yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into build-with-mix

This commit is contained in:
Thales Macedo Garitezi 2021-12-07 15:09:09 -03:00
parent 2e5596f6ee 3fdc075b24
commit 51bd361b16
No known key found for this signature in database
GPG Key ID: DD279F8152A9B6DD
133 changed files with 3432 additions and 1449 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.ci/build_packages/tests.sh
View File

@ -175,26 +175,6 @@ EOF
cat /var/log/emqx/emqx.log.1 || true
exit 1
fi
if [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = ubuntu ] \
|| [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = debian ] ;then
if ! service emqx start; then
cat /var/log/emqx/erlang.log.1 || true
cat /var/log/emqx/emqx.log.1 || true
exit 1
fi
IDLE_TIME=0
while ! curl http://127.0.0.1:18083/api/v5/status >/dev/null 2>&1; do
if [ $IDLE_TIME -gt 10 ]
then
echo "emqx service error"
exit 1
fi
sleep 10
IDLE_TIME=$((IDLE_TIME+1))
done
service emqx stop
fi
}
relup_test(){

2
.ci/docker-compose-file/.env
View File

@ -1,6 +1,6 @@
MYSQL_TAG=8
REDIS_TAG=6
MONGO_TAG=4
MONGO_TAG=5
PGSQL_TAG=13
LDAP_TAG=2.4.50

2
.ci/docker-compose-file/Makefile.local
View File

@ -14,7 +14,7 @@ up:
env \
MYSQL_TAG=8 \
REDIS_TAG=6 \
MONGO_TAG=4 \
MONGO_TAG=5 \
PGSQL_TAG=13 \
LDAP_TAG=2.4.50 \
docker-compose \

4
.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml
View File

@ -2,11 +2,9 @@ version: '3.9'
services:
mongo_server:
container_name: mongo
container_name: mongo
image: mongo:${MONGO_TAG}
restart: always
environment:
MONGO_INITDB_DATABASE: mqtt
networks:
- emqx_bridge
ports:

4
.ci/docker-compose-file/docker-compose.yaml
View File

@ -3,7 +3,7 @@ version: '3.9'
services:
erlang23:
container_name: erlang23
image: ghcr.io/emqx/emqx-builder/5.0-2:23.3.4.9-3-ubuntu20.04
image: ghcr.io/emqx/emqx-builder/5.0-3:23.3.4.9-3-ubuntu20.04
env_file:
- conf.env
environment:
@ -23,7 +23,7 @@ services:
erlang24:
container_name: erlang24
image: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04
image: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04
env_file:
- conf.env
environment:

14
.github/workflows/build_packages.yaml vendored
View File

@ -19,7 +19,7 @@ jobs:
prepare:
runs-on: ubuntu-20.04
# prepare source with any OTP version, no need for a matrix
container: "ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04"
container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04"
outputs:
old_vsns: ${{ steps.find_old_versons.outputs.old_vsns }}
@ -129,7 +129,7 @@ jobs:
- emqx
- emqx-enterprise
otp:
- 24.1.5-2
- 24.1.5-3
macos:
- macos-11
- macos-10.15
@ -215,7 +215,7 @@ jobs:
- emqx
- emqx-enterprise
otp:
- 24.1.5-2 # we test with OTP 23, but only build package on OTP 24 versions
- 24.1.5-3 # we test with OTP 23, but only build package on OTP 24 versions
arch:
- amd64
- arm64
@ -301,7 +301,7 @@ jobs:
-v $(pwd):/emqx \
--workdir /emqx \
--platform linux/$ARCH \
ghcr.io/emqx/emqx-builder/5.0-2:$OTP-$SYSTEM \
ghcr.io/emqx/emqx-builder/5.0-3:$OTP-$SYSTEM \
bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \
make $PROFILE-pkg || cat rebar3.crashdump; \
EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh"
@ -336,7 +336,7 @@ jobs:
- emqx-enterprise
# NOTE: for docker, only support latest otp version, not a matrix
otp:
- 24.1.5-2 # update to latest
- 24.1.5-3 # update to latest
steps:
- uses: actions/download-artifact@v2
@ -377,7 +377,7 @@ jobs:
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp }}-alpine3.14
BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-alpine3.14
RUN_FROM=alpine:3.14
EMQX_NAME=${{ matrix.profile }}
file: source/deploy/docker/Dockerfile
@ -405,7 +405,7 @@ jobs:
- emqx
- emqx-enterprise
otp:
- 24.1.5-2
- 24.1.5-3
steps:
- uses: actions/checkout@v2

6
.github/workflows/build_slim_packages.yaml vendored
View File

@ -24,12 +24,12 @@ jobs:
- emqx
- emqx-enterprise
otp:
- 24.1.5-2
- 24.1.5-3
os:
- ubuntu20.04
- centos7
container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp }}-${{ matrix.os }}"
container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}"
steps:
- uses: actions/checkout@v1
@ -55,7 +55,7 @@ jobs:
- emqx
- emqx-enterprise
otp:
- 24.1.5-2
- 24.1.5-3
macos:
- macos-11
- macos-10.15

2
.github/workflows/check_deps_integrity.yaml vendored
View File

@ -5,7 +5,7 @@ on: [pull_request]
jobs:
check_deps_integrity:
runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04"
container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04"
steps:
- uses: actions/checkout@v2

4
.github/workflows/run_emqx_app_tests.yaml vendored
View File

@ -13,10 +13,10 @@ jobs:
matrix:
otp:
- 23.3.4.9-3
- 24.1.5-2
- 24.1.5-3
runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp }}-ubuntu20.04"
container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-ubuntu20.04"
steps:
- uses: actions/checkout@v2

6
.github/workflows/run_fvt_tests.yaml vendored
View File

@ -14,7 +14,7 @@ jobs:
prepare:
runs-on: ubuntu-20.04
# prepare source with any OTP version, no need for a matrix
container: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14
container: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14
steps:
- uses: actions/checkout@v2
@ -55,7 +55,7 @@ jobs:
- name: make docker image
working-directory: source
env:
EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14
EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14
run: |
make ${{ matrix.profile }}-docker
- name: run emqx
@ -100,7 +100,7 @@ jobs:
- name: make docker image
working-directory: source
env:
EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14
EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14
run: |
make ${{ matrix.profile }}-docker
echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV

4
.github/workflows/run_relup_tests.yaml vendored
View File

@ -19,10 +19,10 @@ jobs:
- emqx
- emqx-enterprise
otp_vsn:
- 24.1.5-2
- 24.1.5-3
runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp_vsn }}-ubuntu20.04"
container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp_vsn }}-ubuntu20.04"
defaults:
run:

6
.github/workflows/run_test_cases.yaml vendored
View File

@ -16,7 +16,7 @@ jobs:
strategy:
matrix:
emqx_builder:
- 5.0-2:24.1.5-2 # run dialyzer on latest OTP
- 5.0-3:24.1.5-3 # run dialyzer on latest OTP
runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/${{ matrix.emqx_builder }}-ubuntu20.04"
@ -32,7 +32,7 @@ jobs:
strategy:
matrix:
emqx_builder:
- 5.0-2:24.1.5-2
- 5.0-3:24.1.5-3
runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/${{ matrix.emqx_builder }}-ubuntu20.04"
@ -55,12 +55,14 @@ jobs:
- uses: actions/checkout@v2
- name: docker compose up
env:
MONGO_TAG: 5
MYSQL_TAG: 8
PGSQL_TAG: 13
REDIS_TAG: 6
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
docker-compose \
-f .ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \

2
.tool-versions
View File

@ -1 +1 @@
erlang 24.1.5-2
erlang 24.1.5-3

2
Makefile
View File

@ -57,7 +57,7 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh)
## app/name-ct targets are intended for local tests hence cover is not enabled
.PHONY: $(APPS:%=%-ct)
define gen-app-ct-target
$1-ct:
$1-ct: conf-segs
$(REBAR) ct --name $(CT_NODE_NAME) -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
endef
$(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))

31
apps/emqx/include/emqx_authentication.hrl Normal file
View File

@ -0,0 +1,31 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-ifndef(EMQX_AUTHENTICATION_HRL).
-define(EMQX_AUTHENTICATION_HRL, true).
%% config root name all auth providers have to agree on.
-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, "authentication").
-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM, authentication).
-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, <<"authentication">>).
%% key to a persistent term which stores a module name in order to inject
%% schema module at run-time to keep emqx app's compile time purity.
%% see emqx_schema.erl for more details
%% and emqx_conf_schema for an examples
-define(EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, emqx_authentication_schema_module).
-endif.

4
apps/emqx/rebar.config
View File

@ -17,10 +17,10 @@
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.20.6"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}}
, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}
, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.15.0"}}}
, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.16.0"}}}
]}.
{plugins, [{rebar3_proper, "0.12.1"}]}.

3
apps/emqx/src/emqx_access_control.erl
View File

@ -63,4 +63,5 @@ do_authorize(ClientInfo, PubSub, Topic) ->
-compile({inline, [run_hooks/3]}).
run_hooks(Name, Args, Acc) ->
ok = emqx_metrics:inc(Name), emqx_hooks:run_fold(Name, Args, Acc).
ok = emqx_metrics:inc(Name),
emqx_hooks:run_fold(Name, Args, Acc).

11
apps/emqx/src/emqx_authentication.erl
View File

@ -24,9 +24,12 @@
-include("emqx.hrl").
-include("logger.hrl").
-include("emqx_authentication.hrl").
-include_lib("stdlib/include/ms_transform.hrl").
-define(CONF_ROOT, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
%% The authentication entrypoint.
-export([ authenticate/2
]).
@ -383,8 +386,8 @@ list_users(ChainName, AuthenticatorID, Params) ->
%%--------------------------------------------------------------------
init(_Opts) ->
ok = emqx_config_handler:add_handler([authentication], ?MODULE),
ok = emqx_config_handler:add_handler([listeners, '?', '?', authentication], ?MODULE),
ok = emqx_config_handler:add_handler([?CONF_ROOT], ?MODULE),
ok = emqx_config_handler:add_handler([listeners, '?', '?', ?CONF_ROOT], ?MODULE),
{ok, #{hooked => false, providers => #{}}}.
handle_call(get_providers, _From, #{providers := Providers} = State) ->
@ -496,8 +499,8 @@ terminate(Reason, _State) ->
Other -> ?SLOG(error, #{msg => "emqx_authentication_terminating",
reason => Other})
end,
emqx_config_handler:remove_handler([authentication]),
emqx_config_handler:remove_handler([listeners, '?', '?', authentication]),
emqx_config_handler:remove_handler([?CONF_ROOT]),
emqx_config_handler:remove_handler([listeners, '?', '?', ?CONF_ROOT]),
ok.
code_change(_OldVsn, State, _Extra) ->

20
apps/emqx/src/emqx_authentication_config.erl
View File

@ -34,6 +34,7 @@
-export_type([config/0]).
-include("logger.hrl").
-include("emqx_authentication.hrl").
-type parsed_config() :: #{mechanism := atom(),
backend => atom(),
@ -132,9 +133,9 @@ do_post_config_update({move_authenticator, ChainName, AuthenticatorID, Position}
check_configs(Configs) ->
Providers = emqx_authentication:get_providers(),
lists:map(fun(C) -> do_check_conifg(C, Providers) end, Configs).
lists:map(fun(C) -> do_check_config(C, Providers) end, Configs).
do_check_conifg(Config, Providers) ->
do_check_config(Config, Providers) ->
Type = authn_type(Config),
case maps:get(Type, Providers, false) of
false ->
@ -143,19 +144,20 @@ do_check_conifg(Config, Providers) ->
providers => Providers}),
throw({unknown_authn_type, Type});
Module ->
do_check_conifg(Type, Config, Module)
do_check_config(Type, Config, Module)
end.
do_check_conifg(Type, Config, Module) ->
do_check_config(Type, Config, Module) ->
F = case erlang:function_exported(Module, check_config, 1) of
true ->
fun Module:check_config/1;
false ->
fun(C) ->
#{config := R} =
hocon_schema:check_plain(Module, #{<<"config">> => C},
Key = list_to_binary(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME),
AtomKey = list_to_atom(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME),
R = hocon_schema:check_plain(Module, #{Key => C},
#{atom_key => true}),
R
maps:get(AtomKey, R)
end
end,
try
@ -261,8 +263,8 @@ authn_type(#{mechanism := M}) -> atom(M);
authn_type(#{<<"mechanism">> := M, <<"backend">> := B}) -> {atom(M), atom(B)};
authn_type(#{<<"mechanism">> := M}) -> atom(M).
atom(Bin) ->
binary_to_existing_atom(Bin, utf8).
atom(A) when is_atom(A) -> A;
atom(Bin) -> binary_to_existing_atom(Bin, utf8).
%% The relative dir for ssl files.
certs_dir(ChainName, ConfigOrID) ->

38
apps/emqx/src/emqx_config.erl
View File

@ -268,23 +268,39 @@ init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) ->
}),
error(failed_to_load_hocon_conf)
end;
init_load(SchemaMod, RawConf0) when is_map(RawConf0) ->
init_load(SchemaMod, RawConf) when is_map(RawConf) ->
ok = save_schema_mod_and_names(SchemaMod),
%% check and save configs
{_AppEnvs, CheckedConf} = check_config(SchemaMod, RawConf0),
%% check configs agains the schema, with environment variables applied on top
{_AppEnvs, CheckedConf} =
check_config(SchemaMod, RawConf, #{apply_override_envs => true}),
%% fill default values for raw config
RawConfWithEnvs = merge_envs(SchemaMod, RawConf),
RootNames = get_root_names(),
ok = save_to_config_map(maps:with(get_atom_root_names(), CheckedConf),
maps:with(get_root_names(), RawConf0)).
maps:with(RootNames, RawConfWithEnvs)).
include_dirs() ->
[filename:join(emqx:data_dir(), "configs")].
merge_envs(SchemaMod, RawConf) ->
Opts = #{logger => fun(_, _) -> ok end, %% everything should have been logged already when check_config
nullable => true, %% TODO: evil, remove, nullable should be declared in schema
format => map,
apply_override_envs => true
},
hocon_schema:merge_env_overrides(SchemaMod, RawConf, all, Opts).
-spec check_config(module(), raw_config()) -> {AppEnvs, CheckedConf}
when AppEnvs :: app_envs(), CheckedConf :: config().
check_config(SchemaMod, RawConf) ->
Opts = #{return_plain => true,
nullable => true,
format => map
},
check_config(SchemaMod, RawConf, #{}).
check_config(SchemaMod, RawConf, Opts0) ->
Opts1 = #{return_plain => true,
nullable => true, %% TODO: evil, remove, nullable should be declared in schema
format => map
},
Opts = maps:merge(Opts0, Opts1),
{AppEnvs, CheckedConf} =
hocon_schema:map_translate(SchemaMod, RawConf, Opts),
{AppEnvs, emqx_map_lib:unsafe_atom_key_map(CheckedConf)}.
@ -312,13 +328,15 @@ read_override_conf(#{} = Opts) ->
File = override_conf_file(Opts),
load_hocon_file(File, map).
override_conf_file(Opts) ->
override_conf_file(Opts) when is_map(Opts) ->
Key =
case maps:get(override_to, Opts, local) of
local -> local_override_conf_file;
cluster -> cluster_override_conf_file
end,
application:get_env(emqx, Key, undefined).
application:get_env(emqx, Key, undefined);
override_conf_file(Which) when is_atom(Which) ->
application:get_env(emqx, Which, undefined).
-spec save_schema_mod_and_names(module()) -> ok.
save_schema_mod_and_names(SchemaMod) ->

4
apps/emqx/src/emqx_frame.erl
View File

@ -248,8 +248,8 @@ parse_packet(#mqtt_packet_header{type = ?CONNECT}, FrameBin, _Options) ->
},
{ConnPacket1, Rest5} = parse_will_message(ConnPacket, Rest4),
{Username, Rest6} = parse_utf8_string(Rest5, bool(UsernameFlag)),
{Passsword, <<>>} = parse_utf8_string(Rest6, bool(PasswordFlag)),
ConnPacket1#mqtt_packet_connect{username = Username, password = Passsword};
{Password, <<>>} = parse_utf8_string(Rest6, bool(PasswordFlag)),
ConnPacket1#mqtt_packet_connect{username = Username, password = Password};
parse_packet(#mqtt_packet_header{type = ?CONNACK},
<<AckFlags:8, ReasonCode:8, Rest/binary>>, #{version := Ver}) ->

4
apps/emqx/src/emqx_listeners.erl
View File

@ -111,7 +111,7 @@ current_conns(ID, ListenOn) ->
{Type, Name} = parse_listener_id(ID),
current_conns(Type, Name, ListenOn).
current_conns(Type, Name, ListenOn) when Type == tcl; Type == ssl ->
current_conns(Type, Name, ListenOn) when Type == tcp; Type == ssl ->
esockd:get_current_connections({listener_id(Type, Name), ListenOn});
current_conns(Type, Name, _ListenOn) when Type =:= ws; Type =:= wss ->
proplists:get_value(all_connections, ranch:info(listener_id(Type, Name)));
@ -122,7 +122,7 @@ max_conns(ID, ListenOn) ->
{Type, Name} = parse_listener_id(ID),
max_conns(Type, Name, ListenOn).
max_conns(Type, Name, ListenOn) when Type == tcl; Type == ssl ->
max_conns(Type, Name, ListenOn) when Type == tcp; Type == ssl ->
esockd:get_max_connections({listener_id(Type, Name), ListenOn});
max_conns(Type, Name, _ListenOn) when Type =:= ws; Type =:= wss ->
proplists:get_value(max_connections, ranch:info(listener_id(Type, Name)));

8
apps/emqx/src/emqx_os_mon.erl
View File

@ -96,8 +96,8 @@ handle_info({timeout, _Timer, check}, State) ->
_ = case emqx_vm:cpu_util() of %% TODO: should be improved?
0 -> ok;
Busy when Busy >= CPUHighWatermark ->
Usage = io_lib:format("~p%", [Busy]),
Message = [Usage, " cpu usage"],
Usage = list_to_binary(io_lib:format("~.2f%", [Busy])),
Message = <<Usage/binary, " cpu usage">>,
emqx_alarm:activate(high_cpu_usage,
#{
usage => Usage,
@ -107,8 +107,8 @@ handle_info({timeout, _Timer, check}, State) ->
Message),
start_check_timer();
Busy when Busy =< CPULowWatermark ->
Usage = io_lib:format("~p%", [Busy]),
Message = [Usage, " cpu usage"],
Usage = list_to_binary(io_lib:format("~.2f%", [Busy])),
Message = <<Usage/binary, " cpu usage">>,
emqx_alarm:deactivate(high_cpu_usage,
#{
usage => Usage,

58
apps/emqx/src/emqx_schema.erl
View File

@ -22,6 +22,7 @@
-dialyzer(no_unused).
-dialyzer(no_fail_call).
-include("emqx_authentication.hrl").
-include_lib("typerefl/include/types.hrl").
-type duration() :: integer().
@ -105,11 +106,29 @@ and can not be deleted."""
The configs here work as default values which can be overriden
in <code>zone</code> configs"""
})}
, {"authentication",
, {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME,
authentication(
"""Default authentication configs for all MQTT listeners.<br>
"""Default authentication configs for all MQTT listeners.
<br>
For per-listener overrides see <code>authentication</code>
in listener configs""")}
in listener configs
<br>
<br>
EMQ X can be configured with:
<br>
<ul>
<li><code>[]</code>: The default value, it allows *ALL* logins</li>
<li>one: For example <code>{enable:true,backend:\"built-in-database\",mechanism=\"password-based\"}</code></li>
<li>chain: An array of structs.</li>
</ul>
<br>
When a chain is configured, the login credentials are checked against the backends
per the configured order, until an 'allow' or 'deny' decision can be made.
<br>
If there is no decision after a full chain exhaustion, the login is rejected.
""")}
%% NOTE: authorization schema here is only to keep emqx app prue
%% the full schema for EMQ X node is injected in emqx_conf_schema.
, {"authorization",
sc(ref("authorization"),
#{})}
@ -972,7 +991,7 @@ mqtt_listener() ->
sc(duration(),
#{})
}
, {"authentication",
, {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME,
authentication("Per-listener authentication override")
}
].
@ -1231,16 +1250,18 @@ ciphers_schema(Default) ->
false -> fun validate_ciphers/1
end
, desc =>
"""TLS cipher suite names separated by comma, or as an array of strings
"""This config holds TLS cipher suite names separated by comma,
or as an array of strings. e.g.
<code>\"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256\"</code> or
<code>[\"TLS_AES_256_GCM_SHA384\",\"TLS_AES_128_GCM_SHA256\"]</code].
<code>[\"TLS_AES_256_GCM_SHA384\",\"TLS_AES_128_GCM_SHA256\"]</code>.
<br>
Ciphers (and their ordering) define the way in which the
client and server encrypts information over the wire.
client and server encrypts information over the network connection.
Selecting a good cipher suite is critical for the
application's data security, confidentiality and performance.
The names should be in OpenSSL sting format (not RFC format).
Default values and examples proveded by EMQ X config
The names should be in OpenSSL string format (not RFC format).
All default values and examples proveded by EMQ X config
documentation are all in OpenSSL format.<br>
NOTE: Certain cipher suites are only compatible with
@ -1436,12 +1457,23 @@ str(S) when is_list(S) ->
S.
authentication(Desc) ->
#{ type => hoconsc:lazy(hoconsc:union([typerefl:map(), hoconsc:array(typerefl:map())]))
, desc => iolist_to_binary([Desc, "<br>", """
%% authentication schemais lazy to make it more 'plugable'
%% the type checks are done in emqx_auth application when it boots.
%% and in emqx_authentication_config module for rutime changes.
Default = hoconsc:lazy(hoconsc:union([typerefl:map(), hoconsc:array(typerefl:map())])),
%% as the type is lazy, the runtime module injection from EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY
%% is for now only affecting document generation.
%% maybe in the future, we can find a more straightforward way to support
%% * document generation (at compile time)
%% * type checks before boot (in bin/emqx config generation)
%% * type checks at runtime (when changing configs via management API)
#{ type => case persistent_term:get(?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, undefined) of
undefined -> Default;
Module -> hoconsc:lazy(Module:root_type())
end
, desc => iolist_to_binary([Desc, """
Authentication can be one single authenticator instance or a chain of authenticators as an array.
When authenticating a login (username, client ID, etc.) the authenticators are checked
in the configured order.<br>
EMQ X comes with a set of pre-built autenticators, for more details, see
<a href=\"#root-authenticator_config\">autenticator_config<a>
"""])
}.

35
apps/emqx/src/emqx_topic.erl
View File

@ -54,11 +54,11 @@ wildcard(Topic) when is_binary(Topic) ->
wildcard(words(Topic));
wildcard([]) ->
false;
wildcard(['#'|_]) ->
wildcard(['#' | _]) ->
true;
wildcard(['+'|_]) ->
wildcard(['+' | _]) ->
true;
wildcard([_H|T]) ->
wildcard([_H | T]) ->
wildcard(T).
%% @doc Match Topic name with filter.
@ -73,17 +73,17 @@ match(Name, Filter) when is_binary(Name), is_binary(Filter) ->
match(words(Name), words(Filter));
match([], []) ->
true;
match([H|T1], [H|T2]) ->
match([H | T1], [H | T2]) ->
match(T1, T2);
match([_H|T1], ['+'|T2]) ->
match([_H | T1], ['+' | T2]) ->
match(T1, T2);
match(_, ['#']) ->
true;
match([_H1|_], [_H2|_]) ->
match([_H1 | _], [_H2 | _]) ->
false;
match([_H1|_], []) ->
match([_H1 | _], []) ->
false;
match([], [_H|_T2]) ->
match([], [_H | _T2]) ->
false.
%% @doc Validate topic name or filter
@ -110,13 +110,13 @@ validate2([]) ->
true;
validate2(['#']) -> % end with '#'
true;
validate2(['#'|Words]) when length(Words) > 0 ->
validate2(['#' | Words]) when length(Words) > 0 ->
error('topic_invalid_#');
validate2([''|Words]) ->
validate2(['' | Words]) ->
validate2(Words);
validate2(['+'|Words]) ->
validate2(['+' | Words]) ->
validate2(Words);
validate2([W|Words]) ->
validate2([W | Words]) ->
validate3(W) andalso validate2(Words).
validate3(<<>>) ->
@ -164,7 +164,7 @@ word(<<"#">>) -> '#';
word(Bin) -> Bin.
%% @doc '$SYS' Topic.
-spec(systop(atom()|string()|binary()) -> topic()).
-spec(systop(atom() | string() | binary()) -> topic()).
systop(Name) when is_atom(Name); is_list(Name) ->
iolist_to_binary(lists:concat(["$SYS/brokers/", node(), "/", Name]));
systop(Name) when is_binary(Name) ->
@ -175,10 +175,10 @@ feed_var(Var, Val, Topic) ->
feed_var(Var, Val, words(Topic), []).
feed_var(_Var, _Val, [], Acc) ->
join(lists:reverse(Acc));
feed_var(Var, Val, [Var|Words], Acc) ->
feed_var(Var, Val, Words, [Val|Acc]);
feed_var(Var, Val, [W|Words], Acc) ->
feed_var(Var, Val, Words, [W|Acc]).
feed_var(Var, Val, [Var | Words], Acc) ->
feed_var(Var, Val, Words, [Val | Acc]);
feed_var(Var, Val, [W | Words], Acc) ->
feed_var(Var, Val, Words, [W | Acc]).
-spec(join(list(binary())) -> binary()).
join([]) ->
@ -218,4 +218,3 @@ parse(TopicFilter = <<"$share/", Rest/binary>>, Options) ->
end;
parse(TopicFilter, Options) ->
{TopicFilter, Options}.

1
apps/emqx/src/emqx_vm.erl
View File

@ -57,6 +57,7 @@
sl_alloc,
ll_alloc,
fix_alloc,
literal_alloc,
std_alloc
]).

27
apps/emqx/test/emqx_authentication_SUITE.erl
View File

@ -25,18 +25,11 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("typerefl/include/types.hrl").
-export([ roots/0, fields/1 ]).
-export([ create/2
, update/2
, authenticate/2
, destroy/1
, check_config/1
]).
-include("emqx_authentication.hrl").
-define(AUTHN, emqx_authentication).
-define(config(KEY), (fun() -> {KEY, _V_} = lists:keyfind(KEY, 1, Config), _V_ end)()).
-define(CONF_ROOT, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
%%------------------------------------------------------------------------------
%% Hocon Schema
@ -250,7 +243,7 @@ t_update_config({init, Config}) ->
{"auth2", AuthNType2} | Config];
t_update_config(Config) when is_list(Config) ->
emqx_config_handler:add_handler([authentication], emqx_authentication),
emqx_config_handler:add_handler([?CONF_ROOT], emqx_authentication),
ok = register_provider(?config("auth1"), ?MODULE),
ok = register_provider(?config("auth2"), ?MODULE),
Global = ?config(global),
@ -267,7 +260,7 @@ t_update_config(Config) when is_list(Config) ->
?assertMatch(
{ok, _},
update_config([authentication], {create_authenticator, Global, AuthenticatorConfig1})),
update_config([?CONF_ROOT], {create_authenticator, Global, AuthenticatorConfig1})),
?assertMatch(
{ok, #{id := ID1, state := #{mark := 1}}},
@ -275,7 +268,7 @@ t_update_config(Config) when is_list(Config) ->
?assertMatch(
{ok, _},
update_config([authentication], {create_authenticator, Global, AuthenticatorConfig2})),
update_config([?CONF_ROOT], {create_authenticator, Global, AuthenticatorConfig2})),
?assertMatch(
{ok, #{id := ID2, state := #{mark := 1}}},
@ -283,7 +276,7 @@ t_update_config(Config) when is_list(Config) ->
?assertMatch(
{ok, _},
update_config([authentication],
update_config([?CONF_ROOT],
{update_authenticator,
Global,
ID1,
@ -296,25 +289,25 @@ t_update_config(Config) when is_list(Config) ->
?assertMatch(
{ok, _},
update_config([authentication], {move_authenticator, Global, ID2, top})),
update_config([?CONF_ROOT], {move_authenticator, Global, ID2, top})),
?assertMatch({ok, [#{id := ID2}, #{id := ID1}]}, ?AUTHN:list_authenticators(Global)),
?assertMatch({ok, _}, update_config([authentication], {delete_authenticator, Global, ID1})),
?assertMatch({ok, _}, update_config([?CONF_ROOT], {delete_authenticator, Global, ID1})),
?assertEqual(
{error, {not_found, {authenticator, ID1}}},
?AUTHN:lookup_authenticator(Global, ID1)),
?assertMatch(
{ok, _},
update_config([authentication], {delete_authenticator, Global, ID2})),
update_config([?CONF_ROOT], {delete_authenticator, Global, ID2})),
?assertEqual(
{error, {not_found, {authenticator, ID2}}},
?AUTHN:lookup_authenticator(Global, ID2)),
ListenerID = 'tcp:default',
ConfKeyPath = [listeners, tcp, default, authentication],
ConfKeyPath = [listeners, tcp, default, ?CONF_ROOT],
?assertMatch(
{ok, _},

4
apps/emqx/test/emqx_channel_SUITE.erl
View File

@ -132,7 +132,7 @@ basic_conf() ->
zones => zone_conf()
}.
set_test_listenser_confs() ->
set_test_listener_confs() ->
Conf = emqx_config:get([]),
emqx_config:put(basic_conf()),
Conf.
@ -179,7 +179,7 @@ end_per_suite(_Config) ->
]).
init_per_testcase(_TestCase, Config) ->
NewConf = set_test_listenser_confs(),
NewConf = set_test_listener_confs(),
[{config, NewConf}|Config].
end_per_testcase(_TestCase, Config) ->

2
apps/emqx/test/emqx_connection_SUITE.erl
View File

@ -59,7 +59,7 @@ init_per_suite(Config) ->
ok = meck:expect(emqx_alarm, deactivate, fun(_) -> ok end),
ok = meck:expect(emqx_alarm, deactivate, fun(_, _) -> ok end),
emqx_channel_SUITE:set_test_listenser_confs(),
emqx_channel_SUITE:set_test_listener_confs(),
Config.
end_per_suite(_Config) ->

37
apps/emqx/test/emqx_listeners_SUITE.erl
View File

@ -22,6 +22,7 @@
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
all() -> emqx_common_test_helpers:all(?MODULE).
@ -37,10 +38,37 @@ end_per_suite(_Config) ->
application:stop(esockd),
application:stop(cowboy).
init_per_testcase(Case, Config)
when Case =:= t_max_conns_tcp; Case =:= t_current_conns_tcp ->
{ok, _} = emqx_config_handler:start_link(),
PrevListeners = emqx_config:get([listeners, tcp], #{}),
PrevRateLimit = emqx_config:get([rate_limit], #{}),
emqx_config:put([listeners, tcp], #{ listener_test =>
#{ bind => {"127.0.0.1", 9999}
, max_connections => 4321
}
}),
emqx_config:put([rate_limit], #{max_conn_rate => 1000}),
ListenerConf = #{ bind => {"127.0.0.1", 9999}
},
ok = emqx_listeners:start(),
[ {listener_conf, ListenerConf}
, {prev_listener_conf, PrevListeners}
, {prev_rate_limit_conf, PrevRateLimit}
| Config];
init_per_testcase(_, Config) ->
{ok, _} = emqx_config_handler:start_link(),
Config.
end_per_testcase(Case, Config)
when Case =:= t_max_conns_tcp; Case =:= t_current_conns_tcp ->
PrevListener = ?config(prev_listener_conf, Config),
PrevRateLimit = ?config(prev_rate_limit_conf, Config),
emqx_config:put([listeners, tcp], PrevListener),
emqx_config:put([rate_limit], PrevRateLimit),
emqx_listeners:stop(),
_ = emqx_config_handler:stop(),
ok;
end_per_testcase(_, _Config) ->
_ = emqx_config_handler:stop(),
ok.
@ -56,6 +84,14 @@ t_restart_listeners(_) ->
ok = emqx_listeners:restart(),
ok = emqx_listeners:stop().
t_max_conns_tcp(_) ->
%% Note: Using a string representation for the bind address like
%% "127.0.0.1" does not work
?assertEqual(4321, emqx_listeners:max_conns('tcp:listener_test', {{127,0,0,1}, 9999})).
t_current_conns_tcp(_) ->
?assertEqual(0, emqx_listeners:current_conns('tcp:listener_test', {{127,0,0,1}, 9999})).
render_config_file() ->
Path = local_path(["etc", "emqx.conf"]),
{ok, Temp} = file:read_file(Path),
@ -101,4 +137,3 @@ get_base_dir(Module) ->
get_base_dir() ->
get_base_dir(?MODULE).

10
apps/emqx/test/emqx_persistent_session_SUITE.erl
View File

@ -245,7 +245,7 @@ receive_messages(Count, Msgs) ->
receive_messages(Count-1, [Msg|Msgs]);
_Other ->
receive_messages(Count, Msgs)
after 1000 ->
after 5000 ->
Msgs
end.
@ -576,7 +576,7 @@ t_publish_while_client_is_gone(Config) ->
| Config]),
{ok, _} = emqtt:ConnFun(Client2),
Msgs = receive_messages(2),
?assertEqual(length(Msgs), 2),
?assertMatch([_, _], Msgs),
[Msg2, Msg1] = Msgs,
?assertEqual({ok, iolist_to_binary(Payload1)}, maps:find(payload, Msg1)),
?assertEqual({ok, 2}, maps:find(qos, Msg1)),
@ -768,7 +768,7 @@ t_lost_messages_because_of_gc(Config) ->
check_snabbkaffe_vanilla(Trace) ->
ResumeTrace = [T || #{?snk_kind := K} = T <- Trace,
re:run(atom_to_list(K), "^ps_") /= nomatch],
re:run(to_list(K), "^ps_") /= nomatch],
?assertMatch([_|_], ResumeTrace),
[_Sid] = lists:usort(?projection(sid, ResumeTrace)),
%% Check internal flow of the emqx_cm resuming
@ -811,6 +811,10 @@ check_snabbkaffe_vanilla(Trace) ->
[Markers] = ?projection(markers, ?of_kind(ps_node_markers, Trace)),
?assertMatch([_], Markers).
to_list(L) when is_list(L) -> L;
to_list(A) when is_atom(A) -> atom_to_list(A);
to_list(B) when is_binary(B) -> binary_to_list(B).
%%--------------------------------------------------------------------
%% Snabbkaffe tests
%%--------------------------------------------------------------------

2
apps/emqx/test/emqx_session_SUITE.erl
View File

@ -29,7 +29,7 @@ all() -> emqx_common_test_helpers:all(?MODULE).
%%--------------------------------------------------------------------
init_per_suite(Config) ->
emqx_channel_SUITE:set_test_listenser_confs(),
emqx_channel_SUITE:set_test_listener_confs(),
ok = meck:new([emqx_hooks, emqx_metrics, emqx_broker],
[passthrough, no_history, no_link]),
ok = meck:expect(emqx_metrics, inc, fun(_) -> ok end),

7
apps/emqx/test/emqx_topic_SUITE.erl
View File

@ -20,6 +20,7 @@
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").
-import(emqx_topic,
[ wildcard/1
@ -183,9 +184,11 @@ t_feed_var(_) ->
?assertEqual(<<"$queue/client/clientId">>,
feed_var(<<"$c">>, <<"clientId">>, <<"$queue/client/$c">>)),
?assertEqual(<<"username/test/client/x">>,
feed_var(<<"%u">>, <<"test">>, <<"username/%u/client/x">>)),
feed_var( ?PH_USERNAME, <<"test">>
, <<"username/", ?PH_USERNAME/binary, "/client/x">>)),
?assertEqual(<<"username/test/client/clientId">>,
feed_var(<<"%c">>, <<"clientId">>, <<"username/test/client/%c">>)).
feed_var( ?PH_CLIENTID, <<"clientId">>
, <<"username/test/client/", ?PH_CLIENTID/binary>>)).
long_topic() ->
iolist_to_binary([[integer_to_list(I), "/"] || I <- lists:seq(0, 66666)]).

1
apps/emqx/test/emqx_trace_handler_SUITE.erl
View File

@ -199,6 +199,7 @@ t_trace_ip_address(_Config) ->
?assertEqual([], emqx_trace_handler:running()).
filesync(Name, Type) ->
ct:sleep(50),
filesync(Name, Type, 3).
%% sometime the handler process is not started yet.

7
apps/emqx_authn/etc/emqx_authn.conf
View File

@ -1,6 +1 @@
# authentication: {
# mechanism: password-based
# backend: built-in-database
# user_id_type: clientid
# }
authentication: []

7
apps/emqx_authn/include/emqx_authn.hrl
View File

@ -17,6 +17,8 @@
-ifndef(EMQX_AUTHN_HRL).
-define(EMQX_AUTHN_HRL, true).
-include_lib("emqx/include/emqx_authentication.hrl").
-define(APP, emqx_authn).
-define(AUTHN, emqx_authentication).
@ -27,4 +29,9 @@
-define(AUTH_SHARD, emqx_authn_shard).
%% has to be the same as the root field name defined in emqx_schema
-define(CONF_NS, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME).
-define(CONF_NS_ATOM, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
-define(CONF_NS_BINARY, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY).
-endif.

17
apps/emqx_authn/src/emqx_authn.erl
View File

@ -22,6 +22,8 @@
, check_configs/1
]).
-include("emqx_authn.hrl").
providers() ->
[ {{'password-based', 'built-in-database'}, emqx_authn_mnesia}
, {{'password-based', mysql}, emqx_authn_mysql}
@ -44,8 +46,8 @@ check_config(Config) ->
check_config(Config, Opts) ->
case do_check_config(Config, Opts) of
#{config := Checked} -> Checked;
#{<<"config">> := WithDefaults} -> WithDefaults
#{?CONF_NS_ATOM := Checked} -> Checked;
#{?CONF_NS_BINARY := WithDefaults} -> WithDefaults
end.
do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) ->
@ -56,10 +58,15 @@ do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) ->
case lists:keyfind(Key, 1, providers()) of
false ->
throw({unknown_handler, Key});
{_, Provider} ->
hocon_schema:check_plain(Provider, #{<<"config">> => Config},
{_, ProviderModule} ->
hocon_schema:check_plain(ProviderModule, #{?CONF_NS_BINARY => Config},
Opts#{atom_key => true})
end.
atom(Bin) ->
binary_to_existing_atom(Bin, utf8).
try
binary_to_existing_atom(Bin, utf8)
catch
_ : _ ->
throw({unknown_auth_provider, Bin})
end.

19
apps/emqx_authn/src/emqx_authn_api.erl
View File

@ -22,6 +22,7 @@
-include("emqx_authn.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx/include/emqx_authentication.hrl").
-import(hoconsc, [mk/2, ref/1]).
-import(emqx_dashboard_swagger, [error_codes/2]).
@ -32,8 +33,10 @@
% Swagger
-define(API_TAGS_GLOBAL, [<<"authentication">>, <<"authentication config(global)">>]).
-define(API_TAGS_SINGLE, [<<"authentication">>, <<"authentication config(single listener)">>]).
-define(API_TAGS_GLOBAL, [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY,
<<"authentication config(global)">>]).
-define(API_TAGS_SINGLE, [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY,
<<"authentication config(single listener)">>]).
-export([ api_spec/0
, paths/0
@ -793,9 +796,10 @@ add_user(ChainName,
AuthenticatorID,
#{<<"user_id">> := UserID, <<"password">> := Password} = UserInfo) ->
IsSuperuser = maps:get(<<"is_superuser">>, UserInfo, false),
case emqx_authentication:add_user(ChainName, AuthenticatorID, #{ user_id => UserID
, password => Password
, is_superuser => IsSuperuser}) of
case emqx_authentication:add_user(ChainName, AuthenticatorID,
#{ user_id => UserID
, password => Password
, is_superuser => IsSuperuser}) of
{ok, User} ->
{201, User};
{error, Reason} ->
@ -845,7 +849,8 @@ list_users(ChainName, AuthenticatorID, PageParams) ->
end.
update_config(Path, ConfigRequest) ->
emqx:update_config(Path, ConfigRequest, #{rawconf_with_defaults => true}).
emqx_conf:update(Path, ConfigRequest, #{rawconf_with_defaults => true,
override_to => cluster}).
get_raw_config_with_defaults(ConfKeyPath) ->
NConfKeyPath = [atom_to_binary(Key, utf8) || Key <- ConfKeyPath],
@ -1027,7 +1032,7 @@ authenticator_examples() ->
backend => <<"redis">>,
server => <<"127.0.0.1:6379">>,
database => 0,
query => <<"HMGET ${username} password_hash salt">>,
cmd => <<"HMGET ${username} password_hash salt">>,
password_hash_algorithm => <<"sha256">>,
salt_position => <<"prefix">>
}

6
apps/emqx_authn/src/emqx_authn_app.erl
View File

@ -25,6 +25,8 @@
, stop/1
]).
-include_lib("emqx/include/emqx_authentication.hrl").
-dialyzer({nowarn_function, [start/2]}).
%%------------------------------------------------------------------------------
@ -65,7 +67,7 @@ chain_configs() ->
[global_chain_config() | listener_chain_configs()].
global_chain_config() ->
{?GLOBAL, emqx:get_raw_config([<<"authentication">>], [])}.
{?GLOBAL, emqx:get_raw_config([?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY], [])}.
listener_chain_configs() ->
lists:map(
@ -77,7 +79,7 @@ listener_chain_configs() ->
auth_config_path(ListenerID) ->
[<<"listeners">>]
++ binary:split(atom_to_binary(ListenerID), <<":">>)
++ [<<"authentication">>].
++ [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY].
provider_types() ->
lists:map(fun({Type, _Module}) -> Type end, emqx_authn:providers()).

22
apps/emqx_authn/src/emqx_authn_schema.erl
View File

@ -22,10 +22,12 @@
, roots/0
, fields/1
, authenticator_type/0
, root_type/0
, mechanism/1
, backend/1
]).
%% only for doc generation
roots() -> [{authenticator_config, hoconsc:mk(authenticator_type())}].
roots() -> [].
fields(_) -> [].
@ -35,6 +37,7 @@ common_fields() ->
enable(type) -> boolean();
enable(default) -> true;
enable(desc) -> "Set to <code>false</code> to disable this auth provider";
enable(_) -> undefined.
authenticator_type() ->
@ -42,3 +45,18 @@ authenticator_type() ->
config_refs(Modules) ->
lists:append([Module:refs() || Module <- Modules]).
%% authn is a core functionality however implemented outside fo emqx app
%% in emqx_schema, 'authentication' is a map() type which is to allow
%% EMQ X more plugable.
root_type() ->
T = authenticator_type(),
hoconsc:union([T, hoconsc:array(T)]).
mechanism(Name) ->
hoconsc:mk(hoconsc:enum([Name]),
#{nullable => false}).
backend(Name) ->
hoconsc:mk(hoconsc:enum([Name]),
#{nullable => false}).

2
apps/emqx_authn/src/emqx_authn_utils.erl
View File

@ -93,6 +93,8 @@ is_superuser(#{<<"is_superuser">> := 0}) ->
#{is_superuser => false};
is_superuser(#{<<"is_superuser">> := null}) ->
#{is_superuser => false};
is_superuser(#{<<"is_superuser">> := undefined}) ->
#{is_superuser => false};
is_superuser(#{<<"is_superuser">> := false}) ->
#{is_superuser => false};
is_superuser(#{<<"is_superuser">> := _}) ->

33
apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
View File

@ -83,11 +83,11 @@ mnesia(boot) ->
namespace() -> "authn-scram-builtin_db".
roots() -> [config].
roots() -> [?CONF_NS].
fields(config) ->
[ {mechanism, {enum, [scram]}}
, {backend, {enum, ['built-in-database']}}
fields(?CONF_NS) ->
[ {mechanism, emqx_authn_schema:mechanism('scram')}
, {backend, emqx_authn_schema:backend('built-in-database')}
, {algorithm, fun algorithm/1}
, {iteration_count, fun iteration_count/1}
] ++ emqx_authn_schema:common_fields().
@ -105,7 +105,7 @@ iteration_count(_) -> undefined.
%%------------------------------------------------------------------------------
refs() ->
[hoconsc:ref(?MODULE, config)].
[hoconsc:ref(?MODULE, ?CONF_NS)].
create(AuthenticatorID,
#{algorithm := Algorithm,
@ -137,10 +137,7 @@ authenticate(_Credential, _State) ->
ignore.
destroy(#{user_group := UserGroup}) ->
MatchSpec = ets:fun2ms(
fun(#user_info{user_id = {Group, _}} = User) when Group =:= UserGroup ->
User
end),
MatchSpec = group_match_spec(UserGroup),
trans(
fun() ->
ok = lists:foreach(fun(UserInfo) ->
@ -205,16 +202,16 @@ lookup_user(UserID, #{user_group := UserGroup}) ->
end.
list_users(PageParams, #{user_group := UserGroup}) ->
MatchSpec = [{{user_info, {UserGroup, '_'}, '_', '_', '_', '_'}, [], ['$_']}],
MatchSpec = group_match_spec(UserGroup),
{ok, emqx_mgmt_api:paginate(?TAB, MatchSpec, PageParams, ?FORMAT_FUN)}.
%%------------------------------------------------------------------------------
%% Internal functions
%%------------------------------------------------------------------------------
ensure_auth_method('SCRAM-SHA-256', #{algorithm := sha256}) ->
ensure_auth_method(<<"SCRAM-SHA-256">>, #{algorithm := sha256}) ->
true;
ensure_auth_method('SCRAM-SHA-512', #{algorithm := sha512}) ->
ensure_auth_method(<<"SCRAM-SHA-512">>, #{algorithm := sha512}) ->
true;
ensure_auth_method(_, _) ->
false.
@ -228,8 +225,10 @@ check_client_first_message(Bin, _Cache, #{iteration_count := IterationCount} = S
#{iteration_count => IterationCount,
retrieve => RetrieveFun}
) of
{cotinue, ServerFirstMessage, Cache} ->
{cotinue, ServerFirstMessage, Cache};
{continue, ServerFirstMessage, Cache} ->
{continue, ServerFirstMessage, Cache};
ignore ->
ignore;
{error, _Reason} ->
{error, not_authorized}
end.
@ -280,3 +279,9 @@ trans(Fun, Args) ->
format_user_info(#user_info{user_id = {_, UserID}, is_superuser = IsSuperuser}) ->
#{user_id => UserID, is_superuser => IsSuperuser}.
group_match_spec(UserGroup) ->
ets:fun2ms(
fun(#user_info{user_id = {Group, _}} = User) when Group =:= UserGroup ->
User
end).

22
apps/emqx_authn/src/simple_authn/emqx_authn_http.erl
View File

@ -43,8 +43,9 @@
namespace() -> "authn-http".
roots() ->
[ {config, hoconsc:mk(hoconsc:union(refs()),
#{})}
[ {?CONF_NS,
hoconsc:mk(hoconsc:union(refs()),
#{})}
].
fields(get) ->
@ -60,8 +61,8 @@ fields(post) ->
] ++ common_fields().
common_fields() ->
[ {mechanism, hoconsc:enum(['password-based'])}
, {backend, hoconsc:enum(['http'])}
[ {mechanism, emqx_authn_schema:mechanism('password-based')}
, {backend, emqx_authn_schema:backend(http)}
, {url, fun url/1}
, {body, fun body/1}
, {request_timeout, fun request_timeout/1}
@ -233,9 +234,9 @@ transform_header_name(Headers) ->
end, #{}, Headers).
check_ssl_opts(Conf) ->
case parse_url(hocon_schema:get_value("config.url", Conf)) of
case parse_url(get_conf_val("url", Conf)) of
#{scheme := https} ->
case hocon_schema:get_value("config.ssl.enable", Conf) of
case get_conf_val("ssl.enable", Conf) of
true -> ok;
false -> false
end;
@ -244,8 +245,8 @@ check_ssl_opts(Conf) ->
end.
check_headers(Conf) ->
Method = to_bin(hocon_schema:get_value("config.method", Conf)),
Headers = hocon_schema:get_value("config.headers", Conf),
Method = to_bin(get_conf_val("method", Conf)),
Headers = get_conf_val("headers", Conf),
Method =:= <<"post">> orelse (not maps:is_key(<<"content-type">>, Headers)).
parse_url(URL) ->
@ -284,7 +285,7 @@ replace_placeholders([{K, V0} | More], Credential, Acc) ->
undefined ->
error({cannot_get_variable, V0});
V ->
replace_placeholders(More, Credential, [{K, emqx_authn_utils:bin(V)} | Acc])
replace_placeholders(More, Credential, [{K, to_bin(V)} | Acc])
end.
append_query(Path, []) ->
@ -340,3 +341,6 @@ to_bin(B) when is_binary(B) ->
B;
to_bin(L) when is_list(L) ->
list_to_binary(L).
get_conf_val(Name, Conf) ->
hocon_schema:get_value(?CONF_NS ++ "." ++ Name, Conf).

55
apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl
View File

@ -20,6 +20,8 @@
-include_lib("emqx/include/logger.hrl").
-include_lib("jose/include/jose_jwk.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-export([ start_link/1
, stop/1
@ -66,9 +68,9 @@ init([Opts]) ->
handle_call(get_cached_jwks, _From, #{jwks := Jwks} = State) ->
{reply, {ok, Jwks}, State};
handle_call({update, Opts}, _From, State) ->
State = handle_options(Opts),
{reply, ok, refresh_jwks(State)};
handle_call({update, Opts}, _From, _State) ->
NewState = handle_options(Opts),
{reply, ok, refresh_jwks(NewState)};
handle_call(_Req, _From, State) ->
{reply, ok, State}.
@ -91,25 +93,27 @@ handle_info({refresh_jwks, _TRef, refresh}, #{request_id := RequestID} = State)
handle_info({http, {RequestID, Result}},
#{request_id := RequestID, endpoint := Endpoint} = State0) ->
?tp(debug, jwks_endpoint_response, #{request_id => RequestID}),
State1 = State0#{request_id := undefined},
case Result of
{error, Reason} ->
?SLOG(warning, #{msg => "failed_to_request_jwks_endpoint",
endpoint => Endpoint,
reason => Reason}),
State1;
{_StatusLine, _Headers, Body} ->
try
JWKS = jose_jwk:from(emqx_json:decode(Body, [return_maps])),
{_, JWKs} = JWKS#jose_jwk.keys,
State1#{jwks := JWKs}
catch _:_ ->
?SLOG(warning, #{msg => "invalid_jwks_returned",
endpoint => Endpoint,
body => Body}),
State1
end
end;
NewState = case Result of
{error, Reason} ->
?SLOG(warning, #{msg => "failed_to_request_jwks_endpoint",
endpoint => Endpoint,
reason => Reason}),
State1;
{_StatusLine, _Headers, Body} ->
try
JWKS = jose_jwk:from(emqx_json:decode(Body, [return_maps])),
{_, JWKs} = JWKS#jose_jwk.keys,
State1#{jwks := JWKs}
catch _:_ ->
?SLOG(warning, #{msg => "invalid_jwks_returned",
endpoint => Endpoint,
body => Body}),
State1
end
end,
{noreply, NewState};
handle_info({http, {_, _}}, State) ->
%% ignore
@ -147,17 +151,18 @@ refresh_jwks(#{endpoint := Endpoint,
NState = case httpc:request(get, {Endpoint, [{"Accept", "application/json"}]}, HTTPOpts,
[{body_format, binary}, {sync, false}, {receiver, self()}]) of
{error, Reason} ->
?SLOG(warning, #{msg => "failed_to_request_jwks_endpoint",
endpoint => Endpoint,
reason => Reason}),
?tp(warning, jwks_endpoint_request_fail, #{endpoint => Endpoint,
http_opts => HTTPOpts,
reason => Reason}),
State;
{ok, RequestID} ->
?tp(debug, jwks_endpoint_request_ok, #{request_id => RequestID}),
State#{request_id := RequestID}
end,
ensure_expiry_timer(NState).
ensure_expiry_timer(State = #{refresh_interval := Interval}) ->
State#{refresh_timer := emqx_misc:start_timer(timer:seconds(Interval), refresh_jwks)}.
State#{refresh_timer => emqx_misc:start_timer(timer:seconds(Interval), refresh_jwks)}.
cancel_timer(State = #{refresh_timer := undefined}) ->
State;

32
apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl
View File

@ -16,6 +16,7 @@
-module(emqx_authn_jwt).
-include("emqx_authn.hrl").
-include_lib("typerefl/include/types.hrl").
-behaviour(hocon_schema).
@ -40,9 +41,9 @@
namespace() -> "authn-jwt".
roots() ->
[ {config, hoconsc:mk(hoconsc:union(refs()),
#{}
)}
[ {?CONF_NS,
hoconsc:mk(hoconsc:union(refs()),
#{})}
].
fields('hmac-based') ->
@ -82,7 +83,7 @@ fields(ssl_disable) ->
[ {enable, #{type => false}} ].
common_fields() ->
[ {mechanism, {enum, [jwt]}}
[ {mechanism, emqx_authn_schema:mechanism('jwt')}
, {verify_claims, fun verify_claims/1}
] ++ emqx_authn_schema:common_fields().
@ -157,7 +158,7 @@ update(#{use_jwks := false} = Config, _State) ->
update(#{use_jwks := true} = Config,
#{jwk := Connector} = State)
when is_pid(Connector) ->
ok = emqx_authn_jwks_connector:update(Connector, Config),
ok = emqx_authn_jwks_connector:update(Connector, connector_opts(Config)),
case maps:get(verify_cliams, Config, undefined) of
undefined ->
{ok, State};
@ -208,7 +209,7 @@ create2(#{use_jwks := false,
JWK = jose_jwk:from_oct(Secret),
{ok, #{jwk => JWK,
verify_claims => VerifyClaims}}
end;
end;
create2(#{use_jwks := false,
algorithm := 'public-key',
@ -219,13 +220,8 @@ create2(#{use_jwks := false,
verify_claims => VerifyClaims}};
create2(#{use_jwks := true,
verify_claims := VerifyClaims,
ssl := #{enable := Enable} = SSL} = Config) ->
SSLOpts = case Enable of
true -> maps:without([enable], SSL);
false -> #{}
end,
case emqx_authn_jwks_connector:start_link(Config#{ssl_opts => SSLOpts}) of
verify_claims := VerifyClaims} = Config) ->
case emqx_authn_jwks_connector:start_link(connector_opts(Config)) of
{ok, Connector} ->
{ok, #{jwk => Connector,
verify_claims => VerifyClaims}};
@ -233,6 +229,14 @@ create2(#{use_jwks := true,
{error, Reason}
end.
connector_opts(#{ssl := #{enable := Enable} = SSL} = Config) ->
SSLOpts = case Enable of
true -> maps:without([enable], SSL);
false -> #{}
end,
Config#{ssl_opts => SSLOpts}.
may_decode_secret(false, Secret) -> Secret;
may_decode_secret(true, Secret) ->
try base64:decode(Secret)
@ -260,7 +264,7 @@ verify(JWS, [JWK | More], VerifyClaims) ->
Claims = emqx_json:decode(Payload, [return_maps]),
case verify_claims(Claims, VerifyClaims) of
ok ->
{ok, #{is_superuser => maps:get(<<"is_superuser">>, Claims, false)}};
{ok, emqx_authn_utils:is_superuser(Claims)};
{error, Reason} ->
{error, Reason}
end;

12
apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
View File

@ -85,11 +85,11 @@ mnesia(boot) ->
namespace() -> "authn-builtin_db".
roots() -> [config].
roots() -> [?CONF_NS].
fields(config) ->
[ {mechanism, {enum, ['password-based']}}
, {backend, {enum, ['built-in-database']}}
fields(?CONF_NS) ->
[ {mechanism, emqx_authn_schema:mechanism('password-based')}
, {backend, emqx_authn_schema:backend('built-in-database')}
, {user_id_type, fun user_id_type/1}
, {password_hash_algorithm, fun password_hash_algorithm/1}
] ++ emqx_authn_schema:common_fields();
@ -104,7 +104,7 @@ fields(other_algorithms) ->
].
user_id_type(type) -> user_id_type();
user_id_type(default) -> username;
user_id_type(default) -> <<"username">>;
user_id_type(_) -> undefined.
password_hash_algorithm(type) -> hoconsc:union([hoconsc:ref(?MODULE, bcrypt),
@ -121,7 +121,7 @@ salt_rounds(_) -> undefined.
%%------------------------------------------------------------------------------
refs() ->
[hoconsc:ref(?MODULE, config)].
[hoconsc:ref(?MODULE, ?CONF_NS)].
create(AuthenticatorID,
#{user_id_type := Type,

17
apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl
View File

@ -42,8 +42,8 @@
namespace() -> "authn-mongodb".
roots() ->
[ {config, hoconsc:mk(hoconsc:union(refs()),
#{})}
[ {?CONF_NS, hoconsc:mk(hoconsc:union(refs()),
#{})}
].
fields(standalone) ->
@ -56,8 +56,8 @@ fields('sharded-cluster') ->
common_fields() ++ emqx_connector_mongo:fields(sharded).
common_fields() ->
[ {mechanism, {enum, ['password-based']}}
, {backend, {enum, [mongodb]}}
[ {mechanism, emqx_authn_schema:mechanism('password-based')}
, {backend, emqx_authn_schema:backend(mongodb)}
, {collection, fun collection/1}
, {selector, fun selector/1}
, {password_hash_field, fun password_hash_field/1}
@ -115,6 +115,8 @@ create(#{selector := Selector} = Config) ->
password_hash_algorithm,
salt_position],
Config),
#{password_hash_algorithm := Algorithm} = State,
ok = emqx_authn_utils:ensure_apps_started(Algorithm),
ResourceId = emqx_authn_utils:make_resource_id(?MODULE),
NState = State#{
selector => NSelector,
@ -155,7 +157,7 @@ authenticate(#{password := Password} = Credential,
Doc ->
case check_password(Password, Doc, State) of
ok ->
{ok, #{is_superuser => is_superuser(Doc, State)}};
{ok, is_superuser(Doc, State)};
{error, {cannot_find_password_hash_field, PasswordHashField}} ->
?SLOG(error, #{msg => "cannot_find_password_hash_field",
resource => ResourceId,
@ -234,9 +236,10 @@ check_password(Password,
end.
is_superuser(Doc, #{is_superuser_field := IsSuperuserField}) ->
maps:get(IsSuperuserField, Doc, false);
IsSuperuser = maps:get(IsSuperuserField, Doc, false),
emqx_authn_utils:is_superuser(#{<<"is_superuser">> => IsSuperuser});
is_superuser(_, _) ->
false.
emqx_authn_utils:is_superuser(#{<<"is_superuser">> => false}).
hash(Algorithm, Password, Salt, prefix) ->
emqx_passwd:hash(Algorithm, <<Salt/binary, Password/binary>>);

10
apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl
View File

@ -41,11 +41,11 @@
namespace() -> "authn-mysql".
roots() -> [config].
roots() -> [?CONF_NS].
fields(config) ->
[ {mechanism, {enum, ['password-based']}}
, {backend, {enum, [mysql]}}
fields(?CONF_NS) ->
[ {mechanism, emqx_authn_schema:mechanism('password-based')}
, {backend, emqx_authn_schema:backend(mysql)}
, {password_hash_algorithm, fun password_hash_algorithm/1}
, {salt_position, fun salt_position/1}
, {query, fun query/1}
@ -74,7 +74,7 @@ query_timeout(_) -> undefined.
%%------------------------------------------------------------------------------
refs() ->
[hoconsc:ref(?MODULE, config)].
[hoconsc:ref(?MODULE, ?CONF_NS)].
create(_AuthenticatorID, Config) ->
create(Config).

10
apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl
View File

@ -47,11 +47,11 @@
namespace() -> "authn-postgresql".
roots() -> [config].
roots() -> [?CONF_NS].
fields(config) ->
[ {mechanism, {enum, ['password-based']}}
, {backend, {enum, [postgresql]}}
fields(?CONF_NS) ->
[ {mechanism, emqx_authn_schema:mechanism('password-based')}
, {backend, emqx_authn_schema:backend(postgresql)}
, {password_hash_algorithm, fun password_hash_algorithm/1}
, {salt_position, fun salt_position/1}
, {query, fun query/1}
@ -75,7 +75,7 @@ query(_) -> undefined.
%%------------------------------------------------------------------------------
refs() ->
[hoconsc:ref(?MODULE, config)].
[hoconsc:ref(?MODULE, ?CONF_NS)].
create(_AuthenticatorID, Config) ->
create(Config).

36
apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl
View File

@ -42,8 +42,8 @@
namespace() -> "authn-redis".
roots() ->
[ {config, hoconsc:mk(hoconsc:union(refs()),
#{})}
[ {?CONF_NS, hoconsc:mk(hoconsc:union(refs()),
#{})}
].
fields(standalone) ->
@ -56,15 +56,15 @@ fields(sentinel) ->
common_fields() ++ emqx_connector_redis:fields(sentinel).
common_fields() ->
[{mechanism, {enum, ['password-based']}},
{backend, {enum, [redis]}},
{query, fun query/1},
{password_hash_algorithm, fun password_hash_algorithm/1},
{salt_position, fun salt_position/1}
[ {mechanism, emqx_authn_schema:mechanism('password-based')}
, {backend, emqx_authn_schema:backend(redis)}
, {cmd, fun cmd/1}
, {password_hash_algorithm, fun password_hash_algorithm/1}
, {salt_position, fun salt_position/1}
] ++ emqx_authn_schema:common_fields().
query(type) -> string();
query(_) -> undefined.
cmd(type) -> string();
cmd(_) -> undefined.
password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt]};
password_hash_algorithm(default) -> sha256;
@ -87,17 +87,17 @@ refs() ->
create(_AuthenticatorID, Config) ->
create(Config).
create(#{query := Query,
create(#{cmd := Cmd,
password_hash_algorithm := Algorithm} = Config) ->
try
NQuery = parse_query(Query),
NCmd = parse_cmd(Cmd),
ok = emqx_authn_utils:ensure_apps_started(Algorithm),
State = maps:with(
[password_hash_algorithm, salt_position],
Config),
ResourceId = emqx_authn_utils:make_resource_id(?MODULE),
NState = State#{
query => NQuery,
cmd => NCmd,
resource_id => ResourceId},
case emqx_resource:create_local(ResourceId, emqx_connector_redis, Config) of
{ok, already_created} ->
@ -108,8 +108,8 @@ create(#{query := Query,
{error, Reason}
end
catch
error:{unsupported_query, _Query} ->
{error, {unsupported_query, Query}};
error:{unsupported_cmd, _Cmd} ->
{error, {unsupported_cmd, Cmd}};
error:missing_password_hash ->
{error, missing_password_hash};
error:{unsupported_fields, Fields} ->
@ -128,7 +128,7 @@ update(Config, State) ->
authenticate(#{auth_method := _}, _) ->
ignore;
authenticate(#{password := Password} = Credential,
#{query := {Command, Key, Fields},
#{cmd := {Command, Key, Fields},
resource_id := ResourceId} = State) ->
NKey = binary_to_list(iolist_to_binary(replace_placeholders(Key, Credential))),
case emqx_resource:query(ResourceId, {cmd, [Command, NKey | Fields]}) of
@ -162,15 +162,15 @@ destroy(#{resource_id := ResourceId}) ->
%%------------------------------------------------------------------------------
%% Only support HGET and HMGET
parse_query(Query) ->
case string:tokens(Query, " ") of
parse_cmd(Cmd) ->
case string:tokens(Cmd, " ") of
[Command, Key, Field | Fields] when Command =:= "HGET" orelse Command =:= "HMGET" ->
NFields = [Field | Fields],
check_fields(NFields),
NKey = parse_key(Key),
{Command, NKey, NFields};
_ ->
error({unsupported_query, Query})
error({unsupported_cmd, Cmd})
end.
check_fields(Fields) ->

22
apps/emqx_authn/test/emqx_authn_SUITE.erl
View File

@ -1,22 +0,0 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_authn_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
all() -> emqx_common_test_helpers:all(?MODULE).

78
apps/emqx_authn/test/emqx_authn_api_SUITE.erl
View File

@ -43,18 +43,20 @@ groups() ->
[].
init_per_testcase(_, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
emqx_authn_test_lib:delete_authenticators(
[authentication],
[?CONF_NS_ATOM],
?GLOBAL),
emqx_authn_test_lib:delete_authenticators(
[listeners, tcp, default, authentication],
[listeners, tcp, default, ?CONF_NS_ATOM],
?TCP_DEFAULT),
{atomic, ok} = mria:clear_table(emqx_authn_mnesia),
Config.
init_per_suite(Config) ->
_ = application:load(emqx_conf),
ok = emqx_common_test_helpers:start_apps(
[emqx_authn, emqx_dashboard],
fun set_special_configs/1),
@ -87,8 +89,8 @@ set_special_configs(_App) ->
%%------------------------------------------------------------------------------
t_invalid_listener(_) ->
{ok, 404, _} = request(get, uri(["listeners", "invalid", "authentication"])),
{ok, 404, _} = request(get, uri(["listeners", "in:valid", "authentication"])).
{ok, 404, _} = request(get, uri(["listeners", "invalid", ?CONF_NS])),
{ok, 404, _} = request(get, uri(["listeners", "in:valid", ?CONF_NS])).
t_authenticators(_) ->
test_authenticators([]).
@ -131,86 +133,86 @@ test_authenticators(PathPrefix) ->
ValidConfig = emqx_authn_test_lib:http_example(),
{ok, 200, _} = request(
post,
uri(PathPrefix ++ ["authentication"]),
uri(PathPrefix ++ [?CONF_NS]),
ValidConfig),
{ok, 409, _} = request(
post,
uri(PathPrefix ++ ["authentication"]),
uri(PathPrefix ++ [?CONF_NS]),
ValidConfig),
InvalidConfig0 = ValidConfig#{method => <<"delete">>},
{ok, 400, _} = request(
post,
uri(PathPrefix ++ ["authentication"]),
uri(PathPrefix ++ [?CONF_NS]),
InvalidConfig0),
InvalidConfig1 = ValidConfig#{method => <<"get">>,
headers => #{<<"content-type">> => <<"application/json">>}},
{ok, 400, _} = request(
post,
uri(PathPrefix ++ ["authentication"]),
uri(PathPrefix ++ [?CONF_NS]),
InvalidConfig1),
?assertAuthenticatorsMatch(
[#{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}],
PathPrefix ++ ["authentication"]).
PathPrefix ++ [?CONF_NS]).
test_authenticator(PathPrefix) ->
ValidConfig0 = emqx_authn_test_lib:http_example(),
{ok, 200, _} = request(
post,
uri(PathPrefix ++ ["authentication"]),
uri(PathPrefix ++ [?CONF_NS]),
ValidConfig0),
{ok, 200, _} = request(
get,
uri(PathPrefix ++ ["authentication", "password-based:http"])),
uri(PathPrefix ++ [?CONF_NS, "password-based:http"])),
{ok, 404, _} = request(
get,
uri(PathPrefix ++ ["authentication", "password-based:redis"])),
uri(PathPrefix ++ [?CONF_NS, "password-based:redis"])),
{ok, 404, _} = request(
put,
uri(PathPrefix ++ ["authentication", "password-based:built-in-database"]),
uri(PathPrefix ++ [?CONF_NS, "password-based:built-in-database"]),
emqx_authn_test_lib:built_in_database_example()),
InvalidConfig0 = ValidConfig0#{method => <<"delete">>},
{ok, 400, _} = request(
put,
uri(PathPrefix ++ ["authentication", "password-based:http"]),
uri(PathPrefix ++ [?CONF_NS, "password-based:http"]),
InvalidConfig0),
InvalidConfig1 = ValidConfig0#{method => <<"get">>,
headers => #{<<"content-type">> => <<"application/json">>}},
{ok, 400, _} = request(
put,
uri(PathPrefix ++ ["authentication", "password-based:http"]),
uri(PathPrefix ++ [?CONF_NS, "password-based:http"]),
InvalidConfig1),
ValidConfig1 = ValidConfig0#{pool_size => 9},
{ok, 200, _} = request(
put,
uri(PathPrefix ++ ["authentication", "password-based:http"]),
uri(PathPrefix ++ [?CONF_NS, "password-based:http"]),
ValidConfig1),
{ok, 404, _} = request(
delete,
uri(PathPrefix ++ ["authentication", "password-based:redis"])),
uri(PathPrefix ++ [?CONF_NS, "password-based:redis"])),
{ok, 204, _} = request(
delete,
uri(PathPrefix ++ ["authentication", "password-based:http"])),
uri(PathPrefix ++ [?CONF_NS, "password-based:http"])),
?assertAuthenticatorsMatch([], PathPrefix ++ ["authentication"]).
?assertAuthenticatorsMatch([], PathPrefix ++ [?CONF_NS]).
test_authenticator_users(PathPrefix) ->
UsersUri = uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]),
UsersUri = uri(PathPrefix ++ [?CONF_NS, "password-based:built-in-database", "users"]),
{ok, 200, _} = request(
post,
uri(PathPrefix ++ ["authentication"]),
uri(PathPrefix ++ [?CONF_NS]),
emqx_authn_test_lib:built_in_database_example()),
InvalidUsers = [
@ -261,11 +263,11 @@ test_authenticator_users(PathPrefix) ->
lists:usort([ UserId || #{<<"user_id">> := UserId} <- Page1Users ++ Page2Users])).
test_authenticator_user(PathPrefix) ->
UsersUri = uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]),
UsersUri = uri(PathPrefix ++ [?CONF_NS, "password-based:built-in-database", "users"]),
{ok, 200, _} = request(
post,
uri(PathPrefix ++ ["authentication"]),
uri(PathPrefix ++ [?CONF_NS]),
emqx_authn_test_lib:built_in_database_example()),
User = #{user_id => <<"u1">>, password => <<"p1">>},
@ -309,7 +311,7 @@ test_authenticator_move(PathPrefix) ->
fun(Conf) ->
{ok, 200, _} = request(
post,
uri(PathPrefix ++ ["authentication"]),
uri(PathPrefix ++ [?CONF_NS]),
Conf)
end,
AuthenticatorConfs),
@ -320,40 +322,40 @@ test_authenticator_move(PathPrefix) ->
#{<<"mechanism">> := <<"jwt">>},
#{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>}
],
PathPrefix ++ ["authentication"]),
PathPrefix ++ [?CONF_NS]),
% Invalid moves
{ok, 400, _} = request(
post,
uri(PathPrefix ++ ["authentication", "jwt", "move"]),
uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]),
#{position => <<"up">>}),
{ok, 400, _} = request(
post,
uri(PathPrefix ++ ["authentication", "jwt", "move"]),
uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]),
#{}),
{ok, 404, _} = request(
post,
uri(PathPrefix ++ ["authentication", "jwt", "move"]),
uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]),
#{position => <<"before:invalid">>}),
{ok, 404, _} = request(
post,
uri(PathPrefix ++ ["authentication", "jwt", "move"]),
uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]),
#{position => <<"before:password-based:redis">>}),
{ok, 404, _} = request(
post,
uri(PathPrefix ++ ["authentication", "jwt", "move"]),
uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]),
#{position => <<"before:password-based:redis">>}),
% Valid moves
{ok, 204, _} = request(
post,
uri(PathPrefix ++ ["authentication", "jwt", "move"]),
uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]),
#{position => <<"top">>}),
?assertAuthenticatorsMatch(
@ -362,11 +364,11 @@ test_authenticator_move(PathPrefix) ->
#{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>},
#{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>}
],
PathPrefix ++ ["authentication"]),
PathPrefix ++ [?CONF_NS]),
{ok, 204, _} = request(
post,
uri(PathPrefix ++ ["authentication", "jwt", "move"]),
uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]),
#{position => <<"bottom">>}),
?assertAuthenticatorsMatch(
@ -375,11 +377,11 @@ test_authenticator_move(PathPrefix) ->
#{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>},
#{<<"mechanism">> := <<"jwt">>}
],
PathPrefix ++ ["authentication"]),
PathPrefix ++ [?CONF_NS]),
{ok, 204, _} = request(
post,
uri(PathPrefix ++ ["authentication", "jwt", "move"]),
uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]),
#{position => <<"before:password-based:built-in-database">>}),
?assertAuthenticatorsMatch(
@ -388,17 +390,17 @@ test_authenticator_move(PathPrefix) ->
#{<<"mechanism">> := <<"jwt">>},
#{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>}
],
PathPrefix ++ ["authentication"]).
PathPrefix ++ [?CONF_NS]).
test_authenticator_import_users(PathPrefix) ->
ImportUri = uri(
PathPrefix ++
["authentication", "password-based:built-in-database", "import_users"]),
[?CONF_NS, "password-based:built-in-database", "import_users"]),
{ok, 200, _} = request(
post,
uri(PathPrefix ++ ["authentication"]),
uri(PathPrefix ++ [?CONF_NS]),
emqx_authn_test_lib:built_in_database_example()),
{ok, 400, _} = request(post, ImportUri, #{}),

4
apps/emqx_authn/test/emqx_authn_http_SUITE.erl
View File

@ -24,7 +24,7 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").
-define(PATH, [authentication]).
-define(PATH, [?CONF_NS_ATOM]).
-define(HTTP_PORT, 33333).
-define(HTTP_PATH, "/auth").
@ -39,6 +39,7 @@ all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
_ = application:load(emqx_conf),
emqx_common_test_helpers:start_apps([emqx_authn]),
application:ensure_all_started(cowboy),
Config.
@ -52,6 +53,7 @@ end_per_suite(_) ->
ok.
init_per_testcase(_Case, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
emqx_authn_test_lib:delete_authenticators(
[authentication],
?GLOBAL),

95
apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl
View File

@ -21,15 +21,25 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include("emqx_authn.hrl").
-define(AUTHN_ID, <<"mechanism:jwt">>).
-define(JWKS_PORT, 33333).
-define(JWKS_PATH, "/jwks.json").
all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_testcase(_, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
Config.
init_per_suite(Config) ->
_ = application:load(emqx_conf),
emqx_common_test_helpers:start_apps([emqx_authn]),
Config.
@ -37,7 +47,11 @@ end_per_suite(_) ->
emqx_common_test_helpers:stop_apps([emqx_authn]),
ok.
t_jwt_authenticator(_) ->
%%------------------------------------------------------------------------------
%% Tests
%%------------------------------------------------------------------------------
t_jwt_authenticator_hmac_based(_) ->
Secret = <<"abcdef">>,
Config = #{mechanism => jwt,
use_jwks => false,
@ -121,10 +135,9 @@ t_jwt_authenticator(_) ->
?assertEqual(ok, emqx_authn_jwt:destroy(State3)),
ok.
t_jwt_authenticator2(_) ->
Dir = code:lib_dir(emqx_authn, test),
PublicKey = list_to_binary(filename:join([Dir, "data/public_key.pem"])),
PrivateKey = list_to_binary(filename:join([Dir, "data/private_key.pem"])),
t_jwt_authenticator_public_key(_) ->
PublicKey = test_rsa_key(public),
PrivateKey = test_rsa_key(private),
Config = #{mechanism => jwt,
use_jwks => false,
algorithm => 'public-key',
@ -142,6 +155,78 @@ t_jwt_authenticator2(_) ->
?assertEqual(ok, emqx_authn_jwt:destroy(State)),
ok.
t_jwks_renewal(_Config) ->
ok = emqx_authn_http_test_server:start(?JWKS_PORT, ?JWKS_PATH),
ok = emqx_authn_http_test_server:set_handler(fun jwks_handler/2),
PrivateKey = test_rsa_key(private),
Payload = #{<<"username">> => <<"myuser">>},
JWS = generate_jws('public-key', Payload, PrivateKey),
Credential = #{username => <<"myuser">>,
password => JWS},
BadConfig = #{mechanism => jwt,
algorithm => 'public-key',
ssl => #{enable => false},
verify_claims => [],
use_jwks => true,
endpoint => "http://127.0.0.1:" ++ integer_to_list(?JWKS_PORT + 1) ++ ?JWKS_PATH,
refresh_interval => 1000
},
ok = snabbkaffe:start_trace(),
{{ok, State0}, _} = ?wait_async_action(
emqx_authn_jwt:create(?AUTHN_ID, BadConfig),
#{?snk_kind := jwks_endpoint_response},
1000),
ok = snabbkaffe:stop(),
?assertEqual(ignore, emqx_authn_jwt:authenticate(Credential, State0)),
?assertEqual(ignore, emqx_authn_jwt:authenticate(Credential#{password => <<"badpassword">>}, State0)),
GoodConfig = BadConfig#{endpoint =>
"http://127.0.0.1:" ++ integer_to_list(?JWKS_PORT) ++ ?JWKS_PATH},
ok = snabbkaffe:start_trace(),
{{ok, State1}, _} = ?wait_async_action(
emqx_authn_jwt:update(GoodConfig, State0),
#{?snk_kind := jwks_endpoint_response},
1000),
ok = snabbkaffe:stop(),
?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential, State1)),
?assertEqual(ignore, emqx_authn_jwt:authenticate(Credential#{password => <<"badpassword">>}, State1)),
?assertEqual(ok, emqx_authn_jwt:destroy(State1)),
ok = emqx_authn_http_test_server:stop().
%%------------------------------------------------------------------------------
%% Helpers
%%------------------------------------------------------------------------------
jwks_handler(Req0, State) ->
JWK = jose_jwk:from_pem_file(test_rsa_key(public)),
JWKS = jose_jwk_set:to_map([JWK], #{}),
Req = cowboy_req:reply(
200,
#{<<"content-type">> => <<"application/json">>},
jiffy:encode(JWKS),
Req0),
{ok, Req, State}.
test_rsa_key(public) ->
Dir = code:lib_dir(emqx_authn, test),
list_to_binary(filename:join([Dir, "data/public_key.pem"]));
test_rsa_key(private) ->
Dir = code:lib_dir(emqx_authn, test),
list_to_binary(filename:join([Dir, "data/private_key.pem"])).
generate_jws('hmac-based', Payload, Secret) ->
JWK = jose_jwk:from_oct(Secret),
Header = #{ <<"alg">> => <<"HS256">>

10
apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl
View File

@ -29,6 +29,7 @@ all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
_ = application:load(emqx_conf),
emqx_common_test_helpers:start_apps([emqx_authn]),
Config.
@ -37,7 +38,8 @@ end_per_suite(_) ->
ok.
init_per_testcase(_Case, Config) ->
mnesia:clear_table(emqx_authn_mnesia),
{ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
mria:clear_table(emqx_authn_mnesia),
Config.
end_per_testcase(_Case, Config) ->
@ -47,6 +49,8 @@ end_per_testcase(_Case, Config) ->
%% Tests
%%------------------------------------------------------------------------------
-define(CONF(Conf), #{?CONF_NS_BINARY => Conf}).
t_check_schema(_Config) ->
ConfigOk = #{
<<"mechanism">> => <<"password-based">>,
@ -58,7 +62,7 @@ t_check_schema(_Config) ->
}
},
hocon_schema:check_plain(emqx_authn_mnesia, #{<<"config">> => ConfigOk}),
hocon_schema:check_plain(emqx_authn_mnesia, ?CONF(ConfigOk)),
ConfigNotOk = #{
<<"mechanism">> => <<"password-based">>,
@ -72,7 +76,7 @@ t_check_schema(_Config) ->
?assertException(
throw,
{emqx_authn_mnesia, _},
hocon_schema:check_plain(emqx_authn_mnesia, #{<<"config">> => ConfigNotOk})).
hocon_schema:check_plain(emqx_authn_mnesia, ?CONF(ConfigNotOk))).
t_create(_) ->
Config0 = config(),

409
apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl Normal file
View File

@ -0,0 +1,409 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_authn_mongo_SUITE).
-compile(nowarn_export_all).
-compile(export_all).
-include("emqx_authn.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-define(MONGO_HOST, "mongo").
-define(MONGO_PORT, 27017).
-define(MONGO_CLIENT, 'emqx_authn_mongo_SUITE_client').
-define(PATH, [authentication]).
all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_testcase(_TestCase, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
emqx_authentication:initialize_authentication(?GLOBAL, []),
emqx_authn_test_lib:delete_authenticators(
[authentication],
?GLOBAL),
{ok, _} = mc_worker_api:connect(mongo_config()),
Config.
end_per_testcase(_TestCase, _Config) ->
ok = mc_worker_api:disconnect(?MONGO_CLIENT).
init_per_suite(Config) ->
_ = application:load(emqx_conf),
case emqx_authn_test_lib:is_tcp_server_available(?MONGO_HOST, ?MONGO_PORT) of
true ->
ok = emqx_common_test_helpers:start_apps([emqx_authn]),
ok = start_apps([emqx_resource, emqx_connector]),
Config;
false ->
{skip, no_mongo}
end.
end_per_suite(_Config) ->
emqx_authn_test_lib:delete_authenticators(
[authentication],
?GLOBAL),
ok = stop_apps([emqx_resource, emqx_connector]),
ok = emqx_common_test_helpers:stop_apps([emqx_authn]).
%%------------------------------------------------------------------------------
%% Tests
%%------------------------------------------------------------------------------
t_create(_Config) ->
AuthConfig = raw_mongo_auth_config(),
{ok, _} = emqx:update_config(
?PATH,
{create_authenticator, ?GLOBAL, AuthConfig}),
{ok, [#{provider := emqx_authn_mongodb}]} = emqx_authentication:list_authenticators(?GLOBAL).
t_create_invalid(_Config) ->
AuthConfig = raw_mongo_auth_config(),
InvalidConfigs =
[
AuthConfig#{mongo_type => <<"unknown">>},
AuthConfig#{selector => <<"{ \"username\": \"${username}\" }">>}
],
lists:foreach(
fun(Config) ->
{error, _} = emqx:update_config(
?PATH,
{create_authenticator, ?GLOBAL, Config}),
{ok, []} = emqx_authentication:list_authenticators(?GLOBAL)
end,
InvalidConfigs).
t_authenticate(_Config) ->
ok = init_seeds(),
ok = lists:foreach(
fun(Sample) ->
ct:pal("test_user_auth sample: ~p", [Sample]),
test_user_auth(Sample)
end,
user_seeds()),
ok = drop_seeds().
test_user_auth(#{credentials := Credentials0,
config_params := SpecificConfigParams,
result := Result}) ->
AuthConfig = maps:merge(raw_mongo_auth_config(), SpecificConfigParams),
{ok, _} = emqx:update_config(
?PATH,
{create_authenticator, ?GLOBAL, AuthConfig}),
Credentials = Credentials0#{
listener => 'tcp:default',
protocol => mqtt
},
?assertEqual(Result, emqx_access_control:authenticate(Credentials)),
emqx_authn_test_lib:delete_authenticators(
[authentication],
?GLOBAL).
t_destroy(_Config) ->
ok = init_seeds(),
AuthConfig = raw_mongo_auth_config(),
{ok, _} = emqx:update_config(
?PATH,
{create_authenticator, ?GLOBAL, AuthConfig}),
{ok, [#{provider := emqx_authn_mongodb, state := State}]}
= emqx_authentication:list_authenticators(?GLOBAL),
{ok, _} = emqx_authn_mongodb:authenticate(
#{username => <<"plain">>,
password => <<"plain">>
},
State),
emqx_authn_test_lib:delete_authenticators(
[authentication],
?GLOBAL),
% Authenticator should not be usable anymore
?assertException(
error,
_,
emqx_authn_mongodb:authenticate(
#{username => <<"plain">>,
password => <<"plain">>
},
State)),
ok = drop_seeds().
t_update(_Config) ->
ok = init_seeds(),
CorrectConfig = raw_mongo_auth_config(),
IncorrectConfig =
CorrectConfig#{selector => #{<<"wrongfield">> => <<"wrongvalue">>}},
{ok, _} = emqx:update_config(
?PATH,
{create_authenticator, ?GLOBAL, IncorrectConfig}),
{error, not_authorized} = emqx_access_control:authenticate(
#{username => <<"plain">>,
password => <<"plain">>,
listener => 'tcp:default',
protocol => mqtt
}),
% We update with config with correct selector, provider should update and work properly
{ok, _} = emqx:update_config(
?PATH,
{update_authenticator, ?GLOBAL, <<"password-based:mongodb">>, CorrectConfig}),
{ok,_} = emqx_access_control:authenticate(
#{username => <<"plain">>,
password => <<"plain">>,
listener => 'tcp:default',
protocol => mqtt
}),
ok = drop_seeds().
t_is_superuser(_Config) ->
Config = raw_mongo_auth_config(),
{ok, _} = emqx:update_config(
?PATH,
{create_authenticator, ?GLOBAL, Config}),
Checks = [
{<<"0">>, false},
{<<"">>, false},
{null, false},
{false, false},
{0, false},
{<<"1">>, true},
{<<"val">>, true},
{1, true},
{123, true},
{true, true}
],
lists:foreach(fun test_is_superuser/1, Checks).
test_is_superuser({Value, ExpectedValue}) ->
{true, _} = mc_worker_api:delete(?MONGO_CLIENT, <<"users">>, #{}),
UserData = #{
username => <<"user">>,
password_hash => <<"plainsalt">>,
salt => <<"salt">>,
is_superuser => Value
},
{{true, _}, _} = mc_worker_api:insert(?MONGO_CLIENT, <<"users">>, [UserData]),
Credentials = #{
listener => 'tcp:default',
protocol => mqtt,
username => <<"user">>,
password => <<"plain">>
},
?assertEqual(
{ok, #{is_superuser => ExpectedValue}},
emqx_access_control:authenticate(Credentials)).
%%------------------------------------------------------------------------------
%% Helpers
%%------------------------------------------------------------------------------
raw_mongo_auth_config() ->
#{
mechanism => <<"password-based">>,
password_hash_algorithm => <<"plain">>,
salt_position => <<"suffix">>,
enable => <<"true">>,
backend => <<"mongodb">>,
mongo_type => <<"single">>,
database => <<"mqtt">>,
collection => <<"users">>,
server => mongo_server(),
selector => #{<<"username">> => <<"${username}">>},
password_hash_field => <<"password_hash">>,
salt_field => <<"salt">>,
is_superuser_field => <<"is_superuser">>
}.
user_seeds() ->
[#{data => #{
username => <<"plain">>,
password_hash => <<"plainsalt">>,
salt => <<"salt">>,
is_superuser => <<"1">>
},
credentials => #{
username => <<"plain">>,
password => <<"plain">>
},
config_params => #{
},
result => {ok,#{is_superuser => true}}
},
#{data => #{
username => <<"md5">>,
password_hash => <<"9b4d0c43d206d48279e69b9ad7132e22">>,
salt => <<"salt">>,
is_superuser => <<"0">>
},
credentials => #{
username => <<"md5">>,
password => <<"md5">>
},
config_params => #{
password_hash_algorithm => <<"md5">>,
salt_position => <<"suffix">>
},
result => {ok,#{is_superuser => false}}
},
#{data => #{
username => <<"sha256">>,
password_hash => <<"ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf">>,
salt => <<"salt">>,
is_superuser => 1
},
credentials => #{
clientid => <<"sha256">>,
password => <<"sha256">>
},
config_params => #{
selector => #{<<"username">> => <<"${clientid}">>},
password_hash_algorithm => <<"sha256">>,
salt_position => <<"prefix">>
},
result => {ok,#{is_superuser => true}}
},
#{data => #{
username => <<"bcrypt">>,
password_hash => <<"$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u">>,
salt => <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>,
is_superuser => 0
},
credentials => #{
username => <<"bcrypt">>,
password => <<"bcrypt">>
},
config_params => #{
password_hash_algorithm => <<"bcrypt">>,
salt_position => <<"suffix">> % should be ignored
},
result => {ok,#{is_superuser => false}}
},
#{data => #{
username => <<"bcrypt0">>,
password_hash => <<"$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u">>,
salt => <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>,
is_superuser => <<"0">>
},
credentials => #{
username => <<"bcrypt0">>,
password => <<"bcrypt">>
},
config_params => #{
% clientid variable & username credentials
selector => #{<<"username">> => <<"${clientid}">>},
password_hash_algorithm => <<"bcrypt">>,
salt_position => <<"suffix">>
},
result => {error,not_authorized}
},
#{data => #{
username => <<"bcrypt1">>,
password_hash => <<"$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u">>,
salt => <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>,
is_superuser => <<"0">>
},
credentials => #{
username => <<"bcrypt1">>,
password => <<"bcrypt">>
},
config_params => #{
selector => #{<<"userid">> => <<"${clientid}">>},
password_hash_algorithm => <<"bcrypt">>,
salt_position => <<"suffix">>
},
result => {error,not_authorized}
},
#{data => #{
username => <<"bcrypt2">>,
password_hash => <<"$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u">>,
salt => <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>,
is_superuser => <<"0">>
},
credentials => #{
username => <<"bcrypt2">>,
% Wrong password
password => <<"wrongpass">>
},
config_params => #{
password_hash_algorithm => <<"bcrypt">>,
salt_position => <<"suffix">>
},
result => {error,bad_username_or_password}
}
].
init_seeds() ->
Users = [Values || #{data := Values} <- user_seeds()],
{{true, _}, _} = mc_worker_api:insert(?MONGO_CLIENT, <<"users">>, Users),
ok.
drop_seeds() ->
{true, _} = mc_worker_api:delete(?MONGO_CLIENT, <<"users">>, #{}),
ok.
mongo_server() ->
iolist_to_binary(
io_lib:format(
"~s:~b",
[?MONGO_HOST, ?MONGO_PORT])).
mongo_config() ->
[
{database, <<"mqtt">>},
{host, ?MONGO_HOST},
{port, ?MONGO_PORT},
{register, ?MONGO_CLIENT}
].
start_apps(Apps) ->
lists:foreach(fun application:ensure_all_started/1, Apps).
stop_apps(Apps) ->
lists:foreach(fun application:stop/1, Apps).

115
apps/emqx_authn/test/emqx_authn_mqtt_test_client.erl Normal file
View File

@ -0,0 +1,115 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_authn_mqtt_test_client).
-behaviour(gen_server).
-include_lib("emqx/include/emqx_mqtt.hrl").
%% API
-export([start_link/2,
stop/1]).
-export([send/2]).
%% gen_server callbacks
-export([init/1,
handle_call/3,
handle_cast/2,
handle_info/2,
terminate/2]).
-define(TIMEOUT, 1000).
-define(TCP_OPTIONS, [binary, {packet, raw}, {active, once},
{nodelay, true}]).
-define(PARSE_OPTIONS,
#{strict_mode => false,
max_size => ?MAX_PACKET_SIZE,
version => ?MQTT_PROTO_V5
}).
%%--------------------------------------------------------------------
%% API
%%--------------------------------------------------------------------
start_link(Host, Port) ->
gen_server:start_link(?MODULE, [Host, Port, self()], []).
stop(Pid) ->
gen_server:call(Pid, stop).
send(Pid, Packet) ->
gen_server:call(Pid, {send, Packet}).
%%--------------------------------------------------------------------
%% gen_server callbacks
%%--------------------------------------------------------------------
init([Host, Port, Owner]) ->
{ok, Socket} = gen_tcp:connect(Host, Port, ?TCP_OPTIONS, ?TIMEOUT),
{ok, #{owner => Owner,
socket => Socket,
parse_state => emqx_frame:initial_parse_state(?PARSE_OPTIONS)
}}.
handle_info({tcp, _Sock, Data}, #{parse_state := PSt,
owner := Owner,
socket := Socket} = St) ->
{NewPSt, Packets} = process_incoming(PSt, Data, []),
ok = deliver(Owner, Packets),
ok = run_sock(Socket),
{noreply, St#{parse_state => NewPSt}};
handle_info({tcp_closed, _Sock}, St) ->
{stop, normal, St}.
handle_call({send, Packet}, _From, #{socket := Socket} = St) ->
ok = gen_tcp:send(Socket, emqx_frame:serialize(Packet, ?MQTT_PROTO_V5)),
{reply, ok, St};
handle_call(stop, _From, #{socket := Socket} = St) ->
ok = gen_tcp:close(Socket),
{stop, normal, ok, St}.
handle_cast(_, St) ->
{noreply, St}.
terminate(_Reason, _St) ->
ok.
%%--------------------------------------------------------------------
%% internal functions
%%--------------------------------------------------------------------
process_incoming(PSt, Data, Packets) ->
case emqx_frame:parse(Data, PSt) of
{more, NewPSt} ->
{NewPSt, lists:reverse(Packets)};
{ok, Packet, Rest, NewPSt} ->
process_incoming(NewPSt, Rest, [Packet | Packets])
end.
deliver(_Owner, []) -> ok;
deliver(Owner, [Packet | Packets]) ->
Owner ! {packet, Packet},
deliver(Owner, Packets).
run_sock(Socket) ->
inet:setopts(Socket, [{active, once}]).

6
apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl
View File

@ -38,6 +38,7 @@ groups() ->
[{require_seeds, [], [t_authenticate, t_update, t_destroy]}].
init_per_testcase(_, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
emqx_authentication:initialize_authentication(?GLOBAL, []),
emqx_authn_test_lib:delete_authenticators(
[authentication],
@ -53,6 +54,7 @@ end_per_group(require_seeds, Config) ->
Config.
init_per_suite(Config) ->
_ = application:load(emqx_conf),
case emqx_authn_test_lib:is_tcp_server_available(?MYSQL_HOST, ?MYSQL_PORT) of
true ->
ok = emqx_common_test_helpers:start_apps([emqx_authn]),
@ -117,9 +119,9 @@ t_authenticate(_Config) ->
user_seeds()).
test_user_auth(#{credentials := Credentials0,
config_params := SpecificConfgParams,
config_params := SpecificConfigParams,
result := Result}) ->
AuthConfig = maps:merge(raw_mysql_auth_config(), SpecificConfgParams),
AuthConfig = maps:merge(raw_mysql_auth_config(), SpecificConfigParams),
{ok, _} = emqx:update_config(
?PATH,

6
apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl
View File

@ -38,6 +38,7 @@ groups() ->
[{require_seeds, [], [t_authenticate, t_update, t_destroy, t_is_superuser]}].
init_per_testcase(_, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
emqx_authentication:initialize_authentication(?GLOBAL, []),
emqx_authn_test_lib:delete_authenticators(
[authentication],
@ -53,6 +54,7 @@ end_per_group(require_seeds, Config) ->
Config.
init_per_suite(Config) ->
_ = application:load(emqx_conf),
case emqx_authn_test_lib:is_tcp_server_available(?PGSQL_HOST, ?PGSQL_PORT) of
true ->
ok = emqx_common_test_helpers:start_apps([emqx_authn]),
@ -117,9 +119,9 @@ t_authenticate(_Config) ->
user_seeds()).
test_user_auth(#{credentials := Credentials0,
config_params := SpecificConfgParams,
config_params := SpecificConfigParams,
result := Result}) ->
AuthConfig = maps:merge(raw_pgsql_auth_config(), SpecificConfgParams),
AuthConfig = maps:merge(raw_pgsql_auth_config(), SpecificConfigParams),
{ok, _} = emqx:update_config(
?PATH,

28
apps/emqx_authn/test/emqx_authn_redis_SUITE.erl
View File

@ -23,12 +23,10 @@
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-define(REDIS_HOST, "redis").
-define(REDIS_PORT, 6379).
-define(REDIS_RESOURCE, <<"emqx_authn_redis_SUITE">>).
-define(PATH, [authentication]).
all() ->
@ -38,6 +36,7 @@ groups() ->
[{require_seeds, [], [t_authenticate, t_update, t_destroy]}].
init_per_testcase(_, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
emqx_authentication:initialize_authentication(?GLOBAL, []),
emqx_authn_test_lib:delete_authenticators(
[authentication],
@ -53,6 +52,7 @@ end_per_group(require_seeds, Config) ->
Config.
init_per_suite(Config) ->
_ = application:load(emqx_conf),
case emqx_authn_test_lib:is_tcp_server_available(?REDIS_HOST, ?REDIS_PORT) of
true ->
ok = emqx_common_test_helpers:start_apps([emqx_authn]),
@ -98,11 +98,11 @@ t_create_invalid(_Config) ->
AuthConfig#{password => <<"wrongpass">>},
AuthConfig#{database => <<"5678">>},
AuthConfig#{
query => <<"MGET password_hash:${username} salt:${username}">>},
cmd => <<"MGET password_hash:${username} salt:${username}">>},
AuthConfig#{
query => <<"HMGET mqtt_user:${username} password_hash invalid_field">>},
cmd => <<"HMGET mqtt_user:${username} password_hash invalid_field">>},
AuthConfig#{
query => <<"HMGET mqtt_user:${username} salt is_superuser">>}
cmd => <<"HMGET mqtt_user:${username} salt is_superuser">>}
],
lists:foreach(
@ -124,9 +124,9 @@ t_authenticate(_Config) ->
user_seeds()).
test_user_auth(#{credentials := Credentials0,
config_params := SpecificConfgParams,
config_params := SpecificConfigParams,
result := Result}) ->
AuthConfig = maps:merge(raw_redis_auth_config(), SpecificConfgParams),
AuthConfig = maps:merge(raw_redis_auth_config(), SpecificConfigParams),
{ok, _} = emqx:update_config(
?PATH,
@ -177,7 +177,7 @@ t_update(_Config) ->
CorrectConfig = raw_redis_auth_config(),
IncorrectConfig =
CorrectConfig#{
query => <<"HMGET invalid_key:${username} password_hash salt is_superuser">>},
cmd => <<"HMGET invalid_key:${username} password_hash salt is_superuser">>},
{ok, _} = emqx:update_config(
?PATH,
@ -214,7 +214,7 @@ raw_redis_auth_config() ->
enable => <<"true">>,
backend => <<"redis">>,
query => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>,
cmd => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>,
database => <<"1">>,
password => <<"public">>,
server => redis_server()
@ -262,7 +262,7 @@ user_seeds() ->
},
key => "mqtt_user:sha256",
config_params => #{
query => <<"HMGET mqtt_user:${clientid} password_hash salt is_superuser">>,
cmd => <<"HMGET mqtt_user:${clientid} password_hash salt is_superuser">>,
password_hash_algorithm => <<"sha256">>,
salt_position => <<"prefix">>
},
@ -298,7 +298,7 @@ user_seeds() ->
key => "mqtt_user:bcrypt0",
config_params => #{
% clientid variable & username credentials
query => <<"HMGET mqtt_client:${clientid} password_hash salt is_superuser">>,
cmd => <<"HMGET mqtt_client:${clientid} password_hash salt is_superuser">>,
password_hash_algorithm => <<"bcrypt">>,
salt_position => <<"suffix">>
},
@ -316,8 +316,8 @@ user_seeds() ->
},
key => "mqtt_user:bcrypt1",
config_params => #{
% Bad key in query
query => <<"HMGET badkey:${username} password_hash salt is_superuser">>,
% Bad key in cmd
cmd => <<"HMGET badkey:${username} password_hash salt is_superuser">>,
password_hash_algorithm => <<"bcrypt">>,
salt_position => <<"suffix">>
},
@ -336,7 +336,7 @@ user_seeds() ->
},
key => "mqtt_user:bcrypt2",
config_params => #{
query => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>,
cmd => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>,
password_hash_algorithm => <<"bcrypt">>,
salt_position => <<"suffix">>
},

375
apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl Normal file
View File

@ -0,0 +1,375 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_enhanced_authn_scram_mnesia_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include("emqx_authn.hrl").
-define(PATH, [authentication]).
-define(USER_MAP, #{user_id := _,
is_superuser := _}).
all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
_ = application:load(emqx_conf),
ok = emqx_common_test_helpers:start_apps([emqx_authn]),
Config.
end_per_suite(_Config) ->
ok = emqx_common_test_helpers:stop_apps([emqx_authn]).
init_per_testcase(_Case, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
mria:clear_table(emqx_enhanced_authn_scram_mnesia),
emqx_authn_test_lib:delete_authenticators(
[authentication],
?GLOBAL),
Config.
end_per_testcase(_Case, Config) ->
Config.
%%------------------------------------------------------------------------------
%% Tests
%%------------------------------------------------------------------------------
t_create(_Config) ->
ValidConfig = #{
<<"mechanism">> => <<"scram">>,
<<"backend">> => <<"built-in-database">>,
<<"algorithm">> => <<"sha512">>,
<<"iteration_count">> => <<"4096">>
},
{ok, _} = emqx:update_config(
?PATH,
{create_authenticator, ?GLOBAL, ValidConfig}),
{ok, [#{provider := emqx_enhanced_authn_scram_mnesia}]}
= emqx_authentication:list_authenticators(?GLOBAL).
t_create_invalid(_Config) ->
InvalidConfig = #{
<<"mechanism">> => <<"scram">>,
<<"backend">> => <<"built-in-database">>,
<<"algorithm">> => <<"sha271828">>,
<<"iteration_count">> => <<"4096">>
},
{error, _} = emqx:update_config(
?PATH,
{create_authenticator, ?GLOBAL, InvalidConfig}),
{ok, []} = emqx_authentication:list_authenticators(?GLOBAL).
t_authenticate(_Config) ->
Algorithm = sha512,
Username = <<"u">>,
Password = <<"p">>,
init_auth(Username, Password, Algorithm),
{ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883),
ClientFirstMessage = esasl_scram:client_first_message(Username),
ConnectPacket = ?CONNECT_PACKET(
#mqtt_packet_connect{
proto_ver = ?MQTT_PROTO_V5,
properties = #{
'Authentication-Method' => <<"SCRAM-SHA-512">>,
'Authentication-Data' => ClientFirstMessage
}
}),
ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket),
?AUTH_PACKET(
?RC_CONTINUE_AUTHENTICATION,
#{'Authentication-Data' := ServerFirstMessage}) = receive_packet(),
{continue, ClientFinalMessage, ClientCache} =
esasl_scram:check_server_first_message(
ServerFirstMessage,
#{client_first_message => ClientFirstMessage,
password => Password,
algorithm => Algorithm}
),
AuthContinuePacket = ?AUTH_PACKET(
?RC_CONTINUE_AUTHENTICATION,
#{'Authentication-Method' => <<"SCRAM-SHA-512">>,
'Authentication-Data' => ClientFinalMessage}),
ok = emqx_authn_mqtt_test_client:send(Pid, AuthContinuePacket),
?CONNACK_PACKET(
?RC_SUCCESS,
_,
#{'Authentication-Data' := ServerFinalMessage}) = receive_packet(),
ok = esasl_scram:check_server_final_message(
ServerFinalMessage, ClientCache#{algorithm => Algorithm}
).
t_authenticate_bad_username(_Config) ->
Algorithm = sha512,
Username = <<"u">>,
Password = <<"p">>,
init_auth(Username, Password, Algorithm),
{ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883),
ClientFirstMessage = esasl_scram:client_first_message(<<"badusername">>),
ConnectPacket = ?CONNECT_PACKET(
#mqtt_packet_connect{
proto_ver = ?MQTT_PROTO_V5,
properties = #{
'Authentication-Method' => <<"SCRAM-SHA-512">>,
'Authentication-Data' => ClientFirstMessage
}
}),
ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket),
?CONNACK_PACKET(?RC_NOT_AUTHORIZED) = receive_packet().
t_authenticate_bad_password(_Config) ->
Algorithm = sha512,
Username = <<"u">>,
Password = <<"p">>,
init_auth(Username, Password, Algorithm),
{ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883),
ClientFirstMessage = esasl_scram:client_first_message(Username),
ConnectPacket = ?CONNECT_PACKET(
#mqtt_packet_connect{
proto_ver = ?MQTT_PROTO_V5,
properties = #{
'Authentication-Method' => <<"SCRAM-SHA-512">>,
'Authentication-Data' => ClientFirstMessage
}
}),
ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket),
?AUTH_PACKET(
?RC_CONTINUE_AUTHENTICATION,
#{'Authentication-Data' := ServerFirstMessage}) = receive_packet(),
{continue, ClientFinalMessage, _ClientCache} =
esasl_scram:check_server_first_message(
ServerFirstMessage,
#{client_first_message => ClientFirstMessage,
password => <<"badpassword">>,
algorithm => Algorithm}
),
AuthContinuePacket = ?AUTH_PACKET(
?RC_CONTINUE_AUTHENTICATION,
#{'Authentication-Method' => <<"SCRAM-SHA-512">>,
'Authentication-Data' => ClientFinalMessage}),
ok = emqx_authn_mqtt_test_client:send(Pid, AuthContinuePacket),
?CONNACK_PACKET(?RC_NOT_AUTHORIZED) = receive_packet().
t_destroy(_) ->
Config = config(),
OtherId = list_to_binary([<<"id-other">>]),
{ok, State0} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config),
{ok, StateOther} = emqx_enhanced_authn_scram_mnesia:create(OtherId, Config),
User = #{user_id => <<"u">>, password => <<"p">>},
{ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(User, State0),
{ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(User, StateOther),
{ok, _} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, State0),
{ok, _} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, StateOther),
ok = emqx_enhanced_authn_scram_mnesia:destroy(State0),
{ok, State1} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config),
{error,not_found} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, State1),
{ok, _} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, StateOther).
t_add_user(_) ->
Config = config(),
{ok, State} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config),
User = #{user_id => <<"u">>, password => <<"p">>},
{ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(User, State),
{error, already_exist} = emqx_enhanced_authn_scram_mnesia:add_user(User, State).
t_delete_user(_) ->
Config = config(),
{ok, State} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config),
{error, not_found} = emqx_enhanced_authn_scram_mnesia:delete_user(<<"u">>, State),
User = #{user_id => <<"u">>, password => <<"p">>},
{ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(User, State),
ok = emqx_enhanced_authn_scram_mnesia:delete_user(<<"u">>, State),
{error, not_found} = emqx_enhanced_authn_scram_mnesia:delete_user(<<"u">>, State).
t_update_user(_) ->
Config = config(),
{ok, State} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config),
User = #{user_id => <<"u">>, password => <<"p">>},
{ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(User, State),
{ok, #{is_superuser := false}} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, State),
{ok,
#{user_id := <<"u">>,
is_superuser := true}} = emqx_enhanced_authn_scram_mnesia:update_user(
<<"u">>,
#{password => <<"p1">>, is_superuser => true},
State),
{ok, #{is_superuser := true}} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, State).
t_list_users(_) ->
Config = config(),
{ok, State} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config),
Users = [#{user_id => <<"u1">>, password => <<"p">>},
#{user_id => <<"u2">>, password => <<"p">>},
#{user_id => <<"u3">>, password => <<"p">>}],
lists:foreach(
fun(U) -> {ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(U, State) end,
Users),
{ok,
#{data := [?USER_MAP, ?USER_MAP],
meta := #{page := 1, limit := 2, count := 3}}} = emqx_enhanced_authn_scram_mnesia:list_users(
#{<<"page">> => 1, <<"limit">> => 2},
State),
{ok,
#{data := [?USER_MAP],
meta := #{page := 2, limit := 2, count := 3}}} = emqx_enhanced_authn_scram_mnesia:list_users(
#{<<"page">> => 2, <<"limit">> => 2},
State).
t_is_superuser(_Config) ->
ok = test_is_superuser(#{is_superuser => false}, false),
ok = test_is_superuser(#{is_superuser => true}, true),
ok = test_is_superuser(#{}, false).
test_is_superuser(UserInfo, ExpectedIsSuperuser) ->
Config = config(),
{ok, State} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config),
Username = <<"u">>,
Password = <<"p">>,
UserInfo0 = UserInfo#{user_id => Username,
password => Password},
{ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(UserInfo0, State),
ClientFirstMessage = esasl_scram:client_first_message(Username),
{continue, ServerFirstMessage, ServerCache}
= emqx_enhanced_authn_scram_mnesia:authenticate(
#{auth_method => <<"SCRAM-SHA-512">>,
auth_data => ClientFirstMessage,
auth_cache => #{}
},
State),
{continue, ClientFinalMessage, ClientCache} =
esasl_scram:check_server_first_message(
ServerFirstMessage,
#{client_first_message => ClientFirstMessage,
password => Password,
algorithm => sha512}
),
{ok, UserInfo1, ServerFinalMessage}
= emqx_enhanced_authn_scram_mnesia:authenticate(
#{auth_method => <<"SCRAM-SHA-512">>,
auth_data => ClientFinalMessage,
auth_cache => ServerCache
},
State),
ok = esasl_scram:check_server_final_message(
ServerFinalMessage, ClientCache#{algorithm => sha512}
),
?assertMatch(#{is_superuser := ExpectedIsSuperuser}, UserInfo1),
ok = emqx_enhanced_authn_scram_mnesia:destroy(State).
%%------------------------------------------------------------------------------
%% Helpers
%%------------------------------------------------------------------------------
config() ->
#{
mechanism => <<"scram">>,
backend => <<"built-in-database">>,
algorithm => sha512,
iteration_count => 4096
}.
raw_config(Algorithm) ->
#{
<<"mechanism">> => <<"scram">>,
<<"backend">> => <<"built-in-database">>,
<<"algorithm">> => atom_to_binary(Algorithm),
<<"iteration_count">> => <<"4096">>
}.
init_auth(Username, Password, Algorithm) ->
Config = raw_config(Algorithm),
{ok, _} = emqx:update_config(
?PATH,
{create_authenticator, ?GLOBAL, Config}),
{ok, [#{state := State}]} = emqx_authentication:list_authenticators(?GLOBAL),
emqx_enhanced_authn_scram_mnesia:add_user(
#{user_id => Username, password => Password},
State).
receive_packet() ->
receive
{packet, Packet} ->
ct:pal("Delivered packet: ~p", [Packet]),
Packet
after 1000 ->
ct:fail("Deliver timeout")
end.

6
apps/emqx_authz/README.md
View File

@ -23,7 +23,7 @@ authz:{
keyfile: "etc/certs/client-key.pem"
}
}
sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or clientid = '%c'"
sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = ${peerhost} or username = ${username} or clientid = ${clientid}"
},
{
type: postgresql
@ -36,7 +36,7 @@ authz:{
auto_reconnect: true
ssl: {enable: false}
}
sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'"
sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = ${peerhost} or username = ${username} or username = '$all' or clientid = ${clientid}"
},
{
type: redis
@ -48,7 +48,7 @@ authz:{
auto_reconnect: true
ssl: {enable: false}
}
cmd: "HGETALL mqtt_authz:%u"
cmd: "HGETALL mqtt_authz:${username}"
},
{
principal: {username: "^admin?"}

8
apps/emqx_authz/etc/emqx_authz.conf
View File

@ -22,7 +22,7 @@ authorization {
# certfile: "{{ platform_etc_dir }}/certs/client-cert.pem"
# keyfile: "{{ platform_etc_dir }}/certs/client-key.pem"
# }
# query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or clientid = '%c'"
# query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = ${peerhost} or username = ${username} or clientid = ${clientid}"
# },
# {
# type: postgresql
@ -33,7 +33,7 @@ authorization {
# password: public
# auto_reconnect: true
# ssl: {enable: false}
# query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'"
# query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = ${peerhost} or username = ${username} or username = '$all' or clientid = ${clientid}"
# },
# {
# type: redis
@ -43,7 +43,7 @@ authorization {
# password: public
# auto_reconnect: true
# ssl: {enable: false}
# cmd: "HGETALL mqtt_authz:%u"
# cmd: "HGETALL mqtt_authz:${username}"
# },
# {
# type: mongodb
@ -53,7 +53,7 @@ authorization {
# database: mqtt
# ssl: {enable: false}
# collection: mqtt_authz
# selector: { "$or": [ { "username": "%u" }, { "clientid": "%c" } ] }
# selector: { "$or": [ { "username": "${username}" }, { "clientid": "${clientid}" } ] }
# },
{
type: built-in-database

53
apps/emqx_authz/src/emqx_authz.erl
View File

@ -40,6 +40,8 @@
-export([acl_conf_file/0]).
-export([ph_to_re/1]).
-spec(register_metrics() -> ok).
register_metrics() ->
lists:foreach(fun emqx_metrics:ensure/1, ?AUTHZ_METRICS).
@ -64,11 +66,14 @@ move(Type, Cmd) ->
move(Type, Cmd, #{}).
move(Type, #{<<"before">> := Before}, Opts) ->
emqx:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))}, Opts);
emqx:update_config( ?CONF_KEY_PATH
, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))}, Opts);
move(Type, #{<<"after">> := After}, Opts) ->
emqx:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_AFTER(type(After))}, Opts);
emqx:update_config( ?CONF_KEY_PATH
, {?CMD_MOVE, type(Type), ?CMD_MOVE_AFTER(type(After))}, Opts);
move(Type, Position, Opts) ->
emqx:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), Position}, Opts).
emqx:update_config( ?CONF_KEY_PATH
, {?CMD_MOVE, type(Type), Position}, Opts).
update(Cmd, Sources) ->
update(Cmd, Sources, #{}).
@ -155,7 +160,8 @@ do_post_update({{?CMD_REPLACE, Type}, Source}, _NewSources) when is_map(Source)
{OldSource, Front, Rear} = take(Type, OldInitedSources),
ok = ensure_resource_deleted(OldSource),
InitedSources = init_sources(check_sources([Source])),
ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Front ++ InitedSources ++ Rear]}, -1),
ok = emqx_hooks:put( 'client.authorize'
, {?MODULE, authorize, [Front ++ InitedSources ++ Rear]}, -1),
ok = emqx_authz_cache:drain_cache();
do_post_update({{?CMD_DELETE, Type}, _Source}, _NewSources) ->
OldInitedSources = lookup(),
@ -201,7 +207,12 @@ check_dup_types([Source | Sources], Checked) ->
create_dry_run(T, Source) ->
case is_connector_source(T) of
true ->
[NSource] = check_sources([Source]),
[CheckedSource] = check_sources([Source]),
case T of
http ->
URIMap = maps:get(url, CheckedSource),
NSource = maps:put(base_url, maps:remove(query, URIMap), CheckedSource)
end,
emqx_resource:create_dry_run(connector_module(T), NSource);
false ->
ok
@ -267,7 +278,7 @@ init_source(#{type := DB,
{error, Reason} -> error({load_config_error, Reason});
Id -> Source#{annotations =>
#{id => Id,
query => Mod:parse_query(SQL)
query => erlang:apply(Mod, parse_query, [SQL])
}
}
end.
@ -277,22 +288,36 @@ init_source(#{type := DB,
%%--------------------------------------------------------------------
%% @doc Check AuthZ
-spec(authorize(emqx_types:clientinfo(), emqx_types:all(), emqx_types:topic(), allow | deny, sources())
-spec(authorize( emqx_types:clientinfo()
, emqx_types:all()
, emqx_types:topic()
, allow | deny
, sources())
-> {stop, allow} | {ok, deny}).
authorize(#{username := Username,
peerhost := IpAddress
} = Client, PubSub, Topic, DefaultResult, Sources) ->
case do_authorize(Client, PubSub, Topic, Sources) of
{matched, allow} ->
?SLOG(info, #{msg => "authorization_permission_allowed", username => Username, ipaddr => IpAddress, topic => Topic}),
?SLOG(info, #{msg => "authorization_permission_allowed",
username => Username,
ipaddr => IpAddress,
topic => Topic}),
emqx_metrics:inc(?AUTHZ_METRICS(allow)),
{stop, allow};
{matched, deny} ->
?SLOG(info, #{msg => "authorization_permission_denied", username => Username, ipaddr => IpAddress, topic => Topic}),
?SLOG(info, #{msg => "authorization_permission_denied",
username => Username,
ipaddr => IpAddress,
topic => Topic}),
emqx_metrics:inc(?AUTHZ_METRICS(deny)),
{stop, deny};
nomatch ->
?SLOG(info, #{msg => "authorization_failed_nomatch", username => Username, ipaddr => IpAddress, topic => Topic, reason => "no-match rule"}),
?SLOG(info, #{msg => "authorization_failed_nomatch",
username => Username,
ipaddr => IpAddress,
topic => Topic,
reason => "no-match rule"}),
{stop, DefaultResult}
end.
@ -309,7 +334,7 @@ do_authorize(Client, PubSub, Topic, [#{type := file} = F | Tail]) ->
do_authorize(Client, PubSub, Topic,
[Connector = #{type := Type} | Tail] ) ->
Mod = authz_module(Type),
case Mod:authorize(Client, PubSub, Topic, Connector) of
case erlang:apply(Mod, authorize, [Client, PubSub, Topic, Connector]) of
nomatch -> do_authorize(Client, PubSub, Topic, Tail);
Matched -> Matched
end.
@ -381,8 +406,12 @@ type(postgresql) -> postgresql;
type(<<"postgresql">>) -> postgresql;
type('built-in-database') -> 'built-in-database';
type(<<"built-in-database">>) -> 'built-in-database';
type(Unknown) -> error({unknown_authz_source_type, Unknown}). % should never happend if the input is type-checked by hocon schema
%% should never happend if the input is type-checked by hocon schema
type(Unknown) -> error({unknown_authz_source_type, Unknown}).
%% @doc where the acl.conf file is stored.
acl_conf_file() ->
filename:join([emqx:data_dir(), "authz", "acl.conf"]).
ph_to_re(VarPH) ->
re:replace(VarPH, "[\\$\\{\\}]", "\\\\&", [global, {return, list}]).

5
apps/emqx_authz/src/emqx_authz_api_sources.erl
View File

@ -440,14 +440,15 @@ read_certs(#{<<"ssl">> := SSL} = Source) ->
{error, Reason} ->
?SLOG(error, Reason#{msg => failed_to_readd_ssl_file}),
throw(failed_to_readd_ssl_file);
NewSSL ->
{ok, NewSSL} ->
Source#{<<"ssl">> => NewSSL}
end;
read_certs(Source) -> Source.
maybe_write_certs(#{<<"ssl">> := #{<<"enable">> := true} = SSL} = Source) ->
Type = maps:get(<<"type">>, Source),
emqx_tls_lib:ensure_ssl_files(filename:join(["authz", Type]), SSL);
{ok, Return} = emqx_tls_lib:ensure_ssl_files(filename:join(["authz", Type]), SSL),
maps:put(<<"ssl">>, Return, Source);
maybe_write_certs(Source) -> Source.
write_file(Filename, Bytes0) ->

55
apps/emqx_authz/src/emqx_authz_http.erl
View File

@ -24,6 +24,7 @@
%% AuthZ Callbacks
-export([ authorize/4
, description/0
, parse_url/1
]).
-ifdef(TEST).
@ -36,7 +37,7 @@ description() ->
authorize(Client, PubSub, Topic,
#{type := http,
url := #{path := Path} = Url,
url := #{path := Path} = URL,
headers := Headers,
method := Method,
request_timeout := RequestTimeout,
@ -44,7 +45,7 @@ authorize(Client, PubSub, Topic,
} = Source) ->
Request = case Method of
get ->
Query = maps:get(query, Url, ""),
Query = maps:get(query, URL, ""),
Path1 = replvar(Path ++ "?" ++ Query, PubSub, Topic, Client),
{Path1, maps:to_list(Headers)};
_ ->
@ -56,10 +57,32 @@ authorize(Client, PubSub, Topic,
Path1 = replvar(Path, PubSub, Topic, Client),
{Path1, maps:to_list(Headers), Body1}
end,
case emqx_resource:query(ResourceID, {Method, Request, RequestTimeout}) of
{ok, 204, _Headers} -> {matched, allow};
{ok, 200, _Headers, _Body} -> {matched, allow};
_ -> nomatch
case emqx_resource:query(ResourceID, {Method, Request, RequestTimeout}) of
{ok, 200, _Headers} ->
{matched, allow};
{ok, 204, _Headers} ->
{matched, allow};
{ok, 200, _Headers, _Body} ->
{matched, allow};
{ok, _Status, _Headers, _Body} ->
nomatch;
{error, Reason} ->
?SLOG(error, #{msg => "http_server_query_failed",
resource => ResourceID,
reason => Reason}),
ignore
end.
parse_url(URL)
when URL =:= undefined ->
#{};
parse_url(URL) ->
{ok, URIMap} = emqx_http_lib:uri_parse(URL),
case maps:get(query, URIMap, undefined) of
undefined ->
URIMap#{query => ""};
_ ->
URIMap
end.
query_string(Body) ->
@ -87,19 +110,19 @@ replvar(Str0, PubSub, Topic,
}) when is_list(Str0);
is_binary(Str0) ->
NTopic = emqx_http_lib:uri_encode(Topic),
Str1 = re:replace( Str0, ?PH_S_CLIENTID
, Clientid, [global, {return, binary}]),
Str2 = re:replace( Str1, ?PH_S_USERNAME
Str1 = re:replace( Str0, emqx_authz:ph_to_re(?PH_S_CLIENTID)
, bin(Clientid), [global, {return, binary}]),
Str2 = re:replace( Str1, emqx_authz:ph_to_re(?PH_S_USERNAME)
, bin(Username), [global, {return, binary}]),
Str3 = re:replace( Str2, ?PH_S_HOST
Str3 = re:replace( Str2, emqx_authz:ph_to_re(?PH_S_HOST)
, inet_parse:ntoa(IpAddress), [global, {return, binary}]),
Str4 = re:replace( Str3, ?PH_S_PROTONAME
Str4 = re:replace( Str3, emqx_authz:ph_to_re(?PH_S_PROTONAME)
, bin(Protocol), [global, {return, binary}]),
Str5 = re:replace( Str4, ?PH_S_MOUNTPOINT
, Mountpoint, [global, {return, binary}]),
Str6 = re:replace( Str5, ?PH_S_TOPIC
, NTopic, [global, {return, binary}]),
Str7 = re:replace( Str6, ?PH_S_ACTION
Str5 = re:replace( Str4, emqx_authz:ph_to_re(?PH_S_MOUNTPOINT)
, bin(Mountpoint), [global, {return, binary}]),
Str6 = re:replace( Str5, emqx_authz:ph_to_re(?PH_S_TOPIC)
, bin(NTopic), [global, {return, binary}]),
Str7 = re:replace( Str6, emqx_authz:ph_to_re(?PH_S_ACTION)
, bin(PubSub), [global, {return, binary}]),
Str7.

6
apps/emqx_authz/src/emqx_authz_mongodb.erl
View File

@ -76,11 +76,11 @@ replvar(Selector, #{clientid := Clientid,
end || M <- V],
AccIn);
InFun(K, V, AccIn) when is_binary(V) ->
V1 = re:replace( V, ?PH_S_CLIENTID
V1 = re:replace( V, emqx_authz:ph_to_re(?PH_S_CLIENTID)
, bin(Clientid), [global, {return, binary}]),
V2 = re:replace( V1, ?PH_S_USERNAME
V2 = re:replace( V1, emqx_authz:ph_to_re(?PH_S_USERNAME)
, bin(Username), [global, {return, binary}]),
V3 = re:replace( V2, ?PH_S_HOST
V3 = re:replace( V2, emqx_authz:ph_to_re(?PH_S_HOST)
, inet_parse:ntoa(IpAddress), [global, {return, binary}]),
maps:put(K, V3, AccIn);
InFun(K, V, AccIn) -> maps:put(K, V, AccIn)

9
apps/emqx_authz/src/emqx_authz_redis.erl
View File

@ -71,8 +71,9 @@ replvar(Cmd, Client = #{username := Username}) ->
replvar(Cmd, _) ->
Cmd.
repl(S, _Var, undefined) ->
repl(S, _VarPH, undefined) ->
S;
repl(S, Var, Val) ->
NVal = re:replace(Val, "&", "\\\\&", [global, {return, list}]),
re:replace(S, Var, NVal, [{return, list}]).
repl(S, VarPH, Val) ->
NVal = re:replace(Val, "&", "\\\\&", [global, {return, list}]),
NVarPH = emqx_authz:ph_to_re(VarPH),
re:replace(S, NVarPH, NVal, [{return, list}]).

181
apps/emqx_authz/src/emqx_authz_schema.erl
View File

@ -32,10 +32,15 @@
-export([ namespace/0
, roots/0
, fields/1
, validations/0
]).
-import(emqx_schema, [mk_duration/2]).
%%--------------------------------------------------------------------
%% Hocon Schema
%%--------------------------------------------------------------------
namespace() -> authz.
%% @doc authorization schema is not exported
@ -98,92 +103,24 @@ and the new rules will override all rules from the old config file.
}}
];
fields(http_get) ->
[ {type, #{type => http}}
, {enable, #{type => boolean(),
default => true}}
, {url, #{type => url()}}
, {method, #{type => get, default => get }}
, {headers, #{type => map(),
default => #{ <<"accept">> => <<"application/json">>
, <<"cache-control">> => <<"no-cache">>
, <<"connection">> => <<"keep-alive">>
, <<"keep-alive">> => <<"timeout=5">>
},
converter => fun (Headers0) ->
Headers1 = maps:fold(fun(K0, V, AccIn) ->
K1 = iolist_to_binary(string:to_lower(to_list(K0))),
maps:put(K1, V, AccIn)
end, #{}, Headers0),
maps:merge(#{ <<"accept">> => <<"application/json">>
, <<"cache-control">> => <<"no-cache">>
, <<"connection">> => <<"keep-alive">>
, <<"keep-alive">> => <<"timeout=5">>
}, Headers1)
end
}
}
, {request_timeout, mk_duration("request timeout", #{default => "30s"})}
] ++ proplists:delete(base_url, emqx_connector_http:fields(config));
[ {method, #{type => get, default => post}}
, {headers, fun headers_no_content_type/1}
] ++ http_common_fields();
fields(http_post) ->
[ {type, #{type => http}}
, {enable, #{type => boolean(),
default => true}}
, {url, #{type => url()}}
, {method, #{type => post,
default => get}}
, {headers, #{type => map(),
default => #{ <<"accept">> => <<"application/json">>
, <<"cache-control">> => <<"no-cache">>
, <<"connection">> => <<"keep-alive">>
, <<"content-type">> => <<"application/json">>
, <<"keep-alive">> => <<"timeout=5">>
},
converter => fun (Headers0) ->
Headers1 = maps:fold(fun(K0, V, AccIn) ->
K1 = iolist_to_binary(string:to_lower(binary_to_list(K0))),
maps:put(K1, V, AccIn)
end, #{}, Headers0),
maps:merge(#{ <<"accept">> => <<"application/json">>
, <<"cache-control">> => <<"no-cache">>
, <<"connection">> => <<"keep-alive">>
, <<"content-type">> => <<"application/json">>
, <<"keep-alive">> => <<"timeout=5">>
}, Headers1)
end
}
}
, {request_timeout, mk_duration("request timeout", #{default => "30s"})}
, {body, #{type => map(),
nullable => true
}
}
] ++ proplists:delete(base_url, emqx_connector_http:fields(config));
[ {method, #{type => post, default => post}}
, {headers, fun headers/1}
] ++ http_common_fields();
fields(mnesia) ->
[ {type, #{type => 'built-in-database'}}
, {enable, #{type => boolean(),
default => true}}
];
fields(mongo_single) ->
[ {collection, #{type => atom()}}
, {selector, #{type => map()}}
, {type, #{type => mongodb}}
, {enable, #{type => boolean(),
default => true}}
] ++ emqx_connector_mongo:fields(single);
mongo_common_fields() ++ emqx_connector_mongo:fields(single);
fields(mongo_rs) ->
[ {collection, #{type => atom()}}
, {selector, #{type => map()}}
, {type, #{type => mongodb}}
, {enable, #{type => boolean(),
default => true}}
] ++ emqx_connector_mongo:fields(rs);
mongo_common_fields() ++ emqx_connector_mongo:fields(rs);
fields(mongo_sharded) ->
[ {collection, #{type => atom()}}
, {selector, #{type => map()}}
, {type, #{type => mongodb}}
, {enable, #{type => boolean(),
default => true}}
] ++ emqx_connector_mongo:fields(sharded);
mongo_common_fields() ++ emqx_connector_mongo:fields(sharded);
fields(mysql) ->
connector_fields(mysql) ++
[ {query, query()} ];
@ -203,10 +140,87 @@ fields(redis_cluster) ->
connector_fields(redis, cluster) ++
[ {cmd, query()} ].
http_common_fields() ->
[ {type, #{type => http}}
, {enable, #{type => boolean(), default => true}}
, {url, #{type => url()}}
, {request_timeout, mk_duration("request timeout", #{default => "30s"})}
, {body, #{type => map(), nullable => true}}
] ++ proplists:delete(base_url, emqx_connector_http:fields(config)).
mongo_common_fields() ->
[ {collection, #{type => atom()}}
, {selector, #{type => map()}}
, {type, #{type => mongodb}}
, {enable, #{type => boolean(),
default => true}}
].
validations() ->
[ {check_ssl_opts, fun check_ssl_opts/1}
, {check_headers, fun check_headers/1}
].
headers(type) -> map();
headers(converter) ->
fun(Headers) ->
maps:merge(default_headers(), transform_header_name(Headers))
end;
headers(default) -> default_headers();
headers(_) -> undefined.
headers_no_content_type(type) -> map();
headers_no_content_type(converter) ->
fun(Headers) ->
maps:merge(default_headers_no_content_type(), transform_header_name(Headers))
end;
headers_no_content_type(default) -> default_headers_no_content_type();
headers_no_content_type(_) -> undefined.
%%--------------------------------------------------------------------
%% Internal functions
%%--------------------------------------------------------------------
default_headers() ->
maps:put(<<"content-type">>,
<<"application/json">>,
default_headers_no_content_type()).
default_headers_no_content_type() ->
#{ <<"accept">> => <<"application/json">>
, <<"cache-control">> => <<"no-cache">>
, <<"connection">> => <<"keep-alive">>
, <<"keep-alive">> => <<"timeout=5">>
}.
transform_header_name(Headers) ->
maps:fold(fun(K0, V, Acc) ->
K = list_to_binary(string:to_lower(to_list(K0))),
maps:put(K, V, Acc)
end, #{}, Headers).
check_ssl_opts(Conf)
when Conf =:= #{} ->
true;
check_ssl_opts(Conf) ->
case emqx_authz_http:parse_url(hocon_schema:get_value("config.url", Conf)) of
#{scheme := https} ->
case hocon_schema:get_value("config.ssl.enable", Conf) of
true -> ok;
false -> false
end;
#{scheme := http} ->
ok
end.
check_headers(Conf)
when Conf =:= #{} ->
true;
check_headers(Conf) ->
Method = to_bin(hocon_schema:get_value("config.method", Conf)),
Headers = hocon_schema:get_value("config.headers", Conf),
Method =:= <<"post">> orelse (not maps:is_key(<<"content-type">>, Headers)).
union_array(Item) when is_list(Item) ->
hoconsc:array(hoconsc:union(Item)).
@ -229,15 +243,22 @@ connector_fields(DB, Fields) ->
catch
error:badarg ->
list_to_atom(Mod0);
Error ->
erlang:error(Error)
error:Reason ->
erlang:error(Reason)
end,
[ {type, #{type => DB}}
, {enable, #{type => boolean(),
default => true}}
] ++ Mod:fields(Fields).
] ++ erlang:apply(Mod, fields, [Fields]).
to_list(A) when is_atom(A) ->
atom_to_list(A);
to_list(B) when is_binary(B) ->
binary_to_list(B).
to_bin(A) when is_atom(A) ->
atom_to_binary(A);
to_bin(B) when is_binary(B) ->
B;
to_bin(L) when is_list(L) ->
list_to_binary(L).

3
apps/emqx_authz/test/emqx_authz_SUITE.erl
View File

@ -36,7 +36,8 @@ init_per_suite(Config) ->
meck:expect(emqx_resource, remove, fun(_) -> ok end ),
ok = emqx_common_test_helpers:start_apps(
[emqx_conf, emqx_authz], fun set_special_configs/1),
[emqx_connector, emqx_conf, emqx_authz],
fun set_special_configs/1),
Config.
end_per_suite(_Config) ->

2
apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl
View File

@ -31,7 +31,7 @@ groups() ->
init_per_suite(Config) ->
ok = emqx_common_test_helpers:start_apps(
[emqx_conf, emqx_authz],
[emqx_connector, emqx_conf, emqx_authz],
fun set_special_configs/1
),
Config.

379
apps/emqx_bridge/src/emqx_bridge_api.erl
View File

@ -17,220 +17,266 @@
-behaviour(minirest_api).
-export([api_spec/0]).
-include_lib("typerefl/include/types.hrl").
-export([ list_create_bridges_in_cluster/2
, list_local_bridges/1
, crud_bridges_in_cluster/2
, manage_bridges/2
-import(hoconsc, [mk/2, array/1, enum/1]).
%% Swagger specs from hocon schema
-export([api_spec/0, paths/0, schema/1, namespace/0]).
%% API callbacks
-export(['/bridges'/2, '/bridges/:id'/2,
'/nodes/:node/bridges/:id/operation/:operation'/2]).
-export([ list_local_bridges/1
, lookup_from_local_node/2
]).
-define(TYPES, [mqtt, http]).
-define(CONN_TYPES, [mqtt]).
-define(TRY_PARSE_ID(ID, EXPR),
try emqx_bridge:parse_bridge_id(Id) of
{BridgeType, BridgeName} -> EXPR
catch
error:{invalid_bridge_id, Id0} ->
{400, #{code => 'INVALID_ID', message => <<"invalid_bridge_id: ", Id0/binary,
". Bridge Ids must be of format <bridge_type>:<name>">>}}
". Bridge Ids must be of format {type}:{name}">>}}
end).
-define(METRICS(MATCH, SUCC, FAILED, RATE, RATE_5, RATE_MAX),
#{ matched => MATCH,
success => SUCC,
failed => FAILED,
speed => RATE,
speed_last5m => RATE_5,
speed_max => RATE_MAX
rate => RATE,
rate_last5m => RATE_5,
rate_max => RATE_MAX
}).
-define(metrics(MATCH, SUCC, FAILED, RATE, RATE_5, RATE_MAX),
#{ matched := MATCH,
success := SUCC,
failed := FAILED,
speed := RATE,
speed_last5m := RATE_5,
speed_max := RATE_MAX
rate := RATE,
rate_last5m := RATE_5,
rate_max := RATE_MAX
}).
req_schema() ->
Schema = [
case maps:to_list(emqx:get_raw_config([bridges, T], #{})) of
%% the bridge is not configured, so we have no method to get the schema
[] -> #{};
[{_K, Conf} | _] ->
emqx_mgmt_api_configs:gen_schema(Conf)
end
|| T <- ?TYPES],
#{'oneOf' => Schema}.
node_schema() ->
#{type => string, example => "emqx@127.0.0.1"}.
status_schema() ->
#{type => string, enum => [connected, disconnected]}.
metrics_schema() ->
#{ type => object
, properties => #{
matched => #{type => integer, example => "0"},
success => #{type => integer, example => "0"},
failed => #{type => integer, example => "0"},
speed => #{type => number, format => float, example => "0.0"},
speed_last5m => #{type => number, format => float, example => "0.0"},
speed_max => #{type => number, format => float, example => "0.0"}
}
}.
per_node_schema(Key, Schema) ->
#{
type => array,
items => #{
type => object,
properties => #{
node => node_schema(),
Key => Schema
}
}
}.
resp_schema() ->
AddMetadata = fun(Prop) ->
Prop#{status => status_schema(),
node_status => per_node_schema(status, status_schema()),
metrics => metrics_schema(),
node_metrics => per_node_schema(metrics, metrics_schema()),
id => #{type => string, example => "http:my_http_bridge"},
bridge_type => #{type => string, enum => ?TYPES},
node => node_schema()
}
end,
more_props_resp_schema(AddMetadata).
more_props_resp_schema(AddMetadata) ->
#{'oneOf' := Schema} = req_schema(),
Schema1 = [S#{properties => AddMetadata(Prop)}
|| S = #{properties := Prop} <- Schema],
#{'oneOf' => Schema1}.
namespace() -> "bridge".
api_spec() ->
{bridge_apis(), []}.
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false}).
bridge_apis() ->
[list_all_bridges_api(), crud_bridges_apis(), operation_apis()].
paths() -> ["/bridges", "/bridges/:id", "/nodes/:node/bridges/:id/operation/:operation"].
list_all_bridges_api() ->
ReqSchema = more_props_resp_schema(fun(Prop) ->
Prop#{id => #{type => string, required => true}}
end),
RespSchema = resp_schema(),
Metadata = #{
error_schema(Code, Message) ->
[ {code, mk(string(), #{example => Code})}
, {message, mk(string(), #{example => Message})}
].
get_response_body_schema() ->
emqx_dashboard_swagger:schema_with_examples(emqx_bridge_schema:get_response(),
bridge_info_examples(get)).
param_path_node() ->
path_param(node, binary(), atom_to_binary(node(), utf8)).
param_path_operation() ->
path_param(operation, enum([start, stop, restart]), <<"start">>).
param_path_id() ->
path_param(id, binary(), <<"http:my_http_bridge">>).
path_param(Name, Type, Example) ->
{Name, mk(Type,
#{ in => path
, required => true
, example => Example
})}.
bridge_info_array_example(Method) ->
[Config || #{value := Config} <- maps:values(bridge_info_examples(Method))].
bridge_info_examples(Method) ->
maps:merge(conn_bridge_examples(Method), #{
<<"http_bridge">> => #{
summary => <<"HTTP Bridge">>,
value => info_example(http, awesome, Method)
}
}).
conn_bridge_examples(Method) ->
lists:foldl(fun(Type, Acc) ->
SType = atom_to_list(Type),
KeyIngress = bin(SType ++ "_ingress"),
KeyEgress = bin(SType ++ "_egress"),
maps:merge(Acc, #{
KeyIngress => #{
summary => bin(string:uppercase(SType) ++ " Ingress Bridge"),
value => info_example(Type, ingress, Method)
},
KeyEgress => #{
summary => bin(string:uppercase(SType) ++ " Egress Bridge"),
value => info_example(Type, egress, Method)
}
})
end, #{}, ?CONN_TYPES).
info_example(Type, Direction, Method) ->
maps:merge(info_example_basic(Type, Direction),
method_example(Type, Direction, Method)).
method_example(Type, Direction, get) ->
SType = atom_to_list(Type),
SDir = atom_to_list(Direction),
SName = "my_" ++ SDir ++ "_" ++ SType ++ "_bridge",
#{
id => bin(SType ++ ":" ++ SName),
type => bin(SType),
name => bin(SName)
};
method_example(Type, Direction, post) ->
SType = atom_to_list(Type),
SDir = atom_to_list(Direction),
SName = "my_" ++ SDir ++ "_" ++ SType ++ "_bridge",
#{
type => bin(SType),
name => bin(SName)
};
method_example(_Type, _Direction, put) ->
#{}.
info_example_basic(http, _) ->
#{
url => <<"http://localhost:9901/messages/${topic}">>,
request_timeout => <<"30s">>,
connect_timeout => <<"30s">>,
max_retries => 3,
retry_interval => <<"10s">>,
pool_type => <<"random">>,
pool_size => 4,
enable_pipelining => true,
ssl => #{enable => false},
from_local_topic => <<"emqx_http/#">>,
method => post,
body => <<"${payload}">>
};
info_example_basic(mqtt, ingress) ->
#{
connector => <<"mqtt:my_mqtt_connector">>,
direction => ingress,
from_remote_topic => <<"aws/#">>,
subscribe_qos => 1,
to_local_topic => <<"from_aws/${topic}">>,
payload => <<"${payload}">>,
qos => <<"${qos}">>,
retain => <<"${retain}">>
};
info_example_basic(mqtt, egress) ->
#{
connector => <<"mqtt:my_mqtt_connector">>,
direction => egress,
from_local_topic => <<"emqx/#">>,
to_remote_topic => <<"from_emqx/${topic}">>,
payload => <<"${payload}">>,
qos => 1,
retain => false
}.
schema("/bridges") ->
#{
operationId => '/bridges',
get => #{
tags => [<<"bridges">>],
summary => <<"List Bridges">>,
description => <<"List all created bridges">>,
responses => #{
<<"200">> => emqx_mgmt_util:array_schema(resp_schema(),
<<"A list of the bridges">>)
200 => emqx_dashboard_swagger:schema_with_example(
array(emqx_bridge_schema:get_response()),
bridge_info_array_example(get))
}
},
post => #{
tags => [<<"bridges">>],
summary => <<"Create Bridge">>,
description => <<"Create a new bridge">>,
'requestBody' => emqx_mgmt_util:schema(ReqSchema),
requestBody => emqx_dashboard_swagger:schema_with_examples(
emqx_bridge_schema:post_request(),
bridge_info_examples(post)),
responses => #{
<<"201">> => emqx_mgmt_util:schema(RespSchema, <<"Bridge created">>),
<<"400">> => emqx_mgmt_util:error_schema(<<"Create bridge failed">>,
['UPDATE_FAILED'])
201 => get_response_body_schema(),
400 => error_schema('BAD_ARG', "Create bridge failed")
}
}
},
{"/bridges/", Metadata, list_create_bridges_in_cluster}.
};
crud_bridges_apis() ->
ReqSchema = req_schema(),
RespSchema = resp_schema(),
Metadata = #{
schema("/bridges/:id") ->
#{
operationId => '/bridges/:id',
get => #{
tags => [<<"bridges">>],
summary => <<"Get Bridge">>,
description => <<"Get a bridge by Id">>,
parameters => [param_path_id()],
responses => #{
<<"200">> => emqx_mgmt_util:array_schema(RespSchema,
<<"The details of the bridge">>),
<<"404">> => emqx_mgmt_util:error_schema(<<"Bridge not found">>, ['NOT_FOUND'])
200 => get_response_body_schema(),
404 => error_schema('NOT_FOUND', "Bridge not found")
}
},
put => #{
tags => [<<"bridges">>],
summary => <<"Update Bridge">>,
description => <<"Update a bridge">>,
parameters => [param_path_id()],
'requestBody' => emqx_mgmt_util:schema(ReqSchema),
requestBody => emqx_dashboard_swagger:schema_with_examples(
emqx_bridge_schema:put_request(),
bridge_info_examples(put)),
responses => #{
<<"200">> => emqx_mgmt_util:array_schema(RespSchema, <<"Bridge updated">>),
<<"400">> => emqx_mgmt_util:error_schema(<<"Update bridge failed">>,
['UPDATE_FAILED'])
200 => get_response_body_schema(),
400 => error_schema('BAD_ARG', "Update bridge failed")
}
},
delete => #{
tags => [<<"bridges">>],
summary => <<"Delete Bridge">>,
description => <<"Delete a bridge">>,
parameters => [param_path_id()],
responses => #{
<<"204">> => emqx_mgmt_util:schema(<<"Bridge deleted">>),
<<"404">> => emqx_mgmt_util:error_schema(<<"Bridge not found">>, ['NOT_FOUND'])
204 => <<"Bridge deleted">>
}
}
},
{"/bridges/:id", Metadata, crud_bridges_in_cluster}.
};
operation_apis() ->
Metadata = #{
schema("/nodes/:node/bridges/:id/operation/:operation") ->
#{
operationId => '/nodes/:node/bridges/:id/operation/:operation',
post => #{
tags => [<<"bridges">>],
summary => <<"Start/Stop/Restart Bridge">>,
description => <<"Start/Stop/Restart bridges on a specific node">>,
parameters => [
param_path_node(),
param_path_id(),
param_path_operation()],
param_path_operation()
],
responses => #{
<<"500">> => emqx_mgmt_util:error_schema(<<"Operation Failed">>,
['INTERNAL_ERROR']),
<<"200">> => emqx_mgmt_util:schema(<<"Operation success">>)}}},
{"/nodes/:node/bridges/:id/operation/:operation", Metadata, manage_bridges}.
param_path_node() ->
#{
name => node,
in => path,
schema => #{type => string},
required => true,
example => node()
500 => error_schema('INTERNAL_ERROR', "Operation Failed"),
200 => <<"Operation success">>
}
}
}.
param_path_id() ->
#{
name => id,
in => path,
schema => #{type => string},
required => true
}.
param_path_operation()->
#{
name => operation,
in => path,
required => true,
schema => #{
type => string,
enum => [start, stop, restart]},
example => restart
}.
list_create_bridges_in_cluster(post, #{body := #{<<"id">> := Id} = Conf}) ->
?TRY_PARSE_ID(Id,
case emqx_bridge:lookup(BridgeType, BridgeName) of
{ok, _} -> {400, #{code => 'ALREADY_EXISTS', message => <<"bridge already exists">>}};
{error, not_found} ->
case ensure_bridge(BridgeType, BridgeName, maps:remove(<<"id">>, Conf)) of
ok -> lookup_from_all_nodes(Id, BridgeType, BridgeName, 201);
{error, Error} -> {400, Error}
end
end);
list_create_bridges_in_cluster(get, _Params) ->
'/bridges'(post, #{body := #{<<"type">> := BridgeType} = Conf}) ->
BridgeName = maps:get(<<"name">>, Conf, emqx_misc:gen_id()),
case emqx_bridge:lookup(BridgeType, BridgeName) of
{ok, _} -> {400, #{code => 'ALREADY_EXISTS', message => <<"bridge already exists">>}};
{error, not_found} ->
case ensure_bridge_created(BridgeType, BridgeName, Conf) of
ok -> lookup_from_all_nodes(BridgeType, BridgeName, 201);
{error, Error} -> {400, Error}
end
end;
'/bridges'(get, _Params) ->
{200, zip_bridges([list_local_bridges(Node) || Node <- mria_mnesia:running_nodes()])}.
list_local_bridges(Node) when Node =:= node() ->
@ -238,22 +284,22 @@ list_local_bridges(Node) when Node =:= node() ->
list_local_bridges(Node) ->
rpc_call(Node, list_local_bridges, [Node]).
crud_bridges_in_cluster(get, #{bindings := #{id := Id}}) ->
?TRY_PARSE_ID(Id, lookup_from_all_nodes(Id, BridgeType, BridgeName, 200));
'/bridges/:id'(get, #{bindings := #{id := Id}}) ->
?TRY_PARSE_ID(Id, lookup_from_all_nodes(BridgeType, BridgeName, 200));
crud_bridges_in_cluster(put, #{bindings := #{id := Id}, body := Conf}) ->
'/bridges/:id'(put, #{bindings := #{id := Id}, body := Conf}) ->
?TRY_PARSE_ID(Id,
case emqx_bridge:lookup(BridgeType, BridgeName) of
{ok, _} ->
case ensure_bridge(BridgeType, BridgeName, Conf) of
ok -> lookup_from_all_nodes(Id, BridgeType, BridgeName, 200);
case ensure_bridge_created(BridgeType, BridgeName, Conf) of
ok -> lookup_from_all_nodes(BridgeType, BridgeName, 200);
{error, Error} -> {400, Error}
end;
{error, not_found} ->
{404, #{code => 'NOT_FOUND', message => <<"bridge not found">>}}
end);
crud_bridges_in_cluster(delete, #{bindings := #{id := Id}}) ->
'/bridges/:id'(delete, #{bindings := #{id := Id}}) ->
?TRY_PARSE_ID(Id,
case emqx_conf:remove(emqx_bridge:config_key_path() ++ [BridgeType, BridgeName],
#{override_to => cluster}) of
@ -262,12 +308,12 @@ crud_bridges_in_cluster(delete, #{bindings := #{id := Id}}) ->
{500, #{code => 102, message => emqx_resource_api:stringify(Reason)}}
end).
lookup_from_all_nodes(Id, BridgeType, BridgeName, SuccCode) ->
lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) ->
case rpc_multicall(lookup_from_local_node, [BridgeType, BridgeName]) of
{ok, [{ok, _} | _] = Results} ->
{SuccCode, format_bridge_info([R || {ok, R} <- Results])};
{ok, [{error, not_found} | _]} ->
{404, error_msg('NOT_FOUND', <<"not_found: ", Id/binary>>)};
{404, error_msg('NOT_FOUND', <<"not_found">>)};
{error, ErrL} ->
{500, error_msg('UNKNOWN_ERROR', ErrL)}
end.
@ -278,7 +324,8 @@ lookup_from_local_node(BridgeType, BridgeName) ->
Error -> Error
end.
manage_bridges(post, #{bindings := #{node := Node, id := Id, operation := Op}}) ->
'/nodes/:node/bridges/:id/operation/:operation'(post, #{bindings :=
#{node := Node, id := Id, operation := Op}}) ->
OperFun =
fun (<<"start">>) -> start;
(<<"stop">>) -> stop;
@ -292,9 +339,10 @@ manage_bridges(post, #{bindings := #{node := Node, id := Id, operation := Op}})
{500, #{code => 102, message => emqx_resource_api:stringify(Reason)}}
end).
ensure_bridge(BridgeType, BridgeName, Conf) ->
case emqx_conf:update(emqx_bridge:config_key_path() ++ [BridgeType, BridgeName], Conf,
#{override_to => cluster}) of
ensure_bridge_created(BridgeType, BridgeName, Conf) ->
Conf1 = maps:without([<<"type">>, <<"name">>], Conf),
case emqx_conf:update(emqx_bridge:config_key_path() ++ [BridgeType, BridgeName],
Conf1, #{override_to => cluster}) of
{ok, _} -> ok;
{error, Reason} ->
{error, error_msg('BAD_ARG', Reason)}
@ -346,12 +394,14 @@ aggregate_metrics(AllMetrics) ->
end, InitMetrics, AllMetrics).
format_resp(#{id := Id, raw_config := RawConf,
resource_data := #{mod := Mod, status := Status, metrics := Metrics}}) ->
resource_data := #{status := Status, metrics := Metrics}}) ->
{Type, Name} = emqx_bridge:parse_bridge_id(Id),
IsConnected = fun(started) -> connected; (_) -> disconnected end,
RawConf#{
id => Id,
type => Type,
name => Name,
node => node(),
bridge_type => emqx_bridge:bridge_type(Mod),
status => IsConnected(Status),
metrics => Metrics
}.
@ -378,4 +428,7 @@ rpc_call(Node, Mod, Fun, Args) ->
error_msg(Code, Msg) when is_binary(Msg) ->
#{code => Code, message => Msg};
error_msg(Code, Msg) ->
#{code => Code, message => list_to_binary(io_lib:format("~p", [Msg]))}.
#{code => Code, message => bin(io_lib:format("~p", [Msg]))}.
bin(S) when is_list(S) ->
list_to_binary(S).

95
apps/emqx_bridge/src/emqx_bridge_http_schema.erl Normal file
View File

@ -0,0 +1,95 @@
-module(emqx_bridge_http_schema).
-include_lib("typerefl/include/types.hrl").
-import(hoconsc, [mk/2, enum/1]).
-export([roots/0, fields/1]).
%%======================================================================================
%% Hocon Schema Definitions
roots() -> [].
fields("bridge") ->
basic_config() ++
[ {url, mk(binary(),
#{ nullable => false
, desc =>"""
The URL of the HTTP Bridge.<br>
Template with variables is allowed in the path, but variables cannot be used in the scheme, host,
or port part.<br>
For example, <code> http://localhost:9901/${topic} </code> is allowed, but
<code> http://${host}:9901/message </code> or <code> http://localhost:${port}/message </code>
is not allowed.
"""
})}
, {from_local_topic, mk(binary(),
#{ desc =>"""
The MQTT topic filter to be forwarded to the HTTP server. All MQTT PUBLISH messages which topic
match the from_local_topic will be forwarded.<br>
NOTE: if this bridge is used as the output of a rule (emqx rule engine), and also from_local_topic is configured, then both the data got from the rule and the MQTT messages that matches
from_local_topic will be forwarded.
"""
})}
, {method, mk(method(),
#{ default => post
, desc =>"""
The method of the HTTP request. All the available methods are: post, put, get, delete.<br>
Template with variables is allowed.<br>
"""
})}
, {headers, mk(map(),
#{ default => #{
<<"accept">> => <<"application/json">>,
<<"cache-control">> => <<"no-cache">>,
<<"connection">> => <<"keep-alive">>,
<<"content-type">> => <<"application/json">>,
<<"keep-alive">> => <<"timeout=5">>}
, desc =>"""
The headers of the HTTP request.<br>
Template with variables is allowed.
"""
})
}
, {body, mk(binary(),
#{ default => <<"${payload}">>
, desc =>"""
The body of the HTTP request.<br>
Template with variables is allowed.
"""
})}
, {request_timeout, mk(emqx_schema:duration_ms(),
#{ default => <<"30s">>
, desc =>"""
How long will the HTTP request timeout.
"""
})}
];
fields("post") ->
[ type_field()
, name_field()
] ++ fields("bridge");
fields("put") ->
fields("bridge");
fields("get") ->
[ id_field()
] ++ fields("post").
basic_config() ->
proplists:delete(base_url, emqx_connector_http:fields(config)).
%%======================================================================================
id_field() ->
{id, mk(binary(), #{desc => "The Bridge Id", example => "http:my_http_bridge"})}.
type_field() ->
{type, mk(http, #{desc => "The Bridge Type"})}.
name_field() ->
{name, mk(binary(), #{desc => "The Bridge Name"})}.
method() ->
enum([post, put, get, delete]).

62
apps/emqx_bridge/src/emqx_bridge_mqtt_schema.erl Normal file
View File

@ -0,0 +1,62 @@
-module(emqx_bridge_mqtt_schema).
-include_lib("typerefl/include/types.hrl").
-import(hoconsc, [mk/2]).
-export([roots/0, fields/1]).
%%======================================================================================
%% Hocon Schema Definitions
roots() -> [].
fields("ingress") ->
[ direction(ingress, emqx_connector_mqtt_schema:ingress_desc())
, emqx_bridge_schema:connector_name()
] ++ proplists:delete(hookpoint, emqx_connector_mqtt_schema:fields("ingress"));
fields("egress") ->
[ direction(egress, emqx_connector_mqtt_schema:egress_desc())
, emqx_bridge_schema:connector_name()
] ++ emqx_connector_mqtt_schema:fields("egress");
fields("post_ingress") ->
[ type_field()
, name_field()
] ++ fields("ingress");
fields("post_egress") ->
[ type_field()
, name_field()
] ++ fields("egress");
fields("put_ingress") ->
fields("ingress");
fields("put_egress") ->
fields("egress");
fields("get_ingress") ->
[ id_field()
] ++ fields("post_ingress");
fields("get_egress") ->
[ id_field()
] ++ fields("post_egress").
%%======================================================================================
direction(Dir, Desc) ->
{direction, mk(Dir,
#{ nullable => false
, desc => "The direction of the bridge. Can be one of 'ingress' or 'egress'.<br>"
++ Desc
})}.
id_field() ->
{id, mk(binary(), #{desc => "The Bridge Id", example => "mqtt:my_mqtt_bridge"})}.
type_field() ->
{type, mk(mqtt, #{desc => "The Bridge Type"})}.
name_field() ->
{name, mk(binary(),
#{ desc => "The Bridge Name"
, example => "some_bridge_name"
})}.

143
apps/emqx_bridge/src/emqx_bridge_schema.erl
View File

@ -2,122 +2,63 @@
-include_lib("typerefl/include/types.hrl").
-import(hoconsc, [mk/2, ref/2]).
-export([roots/0, fields/1]).
-export([ get_response/0
, put_request/0
, post_request/0
]).
-export([ connector_name/0
]).
%%======================================================================================
%% Hocon Schema Definitions
roots() -> [bridges].
-define(CONN_TYPES, [mqtt]).
fields(bridges) ->
[ {mqtt,
sc(hoconsc:map(name, hoconsc:union([ ref("ingress_mqtt_bridge")
, ref("egress_mqtt_bridge")
])),
#{ desc => "MQTT bridges"
})}
, {http,
sc(hoconsc:map(name, ref("http_bridge")),
#{ desc => "HTTP bridges"
})}
];
fields("ingress_mqtt_bridge") ->
[ direction(ingress, emqx_connector_mqtt_schema:ingress_desc())
, connector_name()
] ++ proplists:delete(hookpoint, emqx_connector_mqtt_schema:fields("ingress"));
fields("egress_mqtt_bridge") ->
[ direction(egress, emqx_connector_mqtt_schema:egress_desc())
, connector_name()
] ++ emqx_connector_mqtt_schema:fields("egress");
fields("http_bridge") ->
basic_config_http() ++
[ {url,
sc(binary(),
#{ nullable => false
, desc =>"""
The URL of the HTTP Bridge.<br>
Template with variables is allowed in the path, but variables cannot be used in the scheme, host,
or port part.<br>
For example, <code> http://localhost:9901/${topic} </code> is allowed, but
<code> http://${host}:9901/message </code> or <code> http://localhost:${port}/message </code>
is not allowed.
"""
})}
, {from_local_topic,
sc(binary(),
#{ desc =>"""
The MQTT topic filter to be forwarded to the HTTP server. All MQTT PUBLISH messages which topic
match the from_local_topic will be forwarded.<br>
NOTE: if this bridge is used as the output of a rule (emqx rule engine), and also from_local_topic is configured, then both the data got from the rule and the MQTT messages that matches
from_local_topic will be forwarded.
"""
})}
, {method,
sc(method(),
#{ default => post
, desc =>"""
The method of the HTTP request. All the available methods are: post, put, get, delete.<br>
Template with variables is allowed.<br>
"""
})}
, {headers,
sc(map(),
#{ default => #{
<<"accept">> => <<"application/json">>,
<<"cache-control">> => <<"no-cache">>,
<<"connection">> => <<"keep-alive">>,
<<"content-type">> => <<"application/json">>,
<<"keep-alive">> => <<"timeout=5">>}
, desc =>"""
The headers of the HTTP request.<br>
Template with variables is allowed.
"""
})
}
, {body,
sc(binary(),
#{ default => <<"${payload}">>
, desc =>"""
The body of the HTTP request.<br>
Template with variables is allowed.
"""
})}
, {request_timeout,
sc(emqx_schema:duration_ms(),
#{ default => <<"30s">>
, desc =>"""
How long will the HTTP request timeout.
"""
})}
].
direction(Dir, Desc) ->
{direction,
sc(Dir,
#{ nullable => false
, desc => "The direction of the bridge. Can be one of 'ingress' or 'egress'.<br>" ++
Desc
})}.
%%======================================================================================
%% For HTTP APIs
get_response() ->
http_schema("get").
connector_name() ->
{connector,
sc(binary(),
mk(binary(),
#{ nullable => false
, desc =>"""
The connector name to be used for this bridge.
Connectors are configured by 'connectors.<type>.<name>
Connectors are configured as 'connectors.{type}.{name}',
for example 'connectors.http.mybridge'.
"""
})}.
basic_config_http() ->
proplists:delete(base_url, emqx_connector_http:fields(config)).
put_request() ->
http_schema("put").
method() ->
hoconsc:enum([post, put, get, delete]).
post_request() ->
http_schema("post").
sc(Type, Meta) -> hoconsc:mk(Type, Meta).
http_schema(Method) ->
Schemas = lists:flatmap(fun(Type) ->
[ref(schema_mod(Type), Method ++ "_ingress"),
ref(schema_mod(Type), Method ++ "_egress")]
end, ?CONN_TYPES),
hoconsc:union([ref(emqx_bridge_http_schema, Method)
| Schemas]).
ref(Field) -> hoconsc:ref(?MODULE, Field).
%%======================================================================================
%% For config files
roots() -> [bridges].
fields(bridges) ->
[{http, mk(hoconsc:map(name, ref(emqx_bridge_http_schema, "bridge")), #{})}]
++ [{T, mk(hoconsc:map(name, hoconsc:union([
ref(schema_mod(T), "ingress"),
ref(schema_mod(T), "egress")
])), #{})} || T <- ?CONN_TYPES].
schema_mod(Type) ->
list_to_atom(lists:concat(["emqx_bridge_", Type, "_schema"])).

80
apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
View File

@ -21,7 +21,9 @@
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-define(CONF_DEFAULT, <<"bridges: {}">>).
-define(TEST_ID, <<"http:test_bridge">>).
-define(BRIDGE_TYPE, <<"http">>).
-define(BRIDGE_NAME, <<"test_bridge">>).
-define(BRIDGE_ID, <<"http:test_bridge">>).
-define(URL(PORT, PATH), list_to_binary(
io_lib:format("http://localhost:~s/~s",
[integer_to_list(PORT), PATH]))).
@ -134,11 +136,15 @@ t_http_crud_apis(_) ->
%% POST /bridges/ will create a bridge
URL1 = ?URL(Port, "path1"),
{ok, 201, Bridge} = request(post, uri(["bridges"]),
?HTTP_BRIDGE(URL1)#{<<"id">> => ?TEST_ID}),
?HTTP_BRIDGE(URL1)#{
<<"type">> => ?BRIDGE_TYPE,
<<"name">> => ?BRIDGE_NAME
}),
%ct:pal("---bridge: ~p", [Bridge]),
?assertMatch(#{ <<"id">> := ?TEST_ID
, <<"bridge_type">> := <<"http">>
?assertMatch(#{ <<"id">> := ?BRIDGE_ID
, <<"type">> := ?BRIDGE_TYPE
, <<"name">> := ?BRIDGE_NAME
, <<"status">> := _
, <<"node_status">> := [_|_]
, <<"metrics">> := _
@ -148,7 +154,10 @@ t_http_crud_apis(_) ->
%% create a again returns an error
{ok, 400, RetMsg} = request(post, uri(["bridges"]),
?HTTP_BRIDGE(URL1)#{<<"id">> => ?TEST_ID}),
?HTTP_BRIDGE(URL1)#{
<<"type">> => ?BRIDGE_TYPE,
<<"name">> => ?BRIDGE_NAME
}),
?assertMatch(
#{ <<"code">> := _
, <<"message">> := <<"bridge already exists">>
@ -156,10 +165,11 @@ t_http_crud_apis(_) ->
%% update the request-path of the bridge
URL2 = ?URL(Port, "path2"),
{ok, 200, Bridge2} = request(put, uri(["bridges", ?TEST_ID]),
{ok, 200, Bridge2} = request(put, uri(["bridges", ?BRIDGE_ID]),
?HTTP_BRIDGE(URL2)),
?assertMatch(#{ <<"id">> := ?TEST_ID
, <<"bridge_type">> := <<"http">>
?assertMatch(#{ <<"id">> := ?BRIDGE_ID
, <<"type">> := ?BRIDGE_TYPE
, <<"name">> := ?BRIDGE_NAME
, <<"status">> := _
, <<"node_status">> := [_|_]
, <<"metrics">> := _
@ -169,8 +179,9 @@ t_http_crud_apis(_) ->
%% list all bridges again, assert Bridge2 is in it
{ok, 200, Bridge2Str} = request(get, uri(["bridges"]), []),
?assertMatch([#{ <<"id">> := ?TEST_ID
, <<"bridge_type">> := <<"http">>
?assertMatch([#{ <<"id">> := ?BRIDGE_ID
, <<"type">> := ?BRIDGE_TYPE
, <<"name">> := ?BRIDGE_NAME
, <<"status">> := _
, <<"node_status">> := [_|_]
, <<"metrics">> := _
@ -179,9 +190,10 @@ t_http_crud_apis(_) ->
}], jsx:decode(Bridge2Str)),
%% get the bridge by id
{ok, 200, Bridge3Str} = request(get, uri(["bridges", ?TEST_ID]), []),
?assertMatch(#{ <<"id">> := ?TEST_ID
, <<"bridge_type">> := <<"http">>
{ok, 200, Bridge3Str} = request(get, uri(["bridges", ?BRIDGE_ID]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID
, <<"type">> := ?BRIDGE_TYPE
, <<"name">> := ?BRIDGE_NAME
, <<"status">> := _
, <<"node_status">> := [_|_]
, <<"metrics">> := _
@ -190,11 +202,11 @@ t_http_crud_apis(_) ->
}, jsx:decode(Bridge3Str)),
%% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", ?TEST_ID]), []),
{ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% update a deleted bridge returns an error
{ok, 404, ErrMsg2} = request(put, uri(["bridges", ?TEST_ID]),
{ok, 404, ErrMsg2} = request(put, uri(["bridges", ?BRIDGE_ID]),
?HTTP_BRIDGE(URL2)),
?assertMatch(
#{ <<"code">> := _
@ -206,11 +218,15 @@ t_start_stop_bridges(_) ->
Port = start_http_server(fun handle_fun_200_ok/1),
URL1 = ?URL(Port, "abc"),
{ok, 201, Bridge} = request(post, uri(["bridges"]),
?HTTP_BRIDGE(URL1)#{<<"id">> => ?TEST_ID}),
?HTTP_BRIDGE(URL1)#{
<<"type">> => ?BRIDGE_TYPE,
<<"name">> => ?BRIDGE_NAME
}),
%ct:pal("the bridge ==== ~p", [Bridge]),
?assertMatch(
#{ <<"id">> := ?TEST_ID
, <<"bridge_type">> := <<"http">>
#{ <<"id">> := ?BRIDGE_ID
, <<"type">> := ?BRIDGE_TYPE
, <<"name">> := ?BRIDGE_NAME
, <<"status">> := _
, <<"node_status">> := [_|_]
, <<"metrics">> := _
@ -219,42 +235,42 @@ t_start_stop_bridges(_) ->
}, jsx:decode(Bridge)),
%% stop it
{ok, 200, <<>>} = request(post,
uri(["nodes", node(), "bridges", ?TEST_ID, "operation", "stop"]),
uri(["nodes", node(), "bridges", ?BRIDGE_ID, "operation", "stop"]),
<<"">>),
{ok, 200, Bridge2} = request(get, uri(["bridges", ?TEST_ID]), []),
?assertMatch(#{ <<"id">> := ?TEST_ID
{ok, 200, Bridge2} = request(get, uri(["bridges", ?BRIDGE_ID]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID
, <<"status">> := <<"disconnected">>
}, jsx:decode(Bridge2)),
%% start again
{ok, 200, <<>>} = request(post,
uri(["nodes", node(), "bridges", ?TEST_ID, "operation", "start"]),
uri(["nodes", node(), "bridges", ?BRIDGE_ID, "operation", "start"]),
<<"">>),
{ok, 200, Bridge3} = request(get, uri(["bridges", ?TEST_ID]), []),
?assertMatch(#{ <<"id">> := ?TEST_ID
{ok, 200, Bridge3} = request(get, uri(["bridges", ?BRIDGE_ID]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID
, <<"status">> := <<"connected">>
}, jsx:decode(Bridge3)),
%% restart an already started bridge
{ok, 200, <<>>} = request(post,
uri(["nodes", node(), "bridges", ?TEST_ID, "operation", "restart"]),
uri(["nodes", node(), "bridges", ?BRIDGE_ID, "operation", "restart"]),
<<"">>),
{ok, 200, Bridge3} = request(get, uri(["bridges", ?TEST_ID]), []),
?assertMatch(#{ <<"id">> := ?TEST_ID
{ok, 200, Bridge3} = request(get, uri(["bridges", ?BRIDGE_ID]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID
, <<"status">> := <<"connected">>
}, jsx:decode(Bridge3)),
%% stop it again
{ok, 200, <<>>} = request(post,
uri(["nodes", node(), "bridges", ?TEST_ID, "operation", "stop"]),
uri(["nodes", node(), "bridges", ?BRIDGE_ID, "operation", "stop"]),
<<"">>),
%% restart a stopped bridge
{ok, 200, <<>>} = request(post,
uri(["nodes", node(), "bridges", ?TEST_ID, "operation", "restart"]),
uri(["nodes", node(), "bridges", ?BRIDGE_ID, "operation", "restart"]),
<<"">>),
{ok, 200, Bridge4} = request(get, uri(["bridges", ?TEST_ID]), []),
?assertMatch(#{ <<"id">> := ?TEST_ID
{ok, 200, Bridge4} = request(get, uri(["bridges", ?BRIDGE_ID]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID
, <<"status">> := <<"connected">>
}, jsx:decode(Bridge4)),
%% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", ?TEST_ID]), []),
{ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []).
%%--------------------------------------------------------------------

25
apps/emqx_conf/etc/emqx_conf.conf
View File

@ -27,12 +27,27 @@ node {
## Default: "{{ platform_data_dir }}/"
data_dir = "{{ platform_data_dir }}/"
## Dir of crash dump file.
## Location of crash dump file.
##
## @doc node.crash_dump_dir
## ValueType: Folder
## Default: "{{ platform_log_dir }}/"
crash_dump_dir = "{{ platform_log_dir }}/"
## @doc node.crash_dump_file
## ValueType: File
## Default: "{{ platform_log_dir }}/erl_crash.dump"
crash_dump_file = "{{ platform_log_dir }}/erl_crash.dump"
## The number of seconds that the broker is allowed to spend writing
## a crash dump
##
## @doc node.crash_dump_seconds
## ValueType: seconds
## Default: 30s
crash_dump_seconds = 30s
## The maximum size of a crash dump file in bytes.
##
## @doc node.crash_dump_bytes
## ValueType: bytes
## Default: 100MB
crash_dump_bytes = 100MB
## Global GC Interval.
##

192
apps/emqx_conf/etc/emqx_conf.md Normal file
View File

@ -0,0 +1,192 @@
EMQ X configuration file is in [HOCON](https://github.com/emqx/hocon) format.
HOCON, or Human-Optimized Config Object Notation is a format for human-readable data,
and a superset of JSON.
## Syntax
In config file the values can be notated as JSON like ojbects, such as
```
node {
name = "emqx@127.0.0.1"
cookie = "mysecret"
}
```
Another equivalent representation is flat, suh as
```
node.name="127.0.0.1"
node.cookie="mysecret"
```
This flat format is almost backward compatible with EMQ X's config file format
in 4.x series (the so called 'cuttlefish' format).
It is 'almost' compabile because the often HOCON requires strings to be quoted,
while cuttlefish treats all characters to the right of the `=` mark as the value.
e.g. cuttlefish: `node.name = emqx@127.0.0.1`, HOCON: `node.name = "emqx@127.0.0.1"`
Strings without special characters in them can be unquoted in HOCON too,
e.g. `foo`, `foo_bar`, `foo_bar_1`:
For more HOCON syntax, pelase refer to the [specification](https://github.com/lightbend/config/blob/main/HOCON.md)
## Schema
To make the HOCON objects type-safe, EMQ X introduded a schema for it.
The schema defines data types, and data fields' names and metadata for config value validation
and more. In fact, this config document itself is generated from schema metadata.
### Complex Data Types
There are 4 complex data types in EMQ X's HOCON config:
1. Struct: Named using an unquoted string, followed by a pre-defined list of fields,
fields can not start with a number, and are only allowed to use
lowercase letters and underscores as word separater.
1. Map: Map is like Struct, however the fields are not pre-defined.
1-based index number can also be used as map keys for an alternative
representation of an Array.
1. Union: `MemberType1 | MemberType2 | ...`
1. Array: `[ElementType]`
### Primitive Data Types
Complex types define data 'boxes' wich may contain other complex data
or primitive values.
There are quite some different primitive types, to name a fiew:
* `atom()`
* `boolean()`
* `string()`
* `integer()`
* `float()`
* `number()`
* `binary()` # another format of string()
* `emqx_schema:duration()` # time duration, another format of integer()
* ...
The primitive types are mostly self-describing, some are built-in, such
as `atom()`, some are defiend in EMQ X modules, such as `emqx_schema:duration()`.
### Config Paths
If we consider the whole EMQ X config as a tree,
to reference a primitive value, we can use a dot-separated names form string for
the path from the tree-root (always a Struct) down to the primitive values at tree-leaves.
Each segment of the dotted string is a Struct filed name or Map key.
For Array elements, 1-based index is used.
below are some examples
```
node.name="emqx.127.0.0.1"
zone.zone1.max_packet_size="10M"
authentication.1.enable=true
```
### Environment varialbes
Environment variables can be used to define or override config values.
Due to the fact that dots (`.`) are not allowed in environment variables, dots are
replaced with double-underscores (`__`).
And a the `EMQX_` prefix is used as the namespace.
For example `node.name` can be represented as `EMQX_NODE__NAME`
Environment varialbe values are parsed as hocon values, this allows users
to even set complex values from environment variables.
For example, this environment variable sets an array value.
```
export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS="[\"TLS_AES_256_GCM_SHA384\"]"
```
Unknown environment variables are logged as a `warning` level log, for example:
```
[warning] unknown_env_vars: ["EMQX_AUTHENTICATION__ENABLED"]
```
because the field name is `enable`, not `enabled`.
<strong>NOTE:</strong> Unknown root keys are however silently discarded.
### Config overlay
HOCON values are overlayed, earlier defined values are at layers closer to the bottom.
The overall order of the overlay rules from bottom up are:
1. `emqx.conf` the base config file
1. `EMQX_` prfixed environment variables
1. Cluster override file, the path of which is configured as `cluster_override_conf_file` in the lower layers
1. Local override file, the path of which is configured as `local_override_conf_file` in the lower layers
Below are the rules of config value overlay.
#### Struct Fileds
Later config values overwrites earlier values.
For example, in below config, the last line `debug` overwrites `errro` for
console log handler's `level` config, but leaving `enable` unchanged.
```
log {
console_handler{
enable=true,
level=error
}
}
## ... more configs ...
log.console_handler.level=debug
```
#### Map Values
Maps are like structs, only the files are user-defined rather than
the config schema. For instance, `zone1` in the exampele below.
```
zone {
zone1 {
mqtt.max_packet_size = 1M
}
}
## The maximum packet size can be defined as above,
## then overriden as below
zone.zone1.mqtt.max_packet_size = 10M
```
#### Array Elements
Arrays in EMQ X config have two different representations
* list, such as: `[1, 2, 3]`
* indexed-map, such as: `{"1"=1, "2"=2, "3"=3}`
Dot-separated paths with number in it are parsed to indexed-maps
e.g. `authentication.1={...}` is parsed as `authentication={"1": {...}}`
Indexed-map arrays can be used to override list arrays:
```
authentication=[{enable=true, backend="built-in-database", mechanism="password-based"}]
# we can disable this authentication provider with:
authentication.1.enable=false
```
However, list arrays do not get recursively merged into indexed-map arrays.
e.g.
```
authentication=[{enable=true, backend="built-in-database", mechanism="password-based"}]
## below value will replace the whole array, but not to override just one field.
authentication=[{enable=true}]
```

66
apps/emqx_conf/src/emqx_cluster_rpc.erl
View File

@ -18,8 +18,9 @@
%% API
-export([start_link/0, mnesia/1]).
-export([multicall/3, multicall/5, query/1, reset/0, status/0, skip_failed_commit/1]).
-export([get_node_tnx_id/1]).
-export([multicall/3, multicall/5, query/1, reset/0, status/0,
skip_failed_commit/1, fast_forward_to_commit/2]).
-export([get_node_tnx_id/1, latest_tnx_id/0]).
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
handle_continue/2, code_change/3]).
@ -60,21 +61,28 @@ start_link() ->
start_link(Node, Name, RetryMs) ->
gen_server:start_link({local, Name}, ?MODULE, [Node, RetryMs], []).
-spec multicall(Module, Function, Args) -> {ok, TnxId, term()} | {error, Reason} when
%% @doc return {ok, TnxId, MFARes} the first MFA result when all MFA run ok.
%% return {error, MFARes} when the first MFA result is no ok or {ok, term()}.
%% return {retry, TnxId, MFARes, Nodes} when some Nodes failed and some Node ok.
-spec multicall(Module, Function, Args) ->
{ok, TnxId, term()} | {error, Reason} | {retry, TnxId, MFARes, node()} when
Module :: module(),
Function :: atom(),
Args :: [term()],
MFARes :: term(),
TnxId :: pos_integer(),
Reason :: string().
multicall(M, F, A) ->
multicall(M, F, A, all, timer:minutes(2)).
-spec multicall(Module, Function, Args, SucceedNum, Timeout) -> {ok, TnxId, term()} |{error, Reason} when
-spec multicall(Module, Function, Args, SucceedNum, Timeout) ->
{ok, TnxId, MFARes} | {error, Reason} | {retry, TnxId, MFARes, node()} when
Module :: module(),
Function :: atom(),
Args :: [term()],
SucceedNum :: pos_integer() | all,
TnxId :: pos_integer(),
MFARes :: term(),
Timeout :: timeout(),
Reason :: string().
multicall(M, F, A, RequireNum, Timeout) when RequireNum =:= all orelse RequireNum >= 1 ->
@ -108,7 +116,10 @@ multicall(M, F, A, RequireNum, Timeout) when RequireNum =:= all orelse RequireNu
end,
case OkOrFailed of
ok -> InitRes;
_ -> OkOrFailed
{error, Error0} -> {error, Error0};
{retry, Node0} ->
{ok, TnxId0, MFARes} = InitRes,
{retry, TnxId0, MFARes, Node0}
end.
-spec query(pos_integer()) -> {'atomic', map()} | {'aborted', Reason :: term()}.
@ -122,6 +133,11 @@ reset() -> gen_server:call(?MODULE, reset).
status() ->
transaction(fun trans_status/0, []).
-spec latest_tnx_id() -> pos_integer().
latest_tnx_id() ->
{atomic, TnxId} = transaction(fun get_latest_id/0, []),
TnxId.
-spec get_node_tnx_id(node()) -> integer().
get_node_tnx_id(Node) ->
case mnesia:wread({?CLUSTER_COMMIT, Node}) of
@ -136,6 +152,13 @@ get_node_tnx_id(Node) ->
skip_failed_commit(Node) ->
gen_server:call({?MODULE, Node}, skip_failed_commit).
%% Regardless of what MFA is returned, consider it a success),
%% then skip the specified TnxId.
%% If CurrTnxId >= TnxId, nothing happened.
%% If CurrTnxId < TnxId, the CurrTnxId will skip to TnxId.
-spec fast_forward_to_commit(node(), pos_integer()) -> pos_integer().
fast_forward_to_commit(Node, ToTnxId) ->
gen_server:call({?MODULE, Node}, {fast_forward_to_commit, ToTnxId}).
%%%===================================================================
%%% gen_server callbacks
%%%===================================================================
@ -165,8 +188,13 @@ handle_call({initiate, MFA}, _From, State = #{node := Node}) ->
{aborted, Reason} ->
{reply, {error, Reason}, State, {continue, ?CATCH_UP}}
end;
handle_call(skip_failed_commit, _From, State) ->
{reply, ok, State, catch_up(State, true)};
handle_call(skip_failed_commit, _From, State = #{node := Node}) ->
Timeout = catch_up(State, true),
{atomic, LatestId} = transaction(fun get_node_tnx_id/1, [Node]),
{reply, LatestId, State, Timeout};
handle_call({fast_forward_to_commit, ToTnxId}, _From, State) ->
NodeId = do_fast_forward_to_commit(ToTnxId, State),
{reply, NodeId, State, catch_up(State)};
handle_call(_, _From, State) ->
{reply, ok, State, catch_up(State)}.
@ -245,7 +273,8 @@ do_catch_up(ToTnxId, Node) ->
{false, Error} -> mnesia:abort(Error)
end;
[#cluster_rpc_commit{tnx_id = LastAppliedId}] ->
Reason = lists:flatten(io_lib:format("~p catch up failed by LastAppliedId(~p) > ToTnxId(~p)",
Reason = lists:flatten(
io_lib:format("~p catch up failed by LastAppliedId(~p) > ToTnxId(~p)",
[Node, LastAppliedId, ToTnxId])),
?SLOG(error, #{
msg => "catch up failed!",
@ -258,6 +287,20 @@ do_catch_up(ToTnxId, Node) ->
commit(Node, TnxId) ->
ok = mnesia:write(?CLUSTER_COMMIT, #cluster_rpc_commit{node = Node, tnx_id = TnxId}, write).
do_fast_forward_to_commit(ToTnxId, State = #{node := Node}) ->
{atomic, NodeId} = transaction(fun get_node_tnx_id/1, [Node]),
case NodeId >= ToTnxId of
true -> NodeId;
false ->
{atomic, LatestId} = transaction(fun get_latest_id/0, []),
case LatestId =< NodeId of
true -> NodeId;
false ->
catch_up(State, true),
do_fast_forward_to_commit(ToTnxId, State)
end
end.
get_latest_id() ->
case mnesia:last(?CLUSTER_MFA) of
'$end_of_table' -> 0;
@ -269,7 +312,8 @@ init_mfa(Node, MFA) ->
LatestId = get_latest_id(),
ok = do_catch_up_in_one_trans(LatestId, Node),
TnxId = LatestId + 1,
MFARec = #cluster_rpc_mfa{tnx_id = TnxId, mfa = MFA, initiator = Node, created_at = erlang:localtime()},
MFARec = #cluster_rpc_mfa{tnx_id = TnxId, mfa = MFA,
initiator = Node, created_at = erlang:localtime()},
ok = mnesia:write(?CLUSTER_MFA, MFARec, write),
ok = commit(Node, TnxId),
case apply_mfa(TnxId, MFA) of
@ -344,7 +388,7 @@ wait_for_all_nodes_commit(TnxId, Delay, Remain) ->
ok = timer:sleep(Delay),
wait_for_all_nodes_commit(TnxId, Delay, Remain - Delay);
[] -> ok;
Nodes -> {error, Nodes}
Nodes -> {retry, Nodes}
end.
wait_for_nodes_commit(RequiredNum, TnxId, Delay, Remain) ->
@ -356,7 +400,7 @@ wait_for_nodes_commit(RequiredNum, TnxId, Delay, Remain) ->
false ->
case lagging_node(TnxId) of
[] -> ok; %% All commit but The succeedNum > length(nodes()).
Nodes -> {error, Nodes}
Nodes -> {retry, Nodes}
end
end.

25
apps/emqx_conf/src/emqx_conf.erl
View File

@ -16,6 +16,7 @@
-module(emqx_conf).
-compile({no_auto_import, [get/1, get/2]}).
-include_lib("emqx/include/logger.hrl").
-export([add_handler/2, remove_handler/1]).
-export([get/1, get/2, get_raw/2, get_all/1]).
@ -23,6 +24,7 @@
-export([update/3, update/4]).
-export([remove/2, remove/3]).
-export([reset/2, reset/3]).
-export([gen_doc/1]).
%% for rpc
-export([get_node_and_config/1]).
@ -122,14 +124,29 @@ reset(Node, KeyPath, Opts) when Node =:= node() ->
reset(Node, KeyPath, Opts) ->
rpc:call(Node, ?MODULE, reset, [KeyPath, Opts]).
-spec gen_doc(file:name_all()) -> ok.
gen_doc(File) ->
Version = emqx_release:version(),
Title = "# EMQ X " ++ Version ++ " Configuration",
BodyFile = filename:join([code:lib_dir(emqx_conf), "etc", "emqx_conf.md"]),
{ok, Body} = file:read_file(BodyFile),
Doc = hocon_schema_doc:gen(emqx_conf_schema, #{title => Title,
body => Body}),
file:write_file(File, Doc).
%%--------------------------------------------------------------------
%% Internal funcs
%% Internal functions
%%--------------------------------------------------------------------
multicall(M, F, Args) ->
case emqx_cluster_rpc:multicall(M, F, Args) of
{ok, _TnxId, Res} ->
{ok, _TnxId, Res} -> Res;
{retry, TnxId, Res, Nodes} ->
%% The init MFA return ok, but other nodes failed.
%% We return ok and alert an alarm.
?SLOG(error, #{msg => "failed to update config in cluster", nodes => Nodes,
tnx_id => TnxId, mfa => {M, F, Args}}),
Res;
{error, Reason} ->
{error, Reason}
{error, Error} -> %% all MFA return not ok or {ok, term()}.
Error
end.

92
apps/emqx_conf/src/emqx_conf_cli.erl Normal file
View File

@ -0,0 +1,92 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_conf_cli).
-export([ load/0
, admins/1
, unload/0
]).
-define(CMD, cluster_call).
load() ->
emqx_ctl:register_command(?CMD, {?MODULE, admins}, []).
unload() ->
emqx_ctl:unregister_command(?CMD).
admins(["status"]) -> status();
admins(["skip"]) ->
status(),
Nodes = mria_mnesia:running_nodes(),
lists:foreach(fun emqx_cluster_rpc:skip_failed_commit/1, Nodes),
status();
admins(["skip", Node0]) ->
status(),
Node = list_to_existing_atom(Node0),
emqx_cluster_rpc:skip_failed_commit(Node),
status();
admins(["tnxid", TnxId0]) ->
TnxId = list_to_integer(TnxId0),
emqx_ctl:print("~p~n", [emqx_cluster_rpc:query(TnxId)]);
admins(["fast_forward"]) ->
status(),
Nodes = mria_mnesia:running_nodes(),
TnxId = emqx_cluster_rpc:latest_tnx_id(),
lists:foreach(fun(N) -> emqx_cluster_rpc:fast_forward_to_commit(N, TnxId) end, Nodes),
status();
admins(["fast_forward", ToTnxId]) ->
status(),
Nodes = mria_mnesia:running_nodes(),
TnxId = list_to_integer(ToTnxId),
lists:foreach(fun(N) -> emqx_cluster_rpc:fast_forward_to_commit(N, TnxId) end, Nodes),
status();
admins(["fast_forward", Node0, ToTnxId]) ->
status(),
TnxId = list_to_integer(ToTnxId),
Node = list_to_existing_atom(Node0),
emqx_cluster_rpc:fast_forward_to_commit(Node, TnxId),
status();
admins(_) ->
emqx_ctl:usage(
[
{"cluster_call status", "status"},
{"cluster_call skip [node]", "increase one commit on specific node"},
{"cluster_call tnxid <TnxId>", "get detailed about TnxId"},
{"cluster_call fast_forward [node] [tnx_id]", "fast forwards to tnx_id" }
]).
status() ->
emqx_ctl:print("-----------------------------------------------\n"),
{atomic, Status} = emqx_cluster_rpc:status(),
lists:foreach(fun(S) ->
#{
node := Node,
tnx_id := TnxId,
mfa := {M, F, A},
created_at := CreatedAt
} = S,
emqx_ctl:print("~p:[~w] CreatedAt:~p ~p:~p/~w\n",
[Node, TnxId, CreatedAt, M, F, length(A)])
end, Status),
emqx_ctl:print("-----------------------------------------------\n").

81
apps/emqx_conf/src/emqx_conf_schema.erl
View File

@ -24,6 +24,7 @@
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("emqx/include/emqx_authentication.hrl").
-type log_level() :: debug | info | notice | warning | error | critical | alert | emergency | all.
-type file() :: string().
@ -62,8 +63,12 @@
namespace() -> undefined.
roots() ->
%% authorization configs are merged in THIS schema's "authorization" fields
lists:keydelete("authorization", 1, emqx_schema:roots(high)) ++
PtKey = ?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY,
case persistent_term:get(PtKey, undefined) of
undefined -> persistent_term:put(PtKey, emqx_authn_schema);
_ -> ok
end,
emqx_schema_high_prio_roots() ++
[ {"node",
sc(hoconsc:ref("node"),
#{ desc => "Node name, cookie, config & data directories "
@ -87,20 +92,6 @@ roots() ->
"should work, but in case you need to do performance "
"fine-turning or experiment a bit, this is where to look."
})}
, {"authorization",
sc(hoconsc:ref("authorization"),
#{ desc => """
Authorization a.k.a ACL.<br>
In EMQ X, MQTT client access control is extremly flexible.<br>
An out of the box set of authorization data sources are supported.
For example,<br>
'file' source is to support concise and yet generic ACL rules in a file;<br>
'built-in-database' source can be used to store per-client customisable rule sets,
natively in the EMQ X node;<br>
'http' source to make EMQ X call an external HTTP API to make the decision;<br>
'postgresql' etc. to look up clients or rules from external databases;<br>
"""
})}
, {"db",
sc(ref("db"),
#{ desc => "Settings of the embedded database."
@ -251,14 +242,12 @@ fields("node") ->
[ {"name",
sc(string(),
#{ default => "emqx@127.0.0.1"
, override_env => "EMQX_NODE_NAME"
})}
, {"cookie",
sc(string(),
#{ mapping => "vm_args.-setcookie",
default => "emqxsecretcookie",
sensitive => true,
override_env => "EMQX_NODE_COOKIE"
sensitive => true
})}
, {"data_dir",
sc(string(),
@ -275,9 +264,25 @@ fields("node") ->
#{ mapping => "emqx_machine.global_gc_interval"
, default => "15m"
})}
, {"crash_dump_dir",
, {"crash_dump_file",
sc(file(),
#{ mapping => "vm_args.-env ERL_CRASH_DUMP"
, desc => "Location of the crash dump file"
})}
, {"crash_dump_seconds",
sc(emqx_schema:duration_s(),
#{ mapping => "vm_args.-env ERL_CRASH_DUMP_SECONDS"
, default => "30s"
, desc => """
The number of seconds that the broker is allowed to spend writing
a crash dump
"""
})}
, {"crash_dump_bytes",
sc(emqx_schema:bytesize(),
#{ mapping => "vm_args.-env ERL_CRASH_DUMP_BYTES"
, default => "100MB"
, desc => "The maximum size of a crash dump file in bytes."
})}
, {"dist_net_ticktime",
sc(emqx_schema:duration(),
@ -347,6 +352,23 @@ to <code>rlog</code>.
List of core nodes that the replicant will connect to.<br/>
Note: this parameter only takes effect when the <code>backend</code> is set
to <code>rlog</code> and the <code>role</code> is set to <code>replicant</code>.
"""
})}
, {"rpc_module",
sc(hoconsc:enum([gen_rpc, rpc]),
#{ mapping => "mria.rlog_rpc_module"
, default => gen_rpc
, desc => """
Protocol used for pushing transaction logs to the replicant nodes.
"""
})}
, {"tlog_push_mode",
sc(hoconsc:enum([sync, async]),
#{ mapping => "mria.tlog_push_mode"
, default => async
, desc => """
In sync mode the core node waits for an ack from the replicant nodes before sending the next
transaction log entry.
"""
})}
];
@ -812,3 +834,22 @@ ensure_list(V) ->
roots(Module) ->
lists:map(fun({_BinName, Root}) -> Root end, hocon_schema:roots(Module)).
%% Like authentication schema, authorization schema is incomplete in emqx_schema
%% module, this function replaces the root filed "authorization" with a new schema
emqx_schema_high_prio_roots() ->
Roots = emqx_schema:roots(high),
Authz = {"authorization",
sc(hoconsc:ref("authorization"),
#{ desc => """
Authorization a.k.a ACL.<br>
In EMQ X, MQTT client access control is extremly flexible.<br>
An out of the box set of authorization data sources are supported.
For example,<br>
'file' source is to support concise and yet generic ACL rules in a file;<br>
'built-in-database' source can be used to store per-client customisable rule sets,
natively in the EMQ X node;<br>
'http' source to make EMQ X call an external HTTP API to make the decision;<br>
'postgresql' etc. to look up clients or rules from external databases;<br>
""" })},
lists:keyreplace("authorization", 1, Roots, Authz).

31
apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl
View File

@ -33,7 +33,8 @@ all() -> [
t_commit_ok_but_apply_fail_on_other_node,
t_commit_ok_apply_fail_on_other_node_then_recover,
t_del_stale_mfa,
t_skip_failed_commit
t_skip_failed_commit,
t_fast_forward_commit
].
suite() -> [{timetrap, {minutes, 3}}].
groups() -> [].
@ -183,13 +184,37 @@ t_skip_failed_commit(_Config) ->
?assertEqual([{Node, 1}, {{Node, ?NODE2}, 1}, {{Node, ?NODE3}, 1}],
tnx_ids(List1)),
{M, F, A} = {?MODULE, failed_on_node, [erlang:whereis(?NODE1)]},
{ok, _, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000),
ok = gen_server:call(?NODE2, skip_failed_commit, 5000),
{ok, 2, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000),
2 = gen_server:call(?NODE2, skip_failed_commit, 5000),
{atomic, List2} = emqx_cluster_rpc:status(),
?assertEqual([{Node, 2}, {{Node, ?NODE2}, 2}, {{Node, ?NODE3}, 1}],
tnx_ids(List2)),
ok.
t_fast_forward_commit(_Config) ->
emqx_cluster_rpc:reset(),
{atomic, []} = emqx_cluster_rpc:status(),
{ok, 1, ok} = emqx_cluster_rpc:multicall(io, format, ["test~n"], all, 1000),
ct:sleep(180),
{atomic, List1} = emqx_cluster_rpc:status(),
Node = node(),
?assertEqual([{Node, 1}, {{Node, ?NODE2}, 1}, {{Node, ?NODE3}, 1}],
tnx_ids(List1)),
{M, F, A} = {?MODULE, failed_on_node, [erlang:whereis(?NODE1)]},
{ok, 2, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000),
{ok, 3, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000),
{ok, 4, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000),
{ok, 5, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000),
{retry, 6, ok, _} = emqx_cluster_rpc:multicall(M, F, A, 2, 1000),
3 = gen_server:call(?NODE2, {fast_forward_to_commit, 3}, 5000),
4 = gen_server:call(?NODE2, {fast_forward_to_commit, 4}, 5000),
6 = gen_server:call(?NODE2, {fast_forward_to_commit, 7}, 5000),
2 = gen_server:call(?NODE3, {fast_forward_to_commit, 2}, 5000),
{atomic, List2} = emqx_cluster_rpc:status(),
?assertEqual([{Node, 6}, {{Node, ?NODE2}, 6}, {{Node, ?NODE3}, 2}],
tnx_ids(List2)),
ok.
tnx_ids(Status) ->
lists:sort(lists:map(fun(#{tnx_id := TnxId, node := Node}) ->
{Node, TnxId} end, Status)).

2
apps/emqx_connector/etc/emqx_connector.conf
View File

@ -1,4 +1,5 @@
#connectors.mqtt.my_mqtt_connector {
# mode = cluster_shareload
# server = "127.0.0.1:1883"
# proto_ver = "v4"
# username = "username1"
@ -8,7 +9,6 @@
# retry_interval = "30s"
# max_inflight = 32
# reconnect_interval = "30s"
# bridge_mode = true
# replayq {
# dir = "{{ platform_data_dir }}/replayq/bridge_mqtt/"
# seg_bytes = "100MB"

130
apps/emqx_connector/src/emqx_connector_api.erl
View File

@ -30,6 +30,8 @@
%% API callbacks
-export(['/connectors_test'/2, '/connectors'/2, '/connectors/:id'/2]).
-define(CONN_TYPES, [mqtt]).
-define(TRY_PARSE_ID(ID, EXPR),
try emqx_connector:parse_connector_id(Id) of
{ConnType, ConnName} ->
@ -38,7 +40,7 @@
catch
error:{invalid_bridge_id, Id0} ->
{400, #{code => 'INVALID_ID', message => <<"invalid_bridge_id: ", Id0/binary,
". Bridge Ids must be of format <bridge_type>:<name>">>}}
". Bridge Ids must be of format {type}:{name}">>}}
end).
namespace() -> "connector".
@ -53,17 +55,71 @@ error_schema(Code, Message) ->
, {message, mk(string(), #{example => Message})}
].
connector_info() ->
hoconsc:union([ ref(emqx_connector_schema, "mqtt_connector_info")
]).
put_request_body_schema() ->
emqx_dashboard_swagger:schema_with_examples(
emqx_connector_schema:put_request(), connector_info_examples(put)).
connector_test_info() ->
hoconsc:union([ ref(emqx_connector_schema, "mqtt_connector_test_info")
]).
post_request_body_schema() ->
emqx_dashboard_swagger:schema_with_examples(
emqx_connector_schema:post_request(), connector_info_examples(post)).
connector_req() ->
hoconsc:union([ ref(emqx_connector_schema, "mqtt_connector")
]).
get_response_body_schema() ->
emqx_dashboard_swagger:schema_with_examples(
emqx_connector_schema:get_response(), connector_info_examples(get)).
connector_info_array_example(Method) ->
[Config || #{value := Config} <- maps:values(connector_info_examples(Method))].
connector_info_examples(Method) ->
lists:foldl(fun(Type, Acc) ->
SType = atom_to_list(Type),
maps:merge(Acc, #{
Type => #{
summary => bin(string:uppercase(SType) ++ " Connector"),
value => info_example(Type, Method)
}
})
end, #{}, ?CONN_TYPES).
info_example(Type, Method) ->
maps:merge(info_example_basic(Type),
method_example(Type, Method)).
method_example(Type, get) ->
SType = atom_to_list(Type),
SName = "my_" ++ SType ++ "_connector",
#{
id => bin(SType ++ ":" ++ SName),
type => bin(SType),
name => bin(SName)
};
method_example(Type, post) ->
SType = atom_to_list(Type),
SName = "my_" ++ SType ++ "_connector",
#{
type => bin(SType),
name => bin(SName)
};
method_example(_Type, put) ->
#{}.
info_example_basic(mqtt) ->
#{
mode => cluster_shareload,
server => <<"127.0.0.1:1883">>,
reconnect_interval => <<"30s">>,
proto_ver => <<"v4">>,
username => <<"foo">>,
password => <<"bar">>,
clientid => <<"foo">>,
clean_start => true,
keepalive => <<"300s">>,
retry_interval => <<"30s">>,
max_inflight => 100,
ssl => #{
enable => false
}
}.
param_path_id() ->
[{id, mk(binary(), #{in => path, example => <<"mqtt:my_mqtt_connector">>})}].
@ -74,9 +130,9 @@ schema("/connectors_test") ->
post => #{
tags => [<<"connectors">>],
description => <<"Test creating a new connector by given Id <br>"
"The Id must be of format <type>:<name>">>,
"The ID must be of format '{type}:{name}'">>,
summary => <<"Test creating connector">>,
requestBody => connector_test_info(),
requestBody => post_request_body_schema(),
responses => #{
200 => <<"Test connector OK">>,
400 => error_schema('TEST_FAILED', "connector test failed")
@ -92,17 +148,19 @@ schema("/connectors") ->
description => <<"List all connectors">>,
summary => <<"List connectors">>,
responses => #{
200 => mk(array(connector_info()), #{desc => "List of connectors"})
200 => emqx_dashboard_swagger:schema_with_example(
array(emqx_connector_schema:get_response()),
connector_info_array_example(get))
}
},
post => #{
tags => [<<"connectors">>],
description => <<"Create a new connector by given Id <br>"
"The Id must be of format <type>:<name>">>,
"The ID must be of format '{type}:{name}'">>,
summary => <<"Create connector">>,
requestBody => connector_info(),
requestBody => post_request_body_schema(),
responses => #{
201 => connector_info(),
201 => get_response_body_schema(),
400 => error_schema('ALREADY_EXISTS', "connector already exists")
}
}
@ -117,7 +175,7 @@ schema("/connectors/:id") ->
summary => <<"Get connector">>,
parameters => param_path_id(),
responses => #{
200 => connector_info(),
200 => get_response_body_schema(),
404 => error_schema('NOT_FOUND', "Connector not found")
}
},
@ -126,9 +184,9 @@ schema("/connectors/:id") ->
description => <<"Update an existing connector by Id">>,
summary => <<"Update connector">>,
parameters => param_path_id(),
requestBody => connector_req(),
requestBody => put_request_body_schema(),
responses => #{
200 => <<"Update connector successfully">>,
200 => get_response_body_schema(),
400 => error_schema('UPDATE_FAIL', "Update failed"),
404 => error_schema('NOT_FOUND', "Connector not found")
}},
@ -143,8 +201,8 @@ schema("/connectors/:id") ->
}}
}.
'/connectors_test'(post, #{body := #{<<"bridge_type">> := ConnType} = Params}) ->
case emqx_connector:create_dry_run(ConnType, maps:remove(<<"bridge_type">>, Params)) of
'/connectors_test'(post, #{body := #{<<"type">> := ConnType} = Params}) ->
case emqx_connector:create_dry_run(ConnType, maps:remove(<<"type">>, Params)) of
ok -> {200};
{error, Error} ->
{400, error_msg('BAD_ARG', Error)}
@ -153,17 +211,20 @@ schema("/connectors/:id") ->
'/connectors'(get, _Request) ->
{200, emqx_connector:list()};
'/connectors'(post, #{body := #{<<"id">> := Id} = Params}) ->
?TRY_PARSE_ID(Id,
case emqx_connector:lookup(ConnType, ConnName) of
{ok, _} ->
{400, error_msg('ALREADY_EXISTS', <<"connector already exists">>)};
{error, not_found} ->
case emqx_connector:update(ConnType, ConnName, maps:remove(<<"id">>, Params)) of
{ok, #{raw_config := RawConf}} -> {201, RawConf#{<<"id">> => Id}};
{error, Error} -> {400, error_msg('BAD_ARG', Error)}
end
end).
'/connectors'(post, #{body := #{<<"type">> := ConnType} = Params}) ->
ConnName = maps:get(<<"name">>, Params, emqx_misc:gen_id()),
case emqx_connector:lookup(ConnType, ConnName) of
{ok, _} ->
{400, error_msg('ALREADY_EXISTS', <<"connector already exists">>)};
{error, not_found} ->
case emqx_connector:update(ConnType, ConnName,
maps:without([<<"type">>, <<"name">>], Params)) of
{ok, #{raw_config := RawConf}} ->
{201, RawConf#{<<"id">> =>
emqx_connector:connector_id(ConnType, ConnName)}};
{error, Error} -> {400, error_msg('BAD_ARG', Error)}
end
end.
'/connectors/:id'(get, #{bindings := #{id := Id}}) ->
?TRY_PARSE_ID(Id,
@ -200,4 +261,7 @@ schema("/connectors/:id") ->
error_msg(Code, Msg) when is_binary(Msg) ->
#{code => Code, message => Msg};
error_msg(Code, Msg) ->
#{code => Code, message => list_to_binary(io_lib:format("~p", [Msg]))}.
#{code => Code, message => bin(io_lib:format("~p", [Msg]))}.
bin(S) when is_list(S) ->
list_to_binary(S).

5
apps/emqx_connector/src/emqx_connector_mongo.erl
View File

@ -118,7 +118,7 @@ on_start(InstId, Config = #{mongo_type := Type,
false -> [{ssl, false}]
end,
Topology = maps:get(topology, NConfig, #{}),
Opts = [{type, init_type(NConfig)},
Opts = [{mongo_type, init_type(NConfig)},
{hosts, Hosts},
{pool_size, PoolSize},
{options, init_topology_options(maps:to_list(Topology), [])},
@ -187,6 +187,7 @@ connect(Opts) ->
WorkerOptions = proplists:get_value(worker_options, Opts, []),
mongo_api:connect(Type, Hosts, Options, WorkerOptions).
mongo_query(Conn, find, Collection, Selector, Projector) ->
mongo_api:find(Conn, Collection, Selector, Projector);
@ -268,7 +269,7 @@ srv_record(_) -> undefined.
parse_servers(Type, Servers) when is_binary(Servers) ->
parse_servers(Type, binary_to_list(Servers));
parse_servers(Type, Servers) when is_list(Servers) ->
case string:split(Servers, ",", trailing) of
case string:split(Servers, ",", all) of
[Host | _] when Type =:= single ->
[Host];
Hosts ->

29
apps/emqx_connector/src/emqx_connector_mqtt.erl
View File

@ -40,6 +40,8 @@
-behaviour(hocon_schema).
-import(hoconsc, [mk/2]).
-export([ roots/0
, fields/1]).
@ -49,7 +51,25 @@ roots() ->
fields("config").
fields("config") ->
emqx_connector_mqtt_schema:fields("config").
emqx_connector_mqtt_schema:fields("config");
fields("get") ->
[{id, mk(binary(),
#{ desc => "The connector Id"
, example => <<"mqtt:my_mqtt_connector">>
})}]
++ fields("post");
fields("put") ->
emqx_connector_mqtt_schema:fields("connector");
fields("post") ->
[ {type, mk(mqtt, #{desc => "The Connector Type"})}
, {name, mk(binary(),
#{ desc => "The Connector Name"
, example => <<"my_mqtt_connector">>
})}
] ++ fields("put").
%% ===================================================================
%% supervisor APIs
@ -100,7 +120,7 @@ on_start(InstId, Conf) ->
BasicConf = basic_config(Conf),
BridgeConf = BasicConf#{
name => InstanceId,
clientid => clientid(InstanceId),
clientid => clientid(maps:get(clientid, Conf, InstId)),
subscriptions => make_sub_confs(maps:get(ingress, Conf, undefined)),
forwards => make_forward_confs(maps:get(egress, Conf, undefined))
},
@ -162,7 +182,6 @@ basic_config(#{
server := Server,
reconnect_interval := ReconnIntv,
proto_ver := ProtoVer,
bridge_mode := BridgeMod,
username := User,
password := Password,
clean_start := CleanStart,
@ -177,7 +196,7 @@ basic_config(#{
server => Server,
reconnect_interval => ReconnIntv,
proto_ver => ProtoVer,
bridge_mode => BridgeMod,
bridge_mode => true,
username => User,
password => Password,
clean_start => CleanStart,
@ -190,4 +209,4 @@ basic_config(#{
}.
clientid(Id) ->
list_to_binary(lists:concat([Id, ":", node()])).
iolist_to_binary([Id, ":", atom_to_list(node())]).

48
apps/emqx_connector/src/emqx_connector_schema.erl
View File

@ -4,33 +4,47 @@
-include_lib("typerefl/include/types.hrl").
-import(hoconsc, [mk/2, ref/2]).
-export([roots/0, fields/1]).
-export([ get_response/0
, put_request/0
, post_request/0
]).
-define(CONN_TYPES, [mqtt]).
%%======================================================================================
%% For HTTP APIs
get_response() ->
http_schema("get").
put_request() ->
http_schema("put").
post_request() ->
http_schema("post").
http_schema(Method) ->
Schemas = [ref(schema_mod(Type), Method) || Type <- ?CONN_TYPES],
hoconsc:union(Schemas).
%%======================================================================================
%% Hocon Schema Definitions
roots() -> ["connectors"].
fields(connectors) -> fields("connectors");
fields("connectors") ->
[ {mqtt,
sc(hoconsc:map(name,
hoconsc:union([ ref("mqtt_connector")
mk(hoconsc:map(name,
hoconsc:union([ ref(emqx_connector_mqtt_schema, "connector")
])),
#{ desc => "MQTT bridges"
})}
];
].
fields("mqtt_connector") ->
emqx_connector_mqtt_schema:fields("connector");
fields("mqtt_connector_info") ->
[{id, sc(binary(), #{desc => "The connector Id"})}]
++ fields("mqtt_connector");
fields("mqtt_connector_test_info") ->
[{bridge_type, sc(mqtt, #{desc => "The Bridge Type"})}]
++ fields("mqtt_connector").
sc(Type, Meta) -> hoconsc:mk(Type, Meta).
ref(Field) -> hoconsc:ref(?MODULE, Field).
schema_mod(Type) ->
list_to_atom(lists:concat(["emqx_connector_", Type])).

29
apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
View File

@ -8,7 +8,7 @@
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% cluster_shareload under the License is cluster_shareload on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
@ -38,7 +38,24 @@ fields("config") ->
topic_mappings();
fields("connector") ->
[ {server,
[ {mode,
sc(hoconsc:enum([cluster_singleton, cluster_shareload]),
#{ default => cluster_shareload
, desc => """
The mode of the MQTT Bridge. Can be one of 'cluster_singleton' or 'cluster_shareload'<br>
- cluster_singleton: create an unique MQTT connection within the emqx cluster.<br>
In 'cluster_singleton' node, all messages toward the remote broker go through the same
MQTT connection.<br>
- cluster_shareload: create an MQTT connection on each node in the emqx cluster.<br>
In 'cluster_shareload' mode, the incomming load from the remote broker is shared by
using shared subscription.<br>
Note that the 'clientid' is suffixed by the node name, this is to avoid
clientid conflicts between different nodes. And we can only use shared subscription
topic filters for 'from_remote_topic'.
"""
})}
, {server,
sc(emqx_schema:ip_port(),
#{ default => "127.0.0.1:1883"
, desc => "The host and port of the remote MQTT broker"
@ -49,11 +66,6 @@ fields("connector") ->
#{ default => v4
, desc => "The MQTT protocol version"
})}
, {bridge_mode,
sc(boolean(),
#{ default => true
, desc => "The bridge mode of the MQTT protocol"
})}
, {username,
sc(binary(),
#{ default => "emqx"
@ -66,8 +78,7 @@ fields("connector") ->
})}
, {clientid,
sc(binary(),
#{ default => "emqx_${nodename}"
, desc => "The clientid of the MQTT protocol"
#{ desc => "The clientid of the MQTT protocol"
})}
, {clean_start,
sc(boolean(),

62
apps/emqx_connector/test/emqx_connector_api_SUITE.erl
View File

@ -24,7 +24,11 @@
-define(CONF_DEFAULT, <<"connectors: {}">>).
-define(BRIDGE_CONF_DEFAULT, <<"bridges: {}">>).
-define(CONNECTR_TYPE, <<"mqtt">>).
-define(CONNECTR_NAME, <<"test_connector">>).
-define(CONNECTR_ID, <<"mqtt:test_connector">>).
-define(BRIDGE_NAME_INGRESS, <<"ingress_test_bridge">>).
-define(BRIDGE_NAME_EGRESS, <<"egress_test_bridge">>).
-define(BRIDGE_ID_INGRESS, <<"mqtt:ingress_test_bridge">>).
-define(BRIDGE_ID_EGRESS, <<"mqtt:egress_test_bridge">>).
-define(MQTT_CONNECOTR(Username),
@ -63,8 +67,8 @@
-define(metrics(MATCH, SUCC, FAILED, SPEED, SPEED5M, SPEEDMAX),
#{<<"matched">> := MATCH, <<"success">> := SUCC,
<<"failed">> := FAILED, <<"speed">> := SPEED,
<<"speed_last5m">> := SPEED5M, <<"speed_max">> := SPEEDMAX}).
<<"failed">> := FAILED, <<"rate">> := SPEED,
<<"rate_last5m">> := SPEED5M, <<"rate_max">> := SPEEDMAX}).
all() ->
emqx_common_test_helpers:all(?MODULE).
@ -115,7 +119,9 @@ t_mqtt_crud_apis(_) ->
%% POST /connectors/ will create a connector
User1 = <<"user1">>,
{ok, 201, Connector} = request(post, uri(["connectors"]),
?MQTT_CONNECOTR(User1)#{<<"id">> => ?CONNECTR_ID}),
?MQTT_CONNECOTR(User1)#{ <<"type">> => ?CONNECTR_TYPE
, <<"name">> => ?CONNECTR_NAME
}),
%ct:pal("---connector: ~p", [Connector]),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID
@ -128,7 +134,9 @@ t_mqtt_crud_apis(_) ->
%% create a again returns an error
{ok, 400, RetMsg} = request(post, uri(["connectors"]),
?MQTT_CONNECOTR(User1)#{<<"id">> => ?CONNECTR_ID}),
?MQTT_CONNECOTR(User1)#{ <<"type">> => ?CONNECTR_TYPE
, <<"name">> => ?CONNECTR_NAME
}),
?assertMatch(
#{ <<"code">> := _
, <<"message">> := <<"connector already exists">>
@ -187,7 +195,9 @@ t_mqtt_conn_bridge_ingress(_) ->
%% then we add a mqtt connector, using POST
User1 = <<"user1">>,
{ok, 201, Connector} = request(post, uri(["connectors"]),
?MQTT_CONNECOTR(User1)#{<<"id">> => ?CONNECTR_ID}),
?MQTT_CONNECOTR(User1)#{ <<"type">> => ?CONNECTR_TYPE
, <<"name">> => ?CONNECTR_NAME
}),
%ct:pal("---connector: ~p", [Connector]),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID
@ -201,11 +211,14 @@ t_mqtt_conn_bridge_ingress(_) ->
%% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now
{ok, 201, Bridge} = request(post, uri(["bridges"]),
?MQTT_BRIDGE_INGRESS(?CONNECTR_ID)#{<<"id">> => ?BRIDGE_ID_INGRESS}),
?MQTT_BRIDGE_INGRESS(?CONNECTR_ID)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_INGRESS
}),
%ct:pal("---bridge: ~p", [Bridge]),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID_INGRESS
, <<"bridge_type">> := <<"mqtt">>
, <<"type">> := <<"mqtt">>
, <<"status">> := <<"connected">>
, <<"connector">> := ?CONNECTR_ID
}, jsx:decode(Bridge)),
@ -250,7 +263,9 @@ t_mqtt_conn_bridge_egress(_) ->
%% then we add a mqtt connector, using POST
User1 = <<"user1">>,
{ok, 201, Connector} = request(post, uri(["connectors"]),
?MQTT_CONNECOTR(User1)#{<<"id">> => ?CONNECTR_ID}),
?MQTT_CONNECOTR(User1)#{ <<"type">> => ?CONNECTR_TYPE
, <<"name">> => ?CONNECTR_NAME
}),
%ct:pal("---connector: ~p", [Connector]),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID
@ -264,11 +279,15 @@ t_mqtt_conn_bridge_egress(_) ->
%% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now
{ok, 201, Bridge} = request(post, uri(["bridges"]),
?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{<<"id">> => ?BRIDGE_ID_EGRESS}),
?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS
}),
%ct:pal("---bridge: ~p", [Bridge]),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS
, <<"bridge_type">> := <<"mqtt">>
, <<"type">> := ?CONNECTR_TYPE
, <<"name">> := ?BRIDGE_NAME_EGRESS
, <<"status">> := <<"connected">>
, <<"connector">> := ?CONNECTR_ID
}, jsx:decode(Bridge)),
@ -322,7 +341,10 @@ t_mqtt_conn_update(_) ->
%% then we add a mqtt connector, using POST
{ok, 201, Connector} = request(post, uri(["connectors"]),
?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)#{<<"id">> => ?CONNECTR_ID}),
?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)
#{ <<"type">> => ?CONNECTR_TYPE
, <<"name">> => ?CONNECTR_NAME
}),
%ct:pal("---connector: ~p", [Connector]),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID
@ -332,9 +354,13 @@ t_mqtt_conn_update(_) ->
%% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now
{ok, 201, Bridge} = request(post, uri(["bridges"]),
?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{<<"id">> => ?BRIDGE_ID_EGRESS}),
?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS
}),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS
, <<"bridge_type">> := <<"mqtt">>
, <<"type">> := <<"mqtt">>
, <<"name">> := ?BRIDGE_NAME_EGRESS
, <<"status">> := <<"connected">>
, <<"connector">> := ?CONNECTR_ID
}, jsx:decode(Bridge)),
@ -358,9 +384,15 @@ t_mqtt_conn_testing(_) ->
%% APIs for testing the connectivity
%% then we add a mqtt connector, using POST
{ok, 200, <<>>} = request(post, uri(["connectors_test"]),
?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)#{<<"bridge_type">> => <<"mqtt">>}),
?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS
}),
{ok, 400, _} = request(post, uri(["connectors_test"]),
?MQTT_CONNECOTR2(<<"127.0.0.1:2883">>)#{<<"bridge_type">> => <<"mqtt">>}).
?MQTT_CONNECOTR2(<<"127.0.0.1:2883">>)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS
}).
%%--------------------------------------------------------------------
%% HTTP Request

21
apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
View File

@ -182,12 +182,12 @@ check_parameter([{Name, Type} | Spec], Bindings, QueryStr, Module, BindingsAcc,
Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]},
case hocon_schema:field_schema(Type, in) of
path ->
Option = #{atom_key => true, override_env => false},
Option = #{atom_key => true},
NewBindings = hocon_schema:check_plain(Schema, Bindings, Option),
NewBindingsAcc = maps:merge(BindingsAcc, NewBindings),
check_parameter(Spec, Bindings, QueryStr, Module, NewBindingsAcc, QueryStrAcc);
query ->
Option = #{override_env => false},
Option = #{},
NewQueryStr = hocon_schema:check_plain(Schema, QueryStr, Option),
NewQueryStrAcc = maps:merge(QueryStrAcc, NewQueryStr),
check_parameter(Spec, Bindings, QueryStr, Module,BindingsAcc, NewQueryStrAcc)
@ -201,7 +201,7 @@ check_request_body(#{body := Body}, Schema, Module, CheckFun, true) ->
_ -> Type0
end,
NewSchema = ?INIT_SCHEMA#{roots => [{root, Type}]},
Option = #{override_env => false, nullable => true},
Option = #{nullable => true},
#{<<"root">> := NewBody} = CheckFun(NewSchema, #{<<"root">> => Body}, Option),
NewBody;
%% TODO not support nest object check yet, please use ref!
@ -214,7 +214,7 @@ check_request_body(#{body := Body}, Schema, Module, CheckFun, true) ->
check_request_body(#{body := Body}, Spec, _Module, CheckFun, false) ->
lists:foldl(fun({Name, Type}, Acc) ->
Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]},
maps:merge(Acc, CheckFun(Schema, Body, #{override_env => false}))
maps:merge(Acc, CheckFun(Schema, Body, #{}))
end, #{}, Spec).
%% tags, description, summary, security, deprecated
@ -337,19 +337,28 @@ components(Refs) ->
components([], SpecAcc, []) -> SpecAcc;
components([], SpecAcc, SubRefAcc) -> components(SubRefAcc, SpecAcc, []);
components([{Module, Field} | Refs], SpecAcc, SubRefsAcc) ->
Props = apply(Module, fields, [Field]),
Props = hocon_schema_fields(Module, Field),
Namespace = namespace(Module),
{Object, SubRefs} = parse_object(Props, Module),
NewSpecAcc = SpecAcc#{?TO_REF(Namespace, Field) => Object},
components(Refs, NewSpecAcc, SubRefs ++ SubRefsAcc);
%% parameters in ref only have one value, not array
components([{Module, Field, parameter} | Refs], SpecAcc, SubRefsAcc) ->
Props = apply(Module, fields, [Field]),
Props = hocon_schema_fields(Module, Field),
{[Param], SubRefs} = parameters(Props, Module),
Namespace = namespace(Module),
NewSpecAcc = SpecAcc#{?TO_REF(Namespace, Field) => Param},
components(Refs, NewSpecAcc, SubRefs ++ SubRefsAcc).
hocon_schema_fields(Module, StructName) ->
case apply(Module, fields, [StructName]) of
#{fields := Fields, desc := _} ->
%% evil here, as it's match hocon_schema's internal representation
Fields; %% TODO: make use of desc ?
Other ->
Other
end.
%% Semantic error at components.schemas.xxx:xx:xx
%% Component names can only contain the characters A-Z a-z 0-9 - . _
%% So replace ':' by '-'.

35
apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl
View File

@ -7,7 +7,7 @@
-export([paths/0, api_spec/0, schema/1, fields/1]).
-export([t_object/1, t_nest_object/1, t_api_spec/1,
t_local_ref/1, t_remote_ref/1, t_bad_ref/1, t_none_ref/1, t_nest_ref/1,
t_ref_array_with_key/1, t_ref_array_without_key/1
t_ref_array_with_key/1, t_ref_array_without_key/1, t_sub_fields/1
]).
-export([
t_object_trans/1, t_object_notrans/1, t_nest_object_trans/1, t_local_ref_trans/1,
@ -154,6 +154,17 @@ t_none_ref(_Config) ->
emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path)),
ok.
t_sub_fields(_Config) ->
Spec = #{
post => #{parameters => [],
requestBody => #{<<"content">> => #{<<"application/json">> =>
#{<<"schema">> => #{<<"$ref">> =>
<<"#/components/schemas/emqx_swagger_requestBody_SUITE.sub_fields">>}}}},
responses => #{<<"200">> => #{description => <<"ok">>}}}},
Refs = [{?MODULE, sub_fields}],
validate("/fields/sub", Spec, Refs),
ok.
t_bad_ref(_Config) ->
Path = "/ref/bad",
Spec = #{
@ -483,7 +494,7 @@ trans_requestBody(Path, Body, Filter) ->
api_spec() -> emqx_dashboard_swagger:spec(?MODULE).
paths() ->
["/object", "/nest/object", "/ref/local", "/ref/nest/ref",
["/object", "/nest/object", "/ref/local", "/ref/nest/ref", "/fields/sub",
"/ref/array/with/key", "/ref/array/without/key"].
schema("/object") ->
@ -506,6 +517,8 @@ schema("/nest/object") ->
]);
schema("/ref/local") ->
to_schema(mk(hoconsc:ref(good_ref), #{}));
schema("/fields/sub") ->
to_schema(mk(hoconsc:ref(sub_fields), #{}));
schema("/ref/remote") ->
to_schema(mk(hoconsc:ref(emqx_swagger_remote_schema, "ref2"), #{}));
schema("/ref/bad") ->
@ -544,4 +557,20 @@ fields(bad_ref) -> %% don't support maps
#{
username => mk(string(), #{}),
is_admin => mk(boolean(), #{})
}.
};
fields(sub_fields) ->
#{fields => [
{enable, fun enable/1},
{init_file, fun init_file/1}
],
desc => <<"test sub fields">>}.
enable(type) -> boolean();
enable(desc) -> <<"Whether to enable tls psk support">>;
enable(default) -> false;
enable(_) -> undefined.
init_file(type) -> binary();
init_file(desc) -> <<"test test desc">>;
init_file(nullable) -> true;
init_file(_) -> undefined.

34
apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl
View File

@ -14,7 +14,7 @@
-export([paths/0, api_spec/0, schema/1, fields/1]).
-export([t_simple_binary/1, t_object/1, t_nest_object/1, t_empty/1, t_error/1,
t_raw_local_ref/1, t_raw_remote_ref/1, t_hocon_schema_function/1, t_complicated_type/1,
t_local_ref/1, t_remote_ref/1, t_bad_ref/1, t_none_ref/1, t_nest_ref/1,
t_local_ref/1, t_remote_ref/1, t_bad_ref/1, t_none_ref/1, t_nest_ref/1, t_sub_fields/1,
t_ref_array_with_key/1, t_ref_array_without_key/1, t_api_spec/1]).
all() -> [{group, spec}].
@ -23,7 +23,7 @@ groups() -> [
{spec, [parallel], [
t_api_spec, t_simple_binary, t_object, t_nest_object, t_error, t_complicated_type,
t_raw_local_ref, t_raw_remote_ref, t_empty, t_hocon_schema_function,
t_local_ref, t_remote_ref, t_bad_ref, t_none_ref,
t_local_ref, t_remote_ref, t_bad_ref, t_none_ref, t_sub_fields,
t_ref_array_with_key, t_ref_array_without_key, t_nest_ref]}
].
@ -163,6 +163,14 @@ t_nest_ref(_Config) ->
validate(Path, Object, ExpectRefs),
ok.
t_sub_fields(_Config) ->
Path = "/fields/sub",
Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{
<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.sub_fields">>}}}},
ExpectRefs = [{?MODULE, sub_fields}],
validate(Path, Object, ExpectRefs),
ok.
t_complicated_type(_Config) ->
Path = "/ref/complicated_type",
Object = #{<<"content">> => #{<<"application/json">> =>
@ -366,7 +374,9 @@ schema("/ref/complicated_type") ->
{fix_integer, hoconsc:mk(typerefl:integer(100), #{})}
]
}}
}.
};
schema("/fields/sub") ->
to_schema(hoconsc:ref(sub_fields)).
validate(Path, ExpectObject, ExpectRefs) ->
{OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path),
@ -400,4 +410,20 @@ fields(bad_ref) -> %% don't support maps
#{
username => mk(string(), #{}),
is_admin => mk(boolean(), #{})
}.
};
fields(sub_fields) ->
#{fields => [
{enable, fun enable/1},
{init_file, fun init_file/1}
],
desc => <<"test sub fields">>}.
enable(type) -> boolean();
enable(desc) -> <<"Whether to enable tls psk support">>;
enable(default) -> false;
enable(_) -> undefined.
init_file(type) -> binary();
init_file(desc) -> <<"test test desc">>;
init_file(nullable) -> true;
init_file(_) -> undefined.

23
apps/emqx_gateway/src/coap/emqx_coap_channel.erl
View File

@ -18,9 +18,6 @@
-behaviour(emqx_gateway_channel).
-include_lib("emqx/include/logger.hrl").
-include("emqx_coap.hrl").
%% API
-export([ info/1
, info/2
@ -44,6 +41,12 @@
-export_type([channel/0]).
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl").
-include_lib("emqx/include/emqx_authentication.hrl").
-define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
-record(channel, {
%% Context
ctx :: emqx_gateway_ctx:context(),
@ -98,10 +101,10 @@ info(ctx, #channel{ctx = Ctx}) ->
stats(_) ->
[].
init(ConnInfo = #{peername := {PeerHost, _},
sockname := {_, SockPort}},
init(ConnInfoT = #{peername := {PeerHost, _},
sockname := {_, SockPort}},
#{ctx := Ctx} = Config) ->
Peercert = maps:get(peercert, ConnInfo, undefined),
Peercert = maps:get(peercert, ConnInfoT, undefined),
Mountpoint = maps:get(mountpoint, Config, <<>>),
ListenerId = case maps:get(listener, Config, undefined) of
undefined -> undefined;
@ -123,6 +126,10 @@ init(ConnInfo = #{peername := {PeerHost, _},
}
),
%% because it is possible to disconnect after init, and then trigger the $event.disconnected hook
%% and these two fields are required in the hook
ConnInfo = ConnInfoT#{proto_name => <<"CoAP">>, proto_ver => <<"1">>},
Heartbeat = ?GET_IDLE_TIME(Config),
#channel{ ctx = Ctx
, conninfo = ConnInfo
@ -279,7 +286,7 @@ try_takeover(idle, DesireId, Msg, Channel) ->
%% udp connection baseon the clientid
call_session(handle_request, Msg, Channel);
_ ->
case emqx_conf:get([gateway, coap, authentication], undefined) of
case emqx_conf:get([gateway, coap, ?AUTHN], undefined) of
undefined ->
call_session(handle_request, Msg, Channel);
_ ->
@ -349,8 +356,6 @@ ensure_connected(Channel = #channel{ctx = Ctx,
conninfo = ConnInfo,
clientinfo = ClientInfo}) ->
NConnInfo = ConnInfo#{ connected_at => erlang:system_time(millisecond)
, proto_name => <<"COAP">>
, proto_ver => <<"1">>
},
ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]),
_ = run_hooks(Ctx, 'client.connack', [NConnInfo, connection_accepted, []]),

5
apps/emqx_gateway/src/emqx_gateway_api.erl
View File

@ -17,6 +17,7 @@
-module(emqx_gateway_api).
-include_lib("emqx/include/emqx_placeholder.hrl").
-include_lib("emqx/include/emqx_authentication.hrl").
-behaviour(minirest_api).
@ -243,7 +244,7 @@ schema_gateway_overview_list() ->
%%
%% NOTE: It is a temporary measure to generate swagger-schema
-define(COAP_GATEWAY_CONFS,
#{<<"authentication">> =>
#{?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY =>
#{<<"mechanism">> => <<"password-based">>,
<<"name">> => <<"authenticator1">>,
<<"server_type">> => <<"built-in-database">>,
@ -331,7 +332,7 @@ schema_gateway_overview_list() ->
).
-define(STOMP_GATEWAY_CONFS,
#{<<"authentication">> =>
#{?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY =>
#{<<"mechanism">> => <<"password-based">>,
<<"name">> => <<"authenticator1">>,
<<"server_type">> => <<"built-in-database">>,

36
apps/emqx_gateway/src/emqx_gateway_conf.erl
View File

@ -17,8 +17,6 @@
%% @doc The gateway configuration management module
-module(emqx_gateway_conf).
-include_lib("emqx/include/logger.hrl").
%% Load/Unload
-export([ load/0
, unload/0
@ -56,6 +54,10 @@
, post_config_update/5
]).
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx/include/emqx_authentication.hrl").
-define(AUTHN_BIN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY).
-type atom_or_bin() :: atom() | binary().
-type ok_or_err() :: ok_or_err().
-type listener_ref() :: {ListenerType :: atom_or_bin(),
@ -106,8 +108,9 @@ maps_key_take([K | Ks], M, Acc) ->
-spec update_gateway(atom_or_bin(), map()) -> ok_or_err().
update_gateway(GwName, Conf0) ->
Conf = maps:without([listeners, authentication,
<<"listeners">>, <<"authentication">>], Conf0),
Exclude0 = [listeners, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM],
Exclude1 = [atom_to_binary(K, utf8) || K <- Exclude0],
Conf = maps:without(Exclude0 ++ Exclude1, Conf0),
update({?FUNCTION_NAME, bin(GwName), Conf}).
%% FIXME: delete cert files ??
@ -232,7 +235,7 @@ update(Req) ->
res(emqx_conf:update([gateway], Req, #{override_to => cluster})).
res({ok, _Result}) -> ok;
res({error, {error, {pre_config_update,emqx_gateway_conf,Reason}}}) -> {error, Reason};
res({error, {pre_config_update, emqx_gateway_conf, Reason}}) -> {error, Reason};
res({error, Reason}) -> {error, Reason}.
bin({LType, LName}) ->
@ -263,8 +266,7 @@ pre_config_update(_, {update_gateway, GwName, Conf}, RawConf) ->
undefined ->
{error, not_found};
_ ->
NConf = maps:without([<<"listeners">>,
<<"authentication">>], Conf),
NConf = maps:without([<<"listeners">>, ?AUTHN_BIN], Conf),
{ok, emqx_map_lib:deep_merge(RawConf, #{GwName => NConf})}
end;
pre_config_update(_, {unload_gateway, GwName}, RawConf) ->
@ -311,11 +313,11 @@ pre_config_update(_, {remove_listener, GwName, {LType, LName}}, RawConf) ->
pre_config_update(_, {add_authn, GwName, Conf}, RawConf) ->
case emqx_map_lib:deep_get(
[GwName, <<"authentication">>], RawConf, undefined) of
[GwName, ?AUTHN_BIN], RawConf, undefined) of
undefined ->
{ok, emqx_map_lib:deep_merge(
RawConf,
#{GwName => #{<<"authentication">> => Conf}})};
#{GwName => #{?AUTHN_BIN => Conf}})};
_ ->
{error, already_exist}
end;
@ -326,9 +328,9 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
undefined ->
{error, not_found};
Listener ->
case maps:get(<<"authentication">>, Listener, undefined) of
case maps:get(?AUTHN_BIN, Listener, undefined) of
undefined ->
NListener = maps:put(<<"authentication">>, Conf, Listener),
NListener = maps:put(?AUTHN_BIN, Conf, Listener),
NGateway = #{GwName =>
#{<<"listeners">> =>
#{LType => #{LName => NListener}}}},
@ -339,13 +341,13 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
end;
pre_config_update(_, {update_authn, GwName, Conf}, RawConf) ->
case emqx_map_lib:deep_get(
[GwName, <<"authentication">>], RawConf, undefined) of
[GwName, ?AUTHN_BIN], RawConf, undefined) of
undefined ->
{error, not_found};
_ ->
{ok, emqx_map_lib:deep_merge(
RawConf,
#{GwName => #{<<"authentication">> => Conf}})}
#{GwName => #{?AUTHN_BIN => Conf}})}
end;
pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
case emqx_map_lib:deep_get(
@ -354,12 +356,12 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
undefined ->
{error, not_found};
Listener ->
case maps:get(<<"authentication">>, Listener, undefined) of
case maps:get(?AUTHN_BIN, Listener, undefined) of
undefined ->
{error, not_found};
Auth ->
NListener = maps:put(
<<"authentication">>,
?AUTHN_BIN,
emqx_map_lib:deep_merge(Auth, Conf),
Listener
),
@ -371,9 +373,9 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
end;
pre_config_update(_, {remove_authn, GwName}, RawConf) ->
{ok, emqx_map_lib:deep_remove(
[GwName, <<"authentication">>], RawConf)};
[GwName, ?AUTHN_BIN], RawConf)};
pre_config_update(_, {remove_authn, GwName, {LType, LName}}, RawConf) ->
Path = [GwName, <<"listeners">>, LType, LName, <<"authentication">>],
Path = [GwName, <<"listeners">>, LType, LName, ?AUTHN_BIN],
{ok, emqx_map_lib:deep_remove(Path, RawConf)};
pre_config_update(_, UnknownReq, _RawConf) ->

7
apps/emqx_gateway/src/emqx_gateway_http.erl
View File

@ -19,6 +19,9 @@
-include("include/emqx_gateway.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx/include/emqx_authentication.hrl").
-define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
%% Mgmt APIs - gateway
-export([ gateways/1
@ -166,7 +169,7 @@ remove_listener(ListenerId) ->
-spec authn(gateway_name()) -> map().
authn(GwName) ->
%% XXX: Need append chain-nanme, authenticator-id?
Path = [gateway, GwName, authentication],
Path = [gateway, GwName, ?AUTHN],
ChainName = emqx_gateway_utils:global_chain(GwName),
wrap_chain_name(
ChainName,
@ -176,7 +179,7 @@ authn(GwName) ->
-spec authn(gateway_name(), binary()) -> map().
authn(GwName, ListenerId) ->
{_, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId),
Path = [gateway, GwName, listeners, Type, Name, authentication],
Path = [gateway, GwName, listeners, Type, Name, ?AUTHN],
ChainName = emqx_gateway_utils:listener_chain(GwName, Type, Name),
wrap_chain_name(
ChainName,

33
apps/emqx_gateway/src/emqx_gateway_schema.erl
View File

@ -24,6 +24,7 @@
-dialyzer(no_unused).
-dialyzer(no_fail_call).
-include_lib("emqx/include/emqx_authentication.hrl").
-include_lib("typerefl/include/types.hrl").
-type ip_port() :: tuple().
@ -144,7 +145,7 @@ The client just sends its PUBLISH messages to a GW"
, desc =>
"The Pre-defined topic ids and topic names.<br>
A 'pre-defined' topic id is a topic id whose mapping to a topic name
is known in advance by both the client’s application and the gateway"
is known in advance by both the client's application and the gateway"
})}
, {listeners, sc(ref(udp_listeners))}
] ++ gateway_common_options();
@ -407,30 +408,14 @@ fields(dtls_opts) ->
, ciphers => dtls_all_available
}, false).
authentication() ->
sc(hoconsc:union(
[ hoconsc:ref(emqx_authn_mnesia, config)
, hoconsc:ref(emqx_authn_mysql, config)
, hoconsc:ref(emqx_authn_pgsql, config)
, hoconsc:ref(emqx_authn_mongodb, standalone)
, hoconsc:ref(emqx_authn_mongodb, 'replica-set')
, hoconsc:ref(emqx_authn_mongodb, 'sharded-cluster')
, hoconsc:ref(emqx_authn_redis, standalone)
, hoconsc:ref(emqx_authn_redis, cluster)
, hoconsc:ref(emqx_authn_redis, sentinel)
, hoconsc:ref(emqx_authn_http, get)
, hoconsc:ref(emqx_authn_http, post)
, hoconsc:ref(emqx_authn_jwt, 'hmac-based')
, hoconsc:ref(emqx_authn_jwt, 'public-key')
, hoconsc:ref(emqx_authn_jwt, 'jwks')
, hoconsc:ref(emqx_enhanced_authn_scram_mnesia, config)
]),
#{ nullable => {true, recursively}
, desc =>
authentication_schema() ->
sc(emqx_authn_schema:authenticator_type(),
#{ nullable => {true, recursively}
, desc =>
"""Default authentication configs for all of the gateway listeners.<br>
For per-listener overrides see <code>authentication</code>
in listener configs"""
}).
}).
gateway_common_options() ->
[ {enable,
@ -464,7 +449,7 @@ it has two purposes:
sc(ref(clientinfo_override),
#{ desc => ""
})}
, {authentication, authentication()}
, {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication_schema()}
].
common_listener_opts() ->
@ -483,7 +468,7 @@ common_listener_opts() ->
sc(integer(),
#{ default => 1000
})}
, {authentication, authentication()}
, {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication_schema()}
, {mountpoint,
sc(binary(),
#{ default => undefined

13
apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl
View File

@ -93,10 +93,10 @@ info(ctx, #channel{ctx = Ctx}) ->
stats(_) ->
[].
init(ConnInfo = #{peername := {PeerHost, _},
sockname := {_, SockPort}},
init(ConnInfoT = #{peername := {PeerHost, _},
sockname := {_, SockPort}},
#{ctx := Ctx} = Config) ->
Peercert = maps:get(peercert, ConnInfo, undefined),
Peercert = maps:get(peercert, ConnInfoT, undefined),
Mountpoint = maps:get(mountpoint, Config, undefined),
ListenerId = case maps:get(listener, Config, undefined) of
undefined -> undefined;
@ -118,18 +118,20 @@ init(ConnInfo = #{peername := {PeerHost, _},
}
),
ConnInfo = ConnInfoT#{proto_name => <<"LwM2M">>, proto_ver => <<"0.0">>},
#channel{ ctx = Ctx
, conninfo = ConnInfo
, clientinfo = ClientInfo
, timers = #{}
, session = emqx_lwm2m_session:new()
%% FIXME: don't store anonymouse func
%% FIXME: don't store anonymouse func
, with_context = with_context(Ctx, ClientInfo)
}.
with_context(Ctx, ClientInfo) ->
fun(Type, Topic) ->
with_context(Type, Topic, Ctx, ClientInfo)
with_context(Type, Topic, Ctx, ClientInfo)
end.
lookup_cmd(Channel, Path, Action) ->
@ -293,7 +295,6 @@ check_lwm2m_version(#coap_message{options = Opts},
end,
if IsValid ->
NConnInfo = ConnInfo#{ connected_at => erlang:system_time(millisecond)
, proto_name => <<"LwM2M">>
, proto_ver => Ver
},
{ok, Channel#channel{conninfo = NConnInfo}};

3
apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl
View File

@ -28,6 +28,8 @@
-include_lib("eunit/include/eunit.hrl").
%% this parses to #{}, will not cause config cleanup
%% so we will need call emqx_config:erase
-define(CONF_DEFAULT, <<"
gateway {}
">>).
@ -39,6 +41,7 @@ gateway {}
all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Conf) ->
emqx_config:erase(gateway),
emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]),
Conf.

Some files were not shown because too many files have changed in this diff Show More