yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into dev/ee5.0

This commit is contained in:
Zaiming (Stone) Shi 2022-08-31 17:11:15 +02:00
parent 50cdb0fd05 47648b3874
commit 5a3fb4a011
80 changed files with 1502 additions and 972 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.ci/fvt_tests/http_server/rebar.config
View File

@ -3,7 +3,7 @@
{erl_opts, [debug_info]}.
{deps,
[
{minirest, {git, "https://github.com/emqx/minirest.git", {tag, "1.3.6"}}}
{minirest, {git, "https://github.com/emqx/minirest.git", {tag, "1.3.7"}}}
]}.
{shell, [

36
.github/workflows/release.yaml vendored
View File

@ -57,24 +57,24 @@ jobs:
-X POST \
-d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \
${{ secrets.EMQX_IO_RELEASE_API }}
- name: update repo.emqx.io
if: github.event_name == 'release'
run: |
REF=${{ github.ref_name }}
case "$REF" in
v*)
BOOL_FLAG_NAME="emqx_ce"
;;
e*)
BOOL_FLAG_NAME="emqx_ee"
;;
esac
curl --silent --show-error \
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
-X POST \
-d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ github.ref_name }}\", \"${BOOL_FLAG_NAME}\": \"true\"}}" \
"https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
- uses: emqx/push-helm-action@v1
if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
with:
charts_dir: "${{ github.workspace }}/deploy/charts/emqx"
version: ${{ github.ref_name }}
aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws_region: "us-west-2"
aws_bucket_name: "repos-emqx-io"
- uses: emqx/push-helm-action@v1
if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee'
with:
charts_dir: "${{ github.workspace }}/deploy/charts/emqx-ee"
version: ${{ github.ref_name }}
aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws_region: "us-west-2"
aws_bucket_name: "repos-emqx-io"
- name: update homebrew packages
if: github.event_name == 'release'
run: |

21
CHANGES-5.0.md
View File

@ -1,8 +1,26 @@
# 5.0.8
## Enhancements
* change the `/gateway` API path to plural form. [#8823](https://github.com/emqx/emqx/pull/8823)
# 5.0.7
## Bug fixes
* Remove `will_msg` (not used) field from the client API. [#8721](https://github.com/emqx/emqx/pull/8721)
* Fix `$queue` topic name error in management API return. [#8728](https://github.com/emqx/emqx/pull/8728)
* Fix race condition which may cause `client.connected` and `client.disconnected` out of order. [#8625](https://github.com/emqx/emqx/pull/8625)
## Enhancements
* Do not auto-populate default SSL cipher suites, so that the configs are less bloated. [#8769](https://github.com/emqx/emqx/pull/8769)
# 5.0.6
## Bug fixes
* Remove the needless `will_msg` field from the client API. [#8721](https://github.com/emqx/emqx/pull/8721)
* Upgrade Dashboard version to fix an issue where the node status was not displayed correctly. [#8771](https://github.com/emqx/emqx/pull/8771)
# 5.0.5
@ -19,6 +37,7 @@
* Updated `/nodes` API node_status from `Running/Stopped` to `running/stopped`. [#8642](https://github.com/emqx/emqx/pull/8642)
* Improve handling of placeholder interpolation errors [#8635](https://github.com/emqx/emqx/pull/8635)
* Better logging on unknown object IDs. [#8670](https://github.com/emqx/emqx/pull/8670)
* The bind option support `:1883` style. [#8758](https://github.com/emqx/emqx/pull/8758)
# 5.0.4

2
Makefile
View File

@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d
export EMQX_DEFAULT_RUNNER = debian:11-slim
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
export EMQX_DASHBOARD_VERSION ?= v1.0.6
export EMQX_DASHBOARD_VERSION ?= v1.0.7
export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.1
export EMQX_REL_FORM ?= tgz
export QUICER_DOWNLOAD_FROM_RELEASE = 1

2
apps/emqx/include/emqx_release.hrl
View File

@ -32,7 +32,7 @@
%% `apps/emqx/src/bpapi/README.md'
%% Community edition
-define(EMQX_RELEASE_CE, "5.0.5-beta.1").
-define(EMQX_RELEASE_CE, "5.0.6").
%% Enterprise edition
-define(EMQX_RELEASE_EE, "5.0.0-beta.2").

2
apps/emqx/rebar.config
View File

@ -27,7 +27,7 @@
{jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}},
{cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}},
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.3"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.4"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.30.0"}}},
{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},

7
apps/emqx/src/bpapi/emqx_bpapi.erl
View File

@ -23,6 +23,11 @@
versions_file/1
]).
%% Internal exports (RPC)
-export([
announce_fun/1
]).
-export_type([api/0, api_version/0, var_name/0, call/0, rpc/0, bpapi_meta/0]).
-include("emqx.hrl").
@ -77,7 +82,7 @@ supported_version(API) ->
-spec announce(atom()) -> ok.
announce(App) ->
{ok, Data} = file:consult(?MODULE:versions_file(App)),
{atomic, ok} = mria:transaction(?COMMON_SHARD, fun announce_fun/1, [Data]),
{atomic, ok} = mria:transaction(?COMMON_SHARD, fun ?MODULE:announce_fun/1, [Data]),
ok.
-spec versions_file(atom()) -> file:filename_all().

2
apps/emqx/src/emqx.app.src
View File

@ -3,7 +3,7 @@
{id, "emqx"},
{description, "EMQX Core"},
% strict semver, bump manually!
{vsn, "5.0.5"},
{vsn, "5.0.7"},
{modules, []},
{registered, []},
{applications, [

26
apps/emqx/src/emqx_alarm.erl
View File

@ -54,6 +54,12 @@
code_change/3
]).
%% Internal exports (RPC)
-export([
create_activate_alarm/3,
do_get_alarms/0
]).
-record(activated_alarm, {
name :: binary() | atom(),
details :: map() | list(),
@ -210,7 +216,7 @@ init([]) ->
handle_call({activate_alarm, Name, Details, Message}, _From, State) ->
Res = mria:transaction(
mria:local_content_shard(),
fun create_activate_alarm/3,
fun ?MODULE:create_activate_alarm/3,
[Name, Details, Message]
),
case Res of
@ -234,15 +240,7 @@ handle_call(delete_all_deactivated_alarms, _From, State) ->
handle_call({get_alarms, all}, _From, State) ->
{atomic, Alarms} =
mria:ro_transaction(
mria:local_content_shard(),
fun() ->
[
normalize(Alarm)
|| Alarm <-
ets:tab2list(?ACTIVATED_ALARM) ++
ets:tab2list(?DEACTIVATED_ALARM)
]
end
mria:local_content_shard(), fun ?MODULE:do_get_alarms/0
),
{reply, Alarms, State, get_validity_period()};
handle_call({get_alarms, activated}, _From, State) ->
@ -295,6 +293,14 @@ create_activate_alarm(Name, Details, Message) ->
Alarm
end.
do_get_alarms() ->
[
normalize(Alarm)
|| Alarm <-
ets:tab2list(?ACTIVATED_ALARM) ++
ets:tab2list(?DEACTIVATED_ALARM)
].
deactivate_alarm(
#activated_alarm{
activate_at = ActivateAt,

9
apps/emqx/src/emqx_banned.erl
View File

@ -49,6 +49,11 @@
code_change/3
]).
%% Internal exports (RPC)
-export([
expire_banned_items/1
]).
-elvis([{elvis_style, state_record_and_type, disable}]).
-define(BANNED_TAB, ?MODULE).
@ -224,7 +229,9 @@ handle_cast(Msg, State) ->
{noreply, State}.
handle_info({timeout, TRef, expire}, State = #{expiry_timer := TRef}) ->
_ = mria:transaction(?COMMON_SHARD, fun expire_banned_items/1, [erlang:system_time(second)]),
_ = mria:transaction(?COMMON_SHARD, fun ?MODULE:expire_banned_items/1, [
erlang:system_time(second)
]),
{noreply, ensure_expiry_timer(State), hibernate};
handle_info(Info, State) ->
?SLOG(error, #{msg => "unexpected_info", info => Info}),

7
apps/emqx/src/emqx_cm_registry.erl
View File

@ -44,6 +44,11 @@
code_change/3
]).
%% Internal exports (RPC)
-export([
do_cleanup_channels/1
]).
-define(REGISTRY, ?MODULE).
-define(TAB, emqx_channel_registry).
-define(LOCK, {?MODULE, cleanup_down}).
@ -155,7 +160,7 @@ cleanup_channels(Node) ->
global:trans(
{?LOCK, self()},
fun() ->
mria:transaction(?CM_SHARD, fun do_cleanup_channels/1, [Node])
mria:transaction(?CM_SHARD, fun ?MODULE:do_cleanup_channels/1, [Node])
end
).

12
apps/emqx/src/emqx_exclusive_subscription.erl
View File

@ -35,6 +35,11 @@
unsubscribe/2
]).
%% Internal exports (RPC)
-export([
try_subscribe/2
]).
-record(exclusive_subscription, {
topic :: emqx_types:topic(),
clientid :: emqx_types:clientid()
@ -80,10 +85,7 @@ on_delete_module() ->
-spec check_subscribe(emqx_types:clientinfo(), emqx_types:topic()) ->
allow | deny.
check_subscribe(#{clientid := ClientId}, Topic) ->
Fun = fun() ->
try_subscribe(ClientId, Topic)
end,
case mria:transaction(?EXCLUSIVE_SHARD, Fun) of
case mria:transaction(?EXCLUSIVE_SHARD, fun ?MODULE:try_subscribe/2, [ClientId, Topic]) of
{atomic, Res} ->
Res;
{aborted, Reason} ->
@ -94,7 +96,7 @@ check_subscribe(#{clientid := ClientId}, Topic) ->
end.
unsubscribe(Topic, #{is_exclusive := true}) ->
_ = mria:transaction(?EXCLUSIVE_SHARD, fun() -> mnesia:delete({?TAB, Topic}) end),
_ = mria:transaction(?EXCLUSIVE_SHARD, fun mnesia:delete/1, [{?TAB, Topic}]),
ok;
unsubscribe(_Topic, _SubOpts) ->
ok.

6
apps/emqx/src/emqx_listeners.erl
View File

@ -583,11 +583,7 @@ enable_authn(Opts) ->
maps:get(enable_authn, Opts, true).
ssl_opts(Opts) ->
maps:to_list(
emqx_tls_lib:drop_tls13_for_old_otp(
maps:get(ssl_options, Opts, #{})
)
).
emqx_tls_lib:to_server_opts(tls, maps:get(ssl_options, Opts, #{})).
tcp_opts(Opts) ->
maps:to_list(

7
apps/emqx/src/emqx_router_helper.erl
View File

@ -47,6 +47,11 @@
code_change/3
]).
%% Internal exports (RPC)
-export([
cleanup_routes/1
]).
-record(routing_node, {name, const = unused}).
-define(ROUTE, emqx_route).
@ -145,7 +150,7 @@ handle_info({nodedown, Node}, State = #{nodes := Nodes}) ->
global:trans(
{?LOCK, self()},
fun() ->
mria:transaction(?ROUTE_SHARD, fun cleanup_routes/1, [Node])
mria:transaction(?ROUTE_SHARD, fun ?MODULE:cleanup_routes/1, [Node])
end
),
ok = mria:dirty_delete(?ROUTING_NODE, Node),

41
apps/emqx/src/emqx_schema.erl
View File

@ -102,7 +102,7 @@
-export([namespace/0, roots/0, roots/1, fields/1, desc/1]).
-export([conf_get/2, conf_get/3, keys/2, filter/1]).
-export([server_ssl_opts_schema/2, client_ssl_opts_schema/1, ciphers_schema/1, default_ciphers/1]).
-export([server_ssl_opts_schema/2, client_ssl_opts_schema/1, ciphers_schema/1]).
-export([sc/2, map/2]).
-elvis([{elvis_style, god_modules, disable}]).
@ -1843,6 +1843,8 @@ filter(Opts) ->
common_ssl_opts_schema(Defaults) ->
D = fun(Field) -> maps:get(to_atom(Field), Defaults, undefined) end,
Df = fun(Field, Default) -> maps:get(to_atom(Field), Defaults, Default) end,
Collection = maps:get(versions, Defaults, tls_all_available),
AvailableVersions = default_tls_vsns(Collection),
[
{"cacertfile",
sc(
@ -1910,9 +1912,9 @@ common_ssl_opts_schema(Defaults) ->
sc(
hoconsc:array(typerefl:atom()),
#{
default => default_tls_vsns(maps:get(versions, Defaults, tls_all_available)),
default => AvailableVersions,
desc => ?DESC(common_ssl_opts_schema_versions),
validator => fun validate_tls_versions/1
validator => fun(Inputs) -> validate_tls_versions(AvailableVersions, Inputs) end
}
)},
{"ciphers", ciphers_schema(D("ciphers"))},
@ -2023,9 +2025,9 @@ client_ssl_opts_schema(Defaults) ->
].
default_tls_vsns(dtls_all_available) ->
proplists:get_value(available_dtls, ssl:versions());
emqx_tls_lib:available_versions(dtls);
default_tls_vsns(tls_all_available) ->
emqx_tls_lib:default_versions().
emqx_tls_lib:available_versions(tls).
-spec ciphers_schema(quic | dtls_all_available | tls_all_available | undefined) ->
hocon_schema:field_schema().
@ -2040,6 +2042,10 @@ ciphers_schema(Default) ->
#{
default => default_ciphers(Default),
converter => fun
(<<>>) ->
[];
("") ->
[];
(Ciphers) when is_binary(Ciphers) ->
binary:split(Ciphers, <<",">>, [global]);
(Ciphers) when is_list(Ciphers) ->
@ -2061,19 +2067,15 @@ default_ciphers(Which) ->
do_default_ciphers(Which)
).
do_default_ciphers(undefined) ->
do_default_ciphers(tls_all_available);
do_default_ciphers(quic) ->
[
"TLS_AES_256_GCM_SHA384",
"TLS_AES_128_GCM_SHA256",
"TLS_CHACHA20_POLY1305_SHA256"
];
do_default_ciphers(dtls_all_available) ->
%% as of now, dtls does not support tlsv1.3 ciphers
emqx_tls_lib:selected_ciphers(['dtlsv1.2', 'dtlsv1']);
do_default_ciphers(tls_all_available) ->
emqx_tls_lib:default_ciphers().
do_default_ciphers(_) ->
%% otherwise resolve default ciphers list at runtime
[].
%% @private return a list of keys in a parent field
-spec keys(string(), hocon:config()) -> [string()].
@ -2163,8 +2165,12 @@ to_bar_separated_list(Str) ->
%% - 127.0.0.1:1883
%% - ::1:1883
%% - [::1]:1883
%% - :1883
%% - :::1883
to_ip_port(Str) ->
case split_ip_port(Str) of
{"", Port} ->
{ok, {{0, 0, 0, 0}, list_to_integer(Port)}};
{Ip, Port} ->
PortVal = list_to_integer(Port),
case inet:parse_address(Ip) of
@ -2247,19 +2253,16 @@ parse_user_lookup_fun(StrConf) ->
{fun Mod:Fun/3, undefined}.
validate_ciphers(Ciphers) ->
All = emqx_tls_lib:all_ciphers(),
case lists:filter(fun(Cipher) -> not lists:member(Cipher, All) end, Ciphers) of
Set = emqx_tls_lib:all_ciphers_set_cached(),
case lists:filter(fun(Cipher) -> not sets:is_element(Cipher, Set) end, Ciphers) of
[] -> ok;
Bad -> {error, {bad_ciphers, Bad}}
end.
validate_tls_versions(Versions) ->
AvailableVersions =
proplists:get_value(available, ssl:versions()) ++
proplists:get_value(available_dtls, ssl:versions()),
validate_tls_versions(AvailableVersions, Versions) ->
case lists:filter(fun(V) -> not lists:member(V, AvailableVersions) end, Versions) of
[] -> ok;
Vs -> {error, {unsupported_ssl_versions, Vs}}
Vs -> {error, {unsupported_tls_versions, Vs}}
end.
validations() ->

37
apps/emqx/src/emqx_secret.erl Normal file
View File

@ -0,0 +1,37 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% Note: this module CAN'T be hot-patched to avoid invalidating the
%% closures, so it must not be changed.
-module(emqx_secret).
%% API:
-export([wrap/1, unwrap/1]).
%%================================================================================
%% API funcions
%%================================================================================
wrap(Term) ->
fun() ->
Term
end.
unwrap(Term) when is_function(Term, 0) ->
%% Handle potentially nested funs
unwrap(Term());
unwrap(Term) ->
Term.

7
apps/emqx/src/emqx_shared_sub.erl
View File

@ -67,6 +67,11 @@
code_change/3
]).
%% Internal exports (RPC)
-export([
init_monitors/0
]).
-export_type([strategy/0]).
-type strategy() ::
@ -336,7 +341,7 @@ subscribers(Group, Topic) ->
init([]) ->
ok = mria:wait_for_tables([?TAB]),
{ok, _} = mnesia:subscribe({table, ?TAB, simple}),
{atomic, PMon} = mria:transaction(?SHARED_SUB_SHARD, fun init_monitors/0),
{atomic, PMon} = mria:transaction(?SHARED_SUB_SHARD, fun ?MODULE:init_monitors/0),
ok = emqx_tables:new(?SHARED_SUBS, [protected, bag]),
ok = emqx_tables:new(?ALIVE_SUBS, [protected, set, {read_concurrency, true}]),
ok = emqx_tables:new(?SHARED_SUBS_ROUND_ROBIN_COUNTER, [public, set, {write_concurrency, true}]),

310
apps/emqx/src/emqx_tls_lib.erl
View File

@ -18,13 +18,12 @@
%% version & cipher suites
-export([
default_versions/0,
integral_versions/1,
available_versions/1,
integral_versions/2,
default_ciphers/0,
selected_ciphers/1,
integral_ciphers/2,
drop_tls13_for_old_otp/1,
all_ciphers/0
all_ciphers_set_cached/0
]).
%% SSL files
@ -38,7 +37,9 @@
]).
-export([
to_client_opts/1
to_server_opts/2,
to_client_opts/1,
to_client_opts/2
]).
-include("logger.hrl").
@ -54,27 +55,80 @@
%% non-empty list of strings
-define(IS_STRING_LIST(L), (is_list(L) andalso L =/= [] andalso ?IS_STRING(hd(L)))).
%% @doc Returns the default supported tls versions.
-spec default_versions() -> [atom()].
default_versions() -> available_versions().
%% The ciphers that ssl:cipher_suites(exclusive, 'tlsv1.3', openssl)
%% should return when running on otp 23.
%% But we still have to hard-code them because tlsv1.3 on otp 22 is
%% not trustworthy.
-define(TLSV13_EXCLUSIVE_CIPHERS, [
"TLS_AES_256_GCM_SHA384",
"TLS_AES_128_GCM_SHA256",
"TLS_CHACHA20_POLY1305_SHA256",
"TLS_AES_128_CCM_SHA256",
"TLS_AES_128_CCM_8_SHA256"
]).
-define(SELECTED_CIPHERS, [
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES256-SHA384",
"ECDHE-RSA-AES256-SHA384",
"ECDH-ECDSA-AES256-GCM-SHA384",
"ECDH-RSA-AES256-GCM-SHA384",
"ECDH-ECDSA-AES256-SHA384",
"ECDH-RSA-AES256-SHA384",
"DHE-DSS-AES256-GCM-SHA384",
"DHE-DSS-AES256-SHA256",
"AES256-GCM-SHA384",
"AES256-SHA256",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-AES128-SHA256",
"ECDHE-RSA-AES128-SHA256",
"ECDH-ECDSA-AES128-GCM-SHA256",
"ECDH-RSA-AES128-GCM-SHA256",
"ECDH-ECDSA-AES128-SHA256",
"ECDH-RSA-AES128-SHA256",
"DHE-DSS-AES128-GCM-SHA256",
"DHE-DSS-AES128-SHA256",
"AES128-GCM-SHA256",
"AES128-SHA256",
"ECDHE-ECDSA-AES256-SHA",
"ECDHE-RSA-AES256-SHA",
"DHE-DSS-AES256-SHA",
"ECDH-ECDSA-AES256-SHA",
"ECDH-RSA-AES256-SHA",
"ECDHE-ECDSA-AES128-SHA",
"ECDHE-RSA-AES128-SHA",
"DHE-DSS-AES128-SHA",
"ECDH-ECDSA-AES128-SHA",
"ECDH-RSA-AES128-SHA",
%% psk
"RSA-PSK-AES256-GCM-SHA384",
"RSA-PSK-AES256-CBC-SHA384",
"RSA-PSK-AES128-GCM-SHA256",
"RSA-PSK-AES128-CBC-SHA256",
"RSA-PSK-AES256-CBC-SHA",
"RSA-PSK-AES128-CBC-SHA"
]).
%% @doc Validate a given list of desired tls versions.
%% raise an error exception if non of them are available.
%% The input list can be a string/binary of comma separated versions.
-spec integral_versions(undefined | string() | binary() | [ssl:tls_version()]) ->
-spec integral_versions(tls | dtls, undefined | string() | binary() | [ssl:tls_version()]) ->
[ssl:tls_version()].
integral_versions(undefined) ->
integral_versions(default_versions());
integral_versions([]) ->
integral_versions(default_versions());
integral_versions(<<>>) ->
integral_versions(default_versions());
integral_versions(Desired) when ?IS_STRING(Desired) ->
integral_versions(iolist_to_binary(Desired));
integral_versions(Desired) when is_binary(Desired) ->
integral_versions(parse_versions(Desired));
integral_versions(Desired) ->
Available = available_versions(),
integral_versions(Type, undefined) ->
available_versions(Type);
integral_versions(Type, []) ->
available_versions(Type);
integral_versions(Type, <<>>) ->
available_versions(Type);
integral_versions(Type, Desired) when ?IS_STRING(Desired) ->
integral_versions(Type, iolist_to_binary(Desired));
integral_versions(Type, Desired) when is_binary(Desired) ->
integral_versions(Type, parse_versions(Desired));
integral_versions(Type, Desired) ->
Available = available_versions(Type),
case lists:filter(fun(V) -> lists:member(V, Available) end, Desired) of
[] ->
erlang:error(#{
@ -86,33 +140,36 @@ integral_versions(Desired) ->
Filtered
end.
%% @doc Return a list of all supported ciphers.
all_ciphers() -> all_ciphers(default_versions()).
%% @doc Return a set of all ciphers
all_ciphers_set_cached() ->
case persistent_term:get(?FUNCTION_NAME, false) of
false ->
S = sets:from_list(all_ciphers()),
persistent_term:put(?FUNCTION_NAME, S);
Set ->
Set
end.
%% @doc Return a list of (openssl string format) cipher suites.
%% @hidden Return a list of all supported ciphers.
all_ciphers() ->
all_ciphers(available_versions(all)).
%% @hidden Return a list of (openssl string format) cipher suites.
-spec all_ciphers([ssl:tls_version()]) -> [string()].
all_ciphers(['tlsv1.3']) ->
%% When it's only tlsv1.3 wanted, use 'exclusive' here
%% because 'all' returns legacy cipher suites too,
%% which does not make sense since tlsv1.3 can not use
%% legacy cipher suites.
ssl:cipher_suites(exclusive, 'tlsv1.3', openssl);
?TLSV13_EXCLUSIVE_CIPHERS;
all_ciphers(Versions) ->
%% assert non-empty
List = lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions]),
[_ | _] = dedup(List).
%% @doc All Pre-selected TLS ciphers.
%% ssl:cipher_suites(all, V, openssl) is too slow. so we cache default ciphers.
default_ciphers() ->
case persistent_term:get(default_ciphers, undefined) of
undefined ->
Default = selected_ciphers(available_versions()),
persistent_term:put(default_ciphers, Default),
Default;
Default ->
Default
end.
selected_ciphers(available_versions(all)).
%% @doc Pre-selected TLS ciphers for given versions..
selected_ciphers(Vsns) ->
@ -126,54 +183,11 @@ selected_ciphers(Vsns) ->
do_selected_ciphers('tlsv1.3') ->
case lists:member('tlsv1.3', proplists:get_value(available, ssl:versions())) of
true -> ssl:cipher_suites(exclusive, 'tlsv1.3', openssl);
true -> ?TLSV13_EXCLUSIVE_CIPHERS;
false -> []
end ++ do_selected_ciphers('tlsv1.2');
do_selected_ciphers(_) ->
[
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES256-SHA384",
"ECDHE-RSA-AES256-SHA384",
"ECDH-ECDSA-AES256-GCM-SHA384",
"ECDH-RSA-AES256-GCM-SHA384",
"ECDH-ECDSA-AES256-SHA384",
"ECDH-RSA-AES256-SHA384",
"DHE-DSS-AES256-GCM-SHA384",
"DHE-DSS-AES256-SHA256",
"AES256-GCM-SHA384",
"AES256-SHA256",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-AES128-SHA256",
"ECDHE-RSA-AES128-SHA256",
"ECDH-ECDSA-AES128-GCM-SHA256",
"ECDH-RSA-AES128-GCM-SHA256",
"ECDH-ECDSA-AES128-SHA256",
"ECDH-RSA-AES128-SHA256",
"DHE-DSS-AES128-GCM-SHA256",
"DHE-DSS-AES128-SHA256",
"AES128-GCM-SHA256",
"AES128-SHA256",
"ECDHE-ECDSA-AES256-SHA",
"ECDHE-RSA-AES256-SHA",
"DHE-DSS-AES256-SHA",
"ECDH-ECDSA-AES256-SHA",
"ECDH-RSA-AES256-SHA",
"ECDHE-ECDSA-AES128-SHA",
"ECDHE-RSA-AES128-SHA",
"DHE-DSS-AES128-SHA",
"ECDH-ECDSA-AES128-SHA",
"ECDH-RSA-AES128-SHA",
%% psk
"RSA-PSK-AES256-GCM-SHA384",
"RSA-PSK-AES256-CBC-SHA384",
"RSA-PSK-AES128-GCM-SHA256",
"RSA-PSK-AES128-CBC-SHA256",
"RSA-PSK-AES256-CBC-SHA",
"RSA-PSK-AES128-CBC-SHA"
].
?SELECTED_CIPHERS.
%% @doc Ensure version & cipher-suites integrity.
-spec integral_ciphers([ssl:tls_version()], binary() | string() | [string()]) -> [string()].
@ -201,17 +215,17 @@ ensure_tls13_cipher(true, Ciphers) ->
ensure_tls13_cipher(false, Ciphers) ->
Ciphers.
%% default ssl versions based on available versions.
-spec available_versions() -> [atom()].
available_versions() ->
OtpRelease = list_to_integer(erlang:system_info(otp_release)),
default_versions(OtpRelease).
%% @doc Returns the default available tls/dtls versions.
available_versions(Type) ->
All = ssl:versions(),
available_versions(Type, All).
%% tlsv1.3 is available from OTP-22 but we do not want to use until 23.
default_versions(OtpRelease) when OtpRelease >= 23 ->
proplists:get_value(available, ssl:versions());
default_versions(_) ->
lists:delete('tlsv1.3', proplists:get_value(available, ssl:versions())).
available_versions(tls, All) ->
proplists:get_value(available, All);
available_versions(dtls, All) ->
proplists:get_value(available_dtls, All);
available_versions(all, All) ->
available_versions(tls, All) ++ available_versions(dtls, All).
%% Deduplicate a list without re-ordering the elements.
dedup([]) ->
@ -244,6 +258,8 @@ do_parse_versions([V | More], Acc) ->
do_parse_versions(More, [Parsed | Acc])
end.
parse_version(<<"dtlsv1.2">>) -> 'dtlsv1.2';
parse_version(<<"dtlsv1">>) -> dtlsv1;
parse_version(<<"tlsv", Vsn/binary>>) -> parse_version(Vsn);
parse_version(<<"v", Vsn/binary>>) -> parse_version(Vsn);
parse_version(<<"1.3">>) -> 'tlsv1.3';
@ -259,36 +275,6 @@ split_by_comma(Bin) ->
trim_space(Bin) ->
hd([I || I <- binary:split(Bin, <<" ">>), I =/= <<>>]).
%% @doc Drop tlsv1.3 version and ciphers from ssl options
%% if running on otp 22 or earlier.
drop_tls13_for_old_otp(SslOpts) ->
case list_to_integer(erlang:system_info(otp_release)) < 23 of
true -> drop_tls13(SslOpts);
false -> SslOpts
end.
%% The ciphers that ssl:cipher_suites(exclusive, 'tlsv1.3', openssl)
%% should return when running on otp 23.
%% But we still have to hard-code them because tlsv1.3 on otp 22 is
%% not trustworthy.
-define(TLSV13_EXCLUSIVE_CIPHERS, [
"TLS_AES_256_GCM_SHA384",
"TLS_AES_128_GCM_SHA256",
"TLS_CHACHA20_POLY1305_SHA256",
"TLS_AES_128_CCM_SHA256",
"TLS_AES_128_CCM_8_SHA256"
]).
drop_tls13(SslOpts0) ->
SslOpts1 =
case maps:find(versions, SslOpts0) of
error -> SslOpts0;
{ok, Vsns} -> SslOpts0#{versions => (Vsns -- ['tlsv1.3'])}
end,
case maps:find(ciphers, SslOpts1) of
error -> SslOpts1;
{ok, Ciphers} -> SslOpts1#{ciphers => Ciphers -- ?TLSV13_EXCLUSIVE_CIPHERS}
end.
%% @doc The input map is a HOCON decoded result of a struct defined as
%% emqx_schema:server_ssl_opts_schema. (NOTE: before schema-checked).
%% `keyfile', `certfile' and `cacertfile' can be either pem format key or certificates,
@ -498,27 +484,54 @@ do_drop_invalid_certs([Key | Keys], SSL) ->
end
end.
%% @doc Convert hocon-checked ssl client options (map()) to
%% @doc Convert hocon-checked ssl server options (map()) to
%% proplist accepted by ssl library.
-spec to_server_opts(tls | dtls, map()) -> [{atom(), term()}].
to_server_opts(Type, Opts) ->
Versions = integral_versions(Type, maps:get(versions, Opts, undefined)),
Ciphers = integral_ciphers(Versions, maps:get(ciphers, Opts, undefined)),
maps:to_list(Opts#{
ciphers => Ciphers,
versions => Versions
}).
%% @doc Convert hocon-checked tls client options (map()) to
%% proplist accepted by ssl library.
-spec to_client_opts(map()) -> [{atom(), term()}].
to_client_opts(Opts) ->
to_client_opts(tls, Opts).
%% @doc Convert hocon-checked tls or dtls client options (map()) to
%% proplist accepted by ssl library.
-spec to_client_opts(tls | dtls, map()) -> [{atom(), term()}].
to_client_opts(Type, Opts) ->
GetD = fun(Key, Default) -> fuzzy_map_get(Key, Opts, Default) end,
Get = fun(Key) -> GetD(Key, undefined) end,
KeyFile = ensure_str(Get(keyfile)),
CertFile = ensure_str(Get(certfile)),
CAFile = ensure_str(Get(cacertfile)),
Verify = GetD(verify, verify_none),
SNI = ensure_sni(Get(server_name_indication)),
Versions = integral_versions(Get(versions)),
Ciphers = integral_ciphers(Versions, Get(ciphers)),
filter([
{keyfile, KeyFile},
{certfile, CertFile},
{cacertfile, CAFile},
{verify, Verify},
{server_name_indication, SNI},
{versions, Versions},
{ciphers, Ciphers}
]).
case GetD(enable, false) of
true ->
KeyFile = ensure_str(Get(keyfile)),
CertFile = ensure_str(Get(certfile)),
CAFile = ensure_str(Get(cacertfile)),
Verify = GetD(verify, verify_none),
SNI = ensure_sni(Get(server_name_indication)),
Versions = integral_versions(Type, Get(versions)),
Ciphers = integral_ciphers(Versions, Get(ciphers)),
filter([
{keyfile, KeyFile},
{certfile, CertFile},
{cacertfile, CAFile},
{verify, Verify},
{server_name_indication, SNI},
{versions, Versions},
{ciphers, Ciphers},
{reuse_sessions, Get(reuse_sessions)},
{depth, Get(depth)},
{password, ensure_str(Get(password))},
{secure_renegotiate, Get(secure_renegotiate)}
]);
false ->
[]
end.
filter([]) -> [];
filter([{_, undefined} | T]) -> filter(T);
@ -556,28 +569,3 @@ ensure_ssl_file_key(SSL, RequiredKeys) ->
[] -> ok;
Miss -> {error, #{reason => ssl_file_option_not_found, which_options => Miss}}
end.
-if(?OTP_RELEASE > 22).
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
drop_tls13_test() ->
Versions = default_versions(),
?assert(lists:member('tlsv1.3', Versions)),
Ciphers = all_ciphers(),
?assert(has_tlsv13_cipher(Ciphers)),
Opts0 = #{versions => Versions, ciphers => Ciphers, other => true},
Opts = drop_tls13(Opts0),
?assertNot(lists:member('tlsv1.3', maps:get(versions, Opts, undefined))),
?assertNot(has_tlsv13_cipher(maps:get(ciphers, Opts, undefined))).
drop_tls13_no_versions_cipers_test() ->
Opts0 = #{other => 0, bool => true},
Opts = drop_tls13(Opts0),
?_assertEqual(Opts0, Opts).
has_tlsv13_cipher(Ciphers) ->
lists:any(fun(C) -> lists:member(C, Ciphers) end, ?TLSV13_EXCLUSIVE_CIPHERS).
-endif.
-endif.

83
apps/emqx/src/emqx_trace/emqx_trace.erl
View File

@ -51,10 +51,7 @@
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
-define(TRACE, ?MODULE).
-define(SHARD, ?COMMON_SHARD).
-define(MAX_SIZE, 30).
-define(OWN_KEYS, [level, filters, filter_default, handlers]).
-include("emqx_trace.hrl").
-ifdef(TEST).
-export([
@ -66,15 +63,6 @@
-export_type([ip_address/0]).
-type ip_address() :: string().
-record(?TRACE, {
name :: binary() | undefined | '_',
type :: clientid | topic | ip_address | undefined | '_',
filter :: emqx_types:topic() | emqx_types:clientid() | ip_address() | undefined | '_',
enable = true :: boolean() | '_',
start_at :: integer() | undefined | '_',
end_at :: integer() | undefined | '_'
}).
publish(#message{topic = <<"$SYS/", _/binary>>}) ->
ignore;
publish(#message{from = From, topic = Topic, payload = Payload}) when
@ -172,13 +160,7 @@ create(Trace) ->
-spec delete(Name :: binary()) -> ok | {error, not_found}.
delete(Name) ->
Tran = fun() ->
case mnesia:read(?TRACE, Name) of
[_] -> mnesia:delete(?TRACE, Name, write);
[] -> mnesia:abort(not_found)
end
end,
transaction(Tran).
transaction(fun emqx_trace_dl:delete/1, [Name]).
-spec clear() -> ok | {error, Reason :: term()}.
clear() ->
@ -190,20 +172,7 @@ clear() ->
-spec update(Name :: binary(), Enable :: boolean()) ->
ok | {error, not_found | finished}.
update(Name, Enable) ->
Tran = fun() ->
case mnesia:read(?TRACE, Name) of
[] ->
mnesia:abort(not_found);
[#?TRACE{enable = Enable}] ->
ok;
[Rec] ->
case erlang:system_time(second) >= Rec#?TRACE.end_at of
false -> mnesia:write(?TRACE, Rec#?TRACE{enable = Enable}, write);
true -> mnesia:abort(finished)
end
end
end,
transaction(Tran).
transaction(fun emqx_trace_dl:update/2, [Name, Enable]).
check() ->
gen_server:call(?MODULE, check).
@ -211,13 +180,7 @@ check() ->
-spec get_trace_filename(Name :: binary()) ->
{ok, FileName :: string()} | {error, not_found}.
get_trace_filename(Name) ->
Tran = fun() ->
case mnesia:read(?TRACE, Name, read) of
[] -> mnesia:abort(not_found);
[#?TRACE{start_at = Start}] -> {ok, filename(Name, Start)}
end
end,
transaction(Tran).
transaction(fun emqx_trace_dl:get_trace_filename/1, [Name]).
-spec trace_file(File :: file:filename_all()) ->
{ok, Node :: list(), Binary :: binary()}
@ -309,23 +272,7 @@ code_change(_, State, _Extra) ->
{ok, State}.
insert_new_trace(Trace) ->
Tran = fun() ->
case mnesia:read(?TRACE, Trace#?TRACE.name) of
[] ->
#?TRACE{start_at = StartAt, type = Type, filter = Filter} = Trace,
Match = #?TRACE{_ = '_', start_at = StartAt, type = Type, filter = Filter},
case mnesia:match_object(?TRACE, Match, read) of
[] ->
ok = mnesia:write(?TRACE, Trace, write),
{ok, Trace};
[#?TRACE{name = Name}] ->
mnesia:abort({duplicate_condition, Name})
end;
[#?TRACE{name = Name}] ->
mnesia:abort({already_existed, Name})
end
end,
transaction(Tran).
transaction(fun emqx_trace_dl:insert_new_trace/1, [Trace]).
update_trace(Traces) ->
Now = erlang:system_time(second),
@ -347,9 +294,7 @@ stop_all_trace_handler() ->
get_enabled_trace() ->
{atomic, Traces} =
mria:ro_transaction(?SHARD, fun() ->
mnesia:match_object(?TRACE, #?TRACE{enable = true, _ = '_'}, read)
end),
mria:ro_transaction(?SHARD, fun emqx_trace_dl:get_enabled_trace/0),
Traces.
find_closest_time(Traces, Now) ->
@ -372,17 +317,7 @@ closest(Time, Now, Closest) -> min(Time - Now, Closest).
disable_finished([]) ->
ok;
disable_finished(Traces) ->
transaction(fun() ->
lists:map(
fun(#?TRACE{name = Name}) ->
case mnesia:read(?TRACE, Name, write) of
[] -> ok;
[Trace] -> mnesia:write(?TRACE, Trace#?TRACE{enable = false}, write)
end
end,
Traces
)
end).
transaction(fun emqx_trace_dl:delete_finished/1, [Traces]).
start_trace(Traces, Started0) ->
Started = lists:map(fun(#{name := Name}) -> Name end, Started0),
@ -586,8 +521,8 @@ filename(Name, Start) ->
[Time, _] = string:split(calendar:system_time_to_rfc3339(Start), "T", leading),
lists:flatten(["trace_", binary_to_list(Name), "_", Time, ".log"]).
transaction(Tran) ->
case mria:transaction(?COMMON_SHARD, Tran) of
transaction(Fun, Args) ->
case mria:transaction(?COMMON_SHARD, Fun, Args) of
{atomic, Res} -> Res;
{aborted, Reason} -> {error, Reason}
end.

35
apps/emqx/src/emqx_trace/emqx_trace.hrl Normal file
View File

@ -0,0 +1,35 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-ifndef(EMQX_TRACE_HRL).
-define(EMQX_TRACE_HRL, true).
-define(TRACE, emqx_trace).
-record(?TRACE, {
name :: binary() | undefined | '_',
type :: clientid | topic | ip_address | undefined | '_',
filter ::
emqx_types:topic() | emqx_types:clientid() | emqx_trace:ip_address() | undefined | '_',
enable = true :: boolean() | '_',
start_at :: integer() | undefined | '_',
end_at :: integer() | undefined | '_'
}).
-define(SHARD, ?COMMON_SHARD).
-define(MAX_SIZE, 30).
-define(OWN_KEYS, [level, filters, filter_default, handlers]).
-endif.

103
apps/emqx/src/emqx_trace/emqx_trace_dl.erl Normal file
View File

@ -0,0 +1,103 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% Data layer for emqx_trace
-module(emqx_trace_dl).
%% API:
-export([
update/2,
insert_new_trace/1,
delete/1,
get_trace_filename/1,
delete_finished/1,
get_enabled_trace/0
]).
-include("emqx_trace.hrl").
%%================================================================================
%% API funcions
%%================================================================================
%% Introduced in 5.0
-spec update(Name :: binary(), Enable :: boolean()) ->
ok.
update(Name, Enable) ->
case mnesia:read(?TRACE, Name) of
[] ->
mnesia:abort(not_found);
[#?TRACE{enable = Enable}] ->
ok;
[Rec] ->
case erlang:system_time(second) >= Rec#?TRACE.end_at of
false -> mnesia:write(?TRACE, Rec#?TRACE{enable = Enable}, write);
true -> mnesia:abort(finished)
end
end.
%% Introduced in 5.0
insert_new_trace(Trace) ->
case mnesia:read(?TRACE, Trace#?TRACE.name) of
[] ->
#?TRACE{start_at = StartAt, type = Type, filter = Filter} = Trace,
Match = #?TRACE{_ = '_', start_at = StartAt, type = Type, filter = Filter},
case mnesia:match_object(?TRACE, Match, read) of
[] ->
ok = mnesia:write(?TRACE, Trace, write),
{ok, Trace};
[#?TRACE{name = Name}] ->
mnesia:abort({duplicate_condition, Name})
end;
[#?TRACE{name = Name}] ->
mnesia:abort({already_existed, Name})
end.
%% Introduced in 5.0
-spec delete(Name :: binary()) -> ok.
delete(Name) ->
case mnesia:read(?TRACE, Name) of
[_] -> mnesia:delete(?TRACE, Name, write);
[] -> mnesia:abort(not_found)
end.
%% Introduced in 5.0
-spec get_trace_filename(Name :: binary()) -> {ok, string()}.
get_trace_filename(Name) ->
case mnesia:read(?TRACE, Name, read) of
[] -> mnesia:abort(not_found);
[#?TRACE{start_at = Start}] -> {ok, emqx_trace:filename(Name, Start)}
end.
%% Introduced in 5.0
delete_finished(Traces) ->
lists:map(
fun(#?TRACE{name = Name}) ->
case mnesia:read(?TRACE, Name, write) of
[] -> ok;
[Trace] -> mnesia:write(?TRACE, Trace#?TRACE{enable = false}, write)
end
end,
Traces
).
%% Introduced in 5.0
get_enabled_trace() ->
mnesia:match_object(?TRACE, #?TRACE{enable = true, _ = '_'}, read).
%%================================================================================
%% Internal functions
%%================================================================================

2
apps/emqx/test/emqx_router_helper_SUITE.erl
View File

@ -72,7 +72,7 @@ end_per_testcase(TestCase, Config) when
->
Slave = ?config(slave, Config),
emqx_common_test_helpers:stop_slave(Slave),
mria:transaction(?ROUTE_SHARD, fun() -> mnesia:clear_table(?ROUTE_TAB) end),
mria:clear_table(?ROUTE_TAB),
snabbkaffe:stop(),
ok;
end_per_testcase(_TestCase, _Config) ->

13
apps/emqx/test/emqx_schema_tests.erl
View File

@ -21,8 +21,7 @@
ssl_opts_dtls_test() ->
Sc = emqx_schema:server_ssl_opts_schema(
#{
versions => dtls_all_available,
ciphers => dtls_all_available
versions => dtls_all_available
},
false
),
@ -30,7 +29,7 @@ ssl_opts_dtls_test() ->
?assertMatch(
#{
versions := ['dtlsv1.2', 'dtlsv1'],
ciphers := ["ECDHE-ECDSA-AES256-GCM-SHA384" | _]
ciphers := []
},
Checked
).
@ -42,7 +41,7 @@ ssl_opts_tls_1_3_test() ->
?assertMatch(
#{
versions := ['tlsv1.3'],
ciphers := [_ | _]
ciphers := []
},
Checked
).
@ -53,7 +52,7 @@ ssl_opts_tls_for_ranch_test() ->
?assertMatch(
#{
versions := ['tlsv1.3'],
ciphers := [_ | _],
ciphers := [],
handshake_timeout := _
},
Checked
@ -125,7 +124,7 @@ validate(Schema, Data0) ->
),
Checked.
ciperhs_schema_test() ->
ciphers_schema_test() ->
Sc = emqx_schema:ciphers_schema(undefined),
WSc = #{roots => [{ciphers, Sc}]},
?assertThrow(
@ -135,7 +134,7 @@ ciperhs_schema_test() ->
bad_tls_version_test() ->
Sc = emqx_schema:server_ssl_opts_schema(#{}, false),
Reason = {unsupported_ssl_versions, [foo]},
Reason = {unsupported_tls_versions, [foo]},
?assertThrow(
{_Sc, [#{kind := validation_error, reason := Reason}]},
validate(Sc, #{<<"versions">> => [<<"foo">>]})

44
apps/emqx/test/emqx_tls_lib_tests.erl
View File

@ -51,24 +51,34 @@ test_cipher_format(Input) ->
?assertEqual([?TLS_13_CIPHER, ?TLS_12_CIPHER], Ciphers).
tls_versions_test() ->
?assert(lists:member('tlsv1.3', emqx_tls_lib:default_versions())).
?assert(lists:member('tlsv1.3', emqx_tls_lib:available_versions(tls))).
tls_version_unknown_test() ->
?assertEqual(
emqx_tls_lib:default_versions(),
emqx_tls_lib:integral_versions([])
),
?assertEqual(
emqx_tls_lib:default_versions(),
emqx_tls_lib:integral_versions(<<>>)
),
?assertEqual(
emqx_tls_lib:default_versions(),
emqx_tls_lib:integral_versions("foo")
),
?assertError(
#{reason := no_available_tls_version},
emqx_tls_lib:integral_versions([foo])
tls_version_unknown_test_() ->
lists:flatmap(
fun(Type) ->
[
?_assertEqual(
emqx_tls_lib:available_versions(Type),
emqx_tls_lib:integral_versions(Type, [])
),
?_assertEqual(
emqx_tls_lib:available_versions(Type),
emqx_tls_lib:integral_versions(Type, <<>>)
),
?_assertEqual(
emqx_tls_lib:available_versions(Type),
%% unknown version dropped
emqx_tls_lib:integral_versions(Type, "foo")
),
fun() ->
?assertError(
#{reason := no_available_tls_version},
emqx_tls_lib:integral_versions(Type, [foo])
)
end
]
end,
[tls, dtls]
).
cipher_suites_no_duplication_test() ->

2
apps/emqx_authn/src/emqx_authn.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_authn, [
{description, "EMQX Authentication"},
{vsn, "0.1.4"},
{vsn, "0.1.5"},
{modules, []},
{registered, [emqx_authn_sup, emqx_authn_registry]},
{applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]},

132
apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
View File

@ -52,6 +52,14 @@
group_match_spec/1
]).
%% Internal exports (RPC)
-export([
do_destroy/1,
do_add_user/2,
do_delete_user/2,
do_update_user/3
]).
-define(TAB, ?MODULE).
-define(AUTHN_QSCHEMA, [
{<<"like_user_id">>, binary},
@ -170,83 +178,79 @@ authenticate(_Credential, _State) ->
ignore.
destroy(#{user_group := UserGroup}) ->
trans(fun ?MODULE:do_destroy/1, [UserGroup]).
do_destroy(UserGroup) ->
MatchSpec = group_match_spec(UserGroup),
trans(
fun() ->
ok = lists:foreach(
fun(UserInfo) ->
mnesia:delete_object(?TAB, UserInfo, write)
end,
mnesia:select(?TAB, MatchSpec, write)
)
end
ok = lists:foreach(
fun(UserInfo) ->
mnesia:delete_object(?TAB, UserInfo, write)
end,
mnesia:select(?TAB, MatchSpec, write)
).
add_user(
add_user(UserInfo, State) ->
trans(fun ?MODULE:do_add_user/2, [UserInfo, State]).
do_add_user(
#{
user_id := UserID,
password := Password
} = UserInfo,
#{user_group := UserGroup} = State
) ->
trans(
fun() ->
case mnesia:read(?TAB, {UserGroup, UserID}, write) of
[] ->
IsSuperuser = maps:get(is_superuser, UserInfo, false),
add_user(UserGroup, UserID, Password, IsSuperuser, State),
{ok, #{user_id => UserID, is_superuser => IsSuperuser}};
[_] ->
{error, already_exist}
end
end
).
case mnesia:read(?TAB, {UserGroup, UserID}, write) of
[] ->
IsSuperuser = maps:get(is_superuser, UserInfo, false),
add_user(UserGroup, UserID, Password, IsSuperuser, State),
{ok, #{user_id => UserID, is_superuser => IsSuperuser}};
[_] ->
{error, already_exist}
end.
delete_user(UserID, #{user_group := UserGroup}) ->
trans(
fun() ->
case mnesia:read(?TAB, {UserGroup, UserID}, write) of
[] ->
{error, not_found};
[_] ->
mnesia:delete(?TAB, {UserGroup, UserID}, write)
end
end
).
delete_user(UserID, State) ->
trans(fun ?MODULE:do_delete_user/2, [UserID, State]).
update_user(
do_delete_user(UserID, #{user_group := UserGroup}) ->
case mnesia:read(?TAB, {UserGroup, UserID}, write) of
[] ->
{error, not_found};
[_] ->
mnesia:delete(?TAB, {UserGroup, UserID}, write)
end.
update_user(UserID, User, State) ->
trans(fun ?MODULE:do_update_user/3, [UserID, User, State]).
do_update_user(
UserID,
User,
#{user_group := UserGroup} = State
) ->
trans(
fun() ->
case mnesia:read(?TAB, {UserGroup, UserID}, write) of
[] ->
{error, not_found};
[#user_info{is_superuser = IsSuperuser} = UserInfo] ->
UserInfo1 = UserInfo#user_info{
is_superuser = maps:get(is_superuser, User, IsSuperuser)
},
UserInfo2 =
case maps:get(password, User, undefined) of
undefined ->
UserInfo1;
Password ->
{StoredKey, ServerKey, Salt} = esasl_scram:generate_authentication_info(
Password, State
),
UserInfo1#user_info{
stored_key = StoredKey,
server_key = ServerKey,
salt = Salt
}
end,
mnesia:write(?TAB, UserInfo2, write),
{ok, format_user_info(UserInfo2)}
end
end
).
case mnesia:read(?TAB, {UserGroup, UserID}, write) of
[] ->
{error, not_found};
[#user_info{is_superuser = IsSuperuser} = UserInfo] ->
UserInfo1 = UserInfo#user_info{
is_superuser = maps:get(is_superuser, User, IsSuperuser)
},
UserInfo2 =
case maps:get(password, User, undefined) of
undefined ->
UserInfo1;
Password ->
{StoredKey, ServerKey, Salt} = esasl_scram:generate_authentication_info(
Password, State
),
UserInfo1#user_info{
stored_key = StoredKey,
server_key = ServerKey,
salt = Salt
}
end,
mnesia:write(?TAB, UserInfo2, write),
{ok, format_user_info(UserInfo2)}
end.
lookup_user(UserID, #{user_group := UserGroup}) ->
case mnesia:dirty_read(?TAB, {UserGroup, UserID}) of
@ -386,12 +390,10 @@ retrieve(UserID, #{user_group := UserGroup}) ->
end.
%% TODO: Move to emqx_authn_utils.erl
trans(Fun) ->
trans(Fun, []).
trans(Fun, Args) ->
case mria:transaction(?AUTH_SHARD, Fun, Args) of
{atomic, Res} -> Res;
{aborted, {function_clause, Stack}} -> erlang:raise(error, function_clause, Stack);
{aborted, Reason} -> {error, Reason}
end.

137
apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
View File

@ -54,6 +54,16 @@
group_match_spec/1
]).
%% Internal exports (RPC)
-export([
do_destroy/1,
do_add_user/2,
do_delete_user/2,
do_update_user/3,
import/2,
import_csv/3
]).
-type user_group() :: binary().
-type user_id() :: binary().
@ -175,15 +185,14 @@ authenticate(
end.
destroy(#{user_group := UserGroup}) ->
trans(
fun() ->
ok = lists:foreach(
fun(User) ->
mnesia:delete_object(?TAB, User, write)
end,
mnesia:select(?TAB, group_match_spec(UserGroup), write)
)
end
trans(fun ?MODULE:do_destroy/1, [UserGroup]).
do_destroy(UserGroup) ->
ok = lists:foreach(
fun(User) ->
mnesia:delete_object(?TAB, User, write)
end,
mnesia:select(?TAB, group_match_spec(UserGroup), write)
).
import_users({Filename0, FileData}, State) ->
@ -200,7 +209,10 @@ import_users({Filename0, FileData}, State) ->
{error, {unsupported_file_format, Extension}}
end.
add_user(
add_user(UserInfo, State) ->
trans(fun ?MODULE:do_add_user/2, [UserInfo, State]).
do_add_user(
#{
user_id := UserID,
password := Password
@ -210,33 +222,31 @@ add_user(
password_hash_algorithm := Algorithm
}
) ->
trans(
fun() ->
case mnesia:read(?TAB, {UserGroup, UserID}, write) of
[] ->
{PasswordHash, Salt} = emqx_authn_password_hashing:hash(Algorithm, Password),
IsSuperuser = maps:get(is_superuser, UserInfo, false),
insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser),
{ok, #{user_id => UserID, is_superuser => IsSuperuser}};
[_] ->
{error, already_exist}
end
end
).
case mnesia:read(?TAB, {UserGroup, UserID}, write) of
[] ->
{PasswordHash, Salt} = emqx_authn_password_hashing:hash(Algorithm, Password),
IsSuperuser = maps:get(is_superuser, UserInfo, false),
insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser),
{ok, #{user_id => UserID, is_superuser => IsSuperuser}};
[_] ->
{error, already_exist}
end.
delete_user(UserID, #{user_group := UserGroup}) ->
trans(
fun() ->
case mnesia:read(?TAB, {UserGroup, UserID}, write) of
[] ->
{error, not_found};
[_] ->
mnesia:delete(?TAB, {UserGroup, UserID}, write)
end
end
).
delete_user(UserID, State) ->
trans(fun ?MODULE:do_delete_user/2, [UserID, State]).
update_user(
do_delete_user(UserID, #{user_group := UserGroup}) ->
case mnesia:read(?TAB, {UserGroup, UserID}, write) of
[] ->
{error, not_found};
[_] ->
mnesia:delete(?TAB, {UserGroup, UserID}, write)
end.
update_user(UserID, UserInfo, State) ->
trans(fun ?MODULE:do_update_user/3, [UserID, UserInfo, State]).
do_update_user(
UserID,
UserInfo,
#{
@ -244,33 +254,29 @@ update_user(
password_hash_algorithm := Algorithm
}
) ->
trans(
fun() ->
case mnesia:read(?TAB, {UserGroup, UserID}, write) of
[] ->
{error, not_found};
[
#user_info{
password_hash = PasswordHash,
salt = Salt,
is_superuser = IsSuperuser
}
] ->
NSuperuser = maps:get(is_superuser, UserInfo, IsSuperuser),
{NPasswordHash, NSalt} =
case UserInfo of
#{password := Password} ->
emqx_authn_password_hashing:hash(
Algorithm, Password
);
#{} ->
{PasswordHash, Salt}
end,
insert_user(UserGroup, UserID, NPasswordHash, NSalt, NSuperuser),
{ok, #{user_id => UserID, is_superuser => NSuperuser}}
end
end
).
case mnesia:read(?TAB, {UserGroup, UserID}, write) of
[] ->
{error, not_found};
[
#user_info{
password_hash = PasswordHash,
salt = Salt,
is_superuser = IsSuperuser
}
] ->
NSuperuser = maps:get(is_superuser, UserInfo, IsSuperuser),
{NPasswordHash, NSalt} =
case UserInfo of
#{password := Password} ->
emqx_authn_password_hashing:hash(
Algorithm, Password
);
#{} ->
{PasswordHash, Salt}
end,
insert_user(UserGroup, UserID, NPasswordHash, NSalt, NSuperuser),
{ok, #{user_id => UserID, is_superuser => NSuperuser}}
end.
lookup_user(UserID, #{user_group := UserGroup}) ->
case mnesia:dirty_read(?TAB, {UserGroup, UserID}) of
@ -335,7 +341,7 @@ run_fuzzy_filter(
import_users_from_json(Bin, #{user_group := UserGroup}) ->
case emqx_json:safe_decode(Bin, [return_maps]) of
{ok, List} ->
trans(fun import/2, [UserGroup, List]);
trans(fun ?MODULE:import/2, [UserGroup, List]);
{error, Reason} ->
{error, Reason}
end.
@ -344,7 +350,7 @@ import_users_from_json(Bin, #{user_group := UserGroup}) ->
import_users_from_csv(CSV, #{user_group := UserGroup}) ->
case get_csv_header(CSV) of
{ok, Seq, NewCSV} ->
trans(fun import_csv/3, [UserGroup, NewCSV, Seq]);
trans(fun ?MODULE:import_csv/3, [UserGroup, NewCSV, Seq]);
{error, Reason} ->
{error, Reason}
end.
@ -435,9 +441,6 @@ get_user_identity(#{clientid := ClientID}, clientid) ->
get_user_identity(_, Type) ->
{error, {bad_user_identity_type, Type}}.
trans(Fun) ->
trans(Fun, []).
trans(Fun, Args) ->
case mria:transaction(?AUTH_SHARD, Fun, Args) of
{atomic, Res} -> Res;

34
apps/emqx_conf/src/emqx_cluster_rpc.erl
View File

@ -31,10 +31,16 @@
fast_forward_to_commit/2
]).
-export([
get_node_tnx_id/1,
commit/2,
commit_status_trans/2,
get_cluster_tnx_id/0,
get_node_tnx_id/1,
init_mfa/2,
latest_tnx_id/0,
make_initiate_call_req/3
make_initiate_call_req/3,
read_next_mfa/1,
trans_query/1,
trans_status/0
]).
-export([
@ -194,18 +200,18 @@ do_multicall(M, F, A, RequiredSyncs, Timeout) ->
-spec query(pos_integer()) -> {'atomic', map()} | {'aborted', Reason :: term()}.
query(TnxId) ->
transaction(fun trans_query/1, [TnxId]).
transaction(fun ?MODULE:trans_query/1, [TnxId]).
-spec reset() -> reset.
reset() -> gen_server:call(?MODULE, reset).
-spec status() -> {'atomic', [map()]} | {'aborted', Reason :: term()}.
status() ->
transaction(fun trans_status/0, []).
transaction(fun ?MODULE:trans_status/0, []).
-spec latest_tnx_id() -> pos_integer().
latest_tnx_id() ->
{atomic, TnxId} = transaction(fun get_cluster_tnx_id/0, []),
{atomic, TnxId} = transaction(fun ?MODULE:get_cluster_tnx_id/0, []),
TnxId.
-spec make_initiate_call_req(module(), atom(), list()) -> init_call_req().
@ -280,7 +286,7 @@ handle_call(reset, _From, State) ->
_ = mria:clear_table(?CLUSTER_MFA),
{reply, ok, State, {continue, ?CATCH_UP}};
handle_call(?INITIATE(MFA), _From, State = #{node := Node}) ->
case transaction(fun init_mfa/2, [Node, MFA]) of
case transaction(fun ?MODULE:init_mfa/2, [Node, MFA]) of
{atomic, {ok, TnxId, Result}} ->
{reply, {ok, TnxId, Result}, State, {continue, ?CATCH_UP}};
{aborted, Error} ->
@ -288,7 +294,7 @@ handle_call(?INITIATE(MFA), _From, State = #{node := Node}) ->
end;
handle_call(skip_failed_commit, _From, State = #{node := Node}) ->
Timeout = catch_up(State, true),
{atomic, LatestId} = transaction(fun get_node_tnx_id/1, [Node]),
{atomic, LatestId} = transaction(fun ?MODULE:get_node_tnx_id/1, [Node]),
{reply, LatestId, State, Timeout};
handle_call({fast_forward_to_commit, ToTnxId}, _From, State) ->
NodeId = do_fast_forward_to_commit(ToTnxId, State),
@ -316,14 +322,14 @@ code_change(_OldVsn, State, _Extra) ->
catch_up(State) -> catch_up(State, false).
catch_up(#{node := Node, retry_interval := RetryMs} = State, SkipResult) ->
case transaction(fun read_next_mfa/1, [Node]) of
case transaction(fun ?MODULE:read_next_mfa/1, [Node]) of
{atomic, caught_up} ->
?TIMEOUT;
{atomic, {still_lagging, NextId, MFA}} ->
{Succeed, _} = apply_mfa(NextId, MFA, ?APPLY_KIND_REPLICATE),
case Succeed orelse SkipResult of
true ->
case transaction(fun commit/2, [Node, NextId]) of
case transaction(fun ?MODULE:commit/2, [Node, NextId]) of
{atomic, ok} ->
catch_up(State, false);
Error ->
@ -367,12 +373,12 @@ commit(Node, TnxId) ->
ok = mnesia:write(?CLUSTER_COMMIT, #cluster_rpc_commit{node = Node, tnx_id = TnxId}, write).
do_fast_forward_to_commit(ToTnxId, State = #{node := Node}) ->
{atomic, NodeId} = transaction(fun get_node_tnx_id/1, [Node]),
{atomic, NodeId} = transaction(fun ?MODULE:get_node_tnx_id/1, [Node]),
case NodeId >= ToTnxId of
true ->
NodeId;
false ->
{atomic, LatestId} = transaction(fun get_cluster_tnx_id/0, []),
{atomic, LatestId} = transaction(fun ?MODULE:get_cluster_tnx_id/0, []),
case LatestId =< NodeId of
true ->
NodeId;
@ -529,11 +535,11 @@ wait_for_nodes_commit(RequiredSyncs, TnxId, Delay, Remain) ->
end.
lagging_node(TnxId) ->
{atomic, Nodes} = transaction(fun commit_status_trans/2, ['<', TnxId]),
{atomic, Nodes} = transaction(fun ?MODULE:commit_status_trans/2, ['<', TnxId]),
Nodes.
synced_nodes(TnxId) ->
{atomic, Nodes} = transaction(fun commit_status_trans/2, ['>=', TnxId]),
{atomic, Nodes} = transaction(fun ?MODULE:commit_status_trans/2, ['>=', TnxId]),
Nodes.
commit_status_trans(Operator, TnxId) ->
@ -547,5 +553,5 @@ get_retry_ms() ->
maybe_init_tnx_id(_Node, TnxId) when TnxId < 0 -> ok;
maybe_init_tnx_id(Node, TnxId) ->
{atomic, _} = transaction(fun commit/2, [Node, TnxId]),
{atomic, _} = transaction(fun ?MODULE:commit/2, [Node, TnxId]),
ok.

7
apps/emqx_conf/src/emqx_cluster_rpc_handler.erl
View File

@ -30,6 +30,11 @@
code_change/3
]).
%% Internal exports (RPC)
-export([
del_stale_mfa/1
]).
start_link() ->
MaxHistory = emqx_conf:get(["node", "cluster_call", "max_history"], 100),
CleanupMs = emqx_conf:get(["node", "cluster_call", "cleanup_interval"], 5 * 60 * 1000),
@ -56,7 +61,7 @@ handle_cast(Msg, State) ->
{noreply, State}.
handle_info({timeout, TRef, del_stale_mfa}, State = #{timer := TRef, max_history := MaxHistory}) ->
case mria:transaction(?CLUSTER_RPC_SHARD, fun del_stale_mfa/1, [MaxHistory]) of
case mria:transaction(?CLUSTER_RPC_SHARD, fun ?MODULE:del_stale_mfa/1, [MaxHistory]) of
{atomic, ok} -> ok;
Error -> ?SLOG(error, #{msg => "del_stale_cluster_rpc_mfa_error", error => Error})
end,

2
apps/emqx_connector/src/emqx_connector.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_connector, [
{description, "An OTP application"},
{vsn, "0.1.3"},
{vsn, "0.1.4"},
{registered, []},
{mod, {emqx_connector_app, []}},
{applications, [

4
apps/emqx_connector/src/emqx_connector_pgsql.erl
View File

@ -100,7 +100,7 @@ on_start(
{host, Host},
{port, Port},
{username, User},
{password, Password},
{password, emqx_secret:wrap(Password)},
{database, DB},
{auto_reconnect, reconn_interval(AutoReconn)},
{pool_size, PoolSize},
@ -160,7 +160,7 @@ reconn_interval(false) -> false.
connect(Opts) ->
Host = proplists:get_value(host, Opts),
Username = proplists:get_value(username, Opts),
Password = proplists:get_value(password, Opts),
Password = emqx_secret:unwrap(proplists:get_value(password, Opts)),
PrepareStatement = proplists:get_value(prepare_statement, Opts),
case epgsql:connect(Host, Username, Password, conn_opts(Opts)) of
{ok, Conn} ->

2
apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf
View File

@ -197,7 +197,7 @@ its own from which a browser should permit loading resources."""
zh: "多语言支持"
}
}
bootstrap_user {
bootstrap_users_file {
desc {
en: "Initialize users file."
zh: "初始化用户文件"

19
apps/emqx_dashboard/src/emqx_dashboard_admin.erl
View File

@ -52,7 +52,7 @@
-export([
add_default_user/0,
default_username/0,
add_bootstrap_user/0
add_bootstrap_users/0
]).
-type emqx_admin() :: #?ADMIN{}.
@ -85,16 +85,16 @@ mnesia(boot) ->
add_default_user() ->
add_default_user(binenv(default_username), binenv(default_password)).
-spec add_bootstrap_user() -> ok | {error, _}.
add_bootstrap_user() ->
case emqx:get_config([dashboard, bootstrap_user], undefined) of
-spec add_bootstrap_users() -> ok | {error, _}.
add_bootstrap_users() ->
case emqx:get_config([dashboard, bootstrap_users_file], undefined) of
undefined ->
ok;
File ->
case mnesia:table_info(?ADMIN, size) of
0 ->
?SLOG(debug, #{msg => "Add dashboard bootstrap users", file => File}),
add_bootstrap_user(File);
add_bootstrap_users(File);
_ ->
ok
end
@ -312,7 +312,7 @@ add_default_user(Username, Password) ->
_ -> {ok, default_user_exists}
end.
add_bootstrap_user(File) ->
add_bootstrap_users(File) ->
case file:open(File, [read]) of
{ok, Dev} ->
{ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]),
@ -324,7 +324,12 @@ add_bootstrap_user(File) ->
after
file:close(Dev)
end;
Error ->
{error, Reason} = Error ->
?SLOG(error, #{
msg => "failed to open the dashboard bootstrap users file",
file => File,
reason => Reason
}),
Error
end.

2
apps/emqx_dashboard/src/emqx_dashboard_app.erl
View File

@ -31,7 +31,7 @@ start(_StartType, _StartArgs) ->
case emqx_dashboard:start_listeners() of
ok ->
emqx_dashboard_cli:load(),
case emqx_dashboard_admin:add_bootstrap_user() of
case emqx_dashboard_admin:add_bootstrap_users() of
ok ->
{ok, _} = emqx_dashboard_admin:add_default_user(),
{ok, Sup};

3
apps/emqx_dashboard/src/emqx_dashboard_schema.erl
View File

@ -55,7 +55,8 @@ fields("dashboard") ->
)},
{cors, fun cors/1},
{i18n_lang, fun i18n_lang/1},
{bootstrap_user, ?HOCON(binary(), #{desc => ?DESC(bootstrap_user), required => false})}
{bootstrap_users_file,
?HOCON(binary(), #{desc => ?DESC(bootstrap_users_file), required => false})}
];
fields("listeners") ->
[

2
apps/emqx_exhook/src/emqx_exhook.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_exhook, [
{description, "EMQX Extension for Hook"},
{vsn, "5.0.2"},
{vsn, "5.0.3"},
{modules, []},
{registered, []},
{mod, {emqx_exhook_app, []}},

2
apps/emqx_exhook/src/emqx_exhook_server.erl
View File

@ -231,7 +231,7 @@ resolve_hookspec(HookSpecs) when is_list(HookSpecs) ->
end,
case {lists:member(Name, AvailableHooks), lists:member(Name, MessageHooks)} of
{false, _} ->
error({unknown_hookpoint, Name});
error({unknown_hookpoint, Name0});
{true, false} ->
Acc#{Name => #{}};
{true, true} ->

10
apps/emqx_gateway/src/coap/emqx_coap_api.erl
View File

@ -28,7 +28,7 @@
-export([request/2]).
-define(PREFIX, "/gateway/coap/clients/:clientid").
-define(PREFIX, "/gateways/coap/clients/:clientid").
-import(hoconsc, [mk/2, enum/1]).
-import(emqx_dashboard_swagger, [error_codes/2]).
@ -42,13 +42,13 @@ api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}).
paths() ->
[?PREFIX ++ "/request"].
emqx_gateway_utils:make_deprecated_paths([?PREFIX ++ "/request"]).
schema(?PREFIX ++ "/request") ->
#{
operationId => request,
post => #{
tags => [<<"CoAP gateway">>],
tags => [<<"CoAP">>],
desc => ?DESC(send_coap_request),
parameters => request_parameters(),
requestBody => request_body(),
@ -60,7 +60,9 @@ schema(?PREFIX ++ "/request") ->
)
}
}
}.
};
schema(Path) ->
emqx_gateway_utils:make_compatible_schema(Path, fun schema/1).
request(post, #{body := Body, bindings := Bindings}) ->
ClientId = maps:get(clientid, Bindings, undefined),

2
apps/emqx_gateway/src/emqx_gateway.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_gateway, [
{description, "The Gateway management application"},
{vsn, "0.1.3"},
{vsn, "0.1.4"},
{registered, []},
{mod, {emqx_gateway_app, []}},
{applications, [kernel, stdlib, grpc, emqx, emqx_authn]},

16
apps/emqx_gateway/src/emqx_gateway_api.erl
View File

@ -61,10 +61,10 @@ api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
paths() ->
[
"/gateway",
"/gateway/:name"
].
emqx_gateway_utils:make_deprecated_paths([
"/gateways",
"/gateways/:name"
]).
%%--------------------------------------------------------------------
%% http handlers
@ -159,7 +159,7 @@ gateway_insta(put, #{
%% Swagger defines
%%--------------------------------------------------------------------
schema("/gateway") ->
schema("/gateways") ->
#{
'operationId' => gateway,
get =>
@ -185,7 +185,7 @@ schema("/gateway") ->
?STANDARD_RESP(#{201 => schema_gateways_conf()})
}
};
schema("/gateway/:name") ->
schema("/gateways/:name") ->
#{
'operationId' => gateway_insta,
get =>
@ -210,7 +210,9 @@ schema("/gateway/:name") ->
responses =>
?STANDARD_RESP(#{200 => schema_gateways_conf()})
}
}.
};
schema(Path) ->
emqx_gateway_utils:make_compatible_schema(Path, fun schema/1).
%%--------------------------------------------------------------------
%% params defines

21
apps/emqx_gateway/src/emqx_gateway_api_authn.erl
View File

@ -60,11 +60,11 @@ api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
paths() ->
[
"/gateway/:name/authentication",
"/gateway/:name/authentication/users",
"/gateway/:name/authentication/users/:uid"
].
emqx_gateway_utils:make_deprecated_paths([
"/gateways/:name/authentication",
"/gateways/:name/authentication/users",
"/gateways/:name/authentication/users/:uid"
]).
%%--------------------------------------------------------------------
%% http handlers
@ -176,7 +176,7 @@ parse_qstring(Qs) ->
%% Swagger defines
%%--------------------------------------------------------------------
schema("/gateway/:name/authentication") ->
schema("/gateways/:name/authentication") ->
#{
'operationId' => authn,
get =>
@ -215,7 +215,7 @@ schema("/gateway/:name/authentication") ->
?STANDARD_RESP(#{204 => <<"Deleted">>})
}
};
schema("/gateway/:name/authentication/users") ->
schema("/gateways/:name/authentication/users") ->
#{
'operationId' => users,
get =>
@ -253,7 +253,7 @@ schema("/gateway/:name/authentication/users") ->
)
}
};
schema("/gateway/:name/authentication/users/:uid") ->
schema("/gateways/:name/authentication/users/:uid") ->
#{
'operationId' => users_insta,
get =>
@ -298,8 +298,9 @@ schema("/gateway/:name/authentication/users/:uid") ->
responses =>
?STANDARD_RESP(#{204 => <<"User Deleted">>})
}
}.
};
schema(Path) ->
emqx_gateway_utils:make_compatible_schema(Path, fun schema/1).
%%--------------------------------------------------------------------
%% params defines

17
apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl
View File

@ -53,10 +53,10 @@ api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false}).
paths() ->
[
"/gateway/:name/authentication/import_users",
"/gateway/:name/listeners/:id/authentication/import_users"
].
emqx_gateway_utils:make_deprecated_paths([
"/gateways/:name/authentication/import_users",
"/gateways/:name/listeners/:id/authentication/import_users"
]).
%%--------------------------------------------------------------------
%% http handlers
@ -117,7 +117,7 @@ import_listener_users(post, #{
%% Swagger defines
%%--------------------------------------------------------------------
schema("/gateway/:name/authentication/import_users") ->
schema("/gateways/:name/authentication/import_users") ->
#{
'operationId' => import_users,
post =>
@ -129,7 +129,7 @@ schema("/gateway/:name/authentication/import_users") ->
?STANDARD_RESP(#{204 => <<"Imported">>})
}
};
schema("/gateway/:name/listeners/:id/authentication/import_users") ->
schema("/gateways/:name/listeners/:id/authentication/import_users") ->
#{
'operationId' => import_listener_users,
post =>
@ -141,8 +141,9 @@ schema("/gateway/:name/listeners/:id/authentication/import_users") ->
responses =>
?STANDARD_RESP(#{204 => <<"Imported">>})
}
}.
};
schema(Path) ->
emqx_gateway_utils:make_compatible_schema(Path, fun schema/1).
%%--------------------------------------------------------------------
%% params defines
%%--------------------------------------------------------------------

24
apps/emqx_gateway/src/emqx_gateway_api_clients.erl
View File

@ -67,12 +67,12 @@ api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}).
paths() ->
[
"/gateway/:name/clients",
"/gateway/:name/clients/:clientid",
"/gateway/:name/clients/:clientid/subscriptions",
"/gateway/:name/clients/:clientid/subscriptions/:topic"
].
emqx_gateway_utils:make_deprecated_paths([
"/gateways/:name/clients",
"/gateways/:name/clients/:clientid",
"/gateways/:name/clients/:clientid/subscriptions",
"/gateways/:name/clients/:clientid/subscriptions/:topic"
]).
-define(CLIENT_QSCHEMA, [
{<<"node">>, atom},
@ -462,7 +462,7 @@ conn_state_to_connected(_) -> false.
%% Swagger defines
%%--------------------------------------------------------------------
schema("/gateway/:name/clients") ->
schema("/gateways/:name/clients") ->
#{
'operationId' => clients,
get =>
@ -473,7 +473,7 @@ schema("/gateway/:name/clients") ->
?STANDARD_RESP(#{200 => schema_client_list()})
}
};
schema("/gateway/:name/clients/:clientid") ->
schema("/gateways/:name/clients/:clientid") ->
#{
'operationId' => clients_insta,
get =>
@ -491,7 +491,7 @@ schema("/gateway/:name/clients/:clientid") ->
?STANDARD_RESP(#{204 => <<"Kicked">>})
}
};
schema("/gateway/:name/clients/:clientid/subscriptions") ->
schema("/gateways/:name/clients/:clientid/subscriptions") ->
#{
'operationId' => subscriptions,
get =>
@ -527,7 +527,7 @@ schema("/gateway/:name/clients/:clientid/subscriptions") ->
)
}
};
schema("/gateway/:name/clients/:clientid/subscriptions/:topic") ->
schema("/gateways/:name/clients/:clientid/subscriptions/:topic") ->
#{
'operationId' => subscriptions,
delete =>
@ -537,7 +537,9 @@ schema("/gateway/:name/clients/:clientid/subscriptions/:topic") ->
responses =>
?STANDARD_RESP(#{204 => <<"Unsubscribed">>})
}
}.
};
schema(Path) ->
emqx_gateway_utils:make_compatible_schema(Path, fun schema/1).
params_client_query() ->
params_gateway_name_in_path() ++

29
apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
View File

@ -68,13 +68,13 @@ api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
paths() ->
[
"/gateway/:name/listeners",
"/gateway/:name/listeners/:id",
"/gateway/:name/listeners/:id/authentication",
"/gateway/:name/listeners/:id/authentication/users",
"/gateway/:name/listeners/:id/authentication/users/:uid"
].
emqx_gateway_utils:make_deprecated_paths([
"/gateways/:name/listeners",
"/gateways/:name/listeners/:id",
"/gateways/:name/listeners/:id/authentication",
"/gateways/:name/listeners/:id/authentication/users",
"/gateways/:name/listeners/:id/authentication/users/:uid"
]).
%%--------------------------------------------------------------------
%% http handlers
@ -353,7 +353,7 @@ bind2str(Listener = #{<<"bind">> := Bind}) ->
%% Swagger defines
%%--------------------------------------------------------------------
schema("/gateway/:name/listeners") ->
schema("/gateways/:name/listeners") ->
#{
'operationId' => listeners,
get =>
@ -391,7 +391,7 @@ schema("/gateway/:name/listeners") ->
)
}
};
schema("/gateway/:name/listeners/:id") ->
schema("/gateways/:name/listeners/:id") ->
#{
'operationId' => listeners_insta,
get =>
@ -437,7 +437,7 @@ schema("/gateway/:name/listeners/:id") ->
)
}
};
schema("/gateway/:name/listeners/:id/authentication") ->
schema("/gateways/:name/listeners/:id/authentication") ->
#{
'operationId' => listeners_insta_authn,
get =>
@ -480,7 +480,7 @@ schema("/gateway/:name/listeners/:id/authentication") ->
?STANDARD_RESP(#{200 => <<"Deleted">>})
}
};
schema("/gateway/:name/listeners/:id/authentication/users") ->
schema("/gateways/:name/listeners/:id/authentication/users") ->
#{
'operationId' => users,
get =>
@ -519,7 +519,7 @@ schema("/gateway/:name/listeners/:id/authentication/users") ->
)
}
};
schema("/gateway/:name/listeners/:id/authentication/users/:uid") ->
schema("/gateways/:name/listeners/:id/authentication/users/:uid") ->
#{
'operationId' => users_insta,
get =>
@ -567,8 +567,9 @@ schema("/gateway/:name/listeners/:id/authentication/users/:uid") ->
responses =>
?STANDARD_RESP(#{204 => <<"Deleted">>})
}
}.
};
schema(Path) ->
emqx_gateway_utils:make_compatible_schema(Path, fun schema/1).
%%--------------------------------------------------------------------
%% params defines

7
apps/emqx_gateway/src/emqx_gateway_cm_registry.erl
View File

@ -42,6 +42,11 @@
code_change/3
]).
%% Internal exports (RPC)
-export([
do_cleanup_channels/2
]).
-define(CM_SHARD, emqx_gateway_cm_shard).
-define(LOCK, {?MODULE, cleanup_down}).
@ -148,7 +153,7 @@ cleanup_channels(Node, Name) ->
global:trans(
{?LOCK, self()},
fun() ->
mria:transaction(?CM_SHARD, fun do_cleanup_channels/2, [Node, Tab])
mria:transaction(?CM_SHARD, fun ?MODULE:do_cleanup_channels/2, [Node, Tab])
end
).

6
apps/emqx_gateway/src/emqx_gateway_schema.erl
View File

@ -365,8 +365,7 @@ fields(ssl_server_opts) ->
#{
depth => 10,
reuse_sessions => true,
versions => tls_all_available,
ciphers => tls_all_available
versions => tls_all_available
},
true
);
@ -502,8 +501,7 @@ fields(dtls_opts) ->
#{
depth => 10,
reuse_sessions => true,
versions => dtls_all_available,
ciphers => dtls_all_available
versions => dtls_all_available
},
false
).

51
apps/emqx_gateway/src/emqx_gateway_utils.erl
View File

@ -44,7 +44,9 @@
parse_listener_id/1,
is_running/2,
global_chain/1,
listener_chain/3
listener_chain/3,
make_deprecated_paths/1,
make_compatible_schema/2
]).
-export([stringfy/1]).
@ -455,14 +457,12 @@ esockd_access_rules(StrRules) ->
[Access(R) || R <- StrRules].
ssl_opts(Name, Opts) ->
maps:to_list(
emqx_tls_lib:drop_tls13_for_old_otp(
maps:without(
[enable],
maps:get(Name, Opts, #{})
)
)
).
Type =
case Name of
ssl -> tls;
dtls -> dtls
end,
emqx_tls_lib:to_server_opts(Type, maps:get(Name, Opts, #{})).
sock_opts(Name, Opts) ->
maps:to_list(
@ -540,3 +540,36 @@ default_subopts() ->
qos => 0,
is_new => true
}.
%% Since 5.0.8, the API path of the gateway has been changed from "gateway" to "gateways"
%% and we need to be compatible with the old path
get_compatible_path("/gateway") ->
"/gateways";
get_compatible_path("/gateway/" ++ Rest) ->
"/gateways/" ++ Rest.
get_deprecated_path("/gateways") ->
"/gateway";
get_deprecated_path("/gateways/" ++ Rest) ->
"/gateway/" ++ Rest.
make_deprecated_paths(Paths) ->
Paths ++ [get_deprecated_path(Path) || Path <- Paths].
make_compatible_schema(Path, SchemaFun) ->
OldPath = get_compatible_path(Path),
make_compatible_schema2(OldPath, SchemaFun).
make_compatible_schema2(Path, SchemaFun) ->
Schema = SchemaFun(Path),
maps:map(
fun(Key, Value) ->
case lists:member(Key, [get, delete, put, post]) of
true ->
Value#{deprecated => true};
_ ->
Value
end
end,
Schema
).

18
apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl
View File

@ -25,7 +25,7 @@
-export([lookup/2, observe/2, read/2, write/2]).
-define(PATH(Suffix), "/gateway/lwm2m/clients/:clientid" Suffix).
-define(PATH(Suffix), "/gateways/lwm2m/clients/:clientid" Suffix).
-define(DATA_TYPE, ['Integer', 'Float', 'Time', 'String', 'Boolean', 'Opaque', 'Objlnk']).
-import(hoconsc, [mk/2, ref/1, ref/2]).
@ -37,13 +37,15 @@ api_spec() ->
emqx_dashboard_swagger:spec(?MODULE).
paths() ->
[?PATH("/lookup"), ?PATH("/observe"), ?PATH("/read"), ?PATH("/write")].
emqx_gateway_utils:make_deprecated_paths([
?PATH("/lookup"), ?PATH("/observe"), ?PATH("/read"), ?PATH("/write")
]).
schema(?PATH("/lookup")) ->
#{
'operationId' => lookup,
get => #{
tags => [<<"lwm2m">>],
tags => [<<"LwM2M">>],
desc => ?DESC(lookup_resource),
parameters => [
{clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})},
@ -67,7 +69,7 @@ schema(?PATH("/observe")) ->
#{
'operationId' => observe,
post => #{
tags => [<<"lwm2m">>],
tags => [<<"LwM2M">>],
desc => ?DESC(observe_resource),
parameters => [
{clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})},
@ -85,7 +87,7 @@ schema(?PATH("/read")) ->
#{
'operationId' => read,
post => #{
tags => [<<"lwm2m">>],
tags => [<<"LwM2M">>],
desc => ?DESC(read_resource),
parameters => [
{clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})},
@ -102,7 +104,7 @@ schema(?PATH("/write")) ->
'operationId' => write,
post => #{
desc => ?DESC(write_resource),
tags => [<<"lwm2m">>],
tags => [<<"LwM2M">>],
parameters => [
{clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})},
{path, mk(binary(), #{in => query, required => true, example => "/3/0/7"})},
@ -118,7 +120,9 @@ schema(?PATH("/write")) ->
404 => error_codes(['CLIENT_NOT_FOUND'], <<"Clientid not found">>)
}
}
}.
};
schema(Path) ->
emqx_gateway_utils:make_compatible_schema(Path, fun schema/1).
fields(resource) ->
[

63
apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl
View File

@ -46,6 +46,11 @@
code_change/3
]).
%% Internal exports (RPC)
-export([
do_register/4
]).
-export([lookup_name/1]).
-define(SN_SHARD, emqx_sn_shard).
@ -173,33 +178,11 @@ handle_call(
TopicId when TopicId >= 16#FFFF ->
{reply, {error, too_large}, State};
TopicId ->
Fun = fun() ->
mnesia:write(
Tab,
#emqx_sn_registry{
key = {ClientId, next_topic_id},
value = TopicId + 1
},
write
),
mnesia:write(
Tab,
#emqx_sn_registry{
key = {ClientId, TopicName},
value = TopicId
},
write
),
mnesia:write(
Tab,
#emqx_sn_registry{
key = {ClientId, TopicId},
value = TopicName
},
write
)
end,
case mria:transaction(?SN_SHARD, Fun) of
case
mria:transaction(?SN_SHARD, fun ?MODULE:do_register/4, [
Tab, ClientId, TopicId, TopicName
])
of
{atomic, ok} ->
{reply, TopicId, State};
{aborted, Error} ->
@ -248,6 +231,32 @@ terminate(_Reason, _State) ->
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
do_register(Tab, ClientId, TopicId, TopicName) ->
mnesia:write(
Tab,
#emqx_sn_registry{
key = {ClientId, next_topic_id},
value = TopicId + 1
},
write
),
mnesia:write(
Tab,
#emqx_sn_registry{
key = {ClientId, TopicName},
value = TopicId
},
write
),
mnesia:write(
Tab,
#emqx_sn_registry{
key = {ClientId, TopicId},
value = TopicName
},
write
).
%%-----------------------------------------------------------------------------
next_topic_id(Tab, PredefId, ClientId) ->

16
apps/emqx_gateway/test/emqx_coap_SUITE.erl
View File

@ -255,37 +255,37 @@ t_clients_api(_) ->
Fun = fun(_Channel, _Token) ->
ClientId = <<"client1">>,
%% list
{200, #{data := [Client1]}} = request(get, "/gateway/coap/clients"),
{200, #{data := [Client1]}} = request(get, "/gateways/coap/clients"),
#{clientid := ClientId} = Client1,
%% searching
{200, #{data := [Client2]}} =
request(
get,
"/gateway/coap/clients",
"/gateways/coap/clients",
[{<<"clientid">>, ClientId}]
),
{200, #{data := [Client3]}} =
request(
get,
"/gateway/coap/clients",
"/gateways/coap/clients",
[{<<"like_clientid">>, <<"cli">>}]
),
%% lookup
{200, Client4} =
request(get, "/gateway/coap/clients/client1"),
request(get, "/gateways/coap/clients/client1"),
%% assert
Client1 = Client2 = Client3 = Client4,
%% kickout
{204, _} =
request(delete, "/gateway/coap/clients/client1"),
request(delete, "/gateways/coap/clients/client1"),
timer:sleep(200),
{200, #{data := []}} = request(get, "/gateway/coap/clients")
{200, #{data := []}} = request(get, "/gateways/coap/clients")
end,
with_connection(Fun).
t_clients_subscription_api(_) ->
Fun = fun(_Channel, _Token) ->
Path = "/gateway/coap/clients/client1/subscriptions",
Path = "/gateways/coap/clients/client1/subscriptions",
%% list
{200, []} = request(get, Path),
%% create
@ -312,7 +312,7 @@ t_clients_subscription_api(_) ->
t_clients_get_subscription_api(_) ->
Fun = fun(Channel, Token) ->
Path = "/gateway/coap/clients/client1/subscriptions",
Path = "/gateways/coap/clients/client1/subscriptions",
%% list
{200, []} = request(get, Path),

49
apps/emqx_gateway/test/emqx_coap_api_SUITE.erl
View File

@ -71,28 +71,33 @@ end_per_suite(Config) ->
t_send_request_api(_) ->
ClientId = start_client(),
timer:sleep(200),
Path = emqx_mgmt_api_test_util:api_path(["gateway/coap/clients/client1/request"]),
Token = <<"atoken">>,
Payload = <<"simple echo this">>,
Req = #{
token => Token,
payload => Payload,
timeout => <<"10s">>,
content_type => <<"text/plain">>,
method => <<"get">>
},
Auth = emqx_mgmt_api_test_util:auth_header_(),
{ok, Response} = emqx_mgmt_api_test_util:request_api(
post,
Path,
"method=get",
Auth,
Req
),
#{<<"token">> := RToken, <<"payload">> := RPayload} =
emqx_json:decode(Response, [return_maps]),
?assertEqual(Token, RToken),
?assertEqual(Payload, RPayload),
Test = fun(API) ->
Path = emqx_mgmt_api_test_util:api_path([API]),
Token = <<"atoken">>,
Payload = <<"simple echo this">>,
Req = #{
token => Token,
payload => Payload,
timeout => <<"10s">>,
content_type => <<"text/plain">>,
method => <<"get">>
},
Auth = emqx_mgmt_api_test_util:auth_header_(),
{ok, Response} = emqx_mgmt_api_test_util:request_api(
post,
Path,
"method=get",
Auth,
Req
),
#{<<"token">> := RToken, <<"payload">> := RPayload} =
emqx_json:decode(Response, [return_maps]),
?assertEqual(Token, RToken),
?assertEqual(Payload, RPayload)
end,
Test("gateways/coap/clients/client1/request"),
timer:sleep(100),
Test("gateway/coap/clients/client1/request"),
erlang:exit(ClientId, kill),
ok.

4
apps/emqx_gateway/test/emqx_exproto_SUITE.erl
View File

@ -483,8 +483,8 @@ ssl_opts() ->
maps:merge(
Certs,
#{
versions => emqx_tls_lib:default_versions(),
ciphers => emqx_tls_lib:default_ciphers(),
versions => emqx_tls_lib:available_versions(tls),
ciphers => [],
verify => verify_peer,
fail_if_no_peer_cert => true,
secure_renegotiate => false,

194
apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl
View File

@ -60,6 +60,22 @@ end_per_suite(Conf) ->
%%--------------------------------------------------------------------
t_gateway(_) ->
{200, Gateways} = request(get, "/gateways"),
lists:foreach(fun assert_gw_unloaded/1, Gateways),
{400, BadReq} = request(get, "/gateways/uname_gateway"),
assert_bad_request(BadReq),
{201, _} = request(post, "/gateways", #{name => <<"stomp">>}),
{200, StompGw1} = request(get, "/gateways/stomp"),
assert_feilds_apperence(
[name, status, enable, created_at, started_at],
StompGw1
),
{204, _} = request(delete, "/gateways/stomp"),
{200, StompGw2} = request(get, "/gateways/stomp"),
assert_gw_unloaded(StompGw2),
ok.
t_deprecated_gateway(_) ->
{200, Gateways} = request(get, "/gateway"),
lists:foreach(fun assert_gw_unloaded/1, Gateways),
{400, BadReq} = request(get, "/gateway/uname_gateway"),
@ -76,7 +92,7 @@ t_gateway(_) ->
ok.
t_gateway_stomp(_) ->
{200, Gw} = request(get, "/gateway/stomp"),
{200, Gw} = request(get, "/gateways/stomp"),
assert_gw_unloaded(Gw),
%% post
GwConf = #{
@ -90,18 +106,18 @@ t_gateway_stomp(_) ->
#{name => <<"def">>, type => <<"tcp">>, bind => <<"61613">>}
]
},
{201, _} = request(post, "/gateway", GwConf),
{200, ConfResp} = request(get, "/gateway/stomp"),
{201, _} = request(post, "/gateways", GwConf),
{200, ConfResp} = request(get, "/gateways/stomp"),
assert_confs(GwConf, ConfResp),
%% put
GwConf2 = emqx_map_lib:deep_merge(GwConf, #{frame => #{max_headers => 10}}),
{200, _} = request(put, "/gateway/stomp", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateway/stomp"),
{200, _} = request(put, "/gateways/stomp", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateways/stomp"),
assert_confs(GwConf2, ConfResp2),
{204, _} = request(delete, "/gateway/stomp").
{204, _} = request(delete, "/gateways/stomp").
t_gateway_mqttsn(_) ->
{200, Gw} = request(get, "/gateway/mqttsn"),
{200, Gw} = request(get, "/gateways/mqttsn"),
assert_gw_unloaded(Gw),
%% post
GwConf = #{
@ -114,18 +130,18 @@ t_gateway_mqttsn(_) ->
#{name => <<"def">>, type => <<"udp">>, bind => <<"1884">>}
]
},
{201, _} = request(post, "/gateway", GwConf),
{200, ConfResp} = request(get, "/gateway/mqttsn"),
{201, _} = request(post, "/gateways", GwConf),
{200, ConfResp} = request(get, "/gateways/mqttsn"),
assert_confs(GwConf, ConfResp),
%% put
GwConf2 = emqx_map_lib:deep_merge(GwConf, #{predefined => []}),
{200, _} = request(put, "/gateway/mqttsn", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateway/mqttsn"),
{200, _} = request(put, "/gateways/mqttsn", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateways/mqttsn"),
assert_confs(GwConf2, ConfResp2),
{204, _} = request(delete, "/gateway/mqttsn").
{204, _} = request(delete, "/gateways/mqttsn").
t_gateway_coap(_) ->
{200, Gw} = request(get, "/gateway/coap"),
{200, Gw} = request(get, "/gateways/coap"),
assert_gw_unloaded(Gw),
%% post
GwConf = #{
@ -136,18 +152,18 @@ t_gateway_coap(_) ->
#{name => <<"def">>, type => <<"udp">>, bind => <<"5683">>}
]
},
{201, _} = request(post, "/gateway", GwConf),
{200, ConfResp} = request(get, "/gateway/coap"),
{201, _} = request(post, "/gateways", GwConf),
{200, ConfResp} = request(get, "/gateways/coap"),
assert_confs(GwConf, ConfResp),
%% put
GwConf2 = emqx_map_lib:deep_merge(GwConf, #{heartbeat => <<"10s">>}),
{200, _} = request(put, "/gateway/coap", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateway/coap"),
{200, _} = request(put, "/gateways/coap", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateways/coap"),
assert_confs(GwConf2, ConfResp2),
{204, _} = request(delete, "/gateway/coap").
{204, _} = request(delete, "/gateways/coap").
t_gateway_lwm2m(_) ->
{200, Gw} = request(get, "/gateway/lwm2m"),
{200, Gw} = request(get, "/gateways/lwm2m"),
assert_gw_unloaded(Gw),
%% post
GwConf = #{
@ -168,18 +184,18 @@ t_gateway_lwm2m(_) ->
#{name => <<"def">>, type => <<"udp">>, bind => <<"5783">>}
]
},
{201, _} = request(post, "/gateway", GwConf),
{200, ConfResp} = request(get, "/gateway/lwm2m"),
{201, _} = request(post, "/gateways", GwConf),
{200, ConfResp} = request(get, "/gateways/lwm2m"),
assert_confs(GwConf, ConfResp),
%% put
GwConf2 = emqx_map_lib:deep_merge(GwConf, #{qmode_time_window => <<"10s">>}),
{200, _} = request(put, "/gateway/lwm2m", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateway/lwm2m"),
{200, _} = request(put, "/gateways/lwm2m", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateways/lwm2m"),
assert_confs(GwConf2, ConfResp2),
{204, _} = request(delete, "/gateway/lwm2m").
{204, _} = request(delete, "/gateways/lwm2m").
t_gateway_exproto(_) ->
{200, Gw} = request(get, "/gateway/exproto"),
{200, Gw} = request(get, "/gateways/exproto"),
assert_gw_unloaded(Gw),
%% post
GwConf = #{
@ -190,18 +206,18 @@ t_gateway_exproto(_) ->
#{name => <<"def">>, type => <<"tcp">>, bind => <<"7993">>}
]
},
{201, _} = request(post, "/gateway", GwConf),
{200, ConfResp} = request(get, "/gateway/exproto"),
{201, _} = request(post, "/gateways", GwConf),
{200, ConfResp} = request(get, "/gateways/exproto"),
assert_confs(GwConf, ConfResp),
%% put
GwConf2 = emqx_map_lib:deep_merge(GwConf, #{server => #{bind => <<"9200">>}}),
{200, _} = request(put, "/gateway/exproto", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateway/exproto"),
{200, _} = request(put, "/gateways/exproto", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateways/exproto"),
assert_confs(GwConf2, ConfResp2),
{204, _} = request(delete, "/gateway/exproto").
{204, _} = request(delete, "/gateways/exproto").
t_gateway_exproto_with_ssl(_) ->
{200, Gw} = request(get, "/gateway/exproto"),
{200, Gw} = request(get, "/gateways/exproto"),
assert_gw_unloaded(Gw),
SslSvrOpts = ssl_server_opts(),
@ -221,8 +237,8 @@ t_gateway_exproto_with_ssl(_) ->
#{name => <<"def">>, type => <<"tcp">>, bind => <<"7993">>}
]
},
{201, _} = request(post, "/gateway", GwConf),
{200, ConfResp} = request(get, "/gateway/exproto"),
{201, _} = request(post, "/gateways", GwConf),
{200, ConfResp} = request(get, "/gateways/exproto"),
assert_confs(GwConf, ConfResp),
%% put
GwConf2 = emqx_map_lib:deep_merge(GwConf, #{
@ -231,50 +247,50 @@ t_gateway_exproto_with_ssl(_) ->
ssl_options => SslCliOpts
}
}),
{200, _} = request(put, "/gateway/exproto", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateway/exproto"),
{200, _} = request(put, "/gateways/exproto", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateways/exproto"),
assert_confs(GwConf2, ConfResp2),
{204, _} = request(delete, "/gateway/exproto").
{204, _} = request(delete, "/gateways/exproto").
t_authn(_) ->
GwConf = #{name => <<"stomp">>},
{201, _} = request(post, "/gateway", GwConf),
{201, _} = request(post, "/gateways", GwConf),
ct:sleep(500),
{204, _} = request(get, "/gateway/stomp/authentication"),
{204, _} = request(get, "/gateways/stomp/authentication"),
AuthConf = #{
mechanism => <<"password_based">>,
backend => <<"built_in_database">>,
user_id_type => <<"clientid">>
},
{201, _} = request(post, "/gateway/stomp/authentication", AuthConf),
{200, ConfResp} = request(get, "/gateway/stomp/authentication"),
{201, _} = request(post, "/gateways/stomp/authentication", AuthConf),
{200, ConfResp} = request(get, "/gateways/stomp/authentication"),
assert_confs(AuthConf, ConfResp),
AuthConf2 = maps:merge(AuthConf, #{user_id_type => <<"username">>}),
{200, _} = request(put, "/gateway/stomp/authentication", AuthConf2),
{200, _} = request(put, "/gateways/stomp/authentication", AuthConf2),
{200, ConfResp2} = request(get, "/gateway/stomp/authentication"),
{200, ConfResp2} = request(get, "/gateways/stomp/authentication"),
assert_confs(AuthConf2, ConfResp2),
{204, _} = request(delete, "/gateway/stomp/authentication"),
{204, _} = request(get, "/gateway/stomp/authentication"),
{204, _} = request(delete, "/gateway/stomp").
{204, _} = request(delete, "/gateways/stomp/authentication"),
{204, _} = request(get, "/gateways/stomp/authentication"),
{204, _} = request(delete, "/gateways/stomp").
t_authn_data_mgmt(_) ->
GwConf = #{name => <<"stomp">>},
{201, _} = request(post, "/gateway", GwConf),
{201, _} = request(post, "/gateways", GwConf),
ct:sleep(500),
{204, _} = request(get, "/gateway/stomp/authentication"),
{204, _} = request(get, "/gateways/stomp/authentication"),
AuthConf = #{
mechanism => <<"password_based">>,
backend => <<"built_in_database">>,
user_id_type => <<"clientid">>
},
{201, _} = request(post, "/gateway/stomp/authentication", AuthConf),
{201, _} = request(post, "/gateways/stomp/authentication", AuthConf),
ct:sleep(500),
{200, ConfResp} = request(get, "/gateway/stomp/authentication"),
{200, ConfResp} = request(get, "/gateways/stomp/authentication"),
assert_confs(AuthConf, ConfResp),
User1 = #{
@ -282,19 +298,19 @@ t_authn_data_mgmt(_) ->
password => <<"123456">>,
is_superuser => false
},
{201, _} = request(post, "/gateway/stomp/authentication/users", User1),
{200, #{data := [UserRespd1]}} = request(get, "/gateway/stomp/authentication/users"),
{201, _} = request(post, "/gateways/stomp/authentication/users", User1),
{200, #{data := [UserRespd1]}} = request(get, "/gateways/stomp/authentication/users"),
assert_confs(UserRespd1, User1),
{200, UserRespd2} = request(
get,
"/gateway/stomp/authentication/users/test"
"/gateways/stomp/authentication/users/test"
),
assert_confs(UserRespd2, User1),
{200, UserRespd3} = request(
put,
"/gateway/stomp/authentication/users/test",
"/gateways/stomp/authentication/users/test",
#{
password => <<"654321">>,
is_superuser => true
@ -304,19 +320,19 @@ t_authn_data_mgmt(_) ->
{200, UserRespd4} = request(
get,
"/gateway/stomp/authentication/users/test"
"/gateways/stomp/authentication/users/test"
),
assert_confs(UserRespd4, User1#{is_superuser => true}),
{204, _} = request(delete, "/gateway/stomp/authentication/users/test"),
{204, _} = request(delete, "/gateways/stomp/authentication/users/test"),
{200, #{data := []}} = request(
get,
"/gateway/stomp/authentication/users"
"/gateways/stomp/authentication/users"
),
ImportUri = emqx_dashboard_api_test_helpers:uri(
["gateway", "stomp", "authentication", "import_users"]
["gateways", "stomp", "authentication", "import_users"]
),
Dir = code:lib_dir(emqx_authn, test),
@ -332,38 +348,38 @@ t_authn_data_mgmt(_) ->
{filename, "user-credentials.csv", CSVData}
]),
{204, _} = request(delete, "/gateway/stomp/authentication"),
{204, _} = request(get, "/gateway/stomp/authentication"),
{204, _} = request(delete, "/gateway/stomp").
{204, _} = request(delete, "/gateways/stomp/authentication"),
{204, _} = request(get, "/gateways/stomp/authentication"),
{204, _} = request(delete, "/gateways/stomp").
t_listeners_tcp(_) ->
GwConf = #{name => <<"stomp">>},
{201, _} = request(post, "/gateway", GwConf),
{404, _} = request(get, "/gateway/stomp/listeners"),
{201, _} = request(post, "/gateways", GwConf),
{404, _} = request(get, "/gateways/stomp/listeners"),
LisConf = #{
name => <<"def">>,
type => <<"tcp">>,
bind => <<"127.0.0.1:61613">>
},
{201, _} = request(post, "/gateway/stomp/listeners", LisConf),
{200, ConfResp} = request(get, "/gateway/stomp/listeners"),
{201, _} = request(post, "/gateways/stomp/listeners", LisConf),
{200, ConfResp} = request(get, "/gateways/stomp/listeners"),
assert_confs([LisConf], ConfResp),
{200, ConfResp1} = request(get, "/gateway/stomp/listeners/stomp:tcp:def"),
{200, ConfResp1} = request(get, "/gateways/stomp/listeners/stomp:tcp:def"),
assert_confs(LisConf, ConfResp1),
LisConf2 = maps:merge(LisConf, #{bind => <<"127.0.0.1:61614">>}),
{200, _} = request(
put,
"/gateway/stomp/listeners/stomp:tcp:def",
"/gateways/stomp/listeners/stomp:tcp:def",
LisConf2
),
{200, ConfResp2} = request(get, "/gateway/stomp/listeners/stomp:tcp:def"),
{200, ConfResp2} = request(get, "/gateways/stomp/listeners/stomp:tcp:def"),
assert_confs(LisConf2, ConfResp2),
{204, _} = request(delete, "/gateway/stomp/listeners/stomp:tcp:def"),
{404, _} = request(get, "/gateway/stomp/listeners/stomp:tcp:def"),
{204, _} = request(delete, "/gateway/stomp").
{204, _} = request(delete, "/gateways/stomp/listeners/stomp:tcp:def"),
{404, _} = request(get, "/gateways/stomp/listeners/stomp:tcp:def"),
{204, _} = request(delete, "/gateways/stomp").
t_listeners_authn(_) ->
GwConf = #{
@ -376,9 +392,9 @@ t_listeners_authn(_) ->
}
]
},
{201, _} = request(post, "/gateway", GwConf),
{201, _} = request(post, "/gateways", GwConf),
ct:sleep(500),
{200, ConfResp} = request(get, "/gateway/stomp"),
{200, ConfResp} = request(get, "/gateways/stomp"),
assert_confs(GwConf, ConfResp),
AuthConf = #{
@ -386,7 +402,7 @@ t_listeners_authn(_) ->
backend => <<"built_in_database">>,
user_id_type => <<"clientid">>
},
Path = "/gateway/stomp/listeners/stomp:tcp:def/authentication",
Path = "/gateways/stomp/listeners/stomp:tcp:def/authentication",
{201, _} = request(post, Path, AuthConf),
{200, ConfResp2} = request(get, Path),
assert_confs(AuthConf, ConfResp2),
@ -400,7 +416,7 @@ t_listeners_authn(_) ->
{204, _} = request(delete, Path),
%% FIXME: 204?
{204, _} = request(get, Path),
{204, _} = request(delete, "/gateway/stomp").
{204, _} = request(delete, "/gateways/stomp").
t_listeners_authn_data_mgmt(_) ->
GwConf = #{
@ -413,8 +429,8 @@ t_listeners_authn_data_mgmt(_) ->
}
]
},
{201, _} = request(post, "/gateway", GwConf),
{200, ConfResp} = request(get, "/gateway/stomp"),
{201, _} = request(post, "/gateways", GwConf),
{200, ConfResp} = request(get, "/gateways/stomp"),
assert_confs(GwConf, ConfResp),
AuthConf = #{
@ -422,7 +438,7 @@ t_listeners_authn_data_mgmt(_) ->
backend => <<"built_in_database">>,
user_id_type => <<"clientid">>
},
Path = "/gateway/stomp/listeners/stomp:tcp:def/authentication",
Path = "/gateways/stomp/listeners/stomp:tcp:def/authentication",
{201, _} = request(post, Path, AuthConf),
{200, ConfResp2} = request(get, Path),
assert_confs(AuthConf, ConfResp2),
@ -434,7 +450,7 @@ t_listeners_authn_data_mgmt(_) ->
},
{201, _} = request(
post,
"/gateway/stomp/listeners/stomp:tcp:def/authentication/users",
"/gateways/stomp/listeners/stomp:tcp:def/authentication/users",
User1
),
@ -474,7 +490,7 @@ t_listeners_authn_data_mgmt(_) ->
),
ImportUri = emqx_dashboard_api_test_helpers:uri(
["gateway", "stomp", "listeners", "stomp:tcp:def", "authentication", "import_users"]
["gateways", "stomp", "listeners", "stomp:tcp:def", "authentication", "import_users"]
),
Dir = code:lib_dir(emqx_authn, test),
@ -490,31 +506,31 @@ t_listeners_authn_data_mgmt(_) ->
{filename, "user-credentials.csv", CSVData}
]),
{204, _} = request(delete, "/gateway/stomp").
{204, _} = request(delete, "/gateways/stomp").
t_authn_fuzzy_search(_) ->
GwConf = #{name => <<"stomp">>},
{201, _} = request(post, "/gateway", GwConf),
{204, _} = request(get, "/gateway/stomp/authentication"),
{201, _} = request(post, "/gateways", GwConf),
{204, _} = request(get, "/gateways/stomp/authentication"),
AuthConf = #{
mechanism => <<"password_based">>,
backend => <<"built_in_database">>,
user_id_type => <<"clientid">>
},
{201, _} = request(post, "/gateway/stomp/authentication", AuthConf),
{200, ConfResp} = request(get, "/gateway/stomp/authentication"),
{201, _} = request(post, "/gateways/stomp/authentication", AuthConf),
{200, ConfResp} = request(get, "/gateways/stomp/authentication"),
assert_confs(AuthConf, ConfResp),
Checker = fun({User, Fuzzy}) ->
{200, #{data := [UserRespd]}} = request(
get, "/gateway/stomp/authentication/users", Fuzzy
get, "/gateways/stomp/authentication/users", Fuzzy
),
assert_confs(UserRespd, User)
end,
Create = fun(User) ->
{201, _} = request(post, "/gateway/stomp/authentication/users", User)
{201, _} = request(post, "/gateways/stomp/authentication/users", User)
end,
UserDatas = [
@ -535,9 +551,9 @@ t_authn_fuzzy_search(_) ->
lists:foreach(Create, UserDatas),
lists:foreach(Checker, lists:zip(UserDatas, FuzzyDatas)),
{204, _} = request(delete, "/gateway/stomp/authentication"),
{204, _} = request(get, "/gateway/stomp/authentication"),
{204, _} = request(delete, "/gateway/stomp").
{204, _} = request(delete, "/gateways/stomp/authentication"),
{204, _} = request(get, "/gateways/stomp/authentication"),
{204, _} = request(delete, "/gateways/stomp").
%%--------------------------------------------------------------------
%% Asserts

4
apps/emqx_gateway/test/emqx_gateway_auth_ct.erl
View File

@ -141,7 +141,7 @@ on_start_auth(authn_http) ->
%% set authn for gateway
Setup = fun(Gateway) ->
Path = io_lib:format("/gateway/~ts/authentication", [Gateway]),
Path = io_lib:format("/gateways/~ts/authentication", [Gateway]),
{204, _} = request(delete, Path),
timer:sleep(200),
{201, _} = request(post, Path, http_authn_config()),
@ -198,7 +198,7 @@ on_start_auth(authz_http) ->
on_stop_auth(authn_http) ->
Delete = fun(Gateway) ->
Path = io_lib:format("/gateway/~ts/authentication", [Gateway]),
Path = io_lib:format("/gateways/~ts/authentication", [Gateway]),
{204, _} = request(delete, Path)
end,
lists:foreach(Delete, ?GATEWAYS),

16
apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl
View File

@ -2353,18 +2353,18 @@ case100_clients_api(Config) ->
std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
%% list
{200, #{data := [Client1]}} = request(get, "/gateway/lwm2m/clients"),
{200, #{data := [Client1]}} = request(get, "/gateways/lwm2m/clients"),
%% searching
{200, #{data := [Client2]}} =
request(
get,
"/gateway/lwm2m/clients",
"/gateways/lwm2m/clients",
[{<<"endpoint_name">>, list_to_binary(Epn)}]
),
{200, #{data := [Client3]}} =
request(
get,
"/gateway/lwm2m/clients",
"/gateways/lwm2m/clients",
[
{<<"like_endpoint_name">>, list_to_binary(Epn)},
{<<"gte_lifetime">>, <<"1">>}
@ -2373,14 +2373,14 @@ case100_clients_api(Config) ->
%% lookup
ClientId = maps:get(clientid, Client1),
{200, Client4} =
request(get, "/gateway/lwm2m/clients/" ++ binary_to_list(ClientId)),
request(get, "/gateways/lwm2m/clients/" ++ binary_to_list(ClientId)),
%% assert
Client1 = Client2 = Client3 = Client4,
%% kickout
{204, _} =
request(delete, "/gateway/lwm2m/clients/" ++ binary_to_list(ClientId)),
request(delete, "/gateways/lwm2m/clients/" ++ binary_to_list(ClientId)),
timer:sleep(100),
{200, #{data := []}} = request(get, "/gateway/lwm2m/clients").
{200, #{data := []}} = request(get, "/gateways/lwm2m/clients").
case100_subscription_api(Config) ->
Epn = "urn:oma:lwm2m:oma:3",
@ -2390,10 +2390,10 @@ case100_subscription_api(Config) ->
RespTopic = list_to_binary("lwm2m/" ++ Epn ++ "/up/resp"),
std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic),
{200, #{data := [Client1]}} = request(get, "/gateway/lwm2m/clients"),
{200, #{data := [Client1]}} = request(get, "/gateways/lwm2m/clients"),
ClientId = maps:get(clientid, Client1),
Path =
"/gateway/lwm2m/clients/" ++
"/gateways/lwm2m/clients/" ++
binary_to_list(ClientId) ++
"/subscriptions",

12
apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl
View File

@ -326,7 +326,7 @@ t_observe(Config) ->
test_recv_mqtt_response(RespTopic),
%% step2, call observe API
call_send_api(Epn, "observe", "path=/3/0/1&enable=false"),
call_deprecated_send_api(Epn, "observe", "path=/3/0/1&enable=false"),
timer:sleep(100),
#coap_message{type = Type, method = Method, options = Opts} = test_recv_coap_request(UdpSock),
?assertEqual(con, Type),
@ -338,7 +338,7 @@ t_observe(Config) ->
%%% Internal Functions
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
call_lookup_api(ClientId, Path, Action) ->
ApiPath = emqx_mgmt_api_test_util:api_path(["gateway/lwm2m/clients", ClientId, "lookup"]),
ApiPath = emqx_mgmt_api_test_util:api_path(["gateways/lwm2m/clients", ClientId, "lookup"]),
Auth = emqx_mgmt_api_test_util:auth_header_(),
Query = io_lib:format("path=~ts&action=~ts", [Path, Action]),
{ok, Response} = emqx_mgmt_api_test_util:request_api(get, ApiPath, Query, Auth),
@ -346,7 +346,13 @@ call_lookup_api(ClientId, Path, Action) ->
Response.
call_send_api(ClientId, Cmd, Query) ->
ApiPath = emqx_mgmt_api_test_util:api_path(["gateway/lwm2m/clients", ClientId, Cmd]),
call_send_api(ClientId, Cmd, Query, "gateways/lwm2m/clients").
call_deprecated_send_api(ClientId, Cmd, Query) ->
call_send_api(ClientId, Cmd, Query, "gateway/lwm2m/clients").
call_send_api(ClientId, Cmd, Query, API) ->
ApiPath = emqx_mgmt_api_test_util:api_path([API, ClientId, Cmd]),
Auth = emqx_mgmt_api_test_util:auth_header_(),
{ok, Response} = emqx_mgmt_api_test_util:request_api(post, ApiPath, Query, Auth),
?LOGT("rest api response:~ts~n", [Response]),

14
apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl
View File

@ -2198,15 +2198,15 @@ t_clients_api(_) ->
send_connect_msg(Socket, ClientId),
?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
%% list
{200, #{data := [Client1]}} = request(get, "/gateway/mqttsn/clients"),
{200, #{data := [Client1]}} = request(get, "/gateways/mqttsn/clients"),
#{clientid := ClientId} = Client1,
%% searching
{200, #{data := [Client2]}} =
request(get, "/gateway/mqttsn/clients", [{<<"clientid">>, ClientId}]),
request(get, "/gateways/mqttsn/clients", [{<<"clientid">>, ClientId}]),
{200, #{data := [Client3]}} =
request(
get,
"/gateway/mqttsn/clients",
"/gateways/mqttsn/clients",
[
{<<"like_clientid">>, <<"test1">>},
{<<"proto_ver">>, <<"1.2">>},
@ -2218,21 +2218,21 @@ t_clients_api(_) ->
),
%% lookup
{200, Client4} =
request(get, "/gateway/mqttsn/clients/client_id_test1"),
request(get, "/gateways/mqttsn/clients/client_id_test1"),
%% assert
Client1 = Client2 = Client3 = Client4,
%% kickout
{204, _} =
request(delete, "/gateway/mqttsn/clients/client_id_test1"),
request(delete, "/gateways/mqttsn/clients/client_id_test1"),
timer:sleep(100),
{200, #{data := []}} = request(get, "/gateway/mqttsn/clients"),
{200, #{data := []}} = request(get, "/gateways/mqttsn/clients"),
send_disconnect_msg(Socket, undefined),
gen_udp:close(Socket).
t_clients_subscription_api(_) ->
ClientId = <<"client_id_test1">>,
Path = "/gateway/mqttsn/clients/client_id_test1/subscriptions",
Path = "/gateways/mqttsn/clients/client_id_test1/subscriptions",
{ok, Socket} = gen_udp:open(0, [binary]),
send_connect_msg(Socket, ClientId),
?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),

6
apps/emqx_gateway/test/emqx_stomp_SUITE.erl
View File

@ -721,12 +721,12 @@ t_rest_clienit_info(_) ->
_, _} = parse(Data),
%% client lists
{200, Clients} = request(get, "/gateway/stomp/clients"),
{200, Clients} = request(get, "/gateways/stomp/clients"),
?assertEqual(1, length(maps:get(data, Clients))),
StompClient = lists:nth(1, maps:get(data, Clients)),
ClientId = maps:get(clientid, StompClient),
ClientPath =
"/gateway/stomp/clients/" ++
"/gateways/stomp/clients/" ++
binary_to_list(ClientId),
{200, StompClient1} = request(get, ClientPath),
?assertEqual(StompClient, StompClient1),
@ -811,7 +811,7 @@ t_rest_clienit_info(_) ->
% sync
ignored = gen_server:call(emqx_cm, ignore, infinity),
ok = emqx_pool:flush_async_tasks(),
{200, Clients2} = request(get, "/gateway/stomp/clients"),
{200, Clients2} = request(get, "/gateways/stomp/clients"),
?assertEqual(0, length(maps:get(data, Clients2)))
end).

2
apps/emqx_management/src/emqx_management.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_management, [
{description, "EMQX Management API and CLI"},
% strict semver, bump manually!
{vsn, "5.0.3"},
{vsn, "5.0.4"},
{modules, []},
{registered, [emqx_management_sup]},
{applications, [kernel, stdlib, emqx_plugins, minirest, emqx]},

17
apps/emqx_management/src/emqx_mgmt_api_configs.erl
View File

@ -37,6 +37,7 @@
-define(PREFIX_RESET, "/configs_reset/").
-define(ERR_MSG(MSG), list_to_binary(io_lib:format("~p", [MSG]))).
-define(OPTS, #{rawconf_with_defaults => true, override_to => cluster}).
-define(TAGS, ["Configs"]).
-define(EXCLUDES,
[
@ -85,7 +86,7 @@ schema("/configs") ->
#{
'operationId' => configs,
get => #{
tags => [conf],
tags => ?TAGS,
description =>
<<"Get all the configurations of the specified node, including hot and non-hot updatable items.">>,
parameters => [
@ -111,7 +112,7 @@ schema("/configs_reset/:rootname") ->
#{
'operationId' => config_reset,
post => #{
tags => [conf],
tags => ?TAGS,
description =>
<<
"Reset the config entry specified by the query string parameter `conf_path`.</br>\n"
@ -149,12 +150,12 @@ schema("/configs/global_zone") ->
#{
'operationId' => global_zone_configs,
get => #{
tags => [conf],
tags => ?TAGS,
description => <<"Get the global zone configs">>,
responses => #{200 => Schema}
},
put => #{
tags => [conf],
tags => ?TAGS,
description => <<"Update globbal zone configs">>,
'requestBody' => Schema,
responses => #{
@ -180,7 +181,7 @@ schema("/configs/global_zone") ->
%% #{
%% 'operationId' => config,
%% get => #{
%% tags => [conf],
%% tags => ?TAGS,
%% description => <<"Get config of this limiter">>,
%% parameters => Parameters,
%% responses => #{
@ -189,7 +190,7 @@ schema("/configs/global_zone") ->
%% }
%% },
%% put => #{
%% tags => [conf],
%% tags => ?TAGS,
%% description => <<"Update config of this limiter">>,
%% parameters => Parameters,
%% 'requestBody' => Schema,
@ -204,7 +205,7 @@ schema(Path) ->
#{
'operationId' => config,
get => #{
tags => [conf],
tags => ?TAGS,
description => iolist_to_binary([
<<"Get the sub-configurations under *">>,
RootKey,
@ -216,7 +217,7 @@ schema(Path) ->
}
},
put => #{
tags => [conf],
tags => ?TAGS,
description => iolist_to_binary([
<<"Update the sub-configurations under *">>,
RootKey,

17
apps/emqx_management/src/emqx_mgmt_api_listeners.erl
View File

@ -342,11 +342,18 @@ list_listeners(get, #{query_string := Query}) ->
{200, listener_status_by_id(NodeL)}.
crud_listeners_by_id(get, #{bindings := #{id := Id0}}) ->
Listeners = [
Conf#{<<"id">> => Id, <<"type">> => Type}
|| {Id, Type, Conf} <- emqx_listeners:list_raw(),
Id =:= Id0
],
Listeners =
[
Conf#{
<<"id">> => Id,
<<"type">> => Type,
<<"bind">> := iolist_to_binary(
emqx_listeners:format_bind(maps:get(<<"bind">>, Conf))
)
}
|| {Id, Type, Conf} <- emqx_listeners:list_raw(),
Id =:= Id0
],
case Listeners of
[] -> {404, #{code => 'BAD_LISTENER_ID', message => ?LISTENER_NOT_FOUND}};
[L] -> {200, L}

2
apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
View File

@ -177,6 +177,8 @@ format({_Subscriber, Topic, Options}) ->
maps:with([qos, nl, rap, rh], Options)
).
get_topic(Topic, #{share := <<"$queue">> = Group}) ->
filename:join([Group, Topic]);
get_topic(Topic, #{share := Group}) ->
filename:join([<<"$share">>, Group, Topic]);
get_topic(Topic, _) ->

101
apps/emqx_management/src/emqx_mgmt_auth.erl
View File

@ -30,6 +30,13 @@
-export([authorize/3]).
%% Internal exports (RPC)
-export([
do_update/4,
do_delete/1,
do_create_app/3
]).
-define(APP, emqx_app).
-record(?APP, {
@ -58,40 +65,37 @@ create(Name, Enable, ExpiredAt, Desc) ->
end.
read(Name) ->
Fun = fun() ->
case mnesia:read(?APP, Name) of
[] -> mnesia:abort(not_found);
[App] -> to_map(App)
end
end,
trans(Fun).
case mnesia:dirty_read(?APP, Name) of
[App] -> {ok, to_map(App)};
[] -> {error, not_found}
end.
update(Name, Enable, ExpiredAt, Desc) ->
Fun = fun() ->
case mnesia:read(?APP, Name, write) of
[] ->
mnesia:abort(not_found);
[App0 = #?APP{enable = Enable0, desc = Desc0}] ->
App =
App0#?APP{
expired_at = ExpiredAt,
enable = ensure_not_undefined(Enable, Enable0),
desc = ensure_not_undefined(Desc, Desc0)
},
ok = mnesia:write(App),
to_map(App)
end
end,
trans(Fun).
trans(fun ?MODULE:do_update/4, [Name, Enable, ExpiredAt, Desc]).
do_update(Name, Enable, ExpiredAt, Desc) ->
case mnesia:read(?APP, Name, write) of
[] ->
mnesia:abort(not_found);
[App0 = #?APP{enable = Enable0, desc = Desc0}] ->
App =
App0#?APP{
expired_at = ExpiredAt,
enable = ensure_not_undefined(Enable, Enable0),
desc = ensure_not_undefined(Desc, Desc0)
},
ok = mnesia:write(App),
to_map(App)
end.
delete(Name) ->
Fun = fun() ->
case mnesia:read(?APP, Name) of
[] -> mnesia:abort(not_found);
[_App] -> mnesia:delete({?APP, Name})
end
end,
trans(Fun).
trans(fun ?MODULE:do_delete/1, [Name]).
do_delete(Name) ->
case mnesia:read(?APP, Name) of
[] -> mnesia:abort(not_found);
[_App] -> mnesia:delete({?APP, Name})
end.
list() ->
to_map(ets:match_object(?APP, #?APP{_ = '_'})).
@ -118,8 +122,8 @@ authorize(_Path, ApiKey, ApiSecret) ->
find_by_api_key(ApiKey) ->
Fun = fun() -> mnesia:match_object(#?APP{api_key = ApiKey, _ = '_'}) end,
case trans(Fun) of
{ok, [#?APP{api_secret_hash = SecretHash, enable = Enable, expired_at = ExpiredAt}]} ->
case mria:ro_transaction(?COMMON_SHARD, Fun) of
{atomic, [#?APP{api_secret_hash = SecretHash, enable = Enable, expired_at = ExpiredAt}]} ->
{ok, Enable, ExpiredAt, SecretHash};
_ ->
{error, "not_found"}
@ -163,23 +167,24 @@ create_app(Name, Enable, ExpiredAt, Desc) ->
end.
create_app(App = #?APP{api_key = ApiKey, name = Name}) ->
trans(fun() ->
case mnesia:read(?APP, Name) of
[_] ->
mnesia:abort(name_already_existed);
[] ->
case mnesia:match_object(?APP, #?APP{api_key = ApiKey, _ = '_'}, read) of
[] ->
ok = mnesia:write(App),
to_map(App);
_ ->
mnesia:abort(api_key_already_existed)
end
end
end).
trans(fun ?MODULE:do_create_app/3, [App, ApiKey, Name]).
trans(Fun) ->
case mria:transaction(?COMMON_SHARD, Fun) of
do_create_app(App, ApiKey, Name) ->
case mnesia:read(?APP, Name) of
[_] ->
mnesia:abort(name_already_existed);
[] ->
case mnesia:match_object(?APP, #?APP{api_key = ApiKey, _ = '_'}, read) of
[] ->
ok = mnesia:write(App),
to_map(App);
_ ->
mnesia:abort(api_key_already_existed)
end
end.
trans(Fun, Args) ->
case mria:transaction(?COMMON_SHARD, Fun, Args) of
{atomic, Res} -> {ok, Res};
{aborted, Error} -> {error, Error}
end.

2
apps/emqx_modules/include/emqx_modules.hrl
View File

@ -20,5 +20,5 @@
%% Interval for reporting telemetry data, Default: 7d
-define(REPORT_INTERVAL, 604800).
-define(API_TAG_MQTT, [<<"mqtt">>]).
-define(API_TAG_MQTT, [<<"MQTT">>]).
-define(API_SCHEMA_MODULE, emqx_modules_schema).

2
apps/emqx_modules/src/emqx_modules.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_modules, [
{description, "EMQX Modules"},
{vsn, "5.0.2"},
{vsn, "5.0.3"},
{modules, []},
{applications, [kernel, stdlib, emqx]},
{mod, {emqx_modules_app, []}},

95
apps/emqx_modules/src/emqx_telemetry.erl
View File

@ -54,6 +54,11 @@
-export([official_version/1]).
%% Internal exports (RPC)
-export([
do_ensure_uuids/0
]).
%% internal export
-export([read_raw_build_info/0]).
@ -530,54 +535,56 @@ bin(B) when is_binary(B) ->
B.
ensure_uuids() ->
Txn = fun() ->
NodeUUID =
case mnesia:wread({?TELEMETRY, node()}) of
[] ->
NodeUUID0 =
case get_uuid_from_file(node) of
{ok, NUUID} -> NUUID;
undefined -> generate_uuid()
end,
mnesia:write(
?TELEMETRY,
#telemetry{
id = node(),
uuid = NodeUUID0
},
write
),
NodeUUID0;
[#telemetry{uuid = NodeUUID0}] ->
NodeUUID0
end,
ClusterUUID =
case mnesia:wread({?TELEMETRY, ?CLUSTER_UUID_KEY}) of
[] ->
ClusterUUID0 =
case get_uuid_from_file(cluster) of
{ok, CUUID} -> CUUID;
undefined -> generate_uuid()
end,
mnesia:write(
?TELEMETRY,
#telemetry{
id = ?CLUSTER_UUID_KEY,
uuid = ClusterUUID0
},
write
),
ClusterUUID0;
[#telemetry{uuid = ClusterUUID0}] ->
ClusterUUID0
end,
{NodeUUID, ClusterUUID}
end,
{atomic, {NodeUUID, ClusterUUID}} = mria:transaction(?TELEMETRY_SHARD, Txn),
{atomic, {NodeUUID, ClusterUUID}} = mria:transaction(
?TELEMETRY_SHARD, fun ?MODULE:do_ensure_uuids/0
),
save_uuid_to_file(NodeUUID, node),
save_uuid_to_file(ClusterUUID, cluster),
{NodeUUID, ClusterUUID}.
do_ensure_uuids() ->
NodeUUID =
case mnesia:wread({?TELEMETRY, node()}) of
[] ->
NodeUUID0 =
case get_uuid_from_file(node) of
{ok, NUUID} -> NUUID;
undefined -> generate_uuid()
end,
mnesia:write(
?TELEMETRY,
#telemetry{
id = node(),
uuid = NodeUUID0
},
write
),
NodeUUID0;
[#telemetry{uuid = NodeUUID0}] ->
NodeUUID0
end,
ClusterUUID =
case mnesia:wread({?TELEMETRY, ?CLUSTER_UUID_KEY}) of
[] ->
ClusterUUID0 =
case get_uuid_from_file(cluster) of
{ok, CUUID} -> CUUID;
undefined -> generate_uuid()
end,
mnesia:write(
?TELEMETRY,
#telemetry{
id = ?CLUSTER_UUID_KEY,
uuid = ClusterUUID0
},
write
),
ClusterUUID0;
[#telemetry{uuid = ClusterUUID0}] ->
ClusterUUID0
end,
{NodeUUID, ClusterUUID}.
get_uuid_from_file(Type) ->
Path = uuid_file_path(Type),
case file:read_file(Path) of

10
apps/emqx_plugin_libs/src/emqx_placeholder.erl
View File

@ -39,7 +39,7 @@
sql_data/1
]).
-define(EX_PLACE_HOLDER, "(\\$\\{[a-zA-Z0-9\\._]+\\})").
-define(EX_PLACE_HOLDER, "(\\$\\{[a-zA-Z0-9\\._]+\\}|\"\\$\\{[a-zA-Z0-9\\._]+\\}\")").
%% Space and CRLF
-define(EX_WITHE_CHARS, "\\s").
@ -235,7 +235,9 @@ get_phld_var(Phld, Data) ->
emqx_rule_maps:nested_get(Phld, Data).
preproc_var_re(#{placeholders := PHs}) ->
"(" ++ string:join([ph_to_re(PH) || PH <- PHs], "|") ++ ")";
Res = [ph_to_re(PH) || PH <- PHs],
QuoteRes = ["\"" ++ Re ++ "\"" || Re <- Res],
"(" ++ string:join(Res ++ QuoteRes, "|") ++ ")";
preproc_var_re(#{}) ->
?EX_PLACE_HOLDER.
@ -292,7 +294,9 @@ parse_nested(Attr) ->
end.
unwrap(<<"${", Val/binary>>) ->
binary:part(Val, {0, byte_size(Val) - 1}).
binary:part(Val, {0, byte_size(Val) - 1});
unwrap(<<"\"${", Val/binary>>) ->
binary:part(Val, {0, byte_size(Val) - 2}).
quote_sql(Str) ->
quote(Str, <<"\\\\'">>).

15
apps/emqx_plugin_libs/test/emqx_placeholder_SUITE.erl
View File

@ -150,6 +150,21 @@ t_preproc_sql6(_) ->
emqx_placeholder:proc_sql(ParamsTokens, Selected)
).
t_preproc_sql7(_) ->
Selected = #{a => <<"a">>, b => <<"b">>},
{PrepareStatement, ParamsTokens} = emqx_placeholder:preproc_sql(
<<"a:\"${a}\",b:\"${b}\"">>,
#{
replace_with => '$n',
placeholders => [<<"${a}">>]
}
),
?assertEqual(<<"a:$1,b:\"${b}\"">>, PrepareStatement),
?assertEqual(
[<<"a">>],
emqx_placeholder:proc_sql(ParamsTokens, Selected)
).
t_preproc_tmpl_deep(_) ->
Selected = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}},

2
apps/emqx_psk/src/emqx_psk.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_psk, [
{description, "EMQX PSK"},
% strict semver, bump manually!
{vsn, "5.0.0"},
{vsn, "5.0.1"},
{modules, []},
{registered, [emqx_psk_sup]},
{applications, [kernel, stdlib]},

9
apps/emqx_psk/src/emqx_psk.erl
View File

@ -43,6 +43,11 @@
code_change/3
]).
%% Internal exports (RPC)
-export([
insert_psks/1
]).
-record(psk_entry, {
psk_id :: binary(),
shared_secret :: binary(),
@ -199,10 +204,10 @@ import_psks(SrcFile) ->
import_psks(Io, Delimiter, ChunkSize, NChunk) ->
case get_psks(Io, Delimiter, ChunkSize) of
{ok, Entries} ->
_ = trans(fun insert_psks/1, [Entries]),
_ = trans(fun ?MODULE:insert_psks/1, [Entries]),
import_psks(Io, Delimiter, ChunkSize, NChunk + 1);
{eof, Entries} ->
_ = trans(fun insert_psks/1, [Entries]),
_ = trans(fun ?MODULE:insert_psks/1, [Entries]),
ok;
{error, {bad_format, {line, N}}} ->
{error, {bad_format, {line, NChunk * ChunkSize + N}}};

2
apps/emqx_retainer/src/emqx_retainer.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_retainer, [
{description, "EMQX Retainer"},
% strict semver, bump manually!
{vsn, "5.0.3"},
{vsn, "5.0.4"},
{modules, []},
{registered, [emqx_retainer_sup]},
{applications, [kernel, stdlib, emqx]},

192
apps/emqx_retainer/src/emqx_retainer_mnesia.erl
View File

@ -36,6 +36,15 @@
size/1
]).
%% Internal exports (RPC)
-export([
do_store_retained/1,
do_clear_expired/0,
do_delete_message/1,
do_populate_index_meta/1,
do_reindex_batch/2
]).
%% Management API:
-export([topics/0]).
@ -126,26 +135,8 @@ create_table(Table, RecordName, Attributes, Type, StorageType) ->
ok
end.
store_retained(_, #message{topic = Topic} = Msg) ->
ExpiryTime = emqx_retainer:get_expiry_time(Msg),
Tokens = topic_to_tokens(Topic),
Fun =
case is_table_full() of
false ->
fun() ->
store_retained(db_indices(write), Msg, Tokens, ExpiryTime)
end;
_ ->
fun() ->
case mnesia:read(?TAB_MESSAGE, Tokens, write) of
[_] ->
store_retained(db_indices(write), Msg, Tokens, ExpiryTime);
[] ->
mnesia:abort(table_is_full)
end
end
end,
case mria:transaction(?RETAINER_SHARD, Fun) of
store_retained(_, Msg = #message{topic = Topic}) ->
case mria:transaction(?RETAINER_SHARD, fun ?MODULE:do_store_retained/1, [Msg]) of
{atomic, ok} ->
?tp(debug, message_retained, #{topic => Topic}),
ok;
@ -157,7 +148,26 @@ store_retained(_, #message{topic = Topic} = Msg) ->
})
end.
do_store_retained(#message{topic = Topic} = Msg) ->
ExpiryTime = emqx_retainer:get_expiry_time(Msg),
Tokens = topic_to_tokens(Topic),
case is_table_full() of
false ->
store_retained(db_indices(write), Msg, Tokens, ExpiryTime);
_ ->
case mnesia:read(?TAB_MESSAGE, Tokens, write) of
[_] ->
store_retained(db_indices(write), Msg, Tokens, ExpiryTime);
[] ->
mnesia:abort(table_is_full)
end
end.
clear_expired(_) ->
{atomic, _} = mria:transaction(?RETAINER_SHARD, fun ?MODULE:do_clear_expired/0),
ok.
do_clear_expired() ->
NowMs = erlang:system_time(millisecond),
QH = qlc:q([
TopicTokens
@ -167,36 +177,29 @@ clear_expired(_) ->
} <- mnesia:table(?TAB_MESSAGE, [{lock, write}]),
(ExpiryTime =/= 0) and (ExpiryTime < NowMs)
]),
Fun = fun() ->
QC = qlc:cursor(QH),
clear_batch(db_indices(write), QC)
end,
{atomic, _} = mria:transaction(?RETAINER_SHARD, Fun),
ok.
QC = qlc:cursor(QH),
clear_batch(db_indices(write), QC).
delete_message(_, Topic) ->
Tokens = topic_to_tokens(Topic),
DeleteFun =
case emqx_topic:wildcard(Topic) of
false ->
fun() ->
ok = delete_message_by_topic(Tokens, db_indices(write))
end;
true ->
fun() ->
QH = topic_search_table(Tokens),
qlc:fold(
fun(TopicTokens, _) ->
ok = delete_message_by_topic(TopicTokens, db_indices(write))
end,
undefined,
QH
)
end
end,
{atomic, _} = mria:transaction(?RETAINER_SHARD, DeleteFun),
{atomic, _} = mria:transaction(?RETAINER_SHARD, fun ?MODULE:do_delete_message/1, [Topic]),
ok.
do_delete_message(Topic) ->
Tokens = topic_to_tokens(Topic),
case emqx_topic:wildcard(Topic) of
false ->
ok = delete_message_by_topic(Tokens, db_indices(write));
true ->
QH = topic_search_table(Tokens),
qlc:fold(
fun(TopicTokens, _) ->
ok = delete_message_by_topic(TopicTokens, db_indices(write))
end,
undefined,
QH
)
end.
read_message(_, Topic) ->
{ok, read_messages(Topic)}.
@ -267,16 +270,11 @@ reindex(Force, StatusFun) ->
reindex(config_indices(), Force, StatusFun).
reindex_status() ->
Fun = fun() ->
mnesia:read(?TAB_INDEX_META, ?META_KEY)
end,
case mria:transaction(?RETAINER_SHARD, Fun) of
{atomic, [#retained_index_meta{reindexing = true}]} ->
case mnesia:dirty_read(?TAB_INDEX_META, ?META_KEY) of
[#retained_index_meta{reindexing = true}] ->
true;
{atomic, _} ->
false;
{aborted, Reason} ->
{error, Reason}
_ ->
false
end.
%%--------------------------------------------------------------------
@ -439,37 +437,7 @@ config_indices() ->
populate_index_meta() ->
ConfigIndices = config_indices(),
Fun = fun() ->
case mnesia:read(?TAB_INDEX_META, ?META_KEY, write) of
[
#retained_index_meta{
read_indices = ReadIndices,
write_indices = WriteIndices,
reindexing = Reindexing
}
] ->
case {ReadIndices, WriteIndices, Reindexing} of
{_, _, true} ->
ok;
{ConfigIndices, ConfigIndices, false} ->
ok;
{DBWriteIndices, DBReadIndices, false} ->
{error, DBWriteIndices, DBReadIndices}
end;
[] ->
mnesia:write(
?TAB_INDEX_META,
#retained_index_meta{
key = ?META_KEY,
read_indices = ConfigIndices,
write_indices = ConfigIndices,
reindexing = false
},
write
)
end
end,
case mria:transaction(?RETAINER_SHARD, Fun) of
case mria:transaction(?RETAINER_SHARD, fun ?MODULE:do_populate_index_meta/1, [ConfigIndices]) of
{atomic, ok} ->
ok;
{atomic, {error, DBWriteIndices, DBReadIndices}} ->
@ -488,6 +456,36 @@ populate_index_meta() ->
{error, Reason}
end.
do_populate_index_meta(ConfigIndices) ->
case mnesia:read(?TAB_INDEX_META, ?META_KEY, write) of
[
#retained_index_meta{
read_indices = ReadIndices,
write_indices = WriteIndices,
reindexing = Reindexing
}
] ->
case {ReadIndices, WriteIndices, Reindexing} of
{_, _, true} ->
ok;
{ConfigIndices, ConfigIndices, false} ->
ok;
{DBWriteIndices, DBReadIndices, false} ->
{error, DBWriteIndices, DBReadIndices}
end;
[] ->
mnesia:write(
?TAB_INDEX_META,
#retained_index_meta{
key = ?META_KEY,
read_indices = ConfigIndices,
write_indices = ConfigIndices,
reindexing = false
},
write
)
end.
db_indices(Type) ->
case mnesia:read(?TAB_INDEX_META, ?META_KEY) of
[#retained_index_meta{read_indices = ReadIndices, write_indices = WriteIndices}] ->
@ -533,6 +531,7 @@ reindex(NewIndices, Force, StatusFun) when
end.
try_start_reindex(NewIndices, true) ->
%% Note: we don't expect reindexing during upgrade, so this function is internal
mria:transaction(
?RETAINER_SHARD,
fun() -> start_reindex(NewIndices) end
@ -566,6 +565,7 @@ start_reindex(NewIndices) ->
).
finalize_reindex() ->
%% Note: we don't expect reindexing during upgrade, so this function is internal
{atomic, ok} = mria:transaction(
?RETAINER_SHARD,
fun() ->
@ -601,16 +601,7 @@ reindex_topic(Indices, Topic) ->
end.
reindex_batch(QC, Done, StatusFun) ->
Fun = fun() ->
Indices = db_indices(write),
{Status, Topics} = qlc_next_answers(QC, ?REINDEX_BATCH_SIZE),
ok = lists:foreach(
fun(Topic) -> reindex_topic(Indices, Topic) end,
Topics
),
{Status, Done + length(Topics)}
end,
case mria:transaction(?RETAINER_SHARD, Fun) of
case mria:transaction(?RETAINER_SHARD, fun ?MODULE:do_reindex_batch/2, [QC, Done]) of
{atomic, {more, NewDone}} ->
_ = StatusFun(NewDone),
reindex_batch(QC, NewDone, StatusFun);
@ -625,6 +616,15 @@ reindex_batch(QC, Done, StatusFun) ->
{error, Reason}
end.
do_reindex_batch(QC, Done) ->
Indices = db_indices(write),
{Status, Topics} = qlc_next_answers(QC, ?REINDEX_BATCH_SIZE),
ok = lists:foreach(
fun(Topic) -> reindex_topic(Indices, Topic) end,
Topics
),
{Status, Done + length(Topics)}.
wait_dispatch_complete(Timeout) ->
Nodes = mria_mnesia:running_nodes(),
{Results, []} = emqx_retainer_proto_v1:wait_dispatch_complete(Nodes, Timeout),

4
apps/emqx_retainer/src/emqx_retainer_mnesia_cli.erl
View File

@ -45,9 +45,7 @@ retainer(["reindex", "status"]) ->
true ->
?PRINT_MSG("Reindexing is in progress~n");
false ->
?PRINT_MSG("Reindexing is not running~n");
{error, Reason} ->
?PRINT("Can't get reindex status: ~p~n", [Reason])
?PRINT_MSG("Reindexing is not running~n")
end;
retainer(["reindex", "start"]) ->
retainer(["reindex", "start", "false"]);

77
lib-ee/emqx_license/i18n/emqx_license_schema_i18n.conf Normal file
View File

@ -0,0 +1,77 @@
emqx_license_schema {
license_root {
desc {
en: "Defines the EMQX Enterprise license. \n\n"
"A license is either a `key` or a `file`.\n"
"When `key` and `file` are both configured, `key` is used.\n"
"\n"
"EMQX comes with a default trial license. For production use, please \n"
"visit https://www.emqx.com/apply-licenses/emqx to apply."
zh: "EMQX企业许可证。\n"
"许可证是一个 `key` 或一个 `file`。\n"
"当 `key` 和 `file` 同时被配置时，优先使用 `key`。\n"
"\n"
"EMQX 自带一个默认的试用许可证，若需要在生产环境部署，\n"
"请访问 https://www.emqx.com/apply-licenses/emqx 来申请。\n"
}
label {
en: "License"
zh: "许可证"
}
}
license_type_field {
desc {
en: "License type"
zh: "许可证类型"
}
label {
en: "License type"
zh: "许可证类型"
}
}
key_field {
desc {
en: "License string"
zh: "许可证字符串"
}
label {
en: "License string"
zh: "许可证字符串"
}
}
file_field {
desc {
en: "Path to the license file"
zh: "许可证文件的路径"
}
label {
en: "Path to the license file"
zh: "许可证文件的路径"
}
}
connection_low_watermark_field {
desc {
en: "Low watermark limit below which license connection quota usage alarms are deactivated"
zh: "低水位限制，低于此水位线时系统会清除连接配额使用告警"
}
label {
en: "Connection low watermark"
zh: "连接低水位线"
}
}
connection_high_watermark_field {
desc {
en: "High watermark limit above which license connection quota usage alarms are activated"
zh: "高水位线，连接数超过这个水位线时，系统会触发许可证连接配额使用告警"
}
label {
en: "Connection high watermark"
zh: "连接高水位"
}
}
}

2
lib-ee/emqx_license/src/emqx_license.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_license, [
{description, "EMQX License"},
{vsn, "5.0.1"},
{vsn, "5.0.2"},
{modules, []},
{registered, [emqx_license_sup]},
{applications, [kernel, stdlib]},

29
lib-ee/emqx_license/src/emqx_license_schema.erl
View File

@ -5,6 +5,7 @@
-module(emqx_license_schema).
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
%%------------------------------------------------------------------------------
%% hocon_schema callbacks
@ -26,13 +27,7 @@ roots() ->
hoconsc:mk(
license_type(),
#{
desc =>
"EMQX Enterprise license.\n"
"A license is either a `key` or a `file`.\n"
"When `key` and `file` are both configured, `key` is used.\n"
"\n"
"EMQX by default starts with a trial license. For a different license,\n"
"visit https://www.emqx.com/apply-licenses/emqx to apply.\n"
desc => ?DESC(license_root)
}
)}
].
@ -41,18 +36,20 @@ fields(key_license) ->
[
{type, #{
type => key,
required => true
required => true,
desc => ?DESC(license_type_field)
}},
{key, #{
type => string(),
%% so it's not logged
sensitive => true,
required => true,
desc => "License string"
desc => ?DESC(key_field)
}},
{file, #{
type => string(),
required => false
required => false,
desc => ?DESC(file_field)
}}
| common_fields()
];
@ -60,17 +57,19 @@ fields(file_license) ->
[
{type, #{
type => file,
required => true
required => true,
desc => ?DESC(license_type_field)
}},
{key, #{
type => string(),
%% so it's not logged
sensitive => true,
required => false
required => false,
desc => ?DESC(key_field)
}},
{file, #{
type => string(),
desc => "Path to the license file"
desc => ?DESC(file_field)
}}
| common_fields()
].
@ -87,12 +86,12 @@ common_fields() ->
{connection_low_watermark, #{
type => emqx_schema:percent(),
default => "75%",
desc => ""
desc => ?DESC(connection_low_watermark_field)
}},
{connection_high_watermark, #{
type => emqx_schema:percent(),
default => "80%",
desc => ""
desc => ?DESC(connection_high_watermark_field)
}}
].

4
mix.exs
View File

@ -52,10 +52,10 @@ defmodule EMQXUmbrella.MixProject do
{:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
{:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true},
{:esockd, github: "emqx/esockd", tag: "5.9.4", override: true},
{:ekka, github: "emqx/ekka", tag: "0.13.3", override: true},
{:ekka, github: "emqx/ekka", tag: "0.13.4", override: true},
{:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true},
{:grpc, github: "emqx/grpc-erl", tag: "0.6.6", override: true},
{:minirest, github: "emqx/minirest", tag: "1.3.6", override: true},
{:minirest, github: "emqx/minirest", tag: "1.3.7", override: true},
{:ecpool, github: "emqx/ecpool", tag: "0.5.2", override: true},
{:replayq, "0.3.4", override: true},
{:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},

4
rebar.config
View File

@ -54,10 +54,10 @@
, {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.3"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.4"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}
, {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.6"}}}
, {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.6"}}}
, {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.7"}}}
, {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}}
, {replayq, "0.3.4"}
, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}

73
rel/emqx_conf.template.en.md
View File

@ -233,3 +233,76 @@ authentication=[{enable=true, backend="built_in_database", mechanism="password_b
authentication=[{enable=true}]
```
:::
#### TLS/SSL ciphers
Starting from v5.0.6, EMQX no longer pre-populate the ciphers list with a default
set of cipher suite names.
Instead, the default ciphers are applyed at runtime when starting the listener
for servers, or when establishing a TLS connection as a client.
Below are the default ciphers selected by EMQX.
For tlsv1.3:
```
ciphers =
[ "TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256",
"TLS_CHACHA20_POLY1305_SHA256", "TLS_AES_128_CCM_SHA256",
"TLS_AES_128_CCM_8_SHA256"
]
```
For tlsv1.2 or earlier
```
ciphers =
[ "ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES256-SHA384",
"ECDHE-RSA-AES256-SHA384",
"ECDH-ECDSA-AES256-GCM-SHA384",
"ECDH-RSA-AES256-GCM-SHA384",
"ECDH-ECDSA-AES256-SHA384",
"ECDH-RSA-AES256-SHA384",
"DHE-DSS-AES256-GCM-SHA384",
"DHE-DSS-AES256-SHA256",
"AES256-GCM-SHA384",
"AES256-SHA256",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-AES128-SHA256",
"ECDHE-RSA-AES128-SHA256",
"ECDH-ECDSA-AES128-GCM-SHA256",
"ECDH-RSA-AES128-GCM-SHA256",
"ECDH-ECDSA-AES128-SHA256",
"ECDH-RSA-AES128-SHA256",
"DHE-DSS-AES128-GCM-SHA256",
"DHE-DSS-AES128-SHA256",
"AES128-GCM-SHA256",
"AES128-SHA256",
"ECDHE-ECDSA-AES256-SHA",
"ECDHE-RSA-AES256-SHA",
"DHE-DSS-AES256-SHA",
"ECDH-ECDSA-AES256-SHA",
"ECDH-RSA-AES256-SHA",
"ECDHE-ECDSA-AES128-SHA",
"ECDHE-RSA-AES128-SHA",
"DHE-DSS-AES128-SHA",
"ECDH-ECDSA-AES128-SHA",
"ECDH-RSA-AES128-SHA"
]
```
For PSK enabled listeners
```
ciphers =
[ "RSA-PSK-AES256-GCM-SHA384",
"RSA-PSK-AES256-CBC-SHA384",
"RSA-PSK-AES128-GCM-SHA256",
"RSA-PSK-AES128-CBC-SHA256",
"RSA-PSK-AES256-CBC-SHA",
"RSA-PSK-AES128-CBC-SHA"
]
```

70
rel/emqx_conf.template.zh.md
View File

@ -216,3 +216,73 @@ authentication=[{enable=true, backend="built_in_database", mechanism="password_b
authentication=[{enable=true}]
```
:::
#### TLS/SSL ciphers
从 v5.0.6 开始 EMQX 不在配置文件中详细列出所有默认的密码套件名称。
而是在配置文件中使用一个空列表，然后在运行时替换成默认的密码套件。
下面这些密码套件是 EMQX 默认支持的：
tlsv1.3:
```
ciphers =
[ "TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256",
"TLS_CHACHA20_POLY1305_SHA256", "TLS_AES_128_CCM_SHA256",
"TLS_AES_128_CCM_8_SHA256"
]
```
tlsv1.2 或更早
```
ciphers =
[ "ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES256-SHA384",
"ECDHE-RSA-AES256-SHA384",
"ECDH-ECDSA-AES256-GCM-SHA384",
"ECDH-RSA-AES256-GCM-SHA384",
"ECDH-ECDSA-AES256-SHA384",
"ECDH-RSA-AES256-SHA384",
"DHE-DSS-AES256-GCM-SHA384",
"DHE-DSS-AES256-SHA256",
"AES256-GCM-SHA384",
"AES256-SHA256",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-AES128-SHA256",
"ECDHE-RSA-AES128-SHA256",
"ECDH-ECDSA-AES128-GCM-SHA256",
"ECDH-RSA-AES128-GCM-SHA256",
"ECDH-ECDSA-AES128-SHA256",
"ECDH-RSA-AES128-SHA256",
"DHE-DSS-AES128-GCM-SHA256",
"DHE-DSS-AES128-SHA256",
"AES128-GCM-SHA256",
"AES128-SHA256",
"ECDHE-ECDSA-AES256-SHA",
"ECDHE-RSA-AES256-SHA",
"DHE-DSS-AES256-SHA",
"ECDH-ECDSA-AES256-SHA",
"ECDH-RSA-AES256-SHA",
"ECDHE-ECDSA-AES128-SHA",
"ECDHE-RSA-AES128-SHA",
"DHE-DSS-AES128-SHA",
"ECDH-ECDSA-AES128-SHA",
"ECDH-RSA-AES128-SHA"
]
```
配置 PSK 认证的监听器
```
ciphers = [
[ "RSA-PSK-AES256-GCM-SHA384",
"RSA-PSK-AES256-CBC-SHA384",
"RSA-PSK-AES128-GCM-SHA256",
"RSA-PSK-AES128-CBC-SHA256",
"RSA-PSK-AES256-CBC-SHA",
"RSA-PSK-AES128-CBC-SHA"
]
```