yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into release-51

This commit is contained in:
Zaiming (Stone) Shi 2023-06-08 12:30:02 +02:00
parent b9f1a70214 641166a67a
commit c1cf2365c2
109 changed files with 2547 additions and 763 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -15,8 +15,8 @@ endif
# Dashbord version
# from https://github.com/emqx/emqx-dashboard5
export EMQX_DASHBOARD_VERSION ?= v1.2.5-1
export EMQX_EE_DASHBOARD_VERSION ?= e1.0.8-beta.1
export EMQX_DASHBOARD_VERSION ?= v1.2.6-beta.1
export EMQX_EE_DASHBOARD_VERSION ?= e1.1.0-beta.2
# `:=` should be used here, otherwise the `$(shell ...)` will be executed every time when the variable is used
# In make 4.4+, for backward-compatibility the value from the original environment is used.

10
apps/emqx/src/emqx_cm.hrl → apps/emqx/include/emqx_cm.hrl
View File

@ -13,9 +13,19 @@
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-ifndef(EMQX_CM_HRL).
-define(EMQX_CM_HRL, true).
%% Tables for channel management.
-define(CHAN_TAB, emqx_channel).
-define(CHAN_CONN_TAB, emqx_channel_conn).
-define(CHAN_INFO_TAB, emqx_channel_info).
-define(CHAN_LIVE_TAB, emqx_channel_live).
%% Mria/Mnesia Tables for channel management.
-define(CHAN_REG_TAB, emqx_channel_registry).
-define(T_KICK, 5_000).
-define(T_GET_INFO, 5_000).
-define(T_TAKEOVER, 15_000).

12
apps/emqx/include/emqx_mqtt.hrl
View File

@ -48,6 +48,13 @@
{?MQTT_PROTO_V5, <<"MQTT">>}
]).
%%--------------------------------------------------------------------
%% MQTT Topic and TopitFilter byte length
%%--------------------------------------------------------------------
%% MQTT-3.1.1 and MQTT-5.0 [MQTT-4.7.3-3]
-define(MAX_TOPIC_LEN, 65535).
%%--------------------------------------------------------------------
%% MQTT QoS Levels
%%--------------------------------------------------------------------
@ -662,6 +669,11 @@ end).
end
).
-define(SHARE_EMPTY_FILTER, share_subscription_topic_cannot_be_empty).
-define(SHARE_EMPTY_GROUP, share_subscription_group_name_cannot_be_empty).
-define(SHARE_RECURSIVELY, '$share_cannot_be_used_as_real_topic_filter').
-define(SHARE_NAME_INVALID_CHAR, share_subscription_group_name_cannot_include_wildcard).
-define(FRAME_PARSE_ERROR, frame_parse_error).
-define(FRAME_SERIALIZE_ERROR, frame_serialize_error).
-define(THROW_FRAME_ERROR(Reason), erlang:throw({?FRAME_PARSE_ERROR, Reason})).

4
apps/emqx/include/emqx_release.hrl
View File

@ -31,11 +31,11 @@
%% NOTE: ALso make sure to follow the instructions in end of
%% `apps/emqx/src/bpapi/README.md'
%% Community edition
%% Opensource edition
-define(EMQX_RELEASE_CE, "5.1.0-alpha.3").
%% Enterprise edition
-define(EMQX_RELEASE_EE, "5.1.0-alpha.3").
%% the HTTP API version
%% The HTTP API version
-define(EMQX_API_VERSION, "5.0").

31
apps/emqx/include/emqx_router.hrl Normal file
View File

@ -0,0 +1,31 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-ifndef(EMQX_ROUTER_HRL).
-define(EMQX_ROUTER_HRL, true).
%% ETS table for message routing
-define(ROUTE_TAB, emqx_route).
%% Mnesia table for message routing
-define(ROUTING_NODE, emqx_routing_node).
%% ETS tables for PubSub
-define(SUBOPTION, emqx_suboption).
-define(SUBSCRIBER, emqx_subscriber).
-define(SUBSCRIPTION, emqx_subscription).
-endif.

35
apps/emqx/src/config/emqx_config_zones.erl Normal file
View File

@ -0,0 +1,35 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_config_zones).
-behaviour(emqx_config_handler).
%% API
-export([add_handler/0, remove_handler/0, pre_config_update/3]).
-define(ZONES, [zones]).
add_handler() ->
ok = emqx_config_handler:add_handler(?ZONES, ?MODULE),
ok.
remove_handler() ->
ok = emqx_config_handler:remove_handler(?ZONES),
ok.
%% replace the old config with the new config
pre_config_update(?ZONES, NewRaw, _OldRaw) ->
{ok, NewRaw}.

11
apps/emqx/src/emqx_broker.erl
View File

@ -19,6 +19,8 @@
-behaviour(gen_server).
-include("emqx.hrl").
-include("emqx_router.hrl").
-include("logger.hrl").
-include("types.hrl").
-include("emqx_mqtt.hrl").
@ -80,11 +82,6 @@
-define(BROKER, ?MODULE).
%% ETS tables for PubSub
-define(SUBOPTION, emqx_suboption).
-define(SUBSCRIBER, emqx_subscriber).
-define(SUBSCRIPTION, emqx_subscription).
%% Guards
-define(IS_SUBID(Id), (is_binary(Id) orelse is_atom(Id))).
@ -106,10 +103,10 @@ start_link(Pool, Id) ->
create_tabs() ->
TabOpts = [public, {read_concurrency, true}, {write_concurrency, true}],
%% SubOption: {Topic, SubPid} -> SubOption
%% SubOption: {TopicFilter, SubPid} -> SubOption
ok = emqx_utils_ets:new(?SUBOPTION, [ordered_set | TabOpts]),
%% Subscription: SubPid -> Topic1, Topic2, Topic3, ...
%% Subscription: SubPid -> TopicFilter1, TopicFilter2, TopicFilter3, ...
%% duplicate_bag: o(1) insert
ok = emqx_utils_ets:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]),

27
apps/emqx/src/emqx_channel.erl
View File

@ -484,13 +484,13 @@ handle_in(
{ok, Channel}
end;
handle_in(
Packet = ?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters),
SubPkt = ?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters),
Channel = #channel{clientinfo = ClientInfo}
) ->
case emqx_packet:check(Packet) of
case emqx_packet:check(SubPkt) of
ok ->
TopicFilters0 = parse_topic_filters(TopicFilters),
TopicFilters1 = put_subid_in_subopts(Properties, TopicFilters0),
TopicFilters1 = enrich_subopts_subid(Properties, TopicFilters0),
TupleTopicFilters0 = check_sub_authzs(TopicFilters1, Channel),
HasAuthzDeny = lists:any(
fun({_TopicFilter, ReasonCode}) ->
@ -503,7 +503,10 @@ handle_in(
true ->
handle_out(disconnect, ?RC_NOT_AUTHORIZED, Channel);
false ->
TopicFilters2 = [TopicFilter || {TopicFilter, 0} <- TupleTopicFilters0],
TopicFilters2 = [
TopicFilter
|| {TopicFilter, ?RC_SUCCESS} <- TupleTopicFilters0
],
TopicFilters3 = run_hooks(
'client.subscribe',
[ClientInfo, Properties],
@ -1632,10 +1635,9 @@ check_banned(_ConnPkt, #channel{clientinfo = ClientInfo}) ->
%%--------------------------------------------------------------------
%% Flapping
count_flapping_event(_ConnPkt, Channel = #channel{clientinfo = ClientInfo = #{zone := Zone}}) ->
is_integer(emqx_config:get_zone_conf(Zone, [flapping_detect, window_time])) andalso
emqx_flapping:detect(ClientInfo),
{ok, Channel}.
count_flapping_event(_ConnPkt, #channel{clientinfo = ClientInfo}) ->
_ = emqx_flapping:detect(ClientInfo),
ok.
%%--------------------------------------------------------------------
%% Authenticate
@ -1866,6 +1868,9 @@ check_pub_caps(
%%--------------------------------------------------------------------
%% Check Sub Authorization
%% TODO: not only check topic filter. Qos chould be checked too.
%% Not implemented yet:
%% MQTT-3.1.1 [MQTT-3.8.4-6] and MQTT-5.0 [MQTT-3.8.4-7]
check_sub_authzs(TopicFilters, Channel) ->
check_sub_authzs(TopicFilters, Channel, []).
@ -1876,7 +1881,7 @@ check_sub_authzs(
) ->
case emqx_access_control:authorize(ClientInfo, subscribe, Topic) of
allow ->
check_sub_authzs(More, Channel, [{TopicFilter, 0} | Acc]);
check_sub_authzs(More, Channel, [{TopicFilter, ?RC_SUCCESS} | Acc]);
deny ->
check_sub_authzs(More, Channel, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc])
end;
@ -1892,9 +1897,9 @@ check_sub_caps(TopicFilter, SubOpts, #channel{clientinfo = ClientInfo}) ->
%%--------------------------------------------------------------------
%% Enrich SubId
put_subid_in_subopts(#{'Subscription-Identifier' := SubId}, TopicFilters) ->
enrich_subopts_subid(#{'Subscription-Identifier' := SubId}, TopicFilters) ->
[{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters];
put_subid_in_subopts(_Properties, TopicFilters) ->
enrich_subopts_subid(_Properties, TopicFilters) ->
TopicFilters.
%%--------------------------------------------------------------------

15
apps/emqx/src/emqx_cm.erl
View File

@ -20,6 +20,7 @@
-behaviour(gen_server).
-include("emqx.hrl").
-include("emqx_cm.hrl").
-include("logger.hrl").
-include("types.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
@ -118,14 +119,6 @@
_Stats :: emqx_types:stats()
}.
-include("emqx_cm.hrl").
%% Tables for channel management.
-define(CHAN_TAB, emqx_channel).
-define(CHAN_CONN_TAB, emqx_channel_conn).
-define(CHAN_INFO_TAB, emqx_channel_info).
-define(CHAN_LIVE_TAB, emqx_channel_live).
-define(CHAN_STATS, [
{?CHAN_TAB, 'channels.count', 'channels.max'},
{?CHAN_TAB, 'sessions.count', 'sessions.max'},
@ -669,12 +662,12 @@ lookup_client({username, Username}) ->
MatchSpec = [
{{'_', #{clientinfo => #{username => '$1'}}, '_'}, [{'=:=', '$1', Username}], ['$_']}
],
ets:select(emqx_channel_info, MatchSpec);
ets:select(?CHAN_INFO_TAB, MatchSpec);
lookup_client({clientid, ClientId}) ->
[
Rec
|| Key <- ets:lookup(emqx_channel, ClientId),
Rec <- ets:lookup(emqx_channel_info, Key)
|| Key <- ets:lookup(?CHAN_TAB, ClientId),
Rec <- ets:lookup(?CHAN_INFO_TAB, Key)
].
%% @private

14
apps/emqx/src/emqx_cm_registry.erl
View File

@ -20,6 +20,7 @@
-behaviour(gen_server).
-include("emqx.hrl").
-include("emqx_cm.hrl").
-include("logger.hrl").
-include("types.hrl").
@ -50,7 +51,6 @@
]).
-define(REGISTRY, ?MODULE).
-define(TAB, emqx_channel_registry).
-define(LOCK, {?MODULE, cleanup_down}).
-record(channel, {chid, pid}).
@ -78,7 +78,7 @@ register_channel(ClientId) when is_binary(ClientId) ->
register_channel({ClientId, self()});
register_channel({ClientId, ChanPid}) when is_binary(ClientId), is_pid(ChanPid) ->
case is_enabled() of
true -> mria:dirty_write(?TAB, record(ClientId, ChanPid));
true -> mria:dirty_write(?CHAN_REG_TAB, record(ClientId, ChanPid));
false -> ok
end.
@ -91,14 +91,14 @@ unregister_channel(ClientId) when is_binary(ClientId) ->
unregister_channel({ClientId, self()});
unregister_channel({ClientId, ChanPid}) when is_binary(ClientId), is_pid(ChanPid) ->
case is_enabled() of
true -> mria:dirty_delete_object(?TAB, record(ClientId, ChanPid));
true -> mria:dirty_delete_object(?CHAN_REG_TAB, record(ClientId, ChanPid));
false -> ok
end.
%% @doc Lookup the global channels.
-spec lookup_channels(emqx_types:clientid()) -> list(pid()).
lookup_channels(ClientId) ->
[ChanPid || #channel{pid = ChanPid} <- mnesia:dirty_read(?TAB, ClientId)].
[ChanPid || #channel{pid = ChanPid} <- mnesia:dirty_read(?CHAN_REG_TAB, ClientId)].
record(ClientId, ChanPid) ->
#channel{chid = ClientId, pid = ChanPid}.
@ -109,7 +109,7 @@ record(ClientId, ChanPid) ->
init([]) ->
mria_config:set_dirty_shard(?CM_SHARD, true),
ok = mria:create_table(?TAB, [
ok = mria:create_table(?CHAN_REG_TAB, [
{type, bag},
{rlog_shard, ?CM_SHARD},
{storage, ram_copies},
@ -166,7 +166,7 @@ cleanup_channels(Node) ->
do_cleanup_channels(Node) ->
Pat = [{#channel{pid = '$1', _ = '_'}, [{'==', {node, '$1'}, Node}], ['$_']}],
lists:foreach(fun delete_channel/1, mnesia:select(?TAB, Pat, write)).
lists:foreach(fun delete_channel/1, mnesia:select(?CHAN_REG_TAB, Pat, write)).
delete_channel(Chan) ->
mnesia:delete_object(?TAB, Chan, write).
mnesia:delete_object(?CHAN_REG_TAB, Chan, write).

124
apps/emqx/src/emqx_config.erl
View File

@ -91,7 +91,7 @@
-export([ensure_atom_conf_path/2]).
-ifdef(TEST).
-export([erase_all/0]).
-export([erase_all/0, backup_and_write/2]).
-endif.
-include("logger.hrl").
@ -105,6 +105,7 @@
-define(LISTENER_CONF_PATH(TYPE, LISTENER, PATH), [listeners, TYPE, LISTENER | PATH]).
-define(CONFIG_NOT_FOUND_MAGIC, '$0tFound').
-define(MAX_KEEP_BACKUP_CONFIGS, 10).
-export_type([
update_request/0,
@ -597,51 +598,104 @@ save_to_config_map(Conf, RawConf) ->
-spec save_to_override_conf(boolean(), raw_config(), update_opts()) -> ok | {error, term()}.
save_to_override_conf(_, undefined, _) ->
ok;
%% TODO: Remove deprecated override conf file when 5.1
save_to_override_conf(true, RawConf, Opts) ->
case deprecated_conf_file(Opts) of
undefined ->
ok;
FileName ->
ok = filelib:ensure_dir(FileName),
case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of
ok ->
ok;
{error, Reason} ->
?SLOG(error, #{
msg => "failed_to_write_override_file",
filename => FileName,
reason => Reason
}),
{error, Reason}
end
backup_and_write(FileName, hocon_pp:do(RawConf, #{}))
end;
save_to_override_conf(false, RawConf, _Opts) ->
case cluster_hocon_file() of
undefined ->
ok;
FileName ->
ok = filelib:ensure_dir(FileName),
case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of
backup_and_write(FileName, hocon_pp:do(RawConf, #{}))
end.
%% @private This is the same human-readable timestamp format as
%% hocon-cli generated app.<time>.config file name.
now_time() ->
Ts = os:system_time(millisecond),
{{Y, M, D}, {HH, MM, SS}} = calendar:system_time_to_local_time(Ts, millisecond),
Res = io_lib:format(
"~0p.~2..0b.~2..0b.~2..0b.~2..0b.~2..0b.~3..0b",
[Y, M, D, HH, MM, SS, Ts rem 1000]
),
lists:flatten(Res).
%% @private Backup the current config to a file with a timestamp suffix and
%% then save the new config to the config file.
backup_and_write(Path, Content) ->
%% this may fail, but we don't care
%% e.g. read-only file system
_ = filelib:ensure_dir(Path),
TmpFile = Path ++ ".tmp",
case file:write_file(TmpFile, Content) of
ok ->
backup_and_replace(Path, TmpFile);
{error, Reason} ->
?SLOG(error, #{
msg => "failed_to_save_conf_file",
hint =>
"The updated cluster config is note saved on this node, please check the file system.",
filename => TmpFile,
reason => Reason
}),
%% e.g. read-only, it's not the end of the world
ok
end.
backup_and_replace(Path, TmpPath) ->
Backup = Path ++ "." ++ now_time() ++ ".bak",
case file:rename(Path, Backup) of
ok ->
ok = file:rename(TmpPath, Path),
ok = prune_backup_files(Path);
{error, enoent} ->
%% not created yet
ok = file:rename(TmpPath, Path);
{error, Reason} ->
?SLOG(warning, #{
msg => "failed_to_backup_conf_file",
filename => Backup,
reason => Reason
}),
ok
end.
prune_backup_files(Path) ->
Files0 = filelib:wildcard(Path ++ ".*"),
Re = "\\.[0-9]{4}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{3}\\.bak$",
Files = lists:filter(fun(F) -> re:run(F, Re) =/= nomatch end, Files0),
Sorted = lists:reverse(lists:sort(Files)),
{_Keeps, Deletes} = lists:split(min(?MAX_KEEP_BACKUP_CONFIGS, length(Sorted)), Sorted),
lists:foreach(
fun(F) ->
case file:delete(F) of
ok ->
ok;
{error, Reason} ->
?SLOG(error, #{
msg => "failed_to_save_conf_file",
filename => FileName,
?SLOG(warning, #{
msg => "failed_to_delete_backup_conf_file",
filename => F,
reason => Reason
}),
{error, Reason}
ok
end
end.
end,
Deletes
).
add_handlers() ->
ok = emqx_config_logger:add_handler(),
ok = emqx_config_zones:add_handler(),
emqx_sys_mon:add_handler(),
ok.
remove_handlers() ->
ok = emqx_config_logger:remove_handler(),
ok = emqx_config_zones:remove_handler(),
emqx_sys_mon:remove_handler(),
ok.
@ -707,7 +761,10 @@ do_put(Type, Putter, [], DeepValue) ->
do_put(Type, Putter, [RootName | KeyPath], DeepValue) ->
OldValue = do_get(Type, [RootName], #{}),
NewValue = do_deep_put(Type, Putter, KeyPath, OldValue, DeepValue),
persistent_term:put(?PERSIS_KEY(Type, RootName), NewValue).
Key = ?PERSIS_KEY(Type, RootName),
persistent_term:put(Key, NewValue),
put_config_post_change_actions(Key, NewValue),
ok.
do_deep_get(?CONF, AtomKeyPath, Map, Default) ->
emqx_utils_maps:deep_get(AtomKeyPath, Map, Default);
@ -828,16 +885,14 @@ merge_with_global_defaults(GlobalDefaults, ZoneVal) ->
NewZoneVal :: map().
maybe_update_zone([zones | T], ZonesValue, Value) ->
%% note, do not write to PT, return *New value* instead
NewZonesValue = emqx_utils_maps:deep_put(T, ZonesValue, Value),
ExistingZoneNames = maps:keys(?MODULE:get([zones], #{})),
%% Update only new zones with global defaults
GLD = zone_global_defaults(),
maps:fold(
fun(ZoneName, ZoneValue, Acc) ->
Acc#{ZoneName := merge_with_global_defaults(GLD, ZoneValue)}
NewZonesValue0 = emqx_utils_maps:deep_put(T, ZonesValue, Value),
NewZonesValue1 = emqx_utils_maps:deep_merge(#{default => GLD}, NewZonesValue0),
maps:map(
fun(_ZoneName, ZoneValue) ->
merge_with_global_defaults(GLD, ZoneValue)
end,
NewZonesValue,
maps:without(ExistingZoneNames, NewZonesValue)
NewZonesValue1
);
maybe_update_zone([RootName | T], RootValue, Value) when is_atom(RootName) ->
NewRootValue = emqx_utils_maps:deep_put(T, RootValue, Value),
@ -911,3 +966,12 @@ rawconf_to_conf(SchemaModule, RawPath, RawValue) ->
),
AtomPath = to_atom_conf_path(RawPath, {raise_error, maybe_update_zone_error}),
emqx_utils_maps:deep_get(AtomPath, RawUserDefinedValues).
%% When the global zone change, the zones is updated with the new global zone.
%% The global zone's keys is too many,
%% so we don't choose to write a global zone change emqx_config_handler callback to hook
put_config_post_change_actions(?PERSIS_KEY(?CONF, zones), _Zones) ->
emqx_flapping:update_config(),
ok;
put_config_post_change_actions(_Key, _NewValue) ->
ok.

49
apps/emqx/src/emqx_connection.erl
View File

@ -49,7 +49,7 @@
-export([
async_set_keepalive/3,
async_set_keepalive/4,
async_set_keepalive/5,
async_set_socket_options/2
]).
@ -273,16 +273,30 @@ stats(#state{
%% NOTE: This API sets TCP socket options, which has nothing to do with
%% the MQTT layer's keepalive (PINGREQ and PINGRESP).
async_set_keepalive(Idle, Interval, Probes) ->
async_set_keepalive(self(), Idle, Interval, Probes).
async_set_keepalive(os:type(), self(), Idle, Interval, Probes).
async_set_keepalive(Pid, Idle, Interval, Probes) ->
async_set_keepalive({unix, linux}, Pid, Idle, Interval, Probes) ->
Options = [
{keepalive, true},
{raw, 6, 4, <<Idle:32/native>>},
{raw, 6, 5, <<Interval:32/native>>},
{raw, 6, 6, <<Probes:32/native>>}
],
async_set_socket_options(Pid, Options).
async_set_socket_options(Pid, Options);
async_set_keepalive({unix, darwin}, Pid, Idle, Interval, Probes) ->
Options = [
{keepalive, true},
{raw, 6, 16#10, <<Idle:32/native>>},
{raw, 6, 16#101, <<Interval:32/native>>},
{raw, 6, 16#102, <<Probes:32/native>>}
],
async_set_socket_options(Pid, Options);
async_set_keepalive(OS, _Pid, _Idle, _Interval, _Probes) ->
?SLOG(warning, #{
msg => "Unsupported operation: set TCP keepalive",
os => OS
}),
ok.
%% @doc Set custom socket options.
%% This API is made async because the call might be originated from
@ -353,6 +367,9 @@ init_state(
false -> disabled
end,
IdleTimeout = emqx_channel:get_mqtt_conf(Zone, idle_timeout),
set_tcp_keepalive(Listener),
IdleTimer = start_timer(IdleTimeout, idle_timeout),
#state{
transport = Transport,
@ -948,8 +965,15 @@ handle_cast(
}
) ->
case Transport:setopts(Socket, Opts) of
ok -> ?tp(info, "custom_socket_options_successfully", #{opts => Opts});
Err -> ?tp(error, "failed_to_set_custom_socket_optionn", #{reason => Err})
ok ->
?tp(debug, "custom_socket_options_successfully", #{opts => Opts});
{error, einval} ->
%% socket is already closed, ignore this error
?tp(debug, "socket already closed", #{reason => socket_already_closed}),
ok;
Err ->
%% other errors
?tp(error, "failed_to_set_custom_socket_option", #{reason => Err})
end,
State;
handle_cast(Req, State) ->
@ -1199,6 +1223,19 @@ inc_counter(Key, Inc) ->
_ = emqx_pd:inc_counter(Key, Inc),
ok.
set_tcp_keepalive({quic, _Listener}) ->
ok;
set_tcp_keepalive({Type, Id}) ->
Conf = emqx_config:get_listener_conf(Type, Id, [tcp_options, keepalive], <<"none">>),
case iolist_to_binary(Conf) of
<<"none">> ->
ok;
Value ->
%% the value is already validated by schema, so we do not validate it again.
{Idle, Interval, Probes} = emqx_schema:parse_tcp_keepalive(Value),
async_set_keepalive(Idle, Interval, Probes)
end.
%%--------------------------------------------------------------------
%% For CT tests
%%--------------------------------------------------------------------

89
apps/emqx/src/emqx_flapping.erl
View File

@ -22,13 +22,13 @@
-include("types.hrl").
-include("logger.hrl").
-export([start_link/0, stop/0]).
-export([start_link/0, update_config/0, stop/0]).
%% API
-export([detect/1]).
-ifdef(TEST).
-export([get_policy/2]).
-export([get_policy/1]).
-endif.
%% gen_server callbacks
@ -59,12 +59,17 @@
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
update_config() ->
gen_server:cast(?MODULE, update_config).
stop() -> gen_server:stop(?MODULE).
%% @doc Detect flapping when a MQTT client disconnected.
-spec detect(emqx_types:clientinfo()) -> boolean().
detect(#{clientid := ClientId, peerhost := PeerHost, zone := Zone}) ->
Policy = #{max_count := Threshold} = get_policy([max_count, window_time, ban_time], Zone),
detect(ClientId, PeerHost, get_policy(Zone)).
detect(ClientId, PeerHost, #{enable := true, max_count := Threshold} = Policy) ->
%% The initial flapping record sets the detect_cnt to 0.
InitVal = #flapping{
clientid = ClientId,
@ -82,24 +87,21 @@ detect(#{clientid := ClientId, peerhost := PeerHost, zone := Zone}) ->
[] ->
false
end
end.
end;
detect(_ClientId, _PeerHost, #{enable := false}) ->
false.
get_policy(Keys, Zone) when is_list(Keys) ->
RootKey = flapping_detect,
Conf = emqx_config:get_zone_conf(Zone, [RootKey]),
lists:foldl(
fun(Key, Acc) ->
case maps:find(Key, Conf) of
{ok, V} -> Acc#{Key => V};
error -> Acc#{Key => emqx_config:get([RootKey, Key])}
end
end,
#{},
Keys
);
get_policy(Key, Zone) ->
#{Key := Conf} = get_policy([Key], Zone),
Conf.
get_policy(Zone) ->
Flapping = [flapping_detect],
case emqx_config:get_zone_conf(Zone, Flapping, undefined) of
undefined ->
%% If zone has be deleted at running time,
%% we don't crash the connection and disable flapping detect.
Policy = emqx_config:get(Flapping),
Policy#{enable => false};
Policy ->
Policy
end.
now_diff(TS) -> erlang:system_time(millisecond) - TS.
@ -115,8 +117,8 @@ init([]) ->
{read_concurrency, true},
{write_concurrency, true}
]),
start_timers(),
{ok, #{}, hibernate}.
Timers = start_timers(),
{ok, Timers, hibernate}.
handle_call(Req, _From, State) ->
?SLOG(error, #{msg => "unexpected_call", call => Req}),
@ -169,17 +171,20 @@ handle_cast(
)
end,
{noreply, State};
handle_cast(update_config, State) ->
NState = update_timer(State),
{noreply, NState};
handle_cast(Msg, State) ->
?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
{noreply, State}.
handle_info({timeout, _TRef, {garbage_collect, Zone}}, State) ->
Timestamp =
erlang:system_time(millisecond) - get_policy(window_time, Zone),
Policy = #{window_time := WindowTime} = get_policy(Zone),
Timestamp = erlang:system_time(millisecond) - WindowTime,
MatchSpec = [{{'_', '_', '_', '$1', '_'}, [{'<', '$1', Timestamp}], [true]}],
ets:select_delete(?FLAPPING_TAB, MatchSpec),
_ = start_timer(Zone),
{noreply, State, hibernate};
Timer = start_timer(Policy, Zone),
{noreply, State#{Zone => Timer}, hibernate};
handle_info(Info, State) ->
?SLOG(error, #{msg => "unexpected_info", info => Info}),
{noreply, State}.
@ -190,18 +195,30 @@ terminate(_Reason, _State) ->
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
start_timer(Zone) ->
case get_policy(window_time, Zone) of
WindowTime when is_integer(WindowTime) ->
emqx_utils:start_timer(WindowTime, {garbage_collect, Zone});
disabled ->
ok
end.
start_timer(#{enable := true, window_time := WindowTime}, Zone) ->
emqx_utils:start_timer(WindowTime, {garbage_collect, Zone});
start_timer(_Policy, _Zone) ->
undefined.
start_timers() ->
maps:foreach(
fun(Zone, _ZoneConf) ->
start_timer(Zone)
maps:map(
fun(ZoneName, #{flapping_detect := FlappingDetect}) ->
start_timer(FlappingDetect, ZoneName)
end,
emqx:get_config([zones], #{})
).
update_timer(Timers) ->
maps:map(
fun(ZoneName, #{flapping_detect := FlappingDetect = #{enable := Enable}}) ->
case maps:get(ZoneName, Timers, undefined) of
undefined ->
start_timer(FlappingDetect, ZoneName);
TRef when Enable -> TRef;
TRef ->
_ = erlang:cancel_timer(TRef),
undefined
end
end,
emqx:get_config([zones], #{})
).

2
apps/emqx/src/emqx_frame.erl
View File

@ -300,6 +300,7 @@ parse_connect2(
ConnPacket = #mqtt_packet_connect{
proto_name = ProtoName,
proto_ver = ProtoVer,
%% For bridge mode, non-standard implementation
is_bridge = (BridgeTag =:= 8),
clean_start = bool(CleanStart),
will_flag = bool(WillFlag),
@ -762,6 +763,7 @@ serialize_variable(
#mqtt_packet_connect{
proto_name = ProtoName,
proto_ver = ProtoVer,
%% For bridge mode, non-standard implementation
is_bridge = IsBridge,
clean_start = CleanStart,
will_flag = WillFlag,

11
apps/emqx/src/emqx_hocon.erl
View File

@ -100,7 +100,16 @@ no_stacktrace(Map) ->
%% it's maybe too much when reporting to the user
-spec compact_errors(any(), Stacktrace :: list()) -> {error, any()}.
compact_errors({SchemaModule, Errors}, Stacktrace) ->
compact_errors(SchemaModule, Errors, Stacktrace).
compact_errors(SchemaModule, Errors, Stacktrace);
compact_errors(ErrorContext0, _Stacktrace) when is_map(ErrorContext0) ->
case ErrorContext0 of
#{exception := #{schema_module := _Mod, message := _Msg} = Detail} ->
Error0 = maps:remove(exception, ErrorContext0),
Error = maps:merge(Error0, Detail),
{error, Error};
_ ->
{error, ErrorContext0}
end.
compact_errors(SchemaModule, [Error0 | More], _Stacktrace) when is_map(Error0) ->
Error1 =

190
apps/emqx/src/emqx_listeners.erl
View File

@ -57,6 +57,7 @@
]).
-export([pre_config_update/3, post_config_update/5]).
-export([create_listener/3, remove_listener/3, update_listener/3]).
-export([format_bind/1]).
@ -65,8 +66,8 @@
-endif.
-type listener_id() :: atom() | binary().
-define(CONF_KEY_PATH, [listeners, '?', '?']).
-define(ROOT_KEY, listeners).
-define(CONF_KEY_PATH, [?ROOT_KEY, '?', '?']).
-define(TYPES_STRING, ["tcp", "ssl", "ws", "wss", "quic"]).
-define(MARK_DEL, ?TOMBSTONE_CONFIG_CHANGE_REQ).
@ -212,7 +213,10 @@ shutdown_count(_, _, _) ->
start() ->
%% The ?MODULE:start/0 will be called by emqx_app when emqx get started,
%% so we install the config handler here.
%% callback when http api request
ok = emqx_config_handler:add_handler(?CONF_KEY_PATH, ?MODULE),
%% callback when reload from config file
ok = emqx_config_handler:add_handler([?ROOT_KEY], ?MODULE),
foreach_listeners(fun start_listener/3).
-spec start_listener(listener_id()) -> ok | {error, term()}.
@ -287,7 +291,8 @@ restart_listener(Type, ListenerName, OldConf, NewConf) ->
stop() ->
%% The ?MODULE:stop/0 will be called by emqx_app when emqx is going to shutdown,
%% so we uninstall the config handler here.
_ = emqx_config_handler:remove_handler(?CONF_KEY_PATH),
ok = emqx_config_handler:remove_handler(?CONF_KEY_PATH),
ok = emqx_config_handler:remove_handler([?ROOT_KEY]),
foreach_listeners(fun stop_listener/3).
-spec stop_listener(listener_id()) -> ok | {error, term()}.
@ -463,48 +468,34 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) ->
end.
%% Update the listeners at runtime
pre_config_update([listeners, Type, Name], {create, NewConf}, V) when
pre_config_update([?ROOT_KEY, Type, Name], {create, NewConf}, V) when
V =:= undefined orelse V =:= ?TOMBSTONE_VALUE
->
CertsDir = certs_dir(Type, Name),
{ok, convert_certs(CertsDir, NewConf)};
pre_config_update([listeners, _Type, _Name], {create, _NewConf}, _RawConf) ->
{ok, convert_certs(Type, Name, NewConf)};
pre_config_update([?ROOT_KEY, _Type, _Name], {create, _NewConf}, _RawConf) ->
{error, already_exist};
pre_config_update([listeners, _Type, _Name], {update, _Request}, undefined) ->
pre_config_update([?ROOT_KEY, _Type, _Name], {update, _Request}, undefined) ->
{error, not_found};
pre_config_update([listeners, Type, Name], {update, Request}, RawConf) ->
NewConfT = emqx_utils_maps:deep_merge(RawConf, Request),
NewConf = ensure_override_limiter_conf(NewConfT, Request),
CertsDir = certs_dir(Type, Name),
{ok, convert_certs(CertsDir, NewConf)};
pre_config_update([listeners, _Type, _Name], {action, _Action, Updated}, RawConf) ->
NewConf = emqx_utils_maps:deep_merge(RawConf, Updated),
{ok, NewConf};
pre_config_update([listeners, _Type, _Name], ?MARK_DEL, _RawConf) ->
pre_config_update([?ROOT_KEY, Type, Name], {update, Request}, RawConf) ->
RawConf1 = emqx_utils_maps:deep_merge(RawConf, Request),
RawConf2 = ensure_override_limiter_conf(RawConf1, Request),
{ok, convert_certs(Type, Name, RawConf2)};
pre_config_update([?ROOT_KEY, _Type, _Name], {action, _Action, Updated}, RawConf) ->
{ok, emqx_utils_maps:deep_merge(RawConf, Updated)};
pre_config_update([?ROOT_KEY, _Type, _Name], ?MARK_DEL, _RawConf) ->
{ok, ?TOMBSTONE_VALUE};
pre_config_update(_Path, _Request, RawConf) ->
{ok, RawConf}.
pre_config_update([?ROOT_KEY], RawConf, RawConf) ->
{ok, RawConf};
pre_config_update([?ROOT_KEY], NewConf, _RawConf) ->
{ok, convert_certs(NewConf)}.
post_config_update([listeners, Type, Name], {create, _Request}, NewConf, undefined, _AppEnvs) ->
start_listener(Type, Name, NewConf);
post_config_update([listeners, Type, Name], {update, _Request}, NewConf, OldConf, _AppEnvs) ->
ok = maybe_unregister_ocsp_stapling_refresh(Type, Name, NewConf),
case NewConf of
#{enabled := true} -> restart_listener(Type, Name, {OldConf, NewConf});
_ -> ok
end;
post_config_update([listeners, Type, Name], Op, _, OldConf, _AppEnvs) when
Op =:= ?MARK_DEL andalso is_map(OldConf)
->
ok = unregister_ocsp_stapling_refresh(Type, Name),
case stop_listener(Type, Name, OldConf) of
ok ->
_ = emqx_authentication:delete_chain(listener_id(Type, Name)),
ok;
Err ->
Err
end;
post_config_update([listeners, Type, Name], {action, _Action, _}, NewConf, OldConf, _AppEnvs) ->
post_config_update([?ROOT_KEY, Type, Name], {create, _Request}, NewConf, undefined, _AppEnvs) ->
create_listener(Type, Name, NewConf);
post_config_update([?ROOT_KEY, Type, Name], {update, _Request}, NewConf, OldConf, _AppEnvs) ->
update_listener(Type, Name, {OldConf, NewConf});
post_config_update([?ROOT_KEY, Type, Name], ?MARK_DEL, _, OldConf = #{}, _AppEnvs) ->
remove_listener(Type, Name, OldConf);
post_config_update([?ROOT_KEY, Type, Name], {action, _Action, _}, NewConf, OldConf, _AppEnvs) ->
#{enabled := NewEnabled} = NewConf,
#{enabled := OldEnabled} = OldConf,
case {NewEnabled, OldEnabled} of
@ -521,9 +512,72 @@ post_config_update([listeners, Type, Name], {action, _Action, _}, NewConf, OldCo
ok = unregister_ocsp_stapling_refresh(Type, Name),
stop_listener(Type, Name, OldConf)
end;
post_config_update([?ROOT_KEY], _Request, OldConf, OldConf, _AppEnvs) ->
ok;
post_config_update([?ROOT_KEY], _Request, NewConf, OldConf, _AppEnvs) ->
#{added := Added, removed := Removed, changed := Changed} = diff_confs(NewConf, OldConf),
Updated = lists:map(fun({{{T, N}, Old}, {_, New}}) -> {{T, N}, {Old, New}} end, Changed),
perform_listener_changes([
{fun ?MODULE:remove_listener/3, Removed},
{fun ?MODULE:update_listener/3, Updated},
{fun ?MODULE:create_listener/3, Added}
]);
post_config_update(_Path, _Request, _NewConf, _OldConf, _AppEnvs) ->
ok.
create_listener(Type, Name, NewConf) ->
Res = start_listener(Type, Name, NewConf),
recreate_authenticators(Res, Type, Name, NewConf).
recreate_authenticators(ok, Type, Name, Conf) ->
Chain = listener_id(Type, Name),
_ = emqx_authentication:delete_chain(Chain),
do_create_authneticators(Chain, maps:get(authentication, Conf, []));
recreate_authenticators(Error, _Type, _Name, _NewConf) ->
Error.
do_create_authneticators(Chain, [AuthN | T]) ->
case emqx_authentication:create_authenticator(Chain, AuthN) of
{ok, _} ->
do_create_authneticators(Chain, T);
Error ->
_ = emqx_authentication:delete_chain(Chain),
Error
end;
do_create_authneticators(_Chain, []) ->
ok.
remove_listener(Type, Name, OldConf) ->
ok = unregister_ocsp_stapling_refresh(Type, Name),
case stop_listener(Type, Name, OldConf) of
ok ->
_ = emqx_authentication:delete_chain(listener_id(Type, Name)),
ok;
Err ->
Err
end.
update_listener(Type, Name, {OldConf, NewConf}) ->
ok = maybe_unregister_ocsp_stapling_refresh(Type, Name, NewConf),
Res = restart_listener(Type, Name, {OldConf, NewConf}),
recreate_authenticators(Res, Type, Name, NewConf).
perform_listener_changes([]) ->
ok;
perform_listener_changes([{Action, ConfL} | Tasks]) ->
case perform_listener_changes(Action, ConfL) of
ok -> perform_listener_changes(Tasks);
{error, Reason} -> {error, Reason}
end.
perform_listener_changes(_Action, []) ->
ok;
perform_listener_changes(Action, [{{Type, Name}, Diff} | MapConf]) ->
case Action(Type, Name, Diff) of
ok -> perform_listener_changes(Action, MapConf);
{error, Reason} -> {error, Reason}
end.
esockd_opts(ListenerId, Type, Opts0) ->
Opts1 = maps:with([acceptors, max_connections, proxy_protocol, proxy_protocol_timeout], Opts0),
Limiter = limiter(Opts0),
@ -699,6 +753,29 @@ del_limiter_bucket(Id, Conf) ->
)
end.
diff_confs(NewConfs, OldConfs) ->
emqx_utils:diff_lists(
flatten_confs(NewConfs),
flatten_confs(OldConfs),
fun({Key, _}) -> Key end
).
flatten_confs(Conf0) ->
lists:flatmap(
fun({Type, Conf}) ->
do_flatten_confs(Type, Conf)
end,
maps:to_list(Conf0)
).
do_flatten_confs(Type, Conf0) ->
FilterFun =
fun
({_Name, ?TOMBSTONE_TYPE}) -> false;
({Name, Conf}) -> {true, {{Type, Name}, Conf}}
end,
lists:filtermap(FilterFun, maps:to_list(Conf0)).
enable_authn(Opts) ->
maps:get(enable_authn, Opts, true).
@ -708,7 +785,7 @@ ssl_opts(Opts) ->
tcp_opts(Opts) ->
maps:to_list(
maps:without(
[active_n],
[active_n, keepalive],
maps:get(tcp_options, Opts, #{})
)
).
@ -760,14 +837,32 @@ parse_bind(#{<<"bind">> := Bind}) ->
certs_dir(Type, Name) ->
iolist_to_binary(filename:join(["listeners", Type, Name])).
convert_certs(CertsDir, Conf) ->
convert_certs(ListenerConf) ->
maps:fold(
fun(Type, Listeners0, Acc) ->
Listeners1 =
maps:fold(
fun(Name, Conf, Acc1) ->
Acc1#{Name => convert_certs(Type, Name, Conf)}
end,
#{},
Listeners0
),
Acc#{Type => Listeners1}
end,
#{},
ListenerConf
).
convert_certs(Type, Name, Conf) ->
CertsDir = certs_dir(Type, Name),
case emqx_tls_lib:ensure_ssl_files(CertsDir, get_ssl_options(Conf)) of
{ok, undefined} ->
Conf;
{ok, SSL} ->
Conf#{<<"ssl_options">> => SSL};
{error, Reason} ->
?SLOG(error, Reason#{msg => "bad_ssl_config"}),
?SLOG(error, Reason#{msg => "bad_ssl_config", type => Type, name => Name}),
throw({bad_ssl_config, Reason})
end.
@ -780,13 +875,15 @@ ensure_override_limiter_conf(Conf, #{<<"limiter">> := Limiter}) ->
ensure_override_limiter_conf(Conf, _) ->
Conf.
get_ssl_options(Conf) ->
get_ssl_options(Conf = #{}) ->
case maps:find(ssl_options, Conf) of
{ok, SSL} ->
SSL;
error ->
maps:get(<<"ssl_options">>, Conf, undefined)
end.
end;
get_ssl_options(_) ->
undefined.
%% @doc Get QUIC optional settings for low level tunings.
%% @see quicer:quic_settings()
@ -878,8 +975,5 @@ unregister_ocsp_stapling_refresh(Type, Name) ->
emqx_ocsp_cache:unregister_listener(ListenerId),
ok.
%% There is currently an issue with frontend
%% infinity is not a good value for it, so we use 5m for now
default_max_conn() ->
%% TODO: <<"infinity">>
5_000_000.
<<"infinity">>.

3
apps/emqx/src/emqx_mqueue.erl
View File

@ -75,11 +75,10 @@
-export_type([mqueue/0, options/0]).
-type topic() :: emqx_types:topic().
-type priority() :: infinity | integer().
-type pq() :: emqx_pqueue:q().
-type count() :: non_neg_integer().
-type p_table() :: ?NO_PRIORITY_TABLE | #{topic() := priority()}.
-type p_table() :: ?NO_PRIORITY_TABLE | #{emqx_types:topic() := priority()}.
-type options() :: #{
max_len := count(),
priorities => p_table(),

53
apps/emqx/src/emqx_release.erl
View File

@ -21,7 +21,10 @@
edition_vsn_prefix/0,
edition_longstr/0,
description/0,
version/0
version/0,
version_with_prefix/0,
vsn_compare/1,
vsn_compare/2
]).
-include("emqx_release.hrl").
@ -68,6 +71,10 @@ edition_vsn_prefix() ->
edition_longstr() ->
maps:get(edition(), ?EMQX_REL_NAME).
%% @doc Return the release version with prefix.
version_with_prefix() ->
edition_vsn_prefix() ++ version().
%% @doc Return the release version.
version() ->
case lists:keyfind(emqx_vsn, 1, ?MODULE:module_info(compile)) of
@ -92,3 +99,47 @@ version() ->
build_vsn() ->
maps:get(edition(), ?EMQX_REL_VSNS).
%% @doc Compare the given version with the current running version,
%% return 'newer' 'older' or 'same'.
vsn_compare("v" ++ Vsn) ->
vsn_compare(?EMQX_RELEASE_CE, Vsn);
vsn_compare("e" ++ Vsn) ->
vsn_compare(?EMQX_RELEASE_EE, Vsn).
%% @private Compare the second argument with the first argument, return
%% 'newer' 'older' or 'same' semver comparison result.
vsn_compare(Vsn1, Vsn2) ->
ParsedVsn1 = parse_vsn(Vsn1),
ParsedVsn2 = parse_vsn(Vsn2),
case ParsedVsn1 =:= ParsedVsn2 of
true ->
same;
false when ParsedVsn1 < ParsedVsn2 ->
newer;
false ->
older
end.
%% @private Parse the version string to a tuple.
%% Return {{Major, Minor, Patch}, Suffix}.
%% Where Suffix is either an empty string or a tuple like {"rc", 1}.
%% NOTE: taking the nature ordering of the suffix:
%% {"alpha", _} < {"beta", _} < {"rc", _} < ""
parse_vsn(Vsn) ->
try
[V1, V2, V3 | Suffix0] = string:tokens(Vsn, ".-"),
Suffix =
case Suffix0 of
"" ->
%% For the case like "5.1.0"
"";
[ReleaseStage, Number] ->
%% For the case like "5.1.0-rc.1"
{ReleaseStage, list_to_integer(Number)}
end,
{{list_to_integer(V1), list_to_integer(V2), list_to_integer(V3)}, Suffix}
catch
_:_ ->
erlang:error({invalid_version_string, Vsn})
end.

3
apps/emqx/src/emqx_router.erl
View File

@ -22,6 +22,7 @@
-include("logger.hrl").
-include("types.hrl").
-include_lib("mria/include/mria.hrl").
-include_lib("emqx/include/emqx_router.hrl").
%% Mnesia bootstrap
-export([mnesia/1]).
@ -69,8 +70,6 @@
-type dest() :: node() | {group(), node()}.
-define(ROUTE_TAB, emqx_route).
%%--------------------------------------------------------------------
%% Mnesia bootstrap
%%--------------------------------------------------------------------

9
apps/emqx/src/emqx_router_helper.erl
View File

@ -19,6 +19,7 @@
-behaviour(gen_server).
-include("emqx.hrl").
-include("emqx_router.hrl").
-include("logger.hrl").
-include("types.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
@ -54,8 +55,6 @@
-record(routing_node, {name, const = unused}).
-define(ROUTE, emqx_route).
-define(ROUTING_NODE, emqx_routing_node).
-define(LOCK, {?MODULE, cleanup_routes}).
-dialyzer({nowarn_function, [cleanup_routes/1]}).
@ -185,7 +184,7 @@ code_change(_OldVsn, State, _Extra) ->
%%--------------------------------------------------------------------
stats_fun() ->
case ets:info(?ROUTE, size) of
case ets:info(?ROUTE_TAB, size) of
undefined ->
ok;
Size ->
@ -198,6 +197,6 @@ cleanup_routes(Node) ->
#route{_ = '_', dest = {'_', Node}}
],
[
mnesia:delete_object(?ROUTE, Route, write)
|| Pat <- Patterns, Route <- mnesia:match_object(?ROUTE, Pat, write)
mnesia:delete_object(?ROUTE_TAB, Route, write)
|| Pat <- Patterns, Route <- mnesia:match_object(?ROUTE_TAB, Pat, write)
].

90
apps/emqx/src/emqx_schema.erl
View File

@ -33,6 +33,7 @@
-define(MAX_INT_TIMEOUT_MS, 4294967295).
%% floor(?MAX_INT_TIMEOUT_MS / 1000).
-define(MAX_INT_TIMEOUT_S, 4294967).
-define(DEFAULT_WINDOW_TIME, <<"1m">>).
-type duration() :: integer().
-type duration_s() :: integer().
@ -94,7 +95,10 @@
validate_keepalive_multiplier/1,
non_empty_string/1,
validations/0,
naive_env_interpolation/1
naive_env_interpolation/1,
validate_server_ssl_opts/1,
validate_tcp_keepalive/1,
parse_tcp_keepalive/1
]).
-export([qos/0]).
@ -274,7 +278,10 @@ roots(low) ->
{"flapping_detect",
sc(
ref("flapping_detect"),
#{importance => ?IMPORTANCE_HIDDEN}
#{
importance => ?IMPORTANCE_MEDIUM,
converter => fun flapping_detect_converter/2
}
)},
{"persistent_session_store",
sc(
@ -684,15 +691,14 @@ fields("flapping_detect") ->
boolean(),
#{
default => false,
deprecated => {since, "5.0.23"},
desc => ?DESC(flapping_detect_enable)
}
)},
{"window_time",
sc(
hoconsc:union([disabled, duration()]),
duration(),
#{
default => disabled,
default => ?DEFAULT_WINDOW_TIME,
importance => ?IMPORTANCE_HIGH,
desc => ?DESC(flapping_detect_window_time)
}
@ -958,7 +964,7 @@ fields("mqtt_wss_listener") ->
{"ssl_options",
sc(
ref("listener_wss_opts"),
#{}
#{validator => fun validate_server_ssl_opts/1}
)},
{"websocket",
sc(
@ -1388,6 +1394,15 @@ fields("tcp_opts") ->
default => true,
desc => ?DESC(fields_tcp_opts_reuseaddr)
}
)},
{"keepalive",
sc(
string(),
#{
default => <<"none">>,
desc => ?DESC(fields_tcp_opts_keepalive),
validator => fun validate_tcp_keepalive/1
}
)}
];
fields("listener_ssl_opts") ->
@ -2426,8 +2441,21 @@ server_ssl_opts_schema(Defaults, IsRanchListener) ->
]
].
validate_server_ssl_opts(#{<<"fail_if_no_peer_cert">> := true, <<"verify">> := Verify}) ->
validate_verify(Verify);
validate_server_ssl_opts(#{fail_if_no_peer_cert := true, verify := Verify}) ->
validate_verify(Verify);
validate_server_ssl_opts(_SSLOpts) ->
ok.
validate_verify(verify_peer) ->
ok;
validate_verify(_) ->
{error, "verify must be verify_peer when fail_if_no_peer_cert is true"}.
mqtt_ssl_listener_ssl_options_validator(Conf) ->
Checks = [
fun validate_server_ssl_opts/1,
fun ocsp_outer_validator/1,
fun crl_outer_validator/1
],
@ -2681,7 +2709,11 @@ do_to_timeout_duration(Str, Fn, Max, Unit) ->
Msg = lists:flatten(
io_lib:format("timeout value too large (max: ~b ~s)", [Max, Unit])
),
throw(Msg)
throw(#{
schema_module => ?MODULE,
message => Msg,
kind => validation_error
})
end;
Err ->
Err
@ -2828,6 +2860,44 @@ validate_alarm_actions(Actions) ->
Error -> {error, Error}
end.
validate_tcp_keepalive(Value) ->
case iolist_to_binary(Value) of
<<"none">> ->
ok;
_ ->
_ = parse_tcp_keepalive(Value),
ok
end.
%% @doc This function is used as value validator and also run-time parser.
parse_tcp_keepalive(Str) ->
try
[Idle, Interval, Probes] = binary:split(iolist_to_binary(Str), <<",">>, [global]),
%% use 10 times the Linux defaults as range limit
IdleInt = parse_ka_int(Idle, "Idle", 1, 7200_0),
IntervalInt = parse_ka_int(Interval, "Interval", 1, 75_0),
ProbesInt = parse_ka_int(Probes, "Probes", 1, 9_0),
{IdleInt, IntervalInt, ProbesInt}
catch
error:_ ->
throw(#{
reason => "Not comma separated positive integers of 'Idle,Interval,Probes' format",
value => Str
})
end.
parse_ka_int(Bin, Name, Min, Max) ->
I = binary_to_integer(string:trim(Bin)),
case I >= Min andalso I =< Max of
true ->
I;
false ->
Msg = io_lib:format("TCP-Keepalive '~s' value must be in the rage of [~p, ~p].", [
Name, Min, Max
]),
throw(#{reason => lists:flatten(Msg), value => I})
end.
user_lookup_fun_tr(Lookup, #{make_serializable := true}) ->
fmt_user_lookup_fun(Lookup);
user_lookup_fun_tr(Lookup, _) ->
@ -3482,3 +3552,9 @@ mqtt_converter(#{<<"keepalive_backoff">> := Backoff} = Mqtt, _Opts) ->
Mqtt#{<<"keepalive_multiplier">> => Backoff * 2};
mqtt_converter(Mqtt, _Opts) ->
Mqtt.
%% For backward compatibility with window_time is disable
flapping_detect_converter(Conf = #{<<"window_time">> := <<"disable">>}, _Opts) ->
Conf#{<<"window_time">> => ?DEFAULT_WINDOW_TIME, <<"enable">> => false};
flapping_detect_converter(Conf, _Opts) ->
Conf.

12
apps/emqx/src/emqx_session_router.erl
View File

@ -57,9 +57,7 @@
code_change/3
]).
-type group() :: binary().
-type dest() :: node() | {group(), node()}.
-type dest() :: node() | {emqx_types:group(), node()}.
-define(ROUTE_RAM_TAB, emqx_session_route_ram).
-define(ROUTE_DISC_TAB, emqx_session_route_disc).
@ -114,7 +112,7 @@ start_link(Pool, Id) ->
%% Route APIs
%%--------------------------------------------------------------------
-spec do_add_route(emqx_topic:topic(), dest()) -> ok | {error, term()}.
-spec do_add_route(emqx_types:topic(), dest()) -> ok | {error, term()}.
do_add_route(Topic, SessionID) when is_binary(Topic) ->
Route = #route{topic = Topic, dest = SessionID},
case lists:member(Route, lookup_routes(Topic)) of
@ -135,7 +133,7 @@ do_add_route(Topic, SessionID) when is_binary(Topic) ->
end.
%% @doc Match routes
-spec match_routes(emqx_topic:topic()) -> [emqx_types:route()].
-spec match_routes(emqx_types:topic()) -> [emqx_types:route()].
match_routes(Topic) when is_binary(Topic) ->
case match_trie(Topic) of
[] -> lookup_routes(Topic);
@ -153,7 +151,7 @@ match_trie(Topic) ->
delete_routes(SessionID, Subscriptions) ->
cast(pick(SessionID), {delete_routes, SessionID, Subscriptions}).
-spec do_delete_route(emqx_topic:topic(), dest()) -> ok | {error, term()}.
-spec do_delete_route(emqx_types:topic(), dest()) -> ok | {error, term()}.
do_delete_route(Topic, SessionID) ->
Route = #route{topic = Topic, dest = SessionID},
case emqx_topic:wildcard(Topic) of
@ -165,7 +163,7 @@ do_delete_route(Topic, SessionID) ->
end.
%% @doc Print routes to a topic
-spec print_routes(emqx_topic:topic()) -> ok.
-spec print_routes(emqx_types:topic()) -> ok.
print_routes(Topic) ->
lists:foreach(
fun(#route{topic = To, dest = SessionID}) ->

4
apps/emqx/src/emqx_shared_sub.erl
View File

@ -97,7 +97,7 @@
-define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}).
-define(SUBSCRIBER_DOWN, noproc).
-type redispatch_to() :: ?REDISPATCH_TO(emqx_topic:group(), emqx_topic:topic()).
-type redispatch_to() :: ?REDISPATCH_TO(emqx_types:group(), emqx_types:topic()).
-record(state, {pmon}).
@ -156,7 +156,7 @@ dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) ->
end
end.
-spec strategy(emqx_topic:group()) -> strategy().
-spec strategy(emqx_types:group()) -> strategy().
strategy(Group) ->
try
emqx:get_config([

85
apps/emqx/src/emqx_topic.erl
View File

@ -16,6 +16,8 @@
-module(emqx_topic).
-include("emqx_mqtt.hrl").
%% APIs
-export([
match/2,
@ -33,18 +35,14 @@
parse/2
]).
-export_type([
group/0,
topic/0,
word/0
]).
-type topic() :: emqx_types:topic().
-type word() :: emqx_types:word().
-type words() :: emqx_types:words().
-type group() :: binary().
-type topic() :: binary().
-type word() :: '' | '+' | '#' | binary().
-type words() :: list(word()).
-define(MAX_TOPIC_LEN, 65535).
%% Guards
-define(MULTI_LEVEL_WILDCARD_NOT_LAST(C, REST),
((C =:= '#' orelse C =:= <<"#">>) andalso REST =/= [])
).
%%--------------------------------------------------------------------
%% APIs
@ -97,11 +95,15 @@ validate({Type, Topic}) when Type =:= name; Type =:= filter ->
-spec validate(name | filter, topic()) -> true.
validate(_, <<>>) ->
%% MQTT-5.0 [MQTT-4.7.3-1]
error(empty_topic);
validate(_, Topic) when is_binary(Topic) andalso (size(Topic) > ?MAX_TOPIC_LEN) ->
%% MQTT-5.0 [MQTT-4.7.3-3]
error(topic_too_long);
validate(filter, Topic) when is_binary(Topic) ->
validate2(words(Topic));
validate(filter, SharedFilter = <<"$share/", _Rest/binary>>) ->
validate_share(SharedFilter);
validate(filter, Filter) when is_binary(Filter) ->
validate2(words(Filter));
validate(name, Topic) when is_binary(Topic) ->
Words = words(Topic),
validate2(Words) andalso
@ -113,7 +115,8 @@ validate2([]) ->
% end with '#'
validate2(['#']) ->
true;
validate2(['#' | Words]) when length(Words) > 0 ->
%% MQTT-5.0 [MQTT-4.7.1-1]
validate2([C | Words]) when ?MULTI_LEVEL_WILDCARD_NOT_LAST(C, Words) ->
error('topic_invalid_#');
validate2(['' | Words]) ->
validate2(Words);
@ -129,6 +132,32 @@ validate3(<<C/utf8, _Rest/binary>>) when C == $#; C == $+; C == 0 ->
validate3(<<_/utf8, Rest/binary>>) ->
validate3(Rest).
validate_share(<<"$share/", Rest/binary>>) when
Rest =:= <<>> orelse Rest =:= <<"/">>
->
%% MQTT-5.0 [MQTT-4.8.2-1]
error(?SHARE_EMPTY_FILTER);
validate_share(<<"$share/", Rest/binary>>) ->
case binary:split(Rest, <<"/">>) of
%% MQTT-5.0 [MQTT-4.8.2-1]
[<<>>, _] ->
error(?SHARE_EMPTY_GROUP);
%% MQTT-5.0 [MQTT-4.7.3-1]
[_, <<>>] ->
error(?SHARE_EMPTY_FILTER);
[ShareName, Filter] ->
validate_share(ShareName, Filter)
end.
validate_share(_, <<"$share/", _Rest/binary>>) ->
error(?SHARE_RECURSIVELY);
validate_share(ShareName, Filter) ->
case binary:match(ShareName, [<<"+">>, <<"#">>]) of
%% MQTT-5.0 [MQTT-4.8.2-2]
nomatch -> validate2(words(Filter));
_ -> error(?SHARE_NAME_INVALID_CHAR)
end.
%% @doc Prepend a topic prefix.
%% Ensured to have only one / between prefix and suffix.
prepend(undefined, W) ->
@ -142,6 +171,7 @@ prepend(Parent0, W) ->
_ -> <<Parent/binary, $/, (bin(W))/binary>>
end.
-spec bin(word()) -> binary().
bin('') -> <<>>;
bin('+') -> <<"+">>;
bin('#') -> <<"#">>;
@ -163,6 +193,7 @@ tokens(Topic) ->
words(Topic) when is_binary(Topic) ->
[word(W) || W <- tokens(Topic)].
-spec word(binary()) -> word().
word(<<>>) -> '';
word(<<"+">>) -> '+';
word(<<"#">>) -> '#';
@ -185,23 +216,19 @@ feed_var(Var, Val, [Var | Words], Acc) ->
feed_var(Var, Val, [W | Words], Acc) ->
feed_var(Var, Val, Words, [W | Acc]).
-spec join(list(binary())) -> binary().
-spec join(list(word())) -> binary().
join([]) ->
<<>>;
join([W]) ->
bin(W);
join(Words) ->
{_, Bin} = lists:foldr(
fun
(W, {true, Tail}) ->
{false, <<W/binary, Tail/binary>>};
(W, {false, Tail}) ->
{false, <<W/binary, "/", Tail/binary>>}
end,
{true, <<>>},
[bin(W) || W <- Words]
),
Bin.
join([Word | Words]) ->
do_join(bin(Word), Words).
do_join(TopicAcc, []) ->
TopicAcc;
%% MQTT-5.0 [MQTT-4.7.1-1]
do_join(_TopicAcc, [C | Words]) when ?MULTI_LEVEL_WILDCARD_NOT_LAST(C, Words) ->
error('topic_invalid_#');
do_join(TopicAcc, [Word | Words]) ->
do_join(<<TopicAcc/binary, "/", (bin(Word))/binary>>, Words).
-spec parse(topic() | {topic(), map()}) -> {topic(), #{share => binary()}}.
parse(TopicFilter) when is_binary(TopicFilter) ->

6
apps/emqx/src/emqx_trie.erl
View File

@ -114,7 +114,7 @@ create_session_trie(Type) ->
insert(Topic) when is_binary(Topic) ->
insert(Topic, ?TRIE).
-spec insert_session(emqx_topic:topic()) -> ok.
-spec insert_session(emqx_types:topic()) -> ok.
insert_session(Topic) when is_binary(Topic) ->
insert(Topic, session_trie()).
@ -132,7 +132,7 @@ delete(Topic) when is_binary(Topic) ->
delete(Topic, ?TRIE).
%% @doc Delete a topic filter from the trie.
-spec delete_session(emqx_topic:topic()) -> ok.
-spec delete_session(emqx_types:topic()) -> ok.
delete_session(Topic) when is_binary(Topic) ->
delete(Topic, session_trie()).
@ -148,7 +148,7 @@ delete(Topic, Trie) when is_binary(Topic) ->
match(Topic) when is_binary(Topic) ->
match(Topic, ?TRIE).
-spec match_session(emqx_topic:topic()) -> list(emqx_topic:topic()).
-spec match_session(emqx_types:topic()) -> list(emqx_types:topic()).
match_session(Topic) when is_binary(Topic) ->
match(Topic, session_trie()).

15
apps/emqx/src/emqx_types.erl
View File

@ -29,10 +29,16 @@
-export_type([
zone/0,
pubsub/0,
topic/0,
subid/0
]).
-export_type([
group/0,
topic/0,
word/0,
words/0
]).
-export_type([
socktype/0,
sockstate/0,
@ -122,9 +128,13 @@
-type zone() :: atom().
-type pubsub() :: publish | subscribe.
-type topic() :: emqx_topic:topic().
-type subid() :: binary() | atom().
-type group() :: binary() | undefined.
-type topic() :: binary().
-type word() :: '' | '+' | '#' | binary().
-type words() :: list(word()).
-type socktype() :: tcp | udp | ssl | proxy | atom().
-type sockstate() :: idle | running | blocked | closed.
-type conninfo() :: #{
@ -230,7 +240,6 @@
| {share, topic(), deliver_result()}
].
-type route() :: #route{}.
-type group() :: emqx_topic:group().
-type route_entry() :: {topic(), node()} | {topic, group()}.
-type command() :: #command{}.

3
apps/emqx/src/emqx_ws_connection.erl
View File

@ -18,6 +18,7 @@
-module(emqx_ws_connection).
-include("emqx.hrl").
-include("emqx_cm.hrl").
-include("emqx_mqtt.hrl").
-include("logger.hrl").
-include("types.hrl").
@ -1034,7 +1035,7 @@ check_max_connection(Type, Listener) ->
allow;
Max ->
MatchSpec = [{{'_', emqx_ws_connection}, [], [true]}],
Curr = ets:select_count(emqx_channel_conn, MatchSpec),
Curr = ets:select_count(?CHAN_CONN_TAB, MatchSpec),
case Curr >= Max of
false ->
allow;

3
apps/emqx/src/emqx_zone_schema.erl
View File

@ -58,8 +58,7 @@ hidden() ->
[
"stats",
"overload_protection",
"conn_congestion",
"flapping_detect"
"conn_congestion"
].
%% zone schemas are clones from the same name from root level

2
apps/emqx/src/proto/emqx_cm_proto_v1.erl
View File

@ -33,7 +33,7 @@
]).
-include("bpapi.hrl").
-include("src/emqx_cm.hrl").
-include_lib("emqx/include/emqx_cm.hrl").
introduced_in() ->
"5.0.0".

2
apps/emqx/src/proto/emqx_cm_proto_v2.erl
View File

@ -34,7 +34,7 @@
]).
-include("bpapi.hrl").
-include("src/emqx_cm.hrl").
-include_lib("emqx/include/emqx_cm.hrl").
introduced_in() ->
"5.0.0".

7
apps/emqx/test/emqx_cm_SUITE.erl
View File

@ -20,6 +20,7 @@
-compile(nowarn_export_all).
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_cm.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
@ -200,10 +201,10 @@ t_open_session_race_condition(_) ->
end,
Winner = WaitForDowns(Pids),
?assertMatch([_], ets:lookup(emqx_channel, ClientId)),
?assertMatch([_], ets:lookup(?CHAN_TAB, ClientId)),
?assertEqual([Winner], emqx_cm:lookup_channels(ClientId)),
?assertMatch([_], ets:lookup(emqx_channel_conn, {ClientId, Winner})),
?assertMatch([_], ets:lookup(emqx_channel_registry, ClientId)),
?assertMatch([_], ets:lookup(?CHAN_CONN_TAB, {ClientId, Winner})),
?assertMatch([_], ets:lookup(?CHAN_REG_TAB, ClientId)),
exit(Winner, kill),
receive

93
apps/emqx/test/emqx_config_SUITE.erl
View File

@ -32,7 +32,23 @@ init_per_suite(Config) ->
end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([]).
t_fill_default_values(_) ->
init_per_testcase(TestCase, Config) ->
try
?MODULE:TestCase({init, Config})
catch
error:function_clause ->
Config
end.
end_per_testcase(TestCase, Config) ->
try
?MODULE:TestCase({'end', Config})
catch
error:function_clause ->
ok
end.
t_fill_default_values(C) when is_list(C) ->
Conf = #{
<<"broker">> => #{
<<"perf">> => #{},
@ -61,7 +77,7 @@ t_fill_default_values(_) ->
_ = emqx_utils_json:encode(WithDefaults),
ok.
t_init_load(_Config) ->
t_init_load(C) when is_list(C) ->
ConfFile = "./test_emqx.conf",
ok = file:write_file(ConfFile, <<"">>),
ExpectRootNames = lists:sort(hocon_schema:root_names(emqx_schema)),
@ -80,7 +96,7 @@ t_init_load(_Config) ->
?assertMatch({ok, #{raw_config := 128}}, emqx:update_config([mqtt, max_topic_levels], 128)),
ok = file:delete(DeprecatedFile).
t_unknown_rook_keys(_) ->
t_unknown_root_keys(C) when is_list(C) ->
?check_trace(
#{timetrap => 1000},
begin
@ -98,7 +114,50 @@ t_unknown_rook_keys(_) ->
),
ok.
t_init_load_emqx_schema(Config) ->
t_cluster_hocon_backup({init, C}) ->
C;
t_cluster_hocon_backup({'end', _C}) ->
File = "backup-test.hocon",
Files = [File | filelib:wildcard(File ++ ".*.bak")],
lists:foreach(fun file:delete/1, Files);
t_cluster_hocon_backup(C) when is_list(C) ->
Write = fun(Path, Content) ->
%% avoid name clash
timer:sleep(1),
emqx_config:backup_and_write(Path, Content)
end,
File = "backup-test.hocon",
%% write 12 times, 10 backups should be kept
%% the latest one is File itself without suffix
%% the oldest one is expected to be deleted
N = 12,
Inputs = lists:seq(1, N),
Backups = lists:seq(N - 10, N - 1),
InputContents = [integer_to_binary(I) || I <- Inputs],
BackupContents = [integer_to_binary(I) || I <- Backups],
lists:foreach(
fun(Content) ->
Write(File, Content)
end,
InputContents
),
LatestContent = integer_to_binary(N),
?assertEqual({ok, LatestContent}, file:read_file(File)),
Re = "\\.[0-9]{4}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{3}\\.bak$",
Files = filelib:wildcard(File ++ ".*.bak"),
?assert(lists:all(fun(F) -> re:run(F, Re) =/= nomatch end, Files)),
%% keep only the latest 10
?assertEqual(10, length(Files)),
FilesSorted = lists:zip(lists:sort(Files), BackupContents),
lists:foreach(
fun({BackupFile, ExpectedContent}) ->
?assertEqual({ok, ExpectedContent}, file:read_file(BackupFile))
end,
FilesSorted
),
ok.
t_init_load_emqx_schema(Config) when is_list(Config) ->
emqx_config:erase_all(),
%% Given empty config file
ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"">>, Config),
@ -127,7 +186,7 @@ t_init_load_emqx_schema(Config) ->
Default
).
t_init_zones_load_emqx_schema_no_default_for_none_existing(Config) ->
t_init_zones_load_emqx_schema_no_default_for_none_existing(Config) when is_list(Config) ->
emqx_config:erase_all(),
%% Given empty config file
ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"">>, Config),
@ -140,7 +199,7 @@ t_init_zones_load_emqx_schema_no_default_for_none_existing(Config) ->
emqx_config:get([zones, no_exists])
).
t_init_zones_load_other_schema(Config) ->
t_init_zones_load_other_schema(Config) when is_list(Config) ->
emqx_config:erase_all(),
%% Given empty config file
ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"">>, Config),
@ -159,7 +218,7 @@ t_init_zones_load_other_schema(Config) ->
emqx_config:get([zones, default])
).
t_init_zones_with_user_defined_default_zone(Config) ->
t_init_zones_with_user_defined_default_zone(Config) when is_list(Config) ->
emqx_config:erase_all(),
%% Given user defined config for default zone
ConfFile = prepare_conf_file(
@ -176,7 +235,7 @@ t_init_zones_with_user_defined_default_zone(Config) ->
%% Then others are defaults
?assertEqual(ExpectedOthers, Others).
t_init_zones_with_user_defined_other_zone(Config) ->
t_init_zones_with_user_defined_other_zone(Config) when is_list(Config) ->
emqx_config:erase_all(),
%% Given user defined config for default zone
ConfFile = prepare_conf_file(
@ -196,7 +255,7 @@ t_init_zones_with_user_defined_other_zone(Config) ->
%% Then default zone still have the defaults
?assertEqual(zone_global_defaults(), emqx_config:get([zones, default])).
t_init_zones_with_cust_root_mqtt(Config) ->
t_init_zones_with_cust_root_mqtt(Config) when is_list(Config) ->
emqx_config:erase_all(),
%% Given config file with mqtt user overrides
ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"mqtt.retry_interval=10m">>, Config),
@ -211,7 +270,7 @@ t_init_zones_with_cust_root_mqtt(Config) ->
emqx_config:get([zones, default, mqtt])
).
t_default_zone_is_updated_after_global_defaults_updated(Config) ->
t_default_zone_is_updated_after_global_defaults_updated(Config) when is_list(Config) ->
emqx_config:erase_all(),
%% Given empty emqx conf
ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"">>, Config),
@ -227,7 +286,7 @@ t_default_zone_is_updated_after_global_defaults_updated(Config) ->
emqx_config:get([zones, default, mqtt])
).
t_myzone_is_updated_after_global_defaults_updated(Config) ->
t_myzone_is_updated_after_global_defaults_updated(Config) when is_list(Config) ->
emqx_config:erase_all(),
%% Given emqx conf file with user override in myzone (none default zone)
ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"zones.myzone.mqtt.max_inflight=32">>, Config),
@ -251,7 +310,7 @@ t_myzone_is_updated_after_global_defaults_updated(Config) ->
emqx_config:get([zones, default, mqtt])
).
t_zone_no_user_defined_overrides(Config) ->
t_zone_no_user_defined_overrides(Config) when is_list(Config) ->
emqx_config:erase_all(),
%% Given emqx conf file with user specified myzone
ConfFile = prepare_conf_file(
@ -268,7 +327,7 @@ t_zone_no_user_defined_overrides(Config) ->
%% Then user defined value from config is not overwritten
?assertMatch(600000, emqx_config:get([zones, myzone, mqtt, retry_interval])).
t_zone_no_user_defined_overrides_internal_represent(Config) ->
t_zone_no_user_defined_overrides_internal_represent(Config) when is_list(Config) ->
emqx_config:erase_all(),
%% Given emqx conf file with user specified myzone
ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"zones.myzone.mqtt.max_inflight=1">>, Config),
@ -281,7 +340,7 @@ t_zone_no_user_defined_overrides_internal_represent(Config) ->
?assertMatch(2, emqx_config:get([zones, default, mqtt, max_inflight])),
?assertMatch(1, emqx_config:get([zones, myzone, mqtt, max_inflight])).
t_update_global_defaults_no_updates_on_user_overrides(Config) ->
t_update_global_defaults_no_updates_on_user_overrides(Config) when is_list(Config) ->
emqx_config:erase_all(),
%% Given default zone config in conf file.
ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"zones.default.mqtt.max_inflight=1">>, Config),
@ -293,7 +352,7 @@ t_update_global_defaults_no_updates_on_user_overrides(Config) ->
%% Then the value is not reflected in default `zone'
?assertMatch(1, emqx_config:get([zones, default, mqtt, max_inflight])).
t_zone_update_with_new_zone(Config) ->
t_zone_update_with_new_zone(Config) when is_list(Config) ->
emqx_config:erase_all(),
%% Given loaded an empty conf file
ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"">>, Config),
@ -308,7 +367,7 @@ t_zone_update_with_new_zone(Config) ->
emqx_config:get([zones, myzone, mqtt])
).
t_init_zone_with_global_defaults(_Config) ->
t_init_zone_with_global_defaults(Config) when is_list(Config) ->
%% Given uninitialized empty config
emqx_config:erase_all(),
Zones = #{myzone => #{mqtt => #{max_inflight => 3}}},
@ -344,7 +403,7 @@ zone_global_defaults() ->
conn_congestion =>
#{enable_alarm => true, min_alarm_sustain_duration => 60000},
flapping_detect =>
#{ban_time => 300000, max_count => 15, window_time => disabled},
#{ban_time => 300000, max_count => 15, window_time => 60000, enable => false},
force_gc =>
#{bytes => 16777216, count => 16000, enable => true},
force_shutdown =>

84
apps/emqx/test/emqx_flapping_SUITE.erl
View File

@ -26,15 +26,16 @@ all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
emqx_common_test_helpers:boot_modules(all),
emqx_common_test_helpers:start_apps([]),
emqx_config:put_zone_conf(
default,
%% update global default config
{ok, _} = emqx:update_config(
[flapping_detect],
#{
max_count => 3,
<<"enable">> => true,
<<"max_count">> => 3,
% 0.1s
window_time => 100,
<<"window_time">> => 100,
%% 2s
ban_time => 2000
<<"ban_time">> => "2s"
}
),
Config.
@ -102,20 +103,73 @@ t_expired_detecting(_) ->
)
).
t_conf_without_window_time(_) ->
%% enable is deprecated, so we need to make sure it won't be used.
t_conf_update(_) ->
Global = emqx_config:get([flapping_detect]),
?assertNot(maps:is_key(enable, Global)),
%% zones don't have default value, so we need to make sure fallback to global conf.
%% this new_zone will fallback to global conf.
#{
ban_time := _BanTime,
enable := _Enable,
max_count := _MaxCount,
window_time := _WindowTime
} = Global,
emqx_config:put_zone_conf(new_zone, [flapping_detect], #{}),
?assertEqual(Global, get_policy(new_zone)),
emqx_config:put_zone_conf(new_zone_1, [flapping_detect], #{window_time => 100}),
?assertEqual(100, emqx_flapping:get_policy(window_time, new_zone_1)),
?assertEqual(maps:get(ban_time, Global), emqx_flapping:get_policy(ban_time, new_zone_1)),
?assertEqual(maps:get(max_count, Global), emqx_flapping:get_policy(max_count, new_zone_1)),
emqx_config:put_zone_conf(zone_1, [flapping_detect], #{window_time => 100}),
?assertEqual(Global#{window_time := 100}, emqx_flapping:get_policy(zone_1)),
Zones = #{
<<"zone_1">> => #{<<"flapping_detect">> => #{<<"window_time">> => 123}},
<<"zone_2">> => #{<<"flapping_detect">> => #{<<"window_time">> => 456}}
},
?assertMatch({ok, _}, emqx:update_config([zones], Zones)),
%% new_zone is already deleted
?assertError({config_not_found, _}, get_policy(new_zone)),
%% update zone(zone_1) has default.
?assertEqual(Global#{window_time := 123}, emqx_flapping:get_policy(zone_1)),
%% create zone(zone_2) has default
?assertEqual(Global#{window_time := 456}, emqx_flapping:get_policy(zone_2)),
%% reset to default(empty) andalso get default from global
?assertMatch({ok, _}, emqx:update_config([zones], #{})),
?assertEqual(Global, emqx:get_config([zones, default, flapping_detect])),
?assertError({config_not_found, _}, get_policy(zone_1)),
?assertError({config_not_found, _}, get_policy(zone_2)),
ok.
t_conf_update_timer(_Config) ->
_ = emqx_flapping:start_link(),
validate_timer([default]),
{ok, _} =
emqx:update_config([zones], #{
<<"timer_1">> => #{<<"flapping_detect">> => #{<<"enable">> => true}},
<<"timer_2">> => #{<<"flapping_detect">> => #{<<"enable">> => true}},
<<"timer_3">> => #{<<"flapping_detect">> => #{<<"enable">> => false}}
}),
validate_timer([timer_1, timer_2, timer_3, default]),
ok.
validate_timer(Names) ->
Zones = emqx:get_config([zones]),
?assertEqual(lists:sort(Names), lists:sort(maps:keys(Zones))),
Timers = sys:get_state(emqx_flapping),
maps:foreach(
fun(Name, #{flapping_detect := #{enable := Enable}}) ->
?assertEqual(Enable, is_reference(maps:get(Name, Timers)), Timers)
end,
Zones
),
?assertEqual(maps:keys(Zones), maps:keys(Timers)),
ok.
t_window_compatibility_check(_Conf) ->
Flapping = emqx:get_config([flapping_detect]),
ok = emqx_config:init_load(emqx_schema, <<"flapping_detect {window_time = disable}">>),
?assertMatch(#{window_time := 60000, enable := false}, emqx:get_config([flapping_detect])),
%% reset
FlappingBin = iolist_to_binary(["flapping_detect {", hocon_pp:do(Flapping, #{}), "}"]),
ok = emqx_config:init_load(emqx_schema, FlappingBin),
?assertEqual(Flapping, emqx:get_config([flapping_detect])),
ok.
get_policy(Zone) ->
emqx_flapping:get_policy([window_time, ban_time, max_count], Zone).
emqx_config:get_zone_conf(Zone, [flapping_detect]).

154
apps/emqx/test/emqx_listeners_update_SUITE.erl Normal file
View File

@ -0,0 +1,154 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_listeners_update_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_schema.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-import(emqx_listeners, [current_conns/2, is_running/1]).
-define(LISTENERS, [listeners]).
all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
emqx_common_test_helpers:boot_modules(all),
emqx_common_test_helpers:start_apps([]),
Config.
end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([]).
init_per_testcase(_TestCase, Config) ->
Init = emqx:get_raw_config(?LISTENERS),
[{init_conf, Init} | Config].
end_per_testcase(_TestCase, Config) ->
Conf = ?config(init_conf, Config),
{ok, _} = emqx:update_config(?LISTENERS, Conf),
ok.
t_default_conf(_Config) ->
?assertMatch(
#{
<<"tcp">> := #{<<"default">> := #{<<"bind">> := <<"0.0.0.0:1883">>}},
<<"ssl">> := #{<<"default">> := #{<<"bind">> := <<"0.0.0.0:8883">>}},
<<"ws">> := #{<<"default">> := #{<<"bind">> := <<"0.0.0.0:8083">>}},
<<"wss">> := #{<<"default">> := #{<<"bind">> := <<"0.0.0.0:8084">>}}
},
emqx:get_raw_config(?LISTENERS)
),
?assertMatch(
#{
tcp := #{default := #{bind := {{0, 0, 0, 0}, 1883}}},
ssl := #{default := #{bind := {{0, 0, 0, 0}, 8883}}},
ws := #{default := #{bind := {{0, 0, 0, 0}, 8083}}},
wss := #{default := #{bind := {{0, 0, 0, 0}, 8084}}}
},
emqx:get_config(?LISTENERS)
),
ok.
t_update_conf(_Conf) ->
Raw = emqx:get_raw_config(?LISTENERS),
Raw1 = emqx_utils_maps:deep_put(
[<<"tcp">>, <<"default">>, <<"bind">>], Raw, <<"127.0.0.1:1883">>
),
Raw2 = emqx_utils_maps:deep_put(
[<<"ssl">>, <<"default">>, <<"bind">>], Raw1, <<"127.0.0.1:8883">>
),
Raw3 = emqx_utils_maps:deep_put(
[<<"ws">>, <<"default">>, <<"bind">>], Raw2, <<"0.0.0.0:8083">>
),
Raw4 = emqx_utils_maps:deep_put(
[<<"wss">>, <<"default">>, <<"bind">>], Raw3, <<"127.0.0.1:8084">>
),
?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw4)),
?assertMatch(
#{
<<"tcp">> := #{<<"default">> := #{<<"bind">> := <<"127.0.0.1:1883">>}},
<<"ssl">> := #{<<"default">> := #{<<"bind">> := <<"127.0.0.1:8883">>}},
<<"ws">> := #{<<"default">> := #{<<"bind">> := <<"0.0.0.0:8083">>}},
<<"wss">> := #{<<"default">> := #{<<"bind">> := <<"127.0.0.1:8084">>}}
},
emqx:get_raw_config(?LISTENERS)
),
BindTcp = {{127, 0, 0, 1}, 1883},
BindSsl = {{127, 0, 0, 1}, 8883},
BindWs = {{0, 0, 0, 0}, 8083},
BindWss = {{127, 0, 0, 1}, 8084},
?assertMatch(
#{
tcp := #{default := #{bind := BindTcp}},
ssl := #{default := #{bind := BindSsl}},
ws := #{default := #{bind := BindWs}},
wss := #{default := #{bind := BindWss}}
},
emqx:get_config(?LISTENERS)
),
?assertError(not_found, current_conns(<<"tcp:default">>, {{0, 0, 0, 0}, 1883})),
?assertError(not_found, current_conns(<<"ssl:default">>, {{0, 0, 0, 0}, 8883})),
?assertEqual(0, current_conns(<<"tcp:default">>, BindTcp)),
?assertEqual(0, current_conns(<<"ssl:default">>, BindSsl)),
?assertEqual({0, 0, 0, 0}, proplists:get_value(ip, ranch:info('ws:default'))),
?assertEqual({127, 0, 0, 1}, proplists:get_value(ip, ranch:info('wss:default'))),
?assert(is_running('ws:default')),
?assert(is_running('wss:default')),
ok.
t_add_delete_conf(_Conf) ->
Raw = emqx:get_raw_config(?LISTENERS),
%% add
#{<<"tcp">> := #{<<"default">> := Tcp}} = Raw,
NewBind = <<"127.0.0.1:1987">>,
Raw1 = emqx_utils_maps:deep_put([<<"tcp">>, <<"new">>], Raw, Tcp#{<<"bind">> => NewBind}),
Raw2 = emqx_utils_maps:deep_put([<<"ssl">>, <<"default">>], Raw1, ?TOMBSTONE_VALUE),
?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw2)),
?assertEqual(0, current_conns(<<"tcp:new">>, {{127, 0, 0, 1}, 1987})),
?assertError(not_found, current_conns(<<"ssl:default">>, {{0, 0, 0, 0}, 8883})),
%% deleted
?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw)),
?assertError(not_found, current_conns(<<"tcp:new">>, {{127, 0, 0, 1}, 1987})),
?assertEqual(0, current_conns(<<"ssl:default">>, {{0, 0, 0, 0}, 8883})),
ok.
t_delete_default_conf(_Conf) ->
Raw = emqx:get_raw_config(?LISTENERS),
%% delete default listeners
Raw1 = emqx_utils_maps:deep_put([<<"tcp">>, <<"default">>], Raw, ?TOMBSTONE_VALUE),
Raw2 = emqx_utils_maps:deep_put([<<"ssl">>, <<"default">>], Raw1, ?TOMBSTONE_VALUE),
Raw3 = emqx_utils_maps:deep_put([<<"ws">>, <<"default">>], Raw2, ?TOMBSTONE_VALUE),
Raw4 = emqx_utils_maps:deep_put([<<"wss">>, <<"default">>], Raw3, ?TOMBSTONE_VALUE),
?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw4)),
?assertError(not_found, current_conns(<<"tcp:default">>, {{0, 0, 0, 0}, 1883})),
?assertError(not_found, current_conns(<<"ssl:default">>, {{0, 0, 0, 0}, 8883})),
?assertMatch({error, not_found}, is_running('ws:default')),
?assertMatch({error, not_found}, is_running('wss:default')),
%% reset
?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw)),
?assertEqual(0, current_conns(<<"tcp:default">>, {{0, 0, 0, 0}, 1883})),
?assertEqual(0, current_conns(<<"ssl:default">>, {{0, 0, 0, 0}, 8883})),
?assert(is_running('ws:default')),
?assert(is_running('wss:default')),
ok.

26
apps/emqx/test/emqx_mqtt_SUITE.erl
View File

@ -219,13 +219,15 @@ t_async_set_keepalive('end', _Config) ->
t_async_set_keepalive(_) ->
case os:type() of
{unix, darwin} ->
%% Mac OSX don't support the feature
ok;
do_async_set_keepalive(16#10, 16#101, 16#102);
{unix, linux} ->
do_async_set_keepalive(4, 5, 6);
_ ->
do_async_set_keepalive()
%% don't support the feature on other OS
ok
end.
do_async_set_keepalive() ->
do_async_set_keepalive(OptKeepIdle, OptKeepInterval, OptKeepCount) ->
ClientID = <<"client-tcp-keepalive">>,
{ok, Client} = emqtt:start_link([
{host, "localhost"},
@ -247,19 +249,19 @@ do_async_set_keepalive() ->
Transport = maps:get(transport, State),
Socket = maps:get(socket, State),
?assert(is_port(Socket)),
Opts = [{raw, 6, 4, 4}, {raw, 6, 5, 4}, {raw, 6, 6, 4}],
Opts = [{raw, 6, OptKeepIdle, 4}, {raw, 6, OptKeepInterval, 4}, {raw, 6, OptKeepCount, 4}],
{ok, [
{raw, 6, 4, <<Idle:32/native>>},
{raw, 6, 5, <<Interval:32/native>>},
{raw, 6, 6, <<Probes:32/native>>}
{raw, 6, OptKeepIdle, <<Idle:32/native>>},
{raw, 6, OptKeepInterval, <<Interval:32/native>>},
{raw, 6, OptKeepCount, <<Probes:32/native>>}
]} = Transport:getopts(Socket, Opts),
ct:pal("Idle=~p, Interval=~p, Probes=~p", [Idle, Interval, Probes]),
emqx_connection:async_set_keepalive(Pid, Idle + 1, Interval + 1, Probes + 1),
emqx_connection:async_set_keepalive(os:type(), Pid, Idle + 1, Interval + 1, Probes + 1),
{ok, _} = ?block_until(#{?snk_kind := "custom_socket_options_successfully"}, 1000),
{ok, [
{raw, 6, 4, <<NewIdle:32/native>>},
{raw, 6, 5, <<NewInterval:32/native>>},
{raw, 6, 6, <<NewProbes:32/native>>}
{raw, 6, OptKeepIdle, <<NewIdle:32/native>>},
{raw, 6, OptKeepInterval, <<NewInterval:32/native>>},
{raw, 6, OptKeepCount, <<NewProbes:32/native>>}
]} = Transport:getopts(Socket, Opts),
?assertEqual(NewIdle, Idle + 1),
?assertEqual(NewInterval, Interval + 1),

5
apps/emqx/test/emqx_quic_multistreams_SUITE.erl
View File

@ -23,6 +23,7 @@
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("quicer/include/quicer.hrl").
-include_lib("emqx/include/emqx_cm.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
@ -1465,7 +1466,7 @@ t_multi_streams_emqx_ctrl_kill(Config) ->
),
ClientId = proplists:get_value(clientid, emqtt:info(C)),
[{ClientId, TransPid}] = ets:lookup(emqx_channel, ClientId),
[{ClientId, TransPid}] = ets:lookup(?CHAN_TAB, ClientId),
exit(TransPid, kill),
%% Client should be closed
@ -1518,7 +1519,7 @@ t_multi_streams_emqx_ctrl_exit_normal(Config) ->
),
ClientId = proplists:get_value(clientid, emqtt:info(C)),
[{ClientId, TransPid}] = ets:lookup(emqx_channel, ClientId),
[{ClientId, TransPid}] = ets:lookup(?CHAN_TAB, ClientId),
emqx_connection:stop(TransPid),
%% Client exit normal.

56
apps/emqx/test/emqx_release_tests.erl Normal file
View File

@ -0,0 +1,56 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_release_tests).
-include_lib("eunit/include/eunit.hrl").
vsn_compre_test_() ->
CurrentVersion = emqx_release:version_with_prefix(),
[
{"must be 'same' when comparing with current version", fun() ->
?assertEqual(same, emqx_release:vsn_compare(CurrentVersion))
end},
{"must be 'same' when comparing same version strings", fun() ->
?assertEqual(same, emqx_release:vsn_compare("1.1.1", "1.1.1"))
end},
{"1.1.1 is older than 1.1.2", fun() ->
?assertEqual(older, emqx_release:vsn_compare("1.1.2", "1.1.1")),
?assertEqual(newer, emqx_release:vsn_compare("1.1.1", "1.1.2"))
end},
{"1.1.9 is older than 1.1.10", fun() ->
?assertEqual(older, emqx_release:vsn_compare("1.1.10", "1.1.9")),
?assertEqual(newer, emqx_release:vsn_compare("1.1.9", "1.1.10"))
end},
{"alpha is older than beta", fun() ->
?assertEqual(older, emqx_release:vsn_compare("1.1.1-beta.1", "1.1.1-alpha.2")),
?assertEqual(newer, emqx_release:vsn_compare("1.1.1-alpha.2", "1.1.1-beta.1"))
end},
{"beta is older than rc", fun() ->
?assertEqual(older, emqx_release:vsn_compare("1.1.1-rc.1", "1.1.1-beta.2")),
?assertEqual(newer, emqx_release:vsn_compare("1.1.1-beta.2", "1.1.1-rc.1"))
end},
{"rc is older than official cut", fun() ->
?assertEqual(older, emqx_release:vsn_compare("1.1.1", "1.1.1-rc.1")),
?assertEqual(newer, emqx_release:vsn_compare("1.1.1-rc.1", "1.1.1"))
end},
{"invalid version string will crash", fun() ->
?assertError({invalid_version_string, "1.1.a"}, emqx_release:vsn_compare("v1.1.a")),
?assertError(
{invalid_version_string, "1.1.1-alpha"}, emqx_release:vsn_compare("e1.1.1-alpha")
)
end}
].

3
apps/emqx/test/emqx_router_SUITE.erl
View File

@ -20,6 +20,7 @@
-compile(nowarn_export_all).
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_router.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
@ -127,5 +128,5 @@ t_unexpected(_) ->
clear_tables() ->
lists:foreach(
fun mnesia:clear_table/1,
[emqx_route, emqx_trie, emqx_trie_node]
[?ROUTE_TAB, ?TRIE, emqx_trie_node]
).

6
apps/emqx/test/emqx_router_helper_SUITE.erl
View File

@ -20,11 +20,11 @@
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
-include_lib("emqx/include/emqx_router.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-define(ROUTER_HELPER, emqx_router_helper).
-define(ROUTE_TAB, emqx_route).
all() -> emqx_common_test_helpers:all(?MODULE).
@ -82,9 +82,9 @@ t_monitor(_) ->
emqx_router_helper:monitor(undefined).
t_mnesia(_) ->
?ROUTER_HELPER ! {mnesia_table_event, {delete, {emqx_routing_node, node()}, undefined}},
?ROUTER_HELPER ! {mnesia_table_event, {delete, {?ROUTING_NODE, node()}, undefined}},
?ROUTER_HELPER ! {mnesia_table_event, testing},
?ROUTER_HELPER ! {mnesia_table_event, {write, {emqx_routing_node, node()}, undefined}},
?ROUTER_HELPER ! {mnesia_table_event, {write, {?ROUTING_NODE, node()}, undefined}},
?ROUTER_HELPER ! {membership, testing},
?ROUTER_HELPER ! {membership, {mnesia, down, node()}},
ct:sleep(200).

79
apps/emqx/test/emqx_schema_tests.erl
View File

@ -106,6 +106,67 @@ bad_cipher_test() ->
),
ok.
fail_if_no_peer_cert_test_() ->
Sc = #{
roots => [mqtt_ssl_listener],
fields => #{mqtt_ssl_listener => emqx_schema:fields("mqtt_ssl_listener")}
},
Opts = #{atom_key => false, required => false},
OptsAtomKey = #{atom_key => true, required => false},
InvalidConf = #{
<<"bind">> => <<"0.0.0.0:9883">>,
<<"ssl_options">> => #{
<<"fail_if_no_peer_cert">> => true,
<<"verify">> => <<"verify_none">>
}
},
InvalidListener = #{<<"mqtt_ssl_listener">> => InvalidConf},
ValidListener = #{
<<"mqtt_ssl_listener">> => InvalidConf#{
<<"ssl_options">> =>
#{
<<"fail_if_no_peer_cert">> => true,
<<"verify">> => <<"verify_peer">>
}
}
},
ValidListener1 = #{
<<"mqtt_ssl_listener">> => InvalidConf#{
<<"ssl_options">> =>
#{
<<"fail_if_no_peer_cert">> => false,
<<"verify">> => <<"verify_none">>
}
}
},
Reason = "verify must be verify_peer when fail_if_no_peer_cert is true",
[
?_assertThrow(
{_Sc, [#{kind := validation_error, reason := Reason}]},
hocon_tconf:check_plain(Sc, InvalidListener, Opts)
),
?_assertThrow(
{_Sc, [#{kind := validation_error, reason := Reason}]},
hocon_tconf:check_plain(Sc, InvalidListener, OptsAtomKey)
),
?_assertMatch(
#{mqtt_ssl_listener := #{}},
hocon_tconf:check_plain(Sc, ValidListener, OptsAtomKey)
),
?_assertMatch(
#{mqtt_ssl_listener := #{}},
hocon_tconf:check_plain(Sc, ValidListener1, OptsAtomKey)
),
?_assertMatch(
#{<<"mqtt_ssl_listener">> := #{}},
hocon_tconf:check_plain(Sc, ValidListener, Opts)
),
?_assertMatch(
#{<<"mqtt_ssl_listener">> := #{}},
hocon_tconf:check_plain(Sc, ValidListener1, Opts)
)
].
validate(Schema, Data0) ->
Sc = #{
roots => [ssl_opts],
@ -825,15 +886,27 @@ timeout_types_test_() ->
typerefl:from_string(emqx_schema:timeout_duration_s(), <<"4294967000ms">>)
),
?_assertThrow(
"timeout value too large (max: 4294967295 ms)",
#{
kind := validation_error,
message := "timeout value too large (max: 4294967295 ms)",
schema_module := emqx_schema
},
typerefl:from_string(emqx_schema:timeout_duration(), <<"4294967296ms">>)
),
?_assertThrow(
"timeout value too large (max: 4294967295 ms)",
#{
kind := validation_error,
message := "timeout value too large (max: 4294967295 ms)",
schema_module := emqx_schema
},
typerefl:from_string(emqx_schema:timeout_duration_ms(), <<"4294967296ms">>)
),
?_assertThrow(
"timeout value too large (max: 4294967 s)",
#{
kind := validation_error,
message := "timeout value too large (max: 4294967 s)",
schema_module := emqx_schema
},
typerefl:from_string(emqx_schema:timeout_duration_s(), <<"4294967001ms">>)
)
].

3
apps/emqx/test/emqx_takeover_SUITE.erl
View File

@ -20,6 +20,7 @@
-compile(nowarn_export_all).
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_cm.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
@ -117,7 +118,7 @@ load_meck(ClientId) ->
[ChanPid] = emqx_cm:lookup_channels(ClientId),
ChanInfo = #{conninfo := ConnInfo} = emqx_cm:get_chan_info(ClientId),
NChanInfo = ChanInfo#{conninfo := ConnInfo#{conn_mod := fake_conn_mod}},
true = ets:update_element(emqx_channel_info, {ClientId, ChanPid}, {2, NChanInfo}).
true = ets:update_element(?CHAN_INFO_TAB, {ClientId, ChanPid}, {2, NChanInfo}).
unload_meck(_ClientId) ->
meck:unload(fake_conn_mod).

31
apps/emqx/test/emqx_topic_SUITE.erl
View File

@ -20,6 +20,7 @@
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").
-import(
@ -130,14 +131,35 @@ t_validate(_) ->
true = validate({filter, <<"x">>}),
true = validate({name, <<"x//y">>}),
true = validate({filter, <<"sport/tennis/#">>}),
%% MQTT-5.0 [MQTT-4.7.3-1]
?assertError(empty_topic, validate({name, <<>>})),
?assertError(empty_topic, validate({filter, <<>>})),
?assertError(topic_name_error, validate({name, <<"abc/#">>})),
?assertError(topic_too_long, validate({name, long_topic()})),
?assertError('topic_invalid_#', validate({filter, <<"abc/#/1">>})),
?assertError(topic_invalid_char, validate({filter, <<"abc/#xzy/+">>})),
?assertError(topic_invalid_char, validate({filter, <<"abc/xzy/+9827">>})),
?assertError(topic_invalid_char, validate({filter, <<"sport/tennis#">>})),
?assertError('topic_invalid_#', validate({filter, <<"sport/tennis/#/ranking">>})).
%% MQTT-5.0 [MQTT-4.7.1-1]
?assertError('topic_invalid_#', validate({filter, <<"abc/#/1">>})),
?assertError('topic_invalid_#', validate({filter, <<"sport/tennis/#/ranking">>})),
%% MQTT-5.0 [MQTT-4.8.2-1]
?assertError(?SHARE_EMPTY_FILTER, validate({filter, <<"$share/">>})),
?assertError(?SHARE_EMPTY_FILTER, validate({filter, <<"$share//">>})),
?assertError(?SHARE_EMPTY_GROUP, validate({filter, <<"$share//t">>})),
?assertError(?SHARE_EMPTY_GROUP, validate({filter, <<"$share//test">>})),
%% MQTT-5.0 [MQTT-4.7.3-1] for shared-sub
?assertError(?SHARE_EMPTY_FILTER, validate({filter, <<"$share/g/">>})),
?assertError(?SHARE_EMPTY_FILTER, validate({filter, <<"$share/g2/">>})),
%% MQTT-5.0 [MQTT-4.8.2-2]
?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/p+q/1">>})),
?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/m+/1">>})),
?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/+n/1">>})),
?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/x#y/1">>})),
?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/x#/1">>})),
?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/#y/1">>})),
%% share recursively
?assertError(?SHARE_RECURSIVELY, validate({filter, <<"$share/g1/$share/t">>})),
true = validate({filter, <<"$share/g1/topic/$share">>}).
t_sigle_level_validate(_) ->
true = validate({filter, <<"+">>}),
@ -177,7 +199,10 @@ t_join(_) ->
?assertEqual(<<"+//#">>, join(['+', '', '#'])),
?assertEqual(<<"x/y/z/+">>, join([<<"x">>, <<"y">>, <<"z">>, '+'])),
?assertEqual(<<"/ab/cd/ef/">>, join(words(<<"/ab/cd/ef/">>))),
?assertEqual(<<"ab/+/#">>, join(words(<<"ab/+/#">>))).
?assertEqual(<<"ab/+/#">>, join(words(<<"ab/+/#">>))),
%% MQTT-5.0 [MQTT-4.7.1-1]
?assertError('topic_invalid_#', join(['+', <<"a">>, '#', <<"b">>, '', '+'])),
?assertError('topic_invalid_#', join(['+', <<"c">>, <<"#">>, <<"d">>, '', '+'])).
t_systop(_) ->
SysTop1 = iolist_to_binary(["$SYS/brokers/", atom_to_list(node()), "/xyz"]),

2
apps/emqx_authz/src/emqx_authz_mnesia.erl
View File

@ -33,7 +33,7 @@
-type clientid() :: {clientid, binary()}.
-type who() :: username() | clientid() | all.
-type rule() :: {emqx_authz_rule:permission(), emqx_authz_rule:action(), emqx_topic:topic()}.
-type rule() :: {emqx_authz_rule:permission(), emqx_authz_rule:action(), emqx_types:topic()}.
-type rules() :: [rule()].
-record(emqx_acl, {

4
apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl
View File

@ -143,12 +143,12 @@ on_start(
{error, Reason}
end.
on_stop(InstId, #{pool_name := PoolName}) ->
on_stop(InstId, _State) ->
?SLOG(info, #{
msg => "stopping_cassandra_connector",
connector => InstId
}),
emqx_resource_pool:stop(PoolName).
emqx_resource_pool:stop(InstId).
-type request() ::
% emqx_bridge.erl

4
apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl
View File

@ -274,12 +274,12 @@ connect(Options) ->
-spec on_stop(resource_id(), resource_state()) -> term().
on_stop(InstanceID, #{pool_name := PoolName}) ->
on_stop(InstanceID, _State) ->
?SLOG(info, #{
msg => "stopping clickouse connector",
connector => InstanceID
}),
emqx_resource_pool:stop(PoolName).
emqx_resource_pool:stop(InstanceID).
%% -------------------------------------------------------------------
%% on_get_status emqx_resouce callback and related functions

4
apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl
View File

@ -111,12 +111,12 @@ on_start(
Error
end.
on_stop(InstanceId, #{pool_name := PoolName}) ->
on_stop(InstanceId, _State) ->
?SLOG(info, #{
msg => "stopping_dynamo_connector",
connector => InstanceId
}),
emqx_resource_pool:stop(PoolName).
emqx_resource_pool:stop(InstanceId).
on_query(InstanceId, Query, State) ->
do_query(InstanceId, Query, State).

121
apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl
View File

@ -6,6 +6,7 @@
-behaviour(emqx_resource).
-include_lib("jose/include/jose_jwk.hrl").
-include_lib("emqx_connector/include/emqx_connector_tables.hrl").
-include_lib("emqx_resource/include/emqx_resource.hrl").
-include_lib("typerefl/include/types.hrl").
@ -26,7 +27,6 @@
]).
-export([reply_delegator/3]).
-type jwt_worker() :: binary().
-type service_account_json() :: emqx_bridge_gcp_pubsub:service_account_json().
-type config() :: #{
connect_timeout := emqx_schema:duration_ms(),
@ -38,7 +38,7 @@
}.
-type state() :: #{
connect_timeout := timer:time(),
jwt_worker_id := jwt_worker(),
jwt_config := emqx_connector_jwt:jwt_config(),
max_retries := non_neg_integer(),
payload_template := emqx_plugin_libs_rule:tmpl_token(),
pool_name := binary(),
@ -97,12 +97,12 @@ on_start(
{enable_pipelining, maps:get(enable_pipelining, Config, ?DEFAULT_PIPELINE_SIZE)}
],
#{
jwt_worker_id := JWTWorkerId,
jwt_config := JWTConfig,
project_id := ProjectId
} = ensure_jwt_worker(ResourceId, Config),
} = parse_jwt_config(ResourceId, Config),
State = #{
connect_timeout => ConnectTimeout,
jwt_worker_id => JWTWorkerId,
jwt_config => JWTConfig,
max_retries => MaxRetries,
payload_template => emqx_plugin_libs_rule:preproc_tmpl(PayloadTemplate),
pool_name => ResourceId,
@ -134,18 +134,21 @@ on_start(
end.
-spec on_stop(resource_id(), state()) -> ok | {error, term()}.
on_stop(
ResourceId,
_State = #{jwt_worker_id := JWTWorkerId}
) ->
?tp(gcp_pubsub_stop, #{resource_id => ResourceId, jwt_worker_id => JWTWorkerId}),
on_stop(ResourceId, _State) ->
?tp(gcp_pubsub_stop, #{resource_id => ResourceId}),
?SLOG(info, #{
msg => "stopping_gcp_pubsub_bridge",
connector => ResourceId
}),
emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId),
emqx_connector_jwt:delete_jwt(?JWT_TABLE, ResourceId),
ehttpc_sup:stop_pool(ResourceId).
ok = emqx_connector_jwt:delete_jwt(?JWT_TABLE, ResourceId),
case ehttpc_sup:stop_pool(ResourceId) of
ok ->
ok;
{error, not_found} ->
ok;
Error ->
Error
end.
-spec on_query(
resource_id(),
@ -228,12 +231,12 @@ on_get_status(ResourceId, #{connect_timeout := Timeout} = State) ->
%% Helper fns
%%-------------------------------------------------------------------------------------------------
-spec ensure_jwt_worker(resource_id(), config()) ->
-spec parse_jwt_config(resource_id(), config()) ->
#{
jwt_worker_id := jwt_worker(),
jwt_config := emqx_connector_jwt:jwt_config(),
project_id := binary()
}.
ensure_jwt_worker(ResourceId, #{
parse_jwt_config(ResourceId, #{
service_account_json := ServiceAccountJSON
}) ->
#{
@ -246,8 +249,32 @@ ensure_jwt_worker(ResourceId, #{
Aud = <<"https://pubsub.googleapis.com/">>,
ExpirationMS = timer:hours(1),
Alg = <<"RS256">>,
Config = #{
private_key => PrivateKeyPEM,
JWK =
try jose_jwk:from_pem(PrivateKeyPEM) of
JWK0 = #jose_jwk{} ->
%% Don't wrap the JWK with `emqx_secret:wrap' here;
%% this is stored in mnesia and synchronized among the
%% nodes, and will easily become a bad fun.
JWK0;
[] ->
?tp(error, gcp_pubsub_connector_startup_error, #{error => empty_key}),
throw("empty private in service account json");
{error, Reason} ->
Error = {invalid_private_key, Reason},
?tp(error, gcp_pubsub_connector_startup_error, #{error => Error}),
throw("invalid private key in service account json");
Error0 ->
Error = {invalid_private_key, Error0},
?tp(error, gcp_pubsub_connector_startup_error, #{error => Error}),
throw("invalid private key in service account json")
catch
Kind:Reason ->
Error = {Kind, Reason},
?tp(error, gcp_pubsub_connector_startup_error, #{error => Error}),
throw("invalid private key in service account json")
end,
JWTConfig = #{
jwk => emqx_secret:wrap(JWK),
resource_id => ResourceId,
expiration => ExpirationMS,
table => ?JWT_TABLE,
@ -257,46 +284,8 @@ ensure_jwt_worker(ResourceId, #{
kid => KId,
alg => Alg
},
JWTWorkerId = <<"gcp_pubsub_jwt_worker:", ResourceId/binary>>,
Worker =
case emqx_connector_jwt_sup:ensure_worker_present(JWTWorkerId, Config) of
{ok, Worker0} ->
Worker0;
Error ->
?tp(error, "gcp_pubsub_bridge_jwt_worker_failed_to_start", #{
connector => ResourceId,
reason => Error
}),
_ = emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId),
throw(failed_to_start_jwt_worker)
end,
MRef = monitor(process, Worker),
Ref = emqx_connector_jwt_worker:ensure_jwt(Worker),
%% to ensure that this resource and its actions will be ready to
%% serve when started, we must ensure that the first JWT has been
%% produced by the worker.
receive
{Ref, token_created} ->
?tp(gcp_pubsub_bridge_jwt_created, #{resource_id => ResourceId}),
demonitor(MRef, [flush]),
ok;
{'DOWN', MRef, process, Worker, Reason} ->
?tp(error, "gcp_pubsub_bridge_jwt_worker_failed_to_start", #{
connector => ResourceId,
reason => Reason
}),
_ = emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId),
throw(failed_to_start_jwt_worker)
after 10_000 ->
?tp(warning, "gcp_pubsub_bridge_jwt_timeout", #{connector => ResourceId}),
demonitor(MRef, [flush]),
_ = emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId),
throw(timeout_creating_jwt)
end,
#{
jwt_worker_id => JWTWorkerId,
jwt_config => JWTConfig,
project_id => ProjectId
}.
@ -322,14 +311,10 @@ publish_path(
) ->
<<"/v1/projects/", ProjectId/binary, "/topics/", PubSubTopic/binary, ":publish">>.
-spec get_jwt_authorization_header(resource_id()) -> [{binary(), binary()}].
get_jwt_authorization_header(ResourceId) ->
case emqx_connector_jwt:lookup_jwt(?JWT_TABLE, ResourceId) of
%% Since we synchronize the JWT creation during resource start
%% (see `on_start/2'), this will be always be populated.
{ok, JWT} ->
[{<<"Authorization">>, <<"Bearer ", JWT/binary>>}]
end.
-spec get_jwt_authorization_header(emqx_connector_jwt:jwt_config()) -> [{binary(), binary()}].
get_jwt_authorization_header(JWTConfig) ->
JWT = emqx_connector_jwt:ensure_jwt(JWTConfig),
[{<<"Authorization">>, <<"Bearer ", JWT/binary>>}].
-spec do_send_requests_sync(
state(),
@ -342,6 +327,7 @@ get_jwt_authorization_header(ResourceId) ->
| {error, term()}.
do_send_requests_sync(State, Requests, ResourceId) ->
#{
jwt_config := JWTConfig,
pool_name := PoolName,
max_retries := MaxRetries,
request_ttl := RequestTTL
@ -354,7 +340,7 @@ do_send_requests_sync(State, Requests, ResourceId) ->
requests => Requests
}
),
Headers = get_jwt_authorization_header(ResourceId),
Headers = get_jwt_authorization_header(JWTConfig),
Payloads =
lists:map(
fun({send_message, Selected}) ->
@ -466,6 +452,7 @@ do_send_requests_sync(State, Requests, ResourceId) ->
) -> {ok, pid()}.
do_send_requests_async(State, Requests, ReplyFunAndArgs, ResourceId) ->
#{
jwt_config := JWTConfig,
pool_name := PoolName,
request_ttl := RequestTTL
} = State,
@ -477,7 +464,7 @@ do_send_requests_async(State, Requests, ReplyFunAndArgs, ResourceId) ->
requests => Requests
}
),
Headers = get_jwt_authorization_header(ResourceId),
Headers = get_jwt_authorization_header(JWTConfig),
Payloads =
lists:map(
fun({send_message, Selected}) ->

132
apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl
View File

@ -55,8 +55,9 @@ single_config_tests() ->
t_not_of_service_account_type,
t_json_missing_fields,
t_invalid_private_key,
t_jwt_worker_start_timeout,
t_failed_to_start_jwt_worker,
t_truncated_private_key,
t_jose_error_tuple,
t_jose_other_error,
t_stop,
t_get_status_ok,
t_get_status_down,
@ -580,14 +581,7 @@ t_publish_success(Config) ->
ServiceAccountJSON = ?config(service_account_json, Config),
TelemetryTable = ?config(telemetry_table, Config),
Topic = <<"t/topic">>,
?check_trace(
create_bridge(Config),
fun(Res, Trace) ->
?assertMatch({ok, _}, Res),
?assertMatch([_], ?of_kind(gcp_pubsub_bridge_jwt_created, Trace)),
ok
end
),
?assertMatch({ok, _}, create_bridge(Config)),
{ok, #{<<"id">> := RuleId}} = create_rule_and_action_http(Config),
on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end),
assert_empty_metrics(ResourceId),
@ -686,14 +680,7 @@ t_publish_success_local_topic(Config) ->
ok.
t_create_via_http(Config) ->
?check_trace(
create_bridge_http(Config),
fun(Res, Trace) ->
?assertMatch({ok, _}, Res),
?assertMatch([_, _], ?of_kind(gcp_pubsub_bridge_jwt_created, Trace)),
ok
end
),
?assertMatch({ok, _}, create_bridge_http(Config)),
ok.
t_publish_templated(Config) ->
@ -705,16 +692,12 @@ t_publish_templated(Config) ->
"{\"payload\": \"${payload}\","
" \"pub_props\": ${pub_props}}"
>>,
?check_trace(
?assertMatch(
{ok, _},
create_bridge(
Config,
#{<<"payload_template">> => PayloadTemplate}
),
fun(Res, Trace) ->
?assertMatch({ok, _}, Res),
?assertMatch([_], ?of_kind(gcp_pubsub_bridge_jwt_created, Trace)),
ok
end
)
),
{ok, #{<<"id">> := RuleId}} = create_rule_and_action_http(Config),
on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end),
@ -908,36 +891,26 @@ t_invalid_private_key(Config) ->
#{<<"private_key">> => InvalidPrivateKeyPEM}
}
),
#{?snk_kind := "gcp_pubsub_bridge_jwt_worker_failed_to_start"},
#{?snk_kind := gcp_pubsub_connector_startup_error},
20_000
),
Res
end,
fun(Res, Trace) ->
?assertMatch({ok, _}, Res),
?assertMatch(
[#{reason := Reason}] when
Reason =:= noproc orelse
Reason =:= {shutdown, {error, empty_key}},
?of_kind("gcp_pubsub_bridge_jwt_worker_failed_to_start", Trace)
),
?assertMatch(
[#{error := empty_key}],
?of_kind(connector_jwt_worker_startup_error, Trace)
?of_kind(gcp_pubsub_connector_startup_error, Trace)
),
ok
end
),
ok.
t_jwt_worker_start_timeout(Config) ->
InvalidPrivateKeyPEM = <<"xxxxxx">>,
t_truncated_private_key(Config) ->
InvalidPrivateKeyPEM = <<"-----BEGIN PRIVATE KEY-----\nMIIEvQI...">>,
?check_trace(
begin
?force_ordering(
#{?snk_kind := will_never_happen},
#{?snk_kind := connector_jwt_worker_make_key}
),
{Res, {ok, _Event}} =
?wait_async_action(
create_bridge(
@ -947,14 +920,71 @@ t_jwt_worker_start_timeout(Config) ->
#{<<"private_key">> => InvalidPrivateKeyPEM}
}
),
#{?snk_kind := "gcp_pubsub_bridge_jwt_timeout"},
#{?snk_kind := gcp_pubsub_connector_startup_error},
20_000
),
Res
end,
fun(Res, Trace) ->
?assertMatch({ok, _}, Res),
?assertMatch([_], ?of_kind("gcp_pubsub_bridge_jwt_timeout", Trace)),
?assertMatch(
[#{error := {error, function_clause}}],
?of_kind(gcp_pubsub_connector_startup_error, Trace)
),
ok
end
),
ok.
t_jose_error_tuple(Config) ->
?check_trace(
begin
{Res, {ok, _Event}} =
?wait_async_action(
emqx_common_test_helpers:with_mock(
jose_jwk,
from_pem,
fun(_PrivateKeyPEM) -> {error, some_error} end,
fun() -> create_bridge(Config) end
),
#{?snk_kind := gcp_pubsub_connector_startup_error},
20_000
),
Res
end,
fun(Res, Trace) ->
?assertMatch({ok, _}, Res),
?assertMatch(
[#{error := {invalid_private_key, some_error}}],
?of_kind(gcp_pubsub_connector_startup_error, Trace)
),
ok
end
),
ok.
t_jose_other_error(Config) ->
?check_trace(
begin
{Res, {ok, _Event}} =
?wait_async_action(
emqx_common_test_helpers:with_mock(
jose_jwk,
from_pem,
fun(_PrivateKeyPEM) -> {unknown, error} end,
fun() -> create_bridge(Config) end
),
#{?snk_kind := gcp_pubsub_connector_startup_error},
20_000
),
Res
end,
fun(Res, Trace) ->
?assertMatch({ok, _}, Res),
?assertMatch(
[#{error := {invalid_private_key, {unknown, error}}}],
?of_kind(gcp_pubsub_connector_startup_error, Trace)
),
ok
end
),
@ -1309,26 +1339,6 @@ t_unrecoverable_error(Config) ->
),
ok.
t_failed_to_start_jwt_worker(Config) ->
?check_trace(
emqx_common_test_helpers:with_mock(
emqx_connector_jwt_sup,
ensure_worker_present,
fun(_JWTWorkerId, _Config) -> {error, restarting} end,
fun() ->
?assertMatch({ok, _}, create_bridge(Config))
end
),
fun(Trace) ->
?assertMatch(
[#{reason := {error, restarting}}],
?of_kind("gcp_pubsub_bridge_jwt_worker_failed_to_start", Trace)
),
ok
end
),
ok.
t_stop(Config) ->
Name = ?config(gcp_pubsub_name, Config),
{ok, _} = create_bridge(Config),

2
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl
View File

@ -49,7 +49,7 @@
-type egress() :: #{
local => #{
topic => emqx_topic:topic()
topic => emqx_types:topic()
},
remote := emqx_bridge_mqtt_msg:msgvars()
}.

2
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl
View File

@ -43,7 +43,7 @@
-type ingress() :: #{
server := string(),
remote := #{
topic := emqx_topic:topic(),
topic := emqx_types:topic(),
qos => emqx_types:qos()
},
local := emqx_bridge_mqtt_msg:msgvars(),

4
apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl
View File

@ -89,12 +89,12 @@ on_start(
Error
end.
on_stop(InstanceId, #{pool_name := PoolName} = _State) ->
on_stop(InstanceId, _State) ->
?SLOG(info, #{
msg => "stopping_opents_connector",
connector => InstanceId
}),
emqx_resource_pool:stop(PoolName).
emqx_resource_pool:stop(InstanceId).
on_query(InstanceId, Request, State) ->
on_batch_query(InstanceId, [Request], State).

5
apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl
View File

@ -261,12 +261,15 @@ on_start(
-spec on_stop(resource_id(), resource_state()) -> term().
on_stop(
ResourceID,
#{poolname := PoolName} = _State
_State
) ->
?SLOG(info, #{
msg => "stopping RabbitMQ connector",
connector => ResourceID
}),
stop_clients_and_pool(ResourceID).
stop_clients_and_pool(PoolName) ->
Workers = [Worker || {_WorkerName, Worker} <- ecpool:workers(PoolName)],
Clients = [
begin

8
apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl
View File

@ -108,7 +108,6 @@ on_start(
Prepares = parse_prepare_sql(Config),
State = Prepares#{pool_name => InstanceId, query_opts => query_opts(Config)},
ok = emqx_resource:allocate_resource(InstanceId, pool_name, InstanceId),
case emqx_resource_pool:start(InstanceId, ?MODULE, Options) of
ok ->
{ok, State};
@ -121,12 +120,7 @@ on_stop(InstanceId, _State) ->
msg => "stopping_tdengine_connector",
connector => InstanceId
}),
case emqx_resource:get_allocated_resources(InstanceId) of
#{pool_name := PoolName} ->
emqx_resource_pool:stop(PoolName);
_ ->
ok
end.
emqx_resource_pool:stop(InstanceId).
on_query(InstanceId, {query, SQL}, State) ->
do_query(InstanceId, SQL, State);

2
apps/emqx_conf/src/emqx_cluster_rpc.erl
View File

@ -575,7 +575,7 @@ maybe_init_tnx_id(Node, TnxId) ->
{atomic, _} = transaction(fun ?MODULE:commit/2, [Node, TnxId]),
ok.
%% @priv Cannot proceed until emqx app is ready.
%% @private Cannot proceed until emqx app is ready.
%% Otherwise the committed transaction catch up may fail.
wait_for_emqx_ready() ->
%% wait 10 seconds for emqx to start

195
apps/emqx_conf/src/emqx_conf_app.erl
View File

@ -42,6 +42,8 @@ start(_StartType, _StartArgs) ->
stop(_State) ->
ok.
%% Read the cluster config from the local node.
%% This function is named 'override' due to historical reasons.
get_override_config_file() ->
Node = node(),
case emqx_app:get_init_config_load_done() of
@ -63,7 +65,7 @@ get_override_config_file() ->
tnx_id => TnxId,
node => Node,
has_deprecated_file => HasDeprecateFile,
release => emqx_app:get_release()
release => emqx_release:version_with_prefix()
}
end,
case mria:ro_transaction(?CLUSTER_RPC_SHARD, Fun) of
@ -95,7 +97,7 @@ init_conf() ->
%% Workaround for https://github.com/emqx/mria/issues/94:
_ = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1000),
_ = mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]),
{ok, TnxId} = copy_override_conf_from_core_node(),
{ok, TnxId} = sync_cluster_conf(),
_ = emqx_app:set_init_tnx_id(TnxId),
ok = init_load(),
ok = emqx_app:set_init_config_load_done().
@ -103,88 +105,137 @@ init_conf() ->
cluster_nodes() ->
mria:cluster_nodes(cores) -- [node()].
copy_override_conf_from_core_node() ->
%% @doc Try to sync the cluster config from other core nodes.
sync_cluster_conf() ->
case cluster_nodes() of
%% The first core nodes is self.
[] ->
?SLOG(debug, #{msg => "skip_copy_override_conf_from_core_node"}),
%% The first core nodes is self.
?SLOG(debug, #{
msg => "skip_sync_cluster_conf",
reason => "This is a single node, or the first node in the cluster"
}),
{ok, ?DEFAULT_INIT_TXN_ID};
Nodes ->
{Results, Failed} = emqx_conf_proto_v2:get_override_config_file(Nodes),
{Ready, NotReady0} = lists:partition(fun(Res) -> element(1, Res) =:= ok end, Results),
NotReady = lists:filter(fun(Res) -> element(1, Res) =:= error end, NotReady0),
case (Failed =/= [] orelse NotReady =/= []) andalso Ready =/= [] of
sync_cluster_conf2(Nodes)
end.
%% @private Some core nodes are running, try to sync the cluster config from them.
sync_cluster_conf2(Nodes) ->
{Results, Failed} = emqx_conf_proto_v2:get_override_config_file(Nodes),
{Ready, NotReady0} = lists:partition(fun(Res) -> element(1, Res) =:= ok end, Results),
NotReady = lists:filter(fun(Res) -> element(1, Res) =:= error end, NotReady0),
case (Failed =/= [] orelse NotReady =/= []) of
true when Ready =/= [] ->
%% Some core nodes failed to reply.
Warning = #{
nodes => Nodes,
failed => Failed,
not_ready => NotReady,
msg => "ignored_nodes_when_sync_cluster_conf"
},
?SLOG(warning, Warning);
true ->
%% There are core nodes running but no one was able to reply.
?SLOG(error, #{
msg => "failed_to_sync_cluster_conf",
nodes => Nodes,
failed => Failed,
not_ready => NotReady
});
false ->
ok
end,
case Ready of
[] ->
case should_proceed_with_boot() of
true ->
Warning = #{
nodes => Nodes,
failed => Failed,
not_ready => NotReady,
msg => "ignored_bad_nodes_when_copy_init_config"
},
?SLOG(warning, Warning);
false ->
ok
end,
case Ready of
[] ->
%% Other core nodes running but no one replicated it successfully.
?SLOG(error, #{
msg => "copy_override_conf_from_core_node_failed",
%% Act as if this node is alone, so it can
%% finish the boot sequence and load the
%% config for other nodes to copy it.
?SLOG(info, #{
msg => "skip_sync_cluster_conf",
loading_from_disk => true,
nodes => Nodes,
failed => Failed,
not_ready => NotReady
}),
case should_proceed_with_boot() of
true ->
%% Act as if this node is alone, so it can
%% finish the boot sequence and load the
%% config for other nodes to copy it.
?SLOG(info, #{
msg => "skip_copy_override_conf_from_core_node",
loading_from_disk => true,
nodes => Nodes,
failed => Failed,
not_ready => NotReady
}),
{ok, ?DEFAULT_INIT_TXN_ID};
false ->
%% retry in some time
Jitter = rand:uniform(2000),
Timeout = 10000 + Jitter,
?SLOG(info, #{
msg => "copy_cluster_conf_from_core_node_retry",
timeout => Timeout,
nodes => Nodes,
failed => Failed,
not_ready => NotReady
}),
timer:sleep(Timeout),
copy_override_conf_from_core_node()
end;
_ ->
[{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready),
#{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info,
HasDeprecatedFile = has_deprecated_file(Info),
?SLOG(debug, #{
msg => "copy_cluster_conf_from_core_node_success",
node => Node,
has_deprecated_file => HasDeprecatedFile,
local_release => emqx_app:get_release(),
remote_release => maps:get(release, Info, "before_v5.0.24|e5.0.3"),
data_dir => emqx:data_dir(),
tnx_id => TnxId
{ok, ?DEFAULT_INIT_TXN_ID};
false ->
%% retry in some time
Jitter = rand:uniform(2000),
Timeout = 10000 + Jitter,
timer:sleep(Timeout),
?SLOG(warning, #{
msg => "sync_cluster_conf_retry",
timeout => Timeout,
nodes => Nodes,
failed => Failed,
not_ready => NotReady
}),
ok = emqx_config:save_to_override_conf(
HasDeprecatedFile,
RawOverrideConf,
#{override_to => cluster}
),
ok = sync_data_from_node(Node),
{ok, TnxId}
end
sync_cluster_conf()
end;
_ ->
sync_cluster_conf3(Ready)
end.
%% @private Filter out the nodes which are running a newer version than this node.
sync_cluster_conf3(Ready) ->
NotNewer = fun({ok, #{release := RemoteRelease}}) ->
try
emqx_release:vsn_compare(RemoteRelease) =/= newer
catch
_:_ ->
%% If the version is not valid (without v or e prefix),
%% we know it's older than v5.1.0/e5.1.0
true
end
end,
case lists:filter(NotNewer, Ready) of
[] ->
%% All available core nodes are running a newer version than this node.
%% Start this node without syncing cluster config from them.
%% This is likely a restart of an older version node during cluster upgrade.
NodesAndVersions = lists:map(
fun({ok, #{node := Node, release := Release}}) ->
#{node => Node, version => Release}
end,
Ready
),
?SLOG(warning, #{
msg => "all_available_nodes_running_newer_version",
hint =>
"Booting this node without syncing cluster config from peer core nodes "
"because other nodes are running a newer version",
peer_nodes => NodesAndVersions
}),
{ok, ?DEFAULT_INIT_TXN_ID};
Ready2 ->
sync_cluster_conf4(Ready2)
end.
%% @private Some core nodes are running and replied with their configs successfully.
%% Try to sort the results and save the first one for local use.
sync_cluster_conf4(Ready) ->
[{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready),
#{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info,
HasDeprecatedFile = has_deprecated_file(Info),
?SLOG(debug, #{
msg => "sync_cluster_conf_success",
synced_from_node => Node,
has_deprecated_file => HasDeprecatedFile,
local_release => emqx_app:get_release(),
remote_release => maps:get(release, Info, "before_v5.0.24|e5.0.3"),
data_dir => emqx:data_dir(),
tnx_id => TnxId
}),
ok = emqx_config:save_to_override_conf(
HasDeprecatedFile,
RawOverrideConf,
#{override_to => cluster}
),
ok = sync_data_from_node(Node),
{ok, TnxId}.
should_proceed_with_boot() ->
TablesStatus = emqx_cluster_rpc:get_tables_status(),
LocalNode = node(),

4
apps/emqx_conf/src/emqx_conf_schema.erl
View File

@ -143,7 +143,7 @@ fields("cluster") ->
)},
{"discovery_strategy",
sc(
hoconsc:enum([manual, static, mcast, dns, etcd, k8s]),
hoconsc:enum([manual, static, dns, etcd, k8s, mcast]),
#{
default => manual,
desc => ?DESC(cluster_discovery_strategy),
@ -198,7 +198,7 @@ fields("cluster") ->
{"mcast",
sc(
?R_REF(cluster_mcast),
#{}
#{importance => ?IMPORTANCE_HIDDEN}
)},
{"dns",
sc(

39
apps/emqx_conf/test/emqx_conf_app_SUITE.erl
View File

@ -98,6 +98,34 @@ t_copy_deprecated_data_dir(_Config) ->
stop_cluster(Nodes)
end.
t_no_copy_from_newer_version_node(_Config) ->
net_kernel:start(['master2@127.0.0.1', longnames]),
ct:timetrap({seconds, 120}),
snabbkaffe:fix_ct_logging(),
Cluster = cluster([cluster_spec({core, 10}), cluster_spec({core, 11}), cluster_spec({core, 12})]),
OKs = [ok, ok, ok],
[First | Rest] = Nodes = start_cluster(Cluster),
try
File = "/configs/cluster.hocon",
assert_config_load_done(Nodes),
rpc:call(First, ?MODULE, create_data_dir, [File]),
{OKs, []} = rpc:multicall(Nodes, application, stop, [emqx_conf]),
{OKs, []} = rpc:multicall(Nodes, ?MODULE, set_data_dir_env, []),
{OKs, []} = rpc:multicall(Nodes, meck, new, [
emqx_release, [passthrough, no_history, no_link, non_strict]
]),
%% 99.9.9 is always newer than the current version
{OKs, []} = rpc:multicall(Nodes, meck, expect, [
emqx_release, version_with_prefix, 0, "e99.9.9"
]),
ok = rpc:call(First, application, start, [emqx_conf]),
{[ok, ok], []} = rpc:multicall(Rest, application, start, [emqx_conf]),
ok = assert_no_cluster_conf_copied(Rest, File),
stop_cluster(Nodes),
ok
after
stop_cluster(Nodes)
end.
%%------------------------------------------------------------------------------
%% Helper functions
%%------------------------------------------------------------------------------
@ -158,6 +186,17 @@ assert_data_copy_done([First0 | Rest], File) ->
Rest
).
assert_no_cluster_conf_copied([], _) ->
ok;
assert_no_cluster_conf_copied([Node | Nodes], File) ->
NodeStr = atom_to_list(Node),
?assertEqual(
{error, enoent},
file:read_file(NodeStr ++ File),
#{node => Node}
),
assert_no_cluster_conf_copied(Nodes, File).
assert_config_load_done(Nodes) ->
lists:foreach(
fun(Node) ->

8
apps/emqx_connector/src/emqx_connector_http.erl
View File

@ -219,7 +219,6 @@ on_start(
base_path => BasePath,
request => preprocess_request(maps:get(request, Config, undefined))
},
ok = emqx_resource:allocate_resource(InstId, pool_name, InstId),
case ehttpc_sup:start_pool(InstId, PoolOpts) of
{ok, _} -> {ok, State};
{error, {already_started, _}} -> {ok, State};
@ -231,12 +230,7 @@ on_stop(InstId, _State) ->
msg => "stopping_http_connector",
connector => InstId
}),
case emqx_resource:get_allocated_resources(InstId) of
#{pool_name := PoolName} ->
ehttpc_sup:stop_pool(PoolName);
_ ->
ok
end.
ehttpc_sup:stop_pool(InstId).
on_query(InstId, {send_message, Msg}, State) ->
case maps:get(request, State, undefined) of

87
apps/emqx_connector/src/emqx_connector_jwt.erl
View File

@ -19,15 +19,33 @@
-include_lib("emqx_connector/include/emqx_connector_tables.hrl").
-include_lib("emqx_resource/include/emqx_resource.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("jose/include/jose_jwt.hrl").
-include_lib("jose/include/jose_jws.hrl").
%% API
-export([
lookup_jwt/1,
lookup_jwt/2,
delete_jwt/2
delete_jwt/2,
ensure_jwt/1
]).
-type jwt() :: binary().
-type wrapped_jwk() :: fun(() -> jose_jwk:key()).
-type jwk() :: jose_jwk:key().
-type jwt_config() :: #{
expiration := timer:time(),
resource_id := resource_id(),
table := ets:table(),
jwk := wrapped_jwk() | jwk(),
iss := binary(),
sub := binary(),
aud := binary(),
kid := binary(),
alg := binary()
}.
-export_type([jwt_config/0, jwt/0]).
-spec lookup_jwt(resource_id()) -> {ok, jwt()} | {error, not_found}.
lookup_jwt(ResourceId) ->
@ -57,3 +75,70 @@ delete_jwt(TId, ResourceId) ->
error:badarg ->
ok
end.
%% @doc Attempts to retrieve a valid JWT from the cache. If there is
%% none or if the cached token is expired, generates an caches a fresh
%% one.
-spec ensure_jwt(jwt_config()) -> jwt().
ensure_jwt(JWTConfig) ->
#{resource_id := ResourceId, table := Table} = JWTConfig,
case lookup_jwt(Table, ResourceId) of
{error, not_found} ->
JWT = do_generate_jwt(JWTConfig),
store_jwt(JWTConfig, JWT),
JWT;
{ok, JWT0} ->
case is_about_to_expire(JWT0) of
true ->
JWT = do_generate_jwt(JWTConfig),
store_jwt(JWTConfig, JWT),
JWT;
false ->
JWT0
end
end.
%%-----------------------------------------------------------------------------------------
%% Helper fns
%%-----------------------------------------------------------------------------------------
-spec do_generate_jwt(jwt_config()) -> jwt().
do_generate_jwt(#{
expiration := ExpirationMS,
iss := Iss,
sub := Sub,
aud := Aud,
kid := KId,
alg := Alg,
jwk := WrappedJWK
}) ->
JWK = emqx_secret:unwrap(WrappedJWK),
Headers = #{
<<"alg">> => Alg,
<<"kid">> => KId
},
Now = erlang:system_time(seconds),
ExpirationS = erlang:convert_time_unit(ExpirationMS, millisecond, second),
Claims = #{
<<"iss">> => Iss,
<<"sub">> => Sub,
<<"aud">> => Aud,
<<"iat">> => Now,
<<"exp">> => Now + ExpirationS
},
JWT0 = jose_jwt:sign(JWK, Headers, Claims),
{_, JWT} = jose_jws:compact(JWT0),
JWT.
-spec store_jwt(jwt_config(), jwt()) -> ok.
store_jwt(#{resource_id := ResourceId, table := TId}, JWT) ->
true = ets:insert(TId, {{ResourceId, jwt}, JWT}),
?tp(emqx_connector_jwt_token_stored, #{resource_id => ResourceId}),
ok.
-spec is_about_to_expire(jwt()) -> boolean().
is_about_to_expire(JWT) ->
#jose_jwt{fields = #{<<"exp">> := Exp}} = jose_jwt:peek(JWT),
Now = erlang:system_time(seconds),
GraceExp = Exp - timer:seconds(5),
Now >= GraceExp.

39
apps/emqx_connector/src/emqx_connector_jwt_worker.erl
View File

@ -189,49 +189,14 @@ terminate(_Reason, State) ->
%% Helper fns
%%-----------------------------------------------------------------------------------------
-spec do_generate_jwt(state()) -> jwt().
do_generate_jwt(
#{
expiration := ExpirationMS,
iss := Iss,
sub := Sub,
aud := Aud,
kid := KId,
alg := Alg,
jwk := JWK
} = _State
) ->
Headers = #{
<<"alg">> => Alg,
<<"kid">> => KId
},
Now = erlang:system_time(seconds),
ExpirationS = erlang:convert_time_unit(ExpirationMS, millisecond, second),
Claims = #{
<<"iss">> => Iss,
<<"sub">> => Sub,
<<"aud">> => Aud,
<<"iat">> => Now,
<<"exp">> => Now + ExpirationS
},
JWT0 = jose_jwt:sign(JWK, Headers, Claims),
{_, JWT} = jose_jws:compact(JWT0),
JWT.
-spec generate_and_store_jwt(state()) -> state().
generate_and_store_jwt(State0) ->
JWT = do_generate_jwt(State0),
store_jwt(State0, JWT),
JWTConfig = maps:without([jwt, refresh_timer], State0),
JWT = emqx_connector_jwt:ensure_jwt(JWTConfig),
?tp(connector_jwt_worker_refresh, #{jwt => JWT}),
State1 = State0#{jwt := JWT},
ensure_timer(State1).
-spec store_jwt(state(), jwt()) -> ok.
store_jwt(#{resource_id := ResourceId, table := TId}, JWT) ->
true = ets:insert(TId, {{ResourceId, jwt}, JWT}),
?tp(connector_jwt_worker_token_stored, #{resource_id => ResourceId}),
ok.
-spec ensure_timer(state()) -> state().
ensure_timer(
State = #{

8
apps/emqx_connector/src/emqx_connector_ldap.erl
View File

@ -97,7 +97,6 @@ on_start(
{pool_size, PoolSize},
{auto_reconnect, ?AUTO_RECONNECT_INTERVAL}
],
ok = emqx_resource:allocate_resource(InstId, pool_name, InstId),
case emqx_resource_pool:start(InstId, ?MODULE, Opts ++ SslOpts) of
ok -> {ok, #{pool_name => InstId}};
{error, Reason} -> {error, Reason}
@ -108,12 +107,7 @@ on_stop(InstId, _State) ->
msg => "stopping_ldap_connector",
connector => InstId
}),
case emqx_resource:get_allocated_resources(InstId) of
#{pool_name := PoolName} ->
emqx_resource_pool:stop(PoolName);
_ ->
ok
end.
emqx_resource_pool:stop(InstId).
on_query(InstId, {search, Base, Filter, Attributes}, #{pool_name := PoolName} = State) ->
Request = {Base, Filter, Attributes},

8
apps/emqx_connector/src/emqx_connector_mongo.erl
View File

@ -183,7 +183,6 @@ on_start(
{worker_options, init_worker_options(maps:to_list(NConfig), SslOpts)}
],
Collection = maps:get(collection, Config, <<"mqtt">>),
ok = emqx_resource:allocate_resource(InstId, pool_name, InstId),
case emqx_resource_pool:start(InstId, ?MODULE, Opts) of
ok ->
{ok, #{
@ -200,12 +199,7 @@ on_stop(InstId, _State) ->
msg => "stopping_mongodb_connector",
connector => InstId
}),
case emqx_resource:get_allocated_resources(InstId) of
#{pool_name := PoolName} ->
emqx_resource_pool:stop(PoolName);
_ ->
ok
end.
emqx_resource_pool:stop(InstId).
on_query(
InstId,

8
apps/emqx_connector/src/emqx_connector_mysql.erl
View File

@ -124,7 +124,6 @@ on_start(
]
),
State = parse_prepare_sql(Config),
ok = emqx_resource:allocate_resource(InstId, pool_name, InstId),
case emqx_resource_pool:start(InstId, ?MODULE, Options ++ SslOpts) of
ok ->
{ok, init_prepare(State#{pool_name => InstId})};
@ -146,12 +145,7 @@ on_stop(InstId, _State) ->
msg => "stopping_mysql_connector",
connector => InstId
}),
case emqx_resource:get_allocated_resources(InstId) of
#{pool_name := PoolName} ->
emqx_resource_pool:stop(PoolName);
_ ->
ok
end.
emqx_resource_pool:stop(InstId).
on_query(InstId, {TypeOrKey, SQLOrKey}, State) ->
on_query(InstId, {TypeOrKey, SQLOrKey, [], default_timeout}, State);

8
apps/emqx_connector/src/emqx_connector_pgsql.erl
View File

@ -121,7 +121,6 @@ on_start(
{pool_size, PoolSize}
],
State = parse_prepare_sql(Config),
ok = emqx_resource:allocate_resource(InstId, pool_name, InstId),
case emqx_resource_pool:start(InstId, ?MODULE, Options ++ SslOpts) of
ok ->
{ok, init_prepare(State#{pool_name => InstId, prepare_statement => #{}})};
@ -138,12 +137,7 @@ on_stop(InstId, _State) ->
msg => "stopping postgresql connector",
connector => InstId
}),
case emqx_resource:get_allocated_resources(InstId) of
#{pool_name := PoolName} ->
emqx_resource_pool:stop(PoolName);
_ ->
ok
end.
emqx_resource_pool:stop(InstId).
on_query(InstId, {TypeOrKey, NameOrSQL}, State) ->
on_query(InstId, {TypeOrKey, NameOrSQL, []}, State);

66
apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl
View File

@ -18,7 +18,10 @@
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("jose/include/jose_jwt.hrl").
-include_lib("jose/include/jose_jws.hrl").
-include("emqx_connector_tables.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-compile([export_all, nowarn_export_all]).
@ -51,6 +54,33 @@ end_per_testcase(_TestCase, _Config) ->
insert_jwt(TId, ResourceId, JWT) ->
ets:insert(TId, {{ResourceId, jwt}, JWT}).
generate_private_key_pem() ->
PublicExponent = 65537,
Size = 2048,
Key = public_key:generate_key({rsa, Size, PublicExponent}),
DERKey = public_key:der_encode('PrivateKeyInfo', Key),
public_key:pem_encode([{'PrivateKeyInfo', DERKey, not_encrypted}]).
generate_config() ->
PrivateKeyPEM = generate_private_key_pem(),
ResourceID = emqx_guid:gen(),
#{
private_key => PrivateKeyPEM,
expiration => timer:hours(1),
resource_id => ResourceID,
table => ets:new(test_jwt_table, [ordered_set, public]),
iss => <<"issuer">>,
sub => <<"subject">>,
aud => <<"audience">>,
kid => <<"key id">>,
alg => <<"RS256">>
}.
is_expired(JWT) ->
#jose_jwt{fields = #{<<"exp">> := Exp}} = jose_jwt:peek(JWT),
Now = erlang:system_time(seconds),
Now >= Exp.
%%-----------------------------------------------------------------------------
%% Test cases
%%-----------------------------------------------------------------------------
@ -77,3 +107,39 @@ t_delete_jwt(_Config) ->
?assertEqual(ok, emqx_connector_jwt:delete_jwt(TId, ResourceId)),
?assertEqual({error, not_found}, emqx_connector_jwt:lookup_jwt(TId, ResourceId)),
ok.
t_ensure_jwt(_Config) ->
Config0 =
#{
table := Table,
resource_id := ResourceId,
private_key := PrivateKeyPEM
} = generate_config(),
JWK = jose_jwk:from_pem(PrivateKeyPEM),
Config1 = maps:without([private_key], Config0),
Expiration = timer:seconds(10),
JWTConfig = Config1#{jwk => JWK, expiration := Expiration},
?assertEqual({error, not_found}, emqx_connector_jwt:lookup_jwt(Table, ResourceId)),
?check_trace(
begin
JWT0 = emqx_connector_jwt:ensure_jwt(JWTConfig),
?assertNot(is_expired(JWT0)),
%% should refresh 5 s before expiration
ct:sleep(Expiration - 5500),
JWT1 = emqx_connector_jwt:ensure_jwt(JWTConfig),
?assertNot(is_expired(JWT1)),
%% fully expired
ct:sleep(2 * Expiration),
JWT2 = emqx_connector_jwt:ensure_jwt(JWTConfig),
?assertNot(is_expired(JWT2)),
{JWT0, JWT1, JWT2}
end,
fun({JWT0, JWT1, JWT2}, Trace) ->
?assertNotEqual(JWT0, JWT1),
?assertNotEqual(JWT1, JWT2),
?assertNotEqual(JWT2, JWT0),
?assertMatch([_, _, _], ?of_kind(emqx_connector_jwt_token_stored, Trace)),
ok
end
),
ok.

4
apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl
View File

@ -176,7 +176,7 @@ t_refresh(_Config) ->
{{ok, _Pid}, {ok, _Event}} =
?wait_async_action(
emqx_connector_jwt_worker:start_link(Config),
#{?snk_kind := connector_jwt_worker_token_stored},
#{?snk_kind := emqx_connector_jwt_token_stored},
5_000
),
{ok, FirstJWT} = emqx_connector_jwt:lookup_jwt(Table, ResourceId),
@ -209,7 +209,7 @@ t_refresh(_Config) ->
fun({FirstJWT, SecondJWT, ThirdJWT}, Trace) ->
?assertMatch(
[_, _, _ | _],
?of_kind(connector_jwt_worker_token_stored, Trace)
?of_kind(emqx_connector_jwt_token_stored, Trace)
),
?assertNotEqual(FirstJWT, SecondJWT),
?assertNotEqual(SecondJWT, ThirdJWT),

3
apps/emqx_dashboard/include/emqx_dashboard.hrl
View File

@ -61,7 +61,8 @@
-define(GAUGE_SAMPLER_LIST, [
subscriptions,
topics,
connections
connections,
live_connections
]).
-define(SAMPLER_LIST, ?GAUGE_SAMPLER_LIST ++ ?DELTA_SAMPLER_LIST).

1
apps/emqx_dashboard/src/emqx_dashboard_monitor.erl
View File

@ -401,6 +401,7 @@ getstats(Key) ->
end.
stats(connections) -> emqx_stats:getstat('connections.count');
stats(live_connections) -> emqx_stats:getstat('live_connections.count');
stats(topics) -> emqx_stats:getstat('topics.count');
stats(subscriptions) -> emqx_stats:getstat('subscriptions.count');
stats(received) -> emqx_metrics:val('messages.received');

5
apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl
View File

@ -171,6 +171,11 @@ swagger_desc(topics) ->
" Can only represent the approximate state"
>>;
swagger_desc(connections) ->
<<
"Sessions at the time of sampling."
" Can only represent the approximate state"
>>;
swagger_desc(live_connections) ->
<<
"Connections at the time of sampling."
" Can only represent the approximate state"

21
apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl
View File

@ -90,6 +90,27 @@ t_monitor_current_api(_) ->
],
ok.
t_monitor_current_api_live_connections(_) ->
process_flag(trap_exit, true),
ClientId = <<"live_conn_tests">>,
ClientId1 = <<"live_conn_tests1">>,
{ok, C} = emqtt:start_link([{clean_start, false}, {clientid, ClientId}]),
{ok, _} = emqtt:connect(C),
ok = emqtt:disconnect(C),
{ok, C1} = emqtt:start_link([{clean_start, true}, {clientid, ClientId1}]),
{ok, _} = emqtt:connect(C1),
%% waiting for emqx_stats ticker
timer:sleep(1500),
_ = emqx_dashboard_monitor:current_rate(),
{ok, Rate} = request(["monitor_current"]),
?assertEqual(1, maps:get(<<"live_connections">>, Rate)),
?assertEqual(2, maps:get(<<"connections">>, Rate)),
%% clears
ok = emqtt:disconnect(C1),
{ok, C2} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]),
{ok, _} = emqtt:connect(C2),
ok = emqtt:disconnect(C2).
t_monitor_reset(_) ->
restart_monitor(),
{ok, Rate} = request(["monitor_current"]),

3
apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl
View File

@ -11,6 +11,7 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx/include/asserts.hrl").
-include_lib("emqx/include/emqx_cm.hrl").
-import(
emqx_eviction_agent_test_helpers,
@ -295,7 +296,7 @@ t_session_serialization(_Config) ->
?assertMatch(
#{data := [#{clientid := <<"client_with_session">>}]},
emqx_mgmt_api:cluster_query(
emqx_channel_info,
?CHAN_INFO_TAB,
#{},
[],
fun emqx_mgmt_api_clients:qs2ms/2,

7
apps/emqx_exhook/src/emqx_exhook_app.erl
View File

@ -22,8 +22,7 @@
-export([
start/2,
stop/1,
prep_stop/1
stop/1
]).
%%--------------------------------------------------------------------
@ -34,10 +33,6 @@ start(_StartType, _StartArgs) ->
{ok, Sup} = emqx_exhook_sup:start_link(),
{ok, Sup}.
prep_stop(State) ->
emqx_ctl:unregister_command(exhook),
State.
stop(_State) ->
ok.

162
apps/emqx_exhook/src/emqx_exhook_mgr.erl
View File

@ -23,6 +23,9 @@
-include_lib("emqx/include/logger.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-define(SERVERS, [exhook, servers]).
-define(EXHOOK, [exhook]).
%% APIs
-export([start_link/0]).
@ -148,7 +151,7 @@ update_config(KeyPath, UpdateReq) ->
Error
end.
pre_config_update(_, {add, #{<<"name">> := Name} = Conf}, OldConf) ->
pre_config_update(?SERVERS, {add, #{<<"name">> := Name} = Conf}, OldConf) ->
case lists:any(fun(#{<<"name">> := ExistedName}) -> ExistedName =:= Name end, OldConf) of
true ->
throw(already_exists);
@ -156,47 +159,35 @@ pre_config_update(_, {add, #{<<"name">> := Name} = Conf}, OldConf) ->
NConf = maybe_write_certs(Conf),
{ok, OldConf ++ [NConf]}
end;
pre_config_update(_, {update, Name, Conf}, OldConf) ->
case replace_conf(Name, fun(_) -> Conf end, OldConf) of
not_found -> throw(not_found);
NewConf -> {ok, lists:map(fun maybe_write_certs/1, NewConf)}
end;
pre_config_update(_, {delete, ToDelete}, OldConf) ->
case do_delete(ToDelete, OldConf) of
not_found -> throw(not_found);
NewConf -> {ok, NewConf}
end;
pre_config_update(_, {move, Name, Position}, OldConf) ->
case do_move(Name, Position, OldConf) of
not_found -> throw(not_found);
NewConf -> {ok, NewConf}
end;
pre_config_update(_, {enable, Name, Enable}, OldConf) ->
case
replace_conf(
Name,
fun(Conf) -> Conf#{<<"enable">> => Enable} end,
OldConf
)
of
not_found -> throw(not_found);
NewConf -> {ok, lists:map(fun maybe_write_certs/1, NewConf)}
end.
pre_config_update(?SERVERS, {update, Name, Conf}, OldConf) ->
NewConf = replace_conf(Name, fun(_) -> Conf end, OldConf),
{ok, lists:map(fun maybe_write_certs/1, NewConf)};
pre_config_update(?SERVERS, {delete, ToDelete}, OldConf) ->
{ok, do_delete(ToDelete, OldConf)};
pre_config_update(?SERVERS, {move, Name, Position}, OldConf) ->
{ok, do_move(Name, Position, OldConf)};
pre_config_update(?SERVERS, {enable, Name, Enable}, OldConf) ->
ReplaceFun = fun(Conf) -> Conf#{<<"enable">> => Enable} end,
NewConf = replace_conf(Name, ReplaceFun, OldConf),
{ok, lists:map(fun maybe_write_certs/1, NewConf)};
pre_config_update(?EXHOOK, NewConf, _OldConf) when NewConf =:= #{} ->
{ok, NewConf#{<<"servers">> => []}};
pre_config_update(?EXHOOK, NewConf = #{<<"servers">> := Servers}, _OldConf) ->
{ok, NewConf#{<<"servers">> => lists:map(fun maybe_write_certs/1, Servers)}}.
post_config_update(_KeyPath, UpdateReq, NewConf, _OldConf, _AppEnvs) ->
Result = call({update_config, UpdateReq, NewConf}),
post_config_update(_KeyPath, UpdateReq, NewConf, OldConf, _AppEnvs) ->
Result = call({update_config, UpdateReq, NewConf, OldConf}),
{ok, Result}.
%%=====================================================================
%%--------------------------------------------------------------------
%% gen_server callbacks
%%--------------------------------------------------------------------
init([]) ->
process_flag(trap_exit, true),
emqx_conf:add_handler([exhook, servers], ?MODULE),
ServerL = emqx:get_config([exhook, servers]),
emqx_conf:add_handler(?EXHOOK, ?MODULE),
emqx_conf:add_handler(?SERVERS, ?MODULE),
ServerL = emqx:get_config(?SERVERS),
Servers = load_all_servers(ServerL),
Servers2 = reorder(ServerL, Servers),
refresh_tick(),
@ -221,22 +212,16 @@ handle_call(
OrderServers = sort_name_by_order(Infos, Servers),
{reply, OrderServers, State};
handle_call(
{update_config, {move, _Name, _Position}, NewConfL},
{update_config, {move, _Name, _Position}, NewConfL, _},
_From,
#{servers := Servers} = State
) ->
Servers2 = reorder(NewConfL, Servers),
{reply, ok, State#{servers := Servers2}};
handle_call({update_config, {delete, ToDelete}, _}, _From, State) ->
emqx_exhook_metrics:on_server_deleted(ToDelete),
#{servers := Servers} = State2 = do_unload_server(ToDelete, State),
Servers2 = maps:remove(ToDelete, Servers),
{reply, ok, update_servers(Servers2, State2)};
handle_call({update_config, {delete, ToDelete}, _, _}, _From, State) ->
{reply, ok, remove_server(ToDelete, State)};
handle_call(
{update_config, {add, RawConf}, NewConfL},
{update_config, {add, RawConf}, NewConfL, _},
_From,
#{servers := Servers} = State
) ->
@ -245,14 +230,30 @@ handle_call(
Servers2 = Servers#{Name => Server},
Servers3 = reorder(NewConfL, Servers2),
{reply, Result, State#{servers := Servers3}};
handle_call({update_config, {update, Name, _Conf}, NewConfL, _}, _From, State) ->
{Result, State2} = restart_server(Name, NewConfL, State),
{reply, Result, State2};
handle_call({update_config, {enable, Name, _Enable}, NewConfL, _}, _From, State) ->
{Result, State2} = restart_server(Name, NewConfL, State),
{reply, Result, State2};
handle_call({update_config, _, ConfL, ConfL}, _From, State) ->
{reply, ok, State};
handle_call({update_config, _, #{servers := NewConfL}, #{servers := OldConfL}}, _From, State) ->
#{
removed := Removed,
added := Added,
changed := Updated
} = emqx_utils:diff_lists(NewConfL, OldConfL, fun(#{name := Name}) -> Name end),
State2 = remove_servers(Removed, State),
{UpdateRes, State3} = restart_servers(Updated, NewConfL, State2),
{AddRes, State4 = #{servers := Servers4}} = add_servers(Added, State3),
State5 = State4#{servers => reorder(NewConfL, Servers4)},
case UpdateRes =:= [] andalso AddRes =:= [] of
true -> {reply, ok, State5};
false -> {reply, {error, #{added => AddRes, updated => UpdateRes}}, State5}
end;
handle_call({lookup, Name}, _From, State) ->
{reply, where_is_server(Name, State), State};
handle_call({update_config, {update, Name, _Conf}, NewConfL}, _From, State) ->
{Result, State2} = restart_server(Name, NewConfL, State),
{reply, Result, State2};
handle_call({update_config, {enable, Name, _Enable}, NewConfL}, _From, State) ->
{Result, State2} = restart_server(Name, NewConfL, State),
{reply, Result, State2};
handle_call({server_info, Name}, _From, State) ->
case where_is_server(Name, State) of
not_found ->
@ -286,6 +287,22 @@ handle_call(_Request, _From, State) ->
Reply = ok,
{reply, Reply, State}.
remove_servers(Removes, State) ->
lists:foldl(
fun(Conf, Acc) ->
ToDelete = maps:get(name, Conf),
remove_server(ToDelete, Acc)
end,
State,
Removes
).
remove_server(ToDelete, State) ->
emqx_exhook_metrics:on_server_deleted(ToDelete),
#{servers := Servers} = State2 = do_unload_server(ToDelete, State),
Servers2 = maps:remove(ToDelete, Servers),
update_servers(Servers2, State2).
handle_cast(_Msg, State) ->
{noreply, State}.
@ -309,6 +326,8 @@ terminate(Reason, State = #{servers := Servers}) ->
Servers
),
?tp(info, exhook_mgr_terminated, #{reason => Reason, servers => Servers}),
emqx_conf:remove_handler(?SERVERS),
emqx_conf:remove_handler(?EXHOOK),
ok.
code_change(_OldVsn, State, _Extra) ->
@ -324,6 +343,22 @@ unload_exhooks() ->
|| {Name, {M, F, _A}} <- ?ENABLED_HOOKS
].
add_servers(Added, State) ->
lists:foldl(
fun(Conf = #{name := Name}, {ResAcc, StateAcc}) ->
case do_load_server(options_to_server(Conf)) of
{ok, Server} ->
#{servers := Servers} = StateAcc,
Servers2 = Servers#{Name => Server},
{ResAcc, update_servers(Servers2, StateAcc)};
{Err, StateAcc1} ->
{[Err | ResAcc], StateAcc1}
end
end,
{[], State},
Added
).
do_load_server(#{name := Name} = Server) ->
case emqx_exhook_server:load(Name, Server) of
{ok, ServerState} ->
@ -400,8 +435,7 @@ clean_reload_timer(#{timer := Timer}) ->
_ = erlang:cancel_timer(Timer),
ok.
-spec do_move(binary(), position(), list(server_options())) ->
not_found | list(server_options()).
-spec do_move(binary(), position(), list(server_options())) -> list(server_options()).
do_move(Name, Position, ConfL) ->
move(ConfL, Name, Position, []).
@ -410,7 +444,7 @@ move([#{<<"name">> := Name} = Server | T], Name, Position, HeadL) ->
move([Server | T], Name, Position, HeadL) ->
move(T, Name, Position, [Server | HeadL]);
move([], _Name, _Position, _HeadL) ->
not_found.
throw(not_found).
move_to(?CMD_MOVE_FRONT, Server, ServerL) ->
[Server | ServerL];
@ -428,8 +462,7 @@ move_to([H | T], Position, Server, HeadL) ->
move_to([], _Position, _Server, _HeadL) ->
not_found.
-spec do_delete(binary(), list(server_options())) ->
not_found | list(server_options()).
-spec do_delete(binary(), list(server_options())) -> list(server_options()).
do_delete(ToDelete, OldConf) ->
case lists:any(fun(#{<<"name">> := ExistedName}) -> ExistedName =:= ToDelete end, OldConf) of
true ->
@ -438,7 +471,7 @@ do_delete(ToDelete, OldConf) ->
OldConf
);
false ->
not_found
throw(not_found)
end.
-spec reorder(list(server_options()), servers()) -> servers().
@ -470,9 +503,7 @@ where_is_server(Name, #{servers := Servers}) ->
-type replace_fun() :: fun((server_options()) -> server_options()).
-spec replace_conf(binary(), replace_fun(), list(server_options())) ->
not_found
| list(server_options()).
-spec replace_conf(binary(), replace_fun(), list(server_options())) -> list(server_options()).
replace_conf(Name, ReplaceFun, ConfL) ->
replace_conf(ConfL, Name, ReplaceFun, []).
@ -482,7 +513,20 @@ replace_conf([#{<<"name">> := Name} = H | T], Name, ReplaceFun, HeadL) ->
replace_conf([H | T], Name, ReplaceFun, HeadL) ->
replace_conf(T, Name, ReplaceFun, [H | HeadL]);
replace_conf([], _, _, _) ->
not_found.
throw(not_found).
restart_servers(Updated, NewConfL, State) ->
lists:foldl(
fun({_Old, Conf}, {ResAcc, StateAcc}) ->
Name = maps:get(name, Conf),
case restart_server(Name, NewConfL, StateAcc) of
{ok, StateAcc1} -> {ResAcc, StateAcc1};
{Err, StateAcc1} -> {[Err | ResAcc], StateAcc1}
end
end,
{[], State},
Updated
).
-spec restart_server(binary(), list(server_options()), state()) ->
{ok, state()}

45
apps/emqx_exhook/test/emqx_exhook_SUITE.erl
View File

@ -196,9 +196,9 @@ t_error_update_conf(_) ->
Path = [exhook, servers],
Name = <<"error_update">>,
ErrorCfg = #{<<"name">> => Name},
{error, _} = emqx_exhook_mgr:update_config(Path, {update, Name, ErrorCfg}),
{error, _} = emqx_exhook_mgr:update_config(Path, {move, Name, top, <<>>}),
{error, _} = emqx_exhook_mgr:update_config(Path, {enable, Name, true}),
{error, not_found} = emqx_exhook_mgr:update_config(Path, {update, Name, ErrorCfg}),
{error, not_found} = emqx_exhook_mgr:update_config(Path, {move, Name, top}),
{error, not_found} = emqx_exhook_mgr:update_config(Path, {enable, Name, true}),
ErrorAnd = #{<<"name">> => Name, <<"url">> => <<"http://127.0.0.1:9001">>},
{ok, _} = emqx_exhook_mgr:update_config(Path, {add, ErrorAnd}),
@ -210,12 +210,37 @@ t_error_update_conf(_) ->
},
{ok, _} = emqx_exhook_mgr:update_config(Path, {update, Name, DisableAnd}),
{ok, _} = emqx_exhook_mgr:update_config(Path, {delete, <<"error">>}),
{error, not_found} = emqx_exhook_mgr:update_config(
Path, {delete, <<"delete_not_exists">>}
),
{ok, _} = emqx_exhook_mgr:update_config(Path, {delete, Name}),
{error, not_found} = emqx_exhook_mgr:update_config(Path, {delete, Name}),
ok.
t_update_conf(_Config) ->
Path = [exhook],
Conf = #{<<"servers">> := Servers} = emqx_config:get_raw(Path),
?assert(length(Servers) > 1),
Servers1 = shuffle(Servers),
ReOrderedConf = Conf#{<<"servers">> => Servers1},
validate_servers(Path, ReOrderedConf, Servers1),
[_ | Servers2] = Servers,
DeletedConf = Conf#{<<"servers">> => Servers2},
validate_servers(Path, DeletedConf, Servers2),
[L1, L2 | Servers3] = Servers,
UpdateL2 = L2#{<<"pool_size">> => 1, <<"request_timeout">> => 1000},
UpdatedServers = [L1, UpdateL2 | Servers3],
UpdatedConf = Conf#{<<"servers">> => UpdatedServers},
validate_servers(Path, UpdatedConf, UpdatedServers),
%% reset
validate_servers(Path, Conf, Servers),
ok.
validate_servers(Path, ReOrderConf, Servers1) ->
{ok, _} = emqx_exhook_mgr:update_config(Path, ReOrderConf),
?assertEqual(ReOrderConf, emqx_config:get_raw(Path)),
List = emqx_exhook_mgr:list(),
ExpectL = lists:map(fun(#{<<"name">> := Name}) -> Name end, Servers1),
L1 = lists:map(fun(#{name := Name}) -> Name end, List),
?assertEqual(ExpectL, L1).
t_error_server_info(_) ->
not_found = emqx_exhook_mgr:server_info(<<"not_exists">>),
ok.
@ -483,6 +508,10 @@ data_file(Name) ->
cert_file(Name) ->
data_file(filename:join(["certs", Name])).
%% FIXME: this creats inter-test dependency
%% FIXME: this creates inter-test dependency
stop_apps(Apps) ->
emqx_common_test_helpers:stop_apps(Apps, #{erase_all_configs => false}).
shuffle(List) ->
Sorted = lists:sort(lists:map(fun(L) -> {rand:uniform(), L} end, List)),
lists:map(fun({_, L}) -> L end, Sorted).

2
apps/emqx_gateway/src/emqx_gateway.erl
View File

@ -16,7 +16,7 @@
-module(emqx_gateway).
-include("include/emqx_gateway.hrl").
-include("emqx_gateway.hrl").
%% Gateway APIs
-export([

3
apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
View File

@ -723,7 +723,8 @@ examples_listener() ->
buffer => <<"10KB">>,
high_watermark => <<"1MB">>,
nodelay => false,
reuseaddr => true
reuseaddr => true,
keepalive => "none"
}
}
},

278
apps/emqx_gateway/src/emqx_gateway_conf.erl
View File

@ -74,18 +74,20 @@
-type listener_ref() :: {ListenerType :: atom_or_bin(), ListenerName :: atom_or_bin()}.
-define(IS_SSL(T), (T == <<"ssl_options">> orelse T == <<"dtls_options">>)).
-define(IGNORE_KEYS, [<<"listeners">>, ?AUTHN_BIN]).
%%--------------------------------------------------------------------
%% Load/Unload
%%--------------------------------------------------------------------
-define(GATEWAY, [gateway]).
-spec load() -> ok.
load() ->
emqx_conf:add_handler([gateway], ?MODULE).
emqx_conf:add_handler(?GATEWAY, ?MODULE).
-spec unload() -> ok.
unload() ->
emqx_conf:remove_handler([gateway]).
emqx_conf:remove_handler(?GATEWAY).
%%--------------------------------------------------------------------
%% APIs
@ -104,7 +106,7 @@ unconvert_listeners(Ls) when is_list(Ls) ->
lists:foldl(
fun(Lis, Acc) ->
{[Type, Name], Lis1} = maps_key_take([<<"type">>, <<"name">>], Lis),
_ = vaildate_listener_name(Name),
_ = validate_listener_name(Name),
NLis1 = maps:without([<<"id">>, <<"running">>], Lis1),
emqx_utils_maps:deep_merge(Acc, #{Type => #{Name => NLis1}})
end,
@ -122,7 +124,7 @@ maps_key_take([K | Ks], M, Acc) ->
{V, M1} -> maps_key_take(Ks, M1, [V | Acc])
end.
vaildate_listener_name(Name) ->
validate_listener_name(Name) ->
try
{match, _} = re:run(Name, "^[0-9a-zA-Z_-]+$"),
ok
@ -373,7 +375,7 @@ ret_listener_or_err(_, _, Err) ->
emqx_config:raw_config()
) ->
{ok, emqx_config:update_request()} | {error, term()}.
pre_config_update(_, {load_gateway, GwName, Conf}, RawConf) ->
pre_config_update(?GATEWAY, {load_gateway, GwName, Conf}, RawConf) ->
case maps:get(GwName, RawConf, undefined) of
undefined ->
NConf = tune_gw_certs(fun convert_certs/2, GwName, Conf),
@ -381,24 +383,20 @@ pre_config_update(_, {load_gateway, GwName, Conf}, RawConf) ->
_ ->
badres_gateway(already_exist, GwName)
end;
pre_config_update(_, {update_gateway, GwName, Conf}, RawConf) ->
pre_config_update(?GATEWAY, {update_gateway, GwName, Conf}, RawConf) ->
case maps:get(GwName, RawConf, undefined) of
undefined ->
badres_gateway(not_found, GwName);
GwRawConf ->
Conf1 = maps:without([<<"listeners">>, ?AUTHN_BIN], Conf),
Conf1 = maps:without(?IGNORE_KEYS, Conf),
NConf = tune_gw_certs(fun convert_certs/2, GwName, Conf1),
NConf1 = maps:merge(GwRawConf, NConf),
{ok, emqx_utils_maps:deep_put([GwName], RawConf, NConf1)}
end;
pre_config_update(_, {unload_gateway, GwName}, RawConf) ->
pre_config_update(?GATEWAY, {unload_gateway, GwName}, RawConf) ->
{ok, maps:remove(GwName, RawConf)};
pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) ->
case
emqx_utils_maps:deep_get(
[GwName, <<"listeners">>, LType, LName], RawConf, undefined
)
of
pre_config_update(?GATEWAY, {add_listener, GwName, {LType, LName}, Conf}, RawConf) ->
case get_listener(GwName, LType, LName, RawConf) of
undefined ->
NConf = convert_certs(certs_dir(GwName), Conf),
NListener = #{LType => #{LName => NConf}},
@ -410,12 +408,8 @@ pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) ->
_ ->
badres_listener(already_exist, GwName, LType, LName)
end;
pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) ->
case
emqx_utils_maps:deep_get(
[GwName, <<"listeners">>, LType, LName], RawConf, undefined
)
of
pre_config_update(?GATEWAY, {update_listener, GwName, {LType, LName}, Conf}, RawConf) ->
case get_listener(GwName, LType, LName, RawConf) of
undefined ->
badres_listener(not_found, GwName, LType, LName);
_OldConf ->
@ -427,20 +421,16 @@ pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) -
),
{ok, NRawConf}
end;
pre_config_update(_, {remove_listener, GwName, {LType, LName}}, RawConf) ->
Path = [GwName, <<"listeners">>, LType, LName],
case emqx_utils_maps:deep_get(Path, RawConf, undefined) of
pre_config_update(?GATEWAY, {remove_listener, GwName, {LType, LName}}, RawConf) ->
case get_listener(GwName, LType, LName, RawConf) of
undefined ->
{ok, RawConf};
_OldConf ->
Path = [GwName, <<"listeners">>, LType, LName],
{ok, emqx_utils_maps:deep_remove(Path, RawConf)}
end;
pre_config_update(_, {add_authn, GwName, Conf}, RawConf) ->
case
emqx_utils_maps:deep_get(
[GwName, ?AUTHN_BIN], RawConf, undefined
)
of
pre_config_update(?GATEWAY, {add_authn, GwName, Conf}, RawConf) ->
case get_authn(GwName, RawConf) of
undefined ->
CertsDir = authn_certs_dir(GwName, Conf),
Conf1 = emqx_authentication_config:convert_certs(CertsDir, Conf),
@ -452,14 +442,8 @@ pre_config_update(_, {add_authn, GwName, Conf}, RawConf) ->
_ ->
badres_authn(already_exist, GwName)
end;
pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
case
emqx_utils_maps:deep_get(
[GwName, <<"listeners">>, LType, LName],
RawConf,
undefined
)
of
pre_config_update(?GATEWAY, {add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
case get_listener(GwName, LType, LName, RawConf) of
undefined ->
badres_listener(not_found, GwName, LType, LName);
Listener ->
@ -480,9 +464,9 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
badres_listener_authn(already_exist, GwName, LType, LName)
end
end;
pre_config_update(_, {update_authn, GwName, Conf}, RawConf) ->
pre_config_update(?GATEWAY, {update_authn, GwName, Conf}, RawConf) ->
Path = [GwName, ?AUTHN_BIN],
case emqx_utils_maps:deep_get(Path, RawConf, undefined) of
case get_authn(GwName, RawConf) of
undefined ->
badres_authn(not_found, GwName);
_OldConf ->
@ -490,9 +474,9 @@ pre_config_update(_, {update_authn, GwName, Conf}, RawConf) ->
Conf1 = emqx_authentication_config:convert_certs(CertsDir, Conf),
{ok, emqx_utils_maps:deep_put(Path, RawConf, Conf1)}
end;
pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
pre_config_update(?GATEWAY, {update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
Path = [GwName, <<"listeners">>, LType, LName],
case emqx_utils_maps:deep_get(Path, RawConf, undefined) of
case get_listener(GwName, LType, LName, RawConf) of
undefined ->
badres_listener(not_found, GwName, LType, LName);
Listener ->
@ -510,16 +494,190 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
{ok, emqx_utils_maps:deep_put(Path, RawConf, NListener)}
end
end;
pre_config_update(_, {remove_authn, GwName}, RawConf) ->
pre_config_update(?GATEWAY, {remove_authn, GwName}, RawConf) ->
Path = [GwName, ?AUTHN_BIN],
{ok, emqx_utils_maps:deep_remove(Path, RawConf)};
pre_config_update(_, {remove_authn, GwName, {LType, LName}}, RawConf) ->
pre_config_update(?GATEWAY, {remove_authn, GwName, {LType, LName}}, RawConf) ->
Path = [GwName, <<"listeners">>, LType, LName, ?AUTHN_BIN],
{ok, emqx_utils_maps:deep_remove(Path, RawConf)};
pre_config_update(_, UnknownReq, _RawConf) ->
logger:error("Unknown configuration update request: ~0p", [UnknownReq]),
pre_config_update(?GATEWAY, NewRawConf0 = #{}, OldRawConf = #{}) ->
%% FIXME don't support gateway's listener's authn update.
%% load all authentications
NewRawConf1 = pre_load_authentications(NewRawConf0, OldRawConf),
%% load all listeners
NewRawConf2 = pre_load_listeners(NewRawConf1, OldRawConf),
%% load all gateway
NewRawConf3 = pre_load_gateways(NewRawConf2, OldRawConf),
{ok, NewRawConf3};
pre_config_update(Path, UnknownReq, _RawConf) ->
?SLOG(error, #{
msg => "unknown_gateway_update_request",
request => UnknownReq,
path => Path
}),
{error, badreq}.
pre_load_gateways(NewConf, OldConf) ->
%% unload old gateways
maps:foreach(
fun(GwName, _OldGwConf) ->
case maps:find(GwName, NewConf) of
error -> pre_config_update(?GATEWAY, {unload_gateway, GwName}, OldConf);
_ -> ok
end
end,
OldConf
),
%% load/update gateways
maps:map(
fun(GwName, NewGwConf) ->
case maps:find(GwName, OldConf) of
{ok, NewGwConf} ->
NewGwConf;
{ok, _OldGwConf} ->
{ok, #{GwName := NewGwConf1}} = pre_config_update(
?GATEWAY, {update_gateway, GwName, NewGwConf}, OldConf
),
%% update gateway should pass through ignore keys(listener/authn)
PassThroughConf = maps:with(?IGNORE_KEYS, NewGwConf),
NewGwConf2 = maps:without(?IGNORE_KEYS, NewGwConf1),
maps:merge(NewGwConf2, PassThroughConf);
error ->
{ok, #{GwName := NewGwConf1}} = pre_config_update(
?GATEWAY, {load_gateway, GwName, NewGwConf}, OldConf
),
NewGwConf1
end
end,
NewConf
).
pre_load_listeners(NewConf, OldConf) ->
%% remove listeners
maps:foreach(
fun(GwName, GwConf) ->
Listeners = maps:get(<<"listeners">>, GwConf, #{}),
remove_listeners(GwName, NewConf, OldConf, Listeners)
end,
OldConf
),
%% add/update listeners
maps:map(
fun(GwName, GwConf) ->
Listeners = maps:get(<<"listeners">>, GwConf, #{}),
NewListeners = create_or_update_listeners(GwName, OldConf, Listeners),
maps:put(<<"listeners">>, NewListeners, GwConf)
end,
NewConf
).
create_or_update_listeners(GwName, OldConf, Listeners) ->
maps:map(
fun(LType, LConf) ->
maps:map(
fun(LName, LConf1) ->
NConf =
case get_listener(GwName, LType, LName, OldConf) of
undefined ->
{ok, NConf0} =
pre_config_update(
?GATEWAY,
{add_listener, GwName, {LType, LName}, LConf1},
OldConf
),
NConf0;
_ ->
{ok, NConf0} =
pre_config_update(
?GATEWAY,
{update_listener, GwName, {LType, LName}, LConf1},
OldConf
),
NConf0
end,
get_listener(GwName, LType, LName, NConf)
end,
LConf
)
end,
Listeners
).
remove_listeners(GwName, NewConf, OldConf, Listeners) ->
maps:foreach(
fun(LType, LConf) ->
maps:foreach(
fun(LName, _LConf1) ->
case get_listener(GwName, LType, LName, NewConf) of
undefined ->
pre_config_update(
?GATEWAY, {remove_listener, GwName, {LType, LName}}, OldConf
);
_ ->
ok
end
end,
LConf
)
end,
Listeners
).
get_listener(GwName, LType, LName, NewConf) ->
emqx_utils_maps:deep_get(
[GwName, <<"listeners">>, LType, LName], NewConf, undefined
).
get_authn(GwName, Conf) ->
emqx_utils_maps:deep_get([GwName, ?AUTHN_BIN], Conf, undefined).
pre_load_authentications(NewConf, OldConf) ->
%% remove authentications when not in new config
maps:foreach(
fun(GwName, OldGwConf) ->
case
maps:get(?AUTHN_BIN, OldGwConf, undefined) =/= undefined andalso
get_authn(GwName, NewConf) =:= undefined
of
true ->
pre_config_update(?GATEWAY, {remove_authn, GwName}, OldConf);
false ->
ok
end
end,
OldConf
),
%% add/update authentications
maps:map(
fun(GwName, NewGwConf) ->
case get_authn(GwName, OldConf) of
undefined ->
case maps:get(?AUTHN_BIN, NewGwConf, undefined) of
undefined ->
NewGwConf;
AuthN ->
{ok, #{GwName := #{?AUTHN_BIN := NAuthN}}} =
pre_config_update(?GATEWAY, {add_authn, GwName, AuthN}, OldConf),
maps:put(?AUTHN_BIN, NAuthN, NewGwConf)
end;
OldAuthN ->
case maps:get(?AUTHN_BIN, NewGwConf, undefined) of
undefined ->
NewGwConf;
OldAuthN ->
NewGwConf;
NewAuthN ->
{ok, #{GwName := #{?AUTHN_BIN := NAuthN}}} =
pre_config_update(
?GATEWAY, {update_authn, GwName, NewAuthN}, OldConf
),
maps:put(?AUTHN_BIN, NAuthN, NewGwConf)
end
end
end,
NewConf
).
badres_gateway(not_found, GwName) ->
{error,
{badres, #{
@ -593,7 +751,7 @@ badres_listener_authn(already_exist, GwName, LType, LName) ->
) ->
ok | {ok, Result :: any()} | {error, Reason :: term()}.
post_config_update(_, Req, NewConfig, OldConfig, _AppEnvs) when is_tuple(Req) ->
post_config_update(?GATEWAY, Req, NewConfig, OldConfig, _AppEnvs) when is_tuple(Req) ->
[_Tag, GwName0 | _] = tuple_to_list(Req),
GwName = binary_to_existing_atom(GwName0),
@ -608,11 +766,35 @@ post_config_update(_, Req, NewConfig, OldConfig, _AppEnvs) when is_tuple(Req) ->
{New, Old} when is_map(New), is_map(Old) ->
emqx_gateway:update(GwName, New)
end;
post_config_update(_, _Req, _NewConfig, _OldConfig, _AppEnvs) ->
post_config_update(?GATEWAY, _Req = #{}, NewConfig, OldConfig, _AppEnvs) ->
%% unload gateways
maps:foreach(
fun(GwName, _OldGwConf) ->
case maps:get(GwName, NewConfig, undefined) of
undefined ->
emqx_gateway:unload(GwName);
_ ->
ok
end
end,
OldConfig
),
%% load/update gateways
maps:foreach(
fun(GwName, NewGwConf) ->
case maps:get(GwName, OldConfig, undefined) of
undefined ->
emqx_gateway:load(GwName, NewGwConf);
_ ->
emqx_gateway:update(GwName, NewGwConf)
end
end,
NewConfig
),
ok.
%%--------------------------------------------------------------------
%% Internal funcs
%% Internal functions
%%--------------------------------------------------------------------
tune_gw_certs(Fun, GwName, Conf) ->

2
apps/emqx_gateway/src/emqx_gateway_metrics.erl
View File

@ -18,7 +18,7 @@
-behaviour(gen_server).
-include_lib("emqx_gateway/include/emqx_gateway.hrl").
-include("emqx_gateway.hrl").
%% APIs
-export([start_link/1]).

13
apps/emqx_gateway/src/emqx_gateway_schema.erl
View File

@ -120,7 +120,10 @@ fields(ssl_listener) ->
{ssl_options,
sc(
hoconsc:ref(emqx_schema, "listener_ssl_opts"),
#{desc => ?DESC(ssl_listener_options)}
#{
desc => ?DESC(ssl_listener_options),
validator => fun emqx_schema:validate_server_ssl_opts/1
}
)}
];
fields(udp_listener) ->
@ -132,7 +135,13 @@ fields(udp_listener) ->
fields(dtls_listener) ->
[{acceptors, sc(integer(), #{default => 16, desc => ?DESC(dtls_listener_acceptors)})}] ++
fields(udp_listener) ++
[{dtls_options, sc(ref(dtls_opts), #{desc => ?DESC(dtls_listener_dtls_opts)})}];
[
{dtls_options,
sc(ref(dtls_opts), #{
desc => ?DESC(dtls_listener_dtls_opts),
validator => fun emqx_schema:validate_server_ssl_opts/1
})}
];
fields(udp_opts) ->
[
{active_n,

188
apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl
View File

@ -277,6 +277,48 @@ t_load_unload_gateway(_) ->
{config_not_found, [<<"gateway">>, stomp]},
emqx:get_raw_config([gateway, stomp])
),
%% test update([gateway], Conf)
Raw0 = emqx:get_raw_config([gateway]),
#{<<"listeners">> := StompConfL1} = StompConf1,
StompConf11 = StompConf1#{
<<"listeners">> => emqx_gateway_conf:unconvert_listeners(StompConfL1)
},
#{<<"listeners">> := StompConfL2} = StompConf2,
StompConf22 = StompConf2#{
<<"listeners">> => emqx_gateway_conf:unconvert_listeners(StompConfL2)
},
Raw1 = Raw0#{<<"stomp">> => StompConf11},
Raw2 = Raw0#{<<"stomp">> => StompConf22},
?assertMatch({ok, _}, emqx:update_config([gateway], Raw1)),
assert_confs(StompConf1, emqx:get_raw_config([gateway, stomp])),
?assertMatch(
#{
config := #{
authentication := #{backend := built_in_database, enable := true},
listeners := #{tcp := #{default := #{bind := 61613}}},
mountpoint := <<"t/">>,
idle_timeout := 10000
}
},
emqx_gateway:lookup('stomp')
),
?assertMatch({ok, _}, emqx:update_config([gateway], Raw2)),
assert_confs(StompConf2, emqx:get_raw_config([gateway, stomp])),
?assertMatch(
#{
config :=
#{
authentication := #{backend := built_in_database, enable := true},
listeners := #{tcp := #{default := #{bind := 61613}}},
idle_timeout := 20000,
mountpoint := <<"t2/">>
}
},
emqx_gateway:lookup('stomp')
),
%% reset
?assertMatch({ok, _}, emqx:update_config([gateway], Raw0)),
?assertEqual(undefined, emqx_gateway:lookup('stomp')),
ok.
t_load_remove_authn(_) ->
@ -310,6 +352,40 @@ t_load_remove_authn(_) ->
{config_not_found, [<<"gateway">>, stomp, authentication]},
emqx:get_raw_config([gateway, stomp, authentication])
),
%% test update([gateway], Conf)
Raw0 = emqx:get_raw_config([gateway]),
#{<<"listeners">> := StompConfL} = StompConf,
StompConf1 = StompConf#{
<<"listeners">> => emqx_gateway_conf:unconvert_listeners(StompConfL),
<<"authentication">> => ?CONF_STOMP_AUTHN_1
},
Raw1 = maps:put(<<"stomp">>, StompConf1, Raw0),
?assertMatch({ok, _}, emqx:update_config([gateway], Raw1)),
assert_confs(StompConf1, emqx:get_raw_config([gateway, stomp])),
?assertMatch(
#{
stomp :=
#{
authn := <<"password_based:built_in_database">>,
listeners := [#{authn := <<"undefined">>, type := tcp}],
num_clients := 0
}
},
emqx_gateway:get_basic_usage_info()
),
%% reset(remove authn)
?assertMatch({ok, _}, emqx:update_config([gateway], Raw0)),
?assertMatch(
#{
stomp :=
#{
authn := <<"undefined">>,
listeners := [#{authn := <<"undefined">>, type := tcp}],
num_clients := 0
}
},
emqx_gateway:get_basic_usage_info()
),
ok.
t_load_remove_listeners(_) ->
@ -324,6 +400,7 @@ t_load_remove_listeners(_) ->
{<<"tcp">>, <<"default">>},
?CONF_STOMP_LISTENER_1
),
assert_confs(
maps:merge(StompConf, listener(?CONF_STOMP_LISTENER_1)),
emqx:get_raw_config([gateway, stomp])
@ -355,6 +432,59 @@ t_load_remove_listeners(_) ->
{config_not_found, [<<"gateway">>, stomp, listeners, tcp, default]},
emqx:get_raw_config([gateway, stomp, listeners, tcp, default])
),
%% test update([gateway], Conf)
Raw0 = emqx:get_raw_config([gateway]),
Raw1 = emqx_utils_maps:deep_put(
[<<"stomp">>, <<"listeners">>, <<"tcp">>, <<"default">>], Raw0, ?CONF_STOMP_LISTENER_1
),
?assertMatch({ok, _}, emqx:update_config([gateway], Raw1)),
assert_confs(
maps:merge(StompConf, listener(?CONF_STOMP_LISTENER_1)),
emqx:get_raw_config([gateway, stomp])
),
?assertMatch(
#{
stomp :=
#{
authn := <<"password_based:built_in_database">>,
listeners := [#{authn := <<"undefined">>, type := tcp}],
num_clients := 0
}
},
emqx_gateway:get_basic_usage_info()
),
Raw2 = emqx_utils_maps:deep_put(
[<<"stomp">>, <<"listeners">>, <<"tcp">>, <<"default">>], Raw0, ?CONF_STOMP_LISTENER_2
),
?assertMatch({ok, _}, emqx:update_config([gateway], Raw2)),
assert_confs(
maps:merge(StompConf, listener(?CONF_STOMP_LISTENER_2)),
emqx:get_raw_config([gateway, stomp])
),
?assertMatch(
#{
stomp :=
#{
authn := <<"password_based:built_in_database">>,
listeners := [#{authn := <<"undefined">>, type := tcp}],
num_clients := 0
}
},
emqx_gateway:get_basic_usage_info()
),
%% reset(remove listener)
?assertMatch({ok, _}, emqx:update_config([gateway], Raw0)),
?assertMatch(
#{
stomp :=
#{
authn := <<"password_based:built_in_database">>,
listeners := [],
num_clients := 0
}
},
emqx_gateway:get_basic_usage_info()
),
ok.
t_load_remove_listener_authn(_) ->
@ -417,6 +547,7 @@ t_load_gateway_with_certs_content(_) ->
[<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>],
emqx:get_raw_config([gateway, stomp])
),
assert_ssl_confs_files_exist(SslConf),
ok = emqx_gateway_conf:unload_gateway(<<"stomp">>),
assert_ssl_confs_files_deleted(SslConf),
?assertException(
@ -424,6 +555,25 @@ t_load_gateway_with_certs_content(_) ->
{config_not_found, [<<"gateway">>, stomp]},
emqx:get_raw_config([gateway, stomp])
),
%% test update([gateway], Conf)
Raw0 = emqx:get_raw_config([gateway]),
#{<<"listeners">> := StompConfL} = StompConf,
StompConf1 = StompConf#{
<<"listeners">> => emqx_gateway_conf:unconvert_listeners(StompConfL)
},
Raw1 = emqx_utils_maps:deep_put([<<"stomp">>], Raw0, StompConf1),
?assertMatch({ok, _}, emqx:update_config([gateway], Raw1)),
assert_ssl_confs_files_exist(SslConf),
?assertEqual(
SslConf,
emqx_utils_maps:deep_get(
[<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>],
emqx:get_raw_config([gateway, stomp])
)
),
%% reset
?assertMatch({ok, _}, emqx:update_config([gateway], Raw0)),
assert_ssl_confs_files_deleted(SslConf),
ok.
%% TODO: Comment out this test case for now, because emqx_tls_lib
@ -475,6 +625,7 @@ t_add_listener_with_certs_content(_) ->
[<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>],
emqx:get_raw_config([gateway, stomp])
),
assert_ssl_confs_files_exist(SslConf),
ok = emqx_gateway_conf:remove_listener(
<<"stomp">>, {<<"ssl">>, <<"default">>}
),
@ -492,6 +643,34 @@ t_add_listener_with_certs_content(_) ->
{config_not_found, [<<"gateway">>, stomp, listeners, ssl, default]},
emqx:get_raw_config([gateway, stomp, listeners, ssl, default])
),
%% test update([gateway], Conf)
Raw0 = emqx:get_raw_config([gateway]),
Raw1 = emqx_utils_maps:deep_put(
[<<"stomp">>, <<"listeners">>, <<"ssl">>, <<"default">>], Raw0, ?CONF_STOMP_LISTENER_SSL
),
?assertMatch({ok, _}, emqx:update_config([gateway], Raw1)),
SslConf1 = emqx_utils_maps:deep_get(
[<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>],
emqx:get_raw_config([gateway, stomp])
),
assert_ssl_confs_files_exist(SslConf1),
%% update
Raw2 = emqx_utils_maps:deep_put(
[<<"stomp">>, <<"listeners">>, <<"ssl">>, <<"default">>], Raw0, ?CONF_STOMP_LISTENER_SSL_2
),
?assertMatch({ok, _}, emqx:update_config([gateway], Raw2)),
SslConf2 =
emqx_utils_maps:deep_get(
[<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>],
emqx:get_raw_config([gateway, stomp])
),
assert_ssl_confs_files_exist(SslConf2),
%% reset
?assertMatch({ok, _}, emqx:update_config([gateway], Raw0)),
assert_ssl_confs_files_deleted(SslConf),
assert_ssl_confs_files_deleted(SslConf1),
assert_ssl_confs_files_deleted(SslConf2),
ok.
assert_ssl_confs_files_deleted(SslConf) when is_map(SslConf) ->
@ -504,6 +683,15 @@ assert_ssl_confs_files_deleted(SslConf) when is_map(SslConf) ->
end,
Ks
).
assert_ssl_confs_files_exist(SslConf) when is_map(SslConf) ->
Ks = [<<"cacertfile">>, <<"certfile">>, <<"keyfile">>],
lists:foreach(
fun(K) ->
Path = maps:get(K, SslConf),
{ok, _} = file:read_file(Path)
end,
Ks
).
%%--------------------------------------------------------------------
%% Utils

8
apps/emqx_gateway/test/emqx_gateway_test_utils.erl
View File

@ -83,13 +83,13 @@ do_assert_confs(Key, Expected, Effected) ->
maybe_unconvert_listeners(Conf) when is_map(Conf) ->
case maps:take(<<"listeners">>, Conf) of
error ->
Conf;
{Ls, Conf1} ->
{Ls, Conf1} when is_list(Ls) ->
Conf1#{
<<"listeners">> =>
emqx_gateway_conf:unconvert_listeners(Ls)
}
};
_ ->
Conf
end;
maybe_unconvert_listeners(Conf) ->
Conf.

3
apps/emqx_gateway/test/test_mqtt_broker.erl
View File

@ -24,6 +24,7 @@
-record(state, {subscriber}).
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_router.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
@ -50,7 +51,7 @@ unsubscribe(Topic) ->
gen_server:call(?MODULE, {unsubscribe, Topic}).
get_subscrbied_topics() ->
[Topic || {_Client, Topic} <- ets:tab2list(emqx_subscription)].
[Topic || {_Client, Topic} <- ets:tab2list(?SUBSCRIPTION)].
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).

2
apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_gateway_mqttsn, [
{description, "MQTT-SN Gateway"},
{vsn, "0.1.1"},
{vsn, "0.1.2"},
{registered, []},
{applications, [kernel, stdlib, emqx, emqx_gateway]},
{env, []},

5
apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl
View File

@ -1111,15 +1111,16 @@ check_pub_authz(
convert_pub_to_msg(
{TopicName, Flags, Data},
Channel = #channel{clientinfo = #{clientid := ClientId}}
Channel = #channel{clientinfo = #{clientid := ClientId, mountpoint := Mountpoint}}
) ->
#mqtt_sn_flags{qos = QoS, dup = Dup, retain = Retain} = Flags,
NewQoS = get_corrected_qos(QoS),
NTopicName = emqx_mountpoint:mount(Mountpoint, TopicName),
Message = put_message_headers(
emqx_message:make(
ClientId,
NewQoS,
TopicName,
NTopicName,
Data,
#{dup => Dup, retain => Retain},
#{}

45
apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl
View File

@ -120,6 +120,13 @@ restart_mqttsn_with_subs_resume_off() ->
Conf#{<<"subs_resume">> => <<"false">>}
).
restart_mqttsn_with_mountpoint(Mp) ->
Conf = emqx:get_raw_config([gateway, mqttsn]),
emqx_gateway_conf:update_gateway(
mqttsn,
Conf#{<<"mountpoint">> => Mp}
).
default_config() ->
?CONF_DEFAULT.
@ -990,6 +997,44 @@ t_publish_qos2_case03(_) ->
?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)),
gen_udp:close(Socket).
t_publish_mountpoint(_) ->
restart_mqttsn_with_mountpoint(<<"mp/">>),
Dup = 0,
QoS = 1,
Retain = 0,
Will = 0,
CleanSession = 0,
MsgId = 1,
TopicId1 = ?MAX_PRED_TOPIC_ID + 1,
Topic = <<"abc">>,
{ok, Socket} = gen_udp:open(0, [binary]),
ClientId = ?CLIENTID,
send_connect_msg(Socket, ClientId),
?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId),
?assertEqual(
<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>,
receive_response(Socket)
),
Payload1 = <<20, 21, 22, 23>>,
send_publish_msg_normal_topic(Socket, QoS, MsgId, TopicId1, Payload1),
?assertEqual(
<<7, ?SN_PUBACK, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)
),
timer:sleep(100),
?assertEqual(
<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
TopicId1:16, MsgId:16, <<20, 21, 22, 23>>/binary>>,
receive_response(Socket)
),
send_disconnect_msg(Socket, undefined),
restart_mqttsn_with_mountpoint(<<>>),
gen_udp:close(Socket).
t_delivery_qos1_register_invalid_topic_id(_) ->
Dup = 0,
QoS = 1,

11
apps/emqx_management/src/emqx_mgmt.erl
View File

@ -17,6 +17,8 @@
-module(emqx_mgmt).
-include("emqx_mgmt.hrl").
-include_lib("emqx/include/emqx_cm.hrl").
-elvis([{elvis_style, invalid_dynamic_call, disable}]).
-elvis([{elvis_style, god_modules, disable}]).
@ -139,7 +141,8 @@ node_info() ->
max_fds => proplists:get_value(
max_fds, lists:usort(lists:flatten(erlang:system_info(check_io)))
),
connections => ets:info(emqx_channel, size),
connections => ets:info(?CHAN_TAB, size),
live_connections => ets:info(?CHAN_LIVE_TAB, size),
node_status => 'running',
uptime => proplists:get_value(uptime, BrokerInfo),
version => iolist_to_binary(proplists:get_value(version, BrokerInfo)),
@ -487,7 +490,7 @@ subscribe([], _ClientId, _TopicTables) ->
-spec do_subscribe(emqx_types:clientid(), emqx_types:topic_filters()) ->
{subscribe, _} | {error, atom()}.
do_subscribe(ClientId, TopicTables) ->
case ets:lookup(emqx_channel, ClientId) of
case ets:lookup(?CHAN_TAB, ClientId) of
[] -> {error, channel_not_found};
[{_, Pid}] -> Pid ! {subscribe, TopicTables}
end.
@ -514,7 +517,7 @@ unsubscribe([], _ClientId, _Topic) ->
-spec do_unsubscribe(emqx_types:clientid(), emqx_types:topic()) ->
{unsubscribe, _} | {error, _}.
do_unsubscribe(ClientId, Topic) ->
case ets:lookup(emqx_channel, ClientId) of
case ets:lookup(?CHAN_TAB, ClientId) of
[] -> {error, channel_not_found};
[{_, Pid}] -> Pid ! {unsubscribe, [emqx_topic:parse(Topic)]}
end.
@ -537,7 +540,7 @@ unsubscribe_batch([], _ClientId, _Topics) ->
-spec do_unsubscribe_batch(emqx_types:clientid(), [emqx_types:topic()]) ->
{unsubscribe_batch, _} | {error, _}.
do_unsubscribe_batch(ClientId, Topics) ->
case ets:lookup(emqx_channel, ClientId) of
case ets:lookup(?CHAN_TAB, ClientId) of
[] -> {error, channel_not_found};
[{_, Pid}] -> Pid ! {unsubscribe, [emqx_topic:parse(Topic) || Topic <- Topics]}
end.

8
apps/emqx_management/src/emqx_mgmt_api_clients.erl
View File

@ -20,6 +20,7 @@
-include_lib("typerefl/include/types.hrl").
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_cm.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("emqx/include/logger.hrl").
@ -57,7 +58,6 @@
%% for batch operation
-export([do_subscribe/3]).
-define(CLIENT_QTAB, emqx_channel_info).
-define(TAGS, [<<"Clients">>]).
-define(CLIENT_QSCHEMA, [
@ -666,7 +666,7 @@ list_clients(QString) ->
case maps:get(<<"node">>, QString, undefined) of
undefined ->
emqx_mgmt_api:cluster_query(
?CLIENT_QTAB,
?CHAN_INFO_TAB,
QString,
?CLIENT_QSCHEMA,
fun ?MODULE:qs2ms/2,
@ -678,7 +678,7 @@ list_clients(QString) ->
QStringWithoutNode = maps:without([<<"node">>], QString),
emqx_mgmt_api:node_query(
Node1,
?CLIENT_QTAB,
?CHAN_INFO_TAB,
QStringWithoutNode,
?CLIENT_QSCHEMA,
fun ?MODULE:qs2ms/2,
@ -753,7 +753,7 @@ subscribe_batch(#{clientid := ClientID, topics := Topics}) ->
%% We use emqx_channel instead of emqx_channel_info (used by the emqx_mgmt:lookup_client/2),
%% as the emqx_channel_info table will only be populated after the hook `client.connected`
%% has returned. So if one want to subscribe topics in this hook, it will fail.
case ets:lookup(emqx_channel, ClientID) of
case ets:lookup(?CHAN_TAB, ClientID) of
[] ->
{404, ?CLIENTID_NOT_FOUND};
_ ->

5
apps/emqx_management/src/emqx_mgmt_api_nodes.erl
View File

@ -151,6 +151,11 @@ fields(node_info) ->
#{desc => <<"Node name">>, example => <<"emqx@127.0.0.1">>}
)},
{connections,
mk(
non_neg_integer(),
#{desc => <<"Number of clients session in this node">>, example => 0}
)},
{live_connections,
mk(
non_neg_integer(),
#{desc => <<"Number of clients currently connected to this node">>, example => 0}

3
apps/emqx_management/src/emqx_mgmt_api_topics.erl
View File

@ -17,6 +17,7 @@
-module(emqx_mgmt_api_topics).
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_router.hrl").
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
@ -111,7 +112,7 @@ do_list(Params) ->
case
emqx_mgmt_api:node_query(
node(),
emqx_route,
?ROUTE_TAB,
Params,
?TOPICS_QUERY_SCHEMA,
fun ?MODULE:qs2ms/2,

24
apps/emqx_management/src/emqx_mgmt_cli.erl
View File

@ -17,6 +17,8 @@
-module(emqx_mgmt_cli).
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_cm.hrl").
-include_lib("emqx/include/emqx_router.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx/include/logger.hrl").
@ -168,7 +170,7 @@ sort_map_list_field(Field, Map) ->
%% @doc Query clients
clients(["list"]) ->
dump(emqx_channel, client);
dump(?CHAN_TAB, client);
clients(["show", ClientId]) ->
if_client(ClientId, fun print/1);
clients(["kick", ClientId]) ->
@ -182,7 +184,7 @@ clients(_) ->
]).
if_client(ClientId, Fun) ->
case ets:lookup(emqx_channel, (bin(ClientId))) of
case ets:lookup(?CHAN_TAB, (bin(ClientId))) of
[] -> emqx_ctl:print("Not Found.~n");
[Channel] -> Fun({client, Channel})
end.
@ -191,9 +193,9 @@ if_client(ClientId, Fun) ->
%% @doc Topics Command
topics(["list"]) ->
dump(emqx_route, emqx_topic);
dump(?ROUTE_TAB, emqx_topic);
topics(["show", Topic]) ->
Routes = ets:lookup(emqx_route, bin(Topic)),
Routes = ets:lookup(?ROUTE_TAB, bin(Topic)),
[print({emqx_topic, Route}) || Route <- Routes];
topics(_) ->
emqx_ctl:usage([
@ -204,23 +206,23 @@ topics(_) ->
subscriptions(["list"]) ->
lists:foreach(
fun(Suboption) ->
print({emqx_suboption, Suboption})
print({?SUBOPTION, Suboption})
end,
ets:tab2list(emqx_suboption)
ets:tab2list(?SUBOPTION)
);
subscriptions(["show", ClientId]) ->
case ets:lookup(emqx_subid, bin(ClientId)) of
[] ->
emqx_ctl:print("Not Found.~n");
[{_, Pid}] ->
case ets:match_object(emqx_suboption, {{'_', Pid}, '_'}) of
case ets:match_object(?SUBOPTION, {{'_', Pid}, '_'}) of
[] -> emqx_ctl:print("Not Found.~n");
Suboption -> [print({emqx_suboption, Sub}) || Sub <- Suboption]
Suboption -> [print({?SUBOPTION, Sub}) || Sub <- Suboption]
end
end;
subscriptions(["add", ClientId, Topic, QoS]) ->
if_valid_qos(QoS, fun(IntQos) ->
case ets:lookup(emqx_channel, bin(ClientId)) of
case ets:lookup(?CHAN_TAB, bin(ClientId)) of
[] ->
emqx_ctl:print("Error: Channel not found!");
[{_, Pid}] ->
@ -230,7 +232,7 @@ subscriptions(["add", ClientId, Topic, QoS]) ->
end
end);
subscriptions(["del", ClientId, Topic]) ->
case ets:lookup(emqx_channel, bin(ClientId)) of
case ets:lookup(?CHAN_TAB, bin(ClientId)) of
[] ->
emqx_ctl:print("Error: Channel not found!");
[{_, Pid}] ->
@ -841,7 +843,7 @@ print({emqx_topic, #route{topic = Topic, dest = {_, Node}}}) ->
emqx_ctl:print("~ts -> ~ts~n", [Topic, Node]);
print({emqx_topic, #route{topic = Topic, dest = Node}}) ->
emqx_ctl:print("~ts -> ~ts~n", [Topic, Node]);
print({emqx_suboption, {{Topic, Pid}, Options}}) when is_pid(Pid) ->
print({?SUBOPTION, {{Topic, Pid}, Options}}) when is_pid(Pid) ->
SubId = maps:get(subid, Options),
QoS = maps:get(qos, Options, 0),
NL = maps:get(nl, Options, 0),

2
apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl
View File

@ -209,7 +209,7 @@ t_zones(_Config) ->
?assertEqual(Mqtt1, NewMqtt),
%% delete the new zones
{ok, #{}} = update_config("zones", Zones),
?assertEqual(undefined, emqx_config:get_raw([new_zone, mqtt], undefined)),
?assertEqual(undefined, emqx_config:get_raw([zones, new_zone], undefined)),
ok.
t_dashboard(_Config) ->

5
apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl
View File

@ -60,6 +60,11 @@ t_nodes_api(_) ->
Edition = maps:get(<<"edition">>, LocalNodeInfo),
?assertEqual(emqx_release:edition_longstr(), Edition),
Conns = maps:get(<<"connections">>, LocalNodeInfo),
?assertEqual(0, Conns),
LiveConns = maps:get(<<"live_connections">>, LocalNodeInfo),
?assertEqual(0, LiveConns),
NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(node())]),
{ok, NodeInfo} = emqx_mgmt_api_test_util:request_api(get, NodePath),
NodeNameResponse =

3
apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl
View File

@ -18,11 +18,10 @@
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("emqx/include/emqx_router.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-define(ROUTE_TAB, emqx_route).
all() ->
emqx_common_test_helpers:all(?MODULE).

2
apps/emqx_resource/test/emqx_resource_schema_tests.erl
View File

@ -67,7 +67,7 @@ health_check_interval_validator_test_() ->
parse_and_check_webhook_bridge(webhook_bridge_health_check_hocon(<<"3_600_000ms">>))
)},
?_assertThrow(
#{exception := "timeout value too large" ++ _},
#{exception := #{message := "timeout value too large" ++ _}},
parse_and_check_webhook_bridge(
webhook_bridge_health_check_hocon(<<"150000000000000s">>)
)

14
apps/emqx_retainer/src/emqx_retainer_index.erl
View File

@ -31,7 +31,7 @@
-type index() :: list(pos_integer()).
%% @doc Index key is a term that can be effectively searched in the index table.
-type index_key() :: {index(), {emqx_topic:words(), emqx_topic:words()}}.
-type index_key() :: {index(), {emqx_types:words(), emqx_types:words()}}.
-type match_pattern_part() :: term().
@ -42,7 +42,7 @@
%% @doc Given words of a concrete topic (`Tokens') and a list of `Indices',
%% constructs index keys for the topic and each of the indices.
%% `Fun' is called with each of these keys.
-spec foreach_index_key(fun((index_key()) -> any()), list(index()), emqx_topic:words()) -> ok.
-spec foreach_index_key(fun((index_key()) -> any()), list(index()), emqx_types:words()) -> ok.
foreach_index_key(_Fun, [], _Tokens) ->
ok;
foreach_index_key(Fun, [Index | Indices], Tokens) ->
@ -59,7 +59,7 @@ foreach_index_key(Fun, [Index | Indices], Tokens) ->
%% returns `{[2, 3], {[<<"b">>, <<"c">>], [<<"a">>, <<"d">>]}}' term.
%%
%% @see foreach_index_key/3
-spec to_index_key(index(), emqx_topic:words()) -> index_key().
-spec to_index_key(index(), emqx_types:words()) -> index_key().
to_index_key(Index, Tokens) ->
{Index, split_index_tokens(Index, Tokens, 1, [], [])}.
@ -73,7 +73,7 @@ to_index_key(Index, Tokens) ->
%%
%% @see foreach_index_key/3
%% @see to_index_key/2
-spec index_score(index(), emqx_topic:words()) -> non_neg_integer().
-spec index_score(index(), emqx_types:words()) -> non_neg_integer().
index_score(Index, Tokens) ->
index_score(Index, Tokens, 1, 0).
@ -92,7 +92,7 @@ select_index(Tokens, Indices) ->
%%
%% E.g. for `[2, 3]' index and <code>['+', <<"b">>, '+', <<"d">>]</code> wildcard topic
%% returns <code>{[2, 3], {[<<"b">>, '_'], ['_', <<"d">>]}}</code> pattern.
-spec condition(index(), emqx_topic:words()) -> match_pattern_part().
-spec condition(index(), emqx_types:words()) -> match_pattern_part().
condition(Index, Tokens) ->
{Index, condition(Index, Tokens, 1, [], [])}.
@ -100,7 +100,7 @@ condition(Index, Tokens) ->
%%
%% E.g. for <code>['+', <<"b">>, '+', <<"d">>, '#']</code> wildcard topic
%% returns <code>['_', <<"b">>, '_', <<"d">> | '_']</code> pattern.
-spec condition(emqx_topic:words()) -> match_pattern_part().
-spec condition(emqx_types:words()) -> match_pattern_part().
condition(Tokens) ->
Tokens1 = [
case W =:= '+' of
@ -118,7 +118,7 @@ condition(Tokens) ->
%%
%% E.g given `{[2, 3], {[<<"b">>, <<"c">>], [<<"a">>, <<"d">>]}}' index key
%% returns `[<<"a">>, <<"b">>, <<"c">>, <<"d">>]' topic.
-spec restore_topic(index_key()) -> emqx_topic:words().
-spec restore_topic(index_key()) -> emqx_types:words().
restore_topic({Index, {IndexTokens, OtherTokens}}) ->
restore_topic(Index, IndexTokens, OtherTokens, 1, []).

152
apps/emqx_utils/src/emqx_utils.erl
View File

@ -55,7 +55,8 @@
safe_to_existing_atom/1,
safe_to_existing_atom/2,
pub_props_to_packet/1,
safe_filename/1
safe_filename/1,
diff_lists/3
]).
-export([
@ -753,3 +754,152 @@ safe_filename(Filename) when is_binary(Filename) ->
binary:replace(Filename, <<":">>, <<"-">>, [global]);
safe_filename(Filename) when is_list(Filename) ->
lists:flatten(string:replace(Filename, ":", "-", all)).
%% @doc Compares two lists of maps and returns the differences between them in a
%% map containing four keys – 'removed', 'added', 'identical', and 'changed' –
%% each holding a list of maps. Elements are compared using key function KeyFunc
%% to extract the comparison key used for matching.
%%
%% The return value is a map with the following keys and the list of maps as its values:
%% * 'removed' – a list of maps that were present in the Old list, but not found in the New list.
%% * 'added' – a list of maps that were present in the New list, but not found in the Old list.
%% * 'identical' – a list of maps that were present in both lists and have the same comparison key value.
%% * 'changed' – a list of pairs of maps representing the changes between maps present in the New and Old lists.
%% The first map in the pair represents the map in the Old list, and the second map
%% represents the potential modification in the New list.
%% The KeyFunc parameter is a function that extracts the comparison key used
%% for matching from each map. The function should return a comparable term,
%% such as an atom, a number, or a string. This is used to determine if each
%% element is the same in both lists.
-spec diff_lists(list(T), list(T), Func) ->
#{
added := list(T),
identical := list(T),
removed := list(T),
changed := list({Old :: T, New :: T})
}
when
Func :: fun((T) -> any()),
T :: any().
diff_lists(New, Old, KeyFunc) when is_list(New) andalso is_list(Old) ->
Removed =
lists:foldl(
fun(E, RemovedAcc) ->
case search(KeyFunc(E), KeyFunc, New) of
false -> [E | RemovedAcc];
_ -> RemovedAcc
end
end,
[],
Old
),
{Added, Identical, Changed} =
lists:foldl(
fun(E, Acc) ->
{Added0, Identical0, Changed0} = Acc,
case search(KeyFunc(E), KeyFunc, Old) of
false ->
{[E | Added0], Identical0, Changed0};
E ->
{Added0, [E | Identical0], Changed0};
E1 ->
{Added0, Identical0, [{E1, E} | Changed0]}
end
end,
{[], [], []},
New
),
#{
removed => lists:reverse(Removed),
added => lists:reverse(Added),
identical => lists:reverse(Identical),
changed => lists:reverse(Changed)
}.
search(_ExpectValue, _KeyFunc, []) ->
false;
search(ExpectValue, KeyFunc, [Item | List]) ->
case KeyFunc(Item) =:= ExpectValue of
true -> Item;
false -> search(ExpectValue, KeyFunc, List)
end.
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
diff_lists_test() ->
KeyFunc = fun(#{name := Name}) -> Name end,
?assertEqual(
#{
removed => [],
added => [],
identical => [],
changed => []
},
diff_lists([], [], KeyFunc)
),
%% test removed list
?assertEqual(
#{
removed => [#{name => a, value => 1}],
added => [],
identical => [],
changed => []
},
diff_lists([], [#{name => a, value => 1}], KeyFunc)
),
%% test added list
?assertEqual(
#{
removed => [],
added => [#{name => a, value => 1}],
identical => [],
changed => []
},
diff_lists([#{name => a, value => 1}], [], KeyFunc)
),
%% test identical list
?assertEqual(
#{
removed => [],
added => [],
identical => [#{name => a, value => 1}],
changed => []
},
diff_lists([#{name => a, value => 1}], [#{name => a, value => 1}], KeyFunc)
),
Old = [
#{name => a, value => 1},
#{name => b, value => 4},
#{name => e, value => 2},
#{name => d, value => 4}
],
New = [
#{name => a, value => 1},
#{name => b, value => 2},
#{name => e, value => 2},
#{name => c, value => 3}
],
Diff = diff_lists(New, Old, KeyFunc),
?assertEqual(
#{
added => [
#{name => c, value => 3}
],
identical => [
#{name => a, value => 1},
#{name => e, value => 2}
],
removed => [
#{name => d, value => 4}
],
changed => [{#{name => b, value => 4}, #{name => b, value => 2}}]
},
Diff
),
ok.
-endif.

8
changes/ce/feat-10895.en.md
View File

@ -1,8 +0,0 @@
Refactored some bridges to avoid leaking resources during crashes at creation, including:
- TDEngine
- WebHook
- LDAP
- MongoDB
- MySQL
- PostgreSQL
- Redis

1
changes/ce/feat-10933.en.md Normal file
View File

@ -0,0 +1 @@
Add support for configuring TCP keep-alive in MQTT/TCP and MQTT/SSL listeners

4
changes/ce/feat-10948.en.md Normal file
View File

@ -0,0 +1,4 @@
Add `live_connections` field for some HTTP APIs, i.e:
- `/monitor_current`, `/monitor_current/nodes/{node}`
- `/monitor/nodes/{node}`, `/monitor`
- `/node/{node}`, `/nodes`

6
changes/ce/fix-10902.en.md Normal file
View File

@ -0,0 +1,6 @@
Avoid syncing cluser.hocon file from the nodes runing a newer version than self.
During cluster rolling upgrade, if an older version node has to restart due to whatever reason,
if it copies the cluster.hocon file from a newer version node, it may fail to start.
After this fix, the older version node will not copy the cluster.hocon file from a newer,
so it will use its own cluster.hocon file to start.

Some files were not shown because too many files have changed in this diff Show More