yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(config): new config for emqx 5.0 

new config for emqx 5.0
This commit is contained in:
Shawn 2021-07-17 18:09:08 +08:00 committed by GitHub
parent bb417e4498 f8f7493352
commit 6f58c7815d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
97 changed files with 4249 additions and 4899 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.ci/docker-compose-file/conf.cluster.env
View File

@ -1,7 +1,8 @@
EMQX_NAME=emqx
EMQX_CLUSTER__DISCOVERY=static
EMQX_CLUSTER__STATIC__SEEDS="emqx@node1.emqx.io, emqx@node2.emqx.io"
EMQX_LISTENER__TCP__EXTERNAL__PROXY_PROTOCOL=on
EMQX_LISTENER__WS__EXTERNAL__PROXY_PROTOCOL=on
EMQX_LOG__LEVEL=debug
EMQX_LOADED_PLUGINS=emqx_sn
EMQX_CLUSTER__DISCOVERY_STRATEGY=static
EMQX_CLUSTER__STATIC__SEEDS="[emqx@node1.emqx.io, emqx@node2.emqx.io]"
EMQX_ZONES__DEFAULT__LISTENERS__MQTT_TCP__PROXY_PROTOCOL=true
EMQX_ZONES__DEFAULT__LISTENERS__MQTT_WS__PROXY_PROTOCOL=true
EMQX_LOG__CONSOLE_HANDLER__ENABLE=true
EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug
EMQX_LOG__PRIMARY_LEVEL=debug

1
.ci/docker-compose-file/conf.env
View File

@ -10,5 +10,4 @@ EMQX_AUTH__PGSQL__PASSWORD=public
EMQX_AUTH__PGSQL__DATABASE=mqtt
EMQX_AUTH__REDIS__SERVER=redis_server:6379
EMQX_AUTH__REDIS__PASSWORD=public
CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
HOCON_ENV_OVERRIDE_PREFIX=EMQX_

9
.github/workflows/run_fvt_tests.yaml vendored
View File

@ -36,10 +36,9 @@ jobs:
timeout-minutes: 5
run: |
set -e -u -x
echo "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" >> .ci/docker-compose-file/conf.cluster.env
echo "HOCON_ENV_OVERRIDE_PREFIX=EMQX_" >> .ci/docker-compose-file/conf.cluster.env
echo "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s" >> .ci/docker-compose-file/conf.cluster.env
echo "EMQX_MQTT__MAX_TOPIC_ALIAS=10" >> .ci/docker-compose-file/conf.cluster.env
echo "EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s" >> .ci/docker-compose-file/conf.cluster.env
echo "EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10" >> .ci/docker-compose-file/conf.cluster.env
docker-compose \
-f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
-f .ci/docker-compose-file/docker-compose-python.yaml \
@ -118,8 +117,8 @@ jobs:
--set image.pullPolicy=Never \
--set emqxAclConfig="" \
--set image.pullPolicy=Never \
--set emqxConfig.EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s \
--set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \
--set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s \
--set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \
deploy/charts/emqx \
--debug

4314
apps/emqx/etc/emqx.conf
View File

File diff suppressed because it is too large Load Diff

6
apps/emqx/include/emqx.hrl
View File

@ -35,12 +35,6 @@
-define(ERTS_MINIMUM_REQUIRED, "10.0").
%%--------------------------------------------------------------------
%% Configs
%%--------------------------------------------------------------------
-define(NO_PRIORITY_TABLE, none).
%%--------------------------------------------------------------------
%% Topics' prefix: $SYS | $queue | $share
%%--------------------------------------------------------------------

2
apps/emqx/rebar.config
View File

@ -12,7 +12,7 @@
[ {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
, {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}}
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}}
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.3"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.10.3"}}}

24
apps/emqx/src/emqx_access_control.erl
View File

@ -28,29 +28,23 @@
-spec(authenticate(emqx_types:clientinfo()) ->
ok | {ok, binary()} | {continue, map()} | {continue, binary(), map()} | {error, term()}).
authenticate(Credential = #{zone := Zone}) ->
%% TODO: Rename to bypass_authentication
case emqx_zone:get_env(Zone, bypass_auth_plugins, false) of
true ->
ok;
false ->
run_hooks('client.authenticate', [Credential], ok)
end.
authenticate(Credential) ->
run_hooks('client.authenticate', [Credential], ok).
%% @doc Check ACL
-spec(authorize(emqx_types:clientinfo(), emqx_types:pubsub(), emqx_types:topic())
-> allow | deny).
authorize(ClientInfo, PubSub, Topic) ->
case emqx_acl_cache:is_enabled() of
-spec authorize(emqx_types:clientinfo(), emqx_types:pubsub(), emqx_types:topic())
-> allow | deny.
authorize(ClientInfo = #{zone := Zone, listener := Listener}, PubSub, Topic) ->
case emqx_acl_cache:is_enabled(Zone, Listener) of
true -> check_authorization_cache(ClientInfo, PubSub, Topic);
false -> do_authorize(ClientInfo, PubSub, Topic)
end.
check_authorization_cache(ClientInfo, PubSub, Topic) ->
case emqx_acl_cache:get_acl_cache(PubSub, Topic) of
check_authorization_cache(ClientInfo = #{zone := Zone, listener := Listener}, PubSub, Topic) ->
case emqx_acl_cache:get_acl_cache(Zone, Listener, PubSub, Topic) of
not_found ->
AclResult = do_authorize(ClientInfo, PubSub, Topic),
emqx_acl_cache:put_acl_cache(PubSub, Topic, AclResult),
emqx_acl_cache:put_acl_cache(Zone, Listener, PubSub, Topic, AclResult),
AclResult;
AclResult -> AclResult
end.

71
apps/emqx/src/emqx_acl_cache.erl
View File

@ -18,15 +18,15 @@
-include("emqx.hrl").
-export([ list_acl_cache/0
, get_acl_cache/2
, put_acl_cache/3
, cleanup_acl_cache/0
-export([ list_acl_cache/2
, get_acl_cache/4
, put_acl_cache/5
, cleanup_acl_cache/2
, empty_acl_cache/0
, dump_acl_cache/0
, get_cache_max_size/0
, get_cache_ttl/0
, is_enabled/0
, get_cache_max_size/2
, get_cache_ttl/2
, is_enabled/2
, drain_cache/0
]).
@ -50,43 +50,45 @@ cache_k(PubSub, Topic)-> {PubSub, Topic}.
cache_v(AclResult)-> {AclResult, time_now()}.
drain_k() -> {?MODULE, drain_timestamp}.
-spec(is_enabled() -> boolean()).
is_enabled() ->
application:get_env(emqx, enable_acl_cache, true).
-spec(is_enabled(atom(), atom()) -> boolean()).
is_enabled(Zone, Listener) ->
emqx_config:get_listener_conf(Zone, Listener, [acl, cache, enable]).
-spec(get_cache_max_size() -> integer()).
get_cache_max_size() ->
application:get_env(emqx, acl_cache_max_size, 32).
-spec(get_cache_max_size(atom(), atom()) -> integer()).
get_cache_max_size(Zone, Listener) ->
emqx_config:get_listener_conf(Zone, Listener, [acl, cache, max_size]).
-spec(get_cache_ttl() -> integer()).
get_cache_ttl() ->
application:get_env(emqx, acl_cache_ttl, 60000).
-spec(get_cache_ttl(atom(), atom()) -> integer()).
get_cache_ttl(Zone, Listener) ->
emqx_config:get_listener_conf(Zone, Listener, [acl, cache, ttl]).
-spec(list_acl_cache() -> [acl_cache_entry()]).
list_acl_cache() ->
cleanup_acl_cache(),
-spec(list_acl_cache(atom(), atom()) -> [acl_cache_entry()]).
list_acl_cache(Zone, Listener) ->
cleanup_acl_cache(Zone, Listener),
map_acl_cache(fun(Cache) -> Cache end).
%% We'll cleanup the cache before replacing an expired acl.
-spec(get_acl_cache(emqx_types:pubsub(), emqx_topic:topic()) -> (acl_result() | not_found)).
get_acl_cache(PubSub, Topic) ->
-spec get_acl_cache(atom(), atom(), emqx_types:pubsub(), emqx_topic:topic()) ->
acl_result() | not_found.
get_acl_cache(Zone, Listener, PubSub, Topic) ->
case erlang:get(cache_k(PubSub, Topic)) of
undefined -> not_found;
{AclResult, CachedAt} ->
if_expired(CachedAt,
if_expired(get_cache_ttl(Zone, Listener), CachedAt,
fun(false) ->
AclResult;
(true) ->
cleanup_acl_cache(),
cleanup_acl_cache(Zone, Listener),
not_found
end)
end.
%% If the cache get full, and also the latest one
%% is expired, then delete all the cache entries
-spec(put_acl_cache(emqx_types:pubsub(), emqx_topic:topic(), acl_result()) -> ok).
put_acl_cache(PubSub, Topic, AclResult) ->
MaxSize = get_cache_max_size(), true = (MaxSize =/= 0),
-spec put_acl_cache(atom(), atom(), emqx_types:pubsub(), emqx_topic:topic(), acl_result())
-> ok.
put_acl_cache(Zone, Listener, PubSub, Topic, AclResult) ->
MaxSize = get_cache_max_size(Zone, Listener), true = (MaxSize =/= 0),
Size = get_cache_size(),
case Size < MaxSize of
true ->
@ -94,7 +96,7 @@ put_acl_cache(PubSub, Topic, AclResult) ->
false ->
NewestK = get_newest_key(),
{_AclResult, CachedAt} = erlang:get(NewestK),
if_expired(CachedAt,
if_expired(get_cache_ttl(Zone, Listener), CachedAt,
fun(true) ->
% all cache expired, cleanup first
empty_acl_cache(),
@ -121,10 +123,10 @@ evict_acl_cache() ->
decr_cache_size().
%% cleanup all the expired cache entries
-spec(cleanup_acl_cache() -> ok).
cleanup_acl_cache() ->
-spec(cleanup_acl_cache(atom(), atom()) -> ok).
cleanup_acl_cache(Zone, Listener) ->
keys_queue_set(
cleanup_acl(keys_queue_get())).
cleanup_acl(get_cache_ttl(Zone, Listener), keys_queue_get())).
get_oldest_key() ->
keys_queue_pick(queue_front()).
@ -174,16 +176,16 @@ update_acl(K, V) ->
erlang:put(K, V),
keys_queue_update(K).
cleanup_acl(KeysQ) ->
cleanup_acl(TTL, KeysQ) ->
case queue:out(KeysQ) of
{{value, OldestK}, KeysQ2} ->
{_AclResult, CachedAt} = erlang:get(OldestK),
if_expired(CachedAt,
if_expired(TTL, CachedAt,
fun(false) -> KeysQ;
(true) ->
erlang:erase(OldestK),
decr_cache_size(),
cleanup_acl(KeysQ2)
cleanup_acl(TTL, KeysQ2)
end);
{empty, KeysQ} -> KeysQ
end.
@ -246,8 +248,7 @@ queue_rear() -> fun queue:get_r/1.
time_now() -> erlang:system_time(millisecond).
if_expired(CachedAt, Fun) ->
TTL = get_cache_ttl(),
if_expired(TTL, CachedAt, Fun) ->
Now = time_now(),
CurrentEvictTimestamp = persistent_term:get(drain_k(), 0),
case CachedAt =< CurrentEvictTimestamp orelse (CachedAt + TTL) =< Now of

87
apps/emqx/src/emqx_alarm.erl
View File

@ -17,6 +17,7 @@
-module(emqx_alarm).
-behaviour(gen_server).
-behaviour(emqx_config_handler).
-include("emqx.hrl").
-include("logger.hrl").
@ -29,7 +30,9 @@
-boot_mnesia({mnesia, [boot]}).
-copy_mnesia({mnesia, [copy]}).
-export([ start_link/1
-export([handle_update_config/2]).
-export([ start_link/0
, stop/0
]).
@ -75,17 +78,9 @@
}).
-record(state, {
actions :: [action()],
size_limit :: non_neg_integer(),
validity_period :: non_neg_integer(),
timer = undefined :: undefined | reference()
timer :: reference()
}).
-type action() :: log | publish | event.
-define(ACTIVATED_ALARM, emqx_activated_alarm).
-define(DEACTIVATED_ALARM, emqx_deactivated_alarm).
@ -93,7 +88,6 @@
-rlog_shard({?COMMON_SHARD, ?ACTIVATED_ALARM}).
-rlog_shard({?COMMON_SHARD, ?DEACTIVATED_ALARM}).
-ifdef(TEST).
-compile(export_all).
-compile(nowarn_export_all).
@ -124,8 +118,8 @@ mnesia(copy) ->
%% API
%%--------------------------------------------------------------------
start_link(Opts) ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [Opts], []).
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
stop() ->
gen_server:stop(?MODULE).
@ -157,27 +151,26 @@ get_alarms(activated) ->
get_alarms(deactivated) ->
gen_server:call(?MODULE, {get_alarms, deactivated}).
handle_update_config(#{<<"validity_period">> := Period0} = NewConf, OldConf) ->
?MODULE ! {update_timer, hocon_postprocess:duration(Period0)},
maps:merge(OldConf, NewConf);
handle_update_config(NewConf, OldConf) ->
maps:merge(OldConf, NewConf).
%%--------------------------------------------------------------------
%% gen_server callbacks
%%--------------------------------------------------------------------
init([]) ->
Opts = [{actions, [log, publish]}],
init([Opts]);
init([Opts]) ->
deactivate_all_alarms(),
Actions = proplists:get_value(actions, Opts),
SizeLimit = proplists:get_value(size_limit, Opts),
ValidityPeriod = timer:seconds(proplists:get_value(validity_period, Opts)),
{ok, ensure_delete_timer(#state{actions = Actions,
size_limit = SizeLimit,
validity_period = ValidityPeriod})}.
emqx_config_handler:add_handler([alarm], ?MODULE),
{ok, #state{timer = ensure_timer(undefined, get_validity_period())}}.
%% suppress dialyzer warning due to dirty read/write race condition.
%% TODO: change from dirty_read/write to transactional.
%% TODO: handle mnesia write errors.
-dialyzer([{nowarn_function, [handle_call/3]}]).
handle_call({activate_alarm, Name, Details}, _From, State = #state{actions = Actions}) ->
handle_call({activate_alarm, Name, Details}, _From, State) ->
case mnesia:dirty_read(?ACTIVATED_ALARM, Name) of
[#activated_alarm{name = Name}] ->
{reply, {error, already_existed}, State};
@ -187,17 +180,16 @@ handle_call({activate_alarm, Name, Details}, _From, State = #state{actions = Act
message = normalize_message(Name, Details),
activate_at = erlang:system_time(microsecond)},
ekka_mnesia:dirty_write(?ACTIVATED_ALARM, Alarm),
do_actions(activate, Alarm, Actions),
do_actions(activate, Alarm, emqx_config:get([alarm, actions])),
{reply, ok, State}
end;
handle_call({deactivate_alarm, Name, Details}, _From, State = #state{
actions = Actions, size_limit = SizeLimit}) ->
handle_call({deactivate_alarm, Name, Details}, _From, State) ->
case mnesia:dirty_read(?ACTIVATED_ALARM, Name) of
[] ->
{reply, {error, not_found}, State};
[Alarm] ->
deactivate_alarm(Details, SizeLimit, Actions, Alarm),
deactivate_alarm(Details, Alarm),
{reply, ok, State}
end;
@ -232,11 +224,15 @@ handle_cast(Msg, State) ->
?LOG(error, "Unexpected msg: ~p", [Msg]),
{noreply, State}.
handle_info({timeout, TRef, delete_expired_deactivated_alarm},
State = #state{timer = TRef,
validity_period = ValidityPeriod}) ->
delete_expired_deactivated_alarms(erlang:system_time(microsecond) - ValidityPeriod * 1000),
{noreply, ensure_delete_timer(State)};
handle_info({timeout, _TRef, delete_expired_deactivated_alarm},
#state{timer = TRef} = State) ->
Period = get_validity_period(),
delete_expired_deactivated_alarms(erlang:system_time(microsecond) - Period * 1000),
{noreply, State#state{timer = ensure_timer(TRef, Period)}};
handle_info({update_timer, Period}, #state{timer = TRef} = State) ->
?LOG(warning, "update the 'validity_period' timer to ~p", [Period]),
{noreply, State#state{timer = ensure_timer(TRef, Period)}};
handle_info(Info, State) ->
?LOG(error, "Unexpected info: ~p", [Info]),
@ -252,11 +248,13 @@ code_change(_OldVsn, State, _Extra) ->
%% Internal functions
%%------------------------------------------------------------------------------
deactivate_alarm(Details, SizeLimit, Actions, #activated_alarm{
activate_at = ActivateAt, name = Name, details = Details0,
message = Msg0}) ->
case SizeLimit > 0 andalso
(mnesia:table_info(?DEACTIVATED_ALARM, size) >= SizeLimit) of
get_validity_period() ->
timer:seconds(emqx_config:get([alarm, validity_period])).
deactivate_alarm(Details, #activated_alarm{activate_at = ActivateAt, name = Name,
details = Details0, message = Msg0}) ->
SizeLimit = emqx_config:get([alarm, size_limit]),
case SizeLimit > 0 andalso (mnesia:table_info(?DEACTIVATED_ALARM, size) >= SizeLimit) of
true ->
case mnesia:dirty_first(?DEACTIVATED_ALARM) of
'$end_of_table' -> ok;
@ -272,7 +270,7 @@ deactivate_alarm(Details, SizeLimit, Actions, #activated_alarm{
erlang:system_time(microsecond)),
ekka_mnesia:dirty_write(?DEACTIVATED_ALARM, HistoryAlarm),
ekka_mnesia:dirty_delete(?ACTIVATED_ALARM, Name),
do_actions(deactivate, DeActAlarm, Actions).
do_actions(deactivate, DeActAlarm, emqx_config:get([alarm, actions])).
make_deactivated_alarm(ActivateAt, Name, Details, Message, DeActivateAt) ->
#deactivated_alarm{
@ -308,9 +306,12 @@ clear_table(TableName) ->
ok
end.
ensure_delete_timer(State = #state{validity_period = ValidityPeriod}) ->
TRef = emqx_misc:start_timer(ValidityPeriod, delete_expired_deactivated_alarm),
State#state{timer = TRef}.
ensure_timer(OldTRef, Period) ->
_ = case is_reference(OldTRef) of
true -> erlang:cancel_timer(OldTRef);
false -> ok
end,
emqx_misc:start_timer(Period, delete_expired_deactivated_alarm).
delete_expired_deactivated_alarms(Checkpoint) ->
delete_expired_deactivated_alarms(mnesia:dirty_first(?DEACTIVATED_ALARM), Checkpoint).
@ -381,9 +382,9 @@ normalize_message(high_system_memory_usage, #{high_watermark := HighWatermark})
normalize_message(high_process_memory_usage, #{high_watermark := HighWatermark}) ->
list_to_binary(io_lib:format("Process memory usage is higher than ~p%", [HighWatermark]));
normalize_message(high_cpu_usage, #{usage := Usage}) ->
list_to_binary(io_lib:format("~p% cpu usage", [Usage]));
list_to_binary(io_lib:format("~s cpu usage", [Usage]));
normalize_message(too_many_processes, #{usage := Usage}) ->
list_to_binary(io_lib:format("~p% process usage", [Usage]));
list_to_binary(io_lib:format("~s process usage", [Usage]));
normalize_message(partition, #{occurred := Node}) ->
list_to_binary(io_lib:format("Partition occurs at node ~s", [Node]));
normalize_message(<<"resource", _/binary>>, #{type := Type, id := ID}) ->

16
apps/emqx/src/emqx_alarm_handler.erl
View File

@ -56,20 +56,22 @@ init({_Args, {alarm_handler, _ExistingAlarms}}) ->
init(_) ->
{ok, []}.
handle_event({set_alarm, {system_memory_high_watermark, []}}, State) ->
emqx_alarm:activate(high_system_memory_usage, #{high_watermark => emqx_os_mon:get_sysmem_high_watermark()}),
handle_event({set_alarm, {system_memory_high_watermark, []}}, State) ->
emqx_alarm:activate(high_system_memory_usage,
#{high_watermark => emqx_os_mon:get_sysmem_high_watermark()}),
{ok, State};
handle_event({set_alarm, {process_memory_high_watermark, Pid}}, State) ->
emqx_alarm:activate(high_process_memory_usage, #{pid => Pid,
high_watermark => emqx_os_mon:get_procmem_high_watermark()}),
handle_event({set_alarm, {process_memory_high_watermark, Pid}}, State) ->
emqx_alarm:activate(high_process_memory_usage,
#{pid => list_to_binary(pid_to_list(Pid)),
high_watermark => emqx_os_mon:get_procmem_high_watermark()}),
{ok, State};
handle_event({clear_alarm, system_memory_high_watermark}, State) ->
handle_event({clear_alarm, system_memory_high_watermark}, State) ->
emqx_alarm:deactivate(high_system_memory_usage),
{ok, State};
handle_event({clear_alarm, process_memory_high_watermark}, State) ->
handle_event({clear_alarm, process_memory_high_watermark}, State) ->
emqx_alarm:deactivate(high_process_memory_usage),
{ok, State};

177
apps/emqx/src/emqx_channel.erl
View File

@ -31,6 +31,8 @@
-export([ info/1
, info/2
, get_mqtt_conf/3
, get_mqtt_conf/4
, set_conn_state/2
, get_session/1
, set_session/2
@ -63,7 +65,7 @@
, maybe_apply/2
]).
-export_type([channel/0]).
-export_type([channel/0, opts/0]).
-record(channel, {
%% MQTT ConnInfo
@ -98,6 +100,8 @@
-type(channel() :: #channel{}).
-type(opts() :: #{zone := atom(), listener := atom(), atom() => term()}).
-type(conn_state() :: idle | connecting | connected | reauthenticating | disconnected).
-type(reply() :: {outgoing, emqx_types:packet()}
@ -151,7 +155,9 @@ info(connected_at, #channel{conninfo = ConnInfo}) ->
info(clientinfo, #channel{clientinfo = ClientInfo}) ->
ClientInfo;
info(zone, #channel{clientinfo = ClientInfo}) ->
maps:get(zone, ClientInfo, undefined);
maps:get(zone, ClientInfo);
info(listener, #channel{clientinfo = ClientInfo}) ->
maps:get(listener, ClientInfo);
info(clientid, #channel{clientinfo = ClientInfo}) ->
maps:get(clientid, ClientInfo, undefined);
info(username, #channel{clientinfo = ClientInfo}) ->
@ -187,25 +193,28 @@ stats(#channel{session = Session})->
emqx_session:stats(Session).
-spec(caps(channel()) -> emqx_types:caps()).
caps(#channel{clientinfo = #{zone := Zone}}) ->
emqx_mqtt_caps:get_caps(Zone).
caps(#channel{clientinfo = #{zone := Zone, listener := Listener}}) ->
emqx_mqtt_caps:get_caps(Zone, Listener).
%%--------------------------------------------------------------------
%% Init the channel
%%--------------------------------------------------------------------
-spec(init(emqx_types:conninfo(), proplists:proplist()) -> channel()).
-spec(init(emqx_types:conninfo(), opts()) -> channel()).
init(ConnInfo = #{peername := {PeerHost, _Port},
sockname := {_Host, SockPort}}, Options) ->
Zone = proplists:get_value(zone, Options),
sockname := {_Host, SockPort}}, #{zone := Zone, listener := Listener}) ->
Peercert = maps:get(peercert, ConnInfo, undefined),
Protocol = maps:get(protocol, ConnInfo, mqtt),
MountPoint = emqx_zone:mountpoint(Zone),
QuotaPolicy = emqx_zone:quota_policy(Zone),
ClientInfo = setting_peercert_infos(
MountPoint = case get_mqtt_conf(Zone, Listener, mountpoint) of
<<>> -> undefined;
MP -> MP
end,
QuotaPolicy = emqx_config:get_listener_conf(Zone, Listener, [rate_limit, quota]),
ClientInfo = set_peercert_infos(
Peercert,
#{zone => Zone,
listener => Listener,
protocol => Protocol,
peerhost => PeerHost,
sockport => SockPort,
@ -214,7 +223,7 @@ init(ConnInfo = #{peername := {PeerHost, _Port},
mountpoint => MountPoint,
is_bridge => false,
is_superuser => false
}, Options),
}, Zone, Listener),
{NClientInfo, NConnInfo} = take_ws_cookie(ClientInfo, ConnInfo),
#channel{conninfo = NConnInfo,
clientinfo = NClientInfo,
@ -222,7 +231,7 @@ init(ConnInfo = #{peername := {PeerHost, _Port},
outbound => #{}
},
auth_cache = #{},
quota = emqx_limiter:init(Zone, QuotaPolicy),
quota = emqx_limiter:init(Zone, quota_policy(QuotaPolicy)),
timers = #{},
conn_state = idle,
takeover = false,
@ -230,30 +239,32 @@ init(ConnInfo = #{peername := {PeerHost, _Port},
pendings = []
}.
setting_peercert_infos(NoSSL, ClientInfo, _Options)
quota_policy(RawPolicy) ->
[{Name, {list_to_integer(StrCount),
erlang:trunc(hocon_postprocess:duration(StrWind) / 1000)}}
|| {Name, [StrCount, StrWind]} <- maps:to_list(RawPolicy)].
set_peercert_infos(NoSSL, ClientInfo, _, _)
when NoSSL =:= nossl;
NoSSL =:= undefined ->
ClientInfo#{username => undefined};
setting_peercert_infos(Peercert, ClientInfo, Options) ->
set_peercert_infos(Peercert, ClientInfo, Zone, Listener) ->
{DN, CN} = {esockd_peercert:subject(Peercert),
esockd_peercert:common_name(Peercert)},
Username = peer_cert_as(peer_cert_as_username, Options, Peercert, DN, CN),
ClientId = peer_cert_as(peer_cert_as_clientid, Options, Peercert, DN, CN),
ClientInfo#{username => Username, clientid => ClientId, dn => DN, cn => CN}.
-dialyzer([{nowarn_function, [peer_cert_as/5]}]).
% esockd_peercert:peercert is opaque
% https://github.com/emqx/esockd/blob/master/src/esockd_peercert.erl
peer_cert_as(Key, Options, Peercert, DN, CN) ->
case proplists:get_value(Key, Options) of
PeercetAs = fun(Key) ->
case get_mqtt_conf(Zone, Listener, Key) of
cn -> CN;
dn -> DN;
crt -> Peercert;
pem -> base64:encode(Peercert);
md5 -> emqx_passwd:hash(md5, Peercert);
pem when is_binary(Peercert) -> base64:encode(Peercert);
md5 when is_binary(Peercert) -> emqx_passwd:hash(md5, Peercert);
_ -> undefined
end.
end
end,
Username = PeercetAs(peer_cert_as_username),
ClientId = PeercetAs(peer_cert_as_clientid),
ClientInfo#{username => Username, clientid => ClientId, dn => DN, cn => CN}.
take_ws_cookie(ClientInfo, ConnInfo) ->
case maps:take(ws_cookie, ConnInfo) of
@ -415,16 +426,17 @@ handle_in(?PUBCOMP_PACKET(PacketId, _ReasonCode), Channel = #channel{session = S
end;
handle_in(Packet = ?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters),
Channel = #channel{clientinfo = ClientInfo = #{zone := Zone}}) ->
Channel = #channel{clientinfo = ClientInfo = #{zone := Zone, listener := Listener}}) ->
case emqx_packet:check(Packet) of
ok ->
TopicFilters0 = parse_topic_filters(TopicFilters),
TopicFilters1 = put_subid_in_subopts(Properties, TopicFilters0),
TupleTopicFilters0 = check_sub_acls(TopicFilters1, Channel),
case emqx_zone:get_env(Zone, acl_deny_action, ignore) =:= disconnect andalso
lists:any(fun({_TopicFilter, ReasonCode}) ->
ReasonCode =:= ?RC_NOT_AUTHORIZED
end, TupleTopicFilters0) of
HasAclDeny = lists:any(fun({_TopicFilter, ReasonCode}) ->
ReasonCode =:= ?RC_NOT_AUTHORIZED
end, TupleTopicFilters0),
DenyAction = emqx_config:get_listener_conf(Zone, Listener, [acl, deny_action]),
case DenyAction =:= disconnect andalso HasAclDeny of
true -> handle_out(disconnect, ?RC_NOT_AUTHORIZED, Channel);
false ->
Replace = fun
@ -526,7 +538,7 @@ process_connect(AckProps, Channel = #channel{conninfo = ConnInfo,
%%--------------------------------------------------------------------
process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId),
Channel = #channel{clientinfo = #{zone := Zone}}) ->
Channel = #channel{clientinfo = #{zone := Zone, listener := Listener}}) ->
case pipeline([fun check_quota_exceeded/2,
fun process_alias/2,
fun check_pub_alias/2,
@ -539,7 +551,7 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId),
{error, Rc = ?RC_NOT_AUTHORIZED, NChannel} ->
?LOG(warning, "Cannot publish message to ~s due to ~s.",
[Topic, emqx_reason_codes:text(Rc)]),
case emqx_zone:get_env(Zone, acl_deny_action, ignore) of
case emqx_config:get_listener_conf(Zone, Listener, [acl_deny_action]) of
ignore ->
case QoS of
?QOS_0 -> {ok, NChannel};
@ -944,8 +956,9 @@ handle_call({takeover, 'end'}, Channel = #channel{session = Session,
AllPendings = lists:append(Delivers, Pendings),
disconnect_and_shutdown(takeovered, AllPendings, Channel);
handle_call(list_acl_cache, Channel) ->
{reply, emqx_acl_cache:list_acl_cache(), Channel};
handle_call(list_acl_cache, #channel{clientinfo = #{zone := Zone, listener := Listener}}
= Channel) ->
{reply, emqx_acl_cache:list_acl_cache(Zone, Listener), Channel};
handle_call({quota, Policy}, Channel) ->
Zone = info(zone, Channel),
@ -982,9 +995,9 @@ handle_info({sock_closed, Reason}, Channel = #channel{conn_state = connecting})
handle_info({sock_closed, Reason}, Channel =
#channel{conn_state = ConnState,
clientinfo = ClientInfo = #{zone := Zone}})
when ConnState =:= connected orelse ConnState =:= reauthenticating ->
emqx_zone:enable_flapping_detect(Zone)
clientinfo = ClientInfo = #{zone := Zone, listener := Listener}})
when ConnState =:= connected orelse ConnState =:= reauthenticating ->
emqx_config:get_listener_conf(Zone, Listener, [flapping_detect, enable])
andalso emqx_flapping:detect(ClientInfo),
Channel1 = ensure_disconnected(Reason, mabye_publish_will_msg(Channel)),
case maybe_shutdown(Reason, Channel1) of
@ -1145,9 +1158,9 @@ enrich_conninfo(ConnPkt = #mqtt_packet_connect{
username = Username
},
Channel = #channel{conninfo = ConnInfo,
clientinfo = #{zone := Zone}
clientinfo = #{zone := Zone, listener := Listener}
}) ->
ExpiryInterval = expiry_interval(Zone, ConnPkt),
ExpiryInterval = expiry_interval(Zone, Listener, ConnPkt),
NConnInfo = ConnInfo#{proto_name => ProtoName,
proto_ver => ProtoVer,
clean_start => CleanStart,
@ -1156,22 +1169,21 @@ enrich_conninfo(ConnPkt = #mqtt_packet_connect{
username => Username,
conn_props => ConnProps,
expiry_interval => ExpiryInterval,
receive_maximum => receive_maximum(Zone, ConnProps)
receive_maximum => receive_maximum(Zone, Listener, ConnProps)
},
{ok, Channel#channel{conninfo = NConnInfo}}.
%% If the Session Expiry Interval is absent the value 0 is used.
-compile({inline, [expiry_interval/2]}).
expiry_interval(_Zone, #mqtt_packet_connect{proto_ver = ?MQTT_PROTO_V5,
expiry_interval(_, _, #mqtt_packet_connect{proto_ver = ?MQTT_PROTO_V5,
properties = ConnProps}) ->
emqx_mqtt_props:get('Session-Expiry-Interval', ConnProps, 0);
expiry_interval(Zone, #mqtt_packet_connect{clean_start = false}) ->
emqx_zone:session_expiry_interval(Zone);
expiry_interval(_Zone, #mqtt_packet_connect{clean_start = true}) ->
expiry_interval(Zone, Listener, #mqtt_packet_connect{clean_start = false}) ->
get_mqtt_conf(Zone, Listener, session_expiry_interval);
expiry_interval(_, _, #mqtt_packet_connect{clean_start = true}) ->
0.
receive_maximum(Zone, ConnProps) ->
MaxInflightConfig = case emqx_zone:max_inflight(Zone) of
receive_maximum(Zone, Listener, ConnProps) ->
MaxInflightConfig = case get_mqtt_conf(Zone, Listener, max_inflight) of
0 -> ?RECEIVE_MAXIMUM_LIMIT;
N -> N
end,
@ -1194,8 +1206,8 @@ run_conn_hooks(ConnPkt, Channel = #channel{conninfo = ConnInfo}) ->
%%--------------------------------------------------------------------
%% Check Connect Packet
check_connect(ConnPkt, #channel{clientinfo = #{zone := Zone}}) ->
emqx_packet:check(ConnPkt, emqx_mqtt_caps:get_caps(Zone)).
check_connect(ConnPkt, #channel{clientinfo = #{zone := Zone, listener := Listener}}) ->
emqx_packet:check(ConnPkt, emqx_mqtt_caps:get_caps(Zone, Listener)).
%%--------------------------------------------------------------------
%% Enrich Client Info
@ -1220,8 +1232,9 @@ set_bridge_mode(_ConnPkt, _ClientInfo) -> ok.
maybe_username_as_clientid(_ConnPkt, ClientInfo = #{username := undefined}) ->
{ok, ClientInfo};
maybe_username_as_clientid(_ConnPkt, ClientInfo = #{zone := Zone, username := Username}) ->
case emqx_zone:use_username_as_clientid(Zone) of
maybe_username_as_clientid(_ConnPkt, ClientInfo = #{zone := Zone, listener := Listener,
username := Username}) ->
case get_mqtt_conf(Zone, Listener, use_username_as_clientid) of
true -> {ok, ClientInfo#{clientid => Username}};
false -> ok
end.
@ -1249,8 +1262,8 @@ set_log_meta(_ConnPkt, #channel{clientinfo = #{clientid := ClientId}}) ->
%%--------------------------------------------------------------------
%% Check banned
check_banned(_ConnPkt, #channel{clientinfo = ClientInfo = #{zone := Zone}}) ->
case emqx_zone:enable_ban(Zone) andalso emqx_banned:check(ClientInfo) of
check_banned(_ConnPkt, #channel{clientinfo = ClientInfo}) ->
case emqx_banned:check(ClientInfo) of
true -> {error, ?RC_BANNED};
false -> ok
end.
@ -1419,8 +1432,8 @@ check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS,
retain = Retain},
variable = #mqtt_packet_publish{topic_name = Topic}
},
#channel{clientinfo = #{zone := Zone}}) ->
emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic => Topic}).
#channel{clientinfo = #{zone := Zone, listener := Listener}}) ->
emqx_mqtt_caps:check_pub(Zone, Listener, #{qos => QoS, retain => Retain, topic => Topic}).
%%--------------------------------------------------------------------
%% Check Sub ACL
@ -1448,8 +1461,9 @@ check_sub_acl(TopicFilter, #channel{clientinfo = ClientInfo}) ->
%%--------------------------------------------------------------------
%% Check Sub Caps
check_sub_caps(TopicFilter, SubOpts, #channel{clientinfo = #{zone := Zone}}) ->
emqx_mqtt_caps:check_sub(Zone, TopicFilter, SubOpts).
check_sub_caps(TopicFilter, SubOpts, #channel{clientinfo = #{zone := Zone,
listener := Listener}}) ->
emqx_mqtt_caps:check_sub(Zone, Listener, TopicFilter, SubOpts).
%%--------------------------------------------------------------------
%% Enrich SubId
@ -1463,21 +1477,23 @@ put_subid_in_subopts(_Properties, TopicFilters) -> TopicFilters.
enrich_subopts(SubOpts, _Channel = ?IS_MQTT_V5) ->
SubOpts;
enrich_subopts(SubOpts, #channel{clientinfo = #{zone := Zone, is_bridge := IsBridge}}) ->
NL = flag(emqx_zone:ignore_loop_deliver(Zone)),
enrich_subopts(SubOpts, #channel{clientinfo = #{zone := Zone, is_bridge := IsBridge,
listener := Listener}}) ->
NL = flag(get_mqtt_conf(Zone, Listener, ignore_loop_deliver)),
SubOpts#{rap => flag(IsBridge), nl => NL}.
%%--------------------------------------------------------------------
%% Enrich ConnAck Caps
enrich_connack_caps(AckProps, ?IS_MQTT_V5 = #channel{clientinfo = #{zone := Zone}}) ->
enrich_connack_caps(AckProps, ?IS_MQTT_V5 = #channel{clientinfo = #{
zone := Zone, listener := Listener}}) ->
#{max_packet_size := MaxPktSize,
max_qos_allowed := MaxQoS,
retain_available := Retain,
max_topic_alias := MaxAlias,
shared_subscription := Shared,
wildcard_subscription := Wildcard
} = emqx_mqtt_caps:get_caps(Zone),
} = emqx_mqtt_caps:get_caps(Zone, Listener),
NAckProps = AckProps#{'Retain-Available' => flag(Retain),
'Maximum-Packet-Size' => MaxPktSize,
'Topic-Alias-Maximum' => MaxAlias,
@ -1499,9 +1515,9 @@ enrich_connack_caps(AckProps, _Channel) -> AckProps.
%%--------------------------------------------------------------------
%% Enrich server keepalive
enrich_server_keepalive(AckProps, #channel{clientinfo = #{zone := Zone}}) ->
case emqx_zone:server_keepalive(Zone) of
undefined -> AckProps;
enrich_server_keepalive(AckProps, #channel{clientinfo = #{zone := Zone, listener := Listener}}) ->
case get_mqtt_conf(Zone, Listener, server_keepalive) of
disabled -> AckProps;
Keepalive -> AckProps#{'Server-Keep-Alive' => Keepalive}
end.
@ -1509,10 +1525,14 @@ enrich_server_keepalive(AckProps, #channel{clientinfo = #{zone := Zone}}) ->
%% Enrich response information
enrich_response_information(AckProps, #channel{conninfo = #{conn_props := ConnProps},
clientinfo = #{zone := Zone}}) ->
clientinfo = #{zone := Zone, listener := Listener}}) ->
case emqx_mqtt_props:get('Request-Response-Information', ConnProps, 0) of
0 -> AckProps;
1 -> AckProps#{'Response-Information' => emqx_zone:response_information(Zone)}
1 -> AckProps#{'Response-Information' =>
case get_mqtt_conf(Zone, Listener, response_information, "") of
"" -> undefined;
RspInfo -> RspInfo
end}
end.
%%--------------------------------------------------------------------
@ -1542,9 +1562,9 @@ ensure_connected(Channel = #channel{conninfo = ConnInfo,
init_alias_maximum(#mqtt_packet_connect{proto_ver = ?MQTT_PROTO_V5,
properties = Properties},
#{zone := Zone} = _ClientInfo) ->
#{zone := Zone, listener := Listener} = _ClientInfo) ->
#{outbound => emqx_mqtt_props:get('Topic-Alias-Maximum', Properties, 0),
inbound => emqx_mqtt_caps:get_caps(Zone, max_topic_alias, ?MAX_TOPIC_AlIAS)
inbound => maps:get(max_topic_alias, emqx_mqtt_caps:get_caps(Zone, Listener))
};
init_alias_maximum(_ConnPkt, _ClientInfo) -> undefined.
@ -1560,8 +1580,10 @@ ensure_keepalive(_AckProps, Channel = #channel{conninfo = ConnInfo}) ->
ensure_keepalive_timer(maps:get(keepalive, ConnInfo), Channel).
ensure_keepalive_timer(0, Channel) -> Channel;
ensure_keepalive_timer(Interval, Channel = #channel{clientinfo = #{zone := Zone}}) ->
Backoff = emqx_zone:keepalive_backoff(Zone),
ensure_keepalive_timer(disabled, Channel) -> Channel;
ensure_keepalive_timer(Interval, Channel = #channel{clientinfo = #{zone := Zone,
listener := Listener}}) ->
Backoff = get_mqtt_conf(Zone, Listener, keepalive_backoff),
Keepalive = emqx_keepalive:init(round(timer:seconds(Interval) * Backoff)),
ensure_timer(alive_timer, Channel#channel{keepalive = Keepalive}).
@ -1602,10 +1624,8 @@ maybe_shutdown(Reason, Channel = #channel{conninfo = ConnInfo}) ->
%%--------------------------------------------------------------------
%% Is ACL enabled?
-compile({inline, [is_acl_enabled/1]}).
is_acl_enabled(#{zone := Zone, is_superuser := IsSuperuser}) ->
(not IsSuperuser) andalso emqx_zone:enable_acl(Zone).
is_acl_enabled(#{zone := Zone, listener := Listener, is_superuser := IsSuperuser}) ->
(not IsSuperuser) andalso emqx_config:get_listener_conf(Zone, Listener, [acl, enable]).
%%--------------------------------------------------------------------
%% Parse Topic Filters
@ -1716,6 +1736,12 @@ sp(false) -> 0.
flag(true) -> 1;
flag(false) -> 0.
get_mqtt_conf(Zone, Listener, Key) ->
emqx_config:get_listener_conf(Zone, Listener, [mqtt, Key]).
get_mqtt_conf(Zone, Listener, Key, Default) ->
emqx_config:get_listener_conf(Zone, Listener, [mqtt, Key], Default).
%%--------------------------------------------------------------------
%% For CT tests
%%--------------------------------------------------------------------
@ -1723,4 +1749,3 @@ flag(false) -> 0.
set_field(Name, Value, Channel) ->
Pos = emqx_misc:index_of(Name, record_info(fields, channel)),
setelement(Pos+1, Channel, Value).

128
apps/emqx/src/emqx_config.erl
View File

@ -20,10 +20,17 @@
-export([ get/0
, get/1
, get/2
, find/1
, put/1
, put/2
]).
-export([ get_listener_conf/3
, get_listener_conf/4
, put_listener_conf/4
, find_listener_conf/3
]).
-export([ update_config/2
]).
@ -35,43 +42,67 @@
, put_raw/2
]).
-export([ deep_get/2
, deep_get/3
, deep_put/3
, safe_atom_key_map/1
, unsafe_atom_key_map/1
]).
-define(CONF, ?MODULE).
-define(RAW_CONF, {?MODULE, raw}).
-export_type([update_request/0, raw_config/0, config_key/0, config_key_path/0]).
-export_type([update_request/0, raw_config/0, config/0]).
-type update_request() :: term().
-type raw_config() :: hocon:config() | undefined.
-type config_key() :: atom() | binary().
-type config_key_path() :: [config_key()].
-type raw_config() :: #{binary() => term()} | undefined.
-type config() :: #{atom() => term()} | undefined.
-spec get() -> map().
get() ->
persistent_term:get(?CONF, #{}).
-spec get(config_key_path()) -> term().
-spec get(emqx_map_lib:config_key_path()) -> term().
get(KeyPath) ->
deep_get(KeyPath, get()).
emqx_map_lib:deep_get(KeyPath, get()).
-spec get(config_key_path(), term()) -> term().
-spec get(emqx_map_lib:config_key_path(), term()) -> term().
get(KeyPath, Default) ->
deep_get(KeyPath, get(), Default).
emqx_map_lib:deep_get(KeyPath, get(), Default).
-spec find(emqx_map_lib:config_key_path()) ->
{ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}.
find(KeyPath) ->
emqx_map_lib:deep_find(KeyPath, get()).
-spec get_listener_conf(atom(), atom(), emqx_map_lib:config_key_path()) -> term().
get_listener_conf(Zone, Listener, KeyPath) ->
case find_listener_conf(Zone, Listener, KeyPath) of
{not_found, SubKeyPath, Data} -> error({not_found, SubKeyPath, Data});
{ok, Data} -> Data
end.
-spec get_listener_conf(atom(), atom(), emqx_map_lib:config_key_path(), term()) -> term().
get_listener_conf(Zone, Listener, KeyPath, Default) ->
case find_listener_conf(Zone, Listener, KeyPath) of
{not_found, _, _} -> Default;
{ok, Data} -> Data
end.
-spec put_listener_conf(atom(), atom(), emqx_map_lib:config_key_path(), term()) -> ok.
put_listener_conf(Zone, Listener, KeyPath, Conf) ->
?MODULE:put([zones, Zone, listeners, Listener | KeyPath], Conf).
-spec find_listener_conf(atom(), atom(), emqx_map_lib:config_key_path()) ->
{ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}.
find_listener_conf(Zone, Listener, KeyPath) ->
%% the configs in listener is prior to the ones in the zone
case find([zones, Zone, listeners, Listener | KeyPath]) of
{not_found, _, _} -> find([zones, Zone | KeyPath]);
{ok, Data} -> {ok, Data}
end.
-spec put(map()) -> ok.
put(Config) ->
persistent_term:put(?CONF, Config).
-spec put(config_key_path(), term()) -> ok.
-spec put(emqx_map_lib:config_key_path(), term()) -> ok.
put(KeyPath, Config) ->
put(deep_put(KeyPath, get(), Config)).
put(emqx_map_lib:deep_put(KeyPath, get(), Config)).
-spec update_config(config_key_path(), update_request()) ->
-spec update_config(emqx_map_lib:config_key_path(), update_request()) ->
ok | {error, term()}.
update_config(ConfKeyPath, UpdateReq) ->
emqx_config_handler:update_config(ConfKeyPath, UpdateReq, get_raw()).
@ -80,67 +111,18 @@ update_config(ConfKeyPath, UpdateReq) ->
get_raw() ->
persistent_term:get(?RAW_CONF, #{}).
-spec get_raw(config_key_path()) -> term().
-spec get_raw(emqx_map_lib:config_key_path()) -> term().
get_raw(KeyPath) ->
deep_get(KeyPath, get_raw()).
emqx_map_lib:deep_get(KeyPath, get_raw()).
-spec get_raw(config_key_path(), term()) -> term().
-spec get_raw(emqx_map_lib:config_key_path(), term()) -> term().
get_raw(KeyPath, Default) ->
deep_get(KeyPath, get_raw(), Default).
emqx_map_lib:deep_get(KeyPath, get_raw(), Default).
-spec put_raw(map()) -> ok.
put_raw(Config) ->
persistent_term:put(?RAW_CONF, Config).
-spec put_raw(config_key_path(), term()) -> ok.
-spec put_raw(emqx_map_lib:config_key_path(), term()) -> ok.
put_raw(KeyPath, Config) ->
put_raw(deep_put(KeyPath, get_raw(), Config)).
%%-----------------------------------------------------------------
-dialyzer([{nowarn_function, [deep_get/2]}]).
-spec deep_get(config_key_path(), map()) -> term().
deep_get(ConfKeyPath, Map) ->
do_deep_get(ConfKeyPath, Map, fun(KeyPath, Data) ->
error({not_found, KeyPath, Data}) end).
-spec deep_get(config_key_path(), map(), term()) -> term().
deep_get(ConfKeyPath, Map, Default) ->
do_deep_get(ConfKeyPath, Map, fun(_, _) -> Default end).
-spec deep_put(config_key_path(), map(), term()) -> map().
deep_put([], Map, Config) when is_map(Map) ->
Config;
deep_put([Key | KeyPath], Map, Config) ->
SubMap = deep_put(KeyPath, maps:get(Key, Map, #{}), Config),
Map#{Key => SubMap}.
unsafe_atom_key_map(Map) ->
covert_keys_to_atom(Map, fun(K) -> binary_to_atom(K, utf8) end).
safe_atom_key_map(Map) ->
covert_keys_to_atom(Map, fun(K) -> binary_to_existing_atom(K, utf8) end).
%%---------------------------------------------------------------------------
-spec do_deep_get(config_key_path(), map(), fun((config_key(), term()) -> any())) -> term().
do_deep_get([], Map, _) ->
Map;
do_deep_get([Key | KeyPath], Map, OnNotFound) when is_map(Map) ->
case maps:find(Key, Map) of
{ok, SubMap} -> do_deep_get(KeyPath, SubMap, OnNotFound);
error -> OnNotFound(Key, Map)
end;
do_deep_get([Key | _KeyPath], Data, OnNotFound) ->
OnNotFound(Key, Data).
covert_keys_to_atom(BinKeyMap, Conv) when is_map(BinKeyMap) ->
maps:fold(
fun(K, V, Acc) when is_binary(K) ->
Acc#{Conv(K) => covert_keys_to_atom(V, Conv)};
(K, V, Acc) when is_atom(K) ->
%% richmap keys
Acc#{K => covert_keys_to_atom(V, Conv)}
end, #{}, BinKeyMap);
covert_keys_to_atom(ListV, Conv) when is_list(ListV) ->
[covert_keys_to_atom(V, Conv) || V <- ListV];
covert_keys_to_atom(Val, _) -> Val.
put_raw(emqx_map_lib:deep_put(KeyPath, get_raw(), Config)).

95
apps/emqx/src/emqx_config_handler.erl
View File

@ -45,11 +45,15 @@
-type handler_name() :: module().
-type handlers() :: #{emqx_config:config_key() => handlers(), ?MOD => handler_name()}.
-optional_callbacks([handle_update_config/2]).
-optional_callbacks([ handle_update_config/2
, post_update_config/2
]).
-callback handle_update_config(emqx_config:update_request(), emqx_config:raw_config()) ->
emqx_config:update_request().
-callback post_update_config(emqx_config:config(), emqx_config:config()) -> any().
-type state() :: #{
handlers := handlers(),
atom() => term()
@ -72,22 +76,23 @@ add_handler(ConfKeyPath, HandlerName) ->
-spec init(term()) -> {ok, state()}.
init(_) ->
{ok, RawConf} = hocon:load(emqx_conf_name(), #{format => richmap}),
RawConf = load_config_file(),
{_MappedEnvs, Conf} = hocon_schema:map_translate(emqx_schema, RawConf, #{}),
ok = save_config_to_emqx(to_plainmap(Conf), to_plainmap(RawConf)),
{ok, #{handlers => #{?MOD => ?MODULE}}}.
handle_call({add_child, ConfKeyPath, HandlerName}, _From,
State = #{handlers := Handlers}) ->
{reply, ok, State#{handlers =>
emqx_config:deep_put(ConfKeyPath, Handlers, #{?MOD => HandlerName})}};
emqx_map_lib:deep_put(ConfKeyPath, Handlers, #{?MOD => HandlerName})}};
handle_call({update_config, ConfKeyPath, UpdateReq, RawConf}, _From,
#{handlers := Handlers} = State) ->
OldConf = emqx_config:get(),
try {RootKeys, Conf} = do_update_config(ConfKeyPath, Handlers, RawConf, UpdateReq),
{reply, save_configs(RootKeys, Conf), State}
Result = save_configs(RootKeys, Conf),
do_post_update_config(ConfKeyPath, Handlers, OldConf, emqx_config:get()),
{reply, Result, State}
catch
throw: Reason ->
{reply, {error, Reason}, State};
Error : Reason : ST ->
?LOG(error, "update config failed: ~p", [{Error, Reason, ST}]),
{reply, {error, Reason}, State}
@ -109,29 +114,40 @@ terminate(_Reason, _State) ->
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
do_update_config([], Handlers, OldConf, UpdateReq) ->
call_handle_update_config(Handlers, OldConf, UpdateReq);
do_update_config([ConfKey | ConfKeyPath], Handlers, OldConf, UpdateReq) ->
do_update_config([], Handlers, OldRawConf, UpdateReq) ->
call_handle_update_config(Handlers, OldRawConf, UpdateReq);
do_update_config([ConfKey | ConfKeyPath], Handlers, OldRawConf, UpdateReq) ->
SubOldRawConf = get_sub_config(bin(ConfKey), OldRawConf),
SubHandlers = maps:get(ConfKey, Handlers, #{}),
NewUpdateReq = do_update_config(ConfKeyPath, SubHandlers, SubOldRawConf, UpdateReq),
call_handle_update_config(Handlers, OldRawConf, #{bin(ConfKey) => NewUpdateReq}).
do_post_update_config([], Handlers, OldConf, NewConf) ->
call_post_update_config(Handlers, OldConf, NewConf);
do_post_update_config([ConfKey | ConfKeyPath], Handlers, OldConf, NewConf) ->
SubOldConf = get_sub_config(ConfKey, OldConf),
case maps:find(ConfKey, Handlers) of
error -> throw({handler_not_found, ConfKey});
{ok, SubHandlers} ->
NewUpdateReq = do_update_config(ConfKeyPath, SubHandlers, SubOldConf, UpdateReq),
call_handle_update_config(Handlers, OldConf, #{bin(ConfKey) => NewUpdateReq})
end.
SubNewConf = get_sub_config(ConfKey, NewConf),
SubHandlers = maps:get(ConfKey, Handlers, #{}),
_ = do_post_update_config(ConfKeyPath, SubHandlers, SubOldConf, SubNewConf),
call_post_update_config(Handlers, OldConf, NewConf).
get_sub_config(_, undefined) ->
undefined;
get_sub_config(ConfKey, OldConf) when is_map(OldConf) ->
maps:get(bin(ConfKey), OldConf, undefined);
get_sub_config(_, OldConf) ->
OldConf.
get_sub_config(ConfKey, Conf) when is_map(Conf) ->
maps:get(ConfKey, Conf, undefined);
get_sub_config(_, _Conf) -> %% the Conf is a primitive
undefined.
call_handle_update_config(Handlers, OldConf, UpdateReq) ->
call_handle_update_config(Handlers, OldRawConf, UpdateReq) ->
HandlerName = maps:get(?MOD, Handlers, undefined),
case erlang:function_exported(HandlerName, handle_update_config, 2) of
true -> HandlerName:handle_update_config(UpdateReq, OldConf);
false -> UpdateReq %% the default behaviour is overwriting the old config
true -> HandlerName:handle_update_config(UpdateReq, OldRawConf);
false -> merge_to_old_config(UpdateReq, OldRawConf)
end.
call_post_update_config(Handlers, OldConf, NewConf) ->
HandlerName = maps:get(?MOD, Handlers, undefined),
case erlang:function_exported(HandlerName, post_update_config, 2) of
true -> _ = HandlerName:post_update_config(NewConf, OldConf);
false -> ok
end.
%% callbacks for the top-level handler
@ -139,11 +155,15 @@ handle_update_config(UpdateReq, OldConf) ->
FullRawConf = merge_to_old_config(UpdateReq, OldConf),
{maps:keys(UpdateReq), FullRawConf}.
%% default callback of config handlers
merge_to_old_config(UpdateReq, undefined) ->
merge_to_old_config(UpdateReq, #{});
merge_to_old_config(UpdateReq, RawConf) ->
maps:merge(RawConf, UpdateReq).
%% The default callback of config handlers
%% the behaviour is overwriting the old config if:
%% 1. the old config is undefined
%% 2. either the old or the new config is not of map type
%% the behaviour is merging the new the config to the old config if they are maps.
merge_to_old_config(UpdateReq, RawConf) when is_map(UpdateReq), is_map(RawConf) ->
maps:merge(RawConf, UpdateReq);
merge_to_old_config(UpdateReq, _RawConf) ->
UpdateReq.
%%============================================================================
save_configs(RootKeys, RawConf) ->
@ -160,7 +180,8 @@ save_configs(RootKeys, RawConf) ->
% end, MappedEnvs).
save_config_to_emqx(Conf, RawConf) ->
emqx_config:put(emqx_config:unsafe_atom_key_map(Conf)),
?LOG(debug, "set config: ~p", [Conf]),
emqx_config:put(emqx_map_lib:unsafe_atom_key_map(Conf)),
emqx_config:put_raw(RawConf).
save_config_to_disk(RootKeys, Conf) ->
@ -191,14 +212,16 @@ read_old_config(FileName) ->
_ -> #{}
end.
emqx_conf_name() ->
filename:join([etc_dir(), "emqx.conf"]).
load_config_file() ->
lists:foldl(fun(ConfFile, Acc) ->
{ok, RawConf} = hocon:load(ConfFile, #{format => richmap}),
emqx_map_lib:deep_merge(Acc, RawConf)
end, #{}, emqx:get_env(config_files, [])).
emqx_override_conf_name() ->
filename:join([emqx:get_env(data_dir), "emqx_override.conf"]).
etc_dir() ->
emqx:get_env(etc_dir).
File = filename:join([emqx:get_env(data_dir), "emqx_override.conf"]),
ok = filelib:ensure_dir(File),
File.
to_richmap(Map) ->
{ok, RichMap} = hocon:binary(jsx:encode(Map), #{format => richmap}),

86
apps/emqx/src/emqx_connection.erl
View File

@ -83,8 +83,6 @@
sockname :: emqx_types:peername(),
%% Sock State
sockstate :: emqx_types:sockstate(),
%% The {active, N} option
active_n :: pos_integer(),
%% Limiter
limiter :: maybe(emqx_limiter:limiter()),
%% Limit Timer
@ -102,13 +100,17 @@
%% Idle Timeout
idle_timeout :: integer(),
%% Idle Timer
idle_timer :: maybe(reference())
idle_timer :: maybe(reference()),
%% Zone name
zone :: atom(),
%% Listener Name
listener :: atom()
}).
-type(state() :: #state{}).
-define(ACTIVE_N, 100).
-define(INFO_KEYS, [socktype, peername, sockname, sockstate, active_n]).
-define(INFO_KEYS, [socktype, peername, sockname, sockstate]).
-define(CONN_STATS, [recv_pkt, recv_msg, send_pkt, send_msg]).
-define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]).
@ -134,7 +136,7 @@
, system_code_change/4
]}).
-spec(start_link(esockd:transport(), esockd:socket(), proplists:proplist())
-spec(start_link(esockd:transport(), esockd:socket(), emqx_channel:opts())
-> {ok, pid()}).
start_link(Transport, Socket, Options) ->
Args = [self(), Transport, Socket, Options],
@ -165,8 +167,6 @@ info(sockname, #state{sockname = Sockname}) ->
Sockname;
info(sockstate, #state{sockstate = SockSt}) ->
SockSt;
info(active_n, #state{active_n = ActiveN}) ->
ActiveN;
info(stats_timer, #state{stats_timer = StatsTimer}) ->
StatsTimer;
info(limit_timer, #state{limit_timer = LimitTimer}) ->
@ -243,7 +243,7 @@ init(Parent, Transport, RawSocket, Options) ->
exit_on_sock_error(Reason)
end.
init_state(Transport, Socket, Options) ->
init_state(Transport, Socket, #{zone := Zone, listener := Listener} = Opts) ->
{ok, Peername} = Transport:ensure_ok_or_exit(peername, [Socket]),
{ok, Sockname} = Transport:ensure_ok_or_exit(sockname, [Socket]),
Peercert = Transport:ensure_ok_or_exit(peercert, [Socket]),
@ -253,26 +253,29 @@ init_state(Transport, Socket, Options) ->
peercert => Peercert,
conn_mod => ?MODULE
},
Zone = proplists:get_value(zone, Options),
ActiveN = proplists:get_value(active_n, Options, ?ACTIVE_N),
PubLimit = emqx_zone:publish_limit(Zone),
BytesIn = proplists:get_value(rate_limit, Options),
RateLimit = emqx_zone:ratelimit(Zone),
Limiter = emqx_limiter:init(Zone, PubLimit, BytesIn, RateLimit),
FrameOpts = emqx_zone:mqtt_frame_options(Zone),
Limiter = emqx_limiter:init(Zone, undefined, undefined, []),
FrameOpts = #{
strict_mode => emqx_config:get_listener_conf(Zone, Listener, [mqtt, strict_mode]),
max_size => emqx_config:get_listener_conf(Zone, Listener, [mqtt, max_packet_size])
},
ParseState = emqx_frame:initial_parse_state(FrameOpts),
Serialize = emqx_frame:serialize_opts(),
Channel = emqx_channel:init(ConnInfo, Options),
GcState = emqx_zone:init_gc_state(Zone),
StatsTimer = emqx_zone:stats_timer(Zone),
IdleTimeout = emqx_zone:idle_timeout(Zone),
Channel = emqx_channel:init(ConnInfo, Opts),
GcState = case emqx_config:get_listener_conf(Zone, Listener, [force_gc]) of
#{enable := false} -> undefined;
GcPolicy -> emqx_gc:init(GcPolicy)
end,
StatsTimer = case emqx_config:get_listener_conf(Zone, Listener, [stats, enable]) of
true -> undefined;
false -> disabled
end,
IdleTimeout = emqx_channel:get_mqtt_conf(Zone, Listener, idle_timeout),
IdleTimer = start_timer(IdleTimeout, idle_timeout),
#state{transport = Transport,
socket = Socket,
peername = Peername,
sockname = Sockname,
sockstate = idle,
active_n = ActiveN,
limiter = Limiter,
parse_state = ParseState,
serialize = Serialize,
@ -280,7 +283,9 @@ init_state(Transport, Socket, Options) ->
gc_state = GcState,
stats_timer = StatsTimer,
idle_timeout = IdleTimeout,
idle_timer = IdleTimer
idle_timer = IdleTimer,
zone = Zone,
listener = Listener
}.
run_loop(Parent, State = #state{transport = Transport,
@ -288,8 +293,9 @@ run_loop(Parent, State = #state{transport = Transport,
peername = Peername,
channel = Channel}) ->
emqx_logger:set_metadata_peername(esockd:format(Peername)),
emqx_misc:tune_heap_size(emqx_zone:oom_policy(
emqx_channel:info(zone, Channel))),
ShutdownPolicy = emqx_config:get_listener_conf(emqx_channel:info(zone, Channel),
emqx_channel:info(listener, Channel), [force_shutdown]),
emqx_misc:tune_heap_size(ShutdownPolicy),
case activate_socket(State) of
{ok, NState} -> hibernate(Parent, NState);
{error, Reason} ->
@ -458,14 +464,15 @@ handle_msg({Passive, _Sock}, State)
NState1 = check_oom(run_gc(InStats, NState)),
handle_info(activate_socket, NState1);
handle_msg(Deliver = {deliver, _Topic, _Msg},
#state{active_n = ActiveN} = State) ->
handle_msg(Deliver = {deliver, _Topic, _Msg}, #state{zone = Zone,
listener = Listener} = State) ->
ActiveN = get_active_n(Zone, Listener),
Delivers = [Deliver|emqx_misc:drain_deliver(ActiveN)],
with_channel(handle_deliver, [Delivers], State);
%% Something sent
handle_msg({inet_reply, _Sock, ok}, State = #state{active_n = ActiveN}) ->
case emqx_pd:get_counter(outgoing_pubs) > ActiveN of
handle_msg({inet_reply, _Sock, ok}, State = #state{zone = Zone, listener = Listener}) ->
case emqx_pd:get_counter(outgoing_pubs) > get_active_n(Zone, Listener) of
true ->
Pubs = emqx_pd:reset_counter(outgoing_pubs),
Bytes = emqx_pd:reset_counter(outgoing_bytes),
@ -794,15 +801,14 @@ run_gc(Stats, State = #state{gc_state = GcSt}) ->
end.
check_oom(State = #state{channel = Channel}) ->
Zone = emqx_channel:info(zone, Channel),
OomPolicy = emqx_zone:oom_policy(Zone),
?tp(debug, check_oom, #{policy => OomPolicy}),
case ?ENABLED(OomPolicy) andalso emqx_misc:check_oom(OomPolicy) of
ShutdownPolicy = emqx_config:get_listener_conf(emqx_channel:info(zone, Channel),
emqx_channel:info(listener, Channel), [force_shutdown]),
?tp(debug, check_oom, #{policy => ShutdownPolicy}),
case emqx_misc:check_oom(ShutdownPolicy) of
{shutdown, Reason} ->
%% triggers terminate/2 callback immediately
erlang:exit({shutdown, Reason});
_Other ->
ok
_ -> ok
end,
State.
@ -814,10 +820,10 @@ activate_socket(State = #state{sockstate = closed}) ->
{ok, State};
activate_socket(State = #state{sockstate = blocked}) ->
{ok, State};
activate_socket(State = #state{transport = Transport,
socket = Socket,
active_n = N}) ->
case Transport:setopts(Socket, [{active, N}]) of
activate_socket(State = #state{transport = Transport, socket = Socket,
zone = Zone, listener = Listener}) ->
ActiveN = get_active_n(Zone, Listener),
case Transport:setopts(Socket, [{active, ActiveN}]) of
ok -> {ok, State#state{sockstate = running}};
Error -> Error
end.
@ -898,3 +904,9 @@ get_state(Pid) ->
State = sys:get_state(Pid),
maps:from_list(lists:zip(record_info(fields, state),
tl(tuple_to_list(State)))).
get_active_n(Zone, Listener) ->
case emqx_config:get([zones, Zone, listeners, Listener, type]) of
quic -> 100;
_ -> emqx_config:get_listener_conf(Zone, Listener, [tcp, active_n])
end.

86
apps/emqx/src/emqx_flapping.erl
View File

@ -45,16 +45,16 @@
-define(FLAPPING_DURATION, 60000).
-define(FLAPPING_BANNED_INTERVAL, 300000).
-define(DEFAULT_DETECT_POLICY,
#{threshold => ?FLAPPING_THRESHOLD,
duration => ?FLAPPING_DURATION,
banned_interval => ?FLAPPING_BANNED_INTERVAL
#{max_count => ?FLAPPING_THRESHOLD,
window_time => ?FLAPPING_DURATION,
ban_time => ?FLAPPING_BANNED_INTERVAL
}).
-record(flapping, {
clientid :: emqx_types:clientid(),
peerhost :: emqx_types:peerhost(),
started_at :: pos_integer(),
detect_cnt :: pos_integer()
detect_cnt :: integer()
}).
-opaque(flapping() :: #flapping{}).
@ -69,33 +69,28 @@ stop() -> gen_server:stop(?MODULE).
%% @doc Detect flapping when a MQTT client disconnected.
-spec(detect(emqx_types:clientinfo()) -> boolean()).
detect(Client) -> detect(Client, get_policy()).
detect(#{clientid := ClientId, peerhost := PeerHost}, Policy = #{threshold := Threshold}) ->
try ets:update_counter(?FLAPPING_TAB, ClientId, {#flapping.detect_cnt, 1}) of
detect(#{clientid := ClientId, peerhost := PeerHost, zone := Zone, listener := Listener}) ->
Policy = #{max_count := Threshold} = get_policy(Zone, Listener),
%% The initial flapping record sets the detect_cnt to 0.
InitVal = #flapping{
clientid = ClientId,
peerhost = PeerHost,
started_at = erlang:system_time(millisecond),
detect_cnt = 0
},
case ets:update_counter(?FLAPPING_TAB, ClientId, {#flapping.detect_cnt, 1}, InitVal) of
Cnt when Cnt < Threshold -> false;
_Cnt -> case ets:take(?FLAPPING_TAB, ClientId) of
[Flapping] ->
ok = gen_server:cast(?MODULE, {detected, Flapping, Policy}),
true;
[] -> false
end
catch
error:badarg ->
%% Create a flapping record.
Flapping = #flapping{clientid = ClientId,
peerhost = PeerHost,
started_at = erlang:system_time(millisecond),
detect_cnt = 1
},
true = ets:insert(?FLAPPING_TAB, Flapping),
false
_Cnt ->
case ets:take(?FLAPPING_TAB, ClientId) of
[Flapping] ->
ok = gen_server:cast(?MODULE, {detected, Flapping, Policy}),
true;
[] -> false
end
end.
-compile({inline, [get_policy/0, now_diff/1]}).
get_policy() ->
emqx:get_env(flapping_detect_policy, ?DEFAULT_DETECT_POLICY).
get_policy(Zone, Listener) ->
emqx_config:get_listener_conf(Zone, Listener, [flapping_detect]).
now_diff(TS) -> erlang:system_time(millisecond) - TS.
@ -105,11 +100,12 @@ now_diff(TS) -> erlang:system_time(millisecond) - TS.
init([]) ->
ok = emqx_tables:new(?FLAPPING_TAB, [public, set,
{keypos, 2},
{keypos, #flapping.clientid},
{read_concurrency, true},
{write_concurrency, true}
]),
{ok, ensure_timer(#{}), hibernate}.
start_timers(),
{ok, #{}, hibernate}.
handle_call(Req, _From, State) ->
?LOG(error, "Unexpected call: ~p", [Req]),
@ -119,17 +115,17 @@ handle_cast({detected, #flapping{clientid = ClientId,
peerhost = PeerHost,
started_at = StartedAt,
detect_cnt = DetectCnt},
#{duration := Duration, banned_interval := Interval}}, State) ->
case now_diff(StartedAt) < Duration of
#{window_time := WindTime, ban_time := Interval}}, State) ->
case now_diff(StartedAt) < WindTime of
true -> %% Flapping happened:(
?LOG(error, "Flapping detected: ~s(~s) disconnected ~w times in ~wms",
[ClientId, inet:ntoa(PeerHost), DetectCnt, Duration]),
[ClientId, inet:ntoa(PeerHost), DetectCnt, WindTime]),
Now = erlang:system_time(second),
Banned = #banned{who = {clientid, ClientId},
by = <<"flapping detector">>,
reason = <<"flapping is detected">>,
at = Now,
until = Now + Interval},
until = Now + (Interval div 1000)},
emqx_banned:create(Banned);
false ->
?LOG(warning, "~s(~s) disconnected ~w times in ~wms",
@ -141,11 +137,13 @@ handle_cast(Msg, State) ->
?LOG(error, "Unexpected cast: ~p", [Msg]),
{noreply, State}.
handle_info({timeout, TRef, expired_detecting}, State = #{expired_timer := TRef}) ->
Timestamp = erlang:system_time(millisecond) - maps:get(duration, get_policy()),
handle_info({timeout, _TRef, {garbage_collect, Zone, Listener}}, State) ->
Timestamp = erlang:system_time(millisecond)
- maps:get(window_time, get_policy(Zone, Listener)),
MatchSpec = [{{'_', '_', '_', '$1', '_'},[{'<', '$1', Timestamp}], [true]}],
ets:select_delete(?FLAPPING_TAB, MatchSpec),
{noreply, ensure_timer(State), hibernate};
start_timer(Zone, Listener),
{noreply, State, hibernate};
handle_info(Info, State) ->
?LOG(error, "Unexpected info: ~p", [Info]),
@ -157,7 +155,13 @@ terminate(_Reason, _State) ->
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
ensure_timer(State) ->
Timeout = maps:get(duration, get_policy()),
TRef = emqx_misc:start_timer(Timeout, expired_detecting),
State#{expired_timer => TRef}.
start_timer(Zone, Listener) ->
WindTime = maps:get(window_time, get_policy(Zone, Listener)),
emqx_misc:start_timer(WindTime, {garbage_collect, Zone, Listener}).
start_timers() ->
lists:foreach(fun({Zone, ZoneConf}) ->
lists:foreach(fun({Listener, _}) ->
start_timer(Zone, Listener)
end, maps:to_list(maps:get(listeners, ZoneConf, #{})))
end, maps:to_list(emqx_config:get([zones], #{}))).

8
apps/emqx/src/emqx_frame.erl
View File

@ -81,11 +81,7 @@ initial_parse_state() ->
-spec(initial_parse_state(options()) -> {none, options()}).
initial_parse_state(Options) when is_map(Options) ->
?none(merge_opts(Options)).
%% @pivate
merge_opts(Options) ->
maps:merge(?DEFAULT_OPTIONS, Options).
?none(maps:merge(?DEFAULT_OPTIONS, Options)).
%%--------------------------------------------------------------------
%% Parse MQTT Frame
@ -643,7 +639,7 @@ serialize_properties(Props) when is_map(Props) ->
Bin = << <<(serialize_property(Prop, Val))/binary>> || {Prop, Val} <- maps:to_list(Props) >>,
[serialize_variable_byte_integer(byte_size(Bin)), Bin].
serialize_property(_, undefined) ->
serialize_property(_, Disabled) when Disabled =:= disabled; Disabled =:= undefined ->
<<>>;
serialize_property('Payload-Format-Indicator', Val) ->
<<16#01, Val>>;

17
apps/emqx/src/emqx_kernel_sup.erl
View File

@ -27,14 +27,15 @@ start_link() ->
init([]) ->
{ok, {{one_for_one, 10, 100},
[child_spec(emqx_global_gc, worker),
child_spec(emqx_pool_sup, supervisor),
child_spec(emqx_hooks, worker),
child_spec(emqx_stats, worker),
child_spec(emqx_metrics, worker),
child_spec(emqx_ctl, worker),
child_spec(emqx_zone, worker),
child_spec(emqx_config_handler, worker)
%% always start emqx_config_handler first to load the emqx.conf to emqx_config
[ child_spec(emqx_config_handler, worker)
, child_spec(emqx_global_gc, worker)
, child_spec(emqx_pool_sup, supervisor)
, child_spec(emqx_hooks, worker)
, child_spec(emqx_stats, worker)
, child_spec(emqx_metrics, worker)
, child_spec(emqx_ctl, worker)
, child_spec(emqx_zone, worker)
]}}.
child_spec(M, Type) ->

371
apps/emqx/src/emqx_listeners.erl
View File

@ -21,7 +21,6 @@
%% APIs
-export([ start/0
, ensure_all_started/0
, restart/0
, stop/0
]).
@ -29,90 +28,29 @@
-export([ start_listener/1
, start_listener/3
, stop_listener/1
, stop_listener/3
, restart_listener/1
, restart_listener/3
]).
-export([ find_id_by_listen_on/1
, find_by_listen_on/1
, find_by_id/1
, identifier/1
, format_listen_on/1
]).
-type(listener() :: #{ name := binary()
, proto := esockd:proto()
, listen_on := esockd:listen_on()
, opts := [esockd:option()]
}).
%% @doc Find listener identifier by listen-on.
%% Return empty string (binary) if listener is not found in config.
-spec(find_id_by_listen_on(esockd:listen_on()) -> binary() | false).
find_id_by_listen_on(ListenOn) ->
case find_by_listen_on(ListenOn) of
false -> false;
L -> identifier(L)
end.
%% @doc Find listener by listen-on.
%% Return 'false' if not found.
-spec(find_by_listen_on(esockd:listen_on()) -> listener() | false).
find_by_listen_on(ListenOn) ->
find_by_listen_on(ListenOn, emqx:get_env(listeners, [])).
%% @doc Find listener by identifier.
%% Return 'false' if not found.
-spec(find_by_id(string() | binary()) -> listener() | false).
find_by_id(Id) ->
find_by_id(iolist_to_binary(Id), emqx:get_env(listeners, [])).
%% @doc Return the ID of the given listener.
-spec identifier(listener()) -> binary().
identifier(#{proto := Proto, name := Name}) ->
identifier(Proto, Name).
%% @doc Start all listeners.
-spec(start() -> ok).
start() ->
lists:foreach(fun start_listener/1, emqx:get_env(listeners, [])).
foreach_listeners(fun start_listener/3).
%% @doc Ensure all configured listeners are started.
%% Raise exception if any of them failed to start.
-spec(ensure_all_started() -> ok).
ensure_all_started() ->
ensure_all_started(emqx:get_env(listeners, []), []).
-spec start_listener(atom()) -> ok | {error, term()}.
start_listener(ListenerId) ->
apply_on_listener(ListenerId, fun start_listener/3).
ensure_all_started([], []) -> ok;
ensure_all_started([], Failed) -> error(Failed);
ensure_all_started([L | Rest], Results) ->
#{proto := Proto, listen_on := ListenOn, opts := Options} = L,
NewResults =
case start_listener(Proto, ListenOn, Options) of
{ok, _Pid} ->
Results;
{error, {already_started, _Pid}} ->
Results;
{error, Reason} ->
[{identifier(L), Reason} | Results]
end,
ensure_all_started(Rest, NewResults).
%% @doc Format address:port for logging.
-spec(format_listen_on(esockd:listen_on()) -> [char()]).
format_listen_on(ListenOn) -> format(ListenOn).
-spec(start_listener(listener()) -> ok).
start_listener(#{proto := Proto, name := Name, listen_on := ListenOn, opts := Options}) ->
ID = identifier(Proto, Name),
case start_listener(Proto, ListenOn, Options) of
{ok, skipped} ->
console_print("Start ~s listener on ~s skpped.~n", [ID, format(ListenOn)]);
-spec start_listener(atom(), atom(), map()) -> ok | {error, term()}.
start_listener(ZoneName, ListenerName, #{type := Type, bind := Bind} = Conf) ->
case do_start_listener(ZoneName, ListenerName, Conf) of
{ok, _} ->
console_print("Start ~s listener on ~s successfully.~n", [ID, format(ListenOn)]);
console_print("Start ~s listener ~s on ~s successfully.~n",
[Type, listener_id(ZoneName, ListenerName), format(Bind)]);
{error, Reason} ->
io:format(standard_error, "Failed to start mqtt listener ~s on ~s: ~0p~n",
[ID, format(ListenOn), Reason]),
io:format(standard_error, "Failed to start ~s listener ~s on ~s: ~0p~n",
[Type, listener_id(ZoneName, ListenerName), format(Bind), Reason]),
error(Reason)
end.
@ -124,154 +62,128 @@ console_print(_Fmt, _Args) -> ok.
-endif.
%% Start MQTT/TCP listener
-spec(start_listener(esockd:proto(), esockd:listen_on(), [esockd:option()])
-> {ok, pid() | skipped} | {error, term()}).
start_listener(tcp, ListenOn, Options) ->
start_mqtt_listener('mqtt:tcp', ListenOn, Options);
%% Start MQTT/TLS listener
start_listener(Proto, ListenOn, Options) when Proto == ssl; Proto == tls ->
start_mqtt_listener('mqtt:ssl', ListenOn, Options);
-spec(do_start_listener(atom(), atom(), map())
-> {ok, pid()} | {error, term()}).
do_start_listener(ZoneName, ListenerName, #{type := tcp, bind := ListenOn} = Opts) ->
esockd:open(listener_id(ZoneName, ListenerName), ListenOn, merge_default(esockd_opts(Opts)),
{emqx_connection, start_link,
[#{zone => ZoneName, listener => ListenerName}]});
%% Start MQTT/WS listener
start_listener(Proto, ListenOn, Options) when Proto == http; Proto == ws ->
start_http_listener(fun cowboy:start_clear/3, 'mqtt:ws', ListenOn,
ranch_opts(Options), ws_opts(Options));
%% Start MQTT/WSS listener
start_listener(Proto, ListenOn, Options) when Proto == https; Proto == wss ->
start_http_listener(fun cowboy:start_tls/3, 'mqtt:wss', ListenOn,
ranch_opts(Options), ws_opts(Options));
do_start_listener(ZoneName, ListenerName, #{type := ws, bind := ListenOn} = Opts) ->
Id = listener_id(ZoneName, ListenerName),
RanchOpts = ranch_opts(ListenOn, Opts),
WsOpts = ws_opts(ZoneName, ListenerName, Opts),
case is_ssl(Opts) of
false ->
cowboy:start_clear(Id, RanchOpts, WsOpts);
true ->
cowboy:start_tls(Id, RanchOpts, WsOpts)
end;
%% Start MQTT/QUIC listener
start_listener(quic, ListenOn, Options) ->
case [ A || {quicer, _, _} = A<-application:which_applications() ] of
[_] ->
%% @fixme unsure why we need reopen lib and reopen config.
quicer_nif:open_lib(),
quicer_nif:reg_open(),
SSLOpts = proplists:get_value(ssl_options, Options),
DefAcceptors = erlang:system_info(schedulers_online) * 8,
ListenOpts = [ {cert, proplists:get_value(certfile, SSLOpts)}
, {key, proplists:get_value(keyfile, SSLOpts)}
, {alpn, ["mqtt"]}
, {conn_acceptors, proplists:get_value(acceptors, Options, DefAcceptors)}
, {idle_timeout_ms, proplists:get_value(idle_timeout, Options, 60000)}
],
ConnectionOpts = [ {conn_callback, emqx_quic_connection}
, {peer_unidi_stream_count, 1}
, {peer_bidi_stream_count, 10}
| Options
],
StreamOpts = [],
quicer:start_listener('mqtt:quic', ListenOn, {ListenOpts, ConnectionOpts, StreamOpts});
[] ->
io:format(standard_error, "INFO: quicer application is unavailable/disabled~n",
[]),
{ok, skipped}
end.
do_start_listener(ZoneName, ListenerName, #{type := quic, bind := ListenOn} = Opts) ->
%% @fixme unsure why we need reopen lib and reopen config.
quicer_nif:open_lib(),
quicer_nif:reg_open(),
DefAcceptors = erlang:system_info(schedulers_online) * 8,
ListenOpts = [ {cert, maps:get(certfile, Opts)}
, {key, maps:get(keyfile, Opts)}
, {alpn, ["mqtt"]}
, {conn_acceptors, maps:get(acceptors, Opts, DefAcceptors)}
, {idle_timeout_ms, emqx_config:get_listener_conf(ZoneName, ListenerName,
[mqtt, idle_timeout])}
],
ConnectionOpts = #{conn_callback => emqx_quic_connection
, peer_unidi_stream_count => 1
, peer_bidi_stream_count => 10
, zone => ZoneName
, listener => ListenerName
},
StreamOpts = [],
quicer:start_listener(listener_id(ZoneName, ListenerName),
port(ListenOn), {ListenOpts, ConnectionOpts, StreamOpts}).
replace(Opts, Key, Value) -> [{Key, Value} | proplists:delete(Key, Opts)].
esockd_opts(Opts0) ->
Opts1 = maps:with([acceptors, max_connections, proxy_protocol, proxy_protocol_timeout], Opts0),
Opts2 = case emqx_map_lib:deep_get([rate_limit, max_conn_rate], Opts0) of
infinity -> Opts1;
Rate -> Opts1#{max_conn_rate => Rate}
end,
Opts3 = Opts2#{access_rules => esockd_access_rules(maps:get(access_rules, Opts0, []))},
maps:to_list(case is_ssl(Opts0) of
false ->
Opts3#{tcp_options => tcp_opts(Opts0)};
true ->
Opts3#{ssl_options => ssl_opts(Opts0), tcp_options => tcp_opts(Opts0)}
end).
drop_tls13_for_old_otp(Options) ->
case proplists:get_value(ssl_options, Options) of
undefined -> Options;
SslOpts ->
SslOpts1 = emqx_tls_lib:drop_tls13_for_old_otp(SslOpts),
replace(Options, ssl_options, SslOpts1)
end.
start_mqtt_listener(Name, ListenOn, Options0) ->
Options = drop_tls13_for_old_otp(Options0),
SockOpts = esockd:parse_opt(Options),
esockd:open(Name, ListenOn, merge_default(SockOpts),
{emqx_connection, start_link, [Options -- SockOpts]}).
start_http_listener(Start, Name, ListenOn, RanchOpts, ProtoOpts) ->
Start(ws_name(Name, ListenOn), with_port(ListenOn, RanchOpts), ProtoOpts).
mqtt_path(Options) ->
proplists:get_value(mqtt_path, Options, "/mqtt").
ws_opts(Options) ->
WsPaths = [{mqtt_path(Options), emqx_ws_connection, Options}],
ws_opts(ZoneName, ListenerName, Opts) ->
WsPaths = [{maps:get(mqtt_path, Opts, "/mqtt"), emqx_ws_connection,
#{zone => ZoneName, listener => ListenerName}}],
Dispatch = cowboy_router:compile([{'_', WsPaths}]),
ProxyProto = proplists:get_value(proxy_protocol, Options, false),
ProxyProto = maps:get(proxy_protocol, Opts, false),
#{env => #{dispatch => Dispatch}, proxy_header => ProxyProto}.
ranch_opts(Options0) ->
Options = drop_tls13_for_old_otp(Options0),
NumAcceptors = proplists:get_value(acceptors, Options, 4),
MaxConnections = proplists:get_value(max_connections, Options, 1024),
TcpOptions = proplists:get_value(tcp_options, Options, []),
RanchOpts = #{num_acceptors => NumAcceptors,
max_connections => MaxConnections,
socket_opts => TcpOptions},
case proplists:get_value(ssl_options, Options) of
undefined -> RanchOpts;
SslOptions -> RanchOpts#{socket_opts => TcpOptions ++ SslOptions}
end.
ranch_opts(ListenOn, Opts) ->
NumAcceptors = maps:get(acceptors, Opts, 4),
MaxConnections = maps:get(max_connections, Opts, 1024),
SocketOpts = case is_ssl(Opts) of
true -> tcp_opts(Opts) ++ proplists:delete(handshake_timeout, ssl_opts(Opts));
false -> tcp_opts(Opts)
end,
#{num_acceptors => NumAcceptors,
max_connections => MaxConnections,
handshake_timeout => maps:get(handshake_timeout, Opts, 15000),
socket_opts => ip_port(ListenOn) ++ SocketOpts}.
with_port(Port, Opts = #{socket_opts := SocketOption}) when is_integer(Port) ->
Opts#{socket_opts => [{port, Port}| SocketOption]};
with_port({Addr, Port}, Opts = #{socket_opts := SocketOption}) ->
Opts#{socket_opts => [{ip, Addr}, {port, Port}| SocketOption]}.
ip_port(Port) when is_integer(Port) ->
[{port, Port}];
ip_port({Addr, Port}) ->
[{ip, Addr}, {port, Port}].
port(Port) when is_integer(Port) -> Port;
port({_Addr, Port}) when is_integer(Port) -> Port.
esockd_access_rules(StrRules) ->
Access = fun(S) ->
[A, CIDR] = string:tokens(S, " "),
{list_to_atom(A), case CIDR of "all" -> all; _ -> CIDR end}
end,
[Access(R) || R <- StrRules].
%% @doc Restart all listeners
-spec(restart() -> ok).
restart() ->
lists:foreach(fun restart_listener/1, emqx:get_env(listeners, [])).
foreach_listeners(fun restart_listener/3).
-spec(restart_listener(listener() | string() | binary()) -> ok | {error, any()}).
restart_listener(#{proto := Proto, listen_on := ListenOn, opts := Options}) ->
restart_listener(Proto, ListenOn, Options);
restart_listener(Identifier) ->
case emqx_listeners:find_by_id(Identifier) of
false -> {error, {no_such_listener, Identifier}};
Listener -> restart_listener(Listener)
-spec(restart_listener(atom()) -> ok | {error, term()}).
restart_listener(ListenerId) ->
apply_on_listener(ListenerId, fun restart_listener/3).
-spec(restart_listener(atom(), atom(), map()) -> ok | {error, term()}).
restart_listener(ZoneName, ListenerName, Conf) ->
case stop_listener(ZoneName, ListenerName, Conf) of
ok -> start_listener(ZoneName, ListenerName, Conf);
Error -> Error
end.
-spec(restart_listener(esockd:proto(), esockd:listen_on(), [esockd:option()]) ->
ok | {error, any()}).
restart_listener(tcp, ListenOn, _Options) ->
esockd:reopen('mqtt:tcp', ListenOn);
restart_listener(Proto, ListenOn, _Options) when Proto == ssl; Proto == tls ->
esockd:reopen('mqtt:ssl', ListenOn);
restart_listener(Proto, ListenOn, Options) when Proto == http; Proto == ws ->
_ = cowboy:stop_listener(ws_name('mqtt:ws', ListenOn)),
ok(start_listener(Proto, ListenOn, Options));
restart_listener(Proto, ListenOn, Options) when Proto == https; Proto == wss ->
_ = cowboy:stop_listener(ws_name('mqtt:wss', ListenOn)),
ok(start_listener(Proto, ListenOn, Options));
restart_listener(Proto, ListenOn, _Opts) ->
esockd:reopen(Proto, ListenOn).
ok({ok, _}) -> ok;
ok(Other) -> Other.
%% @doc Stop all listeners.
-spec(stop() -> ok).
stop() ->
lists:foreach(fun stop_listener/1, emqx:get_env(listeners, [])).
foreach_listeners(fun stop_listener/3).
-spec(stop_listener(listener()) -> ok | {error, term()}).
stop_listener(#{proto := Proto, listen_on := ListenOn, opts := Opts}) ->
stop_listener(Proto, ListenOn, Opts).
-spec(stop_listener(atom()) -> ok | {error, term()}).
stop_listener(ListenerId) ->
apply_on_listener(ListenerId, fun stop_listener/3).
-spec(stop_listener(esockd:proto(), esockd:listen_on(), [esockd:option()])
-> ok | {error, term()}).
stop_listener(tcp, ListenOn, _Opts) ->
esockd:close('mqtt:tcp', ListenOn);
stop_listener(Proto, ListenOn, _Opts) when Proto == ssl; Proto == tls ->
esockd:close('mqtt:ssl', ListenOn);
stop_listener(Proto, ListenOn, _Opts) when Proto == http; Proto == ws ->
cowboy:stop_listener(ws_name('mqtt:ws', ListenOn));
stop_listener(Proto, ListenOn, _Opts) when Proto == https; Proto == wss ->
cowboy:stop_listener(ws_name('mqtt:wss', ListenOn));
stop_listener(quic, _ListenOn, _Opts) ->
quicer:stop_listener('mqtt:quic');
stop_listener(Proto, ListenOn, _Opts) ->
esockd:close(Proto, ListenOn).
-spec(stop_listener(atom(), atom(), map()) -> ok | {error, term()}).
stop_listener(ZoneName, ListenerName, #{type := tcp, bind := ListenOn}) ->
esockd:close(listener_id(ZoneName, ListenerName), ListenOn);
stop_listener(ZoneName, ListenerName, #{type := ws}) ->
cowboy:stop_listener(listener_id(ZoneName, ListenerName));
stop_listener(ZoneName, ListenerName, #{type := quic}) ->
quicer:stop_listener(listener_id(ZoneName, ListenerName)).
merge_default(Options) ->
case lists:keytake(tcp_options, 1, Options) of
@ -288,23 +200,46 @@ format({Addr, Port}) when is_list(Addr) ->
format({Addr, Port}) when is_tuple(Addr) ->
io_lib:format("~s:~w", [inet:ntoa(Addr), Port]).
ws_name(Name, {_Addr, Port}) ->
ws_name(Name, Port);
ws_name(Name, Port) ->
list_to_atom(lists:concat([Name, ":", Port])).
listener_id(ZoneName, ListenerName) ->
list_to_atom(lists:append([atom_to_list(ZoneName), ":", atom_to_list(ListenerName)])).
identifier(Proto, Name) when is_atom(Proto) ->
identifier(atom_to_list(Proto), Name);
identifier(Proto, Name) ->
iolist_to_binary(["mqtt", ":", Proto, ":", Name]).
find_by_listen_on(_ListenOn, []) -> false;
find_by_listen_on(ListenOn, [#{listen_on := ListenOn} = L | _]) -> L;
find_by_listen_on(ListenOn, [_ | Rest]) -> find_by_listen_on(ListenOn, Rest).
find_by_id(_Id, []) -> false;
find_by_id(Id, [L | Rest]) ->
case identifier(L) =:= Id of
true -> L;
false -> find_by_id(Id, Rest)
decode_listener_id(Id) ->
case string:split(atom_to_list(Id), ":", leading) of
[Zone, Listen] -> {list_to_atom(Zone), list_to_atom(Listen)};
_ -> error({invalid_listener_id, Id})
end.
ssl_opts(Opts) ->
maps:to_list(
emqx_tls_lib:drop_tls13_for_old_otp(
maps:without([enable],
maps:get(ssl, Opts, #{})))).
tcp_opts(Opts) ->
maps:to_list(
maps:without([active_n],
maps:get(tcp, Opts, #{}))).
is_ssl(Opts) ->
emqx_map_lib:deep_get([ssl, enable], Opts, false).
foreach_listeners(Do) ->
lists:foreach(fun({ZoneName, ZoneConf}) ->
lists:foreach(fun({LName, LConf}) ->
Do(ZoneName, LName, merge_zone_and_listener_confs(ZoneConf, LConf))
end, maps:to_list(maps:get(listeners, ZoneConf, #{})))
end, maps:to_list(emqx_config:get([zones], #{}))).
%% merge the configs in zone and listeners in a manner that
%% all config entries in the listener are prior to the ones in the zone.
merge_zone_and_listener_confs(ZoneConf, ListenerConf) ->
ConfsInZonesOnly = [listeners, overall_max_connections],
BaseConf = maps:without(ConfsInZonesOnly, ZoneConf),
emqx_map_lib:deep_merge(BaseConf, ListenerConf).
apply_on_listener(ListenerId, Do) ->
{ZoneName, ListenerName} = decode_listener_id(ListenerId),
case emqx_config:find([zones, ZoneName, listeners, ListenerName]) of
{not_found, _, _} -> error({not_found, ListenerId});
{ok, Conf} -> Do(ZoneName, ListenerName, Conf)
end.

101
apps/emqx/src/emqx_map_lib.erl Normal file
View File

@ -0,0 +1,101 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_map_lib).
-export([ deep_get/2
, deep_get/3
, deep_find/2
, deep_put/3
, deep_merge/2
, safe_atom_key_map/1
, unsafe_atom_key_map/1
]).
-export_type([config_key/0, config_key_path/0]).
-type config_key() :: atom() | binary().
-type config_key_path() :: [config_key()].
%%-----------------------------------------------------------------
-spec deep_get(config_key_path(), map()) -> term().
deep_get(ConfKeyPath, Map) ->
case deep_find(ConfKeyPath, Map) of
{not_found, KeyPath, Data} -> error({not_found, KeyPath, Data});
{ok, Data} -> Data
end.
-spec deep_get(config_key_path(), map(), term()) -> term().
deep_get(ConfKeyPath, Map, Default) ->
case deep_find(ConfKeyPath, Map) of
{not_found, _KeyPath, _Data} -> Default;
{ok, Data} -> Data
end.
-spec deep_find(config_key_path(), map()) ->
{ok, term()} | {not_found, config_key_path(), term()}.
deep_find([], Map) ->
{ok, Map};
deep_find([Key | KeyPath] = Path, Map) when is_map(Map) ->
case maps:find(Key, Map) of
{ok, SubMap} -> deep_find(KeyPath, SubMap);
error -> {not_found, Path, Map}
end;
deep_find(_KeyPath, Data) ->
{not_found, _KeyPath, Data}.
-spec deep_put(config_key_path(), map(), term()) -> map().
deep_put([], Map, Config) when is_map(Map) ->
Config;
deep_put([], _Map, Config) -> %% not map, replace it
Config;
deep_put([Key | KeyPath], Map, Config) ->
SubMap = deep_put(KeyPath, maps:get(Key, Map, #{}), Config),
Map#{Key => SubMap}.
%% #{a => #{b => 3, c => 2}, d => 4}
%% = deep_merge(#{a => #{b => 1, c => 2}, d => 4}, #{a => #{b => 3}}).
-spec deep_merge(map(), map()) -> map().
deep_merge(BaseMap, NewMap) ->
NewKeys = maps:keys(NewMap) -- maps:keys(BaseMap),
MergedBase = maps:fold(fun(K, V, Acc) ->
case maps:find(K, NewMap) of
error ->
Acc#{K => V};
{ok, NewV} when is_map(V), is_map(NewV) ->
Acc#{K => deep_merge(V, NewV)};
{ok, NewV} ->
Acc#{K => NewV}
end
end, #{}, BaseMap),
maps:merge(MergedBase, maps:with(NewKeys, NewMap)).
unsafe_atom_key_map(Map) ->
covert_keys_to_atom(Map, fun(K) -> binary_to_atom(K, utf8) end).
safe_atom_key_map(Map) ->
covert_keys_to_atom(Map, fun(K) -> binary_to_existing_atom(K, utf8) end).
%%---------------------------------------------------------------------------
covert_keys_to_atom(BinKeyMap, Conv) when is_map(BinKeyMap) ->
maps:fold(
fun(K, V, Acc) when is_binary(K) ->
Acc#{Conv(K) => covert_keys_to_atom(V, Conv)};
(K, V, Acc) when is_atom(K) ->
%% richmap keys
Acc#{K => covert_keys_to_atom(V, Conv)}
end, #{}, BinKeyMap);
covert_keys_to_atom(ListV, Conv) when is_list(ListV) ->
[covert_keys_to_atom(V, Conv) || V <- ListV];
covert_keys_to_atom(Val, _) -> Val.

30
apps/emqx/src/emqx_misc.erl
View File

@ -197,7 +197,8 @@ check_oom(Policy) ->
check_oom(self(), Policy).
-spec(check_oom(pid(), emqx_types:oom_policy()) -> ok | {shutdown, term()}).
check_oom(Pid, #{message_queue_len := MaxQLen,
check_oom(_Pid, #{enable := false}) -> ok;
check_oom(Pid, #{max_message_queue_len := MaxQLen,
max_heap_size := MaxHeapSize}) ->
case process_info(Pid, [message_queue_len, total_heap_size]) of
undefined -> ok;
@ -214,13 +215,26 @@ do_check_oom([{Val, Max, Reason}|Rest]) ->
false -> do_check_oom(Rest)
end.
tune_heap_size(#{max_heap_size := MaxHeapSize}) ->
%% If set to zero, the limit is disabled.
erlang:process_flag(max_heap_size, #{size => MaxHeapSize,
kill => false,
error_logger => true
});
tune_heap_size(undefined) -> ok.
tune_heap_size(#{enable := false}) ->
ok;
%% If the max_heap_size is set to zero, the limit is disabled.
tune_heap_size(#{max_heap_size := MaxHeapSize}) when MaxHeapSize > 0 ->
MaxSize = case erlang:system_info(wordsize) of
8 -> % arch_64
(1 bsl 59) - 1;
4 -> % arch_32
(1 bsl 27) - 1
end,
OverflowedSize = case erlang:trunc(MaxHeapSize * 1.5) of
SZ when SZ > MaxSize -> MaxSize;
SZ -> SZ
end,
erlang:process_flag(max_heap_size, #{
size => OverflowedSize,
kill => true,
error_logger => true
}).
-spec(proc_name(atom(), pos_integer()) -> atom()).
proc_name(Mod, Id) ->

71
apps/emqx/src/emqx_mqtt_caps.erl
View File

@ -20,19 +20,13 @@
-include("emqx_mqtt.hrl").
-include("types.hrl").
-export([ check_pub/2
, check_sub/3
-export([ check_pub/3
, check_sub/4
]).
-export([ get_caps/1
, get_caps/2
, get_caps/3
-export([ get_caps/2
]).
-export([default_caps/0]).
-export([default/0]).
-export_type([caps/0]).
-type(caps() :: #{max_packet_size => integer(),
@ -46,7 +40,7 @@
shared_subscription => boolean()
}).
-define(UNLIMITED, 0).
-define(MAX_TOPIC_LEVELS, 65535).
-define(PUBCAP_KEYS, [max_topic_levels,
max_qos_allowed,
@ -62,7 +56,7 @@
-define(DEFAULT_CAPS, #{max_packet_size => ?MAX_PACKET_SIZE,
max_clientid_len => ?MAX_CLIENTID_LEN,
max_topic_alias => ?MAX_TOPIC_AlIAS,
max_topic_levels => ?UNLIMITED,
max_topic_levels => ?MAX_TOPIC_LEVELS,
max_qos_allowed => ?QOS_2,
retain_available => true,
wildcard_subscription => true,
@ -70,18 +64,18 @@
shared_subscription => true
}).
-spec(check_pub(emqx_types:zone(),
-spec(check_pub(emqx_types:zone(), atom(),
#{qos := emqx_types:qos(),
retain := boolean(),
topic := emqx_topic:topic()})
-> ok_or_error(emqx_types:reason_code())).
check_pub(Zone, Flags) when is_map(Flags) ->
check_pub(Zone, Listener, Flags) when is_map(Flags) ->
do_check_pub(case maps:take(topic, Flags) of
{Topic, Flags1} ->
Flags1#{topic_levels => emqx_topic:levels(Topic)};
error ->
Flags
end, get_caps(Zone, publish)).
end, maps:with(?PUBCAP_KEYS, get_caps(Zone, Listener))).
do_check_pub(#{topic_levels := Levels}, #{max_topic_levels := Limit})
when Limit > 0, Levels > Limit ->
@ -93,12 +87,12 @@ do_check_pub(#{retain := true}, #{retain_available := false}) ->
{error, ?RC_RETAIN_NOT_SUPPORTED};
do_check_pub(_Flags, _Caps) -> ok.
-spec(check_sub(emqx_types:zone(),
-spec(check_sub(emqx_types:zone(), atom(),
emqx_types:topic(),
emqx_types:subopts())
-> ok_or_error(emqx_types:reason_code())).
check_sub(Zone, Topic, SubOpts) ->
Caps = get_caps(Zone, subscribe),
check_sub(Zone, Listener, Topic, SubOpts) ->
Caps = maps:with(?SUBCAP_KEYS, get_caps(Zone, Listener)),
Flags = lists:foldl(
fun(max_topic_levels, Map) ->
Map#{topic_levels => emqx_topic:levels(Topic)};
@ -119,42 +113,7 @@ do_check_sub(#{is_shared := true}, #{shared_subscription := false}) ->
{error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED};
do_check_sub(_Flags, _Caps) -> ok.
default_caps() ->
?DEFAULT_CAPS.
get_caps(Zone, Cap, Def) ->
emqx_zone:get_env(Zone, Cap, Def).
get_caps(Zone, publish) ->
with_env(Zone, '$mqtt_pub_caps',
fun() ->
filter_caps(?PUBCAP_KEYS, get_caps(Zone))
end);
get_caps(Zone, subscribe) ->
with_env(Zone, '$mqtt_sub_caps',
fun() ->
filter_caps(?SUBCAP_KEYS, get_caps(Zone))
end).
get_caps(Zone) ->
with_env(Zone, '$mqtt_caps',
fun() ->
maps:map(fun(Cap, Def) ->
emqx_zone:get_env(Zone, Cap, Def)
end, ?DEFAULT_CAPS)
end).
filter_caps(Keys, Caps) ->
maps:filter(fun(Key, _Val) -> lists:member(Key, Keys) end, Caps).
-spec(default() -> caps()).
default() -> ?DEFAULT_CAPS.
with_env(Zone, Key, InitFun) ->
case emqx_zone:get_env(Zone, Key) of
undefined -> Caps = InitFun(),
ok = emqx_zone:set_env(Zone, Key, Caps),
Caps;
ZoneCaps -> ZoneCaps
end.
get_caps(Zone, Listener) ->
lists:foldl(fun({K, V}, Acc) ->
Acc#{K => emqx_config:get_listener_conf(Zone, Listener, [mqtt, K], V)}
end, #{}, maps:to_list(?DEFAULT_CAPS)).

2
apps/emqx/src/emqx_mqueue.erl
View File

@ -67,6 +67,8 @@
, dropped/1
]).
-define(NO_PRIORITY_TABLE, disabled).
-export_type([mqueue/0, options/0]).
-type(topic() :: emqx_topic:topic()).

105
apps/emqx/src/emqx_os_mon.erl
View File

@ -22,15 +22,9 @@
-logger_header("[OS_MON]").
-export([start_link/1]).
-export([start_link/0]).
-export([ get_cpu_check_interval/0
, set_cpu_check_interval/1
, get_cpu_high_watermark/0
, set_cpu_high_watermark/1
, get_cpu_low_watermark/0
, set_cpu_low_watermark/1
, get_mem_check_interval/0
-export([ get_mem_check_interval/0
, set_mem_check_interval/1
, get_sysmem_high_watermark/0
, set_sysmem_high_watermark/1
@ -51,31 +45,13 @@
-define(OS_MON, ?MODULE).
start_link(Opts) ->
gen_server:start_link({local, ?OS_MON}, ?MODULE, [Opts], []).
start_link() ->
gen_server:start_link({local, ?OS_MON}, ?MODULE, [], []).
%%--------------------------------------------------------------------
%% API
%%--------------------------------------------------------------------
get_cpu_check_interval() ->
call(get_cpu_check_interval).
set_cpu_check_interval(Seconds) ->
call({set_cpu_check_interval, Seconds}).
get_cpu_high_watermark() ->
call(get_cpu_high_watermark).
set_cpu_high_watermark(Float) ->
call({set_cpu_high_watermark, Float}).
get_cpu_low_watermark() ->
call(get_cpu_low_watermark).
set_cpu_low_watermark(Float) ->
call({set_cpu_low_watermark, Float}).
get_mem_check_interval() ->
memsup:get_check_interval() div 1000.
@ -88,47 +64,25 @@ get_sysmem_high_watermark() ->
memsup:get_sysmem_high_watermark().
set_sysmem_high_watermark(Float) ->
memsup:set_sysmem_high_watermark(Float / 100).
memsup:set_sysmem_high_watermark(Float).
get_procmem_high_watermark() ->
memsup:get_procmem_high_watermark().
set_procmem_high_watermark(Float) ->
memsup:set_procmem_high_watermark(Float / 100).
call(Req) ->
gen_server:call(?OS_MON, Req, infinity).
memsup:set_procmem_high_watermark(Float).
%%--------------------------------------------------------------------
%% gen_server callbacks
%%--------------------------------------------------------------------
init([Opts]) ->
set_mem_check_interval(proplists:get_value(mem_check_interval, Opts)),
set_sysmem_high_watermark(proplists:get_value(sysmem_high_watermark, Opts)),
set_procmem_high_watermark(proplists:get_value(procmem_high_watermark, Opts)),
{ok, ensure_check_timer(#{cpu_high_watermark => proplists:get_value(cpu_high_watermark, Opts),
cpu_low_watermark => proplists:get_value(cpu_low_watermark, Opts),
cpu_check_interval => proplists:get_value(cpu_check_interval, Opts),
timer => undefined})}.
handle_call(get_cpu_check_interval, _From, State) ->
{reply, maps:get(cpu_check_interval, State, undefined), State};
handle_call({set_cpu_check_interval, Seconds}, _From, State) ->
{reply, ok, State#{cpu_check_interval := Seconds}};
handle_call(get_cpu_high_watermark, _From, State) ->
{reply, maps:get(cpu_high_watermark, State, undefined), State};
handle_call({set_cpu_high_watermark, Float}, _From, State) ->
{reply, ok, State#{cpu_high_watermark := Float}};
handle_call(get_cpu_low_watermark, _From, State) ->
{reply, maps:get(cpu_low_watermark, State, undefined), State};
handle_call({set_cpu_low_watermark, Float}, _From, State) ->
{reply, ok, State#{cpu_low_watermark := Float}};
init([]) ->
Opts = emqx_config:get([sysmon, os]),
set_mem_check_interval(maps:get(mem_check_interval, Opts)),
set_sysmem_high_watermark(maps:get(sysmem_high_watermark, Opts)),
set_procmem_high_watermark(maps:get(procmem_high_watermark, Opts)),
_ = start_check_timer(),
{ok, #{}}.
handle_call(Req, _From, State) ->
?LOG(error, "Unexpected call: ~p", [Req]),
@ -138,32 +92,30 @@ handle_cast(Msg, State) ->
?LOG(error, "Unexpected cast: ~p", [Msg]),
{noreply, State}.
handle_info({timeout, Timer, check}, State = #{timer := Timer,
cpu_high_watermark := CPUHighWatermark,
cpu_low_watermark := CPULowWatermark}) ->
NState =
case emqx_vm:cpu_util() of %% TODO: should be improved?
0 ->
State#{timer := undefined};
handle_info({timeout, _Timer, check}, State) ->
CPUHighWatermark = emqx_config:get([sysmon, os, cpu_high_watermark]) * 100,
CPULowWatermark = emqx_config:get([sysmon, os, cpu_low_watermark]) * 100,
_ = case emqx_vm:cpu_util() of %% TODO: should be improved?
0 -> ok;
Busy when Busy >= CPUHighWatermark ->
emqx_alarm:activate(high_cpu_usage, #{usage => Busy,
emqx_alarm:activate(high_cpu_usage, #{usage => io_lib:format("~p%", [Busy]),
high_watermark => CPUHighWatermark,
low_watermark => CPULowWatermark}),
ensure_check_timer(State);
start_check_timer();
Busy when Busy =< CPULowWatermark ->
emqx_alarm:deactivate(high_cpu_usage),
ensure_check_timer(State);
start_check_timer();
_Busy ->
ensure_check_timer(State)
start_check_timer()
end,
{noreply, NState};
{noreply, State};
handle_info(Info, State) ->
?LOG(error, "unexpected info: ~p", [Info]),
{noreply, State}.
terminate(_Reason, #{timer := Timer}) ->
emqx_misc:cancel_timer(Timer).
terminate(_Reason, _State) ->
ok.
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
@ -172,8 +124,9 @@ code_change(_OldVsn, State, _Extra) ->
%% Internal functions
%%--------------------------------------------------------------------
ensure_check_timer(State = #{cpu_check_interval := Interval}) ->
start_check_timer() ->
Interval = emqx_config:get([sysmon, os, cpu_check_interval]),
case erlang:system_info(system_architecture) of
"x86_64-pc-linux-musl" -> State;
_ -> State#{timer := emqx_misc:start_timer(timer:seconds(Interval), check)}
"x86_64-pc-linux-musl" -> ok;
_ -> emqx_misc:start_timer(timer:seconds(Interval), check)
end.

2
apps/emqx/src/emqx_quic_connection.erl
View File

@ -21,6 +21,4 @@
]).
new_conn(Conn, {_L, COpts, _S}) when is_map(COpts) ->
new_conn(Conn, maps:to_list(COpts));
new_conn(Conn, COpts) ->
emqx_connection:start_link(emqx_quic_stream, Conn, COpts).

1253
apps/emqx/src/emqx_schema.erl
View File

File diff suppressed because it is too large Load Diff

39
apps/emqx/src/emqx_session.erl
View File

@ -92,13 +92,11 @@
-export_type([session/0]).
-import(emqx_zone, [get_env/3]).
-record(session, {
%% Client’s Subscriptions.
subscriptions :: map(),
%% Max subscriptions allowed
max_subscriptions :: non_neg_integer(),
max_subscriptions :: non_neg_integer() | infinity,
%% Upgrade QoS?
upgrade_qos :: boolean(),
%% Client <- Broker: QoS1/2 messages sent to the client but
@ -117,7 +115,7 @@
%% have not been completely acknowledged
awaiting_rel :: map(),
%% Maximum number of awaiting QoS2 messages allowed
max_awaiting_rel :: non_neg_integer(),
max_awaiting_rel :: non_neg_integer() | infinity,
%% Awaiting PUBREL Timeout (Unit: millsecond)
await_rel_timeout :: timeout(),
%% Created at
@ -159,27 +157,28 @@
%%--------------------------------------------------------------------
-spec(init(emqx_types:clientinfo(), emqx_types:conninfo()) -> session()).
init(#{zone := Zone}, #{receive_maximum := MaxInflight}) ->
#session{max_subscriptions = get_env(Zone, max_subscriptions, 0),
init(#{zone := Zone, listener := Listener}, #{receive_maximum := MaxInflight}) ->
#session{max_subscriptions = get_conf(Zone, Listener, max_subscriptions),
subscriptions = #{},
upgrade_qos = get_env(Zone, upgrade_qos, false),
upgrade_qos = get_conf(Zone, Listener, upgrade_qos),
inflight = emqx_inflight:new(MaxInflight),
mqueue = init_mqueue(Zone),
mqueue = init_mqueue(Zone, Listener),
next_pkt_id = 1,
retry_interval = timer:seconds(get_env(Zone, retry_interval, 0)),
retry_interval = timer:seconds(get_conf(Zone, Listener, retry_interval)),
awaiting_rel = #{},
max_awaiting_rel = get_env(Zone, max_awaiting_rel, 100),
await_rel_timeout = timer:seconds(get_env(Zone, await_rel_timeout, 300)),
max_awaiting_rel = get_conf(Zone, Listener, max_awaiting_rel),
await_rel_timeout = timer:seconds(get_conf(Zone, Listener, await_rel_timeout)),
created_at = erlang:system_time(millisecond)
}.
%% @private init mq
init_mqueue(Zone) ->
emqx_mqueue:init(#{max_len => get_env(Zone, max_mqueue_len, 1000),
store_qos0 => get_env(Zone, mqueue_store_qos0, true),
priorities => get_env(Zone, mqueue_priorities, none),
default_priority => get_env(Zone, mqueue_default_priority, lowest)
}).
init_mqueue(Zone, Listener) ->
emqx_mqueue:init(#{
max_len => get_conf(Zone, Listener, max_mqueue_len),
store_qos0 => get_conf(Zone, Listener, mqueue_store_qos0),
priorities => get_conf(Zone, Listener, mqueue_priorities),
default_priority => get_conf(Zone, Listener, mqueue_default_priority)
}).
%%--------------------------------------------------------------------
%% Info, Stats
@ -253,7 +252,7 @@ subscribe(ClientInfo = #{clientid := ClientId}, TopicFilter, SubOpts,
end.
-compile({inline, [is_subscriptions_full/1]}).
is_subscriptions_full(#session{max_subscriptions = 0}) ->
is_subscriptions_full(#session{max_subscriptions = infinity}) ->
false;
is_subscriptions_full(#session{subscriptions = Subs,
max_subscriptions = MaxLimit}) ->
@ -302,7 +301,7 @@ publish(_PacketId, Msg, Session) ->
{ok, emqx_broker:publish(Msg), Session}.
-compile({inline, [is_awaiting_full/1]}).
is_awaiting_full(#session{max_awaiting_rel = 0}) ->
is_awaiting_full(#session{max_awaiting_rel = infinity}) ->
false;
is_awaiting_full(#session{awaiting_rel = AwaitingRel,
max_awaiting_rel = MaxLimit}) ->
@ -697,3 +696,5 @@ set_field(Name, Value, Session) ->
Pos = emqx_misc:index_of(Name, record_info(fields, session)),
setelement(Pos+1, Session, Value).
get_conf(Zone, Listener, Key) ->
emqx_config:get_listener_conf(Zone, Listener, [mqtt, Key]).

10
apps/emqx/src/emqx_sys.erl
View File

@ -32,8 +32,6 @@
, uptime/0
, datetime/0
, sysdescr/0
, sys_interval/0
, sys_heatbeat_interval/0
]).
-export([info/0]).
@ -104,15 +102,11 @@ datetime() ->
io_lib:format(
"~4..0w-~2..0w-~2..0w ~2..0w:~2..0w:~2..0w", [Y, M, D, H, MM, S])).
%% @doc Get sys interval
-spec(sys_interval() -> pos_integer()).
sys_interval() ->
emqx:get_env(broker_sys_interval, 60000).
emqx_config:get([broker, sys_msg_interval]).
%% @doc Get sys heatbeat interval
-spec(sys_heatbeat_interval() -> pos_integer()).
sys_heatbeat_interval() ->
emqx:get_env(broker_sys_heartbeat, 30000).
emqx_config:get([broker, sys_heartbeat_interval]).
%% @doc Get sys info
-spec(info() -> list(tuple())).

62
apps/emqx/src/emqx_sys_mon.erl
View File

@ -23,7 +23,7 @@
-logger_header("[SYSMON]").
-export([start_link/1]).
-export([start_link/0]).
%% compress unused warning
-export([procinfo/1]).
@ -37,25 +37,19 @@
, code_change/3
]).
-type(option() :: {long_gc, non_neg_integer()}
| {long_schedule, non_neg_integer()}
| {large_heap, non_neg_integer()}
| {busy_port, boolean()}
| {busy_dist_port, boolean()}).
-define(SYSMON, ?MODULE).
%% @doc Start the system monitor.
-spec(start_link(list(option())) -> startlink_ret()).
start_link(Opts) ->
gen_server:start_link({local, ?SYSMON}, ?MODULE, [Opts], []).
-spec(start_link() -> startlink_ret()).
start_link() ->
gen_server:start_link({local, ?SYSMON}, ?MODULE, [], []).
%%--------------------------------------------------------------------
%% gen_server callbacks
%%--------------------------------------------------------------------
init([Opts]) ->
_ = erlang:system_monitor(self(), parse_opt(Opts)),
init([]) ->
_ = erlang:system_monitor(self(), sysm_opts()),
emqx_logger:set_proc_metadata(#{sysmon => true}),
%% Monitor cluster partition event
@ -66,30 +60,28 @@ init([Opts]) ->
start_timer(State) ->
State#{timer := emqx_misc:start_timer(timer:seconds(2), reset)}.
parse_opt(Opts) ->
parse_opt(Opts, []).
parse_opt([], Acc) ->
sysm_opts() ->
sysm_opts(maps:to_list(emqx_config:get([sysmon, vm])), []).
sysm_opts([], Acc) ->
Acc;
parse_opt([{long_gc, 0}|Opts], Acc) ->
parse_opt(Opts, Acc);
parse_opt([{long_gc, Ms}|Opts], Acc) when is_integer(Ms) ->
parse_opt(Opts, [{long_gc, Ms}|Acc]);
parse_opt([{long_schedule, 0}|Opts], Acc) ->
parse_opt(Opts, Acc);
parse_opt([{long_schedule, Ms}|Opts], Acc) when is_integer(Ms) ->
parse_opt(Opts, [{long_schedule, Ms}|Acc]);
parse_opt([{large_heap, Size}|Opts], Acc) when is_integer(Size) ->
parse_opt(Opts, [{large_heap, Size}|Acc]);
parse_opt([{busy_port, true}|Opts], Acc) ->
parse_opt(Opts, [busy_port|Acc]);
parse_opt([{busy_port, false}|Opts], Acc) ->
parse_opt(Opts, Acc);
parse_opt([{busy_dist_port, true}|Opts], Acc) ->
parse_opt(Opts, [busy_dist_port|Acc]);
parse_opt([{busy_dist_port, false}|Opts], Acc) ->
parse_opt(Opts, Acc);
parse_opt([_Opt|Opts], Acc) ->
parse_opt(Opts, Acc).
sysm_opts([{_, disabled}|Opts], Acc) ->
sysm_opts(Opts, Acc);
sysm_opts([{long_gc, Ms}|Opts], Acc) when is_integer(Ms) ->
sysm_opts(Opts, [{long_gc, Ms}|Acc]);
sysm_opts([{long_schedule, Ms}|Opts], Acc) when is_integer(Ms) ->
sysm_opts(Opts, [{long_schedule, Ms}|Acc]);
sysm_opts([{large_heap, Size}|Opts], Acc) when is_integer(Size) ->
sysm_opts(Opts, [{large_heap, Size}|Acc]);
sysm_opts([{busy_port, true}|Opts], Acc) ->
sysm_opts(Opts, [busy_port|Acc]);
sysm_opts([{busy_port, false}|Opts], Acc) ->
sysm_opts(Opts, Acc);
sysm_opts([{busy_dist_port, true}|Opts], Acc) ->
sysm_opts(Opts, [busy_dist_port|Acc]);
sysm_opts([{busy_dist_port, false}|Opts], Acc) ->
sysm_opts(Opts, Acc);
sysm_opts([_Opt|Opts], Acc) ->
sysm_opts(Opts, Acc).
handle_call(Req, _From, State) ->
?LOG(error, "Unexpected call: ~p", [Req]),

11
apps/emqx/src/emqx_sys_sup.erl
View File

@ -27,10 +27,10 @@ start_link() ->
init([]) ->
Childs = [child_spec(emqx_sys),
child_spec(emqx_alarm, [config(alarm)]),
child_spec(emqx_sys_mon, [config(sysmon)]),
child_spec(emqx_os_mon, [config(os_mon)]),
child_spec(emqx_vm_mon, [config(vm_mon)])],
child_spec(emqx_alarm),
child_spec(emqx_sys_mon),
child_spec(emqx_os_mon),
child_spec(emqx_vm_mon)],
{ok, {{one_for_one, 10, 100}, Childs}}.
%%--------------------------------------------------------------------
@ -48,6 +48,3 @@ child_spec(Mod, Args) ->
type => worker,
modules => [Mod]
}.
config(Name) -> emqx:get_env(Name, []).

23
apps/emqx/src/emqx_tls_lib.erl
View File

@ -161,17 +161,16 @@ drop_tls13_for_old_otp(SslOpts) ->
, "TLS_AES_128_CCM_8_SHA256"
]).
drop_tls13(SslOpts0) ->
SslOpts1 = case proplists:get_value(versions, SslOpts0) of
undefined -> SslOpts0;
Vsns -> replace(SslOpts0, versions, Vsns -- ['tlsv1.3'])
SslOpts1 = case maps:find(versions, SslOpts0) of
error -> SslOpts0;
{ok, Vsns} -> SslOpts0#{versions => (Vsns -- ['tlsv1.3'])}
end,
case proplists:get_value(ciphers, SslOpts1) of
undefined -> SslOpts1;
Ciphers -> replace(SslOpts1, ciphers, Ciphers -- ?TLSV13_EXCLUSIVE_CIPHERS)
case maps:find(ciphers, SslOpts1) of
error -> SslOpts1;
{ok, Ciphers} ->
SslOpts1#{ciphers => Ciphers -- ?TLSV13_EXCLUSIVE_CIPHERS}
end.
replace(Opts, Key, Value) -> [{Key, Value} | proplists:delete(Key, Opts)].
-if(?OTP_RELEASE > 22).
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
@ -181,13 +180,13 @@ drop_tls13_test() ->
?assert(lists:member('tlsv1.3', Versions)),
Ciphers = default_ciphers(),
?assert(has_tlsv13_cipher(Ciphers)),
Opts0 = [{versions, Versions}, {ciphers, Ciphers}, other, {bool, true}],
Opts0 = #{versions => Versions, ciphers => Ciphers, other => true},
Opts = drop_tls13(Opts0),
?assertNot(lists:member('tlsv1.3', proplists:get_value(versions, Opts))),
?assertNot(has_tlsv13_cipher(proplists:get_value(ciphers, Opts))).
?assertNot(lists:member('tlsv1.3', maps:get(versions, Opts, undefined))),
?assertNot(has_tlsv13_cipher(maps:get(ciphers, Opts, undefined))).
drop_tls13_no_versions_cipers_test() ->
Opts0 = [other, {bool, true}],
Opts0 = #{other => 0, bool => true},
Opts = drop_tls13(Opts0),
?_assertEqual(Opts0, Opts).

5
apps/emqx/src/emqx_types.erl
View File

@ -209,7 +209,8 @@
-type(infos() :: #{atom() => term()}).
-type(stats() :: [{atom(), term()}]).
-type(oom_policy() :: #{message_queue_len => non_neg_integer(),
max_heap_size => non_neg_integer()
-type(oom_policy() :: #{max_message_queue_len => non_neg_integer(),
max_heap_size => non_neg_integer(),
enable => boolean()
}).

90
apps/emqx/src/emqx_vm_mon.erl
View File

@ -21,15 +21,7 @@
-include("logger.hrl").
%% APIs
-export([start_link/1]).
-export([ get_check_interval/0
, set_check_interval/1
, get_process_high_watermark/0
, set_process_high_watermark/1
, get_process_low_watermark/0
, set_process_low_watermark/1
]).
-export([start_link/0]).
%% gen_server callbacks
-export([ init/1
@ -42,61 +34,19 @@
-define(VM_MON, ?MODULE).
start_link(Opts) ->
gen_server:start_link({local, ?VM_MON}, ?MODULE, [Opts], []).
%%--------------------------------------------------------------------
%% API
%%--------------------------------------------------------------------
get_check_interval() ->
call(get_check_interval).
set_check_interval(Seconds) ->
call({set_check_interval, Seconds}).
get_process_high_watermark() ->
call(get_process_high_watermark).
set_process_high_watermark(Float) ->
call({set_process_high_watermark, Float}).
get_process_low_watermark() ->
call(get_process_low_watermark).
set_process_low_watermark(Float) ->
call({set_process_low_watermark, Float}).
call(Req) ->
gen_server:call(?VM_MON, Req, infinity).
start_link() ->
gen_server:start_link({local, ?VM_MON}, ?MODULE, [], []).
%%--------------------------------------------------------------------
%% gen_server callbacks
%%--------------------------------------------------------------------
init([Opts]) ->
{ok, ensure_check_timer(#{check_interval => proplists:get_value(check_interval, Opts),
process_high_watermark => proplists:get_value(process_high_watermark, Opts),
process_low_watermark => proplists:get_value(process_low_watermark, Opts),
timer => undefined})}.
handle_call(get_check_interval, _From, State) ->
{reply, maps:get(check_interval, State, undefined), State};
handle_call({set_check_interval, Seconds}, _From, State) ->
{reply, ok, State#{check_interval := Seconds}};
handle_call(get_process_high_watermark, _From, State) ->
{reply, maps:get(process_high_watermark, State, undefined), State};
handle_call({set_process_high_watermark, Float}, _From, State) ->
{reply, ok, State#{process_high_watermark := Float}};
handle_call(get_process_low_watermark, _From, State) ->
{reply, maps:get(process_low_watermark, State, undefined), State};
handle_call({set_process_low_watermark, Float}, _From, State) ->
{reply, ok, State#{process_low_watermark := Float}};
init([]) ->
start_check_timer(),
{ok, #{}}.
handle_call(Req, _From, State) ->
?LOG(error, "[VM_MON] Unexpected call: ~p", [Req]),
@ -106,29 +56,30 @@ handle_cast(Msg, State) ->
?LOG(error, "[VM_MON] Unexpected cast: ~p", [Msg]),
{noreply, State}.
handle_info({timeout, Timer, check},
State = #{timer := Timer,
process_high_watermark := ProcHighWatermark,
process_low_watermark := ProcLowWatermark}) ->
handle_info({timeout, _Timer, check}, State) ->
ProcHighWatermark = emqx_config:get([sysmon, vm, process_high_watermark]),
ProcLowWatermark = emqx_config:get([sysmon, vm, process_low_watermark]),
ProcessCount = erlang:system_info(process_count),
case ProcessCount / erlang:system_info(process_limit) * 100 of
case ProcessCount / erlang:system_info(process_limit) of
Percent when Percent >= ProcHighWatermark ->
emqx_alarm:activate(too_many_processes, #{usage => Percent,
high_watermark => ProcHighWatermark,
low_watermark => ProcLowWatermark});
emqx_alarm:activate(too_many_processes, #{
usage => io_lib:format("~p%", [Percent*100]),
high_watermark => ProcHighWatermark,
low_watermark => ProcLowWatermark});
Percent when Percent < ProcLowWatermark ->
emqx_alarm:deactivate(too_many_processes);
_Precent ->
ok
end,
{noreply, ensure_check_timer(State)};
start_check_timer(),
{noreply, State};
handle_info(Info, State) ->
?LOG(error, "[VM_MON] Unexpected info: ~p", [Info]),
{noreply, State}.
terminate(_Reason, #{timer := Timer}) ->
emqx_misc:cancel_timer(Timer).
terminate(_Reason, _State) ->
ok.
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
@ -137,5 +88,6 @@ code_change(_OldVsn, State, _Extra) ->
%% Internal functions
%%--------------------------------------------------------------------
ensure_check_timer(State = #{check_interval := Interval}) ->
State#{timer := emqx_misc:start_timer(timer:seconds(Interval), check)}.
start_check_timer() ->
Interval = emqx_config:get([sysmon, vm, process_check_interval]),
emqx_misc:start_timer(Interval, check).

132
apps/emqx/src/emqx_ws_connection.erl
View File

@ -62,8 +62,6 @@
sockname :: emqx_types:peername(),
%% Sock state
sockstate :: emqx_types:sockstate(),
%% Simulate the active_n opt
active_n :: pos_integer(),
%% MQTT Piggyback
mqtt_piggyback :: single | multiple,
%% Limiter
@ -85,7 +83,11 @@
%% Idle Timeout
idle_timeout :: timeout(),
%% Idle Timer
idle_timer :: maybe(reference())
idle_timer :: maybe(reference()),
%% Zone name
zone :: atom(),
%% Listener Name
listener :: atom()
}).
-type(state() :: #state{}).
@ -93,7 +95,7 @@
-type(ws_cmd() :: {active, boolean()}|close).
-define(ACTIVE_N, 100).
-define(INFO_KEYS, [socktype, peername, sockname, sockstate, active_n]).
-define(INFO_KEYS, [socktype, peername, sockname, sockstate]).
-define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt]).
-define(CONN_STATS, [recv_pkt, recv_msg, send_pkt, send_msg]).
@ -124,8 +126,6 @@ info(sockname, #state{sockname = Sockname}) ->
Sockname;
info(sockstate, #state{sockstate = SockSt}) ->
SockSt;
info(active_n, #state{active_n = ActiveN}) ->
ActiveN;
info(limiter, #state{limiter = Limiter}) ->
maybe_apply(fun emqx_limiter:info/1, Limiter);
info(channel, #state{channel = Channel}) ->
@ -174,21 +174,13 @@ call(WsPid, Req, Timeout) when is_pid(WsPid) ->
%% WebSocket callbacks
%%--------------------------------------------------------------------
init(Req, Opts) ->
init(Req, #{zone := Zone, listener := Listener} = Opts) ->
%% WS Transport Idle Timeout
IdleTimeout = proplists:get_value(idle_timeout, Opts, 7200000),
DeflateOptions = maps:from_list(proplists:get_value(deflate_options, Opts, [])),
MaxFrameSize = case proplists:get_value(max_frame_size, Opts, 0) of
0 -> infinity;
I -> I
end,
Compress = proplists:get_bool(compress, Opts),
WsOpts = #{compress => Compress,
deflate_opts => DeflateOptions,
max_frame_size => MaxFrameSize,
idle_timeout => IdleTimeout
WsOpts = #{compress => get_ws_opts(Zone, Listener, compress),
deflate_opts => get_ws_opts(Zone, Listener, deflate_opts),
max_frame_size => get_ws_opts(Zone, Listener, max_frame_size),
idle_timeout => get_ws_opts(Zone, Listener, idle_timeout)
},
case check_origin_header(Req, Opts) of
{error, Message} ->
?LOG(error, "Invalid Origin Header ~p~n", [Message]),
@ -196,18 +188,17 @@ init(Req, Opts) ->
ok -> parse_sec_websocket_protocol(Req, Opts, WsOpts)
end.
parse_sec_websocket_protocol(Req, Opts, WsOpts) ->
FailIfNoSubprotocol = proplists:get_value(fail_if_no_subprotocol, Opts),
parse_sec_websocket_protocol(Req, #{zone := Zone, listener := Listener} = Opts, WsOpts) ->
case cowboy_req:parse_header(<<"sec-websocket-protocol">>, Req) of
undefined ->
case FailIfNoSubprotocol of
case get_ws_opts(Zone, Listener, fail_if_no_subprotocol) of
true ->
{ok, cowboy_req:reply(400, Req), WsOpts};
false ->
{cowboy_websocket, Req, [Req, Opts], WsOpts}
end;
Subprotocols ->
SupportedSubprotocols = proplists:get_value(supported_subprotocols, Opts),
SupportedSubprotocols = get_ws_opts(Zone, Listener, supported_subprotocols),
NSupportedSubprotocols = [list_to_binary(Subprotocol)
|| Subprotocol <- SupportedSubprotocols],
case pick_subprotocol(Subprotocols, NSupportedSubprotocols) of
@ -231,31 +222,30 @@ pick_subprotocol([Subprotocol | Rest], SupportedSubprotocols) ->
pick_subprotocol(Rest, SupportedSubprotocols)
end.
parse_header_fun_origin(Req, Opts) ->
parse_header_fun_origin(Req, #{zone := Zone, listener := Listener}) ->
case cowboy_req:header(<<"origin">>, Req) of
undefined ->
case proplists:get_bool(allow_origin_absence, Opts) of
case get_ws_opts(Zone, Listener, allow_origin_absence) of
true -> ok;
false -> {error, origin_header_cannot_be_absent}
end;
Value ->
Origins = proplists:get_value(check_origins, Opts, []),
case lists:member(Value, Origins) of
case lists:member(Value, get_ws_opts(Zone, Listener, check_origins)) of
true -> ok;
false -> {origin_not_allowed, Value}
end
end.
check_origin_header(Req, Opts) ->
case proplists:get_bool(check_origin_enable, Opts) of
check_origin_header(Req, #{zone := Zone, listener := Listener} = Opts) ->
case get_ws_opts(Zone, Listener, check_origin_enable) of
true -> parse_header_fun_origin(Req, Opts);
false -> ok
end.
websocket_init([Req, Opts]) ->
websocket_init([Req, #{zone := Zone, listener := Listener} = Opts]) ->
{Peername, Peercert} =
case proplists:get_bool(proxy_protocol, Opts)
andalso maps:get(proxy_header, Req) of
case emqx_config:get_listener_conf(Zone, Listener, [proxy_protocol]) andalso
maps:get(proxy_header, Req) of
#{src_address := SrcAddr, src_port := SrcPort, ssl := SSL} ->
SourceName = {SrcAddr, SrcPort},
%% Notice: Only CN is available in Proxy Protocol V2 additional info
@ -266,7 +256,7 @@ websocket_init([Req, Opts]) ->
{SourceName, SourceSSL};
#{src_address := SrcAddr, src_port := SrcPort} ->
SourceName = {SrcAddr, SrcPort},
{SourceName , nossl};
{SourceName, nossl};
_ ->
{get_peer(Req, Opts), cowboy_req:cert(Req)}
end,
@ -288,28 +278,35 @@ websocket_init([Req, Opts]) ->
ws_cookie => WsCookie,
conn_mod => ?MODULE
},
Zone = proplists:get_value(zone, Opts),
PubLimit = emqx_zone:publish_limit(Zone),
BytesIn = proplists:get_value(rate_limit, Opts),
RateLimit = emqx_zone:ratelimit(Zone),
Limiter = emqx_limiter:init(Zone, PubLimit, BytesIn, RateLimit),
ActiveN = proplists:get_value(active_n, Opts, ?ACTIVE_N),
MQTTPiggyback = proplists:get_value(mqtt_piggyback, Opts, multiple),
FrameOpts = emqx_zone:mqtt_frame_options(Zone),
Limiter = emqx_limiter:init(Zone, undefined, undefined, []),
MQTTPiggyback = get_ws_opts(Zone, Listener, mqtt_piggyback),
FrameOpts = #{
strict_mode => emqx_config:get_listener_conf(Zone, Listener, [mqtt, strict_mode]),
max_size => emqx_config:get_listener_conf(Zone, Listener, [mqtt, max_packet_size])
},
ParseState = emqx_frame:initial_parse_state(FrameOpts),
Serialize = emqx_frame:serialize_opts(),
Channel = emqx_channel:init(ConnInfo, Opts),
GcState = emqx_zone:init_gc_state(Zone),
StatsTimer = emqx_zone:stats_timer(Zone),
GcState = case emqx_config:get_listener_conf(Zone, Listener, [force_gc]) of
#{enable := false} -> undefined;
GcPolicy -> emqx_gc:init(GcPolicy)
end,
StatsTimer = case emqx_config:get_listener_conf(Zone, Listener, [stats, enable]) of
true -> undefined;
false -> disabled
end,
%% MQTT Idle Timeout
IdleTimeout = emqx_zone:idle_timeout(Zone),
IdleTimeout = emqx_channel:get_mqtt_conf(Zone, Listener, idle_timeout),
IdleTimer = start_timer(IdleTimeout, idle_timeout),
emqx_misc:tune_heap_size(emqx_zone:oom_policy(Zone)),
case emqx_config:get_listener_conf(emqx_channel:info(zone, Channel),
emqx_channel:info(listener, Channel), [force_shutdown]) of
#{enable := false} -> ok;
ShutdownPolicy -> emqx_misc:tune_heap_size(ShutdownPolicy)
end,
emqx_logger:set_metadata_peername(esockd:format(Peername)),
{ok, #state{peername = Peername,
sockname = Sockname,
sockstate = running,
active_n = ActiveN,
mqtt_piggyback = MQTTPiggyback,
limiter = Limiter,
parse_state = ParseState,
@ -319,7 +316,9 @@ websocket_init([Req, Opts]) ->
postponed = [],
stats_timer = StatsTimer,
idle_timeout = IdleTimeout,
idle_timer = IdleTimer
idle_timer = IdleTimer,
zone = Zone,
listener = Listener
}, hibernate}.
websocket_handle({binary, Data}, State) when is_list(Data) ->
@ -372,7 +371,8 @@ websocket_info({check_gc, Stats}, State) ->
return(check_oom(run_gc(Stats, State)));
websocket_info(Deliver = {deliver, _Topic, _Msg},
State = #state{active_n = ActiveN}) ->
State = #state{zone = Zone, listener = Listener}) ->
ActiveN = emqx_config:get_listener_conf(Zone, Listener, [tcp, active_n]),
Delivers = [Deliver|emqx_misc:drain_deliver(ActiveN)],
with_channel(handle_deliver, [Delivers], State);
@ -521,11 +521,16 @@ run_gc(Stats, State = #state{gc_state = GcSt}) ->
end.
check_oom(State = #state{channel = Channel}) ->
OomPolicy = emqx_zone:oom_policy(emqx_channel:info(zone, Channel)),
case ?ENABLED(OomPolicy) andalso emqx_misc:check_oom(OomPolicy) of
Shutdown = {shutdown, _Reason} ->
postpone(Shutdown, State);
_Other -> State
ShutdownPolicy = emqx_config:get_listener_conf(emqx_channel:info(zone, Channel),
emqx_channel:info(listener, Channel), [force_shutdown]),
case ShutdownPolicy of
#{enable := false} -> State;
#{enable := true} ->
case emqx_misc:check_oom(ShutdownPolicy) of
Shutdown = {shutdown, _Reason} ->
postpone(Shutdown, State);
_Other -> State
end
end.
%%--------------------------------------------------------------------
@ -554,11 +559,12 @@ parse_incoming(Data, State = #state{parse_state = ParseState}) ->
%% Handle incoming packet
%%--------------------------------------------------------------------
handle_incoming(Packet, State = #state{active_n = ActiveN})
handle_incoming(Packet, State = #state{zone = Zone, listener = Listener})
when is_record(Packet, mqtt_packet) ->
?LOG(debug, "RECV ~s", [emqx_packet:format(Packet)]),
ok = inc_incoming_stats(Packet),
NState = case emqx_pd:get_counter(incoming_pubs) > ActiveN of
NState = case emqx_pd:get_counter(incoming_pubs) >
emqx_config:get_listener_conf(Zone, Listener, [tcp, active_n]) of
true -> postpone({cast, rate_limit}, State);
false -> State
end,
@ -589,11 +595,13 @@ with_channel(Fun, Args, State = #state{channel = Channel}) ->
%% Handle outgoing packets
%%--------------------------------------------------------------------
handle_outgoing(Packets, State = #state{active_n = ActiveN, mqtt_piggyback = MQTTPiggyback}) ->
handle_outgoing(Packets, State = #state{mqtt_piggyback = MQTTPiggyback,
zone = Zone, listener = Listener}) ->
IoData = lists:map(serialize_and_inc_stats_fun(State), Packets),
Oct = iolist_size(IoData),
ok = inc_sent_stats(length(Packets), Oct),
NState = case emqx_pd:get_counter(outgoing_pubs) > ActiveN of
NState = case emqx_pd:get_counter(outgoing_pubs) >
emqx_config:get_listener_conf(Zone, Listener, [tcp, active_n]) of
true ->
Stats = #{cnt => emqx_pd:reset_counter(outgoing_pubs),
oct => emqx_pd:reset_counter(outgoing_bytes)
@ -742,9 +750,10 @@ classify([Event|More], Packets, Cmds, Events) ->
trigger(Event) -> erlang:send(self(), Event).
get_peer(Req, Opts) ->
get_peer(Req, #{zone := Zone, listener := Listener}) ->
{PeerAddr, PeerPort} = cowboy_req:peer(Req),
AddrHeader = cowboy_req:header(proplists:get_value(proxy_address_header, Opts), Req, <<>>),
AddrHeader = cowboy_req:header(
get_ws_opts(Zone, Listener, proxy_address_header), Req, <<>>),
ClientAddr = case string:tokens(binary_to_list(AddrHeader), ", ") of
[] ->
undefined;
@ -757,7 +766,8 @@ get_peer(Req, Opts) ->
_ ->
PeerAddr
end,
PortHeader = cowboy_req:header(proplists:get_value(proxy_port_header, Opts), Req, <<>>),
PortHeader = cowboy_req:header(
get_ws_opts(Zone, Listener, proxy_port_header), Req, <<>>),
ClientPort = case string:tokens(binary_to_list(PortHeader), ", ") of
[] ->
undefined;
@ -778,3 +788,5 @@ set_field(Name, Value, State) ->
Pos = emqx_misc:index_of(Name, record_info(fields, state)),
setelement(Pos+1, State, Value).
get_ws_opts(Zone, Listener, Key) ->
emqx_config:get_listener_conf(Zone, Listener, [websocket, Key]).

1
apps/emqx/test/emqx_SUITE.erl
View File

@ -27,6 +27,7 @@ all() -> emqx_ct:all(?MODULE).
init_per_suite(Config) ->
emqx_ct_helpers:start_apps([]),
ct:pal("------------config: ~p", [emqx_config:get()]),
Config.
end_per_suite(_Config) ->

18
apps/emqx/test/emqx_access_control_SUITE.erl
View File

@ -39,25 +39,14 @@ t_authorize(_) ->
Publish = ?PUBLISH_PACKET(?QOS_0, <<"t">>, 1, <<"payload">>),
?assertEqual(allow, emqx_access_control:authorize(clientinfo(), Publish, <<"t">>)).
t_bypass_auth_plugins(_) ->
ClientInfo = clientinfo(),
emqx_zone:set_env(bypass_zone, bypass_auth_plugins, true),
emqx:hook('client.authenticate',{?MODULE, auth_fun, []}),
?assertMatch(ok, emqx_access_control:authenticate(ClientInfo#{zone => bypass_zone})),
?assertMatch({error, bad_username_or_password}, emqx_access_control:authenticate(ClientInfo)).
%%--------------------------------------------------------------------
%% Helper functions
%%--------------------------------------------------------------------
auth_fun(#{zone := bypass_zone}, _) ->
{stop, ok};
auth_fun(#{zone := _}, _) ->
{stop, {error, bad_username_or_password}}.
clientinfo() -> clientinfo(#{}).
clientinfo(InitProps) ->
maps:merge(#{zone => zone,
maps:merge(#{zone => default,
listener => mqtt_tcp,
protocol => mqtt,
peerhost => {127,0,0,1},
clientid => <<"clientid">>,
@ -67,3 +56,6 @@ clientinfo(InitProps) ->
peercert => undefined,
mountpoint => undefined
}, InitProps).
toggle_auth(Bool) when is_boolean(Bool) ->
emqx_config:put_listener_conf(default, mqtt_tcp, [auth, enable], Bool).

71
apps/emqx/test/emqx_acl_cache_SUITE.erl
View File

@ -26,6 +26,7 @@ all() -> emqx_ct:all(?MODULE).
init_per_suite(Config) ->
emqx_ct_helpers:boot_modules(all),
emqx_ct_helpers:start_apps([]),
toggle_acl(true),
Config.
end_per_suite(_Config) ->
@ -55,7 +56,6 @@ t_clean_acl_cache(_) ->
?assertEqual(0, length(gen_server:call(ClientPid, list_acl_cache))),
emqtt:stop(Client).
t_drain_acl_cache(_) ->
{ok, Client} = emqtt:start_link([{clientid, <<"emqx_c">>}]),
{ok, _} = emqtt:connect(Client),
@ -79,70 +79,5 @@ t_drain_acl_cache(_) ->
?assert(length(gen_server:call(ClientPid, list_acl_cache)) > 0),
emqtt:stop(Client).
% optimize??
t_reload_aclfile_and_cleanall(_Config) ->
RasieMsg = fun() -> Self = self(), #{puback => fun(Msg) -> Self ! {puback, Msg} end,
disconnected => fun(_) -> ok end,
publish => fun(_) -> ok end } end,
{ok, Client} = emqtt:start_link([{clientid, <<"emqx_c">>}, {proto_ver, v5},
{msg_handler, RasieMsg()}]),
{ok, _} = emqtt:connect(Client),
{ok, PktId} = emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, qos1),
%% Success publish to broker
receive
{puback, #{packet_id := PktId, reason_code := Rc}} ->
?assertEqual(16#10, Rc);
_ ->
?assert(false)
end,
%% Check acl cache list
[ClientPid] = emqx_cm:lookup_channels(<<"emqx_c">>),
?assert(length(gen_server:call(ClientPid, list_acl_cache)) > 0),
emqtt:stop(Client).
%% @private
testdir(DataPath) ->
Ls = filename:split(DataPath),
filename:join(lists:sublist(Ls, 1, length(Ls) - 1)).
% t_cache_k(_) ->
% error('TODO').
% t_cache_v(_) ->
% error('TODO').
% t_cleanup_acl_cache(_) ->
% error('TODO').
% t_get_oldest_key(_) ->
% error('TODO').
% t_get_newest_key(_) ->
% error('TODO').
% t_get_cache_max_size(_) ->
% error('TODO').
% t_get_cache_size(_) ->
% error('TODO').
% t_dump_acl_cache(_) ->
% error('TODO').
% t_empty_acl_cache(_) ->
% error('TODO').
% t_put_acl_cache(_) ->
% error('TODO').
% t_get_acl_cache(_) ->
% error('TODO').
% t_is_enabled(_) ->
% error('TODO').
toggle_acl(Bool) when is_boolean(Bool) ->
emqx_config:put_listener_conf(default, mqtt_tcp, [acl, enable], Bool).

28
apps/emqx/test/emqx_alarm_SUITE.erl
View File

@ -27,27 +27,17 @@ all() -> emqx_ct:all(?MODULE).
init_per_testcase(t_size_limit, Config) ->
emqx_ct_helpers:boot_modules(all),
emqx_ct_helpers:start_apps([],
fun(emqx) ->
application:set_env(emqx, alarm, [{actions, [log,publish]},
{size_limit, 2},
{validity_period, 3600}]),
ok;
(_) ->
ok
end),
emqx_ct_helpers:start_apps([]),
emqx_config:update_config([alarm], #{
<<"size_limit">> => 2
}),
Config;
init_per_testcase(t_validity_period, Config) ->
emqx_ct_helpers:boot_modules(all),
emqx_ct_helpers:start_apps([],
fun(emqx) ->
application:set_env(emqx, alarm, [{actions, [log,publish]},
{size_limit, 1000},
{validity_period, 1}]),
ok;
(_) ->
ok
end),
emqx_ct_helpers:start_apps([]),
emqx_config:update_config([alarm], #{
<<"validity_period">> => <<"1s">>
}),
Config;
init_per_testcase(_, Config) ->
emqx_ct_helpers:boot_modules(all),
@ -89,7 +79,7 @@ t_size_limit(_) ->
ok = emqx_alarm:activate(b),
ok = emqx_alarm:deactivate(b),
?assertNotEqual({error, not_found}, get_alarm(a, emqx_alarm:get_alarms(deactivated))),
?assertNotEqual({error, not_found}, get_alarm(a, emqx_alarm:get_alarms(deactivated))),
?assertNotEqual({error, not_found}, get_alarm(b, emqx_alarm:get_alarms(deactivated))),
ok = emqx_alarm:activate(c),
ok = emqx_alarm:deactivate(c),
?assertNotEqual({error, not_found}, get_alarm(c, emqx_alarm:get_alarms(deactivated))),

18
apps/emqx/test/emqx_broker_SUITE.erl
View File

@ -42,19 +42,19 @@ end_per_suite(_Config) ->
%%--------------------------------------------------------------------
t_stats_fun(_) ->
?assertEqual(0, emqx_stats:getstat('subscribers.count')),
?assertEqual(0, emqx_stats:getstat('subscriptions.count')),
?assertEqual(0, emqx_stats:getstat('suboptions.count')),
Subscribers = emqx_stats:getstat('subscribers.count'),
Subscriptions = emqx_stats:getstat('subscriptions.count'),
Subopts = emqx_stats:getstat('suboptions.count'),
ok = emqx_broker:subscribe(<<"topic">>, <<"clientid">>),
ok = emqx_broker:subscribe(<<"topic2">>, <<"clientid">>),
emqx_broker:stats_fun(),
ct:sleep(10),
?assertEqual(2, emqx_stats:getstat('subscribers.count')),
?assertEqual(2, emqx_stats:getstat('subscribers.max')),
?assertEqual(2, emqx_stats:getstat('subscriptions.count')),
?assertEqual(2, emqx_stats:getstat('subscriptions.max')),
?assertEqual(2, emqx_stats:getstat('suboptions.count')),
?assertEqual(2, emqx_stats:getstat('suboptions.max')).
?assertEqual(Subscribers + 2, emqx_stats:getstat('subscribers.count')),
?assertEqual(Subscribers + 2, emqx_stats:getstat('subscribers.max')),
?assertEqual(Subscriptions + 2, emqx_stats:getstat('subscriptions.count')),
?assertEqual(Subscriptions + 2, emqx_stats:getstat('subscriptions.max')),
?assertEqual(Subopts + 2, emqx_stats:getstat('suboptions.count')),
?assertEqual(Subopts + 2, emqx_stats:getstat('suboptions.max')).
t_subscribed(_) ->
emqx_broker:subscribe(<<"topic">>),

189
apps/emqx/test/emqx_channel_SUITE.erl
View File

@ -24,7 +24,152 @@
-include_lib("eunit/include/eunit.hrl").
all() -> emqx_ct:all(?MODULE).
all() ->
emqx_ct:all(?MODULE).
mqtt_conf() ->
#{await_rel_timeout => 300,
idle_timeout => 15000,
ignore_loop_deliver => false,
keepalive_backoff => 0.75,
max_awaiting_rel => 100,
max_clientid_len => 65535,
max_inflight => 32,
max_mqueue_len => 1000,
max_packet_size => 1048576,
max_qos_allowed => 2,
max_subscriptions => infinity,
max_topic_alias => 65535,
max_topic_levels => 65535,
mountpoint => <<>>,
mqueue_default_priority => lowest,
mqueue_priorities => #{},
mqueue_store_qos0 => true,
peer_cert_as_clientid => disabled,
peer_cert_as_username => disabled,
response_information => [],
retain_available => true,
retry_interval => 30,
server_keepalive => disabled,
session_expiry_interval => 7200,
shared_subscription => true,
strict_mode => false,
upgrade_qos => false,
use_username_as_clientid => false,
wildcard_subscription => true}.
listener_mqtt_tcp_conf() ->
#{acceptors => 16,
access_rules => ["allow all"],
bind => {{0,0,0,0},1883},
max_connections => 1024000,
proxy_protocol => false,
proxy_protocol_timeout => 3000,
rate_limit =>
#{conn_bytes_in =>
["100KB","10s"],
conn_messages_in =>
["100","10s"],
max_conn_rate => 1000,
quota =>
#{conn_messages_routing => infinity,
overall_messages_routing => infinity}},
tcp =>
#{active_n => 100,
backlog => 1024,
buffer => 4096,
high_watermark => 1048576,
send_timeout => 15000,
send_timeout_close =>
true},
type => tcp}.
listener_mqtt_ws_conf() ->
#{acceptors => 16,
access_rules => ["allow all"],
bind => {{0,0,0,0},8083},
max_connections => 1024000,
proxy_protocol => false,
proxy_protocol_timeout => 3000,
rate_limit =>
#{conn_bytes_in =>
["100KB","10s"],
conn_messages_in =>
["100","10s"],
max_conn_rate => 1000,
quota =>
#{conn_messages_routing => infinity,
overall_messages_routing => infinity}},
tcp =>
#{active_n => 100,
backlog => 1024,
buffer => 4096,
high_watermark => 1048576,
send_timeout => 15000,
send_timeout_close =>
true},
type => ws,
websocket =>
#{allow_origin_absence =>
true,
check_origin_enable =>
false,
check_origins => [],
compress => false,
deflate_opts =>
#{client_max_window_bits =>
15,
mem_level => 8,
server_max_window_bits =>
15},
fail_if_no_subprotocol =>
true,
idle_timeout => 86400000,
max_frame_size => infinity,
mqtt_path => "/mqtt",
mqtt_piggyback => multiple,
proxy_address_header =>
"x-forwarded-for",
proxy_port_header =>
"x-forwarded-port",
supported_subprotocols =>
["mqtt","mqtt-v3",
"mqtt-v3.1.1",
"mqtt-v5"]}}.
default_zone_conf() ->
#{zones =>
#{default =>
#{ acl => #{
cache => #{enable => true,max_size => 32, ttl => 60000},
deny_action => ignore,
enable => false
},
auth => #{enable => false},
overall_max_connections => infinity,
stats => #{enable => true},
conn_congestion =>
#{enable_alarm => true, min_alarm_sustain_duration => 60000},
flapping_detect =>
#{ban_time => 300000,enable => false,
max_count => 15,window_time => 60000},
force_gc =>
#{bytes => 16777216,count => 16000,
enable => true},
force_shutdown =>
#{enable => true,
max_heap_size => 4194304,
max_message_queue_len => 1000},
mqtt => mqtt_conf(),
listeners =>
#{mqtt_tcp => listener_mqtt_tcp_conf(),
mqtt_ws => listener_mqtt_ws_conf()}
}
}
}.
set_default_zone_conf() ->
emqx_config:put(default_zone_conf()).
%%--------------------------------------------------------------------
%% CT Callbacks
@ -50,6 +195,9 @@ init_per_suite(Config) ->
ok = meck:new(emqx_metrics, [passthrough, no_history, no_link]),
ok = meck:expect(emqx_metrics, inc, fun(_) -> ok end),
ok = meck:expect(emqx_metrics, inc, fun(_, _) -> ok end),
%% Ban
meck:new(emqx_banned, [passthrough, no_history, no_link]),
ok = meck:expect(emqx_banned, check, fun(_ConnInfo) -> false end),
Config.
end_per_suite(_Config) ->
@ -58,15 +206,15 @@ end_per_suite(_Config) ->
emqx_session,
emqx_broker,
emqx_hooks,
emqx_cm
emqx_cm,
emqx_banned
]).
init_per_testcase(_TestCase, Config) ->
meck:new(emqx_zone, [passthrough, no_history, no_link]),
set_default_zone_conf(),
Config.
end_per_testcase(_TestCase, Config) ->
meck:unload([emqx_zone]),
Config.
%%--------------------------------------------------------------------
@ -83,7 +231,7 @@ t_chan_caps(_) ->
#{max_clientid_len := 65535,
max_qos_allowed := 2,
max_topic_alias := 65535,
max_topic_levels := 0,
max_topic_levels := 65535,
retain_available := true,
shared_subscription := true,
subscription_identifiers := true,
@ -250,7 +398,7 @@ t_bad_receive_maximum(_) ->
fun(true, _ClientInfo, _ConnInfo) ->
{ok, #{session => session(), present => false}}
end),
ok = meck:expect(emqx_zone, response_information, fun(_) -> test end),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, response_information], test),
C1 = channel(#{conn_state => idle}),
{shutdown, protocol_error, _, _} =
emqx_channel:handle_in(
@ -263,8 +411,8 @@ t_override_client_receive_maximum(_) ->
fun(true, _ClientInfo, _ConnInfo) ->
{ok, #{session => session(), present => false}}
end),
ok = meck:expect(emqx_zone, response_information, fun(_) -> test end),
ok = meck:expect(emqx_zone, max_inflight, fun(_) -> 0 end),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, response_information], test),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, max_inflight], 0),
C1 = channel(#{conn_state => idle}),
ClientCapacity = 2,
{ok, [{event, connected}, _ConnAck], C2} =
@ -486,7 +634,7 @@ t_handle_deliver_nl(_) ->
Channel = channel(#{clientinfo => ClientInfo, session => Session}),
Msg = emqx_message:make(<<"clientid">>, ?QOS_1, <<"t1">>, <<"qos1">>),
NMsg = emqx_message:set_flag(nl, Msg),
{ok, Channel} = emqx_channel:handle_deliver([{deliver, <<"t1">>, NMsg}], Channel).
{ok, _} = emqx_channel:handle_deliver([{deliver, <<"t1">>, NMsg}], Channel).
%%--------------------------------------------------------------------
%% Test cases for handle_out
@ -515,7 +663,7 @@ t_handle_out_connack_response_information(_) ->
fun(true, _ClientInfo, _ConnInfo) ->
{ok, #{session => session(), present => false}}
end),
ok = meck:expect(emqx_zone, response_information, fun(_) -> test end),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, response_information], test),
IdleChannel = channel(#{conn_state => idle}),
{ok, [{event, connected},
{connack, ?CONNACK_PACKET(?RC_SUCCESS, 0, #{'Response-Information' := test})}],
@ -529,7 +677,7 @@ t_handle_out_connack_not_response_information(_) ->
fun(true, _ClientInfo, _ConnInfo) ->
{ok, #{session => session(), present => false}}
end),
ok = meck:expect(emqx_zone, response_information, fun(_) -> test end),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, response_information], test),
IdleChannel = channel(#{conn_state => idle}),
{ok, [{event, connected}, {connack, ?CONNACK_PACKET(?RC_SUCCESS, 0, AckProps)}], _} =
emqx_channel:handle_in(
@ -669,9 +817,6 @@ t_enrich_conninfo(_) ->
t_enrich_client(_) ->
{ok, _ConnPkt, _Chan} = emqx_channel:enrich_client(connpkt(), channel()).
t_check_banned(_) ->
ok = emqx_channel:check_banned(connpkt(), channel()).
t_auth_connect(_) ->
{ok, _, _Chan} = emqx_channel:authenticate(?CONNECT_PACKET(connpkt()), channel()).
@ -718,7 +863,7 @@ t_packing_alias(_) ->
channel())).
t_check_pub_acl(_) ->
ok = meck:expect(emqx_zone, enable_acl, fun(_) -> true end),
emqx_config:put_listener_conf(default, mqtt_tcp, [acl, enable], true),
Publish = ?PUBLISH_PACKET(?QOS_0, <<"t">>, 1, <<"payload">>),
ok = emqx_channel:check_pub_acl(Publish, channel()).
@ -728,14 +873,14 @@ t_check_pub_alias(_) ->
ok = emqx_channel:check_pub_alias(#mqtt_packet{variable = Publish}, Channel).
t_check_sub_acls(_) ->
ok = meck:expect(emqx_zone, enable_acl, fun(_) -> true end),
emqx_config:put_listener_conf(default, mqtt_tcp, [acl, enable], true),
TopicFilter = {<<"t">>, ?DEFAULT_SUBOPTS},
[{TopicFilter, 0}] = emqx_channel:check_sub_acls([TopicFilter], channel()).
t_enrich_connack_caps(_) ->
ok = meck:new(emqx_mqtt_caps, [passthrough, no_history]),
ok = meck:expect(emqx_mqtt_caps, get_caps,
fun(_Zone) ->
fun(_Zone, _Listener) ->
#{max_packet_size => 1024,
max_qos_allowed => ?QOS_2,
retain_available => true,
@ -772,7 +917,7 @@ t_ws_cookie_init(_) ->
conn_mod => emqx_ws_connection,
ws_cookie => WsCookie
},
Channel = emqx_channel:init(ConnInfo, [{zone, zone}]),
Channel = emqx_channel:init(ConnInfo, #{zone => default, listener => mqtt_tcp}),
?assertMatch(#{ws_cookie := WsCookie}, emqx_channel:info(clientinfo, Channel)).
%%--------------------------------------------------------------------
@ -797,7 +942,7 @@ channel(InitFields) ->
maps:fold(fun(Field, Value, Channel) ->
emqx_channel:set_field(Field, Value, Channel)
end,
emqx_channel:init(ConnInfo, [{zone, zone}]),
emqx_channel:init(ConnInfo, #{zone => default, listener => mqtt_tcp}),
maps:merge(#{clientinfo => clientinfo(),
session => session(),
conn_state => connected
@ -805,7 +950,8 @@ channel(InitFields) ->
clientinfo() -> clientinfo(#{}).
clientinfo(InitProps) ->
maps:merge(#{zone => zone,
maps:merge(#{zone => default,
listener => mqtt_tcp,
protocol => mqtt,
peerhost => {127,0,0,1},
clientid => <<"clientid">>,
@ -837,7 +983,8 @@ session(InitFields) when is_map(InitFields) ->
maps:fold(fun(Field, Value, Session) ->
emqx_session:set_field(Field, Value, Session)
end,
emqx_session:init(#{zone => channel}, #{receive_maximum => 0}),
emqx_session:init(#{zone => default, listener => mqtt_tcp},
#{receive_maximum => 0}),
InitFields).
%% conn: 5/s; overall: 10/s

20
apps/emqx/test/emqx_client_SUITE.erl
View File

@ -78,17 +78,14 @@ groups() ->
init_per_suite(Config) ->
emqx_ct_helpers:boot_modules(all),
emqx_ct_helpers:start_apps([], fun set_special_confs/1),
emqx_ct_helpers:start_apps([]),
emqx_config:put_listener_conf(default, mqtt_ssl, [ssl, verify], verify_peer),
emqx_listeners:restart_listener('default:mqtt_ssl'),
Config.
end_per_suite(_Config) ->
emqx_ct_helpers:stop_apps([]).
set_special_confs(emqx) ->
emqx_ct_helpers:change_emqx_opts(ssl_twoway, [{peer_cert_as_username, cn}]);
set_special_confs(_) ->
ok.
%%--------------------------------------------------------------------
%% Test cases for MQTT v3
%%--------------------------------------------------------------------
@ -104,8 +101,7 @@ t_basic_v4(_Config) ->
t_basic([{proto_ver, v4}]).
t_cm(_) ->
IdleTimeout = emqx_zone:get_env(external, idle_timeout, 30000),
emqx_zone:set_env(external, idle_timeout, 1000),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, idle_timeout], 1000),
ClientId = <<"myclient">>,
{ok, C} = emqtt:start_link([{clientid, ClientId}]),
{ok, _} = emqtt:connect(C),
@ -115,7 +111,7 @@ t_cm(_) ->
ct:sleep(1200),
Stats = emqx_cm:get_chan_stats(ClientId),
?assertEqual(1, proplists:get_value(subscriptions_cnt, Stats)),
emqx_zone:set_env(external, idle_timeout, IdleTimeout).
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, idle_timeout], 15000).
t_cm_registry(_) ->
Info = supervisor:which_children(emqx_cm_sup),
@ -273,15 +269,13 @@ t_basic(_Opts) ->
ok = emqtt:disconnect(C).
t_username_as_clientid(_) ->
emqx_zone:set_env(external, use_username_as_clientid, true),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, use_username_as_clientid], true),
Username = <<"usera">>,
{ok, C} = emqtt:start_link([{username, Username}]),
{ok, _} = emqtt:connect(C),
#{clientinfo := #{clientid := Username}} = emqx_cm:get_chan_info(Username),
emqtt:disconnect(C).
t_certcn_as_clientid_default_config_tls(_) ->
tls_certcn_as_clientid(default).
@ -329,7 +323,7 @@ tls_certcn_as_clientid(TLSVsn) ->
tls_certcn_as_clientid(TLSVsn, RequiredTLSVsn) ->
CN = <<"Client">>,
emqx_zone:set_env(external, use_username_as_clientid, true),
emqx_config:put_listener_conf(default, mqtt_ssl, [mqtt, peer_cert_as_clientid], cn),
SslConf = emqx_ct_helpers:client_ssl_twoway(TLSVsn),
{ok, Client} = emqtt:start_link([{port, 8883}, {ssl, true}, {ssl_opts, SslConf}]),
{ok, _} = emqtt:connect(Client),

4
apps/emqx/test/emqx_cm_SUITE.erl
View File

@ -89,7 +89,7 @@ t_open_session(_) ->
ok = meck:expect(emqx_connection, call, fun(_, _) -> ok end),
ok = meck:expect(emqx_connection, call, fun(_, _, _) -> ok end),
ClientInfo = #{zone => external,
ClientInfo = #{zone => default, listener => mqtt_tcp,
clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127,0,0,1}},
@ -114,7 +114,7 @@ rand_client_id() ->
t_open_session_race_condition(_) ->
ClientId = rand_client_id(),
ClientInfo = #{zone => external,
ClientInfo = #{zone => default, listener => mqtt_tcp,
clientid => ClientId,
username => <<"username">>,
peerhost => {127,0,0,1}},

45
apps/emqx/test/emqx_connection_SUITE.erl
View File

@ -57,6 +57,7 @@ init_per_suite(Config) ->
ok = meck:expect(emqx_alarm, deactivate, fun(_) -> ok end),
ok = meck:expect(emqx_alarm, deactivate, fun(_, _) -> ok end),
emqx_channel_SUITE:set_default_zone_conf(),
Config.
end_per_suite(_Config) ->
@ -120,14 +121,13 @@ t_info(_) ->
end
end),
#{sockinfo := SockInfo} = emqx_connection:info(CPid),
?assertMatch(#{active_n := 100,
peername := {{127,0,0,1},3456},
?assertMatch(#{ peername := {{127,0,0,1},3456},
sockname := {{127,0,0,1},1883},
sockstate := idle,
socktype := tcp}, SockInfo).
t_info_limiter(_) ->
St = st(#{limiter => emqx_limiter:init(external, [])}),
St = st(#{limiter => emqx_limiter:init(default, [])}),
?assertEqual(undefined, emqx_connection:info(limiter, St)).
t_stats(_) ->
@ -219,8 +219,10 @@ t_handle_msg_deliver(_) ->
t_handle_msg_inet_reply(_) ->
ok = meck:expect(emqx_pd, get_counter, fun(_) -> 10 end),
?assertMatch({ok, _St}, handle_msg({inet_reply, for_testing, ok}, st(#{active_n => 0}))),
?assertEqual(ok, handle_msg({inet_reply, for_testing, ok}, st(#{active_n => 100}))),
emqx_config:put_listener_conf(default, mqtt_tcp, [tcp, active_n], 0),
?assertMatch({ok, _St}, handle_msg({inet_reply, for_testing, ok}, st())),
emqx_config:put_listener_conf(default, mqtt_tcp, [tcp, active_n], 100),
?assertEqual(ok, handle_msg({inet_reply, for_testing, ok}, st())),
?assertMatch({stop, {shutdown, for_testing}, _St},
handle_msg({inet_reply, for_testing, {error, for_testing}}, st())).
@ -331,12 +333,12 @@ t_ensure_rate_limit(_) ->
?assertEqual(undefined, emqx_connection:info(limiter, State)),
ok = meck:expect(emqx_limiter, check,
fun(_, _) -> {ok, emqx_limiter:init(external, [])} end),
fun(_, _) -> {ok, emqx_limiter:init(default, [])} end),
State1 = emqx_connection:ensure_rate_limit(#{}, st(#{limiter => #{}})),
?assertEqual(undefined, emqx_connection:info(limiter, State1)),
ok = meck:expect(emqx_limiter, check,
fun(_, _) -> {pause, 3000, emqx_limiter:init(external, [])} end),
fun(_, _) -> {pause, 3000, emqx_limiter:init(default, [])} end),
State2 = emqx_connection:ensure_rate_limit(#{}, st(#{limiter => #{}})),
?assertEqual(undefined, emqx_connection:info(limiter, State2)),
?assertEqual(blocked, emqx_connection:info(sockstate, State2)).
@ -386,8 +388,7 @@ t_start_link_exit_on_activate(_) ->
t_get_conn_info(_) ->
with_conn(fun(CPid) ->
#{sockinfo := SockInfo} = emqx_connection:info(CPid),
?assertEqual(#{active_n => 100,
peername => {{127,0,0,1},3456},
?assertEqual(#{peername => {{127,0,0,1},3456},
sockname => {{127,0,0,1},1883},
sockstate => running,
socktype => tcp
@ -397,16 +398,12 @@ t_get_conn_info(_) ->
t_oom_shutdown(init, Config) ->
ok = snabbkaffe:start_trace(),
ok = meck:new(emqx_misc, [non_strict, passthrough, no_history, no_link]),
ok = meck:new(emqx_zone, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_zone, oom_policy,
fun(_Zone) -> #{message_queue_len => 10, max_heap_size => 8000000} end),
meck:expect(emqx_misc, check_oom,
fun(_) -> {shutdown, "fake_oom"} end),
Config;
t_oom_shutdown('end', _Config) ->
snabbkaffe:stop(),
meck:unload(emqx_misc),
meck:unload(emqx_zone),
ok.
t_oom_shutdown(_) ->
@ -455,13 +452,11 @@ exit_on_activate_error(SockErr, Reason) ->
with_conn(TestFun) ->
with_conn(TestFun, #{trap_exit => false}).
with_conn(TestFun, Options) when is_map(Options) ->
with_conn(TestFun, maps:to_list(Options));
with_conn(TestFun, Options) ->
TrapExit = proplists:get_value(trap_exit, Options, false),
with_conn(TestFun, Opts) when is_map(Opts) ->
TrapExit = maps:get(trap_exit, Opts, false),
process_flag(trap_exit, TrapExit),
{ok, CPid} = emqx_connection:start_link(emqx_transport, sock, Options),
{ok, CPid} = emqx_connection:start_link(emqx_transport, sock,
maps:merge(Opts, #{zone => default, listener => mqtt_tcp})),
TestFun(CPid),
TrapExit orelse emqx_connection:stop(CPid),
ok.
@ -483,7 +478,8 @@ st() -> st(#{}, #{}).
st(InitFields) when is_map(InitFields) ->
st(InitFields, #{}).
st(InitFields, ChannelFields) when is_map(InitFields) ->
St = emqx_connection:init_state(emqx_transport, sock, [#{zone => external}]),
St = emqx_connection:init_state(emqx_transport, sock, #{zone => default,
listener => mqtt_tcp}),
maps:fold(fun(N, V, S) -> emqx_connection:set_field(N, V, S) end,
emqx_connection:set_field(channel, channel(ChannelFields), St),
InitFields
@ -503,7 +499,8 @@ channel(InitFields) ->
receive_maximum => 100,
expiry_interval => 0
},
ClientInfo = #{zone => zone,
ClientInfo = #{zone => default,
listener => mqtt_tcp,
protocol => mqtt,
peerhost => {127,0,0,1},
clientid => <<"clientid">>,
@ -512,13 +509,13 @@ channel(InitFields) ->
peercert => undefined,
mountpoint => undefined
},
Session = emqx_session:init(#{zone => external},
Session = emqx_session:init(#{zone => default, listener => mqtt_tcp},
#{receive_maximum => 0}
),
maps:fold(fun(Field, Value, Channel) ->
emqx_channel:set_field(Field, Value, Channel)
emqx_channel:set_field(Field, Value, Channel)
end,
emqx_channel:init(ConnInfo, [{zone, zone}]),
emqx_channel:init(ConnInfo, #{zone => default, listener => mqtt_tcp}),
maps:merge(#{clientinfo => ClientInfo,
session => Session,
conn_state => connected

32
apps/emqx/test/emqx_flapping_SUITE.erl
View File

@ -25,26 +25,23 @@ all() -> emqx_ct:all(?MODULE).
init_per_suite(Config) ->
emqx_ct_helpers:boot_modules(all),
emqx_ct_helpers:start_apps([], fun set_special_configs/1),
emqx_ct_helpers:start_apps([]),
emqx_config:put_listener_conf(default, mqtt_tcp, [flapping_detect],
#{max_count => 3,
window_time => 100, % 0.1s
ban_time => 2000 %% 2s
}),
Config.
set_special_configs(emqx) ->
emqx_zone:set_env(external, enable_flapping_detect, true),
application:set_env(emqx, flapping_detect_policy,
#{threshold => 3,
duration => 100,
banned_interval => 2
});
set_special_configs(_App) -> ok.
end_per_suite(_Config) ->
emqx_ct_helpers:stop_apps([]),
ekka_mnesia:delete_schema(), %% Clean emqx_banned table
ok.
t_detect_check(_) ->
ClientInfo = #{zone => external,
clientid => <<"clientid">>,
ClientInfo = #{zone => default,
listener => mqtt_tcp,
clientid => <<"client007">>,
peerhost => {127,0,0,1}
},
false = emqx_flapping:detect(ClientInfo),
@ -53,6 +50,8 @@ t_detect_check(_) ->
false = emqx_banned:check(ClientInfo),
true = emqx_flapping:detect(ClientInfo),
timer:sleep(50),
ct:pal("the table emqx_banned: ~p, nowsec: ~p", [ets:tab2list(emqx_banned),
erlang:system_time(second)]),
true = emqx_banned:check(ClientInfo),
timer:sleep(3000),
false = emqx_banned:check(ClientInfo),
@ -64,12 +63,13 @@ t_detect_check(_) ->
ok = emqx_flapping:stop().
t_expired_detecting(_) ->
ClientInfo = #{zone => external,
clientid => <<"clientid">>,
ClientInfo = #{zone => default,
listener => mqtt_tcp,
clientid => <<"client008">>,
peerhost => {127,0,0,1}},
false = emqx_flapping:detect(ClientInfo),
?assertEqual(true, lists:any(fun({flapping, <<"clientid">>, _, _, _}) -> true;
?assertEqual(true, lists:any(fun({flapping, <<"client008">>, _, _, _}) -> true;
(_) -> false end, ets:tab2list(emqx_flapping))),
timer:sleep(200),
?assertEqual(true, lists:all(fun({flapping, <<"clientid">>, _, _, _}) -> false;
?assertEqual(true, lists:all(fun({flapping, <<"client008">>, _, _, _}) -> false;
(_) -> true end, ets:tab2list(emqx_flapping))).

5
apps/emqx/test/emqx_misc_SUITE.erl
View File

@ -119,8 +119,9 @@ t_index_of(_) ->
?assertEqual(3, emqx_misc:index_of(a, [b, c, a, e, f])).
t_check(_) ->
Policy = #{message_queue_len => 10,
max_heap_size => 1024 * 1024 * 8},
Policy = #{max_message_queue_len => 10,
max_heap_size => 1024 * 1024 * 8,
enable => true},
[self() ! {msg, I} || I <- lists:seq(1, 5)],
?assertEqual(ok, emqx_misc:check_oom(Policy)),
[self() ! {msg, I} || I <- lists:seq(1, 6)],

9
apps/emqx/test/emqx_mqtt_SUITE.erl
View File

@ -156,6 +156,15 @@ t_async_set_keepalive('end', _Config) ->
ok.
t_async_set_keepalive(_) ->
case os:type() of
{unix, darwin} ->
%% Mac OSX don't support the feature
ok;
_ ->
do_async_set_keepalive()
end.
do_async_set_keepalive() ->
ClientID = <<"client-tcp-keepalive">>,
{ok, Client} = emqtt:start_link([{host, "localhost"},
{proto_ver,v5},

37
apps/emqx/test/emqx_mqtt_caps_SUITE.erl
View File

@ -25,39 +25,36 @@
all() -> emqx_ct:all(?MODULE).
t_check_pub(_) ->
PubCaps = #{max_qos_allowed => ?QOS_1,
retain_available => false
},
emqx_zone:set_env(zone, '$mqtt_pub_caps', PubCaps),
OldConf = emqx_config:get(),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, max_qos_allowed], ?QOS_1),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, retain_available], false),
timer:sleep(50),
ok = emqx_mqtt_caps:check_pub(zone, #{qos => ?QOS_1,
retain => false}),
ok = emqx_mqtt_caps:check_pub(default, mqtt_tcp, #{qos => ?QOS_1, retain => false}),
PubFlags1 = #{qos => ?QOS_2, retain => false},
?assertEqual({error, ?RC_QOS_NOT_SUPPORTED},
emqx_mqtt_caps:check_pub(zone, PubFlags1)),
emqx_mqtt_caps:check_pub(default, mqtt_tcp, PubFlags1)),
PubFlags2 = #{qos => ?QOS_1, retain => true},
?assertEqual({error, ?RC_RETAIN_NOT_SUPPORTED},
emqx_mqtt_caps:check_pub(zone, PubFlags2)),
emqx_zone:unset_env(zone, '$mqtt_pub_caps').
emqx_mqtt_caps:check_pub(default, mqtt_tcp, PubFlags2)),
emqx_config:put(OldConf).
t_check_sub(_) ->
OldConf = emqx_config:get(),
SubOpts = #{rh => 0,
rap => 0,
nl => 0,
qos => ?QOS_2
},
SubCaps = #{max_topic_levels => 2,
max_qos_allowed => ?QOS_2,
shared_subscription => false,
wildcard_subscription => false
},
emqx_zone:set_env(zone, '$mqtt_sub_caps', SubCaps),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, max_topic_levels], 2),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, max_qos_allowed], ?QOS_1),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, shared_subscription], false),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, wildcard_subscription], false),
timer:sleep(50),
ok = emqx_mqtt_caps:check_sub(zone, <<"topic">>, SubOpts),
ok = emqx_mqtt_caps:check_sub(default, mqtt_tcp, <<"topic">>, SubOpts),
?assertEqual({error, ?RC_TOPIC_FILTER_INVALID},
emqx_mqtt_caps:check_sub(zone, <<"a/b/c/d">>, SubOpts)),
emqx_mqtt_caps:check_sub(default, mqtt_tcp, <<"a/b/c/d">>, SubOpts)),
?assertEqual({error, ?RC_WILDCARD_SUBSCRIPTIONS_NOT_SUPPORTED},
emqx_mqtt_caps:check_sub(zone, <<"+/#">>, SubOpts)),
emqx_mqtt_caps:check_sub(default, mqtt_tcp, <<"+/#">>, SubOpts)),
?assertEqual({error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED},
emqx_mqtt_caps:check_sub(zone, <<"topic">>, SubOpts#{share => true})),
emqx_zone:unset_env(zone, '$mqtt_pub_caps').
emqx_mqtt_caps:check_sub(default, mqtt_tcp, <<"topic">>, SubOpts#{share => true})),
emqx_config:put(OldConf).

68
apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
View File

@ -217,10 +217,14 @@ t_connect_will_message(Config) ->
ok = emqtt:disconnect(Client4).
t_batch_subscribe(init, Config) ->
emqx_config:put_listener_conf(default, mqtt_tcp, [acl, enable], true),
emqx_config:put_listener_conf(default, mqtt_quic, [acl, enable], true),
ok = meck:new(emqx_access_control, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_access_control, authorize, fun(_, _, _) -> deny end),
Config;
t_batch_subscribe('end', _Config) ->
emqx_config:put_listener_conf(default, mqtt_tcp, [acl, enable], false),
emqx_config:put_listener_conf(default, mqtt_quic, [acl, enable], false),
meck:unload(emqx_access_control).
t_batch_subscribe(Config) ->
@ -284,52 +288,22 @@ t_connect_will_retain(Config) ->
t_connect_idle_timeout(_Config) ->
IdleTimeout = 2000,
emqx_zone:set_env(external, idle_timeout, IdleTimeout),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, idle_timeout], IdleTimeout),
emqx_config:put_listener_conf(default, mqtt_quic, [mqtt, idle_timeout], IdleTimeout),
{ok, Sock} = emqtt_sock:connect({127,0,0,1}, 1883, [], 60000),
timer:sleep(IdleTimeout),
?assertMatch({error, closed}, emqtt_sock:recv(Sock,1024)).
t_connect_limit_timeout(init, Config) ->
ok = meck:new(proplists, [non_strict, passthrough, no_history, no_link, unstick]),
meck:expect(proplists, get_value, fun(active_n, _Options, _Default) -> 1;
(Arg1, ARg2, Arg3) -> meck:passthrough([Arg1, ARg2, Arg3])
end),
Config;
t_connect_limit_timeout('end', _Config) ->
catch meck:unload(proplists).
t_connect_limit_timeout(Config) ->
ConnFun = ?config(conn_fun, Config),
Topic = nth(1, ?TOPICS),
emqx_zone:set_env(external, publish_limit, {3, 5}),
{ok, Client} = emqtt:start_link([{proto_ver, v5},{keepalive, 60} | Config]),
{ok, _} = emqtt:ConnFun(Client),
[ClientPid] = emqx_cm:lookup_channels(client_info(clientid, Client)),
?assertEqual(undefined, emqx_connection:info(limit_timer, sys:get_state(ClientPid))),
Payload = <<"t_shared_subscriptions_client_terminates_when_qos_eq_2">>,
{ok, 2} = emqtt:publish(Client, Topic, Payload, 1),
{ok, 3} = emqtt:publish(Client, Topic, Payload, 1),
{ok, 4} = emqtt:publish(Client, Topic, Payload, 1),
timer:sleep(250),
?assert(is_reference(emqx_connection:info(limit_timer, sys:get_state(ClientPid)))),
ok = emqtt:disconnect(Client),
emqx_zone:set_env(external, publish_limit, undefined),
meck:unload(proplists).
t_connect_emit_stats_timeout(init, Config) ->
NewIdleTimeout = 1000,
OldIdleTimeout = emqx_zone:get_env(external, idle_timeout),
emqx_zone:set_env(external, idle_timeout, NewIdleTimeout),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, idle_timeout], NewIdleTimeout),
emqx_config:put_listener_conf(default, mqtt_quic, [mqtt, idle_timeout], NewIdleTimeout),
ok = snabbkaffe:start_trace(),
[{idle_timeout, NewIdleTimeout}, {old_idle_timeout, OldIdleTimeout} | Config];
t_connect_emit_stats_timeout('end', Config) ->
[{idle_timeout, NewIdleTimeout} | Config];
t_connect_emit_stats_timeout('end', _Config) ->
snabbkaffe:stop(),
{_, OldIdleTimeout} = lists:keyfind(old_idle_timeout, 1, Config),
emqx_zone:set_env(external, idle_timeout, OldIdleTimeout),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, idle_timeout], 15000),
emqx_config:put_listener_conf(default, mqtt_quic, [mqtt, idle_timeout], 15000),
ok.
t_connect_emit_stats_timeout(Config) ->
@ -497,7 +471,8 @@ t_connack_session_present(Config) ->
t_connack_max_qos_allowed(init, Config) ->
Config;
t_connack_max_qos_allowed('end', _Config) ->
emqx_zone:set_env(external, max_qos_allowed, 2),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, max_qos_allowed], 2),
emqx_config:put_listener_conf(default, mqtt_quic, [mqtt, max_qos_allowed], 2),
ok.
t_connack_max_qos_allowed(Config) ->
ConnFun = ?config(conn_fun, Config),
@ -505,9 +480,8 @@ t_connack_max_qos_allowed(Config) ->
Topic = nth(1, ?TOPICS),
%% max_qos_allowed = 0
emqx_zone:set_env(external, max_qos_allowed, 0),
persistent_term:erase({emqx_zone, external, '$mqtt_caps'}),
persistent_term:erase({emqx_zone, external, '$mqtt_pub_caps'}),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, max_qos_allowed], 0),
emqx_config:put_listener_conf(default, mqtt_quic, [mqtt, max_qos_allowed], 0),
{ok, Client1} = emqtt:start_link([{proto_ver, v5} | Config]),
{ok, Connack1} = emqtt:ConnFun(Client1),
@ -532,9 +506,8 @@ t_connack_max_qos_allowed(Config) ->
waiting_client_process_exit(Client2),
%% max_qos_allowed = 1
emqx_zone:set_env(external, max_qos_allowed, 1),
persistent_term:erase({emqx_zone, external, '$mqtt_caps'}),
persistent_term:erase({emqx_zone, external, '$mqtt_pub_caps'}),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, max_qos_allowed], 1),
emqx_config:put_listener_conf(default, mqtt_quic, [mqtt, max_qos_allowed], 1),
{ok, Client3} = emqtt:start_link([{proto_ver, v5} | Config]),
{ok, Connack3} = emqtt:ConnFun(Client3),
@ -559,9 +532,8 @@ t_connack_max_qos_allowed(Config) ->
waiting_client_process_exit(Client4),
%% max_qos_allowed = 2
emqx_zone:set_env(external, max_qos_allowed, 2),
persistent_term:erase({emqx_zone, external, '$mqtt_caps'}),
persistent_term:erase({emqx_zone, external, '$mqtt_pub_caps'}),
emqx_config:put_listener_conf(default, mqtt_tcp, [mqtt, max_qos_allowed], 2),
emqx_config:put_listener_conf(default, mqtt_quic, [mqtt, max_qos_allowed], 2),
{ok, Client5} = emqtt:start_link([{proto_ver, v5} | Config]),
{ok, Connack5} = emqtt:ConnFun(Client5),

50
apps/emqx/test/emqx_os_mon_SUITE.erl
View File

@ -24,46 +24,34 @@
all() -> emqx_ct:all(?MODULE).
init_per_suite(Config) ->
emqx_config:put([sysmon, os], #{
cpu_check_interval => 60,cpu_high_watermark => 0.8,
cpu_low_watermark => 0.6,mem_check_interval => 60,
procmem_high_watermark => 0.05,sysmem_high_watermark => 0.7}),
application:ensure_all_started(os_mon),
Config.
end_per_suite(_Config) ->
application:stop(os_mon).
% t_set_mem_check_interval(_) ->
% error('TODO').
% t_set_sysmem_high_watermark(_) ->
% error('TODO').
% t_set_procmem_high_watermark(_) ->
% error('TODO').
t_api(_) ->
gen_event:swap_handler(alarm_handler, {emqx_alarm_handler, swap}, {alarm_handler, []}),
{ok, _} = emqx_os_mon:start_link([{cpu_check_interval, 1},
{cpu_high_watermark, 5},
{cpu_low_watermark, 80},
{mem_check_interval, 60},
{sysmem_high_watermark, 70},
{procmem_high_watermark, 5}]),
?assertEqual(1, emqx_os_mon:get_cpu_check_interval()),
?assertEqual(5, emqx_os_mon:get_cpu_high_watermark()),
?assertEqual(80, emqx_os_mon:get_cpu_low_watermark()),
?assertEqual(60, emqx_os_mon:get_mem_check_interval()),
?assertEqual(70, emqx_os_mon:get_sysmem_high_watermark()),
?assertEqual(5, emqx_os_mon:get_procmem_high_watermark()),
% timer:sleep(2000),
% ?assertEqual(true, lists:keymember(cpu_high_watermark, 1, alarm_handler:get_alarms())),
{ok, _} = emqx_os_mon:start_link(),
?assertEqual(60, emqx_os_mon:get_mem_check_interval()),
?assertEqual(ok, emqx_os_mon:set_mem_check_interval(30)),
?assertEqual(60, emqx_os_mon:get_mem_check_interval()),
?assertEqual(ok, emqx_os_mon:set_mem_check_interval(122)),
?assertEqual(120, emqx_os_mon:get_mem_check_interval()),
?assertEqual(70, emqx_os_mon:get_sysmem_high_watermark()),
?assertEqual(ok, emqx_os_mon:set_sysmem_high_watermark(0.8)),
?assertEqual(80, emqx_os_mon:get_sysmem_high_watermark()),
?assertEqual(5, emqx_os_mon:get_procmem_high_watermark()),
?assertEqual(ok, emqx_os_mon:set_procmem_high_watermark(0.11)),
?assertEqual(11, emqx_os_mon:get_procmem_high_watermark()),
emqx_os_mon:set_cpu_check_interval(0.05),
emqx_os_mon:set_cpu_high_watermark(80),
emqx_os_mon:set_cpu_low_watermark(75),
?assertEqual(0.05, emqx_os_mon:get_cpu_check_interval()),
?assertEqual(80, emqx_os_mon:get_cpu_high_watermark()),
?assertEqual(75, emqx_os_mon:get_cpu_low_watermark()),
% timer:sleep(3000),
% ?assertEqual(false, lists:keymember(cpu_high_watermark, 1, alarm_handler:get_alarms())),
?assertEqual(ignored, gen_server:call(emqx_os_mon, ignored)),
?assertEqual(ok, gen_server:cast(emqx_os_mon, ignored)),
emqx_os_mon ! ignored,

19
apps/emqx/test/emqx_session_SUITE.erl
View File

@ -29,6 +29,7 @@ all() -> emqx_ct:all(?MODULE).
%%--------------------------------------------------------------------
init_per_suite(Config) ->
emqx_channel_SUITE:set_default_zone_conf(),
ok = meck:new([emqx_hooks, emqx_metrics, emqx_broker],
[passthrough, no_history, no_link]),
ok = meck:expect(emqx_metrics, inc, fun(_) -> ok end),
@ -50,15 +51,16 @@ end_per_testcase(_TestCase, Config) ->
%%--------------------------------------------------------------------
t_session_init(_) ->
Session = emqx_session:init(#{zone => zone}, #{receive_maximum => 64}),
Session = emqx_session:init(#{zone => default, listener => mqtt_tcp},
#{receive_maximum => 64}),
?assertEqual(#{}, emqx_session:info(subscriptions, Session)),
?assertEqual(0, emqx_session:info(subscriptions_cnt, Session)),
?assertEqual(0, emqx_session:info(subscriptions_max, Session)),
?assertEqual(infinity, emqx_session:info(subscriptions_max, Session)),
?assertEqual(false, emqx_session:info(upgrade_qos, Session)),
?assertEqual(0, emqx_session:info(inflight_cnt, Session)),
?assertEqual(64, emqx_session:info(inflight_max, Session)),
?assertEqual(1, emqx_session:info(next_pkt_id, Session)),
?assertEqual(0, emqx_session:info(retry_interval, Session)),
?assertEqual(30, emqx_session:info(retry_interval, Session)),
?assertEqual(0, emqx_mqueue:len(emqx_session:info(mqueue, Session))),
?assertEqual(0, emqx_session:info(awaiting_rel_cnt, Session)),
?assertEqual(100, emqx_session:info(awaiting_rel_max, Session)),
@ -72,13 +74,13 @@ t_session_init(_) ->
t_session_info(_) ->
?assertMatch(#{subscriptions := #{},
upgrade_qos := false,
retry_interval := 0,
retry_interval := 30,
await_rel_timeout := 300
}, emqx_session:info(session())).
t_session_stats(_) ->
Stats = emqx_session:stats(session()),
?assertMatch(#{subscriptions_max := 0,
?assertMatch(#{subscriptions_max := infinity,
inflight_max := 0,
mqueue_len := 0,
mqueue_max := 1000,
@ -99,7 +101,7 @@ t_subscribe(_) ->
?assertEqual(1, emqx_session:info(subscriptions_cnt, Session)).
t_is_subscriptions_full_false(_) ->
Session = session(#{max_subscriptions => 0}),
Session = session(#{max_subscriptions => infinity}),
?assertNot(emqx_session:is_subscriptions_full(Session)).
t_is_subscriptions_full_true(_) ->
@ -152,7 +154,7 @@ t_publish_qos2_with_error_return(_) ->
{error, ?RC_RECEIVE_MAXIMUM_EXCEEDED} = emqx_session:publish(3, Msg, Session1).
t_is_awaiting_full_false(_) ->
Session = session(#{max_awaiting_rel => 0}),
Session = session(#{max_awaiting_rel => infinity}),
?assertNot(emqx_session:is_awaiting_full(Session)).
t_is_awaiting_full_true(_) ->
@ -375,7 +377,8 @@ session(InitFields) when is_map(InitFields) ->
maps:fold(fun(Field, Value, Session) ->
emqx_session:set_field(Field, Value, Session)
end,
emqx_session:init(#{zone => channel}, #{receive_maximum => 0}),
emqx_session:init(#{zone => default, listener => mqtt_tcp},
#{receive_maximum => 0}),
InitFields).

35
apps/emqx/test/emqx_vm_mon_SUITE.erl
View File

@ -23,17 +23,16 @@
all() -> emqx_ct:all(?MODULE).
init_per_testcase(t_api, Config) ->
init_per_testcase(t_alarms, Config) ->
emqx_ct_helpers:boot_modules(all),
emqx_ct_helpers:start_apps([],
fun(emqx) ->
application:set_env(emqx, vm_mon, [{check_interval, 1},
{process_high_watermark, 80},
{process_low_watermark, 75}]),
ok;
(_) ->
ok
end),
emqx_ct_helpers:start_apps([]),
emqx_config:put([sysmon, vm], #{
process_high_watermark => 0,
process_low_watermark => 0,
process_check_interval => 100 %% 1s
}),
ok = supervisor:terminate_child(emqx_sys_sup, emqx_vm_mon),
{ok, _} = supervisor:restart_child(emqx_sys_sup, emqx_vm_mon),
Config;
init_per_testcase(_, Config) ->
emqx_ct_helpers:boot_modules(all),
@ -43,18 +42,12 @@ init_per_testcase(_, Config) ->
end_per_testcase(_, _Config) ->
emqx_ct_helpers:stop_apps([]).
t_api(_) ->
?assertEqual(1, emqx_vm_mon:get_check_interval()),
?assertEqual(80, emqx_vm_mon:get_process_high_watermark()),
?assertEqual(75, emqx_vm_mon:get_process_low_watermark()),
emqx_vm_mon:set_process_high_watermark(0),
emqx_vm_mon:set_process_low_watermark(60),
?assertEqual(0, emqx_vm_mon:get_process_high_watermark()),
?assertEqual(60, emqx_vm_mon:get_process_low_watermark()),
timer:sleep(emqx_vm_mon:get_check_interval() * 1000 * 2),
t_alarms(_) ->
timer:sleep(500),
?assert(is_existing(too_many_processes, emqx_alarm:get_alarms(activated))),
emqx_vm_mon:set_process_high_watermark(70),
timer:sleep(emqx_vm_mon:get_check_interval() * 1000 * 2),
emqx_config:put([sysmon, vm, process_high_watermark], 70),
emqx_config:put([sysmon, vm, process_low_watermark], 60),
timer:sleep(500),
?assertNot(is_existing(too_many_processes, emqx_alarm:get_alarms(activated))).
is_existing(Name, [#{name := Name} | _More]) ->

96
apps/emqx/test/emqx_ws_connection_SUITE.erl
View File

@ -48,6 +48,7 @@ init_per_testcase(TestCase, Config) when
TestCase =/= t_ws_pingreq_before_connected,
TestCase =/= t_ws_non_check_origin
->
emqx_channel_SUITE:set_default_zone_conf(),
%% Mock cowboy_req
ok = meck:new(cowboy_req, [passthrough, no_history, no_link]),
ok = meck:expect(cowboy_req, header, fun(_, _, _) -> <<>> end),
@ -55,13 +56,6 @@ init_per_testcase(TestCase, Config) when
ok = meck:expect(cowboy_req, sock, fun(_) -> {{127,0,0,1}, 18083} end),
ok = meck:expect(cowboy_req, cert, fun(_) -> undefined end),
ok = meck:expect(cowboy_req, parse_cookies, fun(_) -> error(badarg) end),
%% Mock emqx_zone
ok = meck:new(emqx_zone, [passthrough, no_history, no_link]),
ok = meck:expect(emqx_zone, oom_policy,
fun(_) -> #{max_heap_size => 838860800,
message_queue_len => 8000
}
end),
%% Mock emqx_access_control
ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]),
ok = meck:expect(emqx_access_control, authorize, fun(_, _, _) -> allow end),
@ -85,6 +79,7 @@ init_per_testcase(TestCase, Config) when
Config;
init_per_testcase(_, Config) ->
ok = emqx_ct_helpers:start_apps([]),
Config.
end_per_testcase(TestCase, _Config) when
@ -96,7 +91,6 @@ end_per_testcase(TestCase, _Config) when
->
lists:foreach(fun meck:unload/1,
[cowboy_req,
emqx_zone,
emqx_access_control,
emqx_broker,
emqx_hooks,
@ -104,6 +98,7 @@ end_per_testcase(TestCase, _Config) when
]);
end_per_testcase(_, Config) ->
emqx_ct_helpers:stop_apps([]),
Config.
%%--------------------------------------------------------------------
@ -118,18 +113,21 @@ t_info(_) ->
end),
#{sockinfo := SockInfo} = ?ws_conn:call(WsPid, info),
#{socktype := ws,
active_n := 100,
peername := {{127,0,0,1}, 3456},
sockname := {{127,0,0,1}, 18083},
sockstate := running
} = SockInfo.
set_ws_opts(Key, Val) ->
emqx_config:put_listener_conf(default, mqtt_ws, [websocket, Key], Val).
t_header(_) ->
ok = meck:expect(cowboy_req, header, fun(<<"x-forwarded-for">>, _, _) -> <<"100.100.100.100, 99.99.99.99">>;
(<<"x-forwarded-port">>, _, _) -> <<"1000">> end),
{ok, St, _} = ?ws_conn:websocket_init([req, [{zone, external},
{proxy_address_header, <<"x-forwarded-for">>},
{proxy_port_header, <<"x-forwarded-port">>}]]),
ok = meck:expect(cowboy_req, header,
fun(<<"x-forwarded-for">>, _, _) -> <<"100.100.100.100, 99.99.99.99">>;
(<<"x-forwarded-port">>, _, _) -> <<"1000">> end),
set_ws_opts(proxy_address_header, <<"x-forwarded-for">>),
set_ws_opts(proxy_port_header, <<"x-forwarded-port">>),
{ok, St, _} = ?ws_conn:websocket_init([req, #{zone => default, listener => mqtt_ws}]),
WsPid = spawn(fun() ->
receive {call, From, info} ->
gen_server:reply(From, ?ws_conn:info(St))
@ -175,12 +173,10 @@ t_call(_) ->
?assertEqual(Info, ?ws_conn:call(WsPid, info)).
t_ws_pingreq_before_connected(_) ->
ok = emqx_ct_helpers:start_apps([]),
{ok, _} = application:ensure_all_started(gun),
{ok, WPID} = gun:open("127.0.0.1", 8083),
ws_pingreq(#{}),
gun:close(WPID),
emqx_ct_helpers:stop_apps([]).
gun:close(WPID).
ws_pingreq(State) ->
receive
@ -209,14 +205,11 @@ ws_pingreq(State) ->
end.
t_ws_sub_protocols_mqtt(_) ->
ok = emqx_ct_helpers:start_apps([]),
{ok, _} = application:ensure_all_started(gun),
?assertMatch({gun_upgrade, _},
start_ws_client(#{protocols => [<<"mqtt">>]})),
emqx_ct_helpers:stop_apps([]).
start_ws_client(#{protocols => [<<"mqtt">>]})).
t_ws_sub_protocols_mqtt_equivalents(_) ->
ok = emqx_ct_helpers:start_apps([]),
{ok, _} = application:ensure_all_started(gun),
%% also support mqtt-v3, mqtt-v3.1.1, mqtt-v5
?assertMatch({gun_upgrade, _},
@ -226,58 +219,39 @@ t_ws_sub_protocols_mqtt_equivalents(_) ->
?assertMatch({gun_upgrade, _},
start_ws_client(#{protocols => [<<"mqtt-v5">>]})),
?assertMatch({gun_response, {_, 400, _}},
start_ws_client(#{protocols => [<<"not-mqtt">>]})),
emqx_ct_helpers:stop_apps([]).
start_ws_client(#{protocols => [<<"not-mqtt">>]})).
t_ws_check_origin(_) ->
emqx_ct_helpers:start_apps([],
fun(emqx) ->
{ok, Listeners} = application:get_env(emqx, listeners),
NListeners = lists:map(fun(#{listen_on := 8083, opts := Opts} = Listener) ->
NOpts = proplists:delete(check_origin_enable, Opts),
Listener#{opts => [{check_origin_enable, true} | NOpts]};
(Listener) ->
Listener
end, Listeners),
application:set_env(emqx, listeners, NListeners),
ok;
(_) -> ok
end),
emqx_config:put_listener_conf(default, mqtt_ws, [websocket, check_origin_enable], true),
emqx_config:put_listener_conf(default, mqtt_ws, [websocket, check_origins],
[<<"http://localhost:18083">>]),
{ok, _} = application:ensure_all_started(gun),
?assertMatch({gun_upgrade, _},
start_ws_client(#{protocols => [<<"mqtt">>],
headers => [{<<"origin">>, <<"http://localhost:18083">>}]})),
?assertMatch({gun_response, {_, 500, _}},
start_ws_client(#{protocols => [<<"mqtt">>],
headers => [{<<"origin">>, <<"http://localhost:18080">>}]})),
emqx_ct_helpers:stop_apps([]).
headers => [{<<"origin">>, <<"http://localhost:18080">>}]})).
t_ws_non_check_origin(_) ->
emqx_ct_helpers:start_apps([]),
emqx_config:put_listener_conf(default, mqtt_ws, [websocket, check_origin_enable], false),
emqx_config:put_listener_conf(default, mqtt_ws, [websocket, check_origins], []),
{ok, _} = application:ensure_all_started(gun),
?assertMatch({gun_upgrade, _},
start_ws_client(#{protocols => [<<"mqtt">>],
headers => [{<<"origin">>, <<"http://localhost:18083">>}]})),
?assertMatch({gun_upgrade, _},
start_ws_client(#{protocols => [<<"mqtt">>],
headers => [{<<"origin">>, <<"http://localhost:18080">>}]})),
emqx_ct_helpers:stop_apps([]).
headers => [{<<"origin">>, <<"http://localhost:18080">>}]})).
t_init(_) ->
Opts = [{idle_timeout, 300000},
{fail_if_no_subprotocol, false},
{supported_subprotocols, ["mqtt"]}],
WsOpts = #{compress => false,
deflate_opts => #{},
max_frame_size => infinity,
idle_timeout => 300000
},
Opts = #{listener => mqtt_ws, zone => default},
ok = meck:expect(cowboy_req, parse_header, fun(_, req) -> undefined end),
{cowboy_websocket, req, [req, Opts], WsOpts} = ?ws_conn:init(req, Opts),
ok = meck:expect(cowboy_req, reply, fun(_, Req) -> Req end),
{ok, req, _} = ?ws_conn:init(req, Opts),
ok = meck:expect(cowboy_req, parse_header, fun(_, req) -> [<<"mqtt">>] end),
ok = meck:expect(cowboy_req, set_resp_header, fun(_, <<"mqtt">>, req) -> resp end),
{cowboy_websocket, resp, [req, Opts], WsOpts} = ?ws_conn:init(req, Opts).
{cowboy_websocket, resp, [req, Opts], _} = ?ws_conn:init(req, Opts).
t_websocket_handle_binary(_) ->
{ok, _} = websocket_handle({binary, <<>>}, st()),
@ -450,15 +424,6 @@ t_run_gc(_) ->
WsSt = st(#{gc_state => GcSt}),
?ws_conn:run_gc(#{cnt => 100, oct => 10000}, WsSt).
t_check_oom(_) ->
%%Policy = #{max_heap_size => 10, message_queue_len => 10},
%%meck:expect(emqx_zone, oom_policy, fun(_) -> Policy end),
_St = ?ws_conn:check_oom(st()),
ok = timer:sleep(10).
%%receive {shutdown, proc_heap_too_large} -> ok
%%after 0 -> error(expect_shutdown)
%%end.
t_enqueue(_) ->
Packet = ?PUBLISH_PACKET(?QOS_0),
St = ?ws_conn:enqueue(Packet, st()),
@ -473,7 +438,7 @@ t_shutdown(_) ->
st() -> st(#{}).
st(InitFields) when is_map(InitFields) ->
{ok, St, _} = ?ws_conn:websocket_init([req, [{zone, external}]]),
{ok, St, _} = ?ws_conn:websocket_init([req, #{zone => default, listener => mqtt_ws}]),
maps:fold(fun(N, V, S) -> ?ws_conn:set_field(N, V, S) end,
?ws_conn:set_field(channel, channel(), St),
InitFields
@ -493,7 +458,8 @@ channel(InitFields) ->
receive_maximum => 100,
expiry_interval => 0
},
ClientInfo = #{zone => zone,
ClientInfo = #{zone => default,
listener => mqtt_ws,
protocol => mqtt,
peerhost => {127,0,0,1},
clientid => <<"clientid">>,
@ -502,13 +468,13 @@ channel(InitFields) ->
peercert => undefined,
mountpoint => undefined
},
Session = emqx_session:init(#{zone => external},
Session = emqx_session:init(#{zone => default, listener => mqtt_ws},
#{receive_maximum => 0}
),
maps:fold(fun(Field, Value, Channel) ->
emqx_channel:set_field(Field, Value, Channel)
end,
emqx_channel:init(ConnInfo, [{zone, zone}]),
emqx_channel:init(ConnInfo, #{zone => default, listener => mqtt_ws}),
maps:merge(#{clientinfo => ClientInfo,
session => Session,
conn_state => connected

4
apps/emqx/test/props/prop_emqx_sys.erl
View File

@ -59,6 +59,8 @@ prop_sys() ->
do_setup() ->
ok = emqx_logger:set_log_level(emergency),
emqx_config:put([broker, sys_msg_interval], 60000),
emqx_config:put([broker, sys_heartbeat_interval], 30000),
[mock(Mod) || Mod <- ?mock_modules],
ok.
@ -98,8 +100,6 @@ command(_State) ->
{call, emqx_sys, uptime, []},
{call, emqx_sys, datetime, []},
{call, emqx_sys, sysdescr, []},
{call, emqx_sys, sys_interval, []},
{call, emqx_sys, sys_heatbeat_interval, []},
%------------ unexpected message ----------------------%
{call, emqx_sys, handle_call, [emqx_sys, other, state]},
{call, emqx_sys, handle_cast, [emqx_sys, other]},

6
apps/emqx_authn/src/emqx_authn.erl
View File

@ -49,6 +49,7 @@
-export([mnesia/1]).
-boot_mnesia({mnesia, [boot]}).
-copy_mnesia({mnesia, [copy]}).
-define(CHAIN_TAB, emqx_authn_chain).
@ -69,7 +70,10 @@ mnesia(boot) ->
{record_name, chain},
{local_content, true},
{attributes, record_info(fields, chain)},
{storage_properties, StoreProps}]).
{storage_properties, StoreProps}]);
mnesia(copy) ->
ok = ekka_mnesia:copy_table(?CHAIN_TAB, ram_copies).
enable() ->
case emqx:hook('client.authenticate', {?MODULE, authenticate, []}) of

2
apps/emqx_authn/test/emqx_authn_SUITE.erl
View File

@ -42,7 +42,7 @@ end_per_suite(_) ->
set_special_configs(emqx_authn) ->
application:set_env(emqx, plugins_etc_dir,
emqx_ct_helpers:deps_path(emqx_authn, "test")),
Conf = #{<<"emqx_authn">> => #{<<"authenticators">> => []}},
Conf = #{<<"emqx_authn">> => #{<<"authenticators">> => [], <<"enable">> => false}},
ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_authn.conf'), jsx:encode(Conf)),
ok;
set_special_configs(_App) ->

2
apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl
View File

@ -41,7 +41,7 @@ end_per_suite(_) ->
set_special_configs(emqx_authn) ->
application:set_env(emqx, plugins_etc_dir,
emqx_ct_helpers:deps_path(emqx_authn, "test")),
Conf = #{<<"emqx_authn">> => #{<<"authenticators">> => []}},
Conf = #{<<"emqx_authn">> => #{<<"authenticators">> => [], <<"enable">> => false}},
ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_authn.conf'), jsx:encode(Conf)),
ok;
set_special_configs(_App) ->

2
apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl
View File

@ -41,7 +41,7 @@ end_per_suite(_) ->
set_special_configs(emqx_authn) ->
application:set_env(emqx, plugins_etc_dir,
emqx_ct_helpers:deps_path(emqx_authn, "test")),
Conf = #{<<"emqx_authn">> => #{<<"authenticators">> => []}},
Conf = #{<<"emqx_authn">> => #{<<"authenticators">> => [], <<"enable">> => false}},
ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_authn.conf'), jsx:encode(Conf)),
ok;
set_special_configs(_App) ->

2
apps/emqx_authz/etc/emqx_authz.conf
View File

@ -57,7 +57,7 @@ emqx_authz:{
# type: mongo
# config: {
# mongo_type: single
# servers: "127.0.0.1:27017"
# server: "127.0.0.1:27017"
# pool_size: 1
# database: mqtt
# ssl: {enable: false}

73
apps/emqx_authz/src/emqx_authz.erl
View File

@ -15,6 +15,7 @@
%%--------------------------------------------------------------------
-module(emqx_authz).
-behaviour(emqx_config_handler).
-include("emqx_authz.hrl").
-include_lib("emqx/include/logger.hrl").
@ -23,33 +24,55 @@
-export([ register_metrics/0
, init/0
, compile/1
, init_rule/1
, lookup/0
, update/1
, update/2
, authorize/5
, match/4
]).
-export([post_update_config/2, handle_update_config/2]).
-define(CONF_KEY_PATH, [emqx_authz, rules]).
-spec(register_metrics() -> ok).
register_metrics() ->
lists:foreach(fun emqx_metrics:ensure/1, ?AUTHZ_METRICS).
init() ->
ok = register_metrics(),
Rules = emqx_config:get([emqx_authz, rules], []),
NRules = [compile(Rule) || Rule <- Rules],
ok = emqx_hooks:add('client.authorize', {?MODULE, authorize, [NRules]}, -1).
emqx_config_handler:add_handler(?CONF_KEY_PATH, ?MODULE),
NRules = [init_rule(Rule) || Rule <- lookup()],
ok = emqx_hooks:add('client.authorize', {?MODULE, authorize, [NRules]}, -1).
lookup() ->
emqx_config:get([emqx_authz, rules], []).
emqx_config:get(?CONF_KEY_PATH, []).
update(Rules) ->
emqx_config:put([emqx_authz], #{rules => Rules}),
NRules = [compile(Rule) || Rule <- Rules],
update(Cmd, Rules) ->
emqx_config:update_config(?CONF_KEY_PATH, {Cmd, Rules}).
%% For now we only support re-creating the entire rule list
handle_update_config({head, Rule}, OldConf) when is_map(Rule), is_list(OldConf) ->
[Rule | OldConf];
handle_update_config({tail, Rule}, OldConf) when is_map(Rule), is_list(OldConf) ->
OldConf ++ [Rule];
handle_update_config({_, NewConf}, _OldConf) ->
%% overwrite the entire config!
case is_list(NewConf) of
true -> NewConf;
false -> [NewConf]
end.
post_update_config(undefined, _OldConf) ->
%_ = [release_rules(Rule) || Rule <- OldConf],
ok;
post_update_config(NewRules, _OldConf) ->
%_ = [release_rules(Rule) || Rule <- OldConf],
InitedRules = [init_rule(Rule) || Rule <- NewRules],
Action = find_action_in_hooks(),
ok = emqx_hooks:del('client.authorize', Action),
ok = emqx_hooks:add('client.authorize', {?MODULE, authorize, [NRules]}, -1),
ok = emqx_acl_cache:empty_acl_cache().
ok = emqx_hooks:add('client.authorize', {?MODULE, authorize, [InitedRules]}, -1),
ok = emqx_acl_cache:drain_cache().
%%--------------------------------------------------------------------
%% Internal functions
@ -77,35 +100,35 @@ create_resource(#{type := DB,
error({load_config_error, Reason})
end.
-spec(compile(rule()) -> rule()).
compile(#{topics := Topics,
action := Action,
permission := Permission,
principal := Principal
-spec(init_rule(rule()) -> rule()).
init_rule(#{topics := Topics,
action := Action,
permission := Permission,
principal := Principal
} = Rule) when ?ALLOW_DENY(Permission), ?PUBSUB(Action), is_list(Topics) ->
NTopics = [compile_topic(Topic) || Topic <- Topics],
Rule#{principal => compile_principal(Principal),
topics => NTopics
};
compile(#{principal := Principal,
type := http,
config := #{url := Url} = Config
} = Rule) ->
init_rule(#{principal := Principal,
type := http,
config := #{url := Url} = Config
} = Rule) ->
NConfig = maps:merge(Config, #{base_url => maps:remove(query, Url)}),
NRule = create_resource(Rule#{config := NConfig}),
NRule#{principal => compile_principal(Principal)};
compile(#{principal := Principal,
type := DB
init_rule(#{principal := Principal,
type := DB
} = Rule) when DB =:= redis;
DB =:= mongo ->
NRule = create_resource(Rule),
NRule#{principal => compile_principal(Principal)};
compile(#{principal := Principal,
type := DB,
sql := SQL
init_rule(#{principal := Principal,
type := DB,
sql := SQL
} = Rule) when DB =:= mysql;
DB =:= pgsql ->
Mod = list_to_existing_atom(io_lib:format("~s_~s",[?APP, DB])),

21
apps/emqx_authz/src/emqx_authz_api.erl
View File

@ -56,28 +56,23 @@ lookup_authz(_Bindings, _Params) ->
return({ok, emqx_authz:lookup()}).
update_authz(_Bindings, Params) ->
Rules = get_rules(Params),
return(emqx_authz:update(Rules)).
Rules = form_rules(Params),
return(emqx_authz:update(replace, Rules)).
append_authz(_Bindings, Params) ->
Rules = get_rules(Params),
NRules = lists:append(emqx_authz:lookup(), Rules),
return(emqx_authz:update(NRules)).
Rules = form_rules(Params),
return(emqx_authz:update(tail, Rules)).
push_authz(_Bindings, Params) ->
Rules = get_rules(Params),
NRules = lists:append(Rules, emqx_authz:lookup()),
return(emqx_authz:update(NRules)).
Rules = form_rules(Params),
return(emqx_authz:update(head, Rules)).
%%------------------------------------------------------------------------------
%% Interval Funcs
%%------------------------------------------------------------------------------
get_rules(Params) ->
{ok, Conf} = hocon:binary(jsx:encode(#{<<"emqx_authz">> => Params}), #{format => richmap}),
CheckConf = hocon_schema:check(emqx_authz_schema, Conf, #{atom_key => true}),
#{emqx_authz := #{rules := Rules}} = hocon_schema:richmap_to_map(CheckConf),
Rules.
form_rules(Params) ->
Params.
%%--------------------------------------------------------------------
%% EUnits

2
apps/emqx_authz/src/emqx_authz_mongo.erl
View File

@ -71,7 +71,7 @@ match(Client, PubSub, Topic,
#{<<"simple_rule">> => Rule},
#{atom_key => true},
[simple_rule]),
case emqx_authz:match(Client, PubSub, Topic, emqx_authz:compile(NRule)) of
case emqx_authz:match(Client, PubSub, Topic, emqx_authz:init_rule(NRule)) of
true -> {matched, NPermission};
false -> nomatch
end.

2
apps/emqx_authz/src/emqx_authz_mysql.erl
View File

@ -90,7 +90,7 @@ match(Client, PubSub, Topic,
#{<<"simple_rule">> => Rule},
#{atom_key => true},
[simple_rule]),
case emqx_authz:match(Client, PubSub, Topic, emqx_authz:compile(NRule)) of
case emqx_authz:match(Client, PubSub, Topic, emqx_authz:init_rule(NRule)) of
true -> {matched, NPermission};
false -> nomatch
end.

2
apps/emqx_authz/src/emqx_authz_pgsql.erl
View File

@ -94,7 +94,7 @@ match(Client, PubSub, Topic,
#{<<"simple_rule">> => Rule},
#{atom_key => true},
[simple_rule]),
case emqx_authz:match(Client, PubSub, Topic, emqx_authz:compile(NRule)) of
case emqx_authz:match(Client, PubSub, Topic, emqx_authz:init_rule(NRule)) of
true -> {matched, NPermission};
false -> nomatch
end.

2
apps/emqx_authz/src/emqx_authz_redis.erl
View File

@ -74,7 +74,7 @@ match(Client, PubSub, Topic,
#{<<"simple_rule">> => Rule},
#{atom_key => true},
[simple_rule]),
case emqx_authz:match(Client, PubSub, Topic, emqx_authz:compile(NRule)) of
case emqx_authz:match(Client, PubSub, Topic, emqx_authz:init_rule(NRule)) of
true -> {matched, allow};
false -> nomatch
end.

21
apps/emqx_authz/src/emqx_authz_schema.erl
View File

@ -2,10 +2,6 @@
-include_lib("typerefl/include/types.hrl").
-type action() :: publish | subscribe | all.
-type permission() :: allow | deny.
-type url() :: emqx_http_lib:uri_map().
-reflect_type([ permission/0
, action/0
, url/0
@ -13,6 +9,18 @@
-typerefl_from_string({url/0, emqx_http_lib, uri_parse}).
-type action() :: publish | subscribe | all.
-type permission() :: allow | deny.
-type url() :: #{
scheme := http | https,
host := string(),
port := non_neg_integer(),
path => string(),
query => string(),
fragment => string(),
userinfo => string()
}.
-export([ structs/0
, fields/1
]).
@ -51,9 +59,8 @@ fields(http_get) ->
end
}
}
, {method, #{type => get,
default => get
}}
, {method, #{type => get, default => get }}
, {request_timeout, #{type => timeout(), default => 30000 }}
] ++ proplists:delete(base_url, emqx_connector_http:fields(config));
fields(http_post) ->
[ {url, #{type => url()}}

49
apps/emqx_authz/test/emqx_authz_SUITE.erl
View File

@ -29,23 +29,16 @@ groups() ->
[].
init_per_suite(Config) ->
ok = emqx_ct_helpers:start_apps([emqx_authz], fun set_special_configs/1),
ok = emqx_ct_helpers:start_apps([emqx_authz]),
ok = emqx_config:update_config([zones, default, acl, cache, enable], false),
ok = emqx_config:update_config([zones, default, acl, enable], true),
emqx_authz:update(replace, []),
Config.
end_per_suite(_Config) ->
file:delete(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf')),
emqx_ct_helpers:stop_apps([emqx_authz]).
set_special_configs(emqx) ->
application:set_env(emqx, allow_anonymous, true),
application:set_env(emqx, enable_acl_cache, false),
ok;
set_special_configs(emqx_authz) ->
emqx_config:put([emqx_authz], #{rules => []}),
ok;
set_special_configs(_App) ->
ok.
-define(RULE1, #{principal => all,
topics => [<<"#">>],
action => all,
@ -81,19 +74,19 @@ set_special_configs(_App) ->
%%------------------------------------------------------------------------------
%% Testcases
%%------------------------------------------------------------------------------
t_compile(_) ->
t_init_rule(_) ->
?assertEqual(#{permission => deny,
action => all,
principal => all,
topics => [['#']]
},emqx_authz:compile(?RULE1)),
}, emqx_authz:init_rule(?RULE1)),
?assertEqual(#{permission => allow,
action => all,
principal =>
#{ipaddress => {{127,0,0,1},{127,0,0,1},32}},
topics => [#{eq => ['#']},
#{eq => ['+']}]
}, emqx_authz:compile(?RULE2)),
}, emqx_authz:init_rule(?RULE2)),
?assertMatch(
#{permission := allow,
action := publish,
@ -103,7 +96,7 @@ t_compile(_) ->
]
},
topics := [[<<"test">>]]
}, emqx_authz:compile(?RULE3)),
}, emqx_authz:init_rule(?RULE3)),
?assertMatch(
#{permission := deny,
action := publish,
@ -115,31 +108,39 @@ t_compile(_) ->
topics := [#{pattern := [<<"%u">>]},
#{pattern := [<<"%c">>]}
]
}, emqx_authz:compile(?RULE4)),
}, emqx_authz:init_rule(?RULE4)),
ok.
t_authz(_) ->
ClientInfo1 = #{clientid => <<"test">>,
username => <<"test">>,
peerhost => {127,0,0,1}
peerhost => {127,0,0,1},
zone => default,
listener => mqtt_tcp
},
ClientInfo2 = #{clientid => <<"test">>,
username => <<"test">>,
peerhost => {192,168,0,10}
peerhost => {192,168,0,10},
zone => default,
listener => mqtt_tcp
},
ClientInfo3 = #{clientid => <<"test">>,
username => <<"fake">>,
peerhost => {127,0,0,1}
peerhost => {127,0,0,1},
zone => default,
listener => mqtt_tcp
},
ClientInfo4 = #{clientid => <<"fake">>,
username => <<"test">>,
peerhost => {127,0,0,1}
peerhost => {127,0,0,1},
zone => default,
listener => mqtt_tcp
},
Rules1 = [emqx_authz:compile(Rule) || Rule <- [?RULE1, ?RULE2]],
Rules2 = [emqx_authz:compile(Rule) || Rule <- [?RULE2, ?RULE1]],
Rules3 = [emqx_authz:compile(Rule) || Rule <- [?RULE3, ?RULE4]],
Rules4 = [emqx_authz:compile(Rule) || Rule <- [?RULE4, ?RULE1]],
Rules1 = [emqx_authz:init_rule(Rule) || Rule <- [?RULE1, ?RULE2]],
Rules2 = [emqx_authz:init_rule(Rule) || Rule <- [?RULE2, ?RULE1]],
Rules3 = [emqx_authz:init_rule(Rule) || Rule <- [?RULE3, ?RULE4]],
Rules4 = [emqx_authz:init_rule(Rule) || Rule <- [?RULE4, ?RULE1]],
?assertEqual({stop, deny},
emqx_authz:authorize(ClientInfo1, subscribe, <<"#">>, deny, [])),

172
apps/emqx_authz/test/emqx_authz_api_SUITE.erl
View File

@ -18,61 +18,66 @@
-compile(nowarn_export_all).
-compile(export_all).
-include("emqx_authz.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
% -include("emqx_authz.hrl").
% -include_lib("eunit/include/eunit.hrl").
% -include_lib("common_test/include/ct.hrl").
-import(emqx_ct_http, [ request_api/3
, request_api/5
, get_http_data/1
, create_default_app/0
, delete_default_app/0
, default_auth_header/0
]).
% -import(emqx_ct_http, [ request_api/3
% , request_api/5
% , get_http_data/1
% , create_default_app/0
% , delete_default_app/0
% , default_auth_header/0
% ]).
-define(HOST, "http://127.0.0.1:8081/").
-define(API_VERSION, "v4").
-define(BASE_PATH, "api").
% -define(HOST, "http://127.0.0.1:8081/").
% -define(API_VERSION, "v4").
% -define(BASE_PATH, "api").
all() ->
%% TODO: V5 API
%% emqx_ct:all(?MODULE).
[].
[t_api_unit_test].
groups() ->
[].
init_per_suite(Config) ->
ok = emqx_ct_helpers:start_apps([emqx_authz, emqx_management], fun set_special_configs/1),
create_default_app(),
%% important! let emqx_schema include the current app!
meck:new(emqx_schema, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_schema, includes, fun() -> ["emqx_authz"] end ),
ok = emqx_ct_helpers:start_apps([emqx_authz]),
%create_default_app(),
Config.
end_per_suite(_Config) ->
delete_default_app(),
%delete_default_app(),
file:delete(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf')),
emqx_ct_helpers:stop_apps([emqx_authz, emqx_management]).
meck:unload(emqx_schema),
emqx_ct_helpers:stop_apps([emqx_authz]).
set_special_configs(emqx) ->
application:set_env(emqx, allow_anonymous, true),
application:set_env(emqx, enable_acl_cache, false),
ok;
set_special_configs(emqx_authz) ->
emqx_config:put([emqx_authz], #{rules => []}),
ok;
% set_special_configs(emqx) ->
% application:set_env(emqx, allow_anonymous, true),
% application:set_env(emqx, enable_acl_cache, false),
% ok;
% set_special_configs(emqx_authz) ->
% emqx_config:put([emqx_authz], #{rules => []}),
% ok;
set_special_configs(emqx_management) ->
emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}],
applications =>[#{id => "admin", secret => "public"}]}),
ok;
% set_special_configs(emqx_management) ->
% emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}],
% applications =>[#{id => "admin", secret => "public"}]}),
% ok;
set_special_configs(_App) ->
ok.
% set_special_configs(_App) ->
% ok.
%%------------------------------------------------------------------------------
%% Testcases
%%------------------------------------------------------------------------------
% %%------------------------------------------------------------------------------
% %% Testcases
% %%------------------------------------------------------------------------------
t_api(_Config) ->
t_api_unit_test(_Config) ->
Rule1 = #{<<"principal">> =>
#{<<"and">> => [#{<<"username">> => <<"^test?">>},
#{<<"clientid">> => <<"^test?">>}
@ -81,53 +86,70 @@ t_api(_Config) ->
<<"topics">> => [<<"%u">>],
<<"permission">> => <<"allow">>
},
{ok, _} = request_http_rest_add(["authz/push"], #{rules => [Rule1]}),
{ok, Result1} = request_http_rest_lookup(["authz"]),
?assertMatch([Rule1 | _ ], get_http_data(Result1)),
ok = emqx_authz_api:push_authz(#{}, Rule1),
[#{action := subscribe,
permission := allow,
principal :=
#{'and' := [#{username := <<"^test?">>},
#{clientid := <<"^test?">>}]},
topics := [<<"%u">>]}] = emqx_config:get([emqx_authz, rules]).
Rule2 = #{<<"principal">> => #{<<"ipaddress">> => <<"127.0.0.1">>},
<<"action">> => <<"publish">>,
<<"topics">> => [#{<<"eq">> => <<"#">>},
#{<<"eq">> => <<"+">>}
],
<<"permission">> => <<"deny">>
},
{ok, _} = request_http_rest_add(["authz/append"], #{rules => [Rule2]}),
{ok, Result2} = request_http_rest_lookup(["authz"]),
?assertEqual(Rule2#{<<"principal">> => #{<<"ipaddress">> => "127.0.0.1"}},
lists:last(get_http_data(Result2))),
% t_api(_Config) ->
% Rule1 = #{<<"principal">> =>
% #{<<"and">> => [#{<<"username">> => <<"^test?">>},
% #{<<"clientid">> => <<"^test?">>}
% ]},
% <<"action">> => <<"subscribe">>,
% <<"topics">> => [<<"%u">>],
% <<"permission">> => <<"allow">>
% },
% {ok, _} = request_http_rest_add(["authz/push"], #{rules => [Rule1]}),
% {ok, Result1} = request_http_rest_lookup(["authz"]),
% ?assertMatch([Rule1 | _ ], get_http_data(Result1)),
{ok, _} = request_http_rest_update(["authz"], #{rules => []}),
{ok, Result3} = request_http_rest_lookup(["authz"]),
?assertEqual([], get_http_data(Result3)),
ok.
% Rule2 = #{<<"principal">> => #{<<"ipaddress">> => <<"127.0.0.1">>},
% <<"action">> => <<"publish">>,
% <<"topics">> => [#{<<"eq">> => <<"#">>},
% #{<<"eq">> => <<"+">>}
% ],
% <<"permission">> => <<"deny">>
% },
% {ok, _} = request_http_rest_add(["authz/append"], #{rules => [Rule2]}),
% {ok, Result2} = request_http_rest_lookup(["authz"]),
% ?assertEqual(Rule2#{<<"principal">> => #{<<"ipaddress">> => "127.0.0.1"}},
% lists:last(get_http_data(Result2))),
%%--------------------------------------------------------------------
%% HTTP Request
%%--------------------------------------------------------------------
% {ok, _} = request_http_rest_update(["authz"], #{rules => []}),
% {ok, Result3} = request_http_rest_lookup(["authz"]),
% ?assertEqual([], get_http_data(Result3)),
% ok.
request_http_rest_list(Path) ->
request_api(get, uri(Path), default_auth_header()).
% %%--------------------------------------------------------------------
% %% HTTP Request
% %%--------------------------------------------------------------------
request_http_rest_lookup(Path) ->
request_api(get, uri([Path]), default_auth_header()).
% request_http_rest_list(Path) ->
% request_api(get, uri(Path), default_auth_header()).
request_http_rest_add(Path, Params) ->
request_api(post, uri(Path), [], default_auth_header(), Params).
% request_http_rest_lookup(Path) ->
% request_api(get, uri([Path]), default_auth_header()).
request_http_rest_update(Path, Params) ->
request_api(put, uri([Path]), [], default_auth_header(), Params).
% request_http_rest_add(Path, Params) ->
% request_api(post, uri(Path), [], default_auth_header(), Params).
request_http_rest_delete(Login) ->
request_api(delete, uri([Login]), default_auth_header()).
% request_http_rest_update(Path, Params) ->
% request_api(put, uri([Path]), [], default_auth_header(), Params).
uri() -> uri([]).
uri(Parts) when is_list(Parts) ->
NParts = [b2l(E) || E <- Parts],
?HOST ++ filename:join([?BASE_PATH, ?API_VERSION | NParts]).
% request_http_rest_delete(Login) ->
% request_api(delete, uri([Login]), default_auth_header()).
%% @private
b2l(B) when is_binary(B) ->
binary_to_list(B);
b2l(L) when is_list(L) ->
L.
% uri() -> uri([]).
% uri(Parts) when is_list(Parts) ->
% NParts = [b2l(E) || E <- Parts],
% ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION | NParts]).
% %% @private
% b2l(B) when is_binary(B) ->
% binary_to_list(B);
% b2l(L) when is_list(L) ->
% L.

48
apps/emqx_authz/test/emqx_authz_http_SUITE.erl
View File

@ -29,41 +29,33 @@ groups() ->
[].
init_per_suite(Config) ->
%% important! let emqx_schema include the current app!
meck:new(emqx_schema, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_schema, includes, fun() -> ["emqx_authz"] end ),
meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end ),
ok = emqx_ct_helpers:start_apps([emqx_authz], fun set_special_configs/1),
ok = emqx_ct_helpers:start_apps([emqx_authz]),
ok = emqx_config:update_config([zones, default, acl, cache, enable], false),
ok = emqx_config:update_config([zones, default, acl, enable], true),
Rules = [#{ <<"config">> => #{
<<"url">> => <<"https://fake.com:443/">>,
<<"headers">> => #{},
<<"method">> => <<"get">>,
<<"request_timeout">> => 5000
},
<<"principal">> => <<"all">>,
<<"type">> => <<"http">>}
],
ok = emqx_authz:update(replace, Rules),
Config.
end_per_suite(_Config) ->
file:delete(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf')),
emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]),
meck:unload(emqx_schema),
meck:unload(emqx_resource).
set_special_configs(emqx) ->
application:set_env(emqx, allow_anonymous, true),
application:set_env(emqx, enable_acl_cache, false),
application:set_env(emqx, acl_nomatch, deny),
application:set_env(emqx, plugins_loaded_file,
emqx_ct_helpers:deps_path(emqx, "test/loaded_plguins")),
ok;
set_special_configs(emqx_authz) ->
Rules = [#{config =>#{
url => #{host => "fake.com",
path => "/",
port => 443,
scheme => https},
headers => #{},
method => get,
request_timeout => 5000
},
principal => all,
type => http}
],
emqx_config:put([emqx_authz], #{rules => Rules}),
ok;
set_special_configs(_App) ->
ok.
%%------------------------------------------------------------------------------
%% Testcases
%%------------------------------------------------------------------------------
@ -73,7 +65,9 @@ t_authz(_) ->
username => <<"username">>,
peerhost => {127,0,0,1},
protocol => mqtt,
mountpoint => <<"fake">>
mountpoint => <<"fake">>,
zone => default,
listener => mqtt_tcp
},
meck:expect(emqx_resource, query, fun(_, _) -> {ok, 204, fake_headers} end),

54
apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl
View File

@ -31,33 +31,35 @@ groups() ->
init_per_suite(Config) ->
meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end ),
ok = emqx_ct_helpers:start_apps([emqx_authz], fun set_special_configs/1),
%% important! let emqx_schema include the current app!
meck:new(emqx_schema, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_schema, includes, fun() -> ["emqx_authz"] end ),
ok = emqx_ct_helpers:start_apps([emqx_authz]),
ct:pal("---- emqx_hooks: ~p", [ets:tab2list(emqx_hooks)]),
ok = emqx_config:update_config([zones, default, acl, cache, enable], false),
ok = emqx_config:update_config([zones, default, acl, enable], true),
Rules = [#{ <<"config">> => #{
<<"mongo_type">> => <<"single">>,
<<"server">> => <<"127.0.0.1:27017">>,
<<"pool_size">> => 1,
<<"database">> => <<"mqtt">>,
<<"ssl">> => #{<<"enable">> => false}},
<<"principal">> => <<"all">>,
<<"collection">> => <<"fake">>,
<<"find">> => #{<<"a">> => <<"b">>},
<<"type">> => <<"mongo">>}
],
ok = emqx_authz:update(replace, Rules),
Config.
end_per_suite(_Config) ->
file:delete(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf')),
emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]),
meck:unload(emqx_schema),
meck:unload(emqx_resource).
set_special_configs(emqx) ->
application:set_env(emqx, allow_anonymous, true),
application:set_env(emqx, enable_acl_cache, false),
application:set_env(emqx, acl_nomatch, deny),
application:set_env(emqx, plugins_loaded_file,
emqx_ct_helpers:deps_path(emqx, "test/loaded_plguins")),
ok;
set_special_configs(emqx_authz) ->
Rules = [#{config =>#{},
principal => all,
collection => <<"fake">>,
find => #{<<"a">> => <<"b">>},
type => mongo}
],
emqx_config:put([emqx_authz], #{rules => Rules}),
ok;
set_special_configs(_App) ->
ok.
-define(RULE1,[#{<<"topics">> => [<<"#">>],
<<"permission">> => <<"deny">>,
<<"action">> => <<"all">>}]).
@ -78,15 +80,21 @@ set_special_configs(_App) ->
t_authz(_) ->
ClientInfo1 = #{clientid => <<"test">>,
username => <<"test">>,
peerhost => {127,0,0,1}
peerhost => {127,0,0,1},
zone => default,
listener => mqtt_tcp
},
ClientInfo2 = #{clientid => <<"test_clientid">>,
username => <<"test_username">>,
peerhost => {192,168,0,10}
peerhost => {192,168,0,10},
zone => default,
listener => mqtt_tcp
},
ClientInfo3 = #{clientid => <<"test_clientid">>,
username => <<"fake_username">>,
peerhost => {127,0,0,1}
peerhost => {127,0,0,1},
zone => default,
listener => mqtt_tcp
},
meck:expect(emqx_resource, query, fun(_, _) -> [] end),

52
apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl
View File

@ -29,34 +29,36 @@ groups() ->
[].
init_per_suite(Config) ->
%% important! let emqx_schema include the current app!
meck:new(emqx_schema, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_schema, includes, fun() -> ["emqx_authz"] end ),
meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end ),
ok = emqx_ct_helpers:start_apps([emqx_authz], fun set_special_configs/1),
ok = emqx_ct_helpers:start_apps([emqx_authz]),
ok = emqx_config:update_config([zones, default, acl, cache, enable], false),
ok = emqx_config:update_config([zones, default, acl, enable], true),
Rules = [#{ <<"config">> => #{
<<"server">> => <<"127.0.0.1:27017">>,
<<"pool_size">> => 1,
<<"database">> => <<"mqtt">>,
<<"username">> => <<"xx">>,
<<"password">> => <<"ee">>,
<<"auto_reconnect">> => true,
<<"ssl">> => #{<<"enable">> => false}
},
<<"principal">> => <<"all">>,
<<"sql">> => <<"abcb">>,
<<"type">> => <<"mysql">> }],
emqx_authz:update(replace, Rules),
Config.
end_per_suite(_Config) ->
file:delete(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf')),
emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]),
meck:unload(emqx_schema),
meck:unload(emqx_resource).
set_special_configs(emqx) ->
application:set_env(emqx, allow_anonymous, false),
application:set_env(emqx, enable_acl_cache, false),
application:set_env(emqx, acl_nomatch, deny),
application:set_env(emqx, plugins_loaded_file,
emqx_ct_helpers:deps_path(emqx, "test/loaded_plguins")),
ok;
set_special_configs(emqx_authz) ->
Rules = [#{config =>#{},
principal => all,
sql => <<"fake">>,
type => mysql}
],
emqx_config:put([emqx_authz], #{rules => Rules}),
ok;
set_special_configs(_App) ->
ok.
-define(COLUMNS, [ <<"ipaddress">>
, <<"username">>
, <<"clientid">>
@ -76,15 +78,21 @@ set_special_configs(_App) ->
t_authz(_) ->
ClientInfo1 = #{clientid => <<"test">>,
username => <<"test">>,
peerhost => {127,0,0,1}
peerhost => {127,0,0,1},
zone => default,
listener => mqtt_tcp
},
ClientInfo2 = #{clientid => <<"test_clientid">>,
username => <<"test_username">>,
peerhost => {192,168,0,10}
peerhost => {192,168,0,10},
zone => default,
listener => mqtt_tcp
},
ClientInfo3 = #{clientid => <<"test_clientid">>,
username => <<"fake_username">>,
peerhost => {127,0,0,1}
peerhost => {127,0,0,1},
zone => default,
listener => mqtt_tcp
},
meck:expect(emqx_resource, query, fun(_, _) -> {ok, ?COLUMNS, []} end),

51
apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl
View File

@ -29,34 +29,35 @@ groups() ->
[].
init_per_suite(Config) ->
%% important! let emqx_schema include the current app!
meck:new(emqx_schema, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_schema, includes, fun() -> ["emqx_authz"] end ),
meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end ),
ok = emqx_ct_helpers:start_apps([emqx_authz], fun set_special_configs/1),
ok = emqx_ct_helpers:start_apps([emqx_authz]),
ok = emqx_config:update_config([zones, default, acl, cache, enable], false),
ok = emqx_config:update_config([zones, default, acl, enable], true),
Rules = [#{ <<"config">> => #{
<<"server">> => <<"127.0.0.1:27017">>,
<<"pool_size">> => 1,
<<"database">> => <<"mqtt">>,
<<"username">> => <<"xx">>,
<<"password">> => <<"ee">>,
<<"auto_reconnect">> => true,
<<"ssl">> => #{<<"enable">> => false}
},
<<"sql">> => <<"abcb">>,
<<"type">> => <<"pgsql">> }],
emqx_authz:update(replace, Rules),
Config.
end_per_suite(_Config) ->
file:delete(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf')),
emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]),
meck:unload(emqx_schema),
meck:unload(emqx_resource).
set_special_configs(emqx) ->
application:set_env(emqx, allow_anonymous, false),
application:set_env(emqx, enable_acl_cache, false),
application:set_env(emqx, acl_nomatch, deny),
application:set_env(emqx, plugins_loaded_file,
emqx_ct_helpers:deps_path(emqx, "test/loaded_plguins")),
ok;
set_special_configs(emqx_authz) ->
Rules = [#{config =>#{},
principal => all,
sql => <<"fake">>,
type => pgsql}
],
emqx_config:put([emqx_authz], #{rules => Rules}),
ok;
set_special_configs(_App) ->
ok.
-define(COLUMNS, [ {column, <<"ipaddress">>, meck, meck, meck, meck, meck, meck, meck}
, {column, <<"username">>, meck, meck, meck, meck, meck, meck, meck}
, {column, <<"clientid">>, meck, meck, meck, meck, meck, meck, meck}
@ -76,15 +77,21 @@ set_special_configs(_App) ->
t_authz(_) ->
ClientInfo1 = #{clientid => <<"test">>,
username => <<"test">>,
peerhost => {127,0,0,1}
peerhost => {127,0,0,1},
zone => default,
listener => mqtt_tcp
},
ClientInfo2 = #{clientid => <<"test_clientid">>,
username => <<"test_username">>,
peerhost => {192,168,0,10}
peerhost => {192,168,0,10},
zone => default,
listener => mqtt_tcp
},
ClientInfo3 = #{clientid => <<"test_clientid">>,
username => <<"fake_username">>,
peerhost => {127,0,0,1}
peerhost => {127,0,0,1},
zone => default,
listener => mqtt_tcp
},
meck:expect(emqx_resource, query, fun(_, _) -> {ok, ?COLUMNS, []} end),

42
apps/emqx_authz/test/emqx_authz_redis_SUITE.erl
View File

@ -29,34 +29,34 @@ groups() ->
[].
init_per_suite(Config) ->
%% important! let emqx_schema include the current app!
meck:new(emqx_schema, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_schema, includes, fun() -> ["emqx_authz"] end ),
meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end ),
ok = emqx_ct_helpers:start_apps([emqx_authz], fun set_special_configs/1),
ok = emqx_ct_helpers:start_apps([emqx_authz]),
ok = emqx_config:update_config([zones, default, acl, cache, enable], false),
ok = emqx_config:update_config([zones, default, acl, enable], true),
Rules = [#{ <<"config">> => #{
<<"server">> => <<"127.0.0.1:27017">>,
<<"pool_size">> => 1,
<<"database">> => 0,
<<"password">> => <<"ee">>,
<<"auto_reconnect">> => true,
<<"ssl">> => #{<<"enable">> => false}
},
<<"cmd">> => <<"HGETALL mqtt_acl:%u">>,
<<"type">> => <<"redis">> }],
emqx_authz:update(replace, Rules),
Config.
end_per_suite(_Config) ->
file:delete(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf')),
emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]),
meck:unload(emqx_schema),
meck:unload(emqx_resource).
set_special_configs(emqx) ->
application:set_env(emqx, allow_anonymous, true),
application:set_env(emqx, enable_acl_cache, false),
application:set_env(emqx, acl_nomatch, deny),
application:set_env(emqx, plugins_loaded_file,
emqx_ct_helpers:deps_path(emqx, "test/loaded_plguins")),
ok;
set_special_configs(emqx_authz) ->
Rules = [#{config =>#{},
principal => all,
cmd => <<"fake">>,
type => redis}
],
emqx_config:put([emqx_authz], #{rules => Rules}),
ok;
set_special_configs(_App) ->
ok.
-define(RULE1, [<<"test/%u">>, <<"publish">>]).
-define(RULE2, [<<"test/%c">>, <<"publish">>]).
-define(RULE3, [<<"#">>, <<"subscribe">>]).
@ -68,7 +68,9 @@ set_special_configs(_App) ->
t_authz(_) ->
ClientInfo = #{clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127,0,0,1}
peerhost => {127,0,0,1},
zone => default,
listener => mqtt_tcp
},
meck:expect(emqx_resource, query, fun(_, _) -> {ok, []} end),

5
apps/emqx_data_bridge/src/emqx_data_bridge_app.erl
View File

@ -34,7 +34,10 @@ stop(_State) ->
handle_update_config({update, Bridge = #{<<"name">> := Name}}, OldConf) ->
[Bridge | remove_bridge(Name, OldConf)];
handle_update_config({delete, Name}, OldConf) ->
remove_bridge(Name, OldConf).
remove_bridge(Name, OldConf);
handle_update_config(NewConf, _OldConf) when is_list(NewConf) ->
%% overwrite the entire config!
NewConf.
remove_bridge(_Name, undefined) ->
[];

70
apps/emqx_gateway/etc/emqx_gateway.conf
View File

@ -29,46 +29,46 @@ emqx_gateway: {
}
}
mqttsn.1: {
## The MQTT-SN Gateway ID in ADVERTISE message.
gateway_id: 1
# mqttsn.1: {
# ## The MQTT-SN Gateway ID in ADVERTISE message.
# gateway_id: 1
## Enable broadcast this gateway to WLAN
broadcast: true
# ## Enable broadcast this gateway to WLAN
# broadcast: true
## To control whether write statistics data into ETS table
## for dashbord to read.
enable_stats: true
# ## To control whether write statistics data into ETS table
# ## for dashbord to read.
# enable_stats: true
## To control whether accept and process the received
## publish message with qos=-1.
enable_qos3: true
# ## To control whether accept and process the received
# ## publish message with qos=-1.
# enable_qos3: true
## Idle timeout for a MQTT-SN channel
idle_timeout: 30s
# ## Idle timeout for a MQTT-SN channel
# idle_timeout: 30s
## The pre-defined topic name corresponding to the pre-defined topic
## id of N.
## Note that the pre-defined topic id of 0 is reserved.
predefined: [
{ id: 1
topic: "/predefined/topic/name/hello"
},
{ id: 2
topic: "/predefined/topic/name/nice"
}
]
# ## The pre-defined topic name corresponding to the pre-defined topic
# ## id of N.
# ## Note that the pre-defined topic id of 0 is reserved.
# predefined: [
# { id: 1
# topic: "/predefined/topic/name/hello"
# },
# { id: 2
# topic: "/predefined/topic/name/nice"
# }
# ]
### ClientInfo override
clientinfo_override: {
username: "mqtt_sn_user"
password: "abc"
}
# ### ClientInfo override
# clientinfo_override: {
# username: "mqtt_sn_user"
# password: "abc"
# }
listener.udp.1: {
bind: 1884
max_connections: 10240000
max_conn_rate: 1000
}
}
# listener.udp.1: {
# bind: 1884
# max_connections: 10240000
# max_conn_rate: 1000
# }
# }
}

3
apps/emqx_gateway/src/coap/emqx_coap_mqtt_adapter.erl
View File

@ -371,7 +371,8 @@ clientinfo(#state{peername = {PeerHost, _},
clientid = ClientId,
username = Username,
password = Password}) ->
#{zone => undefined,
#{zone => default,
listener => mqtt_tcp, %% FIXME: this won't work
protocol => coap,
peerhost => PeerHost,
sockport => 5683, %% FIXME:

5
apps/emqx_gateway/src/emqx_gateway_cli.erl
View File

@ -39,7 +39,10 @@ unload() ->
lists:foreach(fun(Cmd) -> emqx_ctl:unregister_command(Cmd) end, Cmds).
is_cmd(Fun) ->
not lists:member(Fun, [init, load, module_info]).
case atom_to_list(Fun) of
"gateway" ++ _ -> true;
_ -> false
end.
%%--------------------------------------------------------------------

3
apps/emqx_gateway/src/emqx_gateway_ctx.erl
View File

@ -68,7 +68,8 @@
| {error, any()}.
authenticate(_Ctx = #{auth := ChainId}, ClientInfo0) ->
ClientInfo = ClientInfo0#{
zone => undefined,
zone => default,
listener => mqtt_tcp,
chain_id => ChainId
},
case emqx_access_control:authenticate(ClientInfo) of

19
apps/emqx_gateway/src/emqx_gateway_schema.erl
View File

@ -8,13 +8,11 @@
-include_lib("typerefl/include/types.hrl").
-type flag() :: true | false.
-type duration() :: integer().
-type bytesize() :: integer().
-type comma_separated_list() :: list().
-type ip_port() :: tuple().
-typerefl_from_string({flag/0, emqx_schema, to_flag}).
-typerefl_from_string({duration/0, emqx_schema, to_duration}).
-typerefl_from_string({bytesize/0, emqx_schema, to_bytesize}).
-typerefl_from_string({comma_separated_list/0, emqx_schema, to_comma_separated_list}).
@ -22,8 +20,7 @@
-behaviour(hocon_schema).
-reflect_type([ flag/0
, duration/0
-reflect_type([ duration/0
, bytesize/0
, comma_separated_list/0
, ip_port/0
@ -114,16 +111,16 @@ fields(listener_settings) ->
%, {zone, t(string())}
%, {rate_limit, t(comma_separated_list())}
, {access, t(ref(access))}
, {proxy_protocol, t(flag())}
, {proxy_protocol, t(boolean())}
, {proxy_protocol_timeout, t(duration())}
, {backlog, t(integer(), undefined, 1024)}
, {send_timeout, t(duration(), undefined, "15s")}
, {send_timeout_close, t(flag(), undefined, true)}
, {send_timeout_close, t(boolean(), undefined, true)}
, {recbuf, t(bytesize())}
, {sndbuf, t(bytesize())}
, {buffer, t(bytesize())}
, {high_watermark, t(bytesize(), undefined, "1MB")}
, {tune_buffer, t(flag())}
, {tune_buffer, t(boolean())}
, {nodelay, t(boolean())}
, {reuseaddr, t(boolean())}
];
@ -202,15 +199,15 @@ ssl(Mapping, Defaults) ->
_ -> Mapping ++ "." ++ Field
end end,
D = fun (Field) -> maps:get(list_to_atom(Field), Defaults, undefined) end,
[ {"enable", t(flag(), M("enable"), D("enable"))}
[ {"enable", t(boolean(), M("enable"), D("enable"))}
, {"cacertfile", t(string(), M("cacertfile"), D("cacertfile"))}
, {"certfile", t(string(), M("certfile"), D("certfile"))}
, {"keyfile", t(string(), M("keyfile"), D("keyfile"))}
, {"verify", t(union(verify_peer, verify_none), M("verify"), D("verify"))}
, {"fail_if_no_peer_cert", t(boolean(), M("fail_if_no_peer_cert"), D("fail_if_no_peer_cert"))}
, {"secure_renegotiate", t(flag(), M("secure_renegotiate"), D("secure_renegotiate"))}
, {"reuse_sessions", t(flag(), M("reuse_sessions"), D("reuse_sessions"))}
, {"honor_cipher_order", t(flag(), M("honor_cipher_order"), D("honor_cipher_order"))}
, {"secure_renegotiate", t(boolean(), M("secure_renegotiate"), D("secure_renegotiate"))}
, {"reuse_sessions", t(boolean(), M("reuse_sessions"), D("reuse_sessions"))}
, {"honor_cipher_order", t(boolean(), M("honor_cipher_order"), D("honor_cipher_order"))}
, {"handshake_timeout", t(duration(), M("handshake_timeout"), D("handshake_timeout"))}
, {"depth", t(integer(), M("depth"), D("depth"))}
, {"password", hoconsc:t(string(), #{mapping => M("key_password"),

3
apps/emqx_gateway/src/exproto/emqx_exproto_conn.erl
View File

@ -219,7 +219,8 @@ send(Data, #state{socket = {esockd_transport, Sock}}) ->
-define(DEFAULT_GC_OPTS, #{count => 1000, bytes => 1024*1024}).
-define(DEFAULT_IDLE_TIMEOUT, 30000).
-define(DEFAULT_OOM_POLICY, #{max_heap_size => 4194304,message_queue_len => 32000}).
-define(DEFAULT_OOM_POLICY, #{enable => true, max_heap_size => 4194304,
max_message_queue_len => 32000}).
init(Parent, WrappedSock, Peername, Options) ->
case esockd_wait(WrappedSock) of

3
apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl
View File

@ -440,7 +440,8 @@ take_place(Text, Placeholder, Value) ->
clientinfo(#lwm2m_state{peername = {PeerHost, _},
endpoint_name = EndpointName,
mountpoint = Mountpoint}) ->
#{zone => undefined,
#{zone => default,
listener => mqtt_tcp, %% FIXME: this won't work
protocol => lwm2m,
peerhost => PeerHost,
sockport => 5683, %% FIXME:

2
apps/emqx_gateway/src/mqttsn/emqx_sn_gateway.erl
View File

@ -104,7 +104,7 @@
-define(STAT_TIMEOUT, 10000).
-define(IDLE_TIMEOUT, 30000).
-define(DEFAULT_CHAN_OPTIONS, [{max_packet_size, 256}, {zone, external}]).
-define(DEFAULT_CHAN_OPTIONS, #{zone => default, listener => mqtt_tcp}).
-define(NEG_QOS_CLIENT_ID, <<"NegQoS-Client">>).

6
apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl
View File

@ -113,7 +113,8 @@ init(ConnInfo = #{peername := {PeerHost, _},
Mountpoint = maps:get(mountpoint, Option, undefined),
ClientInfo = setting_peercert_infos(
Peercert,
#{ zone => undefined
#{ zone => default
, listener => mqtt_tcp
, protocol => stomp
, peerhost => PeerHost
, sockport => SockPort
@ -583,7 +584,8 @@ handle_call(discard, Channel) ->
% shutdown_and_reply(takeovered, AllPendings, Channel);
handle_call(list_acl_cache, Channel) ->
{reply, emqx_acl_cache:list_acl_cache(), Channel};
%% This won't work
{reply, emqx_acl_cache:list_acl_cache(default, mqtt_tcp), Channel};
%% XXX: No Quota Now
% handle_call({quota, Policy}, Channel) ->

4
apps/emqx_gateway/src/stomp/emqx_stomp_connection.erl
View File

@ -245,7 +245,9 @@ init_state(Transport, Socket, Options) ->
peername => Peername,
sockname => Sockname,
peercert => Peercert,
conn_mod => ?MODULE
conn_mod => ?MODULE,
zone => default,
listener => mqtt_tcp
},
ActiveN = emqx_gateway_utils:active_n(Options),
%% TODO: RateLimit ? How ?

4
apps/emqx_management/etc/emqx_management.conf
View File

@ -14,7 +14,7 @@ emqx_management:{
port: 8081
backlog: 512
send_timeout: 15s
send_timeout_close: on
send_timeout_close: true
inet6: false
ipv6_v6only: false
}
@ -25,7 +25,7 @@ emqx_management:{
## acceptors: 2
## backlog: 512
## send_timeout: 15s
## send_timeout_close: on
## send_timeout_close: true
## inet6: false
## ipv6_v6only: false
## certfile = "etc/certs/cert.pem"

4
apps/emqx_management/src/emqx_management_schema.erl
View File

@ -43,13 +43,13 @@ fields("http") ->
, {"max_connections", emqx_schema:t(integer(), undefined, 512)}
, {"backlog", emqx_schema:t(integer(), undefined, 1024)}
, {"send_timeout", emqx_schema:t(emqx_schema:duration(), undefined, "15s")}
, {"send_timeout_close", emqx_schema:t(emqx_schema:flag(), undefined, true)}
, {"send_timeout_close", emqx_schema:t(boolean(), undefined, true)}
, {"inet6", emqx_schema:t(boolean(), undefined, false)}
, {"ipv6_v6only", emqx_schema:t(boolean(), undefined, false)}
];
fields("https") ->
emqx_schema:ssl(undefined, #{enable => true}) ++ fields("http").
emqx_schema:ssl(#{enable => true}) ++ fields("http").
max_row_limit(type) -> integer();
max_row_limit(default) -> 1000;

3
apps/emqx_management/src/emqx_mgmt.erl
View File

@ -457,7 +457,7 @@ list_listeners(Node) when Node =:= node() ->
Tcp = lists:map(fun({{Protocol, ListenOn}, _Pid}) ->
#{protocol => Protocol,
listen_on => ListenOn,
identifier => emqx_listeners:find_id_by_listen_on(ListenOn),
identifier => Protocol,
acceptors => esockd:get_acceptors({Protocol, ListenOn}),
max_conns => esockd:get_max_connections({Protocol, ListenOn}),
current_conns => esockd:get_current_connections({Protocol, ListenOn}),
@ -476,6 +476,7 @@ list_listeners(Node) when Node =:= node() ->
list_listeners(Node) ->
rpc_call(Node, list_listeners, [Node]).
-spec restart_listener(node(), atom()) -> ok | {error, term()}.
restart_listener(Node, Identifier) when Node =:= node() ->
emqx_listeners:restart_listener(Identifier);

9
apps/emqx_management/src/emqx_mgmt_api_listeners.erl
View File

@ -30,13 +30,13 @@
-rest_api(#{name => restart_listener,
method => 'PUT',
path => "/listeners/:bin:identifier/restart",
path => "/listeners/:atom:identifier/restart",
func => restart,
descr => "Restart a listener in the cluster"}).
-rest_api(#{name => restart_node_listener,
method => 'PUT',
path => "/nodes/:atom:node/listeners/:bin:identifier/restart",
path => "/nodes/:atom:node/listeners/:atom:identifier/restart",
func => restart,
descr => "Restart a listener on a node"}).
@ -57,10 +57,7 @@ restart(#{node := Node, identifier := Identifier}, _Params) ->
ok -> emqx_mgmt:return({ok, "Listener restarted."});
{error, Error} -> emqx_mgmt:return({error, Error})
end;
%% Restart listeners in the cluster.
restart(#{identifier := <<"http", _/binary>>}, _Params) ->
{403, <<"http_listener_restart_unsupported">>};
%% Restart listeners on all nodes in the cluster.
restart(#{identifier := Identifier}, _Params) ->
Results = [{Node, emqx_mgmt:restart_listener(Node, Identifier)} || {Node, _Info} <- emqx_mgmt:list_nodes()],
case lists:filter(fun({_, Result}) -> Result =/= ok end, Results) of

98
apps/emqx_management/src/emqx_mgmt_cli.erl
View File

@ -463,86 +463,57 @@ trace_off(Who, Name) ->
listeners([]) ->
lists:foreach(fun({{Protocol, ListenOn}, _Pid}) ->
Info = [{listen_on, {string, emqx_listeners:format_listen_on(ListenOn)}},
Info = [{listen_on, {string, format_listen_on(ListenOn)}},
{acceptors, esockd:get_acceptors({Protocol, ListenOn})},
{max_conns, esockd:get_max_connections({Protocol, ListenOn})},
{current_conn, esockd:get_current_connections({Protocol, ListenOn})},
{shutdown_count, esockd:get_shutdown_count({Protocol, ListenOn})}
],
emqx_ctl:print("~s~n", [listener_identifier(Protocol, ListenOn)]),
emqx_ctl:print("~s~n", [Protocol]),
lists:foreach(fun indent_print/1, Info)
end, esockd:listeners()),
lists:foreach(fun({Protocol, Opts}) ->
Port = proplists:get_value(port, Opts),
Info = [{listen_on, {string, emqx_listeners:format_listen_on(Port)}},
Info = [{listen_on, {string, format_listen_on(Port)}},
{acceptors, maps:get(num_acceptors, proplists:get_value(transport_options, Opts, #{}), 0)},
{max_conns, proplists:get_value(max_connections, Opts)},
{current_conn, proplists:get_value(all_connections, Opts)},
{shutdown_count, []}],
emqx_ctl:print("~s~n", [listener_identifier(Protocol, Port)]),
emqx_ctl:print("~s~n", [Protocol]),
lists:foreach(fun indent_print/1, Info)
end, ranch:info());
listeners(["stop", Name = "http" ++ _N | _MaybePort]) ->
%% _MaybePort is to be backward compatible, to stop http listener, there is no need for the port number
case minirest:stop(list_to_atom(Name)) of
listeners(["stop", ListenerId]) ->
case emqx_listeners:stop_listener(list_to_atom(ListenerId)) of
ok ->
emqx_ctl:print("Stop ~s listener successfully.~n", [Name]);
emqx_ctl:print("Stop ~s listener successfully.~n", [ListenerId]);
{error, Error} ->
emqx_ctl:print("Failed to stop ~s listener: ~0p~n", [Name, Error])
emqx_ctl:print("Failed to stop ~s listener: ~0p~n", [ListenerId, Error])
end;
listeners(["stop", "mqtt:" ++ _ = Identifier]) ->
stop_listener(emqx_listeners:find_by_id(Identifier), Identifier);
listeners(["stop", _Proto, ListenOn]) ->
%% this clause is kept to be backward compatible
ListenOn1 = case string:tokens(ListenOn, ":") of
[Port] -> list_to_integer(Port);
[IP, Port] -> {IP, list_to_integer(Port)}
end,
stop_listener(emqx_listeners:find_by_listen_on(ListenOn1), ListenOn1);
listeners(["restart", "http:management"]) ->
restart_http_listener(http, emqx_management);
listeners(["restart", "https:management"]) ->
restart_http_listener(https, emqx_management);
listeners(["restart", "http:dashboard"]) ->
restart_http_listener(http, emqx_dashboard);
listeners(["restart", "https:dashboard"]) ->
restart_http_listener(https, emqx_dashboard);
listeners(["restart", Identifier]) ->
case emqx_listeners:restart_listener(Identifier) of
listeners(["start", ListenerId]) ->
case emqx_listeners:start_listener(list_to_atom(ListenerId)) of
ok ->
emqx_ctl:print("Restarted ~s listener successfully.~n", [Identifier]);
emqx_ctl:print("Started ~s listener successfully.~n", [ListenerId]);
{error, Error} ->
emqx_ctl:print("Failed to restart ~s listener: ~0p~n", [Identifier, Error])
emqx_ctl:print("Failed to start ~s listener: ~0p~n", [ListenerId, Error])
end;
listeners(["restart", ListenerId]) ->
case emqx_listeners:restart_listener(list_to_atom(ListenerId)) of
ok ->
emqx_ctl:print("Restarted ~s listener successfully.~n", [ListenerId]);
{error, Error} ->
emqx_ctl:print("Failed to restart ~s listener: ~0p~n", [ListenerId, Error])
end;
listeners(_) ->
emqx_ctl:usage([{"listeners", "List listeners"},
{"listeners stop <Identifier>", "Stop a listener"},
{"listeners stop <Proto> <Port>", "Stop a listener"},
{"listeners start <Identifier>", "Start a listener"},
{"listeners restart <Identifier>", "Restart a listener"}
]).
stop_listener(false, Input) ->
emqx_ctl:print("No such listener ~p~n", [Input]);
stop_listener(#{listen_on := ListenOn} = Listener, _Input) ->
ID = emqx_listeners:identifier(Listener),
ListenOnStr = emqx_listeners:format_listen_on(ListenOn),
case emqx_listeners:stop_listener(Listener) of
ok ->
emqx_ctl:print("Stop ~s listener on ~s successfully.~n", [ID, ListenOnStr]);
{error, Reason} ->
emqx_ctl:print("Failed to stop ~s listener on ~s: ~0p~n",
[ID, ListenOnStr, Reason])
end.
%%--------------------------------------------------------------------
%% @doc acl Command
@ -661,24 +632,9 @@ indent_print({Key, {string, Val}}) ->
indent_print({Key, Val}) ->
emqx_ctl:print(" ~-16s: ~w~n", [Key, Val]).
listener_identifier(Protocol, ListenOn) ->
case emqx_listeners:find_id_by_listen_on(ListenOn) of
false ->
atom_to_list(Protocol);
ID ->
ID
end.
restart_http_listener(Scheme, AppName) ->
Listeners = application:get_env(AppName, listeners, []),
case lists:keyfind(Scheme, 1, Listeners) of
false ->
emqx_ctl:print("Listener ~s not exists!~n", [AppName]);
{Scheme, Port, Options} ->
ModName = http_mod_name(AppName),
ModName:stop_listener({Scheme, Port, Options}),
ModName:start_listener({Scheme, Port, Options})
end.
http_mod_name(emqx_management) -> emqx_mgmt_http;
http_mod_name(Name) -> Name.
format_listen_on(Port) when is_integer(Port) ->
io_lib:format("0.0.0.0:~w", [Port]);
format_listen_on({Addr, Port}) when is_list(Addr) ->
io_lib:format("~s:~w", [Addr, Port]);
format_listen_on({Addr, Port}) when is_tuple(Addr) ->
io_lib:format("~s:~w", [inet:ntoa(Addr), Port]).

18
apps/emqx_management/test/emqx_mgmt_api_test_util.erl
View File

@ -20,24 +20,6 @@
-define(SERVER, "http://127.0.0.1:8081").
-define(BASE_PATH, "/api/v5").
default_init() ->
ekka_mnesia:start(),
emqx_mgmt_auth:mnesia(boot),
emqx_ct_helpers:start_apps([emqx_management], fun set_special_configs/1),
ok.
default_end() ->
emqx_ct_helpers:stop_apps([emqx_management]).
set_special_configs(emqx_management) ->
emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}],
applications =>[#{id => "admin", secret => "public"}]}),
ok;
set_special_configs(_App) ->
ok.
request_api(Method, Url) ->
request_api(Method, Url, [], auth_header_(), []).

13
apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl
View File

@ -25,11 +25,20 @@ all() ->
emqx_ct:all(?MODULE).
init_per_suite(Config) ->
emqx_mgmt_api_test_util:default_init(),
ekka_mnesia:start(),
emqx_mgmt_auth:mnesia(boot),
emqx_ct_helpers:start_apps([emqx_management], fun set_special_configs/1),
Config.
end_per_suite(_) ->
emqx_mgmt_api_test_util:default_end().
emqx_ct_helpers:stop_apps([emqx_management]).
set_special_configs(emqx_management) ->
emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}],
applications =>[#{id => "admin", secret => "public"}]}),
ok;
set_special_configs(_App) ->
ok.
t_clients(_) ->
process_flag(trap_exit, true),

2
apps/emqx_retainer/src/emqx_retainer.erl
View File

@ -112,7 +112,7 @@ dispatch(Context, Pid, Topic, Cursor) ->
case Cursor =/= undefined orelse emqx_topic:wildcard(Topic) of
false ->
{ok, Result} = Mod:read_message(Context, Topic),
deliver(Result, Context, Pid, Topic, undefiend);
deliver(Result, Context, Pid, Topic, undefined);
true ->
{ok, Result, NewCursor} = Mod:match_messages(Context, Topic, Cursor),
deliver(Result, Context, Pid, Topic, NewCursor)

2
apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl
View File

@ -50,7 +50,7 @@ init_per_suite(Config) ->
end_per_suite(_Config) ->
delete_default_app(),
emqx_ct_helpers:stop_apps([emqx_retainer]).
emqx_ct_helpers:stop_apps([emqx_management, emqx_retainer]).
init_per_testcase(_, Config) ->
Config.

5
bin/emqx
View File

@ -205,6 +205,7 @@ relx_nodetool() {
call_hocon() {
export RUNNER_ROOT_DIR
export RUNNER_ETC_DIR
export REL_VSN
"$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" hocon "$@" \
|| die "ERROR: call_hocon failed: $*" $?
@ -592,7 +593,7 @@ case "$1" in
# set before generate_config
if [ "${_EMQX_START_MODE:-}" = '' ]; then
export EMQX_LOG__TO="${EMQX_LOG__TO:-console}"
export EMQX_LOG__CONSOLE_HANDLER__ENABLE="${EMQX_LOG__CONSOLE_HANDLER__ENABLE:-true}"
fi
#generate app.config and vm.args
@ -636,7 +637,7 @@ case "$1" in
# or other supervision services
# set before generate_config
export EMQX_LOG__TO="${EMQX_LOG__TO:-console}"
export EMQX_LOG__CONSOLE_HANDLER__ENABLE="${EMQX_LOG__CONSOLE_HANDLER__ENABLE:-true}"
#generate app.config and vm.args
generate_config

2
bin/emqx.cmd
View File

@ -20,8 +20,6 @@
@set erts_vsn={{ erts_vsn }}
@set erl_opts={{ erl_opts }}
@set "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_"
@set script=%~n0
:: Discover the release root directory from the directory

2
deploy/charts/emqx/templates/StatefulSet.yaml
View File

@ -116,7 +116,7 @@ spec:
value: {{ .Release.Name }}
- name: EMQX_CLUSTER__K8S__APP_NAME
value: {{ .Release.Name }}
- name: EMQX_CLUSTER__DISCOVERY
- name: EMQX_CLUSTER__DISCOVERY_STRATEGY
value: k8s
- name: EMQX_CLUSTER__K8S__SERVICE_NAME
value: {{ include "emqx.fullname" . }}-headless

14
deploy/docker/README.md
View File

@ -48,8 +48,8 @@ You can change the prefix by overriding "HOCON_ENV_OVERRIDE_PREFIX".
Example:
```bash
EMQX_LISTENER__SSL__EXTERNAL__ACCEPTORS <--> listener.ssl.external.acceptors
EMQX_MQTT__MAX_PACKET_SIZE <--> mqtt.max_packet_size
EMQX_ZONES__DEFAULT__LISTENERS__MQTT_SSL__ACCEPTORS <--> zones.default.listeners.mqtt_ssl.acceptors
EMQX_ZONES__DEFAULT__MQTT__MAX_PACKET_SIZE <--> zones.default.mqtt.max_packet_size
```
+ Prefix ``EMQX_`` is removed
@ -87,7 +87,7 @@ If set ``EMQX_NAME`` and ``EMQX_HOST``, and unset ``EMQX_NODE_NAME``, ``EMQX_NOD
For example, set mqtt tcp port to 1883
``docker run -d --name emqx -e EMQX_LISTENER__TCP__EXTERNAL=1883 -p 18083:18083 -p 1883:1883 emqx/emqx:latest``
``docker run -d --name emqx -e EMQX_ZONES__DEFAULT__LISTENERS__MQTT_TCP__BIND=1883 -p 18083:18083 -p 1883:1883 emqx/emqx:latest``
#### EMQ Loaded Modules Configuration
@ -213,8 +213,8 @@ Let's create a static node list cluster from docker-compose.
environment:
- "EMQX_NAME=emqx"
- "EMQX_HOST=node1.emqx.io"
- "EMQX_CLUSTER__DISCOVERY=static"
- "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io"
- "EMQX_CLUSTER__DISCOVERY_STRATEGY=static"
- "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io, emqx@node2.emqx.io]"
networks:
emqx-bridge:
aliases:
@ -225,8 +225,8 @@ Let's create a static node list cluster from docker-compose.
environment:
- "EMQX_NAME=emqx"
- "EMQX_HOST=node2.emqx.io"
- "EMQX_CLUSTER__DISCOVERY=static"
- "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io"
- "EMQX_CLUSTER__DISCOVERY_STRATEGY=static"
- "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io, emqx@node2.emqx.io]"
networks:
emqx-bridge:
aliases:

46
deploy/docker/docker-entrypoint.sh
View File

@ -42,52 +42,6 @@ if [[ -z "$EMQX_NODE_NAME" ]]; then
export EMQX_NODE_NAME="$EMQX_NAME@$EMQX_HOST"
fi
# Set hosts to prevent cluster mode failed
if [[ -z "$EMQX_NODE__PROCESS_LIMIT" ]]; then
export EMQX_NODE__PROCESS_LIMIT=2097152
fi
if [[ -z "$EMQX_NODE__MAX_PORTS" ]]; then
export EMQX_NODE__MAX_PORTS=1048576
fi
if [[ -z "$EMQX_NODE__MAX_ETS_TABLES" ]]; then
export EMQX_NODE__MAX_ETS_TABLES=2097152
fi
if [[ -z "$EMQX_LISTENER__TCP__EXTERNAL__ACCEPTORS" ]]; then
export EMQX_LISTENER__TCP__EXTERNAL__ACCEPTORS=64
fi
if [[ -z "$EMQX_LISTENER__TCP__EXTERNAL__MAX_CONNECTIONS" ]]; then
export EMQX_LISTENER__TCP__EXTERNAL__MAX_CONNECTIONS=1024000
fi
if [[ -z "$EMQX_LISTENER__SSL__EXTERNAL__ACCEPTORS" ]]; then
export EMQX_LISTENER__SSL__EXTERNAL__ACCEPTORS=32
fi
if [[ -z "$EMQX_LISTENER__SSL__EXTERNAL__MAX_CONNECTIONS" ]]; then
export EMQX_LISTENER__SSL__EXTERNAL__MAX_CONNECTIONS=102400
fi
if [[ -z "$EMQX_LISTENER__WS__EXTERNAL__ACCEPTORS" ]]; then
export EMQX_LISTENER__WS__EXTERNAL__ACCEPTORS=16
fi
if [[ -z "$EMQX_LISTENER__WS__EXTERNAL__MAX_CONNECTIONS" ]]; then
export EMQX_LISTENER__WS__EXTERNAL__MAX_CONNECTIONS=102400
fi
if [[ -z "$EMQX_LISTENER__WSS__EXTERNAL__ACCEPTORS" ]]; then
export EMQX_LISTENER__WSS__EXTERNAL__ACCEPTORS=16
fi
if [[ -z "$EMQX_LISTENER__WSS__EXTERNAL__MAX_CONNECTIONS" ]]; then
export EMQX_LISTENER__WSS__EXTERNAL__MAX_CONNECTIONS=102400
fi
# fill tuples on specific file
# SYNOPSIS
# fill_tuples FILE [ELEMENTS ...]

2
rebar.config
View File

@ -47,7 +47,7 @@
, {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
, {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}}
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}}
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.3"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
, {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.1.2"}}}