yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into build-with-mix-mkII

This commit is contained in:
Thales Macedo Garitezi 2021-12-28 11:28:40 -03:00
parent cea54365dd ecbb28fedf
commit 0020cf592f
No known key found for this signature in database
GPG Key ID: DD279F8152A9B6DD
75 changed files with 2431 additions and 1016 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3
export EMQX_DEFAULT_RUNNER = alpine:3.14
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
export EMQX_DASHBOARD_VERSION ?= v0.8.0
export EMQX_DASHBOARD_VERSION ?= v0.10.0
export DOCKERFILE := deploy/docker/Dockerfile
export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing
ifeq ($(OS),Windows_NT)

2
apps/emqx/rebar.config
View File

@ -15,7 +15,7 @@
, {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.2"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}}
, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}

4
apps/emqx/src/emqx_channel.erl
View File

@ -292,7 +292,7 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) ->
fun check_banned/2
], ConnPkt, Channel#channel{conn_state = connecting}) of
{ok, NConnPkt, NChannel = #channel{clientinfo = ClientInfo}} ->
?LOG(debug, "RECV ~s", [emqx_packet:format(Packet)]),
?SLOG(debug, #{msg => "recv_packet", packet => emqx_packet:format(Packet)}),
NChannel1 = NChannel#channel{
will_msg = emqx_packet:will_msg(NConnPkt),
alias_maximum = init_alias_maximum(NConnPkt, ClientInfo)
@ -637,7 +637,7 @@ do_publish(PacketId, Msg = #message{qos = ?QOS_2},
packet_id => PacketId
}),
ok = emqx_metrics:inc('packets.publish.dropped'),
handle_out(pubrec, {PacketId, RC}, Channel)
handle_out(disconnect, RC, Channel)
end.
ensure_quota(_, Channel = #channel{quota = undefined}) ->

6
apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl
View File

@ -138,7 +138,7 @@ init([]) ->
{stop, Reason :: term(), Reply :: term(), NewState :: term()} |
{stop, Reason :: term(), NewState :: term()}.
handle_call(Req, _From, State) ->
?LOG(error, "Unexpected call: ~p", [Req]),
?SLOG(error, #{msg => "unexpected_call", call => Req}),
{reply, ignore, State}.
%%--------------------------------------------------------------------
@ -153,7 +153,7 @@ handle_call(Req, _From, State) ->
{noreply, NewState :: term(), hibernate} |
{stop, Reason :: term(), NewState :: term()}.
handle_cast(Req, State) ->
?LOG(error, "Unexpected cast: ~p", [Req]),
?SLOG(error, #{msg => "unexpected_cast", cast => Req}),
{noreply, State}.
%%--------------------------------------------------------------------
@ -168,7 +168,7 @@ handle_cast(Req, State) ->
{noreply, NewState :: term(), hibernate} |
{stop, Reason :: normal | term(), NewState :: term()}.
handle_info(Info, State) ->
?LOG(error, "Unexpected info: ~p", [Info]),
?SLOG(error, #{msg => "unexpected_info", info => Info}),
{noreply, State}.
%%--------------------------------------------------------------------

10
apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl
View File

@ -98,7 +98,7 @@ connect(Type, BucketName) when is_atom(BucketName) ->
Path = [emqx_limiter, Type, bucket, BucketName],
case emqx:get_config(Path, undefined) of
undefined ->
?LOG(error, "can't find the config of this bucket: ~p~n", [Path]),
?SLOG(error, #{msg => "bucket_config_not_found", path => Path}),
throw("bucket's config not found");
#{zone := Zone,
aggregated := #{rate := AggrRate, capacity := AggrSize},
@ -113,7 +113,7 @@ connect(Type, BucketName) when is_atom(BucketName) ->
emqx_htb_limiter:make_ref_limiter(Cfg, Bucket)
end;
undefined ->
?LOG(error, "can't find the bucket:~p~n", [Path]),
?SLOG(error, #{msg => "bucket_not_found", path => Path}),
throw("invalid bucket")
end
end;
@ -182,7 +182,7 @@ init([Type]) ->
{stop, Reason :: term(), Reply :: term(), NewState :: term()} |
{stop, Reason :: term(), NewState :: term()}.
handle_call(Req, _From, State) ->
?LOG(error, "Unexpected call: ~p", [Req]),
?SLOG(error, #{msg => "unexpected_call", call => Req}),
{reply, ignored, State}.
%%--------------------------------------------------------------------
@ -197,7 +197,7 @@ handle_call(Req, _From, State) ->
{noreply, NewState :: term(), hibernate} |
{stop, Reason :: term(), NewState :: term()}.
handle_cast(Req, State) ->
?LOG(error, "Unexpected cast: ~p", [Req]),
?SLOG(error, #{msg => "unexpected_cast", cast => Req}),
{noreply, State}.
%%--------------------------------------------------------------------
@ -215,7 +215,7 @@ handle_info(oscillate, State) ->
{noreply, oscillation(State)};
handle_info(Info, State) ->
?LOG(error, "Unexpected info: ~p", [Info]),
?SLOG(error, #{msg => "unexpected_info", info => Info}),
{noreply, State}.
%%--------------------------------------------------------------------

1
apps/emqx/src/emqx_trace/emqx_trace.erl
View File

@ -194,6 +194,7 @@ format(Traces) ->
end, Traces).
init([]) ->
ok = mria:wait_for_tables([?TRACE]),
erlang:process_flag(trap_exit, true),
OriginLogLevel = emqx_logger:get_primary_log_level(),
ok = filelib:ensure_dir(trace_dir()),

8
apps/emqx/src/emqx_trace/emqx_trace_handler.erl
View File

@ -103,7 +103,7 @@ uninstall(Type, Name) ->
-spec uninstall(HandlerId :: atom()) -> ok | {error, term()}.
uninstall(HandlerId) ->
Res = logger:remove_handler(HandlerId),
show_prompts(Res, HandlerId, "Stop trace"),
show_prompts(Res, HandlerId, "stop_trace"),
Res.
%% @doc Return all running trace handlers information.
@ -151,7 +151,7 @@ install_handler(Who = #{name := Name, type := Type}, Level, LogFile) ->
config => ?CONFIG(LogFile)
},
Res = logger:add_handler(HandlerId, logger_disk_log_h, Config),
show_prompts(Res, Who, "Start trace"),
show_prompts(Res, Who, "start_trace"),
Res.
filters(#{type := clientid, filter := Filter, name := Name}) ->
@ -223,6 +223,6 @@ ensure_list(Bin) when is_binary(Bin) -> unicode:characters_to_list(Bin, utf8);
ensure_list(List) when is_list(List) -> List.
show_prompts(ok, Who, Msg) ->
?LOG(info, Msg ++ " ~p " ++ "successfully~n", [Who]);
?SLOG(info, #{msg => "trace_action_succeeded", action => Msg, traced => Who});
show_prompts({error, Reason}, Who, Msg) ->
?LOG(error, Msg ++ " ~p " ++ "failed with ~p~n", [Who, Reason]).
?SLOG(info, #{msg => "trace_action_failed", action => Msg, traced => Who, reason => Reason}).

3
apps/emqx/test/emqx_channel_SUITE.erl
View File

@ -370,7 +370,8 @@ t_handle_in_qos2_publish_with_error_return(_) ->
{ok, ?PUBREC_PACKET(2, ?RC_NO_MATCHING_SUBSCRIBERS), Channel1} =
emqx_channel:handle_in(Publish2, Channel),
Publish3 = ?PUBLISH_PACKET(?QOS_2, <<"topic">>, 3, <<"payload">>),
{ok, ?PUBREC_PACKET(3, ?RC_RECEIVE_MAXIMUM_EXCEEDED), Channel1} =
{ok, [{outgoing, ?DISCONNECT_PACKET(?RC_RECEIVE_MAXIMUM_EXCEEDED)},
{close, receive_maximum_exceeded}], Channel1} =
emqx_channel:handle_in(Publish3, Channel1).
t_handle_in_puback_ok(_) ->

6
apps/emqx_authz/src/emqx_authz.erl
View File

@ -59,6 +59,8 @@
-define(METRICS, [?METRIC_ALLOW, ?METRIC_DENY, ?METRIC_NOMATCH]).
-define(IS_ENABLED(Enable), ((Enable =:= true) or (Enable =:= <<"true">>))).
%% Initialize authz backend.
%% Populate the passed configuration map with necessary data,
%% like `ResourceID`s
@ -155,8 +157,8 @@ do_update({?CMD_APPEND, Sources}, Conf) when is_list(Sources), is_list(Conf) ->
NConf = Conf ++ Sources,
ok = check_dup_types(NConf),
NConf;
do_update({{?CMD_REPLACE, Type}, #{<<"enable">> := true} = Source}, Conf) when is_map(Source),
is_list(Conf) ->
do_update({{?CMD_REPLACE, Type}, #{<<"enable">> := Enable} = Source}, Conf)
when is_map(Source), is_list(Conf), ?IS_ENABLED(Enable) ->
case create_dry_run(Type, Source) of
ok ->
{_Old, Front, Rear} = take(Type, Conf),

6
apps/emqx_authz/src/emqx_authz_file.erl
View File

@ -55,7 +55,11 @@ init(#{path := Path} = Source) ->
destroy(_Source) -> ok.
dry_run(_Source) -> ok.
dry_run(#{path := Path}) ->
case file:consult(Path) of
{ok, _} -> ok;
{error, _} = Error -> Error
end.
authorize(Client, PubSub, Topic, #{annotations := #{rules := Rules}}) ->
emqx_authz_rule:matches(Client, PubSub, Topic, Rules).

137
apps/emqx_authz/src/emqx_authz_http.erl
View File

@ -40,9 +40,8 @@
description() ->
"AuthZ with http".
init(#{url := Url} = Source) ->
NSource = maps:put(base_url, maps:remove(query, Url), Source),
case emqx_authz_utils:create_resource(emqx_connector_http, NSource) of
init(Source) ->
case emqx_authz_utils:create_resource(emqx_connector_http, Source) of
{error, Reason} -> error({load_config_error, Reason});
{ok, Id} -> Source#{annotations => #{id => Id}}
end.
@ -51,13 +50,12 @@ destroy(#{annotations := #{id := Id}}) ->
ok = emqx_resource:remove(Id).
dry_run(Source) ->
URIMap = maps:get(url, Source),
NSource = maps:put(base_url, maps:remove(query, URIMap), Source),
emqx_resource:create_dry_run(emqx_connector_http, NSource).
emqx_resource:create_dry_run(emqx_connector_http, Source).
authorize(Client, PubSub, Topic,
#{type := http,
url := #{path := Path} = URL,
query := Query,
path := Path,
headers := Headers,
method := Method,
request_timeout := RequestTimeout,
@ -65,25 +63,47 @@ authorize(Client, PubSub, Topic,
} = Source) ->
Request = case Method of
get ->
Query = maps:get(query, URL, ""),
Path1 = replvar(Path ++ "?" ++ Query, PubSub, Topic, Client),
Path1 = replvar(
Path ++ "?" ++ Query,
PubSub,
Topic,
maps:to_list(Client),
fun var_uri_encode/1),
{Path1, maps:to_list(Headers)};
_ ->
Body0 = serialize_body(
maps:get('Accept', Headers, <<"application/json">>),
maps:get(body, Source, #{})
),
Body1 = replvar(Body0, PubSub, Topic, Client),
Path1 = replvar(Path, PubSub, Topic, Client),
{Path1, maps:to_list(Headers), Body1}
Body0 = maps:get(body, Source, #{}),
Body1 = replvar_deep(
Body0,
PubSub,
Topic,
maps:to_list(Client),
fun var_bin_encode/1),
Body2 = serialize_body(
maps:get(<<"content-type">>, Headers, <<"application/json">>),
Body1),
Path1 = replvar(
Path,
PubSub,
Topic,
maps:to_list(Client),
fun var_uri_encode/1),
{Path1, maps:to_list(Headers), Body2}
end,
case emqx_resource:query(ResourceID, {Method, Request, RequestTimeout}) of
HttpResult = emqx_resource:query(ResourceID, {Method, Request, RequestTimeout}),
case HttpResult of
{ok, 200, _Headers} ->
{matched, allow};
{ok, 204, _Headers} ->
{matched, allow};
{ok, 200, _Headers, _Body} ->
{matched, allow};
{ok, _Status, _Headers} ->
nomatch;
{ok, _Status, _Headers, _Body} ->
nomatch;
{error, Reason} ->
@ -121,30 +141,67 @@ serialize_body(<<"application/json">>, Body) ->
serialize_body(<<"application/x-www-form-urlencoded">>, Body) ->
query_string(Body).
replvar(Str0, PubSub, Topic,
#{username := Username,
clientid := Clientid,
peerhost := IpAddress,
protocol := Protocol,
mountpoint := Mountpoint
}) when is_list(Str0);
is_binary(Str0) ->
replvar_deep(Map, PubSub, Topic, Vars, VarEncode) when is_map(Map) ->
maps:from_list(
lists:map(
fun({Key, Value}) ->
{replvar(Key, PubSub, Topic, Vars, VarEncode),
replvar(Value, PubSub, Topic, Vars, VarEncode)}
end,
maps:to_list(Map)));
replvar_deep(List, PubSub, Topic, Vars, VarEncode) when is_list(List) ->
lists:map(
fun(Value) ->
replvar(Value, PubSub, Topic, Vars, VarEncode)
end,
List);
replvar_deep(Number, _PubSub, _Topic, _Vars, _VarEncode) when is_number(Number) ->
Number;
replvar_deep(Binary, PubSub, Topic, Vars, VarEncode) when is_binary(Binary) ->
replvar(Binary, PubSub, Topic, Vars, VarEncode).
replvar(Str0, PubSub, Topic, [], VarEncode) ->
NTopic = emqx_http_lib:uri_encode(Topic),
Str1 = re:replace( Str0, emqx_authz:ph_to_re(?PH_S_CLIENTID)
, bin(Clientid), [global, {return, binary}]),
Str2 = re:replace( Str1, emqx_authz:ph_to_re(?PH_S_USERNAME)
, bin(Username), [global, {return, binary}]),
Str3 = re:replace( Str2, emqx_authz:ph_to_re(?PH_S_HOST)
, inet_parse:ntoa(IpAddress), [global, {return, binary}]),
Str4 = re:replace( Str3, emqx_authz:ph_to_re(?PH_S_PROTONAME)
, bin(Protocol), [global, {return, binary}]),
Str5 = re:replace( Str4, emqx_authz:ph_to_re(?PH_S_MOUNTPOINT)
, bin(Mountpoint), [global, {return, binary}]),
Str6 = re:replace( Str5, emqx_authz:ph_to_re(?PH_S_TOPIC)
, bin(NTopic), [global, {return, binary}]),
Str7 = re:replace( Str6, emqx_authz:ph_to_re(?PH_S_ACTION)
, bin(PubSub), [global, {return, binary}]),
Str7.
Str1 = re:replace(Str0, emqx_authz:ph_to_re(?PH_S_TOPIC),
VarEncode(NTopic), [global, {return, binary}]),
re:replace(Str1, emqx_authz:ph_to_re(?PH_S_ACTION),
VarEncode(PubSub), [global, {return, binary}]);
replvar(Str, PubSub, Topic, [{username, Username} | Rest], VarEncode) ->
Str1 = re:replace(Str, emqx_authz:ph_to_re(?PH_S_USERNAME),
VarEncode(Username), [global, {return, binary}]),
replvar(Str1, PubSub, Topic, Rest, VarEncode);
replvar(Str, PubSub, Topic, [{clientid, Clientid} | Rest], VarEncode) ->
Str1 = re:replace(Str, emqx_authz:ph_to_re(?PH_S_CLIENTID),
VarEncode(Clientid), [global, {return, binary}]),
replvar(Str1, PubSub, Topic, Rest, VarEncode);
replvar(Str, PubSub, Topic, [{peerhost, IpAddress} | Rest], VarEncode) ->
Str1 = re:replace(Str, emqx_authz:ph_to_re(?PH_S_PEERHOST),
VarEncode(inet_parse:ntoa(IpAddress)), [global, {return, binary}]),
replvar(Str1, PubSub, Topic, Rest, VarEncode);
replvar(Str, PubSub, Topic, [{protocol, Protocol} | Rest], VarEncode) ->
Str1 = re:replace(Str, emqx_authz:ph_to_re(?PH_S_PROTONAME),
VarEncode(Protocol), [global, {return, binary}]),
replvar(Str1, PubSub, Topic, Rest, VarEncode);
replvar(Str, PubSub, Topic, [{mountpoint, Mountpoint} | Rest], VarEncode) ->
Str1 = re:replace(Str, emqx_authz:ph_to_re(?PH_S_MOUNTPOINT),
VarEncode(Mountpoint), [global, {return, binary}]),
replvar(Str1, PubSub, Topic, Rest, VarEncode);
replvar(Str, PubSub, Topic, [_Unknown | Rest], VarEncode) ->
replvar(Str, PubSub, Topic, Rest, VarEncode).
var_uri_encode(S) ->
emqx_http_lib:uri_encode(bin(S)).
var_bin_encode(S) ->
bin(S).
bin(A) when is_atom(A) -> atom_to_binary(A, utf8);
bin(B) when is_binary(B) -> B;

35
apps/emqx_authz/src/emqx_authz_mnesia.erl
View File

@ -114,18 +114,19 @@ authorize(#{username := Username,
%% Management API
%%--------------------------------------------------------------------
-spec(init_tables() -> ok).
init_tables() ->
ok = mria_rlog:wait_for_shards([?ACL_SHARDED], infinity).
-spec(store_rules(who(), rules()) -> ok).
store_rules({username, Username}, Rules) ->
Record = #emqx_acl{who = {?ACL_TABLE_USERNAME, Username}, rules = Rules},
Record = #emqx_acl{who = {?ACL_TABLE_USERNAME, Username}, rules = normalize_rules(Rules)},
mria:dirty_write(Record);
store_rules({clientid, Clientid}, Rules) ->
Record = #emqx_acl{who = {?ACL_TABLE_CLIENTID, Clientid}, rules = Rules},
Record = #emqx_acl{who = {?ACL_TABLE_CLIENTID, Clientid}, rules = normalize_rules(Rules)},
mria:dirty_write(Record);
store_rules(all, Rules) ->
Record = #emqx_acl{who = ?ACL_TABLE_ALL, rules = Rules},
Record = #emqx_acl{who = ?ACL_TABLE_ALL, rules = normalize_rules(Rules)},
mria:dirty_write(Record).
-spec(purge_rules() -> ok).
@ -176,6 +177,29 @@ record_count() ->
%% Internal functions
%%--------------------------------------------------------------------
normalize_rules(Rules) ->
lists:map(fun normalize_rule/1, Rules).
normalize_rule({Permission, Action, Topic}) ->
{normalize_permission(Permission),
normalize_action(Action),
normalize_topic(Topic)};
normalize_rule(Rule) ->
error({invalid_rule, Rule}).
normalize_topic(Topic) when is_list(Topic) -> list_to_binary(Topic);
normalize_topic(Topic) when is_binary(Topic) -> Topic;
normalize_topic(Topic) -> error({invalid_rule_topic, Topic}).
normalize_action(publish) -> publish;
normalize_action(subscribe) -> subscribe;
normalize_action(all) -> all;
normalize_action(Action) -> error({invalid_rule_action, Action}).
normalize_permission(allow) -> allow;
normalize_permission(deny) -> deny;
normalize_permission(Permission) -> error({invalid_rule_permission, Permission}).
do_get_rules(Key) ->
case mnesia:dirty_read(?ACL_TABLE, Key) of
[#emqx_acl{rules = Rules}] -> {ok, Rules};
@ -184,9 +208,8 @@ do_get_rules(Key) ->
do_authorize(_Client, _PubSub, _Topic, []) -> nomatch;
do_authorize(Client, PubSub, Topic, [ {Permission, Action, TopicFilter} | Tail]) ->
case emqx_authz_rule:match(Client, PubSub, Topic,
emqx_authz_rule:compile({Permission, all, Action, [TopicFilter]})
) of
Rule = emqx_authz_rule:compile({Permission, all, Action, [TopicFilter]}),
case emqx_authz_rule:match(Client, PubSub, Topic, Rule) of
{matched, Permission} -> {matched, Permission};
nomatch -> do_authorize(Client, PubSub, Topic, Tail)
end.

11
apps/emqx_authz/src/emqx_authz_schema.erl
View File

@ -20,14 +20,10 @@
-reflect_type([ permission/0
, action/0
, url/0
]).
-typerefl_from_string({url/0, emqx_http_lib, uri_parse}).
-type action() :: publish | subscribe | all.
-type permission() :: allow | deny.
-type url() :: emqx_http_lib:uri_map().
-export([ namespace/0
, roots/0
@ -143,10 +139,11 @@ fields(redis_cluster) ->
http_common_fields() ->
[ {type, #{type => http}}
, {enable, #{type => boolean(), default => true}}
, {url, #{type => url()}}
, {request_timeout, mk_duration("request timeout", #{default => "30s"})}
, {body, #{type => map(), nullable => true}}
] ++ proplists:delete(base_url, emqx_connector_http:fields(config)).
, {path, #{type => string(), default => ""}}
, {query, #{type => string(), default => ""}}
] ++ emqx_connector_http:fields(config).
mongo_common_fields() ->
[ {collection, #{type => atom()}}
@ -203,7 +200,7 @@ check_ssl_opts(Conf)
when Conf =:= #{} ->
true;
check_ssl_opts(Conf) ->
case emqx_authz_http:parse_url(hocon_schema:get_value("config.url", Conf)) of
case emqx_authz_http:parse_url(hocon_schema:get_value("config.base_url", Conf)) of
#{scheme := https} ->
case hocon_schema:get_value("config.ssl.enable", Conf) of
true -> ok;

6
apps/emqx_authz/test/emqx_authz_SUITE.erl
View File

@ -65,7 +65,9 @@ set_special_configs(_App) ->
-define(SOURCE1, #{<<"type">> => <<"http">>,
<<"enable">> => true,
<<"url">> => <<"https://fake.com:443/">>,
<<"base_url">> => <<"https://example.com:443/">>,
<<"path">> => <<"a/b">>,
<<"query">> => <<"c=d">>,
<<"headers">> => #{},
<<"method">> => <<"get">>,
<<"request_timeout">> => 5000
@ -77,7 +79,7 @@ set_special_configs(_App) ->
<<"pool_size">> => 1,
<<"database">> => <<"mqtt">>,
<<"ssl">> => #{<<"enable">> => false},
<<"collection">> => <<"fake">>,
<<"collection">> => <<"authz">>,
<<"selector">> => #{<<"a">> => <<"b">>}
}).
-define(SOURCE3, #{<<"type">> => <<"mysql">>,

130
apps/emqx_authz/test/emqx_authz_file_SUITE.erl Normal file
View File

@ -0,0 +1,130 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_authz_file_SUITE).
-compile(nowarn_export_all).
-compile(export_all).
-include("emqx_authz.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
all() ->
emqx_common_test_helpers:all(?MODULE).
groups() ->
[].
init_per_suite(Config) ->
ok = emqx_common_test_helpers:start_apps(
[emqx_conf, emqx_authz],
fun set_special_configs/1),
Config.
end_per_suite(_Config) ->
ok = emqx_authz_test_lib:restore_authorizers(),
ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
init_per_testcase(_TestCase, Config) ->
ok = emqx_authz_test_lib:reset_authorizers(),
Config.
set_special_configs(emqx_authz) ->
ok = emqx_authz_test_lib:reset_authorizers();
set_special_configs(_) ->
ok.
%%------------------------------------------------------------------------------
%% Testcases
%%------------------------------------------------------------------------------
t_ok(_Config) ->
ClientInfo = #{clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127,0,0,1},
zone => default,
listener => {tcp, default}
},
ok = setup_rules([{allow, {user, "username"}, publish, ["t"]}]),
ok = setup_config(#{}),
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
?assertEqual(
deny,
emqx_access_control:authorize(ClientInfo, subscribe, <<"t">>)).
t_invalid_file(_Config) ->
ok = file:write_file(<<"acl.conf">>, <<"{{invalid term">>),
?assertMatch(
{error, {1, erl_parse, _}},
emqx_authz:update(?CMD_REPLACE, [raw_file_authz_config()])).
t_nonexistent_file(_Config) ->
?assertEqual(
{error, enoent},
emqx_authz:update(?CMD_REPLACE,
[maps:merge(raw_file_authz_config(),
#{<<"path">> => <<"nonexistent.conf">>})
])).
t_update(_Config) ->
ok = setup_rules([{allow, {user, "username"}, publish, ["t"]}]),
ok = setup_config(#{}),
?assertMatch(
{error, _},
emqx_authz:update(
{?CMD_REPLACE, file},
maps:merge(raw_file_authz_config(),
#{<<"path">> => <<"nonexistent.conf">>}))),
?assertMatch(
{ok, _},
emqx_authz:update(
{?CMD_REPLACE, file},
raw_file_authz_config())).
%%------------------------------------------------------------------------------
%% Helpers
%%------------------------------------------------------------------------------
raw_file_authz_config() ->
#{
<<"enable">> => <<"true">>,
<<"type">> => <<"file">>,
<<"path">> => <<"acl.conf">>
}.
setup_rules(Rules) ->
{ok, F} = file:open(<<"acl.conf">>, [write]),
lists:foreach(
fun(Rule) ->
io:format(F, "~p.~n", [Rule])
end,
Rules),
ok = file:close(F).
setup_config(SpecialParams) ->
emqx_authz_test_lib:setup_config(
raw_file_authz_config(),
SpecialParams).

411
apps/emqx_authz/test/emqx_authz_http_SUITE.erl
View File

@ -4,6 +4,7 @@
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
@ -22,75 +23,389 @@
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-define(HTTP_PORT, 33333).
-define(HTTP_PATH, "/authz/[...]").
all() ->
emqx_common_test_helpers:all(?MODULE).
groups() ->
[].
init_per_suite(Config) ->
meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end),
meck:expect(emqx_resource, remove, fun(_) -> ok end ),
ok = emqx_common_test_helpers:start_apps(
[emqx_conf, emqx_authz],
fun set_special_configs/1),
Rules = [#{<<"type">> => <<"http">>,
<<"url">> => <<"https://fake.com:443/">>,
<<"headers">> => #{},
<<"method">> => <<"get">>,
<<"request_timeout">> => 5000
}
],
{ok, _} = emqx_authz:update(replace, Rules),
fun set_special_configs/1
),
ok = start_apps([emqx_resource, emqx_connector, cowboy]),
Config.
end_per_suite(_Config) ->
{ok, _} = emqx:update_config(
[authorization],
#{<<"no_match">> => <<"allow">>,
<<"cache">> => #{<<"enable">> => <<"true">>},
<<"sources">> => []}),
emqx_common_test_helpers:stop_apps([emqx_authz, emqx_conf]),
meck:unload(emqx_resource),
ok.
ok = emqx_authz_test_lib:restore_authorizers(),
ok = stop_apps([emqx_resource, emqx_connector, cowboy]),
ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
set_special_configs(emqx_authz) ->
{ok, _} = emqx:update_config([authorization, cache, enable], false),
{ok, _} = emqx:update_config([authorization, no_match], deny),
{ok, _} = emqx:update_config([authorization, sources], []),
ok;
set_special_configs(_App) ->
ok = emqx_authz_test_lib:reset_authorizers();
set_special_configs(_) ->
ok.
init_per_testcase(_Case, Config) ->
ok = emqx_authz_test_lib:reset_authorizers(),
{ok, _} = emqx_authz_http_test_server:start_link(?HTTP_PORT, ?HTTP_PATH),
Config.
end_per_testcase(_Case, _Config) ->
ok = emqx_authz_http_test_server:stop().
%%------------------------------------------------------------------------------
%% Testcases
%% Tests
%%------------------------------------------------------------------------------
t_authz(_) ->
ClientInfo = #{clientid => <<"my-clientid">>,
username => <<"my-username">>,
t_response_handling(_Config) ->
ClientInfo = #{clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127,0,0,1},
protocol => mqtt,
mountpoint => <<"fake">>,
zone => default,
listener => {tcp, default}
},
meck:expect(emqx_resource, query, fun(_, _) -> {ok, 204, fake_headers} end),
?assertEqual(allow,
emqx_access_control:authorize(ClientInfo, subscribe, <<"#">>)),
%% OK, get, no body
ok = setup_handler_and_config(
fun(Req0, State) ->
Req = cowboy_req:reply(200, Req0),
{ok, Req, State}
end,
#{}),
meck:expect(emqx_resource, query, fun(_, _) -> {ok, 200, fake_headers, fake_body} end),
?assertEqual(allow,
emqx_access_control:authorize(ClientInfo, publish, <<"#">>)),
allow = emqx_access_control:authorize(ClientInfo, publish, <<"t">>),
%% OK, get, body & headers
ok = setup_handler_and_config(
fun(Req0, State) ->
Req = cowboy_req:reply(
200,
#{<<"content-type">> => <<"text/plain">>},
"Response body",
Req0),
{ok, Req, State}
end,
#{}),
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
%% OK, get, 204
ok = setup_handler_and_config(
fun(Req0, State) ->
Req = cowboy_req:reply(204, Req0),
{ok, Req, State}
end,
#{}),
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
%% Not OK, get, 400
ok = setup_handler_and_config(
fun(Req0, State) ->
Req = cowboy_req:reply(400, Req0),
{ok, Req, State}
end,
#{}),
?assertEqual(
deny,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
%% Not OK, get, 400 + body & headers
ok = setup_handler_and_config(
fun(Req0, State) ->
Req = cowboy_req:reply(
400,
#{<<"content-type">> => <<"text/plain">>},
"Response body",
Req0),
{ok, Req, State}
end,
#{}),
?assertEqual(
deny,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)).
t_query_params(_Config) ->
ok = setup_handler_and_config(
fun(Req0, State) ->
#{username := <<"user name">>,
clientid := <<"client id">>,
peerhost := <<"127.0.0.1">>,
proto_name := <<"MQTT">>,
mountpoint := <<"MOUNTPOINT">>,
topic := <<"t">>,
action := <<"publish">>
} = cowboy_req:match_qs(
[username,
clientid,
peerhost,
proto_name,
mountpoint,
topic,
action],
Req0),
Req = cowboy_req:reply(200, Req0),
{ok, Req, State}
end,
#{<<"query">> => <<"username=${username}&"
"clientid=${clientid}&"
"peerhost=${peerhost}&"
"proto_name=${proto_name}&"
"mountpoint=${mountpoint}&"
"topic=${topic}&"
"action=${action}">>
}),
ClientInfo = #{clientid => <<"client id">>,
username => <<"user name">>,
peerhost => {127,0,0,1},
protocol => <<"MQTT">>,
mountpoint => <<"MOUNTPOINT">>,
zone => default,
listener => {tcp, default}
},
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)).
t_path_params(_Config) ->
ok = setup_handler_and_config(
fun(Req0, State) ->
<<"/authz/"
"username/user%20name/"
"clientid/client%20id/"
"peerhost/127.0.0.1/"
"proto_name/MQTT/"
"mountpoint/MOUNTPOINT/"
"topic/t/"
"action/publish">> = cowboy_req:path(Req0),
Req = cowboy_req:reply(200, Req0),
{ok, Req, State}
end,
#{<<"path">> => <<"username/${username}/"
"clientid/${clientid}/"
"peerhost/${peerhost}/"
"proto_name/${proto_name}/"
"mountpoint/${mountpoint}/"
"topic/${topic}/"
"action/${action}">>
}),
ClientInfo = #{clientid => <<"client id">>,
username => <<"user name">>,
peerhost => {127,0,0,1},
protocol => <<"MQTT">>,
mountpoint => <<"MOUNTPOINT">>,
zone => default,
listener => {tcp, default}
},
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)).
t_json_body(_Config) ->
ok = setup_handler_and_config(
fun(Req0, State) ->
?assertEqual(
<<"/authz/"
"username/user%20name/"
"clientid/client%20id/"
"peerhost/127.0.0.1/"
"proto_name/MQTT/"
"mountpoint/MOUNTPOINT/"
"topic/t/"
"action/publish">>,
cowboy_req:path(Req0)),
{ok, RawBody, Req1} = cowboy_req:read_body(Req0),
?assertMatch(
#{<<"username">> := <<"user name">>,
<<"CLIENT_client id">> := <<"client id">>,
<<"peerhost">> := <<"127.0.0.1">>,
<<"proto_name">> := <<"MQTT">>,
<<"mountpoint">> := <<"MOUNTPOINT">>,
<<"topic">> := <<"t">>,
<<"action">> := <<"publish">>},
jiffy:decode(RawBody, [return_maps])),
Req = cowboy_req:reply(200, Req1),
{ok, Req, State}
end,
#{<<"method">> => <<"post">>,
<<"path">> => <<"username/${username}/"
"clientid/${clientid}/"
"peerhost/${peerhost}/"
"proto_name/${proto_name}/"
"mountpoint/${mountpoint}/"
"topic/${topic}/"
"action/${action}">>,
<<"body">> => #{<<"username">> => <<"${username}">>,
<<"CLIENT_${clientid}">> => <<"${clientid}">>,
<<"peerhost">> => <<"${peerhost}">>,
<<"proto_name">> => <<"${proto_name}">>,
<<"mountpoint">> => <<"${mountpoint}">>,
<<"topic">> => <<"${topic}">>,
<<"action">> => <<"${action}">>}
}),
ClientInfo = #{clientid => <<"client id">>,
username => <<"user name">>,
peerhost => {127,0,0,1},
protocol => <<"MQTT">>,
mountpoint => <<"MOUNTPOINT">>,
zone => default,
listener => {tcp, default}
},
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)).
meck:expect(emqx_resource, query, fun(_, _) -> {error, other} end),
?assertEqual(deny,
emqx_access_control:authorize(ClientInfo, subscribe, <<"+">>)),
?assertEqual(deny,
emqx_access_control:authorize(ClientInfo, publish, <<"+">>)),
ok.
t_form_body(_Config) ->
ok = setup_handler_and_config(
fun(Req0, State) ->
?assertEqual(
<<"/authz/"
"username/user%20name/"
"clientid/client%20id/"
"peerhost/127.0.0.1/"
"proto_name/MQTT/"
"mountpoint/MOUNTPOINT/"
"topic/t/"
"action/publish">>,
cowboy_req:path(Req0)),
{ok, PostVars, Req1} = cowboy_req:read_urlencoded_body(Req0),
?assertMatch(
#{<<"username">> := <<"user name">>,
<<"clientid">> := <<"client id">>,
<<"peerhost">> := <<"127.0.0.1">>,
<<"proto_name">> := <<"MQTT">>,
<<"mountpoint">> := <<"MOUNTPOINT">>,
<<"topic">> := <<"t">>,
<<"action">> := <<"publish">>},
maps:from_list(PostVars)),
Req = cowboy_req:reply(200, Req1),
{ok, Req, State}
end,
#{<<"method">> => <<"post">>,
<<"path">> => <<"username/${username}/"
"clientid/${clientid}/"
"peerhost/${peerhost}/"
"proto_name/${proto_name}/"
"mountpoint/${mountpoint}/"
"topic/${topic}/"
"action/${action}">>,
<<"body">> => #{<<"username">> => <<"${username}">>,
<<"clientid">> => <<"${clientid}">>,
<<"peerhost">> => <<"${peerhost}">>,
<<"proto_name">> => <<"${proto_name}">>,
<<"mountpoint">> => <<"${mountpoint}">>,
<<"topic">> => <<"${topic}">>,
<<"action">> => <<"${action}">>},
<<"headers">> => #{<<"content-type">> => <<"application/x-www-form-urlencoded">>}
}),
ClientInfo = #{clientid => <<"client id">>,
username => <<"user name">>,
peerhost => {127,0,0,1},
protocol => <<"MQTT">>,
mountpoint => <<"MOUNTPOINT">>,
zone => default,
listener => {tcp, default}
},
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)).
t_create_replace(_Config) ->
ClientInfo = #{clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127,0,0,1},
zone => default,
listener => {tcp, default}
},
%% Bad URL
ok = setup_handler_and_config(
fun(Req0, State) ->
Req = cowboy_req:reply(200, Req0),
{ok, Req, State}
end,
#{<<"base_url">> => <<"http://127.0.0.1:33331/authz">>}),
?assertEqual(
deny,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
%% Changing to other bad config does not work
BadConfig = maps:merge(
raw_http_authz_config(),
#{<<"base_url">> => <<"http://127.0.0.1:33332/authz">>}),
?assertMatch(
{error, _},
emqx_authz:update({?CMD_REPLACE, http}, BadConfig)),
?assertEqual(
deny,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
%% Changing to valid config
OkConfig = maps:merge(
raw_http_authz_config(),
#{<<"base_url">> => <<"http://127.0.0.1:33333/authz">>}),
?assertMatch(
{ok, _},
emqx_authz:update({?CMD_REPLACE, http}, OkConfig)),
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)).
%%------------------------------------------------------------------------------
%% Helpers
%%------------------------------------------------------------------------------
raw_http_authz_config() ->
#{
<<"enable">> => <<"true">>,
<<"type">> => <<"http">>,
<<"method">> => <<"get">>,
<<"base_url">> => <<"http://127.0.0.1:33333/authz">>,
<<"path">> => <<"users/${username}/">>,
<<"query">> => <<"topic=${topic}&action=${action}">>,
<<"headers">> => #{<<"X-Test-Header">> => <<"Test Value">>}
}.
setup_handler_and_config(Handler, Config) ->
ok = emqx_authz_http_test_server:set_handler(Handler),
ok = emqx_authz_test_lib:setup_config(
raw_http_authz_config(),
Config).
start_apps(Apps) ->
lists:foreach(fun application:ensure_all_started/1, Apps).
stop_apps(Apps) ->
lists:foreach(fun application:stop/1, Apps).

86
apps/emqx_authz/test/emqx_authz_http_test_server.erl Normal file
View File

@ -0,0 +1,86 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_authz_http_test_server).
-behaviour(supervisor).
-behaviour(cowboy_handler).
% cowboy_server callbacks
-export([init/2]).
% supervisor callbacks
-export([init/1]).
% API
-export([start_link/2,
stop/0,
set_handler/1
]).
%%------------------------------------------------------------------------------
%% API
%%------------------------------------------------------------------------------
start_link(Port, Path) ->
supervisor:start_link({local, ?MODULE}, ?MODULE, [Port, Path]).
stop() ->
gen_server:stop(?MODULE).
set_handler(F) when is_function(F, 2) ->
true = ets:insert(?MODULE, {handler, F}),
ok.
%%------------------------------------------------------------------------------
%% supervisor API
%%------------------------------------------------------------------------------
init([Port, Path]) ->
Dispatch = cowboy_router:compile(
[
{'_', [{Path, ?MODULE, []}]}
]),
TransOpts = #{socket_opts => [{port, Port}],
connection_type => supervisor},
ProtoOpts = #{env => #{dispatch => Dispatch}},
Tab = ets:new(?MODULE, [set, named_table, public]),
ets:insert(Tab, {handler, fun default_handler/2}),
ChildSpec = ranch:child_spec(?MODULE, ranch_tcp, TransOpts, cowboy_clear, ProtoOpts),
{ok, {{one_for_one, 10, 10}, [ChildSpec]}}.
%%------------------------------------------------------------------------------
%% cowboy_server API
%%------------------------------------------------------------------------------
init(Req, State) ->
[{handler, Handler}] = ets:lookup(?MODULE, handler),
Handler(Req, State).
%%------------------------------------------------------------------------------
%% Internal functions
%%------------------------------------------------------------------------------
default_handler(Req0, State) ->
Req = cowboy_req:reply(
400,
#{<<"content-type">> => <<"text/plain">>},
<<"">>,
Req0),
{ok, Req, State}.

169
apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl
View File

@ -18,10 +18,8 @@
-compile(nowarn_export_all).
-compile(export_all).
-include("emqx_authz.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").
all() ->
emqx_common_test_helpers:all(?MODULE).
@ -31,86 +29,123 @@ groups() ->
init_per_suite(Config) ->
ok = emqx_common_test_helpers:start_apps(
[emqx_connector, emqx_conf, emqx_authz],
fun set_special_configs/1
),
[emqx_conf, emqx_authz],
fun set_special_configs/1),
Config.
end_per_suite(_Config) ->
{ok, _} = emqx:update_config(
[authorization],
#{<<"no_match">> => <<"allow">>,
<<"cache">> => #{<<"enable">> => <<"true">>},
<<"sources">> => []}),
emqx_common_test_helpers:stop_apps([emqx_authz, emqx_conf]),
ok.
ok = emqx_authz_test_lib:restore_authorizers(),
ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
init_per_testcase(_TestCase, Config) ->
ok = emqx_authz_test_lib:reset_authorizers(),
ok = setup_config(),
Config.
end_per_testcase(_TestCase, _Config) ->
ok = emqx_authz_mnesia:purge_rules().
set_special_configs(emqx_authz) ->
{ok, _} = emqx:update_config([authorization, cache, enable], false),
{ok, _} = emqx:update_config([authorization, no_match], deny),
{ok, _} = emqx:update_config([authorization, sources],
[#{<<"type">> => <<"built-in-database">>}]),
ok;
set_special_configs(_App) ->
ok = emqx_authz_test_lib:reset_authorizers();
set_special_configs(_) ->
ok.
init_per_testcase(t_authz, Config) ->
emqx_authz_mnesia:store_rules(
{username, <<"test_username">>},
[{allow, publish, <<"test/", ?PH_S_USERNAME>>},
{allow, subscribe, <<"eq #">>}]),
emqx_authz_mnesia:store_rules(
{clientid, <<"test_clientid">>},
[{allow, publish, <<"test/", ?PH_S_CLIENTID>>},
{deny, subscribe, <<"eq #">>}]),
emqx_authz_mnesia:store_rules(
all,
[{deny, all, <<"#">>}]),
Config;
init_per_testcase(_, Config) -> Config.
end_per_testcase(t_authz, Config) ->
ok = emqx_authz_mnesia:purge_rules(),
Config;
end_per_testcase(_, Config) -> Config.
%%------------------------------------------------------------------------------
%% Testcases
%%------------------------------------------------------------------------------
t_username_topic_rules(_Config) ->
ok = test_topic_rules(username).
t_authz(_) ->
ClientInfo1 = #{clientid => <<"test">>,
username => <<"test">>,
peerhost => {127,0,0,1},
listener => {tcp, default}
},
ClientInfo2 = #{clientid => <<"fake_clientid">>,
username => <<"test_username">>,
peerhost => {127,0,0,1},
listener => {tcp, default}
},
ClientInfo3 = #{clientid => <<"test_clientid">>,
username => <<"fake_username">>,
t_clientid_topic_rules(_Config) ->
ok = test_topic_rules(clientid).
t_all_topic_rules(_Config) ->
ok = test_topic_rules(all).
test_topic_rules(Key) ->
ClientInfo = #{clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127,0,0,1},
zone => default,
listener => {tcp, default}
},
?assertEqual(deny, emqx_access_control:authorize(
ClientInfo1, subscribe, <<"#">>)),
?assertEqual(deny, emqx_access_control:authorize(
ClientInfo1, publish, <<"#">>)),
SetupSamples = fun(CInfo, Samples) ->
setup_client_samples(CInfo, Samples, Key)
end,
?assertEqual(allow, emqx_access_control:authorize(
ClientInfo2, publish, <<"test/test_username">>)),
?assertEqual(allow, emqx_access_control:authorize(
ClientInfo2, subscribe, <<"#">>)),
ok = emqx_authz_test_lib:test_no_topic_rules(ClientInfo, SetupSamples),
?assertEqual(allow, emqx_access_control:authorize(
ClientInfo3, publish, <<"test/test_clientid">>)),
?assertEqual(deny, emqx_access_control:authorize(
ClientInfo3, subscribe, <<"#">>)),
ok = emqx_authz_test_lib:test_allow_topic_rules(ClientInfo, SetupSamples),
ok.
ok = emqx_authz_test_lib:test_deny_topic_rules(ClientInfo, SetupSamples).
t_normalize_rules(_Config) ->
ClientInfo = #{clientid => <<"clientid">>,
username => <<"username">>,
peerhost => {127,0,0,1},
zone => default,
listener => {tcp, default}
},
ok = emqx_authz_mnesia:store_rules(
{username, <<"username">>},
[{allow, publish, "t"}]),
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
?assertException(
error,
{invalid_rule, _},
emqx_authz_mnesia:store_rules(
{username, <<"username">>},
[[allow, publish, <<"t">>]])),
?assertException(
error,
{invalid_rule_action, _},
emqx_authz_mnesia:store_rules(
{username, <<"username">>},
[{allow, pub, <<"t">>}])),
?assertException(
error,
{invalid_rule_permission, _},
emqx_authz_mnesia:store_rules(
{username, <<"username">>},
[{accept, publish, <<"t">>}])).
%%------------------------------------------------------------------------------
%% Helpers
%%------------------------------------------------------------------------------
raw_mnesia_authz_config() ->
#{
<<"enable">> => <<"true">>,
<<"type">> => <<"built-in-database">>
}.
setup_client_samples(ClientInfo, Samples, Key) ->
ok = emqx_authz_mnesia:purge_rules(),
Rules = lists:flatmap(
fun(#{topics := Topics, permission := Permission, action := Action}) ->
lists:map(
fun(Topic) ->
{binary_to_atom(Permission), binary_to_atom(Action), Topic}
end,
Topics)
end,
Samples),
#{username := Username, clientid := ClientId} = ClientInfo,
Who = case Key of
username -> {username, Username};
clientid -> {clientid, ClientId};
all -> all
end,
ok = emqx_authz_mnesia:store_rules(Who, Rules).
setup_config() ->
emqx_authz_test_lib:setup_config(raw_mnesia_authz_config(), #{}).

2
apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl
View File

@ -56,7 +56,7 @@ end_per_suite(_Config) ->
ok = stop_apps([emqx_resource, emqx_connector]),
ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
init_per_testcase(Config) ->
init_per_testcase(_TestCase, Config) ->
ok = emqx_authz_test_lib:reset_authorizers(),
Config.

2
apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl
View File

@ -56,7 +56,7 @@ end_per_suite(_Config) ->
ok = stop_apps([emqx_resource, emqx_connector]),
ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
init_per_testcase(Config) ->
init_per_testcase(_TestCase, Config) ->
ok = emqx_authz_test_lib:reset_authorizers(),
Config.

2
apps/emqx_authz/test/emqx_authz_redis_SUITE.erl
View File

@ -57,7 +57,7 @@ end_per_suite(_Config) ->
ok = stop_apps([emqx_resource, emqx_connector]),
ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
init_per_testcase(Config) ->
init_per_testcase(_TestCase, Config) ->
ok = emqx_authz_test_lib:reset_authorizers(),
Config.

1
apps/emqx_authz/test/emqx_authz_test_lib.erl
View File

@ -70,6 +70,7 @@ test_samples(ClientInfo, Samples) ->
test_no_topic_rules(ClientInfo, SetupSamples) ->
%% No rules
ok = reset_authorizers(deny, false),
ok = SetupSamples(ClientInfo, []),
ok = test_samples(

12
apps/emqx_dashboard/src/emqx_dashboard_collection.erl
View File

@ -22,7 +22,7 @@
-export([get_collect/0]).
-export([get_local_time/0]).
-export([get_universal_epoch/0]).
-boot_mnesia({mnesia, [boot]}).
@ -108,7 +108,7 @@ handle_info(collect, State = #{count := Count, collect := Collect, temp_collect
handle_info(clear_expire_data, State = #{expire_interval := ExpireInterval}) ->
timer(?CLEAR_INTERVAL, clear_expire_data),
T1 = get_local_time(),
T1 = get_universal_epoch(),
Spec = ets:fun2ms(fun({_, T, _C} = Data) when (T1 - T) > ExpireInterval -> Data end),
Collects = ets:select(?TAB_COLLECT, Spec),
lists:foreach(fun(Collect) ->
@ -161,7 +161,7 @@ flush({Connection, Route, Subscription}, {Received0, Sent0, Dropped0}) ->
diff(Received, Received0),
diff(Sent, Sent0),
diff(Dropped, Dropped0)},
Ts = get_local_time(),
Ts = get_universal_epoch(),
{atomic, ok} = mria:transaction(mria:local_content_shard(),
fun mnesia:write/3,
[ ?TAB_COLLECT
@ -179,8 +179,8 @@ timer(Secs, Msg) ->
erlang:send_after(Secs, self(), Msg).
get_today_remaining_seconds() ->
?CLEAR_INTERVAL - (get_local_time() rem ?CLEAR_INTERVAL).
?CLEAR_INTERVAL - (get_universal_epoch() rem ?CLEAR_INTERVAL).
get_local_time() ->
(calendar:datetime_to_gregorian_seconds(calendar:local_time()) -
get_universal_epoch() ->
(calendar:datetime_to_gregorian_seconds(calendar:universal_time()) -
calendar:datetime_to_gregorian_seconds({{1970,1,1}, {0,0,0}})).

2
apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl
View File

@ -278,7 +278,7 @@ sampling(Node, Counter) ->
rpc:call(Node, ?MODULE, sampling, [Node, Counter]).
select_data() ->
Time = emqx_dashboard_collection:get_local_time() - 7200000,
Time = emqx_dashboard_collection:get_universal_epoch() - 7200000,
ets:select(?TAB_COLLECT, [{{mqtt_collect,'$1','$2'}, [{'>', '$1', Time}], ['$_']}]).
format(Collects) ->

10
apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
View File

@ -211,11 +211,16 @@ check_request_body(#{body := Body}, Schema, Module, CheckFun, true) ->
%% {good_nest_2, mk(ref(?MODULE, good_ref), #{})}
%% ]}
%% ]
check_request_body(#{body := Body}, Spec, _Module, CheckFun, false) ->
check_request_body(#{body := Body}, Spec, _Module, CheckFun, false)when is_list(Spec) ->
lists:foldl(fun({Name, Type}, Acc) ->
Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]},
maps:merge(Acc, CheckFun(Schema, Body, #{}))
end, #{}, Spec).
end, #{}, Spec);
%% requestBody => #{content => #{ 'application/octet-stream' =>
%% #{schema => #{ type => string, format => binary}}}
check_request_body(#{body := Body}, Spec, _Module, _CheckFun, false)when is_map(Spec) ->
Body.
%% tags, description, summary, security, deprecated
meta_to_spec(Meta, Module) ->
@ -287,6 +292,7 @@ trans_desc(Spec, Hocon) ->
Desc -> Spec#{description => to_bin(Desc)}
end.
request_body(#{content := _} = Content, _Module) -> {Content, []};
request_body([], _Module) -> {[], []};
request_body(Schema, Module) ->
{{Props, Refs}, Examples} =

48
apps/emqx_exhook/etc/emqx_exhook.conf
View File

@ -2,43 +2,45 @@
## EMQ X Hooks
##====================================================================
exhook {
## The default value or action will be returned, while the request to
## the gRPC server failed or no available grpc server running.
##
## Default: deny
## Value: ignore | deny
request_failed_action = deny
emqx_exhook {
## The timeout to request grpc server
servers = [
##{
## name = default
##
## Default: 5s
## Value: Duration
request_timeout = 5s
## Whether to automatically reconnect (initialize) the gRPC server
##
## When gRPC is not available, exhook tries to request the gRPC service at
## that interval and reinitialize the list of mounted hooks.
##
## Default: false
## Value: false | Duration
auto_reconnect = 60s
## auto_reconnect = 60s
## The process pool size for gRPC client
## The default value or action will be returned, while the request to
## the gRPC server failed or no available grpc server running.
##
## Default: Equals cpu cores
## Value: Integer
#pool_size = 16
## Default: deny
## Value: ignore | deny
## failed_action = deny
servers = [
# { name: "default"
# url: "http://127.0.0.1:9000"
# #ssl: {
## The timeout to request grpc server
##
## Default: 5s
## Value: Duration
## request_timeout = 5s
## url = "http://127.0.0.1:9000"
## ssl {
## cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem"
## certfile: "{{ platform_etc_dir }}/certs/cert.pem"
## keyfile: "{{ platform_etc_dir }}/certs/key.pem"
## }
# }
##
## The process pool size for gRPC client
##
## Default: Equals cpu cores
## Value: Integer
## pool_size = 16
##}
]
}

66
apps/emqx_exhook/src/emqx_exhook.erl
View File

@ -19,90 +19,56 @@
-include("emqx_exhook.hrl").
-include_lib("emqx/include/logger.hrl").
-export([ enable/1
, disable/1
, list/0
]).
-export([ cast/2
, call_fold/3
]).
%%--------------------------------------------------------------------
%% Mgmt APIs
%%--------------------------------------------------------------------
-spec enable(binary()) -> ok | {error, term()}.
enable(Name) ->
with_mngr(fun(Pid) -> emqx_exhook_mngr:enable(Pid, Name) end).
-spec disable(binary()) -> ok | {error, term()}.
disable(Name) ->
with_mngr(fun(Pid) -> emqx_exhook_mngr:disable(Pid, Name) end).
-spec list() -> [atom() | string()].
list() ->
with_mngr(fun(Pid) -> emqx_exhook_mngr:list(Pid) end).
with_mngr(Fun) ->
case lists:keyfind(emqx_exhook_mngr, 1,
supervisor:which_children(emqx_exhook_sup)) of
{_, Pid, _, _} ->
Fun(Pid);
_ ->
{error, no_manager_svr}
end.
%%--------------------------------------------------------------------
%% Dispatch APIs
%%--------------------------------------------------------------------
-spec cast(atom(), map()) -> ok.
cast(Hookpoint, Req) ->
cast(Hookpoint, Req, emqx_exhook_mngr:running()).
cast(Hookpoint, Req, emqx_exhook_mgr:running()).
cast(_, _, []) ->
ok;
cast(Hookpoint, Req, [ServerName|More]) ->
%% XXX: Need a real asynchronous running
_ = emqx_exhook_server:call(Hookpoint, Req,
emqx_exhook_mngr:server(ServerName)),
emqx_exhook_mgr:server(ServerName)),
cast(Hookpoint, Req, More).
-spec call_fold(atom(), term(), function())
-> {ok, term()}
-spec call_fold(atom(), term(), function()) -> {ok, term()}
| {stop, term()}.
call_fold(Hookpoint, Req, AccFun) ->
FailedAction = emqx_exhook_mngr:get_request_failed_action(),
ServerNames = emqx_exhook_mngr:running(),
case ServerNames == [] andalso FailedAction == deny of
true ->
case emqx_exhook_mgr:running() of
[] ->
{stop, deny_action_result(Hookpoint, Req)};
_ ->
call_fold(Hookpoint, Req, FailedAction, AccFun, ServerNames)
ServerNames ->
call_fold(Hookpoint, Req, AccFun, ServerNames)
end.
call_fold(_, Req, _, _, []) ->
call_fold(_, Req, _, []) ->
{ok, Req};
call_fold(Hookpoint, Req, FailedAction, AccFun, [ServerName|More]) ->
Server = emqx_exhook_mngr:server(ServerName),
call_fold(Hookpoint, Req, AccFun, [ServerName|More]) ->
Server = emqx_exhook_mgr:server(ServerName),
case emqx_exhook_server:call(Hookpoint, Req, Server) of
{ok, Resp} ->
case AccFun(Req, Resp) of
{stop, NReq} ->
{stop, NReq};
{ok, NReq} ->
call_fold(Hookpoint, NReq, FailedAction, AccFun, More);
call_fold(Hookpoint, NReq, AccFun, More);
_ ->
call_fold(Hookpoint, Req, FailedAction, AccFun, More)
call_fold(Hookpoint, Req, AccFun, More)
end;
_ ->
case FailedAction of
case emqx_exhook_server:failed_action(Server) of
ignore ->
call_fold(Hookpoint, Req, AccFun, More);
deny ->
{stop, deny_action_result(Hookpoint, Req)};
_ ->
call_fold(Hookpoint, Req, FailedAction, AccFun, More)
{stop, deny_action_result(Hookpoint, Req)}
end
end.

281
apps/emqx_exhook/src/emqx_exhook_api.erl Normal file
View File

@ -0,0 +1,281 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_exhook_api).
-behaviour(minirest_api).
-include_lib("typerefl/include/types.hrl").
-include_lib("emqx/include/logger.hrl").
-export([api_spec/0, paths/0, schema/1, fields/1, namespace/0]).
-export([exhooks/2, action_with_name/2, move/2]).
-import(hoconsc, [mk/2, ref/1, enum/1, array/1]).
-import(emqx_dashboard_swagger, [schema_with_example/2, error_codes/2]).
-define(TAGS, [<<"exhooks">>]).
-define(BAD_REQUEST, 'BAD_REQUEST').
-define(BAD_RPC, 'BAD_RPC').
namespace() -> "exhook".
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE).
paths() -> ["/exhooks", "/exhooks/:name", "/exhooks/:name/move"].
schema(("/exhooks")) ->
#{
'operationId' => exhooks,
get => #{tags => ?TAGS,
description => <<"List all servers">>,
responses => #{200 => mk(array(ref(detailed_server_info)), #{})}
},
post => #{tags => ?TAGS,
description => <<"Add a servers">>,
'requestBody' => server_conf_schema(),
responses => #{201 => mk(ref(detailed_server_info), #{}),
500 => error_codes([?BAD_RPC], <<"Bad RPC">>)
}
}
};
schema("/exhooks/:name") ->
#{'operationId' => action_with_name,
get => #{tags => ?TAGS,
description => <<"Get the detail information of server">>,
parameters => params_server_name_in_path(),
responses => #{200 => mk(ref(detailed_server_info), #{}),
400 => error_codes([?BAD_REQUEST], <<"Bad Request">>)
}
},
put => #{tags => ?TAGS,
description => <<"Update the server">>,
parameters => params_server_name_in_path(),
'requestBody' => server_conf_schema(),
responses => #{200 => <<>>,
400 => error_codes([?BAD_REQUEST], <<"Bad Request">>),
500 => error_codes([?BAD_RPC], <<"Bad RPC">>)
}
},
delete => #{tags => ?TAGS,
description => <<"Delete the server">>,
parameters => params_server_name_in_path(),
responses => #{204 => <<>>,
500 => error_codes([?BAD_RPC], <<"Bad RPC">>) }
}
};
schema("/exhooks/:name/move") ->
#{'operationId' => move,
post => #{tags => ?TAGS,
description => <<"Move the server">>,
parameters => params_server_name_in_path(),
'requestBody' => mk(ref(move_req), #{}),
responses => #{200 => <<>>,
400 => error_codes([?BAD_REQUEST], <<"Bad Request">>),
500 => error_codes([?BAD_RPC], <<"Bad RPC">>)
}
}
}.
fields(move_req) ->
[
{position, mk(enum([top, bottom, before, 'after']), #{})},
{related, mk(string(), #{desc => <<"Relative position of movement">>,
default => <<>>,
example => <<>>
})}
];
fields(detailed_server_info) ->
[ {status, mk(enum([running, waiting, stopped]), #{})}
, {hooks, mk(array(string()), #{default => []})}
, {node_status, mk(ref(node_status), #{})}
] ++ emqx_exhook_schema:server_config();
fields(node_status) ->
[ {node, mk(string(), #{})}
, {status, mk(enum([running, waiting, stopped, not_found, error]), #{})}
];
fields(server_config) ->
emqx_exhook_schema:server_config().
params_server_name_in_path() ->
[{name, mk(string(), #{in => path,
required => true,
example => <<"default">>})}
].
server_conf_schema() ->
schema_with_example(ref(server_config),
#{ name => "default"
, enable => true
, url => <<"http://127.0.0.1:8081">>
, request_timeout => "5s"
, failed_action => deny
, auto_reconnect => "60s"
, pool_size => 8
, ssl => #{ enable => false
, cacertfile => <<"{{ platform_etc_dir }}/certs/cacert.pem">>
, certfile => <<"{{ platform_etc_dir }}/certs/cert.pem">>
, keyfile => <<"{{ platform_etc_dir }}/certs/key.pem">>
}
}).
exhooks(get, _) ->
ServerL = emqx_exhook_mgr:list(),
ServerL2 = nodes_all_server_status(ServerL),
{200, ServerL2};
exhooks(post, #{body := Body}) ->
case emqx_exhook_mgr:update_config([emqx_exhook, servers], {add, Body}) of
{ok, Result} ->
{201, Result};
{error, Error} ->
{500, #{code => <<"BAD_RPC">>,
message => Error
}}
end.
action_with_name(get, #{bindings := #{name := Name}}) ->
Result = emqx_exhook_mgr:lookup(Name),
NodeStatus = nodes_server_status(Name),
case Result of
not_found ->
{400, #{code => <<"BAD_REQUEST">>,
message => <<"Server not found">>
}};
ServerInfo ->
{200, ServerInfo#{node_status => NodeStatus}}
end;
action_with_name(put, #{bindings := #{name := Name}, body := Body}) ->
case emqx_exhook_mgr:update_config([emqx_exhook, servers],
{update, Name, Body}) of
{ok, not_found} ->
{400, #{code => <<"BAD_REQUEST">>,
message => <<"Server not found">>
}};
{ok, {error, Reason}} ->
{400, #{code => <<"BAD_REQUEST">>,
message => unicode:characters_to_binary(io_lib:format("Error Reason:~p~n", [Reason]))
}};
{ok, _} ->
{200};
{error, Error} ->
{500, #{code => <<"BAD_RPC">>,
message => Error
}}
end;
action_with_name(delete, #{bindings := #{name := Name}}) ->
case emqx_exhook_mgr:update_config([emqx_exhook, servers],
{delete, Name}) of
{ok, _} ->
{200};
{error, Error} ->
{500, #{code => <<"BAD_RPC">>,
message => Error
}}
end.
move(post, #{bindings := #{name := Name}, body := Body}) ->
#{<<"position">> := PositionT, <<"related">> := Related} = Body,
Position = erlang:binary_to_atom(PositionT),
case emqx_exhook_mgr:update_config([emqx_exhook, servers],
{move, Name, Position, Related}) of
{ok, ok} ->
{200};
{ok, not_found} ->
{400, #{code => <<"BAD_REQUEST">>,
message => <<"Server not found">>
}};
{error, Error} ->
{500, #{code => <<"BAD_RPC">>,
message => Error
}}
end.
nodes_server_status(Name) ->
StatusL = call_cluster(emqx_exhook_mgr, server_status, [Name]),
Handler = fun({Node, {error, _}}) ->
#{node => Node,
status => error
};
({Node, Status}) ->
#{node => Node,
status => Status
}
end,
lists:map(Handler, StatusL).
nodes_all_server_status(ServerL) ->
AllStatusL = call_cluster(emqx_exhook_mgr, all_servers_status, []),
AggreMap = lists:foldl(fun(#{name := Name}, Acc) ->
Acc#{Name => []}
end,
#{},
ServerL),
AddToMap = fun(Servers, Node, Status, Map) ->
lists:foldl(fun(Name, Acc) ->
StatusL = maps:get(Name, Acc),
StatusL2 = [#{node => Node,
status => Status
} | StatusL],
Acc#{Name := StatusL2}
end,
Map,
Servers)
end,
AggreMap2 = lists:foldl(fun({Node, #{running := Running,
waiting := Waiting,
stopped := Stopped}},
Acc) ->
AddToMap(Stopped, Node, stopped,
AddToMap(Waiting, Node, waiting,
AddToMap(Running, Node, running, Acc)))
end,
AggreMap,
AllStatusL),
Handler = fun(#{name := Name} = Server) ->
Server#{node_status => maps:get(Name, AggreMap2)}
end,
lists:map(Handler, ServerL).
call_cluster(Module, Fun, Args) ->
Nodes = mria_mnesia:running_nodes(),
[{Node, rpc_call(Node, Module, Fun, Args)} || Node <- Nodes].
rpc_call(Node, Module, Fun, Args) when Node =:= node() ->
erlang:apply(Module, Fun, Args);
rpc_call(Node, Module, Fun, Args) ->
case rpc:call(Node, Module, Fun, Args) of
{badrpc, Reason} -> {error, Reason};
Res -> Res
end.

84
apps/emqx_exhook/src/emqx_exhook_cli.erl
View File

@ -1,84 +0,0 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_exhook_cli).
-include("emqx_exhook.hrl").
-export([cli/1]).
cli(["server", "list"]) ->
if_enabled(fun() ->
ServerNames = emqx_exhook:list(),
[emqx_ctl:print("Server(~ts)~n", [format(Name)]) || Name <- ServerNames]
end);
cli(["server", "enable", Name]) ->
if_enabled(fun() ->
print(emqx_exhook:enable(iolist_to_binary(Name)))
end);
cli(["server", "disable", Name]) ->
if_enabled(fun() ->
print(emqx_exhook:disable(iolist_to_binary(Name)))
end);
cli(["server", "stats"]) ->
if_enabled(fun() ->
[emqx_ctl:print("~-35s:~w~n", [Name, N]) || {Name, N} <- stats()]
end);
cli(_) ->
emqx_ctl:usage([{"exhook server list", "List all running exhook server"},
{"exhook server enable <Name>", "Enable a exhook server in the configuration"},
{"exhook server disable <Name>", "Disable a exhook server"},
{"exhook server stats", "Print exhook server statistic"}]).
print(ok) ->
emqx_ctl:print("ok~n");
print({error, Reason}) ->
emqx_ctl:print("~p~n", [Reason]).
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------
if_enabled(Fun) ->
case lists:keymember(?APP, 1, application:which_applications()) of
true ->
Fun();
_ -> hint()
end.
hint() ->
emqx_ctl:print("Please './bin/emqx_ctl plugins load emqx_exhook' first.~n").
stats() ->
lists:usort(lists:foldr(fun({K, N}, Acc) ->
case atom_to_list(K) of
"exhook." ++ Key -> [{Key, N} | Acc];
_ -> Acc
end
end, [], emqx_metrics:all())).
format(Name) ->
case emqx_exhook_mngr:server(Name) of
undefined ->
lists:flatten(
io_lib:format("name=~ts, hooks=#{}, active=false", [Name]));
Server ->
emqx_exhook_server:format(Server)
end.

596
apps/emqx_exhook/src/emqx_exhook_mgr.erl Normal file
View File

@ -0,0 +1,596 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% @doc Manage the server status and reload strategy
-module(emqx_exhook_mgr).
-behaviour(gen_server).
-include("emqx_exhook.hrl").
-include_lib("emqx/include/logger.hrl").
%% APIs
-export([start_link/0]).
%% Mgmt API
-export([ list/0
, lookup/1
, enable/1
, disable/1
, server_status/1
, all_servers_status/0
]).
%% Helper funcs
-export([ running/0
, server/1
, init_counter_table/0
]).
-export([ update_config/2
, pre_config_update/3
, post_config_update/5
]).
%% gen_server callbacks
-export([ init/1
, handle_call/3
, handle_cast/2
, handle_info/2
, terminate/2
, code_change/3
]).
-export([roots/0]).
-type state() :: #{%% Running servers
running := servers(),
%% Wait to reload servers
waiting := servers(),
%% Marked stopped servers
stopped := servers(),
%% Timer references
trefs := map(),
orders := orders()
}.
-type server_name() :: binary().
-type servers() :: #{server_name() => server()}.
-type server() :: server_options().
-type server_options() :: map().
-type move_direct() :: top
| bottom
| before
| 'after'.
-type orders() :: #{server_name() => integer()}.
-type server_info() :: #{name := server_name(),
status := running | waiting | stopped,
atom() => term()
}.
-define(DEFAULT_TIMEOUT, 60000).
-define(CNTER, emqx_exhook_counter).
-export_type([server_info/0]).
%%--------------------------------------------------------------------
%% APIs
%%--------------------------------------------------------------------
-spec start_link() -> ignore
| {ok, pid()}
| {error, any()}.
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
list() ->
call(list).
-spec lookup(server_name()) -> not_found | server_info().
lookup(Name) ->
call({lookup, Name}).
enable(Name) ->
update_config([emqx_exhook, servers], {enable, Name, true}).
disable(Name) ->
update_config([emqx_exhook, servers], {enable, Name, false}).
server_status(Name) ->
call({server_status, Name}).
all_servers_status() ->
call(all_servers_status).
call(Req) ->
gen_server:call(?MODULE, Req, ?DEFAULT_TIMEOUT).
init_counter_table() ->
_ = ets:new(?CNTER, [named_table, public]).
%%=====================================================================
%% Hocon schema
roots() ->
emqx_exhook_schema:server_config().
update_config(KeyPath, UpdateReq) ->
case emqx_conf:update(KeyPath, UpdateReq, #{override_to => cluster}) of
{ok, UpdateResult} ->
#{post_config_update := #{?MODULE := Result}} = UpdateResult,
{ok, Result};
Error ->
Error
end.
pre_config_update(_, {add, Conf}, OldConf) ->
{ok, OldConf ++ [Conf]};
pre_config_update(_, {update, Name, Conf}, OldConf) ->
case replace_conf(Name, fun(_) -> Conf end, OldConf) of
not_found -> {error, not_found};
NewConf -> {ok, NewConf}
end;
pre_config_update(_, {delete, ToDelete}, OldConf) ->
{ok, lists:dropwhile(fun(#{<<"name">> := Name}) -> Name =:= ToDelete end,
OldConf)};
pre_config_update(_, {move, Name, Position, Relate}, OldConf) ->
case do_move(Name, Position, Relate, OldConf) of
not_found -> {error, not_found};
NewConf -> {ok, NewConf}
end;
pre_config_update(_, {enable, Name, Enable}, OldConf) ->
case replace_conf(Name,
fun(Conf) -> Conf#{<<"enable">> => Enable} end, OldConf) of
not_found -> {error, not_found};
NewConf ->
ct:pal(">>>> enable Name:~p Enable:~p, New:~p~n", [Name, Enable, NewConf]),
{ok, NewConf}
end.
post_config_update(_KeyPath, UpdateReq, NewConf, _OldConf, _AppEnvs) ->
{ok, call({update_config, UpdateReq, NewConf})}.
%%=====================================================================
%%--------------------------------------------------------------------
%% gen_server callbacks
%%--------------------------------------------------------------------
init([]) ->
process_flag(trap_exit, true),
emqx_conf:add_handler([emqx_exhook, servers], ?MODULE),
ServerL = emqx:get_config([emqx_exhook, servers]),
{Waiting, Running, Stopped} = load_all_servers(ServerL),
Orders = reorder(ServerL),
{ok, ensure_reload_timer(
#{waiting => Waiting,
running => Running,
stopped => Stopped,
trefs => #{},
orders => Orders
})}.
-spec load_all_servers(list(server_options())) -> {servers(), servers(), servers()}.
load_all_servers(ServerL) ->
load_all_servers(ServerL, #{}, #{}, #{}).
load_all_servers([#{name := Name} = Options | More], Waiting, Running, Stopped) ->
case emqx_exhook_server:load(Name, Options) of
{ok, ServerState} ->
save(Name, ServerState),
load_all_servers(More, Waiting, Running#{Name => Options}, Stopped);
{error, _} ->
load_all_servers(More, Waiting#{Name => Options}, Running, Stopped);
disable ->
load_all_servers(More, Waiting, Running, Stopped#{Name => Options})
end;
load_all_servers([], Waiting, Running, Stopped) ->
{Waiting, Running, Stopped}.
handle_call(list, _From, State = #{running := Running,
waiting := Waiting,
stopped := Stopped,
orders := Orders}) ->
R = get_servers_info(running, Running),
W = get_servers_info(waiting, Waiting),
S = get_servers_info(stopped, Stopped),
Servers = R ++ W ++ S,
OrderServers = sort_name_by_order(Servers, Orders),
{reply, OrderServers, State};
handle_call({update_config, {move, _Name, _Direct, _Related}, NewConfL},
_From,
State) ->
Orders = reorder(NewConfL),
{reply, ok, State#{orders := Orders}};
handle_call({update_config, {delete, ToDelete}, _}, _From, State) ->
{ok, #{orders := Orders,
stopped := Stopped
} = State2} = do_unload_server(ToDelete, State),
State3 = State2#{stopped := maps:remove(ToDelete, Stopped),
orders := maps:remove(ToDelete, Orders)
},
{reply, ok, State3};
handle_call({update_config, {add, RawConf}, NewConfL},
_From,
#{running := Running, waiting := Waitting, stopped := Stopped} = State) ->
{_, #{name := Name} = Conf} = emqx_config:check_config(?MODULE, RawConf),
case emqx_exhook_server:load(Name, Conf) of
{ok, ServerState} ->
save(Name, ServerState),
Status = running,
Hooks = hooks(Name),
State2 = State#{running := Running#{Name => Conf}};
{error, _} ->
Status = running,
Hooks = [],
StateT = State#{waiting := Waitting#{Name => Conf}},
State2 = ensure_reload_timer(StateT);
disable ->
Status = stopped,
Hooks = [],
State2 = State#{stopped := Stopped#{Name => Conf}}
end,
Orders = reorder(NewConfL),
Resulte = maps:merge(Conf, #{status => Status, hooks => Hooks}),
{reply, Resulte, State2#{orders := Orders}};
handle_call({lookup, Name}, _From, State) ->
case where_is_server(Name, State) of
not_found ->
Result = not_found;
{Where, #{Name := Conf}} ->
Result = maps:merge(Conf,
#{ status => Where
, hooks => hooks(Name)
})
end,
{reply, Result, State};
handle_call({update_config, {update, Name, _Conf}, NewConfL}, _From, State) ->
{Result, State2} = restart_server(Name, NewConfL, State),
{reply, Result, State2};
handle_call({update_config, {enable, Name, _Enable}, NewConfL}, _From, State) ->
{Result, State2} = restart_server(Name, NewConfL, State),
{reply, Result, State2};
handle_call({server_status, Name}, _From, State) ->
case where_is_server(Name, State) of
not_found ->
Result = not_found;
{Status, _} ->
Result = Status
end,
{reply, Result, State};
handle_call(all_servers_status, _From, #{running := Running,
waiting := Waiting,
stopped := Stopped} = State) ->
{reply, #{running => maps:keys(Running),
waiting => maps:keys(Waiting),
stopped => maps:keys(Stopped)}, State};
handle_call(_Request, _From, State) ->
Reply = ok,
{reply, Reply, State}.
handle_cast(_Msg, State) ->
{noreply, State}.
handle_info({timeout, _Ref, {reload, Name}}, State) ->
{Result, NState} = do_load_server(Name, State),
case Result of
ok ->
{noreply, NState};
{error, not_found} ->
{noreply, NState};
{error, Reason} ->
?LOG(warning, "Failed to reload exhook callback server \"~ts\", "
"Reason: ~0p", [Name, Reason]),
{noreply, ensure_reload_timer(NState)}
end;
handle_info(_Info, State) ->
{noreply, State}.
terminate(_Reason, State = #{running := Running}) ->
_ = maps:fold(fun(Name, _, AccIn) ->
{ok, NAccIn} = do_unload_server(Name, AccIn),
NAccIn
end, State, Running),
_ = unload_exhooks(),
ok.
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------
unload_exhooks() ->
[emqx:unhook(Name, {M, F}) ||
{Name, {M, F, _A}} <- ?ENABLED_HOOKS].
-spec do_load_server(server_name(), state()) -> {{error, not_found}, state()}
| {{error, already_started}, state()}
| {ok, state()}.
do_load_server(Name, State = #{orders := Orders}) ->
case where_is_server(Name, State) of
not_found ->
{{error, not_found}, State};
{running, _} ->
{ok, State};
{Where, Map} ->
State2 = clean_reload_timer(Name, State),
{Options, Map2} = maps:take(Name, Map),
State3 = State2#{Where := Map2},
#{running := Running,
stopped := Stopped} = State3,
case emqx_exhook_server:load(Name, Options) of
{ok, ServerState} ->
save(Name, ServerState),
update_order(Orders),
?LOG(info, "Load exhook callback server "
"\"~ts\" successfully!", [Name]),
{ok, State3#{running := maps:put(Name, Options, Running)}};
{error, Reason} ->
{{error, Reason}, State};
disable ->
{ok, State3#{stopped := Stopped#{Name => Options}}}
end
end.
-spec do_unload_server(server_name(), state()) -> {ok, state()}.
do_unload_server(Name, #{stopped := Stopped} = State) ->
case where_is_server(Name, State) of
{stopped, _} -> {ok, State};
{waiting, Waiting} ->
{Options, Waiting2} = maps:take(Name, Waiting),
{ok, clean_reload_timer(Name,
State#{waiting := Waiting2,
stopped := maps:put(Name, Options, Stopped)
}
)};
{running, Running} ->
Service = server(Name),
ok = unsave(Name),
ok = emqx_exhook_server:unload(Service),
{Options, Running2} = maps:take(Name, Running),
{ok, State#{running := Running2,
stopped := maps:put(Name, Options, Stopped)
}};
not_found -> {ok, State}
end.
-spec ensure_reload_timer(state()) -> state().
ensure_reload_timer(State = #{waiting := Waiting,
stopped := Stopped,
trefs := TRefs}) ->
Iter = maps:iterator(Waiting),
{Waitting2, Stopped2, TRefs2} =
ensure_reload_timer(maps:next(Iter), Waiting, Stopped, TRefs),
State#{waiting := Waitting2,
stopped := Stopped2,
trefs := TRefs2}.
ensure_reload_timer(none, Waiting, Stopped, TimerRef) ->
{Waiting, Stopped, TimerRef};
ensure_reload_timer({Name, #{auto_reconnect := Intv}, Iter},
Waiting,
Stopped,
TimerRef) ->
Next = maps:next(Iter),
case maps:is_key(Name, TimerRef) of
true ->
ensure_reload_timer(Next, Waiting, Stopped, TimerRef);
_ ->
Ref = erlang:start_timer(Intv, self(), {reload, Name}),
TimerRef2 = maps:put(Name, Ref, TimerRef),
ensure_reload_timer(Next, Waiting, Stopped, TimerRef2)
end;
ensure_reload_timer({Name, Opts, Iter}, Waiting, Stopped, TimerRef) ->
ensure_reload_timer(maps:next(Iter),
maps:remove(Name, Waiting),
maps:put(Name, Opts, Stopped),
TimerRef).
-spec clean_reload_timer(server_name(), state()) -> state().
clean_reload_timer(Name, State = #{trefs := TRefs}) ->
case maps:take(Name, TRefs) of
error -> State;
{TRef, NTRefs} ->
_ = erlang:cancel_timer(TRef),
State#{trefs := NTRefs}
end.
-spec do_move(binary(), move_direct(), binary(), list(server_options())) ->
not_found | list(server_options()).
do_move(Name, Direct, ToName, ConfL) ->
move(ConfL, Name, Direct, ToName, []).
move([#{<<"name">> := Name} = Server | T], Name, Direct, ToName, HeadL) ->
move_to(Direct, ToName, Server, lists:reverse(HeadL) ++ T);
move([Server | T], Name, Direct, ToName, HeadL) ->
move(T, Name, Direct, ToName, [Server | HeadL]);
move([], _Name, _Direct, _ToName, _HeadL) ->
not_found.
move_to(top, _, Server, ServerL) ->
[Server | ServerL];
move_to(bottom, _, Server, ServerL) ->
ServerL ++ [Server];
move_to(Direct, ToName, Server, ServerL) ->
move_to(ServerL, Direct, ToName, Server, []).
move_to([#{<<"name">> := Name} | _] = T, before, Name, Server, HeadL) ->
lists:reverse(HeadL) ++ [Server | T];
move_to([#{<<"name">> := Name} = H | T], 'after', Name, Server, HeadL) ->
lists:reverse(HeadL) ++ [H, Server | T];
move_to([H | T], Direct, Name, Server, HeadL) ->
move_to(T, Direct, Name, Server, [H | HeadL]);
move_to([], _Direct, _Name, _Server, _HeadL) ->
not_found.
-spec reorder(list(server_options())) -> orders().
reorder(ServerL) ->
Orders = reorder(ServerL, 1, #{}),
update_order(Orders),
Orders.
reorder([#{name := Name} | T], Order, Orders) ->
reorder(T, Order + 1, Orders#{Name => Order});
reorder([], _Order, Orders) ->
Orders.
get_servers_info(Status, Map) ->
Fold = fun(Name, Conf, Acc) ->
[maps:merge(Conf, #{status => Status,
hooks => hooks(Name)}) | Acc]
end,
maps:fold(Fold, [], Map).
where_is_server(Name, #{running := Running}) when is_map_key(Name, Running) ->
{running, Running};
where_is_server(Name, #{waiting := Waiting}) when is_map_key(Name, Waiting) ->
{waiting, Waiting};
where_is_server(Name, #{stopped := Stopped}) when is_map_key(Name, Stopped) ->
{stopped, Stopped};
where_is_server(_, _) ->
not_found.
-type replace_fun() :: fun((server_options()) -> server_options()).
-spec replace_conf(binary(), replace_fun(), list(server_options())) -> not_found
| list(server_options()).
replace_conf(Name, ReplaceFun, ConfL) ->
replace_conf(ConfL, Name, ReplaceFun, []).
replace_conf([#{<<"name">> := Name} = H | T], Name, ReplaceFun, HeadL) ->
New = ReplaceFun(H),
lists:reverse(HeadL) ++ [New | T];
replace_conf([H | T], Name, ReplaceFun, HeadL) ->
replace_conf(T, Name, ReplaceFun, [H | HeadL]);
replace_conf([], _, _, _) ->
not_found.
-spec restart_server(binary(), list(server_options()), state()) -> {ok, state()}
| {{error, term()}, state()}.
restart_server(Name, ConfL, State) ->
case lists:search(fun(#{name := CName}) -> CName =:= Name end, ConfL) of
false ->
{{error, not_found}, State};
{value, Conf} ->
case where_is_server(Name, State) of
not_found ->
{{error, not_found}, State};
{Where, Map} ->
State2 = State#{Where := Map#{Name := Conf}},
{ok, State3} = do_unload_server(Name, State2),
case do_load_server(Name, State3) of
{ok, State4} ->
{ok, State4};
{Error, State4} ->
{Error, State4}
end
end
end.
sort_name_by_order(Names, Orders) ->
lists:sort(fun(A, B) when is_binary(A) ->
maps:get(A, Orders) < maps:get(B, Orders);
(#{name := A}, #{name := B}) ->
maps:get(A, Orders) < maps:get(B, Orders)
end,
Names).
%%--------------------------------------------------------------------
%% Server state persistent
save(Name, ServerState) ->
Saved = persistent_term:get(?APP, []),
persistent_term:put(?APP, lists:reverse([Name | Saved])),
persistent_term:put({?APP, Name}, ServerState).
unsave(Name) ->
case persistent_term:get(?APP, []) of
[] ->
ok;
Saved ->
case lists:member(Name, Saved) of
false ->
ok;
true ->
persistent_term:put(?APP, lists:delete(Name, Saved))
end
end,
persistent_term:erase({?APP, Name}),
ok.
running() ->
persistent_term:get(?APP, []).
server(Name) ->
case persistent_term:get({?APP, Name}, undefined) of
undefined -> undefined;
Service -> Service
end.
update_order(Orders) ->
Running = running(),
Running2 = sort_name_by_order(Running, Orders),
persistent_term:put(?APP, Running2).
hooks(Name) ->
case server(Name) of
undefined ->
[];
Service ->
emqx_exhook_server:hookpoints(Service)
end.

329
apps/emqx_exhook/src/emqx_exhook_mngr.erl
View File

@ -1,329 +0,0 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% @doc Manage the server status and reload strategy
-module(emqx_exhook_mngr).
-behaviour(gen_server).
-include("emqx_exhook.hrl").
-include_lib("emqx/include/logger.hrl").
%% APIs
-export([start_link/3]).
%% Mgmt API
-export([ enable/2
, disable/2
, list/1
]).
%% Helper funcs
-export([ running/0
, server/1
, put_request_failed_action/1
, get_request_failed_action/0
, put_pool_size/1
, get_pool_size/0
]).
%% gen_server callbacks
-export([ init/1
, handle_call/3
, handle_cast/2
, handle_info/2
, terminate/2
, code_change/3
]).
-record(state, {
%% Running servers
running :: map(), %% XXX: server order?
%% Wait to reload servers
waiting :: map(),
%% Marked stopped servers
stopped :: map(),
%% Auto reconnect timer interval
auto_reconnect :: false | non_neg_integer(),
%% Request options
request_options :: grpc_client:options(),
%% Timer references
trefs :: map()
}).
-type servers() :: [{Name :: atom(), server_options()}].
-type server_options() :: [ {scheme, http | https}
| {host, string()}
| {port, inet:port_number()}
].
-define(DEFAULT_TIMEOUT, 60000).
-define(CNTER, emqx_exhook_counter).
%%--------------------------------------------------------------------
%% APIs
%%--------------------------------------------------------------------
-spec start_link(servers(), false | non_neg_integer(), grpc_client:options())
->ignore
| {ok, pid()}
| {error, any()}.
start_link(Servers, AutoReconnect, ReqOpts) ->
gen_server:start_link(?MODULE, [Servers, AutoReconnect, ReqOpts], []).
-spec enable(pid(), binary()) -> ok | {error, term()}.
enable(Pid, Name) ->
call(Pid, {load, Name}).
-spec disable(pid(), binary()) -> ok | {error, term()}.
disable(Pid, Name) ->
call(Pid, {unload, Name}).
list(Pid) ->
call(Pid, list).
call(Pid, Req) ->
gen_server:call(Pid, Req, ?DEFAULT_TIMEOUT).
%%--------------------------------------------------------------------
%% gen_server callbacks
%%--------------------------------------------------------------------
init([Servers, AutoReconnect, ReqOpts0]) ->
process_flag(trap_exit, true),
%% XXX: Due to the ExHook Module in the enterprise,
%% this process may start multiple times and they will share this table
try
_ = ets:new(?CNTER, [named_table, public]), ok
catch
error:badarg:_ ->
ok
end,
%% put the global option
put_request_failed_action(
maps:get(request_failed_action, ReqOpts0, deny)
),
put_pool_size(
maps:get(pool_size, ReqOpts0, erlang:system_info(schedulers))
),
%% Load the hook servers
ReqOpts = maps:without([request_failed_action], ReqOpts0),
{Waiting, Running} = load_all_servers(Servers, ReqOpts),
{ok, ensure_reload_timer(
#state{waiting = Waiting,
running = Running,
stopped = #{},
request_options = ReqOpts,
auto_reconnect = AutoReconnect,
trefs = #{}
}
)}.
%% @private
load_all_servers(Servers, ReqOpts) ->
load_all_servers(Servers, ReqOpts, #{}, #{}).
load_all_servers([], _Request, Waiting, Running) ->
{Waiting, Running};
load_all_servers([#{name := Name0} = Options0 | More], ReqOpts, Waiting, Running) ->
Name = iolist_to_binary(Name0),
Options = Options0#{name => Name},
{NWaiting, NRunning} =
case emqx_exhook_server:load(Name, Options, ReqOpts) of
{ok, ServerState} ->
save(Name, ServerState),
{Waiting, Running#{Name => Options}};
{error, _} ->
{Waiting#{Name => Options}, Running}
end,
load_all_servers(More, ReqOpts, NWaiting, NRunning).
handle_call({load, Name}, _From, State) ->
{Result, NState} = do_load_server(Name, State),
{reply, Result, NState};
handle_call({unload, Name}, _From, State) ->
case do_unload_server(Name, State) of
{error, Reason} ->
{reply, {error, Reason}, State};
{ok, NState} ->
{reply, ok, NState}
end;
handle_call(list, _From, State = #state{
running = Running,
waiting = Waiting,
stopped = Stopped}) ->
ServerNames = maps:keys(Running)
++ maps:keys(Waiting)
++ maps:keys(Stopped),
{reply, ServerNames, State};
handle_call(_Request, _From, State) ->
Reply = ok,
{reply, Reply, State}.
handle_cast(_Msg, State) ->
{noreply, State}.
handle_info({timeout, _Ref, {reload, Name}}, State) ->
{Result, NState} = do_load_server(Name, State),
case Result of
ok ->
{noreply, NState};
{error, not_found} ->
{noreply, NState};
{error, Reason} ->
?SLOG(warning, #{msg => "failed_to_reload_exhook_callback_server",
server_name => Name,
reason => Reason}),
{noreply, ensure_reload_timer(NState)}
end;
handle_info(_Info, State) ->
{noreply, State}.
terminate(_Reason, State = #state{running = Running}) ->
_ = maps:fold(fun(Name, _, AccIn) ->
case do_unload_server(Name, AccIn) of
{ok, NAccIn} -> NAccIn;
_ -> AccIn
end
end, State, Running),
_ = unload_exhooks(),
ok.
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------
unload_exhooks() ->
[emqx:unhook(Name, {M, F}) ||
{Name, {M, F, _A}} <- ?ENABLED_HOOKS].
do_load_server(Name, State0 = #state{
waiting = Waiting,
running = Running,
stopped = Stopped,
request_options = ReqOpts}) ->
State = clean_reload_timer(Name, State0),
case maps:get(Name, Running, undefined) of
undefined ->
case maps:get(Name, Stopped,
maps:get(Name, Waiting, undefined)) of
undefined ->
{{error, not_found}, State};
Options ->
case emqx_exhook_server:load(Name, Options, ReqOpts) of
{ok, ServerState} ->
save(Name, ServerState),
?SLOG(info, #{msg => "load_exhook_callback_server_successfully",
server_name => Name}),
{ok, State#state{
running = maps:put(Name, Options, Running),
waiting = maps:remove(Name, Waiting),
stopped = maps:remove(Name, Stopped)
}
};
{error, Reason} ->
{{error, Reason}, State}
end
end;
_ ->
{{error, already_started}, State}
end.
do_unload_server(Name, State = #state{running = Running, stopped = Stopped}) ->
case maps:take(Name, Running) of
error -> {error, not_running};
{Options, NRunning} ->
ok = emqx_exhook_server:unload(server(Name)),
ok = unsave(Name),
{ok, State#state{running = NRunning,
stopped = maps:put(Name, Options, Stopped)
}}
end.
ensure_reload_timer(State = #state{auto_reconnect = false}) ->
State;
ensure_reload_timer(State = #state{waiting = Waiting,
trefs = TRefs,
auto_reconnect = Intv}) ->
NRefs = maps:fold(fun(Name, _, AccIn) ->
case maps:get(Name, AccIn, undefined) of
undefined ->
Ref = erlang:start_timer(Intv, self(), {reload, Name}),
AccIn#{Name => Ref};
_HasRef ->
AccIn
end
end, TRefs, Waiting),
State#state{trefs = NRefs}.
clean_reload_timer(Name, State = #state{trefs = TRefs}) ->
case maps:take(Name, TRefs) of
error -> State;
{TRef, NTRefs} ->
_ = erlang:cancel_timer(TRef),
State#state{trefs = NTRefs}
end.
%%--------------------------------------------------------------------
%% Server state persistent
put_request_failed_action(Val) ->
persistent_term:put({?APP, request_failed_action}, Val).
get_request_failed_action() ->
persistent_term:get({?APP, request_failed_action}).
put_pool_size(Val) ->
persistent_term:put({?APP, pool_size}, Val).
get_pool_size() ->
%% Avoid the scenario that the parameter is not set after
%% the hot upgrade completed.
persistent_term:get({?APP, pool_size}, erlang:system_info(schedulers)).
save(Name, ServerState) ->
Saved = persistent_term:get(?APP, []),
persistent_term:put(?APP, lists:reverse([Name | Saved])),
persistent_term:put({?APP, Name}, ServerState).
unsave(Name) ->
case persistent_term:get(?APP, []) of
[] ->
persistent_term:erase(?APP);
Saved ->
persistent_term:put(?APP, lists:delete(Name, Saved))
end,
persistent_term:erase({?APP, Name}),
ok.
running() ->
persistent_term:get(?APP, []).
server(Name) ->
case catch persistent_term:get({?APP, Name}) of
{'EXIT', {badarg,_}} -> undefined;
Service -> Service
end.

71
apps/emqx_exhook/src/emqx_exhook_schema.erl
View File

@ -32,61 +32,58 @@
-reflect_type([duration/0]).
-export([namespace/0, roots/0, fields/1]).
-export([namespace/0, roots/0, fields/1, server_config/0]).
namespace() -> exhook.
namespace() -> emqx_exhook.
roots() -> [exhook].
roots() -> [emqx_exhook].
fields(exhook) ->
[ {request_failed_action,
sc(hoconsc:enum([deny, ignore]),
#{default => deny})}
, {request_timeout,
sc(duration(),
#{default => "5s"})}
, {auto_reconnect,
sc(hoconsc:union([false, duration()]),
#{ default => "60s"
})}
, {pool_size,
sc(integer(),
#{ nullable => true
})}
, {servers,
sc(hoconsc:array(ref(servers)),
fields(emqx_exhook) ->
[{servers,
sc(hoconsc:array(ref(server)),
#{default => []})}
];
fields(servers) ->
[ {name,
sc(string(),
#{})}
, {url,
sc(string(),
#{})}
fields(server) ->
[ {name, sc(binary(), #{})}
, {enable, sc(boolean(), #{default => true})}
, {url, sc(binary(), #{})}
, {request_timeout,
sc(duration(), #{default => "5s"})}
, {failed_action, failed_action()}
, {ssl,
sc(ref(ssl_conf),
#{})}
sc(ref(ssl_conf), #{})}
, {auto_reconnect,
sc(hoconsc:union([false, duration()]),
#{default => "60s"})}
, {pool_size,
sc(integer(), #{default => 8, example => 8})}
];
fields(ssl_conf) ->
[ {cacertfile,
sc(string(),
#{})
[ {enable, sc(boolean(), #{default => true})}
, {cacertfile,
sc(binary(),
#{example => <<"{{ platform_etc_dir }}/certs/cacert.pem">>})
}
, {certfile,
sc(string(),
#{})
sc(binary(),
#{example => <<"{{ platform_etc_dir }}/certs/cert.pem">>})
}
, {keyfile,
sc(string(),
#{})}
sc(binary(),
#{example => <<"{{ platform_etc_dir }}/certs/key.pem">>})}
].
%% types
sc(Type, Meta) -> Meta#{type => Type}.
ref(Field) ->
hoconsc:ref(?MODULE, Field).
failed_action() ->
sc(hoconsc:enum([deny, ignore]),
#{default => deny}).
server_config() ->
fields(server).

83
apps/emqx_exhook/src/emqx_exhook_server.erl
View File

@ -24,7 +24,7 @@
-define(PB_CLIENT_MOD, emqx_exhook_v_1_hook_provider_client).
%% Load/Unload
-export([ load/3
-export([ load/2
, unload/1
]).
@ -33,23 +33,24 @@
%% Infos
-export([ name/1
, hookpoints/1
, format/1
, failed_action/1
]).
-record(server, {
%% Server name (equal to grpc client channel name)
name :: binary(),
%% The function options
options :: map(),
%% gRPC channel pid
channel :: pid(),
%% Registered hook names and options
hookspec :: #{hookpoint() => map()},
%% Metrcis name prefix
prefix :: list()
}).
-type server() :: #server{}.
-type server() :: #{%% Server name (equal to grpc client channel name)
name := binary(),
%% The function options
options := map(),
%% gRPC channel pid
channel := pid(),
%% Registered hook names and options
hookspec := #{hookpoint() => map()},
%% Metrcis name prefix
prefix := list()
}.
-type hookpoint() :: 'client.connect'
| 'client.connack'
@ -81,9 +82,13 @@
%% Load/Unload APIs
%%--------------------------------------------------------------------
-spec load(binary(), map(), map()) -> {ok, server()} | {error, term()} .
load(Name, Opts0, ReqOpts) ->
{SvrAddr, ClientOpts} = channel_opts(Opts0),
-spec load(binary(), map()) -> {ok, server()} | {error, term()} | disable.
load(_Name, #{enable := false}) ->
disable;
load(Name, #{request_timeout := Timeout, failed_action := FailedAction} = Opts) ->
ReqOpts = #{timeout => Timeout, failed_action => FailedAction},
{SvrAddr, ClientOpts} = channel_opts(Opts),
case emqx_exhook_sup:start_grpc_client_channel(
Name,
SvrAddr,
@ -92,16 +97,15 @@ load(Name, Opts0, ReqOpts) ->
case do_init(Name, ReqOpts) of
{ok, HookSpecs} ->
%% Reigster metrics
Prefix = lists:flatten(
io_lib:format("exhook.~ts.", [Name])),
Prefix = lists:flatten(io_lib:format("exhook.~ts.", [Name])),
ensure_metrics(Prefix, HookSpecs),
%% Ensure hooks
ensure_hooks(HookSpecs),
{ok, #server{name = Name,
options = ReqOpts,
channel = _ChannPoolPid,
hookspec = HookSpecs,
prefix = Prefix }};
{ok, #{name => Name,
options => ReqOpts,
channel => _ChannPoolPid,
hookspec => HookSpecs,
prefix => Prefix }};
{error, _} = E ->
emqx_exhook_sup:stop_grpc_client_channel(Name), E
end;
@ -110,14 +114,16 @@ load(Name, Opts0, ReqOpts) ->
%% @private
channel_opts(Opts = #{url := URL}) ->
ClientOpts = #{pool_size => emqx_exhook_mngr:get_pool_size()},
ClientOpts = maps:merge(#{pool_size => erlang:system_info(schedulers)},
Opts),
case uri_string:parse(URL) of
#{scheme := "http", host := Host, port := Port} ->
#{scheme := <<"http">>, host := Host, port := Port} ->
{format_http_uri("http", Host, Port), ClientOpts};
#{scheme := "https", host := Host, port := Port} ->
#{scheme := <<"https">>, host := Host, port := Port} ->
SslOpts =
case maps:get(ssl, Opts, undefined) of
undefined -> [];
#{enable := false} -> [];
MapOpts ->
filter(
[{cacertfile, maps:get(cacertfile, MapOpts, undefined)},
@ -131,8 +137,8 @@ channel_opts(Opts = #{url := URL}) ->
transport_opts => SslOpts}
},
{format_http_uri("https", Host, Port), NClientOpts};
_ ->
error(bad_server_url)
Error ->
error({bad_server_url, URL, Error})
end.
format_http_uri(Scheme, Host, Port) ->
@ -142,7 +148,7 @@ filter(Ls) ->
[ E || E <- Ls, E /= undefined].
-spec unload(server()) -> ok.
unload(#server{name = Name, options = ReqOpts, hookspec = HookSpecs}) ->
unload(#{name := Name, options := ReqOpts, hookspec := HookSpecs}) ->
_ = do_deinit(Name, ReqOpts),
_ = may_unload_hooks(HookSpecs),
_ = emqx_exhook_sup:stop_grpc_client_channel(Name),
@ -227,7 +233,7 @@ may_unload_hooks(HookSpecs) ->
end
end, maps:keys(HookSpecs)).
format(#server{name = Name, hookspec = Hooks}) ->
format(#{name := Name, hookspec := Hooks}) ->
lists:flatten(
io_lib:format("name=~ts, hooks=~0p, active=true", [Name, Hooks])).
@ -235,15 +241,17 @@ format(#server{name = Name, hookspec = Hooks}) ->
%% APIs
%%--------------------------------------------------------------------
name(#server{name = Name}) ->
name(#{name := Name}) ->
Name.
-spec call(hookpoint(), map(), server())
-> ignore
hookpoints(#{hookspec := Hooks}) ->
maps:keys(Hooks).
-spec call(hookpoint(), map(), server()) -> ignore
| {ok, Resp :: term()}
| {error, term()}.
call(Hookpoint, Req, #server{name = ChannName, options = ReqOpts,
hookspec = Hooks, prefix = Prefix}) ->
call(Hookpoint, Req, #{name := ChannName, options := ReqOpts,
hookspec := Hooks, prefix := Prefix}) ->
GrpcFunc = hk2func(Hookpoint),
case maps:get(Hookpoint, Hooks, undefined) of
undefined -> ignore;
@ -299,6 +307,9 @@ do_call(ChannName, Fun, Req, ReqOpts) ->
{error, Reason}
end.
failed_action(#{options := Opts}) ->
maps:get(failed_action, Opts).
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------

19
apps/emqx_exhook/src/emqx_exhook_sup.erl
View File

@ -42,25 +42,10 @@ start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
init([]) ->
Mngr = ?CHILD(emqx_exhook_mngr, worker,
[servers(), auto_reconnect(), request_options()]),
_ = emqx_exhook_mgr:init_counter_table(),
Mngr = ?CHILD(emqx_exhook_mgr, worker, []),
{ok, {{one_for_one, 10, 100}, [Mngr]}}.
servers() ->
env(servers, []).
auto_reconnect() ->
env(auto_reconnect, 60000).
request_options() ->
#{timeout => env(request_timeout, 5000),
request_failed_action => env(request_failed_action, deny),
pool_size => env(pool_size, erlang:system_info(schedulers))
}.
env(Key, Def) ->
emqx_conf:get([exhook, Key], Def).
%%--------------------------------------------------------------------
%% APIs
%%--------------------------------------------------------------------

69
apps/emqx_exhook/test/emqx_exhook_SUITE.erl
View File

@ -21,14 +21,14 @@
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-define(CLUSTER_RPC_SHARD, emqx_cluster_rpc_shard).
-define(CONF_DEFAULT, <<"
exhook: {
servers: [
{ name: \"default\"
url: \"http://127.0.0.1:9000\"
}
]
emqx_exhook
{servers = [
{name = default,
url = \"http://127.0.0.1:9000\"
}]
}
">>).
@ -39,27 +39,53 @@ exhook: {
all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Cfg) ->
application:load(emqx_conf),
ok = ekka:start(),
ok = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], infinity),
meck:new(emqx_alarm, [non_strict, passthrough, no_link]),
meck:expect(emqx_alarm, activate, 3, ok),
meck:expect(emqx_alarm, deactivate, 3, ok),
_ = emqx_exhook_demo_svr:start(),
ok = emqx_config:init_load(emqx_exhook_schema, ?CONF_DEFAULT),
emqx_common_test_helpers:start_apps([emqx_exhook]),
Cfg.
end_per_suite(_Cfg) ->
ekka:stop(),
mria:stop(),
mria_mnesia:delete_schema(),
meck:unload(emqx_alarm),
emqx_common_test_helpers:stop_apps([emqx_exhook]),
emqx_exhook_demo_svr:stop().
init_per_testcase(_, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(),
timer:sleep(200),
Config.
end_per_testcase(_, Config) ->
case erlang:whereis(node()) of
undefined -> ok;
P ->
erlang:unlink(P),
erlang:exit(P, kill)
end,
Config.
%%--------------------------------------------------------------------
%% Test cases
%%--------------------------------------------------------------------
t_noserver_nohook(_) ->
emqx_exhook:disable(<<"default">>),
emqx_exhook_mgr:disable(<<"default">>),
?assertEqual([], ets:tab2list(emqx_hooks)),
ok = emqx_exhook:enable(<<"default">>),
{ok, _} = emqx_exhook_mgr:enable(<<"default">>),
?assertNotEqual([], ets:tab2list(emqx_hooks)).
t_access_failed_if_no_server_running(_) ->
emqx_exhook:disable(<<"default">>),
emqx_exhook_mgr:disable(<<"default">>),
ClientInfo = #{clientid => <<"user-id-1">>,
username => <<"usera">>,
peerhost => {127,0,0,1},
@ -76,30 +102,7 @@ t_access_failed_if_no_server_running(_) ->
Message = emqx_message:make(<<"t/1">>, <<"abc">>),
?assertMatch({stop, Message},
emqx_exhook_handler:on_message_publish(Message)),
emqx_exhook:enable(<<"default">>).
t_cli_list(_) ->
meck_print(),
?assertEqual( [[emqx_exhook_server:format(emqx_exhook_mngr:server(Name)) || Name <- emqx_exhook:list()]]
, emqx_exhook_cli:cli(["server", "list"])
),
unmeck_print().
t_cli_enable_disable(_) ->
meck_print(),
?assertEqual([already_started], emqx_exhook_cli:cli(["server", "enable", "default"])),
?assertEqual(ok, emqx_exhook_cli:cli(["server", "disable", "default"])),
?assertEqual([["name=default, hooks=#{}, active=false"]], emqx_exhook_cli:cli(["server", "list"])),
?assertEqual([not_running], emqx_exhook_cli:cli(["server", "disable", "default"])),
?assertEqual(ok, emqx_exhook_cli:cli(["server", "enable", "default"])),
unmeck_print().
t_cli_stats(_) ->
meck_print(),
_ = emqx_exhook_cli:cli(["server", "stats"]),
_ = emqx_exhook_cli:cli(x),
unmeck_print().
emqx_exhook_mgr:enable(<<"default">>).
%%--------------------------------------------------------------------
%% Utils

197
apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl Normal file
View File

@ -0,0 +1,197 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_exhook_api_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
-define(HOST, "http://127.0.0.1:18083/").
-define(API_VERSION, "v5").
-define(BASE_PATH, "api").
-define(CLUSTER_RPC_SHARD, emqx_cluster_rpc_shard).
-define(CONF_DEFAULT, <<"
emqx_exhook {servers = [
{name = default,
url = \"http://127.0.0.1:9000\"
}
]
}
">>).
all() ->
[t_list, t_get, t_add, t_move_1, t_move_2, t_delete, t_update].
init_per_suite(Config) ->
application:load(emqx_conf),
ok = ekka:start(),
ok = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], infinity),
meck:new(emqx_alarm, [non_strict, passthrough, no_link]),
meck:expect(emqx_alarm, activate, 3, ok),
meck:expect(emqx_alarm, deactivate, 3, ok),
_ = emqx_exhook_demo_svr:start(),
ok = emqx_config:init_load(emqx_exhook_schema, ?CONF_DEFAULT),
emqx_mgmt_api_test_util:init_suite([emqx_exhook]),
[Conf] = emqx:get_config([emqx_exhook, servers]),
[{template, Conf} | Config].
end_per_suite(Config) ->
ekka:stop(),
mria:stop(),
mria_mnesia:delete_schema(),
meck:unload(emqx_alarm),
emqx_mgmt_api_test_util:end_suite([emqx_exhook]),
emqx_exhook_demo_svr:stop(),
emqx_exhook_demo_svr:stop(<<"test1">>),
Config.
init_per_testcase(t_add, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(),
_ = emqx_exhook_demo_svr:start(<<"test1">>, 9001),
timer:sleep(200),
Config;
init_per_testcase(_, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(),
timer:sleep(200),
Config.
end_per_testcase(_, Config) ->
case erlang:whereis(node()) of
undefined -> ok;
P ->
erlang:unlink(P),
erlang:exit(P, kill)
end,
Config.
t_list(_) ->
{ok, Data} = request_api(get, api_path(["exhooks"]), "",
auth_header_()),
List = decode_json(Data),
?assertEqual(1, length(List)),
[Svr] = List,
?assertMatch(#{name := <<"default">>,
status := <<"running">>}, Svr).
t_get(_) ->
{ok, Data} = request_api(get, api_path(["exhooks", "default"]), "",
auth_header_()),
Svr = decode_json(Data),
?assertMatch(#{name := <<"default">>,
status := <<"running">>}, Svr).
t_add(Cfg) ->
Template = proplists:get_value(template, Cfg),
Instance = Template#{name => <<"test1">>,
url => "http://127.0.0.1:9001"
},
{ok, Data} = request_api(post, api_path(["exhooks"]), "",
auth_header_(), Instance),
Svr = decode_json(Data),
?assertMatch(#{name := <<"test1">>,
status := <<"running">>}, Svr),
?assertMatch([<<"default">>, <<"test1">>], emqx_exhook_mgr:running()).
t_move_1(_) ->
Result = request_api(post, api_path(["exhooks", "default", "move"]), "",
auth_header_(),
#{position => bottom, related => <<>>}),
?assertMatch({ok, <<>>}, Result),
?assertMatch([<<"test1">>, <<"default">>], emqx_exhook_mgr:running()).
t_move_2(_) ->
Result = request_api(post, api_path(["exhooks", "default", "move"]), "",
auth_header_(),
#{position => before, related => <<"test1">>}),
?assertMatch({ok, <<>>}, Result),
?assertMatch([<<"default">>, <<"test1">>], emqx_exhook_mgr:running()).
t_delete(_) ->
Result = request_api(delete, api_path(["exhooks", "test1"]), "",
auth_header_()),
?assertMatch({ok, <<>>}, Result),
?assertMatch([<<"default">>], emqx_exhook_mgr:running()).
t_update(Cfg) ->
Template = proplists:get_value(template, Cfg),
Instance = Template#{enable => false},
{ok, <<>>} = request_api(put, api_path(["exhooks", "default"]), "",
auth_header_(), Instance),
?assertMatch([], emqx_exhook_mgr:running()).
decode_json(Data) ->
BinJosn = emqx_json:decode(Data, [return_maps]),
emqx_map_lib:unsafe_atom_key_map(BinJosn).
request_api(Method, Url, Auth) ->
request_api(Method, Url, [], Auth, []).
request_api(Method, Url, QueryParams, Auth) ->
request_api(Method, Url, QueryParams, Auth, []).
request_api(Method, Url, QueryParams, Auth, []) ->
NewUrl = case QueryParams of
"" -> Url;
_ -> Url ++ "?" ++ QueryParams
end,
do_request_api(Method, {NewUrl, [Auth]});
request_api(Method, Url, QueryParams, Auth, Body) ->
NewUrl = case QueryParams of
"" -> Url;
_ -> Url ++ "?" ++ QueryParams
end,
do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}).
do_request_api(Method, Request)->
case httpc:request(Method, Request, [], [{body_format, binary}]) of
{error, socket_closed_remotely} ->
{error, socket_closed_remotely};
{ok, {{"HTTP/1.1", Code, _}, _, Return} }
when Code =:= 200 orelse Code =:= 204 orelse Code =:= 201 ->
{ok, Return};
{ok, {Reason, _, _}} ->
{error, Reason}
end.
auth_header_() ->
AppId = <<"admin">>,
AppSecret = <<"public">>,
auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)).
auth_header_(User, Pass) ->
Encoded = base64:encode_to_string(lists:append([User,":",Pass])),
{"Authorization","Basic " ++ Encoded}.
api_path(Parts)->
?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts).

38
apps/emqx_exhook/test/emqx_exhook_demo_svr.erl
View File

@ -20,7 +20,9 @@
%%
-export([ start/0
, start/2
, stop/0
, stop/1
, take/0
, in/1
]).
@ -57,39 +59,45 @@
%%--------------------------------------------------------------------
start() ->
Pid = spawn(fun mngr_main/0),
register(?MODULE, Pid),
start(?NAME, ?PORT).
start(Name, Port) ->
Pid = spawn(fun() -> mgr_main(Name, Port) end),
register(to_atom_name(Name), Pid),
{ok, Pid}.
stop() ->
grpc:stop_server(?NAME),
?MODULE ! stop.
stop(?NAME).
stop(Name) ->
grpc:stop_server(Name),
to_atom_name(Name) ! stop.
take() ->
?MODULE ! {take, self()},
to_atom_name(?NAME) ! {take, self()},
receive {value, V} -> V
after 5000 -> error(timeout) end.
in({FunName, Req}) ->
?MODULE ! {in, FunName, Req}.
to_atom_name(?NAME) ! {in, FunName, Req}.
mngr_main() ->
mgr_main(Name, Port) ->
application:ensure_all_started(grpc),
Services = #{protos => [emqx_exhook_pb],
services => #{'emqx.exhook.v1.HookProvider' => emqx_exhook_demo_svr}
},
Options = [],
Svr = grpc:start_server(?NAME, ?PORT, Services, Options),
mngr_loop([Svr, queue:new(), queue:new()]).
Svr = grpc:start_server(Name, Port, Services, Options),
mgr_loop([Svr, queue:new(), queue:new()]).
mngr_loop([Svr, Q, Takes]) ->
mgr_loop([Svr, Q, Takes]) ->
receive
{in, FunName, Req} ->
{NQ1, NQ2} = reply(queue:in({FunName, Req}, Q), Takes),
mngr_loop([Svr, NQ1, NQ2]);
mgr_loop([Svr, NQ1, NQ2]);
{take, From} ->
{NQ1, NQ2} = reply(Q, queue:in(From, Takes)),
mngr_loop([Svr, NQ1, NQ2]);
mgr_loop([Svr, NQ1, NQ2]);
stop ->
exit(normal)
end.
@ -105,6 +113,12 @@ reply(Q1, Q2) ->
{NQ1, NQ2}
end.
to_atom_name(Name) when is_atom(Name) ->
Name;
to_atom_name(Name) ->
erlang:binary_to_atom(Name).
%%--------------------------------------------------------------------
%% callbacks
%%--------------------------------------------------------------------

11
apps/emqx_exhook/test/props/prop_exhook_hooks.erl
View File

@ -31,12 +31,11 @@
]).
-define(CONF_DEFAULT, <<"
exhook: {
servers: [
{ name: \"default\"
url: \"http://127.0.0.1:9000\"
}
]
emqx_exhook
{servers = [
{name = default,
url = \"http://127.0.0.1:9000\"
}]
}
">>).

10
apps/emqx_gateway/src/coap/emqx_coap_channel.erl
View File

@ -460,8 +460,9 @@ process_connect(#channel{ctx = Ctx,
{ok, _Sess} ->
RandVal = rand:uniform(?TOKEN_MAXIMUM),
Token = erlang:list_to_binary(erlang:integer_to_list(RandVal)),
NResult = Result#{events => [{event, connected}]},
iter(Iter,
reply({ok, created}, Token, Msg, Result),
reply({ok, created}, Token, Msg, NResult),
Channel#channel{token = Token});
{error, Reason} ->
?SLOG(error, #{ msg => "failed_open_session"
@ -568,7 +569,8 @@ process_out(Outs, Result, Channel, _) ->
Reply ->
[Reply | Outs2]
end,
{ok, {outgoing, Outs3}, Channel}.
Events = maps:get(events, Result, []),
{ok, [{outgoing, Outs3}] ++ Events, Channel}.
%% leaf node
process_nothing(_, _, Channel) ->
@ -607,4 +609,6 @@ process_reply(Reply, Result, #channel{session = Session} = Channel, _) ->
Session2 = emqx_coap_session:set_reply(Reply, Session),
Outs = maps:get(out, Result, []),
Outs2 = lists:reverse(Outs),
{ok, {outgoing, [Reply | Outs2]}, Channel#channel{session = Session2}}.
Events = maps:get(events, Result, []),
{ok, [{outgoing, [Reply | Outs2]}] ++ Events,
Channel#channel{session = Session2}}.

2
apps/emqx_gateway/src/emqx_gateway_api.erl
View File

@ -83,7 +83,7 @@ gateway(post, Request) ->
{ok, NGwConf} ->
{201, NGwConf};
{error, Reason} ->
return_http_error(500, Reason)
emqx_gateway_http:reason2resp(Reason)
end
end
catch

15
apps/emqx_gateway/src/emqx_gateway_api_clients.erl
View File

@ -745,7 +745,8 @@ common_client_props() ->
"due to exceeding the length">>})}
, {awaiting_rel_cnt,
mk(integer(),
#{ desc => <<"Number of awaiting PUBREC packet">>})}
%% FIXME: PUBREC ??
#{ desc => <<"Number of awaiting acknowledge packet">>})}
, {awaiting_rel_max,
mk(integer(),
#{ desc => <<"Maximum allowed number of awaiting PUBREC "
@ -755,25 +756,25 @@ common_client_props() ->
#{ desc => <<"Number of bytes received by EMQ X Broker">>})}
, {recv_cnt,
mk(integer(),
#{ desc => <<"Number of TCP packets received">>})}
#{ desc => <<"Number of socket packets received">>})}
, {recv_pkt,
mk(integer(),
#{ desc => <<"Number of MQTT packets received">>})}
#{ desc => <<"Number of protocol packets received">>})}
, {recv_msg,
mk(integer(),
#{ desc => <<"Number of PUBLISH packets received">>})}
#{ desc => <<"Number of message packets received">>})}
, {send_oct,
mk(integer(),
#{ desc => <<"Number of bytes sent">>})}
, {send_cnt,
mk(integer(),
#{ desc => <<"Number of TCP packets sent">>})}
#{ desc => <<"Number of socket packets sent">>})}
, {send_pkt,
mk(integer(),
#{ desc => <<"Number of MQTT packets sent">>})}
#{ desc => <<"Number of protocol packets sent">>})}
, {send_msg,
mk(integer(),
#{ desc => <<"Number of PUBLISH packets sent">>})}
#{ desc => <<"Number of message packets sent">>})}
, {mailbox_len,
mk(integer(),
#{ desc => <<"Process mailbox size">>})}

2
apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
View File

@ -32,7 +32,7 @@
-import(emqx_gateway_api_authn, [schema_authn/0]).
%% minirest/dashbaord_swagger behaviour callbacks
%% minirest/dashboard_swagger behaviour callbacks
-export([ api_spec/0
, paths/0
, schema/1

55
apps/emqx_gateway/src/emqx_gateway_conf.erl
View File

@ -248,7 +248,8 @@ update(Req) ->
res(emqx_conf:update([gateway], Req, #{override_to => cluster})).
res({ok, Result}) -> {ok, Result};
res({error, {pre_config_update,emqx_gateway_conf,Reason}}) -> {error, Reason};
res({error, {pre_config_update,?MODULE,Reason}}) -> {error, Reason};
res({error, {post_config_update,?MODULE,Reason}}) -> {error, Reason};
res({error, Reason}) -> {error, Reason}.
bin({LType, LName}) ->
@ -314,12 +315,12 @@ pre_config_update(_, {load_gateway, GwName, Conf}, RawConf) ->
NConf = tune_gw_certs(fun convert_certs/2, GwName, Conf),
{ok, emqx_map_lib:deep_merge(RawConf, #{GwName => NConf})};
_ ->
{error, already_exist}
badres_gateway(already_exist, GwName)
end;
pre_config_update(_, {update_gateway, GwName, Conf}, RawConf) ->
case maps:get(GwName, RawConf, undefined) of
undefined ->
{error, not_found};
badres_gateway(not_found, GwName);
_ ->
NConf = maps:without([<<"listeners">>, ?AUTHN_BIN], Conf),
{ok, emqx_map_lib:deep_merge(RawConf, #{GwName => NConf})}
@ -341,13 +342,13 @@ pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) ->
RawConf,
#{GwName => #{<<"listeners">> => NListener}})};
_ ->
{error, already_exist}
badres_listener(already_exist, GwName, LType, LName)
end;
pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) ->
case emqx_map_lib:deep_get(
[GwName, <<"listeners">>, LType, LName], RawConf, undefined) of
undefined ->
{error, not_found};
badres_listener(not_found, GwName, LType, LName);
OldConf ->
NConf = convert_certs(certs_dir(GwName), Conf, OldConf),
NListener = #{LType => #{LName => NConf}},
@ -374,14 +375,14 @@ pre_config_update(_, {add_authn, GwName, Conf}, RawConf) ->
RawConf,
#{GwName => #{?AUTHN_BIN => Conf}})};
_ ->
{error, already_exist}
badres_authn(already_exist, GwName)
end;
pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
case emqx_map_lib:deep_get(
[GwName, <<"listeners">>, LType, LName],
RawConf, undefined) of
undefined ->
{error, not_found};
badres_listener(not_found, GwName, LType, LName);
Listener ->
case maps:get(?AUTHN_BIN, Listener, undefined) of
undefined ->
@ -391,14 +392,14 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
#{LType => #{LName => NListener}}}},
{ok, emqx_map_lib:deep_merge(RawConf, NGateway)};
_ ->
{error, already_exist}
badres_listener_authn(already_exist, GwName, LType, LName)
end
end;
pre_config_update(_, {update_authn, GwName, Conf}, RawConf) ->
case emqx_map_lib:deep_get(
[GwName, ?AUTHN_BIN], RawConf, undefined) of
undefined ->
{error, not_found};
badres_authn(not_found, GwName);
_ ->
{ok, emqx_map_lib:deep_merge(
RawConf,
@ -409,11 +410,11 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
[GwName, <<"listeners">>, LType, LName],
RawConf, undefined) of
undefined ->
{error, not_found};
badres_listener(not_found, GwName, LType, LName);
Listener ->
case maps:get(?AUTHN_BIN, Listener, undefined) of
undefined ->
{error, not_found};
badres_listener_authn(not_found, GwName, LType, LName);
Auth ->
NListener = maps:put(
?AUTHN_BIN,
@ -437,6 +438,38 @@ pre_config_update(_, UnknownReq, _RawConf) ->
logger:error("Unknown configuration update request: ~0p", [UnknownReq]),
{error, badreq}.
badres_gateway(not_found, GwName) ->
{error, {badres, #{resource => gateway, gateway => GwName,
reason => not_found}}};
badres_gateway(already_exist, GwName) ->
{error, {badres, #{resource => gateway, gateway => GwName,
reason => already_exist}}}.
badres_listener(not_found, GwName, LType, LName) ->
{error, {badres, #{resource => listener, gateway => GwName,
listener => {GwName, LType, LName},
reason => not_found}}};
badres_listener(already_exist, GwName, LType, LName) ->
{error, {badres, #{resource => listener, gateway => GwName,
listener => {GwName, LType, LName},
reason => already_exist}}}.
badres_authn(not_found, GwName) ->
{error, {badres, #{resource => authn, gateway => GwName,
reason => not_found}}};
badres_authn(already_exist, GwName) ->
{error, {badres, #{resource => authn, gateway => GwName,
reason => already_exist}}}.
badres_listener_authn(not_found, GwName, LType, LName) ->
{error, {badres, #{resource => listener_authn, gateway => GwName,
listener => {GwName, LType, LName},
reason => not_found}}};
badres_listener_authn(already_exist, GwName, LType, LName) ->
{error, {badres, #{resource => listener_authn, gateway => GwName,
listener => {GwName, LType, LName},
reason => already_exist}}}.
-spec post_config_update(list(atom()),
emqx_config:update_request(),
emqx_config:config(),

92
apps/emqx_gateway/src/emqx_gateway_http.erl
View File

@ -23,6 +23,8 @@
-define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
-import(emqx_gateway_utils, [listener_id/3]).
%% Mgmt APIs - gateway
-export([ gateways/1
]).
@ -59,10 +61,7 @@
, with_authn/2
, with_listener_authn/3
, checks/2
, schema_bad_request/0
, schema_not_found/0
, schema_internal_error/0
, schema_no_content/0
, reason2resp/1
]).
-type gateway_summary() ::
@ -131,7 +130,7 @@ current_connections_count(GwName) ->
get_listeners_status(GwName, Config) ->
Listeners = emqx_gateway_utils:normalize_config(Config),
lists:map(fun({Type, LisName, ListenOn, _, _}) ->
Name0 = emqx_gateway_utils:listener_id(GwName, Type, LisName),
Name0 = listener_id(GwName, Type, LisName),
Name = {Name0, ListenOn},
LisO = #{id => Name0, type => Type, name => LisName},
case catch esockd:listener(Name) of
@ -223,12 +222,7 @@ remove_authn(GwName, ListenerId) ->
confexp(ok) -> ok;
confexp({ok, Res}) -> {ok, Res};
confexp({error, badarg}) ->
error({update_conf_error, badarg});
confexp({error, not_found}) ->
error({update_conf_error, not_found});
confexp({error, already_exist}) ->
error({update_conf_error, already_exist}).
confexp({error, Reason}) -> error(Reason).
%%--------------------------------------------------------------------
%% Mgmt APIs - clients
@ -322,6 +316,59 @@ with_channel(GwName, ClientId, Fun) ->
%% Utils
%%--------------------------------------------------------------------
-spec reason2resp({atom(), map()} | any()) -> binary() | any().
reason2resp({badconf, #{key := Key, value := Value, reason := Reason}}) ->
fmt400err("Bad config value '~s' for '~s', reason: ~s",
[Value, Key, Reason]);
reason2resp({badres, #{resource := gateway,
gateway := GwName,
reason := not_found}}) ->
fmt400err("The ~s gateway is unloaded", [GwName]);
reason2resp({badres, #{resource := gateway,
gateway := GwName,
reason := already_exist}}) ->
fmt400err("The ~s gateway has loaded", [GwName]);
reason2resp({badres, #{resource := listener,
listener := {GwName, LType, LName},
reason := not_found}}) ->
fmt400err("Listener ~s not found",
[listener_id(GwName, LType, LName)]);
reason2resp({badres, #{resource := listener,
listener := {GwName, LType, LName},
reason := already_exist}}) ->
fmt400err("The listener ~s of ~s already exist",
[listener_id(GwName, LType, LName), GwName]);
reason2resp({badres, #{resource := authn,
gateway := GwName,
reason := not_found}}) ->
fmt400err("The authentication not found on ~s", [GwName]);
reason2resp({badres, #{resource := authn,
gateway := GwName,
reason := already_exist}}) ->
fmt400err("The authentication already exist on ~s", [GwName]);
reason2resp({badres, #{resource := listener_authn,
listener := {GwName, LType, LName},
reason := not_found}}) ->
fmt400err("The authentication not found on ~s",
[listener_id(GwName, LType, LName)]);
reason2resp({badres, #{resource := listener_authn,
listener := {GwName, LType, LName},
reason := already_exist}}) ->
fmt400err("The authentication already exist on ~s",
[listener_id(GwName, LType, LName)]);
reason2resp(R) -> return_http_error(500, R).
fmt400err(Fmt, Args) ->
return_http_error(400, io_lib:format(Fmt, Args)).
-spec return_http_error(integer(), any()) -> {integer(), binary()}.
return_http_error(Code, Msg) ->
{Code, emqx_json:encode(
@ -378,19 +425,12 @@ with_gateway(GwName0, Fun) ->
Path = lists:concat(
lists:join(".", lists:map(fun to_list/1, Path0))),
return_http_error(404, "Resource not found. path: " ++ Path);
%% Exceptions from: confexp/1
error : {update_conf_error, badarg} ->
return_http_error(400, "Bad arguments");
error : {update_conf_error, not_found} ->
return_http_error(404, "Resource not found");
error : {update_conf_error, already_exist} ->
return_http_error(400, "Resource already exist");
Class : Reason : Stk ->
?SLOG(error, #{ msg => "uncatched_error"
, reason => {Class, Reason}
, stacktrace => Stk
}),
return_http_error(500, {Class, Reason, Stk})
reason2resp(Reason)
end.
-spec checks(list(), map()) -> ok.
@ -408,20 +448,6 @@ to_list(A) when is_atom(A) ->
to_list(B) when is_binary(B) ->
binary_to_list(B).
%%--------------------------------------------------------------------
%% common schemas
schema_bad_request() ->
emqx_mgmt_util:error_schema(
<<"Some Params missed">>, ['PARAMETER_MISSED']).
schema_internal_error() ->
emqx_mgmt_util:error_schema(
<<"Ineternal Server Error">>, ['INTERNAL_SERVER_ERROR']).
schema_not_found() ->
emqx_mgmt_util:error_schema(<<"Resource Not Found">>).
schema_no_content() ->
#{description => <<"No Content">>}.
%%--------------------------------------------------------------------
%% Internal funcs

12
apps/emqx_gateway/src/emqx_gateway_insta_sup.erl
View File

@ -112,7 +112,7 @@ init([Gateway, Ctx, _GwDscrptr]) ->
true ->
case cb_gateway_load(State) of
{error, Reason} ->
{stop, {load_gateway_failure, Reason}};
{stop, Reason};
{ok, NState} ->
{ok, NState}
end
@ -360,7 +360,7 @@ cb_gateway_unload(State = #state{name = GwName,
, reason => {Class, Reason}
, stacktrace => Stk
}),
{error, {Class, Reason, Stk}}
{error, Reason}
after
_ = do_deinit_authn(State#state.authns)
end.
@ -381,7 +381,7 @@ cb_gateway_load(State = #state{name = GwName,
case CbMod:on_gateway_load(Gateway, NCtx) of
{error, Reason} ->
do_deinit_authn(AuthnNames),
throw({callback_return_error, Reason});
{error, Reason};
{ok, ChildPidOrSpecs, GwState} ->
ChildPids = start_child_process(ChildPidOrSpecs),
{ok, State#state{
@ -403,7 +403,7 @@ cb_gateway_load(State = #state{name = GwName,
, reason => {Class, Reason1}
, stacktrace => Stk
}),
{error, {Class, Reason1, Stk}}
{error, Reason1}
end.
cb_gateway_update(Config,
@ -412,7 +412,7 @@ cb_gateway_update(Config,
try
#{cbkmod := CbMod} = emqx_gateway_registry:lookup(GwName),
case CbMod:on_gateway_update(Config, detailed_gateway_info(State), GwState) of
{error, Reason} -> throw({callback_return_error, Reason});
{error, Reason} -> {error, Reason};
{ok, ChildPidOrSpecs, NGwState} ->
%% XXX: Hot-upgrade ???
ChildPids = start_child_process(ChildPidOrSpecs),
@ -430,7 +430,7 @@ cb_gateway_update(Config,
, reason => {Class, Reason1}
, stacktrace => Stk
}),
{error, {Class, Reason1, Stk}}
{error, Reason1}
end.
start_child_process([]) -> [];

21
apps/emqx_gateway/src/emqx_gateway_schema.erl
View File

@ -118,6 +118,7 @@ fields(mqttsn) ->
[ {gateway_id,
sc(integer(),
#{ default => 1
, nullable => false
, desc =>
"MQTT-SN Gateway Id.<br>
When the <code>broadcast</code> option is enabled,
@ -142,6 +143,7 @@ The client just sends its PUBLISH messages to a GW"
, {predefined,
sc(hoconsc:array(ref(mqttsn_predefined)),
#{ default => []
, nullable => {true, recursively}
, desc =>
<<"The Pre-defined topic ids and topic names.<br>
A 'pre-defined' topic id is a topic id whose mapping to a topic name
@ -217,6 +219,7 @@ fields(lwm2m) ->
[ {xml_dir,
sc(binary(),
#{ default =>"etc/lwm2m_xml"
, nullable => false
, desc => "The Directory for LwM2M Resource defination"
})}
, {lifetime_min,
@ -265,18 +268,21 @@ beyond this time window are temporarily stored in memory."
fields(exproto) ->
[ {server,
sc(ref(exproto_grpc_server),
#{ desc => "Configurations for starting the <code>ConnectionAdapter</code> service"
#{ nullable => false
, desc => "Configurations for starting the <code>ConnectionAdapter</code> service"
})}
, {handler,
sc(ref(exproto_grpc_handler),
#{ desc => "Configurations for request to <code>ConnectionHandler</code> service"
#{ nullable => false
, desc => "Configurations for request to <code>ConnectionHandler</code> service"
})}
, {listeners, sc(ref(udp_tcp_listeners))}
] ++ gateway_common_options();
fields(exproto_grpc_server) ->
[ {bind,
sc(hoconsc:union([ip_port(), integer()]))}
sc(hoconsc:union([ip_port(), integer()]),
#{nullable => false})}
, {ssl,
sc(ref(ssl_server_opts),
#{ nullable => {true, recursively}
@ -284,7 +290,7 @@ fields(exproto_grpc_server) ->
];
fields(exproto_grpc_handler) ->
[ {address, sc(binary())}
[ {address, sc(binary(), #{nullable => false})}
, {ssl,
sc(ref(ssl_client_opts),
#{ nullable => {true, recursively}
@ -316,11 +322,13 @@ fields(lwm2m_translators) ->
For each new LwM2M client that succeeds in going online, the gateway creates
a the subscription relationship to receive downstream commands and send it to
the LwM2M client"
, nullable => false
})}
, {response,
sc(ref(translator),
#{ desc =>
"The topic for gateway to publish the acknowledge events from LwM2M client"
, nullable => false
})}
, {notify,
sc(ref(translator),
@ -328,21 +336,24 @@ the LwM2M client"
"The topic for gateway to publish the notify events from LwM2M client.<br>
After succeed observe a resource of LwM2M client, Gateway will send the
notifyevents via this topic, if the client reports any resource changes"
, nullable => false
})}
, {register,
sc(ref(translator),
#{ desc =>
"The topic for gateway to publish the register events from LwM2M client.<br>"
, nullable => false
})}
, {update,
sc(ref(translator),
#{ desc =>
"The topic for gateway to publish the update events from LwM2M client.<br>"
, nullable => false
})}
];
fields(translator) ->
[ {topic, sc(binary())}
[ {topic, sc(binary(), #{nullable => false})}
, {qos, sc(range(0, 2), #{default => 0})}
];

1
apps/emqx_gateway/src/emqx_gateway_utils.erl
View File

@ -90,6 +90,7 @@ childspec(Id, Type, Mod, Args) ->
-> {ok, pid()}
| {error, supervisor:startchild_err()}.
supervisor_ret({ok, Pid, _Info}) -> {ok, Pid};
supervisor_ret({error, {Reason, _Child}}) -> {error, Reason};
supervisor_ret(Ret) -> Ret.
-spec find_sup_child(Sup :: pid() | atom(), ChildId :: supervisor:child_id())

8
apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl
View File

@ -75,7 +75,13 @@ stop_grpc_server(GwName) ->
start_grpc_client_channel(_GwName, undefined) ->
undefined;
start_grpc_client_channel(GwName, Options = #{address := Address}) ->
{Host, Port} = emqx_gateway_utils:parse_address(Address),
{Host, Port} = try emqx_gateway_utils:parse_address(Address)
catch error : badarg ->
throw({badconf, #{key => address,
value => Address,
reason => illegal_grpc_address
}})
end,
case maps:to_list(maps:get(ssl, Options, #{})) of
[] ->
SvrAddr = compose_http_uri(http, Host, Port),

14
apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl
View File

@ -50,14 +50,20 @@ unreg() ->
on_gateway_load(_Gateway = #{ name := GwName,
config := Config
}, Ctx) ->
%% Xml registry
{ok, RegPid} = emqx_lwm2m_xml_object_db:start_link(maps:get(xml_dir, Config)),
XmlDir = maps:get(xml_dir, Config),
case emqx_lwm2m_xml_object_db:start_link(XmlDir) of
{ok, RegPid} ->
Listeners = emqx_gateway_utils:normalize_config(Config),
ListenerPids = lists:map(fun(Lis) ->
start_listener(GwName, Ctx, Lis)
end, Listeners),
{ok, ListenerPids, _GwState = #{ctx => Ctx, registry => RegPid}}.
{ok, ListenerPids, _GwState = #{ctx => Ctx, registry => RegPid}};
{error, Reason} ->
throw({badconf, #{ key => xml_dir
, value => XmlDir
, reason => Reason
}})
end.
on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
GwName = maps:get(name, Gateway),

14
apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl
View File

@ -47,6 +47,11 @@
%% API Function Definitions
%% ------------------------------------------------------------------
-spec start_link(string())
-> {ok, pid()}
| ignore
| {error, no_xml_files_found}
| {error, term()}.
start_link(XmlDir) ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [XmlDir], []).
@ -85,8 +90,11 @@ stop() ->
init([XmlDir]) ->
_ = ets:new(?LWM2M_OBJECT_DEF_TAB, [set, named_table, protected]),
_ = ets:new(?LWM2M_OBJECT_NAME_TO_ID_TAB, [set, named_table, protected]),
load(XmlDir),
{ok, #state{}}.
case load(XmlDir) of
ok ->
{ok, #state{}};
{error, Reason} -> {stop, Reason}
end.
handle_call(_Request, _From, State) ->
{reply, ignored, State}.
@ -116,7 +124,7 @@ load(BaseDir) ->
Wild
end,
case filelib:wildcard(Wild2) of
[] -> error(no_xml_files_found, BaseDir);
[] -> {error, no_xml_files_found};
AllXmlFiles -> load_loop(AllXmlFiles)
end.

32
apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl
View File

@ -245,8 +245,9 @@ t_load_unload_gateway(_) ->
?CONF_STOMP_AUTHN_1,
?CONF_STOMP_LISTENER_1),
{ok, _} = emqx_gateway_conf:load_gateway(stomp, StompConf1),
{error, already_exist} =
emqx_gateway_conf:load_gateway(stomp, StompConf1),
?assertMatch(
{error, {badres, #{reason := already_exist}}},
emqx_gateway_conf:load_gateway(stomp, StompConf1)),
assert_confs(StompConf1, emqx:get_raw_config([gateway, stomp])),
{ok, _} = emqx_gateway_conf:update_gateway(stomp, StompConf2),
@ -255,8 +256,9 @@ t_load_unload_gateway(_) ->
ok = emqx_gateway_conf:unload_gateway(stomp),
ok = emqx_gateway_conf:unload_gateway(stomp),
{error, not_found} =
emqx_gateway_conf:update_gateway(stomp, StompConf2),
?assertMatch(
{error, {badres, #{reason := not_found}}},
emqx_gateway_conf:update_gateway(stomp, StompConf2)),
?assertException(error, {config_not_found, [gateway, stomp]},
emqx:get_raw_config([gateway, stomp])),
@ -280,8 +282,9 @@ t_load_remove_authn(_) ->
ok = emqx_gateway_conf:remove_authn(<<"stomp">>),
{error, not_found} =
emqx_gateway_conf:update_authn(<<"stomp">>, ?CONF_STOMP_AUTHN_2),
?assertMatch(
{error, {badres, #{reason := not_found}}},
emqx_gateway_conf:update_authn(<<"stomp">>, ?CONF_STOMP_AUTHN_2)),
?assertException(
error, {config_not_found, [gateway, stomp, authentication]},
@ -312,9 +315,10 @@ t_load_remove_listeners(_) ->
ok = emqx_gateway_conf:remove_listener(
<<"stomp">>, {<<"tcp">>, <<"default">>}),
{error, not_found} =
?assertMatch(
{error, {badres, #{reason := not_found}}},
emqx_gateway_conf:update_listener(
<<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_LISTENER_2),
<<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_LISTENER_2)),
?assertException(
error, {config_not_found, [gateway, stomp, listeners, tcp, default]},
@ -352,9 +356,10 @@ t_load_remove_listener_authn(_) ->
ok = emqx_gateway_conf:remove_authn(
<<"stomp">>, {<<"tcp">>, <<"default">>}),
{error, not_found} =
?assertMatch(
{error, {badres, #{reason := not_found}}},
emqx_gateway_conf:update_authn(
<<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_AUTHN_2),
<<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_AUTHN_2)),
Path = [gateway, stomp, listeners, tcp, default, authentication],
?assertException(
@ -426,9 +431,12 @@ t_add_listener_with_certs_content(_) ->
ok = emqx_gateway_conf:remove_listener(
<<"stomp">>, {<<"ssl">>, <<"default">>}),
assert_ssl_confs_files_deleted(SslConf),
{error, not_found} =
?assertMatch(
{error, {badres, #{reason := not_found}}},
emqx_gateway_conf:update_listener(
<<"stomp">>, {<<"ssl">>, <<"default">>}, ?CONF_STOMP_LISTENER_SSL_2),
<<"stomp">>, {<<"ssl">>, <<"default">>}, ?CONF_STOMP_LISTENER_SSL_2)),
?assertException(
error, {config_not_found, [gateway, stomp, listeners, ssl, default]},
emqx:get_raw_config([gateway, stomp, listeners, ssl, default])

6
apps/emqx_management/src/emqx_mgmt_api_banned.erl
View File

@ -101,15 +101,15 @@ fields(ban) ->
desc => <<"Banned type clientid, username, peerhost">>,
nullable => false,
example => username})},
{who, hoconsc:mk(binary(), #{
{who, hoconsc:mk(emqx_schema:unicode_binary(), #{
desc => <<"Client info as banned type">>,
nullable => false,
example => <<"Badass">>})},
example => <<"Badass坏"/utf8>>})},
{by, hoconsc:mk(binary(), #{
desc => <<"Commander">>,
nullable => true,
example => <<"mgmt_api">>})},
{reason, hoconsc:mk(binary(), #{
{reason, hoconsc:mk(emqx_schema:unicode_binary(), #{
desc => <<"Banned reason">>,
nullable => true,
example => <<"Too many requests">>})},

2
apps/emqx_management/src/emqx_mgmt_api_trace.erl
View File

@ -310,7 +310,7 @@ group_trace_file(ZipDir, TraceLog, TraceFiles) ->
_ -> Acc
end;
{error, Node, Reason} ->
?LOG(error, "download trace log error:~p", [{Node, TraceLog, Reason}]),
?SLOG(error, #{msg => "download_trace_log_error", node => Node, log => TraceLog, reason => Reason}),
Acc
end
end, [], TraceFiles).

8
apps/emqx_modules/src/emqx_delayed.erl
View File

@ -97,7 +97,7 @@ on_message_publish(Msg = #message{
case store(#delayed_message{key = {PubAt, Id}, delayed = Delayed, msg = PubMsg}) of
ok -> ok;
{error, Error} ->
?LOG(error, "Store delayed message fail: ~p", [Error])
?SLOG(error, #{msg => "store_delayed_message_fail", error => Error})
end,
{stop, PubMsg#message{headers = Headers#{allow_publish => false}}};
@ -230,11 +230,11 @@ handle_call(disable, _From, State) ->
{reply, ok, State};
handle_call(Req, _From, State) ->
?LOG(error, "Unexpected call: ~p", [Req]),
?SLOG(error, #{msg => "unexpected_call", call => Req}),
{reply, ignored, State}.
handle_cast(Msg, State) ->
?LOG(error, "Unexpected cast: ~p", [Msg]),
?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
{noreply, State}.
%% Do Publish...
@ -248,7 +248,7 @@ handle_info(stats, State = #{stats_fun := StatsFun}) ->
{noreply, State, hibernate};
handle_info(Info, State) ->
?LOG(error, "Unexpected info: ~p", [Info]),
?SLOG(error, #{msg => "unexpected_info", info => Info}),
{noreply, State}.
terminate(_Reason, #{timer := TRef}) ->

55
apps/emqx_modules/src/emqx_telemetry.erl
View File

@ -173,15 +173,15 @@ handle_call(get_telemetry, _From, State) ->
{reply, {ok, get_telemetry(State)}, State};
handle_call(Req, _From, State) ->
?LOG(error, "Unexpected call: ~p", [Req]),
?SLOG(error, #{msg => "unexpected_call", call => Req}),
{reply, ignored, State}.
handle_cast(Msg, State) ->
?LOG(error, "Unexpected msg: ~p", [Msg]),
?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
{noreply, State}.
handle_continue(Continue, State) ->
?LOG(error, "Unexpected continue: ~p", [Continue]),
?SLOG(error, #{msg => "unexpected_continue", continue => Continue}),
{noreply, State}.
handle_info({timeout, TRef, time_to_report_telemetry_data}, State = #state{timer = TRef}) ->
@ -192,7 +192,7 @@ handle_info({timeout, TRef, time_to_report_telemetry_data}, State = #state{timer
{noreply, ensure_report_timer(State)};
handle_info(Info, State) ->
?LOG(error, "Unexpected info: ~p", [Info]),
?SLOG(error, #{msg => "unexpected_info", info => Info}),
{noreply, State}.
terminate(_Reason, _State) ->
@ -220,37 +220,24 @@ os_info() ->
[{os_name, Name},
{os_version, Version}];
{unix, _} ->
case file:read_file_info("/etc/os-release") of
case file:read_file("/etc/os-release") of
{error, _} ->
[{os_name, "Unknown"},
{os_version, "Unknown"}];
{ok, FileInfo} ->
case FileInfo#file_info.access of
Access when Access =:= read orelse Access =:= read_write ->
OSInfo = lists:foldl(fun(Line, Acc) ->
[Var, Value] = string:tokens(Line, "="),
NValue = case Value of
_ when is_list(Value) ->
lists:nth(1, string:tokens(Value, "\""));
_ ->
Value
end,
[{Var, NValue} | Acc]
end, [], string:tokens(os:cmd("cat /etc/os-release"), "\n")),
[{os_name, get_value("NAME", OSInfo, "Unknown")},
{ok, FileContent} ->
OSInfo = parse_os_release(FileContent),
[{os_name, get_value("NAME", OSInfo)},
{os_version, get_value("VERSION", OSInfo,
get_value("VERSION_ID", OSInfo, "Unknown"))}];
_ ->
[{os_name, "Unknown"},
{os_version, "Unknown"}]
end
get_value("VERSION_ID", OSInfo,
get_value("PRETTY_NAME", OSInfo)))}]
end;
{win32, nt} ->
Ver = os:cmd("ver"),
case re:run(Ver, "[a-zA-Z ]+ \\[Version ([0-9]+[\.])+[0-9]+\\]", [{capture, none}]) of
match ->
[NVer | _] = string:tokens(Ver, "\r\n"),
{match, [Version]} = re:run(NVer, "([0-9]+[\.])+[0-9]+", [{capture, first, list}]),
{match, [Version]} =
re:run(NVer, "([0-9]+[\.])+[0-9]+", [{capture, first, list}]),
[Name | _] = string:split(NVer, " [Version "),
[{os_name, Name},
{os_version, Version}];
@ -307,7 +294,8 @@ generate_uuid() ->
<<NTimeHigh:16>> = <<16#01:4, TimeHigh:12>>,
<<NClockSeq:16>> = <<1:1, 0:1, ClockSeq:14>>,
<<Node:48>> = <<First:7, 1:1, Last:40>>,
list_to_binary(io_lib:format("~.16B-~.16B-~.16B-~.16B-~.16B", [TimeLow, TimeMid, NTimeHigh, NClockSeq, Node])).
list_to_binary(io_lib:format( "~.16B-~.16B-~.16B-~.16B-~.16B"
, [TimeLow, TimeMid, NTimeHigh, NClockSeq, Node])).
get_telemetry(#state{uuid = UUID}) ->
OSInfo = os_info(),
@ -339,7 +327,22 @@ report_telemetry(State = #state{url = URL}) ->
httpc_request(Method, URL, Headers, Body) ->
httpc:request(Method, {URL, Headers, "application/json", Body}, [], []).
parse_os_release(FileContent) ->
lists:foldl(fun(Line, Acc) ->
[Var, Value] = string:tokens(Line, "="),
NValue = case Value of
_ when is_list(Value) ->
lists:nth(1, string:tokens(Value, "\""));
_ ->
Value
end,
[{Var, NValue} | Acc]
end,
[], string:tokens(binary:bin_to_list(FileContent), "\n")).
bin(L) when is_list(L) ->
list_to_binary(L);
bin(A) when is_atom(A) ->
atom_to_binary(A);
bin(B) when is_binary(B) ->
B.

4
apps/emqx_modules/src/emqx_topic_metrics.erl
View File

@ -261,7 +261,7 @@ handle_call({get_rates, Topic, Metric}, _From, State = #state{speeds = Speeds})
end.
handle_cast(Msg, State) ->
?LOG(error, "Unexpected cast: ~p", [Msg]),
?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
{noreply, State}.
handle_info(ticking, State = #state{speeds = Speeds}) ->
@ -276,7 +276,7 @@ handle_info(ticking, State = #state{speeds = Speeds}) ->
{noreply, State#state{speeds = NSpeeds}};
handle_info(Info, State) ->
?LOG(error, "Unexpected info: ~p", [Info]),
?SLOG(error, #{msg => "unexpected_info", info => Info}),
{noreply, State}.
terminate(_Reason, _State) ->

2
apps/emqx_plugins/test/emqx_plugins_SUITE_data/build-demo-plugin.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
set -euo pipefail

6
apps/emqx_retainer/src/emqx_retainer.erl
View File

@ -217,11 +217,11 @@ handle_call({page_read, Topic, Page, Limit}, _, #{context := Context} = State) -
{reply, Result, State};
handle_call(Req, _From, State) ->
?LOG(error, "Unexpected call: ~p", [Req]),
?SLOG(error, #{msg => "unexpected_call", call => Req}),
{reply, ignored, State}.
handle_cast(Msg, State) ->
?LOG(error, "Unexpected cast: ~p", [Msg]),
?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
{noreply, State}.
handle_info(clear_expired, #{context := Context} = State) ->
@ -248,7 +248,7 @@ handle_info(release_deliver_quota, #{context := Context, wait_quotas := Waits} =
wait_quotas := []}};
handle_info(Info, State) ->
?LOG(error, "Unexpected info: ~p", [Info]),
?SLOG(error, #{msg => "unexpected_info", info => Info}),
{noreply, State}.
terminate(_Reason, #{clear_timer := TRef1, release_quota_timer := TRef2}) ->

10
apps/emqx_retainer/src/emqx_retainer_api.erl
View File

@ -34,6 +34,8 @@
, page_params/0
, properties/1]).
-define(MAX_BASE64_PAYLOAD_SIZE, 1048576). %% 1MB = 1024 x 1024
api_spec() ->
{[lookup_retained_api(), with_topic_api(), config_api()], []}.
@ -179,7 +181,13 @@ format_message(#message{ id = ID, qos = Qos, topic = Topic, from = From
format_detail_message(#message{payload = Payload} = Msg) ->
Base = format_message(Msg),
Base#{payload => Payload}.
EncodePayload = base64:encode(Payload),
case erlang:byte_size(EncodePayload) =< ?MAX_BASE64_PAYLOAD_SIZE of
true ->
Base#{payload => EncodePayload};
_ ->
Base#{payload => base64:encode(<<"PAYLOAD_TOO_LARGE">>)}
end.
to_bin_string(Data) when is_binary(Data) ->
Data;

11
apps/emqx_retainer/src/emqx_retainer_mnesia.erl
View File

@ -91,14 +91,13 @@ store_retained(_, Msg =#message{topic = Topic}) ->
expiry_time = ExpiryTime},
write);
[] ->
?LOG(error,
"Cannot retain message(topic=~ts) for table is full!",
[Topic]),
ok
mnesia:abort(table_is_full)
end
end,
{atomic, ok} = mria:transaction(?RETAINER_SHARD, Fun),
ok
case mria:transaction(?RETAINER_SHARD, Fun) of
{atomic, ok} -> ok;
{aborted, Reason} -> ?SLOG(error, #{msg => "failed_to_retain_message", topic => Topic, reason => Reason})
end
end.
clear_expired(_) ->

8
apps/emqx_retainer/src/emqx_retainer_pool.erl
View File

@ -84,7 +84,7 @@ init([Pool, Id]) ->
{stop, Reason :: term(), Reply :: term(), NewState :: term()} |
{stop, Reason :: term(), NewState :: term()}.
handle_call(Req, _From, State) ->
?LOG(error, "Unexpected call: ~p", [Req]),
?SLOG(error, #{msg => "unexpected_call", call => Req}),
{reply, ignored, State}.
%%--------------------------------------------------------------------
@ -101,12 +101,12 @@ handle_call(Req, _From, State) ->
handle_cast({async_submit, Task}, State) ->
try run(Task)
catch _:Error:Stacktrace ->
?LOG(error, "Error: ~0p, ~0p", [Error, Stacktrace])
?SLOG(error, #{msg => "crashed_handling_async_task", exception => Error, stacktrace => Stacktrace})
end,
{noreply, State};
handle_cast(Msg, State) ->
?LOG(error, "Unexpected cast: ~p", [Msg]),
?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
{noreply, State}.
%%--------------------------------------------------------------------
@ -121,7 +121,7 @@ handle_cast(Msg, State) ->
{noreply, NewState :: term(), hibernate} |
{stop, Reason :: normal | term(), NewState :: term()}.
handle_info(Info, State) ->
?LOG(error, "Unexpected info: ~p", [Info]),
?SLOG(error, #{msg => "unexpected_info", info => Info}),
{noreply, State}.
%%--------------------------------------------------------------------

1
apps/emqx_rule_engine/src/emqx_rule_api_schema.erl
View File

@ -68,6 +68,7 @@ fields("rule_events") ->
fields("rule_test") ->
[ {"context", sc(hoconsc:union([ ref("ctx_pub")
, ref("ctx_sub")
, ref("ctx_unsub")
, ref("ctx_delivered")
, ref("ctx_acked")
, ref("ctx_dropped")

7
apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
View File

@ -257,11 +257,16 @@ format_output(Outputs) ->
[do_format_output(Out) || Out <- Outputs].
do_format_output(#{mod := Mod, func := Func, args := Args}) ->
#{function => list_to_binary(lists:concat([Mod,":",Func])),
#{function => printable_function_name(Mod, Func),
args => maps:remove(preprocessed_tmpl, Args)};
do_format_output(BridgeChannelId) when is_binary(BridgeChannelId) ->
BridgeChannelId.
printable_function_name(emqx_rule_outputs, Func) ->
Func;
printable_function_name(Mod, Func) ->
list_to_binary(lists:concat([Mod,":",Func])).
get_rule_metrics(Id) ->
Format = fun (Node, #{matched := Matched,
rate := Current,

1
apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
View File

@ -182,6 +182,7 @@ rule_name() ->
{"name", sc(binary(),
#{ desc => "The name of the rule"
, default => ""
, nullable => false
, example => "foo"
})}.

6
apps/emqx_slow_subs/src/emqx_slow_subs.erl
View File

@ -153,11 +153,11 @@ handle_call(clear_history, _, State) ->
{reply, ok, State};
handle_call(Req, _From, State) ->
?LOG(error, "Unexpected call: ~p", [Req]),
?SLOG(error, #{msg => "unexpected_call", call => Req}),
{reply, ignored, State}.
handle_cast(Msg, State) ->
?LOG(error, "Unexpected cast: ~p", [Msg]),
?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
{noreply, State}.
handle_info(expire_tick, State) ->
@ -173,7 +173,7 @@ handle_info(notice_tick, State) ->
{noreply, State#{last_tick_at := ?NOW}};
handle_info(Info, State) ->
?LOG(error, "Unexpected info: ~p", [Info]),
?SLOG(error, #{msg => "unexpected_info", info => Info}),
{noreply, State}.
terminate(_Reason, _) ->

2
bin/emqx
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# -*- tab-width:4;indent-tabs-mode:nil -*-
# ex: ts=4 sw=4 et

2
bin/emqx_ctl
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# -*- tab-width:4;indent-tabs-mode:nil -*-
# ex: ts=4 sw=4 et

4
bin/install_upgrade.escript
View File

@ -248,10 +248,6 @@ parse_version(V) when is_list(V) ->
hd(string:tokens(V,"/")).
check_and_install(TargetNode, Vsn) ->
%% Backup the vm.args. VM args should be unchanged during hot upgrade
%% but we still backup it here
{ok, [[CurrVmArgs]]} = rpc:call(TargetNode, init, get_argument, [vm_args], ?TIMEOUT),
{ok, _} = file:copy(CurrVmArgs, filename:join(["releases", Vsn, "vm.args"])),
%% Backup the sys.config, this will be used when we check and install release
%% NOTE: We cannot backup the old sys.config directly, because the
%% configs for plugins are only in app-envs, not in the old sys.config

2
build
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# This script helps to build release artifacts.
# arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg

2
deploy/docker/docker-entrypoint.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
## EMQ docker image start script
# Huang Rui <vowstar@gmail.com>
# EMQ X Team <support@emqx.io>

2
deploy/packages/rpm/init.script
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
#
# emqx
#

2
scripts/apps-version-check.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
set -euo pipefail
latest_release=$(git describe --abbrev=0 --tags)

2
scripts/check-nl-at-eof.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
set -euo pipefail

2
scripts/get-distro.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
## This script prints Linux distro name and its version number
## e.g. macos, centos8, ubuntu20.04

2
scripts/get-otp-vsn.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
set -euo pipefail