yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

style: reformat all remaining apps

This commit is contained in:
Zaiming (Stone) Shi 2022-04-27 15:51:18 +02:00
parent 37be7a4977
commit 02c3f87b31
96 changed files with 9988 additions and 6360 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
apps/emqx_bridge/rebar.config
View File

@ -1,8 +1,9 @@
{erl_opts, [debug_info]}. {erl_opts, [debug_info]}.
{deps, [ {emqx, {path, "../emqx"}} {deps, [{emqx, {path, "../emqx"}}]}.
]}.
{shell, [ {shell, [
% {config, "config/sys.config"}, % {config, "config/sys.config"},
{apps, [emqx_bridge]} {apps, [emqx_bridge]}
]}. ]}.
{project_plugins, [erlfmt]}.

32
apps/emqx_bridge/src/emqx_bridge.app.src
View File

@ -1,18 +1,18 @@
%% -*- mode: erlang -*- %% -*- mode: erlang -*-
{application, emqx_bridge, {application, emqx_bridge, [
[{description, "An OTP application"}, {description, "An OTP application"},
{vsn, "0.1.0"}, {vsn, "0.1.0"},
{registered, []}, {registered, []},
{mod, {emqx_bridge_app, []}}, {mod, {emqx_bridge_app, []}},
{applications, {applications, [
[kernel, kernel,
stdlib, stdlib,
emqx, emqx,
emqx_connector emqx_connector
]}, ]},
{env,[]}, {env, []},
{modules, []}, {modules, []},
{licenses, ["Apache 2.0"]}, {licenses, ["Apache 2.0"]},
{links, []} {links, []}
]}. ]}.

429
apps/emqx_bridge/src/emqx_bridge.erl
View File

@ -18,48 +18,48 @@
-include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/logger.hrl").
-export([ post_config_update/5 -export([post_config_update/5]).
]).
-export([ load_hook/0 -export([
, unload_hook/0 load_hook/0,
]). unload_hook/0
]).
-export([on_message_publish/1]). -export([on_message_publish/1]).
-export([ resource_type/1 -export([
, bridge_type/1 resource_type/1,
, resource_id/1 bridge_type/1,
, resource_id/2 resource_id/1,
, bridge_id/2 resource_id/2,
, parse_bridge_id/1 bridge_id/2,
]). parse_bridge_id/1
]).
-export([ load/0 -export([
, lookup/1 load/0,
, lookup/2 lookup/1,
, lookup/3 lookup/2,
, list/0 lookup/3,
, list_bridges_by_connector/1 list/0,
, create/2 list_bridges_by_connector/1,
, create/3 create/2,
, recreate/2 create/3,
, recreate/3 recreate/2,
, create_dry_run/2 recreate/3,
, remove/1 create_dry_run/2,
, remove/2 remove/1,
, update/2 remove/2,
, update/3 update/2,
, stop/2 update/3,
, restart/2 stop/2,
, reset_metrics/1 restart/2,
]). reset_metrics/1
]).
-export([ send_message/2 -export([send_message/2]).
]).
-export([ config_key_path/0 -export([config_key_path/0]).
]).
%% exported for `emqx_telemetry' %% exported for `emqx_telemetry'
-export([get_basic_usage_info/0]). -export([get_basic_usage_info/0]).
@ -69,18 +69,25 @@ load_hook() ->
load_hook(Bridges). load_hook(Bridges).
load_hook(Bridges) -> load_hook(Bridges) ->
lists:foreach(fun({_Type, Bridge}) -> lists:foreach(
lists:foreach(fun({_Name, BridgeConf}) -> fun({_Type, Bridge}) ->
lists:foreach(
fun({_Name, BridgeConf}) ->
do_load_hook(BridgeConf) do_load_hook(BridgeConf)
end, maps:to_list(Bridge)) end,
end, maps:to_list(Bridges)). maps:to_list(Bridge)
)
end,
maps:to_list(Bridges)
).
do_load_hook(#{local_topic := _} = Conf) -> do_load_hook(#{local_topic := _} = Conf) ->
case maps:get(direction, Conf, egress) of case maps:get(direction, Conf, egress) of
egress -> emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}); egress -> emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []});
ingress -> ok ingress -> ok
end; end;
do_load_hook(_Conf) -> ok. do_load_hook(_Conf) ->
ok.
unload_hook() -> unload_hook() ->
ok = emqx_hooks:del('message.publish', {?MODULE, on_message_publish}). ok = emqx_hooks:del('message.publish', {?MODULE, on_message_publish}).
@ -90,23 +97,36 @@ on_message_publish(Message = #message{topic = Topic, flags = Flags}) ->
false -> false ->
Msg = emqx_rule_events:eventmsg_publish(Message), Msg = emqx_rule_events:eventmsg_publish(Message),
send_to_matched_egress_bridges(Topic, Msg); send_to_matched_egress_bridges(Topic, Msg);
true -> ok true ->
ok
end, end,
{ok, Message}. {ok, Message}.
send_to_matched_egress_bridges(Topic, Msg) -> send_to_matched_egress_bridges(Topic, Msg) ->
lists:foreach(fun (Id) -> lists:foreach(
try send_message(Id, Msg) of fun(Id) ->
{error, Reason} -> try send_message(Id, Msg) of
?SLOG(error, #{msg => "send_message_to_bridge_failed", {error, Reason} ->
bridge => Id, error => Reason}); ?SLOG(error, #{
_ -> ok msg => "send_message_to_bridge_failed",
catch Err:Reason:ST -> bridge => Id,
?SLOG(error, #{msg => "send_message_to_bridge_exception", error => Reason
bridge => Id, error => Err, reason => Reason, });
stacktrace => ST}) _ ->
end ok
end, get_matched_bridges(Topic)). catch
Err:Reason:ST ->
?SLOG(error, #{
msg => "send_message_to_bridge_exception",
bridge => Id,
error => Err,
reason => Reason,
stacktrace => ST
})
end
end,
get_matched_bridges(Topic)
).
send_message(BridgeId, Message) -> send_message(BridgeId, Message) ->
{BridgeType, BridgeName} = parse_bridge_id(BridgeId), {BridgeType, BridgeName} = parse_bridge_id(BridgeId),
@ -132,8 +152,8 @@ bridge_type(emqx_connector_mqtt) -> mqtt;
bridge_type(emqx_connector_http) -> http. bridge_type(emqx_connector_http) -> http.
post_config_update(_, _Req, NewConf, OldConf, _AppEnv) -> post_config_update(_, _Req, NewConf, OldConf, _AppEnv) ->
#{added := Added, removed := Removed, changed := Updated} #{added := Added, removed := Removed, changed := Updated} =
= diff_confs(NewConf, OldConf), diff_confs(NewConf, OldConf),
%% The config update will be failed if any task in `perform_bridge_changes` failed. %% The config update will be failed if any task in `perform_bridge_changes` failed.
Result = perform_bridge_changes([ Result = perform_bridge_changes([
{fun remove/3, Removed}, {fun remove/3, Removed},
@ -150,15 +170,19 @@ perform_bridge_changes(Tasks) ->
perform_bridge_changes([], Result) -> perform_bridge_changes([], Result) ->
Result; Result;
perform_bridge_changes([{Action, MapConfs} | Tasks], Result0) -> perform_bridge_changes([{Action, MapConfs} | Tasks], Result0) ->
Result = maps:fold(fun Result = maps:fold(
({_Type, _Name}, _Conf, {error, Reason}) -> fun
{error, Reason}; ({_Type, _Name}, _Conf, {error, Reason}) ->
({Type, Name}, Conf, _) -> {error, Reason};
case Action(Type, Name, Conf) of ({Type, Name}, Conf, _) ->
{error, Reason} -> {error, Reason}; case Action(Type, Name, Conf) of
Return -> Return {error, Reason} -> {error, Reason};
end Return -> Return
end, Result0, MapConfs), end
end,
Result0,
MapConfs
),
perform_bridge_changes(Tasks, Result). perform_bridge_changes(Tasks, Result).
load() -> load() ->
@ -184,18 +208,29 @@ parse_bridge_id(BridgeId) ->
end. end.
list() -> list() ->
lists:foldl(fun({Type, NameAndConf}, Bridges) -> lists:foldl(
lists:foldl(fun({Name, RawConf}, Acc) -> fun({Type, NameAndConf}, Bridges) ->
lists:foldl(
fun({Name, RawConf}, Acc) ->
case lookup(Type, Name, RawConf) of case lookup(Type, Name, RawConf) of
{error, not_found} -> Acc; {error, not_found} -> Acc;
{ok, Res} -> [Res | Acc] {ok, Res} -> [Res | Acc]
end end
end, Bridges, maps:to_list(NameAndConf)) end,
end, [], maps:to_list(emqx:get_raw_config([bridges], #{}))). Bridges,
maps:to_list(NameAndConf)
)
end,
[],
maps:to_list(emqx:get_raw_config([bridges], #{}))
).
list_bridges_by_connector(ConnectorId) -> list_bridges_by_connector(ConnectorId) ->
[B || B = #{raw_config := #{<<"connector">> := Id}} <- list(), [
ConnectorId =:= Id]. B
|| B = #{raw_config := #{<<"connector">> := Id}} <- list(),
ConnectorId =:= Id
].
lookup(Id) -> lookup(Id) ->
{Type, Name} = parse_bridge_id(Id), {Type, Name} = parse_bridge_id(Id),
@ -206,10 +241,15 @@ lookup(Type, Name) ->
lookup(Type, Name, RawConf). lookup(Type, Name, RawConf).
lookup(Type, Name, RawConf) -> lookup(Type, Name, RawConf) ->
case emqx_resource:get_instance(resource_id(Type, Name)) of case emqx_resource:get_instance(resource_id(Type, Name)) of
{error, not_found} -> {error, not_found}; {error, not_found} ->
{error, not_found};
{ok, _, Data} -> {ok, _, Data} ->
{ok, #{type => Type, name => Name, resource_data => Data, {ok, #{
raw_config => RawConf}} type => Type,
name => Name,
resource_data => Data,
raw_config => RawConf
}}
end. end.
reset_metrics(ResourceId) -> reset_metrics(ResourceId) ->
@ -227,13 +267,21 @@ create(BridgeId, Conf) ->
create(BridgeType, BridgeName, Conf). create(BridgeType, BridgeName, Conf).
create(Type, Name, Conf) -> create(Type, Name, Conf) ->
?SLOG(info, #{msg => "create bridge", type => Type, name => Name, ?SLOG(info, #{
config => Conf}), msg => "create bridge",
case emqx_resource:create_local(resource_id(Type, Name), type => Type,
<<"emqx_bridge">>, name => Name,
emqx_bridge:resource_type(Type), config => Conf
parse_confs(Type, Name, Conf), }),
#{}) of case
emqx_resource:create_local(
resource_id(Type, Name),
<<"emqx_bridge">>,
emqx_bridge:resource_type(Type),
parse_confs(Type, Name, Conf),
#{}
)
of
{ok, already_created} -> maybe_disable_bridge(Type, Name, Conf); {ok, already_created} -> maybe_disable_bridge(Type, Name, Conf);
{ok, _} -> maybe_disable_bridge(Type, Name, Conf); {ok, _} -> maybe_disable_bridge(Type, Name, Conf);
{error, Reason} -> {error, Reason} {error, Reason} -> {error, Reason}
@ -254,15 +302,25 @@ update(Type, Name, {OldConf, Conf}) ->
%% %%
case if_only_to_toggle_enable(OldConf, Conf) of case if_only_to_toggle_enable(OldConf, Conf) of
false -> false ->
?SLOG(info, #{msg => "update bridge", type => Type, name => Name, ?SLOG(info, #{
config => Conf}), msg => "update bridge",
type => Type,
name => Name,
config => Conf
}),
case recreate(Type, Name, Conf) of case recreate(Type, Name, Conf) of
{ok, _} -> maybe_disable_bridge(Type, Name, Conf); {ok, _} ->
maybe_disable_bridge(Type, Name, Conf);
{error, not_found} -> {error, not_found} ->
?SLOG(warning, #{ msg => "updating_a_non-exist_bridge_need_create_a_new_one" ?SLOG(warning, #{
, type => Type, name => Name, config => Conf}), msg => "updating_a_non-exist_bridge_need_create_a_new_one",
type => Type,
name => Name,
config => Conf
}),
create(Type, Name, Conf); create(Type, Name, Conf);
{error, Reason} -> {error, {update_bridge_failed, Reason}} {error, Reason} ->
{error, {update_bridge_failed, Reason}}
end; end;
true -> true ->
%% we don't need to recreate the bridge if this config change is only to %% we don't need to recreate the bridge if this config change is only to
@ -277,22 +335,25 @@ recreate(Type, Name) ->
recreate(Type, Name, emqx:get_config([bridges, Type, Name])). recreate(Type, Name, emqx:get_config([bridges, Type, Name])).
recreate(Type, Name, Conf) -> recreate(Type, Name, Conf) ->
emqx_resource:recreate_local(resource_id(Type, Name), emqx_resource:recreate_local(
resource_id(Type, Name),
emqx_bridge:resource_type(Type), emqx_bridge:resource_type(Type),
parse_confs(Type, Name, Conf), parse_confs(Type, Name, Conf),
#{}). #{}
).
create_dry_run(Type, Conf) -> create_dry_run(Type, Conf) ->
Conf0 = Conf#{
Conf0 = Conf#{<<"egress">> => <<"egress">> =>
#{ <<"remote_topic">> => <<"t">> #{
, <<"remote_qos">> => 0 <<"remote_topic">> => <<"t">>,
, <<"retain">> => true <<"remote_qos">> => 0,
, <<"payload">> => <<"val">> <<"retain">> => true,
}, <<"payload">> => <<"val">>
<<"ingress">> => },
#{ <<"remote_topic">> => <<"t">> <<"ingress">> =>
}}, #{<<"remote_topic">> => <<"t">>}
},
case emqx_resource:check_config(emqx_bridge:resource_type(Type), Conf0) of case emqx_resource:check_config(emqx_bridge:resource_type(Type), Conf0) of
{ok, Conf1} -> {ok, Conf1} ->
emqx_resource:create_dry_run_local(emqx_bridge:resource_type(Type), Conf1); emqx_resource:create_dry_run_local(emqx_bridge:resource_type(Type), Conf1);
@ -313,35 +374,48 @@ remove(Type, Name, _Conf) ->
case emqx_resource:remove_local(resource_id(Type, Name)) of case emqx_resource:remove_local(resource_id(Type, Name)) of
ok -> ok; ok -> ok;
{error, not_found} -> ok; {error, not_found} -> ok;
{error, Reason} -> {error, Reason} -> {error, Reason}
{error, Reason}
end. end.
diff_confs(NewConfs, OldConfs) -> diff_confs(NewConfs, OldConfs) ->
emqx_map_lib:diff_maps(flatten_confs(NewConfs), emqx_map_lib:diff_maps(
flatten_confs(OldConfs)). flatten_confs(NewConfs),
flatten_confs(OldConfs)
).
flatten_confs(Conf0) -> flatten_confs(Conf0) ->
maps:from_list( maps:from_list(
lists:flatmap(fun({Type, Conf}) -> lists:flatmap(
fun({Type, Conf}) ->
do_flatten_confs(Type, Conf) do_flatten_confs(Type, Conf)
end, maps:to_list(Conf0))). end,
maps:to_list(Conf0)
)
).
do_flatten_confs(Type, Conf0) -> do_flatten_confs(Type, Conf0) ->
[{{Type, Name}, Conf} || {Name, Conf} <- maps:to_list(Conf0)]. [{{Type, Name}, Conf} || {Name, Conf} <- maps:to_list(Conf0)].
get_matched_bridges(Topic) -> get_matched_bridges(Topic) ->
Bridges = emqx:get_config([bridges], #{}), Bridges = emqx:get_config([bridges], #{}),
maps:fold(fun (BType, Conf, Acc0) -> maps:fold(
maps:fold(fun fun(BType, Conf, Acc0) ->
%% Confs for MQTT, Kafka bridges have the `direction` flag maps:fold(
(_BName, #{direction := ingress}, Acc1) -> fun
Acc1; %% Confs for MQTT, Kafka bridges have the `direction` flag
(BName, #{direction := egress} = Egress, Acc1) -> (_BName, #{direction := ingress}, Acc1) ->
%% HTTP, MySQL bridges only have egress direction Acc1;
get_matched_bridge_id(Egress, Topic, BType, BName, Acc1) (BName, #{direction := egress} = Egress, Acc1) ->
end, Acc0, Conf) %% HTTP, MySQL bridges only have egress direction
end, [], Bridges). get_matched_bridge_id(Egress, Topic, BType, BName, Acc1)
end,
Acc0,
Conf
)
end,
[],
Bridges
).
get_matched_bridge_id(#{enable := false}, _Topic, _BType, _BName, Acc) -> get_matched_bridge_id(#{enable := false}, _Topic, _BType, _BName, Acc) ->
Acc; Acc;
@ -351,38 +425,56 @@ get_matched_bridge_id(#{local_topic := Filter}, Topic, BType, BName, Acc) ->
false -> Acc false -> Acc
end. end.
parse_confs(http, _Name, parse_confs(
#{ url := Url http,
, method := Method _Name,
, body := Body #{
, headers := Headers url := Url,
, request_timeout := ReqTimeout method := Method,
} = Conf) -> body := Body,
headers := Headers,
request_timeout := ReqTimeout
} = Conf
) ->
{BaseUrl, Path} = parse_url(Url), {BaseUrl, Path} = parse_url(Url),
{ok, BaseUrl2} = emqx_http_lib:uri_parse(BaseUrl), {ok, BaseUrl2} = emqx_http_lib:uri_parse(BaseUrl),
Conf#{ base_url => BaseUrl2 Conf#{
, request => base_url => BaseUrl2,
#{ path => Path request =>
, method => Method #{
, body => Body path => Path,
, headers => Headers method => Method,
, request_timeout => ReqTimeout body => Body,
} headers => Headers,
}; request_timeout => ReqTimeout
parse_confs(Type, Name, #{connector := ConnId, direction := Direction} = Conf) }
when is_binary(ConnId) -> };
parse_confs(Type, Name, #{connector := ConnId, direction := Direction} = Conf) when
is_binary(ConnId)
->
case emqx_connector:parse_connector_id(ConnId) of case emqx_connector:parse_connector_id(ConnId) of
{Type, ConnName} -> {Type, ConnName} ->
ConnectorConfs = emqx:get_config([connectors, Type, ConnName]), ConnectorConfs = emqx:get_config([connectors, Type, ConnName]),
make_resource_confs(Direction, ConnectorConfs, make_resource_confs(
maps:without([connector, direction], Conf), Type, Name); Direction,
ConnectorConfs,
maps:without([connector, direction], Conf),
Type,
Name
);
{_ConnType, _ConnName} -> {_ConnType, _ConnName} ->
error({cannot_use_connector_with_different_type, ConnId}) error({cannot_use_connector_with_different_type, ConnId})
end; end;
parse_confs(Type, Name, #{connector := ConnectorConfs, direction := Direction} = Conf) parse_confs(Type, Name, #{connector := ConnectorConfs, direction := Direction} = Conf) when
when is_map(ConnectorConfs) -> is_map(ConnectorConfs)
make_resource_confs(Direction, ConnectorConfs, ->
maps:without([connector, direction], Conf), Type, Name). make_resource_confs(
Direction,
ConnectorConfs,
maps:without([connector, direction], Conf),
Type,
Name
).
make_resource_confs(ingress, ConnectorConfs, BridgeConf, Type, Name) -> make_resource_confs(ingress, ConnectorConfs, BridgeConf, Type, Name) ->
BName = bridge_id(Type, Name), BName = bridge_id(Type, Name),
@ -417,39 +509,48 @@ if_only_to_toggle_enable(OldConf, Conf) ->
#{added := Added, removed := Removed, changed := Updated} = #{added := Added, removed := Removed, changed := Updated} =
emqx_map_lib:diff_maps(OldConf, Conf), emqx_map_lib:diff_maps(OldConf, Conf),
case {Added, Removed, Updated} of case {Added, Removed, Updated} of
{Added, Removed, #{enable := _}= Updated} {Added, Removed, #{enable := _} = Updated} when
when map_size(Added) =:= 0, map_size(Added) =:= 0,
map_size(Removed) =:= 0, map_size(Removed) =:= 0,
map_size(Updated) =:= 1 -> true; map_size(Updated) =:= 1
{_, _, _} -> false ->
true;
{_, _, _} ->
false
end. end.
-spec get_basic_usage_info() -> -spec get_basic_usage_info() ->
#{ num_bridges => non_neg_integer() #{
, count_by_type => num_bridges => non_neg_integer(),
#{ BridgeType => non_neg_integer() count_by_type =>
} #{BridgeType => non_neg_integer()}
} when BridgeType :: atom(). }
when
BridgeType :: atom().
get_basic_usage_info() -> get_basic_usage_info() ->
InitialAcc = #{num_bridges => 0, count_by_type => #{}}, InitialAcc = #{num_bridges => 0, count_by_type => #{}},
try try
lists:foldl( lists:foldl(
fun(#{resource_data := #{config := #{enable := false}}}, Acc) -> fun
Acc; (#{resource_data := #{config := #{enable := false}}}, Acc) ->
(#{type := BridgeType}, Acc) -> Acc;
NumBridges = maps:get(num_bridges, Acc), (#{type := BridgeType}, Acc) ->
CountByType0 = maps:get(count_by_type, Acc), NumBridges = maps:get(num_bridges, Acc),
CountByType = maps:update_with( CountByType0 = maps:get(count_by_type, Acc),
binary_to_atom(BridgeType, utf8), CountByType = maps:update_with(
fun(X) -> X + 1 end, binary_to_atom(BridgeType, utf8),
1, fun(X) -> X + 1 end,
CountByType0), 1,
Acc#{ num_bridges => NumBridges + 1 CountByType0
, count_by_type => CountByType ),
} Acc#{
end, num_bridges => NumBridges + 1,
InitialAcc, count_by_type => CountByType
list()) }
end,
InitialAcc,
list()
)
catch catch
%% for instance, when the bridge app is not ready yet. %% for instance, when the bridge app is not ready yet.
_:_ -> _:_ ->

458
apps/emqx_bridge/src/emqx_bridge_api.erl
View File

@ -24,22 +24,23 @@
-import(hoconsc, [mk/2, array/1, enum/1]). -import(hoconsc, [mk/2, array/1, enum/1]).
%% Swagger specs from hocon schema %% Swagger specs from hocon schema
-export([ api_spec/0 -export([
, paths/0 api_spec/0,
, schema/1 paths/0,
, namespace/0 schema/1,
]). namespace/0
]).
%% API callbacks %% API callbacks
-export([ '/bridges'/2 -export([
, '/bridges/:id'/2 '/bridges'/2,
, '/bridges/:id/operation/:operation'/2 '/bridges/:id'/2,
, '/nodes/:node/bridges/:id/operation/:operation'/2 '/bridges/:id/operation/:operation'/2,
, '/bridges/:id/reset_metrics'/2 '/nodes/:node/bridges/:id/operation/:operation'/2,
]). '/bridges/:id/reset_metrics'/2
]).
-export([ lookup_from_local_node/2 -export([lookup_from_local_node/2]).
]).
-define(TYPES, [mqtt, http]). -define(TYPES, [mqtt, http]).
@ -51,35 +52,45 @@
EXPR EXPR
catch catch
error:{invalid_bridge_id, Id0} -> error:{invalid_bridge_id, Id0} ->
{400, error_msg('INVALID_ID', <<"invalid_bridge_id: ", Id0/binary, {400,
". Bridge Ids must be of format {type}:{name}">>)} error_msg(
end). 'INVALID_ID',
<<"invalid_bridge_id: ", Id0/binary,
". Bridge Ids must be of format {type}:{name}">>
)}
end
).
-define(METRICS(MATCH, SUCC, FAILED, RATE, RATE_5, RATE_MAX), -define(METRICS(MATCH, SUCC, FAILED, RATE, RATE_5, RATE_MAX), #{
#{ matched => MATCH, matched => MATCH,
success => SUCC, success => SUCC,
failed => FAILED, failed => FAILED,
rate => RATE, rate => RATE,
rate_last5m => RATE_5, rate_last5m => RATE_5,
rate_max => RATE_MAX rate_max => RATE_MAX
}). }).
-define(metrics(MATCH, SUCC, FAILED, RATE, RATE_5, RATE_MAX), -define(metrics(MATCH, SUCC, FAILED, RATE, RATE_5, RATE_MAX), #{
#{ matched := MATCH, matched := MATCH,
success := SUCC, success := SUCC,
failed := FAILED, failed := FAILED,
rate := RATE, rate := RATE,
rate_last5m := RATE_5, rate_last5m := RATE_5,
rate_max := RATE_MAX rate_max := RATE_MAX
}). }).
namespace() -> "bridge". namespace() -> "bridge".
api_spec() -> api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false}). emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false}).
paths() -> ["/bridges", "/bridges/:id", "/bridges/:id/operation/:operation", paths() ->
"/nodes/:node/bridges/:id/operation/:operation", [
"/bridges/:id/reset_metrics"]. "/bridges",
"/bridges/:id",
"/bridges/:id/operation/:operation",
"/nodes/:node/bridges/:id/operation/:operation",
"/bridges/:id/reset_metrics"
].
error_schema(Code, Message) when is_atom(Code) -> error_schema(Code, Message) when is_atom(Code) ->
error_schema([Code], Message); error_schema([Code], Message);
@ -89,40 +100,58 @@ error_schema(Codes, Message) when is_list(Codes) andalso is_binary(Message) ->
emqx_dashboard_swagger:error_codes(Codes, Message). emqx_dashboard_swagger:error_codes(Codes, Message).
get_response_body_schema() -> get_response_body_schema() ->
emqx_dashboard_swagger:schema_with_examples(emqx_bridge_schema:get_response(), emqx_dashboard_swagger:schema_with_examples(
bridge_info_examples(get)). emqx_bridge_schema:get_response(),
bridge_info_examples(get)
).
param_path_operation_cluster() -> param_path_operation_cluster() ->
{operation, mk(enum([enable, disable, stop, restart]), {operation,
#{ in => path mk(
, required => true enum([enable, disable, stop, restart]),
, example => <<"start">> #{
, desc => ?DESC("desc_param_path_operation_cluster") in => path,
})}. required => true,
example => <<"start">>,
desc => ?DESC("desc_param_path_operation_cluster")
}
)}.
param_path_operation_on_node() -> param_path_operation_on_node() ->
{operation, mk(enum([stop, restart]), {operation,
#{ in => path mk(
, required => true enum([stop, restart]),
, example => <<"start">> #{
, desc => ?DESC("desc_param_path_operation_on_node") in => path,
})}. required => true,
example => <<"start">>,
desc => ?DESC("desc_param_path_operation_on_node")
}
)}.
param_path_node() -> param_path_node() ->
{node, mk(binary(), {node,
#{ in => path mk(
, required => true binary(),
, example => <<"emqx@127.0.0.1">> #{
, desc => ?DESC("desc_param_path_node") in => path,
})}. required => true,
example => <<"emqx@127.0.0.1">>,
desc => ?DESC("desc_param_path_node")
}
)}.
param_path_id() -> param_path_id() ->
{id, mk(binary(), {id,
#{ in => path mk(
, required => true binary(),
, example => <<"http:my_http_bridge">> #{
, desc => ?DESC("desc_param_path_id") in => path,
})}. required => true,
example => <<"http:my_http_bridge">>,
desc => ?DESC("desc_param_path_id")
}
)}.
bridge_info_array_example(Method) -> bridge_info_array_example(Method) ->
[Config || #{value := Config} <- maps:values(bridge_info_examples(Method))]. [Config || #{value := Config} <- maps:values(bridge_info_examples(Method))].
@ -136,7 +165,8 @@ bridge_info_examples(Method) ->
}). }).
conn_bridge_examples(Method) -> conn_bridge_examples(Method) ->
lists:foldl(fun(Type, Acc) -> lists:foldl(
fun(Type, Acc) ->
SType = atom_to_list(Type), SType = atom_to_list(Type),
KeyIngress = bin(SType ++ "_ingress"), KeyIngress = bin(SType ++ "_ingress"),
KeyEgress = bin(SType ++ "_egress"), KeyEgress = bin(SType ++ "_egress"),
@ -150,19 +180,25 @@ conn_bridge_examples(Method) ->
value => info_example(Type, egress, Method) value => info_example(Type, egress, Method)
} }
}) })
end, #{}, ?CONN_TYPES). end,
#{},
?CONN_TYPES
).
info_example(Type, Direction, Method) -> info_example(Type, Direction, Method) ->
maps:merge(info_example_basic(Type, Direction), maps:merge(
method_example(Type, Direction, Method)). info_example_basic(Type, Direction),
method_example(Type, Direction, Method)
).
method_example(Type, Direction, Method) when Method == get; Method == post -> method_example(Type, Direction, Method) when Method == get; Method == post ->
SType = atom_to_list(Type), SType = atom_to_list(Type),
SDir = atom_to_list(Direction), SDir = atom_to_list(Direction),
SName = case Type of SName =
http -> "my_" ++ SType ++ "_bridge"; case Type of
_ -> "my_" ++ SDir ++ "_" ++ SType ++ "_bridge" http -> "my_" ++ SType ++ "_bridge";
end, _ -> "my_" ++ SDir ++ "_" ++ SType ++ "_bridge"
end,
TypeNameExamp = #{ TypeNameExamp = #{
type => bin(SType), type => bin(SType),
name => bin(SName) name => bin(SName)
@ -175,8 +211,10 @@ maybe_with_metrics_example(TypeNameExamp, get) ->
TypeNameExamp#{ TypeNameExamp#{
metrics => ?METRICS(0, 0, 0, 0, 0, 0), metrics => ?METRICS(0, 0, 0, 0, 0, 0),
node_metrics => [ node_metrics => [
#{node => node(), #{
metrics => ?METRICS(0, 0, 0, 0, 0, 0)} node => node(),
metrics => ?METRICS(0, 0, 0, 0, 0, 0)
}
] ]
}; };
maybe_with_metrics_example(TypeNameExamp, _) -> maybe_with_metrics_example(TypeNameExamp, _) ->
@ -231,8 +269,9 @@ schema("/bridges") ->
description => ?DESC("desc_api1"), description => ?DESC("desc_api1"),
responses => #{ responses => #{
200 => emqx_dashboard_swagger:schema_with_example( 200 => emqx_dashboard_swagger:schema_with_example(
array(emqx_bridge_schema:get_response()), array(emqx_bridge_schema:get_response()),
bridge_info_array_example(get)) bridge_info_array_example(get)
)
} }
}, },
post => #{ post => #{
@ -240,15 +279,15 @@ schema("/bridges") ->
summary => <<"Create Bridge">>, summary => <<"Create Bridge">>,
description => ?DESC("desc_api2"), description => ?DESC("desc_api2"),
'requestBody' => emqx_dashboard_swagger:schema_with_examples( 'requestBody' => emqx_dashboard_swagger:schema_with_examples(
emqx_bridge_schema:post_request(), emqx_bridge_schema:post_request(),
bridge_info_examples(post)), bridge_info_examples(post)
),
responses => #{ responses => #{
201 => get_response_body_schema(), 201 => get_response_body_schema(),
400 => error_schema('ALREADY_EXISTS', "Bridge already exists") 400 => error_schema('ALREADY_EXISTS', "Bridge already exists")
} }
} }
}; };
schema("/bridges/:id") -> schema("/bridges/:id") ->
#{ #{
'operationId' => '/bridges/:id', 'operationId' => '/bridges/:id',
@ -268,8 +307,9 @@ schema("/bridges/:id") ->
description => ?DESC("desc_api4"), description => ?DESC("desc_api4"),
parameters => [param_path_id()], parameters => [param_path_id()],
'requestBody' => emqx_dashboard_swagger:schema_with_examples( 'requestBody' => emqx_dashboard_swagger:schema_with_examples(
emqx_bridge_schema:put_request(), emqx_bridge_schema:put_request(),
bridge_info_examples(put)), bridge_info_examples(put)
),
responses => #{ responses => #{
200 => get_response_body_schema(), 200 => get_response_body_schema(),
404 => error_schema('NOT_FOUND', "Bridge not found"), 404 => error_schema('NOT_FOUND', "Bridge not found"),
@ -287,7 +327,6 @@ schema("/bridges/:id") ->
} }
} }
}; };
schema("/bridges/:id/reset_metrics") -> schema("/bridges/:id/reset_metrics") ->
#{ #{
'operationId' => '/bridges/:id/reset_metrics', 'operationId' => '/bridges/:id/reset_metrics',
@ -319,7 +358,6 @@ schema("/bridges/:id/operation/:operation") ->
} }
} }
}; };
schema("/nodes/:node/bridges/:id/operation/:operation") -> schema("/nodes/:node/bridges/:id/operation/:operation") ->
#{ #{
'operationId' => '/nodes/:node/bridges/:id/operation/:operation', 'operationId' => '/nodes/:node/bridges/:id/operation/:operation',
@ -336,7 +374,6 @@ schema("/nodes/:node/bridges/:id/operation/:operation") ->
200 => <<"Operation success">>, 200 => <<"Operation success">>,
400 => error_schema('INVALID_ID', "Bad bridge ID"), 400 => error_schema('INVALID_ID', "Bad bridge ID"),
403 => error_schema('FORBIDDEN_REQUEST', "forbidden operation") 403 => error_schema('FORBIDDEN_REQUEST', "forbidden operation")
} }
} }
}. }.
@ -353,15 +390,18 @@ schema("/nodes/:node/bridges/:id/operation/:operation") ->
end end
end; end;
'/bridges'(get, _Params) -> '/bridges'(get, _Params) ->
{200, zip_bridges([[format_resp(Data) || Data <- emqx_bridge_proto_v1:list_bridges(Node)] {200,
|| Node <- mria_mnesia:running_nodes()])}. zip_bridges([
[format_resp(Data) || Data <- emqx_bridge_proto_v1:list_bridges(Node)]
|| Node <- mria_mnesia:running_nodes()
])}.
'/bridges/:id'(get, #{bindings := #{id := Id}}) -> '/bridges/:id'(get, #{bindings := #{id := Id}}) ->
?TRY_PARSE_ID(Id, lookup_from_all_nodes(BridgeType, BridgeName, 200)); ?TRY_PARSE_ID(Id, lookup_from_all_nodes(BridgeType, BridgeName, 200));
'/bridges/:id'(put, #{bindings := #{id := Id}, body := Conf0}) -> '/bridges/:id'(put, #{bindings := #{id := Id}, body := Conf0}) ->
Conf = filter_out_request_body(Conf0), Conf = filter_out_request_body(Conf0),
?TRY_PARSE_ID(Id, ?TRY_PARSE_ID(
Id,
case emqx_bridge:lookup(BridgeType, BridgeName) of case emqx_bridge:lookup(BridgeType, BridgeName) of
{ok, _} -> {ok, _} ->
case ensure_bridge_created(BridgeType, BridgeName, Conf) of case ensure_bridge_created(BridgeType, BridgeName, Conf) of
@ -371,24 +411,31 @@ schema("/nodes/:node/bridges/:id/operation/:operation") ->
{400, Error} {400, Error}
end; end;
{error, not_found} -> {error, not_found} ->
{404, error_msg('NOT_FOUND',<<"bridge not found">>)} {404, error_msg('NOT_FOUND', <<"bridge not found">>)}
end); end
);
'/bridges/:id'(delete, #{bindings := #{id := Id}}) -> '/bridges/:id'(delete, #{bindings := #{id := Id}}) ->
?TRY_PARSE_ID(Id, ?TRY_PARSE_ID(
case emqx_conf:remove(emqx_bridge:config_key_path() ++ [BridgeType, BridgeName], Id,
#{override_to => cluster}) of case
emqx_conf:remove(
emqx_bridge:config_key_path() ++ [BridgeType, BridgeName],
#{override_to => cluster}
)
of
{ok, _} -> {204}; {ok, _} -> {204};
{error, Reason} -> {error, Reason} -> {500, error_msg('INTERNAL_ERROR', Reason)}
{500, error_msg('INTERNAL_ERROR', Reason)} end
end). ).
'/bridges/:id/reset_metrics'(put, #{bindings := #{id := Id}}) -> '/bridges/:id/reset_metrics'(put, #{bindings := #{id := Id}}) ->
?TRY_PARSE_ID(Id, ?TRY_PARSE_ID(
Id,
case emqx_bridge:reset_metrics(emqx_bridge:resource_id(BridgeType, BridgeName)) of case emqx_bridge:reset_metrics(emqx_bridge:resource_id(BridgeType, BridgeName)) of
ok -> {200, <<"Reset success">>}; ok -> {200, <<"Reset success">>};
Reason -> {400, error_msg('BAD_REQUEST', Reason)} Reason -> {400, error_msg('BAD_REQUEST', Reason)}
end). end
).
lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) -> lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) ->
Nodes = mria_mnesia:running_nodes(), Nodes = mria_mnesia:running_nodes(),
@ -407,40 +454,58 @@ lookup_from_local_node(BridgeType, BridgeName) ->
Error -> Error Error -> Error
end. end.
'/bridges/:id/operation/:operation'(post, #{bindings := '/bridges/:id/operation/:operation'(post, #{
#{id := Id, operation := Op}}) -> bindings :=
?TRY_PARSE_ID(Id, case operation_func(Op) of #{id := Id, operation := Op}
invalid -> {400, error_msg('BAD_REQUEST', <<"invalid operation">>)}; }) ->
OperFunc when OperFunc == enable; OperFunc == disable -> ?TRY_PARSE_ID(
case emqx_conf:update(emqx_bridge:config_key_path() ++ [BridgeType, BridgeName], Id,
{OperFunc, BridgeType, BridgeName}, #{override_to => cluster}) of case operation_func(Op) of
{ok, _} -> {200}; invalid ->
{error, {pre_config_update, _, bridge_not_found}} -> {400, error_msg('BAD_REQUEST', <<"invalid operation">>)};
{404, error_msg('NOT_FOUND', <<"bridge not found">>)}; OperFunc when OperFunc == enable; OperFunc == disable ->
{error, Reason} -> case
{500, error_msg('INTERNAL_ERROR', Reason)} emqx_conf:update(
end; emqx_bridge:config_key_path() ++ [BridgeType, BridgeName],
OperFunc -> {OperFunc, BridgeType, BridgeName},
Nodes = mria_mnesia:running_nodes(), #{override_to => cluster}
operation_to_all_nodes(Nodes, OperFunc, BridgeType, BridgeName) )
end). of
{ok, _} ->
{200};
{error, {pre_config_update, _, bridge_not_found}} ->
{404, error_msg('NOT_FOUND', <<"bridge not found">>)};
{error, Reason} ->
{500, error_msg('INTERNAL_ERROR', Reason)}
end;
OperFunc ->
Nodes = mria_mnesia:running_nodes(),
operation_to_all_nodes(Nodes, OperFunc, BridgeType, BridgeName)
end
).
'/nodes/:node/bridges/:id/operation/:operation'(post, #{bindings := '/nodes/:node/bridges/:id/operation/:operation'(post, #{
#{id := Id, operation := Op}}) -> bindings :=
?TRY_PARSE_ID(Id, case operation_func(Op) of #{id := Id, operation := Op}
invalid -> {400, error_msg('BAD_REQUEST', <<"invalid operation">>)}; }) ->
OperFunc when OperFunc == restart; OperFunc == stop -> ?TRY_PARSE_ID(
ConfMap = emqx:get_config([bridges, BridgeType, BridgeName]), Id,
case maps:get(enable, ConfMap, false) of case operation_func(Op) of
false -> {403, error_msg('FORBIDDEN_REQUEST', <<"forbidden operation">>)}; invalid ->
true -> {400, error_msg('BAD_REQUEST', <<"invalid operation">>)};
case emqx_bridge:OperFunc(BridgeType, BridgeName) of OperFunc when OperFunc == restart; OperFunc == stop ->
ok -> {200}; ConfMap = emqx:get_config([bridges, BridgeType, BridgeName]),
{error, Reason} -> case maps:get(enable, ConfMap, false) of
{500, error_msg('INTERNAL_ERROR', Reason)} false ->
end {403, error_msg('FORBIDDEN_REQUEST', <<"forbidden operation">>)};
end true ->
end). case emqx_bridge:OperFunc(BridgeType, BridgeName) of
ok -> {200};
{error, Reason} -> {500, error_msg('INTERNAL_ERROR', Reason)}
end
end
end
).
operation_func(<<"stop">>) -> stop; operation_func(<<"stop">>) -> stop;
operation_func(<<"restart">>) -> restart; operation_func(<<"restart">>) -> restart;
@ -449,10 +514,11 @@ operation_func(<<"disable">>) -> disable;
operation_func(_) -> invalid. operation_func(_) -> invalid.
operation_to_all_nodes(Nodes, OperFunc, BridgeType, BridgeName) -> operation_to_all_nodes(Nodes, OperFunc, BridgeType, BridgeName) ->
RpcFunc = case OperFunc of RpcFunc =
restart -> restart_bridges_to_all_nodes; case OperFunc of
stop -> stop_bridges_to_all_nodes restart -> restart_bridges_to_all_nodes;
end, stop -> stop_bridges_to_all_nodes
end,
case is_ok(emqx_bridge_proto_v1:RpcFunc(Nodes, BridgeType, BridgeName)) of case is_ok(emqx_bridge_proto_v1:RpcFunc(Nodes, BridgeType, BridgeName)) of
{ok, _} -> {ok, _} ->
{200}; {200};
@ -461,48 +527,70 @@ operation_to_all_nodes(Nodes, OperFunc, BridgeType, BridgeName) ->
end. end.
ensure_bridge_created(BridgeType, BridgeName, Conf) -> ensure_bridge_created(BridgeType, BridgeName, Conf) ->
case emqx_conf:update(emqx_bridge:config_key_path() ++ [BridgeType, BridgeName], case
Conf, #{override_to => cluster}) of emqx_conf:update(
emqx_bridge:config_key_path() ++ [BridgeType, BridgeName],
Conf,
#{override_to => cluster}
)
of
{ok, _} -> ok; {ok, _} -> ok;
{error, Reason} -> {error, Reason} -> {error, error_msg('BAD_REQUEST', Reason)}
{error, error_msg('BAD_REQUEST', Reason)}
end. end.
zip_bridges([BridgesFirstNode | _] = BridgesAllNodes) -> zip_bridges([BridgesFirstNode | _] = BridgesAllNodes) ->
lists:foldl(fun(#{type := Type, name := Name}, Acc) -> lists:foldl(
fun(#{type := Type, name := Name}, Acc) ->
Bridges = pick_bridges_by_id(Type, Name, BridgesAllNodes), Bridges = pick_bridges_by_id(Type, Name, BridgesAllNodes),
[format_bridge_info(Bridges) | Acc] [format_bridge_info(Bridges) | Acc]
end, [], BridgesFirstNode). end,
[],
BridgesFirstNode
).
pick_bridges_by_id(Type, Name, BridgesAllNodes) -> pick_bridges_by_id(Type, Name, BridgesAllNodes) ->
lists:foldl(fun(BridgesOneNode, Acc) -> lists:foldl(
case [Bridge || Bridge = #{type := Type0, name := Name0} <- BridgesOneNode, fun(BridgesOneNode, Acc) ->
Type0 == Type, Name0 == Name] of case
[BridgeInfo] -> [BridgeInfo | Acc]; [
Bridge
|| Bridge = #{type := Type0, name := Name0} <- BridgesOneNode,
Type0 == Type,
Name0 == Name
]
of
[BridgeInfo] ->
[BridgeInfo | Acc];
[] -> [] ->
?SLOG(warning, #{msg => "bridge_inconsistent_in_cluster", ?SLOG(warning, #{
bridge => emqx_bridge:bridge_id(Type, Name)}), msg => "bridge_inconsistent_in_cluster",
bridge => emqx_bridge:bridge_id(Type, Name)
}),
Acc Acc
end end
end, [], BridgesAllNodes). end,
[],
BridgesAllNodes
).
format_bridge_info([FirstBridge | _] = Bridges) -> format_bridge_info([FirstBridge | _] = Bridges) ->
Res = maps:remove(node, FirstBridge), Res = maps:remove(node, FirstBridge),
NodeStatus = collect_status(Bridges), NodeStatus = collect_status(Bridges),
NodeMetrics = collect_metrics(Bridges), NodeMetrics = collect_metrics(Bridges),
Res#{ status => aggregate_status(NodeStatus) Res#{
, node_status => NodeStatus status => aggregate_status(NodeStatus),
, metrics => aggregate_metrics(NodeMetrics) node_status => NodeStatus,
, node_metrics => NodeMetrics metrics => aggregate_metrics(NodeMetrics),
}. node_metrics => NodeMetrics
}.
collect_status(Bridges) -> collect_status(Bridges) ->
[maps:with([node, status], B) || B <- Bridges]. [maps:with([node, status], B) || B <- Bridges].
aggregate_status(AllStatus) -> aggregate_status(AllStatus) ->
Head = fun ([A | _]) -> A end, Head = fun([A | _]) -> A end,
HeadVal = maps:get(status, Head(AllStatus), connecting), HeadVal = maps:get(status, Head(AllStatus), connecting),
AllRes = lists:all(fun (#{status := Val}) -> Val == HeadVal end, AllStatus), AllRes = lists:all(fun(#{status := Val}) -> Val == HeadVal end, AllStatus),
case AllRes of case AllRes of
true -> HeadVal; true -> HeadVal;
false -> inconsistent false -> inconsistent
@ -512,15 +600,31 @@ collect_metrics(Bridges) ->
[maps:with([node, metrics], B) || B <- Bridges]. [maps:with([node, metrics], B) || B <- Bridges].
aggregate_metrics(AllMetrics) -> aggregate_metrics(AllMetrics) ->
InitMetrics = ?METRICS(0,0,0,0,0,0), InitMetrics = ?METRICS(0, 0, 0, 0, 0, 0),
lists:foldl(fun(#{metrics := ?metrics(Match1, Succ1, Failed1, Rate1, Rate5m1, RateMax1)}, lists:foldl(
?metrics(Match0, Succ0, Failed0, Rate0, Rate5m0, RateMax0)) -> fun(
?METRICS(Match1 + Match0, Succ1 + Succ0, Failed1 + Failed0, #{metrics := ?metrics(Match1, Succ1, Failed1, Rate1, Rate5m1, RateMax1)},
Rate1 + Rate0, Rate5m1 + Rate5m0, RateMax1 + RateMax0) ?metrics(Match0, Succ0, Failed0, Rate0, Rate5m0, RateMax0)
end, InitMetrics, AllMetrics). ) ->
?METRICS(
Match1 + Match0,
Succ1 + Succ0,
Failed1 + Failed0,
Rate1 + Rate0,
Rate5m1 + Rate5m0,
RateMax1 + RateMax0
)
end,
InitMetrics,
AllMetrics
).
format_resp(#{type := Type, name := BridgeName, raw_config := RawConf, format_resp(#{
resource_data := #{status := Status, metrics := Metrics}}) -> type := Type,
name := BridgeName,
raw_config := RawConf,
resource_data := #{status := Status, metrics := Metrics}
}) ->
RawConfFull = fill_defaults(Type, RawConf), RawConfFull = fill_defaults(Type, RawConf),
RawConfFull#{ RawConfFull#{
type => Type, type => Type,
@ -531,10 +635,11 @@ format_resp(#{type := Type, name := BridgeName, raw_config := RawConf,
}. }.
format_metrics(#{ format_metrics(#{
counters := #{failed := Failed, exception := Ex, matched := Match, success := Succ}, counters := #{failed := Failed, exception := Ex, matched := Match, success := Succ},
rate := #{ rate := #{
matched := #{current := Rate, last5m := Rate5m, max := RateMax} matched := #{current := Rate, last5m := Rate5m, max := RateMax}
} }) -> }
}) ->
?METRICS(Match, Succ, Failed + Ex, Rate, Rate5m, RateMax). ?METRICS(Match, Succ, Failed + Ex, Rate, Rate5m, RateMax).
fill_defaults(Type, RawConf) -> fill_defaults(Type, RawConf) ->
@ -551,14 +656,31 @@ unpack_bridge_conf(Type, PackedConf) ->
RawConf. RawConf.
is_ok(ResL) -> is_ok(ResL) ->
case lists:filter(fun({ok, _}) -> false; (ok) -> false; (_) -> true end, ResL) of case
lists:filter(
fun
({ok, _}) -> false;
(ok) -> false;
(_) -> true
end,
ResL
)
of
[] -> {ok, [Res || {ok, Res} <- ResL]}; [] -> {ok, [Res || {ok, Res} <- ResL]};
ErrL -> {error, ErrL} ErrL -> {error, ErrL}
end. end.
filter_out_request_body(Conf) -> filter_out_request_body(Conf) ->
ExtraConfs = [<<"id">>, <<"type">>, <<"name">>, <<"status">>, <<"node_status">>, ExtraConfs = [
<<"node_metrics">>, <<"metrics">>, <<"node">>], <<"id">>,
<<"type">>,
<<"name">>,
<<"status">>,
<<"node_status">>,
<<"node_metrics">>,
<<"metrics">>,
<<"node">>
],
maps:without(ExtraConfs, Conf). maps:without(ExtraConfs, Conf).
error_msg(Code, Msg) when is_binary(Msg) -> error_msg(Code, Msg) when is_binary(Msg) ->

7
apps/emqx_bridge/src/emqx_bridge_app.erl
View File

@ -19,9 +19,10 @@
-export([start/2, stop/1]). -export([start/2, stop/1]).
-export([ pre_config_update/3 -export([
, post_config_update/5 pre_config_update/3,
]). post_config_update/5
]).
-define(TOP_LELVE_HDLR_PATH, (emqx_bridge:config_key_path())). -define(TOP_LELVE_HDLR_PATH, (emqx_bridge:config_key_path())).
-define(LEAF_NODE_HDLR_PATH, (emqx_bridge:config_key_path() ++ ['?', '?'])). -define(LEAF_NODE_HDLR_PATH, (emqx_bridge:config_key_path() ++ ['?', '?'])).

146
apps/emqx_bridge/src/emqx_bridge_http_schema.erl
View File

@ -15,45 +15,66 @@ roots() -> [].
fields("config") -> fields("config") ->
basic_config() ++ basic_config() ++
[ {url, mk(binary(), [
#{ required => true {url,
, desc => ?DESC("config_url") mk(
})} binary(),
, {local_topic, mk(binary(), #{
#{ desc => ?DESC("config_local_topic") required => true,
})} desc => ?DESC("config_url")
, {method, mk(method(), }
#{ default => post )},
, desc => ?DESC("config_method") {local_topic,
})} mk(
, {headers, mk(map(), binary(),
#{ default => #{ #{desc => ?DESC("config_local_topic")}
<<"accept">> => <<"application/json">>, )},
<<"cache-control">> => <<"no-cache">>, {method,
<<"connection">> => <<"keep-alive">>, mk(
<<"content-type">> => <<"application/json">>, method(),
<<"keep-alive">> => <<"timeout=5">>} #{
, desc => ?DESC("config_headers") default => post,
}) desc => ?DESC("config_method")
} }
, {body, mk(binary(), )},
#{ default => <<"${payload}">> {headers,
, desc => ?DESC("config_body") mk(
})} map(),
, {request_timeout, mk(emqx_schema:duration_ms(), #{
#{ default => <<"15s">> default => #{
, desc => ?DESC("config_request_timeout") <<"accept">> => <<"application/json">>,
})} <<"cache-control">> => <<"no-cache">>,
]; <<"connection">> => <<"keep-alive">>,
<<"content-type">> => <<"application/json">>,
<<"keep-alive">> => <<"timeout=5">>
},
desc => ?DESC("config_headers")
}
)},
{body,
mk(
binary(),
#{
default => <<"${payload}">>,
desc => ?DESC("config_body")
}
)},
{request_timeout,
mk(
emqx_schema:duration_ms(),
#{
default => <<"15s">>,
desc => ?DESC("config_request_timeout")
}
)}
];
fields("post") -> fields("post") ->
[ type_field() [
, name_field() type_field(),
name_field()
] ++ fields("config"); ] ++ fields("config");
fields("put") -> fields("put") ->
fields("config"); fields("config");
fields("get") -> fields("get") ->
emqx_bridge_schema:metrics_status_fields() ++ fields("post"). emqx_bridge_schema:metrics_status_fields() ++ fields("post").
@ -65,32 +86,47 @@ desc(_) ->
undefined. undefined.
basic_config() -> basic_config() ->
[ {enable, [
mk(boolean(), {enable,
#{ desc => ?DESC("config_enable") mk(
, default => true boolean(),
})} #{
, {direction, desc => ?DESC("config_enable"),
mk(egress, default => true
#{ desc => ?DESC("config_direction") }
, default => egress )},
})} {direction,
] mk(
++ proplists:delete(base_url, emqx_connector_http:fields(config)). egress,
#{
desc => ?DESC("config_direction"),
default => egress
}
)}
] ++
proplists:delete(base_url, emqx_connector_http:fields(config)).
%%====================================================================================== %%======================================================================================
type_field() -> type_field() ->
{type, mk(http, {type,
#{ required => true mk(
, desc => ?DESC("desc_type") http,
})}. #{
required => true,
desc => ?DESC("desc_type")
}
)}.
name_field() -> name_field() ->
{name, mk(binary(), {name,
#{ required => true mk(
, desc => ?DESC("desc_name") binary(),
})}. #{
required => true,
desc => ?DESC("desc_name")
}
)}.
method() -> method() ->
enum([post, put, get, delete]). enum([post, put, get, delete]).

49
apps/emqx_bridge/src/emqx_bridge_monitor.erl
View File

@ -22,17 +22,20 @@
-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl").
%% API functions %% API functions
-export([ start_link/0 -export([
, ensure_all_started/1 start_link/0,
]). ensure_all_started/1
]).
%% gen_server callbacks %% gen_server callbacks
-export([init/1, -export([
handle_call/3, init/1,
handle_cast/2, handle_call/3,
handle_info/2, handle_cast/2,
terminate/2, handle_info/2,
code_change/3]). terminate/2,
code_change/3
]).
-record(state, {}). -record(state, {}).
@ -52,7 +55,6 @@ handle_call(_Request, _From, State) ->
handle_cast({start_and_monitor, Configs}, State) -> handle_cast({start_and_monitor, Configs}, State) ->
ok = load_bridges(Configs), ok = load_bridges(Configs),
{noreply, State}; {noreply, State};
handle_cast(_Msg, State) -> handle_cast(_Msg, State) ->
{noreply, State}. {noreply, State}.
@ -67,13 +69,22 @@ code_change(_OldVsn, State, _Extra) ->
%%============================================================================ %%============================================================================
load_bridges(Configs) -> load_bridges(Configs) ->
lists:foreach(fun({Type, NamedConf}) -> lists:foreach(
lists:foreach(fun({Name, Conf}) -> fun({Type, NamedConf}) ->
lists:foreach(
fun({Name, Conf}) ->
_Res = emqx_bridge:create(Type, Name, Conf), _Res = emqx_bridge:create(Type, Name, Conf),
?tp(emqx_bridge_monitor_loaded_bridge, ?tp(
#{ type => Type emqx_bridge_monitor_loaded_bridge,
, name => Name #{
, res => _Res type => Type,
}) name => Name,
end, maps:to_list(NamedConf)) res => _Res
end, maps:to_list(Configs)). }
)
end,
maps:to_list(NamedConf)
)
end,
maps:to_list(Configs)
).

52
apps/emqx_bridge/src/emqx_bridge_mqtt_schema.erl
View File

@ -12,31 +12,27 @@
roots() -> []. roots() -> [].
fields("ingress") -> fields("ingress") ->
[ emqx_bridge_schema:direction_field(ingress, emqx_connector_mqtt_schema:ingress_desc()) [emqx_bridge_schema:direction_field(ingress, emqx_connector_mqtt_schema:ingress_desc())] ++
] emqx_bridge_schema:common_bridge_fields() ++
++ emqx_bridge_schema:common_bridge_fields() proplists:delete(hookpoint, emqx_connector_mqtt_schema:fields("ingress"));
++ proplists:delete(hookpoint, emqx_connector_mqtt_schema:fields("ingress"));
fields("egress") -> fields("egress") ->
[ emqx_bridge_schema:direction_field(egress, emqx_connector_mqtt_schema:egress_desc()) [emqx_bridge_schema:direction_field(egress, emqx_connector_mqtt_schema:egress_desc())] ++
] emqx_bridge_schema:common_bridge_fields() ++
++ emqx_bridge_schema:common_bridge_fields() emqx_connector_mqtt_schema:fields("egress");
++ emqx_connector_mqtt_schema:fields("egress");
fields("post_ingress") -> fields("post_ingress") ->
[ type_field() [
, name_field() type_field(),
name_field()
] ++ proplists:delete(enable, fields("ingress")); ] ++ proplists:delete(enable, fields("ingress"));
fields("post_egress") -> fields("post_egress") ->
[ type_field() [
, name_field() type_field(),
name_field()
] ++ proplists:delete(enable, fields("egress")); ] ++ proplists:delete(enable, fields("egress"));
fields("put_ingress") -> fields("put_ingress") ->
proplists:delete(enable, fields("ingress")); proplists:delete(enable, fields("ingress"));
fields("put_egress") -> fields("put_egress") ->
proplists:delete(enable, fields("egress")); proplists:delete(enable, fields("egress"));
fields("get_ingress") -> fields("get_ingress") ->
emqx_bridge_schema:metrics_status_fields() ++ fields("post_ingress"); emqx_bridge_schema:metrics_status_fields() ++ fields("post_ingress");
fields("get_egress") -> fields("get_egress") ->
@ -49,13 +45,21 @@ desc(_) ->
%%====================================================================================== %%======================================================================================
type_field() -> type_field() ->
{type, mk(mqtt, {type,
#{ required => true mk(
, desc => ?DESC("desc_type") mqtt,
})}. #{
required => true,
desc => ?DESC("desc_type")
}
)}.
name_field() -> name_field() ->
{name, mk(binary(), {name,
#{ required => true mk(
, desc => ?DESC("desc_name") binary(),
})}. #{
required => true,
desc => ?DESC("desc_name")
}
)}.

159
apps/emqx_bridge/src/emqx_bridge_schema.erl
View File

@ -7,15 +7,17 @@
-export([roots/0, fields/1, desc/1, namespace/0]). -export([roots/0, fields/1, desc/1, namespace/0]).
-export([ get_response/0 -export([
, put_request/0 get_response/0,
, post_request/0 put_request/0,
]). post_request/0
]).
-export([ common_bridge_fields/0 -export([
, metrics_status_fields/0 common_bridge_fields/0,
, direction_field/2 metrics_status_fields/0,
]). direction_field/2
]).
%%====================================================================================== %%======================================================================================
%% Hocon Schema Definitions %% Hocon Schema Definitions
@ -34,43 +36,68 @@ post_request() ->
http_schema("post"). http_schema("post").
http_schema(Method) -> http_schema(Method) ->
Schemas = lists:flatmap(fun(Type) -> Schemas = lists:flatmap(
[ref(schema_mod(Type), Method ++ "_ingress"), fun(Type) ->
ref(schema_mod(Type), Method ++ "_egress")] [
end, ?CONN_TYPES), ref(schema_mod(Type), Method ++ "_ingress"),
hoconsc:union([ref(emqx_bridge_http_schema, Method) ref(schema_mod(Type), Method ++ "_egress")
| Schemas]). ]
end,
?CONN_TYPES
),
hoconsc:union([
ref(emqx_bridge_http_schema, Method)
| Schemas
]).
common_bridge_fields() -> common_bridge_fields() ->
[ {enable, [
mk(boolean(), {enable,
#{ desc => ?DESC("desc_enable") mk(
, default => true boolean(),
})} #{
, {connector, desc => ?DESC("desc_enable"),
mk(binary(), default => true
#{ required => true }
, example => <<"mqtt:my_mqtt_connector">> )},
, desc => ?DESC("desc_connector") {connector,
})} mk(
binary(),
#{
required => true,
example => <<"mqtt:my_mqtt_connector">>,
desc => ?DESC("desc_connector")
}
)}
]. ].
metrics_status_fields() -> metrics_status_fields() ->
[ {"metrics", mk(ref(?MODULE, "metrics"), #{desc => ?DESC("desc_metrics")})} [
, {"node_metrics", mk(hoconsc:array(ref(?MODULE, "node_metrics")), {"metrics", mk(ref(?MODULE, "metrics"), #{desc => ?DESC("desc_metrics")})},
#{ desc => ?DESC("desc_node_metrics")})} {"node_metrics",
, {"status", mk(status(), #{desc => ?DESC("desc_status")})} mk(
, {"node_status", mk(hoconsc:array(ref(?MODULE, "node_status")), hoconsc:array(ref(?MODULE, "node_metrics")),
#{ desc => ?DESC("desc_node_status")})} #{desc => ?DESC("desc_node_metrics")}
)},
{"status", mk(status(), #{desc => ?DESC("desc_status")})},
{"node_status",
mk(
hoconsc:array(ref(?MODULE, "node_status")),
#{desc => ?DESC("desc_node_status")}
)}
]. ].
direction_field(Dir, Desc) -> direction_field(Dir, Desc) ->
{direction, mk(Dir, {direction,
#{ required => true mk(
, default => egress Dir,
, desc => "The direction of the bridge. Can be one of 'ingress' or 'egress'.</br>" #{
++ Desc required => true,
})}. default => egress,
desc => "The direction of the bridge. Can be one of 'ingress' or 'egress'.</br>" ++
Desc
}
)}.
%%====================================================================================== %%======================================================================================
%% For config files %% For config files
@ -80,31 +107,49 @@ namespace() -> "bridge".
roots() -> [bridges]. roots() -> [bridges].
fields(bridges) -> fields(bridges) ->
[{http, mk(hoconsc:map(name, ref(emqx_bridge_http_schema, "config")), [
#{desc => ?DESC("bridges_http")})}] {http,
++ [{T, mk(hoconsc:map(name, hoconsc:union([ ref(schema_mod(T), "ingress") mk(
, ref(schema_mod(T), "egress") hoconsc:map(name, ref(emqx_bridge_http_schema, "config")),
])), #{desc => ?DESC("bridges_http")}
#{desc => ?DESC("bridges_name")})} || T <- ?CONN_TYPES]; )}
] ++
[
{T,
mk(
hoconsc:map(
name,
hoconsc:union([
ref(schema_mod(T), "ingress"),
ref(schema_mod(T), "egress")
])
),
#{desc => ?DESC("bridges_name")}
)}
|| T <- ?CONN_TYPES
];
fields("metrics") -> fields("metrics") ->
[ {"matched", mk(integer(), #{desc => ?DESC("metric_matched")})} [
, {"success", mk(integer(), #{desc => ?DESC("metric_success")})} {"matched", mk(integer(), #{desc => ?DESC("metric_matched")})},
, {"failed", mk(integer(), #{desc => ?DESC("metric_failed")})} {"success", mk(integer(), #{desc => ?DESC("metric_success")})},
, {"rate", mk(float(), #{desc => ?DESC("metric_rate")})} {"failed", mk(integer(), #{desc => ?DESC("metric_failed")})},
, {"rate_max", mk(float(), #{desc => ?DESC("metric_rate_max")})} {"rate", mk(float(), #{desc => ?DESC("metric_rate")})},
, {"rate_last5m", mk(float(), {"rate_max", mk(float(), #{desc => ?DESC("metric_rate_max")})},
#{desc => ?DESC("metric_rate_last5m")})} {"rate_last5m",
mk(
float(),
#{desc => ?DESC("metric_rate_last5m")}
)}
]; ];
fields("node_metrics") -> fields("node_metrics") ->
[ node_name() [
, {"metrics", mk(ref(?MODULE, "metrics"), #{})} node_name(),
{"metrics", mk(ref(?MODULE, "metrics"), #{})}
]; ];
fields("node_status") -> fields("node_status") ->
[ node_name() [
, {"status", mk(status(), #{})} node_name(),
{"status", mk(status(), #{})}
]. ].
desc(bridges) -> desc(bridges) ->

20
apps/emqx_bridge/src/emqx_bridge_sup.erl
View File

@ -27,15 +27,19 @@ start_link() ->
supervisor:start_link({local, ?SERVER}, ?MODULE, []). supervisor:start_link({local, ?SERVER}, ?MODULE, []).
init([]) -> init([]) ->
SupFlags = #{strategy => one_for_one, SupFlags = #{
intensity => 10, strategy => one_for_one,
period => 10}, intensity => 10,
period => 10
},
ChildSpecs = [ ChildSpecs = [
#{id => emqx_bridge_monitor, #{
start => {emqx_bridge_monitor, start_link, []}, id => emqx_bridge_monitor,
restart => permanent, start => {emqx_bridge_monitor, start_link, []},
type => worker, restart => permanent,
modules => [emqx_bridge_monitor]} type => worker,
modules => [emqx_bridge_monitor]
}
], ],
{ok, {SupFlags, ChildSpecs}}. {ok, {SupFlags, ChildSpecs}}.

46
apps/emqx_bridge/src/proto/emqx_bridge_proto_v1.erl
View File

@ -18,13 +18,14 @@
-behaviour(emqx_bpapi). -behaviour(emqx_bpapi).
-export([ introduced_in/0 -export([
introduced_in/0,
, list_bridges/1 list_bridges/1,
, lookup_from_all_nodes/3 lookup_from_all_nodes/3,
, restart_bridges_to_all_nodes/3 restart_bridges_to_all_nodes/3,
, stop_bridges_to_all_nodes/3 stop_bridges_to_all_nodes/3
]). ]).
-include_lib("emqx/include/bpapi.hrl"). -include_lib("emqx/include/bpapi.hrl").
@ -40,19 +41,34 @@ list_bridges(Node) ->
-type key() :: atom() | binary() | [byte()]. -type key() :: atom() | binary() | [byte()].
-spec restart_bridges_to_all_nodes([node()], key(), key()) -> -spec restart_bridges_to_all_nodes([node()], key(), key()) ->
emqx_rpc:erpc_multicall(). emqx_rpc:erpc_multicall().
restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) ->
erpc:multicall(Nodes, emqx_bridge, restart, erpc:multicall(
[BridgeType, BridgeName], ?TIMEOUT). Nodes,
emqx_bridge,
restart,
[BridgeType, BridgeName],
?TIMEOUT
).
-spec stop_bridges_to_all_nodes([node()], key(), key()) -> -spec stop_bridges_to_all_nodes([node()], key(), key()) ->
emqx_rpc:erpc_multicall(). emqx_rpc:erpc_multicall().
stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) ->
erpc:multicall(Nodes, emqx_bridge, stop, erpc:multicall(
[BridgeType, BridgeName], ?TIMEOUT). Nodes,
emqx_bridge,
stop,
[BridgeType, BridgeName],
?TIMEOUT
).
-spec lookup_from_all_nodes([node()], key(), key()) -> -spec lookup_from_all_nodes([node()], key(), key()) ->
emqx_rpc:erpc_multicall(). emqx_rpc:erpc_multicall().
lookup_from_all_nodes(Nodes, BridgeType, BridgeName) -> lookup_from_all_nodes(Nodes, BridgeType, BridgeName) ->
erpc:multicall(Nodes, emqx_bridge_api, lookup_from_local_node, erpc:multicall(
[BridgeType, BridgeName], ?TIMEOUT). Nodes,
emqx_bridge_api,
lookup_from_local_node,
[BridgeType, BridgeName],
?TIMEOUT
).

115
apps/emqx_bridge/test/emqx_bridge_SUITE.erl
View File

@ -23,7 +23,7 @@
-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl").
all() -> all() ->
emqx_common_test_helpers:all(?MODULE). emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) -> init_per_suite(Config) ->
%% to avoid inter-suite dependencies %% to avoid inter-suite dependencies
@ -32,8 +32,12 @@ init_per_suite(Config) ->
Config. Config.
end_per_suite(_Config) -> end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([emqx, emqx_bridge, emqx_common_test_helpers:stop_apps([
emqx_resource, emqx_connector]). emqx,
emqx_bridge,
emqx_resource,
emqx_connector
]).
init_per_testcase(t_get_basic_usage_info_1, Config) -> init_per_testcase(t_get_basic_usage_info_1, Config) ->
setup_fake_telemetry_data(), setup_fake_telemetry_data(),
@ -43,13 +47,15 @@ init_per_testcase(_TestCase, Config) ->
end_per_testcase(t_get_basic_usage_info_1, _Config) -> end_per_testcase(t_get_basic_usage_info_1, _Config) ->
lists:foreach( lists:foreach(
fun({BridgeType, BridgeName}) -> fun({BridgeType, BridgeName}) ->
ok = emqx_bridge:remove(BridgeType, BridgeName) ok = emqx_bridge:remove(BridgeType, BridgeName)
end, end,
[ {http, <<"basic_usage_info_http">>} [
, {http, <<"basic_usage_info_http_disabled">>} {http, <<"basic_usage_info_http">>},
, {mqtt, <<"basic_usage_info_mqtt">>} {http, <<"basic_usage_info_http_disabled">>},
]), {mqtt, <<"basic_usage_info_mqtt">>}
]
),
ok = emqx_config:delete_override_conf_files(), ok = emqx_config:delete_override_conf_files(),
ok = emqx_config:put([bridges], #{}), ok = emqx_config:put([bridges], #{}),
ok = emqx_config:put_raw([bridges], #{}), ok = emqx_config:put_raw([bridges], #{}),
@ -59,53 +65,68 @@ end_per_testcase(_TestCase, _Config) ->
t_get_basic_usage_info_0(_Config) -> t_get_basic_usage_info_0(_Config) ->
?assertEqual( ?assertEqual(
#{ num_bridges => 0 #{
, count_by_type => #{} num_bridges => 0,
count_by_type => #{}
}, },
emqx_bridge:get_basic_usage_info()). emqx_bridge:get_basic_usage_info()
).
t_get_basic_usage_info_1(_Config) -> t_get_basic_usage_info_1(_Config) ->
BasicUsageInfo = emqx_bridge:get_basic_usage_info(), BasicUsageInfo = emqx_bridge:get_basic_usage_info(),
?assertEqual( ?assertEqual(
#{ num_bridges => 2 #{
, count_by_type => #{ http => 1 num_bridges => 2,
, mqtt => 1 count_by_type => #{
} http => 1,
mqtt => 1
}
}, },
BasicUsageInfo). BasicUsageInfo
).
setup_fake_telemetry_data() -> setup_fake_telemetry_data() ->
ConnectorConf = ConnectorConf =
#{<<"connectors">> => #{
#{<<"mqtt">> => #{<<"my_mqtt_connector">> => <<"connectors">> =>
#{ server => "127.0.0.1:1883" }}}}, #{
MQTTConfig = #{ connector => <<"mqtt:my_mqtt_connector">> <<"mqtt">> => #{
, enable => true <<"my_mqtt_connector">> =>
, direction => ingress #{server => "127.0.0.1:1883"}
, remote_topic => <<"aws/#">> }
, remote_qos => 1
},
HTTPConfig = #{ url => <<"http://localhost:9901/messages/${topic}">>
, enable => true
, direction => egress
, local_topic => "emqx_http/#"
, method => post
, body => <<"${payload}">>
, headers => #{}
, request_timeout => "15s"
},
Conf =
#{ <<"bridges">> =>
#{ <<"http">> =>
#{ <<"basic_usage_info_http">> => HTTPConfig
, <<"basic_usage_info_http_disabled">> =>
HTTPConfig#{enable => false}
}
, <<"mqtt">> =>
#{ <<"basic_usage_info_mqtt">> => MQTTConfig
}
} }
}, },
MQTTConfig = #{
connector => <<"mqtt:my_mqtt_connector">>,
enable => true,
direction => ingress,
remote_topic => <<"aws/#">>,
remote_qos => 1
},
HTTPConfig = #{
url => <<"http://localhost:9901/messages/${topic}">>,
enable => true,
direction => egress,
local_topic => "emqx_http/#",
method => post,
body => <<"${payload}">>,
headers => #{},
request_timeout => "15s"
},
Conf =
#{
<<"bridges">> =>
#{
<<"http">> =>
#{
<<"basic_usage_info_http">> => HTTPConfig,
<<"basic_usage_info_http_disabled">> =>
HTTPConfig#{enable => false}
},
<<"mqtt">> =>
#{<<"basic_usage_info_mqtt">> => MQTTConfig}
}
},
ok = emqx_common_test_helpers:load_config(emqx_connector_schema, ConnectorConf), ok = emqx_common_test_helpers:load_config(emqx_connector_schema, ConnectorConf),
ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, Conf), ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, Conf),

271
apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
View File

@ -25,11 +25,15 @@
-define(CONF_DEFAULT, <<"bridges: {}">>). -define(CONF_DEFAULT, <<"bridges: {}">>).
-define(BRIDGE_TYPE, <<"http">>). -define(BRIDGE_TYPE, <<"http">>).
-define(BRIDGE_NAME, <<"test_bridge">>). -define(BRIDGE_NAME, <<"test_bridge">>).
-define(URL(PORT, PATH), list_to_binary( -define(URL(PORT, PATH),
io_lib:format("http://localhost:~s/~s", list_to_binary(
[integer_to_list(PORT), PATH]))). io_lib:format(
-define(HTTP_BRIDGE(URL, TYPE, NAME), "http://localhost:~s/~s",
#{ [integer_to_list(PORT), PATH]
)
)
).
-define(HTTP_BRIDGE(URL, TYPE, NAME), #{
<<"type">> => TYPE, <<"type">> => TYPE,
<<"name">> => NAME, <<"name">> => NAME,
<<"url">> => URL, <<"url">> => URL,
@ -40,7 +44,6 @@
<<"headers">> => #{ <<"headers">> => #{
<<"content-type">> => <<"application/json">> <<"content-type">> => <<"application/json">>
} }
}). }).
all() -> all() ->
@ -50,15 +53,17 @@ groups() ->
[]. [].
suite() -> suite() ->
[{timetrap,{seconds,60}}]. [{timetrap, {seconds, 60}}].
init_per_suite(Config) -> init_per_suite(Config) ->
_ = application:load(emqx_conf), _ = application:load(emqx_conf),
%% some testcases (may from other app) already get emqx_connector started %% some testcases (may from other app) already get emqx_connector started
_ = application:stop(emqx_resource), _ = application:stop(emqx_resource),
_ = application:stop(emqx_connector), _ = application:stop(emqx_connector),
ok = emqx_common_test_helpers:start_apps([emqx_bridge, emqx_dashboard], ok = emqx_common_test_helpers:start_apps(
fun set_special_configs/1), [emqx_bridge, emqx_dashboard],
fun set_special_configs/1
),
ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, ?CONF_DEFAULT), ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, ?CONF_DEFAULT),
Config. Config.
@ -79,9 +84,12 @@ end_per_testcase(_, _Config) ->
ok. ok.
clear_resources() -> clear_resources() ->
lists:foreach(fun(#{type := Type, name := Name}) -> lists:foreach(
fun(#{type := Type, name := Name}) ->
ok = emqx_bridge:remove(Type, Name) ok = emqx_bridge:remove(Type, Name)
end, emqx_bridge:list()). end,
emqx_bridge:list()
).
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% HTTP server for testing %% HTTP server for testing
@ -95,12 +103,12 @@ start_http_server(HandleFun) ->
end), end),
receive receive
{port, Port} -> Port {port, Port} -> Port
after after 2000 -> error({timeout, start_http_server})
2000 -> error({timeout, start_http_server})
end. end.
listen_on_random_port() -> listen_on_random_port() ->
Min = 1024, Max = 65000, Min = 1024,
Max = 65000,
Port = rand:uniform(Max - Min) + Min, Port = rand:uniform(Max - Min) + Min,
case gen_tcp:listen(Port, [{active, false}, {reuseaddr, true}, binary]) of case gen_tcp:listen(Port, [{active, false}, {reuseaddr, true}, binary]) of
{ok, Sock} -> {Port, Sock}; {ok, Sock} -> {Port, Sock};
@ -109,16 +117,18 @@ listen_on_random_port() ->
loop(Sock, HandleFun, Parent) -> loop(Sock, HandleFun, Parent) ->
{ok, Conn} = gen_tcp:accept(Sock), {ok, Conn} = gen_tcp:accept(Sock),
Handler = spawn(fun () -> HandleFun(Conn, Parent) end), Handler = spawn(fun() -> HandleFun(Conn, Parent) end),
gen_tcp:controlling_process(Conn, Handler), gen_tcp:controlling_process(Conn, Handler),
loop(Sock, HandleFun, Parent). loop(Sock, HandleFun, Parent).
make_response(CodeStr, Str) -> make_response(CodeStr, Str) ->
B = iolist_to_binary(Str), B = iolist_to_binary(Str),
iolist_to_binary( iolist_to_binary(
io_lib:fwrite( io_lib:fwrite(
"HTTP/1.0 ~s\r\nContent-Type: text/html\r\nContent-Length: ~p\r\n\r\n~s", "HTTP/1.0 ~s\r\nContent-Type: text/html\r\nContent-Length: ~p\r\n\r\n~s",
[CodeStr, size(B), B])). [CodeStr, size(B), B]
)
).
handle_fun_200_ok(Conn, Parent) -> handle_fun_200_ok(Conn, Parent) ->
case gen_tcp:recv(Conn, 0) of case gen_tcp:recv(Conn, 0) of
@ -151,18 +161,22 @@ t_http_crud_apis(_) ->
%% then we add a http bridge, using POST %% then we add a http bridge, using POST
%% POST /bridges/ will create a bridge %% POST /bridges/ will create a bridge
URL1 = ?URL(Port, "path1"), URL1 = ?URL(Port, "path1"),
{ok, 201, Bridge} = request(post, uri(["bridges"]), {ok, 201, Bridge} = request(
?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)), post,
uri(["bridges"]),
?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)
),
%ct:pal("---bridge: ~p", [Bridge]), %ct:pal("---bridge: ~p", [Bridge]),
#{ <<"type">> := ?BRIDGE_TYPE #{
, <<"name">> := ?BRIDGE_NAME <<"type">> := ?BRIDGE_TYPE,
, <<"status">> := _ <<"name">> := ?BRIDGE_NAME,
, <<"node_status">> := [_|_] <<"status">> := _,
, <<"metrics">> := _ <<"node_status">> := [_ | _],
, <<"node_metrics">> := [_|_] <<"metrics">> := _,
, <<"url">> := URL1 <<"node_metrics">> := [_ | _],
} = jsx:decode(Bridge), <<"url">> := URL1
} = jsx:decode(Bridge),
BridgeID = emqx_bridge:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), BridgeID = emqx_bridge:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME),
%% send an message to emqx and the message should be forwarded to the HTTP server %% send an message to emqx and the message should be forwarded to the HTTP server
@ -170,49 +184,70 @@ t_http_crud_apis(_) ->
emqx:publish(emqx_message:make(<<"emqx_http/1">>, Body)), emqx:publish(emqx_message:make(<<"emqx_http/1">>, Body)),
?assert( ?assert(
receive receive
{http_server, received, #{method := <<"POST">>, path := <<"/path1">>, {http_server, received, #{
body := Body}} -> method := <<"POST">>,
path := <<"/path1">>,
body := Body
}} ->
true; true;
Msg -> Msg ->
ct:pal("error: http got unexpected request: ~p", [Msg]), ct:pal("error: http got unexpected request: ~p", [Msg]),
false false
after 100 -> after 100 ->
false false
end), end
),
%% update the request-path of the bridge %% update the request-path of the bridge
URL2 = ?URL(Port, "path2"), URL2 = ?URL(Port, "path2"),
{ok, 200, Bridge2} = request(put, uri(["bridges", BridgeID]), {ok, 200, Bridge2} = request(
?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, ?BRIDGE_NAME)), put,
?assertMatch(#{ <<"type">> := ?BRIDGE_TYPE uri(["bridges", BridgeID]),
, <<"name">> := ?BRIDGE_NAME ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, ?BRIDGE_NAME)
, <<"status">> := _ ),
, <<"node_status">> := [_|_] ?assertMatch(
, <<"metrics">> := _ #{
, <<"node_metrics">> := [_|_] <<"type">> := ?BRIDGE_TYPE,
, <<"url">> := URL2 <<"name">> := ?BRIDGE_NAME,
}, jsx:decode(Bridge2)), <<"status">> := _,
<<"node_status">> := [_ | _],
<<"metrics">> := _,
<<"node_metrics">> := [_ | _],
<<"url">> := URL2
},
jsx:decode(Bridge2)
),
%% list all bridges again, assert Bridge2 is in it %% list all bridges again, assert Bridge2 is in it
{ok, 200, Bridge2Str} = request(get, uri(["bridges"]), []), {ok, 200, Bridge2Str} = request(get, uri(["bridges"]), []),
?assertMatch([#{ <<"type">> := ?BRIDGE_TYPE ?assertMatch(
, <<"name">> := ?BRIDGE_NAME [
, <<"status">> := _ #{
, <<"node_status">> := [_|_] <<"type">> := ?BRIDGE_TYPE,
, <<"metrics">> := _ <<"name">> := ?BRIDGE_NAME,
, <<"node_metrics">> := [_|_] <<"status">> := _,
, <<"url">> := URL2 <<"node_status">> := [_ | _],
}], jsx:decode(Bridge2Str)), <<"metrics">> := _,
<<"node_metrics">> := [_ | _],
<<"url">> := URL2
}
],
jsx:decode(Bridge2Str)
),
%% get the bridge by id %% get the bridge by id
{ok, 200, Bridge3Str} = request(get, uri(["bridges", BridgeID]), []), {ok, 200, Bridge3Str} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"type">> := ?BRIDGE_TYPE ?assertMatch(
, <<"name">> := ?BRIDGE_NAME #{
, <<"status">> := _ <<"type">> := ?BRIDGE_TYPE,
, <<"node_status">> := [_|_] <<"name">> := ?BRIDGE_NAME,
, <<"metrics">> := _ <<"status">> := _,
, <<"node_metrics">> := [_|_] <<"node_status">> := [_ | _],
, <<"url">> := URL2 <<"metrics">> := _,
}, jsx:decode(Bridge3Str)), <<"node_metrics">> := [_ | _],
<<"url">> := URL2
},
jsx:decode(Bridge3Str)
),
%% send an message to emqx again, check the path has been changed %% send an message to emqx again, check the path has been changed
emqx:publish(emqx_message:make(<<"emqx_http/1">>, Body)), emqx:publish(emqx_message:make(<<"emqx_http/1">>, Body)),
@ -225,25 +260,35 @@ t_http_crud_apis(_) ->
false false
after 100 -> after 100 ->
false false
end), end
),
%% delete the bridge %% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% update a deleted bridge returns an error %% update a deleted bridge returns an error
{ok, 404, ErrMsg2} = request(put, uri(["bridges", BridgeID]), {ok, 404, ErrMsg2} = request(
?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, ?BRIDGE_NAME)), put,
uri(["bridges", BridgeID]),
?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, ?BRIDGE_NAME)
),
?assertMatch( ?assertMatch(
#{ <<"code">> := _ #{
, <<"message">> := <<"bridge not found">> <<"code">> := _,
}, jsx:decode(ErrMsg2)), <<"message">> := <<"bridge not found">>
},
jsx:decode(ErrMsg2)
),
ok. ok.
t_start_stop_bridges(_) -> t_start_stop_bridges(_) ->
lists:foreach(fun(Type) -> lists:foreach(
fun(Type) ->
do_start_stop_bridges(Type) do_start_stop_bridges(Type)
end, [node, cluster]). end,
[node, cluster]
).
do_start_stop_bridges(Type) -> do_start_stop_bridges(Type) ->
%% assert we there's no bridges at first %% assert we there's no bridges at first
@ -251,40 +296,40 @@ do_start_stop_bridges(Type) ->
Port = start_http_server(fun handle_fun_200_ok/2), Port = start_http_server(fun handle_fun_200_ok/2),
URL1 = ?URL(Port, "abc"), URL1 = ?URL(Port, "abc"),
{ok, 201, Bridge} = request(post, uri(["bridges"]), {ok, 201, Bridge} = request(
?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)), post,
uri(["bridges"]),
?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)
),
%ct:pal("the bridge ==== ~p", [Bridge]), %ct:pal("the bridge ==== ~p", [Bridge]),
#{ <<"type">> := ?BRIDGE_TYPE #{
, <<"name">> := ?BRIDGE_NAME <<"type">> := ?BRIDGE_TYPE,
, <<"status">> := <<"connected">> <<"name">> := ?BRIDGE_NAME,
, <<"node_status">> := [_|_] <<"status">> := <<"connected">>,
, <<"metrics">> := _ <<"node_status">> := [_ | _],
, <<"node_metrics">> := [_|_] <<"metrics">> := _,
, <<"url">> := URL1 <<"node_metrics">> := [_ | _],
} = jsx:decode(Bridge), <<"url">> := URL1
} = jsx:decode(Bridge),
BridgeID = emqx_bridge:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), BridgeID = emqx_bridge:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME),
%% stop it %% stop it
{ok, 200, <<>>} = request(post, operation_path(Type, stop, BridgeID), <<"">>), {ok, 200, <<>>} = request(post, operation_path(Type, stop, BridgeID), <<"">>),
{ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []), {ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"status">> := <<"disconnected">> ?assertMatch(#{<<"status">> := <<"disconnected">>}, jsx:decode(Bridge2)),
}, jsx:decode(Bridge2)),
%% start again %% start again
{ok, 200, <<>>} = request(post, operation_path(Type, restart, BridgeID), <<"">>), {ok, 200, <<>>} = request(post, operation_path(Type, restart, BridgeID), <<"">>),
{ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []), {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"status">> := <<"connected">> ?assertMatch(#{<<"status">> := <<"connected">>}, jsx:decode(Bridge3)),
}, jsx:decode(Bridge3)),
%% restart an already started bridge %% restart an already started bridge
{ok, 200, <<>>} = request(post, operation_path(Type, restart, BridgeID), <<"">>), {ok, 200, <<>>} = request(post, operation_path(Type, restart, BridgeID), <<"">>),
{ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []), {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"status">> := <<"connected">> ?assertMatch(#{<<"status">> := <<"connected">>}, jsx:decode(Bridge3)),
}, jsx:decode(Bridge3)),
%% stop it again %% stop it again
{ok, 200, <<>>} = request(post, operation_path(Type, stop, BridgeID), <<"">>), {ok, 200, <<>>} = request(post, operation_path(Type, stop, BridgeID), <<"">>),
%% restart a stopped bridge %% restart a stopped bridge
{ok, 200, <<>>} = request(post, operation_path(Type, restart, BridgeID), <<"">>), {ok, 200, <<>>} = request(post, operation_path(Type, restart, BridgeID), <<"">>),
{ok, 200, Bridge4} = request(get, uri(["bridges", BridgeID]), []), {ok, 200, Bridge4} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"status">> := <<"connected">> ?assertMatch(#{<<"status">> := <<"connected">>}, jsx:decode(Bridge4)),
}, jsx:decode(Bridge4)),
%% delete the bridge %% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []). {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []).
@ -295,33 +340,34 @@ t_enable_disable_bridges(_) ->
Port = start_http_server(fun handle_fun_200_ok/2), Port = start_http_server(fun handle_fun_200_ok/2),
URL1 = ?URL(Port, "abc"), URL1 = ?URL(Port, "abc"),
{ok, 201, Bridge} = request(post, uri(["bridges"]), {ok, 201, Bridge} = request(
?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)), post,
uri(["bridges"]),
?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)
),
%ct:pal("the bridge ==== ~p", [Bridge]), %ct:pal("the bridge ==== ~p", [Bridge]),
#{ <<"type">> := ?BRIDGE_TYPE #{
, <<"name">> := ?BRIDGE_NAME <<"type">> := ?BRIDGE_TYPE,
, <<"status">> := <<"connected">> <<"name">> := ?BRIDGE_NAME,
, <<"node_status">> := [_|_] <<"status">> := <<"connected">>,
, <<"metrics">> := _ <<"node_status">> := [_ | _],
, <<"node_metrics">> := [_|_] <<"metrics">> := _,
, <<"url">> := URL1 <<"node_metrics">> := [_ | _],
} = jsx:decode(Bridge), <<"url">> := URL1
} = jsx:decode(Bridge),
BridgeID = emqx_bridge:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), BridgeID = emqx_bridge:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME),
%% disable it %% disable it
{ok, 200, <<>>} = request(post, operation_path(cluster, disable, BridgeID), <<"">>), {ok, 200, <<>>} = request(post, operation_path(cluster, disable, BridgeID), <<"">>),
{ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []), {ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"status">> := <<"disconnected">> ?assertMatch(#{<<"status">> := <<"disconnected">>}, jsx:decode(Bridge2)),
}, jsx:decode(Bridge2)),
%% enable again %% enable again
{ok, 200, <<>>} = request(post, operation_path(cluster, enable, BridgeID), <<"">>), {ok, 200, <<>>} = request(post, operation_path(cluster, enable, BridgeID), <<"">>),
{ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []), {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"status">> := <<"connected">> ?assertMatch(#{<<"status">> := <<"connected">>}, jsx:decode(Bridge3)),
}, jsx:decode(Bridge3)),
%% enable an already started bridge %% enable an already started bridge
{ok, 200, <<>>} = request(post, operation_path(cluster, enable, BridgeID), <<"">>), {ok, 200, <<>>} = request(post, operation_path(cluster, enable, BridgeID), <<"">>),
{ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []), {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"status">> := <<"connected">> ?assertMatch(#{<<"status">> := <<"connected">>}, jsx:decode(Bridge3)),
}, jsx:decode(Bridge3)),
%% disable it again %% disable it again
{ok, 200, <<>>} = request(post, operation_path(cluster, disable, BridgeID), <<"">>), {ok, 200, <<>>} = request(post, operation_path(cluster, disable, BridgeID), <<"">>),
@ -331,8 +377,7 @@ t_enable_disable_bridges(_) ->
%% enable a stopped bridge %% enable a stopped bridge
{ok, 200, <<>>} = request(post, operation_path(cluster, enable, BridgeID), <<"">>), {ok, 200, <<>>} = request(post, operation_path(cluster, enable, BridgeID), <<"">>),
{ok, 200, Bridge4} = request(get, uri(["bridges", BridgeID]), []), {ok, 200, Bridge4} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"status">> := <<"connected">> ?assertMatch(#{<<"status">> := <<"connected">>}, jsx:decode(Bridge4)),
}, jsx:decode(Bridge4)),
%% delete the bridge %% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []). {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []).
@ -343,17 +388,21 @@ t_reset_bridges(_) ->
Port = start_http_server(fun handle_fun_200_ok/2), Port = start_http_server(fun handle_fun_200_ok/2),
URL1 = ?URL(Port, "abc"), URL1 = ?URL(Port, "abc"),
{ok, 201, Bridge} = request(post, uri(["bridges"]), {ok, 201, Bridge} = request(
?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)), post,
uri(["bridges"]),
?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)
),
%ct:pal("the bridge ==== ~p", [Bridge]), %ct:pal("the bridge ==== ~p", [Bridge]),
#{ <<"type">> := ?BRIDGE_TYPE #{
, <<"name">> := ?BRIDGE_NAME <<"type">> := ?BRIDGE_TYPE,
, <<"status">> := <<"connected">> <<"name">> := ?BRIDGE_NAME,
, <<"node_status">> := [_|_] <<"status">> := <<"connected">>,
, <<"metrics">> := _ <<"node_status">> := [_ | _],
, <<"node_metrics">> := [_|_] <<"metrics">> := _,
, <<"url">> := URL1 <<"node_metrics">> := [_ | _],
} = jsx:decode(Bridge), <<"url">> := URL1
} = jsx:decode(Bridge),
BridgeID = emqx_bridge:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), BridgeID = emqx_bridge:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME),
{ok, 200, <<"Reset success">>} = request(put, uri(["bridges", BridgeID, "reset_metrics"]), []), {ok, 200, <<"Reset success">>} = request(put, uri(["bridges", BridgeID, "reset_metrics"]), []),

15
apps/emqx_connector/include/emqx_connector.hrl
View File

@ -24,13 +24,16 @@
-define(REDIS_DEFAULT_PORT, 6379). -define(REDIS_DEFAULT_PORT, 6379).
-define(PGSQL_DEFAULT_PORT, 5432). -define(PGSQL_DEFAULT_PORT, 5432).
-define(SERVERS_DESC, "A Node list for Cluster to connect to. The nodes should be separated with commas, such as: `Node[,Node].` -define(SERVERS_DESC,
For each Node should be: "). "A Node list for Cluster to connect to. The nodes should be separated with commas, such as: `Node[,Node].`\n"
"For each Node should be: "
).
-define(SERVER_DESC(TYPE, DEFAULT_PORT), " -define(SERVER_DESC(TYPE, DEFAULT_PORT),
The IPv4 or IPv6 address or the hostname to connect to.</br> "\n"
A host entry has the following form: `Host[:Port]`.</br> "The IPv4 or IPv6 address or the hostname to connect to.</br>\n"
The " ++ TYPE ++ " default port " ++ DEFAULT_PORT ++ " is used if `[:Port]` is not specified." "A host entry has the following form: `Host[:Port]`.</br>\n"
"The " ++ TYPE ++ " default port " ++ DEFAULT_PORT ++ " is used if `[:Port]` is not specified."
). ).
-define(THROW_ERROR(Str), erlang:throw({error, Str})). -define(THROW_ERROR(Str), erlang:throw({error, Str})).

40
apps/emqx_connector/rebar.config
View File

@ -1,30 +1,32 @@
%% -*- mode: erlang -*- %% -*- mode: erlang -*-
{erl_opts, [ {erl_opts, [
nowarn_unused_import, nowarn_unused_import,
debug_info debug_info
]}. ]}.
{deps, [ {deps, [
{emqx, {path, "../emqx"}}, {emqx, {path, "../emqx"}},
{eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}}, {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}},
{mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}}, {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}},
{epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7-emqx.2"}}}, {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7-emqx.2"}}},
%% NOTE: mind poolboy version when updating mongodb-erlang version %% NOTE: mind poolboy version when updating mongodb-erlang version
{mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.13"}}}, {mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.13"}}},
%% NOTE: mind poolboy version when updating eredis_cluster version %% NOTE: mind poolboy version when updating eredis_cluster version
{eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.1"}}}, {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.1"}}},
%% mongodb-erlang uses a special fork https://github.com/comtihon/poolboy.git %% mongodb-erlang uses a special fork https://github.com/comtihon/poolboy.git
%% (which has overflow_ttl feature added). %% (which has overflow_ttl feature added).
%% However, it references `{branch, "master}` (commit 9c06a9a on 2021-04-07). %% However, it references `{branch, "master}` (commit 9c06a9a on 2021-04-07).
%% By accident, We have always been using the upstream fork due to %% By accident, We have always been using the upstream fork due to
%% eredis_cluster's dependency getting resolved earlier. %% eredis_cluster's dependency getting resolved earlier.
%% Here we pin 1.5.2 to avoid surprises in the future. %% Here we pin 1.5.2 to avoid surprises in the future.
{poolboy, {git, "https://github.com/emqx/poolboy.git", {tag, "1.5.2"}}}, {poolboy, {git, "https://github.com/emqx/poolboy.git", {tag, "1.5.2"}}},
{emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.5.0"}}} {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.5.0"}}}
]}. ]}.
{shell, [ {shell, [
% {config, "config/sys.config"}, % {config, "config/sys.config"},
{apps, [emqx_connector]} {apps, [emqx_connector]}
]}. ]}.
{project_plugins, [erlfmt]}.

50
apps/emqx_connector/src/emqx_connector.app.src
View File

@ -1,27 +1,27 @@
%% -*- mode: erlang -*- %% -*- mode: erlang -*-
{application, emqx_connector, {application, emqx_connector, [
[{description, "An OTP application"}, {description, "An OTP application"},
{vsn, "0.1.1"}, {vsn, "0.1.1"},
{registered, []}, {registered, []},
{mod, {emqx_connector_app, []}}, {mod, {emqx_connector_app, []}},
{applications, {applications, [
[kernel, kernel,
stdlib, stdlib,
ecpool, ecpool,
emqx_resource, emqx_resource,
eredis_cluster, eredis_cluster,
eredis, eredis,
epgsql, epgsql,
eldap2, eldap2,
mysql, mysql,
mongodb, mongodb,
ehttpc, ehttpc,
emqx, emqx,
emqtt emqtt
]}, ]},
{env,[]}, {env, []},
{modules, []}, {modules, []},
{licenses, ["Apache 2.0"]}, {licenses, ["Apache 2.0"]},
{links, []} {links, []}
]}. ]}.

94
apps/emqx_connector/src/emqx_connector.erl
View File

@ -15,24 +15,27 @@
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-module(emqx_connector). -module(emqx_connector).
-export([ config_key_path/0 -export([
, pre_config_update/3 config_key_path/0,
, post_config_update/5 pre_config_update/3,
]). post_config_update/5
]).
-export([ parse_connector_id/1 -export([
, connector_id/2 parse_connector_id/1,
]). connector_id/2
]).
-export([ list_raw/0 -export([
, lookup_raw/1 list_raw/0,
, lookup_raw/2 lookup_raw/1,
, create_dry_run/2 lookup_raw/2,
, update/2 create_dry_run/2,
, update/3 update/2,
, delete/1 update/3,
, delete/2 delete/1,
]). delete/2
]).
config_key_path() -> config_key_path() ->
[connectors]. [connectors].
@ -53,19 +56,27 @@ post_config_update([connectors, Type, Name] = Path, '$remove', _, OldConf, _AppE
throw({dependency_bridges_exist, emqx_bridge:bridge_id(BType, BName)}) throw({dependency_bridges_exist, emqx_bridge:bridge_id(BType, BName)})
end), end),
_ = emqx_connector_ssl:clear_certs(filename:join(Path), OldConf) _ = emqx_connector_ssl:clear_certs(filename:join(Path), OldConf)
catch throw:Error -> {error, Error} catch
throw:Error -> {error, Error}
end; end;
post_config_update([connectors, Type, Name], _Req, NewConf, OldConf, _AppEnvs) -> post_config_update([connectors, Type, Name], _Req, NewConf, OldConf, _AppEnvs) ->
ConnId = connector_id(Type, Name), ConnId = connector_id(Type, Name),
foreach_linked_bridges(ConnId, foreach_linked_bridges(
ConnId,
fun(#{type := BType, name := BName}) -> fun(#{type := BType, name := BName}) ->
BridgeConf = emqx:get_config([bridges, BType, BName]), BridgeConf = emqx:get_config([bridges, BType, BName]),
case emqx_bridge:update(BType, BName, {BridgeConf#{connector => OldConf}, case
BridgeConf#{connector => NewConf}}) of emqx_bridge:update(
BType,
BName,
{BridgeConf#{connector => OldConf}, BridgeConf#{connector => NewConf}}
)
of
ok -> ok; ok -> ok;
{error, Reason} -> error({update_bridge_error, Reason}) {error, Reason} -> error({update_bridge_error, Reason})
end end
end). end
).
connector_id(Type0, Name0) -> connector_id(Type0, Name0) ->
Type = bin(Type0), Type = bin(Type0),
@ -80,13 +91,22 @@ parse_connector_id(ConnectorId) ->
list_raw() -> list_raw() ->
case get_raw_connector_conf() of case get_raw_connector_conf() of
not_found -> []; not_found ->
[];
Config -> Config ->
lists:foldl(fun({Type, NameAndConf}, Connectors) -> lists:foldl(
lists:foldl(fun({Name, RawConf}, Acc) -> fun({Type, NameAndConf}, Connectors) ->
[RawConf#{<<"type">> => Type, <<"name">> => Name} | Acc] lists:foldl(
end, Connectors, maps:to_list(NameAndConf)) fun({Name, RawConf}, Acc) ->
end, [], maps:to_list(Config)) [RawConf#{<<"type">> => Type, <<"name">> => Name} | Acc]
end,
Connectors,
maps:to_list(NameAndConf)
)
end,
[],
maps:to_list(Config)
)
end. end.
lookup_raw(Id) when is_binary(Id) -> lookup_raw(Id) when is_binary(Id) ->
@ -96,7 +116,8 @@ lookup_raw(Id) when is_binary(Id) ->
lookup_raw(Type, Name) -> lookup_raw(Type, Name) ->
Path = [bin(P) || P <- [Type, Name]], Path = [bin(P) || P <- [Type, Name]],
case get_raw_connector_conf() of case get_raw_connector_conf() of
not_found -> {error, not_found}; not_found ->
{error, not_found};
Conf -> Conf ->
case emqx_map_lib:deep_get(Path, Conf, not_found) of case emqx_map_lib:deep_get(Path, Conf, not_found) of
not_found -> {error, not_found}; not_found -> {error, not_found};
@ -123,7 +144,8 @@ delete(Type, Name) ->
get_raw_connector_conf() -> get_raw_connector_conf() ->
case emqx:get_raw_config(config_key_path(), not_found) of case emqx:get_raw_config(config_key_path(), not_found) of
not_found -> not_found; not_found ->
not_found;
RawConf -> RawConf ->
#{<<"connectors">> := Conf} = #{<<"connectors">> := Conf} =
emqx_config:fill_defaults(#{<<"connectors">> => RawConf}), emqx_config:fill_defaults(#{<<"connectors">> => RawConf}),
@ -135,8 +157,12 @@ bin(Str) when is_list(Str) -> list_to_binary(Str);
bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8).
foreach_linked_bridges(ConnId, Do) -> foreach_linked_bridges(ConnId, Do) ->
lists:foreach(fun lists:foreach(
(#{raw_config := #{<<"connector">> := ConnId0}} = Bridge) when ConnId0 == ConnId -> fun
Do(Bridge); (#{raw_config := #{<<"connector">> := ConnId0}} = Bridge) when ConnId0 == ConnId ->
(_) -> ok Do(Bridge);
end, emqx_bridge:list()). (_) ->
ok
end,
emqx_bridge:list()
).

124
apps/emqx_connector/src/emqx_connector_api.erl
View File

@ -40,9 +40,14 @@
EXPR EXPR
catch catch
error:{invalid_connector_id, Id0} -> error:{invalid_connector_id, Id0} ->
{400, #{code => 'INVALID_ID', message => <<"invalid_connector_id: ", Id0/binary, {400, #{
". Connector Ids must be of format {type}:{name}">>}} code => 'INVALID_ID',
end). message =>
<<"invalid_connector_id: ", Id0/binary,
". Connector Ids must be of format {type}:{name}">>
}}
end
).
namespace() -> "connector". namespace() -> "connector".
@ -58,21 +63,25 @@ error_schema(Codes, Message) when is_binary(Message) ->
put_request_body_schema() -> put_request_body_schema() ->
emqx_dashboard_swagger:schema_with_examples( emqx_dashboard_swagger:schema_with_examples(
emqx_connector_schema:put_request(), connector_info_examples(put)). emqx_connector_schema:put_request(), connector_info_examples(put)
).
post_request_body_schema() -> post_request_body_schema() ->
emqx_dashboard_swagger:schema_with_examples( emqx_dashboard_swagger:schema_with_examples(
emqx_connector_schema:post_request(), connector_info_examples(post)). emqx_connector_schema:post_request(), connector_info_examples(post)
).
get_response_body_schema() -> get_response_body_schema() ->
emqx_dashboard_swagger:schema_with_examples( emqx_dashboard_swagger:schema_with_examples(
emqx_connector_schema:get_response(), connector_info_examples(get)). emqx_connector_schema:get_response(), connector_info_examples(get)
).
connector_info_array_example(Method) -> connector_info_array_example(Method) ->
[Config || #{value := Config} <- maps:values(connector_info_examples(Method))]. [Config || #{value := Config} <- maps:values(connector_info_examples(Method))].
connector_info_examples(Method) -> connector_info_examples(Method) ->
lists:foldl(fun(Type, Acc) -> lists:foldl(
fun(Type, Acc) ->
SType = atom_to_list(Type), SType = atom_to_list(Type),
maps:merge(Acc, #{ maps:merge(Acc, #{
Type => #{ Type => #{
@ -80,11 +89,16 @@ connector_info_examples(Method) ->
value => info_example(Type, Method) value => info_example(Type, Method)
} }
}) })
end, #{}, ?CONN_TYPES). end,
#{},
?CONN_TYPES
).
info_example(Type, Method) -> info_example(Type, Method) ->
maps:merge(info_example_basic(Type), maps:merge(
method_example(Type, Method)). info_example_basic(Type),
method_example(Type, Method)
).
method_example(Type, Method) when Method == get; Method == post -> method_example(Type, Method) when Method == get; Method == post ->
SType = atom_to_list(Type), SType = atom_to_list(Type),
@ -115,11 +129,17 @@ info_example_basic(mqtt) ->
}. }.
param_path_id() -> param_path_id() ->
[{id, mk(binary(), [
#{ in => path {id,
, example => <<"mqtt:my_mqtt_connector">> mk(
, desc => ?DESC("id") binary(),
})}]. #{
in => path,
example => <<"mqtt:my_mqtt_connector">>,
desc => ?DESC("id")
}
)}
].
schema("/connectors_test") -> schema("/connectors_test") ->
#{ #{
@ -135,7 +155,6 @@ schema("/connectors_test") ->
} }
} }
}; };
schema("/connectors") -> schema("/connectors") ->
#{ #{
'operationId' => '/connectors', 'operationId' => '/connectors',
@ -145,8 +164,9 @@ schema("/connectors") ->
summary => <<"List connectors">>, summary => <<"List connectors">>,
responses => #{ responses => #{
200 => emqx_dashboard_swagger:schema_with_example( 200 => emqx_dashboard_swagger:schema_with_example(
array(emqx_connector_schema:get_response()), array(emqx_connector_schema:get_response()),
connector_info_array_example(get)) connector_info_array_example(get)
)
} }
}, },
post => #{ post => #{
@ -160,7 +180,6 @@ schema("/connectors") ->
} }
} }
}; };
schema("/connectors/:id") -> schema("/connectors/:id") ->
#{ #{
'operationId' => '/connectors/:id', 'operationId' => '/connectors/:id',
@ -185,7 +204,8 @@ schema("/connectors/:id") ->
200 => get_response_body_schema(), 200 => get_response_body_schema(),
404 => error_schema(['NOT_FOUND'], "Connector not found"), 404 => error_schema(['NOT_FOUND'], "Connector not found"),
400 => error_schema(['INVALID_ID'], "Bad connector ID") 400 => error_schema(['INVALID_ID'], "Bad connector ID")
}}, }
},
delete => #{ delete => #{
tags => [<<"connectors">>], tags => [<<"connectors">>],
desc => ?DESC("conn_id_delete"), desc => ?DESC("conn_id_delete"),
@ -196,7 +216,8 @@ schema("/connectors/:id") ->
403 => error_schema(['DEPENDENCY_EXISTS'], "Cannot remove dependent connector"), 403 => error_schema(['DEPENDENCY_EXISTS'], "Cannot remove dependent connector"),
404 => error_schema(['NOT_FOUND'], "Delete failed, not found"), 404 => error_schema(['NOT_FOUND'], "Delete failed, not found"),
400 => error_schema(['INVALID_ID'], "Bad connector ID") 400 => error_schema(['INVALID_ID'], "Bad connector ID")
}} }
}
}. }.
'/connectors_test'(post, #{body := #{<<"type">> := ConnType} = Params}) -> '/connectors_test'(post, #{body := #{<<"type">> := ConnType} = Params}) ->
@ -209,67 +230,83 @@ schema("/connectors/:id") ->
'/connectors'(get, _Request) -> '/connectors'(get, _Request) ->
{200, [format_resp(Conn) || Conn <- emqx_connector:list_raw()]}; {200, [format_resp(Conn) || Conn <- emqx_connector:list_raw()]};
'/connectors'(post, #{body := #{<<"type">> := ConnType, <<"name">> := ConnName} = Params}) -> '/connectors'(post, #{body := #{<<"type">> := ConnType, <<"name">> := ConnName} = Params}) ->
case emqx_connector:lookup_raw(ConnType, ConnName) of case emqx_connector:lookup_raw(ConnType, ConnName) of
{ok, _} -> {ok, _} ->
{400, error_msg('ALREADY_EXISTS', <<"connector already exists">>)}; {400, error_msg('ALREADY_EXISTS', <<"connector already exists">>)};
{error, not_found} -> {error, not_found} ->
case emqx_connector:update(ConnType, ConnName, case
filter_out_request_body(Params)) of emqx_connector:update(
ConnType,
ConnName,
filter_out_request_body(Params)
)
of
{ok, #{raw_config := RawConf}} -> {ok, #{raw_config := RawConf}} ->
{201, format_resp(RawConf#{<<"type">> => ConnType, {201,
<<"name">> => ConnName})}; format_resp(RawConf#{
<<"type">> => ConnType,
<<"name">> => ConnName
})};
{error, Error} -> {error, Error} ->
{400, error_msg('BAD_REQUEST', Error)} {400, error_msg('BAD_REQUEST', Error)}
end end
end; end;
'/connectors'(post, _) -> '/connectors'(post, _) ->
{400, error_msg('BAD_REQUEST', <<"missing some required fields: [name, type]">>)}. {400, error_msg('BAD_REQUEST', <<"missing some required fields: [name, type]">>)}.
'/connectors/:id'(get, #{bindings := #{id := Id}}) -> '/connectors/:id'(get, #{bindings := #{id := Id}}) ->
?TRY_PARSE_ID(Id, ?TRY_PARSE_ID(
Id,
case emqx_connector:lookup_raw(ConnType, ConnName) of case emqx_connector:lookup_raw(ConnType, ConnName) of
{ok, Conf} -> {ok, Conf} ->
{200, format_resp(Conf)}; {200, format_resp(Conf)};
{error, not_found} -> {error, not_found} ->
{404, error_msg('NOT_FOUND', <<"connector not found">>)} {404, error_msg('NOT_FOUND', <<"connector not found">>)}
end); end
);
'/connectors/:id'(put, #{bindings := #{id := Id}, body := Params0}) -> '/connectors/:id'(put, #{bindings := #{id := Id}, body := Params0}) ->
Params = filter_out_request_body(Params0), Params = filter_out_request_body(Params0),
?TRY_PARSE_ID(Id, ?TRY_PARSE_ID(
Id,
case emqx_connector:lookup_raw(ConnType, ConnName) of case emqx_connector:lookup_raw(ConnType, ConnName) of
{ok, _} -> {ok, _} ->
case emqx_connector:update(ConnType, ConnName, Params) of case emqx_connector:update(ConnType, ConnName, Params) of
{ok, #{raw_config := RawConf}} -> {ok, #{raw_config := RawConf}} ->
{200, format_resp(RawConf#{<<"type">> => ConnType, {200,
<<"name">> => ConnName})}; format_resp(RawConf#{
<<"type">> => ConnType,
<<"name">> => ConnName
})};
{error, Error} -> {error, Error} ->
{500, error_msg('INTERNAL_ERROR', Error)} {500, error_msg('INTERNAL_ERROR', Error)}
end; end;
{error, not_found} -> {error, not_found} ->
{404, error_msg('NOT_FOUND', <<"connector not found">>)} {404, error_msg('NOT_FOUND', <<"connector not found">>)}
end); end
);
'/connectors/:id'(delete, #{bindings := #{id := Id}}) -> '/connectors/:id'(delete, #{bindings := #{id := Id}}) ->
?TRY_PARSE_ID(Id, ?TRY_PARSE_ID(
Id,
case emqx_connector:lookup_raw(ConnType, ConnName) of case emqx_connector:lookup_raw(ConnType, ConnName) of
{ok, _} -> {ok, _} ->
case emqx_connector:delete(ConnType, ConnName) of case emqx_connector:delete(ConnType, ConnName) of
{ok, _} -> {ok, _} ->
{204}; {204};
{error, {post_config_update, _, {dependency_bridges_exist, BridgeID}}} -> {error, {post_config_update, _, {dependency_bridges_exist, BridgeID}}} ->
{403, error_msg('DEPENDENCY_EXISTS', {403,
<<"Cannot remove the connector as it's in use by a bridge: ", error_msg(
BridgeID/binary>>)}; 'DEPENDENCY_EXISTS',
<<"Cannot remove the connector as it's in use by a bridge: ",
BridgeID/binary>>
)};
{error, Error} -> {error, Error} ->
{500, error_msg('INTERNAL_ERROR', Error)} {500, error_msg('INTERNAL_ERROR', Error)}
end; end;
{error, not_found} -> {error, not_found} ->
{404, error_msg('NOT_FOUND', <<"connector not found">>)} {404, error_msg('NOT_FOUND', <<"connector not found">>)}
end). end
).
error_msg(Code, Msg) when is_binary(Msg) -> error_msg(Code, Msg) when is_binary(Msg) ->
#{code => Code, message => Msg}; #{code => Code, message => Msg};
@ -277,8 +314,11 @@ error_msg(Code, Msg) ->
#{code => Code, message => bin(io_lib:format("~p", [Msg]))}. #{code => Code, message => bin(io_lib:format("~p", [Msg]))}.
format_resp(#{<<"type">> := ConnType, <<"name">> := ConnName} = RawConf) -> format_resp(#{<<"type">> := ConnType, <<"name">> := ConnName} = RawConf) ->
NumOfBridges = length(emqx_bridge:list_bridges_by_connector( NumOfBridges = length(
emqx_connector:connector_id(ConnType, ConnName))), emqx_bridge:list_bridges_by_connector(
emqx_connector:connector_id(ConnType, ConnName)
)
),
RawConf#{ RawConf#{
<<"type">> => ConnType, <<"type">> => ConnType,
<<"name">> => ConnName, <<"name">> => ConnName,

423
apps/emqx_connector/src/emqx_connector_http.erl
View File

@ -25,32 +25,34 @@
-behaviour(emqx_resource). -behaviour(emqx_resource).
%% callbacks of behaviour emqx_resource %% callbacks of behaviour emqx_resource
-export([ on_start/2 -export([
, on_stop/2 on_start/2,
, on_query/4 on_stop/2,
, on_health_check/2 on_query/4,
]). on_health_check/2
]).
-type url() :: emqx_http_lib:uri_map(). -type url() :: emqx_http_lib:uri_map().
-reflect_type([url/0]). -reflect_type([url/0]).
-typerefl_from_string({url/0, emqx_http_lib, uri_parse}). -typerefl_from_string({url/0, emqx_http_lib, uri_parse}).
-export([ roots/0 -export([
, fields/1 roots/0,
, desc/1 fields/1,
, validations/0 desc/1,
, namespace/0 validations/0,
]). namespace/0
]).
-export([ check_ssl_opts/2 -export([check_ssl_opts/2]).
]).
-type connect_timeout() :: emqx_schema:duration() | infinity. -type connect_timeout() :: emqx_schema:duration() | infinity.
-type pool_type() :: random | hash. -type pool_type() :: random | hash.
-reflect_type([ connect_timeout/0 -reflect_type([
, pool_type/0 connect_timeout/0,
]). pool_type/0
]).
%%===================================================================== %%=====================================================================
%% Hocon schema %% Hocon schema
@ -61,63 +63,96 @@ roots() ->
fields(config). fields(config).
fields(config) -> fields(config) ->
[ {base_url, [
sc(url(), {base_url,
#{ required => true sc(
, validator => fun(#{query := _Query}) -> url(),
#{
required => true,
validator => fun
(#{query := _Query}) ->
{error, "There must be no query in the base_url"}; {error, "There must be no query in the base_url"};
(_) -> ok (_) ->
end ok
, desc => ?DESC("base_url") end,
})} desc => ?DESC("base_url")
, {connect_timeout, }
sc(emqx_schema:duration_ms(), )},
#{ default => "15s" {connect_timeout,
, desc => ?DESC("connect_timeout") sc(
})} emqx_schema:duration_ms(),
, {max_retries, #{
sc(non_neg_integer(), default => "15s",
#{ default => 5 desc => ?DESC("connect_timeout")
, desc => ?DESC("max_retries") }
})} )},
, {retry_interval, {max_retries,
sc(emqx_schema:duration(), sc(
#{ default => "1s" non_neg_integer(),
, desc => ?DESC("retry_interval") #{
})} default => 5,
, {pool_type, desc => ?DESC("max_retries")
sc(pool_type(), }
#{ default => random )},
, desc => ?DESC("pool_type") {retry_interval,
})} sc(
, {pool_size, emqx_schema:duration(),
sc(pos_integer(), #{
#{ default => 8 default => "1s",
, desc => ?DESC("pool_size") desc => ?DESC("retry_interval")
})} }
, {enable_pipelining, )},
sc(boolean(), {pool_type,
#{ default => true sc(
, desc => ?DESC("enable_pipelining") pool_type(),
})} #{
, {request, hoconsc:mk( default => random,
ref("request"), desc => ?DESC("pool_type")
#{ default => undefined }
, required => false )},
, desc => ?DESC("request") {pool_size,
})} sc(
pos_integer(),
#{
default => 8,
desc => ?DESC("pool_size")
}
)},
{enable_pipelining,
sc(
boolean(),
#{
default => true,
desc => ?DESC("enable_pipelining")
}
)},
{request,
hoconsc:mk(
ref("request"),
#{
default => undefined,
required => false,
desc => ?DESC("request")
}
)}
] ++ emqx_connector_schema_lib:ssl_fields(); ] ++ emqx_connector_schema_lib:ssl_fields();
fields("request") -> fields("request") ->
[ {method, hoconsc:mk(hoconsc:enum([post, put, get, delete]), #{required => false, desc => ?DESC("method")})} [
, {path, hoconsc:mk(binary(), #{required => false, desc => ?DESC("path")})} {method,
, {body, hoconsc:mk(binary(), #{required => false, desc => ?DESC("body")})} hoconsc:mk(hoconsc:enum([post, put, get, delete]), #{
, {headers, hoconsc:mk(map(), #{required => false, desc => ?DESC("headers")})} required => false, desc => ?DESC("method")
, {request_timeout, })},
sc(emqx_schema:duration_ms(), {path, hoconsc:mk(binary(), #{required => false, desc => ?DESC("path")})},
#{ required => false {body, hoconsc:mk(binary(), #{required => false, desc => ?DESC("body")})},
, desc => ?DESC("request_timeout") {headers, hoconsc:mk(map(), #{required => false, desc => ?DESC("headers")})},
})} {request_timeout,
sc(
emqx_schema:duration_ms(),
#{
required => false,
desc => ?DESC("request_timeout")
}
)}
]. ].
desc(config) -> desc(config) ->
@ -128,24 +163,34 @@ desc(_) ->
undefined. undefined.
validations() -> validations() ->
[ {check_ssl_opts, fun check_ssl_opts/1} ]. [{check_ssl_opts, fun check_ssl_opts/1}].
sc(Type, Meta) -> hoconsc:mk(Type, Meta). sc(Type, Meta) -> hoconsc:mk(Type, Meta).
ref(Field) -> hoconsc:ref(?MODULE, Field). ref(Field) -> hoconsc:ref(?MODULE, Field).
%% =================================================================== %% ===================================================================
on_start(InstId, #{base_url := #{scheme := Scheme, on_start(
host := Host, InstId,
port := Port, #{
path := BasePath}, base_url := #{
connect_timeout := ConnectTimeout, scheme := Scheme,
max_retries := MaxRetries, host := Host,
retry_interval := RetryInterval, port := Port,
pool_type := PoolType, path := BasePath
pool_size := PoolSize} = Config) -> },
?SLOG(info, #{msg => "starting_http_connector", connect_timeout := ConnectTimeout,
connector => InstId, config => Config}), max_retries := MaxRetries,
retry_interval := RetryInterval,
pool_type := PoolType,
pool_size := PoolSize
} = Config
) ->
?SLOG(info, #{
msg => "starting_http_connector",
connector => InstId,
config => Config
}),
{Transport, TransportOpts} = {Transport, TransportOpts} =
case Scheme of case Scheme of
http -> http ->
@ -155,16 +200,18 @@ on_start(InstId, #{base_url := #{scheme := Scheme,
{tls, SSLOpts} {tls, SSLOpts}
end, end,
NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), NTransportOpts = emqx_misc:ipv6_probe(TransportOpts),
PoolOpts = [ {host, Host} PoolOpts = [
, {port, Port} {host, Host},
, {connect_timeout, ConnectTimeout} {port, Port},
, {retry, MaxRetries} {connect_timeout, ConnectTimeout},
, {retry_timeout, RetryInterval} {retry, MaxRetries},
, {keepalive, 30000} {retry_timeout, RetryInterval},
, {pool_type, PoolType} {keepalive, 30000},
, {pool_size, PoolSize} {pool_type, PoolType},
, {transport, Transport} {pool_size, PoolSize},
, {transport_opts, NTransportOpts}], {transport, Transport},
{transport_opts, NTransportOpts}
],
PoolName = emqx_plugin_libs_pool:pool_name(InstId), PoolName = emqx_plugin_libs_pool:pool_name(InstId),
State = #{ State = #{
pool_name => PoolName, pool_name => PoolName,
@ -177,54 +224,84 @@ on_start(InstId, #{base_url := #{scheme := Scheme,
case ehttpc_sup:start_pool(PoolName, PoolOpts) of case ehttpc_sup:start_pool(PoolName, PoolOpts) of
{ok, _} -> {ok, State}; {ok, _} -> {ok, State};
{error, {already_started, _}} -> {ok, State}; {error, {already_started, _}} -> {ok, State};
{error, Reason} -> {error, Reason} -> {error, Reason}
{error, Reason}
end. end.
on_stop(InstId, #{pool_name := PoolName}) -> on_stop(InstId, #{pool_name := PoolName}) ->
?SLOG(info, #{msg => "stopping_http_connector", ?SLOG(info, #{
connector => InstId}), msg => "stopping_http_connector",
connector => InstId
}),
ehttpc_sup:stop_pool(PoolName). ehttpc_sup:stop_pool(PoolName).
on_query(InstId, {send_message, Msg}, AfterQuery, State) -> on_query(InstId, {send_message, Msg}, AfterQuery, State) ->
case maps:get(request, State, undefined) of case maps:get(request, State, undefined) of
undefined -> ?SLOG(error, #{msg => "request_not_found", connector => InstId}); undefined ->
?SLOG(error, #{msg => "request_not_found", connector => InstId});
Request -> Request ->
#{method := Method, path := Path, body := Body, headers := Headers, #{
request_timeout := Timeout} = process_request(Request, Msg), method := Method,
path := Path,
body := Body,
headers := Headers,
request_timeout := Timeout
} = process_request(Request, Msg),
on_query(InstId, {Method, {Path, Headers, Body}, Timeout}, AfterQuery, State) on_query(InstId, {Method, {Path, Headers, Body}, Timeout}, AfterQuery, State)
end; end;
on_query(InstId, {Method, Request}, AfterQuery, State) -> on_query(InstId, {Method, Request}, AfterQuery, State) ->
on_query(InstId, {undefined, Method, Request, 5000}, AfterQuery, State); on_query(InstId, {undefined, Method, Request, 5000}, AfterQuery, State);
on_query(InstId, {Method, Request, Timeout}, AfterQuery, State) -> on_query(InstId, {Method, Request, Timeout}, AfterQuery, State) ->
on_query(InstId, {undefined, Method, Request, Timeout}, AfterQuery, State); on_query(InstId, {undefined, Method, Request, Timeout}, AfterQuery, State);
on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery, on_query(
#{pool_name := PoolName, base_path := BasePath} = State) -> InstId,
?TRACE("QUERY", "http_connector_received", {KeyOrNum, Method, Request, Timeout},
#{request => Request, connector => InstId, state => State}), AfterQuery,
#{pool_name := PoolName, base_path := BasePath} = State
) ->
?TRACE(
"QUERY",
"http_connector_received",
#{request => Request, connector => InstId, state => State}
),
NRequest = formalize_request(Method, BasePath, Request), NRequest = formalize_request(Method, BasePath, Request),
case Result = ehttpc:request(case KeyOrNum of case
undefined -> PoolName; Result = ehttpc:request(
_ -> {PoolName, KeyOrNum} case KeyOrNum of
end, Method, NRequest, Timeout) of undefined -> PoolName;
_ -> {PoolName, KeyOrNum}
end,
Method,
NRequest,
Timeout
)
of
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => "http_connector_do_reqeust_failed", ?SLOG(error, #{
request => NRequest, reason => Reason, msg => "http_connector_do_reqeust_failed",
connector => InstId}), request => NRequest,
reason => Reason,
connector => InstId
}),
emqx_resource:query_failed(AfterQuery); emqx_resource:query_failed(AfterQuery);
{ok, StatusCode, _} when StatusCode >= 200 andalso StatusCode < 300 -> {ok, StatusCode, _} when StatusCode >= 200 andalso StatusCode < 300 ->
emqx_resource:query_success(AfterQuery); emqx_resource:query_success(AfterQuery);
{ok, StatusCode, _, _} when StatusCode >= 200 andalso StatusCode < 300 -> {ok, StatusCode, _, _} when StatusCode >= 200 andalso StatusCode < 300 ->
emqx_resource:query_success(AfterQuery); emqx_resource:query_success(AfterQuery);
{ok, StatusCode, _} -> {ok, StatusCode, _} ->
?SLOG(error, #{msg => "http connector do request, received error response", ?SLOG(error, #{
request => NRequest, connector => InstId, msg => "http connector do request, received error response",
status_code => StatusCode}), request => NRequest,
connector => InstId,
status_code => StatusCode
}),
emqx_resource:query_failed(AfterQuery); emqx_resource:query_failed(AfterQuery);
{ok, StatusCode, _, _} -> {ok, StatusCode, _, _} ->
?SLOG(error, #{msg => "http connector do request, received error response", ?SLOG(error, #{
request => NRequest, connector => InstId, msg => "http connector do request, received error response",
status_code => StatusCode}), request => NRequest,
connector => InstId,
status_code => StatusCode
}),
emqx_resource:query_failed(AfterQuery) emqx_resource:query_failed(AfterQuery)
end, end,
Result. Result.
@ -232,14 +309,16 @@ on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery,
on_health_check(_InstId, #{host := Host, port := Port, connect_timeout := Timeout} = State) -> on_health_check(_InstId, #{host := Host, port := Port, connect_timeout := Timeout} = State) ->
case do_health_check(Host, Port, Timeout) of case do_health_check(Host, Port, Timeout) of
ok -> {ok, State}; ok -> {ok, State};
{error, Reason} -> {error, Reason} -> {error, {http_health_check_failed, Reason}, State}
{error, {http_health_check_failed, Reason}, State}
end. end.
do_health_check(Host, Port, Timeout) -> do_health_check(Host, Port, Timeout) ->
case gen_tcp:connect(Host, Port, emqx_misc:ipv6_probe([]), Timeout) of case gen_tcp:connect(Host, Port, emqx_misc:ipv6_probe([]), Timeout) of
{ok, Sock} -> gen_tcp:close(Sock), ok; {ok, Sock} ->
{error, Reason} -> {error, Reason} gen_tcp:close(Sock),
ok;
{error, Reason} ->
{error, Reason}
end. end.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
@ -250,47 +329,64 @@ preprocess_request(undefined) ->
undefined; undefined;
preprocess_request(Req) when map_size(Req) == 0 -> preprocess_request(Req) when map_size(Req) == 0 ->
undefined; undefined;
preprocess_request(#{ preprocess_request(
method := Method, #{
path := Path, method := Method,
body := Body, path := Path,
headers := Headers body := Body,
} = Req) -> headers := Headers
#{ method => emqx_plugin_libs_rule:preproc_tmpl(bin(Method)) } = Req
, path => emqx_plugin_libs_rule:preproc_tmpl(Path) ) ->
, body => emqx_plugin_libs_rule:preproc_tmpl(Body) #{
, headers => preproc_headers(Headers) method => emqx_plugin_libs_rule:preproc_tmpl(bin(Method)),
, request_timeout => maps:get(request_timeout, Req, 30000) path => emqx_plugin_libs_rule:preproc_tmpl(Path),
}. body => emqx_plugin_libs_rule:preproc_tmpl(Body),
headers => preproc_headers(Headers),
request_timeout => maps:get(request_timeout, Req, 30000)
}.
preproc_headers(Headers) when is_map(Headers) -> preproc_headers(Headers) when is_map(Headers) ->
maps:fold(fun(K, V, Acc) -> maps:fold(
[{ fun(K, V, Acc) ->
[
{
emqx_plugin_libs_rule:preproc_tmpl(bin(K)),
emqx_plugin_libs_rule:preproc_tmpl(bin(V))
}
| Acc
]
end,
[],
Headers
);
preproc_headers(Headers) when is_list(Headers) ->
lists:map(
fun({K, V}) ->
{
emqx_plugin_libs_rule:preproc_tmpl(bin(K)), emqx_plugin_libs_rule:preproc_tmpl(bin(K)),
emqx_plugin_libs_rule:preproc_tmpl(bin(V)) emqx_plugin_libs_rule:preproc_tmpl(bin(V))
} | Acc] }
end, [], Headers); end,
preproc_headers(Headers) when is_list(Headers) -> Headers
lists:map(fun({K, V}) -> ).
{
emqx_plugin_libs_rule:preproc_tmpl(bin(K)),
emqx_plugin_libs_rule:preproc_tmpl(bin(V))
}
end, Headers).
process_request(#{ process_request(
method := MethodTks, #{
path := PathTks, method := MethodTks,
body := BodyTks, path := PathTks,
headers := HeadersTks, body := BodyTks,
request_timeout := ReqTimeout headers := HeadersTks,
} = Conf, Msg) -> request_timeout := ReqTimeout
Conf#{ method => make_method(emqx_plugin_libs_rule:proc_tmpl(MethodTks, Msg)) } = Conf,
, path => emqx_plugin_libs_rule:proc_tmpl(PathTks, Msg) Msg
, body => process_request_body(BodyTks, Msg) ) ->
, headers => proc_headers(HeadersTks, Msg) Conf#{
, request_timeout => ReqTimeout method => make_method(emqx_plugin_libs_rule:proc_tmpl(MethodTks, Msg)),
}. path => emqx_plugin_libs_rule:proc_tmpl(PathTks, Msg),
body => process_request_body(BodyTks, Msg),
headers => proc_headers(HeadersTks, Msg),
request_timeout => ReqTimeout
}.
process_request_body([], Msg) -> process_request_body([], Msg) ->
emqx_json:encode(Msg); emqx_json:encode(Msg);
@ -298,12 +394,15 @@ process_request_body(BodyTks, Msg) ->
emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg). emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg).
proc_headers(HeaderTks, Msg) -> proc_headers(HeaderTks, Msg) ->
lists:map(fun({K, V}) -> lists:map(
fun({K, V}) ->
{ {
emqx_plugin_libs_rule:proc_tmpl(K, Msg), emqx_plugin_libs_rule:proc_tmpl(K, Msg),
emqx_plugin_libs_rule:proc_tmpl(V, Msg) emqx_plugin_libs_rule:proc_tmpl(V, Msg)
} }
end, HeaderTks). end,
HeaderTks
).
make_method(M) when M == <<"POST">>; M == <<"post">> -> post; make_method(M) when M == <<"POST">>; M == <<"post">> -> post;
make_method(M) when M == <<"PUT">>; M == <<"put">> -> put; make_method(M) when M == <<"PUT">>; M == <<"put">> -> put;
@ -315,19 +414,19 @@ check_ssl_opts(Conf) ->
check_ssl_opts(URLFrom, Conf) -> check_ssl_opts(URLFrom, Conf) ->
#{scheme := Scheme} = hocon_maps:get(URLFrom, Conf), #{scheme := Scheme} = hocon_maps:get(URLFrom, Conf),
SSL= hocon_maps:get("ssl", Conf), SSL = hocon_maps:get("ssl", Conf),
case {Scheme, maps:get(enable, SSL, false)} of case {Scheme, maps:get(enable, SSL, false)} of
{http, false} -> true; {http, false} -> true;
{https, true} -> true; {https, true} -> true;
{_, _} -> false {_, _} -> false
end. end.
formalize_request(Method, BasePath, {Path, Headers, _Body}) formalize_request(Method, BasePath, {Path, Headers, _Body}) when
when Method =:= get; Method =:= delete -> Method =:= get; Method =:= delete
->
formalize_request(Method, BasePath, {Path, Headers}); formalize_request(Method, BasePath, {Path, Headers});
formalize_request(_Method, BasePath, {Path, Headers, Body}) -> formalize_request(_Method, BasePath, {Path, Headers, Body}) ->
{filename:join(BasePath, Path), Headers, Body}; {filename:join(BasePath, Path), Headers, Body};
formalize_request(_Method, BasePath, {Path, Headers}) -> formalize_request(_Method, BasePath, {Path, Headers}) ->
{filename:join(BasePath, Path), Headers}. {filename:join(BasePath, Path), Headers}.

188
apps/emqx_connector/src/emqx_connector_ldap.erl
View File

@ -24,11 +24,12 @@
-behaviour(emqx_resource). -behaviour(emqx_resource).
%% callbacks of behaviour emqx_resource %% callbacks of behaviour emqx_resource
-export([ on_start/2 -export([
, on_stop/2 on_start/2,
, on_query/4 on_stop/2,
, on_health_check/2 on_query/4,
]). on_health_check/2
]).
-export([do_health_check/1]). -export([do_health_check/1]).
@ -43,54 +44,84 @@ roots() ->
fields(_) -> []. fields(_) -> [].
%% =================================================================== %% ===================================================================
on_start(InstId, #{servers := Servers0, on_start(
port := Port, InstId,
bind_dn := BindDn, #{
bind_password := BindPassword, servers := Servers0,
timeout := Timeout, port := Port,
pool_size := PoolSize, bind_dn := BindDn,
auto_reconnect := AutoReconn, bind_password := BindPassword,
ssl := SSL} = Config) -> timeout := Timeout,
?SLOG(info, #{msg => "starting_ldap_connector", pool_size := PoolSize,
connector => InstId, config => Config}), auto_reconnect := AutoReconn,
Servers = [begin proplists:get_value(host, S) end || S <- Servers0], ssl := SSL
SslOpts = case maps:get(enable, SSL) of } = Config
true -> ) ->
[{ssl, true}, ?SLOG(info, #{
{sslopts, emqx_tls_lib:to_client_opts(SSL)} msg => "starting_ldap_connector",
]; connector => InstId,
false -> [{ssl, false}] config => Config
end, }),
Opts = [{servers, Servers}, Servers = [
{port, Port}, begin
{bind_dn, BindDn}, proplists:get_value(host, S)
{bind_password, BindPassword}, end
{timeout, Timeout}, || S <- Servers0
{pool_size, PoolSize}, ],
{auto_reconnect, reconn_interval(AutoReconn)}, SslOpts =
{servers, Servers}], case maps:get(enable, SSL) of
true ->
[
{ssl, true},
{sslopts, emqx_tls_lib:to_client_opts(SSL)}
];
false ->
[{ssl, false}]
end,
Opts = [
{servers, Servers},
{port, Port},
{bind_dn, BindDn},
{bind_password, BindPassword},
{timeout, Timeout},
{pool_size, PoolSize},
{auto_reconnect, reconn_interval(AutoReconn)},
{servers, Servers}
],
PoolName = emqx_plugin_libs_pool:pool_name(InstId), PoolName = emqx_plugin_libs_pool:pool_name(InstId),
case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts ++ SslOpts) of case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts ++ SslOpts) of
ok -> {ok, #{poolname => PoolName}}; ok -> {ok, #{poolname => PoolName}};
{error, Reason} -> {error, Reason} {error, Reason} -> {error, Reason}
end. end.
on_stop(InstId, #{poolname := PoolName}) -> on_stop(InstId, #{poolname := PoolName}) ->
?SLOG(info, #{msg => "stopping_ldap_connector", ?SLOG(info, #{
connector => InstId}), msg => "stopping_ldap_connector",
connector => InstId
}),
emqx_plugin_libs_pool:stop_pool(PoolName). emqx_plugin_libs_pool:stop_pool(PoolName).
on_query(InstId, {search, Base, Filter, Attributes}, AfterQuery, #{poolname := PoolName} = State) -> on_query(InstId, {search, Base, Filter, Attributes}, AfterQuery, #{poolname := PoolName} = State) ->
Request = {Base, Filter, Attributes}, Request = {Base, Filter, Attributes},
?TRACE("QUERY", "ldap_connector_received", ?TRACE(
#{request => Request, connector => InstId, state => State}), "QUERY",
case Result = ecpool:pick_and_do( "ldap_connector_received",
PoolName, #{request => Request, connector => InstId, state => State}
{?MODULE, search, [Base, Filter, Attributes]}, ),
no_handover) of case
Result = ecpool:pick_and_do(
PoolName,
{?MODULE, search, [Base, Filter, Attributes]},
no_handover
)
of
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => "ldap_connector_do_request_failed", ?SLOG(error, #{
request => Request, connector => InstId, reason => Reason}), msg => "ldap_connector_do_request_failed",
request => Request,
connector => InstId,
reason => Reason
}),
emqx_resource:query_failed(AfterQuery); emqx_resource:query_failed(AfterQuery);
_ -> _ ->
emqx_resource:query_success(AfterQuery) emqx_resource:query_success(AfterQuery)
@ -107,38 +138,45 @@ reconn_interval(true) -> 15;
reconn_interval(false) -> false. reconn_interval(false) -> false.
search(Conn, Base, Filter, Attributes) -> search(Conn, Base, Filter, Attributes) ->
eldap2:search(Conn, [{base, Base}, eldap2:search(Conn, [
{filter, Filter}, {base, Base},
{attributes, Attributes}, {filter, Filter},
{deref, eldap2:'derefFindingBaseObj'()}]). {attributes, Attributes},
{deref, eldap2:'derefFindingBaseObj'()}
]).
%% =================================================================== %% ===================================================================
connect(Opts) -> connect(Opts) ->
Servers = proplists:get_value(servers, Opts, ["localhost"]), Servers = proplists:get_value(servers, Opts, ["localhost"]),
Port = proplists:get_value(port, Opts, 389), Port = proplists:get_value(port, Opts, 389),
Timeout = proplists:get_value(timeout, Opts, 30), Timeout = proplists:get_value(timeout, Opts, 30),
BindDn = proplists:get_value(bind_dn, Opts), BindDn = proplists:get_value(bind_dn, Opts),
BindPassword = proplists:get_value(bind_password, Opts), BindPassword = proplists:get_value(bind_password, Opts),
SslOpts = case proplists:get_value(ssl, Opts, false) of SslOpts =
true -> case proplists:get_value(ssl, Opts, false) of
[{sslopts, proplists:get_value(sslopts, Opts, [])}, {ssl, true}]; true ->
false -> [{sslopts, proplists:get_value(sslopts, Opts, [])}, {ssl, true}];
[{ssl, false}] false ->
end, [{ssl, false}]
LdapOpts = [{port, Port}, end,
{timeout, Timeout}] ++ SslOpts, LdapOpts =
[
{port, Port},
{timeout, Timeout}
] ++ SslOpts,
{ok, LDAP} = eldap2:open(Servers, LdapOpts), {ok, LDAP} = eldap2:open(Servers, LdapOpts),
ok = eldap2:simple_bind(LDAP, BindDn, BindPassword), ok = eldap2:simple_bind(LDAP, BindDn, BindPassword),
{ok, LDAP}. {ok, LDAP}.
ldap_fields() -> ldap_fields() ->
[ {servers, fun servers/1} [
, {port, fun port/1} {servers, fun servers/1},
, {pool_size, fun emqx_connector_schema_lib:pool_size/1} {port, fun port/1},
, {bind_dn, fun bind_dn/1} {pool_size, fun emqx_connector_schema_lib:pool_size/1},
, {bind_password, fun emqx_connector_schema_lib:password/1} {bind_dn, fun bind_dn/1},
, {timeout, fun duration/1} {bind_password, fun emqx_connector_schema_lib:password/1},
, {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1} {timeout, fun duration/1},
{auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1}
]. ].
servers(type) -> list(); servers(type) -> list();
@ -159,14 +197,18 @@ duration(type) -> emqx_schema:duration_ms();
duration(_) -> undefined. duration(_) -> undefined.
to_servers_raw(Servers) -> to_servers_raw(Servers) ->
{ok, lists:map( fun(Server) -> {ok,
case string:tokens(Server, ": ") of lists:map(
[Ip] -> fun(Server) ->
[{host, Ip}]; case string:tokens(Server, ": ") of
[Ip, Port] -> [Ip] ->
[{host, Ip}, {port, list_to_integer(Port)}] [{host, Ip}];
end [Ip, Port] ->
end, string:tokens(str(Servers), ", "))}. [{host, Ip}, {port, list_to_integer(Port)}]
end
end,
string:tokens(str(Servers), ", ")
)}.
str(A) when is_atom(A) -> str(A) when is_atom(A) ->
atom_to_list(A); atom_to_list(A);

389
apps/emqx_connector/src/emqx_connector_mongo.erl
View File

@ -24,11 +24,12 @@
-behaviour(emqx_resource). -behaviour(emqx_resource).
%% callbacks of behaviour emqx_resource %% callbacks of behaviour emqx_resource
-export([ on_start/2 -export([
, on_stop/2 on_start/2,
, on_query/4 on_stop/2,
, on_health_check/2 on_query/4,
]). on_health_check/2
]).
%% ecpool callback %% ecpool callback
-export([connect/1]). -export([connect/1]).
@ -40,57 +41,73 @@
-define(HEALTH_CHECK_TIMEOUT, 10000). -define(HEALTH_CHECK_TIMEOUT, 10000).
%% mongo servers don't need parse %% mongo servers don't need parse
-define( MONGO_HOST_OPTIONS -define(MONGO_HOST_OPTIONS, #{
, #{ host_type => hostname host_type => hostname,
, default_port => ?MONGO_DEFAULT_PORT}). default_port => ?MONGO_DEFAULT_PORT
}).
%%===================================================================== %%=====================================================================
roots() -> roots() ->
[ {config, #{type => hoconsc:union( [
[ hoconsc:ref(?MODULE, single) {config, #{
, hoconsc:ref(?MODULE, rs) type => hoconsc:union(
, hoconsc:ref(?MODULE, sharded) [
])}} hoconsc:ref(?MODULE, single),
hoconsc:ref(?MODULE, rs),
hoconsc:ref(?MODULE, sharded)
]
)
}}
]. ].
fields(single) -> fields(single) ->
[ {mongo_type, #{type => single, [
default => single, {mongo_type, #{
required => true, type => single,
desc => ?DESC("single_mongo_type")}} default => single,
, {server, fun server/1} required => true,
, {w_mode, fun w_mode/1} desc => ?DESC("single_mongo_type")
}},
{server, fun server/1},
{w_mode, fun w_mode/1}
] ++ mongo_fields(); ] ++ mongo_fields();
fields(rs) -> fields(rs) ->
[ {mongo_type, #{type => rs, [
default => rs, {mongo_type, #{
required => true, type => rs,
desc => ?DESC("rs_mongo_type")}} default => rs,
, {servers, fun servers/1} required => true,
, {w_mode, fun w_mode/1} desc => ?DESC("rs_mongo_type")
, {r_mode, fun r_mode/1} }},
, {replica_set_name, fun replica_set_name/1} {servers, fun servers/1},
{w_mode, fun w_mode/1},
{r_mode, fun r_mode/1},
{replica_set_name, fun replica_set_name/1}
] ++ mongo_fields(); ] ++ mongo_fields();
fields(sharded) -> fields(sharded) ->
[ {mongo_type, #{type => sharded, [
default => sharded, {mongo_type, #{
required => true, type => sharded,
desc => ?DESC("sharded_mongo_type")}} default => sharded,
, {servers, fun servers/1} required => true,
, {w_mode, fun w_mode/1} desc => ?DESC("sharded_mongo_type")
}},
{servers, fun servers/1},
{w_mode, fun w_mode/1}
] ++ mongo_fields(); ] ++ mongo_fields();
fields(topology) -> fields(topology) ->
[ {pool_size, fun emqx_connector_schema_lib:pool_size/1} [
, {max_overflow, fun max_overflow/1} {pool_size, fun emqx_connector_schema_lib:pool_size/1},
, {overflow_ttl, fun duration/1} {max_overflow, fun max_overflow/1},
, {overflow_check_period, fun duration/1} {overflow_ttl, fun duration/1},
, {local_threshold_ms, fun duration/1} {overflow_check_period, fun duration/1},
, {connect_timeout_ms, fun duration/1} {local_threshold_ms, fun duration/1},
, {socket_timeout_ms, fun duration/1} {connect_timeout_ms, fun duration/1},
, {server_selection_timeout_ms, fun duration/1} {socket_timeout_ms, fun duration/1},
, {wait_queue_timeout_ms, fun duration/1} {server_selection_timeout_ms, fun duration/1},
, {heartbeat_frequency_ms, fun duration/1} {wait_queue_timeout_ms, fun duration/1},
, {min_heartbeat_frequency_ms, fun duration/1} {heartbeat_frequency_ms, fun duration/1},
{min_heartbeat_frequency_ms, fun duration/1}
]. ].
desc(single) -> desc(single) ->
@ -105,69 +122,96 @@ desc(_) ->
undefined. undefined.
mongo_fields() -> mongo_fields() ->
[ {srv_record, fun srv_record/1} [
, {pool_size, fun emqx_connector_schema_lib:pool_size/1} {srv_record, fun srv_record/1},
, {username, fun emqx_connector_schema_lib:username/1} {pool_size, fun emqx_connector_schema_lib:pool_size/1},
, {password, fun emqx_connector_schema_lib:password/1} {username, fun emqx_connector_schema_lib:username/1},
, {auth_source, #{ type => binary() {password, fun emqx_connector_schema_lib:password/1},
, required => false {auth_source, #{
, desc => ?DESC("auth_source") type => binary(),
}} required => false,
, {database, fun emqx_connector_schema_lib:database/1} desc => ?DESC("auth_source")
, {topology, #{type => hoconsc:ref(?MODULE, topology), required => false}} }},
{database, fun emqx_connector_schema_lib:database/1},
{topology, #{type => hoconsc:ref(?MODULE, topology), required => false}}
] ++ ] ++
emqx_connector_schema_lib:ssl_fields(). emqx_connector_schema_lib:ssl_fields().
%% =================================================================== %% ===================================================================
on_start(InstId, Config = #{mongo_type := Type, on_start(
pool_size := PoolSize, InstId,
ssl := SSL}) -> Config = #{
Msg = case Type of mongo_type := Type,
single -> "starting_mongodb_single_connector"; pool_size := PoolSize,
rs -> "starting_mongodb_replica_set_connector"; ssl := SSL
sharded -> "starting_mongodb_sharded_connector" }
end, ) ->
Msg =
case Type of
single -> "starting_mongodb_single_connector";
rs -> "starting_mongodb_replica_set_connector";
sharded -> "starting_mongodb_sharded_connector"
end,
?SLOG(info, #{msg => Msg, connector => InstId, config => Config}), ?SLOG(info, #{msg => Msg, connector => InstId, config => Config}),
NConfig = #{hosts := Hosts} = may_parse_srv_and_txt_records(Config), NConfig = #{hosts := Hosts} = may_parse_srv_and_txt_records(Config),
SslOpts = case maps:get(enable, SSL) of SslOpts =
true -> case maps:get(enable, SSL) of
[{ssl, true}, true ->
{ssl_opts, emqx_tls_lib:to_client_opts(SSL)} [
]; {ssl, true},
false -> [{ssl, false}] {ssl_opts, emqx_tls_lib:to_client_opts(SSL)}
end, ];
false ->
[{ssl, false}]
end,
Topology = maps:get(topology, NConfig, #{}), Topology = maps:get(topology, NConfig, #{}),
Opts = [{mongo_type, init_type(NConfig)}, Opts = [
{hosts, Hosts}, {mongo_type, init_type(NConfig)},
{pool_size, PoolSize}, {hosts, Hosts},
{options, init_topology_options(maps:to_list(Topology), [])}, {pool_size, PoolSize},
{worker_options, init_worker_options(maps:to_list(NConfig), SslOpts)}], {options, init_topology_options(maps:to_list(Topology), [])},
{worker_options, init_worker_options(maps:to_list(NConfig), SslOpts)}
],
PoolName = emqx_plugin_libs_pool:pool_name(InstId), PoolName = emqx_plugin_libs_pool:pool_name(InstId),
case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts) of case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts) of
ok -> {ok, #{poolname => PoolName, type => Type}}; ok -> {ok, #{poolname => PoolName, type => Type}};
{error, Reason} -> {error, Reason} {error, Reason} -> {error, Reason}
end. end.
on_stop(InstId, #{poolname := PoolName}) -> on_stop(InstId, #{poolname := PoolName}) ->
?SLOG(info, #{msg => "stopping_mongodb_connector", ?SLOG(info, #{
connector => InstId}), msg => "stopping_mongodb_connector",
connector => InstId
}),
emqx_plugin_libs_pool:stop_pool(PoolName). emqx_plugin_libs_pool:stop_pool(PoolName).
on_query(InstId, on_query(
{Action, Collection, Selector, Projector}, InstId,
AfterQuery, {Action, Collection, Selector, Projector},
#{poolname := PoolName} = State) -> AfterQuery,
#{poolname := PoolName} = State
) ->
Request = {Action, Collection, Selector, Projector}, Request = {Action, Collection, Selector, Projector},
?TRACE("QUERY", "mongodb_connector_received", ?TRACE(
#{request => Request, connector => InstId, state => State}), "QUERY",
case ecpool:pick_and_do(PoolName, "mongodb_connector_received",
{?MODULE, mongo_query, [Action, Collection, Selector, Projector]}, #{request => Request, connector => InstId, state => State}
no_handover) of ),
case
ecpool:pick_and_do(
PoolName,
{?MODULE, mongo_query, [Action, Collection, Selector, Projector]},
no_handover
)
of
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => "mongodb_connector_do_query_failed", ?SLOG(error, #{
request => Request, reason => Reason, msg => "mongodb_connector_do_query_failed",
connector => InstId}), request => Request,
reason => Reason,
connector => InstId
}),
emqx_resource:query_failed(AfterQuery), emqx_resource:query_failed(AfterQuery),
{error, Reason}; {error, Reason};
{ok, Cursor} when is_pid(Cursor) -> {ok, Cursor} when is_pid(Cursor) ->
@ -182,12 +226,16 @@ on_query(InstId,
on_health_check(InstId, #{poolname := PoolName} = State) -> on_health_check(InstId, #{poolname := PoolName} = State) ->
case health_check(PoolName) of case health_check(PoolName) of
true -> true ->
?tp(debug, emqx_connector_mongo_health_check, #{instance_id => InstId, ?tp(debug, emqx_connector_mongo_health_check, #{
status => ok}), instance_id => InstId,
status => ok
}),
{ok, State}; {ok, State};
false -> false ->
?tp(warning, emqx_connector_mongo_health_check, #{instance_id => InstId, ?tp(warning, emqx_connector_mongo_health_check, #{
status => failed}), instance_id => InstId,
status => failed
}),
{error, health_check_failed, State} {error, health_check_failed, State}
end. end.
@ -204,36 +252,43 @@ check_worker_health(Worker) ->
%% we don't care if this returns something or not, we just to test the connection %% we don't care if this returns something or not, we just to test the connection
try do_test_query(Conn) of try do_test_query(Conn) of
{error, Reason} -> {error, Reason} ->
?SLOG(warning, #{msg => "mongo_connection_health_check_error", ?SLOG(warning, #{
worker => Worker, msg => "mongo_connection_health_check_error",
reason => Reason}), worker => Worker,
reason => Reason
}),
false; false;
_ -> _ ->
true true
catch catch
Class:Error -> Class:Error ->
?SLOG(warning, #{msg => "mongo_connection_health_check_exception", ?SLOG(warning, #{
worker => Worker, msg => "mongo_connection_health_check_exception",
class => Class, worker => Worker,
error => Error}), class => Class,
error => Error
}),
false false
end; end;
_ -> _ ->
?SLOG(warning, #{msg => "mongo_connection_health_check_error", ?SLOG(warning, #{
worker => Worker, msg => "mongo_connection_health_check_error",
reason => worker_not_found}), worker => Worker,
reason => worker_not_found
}),
false false
end. end.
do_test_query(Conn) -> do_test_query(Conn) ->
mongoc:transaction_query( mongoc:transaction_query(
Conn, Conn,
fun(Conf = #{pool := Worker}) -> fun(Conf = #{pool := Worker}) ->
Query = mongoc:find_one_query(Conf, <<"foo">>, #{}, #{}, 0), Query = mongoc:find_one_query(Conf, <<"foo">>, #{}, #{}, 0),
mc_worker_api:find_one(Worker, Query) mc_worker_api:find_one(Worker, Query)
end, end,
#{}, #{},
?HEALTH_CHECK_TIMEOUT). ?HEALTH_CHECK_TIMEOUT
).
connect(Opts) -> connect(Opts) ->
Type = proplists:get_value(mongo_type, Opts, single), Type = proplists:get_value(mongo_type, Opts, single),
@ -244,10 +299,8 @@ connect(Opts) ->
mongo_query(Conn, find, Collection, Selector, Projector) -> mongo_query(Conn, find, Collection, Selector, Projector) ->
mongo_api:find(Conn, Collection, Selector, Projector); mongo_api:find(Conn, Collection, Selector, Projector);
mongo_query(Conn, find_one, Collection, Selector, Projector) -> mongo_query(Conn, find_one, Collection, Selector, Projector) ->
mongo_api:find_one(Conn, Collection, Selector, Projector); mongo_api:find_one(Conn, Collection, Selector, Projector);
%% Todo xxx %% Todo xxx
mongo_query(_Conn, _Action, _Collection, _Selector, _Projector) -> mongo_query(_Conn, _Action, _Collection, _Selector, _Projector) ->
ok. ok.
@ -298,7 +351,8 @@ init_worker_options([{r_mode, V} | R], Acc) ->
init_worker_options(R, [{r_mode, V} | Acc]); init_worker_options(R, [{r_mode, V} | Acc]);
init_worker_options([_ | R], Acc) -> init_worker_options([_ | R], Acc) ->
init_worker_options(R, Acc); init_worker_options(R, Acc);
init_worker_options([], Acc) -> Acc. init_worker_options([], Acc) ->
Acc.
%% =================================================================== %% ===================================================================
%% Schema funcs %% Schema funcs
@ -356,59 +410,76 @@ may_parse_srv_and_txt_records(#{server := Server} = Config) ->
may_parse_srv_and_txt_records(Config) -> may_parse_srv_and_txt_records(Config) ->
may_parse_srv_and_txt_records_(Config). may_parse_srv_and_txt_records_(Config).
may_parse_srv_and_txt_records_(#{mongo_type := Type, may_parse_srv_and_txt_records_(
srv_record := false, #{
servers := Servers} = Config) -> mongo_type := Type,
srv_record := false,
servers := Servers
} = Config
) ->
case Type =:= rs andalso maps:is_key(replica_set_name, Config) =:= false of case Type =:= rs andalso maps:is_key(replica_set_name, Config) =:= false of
true -> true ->
error({missing_parameter, replica_set_name}); error({missing_parameter, replica_set_name});
false -> false ->
Config#{hosts => servers_to_bin(Servers)} Config#{hosts => servers_to_bin(Servers)}
end; end;
may_parse_srv_and_txt_records_(#{mongo_type := Type, may_parse_srv_and_txt_records_(
srv_record := true, #{
servers := Servers} = Config) -> mongo_type := Type,
srv_record := true,
servers := Servers
} = Config
) ->
Hosts = parse_srv_records(Type, Servers), Hosts = parse_srv_records(Type, Servers),
ExtraOpts = parse_txt_records(Type, Servers), ExtraOpts = parse_txt_records(Type, Servers),
maps:merge(Config#{hosts => Hosts}, ExtraOpts). maps:merge(Config#{hosts => Hosts}, ExtraOpts).
parse_srv_records(Type, Servers) -> parse_srv_records(Type, Servers) ->
Fun = fun(AccIn, {IpOrHost, _Port}) -> Fun = fun(AccIn, {IpOrHost, _Port}) ->
case inet_res:lookup("_mongodb._tcp." case
++ ip_or_host_to_string(IpOrHost), in, srv) of inet_res:lookup(
[] -> "_mongodb._tcp." ++
error(service_not_found); ip_or_host_to_string(IpOrHost),
Services -> in,
[ [server_to_bin({Host, Port}) || {_, _, Port, Host} <- Services] srv
| AccIn] )
end of
end, [] ->
error(service_not_found);
Services ->
[
[server_to_bin({Host, Port}) || {_, _, Port, Host} <- Services]
| AccIn
]
end
end,
Res = lists:foldl(Fun, [], Servers), Res = lists:foldl(Fun, [], Servers),
case Type of case Type of
single -> lists:nth(1, Res); single -> lists:nth(1, Res);
_ -> Res _ -> Res
end. end.
parse_txt_records(Type, Servers) -> parse_txt_records(Type, Servers) ->
Fields = case Type of Fields =
rs -> ["authSource", "replicaSet"]; case Type of
_ -> ["authSource"] rs -> ["authSource", "replicaSet"];
end, _ -> ["authSource"]
end,
Fun = fun(AccIn, {IpOrHost, _Port}) -> Fun = fun(AccIn, {IpOrHost, _Port}) ->
case inet_res:lookup(IpOrHost, in, txt) of case inet_res:lookup(IpOrHost, in, txt) of
[] -> [] ->
#{}; #{};
[[QueryString]] -> [[QueryString]] ->
case uri_string:dissect_query(QueryString) of case uri_string:dissect_query(QueryString) of
{error, _, _} -> {error, _, _} ->
error({invalid_txt_record, invalid_query_string}); error({invalid_txt_record, invalid_query_string});
Options -> Options ->
maps:merge(AccIn, take_and_convert(Fields, Options)) maps:merge(AccIn, take_and_convert(Fields, Options))
end; end;
_ -> _ ->
error({invalid_txt_record, multiple_records}) error({invalid_txt_record, multiple_records})
end end
end, end,
lists:foldl(Fun, #{}, Servers). lists:foldl(Fun, #{}, Servers).
take_and_convert(Fields, Options) -> take_and_convert(Fields, Options) ->
@ -430,8 +501,8 @@ take_and_convert([Field | More], Options, Acc) ->
take_and_convert(More, Options, Acc) take_and_convert(More, Options, Acc)
end. end.
-spec ip_or_host_to_string(binary() | string() | tuple()) -spec ip_or_host_to_string(binary() | string() | tuple()) ->
-> string(). string().
ip_or_host_to_string(Ip) when is_tuple(Ip) -> ip_or_host_to_string(Ip) when is_tuple(Ip) ->
inet:ntoa(Ip); inet:ntoa(Ip);
ip_or_host_to_string(Host) -> ip_or_host_to_string(Host) ->
@ -448,18 +519,20 @@ server_to_bin({IpOrHost, Port}) ->
%% =================================================================== %% ===================================================================
%% typereflt funcs %% typereflt funcs
-spec to_server_raw(string()) -spec to_server_raw(string()) ->
-> {string(), pos_integer()}. {string(), pos_integer()}.
to_server_raw(Server) -> to_server_raw(Server) ->
emqx_connector_schema_lib:parse_server(Server, ?MONGO_HOST_OPTIONS). emqx_connector_schema_lib:parse_server(Server, ?MONGO_HOST_OPTIONS).
-spec to_servers_raw(string()) -spec to_servers_raw(string()) ->
-> [{string(), pos_integer()}]. [{string(), pos_integer()}].
to_servers_raw(Servers) -> to_servers_raw(Servers) ->
lists:map( fun(Server) -> lists:map(
emqx_connector_schema_lib:parse_server(Server, ?MONGO_HOST_OPTIONS) fun(Server) ->
end emqx_connector_schema_lib:parse_server(Server, ?MONGO_HOST_OPTIONS)
, string:tokens(str(Servers), ", ")). end,
string:tokens(str(Servers), ", ")
).
str(A) when is_atom(A) -> str(A) when is_atom(A) ->
atom_to_list(A); atom_to_list(A);

153
apps/emqx_connector/src/emqx_connector_mqtt.erl
View File

@ -23,28 +23,32 @@
-behaviour(emqx_resource). -behaviour(emqx_resource).
%% API and callbacks for supervisor %% API and callbacks for supervisor
-export([ start_link/0 -export([
, init/1 start_link/0,
, create_bridge/1 init/1,
, drop_bridge/1 create_bridge/1,
, bridges/0 drop_bridge/1,
]). bridges/0
]).
-export([on_message_received/3]). -export([on_message_received/3]).
%% callbacks of behaviour emqx_resource %% callbacks of behaviour emqx_resource
-export([ on_start/2 -export([
, on_stop/2 on_start/2,
, on_query/4 on_stop/2,
, on_health_check/2 on_query/4,
]). on_health_check/2
]).
-behaviour(hocon_schema). -behaviour(hocon_schema).
-import(hoconsc, [mk/2]). -import(hoconsc, [mk/2]).
-export([ roots/0 -export([
, fields/1]). roots/0,
fields/1
]).
%%===================================================================== %%=====================================================================
%% Hocon schema %% Hocon schema
@ -53,25 +57,34 @@ roots() ->
fields("config") -> fields("config") ->
emqx_connector_mqtt_schema:fields("config"); emqx_connector_mqtt_schema:fields("config");
fields("get") -> fields("get") ->
[ {num_of_bridges, mk(integer(), [
#{ desc => ?DESC("num_of_bridges") {num_of_bridges,
})} mk(
integer(),
#{desc => ?DESC("num_of_bridges")}
)}
] ++ fields("post"); ] ++ fields("post");
fields("put") -> fields("put") ->
emqx_connector_mqtt_schema:fields("connector"); emqx_connector_mqtt_schema:fields("connector");
fields("post") -> fields("post") ->
[ {type, mk(mqtt, [
#{ required => true {type,
, desc => ?DESC("type") mk(
})} mqtt,
, {name, mk(binary(), #{
#{ required => true required => true,
, desc => ?DESC("name") desc => ?DESC("type")
})} }
)},
{name,
mk(
binary(),
#{
required => true,
desc => ?DESC("name")
}
)}
] ++ fields("put"). ] ++ fields("put").
%% =================================================================== %% ===================================================================
@ -80,23 +93,29 @@ start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []). supervisor:start_link({local, ?MODULE}, ?MODULE, []).
init([]) -> init([]) ->
SupFlag = #{strategy => one_for_one, SupFlag = #{
intensity => 100, strategy => one_for_one,
period => 10}, intensity => 100,
period => 10
},
{ok, {SupFlag, []}}. {ok, {SupFlag, []}}.
bridge_spec(Config) -> bridge_spec(Config) ->
#{id => maps:get(name, Config), #{
start => {emqx_connector_mqtt_worker, start_link, [Config]}, id => maps:get(name, Config),
restart => permanent, start => {emqx_connector_mqtt_worker, start_link, [Config]},
shutdown => 5000, restart => permanent,
type => worker, shutdown => 5000,
modules => [emqx_connector_mqtt_worker]}. type => worker,
modules => [emqx_connector_mqtt_worker]
}.
-spec(bridges() -> [{node(), map()}]). -spec bridges() -> [{node(), map()}].
bridges() -> bridges() ->
[{Name, emqx_connector_mqtt_worker:status(Name)} [
|| {Name, _Pid, _, _} <- supervisor:which_children(?MODULE)]. {Name, emqx_connector_mqtt_worker:status(Name)}
|| {Name, _Pid, _, _} <- supervisor:which_children(?MODULE)
].
create_bridge(Config) -> create_bridge(Config) ->
supervisor:start_child(?MODULE, bridge_spec(Config)). supervisor:start_child(?MODULE, bridge_spec(Config)).
@ -121,8 +140,11 @@ on_message_received(Msg, HookPoint, InstId) ->
%% =================================================================== %% ===================================================================
on_start(InstId, Conf) -> on_start(InstId, Conf) ->
InstanceId = binary_to_atom(InstId, utf8), InstanceId = binary_to_atom(InstId, utf8),
?SLOG(info, #{msg => "starting_mqtt_connector", ?SLOG(info, #{
connector => InstanceId, config => Conf}), msg => "starting_mqtt_connector",
connector => InstanceId,
config => Conf
}),
BasicConf = basic_config(Conf), BasicConf = basic_config(Conf),
BridgeConf = BasicConf#{ BridgeConf = BasicConf#{
name => InstanceId, name => InstanceId,
@ -142,19 +164,25 @@ on_start(InstId, Conf) ->
end. end.
on_stop(_InstId, #{name := InstanceId}) -> on_stop(_InstId, #{name := InstanceId}) ->
?SLOG(info, #{msg => "stopping_mqtt_connector", ?SLOG(info, #{
connector => InstanceId}), msg => "stopping_mqtt_connector",
connector => InstanceId
}),
case ?MODULE:drop_bridge(InstanceId) of case ?MODULE:drop_bridge(InstanceId) of
ok -> ok; ok ->
{error, not_found} -> ok; ok;
{error, not_found} ->
ok;
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => "stop_mqtt_connector", ?SLOG(error, #{
connector => InstanceId, reason => Reason}) msg => "stop_mqtt_connector",
connector => InstanceId,
reason => Reason
})
end. end.
on_query(_InstId, {message_received, _Msg}, AfterQuery, _State) -> on_query(_InstId, {message_received, _Msg}, AfterQuery, _State) ->
emqx_resource:query_success(AfterQuery); emqx_resource:query_success(AfterQuery);
on_query(_InstId, {send_message, Msg}, AfterQuery, #{name := InstanceId}) -> on_query(_InstId, {send_message, Msg}, AfterQuery, #{name := InstanceId}) ->
?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => InstanceId}), ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => InstanceId}),
emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg), emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg),
@ -178,7 +206,8 @@ make_sub_confs(undefined, _) ->
undefined; undefined;
make_sub_confs(SubRemoteConf, InstId) -> make_sub_confs(SubRemoteConf, InstId) ->
case maps:take(hookpoint, SubRemoteConf) of case maps:take(hookpoint, SubRemoteConf) of
error -> SubRemoteConf; error ->
SubRemoteConf;
{HookPoint, SubConf} -> {HookPoint, SubConf} ->
MFA = {?MODULE, on_message_received, [HookPoint, InstId]}, MFA = {?MODULE, on_message_received, [HookPoint, InstId]},
SubConf#{on_message_received => MFA} SubConf#{on_message_received => MFA}
@ -192,22 +221,24 @@ make_forward_confs(FrowardConf) ->
FrowardConf. FrowardConf.
basic_config(#{ basic_config(#{
server := Server, server := Server,
reconnect_interval := ReconnIntv, reconnect_interval := ReconnIntv,
proto_ver := ProtoVer, proto_ver := ProtoVer,
username := User, username := User,
password := Password, password := Password,
clean_start := CleanStart, clean_start := CleanStart,
keepalive := KeepAlive, keepalive := KeepAlive,
retry_interval := RetryIntv, retry_interval := RetryIntv,
max_inflight := MaxInflight, max_inflight := MaxInflight,
replayq := ReplayQ, replayq := ReplayQ,
ssl := #{enable := EnableSsl} = Ssl}) -> ssl := #{enable := EnableSsl} = Ssl
}) ->
#{ #{
replayq => ReplayQ, replayq => ReplayQ,
%% connection opts %% connection opts
server => Server, server => Server,
connect_timeout => 30, %% 30s %% 30s
connect_timeout => 30,
reconnect_interval => ReconnIntv, reconnect_interval => ReconnIntv,
proto_ver => ProtoVer, proto_ver => ProtoVer,
bridge_mode => true, bridge_mode => true,

149
apps/emqx_connector/src/emqx_connector_mysql.erl
View File

@ -23,11 +23,12 @@
-behaviour(emqx_resource). -behaviour(emqx_resource).
%% callbacks of behaviour emqx_resource %% callbacks of behaviour emqx_resource
-export([ on_start/2 -export([
, on_stop/2 on_start/2,
, on_query/4 on_stop/2,
, on_health_check/2 on_query/4,
]). on_health_check/2
]).
%% ecpool connect & reconnect %% ecpool connect & reconnect
-export([connect/1, prepare_sql_to_conn/2]). -export([connect/1, prepare_sql_to_conn/2]).
@ -38,9 +39,10 @@
-export([do_health_check/1]). -export([do_health_check/1]).
-define( MYSQL_HOST_OPTIONS -define(MYSQL_HOST_OPTIONS, #{
, #{ host_type => inet_addr host_type => inet_addr,
, default_port => ?MYSQL_DEFAULT_PORT}). default_port => ?MYSQL_DEFAULT_PORT
}).
%%===================================================================== %%=====================================================================
%% Hocon schema %% Hocon schema
@ -48,11 +50,10 @@ roots() ->
[{config, #{type => hoconsc:ref(?MODULE, config)}}]. [{config, #{type => hoconsc:ref(?MODULE, config)}}].
fields(config) -> fields(config) ->
[ {server, fun server/1} [{server, fun server/1}] ++
] ++ emqx_connector_schema_lib:relational_db_fields() ++
emqx_connector_schema_lib:relational_db_fields() ++ emqx_connector_schema_lib:ssl_fields() ++
emqx_connector_schema_lib:ssl_fields() ++ emqx_connector_schema_lib:prepare_statement_fields().
emqx_connector_schema_lib:prepare_statement_fields().
server(type) -> emqx_schema:ip_port(); server(type) -> emqx_schema:ip_port();
server(required) -> true; server(required) -> true;
@ -62,47 +63,64 @@ server(desc) -> ?DESC("server");
server(_) -> undefined. server(_) -> undefined.
%% =================================================================== %% ===================================================================
on_start(InstId, #{server := {Host, Port}, on_start(
database := DB, InstId,
username := User, #{
password := Password, server := {Host, Port},
auto_reconnect := AutoReconn, database := DB,
pool_size := PoolSize, username := User,
ssl := SSL } = Config) -> password := Password,
?SLOG(info, #{msg => "starting_mysql_connector", auto_reconnect := AutoReconn,
connector => InstId, config => Config}), pool_size := PoolSize,
SslOpts = case maps:get(enable, SSL) of ssl := SSL
true -> } = Config
[{ssl, emqx_tls_lib:to_client_opts(SSL)}]; ) ->
false -> ?SLOG(info, #{
[] msg => "starting_mysql_connector",
end, connector => InstId,
Options = [{host, Host}, config => Config
{port, Port}, }),
{user, User}, SslOpts =
{password, Password}, case maps:get(enable, SSL) of
{database, DB}, true ->
{auto_reconnect, reconn_interval(AutoReconn)}, [{ssl, emqx_tls_lib:to_client_opts(SSL)}];
{pool_size, PoolSize}], false ->
[]
end,
Options = [
{host, Host},
{port, Port},
{user, User},
{password, Password},
{database, DB},
{auto_reconnect, reconn_interval(AutoReconn)},
{pool_size, PoolSize}
],
PoolName = emqx_plugin_libs_pool:pool_name(InstId), PoolName = emqx_plugin_libs_pool:pool_name(InstId),
Prepares = maps:get(prepare_statement, Config, #{}), Prepares = maps:get(prepare_statement, Config, #{}),
State = init_prepare(#{poolname => PoolName, prepare_statement => Prepares}), State = init_prepare(#{poolname => PoolName, prepare_statement => Prepares}),
case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of
ok -> {ok, State}; ok -> {ok, State};
{error, Reason} -> {error, Reason} {error, Reason} -> {error, Reason}
end. end.
on_stop(InstId, #{poolname := PoolName}) -> on_stop(InstId, #{poolname := PoolName}) ->
?SLOG(info, #{msg => "stopping_mysql_connector", ?SLOG(info, #{
connector => InstId}), msg => "stopping_mysql_connector",
connector => InstId
}),
emqx_plugin_libs_pool:stop_pool(PoolName). emqx_plugin_libs_pool:stop_pool(PoolName).
on_query(InstId, {Type, SQLOrKey}, AfterQuery, State) -> on_query(InstId, {Type, SQLOrKey}, AfterQuery, State) ->
on_query(InstId, {Type, SQLOrKey, [], default_timeout}, AfterQuery, State); on_query(InstId, {Type, SQLOrKey, [], default_timeout}, AfterQuery, State);
on_query(InstId, {Type, SQLOrKey, Params}, AfterQuery, State) -> on_query(InstId, {Type, SQLOrKey, Params}, AfterQuery, State) ->
on_query(InstId, {Type, SQLOrKey, Params, default_timeout}, AfterQuery, State); on_query(InstId, {Type, SQLOrKey, Params, default_timeout}, AfterQuery, State);
on_query(InstId, {Type, SQLOrKey, Params, Timeout}, AfterQuery, on_query(
#{poolname := PoolName, prepare_statement := Prepares} = State) -> InstId,
{Type, SQLOrKey, Params, Timeout},
AfterQuery,
#{poolname := PoolName, prepare_statement := Prepares} = State
) ->
LogMeta = #{connector => InstId, sql => SQLOrKey, state => State}, LogMeta = #{connector => InstId, sql => SQLOrKey, state => State},
?TRACE("QUERY", "mysql_connector_received", LogMeta), ?TRACE("QUERY", "mysql_connector_received", LogMeta),
Worker = ecpool:get_client(PoolName), Worker = ecpool:get_client(PoolName),
@ -111,28 +129,36 @@ on_query(InstId, {Type, SQLOrKey, Params, Timeout}, AfterQuery,
Result = erlang:apply(mysql, MySqlFunction, [Conn, SQLOrKey, Params, Timeout]), Result = erlang:apply(mysql, MySqlFunction, [Conn, SQLOrKey, Params, Timeout]),
case Result of case Result of
{error, disconnected} -> {error, disconnected} ->
?SLOG(error, ?SLOG(
LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => disconnected}), error,
LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => disconnected}
),
%% kill the poll worker to trigger reconnection %% kill the poll worker to trigger reconnection
_ = exit(Conn, restart), _ = exit(Conn, restart),
emqx_resource:query_failed(AfterQuery), emqx_resource:query_failed(AfterQuery),
Result; Result;
{error, not_prepared} -> {error, not_prepared} ->
?SLOG(warning, ?SLOG(
LogMeta#{msg => "mysql_connector_prepare_query_failed", reason => not_prepared}), warning,
LogMeta#{msg => "mysql_connector_prepare_query_failed", reason => not_prepared}
),
case prepare_sql(Prepares, PoolName) of case prepare_sql(Prepares, PoolName) of
ok -> ok ->
%% not return result, next loop will try again %% not return result, next loop will try again
on_query(InstId, {Type, SQLOrKey, Params, Timeout}, AfterQuery, State); on_query(InstId, {Type, SQLOrKey, Params, Timeout}, AfterQuery, State);
{error, Reason} -> {error, Reason} ->
?SLOG(error, ?SLOG(
LogMeta#{msg => "mysql_connector_do_prepare_failed", reason => Reason}), error,
LogMeta#{msg => "mysql_connector_do_prepare_failed", reason => Reason}
),
emqx_resource:query_failed(AfterQuery), emqx_resource:query_failed(AfterQuery),
{error, Reason} {error, Reason}
end; end;
{error, Reason} -> {error, Reason} ->
?SLOG(error, ?SLOG(
LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => Reason}), error,
LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => Reason}
),
emqx_resource:query_failed(AfterQuery), emqx_resource:query_failed(AfterQuery),
Result; Result;
_ -> _ ->
@ -147,7 +173,7 @@ on_health_check(_InstId, #{poolname := PoolName} = State) ->
case emqx_plugin_libs_pool:health_check(PoolName, fun ?MODULE:do_health_check/1, State) of case emqx_plugin_libs_pool:health_check(PoolName, fun ?MODULE:do_health_check/1, State) of
{ok, State} -> {ok, State} ->
case do_health_check_prepares(State) of case do_health_check_prepares(State) of
ok-> ok ->
{ok, State}; {ok, State};
{ok, NState} -> {ok, NState} ->
{ok, NState}; {ok, NState};
@ -161,7 +187,7 @@ on_health_check(_InstId, #{poolname := PoolName} = State) ->
do_health_check(Conn) -> do_health_check(Conn) ->
ok == element(1, mysql:query(Conn, <<"SELECT count(1) AS T">>)). ok == element(1, mysql:query(Conn, <<"SELECT count(1) AS T">>)).
do_health_check_prepares(#{prepare_statement := Prepares})when is_map(Prepares) -> do_health_check_prepares(#{prepare_statement := Prepares}) when is_map(Prepares) ->
ok; ok;
do_health_check_prepares(State = #{poolname := PoolName, prepare_statement := {error, Prepares}}) -> do_health_check_prepares(State = #{poolname := PoolName, prepare_statement := {error, Prepares}}) ->
%% retry to prepare %% retry to prepare
@ -180,8 +206,8 @@ reconn_interval(false) -> false.
connect(Options) -> connect(Options) ->
mysql:start_link(Options). mysql:start_link(Options).
-spec to_server(string()) -spec to_server(string()) ->
-> {inet:ip_address() | inet:hostname(), pos_integer()}. {inet:ip_address() | inet:hostname(), pos_integer()}.
to_server(Str) -> to_server(Str) ->
emqx_connector_schema_lib:parse_server(Str, ?MYSQL_HOST_OPTIONS). emqx_connector_schema_lib:parse_server(Str, ?MYSQL_HOST_OPTIONS).
@ -215,20 +241,27 @@ prepare_sql(Prepares, PoolName) ->
do_prepare_sql(Prepares, PoolName) -> do_prepare_sql(Prepares, PoolName) ->
Conns = Conns =
[begin [
{ok, Conn} = ecpool_worker:client(Worker), begin
Conn {ok, Conn} = ecpool_worker:client(Worker),
end || {_Name, Worker} <- ecpool:workers(PoolName)], Conn
end
|| {_Name, Worker} <- ecpool:workers(PoolName)
],
prepare_sql_to_conn_list(Conns, Prepares). prepare_sql_to_conn_list(Conns, Prepares).
prepare_sql_to_conn_list([], _PrepareList) -> ok; prepare_sql_to_conn_list([], _PrepareList) ->
ok;
prepare_sql_to_conn_list([Conn | ConnList], PrepareList) -> prepare_sql_to_conn_list([Conn | ConnList], PrepareList) ->
case prepare_sql_to_conn(Conn, PrepareList) of case prepare_sql_to_conn(Conn, PrepareList) of
ok -> ok ->
prepare_sql_to_conn_list(ConnList, PrepareList); prepare_sql_to_conn_list(ConnList, PrepareList);
{error, R} -> {error, R} ->
%% rollback %% rollback
Fun = fun({Key, _}) -> _ = unprepare_sql_to_conn(Conn, Key), ok end, Fun = fun({Key, _}) ->
_ = unprepare_sql_to_conn(Conn, Key),
ok
end,
lists:foreach(Fun, PrepareList), lists:foreach(Fun, PrepareList),
{error, R} {error, R}
end. end.

121
apps/emqx_connector/src/emqx_connector_pgsql.erl
View File

@ -26,24 +26,26 @@
-behaviour(emqx_resource). -behaviour(emqx_resource).
%% callbacks of behaviour emqx_resource %% callbacks of behaviour emqx_resource
-export([ on_start/2 -export([
, on_stop/2 on_start/2,
, on_query/4 on_stop/2,
, on_health_check/2 on_query/4,
]). on_health_check/2
]).
-export([connect/1]). -export([connect/1]).
-export([ query/3 -export([
, prepared_query/3 query/3,
]). prepared_query/3
]).
-export([do_health_check/1]). -export([do_health_check/1]).
-define( PGSQL_HOST_OPTIONS -define(PGSQL_HOST_OPTIONS, #{
, #{ host_type => inet_addr host_type => inet_addr,
, default_port => ?PGSQL_DEFAULT_PORT}). default_port => ?PGSQL_DEFAULT_PORT
}).
%%===================================================================== %%=====================================================================
@ -52,9 +54,9 @@ roots() ->
fields(config) -> fields(config) ->
[{server, fun server/1}] ++ [{server, fun server/1}] ++
emqx_connector_schema_lib:relational_db_fields() ++ emqx_connector_schema_lib:relational_db_fields() ++
emqx_connector_schema_lib:ssl_fields() ++ emqx_connector_schema_lib:ssl_fields() ++
emqx_connector_schema_lib:prepare_statement_fields(). emqx_connector_schema_lib:prepare_statement_fields().
server(type) -> emqx_schema:ip_port(); server(type) -> emqx_schema:ip_port();
server(required) -> true; server(required) -> true;
@ -64,52 +66,73 @@ server(desc) -> ?DESC("server");
server(_) -> undefined. server(_) -> undefined.
%% =================================================================== %% ===================================================================
on_start(InstId, #{server := {Host, Port}, on_start(
database := DB, InstId,
username := User, #{
password := Password, server := {Host, Port},
auto_reconnect := AutoReconn, database := DB,
pool_size := PoolSize, username := User,
ssl := SSL} = Config) -> password := Password,
?SLOG(info, #{msg => "starting_postgresql_connector", auto_reconnect := AutoReconn,
connector => InstId, config => Config}), pool_size := PoolSize,
SslOpts = case maps:get(enable, SSL) of ssl := SSL
true -> } = Config
[{ssl, true}, ) ->
{ssl_opts, emqx_tls_lib:to_client_opts(SSL)}]; ?SLOG(info, #{
false -> msg => "starting_postgresql_connector",
[{ssl, false}] connector => InstId,
end, config => Config
Options = [{host, Host}, }),
{port, Port}, SslOpts =
{username, User}, case maps:get(enable, SSL) of
{password, Password}, true ->
{database, DB}, [
{auto_reconnect, reconn_interval(AutoReconn)}, {ssl, true},
{pool_size, PoolSize}, {ssl_opts, emqx_tls_lib:to_client_opts(SSL)}
{prepare_statement, maps:to_list(maps:get(prepare_statement, Config, #{}))}], ];
false ->
[{ssl, false}]
end,
Options = [
{host, Host},
{port, Port},
{username, User},
{password, Password},
{database, DB},
{auto_reconnect, reconn_interval(AutoReconn)},
{pool_size, PoolSize},
{prepare_statement, maps:to_list(maps:get(prepare_statement, Config, #{}))}
],
PoolName = emqx_plugin_libs_pool:pool_name(InstId), PoolName = emqx_plugin_libs_pool:pool_name(InstId),
case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of
ok -> {ok, #{poolname => PoolName}}; ok -> {ok, #{poolname => PoolName}};
{error, Reason} -> {error, Reason} {error, Reason} -> {error, Reason}
end. end.
on_stop(InstId, #{poolname := PoolName}) -> on_stop(InstId, #{poolname := PoolName}) ->
?SLOG(info, #{msg => "stopping postgresql connector", ?SLOG(info, #{
connector => InstId}), msg => "stopping postgresql connector",
connector => InstId
}),
emqx_plugin_libs_pool:stop_pool(PoolName). emqx_plugin_libs_pool:stop_pool(PoolName).
on_query(InstId, {Type, NameOrSQL}, AfterQuery, #{poolname := _PoolName} = State) -> on_query(InstId, {Type, NameOrSQL}, AfterQuery, #{poolname := _PoolName} = State) ->
on_query(InstId, {Type, NameOrSQL, []}, AfterQuery, State); on_query(InstId, {Type, NameOrSQL, []}, AfterQuery, State);
on_query(InstId, {Type, NameOrSQL, Params}, AfterQuery, #{poolname := PoolName} = State) -> on_query(InstId, {Type, NameOrSQL, Params}, AfterQuery, #{poolname := PoolName} = State) ->
?SLOG(debug, #{msg => "postgresql connector received sql query", ?SLOG(debug, #{
connector => InstId, sql => NameOrSQL, state => State}), msg => "postgresql connector received sql query",
connector => InstId,
sql => NameOrSQL,
state => State
}),
case Result = ecpool:pick_and_do(PoolName, {?MODULE, Type, [NameOrSQL, Params]}, no_handover) of case Result = ecpool:pick_and_do(PoolName, {?MODULE, Type, [NameOrSQL, Params]}, no_handover) of
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{ ?SLOG(error, #{
msg => "postgresql connector do sql query failed", msg => "postgresql connector do sql query failed",
connector => InstId, sql => NameOrSQL, reason => Reason}), connector => InstId,
sql => NameOrSQL,
reason => Reason
}),
emqx_resource:query_failed(AfterQuery); emqx_resource:query_failed(AfterQuery);
_ -> _ ->
emqx_resource:query_success(AfterQuery) emqx_resource:query_success(AfterQuery)
@ -127,7 +150,7 @@ reconn_interval(true) -> 15;
reconn_interval(false) -> false. reconn_interval(false) -> false.
connect(Opts) -> connect(Opts) ->
Host = proplists:get_value(host, Opts), Host = proplists:get_value(host, Opts),
Username = proplists:get_value(username, Opts), Username = proplists:get_value(username, Opts),
Password = proplists:get_value(password, Opts), Password = proplists:get_value(password, Opts),
PrepareStatement = proplists:get_value(prepare_statement, Opts), PrepareStatement = proplists:get_value(prepare_statement, Opts),
@ -177,7 +200,7 @@ conn_opts([_Opt | Opts], Acc) ->
%% =================================================================== %% ===================================================================
%% typereflt funcs %% typereflt funcs
-spec to_server(string()) -spec to_server(string()) ->
-> {inet:ip_address() | inet:hostname(), pos_integer()}. {inet:ip_address() | inet:hostname(), pos_integer()}.
to_server(Str) -> to_server(Str) ->
emqx_connector_schema_lib:parse_server(Str, ?PGSQL_HOST_OPTIONS). emqx_connector_schema_lib:parse_server(Str, ?PGSQL_HOST_OPTIONS).

235
apps/emqx_connector/src/emqx_connector_redis.erl
View File

@ -25,11 +25,12 @@
-behaviour(emqx_resource). -behaviour(emqx_resource).
%% callbacks of behaviour emqx_resource %% callbacks of behaviour emqx_resource
-export([ on_start/2 -export([
, on_stop/2 on_start/2,
, on_query/4 on_stop/2,
, on_health_check/2 on_query/4,
]). on_health_check/2
]).
-export([do_health_check/1]). -export([do_health_check/1]).
@ -38,50 +39,59 @@
-export([cmd/3]). -export([cmd/3]).
%% redis host don't need parse %% redis host don't need parse
-define( REDIS_HOST_OPTIONS -define(REDIS_HOST_OPTIONS, #{
, #{ host_type => hostname host_type => hostname,
, default_port => ?REDIS_DEFAULT_PORT}). default_port => ?REDIS_DEFAULT_PORT
}).
%%===================================================================== %%=====================================================================
roots() -> roots() ->
[ {config, #{type => hoconsc:union( [
[ hoconsc:ref(?MODULE, cluster) {config, #{
, hoconsc:ref(?MODULE, single) type => hoconsc:union(
, hoconsc:ref(?MODULE, sentinel) [
])} hoconsc:ref(?MODULE, cluster),
} hoconsc:ref(?MODULE, single),
hoconsc:ref(?MODULE, sentinel)
]
)
}}
]. ].
fields(single) -> fields(single) ->
[ {server, fun server/1} [
, {redis_type, #{type => hoconsc:enum([single]), {server, fun server/1},
required => true, {redis_type, #{
desc => ?DESC("single") type => hoconsc:enum([single]),
}} required => true,
desc => ?DESC("single")
}}
] ++ ] ++
redis_fields() ++ redis_fields() ++
emqx_connector_schema_lib:ssl_fields(); emqx_connector_schema_lib:ssl_fields();
fields(cluster) -> fields(cluster) ->
[ {servers, fun servers/1} [
, {redis_type, #{type => hoconsc:enum([cluster]), {servers, fun servers/1},
required => true, {redis_type, #{
desc => ?DESC("cluster") type => hoconsc:enum([cluster]),
}} required => true,
desc => ?DESC("cluster")
}}
] ++ ] ++
redis_fields() ++ redis_fields() ++
emqx_connector_schema_lib:ssl_fields(); emqx_connector_schema_lib:ssl_fields();
fields(sentinel) -> fields(sentinel) ->
[ {servers, fun servers/1} [
, {redis_type, #{type => hoconsc:enum([sentinel]), {servers, fun servers/1},
required => true, {redis_type, #{
desc => ?DESC("sentinel") type => hoconsc:enum([sentinel]),
}} required => true,
, {sentinel, #{type => string(), desc => ?DESC("sentinel_desc") desc => ?DESC("sentinel")
}} }},
{sentinel, #{type => string(), desc => ?DESC("sentinel_desc")}}
] ++ ] ++
redis_fields() ++ redis_fields() ++
emqx_connector_schema_lib:ssl_fields(). emqx_connector_schema_lib:ssl_fields().
server(type) -> emqx_schema:ip_port(); server(type) -> emqx_schema:ip_port();
server(required) -> true; server(required) -> true;
@ -98,62 +108,89 @@ servers(desc) -> ?DESC("servers");
servers(_) -> undefined. servers(_) -> undefined.
%% =================================================================== %% ===================================================================
on_start(InstId, #{redis_type := Type, on_start(
database := Database, InstId,
pool_size := PoolSize, #{
auto_reconnect := AutoReconn, redis_type := Type,
ssl := SSL } = Config) -> database := Database,
?SLOG(info, #{msg => "starting_redis_connector", pool_size := PoolSize,
connector => InstId, config => Config}), auto_reconnect := AutoReconn,
Servers = case Type of ssl := SSL
single -> [{servers, [maps:get(server, Config)]}]; } = Config
_ ->[{servers, maps:get(servers, Config)}] ) ->
end, ?SLOG(info, #{
Opts = [{pool_size, PoolSize}, msg => "starting_redis_connector",
connector => InstId,
config => Config
}),
Servers =
case Type of
single -> [{servers, [maps:get(server, Config)]}];
_ -> [{servers, maps:get(servers, Config)}]
end,
Opts =
[
{pool_size, PoolSize},
{database, Database}, {database, Database},
{password, maps:get(password, Config, "")}, {password, maps:get(password, Config, "")},
{auto_reconnect, reconn_interval(AutoReconn)} {auto_reconnect, reconn_interval(AutoReconn)}
] ++ Servers, ] ++ Servers,
Options = case maps:get(enable, SSL) of Options =
true -> case maps:get(enable, SSL) of
[{ssl, true}, true ->
{ssl_options, emqx_tls_lib:to_client_opts(SSL)}]; [
false -> [{ssl, false}] {ssl, true},
end ++ [{sentinel, maps:get(sentinel, Config, undefined)}], {ssl_options, emqx_tls_lib:to_client_opts(SSL)}
];
false ->
[{ssl, false}]
end ++ [{sentinel, maps:get(sentinel, Config, undefined)}],
PoolName = emqx_plugin_libs_pool:pool_name(InstId), PoolName = emqx_plugin_libs_pool:pool_name(InstId),
case Type of case Type of
cluster -> cluster ->
case eredis_cluster:start_pool(PoolName, Opts ++ [{options, Options}]) of case eredis_cluster:start_pool(PoolName, Opts ++ [{options, Options}]) of
{ok, _} -> {ok, #{poolname => PoolName, type => Type}}; {ok, _} -> {ok, #{poolname => PoolName, type => Type}};
{ok, _, _} -> {ok, #{poolname => PoolName, type => Type}}; {ok, _, _} -> {ok, #{poolname => PoolName, type => Type}};
{error, Reason} -> {error, Reason} {error, Reason} -> {error, Reason}
end; end;
_ -> _ ->
case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts ++ [{options, Options}]) of case
ok -> {ok, #{poolname => PoolName, type => Type}}; emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts ++ [{options, Options}])
of
ok -> {ok, #{poolname => PoolName, type => Type}};
{error, Reason} -> {error, Reason} {error, Reason} -> {error, Reason}
end end
end. end.
on_stop(InstId, #{poolname := PoolName, type := Type}) -> on_stop(InstId, #{poolname := PoolName, type := Type}) ->
?SLOG(info, #{msg => "stopping_redis_connector", ?SLOG(info, #{
connector => InstId}), msg => "stopping_redis_connector",
connector => InstId
}),
case Type of case Type of
cluster -> eredis_cluster:stop_pool(PoolName); cluster -> eredis_cluster:stop_pool(PoolName);
_ -> emqx_plugin_libs_pool:stop_pool(PoolName) _ -> emqx_plugin_libs_pool:stop_pool(PoolName)
end. end.
on_query(InstId, {cmd, Command}, AfterCommand, #{poolname := PoolName, type := Type} = State) -> on_query(InstId, {cmd, Command}, AfterCommand, #{poolname := PoolName, type := Type} = State) ->
?TRACE("QUERY", "redis_connector_received", ?TRACE(
#{connector => InstId, sql => Command, state => State}), "QUERY",
Result = case Type of "redis_connector_received",
cluster -> eredis_cluster:q(PoolName, Command); #{connector => InstId, sql => Command, state => State}
_ -> ecpool:pick_and_do(PoolName, {?MODULE, cmd, [Type, Command]}, no_handover) ),
end, Result =
case Type of
cluster -> eredis_cluster:q(PoolName, Command);
_ -> ecpool:pick_and_do(PoolName, {?MODULE, cmd, [Type, Command]}, no_handover)
end,
case Result of case Result of
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => "redis_connector_do_cmd_query_failed", ?SLOG(error, #{
connector => InstId, sql => Command, reason => Reason}), msg => "redis_connector_do_cmd_query_failed",
connector => InstId,
sql => Command,
reason => Reason
}),
emqx_resource:query_failed(AfterCommand); emqx_resource:query_failed(AfterCommand);
_ -> _ ->
emqx_resource:query_success(AfterCommand) emqx_resource:query_success(AfterCommand)
@ -161,14 +198,19 @@ on_query(InstId, {cmd, Command}, AfterCommand, #{poolname := PoolName, type := T
Result. Result.
extract_eredis_cluster_workers(PoolName) -> extract_eredis_cluster_workers(PoolName) ->
lists:flatten([gen_server:call(PoolPid, get_all_workers) || lists:flatten([
PoolPid <- eredis_cluster_monitor:get_all_pools(PoolName)]). gen_server:call(PoolPid, get_all_workers)
|| PoolPid <- eredis_cluster_monitor:get_all_pools(PoolName)
]).
eredis_cluster_workers_exist_and_are_connected(Workers) -> eredis_cluster_workers_exist_and_are_connected(Workers) ->
length(Workers) > 0 andalso lists:all( length(Workers) > 0 andalso
fun({_, Pid, _, _}) -> lists:all(
eredis_cluster_pool_worker:is_connected(Pid) =:= true fun({_, Pid, _, _}) ->
end, Workers). eredis_cluster_pool_worker:is_connected(Pid) =:= true
end,
Workers
).
on_health_check(_InstId, #{type := cluster, poolname := PoolName} = State) -> on_health_check(_InstId, #{type := cluster, poolname := PoolName} = State) ->
case eredis_cluster:pool_exists(PoolName) of case eredis_cluster:pool_exists(PoolName) of
@ -178,12 +220,9 @@ on_health_check(_InstId, #{type := cluster, poolname := PoolName} = State) ->
true -> {ok, State}; true -> {ok, State};
false -> {error, health_check_failed, State} false -> {error, health_check_failed, State}
end; end;
false -> false ->
{error, health_check_failed, State} {error, health_check_failed, State}
end; end;
on_health_check(_InstId, #{poolname := PoolName} = State) -> on_health_check(_InstId, #{poolname := PoolName} = State) ->
emqx_plugin_libs_pool:health_check(PoolName, fun ?MODULE:do_health_check/1, State). emqx_plugin_libs_pool:health_check(PoolName, fun ?MODULE:do_health_check/1, State).
@ -206,28 +245,32 @@ connect(Opts) ->
eredis:start_link(Opts). eredis:start_link(Opts).
redis_fields() -> redis_fields() ->
[ {pool_size, fun emqx_connector_schema_lib:pool_size/1} [
, {password, fun emqx_connector_schema_lib:password/1} {pool_size, fun emqx_connector_schema_lib:pool_size/1},
, {database, #{type => integer(), {password, fun emqx_connector_schema_lib:password/1},
default => 0, {database, #{
required => true, type => integer(),
desc => ?DESC("database") default => 0,
}} required => true,
, {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1} desc => ?DESC("database")
}},
{auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1}
]. ].
-spec to_server_raw(string()) -spec to_server_raw(string()) ->
-> {string(), pos_integer()}. {string(), pos_integer()}.
to_server_raw(Server) -> to_server_raw(Server) ->
emqx_connector_schema_lib:parse_server(Server, ?REDIS_HOST_OPTIONS). emqx_connector_schema_lib:parse_server(Server, ?REDIS_HOST_OPTIONS).
-spec to_servers_raw(string()) -spec to_servers_raw(string()) ->
-> [{string(), pos_integer()}]. [{string(), pos_integer()}].
to_servers_raw(Servers) -> to_servers_raw(Servers) ->
lists:map( fun(Server) -> lists:map(
emqx_connector_schema_lib:parse_server(Server, ?REDIS_HOST_OPTIONS) fun(Server) ->
end emqx_connector_schema_lib:parse_server(Server, ?REDIS_HOST_OPTIONS)
, string:tokens(str(Servers), ", ")). end,
string:tokens(str(Servers), ", ")
).
str(A) when is_atom(A) -> str(A) when is_atom(A) ->
atom_to_list(A); atom_to_list(A);

34
apps/emqx_connector/src/emqx_connector_schema.erl
View File

@ -24,10 +24,11 @@
-export([namespace/0, roots/0, fields/1, desc/1]). -export([namespace/0, roots/0, fields/1, desc/1]).
-export([ get_response/0 -export([
, put_request/0 get_response/0,
, post_request/0 put_request/0,
]). post_request/0
]).
%% the config for http bridges do not need connectors %% the config for http bridges do not need connectors
-define(CONN_TYPES, [mqtt]). -define(CONN_TYPES, [mqtt]).
@ -55,18 +56,25 @@ namespace() -> connector.
roots() -> ["connectors"]. roots() -> ["connectors"].
fields(connectors) -> fields("connectors"); fields(connectors) ->
fields("connectors");
fields("connectors") -> fields("connectors") ->
[ {mqtt, [
mk(hoconsc:map(name, {mqtt,
hoconsc:union([ ref(emqx_connector_mqtt_schema, "connector") mk(
])), hoconsc:map(
#{ desc => ?DESC("mqtt") name,
})} hoconsc:union([ref(emqx_connector_mqtt_schema, "connector")])
),
#{desc => ?DESC("mqtt")}
)}
]. ].
desc(Record) when Record =:= connectors; desc(Record) when
Record =:= "connectors" -> ?DESC("desc_connector"); Record =:= connectors;
Record =:= "connectors"
->
?DESC("desc_connector");
desc(_) -> desc(_) ->
undefined. undefined.

73
apps/emqx_connector/src/emqx_connector_schema_lib.erl
View File

@ -19,32 +19,36 @@
-include_lib("typerefl/include/types.hrl"). -include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl"). -include_lib("hocon/include/hoconsc.hrl").
-export([ relational_db_fields/0 -export([
, ssl_fields/0 relational_db_fields/0,
, prepare_statement_fields/0 ssl_fields/0,
]). prepare_statement_fields/0
]).
-export([ ip_port_to_string/1 -export([
, parse_server/2 ip_port_to_string/1,
]). parse_server/2
]).
-export([ pool_size/1 -export([
, database/1 pool_size/1,
, username/1 database/1,
, password/1 username/1,
, auto_reconnect/1 password/1,
]). auto_reconnect/1
]).
-type database() :: binary(). -type database() :: binary().
-type pool_size() :: pos_integer(). -type pool_size() :: pos_integer().
-type username() :: binary(). -type username() :: binary().
-type password() :: binary(). -type password() :: binary().
-reflect_type([ database/0 -reflect_type([
, pool_size/0 database/0,
, username/0 pool_size/0,
, password/0 username/0,
]). password/0
]).
-export([roots/0, fields/1]). -export([roots/0, fields/1]).
@ -53,24 +57,25 @@ roots() -> [].
fields(_) -> []. fields(_) -> [].
ssl_fields() -> ssl_fields() ->
[ {ssl, #{type => hoconsc:ref(emqx_schema, "ssl_client_opts"), [
default => #{<<"enable">> => false}, {ssl, #{
desc => ?DESC("ssl") type => hoconsc:ref(emqx_schema, "ssl_client_opts"),
} default => #{<<"enable">> => false},
} desc => ?DESC("ssl")
}}
]. ].
relational_db_fields() -> relational_db_fields() ->
[ {database, fun database/1} [
, {pool_size, fun pool_size/1} {database, fun database/1},
, {username, fun username/1} {pool_size, fun pool_size/1},
, {password, fun password/1} {username, fun username/1},
, {auto_reconnect, fun auto_reconnect/1} {password, fun password/1},
{auto_reconnect, fun auto_reconnect/1}
]. ].
prepare_statement_fields() -> prepare_statement_fields() ->
[ {prepare_statement, fun prepare_statement/1} [{prepare_statement, fun prepare_statement/1}].
].
prepare_statement(type) -> map(); prepare_statement(type) -> map();
prepare_statement(desc) -> ?DESC("prepare_statement"); prepare_statement(desc) -> ?DESC("prepare_statement");
@ -113,16 +118,16 @@ parse_server(Str, #{host_type := inet_addr, default_port := DefaultPort}) ->
try string:tokens(str(Str), ": ") of try string:tokens(str(Str), ": ") of
[Ip, Port] -> [Ip, Port] ->
case parse_ip(Ip) of case parse_ip(Ip) of
{ok, R} -> {R, list_to_integer(Port)} {ok, R} -> {R, list_to_integer(Port)}
end; end;
[Ip] -> [Ip] ->
case parse_ip(Ip) of case parse_ip(Ip) of
{ok, R} -> {R, DefaultPort} {ok, R} -> {R, DefaultPort}
end; end;
_ -> _ ->
?THROW_ERROR("Bad server schema.") ?THROW_ERROR("Bad server schema.")
catch catch
error : Reason -> error:Reason ->
?THROW_ERROR(Reason) ?THROW_ERROR(Reason)
end; end;
parse_server(Str, #{host_type := hostname, default_port := DefaultPort}) -> parse_server(Str, #{host_type := hostname, default_port := DefaultPort}) ->
@ -134,7 +139,7 @@ parse_server(Str, #{host_type := hostname, default_port := DefaultPort}) ->
_ -> _ ->
?THROW_ERROR("Bad server schema.") ?THROW_ERROR("Bad server schema.")
catch catch
error : Reason -> error:Reason ->
?THROW_ERROR(Reason) ?THROW_ERROR(Reason)
end; end;
parse_server(_, _) -> parse_server(_, _) ->

11
apps/emqx_connector/src/emqx_connector_ssl.erl
View File

@ -1,4 +1,3 @@
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. %% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%% %%
@ -17,9 +16,10 @@
-module(emqx_connector_ssl). -module(emqx_connector_ssl).
-export([ convert_certs/2 -export([
, clear_certs/2 convert_certs/2,
]). clear_certs/2
]).
convert_certs(RltvDir, NewConfig) -> convert_certs(RltvDir, NewConfig) ->
NewSSL = drop_invalid_certs(maps:get(<<"ssl">>, NewConfig, undefined)), NewSSL = drop_invalid_certs(maps:get(<<"ssl">>, NewConfig, undefined)),
@ -40,7 +40,8 @@ new_ssl_config(Config, SSL) -> Config#{<<"ssl">> => SSL}.
drop_invalid_certs(undefined) -> undefined; drop_invalid_certs(undefined) -> undefined;
drop_invalid_certs(SSL) -> emqx_tls_lib:drop_invalid_certs(SSL). drop_invalid_certs(SSL) -> emqx_tls_lib:drop_invalid_certs(SSL).
map_get_oneof([], _Map, Default) -> Default; map_get_oneof([], _Map, Default) ->
Default;
map_get_oneof([Key | Keys], Map, Default) -> map_get_oneof([Key | Keys], Map, Default) ->
case maps:find(Key, Map) of case maps:find(Key, Map) of
error -> error ->

22
apps/emqx_connector/src/emqx_connector_sup.erl
View File

@ -27,20 +27,24 @@ start_link() ->
supervisor:start_link({local, ?SERVER}, ?MODULE, []). supervisor:start_link({local, ?SERVER}, ?MODULE, []).
init([]) -> init([]) ->
SupFlags = #{strategy => one_for_all, SupFlags = #{
intensity => 5, strategy => one_for_all,
period => 20}, intensity => 5,
period => 20
},
ChildSpecs = [ ChildSpecs = [
child_spec(emqx_connector_mqtt) child_spec(emqx_connector_mqtt)
], ],
{ok, {SupFlags, ChildSpecs}}. {ok, {SupFlags, ChildSpecs}}.
child_spec(Mod) -> child_spec(Mod) ->
#{id => Mod, #{
start => {Mod, start_link, []}, id => Mod,
restart => permanent, start => {Mod, start_link, []},
shutdown => 3000, restart => permanent,
type => supervisor, shutdown => 3000,
modules => [Mod]}. type => supervisor,
modules => [Mod]
}.
%% internal functions %% internal functions

181
apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl
View File

@ -18,21 +18,24 @@
-module(emqx_connector_mqtt_mod). -module(emqx_connector_mqtt_mod).
-export([ start/1 -export([
, send/2 start/1,
, stop/1 send/2,
, ping/1 stop/1,
]). ping/1
]).
-export([ ensure_subscribed/3 -export([
, ensure_unsubscribed/2 ensure_subscribed/3,
]). ensure_unsubscribed/2
]).
%% callbacks for emqtt %% callbacks for emqtt
-export([ handle_puback/2 -export([
, handle_publish/3 handle_puback/2,
, handle_disconnected/2 handle_publish/3,
]). handle_disconnected/2
]).
-include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/logger.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl").
@ -69,7 +72,7 @@ start(Config) ->
ok = sub_remote_topics(Pid, Subscriptions), ok = sub_remote_topics(Pid, Subscriptions),
{ok, #{client_pid => Pid, subscriptions => Subscriptions}} {ok, #{client_pid => Pid, subscriptions => Subscriptions}}
catch catch
throw : Reason -> throw:Reason ->
ok = stop(#{client_pid => Pid}), ok = stop(#{client_pid => Pid}),
{error, error_reason(Reason, ServerStr)} {error, error_reason(Reason, ServerStr)}
end; end;
@ -90,13 +93,14 @@ stop(#{client_pid := Pid}) ->
ping(undefined) -> ping(undefined) ->
pang; pang;
ping(#{client_pid := Pid}) -> ping(#{client_pid := Pid}) ->
emqtt:ping(Pid). emqtt:ping(Pid).
ensure_subscribed(#{client_pid := Pid, subscriptions := Subs} = Conn, Topic, QoS) when is_pid(Pid) -> ensure_subscribed(#{client_pid := Pid, subscriptions := Subs} = Conn, Topic, QoS) when
is_pid(Pid)
->
case emqtt:subscribe(Pid, Topic, QoS) of case emqtt:subscribe(Pid, Topic, QoS) of
{ok, _, _} -> Conn#{subscriptions => [{Topic, QoS}|Subs]}; {ok, _, _} -> Conn#{subscriptions => [{Topic, QoS} | Subs]};
Error -> {error, Error} Error -> {error, Error}
end; end;
ensure_subscribed(_Conn, _Topic, _QoS) -> ensure_subscribed(_Conn, _Topic, _QoS) ->
@ -120,15 +124,14 @@ safe_stop(Pid, StopF, Timeout) ->
try try
StopF() StopF()
catch catch
_ : _ -> _:_ ->
ok ok
end, end,
receive receive
{'DOWN', MRef, _, _, _} -> {'DOWN', MRef, _, _, _} ->
ok ok
after after Timeout ->
Timeout -> exit(Pid, kill)
exit(Pid, kill)
end. end.
send(Conn, Msgs) -> send(Conn, Msgs) ->
@ -157,26 +160,38 @@ send(#{client_pid := ClientPid} = Conn, [Msg | Rest], PktIds) ->
{error, Reason} {error, Reason}
end. end.
handle_puback(#{packet_id := PktId, reason_code := RC}, Parent) handle_puback(#{packet_id := PktId, reason_code := RC}, Parent) when
when RC =:= ?RC_SUCCESS; RC =:= ?RC_SUCCESS;
RC =:= ?RC_NO_MATCHING_SUBSCRIBERS -> RC =:= ?RC_NO_MATCHING_SUBSCRIBERS
Parent ! {batch_ack, PktId}, ok; ->
Parent ! {batch_ack, PktId},
ok;
handle_puback(#{packet_id := PktId, reason_code := RC}, _Parent) -> handle_puback(#{packet_id := PktId, reason_code := RC}, _Parent) ->
?SLOG(warning, #{msg => "publish_to_remote_node_falied", ?SLOG(warning, #{
packet_id => PktId, reason_code => RC}). msg => "publish_to_remote_node_falied",
packet_id => PktId,
reason_code => RC
}).
handle_publish(Msg, undefined, _Opts) -> handle_publish(Msg, undefined, _Opts) ->
?SLOG(error, #{msg => "cannot_publish_to_local_broker_as" ?SLOG(error, #{
"_'ingress'_is_not_configured", msg =>
message => Msg}); "cannot_publish_to_local_broker_as"
"_'ingress'_is_not_configured",
message => Msg
});
handle_publish(#{properties := Props} = Msg0, Vars, Opts) -> handle_publish(#{properties := Props} = Msg0, Vars, Opts) ->
Msg = format_msg_received(Msg0, Opts), Msg = format_msg_received(Msg0, Opts),
?SLOG(debug, #{msg => "publish_to_local_broker", ?SLOG(debug, #{
message => Msg, vars => Vars}), msg => "publish_to_local_broker",
message => Msg,
vars => Vars
}),
case Vars of case Vars of
#{on_message_received := {Mod, Func, Args}} -> #{on_message_received := {Mod, Func, Args}} ->
_ = erlang:apply(Mod, Func, [Msg | Args]); _ = erlang:apply(Mod, Func, [Msg | Args]);
_ -> ok _ ->
ok
end, end,
maybe_publish_to_local_broker(Msg, Vars, Props). maybe_publish_to_local_broker(Msg, Vars, Props).
@ -184,12 +199,14 @@ handle_disconnected(Reason, Parent) ->
Parent ! {disconnected, self(), Reason}. Parent ! {disconnected, self(), Reason}.
make_hdlr(Parent, Vars, Opts) -> make_hdlr(Parent, Vars, Opts) ->
#{puback => {fun ?MODULE:handle_puback/2, [Parent]}, #{
publish => {fun ?MODULE:handle_publish/3, [Vars, Opts]}, puback => {fun ?MODULE:handle_puback/2, [Parent]},
disconnected => {fun ?MODULE:handle_disconnected/2, [Parent]} publish => {fun ?MODULE:handle_publish/3, [Vars, Opts]},
}. disconnected => {fun ?MODULE:handle_disconnected/2, [Parent]}
}.
sub_remote_topics(_ClientPid, undefined) -> ok; sub_remote_topics(_ClientPid, undefined) ->
ok;
sub_remote_topics(ClientPid, #{remote_topic := FromTopic, remote_qos := QoS}) -> sub_remote_topics(ClientPid, #{remote_topic := FromTopic, remote_qos := QoS}) ->
case emqtt:subscribe(ClientPid, FromTopic, QoS) of case emqtt:subscribe(ClientPid, FromTopic, QoS) of
{ok, _, _} -> ok; {ok, _, _} -> ok;
@ -199,52 +216,82 @@ sub_remote_topics(ClientPid, #{remote_topic := FromTopic, remote_qos := QoS}) ->
process_config(Config) -> process_config(Config) ->
maps:without([conn_type, address, receive_mountpoint, subscriptions, name], Config). maps:without([conn_type, address, receive_mountpoint, subscriptions, name], Config).
maybe_publish_to_local_broker(#{topic := Topic} = Msg, #{remote_topic := SubTopic} = Vars, maybe_publish_to_local_broker(
Props) -> #{topic := Topic} = Msg,
#{remote_topic := SubTopic} = Vars,
Props
) ->
case maps:get(local_topic, Vars, undefined) of case maps:get(local_topic, Vars, undefined) of
undefined -> undefined ->
ok; %% local topic is not set, discard it %% local topic is not set, discard it
ok;
_ -> _ ->
case emqx_topic:match(Topic, SubTopic) of case emqx_topic:match(Topic, SubTopic) of
true -> true ->
_ = emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars, Props)), _ = emqx_broker:publish(
emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars, Props)
),
ok; ok;
false -> false ->
?SLOG(warning, #{msg => "discard_message_as_topic_not_matched", ?SLOG(warning, #{
message => Msg, subscribed => SubTopic, got_topic => Topic}) msg => "discard_message_as_topic_not_matched",
message => Msg,
subscribed => SubTopic,
got_topic => Topic
})
end end
end. end.
format_msg_received(#{dup := Dup, payload := Payload, properties := Props, format_msg_received(
qos := QoS, retain := Retain, topic := Topic}, #{server := Server}) -> #{
#{ id => emqx_guid:to_hexstr(emqx_guid:gen()) dup := Dup,
, server => Server payload := Payload,
, payload => Payload properties := Props,
, topic => Topic qos := QoS,
, qos => QoS retain := Retain,
, dup => Dup topic := Topic
, retain => Retain },
, pub_props => printable_maps(Props) #{server := Server}
, message_received_at => erlang:system_time(millisecond) ) ->
}. #{
id => emqx_guid:to_hexstr(emqx_guid:gen()),
server => Server,
payload => Payload,
topic => Topic,
qos => QoS,
dup => Dup,
retain => Retain,
pub_props => printable_maps(Props),
message_received_at => erlang:system_time(millisecond)
}.
printable_maps(undefined) -> #{}; printable_maps(undefined) ->
#{};
printable_maps(Headers) -> printable_maps(Headers) ->
maps:fold( maps:fold(
fun ('User-Property', V0, AccIn) when is_list(V0) -> fun
('User-Property', V0, AccIn) when is_list(V0) ->
AccIn#{ AccIn#{
'User-Property' => maps:from_list(V0), 'User-Property' => maps:from_list(V0),
'User-Property-Pairs' => [#{ 'User-Property-Pairs' => [
key => Key, #{
value => Value key => Key,
} || {Key, Value} <- V0] value => Value
}
|| {Key, Value} <- V0
]
}; };
(K, V0, AccIn) -> AccIn#{K => V0} (K, V0, AccIn) ->
end, #{}, Headers). AccIn#{K => V0}
end,
#{},
Headers
).
ip_port_to_server_str(Host, Port) -> ip_port_to_server_str(Host, Port) ->
HostStr = case inet:ntoa(Host) of HostStr =
{error, einval} -> Host; case inet:ntoa(Host) of
IPStr -> IPStr {error, einval} -> Host;
end, IPStr -> IPStr
end,
list_to_binary(io_lib:format("~s:~w", [HostStr, Port])). list_to_binary(io_lib:format("~s:~w", [HostStr, Port])).

75
apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl
View File

@ -16,17 +16,19 @@
-module(emqx_connector_mqtt_msg). -module(emqx_connector_mqtt_msg).
-export([ to_binary/1 -export([
, from_binary/1 to_binary/1,
, make_pub_vars/2 from_binary/1,
, to_remote_msg/2 make_pub_vars/2,
, to_broker_msg/3 to_remote_msg/2,
, estimate_size/1 to_broker_msg/3,
]). estimate_size/1
]).
-export([ replace_vars_in_str/2 -export([
, replace_simple_var/2 replace_vars_in_str/2,
]). replace_simple_var/2
]).
-export_type([msg/0]). -export_type([msg/0]).
@ -34,7 +36,6 @@
-include_lib("emqtt/include/emqtt.hrl"). -include_lib("emqtt/include/emqtt.hrl").
-type msg() :: emqx_types:message(). -type msg() :: emqx_types:message().
-type exp_msg() :: emqx_types:message() | #mqtt_msg{}. -type exp_msg() :: emqx_types:message() | #mqtt_msg{}.
@ -46,7 +47,8 @@
payload := binary() payload := binary()
}. }.
make_pub_vars(_, undefined) -> undefined; make_pub_vars(_, undefined) ->
undefined;
make_pub_vars(Mountpoint, Conf) when is_map(Conf) -> make_pub_vars(Mountpoint, Conf) when is_map(Conf) ->
Conf#{mountpoint => Mountpoint}. Conf#{mountpoint => Mountpoint}.
@ -57,37 +59,56 @@ make_pub_vars(Mountpoint, Conf) when is_map(Conf) ->
%% Shame that we have to know the callback module here %% Shame that we have to know the callback module here
%% would be great if we can get rid of #mqtt_msg{} record %% would be great if we can get rid of #mqtt_msg{} record
%% and use #message{} in all places. %% and use #message{} in all places.
-spec to_remote_msg(msg() | map(), variables()) -spec to_remote_msg(msg() | map(), variables()) ->
-> exp_msg(). exp_msg().
to_remote_msg(#message{flags = Flags0} = Msg, Vars) -> to_remote_msg(#message{flags = Flags0} = Msg, Vars) ->
Retain0 = maps:get(retain, Flags0, false), Retain0 = maps:get(retain, Flags0, false),
MapMsg = maps:put(retain, Retain0, emqx_rule_events:eventmsg_publish(Msg)), MapMsg = maps:put(retain, Retain0, emqx_rule_events:eventmsg_publish(Msg)),
to_remote_msg(MapMsg, Vars); to_remote_msg(MapMsg, Vars);
to_remote_msg(MapMsg, #{remote_topic := TopicToken, payload := PayloadToken, to_remote_msg(MapMsg, #{
remote_qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) when is_map(MapMsg) -> remote_topic := TopicToken,
payload := PayloadToken,
remote_qos := QoSToken,
retain := RetainToken,
mountpoint := Mountpoint
}) when is_map(MapMsg) ->
Topic = replace_vars_in_str(TopicToken, MapMsg), Topic = replace_vars_in_str(TopicToken, MapMsg),
Payload = process_payload(PayloadToken, MapMsg), Payload = process_payload(PayloadToken, MapMsg),
QoS = replace_simple_var(QoSToken, MapMsg), QoS = replace_simple_var(QoSToken, MapMsg),
Retain = replace_simple_var(RetainToken, MapMsg), Retain = replace_simple_var(RetainToken, MapMsg),
#mqtt_msg{qos = QoS, #mqtt_msg{
retain = Retain, qos = QoS,
topic = topic(Mountpoint, Topic), retain = Retain,
props = #{}, topic = topic(Mountpoint, Topic),
payload = Payload}; props = #{},
payload = Payload
};
to_remote_msg(#message{topic = Topic} = Msg, #{mountpoint := Mountpoint}) -> to_remote_msg(#message{topic = Topic} = Msg, #{mountpoint := Mountpoint}) ->
Msg#message{topic = topic(Mountpoint, Topic)}. Msg#message{topic = topic(Mountpoint, Topic)}.
%% published from remote node over a MQTT connection %% published from remote node over a MQTT connection
to_broker_msg(#{dup := Dup} = MapMsg, to_broker_msg(
#{local_topic := TopicToken, payload := PayloadToken, #{dup := Dup} = MapMsg,
local_qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}, Props) -> #{
local_topic := TopicToken,
payload := PayloadToken,
local_qos := QoSToken,
retain := RetainToken,
mountpoint := Mountpoint
},
Props
) ->
Topic = replace_vars_in_str(TopicToken, MapMsg), Topic = replace_vars_in_str(TopicToken, MapMsg),
Payload = process_payload(PayloadToken, MapMsg), Payload = process_payload(PayloadToken, MapMsg),
QoS = replace_simple_var(QoSToken, MapMsg), QoS = replace_simple_var(QoSToken, MapMsg),
Retain = replace_simple_var(RetainToken, MapMsg), Retain = replace_simple_var(RetainToken, MapMsg),
set_headers(Props, set_headers(
emqx_message:set_flags(#{dup => Dup, retain => Retain}, Props,
emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload))). emqx_message:set_flags(
#{dup => Dup, retain => Retain},
emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload)
)
).
process_payload([], Msg) -> process_payload([], Msg) ->
emqx_json:encode(Msg); emqx_json:encode(Msg);

389
apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
View File

@ -21,15 +21,17 @@
-behaviour(hocon_schema). -behaviour(hocon_schema).
-export([ namespace/0 -export([
, roots/0 namespace/0,
, fields/1 roots/0,
, desc/1 fields/1,
]). desc/1
]).
-export([ ingress_desc/0 -export([
, egress_desc/0 ingress_desc/0,
]). egress_desc/0
]).
-import(emqx_schema, [mk_duration/2]). -import(emqx_schema, [mk_duration/2]).
@ -40,146 +42,210 @@ roots() ->
fields("config") -> fields("config") ->
fields("connector") ++ fields("connector") ++
topic_mappings(); topic_mappings();
fields("connector") -> fields("connector") ->
[ {mode, [
sc(hoconsc:enum([cluster_shareload]), {mode,
#{ default => cluster_shareload sc(
, desc => ?DESC("mode") hoconsc:enum([cluster_shareload]),
})} #{
, {server, default => cluster_shareload,
sc(emqx_schema:ip_port(), desc => ?DESC("mode")
#{ required => true }
, desc => ?DESC("server") )},
})} {server,
, {reconnect_interval, mk_duration( sc(
"Reconnect interval. Delay for the MQTT bridge to retry establishing the connection " emqx_schema:ip_port(),
"in case of transportation failure.", #{
#{default => "15s"})} required => true,
, {proto_ver, desc => ?DESC("server")
sc(hoconsc:enum([v3, v4, v5]), }
#{ default => v4 )},
, desc => ?DESC("proto_ver") {reconnect_interval,
})} mk_duration(
, {username, "Reconnect interval. Delay for the MQTT bridge to retry establishing the connection "
sc(binary(), "in case of transportation failure.",
#{ default => "emqx" #{default => "15s"}
, desc => ?DESC("username") )},
})} {proto_ver,
, {password, sc(
sc(binary(), hoconsc:enum([v3, v4, v5]),
#{ default => "emqx" #{
, desc => ?DESC("password") default => v4,
})} desc => ?DESC("proto_ver")
, {clean_start, }
sc(boolean(), )},
#{ default => true {username,
, desc => ?DESC("clean_start") sc(
})} binary(),
, {keepalive, mk_duration("MQTT Keepalive.", #{default => "300s"})} #{
, {retry_interval, mk_duration( default => "emqx",
"Message retry interval. Delay for the MQTT bridge to retry sending the QoS1/QoS2 " desc => ?DESC("username")
"messages in case of ACK not received.", }
#{default => "15s"})} )},
, {max_inflight, {password,
sc(non_neg_integer(), sc(
#{ default => 32 binary(),
, desc => ?DESC("max_inflight") #{
})} default => "emqx",
, {replayq, desc => ?DESC("password")
sc(ref("replayq"), #{})} }
)},
{clean_start,
sc(
boolean(),
#{
default => true,
desc => ?DESC("clean_start")
}
)},
{keepalive, mk_duration("MQTT Keepalive.", #{default => "300s"})},
{retry_interval,
mk_duration(
"Message retry interval. Delay for the MQTT bridge to retry sending the QoS1/QoS2 "
"messages in case of ACK not received.",
#{default => "15s"}
)},
{max_inflight,
sc(
non_neg_integer(),
#{
default => 32,
desc => ?DESC("max_inflight")
}
)},
{replayq, sc(ref("replayq"), #{})}
] ++ emqx_connector_schema_lib:ssl_fields(); ] ++ emqx_connector_schema_lib:ssl_fields();
fields("ingress") -> fields("ingress") ->
%% the message maybe subscribed by rules, in this case 'local_topic' is not necessary %% the message maybe subscribed by rules, in this case 'local_topic' is not necessary
[ {remote_topic, [
sc(binary(), {remote_topic,
#{ required => true sc(
, validator => fun emqx_schema:non_empty_string/1 binary(),
, desc => ?DESC("ingress_remote_topic") #{
})} required => true,
, {remote_qos, validator => fun emqx_schema:non_empty_string/1,
sc(qos(), desc => ?DESC("ingress_remote_topic")
#{ default => 1 }
, desc => ?DESC("ingress_remote_qos") )},
})} {remote_qos,
, {local_topic, sc(
sc(binary(), qos(),
#{ validator => fun emqx_schema:non_empty_string/1 #{
, desc => ?DESC("ingress_local_topic") default => 1,
})} desc => ?DESC("ingress_remote_qos")
, {local_qos, }
sc(qos(), )},
#{ default => <<"${qos}">> {local_topic,
, desc => ?DESC("ingress_local_qos") sc(
})} binary(),
, {hookpoint, #{
sc(binary(), validator => fun emqx_schema:non_empty_string/1,
#{ desc => ?DESC("ingress_hookpoint") desc => ?DESC("ingress_local_topic")
})} }
)},
{local_qos,
sc(
qos(),
#{
default => <<"${qos}">>,
desc => ?DESC("ingress_local_qos")
}
)},
{hookpoint,
sc(
binary(),
#{desc => ?DESC("ingress_hookpoint")}
)},
, {retain, {retain,
sc(hoconsc:union([boolean(), binary()]), sc(
#{ default => <<"${retain}">> hoconsc:union([boolean(), binary()]),
, desc => ?DESC("retain") #{
})} default => <<"${retain}">>,
desc => ?DESC("retain")
}
)},
, {payload, {payload,
sc(binary(), sc(
#{ default => <<"${payload}">> binary(),
, desc => ?DESC("payload") #{
})} default => <<"${payload}">>,
desc => ?DESC("payload")
}
)}
]; ];
fields("egress") -> fields("egress") ->
%% the message maybe sent from rules, in this case 'local_topic' is not necessary %% the message maybe sent from rules, in this case 'local_topic' is not necessary
[ {local_topic, [
sc(binary(), {local_topic,
#{ desc => ?DESC("egress_local_topic") sc(
, validator => fun emqx_schema:non_empty_string/1 binary(),
})} #{
, {remote_topic, desc => ?DESC("egress_local_topic"),
sc(binary(), validator => fun emqx_schema:non_empty_string/1
#{ required => true }
, validator => fun emqx_schema:non_empty_string/1 )},
, desc => ?DESC("egress_remote_topic") {remote_topic,
})} sc(
, {remote_qos, binary(),
sc(qos(), #{
#{ required => true required => true,
, desc => ?DESC("egress_remote_qos") validator => fun emqx_schema:non_empty_string/1,
})} desc => ?DESC("egress_remote_topic")
}
)},
{remote_qos,
sc(
qos(),
#{
required => true,
desc => ?DESC("egress_remote_qos")
}
)},
, {retain, {retain,
sc(hoconsc:union([boolean(), binary()]), sc(
#{ required => true hoconsc:union([boolean(), binary()]),
, desc => ?DESC("retain") #{
})} required => true,
desc => ?DESC("retain")
}
)},
, {payload, {payload,
sc(binary(), sc(
#{ required => true binary(),
, desc => ?DESC("payload") #{
})} required => true,
desc => ?DESC("payload")
}
)}
]; ];
fields("replayq") -> fields("replayq") ->
[ {dir, [
sc(hoconsc:union([boolean(), string()]), {dir,
#{ desc => ?DESC("dir") sc(
})} hoconsc:union([boolean(), string()]),
, {seg_bytes, #{desc => ?DESC("dir")}
sc(emqx_schema:bytesize(), )},
#{ default => "100MB" {seg_bytes,
, desc => ?DESC("seg_bytes") sc(
})} emqx_schema:bytesize(),
, {offload, #{
sc(boolean(), default => "100MB",
#{ default => false desc => ?DESC("seg_bytes")
, desc => ?DESC("offload") }
})} )},
{offload,
sc(
boolean(),
#{
default => false,
desc => ?DESC("offload")
}
)}
]. ].
desc("connector") -> desc("connector") ->
@ -194,34 +260,37 @@ desc(_) ->
undefined. undefined.
topic_mappings() -> topic_mappings() ->
[ {ingress, [
sc(ref("ingress"), {ingress,
#{ default => #{} sc(
})} ref("ingress"),
, {egress, #{default => #{}}
sc(ref("egress"), )},
#{ default => #{} {egress,
})} sc(
ref("egress"),
#{default => #{}}
)}
]. ].
ingress_desc() -> " ingress_desc() ->
The ingress config defines how this bridge receive messages from the remote MQTT broker, and then "\n"
send them to the local broker.</br> "The ingress config defines how this bridge receive messages from the remote MQTT broker, and then\n"
Template with variables is allowed in 'local_topic', 'remote_qos', 'qos', 'retain', "send them to the local broker.</br>\n"
'payload'.</br> "Template with variables is allowed in 'local_topic', 'remote_qos', 'qos', 'retain',\n"
NOTE: if this bridge is used as the input of a rule (emqx rule engine), and also local_topic is "'payload'.</br>\n"
configured, then messages got from the remote broker will be sent to both the 'local_topic' and "NOTE: if this bridge is used as the input of a rule (emqx rule engine), and also local_topic is\n"
the rule. "configured, then messages got from the remote broker will be sent to both the 'local_topic' and\n"
". "the rule.\n".
egress_desc() -> " egress_desc() ->
The egress config defines how this bridge forwards messages from the local broker to the remote "\n"
broker.</br> "The egress config defines how this bridge forwards messages from the local broker to the remote\n"
Template with variables is allowed in 'remote_topic', 'qos', 'retain', 'payload'.</br> "broker.</br>\n"
NOTE: if this bridge is used as the output of a rule (emqx rule engine), and also local_topic "Template with variables is allowed in 'remote_topic', 'qos', 'retain', 'payload'.</br>\n"
is configured, then both the data got from the rule and the MQTT messages that matches "NOTE: if this bridge is used as the output of a rule (emqx rule engine), and also local_topic\n"
local_topic will be forwarded. "is configured, then both the data got from the rule and the MQTT messages that matches\n"
". "local_topic will be forwarded.\n".
qos() -> qos() ->
hoconsc:union([emqx_schema:qos(), binary()]). hoconsc:union([emqx_schema:qos(), binary()]).

256
apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
View File

@ -66,43 +66,46 @@
-include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/logger.hrl").
%% APIs %% APIs
-export([ start_link/1 -export([
, register_metrics/0 start_link/1,
, stop/1 register_metrics/0,
]). stop/1
]).
%% gen_statem callbacks %% gen_statem callbacks
-export([ terminate/3 -export([
, code_change/4 terminate/3,
, init/1 code_change/4,
, callback_mode/0 init/1,
]). callback_mode/0
]).
%% state functions %% state functions
-export([ idle/3 -export([
, connected/3 idle/3,
]). connected/3
]).
%% management APIs %% management APIs
-export([ ensure_started/1 -export([
, ensure_stopped/1 ensure_started/1,
, status/1 ensure_stopped/1,
, ping/1 status/1,
, send_to_remote/2 ping/1,
]). send_to_remote/2
]).
-export([ get_forwards/1 -export([get_forwards/1]).
]).
-export([ get_subscriptions/1 -export([get_subscriptions/1]).
]).
%% Internal %% Internal
-export([msg_marshaller/1]). -export([msg_marshaller/1]).
-export_type([ config/0 -export_type([
, ack_ref/0 config/0,
]). ack_ref/0
]).
-type id() :: atom() | string() | pid(). -type id() :: atom() | string() | pid().
-type qos() :: emqx_types:qos(). -type qos() :: emqx_types:qos().
@ -113,7 +116,6 @@
-include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/logger.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl").
%% same as default in-flight limit for emqtt %% same as default in-flight limit for emqtt
-define(DEFAULT_INFLIGHT_SIZE, 32). -define(DEFAULT_INFLIGHT_SIZE, 32).
-define(DEFAULT_RECONNECT_DELAY_MS, timer:seconds(5)). -define(DEFAULT_RECONNECT_DELAY_MS, timer:seconds(5)).
@ -188,8 +190,10 @@ callback_mode() -> [state_functions].
%% @doc Config should be a map(). %% @doc Config should be a map().
init(#{name := Name} = ConnectOpts) -> init(#{name := Name} = ConnectOpts) ->
?SLOG(debug, #{msg => "starting_bridge_worker", ?SLOG(debug, #{
name => Name}), msg => "starting_bridge_worker",
name => Name
}),
erlang:process_flag(trap_exit, true), erlang:process_flag(trap_exit, true),
Queue = open_replayq(Name, maps:get(replayq, ConnectOpts, #{})), Queue = open_replayq(Name, maps:get(replayq, ConnectOpts, #{})),
State = init_state(ConnectOpts), State = init_state(ConnectOpts),
@ -205,31 +209,44 @@ init_state(Opts) ->
Mountpoint = maps:get(forward_mountpoint, Opts, undefined), Mountpoint = maps:get(forward_mountpoint, Opts, undefined),
MaxInflightSize = maps:get(max_inflight, Opts, ?DEFAULT_INFLIGHT_SIZE), MaxInflightSize = maps:get(max_inflight, Opts, ?DEFAULT_INFLIGHT_SIZE),
Name = maps:get(name, Opts, undefined), Name = maps:get(name, Opts, undefined),
#{start_type => StartType, #{
reconnect_interval => ReconnDelayMs, start_type => StartType,
mountpoint => format_mountpoint(Mountpoint), reconnect_interval => ReconnDelayMs,
inflight => [], mountpoint => format_mountpoint(Mountpoint),
max_inflight => MaxInflightSize, inflight => [],
connection => undefined, max_inflight => MaxInflightSize,
name => Name}. connection => undefined,
name => Name
}.
open_replayq(Name, QCfg) -> open_replayq(Name, QCfg) ->
Dir = maps:get(dir, QCfg, undefined), Dir = maps:get(dir, QCfg, undefined),
SegBytes = maps:get(seg_bytes, QCfg, ?DEFAULT_SEG_BYTES), SegBytes = maps:get(seg_bytes, QCfg, ?DEFAULT_SEG_BYTES),
MaxTotalSize = maps:get(max_total_size, QCfg, ?DEFAULT_MAX_TOTAL_SIZE), MaxTotalSize = maps:get(max_total_size, QCfg, ?DEFAULT_MAX_TOTAL_SIZE),
QueueConfig = case Dir =:= undefined orelse Dir =:= "" of QueueConfig =
true -> #{mem_only => true}; case Dir =:= undefined orelse Dir =:= "" of
false -> #{dir => filename:join([Dir, node(), Name]), true ->
seg_bytes => SegBytes, max_total_size => MaxTotalSize} #{mem_only => true};
end, false ->
replayq:open(QueueConfig#{sizer => fun emqx_connector_mqtt_msg:estimate_size/1, #{
marshaller => fun ?MODULE:msg_marshaller/1}). dir => filename:join([Dir, node(), Name]),
seg_bytes => SegBytes,
max_total_size => MaxTotalSize
}
end,
replayq:open(QueueConfig#{
sizer => fun emqx_connector_mqtt_msg:estimate_size/1,
marshaller => fun ?MODULE:msg_marshaller/1
}).
pre_process_opts(#{subscriptions := InConf, forwards := OutConf} = ConnectOpts) -> pre_process_opts(#{subscriptions := InConf, forwards := OutConf} = ConnectOpts) ->
ConnectOpts#{subscriptions => pre_process_in_out(in, InConf), ConnectOpts#{
forwards => pre_process_in_out(out, OutConf)}. subscriptions => pre_process_in_out(in, InConf),
forwards => pre_process_in_out(out, OutConf)
}.
pre_process_in_out(_, undefined) -> undefined; pre_process_in_out(_, undefined) ->
undefined;
pre_process_in_out(in, Conf) when is_map(Conf) -> pre_process_in_out(in, Conf) when is_map(Conf) ->
Conf1 = pre_process_conf(local_topic, Conf), Conf1 = pre_process_conf(local_topic, Conf),
Conf2 = pre_process_conf(local_qos, Conf1), Conf2 = pre_process_conf(local_qos, Conf1),
@ -245,7 +262,8 @@ pre_process_in_out_common(Conf) ->
pre_process_conf(Key, Conf) -> pre_process_conf(Key, Conf) ->
case maps:find(Key, Conf) of case maps:find(Key, Conf) of
error -> Conf; error ->
Conf;
{ok, Val} when is_binary(Val) -> {ok, Val} when is_binary(Val) ->
Conf#{Key => emqx_plugin_libs_rule:preproc_tmpl(Val)}; Conf#{Key => emqx_plugin_libs_rule:preproc_tmpl(Val)};
{ok, Val} -> {ok, Val} ->
@ -276,7 +294,6 @@ idle(info, idle, #{start_type := auto} = State) ->
connecting(State); connecting(State);
idle(state_timeout, reconnect, State) -> idle(state_timeout, reconnect, State) ->
connecting(State); connecting(State);
idle(Type, Content, State) -> idle(Type, Content, State) ->
common(idle, Type, Content, State). common(idle, Type, Content, State).
@ -298,13 +315,16 @@ connected(state_timeout, connected, #{inflight := Inflight} = State) ->
connected(internal, maybe_send, State) -> connected(internal, maybe_send, State) ->
{_, NewState} = pop_and_send(State), {_, NewState} = pop_and_send(State),
{keep_state, NewState}; {keep_state, NewState};
connected(
connected(info, {disconnected, Conn, Reason}, info,
#{connection := Connection, name := Name, reconnect_interval := ReconnectDelayMs} = State) -> {disconnected, Conn, Reason},
#{connection := Connection, name := Name, reconnect_interval := ReconnectDelayMs} = State
) ->
?tp(info, disconnected, #{name => Name, reason => Reason}), ?tp(info, disconnected, #{name => Name, reason => Reason}),
case Conn =:= maps:get(client_pid, Connection, undefined) of case Conn =:= maps:get(client_pid, Connection, undefined) of
true -> true ->
{next_state, idle, State#{connection => undefined}, {state_timeout, ReconnectDelayMs, reconnect}}; {next_state, idle, State#{connection => undefined},
{state_timeout, ReconnectDelayMs, reconnect}};
false -> false ->
keep_state_and_data keep_state_and_data
end; end;
@ -317,7 +337,7 @@ connected(Type, Content, State) ->
%% Common handlers %% Common handlers
common(StateName, {call, From}, status, _State) -> common(StateName, {call, From}, status, _State) ->
{keep_state_and_data, [{reply, From, StateName}]}; {keep_state_and_data, [{reply, From, StateName}]};
common(_StateName, {call, From}, ping, #{connection := Conn} =_State) -> common(_StateName, {call, From}, ping, #{connection := Conn} = _State) ->
Reply = emqx_connector_mqtt_mod:ping(Conn), Reply = emqx_connector_mqtt_mod:ping(Conn),
{keep_state_and_data, [{reply, From, Reply}]}; {keep_state_and_data, [{reply, From, Reply}]};
common(_StateName, {call, From}, ensure_stopped, #{connection := undefined} = _State) -> common(_StateName, {call, From}, ensure_stopped, #{connection := undefined} = _State) ->
@ -335,27 +355,39 @@ common(_StateName, cast, {send_to_remote, Msg}, #{replayq := Q} = State) ->
NewQ = replayq:append(Q, [Msg]), NewQ = replayq:append(Q, [Msg]),
{keep_state, State#{replayq => NewQ}, {next_event, internal, maybe_send}}; {keep_state, State#{replayq => NewQ}, {next_event, internal, maybe_send}};
common(StateName, Type, Content, #{name := Name} = State) -> common(StateName, Type, Content, #{name := Name} = State) ->
?SLOG(notice, #{msg => "bridge_discarded_event", ?SLOG(notice, #{
name => Name, type => Type, state_name => StateName, msg => "bridge_discarded_event",
content => Content}), name => Name,
type => Type,
state_name => StateName,
content => Content
}),
{keep_state, State}. {keep_state, State}.
do_connect(#{connect_opts := ConnectOpts, do_connect(
inflight := Inflight, #{
name := Name} = State) -> connect_opts := ConnectOpts,
inflight := Inflight,
name := Name
} = State
) ->
case emqx_connector_mqtt_mod:start(ConnectOpts) of case emqx_connector_mqtt_mod:start(ConnectOpts) of
{ok, Conn} -> {ok, Conn} ->
?tp(info, connected, #{name => Name, inflight => length(Inflight)}), ?tp(info, connected, #{name => Name, inflight => length(Inflight)}),
{ok, State#{connection => Conn}}; {ok, State#{connection => Conn}};
{error, Reason} -> {error, Reason} ->
ConnectOpts1 = obfuscate(ConnectOpts), ConnectOpts1 = obfuscate(ConnectOpts),
?SLOG(error, #{msg => "failed_to_connect", ?SLOG(error, #{
config => ConnectOpts1, reason => Reason}), msg => "failed_to_connect",
config => ConnectOpts1,
reason => Reason
}),
{error, Reason, State} {error, Reason, State}
end. end.
%% Retry all inflight (previously sent but not acked) batches. %% Retry all inflight (previously sent but not acked) batches.
retry_inflight(State, []) -> {ok, State}; retry_inflight(State, []) ->
{ok, State};
retry_inflight(State, [#{q_ack_ref := QAckRef, msg := Msg} | Rest] = OldInf) -> retry_inflight(State, [#{q_ack_ref := QAckRef, msg := Msg} | Rest] = OldInf) ->
case do_send(State, QAckRef, Msg) of case do_send(State, QAckRef, Msg) of
{ok, State1} -> {ok, State1} ->
@ -386,28 +418,49 @@ pop_and_send_loop(#{replayq := Q} = State, N) ->
end. end.
do_send(#{connect_opts := #{forwards := undefined}}, _QAckRef, Msg) -> do_send(#{connect_opts := #{forwards := undefined}}, _QAckRef, Msg) ->
?SLOG(error, #{msg => "cannot_forward_messages_to_remote_broker" ?SLOG(error, #{
"_as_'egress'_is_not_configured", msg =>
messages => Msg}); "cannot_forward_messages_to_remote_broker"
do_send(#{inflight := Inflight, "_as_'egress'_is_not_configured",
connection := Connection, messages => Msg
mountpoint := Mountpoint, });
connect_opts := #{forwards := Forwards}} = State, QAckRef, Msg) -> do_send(
#{
inflight := Inflight,
connection := Connection,
mountpoint := Mountpoint,
connect_opts := #{forwards := Forwards}
} = State,
QAckRef,
Msg
) ->
Vars = emqx_connector_mqtt_msg:make_pub_vars(Mountpoint, Forwards), Vars = emqx_connector_mqtt_msg:make_pub_vars(Mountpoint, Forwards),
ExportMsg = fun(Message) -> ExportMsg = fun(Message) ->
emqx_metrics:inc('bridge.mqtt.message_sent_to_remote'), emqx_metrics:inc('bridge.mqtt.message_sent_to_remote'),
emqx_connector_mqtt_msg:to_remote_msg(Message, Vars) emqx_connector_mqtt_msg:to_remote_msg(Message, Vars)
end, end,
?SLOG(debug, #{msg => "publish_to_remote_broker", ?SLOG(debug, #{
message => Msg, vars => Vars}), msg => "publish_to_remote_broker",
message => Msg,
vars => Vars
}),
case emqx_connector_mqtt_mod:send(Connection, [ExportMsg(Msg)]) of case emqx_connector_mqtt_mod:send(Connection, [ExportMsg(Msg)]) of
{ok, Refs} -> {ok, Refs} ->
{ok, State#{inflight := Inflight ++ [#{q_ack_ref => QAckRef, {ok, State#{
send_ack_ref => map_set(Refs), inflight := Inflight ++
msg => Msg}]}}; [
#{
q_ack_ref => QAckRef,
send_ack_ref => map_set(Refs),
msg => Msg
}
]
}};
{error, Reason} -> {error, Reason} ->
?SLOG(info, #{msg => "mqtt_bridge_produce_failed", ?SLOG(info, #{
reason => Reason}), msg => "mqtt_bridge_produce_failed",
reason => Reason
}),
{error, State} {error, State}
end. end.
@ -427,8 +480,10 @@ handle_batch_ack(#{inflight := Inflight0, replayq := Q} = State, Ref) ->
State#{inflight := Inflight}. State#{inflight := Inflight}.
do_ack([], Ref) -> do_ack([], Ref) ->
?SLOG(debug, #{msg => "stale_batch_ack_reference", ?SLOG(debug, #{
ref => Ref}), msg => "stale_batch_ack_reference",
ref => Ref
}),
[]; [];
do_ack([#{send_ack_ref := Refs} = First | Rest], Ref) -> do_ack([#{send_ack_ref := Refs} = First | Rest], Ref) ->
case maps:is_key(Ref, Refs) of case maps:is_key(Ref, Refs) of
@ -443,8 +498,16 @@ do_ack([#{send_ack_ref := Refs} = First | Rest], Ref) ->
drop_acked_batches(_Q, []) -> drop_acked_batches(_Q, []) ->
?tp(debug, inflight_drained, #{}), ?tp(debug, inflight_drained, #{}),
[]; [];
drop_acked_batches(Q, [#{send_ack_ref := Refs, drop_acked_batches(
q_ack_ref := QAckRef} | Rest] = All) -> Q,
[
#{
send_ack_ref := Refs,
q_ack_ref := QAckRef
}
| Rest
] = All
) ->
case maps:size(Refs) of case maps:size(Refs) of
0 -> 0 ->
%% all messages are acked by bridge target %% all messages are acked by bridge target
@ -475,18 +538,25 @@ format_mountpoint(Prefix) ->
name(Id) -> list_to_atom(str(Id)). name(Id) -> list_to_atom(str(Id)).
register_metrics() -> register_metrics() ->
lists:foreach(fun emqx_metrics:ensure/1, lists:foreach(
['bridge.mqtt.message_sent_to_remote', fun emqx_metrics:ensure/1,
'bridge.mqtt.message_received_from_remote' [
]). 'bridge.mqtt.message_sent_to_remote',
'bridge.mqtt.message_received_from_remote'
]
).
obfuscate(Map) -> obfuscate(Map) ->
maps:fold(fun(K, V, Acc) -> maps:fold(
case is_sensitive(K) of fun(K, V, Acc) ->
true -> [{K, '***'} | Acc]; case is_sensitive(K) of
false -> [{K, V} | Acc] true -> [{K, '***'} | Acc];
end false -> [{K, V} | Acc]
end, [], Map). end
end,
[],
Map
).
is_sensitive(password) -> true; is_sensitive(password) -> true;
is_sensitive(_) -> false. is_sensitive(_) -> false.

641
apps/emqx_connector/test/emqx_connector_api_SUITE.erl
View File

@ -26,27 +26,23 @@
-include("emqx_dashboard/include/emqx_dashboard.hrl"). -include("emqx_dashboard/include/emqx_dashboard.hrl").
%% output functions %% output functions
-export([ inspect/3 -export([inspect/3]).
]).
-define(BRIDGE_CONF_DEFAULT, <<"bridges: {}">>). -define(BRIDGE_CONF_DEFAULT, <<"bridges: {}">>).
-define(CONNECTR_TYPE, <<"mqtt">>). -define(CONNECTR_TYPE, <<"mqtt">>).
-define(CONNECTR_NAME, <<"test_connector">>). -define(CONNECTR_NAME, <<"test_connector">>).
-define(BRIDGE_NAME_INGRESS, <<"ingress_test_bridge">>). -define(BRIDGE_NAME_INGRESS, <<"ingress_test_bridge">>).
-define(BRIDGE_NAME_EGRESS, <<"egress_test_bridge">>). -define(BRIDGE_NAME_EGRESS, <<"egress_test_bridge">>).
-define(MQTT_CONNECTOR(Username), -define(MQTT_CONNECTOR(Username), #{
#{
<<"server">> => <<"127.0.0.1:1883">>, <<"server">> => <<"127.0.0.1:1883">>,
<<"username">> => Username, <<"username">> => Username,
<<"password">> => <<"">>, <<"password">> => <<"">>,
<<"proto_ver">> => <<"v4">>, <<"proto_ver">> => <<"v4">>,
<<"ssl">> => #{<<"enable">> => false} <<"ssl">> => #{<<"enable">> => false}
}). }).
-define(MQTT_CONNECTOR2(Server), -define(MQTT_CONNECTOR2(Server), ?MQTT_CONNECTOR(<<"user1">>)#{<<"server">> => Server}).
?MQTT_CONNECTOR(<<"user1">>)#{<<"server">> => Server}).
-define(MQTT_BRIDGE_INGRESS(ID), -define(MQTT_BRIDGE_INGRESS(ID), #{
#{
<<"connector">> => ID, <<"connector">> => ID,
<<"direction">> => <<"ingress">>, <<"direction">> => <<"ingress">>,
<<"remote_topic">> => <<"remote_topic/#">>, <<"remote_topic">> => <<"remote_topic/#">>,
@ -57,8 +53,7 @@
<<"retain">> => <<"${retain}">> <<"retain">> => <<"${retain}">>
}). }).
-define(MQTT_BRIDGE_EGRESS(ID), -define(MQTT_BRIDGE_EGRESS(ID), #{
#{
<<"connector">> => ID, <<"connector">> => ID,
<<"direction">> => <<"egress">>, <<"direction">> => <<"egress">>,
<<"local_topic">> => <<"local_topic/#">>, <<"local_topic">> => <<"local_topic/#">>,
@ -68,10 +63,14 @@
<<"retain">> => <<"${retain}">> <<"retain">> => <<"${retain}">>
}). }).
-define(metrics(MATCH, SUCC, FAILED, SPEED, SPEED5M, SPEEDMAX), -define(metrics(MATCH, SUCC, FAILED, SPEED, SPEED5M, SPEEDMAX), #{
#{<<"matched">> := MATCH, <<"success">> := SUCC, <<"matched">> := MATCH,
<<"failed">> := FAILED, <<"rate">> := SPEED, <<"success">> := SUCC,
<<"rate_last5m">> := SPEED5M, <<"rate_max">> := SPEEDMAX}). <<"failed">> := FAILED,
<<"rate">> := SPEED,
<<"rate_last5m">> := SPEED5M,
<<"rate_max">> := SPEEDMAX
}).
inspect(Selected, _Envs, _Args) -> inspect(Selected, _Envs, _Args) ->
persistent_term:put(?MODULE, #{inspect => Selected}). persistent_term:put(?MODULE, #{inspect => Selected}).
@ -83,24 +82,37 @@ groups() ->
[]. [].
suite() -> suite() ->
[{timetrap,{seconds,30}}]. [{timetrap, {seconds, 30}}].
init_per_suite(Config) -> init_per_suite(Config) ->
_ = application:load(emqx_conf), _ = application:load(emqx_conf),
%% some testcases (may from other app) already get emqx_connector started %% some testcases (may from other app) already get emqx_connector started
_ = application:stop(emqx_resource), _ = application:stop(emqx_resource),
_ = application:stop(emqx_connector), _ = application:stop(emqx_connector),
ok = emqx_common_test_helpers:start_apps([emqx_rule_engine, emqx_connector, ok = emqx_common_test_helpers:start_apps(
emqx_bridge, emqx_dashboard], fun set_special_configs/1), [
emqx_rule_engine,
emqx_connector,
emqx_bridge,
emqx_dashboard
],
fun set_special_configs/1
),
ok = emqx_common_test_helpers:load_config(emqx_connector_schema, <<"connectors: {}">>), ok = emqx_common_test_helpers:load_config(emqx_connector_schema, <<"connectors: {}">>),
ok = emqx_common_test_helpers:load_config(emqx_rule_engine_schema, ok = emqx_common_test_helpers:load_config(
<<"rule_engine {rules {}}">>), emqx_rule_engine_schema,
<<"rule_engine {rules {}}">>
),
ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, ?BRIDGE_CONF_DEFAULT), ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, ?BRIDGE_CONF_DEFAULT),
Config. Config.
end_per_suite(_Config) -> end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([emqx_rule_engine, emqx_connector, emqx_bridge, emqx_common_test_helpers:stop_apps([
emqx_dashboard]), emqx_rule_engine,
emqx_connector,
emqx_bridge,
emqx_dashboard
]),
ok. ok.
set_special_configs(emqx_dashboard) -> set_special_configs(emqx_dashboard) ->
@ -116,15 +128,24 @@ end_per_testcase(_, _Config) ->
ok. ok.
clear_resources() -> clear_resources() ->
lists:foreach(fun(#{id := Id}) -> lists:foreach(
fun(#{id := Id}) ->
ok = emqx_rule_engine:delete_rule(Id) ok = emqx_rule_engine:delete_rule(Id)
end, emqx_rule_engine:get_rules()), end,
lists:foreach(fun(#{type := Type, name := Name}) -> emqx_rule_engine:get_rules()
),
lists:foreach(
fun(#{type := Type, name := Name}) ->
ok = emqx_bridge:remove(Type, Name) ok = emqx_bridge:remove(Type, Name)
end, emqx_bridge:list()), end,
lists:foreach(fun(#{<<"type">> := Type, <<"name">> := Name}) -> emqx_bridge:list()
),
lists:foreach(
fun(#{<<"type">> := Type, <<"name">> := Name}) ->
ok = emqx_connector:delete(Type, Name) ok = emqx_connector:delete(Type, Name)
end, emqx_connector:list_raw()). end,
emqx_connector:list_raw()
).
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% Testcases %% Testcases
@ -137,103 +158,144 @@ t_mqtt_crud_apis(_) ->
%% then we add a mqtt connector, using POST %% then we add a mqtt connector, using POST
%% POST /connectors/ will create a connector %% POST /connectors/ will create a connector
User1 = <<"user1">>, User1 = <<"user1">>,
{ok, 400, <<"{\"code\":\"BAD_REQUEST\",\"message\"" {ok, 400, <<
":\"missing some required fields: [name, type]\"}">>} "{\"code\":\"BAD_REQUEST\",\"message\""
= request(post, uri(["connectors"]), ":\"missing some required fields: [name, type]\"}"
?MQTT_CONNECTOR(User1)#{ <<"type">> => ?CONNECTR_TYPE >>} =
}), request(
{ok, 201, Connector} = request(post, uri(["connectors"]), post,
?MQTT_CONNECTOR(User1)#{ <<"type">> => ?CONNECTR_TYPE uri(["connectors"]),
, <<"name">> => ?CONNECTR_NAME ?MQTT_CONNECTOR(User1)#{<<"type">> => ?CONNECTR_TYPE}
}), ),
{ok, 201, Connector} = request(
post,
uri(["connectors"]),
?MQTT_CONNECTOR(User1)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?CONNECTR_NAME
}
),
#{ <<"type">> := ?CONNECTR_TYPE #{
, <<"name">> := ?CONNECTR_NAME <<"type">> := ?CONNECTR_TYPE,
, <<"server">> := <<"127.0.0.1:1883">> <<"name">> := ?CONNECTR_NAME,
, <<"username">> := User1 <<"server">> := <<"127.0.0.1:1883">>,
, <<"password">> := <<"">> <<"username">> := User1,
, <<"proto_ver">> := <<"v4">> <<"password">> := <<"">>,
, <<"ssl">> := #{<<"enable">> := false} <<"proto_ver">> := <<"v4">>,
} = jsx:decode(Connector), <<"ssl">> := #{<<"enable">> := false}
} = jsx:decode(Connector),
ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME), ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME),
%% update the request-path of the connector %% update the request-path of the connector
User2 = <<"user2">>, User2 = <<"user2">>,
{ok, 200, Connector2} = request(put, uri(["connectors", ConnctorID]), {ok, 200, Connector2} = request(
?MQTT_CONNECTOR(User2)), put,
?assertMatch(#{ <<"type">> := ?CONNECTR_TYPE uri(["connectors", ConnctorID]),
, <<"name">> := ?CONNECTR_NAME ?MQTT_CONNECTOR(User2)
, <<"server">> := <<"127.0.0.1:1883">> ),
, <<"username">> := User2 ?assertMatch(
, <<"password">> := <<"">> #{
, <<"proto_ver">> := <<"v4">> <<"type">> := ?CONNECTR_TYPE,
, <<"ssl">> := #{<<"enable">> := false} <<"name">> := ?CONNECTR_NAME,
}, jsx:decode(Connector2)), <<"server">> := <<"127.0.0.1:1883">>,
<<"username">> := User2,
<<"password">> := <<"">>,
<<"proto_ver">> := <<"v4">>,
<<"ssl">> := #{<<"enable">> := false}
},
jsx:decode(Connector2)
),
%% list all connectors again, assert Connector2 is in it %% list all connectors again, assert Connector2 is in it
{ok, 200, Connector2Str} = request(get, uri(["connectors"]), []), {ok, 200, Connector2Str} = request(get, uri(["connectors"]), []),
?assertMatch([#{ <<"type">> := ?CONNECTR_TYPE ?assertMatch(
, <<"name">> := ?CONNECTR_NAME [
, <<"server">> := <<"127.0.0.1:1883">> #{
, <<"username">> := User2 <<"type">> := ?CONNECTR_TYPE,
, <<"password">> := <<"">> <<"name">> := ?CONNECTR_NAME,
, <<"proto_ver">> := <<"v4">> <<"server">> := <<"127.0.0.1:1883">>,
, <<"ssl">> := #{<<"enable">> := false} <<"username">> := User2,
}], jsx:decode(Connector2Str)), <<"password">> := <<"">>,
<<"proto_ver">> := <<"v4">>,
<<"ssl">> := #{<<"enable">> := false}
}
],
jsx:decode(Connector2Str)
),
%% get the connector by id %% get the connector by id
{ok, 200, Connector3Str} = request(get, uri(["connectors", ConnctorID]), []), {ok, 200, Connector3Str} = request(get, uri(["connectors", ConnctorID]), []),
?assertMatch(#{ <<"type">> := ?CONNECTR_TYPE ?assertMatch(
, <<"name">> := ?CONNECTR_NAME #{
, <<"server">> := <<"127.0.0.1:1883">> <<"type">> := ?CONNECTR_TYPE,
, <<"username">> := User2 <<"name">> := ?CONNECTR_NAME,
, <<"password">> := <<"">> <<"server">> := <<"127.0.0.1:1883">>,
, <<"proto_ver">> := <<"v4">> <<"username">> := User2,
, <<"ssl">> := #{<<"enable">> := false} <<"password">> := <<"">>,
}, jsx:decode(Connector3Str)), <<"proto_ver">> := <<"v4">>,
<<"ssl">> := #{<<"enable">> := false}
},
jsx:decode(Connector3Str)
),
%% delete the connector %% delete the connector
{ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []), {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []), {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
%% update a deleted connector returns an error %% update a deleted connector returns an error
{ok, 404, ErrMsg2} = request(put, uri(["connectors", ConnctorID]), {ok, 404, ErrMsg2} = request(
?MQTT_CONNECTOR(User2)), put,
uri(["connectors", ConnctorID]),
?MQTT_CONNECTOR(User2)
),
?assertMatch( ?assertMatch(
#{ <<"code">> := _ #{
, <<"message">> := <<"connector not found">> <<"code">> := _,
}, jsx:decode(ErrMsg2)), <<"message">> := <<"connector not found">>
},
jsx:decode(ErrMsg2)
),
ok. ok.
t_mqtt_conn_bridge_ingress(_) -> t_mqtt_conn_bridge_ingress(_) ->
%% then we add a mqtt connector, using POST %% then we add a mqtt connector, using POST
User1 = <<"user1">>, User1 = <<"user1">>,
{ok, 201, Connector} = request(post, uri(["connectors"]), {ok, 201, Connector} = request(
?MQTT_CONNECTOR(User1)#{ <<"type">> => ?CONNECTR_TYPE post,
, <<"name">> => ?CONNECTR_NAME uri(["connectors"]),
}), ?MQTT_CONNECTOR(User1)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?CONNECTR_NAME
}
),
#{ <<"type">> := ?CONNECTR_TYPE #{
, <<"name">> := ?CONNECTR_NAME <<"type">> := ?CONNECTR_TYPE,
, <<"server">> := <<"127.0.0.1:1883">> <<"name">> := ?CONNECTR_NAME,
, <<"num_of_bridges">> := 0 <<"server">> := <<"127.0.0.1:1883">>,
, <<"username">> := User1 <<"num_of_bridges">> := 0,
, <<"password">> := <<"">> <<"username">> := User1,
, <<"proto_ver">> := <<"v4">> <<"password">> := <<"">>,
, <<"ssl">> := #{<<"enable">> := false} <<"proto_ver">> := <<"v4">>,
} = jsx:decode(Connector), <<"ssl">> := #{<<"enable">> := false}
} = jsx:decode(Connector),
ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME), ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME),
%% ... and a MQTT bridge, using POST %% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now %% we bind this bridge to the connector created just now
timer:sleep(50), timer:sleep(50),
{ok, 201, Bridge} = request(post, uri(["bridges"]), {ok, 201, Bridge} = request(
post,
uri(["bridges"]),
?MQTT_BRIDGE_INGRESS(ConnctorID)#{ ?MQTT_BRIDGE_INGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE, <<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_INGRESS <<"name">> => ?BRIDGE_NAME_INGRESS
}), }
#{ <<"type">> := ?CONNECTR_TYPE ),
, <<"name">> := ?BRIDGE_NAME_INGRESS #{
, <<"connector">> := ConnctorID <<"type">> := ?CONNECTR_TYPE,
} = jsx:decode(Bridge), <<"name">> := ?BRIDGE_NAME_INGRESS,
<<"connector">> := ConnctorID
} = jsx:decode(Bridge),
BridgeIDIngress = emqx_bridge:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_INGRESS), BridgeIDIngress = emqx_bridge:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_INGRESS),
wait_for_resource_ready(BridgeIDIngress, 5), wait_for_resource_ready(BridgeIDIngress, 5),
@ -257,12 +319,12 @@ t_mqtt_conn_bridge_ingress(_) ->
false false
after 100 -> after 100 ->
false false
end), end
),
%% get the connector by id, verify the num_of_bridges now is 1 %% get the connector by id, verify the num_of_bridges now is 1
{ok, 200, Connector1Str} = request(get, uri(["connectors", ConnctorID]), []), {ok, 200, Connector1Str} = request(get, uri(["connectors", ConnctorID]), []),
?assertMatch(#{ <<"num_of_bridges">> := 1 ?assertMatch(#{<<"num_of_bridges">> := 1}, jsx:decode(Connector1Str)),
}, jsx:decode(Connector1Str)),
%% delete the bridge %% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []),
@ -276,30 +338,39 @@ t_mqtt_conn_bridge_ingress(_) ->
t_mqtt_conn_bridge_egress(_) -> t_mqtt_conn_bridge_egress(_) ->
%% then we add a mqtt connector, using POST %% then we add a mqtt connector, using POST
User1 = <<"user1">>, User1 = <<"user1">>,
{ok, 201, Connector} = request(post, uri(["connectors"]), {ok, 201, Connector} = request(
?MQTT_CONNECTOR(User1)#{ <<"type">> => ?CONNECTR_TYPE post,
, <<"name">> => ?CONNECTR_NAME uri(["connectors"]),
}), ?MQTT_CONNECTOR(User1)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?CONNECTR_NAME
}
),
%ct:pal("---connector: ~p", [Connector]), %ct:pal("---connector: ~p", [Connector]),
#{ <<"server">> := <<"127.0.0.1:1883">> #{
, <<"username">> := User1 <<"server">> := <<"127.0.0.1:1883">>,
, <<"password">> := <<"">> <<"username">> := User1,
, <<"proto_ver">> := <<"v4">> <<"password">> := <<"">>,
, <<"ssl">> := #{<<"enable">> := false} <<"proto_ver">> := <<"v4">>,
} = jsx:decode(Connector), <<"ssl">> := #{<<"enable">> := false}
} = jsx:decode(Connector),
ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME), ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME),
%% ... and a MQTT bridge, using POST %% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now %% we bind this bridge to the connector created just now
{ok, 201, Bridge} = request(post, uri(["bridges"]), {ok, 201, Bridge} = request(
post,
uri(["bridges"]),
?MQTT_BRIDGE_EGRESS(ConnctorID)#{ ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE, <<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS <<"name">> => ?BRIDGE_NAME_EGRESS
}), }
#{ <<"type">> := ?CONNECTR_TYPE ),
, <<"name">> := ?BRIDGE_NAME_EGRESS #{
, <<"connector">> := ConnctorID <<"type">> := ?CONNECTR_TYPE,
} = jsx:decode(Bridge), <<"name">> := ?BRIDGE_NAME_EGRESS,
<<"connector">> := ConnctorID
} = jsx:decode(Bridge),
BridgeIDEgress = emqx_bridge:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_EGRESS), BridgeIDEgress = emqx_bridge:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_EGRESS),
wait_for_resource_ready(BridgeIDEgress, 5), wait_for_resource_ready(BridgeIDEgress, 5),
@ -324,14 +395,19 @@ t_mqtt_conn_bridge_egress(_) ->
false false
after 100 -> after 100 ->
false false
end), end
),
%% verify the metrics of the bridge %% verify the metrics of the bridge
{ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []), {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
?assertMatch(#{ <<"metrics">> := ?metrics(1, 1, 0, _, _, _) ?assertMatch(
, <<"node_metrics">> := #{
[#{<<"node">> := _, <<"metrics">> := ?metrics(1, 1, 0, _, _, _)}] <<"metrics">> := ?metrics(1, 1, 0, _, _, _),
}, jsx:decode(BridgeStr)), <<"node_metrics">> :=
[#{<<"node">> := _, <<"metrics">> := ?metrics(1, 1, 0, _, _, _)}]
},
jsx:decode(BridgeStr)
),
%% delete the bridge %% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
@ -347,38 +423,50 @@ t_mqtt_conn_bridge_egress(_) ->
%% - cannot delete a connector that is used by at least one bridge %% - cannot delete a connector that is used by at least one bridge
t_mqtt_conn_update(_) -> t_mqtt_conn_update(_) ->
%% then we add a mqtt connector, using POST %% then we add a mqtt connector, using POST
{ok, 201, Connector} = request(post, uri(["connectors"]), {ok, 201, Connector} = request(
?MQTT_CONNECTOR2(<<"127.0.0.1:1883">>) post,
#{ <<"type">> => ?CONNECTR_TYPE uri(["connectors"]),
, <<"name">> => ?CONNECTR_NAME ?MQTT_CONNECTOR2(<<"127.0.0.1:1883">>)#{
}), <<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?CONNECTR_NAME
}
),
%ct:pal("---connector: ~p", [Connector]), %ct:pal("---connector: ~p", [Connector]),
#{ <<"server">> := <<"127.0.0.1:1883">> #{<<"server">> := <<"127.0.0.1:1883">>} = jsx:decode(Connector),
} = jsx:decode(Connector),
ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME), ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME),
%% ... and a MQTT bridge, using POST %% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now %% we bind this bridge to the connector created just now
{ok, 201, Bridge} = request(post, uri(["bridges"]), {ok, 201, Bridge} = request(
post,
uri(["bridges"]),
?MQTT_BRIDGE_EGRESS(ConnctorID)#{ ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE, <<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS <<"name">> => ?BRIDGE_NAME_EGRESS
}), }
#{ <<"type">> := ?CONNECTR_TYPE ),
, <<"name">> := ?BRIDGE_NAME_EGRESS #{
, <<"connector">> := ConnctorID <<"type">> := ?CONNECTR_TYPE,
} = jsx:decode(Bridge), <<"name">> := ?BRIDGE_NAME_EGRESS,
<<"connector">> := ConnctorID
} = jsx:decode(Bridge),
BridgeIDEgress = emqx_bridge:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_EGRESS), BridgeIDEgress = emqx_bridge:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_EGRESS),
wait_for_resource_ready(BridgeIDEgress, 5), wait_for_resource_ready(BridgeIDEgress, 5),
%% Then we try to update 'server' of the connector, to an unavailable IP address %% Then we try to update 'server' of the connector, to an unavailable IP address
%% The update OK, we recreate the resource even if the resource is current connected, %% The update OK, we recreate the resource even if the resource is current connected,
%% and the target resource we're going to update is unavailable. %% and the target resource we're going to update is unavailable.
{ok, 200, _} = request(put, uri(["connectors", ConnctorID]), {ok, 200, _} = request(
?MQTT_CONNECTOR2(<<"127.0.0.1:2603">>)), put,
uri(["connectors", ConnctorID]),
?MQTT_CONNECTOR2(<<"127.0.0.1:2603">>)
),
%% we fix the 'server' parameter to a normal one, it should work %% we fix the 'server' parameter to a normal one, it should work
{ok, 200, _} = request(put, uri(["connectors", ConnctorID]), {ok, 200, _} = request(
?MQTT_CONNECTOR2(<<"127.0.0.1 : 1883">>)), put,
uri(["connectors", ConnctorID]),
?MQTT_CONNECTOR2(<<"127.0.0.1 : 1883">>)
),
%% delete the bridge %% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
@ -390,40 +478,51 @@ t_mqtt_conn_update(_) ->
t_mqtt_conn_update2(_) -> t_mqtt_conn_update2(_) ->
%% then we add a mqtt connector, using POST %% then we add a mqtt connector, using POST
%% but this connector is point to a unreachable server "2603" %% but this connector is point to a unreachable server "2603"
{ok, 201, Connector} = request(post, uri(["connectors"]), {ok, 201, Connector} = request(
?MQTT_CONNECTOR2(<<"127.0.0.1:2603">>) post,
#{ <<"type">> => ?CONNECTR_TYPE uri(["connectors"]),
, <<"name">> => ?CONNECTR_NAME ?MQTT_CONNECTOR2(<<"127.0.0.1:2603">>)#{
}), <<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?CONNECTR_NAME
}
),
#{ <<"server">> := <<"127.0.0.1:2603">> #{<<"server">> := <<"127.0.0.1:2603">>} = jsx:decode(Connector),
} = jsx:decode(Connector),
ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME), ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME),
%% ... and a MQTT bridge, using POST %% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now %% we bind this bridge to the connector created just now
{ok, 201, Bridge} = request(post, uri(["bridges"]), {ok, 201, Bridge} = request(
post,
uri(["bridges"]),
?MQTT_BRIDGE_EGRESS(ConnctorID)#{ ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE, <<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS <<"name">> => ?BRIDGE_NAME_EGRESS
}), }
#{ <<"type">> := ?CONNECTR_TYPE ),
, <<"name">> := ?BRIDGE_NAME_EGRESS #{
, <<"status">> := <<"disconnected">> <<"type">> := ?CONNECTR_TYPE,
, <<"connector">> := ConnctorID <<"name">> := ?BRIDGE_NAME_EGRESS,
} = jsx:decode(Bridge), <<"status">> := <<"disconnected">>,
<<"connector">> := ConnctorID
} = jsx:decode(Bridge),
BridgeIDEgress = emqx_bridge:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_EGRESS), BridgeIDEgress = emqx_bridge:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_EGRESS),
%% We try to fix the 'server' parameter, to another unavailable server.. %% We try to fix the 'server' parameter, to another unavailable server..
%% The update should success: we don't check the connectivity of the new config %% The update should success: we don't check the connectivity of the new config
%% if the resource is now disconnected. %% if the resource is now disconnected.
{ok, 200, _} = request(put, uri(["connectors", ConnctorID]), {ok, 200, _} = request(
?MQTT_CONNECTOR2(<<"127.0.0.1:2604">>)), put,
uri(["connectors", ConnctorID]),
?MQTT_CONNECTOR2(<<"127.0.0.1:2604">>)
),
%% we fix the 'server' parameter to a normal one, it should work %% we fix the 'server' parameter to a normal one, it should work
{ok, 200, _} = request(put, uri(["connectors", ConnctorID]), {ok, 200, _} = request(
?MQTT_CONNECTOR2(<<"127.0.0.1:1883">>)), put,
uri(["connectors", ConnctorID]),
?MQTT_CONNECTOR2(<<"127.0.0.1:1883">>)
),
wait_for_resource_ready(BridgeIDEgress, 5), wait_for_resource_ready(BridgeIDEgress, 5),
{ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []), {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
?assertMatch(#{ <<"status">> := <<"connected">> ?assertMatch(#{<<"status">> := <<"connected">>}, jsx:decode(BridgeStr)),
}, jsx:decode(BridgeStr)),
%% delete the bridge %% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
@ -434,21 +533,26 @@ t_mqtt_conn_update2(_) ->
t_mqtt_conn_update3(_) -> t_mqtt_conn_update3(_) ->
%% we add a mqtt connector, using POST %% we add a mqtt connector, using POST
{ok, 201, _} = request(post, uri(["connectors"]), {ok, 201, _} = request(
?MQTT_CONNECTOR2(<<"127.0.0.1:1883">>) post,
#{ <<"type">> => ?CONNECTR_TYPE uri(["connectors"]),
, <<"name">> => ?CONNECTR_NAME ?MQTT_CONNECTOR2(<<"127.0.0.1:1883">>)#{
}), <<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?CONNECTR_NAME
}
),
ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME), ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME),
%% ... and a MQTT bridge, using POST %% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now %% we bind this bridge to the connector created just now
{ok, 201, Bridge} = request(post, uri(["bridges"]), {ok, 201, Bridge} = request(
post,
uri(["bridges"]),
?MQTT_BRIDGE_EGRESS(ConnctorID)#{ ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE, <<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS <<"name">> => ?BRIDGE_NAME_EGRESS
}), }
#{ <<"connector">> := ConnctorID ),
} = jsx:decode(Bridge), #{<<"connector">> := ConnctorID} = jsx:decode(Bridge),
BridgeIDEgress = emqx_bridge:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_EGRESS), BridgeIDEgress = emqx_bridge:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_EGRESS),
wait_for_resource_ready(BridgeIDEgress, 5), wait_for_resource_ready(BridgeIDEgress, 5),
@ -462,37 +566,54 @@ t_mqtt_conn_update3(_) ->
t_mqtt_conn_testing(_) -> t_mqtt_conn_testing(_) ->
%% APIs for testing the connectivity %% APIs for testing the connectivity
%% then we add a mqtt connector, using POST %% then we add a mqtt connector, using POST
{ok, 204, <<>>} = request(post, uri(["connectors_test"]), {ok, 204, <<>>} = request(
post,
uri(["connectors_test"]),
?MQTT_CONNECTOR2(<<"127.0.0.1:1883">>)#{ ?MQTT_CONNECTOR2(<<"127.0.0.1:1883">>)#{
<<"type">> => ?CONNECTR_TYPE, <<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS <<"name">> => ?BRIDGE_NAME_EGRESS
}), }
{ok, 400, _} = request(post, uri(["connectors_test"]), ),
{ok, 400, _} = request(
post,
uri(["connectors_test"]),
?MQTT_CONNECTOR2(<<"127.0.0.1:2883">>)#{ ?MQTT_CONNECTOR2(<<"127.0.0.1:2883">>)#{
<<"type">> => ?CONNECTR_TYPE, <<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS <<"name">> => ?BRIDGE_NAME_EGRESS
}). }
).
t_ingress_mqtt_bridge_with_rules(_) -> t_ingress_mqtt_bridge_with_rules(_) ->
{ok, 201, _} = request(post, uri(["connectors"]), {ok, 201, _} = request(
?MQTT_CONNECTOR(<<"user1">>)#{ <<"type">> => ?CONNECTR_TYPE post,
, <<"name">> => ?CONNECTR_NAME uri(["connectors"]),
}), ?MQTT_CONNECTOR(<<"user1">>)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?CONNECTR_NAME
}
),
ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME), ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME),
{ok, 201, _} = request(post, uri(["bridges"]), {ok, 201, _} = request(
post,
uri(["bridges"]),
?MQTT_BRIDGE_INGRESS(ConnctorID)#{ ?MQTT_BRIDGE_INGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE, <<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_INGRESS <<"name">> => ?BRIDGE_NAME_INGRESS
}), }
),
BridgeIDIngress = emqx_bridge:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_INGRESS), BridgeIDIngress = emqx_bridge:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_INGRESS),
{ok, 201, Rule} = request(post, uri(["rules"]), {ok, 201, Rule} = request(
#{<<"name">> => <<"A rule get messages from a source mqtt bridge">>, post,
<<"enable">> => true, uri(["rules"]),
<<"outputs">> => [#{<<"function">> => "emqx_connector_api_SUITE:inspect"}], #{
<<"sql">> => <<"SELECT * from \"$bridges/", BridgeIDIngress/binary, "\"">> <<"name">> => <<"A rule get messages from a source mqtt bridge">>,
}), <<"enable">> => true,
<<"outputs">> => [#{<<"function">> => "emqx_connector_api_SUITE:inspect"}],
<<"sql">> => <<"SELECT * from \"$bridges/", BridgeIDIngress/binary, "\"">>
}
),
#{<<"id">> := RuleId} = jsx:decode(Rule), #{<<"id">> := RuleId} = jsx:decode(Rule),
%% we now test if the bridge works as expected %% we now test if the bridge works as expected
@ -517,63 +638,81 @@ t_ingress_mqtt_bridge_with_rules(_) ->
false false
after 100 -> after 100 ->
false false
end), end
),
%% and also the rule should be matched, with matched + 1: %% and also the rule should be matched, with matched + 1:
{ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []), {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []),
#{ <<"id">> := RuleId #{
, <<"metrics">> := #{ <<"id">> := RuleId,
<<"sql.matched">> := 1, <<"metrics">> := #{
<<"sql.passed">> := 1, <<"sql.matched">> := 1,
<<"sql.failed">> := 0, <<"sql.passed">> := 1,
<<"sql.failed.exception">> := 0, <<"sql.failed">> := 0,
<<"sql.failed.no_result">> := 0, <<"sql.failed.exception">> := 0,
<<"sql.matched.rate">> := _, <<"sql.failed.no_result">> := 0,
<<"sql.matched.rate.max">> := _, <<"sql.matched.rate">> := _,
<<"sql.matched.rate.last5m">> := _, <<"sql.matched.rate.max">> := _,
<<"outputs.total">> := 1, <<"sql.matched.rate.last5m">> := _,
<<"outputs.success">> := 1, <<"outputs.total">> := 1,
<<"outputs.failed">> := 0, <<"outputs.success">> := 1,
<<"outputs.failed.out_of_service">> := 0, <<"outputs.failed">> := 0,
<<"outputs.failed.unknown">> := 0 <<"outputs.failed.out_of_service">> := 0,
} <<"outputs.failed.unknown">> := 0
} = jsx:decode(Rule1), }
} = jsx:decode(Rule1),
%% we also check if the outputs of the rule is triggered %% we also check if the outputs of the rule is triggered
?assertMatch(#{inspect := #{ ?assertMatch(
event := <<"$bridges/mqtt", _/binary>>, #{
id := MsgId, inspect := #{
payload := Payload, event := <<"$bridges/mqtt", _/binary>>,
topic := RemoteTopic, id := MsgId,
qos := 0, payload := Payload,
dup := false, topic := RemoteTopic,
retain := false, qos := 0,
pub_props := #{}, dup := false,
timestamp := _ retain := false,
}} when is_binary(MsgId), persistent_term:get(?MODULE)), pub_props := #{},
timestamp := _
}
} when is_binary(MsgId),
persistent_term:get(?MODULE)
),
{ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []), {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []),
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []),
{ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []). {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []).
t_egress_mqtt_bridge_with_rules(_) -> t_egress_mqtt_bridge_with_rules(_) ->
{ok, 201, _} = request(post, uri(["connectors"]), {ok, 201, _} = request(
?MQTT_CONNECTOR(<<"user1">>)#{ <<"type">> => ?CONNECTR_TYPE post,
, <<"name">> => ?CONNECTR_NAME uri(["connectors"]),
}), ?MQTT_CONNECTOR(<<"user1">>)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?CONNECTR_NAME
}
),
ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME), ConnctorID = emqx_connector:connector_id(?CONNECTR_TYPE, ?CONNECTR_NAME),
{ok, 201, Bridge} = request(post, uri(["bridges"]), {ok, 201, Bridge} = request(
post,
uri(["bridges"]),
?MQTT_BRIDGE_EGRESS(ConnctorID)#{ ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE, <<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS <<"name">> => ?BRIDGE_NAME_EGRESS
}), }
#{ <<"type">> := ?CONNECTR_TYPE, <<"name">> := ?BRIDGE_NAME_EGRESS } = jsx:decode(Bridge), ),
#{<<"type">> := ?CONNECTR_TYPE, <<"name">> := ?BRIDGE_NAME_EGRESS} = jsx:decode(Bridge),
BridgeIDEgress = emqx_bridge:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_EGRESS), BridgeIDEgress = emqx_bridge:bridge_id(?CONNECTR_TYPE, ?BRIDGE_NAME_EGRESS),
{ok, 201, Rule} = request(post, uri(["rules"]), {ok, 201, Rule} = request(
#{<<"name">> => <<"A rule send messages to a sink mqtt bridge">>, post,
<<"enable">> => true, uri(["rules"]),
<<"outputs">> => [BridgeIDEgress], #{
<<"sql">> => <<"SELECT * from \"t/1\"">> <<"name">> => <<"A rule send messages to a sink mqtt bridge">>,
}), <<"enable">> => true,
<<"outputs">> => [BridgeIDEgress],
<<"sql">> => <<"SELECT * from \"t/1\"">>
}
),
#{<<"id">> := RuleId} = jsx:decode(Rule), #{<<"id">> := RuleId} = jsx:decode(Rule),
%% we now test if the bridge works as expected %% we now test if the bridge works as expected
@ -597,7 +736,8 @@ t_egress_mqtt_bridge_with_rules(_) ->
false false
after 100 -> after 100 ->
false false
end), end
),
emqx:unsubscribe(RemoteTopic), emqx:unsubscribe(RemoteTopic),
%% PUBLISH a message to the rule. %% PUBLISH a message to the rule.
@ -609,23 +749,24 @@ t_egress_mqtt_bridge_with_rules(_) ->
wait_for_resource_ready(BridgeIDEgress, 5), wait_for_resource_ready(BridgeIDEgress, 5),
emqx:publish(emqx_message:make(RuleTopic, Payload2)), emqx:publish(emqx_message:make(RuleTopic, Payload2)),
{ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []), {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []),
#{ <<"id">> := RuleId #{
, <<"metrics">> := #{ <<"id">> := RuleId,
<<"sql.matched">> := 1, <<"metrics">> := #{
<<"sql.passed">> := 1, <<"sql.matched">> := 1,
<<"sql.failed">> := 0, <<"sql.passed">> := 1,
<<"sql.failed.exception">> := 0, <<"sql.failed">> := 0,
<<"sql.failed.no_result">> := 0, <<"sql.failed.exception">> := 0,
<<"sql.matched.rate">> := _, <<"sql.failed.no_result">> := 0,
<<"sql.matched.rate.max">> := _, <<"sql.matched.rate">> := _,
<<"sql.matched.rate.last5m">> := _, <<"sql.matched.rate.max">> := _,
<<"outputs.total">> := 1, <<"sql.matched.rate.last5m">> := _,
<<"outputs.success">> := 1, <<"outputs.total">> := 1,
<<"outputs.failed">> := 0, <<"outputs.success">> := 1,
<<"outputs.failed.out_of_service">> := 0, <<"outputs.failed">> := 0,
<<"outputs.failed.unknown">> := 0 <<"outputs.failed.out_of_service">> := 0,
} <<"outputs.failed.unknown">> := 0
} = jsx:decode(Rule1), }
} = jsx:decode(Rule1),
%% we should receive a message on the "remote" broker, with specified topic %% we should receive a message on the "remote" broker, with specified topic
?assert( ?assert(
receive receive
@ -637,14 +778,19 @@ t_egress_mqtt_bridge_with_rules(_) ->
false false
after 100 -> after 100 ->
false false
end), end
),
%% verify the metrics of the bridge %% verify the metrics of the bridge
{ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []), {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
?assertMatch(#{ <<"metrics">> := ?metrics(2, 2, 0, _, _, _) ?assertMatch(
, <<"node_metrics">> := #{
[#{<<"node">> := _, <<"metrics">> := ?metrics(2, 2, 0, _, _, _)}] <<"metrics">> := ?metrics(2, 2, 0, _, _, _),
}, jsx:decode(BridgeStr)), <<"node_metrics">> :=
[#{<<"node">> := _, <<"metrics">> := ?metrics(2, 2, 0, _, _, _)}]
},
jsx:decode(BridgeStr)
),
{ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []), {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []),
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
@ -658,8 +804,9 @@ wait_for_resource_ready(InstId, 0) ->
ct:fail(wait_resource_timeout); ct:fail(wait_resource_timeout);
wait_for_resource_ready(InstId, Retry) -> wait_for_resource_ready(InstId, Retry) ->
case emqx_bridge:lookup(InstId) of case emqx_bridge:lookup(InstId) of
{ok, #{resource_data := #{status := connected}}} -> ok; {ok, #{resource_data := #{status := connected}}} ->
ok;
_ -> _ ->
timer:sleep(100), timer:sleep(100),
wait_for_resource_ready(InstId, Retry-1) wait_for_resource_ready(InstId, Retry - 1)
end. end.

61
apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl
View File

@ -65,20 +65,24 @@ t_lifecycle(_Config) ->
perform_lifecycle_check(PoolName, InitialConfig) -> perform_lifecycle_check(PoolName, InitialConfig) ->
{ok, #{config := CheckedConfig}} = {ok, #{config := CheckedConfig}} =
emqx_resource:check_config(?MONGO_RESOURCE_MOD, InitialConfig), emqx_resource:check_config(?MONGO_RESOURCE_MOD, InitialConfig),
{ok, #{state := #{poolname := ReturnedPoolName} = State, {ok, #{
status := InitialStatus}} state := #{poolname := ReturnedPoolName} = State,
= emqx_resource:create_local( status := InitialStatus
PoolName, }} =
?CONNECTOR_RESOURCE_GROUP, emqx_resource:create_local(
?MONGO_RESOURCE_MOD, PoolName,
CheckedConfig, ?CONNECTOR_RESOURCE_GROUP,
#{} ?MONGO_RESOURCE_MOD,
), CheckedConfig,
#{}
),
?assertEqual(InitialStatus, connected), ?assertEqual(InitialStatus, connected),
% Instance should match the state and status of the just started resource % Instance should match the state and status of the just started resource
{ok, ?CONNECTOR_RESOURCE_GROUP, #{state := State, {ok, ?CONNECTOR_RESOURCE_GROUP, #{
status := InitialStatus}} state := State,
= emqx_resource:get_instance(PoolName), status := InitialStatus
}} =
emqx_resource:get_instance(PoolName),
?assertEqual(ok, emqx_resource:health_check(PoolName)), ?assertEqual(ok, emqx_resource:health_check(PoolName)),
% % Perform query as further check that the resource is working as expected % % Perform query as further check that the resource is working as expected
?assertMatch([], emqx_resource:query(PoolName, test_query_find())), ?assertMatch([], emqx_resource:query(PoolName, test_query_find())),
@ -86,11 +90,13 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
?assertEqual(ok, emqx_resource:stop(PoolName)), ?assertEqual(ok, emqx_resource:stop(PoolName)),
% Resource will be listed still, but state will be changed and healthcheck will fail % Resource will be listed still, but state will be changed and healthcheck will fail
% as the worker no longer exists. % as the worker no longer exists.
{ok, ?CONNECTOR_RESOURCE_GROUP, #{state := State, {ok, ?CONNECTOR_RESOURCE_GROUP, #{
status := StoppedStatus}} state := State,
= emqx_resource:get_instance(PoolName), status := StoppedStatus
}} =
emqx_resource:get_instance(PoolName),
?assertEqual(StoppedStatus, disconnected), ?assertEqual(StoppedStatus, disconnected),
?assertEqual({error,health_check_failed}, emqx_resource:health_check(PoolName)), ?assertEqual({error, health_check_failed}, emqx_resource:health_check(PoolName)),
% Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself.
?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)),
% Can call stop/1 again on an already stopped instance % Can call stop/1 again on an already stopped instance
@ -99,8 +105,8 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
?assertEqual(ok, emqx_resource:restart(PoolName)), ?assertEqual(ok, emqx_resource:restart(PoolName)),
% async restart, need to wait resource % async restart, need to wait resource
timer:sleep(500), timer:sleep(500),
{ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} =
= emqx_resource:get_instance(PoolName), emqx_resource:get_instance(PoolName),
?assertEqual(ok, emqx_resource:health_check(PoolName)), ?assertEqual(ok, emqx_resource:health_check(PoolName)),
?assertMatch([], emqx_resource:query(PoolName, test_query_find())), ?assertMatch([], emqx_resource:query(PoolName, test_query_find())),
?assertMatch(undefined, emqx_resource:query(PoolName, test_query_find_one())), ?assertMatch(undefined, emqx_resource:query(PoolName, test_query_find_one())),
@ -115,12 +121,19 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
% %%------------------------------------------------------------------------------ % %%------------------------------------------------------------------------------
mongo_config() -> mongo_config() ->
RawConfig = list_to_binary(io_lib:format(""" RawConfig = list_to_binary(
mongo_type = single io_lib:format(
database = mqtt ""
pool_size = 8 "\n"
server = \"~s:~b\" " mongo_type = single\n"
""", [?MONGO_HOST, ?MONGO_DEFAULT_PORT])), " database = mqtt\n"
" pool_size = 8\n"
" server = \"~s:~b\"\n"
" "
"",
[?MONGO_HOST, ?MONGO_DEFAULT_PORT]
)
),
{ok, Config} = hocon:binary(RawConfig), {ok, Config} = hocon:binary(RawConfig),
#{<<"config">> => Config}. #{<<"config">> => Config}.

39
apps/emqx_connector/test/emqx_connector_mqtt_tests.erl
View File

@ -22,23 +22,36 @@
send_and_ack_test() -> send_and_ack_test() ->
%% delegate from gen_rpc to rpc for unit test %% delegate from gen_rpc to rpc for unit test
meck:new(emqtt, [passthrough, no_history]), meck:new(emqtt, [passthrough, no_history]),
meck:expect(emqtt, start_link, 1, meck:expect(
fun(_) -> emqtt,
{ok, spawn_link(fun() -> ok end)} start_link,
end), 1,
fun(_) ->
{ok, spawn_link(fun() -> ok end)}
end
),
meck:expect(emqtt, connect, 1, {ok, dummy}), meck:expect(emqtt, connect, 1, {ok, dummy}),
meck:expect(emqtt, stop, 1, meck:expect(
fun(Pid) -> Pid ! stop end), emqtt,
meck:expect(emqtt, publish, 2, stop,
fun(Client, Msg) -> 1,
Client ! {publish, Msg}, fun(Pid) -> Pid ! stop end
{ok, Msg} %% as packet id ),
end), meck:expect(
emqtt,
publish,
2,
fun(Client, Msg) ->
Client ! {publish, Msg},
%% as packet id
{ok, Msg}
end
),
try try
Max = 1, Max = 1,
Batch = lists:seq(1, Max), Batch = lists:seq(1, Max),
{ok, Conn} = emqx_connector_mqtt_mod:start(#{server => {{127,0,0,1}, 1883}}), {ok, Conn} = emqx_connector_mqtt_mod:start(#{server => {{127, 0, 0, 1}, 1883}}),
% %% return last packet id as batch reference % %% return last packet id as batch reference
{ok, _AckRef} = emqx_connector_mqtt_mod:send(Conn, Batch), {ok, _AckRef} = emqx_connector_mqtt_mod:send(Conn, Batch),
ok = emqx_connector_mqtt_mod:stop(Conn) ok = emqx_connector_mqtt_mod:stop(Conn)

16
apps/emqx_connector/test/emqx_connector_mqtt_worker_tests.erl
View File

@ -23,13 +23,13 @@
-define(BRIDGE_NAME, test). -define(BRIDGE_NAME, test).
-define(BRIDGE_REG_NAME, emqx_connector_mqtt_worker_test). -define(BRIDGE_REG_NAME, emqx_connector_mqtt_worker_test).
-define(WAIT(PATTERN, TIMEOUT), -define(WAIT(PATTERN, TIMEOUT),
receive receive
PATTERN -> PATTERN ->
ok ok
after after TIMEOUT ->
TIMEOUT -> error(timeout)
error(timeout) end
end). ).
-export([start/1, send/2, stop/1]). -export([start/1, send/2, stop/1]).
@ -125,7 +125,7 @@ manual_start_stop_test() ->
Ref = make_ref(), Ref = make_ref(),
TestPid = self(), TestPid = self(),
BridgeName = manual_start_stop, BridgeName = manual_start_stop,
Config0 = make_config(Ref, TestPid, {ok, #{client_pid => TestPid}}), Config0 = make_config(Ref, TestPid, {ok, #{client_pid => TestPid}}),
Config = Config0#{start_type := manual}, Config = Config0#{start_type := manual},
{ok, Pid} = emqx_connector_mqtt_worker:start_link(Config#{name => BridgeName}), {ok, Pid} = emqx_connector_mqtt_worker:start_link(Config#{name => BridgeName}),
%% call ensure_started again should yield the same result %% call ensure_started again should yield the same result

67
apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl
View File

@ -64,9 +64,11 @@ t_lifecycle(_Config) ->
perform_lifecycle_check(PoolName, InitialConfig) -> perform_lifecycle_check(PoolName, InitialConfig) ->
{ok, #{config := CheckedConfig}} = {ok, #{config := CheckedConfig}} =
emqx_resource:check_config(?MYSQL_RESOURCE_MOD, InitialConfig), emqx_resource:check_config(?MYSQL_RESOURCE_MOD, InitialConfig),
{ok, #{state := #{poolname := ReturnedPoolName} = State, {ok, #{
status := InitialStatus}} = emqx_resource:create_local( state := #{poolname := ReturnedPoolName} = State,
status := InitialStatus
}} = emqx_resource:create_local(
PoolName, PoolName,
?CONNECTOR_RESOURCE_GROUP, ?CONNECTOR_RESOURCE_GROUP,
?MYSQL_RESOURCE_MOD, ?MYSQL_RESOURCE_MOD,
@ -75,23 +77,32 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
), ),
?assertEqual(InitialStatus, connected), ?assertEqual(InitialStatus, connected),
% Instance should match the state and status of the just started resource % Instance should match the state and status of the just started resource
{ok, ?CONNECTOR_RESOURCE_GROUP, #{state := State, {ok, ?CONNECTOR_RESOURCE_GROUP, #{
status := InitialStatus}} state := State,
= emqx_resource:get_instance(PoolName), status := InitialStatus
}} =
emqx_resource:get_instance(PoolName),
?assertEqual(ok, emqx_resource:health_check(PoolName)), ?assertEqual(ok, emqx_resource:health_check(PoolName)),
% % Perform query as further check that the resource is working as expected % % Perform query as further check that the resource is working as expected
?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, test_query_no_params())), ?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, test_query_no_params())),
?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, test_query_with_params())), ?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, test_query_with_params())),
?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, ?assertMatch(
test_query_with_params_and_timeout())), {ok, _, [[1]]},
emqx_resource:query(
PoolName,
test_query_with_params_and_timeout()
)
),
?assertEqual(ok, emqx_resource:stop(PoolName)), ?assertEqual(ok, emqx_resource:stop(PoolName)),
% Resource will be listed still, but state will be changed and healthcheck will fail % Resource will be listed still, but state will be changed and healthcheck will fail
% as the worker no longer exists. % as the worker no longer exists.
{ok, ?CONNECTOR_RESOURCE_GROUP, #{state := State, {ok, ?CONNECTOR_RESOURCE_GROUP, #{
status := StoppedStatus}} state := State,
= emqx_resource:get_instance(PoolName), status := StoppedStatus
}} =
emqx_resource:get_instance(PoolName),
?assertEqual(StoppedStatus, disconnected), ?assertEqual(StoppedStatus, disconnected),
?assertEqual({error,health_check_failed}, emqx_resource:health_check(PoolName)), ?assertEqual({error, health_check_failed}, emqx_resource:health_check(PoolName)),
% Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself.
?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)),
% Can call stop/1 again on an already stopped instance % Can call stop/1 again on an already stopped instance
@ -105,8 +116,13 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
?assertEqual(ok, emqx_resource:health_check(PoolName)), ?assertEqual(ok, emqx_resource:health_check(PoolName)),
?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, test_query_no_params())), ?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, test_query_no_params())),
?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, test_query_with_params())), ?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, test_query_with_params())),
?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, ?assertMatch(
test_query_with_params_and_timeout())), {ok, _, [[1]]},
emqx_resource:query(
PoolName,
test_query_with_params_and_timeout()
)
),
% Stop and remove the resource in one go. % Stop and remove the resource in one go.
?assertEqual(ok, emqx_resource:remove_local(PoolName)), ?assertEqual(ok, emqx_resource:remove_local(PoolName)),
?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)),
@ -118,14 +134,21 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
% %%------------------------------------------------------------------------------ % %%------------------------------------------------------------------------------
mysql_config() -> mysql_config() ->
RawConfig = list_to_binary(io_lib:format(""" RawConfig = list_to_binary(
auto_reconnect = true io_lib:format(
database = mqtt ""
username= root "\n"
password = public " auto_reconnect = true\n"
pool_size = 8 " database = mqtt\n"
server = \"~s:~b\" " username= root\n"
""", [?MYSQL_HOST, ?MYSQL_DEFAULT_PORT])), " password = public\n"
" pool_size = 8\n"
" server = \"~s:~b\"\n"
" "
"",
[?MYSQL_HOST, ?MYSQL_DEFAULT_PORT]
)
),
{ok, Config} = hocon:binary(RawConfig), {ok, Config} = hocon:binary(RawConfig),
#{<<"config">> => Config}. #{<<"config">> => Config}.

65
apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl
View File

@ -65,20 +65,24 @@ t_lifecycle(_Config) ->
perform_lifecycle_check(PoolName, InitialConfig) -> perform_lifecycle_check(PoolName, InitialConfig) ->
{ok, #{config := CheckedConfig}} = {ok, #{config := CheckedConfig}} =
emqx_resource:check_config(?PGSQL_RESOURCE_MOD, InitialConfig), emqx_resource:check_config(?PGSQL_RESOURCE_MOD, InitialConfig),
{ok, #{state := #{poolname := ReturnedPoolName} = State, {ok, #{
status := InitialStatus}} state := #{poolname := ReturnedPoolName} = State,
= emqx_resource:create_local( status := InitialStatus
PoolName, }} =
?CONNECTOR_RESOURCE_GROUP, emqx_resource:create_local(
?PGSQL_RESOURCE_MOD, PoolName,
CheckedConfig, ?CONNECTOR_RESOURCE_GROUP,
#{} ?PGSQL_RESOURCE_MOD,
), CheckedConfig,
#{}
),
?assertEqual(InitialStatus, connected), ?assertEqual(InitialStatus, connected),
% Instance should match the state and status of the just started resource % Instance should match the state and status of the just started resource
{ok, ?CONNECTOR_RESOURCE_GROUP, #{state := State, {ok, ?CONNECTOR_RESOURCE_GROUP, #{
status := InitialStatus}} state := State,
= emqx_resource:get_instance(PoolName), status := InitialStatus
}} =
emqx_resource:get_instance(PoolName),
?assertEqual(ok, emqx_resource:health_check(PoolName)), ?assertEqual(ok, emqx_resource:health_check(PoolName)),
% % Perform query as further check that the resource is working as expected % % Perform query as further check that the resource is working as expected
?assertMatch({ok, _, [{1}]}, emqx_resource:query(PoolName, test_query_no_params())), ?assertMatch({ok, _, [{1}]}, emqx_resource:query(PoolName, test_query_no_params())),
@ -86,11 +90,13 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
?assertEqual(ok, emqx_resource:stop(PoolName)), ?assertEqual(ok, emqx_resource:stop(PoolName)),
% Resource will be listed still, but state will be changed and healthcheck will fail % Resource will be listed still, but state will be changed and healthcheck will fail
% as the worker no longer exists. % as the worker no longer exists.
{ok, ?CONNECTOR_RESOURCE_GROUP, #{state := State, {ok, ?CONNECTOR_RESOURCE_GROUP, #{
status := StoppedStatus}} state := State,
= emqx_resource:get_instance(PoolName), status := StoppedStatus
}} =
emqx_resource:get_instance(PoolName),
?assertEqual(StoppedStatus, disconnected), ?assertEqual(StoppedStatus, disconnected),
?assertEqual({error,health_check_failed}, emqx_resource:health_check(PoolName)), ?assertEqual({error, health_check_failed}, emqx_resource:health_check(PoolName)),
% Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself.
?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)),
% Can call stop/1 again on an already stopped instance % Can call stop/1 again on an already stopped instance
@ -99,8 +105,8 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
?assertEqual(ok, emqx_resource:restart(PoolName)), ?assertEqual(ok, emqx_resource:restart(PoolName)),
% async restart, need to wait resource % async restart, need to wait resource
timer:sleep(500), timer:sleep(500),
{ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} =
= emqx_resource:get_instance(PoolName), emqx_resource:get_instance(PoolName),
?assertEqual(ok, emqx_resource:health_check(PoolName)), ?assertEqual(ok, emqx_resource:health_check(PoolName)),
?assertMatch({ok, _, [{1}]}, emqx_resource:query(PoolName, test_query_no_params())), ?assertMatch({ok, _, [{1}]}, emqx_resource:query(PoolName, test_query_no_params())),
?assertMatch({ok, _, [{1}]}, emqx_resource:query(PoolName, test_query_with_params())), ?assertMatch({ok, _, [{1}]}, emqx_resource:query(PoolName, test_query_with_params())),
@ -115,14 +121,21 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
% %%------------------------------------------------------------------------------ % %%------------------------------------------------------------------------------
pgsql_config() -> pgsql_config() ->
RawConfig = list_to_binary(io_lib:format(""" RawConfig = list_to_binary(
auto_reconnect = true io_lib:format(
database = mqtt ""
username= root "\n"
password = public " auto_reconnect = true\n"
pool_size = 8 " database = mqtt\n"
server = \"~s:~b\" " username= root\n"
""", [?PGSQL_HOST, ?PGSQL_DEFAULT_PORT])), " password = public\n"
" pool_size = 8\n"
" server = \"~s:~b\"\n"
" "
"",
[?PGSQL_HOST, ?PGSQL_DEFAULT_PORT]
)
),
{ok, Config} = hocon:binary(RawConfig), {ok, Config} = hocon:binary(RawConfig),
#{<<"config">> => Config}. #{<<"config">> => Config}.

51
apps/emqx_connector/test/emqx_connector_redis_SUITE.erl
View File

@ -80,8 +80,10 @@ t_sentinel_lifecycle(_Config) ->
perform_lifecycle_check(PoolName, InitialConfig, RedisCommand) -> perform_lifecycle_check(PoolName, InitialConfig, RedisCommand) ->
{ok, #{config := CheckedConfig}} = {ok, #{config := CheckedConfig}} =
emqx_resource:check_config(?REDIS_RESOURCE_MOD, InitialConfig), emqx_resource:check_config(?REDIS_RESOURCE_MOD, InitialConfig),
{ok, #{state := #{poolname := ReturnedPoolName} = State, {ok, #{
status := InitialStatus}} = emqx_resource:create_local( state := #{poolname := ReturnedPoolName} = State,
status := InitialStatus
}} = emqx_resource:create_local(
PoolName, PoolName,
?CONNECTOR_RESOURCE_GROUP, ?CONNECTOR_RESOURCE_GROUP,
?REDIS_RESOURCE_MOD, ?REDIS_RESOURCE_MOD,
@ -90,20 +92,24 @@ perform_lifecycle_check(PoolName, InitialConfig, RedisCommand) ->
), ),
?assertEqual(InitialStatus, connected), ?assertEqual(InitialStatus, connected),
% Instance should match the state and status of the just started resource % Instance should match the state and status of the just started resource
{ok, ?CONNECTOR_RESOURCE_GROUP, #{state := State, {ok, ?CONNECTOR_RESOURCE_GROUP, #{
status := InitialStatus}} state := State,
= emqx_resource:get_instance(PoolName), status := InitialStatus
}} =
emqx_resource:get_instance(PoolName),
?assertEqual(ok, emqx_resource:health_check(PoolName)), ?assertEqual(ok, emqx_resource:health_check(PoolName)),
% Perform query as further check that the resource is working as expected % Perform query as further check that the resource is working as expected
?assertEqual({ok, <<"PONG">>}, emqx_resource:query(PoolName, {cmd, RedisCommand})), ?assertEqual({ok, <<"PONG">>}, emqx_resource:query(PoolName, {cmd, RedisCommand})),
?assertEqual(ok, emqx_resource:stop(PoolName)), ?assertEqual(ok, emqx_resource:stop(PoolName)),
% Resource will be listed still, but state will be changed and healthcheck will fail % Resource will be listed still, but state will be changed and healthcheck will fail
% as the worker no longer exists. % as the worker no longer exists.
{ok, ?CONNECTOR_RESOURCE_GROUP, #{state := State, {ok, ?CONNECTOR_RESOURCE_GROUP, #{
status := StoppedStatus}} state := State,
= emqx_resource:get_instance(PoolName), status := StoppedStatus
}} =
emqx_resource:get_instance(PoolName),
?assertEqual(StoppedStatus, disconnected), ?assertEqual(StoppedStatus, disconnected),
?assertEqual({error,health_check_failed}, emqx_resource:health_check(PoolName)), ?assertEqual({error, health_check_failed}, emqx_resource:health_check(PoolName)),
% Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself.
?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)),
% Can call stop/1 again on an already stopped instance % Can call stop/1 again on an already stopped instance
@ -112,8 +118,8 @@ perform_lifecycle_check(PoolName, InitialConfig, RedisCommand) ->
?assertEqual(ok, emqx_resource:restart(PoolName)), ?assertEqual(ok, emqx_resource:restart(PoolName)),
% async restart, need to wait resource % async restart, need to wait resource
timer:sleep(500), timer:sleep(500),
{ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} =
= emqx_resource:get_instance(PoolName), emqx_resource:get_instance(PoolName),
?assertEqual(ok, emqx_resource:health_check(PoolName)), ?assertEqual(ok, emqx_resource:health_check(PoolName)),
?assertEqual({ok, <<"PONG">>}, emqx_resource:query(PoolName, {cmd, RedisCommand})), ?assertEqual({ok, <<"PONG">>}, emqx_resource:query(PoolName, {cmd, RedisCommand})),
% Stop and remove the resource in one go. % Stop and remove the resource in one go.
@ -136,14 +142,21 @@ redis_config_sentinel() ->
redis_config_base("sentinel", "servers"). redis_config_base("sentinel", "servers").
redis_config_base(Type, ServerKey) -> redis_config_base(Type, ServerKey) ->
RawConfig = list_to_binary(io_lib:format(""" RawConfig = list_to_binary(
auto_reconnect = true io_lib:format(
database = 1 ""
pool_size = 8 "\n"
redis_type = ~s " auto_reconnect = true\n"
password = public " database = 1\n"
~s = \"~s:~b\" " pool_size = 8\n"
""", [Type, ServerKey, ?REDIS_HOST, ?REDIS_PORT])), " redis_type = ~s\n"
" password = public\n"
" ~s = \"~s:~b\"\n"
" "
"",
[Type, ServerKey, ?REDIS_HOST, ?REDIS_PORT]
)
),
{ok, Config} = hocon:binary(RawConfig), {ok, Config} = hocon:binary(RawConfig),
#{<<"config">> => Config}. #{<<"config">> => Config}.

17
apps/emqx_connector/test/emqx_connector_test_helpers.erl
View File

@ -19,10 +19,11 @@
-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct.hrl").
-include_lib("eunit/include/eunit.hrl"). -include_lib("eunit/include/eunit.hrl").
-export([ check_fields/1 -export([
, start_apps/1 check_fields/1,
, stop_apps/1 start_apps/1,
]). stop_apps/1
]).
check_fields({FieldName, FieldValue}) -> check_fields({FieldName, FieldValue}) ->
?assert(is_atom(FieldName)), ?assert(is_atom(FieldName)),
@ -30,10 +31,10 @@ check_fields({FieldName, FieldValue}) ->
is_map(FieldValue) -> is_map(FieldValue) ->
ct:pal("~p~n", [{FieldName, FieldValue}]), ct:pal("~p~n", [{FieldName, FieldValue}]),
?assert( ?assert(
(maps:is_key(type, FieldValue) (maps:is_key(type, FieldValue) andalso
andalso maps:is_key(default, FieldValue)) maps:is_key(default, FieldValue)) orelse
orelse ((maps:is_key(required, FieldValue) (maps:is_key(required, FieldValue) andalso
andalso maps:get(required, FieldValue) =:= false)) maps:get(required, FieldValue) =:= false)
); );
true -> true ->
?assert(is_function(FieldValue)) ?assert(is_function(FieldValue))

5
apps/emqx_plugins/rebar.config
View File

@ -1,4 +1,5 @@
%% -*- mode: erlang -*- %% -*- mode: erlang -*-
{deps, [ {emqx, {path, "../emqx"}} {deps, [{emqx, {path, "../emqx"}}]}.
]}.
{project_plugins, [erlfmt]}.

16
apps/emqx_plugins/src/emqx_plugins.app.src
View File

@ -1,9 +1,9 @@
%% -*- mode: erlang -*- %% -*- mode: erlang -*-
{application, emqx_plugins, {application, emqx_plugins, [
[{description, "EMQX Plugin Management"}, {description, "EMQX Plugin Management"},
{vsn, "0.1.0"}, {vsn, "0.1.0"},
{modules, []}, {modules, []},
{mod, {emqx_plugins_app,[]}}, {mod, {emqx_plugins_app, []}},
{applications, [kernel,stdlib,emqx]}, {applications, [kernel, stdlib, emqx]},
{env, []} {env, []}
]}. ]}.

442
apps/emqx_plugins/src/emqx_plugins.erl
View File

@ -19,35 +19,37 @@
-include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/logger.hrl").
-export([ ensure_installed/1 -export([
, ensure_uninstalled/1 ensure_installed/1,
, ensure_enabled/1 ensure_uninstalled/1,
, ensure_enabled/2 ensure_enabled/1,
, ensure_disabled/1 ensure_enabled/2,
, purge/1 ensure_disabled/1,
, delete_package/1 purge/1,
]). delete_package/1
]).
-export([ ensure_started/0 -export([
, ensure_started/1 ensure_started/0,
, ensure_stopped/0 ensure_started/1,
, ensure_stopped/1 ensure_stopped/0,
, restart/1 ensure_stopped/1,
, list/0 restart/1,
, describe/1 list/0,
, parse_name_vsn/1 describe/1,
]). parse_name_vsn/1
]).
-export([ get_config/2 -export([
, put_config/2 get_config/2,
]). put_config/2
]).
%% internal %% internal
-export([ do_ensure_started/1 -export([do_ensure_started/1]).
]).
-export([ -export([
install_dir/0 install_dir/0
]). ]).
-ifdef(TEST). -ifdef(TEST).
-compile(export_all). -compile(export_all).
@ -58,8 +60,10 @@
-include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/logger.hrl").
-include("emqx_plugins.hrl"). -include("emqx_plugins.hrl").
-type name_vsn() :: binary() | string(). %% "my_plugin-0.1.0" %% "my_plugin-0.1.0"
-type plugin() :: map(). %% the parse result of the JSON info file -type name_vsn() :: binary() | string().
%% the parse result of the JSON info file
-type plugin() :: map().
-type position() :: no_move | front | rear | {before, name_vsn()} | {behind, name_vsn()}. -type position() :: no_move | front | rear | {before, name_vsn()} | {behind, name_vsn()}.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
@ -86,22 +90,25 @@ do_ensure_installed(NameVsn) ->
case erl_tar:extract(TarGz, [{cwd, install_dir()}, compressed]) of case erl_tar:extract(TarGz, [{cwd, install_dir()}, compressed]) of
ok -> ok ->
case read_plugin(NameVsn, #{}) of case read_plugin(NameVsn, #{}) of
{ok, _} -> ok; {ok, _} ->
ok;
{error, Reason} -> {error, Reason} ->
?SLOG(warning, Reason#{msg => "failed_to_read_after_install"}), ?SLOG(warning, Reason#{msg => "failed_to_read_after_install"}),
_ = ensure_uninstalled(NameVsn), _ = ensure_uninstalled(NameVsn),
{error, Reason} {error, Reason}
end; end;
{error, {_, enoent}} -> {error, {_, enoent}} ->
{error, #{ reason => "failed_to_extract_plugin_package" {error, #{
, path => TarGz reason => "failed_to_extract_plugin_package",
, return => not_found path => TarGz,
}}; return => not_found
}};
{error, Reason} -> {error, Reason} ->
{error, #{ reason => "bad_plugin_package" {error, #{
, path => TarGz reason => "bad_plugin_package",
, return => Reason path => TarGz,
}} return => Reason
}}
end. end.
%% @doc Ensure files and directories for the given plugin are delete. %% @doc Ensure files and directories for the given plugin are delete.
@ -110,13 +117,15 @@ do_ensure_installed(NameVsn) ->
ensure_uninstalled(NameVsn) -> ensure_uninstalled(NameVsn) ->
case read_plugin(NameVsn, #{}) of case read_plugin(NameVsn, #{}) of
{ok, #{running_status := RunningSt}} when RunningSt =/= stopped -> {ok, #{running_status := RunningSt}} when RunningSt =/= stopped ->
{error, #{reason => "bad_plugin_running_status", {error, #{
hint => "stop_the_plugin_first" reason => "bad_plugin_running_status",
}}; hint => "stop_the_plugin_first"
}};
{ok, #{config_status := enabled}} -> {ok, #{config_status := enabled}} ->
{error, #{reason => "bad_plugin_config_status", {error, #{
hint => "disable_the_plugin_first" reason => "bad_plugin_config_status",
}}; hint => "disable_the_plugin_first"
}};
_ -> _ ->
purge(NameVsn) purge(NameVsn)
end. end.
@ -141,9 +150,10 @@ ensure_state(NameVsn, Position, State) when is_binary(NameVsn) ->
ensure_state(NameVsn, Position, State) -> ensure_state(NameVsn, Position, State) ->
case read_plugin(NameVsn, #{}) of case read_plugin(NameVsn, #{}) of
{ok, _} -> {ok, _} ->
Item = #{ name_vsn => NameVsn Item = #{
, enable => State name_vsn => NameVsn,
}, enable => State
},
tryit("ensure_state", fun() -> ensure_configured(Item, Position) end); tryit("ensure_state", fun() -> ensure_configured(Item, Position) end);
{error, Reason} -> {error, Reason} ->
{error, Reason} {error, Reason}
@ -175,18 +185,19 @@ add_new_configured(Configured, {Action, NameVsn}, Item) ->
SplitFun = fun(#{name_vsn := Nv}) -> bin(Nv) =/= bin(NameVsn) end, SplitFun = fun(#{name_vsn := Nv}) -> bin(Nv) =/= bin(NameVsn) end,
{Front, Rear} = lists:splitwith(SplitFun, Configured), {Front, Rear} = lists:splitwith(SplitFun, Configured),
Rear =:= [] andalso Rear =:= [] andalso
throw(#{error => "position_anchor_plugin_not_configured", throw(#{
hint => "maybe_install_and_configure", error => "position_anchor_plugin_not_configured",
name_vsn => NameVsn hint => "maybe_install_and_configure",
}), name_vsn => NameVsn
}),
case Action of case Action of
before -> Front ++ [Item | Rear]; before ->
Front ++ [Item | Rear];
behind -> behind ->
[Anchor | Rear0] = Rear, [Anchor | Rear0] = Rear,
Front ++ [Anchor, Item | Rear0] Front ++ [Anchor, Item | Rear0]
end. end.
%% @doc Delete the package file. %% @doc Delete the package file.
-spec delete_package(name_vsn()) -> ok. -spec delete_package(name_vsn()) -> ok.
delete_package(NameVsn) -> delete_package(NameVsn) ->
@ -198,9 +209,11 @@ delete_package(NameVsn) ->
{error, enoent} -> {error, enoent} ->
ok; ok;
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => "failed_to_delete_package_file", ?SLOG(error, #{
path => File, msg => "failed_to_delete_package_file",
reason => Reason}), path => File,
reason => Reason
}),
{error, Reason} {error, Reason}
end. end.
@ -219,9 +232,11 @@ purge(NameVsn) ->
{error, enoent} -> {error, enoent} ->
ok; ok;
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => "failed_to_purge_plugin_dir", ?SLOG(error, #{
dir => Dir, msg => "failed_to_purge_plugin_dir",
reason => Reason}), dir => Dir,
reason => Reason
}),
{error, Reason} {error, Reason}
end. end.
@ -235,10 +250,13 @@ ensure_started() ->
-spec ensure_started(name_vsn()) -> ok | {error, term()}. -spec ensure_started(name_vsn()) -> ok | {error, term()}.
ensure_started(NameVsn) -> ensure_started(NameVsn) ->
case do_ensure_started(NameVsn) of case do_ensure_started(NameVsn) of
ok -> ok; ok ->
ok;
{error, Reason} -> {error, Reason} ->
?SLOG(alert, #{msg => "failed_to_start_plugin", ?SLOG(alert, #{
reason => Reason}), msg => "failed_to_start_plugin",
reason => Reason
}),
{error, Reason} {error, Reason}
end. end.
@ -250,11 +268,13 @@ ensure_stopped() ->
%% @doc Stop a plugin from Management API or CLI. %% @doc Stop a plugin from Management API or CLI.
-spec ensure_stopped(name_vsn()) -> ok | {error, term()}. -spec ensure_stopped(name_vsn()) -> ok | {error, term()}.
ensure_stopped(NameVsn) -> ensure_stopped(NameVsn) ->
tryit("stop_plugin", tryit(
fun() -> "stop_plugin",
Plugin = do_read_plugin(NameVsn), fun() ->
ensure_apps_stopped(Plugin) Plugin = do_read_plugin(NameVsn),
end). ensure_apps_stopped(Plugin)
end
).
%% @doc Stop and then start the plugin. %% @doc Stop and then start the plugin.
restart(NameVsn) -> restart(NameVsn) ->
@ -269,39 +289,45 @@ restart(NameVsn) ->
list() -> list() ->
Pattern = filename:join([install_dir(), "*", "release.json"]), Pattern = filename:join([install_dir(), "*", "release.json"]),
All = lists:filtermap( All = lists:filtermap(
fun(JsonFile) -> fun(JsonFile) ->
case read_plugin({file, JsonFile}, #{}) of case read_plugin({file, JsonFile}, #{}) of
{ok, Info} -> {ok, Info} ->
{true, Info}; {true, Info};
{error, Reason} -> {error, Reason} ->
?SLOG(warning, Reason), ?SLOG(warning, Reason),
false false
end end
end, filelib:wildcard(Pattern)), end,
filelib:wildcard(Pattern)
),
list(configured(), All). list(configured(), All).
%% Make sure configured ones are ordered in front. %% Make sure configured ones are ordered in front.
list([], All) -> All; list([], All) ->
All;
list([#{name_vsn := NameVsn} | Rest], All) -> list([#{name_vsn := NameVsn} | Rest], All) ->
SplitF = fun(#{<<"name">> := Name, <<"rel_vsn">> := Vsn}) -> SplitF = fun(#{<<"name">> := Name, <<"rel_vsn">> := Vsn}) ->
bin([Name, "-", Vsn]) =/= bin(NameVsn) bin([Name, "-", Vsn]) =/= bin(NameVsn)
end, end,
case lists:splitwith(SplitF, All) of case lists:splitwith(SplitF, All) of
{_, []} -> {_, []} ->
?SLOG(warning, #{msg => "configured_plugin_not_installed", ?SLOG(warning, #{
name_vsn => NameVsn msg => "configured_plugin_not_installed",
}), name_vsn => NameVsn
}),
list(Rest, All); list(Rest, All);
{Front, [I | Rear]} -> {Front, [I | Rear]} ->
[I | list(Rest, Front ++ Rear)] [I | list(Rest, Front ++ Rear)]
end. end.
do_ensure_started(NameVsn) -> do_ensure_started(NameVsn) ->
tryit("start_plugins", tryit(
fun() -> "start_plugins",
Plugin = do_read_plugin(NameVsn), fun() ->
ok = load_code_start_apps(NameVsn, Plugin) Plugin = do_read_plugin(NameVsn),
end). ok = load_code_start_apps(NameVsn, Plugin)
end
).
%% try the function, catch 'throw' exceptions as normal 'error' return %% try the function, catch 'throw' exceptions as normal 'error' return
%% other exceptions with stacktrace returned. %% other exceptions with stacktrace returned.
@ -309,25 +335,28 @@ tryit(WhichOp, F) ->
try try
F() F()
catch catch
throw : Reason -> throw:Reason ->
%% thrown exceptions are known errors %% thrown exceptions are known errors
%% translate to a return value without stacktrace %% translate to a return value without stacktrace
{error, Reason}; {error, Reason};
error : Reason : Stacktrace -> error:Reason:Stacktrace ->
%% unexpected errors, log stacktrace %% unexpected errors, log stacktrace
?SLOG(warning, #{ msg => "plugin_op_failed" ?SLOG(warning, #{
, which_op => WhichOp msg => "plugin_op_failed",
, exception => Reason which_op => WhichOp,
, stacktrace => Stacktrace exception => Reason,
}), stacktrace => Stacktrace
}),
{error, {failed, WhichOp}} {error, {failed, WhichOp}}
end. end.
%% read plugin info from the JSON file %% read plugin info from the JSON file
%% returns {ok, Info} or {error, Reason} %% returns {ok, Info} or {error, Reason}
read_plugin(NameVsn, Options) -> read_plugin(NameVsn, Options) ->
tryit("read_plugin_info", tryit(
fun() -> {ok, do_read_plugin(NameVsn, Options)} end). "read_plugin_info",
fun() -> {ok, do_read_plugin(NameVsn, Options)} end
).
do_read_plugin(Plugin) -> do_read_plugin(Plugin, #{}). do_read_plugin(Plugin) -> do_read_plugin(Plugin, #{}).
@ -339,10 +368,11 @@ do_read_plugin({file, InfoFile}, Options) ->
Info1 = plugins_readme(NameVsn, Options, Info0), Info1 = plugins_readme(NameVsn, Options, Info0),
plugin_status(NameVsn, Info1); plugin_status(NameVsn, Info1);
{error, Reason} -> {error, Reason} ->
throw(#{error => "bad_info_file", throw(#{
path => InfoFile, error => "bad_info_file",
return => Reason path => InfoFile,
}) return => Reason
})
end; end;
do_read_plugin(NameVsn, Options) -> do_read_plugin(NameVsn, Options) ->
do_read_plugin({file, info_file(NameVsn)}, Options). do_read_plugin({file, info_file(NameVsn)}, Options).
@ -352,7 +382,8 @@ plugins_readme(NameVsn, #{fill_readme := true}, Info) ->
{ok, Bin} -> Info#{readme => Bin}; {ok, Bin} -> Info#{readme => Bin};
_ -> Info#{readme => <<>>} _ -> Info#{readme => <<>>}
end; end;
plugins_readme(_NameVsn, _Options, Info) -> Info. plugins_readme(_NameVsn, _Options, Info) ->
Info.
plugin_status(NameVsn, Info) -> plugin_status(NameVsn, Info) ->
{AppName, _AppVsn} = parse_name_vsn(NameVsn), {AppName, _AppVsn} = parse_name_vsn(NameVsn),
@ -368,74 +399,91 @@ plugin_status(NameVsn, Info) ->
end, end,
Configured = lists:filtermap( Configured = lists:filtermap(
fun(#{name_vsn := Nv, enable := St}) -> fun(#{name_vsn := Nv, enable := St}) ->
case bin(Nv) =:= bin(NameVsn) of case bin(Nv) =:= bin(NameVsn) of
true -> {true, St}; true -> {true, St};
false -> false false -> false
end end
end, configured()), end,
ConfSt = case Configured of configured()
[] -> not_configured; ),
[true] -> enabled; ConfSt =
[false] -> disabled case Configured of
end, [] -> not_configured;
Info#{ running_status => RunningSt [true] -> enabled;
, config_status => ConfSt [false] -> disabled
end,
Info#{
running_status => RunningSt,
config_status => ConfSt
}. }.
bin(A) when is_atom(A) -> atom_to_binary(A, utf8); bin(A) when is_atom(A) -> atom_to_binary(A, utf8);
bin(L) when is_list(L) -> unicode:characters_to_binary(L, utf8); bin(L) when is_list(L) -> unicode:characters_to_binary(L, utf8);
bin(B) when is_binary(B) -> B. bin(B) when is_binary(B) -> B.
check_plugin(#{ <<"name">> := Name check_plugin(
, <<"rel_vsn">> := Vsn #{
, <<"rel_apps">> := Apps <<"name">> := Name,
, <<"description">> := _ <<"rel_vsn">> := Vsn,
} = Info, NameVsn, File) -> <<"rel_apps">> := Apps,
<<"description">> := _
} = Info,
NameVsn,
File
) ->
case bin(NameVsn) =:= bin([Name, "-", Vsn]) of case bin(NameVsn) =:= bin([Name, "-", Vsn]) of
true -> true ->
try try
[_ | _ ] = Apps, %% assert %% assert
[_ | _] = Apps,
%% validate if the list is all <app>-<vsn> strings %% validate if the list is all <app>-<vsn> strings
lists:foreach(fun parse_name_vsn/1, Apps) lists:foreach(fun parse_name_vsn/1, Apps)
catch catch
_ : _ -> _:_ ->
throw(#{ error => "bad_rel_apps" throw(#{
, rel_apps => Apps error => "bad_rel_apps",
, hint => "A non-empty string list of app_name-app_vsn format" rel_apps => Apps,
}) hint => "A non-empty string list of app_name-app_vsn format"
})
end, end,
Info; Info;
false -> false ->
throw(#{ error => "name_vsn_mismatch" throw(#{
, name_vsn => NameVsn error => "name_vsn_mismatch",
, path => File name_vsn => NameVsn,
, name => Name path => File,
, rel_vsn => Vsn name => Name,
}) rel_vsn => Vsn
})
end; end;
check_plugin(_What, NameVsn, File) -> check_plugin(_What, NameVsn, File) ->
throw(#{ error => "bad_info_file_content" throw(#{
, mandatory_fields => [rel_vsn, name, rel_apps, description] error => "bad_info_file_content",
, name_vsn => NameVsn mandatory_fields => [rel_vsn, name, rel_apps, description],
, path => File name_vsn => NameVsn,
}). path => File
}).
load_code_start_apps(RelNameVsn, #{<<"rel_apps">> := Apps}) -> load_code_start_apps(RelNameVsn, #{<<"rel_apps">> := Apps}) ->
LibDir = filename:join([install_dir(), RelNameVsn]), LibDir = filename:join([install_dir(), RelNameVsn]),
RunningApps = running_apps(), RunningApps = running_apps(),
%% load plugin apps and beam code %% load plugin apps and beam code
AppNames = AppNames =
lists:map(fun(AppNameVsn) -> lists:map(
{AppName, AppVsn} = parse_name_vsn(AppNameVsn), fun(AppNameVsn) ->
EbinDir = filename:join([LibDir, AppNameVsn, "ebin"]), {AppName, AppVsn} = parse_name_vsn(AppNameVsn),
ok = load_plugin_app(AppName, AppVsn, EbinDir, RunningApps), EbinDir = filename:join([LibDir, AppNameVsn, "ebin"]),
AppName ok = load_plugin_app(AppName, AppVsn, EbinDir, RunningApps),
end, Apps), AppName
end,
Apps
),
lists:foreach(fun start_app/1, AppNames). lists:foreach(fun start_app/1, AppNames).
load_plugin_app(AppName, AppVsn, Ebin, RunningApps) -> load_plugin_app(AppName, AppVsn, Ebin, RunningApps) ->
case lists:keyfind(AppName, 1, RunningApps) of case lists:keyfind(AppName, 1, RunningApps) of
false -> do_load_plugin_app(AppName, Ebin); false ->
do_load_plugin_app(AppName, Ebin);
{_, Vsn} -> {_, Vsn} ->
case bin(Vsn) =:= bin(AppVsn) of case bin(Vsn) =:= bin(AppVsn) of
true -> true ->
@ -443,10 +491,12 @@ load_plugin_app(AppName, AppVsn, Ebin, RunningApps) ->
ok; ok;
false -> false ->
%% running but a different version %% running but a different version
?SLOG(warning, #{msg => "plugin_app_already_running", name => AppName, ?SLOG(warning, #{
running_vsn => Vsn, msg => "plugin_app_already_running",
loading_vsn => AppVsn name => AppName,
}) running_vsn => Vsn,
loading_vsn => AppVsn
})
end end
end. end.
@ -457,21 +507,31 @@ do_load_plugin_app(AppName, Ebin) ->
Modules = filelib:wildcard(filename:join([Ebin, "*.beam"])), Modules = filelib:wildcard(filename:join([Ebin, "*.beam"])),
lists:foreach( lists:foreach(
fun(BeamFile) -> fun(BeamFile) ->
Module = list_to_atom(filename:basename(BeamFile, ".beam")), Module = list_to_atom(filename:basename(BeamFile, ".beam")),
case code:load_file(Module) of case code:load_file(Module) of
{module, _} -> ok; {module, _} ->
{error, Reason} -> throw(#{error => "failed_to_load_plugin_beam", ok;
path => BeamFile, {error, Reason} ->
reason => Reason throw(#{
}) error => "failed_to_load_plugin_beam",
end path => BeamFile,
end, Modules), reason => Reason
})
end
end,
Modules
),
case application:load(AppName) of case application:load(AppName) of
ok -> ok; ok ->
{error, {already_loaded, _}} -> ok; ok;
{error, Reason} -> throw(#{error => "failed_to_load_plugin_app", {error, {already_loaded, _}} ->
name => AppName, ok;
reason => Reason}) {error, Reason} ->
throw(#{
error => "failed_to_load_plugin_app",
name => AppName,
reason => Reason
})
end. end.
start_app(App) -> start_app(App) ->
@ -484,11 +544,12 @@ start_app(App) ->
?SLOG(debug, #{msg => "started_plugin_app", app => App}), ?SLOG(debug, #{msg => "started_plugin_app", app => App}),
ok; ok;
{error, {ErrApp, Reason}} -> {error, {ErrApp, Reason}} ->
throw(#{error => "failed_to_start_plugin_app", throw(#{
app => App, error => "failed_to_start_plugin_app",
err_app => ErrApp, app => App,
reason => Reason err_app => ErrApp,
}) reason => Reason
})
end. end.
%% Stop all apps installed by the plugin package, %% Stop all apps installed by the plugin package,
@ -496,18 +557,22 @@ start_app(App) ->
ensure_apps_stopped(#{<<"rel_apps">> := Apps}) -> ensure_apps_stopped(#{<<"rel_apps">> := Apps}) ->
%% load plugin apps and beam code %% load plugin apps and beam code
AppsToStop = AppsToStop =
lists:map(fun(NameVsn) -> lists:map(
{AppName, _AppVsn} = parse_name_vsn(NameVsn), fun(NameVsn) ->
AppName {AppName, _AppVsn} = parse_name_vsn(NameVsn),
end, Apps), AppName
end,
Apps
),
case tryit("stop_apps", fun() -> stop_apps(AppsToStop) end) of case tryit("stop_apps", fun() -> stop_apps(AppsToStop) end) of
{ok, []} -> {ok, []} ->
%% all apps stopped %% all apps stopped
ok; ok;
{ok, Left} -> {ok, Left} ->
?SLOG(warning, #{msg => "unabled_to_stop_plugin_apps", ?SLOG(warning, #{
apps => Left msg => "unabled_to_stop_plugin_apps",
}), apps => Left
}),
ok; ok;
{error, Reason} -> {error, Reason} ->
{error, Reason} {error, Reason}
@ -516,9 +581,12 @@ ensure_apps_stopped(#{<<"rel_apps">> := Apps}) ->
stop_apps(Apps) -> stop_apps(Apps) ->
RunningApps = running_apps(), RunningApps = running_apps(),
case do_stop_apps(Apps, [], RunningApps) of case do_stop_apps(Apps, [], RunningApps) of
{ok, []} -> {ok, []}; %% all stopped %% all stopped
{ok, Remain} when Remain =:= Apps -> {ok, Apps}; %% no progress {ok, []} -> {ok, []};
{ok, Remain} -> stop_apps(Remain) %% try again %% no progress
{ok, Remain} when Remain =:= Apps -> {ok, Apps};
%% try again
{ok, Remain} -> stop_apps(Remain)
end. end.
do_stop_apps([], Remain, _AllApps) -> do_stop_apps([], Remain, _AllApps) ->
@ -553,11 +621,15 @@ unload_moudle_and_app(App) ->
ok. ok.
is_needed_by_any(AppToStop, RunningApps) -> is_needed_by_any(AppToStop, RunningApps) ->
lists:any(fun({RunningApp, _RunningAppVsn}) -> lists:any(
is_needed_by(AppToStop, RunningApp) fun({RunningApp, _RunningAppVsn}) ->
end, RunningApps). is_needed_by(AppToStop, RunningApp)
end,
RunningApps
).
is_needed_by(AppToStop, AppToStop) -> false; is_needed_by(AppToStop, AppToStop) ->
false;
is_needed_by(AppToStop, RunningApp) -> is_needed_by(AppToStop, RunningApp) ->
case application:get_key(RunningApp, applications) of case application:get_key(RunningApp, applications) of
{ok, Deps} -> lists:member(AppToStop, Deps); {ok, Deps} -> lists:member(AppToStop, Deps);
@ -577,7 +649,8 @@ bin_key(Map) when is_map(Map) ->
maps:fold(fun(K, V, Acc) -> Acc#{bin(K) => V} end, #{}, Map); maps:fold(fun(K, V, Acc) -> Acc#{bin(K) => V} end, #{}, Map);
bin_key(List = [#{} | _]) -> bin_key(List = [#{} | _]) ->
lists:map(fun(M) -> bin_key(M) end, List); lists:map(fun(M) -> bin_key(M) end, List);
bin_key(Term) -> Term. bin_key(Term) ->
Term.
get_config(Key, Default) when is_atom(Key) -> get_config(Key, Default) when is_atom(Key) ->
get_config([Key], Default); get_config([Key], Default);
@ -604,8 +677,10 @@ for_plugin(#{name_vsn := NameVsn, enable := true}, Fun) ->
{error, Reason} -> [{NameVsn, Reason}] {error, Reason} -> [{NameVsn, Reason}]
end; end;
for_plugin(#{name_vsn := NameVsn, enable := false}, _Fun) -> for_plugin(#{name_vsn := NameVsn, enable := false}, _Fun) ->
?SLOG(debug, #{msg => "plugin_disabled", ?SLOG(debug, #{
name_vsn => NameVsn}), msg => "plugin_disabled",
name_vsn => NameVsn
}),
[]. [].
parse_name_vsn(NameVsn) when is_binary(NameVsn) -> parse_name_vsn(NameVsn) when is_binary(NameVsn) ->
@ -627,6 +702,9 @@ readme_file(NameVsn) ->
filename:join([dir(NameVsn), "README.md"]). filename:join([dir(NameVsn), "README.md"]).
running_apps() -> running_apps() ->
lists:map(fun({N, _, V}) -> lists:map(
{N, V} fun({N, _, V}) ->
end, application:which_applications(infinity)). {N, V}
end,
application:which_applications(infinity)
).

10
apps/emqx_plugins/src/emqx_plugins_app.erl
View File

@ -18,12 +18,14 @@
-behaviour(application). -behaviour(application).
-export([ start/2 -export([
, stop/1 start/2,
]). stop/1
]).
start(_Type, _Args) -> start(_Type, _Args) ->
ok = emqx_plugins:ensure_started(), %% load all pre-configured %% load all pre-configured
ok = emqx_plugins:ensure_started(),
{ok, Sup} = emqx_plugins_sup:start_link(), {ok, Sup} = emqx_plugins_sup:start_link(),
{ok, Sup}. {ok, Sup}.

44
apps/emqx_plugins/src/emqx_plugins_cli.erl
View File

@ -16,21 +16,23 @@
-module(emqx_plugins_cli). -module(emqx_plugins_cli).
-export([ list/1 -export([
, describe/2 list/1,
, ensure_installed/2 describe/2,
, ensure_uninstalled/2 ensure_installed/2,
, ensure_started/2 ensure_uninstalled/2,
, ensure_stopped/2 ensure_started/2,
, restart/2 ensure_stopped/2,
, ensure_disabled/2 restart/2,
, ensure_enabled/3 ensure_disabled/2,
]). ensure_enabled/3
]).
-include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/logger.hrl").
-define(PRINT(EXPR, LOG_FUN), -define(PRINT(EXPR, LOG_FUN),
print(NameVsn, fun()-> EXPR end(), LOG_FUN, ?FUNCTION_NAME)). print(NameVsn, fun() -> EXPR end(), LOG_FUN, ?FUNCTION_NAME)
).
list(LogFun) -> list(LogFun) ->
LogFun("~ts~n", [to_json(emqx_plugins:list())]). LogFun("~ts~n", [to_json(emqx_plugins:list())]).
@ -43,9 +45,11 @@ describe(NameVsn, LogFun) ->
%% this should not happen unless the package is manually installed %% this should not happen unless the package is manually installed
%% corrupted packages installed from emqx_plugins:ensure_installed %% corrupted packages installed from emqx_plugins:ensure_installed
%% should not leave behind corrupted files %% should not leave behind corrupted files
?SLOG(error, #{msg => "failed_to_describe_plugin", ?SLOG(error, #{
name_vsn => NameVsn, msg => "failed_to_describe_plugin",
cause => Reason}), name_vsn => NameVsn,
cause => Reason
}),
%% do nothing to the CLI console %% do nothing to the CLI console
ok ok
end. end.
@ -75,14 +79,18 @@ to_json(Input) ->
emqx_logger_jsonfmt:best_effort_json(Input). emqx_logger_jsonfmt:best_effort_json(Input).
print(NameVsn, Res, LogFun, Action) -> print(NameVsn, Res, LogFun, Action) ->
Obj = #{action => Action, Obj = #{
name_vsn => NameVsn}, action => Action,
name_vsn => NameVsn
},
JsonReady = JsonReady =
case Res of case Res of
ok -> ok ->
Obj#{result => ok}; Obj#{result => ok};
{error, Reason} -> {error, Reason} ->
Obj#{result => not_ok, Obj#{
cause => Reason} result => not_ok,
cause => Reason
}
end, end,
LogFun("~ts~n", [to_json(JsonReady)]). LogFun("~ts~n", [to_json(JsonReady)]).

60
apps/emqx_plugins/src/emqx_plugins_schema.erl
View File

@ -18,10 +18,11 @@
-behaviour(hocon_schema). -behaviour(hocon_schema).
-export([ roots/0 -export([
, fields/1 roots/0,
, namespace/0 fields/1,
]). namespace/0
]).
-include_lib("hocon/include/hoconsc.hrl"). -include_lib("hocon/include/hoconsc.hrl").
-include("emqx_plugins.hrl"). -include("emqx_plugins.hrl").
@ -31,31 +32,41 @@ namespace() -> "plugin".
roots() -> [?CONF_ROOT]. roots() -> [?CONF_ROOT].
fields(?CONF_ROOT) -> fields(?CONF_ROOT) ->
#{fields => root_fields(), #{
desc => ?DESC(?CONF_ROOT) fields => root_fields(),
}; desc => ?DESC(?CONF_ROOT)
};
fields(state) -> fields(state) ->
#{ fields => state_fields(), #{
desc => ?DESC(state) fields => state_fields(),
}. desc => ?DESC(state)
}.
state_fields() -> state_fields() ->
[ {name_vsn, [
hoconsc:mk(string(), {name_vsn,
#{ desc => ?DESC(name_vsn) hoconsc:mk(
, required => true string(),
})} #{
, {enable, desc => ?DESC(name_vsn),
hoconsc:mk(boolean(), required => true
#{ desc => ?DESC(enable) }
, required => true )},
})} {enable,
hoconsc:mk(
boolean(),
#{
desc => ?DESC(enable),
required => true
}
)}
]. ].
root_fields() -> root_fields() ->
[ {states, fun states/1} [
, {install_dir, fun install_dir/1} {states, fun states/1},
, {check_interval, fun check_interval/1} {install_dir, fun install_dir/1},
{check_interval, fun check_interval/1}
]. ].
states(type) -> hoconsc:array(hoconsc:ref(?MODULE, state)); states(type) -> hoconsc:array(hoconsc:ref(?MODULE, state));
@ -66,7 +77,8 @@ states(_) -> undefined.
install_dir(type) -> string(); install_dir(type) -> string();
install_dir(required) -> false; install_dir(required) -> false;
install_dir(default) -> "plugins"; %% runner's root dir %% runner's root dir
install_dir(default) -> "plugins";
install_dir(T) when T =/= desc -> undefined; install_dir(T) when T =/= desc -> undefined;
install_dir(desc) -> ?DESC(install_dir). install_dir(desc) -> ?DESC(install_dir).

3
apps/emqx_plugins/src/emqx_plugins_sup.erl
View File

@ -29,7 +29,8 @@ init([]) ->
%% TODO: Add monitor plugins change. %% TODO: Add monitor plugins change.
Monitor = emqx_plugins_monitor, Monitor = emqx_plugins_monitor,
_Children = [ _Children = [
#{id => Monitor, #{
id => Monitor,
start => {Monitor, start_link, []}, start => {Monitor, start_link, []},
restart => permanent, restart => permanent,
shutdown => brutal_kill, shutdown => brutal_kill,

263
apps/emqx_plugins/test/emqx_plugins_SUITE.erl
View File

@ -48,9 +48,12 @@ end_per_suite(Config) ->
init_per_testcase(TestCase, Config) -> init_per_testcase(TestCase, Config) ->
emqx_plugins:put_configured([]), emqx_plugins:put_configured([]),
lists:foreach(fun(#{<<"name">> := Name, <<"rel_vsn">> := Vsn}) -> lists:foreach(
emqx_plugins:purge(bin([Name, "-", Vsn])) fun(#{<<"name">> := Name, <<"rel_vsn">> := Vsn}) ->
end, emqx_plugins:list()), emqx_plugins:purge(bin([Name, "-", Vsn]))
end,
emqx_plugins:list()
),
?MODULE:TestCase({init, Config}). ?MODULE:TestCase({init, Config}).
end_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) ->
@ -59,35 +62,46 @@ end_per_testcase(TestCase, Config) ->
build_demo_plugin_package() -> build_demo_plugin_package() ->
build_demo_plugin_package( build_demo_plugin_package(
#{ target_path => "_build/default/emqx_plugrel" #{
, release_name => "emqx_plugin_template" target_path => "_build/default/emqx_plugrel",
, git_url => "https://github.com/emqx/emqx-plugin-template.git" release_name => "emqx_plugin_template",
, vsn => ?EMQX_PLUGIN_TEMPLATE_VSN git_url => "https://github.com/emqx/emqx-plugin-template.git",
, workdir => "demo_src" vsn => ?EMQX_PLUGIN_TEMPLATE_VSN,
, shdir => emqx_plugins:install_dir() workdir => "demo_src",
}). shdir => emqx_plugins:install_dir()
}
).
build_demo_plugin_package(#{ target_path := TargetPath build_demo_plugin_package(
, release_name := ReleaseName #{
, git_url := GitUrl target_path := TargetPath,
, vsn := PluginVsn release_name := ReleaseName,
, workdir := DemoWorkDir git_url := GitUrl,
, shdir := WorkDir vsn := PluginVsn,
} = Opts) -> workdir := DemoWorkDir,
shdir := WorkDir
} = Opts
) ->
BuildSh = filename:join([WorkDir, "build-demo-plugin.sh"]), BuildSh = filename:join([WorkDir, "build-demo-plugin.sh"]),
Cmd = string:join([ BuildSh Cmd = string:join(
, PluginVsn [
, TargetPath BuildSh,
, ReleaseName PluginVsn,
, GitUrl TargetPath,
, DemoWorkDir ReleaseName,
], GitUrl,
" "), DemoWorkDir
],
" "
),
case emqx_run_sh:do(Cmd, [{cd, WorkDir}]) of case emqx_run_sh:do(Cmd, [{cd, WorkDir}]) of
{ok, _} -> {ok, _} ->
Pkg = filename:join([WorkDir, ReleaseName ++ "-" ++ Pkg = filename:join([
PluginVsn ++ WorkDir,
?PACKAGE_SUFFIX]), ReleaseName ++ "-" ++
PluginVsn ++
?PACKAGE_SUFFIX
]),
case filelib:is_regular(Pkg) of case filelib:is_regular(Pkg) of
true -> Opts#{package => Pkg}; true -> Opts#{package => Pkg};
false -> error(#{reason => unexpected_build_result, not_found => Pkg}) false -> error(#{reason => unexpected_build_result, not_found => Pkg})
@ -104,16 +118,19 @@ bin(B) when is_binary(B) -> B.
t_demo_install_start_stop_uninstall({init, Config}) -> t_demo_install_start_stop_uninstall({init, Config}) ->
Opts = #{package := Package} = build_demo_plugin_package(), Opts = #{package := Package} = build_demo_plugin_package(),
NameVsn = filename:basename(Package, ?PACKAGE_SUFFIX), NameVsn = filename:basename(Package, ?PACKAGE_SUFFIX),
[ {name_vsn, NameVsn} [
, {plugin_opts, Opts} {name_vsn, NameVsn},
| Config {plugin_opts, Opts}
| Config
]; ];
t_demo_install_start_stop_uninstall({'end', _Config}) -> ok; t_demo_install_start_stop_uninstall({'end', _Config}) ->
ok;
t_demo_install_start_stop_uninstall(Config) -> t_demo_install_start_stop_uninstall(Config) ->
NameVsn = proplists:get_value(name_vsn, Config), NameVsn = proplists:get_value(name_vsn, Config),
#{ release_name := ReleaseName #{
, vsn := PluginVsn release_name := ReleaseName,
} = proplists:get_value(plugin_opts, Config), vsn := PluginVsn
} = proplists:get_value(plugin_opts, Config),
ok = emqx_plugins:ensure_installed(NameVsn), ok = emqx_plugins:ensure_installed(NameVsn),
%% idempotent %% idempotent
ok = emqx_plugins:ensure_installed(NameVsn), ok = emqx_plugins:ensure_installed(NameVsn),
@ -129,8 +146,10 @@ t_demo_install_start_stop_uninstall(Config) ->
ok = assert_app_running(map_sets, true), ok = assert_app_running(map_sets, true),
%% running app can not be un-installed %% running app can not be un-installed
?assertMatch({error, _}, ?assertMatch(
emqx_plugins:ensure_uninstalled(NameVsn)), {error, _},
emqx_plugins:ensure_uninstalled(NameVsn)
),
%% stop %% stop
ok = emqx_plugins:ensure_stopped(NameVsn), ok = emqx_plugins:ensure_stopped(NameVsn),
@ -143,9 +162,15 @@ t_demo_install_start_stop_uninstall(Config) ->
%% still listed after stopped %% still listed after stopped
ReleaseNameBin = list_to_binary(ReleaseName), ReleaseNameBin = list_to_binary(ReleaseName),
PluginVsnBin = list_to_binary(PluginVsn), PluginVsnBin = list_to_binary(PluginVsn),
?assertMatch([#{<<"name">> := ReleaseNameBin, ?assertMatch(
<<"rel_vsn">> := PluginVsnBin [
}], emqx_plugins:list()), #{
<<"name">> := ReleaseNameBin,
<<"rel_vsn">> := PluginVsnBin
}
],
emqx_plugins:list()
),
ok = emqx_plugins:ensure_uninstalled(NameVsn), ok = emqx_plugins:ensure_uninstalled(NameVsn),
?assertEqual([], emqx_plugins:list()), ?assertEqual([], emqx_plugins:list()),
ok. ok.
@ -164,23 +189,29 @@ t_position({init, Config}) ->
#{package := Package} = build_demo_plugin_package(), #{package := Package} = build_demo_plugin_package(),
NameVsn = filename:basename(Package, ?PACKAGE_SUFFIX), NameVsn = filename:basename(Package, ?PACKAGE_SUFFIX),
[{name_vsn, NameVsn} | Config]; [{name_vsn, NameVsn} | Config];
t_position({'end', _Config}) -> ok; t_position({'end', _Config}) ->
ok;
t_position(Config) -> t_position(Config) ->
NameVsn = proplists:get_value(name_vsn, Config), NameVsn = proplists:get_value(name_vsn, Config),
ok = emqx_plugins:ensure_installed(NameVsn), ok = emqx_plugins:ensure_installed(NameVsn),
ok = emqx_plugins:ensure_enabled(NameVsn), ok = emqx_plugins:ensure_enabled(NameVsn),
FakeInfo = "name=position, rel_vsn=\"2\", rel_apps=[\"position-9\"]," FakeInfo =
"description=\"desc fake position app\"", "name=position, rel_vsn=\"2\", rel_apps=[\"position-9\"],"
"description=\"desc fake position app\"",
PosApp2 = <<"position-2">>, PosApp2 = <<"position-2">>,
ok = write_info_file(Config, PosApp2, FakeInfo), ok = write_info_file(Config, PosApp2, FakeInfo),
%% fake a disabled plugin in config %% fake a disabled plugin in config
ok = emqx_plugins:ensure_state(PosApp2, {before, NameVsn}, false), ok = emqx_plugins:ensure_state(PosApp2, {before, NameVsn}, false),
ListFun = fun() -> ListFun = fun() ->
lists:map(fun( lists:map(
#{<<"name">> := Name, <<"rel_vsn">> := Vsn}) -> fun(
<<Name/binary, "-", Vsn/binary>> #{<<"name">> := Name, <<"rel_vsn">> := Vsn}
end, emqx_plugins:list()) ) ->
end, <<Name/binary, "-", Vsn/binary>>
end,
emqx_plugins:list()
)
end,
?assertEqual([PosApp2, list_to_binary(NameVsn)], ListFun()), ?assertEqual([PosApp2, list_to_binary(NameVsn)], ListFun()),
emqx_plugins:ensure_enabled(PosApp2, {behind, NameVsn}), emqx_plugins:ensure_enabled(PosApp2, {behind, NameVsn}),
?assertEqual([list_to_binary(NameVsn), PosApp2], ListFun()), ?assertEqual([list_to_binary(NameVsn), PosApp2], ListFun()),
@ -197,13 +228,15 @@ t_start_restart_and_stop({init, Config}) ->
#{package := Package} = build_demo_plugin_package(), #{package := Package} = build_demo_plugin_package(),
NameVsn = filename:basename(Package, ?PACKAGE_SUFFIX), NameVsn = filename:basename(Package, ?PACKAGE_SUFFIX),
[{name_vsn, NameVsn} | Config]; [{name_vsn, NameVsn} | Config];
t_start_restart_and_stop({'end', _Config}) -> ok; t_start_restart_and_stop({'end', _Config}) ->
ok;
t_start_restart_and_stop(Config) -> t_start_restart_and_stop(Config) ->
NameVsn = proplists:get_value(name_vsn, Config), NameVsn = proplists:get_value(name_vsn, Config),
ok = emqx_plugins:ensure_installed(NameVsn), ok = emqx_plugins:ensure_installed(NameVsn),
ok = emqx_plugins:ensure_enabled(NameVsn), ok = emqx_plugins:ensure_enabled(NameVsn),
FakeInfo = "name=bar, rel_vsn=\"2\", rel_apps=[\"bar-9\"]," FakeInfo =
"description=\"desc bar\"", "name=bar, rel_vsn=\"2\", rel_apps=[\"bar-9\"],"
"description=\"desc bar\"",
Bar2 = <<"bar-2">>, Bar2 = <<"bar-2">>,
ok = write_info_file(Config, Bar2, FakeInfo), ok = write_info_file(Config, Bar2, FakeInfo),
%% fake a disabled plugin in config %% fake a disabled plugin in config
@ -216,8 +249,10 @@ t_start_restart_and_stop(Config) ->
%% fake enable bar-2 %% fake enable bar-2
ok = emqx_plugins:ensure_state(Bar2, rear, true), ok = emqx_plugins:ensure_state(Bar2, rear, true),
%% should cause an error %% should cause an error
?assertError(#{function := _, errors := [_ | _]}, ?assertError(
emqx_plugins:ensure_started()), #{function := _, errors := [_ | _]},
emqx_plugins:ensure_started()
),
%% but demo plugin should still be running %% but demo plugin should still be running
assert_app_running(emqx_plugin_template, true), assert_app_running(emqx_plugin_template, true),
@ -255,9 +290,13 @@ t_enable_disable(Config) ->
?assertEqual([#{name_vsn => NameVsn, enable => false}], emqx_plugins:configured()), ?assertEqual([#{name_vsn => NameVsn, enable => false}], emqx_plugins:configured()),
ok = emqx_plugins:ensure_enabled(bin(NameVsn)), ok = emqx_plugins:ensure_enabled(bin(NameVsn)),
?assertEqual([#{name_vsn => NameVsn, enable => true}], emqx_plugins:configured()), ?assertEqual([#{name_vsn => NameVsn, enable => true}], emqx_plugins:configured()),
?assertMatch({error, #{reason := "bad_plugin_config_status", ?assertMatch(
hint := "disable_the_plugin_first" {error, #{
}}, emqx_plugins:ensure_uninstalled(NameVsn)), reason := "bad_plugin_config_status",
hint := "disable_the_plugin_first"
}},
emqx_plugins:ensure_uninstalled(NameVsn)
),
ok = emqx_plugins:ensure_disabled(bin(NameVsn)), ok = emqx_plugins:ensure_disabled(bin(NameVsn)),
ok = emqx_plugins:ensure_uninstalled(NameVsn), ok = emqx_plugins:ensure_uninstalled(NameVsn),
?assertMatch({error, _}, emqx_plugins:ensure_enabled(NameVsn)), ?assertMatch({error, _}, emqx_plugins:ensure_enabled(NameVsn)),
@ -271,20 +310,28 @@ assert_app_running(Name, false) ->
AllApps = application:which_applications(), AllApps = application:which_applications(),
?assertEqual(false, lists:keyfind(Name, 1, AllApps)). ?assertEqual(false, lists:keyfind(Name, 1, AllApps)).
t_bad_tar_gz({init, Config}) -> Config; t_bad_tar_gz({init, Config}) ->
t_bad_tar_gz({'end', _Config}) -> ok; Config;
t_bad_tar_gz({'end', _Config}) ->
ok;
t_bad_tar_gz(Config) -> t_bad_tar_gz(Config) ->
WorkDir = proplists:get_value(data_dir, Config), WorkDir = proplists:get_value(data_dir, Config),
FakeTarTz = filename:join([WorkDir, "fake-vsn.tar.gz"]), FakeTarTz = filename:join([WorkDir, "fake-vsn.tar.gz"]),
ok = file:write_file(FakeTarTz, "a\n"), ok = file:write_file(FakeTarTz, "a\n"),
?assertMatch({error, #{reason := "bad_plugin_package", ?assertMatch(
return := eof {error, #{
}}, reason := "bad_plugin_package",
emqx_plugins:ensure_installed("fake-vsn")), return := eof
?assertMatch({error, #{reason := "failed_to_extract_plugin_package", }},
return := not_found emqx_plugins:ensure_installed("fake-vsn")
}}, ),
emqx_plugins:ensure_installed("nonexisting")), ?assertMatch(
{error, #{
reason := "failed_to_extract_plugin_package",
return := not_found
}},
emqx_plugins:ensure_installed("nonexisting")
),
?assertEqual([], emqx_plugins:list()), ?assertEqual([], emqx_plugins:list()),
ok = emqx_plugins:delete_package("fake-vsn"), ok = emqx_plugins:delete_package("fake-vsn"),
%% idempotent %% idempotent
@ -292,8 +339,10 @@ t_bad_tar_gz(Config) ->
%% create a corrupted .tar.gz %% create a corrupted .tar.gz
%% failed install attempts should not leave behind extracted dir %% failed install attempts should not leave behind extracted dir
t_bad_tar_gz2({init, Config}) -> Config; t_bad_tar_gz2({init, Config}) ->
t_bad_tar_gz2({'end', _Config}) -> ok; Config;
t_bad_tar_gz2({'end', _Config}) ->
ok;
t_bad_tar_gz2(Config) -> t_bad_tar_gz2(Config) ->
WorkDir = proplists:get_value(data_dir, Config), WorkDir = proplists:get_value(data_dir, Config),
NameVsn = "foo-0.2", NameVsn = "foo-0.2",
@ -310,45 +359,57 @@ t_bad_tar_gz2(Config) ->
?assertEqual({error, enoent}, file:read_file_info(emqx_plugins:dir(NameVsn))), ?assertEqual({error, enoent}, file:read_file_info(emqx_plugins:dir(NameVsn))),
ok = emqx_plugins:delete_package(NameVsn). ok = emqx_plugins:delete_package(NameVsn).
t_bad_info_json({init, Config}) -> Config; t_bad_info_json({init, Config}) ->
t_bad_info_json({'end', _}) -> ok; Config;
t_bad_info_json({'end', _}) ->
ok;
t_bad_info_json(Config) -> t_bad_info_json(Config) ->
NameVsn = "test-2", NameVsn = "test-2",
ok = write_info_file(Config, NameVsn, "bad-syntax"), ok = write_info_file(Config, NameVsn, "bad-syntax"),
?assertMatch({error, #{error := "bad_info_file", ?assertMatch(
return := {parse_error, _} {error, #{
}}, error := "bad_info_file",
emqx_plugins:describe(NameVsn)), return := {parse_error, _}
}},
emqx_plugins:describe(NameVsn)
),
ok = write_info_file(Config, NameVsn, "{\"bad\": \"obj\"}"), ok = write_info_file(Config, NameVsn, "{\"bad\": \"obj\"}"),
?assertMatch({error, #{error := "bad_info_file_content", ?assertMatch(
mandatory_fields := _ {error, #{
}}, error := "bad_info_file_content",
emqx_plugins:describe(NameVsn)), mandatory_fields := _
}},
emqx_plugins:describe(NameVsn)
),
?assertEqual([], emqx_plugins:list()), ?assertEqual([], emqx_plugins:list()),
emqx_plugins:purge(NameVsn), emqx_plugins:purge(NameVsn),
ok. ok.
t_elixir_plugin({init, Config}) -> t_elixir_plugin({init, Config}) ->
Opts0 = Opts0 =
#{ target_path => "_build/prod/plugrelex/elixir_plugin_template" #{
, release_name => "elixir_plugin_template" target_path => "_build/prod/plugrelex/elixir_plugin_template",
, git_url => "https://github.com/emqx/emqx-elixir-plugin.git" release_name => "elixir_plugin_template",
, vsn => ?EMQX_ELIXIR_PLUGIN_TEMPLATE_VSN git_url => "https://github.com/emqx/emqx-elixir-plugin.git",
, workdir => "demo_src_elixir" vsn => ?EMQX_ELIXIR_PLUGIN_TEMPLATE_VSN,
, shdir => emqx_plugins:install_dir() workdir => "demo_src_elixir",
}, shdir => emqx_plugins:install_dir()
},
Opts = #{package := Package} = build_demo_plugin_package(Opts0), Opts = #{package := Package} = build_demo_plugin_package(Opts0),
NameVsn = filename:basename(Package, ?PACKAGE_SUFFIX), NameVsn = filename:basename(Package, ?PACKAGE_SUFFIX),
[ {name_vsn, NameVsn} [
, {plugin_opts, Opts} {name_vsn, NameVsn},
| Config {plugin_opts, Opts}
| Config
]; ];
t_elixir_plugin({'end', _Config}) -> ok; t_elixir_plugin({'end', _Config}) ->
ok;
t_elixir_plugin(Config) -> t_elixir_plugin(Config) ->
NameVsn = proplists:get_value(name_vsn, Config), NameVsn = proplists:get_value(name_vsn, Config),
#{ release_name := ReleaseName #{
, vsn := PluginVsn release_name := ReleaseName,
} = proplists:get_value(plugin_opts, Config), vsn := PluginVsn
} = proplists:get_value(plugin_opts, Config),
ok = emqx_plugins:ensure_installed(NameVsn), ok = emqx_plugins:ensure_installed(NameVsn),
%% idempotent %% idempotent
ok = emqx_plugins:ensure_installed(NameVsn), ok = emqx_plugins:ensure_installed(NameVsn),
@ -368,8 +429,10 @@ t_elixir_plugin(Config) ->
3 = 'Elixir.Kernel':'+'(1, 2), 3 = 'Elixir.Kernel':'+'(1, 2),
%% running app can not be un-installed %% running app can not be un-installed
?assertMatch({error, _}, ?assertMatch(
emqx_plugins:ensure_uninstalled(NameVsn)), {error, _},
emqx_plugins:ensure_uninstalled(NameVsn)
),
%% stop %% stop
ok = emqx_plugins:ensure_stopped(NameVsn), ok = emqx_plugins:ensure_stopped(NameVsn),
@ -382,9 +445,15 @@ t_elixir_plugin(Config) ->
%% still listed after stopped %% still listed after stopped
ReleaseNameBin = list_to_binary(ReleaseName), ReleaseNameBin = list_to_binary(ReleaseName),
PluginVsnBin = list_to_binary(PluginVsn), PluginVsnBin = list_to_binary(PluginVsn),
?assertMatch([#{<<"name">> := ReleaseNameBin, ?assertMatch(
<<"rel_vsn">> := PluginVsnBin [
}], emqx_plugins:list()), #{
<<"name">> := ReleaseNameBin,
<<"rel_vsn">> := PluginVsnBin
}
],
emqx_plugins:list()
),
ok = emqx_plugins:ensure_uninstalled(NameVsn), ok = emqx_plugins:ensure_uninstalled(NameVsn),
?assertEqual([], emqx_plugins:list()), ?assertEqual([], emqx_plugins:list()),
ok. ok.

46
apps/emqx_plugins/test/emqx_plugins_tests.erl
View File

@ -23,23 +23,26 @@
ensure_configured_test_todo() -> ensure_configured_test_todo() ->
meck_emqx(), meck_emqx(),
try test_ensure_configured() try
after emqx_plugins:put_configured([]) test_ensure_configured()
after
emqx_plugins:put_configured([])
end, end,
meck:unload(emqx). meck:unload(emqx).
test_ensure_configured() -> test_ensure_configured() ->
ok = emqx_plugins:put_configured([]), ok = emqx_plugins:put_configured([]),
P1 =#{name_vsn => "p-1", enable => true}, P1 = #{name_vsn => "p-1", enable => true},
P2 =#{name_vsn => "p-2", enable => true}, P2 = #{name_vsn => "p-2", enable => true},
P3 =#{name_vsn => "p-3", enable => false}, P3 = #{name_vsn => "p-3", enable => false},
emqx_plugins:ensure_configured(P1, front), emqx_plugins:ensure_configured(P1, front),
emqx_plugins:ensure_configured(P2, {before, <<"p-1">>}), emqx_plugins:ensure_configured(P2, {before, <<"p-1">>}),
emqx_plugins:ensure_configured(P3, {before, <<"p-1">>}), emqx_plugins:ensure_configured(P3, {before, <<"p-1">>}),
?assertEqual([P2, P3, P1], emqx_plugins:configured()), ?assertEqual([P2, P3, P1], emqx_plugins:configured()),
?assertThrow(#{error := "position_anchor_plugin_not_configured"}, ?assertThrow(
emqx_plugins:ensure_configured(P3, {before, <<"unknown-x">>})). #{error := "position_anchor_plugin_not_configured"},
emqx_plugins:ensure_configured(P3, {before, <<"unknown-x">>})
).
read_plugin_test() -> read_plugin_test() ->
meck_emqx(), meck_emqx(),
@ -47,16 +50,20 @@ read_plugin_test() ->
fun(_Dir) -> fun(_Dir) ->
NameVsn = "bar-5", NameVsn = "bar-5",
InfoFile = emqx_plugins:info_file(NameVsn), InfoFile = emqx_plugins:info_file(NameVsn),
FakeInfo = "name=bar, rel_vsn=\"5\", rel_apps=[justname_no_vsn]," FakeInfo =
"description=\"desc bar\"", "name=bar, rel_vsn=\"5\", rel_apps=[justname_no_vsn],"
"description=\"desc bar\"",
try try
ok = write_file(InfoFile, FakeInfo), ok = write_file(InfoFile, FakeInfo),
?assertMatch({error, #{error := "bad_rel_apps"}}, ?assertMatch(
emqx_plugins:read_plugin(NameVsn, #{})) {error, #{error := "bad_rel_apps"}},
emqx_plugins:read_plugin(NameVsn, #{})
)
after after
emqx_plugins:purge(NameVsn) emqx_plugins:purge(NameVsn)
end end
end), end
),
meck:unload(emqx). meck:unload(emqx).
with_rand_install_dir(F) -> with_rand_install_dir(F) ->
@ -91,7 +98,8 @@ delete_package_test() ->
Dir = File, Dir = File,
ok = filelib:ensure_dir(filename:join([Dir, "foo"])), ok = filelib:ensure_dir(filename:join([Dir, "foo"])),
?assertMatch({error, _}, emqx_plugins:delete_package("a-1")) ?assertMatch({error, _}, emqx_plugins:delete_package("a-1"))
end), end
),
meck:unload(emqx). meck:unload(emqx).
%% purge plugin's install dir should mostly work and return ok %% purge plugin's install dir should mostly work and return ok
@ -110,15 +118,19 @@ purge_test() ->
%% write a file for the dir path %% write a file for the dir path
ok = file:write_file(Dir, "a"), ok = file:write_file(Dir, "a"),
?assertEqual(ok, emqx_plugins:purge("a-1")) ?assertEqual(ok, emqx_plugins:purge("a-1"))
end), end
),
meck:unload(emqx). meck:unload(emqx).
meck_emqx() -> meck_emqx() ->
meck:new(emqx, [unstick, passthrough]), meck:new(emqx, [unstick, passthrough]),
meck:expect(emqx, update_config, meck:expect(
emqx,
update_config,
fun(Path, Values, _Opts) -> fun(Path, Values, _Opts) ->
emqx_config:put(Path, Values) emqx_config:put(Path, Values)
end), end
),
%meck:expect(emqx, get_config, %meck:expect(emqx, get_config,
% fun(KeyPath, Default) -> % fun(KeyPath, Default) ->
% Map = emqx:get_raw_config(KeyPath, Default), % Map = emqx:get_raw_config(KeyPath, Default),

39
apps/emqx_prometheus/rebar.config
View File

@ -1,23 +1,32 @@
%% -*- mode: erlang -*- %% -*- mode: erlang -*-
{deps, {deps, [
[ {emqx, {path, "../emqx"}}, {emqx, {path, "../emqx"}},
%% FIXME: tag this as v3.1.3 %% FIXME: tag this as v3.1.3
{prometheus, {git, "https://github.com/deadtrickster/prometheus.erl", {tag, "v4.8.1"}}}, {prometheus, {git, "https://github.com/deadtrickster/prometheus.erl", {tag, "v4.8.1"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.27.4"}}} {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.27.4"}}}
]}. ]}.
{edoc_opts, [{preprocess, true}]}. {edoc_opts, [{preprocess, true}]}.
{erl_opts, [warn_unused_vars, {erl_opts, [
warn_shadow_vars, warn_unused_vars,
warn_unused_import, warn_shadow_vars,
warn_obsolete_guard, warn_unused_import,
debug_info, warn_obsolete_guard,
{parse_transform}]}. debug_info,
{parse_transform}
]}.
{xref_checks, [undefined_function_calls, undefined_functions, {xref_checks, [
locals_not_used, deprecated_function_calls, undefined_function_calls,
warnings_as_errors, deprecated_functions]}. undefined_functions,
locals_not_used,
deprecated_function_calls,
warnings_as_errors,
deprecated_functions
]}.
{cover_enabled, true}. {cover_enabled, true}.
{cover_opts, [verbose]}. {cover_opts, [verbose]}.
{cover_export_enabled, true}. {cover_export_enabled, true}.
{project_plugins, [erlfmt]}.

30
apps/emqx_prometheus/src/emqx_prometheus.app.src
View File

@ -1,15 +1,17 @@
%% -*- mode: erlang -*- %% -*- mode: erlang -*-
{application, emqx_prometheus, {application, emqx_prometheus, [
[{description, "Prometheus for EMQX"}, {description, "Prometheus for EMQX"},
{vsn, "5.0.0"}, % strict semver, bump manually! % strict semver, bump manually!
{modules, []}, {vsn, "5.0.0"},
{registered, [emqx_prometheus_sup]}, {modules, []},
{applications, [kernel,stdlib,prometheus,emqx]}, {registered, [emqx_prometheus_sup]},
{mod, {emqx_prometheus_app,[]}}, {applications, [kernel, stdlib, prometheus, emqx]},
{env, []}, {mod, {emqx_prometheus_app, []}},
{licenses, ["Apache-2.0"]}, {env, []},
{maintainers, ["EMQX Team <contact@emqx.io>"]}, {licenses, ["Apache-2.0"]},
{links, [{"Homepage", "https://emqx.io/"}, {maintainers, ["EMQX Team <contact@emqx.io>"]},
{"Github", "https://github.com/emqx/emqx-prometheus"} {links, [
]} {"Homepage", "https://emqx.io/"},
]}. {"Github", "https://github.com/emqx/emqx-prometheus"}
]}
]}.

362
apps/emqx_prometheus/src/emqx_prometheus.erl
View File

@ -28,38 +28,44 @@
-include_lib("prometheus/include/prometheus_model.hrl"). -include_lib("prometheus/include/prometheus_model.hrl").
-include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/logger.hrl").
-import(prometheus_model_helpers, -import(
[ create_mf/5 prometheus_model_helpers,
, gauge_metric/1 [
, counter_metric/1 create_mf/5,
]). gauge_metric/1,
counter_metric/1
]
).
-export([ update/1 -export([
, start/0 update/1,
, stop/0 start/0,
, restart/0 stop/0,
% for rpc restart/0,
, do_start/0 % for rpc
, do_stop/0 do_start/0,
]). do_stop/0
]).
%% APIs %% APIs
-export([start_link/1]). -export([start_link/1]).
%% gen_server callbacks %% gen_server callbacks
-export([ init/1 -export([
, handle_call/3 init/1,
, handle_cast/2 handle_call/3,
, handle_info/2 handle_cast/2,
, code_change/3 handle_info/2,
, terminate/2 code_change/3,
]). terminate/2
]).
%% prometheus_collector callback %% prometheus_collector callback
-export([ deregister_cleanup/1 -export([
, collect_mf/2 deregister_cleanup/1,
, collect_metrics/2 collect_mf/2,
]). collect_metrics/2
]).
-export([collect/1]). -export([collect/1]).
@ -72,8 +78,13 @@
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% update new config %% update new config
update(Config) -> update(Config) ->
case emqx_conf:update([prometheus], Config, case
#{rawconf_with_defaults => true, override_to => cluster}) of emqx_conf:update(
[prometheus],
Config,
#{rawconf_with_defaults => true, override_to => cluster}
)
of
{ok, #{raw_config := NewConfigRows}} -> {ok, #{raw_config := NewConfigRows}} ->
case maps:get(<<"enable">>, Config, true) of case maps:get(<<"enable">>, Config, true) of
true -> true ->
@ -131,13 +142,12 @@ handle_call(_Msg, _From, State) ->
handle_cast(_Msg, State) -> handle_cast(_Msg, State) ->
{noreply, State}. {noreply, State}.
handle_info({timeout, R, ?TIMER_MSG}, State = #state{timer=R, push_gateway=Uri}) -> handle_info({timeout, R, ?TIMER_MSG}, State = #state{timer = R, push_gateway = Uri}) ->
[Name, Ip] = string:tokens(atom_to_list(node()), "@"), [Name, Ip] = string:tokens(atom_to_list(node()), "@"),
Url = lists:concat([Uri, "/metrics/job/", Name, "/instance/",Name, "~", Ip]), Url = lists:concat([Uri, "/metrics/job/", Name, "/instance/", Name, "~", Ip]),
Data = prometheus_text_format:format(), Data = prometheus_text_format:format(),
httpc:request(post, {Url, [], "text/plain", Data}, [{autoredirect, true}], []), httpc:request(post, {Url, [], "text/plain", Data}, [{autoredirect, true}], []),
{noreply, ensure_timer(State)}; {noreply, ensure_timer(State)};
handle_info(_Msg, State) -> handle_info(_Msg, State) ->
{noreply, State}. {noreply, State}.
@ -176,14 +186,15 @@ collect(<<"json">>) ->
Metrics = emqx_metrics:all(), Metrics = emqx_metrics:all(),
Stats = emqx_stats:getstats(), Stats = emqx_stats:getstats(),
VMData = emqx_vm_data(), VMData = emqx_vm_data(),
#{stats => maps:from_list([collect_stats(Name, Stats) || Name <- emqx_stats()]), #{
metrics => maps:from_list([collect_stats(Name, VMData) || Name <- emqx_vm()]), stats => maps:from_list([collect_stats(Name, Stats) || Name <- emqx_stats()]),
packets => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_packets()]), metrics => maps:from_list([collect_stats(Name, VMData) || Name <- emqx_vm()]),
messages => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_messages()]), packets => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_packets()]),
delivery => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_delivery()]), messages => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_messages()]),
client => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_client()]), delivery => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_delivery()]),
session => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_session()])}; client => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_client()]),
session => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_session()])
};
collect(<<"prometheus">>) -> collect(<<"prometheus">>) ->
prometheus_text_format:format(). prometheus_text_format:format().
@ -219,13 +230,11 @@ emqx_collect(emqx_connections_count, Stats) ->
gauge_metric(?C('connections.count', Stats)); gauge_metric(?C('connections.count', Stats));
emqx_collect(emqx_connections_max, Stats) -> emqx_collect(emqx_connections_max, Stats) ->
gauge_metric(?C('connections.max', Stats)); gauge_metric(?C('connections.max', Stats));
%% sessions %% sessions
emqx_collect(emqx_sessions_count, Stats) -> emqx_collect(emqx_sessions_count, Stats) ->
gauge_metric(?C('sessions.count', Stats)); gauge_metric(?C('sessions.count', Stats));
emqx_collect(emqx_sessions_max, Stats) -> emqx_collect(emqx_sessions_max, Stats) ->
gauge_metric(?C('sessions.max', Stats)); gauge_metric(?C('sessions.max', Stats));
%% pub/sub stats %% pub/sub stats
emqx_collect(emqx_topics_count, Stats) -> emqx_collect(emqx_topics_count, Stats) ->
gauge_metric(?C('topics.count', Stats)); gauge_metric(?C('topics.count', Stats));
@ -247,13 +256,11 @@ emqx_collect(emqx_subscriptions_shared_count, Stats) ->
gauge_metric(?C('subscriptions.shared.count', Stats)); gauge_metric(?C('subscriptions.shared.count', Stats));
emqx_collect(emqx_subscriptions_shared_max, Stats) -> emqx_collect(emqx_subscriptions_shared_max, Stats) ->
gauge_metric(?C('subscriptions.shared.max', Stats)); gauge_metric(?C('subscriptions.shared.max', Stats));
%% retained %% retained
emqx_collect(emqx_retained_count, Stats) -> emqx_collect(emqx_retained_count, Stats) ->
gauge_metric(?C('retained.count', Stats)); gauge_metric(?C('retained.count', Stats));
emqx_collect(emqx_retained_max, Stats) -> emqx_collect(emqx_retained_max, Stats) ->
gauge_metric(?C('retained.max', Stats)); gauge_metric(?C('retained.max', Stats));
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% Metrics - packets & bytes %% Metrics - packets & bytes
@ -262,13 +269,11 @@ emqx_collect(emqx_bytes_received, Metrics) ->
counter_metric(?C('bytes.received', Metrics)); counter_metric(?C('bytes.received', Metrics));
emqx_collect(emqx_bytes_sent, Metrics) -> emqx_collect(emqx_bytes_sent, Metrics) ->
counter_metric(?C('bytes.sent', Metrics)); counter_metric(?C('bytes.sent', Metrics));
%% received.sent %% received.sent
emqx_collect(emqx_packets_received, Metrics) -> emqx_collect(emqx_packets_received, Metrics) ->
counter_metric(?C('packets.received', Metrics)); counter_metric(?C('packets.received', Metrics));
emqx_collect(emqx_packets_sent, Metrics) -> emqx_collect(emqx_packets_sent, Metrics) ->
counter_metric(?C('packets.sent', Metrics)); counter_metric(?C('packets.sent', Metrics));
%% connect %% connect
emqx_collect(emqx_packets_connect, Metrics) -> emqx_collect(emqx_packets_connect, Metrics) ->
counter_metric(?C('packets.connect.received', Metrics)); counter_metric(?C('packets.connect.received', Metrics));
@ -278,7 +283,6 @@ emqx_collect(emqx_packets_connack_error, Metrics) ->
counter_metric(?C('packets.connack.error', Metrics)); counter_metric(?C('packets.connack.error', Metrics));
emqx_collect(emqx_packets_connack_auth_error, Metrics) -> emqx_collect(emqx_packets_connack_auth_error, Metrics) ->
counter_metric(?C('packets.connack.auth_error', Metrics)); counter_metric(?C('packets.connack.auth_error', Metrics));
%% sub.unsub %% sub.unsub
emqx_collect(emqx_packets_subscribe_received, Metrics) -> emqx_collect(emqx_packets_subscribe_received, Metrics) ->
counter_metric(?C('packets.subscribe.received', Metrics)); counter_metric(?C('packets.subscribe.received', Metrics));
@ -294,7 +298,6 @@ emqx_collect(emqx_packets_unsubscribe_error, Metrics) ->
counter_metric(?C('packets.unsubscribe.error', Metrics)); counter_metric(?C('packets.unsubscribe.error', Metrics));
emqx_collect(emqx_packets_unsuback_sent, Metrics) -> emqx_collect(emqx_packets_unsuback_sent, Metrics) ->
counter_metric(?C('packets.unsuback.sent', Metrics)); counter_metric(?C('packets.unsuback.sent', Metrics));
%% publish.puback %% publish.puback
emqx_collect(emqx_packets_publish_received, Metrics) -> emqx_collect(emqx_packets_publish_received, Metrics) ->
counter_metric(?C('packets.publish.received', Metrics)); counter_metric(?C('packets.publish.received', Metrics));
@ -308,7 +311,6 @@ emqx_collect(emqx_packets_publish_auth_error, Metrics) ->
counter_metric(?C('packets.publish.auth_error', Metrics)); counter_metric(?C('packets.publish.auth_error', Metrics));
emqx_collect(emqx_packets_publish_dropped, Metrics) -> emqx_collect(emqx_packets_publish_dropped, Metrics) ->
counter_metric(?C('packets.publish.dropped', Metrics)); counter_metric(?C('packets.publish.dropped', Metrics));
%% puback %% puback
emqx_collect(emqx_packets_puback_received, Metrics) -> emqx_collect(emqx_packets_puback_received, Metrics) ->
counter_metric(?C('packets.puback.received', Metrics)); counter_metric(?C('packets.puback.received', Metrics));
@ -318,7 +320,6 @@ emqx_collect(emqx_packets_puback_inuse, Metrics) ->
counter_metric(?C('packets.puback.inuse', Metrics)); counter_metric(?C('packets.puback.inuse', Metrics));
emqx_collect(emqx_packets_puback_missed, Metrics) -> emqx_collect(emqx_packets_puback_missed, Metrics) ->
counter_metric(?C('packets.puback.missed', Metrics)); counter_metric(?C('packets.puback.missed', Metrics));
%% pubrec %% pubrec
emqx_collect(emqx_packets_pubrec_received, Metrics) -> emqx_collect(emqx_packets_pubrec_received, Metrics) ->
counter_metric(?C('packets.pubrec.received', Metrics)); counter_metric(?C('packets.pubrec.received', Metrics));
@ -328,7 +329,6 @@ emqx_collect(emqx_packets_pubrec_inuse, Metrics) ->
counter_metric(?C('packets.pubrec.inuse', Metrics)); counter_metric(?C('packets.pubrec.inuse', Metrics));
emqx_collect(emqx_packets_pubrec_missed, Metrics) -> emqx_collect(emqx_packets_pubrec_missed, Metrics) ->
counter_metric(?C('packets.pubrec.missed', Metrics)); counter_metric(?C('packets.pubrec.missed', Metrics));
%% pubrel %% pubrel
emqx_collect(emqx_packets_pubrel_received, Metrics) -> emqx_collect(emqx_packets_pubrel_received, Metrics) ->
counter_metric(?C('packets.pubrel.received', Metrics)); counter_metric(?C('packets.pubrel.received', Metrics));
@ -336,7 +336,6 @@ emqx_collect(emqx_packets_pubrel_sent, Metrics) ->
counter_metric(?C('packets.pubrel.sent', Metrics)); counter_metric(?C('packets.pubrel.sent', Metrics));
emqx_collect(emqx_packets_pubrel_missed, Metrics) -> emqx_collect(emqx_packets_pubrel_missed, Metrics) ->
counter_metric(?C('packets.pubrel.missed', Metrics)); counter_metric(?C('packets.pubrel.missed', Metrics));
%% pubcomp %% pubcomp
emqx_collect(emqx_packets_pubcomp_received, Metrics) -> emqx_collect(emqx_packets_pubcomp_received, Metrics) ->
counter_metric(?C('packets.pubcomp.received', Metrics)); counter_metric(?C('packets.pubcomp.received', Metrics));
@ -346,77 +345,59 @@ emqx_collect(emqx_packets_pubcomp_inuse, Metrics) ->
counter_metric(?C('packets.pubcomp.inuse', Metrics)); counter_metric(?C('packets.pubcomp.inuse', Metrics));
emqx_collect(emqx_packets_pubcomp_missed, Metrics) -> emqx_collect(emqx_packets_pubcomp_missed, Metrics) ->
counter_metric(?C('packets.pubcomp.missed', Metrics)); counter_metric(?C('packets.pubcomp.missed', Metrics));
%% pingreq %% pingreq
emqx_collect(emqx_packets_pingreq_received, Metrics) -> emqx_collect(emqx_packets_pingreq_received, Metrics) ->
counter_metric(?C('packets.pingreq.received', Metrics)); counter_metric(?C('packets.pingreq.received', Metrics));
emqx_collect(emqx_packets_pingresp_sent, Metrics) -> emqx_collect(emqx_packets_pingresp_sent, Metrics) ->
counter_metric(?C('packets.pingresp.sent', Metrics)); counter_metric(?C('packets.pingresp.sent', Metrics));
%% disconnect %% disconnect
emqx_collect(emqx_packets_disconnect_received, Metrics) -> emqx_collect(emqx_packets_disconnect_received, Metrics) ->
counter_metric(?C('packets.disconnect.received', Metrics)); counter_metric(?C('packets.disconnect.received', Metrics));
emqx_collect(emqx_packets_disconnect_sent, Metrics) -> emqx_collect(emqx_packets_disconnect_sent, Metrics) ->
counter_metric(?C('packets.disconnect.sent', Metrics)); counter_metric(?C('packets.disconnect.sent', Metrics));
%% auth %% auth
emqx_collect(emqx_packets_auth_received, Metrics) -> emqx_collect(emqx_packets_auth_received, Metrics) ->
counter_metric(?C('packets.auth.received', Metrics)); counter_metric(?C('packets.auth.received', Metrics));
emqx_collect(emqx_packets_auth_sent, Metrics) -> emqx_collect(emqx_packets_auth_sent, Metrics) ->
counter_metric(?C('packets.auth.sent', Metrics)); counter_metric(?C('packets.auth.sent', Metrics));
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% Metrics - messages %% Metrics - messages
%% messages %% messages
emqx_collect(emqx_messages_received, Metrics) -> emqx_collect(emqx_messages_received, Metrics) ->
counter_metric(?C('messages.received', Metrics)); counter_metric(?C('messages.received', Metrics));
emqx_collect(emqx_messages_sent, Metrics) -> emqx_collect(emqx_messages_sent, Metrics) ->
counter_metric(?C('messages.sent', Metrics)); counter_metric(?C('messages.sent', Metrics));
emqx_collect(emqx_messages_qos0_received, Metrics) -> emqx_collect(emqx_messages_qos0_received, Metrics) ->
counter_metric(?C('messages.qos0.received', Metrics)); counter_metric(?C('messages.qos0.received', Metrics));
emqx_collect(emqx_messages_qos0_sent, Metrics) -> emqx_collect(emqx_messages_qos0_sent, Metrics) ->
counter_metric(?C('messages.qos0.sent', Metrics)); counter_metric(?C('messages.qos0.sent', Metrics));
emqx_collect(emqx_messages_qos1_received, Metrics) -> emqx_collect(emqx_messages_qos1_received, Metrics) ->
counter_metric(?C('messages.qos1.received', Metrics)); counter_metric(?C('messages.qos1.received', Metrics));
emqx_collect(emqx_messages_qos1_sent, Metrics) -> emqx_collect(emqx_messages_qos1_sent, Metrics) ->
counter_metric(?C('messages.qos1.sent', Metrics)); counter_metric(?C('messages.qos1.sent', Metrics));
emqx_collect(emqx_messages_qos2_received, Metrics) -> emqx_collect(emqx_messages_qos2_received, Metrics) ->
counter_metric(?C('messages.qos2.received', Metrics)); counter_metric(?C('messages.qos2.received', Metrics));
emqx_collect(emqx_messages_qos2_sent, Metrics) -> emqx_collect(emqx_messages_qos2_sent, Metrics) ->
counter_metric(?C('messages.qos2.sent', Metrics)); counter_metric(?C('messages.qos2.sent', Metrics));
emqx_collect(emqx_messages_publish, Metrics) -> emqx_collect(emqx_messages_publish, Metrics) ->
counter_metric(?C('messages.publish', Metrics)); counter_metric(?C('messages.publish', Metrics));
emqx_collect(emqx_messages_dropped, Metrics) -> emqx_collect(emqx_messages_dropped, Metrics) ->
counter_metric(?C('messages.dropped', Metrics)); counter_metric(?C('messages.dropped', Metrics));
emqx_collect(emqx_messages_dropped_expired, Metrics) -> emqx_collect(emqx_messages_dropped_expired, Metrics) ->
counter_metric(?C('messages.dropped.await_pubrel_timeout', Metrics)); counter_metric(?C('messages.dropped.await_pubrel_timeout', Metrics));
emqx_collect(emqx_messages_dropped_no_subscribers, Metrics) -> emqx_collect(emqx_messages_dropped_no_subscribers, Metrics) ->
counter_metric(?C('messages.dropped.no_subscribers', Metrics)); counter_metric(?C('messages.dropped.no_subscribers', Metrics));
emqx_collect(emqx_messages_forward, Metrics) -> emqx_collect(emqx_messages_forward, Metrics) ->
counter_metric(?C('messages.forward', Metrics)); counter_metric(?C('messages.forward', Metrics));
emqx_collect(emqx_messages_retained, Metrics) -> emqx_collect(emqx_messages_retained, Metrics) ->
counter_metric(?C('messages.retained', Metrics)); counter_metric(?C('messages.retained', Metrics));
emqx_collect(emqx_messages_delayed, Stats) -> emqx_collect(emqx_messages_delayed, Stats) ->
counter_metric(?C('messages.delayed', Stats)); counter_metric(?C('messages.delayed', Stats));
emqx_collect(emqx_messages_delivered, Stats) -> emqx_collect(emqx_messages_delivered, Stats) ->
counter_metric(?C('messages.delivered', Stats)); counter_metric(?C('messages.delivered', Stats));
emqx_collect(emqx_messages_acked, Stats) -> emqx_collect(emqx_messages_acked, Stats) ->
counter_metric(?C('messages.acked', Stats)); counter_metric(?C('messages.acked', Stats));
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% Metrics - delivery %% Metrics - delivery
@ -432,7 +413,6 @@ emqx_collect(emqx_delivery_dropped_queue_full, Stats) ->
counter_metric(?C('delivery.dropped.queue_full', Stats)); counter_metric(?C('delivery.dropped.queue_full', Stats));
emqx_collect(emqx_delivery_dropped_expired, Stats) -> emqx_collect(emqx_delivery_dropped_expired, Stats) ->
counter_metric(?C('delivery.dropped.expired', Stats)); counter_metric(?C('delivery.dropped.expired', Stats));
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% Metrics - client %% Metrics - client
@ -450,7 +430,6 @@ emqx_collect(emqx_client_unsubscribe, Stats) ->
counter_metric(?C('client.unsubscribe', Stats)); counter_metric(?C('client.unsubscribe', Stats));
emqx_collect(emqx_client_disconnected, Stats) -> emqx_collect(emqx_client_disconnected, Stats) ->
counter_metric(?C('client.disconnected', Stats)); counter_metric(?C('client.disconnected', Stats));
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% Metrics - session %% Metrics - session
@ -464,31 +443,23 @@ emqx_collect(emqx_session_discarded, Stats) ->
counter_metric(?C('session.discarded', Stats)); counter_metric(?C('session.discarded', Stats));
emqx_collect(emqx_session_terminated, Stats) -> emqx_collect(emqx_session_terminated, Stats) ->
counter_metric(?C('session.terminated', Stats)); counter_metric(?C('session.terminated', Stats));
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% VM %% VM
emqx_collect(emqx_vm_cpu_use, VMData) -> emqx_collect(emqx_vm_cpu_use, VMData) ->
gauge_metric(?C(cpu_use, VMData)); gauge_metric(?C(cpu_use, VMData));
emqx_collect(emqx_vm_cpu_idle, VMData) -> emqx_collect(emqx_vm_cpu_idle, VMData) ->
gauge_metric(?C(cpu_idle, VMData)); gauge_metric(?C(cpu_idle, VMData));
emqx_collect(emqx_vm_run_queue, VMData) -> emqx_collect(emqx_vm_run_queue, VMData) ->
gauge_metric(?C(run_queue, VMData)); gauge_metric(?C(run_queue, VMData));
emqx_collect(emqx_vm_process_messages_in_queues, VMData) -> emqx_collect(emqx_vm_process_messages_in_queues, VMData) ->
gauge_metric(?C(process_total_messages, VMData)); gauge_metric(?C(process_total_messages, VMData));
emqx_collect(emqx_vm_total_memory, VMData) -> emqx_collect(emqx_vm_total_memory, VMData) ->
gauge_metric(?C(total_memory, VMData)); gauge_metric(?C(total_memory, VMData));
emqx_collect(emqx_vm_used_memory, VMData) -> emqx_collect(emqx_vm_used_memory, VMData) ->
gauge_metric(?C(used_memory, VMData)); gauge_metric(?C(used_memory, VMData));
emqx_collect(emqx_cluster_nodes_running, ClusterData) -> emqx_collect(emqx_cluster_nodes_running, ClusterData) ->
gauge_metric(?C(nodes_running, ClusterData)); gauge_metric(?C(nodes_running, ClusterData));
emqx_collect(emqx_cluster_nodes_stopped, ClusterData) -> emqx_collect(emqx_cluster_nodes_stopped, ClusterData) ->
gauge_metric(?C(nodes_stopped, ClusterData)). gauge_metric(?C(nodes_stopped, ClusterData)).
@ -497,142 +468,157 @@ emqx_collect(emqx_cluster_nodes_stopped, ClusterData) ->
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
emqx_stats() -> emqx_stats() ->
[ emqx_connections_count [
, emqx_connections_max emqx_connections_count,
, emqx_sessions_count emqx_connections_max,
, emqx_sessions_max emqx_sessions_count,
, emqx_topics_count emqx_sessions_max,
, emqx_topics_max emqx_topics_count,
, emqx_suboptions_count emqx_topics_max,
, emqx_suboptions_max emqx_suboptions_count,
, emqx_subscribers_count emqx_suboptions_max,
, emqx_subscribers_max emqx_subscribers_count,
, emqx_subscriptions_count emqx_subscribers_max,
, emqx_subscriptions_max emqx_subscriptions_count,
, emqx_subscriptions_shared_count emqx_subscriptions_max,
, emqx_subscriptions_shared_max emqx_subscriptions_shared_count,
, emqx_retained_count emqx_subscriptions_shared_max,
, emqx_retained_max emqx_retained_count,
emqx_retained_max
]. ].
emqx_metrics_packets() -> emqx_metrics_packets() ->
[ emqx_bytes_received [
, emqx_bytes_sent emqx_bytes_received,
, emqx_packets_received emqx_bytes_sent,
, emqx_packets_sent emqx_packets_received,
, emqx_packets_connect emqx_packets_sent,
, emqx_packets_connack_sent emqx_packets_connect,
, emqx_packets_connack_error emqx_packets_connack_sent,
, emqx_packets_connack_auth_error emqx_packets_connack_error,
, emqx_packets_publish_received emqx_packets_connack_auth_error,
, emqx_packets_publish_sent emqx_packets_publish_received,
, emqx_packets_publish_inuse emqx_packets_publish_sent,
, emqx_packets_publish_error emqx_packets_publish_inuse,
, emqx_packets_publish_auth_error emqx_packets_publish_error,
, emqx_packets_publish_dropped emqx_packets_publish_auth_error,
, emqx_packets_puback_received emqx_packets_publish_dropped,
, emqx_packets_puback_sent emqx_packets_puback_received,
, emqx_packets_puback_inuse emqx_packets_puback_sent,
, emqx_packets_puback_missed emqx_packets_puback_inuse,
, emqx_packets_pubrec_received emqx_packets_puback_missed,
, emqx_packets_pubrec_sent emqx_packets_pubrec_received,
, emqx_packets_pubrec_inuse emqx_packets_pubrec_sent,
, emqx_packets_pubrec_missed emqx_packets_pubrec_inuse,
, emqx_packets_pubrel_received emqx_packets_pubrec_missed,
, emqx_packets_pubrel_sent emqx_packets_pubrel_received,
, emqx_packets_pubrel_missed emqx_packets_pubrel_sent,
, emqx_packets_pubcomp_received emqx_packets_pubrel_missed,
, emqx_packets_pubcomp_sent emqx_packets_pubcomp_received,
, emqx_packets_pubcomp_inuse emqx_packets_pubcomp_sent,
, emqx_packets_pubcomp_missed emqx_packets_pubcomp_inuse,
, emqx_packets_subscribe_received emqx_packets_pubcomp_missed,
, emqx_packets_subscribe_error emqx_packets_subscribe_received,
, emqx_packets_subscribe_auth_error emqx_packets_subscribe_error,
, emqx_packets_suback_sent emqx_packets_subscribe_auth_error,
, emqx_packets_unsubscribe_received emqx_packets_suback_sent,
, emqx_packets_unsubscribe_error emqx_packets_unsubscribe_received,
, emqx_packets_unsuback_sent emqx_packets_unsubscribe_error,
, emqx_packets_pingreq_received emqx_packets_unsuback_sent,
, emqx_packets_pingresp_sent emqx_packets_pingreq_received,
, emqx_packets_disconnect_received emqx_packets_pingresp_sent,
, emqx_packets_disconnect_sent emqx_packets_disconnect_received,
, emqx_packets_auth_received emqx_packets_disconnect_sent,
, emqx_packets_auth_sent emqx_packets_auth_received,
emqx_packets_auth_sent
]. ].
emqx_metrics_messages() -> emqx_metrics_messages() ->
[ emqx_messages_received [
, emqx_messages_sent emqx_messages_received,
, emqx_messages_qos0_received emqx_messages_sent,
, emqx_messages_qos0_sent emqx_messages_qos0_received,
, emqx_messages_qos1_received emqx_messages_qos0_sent,
, emqx_messages_qos1_sent emqx_messages_qos1_received,
, emqx_messages_qos2_received emqx_messages_qos1_sent,
, emqx_messages_qos2_sent emqx_messages_qos2_received,
, emqx_messages_publish emqx_messages_qos2_sent,
, emqx_messages_dropped emqx_messages_publish,
, emqx_messages_dropped_expired emqx_messages_dropped,
, emqx_messages_dropped_no_subscribers emqx_messages_dropped_expired,
, emqx_messages_forward emqx_messages_dropped_no_subscribers,
, emqx_messages_retained emqx_messages_forward,
, emqx_messages_delayed emqx_messages_retained,
, emqx_messages_delivered emqx_messages_delayed,
, emqx_messages_acked emqx_messages_delivered,
emqx_messages_acked
]. ].
emqx_metrics_delivery() -> emqx_metrics_delivery() ->
[ emqx_delivery_dropped [
, emqx_delivery_dropped_no_local emqx_delivery_dropped,
, emqx_delivery_dropped_too_large emqx_delivery_dropped_no_local,
, emqx_delivery_dropped_qos0_msg emqx_delivery_dropped_too_large,
, emqx_delivery_dropped_queue_full emqx_delivery_dropped_qos0_msg,
, emqx_delivery_dropped_expired emqx_delivery_dropped_queue_full,
emqx_delivery_dropped_expired
]. ].
emqx_metrics_client() -> emqx_metrics_client() ->
[ emqx_client_connected [
, emqx_client_authenticate emqx_client_connected,
, emqx_client_auth_anonymous emqx_client_authenticate,
, emqx_client_authorize emqx_client_auth_anonymous,
, emqx_client_subscribe emqx_client_authorize,
, emqx_client_unsubscribe emqx_client_subscribe,
, emqx_client_disconnected emqx_client_unsubscribe,
emqx_client_disconnected
]. ].
emqx_metrics_session() -> emqx_metrics_session() ->
[ emqx_session_created [
, emqx_session_resumed emqx_session_created,
, emqx_session_takenover emqx_session_resumed,
, emqx_session_discarded emqx_session_takenover,
, emqx_session_terminated emqx_session_discarded,
emqx_session_terminated
]. ].
emqx_vm() -> emqx_vm() ->
[ emqx_vm_cpu_use [
, emqx_vm_cpu_idle emqx_vm_cpu_use,
, emqx_vm_run_queue emqx_vm_cpu_idle,
, emqx_vm_process_messages_in_queues emqx_vm_run_queue,
, emqx_vm_total_memory emqx_vm_process_messages_in_queues,
, emqx_vm_used_memory emqx_vm_total_memory,
emqx_vm_used_memory
]. ].
emqx_vm_data() -> emqx_vm_data() ->
Idle = case cpu_sup:util([detailed]) of Idle =
{_, 0, 0, _} -> 0; %% Not support for Windows case cpu_sup:util([detailed]) of
{_Num, _Use, IdleList, _} -> ?C(idle, IdleList) %% Not support for Windows
end, {_, 0, 0, _} -> 0;
{_Num, _Use, IdleList, _} -> ?C(idle, IdleList)
end,
RunQueue = erlang:statistics(run_queue), RunQueue = erlang:statistics(run_queue),
[{run_queue, RunQueue}, [
{process_total_messages, 0}, %% XXX: Plan removed at v5.0 {run_queue, RunQueue},
{cpu_idle, Idle}, %% XXX: Plan removed at v5.0
{cpu_use, 100 - Idle}] ++ emqx_vm:mem_info(). {process_total_messages, 0},
{cpu_idle, Idle},
{cpu_use, 100 - Idle}
] ++ emqx_vm:mem_info().
emqx_cluster() -> emqx_cluster() ->
[ emqx_cluster_nodes_running [
, emqx_cluster_nodes_stopped emqx_cluster_nodes_running,
emqx_cluster_nodes_stopped
]. ].
emqx_cluster_data() -> emqx_cluster_data() ->
#{running_nodes := Running, stopped_nodes := Stopped} = mria_mnesia:cluster_info(), #{running_nodes := Running, stopped_nodes := Stopped} = mria_mnesia:cluster_info(),
[{nodes_running, length(Running)}, [
{nodes_stopped, length(Stopped)}]. {nodes_running, length(Running)},
{nodes_stopped, length(Stopped)}
].

93
apps/emqx_prometheus/src/emqx_prometheus_api.erl
View File

@ -22,14 +22,16 @@
-import(hoconsc, [ref/2]). -import(hoconsc, [ref/2]).
-export([ api_spec/0 -export([
, paths/0 api_spec/0,
, schema/1 paths/0,
]). schema/1
]).
-export([ prometheus/2 -export([
, stats/2 prometheus/2,
]). stats/2
]).
-define(SCHEMA_MODULE, emqx_prometheus_schema). -define(SCHEMA_MODULE, emqx_prometheus_schema).
@ -37,32 +39,38 @@ api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
paths() -> paths() ->
[ "/prometheus" [
, "/prometheus/stats" "/prometheus",
"/prometheus/stats"
]. ].
schema("/prometheus") -> schema("/prometheus") ->
#{ 'operationId' => prometheus #{
, get => 'operationId' => prometheus,
#{ description => <<"Get Prometheus config info">> get =>
, responses => #{
#{200 => prometheus_config_schema()} description => <<"Get Prometheus config info">>,
responses =>
#{200 => prometheus_config_schema()}
},
put =>
#{
description => <<"Update Prometheus config">>,
'requestBody' => prometheus_config_schema(),
responses =>
#{200 => prometheus_config_schema()}
} }
, put => };
#{ description => <<"Update Prometheus config">>
, 'requestBody' => prometheus_config_schema()
, responses =>
#{200 => prometheus_config_schema()}
}
};
schema("/prometheus/stats") -> schema("/prometheus/stats") ->
#{ 'operationId' => stats #{
, get => 'operationId' => stats,
#{ description => <<"Get Prometheus Data">> get =>
, responses => #{
#{200 => prometheus_data_schema()} description => <<"Get Prometheus Data">>,
responses =>
#{200 => prometheus_data_schema()}
} }
}. }.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% API Handler funcs %% API Handler funcs
@ -70,7 +78,6 @@ schema("/prometheus/stats") ->
prometheus(get, _Params) -> prometheus(get, _Params) ->
{200, emqx:get_raw_config([<<"prometheus">>], #{})}; {200, emqx:get_raw_config([<<"prometheus">>], #{})};
prometheus(put, #{body := Body}) -> prometheus(put, #{body := Body}) ->
case emqx_prometheus:update(Body) of case emqx_prometheus:update(Body) of
{ok, NewConfig} -> {ok, NewConfig} ->
@ -100,21 +107,25 @@ stats(get, #{headers := Headers}) ->
prometheus_config_schema() -> prometheus_config_schema() ->
emqx_dashboard_swagger:schema_with_example( emqx_dashboard_swagger:schema_with_example(
ref(?SCHEMA_MODULE, "prometheus"), ref(?SCHEMA_MODULE, "prometheus"),
prometheus_config_example()). prometheus_config_example()
).
prometheus_config_example() -> prometheus_config_example() ->
#{ enable => true #{
, interval => "15s" enable => true,
, push_gateway_server => <<"http://127.0.0.1:9091">> interval => "15s",
}. push_gateway_server => <<"http://127.0.0.1:9091">>
}.
prometheus_data_schema() -> prometheus_data_schema() ->
#{ description => <<"Get Prometheus Data">> #{
, content => description => <<"Get Prometheus Data">>,
#{ 'application/json' => content =>
#{schema => #{type => object}} #{
, 'text/plain' => 'application/json' =>
#{schema => #{type => string}} #{schema => #{type => object}},
'text/plain' =>
#{schema => #{type => string}}
} }
}. }.

7
apps/emqx_prometheus/src/emqx_prometheus_app.erl
View File

@ -21,9 +21,10 @@
-include("emqx_prometheus.hrl"). -include("emqx_prometheus.hrl").
%% Application callbacks %% Application callbacks
-export([ start/2 -export([
, stop/1 start/2,
]). stop/1
]).
start(_StartType, _StartArgs) -> start(_StartType, _StartArgs) ->
{ok, Sup} = emqx_prometheus_sup:start_link(), {ok, Sup} = emqx_prometheus_sup:start_link(),

61
apps/emqx_prometheus/src/emqx_prometheus_mria.erl
View File

@ -15,9 +15,10 @@
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-module(emqx_prometheus_mria). -module(emqx_prometheus_mria).
-export([deregister_cleanup/1, -export([
collect_mf/2 deregister_cleanup/1,
]). collect_mf/2
]).
-include_lib("prometheus/include/prometheus.hrl"). -include_lib("prometheus/include/prometheus.hrl").
@ -43,39 +44,45 @@ deregister_cleanup(_) -> ok.
_Registry :: prometheus_registry:registry(), _Registry :: prometheus_registry:registry(),
Callback :: prometheus_collector:callback(). Callback :: prometheus_collector:callback().
collect_mf(_Registry, Callback) -> collect_mf(_Registry, Callback) ->
case mria_rlog:backend() of case mria_rlog:backend() of
rlog -> rlog ->
Metrics = metrics(), Metrics = metrics(),
_ = [add_metric_family(Metric, Callback) || Metric <- Metrics], _ = [add_metric_family(Metric, Callback) || Metric <- Metrics],
ok; ok;
mnesia -> mnesia ->
ok ok
end. end.
add_metric_family({Name, Metrics}, Callback) -> add_metric_family({Name, Metrics}, Callback) ->
Callback(prometheus_model_helpers:create_mf( ?METRIC_NAME(Name) Callback(
, <<"">> prometheus_model_helpers:create_mf(
, gauge ?METRIC_NAME(Name),
, catch_all(Metrics) <<"">>,
)). gauge,
catch_all(Metrics)
)
).
%%==================================================================== %%====================================================================
%% Internal functions %% Internal functions
%%==================================================================== %%====================================================================
metrics() -> metrics() ->
Metrics = case mria_rlog:role() of Metrics =
replicant -> case mria_rlog:role() of
[lag, bootstrap_time, bootstrap_num_keys, message_queue_len, replayq_len]; replicant ->
core -> [lag, bootstrap_time, bootstrap_num_keys, message_queue_len, replayq_len];
[last_intercepted_trans, weight, replicants, server_mql] core ->
end, [last_intercepted_trans, weight, replicants, server_mql]
end,
[{MetricId, fun() -> get_shard_metric(MetricId) end} || MetricId <- Metrics]. [{MetricId, fun() -> get_shard_metric(MetricId) end} || MetricId <- Metrics].
get_shard_metric(Metric) -> get_shard_metric(Metric) ->
%% TODO: only report shards that are up %% TODO: only report shards that are up
[{[{shard, Shard}], get_shard_metric(Metric, Shard)} || [
Shard <- mria_schema:shards(), Shard =/= undefined]. {[{shard, Shard}], get_shard_metric(Metric, Shard)}
|| Shard <- mria_schema:shards(), Shard =/= undefined
].
get_shard_metric(replicants, Shard) -> get_shard_metric(replicants, Shard) ->
length(mria_status:agents(Shard)); length(mria_status:agents(Shard));
@ -88,6 +95,8 @@ get_shard_metric(Metric, Shard) ->
end. end.
catch_all(DataFun) -> catch_all(DataFun) ->
try DataFun() try
catch _:_ -> undefined DataFun()
catch
_:_ -> undefined
end. end.

56
apps/emqx_prometheus/src/emqx_prometheus_schema.erl
View File

@ -20,11 +20,12 @@
-behaviour(hocon_schema). -behaviour(hocon_schema).
-export([ namespace/0 -export([
, roots/0 namespace/0,
, fields/1 roots/0,
, desc/1 fields/1,
]). desc/1
]).
namespace() -> "prometheus". namespace() -> "prometheus".
@ -32,25 +33,36 @@ roots() -> ["prometheus"].
fields("prometheus") -> fields("prometheus") ->
[ [
{push_gateway_server, sc(string(), {push_gateway_server,
#{ default => "http://127.0.0.1:9091" sc(
, required => true string(),
, desc => ?DESC(push_gateway_server) #{
})}, default => "http://127.0.0.1:9091",
{interval, sc(emqx_schema:duration_ms(), required => true,
#{ default => "15s" desc => ?DESC(push_gateway_server)
, required => true }
, desc => ?DESC(interval) )},
})}, {interval,
{enable, sc(boolean(), sc(
#{ default => false emqx_schema:duration_ms(),
, required => true #{
, desc => ?DESC(enable) default => "15s",
})} required => true,
desc => ?DESC(interval)
}
)},
{enable,
sc(
boolean(),
#{
default => false,
required => true,
desc => ?DESC(enable)
}
)}
]. ].
desc("prometheus") -> ?DESC(prometheus); desc("prometheus") -> ?DESC(prometheus);
desc(_) -> desc(_) -> undefined.
undefined.
sc(Type, Meta) -> hoconsc:mk(Type, Meta). sc(Type, Meta) -> hoconsc:mk(Type, Meta).

27
apps/emqx_prometheus/src/emqx_prometheus_sup.erl
View File

@ -18,21 +18,24 @@
-behaviour(supervisor). -behaviour(supervisor).
-export([ start_link/0 -export([
, start_child/1 start_link/0,
, start_child/2 start_child/1,
, stop_child/1 start_child/2,
]). stop_child/1
]).
-export([init/1]). -export([init/1]).
%% Helper macro for declaring children of supervisor %% Helper macro for declaring children of supervisor
-define(CHILD(Mod, Opts), #{id => Mod, -define(CHILD(Mod, Opts), #{
start => {Mod, start_link, [Opts]}, id => Mod,
restart => permanent, start => {Mod, start_link, [Opts]},
shutdown => 5000, restart => permanent,
type => worker, shutdown => 5000,
modules => [Mod]}). type => worker,
modules => [Mod]
}).
start_link() -> start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []). supervisor:start_link({local, ?MODULE}, ?MODULE, []).
@ -45,7 +48,7 @@ start_child(ChildSpec) when is_map(ChildSpec) ->
start_child(Mod, Opts) when is_atom(Mod) andalso is_map(Opts) -> start_child(Mod, Opts) when is_atom(Mod) andalso is_map(Opts) ->
assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Opts))). assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Opts))).
-spec(stop_child(any()) -> ok | {error, term()}). -spec stop_child(any()) -> ok | {error, term()}.
stop_child(ChildId) -> stop_child(ChildId) ->
case supervisor:terminate_child(?MODULE, ChildId) of case supervisor:terminate_child(?MODULE, ChildId) of
ok -> supervisor:delete_child(?MODULE, ChildId); ok -> supervisor:delete_child(?MODULE, ChildId);

9
apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl
View File

@ -18,11 +18,12 @@
-behaviour(emqx_bpapi). -behaviour(emqx_bpapi).
-export([ introduced_in/0 -export([
introduced_in/0,
, start/1 start/1,
, stop/1 stop/1
]). ]).
-include_lib("emqx/include/bpapi.hrl"). -include_lib("emqx/include/bpapi.hrl").

15
apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl
View File

@ -22,13 +22,14 @@
-compile(export_all). -compile(export_all).
-define(CLUSTER_RPC_SHARD, emqx_cluster_rpc_shard). -define(CLUSTER_RPC_SHARD, emqx_cluster_rpc_shard).
-define(CONF_DEFAULT, <<" -define(CONF_DEFAULT,
prometheus { <<"\n"
push_gateway_server = \"http://127.0.0.1:9091\" "prometheus {\n"
interval = \"1s\" " push_gateway_server = \"http://127.0.0.1:9091\"\n"
enable = true " interval = \"1s\"\n"
} " enable = true\n"
">>). "}\n">>
).
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% Setups %% Setups

11
apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl
View File

@ -67,9 +67,14 @@ t_prometheus_api(_) ->
{ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, "", Auth), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, "", Auth),
Conf = emqx_json:decode(Response, [return_maps]), Conf = emqx_json:decode(Response, [return_maps]),
?assertMatch(#{<<"push_gateway_server">> := _, ?assertMatch(
<<"interval">> := _, #{
<<"enable">> := _}, Conf), <<"push_gateway_server">> := _,
<<"interval">> := _,
<<"enable">> := _
},
Conf
),
NewConf = Conf#{<<"interval">> := <<"2s">>}, NewConf = Conf#{<<"interval">> := <<"2s">>},
{ok, Response2} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf), {ok, Response2} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf),

13
apps/emqx_resource/include/emqx_resource.hrl
View File

@ -30,12 +30,13 @@
}. }.
-type resource_group() :: binary(). -type resource_group() :: binary().
-type create_opts() :: #{ -type create_opts() :: #{
health_check_interval => integer(), health_check_interval => integer(),
health_check_timeout => integer(), health_check_timeout => integer(),
waiting_connect_complete => integer() waiting_connect_complete => integer()
}. }.
-type after_query() :: {[OnSuccess :: after_query_fun()], [OnFailed :: after_query_fun()]} | -type after_query() ::
undefined. {[OnSuccess :: after_query_fun()], [OnFailed :: after_query_fun()]}
| undefined.
%% the `after_query_fun()` is mainly for callbacks that increment counters or do some fallback %% the `after_query_fun()` is mainly for callbacks that increment counters or do some fallback
%% actions upon query failure %% actions upon query failure

16
apps/emqx_resource/include/emqx_resource_utils.hrl
View File

@ -15,13 +15,17 @@
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-define(SAFE_CALL(_EXP_), -define(SAFE_CALL(_EXP_),
?SAFE_CALL(_EXP_, ok)). ?SAFE_CALL(_EXP_, ok)
).
-define(SAFE_CALL(_EXP_, _EXP_ON_FAIL_), -define(SAFE_CALL(_EXP_, _EXP_ON_FAIL_),
fun() -> fun() ->
try (_EXP_) try
catch _EXCLASS_:_EXCPTION_:_ST_ -> (_EXP_)
catch
_EXCLASS_:_EXCPTION_:_ST_ ->
_EXP_ON_FAIL_, _EXP_ON_FAIL_,
{error, {_EXCLASS_, _EXCPTION_, _ST_}} {error, {_EXCLASS_, _EXCPTION_, _ST_}}
end end
end()). end()
).

21
apps/emqx_resource/rebar.config
View File

@ -1,9 +1,10 @@
%% -*- mode: erlang -*- %% -*- mode: erlang -*-
{erl_opts, [ debug_info {erl_opts, [
, nowarn_unused_import debug_info,
%, {d, 'RESOURCE_DEBUG'} nowarn_unused_import
]}. %, {d, 'RESOURCE_DEBUG'}
]}.
{erl_first_files, ["src/emqx_resource_transform.erl"]}. {erl_first_files, ["src/emqx_resource_transform.erl"]}.
@ -11,9 +12,11 @@
%% try to override the dialyzer 'race_conditions' defined in the top-level dir, %% try to override the dialyzer 'race_conditions' defined in the top-level dir,
%% but it doesn't work %% but it doesn't work
{dialyzer, [{warnings, [unmatched_returns, error_handling]} {dialyzer, [{warnings, [unmatched_returns, error_handling]}]}.
]}.
{deps, [ {jsx, {git, "https://github.com/talentdeficit/jsx", {tag, "v3.1.0"}}} {deps, [
, {emqx, {path, "../emqx"}} {jsx, {git, "https://github.com/talentdeficit/jsx", {tag, "v3.1.0"}}},
]}. {emqx, {path, "../emqx"}}
]}.
{project_plugins, [erlfmt]}.

34
apps/emqx_resource/src/emqx_resource.app.src
View File

@ -1,19 +1,19 @@
%% -*- mode: erlang -*- %% -*- mode: erlang -*-
{application, emqx_resource, {application, emqx_resource, [
[{description, "An OTP application"}, {description, "An OTP application"},
{vsn, "0.1.0"}, {vsn, "0.1.0"},
{registered, []}, {registered, []},
{mod, {emqx_resource_app, []}}, {mod, {emqx_resource_app, []}},
{applications, {applications, [
[kernel, kernel,
stdlib, stdlib,
gproc, gproc,
jsx, jsx,
emqx emqx
]}, ]},
{env,[]}, {env, []},
{modules, []}, {modules, []},
{licenses, ["Apache 2.0"]}, {licenses, ["Apache 2.0"]},
{links, []} {links, []}
]}. ]}.

279
apps/emqx_resource/src/emqx_resource.erl
View File

@ -25,66 +25,93 @@
%% APIs for behaviour implementations %% APIs for behaviour implementations
-export([ query_success/1 -export([
, query_failed/1 query_success/1,
]). query_failed/1
]).
%% APIs for instances %% APIs for instances
-export([ check_config/2 -export([
, check_and_create/4 check_config/2,
, check_and_create/5 check_and_create/4,
, check_and_create_local/4 check_and_create/5,
, check_and_create_local/5 check_and_create_local/4,
, check_and_recreate/4 check_and_create_local/5,
, check_and_recreate_local/4 check_and_recreate/4,
]). check_and_recreate_local/4
]).
%% Sync resource instances and files %% Sync resource instances and files
%% provisional solution: rpc:multicall to all the nodes for creating/updating/removing %% provisional solution: rpc:multicall to all the nodes for creating/updating/removing
%% todo: replicate operations %% todo: replicate operations
-export([ create/4 %% store the config and start the instance
, create/5 %% store the config and start the instance
, create_local/4 -export([
, create_local/5 create/4,
, create_dry_run/2 %% run start/2, health_check/2 and stop/1 sequentially create/5,
, create_dry_run_local/2 create_local/4,
, recreate/4 %% this will do create_dry_run, stop the old instance and start a new one create_local/5,
, recreate_local/4 %% run start/2, health_check/2 and stop/1 sequentially
, remove/1 %% remove the config and stop the instance create_dry_run/2,
, remove_local/1 create_dry_run_local/2,
, reset_metrics/1 %% this will do create_dry_run, stop the old instance and start a new one
, reset_metrics_local/1 recreate/4,
]). recreate_local/4,
%% remove the config and stop the instance
remove/1,
remove_local/1,
reset_metrics/1,
reset_metrics_local/1
]).
%% Calls to the callback module with current resource state %% Calls to the callback module with current resource state
%% They also save the state after the call finished (except query/2,3). %% They also save the state after the call finished (except query/2,3).
-export([ restart/1 %% restart the instance.
, restart/2 %% restart the instance.
, health_check/1 %% verify if the resource is working normally -export([
, set_resource_status_connecting/1 %% set resource status to disconnected restart/1,
, stop/1 %% stop the instance restart/2,
, query/2 %% query the instance %% verify if the resource is working normally
, query/3 %% query the instance with after_query() health_check/1,
]). %% set resource status to disconnected
set_resource_status_connecting/1,
%% stop the instance
stop/1,
%% query the instance
query/2,
%% query the instance with after_query()
query/3
]).
%% Direct calls to the callback module %% Direct calls to the callback module
-export([ call_start/3 %% start the instance
, call_health_check/3 %% verify if the resource is working normally
, call_stop/3 %% stop the instance
]).
-export([ list_instances/0 %% list all the instances, id only. %% start the instance
, list_instances_verbose/0 %% list all the instances -export([
, get_instance/1 %% return the data of the instance call_start/3,
, list_instances_by_type/1 %% return all the instances of the same resource type %% verify if the resource is working normally
, generate_id/1 call_health_check/3,
, list_group_instances/1 %% stop the instance
]). call_stop/3
]).
-optional_callbacks([ on_query/4 %% list all the instances, id only.
, on_health_check/2 -export([
]). list_instances/0,
%% list all the instances
list_instances_verbose/0,
%% return the data of the instance
get_instance/1,
%% return all the instances of the same resource type
list_instances_by_type/1,
generate_id/1,
list_group_instances/1
]).
-optional_callbacks([
on_query/4,
on_health_check/2
]).
%% when calling emqx_resource:start/1 %% when calling emqx_resource:start/1
-callback on_start(instance_id(), resource_config()) -> -callback on_start(instance_id(), resource_config()) ->
@ -98,7 +125,7 @@
%% when calling emqx_resource:health_check/2 %% when calling emqx_resource:health_check/2
-callback on_health_check(instance_id(), resource_state()) -> -callback on_health_check(instance_id(), resource_state()) ->
{ok, resource_state()} | {error, Reason:: term(), resource_state()}. {ok, resource_state()} | {error, Reason :: term(), resource_state()}.
-spec list_types() -> [module()]. -spec list_types() -> [module()].
list_types() -> list_types() ->
@ -111,24 +138,26 @@ discover_resource_mods() ->
-spec is_resource_mod(module()) -> boolean(). -spec is_resource_mod(module()) -> boolean().
is_resource_mod(Module) -> is_resource_mod(Module) ->
Info = Module:module_info(attributes), Info = Module:module_info(attributes),
Behaviour = proplists:get_value(behavior, Info, []) ++ Behaviour =
proplists:get_value(behaviour, Info, []), proplists:get_value(behavior, Info, []) ++
proplists:get_value(behaviour, Info, []),
lists:member(?MODULE, Behaviour). lists:member(?MODULE, Behaviour).
-spec query_success(after_query()) -> ok. -spec query_success(after_query()) -> ok.
query_success(undefined) -> ok; query_success(undefined) -> ok;
query_success({OnSucc, _}) -> query_success({OnSucc, _}) -> apply_query_after_calls(OnSucc).
apply_query_after_calls(OnSucc).
-spec query_failed(after_query()) -> ok. -spec query_failed(after_query()) -> ok.
query_failed(undefined) -> ok; query_failed(undefined) -> ok;
query_failed({_, OnFailed}) -> query_failed({_, OnFailed}) -> apply_query_after_calls(OnFailed).
apply_query_after_calls(OnFailed).
apply_query_after_calls(Funcs) -> apply_query_after_calls(Funcs) ->
lists:foreach(fun({Fun, Args}) -> lists:foreach(
fun({Fun, Args}) ->
safe_apply(Fun, Args) safe_apply(Fun, Args)
end, Funcs). end,
Funcs
).
%% ================================================================================= %% =================================================================================
%% APIs for resource instances %% APIs for resource instances
@ -149,11 +178,13 @@ create(InstId, Group, ResourceType, Config, Opts) ->
create_local(InstId, Group, ResourceType, Config) -> create_local(InstId, Group, ResourceType, Config) ->
create_local(InstId, Group, ResourceType, Config, #{}). create_local(InstId, Group, ResourceType, Config, #{}).
-spec create_local(instance_id(), -spec create_local(
resource_group(), instance_id(),
resource_type(), resource_group(),
resource_config(), resource_type(),
create_opts()) -> resource_config(),
create_opts()
) ->
{ok, resource_data() | 'already_created'} | {error, Reason :: term()}. {ok, resource_data() | 'already_created'} | {error, Reason :: term()}.
create_local(InstId, Group, ResourceType, Config, Opts) -> create_local(InstId, Group, ResourceType, Config, Opts) ->
call_instance(InstId, {create, InstId, Group, ResourceType, Config, Opts}). call_instance(InstId, {create, InstId, Group, ResourceType, Config, Opts}).
@ -206,19 +237,25 @@ query(InstId, Request) ->
query(InstId, Request, AfterQuery) -> query(InstId, Request, AfterQuery) ->
case get_instance(InstId) of case get_instance(InstId) of
{ok, _Group, #{status := connecting}} -> {ok, _Group, #{status := connecting}} ->
query_error(connecting, <<"cannot serve query when the resource " query_error(connecting, <<
"instance is still connecting">>); "cannot serve query when the resource "
"instance is still connecting"
>>);
{ok, _Group, #{status := disconnected}} -> {ok, _Group, #{status := disconnected}} ->
query_error(disconnected, <<"cannot serve query when the resource " query_error(disconnected, <<
"instance is disconnected">>); "cannot serve query when the resource "
"instance is disconnected"
>>);
{ok, _Group, #{mod := Mod, state := ResourceState, status := connected}} -> {ok, _Group, #{mod := Mod, state := ResourceState, status := connected}} ->
%% the resource state is readonly to Module:on_query/4 %% the resource state is readonly to Module:on_query/4
%% and the `after_query()` functions should be thread safe %% and the `after_query()` functions should be thread safe
ok = emqx_plugin_libs_metrics:inc(resource_metrics, InstId, matched), ok = emqx_plugin_libs_metrics:inc(resource_metrics, InstId, matched),
try Mod:on_query(InstId, Request, AfterQuery, ResourceState) try
catch Err:Reason:ST -> Mod:on_query(InstId, Request, AfterQuery, ResourceState)
emqx_plugin_libs_metrics:inc(resource_metrics, InstId, exception), catch
erlang:raise(Err, Reason, ST) Err:Reason:ST ->
emqx_plugin_libs_metrics:inc(resource_metrics, InstId, exception),
erlang:raise(Err, Reason, ST)
end; end;
{error, not_found} -> {error, not_found} ->
query_error(not_found, <<"the resource id not exists">>) query_error(not_found, <<"the resource id not exists">>)
@ -258,9 +295,10 @@ list_instances_verbose() ->
-spec list_instances_by_type(module()) -> [instance_id()]. -spec list_instances_by_type(module()) -> [instance_id()].
list_instances_by_type(ResourceType) -> list_instances_by_type(ResourceType) ->
filter_instances(fun(_, RT) when RT =:= ResourceType -> true; filter_instances(fun
(_, _) -> false (_, RT) when RT =:= ResourceType -> true;
end). (_, _) -> false
end).
-spec generate_id(term()) -> instance_id(). -spec generate_id(term()) -> instance_id().
generate_id(Name) when is_binary(Name) -> generate_id(Name) when is_binary(Name) ->
@ -276,7 +314,9 @@ call_start(InstId, Mod, Config) ->
?SAFE_CALL(Mod:on_start(InstId, Config)). ?SAFE_CALL(Mod:on_start(InstId, Config)).
-spec call_health_check(instance_id(), module(), resource_state()) -> -spec call_health_check(instance_id(), module(), resource_state()) ->
{ok, resource_state()} | {error, Reason:: term()} | {error, Reason:: term(), resource_state()}. {ok, resource_state()}
| {error, Reason :: term()}
| {error, Reason :: term(), resource_state()}.
call_health_check(InstId, Mod, ResourceState) -> call_health_check(InstId, Mod, ResourceState) ->
?SAFE_CALL(Mod:on_health_check(InstId, ResourceState)). ?SAFE_CALL(Mod:on_health_check(InstId, ResourceState)).
@ -289,58 +329,82 @@ call_stop(InstId, Mod, ResourceState) ->
check_config(ResourceType, Conf) -> check_config(ResourceType, Conf) ->
emqx_hocon:check(ResourceType, Conf). emqx_hocon:check(ResourceType, Conf).
-spec check_and_create(instance_id(), -spec check_and_create(
resource_group(), instance_id(),
resource_type(), resource_group(),
raw_resource_config()) -> resource_type(),
raw_resource_config()
) ->
{ok, resource_data() | 'already_created'} | {error, term()}. {ok, resource_data() | 'already_created'} | {error, term()}.
check_and_create(InstId, Group, ResourceType, RawConfig) -> check_and_create(InstId, Group, ResourceType, RawConfig) ->
check_and_create(InstId, Group, ResourceType, RawConfig, #{}). check_and_create(InstId, Group, ResourceType, RawConfig, #{}).
-spec check_and_create(instance_id(), -spec check_and_create(
resource_group(), instance_id(),
resource_type(), resource_group(),
raw_resource_config(), resource_type(),
create_opts()) -> raw_resource_config(),
create_opts()
) ->
{ok, resource_data() | 'already_created'} | {error, term()}. {ok, resource_data() | 'already_created'} | {error, term()}.
check_and_create(InstId, Group, ResourceType, RawConfig, Opts) -> check_and_create(InstId, Group, ResourceType, RawConfig, Opts) ->
check_and_do(ResourceType, RawConfig, check_and_do(
fun(InstConf) -> create(InstId, Group, ResourceType, InstConf, Opts) end). ResourceType,
RawConfig,
fun(InstConf) -> create(InstId, Group, ResourceType, InstConf, Opts) end
).
-spec check_and_create_local(instance_id(), -spec check_and_create_local(
resource_group(), instance_id(),
resource_type(), resource_group(),
raw_resource_config()) -> resource_type(),
raw_resource_config()
) ->
{ok, resource_data()} | {error, term()}. {ok, resource_data()} | {error, term()}.
check_and_create_local(InstId, Group, ResourceType, RawConfig) -> check_and_create_local(InstId, Group, ResourceType, RawConfig) ->
check_and_create_local(InstId, Group, ResourceType, RawConfig, #{}). check_and_create_local(InstId, Group, ResourceType, RawConfig, #{}).
-spec check_and_create_local(instance_id(), -spec check_and_create_local(
resource_group(), instance_id(),
resource_type(), resource_group(),
raw_resource_config(), resource_type(),
create_opts()) -> {ok, resource_data()} | {error, term()}. raw_resource_config(),
create_opts()
) -> {ok, resource_data()} | {error, term()}.
check_and_create_local(InstId, Group, ResourceType, RawConfig, Opts) -> check_and_create_local(InstId, Group, ResourceType, RawConfig, Opts) ->
check_and_do(ResourceType, RawConfig, check_and_do(
fun(InstConf) -> create_local(InstId, Group, ResourceType, InstConf, Opts) end). ResourceType,
RawConfig,
fun(InstConf) -> create_local(InstId, Group, ResourceType, InstConf, Opts) end
).
-spec check_and_recreate(instance_id(), -spec check_and_recreate(
resource_type(), instance_id(),
raw_resource_config(), resource_type(),
create_opts()) -> raw_resource_config(),
create_opts()
) ->
{ok, resource_data()} | {error, term()}. {ok, resource_data()} | {error, term()}.
check_and_recreate(InstId, ResourceType, RawConfig, Opts) -> check_and_recreate(InstId, ResourceType, RawConfig, Opts) ->
check_and_do(ResourceType, RawConfig, check_and_do(
fun(InstConf) -> recreate(InstId, ResourceType, InstConf, Opts) end). ResourceType,
RawConfig,
fun(InstConf) -> recreate(InstId, ResourceType, InstConf, Opts) end
).
-spec check_and_recreate_local(instance_id(), -spec check_and_recreate_local(
resource_type(), instance_id(),
raw_resource_config(), resource_type(),
create_opts()) -> raw_resource_config(),
create_opts()
) ->
{ok, resource_data()} | {error, term()}. {ok, resource_data()} | {error, term()}.
check_and_recreate_local(InstId, ResourceType, RawConfig, Opts) -> check_and_recreate_local(InstId, ResourceType, RawConfig, Opts) ->
check_and_do(ResourceType, RawConfig, check_and_do(
fun(InstConf) -> recreate_local(InstId, ResourceType, InstConf, Opts) end). ResourceType,
RawConfig,
fun(InstConf) -> recreate_local(InstId, ResourceType, InstConf, Opts) end
).
check_and_do(ResourceType, RawConfig, Do) when is_function(Do) -> check_and_do(ResourceType, RawConfig, Do) when is_function(Do) ->
case check_config(ResourceType, RawConfig) of case check_config(ResourceType, RawConfig) of
@ -355,8 +419,7 @@ filter_instances(Filter) ->
inc_metrics_funcs(InstId) -> inc_metrics_funcs(InstId) ->
OnFailed = [{fun emqx_plugin_libs_metrics:inc/3, [resource_metrics, InstId, failed]}], OnFailed = [{fun emqx_plugin_libs_metrics:inc/3, [resource_metrics, InstId, failed]}],
OnSucc = [ {fun emqx_plugin_libs_metrics:inc/3, [resource_metrics, InstId, success]} OnSucc = [{fun emqx_plugin_libs_metrics:inc/3, [resource_metrics, InstId, success]}],
],
{OnSucc, OnFailed}. {OnSucc, OnFailed}.
call_instance(InstId, Query) -> call_instance(InstId, Query) ->

55
apps/emqx_resource/src/emqx_resource_health_check.erl
View File

@ -15,23 +15,29 @@
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-module(emqx_resource_health_check). -module(emqx_resource_health_check).
-export([ start_link/3 -export([
, create_checker/3 start_link/3,
, delete_checker/1 create_checker/3,
]). delete_checker/1
]).
-export([ start_health_check/3 -export([
, health_check_timeout_checker/4 start_health_check/3,
]). health_check_timeout_checker/4
]).
-define(SUP, emqx_resource_health_check_sup). -define(SUP, emqx_resource_health_check_sup).
-define(ID(NAME), {resource_health_check, NAME}). -define(ID(NAME), {resource_health_check, NAME}).
child_spec(Name, Sleep, Timeout) -> child_spec(Name, Sleep, Timeout) ->
#{id => ?ID(Name), #{
start => {?MODULE, start_link, [Name, Sleep, Timeout]}, id => ?ID(Name),
restart => transient, start => {?MODULE, start_link, [Name, Sleep, Timeout]},
shutdown => 5000, type => worker, modules => [?MODULE]}. restart => transient,
shutdown => 5000,
type => worker,
modules => [?MODULE]
}.
start_link(Name, Sleep, Timeout) -> start_link(Name, Sleep, Timeout) ->
Pid = proc_lib:spawn_link(?MODULE, start_health_check, [Name, Sleep, Timeout]), Pid = proc_lib:spawn_link(?MODULE, start_health_check, [Name, Sleep, Timeout]),
@ -42,19 +48,22 @@ create_checker(Name, Sleep, Timeout) ->
create_checker(Name, Sleep, Retry, Timeout) -> create_checker(Name, Sleep, Retry, Timeout) ->
case supervisor:start_child(?SUP, child_spec(Name, Sleep, Timeout)) of case supervisor:start_child(?SUP, child_spec(Name, Sleep, Timeout)) of
{ok, _} -> ok; {ok, _} ->
{error, already_present} -> ok; ok;
{error, already_present} ->
ok;
{error, {already_started, _}} when Retry == false -> {error, {already_started, _}} when Retry == false ->
ok = delete_checker(Name), ok = delete_checker(Name),
create_checker(Name, Sleep, true, Timeout); create_checker(Name, Sleep, true, Timeout);
Error -> Error Error ->
Error
end. end.
delete_checker(Name) -> delete_checker(Name) ->
case supervisor:terminate_child(?SUP, ?ID(Name)) of case supervisor:terminate_child(?SUP, ?ID(Name)) of
ok -> supervisor:delete_child(?SUP, ?ID(Name)); ok -> supervisor:delete_child(?SUP, ?ID(Name));
Error -> Error Error -> Error
end. end.
start_health_check(Name, Sleep, Timeout) -> start_health_check(Name, Sleep, Timeout) ->
Pid = self(), Pid = self(),
@ -63,13 +72,16 @@ start_health_check(Name, Sleep, Timeout) ->
health_check(Name) -> health_check(Name) ->
receive receive
{Pid, begin_health_check} -> {Pid, begin_health_check} ->
case emqx_resource:health_check(Name) of case emqx_resource:health_check(Name) of
ok -> ok ->
emqx_alarm:deactivate(Name); emqx_alarm:deactivate(Name);
{error, _} -> {error, _} ->
emqx_alarm:activate(Name, #{name => Name}, emqx_alarm:activate(
<<Name/binary, " health check failed">>) Name,
#{name => Name},
<<Name/binary, " health check failed">>
)
end, end,
Pid ! health_check_finish Pid ! health_check_finish
end, end,
@ -81,8 +93,11 @@ health_check_timeout_checker(Pid, Name, SleepTime, Timeout) ->
receive receive
health_check_finish -> timer:sleep(SleepTime) health_check_finish -> timer:sleep(SleepTime)
after Timeout -> after Timeout ->
emqx_alarm:activate(Name, #{name => Name}, emqx_alarm:activate(
<<Name/binary, " health check timeout">>), Name,
#{name => Name},
<<Name/binary, " health check timeout">>
),
emqx_resource:set_resource_status_connecting(Name), emqx_resource:set_resource_status_connecting(Name),
receive receive
health_check_finish -> timer:sleep(SleepTime) health_check_finish -> timer:sleep(SleepTime)

136
apps/emqx_resource/src/emqx_resource_instance.erl
View File

@ -23,25 +23,28 @@
-export([start_link/2]). -export([start_link/2]).
%% load resource instances from *.conf files %% load resource instances from *.conf files
-export([ lookup/1 -export([
, get_metrics/1 lookup/1,
, reset_metrics/1 get_metrics/1,
, list_all/0 reset_metrics/1,
, list_group/1 list_all/0,
]). list_group/1
]).
-export([ hash_call/2 -export([
, hash_call/3 hash_call/2,
]). hash_call/3
]).
%% gen_server Callbacks %% gen_server Callbacks
-export([ init/1 -export([
, handle_call/3 init/1,
, handle_cast/2 handle_call/3,
, handle_info/2 handle_cast/2,
, terminate/2 handle_info/2,
, code_change/3 terminate/2,
]). code_change/3
]).
-record(state, {worker_pool, worker_id}). -record(state, {worker_pool, worker_id}).
@ -52,8 +55,12 @@
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
start_link(Pool, Id) -> start_link(Pool, Id) ->
gen_server:start_link({local, proc_name(?MODULE, Id)}, gen_server:start_link(
?MODULE, {Pool, Id}, []). {local, proc_name(?MODULE, Id)},
?MODULE,
{Pool, Id},
[]
).
%% call the worker by the hash of resource-instance-id, to make sure we always handle %% call the worker by the hash of resource-instance-id, to make sure we always handle
%% operations on the same instance in the same worker. %% operations on the same instance in the same worker.
@ -67,8 +74,7 @@ hash_call(InstId, Request, Timeout) ->
lookup(InstId) -> lookup(InstId) ->
case ets:lookup(emqx_resource_instance, InstId) of case ets:lookup(emqx_resource_instance, InstId) of
[] -> {error, not_found}; [] -> {error, not_found};
[{_, Group, Data}] -> [{_, Group, Data}] -> {ok, Group, Data#{id => InstId, metrics => get_metrics(InstId)}}
{ok, Group, Data#{id => InstId, metrics => get_metrics(InstId)}}
end. end.
make_test_id() -> make_test_id() ->
@ -103,39 +109,32 @@ list_group(Group) ->
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
-spec init({atom(), integer()}) -> -spec init({atom(), integer()}) ->
{ok, State :: state()} | {ok, State :: state(), timeout() | hibernate | {continue, term()}} | {ok, State :: state()}
{stop, Reason :: term()} | ignore. | {ok, State :: state(), timeout() | hibernate | {continue, term()}}
| {stop, Reason :: term()}
| ignore.
init({Pool, Id}) -> init({Pool, Id}) ->
true = gproc_pool:connect_worker(Pool, {Pool, Id}), true = gproc_pool:connect_worker(Pool, {Pool, Id}),
{ok, #state{worker_pool = Pool, worker_id = Id}}. {ok, #state{worker_pool = Pool, worker_id = Id}}.
handle_call({create, InstId, Group, ResourceType, Config, Opts}, _From, State) -> handle_call({create, InstId, Group, ResourceType, Config, Opts}, _From, State) ->
{reply, do_create(InstId, Group, ResourceType, Config, Opts), State}; {reply, do_create(InstId, Group, ResourceType, Config, Opts), State};
handle_call({create_dry_run, ResourceType, Config}, _From, State) -> handle_call({create_dry_run, ResourceType, Config}, _From, State) ->
{reply, do_create_dry_run(ResourceType, Config), State}; {reply, do_create_dry_run(ResourceType, Config), State};
handle_call({recreate, InstId, ResourceType, Config, Opts}, _From, State) -> handle_call({recreate, InstId, ResourceType, Config, Opts}, _From, State) ->
{reply, do_recreate(InstId, ResourceType, Config, Opts), State}; {reply, do_recreate(InstId, ResourceType, Config, Opts), State};
handle_call({reset_metrics, InstId}, _From, State) -> handle_call({reset_metrics, InstId}, _From, State) ->
{reply, do_reset_metrics(InstId), State}; {reply, do_reset_metrics(InstId), State};
handle_call({remove, InstId}, _From, State) -> handle_call({remove, InstId}, _From, State) ->
{reply, do_remove(InstId), State}; {reply, do_remove(InstId), State};
handle_call({restart, InstId, Opts}, _From, State) -> handle_call({restart, InstId, Opts}, _From, State) ->
{reply, do_restart(InstId, Opts), State}; {reply, do_restart(InstId, Opts), State};
handle_call({stop, InstId}, _From, State) -> handle_call({stop, InstId}, _From, State) ->
{reply, do_stop(InstId), State}; {reply, do_stop(InstId), State};
handle_call({health_check, InstId}, _From, State) -> handle_call({health_check, InstId}, _From, State) ->
{reply, do_health_check(InstId), State}; {reply, do_health_check(InstId), State};
handle_call({set_resource_status_connecting, InstId}, _From, State) -> handle_call({set_resource_status_connecting, InstId}, _From, State) ->
{reply, do_set_resource_status_connecting(InstId), State}; {reply, do_set_resource_status_connecting(InstId), State};
handle_call(Req, _From, State) -> handle_call(Req, _From, State) ->
logger:error("Received unexpected call: ~p", [Req]), logger:error("Received unexpected call: ~p", [Req]),
{reply, ignored, State}. {reply, ignored, State}.
@ -155,14 +154,17 @@ code_change(_OldVsn, State, _Extra) ->
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% suppress the race condition check, as these functions are protected in gproc workers %% suppress the race condition check, as these functions are protected in gproc workers
-dialyzer({nowarn_function, [ do_recreate/4 -dialyzer(
, do_create/5 {nowarn_function, [
, do_restart/2 do_recreate/4,
, do_start/5 do_create/5,
, do_stop/1 do_restart/2,
, do_health_check/1 do_start/5,
, start_and_check/6 do_stop/1,
]}). do_health_check/1,
start_and_check/6
]}
).
do_recreate(InstId, ResourceType, NewConfig, Opts) -> do_recreate(InstId, ResourceType, NewConfig, Opts) ->
case lookup(InstId) of case lookup(InstId) of
@ -185,10 +187,11 @@ do_wait_for_resource_ready(_InstId, 0) ->
timeout; timeout;
do_wait_for_resource_ready(InstId, Retry) -> do_wait_for_resource_ready(InstId, Retry) ->
case force_lookup(InstId) of case force_lookup(InstId) of
#{status := connected} -> ok; #{status := connected} ->
ok;
_ -> _ ->
timer:sleep(100), timer:sleep(100),
do_wait_for_resource_ready(InstId, Retry-1) do_wait_for_resource_ready(InstId, Retry - 1)
end. end.
do_create(InstId, Group, ResourceType, Config, Opts) -> do_create(InstId, Group, ResourceType, Config, Opts) ->
@ -197,8 +200,12 @@ do_create(InstId, Group, ResourceType, Config, Opts) ->
{ok, already_created}; {ok, already_created};
{error, not_found} -> {error, not_found} ->
ok = do_start(InstId, Group, ResourceType, Config, Opts), ok = do_start(InstId, Group, ResourceType, Config, Opts),
ok = emqx_plugin_libs_metrics:create_metrics(resource_metrics, InstId, ok = emqx_plugin_libs_metrics:create_metrics(
[matched, success, failed, exception], [matched]), resource_metrics,
InstId,
[matched, success, failed, exception],
[matched]
),
{ok, force_lookup(InstId)} {ok, force_lookup(InstId)}
end. end.
@ -212,7 +219,8 @@ do_create_dry_run(ResourceType, Config) ->
{error, _} = Error -> Error; {error, _} = Error -> Error;
_ -> ok _ -> ok
end; end;
{error, Reason, _} -> {error, Reason} {error, Reason, _} ->
{error, Reason}
end; end;
{error, Reason} -> {error, Reason} ->
{error, Reason} {error, Reason}
@ -246,13 +254,18 @@ do_restart(InstId, Opts) ->
end. end.
do_start(InstId, Group, ResourceType, Config, Opts) when is_binary(InstId) -> do_start(InstId, Group, ResourceType, Config, Opts) when is_binary(InstId) ->
InitData = #{id => InstId, mod => ResourceType, config => Config, InitData = #{
status => connecting, state => undefined}, id => InstId,
mod => ResourceType,
config => Config,
status => connecting,
state => undefined
},
%% The `emqx_resource:call_start/3` need the instance exist beforehand %% The `emqx_resource:call_start/3` need the instance exist beforehand
ets:insert(emqx_resource_instance, {InstId, Group, InitData}), ets:insert(emqx_resource_instance, {InstId, Group, InitData}),
spawn(fun() -> spawn(fun() ->
start_and_check(InstId, Group, ResourceType, Config, Opts, InitData) start_and_check(InstId, Group, ResourceType, Config, Opts, InitData)
end), end),
_ = wait_for_resource_ready(InstId, maps:get(wait_for_resource_ready, Opts, 5000)), _ = wait_for_resource_ready(InstId, maps:get(wait_for_resource_ready, Opts, 5000)),
ok. ok.
@ -268,9 +281,11 @@ start_and_check(InstId, Group, ResourceType, Config, Opts, Data) ->
end. end.
create_default_checker(InstId, Opts) -> create_default_checker(InstId, Opts) ->
emqx_resource_health_check:create_checker(InstId, emqx_resource_health_check:create_checker(
InstId,
maps:get(health_check_interval, Opts, 15000), maps:get(health_check_interval, Opts, 15000),
maps:get(health_check_timeout, Opts, 10000)). maps:get(health_check_timeout, Opts, 10000)
).
do_stop(InstId) when is_binary(InstId) -> do_stop(InstId) when is_binary(InstId) ->
do_with_group_and_instance_data(InstId, fun do_stop/2, []). do_with_group_and_instance_data(InstId, fun do_stop/2, []).
@ -291,18 +306,24 @@ do_health_check(_Group, #{state := undefined}) ->
do_health_check(Group, #{id := InstId, mod := Mod, state := ResourceState0} = Data) -> do_health_check(Group, #{id := InstId, mod := Mod, state := ResourceState0} = Data) ->
case emqx_resource:call_health_check(InstId, Mod, ResourceState0) of case emqx_resource:call_health_check(InstId, Mod, ResourceState0) of
{ok, ResourceState1} -> {ok, ResourceState1} ->
ets:insert(emqx_resource_instance, ets:insert(
{InstId, Group, Data#{status => connected, state => ResourceState1}}), emqx_resource_instance,
{InstId, Group, Data#{status => connected, state => ResourceState1}}
),
ok; ok;
{error, Reason} -> {error, Reason} ->
logger:error("health check for ~p failed: ~p", [InstId, Reason]), logger:error("health check for ~p failed: ~p", [InstId, Reason]),
ets:insert(emqx_resource_instance, ets:insert(
{InstId, Group, Data#{status => connecting}}), emqx_resource_instance,
{InstId, Group, Data#{status => connecting}}
),
{error, Reason}; {error, Reason};
{error, Reason, ResourceState1} -> {error, Reason, ResourceState1} ->
logger:error("health check for ~p failed: ~p", [InstId, Reason]), logger:error("health check for ~p failed: ~p", [InstId, Reason]),
ets:insert(emqx_resource_instance, ets:insert(
{InstId, Group, Data#{status => connecting, state => ResourceState1}}), emqx_resource_instance,
{InstId, Group, Data#{status => connecting, state => ResourceState1}}
),
{error, Reason} {error, Reason}
end. end.
@ -311,7 +332,8 @@ do_set_resource_status_connecting(InstId) ->
{ok, Group, #{id := InstId} = Data} -> {ok, Group, #{id := InstId} = Data} ->
logger:error("health check for ~p failed: timeout", [InstId]), logger:error("health check for ~p failed: timeout", [InstId]),
ets:insert(emqx_resource_instance, {InstId, Group, Data#{status => connecting}}); ets:insert(emqx_resource_instance, {InstId, Group, Data#{status => connecting}});
Error -> {error, Error} Error ->
{error, Error}
end. end.
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------

39
apps/emqx_resource/src/emqx_resource_sup.erl
View File

@ -22,7 +22,8 @@
-export([init/1]). -export([init/1]).
-define(RESOURCE_INST_MOD, emqx_resource_instance). -define(RESOURCE_INST_MOD, emqx_resource_instance).
-define(POOL_SIZE, 64). %% set a very large pool size in case all the workers busy %% set a very large pool size in case all the workers busy
-define(POOL_SIZE, 64).
start_link() -> start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []). supervisor:start_link({local, ?MODULE}, ?MODULE, []).
@ -40,27 +41,39 @@ init([]) ->
ResourceInsts = [ ResourceInsts = [
begin begin
ensure_pool_worker(Pool, {Pool, Idx}, Idx), ensure_pool_worker(Pool, {Pool, Idx}, Idx),
#{id => {Mod, Idx}, #{
start => {Mod, start_link, [Pool, Idx]}, id => {Mod, Idx},
restart => transient, start => {Mod, start_link, [Pool, Idx]},
shutdown => 5000, type => worker, modules => [Mod]} restart => transient,
end || Idx <- lists:seq(1, ?POOL_SIZE)], shutdown => 5000,
HealthCheck = type => worker,
#{id => emqx_resource_health_check_sup, modules => [Mod]
start => {emqx_resource_health_check_sup, start_link, []}, }
restart => transient, end
shutdown => infinity, type => supervisor, modules => [emqx_resource_health_check_sup]}, || Idx <- lists:seq(1, ?POOL_SIZE)
],
HealthCheck =
#{
id => emqx_resource_health_check_sup,
start => {emqx_resource_health_check_sup, start_link, []},
restart => transient,
shutdown => infinity,
type => supervisor,
modules => [emqx_resource_health_check_sup]
},
{ok, {SupFlags, [HealthCheck, Metrics | ResourceInsts]}}. {ok, {SupFlags, [HealthCheck, Metrics | ResourceInsts]}}.
%% internal functions %% internal functions
ensure_pool(Pool, Type, Opts) -> ensure_pool(Pool, Type, Opts) ->
try gproc_pool:new(Pool, Type, Opts) try
gproc_pool:new(Pool, Type, Opts)
catch catch
error:exists -> ok error:exists -> ok
end. end.
ensure_pool_worker(Pool, Name, Slot) -> ensure_pool_worker(Pool, Name, Slot) ->
try gproc_pool:add_worker(Pool, Name, Slot) try
gproc_pool:add_worker(Pool, Name, Slot)
catch catch
error:exists -> ok error:exists -> ok
end. end.

23
apps/emqx_resource/src/emqx_resource_validator.erl
View File

@ -16,10 +16,11 @@
-module(emqx_resource_validator). -module(emqx_resource_validator).
-export([ min/2 -export([
, max/2 min/2,
, not_empty/1 max/2,
]). not_empty/1
]).
max(Type, Max) -> max(Type, Max) ->
limit(Type, '=<', Max). limit(Type, '=<', Max).
@ -28,16 +29,19 @@ min(Type, Min) ->
limit(Type, '>=', Min). limit(Type, '>=', Min).
not_empty(ErrMsg) -> not_empty(ErrMsg) ->
fun(<<>>) -> {error, ErrMsg}; fun
(_) -> ok (<<>>) -> {error, ErrMsg};
(_) -> ok
end. end.
limit(Type, Op, Expected) -> limit(Type, Op, Expected) ->
L = len(Type), L = len(Type),
fun(Value) -> fun(Value) ->
Got = L(Value), Got = L(Value),
return(erlang:Op(Got, Expected), return(
err_limit({Type, {Op, Expected}, {got, Got}})) erlang:Op(Got, Expected),
err_limit({Type, {Op, Expected}, {got, Got}})
)
end. end.
len(array) -> fun erlang:length/1; len(array) -> fun erlang:length/1;
@ -48,5 +52,4 @@ err_limit({Type, {Op, Expected}, {got, Got}}) ->
io_lib:format("Expect the ~ts value ~ts ~p but got: ~p", [Type, Op, Expected, Got]). io_lib:format("Expect the ~ts value ~ts ~p but got: ~p", [Type, Op, Expected, Got]).
return(true, _) -> ok; return(true, _) -> ok;
return(false, Error) -> return(false, Error) -> {error, Error}.
{error, Error}.

60
apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl
View File

@ -18,52 +18,58 @@
-behaviour(emqx_bpapi). -behaviour(emqx_bpapi).
-export([ introduced_in/0 -export([
introduced_in/0,
, create/5 create/5,
, create_dry_run/2 create_dry_run/2,
, recreate/4 recreate/4,
, remove/1 remove/1,
, reset_metrics/1 reset_metrics/1
]). ]).
-include_lib("emqx/include/bpapi.hrl"). -include_lib("emqx/include/bpapi.hrl").
introduced_in() -> introduced_in() ->
"5.0.0". "5.0.0".
-spec create( emqx_resource:instance_id() -spec create(
, emqx_resource:resource_group() emqx_resource:instance_id(),
, emqx_resource:resource_type() emqx_resource:resource_group(),
, emqx_resource:resource_config() emqx_resource:resource_type(),
, emqx_resource:create_opts() emqx_resource:resource_config(),
) -> emqx_resource:create_opts()
emqx_cluster_rpc:multicall_return(emqx_resource:resource_data()). ) ->
emqx_cluster_rpc:multicall_return(emqx_resource:resource_data()).
create(InstId, Group, ResourceType, Config, Opts) -> create(InstId, Group, ResourceType, Config, Opts) ->
emqx_cluster_rpc:multicall(emqx_resource, create_local, [InstId, Group, ResourceType, Config, Opts]). emqx_cluster_rpc:multicall(emqx_resource, create_local, [
InstId, Group, ResourceType, Config, Opts
]).
-spec create_dry_run( emqx_resource:resource_type() -spec create_dry_run(
, emqx_resource:resource_config() emqx_resource:resource_type(),
) -> emqx_resource:resource_config()
emqx_cluster_rpc:multicall_return(emqx_resource:resource_data()). ) ->
emqx_cluster_rpc:multicall_return(emqx_resource:resource_data()).
create_dry_run(ResourceType, Config) -> create_dry_run(ResourceType, Config) ->
emqx_cluster_rpc:multicall(emqx_resource, create_dry_run_local, [ResourceType, Config]). emqx_cluster_rpc:multicall(emqx_resource, create_dry_run_local, [ResourceType, Config]).
-spec recreate( emqx_resource:instance_id() -spec recreate(
, emqx_resource:resource_type() emqx_resource:instance_id(),
, emqx_resource:resource_config() emqx_resource:resource_type(),
, emqx_resource:create_opts() emqx_resource:resource_config(),
) -> emqx_resource:create_opts()
emqx_cluster_rpc:multicall_return(emqx_resource:resource_data()). ) ->
emqx_cluster_rpc:multicall_return(emqx_resource:resource_data()).
recreate(InstId, ResourceType, Config, Opts) -> recreate(InstId, ResourceType, Config, Opts) ->
emqx_cluster_rpc:multicall(emqx_resource, recreate_local, [InstId, ResourceType, Config, Opts]). emqx_cluster_rpc:multicall(emqx_resource, recreate_local, [InstId, ResourceType, Config, Opts]).
-spec remove(emqx_resource:instance_id()) -> -spec remove(emqx_resource:instance_id()) ->
emqx_cluster_rpc:multicall_return(ok). emqx_cluster_rpc:multicall_return(ok).
remove(InstId) -> remove(InstId) ->
emqx_cluster_rpc:multicall(emqx_resource, remove_local, [InstId]). emqx_cluster_rpc:multicall(emqx_resource, remove_local, [InstId]).
-spec reset_metrics(emqx_resource:instance_id()) -> -spec reset_metrics(emqx_resource:instance_id()) ->
emqx_cluster_rpc:multicall_return(ok). emqx_cluster_rpc:multicall_return(ok).
reset_metrics(InstId) -> reset_metrics(InstId) ->
emqx_cluster_rpc:multicall(emqx_resource, reset_metrics_local, [InstId]). emqx_cluster_rpc:multicall(emqx_resource, reset_metrics_local, [InstId]).

236
apps/emqx_resource/test/emqx_resource_SUITE.erl
View File

@ -60,22 +60,25 @@ t_check_config(_) ->
t_create_remove(_) -> t_create_remove(_) ->
{error, _} = emqx_resource:check_and_create_local( {error, _} = emqx_resource:check_and_create_local(
?ID, ?ID,
?DEFAULT_RESOURCE_GROUP, ?DEFAULT_RESOURCE_GROUP,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{unknown => test_resource}), #{unknown => test_resource}
),
{ok, _} = emqx_resource:create( {ok, _} = emqx_resource:create(
?ID, ?ID,
?DEFAULT_RESOURCE_GROUP, ?DEFAULT_RESOURCE_GROUP,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{name => test_resource}), #{name => test_resource}
),
emqx_resource:recreate( emqx_resource:recreate(
?ID, ?ID,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{name => test_resource}, #{name => test_resource},
#{}), #{}
),
#{pid := Pid} = emqx_resource:query(?ID, get_state), #{pid := Pid} = emqx_resource:query(?ID, get_state),
?assert(is_process_alive(Pid)), ?assert(is_process_alive(Pid)),
@ -87,22 +90,25 @@ t_create_remove(_) ->
t_create_remove_local(_) -> t_create_remove_local(_) ->
{error, _} = emqx_resource:check_and_create_local( {error, _} = emqx_resource:check_and_create_local(
?ID, ?ID,
?DEFAULT_RESOURCE_GROUP, ?DEFAULT_RESOURCE_GROUP,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{unknown => test_resource}), #{unknown => test_resource}
),
{ok, _} = emqx_resource:create_local( {ok, _} = emqx_resource:create_local(
?ID, ?ID,
?DEFAULT_RESOURCE_GROUP, ?DEFAULT_RESOURCE_GROUP,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{name => test_resource}), #{name => test_resource}
),
emqx_resource:recreate_local( emqx_resource:recreate_local(
?ID, ?ID,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{name => test_resource}, #{name => test_resource},
#{}), #{}
),
#{pid := Pid} = emqx_resource:query(?ID, get_state), #{pid := Pid} = emqx_resource:query(?ID, get_state),
?assert(is_process_alive(Pid)), ?assert(is_process_alive(Pid)),
@ -110,10 +116,11 @@ t_create_remove_local(_) ->
emqx_resource:set_resource_status_connecting(?ID), emqx_resource:set_resource_status_connecting(?ID),
emqx_resource:recreate_local( emqx_resource:recreate_local(
?ID, ?ID,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{name => test_resource}, #{name => test_resource},
#{}), #{}
),
ok = emqx_resource:remove_local(?ID), ok = emqx_resource:remove_local(?ID),
{error, _} = emqx_resource:remove_local(?ID), {error, _} = emqx_resource:remove_local(?ID),
@ -122,10 +129,11 @@ t_create_remove_local(_) ->
t_query(_) -> t_query(_) ->
{ok, _} = emqx_resource:create_local( {ok, _} = emqx_resource:create_local(
?ID, ?ID,
?DEFAULT_RESOURCE_GROUP, ?DEFAULT_RESOURCE_GROUP,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{name => test_resource}), #{name => test_resource}
),
Pid = self(), Pid = self(),
Success = fun() -> Pid ! success end, Success = fun() -> Pid ! success end,
@ -142,28 +150,32 @@ t_query(_) ->
?assert(false) ?assert(false)
end, end,
?assertMatch({error, {emqx_resource, #{reason := not_found}}}, ?assertMatch(
emqx_resource:query(<<"unknown">>, get_state)), {error, {emqx_resource, #{reason := not_found}}},
emqx_resource:query(<<"unknown">>, get_state)
),
ok = emqx_resource:remove_local(?ID). ok = emqx_resource:remove_local(?ID).
t_healthy_timeout(_) -> t_healthy_timeout(_) ->
{ok, _} = emqx_resource:create_local( {ok, _} = emqx_resource:create_local(
?ID, ?ID,
?DEFAULT_RESOURCE_GROUP, ?DEFAULT_RESOURCE_GROUP,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{name => <<"test_resource">>}, #{name => <<"test_resource">>},
#{health_check_timeout => 200}), #{health_check_timeout => 200}
),
timer:sleep(500), timer:sleep(500),
ok = emqx_resource:remove_local(?ID). ok = emqx_resource:remove_local(?ID).
t_healthy(_) -> t_healthy(_) ->
{ok, _} = emqx_resource:create_local( {ok, _} = emqx_resource:create_local(
?ID, ?ID,
?DEFAULT_RESOURCE_GROUP, ?DEFAULT_RESOURCE_GROUP,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{name => <<"test_resource">>}), #{name => <<"test_resource">>}
),
timer:sleep(400), timer:sleep(400),
emqx_resource_health_check:create_checker(?ID, 15000, 10000), emqx_resource_health_check:create_checker(?ID, 15000, 10000),
@ -175,38 +187,44 @@ t_healthy(_) ->
?assertMatch( ?assertMatch(
[#{status := connected}], [#{status := connected}],
emqx_resource:list_instances_verbose()), emqx_resource:list_instances_verbose()
),
erlang:exit(Pid, shutdown), erlang:exit(Pid, shutdown),
?assertEqual( ?assertEqual(
{error, dead}, {error, dead},
emqx_resource:health_check(?ID)), emqx_resource:health_check(?ID)
),
?assertMatch( ?assertMatch(
[#{status := connecting}], [#{status := connecting}],
emqx_resource:list_instances_verbose()), emqx_resource:list_instances_verbose()
),
ok = emqx_resource:remove_local(?ID). ok = emqx_resource:remove_local(?ID).
t_stop_start(_) -> t_stop_start(_) ->
{error, _} = emqx_resource:check_and_create( {error, _} = emqx_resource:check_and_create(
?ID, ?ID,
?DEFAULT_RESOURCE_GROUP, ?DEFAULT_RESOURCE_GROUP,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{unknown => test_resource}), #{unknown => test_resource}
),
{ok, _} = emqx_resource:check_and_create( {ok, _} = emqx_resource:check_and_create(
?ID, ?ID,
?DEFAULT_RESOURCE_GROUP, ?DEFAULT_RESOURCE_GROUP,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{<<"name">> => <<"test_resource">>}), #{<<"name">> => <<"test_resource">>}
),
{ok, _} = emqx_resource:check_and_recreate( {ok, _} = emqx_resource:check_and_recreate(
?ID, ?ID,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{<<"name">> => <<"test_resource">>}, #{<<"name">> => <<"test_resource">>},
#{}), #{}
),
#{pid := Pid0} = emqx_resource:query(?ID, get_state), #{pid := Pid0} = emqx_resource:query(?ID, get_state),
@ -216,8 +234,10 @@ t_stop_start(_) ->
?assertNot(is_process_alive(Pid0)), ?assertNot(is_process_alive(Pid0)),
?assertMatch({error, {emqx_resource, #{reason := disconnected}}}, ?assertMatch(
emqx_resource:query(?ID, get_state)), {error, {emqx_resource, #{reason := disconnected}}},
emqx_resource:query(?ID, get_state)
),
ok = emqx_resource:restart(?ID), ok = emqx_resource:restart(?ID),
@ -229,22 +249,25 @@ t_stop_start(_) ->
t_stop_start_local(_) -> t_stop_start_local(_) ->
{error, _} = emqx_resource:check_and_create_local( {error, _} = emqx_resource:check_and_create_local(
?ID, ?ID,
?DEFAULT_RESOURCE_GROUP, ?DEFAULT_RESOURCE_GROUP,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{unknown => test_resource}), #{unknown => test_resource}
),
{ok, _} = emqx_resource:check_and_create_local( {ok, _} = emqx_resource:check_and_create_local(
?ID, ?ID,
?DEFAULT_RESOURCE_GROUP, ?DEFAULT_RESOURCE_GROUP,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{<<"name">> => <<"test_resource">>}), #{<<"name">> => <<"test_resource">>}
),
{ok, _} = emqx_resource:check_and_recreate_local( {ok, _} = emqx_resource:check_and_recreate_local(
?ID, ?ID,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{<<"name">> => <<"test_resource">>}, #{<<"name">> => <<"test_resource">>},
#{}), #{}
),
#{pid := Pid0} = emqx_resource:query(?ID, get_state), #{pid := Pid0} = emqx_resource:query(?ID, get_state),
@ -254,8 +277,10 @@ t_stop_start_local(_) ->
?assertNot(is_process_alive(Pid0)), ?assertNot(is_process_alive(Pid0)),
?assertMatch({error, {emqx_resource, #{reason := disconnected}}}, ?assertMatch(
emqx_resource:query(?ID, get_state)), {error, {emqx_resource, #{reason := disconnected}}},
emqx_resource:query(?ID, get_state)
),
ok = emqx_resource:restart(?ID), ok = emqx_resource:restart(?ID),
@ -265,60 +290,73 @@ t_stop_start_local(_) ->
t_list_filter(_) -> t_list_filter(_) ->
{ok, _} = emqx_resource:create_local( {ok, _} = emqx_resource:create_local(
emqx_resource:generate_id(<<"a">>), emqx_resource:generate_id(<<"a">>),
<<"group1">>, <<"group1">>,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{name => a}), #{name => a}
),
{ok, _} = emqx_resource:create_local( {ok, _} = emqx_resource:create_local(
emqx_resource:generate_id(<<"a">>), emqx_resource:generate_id(<<"a">>),
<<"group2">>, <<"group2">>,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{name => grouped_a}), #{name => grouped_a}
),
[Id1] = emqx_resource:list_group_instances(<<"group1">>), [Id1] = emqx_resource:list_group_instances(<<"group1">>),
?assertMatch( ?assertMatch(
{ok, <<"group1">>, #{config := #{name := a}}}, {ok, <<"group1">>, #{config := #{name := a}}},
emqx_resource:get_instance(Id1)), emqx_resource:get_instance(Id1)
),
[Id2] = emqx_resource:list_group_instances(<<"group2">>), [Id2] = emqx_resource:list_group_instances(<<"group2">>),
?assertMatch( ?assertMatch(
{ok, <<"group2">>, #{config := #{name := grouped_a}}}, {ok, <<"group2">>, #{config := #{name := grouped_a}}},
emqx_resource:get_instance(Id2)). emqx_resource:get_instance(Id2)
).
t_create_dry_run_local(_) -> t_create_dry_run_local(_) ->
?assertEqual( ?assertEqual(
ok, ok,
emqx_resource:create_dry_run_local( emqx_resource:create_dry_run_local(
?TEST_RESOURCE, ?TEST_RESOURCE,
#{name => test_resource, register => true})), #{name => test_resource, register => true}
)
),
?assertEqual(undefined, whereis(test_resource)). ?assertEqual(undefined, whereis(test_resource)).
t_create_dry_run_local_failed(_) -> t_create_dry_run_local_failed(_) ->
{Res, _} = emqx_resource:create_dry_run_local(?TEST_RESOURCE, {Res, _} = emqx_resource:create_dry_run_local(
#{cteate_error => true}), ?TEST_RESOURCE,
#{cteate_error => true}
),
?assertEqual(error, Res), ?assertEqual(error, Res),
{Res, _} = emqx_resource:create_dry_run_local(?TEST_RESOURCE, {Res, _} = emqx_resource:create_dry_run_local(
#{name => test_resource, health_check_error => true}), ?TEST_RESOURCE,
#{name => test_resource, health_check_error => true}
),
?assertEqual(error, Res), ?assertEqual(error, Res),
{Res, _} = emqx_resource:create_dry_run_local(?TEST_RESOURCE, {Res, _} = emqx_resource:create_dry_run_local(
#{name => test_resource, stop_error => true}), ?TEST_RESOURCE,
#{name => test_resource, stop_error => true}
),
?assertEqual(error, Res). ?assertEqual(error, Res).
t_test_func(_) -> t_test_func(_) ->
?assertEqual(ok, erlang:apply(emqx_resource_validator:not_empty("not_empty"), [<<"someval">>])), ?assertEqual(ok, erlang:apply(emqx_resource_validator:not_empty("not_empty"), [<<"someval">>])),
?assertEqual(ok, erlang:apply(emqx_resource_validator:min(int, 3), [4])), ?assertEqual(ok, erlang:apply(emqx_resource_validator:min(int, 3), [4])),
?assertEqual(ok, erlang:apply(emqx_resource_validator:max(array, 10), [[a,b,c,d]])), ?assertEqual(ok, erlang:apply(emqx_resource_validator:max(array, 10), [[a, b, c, d]])),
?assertEqual(ok, erlang:apply(emqx_resource_validator:max(string, 10), ["less10"])). ?assertEqual(ok, erlang:apply(emqx_resource_validator:max(string, 10), ["less10"])).
t_reset_metrics(_) -> t_reset_metrics(_) ->
{ok, _} = emqx_resource:create( {ok, _} = emqx_resource:create(
?ID, ?ID,
?DEFAULT_RESOURCE_GROUP, ?DEFAULT_RESOURCE_GROUP,
?TEST_RESOURCE, ?TEST_RESOURCE,
#{name => test_resource}), #{name => test_resource}
),
#{pid := Pid} = emqx_resource:query(?ID, get_state), #{pid := Pid} = emqx_resource:query(?ID, get_state),
emqx_resource:reset_metrics(?ID), emqx_resource:reset_metrics(?ID),

68
apps/emqx_resource/test/emqx_test_resource.erl
View File

@ -21,17 +21,21 @@
-behaviour(emqx_resource). -behaviour(emqx_resource).
%% callbacks of behaviour emqx_resource %% callbacks of behaviour emqx_resource
-export([ on_start/2 -export([
, on_stop/2 on_start/2,
, on_query/4 on_stop/2,
, on_health_check/2 on_query/4,
]). on_health_check/2
]).
%% callbacks for emqx_resource config schema %% callbacks for emqx_resource config schema
-export([roots/0]). -export([roots/0]).
roots() -> [{name, fun name/1}, roots() ->
{register, fun register/1}]. [
{name, fun name/1},
{register, fun register/1}
].
name(type) -> atom(); name(type) -> atom();
name(required) -> true; name(required) -> true;
@ -46,21 +50,27 @@ on_start(_InstId, #{create_error := true}) ->
error("some error"); error("some error");
on_start(InstId, #{name := Name, stop_error := true} = Opts) -> on_start(InstId, #{name := Name, stop_error := true} = Opts) ->
Register = maps:get(register, Opts, false), Register = maps:get(register, Opts, false),
{ok, #{name => Name, {ok, #{
id => InstId, name => Name,
stop_error => true, id => InstId,
pid => spawn_dummy_process(Name, Register)}}; stop_error => true,
pid => spawn_dummy_process(Name, Register)
}};
on_start(InstId, #{name := Name, health_check_error := true} = Opts) -> on_start(InstId, #{name := Name, health_check_error := true} = Opts) ->
Register = maps:get(register, Opts, false), Register = maps:get(register, Opts, false),
{ok, #{name => Name, {ok, #{
id => InstId, name => Name,
health_check_error => true, id => InstId,
pid => spawn_dummy_process(Name, Register)}}; health_check_error => true,
pid => spawn_dummy_process(Name, Register)
}};
on_start(InstId, #{name := Name} = Opts) -> on_start(InstId, #{name := Name} = Opts) ->
Register = maps:get(register, Opts, false), Register = maps:get(register, Opts, false),
{ok, #{name => Name, {ok, #{
id => InstId, name => Name,
pid => spawn_dummy_process(Name, Register)}}. id => InstId,
pid => spawn_dummy_process(Name, Register)
}}.
on_stop(_InstId, #{stop_error := true}) -> on_stop(_InstId, #{stop_error := true}) ->
{error, stop_error}; {error, stop_error};
@ -86,13 +96,15 @@ on_health_check(_InstId, State = #{pid := Pid}) ->
spawn_dummy_process(Name, Register) -> spawn_dummy_process(Name, Register) ->
spawn( spawn(
fun() -> fun() ->
true = case Register of true =
true -> register(Name, self()); case Register of
_ -> true true -> register(Name, self());
end, _ -> true
Ref = make_ref(), end,
receive Ref = make_ref(),
Ref -> ok receive
end Ref -> ok
end). end
end
).

93
apps/emqx_rule_engine/include/rule_engine.hrl
View File

@ -23,7 +23,7 @@
-type rule_id() :: binary(). -type rule_id() :: binary().
-type rule_name() :: binary(). -type rule_name() :: binary().
-type mf() :: {Module::atom(), Fun::atom()}. -type mf() :: {Module :: atom(), Fun :: atom()}.
-type hook() :: atom() | 'any'. -type hook() :: atom() | 'any'.
-type topic() :: binary(). -type topic() :: binary().
@ -36,60 +36,73 @@
-type bridge_channel_id() :: binary(). -type bridge_channel_id() :: binary().
-type output_fun_args() :: map(). -type output_fun_args() :: map().
-type output() :: #{ -type output() ::
mod := builtin_output_module() | module(), #{
func := builtin_output_func() | atom(), mod := builtin_output_module() | module(),
args => output_fun_args() func := builtin_output_func() | atom(),
} | bridge_channel_id(). args => output_fun_args()
}
| bridge_channel_id().
-type rule() :: -type rule() ::
#{ id := rule_id() #{
, name := binary() id := rule_id(),
, sql := binary() name := binary(),
, outputs := [output()] sql := binary(),
, enable := boolean() outputs := [output()],
, description => binary() enable := boolean(),
, created_at := integer() %% epoch in millisecond precision description => binary(),
, updated_at := integer() %% epoch in millisecond precision %% epoch in millisecond precision
, from := list(topic()) created_at := integer(),
, is_foreach := boolean() %% epoch in millisecond precision
, fields := list() updated_at := integer(),
, doeach := term() from := list(topic()),
, incase := term() is_foreach := boolean(),
, conditions := tuple() fields := list(),
}. doeach := term(),
incase := term(),
conditions := tuple()
}.
%% Arithmetic operators %% Arithmetic operators
-define(is_arith(Op), (Op =:= '+' orelse -define(is_arith(Op),
Op =:= '-' orelse (Op =:= '+' orelse
Op =:= '*' orelse Op =:= '-' orelse
Op =:= '/' orelse Op =:= '*' orelse
Op =:= 'div')). Op =:= '/' orelse
Op =:= 'div')
).
%% Compare operators %% Compare operators
-define(is_comp(Op), (Op =:= '=' orelse -define(is_comp(Op),
Op =:= '=~' orelse (Op =:= '=' orelse
Op =:= '>' orelse Op =:= '=~' orelse
Op =:= '<' orelse Op =:= '>' orelse
Op =:= '<=' orelse Op =:= '<' orelse
Op =:= '>=' orelse Op =:= '<=' orelse
Op =:= '<>' orelse Op =:= '>=' orelse
Op =:= '!=')). Op =:= '<>' orelse
Op =:= '!=')
).
%% Logical operators %% Logical operators
-define(is_logical(Op), (Op =:= 'and' orelse Op =:= 'or')). -define(is_logical(Op), (Op =:= 'and' orelse Op =:= 'or')).
-define(RAISE(_EXP_, _ERROR_), -define(RAISE(_EXP_, _ERROR_),
?RAISE(_EXP_, _ = do_nothing, _ERROR_)). ?RAISE(_EXP_, _ = do_nothing, _ERROR_)
).
-define(RAISE(_EXP_, _EXP_ON_FAIL_, _ERROR_), -define(RAISE(_EXP_, _EXP_ON_FAIL_, _ERROR_),
fun() -> fun() ->
try (_EXP_) try
catch _EXCLASS_:_EXCPTION_:_ST_ -> (_EXP_)
catch
_EXCLASS_:_EXCPTION_:_ST_ ->
_EXP_ON_FAIL_, _EXP_ON_FAIL_,
throw(_ERROR_) throw(_ERROR_)
end end
end()). end()
).
%% Tables %% Tables
-define(RULE_TAB, emqx_rule_engine). -define(RULE_TAB, emqx_rule_engine).

35
apps/emqx_rule_engine/rebar.config
View File

@ -1,28 +1,35 @@
%% -*- mode: erlang -*- %% -*- mode: erlang -*-
{deps, [ {emqx, {path, "../emqx"}} {deps, [{emqx, {path, "../emqx"}}]}.
]}.
{erl_opts, [warn_unused_vars, {erl_opts, [
warn_shadow_vars, warn_unused_vars,
warn_unused_import, warn_shadow_vars,
warn_obsolete_guard, warn_unused_import,
no_debug_info, warn_obsolete_guard,
compressed, %% for edge no_debug_info,
{parse_transform} %% for edge
]}. compressed,
{parse_transform}
]}.
{overrides, [{add, [{erl_opts, [no_debug_info, compressed]}]}]}. {overrides, [{add, [{erl_opts, [no_debug_info, compressed]}]}]}.
{edoc_opts, [{preprocess, true}]}. {edoc_opts, [{preprocess, true}]}.
{xref_checks, [undefined_function_calls, undefined_functions, {xref_checks, [
locals_not_used, deprecated_function_calls, undefined_function_calls,
warnings_as_errors, deprecated_functions undefined_functions,
]}. locals_not_used,
deprecated_function_calls,
warnings_as_errors,
deprecated_functions
]}.
{cover_enabled, true}. {cover_enabled, true}.
{cover_opts, [verbose]}. {cover_opts, [verbose]}.
{cover_export_enabled, true}. {cover_export_enabled, true}.
{plugins, [rebar3_proper]}. {plugins, [rebar3_proper]}.
{project_plugins, [erlfmt]}.

431
apps/emqx_rule_engine/src/emqx_rule_api_schema.erl
View File

@ -6,8 +6,7 @@
-include_lib("hocon/include/hoconsc.hrl"). -include_lib("hocon/include/hoconsc.hrl").
-include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/logger.hrl").
-export([ check_params/2 -export([check_params/2]).
]).
-export([roots/0, fields/1]). -export([roots/0, fields/1]).
@ -20,10 +19,11 @@ check_params(Params, Tag) ->
try hocon_tconf:check_plain(?MODULE, #{BTag => Params}, Opts, [Tag]) of try hocon_tconf:check_plain(?MODULE, #{BTag => Params}, Opts, [Tag]) of
#{Tag := Checked} -> {ok, Checked} #{Tag := Checked} -> {ok, Checked}
catch catch
throw : Reason -> throw:Reason ->
?SLOG(error, #{msg => "check_rule_params_failed", ?SLOG(error, #{
reason => Reason msg => "check_rule_params_failed",
}), reason => Reason
}),
{error, Reason} {error, Reason}
end. end.
@ -31,226 +31,285 @@ check_params(Params, Tag) ->
%% Hocon Schema Definitions %% Hocon Schema Definitions
roots() -> roots() ->
[ {"rule_creation", sc(ref("rule_creation"), #{desc => ?DESC("root_rule_creation")})} [
, {"rule_info", sc(ref("rule_info"), #{desc => ?DESC("root_rule_info")})} {"rule_creation", sc(ref("rule_creation"), #{desc => ?DESC("root_rule_creation")})},
, {"rule_events", sc(ref("rule_events"), #{desc => ?DESC("root_rule_events")})} {"rule_info", sc(ref("rule_info"), #{desc => ?DESC("root_rule_info")})},
, {"rule_test", sc(ref("rule_test"), #{desc => ?DESC("root_rule_test")})} {"rule_events", sc(ref("rule_events"), #{desc => ?DESC("root_rule_events")})},
{"rule_test", sc(ref("rule_test"), #{desc => ?DESC("root_rule_test")})}
]. ].
fields("rule_creation") -> fields("rule_creation") ->
emqx_rule_engine_schema:fields("rules"); emqx_rule_engine_schema:fields("rules");
fields("rule_info") -> fields("rule_info") ->
[ rule_id() [
, {"metrics", sc(ref("metrics"), #{desc => ?DESC("ri_metrics")})} rule_id(),
, {"node_metrics", sc(hoconsc:array(ref("node_metrics")), {"metrics", sc(ref("metrics"), #{desc => ?DESC("ri_metrics")})},
#{ desc => ?DESC("ri_node_metrics") {"node_metrics",
})} sc(
, {"from", sc(hoconsc:array(binary()), hoconsc:array(ref("node_metrics")),
#{desc => ?DESC("ri_from"), example => "t/#"})} #{desc => ?DESC("ri_node_metrics")}
, {"created_at", sc(binary(), )},
#{ desc => ?DESC("ri_created_at") {"from",
, example => "2021-12-01T15:00:43.153+08:00" sc(
})} hoconsc:array(binary()),
#{desc => ?DESC("ri_from"), example => "t/#"}
)},
{"created_at",
sc(
binary(),
#{
desc => ?DESC("ri_created_at"),
example => "2021-12-01T15:00:43.153+08:00"
}
)}
] ++ fields("rule_creation"); ] ++ fields("rule_creation");
%% TODO: we can delete this API if the Dashboard not depends on it %% TODO: we can delete this API if the Dashboard not depends on it
fields("rule_events") -> fields("rule_events") ->
ETopics = [binary_to_atom(emqx_rule_events:event_topic(E)) || E <- emqx_rule_events:event_names()], ETopics = [
[ {"event", sc(hoconsc:enum(ETopics), #{desc => ?DESC("rs_event"), required => true})} binary_to_atom(emqx_rule_events:event_topic(E))
, {"title", sc(binary(), #{desc => ?DESC("rs_title"), example => "some title"})} || E <- emqx_rule_events:event_names()
, {"description", sc(binary(), #{desc => ?DESC("rs_description"), example => "some desc"})} ],
, {"columns", sc(map(), #{desc => ?DESC("rs_columns")})} [
, {"test_columns", sc(map(), #{desc => ?DESC("rs_test_columns")})} {"event", sc(hoconsc:enum(ETopics), #{desc => ?DESC("rs_event"), required => true})},
, {"sql_example", sc(binary(), #{desc => ?DESC("rs_sql_example")})} {"title", sc(binary(), #{desc => ?DESC("rs_title"), example => "some title"})},
{"description", sc(binary(), #{desc => ?DESC("rs_description"), example => "some desc"})},
{"columns", sc(map(), #{desc => ?DESC("rs_columns")})},
{"test_columns", sc(map(), #{desc => ?DESC("rs_test_columns")})},
{"sql_example", sc(binary(), #{desc => ?DESC("rs_sql_example")})}
]; ];
fields("rule_test") -> fields("rule_test") ->
[ {"context", sc(hoconsc:union([ ref("ctx_pub") [
, ref("ctx_sub") {"context",
, ref("ctx_unsub") sc(
, ref("ctx_delivered") hoconsc:union([
, ref("ctx_acked") ref("ctx_pub"),
, ref("ctx_dropped") ref("ctx_sub"),
, ref("ctx_connected") ref("ctx_unsub"),
, ref("ctx_disconnected") ref("ctx_delivered"),
, ref("ctx_connack") ref("ctx_acked"),
, ref("ctx_check_authz_complete") ref("ctx_dropped"),
, ref("ctx_bridge_mqtt") ref("ctx_connected"),
]), ref("ctx_disconnected"),
#{desc => ?DESC("test_context"), ref("ctx_connack"),
default => #{}})} ref("ctx_check_authz_complete"),
, {"sql", sc(binary(), #{desc => ?DESC("test_sql"), required => true})} ref("ctx_bridge_mqtt")
]),
#{
desc => ?DESC("test_context"),
default => #{}
}
)},
{"sql", sc(binary(), #{desc => ?DESC("test_sql"), required => true})}
]; ];
fields("metrics") -> fields("metrics") ->
[ {"sql.matched", sc(non_neg_integer(), #{ [
desc => ?DESC("metrics_sql_matched") {"sql.matched",
})} sc(non_neg_integer(), #{
, {"sql.matched.rate", sc(float(), #{desc => ?DESC("metrics_sql_matched_rate") })} desc => ?DESC("metrics_sql_matched")
, {"sql.matched.rate.max", sc(float(), #{desc => ?DESC("metrics_sql_matched_rate_max") })} })},
, {"sql.matched.rate.last5m", sc(float(), {"sql.matched.rate", sc(float(), #{desc => ?DESC("metrics_sql_matched_rate")})},
#{desc => ?DESC("metrics_sql_matched_rate_last5m") })} {"sql.matched.rate.max", sc(float(), #{desc => ?DESC("metrics_sql_matched_rate_max")})},
, {"sql.passed", sc(non_neg_integer(), #{desc => ?DESC("metrics_sql_passed") })} {"sql.matched.rate.last5m",
, {"sql.failed", sc(non_neg_integer(), #{desc => ?DESC("metrics_sql_failed") })} sc(
, {"sql.failed.exception", sc(non_neg_integer(), #{ float(),
desc => ?DESC("metrics_sql_failed_exception") #{desc => ?DESC("metrics_sql_matched_rate_last5m")}
})} )},
, {"sql.failed.unknown", sc(non_neg_integer(), #{ {"sql.passed", sc(non_neg_integer(), #{desc => ?DESC("metrics_sql_passed")})},
desc => ?DESC("metrics_sql_failed_unknown") {"sql.failed", sc(non_neg_integer(), #{desc => ?DESC("metrics_sql_failed")})},
})} {"sql.failed.exception",
, {"outputs.total", sc(non_neg_integer(), #{ sc(non_neg_integer(), #{
desc => ?DESC("metrics_outputs_total") desc => ?DESC("metrics_sql_failed_exception")
})} })},
, {"outputs.success", sc(non_neg_integer(), #{ {"sql.failed.unknown",
desc => ?DESC("metrics_outputs_success") sc(non_neg_integer(), #{
})} desc => ?DESC("metrics_sql_failed_unknown")
, {"outputs.failed", sc(non_neg_integer(), #{ })},
desc => ?DESC("metrics_outputs_failed") {"outputs.total",
})} sc(non_neg_integer(), #{
, {"outputs.failed.out_of_service", sc(non_neg_integer(), #{ desc => ?DESC("metrics_outputs_total")
desc => ?DESC("metrics_outputs_failed_out_of_service") })},
})} {"outputs.success",
, {"outputs.failed.unknown", sc(non_neg_integer(), #{ sc(non_neg_integer(), #{
desc => ?DESC("metrics_outputs_failed_unknown") desc => ?DESC("metrics_outputs_success")
})} })},
{"outputs.failed",
sc(non_neg_integer(), #{
desc => ?DESC("metrics_outputs_failed")
})},
{"outputs.failed.out_of_service",
sc(non_neg_integer(), #{
desc => ?DESC("metrics_outputs_failed_out_of_service")
})},
{"outputs.failed.unknown",
sc(non_neg_integer(), #{
desc => ?DESC("metrics_outputs_failed_unknown")
})}
]; ];
fields("node_metrics") -> fields("node_metrics") ->
[ {"node", sc(binary(), #{desc => ?DESC("node_node"), example => "emqx@127.0.0.1"})} [{"node", sc(binary(), #{desc => ?DESC("node_node"), example => "emqx@127.0.0.1"})}] ++
] ++ fields("metrics"); fields("metrics");
fields("ctx_pub") -> fields("ctx_pub") ->
[ {"event_type", sc(message_publish, #{desc => ?DESC("event_event_type"), required => true})} [
, {"id", sc(binary(), #{desc => ?DESC("event_id")})} {"event_type", sc(message_publish, #{desc => ?DESC("event_event_type"), required => true})},
, {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})} {"id", sc(binary(), #{desc => ?DESC("event_id")})},
, {"username", sc(binary(), #{desc => ?DESC("event_username")})} {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})},
, {"payload", sc(binary(), #{desc => ?DESC("event_payload")})} {"username", sc(binary(), #{desc => ?DESC("event_username")})},
, {"peerhost", sc(binary(), #{desc => ?DESC("event_peerhost")})} {"payload", sc(binary(), #{desc => ?DESC("event_payload")})},
, {"topic", sc(binary(), #{desc => ?DESC("event_topic")})} {"peerhost", sc(binary(), #{desc => ?DESC("event_peerhost")})},
, {"publish_received_at", sc(integer(), #{ {"topic", sc(binary(), #{desc => ?DESC("event_topic")})},
desc => ?DESC("event_publish_received_at")})} {"publish_received_at",
sc(integer(), #{
desc => ?DESC("event_publish_received_at")
})}
] ++ [qos()]; ] ++ [qos()];
fields("ctx_sub") -> fields("ctx_sub") ->
[ {"event_type", sc(session_subscribed, #{desc => ?DESC("event_event_type"), required => true})} [
, {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})} {"event_type",
, {"username", sc(binary(), #{desc => ?DESC("event_username")})} sc(session_subscribed, #{desc => ?DESC("event_event_type"), required => true})},
, {"payload", sc(binary(), #{desc => ?DESC("event_payload")})} {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})},
, {"peerhost", sc(binary(), #{desc => ?DESC("event_peerhost")})} {"username", sc(binary(), #{desc => ?DESC("event_username")})},
, {"topic", sc(binary(), #{desc => ?DESC("event_topic")})} {"payload", sc(binary(), #{desc => ?DESC("event_payload")})},
, {"publish_received_at", sc(integer(), #{ {"peerhost", sc(binary(), #{desc => ?DESC("event_peerhost")})},
desc => ?DESC("event_publish_received_at")})} {"topic", sc(binary(), #{desc => ?DESC("event_topic")})},
{"publish_received_at",
sc(integer(), #{
desc => ?DESC("event_publish_received_at")
})}
] ++ [qos()]; ] ++ [qos()];
fields("ctx_unsub") -> fields("ctx_unsub") ->
[{"event_type", sc(session_unsubscribed, #{desc => ?DESC("event_event_type"), required => true})}] ++ [
proplists:delete("event_type", fields("ctx_sub")); {"event_type",
sc(session_unsubscribed, #{desc => ?DESC("event_event_type"), required => true})}
] ++
proplists:delete("event_type", fields("ctx_sub"));
fields("ctx_delivered") -> fields("ctx_delivered") ->
[ {"event_type", sc(message_delivered, #{desc => ?DESC("event_event_type"), required => true})} [
, {"id", sc(binary(), #{desc => ?DESC("event_id")})} {"event_type",
, {"from_clientid", sc(binary(), #{desc => ?DESC("event_from_clientid")})} sc(message_delivered, #{desc => ?DESC("event_event_type"), required => true})},
, {"from_username", sc(binary(), #{desc => ?DESC("event_from_username")})} {"id", sc(binary(), #{desc => ?DESC("event_id")})},
, {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})} {"from_clientid", sc(binary(), #{desc => ?DESC("event_from_clientid")})},
, {"username", sc(binary(), #{desc => ?DESC("event_username")})} {"from_username", sc(binary(), #{desc => ?DESC("event_from_username")})},
, {"payload", sc(binary(), #{desc => ?DESC("event_payload")})} {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})},
, {"peerhost", sc(binary(), #{desc => ?DESC("event_peerhost")})} {"username", sc(binary(), #{desc => ?DESC("event_username")})},
, {"topic", sc(binary(), #{desc => ?DESC("event_topic")})} {"payload", sc(binary(), #{desc => ?DESC("event_payload")})},
, {"publish_received_at", sc(integer(), #{ {"peerhost", sc(binary(), #{desc => ?DESC("event_peerhost")})},
desc => ?DESC("event_publish_received_at")})} {"topic", sc(binary(), #{desc => ?DESC("event_topic")})},
{"publish_received_at",
sc(integer(), #{
desc => ?DESC("event_publish_received_at")
})}
] ++ [qos()]; ] ++ [qos()];
fields("ctx_acked") -> fields("ctx_acked") ->
[{"event_type", sc(message_acked, #{desc => ?DESC("event_event_type"), required => true})}] ++ [{"event_type", sc(message_acked, #{desc => ?DESC("event_event_type"), required => true})}] ++
proplists:delete("event_type", fields("ctx_delivered")); proplists:delete("event_type", fields("ctx_delivered"));
fields("ctx_dropped") -> fields("ctx_dropped") ->
[ {"event_type", sc(message_dropped, #{desc => ?DESC("event_event_type"), required => true})} [
, {"id", sc(binary(), #{desc => ?DESC("event_id")})} {"event_type", sc(message_dropped, #{desc => ?DESC("event_event_type"), required => true})},
, {"reason", sc(binary(), #{desc => ?DESC("event_ctx_dropped")})} {"id", sc(binary(), #{desc => ?DESC("event_id")})},
, {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})} {"reason", sc(binary(), #{desc => ?DESC("event_ctx_dropped")})},
, {"username", sc(binary(), #{desc => ?DESC("event_username")})} {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})},
, {"payload", sc(binary(), #{desc => ?DESC("event_payload")})} {"username", sc(binary(), #{desc => ?DESC("event_username")})},
, {"peerhost", sc(binary(), #{desc => ?DESC("event_peerhost")})} {"payload", sc(binary(), #{desc => ?DESC("event_payload")})},
, {"topic", sc(binary(), #{desc => ?DESC("event_topic")})} {"peerhost", sc(binary(), #{desc => ?DESC("event_peerhost")})},
, {"publish_received_at", sc(integer(), #{ {"topic", sc(binary(), #{desc => ?DESC("event_topic")})},
desc => ?DESC("event_publish_received_at")})} {"publish_received_at",
sc(integer(), #{
desc => ?DESC("event_publish_received_at")
})}
] ++ [qos()]; ] ++ [qos()];
fields("ctx_connected") -> fields("ctx_connected") ->
[ {"event_type", sc(client_connected, #{desc => ?DESC("event_event_type"), required => true})} [
, {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})} {"event_type",
, {"username", sc(binary(), #{desc => ?DESC("event_username")})} sc(client_connected, #{desc => ?DESC("event_event_type"), required => true})},
, {"mountpoint", sc(binary(), #{desc => ?DESC("event_mountpoint")})} {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})},
, {"peername", sc(binary(), #{desc => ?DESC("event_peername")})} {"username", sc(binary(), #{desc => ?DESC("event_username")})},
, {"sockname", sc(binary(), #{desc => ?DESC("event_sockname")})} {"mountpoint", sc(binary(), #{desc => ?DESC("event_mountpoint")})},
, {"proto_name", sc(binary(), #{desc => ?DESC("event_proto_name")})} {"peername", sc(binary(), #{desc => ?DESC("event_peername")})},
, {"proto_ver", sc(binary(), #{desc => ?DESC("event_proto_ver")})} {"sockname", sc(binary(), #{desc => ?DESC("event_sockname")})},
, {"keepalive", sc(integer(), #{desc => ?DESC("event_keepalive")})} {"proto_name", sc(binary(), #{desc => ?DESC("event_proto_name")})},
, {"clean_start", sc(boolean(), #{desc => ?DESC("event_clean_start"), default => true})} {"proto_ver", sc(binary(), #{desc => ?DESC("event_proto_ver")})},
, {"expiry_interval", sc(integer(), #{desc => ?DESC("event_expiry_interval")})} {"keepalive", sc(integer(), #{desc => ?DESC("event_keepalive")})},
, {"is_bridge", sc(boolean(), #{desc => ?DESC("event_is_bridge"), default => false})} {"clean_start", sc(boolean(), #{desc => ?DESC("event_clean_start"), default => true})},
, {"connected_at", sc(integer(), #{ {"expiry_interval", sc(integer(), #{desc => ?DESC("event_expiry_interval")})},
desc => ?DESC("event_connected_at")})} {"is_bridge", sc(boolean(), #{desc => ?DESC("event_is_bridge"), default => false})},
{"connected_at",
sc(integer(), #{
desc => ?DESC("event_connected_at")
})}
]; ];
fields("ctx_disconnected") -> fields("ctx_disconnected") ->
[ {"event_type", sc(client_disconnected, #{desc => ?DESC("event_event_type"), required => true})} [
, {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})} {"event_type",
, {"username", sc(binary(), #{desc => ?DESC("event_username")})} sc(client_disconnected, #{desc => ?DESC("event_event_type"), required => true})},
, {"reason", sc(binary(), #{desc => ?DESC("event_ctx_disconnected_reason")})} {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})},
, {"peername", sc(binary(), #{desc => ?DESC("event_peername")})} {"username", sc(binary(), #{desc => ?DESC("event_username")})},
, {"sockname", sc(binary(), #{desc => ?DESC("event_sockname")})} {"reason", sc(binary(), #{desc => ?DESC("event_ctx_disconnected_reason")})},
, {"disconnected_at", sc(integer(), #{ {"peername", sc(binary(), #{desc => ?DESC("event_peername")})},
desc => ?DESC("event_ctx_disconnected_da")})} {"sockname", sc(binary(), #{desc => ?DESC("event_sockname")})},
{"disconnected_at",
sc(integer(), #{
desc => ?DESC("event_ctx_disconnected_da")
})}
]; ];
fields("ctx_connack") -> fields("ctx_connack") ->
[ {"event_type", sc(client_connack, #{desc => ?DESC("event_event_type"), required => true})} [
, {"reason_code", sc(binary(), #{desc => ?DESC("event_ctx_connack_reason_code")})} {"event_type", sc(client_connack, #{desc => ?DESC("event_event_type"), required => true})},
, {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})} {"reason_code", sc(binary(), #{desc => ?DESC("event_ctx_connack_reason_code")})},
, {"clean_start", sc(boolean(), #{desc => ?DESC("event_clean_start"), default => true})} {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})},
, {"username", sc(binary(), #{desc => ?DESC("event_username")})} {"clean_start", sc(boolean(), #{desc => ?DESC("event_clean_start"), default => true})},
, {"peername", sc(binary(), #{desc => ?DESC("event_peername")})} {"username", sc(binary(), #{desc => ?DESC("event_username")})},
, {"sockname", sc(binary(), #{desc => ?DESC("event_sockname")})} {"peername", sc(binary(), #{desc => ?DESC("event_peername")})},
, {"proto_name", sc(binary(), #{desc => ?DESC("event_proto_name")})} {"sockname", sc(binary(), #{desc => ?DESC("event_sockname")})},
, {"proto_ver", sc(binary(), #{desc => ?DESC("event_proto_ver")})} {"proto_name", sc(binary(), #{desc => ?DESC("event_proto_name")})},
, {"keepalive", sc(integer(), #{desc => ?DESC("event_keepalive")})} {"proto_ver", sc(binary(), #{desc => ?DESC("event_proto_ver")})},
, {"expiry_interval", sc(integer(), #{desc => ?DESC("event_expiry_interval")})} {"keepalive", sc(integer(), #{desc => ?DESC("event_keepalive")})},
, {"connected_at", sc(integer(), #{ {"expiry_interval", sc(integer(), #{desc => ?DESC("event_expiry_interval")})},
desc => ?DESC("event_connected_at")})} {"connected_at",
sc(integer(), #{
desc => ?DESC("event_connected_at")
})}
]; ];
fields("ctx_check_authz_complete") -> fields("ctx_check_authz_complete") ->
[ {"event_type", sc(client_check_authz_complete, #{desc => ?DESC("event_event_type"), required => true})} [
, {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})} {"event_type",
, {"username", sc(binary(), #{desc => ?DESC("event_username")})} sc(client_check_authz_complete, #{desc => ?DESC("event_event_type"), required => true})},
, {"peerhost", sc(binary(), #{desc => ?DESC("event_peerhost")})} {"clientid", sc(binary(), #{desc => ?DESC("event_clientid")})},
, {"topic", sc(binary(), #{desc => ?DESC("event_topic")})} {"username", sc(binary(), #{desc => ?DESC("event_username")})},
, {"action", sc(binary(), #{desc => ?DESC("event_action")})} {"peerhost", sc(binary(), #{desc => ?DESC("event_peerhost")})},
, {"authz_source", sc(binary(), #{desc => ?DESC("event_authz_source")})} {"topic", sc(binary(), #{desc => ?DESC("event_topic")})},
, {"result", sc(binary(), #{desc => ?DESC("event_result")})} {"action", sc(binary(), #{desc => ?DESC("event_action")})},
{"authz_source", sc(binary(), #{desc => ?DESC("event_authz_source")})},
{"result", sc(binary(), #{desc => ?DESC("event_result")})}
]; ];
fields("ctx_bridge_mqtt") -> fields("ctx_bridge_mqtt") ->
[ {"event_type", sc('$bridges/mqtt:*', #{desc => ?DESC("event_event_type"), required => true})} [
, {"id", sc(binary(), #{desc => ?DESC("event_id")})} {"event_type",
, {"payload", sc(binary(), #{desc => ?DESC("event_payload")})} sc('$bridges/mqtt:*', #{desc => ?DESC("event_event_type"), required => true})},
, {"topic", sc(binary(), #{desc => ?DESC("event_topic")})} {"id", sc(binary(), #{desc => ?DESC("event_id")})},
, {"server", sc(binary(), #{desc => ?DESC("event_server")})} {"payload", sc(binary(), #{desc => ?DESC("event_payload")})},
, {"dup", sc(binary(), #{desc => ?DESC("event_dup")})} {"topic", sc(binary(), #{desc => ?DESC("event_topic")})},
, {"retain", sc(binary(), #{desc => ?DESC("event_retain")})} {"server", sc(binary(), #{desc => ?DESC("event_server")})},
, {"message_received_at", sc(integer(), #{ {"dup", sc(binary(), #{desc => ?DESC("event_dup")})},
desc => ?DESC("event_publish_received_at")})} {"retain", sc(binary(), #{desc => ?DESC("event_retain")})},
{"message_received_at",
sc(integer(), #{
desc => ?DESC("event_publish_received_at")
})}
] ++ [qos()]. ] ++ [qos()].
qos() -> qos() ->
{"qos", sc(emqx_schema:qos(), #{desc => ?DESC("event_qos")})}. {"qos", sc(emqx_schema:qos(), #{desc => ?DESC("event_qos")})}.
rule_id() -> rule_id() ->
{"id", sc(binary(), {"id",
#{ desc => ?DESC("rule_id"), required => true sc(
, example => "293fb66f" binary(),
})}. #{
desc => ?DESC("rule_id"),
required => true,
example => "293fb66f"
}
)}.
sc(Type, Meta) -> hoconsc:mk(Type, Meta). sc(Type, Meta) -> hoconsc:mk(Type, Meta).
ref(Field) -> hoconsc:ref(?MODULE, Field). ref(Field) -> hoconsc:ref(?MODULE, Field).

244
apps/emqx_rule_engine/src/emqx_rule_date.erl
View File

@ -18,19 +18,27 @@
-export([date/3, date/4, parse_date/4]). -export([date/3, date/4, parse_date/4]).
-export([ is_int_char/1 -export([
, is_symbol_char/1 is_int_char/1,
, is_m_char/1 is_symbol_char/1,
]). is_m_char/1
]).
-record(result, { -record(result, {
year = "1970" :: string() %%year() %%year()
, month = "1" :: string() %%month() year = "1970" :: string(),
, day = "1" :: string() %%day() %%month()
, hour = "0" :: string() %%hour() month = "1" :: string(),
, minute = "0" :: string() %%minute() %% epoch in millisecond precision %%day()
, second = "0" :: string() %%second() %% epoch in millisecond precision day = "1" :: string(),
, zone = "+00:00" :: string() %%integer() %% zone maybe some value %%hour()
hour = "0" :: string(),
%%minute() %% epoch in millisecond precision
minute = "0" :: string(),
%%second() %% epoch in millisecond precision
second = "0" :: string(),
%%integer() %% zone maybe some value
zone = "+00:00" :: string()
}). }).
%% -type time_unit() :: 'microsecond' %% -type time_unit() :: 'microsecond'
@ -42,43 +50,59 @@ date(TimeUnit, Offset, FormatString) ->
date(TimeUnit, Offset, FormatString, erlang:system_time(TimeUnit)). date(TimeUnit, Offset, FormatString, erlang:system_time(TimeUnit)).
date(TimeUnit, Offset, FormatString, TimeEpoch) -> date(TimeUnit, Offset, FormatString, TimeEpoch) ->
[Head|Other] = string:split(FormatString, "%", all), [Head | Other] = string:split(FormatString, "%", all),
R = create_tag([{st, Head}], Other), R = create_tag([{st, Head}], Other),
Res = lists:map(fun(Expr) -> Res = lists:map(
eval_tag(rmap(make_time(TimeUnit, Offset, TimeEpoch)), Expr) end, R), fun(Expr) ->
eval_tag(rmap(make_time(TimeUnit, Offset, TimeEpoch)), Expr)
end,
R
),
lists:concat(Res). lists:concat(Res).
parse_date(TimeUnit, Offset, FormatString, InputString) -> parse_date(TimeUnit, Offset, FormatString, InputString) ->
[Head|Other] = string:split(FormatString, "%", all), [Head | Other] = string:split(FormatString, "%", all),
R = create_tag([{st, Head}], Other), R = create_tag([{st, Head}], Other),
IsZ = fun(V) -> case V of IsZ = fun(V) ->
{tag, $Z} -> true; case V of
_ -> false {tag, $Z} -> true;
end end, _ -> false
end
end,
R1 = lists:filter(IsZ, R), R1 = lists:filter(IsZ, R),
IfFun = fun(Con, A, B) -> IfFun = fun(Con, A, B) ->
case Con of case Con of
[] -> A; [] -> A;
_ -> B _ -> B
end end, end
end,
Res = parse_input(FormatString, InputString), Res = parse_input(FormatString, InputString),
Str = Res#result.year ++ "-" Str =
++ Res#result.month ++ "-" Res#result.year ++ "-" ++
++ Res#result.day ++ "T" Res#result.month ++ "-" ++
++ Res#result.hour ++ ":" Res#result.day ++ "T" ++
++ Res#result.minute ++ ":" Res#result.hour ++ ":" ++
++ Res#result.second ++ Res#result.minute ++ ":" ++
IfFun(R1, Offset, Res#result.zone), Res#result.second ++
IfFun(R1, Offset, Res#result.zone),
calendar:rfc3339_to_system_time(Str, [{unit, TimeUnit}]). calendar:rfc3339_to_system_time(Str, [{unit, TimeUnit}]).
mlist(R)-> mlist(R) ->
[ {$H, R#result.hour} %% %H Shows hour in 24-hour format [15] %% %H Shows hour in 24-hour format [15]
, {$M, R#result.minute} %% %M Displays minutes [00-59] [
, {$S, R#result.second} %% %S Displays seconds [00-59] {$H, R#result.hour},
, {$y, R#result.year} %% %y Displays year YYYY [2021] %% %M Displays minutes [00-59]
, {$m, R#result.month} %% %m Displays the number of the month [01-12] {$M, R#result.minute},
, {$d, R#result.day} %% %d Displays the number of the month [01-12] %% %S Displays seconds [00-59]
, {$Z, R#result.zone} %% %Z Displays Time zone {$S, R#result.second},
%% %y Displays year YYYY [2021]
{$y, R#result.year},
%% %m Displays the number of the month [01-12]
{$m, R#result.month},
%% %d Displays the number of the month [01-12]
{$d, R#result.day},
%% %Z Displays Time zone
{$Z, R#result.zone}
]. ].
rmap(Result) -> rmap(Result) ->
@ -88,69 +112,95 @@ support_char() -> "HMSymdZ".
create_tag(Head, []) -> create_tag(Head, []) ->
Head; Head;
create_tag(Head, [Val1|RVal]) -> create_tag(Head, [Val1 | RVal]) ->
case Val1 of case Val1 of
[] -> create_tag(Head ++ [{st, [$%]}], RVal); [] ->
[H| Other] -> create_tag(Head ++ [{st, [$%]}], RVal);
[H | Other] ->
case lists:member(H, support_char()) of case lists:member(H, support_char()) of
true -> create_tag(Head ++ [{tag, H}, {st, Other}], RVal); true -> create_tag(Head ++ [{tag, H}, {st, Other}], RVal);
false -> create_tag(Head ++ [{st, [$%|Val1]}], RVal) false -> create_tag(Head ++ [{st, [$% | Val1]}], RVal)
end end
end. end.
eval_tag(_,{st, Str}) -> eval_tag(_, {st, Str}) ->
Str; Str;
eval_tag(Map,{tag, Char}) -> eval_tag(Map, {tag, Char}) ->
maps:get(Char, Map, "undefined"). maps:get(Char, Map, "undefined").
%% make_time(TimeUnit, Offset) -> %% make_time(TimeUnit, Offset) ->
%% make_time(TimeUnit, Offset, erlang:system_time(TimeUnit)). %% make_time(TimeUnit, Offset, erlang:system_time(TimeUnit)).
make_time(TimeUnit, Offset, TimeEpoch) -> make_time(TimeUnit, Offset, TimeEpoch) ->
Res = calendar:system_time_to_rfc3339(TimeEpoch, Res = calendar:system_time_to_rfc3339(
[{unit, TimeUnit}, {offset, Offset}]), TimeEpoch,
[Y1, Y2, Y3, Y4, $-, Mon1, Mon2, $-, D1, D2, _T, [{unit, TimeUnit}, {offset, Offset}]
H1, H2, $:, Min1, Min2, $:, S1, S2 | TimeStr] = Res, ),
[
Y1,
Y2,
Y3,
Y4,
$-,
Mon1,
Mon2,
$-,
D1,
D2,
_T,
H1,
H2,
$:,
Min1,
Min2,
$:,
S1,
S2
| TimeStr
] = Res,
IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end,
{FractionStr, UtcOffset} = lists:splitwith(IsFractionChar, TimeStr), {FractionStr, UtcOffset} = lists:splitwith(IsFractionChar, TimeStr),
#result{ #result{
year = [Y1, Y2, Y3, Y4] year = [Y1, Y2, Y3, Y4],
, month = [Mon1, Mon2] month = [Mon1, Mon2],
, day = [D1, D2] day = [D1, D2],
, hour = [H1, H2] hour = [H1, H2],
, minute = [Min1, Min2] minute = [Min1, Min2],
, second = [S1, S2] ++ FractionStr second = [S1, S2] ++ FractionStr,
, zone = UtcOffset zone = UtcOffset
}. }.
is_int_char(C) -> is_int_char(C) ->
C >= $0 andalso C =< $9 . C >= $0 andalso C =< $9.
is_symbol_char(C) -> is_symbol_char(C) ->
C =:= $- orelse C =:= $+ . C =:= $- orelse C =:= $+.
is_m_char(C) -> is_m_char(C) ->
C =:= $:. C =:= $:.
parse_char_with_fun(_, []) -> error(null_input); parse_char_with_fun(_, []) ->
parse_char_with_fun(ValidFun, [C|Other]) -> error(null_input);
Res = case erlang:is_function(ValidFun) of parse_char_with_fun(ValidFun, [C | Other]) ->
true -> ValidFun(C); Res =
false -> erlang:apply(emqx_rule_date, ValidFun, [C]) case erlang:is_function(ValidFun) of
end, true -> ValidFun(C);
false -> erlang:apply(emqx_rule_date, ValidFun, [C])
end,
case Res of case Res of
true -> {C, Other}; true -> {C, Other};
false -> error({unexpected,[C|Other]}) false -> error({unexpected, [C | Other]})
end. end.
parse_string([], Input) -> {[], Input}; parse_string([], Input) ->
parse_string([C|Other], Input) -> {[], Input};
parse_string([C | Other], Input) ->
{C1, Input1} = parse_char_with_fun(fun(V) -> V =:= C end, Input), {C1, Input1} = parse_char_with_fun(fun(V) -> V =:= C end, Input),
{Res, Input2} = parse_string(Other, Input1), {Res, Input2} = parse_string(Other, Input1),
{[C1|Res], Input2}. {[C1 | Res], Input2}.
parse_times(0, _, Input) -> {[], Input}; parse_times(0, _, Input) ->
{[], Input};
parse_times(Times, Fun, Input) -> parse_times(Times, Fun, Input) ->
{C1, Input1} = parse_char_with_fun(Fun, Input), {C1, Input1} = parse_char_with_fun(Fun, Input),
{Res, Input2} = parse_times((Times - 1), Fun, Input1), {Res, Input2} = parse_times((Times - 1), Fun, Input1),
{[C1|Res], Input2}. {[C1 | Res], Input2}.
parse_int_times(Times, Input) -> parse_int_times(Times, Input) ->
parse_times(Times, is_int_char, Input). parse_times(Times, is_int_char, Input).
@ -162,33 +212,42 @@ parse_fraction(Input) ->
parse_second(Input) -> parse_second(Input) ->
{M, Input1} = parse_int_times(2, Input), {M, Input1} = parse_int_times(2, Input),
{M1, Input2} = parse_fraction(Input1), {M1, Input2} = parse_fraction(Input1),
{M++M1, Input2}. {M ++ M1, Input2}.
parse_zone(Input) -> parse_zone(Input) ->
{S, Input1} = parse_char_with_fun(is_symbol_char, Input), {S, Input1} = parse_char_with_fun(is_symbol_char, Input),
{M, Input2} = parse_int_times(2, Input1), {M, Input2} = parse_int_times(2, Input1),
{C, Input3} = parse_char_with_fun(is_m_char, Input2), {C, Input3} = parse_char_with_fun(is_m_char, Input2),
{V, Input4} = parse_int_times(2, Input3), {V, Input4} = parse_int_times(2, Input3),
{[S|M++[C|V]], Input4}. {[S | M ++ [C | V]], Input4}.
mlist1()-> mlist1() ->
maps:from_list( maps:from_list(
[ {$H, fun(Input) -> parse_int_times(2, Input) end} %% %H Shows hour in 24-hour format [15] %% %H Shows hour in 24-hour format [15]
, {$M, fun(Input) -> parse_int_times(2, Input) end} %% %M Displays minutes [00-59] [
, {$S, fun(Input) -> parse_second(Input) end} %% %S Displays seconds [00-59] {$H, fun(Input) -> parse_int_times(2, Input) end},
, {$y, fun(Input) -> parse_int_times(4, Input) end} %% %y Displays year YYYY [2021] %% %M Displays minutes [00-59]
, {$m, fun(Input) -> parse_int_times(2, Input) end} %% %m Displays the number of the month [01-12] {$M, fun(Input) -> parse_int_times(2, Input) end},
, {$d, fun(Input) -> parse_int_times(2, Input) end} %% %d Displays the number of the month [01-12] %% %S Displays seconds [00-59]
, {$Z, fun(Input) -> parse_zone(Input) end} %% %Z Displays Time zone {$S, fun(Input) -> parse_second(Input) end},
]). %% %y Displays year YYYY [2021]
{$y, fun(Input) -> parse_int_times(4, Input) end},
%% %m Displays the number of the month [01-12]
{$m, fun(Input) -> parse_int_times(2, Input) end},
%% %d Displays the number of the month [01-12]
{$d, fun(Input) -> parse_int_times(2, Input) end},
%% %Z Displays Time zone
{$Z, fun(Input) -> parse_zone(Input) end}
]
).
update_result($H, Res, Str) -> Res#result{hour=Str}; update_result($H, Res, Str) -> Res#result{hour = Str};
update_result($M, Res, Str) -> Res#result{minute=Str}; update_result($M, Res, Str) -> Res#result{minute = Str};
update_result($S, Res, Str) -> Res#result{second=Str}; update_result($S, Res, Str) -> Res#result{second = Str};
update_result($y, Res, Str) -> Res#result{year=Str}; update_result($y, Res, Str) -> Res#result{year = Str};
update_result($m, Res, Str) -> Res#result{month=Str}; update_result($m, Res, Str) -> Res#result{month = Str};
update_result($d, Res, Str) -> Res#result{day=Str}; update_result($d, Res, Str) -> Res#result{day = Str};
update_result($Z, Res, Str) -> Res#result{zone=Str}. update_result($Z, Res, Str) -> Res#result{zone = Str}.
parse_tag(Res, {st, St}, InputString) -> parse_tag(Res, {st, St}, InputString) ->
{_A, B} = parse_string(St, InputString), {_A, B} = parse_string(St, InputString),
@ -199,12 +258,13 @@ parse_tag(Res, {tag, St}, InputString) ->
NRes = update_result(St, Res, A), NRes = update_result(St, Res, A),
{NRes, B}. {NRes, B}.
parse_tags(Res, [], _) -> Res; parse_tags(Res, [], _) ->
parse_tags(Res, [Tag|Others], InputString) -> Res;
parse_tags(Res, [Tag | Others], InputString) ->
{NRes, B} = parse_tag(Res, Tag, InputString), {NRes, B} = parse_tag(Res, Tag, InputString),
parse_tags(NRes, Others, B). parse_tags(NRes, Others, B).
parse_input(FormatString, InputString) -> parse_input(FormatString, InputString) ->
[Head|Other] = string:split(FormatString, "%", all), [Head | Other] = string:split(FormatString, "%", all),
R = create_tag([{st, Head}], Other), R = create_tag([{st, Head}], Other),
parse_tags(#result{}, R, InputString). parse_tags(#result{}, R, InputString).

30
apps/emqx_rule_engine/src/emqx_rule_engine.app.src
View File

@ -1,15 +1,17 @@
%% -*- mode: erlang -*- %% -*- mode: erlang -*-
{application, emqx_rule_engine, {application, emqx_rule_engine, [
[{description, "EMQX Rule Engine"}, {description, "EMQX Rule Engine"},
{vsn, "5.0.0"}, % strict semver, bump manually! % strict semver, bump manually!
{modules, []}, {vsn, "5.0.0"},
{registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {modules, []},
{applications, [kernel,stdlib,rulesql,getopt]}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]},
{mod, {emqx_rule_engine_app, []}}, {applications, [kernel, stdlib, rulesql, getopt]},
{env, []}, {mod, {emqx_rule_engine_app, []}},
{licenses, ["Apache-2.0"]}, {env, []},
{maintainers, ["EMQX Team <contact@emqx.io>"]}, {licenses, ["Apache-2.0"]},
{links, [{"Homepage", "https://emqx.io/"}, {maintainers, ["EMQX Team <contact@emqx.io>"]},
{"Github", "https://github.com/emqx/emqx-rule-engine"} {links, [
]} {"Homepage", "https://emqx.io/"},
]}. {"Github", "https://github.com/emqx/emqx-rule-engine"}
]}
]}.

269
apps/emqx_rule_engine/src/emqx_rule_engine.erl
View File

@ -25,51 +25,56 @@
-export([start_link/0]). -export([start_link/0]).
-export([ post_config_update/5 -export([
, config_key_path/0 post_config_update/5,
]). config_key_path/0
]).
%% Rule Management %% Rule Management
-export([ load_rules/0 -export([load_rules/0]).
]).
-export([ create_rule/1 -export([
, insert_rule/1 create_rule/1,
, update_rule/1 insert_rule/1,
, delete_rule/1 update_rule/1,
, get_rule/1 delete_rule/1,
]). get_rule/1
]).
-export([ get_rules/0 -export([
, get_rules_for_topic/1 get_rules/0,
, get_rules_with_same_event/1 get_rules_for_topic/1,
, get_rules_ordered_by_ts/0 get_rules_with_same_event/1,
]). get_rules_ordered_by_ts/0
]).
%% exported for cluster_call %% exported for cluster_call
-export([ do_delete_rule/1 -export([
, do_insert_rule/1 do_delete_rule/1,
]). do_insert_rule/1
]).
-export([ load_hooks_for_rule/1 -export([
, unload_hooks_for_rule/1 load_hooks_for_rule/1,
, maybe_add_metrics_for_rule/1 unload_hooks_for_rule/1,
, clear_metrics_for_rule/1 maybe_add_metrics_for_rule/1,
, reset_metrics_for_rule/1 clear_metrics_for_rule/1,
]). reset_metrics_for_rule/1
]).
%% exported for `emqx_telemetry' %% exported for `emqx_telemetry'
-export([get_basic_usage_info/0]). -export([get_basic_usage_info/0]).
%% gen_server Callbacks %% gen_server Callbacks
-export([ init/1 -export([
, handle_call/3 init/1,
, handle_cast/2 handle_call/3,
, handle_info/2 handle_cast/2,
, terminate/2 handle_info/2,
, code_change/3 terminate/2,
]). code_change/3
]).
-define(RULE_ENGINE, ?MODULE). -define(RULE_ENGINE, ?MODULE).
@ -77,24 +82,25 @@
%% NOTE: This order cannot be changed! This is to make the metric working during relup. %% NOTE: This order cannot be changed! This is to make the metric working during relup.
%% Append elements to this list to add new metrics. %% Append elements to this list to add new metrics.
-define(METRICS, [ 'sql.matched' -define(METRICS, [
, 'sql.passed' 'sql.matched',
, 'sql.failed' 'sql.passed',
, 'sql.failed.exception' 'sql.failed',
, 'sql.failed.no_result' 'sql.failed.exception',
, 'outputs.total' 'sql.failed.no_result',
, 'outputs.success' 'outputs.total',
, 'outputs.failed' 'outputs.success',
, 'outputs.failed.out_of_service' 'outputs.failed',
, 'outputs.failed.unknown' 'outputs.failed.out_of_service',
]). 'outputs.failed.unknown'
]).
-define(RATE_METRICS, ['sql.matched']). -define(RATE_METRICS, ['sql.matched']).
config_key_path() -> config_key_path() ->
[rule_engine, rules]. [rule_engine, rules].
-spec(start_link() -> {ok, pid()} | ignore | {error, Reason :: term()}). -spec start_link() -> {ok, pid()} | ignore | {error, Reason :: term()}.
start_link() -> start_link() ->
gen_server:start_link({local, ?RULE_ENGINE}, ?MODULE, [], []). gen_server:start_link({local, ?RULE_ENGINE}, ?MODULE, [], []).
@ -102,17 +108,26 @@ start_link() ->
%% The config handler for emqx_rule_engine %% The config handler for emqx_rule_engine
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
post_config_update(_, _Req, NewRules, OldRules, _AppEnvs) -> post_config_update(_, _Req, NewRules, OldRules, _AppEnvs) ->
#{added := Added, removed := Removed, changed := Updated} #{added := Added, removed := Removed, changed := Updated} =
= emqx_map_lib:diff_maps(NewRules, OldRules), emqx_map_lib:diff_maps(NewRules, OldRules),
maps_foreach(fun({Id, {_Old, New}}) -> maps_foreach(
fun({Id, {_Old, New}}) ->
{ok, _} = update_rule(New#{id => bin(Id)}) {ok, _} = update_rule(New#{id => bin(Id)})
end, Updated), end,
maps_foreach(fun({Id, _Rule}) -> Updated
),
maps_foreach(
fun({Id, _Rule}) ->
ok = delete_rule(bin(Id)) ok = delete_rule(bin(Id))
end, Removed), end,
maps_foreach(fun({Id, Rule}) -> Removed
),
maps_foreach(
fun({Id, Rule}) ->
{ok, _} = create_rule(Rule#{id => bin(Id)}) {ok, _} = create_rule(Rule#{id => bin(Id)})
end, Added), end,
Added
),
{ok, get_rules()}. {ok, get_rules()}.
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
@ -121,9 +136,12 @@ post_config_update(_, _Req, NewRules, OldRules, _AppEnvs) ->
-spec load_rules() -> ok. -spec load_rules() -> ok.
load_rules() -> load_rules() ->
maps_foreach(fun({Id, Rule}) -> maps_foreach(
fun({Id, Rule}) ->
{ok, _} = create_rule(Rule#{id => bin(Id)}) {ok, _} = create_rule(Rule#{id => bin(Id)})
end, emqx:get_config([rule_engine, rules], #{})). end,
emqx:get_config([rule_engine, rules], #{})
).
-spec create_rule(map()) -> {ok, rule()} | {error, term()}. -spec create_rule(map()) -> {ok, rule()} | {error, term()}.
create_rule(Params = #{id := RuleId}) when is_binary(RuleId) -> create_rule(Params = #{id := RuleId}) when is_binary(RuleId) ->
@ -141,11 +159,11 @@ update_rule(Params = #{id := RuleId}) when is_binary(RuleId) ->
parse_and_insert(Params, CreatedAt) parse_and_insert(Params, CreatedAt)
end. end.
-spec(delete_rule(RuleId :: rule_id()) -> ok). -spec delete_rule(RuleId :: rule_id()) -> ok.
delete_rule(RuleId) when is_binary(RuleId) -> delete_rule(RuleId) when is_binary(RuleId) ->
gen_server:call(?RULE_ENGINE, {delete_rule, RuleId}, ?T_CALL). gen_server:call(?RULE_ENGINE, {delete_rule, RuleId}, ?T_CALL).
-spec(insert_rule(Rule :: rule()) -> ok). -spec insert_rule(Rule :: rule()) -> ok.
insert_rule(Rule) -> insert_rule(Rule) ->
gen_server:call(?RULE_ENGINE, {insert_rule, Rule}, ?T_CALL). gen_server:call(?RULE_ENGINE, {insert_rule, Rule}, ?T_CALL).
@ -153,30 +171,39 @@ insert_rule(Rule) ->
%% Rule Management %% Rule Management
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
-spec(get_rules() -> [rule()]). -spec get_rules() -> [rule()].
get_rules() -> get_rules() ->
get_all_records(?RULE_TAB). get_all_records(?RULE_TAB).
get_rules_ordered_by_ts() -> get_rules_ordered_by_ts() ->
lists:sort(fun(#{created_at := CreatedA}, #{created_at := CreatedB}) -> lists:sort(
fun(#{created_at := CreatedA}, #{created_at := CreatedB}) ->
CreatedA =< CreatedB CreatedA =< CreatedB
end, get_rules()). end,
get_rules()
).
-spec(get_rules_for_topic(Topic :: binary()) -> [rule()]). -spec get_rules_for_topic(Topic :: binary()) -> [rule()].
get_rules_for_topic(Topic) -> get_rules_for_topic(Topic) ->
[Rule || Rule = #{from := From} <- get_rules(), [
emqx_plugin_libs_rule:can_topic_match_oneof(Topic, From)]. Rule
|| Rule = #{from := From} <- get_rules(),
emqx_plugin_libs_rule:can_topic_match_oneof(Topic, From)
].
-spec(get_rules_with_same_event(Topic :: binary()) -> [rule()]). -spec get_rules_with_same_event(Topic :: binary()) -> [rule()].
get_rules_with_same_event(Topic) -> get_rules_with_same_event(Topic) ->
EventName = emqx_rule_events:event_name(Topic), EventName = emqx_rule_events:event_name(Topic),
[Rule || Rule = #{from := From} <- get_rules(), [
lists:any(fun(T) -> is_of_event_name(EventName, T) end, From)]. Rule
|| Rule = #{from := From} <- get_rules(),
lists:any(fun(T) -> is_of_event_name(EventName, T) end, From)
].
is_of_event_name(EventName, Topic) -> is_of_event_name(EventName, Topic) ->
EventName =:= emqx_rule_events:event_name(Topic). EventName =:= emqx_rule_events:event_name(Topic).
-spec(get_rule(Id :: rule_id()) -> {ok, rule()} | not_found). -spec get_rule(Id :: rule_id()) -> {ok, rule()} | not_found.
get_rule(Id) -> get_rule(Id) ->
case ets:lookup(?RULE_TAB, Id) of case ets:lookup(?RULE_TAB, Id) of
[{Id, Rule}] -> {ok, Rule#{id => Id}}; [{Id, Rule}] -> {ok, Rule#{id => Id}};
@ -188,7 +215,8 @@ load_hooks_for_rule(#{from := Topics}) ->
maybe_add_metrics_for_rule(Id) -> maybe_add_metrics_for_rule(Id) ->
case emqx_plugin_libs_metrics:has_metrics(rule_metrics, Id) of case emqx_plugin_libs_metrics:has_metrics(rule_metrics, Id) of
true -> ok; true ->
ok;
false -> false ->
ok = emqx_plugin_libs_metrics:create_metrics(rule_metrics, Id, ?METRICS, ?RATE_METRICS) ok = emqx_plugin_libs_metrics:create_metrics(rule_metrics, Id, ?METRICS, ?RATE_METRICS)
end. end.
@ -196,86 +224,101 @@ maybe_add_metrics_for_rule(Id) ->
clear_metrics_for_rule(Id) -> clear_metrics_for_rule(Id) ->
ok = emqx_plugin_libs_metrics:clear_metrics(rule_metrics, Id). ok = emqx_plugin_libs_metrics:clear_metrics(rule_metrics, Id).
-spec(reset_metrics_for_rule(rule_id()) -> ok). -spec reset_metrics_for_rule(rule_id()) -> ok.
reset_metrics_for_rule(Id) -> reset_metrics_for_rule(Id) ->
emqx_plugin_libs_metrics:reset_metrics(rule_metrics, Id). emqx_plugin_libs_metrics:reset_metrics(rule_metrics, Id).
unload_hooks_for_rule(#{id := Id, from := Topics}) -> unload_hooks_for_rule(#{id := Id, from := Topics}) ->
lists:foreach(fun(Topic) -> lists:foreach(
case get_rules_with_same_event(Topic) of fun(Topic) ->
[#{id := Id0}] when Id0 == Id -> %% we are now deleting the last rule case get_rules_with_same_event(Topic) of
emqx_rule_events:unload(Topic); %% we are now deleting the last rule
_ -> ok [#{id := Id0}] when Id0 == Id ->
end emqx_rule_events:unload(Topic);
end, Topics). _ ->
ok
end
end,
Topics
).
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% Telemetry helper functions %% Telemetry helper functions
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
-spec get_basic_usage_info() -> #{ num_rules => non_neg_integer() -spec get_basic_usage_info() ->
, referenced_bridges => #{
#{ BridgeType => non_neg_integer() num_rules => non_neg_integer(),
} referenced_bridges =>
} #{BridgeType => non_neg_integer()}
when BridgeType :: atom(). }
when
BridgeType :: atom().
get_basic_usage_info() -> get_basic_usage_info() ->
try try
Rules = get_rules(), Rules = get_rules(),
EnabledRules = EnabledRules =
lists:filter( lists:filter(
fun(#{enable := Enabled}) -> Enabled end, fun(#{enable := Enabled}) -> Enabled end,
Rules), Rules
),
NumRules = length(EnabledRules), NumRules = length(EnabledRules),
ReferencedBridges = ReferencedBridges =
lists:foldl( lists:foldl(
fun(#{outputs := Outputs}, Acc) -> fun(#{outputs := Outputs}, Acc) ->
BridgeIDs = lists:filter(fun is_binary/1, Outputs), BridgeIDs = lists:filter(fun is_binary/1, Outputs),
tally_referenced_bridges(BridgeIDs, Acc) tally_referenced_bridges(BridgeIDs, Acc)
end, end,
#{}, #{},
EnabledRules), EnabledRules
#{ num_rules => NumRules ),
, referenced_bridges => ReferencedBridges #{
} num_rules => NumRules,
referenced_bridges => ReferencedBridges
}
catch catch
_:_ -> _:_ ->
#{ num_rules => 0 #{
, referenced_bridges => #{} num_rules => 0,
} referenced_bridges => #{}
}
end. end.
tally_referenced_bridges(BridgeIDs, Acc0) -> tally_referenced_bridges(BridgeIDs, Acc0) ->
lists:foldl( lists:foldl(
fun(BridgeID, Acc) -> fun(BridgeID, Acc) ->
{BridgeType, _BridgeName} = emqx_bridge:parse_bridge_id(BridgeID), {BridgeType, _BridgeName} = emqx_bridge:parse_bridge_id(BridgeID),
maps:update_with( maps:update_with(
BridgeType, BridgeType,
fun(X) -> X + 1 end, fun(X) -> X + 1 end,
1, 1,
Acc) Acc
end, )
Acc0, end,
BridgeIDs). Acc0,
BridgeIDs
).
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% gen_server callbacks %% gen_server callbacks
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
init([]) -> init([]) ->
_TableId = ets:new(?KV_TAB, [named_table, set, public, {write_concurrency, true}, _TableId = ets:new(?KV_TAB, [
{read_concurrency, true}]), named_table,
set,
public,
{write_concurrency, true},
{read_concurrency, true}
]),
{ok, #{}}. {ok, #{}}.
handle_call({insert_rule, Rule}, _From, State) -> handle_call({insert_rule, Rule}, _From, State) ->
do_insert_rule(Rule), do_insert_rule(Rule),
{reply, ok, State}; {reply, ok, State};
handle_call({delete_rule, Rule}, _From, State) -> handle_call({delete_rule, Rule}, _From, State) ->
do_delete_rule(Rule), do_delete_rule(Rule),
{reply, ok, State}; {reply, ok, State};
handle_call(Req, _From, State) -> handle_call(Req, _From, State) ->
?SLOG(error, #{msg => "unexpected_call", request => Req}), ?SLOG(error, #{msg => "unexpected_call", request => Req}),
{reply, ignored, State}. {reply, ignored, State}.
@ -321,7 +364,8 @@ parse_and_insert(Params = #{id := RuleId, sql := Sql, outputs := Outputs}, Creat
}, },
ok = insert_rule(Rule), ok = insert_rule(Rule),
{ok, Rule}; {ok, Rule};
{error, Reason} -> {error, Reason} {error, Reason} ->
{error, Reason}
end. end.
do_insert_rule(#{id := Id} = Rule) -> do_insert_rule(#{id := Id} = Rule) ->
@ -337,7 +381,8 @@ do_delete_rule(RuleId) ->
ok = clear_metrics_for_rule(RuleId), ok = clear_metrics_for_rule(RuleId),
true = ets:delete(?RULE_TAB, RuleId), true = ets:delete(?RULE_TAB, RuleId),
ok; ok;
not_found -> ok not_found ->
ok
end. end.
parse_outputs(Outputs) -> parse_outputs(Outputs) ->

310
apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
View File

@ -32,20 +32,33 @@
-export(['/rule_events'/2, '/rule_test'/2, '/rules'/2, '/rules/:id'/2, '/rules/:id/reset_metrics'/2]). -export(['/rule_events'/2, '/rule_test'/2, '/rules'/2, '/rules/:id'/2, '/rules/:id/reset_metrics'/2]).
-define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~ts Not Found", [(ID)]))). -define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~ts Not Found", [(ID)]))).
-define(ERR_BADARGS(REASON), -define(ERR_BADARGS(REASON), begin
begin R0 = err_msg(REASON),
R0 = err_msg(REASON), <<"Bad Arguments: ", R0/binary>>
<<"Bad Arguments: ", R0/binary>> end).
end).
-define(CHECK_PARAMS(PARAMS, TAG, EXPR), -define(CHECK_PARAMS(PARAMS, TAG, EXPR),
case emqx_rule_api_schema:check_params(PARAMS, TAG) of case emqx_rule_api_schema:check_params(PARAMS, TAG) of
{ok, CheckedParams} -> {ok, CheckedParams} ->
EXPR; EXPR;
{error, REASON} -> {error, REASON} ->
{400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(REASON)}} {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(REASON)}}
end). end
-define(METRICS(MATCH, PASS, FAIL, FAIL_EX, FAIL_NORES, O_TOTAL, O_FAIL, O_FAIL_OOS, ).
O_FAIL_UNKNOWN, O_SUCC, RATE, RATE_MAX, RATE_5), -define(METRICS(
MATCH,
PASS,
FAIL,
FAIL_EX,
FAIL_NORES,
O_TOTAL,
O_FAIL,
O_FAIL_OOS,
O_FAIL_UNKNOWN,
O_SUCC,
RATE,
RATE_MAX,
RATE_5
),
#{ #{
'sql.matched' => MATCH, 'sql.matched' => MATCH,
'sql.passed' => PASS, 'sql.passed' => PASS,
@ -60,9 +73,23 @@
'sql.matched.rate' => RATE, 'sql.matched.rate' => RATE,
'sql.matched.rate.max' => RATE_MAX, 'sql.matched.rate.max' => RATE_MAX,
'sql.matched.rate.last5m' => RATE_5 'sql.matched.rate.last5m' => RATE_5
}). }
-define(metrics(MATCH, PASS, FAIL, FAIL_EX, FAIL_NORES, O_TOTAL, O_FAIL, O_FAIL_OOS, ).
O_FAIL_UNKNOWN, O_SUCC, RATE, RATE_MAX, RATE_5), -define(metrics(
MATCH,
PASS,
FAIL,
FAIL_EX,
FAIL_NORES,
O_TOTAL,
O_FAIL,
O_FAIL_OOS,
O_FAIL_UNKNOWN,
O_SUCC,
RATE,
RATE_MAX,
RATE_5
),
#{ #{
'sql.matched' := MATCH, 'sql.matched' := MATCH,
'sql.passed' := PASS, 'sql.passed' := PASS,
@ -77,7 +104,8 @@
'sql.matched.rate' := RATE, 'sql.matched.rate' := RATE,
'sql.matched.rate.max' := RATE_MAX, 'sql.matched.rate.max' := RATE_MAX,
'sql.matched.rate.last5m' := RATE_5 'sql.matched.rate.last5m' := RATE_5
}). }
).
namespace() -> "rule". namespace() -> "rule".
@ -107,7 +135,8 @@ schema("/rules") ->
summary => <<"List Rules">>, summary => <<"List Rules">>,
responses => #{ responses => #{
200 => mk(array(rule_info_schema()), #{desc => ?DESC("desc9")}) 200 => mk(array(rule_info_schema()), #{desc => ?DESC("desc9")})
}}, }
},
post => #{ post => #{
tags => [<<"rules">>], tags => [<<"rules">>],
description => ?DESC("api2"), description => ?DESC("api2"),
@ -116,9 +145,9 @@ schema("/rules") ->
responses => #{ responses => #{
400 => error_schema('BAD_REQUEST', "Invalid Parameters"), 400 => error_schema('BAD_REQUEST', "Invalid Parameters"),
201 => rule_info_schema() 201 => rule_info_schema()
}} }
}
}; };
schema("/rule_events") -> schema("/rule_events") ->
#{ #{
'operationId' => '/rule_events', 'operationId' => '/rule_events',
@ -131,7 +160,6 @@ schema("/rule_events") ->
} }
} }
}; };
schema("/rules/:id") -> schema("/rules/:id") ->
#{ #{
'operationId' => '/rules/:id', 'operationId' => '/rules/:id',
@ -166,7 +194,6 @@ schema("/rules/:id") ->
} }
} }
}; };
schema("/rules/:id/reset_metrics") -> schema("/rules/:id/reset_metrics") ->
#{ #{
'operationId' => '/rules/:id/reset_metrics', 'operationId' => '/rules/:id/reset_metrics',
@ -181,7 +208,6 @@ schema("/rules/:id/reset_metrics") ->
} }
} }
}; };
schema("/rule_test") -> schema("/rule_test") ->
#{ #{
'operationId' => '/rule_test', 'operationId' => '/rule_test',
@ -206,7 +232,7 @@ param_path_id() ->
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% To get around the hocon bug, we replace crlf with spaces %% To get around the hocon bug, we replace crlf with spaces
replace_sql_clrf(#{ <<"sql">> := SQL } = Params) -> replace_sql_clrf(#{<<"sql">> := SQL} = Params) ->
NewSQL = re:replace(SQL, "[\r\n]", " ", [{return, binary}, global]), NewSQL = re:replace(SQL, "[\r\n]", " ", [{return, binary}, global]),
Params#{<<"sql">> => NewSQL}. Params#{<<"sql">> => NewSQL}.
@ -216,7 +242,6 @@ replace_sql_clrf(#{ <<"sql">> := SQL } = Params) ->
'/rules'(get, _Params) -> '/rules'(get, _Params) ->
Records = emqx_rule_engine:get_rules_ordered_by_ts(), Records = emqx_rule_engine:get_rules_ordered_by_ts(),
{200, format_rule_resp(Records)}; {200, format_rule_resp(Records)};
'/rules'(post, #{body := Params0}) -> '/rules'(post, #{body := Params0}) ->
case maps:get(<<"id">>, Params0, list_to_binary(emqx_misc:gen_id(8))) of case maps:get(<<"id">>, Params0, list_to_binary(emqx_misc:gen_id(8))) of
<<>> -> <<>> ->
@ -233,20 +258,29 @@ replace_sql_clrf(#{ <<"sql">> := SQL } = Params) ->
[Rule] = get_one_rule(AllRules, Id), [Rule] = get_one_rule(AllRules, Id),
{201, format_rule_resp(Rule)}; {201, format_rule_resp(Rule)};
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => "create_rule_failed", ?SLOG(error, #{
id => Id, reason => Reason}), msg => "create_rule_failed",
id => Id,
reason => Reason
}),
{400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}}
end end
end end
end. end.
'/rule_test'(post, #{body := Params}) -> '/rule_test'(post, #{body := Params}) ->
?CHECK_PARAMS(Params, rule_test, case emqx_rule_sqltester:test(CheckedParams) of ?CHECK_PARAMS(
{ok, Result} -> {200, Result}; Params,
{error, {parse_error, Reason}} -> rule_test,
{400, #{code => 'BAD_REQUEST', message => err_msg(Reason)}}; case emqx_rule_sqltester:test(CheckedParams) of
{error, nomatch} -> {412, #{code => 'NOT_MATCH', message => <<"SQL Not Match">>}} {ok, Result} ->
end). {200, Result};
{error, {parse_error, Reason}} ->
{400, #{code => 'BAD_REQUEST', message => err_msg(Reason)}};
{error, nomatch} ->
{412, #{code => 'NOT_MATCH', message => <<"SQL Not Match">>}}
end
).
'/rules/:id'(get, #{bindings := #{id := Id}}) -> '/rules/:id'(get, #{bindings := #{id := Id}}) ->
case emqx_rule_engine:get_rule(Id) of case emqx_rule_engine:get_rule(Id) of
@ -255,7 +289,6 @@ replace_sql_clrf(#{ <<"sql">> := SQL } = Params) ->
not_found -> not_found ->
{404, #{code => 'NOT_FOUND', message => <<"Rule Id Not Found">>}} {404, #{code => 'NOT_FOUND', message => <<"Rule Id Not Found">>}}
end; end;
'/rules/:id'(put, #{bindings := #{id := Id}, body := Params0}) -> '/rules/:id'(put, #{bindings := #{id := Id}, body := Params0}) ->
Params = filter_out_request_body(Params0), Params = filter_out_request_body(Params0),
ConfPath = emqx_rule_engine:config_key_path() ++ [Id], ConfPath = emqx_rule_engine:config_key_path() ++ [Id],
@ -264,25 +297,35 @@ replace_sql_clrf(#{ <<"sql">> := SQL } = Params) ->
[Rule] = get_one_rule(AllRules, Id), [Rule] = get_one_rule(AllRules, Id),
{200, format_rule_resp(Rule)}; {200, format_rule_resp(Rule)};
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => "update_rule_failed", ?SLOG(error, #{
id => Id, reason => Reason}), msg => "update_rule_failed",
id => Id,
reason => Reason
}),
{400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}}
end; end;
'/rules/:id'(delete, #{bindings := #{id := Id}}) -> '/rules/:id'(delete, #{bindings := #{id := Id}}) ->
ConfPath = emqx_rule_engine:config_key_path() ++ [Id], ConfPath = emqx_rule_engine:config_key_path() ++ [Id],
case emqx_conf:remove(ConfPath, #{override_to => cluster}) of case emqx_conf:remove(ConfPath, #{override_to => cluster}) of
{ok, _} -> {204}; {ok, _} ->
{204};
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => "delete_rule_failed", ?SLOG(error, #{
id => Id, reason => Reason}), msg => "delete_rule_failed",
id => Id,
reason => Reason
}),
{500, #{code => 'INTERNAL_ERROR', message => ?ERR_BADARGS(Reason)}} {500, #{code => 'INTERNAL_ERROR', message => ?ERR_BADARGS(Reason)}}
end. end.
'/rules/:id/reset_metrics'(put, #{bindings := #{id := RuleId}}) -> '/rules/:id/reset_metrics'(put, #{bindings := #{id := RuleId}}) ->
case emqx_rule_engine_proto_v1:reset_metrics(RuleId) of case emqx_rule_engine_proto_v1:reset_metrics(RuleId) of
{ok, _TxnId, _Result} -> {200, <<"Reset Success">>}; {ok, _TxnId, _Result} ->
Failed -> {400, #{code => 'BAD_REQUEST', {200, <<"Reset Success">>};
message => err_msg(Failed)}} Failed ->
{400, #{
code => 'BAD_REQUEST',
message => err_msg(Failed)
}}
end. end.
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
@ -292,29 +335,31 @@ replace_sql_clrf(#{ <<"sql">> := SQL } = Params) ->
err_msg(Msg) -> err_msg(Msg) ->
list_to_binary(io_lib:format("~0p", [Msg])). list_to_binary(io_lib:format("~0p", [Msg])).
format_rule_resp(Rules) when is_list(Rules) -> format_rule_resp(Rules) when is_list(Rules) ->
[format_rule_resp(R) || R <- Rules]; [format_rule_resp(R) || R <- Rules];
format_rule_resp(#{
format_rule_resp(#{ id := Id, name := Name, id := Id,
created_at := CreatedAt, name := Name,
from := Topics, created_at := CreatedAt,
outputs := Output, from := Topics,
sql := SQL, outputs := Output,
enable := Enable, sql := SQL,
description := Descr}) -> enable := Enable,
description := Descr
}) ->
NodeMetrics = get_rule_metrics(Id), NodeMetrics = get_rule_metrics(Id),
#{id => Id, #{
name => Name, id => Id,
from => Topics, name => Name,
outputs => format_output(Output), from => Topics,
sql => SQL, outputs => format_output(Output),
metrics => aggregate_metrics(NodeMetrics), sql => SQL,
node_metrics => NodeMetrics, metrics => aggregate_metrics(NodeMetrics),
enable => Enable, node_metrics => NodeMetrics,
created_at => format_datetime(CreatedAt, millisecond), enable => Enable,
description => Descr created_at => format_datetime(CreatedAt, millisecond),
}. description => Descr
}.
format_datetime(Timestamp, Unit) -> format_datetime(Timestamp, Unit) ->
list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, Unit}])). list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, Unit}])).
@ -323,62 +368,133 @@ format_output(Outputs) ->
[do_format_output(Out) || Out <- Outputs]. [do_format_output(Out) || Out <- Outputs].
do_format_output(#{mod := Mod, func := Func, args := Args}) -> do_format_output(#{mod := Mod, func := Func, args := Args}) ->
#{function => printable_function_name(Mod, Func), #{
args => maps:remove(preprocessed_tmpl, Args)}; function => printable_function_name(Mod, Func),
args => maps:remove(preprocessed_tmpl, Args)
};
do_format_output(BridgeChannelId) when is_binary(BridgeChannelId) -> do_format_output(BridgeChannelId) when is_binary(BridgeChannelId) ->
BridgeChannelId. BridgeChannelId.
printable_function_name(emqx_rule_outputs, Func) -> printable_function_name(emqx_rule_outputs, Func) ->
Func; Func;
printable_function_name(Mod, Func) -> printable_function_name(Mod, Func) ->
list_to_binary(lists:concat([Mod,":",Func])). list_to_binary(lists:concat([Mod, ":", Func])).
get_rule_metrics(Id) -> get_rule_metrics(Id) ->
Format = fun (Node, #{ Format = fun(
Node,
#{
counters := counters :=
#{'sql.matched' := Matched, 'sql.passed' := Passed, 'sql.failed' := Failed, #{
'sql.failed.exception' := FailedEx, 'sql.matched' := Matched,
'sql.failed.no_result' := FailedNoRes, 'sql.passed' := Passed,
'outputs.total' := OTotal, 'sql.failed' := Failed,
'outputs.failed' := OFailed, 'sql.failed.exception' := FailedEx,
'outputs.failed.out_of_service' := OFailedOOS, 'sql.failed.no_result' := FailedNoRes,
'outputs.failed.unknown' := OFailedUnknown, 'outputs.total' := OTotal,
'outputs.success' := OFailedSucc 'outputs.failed' := OFailed,
}, 'outputs.failed.out_of_service' := OFailedOOS,
'outputs.failed.unknown' := OFailedUnknown,
'outputs.success' := OFailedSucc
},
rate := rate :=
#{'sql.matched' := #{
#{current := Current, max := Max, last5m := Last5M} 'sql.matched' :=
}}) -> #{current := Current, max := Max, last5m := Last5M}
#{ metrics => ?METRICS(Matched, Passed, Failed, FailedEx, FailedNoRes, }
OTotal, OFailed, OFailedOOS, OFailedUnknown, OFailedSucc, Current, Max, Last5M) }
, node => Node ) ->
} #{
metrics => ?METRICS(
Matched,
Passed,
Failed,
FailedEx,
FailedNoRes,
OTotal,
OFailed,
OFailedOOS,
OFailedUnknown,
OFailedSucc,
Current,
Max,
Last5M
),
node => Node
}
end, end,
[Format(Node, emqx_plugin_libs_proto_v1:get_metrics(Node, rule_metrics, Id)) [
|| Node <- mria_mnesia:running_nodes()]. Format(Node, emqx_plugin_libs_proto_v1:get_metrics(Node, rule_metrics, Id))
|| Node <- mria_mnesia:running_nodes()
].
aggregate_metrics(AllMetrics) -> aggregate_metrics(AllMetrics) ->
InitMetrics = ?METRICS(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), InitMetrics = ?METRICS(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
lists:foldl(fun lists:foldl(
(#{metrics := ?metrics(Match1, Passed1, Failed1, FailedEx1, FailedNoRes1, fun(
OTotal1, OFailed1, OFailedOOS1, OFailedUnknown1, OFailedSucc1, #{
Rate1, RateMax1, Rate5m1)}, metrics := ?metrics(
?metrics(Match0, Passed0, Failed0, FailedEx0, FailedNoRes0, Match1,
OTotal0, OFailed0, OFailedOOS0, OFailedUnknown0, OFailedSucc0, Passed1,
Rate0, RateMax0, Rate5m0)) -> Failed1,
?METRICS(Match1 + Match0, Passed1 + Passed0, Failed1 + Failed0, FailedEx1,
FailedEx1 + FailedEx0, FailedNoRes1 + FailedNoRes0, FailedNoRes1,
OTotal1 + OTotal0, OFailed1 + OFailed0, OTotal1,
OFailedOOS1 + OFailedOOS0, OFailed1,
OFailedUnknown1 + OFailedUnknown0, OFailedOOS1,
OFailedSucc1 + OFailedSucc0, OFailedUnknown1,
Rate1 + Rate0, RateMax1 + RateMax0, Rate5m1 + Rate5m0) OFailedSucc1,
end, InitMetrics, AllMetrics). Rate1,
RateMax1,
Rate5m1
)
},
?metrics(
Match0,
Passed0,
Failed0,
FailedEx0,
FailedNoRes0,
OTotal0,
OFailed0,
OFailedOOS0,
OFailedUnknown0,
OFailedSucc0,
Rate0,
RateMax0,
Rate5m0
)
) ->
?METRICS(
Match1 + Match0,
Passed1 + Passed0,
Failed1 + Failed0,
FailedEx1 + FailedEx0,
FailedNoRes1 + FailedNoRes0,
OTotal1 + OTotal0,
OFailed1 + OFailed0,
OFailedOOS1 + OFailedOOS0,
OFailedUnknown1 + OFailedUnknown0,
OFailedSucc1 + OFailedSucc0,
Rate1 + Rate0,
RateMax1 + RateMax0,
Rate5m1 + Rate5m0
)
end,
InitMetrics,
AllMetrics
).
get_one_rule(AllRules, Id) -> get_one_rule(AllRules, Id) ->
[R || R = #{id := Id0} <- AllRules, Id0 == Id]. [R || R = #{id := Id0} <- AllRules, Id0 == Id].
filter_out_request_body(Conf) -> filter_out_request_body(Conf) ->
ExtraConfs = [<<"id">>, <<"status">>, <<"node_status">>, <<"node_metrics">>, ExtraConfs = [
<<"metrics">>, <<"node">>], <<"id">>,
<<"status">>,
<<"node_status">>,
<<"node_metrics">>,
<<"metrics">>,
<<"node">>
],
maps:without(ExtraConfs, Conf). maps:without(ExtraConfs, Conf).

215
apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
View File

@ -21,96 +21,140 @@
-behaviour(hocon_schema). -behaviour(hocon_schema).
-export([ namespace/0 -export([
, roots/0 namespace/0,
, fields/1 roots/0,
, desc/1 fields/1,
]). desc/1
]).
-export([ validate_sql/1 -export([validate_sql/1]).
]).
namespace() -> rule_engine. namespace() -> rule_engine.
roots() -> ["rule_engine"]. roots() -> ["rule_engine"].
fields("rule_engine") -> fields("rule_engine") ->
[ {ignore_sys_message, sc(boolean(), #{default => true, desc => ?DESC("rule_engine_ignore_sys_message") [
})} {ignore_sys_message,
, {rules, sc(hoconsc:map("id", ref("rules")), #{desc => ?DESC("rule_engine_rules"), default => #{}})} sc(boolean(), #{default => true, desc => ?DESC("rule_engine_ignore_sys_message")})},
{rules,
sc(hoconsc:map("id", ref("rules")), #{
desc => ?DESC("rule_engine_rules"), default => #{}
})}
]; ];
fields("rules") -> fields("rules") ->
[ rule_name() [
, {"sql", sc(binary(), rule_name(),
#{ desc => ?DESC("rules_sql") {"sql",
, example => "SELECT * FROM \"test/topic\" WHERE payload.x = 1" sc(
, required => true binary(),
, validator => fun ?MODULE:validate_sql/1 #{
})} desc => ?DESC("rules_sql"),
, {"outputs", sc(hoconsc:array(hoconsc:union(outputs())), example => "SELECT * FROM \"test/topic\" WHERE payload.x = 1",
#{ desc => ?DESC("rules_outputs") required => true,
, default => [] validator => fun ?MODULE:validate_sql/1
, example => [ }
<<"http:my_http_bridge">>, )},
#{function => republish, args => #{ {"outputs",
topic => <<"t/1">>, payload => <<"${payload}">>}}, sc(
#{function => console} hoconsc:array(hoconsc:union(outputs())),
] #{
})} desc => ?DESC("rules_outputs"),
, {"enable", sc(boolean(), #{desc => ?DESC("rules_enable"), default => true})} default => [],
, {"description", sc(binary(), example => [
#{ desc => ?DESC("rules_description") <<"http:my_http_bridge">>,
, example => "Some description" #{
, default => <<>> function => republish,
})} args => #{
topic => <<"t/1">>, payload => <<"${payload}">>
}
},
#{function => console}
]
}
)},
{"enable", sc(boolean(), #{desc => ?DESC("rules_enable"), default => true})},
{"description",
sc(
binary(),
#{
desc => ?DESC("rules_description"),
example => "Some description",
default => <<>>
}
)}
]; ];
fields("builtin_output_republish") -> fields("builtin_output_republish") ->
[ {function, sc(republish, #{desc => ?DESC("republish_function")})} [
, {args, sc(ref("republish_args"), #{default => #{}})} {function, sc(republish, #{desc => ?DESC("republish_function")})},
{args, sc(ref("republish_args"), #{default => #{}})}
]; ];
fields("builtin_output_console") -> fields("builtin_output_console") ->
[ {function, sc(console, #{desc => ?DESC("console_function")})} [
%% we may support some args for the console output in the future {function, sc(console, #{desc => ?DESC("console_function")})}
%, {args, sc(map(), #{desc => "The arguments of the built-in 'console' output", %% we may support some args for the console output in the future
% default => #{}})} %, {args, sc(map(), #{desc => "The arguments of the built-in 'console' output",
% default => #{}})}
]; ];
fields("user_provided_function") -> fields("user_provided_function") ->
[ {function, sc(binary(), [
#{ desc => ?DESC("user_provided_function_function") {function,
, required => true sc(
, example => "module:function" binary(),
})} #{
, {args, sc(map(), desc => ?DESC("user_provided_function_function"),
#{ desc => ?DESC("user_provided_function_args") required => true,
, default => #{} example => "module:function"
})} }
)},
{args,
sc(
map(),
#{
desc => ?DESC("user_provided_function_args"),
default => #{}
}
)}
]; ];
fields("republish_args") -> fields("republish_args") ->
[ {topic, sc(binary(), [
#{ desc => ?DESC("republish_args_topic") {topic,
, required => true sc(
, example => <<"a/1">> binary(),
})} #{
, {qos, sc(qos(), desc => ?DESC("republish_args_topic"),
#{ desc => ?DESC("republish_args_qos") required => true,
, default => <<"${qos}">> example => <<"a/1">>
, example => <<"${qos}">> }
})} )},
, {retain, sc(hoconsc:union([binary(), boolean()]), {qos,
#{ desc => ?DESC("republish_args_retain") sc(
, default => <<"${retain}">> qos(),
, example => <<"${retain}">> #{
})} desc => ?DESC("republish_args_qos"),
, {payload, sc(binary(), default => <<"${qos}">>,
#{ desc => ?DESC("republish_args_payload") example => <<"${qos}">>
, default => <<"${payload}">> }
, example => <<"${payload}">> )},
})} {retain,
sc(
hoconsc:union([binary(), boolean()]),
#{
desc => ?DESC("republish_args_retain"),
default => <<"${retain}">>,
example => <<"${retain}">>
}
)},
{payload,
sc(
binary(),
#{
desc => ?DESC("republish_args_payload"),
default => <<"${payload}">>,
example => <<"${payload}">>
}
)}
]. ].
desc("rule_engine") -> desc("rule_engine") ->
@ -129,18 +173,23 @@ desc(_) ->
undefined. undefined.
rule_name() -> rule_name() ->
{"name", sc(binary(), {"name",
#{ desc => ?DESC("rules_name") sc(
, default => "" binary(),
, required => true #{
, example => "foo" desc => ?DESC("rules_name"),
})}. default => "",
required => true,
example => "foo"
}
)}.
outputs() -> outputs() ->
[ binary() [
, ref("builtin_output_republish") binary(),
, ref("builtin_output_console") ref("builtin_output_republish"),
, ref("user_provided_function") ref("builtin_output_console"),
ref("user_provided_function")
]. ].
qos() -> qos() ->

14
apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl
View File

@ -28,11 +28,13 @@ start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []). supervisor:start_link({local, ?MODULE}, ?MODULE, []).
init([]) -> init([]) ->
Registry = #{id => emqx_rule_engine, Registry = #{
start => {emqx_rule_engine, start_link, []}, id => emqx_rule_engine,
restart => permanent, start => {emqx_rule_engine, start_link, []},
shutdown => 5000, restart => permanent,
type => worker, shutdown => 5000,
modules => [emqx_rule_engine]}, type => worker,
modules => [emqx_rule_engine]
},
Metrics = emqx_plugin_libs_metrics:child_spec(rule_metrics), Metrics = emqx_plugin_libs_metrics:child_spec(rule_metrics),
{ok, {{one_for_one, 10, 10}, [Registry, Metrics]}}. {ok, {{one_for_one, 10, 10}, [Registry, Metrics]}}.

1215
apps/emqx_rule_engine/src/emqx_rule_events.erl
View File

File diff suppressed because it is too large Load Diff

637
apps/emqx_rule_engine/src/emqx_rule_funcs.erl
View File

@ -21,257 +21,303 @@
-include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/logger.hrl").
%% IoT Funcs %% IoT Funcs
-export([ msgid/0 -export([
, qos/0 msgid/0,
, flags/0 qos/0,
, flag/1 flags/0,
, topic/0 flag/1,
, topic/1 topic/0,
, clientid/0 topic/1,
, clientip/0 clientid/0,
, peerhost/0 clientip/0,
, username/0 peerhost/0,
, payload/0 username/0,
, payload/1 payload/0,
, contains_topic/2 payload/1,
, contains_topic/3 contains_topic/2,
, contains_topic_match/2 contains_topic/3,
, contains_topic_match/3 contains_topic_match/2,
, null/0 contains_topic_match/3,
]). null/0
]).
%% Arithmetic Funcs %% Arithmetic Funcs
-export([ '+'/2 -export([
, '-'/2 '+'/2,
, '*'/2 '-'/2,
, '/'/2 '*'/2,
, 'div'/2 '/'/2,
, mod/2 'div'/2,
, eq/2 mod/2,
]). eq/2
]).
%% Math Funcs %% Math Funcs
-export([ abs/1 -export([
, acos/1 abs/1,
, acosh/1 acos/1,
, asin/1 acosh/1,
, asinh/1 asin/1,
, atan/1 asinh/1,
, atanh/1 atan/1,
, ceil/1 atanh/1,
, cos/1 ceil/1,
, cosh/1 cos/1,
, exp/1 cosh/1,
, floor/1 exp/1,
, fmod/2 floor/1,
, log/1 fmod/2,
, log10/1 log/1,
, log2/1 log10/1,
, power/2 log2/1,
, round/1 power/2,
, sin/1 round/1,
, sinh/1 sin/1,
, sqrt/1 sinh/1,
, tan/1 sqrt/1,
, tanh/1 tan/1,
]). tanh/1
]).
%% Bits Funcs %% Bits Funcs
-export([ bitnot/1 -export([
, bitand/2 bitnot/1,
, bitor/2 bitand/2,
, bitxor/2 bitor/2,
, bitsl/2 bitxor/2,
, bitsr/2 bitsl/2,
, bitsize/1 bitsr/2,
, subbits/2 bitsize/1,
, subbits/3 subbits/2,
, subbits/6 subbits/3,
]). subbits/6
]).
%% Data Type Conversion %% Data Type Conversion
-export([ str/1 -export([
, str_utf8/1 str/1,
, bool/1 str_utf8/1,
, int/1 bool/1,
, float/1 int/1,
, float/2 float/1,
, map/1 float/2,
, bin2hexstr/1 map/1,
, hexstr2bin/1 bin2hexstr/1,
]). hexstr2bin/1
]).
%% Data Type Validation Funcs %% Data Type Validation Funcs
-export([ is_null/1 -export([
, is_not_null/1 is_null/1,
, is_str/1 is_not_null/1,
, is_bool/1 is_str/1,
, is_int/1 is_bool/1,
, is_float/1 is_int/1,
, is_num/1 is_float/1,
, is_map/1 is_num/1,
, is_array/1 is_map/1,
]). is_array/1
]).
%% String Funcs %% String Funcs
-export([ lower/1 -export([
, ltrim/1 lower/1,
, reverse/1 ltrim/1,
, rtrim/1 reverse/1,
, strlen/1 rtrim/1,
, substr/2 strlen/1,
, substr/3 substr/2,
, trim/1 substr/3,
, upper/1 trim/1,
, split/2 upper/1,
, split/3 split/2,
, concat/2 split/3,
, tokens/2 concat/2,
, tokens/3 tokens/2,
, sprintf_s/2 tokens/3,
, pad/2 sprintf_s/2,
, pad/3 pad/2,
, pad/4 pad/3,
, replace/3 pad/4,
, replace/4 replace/3,
, regex_match/2 replace/4,
, regex_replace/3 regex_match/2,
, ascii/1 regex_replace/3,
, find/2 ascii/1,
, find/3 find/2,
]). find/3
]).
%% Map Funcs %% Map Funcs
-export([ map_new/0 -export([map_new/0]).
]).
-export([ map_get/2 -export([
, map_get/3 map_get/2,
, map_put/3 map_get/3,
]). map_put/3
]).
%% For backward compatibility %% For backward compatibility
-export([ mget/2 -export([
, mget/3 mget/2,
, mput/3 mget/3,
]). mput/3
]).
%% Array Funcs %% Array Funcs
-export([ nth/2 -export([
, length/1 nth/2,
, sublist/2 length/1,
, sublist/3 sublist/2,
, first/1 sublist/3,
, last/1 first/1,
, contains/2 last/1,
]). contains/2
]).
%% Hash Funcs %% Hash Funcs
-export([ md5/1 -export([
, sha/1 md5/1,
, sha256/1 sha/1,
]). sha256/1
]).
%% Data encode and decode %% Data encode and decode
-export([ base64_encode/1 -export([
, base64_decode/1 base64_encode/1,
, json_decode/1 base64_decode/1,
, json_encode/1 json_decode/1,
, term_decode/1 json_encode/1,
, term_encode/1 term_decode/1,
]). term_encode/1
]).
%% Date functions %% Date functions
-export([ now_rfc3339/0 -export([
, now_rfc3339/1 now_rfc3339/0,
, unix_ts_to_rfc3339/1 now_rfc3339/1,
, unix_ts_to_rfc3339/2 unix_ts_to_rfc3339/1,
, rfc3339_to_unix_ts/1 unix_ts_to_rfc3339/2,
, rfc3339_to_unix_ts/2 rfc3339_to_unix_ts/1,
, now_timestamp/0 rfc3339_to_unix_ts/2,
, now_timestamp/1 now_timestamp/0,
, format_date/3 now_timestamp/1,
, format_date/4 format_date/3,
, date_to_unix_ts/4 format_date/4,
]). date_to_unix_ts/4
]).
%% Proc Dict Func %% Proc Dict Func
-export([ proc_dict_get/1 -export([
, proc_dict_put/2 proc_dict_get/1,
, proc_dict_del/1 proc_dict_put/2,
, kv_store_get/1 proc_dict_del/1,
, kv_store_get/2 kv_store_get/1,
, kv_store_put/2 kv_store_get/2,
, kv_store_del/1 kv_store_put/2,
]). kv_store_del/1
]).
-export(['$handle_undefined_function'/2]). -export(['$handle_undefined_function'/2]).
-compile({no_auto_import, -compile(
[ abs/1 {no_auto_import, [
, ceil/1 abs/1,
, floor/1 ceil/1,
, round/1 floor/1,
, map_get/2 round/1,
]}). map_get/2
]}
).
-define(is_var(X), is_binary(X)). -define(is_var(X), is_binary(X)).
%% @doc "msgid()" Func %% @doc "msgid()" Func
msgid() -> msgid() ->
fun(#{id := MsgId}) -> MsgId; (_) -> undefined end. fun
(#{id := MsgId}) -> MsgId;
(_) -> undefined
end.
%% @doc "qos()" Func %% @doc "qos()" Func
qos() -> qos() ->
fun(#{qos := QoS}) -> QoS; (_) -> undefined end. fun
(#{qos := QoS}) -> QoS;
(_) -> undefined
end.
%% @doc "topic()" Func %% @doc "topic()" Func
topic() -> topic() ->
fun(#{topic := Topic}) -> Topic; (_) -> undefined end. fun
(#{topic := Topic}) -> Topic;
(_) -> undefined
end.
%% @doc "topic(N)" Func %% @doc "topic(N)" Func
topic(I) when is_integer(I) -> topic(I) when is_integer(I) ->
fun(#{topic := Topic}) -> fun
(#{topic := Topic}) ->
lists:nth(I, emqx_topic:tokens(Topic)); lists:nth(I, emqx_topic:tokens(Topic));
(_) -> undefined (_) ->
undefined
end. end.
%% @doc "flags()" Func %% @doc "flags()" Func
flags() -> flags() ->
fun(#{flags := Flags}) -> Flags; (_) -> #{} end. fun
(#{flags := Flags}) -> Flags;
(_) -> #{}
end.
%% @doc "flags(Name)" Func %% @doc "flags(Name)" Func
flag(Name) -> flag(Name) ->
fun(#{flags := Flags}) -> emqx_rule_maps:nested_get({var,Name}, Flags); (_) -> undefined end. fun
(#{flags := Flags}) -> emqx_rule_maps:nested_get({var, Name}, Flags);
(_) -> undefined
end.
%% @doc "clientid()" Func %% @doc "clientid()" Func
clientid() -> clientid() ->
fun(#{from := ClientId}) -> ClientId; (_) -> undefined end. fun
(#{from := ClientId}) -> ClientId;
(_) -> undefined
end.
%% @doc "username()" Func %% @doc "username()" Func
username() -> username() ->
fun(#{username := Username}) -> Username; (_) -> undefined end. fun
(#{username := Username}) -> Username;
(_) -> undefined
end.
%% @doc "clientip()" Func %% @doc "clientip()" Func
clientip() -> clientip() ->
peerhost(). peerhost().
peerhost() -> peerhost() ->
fun(#{peerhost := Addr}) -> Addr; (_) -> undefined end. fun
(#{peerhost := Addr}) -> Addr;
(_) -> undefined
end.
payload() -> payload() ->
fun(#{payload := Payload}) -> Payload; (_) -> undefined end. fun
(#{payload := Payload}) -> Payload;
(_) -> undefined
end.
payload(Path) -> payload(Path) ->
fun(#{payload := Payload}) when erlang:is_map(Payload) -> fun
(#{payload := Payload}) when erlang:is_map(Payload) ->
emqx_rule_maps:nested_get(map_path(Path), Payload); emqx_rule_maps:nested_get(map_path(Path), Payload);
(_) -> undefined (_) ->
undefined
end. end.
%% @doc Check if a topic_filter contains a specific topic %% @doc Check if a topic_filter contains a specific topic
%% TopicFilters = [{<<"t/a">>, #{qos => 0}]. %% TopicFilters = [{<<"t/a">>, #{qos => 0}].
-spec(contains_topic(emqx_types:topic_filters(), emqx_types:topic()) -spec contains_topic(emqx_types:topic_filters(), emqx_types:topic()) ->
-> true | false). true | false.
contains_topic(TopicFilters, Topic) -> contains_topic(TopicFilters, Topic) ->
case find_topic_filter(Topic, TopicFilters, fun eq/2) of case find_topic_filter(Topic, TopicFilters, fun eq/2) of
not_found -> false; not_found -> false;
@ -283,8 +329,8 @@ contains_topic(TopicFilters, Topic, QoS) ->
_ -> false _ -> false
end. end.
-spec(contains_topic_match(emqx_types:topic_filters(), emqx_types:topic()) -spec contains_topic_match(emqx_types:topic_filters(), emqx_types:topic()) ->
-> true | false). true | false.
contains_topic_match(TopicFilters, Topic) -> contains_topic_match(TopicFilters, Topic) ->
case find_topic_filter(Topic, TopicFilters, fun emqx_topic:match/2) of case find_topic_filter(Topic, TopicFilters, fun emqx_topic:match/2) of
not_found -> false; not_found -> false;
@ -298,10 +344,13 @@ contains_topic_match(TopicFilters, Topic, QoS) ->
find_topic_filter(Filter, TopicFilters, Func) -> find_topic_filter(Filter, TopicFilters, Func) ->
try try
[case Func(Topic, Filter) of [
true -> throw(Result); case Func(Topic, Filter) of
false -> ok true -> throw(Result);
end || Result = #{topic := Topic} <- TopicFilters], false -> ok
end
|| Result = #{topic := Topic} <- TopicFilters
],
not_found not_found
catch catch
throw:Result -> Result throw:Result -> Result
@ -317,7 +366,6 @@ null() ->
%% plus 2 numbers %% plus 2 numbers
'+'(X, Y) when is_number(X), is_number(Y) -> '+'(X, Y) when is_number(X), is_number(Y) ->
X + Y; X + Y;
%% string concatenation %% string concatenation
%% this requires one of the arguments is string, the other argument will be converted %% this requires one of the arguments is string, the other argument will be converted
%% to string automatically (implicit conversion) %% to string automatically (implicit conversion)
@ -355,7 +403,7 @@ acos(N) when is_number(N) ->
acosh(N) when is_number(N) -> acosh(N) when is_number(N) ->
math:acosh(N). math:acosh(N).
asin(N) when is_number(N)-> asin(N) when is_number(N) ->
math:asin(N). math:asin(N).
asinh(N) when is_number(N) -> asinh(N) when is_number(N) ->
@ -364,19 +412,19 @@ asinh(N) when is_number(N) ->
atan(N) when is_number(N) -> atan(N) when is_number(N) ->
math:atan(N). math:atan(N).
atanh(N) when is_number(N)-> atanh(N) when is_number(N) ->
math:atanh(N). math:atanh(N).
ceil(N) when is_number(N) -> ceil(N) when is_number(N) ->
erlang:ceil(N). erlang:ceil(N).
cos(N) when is_number(N)-> cos(N) when is_number(N) ->
math:cos(N). math:cos(N).
cosh(N) when is_number(N) -> cosh(N) when is_number(N) ->
math:cosh(N). math:cosh(N).
exp(N) when is_number(N)-> exp(N) when is_number(N) ->
math:exp(N). math:exp(N).
floor(N) when is_number(N) -> floor(N) when is_number(N) ->
@ -391,7 +439,7 @@ log(N) when is_number(N) ->
log10(N) when is_number(N) -> log10(N) when is_number(N) ->
math:log10(N). math:log10(N).
log2(N) when is_number(N)-> log2(N) when is_number(N) ->
math:log2(N). math:log2(N).
power(X, Y) when is_number(X), is_number(Y) -> power(X, Y) when is_number(X), is_number(Y) ->
@ -446,7 +494,9 @@ subbits(Bits, Len) when is_integer(Len), is_bitstring(Bits) ->
subbits(Bits, Start, Len) when is_integer(Start), is_integer(Len), is_bitstring(Bits) -> subbits(Bits, Start, Len) when is_integer(Start), is_integer(Len), is_bitstring(Bits) ->
get_subbits(Bits, Start, Len, <<"integer">>, <<"unsigned">>, <<"big">>). get_subbits(Bits, Start, Len, <<"integer">>, <<"unsigned">>, <<"big">>).
subbits(Bits, Start, Len, Type, Signedness, Endianness) when is_integer(Start), is_integer(Len), is_bitstring(Bits) -> subbits(Bits, Start, Len, Type, Signedness, Endianness) when
is_integer(Start), is_integer(Len), is_bitstring(Bits)
->
get_subbits(Bits, Start, Len, Type, Signedness, Endianness). get_subbits(Bits, Start, Len, Type, Signedness, Endianness).
get_subbits(Bits, Start, Len, Type, Signedness, Endianness) -> get_subbits(Bits, Start, Len, Type, Signedness, Endianness) ->
@ -455,7 +505,8 @@ get_subbits(Bits, Start, Len, Type, Signedness, Endianness) ->
<<_:Begin, Rem/bits>> when Rem =/= <<>> -> <<_:Begin, Rem/bits>> when Rem =/= <<>> ->
Sz = bit_size(Rem), Sz = bit_size(Rem),
do_get_subbits(Rem, Sz, Len, Type, Signedness, Endianness); do_get_subbits(Rem, Sz, Len, Type, Signedness, Endianness);
_ -> undefined _ ->
undefined
end. end.
-define(match_bits(Bits0, Pattern, ElesePattern), -define(match_bits(Bits0, Pattern, ElesePattern),
@ -464,46 +515,80 @@ get_subbits(Bits, Start, Len, Type, Signedness, Endianness) ->
SubBits; SubBits;
ElesePattern -> ElesePattern ->
SubBits SubBits
end). end
).
do_get_subbits(Bits, Sz, Len, <<"integer">>, <<"unsigned">>, <<"big">>) -> do_get_subbits(Bits, Sz, Len, <<"integer">>, <<"unsigned">>, <<"big">>) ->
?match_bits(Bits, <<SubBits:Len/integer-unsigned-big-unit:1, _/bits>>, ?match_bits(
<<SubBits:Sz/integer-unsigned-big-unit:1>>); Bits,
<<SubBits:Len/integer-unsigned-big-unit:1, _/bits>>,
<<SubBits:Sz/integer-unsigned-big-unit:1>>
);
do_get_subbits(Bits, Sz, Len, <<"float">>, <<"unsigned">>, <<"big">>) -> do_get_subbits(Bits, Sz, Len, <<"float">>, <<"unsigned">>, <<"big">>) ->
?match_bits(Bits, <<SubBits:Len/float-unsigned-big-unit:1, _/bits>>, ?match_bits(
<<SubBits:Sz/float-unsigned-big-unit:1>>); Bits,
<<SubBits:Len/float-unsigned-big-unit:1, _/bits>>,
<<SubBits:Sz/float-unsigned-big-unit:1>>
);
do_get_subbits(Bits, Sz, Len, <<"bits">>, <<"unsigned">>, <<"big">>) -> do_get_subbits(Bits, Sz, Len, <<"bits">>, <<"unsigned">>, <<"big">>) ->
?match_bits(Bits, <<SubBits:Len/bits-unsigned-big-unit:1, _/bits>>, ?match_bits(
<<SubBits:Sz/bits-unsigned-big-unit:1>>); Bits,
<<SubBits:Len/bits-unsigned-big-unit:1, _/bits>>,
<<SubBits:Sz/bits-unsigned-big-unit:1>>
);
do_get_subbits(Bits, Sz, Len, <<"integer">>, <<"signed">>, <<"big">>) -> do_get_subbits(Bits, Sz, Len, <<"integer">>, <<"signed">>, <<"big">>) ->
?match_bits(Bits, <<SubBits:Len/integer-signed-big-unit:1, _/bits>>, ?match_bits(
<<SubBits:Sz/integer-signed-big-unit:1>>); Bits,
<<SubBits:Len/integer-signed-big-unit:1, _/bits>>,
<<SubBits:Sz/integer-signed-big-unit:1>>
);
do_get_subbits(Bits, Sz, Len, <<"float">>, <<"signed">>, <<"big">>) -> do_get_subbits(Bits, Sz, Len, <<"float">>, <<"signed">>, <<"big">>) ->
?match_bits(Bits, <<SubBits:Len/float-signed-big-unit:1, _/bits>>, ?match_bits(
<<SubBits:Sz/float-signed-big-unit:1>>); Bits,
<<SubBits:Len/float-signed-big-unit:1, _/bits>>,
<<SubBits:Sz/float-signed-big-unit:1>>
);
do_get_subbits(Bits, Sz, Len, <<"bits">>, <<"signed">>, <<"big">>) -> do_get_subbits(Bits, Sz, Len, <<"bits">>, <<"signed">>, <<"big">>) ->
?match_bits(Bits, <<SubBits:Len/bits-signed-big-unit:1, _/bits>>, ?match_bits(
<<SubBits:Sz/bits-signed-big-unit:1>>); Bits,
<<SubBits:Len/bits-signed-big-unit:1, _/bits>>,
<<SubBits:Sz/bits-signed-big-unit:1>>
);
do_get_subbits(Bits, Sz, Len, <<"integer">>, <<"unsigned">>, <<"little">>) -> do_get_subbits(Bits, Sz, Len, <<"integer">>, <<"unsigned">>, <<"little">>) ->
?match_bits(Bits, <<SubBits:Len/integer-unsigned-little-unit:1, _/bits>>, ?match_bits(
<<SubBits:Sz/integer-unsigned-little-unit:1>>); Bits,
<<SubBits:Len/integer-unsigned-little-unit:1, _/bits>>,
<<SubBits:Sz/integer-unsigned-little-unit:1>>
);
do_get_subbits(Bits, Sz, Len, <<"float">>, <<"unsigned">>, <<"little">>) -> do_get_subbits(Bits, Sz, Len, <<"float">>, <<"unsigned">>, <<"little">>) ->
?match_bits(Bits, <<SubBits:Len/float-unsigned-little-unit:1, _/bits>>, ?match_bits(
<<SubBits:Sz/float-unsigned-little-unit:1>>); Bits,
<<SubBits:Len/float-unsigned-little-unit:1, _/bits>>,
<<SubBits:Sz/float-unsigned-little-unit:1>>
);
do_get_subbits(Bits, Sz, Len, <<"bits">>, <<"unsigned">>, <<"little">>) -> do_get_subbits(Bits, Sz, Len, <<"bits">>, <<"unsigned">>, <<"little">>) ->
?match_bits(Bits, <<SubBits:Len/bits-unsigned-little-unit:1, _/bits>>, ?match_bits(
<<SubBits:Sz/bits-unsigned-little-unit:1>>); Bits,
<<SubBits:Len/bits-unsigned-little-unit:1, _/bits>>,
<<SubBits:Sz/bits-unsigned-little-unit:1>>
);
do_get_subbits(Bits, Sz, Len, <<"integer">>, <<"signed">>, <<"little">>) -> do_get_subbits(Bits, Sz, Len, <<"integer">>, <<"signed">>, <<"little">>) ->
?match_bits(Bits, <<SubBits:Len/integer-signed-little-unit:1, _/bits>>, ?match_bits(
<<SubBits:Sz/integer-signed-little-unit:1>>); Bits,
<<SubBits:Len/integer-signed-little-unit:1, _/bits>>,
<<SubBits:Sz/integer-signed-little-unit:1>>
);
do_get_subbits(Bits, Sz, Len, <<"float">>, <<"signed">>, <<"little">>) -> do_get_subbits(Bits, Sz, Len, <<"float">>, <<"signed">>, <<"little">>) ->
?match_bits(Bits, <<SubBits:Len/float-signed-little-unit:1, _/bits>>, ?match_bits(
<<SubBits:Sz/float-signed-little-unit:1>>); Bits,
<<SubBits:Len/float-signed-little-unit:1, _/bits>>,
<<SubBits:Sz/float-signed-little-unit:1>>
);
do_get_subbits(Bits, Sz, Len, <<"bits">>, <<"signed">>, <<"little">>) -> do_get_subbits(Bits, Sz, Len, <<"bits">>, <<"signed">>, <<"little">>) ->
?match_bits(Bits, <<SubBits:Len/bits-signed-little-unit:1, _/bits>>, ?match_bits(
<<SubBits:Sz/bits-signed-little-unit:1>>). Bits,
<<SubBits:Len/bits-signed-little-unit:1, _/bits>>,
<<SubBits:Sz/bits-signed-little-unit:1>>
).
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% Data Type Conversion Funcs %% Data Type Conversion Funcs
@ -590,9 +675,11 @@ strlen(S) when is_binary(S) ->
substr(S, Start) when is_binary(S), is_integer(Start) -> substr(S, Start) when is_binary(S), is_integer(Start) ->
string:slice(S, Start). string:slice(S, Start).
substr(S, Start, Length) when is_binary(S), substr(S, Start, Length) when
is_integer(Start), is_binary(S),
is_integer(Length) -> is_integer(Start),
is_integer(Length)
->
string:slice(S, Start, Length). string:slice(S, Start, Length).
trim(S) when is_binary(S) -> trim(S) when is_binary(S) ->
@ -601,26 +688,28 @@ trim(S) when is_binary(S) ->
upper(S) when is_binary(S) -> upper(S) when is_binary(S) ->
string:uppercase(S). string:uppercase(S).
split(S, P) when is_binary(S),is_binary(P) -> split(S, P) when is_binary(S), is_binary(P) ->
[R || R <- string:split(S, P, all), R =/= <<>> andalso R =/= ""]. [R || R <- string:split(S, P, all), R =/= <<>> andalso R =/= ""].
split(S, P, <<"notrim">>) -> split(S, P, <<"notrim">>) ->
string:split(S, P, all); string:split(S, P, all);
split(S, P, <<"leading_notrim">>) -> split(S, P, <<"leading_notrim">>) ->
string:split(S, P, leading); string:split(S, P, leading);
split(S, P, <<"leading">>) when is_binary(S),is_binary(P) -> split(S, P, <<"leading">>) when is_binary(S), is_binary(P) ->
[R || R <- string:split(S, P, leading), R =/= <<>> andalso R =/= ""]; [R || R <- string:split(S, P, leading), R =/= <<>> andalso R =/= ""];
split(S, P, <<"trailing_notrim">>) -> split(S, P, <<"trailing_notrim">>) ->
string:split(S, P, trailing); string:split(S, P, trailing);
split(S, P, <<"trailing">>) when is_binary(S),is_binary(P) -> split(S, P, <<"trailing">>) when is_binary(S), is_binary(P) ->
[R || R <- string:split(S, P, trailing), R =/= <<>> andalso R =/= ""]. [R || R <- string:split(S, P, trailing), R =/= <<>> andalso R =/= ""].
tokens(S, Separators) -> tokens(S, Separators) ->
[list_to_binary(R) || R <- string:lexemes(binary_to_list(S), binary_to_list(Separators))]. [list_to_binary(R) || R <- string:lexemes(binary_to_list(S), binary_to_list(Separators))].
tokens(S, Separators, <<"nocrlf">>) -> tokens(S, Separators, <<"nocrlf">>) ->
[list_to_binary(R) || R <- string:lexemes(binary_to_list(S), binary_to_list(Separators) ++ [$\r,$\n,[$\r,$\n]])]. [
list_to_binary(R)
|| R <- string:lexemes(binary_to_list(S), binary_to_list(Separators) ++ [$\r, $\n, [$\r, $\n]])
].
%% implicit convert args to strings, and then do concatenation %% implicit convert args to strings, and then do concatenation
concat(S1, S2) -> concat(S1, S2) ->
@ -634,21 +723,17 @@ pad(S, Len) when is_binary(S), is_integer(Len) ->
pad(S, Len, <<"trailing">>) when is_binary(S), is_integer(Len) -> pad(S, Len, <<"trailing">>) when is_binary(S), is_integer(Len) ->
iolist_to_binary(string:pad(S, Len, trailing)); iolist_to_binary(string:pad(S, Len, trailing));
pad(S, Len, <<"both">>) when is_binary(S), is_integer(Len) -> pad(S, Len, <<"both">>) when is_binary(S), is_integer(Len) ->
iolist_to_binary(string:pad(S, Len, both)); iolist_to_binary(string:pad(S, Len, both));
pad(S, Len, <<"leading">>) when is_binary(S), is_integer(Len) -> pad(S, Len, <<"leading">>) when is_binary(S), is_integer(Len) ->
iolist_to_binary(string:pad(S, Len, leading)). iolist_to_binary(string:pad(S, Len, leading)).
pad(S, Len, <<"trailing">>, Char) when is_binary(S), is_integer(Len), is_binary(Char) -> pad(S, Len, <<"trailing">>, Char) when is_binary(S), is_integer(Len), is_binary(Char) ->
Chars = unicode:characters_to_list(Char, utf8), Chars = unicode:characters_to_list(Char, utf8),
iolist_to_binary(string:pad(S, Len, trailing, Chars)); iolist_to_binary(string:pad(S, Len, trailing, Chars));
pad(S, Len, <<"both">>, Char) when is_binary(S), is_integer(Len), is_binary(Char) -> pad(S, Len, <<"both">>, Char) when is_binary(S), is_integer(Len), is_binary(Char) ->
Chars = unicode:characters_to_list(Char, utf8), Chars = unicode:characters_to_list(Char, utf8),
iolist_to_binary(string:pad(S, Len, both, Chars)); iolist_to_binary(string:pad(S, Len, both, Chars));
pad(S, Len, <<"leading">>, Char) when is_binary(S), is_integer(Len), is_binary(Char) -> pad(S, Len, <<"leading">>, Char) when is_binary(S), is_integer(Len), is_binary(Char) ->
Chars = unicode:characters_to_list(Char, utf8), Chars = unicode:characters_to_list(Char, utf8),
iolist_to_binary(string:pad(S, Len, leading, Chars)). iolist_to_binary(string:pad(S, Len, leading, Chars)).
@ -658,24 +743,24 @@ replace(SrcStr, P, RepStr) when is_binary(SrcStr), is_binary(P), is_binary(RepSt
replace(SrcStr, P, RepStr, <<"all">>) when is_binary(SrcStr), is_binary(P), is_binary(RepStr) -> replace(SrcStr, P, RepStr, <<"all">>) when is_binary(SrcStr), is_binary(P), is_binary(RepStr) ->
iolist_to_binary(string:replace(SrcStr, P, RepStr, all)); iolist_to_binary(string:replace(SrcStr, P, RepStr, all));
replace(SrcStr, P, RepStr, <<"trailing">>) when
replace(SrcStr, P, RepStr, <<"trailing">>) when is_binary(SrcStr), is_binary(P), is_binary(RepStr) -> is_binary(SrcStr), is_binary(P), is_binary(RepStr)
->
iolist_to_binary(string:replace(SrcStr, P, RepStr, trailing)); iolist_to_binary(string:replace(SrcStr, P, RepStr, trailing));
replace(SrcStr, P, RepStr, <<"leading">>) when is_binary(SrcStr), is_binary(P), is_binary(RepStr) -> replace(SrcStr, P, RepStr, <<"leading">>) when is_binary(SrcStr), is_binary(P), is_binary(RepStr) ->
iolist_to_binary(string:replace(SrcStr, P, RepStr, leading)). iolist_to_binary(string:replace(SrcStr, P, RepStr, leading)).
regex_match(Str, RE) -> regex_match(Str, RE) ->
case re:run(Str, RE, [global,{capture,none}]) of case re:run(Str, RE, [global, {capture, none}]) of
match -> true; match -> true;
nomatch -> false nomatch -> false
end. end.
regex_replace(SrcStr, RE, RepStr) -> regex_replace(SrcStr, RE, RepStr) ->
re:replace(SrcStr, RE, RepStr, [global, {return,binary}]). re:replace(SrcStr, RE, RepStr, [global, {return, binary}]).
ascii(Char) when is_binary(Char) -> ascii(Char) when is_binary(Char) ->
[FirstC| _] = binary_to_list(Char), [FirstC | _] = binary_to_list(Char),
FirstC. FirstC.
find(S, P) when is_binary(S), is_binary(P) -> find(S, P) when is_binary(S), is_binary(P) ->
@ -683,7 +768,6 @@ find(S, P) when is_binary(S), is_binary(P) ->
find(S, P, <<"trailing">>) when is_binary(S), is_binary(P) -> find(S, P, <<"trailing">>) when is_binary(S), is_binary(P) ->
find_s(S, P, trailing); find_s(S, P, trailing);
find(S, P, <<"leading">>) when is_binary(S), is_binary(P) -> find(S, P, <<"leading">>) when is_binary(S), is_binary(P) ->
find_s(S, P, leading). find_s(S, P, leading).
@ -735,7 +819,8 @@ mget(Key, Map) ->
mget(Key, Map, Default) -> mget(Key, Map, Default) ->
case maps:find(Key, Map) of case maps:find(Key, Map) of
{ok, Val} -> Val; {ok, Val} ->
Val;
error when is_atom(Key) -> error when is_atom(Key) ->
%% the map may have an equivalent binary-form key %% the map may have an equivalent binary-form key
BinKey = emqx_plugin_libs_rule:bin(Key), BinKey = emqx_plugin_libs_rule:bin(Key),
@ -744,14 +829,16 @@ mget(Key, Map, Default) ->
error -> Default error -> Default
end; end;
error when is_binary(Key) -> error when is_binary(Key) ->
try %% the map may have an equivalent atom-form key %% the map may have an equivalent atom-form key
try
AtomKey = list_to_existing_atom(binary_to_list(Key)), AtomKey = list_to_existing_atom(binary_to_list(Key)),
case maps:find(AtomKey, Map) of case maps:find(AtomKey, Map) of
{ok, Val} -> Val; {ok, Val} -> Val;
error -> Default error -> Default
end end
catch error:badarg -> catch
Default error:badarg ->
Default
end; end;
error -> error ->
Default Default
@ -759,7 +846,8 @@ mget(Key, Map, Default) ->
mput(Key, Val, Map) -> mput(Key, Val, Map) ->
case maps:find(Key, Map) of case maps:find(Key, Map) of
{ok, _} -> maps:put(Key, Val, Map); {ok, _} ->
maps:put(Key, Val, Map);
error when is_atom(Key) -> error when is_atom(Key) ->
%% the map may have an equivalent binary-form key %% the map may have an equivalent binary-form key
BinKey = emqx_plugin_libs_rule:bin(Key), BinKey = emqx_plugin_libs_rule:bin(Key),
@ -768,14 +856,16 @@ mput(Key, Val, Map) ->
error -> maps:put(Key, Val, Map) error -> maps:put(Key, Val, Map)
end; end;
error when is_binary(Key) -> error when is_binary(Key) ->
try %% the map may have an equivalent atom-form key %% the map may have an equivalent atom-form key
try
AtomKey = list_to_existing_atom(binary_to_list(Key)), AtomKey = list_to_existing_atom(binary_to_list(Key)),
case maps:find(AtomKey, Map) of case maps:find(AtomKey, Map) of
{ok, _} -> maps:put(AtomKey, Val, Map); {ok, _} -> maps:put(AtomKey, Val, Map);
error -> maps:put(Key, Val, Map) error -> maps:put(Key, Val, Map)
end end
catch error:badarg -> catch
maps:put(Key, Val, Map) error:badarg ->
maps:put(Key, Val, Map)
end; end;
error -> error ->
maps:put(Key, Val, Map) maps:put(Key, Val, Map)
@ -863,14 +953,18 @@ unix_ts_to_rfc3339(Epoch) ->
unix_ts_to_rfc3339(Epoch, Unit) when is_integer(Epoch) -> unix_ts_to_rfc3339(Epoch, Unit) when is_integer(Epoch) ->
emqx_plugin_libs_rule:bin( emqx_plugin_libs_rule:bin(
calendar:system_time_to_rfc3339( calendar:system_time_to_rfc3339(
Epoch, [{unit, time_unit(Unit)}])). Epoch, [{unit, time_unit(Unit)}]
)
).
rfc3339_to_unix_ts(DateTime) -> rfc3339_to_unix_ts(DateTime) ->
rfc3339_to_unix_ts(DateTime, <<"second">>). rfc3339_to_unix_ts(DateTime, <<"second">>).
rfc3339_to_unix_ts(DateTime, Unit) when is_binary(DateTime) -> rfc3339_to_unix_ts(DateTime, Unit) when is_binary(DateTime) ->
calendar:rfc3339_to_system_time(binary_to_list(DateTime), calendar:rfc3339_to_system_time(
[{unit, time_unit(Unit)}]). binary_to_list(DateTime),
[{unit, time_unit(Unit)}]
).
now_timestamp() -> now_timestamp() ->
erlang:system_time(second). erlang:system_time(second).
@ -885,22 +979,30 @@ time_unit(<<"nanosecond">>) -> nanosecond.
format_date(TimeUnit, Offset, FormatString) -> format_date(TimeUnit, Offset, FormatString) ->
emqx_plugin_libs_rule:bin( emqx_plugin_libs_rule:bin(
emqx_rule_date:date(time_unit(TimeUnit), emqx_rule_date:date(
emqx_plugin_libs_rule:str(Offset), time_unit(TimeUnit),
emqx_plugin_libs_rule:str(FormatString))). emqx_plugin_libs_rule:str(Offset),
emqx_plugin_libs_rule:str(FormatString)
)
).
format_date(TimeUnit, Offset, FormatString, TimeEpoch) -> format_date(TimeUnit, Offset, FormatString, TimeEpoch) ->
emqx_plugin_libs_rule:bin( emqx_plugin_libs_rule:bin(
emqx_rule_date:date(time_unit(TimeUnit), emqx_rule_date:date(
emqx_plugin_libs_rule:str(Offset), time_unit(TimeUnit),
emqx_plugin_libs_rule:str(FormatString), emqx_plugin_libs_rule:str(Offset),
TimeEpoch)). emqx_plugin_libs_rule:str(FormatString),
TimeEpoch
)
).
date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) -> date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) ->
emqx_rule_date:parse_date(time_unit(TimeUnit), emqx_rule_date:parse_date(
emqx_plugin_libs_rule:str(Offset), time_unit(TimeUnit),
emqx_plugin_libs_rule:str(FormatString), emqx_plugin_libs_rule:str(Offset),
emqx_plugin_libs_rule:str(InputString)). emqx_plugin_libs_rule:str(FormatString),
emqx_plugin_libs_rule:str(InputString)
).
%% @doc This is for sql funcs that should be handled in the specific modules. %% @doc This is for sql funcs that should be handled in the specific modules.
%% Here the emqx_rule_funcs module acts as a proxy, forwarding %% Here the emqx_rule_funcs module acts as a proxy, forwarding
@ -922,9 +1024,8 @@ date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) ->
% '$handle_undefined_function'(Fun, Args) -> % '$handle_undefined_function'(Fun, Args) ->
% error({sql_function_not_supported, function_literal(Fun, Args)}). % error({sql_function_not_supported, function_literal(Fun, Args)}).
'$handle_undefined_function'(sprintf, [Format|Args]) -> '$handle_undefined_function'(sprintf, [Format | Args]) ->
erlang:apply(fun sprintf_s/2, [Format, Args]); erlang:apply(fun sprintf_s/2, [Format, Args]);
'$handle_undefined_function'(Fun, Args) -> '$handle_undefined_function'(Fun, Args) ->
error({sql_function_not_supported, function_literal(Fun, Args)}). error({sql_function_not_supported, function_literal(Fun, Args)}).
@ -935,8 +1036,12 @@ function_literal(Fun, []) when is_atom(Fun) ->
atom_to_list(Fun) ++ "()"; atom_to_list(Fun) ++ "()";
function_literal(Fun, [FArg | Args]) when is_atom(Fun), is_list(Args) -> function_literal(Fun, [FArg | Args]) when is_atom(Fun), is_list(Args) ->
WithFirstArg = io_lib:format("~ts(~0p", [atom_to_list(Fun), FArg]), WithFirstArg = io_lib:format("~ts(~0p", [atom_to_list(Fun), FArg]),
lists:foldl(fun(Arg, Literal) -> lists:foldl(
io_lib:format("~ts, ~0p", [Literal, Arg]) fun(Arg, Literal) ->
end, WithFirstArg, Args) ++ ")"; io_lib:format("~ts, ~0p", [Literal, Arg])
end,
WithFirstArg,
Args
) ++ ")";
function_literal(Fun, Args) -> function_literal(Fun, Args) ->
{invalid_func, {Fun, Args}}. {invalid_func, {Fun, Args}}.

112
apps/emqx_rule_engine/src/emqx_rule_maps.erl
View File

@ -16,14 +16,15 @@
-module(emqx_rule_maps). -module(emqx_rule_maps).
-export([ nested_get/2 -export([
, nested_get/3 nested_get/2,
, nested_put/3 nested_get/3,
, range_gen/2 nested_put/3,
, range_get/3 range_gen/2,
, atom_key_map/1 range_get/3,
, unsafe_atom_key_map/1 atom_key_map/1,
]). unsafe_atom_key_map/1
]).
nested_get(Key, Data) -> nested_get(Key, Data) ->
nested_get(Key, Data, undefined). nested_get(Key, Data, undefined).
@ -41,8 +42,10 @@ do_nested_get([Key | More], Data, OrgData, Default) ->
do_nested_get([], Val, _OrgData, _Default) -> do_nested_get([], Val, _OrgData, _Default) ->
Val. Val.
nested_put(Key, Val, Data) when not is_map(Data), nested_put(Key, Val, Data) when
not is_list(Data) -> not is_map(Data),
not is_list(Data)
->
nested_put(Key, Val, #{}); nested_put(Key, Val, #{});
nested_put({var, Key}, Val, Map) -> nested_put({var, Key}, Val, Map) ->
general_map_put({key, Key}, Val, Map, Map); general_map_put({key, Key}, Val, Map, Map);
@ -56,19 +59,27 @@ do_nested_put([], Val, _Map, _OrgData) ->
Val. Val.
general_map_get(Key, Map, OrgData, Default) -> general_map_get(Key, Map, OrgData, Default) ->
general_find(Key, Map, OrgData, general_find(
Key,
Map,
OrgData,
fun fun
({equivalent, {_EquiKey, Val}}) -> Val; ({equivalent, {_EquiKey, Val}}) -> Val;
({found, {_Key, Val}}) -> Val; ({found, {_Key, Val}}) -> Val;
(not_found) -> Default (not_found) -> Default
end). end
).
general_map_put(Key, Val, Map, OrgData) -> general_map_put(Key, Val, Map, OrgData) ->
general_find(Key, Map, OrgData, general_find(
Key,
Map,
OrgData,
fun fun
({equivalent, {EquiKey, _Val}}) -> do_put(EquiKey, Val, Map, OrgData); ({equivalent, {EquiKey, _Val}}) -> do_put(EquiKey, Val, Map, OrgData);
(_) -> do_put(Key, Val, Map, OrgData) (_) -> do_put(Key, Val, Map, OrgData)
end). end
).
general_find(KeyOrIndex, Data, OrgData, Handler) when is_binary(Data) -> general_find(KeyOrIndex, Data, OrgData, Handler) when is_binary(Data) ->
try emqx_json:decode(Data, [return_maps]) of try emqx_json:decode(Data, [return_maps]) of
@ -78,7 +89,8 @@ general_find(KeyOrIndex, Data, OrgData, Handler) when is_binary(Data) ->
end; end;
general_find({key, Key}, Map, _OrgData, Handler) when is_map(Map) -> general_find({key, Key}, Map, _OrgData, Handler) when is_map(Map) ->
case maps:find(Key, Map) of case maps:find(Key, Map) of
{ok, Val} -> Handler({found, {{key, Key}, Val}}); {ok, Val} ->
Handler({found, {{key, Key}, Val}});
error when is_atom(Key) -> error when is_atom(Key) ->
%% the map may have an equivalent binary-form key %% the map may have an equivalent binary-form key
BinKey = emqx_plugin_libs_rule:bin(Key), BinKey = emqx_plugin_libs_rule:bin(Key),
@ -87,14 +99,16 @@ general_find({key, Key}, Map, _OrgData, Handler) when is_map(Map) ->
error -> Handler(not_found) error -> Handler(not_found)
end; end;
error when is_binary(Key) -> error when is_binary(Key) ->
try %% the map may have an equivalent atom-form key %% the map may have an equivalent atom-form key
try
AtomKey = list_to_existing_atom(binary_to_list(Key)), AtomKey = list_to_existing_atom(binary_to_list(Key)),
case maps:find(AtomKey, Map) of case maps:find(AtomKey, Map) of
{ok, Val} -> Handler({equivalent, {{key, AtomKey}, Val}}); {ok, Val} -> Handler({equivalent, {{key, AtomKey}, Val}});
error -> Handler(not_found) error -> Handler(not_found)
end end
catch error:badarg -> catch
Handler(not_found) error:badarg ->
Handler(not_found)
end; end;
error -> error ->
Handler(not_found) Handler(not_found)
@ -122,18 +136,21 @@ do_put({index, Index0}, Val, List, OrgData) ->
setnth(_, Data, Val) when not is_list(Data) -> setnth(_, Data, Val) when not is_list(Data) ->
setnth(head, [], Val); setnth(head, [], Val);
setnth(head, List, Val) when is_list(List) -> [Val | List]; setnth(head, List, Val) when is_list(List) -> [Val | List];
setnth(head, _List, Val) -> [Val]; setnth(head, _List, Val) ->
[Val];
setnth(tail, List, Val) when is_list(List) -> List ++ [Val]; setnth(tail, List, Val) when is_list(List) -> List ++ [Val];
setnth(tail, _List, Val) -> [Val]; setnth(tail, _List, Val) ->
[Val];
setnth(I, List, _Val) when not is_integer(I) -> List; setnth(I, List, _Val) when not is_integer(I) -> List;
setnth(0, List, _Val) -> List; setnth(0, List, _Val) ->
List;
setnth(I, List, Val) when is_integer(I), I > 0 -> setnth(I, List, Val) when is_integer(I), I > 0 ->
do_setnth(I, List, Val); do_setnth(I, List, Val);
setnth(I, List, Val) when is_integer(I), I < 0 -> setnth(I, List, Val) when is_integer(I), I < 0 ->
lists:reverse(do_setnth(-I, lists:reverse(List), Val)). lists:reverse(do_setnth(-I, lists:reverse(List), Val)).
do_setnth(1, [_ | Rest], Val) -> [Val | Rest]; do_setnth(1, [_ | Rest], Val) -> [Val | Rest];
do_setnth(I, [E | Rest], Val) -> [E | setnth(I-1, Rest, Val)]; do_setnth(I, [E | Rest], Val) -> [E | setnth(I - 1, Rest, Val)];
do_setnth(_, [], _Val) -> []. do_setnth(_, [], _Val) -> [].
getnth(0, _) -> getnth(0, _) ->
@ -144,8 +161,10 @@ getnth(I, L) when I < 0 ->
do_getnth(-I, lists:reverse(L)). do_getnth(-I, lists:reverse(L)).
do_getnth(I, L) -> do_getnth(I, L) ->
try {ok, lists:nth(I, L)} try
catch error:_ -> {error, not_found} {ok, lists:nth(I, L)}
catch
error:_ -> {error, not_found}
end. end.
handle_getnth(Index, List, IndexPattern, Handler) -> handle_getnth(Index, List, IndexPattern, Handler) ->
@ -170,7 +189,8 @@ do_range_get(Begin, End, List) ->
EndIndex = index(End, TotalLen), EndIndex = index(End, TotalLen),
lists:sublist(List, BeginIndex, (EndIndex - BeginIndex + 1)). lists:sublist(List, BeginIndex, (EndIndex - BeginIndex + 1)).
index(0, _) -> error({invalid_index, 0}); index(0, _) ->
error({invalid_index, 0});
index(Index, _) when Index > 0 -> Index; index(Index, _) when Index > 0 -> Index;
index(Index, Len) when Index < 0 -> index(Index, Len) when Index < 0 ->
Len + Index + 1. Len + Index + 1.
@ -180,26 +200,36 @@ index(Index, Len) when Index < 0 ->
%%%------------------------------------------------------------------- %%%-------------------------------------------------------------------
atom_key_map(BinKeyMap) when is_map(BinKeyMap) -> atom_key_map(BinKeyMap) when is_map(BinKeyMap) ->
maps:fold( maps:fold(
fun(K, V, Acc) when is_binary(K) -> fun
Acc#{binary_to_existing_atom(K, utf8) => atom_key_map(V)}; (K, V, Acc) when is_binary(K) ->
(K, V, Acc) when is_list(K) -> Acc#{binary_to_existing_atom(K, utf8) => atom_key_map(V)};
Acc#{list_to_existing_atom(K) => atom_key_map(V)}; (K, V, Acc) when is_list(K) ->
(K, V, Acc) when is_atom(K) -> Acc#{list_to_existing_atom(K) => atom_key_map(V)};
Acc#{K => atom_key_map(V)} (K, V, Acc) when is_atom(K) ->
end, #{}, BinKeyMap); Acc#{K => atom_key_map(V)}
end,
#{},
BinKeyMap
);
atom_key_map(ListV) when is_list(ListV) -> atom_key_map(ListV) when is_list(ListV) ->
[atom_key_map(V) || V <- ListV]; [atom_key_map(V) || V <- ListV];
atom_key_map(Val) -> Val. atom_key_map(Val) ->
Val.
unsafe_atom_key_map(BinKeyMap) when is_map(BinKeyMap) -> unsafe_atom_key_map(BinKeyMap) when is_map(BinKeyMap) ->
maps:fold( maps:fold(
fun(K, V, Acc) when is_binary(K) -> fun
Acc#{binary_to_atom(K, utf8) => unsafe_atom_key_map(V)}; (K, V, Acc) when is_binary(K) ->
(K, V, Acc) when is_list(K) -> Acc#{binary_to_atom(K, utf8) => unsafe_atom_key_map(V)};
Acc#{list_to_atom(K) => unsafe_atom_key_map(V)}; (K, V, Acc) when is_list(K) ->
(K, V, Acc) when is_atom(K) -> Acc#{list_to_atom(K) => unsafe_atom_key_map(V)};
Acc#{K => unsafe_atom_key_map(V)} (K, V, Acc) when is_atom(K) ->
end, #{}, BinKeyMap); Acc#{K => unsafe_atom_key_map(V)}
end,
#{},
BinKeyMap
);
unsafe_atom_key_map(ListV) when is_list(ListV) -> unsafe_atom_key_map(ListV) when is_list(ListV) ->
[unsafe_atom_key_map(V) || V <- ListV]; [unsafe_atom_key_map(V) || V <- ListV];
unsafe_atom_key_map(Val) -> Val. unsafe_atom_key_map(Val) ->
Val.

101
apps/emqx_rule_engine/src/emqx_rule_outputs.erl
View File

@ -22,20 +22,18 @@
-include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx.hrl").
%% APIs %% APIs
-export([ parse_output/1 -export([parse_output/1]).
]).
%% callbacks of emqx_rule_output %% callbacks of emqx_rule_output
-export([ pre_process_output_args/2 -export([pre_process_output_args/2]).
]).
%% output functions %% output functions
-export([ console/3 -export([
, republish/3 console/3,
]). republish/3
]).
-optional_callbacks([ pre_process_output_args/2 -optional_callbacks([pre_process_output_args/2]).
]).
-callback pre_process_output_args(FuncName :: atom(), output_fun_args()) -> output_fun_args(). -callback pre_process_output_args(FuncName :: atom(), output_fun_args()) -> output_fun_args().
@ -44,20 +42,32 @@
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
parse_output(#{function := OutputFunc} = Output) -> parse_output(#{function := OutputFunc} = Output) ->
{Mod, Func} = parse_output_func(OutputFunc), {Mod, Func} = parse_output_func(OutputFunc),
#{mod => Mod, func => Func, #{
args => pre_process_args(Mod, Func, maps:get(args, Output, #{}))}. mod => Mod,
func => Func,
args => pre_process_args(Mod, Func, maps:get(args, Output, #{}))
}.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% callbacks of emqx_rule_output %% callbacks of emqx_rule_output
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
pre_process_output_args(republish, #{topic := Topic, qos := QoS, retain := Retain, pre_process_output_args(
payload := Payload} = Args) -> republish,
Args#{preprocessed_tmpl => #{ #{
topic := Topic,
qos := QoS,
retain := Retain,
payload := Payload
} = Args
) ->
Args#{
preprocessed_tmpl => #{
topic => emqx_plugin_libs_rule:preproc_tmpl(Topic), topic => emqx_plugin_libs_rule:preproc_tmpl(Topic),
qos => preproc_vars(QoS), qos => preproc_vars(QoS),
retain => preproc_vars(Retain), retain => preproc_vars(Retain),
payload => emqx_plugin_libs_rule:preproc_tmpl(Payload) payload => emqx_plugin_libs_rule:preproc_tmpl(Payload)
}}; }
};
pre_process_output_args(_, Args) -> pre_process_output_args(_, Args) ->
Args. Args.
@ -66,35 +76,55 @@ pre_process_output_args(_, Args) ->
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-spec console(map(), map(), map()) -> any(). -spec console(map(), map(), map()) -> any().
console(Selected, #{metadata := #{rule_id := RuleId}} = Envs, _Args) -> console(Selected, #{metadata := #{rule_id := RuleId}} = Envs, _Args) ->
?ULOG("[rule output] ~ts~n" ?ULOG(
"\tOutput Data: ~p~n" "[rule output] ~ts~n"
"\tEnvs: ~p~n", [RuleId, Selected, Envs]). "\tOutput Data: ~p~n"
"\tEnvs: ~p~n",
[RuleId, Selected, Envs]
).
republish(_Selected, #{topic := Topic, headers := #{republish_by := RuleId}, republish(
metadata := #{rule_id := RuleId}}, _Args) -> _Selected,
#{
topic := Topic,
headers := #{republish_by := RuleId},
metadata := #{rule_id := RuleId}
},
_Args
) ->
?SLOG(error, #{msg => "recursive_republish_detected", topic => Topic}); ?SLOG(error, #{msg => "recursive_republish_detected", topic => Topic});
%% republish a PUBLISH message %% republish a PUBLISH message
republish(Selected, #{flags := Flags, metadata := #{rule_id := RuleId}}, republish(
#{preprocessed_tmpl := #{ Selected,
#{flags := Flags, metadata := #{rule_id := RuleId}},
#{
preprocessed_tmpl := #{
qos := QoSTks, qos := QoSTks,
retain := RetainTks, retain := RetainTks,
topic := TopicTks, topic := TopicTks,
payload := PayloadTks}}) -> payload := PayloadTks
}
}
) ->
Topic = emqx_plugin_libs_rule:proc_tmpl(TopicTks, Selected), Topic = emqx_plugin_libs_rule:proc_tmpl(TopicTks, Selected),
Payload = format_msg(PayloadTks, Selected), Payload = format_msg(PayloadTks, Selected),
QoS = replace_simple_var(QoSTks, Selected, 0), QoS = replace_simple_var(QoSTks, Selected, 0),
Retain = replace_simple_var(RetainTks, Selected, false), Retain = replace_simple_var(RetainTks, Selected, false),
?TRACE("RULE", "republish_message", #{topic => Topic, payload => Payload}), ?TRACE("RULE", "republish_message", #{topic => Topic, payload => Payload}),
safe_publish(RuleId, Topic, QoS, Flags#{retain => Retain}, Payload); safe_publish(RuleId, Topic, QoS, Flags#{retain => Retain}, Payload);
%% in case this is a "$events/" event %% in case this is a "$events/" event
republish(Selected, #{metadata := #{rule_id := RuleId}}, republish(
#{preprocessed_tmpl := #{ Selected,
qos := QoSTks, #{metadata := #{rule_id := RuleId}},
retain := RetainTks, #{
topic := TopicTks, preprocessed_tmpl := #{
payload := PayloadTks}}) -> qos := QoSTks,
retain := RetainTks,
topic := TopicTks,
payload := PayloadTks
}
}
) ->
Topic = emqx_plugin_libs_rule:proc_tmpl(TopicTks, Selected), Topic = emqx_plugin_libs_rule:proc_tmpl(TopicTks, Selected),
Payload = format_msg(PayloadTks, Selected), Payload = format_msg(PayloadTks, Selected),
QoS = replace_simple_var(QoSTks, Selected, 0), QoS = replace_simple_var(QoSTks, Selected, 0),
@ -114,8 +144,10 @@ get_output_mod_func(OutputFunc) when is_atom(OutputFunc) ->
{emqx_rule_outputs, OutputFunc}; {emqx_rule_outputs, OutputFunc};
get_output_mod_func(OutputFunc) when is_binary(OutputFunc) -> get_output_mod_func(OutputFunc) when is_binary(OutputFunc) ->
ToAtom = fun(Bin) -> ToAtom = fun(Bin) ->
try binary_to_existing_atom(Bin) of Atom -> Atom try binary_to_existing_atom(Bin) of
catch error:badarg -> error({unknown_output_function, OutputFunc}) Atom -> Atom
catch
error:badarg -> error({unknown_output_function, OutputFunc})
end end
end, end,
case string:split(OutputFunc, ":", all) of case string:split(OutputFunc, ":", all) of
@ -158,7 +190,8 @@ preproc_vars(Data) ->
replace_simple_var(Tokens, Data, Default) when is_list(Tokens) -> replace_simple_var(Tokens, Data, Default) when is_list(Tokens) ->
[Var] = emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => rawlist}), [Var] = emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => rawlist}),
case Var of case Var of
undefined -> Default; %% cannot find the variable from Data %% cannot find the variable from Data
undefined -> Default;
_ -> Var _ -> Var
end; end;
replace_simple_var(Val, _Data, _Default) -> replace_simple_var(Val, _Data, _Default) ->

225
apps/emqx_rule_engine/src/emqx_rule_runtime.erl
View File

@ -20,32 +20,37 @@
-include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/logger.hrl").
-export([ apply_rule/2 -export([
, apply_rules/2 apply_rule/2,
, clear_rule_payload/0 apply_rules/2,
]). clear_rule_payload/0
]).
-import(emqx_rule_maps, -import(
[ nested_get/2 emqx_rule_maps,
, range_gen/2 [
, range_get/3 nested_get/2,
]). range_gen/2,
range_get/3
]
).
-compile({no_auto_import,[alias/1]}). -compile({no_auto_import, [alias/1]}).
-type input() :: map(). -type input() :: map().
-type alias() :: atom(). -type alias() :: atom().
-type collection() :: {alias(), [term()]}. -type collection() :: {alias(), [term()]}.
-define(ephemeral_alias(TYPE, NAME), -define(ephemeral_alias(TYPE, NAME),
iolist_to_binary(io_lib:format("_v_~ts_~p_~p", [TYPE, NAME, erlang:system_time()]))). iolist_to_binary(io_lib:format("_v_~ts_~p_~p", [TYPE, NAME, erlang:system_time()]))
).
-define(ActionMaxRetry, 3). -define(ActionMaxRetry, 3).
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% Apply rules %% Apply rules
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
-spec(apply_rules(list(rule()), input()) -> ok). -spec apply_rules(list(rule()), input()) -> ok.
apply_rules([], _Input) -> apply_rules([], _Input) ->
ok; ok;
apply_rules([#{enable := false} | More], Input) -> apply_rules([#{enable := false} | More], Input) ->
@ -61,54 +66,77 @@ apply_rule_discard_result(Rule, Input) ->
apply_rule(Rule = #{id := RuleID}, Input) -> apply_rule(Rule = #{id := RuleID}, Input) ->
ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleID, 'sql.matched'), ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleID, 'sql.matched'),
clear_rule_payload(), clear_rule_payload(),
try do_apply_rule(Rule, add_metadata(Input, #{rule_id => RuleID})) try
do_apply_rule(Rule, add_metadata(Input, #{rule_id => RuleID}))
catch catch
%% ignore the errors if select or match failed %% ignore the errors if select or match failed
_:Reason = {select_and_transform_error, Error} -> _:Reason = {select_and_transform_error, Error} ->
ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleID, 'sql.failed.exception'), ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleID, 'sql.failed.exception'),
?SLOG(warning, #{msg => "SELECT_clause_exception", ?SLOG(warning, #{
rule_id => RuleID, reason => Error}), msg => "SELECT_clause_exception",
rule_id => RuleID,
reason => Error
}),
{error, Reason}; {error, Reason};
_:Reason = {match_conditions_error, Error} -> _:Reason = {match_conditions_error, Error} ->
ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleID, 'sql.failed.exception'), ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleID, 'sql.failed.exception'),
?SLOG(warning, #{msg => "WHERE_clause_exception", ?SLOG(warning, #{
rule_id => RuleID, reason => Error}), msg => "WHERE_clause_exception",
rule_id => RuleID,
reason => Error
}),
{error, Reason}; {error, Reason};
_:Reason = {select_and_collect_error, Error} -> _:Reason = {select_and_collect_error, Error} ->
ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleID, 'sql.failed.exception'), ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleID, 'sql.failed.exception'),
?SLOG(warning, #{msg => "FOREACH_clause_exception", ?SLOG(warning, #{
rule_id => RuleID, reason => Error}), msg => "FOREACH_clause_exception",
rule_id => RuleID,
reason => Error
}),
{error, Reason}; {error, Reason};
_:Reason = {match_incase_error, Error} -> _:Reason = {match_incase_error, Error} ->
ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleID, 'sql.failed.exception'), ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleID, 'sql.failed.exception'),
?SLOG(warning, #{msg => "INCASE_clause_exception", ?SLOG(warning, #{
rule_id => RuleID, reason => Error}), msg => "INCASE_clause_exception",
rule_id => RuleID,
reason => Error
}),
{error, Reason}; {error, Reason};
Class:Error:StkTrace -> Class:Error:StkTrace ->
ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleID, 'sql.failed.exception'), ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleID, 'sql.failed.exception'),
?SLOG(error, #{msg => "apply_rule_failed", ?SLOG(error, #{
rule_id => RuleID, msg => "apply_rule_failed",
exception => Class, rule_id => RuleID,
reason => Error, exception => Class,
stacktrace => StkTrace reason => Error,
}), stacktrace => StkTrace
}),
{error, {Error, StkTrace}} {error, {Error, StkTrace}}
end. end.
do_apply_rule(#{ do_apply_rule(
id := RuleId, #{
is_foreach := true, id := RuleId,
fields := Fields, is_foreach := true,
doeach := DoEach, fields := Fields,
incase := InCase, doeach := DoEach,
conditions := Conditions, incase := InCase,
outputs := Outputs conditions := Conditions,
}, Input) -> outputs := Outputs
{Selected, Collection} = ?RAISE(select_and_collect(Fields, Input), },
{select_and_collect_error, {_EXCLASS_,_EXCPTION_,_ST_}}), Input
) ->
{Selected, Collection} = ?RAISE(
select_and_collect(Fields, Input),
{select_and_collect_error, {_EXCLASS_, _EXCPTION_, _ST_}}
),
ColumnsAndSelected = maps:merge(Input, Selected), ColumnsAndSelected = maps:merge(Input, Selected),
case ?RAISE(match_conditions(Conditions, ColumnsAndSelected), case
{match_conditions_error, {_EXCLASS_,_EXCPTION_,_ST_}}) of ?RAISE(
match_conditions(Conditions, ColumnsAndSelected),
{match_conditions_error, {_EXCLASS_, _EXCPTION_, _ST_}}
)
of
true -> true ->
Collection2 = filter_collection(Input, InCase, DoEach, Collection), Collection2 = filter_collection(Input, InCase, DoEach, Collection),
case Collection2 of case Collection2 of
@ -122,17 +150,26 @@ do_apply_rule(#{
ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleId, 'sql.failed.no_result'), ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleId, 'sql.failed.no_result'),
{error, nomatch} {error, nomatch}
end; end;
do_apply_rule(
do_apply_rule(#{id := RuleId, #{
is_foreach := false, id := RuleId,
fields := Fields, is_foreach := false,
conditions := Conditions, fields := Fields,
outputs := Outputs conditions := Conditions,
}, Input) -> outputs := Outputs
Selected = ?RAISE(select_and_transform(Fields, Input), },
{select_and_transform_error, {_EXCLASS_,_EXCPTION_,_ST_}}), Input
case ?RAISE(match_conditions(Conditions, maps:merge(Input, Selected)), ) ->
{match_conditions_error, {_EXCLASS_,_EXCPTION_,_ST_}}) of Selected = ?RAISE(
select_and_transform(Fields, Input),
{select_and_transform_error, {_EXCLASS_, _EXCPTION_, _ST_}}
),
case
?RAISE(
match_conditions(Conditions, maps:merge(Input, Selected)),
{match_conditions_error, {_EXCLASS_, _EXCPTION_, _ST_}}
)
of
true -> true ->
ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleId, 'sql.passed'), ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleId, 'sql.passed'),
{ok, handle_output_list(RuleId, Outputs, Selected, Input)}; {ok, handle_output_list(RuleId, Outputs, Selected, Input)};
@ -154,15 +191,19 @@ select_and_transform(['*' | More], Input, Output) ->
select_and_transform(More, Input, maps:merge(Output, Input)); select_and_transform(More, Input, maps:merge(Output, Input));
select_and_transform([{as, Field, Alias} | More], Input, Output) -> select_and_transform([{as, Field, Alias} | More], Input, Output) ->
Val = eval(Field, Input), Val = eval(Field, Input),
select_and_transform(More, select_and_transform(
More,
nested_put(Alias, Val, Input), nested_put(Alias, Val, Input),
nested_put(Alias, Val, Output)); nested_put(Alias, Val, Output)
);
select_and_transform([Field | More], Input, Output) -> select_and_transform([Field | More], Input, Output) ->
Val = eval(Field, Input), Val = eval(Field, Input),
Key = alias(Field), Key = alias(Field),
select_and_transform(More, select_and_transform(
More,
nested_put(Key, Val, Input), nested_put(Key, Val, Input),
nested_put(Key, Val, Output)). nested_put(Key, Val, Output)
).
%% FOREACH Clause %% FOREACH Clause
-spec select_and_collect(list(), input()) -> {input(), collection()}. -spec select_and_collect(list(), input()) -> {input(), collection()}.
@ -174,9 +215,11 @@ select_and_collect([{as, Field, {_, A} = Alias}], Input, {Output, _}) ->
{nested_put(Alias, Val, Output), {A, ensure_list(Val)}}; {nested_put(Alias, Val, Output), {A, ensure_list(Val)}};
select_and_collect([{as, Field, Alias} | More], Input, {Output, LastKV}) -> select_and_collect([{as, Field, Alias} | More], Input, {Output, LastKV}) ->
Val = eval(Field, Input), Val = eval(Field, Input),
select_and_collect(More, select_and_collect(
More,
nested_put(Alias, Val, Input), nested_put(Alias, Val, Input),
{nested_put(Alias, Val, Output), LastKV}); {nested_put(Alias, Val, Output), LastKV}
);
select_and_collect([Field], Input, {Output, _}) -> select_and_collect([Field], Input, {Output, _}) ->
Val = eval(Field, Input), Val = eval(Field, Input),
Key = alias(Field), Key = alias(Field),
@ -184,24 +227,36 @@ select_and_collect([Field], Input, {Output, _}) ->
select_and_collect([Field | More], Input, {Output, LastKV}) -> select_and_collect([Field | More], Input, {Output, LastKV}) ->
Val = eval(Field, Input), Val = eval(Field, Input),
Key = alias(Field), Key = alias(Field),
select_and_collect(More, select_and_collect(
More,
nested_put(Key, Val, Input), nested_put(Key, Val, Input),
{nested_put(Key, Val, Output), LastKV}). {nested_put(Key, Val, Output), LastKV}
).
%% Filter each item got from FOREACH %% Filter each item got from FOREACH
filter_collection(Input, InCase, DoEach, {CollKey, CollVal}) -> filter_collection(Input, InCase, DoEach, {CollKey, CollVal}) ->
lists:filtermap( lists:filtermap(
fun(Item) -> fun(Item) ->
InputAndItem = maps:merge(Input, #{CollKey => Item}), InputAndItem = maps:merge(Input, #{CollKey => Item}),
case ?RAISE(match_conditions(InCase, InputAndItem), case
{match_incase_error, {_EXCLASS_,_EXCPTION_,_ST_}}) of ?RAISE(
match_conditions(InCase, InputAndItem),
{match_incase_error, {_EXCLASS_, _EXCPTION_, _ST_}}
)
of
true when DoEach == [] -> {true, InputAndItem}; true when DoEach == [] -> {true, InputAndItem};
true -> true ->
{true, ?RAISE(select_and_transform(DoEach, InputAndItem), {true,
{doeach_error, {_EXCLASS_,_EXCPTION_,_ST_}})}; ?RAISE(
false -> false select_and_transform(DoEach, InputAndItem),
{doeach_error, {_EXCLASS_, _EXCPTION_, _ST_}}
)};
false ->
false
end end
end, CollVal). end,
CollVal
).
%% Conditional Clauses such as WHERE, WHEN. %% Conditional Clauses such as WHERE, WHEN.
match_conditions({'and', L, R}, Data) -> match_conditions({'and', L, R}, Data) ->
@ -212,7 +267,8 @@ match_conditions({'not', Var}, Data) ->
case eval(Var, Data) of case eval(Var, Data) of
Bool when is_boolean(Bool) -> Bool when is_boolean(Bool) ->
not Bool; not Bool;
_other -> false _other ->
false
end; end;
match_conditions({in, Var, {list, Vals}}, Data) -> match_conditions({in, Var, {list, Vals}}, Data) ->
lists:member(eval(Var, Data), [eval(V, Data) || V <- Vals]); lists:member(eval(Var, Data), [eval(V, Data) || V <- Vals]);
@ -250,8 +306,10 @@ do_compare('!=', L, R) -> L /= R;
do_compare('=~', T, F) -> emqx_topic:match(T, F). do_compare('=~', T, F) -> emqx_topic:match(T, F).
number(Bin) -> number(Bin) ->
try binary_to_integer(Bin) try
catch error:badarg -> binary_to_float(Bin) binary_to_integer(Bin)
catch
error:badarg -> binary_to_float(Bin)
end. end.
handle_output_list(RuleId, Outputs, Selected, Envs) -> handle_output_list(RuleId, Outputs, Selected, Envs) ->
@ -266,13 +324,20 @@ handle_output(RuleId, OutId, Selected, Envs) ->
catch catch
throw:out_of_service -> throw:out_of_service ->
ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleId, 'outputs.failed'), ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleId, 'outputs.failed'),
ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleId, 'outputs.failed.out_of_service'), ok = emqx_plugin_libs_metrics:inc(
rule_metrics, RuleId, 'outputs.failed.out_of_service'
),
?SLOG(warning, #{msg => "out_of_service", output => OutId}); ?SLOG(warning, #{msg => "out_of_service", output => OutId});
Err:Reason:ST -> Err:Reason:ST ->
ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleId, 'outputs.failed'), ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleId, 'outputs.failed'),
ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleId, 'outputs.failed.unknown'), ok = emqx_plugin_libs_metrics:inc(rule_metrics, RuleId, 'outputs.failed.unknown'),
?SLOG(error, #{msg => "output_failed", output => OutId, exception => Err, ?SLOG(error, #{
reason => Reason, stacktrace => ST}) msg => "output_failed",
output => OutId,
exception => Err,
reason => Reason,
stacktrace => ST
})
end. end.
do_handle_output(BridgeId, Selected, _Envs) when is_binary(BridgeId) -> do_handle_output(BridgeId, Selected, _Envs) when is_binary(BridgeId) ->
@ -280,7 +345,8 @@ do_handle_output(BridgeId, Selected, _Envs) when is_binary(BridgeId) ->
case emqx_bridge:send_message(BridgeId, Selected) of case emqx_bridge:send_message(BridgeId, Selected) of
{error, {Err, _}} when Err == bridge_not_found; Err == bridge_stopped -> {error, {Err, _}} when Err == bridge_not_found; Err == bridge_stopped ->
throw(out_of_service); throw(out_of_service);
Result -> Result Result ->
Result
end; end;
do_handle_output(#{mod := Mod, func := Func, args := Args}, Selected, Envs) -> do_handle_output(#{mod := Mod, func := Func, args := Args}, Selected, Envs) ->
%% the function can also throw 'out_of_service' %% the function can also throw 'out_of_service'
@ -382,8 +448,10 @@ apply_func(Name, Args, Input) when is_atom(Name) ->
do_apply_func(Name, Args, Input); do_apply_func(Name, Args, Input);
apply_func(Name, Args, Input) when is_binary(Name) -> apply_func(Name, Args, Input) when is_binary(Name) ->
FunName = FunName =
try binary_to_existing_atom(Name, utf8) try
catch error:badarg -> error({sql_function_not_supported, Name}) binary_to_existing_atom(Name, utf8)
catch
error:badarg -> error({sql_function_not_supported, Name})
end, end,
do_apply_func(FunName, Args, Input). do_apply_func(FunName, Args, Input).
@ -391,7 +459,8 @@ do_apply_func(Name, Args, Input) ->
case erlang:apply(emqx_rule_funcs, Name, Args) of case erlang:apply(emqx_rule_funcs, Name, Args) of
Func when is_function(Func) -> Func when is_function(Func) ->
erlang:apply(Func, [Input]); erlang:apply(Func, [Input]);
Result -> Result Result ->
Result
end. end.
add_metadata(Input, Metadata) when is_map(Input), is_map(Metadata) -> add_metadata(Input, Metadata) when is_map(Input), is_map(Metadata) ->
@ -417,8 +486,10 @@ cache_payload(DecodedP) ->
DecodedP. DecodedP.
safe_decode_and_cache(MaybeJson) -> safe_decode_and_cache(MaybeJson) ->
try cache_payload(emqx_json:decode(MaybeJson, [return_maps])) try
catch _:_ -> error({decode_json_failed, MaybeJson}) cache_payload(emqx_json:decode(MaybeJson, [return_maps]))
catch
_:_ -> error({decode_json_failed, MaybeJson})
end. end.
ensure_list(List) when is_list(List) -> List; ensure_list(List) when is_list(List) -> List;

83
apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl
View File

@ -20,84 +20,89 @@
-export([parse/1]). -export([parse/1]).
-export([ select_fields/1 -export([
, select_is_foreach/1 select_fields/1,
, select_doeach/1 select_is_foreach/1,
, select_incase/1 select_doeach/1,
, select_from/1 select_incase/1,
, select_where/1 select_from/1,
]). select_where/1
]).
-import(proplists, [ get_value/2 -import(proplists, [
, get_value/3 get_value/2,
]). get_value/3
]).
-record(select, {fields, from, where, is_foreach, doeach, incase}). -record(select, {fields, from, where, is_foreach, doeach, incase}).
-opaque(select() :: #select{}). -opaque select() :: #select{}.
-type const() :: {const, number()|binary()}. -type const() :: {const, number() | binary()}.
-type variable() :: binary() | list(binary()). -type variable() :: binary() | list(binary()).
-type alias() :: binary() | list(binary()). -type alias() :: binary() | list(binary()).
-type field() :: const() | variable() -type field() ::
| {as, field(), alias()} const()
| {'fun', atom(), list(field())}. | variable()
| {as, field(), alias()}
| {'fun', atom(), list(field())}.
-export_type([select/0]). -export_type([select/0]).
%% Parse one select statement. %% Parse one select statement.
-spec(parse(string() | binary()) -> {ok, select()} | {error, term()}). -spec parse(string() | binary()) -> {ok, select()} | {error, term()}.
parse(Sql) -> parse(Sql) ->
try case rulesql:parsetree(Sql) of try
case rulesql:parsetree(Sql) of
{ok, {select, Clauses}} -> {ok, {select, Clauses}} ->
{ok, #select{ {ok, #select{
is_foreach = false, is_foreach = false,
fields = get_value(fields, Clauses), fields = get_value(fields, Clauses),
doeach = [], doeach = [],
incase = {}, incase = {},
from = get_value(from, Clauses), from = get_value(from, Clauses),
where = get_value(where, Clauses) where = get_value(where, Clauses)
}}; }};
{ok, {foreach, Clauses}} -> {ok, {foreach, Clauses}} ->
{ok, #select{ {ok, #select{
is_foreach = true, is_foreach = true,
fields = get_value(fields, Clauses), fields = get_value(fields, Clauses),
doeach = get_value(do, Clauses, []), doeach = get_value(do, Clauses, []),
incase = get_value(incase, Clauses, {}), incase = get_value(incase, Clauses, {}),
from = get_value(from, Clauses), from = get_value(from, Clauses),
where = get_value(where, Clauses) where = get_value(where, Clauses)
}}; }};
Error -> {error, Error} Error ->
{error, Error}
end end
catch catch
_Error:Reason:StackTrace -> _Error:Reason:StackTrace ->
{error, {Reason, StackTrace}} {error, {Reason, StackTrace}}
end. end.
-spec(select_fields(select()) -> list(field())). -spec select_fields(select()) -> list(field()).
select_fields(#select{fields = Fields}) -> select_fields(#select{fields = Fields}) ->
Fields. Fields.
-spec(select_is_foreach(select()) -> boolean()). -spec select_is_foreach(select()) -> boolean().
select_is_foreach(#select{is_foreach = IsForeach}) -> select_is_foreach(#select{is_foreach = IsForeach}) ->
IsForeach. IsForeach.
-spec(select_doeach(select()) -> list(field())). -spec select_doeach(select()) -> list(field()).
select_doeach(#select{doeach = DoEach}) -> select_doeach(#select{doeach = DoEach}) ->
DoEach. DoEach.
-spec(select_incase(select()) -> list(field())). -spec select_incase(select()) -> list(field()).
select_incase(#select{incase = InCase}) -> select_incase(#select{incase = InCase}) ->
InCase. InCase.
-spec(select_from(select()) -> list(binary())). -spec select_from(select()) -> list(binary()).
select_from(#select{from = From}) -> select_from(#select{from = From}) ->
From. From.
-spec(select_where(select()) -> tuple()). -spec select_where(select()) -> tuple().
select_where(#select{where = Where}) -> select_where(#select{where = Where}) ->
Where. Where.

23
apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
View File

@ -17,10 +17,11 @@
-include("rule_engine.hrl"). -include("rule_engine.hrl").
-include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/logger.hrl").
-export([ test/1 -export([
, echo_action/2 test/1,
, get_selected_data/3 echo_action/2,
]). get_selected_data/3
]).
-spec test(#{sql := binary(), context := map()}) -> {ok, map() | list()} | {error, term()}. -spec test(#{sql := binary(), context := map()}) -> {ok, map() | list()} | {error, term()}.
test(#{sql := Sql, context := Context}) -> test(#{sql := Sql, context := Context}) ->
@ -60,9 +61,7 @@ test_rule(Sql, Select, Context, EventTopics) ->
created_at => erlang:system_time(millisecond) created_at => erlang:system_time(millisecond)
}, },
FullContext = fill_default_values(hd(EventTopics), emqx_rule_maps:atom_key_map(Context)), FullContext = fill_default_values(hd(EventTopics), emqx_rule_maps:atom_key_map(Context)),
try try emqx_rule_runtime:apply_rule(Rule, FullContext) of
emqx_rule_runtime:apply_rule(Rule, FullContext)
of
{ok, Data} -> {ok, flatten(Data)}; {ok, Data} -> {ok, flatten(Data)};
{error, Reason} -> {error, Reason} {error, Reason} -> {error, Reason}
after after
@ -76,8 +75,10 @@ is_publish_topic(<<"$events/", _/binary>>) -> false;
is_publish_topic(<<"$bridges/", _/binary>>) -> false; is_publish_topic(<<"$bridges/", _/binary>>) -> false;
is_publish_topic(_Topic) -> true. is_publish_topic(_Topic) -> true.
flatten([]) -> []; flatten([]) ->
flatten([D1]) -> D1; [];
flatten([D1]) ->
D1;
flatten([D1 | L]) when is_list(D1) -> flatten([D1 | L]) when is_list(D1) ->
D1 ++ flatten(L). D1 ++ flatten(L).
@ -92,4 +93,6 @@ envs_examp(EventTopic) ->
EventName = emqx_rule_events:event_name(EventTopic), EventName = emqx_rule_events:event_name(EventTopic),
emqx_rule_maps:atom_key_map( emqx_rule_maps:atom_key_map(
maps:from_list( maps:from_list(
emqx_rule_events:columns_with_exam(EventName))). emqx_rule_events:columns_with_exam(EventName)
)
).

9
apps/emqx_rule_engine/src/proto/emqx_rule_engine_proto_v1.erl
View File

@ -18,10 +18,11 @@
-behaviour(emqx_bpapi). -behaviour(emqx_bpapi).
-export([ introduced_in/0 -export([
introduced_in/0,
, reset_metrics/1 reset_metrics/1
]). ]).
-include_lib("emqx/include/bpapi.hrl"). -include_lib("emqx/include/bpapi.hrl").
-include_lib("emqx_rule_engine/include/rule_engine.hrl"). -include_lib("emqx_rule_engine/include/rule_engine.hrl").
@ -30,6 +31,6 @@ introduced_in() ->
"5.0.0". "5.0.0".
-spec reset_metrics(rule_id()) -> -spec reset_metrics(rule_id()) ->
emqx_cluster_rpc:multicall_return(ok). emqx_cluster_rpc:multicall_return(ok).
reset_metrics(RuleId) -> reset_metrics(RuleId) ->
emqx_cluster_rpc:multicall(emqx_rule_engine, reset_metrics_for_rule, [RuleId]). emqx_cluster_rpc:multicall(emqx_rule_engine, reset_metrics_for_rule, [RuleId]).

2954
apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
View File

File diff suppressed because it is too large Load Diff

31
apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl
View File

@ -38,15 +38,19 @@ t_crud_rule_api(_Config) ->
}, },
{201, Rule} = emqx_rule_engine_api:'/rules'(post, #{body => Params0}), {201, Rule} = emqx_rule_engine_api:'/rules'(post, #{body => Params0}),
%% if we post again with the same params, it return with 400 "rule id already exists" %% if we post again with the same params, it return with 400 "rule id already exists"
?assertMatch({400, #{code := _, message := _Message}}, ?assertMatch(
emqx_rule_engine_api:'/rules'(post, #{body => Params0})), {400, #{code := _, message := _Message}},
emqx_rule_engine_api:'/rules'(post, #{body => Params0})
),
?assertEqual(RuleID, maps:get(id, Rule)), ?assertEqual(RuleID, maps:get(id, Rule)),
{200, Rules} = emqx_rule_engine_api:'/rules'(get, #{}), {200, Rules} = emqx_rule_engine_api:'/rules'(get, #{}),
ct:pal("RList : ~p", [Rules]), ct:pal("RList : ~p", [Rules]),
?assert(length(Rules) > 0), ?assert(length(Rules) > 0),
{200, Rule0} = emqx_rule_engine_api:'/rules/:id/reset_metrics'(put, #{bindings => #{id => RuleID}}), {200, Rule0} = emqx_rule_engine_api:'/rules/:id/reset_metrics'(put, #{
bindings => #{id => RuleID}
}),
?assertEqual(<<"Reset Success">>, Rule0), ?assertEqual(<<"Reset Success">>, Rule0),
{200, Rule1} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}}), {200, Rule1} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}}),
@ -54,19 +58,26 @@ t_crud_rule_api(_Config) ->
?assertEqual(Rule, Rule1), ?assertEqual(Rule, Rule1),
{200, Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{ {200, Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{
bindings => #{id => RuleID}, bindings => #{id => RuleID},
body => Params0#{<<"sql">> => <<"select * from \"t/b\"">>} body => Params0#{<<"sql">> => <<"select * from \"t/b\"">>}
}), }),
{200, Rule3} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}}), {200, Rule3} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}}),
%ct:pal("RShow : ~p", [Rule3]), %ct:pal("RShow : ~p", [Rule3]),
?assertEqual(Rule3, Rule2), ?assertEqual(Rule3, Rule2),
?assertEqual(<<"select * from \"t/b\"">>, maps:get(sql, Rule3)), ?assertEqual(<<"select * from \"t/b\"">>, maps:get(sql, Rule3)),
?assertMatch({204}, emqx_rule_engine_api:'/rules/:id'(delete, ?assertMatch(
#{bindings => #{id => RuleID}})), {204},
emqx_rule_engine_api:'/rules/:id'(
delete,
#{bindings => #{id => RuleID}}
)
),
%ct:pal("Show After Deleted: ~p", [NotFound]), %ct:pal("Show After Deleted: ~p", [NotFound]),
?assertMatch({404, #{code := _, message := _Message}}, ?assertMatch(
emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}})), {404, #{code := _, message := _Message}},
emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}})
),
ok. ok.

41
apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl
View File

@ -9,23 +9,30 @@ all() -> emqx_common_test_helpers:all(?MODULE).
t_mod_hook_fun(_) -> t_mod_hook_fun(_) ->
Funcs = emqx_rule_events:module_info(exports), Funcs = emqx_rule_events:module_info(exports),
[?assert(lists:keymember(emqx_rule_events:hook_fun(Event), 1, Funcs)) || [
Event <- ['client.connected', ?assert(lists:keymember(emqx_rule_events:hook_fun(Event), 1, Funcs))
'client.disconnected', || Event <- [
'session.subscribed', 'client.connected',
'session.unsubscribed', 'client.disconnected',
'message.acked', 'session.subscribed',
'message.dropped', 'session.unsubscribed',
'message.delivered' 'message.acked',
]]. 'message.dropped',
'message.delivered'
]
].
t_printable_maps(_) -> t_printable_maps(_) ->
Headers = #{peerhost => {127,0,0,1}, Headers = #{
peername => {{127,0,0,1}, 9980}, peerhost => {127, 0, 0, 1},
sockname => {{127,0,0,1}, 1883} peername => {{127, 0, 0, 1}, 9980},
}, sockname => {{127, 0, 0, 1}, 1883}
},
?assertMatch( ?assertMatch(
#{peerhost := <<"127.0.0.1">>, #{
peername := <<"127.0.0.1:9980">>, peerhost := <<"127.0.0.1">>,
sockname := <<"127.0.0.1:1883">> peername := <<"127.0.0.1:9980">>,
}, emqx_rule_events:printable_maps(Headers)). sockname := <<"127.0.0.1:1883">>
},
emqx_rule_events:printable_maps(Headers)
).

632
apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl
View File

@ -35,7 +35,9 @@
t_msgid(_) -> t_msgid(_) ->
Msg = message(), Msg = message(),
?assertEqual(undefined, apply_func(msgid, [], #{})), ?assertEqual(undefined, apply_func(msgid, [], #{})),
?assertEqual(emqx_guid:to_hexstr(emqx_message:id(Msg)), apply_func(msgid, [], eventmsg_publish(Msg))). ?assertEqual(
emqx_guid:to_hexstr(emqx_message:id(Msg)), apply_func(msgid, [], eventmsg_publish(Msg))
).
t_qos(_) -> t_qos(_) ->
?assertEqual(undefined, apply_func(qos, [], #{})), ?assertEqual(undefined, apply_func(qos, [], #{})),
@ -61,12 +63,12 @@ t_clientid(_) ->
?assertEqual(<<"clientid">>, apply_func(clientid, [], Msg)). ?assertEqual(<<"clientid">>, apply_func(clientid, [], Msg)).
t_clientip(_) -> t_clientip(_) ->
Msg = emqx_message:set_header(peerhost, {127,0,0,1}, message()), Msg = emqx_message:set_header(peerhost, {127, 0, 0, 1}, message()),
?assertEqual(undefined, apply_func(clientip, [], #{})), ?assertEqual(undefined, apply_func(clientip, [], #{})),
?assertEqual(<<"127.0.0.1">>, apply_func(clientip, [], eventmsg_publish(Msg))). ?assertEqual(<<"127.0.0.1">>, apply_func(clientip, [], eventmsg_publish(Msg))).
t_peerhost(_) -> t_peerhost(_) ->
Msg = emqx_message:set_header(peerhost, {127,0,0,1}, message()), Msg = emqx_message:set_header(peerhost, {127, 0, 0, 1}, message()),
?assertEqual(undefined, apply_func(peerhost, [], #{})), ?assertEqual(undefined, apply_func(peerhost, [], #{})),
?assertEqual(<<"127.0.0.1">>, apply_func(peerhost, [], eventmsg_publish(Msg))). ?assertEqual(<<"127.0.0.1">>, apply_func(peerhost, [], eventmsg_publish(Msg))).
@ -87,7 +89,7 @@ t_str(_) ->
?assertEqual(<<"abc">>, emqx_rule_funcs:str("abc")), ?assertEqual(<<"abc">>, emqx_rule_funcs:str("abc")),
?assertEqual(<<"abc">>, emqx_rule_funcs:str(abc)), ?assertEqual(<<"abc">>, emqx_rule_funcs:str(abc)),
?assertEqual(<<"{\"a\":1}">>, emqx_rule_funcs:str(#{a => 1})), ?assertEqual(<<"{\"a\":1}">>, emqx_rule_funcs:str(#{a => 1})),
?assertEqual(<<"[{\"a\":1},{\"b\":1}]">>, emqx_rule_funcs:str([#{a => 1},#{b => 1}])), ?assertEqual(<<"[{\"a\":1},{\"b\":1}]">>, emqx_rule_funcs:str([#{a => 1}, #{b => 1}])),
?assertEqual(<<"1">>, emqx_rule_funcs:str(1)), ?assertEqual(<<"1">>, emqx_rule_funcs:str(1)),
?assertEqual(<<"2.0">>, emqx_rule_funcs:str(2.0)), ?assertEqual(<<"2.0">>, emqx_rule_funcs:str(2.0)),
?assertEqual(<<"true">>, emqx_rule_funcs:str(true)), ?assertEqual(<<"true">>, emqx_rule_funcs:str(true)),
@ -97,7 +99,9 @@ t_str(_) ->
?assertEqual(<<"abc 你好"/utf8>>, emqx_rule_funcs:str_utf8("abc 你好")), ?assertEqual(<<"abc 你好"/utf8>>, emqx_rule_funcs:str_utf8("abc 你好")),
?assertEqual(<<"abc 你好"/utf8>>, emqx_rule_funcs:str_utf8(<<"abc 你好"/utf8>>)), ?assertEqual(<<"abc 你好"/utf8>>, emqx_rule_funcs:str_utf8(<<"abc 你好"/utf8>>)),
?assertEqual(<<"abc">>, emqx_rule_funcs:str_utf8(abc)), ?assertEqual(<<"abc">>, emqx_rule_funcs:str_utf8(abc)),
?assertEqual(<<"{\"a\":\"abc 你好\"}"/utf8>>, emqx_rule_funcs:str_utf8(#{a => <<"abc 你好"/utf8>>})), ?assertEqual(
<<"{\"a\":\"abc 你好\"}"/utf8>>, emqx_rule_funcs:str_utf8(#{a => <<"abc 你好"/utf8>>})
),
?assertEqual(<<"1">>, emqx_rule_funcs:str_utf8(1)), ?assertEqual(<<"1">>, emqx_rule_funcs:str_utf8(1)),
?assertEqual(<<"2.0">>, emqx_rule_funcs:str_utf8(2.0)), ?assertEqual(<<"2.0">>, emqx_rule_funcs:str_utf8(2.0)),
?assertEqual(<<"true">>, emqx_rule_funcs:str_utf8(true)), ?assertEqual(<<"true">>, emqx_rule_funcs:str_utf8(true)),
@ -126,7 +130,9 @@ t_float(_) ->
?assertError(_, emqx_rule_funcs:float("a")). ?assertError(_, emqx_rule_funcs:float("a")).
t_map(_) -> t_map(_) ->
?assertEqual(#{ver => <<"1.0">>, name => "emqx"}, emqx_rule_funcs:map([{ver, <<"1.0">>}, {name, "emqx"}])), ?assertEqual(
#{ver => <<"1.0">>, name => "emqx"}, emqx_rule_funcs:map([{ver, <<"1.0">>}, {name, "emqx"}])
),
?assertEqual(#{<<"a">> => 1}, emqx_rule_funcs:map(<<"{\"a\":1}">>)), ?assertEqual(#{<<"a">> => 1}, emqx_rule_funcs:map(<<"{\"a\":1}">>)),
?assertError(_, emqx_rule_funcs:map(<<"a">>)), ?assertError(_, emqx_rule_funcs:map(<<"a">>)),
?assertError(_, emqx_rule_funcs:map("a")), ?assertError(_, emqx_rule_funcs:map("a")),
@ -151,31 +157,42 @@ t_proc_dict_put_get_del(_) ->
?assertEqual(undefined, emqx_rule_funcs:proc_dict_get(<<"abc">>)). ?assertEqual(undefined, emqx_rule_funcs:proc_dict_get(<<"abc">>)).
t_term_encode(_) -> t_term_encode(_) ->
TestData = [<<"abc">>, #{a => 1}, #{<<"3">> => [1,2,4]}], TestData = [<<"abc">>, #{a => 1}, #{<<"3">> => [1, 2, 4]}],
lists:foreach(fun(Data) -> lists:foreach(
?assertEqual(Data, fun(Data) ->
?assertEqual(
Data,
emqx_rule_funcs:term_decode( emqx_rule_funcs:term_decode(
emqx_rule_funcs:term_encode(Data))) emqx_rule_funcs:term_encode(Data)
end, TestData). )
)
end,
TestData
).
t_hexstr2bin(_) -> t_hexstr2bin(_) ->
?assertEqual(<<1,2>>, emqx_rule_funcs:hexstr2bin(<<"0102">>)), ?assertEqual(<<1, 2>>, emqx_rule_funcs:hexstr2bin(<<"0102">>)),
?assertEqual(<<17,33>>, emqx_rule_funcs:hexstr2bin(<<"1121">>)). ?assertEqual(<<17, 33>>, emqx_rule_funcs:hexstr2bin(<<"1121">>)).
t_bin2hexstr(_) -> t_bin2hexstr(_) ->
?assertEqual(<<"0102">>, emqx_rule_funcs:bin2hexstr(<<1,2>>)), ?assertEqual(<<"0102">>, emqx_rule_funcs:bin2hexstr(<<1, 2>>)),
?assertEqual(<<"1121">>, emqx_rule_funcs:bin2hexstr(<<17,33>>)). ?assertEqual(<<"1121">>, emqx_rule_funcs:bin2hexstr(<<17, 33>>)).
t_hex_convert(_) -> t_hex_convert(_) ->
?PROPTEST(hex_convert). ?PROPTEST(hex_convert).
hex_convert() -> hex_convert() ->
?FORALL(L, list(range(0, 255)), ?FORALL(
begin L,
AbitraryBin = list_to_binary(L), list(range(0, 255)),
AbitraryBin == emqx_rule_funcs:hexstr2bin( begin
emqx_rule_funcs:bin2hexstr(AbitraryBin)) AbitraryBin = list_to_binary(L),
end). AbitraryBin ==
emqx_rule_funcs:hexstr2bin(
emqx_rule_funcs:bin2hexstr(AbitraryBin)
)
end
).
t_is_null(_) -> t_is_null(_) ->
?assertEqual(true, emqx_rule_funcs:is_null(undefined)), ?assertEqual(true, emqx_rule_funcs:is_null(undefined)),
@ -184,50 +201,80 @@ t_is_null(_) ->
?assertEqual(false, emqx_rule_funcs:is_null(<<"a">>)). ?assertEqual(false, emqx_rule_funcs:is_null(<<"a">>)).
t_is_not_null(_) -> t_is_not_null(_) ->
[?assertEqual(emqx_rule_funcs:is_not_null(T), not emqx_rule_funcs:is_null(T)) [
|| T <- [undefined, a, <<"a">>, <<>>]]. ?assertEqual(emqx_rule_funcs:is_not_null(T), not emqx_rule_funcs:is_null(T))
|| T <- [undefined, a, <<"a">>, <<>>]
].
t_is_str(_) -> t_is_str(_) ->
[?assertEqual(true, emqx_rule_funcs:is_str(T)) [
|| T <- [<<"a">>, <<>>, <<"abc">>]], ?assertEqual(true, emqx_rule_funcs:is_str(T))
[?assertEqual(false, emqx_rule_funcs:is_str(T)) || T <- [<<"a">>, <<>>, <<"abc">>]
|| T <- ["a", a, 1]]. ],
[
?assertEqual(false, emqx_rule_funcs:is_str(T))
|| T <- ["a", a, 1]
].
t_is_bool(_) -> t_is_bool(_) ->
[?assertEqual(true, emqx_rule_funcs:is_bool(T)) [
|| T <- [true, false]], ?assertEqual(true, emqx_rule_funcs:is_bool(T))
[?assertEqual(false, emqx_rule_funcs:is_bool(T)) || T <- [true, false]
|| T <- ["a", <<>>, a, 2]]. ],
[
?assertEqual(false, emqx_rule_funcs:is_bool(T))
|| T <- ["a", <<>>, a, 2]
].
t_is_int(_) -> t_is_int(_) ->
[?assertEqual(true, emqx_rule_funcs:is_int(T)) [
|| T <- [1, 2, -1]], ?assertEqual(true, emqx_rule_funcs:is_int(T))
[?assertEqual(false, emqx_rule_funcs:is_int(T)) || T <- [1, 2, -1]
|| T <- [1.1, "a", a]]. ],
[
?assertEqual(false, emqx_rule_funcs:is_int(T))
|| T <- [1.1, "a", a]
].
t_is_float(_) -> t_is_float(_) ->
[?assertEqual(true, emqx_rule_funcs:is_float(T)) [
|| T <- [1.1, 2.0, -1.2]], ?assertEqual(true, emqx_rule_funcs:is_float(T))
[?assertEqual(false, emqx_rule_funcs:is_float(T)) || T <- [1.1, 2.0, -1.2]
|| T <- [1, "a", a, <<>>]]. ],
[
?assertEqual(false, emqx_rule_funcs:is_float(T))
|| T <- [1, "a", a, <<>>]
].
t_is_num(_) -> t_is_num(_) ->
[?assertEqual(true, emqx_rule_funcs:is_num(T)) [
|| T <- [1.1, 2.0, -1.2, 1]], ?assertEqual(true, emqx_rule_funcs:is_num(T))
[?assertEqual(false, emqx_rule_funcs:is_num(T)) || T <- [1.1, 2.0, -1.2, 1]
|| T <- ["a", a, <<>>]]. ],
[
?assertEqual(false, emqx_rule_funcs:is_num(T))
|| T <- ["a", a, <<>>]
].
t_is_map(_) -> t_is_map(_) ->
[?assertEqual(true, emqx_rule_funcs:is_map(T)) [
|| T <- [#{}, #{a =>1}]], ?assertEqual(true, emqx_rule_funcs:is_map(T))
[?assertEqual(false, emqx_rule_funcs:is_map(T)) || T <- [#{}, #{a => 1}]
|| T <- ["a", a, <<>>]]. ],
[
?assertEqual(false, emqx_rule_funcs:is_map(T))
|| T <- ["a", a, <<>>]
].
t_is_array(_) -> t_is_array(_) ->
[?assertEqual(true, emqx_rule_funcs:is_array(T)) [
|| T <- [[], [1,2]]], ?assertEqual(true, emqx_rule_funcs:is_array(T))
[?assertEqual(false, emqx_rule_funcs:is_array(T)) || T <- [[], [1, 2]]
|| T <- [<<>>, a]]. ],
[
?assertEqual(false, emqx_rule_funcs:is_array(T))
|| T <- [<<>>, a]
].
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% Test cases for arith op %% Test cases for arith op
@ -237,22 +284,30 @@ t_arith_op(_) ->
?PROPTEST(prop_arith_op). ?PROPTEST(prop_arith_op).
prop_arith_op() -> prop_arith_op() ->
?FORALL({X, Y}, {number(), number()}, ?FORALL(
begin {X, Y},
(X + Y) == apply_func('+', [X, Y]) andalso {number(), number()},
begin
(X + Y) == apply_func('+', [X, Y]) andalso
(X - Y) == apply_func('-', [X, Y]) andalso (X - Y) == apply_func('-', [X, Y]) andalso
(X * Y) == apply_func('*', [X, Y]) andalso (X * Y) == apply_func('*', [X, Y]) andalso
(if Y =/= 0 -> (if
Y =/= 0 ->
(X / Y) == apply_func('/', [X, Y]); (X / Y) == apply_func('/', [X, Y]);
true -> true true ->
end) andalso true
(case is_integer(X) end) andalso
andalso is_pos_integer(Y) of (case
true -> is_integer(X) andalso
(X rem Y) == apply_func('mod', [X, Y]); is_pos_integer(Y)
false -> true of
true ->
(X rem Y) == apply_func('mod', [X, Y]);
false ->
true
end) end)
end). end
).
is_pos_integer(X) -> is_pos_integer(X) ->
is_integer(X) andalso X > 0. is_integer(X) andalso X > 0.
@ -266,29 +321,45 @@ t_math_fun(_) ->
prop_math_fun() -> prop_math_fun() ->
Excluded = [module_info, atanh, asin, acos], Excluded = [module_info, atanh, asin, acos],
MathFuns = [{F, A} || {F, A} <- math:module_info(exports), MathFuns = [
not lists:member(F, Excluded), {F, A}
erlang:function_exported(emqx_rule_funcs, F, A)], || {F, A} <- math:module_info(exports),
?FORALL({X, Y}, {pos_integer(), pos_integer()}, not lists:member(F, Excluded),
begin erlang:function_exported(emqx_rule_funcs, F, A)
lists:foldl(fun({F, 1}, True) -> ],
True andalso comp_with_math(F, X); ?FORALL(
({F = fmod, 2}, True) -> {X, Y},
True andalso (if Y =/= 0 -> {pos_integer(), pos_integer()},
comp_with_math(F, X, Y); begin
true -> true lists:foldl(
end); fun
({F, 2}, True) -> ({F, 1}, True) ->
True andalso comp_with_math(F, X, Y) True andalso comp_with_math(F, X);
end, true, MathFuns) ({F = fmod, 2}, True) ->
end). True andalso
(if
Y =/= 0 ->
comp_with_math(F, X, Y);
true ->
true
end);
({F, 2}, True) ->
True andalso comp_with_math(F, X, Y)
end,
true,
MathFuns
)
end
).
comp_with_math(Fun, X) comp_with_math(Fun, X) when
when Fun =:= exp; Fun =:= exp;
Fun =:= sinh; Fun =:= sinh;
Fun =:= cosh -> Fun =:= cosh
if X < 710 -> math:Fun(X) == apply_func(Fun, [X]); ->
true -> true if
X < 710 -> math:Fun(X) == apply_func(Fun, [X]);
true -> true
end; end;
comp_with_math(F, X) -> comp_with_math(F, X) ->
math:F(X) == apply_func(F, [X]). math:F(X) == apply_func(F, [X]).
@ -304,15 +375,18 @@ t_bits_op(_) ->
?PROPTEST(prop_bits_op). ?PROPTEST(prop_bits_op).
prop_bits_op() -> prop_bits_op() ->
?FORALL({X, Y}, {integer(), integer()}, ?FORALL(
begin {X, Y},
(bnot X) == apply_func(bitnot, [X]) andalso {integer(), integer()},
begin
(bnot X) == apply_func(bitnot, [X]) andalso
(X band Y) == apply_func(bitand, [X, Y]) andalso (X band Y) == apply_func(bitand, [X, Y]) andalso
(X bor Y) == apply_func(bitor, [X, Y]) andalso (X bor Y) == apply_func(bitor, [X, Y]) andalso
(X bxor Y) == apply_func(bitxor, [X, Y]) andalso (X bxor Y) == apply_func(bitxor, [X, Y]) andalso
(X bsl Y) == apply_func(bitsl, [X, Y]) andalso (X bsl Y) == apply_func(bitsl, [X, Y]) andalso
(X bsr Y) == apply_func(bitsr, [X, Y]) (X bsr Y) == apply_func(bitsr, [X, Y])
end). end
).
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% Test cases for string %% Test cases for string
@ -346,77 +420,140 @@ t_trim(_) ->
t_split_all(_) -> t_split_all(_) ->
?assertEqual([], apply_func(split, [<<>>, <<"/">>])), ?assertEqual([], apply_func(split, [<<>>, <<"/">>])),
?assertEqual([], apply_func(split, [<<"/">>, <<"/">>])), ?assertEqual([], apply_func(split, [<<"/">>, <<"/">>])),
?assertEqual([<<"a">>,<<"b">>,<<"c">>], apply_func(split, [<<"/a/b/c">>, <<"/">>])), ?assertEqual([<<"a">>, <<"b">>, <<"c">>], apply_func(split, [<<"/a/b/c">>, <<"/">>])),
?assertEqual([<<"a">>,<<"b">>,<<"c">>], apply_func(split, [<<"/a/b//c/">>, <<"/">>])), ?assertEqual([<<"a">>, <<"b">>, <<"c">>], apply_func(split, [<<"/a/b//c/">>, <<"/">>])),
?assertEqual([<<"a">>,<<"b">>,<<"c">>], apply_func(split, [<<"a,b,c">>, <<",">>])), ?assertEqual([<<"a">>, <<"b">>, <<"c">>], apply_func(split, [<<"a,b,c">>, <<",">>])),
?assertEqual([<<"a">>,<<" b ">>,<<"c">>], apply_func(split, [<<"a, b ,c">>, <<",">>])), ?assertEqual([<<"a">>, <<" b ">>, <<"c">>], apply_func(split, [<<"a, b ,c">>, <<",">>])),
?assertEqual([<<"a">>,<<"b">>,<<"c\n">>], apply_func(split, [<<"a,b,c\n">>, <<",">>])), ?assertEqual([<<"a">>, <<"b">>, <<"c\n">>], apply_func(split, [<<"a,b,c\n">>, <<",">>])),
?assertEqual([<<"a">>,<<"b">>,<<"c\r\n">>], apply_func(split, [<<"a,b,c\r\n">>, <<",">>])). ?assertEqual([<<"a">>, <<"b">>, <<"c\r\n">>], apply_func(split, [<<"a,b,c\r\n">>, <<",">>])).
t_split_notrim_all(_) -> t_split_notrim_all(_) ->
?assertEqual([<<>>], apply_func(split, [<<>>, <<"/">>, <<"notrim">>])), ?assertEqual([<<>>], apply_func(split, [<<>>, <<"/">>, <<"notrim">>])),
?assertEqual([<<>>,<<>>], apply_func(split, [<<"/">>, <<"/">>, <<"notrim">>])), ?assertEqual([<<>>, <<>>], apply_func(split, [<<"/">>, <<"/">>, <<"notrim">>])),
?assertEqual([<<>>, <<"a">>,<<"b">>,<<"c">>], apply_func(split, [<<"/a/b/c">>, <<"/">>, <<"notrim">>])), ?assertEqual(
?assertEqual([<<>>, <<"a">>,<<"b">>, <<>>, <<"c">>, <<>>], apply_func(split, [<<"/a/b//c/">>, <<"/">>, <<"notrim">>])), [<<>>, <<"a">>, <<"b">>, <<"c">>], apply_func(split, [<<"/a/b/c">>, <<"/">>, <<"notrim">>])
?assertEqual([<<>>, <<"a">>,<<"b">>,<<"c\n">>], apply_func(split, [<<",a,b,c\n">>, <<",">>, <<"notrim">>])), ),
?assertEqual([<<"a">>,<<" b">>,<<"c\r\n">>], apply_func(split, [<<"a, b,c\r\n">>, <<",">>, <<"notrim">>])), ?assertEqual(
?assertEqual([<<"哈哈"/utf8>>,<<" 你好"/utf8>>,<<" 是的\r\n"/utf8>>], apply_func(split, [<<"哈哈, 你好, 是的\r\n"/utf8>>, <<",">>, <<"notrim">>])). [<<>>, <<"a">>, <<"b">>, <<>>, <<"c">>, <<>>],
apply_func(split, [<<"/a/b//c/">>, <<"/">>, <<"notrim">>])
),
?assertEqual(
[<<>>, <<"a">>, <<"b">>, <<"c\n">>],
apply_func(split, [<<",a,b,c\n">>, <<",">>, <<"notrim">>])
),
?assertEqual(
[<<"a">>, <<" b">>, <<"c\r\n">>],
apply_func(split, [<<"a, b,c\r\n">>, <<",">>, <<"notrim">>])
),
?assertEqual(
[<<"哈哈"/utf8>>, <<" 你好"/utf8>>, <<" 是的\r\n"/utf8>>],
apply_func(split, [<<"哈哈, 你好, 是的\r\n"/utf8>>, <<",">>, <<"notrim">>])
).
t_split_leading(_) -> t_split_leading(_) ->
?assertEqual([], apply_func(split, [<<>>, <<"/">>, <<"leading">>])), ?assertEqual([], apply_func(split, [<<>>, <<"/">>, <<"leading">>])),
?assertEqual([], apply_func(split, [<<"/">>, <<"/">>, <<"leading">>])), ?assertEqual([], apply_func(split, [<<"/">>, <<"/">>, <<"leading">>])),
?assertEqual([<<"a/b/c">>], apply_func(split, [<<"/a/b/c">>, <<"/">>, <<"leading">>])), ?assertEqual([<<"a/b/c">>], apply_func(split, [<<"/a/b/c">>, <<"/">>, <<"leading">>])),
?assertEqual([<<"a">>,<<"b//c/">>], apply_func(split, [<<"a/b//c/">>, <<"/">>, <<"leading">>])), ?assertEqual(
?assertEqual([<<"a">>,<<"b,c\n">>], apply_func(split, [<<"a,b,c\n">>, <<",">>, <<"leading">>])), [<<"a">>, <<"b//c/">>], apply_func(split, [<<"a/b//c/">>, <<"/">>, <<"leading">>])
?assertEqual([<<"a b">>,<<"c\r\n">>], apply_func(split, [<<"a b,c\r\n">>, <<",">>, <<"leading">>])), ),
?assertEqual([<<"哈哈"/utf8>>,<<" 你好, 是的\r\n"/utf8>>], apply_func(split, [<<"哈哈, 你好, 是的\r\n"/utf8>>, <<",">>, <<"leading">>])). ?assertEqual(
[<<"a">>, <<"b,c\n">>], apply_func(split, [<<"a,b,c\n">>, <<",">>, <<"leading">>])
),
?assertEqual(
[<<"a b">>, <<"c\r\n">>], apply_func(split, [<<"a b,c\r\n">>, <<",">>, <<"leading">>])
),
?assertEqual(
[<<"哈哈"/utf8>>, <<" 你好, 是的\r\n"/utf8>>],
apply_func(split, [<<"哈哈, 你好, 是的\r\n"/utf8>>, <<",">>, <<"leading">>])
).
t_split_leading_notrim(_) -> t_split_leading_notrim(_) ->
?assertEqual([<<>>], apply_func(split, [<<>>, <<"/">>, <<"leading_notrim">>])), ?assertEqual([<<>>], apply_func(split, [<<>>, <<"/">>, <<"leading_notrim">>])),
?assertEqual([<<>>,<<>>], apply_func(split, [<<"/">>, <<"/">>, <<"leading_notrim">>])), ?assertEqual([<<>>, <<>>], apply_func(split, [<<"/">>, <<"/">>, <<"leading_notrim">>])),
?assertEqual([<<>>, <<"a/b/c">>], apply_func(split, [<<"/a/b/c">>, <<"/">>, <<"leading_notrim">>])), ?assertEqual(
?assertEqual([<<"a">>,<<"b//c/">>], apply_func(split, [<<"a/b//c/">>, <<"/">>, <<"leading_notrim">>])), [<<>>, <<"a/b/c">>], apply_func(split, [<<"/a/b/c">>, <<"/">>, <<"leading_notrim">>])
?assertEqual([<<"a">>,<<"b,c\n">>], apply_func(split, [<<"a,b,c\n">>, <<",">>, <<"leading_notrim">>])), ),
?assertEqual([<<"a b">>,<<"c\r\n">>], apply_func(split, [<<"a b,c\r\n">>, <<",">>, <<"leading_notrim">>])), ?assertEqual(
?assertEqual([<<"哈哈"/utf8>>,<<" 你好, 是的\r\n"/utf8>>], apply_func(split, [<<"哈哈, 你好, 是的\r\n"/utf8>>, <<",">>, <<"leading_notrim">>])). [<<"a">>, <<"b//c/">>], apply_func(split, [<<"a/b//c/">>, <<"/">>, <<"leading_notrim">>])
),
?assertEqual(
[<<"a">>, <<"b,c\n">>], apply_func(split, [<<"a,b,c\n">>, <<",">>, <<"leading_notrim">>])
),
?assertEqual(
[<<"a b">>, <<"c\r\n">>],
apply_func(split, [<<"a b,c\r\n">>, <<",">>, <<"leading_notrim">>])
),
?assertEqual(
[<<"哈哈"/utf8>>, <<" 你好, 是的\r\n"/utf8>>],
apply_func(split, [<<"哈哈, 你好, 是的\r\n"/utf8>>, <<",">>, <<"leading_notrim">>])
).
t_split_trailing(_) -> t_split_trailing(_) ->
?assertEqual([], apply_func(split, [<<>>, <<"/">>, <<"trailing">>])), ?assertEqual([], apply_func(split, [<<>>, <<"/">>, <<"trailing">>])),
?assertEqual([], apply_func(split, [<<"/">>, <<"/">>, <<"trailing">>])), ?assertEqual([], apply_func(split, [<<"/">>, <<"/">>, <<"trailing">>])),
?assertEqual([<<"/a/b">>, <<"c">>], apply_func(split, [<<"/a/b/c">>, <<"/">>, <<"trailing">>])), ?assertEqual([<<"/a/b">>, <<"c">>], apply_func(split, [<<"/a/b/c">>, <<"/">>, <<"trailing">>])),
?assertEqual([<<"a/b//c">>], apply_func(split, [<<"a/b//c/">>, <<"/">>, <<"trailing">>])), ?assertEqual([<<"a/b//c">>], apply_func(split, [<<"a/b//c/">>, <<"/">>, <<"trailing">>])),
?assertEqual([<<"a,b">>,<<"c\n">>], apply_func(split, [<<"a,b,c\n">>, <<",">>, <<"trailing">>])), ?assertEqual(
?assertEqual([<<"a b">>,<<"c\r\n">>], apply_func(split, [<<"a b,c\r\n">>, <<",">>, <<"trailing">>])), [<<"a,b">>, <<"c\n">>], apply_func(split, [<<"a,b,c\n">>, <<",">>, <<"trailing">>])
?assertEqual([<<"哈哈, 你好"/utf8>>,<<" 是的\r\n"/utf8>>], apply_func(split, [<<"哈哈, 你好, 是的\r\n"/utf8>>, <<",">>, <<"trailing">>])). ),
?assertEqual(
[<<"a b">>, <<"c\r\n">>], apply_func(split, [<<"a b,c\r\n">>, <<",">>, <<"trailing">>])
),
?assertEqual(
[<<"哈哈, 你好"/utf8>>, <<" 是的\r\n"/utf8>>],
apply_func(split, [<<"哈哈, 你好, 是的\r\n"/utf8>>, <<",">>, <<"trailing">>])
).
t_split_trailing_notrim(_) -> t_split_trailing_notrim(_) ->
?assertEqual([<<>>], apply_func(split, [<<>>, <<"/">>, <<"trailing_notrim">>])), ?assertEqual([<<>>], apply_func(split, [<<>>, <<"/">>, <<"trailing_notrim">>])),
?assertEqual([<<>>, <<>>], apply_func(split, [<<"/">>, <<"/">>, <<"trailing_notrim">>])), ?assertEqual([<<>>, <<>>], apply_func(split, [<<"/">>, <<"/">>, <<"trailing_notrim">>])),
?assertEqual([<<"/a/b">>, <<"c">>], apply_func(split, [<<"/a/b/c">>, <<"/">>, <<"trailing_notrim">>])), ?assertEqual(
?assertEqual([<<"a/b//c">>, <<>>], apply_func(split, [<<"a/b//c/">>, <<"/">>, <<"trailing_notrim">>])), [<<"/a/b">>, <<"c">>], apply_func(split, [<<"/a/b/c">>, <<"/">>, <<"trailing_notrim">>])
?assertEqual([<<"a,b">>,<<"c\n">>], apply_func(split, [<<"a,b,c\n">>, <<",">>, <<"trailing_notrim">>])), ),
?assertEqual([<<"a b">>,<<"c\r\n">>], apply_func(split, [<<"a b,c\r\n">>, <<",">>, <<"trailing_notrim">>])), ?assertEqual(
?assertEqual([<<"哈哈, 你好"/utf8>>,<<" 是的\r\n"/utf8>>], apply_func(split, [<<"哈哈, 你好, 是的\r\n"/utf8>>, <<",">>, <<"trailing_notrim">>])). [<<"a/b//c">>, <<>>], apply_func(split, [<<"a/b//c/">>, <<"/">>, <<"trailing_notrim">>])
),
?assertEqual(
[<<"a,b">>, <<"c\n">>], apply_func(split, [<<"a,b,c\n">>, <<",">>, <<"trailing_notrim">>])
),
?assertEqual(
[<<"a b">>, <<"c\r\n">>],
apply_func(split, [<<"a b,c\r\n">>, <<",">>, <<"trailing_notrim">>])
),
?assertEqual(
[<<"哈哈, 你好"/utf8>>, <<" 是的\r\n"/utf8>>],
apply_func(split, [<<"哈哈, 你好, 是的\r\n"/utf8>>, <<",">>, <<"trailing_notrim">>])
).
t_tokens(_) -> t_tokens(_) ->
?assertEqual([], apply_func(tokens, [<<>>, <<"/">>])), ?assertEqual([], apply_func(tokens, [<<>>, <<"/">>])),
?assertEqual([], apply_func(tokens, [<<"/">>, <<"/">>])), ?assertEqual([], apply_func(tokens, [<<"/">>, <<"/">>])),
?assertEqual([<<"a">>,<<"b">>,<<"c">>], apply_func(tokens, [<<"/a/b/c">>, <<"/">>])), ?assertEqual([<<"a">>, <<"b">>, <<"c">>], apply_func(tokens, [<<"/a/b/c">>, <<"/">>])),
?assertEqual([<<"a">>,<<"b">>,<<"c">>], apply_func(tokens, [<<"/a/b//c/">>, <<"/">>])), ?assertEqual([<<"a">>, <<"b">>, <<"c">>], apply_func(tokens, [<<"/a/b//c/">>, <<"/">>])),
?assertEqual([<<"a">>,<<"b">>,<<"c">>], apply_func(tokens, [<<" /a/ b /c">>, <<" /">>])), ?assertEqual([<<"a">>, <<"b">>, <<"c">>], apply_func(tokens, [<<" /a/ b /c">>, <<" /">>])),
?assertEqual([<<"a">>,<<"\nb">>,<<"c\n">>], apply_func(tokens, [<<"a ,\nb,c\n">>, <<", ">>])), ?assertEqual([<<"a">>, <<"\nb">>, <<"c\n">>], apply_func(tokens, [<<"a ,\nb,c\n">>, <<", ">>])),
?assertEqual([<<"a">>,<<"b">>,<<"c\r\n">>], apply_func(tokens, [<<"a ,b,c\r\n">>, <<", ">>])), ?assertEqual([<<"a">>, <<"b">>, <<"c\r\n">>], apply_func(tokens, [<<"a ,b,c\r\n">>, <<", ">>])),
?assertEqual([<<"a">>,<<"b">>,<<"c">>], apply_func(tokens, [<<"a,b, c\n">>, <<", ">>, <<"nocrlf">>])), ?assertEqual(
?assertEqual([<<"a">>,<<"b">>,<<"c">>], apply_func(tokens, [<<"a,b,c\r\n">>, <<",">>, <<"nocrlf">>])), [<<"a">>, <<"b">>, <<"c">>], apply_func(tokens, [<<"a,b, c\n">>, <<", ">>, <<"nocrlf">>])
?assertEqual([<<"a">>,<<"b">>,<<"c">>], apply_func(tokens, [<<"a,b\r\n,c\n">>, <<",">>, <<"nocrlf">>])), ),
?assertEqual(
[<<"a">>, <<"b">>, <<"c">>], apply_func(tokens, [<<"a,b,c\r\n">>, <<",">>, <<"nocrlf">>])
),
?assertEqual(
[<<"a">>, <<"b">>, <<"c">>], apply_func(tokens, [<<"a,b\r\n,c\n">>, <<",">>, <<"nocrlf">>])
),
?assertEqual([], apply_func(tokens, [<<"\r\n">>, <<",">>, <<"nocrlf">>])), ?assertEqual([], apply_func(tokens, [<<"\r\n">>, <<",">>, <<"nocrlf">>])),
?assertEqual([], apply_func(tokens, [<<"\r\n">>, <<",">>, <<"nocrlf">>])), ?assertEqual([], apply_func(tokens, [<<"\r\n">>, <<",">>, <<"nocrlf">>])),
?assertEqual([<<"哈哈"/utf8>>,<<"你好"/utf8>>,<<"是的"/utf8>>], apply_func(tokens, [<<"哈哈, 你好, 是的\r\n"/utf8>>, <<", ">>, <<"nocrlf">>])). ?assertEqual(
[<<"哈哈"/utf8>>, <<"你好"/utf8>>, <<"是的"/utf8>>],
apply_func(tokens, [<<"哈哈, 你好, 是的\r\n"/utf8>>, <<", ">>, <<"nocrlf">>])
).
t_concat(_) -> t_concat(_) ->
?assertEqual(<<"ab">>, apply_func(concat, [<<"a">>, <<"b">>])), ?assertEqual(<<"ab">>, apply_func(concat, [<<"a">>, <<"b">>])),
?assertEqual(<<"ab">>, apply_func('+', [<<"a">>, <<"b">>])), ?assertEqual(<<"ab">>, apply_func('+', [<<"a">>, <<"b">>])),
?assertEqual(<<"哈哈你好"/utf8>>, apply_func(concat, [<<"哈哈"/utf8>>,<<"你好"/utf8>>])), ?assertEqual(<<"哈哈你好"/utf8>>, apply_func(concat, [<<"哈哈"/utf8>>, <<"你好"/utf8>>])),
?assertEqual(<<"abc">>, apply_func(concat, [apply_func(concat, [<<"a">>, <<"b">>]), <<"c">>])), ?assertEqual(<<"abc">>, apply_func(concat, [apply_func(concat, [<<"a">>, <<"b">>]), <<"c">>])),
?assertEqual(<<"a">>, apply_func(concat, [<<"">>, <<"a">>])), ?assertEqual(<<"a">>, apply_func(concat, [<<"">>, <<"a">>])),
?assertEqual(<<"a">>, apply_func(concat, [<<"a">>, <<"">>])), ?assertEqual(<<"a">>, apply_func(concat, [<<"a">>, <<"">>])),
@ -424,8 +561,13 @@ t_concat(_) ->
t_sprintf(_) -> t_sprintf(_) ->
?assertEqual(<<"Hello Shawn!">>, apply_func(sprintf, [<<"Hello ~ts!">>, <<"Shawn">>])), ?assertEqual(<<"Hello Shawn!">>, apply_func(sprintf, [<<"Hello ~ts!">>, <<"Shawn">>])),
?assertEqual(<<"Name: ABC, Count: 2">>, apply_func(sprintf, [<<"Name: ~ts, Count: ~p">>, <<"ABC">>, 2])), ?assertEqual(
?assertEqual(<<"Name: ABC, Count: 2, Status: {ok,running}">>, apply_func(sprintf, [<<"Name: ~ts, Count: ~p, Status: ~p">>, <<"ABC">>, 2, {ok, running}])). <<"Name: ABC, Count: 2">>, apply_func(sprintf, [<<"Name: ~ts, Count: ~p">>, <<"ABC">>, 2])
),
?assertEqual(
<<"Name: ABC, Count: 2, Status: {ok,running}">>,
apply_func(sprintf, [<<"Name: ~ts, Count: ~p, Status: ~p">>, <<"ABC">>, 2, {ok, running}])
).
t_pad(_) -> t_pad(_) ->
?assertEqual(<<"abc ">>, apply_func(pad, [<<"abc">>, 5])), ?assertEqual(<<"abc ">>, apply_func(pad, [<<"abc">>, 5])),
@ -449,8 +591,12 @@ t_replace(_) ->
?assertEqual(<<"ab-c--">>, apply_func(replace, [<<"ab c ">>, <<" ">>, <<"-">>])), ?assertEqual(<<"ab-c--">>, apply_func(replace, [<<"ab c ">>, <<" ">>, <<"-">>])),
?assertEqual(<<"ab::c::::">>, apply_func(replace, [<<"ab c ">>, <<" ">>, <<"::">>])), ?assertEqual(<<"ab::c::::">>, apply_func(replace, [<<"ab c ">>, <<" ">>, <<"::">>])),
?assertEqual(<<"ab-c--">>, apply_func(replace, [<<"ab c ">>, <<" ">>, <<"-">>, <<"all">>])), ?assertEqual(<<"ab-c--">>, apply_func(replace, [<<"ab c ">>, <<" ">>, <<"-">>, <<"all">>])),
?assertEqual(<<"ab-c ">>, apply_func(replace, [<<"ab c ">>, <<" ">>, <<"-">>, <<"leading">>])), ?assertEqual(
?assertEqual(<<"ab c -">>, apply_func(replace, [<<"ab c ">>, <<" ">>, <<"-">>, <<"trailing">>])). <<"ab-c ">>, apply_func(replace, [<<"ab c ">>, <<" ">>, <<"-">>, <<"leading">>])
),
?assertEqual(
<<"ab c -">>, apply_func(replace, [<<"ab c ">>, <<" ">>, <<"-">>, <<"trailing">>])
).
t_ascii(_) -> t_ascii(_) ->
?assertEqual(97, apply_func(ascii, [<<"a">>])), ?assertEqual(97, apply_func(ascii, [<<"a">>])),
@ -483,7 +629,7 @@ t_regex_replace(_) ->
?assertEqual(<<"aebed">>, apply_func(regex_replace, [<<"accbcd">>, <<"c+">>, <<"e">>])), ?assertEqual(<<"aebed">>, apply_func(regex_replace, [<<"accbcd">>, <<"c+">>, <<"e">>])),
?assertEqual(<<"a[cc]b[c]d">>, apply_func(regex_replace, [<<"accbcd">>, <<"c+">>, <<"[&]">>])). ?assertEqual(<<"a[cc]b[c]d">>, apply_func(regex_replace, [<<"accbcd">>, <<"c+">>, <<"[&]">>])).
ascii_string() -> list(range(0,127)). ascii_string() -> list(range(0, 127)).
bin(S) -> iolist_to_binary(S). bin(S) -> iolist_to_binary(S).
@ -492,34 +638,34 @@ bin(S) -> iolist_to_binary(S).
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
t_nth(_) -> t_nth(_) ->
?assertEqual(2, apply_func(nth, [2, [1,2,3,4]])), ?assertEqual(2, apply_func(nth, [2, [1, 2, 3, 4]])),
?assertEqual(4, apply_func(nth, [4, [1,2,3,4]])). ?assertEqual(4, apply_func(nth, [4, [1, 2, 3, 4]])).
t_length(_) -> t_length(_) ->
?assertEqual(4, apply_func(length, [[1,2,3,4]])), ?assertEqual(4, apply_func(length, [[1, 2, 3, 4]])),
?assertEqual(0, apply_func(length, [[]])). ?assertEqual(0, apply_func(length, [[]])).
t_slice(_) -> t_slice(_) ->
?assertEqual([1,2,3,4], apply_func(sublist, [4, [1,2,3,4]])), ?assertEqual([1, 2, 3, 4], apply_func(sublist, [4, [1, 2, 3, 4]])),
?assertEqual([1,2], apply_func(sublist, [2, [1,2,3,4]])), ?assertEqual([1, 2], apply_func(sublist, [2, [1, 2, 3, 4]])),
?assertEqual([4], apply_func(sublist, [4, 1, [1,2,3,4]])), ?assertEqual([4], apply_func(sublist, [4, 1, [1, 2, 3, 4]])),
?assertEqual([4], apply_func(sublist, [4, 2, [1,2,3,4]])), ?assertEqual([4], apply_func(sublist, [4, 2, [1, 2, 3, 4]])),
?assertEqual([], apply_func(sublist, [5, 2, [1,2,3,4]])), ?assertEqual([], apply_func(sublist, [5, 2, [1, 2, 3, 4]])),
?assertEqual([2,3], apply_func(sublist, [2, 2, [1,2,3,4]])), ?assertEqual([2, 3], apply_func(sublist, [2, 2, [1, 2, 3, 4]])),
?assertEqual([1], apply_func(sublist, [1, 1, [1,2,3,4]])). ?assertEqual([1], apply_func(sublist, [1, 1, [1, 2, 3, 4]])).
t_first_last(_) -> t_first_last(_) ->
?assertEqual(1, apply_func(first, [[1,2,3,4]])), ?assertEqual(1, apply_func(first, [[1, 2, 3, 4]])),
?assertEqual(4, apply_func(last, [[1,2,3,4]])). ?assertEqual(4, apply_func(last, [[1, 2, 3, 4]])).
t_contains(_) -> t_contains(_) ->
?assertEqual(true, apply_func(contains, [1, [1,2,3,4]])), ?assertEqual(true, apply_func(contains, [1, [1, 2, 3, 4]])),
?assertEqual(true, apply_func(contains, [3, [1,2,3,4]])), ?assertEqual(true, apply_func(contains, [3, [1, 2, 3, 4]])),
?assertEqual(true, apply_func(contains, [<<"a">>, [<<>>,<<"ab">>,3,<<"a">>]])), ?assertEqual(true, apply_func(contains, [<<"a">>, [<<>>, <<"ab">>, 3, <<"a">>]])),
?assertEqual(true, apply_func(contains, [#{a=>b}, [#{a=>1}, #{a=>b}]])), ?assertEqual(true, apply_func(contains, [#{a => b}, [#{a => 1}, #{a => b}]])),
?assertEqual(false, apply_func(contains, [#{a=>b}, [#{a=>1}]])), ?assertEqual(false, apply_func(contains, [#{a => b}, [#{a => 1}]])),
?assertEqual(false, apply_func(contains, [3, [1, 2]])), ?assertEqual(false, apply_func(contains, [3, [1, 2]])),
?assertEqual(false, apply_func(contains, [<<"c">>, [<<>>,<<"ab">>,3,<<"a">>]])). ?assertEqual(false, apply_func(contains, [<<"c">>, [<<>>, <<"ab">>, 3, <<"a">>]])).
t_map_get(_) -> t_map_get(_) ->
?assertEqual(1, apply_func(map_get, [<<"a">>, #{a => 1}])), ?assertEqual(1, apply_func(map_get, [<<"a">>, #{a => 1}])),
@ -532,7 +678,9 @@ t_map_put(_) ->
?assertEqual(#{<<"a">> => 1}, apply_func(map_put, [<<"a">>, 1, #{}])), ?assertEqual(#{<<"a">> => 1}, apply_func(map_put, [<<"a">>, 1, #{}])),
?assertEqual(#{a => 2}, apply_func(map_put, [<<"a">>, 2, #{a => 1}])), ?assertEqual(#{a => 2}, apply_func(map_put, [<<"a">>, 2, #{a => 1}])),
?assertEqual(#{<<"a">> => #{<<"b">> => 1}}, apply_func(map_put, [<<"a.b">>, 1, #{}])), ?assertEqual(#{<<"a">> => #{<<"b">> => 1}}, apply_func(map_put, [<<"a.b">>, 1, #{}])),
?assertEqual(#{a => #{b => 1, <<"c">> => 1}}, apply_func(map_put, [<<"a.c">>, 1, #{a => #{b => 1}}])), ?assertEqual(
#{a => #{b => 1, <<"c">> => 1}}, apply_func(map_put, [<<"a.c">>, 1, #{a => #{b => 1}}])
),
?assertEqual(#{a => 2}, apply_func(map_put, [<<"a">>, 2, #{a => 1}])). ?assertEqual(#{a => 2}, apply_func(map_put, [<<"a">>, 2, #{a => 1}])).
t_mget(_) -> t_mget(_) ->
@ -579,20 +727,26 @@ t_subbits2_1(_) ->
?assertEqual(127, apply_func(subbits, [<<255:8>>, 2, 7])), ?assertEqual(127, apply_func(subbits, [<<255:8>>, 2, 7])),
?assertEqual(127, apply_func(subbits, [<<255:8>>, 2, 8])). ?assertEqual(127, apply_func(subbits, [<<255:8>>, 2, 8])).
t_subbits2_integer(_) -> t_subbits2_integer(_) ->
?assertEqual(456, apply_func(subbits, [<<456:32/integer>>, 1, 32, <<"integer">>, <<"signed">>, <<"big">>])), ?assertEqual(
?assertEqual(-456, apply_func(subbits, [<<-456:32/integer>>, 1, 32, <<"integer">>, <<"signed">>, <<"big">>])). 456,
apply_func(subbits, [<<456:32/integer>>, 1, 32, <<"integer">>, <<"signed">>, <<"big">>])
),
?assertEqual(
-456,
apply_func(subbits, [<<-456:32/integer>>, 1, 32, <<"integer">>, <<"signed">>, <<"big">>])
).
t_subbits2_float(_) -> t_subbits2_float(_) ->
R = apply_func(subbits, [<<5.3:64/float>>, 1, 64, <<"float">>, <<"unsigned">>, <<"big">>]), R = apply_func(subbits, [<<5.3:64/float>>, 1, 64, <<"float">>, <<"unsigned">>, <<"big">>]),
RL = (5.3 - R), RL = (5.3 - R),
ct:pal(";;;;~p", [R]), ct:pal(";;;;~p", [R]),
?assert( (RL >= 0 andalso RL < 0.0001) orelse (RL =< 0 andalso RL > -0.0001)), ?assert((RL >= 0 andalso RL < 0.0001) orelse (RL =< 0 andalso RL > -0.0001)),
R2 = apply_func(subbits, [<<-5.3:64/float>>, 1, 64, <<"float">>, <<"signed">>, <<"big">>]), R2 = apply_func(subbits, [<<-5.3:64/float>>, 1, 64, <<"float">>, <<"signed">>, <<"big">>]),
RL2 = (5.3 + R2), RL2 = (5.3 + R2),
ct:pal(";;;;~p", [R2]), ct:pal(";;;;~p", [R2]),
?assert( (RL2 >= 0 andalso RL2 < 0.0001) orelse (RL2 =< 0 andalso RL2 > -0.0001)). ?assert((RL2 >= 0 andalso RL2 < 0.0001) orelse (RL2 =< 0 andalso RL2 > -0.0001)).
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% Test cases for Hash funcs %% Test cases for Hash funcs
@ -602,12 +756,15 @@ t_hash_funcs(_) ->
?PROPTEST(prop_hash_fun). ?PROPTEST(prop_hash_fun).
prop_hash_fun() -> prop_hash_fun() ->
?FORALL(S, binary(), ?FORALL(
begin S,
(32 == byte_size(apply_func(md5, [S]))) andalso binary(),
begin
(32 == byte_size(apply_func(md5, [S]))) andalso
(40 == byte_size(apply_func(sha, [S]))) andalso (40 == byte_size(apply_func(sha, [S]))) andalso
(64 == byte_size(apply_func(sha256, [S]))) (64 == byte_size(apply_func(sha256, [S])))
end). end
).
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% Test cases for base64 %% Test cases for base64
@ -617,72 +774,131 @@ t_base64_encode(_) ->
?PROPTEST(prop_base64_encode). ?PROPTEST(prop_base64_encode).
prop_base64_encode() -> prop_base64_encode() ->
?FORALL(S, list(range(0, 255)), ?FORALL(
begin S,
Bin = iolist_to_binary(S), list(range(0, 255)),
Bin == base64:decode(apply_func(base64_encode, [Bin])) begin
end). Bin = iolist_to_binary(S),
Bin == base64:decode(apply_func(base64_encode, [Bin]))
end
).
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% Date functions %% Date functions
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
t_now_rfc3339(_) -> t_now_rfc3339(_) ->
?assert(is_integer( ?assert(
calendar:rfc3339_to_system_time( is_integer(
binary_to_list(apply_func(now_rfc3339, []))))). calendar:rfc3339_to_system_time(
binary_to_list(apply_func(now_rfc3339, []))
)
)
).
t_now_rfc3339_1(_) -> t_now_rfc3339_1(_) ->
[?assert(is_integer( [
calendar:rfc3339_to_system_time( ?assert(
binary_to_list(apply_func(now_rfc3339, [atom_to_binary(Unit, utf8)])), is_integer(
[{unit, Unit}]))) calendar:rfc3339_to_system_time(
|| Unit <- [second,millisecond,microsecond,nanosecond]]. binary_to_list(apply_func(now_rfc3339, [atom_to_binary(Unit, utf8)])),
[{unit, Unit}]
)
)
)
|| Unit <- [second, millisecond, microsecond, nanosecond]
].
t_now_timestamp(_) -> t_now_timestamp(_) ->
?assert(is_integer(apply_func(now_timestamp, []))). ?assert(is_integer(apply_func(now_timestamp, []))).
t_now_timestamp_1(_) -> t_now_timestamp_1(_) ->
[?assert(is_integer( [
apply_func(now_timestamp, [atom_to_binary(Unit, utf8)]))) ?assert(
|| Unit <- [second,millisecond,microsecond,nanosecond]]. is_integer(
apply_func(now_timestamp, [atom_to_binary(Unit, utf8)])
)
)
|| Unit <- [second, millisecond, microsecond, nanosecond]
].
t_unix_ts_to_rfc3339(_) -> t_unix_ts_to_rfc3339(_) ->
[begin [
BUnit = atom_to_binary(Unit, utf8), begin
Epoch = apply_func(now_timestamp, [BUnit]), BUnit = atom_to_binary(Unit, utf8),
DateTime = apply_func(unix_ts_to_rfc3339, [Epoch, BUnit]), Epoch = apply_func(now_timestamp, [BUnit]),
?assertEqual(Epoch, DateTime = apply_func(unix_ts_to_rfc3339, [Epoch, BUnit]),
calendar:rfc3339_to_system_time(binary_to_list(DateTime), [{unit, Unit}])) ?assertEqual(
end || Unit <- [second,millisecond,microsecond,nanosecond]]. Epoch,
calendar:rfc3339_to_system_time(binary_to_list(DateTime), [{unit, Unit}])
)
end
|| Unit <- [second, millisecond, microsecond, nanosecond]
].
t_rfc3339_to_unix_ts(_) -> t_rfc3339_to_unix_ts(_) ->
[begin [
BUnit = atom_to_binary(Unit, utf8), begin
Epoch = apply_func(now_timestamp, [BUnit]), BUnit = atom_to_binary(Unit, utf8),
DateTime = apply_func(unix_ts_to_rfc3339, [Epoch, BUnit]), Epoch = apply_func(now_timestamp, [BUnit]),
?assertEqual(Epoch, emqx_rule_funcs:rfc3339_to_unix_ts(DateTime, BUnit)) DateTime = apply_func(unix_ts_to_rfc3339, [Epoch, BUnit]),
end || Unit <- [second,millisecond,microsecond,nanosecond]]. ?assertEqual(Epoch, emqx_rule_funcs:rfc3339_to_unix_ts(DateTime, BUnit))
end
|| Unit <- [second, millisecond, microsecond, nanosecond]
].
t_format_date_funcs(_) -> t_format_date_funcs(_) ->
?PROPTEST(prop_format_date_fun). ?PROPTEST(prop_format_date_fun).
prop_format_date_fun() -> prop_format_date_fun() ->
Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%y---%H:%M:%S%Z">>], Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%y---%H:%M:%S%Z">>],
?FORALL(S, erlang:system_time(second), ?FORALL(
S == apply_func(date_to_unix_ts, S,
Args1 ++ [apply_func(format_date, erlang:system_time(second),
Args1 ++ [S])])), S ==
apply_func(
date_to_unix_ts,
Args1 ++
[
apply_func(
format_date,
Args1 ++ [S]
)
]
)
),
Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%y---%H:%M:%S%Z">>], Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%y---%H:%M:%S%Z">>],
?FORALL(S, erlang:system_time(millisecond), ?FORALL(
S == apply_func(date_to_unix_ts, S,
Args2 ++ [apply_func(format_date, erlang:system_time(millisecond),
Args2 ++ [S])])), S ==
apply_func(
date_to_unix_ts,
Args2 ++
[
apply_func(
format_date,
Args2 ++ [S]
)
]
)
),
Args = [<<"second">>, <<"+08:00">>, <<"%y-%m-%d-%H:%M:%S%Z">>], Args = [<<"second">>, <<"+08:00">>, <<"%y-%m-%d-%H:%M:%S%Z">>],
?FORALL(S, erlang:system_time(second), ?FORALL(
S == apply_func(date_to_unix_ts, S,
Args ++ [apply_func(format_date, erlang:system_time(second),
Args ++ [S])])). S ==
apply_func(
date_to_unix_ts,
Args ++
[
apply_func(
format_date,
Args ++ [S]
)
]
)
).
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% Utility functions %% Utility functions
@ -699,8 +915,10 @@ apply_func(Name, Args, Msg) ->
apply_func(Name, Args, emqx_message:to_map(Msg)). apply_func(Name, Args, emqx_message:to_map(Msg)).
message() -> message() ->
emqx_message:set_flags(#{dup => false}, emqx_message:set_flags(
emqx_message:make(<<"clientid">>, 1, <<"topic/#">>, <<"payload">>)). #{dup => false},
emqx_message:make(<<"clientid">>, 1, <<"topic/#">>, <<"payload">>)
).
% t_contains_topic(_) -> % t_contains_topic(_) ->
% error('TODO'). % error('TODO').
@ -831,13 +1049,15 @@ message() ->
% t_json_decode(_) -> % t_json_decode(_) ->
% error('TODO'). % error('TODO').
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% CT functions %% CT functions
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
all() -> all() ->
IsTestCase = fun("t_" ++ _) -> true; (_) -> false end, IsTestCase = fun
("t_" ++ _) -> true;
(_) -> false
end,
[F || {F, _A} <- module_info(exports), IsTestCase(atom_to_list(F))]. [F || {F, _A} <- module_info(exports), IsTestCase(atom_to_list(F))].
suite() -> suite() ->

279
apps/emqx_rule_engine/test/emqx_rule_maps_SUITE.erl
View File

@ -22,20 +22,27 @@
-compile(export_all). -compile(export_all).
-compile(nowarn_export_all). -compile(nowarn_export_all).
-import(emqx_rule_maps, -import(
[ nested_get/2 emqx_rule_maps,
, nested_get/3 [
, nested_put/3 nested_get/2,
, atom_key_map/1 nested_get/3,
]). nested_put/3,
atom_key_map/1
]
).
-define(path(Path), {path, -define(path(Path),
[case K of {path, [
{ic, Key} -> {index, {const, Key}}; case K of
{iv, Key} -> {index, {var, Key}}; {ic, Key} -> {index, {const, Key}};
{i, Path1} -> {index, Path1}; {iv, Key} -> {index, {var, Key}};
_ -> {key, K} {i, Path1} -> {index, Path1};
end || K <- Path]}). _ -> {key, K}
end
|| K <- Path
]}
).
-define(PROPTEST(Prop), true = proper:quickcheck(Prop)). -define(PROPTEST(Prop), true = proper:quickcheck(Prop)).
@ -44,8 +51,8 @@ t_nested_put_map(_) ->
?assertEqual(#{a => a}, nested_put(?path([a]), a, #{})), ?assertEqual(#{a => a}, nested_put(?path([a]), a, #{})),
?assertEqual(#{a => undefined}, nested_put(?path([a]), undefined, #{})), ?assertEqual(#{a => undefined}, nested_put(?path([a]), undefined, #{})),
?assertEqual(#{a => 1}, nested_put(?path([a]), 1, not_map)), ?assertEqual(#{a => 1}, nested_put(?path([a]), 1, not_map)),
?assertEqual(#{a => #{b => b}}, nested_put(?path([a,b]), b, #{})), ?assertEqual(#{a => #{b => b}}, nested_put(?path([a, b]), b, #{})),
?assertEqual(#{a => #{b => #{c => c}}}, nested_put(?path([a,b,c]), c, #{})), ?assertEqual(#{a => #{b => #{c => c}}}, nested_put(?path([a, b, c]), c, #{})),
?assertEqual(#{<<"k">> => v1}, nested_put(?path([k]), v1, #{<<"k">> => v0})), ?assertEqual(#{<<"k">> => v1}, nested_put(?path([k]), v1, #{<<"k">> => v0})),
?assertEqual(#{k => v1}, nested_put(?path([k]), v1, #{k => v0})), ?assertEqual(#{k => v1}, nested_put(?path([k]), v1, #{k => v0})),
?assertEqual(#{<<"k">> => v1, a => b}, nested_put(?path([k]), v1, #{<<"k">> => v0, a => b})), ?assertEqual(#{<<"k">> => v1, a => b}, nested_put(?path([k]), v1, #{<<"k">> => v0, a => b})),
@ -53,122 +60,194 @@ t_nested_put_map(_) ->
?assertEqual(#{k => v1}, nested_put(?path([k]), v1, #{k => v0})), ?assertEqual(#{k => v1}, nested_put(?path([k]), v1, #{k => v0})),
?assertEqual(#{k => v1, a => b}, nested_put(?path([k]), v1, #{k => v0, a => b})), ?assertEqual(#{k => v1, a => b}, nested_put(?path([k]), v1, #{k => v0, a => b})),
?assertEqual(#{<<"k">> => v1, a => b}, nested_put(?path([k]), v1, #{<<"k">> => v0, a => b})), ?assertEqual(#{<<"k">> => v1, a => b}, nested_put(?path([k]), v1, #{<<"k">> => v0, a => b})),
?assertEqual(#{<<"k">> => #{<<"t">> => v1}}, nested_put(?path([k,t]), v1, #{<<"k">> => #{<<"t">> => v0}})), ?assertEqual(
?assertEqual(#{<<"k">> => #{t => v1}}, nested_put(?path([k,t]), v1, #{<<"k">> => #{t => v0}})), #{<<"k">> => #{<<"t">> => v1}},
?assertEqual(#{k => #{<<"t">> => #{a => v1}}}, nested_put(?path([k,t,a]), v1, #{k => #{<<"t">> => v0}})), nested_put(?path([k, t]), v1, #{<<"k">> => #{<<"t">> => v0}})
?assertEqual(#{k => #{<<"t">> => #{<<"a">> => v1}}}, nested_put(?path([k,t,<<"a">>]), v1, #{k => #{<<"t">> => v0}})). ),
?assertEqual(#{<<"k">> => #{t => v1}}, nested_put(?path([k, t]), v1, #{<<"k">> => #{t => v0}})),
?assertEqual(
#{k => #{<<"t">> => #{a => v1}}}, nested_put(?path([k, t, a]), v1, #{k => #{<<"t">> => v0}})
),
?assertEqual(
#{k => #{<<"t">> => #{<<"a">> => v1}}},
nested_put(?path([k, t, <<"a">>]), v1, #{k => #{<<"t">> => v0}})
).
t_nested_put_index(_) -> t_nested_put_index(_) ->
?assertEqual([1,a,3], nested_put(?path([{ic,2}]), a, [1,2,3])), ?assertEqual([1, a, 3], nested_put(?path([{ic, 2}]), a, [1, 2, 3])),
?assertEqual([1,2,3], nested_put(?path([{ic,0}]), a, [1,2,3])), ?assertEqual([1, 2, 3], nested_put(?path([{ic, 0}]), a, [1, 2, 3])),
?assertEqual([1,2,3], nested_put(?path([{ic,4}]), a, [1,2,3])), ?assertEqual([1, 2, 3], nested_put(?path([{ic, 4}]), a, [1, 2, 3])),
?assertEqual([1,[a],3], nested_put(?path([{ic,2}, {ic,1}]), a, [1,[2],3])), ?assertEqual([1, [a], 3], nested_put(?path([{ic, 2}, {ic, 1}]), a, [1, [2], 3])),
?assertEqual([1,[[a]],3], nested_put(?path([{ic,2}, {ic,1}, {ic,1}]), a, [1,[[2]],3])), ?assertEqual([1, [[a]], 3], nested_put(?path([{ic, 2}, {ic, 1}, {ic, 1}]), a, [1, [[2]], 3])),
?assertEqual([1,[[2]],3], nested_put(?path([{ic,2}, {ic,1}, {ic,2}]), a, [1,[[2]],3])), ?assertEqual([1, [[2]], 3], nested_put(?path([{ic, 2}, {ic, 1}, {ic, 2}]), a, [1, [[2]], 3])),
?assertEqual([1,[a],1], nested_put(?path([{ic,2}, {i,?path([{ic,3}])}]), a, [1,[2],1])), ?assertEqual([1, [a], 1], nested_put(?path([{ic, 2}, {i, ?path([{ic, 3}])}]), a, [1, [2], 1])),
%% nested_put to the first or tail of a list: %% nested_put to the first or tail of a list:
?assertEqual([a], nested_put(?path([{ic,head}]), a, not_list)), ?assertEqual([a], nested_put(?path([{ic, head}]), a, not_list)),
?assertEqual([a], nested_put(?path([{ic,head}]), a, [])), ?assertEqual([a], nested_put(?path([{ic, head}]), a, [])),
?assertEqual([a,1,2,3], nested_put(?path([{ic,head}]), a, [1,2,3])), ?assertEqual([a, 1, 2, 3], nested_put(?path([{ic, head}]), a, [1, 2, 3])),
?assertEqual([a], nested_put(?path([{ic,tail}]), a, not_list)), ?assertEqual([a], nested_put(?path([{ic, tail}]), a, not_list)),
?assertEqual([a], nested_put(?path([{ic,tail}]), a, [])), ?assertEqual([a], nested_put(?path([{ic, tail}]), a, [])),
?assertEqual([1,2,3,a], nested_put(?path([{ic,tail}]), a, [1,2,3])). ?assertEqual([1, 2, 3, a], nested_put(?path([{ic, tail}]), a, [1, 2, 3])).
t_nested_put_negative_index(_) -> t_nested_put_negative_index(_) ->
?assertEqual([1,2,a], nested_put(?path([{ic,-1}]), a, [1,2,3])), ?assertEqual([1, 2, a], nested_put(?path([{ic, -1}]), a, [1, 2, 3])),
?assertEqual([1,a,3], nested_put(?path([{ic,-2}]), a, [1,2,3])), ?assertEqual([1, a, 3], nested_put(?path([{ic, -2}]), a, [1, 2, 3])),
?assertEqual([a,2,3], nested_put(?path([{ic,-3}]), a, [1,2,3])), ?assertEqual([a, 2, 3], nested_put(?path([{ic, -3}]), a, [1, 2, 3])),
?assertEqual([1,2,3], nested_put(?path([{ic,-4}]), a, [1,2,3])). ?assertEqual([1, 2, 3], nested_put(?path([{ic, -4}]), a, [1, 2, 3])).
t_nested_put_mix_map_index(_) -> t_nested_put_mix_map_index(_) ->
?assertEqual(#{a => [a]}, nested_put(?path([a, {ic,2}]), a, #{})), ?assertEqual(#{a => [a]}, nested_put(?path([a, {ic, 2}]), a, #{})),
?assertEqual(#{a => [#{b => 0}]}, nested_put(?path([a, {ic,2}, b]), 0, #{})), ?assertEqual(#{a => [#{b => 0}]}, nested_put(?path([a, {ic, 2}, b]), 0, #{})),
?assertEqual(#{a => [1,a,3]}, nested_put(?path([a, {ic,2}]), a, #{a => [1,2,3]})), ?assertEqual(#{a => [1, a, 3]}, nested_put(?path([a, {ic, 2}]), a, #{a => [1, 2, 3]})),
?assertEqual([1,#{a => c},3], nested_put(?path([{ic,2}, a]), c, [1,#{a => b},3])), ?assertEqual([1, #{a => c}, 3], nested_put(?path([{ic, 2}, a]), c, [1, #{a => b}, 3])),
?assertEqual([1,#{a => [c]},3], nested_put(?path([{ic,2}, a, {ic, 1}]), c, [1,#{a => [b]},3])), ?assertEqual(
?assertEqual(#{a => [1,a,3], b => 2}, nested_put(?path([a, {iv,b}]), a, #{a => [1,2,3], b => 2})), [1, #{a => [c]}, 3], nested_put(?path([{ic, 2}, a, {ic, 1}]), c, [1, #{a => [b]}, 3])
?assertEqual(#{a => [1,2,3], b => 2}, nested_put(?path([a, {iv,c}]), a, #{a => [1,2,3], b => 2})), ),
?assertEqual(#{a => [#{c => a},1,2,3]}, nested_put(?path([a, {ic,head}, c]), a, #{a => [1,2,3]})). ?assertEqual(
#{a => [1, a, 3], b => 2}, nested_put(?path([a, {iv, b}]), a, #{a => [1, 2, 3], b => 2})
),
?assertEqual(
#{a => [1, 2, 3], b => 2}, nested_put(?path([a, {iv, c}]), a, #{a => [1, 2, 3], b => 2})
),
?assertEqual(
#{a => [#{c => a}, 1, 2, 3]}, nested_put(?path([a, {ic, head}, c]), a, #{a => [1, 2, 3]})
).
t_nested_get_map(_) -> t_nested_get_map(_) ->
?assertEqual(undefined, nested_get(?path([a]), not_map)), ?assertEqual(undefined, nested_get(?path([a]), not_map)),
?assertEqual(#{a => 1}, nested_get(?path([]), #{a => 1})), ?assertEqual(#{a => 1}, nested_get(?path([]), #{a => 1})),
?assertEqual(#{b => c}, nested_get(?path([a]), #{a => #{b => c}})), ?assertEqual(#{b => c}, nested_get(?path([a]), #{a => #{b => c}})),
?assertEqual(undefined, nested_get(?path([a,b,c]), not_map)), ?assertEqual(undefined, nested_get(?path([a, b, c]), not_map)),
?assertEqual(undefined, nested_get(?path([a,b,c]), #{})), ?assertEqual(undefined, nested_get(?path([a, b, c]), #{})),
?assertEqual(undefined, nested_get(?path([a,b,c]), #{a => #{}})), ?assertEqual(undefined, nested_get(?path([a, b, c]), #{a => #{}})),
?assertEqual(undefined, nested_get(?path([a,b,c]), #{a => #{b => #{}}})), ?assertEqual(undefined, nested_get(?path([a, b, c]), #{a => #{b => #{}}})),
?assertEqual(v1, nested_get(?path([p,x]), #{p => #{x => v1}})), ?assertEqual(v1, nested_get(?path([p, x]), #{p => #{x => v1}})),
?assertEqual(v1, nested_get(?path([<<"p">>,<<"x">>]), #{p => #{x => v1}})), ?assertEqual(v1, nested_get(?path([<<"p">>, <<"x">>]), #{p => #{x => v1}})),
?assertEqual(c, nested_get(?path([a,b,c]), #{a => #{b => #{c => c}}})). ?assertEqual(c, nested_get(?path([a, b, c]), #{a => #{b => #{c => c}}})).
t_nested_get_map_1(_) -> t_nested_get_map_1(_) ->
?assertEqual(1, nested_get(?path([a]), <<"{\"a\": 1}">>)), ?assertEqual(1, nested_get(?path([a]), <<"{\"a\": 1}">>)),
?assertEqual(<<"{\"b\": 1}">>, nested_get(?path([a]), #{a => <<"{\"b\": 1}">>})), ?assertEqual(<<"{\"b\": 1}">>, nested_get(?path([a]), #{a => <<"{\"b\": 1}">>})),
?assertEqual(1, nested_get(?path([a,b]), #{a => <<"{\"b\": 1}">>})). ?assertEqual(1, nested_get(?path([a, b]), #{a => <<"{\"b\": 1}">>})).
t_nested_get_index(_) -> t_nested_get_index(_) ->
%% single index get %% single index get
?assertEqual(1, nested_get(?path([{ic,1}]), [1,2,3])), ?assertEqual(1, nested_get(?path([{ic, 1}]), [1, 2, 3])),
?assertEqual(2, nested_get(?path([{ic,2}]), [1,2,3])), ?assertEqual(2, nested_get(?path([{ic, 2}]), [1, 2, 3])),
?assertEqual(3, nested_get(?path([{ic,3}]), [1,2,3])), ?assertEqual(3, nested_get(?path([{ic, 3}]), [1, 2, 3])),
?assertEqual(undefined, nested_get(?path([{ic,0}]), [1,2,3])), ?assertEqual(undefined, nested_get(?path([{ic, 0}]), [1, 2, 3])),
?assertEqual("not_found", nested_get(?path([{ic,0}]), [1,2,3], "not_found")), ?assertEqual("not_found", nested_get(?path([{ic, 0}]), [1, 2, 3], "not_found")),
?assertEqual(undefined, nested_get(?path([{ic,4}]), [1,2,3])), ?assertEqual(undefined, nested_get(?path([{ic, 4}]), [1, 2, 3])),
?assertEqual("not_found", nested_get(?path([{ic,4}]), [1,2,3], "not_found")), ?assertEqual("not_found", nested_get(?path([{ic, 4}]), [1, 2, 3], "not_found")),
%% multiple index get %% multiple index get
?assertEqual(c, nested_get(?path([{ic,2}, {ic,3}]), [1,[a,b,c],3])), ?assertEqual(c, nested_get(?path([{ic, 2}, {ic, 3}]), [1, [a, b, c], 3])),
?assertEqual("I", nested_get(?path([{ic,2}, {ic,3}, {ic,1}]), [1,[a,b,["I","II","III"]],3])), ?assertEqual(
?assertEqual(undefined, nested_get(?path([{ic,2}, {ic,1}, {ic,1}]), [1,[a,b,["I","II","III"]],3])), "I", nested_get(?path([{ic, 2}, {ic, 3}, {ic, 1}]), [1, [a, b, ["I", "II", "III"]], 3])
?assertEqual(default, nested_get(?path([{ic,2}, {ic,1}, {ic,1}]), [1,[a,b,["I","II","III"]],3], default)). ),
?assertEqual(
undefined,
nested_get(?path([{ic, 2}, {ic, 1}, {ic, 1}]), [1, [a, b, ["I", "II", "III"]], 3])
),
?assertEqual(
default,
nested_get(?path([{ic, 2}, {ic, 1}, {ic, 1}]), [1, [a, b, ["I", "II", "III"]], 3], default)
).
t_nested_get_negative_index(_) -> t_nested_get_negative_index(_) ->
?assertEqual(3, nested_get(?path([{ic,-1}]), [1,2,3])), ?assertEqual(3, nested_get(?path([{ic, -1}]), [1, 2, 3])),
?assertEqual(2, nested_get(?path([{ic,-2}]), [1,2,3])), ?assertEqual(2, nested_get(?path([{ic, -2}]), [1, 2, 3])),
?assertEqual(1, nested_get(?path([{ic,-3}]), [1,2,3])), ?assertEqual(1, nested_get(?path([{ic, -3}]), [1, 2, 3])),
?assertEqual(undefined, nested_get(?path([{ic,-4}]), [1,2,3])). ?assertEqual(undefined, nested_get(?path([{ic, -4}]), [1, 2, 3])).
t_nested_get_mix_map_index(_) -> t_nested_get_mix_map_index(_) ->
%% index const %% index const
?assertEqual(1, nested_get(?path([a, {ic,1}]), #{a => [1,2,3]})), ?assertEqual(1, nested_get(?path([a, {ic, 1}]), #{a => [1, 2, 3]})),
?assertEqual(2, nested_get(?path([{ic,2}, a]), [1,#{a => 2},3])), ?assertEqual(2, nested_get(?path([{ic, 2}, a]), [1, #{a => 2}, 3])),
?assertEqual(undefined, nested_get(?path([a, {ic,0}]), #{a => [1,2,3]})), ?assertEqual(undefined, nested_get(?path([a, {ic, 0}]), #{a => [1, 2, 3]})),
?assertEqual("not_found", nested_get(?path([a, {ic,0}]), #{a => [1,2,3]}, "not_found")), ?assertEqual("not_found", nested_get(?path([a, {ic, 0}]), #{a => [1, 2, 3]}, "not_found")),
?assertEqual("not_found", nested_get(?path([b, {ic,1}]), #{a => [1,2,3]}, "not_found")), ?assertEqual("not_found", nested_get(?path([b, {ic, 1}]), #{a => [1, 2, 3]}, "not_found")),
?assertEqual(undefined, nested_get(?path([{ic,4}, a]), [1,2,3,4])), ?assertEqual(undefined, nested_get(?path([{ic, 4}, a]), [1, 2, 3, 4])),
?assertEqual("not_found", nested_get(?path([{ic,4}, a]), [1,2,3,4], "not_found")), ?assertEqual("not_found", nested_get(?path([{ic, 4}, a]), [1, 2, 3, 4], "not_found")),
?assertEqual(c, nested_get(?path([a, {ic,2}, {ic,3}]), #{a => [1,[a,b,c],3]})), ?assertEqual(c, nested_get(?path([a, {ic, 2}, {ic, 3}]), #{a => [1, [a, b, c], 3]})),
?assertEqual("I", nested_get(?path([{ic,2}, c, {ic,1}]), [1,#{a => a, b => b, c => ["I","II","III"]},3])), ?assertEqual(
?assertEqual("I", nested_get(?path([{ic,2}, c, d]), [1,#{a => a, b => b, c => #{d => "I"}},3])), "I",
?assertEqual(undefined, nested_get(?path([{ic,2}, c, e]), [1,#{a => a, b => b, c => #{d => "I"}},3])), nested_get(?path([{ic, 2}, c, {ic, 1}]), [1, #{a => a, b => b, c => ["I", "II", "III"]}, 3])
?assertEqual(default, nested_get(?path([{ic,2}, c, e]), [1,#{a => a, b => b, c => #{d => "I"}},3], default)), ),
?assertEqual(
"I", nested_get(?path([{ic, 2}, c, d]), [1, #{a => a, b => b, c => #{d => "I"}}, 3])
),
?assertEqual(
undefined, nested_get(?path([{ic, 2}, c, e]), [1, #{a => a, b => b, c => #{d => "I"}}, 3])
),
?assertEqual(
default,
nested_get(?path([{ic, 2}, c, e]), [1, #{a => a, b => b, c => #{d => "I"}}, 3], default)
),
%% index var %% index var
?assertEqual(1, nested_get(?path([a, {iv,<<"b">>}]), #{a => [1,2,3], b => 1})), ?assertEqual(1, nested_get(?path([a, {iv, <<"b">>}]), #{a => [1, 2, 3], b => 1})),
?assertEqual(1, nested_get(?path([a, {iv,b}]), #{a => [1,2,3], b => 1})), ?assertEqual(1, nested_get(?path([a, {iv, b}]), #{a => [1, 2, 3], b => 1})),
?assertEqual(undefined, nested_get(?path([a, {iv,c}]), #{a => [1,2,3], b => 1})), ?assertEqual(undefined, nested_get(?path([a, {iv, c}]), #{a => [1, 2, 3], b => 1})),
?assertEqual(undefined, nested_get(?path([a, {iv,b}]), #{a => [1,2,3], b => 4})), ?assertEqual(undefined, nested_get(?path([a, {iv, b}]), #{a => [1, 2, 3], b => 4})),
?assertEqual("I", nested_get(?path([{i,?path([{ic, 3}])}, c, d]), ?assertEqual(
[1,#{a => a, b => b, c => #{d => "I"}},2], default)), "I",
?assertEqual(3, nested_get(?path([a, {i,?path([b,{ic,1},c])}]), nested_get(
#{a => [1,2,3], b => [#{c => 3}]})), ?path([{i, ?path([{ic, 3}])}, c, d]),
?assertEqual(3, nested_get(?path([a, {i,?path([b,{ic,1},c])}]), [1, #{a => a, b => b, c => #{d => "I"}}, 2],
#{a => [1,2,3], b => [#{c => 3}]}, default)), default
?assertEqual(default, nested_get(?path([a, {i,?path([b,{ic,1},c])}]), )
#{a => [1,2,3], b => [#{c => 4}]}, default)), ),
?assertEqual(default, nested_get(?path([a, {i,?path([b,{ic,2},c])}]), ?assertEqual(
#{a => [1,2,3], b => [#{c => 3}]}, default)). 3,
nested_get(
?path([a, {i, ?path([b, {ic, 1}, c])}]),
#{a => [1, 2, 3], b => [#{c => 3}]}
)
),
?assertEqual(
3,
nested_get(
?path([a, {i, ?path([b, {ic, 1}, c])}]),
#{a => [1, 2, 3], b => [#{c => 3}]},
default
)
),
?assertEqual(
default,
nested_get(
?path([a, {i, ?path([b, {ic, 1}, c])}]),
#{a => [1, 2, 3], b => [#{c => 4}]},
default
)
),
?assertEqual(
default,
nested_get(
?path([a, {i, ?path([b, {ic, 2}, c])}]),
#{a => [1, 2, 3], b => [#{c => 3}]},
default
)
).
t_atom_key_map(_) -> t_atom_key_map(_) ->
?assertEqual(#{a => 1}, atom_key_map(#{<<"a">> => 1})), ?assertEqual(#{a => 1}, atom_key_map(#{<<"a">> => 1})),
?assertEqual(#{a => 1, b => #{a => 2}}, ?assertEqual(
atom_key_map(#{<<"a">> => 1, <<"b">> => #{<<"a">> => 2}})), #{a => 1, b => #{a => 2}},
?assertEqual([#{a => 1}, #{b => #{a => 2}}], atom_key_map(#{<<"a">> => 1, <<"b">> => #{<<"a">> => 2}})
atom_key_map([#{<<"a">> => 1}, #{<<"b">> => #{<<"a">> => 2}}])), ),
?assertEqual(#{a => 1, b => [#{a => 2}, #{c => 2}]}, ?assertEqual(
atom_key_map(#{<<"a">> => 1, <<"b">> => [#{<<"a">> => 2}, #{<<"c">> => 2}]})). [#{a => 1}, #{b => #{a => 2}}],
atom_key_map([#{<<"a">> => 1}, #{<<"b">> => #{<<"a">> => 2}}])
),
?assertEqual(
#{a => 1, b => [#{a => 2}, #{c => 2}]},
atom_key_map(#{<<"a">> => 1, <<"b">> => [#{<<"a">> => 2}, #{<<"c">> => 2}]})
).
all() -> all() ->
IsTestCase = fun("t_" ++ _) -> true; (_) -> false end, IsTestCase = fun
("t_" ++ _) -> true;
(_) -> false
end,
[F || {F, _A} <- module_info(exports), IsTestCase(atom_to_list(F))]. [F || {F, _A} <- module_info(exports), IsTestCase(atom_to_list(F))].
suite() -> suite() ->

16
apps/emqx_rule_engine/test/prop_rule_maps.erl
View File

@ -3,8 +3,14 @@
-include_lib("proper/include/proper.hrl"). -include_lib("proper/include/proper.hrl").
prop_get_put_single_key() -> prop_get_put_single_key() ->
?FORALL({Key, Val}, {term(), term()}, ?FORALL(
begin {Key, Val},
Val =:= emqx_rule_maps:nested_get({var, Key}, {term(), term()},
emqx_rule_maps:nested_put({var, Key}, Val, #{})) begin
end). Val =:=
emqx_rule_maps:nested_get(
{var, Key},
emqx_rule_maps:nested_put({var, Key}, Val, #{})
)
end
).