yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(pluglib): move conversion utils to `emqx_utils_conv`

This commit is contained in:
Andrew Mayorov 2023-06-08 14:42:20 +03:00
parent 7d0abb6146
commit a51baaa206
No known key found for this signature in database
GPG Key ID: 2837C62ACFBFED5D
14 changed files with 215 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl
View File

@ -637,7 +637,7 @@ value_type(Val) ->
Val.
key_filter(undefined) -> undefined;
key_filter(Value) -> emqx_plugin_libs_rule:bin(Value).
key_filter(Value) -> emqx_utils_conv:bin(Value).
data_filter(undefined) -> undefined;
data_filter(Int) when is_integer(Int) -> Int;
@ -645,7 +645,7 @@ data_filter(Number) when is_number(Number) -> Number;
data_filter(Bool) when is_boolean(Bool) -> Bool;
data_filter(Data) -> bin(Data).
bin(Data) -> emqx_plugin_libs_rule:bin(Data).
bin(Data) -> emqx_utils_conv:bin(Data).
%% helper funcs
log_error_points(InstId, Errs) ->

2
apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl
View File

@ -556,7 +556,7 @@ render(FullMessage, PayloadTemplate) ->
(undefined) ->
<<>>;
(X) ->
emqx_plugin_libs_rule:bin(X)
emqx_utils_conv:bin(X)
end
},
emqx_placeholder:proc_tmpl(PayloadTemplate, FullMessage, Opts).

2
apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
View File

@ -255,7 +255,7 @@ render(Template, Message) ->
Opts = #{
var_trans => fun
(undefined) -> <<"">>;
(X) -> emqx_plugin_libs_rule:bin(X)
(X) -> emqx_utils_conv:bin(X)
end,
return => full_binary
},

2
apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl
View File

@ -435,7 +435,7 @@ render(Message, Template) ->
Opts = #{
var_trans => fun
(undefined) -> <<"">>;
(X) -> emqx_plugin_libs_rule:bin(X)
(X) -> emqx_utils_conv:bin(X)
end,
return => full_binary
},

2
apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl
View File

@ -44,7 +44,7 @@
%% Internal exports used to execute code with ecpool worker
-export([do_get_status/1, worker_do_insert/3]).
-import(emqx_plugin_libs_rule, [str/1]).
-import(emqx_utils_conv, [str/1]).
-import(hoconsc, [mk/2, enum/1, ref/2]).
-define(ACTION_SEND_MESSAGE, send_message).

10
apps/emqx_oracle/src/emqx_oracle.erl
View File

@ -93,14 +93,14 @@ on_start(
ServiceName =
case maps:get(service_name, Config, undefined) of
undefined -> undefined;
ServiceName0 -> emqx_plugin_libs_rule:str(ServiceName0)
ServiceName0 -> emqx_utils_conv:str(ServiceName0)
end,
Options = [
{host, Host},
{port, Port},
{user, emqx_plugin_libs_rule:str(User)},
{user, emqx_utils_conv:str(User)},
{password, jamdb_secret:wrap(maps:get(password, Config, ""))},
{sid, emqx_plugin_libs_rule:str(Sid)},
{sid, emqx_utils_conv:str(Sid)},
{service_name, ServiceName},
{pool_size, maps:get(<<"pool_size">>, Config, ?DEFAULT_POOL_SIZE)},
{timeout, ?OPT_TIMEOUT},
@ -268,14 +268,14 @@ connect(Opts) ->
jamdb_oracle:start_link(Opts).
sql_query_to_str(SqlQuery) ->
emqx_plugin_libs_rule:str(SqlQuery).
emqx_utils_conv:str(SqlQuery).
sql_params_to_str(Params) when is_list(Params) ->
lists:map(
fun
(false) -> "0";
(true) -> "1";
(Value) -> emqx_plugin_libs_rule:str(Value)
(Value) -> emqx_utils_conv:str(Value)
end,
Params
).

41
apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
View File

@ -27,8 +27,6 @@
%% type converting
-export([
str/1,
bin/1,
bool/1,
int/1,
float/1,
@ -36,7 +34,6 @@
map/1,
utf8_bin/1,
utf8_str/1,
number_to_binary/1,
atom_key/1,
unsafe_atom_key/1
]).
@ -172,39 +169,15 @@ tcp_connectivity(Host, Port, Timeout) ->
{error, Reason}
end.
str(Bin) when is_binary(Bin) -> binary_to_list(Bin);
str(Num) when is_number(Num) -> number_to_list(Num);
str(Atom) when is_atom(Atom) -> atom_to_list(Atom);
str(Map) when is_map(Map) -> binary_to_list(emqx_utils_json:encode(Map));
str(List) when is_list(List) ->
case io_lib:printable_list(List) of
true -> List;
false -> binary_to_list(emqx_utils_json:encode(List))
end;
str(Data) ->
error({invalid_str, Data}).
utf8_bin(Str) when is_binary(Str); is_list(Str) ->
unicode:characters_to_binary(Str);
utf8_bin(Str) ->
unicode:characters_to_binary(bin(Str)).
unicode:characters_to_binary(emqx_utils_conv:bin(Str)).
utf8_str(Str) when is_binary(Str); is_list(Str) ->
unicode:characters_to_list(Str);
utf8_str(Str) ->
unicode:characters_to_list(str(Str)).
bin(Bin) when is_binary(Bin) -> Bin;
bin(Num) when is_number(Num) -> number_to_binary(Num);
bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
bin(Map) when is_map(Map) -> emqx_utils_json:encode(Map);
bin(List) when is_list(List) ->
case io_lib:printable_list(List) of
true -> list_to_binary(List);
false -> emqx_utils_json:encode(List)
end;
bin(Data) ->
error({invalid_bin, Data}).
unicode:characters_to_list(emqx_utils_conv:str(Str)).
int(List) when is_list(List) ->
try
@ -274,13 +247,3 @@ bool(Bool) when
false;
bool(Bool) ->
error({invalid_boolean, Bool}).
number_to_binary(Int) when is_integer(Int) ->
integer_to_binary(Int);
number_to_binary(Float) when is_float(Float) ->
float_to_binary(Float, [{decimals, 10}, compact]).
number_to_list(Int) when is_integer(Int) ->
integer_to_list(Int);
number_to_list(Float) when is_float(Float) ->
float_to_list(Float, [{decimals, 10}, compact]).

23
apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl
View File

@ -28,11 +28,11 @@ all() -> emqx_common_test_helpers:all(?MODULE).
t_http_connectivity(_) ->
{ok, Socket} = gen_tcp:listen(?PORT, []),
ok = emqx_plugin_libs_rule:http_connectivity(
"http://127.0.0.1:" ++ emqx_plugin_libs_rule:str(?PORT), 1000
"http://127.0.0.1:" ++ integer_to_list(?PORT), 1000
),
gen_tcp:close(Socket),
{error, _} = emqx_plugin_libs_rule:http_connectivity(
"http://127.0.0.1:" ++ emqx_plugin_libs_rule:str(?PORT), 1000
"http://127.0.0.1:" ++ integer_to_list(?PORT), 1000
).
t_tcp_connectivity(_) ->
@ -41,25 +41,6 @@ t_tcp_connectivity(_) ->
gen_tcp:close(Socket),
{error, _} = emqx_plugin_libs_rule:tcp_connectivity("127.0.0.1", ?PORT, 1000).
t_str(_) ->
?assertEqual("abc", emqx_plugin_libs_rule:str("abc")),
?assertEqual("abc", emqx_plugin_libs_rule:str(abc)),
?assertEqual("{\"a\":1}", emqx_plugin_libs_rule:str(#{a => 1})),
?assertEqual("1", emqx_plugin_libs_rule:str(1)),
?assertEqual("2.0", emqx_plugin_libs_rule:str(2.0)),
?assertEqual("true", emqx_plugin_libs_rule:str(true)),
?assertError(_, emqx_plugin_libs_rule:str({a, v})).
t_bin(_) ->
?assertEqual(<<"abc">>, emqx_plugin_libs_rule:bin("abc")),
?assertEqual(<<"abc">>, emqx_plugin_libs_rule:bin(abc)),
?assertEqual(<<"{\"a\":1}">>, emqx_plugin_libs_rule:bin(#{a => 1})),
?assertEqual(<<"[{\"a\":1}]">>, emqx_plugin_libs_rule:bin([#{a => 1}])),
?assertEqual(<<"1">>, emqx_plugin_libs_rule:bin(1)),
?assertEqual(<<"2.0">>, emqx_plugin_libs_rule:bin(2.0)),
?assertEqual(<<"true">>, emqx_plugin_libs_rule:bin(true)),
?assertError(_, emqx_plugin_libs_rule:bin({a, v})).
t_atom_key(_) ->
_ = erlang,
_ = port,

40
apps/emqx_rule_engine/src/emqx_rule_funcs.erl
View File

@ -17,7 +17,6 @@
-module(emqx_rule_funcs).
-include("rule_engine.hrl").
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/logger.hrl").
-elvis([{elvis_style, god_modules, disable}]).
@ -266,8 +265,6 @@
]}
).
-define(is_var(X), is_binary(X)).
%% @doc "msgid()" Func
msgid() ->
fun
@ -631,29 +628,42 @@ do_get_subbits(Bits, Sz, Len, <<"bits">>, <<"signed">>, <<"little">>) ->
%%------------------------------------------------------------------------------
str(Data) ->
emqx_plugin_libs_rule:bin(Data).
emqx_utils_conv:bin(Data).
str_utf8(Data) when is_binary(Data); is_list(Data) ->
unicode:characters_to_binary(Data);
str_utf8(Data) ->
emqx_plugin_libs_rule:utf8_bin(Data).
unicode:characters_to_binary(str(Data)).
bool(Data) ->
emqx_plugin_libs_rule:bool(Data).
emqx_utils_conv:bool(Data).
int(Data) ->
emqx_plugin_libs_rule:int(Data).
emqx_utils_conv:int(Data).
float(Data) ->
emqx_plugin_libs_rule:float(Data).
emqx_utils_conv:float(Data).
float(Data, Decimals) when Decimals > 0 ->
Data1 = ?MODULE:float(Data),
Data1 = emqx_utils_conv:float(Data),
list_to_float(float_to_list(Data1, [{decimals, Decimals}])).
float2str(Float, Precision) ->
emqx_plugin_libs_rule:float2str(Float, Precision).
float_to_binary(Float, [{decimals, Precision}, compact]).
map(Bin) when is_binary(Bin) ->
case emqx_utils_json:decode(Bin) of
Map = #{} ->
Map;
_ ->
error(badarg, [Bin])
end;
map(List) when is_list(List) ->
maps:from_list(List);
map(Map = #{}) ->
Map;
map(Data) ->
emqx_plugin_libs_rule:map(Data).
error(badarg, [Data]).
bin2hexstr(Bin) when is_binary(Bin) ->
emqx_utils:bin_to_hexstr(Bin, upper).
@ -895,7 +905,7 @@ mget(Key, Map, Default) ->
Val;
error when is_atom(Key) ->
%% the map may have an equivalent binary-form key
BinKey = emqx_plugin_libs_rule:bin(Key),
BinKey = emqx_utils_conv:bin(Key),
case maps:find(BinKey, Map) of
{ok, Val} -> Val;
error -> Default
@ -922,7 +932,7 @@ mput(Key, Val, Map) ->
maps:put(Key, Val, Map);
error when is_atom(Key) ->
%% the map may have an equivalent binary-form key
BinKey = emqx_plugin_libs_rule:bin(Key),
BinKey = emqx_utils_conv:bin(Key),
case maps:find(BinKey, Map) of
{ok, _} -> maps:put(BinKey, Val, Map);
error -> maps:put(Key, Val, Map)
@ -1053,7 +1063,7 @@ unix_ts_to_rfc3339(Epoch) ->
unix_ts_to_rfc3339(Epoch, <<"second">>).
unix_ts_to_rfc3339(Epoch, Unit) when is_integer(Epoch) ->
emqx_plugin_libs_rule:bin(
emqx_utils_conv:bin(
calendar:system_time_to_rfc3339(
Epoch, [{unit, time_unit(Unit)}]
)
@ -1090,7 +1100,7 @@ format_date(TimeUnit, Offset, FormatString) ->
format_date(TimeUnit, Offset, FormatString, TimeEpoch) ->
Unit = time_unit(TimeUnit),
emqx_plugin_libs_rule:bin(
emqx_utils_conv:bin(
lists:concat(
emqx_calendar:format(TimeEpoch, Unit, Offset, FormatString)
)

2
apps/emqx_rule_engine/src/emqx_rule_maps.erl
View File

@ -97,7 +97,7 @@ general_find({key, Key}, Map, _OrgData, Handler) when is_map(Map) ->
Handler({found, {{key, Key}, Val}});
error when is_atom(Key) ->
%% the map may have an equivalent binary-form key
BinKey = emqx_plugin_libs_rule:bin(Key),
BinKey = atom_to_binary(Key),
case maps:find(BinKey, Map) of
{ok, Val} -> Handler({equivalent, {{key, BinKey}, Val}});
error -> Handler(not_found)

6
apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl
View File

@ -126,7 +126,7 @@ t_int(_) ->
?assertEqual(1, emqx_rule_funcs:int(1.0001)),
?assertEqual(1, emqx_rule_funcs:int(true)),
?assertEqual(0, emqx_rule_funcs:int(false)),
?assertError({invalid_number, {a, v}}, emqx_rule_funcs:int({a, v})),
?assertError(badarg, emqx_rule_funcs:int({a, v})),
?assertError(_, emqx_rule_funcs:int("a")).
t_float(_) ->
@ -137,7 +137,7 @@ t_float(_) ->
?assertEqual(1.9, emqx_rule_funcs:float(1.9)),
?assertEqual(1.0001, emqx_rule_funcs:float(1.0001)),
?assertEqual(1.0000000001, emqx_rule_funcs:float(1.0000000001)),
?assertError({invalid_number, {a, v}}, emqx_rule_funcs:float({a, v})),
?assertError(badarg, emqx_rule_funcs:float({a, v})),
?assertError(_, emqx_rule_funcs:float("a")).
t_map(_) ->
@ -158,7 +158,7 @@ t_bool(_) ->
?assertEqual(true, emqx_rule_funcs:bool(<<"true">>)),
?assertEqual(false, emqx_rule_funcs:bool(false)),
?assertEqual(false, emqx_rule_funcs:bool(<<"false">>)),
?assertError({invalid_boolean, _}, emqx_rule_funcs:bool(3)).
?assertError(badarg, emqx_rule_funcs:bool(3)).
t_proc_dict_put_get_del(_) ->
?assertEqual(undefined, emqx_rule_funcs:proc_dict_get(<<"abc">>)),

4
apps/emqx_utils/src/emqx_placeholder.erl
View File

@ -112,7 +112,7 @@ proc_tmpl(Tokens, Data) ->
-spec proc_tmpl(tmpl_token(), map(), proc_tmpl_opts()) -> binary() | list().
proc_tmpl(Tokens, Data, Opts = #{return := full_binary}) ->
Trans = maps:get(var_trans, Opts, fun emqx_plugin_libs_rule:bin/1),
Trans = maps:get(var_trans, Opts, fun emqx_utils_conv:bin/1),
list_to_binary(
proc_tmpl(Tokens, Data, #{return => rawlist, var_trans => Trans})
);
@ -243,7 +243,7 @@ sql_data(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
sql_data(Map) when is_map(Map) -> emqx_utils_json:encode(Map).
-spec bin(term()) -> binary().
bin(Val) -> emqx_plugin_libs_rule:bin(Val).
bin(Val) -> emqx_utils_conv:bin(Val).
-spec quote_sql(_Value) -> iolist().
quote_sql(Str) ->

125
apps/emqx_utils/src/emqx_utils_conv.erl Normal file
View File

@ -0,0 +1,125 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_utils_conv).
-export([bin/1]).
-export([str/1]).
-export([bool/1]).
-export([int/1]).
-export([float/1]).
-compile({no_auto_import, [float/1]}).
-type scalar() :: binary() | number() | atom() | string().
-spec bin(Term) -> binary() when
Term :: scalar() | #{scalar() => Term} | [Term].
bin(Bin) when is_binary(Bin) -> Bin;
bin(Num) when is_number(Num) -> number_to_binary(Num);
bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
bin(Map) when is_map(Map) -> emqx_utils_json:encode(Map);
bin(List) when is_list(List) ->
case io_lib:printable_list(List) of
true -> list_to_binary(List);
false -> emqx_utils_json:encode(List)
end;
bin(Data) ->
error({invalid_bin, Data}).
-spec str(Term) -> string() when
Term :: scalar() | #{scalar() => Term} | [Term].
str(Bin) when is_binary(Bin) -> binary_to_list(Bin);
str(Num) when is_number(Num) -> number_to_list(Num);
str(Atom) when is_atom(Atom) -> atom_to_list(Atom);
str(Map) when is_map(Map) -> binary_to_list(emqx_utils_json:encode(Map));
str(List) when is_list(List) ->
case io_lib:printable_list(List) of
true -> List;
false -> binary_to_list(emqx_utils_json:encode(List))
end;
str(Data) ->
error({invalid_str, Data}).
-spec number_to_binary(number()) -> binary().
number_to_binary(Int) when is_integer(Int) ->
integer_to_binary(Int);
number_to_binary(Float) when is_float(Float) ->
float_to_binary(Float, [{decimals, 10}, compact]).
-spec number_to_list(number()) -> string().
number_to_list(Int) when is_integer(Int) ->
integer_to_list(Int);
number_to_list(Float) when is_float(Float) ->
float_to_list(Float, [{decimals, 10}, compact]).
-spec bool(Term) -> boolean() when
Term :: boolean() | binary() | 0..1.
bool(true) -> true;
bool(<<"true">>) -> true;
bool(N) when N == 1 -> true;
bool(false) -> false;
bool(<<"false">>) -> false;
bool(N) when N == 0 -> false;
bool(Data) -> error(badarg, [Data]).
-spec int(Term) -> integer() when
Term :: binary() | string() | number() | boolean().
int(List) when is_list(List) ->
try
list_to_integer(List)
catch
error:badarg ->
int(list_to_float(List))
end;
int(Bin) when is_binary(Bin) ->
try
binary_to_integer(Bin)
catch
error:badarg ->
int(binary_to_float(Bin))
end;
int(Int) when is_integer(Int) ->
Int;
int(Float) when is_float(Float) ->
erlang:floor(Float);
int(true) ->
1;
int(false) ->
0;
int(Data) ->
error(badarg, [Data]).
-spec float(Term) -> float() when
Term :: binary() | string() | number().
float(List) when is_list(List) ->
try
list_to_float(List)
catch
error:badarg ->
float(list_to_integer(List))
end;
float(Bin) when is_binary(Bin) ->
try
binary_to_float(Bin)
catch
error:badarg ->
float(binary_to_integer(Bin))
end;
float(Num) when is_number(Num) ->
erlang:float(Num);
float(Data) ->
error(badarg, [Data]).

44
apps/emqx_utils/test/emqx_utils_conv_tests.erl Normal file
View File

@ -0,0 +1,44 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_utils_conv_tests).
-import(emqx_utils_conv, [bin/1, str/1]).
-include_lib("eunit/include/eunit.hrl").
bin_test_() ->
[
?_assertEqual(<<"abc">>, bin("abc")),
?_assertEqual(<<"abc">>, bin(abc)),
?_assertEqual(<<"{\"a\":1}">>, bin(#{a => 1})),
?_assertEqual(<<"[{\"a\":1}]">>, bin([#{a => 1}])),
?_assertEqual(<<"1">>, bin(1)),
?_assertEqual(<<"2.0">>, bin(2.0)),
?_assertEqual(<<"true">>, bin(true)),
?_assertError(_, bin({a, v}))
].
str_test_() ->
[
?_assertEqual("abc", str("abc")),
?_assertEqual("abc", str(abc)),
?_assertEqual("{\"a\":1}", str(#{a => 1})),
?_assertEqual("1", str(1)),
?_assertEqual("2.0", str(2.0)),
?_assertEqual("true", str(true)),
?_assertError(_, str({a, v}))
].