yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #12316 from lafirest/fix/iotdb 

fix(iotdb): move the iot_version into IoTDB connector
This commit is contained in:
lafirest 2024-01-12 20:56:10 +08:00 committed by GitHub
parent 41e931d90a b15106c753
commit 1c0ca72877
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 120 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl
View File

@ -89,14 +89,6 @@ fields(action_parameters) ->
desc => ?DESC("config_device_id") desc => ?DESC("config_device_id")
} }
)}, )},
{iotdb_version,
mk(
hoconsc:enum([?VSN_1_1_X, ?VSN_1_0_X, ?VSN_0_13_X]),
#{
desc => ?DESC("config_iotdb_version"),
default => ?VSN_1_1_X
}
)},
{data, {data,
mk( mk(
array(ref(?MODULE, action_parameters_data)), array(ref(?MODULE, action_parameters_data)),
@ -310,8 +302,7 @@ action_values() ->
} }
], ],
is_aligned => false, is_aligned => false,
device_id => <<"my_device">>, device_id => <<"my_device">>
iotdb_version => ?VSN_1_1_X
} }
}. }.

123
apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl
View File

@ -47,6 +47,7 @@
connect_timeout := pos_integer(), connect_timeout := pos_integer(),
pool_type := random | hash, pool_type := random | hash,
pool_size := pos_integer(), pool_size := pos_integer(),
iotdb_version := atom(),
request => undefined | map(), request => undefined | map(),
atom() => _ atom() => _
}. }.
@ -57,6 +58,7 @@
connect_timeout := pos_integer(), connect_timeout := pos_integer(),
pool_type := random | hash, pool_type := random | hash,
channels := map(), channels := map(),
iotdb_version := atom(),
request => undefined | map(), request => undefined | map(),
atom() => _ atom() => _
}. }.
@ -88,6 +90,7 @@ connector_example_values() ->
name => <<"iotdb_connector">>, name => <<"iotdb_connector">>,
type => iotdb, type => iotdb,
enable => true, enable => true,
iotdb_version => ?VSN_1_1_X,
authentication => #{ authentication => #{
<<"username">> => <<"root">>, <<"username">> => <<"root">>,
<<"password">> => <<"*****">> <<"password">> => <<"*****">>
@ -121,6 +124,14 @@ fields("connection_fields") ->
desc => ?DESC(emqx_bridge_iotdb, "config_base_url") desc => ?DESC(emqx_bridge_iotdb, "config_base_url")
} }
)}, )},
{iotdb_version,
mk(
hoconsc:enum([?VSN_1_1_X, ?VSN_1_0_X, ?VSN_0_13_X]),
#{
desc => ?DESC(emqx_bridge_iotdb, "config_iotdb_version"),
default => ?VSN_1_1_X
}
)},
{authentication, {authentication,
mk( mk(
hoconsc:union([ref(?MODULE, auth_basic)]), hoconsc:union([ref(?MODULE, auth_basic)]),
@ -190,7 +201,7 @@ proplists_without(Keys, List) ->
callback_mode() -> async_if_possible. callback_mode() -> async_if_possible.
-spec on_start(manager_id(), config()) -> {ok, state()} | no_return(). -spec on_start(manager_id(), config()) -> {ok, state()} | no_return().
on_start(InstanceId, Config) -> on_start(InstanceId, #{iotdb_version := Version} = Config) ->
%% [FIXME] The configuration passed in here is pre-processed and transformed %% [FIXME] The configuration passed in here is pre-processed and transformed
%% in emqx_bridge_resource:parse_confs/2. %% in emqx_bridge_resource:parse_confs/2.
case emqx_bridge_http_connector:on_start(InstanceId, Config) of case emqx_bridge_http_connector:on_start(InstanceId, Config) of
@ -201,7 +212,7 @@ on_start(InstanceId, Config) ->
request => maps:get(request, State, <<>>) request => maps:get(request, State, <<>>)
}), }),
?tp(iotdb_bridge_started, #{instance_id => InstanceId}), ?tp(iotdb_bridge_started, #{instance_id => InstanceId}),
{ok, State#{channels => #{}}}; {ok, State#{iotdb_version => Version, channels => #{}}};
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{ ?SLOG(error, #{
msg => "failed_to_start_iotdb_bridge", msg => "failed_to_start_iotdb_bridge",
@ -231,7 +242,11 @@ on_get_status(InstanceId, State) ->
{ok, pos_integer(), [term()], term()} {ok, pos_integer(), [term()], term()}
| {ok, pos_integer(), [term()]} | {ok, pos_integer(), [term()]}
| {error, term()}. | {error, term()}.
on_query(InstanceId, {ChannelId, _Message} = Req, #{channels := Channels} = State) -> on_query(
InstanceId,
{ChannelId, _Message} = Req,
#{iotdb_version := IoTDBVsn, channels := Channels} = State
) ->
?tp(iotdb_bridge_on_query, #{instance_id => InstanceId}), ?tp(iotdb_bridge_on_query, #{instance_id => InstanceId}),
?SLOG(debug, #{ ?SLOG(debug, #{
msg => "iotdb_bridge_on_query_called", msg => "iotdb_bridge_on_query_called",
@ -240,7 +255,7 @@ on_query(InstanceId, {ChannelId, _Message} = Req, #{channels := Channels} = Stat
state => emqx_utils:redact(State) state => emqx_utils:redact(State)
}), }),
case try_render_message(Req, Channels) of case try_render_message(Req, IoTDBVsn, Channels) of
{ok, IoTDBPayload} -> {ok, IoTDBPayload} ->
handle_response( handle_response(
emqx_bridge_http_connector:on_query( emqx_bridge_http_connector:on_query(
@ -254,7 +269,10 @@ on_query(InstanceId, {ChannelId, _Message} = Req, #{channels := Channels} = Stat
-spec on_query_async(manager_id(), {send_message, map()}, {function(), [term()]}, state()) -> -spec on_query_async(manager_id(), {send_message, map()}, {function(), [term()]}, state()) ->
{ok, pid()} | {error, empty_request}. {ok, pid()} | {error, empty_request}.
on_query_async( on_query_async(
InstanceId, {ChannelId, _Message} = Req, ReplyFunAndArgs0, #{channels := Channels} = State InstanceId,
{ChannelId, _Message} = Req,
ReplyFunAndArgs0,
#{iotdb_version := IoTDBVsn, channels := Channels} = State
) -> ) ->
?tp(iotdb_bridge_on_query_async, #{instance_id => InstanceId}), ?tp(iotdb_bridge_on_query_async, #{instance_id => InstanceId}),
?SLOG(debug, #{ ?SLOG(debug, #{
@ -263,7 +281,7 @@ on_query_async(
send_message => Req, send_message => Req,
state => emqx_utils:redact(State) state => emqx_utils:redact(State)
}), }),
case try_render_message(Req, Channels) of case try_render_message(Req, IoTDBVsn, Channels) of
{ok, IoTDBPayload} -> {ok, IoTDBPayload} ->
ReplyFunAndArgs = ReplyFunAndArgs =
{ {
@ -282,10 +300,10 @@ on_query_async(
on_add_channel( on_add_channel(
InstanceId, InstanceId,
#{channels := Channels} = OldState0, #{iotdb_version := Version, channels := Channels} = OldState0,
ChannelId, ChannelId,
#{ #{
parameters := #{iotdb_version := Version, data := Data} = Parameter parameters := #{data := Data} = Parameter
} }
) -> ) ->
case maps:is_key(ChannelId, Channels) of case maps:is_key(ChannelId, Channels) of
@ -404,25 +422,41 @@ proc_data(PreProcessedData, Msg) ->
now_us => erlang:convert_time_unit(NowNS, nanosecond, microsecond), now_us => erlang:convert_time_unit(NowNS, nanosecond, microsecond),
now_ns => NowNS now_ns => NowNS
}, },
lists:map( proc_data(PreProcessedData, Msg, Nows, []).
fun(
proc_data(
[
#{ #{
timestamp := TimestampTkn, timestamp := TimestampTkn,
measurement := Measurement, measurement := Measurement,
data_type := DataType0, data_type := DataType0,
value := ValueTkn value := ValueTkn
} }
) -> | T
DataType = emqx_placeholder:proc_tmpl(DataType0, Msg), ],
Msg,
Nows,
Acc
) ->
DataType = list_to_binary(
string:uppercase(binary_to_list(emqx_placeholder:proc_tmpl(DataType0, Msg)))
),
case proc_value(DataType, ValueTkn, Msg) of
{ok, Value} ->
proc_data(T, Msg, Nows, [
#{ #{
timestamp => iot_timestamp(TimestampTkn, Msg, Nows), timestamp => iot_timestamp(TimestampTkn, Msg, Nows),
measurement => emqx_placeholder:proc_tmpl(Measurement, Msg), measurement => emqx_placeholder:proc_tmpl(Measurement, Msg),
data_type => DataType, data_type => DataType,
value => proc_value(DataType, ValueTkn, Msg) value => Value
} }
end, | Acc
PreProcessedData ]);
). Error ->
Error
end;
proc_data([], _Msg, _Nows, Acc) ->
{ok, lists:reverse(Acc)}.
iot_timestamp(Timestamp, _, _) when is_integer(Timestamp) -> iot_timestamp(Timestamp, _, _) when is_integer(Timestamp) ->
Timestamp; Timestamp;
@ -441,16 +475,19 @@ iot_timestamp(Timestamp, _) when is_binary(Timestamp) ->
binary_to_integer(Timestamp). binary_to_integer(Timestamp).
proc_value(<<"TEXT">>, ValueTkn, Msg) -> proc_value(<<"TEXT">>, ValueTkn, Msg) ->
{ok,
case emqx_placeholder:proc_tmpl(ValueTkn, Msg) of case emqx_placeholder:proc_tmpl(ValueTkn, Msg) of
<<"undefined">> -> null; <<"undefined">> -> null;
Val -> Val Val -> Val
end; end};
proc_value(<<"BOOLEAN">>, ValueTkn, Msg) -> proc_value(<<"BOOLEAN">>, ValueTkn, Msg) ->
convert_bool(replace_var(ValueTkn, Msg)); {ok, convert_bool(replace_var(ValueTkn, Msg))};
proc_value(Int, ValueTkn, Msg) when Int =:= <<"INT32">>; Int =:= <<"INT64">> -> proc_value(Int, ValueTkn, Msg) when Int =:= <<"INT32">>; Int =:= <<"INT64">> ->
convert_int(replace_var(ValueTkn, Msg)); {ok, convert_int(replace_var(ValueTkn, Msg))};
proc_value(Int, ValueTkn, Msg) when Int =:= <<"FLOAT">>; Int =:= <<"DOUBLE">> -> proc_value(Int, ValueTkn, Msg) when Int =:= <<"FLOAT">>; Int =:= <<"DOUBLE">> ->
convert_float(replace_var(ValueTkn, Msg)). {ok, convert_float(replace_var(ValueTkn, Msg))};
proc_value(Type, _, _) ->
{error, {invalid_type, Type}}.
replace_var(Tokens, Data) when is_list(Tokens) -> replace_var(Tokens, Data) when is_list(Tokens) ->
[Val] = emqx_placeholder:proc_tmpl(Tokens, Data, #{return => rawlist}), [Val] = emqx_placeholder:proc_tmpl(Tokens, Data, #{return => rawlist}),
@ -495,18 +532,18 @@ convert_float(Str) when is_binary(Str) ->
convert_float(undefined) -> convert_float(undefined) ->
null. null.
make_iotdb_insert_request(DataList, IsAligned, DeviceId, IotDBVsn) -> make_iotdb_insert_request(DataList, IsAligned, DeviceId, IoTDBVsn) ->
InitAcc = #{timestamps => [], measurements => [], dtypes => [], values => []}, InitAcc = #{timestamps => [], measurements => [], dtypes => [], values => []},
Rows = replace_dtypes(aggregate_rows(DataList, InitAcc), IotDBVsn), Rows = replace_dtypes(aggregate_rows(DataList, InitAcc), IoTDBVsn),
{ok, {ok,
maps:merge(Rows, #{ maps:merge(Rows, #{
iotdb_field_key(is_aligned, IotDBVsn) => IsAligned, iotdb_field_key(is_aligned, IoTDBVsn) => IsAligned,
iotdb_field_key(device_id, IotDBVsn) => DeviceId iotdb_field_key(device_id, IoTDBVsn) => DeviceId
})}. })}.
replace_dtypes(Rows0, IotDBVsn) -> replace_dtypes(Rows0, IoTDBVsn) ->
{Types, Rows} = maps:take(dtypes, Rows0), {Types, Rows} = maps:take(dtypes, Rows0),
Rows#{iotdb_field_key(data_types, IotDBVsn) => Types}. Rows#{iotdb_field_key(data_types, IoTDBVsn) => Types}.
aggregate_rows(DataList, InitAcc) -> aggregate_rows(DataList, InitAcc) ->
lists:foldr( lists:foldr(
@ -612,9 +649,9 @@ eval_response_body(Body, Resp) ->
preproc_data_template(DataList) -> preproc_data_template(DataList) ->
Atom2Bin = fun Atom2Bin = fun
(Atom, Converter) when is_atom(Atom) -> (Atom) when is_atom(Atom) ->
Converter(Atom); erlang:atom_to_binary(Atom);
(Bin, _) -> (Bin) ->
Bin Bin
end, end,
lists:map( lists:map(
@ -627,33 +664,24 @@ preproc_data_template(DataList) ->
} }
) -> ) ->
#{ #{
timestamp => emqx_placeholder:preproc_tmpl( timestamp => emqx_placeholder:preproc_tmpl(Atom2Bin(Timestamp)),
Atom2Bin(Timestamp, fun erlang:atom_to_binary/1)
),
measurement => emqx_placeholder:preproc_tmpl(Measurement), measurement => emqx_placeholder:preproc_tmpl(Measurement),
data_type => emqx_placeholder:preproc_tmpl( data_type => emqx_placeholder:preproc_tmpl(Atom2Bin(DataType)),
Atom2Bin(
DataType,
fun(Atom) ->
erlang:list_to_binary(string:uppercase(erlang:atom_to_list(Atom)))
end
)
),
value => emqx_placeholder:preproc_tmpl(Value) value => emqx_placeholder:preproc_tmpl(Value)
} }
end, end,
DataList DataList
). ).
try_render_message({ChannelId, Msg}, Channels) -> try_render_message({ChannelId, Msg}, IoTDBVsn, Channels) ->
case maps:find(ChannelId, Channels) of case maps:find(ChannelId, Channels) of
{ok, Channel} -> {ok, Channel} ->
render_channel_message(Channel, Msg); render_channel_message(Channel, IoTDBVsn, Msg);
_ -> _ ->
{error, {unrecoverable_error, {invalid_channel_id, ChannelId}}} {error, {unrecoverable_error, {invalid_channel_id, ChannelId}}}
end. end.
render_channel_message(#{is_aligned := IsAligned, iotdb_version := IoTDBVsn} = Channel, Message) -> render_channel_message(#{is_aligned := IsAligned} = Channel, IoTDBVsn, Message) ->
Payloads = to_list(parse_payload(get_payload(Message))), Payloads = to_list(parse_payload(get_payload(Message))),
case device_id(Message, Payloads, Channel) of case device_id(Message, Payloads, Channel) of
undefined -> undefined ->
@ -663,9 +691,12 @@ render_channel_message(#{is_aligned := IsAligned, iotdb_version := IoTDBVsn} = C
[] -> [] ->
{error, invalid_data}; {error, invalid_data};
DataTemplate -> DataTemplate ->
DataList = proc_data(DataTemplate, Message), case proc_data(DataTemplate, Message) of
{ok, DataList} ->
make_iotdb_insert_request(DataList, IsAligned, DeviceId, IoTDBVsn) make_iotdb_insert_request(DataList, IsAligned, DeviceId, IoTDBVsn);
Error ->
Error
end
end end
end. end.

33
apps/emqx_bridge_iotdb/test/emqx_bridge_iotdb_impl_SUITE.erl
View File

@ -255,7 +255,6 @@ is_error_check(Reason) ->
end. end.
action_config(Name, Config) -> action_config(Name, Config) ->
Version = ?config(iotdb_version, Config),
Type = ?config(bridge_type, Config), Type = ?config(bridge_type, Config),
ConfigString = ConfigString =
io_lib:format( io_lib:format(
@ -263,15 +262,13 @@ action_config(Name, Config) ->
" enable = true\n" " enable = true\n"
" connector = \"~s\"\n" " connector = \"~s\"\n"
" parameters = {\n" " parameters = {\n"
" iotdb_version = \"~s\"\n"
" data = []\n" " data = []\n"
" }\n" " }\n"
"}\n", "}\n",
[ [
Type, Type,
Name, Name,
Name, Name
Version
] ]
), ),
ct:pal("ActionConfig:~ts~n", [ConfigString]), ct:pal("ActionConfig:~ts~n", [ConfigString]),
@ -281,12 +278,14 @@ connector_config(Name, Config) ->
Host = ?config(bridge_host, Config), Host = ?config(bridge_host, Config),
Port = ?config(bridge_port, Config), Port = ?config(bridge_port, Config),
Type = ?config(bridge_type, Config), Type = ?config(bridge_type, Config),
Version = ?config(iotdb_version, Config),
ServerURL = iotdb_server_url(Host, Port), ServerURL = iotdb_server_url(Host, Port),
ConfigString = ConfigString =
io_lib:format( io_lib:format(
"connectors.~s.~s {\n" "connectors.~s.~s {\n"
" enable = true\n" " enable = true\n"
" base_url = \"~s\"\n" " base_url = \"~s\"\n"
" iotdb_version = \"~s\"\n"
" authentication = {\n" " authentication = {\n"
" username = \"root\"\n" " username = \"root\"\n"
" password = \"root\"\n" " password = \"root\"\n"
@ -295,7 +294,8 @@ connector_config(Name, Config) ->
[ [
Type, Type,
Name, Name,
ServerURL ServerURL,
Version
] ]
), ),
ct:pal("ConnectorConfig:~ts~n", [ConfigString]), ct:pal("ConnectorConfig:~ts~n", [ConfigString]),
@ -646,6 +646,29 @@ t_template(Config) ->
iotdb_reset(Config, TemplateDeviceId), iotdb_reset(Config, TemplateDeviceId),
ok. ok.
t_sync_query_case(Config) ->
DeviceId = iotdb_device(Config),
Payload = make_iotdb_payload(DeviceId, "temp", "InT32", "36"),
MakeMessageFun = make_message_fun(iotdb_topic(Config), Payload),
ok = emqx_bridge_v2_testlib:t_sync_query(
Config, MakeMessageFun, fun is_success_check/1, iotdb_bridge_on_query
),
Query = <<"select temp from ", DeviceId/binary>>,
{ok, {{_, 200, _}, _, IoTDBResult}} = iotdb_query(Config, Query),
?assertMatch(
#{<<"values">> := [[36]]},
emqx_utils_json:decode(IoTDBResult)
).
t_sync_query_invalid_type(Config) ->
DeviceId = iotdb_device(Config),
Payload = make_iotdb_payload(DeviceId, "temp", "IxT32", "36"),
MakeMessageFun = make_message_fun(iotdb_topic(Config), Payload),
IsInvalidType = fun(Result) -> ?assertMatch({error, {invalid_type, _}}, Result) end,
ok = emqx_bridge_v2_testlib:t_sync_query(
Config, MakeMessageFun, IsInvalidType, iotdb_bridge_on_query
).
is_empty(null) -> true; is_empty(null) -> true;
is_empty([]) -> true; is_empty([]) -> true;
is_empty([[]]) -> true; is_empty([[]]) -> true;