Merge pull request #11301 from id/0719-sync-release-51-to-master

This commit is contained in:
Ivan Dyachkov 2023-07-19 14:21:26 +02:00 committed by GitHub
commit f29a9ed9d5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 550 additions and 152 deletions

View File

@ -17,8 +17,15 @@ jobs:
- uses: erlef/setup-beam@v1.15.4
with:
otp-version: 25.3.2
- name: Cache Jmeter
id: cache-jmeter
uses: actions/cache@v3
with:
path: /tmp/apache-jmeter.tgz
key: apache-jmeter-5.4.3.tgz
- name: download jmeter
timeout-minutes: 3
if: steps.cache-jmeter.outputs.cache-hit != 'true'
timeout-minutes: 15
env:
JMETER_VERSION: 5.4.3
run: |

View File

@ -16,7 +16,7 @@ endif
# Dashboard version
# from https://github.com/emqx/emqx-dashboard5
export EMQX_DASHBOARD_VERSION ?= v1.3.2
export EMQX_EE_DASHBOARD_VERSION ?= e1.1.1-beta.3
export EMQX_EE_DASHBOARD_VERSION ?= e1.1.1-beta.4
# `:=` should be used here, otherwise the `$(shell ...)` will be executed every time when the variable is used
# In make 4.4+, for backward-compatibility the value from the original environment is used.

View File

@ -33,12 +33,15 @@
-define(ERTS_MINIMUM_REQUIRED, "10.0").
%%--------------------------------------------------------------------
%% Topics' prefix: $SYS | $share
%% Topics' prefix: $SYS | $queue | $share
%%--------------------------------------------------------------------
%% System topic
-define(SYSTOP, <<"$SYS/">>).
%% Queue topic
-define(QUEUE, <<"$queue/">>).
%%--------------------------------------------------------------------
%% alarms
%%--------------------------------------------------------------------

View File

@ -35,7 +35,7 @@
-define(EMQX_RELEASE_CE, "5.1.1").
%% Enterprise edition
-define(EMQX_RELEASE_EE, "5.1.1-alpha.1").
-define(EMQX_RELEASE_EE, "5.1.1-alpha.2").
%% The HTTP API version
-define(EMQX_API_VERSION, "5.0").

View File

@ -455,7 +455,7 @@ handle_in(
NChannel = Channel#channel{session = NSession},
handle_out(pubcomp, {PacketId, ?RC_SUCCESS}, NChannel);
{error, RC = ?RC_PACKET_IDENTIFIER_NOT_FOUND} ->
?SLOG(warning, #{msg => "pubrec_packetId_not_found", packetId => PacketId}),
?SLOG(warning, #{msg => "pubrel_packetId_not_found", packetId => PacketId}),
ok = emqx_metrics:inc('packets.pubrel.missed'),
handle_out(pubcomp, {PacketId, RC}, Channel)
end;

View File

@ -2504,7 +2504,11 @@ to_integer(Str) ->
end.
to_percent(Str) ->
{ok, hocon_postprocess:percent(Str)}.
Percent = hocon_postprocess:percent(Str),
case is_number(Percent) andalso Percent >= 0.0 andalso Percent =< 1.0 of
true -> {ok, Percent};
false -> {error, Str}
end.
to_comma_separated_list(Str) ->
{ok, string:tokens(Str, ", ")}.
@ -2732,15 +2736,17 @@ check_cpu_watermark(Conf) ->
check_watermark("sysmon.os.cpu_low_watermark", "sysmon.os.cpu_high_watermark", Conf).
check_watermark(LowKey, HighKey, Conf) ->
case hocon_maps:get(LowKey, Conf) of
undefined ->
case to_percent(hocon_maps:get(LowKey, Conf)) of
{error, undefined} ->
true;
Low ->
High = hocon_maps:get(HighKey, Conf),
case Low < High of
true -> true;
false -> {bad_watermark, #{LowKey => Low, HighKey => High}}
end
{ok, Low} ->
case to_percent(hocon_maps:get(HighKey, Conf)) of
{ok, High} when High > Low -> true;
{ok, High} -> {bad_watermark, #{LowKey => Low, HighKey => High}};
{error, HighVal} -> {bad_watermark, #{HighKey => HighVal}}
end;
{error, Low} ->
{bad_watermark, #{LowKey => Low}}
end.
str(A) when is_atom(A) ->

View File

@ -244,8 +244,12 @@ parse({TopicFilter, Options}) when is_binary(TopicFilter) ->
parse(TopicFilter, Options).
-spec parse(topic(), map()) -> {topic(), map()}.
parse(TopicFilter = <<"$queue/", _/binary>>, #{share := _Group}) ->
error({invalid_topic_filter, TopicFilter});
parse(TopicFilter = <<"$share/", _/binary>>, #{share := _Group}) ->
error({invalid_topic_filter, TopicFilter});
parse(<<"$queue/", TopicFilter/binary>>, Options) ->
parse(TopicFilter, Options#{share => <<"$queue">>});
parse(TopicFilter = <<"$share/", Rest/binary>>, Options) ->
case binary:split(Rest, <<"/">>) of
[_Any] ->

View File

@ -444,7 +444,7 @@ systopic_mon() ->
sharetopic() ->
?LET(
{Type, Grp, T},
{oneof([<<"$share">>]), list(latin_char()), normal_topic()},
{oneof([<<"$queue">>, <<"$share">>]), list(latin_char()), normal_topic()},
<<Type/binary, "/", (iolist_to_binary(Grp))/binary, "/", T/binary>>
).

View File

@ -20,8 +20,10 @@
-compile(nowarn_export_all).
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-define(SUITE, ?MODULE).
@ -986,6 +988,112 @@ t_session_kicked(Config) when is_list(Config) ->
?assertEqual([], collect_msgs(0)),
ok.
%% FIXME: currently doesn't work
%% t_different_groups_same_topic({init, Config}) ->
%% TestName = atom_to_binary(?FUNCTION_NAME),
%% ClientId = <<TestName/binary, (integer_to_binary(erlang:unique_integer()))/binary>>,
%% {ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]),
%% {ok, _} = emqtt:connect(C),
%% [{client, C}, {clientid, ClientId} | Config];
%% t_different_groups_same_topic({'end', Config}) ->
%% C = ?config(client, Config),
%% emqtt:stop(C),
%% ok;
%% t_different_groups_same_topic(Config) when is_list(Config) ->
%% C = ?config(client, Config),
%% ClientId = ?config(clientid, Config),
%% %% Subscribe and unsubscribe to both $queue and $shared topics
%% Topic = <<"t/1">>,
%% SharedTopic0 = <<"$share/aa/", Topic/binary>>,
%% SharedTopic1 = <<"$share/bb/", Topic/binary>>,
%% {ok, _, [2]} = emqtt:subscribe(C, {SharedTopic0, 2}),
%% {ok, _, [2]} = emqtt:subscribe(C, {SharedTopic1, 2}),
%% Message0 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hi">>),
%% emqx:publish(Message0),
%% ?assertMatch([ {publish, #{payload := <<"hi">>}}
%% , {publish, #{payload := <<"hi">>}}
%% ], collect_msgs(5_000), #{routes => ets:tab2list(emqx_route)}),
%% {ok, _, [0]} = emqtt:unsubscribe(C, SharedTopic0),
%% {ok, _, [0]} = emqtt:unsubscribe(C, SharedTopic1),
%% ok.
t_queue_subscription({init, Config}) ->
TestName = atom_to_binary(?FUNCTION_NAME),
ClientId = <<TestName/binary, (integer_to_binary(erlang:unique_integer()))/binary>>,
{ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]),
{ok, _} = emqtt:connect(C),
[{client, C}, {clientid, ClientId} | Config];
t_queue_subscription({'end', Config}) ->
C = ?config(client, Config),
emqtt:stop(C),
ok;
t_queue_subscription(Config) when is_list(Config) ->
C = ?config(client, Config),
ClientId = ?config(clientid, Config),
%% Subscribe and unsubscribe to both $queue and $shared topics
Topic = <<"t/1">>,
QueueTopic = <<"$queue/", Topic/binary>>,
SharedTopic = <<"$share/aa/", Topic/binary>>,
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(C, {QueueTopic, 2}),
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(C, {SharedTopic, 2}),
%% FIXME: we should actually see 2 routes, one for each group
%% ($queue and aa), but currently the latest subscription
%% overwrites the existing one.
?retry(
_Sleep0 = 100,
_Attempts0 = 50,
begin
ct:pal("routes: ~p", [ets:tab2list(emqx_route)]),
%% FIXME: should ensure we have 2 subscriptions
true = emqx_router:has_routes(Topic)
end
),
%% now publish to the underlying topic
Message0 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hi">>),
emqx:publish(Message0),
?assertMatch(
[
{publish, #{payload := <<"hi">>}}
%% FIXME: should receive one message from each group
%% , {publish, #{payload := <<"hi">>}}
],
collect_msgs(5_000)
),
{ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, QueueTopic),
%% FIXME: return code should be success instead of 17 ("no_subscription_existed")
{ok, _, [?RC_NO_SUBSCRIPTION_EXISTED]} = emqtt:unsubscribe(C, SharedTopic),
%% FIXME: this should eventually be true, but currently we leak
%% the previous group subscription...
%% ?retry(
%% _Sleep0 = 100,
%% _Attempts0 = 50,
%% begin
%% ct:pal("routes: ~p", [ets:tab2list(emqx_route)]),
%% false = emqx_router:has_routes(Topic)
%% end
%% ),
ct:sleep(500),
Message1 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hello">>),
emqx:publish(Message1),
%% FIXME: we should *not* receive any messages...
%% ?assertEqual([], collect_msgs(1_000), #{routes => ets:tab2list(emqx_route)}),
%% This is from the leaked group...
?assertMatch([{publish, #{topic := Topic}}], collect_msgs(1_000), #{
routes => ets:tab2list(emqx_route)
}),
ok.
%%--------------------------------------------------------------------
%% help functions
%%--------------------------------------------------------------------

View File

@ -211,6 +211,10 @@ t_systop(_) ->
?assertEqual(SysTop2, systop(<<"abc">>)).
t_feed_var(_) ->
?assertEqual(
<<"$queue/client/clientId">>,
feed_var(<<"$c">>, <<"clientId">>, <<"$queue/client/$c">>)
),
?assertEqual(
<<"username/test/client/x">>,
feed_var(
@ -232,6 +236,10 @@ long_topic() ->
iolist_to_binary([[integer_to_list(I), "/"] || I <- lists:seq(0, 66666)]).
t_parse(_) ->
?assertError(
{invalid_topic_filter, <<"$queue/t">>},
parse(<<"$queue/t">>, #{share => <<"g">>})
),
?assertError(
{invalid_topic_filter, <<"$share/g/t">>},
parse(<<"$share/g/t">>, #{share => <<"g">>})
@ -246,9 +254,11 @@ t_parse(_) ->
),
?assertEqual({<<"a/b/+/#">>, #{}}, parse(<<"a/b/+/#">>)),
?assertEqual({<<"a/b/+/#">>, #{qos => 1}}, parse({<<"a/b/+/#">>, #{qos => 1}})),
?assertEqual({<<"topic">>, #{share => <<"$queue">>}}, parse(<<"$queue/topic">>)),
?assertEqual({<<"topic">>, #{share => <<"group">>}}, parse(<<"$share/group/topic">>)),
%% The '$local' and '$fastlane' topics have been deprecated.
?assertEqual({<<"$local/topic">>, #{}}, parse(<<"$local/topic">>)),
?assertEqual({<<"$local/$queue/topic">>, #{}}, parse(<<"$local/$queue/topic">>)),
?assertEqual({<<"$local/$share/group/a/b/c">>, #{}}, parse(<<"$local/$share/group/a/b/c">>)),
?assertEqual({<<"$fastlane/topic">>, #{}}, parse(<<"$fastlane/topic">>)).

View File

@ -988,15 +988,10 @@ call_operation(NodeOrAll, OperFunc, Args = [_Nodes, BridgeType, BridgeName]) ->
%% still on an older bpapi version that doesn't support it.
maybe_try_restart(NodeOrAll, OperFunc, Args);
{error, timeout} ->
?SERVICE_UNAVAILABLE(<<"Request timeout">>);
?BAD_REQUEST(<<"Request timeout">>);
{error, {start_pool_failed, Name, Reason}} ->
Msg = bin(io_lib:format("Failed to start ~p pool for reason ~p", [Name, Reason])),
case Reason of
nxdomain ->
?BAD_REQUEST(Msg);
_ ->
?SERVICE_UNAVAILABLE(Msg)
end;
?BAD_REQUEST(Msg);
{error, not_found} ->
BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName),
?SLOG(warning, #{

View File

@ -834,7 +834,8 @@ do_start_stop_bridges(Type, Config) ->
),
BadBridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_MQTT, BadName),
?assertMatch(
{ok, SC, _} when SC == 500 orelse SC == 503,
%% request from product: return 400 on such errors
{ok, SC, _} when SC == 500 orelse SC == 400,
request(post, {operation, Type, start, BadBridgeID}, Config)
),
ok = gen_tcp:close(Sock),

View File

@ -10,6 +10,8 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-import(emqx_common_test_helpers, [on_exit/1]).
%% ct setup helpers
init_per_suite(Config, Apps) ->
@ -211,19 +213,27 @@ probe_bridge_api(BridgeType, BridgeName, BridgeConfig) ->
Res.
create_rule_and_action_http(BridgeType, RuleTopic, Config) ->
create_rule_and_action_http(BridgeType, RuleTopic, Config, _Opts = #{}).
create_rule_and_action_http(BridgeType, RuleTopic, Config, Opts) ->
BridgeName = ?config(bridge_name, Config),
BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName),
SQL = maps:get(sql, Opts, <<"SELECT * FROM \"", RuleTopic/binary, "\"">>),
Params = #{
enable => true,
sql => <<"SELECT * FROM \"", RuleTopic/binary, "\"">>,
sql => SQL,
actions => [BridgeId]
},
Path = emqx_mgmt_api_test_util:api_path(["rules"]),
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
ct:pal("rule action params: ~p", [Params]),
case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of
{ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])};
Error -> Error
{ok, Res0} ->
Res = #{<<"id">> := RuleId} = emqx_utils_json:decode(Res0, [return_maps]),
on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end),
{ok, Res};
Error ->
Error
end.
%%------------------------------------------------------------------------------

View File

@ -119,10 +119,13 @@ roots() ->
fields(config) ->
[
{url, mk(binary(), #{required => true, desc => ?DESC("url")})},
{url,
mk(binary(), #{
required => true, desc => ?DESC("url"), default => <<"http://127.0.0.1:6570">>
})},
{stream, mk(binary(), #{required => true, desc => ?DESC("stream_name")})},
{partition_key, mk(binary(), #{required => false, desc => ?DESC("partition_key")})},
{pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})},
{pool_size, fun emqx_connector_schema_lib:pool_size/1},
{grpc_timeout, fun grpc_timeout/1}
] ++ emqx_connector_schema_lib:ssl_fields().

View File

@ -379,6 +379,41 @@ t_sync_device_id_missing(Config) ->
iotdb_bridge_on_query
).
t_extract_device_id_from_rule_engine_message(Config) ->
BridgeType = ?config(bridge_type, Config),
RuleTopic = <<"t/iotdb">>,
DeviceId = iotdb_device(Config),
Payload = make_iotdb_payload(DeviceId, "temp", "INT32", "12"),
Message = emqx_message:make(RuleTopic, emqx_utils_json:encode(Payload)),
?check_trace(
begin
{ok, _} = emqx_bridge_testlib:create_bridge(Config),
SQL = <<
"SELECT\n"
" payload.measurement, payload.data_type, payload.value, payload.device_id\n"
"FROM\n"
" \"",
RuleTopic/binary,
"\""
>>,
Opts = #{sql => SQL},
{ok, _} = emqx_bridge_testlib:create_rule_and_action_http(
BridgeType, RuleTopic, Config, Opts
),
emqx:publish(Message),
?block_until(handle_async_reply, 5_000),
ok
end,
fun(Trace) ->
?assertMatch(
[#{action := ack, result := {ok, 200, _, _}}],
?of_kind(handle_async_reply, Trace)
),
ok
end
),
ok.
t_sync_invalid_data(Config) ->
emqx_bridge_testlib:t_sync_query(
Config,

View File

@ -23,7 +23,7 @@
-define(DEFAULT_SQL, <<
"insert into t_mqtt_msg(ts, msgid, mqtt_topic, qos, payload, arrived) "
"values (${ts}, ${id}, ${topic}, ${qos}, ${payload}, ${timestamp})"
"values (${ts}, '${id}', '${topic}', ${qos}, '${payload}', ${timestamp})"
>>).
%% -------------------------------------------------------------------------------------------------

View File

@ -124,7 +124,7 @@ on_query(InstanceId, {query, SQL}, State) ->
on_query(InstanceId, {Key, Data}, #{insert_tokens := InsertTksMap} = State) ->
case maps:find(Key, InsertTksMap) of
{ok, Tokens} when is_map(Data) ->
SQL = emqx_placeholder:proc_sql_param_str(Tokens, Data),
SQL = emqx_placeholder:proc_tmpl(Tokens, Data),
do_query(InstanceId, SQL, State);
_ ->
{error, {unrecoverable_error, invalid_request}}
@ -209,31 +209,16 @@ execute(Conn, Query, Opts) ->
tdengine:insert(Conn, Query, Opts).
do_batch_insert(Conn, Tokens, BatchReqs, Opts) ->
Queries = aggregate_query(Tokens, BatchReqs),
SQL = maps:fold(
fun(InsertPart, Values, Acc) ->
lists:foldl(
fun(ValuePart, IAcc) ->
<<IAcc/binary, " ", ValuePart/binary>>
end,
<<Acc/binary, " ", InsertPart/binary, " VALUES">>,
Values
)
end,
<<"INSERT INTO">>,
Queries
),
SQL = aggregate_query(Tokens, BatchReqs, <<"INSERT INTO">>),
execute(Conn, SQL, Opts).
aggregate_query({InsertPartTks, ParamsPartTks}, BatchReqs) ->
aggregate_query(BatchTks, BatchReqs, Acc) ->
lists:foldl(
fun({_, Data}, Acc) ->
InsertPart = emqx_placeholder:proc_sql_param_str(InsertPartTks, Data),
ParamsPart = emqx_placeholder:proc_sql_param_str(ParamsPartTks, Data),
Values = maps:get(InsertPart, Acc, []),
maps:put(InsertPart, [ParamsPart | Values], Acc)
fun({_, Data}, InAcc) ->
InsertPart = emqx_placeholder:proc_tmpl(BatchTks, Data),
<<InAcc/binary, " ", InsertPart/binary>>
end,
#{},
Acc,
BatchReqs
).
@ -260,13 +245,12 @@ parse_batch_prepare_sql([{Key, H} | T], InsertTksMap, BatchTksMap) ->
InsertTks = emqx_placeholder:preproc_tmpl(H),
H1 = string:trim(H, trailing, ";"),
case split_insert_sql(H1) of
[_InsertStr, InsertPart, _ValuesStr, ParamsPart] ->
InsertPartTks = emqx_placeholder:preproc_tmpl(InsertPart),
ParamsPartTks = emqx_placeholder:preproc_tmpl(ParamsPart),
[_InsertPart, BatchDesc] ->
BatchTks = emqx_placeholder:preproc_tmpl(BatchDesc),
parse_batch_prepare_sql(
T,
InsertTksMap#{Key => InsertTks},
BatchTksMap#{Key => {InsertPartTks, ParamsPartTks}}
BatchTksMap#{Key => BatchTks}
);
Result ->
?SLOG(error, #{msg => "split sql failed", sql => H, result => Result}),
@ -299,7 +283,7 @@ split_insert_sql(SQL0) ->
{true, E1}
end
end,
re:split(SQL, "(?i)(insert into)|(?i)(values)")
re:split(SQL, "(?i)(insert into)")
).
formalize_sql(Input) ->

View File

@ -13,7 +13,8 @@
% SQL definitions
-define(SQL_BRIDGE,
"insert into mqtt.t_mqtt_msg(ts, payload) values (${timestamp}, ${payload})"
"insert into t_mqtt_msg(ts, payload) values (${timestamp}, '${payload}')"
"t_mqtt_msg(ts, payload) values (${second_ts}, '${payload}')"
).
-define(SQL_CREATE_DATABASE, "CREATE DATABASE IF NOT EXISTS mqtt; USE mqtt;").
@ -29,7 +30,8 @@
-define(SQL_SELECT, "SELECT payload FROM t_mqtt_msg").
-define(AUTO_CREATE_BRIDGE,
"insert into ${clientid} USING s_tab TAGS (${clientid}) values (${timestamp}, ${payload})"
"insert into ${clientid} USING s_tab TAGS ('${clientid}') values (${timestamp}, '${payload}')"
"test_${clientid} USING s_tab TAGS ('${clientid}') values (${second_ts}, '${payload}')"
).
-define(SQL_CREATE_STABLE,
@ -301,7 +303,7 @@ connect_and_clear_table(Config) ->
connect_and_get_payload(Config) ->
?WITH_CON(
{ok, #{<<"code">> := 0, <<"data">> := [[Result]]}} = directly_query(Con, ?SQL_SELECT)
{ok, #{<<"code">> := 0, <<"data">> := Result}} = directly_query(Con, ?SQL_SELECT)
),
Result.
@ -329,7 +331,7 @@ t_setup_via_config_and_publish(Config) ->
{ok, _},
create_bridge(Config)
),
SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000},
SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000, second_ts => 1668602148010},
?check_trace(
begin
{_, {ok, #{result := Result}}} =
@ -342,7 +344,7 @@ t_setup_via_config_and_publish(Config) ->
{ok, #{<<"code">> := 0, <<"rows">> := 1}}, Result
),
?assertMatch(
?PAYLOAD,
[[?PAYLOAD], [?PAYLOAD]],
connect_and_get_payload(Config)
),
ok
@ -368,7 +370,8 @@ t_setup_via_http_api_and_publish(Config) ->
{ok, _},
create_bridge_http(TDengineConfig)
),
SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000},
SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000, second_ts => 1668602148010},
?check_trace(
begin
Request = {send_message, SentData},
@ -386,7 +389,7 @@ t_setup_via_http_api_and_publish(Config) ->
{ok, #{<<"code">> := 0, <<"rows">> := 1}}, Res0
),
?assertMatch(
?PAYLOAD,
[[?PAYLOAD], [?PAYLOAD]],
connect_and_get_payload(Config)
),
ok
@ -426,7 +429,7 @@ t_write_failure(Config) ->
ProxyPort = ?config(proxy_port, Config),
ProxyHost = ?config(proxy_host, Config),
{ok, _} = create_bridge(Config),
SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000},
SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000, second_ts => 1668602148010},
emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() ->
{_, {ok, #{result := Result}}} =
?wait_async_action(
@ -461,7 +464,7 @@ t_write_timeout(Config) ->
}
}
),
SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000},
SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000, second_ts => 1668602148010},
%% FIXME: TDengine connector hangs indefinetily during
%% `call_query' while the connection is unresponsive. Should add
%% a timeout to `APPLY_RESOURCE' in buffer worker??
@ -486,7 +489,7 @@ t_simple_sql_query(Config) ->
{ok, _},
create_bridge(Config)
),
Request = {query, <<"SELECT count(1) AS T">>},
Request = {query, <<"SELECT 1 AS T">>},
{_, {ok, #{result := Result}}} =
?wait_async_action(
query_resource(Config, Request),
@ -537,37 +540,41 @@ t_bad_sql_parameter(Config) ->
?assertMatch({error, {unrecoverable_error, invalid_request}}, Result),
ok.
t_nasty_sql_string(Config) ->
?assertMatch(
{ok, _},
create_bridge(Config)
),
% NOTE
% Column `payload` has BINARY type, so we would certainly like to test it
% with `lists:seq(1, 127)`, but:
% 1. There's no way to insert zero byte in an SQL string, seems that TDengine's
% parser[1] has no escaping sequence for it so a zero byte probably confuses
% interpreter somewhere down the line.
% 2. Bytes > 127 come back as U+FFFDs (i.e. replacement characters) in UTF-8 for
% some reason.
%
% [1]: https://github.com/taosdata/TDengine/blob/066cb34a/source/libs/parser/src/parUtil.c#L279-L301
Payload = list_to_binary(lists:seq(1, 127)),
Message = #{payload => Payload, timestamp => erlang:system_time(millisecond)},
{_, {ok, #{result := Result}}} =
?wait_async_action(
send_message(Config, Message),
#{?snk_kind := buffer_worker_flush_ack},
2_000
),
?assertMatch(
{ok, #{<<"code">> := 0, <<"rows">> := 1}},
Result
),
?assertEqual(
Payload,
connect_and_get_payload(Config)
).
%% TODO
%% For supporting to generate a subtable name by mixing prefixes/suffixes with placeholders,
%% the SQL quote(escape) is removed now,
%% we should introduce a new syntax for placeholders to allow some vars to keep unquote.
%% t_nasty_sql_string(Config) ->
%% ?assertMatch(
%% {ok, _},
%% create_bridge(Config)
%% ),
%% % NOTE
%% % Column `payload` has BINARY type, so we would certainly like to test it
%% % with `lists:seq(1, 127)`, but:
%% % 1. There's no way to insert zero byte in an SQL string, seems that TDengine's
%% % parser[1] has no escaping sequence for it so a zero byte probably confuses
%% % interpreter somewhere down the line.
%% % 2. Bytes > 127 come back as U+FFFDs (i.e. replacement characters) in UTF-8 for
%% % some reason.
%% %
%% % [1]: https://github.com/taosdata/TDengine/blob/066cb34a/source/libs/parser/src/parUtil.c#L279-L301
%% Payload = list_to_binary(lists:seq(1, 127)),
%% Message = #{payload => Payload, timestamp => erlang:system_time(millisecond)},
%% {_, {ok, #{result := Result}}} =
%% ?wait_async_action(
%% send_message(Config, Message),
%% #{?snk_kind := buffer_worker_flush_ack},
%% 2_000
%% ),
%% ?assertMatch(
%% {ok, #{<<"code">> := 0, <<"rows">> := 1}},
%% Result
%% ),
%% ?assertEqual(
%% Payload,
%% connect_and_get_payload(Config)
%% ).
t_simple_insert(Config) ->
connect_and_clear_table(Config),
@ -576,7 +583,7 @@ t_simple_insert(Config) ->
create_bridge(Config)
),
SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000},
SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000, second_ts => 1668602148010},
Request = {send_message, SentData},
{_, {ok, #{result := _Result}}} =
?wait_async_action(
@ -585,7 +592,7 @@ t_simple_insert(Config) ->
2_000
),
?assertMatch(
?PAYLOAD,
[[?PAYLOAD], [?PAYLOAD]],
connect_and_get_payload(Config)
).
@ -602,7 +609,9 @@ t_batch_insert(Config) ->
?wait_async_action(
lists:foreach(
fun(Idx) ->
SentData = #{payload => ?PAYLOAD, timestamp => Ts + Idx},
SentData = #{
payload => ?PAYLOAD, timestamp => Ts + Idx, second_ts => Ts + Idx + 5000
},
Request = {send_message, SentData},
query_resource(Config, Request)
end,
@ -613,11 +622,12 @@ t_batch_insert(Config) ->
2_000
),
DoubleSize = Size * 2,
?retry(
_Sleep = 50,
_Attempts = 30,
?assertMatch(
[[Size]],
[[DoubleSize]],
connect_and_query(Config, "SELECT COUNT(1) FROM t_mqtt_msg")
)
).
@ -633,6 +643,7 @@ t_auto_create_simple_insert(Config0) ->
SentData = #{
payload => ?PAYLOAD,
timestamp => 1668602148000,
second_ts => 1668602148000 + 100,
clientid => ClientId
},
Request = {send_message, SentData},
@ -647,9 +658,19 @@ t_auto_create_simple_insert(Config0) ->
connect_and_query(Config, "SELECT payload FROM " ++ ClientId)
),
?assertMatch(
[[?PAYLOAD]],
connect_and_query(Config, "SELECT payload FROM test_" ++ ClientId)
),
?assertMatch(
[[0]],
connect_and_query(Config, "DROP TABLE " ++ ClientId)
),
?assertMatch(
[[0]],
connect_and_query(Config, "DROP TABLE test_" ++ ClientId)
).
t_auto_create_batch_insert(Config0) ->
@ -675,6 +696,7 @@ t_auto_create_batch_insert(Config0) ->
SentData = #{
payload => ?PAYLOAD,
timestamp => Ts + Idx + Offset,
second_ts => Ts + Idx + Offset + 5000,
clientid => ClientId
},
Request = {send_message, SentData},
@ -693,29 +715,28 @@ t_auto_create_batch_insert(Config0) ->
_Sleep = 50,
_Attempts = 30,
?assertMatch(
[[Size1]],
connect_and_query(Config, "SELECT COUNT(1) FROM " ++ ClientId1)
lists:foreach(
fun({Table, Size}) ->
?assertMatch(
[[Size]],
connect_and_query(Config, "SELECT COUNT(1) FROM " ++ Table)
)
end,
lists:zip(
[ClientId1, "test_" ++ ClientId1, ClientId2, "test_" ++ ClientId2],
[Size1, Size1, Size2, Size2]
)
)
),
?retry(
50,
30,
?assertMatch(
[[Size2]],
connect_and_query(Config, "SELECT COUNT(1) FROM " ++ ClientId2)
)
),
?assertMatch(
[[0]],
connect_and_query(Config, "DROP TABLE " ++ ClientId1)
),
?assertMatch(
[[0]],
connect_and_query(Config, "DROP TABLE " ++ ClientId2)
lists:foreach(
fun(E) ->
?assertMatch(
[[0]],
connect_and_query(Config, "DROP TABLE " ++ E)
)
end,
[ClientId1, ClientId2, "test_" ++ ClientId1, "test_" ++ ClientId2]
).
to_bin(List) when is_list(List) ->

View File

@ -332,8 +332,8 @@ load_etc_config_file() ->
filter_readonly_config(Raw) ->
SchemaMod = emqx_conf:schema_module(),
RawDefault = fill_defaults(Raw),
try
RawDefault = fill_defaults(Raw),
_ = emqx_config:check_config(SchemaMod, RawDefault),
ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS],
{ok, maps:without(ReadOnlyKeys, Raw)}

View File

@ -86,8 +86,7 @@ roots() ->
sc(
?R_REF("node"),
#{
translate_to => ["emqx"],
converter => fun node_converter/2
translate_to => ["emqx"]
}
)},
{"cluster",
@ -446,9 +445,11 @@ fields("node") ->
sc(
range(1024, 134217727),
#{
mapping => "vm_args.+P",
%% deprecated make sure it's disappeared in raw_conf user(HTTP API)
%% but still in vm.args via translation/1
%% ProcessLimit is always equal to MaxPort * 2 when translation/1.
deprecated => true,
desc => ?DESC(process_limit),
default => ?DEFAULT_MAX_PORTS * 2,
importance => ?IMPORTANCE_HIDDEN,
'readOnly' => true
}
@ -1052,7 +1053,7 @@ desc("authorization") ->
desc(_) ->
undefined.
translations() -> ["ekka", "kernel", "emqx", "gen_rpc", "prometheus"].
translations() -> ["ekka", "kernel", "emqx", "gen_rpc", "prometheus", "vm_args"].
translation("ekka") ->
[{"cluster_discovery", fun tr_cluster_discovery/1}];
@ -1079,8 +1080,15 @@ translation("prometheus") ->
{"vm_system_info_collector_metrics", fun tr_vm_system_info_collector/1},
{"vm_memory_collector_metrics", fun tr_vm_memory_collector/1},
{"vm_msacc_collector_metrics", fun tr_vm_msacc_collector/1}
];
translation("vm_args") ->
[
{"+P", fun tr_vm_args_process_limit/1}
].
tr_vm_args_process_limit(Conf) ->
2 * conf_get("node.max_ports", Conf, ?DEFAULT_MAX_PORTS).
tr_vm_dist_collector(Conf) ->
metrics_enabled(conf_get("prometheus.vm_dist_collector", Conf, enabled)).
@ -1395,10 +1403,3 @@ ensure_unicode_path(Path, _) when is_list(Path) ->
Path;
ensure_unicode_path(Path, _) ->
throw({"not_string", Path}).
node_converter(#{<<"process_limit">> := _} = Conf, _Opts) ->
Conf;
node_converter(#{<<"max_ports">> := MaxPorts} = Conf, _Opts) ->
Conf#{<<"process_limit">> => MaxPorts * 2};
node_converter(Conf, _Opts) ->
Conf.

View File

@ -25,6 +25,8 @@
name = \"emqx1@127.0.0.1\"
cookie = \"emqxsecretcookie\"
data_dir = \"data\"
max_ports = 2048
process_limit = 10240
}
cluster {
name = emqxcl
@ -42,6 +44,12 @@ array_nodes_test() ->
ConfFile = to_bin(?BASE_CONF, [Nodes, Nodes]),
{ok, Conf} = hocon:binary(ConfFile, #{format => richmap}),
ConfList = hocon_tconf:generate(emqx_conf_schema, Conf),
VMArgs = proplists:get_value(vm_args, ConfList),
ProcLimit = proplists:get_value('+P', VMArgs),
MaxPort = proplists:get_value('+Q', VMArgs),
?assertEqual(2048, MaxPort),
?assertEqual(MaxPort * 2, ProcLimit),
ClusterDiscovery = proplists:get_value(
cluster_discovery, proplists:get_value(ekka, ConfList)
),

View File

@ -187,6 +187,8 @@ format(WhichNode, {{Topic, _Subscriber}, Options}) ->
maps:with([qos, nl, rap, rh], Options)
).
get_topic(Topic, #{share := <<"$queue">> = Group}) ->
emqx_topic:join([Group, Topic]);
get_topic(Topic, #{share := Group}) ->
emqx_topic:join([<<"$share">>, Group, Topic]);
get_topic(Topic, _) ->

View File

@ -23,7 +23,6 @@
-export([
apply_rule/3,
apply_rules/3,
clear_rule_payload/0,
inc_action_metrics/2
]).
@ -196,18 +195,18 @@ select_and_transform([], _Columns, Action) ->
select_and_transform(['*' | More], Columns, Action) ->
select_and_transform(More, Columns, maps:merge(Action, Columns));
select_and_transform([{as, Field, Alias} | More], Columns, Action) ->
Val = eval(Field, Columns),
Val = eval(Field, [Action, Columns]),
select_and_transform(
More,
nested_put(Alias, Val, Columns),
Columns,
nested_put(Alias, Val, Action)
);
select_and_transform([Field | More], Columns, Action) ->
Val = eval(Field, Columns),
Val = eval(Field, [Action, Columns]),
Key = alias(Field, Columns),
select_and_transform(
More,
nested_put(Key, Val, Columns),
Columns,
nested_put(Key, Val, Action)
).
@ -217,25 +216,25 @@ select_and_collect(Fields, Columns) ->
select_and_collect(Fields, Columns, {#{}, {'item', []}}).
select_and_collect([{as, Field, {_, A} = Alias}], Columns, {Action, _}) ->
Val = eval(Field, Columns),
Val = eval(Field, [Action, Columns]),
{nested_put(Alias, Val, Action), {A, ensure_list(Val)}};
select_and_collect([{as, Field, Alias} | More], Columns, {Action, LastKV}) ->
Val = eval(Field, Columns),
Val = eval(Field, [Action, Columns]),
select_and_collect(
More,
nested_put(Alias, Val, Columns),
{nested_put(Alias, Val, Action), LastKV}
);
select_and_collect([Field], Columns, {Action, _}) ->
Val = eval(Field, Columns),
Val = eval(Field, [Action, Columns]),
Key = alias(Field, Columns),
{nested_put(Key, Val, Action), {'item', ensure_list(Val)}};
select_and_collect([Field | More], Columns, {Action, LastKV}) ->
Val = eval(Field, Columns),
Val = eval(Field, [Action, Columns]),
Key = alias(Field, Columns),
select_and_collect(
More,
nested_put(Key, Val, Columns),
Columns,
{nested_put(Key, Val, Action), LastKV}
).
@ -368,6 +367,16 @@ do_handle_action(RuleId, #{mod := Mod, func := Func, args := Args}, Selected, En
inc_action_metrics(RuleId, Result),
Result.
eval({Op, _} = Exp, Context) when is_list(Context) andalso (Op == path orelse Op == var) ->
case Context of
[Columns] ->
eval(Exp, Columns);
[Columns | Rest] ->
case eval(Exp, Columns) of
undefined -> eval(Exp, Rest);
Val -> Val
end
end;
eval({path, [{key, <<"payload">>} | Path]}, #{payload := Payload}) ->
nested_get({path, Path}, may_decode_payload(Payload));
eval({path, [{key, <<"payload">>} | Path]}, #{<<"payload">> := Payload}) ->

View File

@ -21,6 +21,7 @@
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("emqx/include/emqx.hrl").
@ -583,6 +584,122 @@ t_ensure_action_removed(_) ->
%% Test cases for rule runtime
%%------------------------------------------------------------------------------
t_json_payload_decoding(_Config) ->
{ok, C} = emqtt:start_link(),
on_exit(fun() -> emqtt:stop(C) end),
{ok, _} = emqtt:connect(C),
Cases =
[
#{
select_fields =>
<<"payload.measurement, payload.data_type, payload.value, payload.device_id">>,
payload => emqx_utils_json:encode(#{
measurement => <<"temp">>,
data_type => <<"FLOAT">>,
value => <<"32.12">>,
device_id => <<"devid">>
}),
expected => #{
payload => #{
<<"measurement">> => <<"temp">>,
<<"data_type">> => <<"FLOAT">>,
<<"value">> => <<"32.12">>,
<<"device_id">> => <<"devid">>
}
}
},
%% "last write wins" examples
#{
select_fields => <<"payload as p, payload.f as p.answer">>,
payload => emqx_utils_json:encode(#{f => 42, keep => <<"that?">>}),
expected => #{
<<"p">> => #{
<<"answer">> => 42
}
}
},
#{
select_fields => <<"payload as p, payload.f as p.jsonlike.f">>,
payload => emqx_utils_json:encode(#{
jsonlike => emqx_utils_json:encode(#{a => 0}),
f => <<"huh">>
}),
%% behavior from 4.4: jsonlike gets wiped without preserving old "keys"
%% here we overwrite it since we don't explicitly decode it
expected => #{
<<"p">> => #{
<<"jsonlike">> => #{<<"f">> => <<"huh">>}
}
}
},
#{
select_fields =>
<<"payload as p, 42 as p, payload.measurement as p.measurement, 51 as p">>,
payload => emqx_utils_json:encode(#{
measurement => <<"temp">>,
data_type => <<"FLOAT">>,
value => <<"32.12">>,
device_id => <<"devid">>
}),
expected => #{
<<"p">> => 51
}
},
%% if selected field is already structured, new values are inserted into it
#{
select_fields =>
<<"json_decode(payload) as p, payload.a as p.z">>,
payload => emqx_utils_json:encode(#{
a => 1,
b => <<"2">>
}),
expected => #{
<<"p">> => #{
<<"a">> => 1,
<<"b">> => <<"2">>,
<<"z">> => 1
}
}
}
],
ActionFn = <<(atom_to_binary(?MODULE))/binary, ":action_response">>,
Topic = <<"some/topic">>,
ok = snabbkaffe:start_trace(),
on_exit(fun() -> snabbkaffe:stop() end),
on_exit(fun() -> delete_rule(?TMP_RULEID) end),
lists:foreach(
fun(#{select_fields := Fs, payload := P, expected := E} = Case) ->
ct:pal("testing case ~p", [Case]),
SQL = <<"select ", Fs/binary, " from \"", Topic/binary, "\"">>,
delete_rule(?TMP_RULEID),
{ok, _Rule} = emqx_rule_engine:create_rule(
#{
sql => SQL,
id => ?TMP_RULEID,
actions => [#{function => ActionFn}]
}
),
{_, {ok, Event}} =
?wait_async_action(
emqtt:publish(C, Topic, P, 0),
#{?snk_kind := action_response},
5_000
),
?assertMatch(
#{selected := E},
Event,
#{payload => P, fields => Fs, expected => E}
),
ok
end,
Cases
),
snabbkaffe:stop(),
ok.
t_events(_Config) ->
{ok, Client} = emqtt:start_link(
[
@ -3065,6 +3182,14 @@ republish_action(Topic, Payload, UserProperties) ->
}
}.
action_response(Selected, Envs, Args) ->
?tp(action_response, #{
selected => Selected,
envs => Envs,
args => Args
}),
ok.
make_simple_rule_with_ts(RuleId, Ts) when is_binary(RuleId) ->
SQL = <<"select * from \"simple/topic\"">>,
make_simple_rule(RuleId, SQL, Ts).

View File

@ -71,7 +71,49 @@ t_nested_put_map(_) ->
?assertEqual(
#{k => #{<<"t">> => #{<<"a">> => v1}}},
nested_put(?path([k, t, <<"a">>]), v1, #{k => #{<<"t">> => v0}})
).
),
%% note: since we handle json-encoded binaries when evaluating the
%% rule rather than baking the decoding in `nested_put`, we test
%% this corner case that _would_ otherwise lose data to
%% demonstrate this behavior.
?assertEqual(
#{payload => #{<<"a">> => v1}},
nested_put(
?path([payload, <<"a">>]),
v1,
#{payload => emqx_utils_json:encode(#{b => <<"v2">>})}
)
),
%% We have an asymmetry in the behavior here because `nested_put'
%% currently, at each key, will use `general_find' to get the
%% current value of the eky, and that attempts JSON decoding the
%% such value... So, the cases below, `old' gets preserved
%% because it's in this direct path.
?assertEqual(
#{payload => #{<<"a">> => #{<<"new">> => v1, <<"old">> => <<"v2">>}}},
nested_put(
?path([payload, <<"a">>, <<"new">>]),
v1,
#{payload => emqx_utils_json:encode(#{a => #{old => <<"v2">>}})}
)
),
?assertEqual(
#{payload => #{<<"a">> => #{<<"new">> => v1, <<"old">> => <<"{}">>}}},
nested_put(
?path([payload, <<"a">>, <<"new">>]),
v1,
#{payload => emqx_utils_json:encode(#{a => #{old => <<"{}">>}, b => <<"{}">>})}
)
),
?assertEqual(
#{payload => #{<<"a">> => #{<<"new">> => v1}}},
nested_put(
?path([payload, <<"a">>, <<"new">>]),
v1,
#{payload => <<"{}">>}
)
),
ok.
t_nested_put_index(_) ->
?assertEqual([1, a, 3], nested_put(?path([{ic, 2}]), a, [1, 2, 3])),

View File

@ -0,0 +1 @@
Ensure that the range of percentage type is from 0% to 100%.

View File

@ -0,0 +1 @@
Fix a typo in the log, when EMQX received an abnormal `PUBREL` packet, the `pubrel` was mistakenly typo as `pubrec`.

View File

@ -0,0 +1 @@
Restored support for the special `$queue/` shared subscription.

View File

@ -0,0 +1,20 @@
Fix and improve support for TDEngine `insert` syntax.
1. Support inserting into multi-table in the template
For example:
`insert into table_1 values (${ts}, '${id}', '${topic}')
table_2 values (${ts}, '${id}', '${topic}')`
2. Support mixing prefixes/suffixes and placeholders in the template
For example:
`insert into table_${topic} values (${ts}, '${id}', '${topic}')`
Note: This is a breaking change. Previously the values of string type were quoted automatically, but now they must be quoted explicitly.
For example:
`insert into table values (${ts}, '${a_string}')`

View File

@ -41,7 +41,8 @@ local_topic.label:
"""Local Topic"""
record_template.desc:
"""The HStream Record template to be forwarded to the HStreamDB. Placeholders supported."""
"""The HStream Record template to be forwarded to the HStreamDB. Placeholders supported.<br>
NOTE: When you use `raw record` template (which means the data is not a valid JSON), you should use `read` or `subscription` in HStream to get the data."""
record_template.label:
"""HStream Record"""

View File

@ -19,31 +19,31 @@ name.label:
"""Connector Name"""
url.desc:
"""HStreamDB Server URL"""
"""HStreamDB Server URL. Using gRPC http server address."""
url.label:
"""HStreamDB Server URL"""
stream_name.desc:
"""HStreamDB Stream Name"""
"""HStreamDB Stream Name."""
stream_name.label:
"""HStreamDB Stream Name"""
partition_key.desc:
"""HStreamDB Ordering Key"""
"""HStreamDB Partition Key. Placeholders supported."""
partition_key.label:
"""HStreamDB Ordering Key"""
"""HStreamDB Partition Key"""
pool_size.desc:
"""HStreamDB Pool Size"""
"""HStreamDB Pool Size."""
pool_size.label:
"""HStreamDB Pool Size"""
grpc_timeout.desc:
"""HStreamDB gRPC Timeout"""
"""HStreamDB gRPC Timeout."""
grpc_timeout.label:
"""HStreamDB gRPC Timeout"""