yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #12076 from id/1201-sync-master 

sync master to release-54
This commit is contained in:
Zaiming (Stone) Shi 2023-12-01 15:37:48 +01:00 committed by GitHub
parent 2bdbbc2046 956b25cc4c
commit abed1cbb80
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
152 changed files with 3842 additions and 1205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/pull_request_template.md vendored
View File

@ -1,6 +1,8 @@
Fixes <issue-or-jira-number>
<!-- Make sure to target release-5[0-9] branch if this PR is intended to fix the issues for the release candidate. -->
Release version: v/e5.?
## Summary
## PR Checklist
Please convert it to a draft if any of the following conditions are not met. Reviewers may skip over until all the items are checked:

20
.github/workflows/release.yaml vendored
View File

@ -20,7 +20,14 @@ jobs:
upload:
runs-on: ubuntu-22.04
permissions:
contents: write
checks: write
packages: write
actions: read
issues: read
pull-requests: read
repository-projects: read
statuses: read
strategy:
fail-fast: false
steps:
@ -45,11 +52,13 @@ jobs:
v*)
echo "profile=emqx" >> $GITHUB_OUTPUT
echo "version=$(./pkg-vsn.sh emqx)" >> $GITHUB_OUTPUT
echo "ref_name=v$(./pkg-vsn.sh emqx)" >> "$GITHUB_ENV"
echo "s3dir=emqx-ce" >> $GITHUB_OUTPUT
;;
e*)
echo "profile=emqx-enterprise" >> $GITHUB_OUTPUT
echo "version=$(./pkg-vsn.sh emqx-enterprise)" >> $GITHUB_OUTPUT
echo "ref_name=e$(./pkg-vsn.sh emqx-enterprise)" >> "$GITHUB_ENV"
echo "s3dir=emqx-ee" >> $GITHUB_OUTPUT
;;
esac
@ -57,14 +66,15 @@ jobs:
run: |
BUCKET=${{ secrets.AWS_S3_BUCKET }}
OUTPUT_DIR=${{ steps.profile.outputs.s3dir }}
aws s3 cp --recursive s3://$BUCKET/$OUTPUT_DIR/${{ github.ref_name }} packages
- uses: alexellis/upload-assets@0.4.0
aws s3 cp --recursive s3://$BUCKET/$OUTPUT_DIR/${{ env.ref_name }} packages
- uses: emqx/upload-assets@8d2083b4dbe3151b0b735572eaa153b6acb647fe # 0.5.0
env:
GITHUB_TOKEN: ${{ github.token }}
with:
asset_paths: '["packages/*"]'
tag_name: "${{ env.ref_name }}"
- name: update to emqx.io
if: startsWith(github.ref_name, 'v') && ((github.event_name == 'release' && !github.event.prerelease) || inputs.publish_release_artefacts)
if: startsWith(env.ref_name, 'v') && ((github.event_name == 'release' && !github.event.release.prerelease) || inputs.publish_release_artefacts)
run: |
set -eux
curl -w %{http_code} \
@ -72,10 +82,10 @@ jobs:
-H "Content-Type: application/json" \
-H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
-X POST \
-d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \
-d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.ref_name }}\" }" \
${{ secrets.EMQX_IO_RELEASE_API }}
- name: Push to packagecloud.io
if: (github.event_name == 'release' && !github.event.prerelease) || inputs.publish_release_artefacts
if: (github.event_name == 'release' && !github.event.release.prerelease) || inputs.publish_release_artefacts
env:
PROFILE: ${{ steps.profile.outputs.profile }}
VERSION: ${{ steps.profile.outputs.version }}

2
.github/workflows/upload-helm-charts.yaml vendored
View File

@ -43,7 +43,7 @@ jobs:
;;
esac
- uses: emqx/push-helm-action@v1.1
if: github.event_name == 'release' && !github.event.prerelease
if: github.event_name == 'release' && !github.event.release.prerelease
with:
charts_dir: "${{ github.workspace }}/deploy/charts/${{ steps.profile.outputs.profile }}"
version: ${{ steps.profile.outputs.version }}

4
.tool-versions
View File

@ -1,2 +1,2 @@
erlang 25.3.2-2
elixir 1.14.5-otp-25
erlang 26.1.2-1
elixir 1.15.7-otp-26

4
Makefile
View File

@ -20,8 +20,8 @@ endif
# Dashboard version
# from https://github.com/emqx/emqx-dashboard5
export EMQX_DASHBOARD_VERSION ?= v1.5.1
export EMQX_EE_DASHBOARD_VERSION ?= e1.3.2-beta.1
export EMQX_DASHBOARD_VERSION ?= v1.5.2
export EMQX_EE_DASHBOARD_VERSION ?= e1.3.2
PROFILE ?= emqx
REL_PROFILES := emqx emqx-enterprise

2
apps/emqx/include/emqx_release.hrl
View File

@ -35,7 +35,7 @@
-define(EMQX_RELEASE_CE, "5.3.2").
%% Enterprise edition
-define(EMQX_RELEASE_EE, "5.3.2-alpha.1").
-define(EMQX_RELEASE_EE, "5.3.2").
%% The HTTP API version
-define(EMQX_API_VERSION, "5.0").

297
apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl
View File

@ -9,6 +9,7 @@
-include_lib("stdlib/include/assert.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("emqx/include/asserts.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-import(emqx_common_test_helpers, [on_exit/1]).
@ -39,7 +40,7 @@ init_per_testcase(TestCase, Config) when
Cluster = cluster(#{n => 1}),
ClusterOpts = #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)},
NodeSpecs = emqx_cth_cluster:mk_nodespecs(Cluster, ClusterOpts),
Nodes = emqx_cth_cluster:start(Cluster, ClusterOpts),
Nodes = emqx_cth_cluster:start(NodeSpecs),
[
{cluster, Cluster},
{node_specs, NodeSpecs},
@ -47,12 +48,36 @@ init_per_testcase(TestCase, Config) when
{nodes, Nodes}
| Config
];
init_per_testcase(t_session_gc = TestCase, Config) ->
Opts = #{
n => 3,
roles => [core, core, replicant],
extra_emqx_conf =>
"\n session_persistence {"
"\n last_alive_update_interval = 500ms "
"\n session_gc_interval = 2s "
"\n session_gc_batch_size = 1 "
"\n }"
},
Cluster = cluster(Opts),
ClusterOpts = #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)},
NodeSpecs = emqx_cth_cluster:mk_nodespecs(Cluster, ClusterOpts),
Nodes = emqx_cth_cluster:start(Cluster, ClusterOpts),
[
{cluster, Cluster},
{node_specs, NodeSpecs},
{cluster_opts, ClusterOpts},
{nodes, Nodes},
{gc_interval, timer:seconds(2)}
| Config
];
init_per_testcase(_TestCase, Config) ->
Config.
end_per_testcase(TestCase, Config) when
TestCase =:= t_session_subscription_idempotency;
TestCase =:= t_session_unsubscription_idempotency
TestCase =:= t_session_unsubscription_idempotency;
TestCase =:= t_session_gc
->
Nodes = ?config(nodes, Config),
emqx_common_test_helpers:call_janitor(60_000),
@ -66,20 +91,32 @@ end_per_testcase(_TestCase, _Config) ->
%% Helper fns
%%------------------------------------------------------------------------------
cluster(#{n := N}) ->
Spec = #{role => core, apps => app_specs()},
cluster(#{n := N} = Opts) ->
MkRole = fun(M) ->
case maps:get(roles, Opts, undefined) of
undefined ->
core;
Roles ->
lists:nth(M, Roles)
end
end,
MkSpec = fun(M) -> #{role => MkRole(M), apps => app_specs(Opts)} end,
lists:map(
fun(M) ->
Name = list_to_atom("ds_SUITE" ++ integer_to_list(M)),
{Name, Spec}
{Name, MkSpec(M)}
end,
lists:seq(1, N)
).
app_specs() ->
app_specs(_Opts = #{}).
app_specs(Opts) ->
ExtraEMQXConf = maps:get(extra_emqx_conf, Opts, ""),
[
emqx_durable_storage,
{emqx, "session_persistence = {enable = true}"}
{emqx, "session_persistence = {enable = true}" ++ ExtraEMQXConf}
].
get_mqtt_port(Node, Type) ->
@ -115,33 +152,37 @@ start_client(Opts0 = #{}) ->
restart_node(Node, NodeSpec) ->
?tp(will_restart_node, #{}),
?tp(notice, "restarting node", #{node => Node}),
true = monitor_node(Node, true),
ok = erpc:call(Node, init, restart, []),
receive
{nodedown, Node} ->
ok
after 10_000 ->
ct:fail("node ~p didn't stop", [Node])
end,
?tp(notice, "waiting for nodeup", #{node => Node}),
emqx_cth_cluster:restart(Node, NodeSpec),
wait_nodeup(Node),
wait_gen_rpc_down(NodeSpec),
?tp(notice, "restarting apps", #{node => Node}),
Apps = maps:get(apps, NodeSpec),
ok = erpc:call(Node, emqx_cth_suite, load_apps, [Apps]),
_ = erpc:call(Node, emqx_cth_suite, start_apps, [Apps, NodeSpec]),
%% have to re-inject this so that we may stop the node succesfully at the
%% end....
ok = emqx_cth_cluster:set_node_opts(Node, NodeSpec),
ok = snabbkaffe:forward_trace(Node),
?tp(notice, "node restarted", #{node => Node}),
?tp(restarted_node, #{}),
ok.
is_persistent_connect_opts(#{properties := #{'Session-Expiry-Interval' := EI}}) ->
EI > 0.
list_all_sessions(Node) ->
erpc:call(Node, emqx_persistent_session_ds, list_all_sessions, []).
list_all_subscriptions(Node) ->
erpc:call(Node, emqx_persistent_session_ds, list_all_subscriptions, []).
list_all_pubranges(Node) ->
erpc:call(Node, emqx_persistent_session_ds, list_all_pubranges, []).
prop_only_cores_run_gc(CoreNodes) ->
{"only core nodes run gc", fun(Trace) -> ?MODULE:prop_only_cores_run_gc(Trace, CoreNodes) end}.
prop_only_cores_run_gc(Trace, CoreNodes) ->
GCNodes = lists:usort([
N
|| #{
?snk_kind := K,
?snk_meta := #{node := N}
} <- Trace,
lists:member(K, [ds_session_gc, ds_session_gc_lock_taken]),
N =/= node()
]),
?assertEqual(lists:usort(CoreNodes), GCNodes).
%%------------------------------------------------------------------------------
%% Testcases
%%------------------------------------------------------------------------------
@ -221,9 +262,10 @@ t_session_subscription_idempotency(Config) ->
end,
fun(Trace) ->
ct:pal("trace:\n ~p", [Trace]),
ConnInfo = #{},
?assertMatch(
#{subscriptions := #{SubTopicFilter := #{}}},
erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId])
erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId, ConnInfo])
)
end
),
@ -294,9 +336,10 @@ t_session_unsubscription_idempotency(Config) ->
end,
fun(Trace) ->
ct:pal("trace:\n ~p", [Trace]),
ConnInfo = #{},
?assertMatch(
#{subscriptions := Subs = #{}} when map_size(Subs) =:= 0,
erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId])
erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId, ConnInfo])
),
ok
end
@ -387,3 +430,201 @@ do_t_session_discard(Params) ->
end
),
ok.
t_session_expiration1(Config) ->
ClientId = atom_to_binary(?FUNCTION_NAME),
Opts = #{
clientid => ClientId,
sequence => [
{#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{}},
{#{clean_start => false, properties => #{'Session-Expiry-Interval' => 1}}, #{}},
{#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{}}
]
},
do_t_session_expiration(Config, Opts).
t_session_expiration2(Config) ->
ClientId = atom_to_binary(?FUNCTION_NAME),
Opts = #{
clientid => ClientId,
sequence => [
{#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{}},
{#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{
'Session-Expiry-Interval' => 1
}},
{#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{}}
]
},
do_t_session_expiration(Config, Opts).
do_t_session_expiration(_Config, Opts) ->
#{
clientid := ClientId,
sequence := [
{FirstConn, FirstDisconn},
{SecondConn, SecondDisconn},
{ThirdConn, ThirdDisconn}
]
} = Opts,
CommonParams = #{proto_ver => v5, clientid => ClientId},
?check_trace(
begin
Topic = <<"some/topic">>,
Params0 = maps:merge(CommonParams, FirstConn),
Client0 = start_client(Params0),
{ok, _} = emqtt:connect(Client0),
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(Client0, Topic, ?QOS_2),
Subs0 = emqx_persistent_session_ds:list_all_subscriptions(),
?assertEqual(1, map_size(Subs0), #{subs => Subs0}),
Info0 = maps:from_list(emqtt:info(Client0)),
?assertEqual(0, maps:get(session_present, Info0), #{info => Info0}),
emqtt:disconnect(Client0, ?RC_NORMAL_DISCONNECTION, FirstDisconn),
Params1 = maps:merge(CommonParams, SecondConn),
Client1 = start_client(Params1),
{ok, _} = emqtt:connect(Client1),
Info1 = maps:from_list(emqtt:info(Client1)),
?assertEqual(1, maps:get(session_present, Info1), #{info => Info1}),
Subs1 = emqtt:subscriptions(Client1),
?assertEqual([], Subs1),
emqtt:disconnect(Client1, ?RC_NORMAL_DISCONNECTION, SecondDisconn),
ct:sleep(1_500),
Params2 = maps:merge(CommonParams, ThirdConn),
Client2 = start_client(Params2),
{ok, _} = emqtt:connect(Client2),
Info2 = maps:from_list(emqtt:info(Client2)),
?assertEqual(0, maps:get(session_present, Info2), #{info => Info2}),
Subs2 = emqtt:subscriptions(Client2),
?assertEqual([], Subs2),
emqtt:publish(Client2, Topic, <<"payload">>),
?assertNotReceive({publish, #{topic := Topic}}),
%% ensure subscriptions are absent from table.
?assertEqual(#{}, emqx_persistent_session_ds:list_all_subscriptions()),
emqtt:disconnect(Client2, ?RC_NORMAL_DISCONNECTION, ThirdDisconn),
ok
end,
[]
),
ok.
t_session_gc(Config) ->
GCInterval = ?config(gc_interval, Config),
[Node1, Node2, Node3] = Nodes = ?config(nodes, Config),
CoreNodes = [Node1, Node2],
[
Port1,
Port2,
Port3
] = lists:map(fun(N) -> get_mqtt_port(N, tcp) end, Nodes),
CommonParams = #{
clean_start => false,
proto_ver => v5
},
StartClient = fun(ClientId, Port, ExpiryInterval) ->
Params = maps:merge(CommonParams, #{
clientid => ClientId,
port => Port,
properties => #{'Session-Expiry-Interval' => ExpiryInterval}
}),
Client = start_client(Params),
{ok, _} = emqtt:connect(Client),
Client
end,
?check_trace(
begin
ClientId0 = <<"session_gc0">>,
Client0 = StartClient(ClientId0, Port1, 30),
ClientId1 = <<"session_gc1">>,
Client1 = StartClient(ClientId1, Port2, 1),
ClientId2 = <<"session_gc2">>,
Client2 = StartClient(ClientId2, Port3, 1),
lists:foreach(
fun(Client) ->
Topic = <<"some/topic">>,
Payload = <<"hi">>,
{ok, _, [?RC_GRANTED_QOS_1]} = emqtt:subscribe(Client, Topic, ?QOS_1),
{ok, _} = emqtt:publish(Client, Topic, Payload, ?QOS_1),
ok
end,
[Client0, Client1, Client2]
),
%% Clients are still alive; no session is garbage collected.
Res0 = ?block_until(
#{
?snk_kind := ds_session_gc,
?snk_span := {complete, _},
?snk_meta := #{node := N}
} when
N =/= node(),
3 * GCInterval + 1_000
),
?assertMatch({ok, _}, Res0),
{ok, #{?snk_meta := #{time := T0}}} = Res0,
Sessions0 = list_all_sessions(Node1),
Subs0 = list_all_subscriptions(Node1),
?assertEqual(3, map_size(Sessions0), #{sessions => Sessions0}),
?assertEqual(3, map_size(Subs0), #{subs => Subs0}),
%% Now we disconnect 2 of them; only those should be GC'ed.
?assertMatch(
{ok, {ok, _}},
?wait_async_action(
emqtt:stop(Client1),
#{?snk_kind := terminate},
1_000
)
),
ct:pal("disconnected client1"),
?assertMatch(
{ok, {ok, _}},
?wait_async_action(
emqtt:stop(Client2),
#{?snk_kind := terminate},
1_000
)
),
ct:pal("disconnected client2"),
?assertMatch(
{ok, _},
?block_until(
#{
?snk_kind := ds_session_gc_cleaned,
?snk_meta := #{node := N, time := T},
session_ids := [ClientId1]
} when
N =/= node() andalso T > T0,
4 * GCInterval + 1_000
)
),
?assertMatch(
{ok, _},
?block_until(
#{
?snk_kind := ds_session_gc_cleaned,
?snk_meta := #{node := N, time := T},
session_ids := [ClientId2]
} when
N =/= node() andalso T > T0,
4 * GCInterval + 1_000
)
),
Sessions1 = list_all_sessions(Node1),
Subs1 = list_all_subscriptions(Node1),
?assertEqual(1, map_size(Sessions1), #{sessions => Sessions1}),
?assertEqual(1, map_size(Subs1), #{subs => Subs1}),
ok
end,
[
prop_only_cores_run_gc(CoreNodes)
]
),
ok.

4
apps/emqx/rebar.config
View File

@ -27,9 +27,9 @@
{lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}},
{gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}},
{cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}},
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}},
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.8"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.1"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.2"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.0"}}},
{emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},
{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},

2
apps/emqx/src/emqx.app.src
View File

@ -2,7 +2,7 @@
{application, emqx, [
{id, "emqx"},
{description, "EMQX Core"},
{vsn, "5.1.14"},
{vsn, "5.1.15"},
{modules, []},
{registered, []},
{applications, [

7
apps/emqx/src/emqx_channel.erl
View File

@ -423,6 +423,7 @@ handle_in(
{ok, Channel}
end;
handle_in(
%% TODO: Why discard the Reason Code?
?PUBREC_PACKET(PacketId, _ReasonCode, Properties),
Channel =
#channel{clientinfo = ClientInfo, session = Session}
@ -1204,12 +1205,13 @@ handle_info(
#channel{
conn_state = ConnState,
clientinfo = ClientInfo,
conninfo = ConnInfo,
session = Session
}
) when
ConnState =:= connected orelse ConnState =:= reauthenticating
->
{Intent, Session1} = emqx_session:disconnect(ClientInfo, Session),
{Intent, Session1} = emqx_session:disconnect(ClientInfo, ConnInfo, Session),
Channel1 = ensure_disconnected(Reason, maybe_publish_will_msg(Channel)),
Channel2 = Channel1#channel{session = Session1},
case maybe_shutdown(Reason, Intent, Channel2) of
@ -1321,7 +1323,8 @@ handle_timeout(
{ok, Replies, NSession} ->
handle_out(publish, Replies, Channel#channel{session = NSession})
end;
handle_timeout(_TRef, expire_session, Channel) ->
handle_timeout(_TRef, expire_session, Channel = #channel{session = Session}) ->
ok = emqx_session:destroy(Session),
shutdown(expired, Channel);
handle_timeout(
_TRef,

1
apps/emqx/src/emqx_cm.erl
View File

@ -91,6 +91,7 @@
clean_down/1,
mark_channel_connected/1,
mark_channel_disconnected/1,
is_channel_connected/1,
get_connected_client_count/0
]).

12
apps/emqx/src/emqx_cm_sup.erl
View File

@ -47,7 +47,17 @@ init([]) ->
Locker = child_spec(emqx_cm_locker, 5000, worker),
Registry = child_spec(emqx_cm_registry, 5000, worker),
Manager = child_spec(emqx_cm, 5000, worker),
{ok, {SupFlags, [Banned, Flapping, Locker, Registry, Manager]}}.
DSSessionGCSup = child_spec(emqx_persistent_session_ds_sup, infinity, supervisor),
Children =
[
Banned,
Flapping,
Locker,
Registry,
Manager,
DSSessionGCSup
],
{ok, {SupFlags, Children}}.
%%--------------------------------------------------------------------
%% Internal functions

489
apps/emqx/src/emqx_persistent_message_ds_replayer.erl
View File

@ -19,7 +19,13 @@
-module(emqx_persistent_message_ds_replayer).
%% API:
-export([new/0, open/1, next_packet_id/1, replay/1, commit_offset/3, poll/3, n_inflight/1]).
-export([new/0, open/1, next_packet_id/1, n_inflight/1]).
-export([poll/4, replay/2, commit_offset/4]).
-export([seqno_to_packet_id/1, packet_id_to_seqno/2]).
-export([committed_until/2]).
%% internal exports:
-export([]).
@ -27,7 +33,6 @@
-export_type([inflight/0, seqno/0]).
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx_utils/include/emqx_message.hrl").
-include("emqx_persistent_session_ds.hrl").
-ifdef(TEST).
@ -35,6 +40,13 @@
-include_lib("eunit/include/eunit.hrl").
-endif.
-define(EPOCH_SIZE, 16#10000).
-define(ACK, 0).
-define(COMP, 1).
-define(TRACK_FLAG(WHICH), (1 bsl WHICH)).
%%================================================================================
%% Type declarations
%%================================================================================
@ -42,15 +54,23 @@
%% Note: sequence numbers are monotonic; they don't wrap around:
-type seqno() :: non_neg_integer().
-type track() :: ack | comp.
-type commit_type() :: rec.
-record(inflight, {
next_seqno = 1 :: seqno(),
acked_until = 1 :: seqno(),
commits = #{ack => 1, comp => 1, rec => 1} :: #{track() | commit_type() => seqno()},
%% Ranges are sorted in ascending order of their sequence numbers.
offset_ranges = [] :: [ds_pubrange()]
}).
-opaque inflight() :: #inflight{}.
-type reply_fun() :: fun(
(seqno(), emqx_types:message()) ->
emqx_session:replies() | {_AdvanceSeqno :: false, emqx_session:replies()}
).
%%================================================================================
%% API funcions
%%================================================================================
@ -61,10 +81,12 @@ new() ->
-spec open(emqx_persistent_session_ds:id()) -> inflight().
open(SessionId) ->
Ranges = ro_transaction(fun() -> get_ranges(SessionId) end),
{AckedUntil, NextSeqno} = compute_inflight_range(Ranges),
{Ranges, RecUntil} = ro_transaction(
fun() -> {get_ranges(SessionId), get_committed_offset(SessionId, rec)} end
),
{Commits, NextSeqno} = compute_inflight_range(Ranges),
#inflight{
acked_until = AckedUntil,
commits = Commits#{rec => RecUntil},
next_seqno = NextSeqno,
offset_ranges = Ranges
}.
@ -75,15 +97,30 @@ next_packet_id(Inflight0 = #inflight{next_seqno = LastSeqno}) ->
{seqno_to_packet_id(LastSeqno), Inflight}.
-spec n_inflight(inflight()) -> non_neg_integer().
n_inflight(#inflight{next_seqno = NextSeqno, acked_until = AckedUntil}) ->
range_size(AckedUntil, NextSeqno).
n_inflight(#inflight{offset_ranges = Ranges}) ->
%% TODO
%% This is not very efficient. Instead, we can take the maximum of
%% `range_size(AckedUntil, NextSeqno)` and `range_size(CompUntil, NextSeqno)`.
%% This won't be exact number but a pessimistic estimate, but this way we
%% will penalize clients that PUBACK QoS 1 messages but don't PUBCOMP QoS 2
%% messages for some reason. For that to work, we need to additionally track
%% actual `AckedUntil` / `CompUntil` during `commit_offset/4`.
lists:foldl(
fun
(#ds_pubrange{type = ?T_CHECKPOINT}, N) ->
N;
(#ds_pubrange{type = ?T_INFLIGHT, id = {_, First}, until = Until}, N) ->
N + range_size(First, Until)
end,
0,
Ranges
).
-spec replay(inflight()) ->
{emqx_session:replies(), inflight()}.
replay(Inflight0 = #inflight{acked_until = AckedUntil, offset_ranges = Ranges0}) ->
-spec replay(reply_fun(), inflight()) -> {emqx_session:replies(), inflight()}.
replay(ReplyFun, Inflight0 = #inflight{offset_ranges = Ranges0}) ->
{Ranges, Replies} = lists:mapfoldr(
fun(Range, Acc) ->
replay_range(Range, AckedUntil, Acc)
replay_range(ReplyFun, Range, Acc)
end,
[],
Ranges0
@ -91,43 +128,49 @@ replay(Inflight0 = #inflight{acked_until = AckedUntil, offset_ranges = Ranges0})
Inflight = Inflight0#inflight{offset_ranges = Ranges},
{Replies, Inflight}.
-spec commit_offset(emqx_persistent_session_ds:id(), emqx_types:packet_id(), inflight()) ->
{_IsValidOffset :: boolean(), inflight()}.
-spec commit_offset(emqx_persistent_session_ds:id(), Offset, emqx_types:packet_id(), inflight()) ->
{_IsValidOffset :: boolean(), inflight()}
when
Offset :: track() | commit_type().
commit_offset(
SessionId,
Track,
PacketId,
Inflight0 = #inflight{
acked_until = AckedUntil, next_seqno = NextSeqno
}
) ->
case packet_id_to_seqno(NextSeqno, PacketId) of
Seqno when Seqno >= AckedUntil andalso Seqno < NextSeqno ->
Inflight0 = #inflight{commits = Commits}
) when Track == ack orelse Track == comp ->
case validate_commit(Track, PacketId, Inflight0) of
CommitUntil when is_integer(CommitUntil) ->
%% TODO
%% We do not preserve `acked_until` in the database. Instead, we discard
%% We do not preserve `CommitUntil` in the database. Instead, we discard
%% fully acked ranges from the database. In effect, this means that the
%% most recent `acked_until` the client has sent may be lost in case of a
%% most recent `CommitUntil` the client has sent may be lost in case of a
%% crash or client loss.
Inflight1 = Inflight0#inflight{acked_until = next_seqno(Seqno)},
Inflight = discard_acked(SessionId, Inflight1),
Inflight1 = Inflight0#inflight{commits = Commits#{Track := CommitUntil}},
Inflight = discard_committed(SessionId, Inflight1),
{true, Inflight};
OutOfRange ->
?SLOG(warning, #{
msg => "out-of-order_ack",
acked_until => AckedUntil,
acked_seqno => OutOfRange,
next_seqno => NextSeqno,
packet_id => PacketId
}),
false ->
{false, Inflight0}
end;
commit_offset(
SessionId,
CommitType = rec,
PacketId,
Inflight0 = #inflight{commits = Commits}
) ->
case validate_commit(CommitType, PacketId, Inflight0) of
CommitUntil when is_integer(CommitUntil) ->
update_committed_offset(SessionId, CommitType, CommitUntil),
Inflight = Inflight0#inflight{commits = Commits#{CommitType := CommitUntil}},
{true, Inflight};
false ->
{false, Inflight0}
end.
-spec poll(emqx_persistent_session_ds:id(), inflight(), pos_integer()) ->
-spec poll(reply_fun(), emqx_persistent_session_ds:id(), inflight(), pos_integer()) ->
{emqx_session:replies(), inflight()}.
poll(SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < 16#7fff ->
#inflight{next_seqno = NextSeqNo0, acked_until = AckedSeqno} =
Inflight0,
poll(ReplyFun, SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < ?EPOCH_SIZE ->
FetchThreshold = max(1, WindowSize div 2),
FreeSpace = AckedSeqno + WindowSize - NextSeqNo0,
FreeSpace = WindowSize - n_inflight(Inflight0),
case FreeSpace >= FetchThreshold of
false ->
%% TODO: this branch is meant to avoid fetching data from
@ -138,9 +181,25 @@ poll(SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < 16#7fff
true ->
%% TODO: Wrap this in `mria:async_dirty/2`?
Streams = shuffle(get_streams(SessionId)),
fetch(SessionId, Inflight0, Streams, FreeSpace, [])
fetch(ReplyFun, SessionId, Inflight0, Streams, FreeSpace, [])
end.
%% Which seqno this track is committed until.
%% "Until" means this is first seqno that is _not yet committed_ for this track.
-spec committed_until(track() | commit_type(), inflight()) -> seqno().
committed_until(Track, #inflight{commits = Commits}) ->
maps:get(Track, Commits).
-spec seqno_to_packet_id(seqno()) -> emqx_types:packet_id() | 0.
seqno_to_packet_id(Seqno) ->
Seqno rem ?EPOCH_SIZE.
%% Reconstruct session counter by adding most significant bits from
%% the current counter to the packet id.
-spec packet_id_to_seqno(emqx_types:packet_id(), inflight()) -> seqno().
packet_id_to_seqno(PacketId, #inflight{next_seqno = NextSeqno}) ->
packet_id_to_seqno_(NextSeqno, PacketId).
%%================================================================================
%% Internal exports
%%================================================================================
@ -150,18 +209,34 @@ poll(SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < 16#7fff
%%================================================================================
compute_inflight_range([]) ->
{1, 1};
{#{ack => 1, comp => 1}, 1};
compute_inflight_range(Ranges) ->
_RangeLast = #ds_pubrange{until = LastSeqno} = lists:last(Ranges),
RangesUnacked = lists:dropwhile(
fun(#ds_pubrange{type = T}) -> T == checkpoint end,
AckedUntil = find_committed_until(ack, Ranges),
CompUntil = find_committed_until(comp, Ranges),
Commits = #{
ack => emqx_maybe:define(AckedUntil, LastSeqno),
comp => emqx_maybe:define(CompUntil, LastSeqno)
},
{Commits, LastSeqno}.
find_committed_until(Track, Ranges) ->
RangesUncommitted = lists:dropwhile(
fun(Range) ->
case Range of
#ds_pubrange{type = ?T_CHECKPOINT} ->
true;
#ds_pubrange{type = ?T_INFLIGHT, tracks = Tracks} ->
not has_track(Track, Tracks)
end
end,
Ranges
),
case RangesUnacked of
[#ds_pubrange{id = {_, AckedUntil}} | _] ->
{AckedUntil, LastSeqno};
case RangesUncommitted of
[#ds_pubrange{id = {_, CommittedUntil}} | _] ->
CommittedUntil;
[] ->
{LastSeqno, LastSeqno}
undefined
end.
-spec get_ranges(emqx_persistent_session_ds:id()) -> [ds_pubrange()].
@ -173,21 +248,22 @@ get_ranges(SessionId) ->
),
mnesia:match_object(?SESSION_PUBRANGE_TAB, Pat, read).
fetch(SessionId, Inflight0, [DSStream | Streams], N, Acc) when N > 0 ->
fetch(ReplyFun, SessionId, Inflight0, [DSStream | Streams], N, Acc) when N > 0 ->
#inflight{next_seqno = FirstSeqno, offset_ranges = Ranges} = Inflight0,
ItBegin = get_last_iterator(DSStream, Ranges),
{ok, ItEnd, Messages} = emqx_ds:next(?PERSISTENT_MESSAGE_DB, ItBegin, N),
case Messages of
[] ->
fetch(SessionId, Inflight0, Streams, N, Acc);
fetch(ReplyFun, SessionId, Inflight0, Streams, N, Acc);
_ ->
{Publishes, UntilSeqno} = publish(FirstSeqno, Messages, _PreserveQoS0 = true),
Size = range_size(FirstSeqno, UntilSeqno),
%% We need to preserve the iterator pointing to the beginning of the
%% range, so that we can replay it if needed.
{Publishes, {UntilSeqno, Tracks}} = publish(ReplyFun, FirstSeqno, Messages),
Size = range_size(FirstSeqno, UntilSeqno),
Range0 = #ds_pubrange{
id = {SessionId, FirstSeqno},
type = inflight,
type = ?T_INFLIGHT,
tracks = Tracks,
until = UntilSeqno,
stream = DSStream#ds_stream.ref,
iterator = ItBegin
@ -196,25 +272,25 @@ fetch(SessionId, Inflight0, [DSStream | Streams], N, Acc) when N > 0 ->
%% ...Yet we need to keep the iterator pointing past the end of the
%% range, so that we can pick up where we left off: it will become
%% `ItBegin` of the next range for this stream.
Range = Range0#ds_pubrange{iterator = ItEnd},
Range = keep_next_iterator(ItEnd, Range0),
Inflight = Inflight0#inflight{
next_seqno = UntilSeqno,
offset_ranges = Ranges ++ [Range]
},
fetch(SessionId, Inflight, Streams, N - Size, [Publishes | Acc])
fetch(ReplyFun, SessionId, Inflight, Streams, N - Size, [Publishes | Acc])
end;
fetch(_SessionId, Inflight, _Streams, _N, Acc) ->
fetch(_ReplyFun, _SessionId, Inflight, _Streams, _N, Acc) ->
Publishes = lists:append(lists:reverse(Acc)),
{Publishes, Inflight}.
discard_acked(
discard_committed(
SessionId,
Inflight0 = #inflight{acked_until = AckedUntil, offset_ranges = Ranges0}
Inflight0 = #inflight{commits = Commits, offset_ranges = Ranges0}
) ->
%% TODO: This could be kept and incrementally updated in the inflight state.
Checkpoints = find_checkpoints(Ranges0),
%% TODO: Wrap this in `mria:async_dirty/2`?
Ranges = discard_acked_ranges(SessionId, AckedUntil, Checkpoints, Ranges0),
Ranges = discard_committed_ranges(SessionId, Commits, Checkpoints, Ranges0),
Inflight0#inflight{offset_ranges = Ranges}.
find_checkpoints(Ranges) ->
@ -227,84 +303,178 @@ find_checkpoints(Ranges) ->
Ranges
).
discard_acked_ranges(
discard_committed_ranges(
SessionId,
AckedUntil,
Commits,
Checkpoints,
[Range = #ds_pubrange{until = Until, stream = StreamRef} | Rest]
) when Until =< AckedUntil ->
%% This range has been fully acked.
%% Either discard it completely, or preserve the iterator for the next range
%% over this stream (i.e. a checkpoint).
RangeKept =
case maps:get(StreamRef, Checkpoints) of
CP when CP > Until ->
discard_range(Range),
[];
Until ->
[checkpoint_range(Range)]
Ranges = [Range = #ds_pubrange{until = Until, stream = StreamRef} | Rest]
) ->
case discard_committed_range(Commits, Range) of
discard ->
%% This range has been fully committed.
%% Either discard it completely, or preserve the iterator for the next range
%% over this stream (i.e. a checkpoint).
RangeKept =
case maps:get(StreamRef, Checkpoints) of
CP when CP > Until ->
discard_range(Range),
[];
Until ->
[checkpoint_range(Range)]
end,
%% Since we're (intentionally) not using transactions here, it's important to
%% issue database writes in the same order in which ranges are stored: from
%% the oldest to the newest. This is also why we need to compute which ranges
%% should become checkpoints before we start writing anything.
RangeKept ++ discard_committed_ranges(SessionId, Commits, Checkpoints, Rest);
keep ->
%% This range has not been fully committed.
[Range | discard_committed_ranges(SessionId, Commits, Checkpoints, Rest)];
keep_all ->
%% The rest of ranges (if any) still have uncommitted messages.
Ranges;
TracksLeft ->
%% Only some track has been committed.
%% Preserve the uncommitted tracks in the database.
RangeKept = Range#ds_pubrange{tracks = TracksLeft},
preserve_range(restore_first_iterator(RangeKept)),
[RangeKept | discard_committed_ranges(SessionId, Commits, Checkpoints, Rest)]
end;
discard_committed_ranges(_SessionId, _Commits, _Checkpoints, []) ->
[].
discard_committed_range(_Commits, #ds_pubrange{type = ?T_CHECKPOINT}) ->
discard;
discard_committed_range(
#{ack := AckedUntil, comp := CompUntil},
#ds_pubrange{until = Until}
) when Until > AckedUntil andalso Until > CompUntil ->
keep_all;
discard_committed_range(Commits, #ds_pubrange{until = Until, tracks = Tracks}) ->
case discard_tracks(Commits, Until, Tracks) of
0 ->
discard;
Tracks ->
keep;
TracksLeft ->
TracksLeft
end.
discard_tracks(#{ack := AckedUntil, comp := CompUntil}, Until, Tracks) ->
TAck =
case Until > AckedUntil of
true -> ?TRACK_FLAG(?ACK) band Tracks;
false -> 0
end,
%% Since we're (intentionally) not using transactions here, it's important to
%% issue database writes in the same order in which ranges are stored: from
%% the oldest to the newest. This is also why we need to compute which ranges
%% should become checkpoints before we start writing anything.
RangeKept ++ discard_acked_ranges(SessionId, AckedUntil, Checkpoints, Rest);
discard_acked_ranges(_SessionId, _AckedUntil, _Checkpoints, Ranges) ->
%% The rest of ranges (if any) still have unacked messages.
Ranges.
TComp =
case Until > CompUntil of
true -> ?TRACK_FLAG(?COMP) band Tracks;
false -> 0
end,
TAck bor TComp.
replay_range(
Range0 = #ds_pubrange{type = inflight, id = {_, First}, until = Until, iterator = It},
AckedUntil,
ReplyFun,
Range0 = #ds_pubrange{type = ?T_INFLIGHT, id = {_, First}, until = Until, iterator = It},
Acc
) ->
Size = range_size(First, Until),
FirstUnacked = max(First, AckedUntil),
{ok, ItNext, Messages} = emqx_ds:next(?PERSISTENT_MESSAGE_DB, It, Size),
MessagesUnacked =
case FirstUnacked of
First ->
Messages;
_ ->
lists:nthtail(range_size(First, FirstUnacked), Messages)
end,
MessagesReplay = [emqx_message:set_flag(dup, true, Msg) || Msg <- MessagesUnacked],
{ok, ItNext, MessagesUnacked} = emqx_ds:next(?PERSISTENT_MESSAGE_DB, It, Size),
%% Asserting that range is consistent with the message storage state.
{Replies, Until} = publish(FirstUnacked, MessagesReplay, _PreserveQoS0 = false),
{Replies, {Until, _TracksInitial}} = publish(ReplyFun, First, MessagesUnacked),
%% Again, we need to keep the iterator pointing past the end of the
%% range, so that we can pick up where we left off.
Range = Range0#ds_pubrange{iterator = ItNext},
Range = keep_next_iterator(ItNext, Range0),
{Range, Replies ++ Acc};
replay_range(Range0 = #ds_pubrange{type = checkpoint}, _AckedUntil, Acc) ->
replay_range(_ReplyFun, Range0 = #ds_pubrange{type = ?T_CHECKPOINT}, Acc) ->
{Range0, Acc}.
publish(FirstSeqNo, Messages, PreserveQos0) ->
do_publish(FirstSeqNo, Messages, PreserveQos0, []).
validate_commit(
Track,
PacketId,
Inflight = #inflight{commits = Commits, next_seqno = NextSeqno}
) ->
Seqno = packet_id_to_seqno_(NextSeqno, PacketId),
CommittedUntil = maps:get(Track, Commits),
CommitNext = get_commit_next(Track, Inflight),
case Seqno >= CommittedUntil andalso Seqno < CommitNext of
true ->
next_seqno(Seqno);
false ->
?SLOG(warning, #{
msg => "out-of-order_commit",
track => Track,
packet_id => PacketId,
commit_seqno => Seqno,
committed_until => CommittedUntil,
commit_next => CommitNext
}),
false
end.
do_publish(SeqNo, [], _, Acc) ->
{lists:reverse(Acc), SeqNo};
do_publish(SeqNo, [#message{qos = 0} | Messages], false, Acc) ->
do_publish(SeqNo, Messages, false, Acc);
do_publish(SeqNo, [#message{qos = 0} = Message | Messages], true, Acc) ->
do_publish(SeqNo, Messages, true, [{undefined, Message} | Acc]);
do_publish(SeqNo, [Message | Messages], PreserveQos0, Acc) ->
PacketId = seqno_to_packet_id(SeqNo),
do_publish(next_seqno(SeqNo), Messages, PreserveQos0, [{PacketId, Message} | Acc]).
get_commit_next(ack, #inflight{next_seqno = NextSeqno}) ->
NextSeqno;
get_commit_next(rec, #inflight{next_seqno = NextSeqno}) ->
NextSeqno;
get_commit_next(comp, #inflight{commits = Commits}) ->
maps:get(rec, Commits).
publish(ReplyFun, FirstSeqno, Messages) ->
lists:mapfoldl(
fun(Message, {Seqno, TAcc}) ->
case ReplyFun(Seqno, Message) of
{_Advance = false, Reply} ->
{Reply, {Seqno, TAcc}};
Reply ->
NextSeqno = next_seqno(Seqno),
NextTAcc = add_msg_track(Message, TAcc),
{Reply, {NextSeqno, NextTAcc}}
end
end,
{FirstSeqno, 0},
Messages
).
add_msg_track(Message, Tracks) ->
case emqx_message:qos(Message) of
1 -> ?TRACK_FLAG(?ACK) bor Tracks;
2 -> ?TRACK_FLAG(?COMP) bor Tracks;
_ -> Tracks
end.
keep_next_iterator(ItNext, Range = #ds_pubrange{iterator = ItFirst, misc = Misc}) ->
Range#ds_pubrange{
iterator = ItNext,
%% We need to keep the first iterator around, in case we need to preserve
%% this range again, updating still uncommitted tracks it's part of.
misc = Misc#{iterator_first => ItFirst}
}.
restore_first_iterator(Range = #ds_pubrange{misc = Misc = #{iterator_first := ItFirst}}) ->
Range#ds_pubrange{
iterator = ItFirst,
misc = maps:remove(iterator_first, Misc)
}.
-spec preserve_range(ds_pubrange()) -> ok.
preserve_range(Range = #ds_pubrange{type = inflight}) ->
preserve_range(Range = #ds_pubrange{type = ?T_INFLIGHT}) ->
mria:dirty_write(?SESSION_PUBRANGE_TAB, Range).
has_track(ack, Tracks) ->
(?TRACK_FLAG(?ACK) band Tracks) > 0;
has_track(comp, Tracks) ->
(?TRACK_FLAG(?COMP) band Tracks) > 0.
-spec discard_range(ds_pubrange()) -> ok.
discard_range(#ds_pubrange{id = RangeId}) ->
mria:dirty_delete(?SESSION_PUBRANGE_TAB, RangeId).
-spec checkpoint_range(ds_pubrange()) -> ds_pubrange().
checkpoint_range(Range0 = #ds_pubrange{type = inflight}) ->
Range = Range0#ds_pubrange{type = checkpoint},
checkpoint_range(Range0 = #ds_pubrange{type = ?T_INFLIGHT}) ->
Range = Range0#ds_pubrange{type = ?T_CHECKPOINT, misc = #{}},
ok = mria:dirty_write(?SESSION_PUBRANGE_TAB, Range),
Range;
checkpoint_range(Range = #ds_pubrange{type = checkpoint}) ->
checkpoint_range(Range = #ds_pubrange{type = ?T_CHECKPOINT}) ->
%% This range should have been checkpointed already.
Range.
@ -320,6 +490,21 @@ get_last_iterator(DSStream = #ds_stream{ref = StreamRef}, Ranges) ->
get_streams(SessionId) ->
mnesia:dirty_read(?SESSION_STREAM_TAB, SessionId).
-spec get_committed_offset(emqx_persistent_session_ds:id(), _Name) -> seqno().
get_committed_offset(SessionId, Name) ->
case mnesia:read(?SESSION_COMMITTED_OFFSET_TAB, {SessionId, Name}) of
[] ->
1;
[#ds_committed_offset{until = Seqno}] ->
Seqno
end.
-spec update_committed_offset(emqx_persistent_session_ds:id(), _Name, seqno()) -> ok.
update_committed_offset(SessionId, Name, Until) ->
mria:dirty_write(?SESSION_COMMITTED_OFFSET_TAB, #ds_committed_offset{
id = {SessionId, Name}, until = Until
}).
next_seqno(Seqno) ->
NextSeqno = Seqno + 1,
case seqno_to_packet_id(NextSeqno) of
@ -332,26 +517,15 @@ next_seqno(Seqno) ->
NextSeqno
end.
%% Reconstruct session counter by adding most significant bits from
%% the current counter to the packet id.
-spec packet_id_to_seqno(_Next :: seqno(), emqx_types:packet_id()) -> seqno().
packet_id_to_seqno(NextSeqNo, PacketId) ->
Epoch = NextSeqNo bsr 16,
case packet_id_to_seqno_(Epoch, PacketId) of
N when N =< NextSeqNo ->
packet_id_to_seqno_(NextSeqno, PacketId) ->
Epoch = NextSeqno bsr 16,
case (Epoch bsl 16) + PacketId of
N when N =< NextSeqno ->
N;
_ ->
packet_id_to_seqno_(Epoch - 1, PacketId)
N ->
N - ?EPOCH_SIZE
end.
-spec packet_id_to_seqno_(non_neg_integer(), emqx_types:packet_id()) -> seqno().
packet_id_to_seqno_(Epoch, PacketId) ->
(Epoch bsl 16) + PacketId.
-spec seqno_to_packet_id(seqno()) -> emqx_types:packet_id() | 0.
seqno_to_packet_id(Seqno) ->
Seqno rem 16#10000.
range_size(FirstSeqno, UntilSeqno) ->
%% This function assumes that gaps in the sequence ID occur _only_ when the
%% packet ID wraps.
@ -379,19 +553,19 @@ ro_transaction(Fun) ->
%% This test only tests boundary conditions (to make sure property-based test didn't skip them):
packet_id_to_seqno_test() ->
%% Packet ID = 1; first epoch:
?assertEqual(1, packet_id_to_seqno(1, 1)),
?assertEqual(1, packet_id_to_seqno(10, 1)),
?assertEqual(1, packet_id_to_seqno(1 bsl 16 - 1, 1)),
?assertEqual(1, packet_id_to_seqno(1 bsl 16, 1)),
?assertEqual(1, packet_id_to_seqno_(1, 1)),
?assertEqual(1, packet_id_to_seqno_(10, 1)),
?assertEqual(1, packet_id_to_seqno_(1 bsl 16 - 1, 1)),
?assertEqual(1, packet_id_to_seqno_(1 bsl 16, 1)),
%% Packet ID = 1; second and 3rd epochs:
?assertEqual(1 bsl 16 + 1, packet_id_to_seqno(1 bsl 16 + 1, 1)),
?assertEqual(1 bsl 16 + 1, packet_id_to_seqno(2 bsl 16, 1)),
?assertEqual(2 bsl 16 + 1, packet_id_to_seqno(2 bsl 16 + 1, 1)),
?assertEqual(1 bsl 16 + 1, packet_id_to_seqno_(1 bsl 16 + 1, 1)),
?assertEqual(1 bsl 16 + 1, packet_id_to_seqno_(2 bsl 16, 1)),
?assertEqual(2 bsl 16 + 1, packet_id_to_seqno_(2 bsl 16 + 1, 1)),
%% Packet ID = 16#ffff:
PID = 1 bsl 16 - 1,
?assertEqual(PID, packet_id_to_seqno(PID, PID)),
?assertEqual(PID, packet_id_to_seqno(1 bsl 16, PID)),
?assertEqual(1 bsl 16 + PID, packet_id_to_seqno(2 bsl 16, PID)),
?assertEqual(PID, packet_id_to_seqno_(PID, PID)),
?assertEqual(PID, packet_id_to_seqno_(1 bsl 16, PID)),
?assertEqual(1 bsl 16 + PID, packet_id_to_seqno_(2 bsl 16, PID)),
ok.
packet_id_to_seqno_test_() ->
@ -406,8 +580,8 @@ packet_id_to_seqno_prop() ->
SeqNo,
seqno_gen(NextSeqNo),
begin
PacketId = SeqNo rem 16#10000,
?assertEqual(SeqNo, packet_id_to_seqno(NextSeqNo, PacketId)),
PacketId = seqno_to_packet_id(SeqNo),
?assertEqual(SeqNo, packet_id_to_seqno_(NextSeqNo, PacketId)),
true
end
)
@ -437,27 +611,42 @@ range_size_test_() ->
compute_inflight_range_test_() ->
[
?_assertEqual(
{1, 1},
{#{ack => 1, comp => 1}, 1},
compute_inflight_range([])
),
?_assertEqual(
{12, 42},
{#{ack => 12, comp => 13}, 42},
compute_inflight_range([
#ds_pubrange{id = {<<>>, 1}, until = 2, type = checkpoint},
#ds_pubrange{id = {<<>>, 4}, until = 8, type = checkpoint},
#ds_pubrange{id = {<<>>, 11}, until = 12, type = checkpoint},
#ds_pubrange{id = {<<>>, 12}, until = 13, type = inflight},
#ds_pubrange{id = {<<>>, 13}, until = 20, type = inflight},
#ds_pubrange{id = {<<>>, 20}, until = 42, type = inflight}
#ds_pubrange{id = {<<>>, 1}, until = 2, type = ?T_CHECKPOINT},
#ds_pubrange{id = {<<>>, 4}, until = 8, type = ?T_CHECKPOINT},
#ds_pubrange{id = {<<>>, 11}, until = 12, type = ?T_CHECKPOINT},
#ds_pubrange{
id = {<<>>, 12},
until = 13,
type = ?T_INFLIGHT,
tracks = ?TRACK_FLAG(?ACK)
},
#ds_pubrange{
id = {<<>>, 13},
until = 20,
type = ?T_INFLIGHT,
tracks = ?TRACK_FLAG(?COMP)
},
#ds_pubrange{
id = {<<>>, 20},
until = 42,
type = ?T_INFLIGHT,
tracks = ?TRACK_FLAG(?ACK) bor ?TRACK_FLAG(?COMP)
}
])
),
?_assertEqual(
{13, 13},
{#{ack => 13, comp => 13}, 13},
compute_inflight_range([
#ds_pubrange{id = {<<>>, 1}, until = 2, type = checkpoint},
#ds_pubrange{id = {<<>>, 4}, until = 8, type = checkpoint},
#ds_pubrange{id = {<<>>, 11}, until = 12, type = checkpoint},
#ds_pubrange{id = {<<>>, 12}, until = 13, type = checkpoint}
#ds_pubrange{id = {<<>>, 1}, until = 2, type = ?T_CHECKPOINT},
#ds_pubrange{id = {<<>>, 4}, until = 8, type = ?T_CHECKPOINT},
#ds_pubrange{id = {<<>>, 11}, until = 12, type = ?T_CHECKPOINT},
#ds_pubrange{id = {<<>>, 12}, until = 13, type = ?T_CHECKPOINT}
])
)
].

282
apps/emqx/src/emqx_persistent_session_ds.erl
View File

@ -56,13 +56,16 @@
deliver/3,
replay/3,
handle_timeout/3,
disconnect/1,
disconnect/2,
terminate/2
]).
%% session table operations
-export([create_tables/0]).
%% internal export used by session GC process
-export([destroy_session/1]).
%% Remove me later (satisfy checks for an unused BPAPI)
-export([
do_open_iterator/3,
@ -74,7 +77,7 @@
-ifdef(TEST).
-export([
session_open/1,
session_open/2,
list_all_sessions/0,
list_all_subscriptions/0,
list_all_streams/0,
@ -98,22 +101,26 @@
id := id(),
%% When the session was created
created_at := timestamp(),
%% When the session should expire
expires_at := timestamp() | never,
%% When the client was last considered alive
last_alive_at := timestamp(),
%% Client’s Subscriptions.
subscriptions := #{topic_filter() => subscription()},
%% Inflight messages
inflight := emqx_persistent_message_ds_replayer:inflight(),
%% Receive maximum
receive_maximum := pos_integer(),
%% Connection Info
conninfo := emqx_types:conninfo(),
%%
props := map()
}.
-type timestamp() :: emqx_utils_calendar:epoch_millisecond().
-type millisecond() :: non_neg_integer().
-type clientinfo() :: emqx_types:clientinfo().
-type conninfo() :: emqx_session:conninfo().
-type replies() :: emqx_session:replies().
-type timer() :: pull | get_streams | bump_last_alive_at.
-define(STATS_KEYS, [
subscriptions_cnt,
@ -123,6 +130,12 @@
next_pkt_id
]).
-define(IS_EXPIRED(NOW_MS, LAST_ALIVE_AT, EI),
(is_number(LAST_ALIVE_AT) andalso
is_number(EI) andalso
(NOW_MS >= LAST_ALIVE_AT + EI))
).
-export_type([id/0]).
%%
@ -144,26 +157,24 @@ open(#{clientid := ClientID} = _ClientInfo, ConnInfo) ->
%% somehow isolate those idling not-yet-expired sessions into a separate process
%% space, and move this call back into `emqx_cm` where it belongs.
ok = emqx_cm:discard_session(ClientID),
case maps:get(clean_start, ConnInfo, false) of
case session_open(ClientID, ConnInfo) of
Session0 = #{} ->
ensure_timers(),
ReceiveMaximum = receive_maximum(ConnInfo),
Session = Session0#{receive_maximum => ReceiveMaximum},
{true, Session, []};
false ->
case session_open(ClientID) of
Session0 = #{} ->
ensure_timers(),
ReceiveMaximum = receive_maximum(ConnInfo),
Session = Session0#{receive_maximum => ReceiveMaximum},
{true, Session, []};
false ->
false
end;
true ->
session_drop(ClientID),
false
end.
ensure_session(ClientID, ConnInfo, Conf) ->
Session = session_ensure_new(ClientID, Conf),
Session = session_ensure_new(ClientID, ConnInfo, Conf),
ReceiveMaximum = receive_maximum(ConnInfo),
Session#{subscriptions => #{}, receive_maximum => ReceiveMaximum}.
Session#{
conninfo => ConnInfo,
receive_maximum => ReceiveMaximum,
subscriptions => #{}
}.
-spec destroy(session() | clientinfo()) -> ok.
destroy(#{id := ClientID}) ->
@ -239,6 +250,7 @@ print_session(ClientId) ->
session => Session,
streams => mnesia:read(?SESSION_STREAM_TAB, ClientId),
pubranges => session_read_pubranges(ClientId),
offsets => session_read_offsets(ClientId),
subscriptions => session_read_subscriptions(ClientId)
};
[] ->
@ -319,12 +331,13 @@ publish(_PacketId, Msg, Session) ->
{ok, emqx_types:message(), replies(), session()}
| {error, emqx_types:reason_code()}.
puback(_ClientInfo, PacketId, Session = #{id := Id, inflight := Inflight0}) ->
case emqx_persistent_message_ds_replayer:commit_offset(Id, PacketId, Inflight0) of
case emqx_persistent_message_ds_replayer:commit_offset(Id, ack, PacketId, Inflight0) of
{true, Inflight} ->
%% TODO
Msg = #message{},
Msg = emqx_message:make(Id, <<>>, <<>>),
{ok, Msg, [], Session#{inflight => Inflight}};
{false, _} ->
%% Invalid Packet Id
{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}
end.
@ -335,9 +348,16 @@ puback(_ClientInfo, PacketId, Session = #{id := Id, inflight := Inflight0}) ->
-spec pubrec(emqx_types:packet_id(), session()) ->
{ok, emqx_types:message(), session()}
| {error, emqx_types:reason_code()}.
pubrec(_PacketId, _Session = #{}) ->
% TODO: stub
{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}.
pubrec(PacketId, Session = #{id := Id, inflight := Inflight0}) ->
case emqx_persistent_message_ds_replayer:commit_offset(Id, rec, PacketId, Inflight0) of
{true, Inflight} ->
%% TODO
Msg = emqx_message:make(Id, <<>>, <<>>),
{ok, Msg, Session#{inflight => Inflight}};
{false, _} ->
%% Invalid Packet Id
{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}
end.
%%--------------------------------------------------------------------
%% Client -> Broker: PUBREL
@ -356,9 +376,16 @@ pubrel(_PacketId, Session = #{}) ->
-spec pubcomp(clientinfo(), emqx_types:packet_id(), session()) ->
{ok, emqx_types:message(), replies(), session()}
| {error, emqx_types:reason_code()}.
pubcomp(_ClientInfo, _PacketId, _Session = #{}) ->
% TODO: stub
{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}.
pubcomp(_ClientInfo, PacketId, Session = #{id := Id, inflight := Inflight0}) ->
case emqx_persistent_message_ds_replayer:commit_offset(Id, comp, PacketId, Inflight0) of
{true, Inflight} ->
%% TODO
Msg = emqx_message:make(Id, <<>>, <<>>),
{ok, Msg, [], Session#{inflight => Inflight}};
{false, _} ->
%% Invalid Packet Id
{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}
end.
%%--------------------------------------------------------------------
@ -375,7 +402,18 @@ handle_timeout(
pull,
Session = #{id := Id, inflight := Inflight0, receive_maximum := ReceiveMaximum}
) ->
{Publishes, Inflight} = emqx_persistent_message_ds_replayer:poll(Id, Inflight0, ReceiveMaximum),
{Publishes, Inflight} = emqx_persistent_message_ds_replayer:poll(
fun
(_Seqno, Message = #message{qos = ?QOS_0}) ->
{false, {undefined, Message}};
(Seqno, Message) ->
PacketId = emqx_persistent_message_ds_replayer:seqno_to_packet_id(Seqno),
{PacketId, Message}
end,
Id,
Inflight0,
ReceiveMaximum
),
IdlePollInterval = emqx_config:get([session_persistence, idle_poll_interval]),
Timeout =
case Publishes of
@ -385,22 +423,50 @@ handle_timeout(
0
end,
ensure_timer(pull, Timeout),
{ok, Publishes, Session#{inflight => Inflight}};
{ok, Publishes, Session#{inflight := Inflight}};
handle_timeout(_ClientInfo, get_streams, Session) ->
renew_streams(Session),
ensure_timer(get_streams),
{ok, [], Session};
handle_timeout(_ClientInfo, bump_last_alive_at, Session0) ->
%% Note: we take a pessimistic approach here and assume that the client will be alive
%% until the next bump timeout. With this, we avoid garbage collecting this session
%% too early in case the session/connection/node crashes earlier without having time
%% to commit the time.
BumpInterval = emqx_config:get([session_persistence, last_alive_update_interval]),
EstimatedLastAliveAt = now_ms() + BumpInterval,
Session = session_set_last_alive_at_trans(Session0, EstimatedLastAliveAt),
ensure_timer(bump_last_alive_at),
{ok, [], Session}.
-spec replay(clientinfo(), [], session()) ->
{ok, replies(), session()}.
replay(_ClientInfo, [], Session = #{inflight := Inflight0}) ->
{Replies, Inflight} = emqx_persistent_message_ds_replayer:replay(Inflight0),
AckedUntil = emqx_persistent_message_ds_replayer:committed_until(ack, Inflight0),
RecUntil = emqx_persistent_message_ds_replayer:committed_until(rec, Inflight0),
CompUntil = emqx_persistent_message_ds_replayer:committed_until(comp, Inflight0),
ReplyFun = fun
(_Seqno, #message{qos = ?QOS_0}) ->
{false, []};
(Seqno, #message{qos = ?QOS_1}) when Seqno < AckedUntil ->
[];
(Seqno, #message{qos = ?QOS_2}) when Seqno < CompUntil ->
[];
(Seqno, #message{qos = ?QOS_2}) when Seqno < RecUntil ->
PacketId = emqx_persistent_message_ds_replayer:seqno_to_packet_id(Seqno),
{pubrel, PacketId};
(Seqno, Message) ->
PacketId = emqx_persistent_message_ds_replayer:seqno_to_packet_id(Seqno),
{PacketId, emqx_message:set_flag(dup, true, Message)}
end,
{Replies, Inflight} = emqx_persistent_message_ds_replayer:replay(ReplyFun, Inflight0),
{ok, Replies, Session#{inflight := Inflight}}.
%%--------------------------------------------------------------------
-spec disconnect(session()) -> {shutdown, session()}.
disconnect(Session = #{}) ->
-spec disconnect(session(), emqx_types:conninfo()) -> {shutdown, session()}.
disconnect(Session0, ConnInfo) ->
Session = session_set_last_alive_at_trans(Session0, ConnInfo, now_ms()),
{shutdown, Session}.
-spec terminate(Reason :: term(), session()) -> ok.
@ -507,11 +573,22 @@ create_tables() ->
{attributes, record_info(fields, ds_pubrange)}
]
),
ok = mria:create_table(
?SESSION_COMMITTED_OFFSET_TAB,
[
{rlog_shard, ?DS_MRIA_SHARD},
{type, set},
{storage, storage()},
{record_name, ds_committed_offset},
{attributes, record_info(fields, ds_committed_offset)}
]
),
ok = mria:wait_for_tables([
?SESSION_TAB,
?SESSION_SUBSCRIPTIONS_TAB,
?SESSION_STREAM_TAB,
?SESSION_PUBRANGE_TAB
?SESSION_PUBRANGE_TAB,
?SESSION_COMMITTED_OFFSET_TAB
]),
ok.
@ -530,47 +607,84 @@ storage() ->
%%
%% Note: session API doesn't handle session takeovers, it's the job of
%% the broker.
-spec session_open(id()) ->
-spec session_open(id(), emqx_types:conninfo()) ->
session() | false.
session_open(SessionId) ->
ro_transaction(fun() ->
session_open(SessionId, NewConnInfo) ->
NowMS = now_ms(),
transaction(fun() ->
case mnesia:read(?SESSION_TAB, SessionId, write) of
[Record = #session{}] ->
Session = export_session(Record),
DSSubs = session_read_subscriptions(SessionId),
Subscriptions = export_subscriptions(DSSubs),
Inflight = emqx_persistent_message_ds_replayer:open(SessionId),
Session#{
subscriptions => Subscriptions,
inflight => Inflight
};
[] ->
[Record0 = #session{last_alive_at = LastAliveAt, conninfo = ConnInfo}] ->
EI = expiry_interval(ConnInfo),
case ?IS_EXPIRED(NowMS, LastAliveAt, EI) of
true ->
session_drop(SessionId),
false;
false ->
%% new connection being established
Record1 = Record0#session{conninfo = NewConnInfo},
Record = session_set_last_alive_at(Record1, NowMS),
Session = export_session(Record),
DSSubs = session_read_subscriptions(SessionId),
Subscriptions = export_subscriptions(DSSubs),
Inflight = emqx_persistent_message_ds_replayer:open(SessionId),
Session#{
conninfo => NewConnInfo,
inflight => Inflight,
subscriptions => Subscriptions
}
end;
_ ->
false
end
end).
-spec session_ensure_new(id(), _Props :: map()) ->
-spec session_ensure_new(id(), emqx_types:conninfo(), _Props :: map()) ->
session().
session_ensure_new(SessionId, Props) ->
session_ensure_new(SessionId, ConnInfo, Props) ->
transaction(fun() ->
ok = session_drop_subscriptions(SessionId),
Session = export_session(session_create(SessionId, Props)),
Session = export_session(session_create(SessionId, ConnInfo, Props)),
Session#{
subscriptions => #{},
inflight => emqx_persistent_message_ds_replayer:new()
}
end).
session_create(SessionId, Props) ->
session_create(SessionId, ConnInfo, Props) ->
Session = #session{
id = SessionId,
created_at = erlang:system_time(millisecond),
expires_at = never,
created_at = now_ms(),
last_alive_at = now_ms(),
conninfo = ConnInfo,
props = Props
},
ok = mnesia:write(?SESSION_TAB, Session, write),
Session.
session_set_last_alive_at_trans(Session, LastAliveAt) ->
#{conninfo := ConnInfo} = Session,
session_set_last_alive_at_trans(Session, ConnInfo, LastAliveAt).
session_set_last_alive_at_trans(Session, NewConnInfo, LastAliveAt) ->
#{id := SessionId} = Session,
transaction(fun() ->
case mnesia:read(?SESSION_TAB, SessionId, write) of
[#session{} = SessionRecord0] ->
SessionRecord = SessionRecord0#session{conninfo = NewConnInfo},
_ = session_set_last_alive_at(SessionRecord, LastAliveAt),
ok;
_ ->
%% log and crash?
ok
end
end),
Session#{conninfo := NewConnInfo, last_alive_at := LastAliveAt}.
session_set_last_alive_at(SessionRecord0, LastAliveAt) ->
SessionRecord = SessionRecord0#session{last_alive_at = LastAliveAt},
ok = mnesia:write(?SESSION_TAB, SessionRecord, write),
SessionRecord.
%% @doc Called when a client reconnects with `clean session=true' or
%% during session GC
-spec session_drop(id()) -> ok.
@ -578,6 +692,7 @@ session_drop(DSSessionId) ->
transaction(fun() ->
ok = session_drop_subscriptions(DSSessionId),
ok = session_drop_pubranges(DSSessionId),
ok = session_drop_offsets(DSSessionId),
ok = session_drop_streams(DSSessionId),
ok = mnesia:delete(?SESSION_TAB, DSSessionId, write)
end).
@ -669,11 +784,22 @@ session_read_pubranges(DSSessionId, LockKind) ->
),
mnesia:select(?SESSION_PUBRANGE_TAB, MS, LockKind).
session_read_offsets(DSSessionID) ->
session_read_offsets(DSSessionID, read).
session_read_offsets(DSSessionId, LockKind) ->
MS = ets:fun2ms(
fun(#ds_committed_offset{id = {Sess, Type}}) when Sess =:= DSSessionId ->
{DSSessionId, Type}
end
),
mnesia:select(?SESSION_COMMITTED_OFFSET_TAB, MS, LockKind).
-spec new_subscription_id(id(), topic_filter()) -> {subscription_id(), integer()}.
new_subscription_id(DSSessionId, TopicFilter) ->
%% Note: here we use _milliseconds_ to match with the timestamp
%% field of `#message' record.
NowMS = erlang:system_time(millisecond),
NowMS = now_ms(),
DSSubId = {DSSessionId, TopicFilter},
{DSSubId, NowMS}.
@ -681,6 +807,9 @@ new_subscription_id(DSSessionId, TopicFilter) ->
subscription_id_to_topic_filter({_DSSessionId, TopicFilter}) ->
TopicFilter.
now_ms() ->
erlang:system_time(millisecond).
%%--------------------------------------------------------------------
%% RPC targets (v1)
%%--------------------------------------------------------------------
@ -778,11 +907,27 @@ session_drop_pubranges(DSSessionId) ->
RangeIds
).
%% must be called inside a transaction
-spec session_drop_offsets(id()) -> ok.
session_drop_offsets(DSSessionId) ->
OffsetIds = session_read_offsets(DSSessionId, write),
lists:foreach(
fun(OffsetId) ->
mnesia:delete(?SESSION_COMMITTED_OFFSET_TAB, OffsetId, write)
end,
OffsetIds
).
%%--------------------------------------------------------------------------------
transaction(Fun) ->
{atomic, Res} = mria:transaction(?DS_MRIA_SHARD, Fun),
Res.
case mnesia:is_transaction() of
true ->
Fun();
false ->
{atomic, Res} = mria:transaction(?DS_MRIA_SHARD, Fun),
Res
end.
ro_transaction(Fun) ->
{atomic, Res} = mria:ro_transaction(?DS_MRIA_SHARD, Fun),
@ -800,7 +945,7 @@ export_subscriptions(DSSubs) ->
).
export_session(#session{} = Record) ->
export_record(Record, #session.id, [id, created_at, expires_at, props], #{}).
export_record(Record, #session.id, [id, created_at, last_alive_at, conninfo, props], #{}).
export_subscription(#ds_sub{} = Record) ->
export_record(Record, #ds_sub.start_time, [start_time, props, extra], #{}).
@ -814,13 +959,17 @@ export_record(_, _, [], Acc) ->
%% effects. Add `CBM:init' callback to the session behavior?
ensure_timers() ->
ensure_timer(pull),
ensure_timer(get_streams).
ensure_timer(get_streams),
ensure_timer(bump_last_alive_at).
-spec ensure_timer(pull | get_streams) -> ok.
-spec ensure_timer(timer()) -> ok.
ensure_timer(bump_last_alive_at = Type) ->
BumpInterval = emqx_config:get([session_persistence, last_alive_update_interval]),
ensure_timer(Type, BumpInterval);
ensure_timer(Type) ->
ensure_timer(Type, 100).
-spec ensure_timer(pull | get_streams, non_neg_integer()) -> ok.
-spec ensure_timer(timer(), non_neg_integer()) -> ok.
ensure_timer(Type, Timeout) ->
_ = emqx_utils:start_timer(Timeout, {emqx_session, Type}),
ok.
@ -832,11 +981,24 @@ receive_maximum(ConnInfo) ->
%% indicates that it's optional.
maps:get(receive_maximum, ConnInfo, 65_535).
-spec expiry_interval(conninfo()) -> millisecond().
expiry_interval(ConnInfo) ->
maps:get(expiry_interval, ConnInfo, 0).
-ifdef(TEST).
list_all_sessions() ->
DSSessionIds = mnesia:dirty_all_keys(?SESSION_TAB),
Sessions = lists:map(
fun(SessionID) -> {SessionID, session_open(SessionID)} end,
ConnInfo = #{},
Sessions = lists:filtermap(
fun(SessionID) ->
Sess = session_open(SessionID, ConnInfo),
case Sess of
false ->
false;
_ ->
{true, {SessionID, Sess}}
end
end,
DSSessionIds
),
maps:from_list(Sessions).

24
apps/emqx/src/emqx_persistent_session_ds.hrl
View File

@ -22,8 +22,12 @@
-define(SESSION_SUBSCRIPTIONS_TAB, emqx_ds_session_subscriptions).
-define(SESSION_STREAM_TAB, emqx_ds_stream_tab).
-define(SESSION_PUBRANGE_TAB, emqx_ds_pubrange_tab).
-define(SESSION_COMMITTED_OFFSET_TAB, emqx_ds_committed_offset_tab).
-define(DS_MRIA_SHARD, emqx_ds_session_shard).
-define(T_INFLIGHT, 1).
-define(T_CHECKPOINT, 2).
-record(ds_sub, {
id :: emqx_persistent_session_ds:subscription_id(),
start_time :: emqx_ds:time(),
@ -56,7 +60,11 @@
%% * Inflight range is a range of yet unacked messages from this stream.
%% * Checkpoint range was already acked, its purpose is to keep track of the
%% very last iterator for this stream.
type :: inflight | checkpoint,
type :: ?T_INFLIGHT | ?T_CHECKPOINT,
%% What commit tracks this range is part of.
%% This is rarely stored: we only need to persist it when the range contains
%% QoS 2 messages.
tracks = 0 :: non_neg_integer(),
%% Meaning of this depends on the type of the range:
%% * For inflight range, this is the iterator pointing to the first message in
%% the range.
@ -68,12 +76,24 @@
}).
-type ds_pubrange() :: #ds_pubrange{}.
-record(ds_committed_offset, {
id :: {
%% What session this marker belongs to.
_Session :: emqx_persistent_session_ds:id(),
%% Marker name.
_CommitType
},
%% Where this marker is pointing to: the first seqno that is not marked.
until :: emqx_persistent_message_ds_replayer:seqno()
}).
-record(session, {
%% same as clientid
id :: emqx_persistent_session_ds:id(),
%% creation time
created_at :: _Millisecond :: non_neg_integer(),
expires_at = never :: _Millisecond :: non_neg_integer() | never,
last_alive_at :: _Millisecond :: non_neg_integer(),
conninfo :: emqx_types:conninfo(),
%% for future usage
props = #{} :: map()
}).

161
apps/emqx/src/emqx_persistent_session_ds_gc_worker.erl Normal file
View File

@ -0,0 +1,161 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_persistent_session_ds_gc_worker).
-behaviour(gen_server).
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("stdlib/include/qlc.hrl").
-include_lib("stdlib/include/ms_transform.hrl").
-include("emqx_persistent_session_ds.hrl").
%% API
-export([
start_link/0
]).
%% `gen_server' API
-export([
init/1,
handle_call/3,
handle_cast/2,
handle_info/2
]).
%% call/cast/info records
-record(gc, {}).
%%--------------------------------------------------------------------------------
%% API
%%--------------------------------------------------------------------------------
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
%%--------------------------------------------------------------------------------
%% `gen_server' API
%%--------------------------------------------------------------------------------
init(_Opts) ->
ensure_gc_timer(),
State = #{},
{ok, State}.
handle_call(_Call, _From, State) ->
{reply, error, State}.
handle_cast(_Cast, State) ->
{noreply, State}.
handle_info(#gc{}, State) ->
try_gc(),
ensure_gc_timer(),
{noreply, State};
handle_info(_Info, State) ->
{noreply, State}.
%%--------------------------------------------------------------------------------
%% Internal fns
%%--------------------------------------------------------------------------------
ensure_gc_timer() ->
Timeout = emqx_config:get([session_persistence, session_gc_interval]),
_ = erlang:send_after(Timeout, self(), #gc{}),
ok.
try_gc() ->
%% Only cores should run GC.
CoreNodes = mria_membership:running_core_nodelist(),
Res = global:trans(
{?MODULE, self()},
fun() -> ?tp_span(ds_session_gc, #{}, start_gc()) end,
CoreNodes,
%% Note: we set retries to 1 here because, in rare occasions, GC might start at the
%% same time in more than one node, and each one will abort the other. By allowing
%% one retry, at least one node will (hopefully) get to enter the transaction and
%% the other will abort. If GC runs too fast, both nodes might run in sequence.
%% But, in that case, GC is clearly not too costly, and that shouldn't be a problem,
%% resource-wise.
_Retries = 1
),
case Res of
aborted ->
?tp(ds_session_gc_lock_taken, #{}),
ok;
ok ->
ok
end.
now_ms() ->
erlang:system_time(millisecond).
start_gc() ->
do_gc(more).
zombie_session_ms() ->
NowMS = now_ms(),
GCInterval = emqx_config:get([session_persistence, session_gc_interval]),
BumpInterval = emqx_config:get([session_persistence, last_alive_update_interval]),
TimeThreshold = max(GCInterval, BumpInterval) * 3,
ets:fun2ms(
fun(
#session{
id = DSSessionId,
last_alive_at = LastAliveAt,
conninfo = #{expiry_interval := EI}
}
) when
LastAliveAt + EI + TimeThreshold =< NowMS
->
DSSessionId
end
).
do_gc(more) ->
GCBatchSize = emqx_config:get([session_persistence, session_gc_batch_size]),
MS = zombie_session_ms(),
{atomic, Next} = mria:transaction(?DS_MRIA_SHARD, fun() ->
Res = mnesia:select(?SESSION_TAB, MS, GCBatchSize, write),
case Res of
'$end_of_table' ->
done;
{[], Cont} ->
%% since `GCBatchsize' is just a "recommendation" for `select', we try only
%% _once_ the continuation and then stop if it yields nothing, to avoid a
%% dead loop.
case mnesia:select(Cont) of
'$end_of_table' ->
done;
{[], _Cont} ->
done;
{DSSessionIds0, _Cont} ->
do_gc_(DSSessionIds0),
more
end;
{DSSessionIds0, _Cont} ->
do_gc_(DSSessionIds0),
more
end
end),
do_gc(Next);
do_gc(done) ->
ok.
do_gc_(DSSessionIds) ->
lists:foreach(fun emqx_persistent_session_ds:destroy_session/1, DSSessionIds),
?tp(ds_session_gc_cleaned, #{session_ids => DSSessionIds}),
ok.

78
apps/emqx/src/emqx_persistent_session_ds_sup.erl Normal file
View File

@ -0,0 +1,78 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_persistent_session_ds_sup).
-behaviour(supervisor).
%% API
-export([
start_link/0
]).
%% `supervisor' API
-export([
init/1
]).
%%--------------------------------------------------------------------------------
%% API
%%--------------------------------------------------------------------------------
start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
%%--------------------------------------------------------------------------------
%% `supervisor' API
%%--------------------------------------------------------------------------------
init(Opts) ->
case emqx_persistent_message:is_persistence_enabled() of
true ->
do_init(Opts);
false ->
ignore
end.
do_init(_Opts) ->
SupFlags = #{
strategy => rest_for_one,
intensity => 10,
period => 2,
auto_shutdown => never
},
CoreChildren = [
worker(gc_worker, emqx_persistent_session_ds_gc_worker, [])
],
Children =
case mria_rlog:role() of
core -> CoreChildren;
replicant -> []
end,
{ok, {SupFlags, Children}}.
%%--------------------------------------------------------------------------------
%% Internal fns
%%--------------------------------------------------------------------------------
worker(Id, Mod, Args) ->
#{
id => Id,
start => {Mod, start_link, Args},
type => worker,
restart => permanent,
shutdown => 10_000,
significant => false
}.

25
apps/emqx/src/emqx_schema.erl
View File

@ -1781,6 +1781,31 @@ fields("session_persistence") ->
desc => ?DESC(session_ds_idle_poll_interval)
}
)},
{"last_alive_update_interval",
sc(
timeout_duration(),
#{
default => <<"5000ms">>,
desc => ?DESC(session_ds_last_alive_update_interval)
}
)},
{"session_gc_interval",
sc(
timeout_duration(),
#{
default => <<"10m">>,
desc => ?DESC(session_ds_session_gc_interval)
}
)},
{"session_gc_batch_size",
sc(
pos_integer(),
#{
default => 100,
importance => ?IMPORTANCE_LOW,
desc => ?DESC(session_ds_session_gc_batch_size)
}
)},
{"force_persistence",
sc(
boolean(),

8
apps/emqx/src/emqx_session.erl
View File

@ -84,7 +84,7 @@
-export([
deliver/3,
handle_timeout/3,
disconnect/2,
disconnect/3,
terminate/3
]).
@ -503,10 +503,10 @@ cancel_timer(Name, Timers) ->
%%--------------------------------------------------------------------
-spec disconnect(clientinfo(), t()) ->
-spec disconnect(clientinfo(), eqmx_types:conninfo(), t()) ->
{idle | shutdown, t()}.
disconnect(_ClientInfo, Session) ->
?IMPL(Session):disconnect(Session).
disconnect(_ClientInfo, ConnInfo, Session) ->
?IMPL(Session):disconnect(Session, ConnInfo).
-spec terminate(clientinfo(), Reason :: term(), t()) ->
ok.

6
apps/emqx/src/emqx_session_mem.erl
View File

@ -87,7 +87,7 @@
deliver/3,
replay/3,
handle_timeout/3,
disconnect/1,
disconnect/2,
terminate/2
]).
@ -725,8 +725,8 @@ append(L1, L2) -> L1 ++ L2.
%%--------------------------------------------------------------------
-spec disconnect(session()) -> {idle, session()}.
disconnect(Session = #session{}) ->
-spec disconnect(session(), emqx_types:conninfo()) -> {idle, session()}.
disconnect(Session = #session{}, _ConnInfo) ->
% TODO: isolate expiry timer / timeout handling here?
{idle, Session}.

3
apps/emqx/src/emqx_vm.erl
View File

@ -418,6 +418,9 @@ get_otp_version() ->
end.
read_otp_version() ->
string:trim(do_read_otp_version()).
do_read_otp_version() ->
ReleasesDir = filename:join([code:root_dir(), "releases"]),
Filename = filename:join([ReleasesDir, emqx_app:get_release(), "BUILD_INFO"]),
case file:read_file(Filename) of

39
apps/emqx/test/emqx_common_test_helpers.erl
View File

@ -753,24 +753,15 @@ start_slave(Name, Opts) when is_map(Opts) ->
case SlaveMod of
ct_slave ->
ct:pal("~p: node data dir: ~s", [Node, NodeDataDir]),
ct_slave:start(
Node,
[
{kill_if_fail, true},
{monitor_master, true},
{init_timeout, 20_000},
{startup_timeout, 20_000},
{erl_flags, erl_flags()},
{env, [
{"HOCON_ENV_OVERRIDE_PREFIX", "EMQX_"},
{"EMQX_NODE__COOKIE", Cookie},
{"EMQX_NODE__DATA_DIR", NodeDataDir}
]}
]
);
Envs = [
{"HOCON_ENV_OVERRIDE_PREFIX", "EMQX_"},
{"EMQX_NODE__COOKIE", Cookie},
{"EMQX_NODE__DATA_DIR", NodeDataDir}
],
emqx_cth_peer:start(Node, erl_flags(), Envs);
slave ->
Env = " -env HOCON_ENV_OVERRIDE_PREFIX EMQX_",
slave:start_link(host(), Name, ebin_path() ++ Env)
Envs = [{"HOCON_ENV_OVERRIDE_PREFIX", "EMQX_"}],
emqx_cth_peer:start(Node, ebin_path(), Envs)
end
end,
case DoStart() of
@ -789,13 +780,7 @@ start_slave(Name, Opts) when is_map(Opts) ->
%% Node stopping
stop_slave(Node0) ->
Node = node_name(Node0),
SlaveMod = get_peer_mod(Node),
erase_peer_mod(Node),
case SlaveMod:stop(Node) of
ok -> ok;
{ok, _} -> ok;
{error, not_started, _} -> ok
end.
emqx_cth_peer:stop(Node).
%% EPMD starting
start_epmd() ->
@ -1022,11 +1007,11 @@ set_envs(Node, Env) ->
).
erl_flags() ->
%% One core and redirecting logs to master
"+S 1:1 -master " ++ atom_to_list(node()) ++ " " ++ ebin_path().
%% One core
["+S", "1:1"] ++ ebin_path().
ebin_path() ->
string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " ").
["-pa" | lists:filter(fun is_lib/1, code:get_path())].
is_lib(Path) ->
string:prefix(Path, code:lib_dir()) =:= nomatch andalso

124
apps/emqx/test/emqx_cth_cluster.erl
View File

@ -38,14 +38,14 @@
%% in `end_per_suite/1` or `end_per_group/2`) with the result from step 2.
-module(emqx_cth_cluster).
-export([start/2]).
-export([start/1, start/2, restart/2]).
-export([stop/1, stop_node/1]).
-export([start_bare_node/2]).
-export([start_bare_nodes/1, start_bare_nodes/2]).
-export([share_load_module/2]).
-export([node_name/1, mk_nodespecs/2]).
-export([start_apps/2, set_node_opts/2]).
-export([start_apps/2]).
-define(APPS_CLUSTERING, [gen_rpc, mria, ekka]).
@ -109,9 +109,12 @@ when
}.
start(Nodes, ClusterOpts) ->
NodeSpecs = mk_nodespecs(Nodes, ClusterOpts),
ct:pal("Starting cluster:\n ~p", [NodeSpecs]),
start(NodeSpecs).
start(NodeSpecs) ->
ct:pal("(Re)starting nodes:\n ~p", [NodeSpecs]),
% 1. Start bare nodes with only basic applications running
_ = emqx_utils:pmap(fun start_node_init/1, NodeSpecs, ?TIMEOUT_NODE_START_MS),
ok = start_nodes_init(NodeSpecs, ?TIMEOUT_NODE_START_MS),
% 2. Start applications needed to enable clustering
% Generally, this causes some applications to restart, but we deliberately don't
% start them yet.
@ -121,6 +124,11 @@ start(Nodes, ClusterOpts) ->
_ = emqx_utils:pmap(fun run_node_phase_apps/1, NodeSpecs, ?TIMEOUT_APPS_START_MS),
[Node || #{name := Node} <- NodeSpecs].
restart(Node, Spec) ->
ct:pal("Stopping peer node ~p", [Node]),
ok = emqx_cth_peer:stop(Node),
start([Spec#{boot_type => restart}]).
mk_nodespecs(Nodes, ClusterOpts) ->
NodeSpecs = lists:zipwith(
fun(N, {Name, Opts}) -> mk_init_nodespec(N, Name, Opts, ClusterOpts) end,
@ -282,8 +290,50 @@ allocate_listener_port(Type, #{base_port := BasePort}) ->
allocate_listener_ports(Types, Spec) ->
lists:foldl(fun maps:merge/2, #{}, [allocate_listener_port(Type, Spec) || Type <- Types]).
start_node_init(Spec = #{name := Node}) ->
Node = start_bare_node(Node, Spec),
start_nodes_init(Specs, Timeout) ->
Names = lists:map(fun(#{name := Name}) -> Name end, Specs),
Nodes = start_bare_nodes(Names, Timeout),
lists:foreach(fun node_init/1, Nodes).
start_bare_nodes(Names) ->
start_bare_nodes(Names, ?TIMEOUT_NODE_START_MS).
start_bare_nodes(Names, Timeout) ->
Args = erl_flags(),
Envs = [],
Waits = lists:map(
fun(Name) ->
WaitTag = {boot_complete, Name},
WaitBoot = {self(), WaitTag},
{ok, _} = emqx_cth_peer:start(Name, Args, Envs, WaitBoot),
WaitTag
end,
Names
),
Deadline = erlang:monotonic_time() + erlang:convert_time_unit(Timeout, millisecond, nanosecond),
Nodes = wait_boot_complete(Waits, Deadline),
lists:foreach(fun(Node) -> pong = net_adm:ping(Node) end, Nodes),
Nodes.
wait_boot_complete([], _) ->
[];
wait_boot_complete(Waits, Deadline) ->
case erlang:monotonic_time() > Deadline of
true ->
error({timeout, Waits});
false ->
ok
end,
receive
{{boot_complete, _Name} = Wait, {started, Node, _Pid}} ->
ct:pal("~p", [Wait]),
[Node | wait_boot_complete(Waits -- [Wait], Deadline)];
{{boot_complete, _Name}, Otherwise} ->
error({unexpected, Otherwise})
after 100 ->
wait_boot_complete(Waits, Deadline)
end.
node_init(Node) ->
% Make it possible to call `ct:pal` and friends (if running under rebar3)
_ = share_load_module(Node, cthr),
% Enable snabbkaffe trace forwarding
@ -300,12 +350,6 @@ run_node_phase_apps(Spec = #{name := Node}) ->
ok = start_apps(Node, Spec),
ok.
set_node_opts(Node, Spec) ->
erpc:call(Node, persistent_term, put, [{?MODULE, opts}, Spec]).
get_node_opts(Node) ->
erpc:call(Node, persistent_term, get, [{?MODULE, opts}]).
load_apps(Node, #{apps := Apps}) ->
erpc:call(Node, emqx_cth_suite, load_apps, [Apps]).
@ -322,8 +366,12 @@ start_apps(Node, #{apps := Apps} = Spec) ->
ok.
suite_opts(Spec) ->
maps:with([work_dir], Spec).
maps:with([work_dir, boot_type], Spec).
maybe_join_cluster(_Node, #{boot_type := restart}) ->
%% when restart, the node should already be in the cluster
%% hence no need to (re)join
ok;
maybe_join_cluster(_Node, #{role := replicant}) ->
ok;
maybe_join_cluster(Node, Spec) ->
@ -352,23 +400,7 @@ stop(Nodes) ->
stop_node(Name) ->
Node = node_name(Name),
try get_node_opts(Node) of
Opts ->
stop_node(Name, Opts)
catch
error:{erpc, _} ->
ok
end.
stop_node(Node, #{driver := ct_slave}) ->
case ct_slave:stop(Node, [{stop_timeout, ?TIMEOUT_NODE_STOP_S}]) of
{ok, _} ->
ok;
{error, Reason, _} when Reason == not_connected; Reason == not_started ->
ok
end;
stop_node(Node, #{driver := slave}) ->
slave:stop(Node).
ok = emqx_cth_peer:stop(Node).
%% Ports
@ -391,36 +423,12 @@ listener_port(BasePort, wss) ->
%%
-spec start_bare_node(atom(), map()) -> node().
start_bare_node(Name, Spec = #{driver := ct_slave}) ->
{ok, Node} = ct_slave:start(
node_name(Name),
[
{kill_if_fail, true},
{monitor_master, true},
{init_timeout, 20_000},
{startup_timeout, 20_000},
{erl_flags, erl_flags()},
{env, []}
]
),
init_bare_node(Node, Spec);
start_bare_node(Name, Spec = #{driver := slave}) ->
{ok, Node} = slave:start_link(host(), Name, ebin_path()),
init_bare_node(Node, Spec).
init_bare_node(Node, Spec) ->
pong = net_adm:ping(Node),
% Preserve node spec right on the remote node
ok = set_node_opts(Node, Spec),
Node.
erl_flags() ->
%% One core and redirecting logs to master
"+S 1:1 -master " ++ atom_to_list(node()) ++ " " ++ ebin_path().
%% One core
["+S", "1:1"] ++ ebin_path().
ebin_path() ->
string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " ").
["-pa" | lists:filter(fun is_lib/1, code:get_path())].
is_lib(Path) ->
string:prefix(Path, code:lib_dir()) =:= nomatch andalso

79
apps/emqx/test/emqx_cth_peer.erl Normal file
View File

@ -0,0 +1,79 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% @doc Common Test Helper proxy module for slave -> peer migration.
%% OTP 26 has slave module deprecated, use peer instead.
-module(emqx_cth_peer).
-export([start/2, start/3, start/4]).
-export([start_link/2, start_link/3, start_link/4]).
-export([stop/1]).
start(Name, Args) ->
start(Name, Args, []).
start(Name, Args, Envs) ->
start(Name, Args, Envs, timer:seconds(20)).
start(Name, Args, Envs, Timeout) when is_atom(Name) ->
do_start(Name, Args, Envs, Timeout, start).
start_link(Name, Args) ->
start_link(Name, Args, []).
start_link(Name, Args, Envs) ->
start_link(Name, Args, Envs, timer:seconds(20)).
start_link(Name, Args, Envs, Timeout) when is_atom(Name) ->
do_start(Name, Args, Envs, Timeout, start_link).
do_start(Name0, Args, Envs, Timeout, Func) when is_atom(Name0) ->
{Name, Host} = parse_node_name(Name0),
{ok, Pid, Node} = peer:Func(#{
name => Name,
host => Host,
args => Args,
env => Envs,
wait_boot => Timeout,
longnames => true,
shutdown => {halt, 1000}
}),
true = register(Node, Pid),
{ok, Node}.
stop(Node) when is_atom(Node) ->
Pid = whereis(Node),
case is_pid(Pid) of
true ->
unlink(Pid),
ok = peer:stop(Pid);
false ->
ct:pal("The control process for node ~p is unexpetedly down", [Node]),
ok
end.
parse_node_name(NodeName) ->
case string:tokens(atom_to_list(NodeName), "@") of
[Name, Host] ->
{list_to_atom(Name), Host};
[_] ->
{NodeName, host()}
end.
host() ->
[_Name, Host] = string:tokens(atom_to_list(node()), "@"),
Host.

3
apps/emqx/test/emqx_cth_suite.erl
View File

@ -453,6 +453,9 @@ stop_apps(Apps) ->
%%
verify_clean_suite_state(#{boot_type := restart}) ->
%% when testing node restart, we do not need to verify clean state
ok;
verify_clean_suite_state(#{work_dir := WorkDir}) ->
{ok, []} = file:list_dir(WorkDir),
false = emqx_schema_hooks:any_injections(),

20
apps/emqx/test/emqx_metrics_worker_SUITE.erl
View File

@ -53,9 +53,9 @@ t_get_metrics(_) ->
?assertMatch(
#{
rate := #{
a := #{current := 0.0, max := 0.0, last5m := 0.0},
b := #{current := 0.0, max := 0.0, last5m := 0.0},
c := #{current := 0.0, max := 0.0, last5m := 0.0}
a := #{current := +0.0, max := +0.0, last5m := +0.0},
b := #{current := +0.0, max := +0.0, last5m := +0.0},
c := #{current := +0.0, max := +0.0, last5m := +0.0}
},
gauges := #{},
counters := #{
@ -118,9 +118,9 @@ t_clear_metrics(_Config) ->
?assertMatch(
#{
rate := #{
a := #{current := 0.0, max := 0.0, last5m := 0.0},
b := #{current := 0.0, max := 0.0, last5m := 0.0},
c := #{current := 0.0, max := 0.0, last5m := 0.0}
a := #{current := +0.0, max := +0.0, last5m := +0.0},
b := #{current := +0.0, max := +0.0, last5m := +0.0},
c := #{current := +0.0, max := +0.0, last5m := +0.0}
},
gauges := #{},
slides := #{},
@ -145,7 +145,7 @@ t_clear_metrics(_Config) ->
#{
counters => #{},
gauges => #{},
rate => #{current => 0.0, last5m => 0.0, max => 0.0},
rate => #{current => +0.0, last5m => +0.0, max => +0.0},
slides => #{}
},
emqx_metrics_worker:get_metrics(?NAME, Id)
@ -160,9 +160,9 @@ t_reset_metrics(_) ->
?assertMatch(
#{
rate := #{
a := #{current := 0.0, max := 0.0, last5m := 0.0},
b := #{current := 0.0, max := 0.0, last5m := 0.0},
c := #{current := 0.0, max := 0.0, last5m := 0.0}
a := #{current := +0.0, max := +0.0, last5m := +0.0},
b := #{current := +0.0, max := +0.0, last5m := +0.0},
c := #{current := +0.0, max := +0.0, last5m := +0.0}
},
gauges := #{},
counters := #{

5
apps/emqx/test/emqx_mountpoint_SUITE.erl
View File

@ -58,9 +58,6 @@ t_mount_share(_) ->
TopicFilters = [T],
?assertEqual(TopicFilter, #share{group = <<"group">>, topic = <<"topic">>}),
%% should not mount share topic when make message.
Msg = emqx_message:make(<<"clientid">>, TopicFilter, <<"payload">>),
?assertEqual(
TopicFilter,
mount(undefined, TopicFilter)
@ -89,8 +86,6 @@ t_unmount_share(_) ->
?assertEqual(TopicFilter, #share{group = <<"group">>, topic = <<"topic">>}),
%% should not unmount share topic when make message.
Msg = emqx_message:make(<<"clientid">>, TopicFilter, <<"payload">>),
?assertEqual(
TopicFilter,
unmount(undefined, TopicFilter)

9
apps/emqx/test/emqx_persistent_messages_SUITE.erl
View File

@ -233,7 +233,7 @@ t_session_subscription_iterators(Config) ->
),
ok.
t_qos0(Config) ->
t_qos0(_Config) ->
Sub = connect(<<?MODULE_STRING "1">>, true, 30),
Pub = connect(<<?MODULE_STRING "2">>, true, 0),
try
@ -258,7 +258,7 @@ t_qos0(Config) ->
emqtt:stop(Pub)
end.
t_publish_as_persistent(Config) ->
t_publish_as_persistent(_Config) ->
Sub = connect(<<?MODULE_STRING "1">>, true, 30),
Pub = connect(<<?MODULE_STRING "2">>, true, 30),
try
@ -272,9 +272,8 @@ t_publish_as_persistent(Config) ->
?assertMatch(
[
#{qos := 0, topic := <<"t/1">>, payload := <<"1">>},
#{qos := 1, topic := <<"t/1">>, payload := <<"2">>}
%% TODO: QoS 2
%% #{qos := 2, topic := <<"t/1">>, payload := <<"3">>}
#{qos := 1, topic := <<"t/1">>, payload := <<"2">>},
#{qos := 2, topic := <<"t/1">>, payload := <<"3">>}
],
receive_messages(3)
)

366
apps/emqx/test/emqx_persistent_session_SUITE.erl
View File

@ -17,6 +17,7 @@
-module(emqx_persistent_session_SUITE).
-include_lib("stdlib/include/assert.hrl").
-include_lib("emqx/include/asserts.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
@ -53,10 +54,10 @@ all() ->
groups() ->
TCs = emqx_common_test_helpers:all(?MODULE),
TCsNonGeneric = [t_choose_impl],
TCGroups = [{group, tcp}, {group, quic}, {group, ws}],
[
{persistence_disabled, [{group, no_kill_connection_process}]},
{persistence_enabled, [{group, no_kill_connection_process}]},
{no_kill_connection_process, [], [{group, tcp}, {group, quic}, {group, ws}]},
{persistence_disabled, TCGroups},
{persistence_enabled, TCGroups},
{tcp, [], TCs},
{quic, [], TCs -- TCsNonGeneric},
{ws, [], TCs -- TCsNonGeneric}
@ -74,7 +75,7 @@ init_per_group(persistence_enabled, Config) ->
{persistence, ds}
| Config
];
init_per_group(Group, Config) when Group == tcp ->
init_per_group(tcp, Config) ->
Apps = emqx_cth_suite:start(
[{emqx, ?config(emqx_config, Config)}],
#{work_dir => emqx_cth_suite:work_dir(Config)}
@ -85,7 +86,7 @@ init_per_group(Group, Config) when Group == tcp ->
{group_apps, Apps}
| Config
];
init_per_group(Group, Config) when Group == ws ->
init_per_group(ws, Config) ->
Apps = emqx_cth_suite:start(
[{emqx, ?config(emqx_config, Config)}],
#{work_dir => emqx_cth_suite:work_dir(Config)}
@ -99,7 +100,7 @@ init_per_group(Group, Config) when Group == ws ->
{group_apps, Apps}
| Config
];
init_per_group(Group, Config) when Group == quic ->
init_per_group(quic, Config) ->
Apps = emqx_cth_suite:start(
[
{emqx,
@ -118,11 +119,7 @@ init_per_group(Group, Config) when Group == quic ->
{ssl, true},
{group_apps, Apps}
| Config
];
init_per_group(no_kill_connection_process, Config) ->
[{kill_connection_process, false} | Config];
init_per_group(kill_connection_process, Config) ->
[{kill_connection_process, true} | Config].
].
get_listener_port(Type, Name) ->
case emqx_config:get([listeners, Type, Name, bind]) of
@ -194,6 +191,8 @@ receive_message_loop(Count, Deadline) ->
receive
{publish, Msg} ->
[Msg | receive_message_loop(Count - 1, Deadline)];
{pubrel, Msg} ->
[{pubrel, Msg} | receive_message_loop(Count - 1, Deadline)];
_Other ->
receive_message_loop(Count, Deadline)
after Timeout ->
@ -201,39 +200,62 @@ receive_message_loop(Count, Deadline) ->
end.
maybe_kill_connection_process(ClientId, Config) ->
case ?config(kill_connection_process, Config) of
true ->
case emqx_cm:lookup_channels(ClientId) of
[] ->
ok;
[ConnectionPid] ->
?assert(is_pid(ConnectionPid)),
Ref = monitor(process, ConnectionPid),
ConnectionPid ! die_if_test,
receive
{'DOWN', Ref, process, ConnectionPid, normal} -> ok
after 3000 -> error(process_did_not_die)
end,
wait_for_cm_unregister(ClientId)
end;
false ->
ok
end.
wait_for_cm_unregister(ClientId) ->
wait_for_cm_unregister(ClientId, 100).
wait_for_cm_unregister(_ClientId, 0) ->
error(cm_did_not_unregister);
wait_for_cm_unregister(ClientId, N) ->
Persistence = ?config(persistence, Config),
case emqx_cm:lookup_channels(ClientId) of
[] ->
ok;
[_] ->
timer:sleep(100),
wait_for_cm_unregister(ClientId, N - 1)
[ConnectionPid] when Persistence == ds ->
Ref = monitor(process, ConnectionPid),
ConnectionPid ! die_if_test,
?assertReceive(
{'DOWN', Ref, process, ConnectionPid, Reason} when
Reason == normal orelse Reason == noproc,
3000
),
wait_connection_process_unregistered(ClientId);
_ ->
ok
end.
wait_connection_process_dies(ClientId) ->
case emqx_cm:lookup_channels(ClientId) of
[] ->
ok;
[ConnectionPid] ->
Ref = monitor(process, ConnectionPid),
?assertReceive(
{'DOWN', Ref, process, ConnectionPid, Reason} when
Reason == normal orelse Reason == noproc,
3000
),
wait_connection_process_unregistered(ClientId)
end.
wait_connection_process_unregistered(ClientId) ->
?retry(
_Timeout = 100,
_Retries = 20,
?assertEqual([], emqx_cm:lookup_channels(ClientId))
).
wait_channel_disconnected(ClientId) ->
?retry(
_Timeout = 100,
_Retries = 20,
case emqx_cm:lookup_channels(ClientId) of
[] ->
false;
[ChanPid] ->
false = emqx_cm:is_channel_connected(ChanPid)
end
).
disconnect_client(ClientPid) ->
ClientId = proplists:get_value(clientid, emqtt:info(ClientPid)),
ok = emqtt:disconnect(ClientPid),
false = wait_channel_disconnected(ClientId),
ok.
messages(Topic, Payloads) ->
messages(Topic, Payloads, ?QOS_2).
@ -272,23 +294,7 @@ do_publish(Messages = [_ | _], PublishFun, WaitForUnregister) ->
lists:foreach(fun(Message) -> PublishFun(Client, Message) end, Messages),
ok = emqtt:disconnect(Client),
%% Snabbkaffe sometimes fails unless all processes are gone.
case WaitForUnregister of
false ->
ok;
true ->
case emqx_cm:lookup_channels(ClientID) of
[] ->
ok;
[ConnectionPid] ->
?assert(is_pid(ConnectionPid)),
Ref1 = monitor(process, ConnectionPid),
receive
{'DOWN', Ref1, process, ConnectionPid, _} -> ok
after 3000 -> error(process_did_not_die)
end,
wait_for_cm_unregister(ClientID)
end
end
WaitForUnregister andalso wait_connection_process_dies(ClientID)
end
),
receive
@ -347,8 +353,6 @@ t_connect_discards_existing_client(Config) ->
end.
%% [MQTT-3.1.2-23]
t_connect_session_expiry_interval(init, Config) -> skip_ds_tc(Config);
t_connect_session_expiry_interval('end', _Config) -> ok.
t_connect_session_expiry_interval(Config) ->
ConnFun = ?config(conn_fun, Config),
Topic = ?config(topic, Config),
@ -356,6 +360,45 @@ t_connect_session_expiry_interval(Config) ->
Payload = <<"test message">>,
ClientId = ?config(client_id, Config),
{ok, Client1} = emqtt:start_link([
{clientid, ClientId},
{proto_ver, v5},
{properties, #{'Session-Expiry-Interval' => 30}}
| Config
]),
{ok, _} = emqtt:ConnFun(Client1),
{ok, _, [?RC_GRANTED_QOS_1]} = emqtt:subscribe(Client1, STopic, ?QOS_1),
ok = emqtt:disconnect(Client1),
maybe_kill_connection_process(ClientId, Config),
publish(Topic, Payload, ?QOS_1),
{ok, Client2} = emqtt:start_link([
{clientid, ClientId},
{proto_ver, v5},
{properties, #{'Session-Expiry-Interval' => 30}},
{clean_start, false}
| Config
]),
{ok, _} = emqtt:ConnFun(Client2),
[Msg | _] = receive_messages(1),
?assertEqual({ok, iolist_to_binary(Topic)}, maps:find(topic, Msg)),
?assertEqual({ok, iolist_to_binary(Payload)}, maps:find(payload, Msg)),
?assertEqual({ok, ?QOS_1}, maps:find(qos, Msg)),
ok = emqtt:disconnect(Client2).
%% [MQTT-3.1.2-23]
%% TODO: un-skip after QoS 2 support is implemented in DS.
t_connect_session_expiry_interval_qos2(init, Config) -> skip_ds_tc(Config);
t_connect_session_expiry_interval_qos2('end', _Config) -> ok.
t_connect_session_expiry_interval_qos2(Config) ->
ConnFun = ?config(conn_fun, Config),
Topic = ?config(topic, Config),
STopic = ?config(stopic, Config),
Payload = <<"test message">>,
ClientId = ?config(client_id, Config),
{ok, Client1} = emqtt:start_link([
{clientid, ClientId},
{proto_ver, v5},
@ -438,7 +481,7 @@ t_cancel_on_disconnect(Config) ->
{ok, _} = emqtt:ConnFun(Client1),
ok = emqtt:disconnect(Client1, 0, #{'Session-Expiry-Interval' => 0}),
wait_for_cm_unregister(ClientId),
wait_connection_process_unregistered(ClientId),
{ok, Client2} = emqtt:start_link([
{clientid, ClientId},
@ -470,7 +513,7 @@ t_persist_on_disconnect(Config) ->
%% Strangely enough, the disconnect is reported as successful by emqtt.
ok = emqtt:disconnect(Client1, 0, #{'Session-Expiry-Interval' => 30}),
wait_for_cm_unregister(ClientId),
wait_connection_process_unregistered(ClientId),
{ok, Client2} = emqtt:start_link([
{clientid, ClientId},
@ -485,8 +528,6 @@ t_persist_on_disconnect(Config) ->
?assertEqual(0, client_info(session_present, Client2)),
ok = emqtt:disconnect(Client2).
t_process_dies_session_expires(init, Config) -> skip_ds_tc(Config);
t_process_dies_session_expires('end', _Config) -> ok.
t_process_dies_session_expires(Config) ->
%% Emulate an error in the connect process,
%% or that the node of the process goes down.
@ -582,7 +623,7 @@ t_publish_many_while_client_is_gone_qos1(Config) ->
{clientid, ClientId},
{properties, #{'Session-Expiry-Interval' => 30}},
{clean_start, true},
{auto_ack, false}
{auto_ack, never}
| Config
]),
{ok, _} = emqtt:ConnFun(Client1),
@ -629,8 +670,7 @@ t_publish_many_while_client_is_gone_qos1(Config) ->
?assertEqual(
get_topicwise_order(Pubs1),
get_topicwise_order(Msgs1),
Msgs1
get_topicwise_order(Msgs1)
),
NAcked = 4,
@ -639,7 +679,7 @@ t_publish_many_while_client_is_gone_qos1(Config) ->
%% Ensure that PUBACKs are propagated to the channel.
pong = emqtt:ping(Client1),
ok = emqtt:disconnect(Client1),
ok = disconnect_client(Client1),
maybe_kill_connection_process(ClientId, Config),
Pubs2 = [
@ -686,23 +726,8 @@ t_publish_many_while_client_is_gone_qos1(Config) ->
[maps:with([packet_id, topic, payload], M) || M <- lists:sublist(Msgs2, NSame)]
),
ok = emqtt:disconnect(Client2).
ok = disconnect_client(Client2).
get_topicwise_order(Msgs) ->
maps:groups_from_list(fun get_msgpub_topic/1, fun get_msgpub_payload/1, Msgs).
get_msgpub_topic(#mqtt_msg{topic = Topic}) ->
Topic;
get_msgpub_topic(#{topic := Topic}) ->
Topic.
get_msgpub_payload(#mqtt_msg{payload = Payload}) ->
Payload;
get_msgpub_payload(#{payload := Payload}) ->
Payload.
t_publish_while_client_is_gone(init, Config) -> skip_ds_tc(Config);
t_publish_while_client_is_gone('end', _Config) -> ok.
t_publish_while_client_is_gone(Config) ->
%% A persistent session should receive messages in its
%% subscription even if the process owning the session dies.
@ -745,6 +770,157 @@ t_publish_while_client_is_gone(Config) ->
ok = emqtt:disconnect(Client2).
t_publish_many_while_client_is_gone(Config) ->
%% A persistent session should receive all of the still unacked messages
%% for its subscriptions after the client dies or reconnects, in addition
%% to PUBRELs for the messages it has PUBRECed. While client must send
%% PUBACKs and PUBRECs in order, those orders are independent of each other.
ClientId = ?config(client_id, Config),
ConnFun = ?config(conn_fun, Config),
ClientOpts = [
{proto_ver, v5},
{clientid, ClientId},
{properties, #{'Session-Expiry-Interval' => 30}},
{auto_ack, never}
| Config
],
{ok, Client1} = emqtt:start_link([{clean_start, true} | ClientOpts]),
{ok, _} = emqtt:ConnFun(Client1),
{ok, _, [?QOS_1]} = emqtt:subscribe(Client1, <<"t/+/foo">>, ?QOS_1),
{ok, _, [?QOS_2]} = emqtt:subscribe(Client1, <<"msg/feed/#">>, ?QOS_2),
{ok, _, [?QOS_2]} = emqtt:subscribe(Client1, <<"loc/+/+/+">>, ?QOS_2),
Pubs1 = [
#mqtt_msg{topic = <<"t/42/foo">>, payload = <<"M1">>, qos = 1},
#mqtt_msg{topic = <<"t/42/foo">>, payload = <<"M2">>, qos = 1},
#mqtt_msg{topic = <<"msg/feed/me">>, payload = <<"M3">>, qos = 2},
#mqtt_msg{topic = <<"loc/1/2/42">>, payload = <<"M4">>, qos = 2},
#mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M5">>, qos = 2},
#mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M6">>, qos = 1},
#mqtt_msg{topic = <<"loc/3/4/5">>, payload = <<"M7">>, qos = 2},
#mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M8">>, qos = 1},
#mqtt_msg{topic = <<"msg/feed/me">>, payload = <<"M9">>, qos = 2}
],
ok = publish_many(Pubs1),
NPubs1 = length(Pubs1),
Msgs1 = receive_messages(NPubs1),
ct:pal("Msgs1 = ~p", [Msgs1]),
NMsgs1 = length(Msgs1),
?assertEqual(NPubs1, NMsgs1),
?assertEqual(
get_topicwise_order(Pubs1),
get_topicwise_order(Msgs1)
),
%% PUBACK every QoS 1 message.
lists:foreach(
fun(PktId) -> ok = emqtt:puback(Client1, PktId) end,
[PktId || #{qos := 1, packet_id := PktId} <- Msgs1]
),
%% PUBREC first `NRecs` QoS 2 messages.
NRecs = 3,
PubRecs1 = lists:sublist([PktId || #{qos := 2, packet_id := PktId} <- Msgs1], NRecs),
lists:foreach(
fun(PktId) -> ok = emqtt:pubrec(Client1, PktId) end,
PubRecs1
),
%% Ensure that PUBACKs / PUBRECs are propagated to the channel.
pong = emqtt:ping(Client1),
%% Receive PUBRELs for the sent PUBRECs.
PubRels1 = receive_messages(NRecs),
ct:pal("PubRels1 = ~p", [PubRels1]),
?assertEqual(
PubRecs1,
[PktId || {pubrel, #{packet_id := PktId}} <- PubRels1],
PubRels1
),
ok = disconnect_client(Client1),
maybe_kill_connection_process(ClientId, Config),
Pubs2 = [
#mqtt_msg{topic = <<"loc/3/4/5">>, payload = <<"M10">>, qos = 2},
#mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M11">>, qos = 1},
#mqtt_msg{topic = <<"msg/feed/friend">>, payload = <<"M12">>, qos = 2}
],
ok = publish_many(Pubs2),
NPubs2 = length(Pubs2),
{ok, Client2} = emqtt:start_link([{clean_start, false} | ClientOpts]),
{ok, _} = emqtt:ConnFun(Client2),
%% Try to receive _at most_ `NPubs` messages.
%% There shouldn't be that much unacked messages in the replay anyway,
%% but it's an easy number to pick.
NPubs = NPubs1 + NPubs2,
Msgs2 = receive_messages(NPubs, _Timeout = 2000),
ct:pal("Msgs2 = ~p", [Msgs2]),
%% We should again receive PUBRELs for the PUBRECs we sent earlier.
?assertEqual(
get_msgs_essentials(PubRels1),
[get_msg_essentials(PubRel) || PubRel = {pubrel, _} <- Msgs2]
),
%% We should receive duplicates only for QoS 2 messages where PUBRELs were
%% not sent, in the same order as the original messages.
Msgs2Dups = [get_msg_essentials(M) || M = #{dup := true} <- Msgs2],
?assertEqual(
Msgs2Dups,
[M || M = #{qos := 2} <- Msgs2Dups]
),
?assertEqual(
get_msgs_essentials(pick_respective_msgs(Msgs2Dups, Msgs1)),
Msgs2Dups
),
%% Now complete all yet incomplete QoS 2 message flows instead.
PubRecs2 = [PktId || #{qos := 2, packet_id := PktId} <- Msgs2],
lists:foreach(
fun(PktId) -> ok = emqtt:pubrec(Client2, PktId) end,
PubRecs2
),
PubRels2 = receive_messages(length(PubRecs2)),
ct:pal("PubRels2 = ~p", [PubRels2]),
?assertEqual(
PubRecs2,
[PktId || {pubrel, #{packet_id := PktId}} <- PubRels2],
PubRels2
),
%% PUBCOMP every PUBREL.
PubComps = [PktId || {pubrel, #{packet_id := PktId}} <- PubRels1 ++ PubRels2],
lists:foreach(
fun(PktId) -> ok = emqtt:pubcomp(Client2, PktId) end,
PubComps
),
%% Ensure that PUBCOMPs are propagated to the channel.
pong = emqtt:ping(Client2),
ok = disconnect_client(Client2),
maybe_kill_connection_process(ClientId, Config),
{ok, Client3} = emqtt:start_link([{clean_start, false} | ClientOpts]),
{ok, _} = emqtt:ConnFun(Client3),
%% Only the last unacked QoS 1 message should be retransmitted.
Msgs3 = receive_messages(NPubs, _Timeout = 2000),
ct:pal("Msgs3 = ~p", [Msgs3]),
?assertMatch(
[#{topic := <<"t/100/foo">>, payload := <<"M11">>, qos := 1, dup := true}],
Msgs3
),
ok = disconnect_client(Client3).
t_clean_start_drops_subscriptions(Config) ->
%% 1. A persistent session is started and disconnected.
%% 2. While disconnected, a message is published and persisted.
@ -795,6 +971,7 @@ t_clean_start_drops_subscriptions(Config) ->
[Msg1] = receive_messages(1),
?assertEqual({ok, iolist_to_binary(Payload2)}, maps:find(payload, Msg1)),
pong = emqtt:ping(Client2),
ok = emqtt:disconnect(Client2),
maybe_kill_connection_process(ClientId, Config),
@ -812,6 +989,7 @@ t_clean_start_drops_subscriptions(Config) ->
[Msg2] = receive_messages(1),
?assertEqual({ok, iolist_to_binary(Payload3)}, maps:find(payload, Msg2)),
pong = emqtt:ping(Client3),
ok = emqtt:disconnect(Client3).
t_unsubscribe(Config) ->
@ -875,6 +1053,30 @@ t_multiple_subscription_matches(Config) ->
?assertEqual({ok, 2}, maps:find(qos, Msg2)),
ok = emqtt:disconnect(Client2).
get_topicwise_order(Msgs) ->
maps:groups_from_list(fun get_msgpub_topic/1, fun get_msgpub_payload/1, Msgs).
get_msgpub_topic(#mqtt_msg{topic = Topic}) ->
Topic;
get_msgpub_topic(#{topic := Topic}) ->
Topic.
get_msgpub_payload(#mqtt_msg{payload = Payload}) ->
Payload;
get_msgpub_payload(#{payload := Payload}) ->
Payload.
get_msg_essentials(Msg = #{}) ->
maps:with([packet_id, topic, payload, qos], Msg);
get_msg_essentials({pubrel, Msg}) ->
{pubrel, maps:with([packet_id, reason_code], Msg)}.
get_msgs_essentials(Msgs) ->
[get_msg_essentials(M) || M <- Msgs].
pick_respective_msgs(MsgRefs, Msgs) ->
[M || M <- Msgs, Ref <- MsgRefs, maps:get(packet_id, M) =:= maps:get(packet_id, Ref)].
skip_ds_tc(Config) ->
case ?config(persistence, Config) of
ds ->

6
apps/emqx/test/emqx_router_helper_SUITE.erl
View File

@ -80,7 +80,7 @@ t_mnesia(_) ->
ct:sleep(200).
t_cleanup_membership_mnesia_down(_Config) ->
Slave = emqx_cth_cluster:node_name(?FUNCTION_NAME),
Slave = emqx_cth_cluster:node_name(node2),
emqx_router:add_route(<<"a/b/c">>, Slave),
emqx_router:add_route(<<"d/e/f">>, node()),
?assertMatch([_, _], emqx_router:topics()),
@ -92,7 +92,7 @@ t_cleanup_membership_mnesia_down(_Config) ->
?assertEqual([<<"d/e/f">>], emqx_router:topics()).
t_cleanup_membership_node_down(_Config) ->
Slave = emqx_cth_cluster:node_name(?FUNCTION_NAME),
Slave = emqx_cth_cluster:node_name(node3),
emqx_router:add_route(<<"a/b/c">>, Slave),
emqx_router:add_route(<<"d/e/f">>, node()),
?assertMatch([_, _], emqx_router:topics()),
@ -104,7 +104,7 @@ t_cleanup_membership_node_down(_Config) ->
?assertEqual([<<"d/e/f">>], emqx_router:topics()).
t_cleanup_monitor_node_down(_Config) ->
Slave = emqx_cth_cluster:start_bare_node(?FUNCTION_NAME, #{driver => ct_slave}),
[Slave] = emqx_cth_cluster:start_bare_nodes([node4]),
emqx_router:add_route(<<"a/b/c">>, Slave),
emqx_router:add_route(<<"d/e/f">>, node()),
?assertMatch([_, _], emqx_router:topics()),

65
apps/emqx/test/emqx_routing_SUITE.erl
View File

@ -218,38 +218,41 @@ t_routing_schema_switch(VFrom, VTo, Config) ->
],
#{work_dir => WorkDir}
),
% Verify that new nodes switched to schema v1/v2 in presence of v1/v2 routes respectively
Nodes = [Node1, Node2, Node3],
?assertEqual(
[{ok, VTo}, {ok, VTo}, {ok, VTo}],
erpc:multicall(Nodes, emqx_router, get_schema_vsn, [])
),
% Wait for all nodes to agree on cluster state
?retry(
500,
10,
?assertMatch(
[{ok, [Node1, Node2, Node3]}],
lists:usort(erpc:multicall(Nodes, emqx, running_nodes, []))
)
),
% Verify that routing works as expected
C2 = start_client(Node2),
ok = subscribe(C2, <<"a/+/d">>),
C3 = start_client(Node3),
ok = subscribe(C3, <<"d/e/f/#">>),
{ok, _} = publish(C1, <<"a/b/d">>, <<"hey-newbies">>),
{ok, _} = publish(C2, <<"a/b/c">>, <<"hi">>),
{ok, _} = publish(C3, <<"d/e/f/42">>, <<"hello">>),
?assertReceive({pub, C2, #{topic := <<"a/b/d">>, payload := <<"hey-newbies">>}}),
?assertReceive({pub, C1, #{topic := <<"a/b/c">>, payload := <<"hi">>}}),
?assertReceive({pub, C1, #{topic := <<"d/e/f/42">>, payload := <<"hello">>}}),
?assertReceive({pub, C3, #{topic := <<"d/e/f/42">>, payload := <<"hello">>}}),
?assertNotReceive(_),
ok = emqtt:stop(C1),
ok = emqtt:stop(C2),
ok = emqtt:stop(C3),
ok = emqx_cth_cluster:stop(Nodes).
try
% Verify that new nodes switched to schema v1/v2 in presence of v1/v2 routes respectively
?assertEqual(
[{ok, VTo}, {ok, VTo}, {ok, VTo}],
erpc:multicall(Nodes, emqx_router, get_schema_vsn, [])
),
% Wait for all nodes to agree on cluster state
?retry(
500,
10,
?assertMatch(
[{ok, [Node1, Node2, Node3]}],
lists:usort(erpc:multicall(Nodes, emqx, running_nodes, []))
)
),
% Verify that routing works as expected
C2 = start_client(Node2),
ok = subscribe(C2, <<"a/+/d">>),
C3 = start_client(Node3),
ok = subscribe(C3, <<"d/e/f/#">>),
{ok, _} = publish(C1, <<"a/b/d">>, <<"hey-newbies">>),
{ok, _} = publish(C2, <<"a/b/c">>, <<"hi">>),
{ok, _} = publish(C3, <<"d/e/f/42">>, <<"hello">>),
?assertReceive({pub, C2, #{topic := <<"a/b/d">>, payload := <<"hey-newbies">>}}),
?assertReceive({pub, C1, #{topic := <<"a/b/c">>, payload := <<"hi">>}}),
?assertReceive({pub, C1, #{topic := <<"d/e/f/42">>, payload := <<"hello">>}}),
?assertReceive({pub, C3, #{topic := <<"d/e/f/42">>, payload := <<"hello">>}}),
?assertNotReceive(_),
ok = emqtt:stop(C1),
ok = emqtt:stop(C2),
ok = emqtt:stop(C3)
after
ok = emqx_cth_cluster:stop(Nodes)
end.
%%

27
apps/emqx/test/emqx_shared_sub_SUITE.erl
View File

@ -63,6 +63,7 @@ init_per_suite(Config) ->
end,
emqx_common_test_helpers:boot_modules(all),
emqx_common_test_helpers:start_apps([]),
emqx_logger:set_log_level(debug),
[{dist_pid, DistPid} | Config].
end_per_suite(Config) ->
@ -574,7 +575,7 @@ t_local(Config) when is_list(Config) ->
<<"sticky_group">> => sticky
},
Node = start_slave('local_shared_sub_testtesttest', 21999),
Node = start_slave('local_shared_sub_local_1', 21999),
ok = ensure_group_config(GroupConfig),
ok = ensure_group_config(Node, GroupConfig),
@ -627,7 +628,7 @@ t_remote(Config) when is_list(Config) ->
<<"sticky_group">> => sticky
},
Node = start_slave('remote_shared_sub_testtesttest', 21999),
Node = start_slave('remote_shared_sub_remote_1', 21999),
ok = ensure_group_config(GroupConfig),
ok = ensure_group_config(Node, GroupConfig),
@ -676,7 +677,7 @@ t_local_fallback(Config) when is_list(Config) ->
Topic = <<"local_foo/bar">>,
ClientId1 = <<"ClientId1">>,
ClientId2 = <<"ClientId2">>,
Node = start_slave('local_fallback_shared_sub_test', 11888),
Node = start_slave('local_fallback_shared_sub_1', 11888),
{ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}]),
{ok, _} = emqtt:connect(ConnPid1),
@ -1253,34 +1254,24 @@ recv_msgs(Count, Msgs) ->
end.
start_slave(Name, Port) ->
{ok, Node} = ct_slave:start(
list_to_atom(atom_to_list(Name) ++ "@" ++ host()),
[
{kill_if_fail, true},
{monitor_master, true},
{init_timeout, 10000},
{startup_timeout, 10000},
{erl_flags, ebin_path()}
]
{ok, Node} = emqx_cth_peer:start_link(
Name,
ebin_path()
),
pong = net_adm:ping(Node),
setup_node(Node, Port),
Node.
stop_slave(Node) ->
rpc:call(Node, mria, leave, []),
ct_slave:stop(Node).
emqx_cth_peer:stop(Node).
host() ->
[_, Host] = string:tokens(atom_to_list(node()), "@"),
Host.
ebin_path() ->
string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " ").
is_lib(Path) ->
string:prefix(Path, code:lib_dir()) =:= nomatch.
["-pa" | code:get_path()].
setup_node(Node, Port) ->
EnvHandler =

2
apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_auth_ldap, [
{description, "EMQX LDAP Authentication and Authorization"},
{vsn, "0.1.1"},
{vsn, "0.1.2"},
{registered, []},
{mod, {emqx_auth_ldap_app, []}},
{applications, [

2
apps/emqx_auth_redis/src/emqx_auth_redis.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_auth_redis, [
{description, "EMQX Redis Authentication and Authorization"},
{vsn, "0.1.1"},
{vsn, "0.1.2"},
{registered, []},
{mod, {emqx_auth_redis_app, []}},
{applications, [

2
apps/emqx_bridge/src/emqx_action_info.erl
View File

@ -90,7 +90,7 @@ hard_coded_action_info_modules_ee() ->
-endif.
hard_coded_action_info_modules_common() ->
[].
[emqx_bridge_http_action_info].
hard_coded_action_info_modules() ->
hard_coded_action_info_modules_common() ++ hard_coded_action_info_modules_ee().

2
apps/emqx_bridge/src/emqx_bridge.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_bridge, [
{description, "EMQX bridges"},
{vsn, "0.1.30"},
{vsn, "0.1.31"},
{registered, [emqx_bridge_sup]},
{mod, {emqx_bridge_app, []}},
{applications, [

2
apps/emqx_bridge/src/emqx_bridge.erl
View File

@ -364,7 +364,7 @@ get_metrics(Type, Name) ->
maybe_upgrade(mqtt, Config) ->
emqx_bridge_compatible_config:maybe_upgrade(Config);
maybe_upgrade(webhook, Config) ->
emqx_bridge_compatible_config:webhook_maybe_upgrade(Config);
emqx_bridge_compatible_config:http_maybe_upgrade(Config);
maybe_upgrade(_Other, Config) ->
Config.

15
apps/emqx_bridge/src/emqx_bridge_api.erl
View File

@ -143,7 +143,7 @@ param_path_id() ->
#{
in => path,
required => true,
example => <<"webhook:webhook_example">>,
example => <<"http:http_example">>,
desc => ?DESC("desc_param_path_id")
}
)}.
@ -166,9 +166,9 @@ bridge_info_array_example(Method) ->
bridge_info_examples(Method) ->
maps:merge(
#{
<<"webhook_example">> => #{
summary => <<"WebHook">>,
value => info_example(webhook, Method)
<<"http_example">> => #{
summary => <<"HTTP">>,
value => info_example(http, Method)
},
<<"mqtt_example">> => #{
summary => <<"MQTT Bridge">>,
@ -201,7 +201,7 @@ method_example(Type, Method) when Method == get; Method == post ->
method_example(_Type, put) ->
#{}.
info_example_basic(webhook) ->
info_example_basic(http) ->
#{
enable => true,
url => <<"http://localhost:9901/messages/${topic}">>,
@ -212,7 +212,7 @@ info_example_basic(webhook) ->
pool_size => 4,
enable_pipelining => 100,
ssl => #{enable => false},
local_topic => <<"emqx_webhook/#">>,
local_topic => <<"emqx_http/#">>,
method => post,
body => <<"${payload}">>,
resource_opts => #{
@ -650,7 +650,8 @@ create_or_update_bridge(BridgeType0, BridgeName, Conf, HttpStatusCode) ->
get_metrics_from_local_node(BridgeType0, BridgeName) ->
BridgeType = upgrade_type(BridgeType0),
format_metrics(emqx_bridge:get_metrics(BridgeType, BridgeName)).
MetricsResult = emqx_bridge:get_metrics(BridgeType, BridgeName),
format_metrics(MetricsResult).
'/bridges/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) ->
?TRY_PARSE_ID(

24
apps/emqx_bridge/src/emqx_bridge_resource.erl
View File

@ -63,18 +63,23 @@
).
-if(?EMQX_RELEASE_EDITION == ee).
bridge_to_resource_type(<<"mqtt">>) -> emqx_bridge_mqtt_connector;
bridge_to_resource_type(mqtt) -> emqx_bridge_mqtt_connector;
bridge_to_resource_type(<<"webhook">>) -> emqx_bridge_http_connector;
bridge_to_resource_type(webhook) -> emqx_bridge_http_connector;
bridge_to_resource_type(BridgeType) -> emqx_bridge_enterprise:resource_type(BridgeType).
bridge_to_resource_type(BridgeType) when is_binary(BridgeType) ->
bridge_to_resource_type(binary_to_existing_atom(BridgeType, utf8));
bridge_to_resource_type(mqtt) ->
emqx_bridge_mqtt_connector;
bridge_to_resource_type(webhook) ->
emqx_bridge_http_connector;
bridge_to_resource_type(BridgeType) ->
emqx_bridge_enterprise:resource_type(BridgeType).
bridge_impl_module(BridgeType) -> emqx_bridge_enterprise:bridge_impl_module(BridgeType).
-else.
bridge_to_resource_type(<<"mqtt">>) -> emqx_bridge_mqtt_connector;
bridge_to_resource_type(mqtt) -> emqx_bridge_mqtt_connector;
bridge_to_resource_type(<<"webhook">>) -> emqx_bridge_http_connector;
bridge_to_resource_type(webhook) -> emqx_bridge_http_connector.
bridge_to_resource_type(BridgeType) when is_binary(BridgeType) ->
bridge_to_resource_type(binary_to_existing_atom(BridgeType, utf8));
bridge_to_resource_type(mqtt) ->
emqx_bridge_mqtt_connector;
bridge_to_resource_type(webhook) ->
emqx_bridge_http_connector.
bridge_impl_module(_BridgeType) -> undefined.
-endif.
@ -309,6 +314,7 @@ remove(Type, Name, _Conf, _Opts) ->
emqx_resource:remove_local(resource_id(Type, Name)).
%% convert bridge configs to what the connector modules want
%% TODO: remove it, if the http_bridge already ported to v2
parse_confs(
<<"webhook">>,
_Name,

86
apps/emqx_bridge/src/emqx_bridge_v2.erl
View File

@ -1188,7 +1188,7 @@ bridge_v1_split_config_and_create(BridgeV1Type, BridgeName, RawConf) ->
%% If the bridge v2 does not exist, it is a valid bridge v1
PreviousRawConf = undefined,
split_bridge_v1_config_and_create_helper(
BridgeV1Type, BridgeName, RawConf, PreviousRawConf
BridgeV1Type, BridgeName, RawConf, PreviousRawConf, fun() -> ok end
);
_Conf ->
case ?MODULE:bridge_v1_is_valid(BridgeV1Type, BridgeName) of
@ -1198,9 +1198,13 @@ bridge_v1_split_config_and_create(BridgeV1Type, BridgeName, RawConf) ->
PreviousRawConf = emqx:get_raw_config(
[?ROOT_KEY, BridgeV2Type, BridgeName], undefined
),
bridge_v1_check_deps_and_remove(BridgeV1Type, BridgeName, RemoveDeps),
%% To avoid losing configurations. We have to make sure that no crash occurs
%% during deletion and creation of configurations.
PreCreateFun = fun() ->
bridge_v1_check_deps_and_remove(BridgeV1Type, BridgeName, RemoveDeps)
end,
split_bridge_v1_config_and_create_helper(
BridgeV1Type, BridgeName, RawConf, PreviousRawConf
BridgeV1Type, BridgeName, RawConf, PreviousRawConf, PreCreateFun
);
false ->
%% If the bridge v2 exists, it is not a valid bridge v1
@ -1208,16 +1212,49 @@ bridge_v1_split_config_and_create(BridgeV1Type, BridgeName, RawConf) ->
end
end.
split_bridge_v1_config_and_create_helper(BridgeV1Type, BridgeName, RawConf, PreviousRawConf) ->
#{
connector_type := ConnectorType,
connector_name := NewConnectorName,
connector_conf := NewConnectorRawConf,
bridge_v2_type := BridgeType,
bridge_v2_name := BridgeName,
bridge_v2_conf := NewBridgeV2RawConf
} =
split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousRawConf),
split_bridge_v1_config_and_create_helper(
BridgeV1Type, BridgeName, RawConf, PreviousRawConf, PreCreateFun
) ->
try
#{
connector_type := ConnectorType,
connector_name := NewConnectorName,
connector_conf := NewConnectorRawConf,
bridge_v2_type := BridgeType,
bridge_v2_name := BridgeName,
bridge_v2_conf := NewBridgeV2RawConf
} = split_and_validate_bridge_v1_config(
BridgeV1Type,
BridgeName,
RawConf,
PreviousRawConf
),
_ = PreCreateFun(),
do_connector_and_bridge_create(
ConnectorType,
NewConnectorName,
NewConnectorRawConf,
BridgeType,
BridgeName,
NewBridgeV2RawConf,
RawConf
)
catch
throw:Reason ->
{error, Reason}
end.
do_connector_and_bridge_create(
ConnectorType,
NewConnectorName,
NewConnectorRawConf,
BridgeType,
BridgeName,
NewBridgeV2RawConf,
RawConf
) ->
case emqx_connector:create(ConnectorType, NewConnectorName, NewConnectorRawConf) of
{ok, _} ->
case create(BridgeType, BridgeName, NewBridgeV2RawConf) of
@ -1335,15 +1372,20 @@ bridge_v1_create_dry_run(BridgeType, RawConfig0) ->
RawConf = maps:without([<<"name">>], RawConfig0),
TmpName = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]),
PreviousRawConf = undefined,
#{
connector_type := _ConnectorType,
connector_name := _NewConnectorName,
connector_conf := ConnectorRawConf,
bridge_v2_type := BridgeV2Type,
bridge_v2_name := _BridgeName,
bridge_v2_conf := BridgeV2RawConf
} = split_and_validate_bridge_v1_config(BridgeType, TmpName, RawConf, PreviousRawConf),
create_dry_run_helper(BridgeV2Type, ConnectorRawConf, BridgeV2RawConf).
try
#{
connector_type := _ConnectorType,
connector_name := _NewConnectorName,
connector_conf := ConnectorRawConf,
bridge_v2_type := BridgeV2Type,
bridge_v2_name := _BridgeName,
bridge_v2_conf := BridgeV2RawConf
} = split_and_validate_bridge_v1_config(BridgeType, TmpName, RawConf, PreviousRawConf),
create_dry_run_helper(BridgeV2Type, ConnectorRawConf, BridgeV2RawConf)
catch
throw:Reason ->
{error, Reason}
end.
%% Only called by test cases (may create broken references)
bridge_v1_remove(BridgeV1Type, BridgeName) ->

2
apps/emqx_bridge/src/emqx_bridge_v2_api.erl
View File

@ -117,7 +117,7 @@ param_path_id() ->
#{
in => path,
required => true,
example => <<"webhook:webhook_example">>,
example => <<"http:my_http_action">>,
desc => ?DESC("desc_param_path_id")
}
)}.

6
apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl
View File

@ -21,7 +21,7 @@
-export([
upgrade_pre_ee/2,
maybe_upgrade/1,
webhook_maybe_upgrade/1
http_maybe_upgrade/1
]).
upgrade_pre_ee(undefined, _UpgradeFunc) ->
@ -40,10 +40,10 @@ maybe_upgrade(#{<<"connector">> := _} = Config0) ->
maybe_upgrade(NewVersion) ->
NewVersion.
webhook_maybe_upgrade(#{<<"direction">> := _} = Config0) ->
http_maybe_upgrade(#{<<"direction">> := _} = Config0) ->
Config1 = maps:remove(<<"direction">>, Config0),
Config1#{<<"resource_opts">> => default_resource_opts()};
webhook_maybe_upgrade(NewVersion) ->
http_maybe_upgrade(NewVersion) ->
NewVersion.
binary_key({K, V}) ->

9
apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
View File

@ -162,13 +162,14 @@ roots() -> [{bridges, ?HOCON(?R_REF(bridges), #{importance => ?IMPORTANCE_LOW})}
fields(bridges) ->
[
{webhook,
{http,
mk(
hoconsc:map(name, ref(emqx_bridge_http_schema, "config")),
#{
aliases => [webhook],
desc => ?DESC("bridges_webhook"),
required => false,
converter => fun webhook_bridge_converter/2
converter => fun http_bridge_converter/2
}
)},
{mqtt,
@ -243,7 +244,7 @@ status() ->
node_name() ->
{"node", mk(binary(), #{desc => ?DESC("desc_node_name"), example => "emqx@127.0.0.1"})}.
webhook_bridge_converter(Conf0, _HoconOpts) ->
http_bridge_converter(Conf0, _HoconOpts) ->
emqx_bridge_compatible_config:upgrade_pre_ee(
Conf0, fun emqx_bridge_compatible_config:webhook_maybe_upgrade/1
Conf0, fun emqx_bridge_compatible_config:http_maybe_upgrade/1
).

58
apps/emqx_bridge/test/emqx_bridge_SUITE.erl
View File

@ -30,14 +30,18 @@ init_per_suite(Config) ->
[
emqx,
emqx_conf,
emqx_connector,
emqx_bridge_http,
emqx_bridge
],
#{work_dir => ?config(priv_dir, Config)}
),
emqx_mgmt_api_test_util:init_suite(),
[{apps, Apps} | Config].
end_per_suite(Config) ->
Apps = ?config(apps, Config),
emqx_mgmt_api_test_util:end_suite(),
ok = emqx_cth_suite:stop(Apps),
ok.
@ -58,6 +62,7 @@ end_per_testcase(t_get_basic_usage_info_1, _Config) ->
ok = emqx_bridge:remove(BridgeType, BridgeName)
end,
[
%% Keep using the old bridge names to avoid breaking the tests
{webhook, <<"basic_usage_info_webhook">>},
{webhook, <<"basic_usage_info_webhook_disabled">>},
{mqtt, <<"basic_usage_info_mqtt">>}
@ -88,7 +93,7 @@ t_get_basic_usage_info_1(_Config) ->
#{
num_bridges => 3,
count_by_type => #{
webhook => 1,
http => 1,
mqtt => 2
}
},
@ -119,40 +124,33 @@ setup_fake_telemetry_data() ->
HTTPConfig = #{
url => <<"http://localhost:9901/messages/${topic}">>,
enable => true,
local_topic => "emqx_webhook/#",
local_topic => "emqx_http/#",
method => post,
body => <<"${payload}">>,
headers => #{},
request_timeout => "15s"
},
Conf =
#{
<<"bridges">> =>
#{
<<"webhook">> =>
#{
<<"basic_usage_info_webhook">> => HTTPConfig,
<<"basic_usage_info_webhook_disabled">> =>
HTTPConfig#{enable => false}
},
<<"mqtt">> =>
#{
<<"basic_usage_info_mqtt">> => MQTTConfig1,
<<"basic_usage_info_mqtt_from_select">> => MQTTConfig2
}
}
},
ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, Conf),
ok = snabbkaffe:start_trace(),
Predicate = fun(#{?snk_kind := K}) -> K =:= emqx_bridge_loaded end,
NEvents = 3,
BackInTime = 0,
Timeout = 11_000,
{ok, Sub} = snabbkaffe_collector:subscribe(Predicate, NEvents, Timeout, BackInTime),
ok = emqx_bridge:load(),
{ok, _} = snabbkaffe_collector:receive_events(Sub),
ok = snabbkaffe:stop(),
%% Keep use the old bridge names to test the backward compatibility
{ok, _} = emqx_bridge_testlib:create_bridge_api(
<<"webhook">>,
<<"basic_usage_info_webhook">>,
HTTPConfig
),
{ok, _} = emqx_bridge_testlib:create_bridge_api(
<<"webhook">>,
<<"basic_usage_info_webhook_disabled">>,
HTTPConfig#{enable => false}
),
{ok, _} = emqx_bridge_testlib:create_bridge_api(
<<"mqtt">>,
<<"basic_usage_info_mqtt">>,
MQTTConfig1
),
{ok, _} = emqx_bridge_testlib:create_bridge_api(
<<"mqtt">>,
<<"basic_usage_info_mqtt_from_select">>,
MQTTConfig2
),
ok.
t_update_ssl_conf(Config) ->

81
apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
View File

@ -73,13 +73,15 @@
-define(HTTP_BRIDGE(URL), ?HTTP_BRIDGE(URL, ?BRIDGE_NAME)).
-define(APPSPECS, [
emqx_conf,
emqx,
emqx_conf,
emqx_auth,
emqx_auth_mnesia,
emqx_management,
{emqx_rule_engine, "rule_engine { rules {} }"},
{emqx_bridge, "bridges {}"}
emqx_connector,
emqx_bridge_http,
{emqx_bridge, "actions {}\n bridges {}"},
{emqx_rule_engine, "rule_engine { rules {} }"}
]).
-define(APPSPEC_DASHBOARD,
@ -108,7 +110,7 @@ groups() ->
].
suite() ->
[{timetrap, {seconds, 60}}].
[{timetrap, {seconds, 120}}].
init_per_suite(Config) ->
Config.
@ -117,10 +119,10 @@ end_per_suite(_Config) ->
ok.
init_per_group(cluster = Name, Config) ->
Nodes = [NodePrimary | _] = mk_cluster(Config),
Nodes = [NodePrimary | _] = mk_cluster(Name, Config),
init_api([{group, Name}, {cluster_nodes, Nodes}, {node, NodePrimary} | Config]);
init_per_group(cluster_later_join = Name, Config) ->
Nodes = [NodePrimary | _] = mk_cluster(Config, #{join_to => undefined}),
Nodes = [NodePrimary | _] = mk_cluster(Name, Config, #{join_to => undefined}),
init_api([{group, Name}, {cluster_nodes, Nodes}, {node, NodePrimary} | Config]);
init_per_group(_Name, Config) ->
WorkDir = emqx_cth_suite:work_dir(Config),
@ -132,10 +134,10 @@ init_api(Config) ->
{ok, App} = erpc:call(APINode, emqx_common_test_http, create_default_app, []),
[{api, App} | Config].
mk_cluster(Config) ->
mk_cluster(Config, #{}).
mk_cluster(Name, Config) ->
mk_cluster(Name, Config, #{}).
mk_cluster(Config, Opts) ->
mk_cluster(Name, Config, Opts) ->
Node1Apps = ?APPSPECS ++ [?APPSPEC_DASHBOARD],
Node2Apps = ?APPSPECS,
emqx_cth_cluster:start(
@ -143,7 +145,7 @@ mk_cluster(Config, Opts) ->
{emqx_bridge_api_SUITE1, Opts#{role => core, apps => Node1Apps}},
{emqx_bridge_api_SUITE2, Opts#{role => core, apps => Node2Apps}}
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
#{work_dir => emqx_cth_suite:work_dir(Name, Config)}
).
end_per_group(Group, Config) when
@ -159,7 +161,7 @@ init_per_testcase(t_broken_bpapi_vsn, Config) ->
meck:new(emqx_bpapi, [passthrough]),
meck:expect(emqx_bpapi, supported_version, 1, -1),
meck:expect(emqx_bpapi, supported_version, 2, -1),
init_per_testcase(commong, Config);
init_per_testcase(common, Config);
init_per_testcase(t_old_bpapi_vsn, Config) ->
meck:new(emqx_bpapi, [passthrough]),
meck:expect(emqx_bpapi, supported_version, 1, 1),
@ -185,6 +187,18 @@ end_per_testcase(_, Config) ->
ok.
clear_resources() ->
lists:foreach(
fun(#{type := Type, name := Name}) ->
ok = emqx_bridge_v2:remove(Type, Name)
end,
emqx_bridge_v2:list()
),
lists:foreach(
fun(#{type := Type, name := Name}) ->
ok = emqx_connector:remove(Type, Name)
end,
emqx_connector:list()
),
lists:foreach(
fun(#{type := Type, name := Name}) ->
ok = emqx_bridge:remove(Type, Name)
@ -407,10 +421,7 @@ t_http_crud_apis(Config) ->
Config
),
?assertMatch(
#{
<<"reason">> := <<"unknown_fields">>,
<<"unknown">> := <<"curl">>
},
#{<<"reason">> := <<"required_field">>},
json(maps:get(<<"message">>, PutFail2))
),
{ok, 400, _} = request_json(
@ -419,12 +430,16 @@ t_http_crud_apis(Config) ->
?HTTP_BRIDGE(<<"localhost:1234/foo">>, Name),
Config
),
{ok, 400, _} = request_json(
{ok, 400, PutFail3} = request_json(
put,
uri(["bridges", BridgeID]),
?HTTP_BRIDGE(<<"htpp://localhost:12341234/foo">>, Name),
Config
),
?assertMatch(
#{<<"kind">> := <<"validation_error">>},
json(maps:get(<<"message">>, PutFail3))
),
%% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), Config),
@ -463,7 +478,7 @@ t_http_crud_apis(Config) ->
),
%% Create non working bridge
BrokenURL = ?URL(Port + 1, "/foo"),
BrokenURL = ?URL(Port + 1, "foo"),
{ok, 201, BrokenBridge} = request(
post,
uri(["bridges"]),
@ -471,6 +486,7 @@ t_http_crud_apis(Config) ->
fun json/1,
Config
),
?assertMatch(
#{
<<"type">> := ?BRIDGE_TYPE_HTTP,
@ -1307,7 +1323,9 @@ t_cluster_later_join_metrics(Config) ->
Name = ?BRIDGE_NAME,
BridgeParams = ?HTTP_BRIDGE(URL1, Name),
BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name),
?check_trace(
#{timetrap => 15_000},
begin
%% Create a bridge on only one of the nodes.
?assertMatch({ok, 201, _}, request_json(post, uri(["bridges"]), BridgeParams, Config)),
@ -1319,8 +1337,26 @@ t_cluster_later_join_metrics(Config) ->
}},
request_json(get, uri(["bridges", BridgeID, "metrics"]), Config)
),
ct:print("node joining cluster"),
%% Now join the other node join with the api node.
ok = erpc:call(OtherNode, ekka, join, [PrimaryNode]),
%% Hack / workaround for the fact that `emqx_machine_boot' doesn't restart the
%% applications, in particular `emqx_conf' doesn't restart and synchronize the
%% transaction id. It's also unclear at the moment why the equivalent test in
%% `emqx_bridge_v2_api_SUITE' doesn't need this hack.
ok = erpc:call(OtherNode, application, stop, [emqx_conf]),
ok = erpc:call(OtherNode, application, start, [emqx_conf]),
ct:print("node joined cluster"),
%% assert: wait for the bridge to be ready on the other node.
{_, {ok, _}} =
?wait_async_action(
{emqx_cluster_rpc, OtherNode} ! wake_up,
#{?snk_kind := cluster_rpc_caught_up, ?snk_meta := #{node := OtherNode}},
10_000
),
%% Check metrics; shouldn't crash even if the bridge is not
%% ready on the node that just joined the cluster.
?assertMatch(
@ -1373,17 +1409,16 @@ t_create_with_bad_name(Config) ->
validate_resource_request_ttl(single, Timeout, Name) ->
SentData = #{payload => <<"Hello EMQX">>, timestamp => 1668602148000},
BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name),
ResId = emqx_bridge_resource:resource_id(<<"webhook">>, Name),
_BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name),
?check_trace(
begin
{ok, Res} =
?wait_async_action(
emqx_bridge:send_message(BridgeID, SentData),
do_send_message(?BRIDGE_TYPE_HTTP, Name, SentData),
#{?snk_kind := async_query},
1000
),
?assertMatch({ok, #{id := ResId, query_opts := #{timeout := Timeout}}}, Res)
?assertMatch({ok, #{id := _ResId, query_opts := #{timeout := Timeout}}}, Res)
end,
fun(Trace0) ->
Trace = ?of_kind(async_query, Trace0),
@ -1394,6 +1429,10 @@ validate_resource_request_ttl(single, Timeout, Name) ->
validate_resource_request_ttl(_Cluster, _Timeout, _Name) ->
ignore.
do_send_message(BridgeV1Type, Name, Message) ->
Type = emqx_bridge_v2:bridge_v1_type_to_bridge_v2_type(BridgeV1Type),
emqx_bridge_v2:send_message(Type, Name, Message, #{}).
%%
request(Method, URL, Config) ->

22
apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl
View File

@ -21,7 +21,7 @@ empty_config_test() ->
Conf1 = #{<<"bridges">> => #{}},
Conf2 = #{<<"bridges">> => #{<<"webhook">> => #{}}},
?assertEqual(Conf1, check(Conf1)),
?assertEqual(Conf2, check(Conf2)),
?assertEqual(#{<<"bridges">> => #{<<"http">> => #{}}}, check(Conf2)),
ok.
%% ensure webhook config can be checked
@ -33,7 +33,7 @@ webhook_config_test() ->
?assertMatch(
#{
<<"bridges">> := #{
<<"webhook">> := #{
<<"http">> := #{
<<"the_name">> :=
#{
<<"method">> := get,
@ -48,7 +48,7 @@ webhook_config_test() ->
?assertMatch(
#{
<<"bridges">> := #{
<<"webhook">> := #{
<<"http">> := #{
<<"the_name">> :=
#{
<<"method">> := get,
@ -61,7 +61,7 @@ webhook_config_test() ->
),
#{
<<"bridges">> := #{
<<"webhook">> := #{
<<"http">> := #{
<<"the_name">> :=
#{
<<"method">> := get,
@ -84,7 +84,7 @@ up(#{<<"mqtt">> := MqttBridges0} = Bridges) ->
Bridges#{<<"mqtt">> := MqttBridges};
up(#{<<"webhook">> := WebhookBridges0} = Bridges) ->
WebhookBridges = emqx_bridge_compatible_config:upgrade_pre_ee(
WebhookBridges0, fun emqx_bridge_compatible_config:webhook_maybe_upgrade/1
WebhookBridges0, fun emqx_bridge_compatible_config:http_maybe_upgrade/1
),
Bridges#{<<"webhook">> := WebhookBridges}.
@ -126,7 +126,7 @@ check(Conf) when is_map(Conf) ->
%% erlfmt-ignore
%% this is config generated from v5.0.11
webhook_v5011_hocon() ->
"""
"
bridges{
webhook {
the_name{
@ -143,7 +143,7 @@ bridges{
}
}
}
""".
".
full_webhook_v5011_hocon() ->
""
@ -215,7 +215,7 @@ full_webhook_v5019_hocon() ->
%% erlfmt-ignore
%% this is a generated from v5.0.11
mqtt_v5011_hocon() ->
"""
"
bridges {
mqtt {
bridge_one {
@ -257,12 +257,12 @@ bridges {
}
}
}
""".
".
%% erlfmt-ignore
%% a more complete version
mqtt_v5011_full_hocon() ->
"""
"
bridges {
mqtt {
bridge_one {
@ -330,4 +330,4 @@ bridges {
}
}
}
""".
".

2
apps/emqx_bridge/test/emqx_bridge_testlib.erl
View File

@ -92,7 +92,7 @@ end_per_testcase(_Testcase, Config) ->
delete_all_bridges() ->
lists:foreach(
fun(#{name := Name, type := Type}) ->
emqx_bridge:remove(Type, Name)
ok = emqx_bridge:remove(Type, Name)
end,
emqx_bridge:list()
).

2
apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl
View File

@ -185,7 +185,7 @@ mk_cluster(Name, Config, Opts) ->
{emqx_bridge_v2_api_SUITE_1, Opts#{role => core, apps => Node1Apps}},
{emqx_bridge_v2_api_SUITE_2, Opts#{role => core, apps => Node2Apps}}
],
#{work_dir => filename:join(?config(priv_dir, Config), Name)}
#{work_dir => emqx_cth_suite:work_dir(Name, Config)}
).
end_per_group(Group, Config) when

73
apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl
View File

@ -146,6 +146,35 @@ create_bridge(Config, Overrides) ->
ct:pal("creating bridge with config: ~p", [BridgeConfig]),
emqx_bridge_v2:create(BridgeType, BridgeName, BridgeConfig).
maybe_json_decode(X) ->
case emqx_utils_json:safe_decode(X, [return_maps]) of
{ok, Decoded} -> Decoded;
{error, _} -> X
end.
request(Method, Path, Params) ->
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
Opts = #{return_all => true},
case emqx_mgmt_api_test_util:request_api(Method, Path, "", AuthHeader, Params, Opts) of
{ok, {Status, Headers, Body0}} ->
Body = maybe_json_decode(Body0),
{ok, {Status, Headers, Body}};
{error, {Status, Headers, Body0}} ->
Body =
case emqx_utils_json:safe_decode(Body0, [return_maps]) of
{ok, Decoded0 = #{<<"message">> := Msg0}} ->
Msg = maybe_json_decode(Msg0),
Decoded0#{<<"message">> := Msg};
{ok, Decoded0} ->
Decoded0;
{error, _} ->
Body0
end,
{error, {Status, Headers, Body}};
Error ->
Error
end.
list_bridges_api() ->
Params = [],
Path = emqx_mgmt_api_test_util:api_path(["actions"]),
@ -209,6 +238,50 @@ create_bridge_api(Config, Overrides) ->
ct:pal("bridge create result: ~p", [Res]),
Res.
create_connector_api(Config) ->
create_connector_api(Config, _Overrides = #{}).
create_connector_api(Config, Overrides) ->
ConnectorConfig0 = ?config(connector_config, Config),
ConnectorName = ?config(connector_name, Config),
ConnectorType = ?config(connector_type, Config),
Method = post,
Path = emqx_mgmt_api_test_util:api_path(["connectors"]),
ConnectorConfig = emqx_utils_maps:deep_merge(ConnectorConfig0, Overrides),
Params = ConnectorConfig#{<<"type">> => ConnectorType, <<"name">> => ConnectorName},
ct:pal("creating connector (http):\n ~p", [Params]),
Res = request(Method, Path, Params),
ct:pal("connector create (http) result:\n ~p", [Res]),
Res.
create_action_api(Config) ->
create_action_api(Config, _Overrides = #{}).
create_action_api(Config, Overrides) ->
ActionName = ?config(action_name, Config),
ActionType = ?config(action_type, Config),
ActionConfig0 = ?config(action_config, Config),
ActionConfig = emqx_utils_maps:deep_merge(ActionConfig0, Overrides),
Params = ActionConfig#{<<"type">> => ActionType, <<"name">> => ActionName},
Method = post,
Path = emqx_mgmt_api_test_util:api_path(["actions"]),
ct:pal("creating action (http):\n ~p", [Params]),
Res = request(Method, Path, Params),
ct:pal("action create (http) result:\n ~p", [Res]),
Res.
get_action_api(Config) ->
ActionName = ?config(action_name, Config),
ActionType = ?config(action_type, Config),
ActionId = emqx_bridge_resource:bridge_id(ActionType, ActionName),
Params = [],
Method = get,
Path = emqx_mgmt_api_test_util:api_path(["actions", ActionId]),
ct:pal("getting action (http)"),
Res = request(Method, Path, Params),
ct:pal("get action (http) result:\n ~p", [Res]),
Res.
update_bridge_api(Config) ->
update_bridge_api(Config, _Overrides = #{}).

2
apps/emqx_bridge_azure_event_hub/rebar.config
View File

@ -2,7 +2,7 @@
{erl_opts, [debug_info]}.
{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.8.0"}}}
, {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.3"}}}
, {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0"}}}
, {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.1"}}}
, {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.8"}}}
, {snappyer, "1.2.9"}
, {emqx_connector, {path, "../../apps/emqx_connector"}}

2
apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_azure_event_hub, [
{description, "EMQX Enterprise Azure Event Hub Bridge"},
{vsn, "0.1.4"},
{vsn, "0.1.5"},
{registered, []},
{applications, [
kernel,

4
apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_tests.erl
View File

@ -12,7 +12,7 @@
%% erlfmt-ignore
aeh_producer_hocon() ->
"""
"
bridges.azure_event_hub_producer.my_producer {
enable = true
authentication {
@ -62,7 +62,7 @@ bridges.azure_event_hub_producer.my_producer {
server_name_indication = auto
}
}
""".
".
%%===========================================================================
%% Helper functions

2
apps/emqx_bridge_confluent/rebar.config
View File

@ -2,7 +2,7 @@
{erl_opts, [debug_info]}.
{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.8.0"}}}
, {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.3"}}}
, {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0"}}}
, {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.1"}}}
, {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.8"}}}
, {snappyer, "1.2.9"}
, {emqx_connector, {path, "../../apps/emqx_connector"}}

4
apps/emqx_bridge_confluent/test/emqx_bridge_confluent_tests.erl
View File

@ -12,7 +12,7 @@
%% erlfmt-ignore
confluent_producer_action_hocon() ->
"""
"
actions.confluent_producer.my_producer {
enable = true
connector = my_connector
@ -40,7 +40,7 @@ actions.confluent_producer.my_producer {
}
local_topic = \"t/confluent\"
}
""".
".
confluent_producer_connector_hocon() ->
""

2
apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_gcp_pubsub, [
{description, "EMQX Enterprise GCP Pub/Sub Bridge"},
{vsn, "0.1.10"},
{vsn, "0.1.11"},
{registered, []},
{applications, [
kernel,

5
apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_worker.erl
View File

@ -237,7 +237,10 @@ handle_continue(?patch_subscription, State0) ->
),
{noreply, State0};
error ->
%% retry
%% retry; add a random delay for the case where multiple workers step on each
%% other's toes before retrying.
RandomMS = rand:uniform(500),
timer:sleep(RandomMS),
{noreply, State0, {continue, ?patch_subscription}}
end.

70
apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl
View File

@ -196,7 +196,7 @@ consumer_config(TestCase, Config) ->
" connect_timeout = \"5s\"\n"
" service_account_json = ~s\n"
" consumer {\n"
" ack_deadline = \"60s\"\n"
" ack_deadline = \"10s\"\n"
" ack_retry_interval = \"1s\"\n"
" pull_max_messages = 10\n"
" consumer_workers_per_topic = 1\n"
@ -520,7 +520,14 @@ wait_acked(Opts) ->
{ok, _} ->
ok;
{timeout, Evts} ->
ct:pal("timed out waiting for acks; received:\n ~p", [Evts])
%% Fixme: apparently, snabbkaffe may timeout but still return the expected
%% events here.
case length(Evts) >= N of
true ->
ok;
false ->
ct:pal("timed out waiting for acks;\n expected: ~b\n received:\n ~p", [N, Evts])
end
end,
ok.
@ -658,25 +665,24 @@ setup_and_start_listeners(Node, NodeOpts) ->
end
).
dedup([]) ->
[];
dedup([X]) ->
[X];
dedup([X | Rest]) ->
[X | dedup(X, Rest)].
dedup(X, [X | Rest]) ->
dedup(X, Rest);
dedup(_X, [Y | Rest]) ->
[Y | dedup(Y, Rest)];
dedup(_X, []) ->
[].
%%------------------------------------------------------------------------------
%% Trace properties
%%------------------------------------------------------------------------------
prop_pulled_only_once() ->
{"all pulled message ids are unique", fun ?MODULE:prop_pulled_only_once/1}.
prop_pulled_only_once(Trace) ->
PulledIds =
[
MsgId
|| #{messages := Msgs} <- ?of_kind(gcp_pubsub_consumer_worker_decoded_messages, Trace),
#{<<"message">> := #{<<"messageId">> := MsgId}} <- Msgs
],
NumPulled = length(PulledIds),
UniquePulledIds = sets:from_list(PulledIds, [{version, 2}]),
UniqueNumPulled = sets:size(UniquePulledIds),
?assertEqual(UniqueNumPulled, NumPulled, #{pulled_ids => PulledIds}),
ok.
prop_handled_only_once() ->
{"all pulled message are processed only once", fun ?MODULE:prop_handled_only_once/1}.
prop_handled_only_once(Trace) ->
@ -1052,7 +1058,6 @@ t_consume_ok(Config) ->
end,
[
prop_all_pulled_are_acked(),
prop_pulled_only_once(),
prop_handled_only_once(),
prop_acked_ids_eventually_forgotten()
]
@ -1125,7 +1130,6 @@ t_bridge_rule_action_source(Config) ->
#{payload => Payload0}
end,
[
prop_pulled_only_once(),
prop_handled_only_once()
]
),
@ -1243,7 +1247,6 @@ t_multiple_topic_mappings(Config) ->
end,
[
prop_all_pulled_are_acked(),
prop_pulled_only_once(),
prop_handled_only_once()
]
),
@ -1276,7 +1279,7 @@ t_multiple_pull_workers(Config) ->
},
<<"resource_opts">> => #{
%% reduce flakiness
<<"request_ttl">> => <<"11s">>
<<"request_ttl">> => <<"20s">>
}
}
),
@ -1304,7 +1307,6 @@ t_multiple_pull_workers(Config) ->
end,
[
prop_all_pulled_are_acked(),
prop_pulled_only_once(),
prop_handled_only_once(),
{"message is processed only once", fun(Trace) ->
?assertMatch({timeout, _}, receive_published(#{timeout => 5_000})),
@ -1543,7 +1545,7 @@ t_async_worker_death_mid_pull(Config) ->
fun(AsyncWorkerPid) ->
Ref = monitor(process, AsyncWorkerPid),
ct:pal("killing pid ~p", [AsyncWorkerPid]),
sys:terminate(AsyncWorkerPid, die, Timeout),
exit(AsyncWorkerPid, kill),
receive
{'DOWN', Ref, process, AsyncWorkerPid, _} ->
ct:pal("killed pid ~p", [AsyncWorkerPid]),
@ -1605,18 +1607,19 @@ t_async_worker_death_mid_pull(Config) ->
],
Trace
),
SubTraceEvts = ?projection(?snk_kind, SubTrace),
?assertMatch(
[
#{?snk_kind := gcp_pubsub_consumer_worker_handled_async_worker_down},
#{?snk_kind := gcp_pubsub_consumer_worker_reply_delegator}
gcp_pubsub_consumer_worker_handled_async_worker_down,
gcp_pubsub_consumer_worker_reply_delegator
| _
],
SubTrace,
dedup(SubTraceEvts),
#{sub_trace => projection_optional_span(SubTrace)}
),
?assertMatch(
#{?snk_kind := gcp_pubsub_consumer_worker_pull_response_received},
lists:last(SubTrace)
gcp_pubsub_consumer_worker_pull_response_received,
lists:last(SubTraceEvts)
),
ok
end
@ -1948,7 +1951,6 @@ t_connection_down_during_ack(Config) ->
end,
[
prop_all_pulled_are_acked(),
prop_pulled_only_once(),
prop_handled_only_once(),
{"message is processed only once", fun(Trace) ->
?assertMatch({timeout, _}, receive_published(#{timeout => 5_000})),
@ -1973,7 +1975,15 @@ t_connection_down_during_ack_redeliver(Config) ->
?wait_async_action(
create_bridge(
Config,
#{<<"consumer">> => #{<<"ack_deadline">> => <<"10s">>}}
#{
<<"consumer">> => #{
<<"ack_deadline">> => <<"12s">>,
<<"ack_retry_interval">> => <<"1s">>
},
<<"resource_opts">> => #{
<<"request_ttl">> => <<"11s">>
}
}
),
#{?snk_kind := "gcp_pubsub_consumer_worker_subscription_ready"},
10_000

4
apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_tests.erl
View File

@ -12,7 +12,7 @@
%% erlfmt-ignore
gcp_pubsub_producer_hocon() ->
"""
"
bridges.gcp_pubsub.my_producer {
attributes_template = [
{key = \"${payload.key}\", value = fixed_value}
@ -54,7 +54,7 @@ bridges.gcp_pubsub.my_producer {
type = service_account
}
}
""".
".
%%===========================================================================
%% Helper functions

2
apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_greptimedb, [
{description, "EMQX GreptimeDB Bridge"},
{vsn, "0.1.4"},
{vsn, "0.1.5"},
{registered, []},
{applications, [
kernel,

2
apps/emqx_bridge_http/src/emqx_bridge_http.app.src
View File

@ -3,7 +3,7 @@
{vsn, "0.1.5"},
{registered, []},
{applications, [kernel, stdlib, emqx_connector, emqx_resource, ehttpc]},
{env, []},
{env, [{emqx_action_info_modules, [emqx_bridge_http_action_info]}]},
{modules, []},
{links, []}
]}.

102
apps/emqx_bridge_http/src/emqx_bridge_http_action_info.erl Normal file
View File

@ -0,0 +1,102 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_bridge_http_action_info).
-behaviour(emqx_action_info).
-export([
bridge_v1_type_name/0,
action_type_name/0,
connector_type_name/0,
schema_module/0,
connector_action_config_to_bridge_v1_config/2,
bridge_v1_config_to_action_config/2,
bridge_v1_config_to_connector_config/1
]).
-define(REMOVED_KEYS, [<<"direction">>]).
-define(ACTION_KEYS, [<<"local_topic">>, <<"resource_opts">>]).
-define(PARAMETER_KEYS, [<<"body">>, <<"max_retries">>, <<"method">>, <<"request_timeout">>]).
bridge_v1_type_name() -> webhook.
action_type_name() -> http.
connector_type_name() -> http.
schema_module() -> emqx_bridge_http_schema.
connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) ->
BridgeV1Config1 = maps:remove(<<"connector">>, ActionConfig),
%% Move parameters to the top level
ParametersMap1 = maps:get(<<"parameters">>, BridgeV1Config1, #{}),
ParametersMap2 = maps:without([<<"path">>, <<"headers">>], ParametersMap1),
BridgeV1Config2 = maps:remove(<<"parameters">>, BridgeV1Config1),
BridgeV1Config3 = emqx_utils_maps:deep_merge(BridgeV1Config2, ParametersMap2),
BridgeV1Config4 = emqx_utils_maps:deep_merge(ConnectorConfig, BridgeV1Config3),
Url = maps:get(<<"url">>, ConnectorConfig),
Path = maps:get(<<"path">>, ParametersMap1, <<>>),
Headers1 = maps:get(<<"headers">>, ConnectorConfig, #{}),
Headers2 = maps:get(<<"headers">>, ParametersMap1, #{}),
Url1 =
case Path of
<<>> -> Url;
_ -> iolist_to_binary(emqx_bridge_http_connector:join_paths(Url, Path))
end,
BridgeV1Config4#{
<<"headers">> => maps:merge(Headers1, Headers2),
<<"url">> => Url1
}.
bridge_v1_config_to_connector_config(BridgeV1Conf) ->
%% To statisfy the emqx_bridge_api_SUITE:t_http_crud_apis/1
ok = validate_webhook_url(maps:get(<<"url">>, BridgeV1Conf, undefined)),
maps:without(?REMOVED_KEYS ++ ?ACTION_KEYS ++ ?PARAMETER_KEYS, BridgeV1Conf).
bridge_v1_config_to_action_config(BridgeV1Conf, ConnectorName) ->
Parameters = maps:with(?PARAMETER_KEYS, BridgeV1Conf),
Parameters1 = Parameters#{<<"path">> => <<>>, <<"headers">> => #{}},
CommonKeys = [<<"enable">>, <<"description">>],
ActionConfig = maps:with(?ACTION_KEYS ++ CommonKeys, BridgeV1Conf),
ActionConfig#{<<"parameters">> => Parameters1, <<"connector">> => ConnectorName}.
%%--------------------------------------------------------------------
%% helpers
validate_webhook_url(undefined) ->
throw(#{
kind => validation_error,
reason => required_field,
required_field => <<"url">>
});
validate_webhook_url(Url) ->
{BaseUrl, _Path} = emqx_connector_resource:parse_url(Url),
case emqx_http_lib:uri_parse(BaseUrl) of
{ok, _} ->
ok;
{error, Reason} ->
throw(#{
kind => validation_error,
reason => invalid_url,
url => Url,
error => emqx_utils:readable_error_msg(Reason)
})
end.

155
apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl
View File

@ -31,9 +31,14 @@
on_query/3,
on_query_async/4,
on_get_status/2,
reply_delegator/3
on_add_channel/4,
on_remove_channel/3,
on_get_channels/1,
on_get_channel_status/3
]).
-export([reply_delegator/3]).
-export([
roots/0,
fields/1,
@ -41,7 +46,7 @@
namespace/0
]).
%% for other webhook-like connectors.
%% for other http-like connectors.
-export([redact_request/1]).
-export([validate_method/1, join_paths/2]).
@ -251,6 +256,21 @@ start_pool(PoolName, PoolOpts) ->
Error
end.
on_add_channel(
_InstId,
OldState,
ActionId,
ActionConfig
) ->
InstalledActions = maps:get(installed_actions, OldState, #{}),
{ok, ActionState} = do_create_http_action(ActionConfig),
NewInstalledActions = maps:put(ActionId, ActionState, InstalledActions),
NewState = maps:put(installed_actions, NewInstalledActions, OldState),
{ok, NewState}.
do_create_http_action(_ActionConfig = #{parameters := Params}) ->
{ok, preprocess_request(Params)}.
on_stop(InstId, _State) ->
?SLOG(info, #{
msg => "stopping_http_connector",
@ -260,6 +280,16 @@ on_stop(InstId, _State) ->
?tp(emqx_connector_http_stopped, #{instance_id => InstId}),
Res.
on_remove_channel(
_InstId,
OldState = #{installed_actions := InstalledActions},
ActionId
) ->
NewInstalledActions = maps:remove(ActionId, InstalledActions),
NewState = maps:put(installed_actions, NewInstalledActions, OldState),
{ok, NewState}.
%% BridgeV1 entrypoint
on_query(InstId, {send_message, Msg}, State) ->
case maps:get(request, State, undefined) of
undefined ->
@ -282,6 +312,36 @@ on_query(InstId, {send_message, Msg}, State) ->
State
)
end;
%% BridgeV2 entrypoint
on_query(
InstId,
{ActionId, Msg},
State = #{installed_actions := InstalledActions}
) when is_binary(ActionId) ->
case {maps:get(request, State, undefined), maps:get(ActionId, InstalledActions, undefined)} of
{undefined, _} ->
?SLOG(error, #{msg => "arg_request_not_found", connector => InstId}),
{error, arg_request_not_found};
{_, undefined} ->
?SLOG(error, #{msg => "action_not_found", connector => InstId, action_id => ActionId}),
{error, action_not_found};
{Request, ActionState} ->
#{
method := Method,
path := Path,
body := Body,
headers := Headers,
request_timeout := Timeout
} = process_request_and_action(Request, ActionState, Msg),
%% bridge buffer worker has retry, do not let ehttpc retry
Retry = 2,
ClientId = maps:get(clientid, Msg, undefined),
on_query(
InstId,
{ClientId, Method, {Path, Headers, Body}, Timeout, Retry},
State
)
end;
on_query(InstId, {Method, Request}, State) ->
%% TODO: Get retry from State
on_query(InstId, {undefined, Method, Request, 5000, _Retry = 2}, State);
@ -343,6 +403,7 @@ on_query(
Result
end.
%% BridgeV1 entrypoint
on_query_async(InstId, {send_message, Msg}, ReplyFunAndArgs, State) ->
case maps:get(request, State, undefined) of
undefined ->
@ -364,6 +425,36 @@ on_query_async(InstId, {send_message, Msg}, ReplyFunAndArgs, State) ->
State
)
end;
%% BridgeV2 entrypoint
on_query_async(
InstId,
{ActionId, Msg},
ReplyFunAndArgs,
State = #{installed_actions := InstalledActions}
) when is_binary(ActionId) ->
case {maps:get(request, State, undefined), maps:get(ActionId, InstalledActions, undefined)} of
{undefined, _} ->
?SLOG(error, #{msg => "arg_request_not_found", connector => InstId}),
{error, arg_request_not_found};
{_, undefined} ->
?SLOG(error, #{msg => "action_not_found", connector => InstId, action_id => ActionId}),
{error, action_not_found};
{Request, ActionState} ->
#{
method := Method,
path := Path,
body := Body,
headers := Headers,
request_timeout := Timeout
} = process_request_and_action(Request, ActionState, Msg),
ClientId = maps:get(clientid, Msg, undefined),
on_query_async(
InstId,
{ClientId, Method, {Path, Headers, Body}, Timeout},
ReplyFunAndArgs,
State
)
end;
on_query_async(
InstId,
{KeyOrNum, Method, Request, Timeout},
@ -411,6 +502,9 @@ resolve_pool_worker(#{pool_name := PoolName} = State, Key) ->
ehttpc_pool:pick_worker(PoolName, Key)
end.
on_get_channels(ResId) ->
emqx_bridge_v2:get_channels_for_connector(ResId).
on_get_status(_InstId, #{pool_name := PoolName, connect_timeout := Timeout} = State) ->
case do_get_status(PoolName, Timeout) of
ok ->
@ -456,6 +550,14 @@ do_get_status(PoolName, Timeout) ->
{error, timeout}
end.
on_get_channel_status(
InstId,
_ChannelId,
State
) ->
%% XXX: Reuse the connector status
on_get_status(InstId, State).
%%--------------------------------------------------------------------
%% Internal functions
%%--------------------------------------------------------------------
@ -466,10 +568,10 @@ preprocess_request(Req) when map_size(Req) == 0 ->
preprocess_request(
#{
method := Method,
path := Path,
headers := Headers
path := Path
} = Req
) ->
Headers = maps:get(headers, Req, []),
#{
method => parse_template(to_bin(Method)),
path => parse_template(Path),
@ -529,6 +631,49 @@ maybe_parse_template(Key, Conf) ->
parse_template(String) ->
emqx_template:parse(String).
process_request_and_action(Request, ActionState, Msg) ->
MethodTemplate = maps:get(method, ActionState),
Method = make_method(render_template_string(MethodTemplate, Msg)),
BodyTemplate = maps:get(body, ActionState),
Body = render_request_body(BodyTemplate, Msg),
PathPrefix = unicode:characters_to_list(render_template(maps:get(path, Request), Msg)),
PathSuffix = unicode:characters_to_list(render_template(maps:get(path, ActionState), Msg)),
Path =
case PathSuffix of
"" -> PathPrefix;
_ -> join_paths(PathPrefix, PathSuffix)
end,
HeadersTemplate1 = maps:get(headers, Request),
HeadersTemplate2 = maps:get(headers, ActionState),
Headers = merge_proplist(
render_headers(HeadersTemplate1, Msg),
render_headers(HeadersTemplate2, Msg)
),
#{
method => Method,
path => Path,
body => Body,
headers => Headers,
request_timeout => maps:get(request_timeout, ActionState)
}.
merge_proplist(Proplist1, Proplist2) ->
lists:foldl(
fun({K, V}, Acc) ->
case lists:keyfind(K, 1, Acc) of
false ->
[{K, V} | Acc];
{K, _} = {K, V1} ->
[{K, V1} | Acc]
end
end,
Proplist2,
Proplist1
).
process_request(
#{
method := MethodTemplate,
@ -691,7 +836,7 @@ maybe_retry({error, Reason}, Context, ReplyFunAndArgs) ->
true -> Context;
false -> Context#{attempt := Attempt + 1}
end,
?tp(webhook_will_retry_async, #{}),
?tp(http_will_retry_async, #{}),
Worker = resolve_pool_worker(State, KeyOrNum),
ok = ehttpc:request_async(
Worker,

419
apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl
View File

@ -18,69 +18,162 @@
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-import(hoconsc, [mk/2, enum/1, ref/2]).
-import(hoconsc, [mk/2, enum/1, ref/1, ref/2]).
-export([roots/0, fields/1, namespace/0, desc/1]).
-export([
bridge_v2_examples/1,
%%conn_bridge_examples/1,
connector_examples/1
]).
%%======================================================================================
%% Hocon Schema Definitions
namespace() -> "bridge_webhook".
namespace() -> "bridge_http".
roots() -> [].
fields("config") ->
basic_config() ++ request_config();
%%--------------------------------------------------------------------
%% v1 bridges http api
%% see: emqx_bridge_schema:get_response/0, put_request/0, post_request/0
fields("post") ->
[
type_field(),
old_type_field(),
name_field()
] ++ fields("config");
fields("put") ->
fields("config");
fields("get") ->
emqx_bridge_schema:status_fields() ++ fields("post");
fields("creation_opts") ->
%%--- v1 bridges config file
%% see: emqx_bridge_schema:fields(bridges)
fields("config") ->
basic_config() ++ request_config();
%%--------------------------------------------------------------------
%% v2: configuration
fields(action) ->
{http,
mk(
hoconsc:map(name, ref(?MODULE, "http_action")),
#{
aliases => [webhook],
desc => <<"HTTP Action Config">>,
required => false
}
)};
fields("http_action") ->
[
{enable, mk(boolean(), #{desc => ?DESC("config_enable_bridge"), default => true})},
{connector,
mk(binary(), #{
desc => ?DESC(emqx_connector_schema, "connector_field"), required => true
})},
{description, emqx_schema:description_schema()},
%% Note: there's an implicit convention in `emqx_bridge' that,
%% for egress bridges with this config, the published messages
%% will be forwarded to such bridges.
{local_topic,
mk(
binary(),
#{
required => false,
desc => ?DESC("config_local_topic"),
importance => ?IMPORTANCE_HIDDEN
}
)},
%% Since e5.3.2, we split the http bridge to two parts: a) connector. b) actions.
%% some fields are moved to connector, some fields are moved to actions and composed into the
%% `parameters` field.
{parameters,
mk(ref("parameters_opts"), #{
required => true,
desc => ?DESC("config_parameters_opts")
})}
] ++ http_resource_opts();
fields("parameters_opts") ->
[
{path,
mk(
binary(),
#{
desc => ?DESC("config_path"),
required => false
}
)},
method_field(),
headers_field(),
body_field(),
max_retries_field(),
request_timeout_field()
];
%% v2: api schema
%% The parameter equls to
%% `get_bridge_v2`, `post_bridge_v2`, `put_bridge_v2` from emqx_bridge_v2_schema:api_schema/1
%% `get_connector`, `post_connector`, `put_connector` from emqx_connector_schema:api_schema/1
fields("post_" ++ Type) ->
[type_field(), name_field() | fields("config_" ++ Type)];
fields("put_" ++ Type) ->
fields("config_" ++ Type);
fields("get_" ++ Type) ->
emqx_bridge_schema:status_fields() ++ fields("post_" ++ Type);
fields("config_bridge_v2") ->
fields("http_action");
fields("config_connector") ->
[
{enable,
mk(
boolean(),
#{
desc => <<"Enable or disable this connector">>,
default => true
}
)},
{description, emqx_schema:description_schema()}
] ++ connector_url_headers() ++ connector_opts();
%%--------------------------------------------------------------------
%% v1/v2
fields("resource_opts") ->
UnsupportedOpts = [enable_batch, batch_size, batch_time],
lists:filter(
fun({K, _V}) ->
not lists:member(K, unsupported_opts())
end,
fun({K, _V}) -> not lists:member(K, UnsupportedOpts) end,
emqx_resource_schema:fields("creation_opts")
).
desc("config") ->
?DESC("desc_config");
desc("creation_opts") ->
desc("resource_opts") ->
?DESC(emqx_resource_schema, "creation_opts");
desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
["Configuration for WebHook using `", string:to_upper(Method), "` method."];
desc("config_connector") ->
?DESC("desc_config");
desc("http_action") ->
?DESC("desc_config");
desc("parameters_opts") ->
?DESC("config_parameters_opts");
desc(_) ->
undefined.
%%--------------------------------------------------------------------
%% helpers for v1 only
basic_config() ->
[
{enable,
mk(
boolean(),
#{
desc => ?DESC("config_enable"),
desc => ?DESC("config_enable_bridge"),
default => true
}
)}
] ++ webhook_creation_opts() ++
proplists:delete(
max_retries, emqx_bridge_http_connector:fields(config)
).
)},
{description, emqx_schema:description_schema()}
] ++ http_resource_opts() ++ connector_opts().
request_config() ->
[
{url,
mk(
binary(),
#{
required => true,
desc => ?DESC("config_url")
}
)},
url_field(),
{direction,
mk(
egress,
@ -98,81 +191,37 @@ request_config() ->
required => false
}
)},
{method,
mk(
method(),
#{
default => post,
desc => ?DESC("config_method")
}
)},
{headers,
mk(
map(),
#{
default => #{
<<"accept">> => <<"application/json">>,
<<"cache-control">> => <<"no-cache">>,
<<"connection">> => <<"keep-alive">>,
<<"content-type">> => <<"application/json">>,
<<"keep-alive">> => <<"timeout=5">>
},
desc => ?DESC("config_headers")
}
)},
{body,
mk(
binary(),
#{
default => undefined,
desc => ?DESC("config_body")
}
)},
{max_retries,
mk(
non_neg_integer(),
#{
default => 2,
desc => ?DESC("config_max_retries")
}
)},
{request_timeout,
mk(
emqx_schema:duration_ms(),
#{
default => <<"15s">>,
deprecated => {since, "v5.0.26"},
desc => ?DESC("config_request_timeout")
}
)}
method_field(),
headers_field(),
body_field(),
max_retries_field(),
request_timeout_field()
].
webhook_creation_opts() ->
[
{resource_opts,
mk(
ref(?MODULE, "creation_opts"),
#{
required => false,
default => #{},
desc => ?DESC(emqx_resource_schema, <<"resource_opts">>)
}
)}
].
%%--------------------------------------------------------------------
%% helpers for v2 only
unsupported_opts() ->
[
enable_batch,
batch_size,
batch_time
].
connector_url_headers() ->
[url_field(), headers_field()].
%%======================================================================================
%%--------------------------------------------------------------------
%% common funcs
%% `webhook` is kept for backward compatibility.
old_type_field() ->
{type,
mk(
enum([webhook, http]),
#{
required => true,
desc => ?DESC("desc_type")
}
)}.
type_field() ->
{type,
mk(
webhook,
http,
#{
required => true,
desc => ?DESC("desc_type")
@ -189,5 +238,189 @@ name_field() ->
}
)}.
method() ->
enum([post, put, get, delete]).
url_field() ->
{url,
mk(
binary(),
#{
required => true,
desc => ?DESC("config_url")
}
)}.
headers_field() ->
{headers,
mk(
map(),
#{
default => #{
<<"accept">> => <<"application/json">>,
<<"cache-control">> => <<"no-cache">>,
<<"connection">> => <<"keep-alive">>,
<<"content-type">> => <<"application/json">>,
<<"keep-alive">> => <<"timeout=5">>
},
desc => ?DESC("config_headers")
}
)}.
method_field() ->
{method,
mk(
enum([post, put, get, delete]),
#{
default => post,
desc => ?DESC("config_method")
}
)}.
body_field() ->
{body,
mk(
binary(),
#{
default => undefined,
desc => ?DESC("config_body")
}
)}.
max_retries_field() ->
{max_retries,
mk(
non_neg_integer(),
#{
default => 2,
desc => ?DESC("config_max_retries")
}
)}.
request_timeout_field() ->
{request_timeout,
mk(
emqx_schema:duration_ms(),
#{
default => <<"15s">>,
deprecated => {since, "v5.0.26"},
desc => ?DESC("config_request_timeout")
}
)}.
http_resource_opts() ->
[
{resource_opts,
mk(
ref(?MODULE, "resource_opts"),
#{
required => false,
default => #{},
desc => ?DESC(emqx_resource_schema, <<"resource_opts">>)
}
)}
].
connector_opts() ->
mark_request_field_deperecated(
proplists:delete(max_retries, emqx_bridge_http_connector:fields(config))
).
mark_request_field_deperecated(Fields) ->
lists:map(
fun({K, V}) ->
case K of
request ->
{K, V#{
%% Note: if we want to deprecate a reference type, we have to change
%% it to a direct type first.
type => typerefl:map(),
deprecated => {since, "5.3.2"},
desc => <<"This field is never used, so we deprecated it since 5.3.2.">>
}};
_ ->
{K, V}
end
end,
Fields
).
%%--------------------------------------------------------------------
%% Examples
bridge_v2_examples(Method) ->
[
#{
<<"http">> => #{
summary => <<"HTTP Action">>,
value => values({Method, bridge_v2})
}
}
].
connector_examples(Method) ->
[
#{
<<"http">> => #{
summary => <<"HTTP Connector">>,
value => values({Method, connector})
}
}
].
values({get, Type}) ->
maps:merge(
#{
status => <<"connected">>,
node_status => [
#{
node => <<"emqx@localhost">>,
status => <<"connected">>
}
]
},
values({post, Type})
);
values({post, bridge_v2}) ->
maps:merge(
#{
name => <<"my_http_action">>,
type => <<"http">>
},
values({put, bridge_v2})
);
values({post, connector}) ->
maps:merge(
#{
name => <<"my_http_connector">>,
type => <<"http">>
},
values({put, connector})
);
values({put, bridge_v2}) ->
values(bridge_v2);
values({put, connector}) ->
values(connector);
values(bridge_v2) ->
#{
enable => true,
connector => <<"my_http_connector">>,
parameters => #{
path => <<"/room/${room_no}">>,
method => <<"post">>,
headers => #{},
body => <<"${.}">>
},
resource_opts => #{
worker_pool_size => 16,
health_check_interval => <<"15s">>,
query_mode => <<"async">>
}
};
values(connector) ->
#{
enable => true,
url => <<"http://localhost:8080/api/v1">>,
headers => #{<<"content-type">> => <<"application/json">>},
connect_timeout => <<"15s">>,
pool_type => <<"hash">>,
pool_size => 1,
enable_pipelining => 100
}.

64
apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl
View File

@ -39,18 +39,33 @@ all() ->
groups() ->
[].
init_per_suite(_Config) ->
emqx_common_test_helpers:render_and_load_app_config(emqx_conf),
ok = emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_bridge, emqx_rule_engine]),
ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
{ok, _} = application:ensure_all_started(emqx_connector),
[].
init_per_suite(Config0) ->
Config =
case os:getenv("DEBUG_CASE") of
[_ | _] = DebugCase ->
CaseName = list_to_atom(DebugCase),
[{debug_case, CaseName} | Config0];
_ ->
Config0
end,
Apps = emqx_cth_suite:start(
[
emqx,
emqx_conf,
emqx_connector,
emqx_bridge_http,
emqx_bridge,
emqx_rule_engine
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
emqx_mgmt_api_test_util:init_suite(),
[{apps, Apps} | Config].
end_per_suite(_Config) ->
ok = emqx_mgmt_api_test_util:end_suite([emqx_rule_engine, emqx_bridge, emqx_conf]),
ok = emqx_connector_test_helpers:stop_apps([emqx_resource]),
_ = application:stop(emqx_connector),
_ = application:stop(emqx_bridge),
end_per_suite(Config) ->
Apps = ?config(apps, Config),
emqx_mgmt_api_test_util:end_suite(),
ok = emqx_cth_suite:stop(Apps),
ok.
suite() ->
@ -115,7 +130,8 @@ end_per_testcase(TestCase, _Config) when
->
ok = emqx_bridge_http_connector_test_server:stop(),
persistent_term:erase({?MODULE, times_called}),
emqx_bridge_testlib:delete_all_bridges(),
emqx_bridge_v2_testlib:delete_all_bridges(),
emqx_bridge_v2_testlib:delete_all_connectors(),
emqx_common_test_helpers:call_janitor(),
ok;
end_per_testcase(_TestCase, Config) ->
@ -123,7 +139,8 @@ end_per_testcase(_TestCase, Config) ->
undefined -> ok;
Server -> stop_http_server(Server)
end,
emqx_bridge_testlib:delete_all_bridges(),
emqx_bridge_v2_testlib:delete_all_bridges(),
emqx_bridge_v2_testlib:delete_all_connectors(),
emqx_common_test_helpers:call_janitor(),
ok.
@ -420,7 +437,7 @@ t_send_async_connection_timeout(Config) ->
),
NumberOfMessagesToSend = 10,
[
emqx_bridge:send_message(BridgeID, #{<<"id">> => Id})
do_send_message(#{<<"id">> => Id})
|| Id <- lists:seq(1, NumberOfMessagesToSend)
],
%% Make sure server receives all messages
@ -431,7 +448,7 @@ t_send_async_connection_timeout(Config) ->
t_async_free_retries(Config) ->
#{port := Port} = ?config(http_server, Config),
BridgeID = make_bridge(#{
_BridgeID = make_bridge(#{
port => Port,
pool_size => 1,
query_mode => "sync",
@ -445,7 +462,7 @@ t_async_free_retries(Config) ->
Fn = fun(Get, Error) ->
?assertMatch(
{ok, 200, _, _},
emqx_bridge:send_message(BridgeID, #{<<"hello">> => <<"world">>}),
do_send_message(#{<<"hello">> => <<"world">>}),
#{error => Error}
),
?assertEqual(ExpectedAttempts, Get(), #{error => Error})
@ -456,7 +473,7 @@ t_async_free_retries(Config) ->
t_async_common_retries(Config) ->
#{port := Port} = ?config(http_server, Config),
BridgeID = make_bridge(#{
_BridgeID = make_bridge(#{
port => Port,
pool_size => 1,
query_mode => "sync",
@ -471,7 +488,7 @@ t_async_common_retries(Config) ->
FnSucceed = fun(Get, Error) ->
?assertMatch(
{ok, 200, _, _},
emqx_bridge:send_message(BridgeID, #{<<"hello">> => <<"world">>}),
do_send_message(#{<<"hello">> => <<"world">>}),
#{error => Error, attempts => Get()}
),
?assertEqual(ExpectedAttempts, Get(), #{error => Error})
@ -479,7 +496,7 @@ t_async_common_retries(Config) ->
FnFail = fun(Get, Error) ->
?assertMatch(
Error,
emqx_bridge:send_message(BridgeID, #{<<"hello">> => <<"world">>}),
do_send_message(#{<<"hello">> => <<"world">>}),
#{error => Error, attempts => Get()}
),
?assertEqual(ExpectedAttempts, Get(), #{error => Error})
@ -559,7 +576,7 @@ t_path_not_found(Config) ->
ok
end,
fun(Trace) ->
?assertEqual([], ?of_kind(webhook_will_retry_async, Trace)),
?assertEqual([], ?of_kind(http_will_retry_async, Trace)),
ok
end
),
@ -600,7 +617,7 @@ t_too_many_requests(Config) ->
ok
end,
fun(Trace) ->
?assertMatch([_ | _], ?of_kind(webhook_will_retry_async, Trace)),
?assertMatch([_ | _], ?of_kind(http_will_retry_async, Trace)),
ok
end
),
@ -711,6 +728,11 @@ t_bridge_probes_header_atoms(Config) ->
ok.
%% helpers
do_send_message(Message) ->
Type = emqx_bridge_v2:bridge_v1_type_to_bridge_v2_type(?BRIDGE_TYPE),
emqx_bridge_v2:send_message(Type, ?BRIDGE_NAME, Message, #{}).
do_t_async_retries(TestCase, TestContext, Error, Fn) ->
#{error_attempts := ErrorAttempts} = TestContext,
PTKey = {?MODULE, TestCase, attempts},

4
apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl
View File

@ -175,7 +175,7 @@ check_atom_key(Conf) when is_map(Conf) ->
%% erlfmt-ignore
webhook_config_hocon() ->
"""
"
bridges.webhook.a {
body = \"${.}\"
connect_timeout = 15s
@ -209,4 +209,4 @@ bridges.webhook.a {
}
url = \"http://some.host:4000/api/echo\"
}
""".
".

140
apps/emqx_bridge_http/test/emqx_bridge_http_v2_SUITE.erl Normal file
View File

@ -0,0 +1,140 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_bridge_http_v2_SUITE).
-compile(nowarn_export_all).
-compile(export_all).
-import(emqx_mgmt_api_test_util, [request/3]).
-import(emqx_common_test_helpers, [on_exit/1]).
-import(emqx_bridge_http_SUITE, [start_http_server/1, stop_http_server/1]).
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("emqx/include/asserts.hrl").
-define(BRIDGE_TYPE, <<"http">>).
-define(BRIDGE_NAME, atom_to_binary(?MODULE)).
-define(CONNECTOR_NAME, atom_to_binary(?MODULE)).
all() ->
emqx_common_test_helpers:all(?MODULE).
groups() ->
[].
init_per_suite(Config0) ->
Config =
case os:getenv("DEBUG_CASE") of
[_ | _] = DebugCase ->
CaseName = list_to_atom(DebugCase),
[{debug_case, CaseName} | Config0];
_ ->
Config0
end,
Apps = emqx_cth_suite:start(
[
emqx,
emqx_conf,
emqx_connector,
emqx_bridge_http,
emqx_bridge,
emqx_rule_engine
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
emqx_mgmt_api_test_util:init_suite(),
[{apps, Apps} | Config].
end_per_suite(Config) ->
Apps = ?config(apps, Config),
emqx_mgmt_api_test_util:end_suite(),
ok = emqx_cth_suite:stop(Apps),
ok.
suite() ->
[{timetrap, {seconds, 60}}].
init_per_testcase(_TestCase, Config) ->
Server = start_http_server(#{response_delay_ms => 0}),
[{http_server, Server} | Config].
end_per_testcase(_TestCase, Config) ->
case ?config(http_server, Config) of
undefined -> ok;
Server -> stop_http_server(Server)
end,
emqx_bridge_v2_testlib:delete_all_bridges(),
emqx_bridge_v2_testlib:delete_all_connectors(),
emqx_common_test_helpers:call_janitor(),
ok.
%%--------------------------------------------------------------------
%% tests
%%--------------------------------------------------------------------
t_compose_connector_url_and_action_path(Config) ->
Path = <<"/foo/bar">>,
ConnectorCfg = make_connector_config(Config),
ActionCfg = make_action_config([{path, Path} | Config]),
CreateConfig = [
{bridge_type, ?BRIDGE_TYPE},
{bridge_name, ?BRIDGE_NAME},
{bridge_config, ActionCfg},
{connector_type, ?BRIDGE_TYPE},
{connector_name, ?CONNECTOR_NAME},
{connector_config, ConnectorCfg}
],
{ok, _} = emqx_bridge_v2_testlib:create_bridge(CreateConfig),
%% assert: the url returned v1 api is composed by the url of the connector and the
%% path of the action
#{port := Port} = ?config(http_server, Config),
ExpectedUrl = iolist_to_binary(io_lib:format("http://localhost:~p/foo/bar", [Port])),
{ok, {_, _, [Bridge]}} = emqx_bridge_testlib:list_bridges_api(),
?assertMatch(
#{<<"url">> := ExpectedUrl},
Bridge
),
ok.
%%--------------------------------------------------------------------
%% helpers
%%--------------------------------------------------------------------
make_connector_config(Config) ->
#{port := Port} = ?config(http_server, Config),
#{
<<"enable">> => true,
<<"url">> => iolist_to_binary(io_lib:format("http://localhost:~p", [Port])),
<<"headers">> => #{},
<<"pool_type">> => <<"hash">>,
<<"pool_size">> => 1
}.
make_action_config(Config) ->
Path = ?config(path, Config),
#{
<<"enable">> => true,
<<"connector">> => ?CONNECTOR_NAME,
<<"parameters">> => #{
<<"path">> => Path,
<<"method">> => <<"post">>,
<<"headers">> => #{},
<<"body">> => <<"${.}">>
}
}.

2
apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_influxdb, [
{description, "EMQX Enterprise InfluxDB Bridge"},
{vsn, "0.1.6"},
{vsn, "0.1.7"},
{registered, []},
{applications, [
kernel,

2
apps/emqx_bridge_kafka/rebar.config
View File

@ -2,7 +2,7 @@
{erl_opts, [debug_info]}.
{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.8.0"}}}
, {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.3"}}}
, {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0"}}}
, {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.1"}}}
, {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.8"}}}
, {snappyer, "1.2.9"}
, {emqx_connector, {path, "../../apps/emqx_connector"}}

2
apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_bridge_kafka, [
{description, "EMQX Enterprise Kafka Bridge"},
{vsn, "0.1.12"},
{vsn, "0.1.13"},
{registered, [emqx_bridge_kafka_consumer_sup]},
{applications, [
kernel,

19
apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl
View File

@ -269,7 +269,11 @@ fields(Field) when
Field == "put_connector";
Field == "post_connector"
->
emqx_connector_schema:api_fields(Field, ?CONNECTOR_TYPE, kafka_connector_config_fields());
emqx_connector_schema:api_fields(
Field,
?CONNECTOR_TYPE,
kafka_connector_config_fields()
);
fields("post_" ++ Type) ->
[type_field(Type), name_field() | fields("config_" ++ Type)];
fields("put_" ++ Type) ->
@ -508,8 +512,7 @@ fields(consumer_opts) ->
{value_encoding_mode,
mk(enum([none, base64]), #{
default => none, desc => ?DESC(consumer_value_encoding_mode)
})},
{resource_opts, mk(ref(resource_opts), #{default => #{}})}
})}
];
fields(consumer_topic_mapping) ->
[
@ -623,7 +626,7 @@ kafka_connector_config_fields() ->
})},
{socket_opts, mk(ref(socket_opts), #{required => false, desc => ?DESC(socket_opts)})},
{ssl, mk(ref(ssl_client_opts), #{})}
].
] ++ [resource_opts()].
producer_opts(ActionOrBridgeV1) ->
[
@ -631,9 +634,11 @@ producer_opts(ActionOrBridgeV1) ->
%% for egress bridges with this config, the published messages
%% will be forwarded to such bridges.
{local_topic, mk(binary(), #{required => false, desc => ?DESC(mqtt_topic)})},
parameters_field(ActionOrBridgeV1),
{resource_opts, mk(ref(resource_opts), #{default => #{}, desc => ?DESC(resource_opts)})}
].
parameters_field(ActionOrBridgeV1)
] ++ [resource_opts() || ActionOrBridgeV1 =:= action].
resource_opts() ->
{resource_opts, mk(ref(resource_opts), #{default => #{}, desc => ?DESC(resource_opts)})}.
%% Since e5.3.1, we want to rename the field 'kafka' to 'parameters'
%% However we need to keep it backward compatible for generated schema json (version 0.1.0)

68
apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
View File

@ -81,11 +81,24 @@ on_start(InstId, Config) ->
ClientId = InstId,
emqx_resource:allocate_resource(InstId, ?kafka_client_id, ClientId),
ok = ensure_client(ClientId, Hosts, ClientConfig),
%% Check if this is a dry run
{ok, #{
client_id => ClientId,
installed_bridge_v2s => #{}
}}.
%% Note: we must return `{error, _}' here if the client cannot connect so that the
%% connector will immediately enter the `?status_disconnected' state, and then avoid
%% giving the impression that channels/actions may be added immediately and start
%% buffering, which won't happen if it's `?status_connecting'. That would lead to
%% data loss, since Kafka Producer uses wolff's internal buffering, which is started
%% only when its producers start.
case check_client_connectivity(ClientId) of
ok ->
{ok, #{
client_id => ClientId,
installed_bridge_v2s => #{}
}};
{error, {find_client, Reason}} ->
%% Race condition? Crash? We just checked it with `ensure_client'...
{error, Reason};
{error, {connectivity, Reason}} ->
{error, Reason}
end.
on_add_channel(
InstId,
@ -478,14 +491,18 @@ on_get_status(
_InstId,
#{client_id := ClientId} = State
) ->
case wolff_client_sup:find_client(ClientId) of
{ok, Pid} ->
case wolff_client:check_connectivity(Pid) of
ok -> ?status_connected;
{error, Error} -> {?status_connecting, State, Error}
end;
{error, _Reason} ->
?status_connecting
%% Note: we must avoid returning `?status_disconnected' here if the connector ever was
%% connected. If the connector ever connected, wolff producers might have been
%% sucessfully started, and returning `?status_disconnected' will make resource
%% manager try to restart the producers / connector, thus potentially dropping data
%% held in wolff producer's replayq.
case check_client_connectivity(ClientId) of
ok ->
?status_connected;
{error, {find_client, _Error}} ->
?status_connecting;
{error, {connectivity, Error}} ->
{?status_connecting, State, Error}
end.
on_get_channel_status(
@ -496,13 +513,19 @@ on_get_channel_status(
installed_bridge_v2s := Channels
} = _State
) ->
%% Note: we must avoid returning `?status_disconnected' here. Returning
%% `?status_disconnected' will make resource manager try to restart the producers /
%% connector, thus potentially dropping data held in wolff producer's replayq. The
%% only exception is if the topic does not exist ("unhealthy target").
#{kafka_topic := KafkaTopic} = maps:get(ChannelId, Channels),
try
ok = check_topic_and_leader_connections(ClientId, KafkaTopic),
?status_connected
catch
throw:#{reason := restarting} ->
?status_connecting
throw:{unhealthy_target, Msg} ->
throw({unhealthy_target, Msg});
K:E ->
{?status_connecting, {K, E}}
end.
check_topic_and_leader_connections(ClientId, KafkaTopic) ->
@ -524,6 +547,21 @@ check_topic_and_leader_connections(ClientId, KafkaTopic) ->
})
end.
-spec check_client_connectivity(wolff:client_id()) ->
ok | {error, {connectivity | find_client, term()}}.
check_client_connectivity(ClientId) ->
case wolff_client_sup:find_client(ClientId) of
{ok, Pid} ->
case wolff_client:check_connectivity(Pid) of
ok ->
ok;
{error, Error} ->
{error, {connectivity, Error}}
end;
{error, Reason} ->
{error, {find_client, Reason}}
end.
check_if_healthy_leaders(ClientId, ClientPid, KafkaTopic) when is_pid(ClientPid) ->
Leaders =
case wolff_client:get_leader_connections(ClientPid, KafkaTopic) of

267
apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl
View File

@ -22,6 +22,7 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("brod/include/brod.hrl").
-include_lib("emqx_resource/include/emqx_resource.hrl").
-import(emqx_common_test_helpers, [on_exit/1]).
@ -35,6 +36,14 @@ all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
KafkaHost = os:getenv("KAFKA_PLAIN_HOST", "toxiproxy.emqx.net"),
KafkaPort = list_to_integer(os:getenv("KAFKA_PLAIN_PORT", "9292")),
ProxyName = "kafka_plain",
DirectKafkaHost = os:getenv("KAFKA_DIRECT_PLAIN_HOST", "kafka-1.emqx.net"),
DirectKafkaPort = list_to_integer(os:getenv("KAFKA_DIRECT_PLAIN_PORT", "9092")),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
Apps = emqx_cth_suite:start(
[
emqx,
@ -50,17 +59,34 @@ init_per_suite(Config) ->
),
{ok, _} = emqx_common_test_http:create_default_app(),
emqx_bridge_kafka_impl_producer_SUITE:wait_until_kafka_is_up(),
[{apps, Apps} | Config].
[
{apps, Apps},
{proxy_host, ProxyHost},
{proxy_port, ProxyPort},
{proxy_name, ProxyName},
{kafka_host, KafkaHost},
{kafka_port, KafkaPort},
{direct_kafka_host, DirectKafkaHost},
{direct_kafka_port, DirectKafkaPort}
| Config
].
end_per_suite(Config) ->
Apps = ?config(apps, Config),
ProxyHost = ?config(proxy_host, Config),
ProxyPort = ?config(proxy_port, Config),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
emqx_cth_suite:stop(Apps),
ok.
init_per_testcase(_TestCase, Config) ->
Config.
end_per_testcase(_TestCase, _Config) ->
end_per_testcase(_TestCase, Config) ->
ProxyHost = ?config(proxy_host, Config),
ProxyPort = ?config(proxy_port, Config),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
emqx_bridge_v2_testlib:delete_all_bridges_and_connectors(),
emqx_common_test_helpers:call_janitor(60_000),
ok.
@ -69,6 +95,13 @@ end_per_testcase(_TestCase, _Config) ->
%%-------------------------------------------------------------------------------------
check_send_message_with_bridge(BridgeName) ->
#{offset := Offset, payload := Payload} = send_message(BridgeName),
%% ######################################
%% Check if message is sent to Kafka
%% ######################################
check_kafka_message_payload(Offset, Payload).
send_message(ActionName) ->
%% ######################################
%% Create Kafka message
%% ######################################
@ -84,11 +117,8 @@ check_send_message_with_bridge(BridgeName) ->
%% ######################################
%% Send message
%% ######################################
emqx_bridge_v2:send_message(?TYPE, BridgeName, Msg, #{}),
%% ######################################
%% Check if message is sent to Kafka
%% ######################################
check_kafka_message_payload(Offset, Payload).
emqx_bridge_v2:send_message(?TYPE, ActionName, Msg, #{}),
#{offset => Offset, payload => Payload}.
resolve_kafka_offset() ->
KafkaTopic = emqx_bridge_kafka_impl_producer_SUITE:test_topic_one_partition(),
@ -106,6 +136,14 @@ check_kafka_message_payload(Offset, ExpectedPayload) ->
{ok, {_, [KafkaMsg0]}} = brod:fetch(Hosts, KafkaTopic, Partition, Offset),
?assertMatch(#kafka_message{value = ExpectedPayload}, KafkaMsg0).
action_config(ConnectorName) ->
action_config(ConnectorName, _Overrides = #{}).
action_config(ConnectorName, Overrides) ->
Cfg0 = bridge_v2_config(ConnectorName),
Cfg1 = emqx_utils_maps:rename(<<"kafka">>, <<"parameters">>, Cfg0),
emqx_utils_maps:deep_merge(Cfg1, Overrides).
bridge_v2_config(ConnectorName) ->
#{
<<"connector">> => ConnectorName,
@ -131,7 +169,9 @@ bridge_v2_config(ConnectorName) ->
<<"query_mode">> => <<"sync">>,
<<"required_acks">> => <<"all_isr">>,
<<"sync_query_timeout">> => <<"5s">>,
<<"topic">> => emqx_bridge_kafka_impl_producer_SUITE:test_topic_one_partition()
<<"topic">> => list_to_binary(
emqx_bridge_kafka_impl_producer_SUITE:test_topic_one_partition()
)
},
<<"local_topic">> => <<"kafka_t/#">>,
<<"resource_opts">> => #{
@ -140,32 +180,37 @@ bridge_v2_config(ConnectorName) ->
}.
connector_config() ->
#{
<<"authentication">> => <<"none">>,
<<"bootstrap_hosts">> => iolist_to_binary(kafka_hosts_string()),
<<"connect_timeout">> => <<"5s">>,
<<"enable">> => true,
<<"metadata_request_timeout">> => <<"5s">>,
<<"min_metadata_refresh_interval">> => <<"3s">>,
<<"socket_opts">> =>
#{
<<"recbuf">> => <<"1024KB">>,
<<"sndbuf">> => <<"1024KB">>,
<<"tcp_keepalive">> => <<"none">>
},
<<"ssl">> =>
#{
<<"ciphers">> => [],
<<"depth">> => 10,
<<"enable">> => false,
<<"hibernate_after">> => <<"5s">>,
<<"log_level">> => <<"notice">>,
<<"reuse_sessions">> => true,
<<"secure_renegotiate">> => true,
<<"verify">> => <<"verify_peer">>,
<<"versions">> => [<<"tlsv1.3">>, <<"tlsv1.2">>]
}
}.
connector_config(_Overrides = #{}).
connector_config(Overrides) ->
Defaults =
#{
<<"authentication">> => <<"none">>,
<<"bootstrap_hosts">> => iolist_to_binary(kafka_hosts_string()),
<<"connect_timeout">> => <<"5s">>,
<<"enable">> => true,
<<"metadata_request_timeout">> => <<"5s">>,
<<"min_metadata_refresh_interval">> => <<"3s">>,
<<"socket_opts">> =>
#{
<<"recbuf">> => <<"1024KB">>,
<<"sndbuf">> => <<"1024KB">>,
<<"tcp_keepalive">> => <<"none">>
},
<<"ssl">> =>
#{
<<"ciphers">> => [],
<<"depth">> => 10,
<<"enable">> => false,
<<"hibernate_after">> => <<"5s">>,
<<"log_level">> => <<"notice">>,
<<"reuse_sessions">> => true,
<<"secure_renegotiate">> => true,
<<"verify">> => <<"verify_peer">>,
<<"versions">> => [<<"tlsv1.3">>, <<"tlsv1.2">>]
}
},
emqx_utils_maps:deep_merge(Defaults, Overrides).
kafka_hosts_string() ->
KafkaHost = os:getenv("KAFKA_PLAIN_HOST", "kafka-1.emqx.net"),
@ -350,13 +395,13 @@ t_bad_url(_Config) ->
{ok, #{
resource_data :=
#{
status := connecting,
status := ?status_disconnected,
error := [#{reason := unresolvable_hostname}]
}
}},
emqx_connector:lookup(?TYPE, ConnectorName)
),
?assertMatch({ok, #{status := connecting}}, emqx_bridge_v2:lookup(?TYPE, ActionName)),
?assertMatch({ok, #{status := ?status_disconnected}}, emqx_bridge_v2:lookup(?TYPE, ActionName)),
ok.
t_parameters_key_api_spec(_Config) ->
@ -383,3 +428,153 @@ t_http_api_get(_Config) ->
emqx_bridge_testlib:list_bridges_api()
),
ok.
t_create_connector_while_connection_is_down(Config) ->
ProxyName = ?config(proxy_name, Config),
ProxyHost = ?config(proxy_host, Config),
ProxyPort = ?config(proxy_port, Config),
KafkaHost = ?config(kafka_host, Config),
KafkaPort = ?config(kafka_port, Config),
Host = iolist_to_binary([KafkaHost, ":", integer_to_binary(KafkaPort)]),
?check_trace(
begin
Type = ?TYPE,
ConnectorConfig = connector_config(#{
<<"bootstrap_hosts">> => Host,
<<"resource_opts">> =>
#{<<"health_check_interval">> => <<"500ms">>}
}),
ConnectorName = <<"c1">>,
ConnectorId = emqx_connector_resource:resource_id(Type, ConnectorName),
ConnectorParams = [
{connector_config, ConnectorConfig},
{connector_name, ConnectorName},
{connector_type, Type}
],
ActionName = ConnectorName,
ActionId = emqx_bridge_v2:id(?TYPE, ActionName, ConnectorName),
ActionConfig = action_config(
ConnectorName
),
ActionParams = [
{action_config, ActionConfig},
{action_name, ActionName},
{action_type, Type}
],
Disconnected = atom_to_binary(?status_disconnected),
%% Initially, the connection cannot be stablished. Messages are not buffered,
%% hence the status is `?status_disconnected'.
emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() ->
{ok, {{_, 201, _}, _, #{<<"status">> := Disconnected}}} =
emqx_bridge_v2_testlib:create_connector_api(ConnectorParams),
{ok, {{_, 201, _}, _, #{<<"status">> := Disconnected}}} =
emqx_bridge_v2_testlib:create_action_api(ActionParams),
#{offset := Offset1} = send_message(ActionName),
#{offset := Offset2} = send_message(ActionName),
#{offset := Offset3} = send_message(ActionName),
?assertEqual([Offset1], lists:usort([Offset1, Offset2, Offset3])),
?assertEqual(3, emqx_resource_metrics:matched_get(ActionId)),
?assertEqual(3, emqx_resource_metrics:failed_get(ActionId)),
?assertEqual(0, emqx_resource_metrics:queuing_get(ActionId)),
?assertEqual(0, emqx_resource_metrics:inflight_get(ActionId)),
?assertEqual(0, emqx_resource_metrics:dropped_get(ActionId)),
ok
end),
%% Let the connector and action recover
Connected = atom_to_binary(?status_connected),
?retry(
_Sleep0 = 1_100,
_Attempts0 = 10,
begin
_ = emqx_resource:health_check(ConnectorId),
_ = emqx_resource:health_check(ActionId),
?assertMatch(
{ok, #{
status := ?status_connected,
resource_data :=
#{
status := ?status_connected,
added_channels :=
#{
ActionId := #{
status := ?status_connected
}
}
}
}},
emqx_bridge_v2:lookup(Type, ActionName),
#{action_id => ActionId}
),
?assertMatch(
{ok, {{_, 200, _}, _, #{<<"status">> := Connected}}},
emqx_bridge_v2_testlib:get_action_api(ActionParams)
)
end
),
%% Now the connection drops again; this time, status should be
%% `?status_connecting' to avoid destroying wolff_producers and their replayq
%% buffers.
Connecting = atom_to_binary(?status_connecting),
emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() ->
?retry(
_Sleep0 = 1_100,
_Attempts0 = 10,
begin
_ = emqx_resource:health_check(ConnectorId),
_ = emqx_resource:health_check(ActionId),
?assertMatch(
{ok, #{
status := ?status_connecting,
resource_data :=
#{
status := ?status_connecting,
added_channels :=
#{
ActionId := #{
status := ?status_connecting
}
}
}
}},
emqx_bridge_v2:lookup(Type, ActionName),
#{action_id => ActionId}
),
?assertMatch(
{ok, {{_, 200, _}, _, #{<<"status">> := Connecting}}},
emqx_bridge_v2_testlib:get_action_api(ActionParams)
)
end
),
%% This should get enqueued by wolff producers.
spawn_link(fun() -> send_message(ActionName) end),
PreviousMatched = 3,
PreviousFailed = 3,
?retry(
_Sleep2 = 100,
_Attempts2 = 10,
?assertEqual(PreviousMatched + 1, emqx_resource_metrics:matched_get(ActionId))
),
?assertEqual(PreviousFailed, emqx_resource_metrics:failed_get(ActionId)),
?assertEqual(1, emqx_resource_metrics:queuing_get(ActionId)),
?assertEqual(0, emqx_resource_metrics:inflight_get(ActionId)),
?assertEqual(0, emqx_resource_metrics:dropped_get(ActionId)),
?assertEqual(0, emqx_resource_metrics:success_get(ActionId)),
ok
end),
?retry(
_Sleep2 = 600,
_Attempts2 = 20,
begin
_ = emqx_resource:health_check(ConnectorId),
_ = emqx_resource:health_check(ActionId),
?assertEqual(1, emqx_resource_metrics:success_get(ActionId), #{
metrics => emqx_bridge_v2:get_metrics(Type, ActionName)
}),
ok
end
),
ok
end,
[]
),
ok.

2
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_bridge_mqtt, [
{description, "EMQX MQTT Broker Bridge"},
{vsn, "0.1.5"},
{vsn, "0.1.6"},
{registered, []},
{applications, [
kernel,

2
apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_mysql, [
{description, "EMQX Enterprise MySQL Bridge"},
{vsn, "0.1.2"},
{vsn, "0.1.3"},
{registered, []},
{applications, [
kernel,

2
apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_pgsql, [
{description, "EMQX Enterprise PostgreSQL Bridge"},
{vsn, "0.1.4"},
{vsn, "0.1.5"},
{registered, []},
{applications, [
kernel,

4
apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_tests.erl
View File

@ -73,7 +73,7 @@ check_atom_key(Conf) when is_map(Conf) ->
%% erlfmt-ignore
pulsar_producer_hocon() ->
"""
"
bridges.pulsar_producer.my_producer {
enable = true
servers = \"localhost:6650\"
@ -90,4 +90,4 @@ bridges.pulsar_producer.my_producer {
server_name_indication = \"auto\"
}
}
""".
".

2
apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_rabbitmq, [
{description, "EMQX Enterprise RabbitMQ Bridge"},
{vsn, "0.1.6"},
{vsn, "0.1.7"},
{registered, []},
{applications, [
kernel,

4
apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl
View File

@ -12,7 +12,7 @@
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-export([roots/0, fields/1, desc/1, connector_examples/1]).
-export([namespace/0, roots/0, fields/1, desc/1, connector_examples/1]).
%% `emqx_resource' API
-export([
@ -44,6 +44,8 @@
%% -------------------------------------------------------------------------------------------------
%% api
namespace() -> "syskeeper_forwarder".
connector_examples(Method) ->
[
#{

2
apps/emqx_conf/src/emqx_cluster_rpc.erl
View File

@ -66,6 +66,7 @@
-boot_mnesia({mnesia, [boot]}).
-include_lib("emqx/include/logger.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include("emqx_conf.hrl").
-ifdef(TEST).
@ -384,6 +385,7 @@ catch_up(State) -> catch_up(State, false).
catch_up(#{node := Node, retry_interval := RetryMs, is_leaving := false} = State, SkipResult) ->
case transaction(fun ?MODULE:read_next_mfa/1, [Node]) of
{atomic, caught_up} ->
?tp(cluster_rpc_caught_up, #{}),
?TIMEOUT;
{atomic, {still_lagging, NextId, MFA}} ->
{Succeed, _} = apply_mfa(NextId, MFA, ?APPLY_KIND_REPLICATE),

2
apps/emqx_conf/src/emqx_conf.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_conf, [
{description, "EMQX configuration management"},
{vsn, "0.1.31"},
{vsn, "0.1.32"},
{registered, []},
{mod, {emqx_conf_app, []}},
{applications, [kernel, stdlib, emqx_ctl]},

2
apps/emqx_conf/src/emqx_conf.erl
View File

@ -306,7 +306,7 @@ gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S) ->
ShortName = short_name(FullName),
case is_missing_namespace(ShortName, to_bin(FullName), RootNames) of
true ->
io:format(standard_error, "WARN: no_namespace_for: ~s~n", [FullName]);
error({no_namespace, FullName, S});
false ->
ok
end,

4
apps/emqx_conf/test/emqx_conf_logger_SUITE.erl
View File

@ -24,7 +24,7 @@
%% erlfmt-ignore
-define(BASE_CONF,
"""
"
log {
console {
enable = true
@ -36,7 +36,7 @@
path = \"log/emqx.log\"
}
}
""").
").
all() ->
emqx_common_test_helpers:all(?MODULE).

28
apps/emqx_conf/test/emqx_conf_schema_tests.erl
View File

@ -20,7 +20,7 @@
%% erlfmt-ignore
-define(BASE_CONF,
"""
"
node {
name = \"emqx1@127.0.0.1\"
cookie = \"emqxsecretcookie\"
@ -34,7 +34,7 @@
static.seeds = ~p
core_nodes = ~p
}
""").
").
array_nodes_test() ->
ensure_acl_conf(),
@ -70,7 +70,7 @@ array_nodes_test() ->
%% erlfmt-ignore
-define(OUTDATED_LOG_CONF,
"""
"
log.console_handler {
burst_limit {
enable = true
@ -124,7 +124,7 @@ log.file_handlers {
time_offset = \"+01:00\"
}
}
"""
"
).
-define(FORMATTER(TimeOffset),
{emqx_logger_textfmt, #{
@ -196,7 +196,7 @@ validate_log(Conf) ->
%% erlfmt-ignore
-define(FILE_LOG_BASE_CONF,
"""
"
log.file.default {
enable = true
file = \"log/xx-emqx.log\"
@ -206,7 +206,7 @@ validate_log(Conf) ->
rotation_size = ~s
time_offset = \"+01:00\"
}
"""
"
).
file_log_infinity_rotation_size_test_() ->
@ -249,7 +249,7 @@ file_log_infinity_rotation_size_test_() ->
%% erlfmt-ignore
-define(KERNEL_LOG_CONF,
"""
"
log.console {
enable = true
formatter = text
@ -269,7 +269,7 @@ file_log_infinity_rotation_size_test_() ->
enable = true
file = \"log/my-emqx.log\"
}
"""
"
).
log_test() ->
@ -279,7 +279,7 @@ log_test() ->
log_rotation_count_limit_test() ->
ensure_acl_conf(),
Format =
"""
"
log.file {
enable = true
path = \"log/emqx.log\"
@ -288,7 +288,7 @@ log_rotation_count_limit_test() ->
rotation = {count = ~w}
rotation_size = \"1024MB\"
}
""",
",
BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]),
lists:foreach(fun({Conf, Count}) ->
Conf0 = <<BaseConf/binary, Conf/binary>>,
@ -320,7 +320,7 @@ log_rotation_count_limit_test() ->
%% erlfmt-ignore
-define(BASE_AUTHN_ARRAY,
"""
"
authentication = [
{backend = \"http\"
body {password = \"${password}\", username = \"${username}\"}
@ -335,7 +335,7 @@ log_rotation_count_limit_test() ->
url = \"~ts\"
}
]
"""
"
).
-define(ERROR(Error),
@ -396,13 +396,13 @@ authn_validations_test() ->
%% erlfmt-ignore
-define(LISTENERS,
"""
"
listeners.ssl.default.bind = 9999
listeners.wss.default.bind = 9998
listeners.wss.default.ssl_options.cacertfile = \"mytest/certs/cacert.pem\"
listeners.wss.new.bind = 9997
listeners.wss.new.websocket.mqtt_path = \"/my-mqtt\"
"""
"
).
listeners_test() ->

2
apps/emqx_connector/src/emqx_connector.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_connector, [
{description, "EMQX Data Integration Connectors"},
{vsn, "0.1.34"},
{vsn, "0.1.35"},
{registered, []},
{mod, {emqx_connector_app, []}},
{applications, [

14
apps/emqx_connector/src/emqx_connector_api.erl
View File

@ -137,7 +137,7 @@ param_path_id() ->
#{
in => path,
required => true,
example => <<"webhook:webhook_example">>,
example => <<"http:my_http_connector">>,
desc => ?DESC("desc_param_path_id")
}
)}.
@ -158,17 +158,7 @@ connector_info_array_example(Method) ->
lists:map(fun(#{value := Config}) -> Config end, maps:values(connector_info_examples(Method))).
connector_info_examples(Method) ->
maps:merge(
#{},
emqx_enterprise_connector_examples(Method)
).
-if(?EMQX_RELEASE_EDITION == ee).
emqx_enterprise_connector_examples(Method) ->
emqx_connector_ee_schema:examples(Method).
-else.
emqx_enterprise_connector_examples(_Method) -> #{}.
-endif.
emqx_connector_schema:examples(Method).
schema("/connectors") ->
#{

24
apps/emqx_connector/src/emqx_connector_resource.erl
View File

@ -49,6 +49,8 @@
get_channels/2
]).
-export([parse_url/1]).
-callback connector_config(ParsedConfig) ->
ParsedConfig
when
@ -77,8 +79,10 @@ connector_impl_module(_ConnectorType) ->
-endif.
connector_to_resource_type_ce(_ConnectorType) ->
no_bridge_v2_for_c2_so_far.
connector_to_resource_type_ce(http) ->
emqx_bridge_http_connector;
connector_to_resource_type_ce(ConnectorType) ->
error({no_bridge_v2, ConnectorType}).
resource_id(ConnectorId) when is_binary(ConnectorId) ->
<<"connector:", ConnectorId/binary>>.
@ -271,13 +275,11 @@ remove(Type, Name, _Conf, _Opts) ->
%% convert connector configs to what the connector modules want
parse_confs(
<<"webhook">>,
<<"http">>,
_Name,
#{
url := Url,
method := Method,
headers := Headers,
max_retries := Retry
headers := Headers
} = Conf
) ->
Url1 = bin(Url),
@ -290,20 +292,14 @@ parse_confs(
Reason1 = emqx_utils:readable_error_msg(Reason),
invalid_data(<<"Invalid URL: ", Url1/binary, ", details: ", Reason1/binary>>)
end,
RequestTTL = emqx_utils_maps:deep_get(
[resource_opts, request_ttl],
Conf
),
Conf#{
base_url => BaseUrl1,
request =>
#{
path => Path,
method => Method,
body => maps:get(body, Conf, undefined),
headers => Headers,
request_ttl => RequestTTL,
max_retries => Retry
body => undefined,
method => undefined
}
};
parse_confs(<<"iotdb">>, Name, Conf) ->

15
apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl
View File

@ -15,7 +15,8 @@
-export([
api_schemas/1,
fields/1,
examples/1
%%examples/1
schema_modules/0
]).
resource_type(Type) when is_binary(Type) ->
@ -151,18 +152,6 @@ connector_structs() ->
)}
].
examples(Method) ->
MergeFun =
fun(Example, Examples) ->
maps:merge(Examples, Example)
end,
Fun =
fun(Module, Examples) ->
ConnectorExamples = erlang:apply(Module, connector_examples, [Method]),
lists:foldl(MergeFun, Examples, ConnectorExamples)
end,
lists:foldl(Fun, #{}, schema_modules()).
schema_modules() ->
[
emqx_bridge_azure_event_hub,

51
apps/emqx_connector/src/schema/emqx_connector_schema.erl
View File

@ -42,6 +42,8 @@
-export([resource_opts_fields/0, resource_opts_fields/1]).
-export([examples/1]).
-if(?EMQX_RELEASE_EDITION == ee).
enterprise_api_schemas(Method) ->
%% We *must* do this to ensure the module is really loaded, especially when we use
@ -71,6 +73,40 @@ enterprise_fields_connectors() -> [].
-endif.
api_schemas(Method) ->
[
%% We need to map the `type' field of a request (binary) to a
%% connector schema module.
api_ref(emqx_bridge_http_schema, <<"http">>, Method ++ "_connector")
].
api_ref(Module, Type, Method) ->
{Type, ref(Module, Method)}.
examples(Method) ->
MergeFun =
fun(Example, Examples) ->
maps:merge(Examples, Example)
end,
Fun =
fun(Module, Examples) ->
ConnectorExamples = erlang:apply(Module, connector_examples, [Method]),
lists:foldl(MergeFun, Examples, ConnectorExamples)
end,
lists:foldl(Fun, #{}, schema_modules()).
-if(?EMQX_RELEASE_EDITION == ee).
schema_modules() ->
[emqx_bridge_http_schema] ++ emqx_connector_ee_schema:schema_modules().
-else.
schema_modules() ->
[emqx_bridge_http_schema].
-endif.
%% @doc Return old bridge(v1) and/or connector(v2) type
%% from the latest connector type name.
connector_type_to_bridge_types(http) ->
[webhook, http];
connector_type_to_bridge_types(azure_event_hub_producer) ->
[azure_event_hub_producer];
connector_type_to_bridge_types(confluent_producer) ->
@ -323,8 +359,9 @@ post_request() ->
api_schema("post").
api_schema(Method) ->
CE = api_schemas(Method),
EE = enterprise_api_schemas(Method),
hoconsc:union(connector_api_union(EE)).
hoconsc:union(connector_api_union(CE ++ EE)).
connector_api_union(Refs) ->
Index = maps:from_list(Refs),
@ -369,7 +406,17 @@ roots() ->
end.
fields(connectors) ->
[] ++ enterprise_fields_connectors();
[
{http,
mk(
hoconsc:map(name, ref(emqx_bridge_http_schema, "config_connector")),
#{
alias => [webhook],
desc => <<"HTTP Connector Config">>,
required => false
}
)}
] ++ enterprise_fields_connectors();
fields("node_status") ->
[
node_name(),

2
apps/emqx_dashboard/src/emqx_dashboard.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_dashboard, [
{description, "EMQX Web Dashboard"},
% strict semver, bump manually!
{vsn, "5.0.30"},
{vsn, "5.0.31"},
{modules, []},
{registered, [emqx_dashboard_sup]},
{applications, [

2
apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_dashboard_rbac, [
{description, "EMQX Dashboard RBAC"},
{vsn, "0.1.1"},
{vsn, "0.1.2"},
{registered, []},
{applications, [
kernel,

2
apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_dashboard_sso, [
{description, "EMQX Dashboard Single Sign-On"},
{vsn, "0.1.2"},
{vsn, "0.1.3"},
{registered, [emqx_dashboard_sso_sup]},
{applications, [
kernel,

6
apps/emqx_durable_storage/src/emqx_ds_replication_layer_meta.erl
View File

@ -192,9 +192,9 @@ sites() ->
{ok, node()} | {error, no_leader_for_shard}.
shard_leader(DB, Shard) ->
case mnesia:dirty_read(?SHARD_TAB, {DB, Shard}) of
[#?SHARD_TAB{leader = Leader}] ->
[#?SHARD_TAB{leader = Leader}] when Leader =/= undefined ->
{ok, Leader};
[] ->
_ ->
{error, no_leader_for_shard}
end.
@ -314,7 +314,7 @@ ensure_tables() ->
{rlog_shard, ?SHARD},
{majority, Majority},
{type, ordered_set},
{storage, ram_copies},
{storage, disc_copies},
{record_name, ?SHARD_TAB},
{attributes, record_info(fields, ?SHARD_TAB)}
]),

2
apps/emqx_durable_storage/src/emqx_durable_storage.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_durable_storage, [
{description, "Message persistence and subscription replays for EMQX"},
% strict semver, bump manually!
{vsn, "0.1.7"},
{vsn, "0.1.8"},
{modules, []},
{registered, []},
{applications, [kernel, stdlib, rocksdb, gproc, mria, emqx_utils]},

2
apps/emqx_enterprise/src/emqx_enterprise.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_enterprise, [
{description, "EMQX Enterprise Edition"},
{vsn, "0.1.5"},
{vsn, "0.1.6"},
{registered, []},
{applications, [
kernel,

2
apps/emqx_gateway/src/emqx_gateway.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_gateway, [
{description, "The Gateway management application"},
{vsn, "0.1.27"},
{vsn, "0.1.28"},
{registered, []},
{mod, {emqx_gateway_app, []}},
{applications, [kernel, stdlib, emqx, emqx_auth, emqx_ctl]},

8
apps/emqx_gateway_gbt32960/src/emqx_gbt32960_schema.erl
View File

@ -4,12 +4,18 @@
-module(emqx_gbt32960_schema).
-behaviour(hocon_schema).
-include("emqx_gbt32960.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("typerefl/include/types.hrl").
%% config schema provides
-export([fields/1, desc/1]).
-export([namespace/0, roots/0, fields/1, desc/1]).
namespace() -> "gateway_gbt32960".
roots() -> [].
fields(gbt32960) ->
[

2
apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_gateway_lwm2m, [
{description, "LwM2M Gateway"},
{vsn, "0.1.4"},
{vsn, "0.1.5"},
{registered, []},
{applications, [kernel, stdlib, emqx, emqx_gateway, emqx_gateway_coap]},
{env, []},

6
apps/emqx_gateway_ocpp/src/emqx_ocpp_schema.erl
View File

@ -10,7 +10,11 @@
-define(DEFAULT_MOUNTPOINT, <<"ocpp/">>).
%% config schema provides
-export([fields/1, desc/1]).
-export([namespace/0, roots/0, fields/1, desc/1]).
namespace() -> "gateway_ocpp".
roots() -> [].
fields(ocpp) ->
[

2
apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_gateway_stomp, [
{description, "Stomp Gateway"},
{vsn, "0.1.4"},
{vsn, "0.1.5"},
{registered, []},
{applications, [kernel, stdlib, emqx, emqx_gateway]},
{env, []},

2
apps/emqx_ldap/src/emqx_ldap.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_ldap, [
{description, "EMQX LDAP Connector"},
{vsn, "0.1.5"},
{vsn, "0.1.6"},
{registered, []},
{applications, [
kernel,

Some files were not shown because too many files have changed in this diff Show More