yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore(ps_ds): make persistent session module use new `emqx_ds` APIs

This commit is contained in:
Thales Macedo Garitezi 2023-10-05 17:17:08 -03:00 committed by ieQu1
parent 2972bf14ee
commit 903b3863d1
5 changed files with 203 additions and 286 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl
View File

@ -14,7 +14,6 @@
-define(DEFAULT_KEYSPACE, default). -define(DEFAULT_KEYSPACE, default).
-define(DS_SHARD_ID, <<"local">>). -define(DS_SHARD_ID, <<"local">>).
-define(DS_SHARD, {?DEFAULT_KEYSPACE, ?DS_SHARD_ID}). -define(DS_SHARD, {?DEFAULT_KEYSPACE, ?DS_SHARD_ID}).
-define(ITERATOR_REF_TAB, emqx_ds_iterator_ref).
-import(emqx_common_test_helpers, [on_exit/1]). -import(emqx_common_test_helpers, [on_exit/1]).
@ -91,9 +90,6 @@ get_mqtt_port(Node, Type) ->
{_IP, Port} = erpc:call(Node, emqx_config, get, [[listeners, Type, default, bind]]), {_IP, Port} = erpc:call(Node, emqx_config, get, [[listeners, Type, default, bind]]),
Port. Port.
get_all_iterator_refs(Node) ->
erpc:call(Node, mnesia, dirty_all_keys, [?ITERATOR_REF_TAB]).
get_all_iterator_ids(Node) -> get_all_iterator_ids(Node) ->
Fn = fun(K, _V, Acc) -> [K | Acc] end, Fn = fun(K, _V, Acc) -> [K | Acc] end,
erpc:call(Node, fun() -> erpc:call(Node, fun() ->
@ -126,6 +122,32 @@ start_client(Opts0 = #{}) ->
on_exit(fun() -> catch emqtt:stop(Client) end), on_exit(fun() -> catch emqtt:stop(Client) end),
Client. Client.
restart_node(Node, NodeSpec) ->
?tp(will_restart_node, #{}),
?tp(notice, "restarting node", #{node => Node}),
true = monitor_node(Node, true),
ok = erpc:call(Node, init, restart, []),
receive
{nodedown, Node} ->
ok
after 10_000 ->
ct:fail("node ~p didn't stop", [Node])
end,
?tp(notice, "waiting for nodeup", #{node => Node}),
wait_nodeup(Node),
wait_gen_rpc_down(NodeSpec),
?tp(notice, "restarting apps", #{node => Node}),
Apps = maps:get(apps, NodeSpec),
ok = erpc:call(Node, emqx_cth_suite, load_apps, [Apps]),
_ = erpc:call(Node, emqx_cth_suite, start_apps, [Apps, NodeSpec]),
%% have to re-inject this so that we may stop the node succesfully at the
%% end....
ok = emqx_cth_cluster:set_node_opts(Node, NodeSpec),
ok = snabbkaffe:forward_trace(Node),
?tp(notice, "node restarted", #{node => Node}),
?tp(restarted_node, #{}),
ok.
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% Testcases %% Testcases
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
@ -143,24 +165,14 @@ t_non_persistent_session_subscription(_Config) ->
{ok, _} = emqtt:connect(Client), {ok, _} = emqtt:connect(Client),
?tp(notice, "subscribing", #{}), ?tp(notice, "subscribing", #{}),
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(Client, SubTopicFilter, qos2), {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(Client, SubTopicFilter, qos2),
IteratorRefs = get_all_iterator_refs(node()),
IteratorIds = get_all_iterator_ids(node()),
ok = emqtt:stop(Client), ok = emqtt:stop(Client),
#{ ok
iterator_refs => IteratorRefs,
iterator_ids => IteratorIds
}
end, end,
fun(Res, Trace) -> fun(Trace) ->
ct:pal("trace:\n ~p", [Trace]), ct:pal("trace:\n ~p", [Trace]),
#{ ?assertEqual([], ?of_kind(ds_session_subscription_added, Trace)),
iterator_refs := IteratorRefs,
iterator_ids := IteratorIds
} = Res,
?assertEqual([], IteratorRefs),
?assertEqual({ok, []}, IteratorIds),
ok ok
end end
), ),
@ -175,7 +187,7 @@ t_session_subscription_idempotency(Config) ->
?check_trace( ?check_trace(
begin begin
?force_ordering( ?force_ordering(
#{?snk_kind := persistent_session_ds_iterator_added}, #{?snk_kind := persistent_session_ds_subscription_added},
_NEvents0 = 1, _NEvents0 = 1,
#{?snk_kind := will_restart_node}, #{?snk_kind := will_restart_node},
_Guard0 = true _Guard0 = true
@ -187,32 +199,7 @@ t_session_subscription_idempotency(Config) ->
_Guard1 = true _Guard1 = true
), ),
spawn_link(fun() -> spawn_link(fun() -> restart_node(Node1, Node1Spec) end),
?tp(will_restart_node, #{}),
?tp(notice, "restarting node", #{node => Node1}),
true = monitor_node(Node1, true),
ok = erpc:call(Node1, init, restart, []),
receive
{nodedown, Node1} ->
ok
after 10_000 ->
ct:fail("node ~p didn't stop", [Node1])
end,
?tp(notice, "waiting for nodeup", #{node => Node1}),
wait_nodeup(Node1),
wait_gen_rpc_down(Node1Spec),
?tp(notice, "restarting apps", #{node => Node1}),
Apps = maps:get(apps, Node1Spec),
ok = erpc:call(Node1, emqx_cth_suite, load_apps, [Apps]),
_ = erpc:call(Node1, emqx_cth_suite, start_apps, [Apps, Node1Spec]),
%% have to re-inject this so that we may stop the node succesfully at the
%% end....
ok = emqx_cth_cluster:set_node_opts(Node1, Node1Spec),
ok = snabbkaffe:forward_trace(Node1),
?tp(notice, "node restarted", #{node => Node1}),
?tp(restarted_node, #{}),
ok
end),
?tp(notice, "starting 1", #{}), ?tp(notice, "starting 1", #{}),
Client0 = start_client(#{port => Port, clientid => ClientId}), Client0 = start_client(#{port => Port, clientid => ClientId}),
@ -223,7 +210,7 @@ t_session_subscription_idempotency(Config) ->
receive receive
{'EXIT', {shutdown, _}} -> {'EXIT', {shutdown, _}} ->
ok ok
after 0 -> ok after 100 -> ok
end, end,
process_flag(trap_exit, false), process_flag(trap_exit, false),
@ -240,10 +227,7 @@ t_session_subscription_idempotency(Config) ->
end, end,
fun(Trace) -> fun(Trace) ->
ct:pal("trace:\n ~p", [Trace]), ct:pal("trace:\n ~p", [Trace]),
%% Exactly one iterator should have been opened.
SubTopicFilterWords = emqx_topic:words(SubTopicFilter), SubTopicFilterWords = emqx_topic:words(SubTopicFilter),
?assertEqual([{ClientId, SubTopicFilterWords}], get_all_iterator_refs(Node1)),
?assertMatch({ok, [_]}, get_all_iterator_ids(Node1)),
?assertMatch( ?assertMatch(
{ok, #{}, #{SubTopicFilterWords := #{}}}, {ok, #{}, #{SubTopicFilterWords := #{}}},
erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId]) erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId])
@ -262,7 +246,10 @@ t_session_unsubscription_idempotency(Config) ->
?check_trace( ?check_trace(
begin begin
?force_ordering( ?force_ordering(
#{?snk_kind := persistent_session_ds_close_iterators, ?snk_span := {complete, _}}, #{
?snk_kind := persistent_session_ds_subscription_delete,
?snk_span := {complete, _}
},
_NEvents0 = 1, _NEvents0 = 1,
#{?snk_kind := will_restart_node}, #{?snk_kind := will_restart_node},
_Guard0 = true _Guard0 = true
@ -270,36 +257,11 @@ t_session_unsubscription_idempotency(Config) ->
?force_ordering( ?force_ordering(
#{?snk_kind := restarted_node}, #{?snk_kind := restarted_node},
_NEvents1 = 1, _NEvents1 = 1,
#{?snk_kind := persistent_session_ds_iterator_delete, ?snk_span := start}, #{?snk_kind := persistent_session_ds_subscription_route_delete, ?snk_span := start},
_Guard1 = true _Guard1 = true
), ),
spawn_link(fun() -> spawn_link(fun() -> restart_node(Node1, Node1Spec) end),
?tp(will_restart_node, #{}),
?tp(notice, "restarting node", #{node => Node1}),
true = monitor_node(Node1, true),
ok = erpc:call(Node1, init, restart, []),
receive
{nodedown, Node1} ->
ok
after 10_000 ->
ct:fail("node ~p didn't stop", [Node1])
end,
?tp(notice, "waiting for nodeup", #{node => Node1}),
wait_nodeup(Node1),
wait_gen_rpc_down(Node1Spec),
?tp(notice, "restarting apps", #{node => Node1}),
Apps = maps:get(apps, Node1Spec),
ok = erpc:call(Node1, emqx_cth_suite, load_apps, [Apps]),
_ = erpc:call(Node1, emqx_cth_suite, start_apps, [Apps, Node1Spec]),
%% have to re-inject this so that we may stop the node succesfully at the
%% end....
ok = emqx_cth_cluster:set_node_opts(Node1, Node1Spec),
ok = snabbkaffe:forward_trace(Node1),
?tp(notice, "node restarted", #{node => Node1}),
?tp(restarted_node, #{}),
ok
end),
?tp(notice, "starting 1", #{}), ?tp(notice, "starting 1", #{}),
Client0 = start_client(#{port => Port, clientid => ClientId}), Client0 = start_client(#{port => Port, clientid => ClientId}),
@ -312,7 +274,7 @@ t_session_unsubscription_idempotency(Config) ->
receive receive
{'EXIT', {shutdown, _}} -> {'EXIT', {shutdown, _}} ->
ok ok
after 0 -> ok after 100 -> ok
end, end,
process_flag(trap_exit, false), process_flag(trap_exit, false),
@ -327,7 +289,7 @@ t_session_unsubscription_idempotency(Config) ->
?wait_async_action( ?wait_async_action(
emqtt:unsubscribe(Client1, SubTopicFilter), emqtt:unsubscribe(Client1, SubTopicFilter),
#{ #{
?snk_kind := persistent_session_ds_iterator_delete, ?snk_kind := persistent_session_ds_subscription_route_delete,
?snk_span := {complete, _} ?snk_span := {complete, _}
}, },
15_000 15_000
@ -339,9 +301,10 @@ t_session_unsubscription_idempotency(Config) ->
end, end,
fun(Trace) -> fun(Trace) ->
ct:pal("trace:\n ~p", [Trace]), ct:pal("trace:\n ~p", [Trace]),
%% No iterators remaining ?assertMatch(
?assertEqual([], get_all_iterator_refs(Node1)), {ok, #{}, Subs = #{}} when map_size(Subs) =:= 0,
?assertEqual({ok, []}, get_all_iterator_ids(Node1)), erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId])
),
ok ok
end end
), ),

2
apps/emqx/src/emqx_persistent_message.erl
View File

@ -42,7 +42,7 @@ init() ->
?WHEN_ENABLED(begin ?WHEN_ENABLED(begin
ok = emqx_ds:open_db(?PERSISTENT_MESSAGE_DB, #{}), ok = emqx_ds:open_db(?PERSISTENT_MESSAGE_DB, #{}),
ok = emqx_persistent_session_ds_router:init_tables(), ok = emqx_persistent_session_ds_router:init_tables(),
%ok = emqx_persistent_session_ds:create_tables(), ok = emqx_persistent_session_ds:create_tables(),
ok ok
end). end).

286
apps/emqx/src/emqx_persistent_session_ds.erl_ → apps/emqx/src/emqx_persistent_session_ds.erl
View File

@ -65,10 +65,13 @@
%% Currently, this is the clientid. We avoid `emqx_types:clientid()' because that can be %% Currently, this is the clientid. We avoid `emqx_types:clientid()' because that can be
%% an atom, in theory (?). %% an atom, in theory (?).
-type id() :: binary(). -type id() :: binary().
-type iterator() :: emqx_ds:iterator().
-type iterator_id() :: emqx_ds:iterator_id().
-type topic_filter() :: emqx_ds:topic_filter(). -type topic_filter() :: emqx_ds:topic_filter().
-type iterators() :: #{topic_filter() => iterator()}. -type subscription_id() :: {id(), topic_filter()}.
-type subscription() :: #{
start_time := emqx_ds:time(),
propts := map(),
extra := map()
}.
-type session() :: #{ -type session() :: #{
%% Client ID %% Client ID
id := id(), id := id(),
@ -77,7 +80,7 @@
%% When the session should expire %% When the session should expire
expires_at := timestamp() | never, expires_at := timestamp() | never,
%% Client’s Subscriptions. %% Client’s Subscriptions.
iterators := #{topic() => iterator()}, iterators := #{topic() => subscription()},
%% %%
props := map() props := map()
}. }.
@ -90,6 +93,8 @@
-export_type([id/0]). -export_type([id/0]).
-define(PERSISTENT_MESSAGE_DB, emqx_persistent_message).
%% %%
-spec create(clientinfo(), conninfo(), emqx_session:conf()) -> -spec create(clientinfo(), conninfo(), emqx_session:conf()) ->
@ -121,17 +126,17 @@ ensure_session(ClientID, Conf) ->
open_session(ClientID) -> open_session(ClientID) ->
case session_open(ClientID) of case session_open(ClientID) of
{ok, Session, Iterators} -> {ok, Session, Subscriptions} ->
Session#{iterators => prep_iterators(Iterators)}; Session#{iterators => prep_subscriptions(Subscriptions)};
false -> false ->
false false
end. end.
prep_iterators(Iterators) -> prep_subscriptions(Subscriptions) ->
maps:fold( maps:fold(
fun(Topic, Iterator, Acc) -> Acc#{emqx_topic:join(Topic) => Iterator} end, fun(Topic, Subscription, Acc) -> Acc#{emqx_topic:join(Topic) => Subscription} end,
#{}, #{},
Iterators Subscriptions
). ).
-spec destroy(session() | clientinfo()) -> ok. -spec destroy(session() | clientinfo()) -> ok.
@ -228,7 +233,7 @@ unsubscribe(
) when is_map_key(TopicFilter, Iters) -> ) when is_map_key(TopicFilter, Iters) ->
Iterator = maps:get(TopicFilter, Iters), Iterator = maps:get(TopicFilter, Iters),
SubOpts = maps:get(props, Iterator), SubOpts = maps:get(props, Iterator),
ok = del_subscription(TopicFilter, Iterator, ID), ok = del_subscription(TopicFilter, ID),
{ok, Session#{iterators := maps:remove(TopicFilter, Iters)}, SubOpts}; {ok, Session#{iterators := maps:remove(TopicFilter, Iters)}, SubOpts};
unsubscribe( unsubscribe(
_TopicFilter, _TopicFilter,
@ -327,91 +332,67 @@ terminate(_Reason, _Session = #{}) ->
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-spec add_subscription(topic(), emqx_types:subopts(), id()) -> -spec add_subscription(topic(), emqx_types:subopts(), id()) ->
emqx_ds:iterator(). subscription().
add_subscription(TopicFilterBin, SubOpts, DSSessionID) -> add_subscription(TopicFilterBin, SubOpts, DSSessionID) ->
% N.B.: we chose to update the router before adding the subscription to the %% N.B.: we chose to update the router before adding the subscription to the
% session/iterator table. The reasoning for this is as follows: %% session/iterator table. The reasoning for this is as follows:
% %%
% Messages matching this topic filter should start to be persisted as soon as %% Messages matching this topic filter should start to be persisted as soon as
% possible to avoid missing messages. If this is the first such persistent %% possible to avoid missing messages. If this is the first such persistent
% session subscription, it's important to do so early on. %% session subscription, it's important to do so early on.
% %%
% This could, in turn, lead to some inconsistency: if such a route gets %% This could, in turn, lead to some inconsistency: if such a route gets
% created but the session/iterator data fails to be updated accordingly, we %% created but the session/iterator data fails to be updated accordingly, we
% have a dangling route. To remove such dangling routes, we may have a %% have a dangling route. To remove such dangling routes, we may have a
% periodic GC process that removes routes that do not have a matching %% periodic GC process that removes routes that do not have a matching
% persistent subscription. Also, route operations use dirty mnesia %% persistent subscription. Also, route operations use dirty mnesia
% operations, which inherently have room for inconsistencies. %% operations, which inherently have room for inconsistencies.
% %%
% In practice, we use the iterator reference table as a source of truth, %% In practice, we use the iterator reference table as a source of truth,
% since it is guarded by a transaction context: we consider a subscription %% since it is guarded by a transaction context: we consider a subscription
% operation to be successful if it ended up changing this table. Both router %% operation to be successful if it ended up changing this table. Both router
% and iterator information can be reconstructed from this table, if needed. %% and iterator information can be reconstructed from this table, if needed.
ok = emqx_persistent_session_ds_router:do_add_route(TopicFilterBin, DSSessionID), ok = emqx_persistent_session_ds_router:do_add_route(TopicFilterBin, DSSessionID),
TopicFilter = emqx_topic:words(TopicFilterBin), TopicFilter = emqx_topic:words(TopicFilterBin),
{ok, Iterator, IsNew} = session_add_iterator( {ok, DSSubExt, IsNew} = session_add_subscription(
DSSessionID, TopicFilter, SubOpts DSSessionID, TopicFilter, SubOpts
), ),
Ctx = #{iterator => Iterator, is_new => IsNew}, ?tp(persistent_session_ds_subscription_added, #{sub => DSSubExt, is_new => IsNew}),
?tp(persistent_session_ds_iterator_added, Ctx), %% we'll list streams and open iterators when implementing message replay.
DSSubExt.
-spec update_subscription(topic(), subscription(), emqx_types:subopts(), id()) ->
subscription().
update_subscription(TopicFilterBin, DSSubExt, SubOpts, DSSessionID) ->
TopicFilter = emqx_topic:words(TopicFilterBin),
{ok, NDSSubExt, false} = session_add_subscription(
DSSessionID, TopicFilter, SubOpts
),
ok = ?tp(persistent_session_ds_iterator_updated, #{sub => DSSubExt}),
NDSSubExt.
-spec del_subscription(topic(), id()) ->
ok.
del_subscription(TopicFilterBin, DSSessionId) ->
TopicFilter = emqx_topic:words(TopicFilterBin),
?tp_span( ?tp_span(
persistent_session_ds_open_iterators, persistent_session_ds_subscription_delete,
Ctx, #{session_id => DSSessionId},
ok = open_iterator_on_all_shards(TopicFilter, Iterator) ok = session_del_subscription(DSSessionId, TopicFilter)
), ),
Iterator. ?tp_span(
persistent_session_ds_subscription_route_delete,
-spec update_subscription(topic(), iterator(), emqx_types:subopts(), id()) -> #{session_id => DSSessionId},
iterator(). ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilterBin, DSSessionId)
update_subscription(TopicFilterBin, Iterator, SubOpts, DSSessionID) -> ).
TopicFilter = emqx_topic:words(TopicFilterBin),
{ok, NIterator, false} = session_add_iterator(
DSSessionID, TopicFilter, SubOpts
),
ok = ?tp(persistent_session_ds_iterator_updated, #{iterator => Iterator}),
NIterator.
-spec open_iterator_on_all_shards(emqx_types:words(), emqx_ds:iterator()) -> ok.
open_iterator_on_all_shards(TopicFilter, Iterator) ->
?tp(persistent_session_ds_will_open_iterators, #{iterator => Iterator}),
%% Note: currently, shards map 1:1 to nodes, but this will change in the future.
Nodes = emqx:running_nodes(),
Results = emqx_persistent_session_ds_proto_v1:open_iterator(
Nodes,
TopicFilter,
maps:get(start_time, Iterator),
maps:get(id, Iterator)
),
%% TODO
%% 1. Handle errors.
%% 2. Iterator handles are rocksdb resources, it's doubtful they survive RPC.
%% Even if they do, we throw them away here anyway. All in all, we probably should
%% hold each of them in a process on the respective node.
true = lists:all(fun(Res) -> element(1, Res) =:= ok end, Results),
ok.
%% RPC target.
-spec do_open_iterator(emqx_types:words(), emqx_ds:time(), emqx_ds:iterator_id()) ->
{ok, emqx_ds_storage_layer:iterator()} | {error, _Reason}.
do_open_iterator(TopicFilter, StartMS, _IteratorID) ->
%% TODO: wrong
{ok, emqx_ds:make_iterator(TopicFilter, StartMS)}.
-spec del_subscription(topic(), iterator(), id()) ->
ok.
del_subscription(TopicFilterBin, #{id := IteratorID}, DSSessionID) ->
% N.B.: see comments in `?MODULE:add_subscription' for a discussion about the
% order of operations here.
TopicFilter = emqx_topic:words(TopicFilterBin),
ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilterBin, DSSessionID).
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% Session tables operations %% Session tables operations
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-define(SESSION_TAB, emqx_ds_session). -define(SESSION_TAB, emqx_ds_session).
-define(ITERATOR_REF_TAB, emqx_ds_iterator_ref). -define(SESSION_SUBSCRIPTIONS_TAB, emqx_ds_session_subscriptions).
-define(DS_MRIA_SHARD, emqx_ds_shard). -define(DS_MRIA_SHARD, emqx_ds_session_shard).
-record(session, { -record(session, {
%% same as clientid %% same as clientid
@ -423,12 +404,13 @@ del_subscription(TopicFilterBin, #{id := IteratorID}, DSSessionID) ->
props = #{} :: map() props = #{} :: map()
}). }).
-record(iterator_ref, { -record(ds_sub, {
ref_id :: {id(), emqx_ds:topic_filter()}, id :: subscription_id(),
it_id :: emqx_ds:iterator_id(),
start_time :: emqx_ds:time(), start_time :: emqx_ds:time(),
props = #{} :: map() props = #{} :: map(),
extra = #{} :: map()
}). }).
-type ds_sub() :: #ds_sub{}.
create_tables() -> create_tables() ->
ok = mria:create_table( ok = mria:create_table(
@ -442,15 +424,16 @@ create_tables() ->
] ]
), ),
ok = mria:create_table( ok = mria:create_table(
?ITERATOR_REF_TAB, ?SESSION_SUBSCRIPTIONS_TAB,
[ [
{rlog_shard, ?DS_MRIA_SHARD}, {rlog_shard, ?DS_MRIA_SHARD},
{type, ordered_set}, {type, ordered_set},
{storage, storage()}, {storage, storage()},
{record_name, iterator_ref}, {record_name, ds_sub},
{attributes, record_info(fields, iterator_ref)} {attributes, record_info(fields, ds_sub)}
] ]
), ),
ok = mria:wait_for_tables([?SESSION_TAB, ?SESSION_SUBSCRIPTIONS_TAB]),
ok. ok.
-dialyzer({nowarn_function, storage/0}). -dialyzer({nowarn_function, storage/0}).
@ -471,26 +454,26 @@ storage() ->
%% Note: session API doesn't handle session takeovers, it's the job of %% Note: session API doesn't handle session takeovers, it's the job of
%% the broker. %% the broker.
-spec session_open(id()) -> -spec session_open(id()) ->
{ok, session(), iterators()} | false. {ok, session(), #{topic() => subscription()}} | false.
session_open(SessionId) -> session_open(SessionId) ->
transaction(fun() -> transaction(fun() ->
case mnesia:read(?SESSION_TAB, SessionId, write) of case mnesia:read(?SESSION_TAB, SessionId, write) of
[Record = #session{}] -> [Record = #session{}] ->
Session = export_record(Record), Session = export_session(Record),
IteratorRefs = session_read_iterators(SessionId), DSSubs = session_read_subscriptions(SessionId),
Iterators = export_iterators(IteratorRefs), Subscriptions = export_subscriptions(DSSubs),
{ok, Session, Iterators}; {ok, Session, Subscriptions};
[] -> [] ->
false false
end end
end). end).
-spec session_ensure_new(id(), _Props :: map()) -> -spec session_ensure_new(id(), _Props :: map()) ->
{ok, session(), iterators()}. {ok, session(), #{topic() => subscription()}}.
session_ensure_new(SessionId, Props) -> session_ensure_new(SessionId, Props) ->
transaction(fun() -> transaction(fun() ->
ok = session_drop_iterators(SessionId), ok = session_drop_subscriptions(SessionId),
Session = export_record(session_create(SessionId, Props)), Session = export_session(session_create(SessionId, Props)),
{ok, Session, #{}} {ok, Session, #{}}
end). end).
@ -510,80 +493,80 @@ session_create(SessionId, Props) ->
session_drop(DSSessionId) -> session_drop(DSSessionId) ->
transaction(fun() -> transaction(fun() ->
%% TODO: ensure all iterators from this clientid are closed? %% TODO: ensure all iterators from this clientid are closed?
ok = session_drop_iterators(DSSessionId), ok = session_drop_subscriptions(DSSessionId),
ok = mnesia:delete(?SESSION_TAB, DSSessionId, write) ok = mnesia:delete(?SESSION_TAB, DSSessionId, write)
end). end).
session_drop_iterators(DSSessionId) -> session_drop_subscriptions(DSSessionId) ->
IteratorRefs = session_read_iterators(DSSessionId), IteratorRefs = session_read_subscriptions(DSSessionId),
ok = lists:foreach(fun session_del_iterator/1, IteratorRefs). ok = lists:foreach(fun session_del_subscription/1, IteratorRefs).
%% @doc Called when a client subscribes to a topic. Idempotent. %% @doc Called when a client subscribes to a topic. Idempotent.
-spec session_add_iterator(id(), topic_filter(), _Props :: map()) -> -spec session_add_subscription(id(), topic_filter(), _Props :: map()) ->
{ok, iterator(), _IsNew :: boolean()}. {ok, subscription(), _IsNew :: boolean()}.
session_add_iterator(DSSessionId, TopicFilter, Props) -> session_add_subscription(DSSessionId, TopicFilter, Props) ->
IteratorRefId = {DSSessionId, TopicFilter}, DSSubId = {DSSessionId, TopicFilter},
transaction(fun() -> transaction(fun() ->
case mnesia:read(?ITERATOR_REF_TAB, IteratorRefId, write) of case mnesia:read(?SESSION_SUBSCRIPTIONS_TAB, DSSubId, write) of
[] -> [] ->
IteratorRef = session_insert_iterator(DSSessionId, TopicFilter, Props), DSSub = session_insert_subscription(DSSessionId, TopicFilter, Props),
Iterator = export_record(IteratorRef), DSSubExt = export_subscription(DSSub),
?tp( ?tp(
ds_session_subscription_added, ds_session_subscription_added,
#{iterator => Iterator, session_id => DSSessionId} #{sub => DSSubExt, session_id => DSSessionId}
), ),
{ok, Iterator, _IsNew = true}; {ok, DSSubExt, _IsNew = true};
[#iterator_ref{} = IteratorRef] -> [#ds_sub{} = DSSub] ->
NIteratorRef = session_update_iterator(IteratorRef, Props), NDSSub = session_update_subscription(DSSub, Props),
NIterator = export_record(NIteratorRef), NDSSubExt = export_subscription(NDSSub),
?tp( ?tp(
ds_session_subscription_present, ds_session_subscription_present,
#{iterator => NIterator, session_id => DSSessionId} #{sub => NDSSubExt, session_id => DSSessionId}
), ),
{ok, NIterator, _IsNew = false} {ok, NDSSubExt, _IsNew = false}
end end
end). end).
session_insert_iterator(DSSessionId, TopicFilter, Props) -> -spec session_insert_subscription(id(), topic_filter(), map()) -> ds_sub().
{IteratorId, StartMS} = new_iterator_id(DSSessionId), session_insert_subscription(DSSessionId, TopicFilter, Props) ->
IteratorRef = #iterator_ref{ {DSSubId, StartMS} = new_subscription_id(DSSessionId, TopicFilter),
ref_id = {DSSessionId, TopicFilter}, DSSub = #ds_sub{
it_id = IteratorId, id = DSSubId,
start_time = StartMS, start_time = StartMS,
props = Props props = Props,
extra = #{}
}, },
ok = mnesia:write(?ITERATOR_REF_TAB, IteratorRef, write), ok = mnesia:write(?SESSION_SUBSCRIPTIONS_TAB, DSSub, write),
IteratorRef. DSSub.
session_update_iterator(IteratorRef, Props) -> -spec session_update_subscription(ds_sub(), map()) -> ds_sub().
NIteratorRef = IteratorRef#iterator_ref{props = Props}, session_update_subscription(DSSub, Props) ->
ok = mnesia:write(?ITERATOR_REF_TAB, NIteratorRef, write), NDSSub = DSSub#ds_sub{props = Props},
NIteratorRef. ok = mnesia:write(?SESSION_SUBSCRIPTIONS_TAB, NDSSub, write),
NDSSub.
%% @doc Called when a client unsubscribes from a topic. session_del_subscription(DSSessionId, TopicFilter) ->
-spec session_del_iterator(id(), topic_filter()) -> ok. DSSubId = {DSSessionId, TopicFilter},
session_del_iterator(DSSessionId, TopicFilter) ->
IteratorRefId = {DSSessionId, TopicFilter},
transaction(fun() -> transaction(fun() ->
mnesia:delete(?ITERATOR_REF_TAB, IteratorRefId, write) mnesia:delete(?SESSION_SUBSCRIPTIONS_TAB, DSSubId, write)
end). end).
session_del_iterator(#iterator_ref{ref_id = IteratorRefId}) -> session_del_subscription(#ds_sub{id = DSSubId}) ->
mnesia:delete(?ITERATOR_REF_TAB, IteratorRefId, write). mnesia:delete(?SESSION_SUBSCRIPTIONS_TAB, DSSubId, write).
session_read_iterators(DSSessionId) -> session_read_subscriptions(DSSessionId) ->
% NOTE: somewhat convoluted way to trick dialyzer % NOTE: somewhat convoluted way to trick dialyzer
Pat = erlang:make_tuple(record_info(size, iterator_ref), '_', [ Pat = erlang:make_tuple(record_info(size, ds_sub), '_', [
{1, iterator_ref}, {1, ds_sub},
{#iterator_ref.ref_id, {DSSessionId, '_'}} {#ds_sub.id, {DSSessionId, '_'}}
]), ]),
mnesia:match_object(?ITERATOR_REF_TAB, Pat, read). mnesia:match_object(?SESSION_SUBSCRIPTIONS_TAB, Pat, read).
-spec new_iterator_id(id()) -> {iterator_id(), emqx_ds:time()}. -spec new_subscription_id(id(), topic_filter()) -> {subscription_id(), emqx_ds:time()}.
new_iterator_id(DSSessionId) -> new_subscription_id(DSSessionId, TopicFilter) ->
NowMS = erlang:system_time(microsecond), NowMS = erlang:system_time(microsecond),
IteratorId = <<DSSessionId/binary, (emqx_guid:gen())/binary>>, DSSubId = {DSSessionId, TopicFilter},
{IteratorId, NowMS}. {DSSubId, NowMS}.
%%-------------------------------------------------------------------------------- %%--------------------------------------------------------------------------------
@ -593,19 +576,20 @@ transaction(Fun) ->
%%-------------------------------------------------------------------------------- %%--------------------------------------------------------------------------------
export_iterators(IteratorRefs) -> export_subscriptions(DSSubs) ->
lists:foldl( lists:foldl(
fun(IteratorRef = #iterator_ref{ref_id = {_DSSessionId, TopicFilter}}, Acc) -> fun(DSSub = #ds_sub{id = {_DSSessionId, TopicFilter}}, Acc) ->
Acc#{TopicFilter => export_record(IteratorRef)} Acc#{TopicFilter => export_subscription(DSSub)}
end, end,
#{}, #{},
IteratorRefs DSSubs
). ).
export_record(#session{} = Record) -> export_session(#session{} = Record) ->
export_record(Record, #session.id, [id, created_at, expires_at, props], #{}); export_record(Record, #session.id, [id, created_at, expires_at, props], #{}).
export_record(#iterator_ref{} = Record) ->
export_record(Record, #iterator_ref.it_id, [id, start_time, props], #{}). export_subscription(#ds_sub{} = Record) ->
export_record(Record, #ds_sub.start_time, [start_time, props, extra], #{}).
export_record(Record, I, [Field | Rest], Acc) -> export_record(Record, I, [Field | Rest], Acc) ->
export_record(Record, I + 1, Rest, Acc#{Field => element(I, Record)}); export_record(Record, I + 1, Rest, Acc#{Field => element(I, Record)});

66
apps/emqx/test/emqx_persistent_messages_SUITE.erl
View File

@ -29,6 +29,7 @@
-define(DEFAULT_KEYSPACE, default). -define(DEFAULT_KEYSPACE, default).
-define(DS_SHARD_ID, <<"local">>). -define(DS_SHARD_ID, <<"local">>).
-define(DS_SHARD, {?DEFAULT_KEYSPACE, ?DS_SHARD_ID}). -define(DS_SHARD, {?DEFAULT_KEYSPACE, ?DS_SHARD_ID}).
-define(PERSISTENT_MESSAGE_DB, emqx_persistent_message).
all() -> all() ->
emqx_common_test_helpers:all(?MODULE). emqx_common_test_helpers:all(?MODULE).
@ -62,6 +63,7 @@ end_per_testcase(t_session_subscription_iterators, Config) ->
end_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, Config) ->
Apps = ?config(apps, Config), Apps = ?config(apps, Config),
emqx_common_test_helpers:call_janitor(60_000), emqx_common_test_helpers:call_janitor(60_000),
clear_db(),
emqx_cth_suite:stop(Apps), emqx_cth_suite:stop(Apps),
ok. ok.
@ -96,7 +98,7 @@ t_messages_persisted(_Config) ->
ct:pal("Results = ~p", [Results]), ct:pal("Results = ~p", [Results]),
Persisted = consume(?DS_SHARD, {['#'], 0}), Persisted = consume(['#'], 0),
ct:pal("Persisted = ~p", [Persisted]), ct:pal("Persisted = ~p", [Persisted]),
@ -139,7 +141,7 @@ t_messages_persisted_2(_Config) ->
{ok, #{reason_code := ?RC_NO_MATCHING_SUBSCRIBERS}} = {ok, #{reason_code := ?RC_NO_MATCHING_SUBSCRIBERS}} =
emqtt:publish(CP, T(<<"client/2/topic">>), <<"8">>, 1), emqtt:publish(CP, T(<<"client/2/topic">>), <<"8">>, 1),
Persisted = consume(?DS_SHARD, {['#'], 0}), Persisted = consume(['#'], 0),
ct:pal("Persisted = ~p", [Persisted]), ct:pal("Persisted = ~p", [Persisted]),
@ -155,7 +157,7 @@ t_messages_persisted_2(_Config) ->
%% TODO: test quic and ws too %% TODO: test quic and ws too
t_session_subscription_iterators(Config) -> t_session_subscription_iterators(Config) ->
[Node1, Node2] = ?config(nodes, Config), [Node1, _Node2] = ?config(nodes, Config),
Port = get_mqtt_port(Node1, tcp), Port = get_mqtt_port(Node1, tcp),
Topic = <<"t/topic">>, Topic = <<"t/topic">>,
SubTopicFilter = <<"t/+">>, SubTopicFilter = <<"t/+">>,
@ -202,11 +204,8 @@ t_session_subscription_iterators(Config) ->
messages => [Message1, Message2, Message3, Message4] messages => [Message1, Message2, Message3, Message4]
} }
end, end,
fun(Results, Trace) -> fun(Trace) ->
ct:pal("trace:\n ~p", [Trace]), ct:pal("trace:\n ~p", [Trace]),
#{
messages := [_Message1, Message2, Message3 | _]
} = Results,
case ?of_kind(ds_session_subscription_added, Trace) of case ?of_kind(ds_session_subscription_added, Trace) of
[] -> [] ->
%% Since `emqx_durable_storage' is a dependency of `emqx', it gets %% Since `emqx_durable_storage' is a dependency of `emqx', it gets
@ -228,17 +227,6 @@ t_session_subscription_iterators(Config) ->
), ),
ok ok
end, end,
?assertMatch({ok, [_]}, get_all_iterator_ids(Node1)),
{ok, [IteratorId]} = get_all_iterator_ids(Node1),
?assertMatch({ok, [IteratorId]}, get_all_iterator_ids(Node2)),
ReplayMessages1 = erpc:call(Node1, fun() -> consume(?DS_SHARD, IteratorId) end),
ExpectedMessages = [Message2, Message3],
%% Note: it is expected that this will break after replayers are in place.
%% They might have consumed all the messages by this time.
?assertEqual(ExpectedMessages, ReplayMessages1),
%% Different DS shard
ReplayMessages2 = erpc:call(Node2, fun() -> consume(?DS_SHARD, IteratorId) end),
?assertEqual([], ReplayMessages2),
ok ok
end end
), ),
@ -263,33 +251,21 @@ connect(Opts0 = #{}) ->
{ok, _} = emqtt:connect(Client), {ok, _} = emqtt:connect(Client),
Client. Client.
consume(Shard, Replay = {_TopicFiler, _StartMS}) -> consume(TopicFiler, StartMS) ->
{ok, It} = emqx_ds_storage_layer:make_iterator(Shard, Replay), [{_, Stream}] = emqx_ds:get_streams(?PERSISTENT_MESSAGE_DB, TopicFiler, StartMS),
consume(It); {ok, It} = emqx_ds:make_iterator(Stream, StartMS),
consume(Shard, IteratorId) when is_binary(IteratorId) ->
{ok, It} = emqx_ds_storage_layer:restore_iterator(Shard, IteratorId),
consume(It). consume(It).
consume(It) -> consume(It) ->
case emqx_ds_storage_layer:next(It) of case emqx_ds:next(It, 100) of
{ok, NIt, [Msg]} -> {ok, _NIt, _Msgs = []} ->
[emqx_persistent_message:deserialize(Msg) | consume(NIt)]; [];
end_of_stream -> {ok, NIt, Msgs} ->
Msgs ++ consume(NIt);
{ok, end_of_stream} ->
[] []
end. end.
delete_all_messages() ->
Persisted = consume(?DS_SHARD, {['#'], 0}),
lists:foreach(
fun(Msg) ->
GUID = emqx_message:id(Msg),
Topic = emqx_topic:words(emqx_message:topic(Msg)),
Timestamp = emqx_guid:timestamp(GUID),
ok = emqx_ds_storage_layer:delete(?DS_SHARD, GUID, Timestamp, Topic)
end,
Persisted
).
receive_messages(Count) -> receive_messages(Count) ->
receive_messages(Count, []). receive_messages(Count, []).
@ -306,13 +282,6 @@ receive_messages(Count, Msgs) ->
publish(Node, Message) -> publish(Node, Message) ->
erpc:call(Node, emqx, publish, [Message]). erpc:call(Node, emqx, publish, [Message]).
get_iterator_ids(Node, ClientId) ->
Channel = erpc:call(Node, fun() ->
[ConnPid] = emqx_cm:lookup_channels(ClientId),
sys:get_state(ConnPid)
end),
emqx_connection:info({channel, {session, iterators}}, Channel).
app_specs() -> app_specs() ->
[ [
emqx_durable_storage, emqx_durable_storage,
@ -330,5 +299,6 @@ get_mqtt_port(Node, Type) ->
{_IP, Port} = erpc:call(Node, emqx_config, get, [[listeners, Type, default, bind]]), {_IP, Port} = erpc:call(Node, emqx_config, get, [[listeners, Type, default, bind]]),
Port. Port.
get_all_iterator_ids(Node) -> clear_db() ->
erpc:call(Node, emqx_ds_storage_layer, list_iterator_prefix, [?DS_SHARD, <<>>]). ok = emqx_ds:drop_db(?PERSISTENT_MESSAGE_DB),
ok.

10
apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl
View File

@ -48,19 +48,19 @@
%% level. %% level.
%% %%
%% TODO: currently the stream is hardwired to only support the %% TODO: currently the stream is hardwired to only support the
%% internal rocksdb storage. In t he future we want to add another %% internal rocksdb storage. In the future we want to add another
%% implementations for emqx_ds, so this type has to take this into %% implementations for emqx_ds, so this type has to take this into
%% account. %% account.
-record(stream, { -record(stream, {
shard :: emqx_ds_replication_layer:shard_id(), shard :: emqx_ds_replication_layer:shard_id(),
enc :: emqx_ds_replication_layer:stream() enc :: emqx_ds_storage_layer:stream()
}). }).
-opaque stream() :: stream(). -opaque stream() :: #stream{}.
-record(iterator, { -record(iterator, {
shard :: emqx_ds_replication_layer:shard_id(), shard :: emqx_ds_replication_layer:shard_id(),
enc :: enqx_ds_replication_layer:iterator() enc :: enqx_ds_storage_layer:iterator()
}). }).
-opaque iterator() :: #iterator{}. -opaque iterator() :: #iterator{}.
@ -154,7 +154,7 @@ next(Iter0, BatchSize) ->
%% messages on the receiving node, hence saving some network. %% messages on the receiving node, hence saving some network.
%% %%
%% This kind of trickery should be probably done here in the %% This kind of trickery should be probably done here in the
%% replication layer. Or, perhaps, in the logic lary. %% replication layer. Or, perhaps, in the logic layer.
case emqx_ds_proto_v1:next(Node, Shard, StorageIter0, BatchSize) of case emqx_ds_proto_v1:next(Node, Shard, StorageIter0, BatchSize) of
{ok, StorageIter, Batch} -> {ok, StorageIter, Batch} ->
Iter = #iterator{shard = Shard, enc = StorageIter}, Iter = #iterator{shard = Shard, enc = StorageIter},