yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(sessds): make existing parts of persistent session impl work

This commit is contained in:
Andrew Mayorov 2023-09-18 14:49:00 +04:00
parent e713fc38aa
commit 7326ef550b
No known key found for this signature in database
GPG Key ID: 2837C62ACFBFED5D
9 changed files with 278 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/emqx/include/emqx_session.hrl
View File

@ -18,6 +18,6 @@
-define(EMQX_SESSION_HRL, true). -define(EMQX_SESSION_HRL, true).
-define(IS_SESSION_IMPL_MEM(S), (is_tuple(S) andalso element(1, S) =:= session)). -define(IS_SESSION_IMPL_MEM(S), (is_tuple(S) andalso element(1, S) =:= session)).
-define(IS_SESSION_IMPL_DS(S), (is_tuple(S) andalso element(1, S) =:= sessionds)). -define(IS_SESSION_IMPL_DS(S), (is_map_key(id, S))).
-endif. -endif.

7
apps/emqx/integration_test/emqx_ds_SUITE.erl
View File

@ -245,10 +245,9 @@ t_session_subscription_idempotency(Config) ->
?assertEqual([{ClientId, SubTopicFilterWords}], get_all_iterator_refs(Node1)), ?assertEqual([{ClientId, SubTopicFilterWords}], get_all_iterator_refs(Node1)),
?assertMatch({ok, [_]}, get_all_iterator_ids(Node1)), ?assertMatch({ok, [_]}, get_all_iterator_ids(Node1)),
?assertMatch( ?assertMatch(
{_IsNew = false, ClientId}, {_IsNew = false, #{}},
erpc:call(Node1, emqx_ds, session_open, [ClientId]) erpc:call(Node1, emqx_ds, session_open, [ClientId, #{}])
), )
ok
end end
), ),
ok. ok.

5
apps/emqx/src/emqx_cm.erl
View File

@ -281,8 +281,9 @@ open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) ->
{true, Session, ReplayContext} -> {true, Session, ReplayContext} ->
ok = register_channel(ClientId, Self, ConnInfo), ok = register_channel(ClientId, Self, ConnInfo),
{ok, #{session => Session, present => true, replay => ReplayContext}}; {ok, #{session => Session, present => true, replay => ReplayContext}};
false -> {false, Session} ->
create_register_session(ClientInfo, ConnInfo, Self) ok = register_channel(ClientId, Self, ConnInfo),
{ok, #{session => Session, present => false}}
end end
end). end).

214
apps/emqx/src/emqx_persistent_session_ds.erl
View File

@ -24,7 +24,7 @@
%% Session API %% Session API
-export([ -export([
create/3, create/3,
open/2, open/3,
destroy/1 destroy/1
]). ]).
@ -49,6 +49,7 @@
-export([ -export([
deliver/3, deliver/3,
replay/3,
% handle_timeout/3, % handle_timeout/3,
disconnect/1, disconnect/1,
terminate/2 terminate/2
@ -70,20 +71,25 @@
-define(DEFAULT_KEYSPACE, default). -define(DEFAULT_KEYSPACE, default).
-define(DS_SHARD, {?DEFAULT_KEYSPACE, ?DS_SHARD_ID}). -define(DS_SHARD, {?DEFAULT_KEYSPACE, ?DS_SHARD_ID}).
-record(sessionds, { -type id() :: emqx_ds:session_id().
-type iterator() :: emqx_ds:iterator().
-type session() :: #{
%% Client ID %% Client ID
id :: binary(), id := id(),
%% When the session was created
created_at := timestamp(),
%% When the session should expire
expires_at := timestamp() | never,
%% Client’s Subscriptions. %% Client’s Subscriptions.
subscriptions :: map(), iterators := #{topic() => iterator()},
iterators :: map(),
%% %%
conf props := map()
}). }.
-type session() :: #sessionds{}.
-type timestamp() :: emqx_utils_calendar:epoch_millisecond().
-type topic() :: emqx_types:topic().
-type clientinfo() :: emqx_types:clientinfo(). -type clientinfo() :: emqx_types:clientinfo().
-type conninfo() :: emqx_types:conninfo(). -type conninfo() :: emqx_session:conninfo().
-type replies() :: emqx_session:replies(). -type replies() :: emqx_session:replies().
%% %%
@ -91,18 +97,31 @@
-spec create(clientinfo(), conninfo(), emqx_session:conf()) -> -spec create(clientinfo(), conninfo(), emqx_session:conf()) ->
session(). session().
create(#{clientid := ClientID}, _ConnInfo, Conf) -> create(#{clientid := ClientID}, _ConnInfo, Conf) ->
#sessionds{ % TODO: expiration
id = ClientID, {true, Session} = emqx_ds:session_open(ClientID, Conf),
subscriptions = #{}, Session.
conf = Conf
}.
-spec open(clientinfo(), conninfo()) -> -spec open(clientinfo(), conninfo(), emqx_session:conf()) ->
{true, session()} | false. {true, session(), []} | {false, session()}.
open(#{clientid := ClientID}, _ConnInfo) -> open(#{clientid := ClientID}, _ConnInfo, Conf) ->
open_session(ClientID). % NOTE
% The fact that we need to concern about discarding all live channels here
% is essentially a consequence of the in-memory session design, where we
% have disconnected channels holding onto session state. Ideally, we should
% somehow isolate those idling not-yet-expired sessions into a separate process
% space, and move this call back into `emqx_cm` where it belongs.
ok = emqx_cm:discard_session(ClientID),
{IsNew, Session} = emqx_ds:session_open(ClientID, Conf),
case IsNew of
false ->
{true, Session, []};
true ->
{false, Session}
end.
-spec destroy(session() | clientinfo()) -> ok. -spec destroy(session() | clientinfo()) -> ok.
destroy(#{id := ClientID}) ->
emqx_ds:session_drop(ClientID);
destroy(#{clientid := ClientID}) -> destroy(#{clientid := ClientID}) ->
emqx_ds:session_drop(ClientID). emqx_ds:session_drop(ClientID).
@ -112,21 +131,21 @@ destroy(#{clientid := ClientID}) ->
info(Keys, Session) when is_list(Keys) -> info(Keys, Session) when is_list(Keys) ->
[{Key, info(Key, Session)} || Key <- Keys]; [{Key, info(Key, Session)} || Key <- Keys];
info(id, #sessionds{id = ClientID}) -> info(id, #{id := ClientID}) ->
ClientID; ClientID;
info(clientid, #sessionds{id = ClientID}) -> info(clientid, #{id := ClientID}) ->
ClientID; ClientID;
% info(created_at, #sessionds{created_at = CreatedAt}) -> info(created_at, #{created_at := CreatedAt}) ->
% CreatedAt; CreatedAt;
info(is_persistent, #sessionds{}) -> info(is_persistent, #{}) ->
true; true;
info(subscriptions, #sessionds{subscriptions = Subs}) -> info(subscriptions, #{iterators := Iters}) ->
Subs; maps:map(fun(_, #{props := SubOpts}) -> SubOpts end, Iters);
info(subscriptions_cnt, #sessionds{subscriptions = Subs}) -> info(subscriptions_cnt, #{iterators := Iters}) ->
maps:size(Subs); maps:size(Iters);
info(subscriptions_max, #sessionds{conf = Conf}) -> info(subscriptions_max, #{props := Conf}) ->
maps:get(max_subscriptions, Conf); maps:get(max_subscriptions, Conf);
info(upgrade_qos, #sessionds{conf = Conf}) -> info(upgrade_qos, #{props := Conf}) ->
maps:get(upgrade_qos, Conf); maps:get(upgrade_qos, Conf);
% info(inflight, #sessmem{inflight = Inflight}) -> % info(inflight, #sessmem{inflight = Inflight}) ->
% Inflight; % Inflight;
@ -134,7 +153,7 @@ info(upgrade_qos, #sessionds{conf = Conf}) ->
% emqx_inflight:size(Inflight); % emqx_inflight:size(Inflight);
% info(inflight_max, #sessmem{inflight = Inflight}) -> % info(inflight_max, #sessmem{inflight = Inflight}) ->
% emqx_inflight:max_size(Inflight); % emqx_inflight:max_size(Inflight);
info(retry_interval, #sessionds{conf = Conf}) -> info(retry_interval, #{props := Conf}) ->
maps:get(retry_interval, Conf); maps:get(retry_interval, Conf);
% info(mqueue, #sessmem{mqueue = MQueue}) -> % info(mqueue, #sessmem{mqueue = MQueue}) ->
% MQueue; % MQueue;
@ -144,15 +163,15 @@ info(retry_interval, #sessionds{conf = Conf}) ->
% emqx_mqueue:max_len(MQueue); % emqx_mqueue:max_len(MQueue);
% info(mqueue_dropped, #sessmem{mqueue = MQueue}) -> % info(mqueue_dropped, #sessmem{mqueue = MQueue}) ->
% emqx_mqueue:dropped(MQueue); % emqx_mqueue:dropped(MQueue);
info(next_pkt_id, #sessionds{}) -> info(next_pkt_id, #{}) ->
_PacketId = 'TODO'; _PacketId = 'TODO';
% info(awaiting_rel, #sessmem{awaiting_rel = AwaitingRel}) -> % info(awaiting_rel, #sessmem{awaiting_rel = AwaitingRel}) ->
% AwaitingRel; % AwaitingRel;
% info(awaiting_rel_cnt, #sessmem{awaiting_rel = AwaitingRel}) -> % info(awaiting_rel_cnt, #sessmem{awaiting_rel = AwaitingRel}) ->
% maps:size(AwaitingRel); % maps:size(AwaitingRel);
info(awaiting_rel_max, #sessionds{conf = Conf}) -> info(awaiting_rel_max, #{props := Conf}) ->
maps:get(max_awaiting_rel, Conf); maps:get(max_awaiting_rel, Conf);
info(await_rel_timeout, #sessionds{conf = Conf}) -> info(await_rel_timeout, #{props := Conf}) ->
maps:get(await_rel_timeout, Conf). maps:get(await_rel_timeout, Conf).
-spec stats(session()) -> emqx_types:stats(). -spec stats(session()) -> emqx_types:stats().
@ -164,50 +183,50 @@ stats(Session) ->
%% Client -> Broker: SUBSCRIBE / UNSUBSCRIBE %% Client -> Broker: SUBSCRIBE / UNSUBSCRIBE
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-spec subscribe(emqx_types:topic(), emqx_types:subopts(), session()) -> -spec subscribe(topic(), emqx_types:subopts(), session()) ->
{ok, session()} | {error, emqx_types:reason_code()}. {ok, session()} | {error, emqx_types:reason_code()}.
subscribe( subscribe(
TopicFilter, TopicFilter,
SubOpts, SubOpts,
Session = #sessionds{subscriptions = Subs} Session = #{id := ID, iterators := Iters}
) when is_map_key(TopicFilter, Subs) -> ) when is_map_key(TopicFilter, Iters) ->
{ok, Session#sessionds{ Iterator = maps:get(TopicFilter, Iters),
subscriptions = Subs#{TopicFilter => SubOpts} NIterator = update_subscription(TopicFilter, Iterator, SubOpts, ID),
}}; {ok, Session#{iterators := Iters#{TopicFilter => NIterator}}};
subscribe( subscribe(
TopicFilter, TopicFilter,
SubOpts, SubOpts,
Session = #sessionds{id = ClientID, subscriptions = Subs, iterators = Iters} Session = #{id := ID, iterators := Iters}
) -> ) ->
% TODO: max_subscriptions % TODO: max_subscriptions
IteratorID = add_subscription(TopicFilter, ClientID), Iterator = add_subscription(TopicFilter, SubOpts, ID),
{ok, Session#sessionds{ {ok, Session#{iterators := Iters#{TopicFilter => Iterator}}}.
subscriptions = Subs#{TopicFilter => SubOpts},
iterators = Iters#{TopicFilter => IteratorID}
}}.
-spec unsubscribe(emqx_types:topic(), session()) -> -spec unsubscribe(topic(), session()) ->
{ok, session(), emqx_types:subopts()} | {error, emqx_types:reason_code()}. {ok, session(), emqx_types:subopts()} | {error, emqx_types:reason_code()}.
unsubscribe( unsubscribe(
TopicFilter, TopicFilter,
Session = #sessionds{id = ClientID, subscriptions = Subs, iterators = Iters} Session = #{id := ID, iterators := Iters}
) when is_map_key(TopicFilter, Subs) -> ) when is_map_key(TopicFilter, Iters) ->
IteratorID = maps:get(TopicFilter, Iters), Iterator = maps:get(TopicFilter, Iters),
ok = del_subscription(IteratorID, TopicFilter, ClientID), SubOpts = maps:get(props, Iterator),
{ok, Session#sessionds{ ok = del_subscription(TopicFilter, Iterator, ID),
subscriptions = maps:remove(TopicFilter, Subs), {ok, Session#{iterators := maps:remove(TopicFilter, Iters)}, SubOpts};
iterators = maps:remove(TopicFilter, Iters)
}};
unsubscribe( unsubscribe(
_TopicFilter, _TopicFilter,
_Session = #sessionds{} _Session = #{}
) -> ) ->
{error, ?RC_NO_SUBSCRIPTION_EXISTED}. {error, ?RC_NO_SUBSCRIPTION_EXISTED}.
-spec get_subscription(emqx_types:topic(), session()) -> -spec get_subscription(emqx_types:topic(), session()) ->
emqx_types:subopts() | undefined. emqx_types:subopts() | undefined.
get_subscription(TopicFilter, #sessionds{subscriptions = Subs}) -> get_subscription(TopicFilter, #{iterators := Iters}) ->
maps:get(TopicFilter, Subs, undefined). case maps:get(TopicFilter, Iters, undefined) of
Iterator = #{} ->
maps:get(props, Iterator);
undefined ->
undefined
end.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% Client -> Broker: PUBLISH %% Client -> Broker: PUBLISH
@ -227,7 +246,7 @@ publish(_PacketId, Msg, Session) ->
-spec puback(clientinfo(), emqx_types:packet_id(), session()) -> -spec puback(clientinfo(), emqx_types:packet_id(), session()) ->
{ok, emqx_types:message(), replies(), session()} {ok, emqx_types:message(), replies(), session()}
| {error, emqx_types:reason_code()}. | {error, emqx_types:reason_code()}.
puback(_ClientInfo, _PacketId, _Session = #sessionds{}) -> puback(_ClientInfo, _PacketId, _Session = #{}) ->
% TODO: stub % TODO: stub
{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}. {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}.
@ -238,7 +257,7 @@ puback(_ClientInfo, _PacketId, _Session = #sessionds{}) ->
-spec pubrec(emqx_types:packet_id(), session()) -> -spec pubrec(emqx_types:packet_id(), session()) ->
{ok, emqx_types:message(), session()} {ok, emqx_types:message(), session()}
| {error, emqx_types:reason_code()}. | {error, emqx_types:reason_code()}.
pubrec(_PacketId, _Session = #sessionds{}) -> pubrec(_PacketId, _Session = #{}) ->
% TODO: stub % TODO: stub
{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}. {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}.
@ -248,7 +267,7 @@ pubrec(_PacketId, _Session = #sessionds{}) ->
-spec pubrel(emqx_types:packet_id(), session()) -> -spec pubrel(emqx_types:packet_id(), session()) ->
{ok, session()} | {error, emqx_types:reason_code()}. {ok, session()} | {error, emqx_types:reason_code()}.
pubrel(_PacketId, Session = #sessionds{}) -> pubrel(_PacketId, Session = #{}) ->
% TODO: stub % TODO: stub
{ok, Session}. {ok, Session}.
@ -259,37 +278,39 @@ pubrel(_PacketId, Session = #sessionds{}) ->
-spec pubcomp(clientinfo(), emqx_types:packet_id(), session()) -> -spec pubcomp(clientinfo(), emqx_types:packet_id(), session()) ->
{ok, emqx_types:message(), replies(), session()} {ok, emqx_types:message(), replies(), session()}
| {error, emqx_types:reason_code()}. | {error, emqx_types:reason_code()}.
pubcomp(_ClientInfo, _PacketId, _Session = #sessionds{}) -> pubcomp(_ClientInfo, _PacketId, _Session = #{}) ->
% TODO: stub % TODO: stub
{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}. {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-spec deliver(clientinfo(), [emqx_types:deliver()], session()) -> -spec deliver(clientinfo(), [emqx_types:deliver()], session()) ->
{ok, replies(), session()}. no_return().
deliver(_ClientInfo, _Delivers, _Session = #sessionds{}) -> deliver(_ClientInfo, _Delivers, _Session = #{}) ->
% TODO: ensure it's unreachable somehow % TODO: ensure it's unreachable somehow
error(unexpected). error(unexpected).
-spec replay(clientinfo(), [], session()) ->
{ok, replies(), session()}.
replay(_ClientInfo, [], Session = #{}) ->
{ok, [], Session}.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-spec disconnect(session()) -> {shutdown, session()}. -spec disconnect(session()) -> {shutdown, session()}.
disconnect(Session = #sessionds{}) -> disconnect(Session = #{}) ->
{shutdown, Session}. {shutdown, Session}.
-spec terminate(Reason :: term(), session()) -> ok. -spec terminate(Reason :: term(), session()) -> ok.
terminate(_Reason, _Session = #sessionds{}) -> terminate(_Reason, _Session = #{}) ->
% TODO: close iterators % TODO: close iterators
ok. ok.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
open_session(ClientID) -> -spec add_subscription(topic(), emqx_types:subopts(), id()) ->
emqx_ds:session_open(ClientID). emqx_ds:iterator().
add_subscription(TopicFilterBin, SubOpts, DSSessionID) ->
-spec add_subscription(emqx_types:topic(), emqx_ds:session_id()) ->
emqx_ds:iterator_id().
add_subscription(TopicFilterBin, DSSessionID) ->
% N.B.: we chose to update the router before adding the subscription to the % N.B.: we chose to update the router before adding the subscription to the
% session/iterator table. The reasoning for this is as follows: % session/iterator table. The reasoning for this is as follows:
% %
@ -310,32 +331,38 @@ add_subscription(TopicFilterBin, DSSessionID) ->
% and iterator information can be reconstructed from this table, if needed. % and iterator information can be reconstructed from this table, if needed.
ok = emqx_persistent_session_ds_router:do_add_route(TopicFilterBin, DSSessionID), ok = emqx_persistent_session_ds_router:do_add_route(TopicFilterBin, DSSessionID),
TopicFilter = emqx_topic:words(TopicFilterBin), TopicFilter = emqx_topic:words(TopicFilterBin),
{ok, IteratorID, StartMS, IsNew} = emqx_ds:session_add_iterator( {ok, Iterator, IsNew} = emqx_ds:session_add_iterator(
DSSessionID, TopicFilter DSSessionID, TopicFilter, SubOpts
), ),
Ctx = #{ Ctx = #{iterator => Iterator, is_new => IsNew},
iterator_id => IteratorID,
start_time => StartMS,
is_new => IsNew
},
?tp(persistent_session_ds_iterator_added, Ctx), ?tp(persistent_session_ds_iterator_added, Ctx),
?tp_span( ?tp_span(
persistent_session_ds_open_iterators, persistent_session_ds_open_iterators,
Ctx, Ctx,
ok = open_iterator_on_all_shards(TopicFilter, StartMS, IteratorID) ok = open_iterator_on_all_shards(TopicFilter, Iterator)
), ),
IteratorID. Iterator.
-spec open_iterator_on_all_shards(emqx_topic:words(), emqx_ds:time(), emqx_ds:iterator_id()) -> ok. -spec update_subscription(topic(), iterator(), emqx_types:subopts(), id()) ->
open_iterator_on_all_shards(TopicFilter, StartMS, IteratorID) -> iterator().
?tp(persistent_session_ds_will_open_iterators, #{ update_subscription(TopicFilterBin, Iterator, SubOpts, DSSessionID) ->
iterator_id => IteratorID, TopicFilter = emqx_topic:words(TopicFilterBin),
start_time => StartMS {ok, NIterator, false} = emqx_ds:session_add_iterator(
}), DSSessionID, TopicFilter, SubOpts
),
ok = ?tp(persistent_session_ds_iterator_updated, #{iterator => Iterator}),
NIterator.
-spec open_iterator_on_all_shards(emqx_topic:words(), emqx_ds:iterator()) -> ok.
open_iterator_on_all_shards(TopicFilter, Iterator) ->
?tp(persistent_session_ds_will_open_iterators, #{iterator => Iterator}),
%% Note: currently, shards map 1:1 to nodes, but this will change in the future. %% Note: currently, shards map 1:1 to nodes, but this will change in the future.
Nodes = emqx:running_nodes(), Nodes = emqx:running_nodes(),
Results = emqx_persistent_session_ds_proto_v1:open_iterator( Results = emqx_persistent_session_ds_proto_v1:open_iterator(
Nodes, TopicFilter, StartMS, IteratorID Nodes,
TopicFilter,
maps:get(start_time, Iterator),
maps:get(id, Iterator)
), ),
%% TODO %% TODO
%% 1. Handle errors. %% 1. Handle errors.
@ -346,14 +373,15 @@ open_iterator_on_all_shards(TopicFilter, StartMS, IteratorID) ->
ok. ok.
%% RPC target. %% RPC target.
-spec do_open_iterator(emqx_topic:words(), emqx_ds:time(), emqx_ds:iterator_id()) -> ok. -spec do_open_iterator(emqx_topic:words(), emqx_ds:time(), emqx_ds:iterator_id()) ->
{ok, emqx_ds_storage_layer:iterator()} | {error, _Reason}.
do_open_iterator(TopicFilter, StartMS, IteratorID) -> do_open_iterator(TopicFilter, StartMS, IteratorID) ->
Replay = {TopicFilter, StartMS}, Replay = {TopicFilter, StartMS},
emqx_ds_storage_layer:ensure_iterator(?DS_SHARD, IteratorID, Replay). emqx_ds_storage_layer:ensure_iterator(?DS_SHARD, IteratorID, Replay).
-spec del_subscription(emqx_ds:iterator_id() | undefined, emqx_types:topic(), emqx_ds:session_id()) -> -spec del_subscription(topic(), iterator(), id()) ->
ok. ok.
del_subscription(IteratorID, TopicFilterBin, DSSessionID) -> del_subscription(TopicFilterBin, #{id := IteratorID}, DSSessionID) ->
% N.B.: see comments in `?MODULE:add_subscription' for a discussion about the % N.B.: see comments in `?MODULE:add_subscription' for a discussion about the
% order of operations here. % order of operations here.
TopicFilter = emqx_topic:words(TopicFilterBin), TopicFilter = emqx_topic:words(TopicFilterBin),
@ -385,7 +413,7 @@ do_ensure_iterator_closed(IteratorID) ->
ok = emqx_ds_storage_layer:discard_iterator(?DS_SHARD, IteratorID), ok = emqx_ds_storage_layer:discard_iterator(?DS_SHARD, IteratorID),
ok. ok.
-spec ensure_all_iterators_closed(emqx_ds:session_id()) -> ok. -spec ensure_all_iterators_closed(id()) -> ok.
ensure_all_iterators_closed(DSSessionID) -> ensure_all_iterators_closed(DSSessionID) ->
%% Note: currently, shards map 1:1 to nodes, but this will change in the future. %% Note: currently, shards map 1:1 to nodes, but this will change in the future.
Nodes = emqx:running_nodes(), Nodes = emqx:running_nodes(),
@ -395,7 +423,7 @@ ensure_all_iterators_closed(DSSessionID) ->
ok. ok.
%% RPC target. %% RPC target.
-spec do_ensure_all_iterators_closed(emqx_ds:session_id()) -> ok. -spec do_ensure_all_iterators_closed(id()) -> ok.
do_ensure_all_iterators_closed(DSSessionID) -> do_ensure_all_iterators_closed(DSSessionID) ->
ok = emqx_ds_storage_layer:discard_iterator_prefix(?DS_SHARD, DSSessionID), ok = emqx_ds_storage_layer:discard_iterator_prefix(?DS_SHARD, DSSessionID),
ok. ok.

17
apps/emqx/src/emqx_session.erl
View File

@ -163,15 +163,28 @@
-spec create(clientinfo(), conninfo()) -> t(). -spec create(clientinfo(), conninfo()) -> t().
create(ClientInfo, ConnInfo) -> create(ClientInfo, ConnInfo) ->
Conf = get_session_conf(ClientInfo, ConnInfo), Conf = get_session_conf(ClientInfo, ConnInfo),
create(ClientInfo, ConnInfo, Conf).
create(ClientInfo, ConnInfo, Conf) ->
% FIXME error conditions % FIXME error conditions
Session = (choose_impl_mod(ConnInfo)):create(ClientInfo, ConnInfo, Conf), Session = (choose_impl_mod(ConnInfo)):create(ClientInfo, ConnInfo, Conf),
ok = emqx_metrics:inc('session.created'), ok = emqx_metrics:inc('session.created'),
ok = emqx_hooks:run('session.created', [ClientInfo, info(Session)]), ok = emqx_hooks:run('session.created', [ClientInfo, info(Session)]),
Session. Session.
-spec open(clientinfo(), conninfo()) -> {true, t(), _ReplayContext} | false. -spec open(clientinfo(), conninfo()) -> {true, t(), _ReplayContext} | {false, t()}.
open(ClientInfo, ConnInfo) -> open(ClientInfo, ConnInfo) ->
(choose_impl_mod(ConnInfo)):open(ClientInfo, ConnInfo). Conf = get_session_conf(ClientInfo, ConnInfo),
case (choose_impl_mod(ConnInfo)):open(ClientInfo, ConnInfo, Conf) of
{true, Session, ReplayContext} ->
{true, Session, ReplayContext};
{false, Session} ->
ok = emqx_metrics:inc('session.created'),
ok = emqx_hooks:run('session.created', [ClientInfo, info(Session)]),
{false, Session};
false ->
{false, create(ClientInfo, ConnInfo, Conf)}
end.
-spec get_session_conf(clientinfo(), conninfo()) -> conf(). -spec get_session_conf(clientinfo(), conninfo()) -> conf().
get_session_conf( get_session_conf(

6
apps/emqx/src/emqx_session_mem.erl
View File

@ -57,7 +57,7 @@
-export([ -export([
create/3, create/3,
open/2, open/3,
destroy/1 destroy/1
]). ]).
@ -193,9 +193,9 @@ destroy(_Session) ->
%% Open a (possibly existing) Session %% Open a (possibly existing) Session
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-spec open(clientinfo(), emqx_types:conninfo()) -> -spec open(clientinfo(), conninfo(), emqx_session:conf()) ->
{true, session(), replayctx()} | false. {true, session(), replayctx()} | false.
open(ClientInfo = #{clientid := ClientId}, _ConnInfo) -> open(ClientInfo = #{clientid := ClientId}, _ConnInfo, _Conf) ->
case emqx_cm:takeover_session_begin(ClientId) of case emqx_cm:takeover_session_begin(ClientId) of
{ok, SessionRemote, TakeoverState} -> {ok, SessionRemote, TakeoverState} ->
Session = resume(ClientInfo, SessionRemote), Session = resume(ClientInfo, SessionRemote),

5
apps/emqx/test/emqx_persistent_messages_SUITE.erl
View File

@ -186,13 +186,14 @@ t_session_subscription_iterators(Config) ->
ct:pal("publishing 2"), ct:pal("publishing 2"),
Message2 = emqx_message:make(Topic, Payload2), Message2 = emqx_message:make(Topic, Payload2),
publish(Node1, Message2), publish(Node1, Message2),
[_] = receive_messages(1), % TODO: no incoming publishes at the moment
% [_] = receive_messages(1),
ct:pal("subscribing 2"), ct:pal("subscribing 2"),
{ok, _, [1]} = emqtt:subscribe(Client, SubTopicFilter, qos1), {ok, _, [1]} = emqtt:subscribe(Client, SubTopicFilter, qos1),
ct:pal("publishing 3"), ct:pal("publishing 3"),
Message3 = emqx_message:make(Topic, Payload3), Message3 = emqx_message:make(Topic, Payload3),
publish(Node1, Message3), publish(Node1, Message3),
[_] = receive_messages(1), % [_] = receive_messages(1),
ct:pal("publishing 4"), ct:pal("publishing 4"),
Message4 = emqx_message:make(AnotherTopic, Payload4), Message4 = emqx_message:make(AnotherTopic, Payload4),
publish(Node1, Message4), publish(Node1, Message4),

159
apps/emqx_durable_storage/src/emqx_ds.erl
View File

@ -26,10 +26,10 @@
-export([iterator_update/2, iterator_next/1, iterator_stats/0]). -export([iterator_update/2, iterator_next/1, iterator_stats/0]).
%% Session: %% Session:
-export([ -export([
session_open/1, session_open/2,
session_drop/1, session_drop/1,
session_suspend/1, session_suspend/1,
session_add_iterator/2, session_add_iterator/3,
session_get_iterator_id/2, session_get_iterator_id/2,
session_del_iterator/2, session_del_iterator/2,
session_stats/0 session_stats/0
@ -60,6 +60,16 @@
%% Type declarations %% Type declarations
%%================================================================================ %%================================================================================
%% Session
%% See also: `#session{}`.
-type session() :: #{
id := emqx_ds:session_id(),
created_at := _Millisecond :: non_neg_integer(),
expires_at := _Millisecond :: non_neg_integer() | never,
iterators := map(),
props := map()
}.
%% Currently, this is the clientid. We avoid `emqx_types:clientid()' because that can be %% Currently, this is the clientid. We avoid `emqx_types:clientid()' because that can be
%% an atom, in theory (?). %% an atom, in theory (?).
-type session_id() :: binary(). -type session_id() :: binary().
@ -141,33 +151,41 @@ message_stats() ->
%% %%
%% Note: session API doesn't handle session takeovers, it's the job of %% Note: session API doesn't handle session takeovers, it's the job of
%% the broker. %% the broker.
-spec session_open(emqx_types:clientid()) -> {_New :: boolean(), session_id()}. -spec session_open(session_id(), _Props :: map()) -> {_New :: boolean(), session()}.
session_open(ClientID) -> session_open(SessionId, Props) ->
{atomic, Res} = transaction(fun() ->
mria:transaction(?DS_MRIA_SHARD, fun() -> case mnesia:read(?SESSION_TAB, SessionId, write) of
case mnesia:read(?SESSION_TAB, ClientID, write) of [Record = #session{}] ->
[#session{}] -> Session = export_record(Record),
{false, ClientID}; IteratorRefs = session_read_iterators(SessionId),
Iterators = export_iterators(IteratorRefs),
{false, Session#{iterators => Iterators}};
[] -> [] ->
Session = #session{id = ClientID}, Session = export_record(session_create(SessionId, Props)),
mnesia:write(?SESSION_TAB, Session, write), {true, Session#{iterators => #{}}}
{true, ClientID}
end end
end), end).
Res.
session_create(SessionId, Props) ->
Session = #session{
id = SessionId,
created_at = erlang:system_time(millisecond),
expires_at = never,
props = Props
},
ok = mnesia:write(?SESSION_TAB, Session, write),
Session.
%% @doc Called when a client reconnects with `clean session=true' or %% @doc Called when a client reconnects with `clean session=true' or
%% during session GC %% during session GC
-spec session_drop(emqx_types:clientid()) -> ok. -spec session_drop(session_id()) -> ok.
session_drop(ClientID) -> session_drop(DSSessionId) ->
{atomic, ok} = mria:transaction( transaction(fun() ->
?DS_MRIA_SHARD,
fun() ->
%% TODO: ensure all iterators from this clientid are closed? %% TODO: ensure all iterators from this clientid are closed?
mnesia:delete({?SESSION_TAB, ClientID}) IteratorRefs = session_read_iterators(DSSessionId),
end ok = lists:foreach(fun session_del_iterator/1, IteratorRefs),
), ok = mnesia:delete(?SESSION_TAB, DSSessionId, write)
ok. end).
%% @doc Called when a client disconnects. This function terminates all %% @doc Called when a client disconnects. This function terminates all
%% active processes related to the session. %% active processes related to the session.
@ -177,37 +195,46 @@ session_suspend(_SessionId) ->
ok. ok.
%% @doc Called when a client subscribes to a topic. Idempotent. %% @doc Called when a client subscribes to a topic. Idempotent.
-spec session_add_iterator(session_id(), emqx_topic:words()) -> -spec session_add_iterator(session_id(), emqx_topic:words(), _Props :: map()) ->
{ok, iterator_id(), time(), _IsNew :: boolean()}. {ok, iterator(), _IsNew :: boolean()}.
session_add_iterator(DSSessionId, TopicFilter) -> session_add_iterator(DSSessionId, TopicFilter, Props) ->
IteratorRefId = {DSSessionId, TopicFilter}, IteratorRefId = {DSSessionId, TopicFilter},
{atomic, Res} = transaction(fun() ->
mria:transaction(?DS_MRIA_SHARD, fun() ->
case mnesia:read(?ITERATOR_REF_TAB, IteratorRefId, write) of case mnesia:read(?ITERATOR_REF_TAB, IteratorRefId, write) of
[] -> [] ->
{IteratorId, StartMS} = new_iterator_id(DSSessionId), IteratorRef = session_insert_iterator(DSSessionId, TopicFilter, Props),
IteratorRef = #iterator_ref{ Iterator = export_record(IteratorRef),
ref_id = IteratorRefId,
it_id = IteratorId,
start_time = StartMS
},
ok = mnesia:write(?ITERATOR_REF_TAB, IteratorRef, write),
?tp( ?tp(
ds_session_subscription_added, ds_session_subscription_added,
#{iterator_id => IteratorId, session_id => DSSessionId} #{iterator => Iterator, session_id => DSSessionId}
), ),
IsNew = true, {ok, Iterator, _IsNew = true};
{ok, IteratorId, StartMS, IsNew}; [#iterator_ref{} = IteratorRef] ->
[#iterator_ref{it_id = IteratorId, start_time = StartMS}] -> NIteratorRef = session_update_iterator(IteratorRef, Props),
NIterator = export_record(NIteratorRef),
?tp( ?tp(
ds_session_subscription_present, ds_session_subscription_present,
#{iterator_id => IteratorId, session_id => DSSessionId} #{iterator => NIterator, session_id => DSSessionId}
), ),
IsNew = false, {ok, NIterator, _IsNew = false}
{ok, IteratorId, StartMS, IsNew}
end end
end), end).
Res.
session_insert_iterator(DSSessionId, TopicFilter, Props) ->
{IteratorId, StartMS} = new_iterator_id(DSSessionId),
IteratorRef = #iterator_ref{
ref_id = {DSSessionId, TopicFilter},
it_id = IteratorId,
start_time = StartMS,
props = Props
},
ok = mnesia:write(?ITERATOR_REF_TAB, IteratorRef, write),
IteratorRef.
session_update_iterator(IteratorRef, Props) ->
NIteratorRef = IteratorRef#iterator_ref{props = Props},
ok = mnesia:write(?ITERATOR_REF_TAB, NIteratorRef, write),
NIteratorRef.
-spec session_get_iterator_id(session_id(), emqx_topic:words()) -> -spec session_get_iterator_id(session_id(), emqx_topic:words()) ->
{ok, iterator_id()} | {error, not_found}. {ok, iterator_id()} | {error, not_found}.
@ -224,11 +251,20 @@ session_get_iterator_id(DSSessionId, TopicFilter) ->
-spec session_del_iterator(session_id(), emqx_topic:words()) -> ok. -spec session_del_iterator(session_id(), emqx_topic:words()) -> ok.
session_del_iterator(DSSessionId, TopicFilter) -> session_del_iterator(DSSessionId, TopicFilter) ->
IteratorRefId = {DSSessionId, TopicFilter}, IteratorRefId = {DSSessionId, TopicFilter},
{atomic, ok} = transaction(fun() ->
mria:transaction(?DS_MRIA_SHARD, fun() ->
mnesia:delete(?ITERATOR_REF_TAB, IteratorRefId, write) mnesia:delete(?ITERATOR_REF_TAB, IteratorRefId, write)
end), end).
ok.
session_read_iterators(DSSessionId) ->
% NOTE: somewhat convoluted way to trick dialyzer
Pat = erlang:make_tuple(record_info(size, iterator_ref), '_', [
{1, iterator_ref},
{#iterator_ref.ref_id, {DSSessionId, '_'}}
]),
mnesia:match_object(?ITERATOR_REF_TAB, Pat, read).
session_del_iterator(#iterator_ref{ref_id = IteratorRefId}) ->
mnesia:delete(?ITERATOR_REF_TAB, IteratorRefId, write).
-spec session_stats() -> #{}. -spec session_stats() -> #{}.
session_stats() -> session_stats() ->
@ -263,3 +299,30 @@ new_iterator_id(DSSessionId) ->
NowMS = erlang:system_time(microsecond), NowMS = erlang:system_time(microsecond),
IteratorId = <<DSSessionId/binary, (emqx_guid:gen())/binary>>, IteratorId = <<DSSessionId/binary, (emqx_guid:gen())/binary>>,
{IteratorId, NowMS}. {IteratorId, NowMS}.
%%--------------------------------------------------------------------------------
transaction(Fun) ->
{atomic, Res} = mria:transaction(?DS_MRIA_SHARD, Fun),
Res.
%%--------------------------------------------------------------------------------
export_iterators(IteratorRefs) ->
lists:foldl(
fun(IteratorRef = #iterator_ref{ref_id = {_DSSessionId, TopicFilter}}, Acc) ->
Acc#{TopicFilter => export_record(IteratorRef)}
end,
#{},
IteratorRefs
).
export_record(#session{} = Record) ->
export_record(Record, #session.id, [id, created_at, expires_at, props], #{});
export_record(#iterator_ref{} = Record) ->
export_record(Record, #iterator_ref.it_id, [id, start_time, props], #{}).
export_record(Record, I, [Field | Rest], Acc) ->
export_record(Record, I + 1, Rest, Acc#{Field => element(I, Record)});
export_record(_, _, [], Acc) ->
Acc.

6
apps/emqx_durable_storage/src/emqx_ds_int.hrl
View File

@ -23,6 +23,9 @@
-record(session, { -record(session, {
%% same as clientid %% same as clientid
id :: emqx_ds:session_id(), id :: emqx_ds:session_id(),
%% creation time
created_at :: _Millisecond :: non_neg_integer(),
expires_at = never :: _Millisecond :: non_neg_integer() | never,
%% for future usage %% for future usage
props = #{} :: map() props = #{} :: map()
}). }).
@ -30,7 +33,8 @@
-record(iterator_ref, { -record(iterator_ref, {
ref_id :: {emqx_ds:session_id(), emqx_topic:words()}, ref_id :: {emqx_ds:session_id(), emqx_topic:words()},
it_id :: emqx_ds:iterator_id(), it_id :: emqx_ds:iterator_id(),
start_time :: emqx_ds:time() start_time :: emqx_ds:time(),
props = #{} :: map()
}). }).
-endif. -endif.