yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(dsrepl): Don't reverse elements of batches

This commit is contained in:
ieQu1 2024-04-23 01:01:11 +02:00
parent bcfa7b2209
commit 86d45522e3
No known key found for this signature in database
GPG Key ID: 488654DF3FED6FDE
1 changed files with 4 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl
View File

@ -672,22 +672,15 @@ apply(
?tag := ?BATCH, ?tag := ?BATCH,
?batch_messages := MessagesIn ?batch_messages := MessagesIn
}, },
#{db_shard := DBShard, latest := Latest} = State #{db_shard := DBShard, latest := Latest0} = State
) -> ) ->
%% NOTE %% NOTE
%% Unique timestamp tracking real time closely. %% Unique timestamp tracking real time closely.
%% With microsecond granularity it should be nearly impossible for it to run %% With microsecond granularity it should be nearly impossible for it to run
%% too far ahead than the real time clock. %% too far ahead than the real time clock.
{NLatest, Messages} = assign_timestamps(Latest, MessagesIn), {Latest, Messages} = assign_timestamps(Latest0, MessagesIn),
%% TODO
%% Batch is now reversed, but it should not make a lot of difference.
%% Even if it would be in order, it's still possible to write messages far away
%% in the past, i.e. when replica catches up with the leader. Storage layer
%% currently relies on wall clock time to decide if it's safe to iterate over
%% next epoch, this is likely wrong. Ideally it should rely on consensus clock
%% time instead.
Result = emqx_ds_storage_layer:store_batch(DBShard, Messages, #{}), Result = emqx_ds_storage_layer:store_batch(DBShard, Messages, #{}),
NState = State#{latest := NLatest}, NState = State#{latest := Latest},
%% TODO: Need to measure effects of changing frequency of `release_cursor`. %% TODO: Need to measure effects of changing frequency of `release_cursor`.
Effect = {release_cursor, RaftIdx, NState}, Effect = {release_cursor, RaftIdx, NState},
{NState, Result, Effect}; {NState, Result, Effect};
@ -730,7 +723,7 @@ assign_timestamps(Latest, [MessageIn | Rest], Acc) ->
assign_timestamps(Latest + 1, Rest, [Message | Acc]) assign_timestamps(Latest + 1, Rest, [Message | Acc])
end; end;
assign_timestamps(Latest, [], Acc) -> assign_timestamps(Latest, [], Acc) ->
{Latest, Acc}. {Latest, lists:reverse(Acc)}.
assign_timestamp(TimestampUs, Message) -> assign_timestamp(TimestampUs, Message) ->
{TimestampUs, Message}. {TimestampUs, Message}.