fix(ds): Allow to write batches to older generations

This commit is contained in:
ieQu1 2024-05-22 20:28:16 +02:00
parent 59a09fb86f
commit 6eb04f90a3
No known key found for this signature in database
GPG Key ID: 488654DF3FED6FDE
1 changed files with 38 additions and 15 deletions

View File

@ -301,14 +301,15 @@ store_batch(Shard, Messages, Options) ->
[{emqx_ds:time(), emqx_types:message()}], [{emqx_ds:time(), emqx_types:message()}],
emqx_ds:message_store_opts() emqx_ds:message_store_opts()
) -> {ok, cooked_batch()} | ignore | emqx_ds:error(_). ) -> {ok, cooked_batch()} | ignore | emqx_ds:error(_).
prepare_batch(Shard, Messages = [_ | _], Options) -> prepare_batch(Shard, Messages = [{Time, _} | _], Options) ->
%% NOTE %% NOTE
%% We assume that batches do not span generations. Callers should enforce this. %% We assume that batches do not span generations. Callers should enforce this.
?tp(emqx_ds_storage_layer_prepare_batch, #{ ?tp(emqx_ds_storage_layer_prepare_batch, #{
shard => Shard, messages => Messages, options => Options shard => Shard, messages => Messages, options => Options
}), }),
GenId = generation_current(Shard), %% FIXME: always store messages in the current generation
#{module := Mod, data := GenData} = generation_get(Shard, GenId), case generation_at(Shard, Time) of
{GenId, #{module := Mod, data := GenData}} ->
T0 = erlang:monotonic_time(microsecond), T0 = erlang:monotonic_time(microsecond),
Result = Result =
case Mod:prepare_batch(Shard, GenData, Messages, Options) of case Mod:prepare_batch(Shard, GenData, Messages, Options) of
@ -321,6 +322,9 @@ prepare_batch(Shard, Messages = [_ | _], Options) ->
%% TODO store->prepare %% TODO store->prepare
emqx_ds_builtin_metrics:observe_store_batch_time(Shard, T1 - T0), emqx_ds_builtin_metrics:observe_store_batch_time(Shard, T1 - T0),
Result; Result;
not_found ->
ignore
end;
prepare_batch(_Shard, [], _Options) -> prepare_batch(_Shard, [], _Options) ->
ignore. ignore.
@ -964,6 +968,25 @@ generation_current(Shard) ->
#{current_generation := Current} = get_schema_runtime(Shard), #{current_generation := Current} = get_schema_runtime(Shard),
Current. Current.
%% TODO: remove me
-spec generation_at(shard_id(), emqx_ds:time()) -> {gen_id(), generation()} | not_found.
generation_at(Shard, Time) ->
Schema = #{current_generation := Current} = get_schema_runtime(Shard),
generation_at(Time, Current, Schema).
generation_at(Time, GenId, Schema) ->
case Schema of
#{?GEN_KEY(GenId) := Gen} ->
case Gen of
#{since := Since} when Time < Since andalso GenId > 0 ->
generation_at(Time, GenId - 1, Schema);
_ ->
{GenId, Gen}
end;
_ ->
not_found
end.
-spec generation_get(shard_id(), gen_id()) -> generation() | not_found. -spec generation_get(shard_id(), gen_id()) -> generation() | not_found.
generation_get(Shard, GenId) -> generation_get(Shard, GenId) ->
case get_schema_runtime(Shard) of case get_schema_runtime(Shard) of