yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(ds): Clarify the language used in ds_bitmapper

This commit is contained in:
ieQu1 2024-01-29 20:07:47 +01:00
parent 75023f2ca3
commit 8edbec5929
1 changed files with 68 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
apps/emqx_durable_storage/src/emqx_ds_bitmask_keymapper.erl
View File

@ -16,8 +16,8 @@
-module(emqx_ds_bitmask_keymapper). -module(emqx_ds_bitmask_keymapper).
%%================================================================================ %%================================================================================
%% @doc This module is used to map N-dimensional coordinates to a %% @doc This module is used to map an N-dimensional vector to a
%% 1-dimensional space. %% 1-dimensional scalar.
%% %%
%% Example: %% Example:
%% %%
@ -59,7 +59,10 @@
%% The resulting index is a space-filling curve that looks like %% The resulting index is a space-filling curve that looks like
%% this in the topic-time 2D space: %% this in the topic-time 2D space:
%% %%
%% T ^ ---->------ |---->------ |---->------ %%
%% topic
%%
%% ^ ---->------ |---->------ |---->------
%% | --/ / --/ / --/ %% | --/ / --/ / --/
%% | -<-/ | -<-/ | -<-/ %% | -<-/ | -<-/ | -<-/
%% | -/ | -/ | -/ %% | -/ | -/ | -/
@ -73,8 +76,8 @@
%% | -/ | -/ | -/ %% | -/ | -/ | -/
%% | ---->------| ---->------| ----------> %% | ---->------| ---->------| ---------->
%% | %% |
%% -+------------+-----------------------------> t %% -+--------------+-------------+-------------> time
%% epoch %% epoch epoch epoch
%% %%
%% This structure allows to quickly seek to a the first message that %% This structure allows to quickly seek to a the first message that
%% was recorded in a certain epoch in a certain topic or a %% was recorded in a certain epoch in a certain topic or a
@ -89,6 +92,18 @@
%% This property doesn't hold between different topics, but it's not deemed %% This property doesn't hold between different topics, but it's not deemed
%% a problem right now. %% a problem right now.
%% %%
%% Notes on the terminology:
%%
%% - "Coordinates" of the original message (usually topic and the
%% timestamp, like in the example above) will be referred as the
%% "vector".
%%
%% - The 1D scalar that these coordinates are transformed to will be
%% referred as the "scalar".
%%
%% - Binary representation of the scalar if fixed size will be
%% referred as the "key".
%%
%%================================================================================ %%================================================================================
%% API: %% API:
@ -128,9 +143,9 @@
%% Type declarations %% Type declarations
%%================================================================================ %%================================================================================
-type scalar() :: integer(). -type coord() :: integer().
-type vector() :: [scalar()]. -type vector() :: [coord()].
%% N-th coordinate of a vector: %% N-th coordinate of a vector:
-type dimension() :: pos_integer(). -type dimension() :: pos_integer().
@ -147,10 +162,17 @@
%% bit from N-th element of the input vector: %% bit from N-th element of the input vector:
{dimension(), offset(), bitsize()}. {dimension(), offset(), bitsize()}.
%% This record is used during transformation of the source vector into
%% a key.
%%
%% Every dimension of the source vector has a list of `scan_action's
%% associated with it, and the key is formed by applying the scan
%% actions to the respective coordinate of the vector using `extract'
%% function.
-record(scan_action, { -record(scan_action, {
src_bitmask :: integer(), vec_coord_bitmask :: integer(),
src_offset :: offset(), vec_coord_offset :: offset(),
dst_offset :: offset() scalar_offset :: offset()
}). }).
-type scan_action() :: #scan_action{}. -type scan_action() :: #scan_action{}.
@ -158,16 +180,20 @@
-type scanner() :: [[scan_action()]]. -type scanner() :: [[scan_action()]].
-record(keymapper, { -record(keymapper, {
%% The original schema of the transformation:
schema :: [bitsource()], schema :: [bitsource()],
scanner :: scanner(), %% List of operations used to map a vector to the scalar
size :: non_neg_integer(), vec_scanner :: scanner(),
%% Total size of the resulting key, in bits:
key_size :: non_neg_integer(),
%% Bit size of each dimenstion of the vector:
dim_sizeof :: [non_neg_integer()] dim_sizeof :: [non_neg_integer()]
}). }).
-opaque keymapper() :: #keymapper{}. -opaque keymapper() :: #keymapper{}.
-type scalar_range() :: -type coord_range() ::
any | {'=', scalar() | infinity} | {'>=', scalar()} | {scalar(), '..', scalar()}. any | {'=', coord() | infinity} | {'>=', coord()} | {coord(), '..', coord()}.
-include("emqx_ds_bitmask.hrl"). -include("emqx_ds_bitmask.hrl").
@ -192,7 +218,7 @@ make_keymapper(Bitsources) ->
fun(DestOffset, {Dim0, Offset, Size}, Acc) -> fun(DestOffset, {Dim0, Offset, Size}, Acc) ->
Dim = Dim0 - 1, Dim = Dim0 - 1,
Action = #scan_action{ Action = #scan_action{
src_bitmask = ones(Size), src_offset = Offset, dst_offset = DestOffset vec_coord_bitmask = ones(Size), vec_coord_offset = Offset, scalar_offset = DestOffset
}, },
{DimSizeof, Actions} = array:get(Dim, Acc), {DimSizeof, Actions} = array:get(Dim, Acc),
array:set(Dim, {DimSizeof + Size, [Action | Actions]}, Acc) array:set(Dim, {DimSizeof + Size, [Action | Actions]}, Acc)
@ -203,27 +229,27 @@ make_keymapper(Bitsources) ->
{DimSizeof, Scanner} = lists:unzip(array:to_list(Arr)), {DimSizeof, Scanner} = lists:unzip(array:to_list(Arr)),
#keymapper{ #keymapper{
schema = Bitsources, schema = Bitsources,
scanner = Scanner, vec_scanner = Scanner,
size = Size, key_size = Size,
dim_sizeof = DimSizeof dim_sizeof = DimSizeof
}. }.
-spec bitsize(keymapper()) -> pos_integer(). -spec bitsize(keymapper()) -> pos_integer().
bitsize(#keymapper{size = Size}) -> bitsize(#keymapper{key_size = Size}) ->
Size. Size.
%% @doc Map N-dimensional vector to a scalar key. %% @doc Map N-dimensional vector to a scalar key.
%% %%
%% Note: this function is not injective. %% Note: this function is not injective.
-spec vector_to_key(keymapper(), vector()) -> key(). -spec vector_to_key(keymapper(), vector()) -> key().
vector_to_key(#keymapper{scanner = []}, []) -> vector_to_key(#keymapper{vec_scanner = []}, []) ->
0; 0;
vector_to_key(#keymapper{scanner = [Actions | Scanner]}, [Coord | Vector]) -> vector_to_key(#keymapper{vec_scanner = [Actions | Scanner]}, [Coord | Vector]) ->
do_vector_to_key(Actions, Scanner, Coord, Vector, 0). do_vector_to_key(Actions, Scanner, Coord, Vector, 0).
%% @doc Same as `vector_to_key', but it works with binaries, and outputs a binary. %% @doc Same as `vector_to_key', but it works with binaries, and outputs a binary.
-spec bin_vector_to_key(keymapper(), [binary()]) -> binary(). -spec bin_vector_to_key(keymapper(), [binary()]) -> binary().
bin_vector_to_key(Keymapper = #keymapper{dim_sizeof = DimSizeof, size = Size}, Binaries) -> bin_vector_to_key(Keymapper = #keymapper{dim_sizeof = DimSizeof, key_size = Size}, Binaries) ->
Vec = lists:zipwith( Vec = lists:zipwith(
fun(Bin, SizeOf) -> fun(Bin, SizeOf) ->
<<Int:SizeOf>> = Bin, <<Int:SizeOf>> = Bin,
@ -240,7 +266,7 @@ bin_vector_to_key(Keymapper = #keymapper{dim_sizeof = DimSizeof, size = Size}, B
%% Note: `vector_to_key(key_to_vector(K)) = K' but %% Note: `vector_to_key(key_to_vector(K)) = K' but
%% `key_to_vector(vector_to_key(V)) = V' is not guaranteed. %% `key_to_vector(vector_to_key(V)) = V' is not guaranteed.
-spec key_to_vector(keymapper(), key()) -> vector(). -spec key_to_vector(keymapper(), key()) -> vector().
key_to_vector(#keymapper{scanner = Scanner}, Key) -> key_to_vector(#keymapper{vec_scanner = Scanner}, Key) ->
lists:map( lists:map(
fun(Actions) -> fun(Actions) ->
lists:foldl( lists:foldl(
@ -256,7 +282,7 @@ key_to_vector(#keymapper{scanner = Scanner}, Key) ->
%% @doc Same as `key_to_vector', but it works with binaries. %% @doc Same as `key_to_vector', but it works with binaries.
-spec bin_key_to_vector(keymapper(), binary()) -> [binary()]. -spec bin_key_to_vector(keymapper(), binary()) -> [binary()].
bin_key_to_vector(Keymapper = #keymapper{dim_sizeof = DimSizeof, size = Size}, BinKey) -> bin_key_to_vector(Keymapper = #keymapper{dim_sizeof = DimSizeof, key_size = Size}, BinKey) ->
<<Key:Size>> = BinKey, <<Key:Size>> = BinKey,
Vector = key_to_vector(Keymapper, Key), Vector = key_to_vector(Keymapper, Key),
lists:zipwith( lists:zipwith(
@ -269,7 +295,7 @@ bin_key_to_vector(Keymapper = #keymapper{dim_sizeof = DimSizeof, size = Size}, B
%% @doc Transform a bitstring to a key %% @doc Transform a bitstring to a key
-spec bitstring_to_key(keymapper(), bitstring()) -> key(). -spec bitstring_to_key(keymapper(), bitstring()) -> key().
bitstring_to_key(#keymapper{size = Size}, Bin) -> bitstring_to_key(#keymapper{key_size = Size}, Bin) ->
case Bin of case Bin of
<<Key:Size>> -> <<Key:Size>> ->
Key; Key;
@ -279,13 +305,13 @@ bitstring_to_key(#keymapper{size = Size}, Bin) ->
%% @doc Transform key to a fixed-size bistring %% @doc Transform key to a fixed-size bistring
-spec key_to_bitstring(keymapper(), key()) -> bitstring(). -spec key_to_bitstring(keymapper(), key()) -> bitstring().
key_to_bitstring(#keymapper{size = Size}, Key) -> key_to_bitstring(#keymapper{key_size = Size}, Key) ->
<<Key:Size>>. <<Key:Size>>.
%% @doc Create a filter object that facilitates range scans. %% @doc Create a filter object that facilitates range scans.
-spec make_filter(keymapper(), [scalar_range()]) -> filter(). -spec make_filter(keymapper(), [coord_range()]) -> filter().
make_filter( make_filter(
KeyMapper = #keymapper{schema = Schema, dim_sizeof = DimSizeof, size = TotalSize}, Filter0 KeyMapper = #keymapper{schema = Schema, dim_sizeof = DimSizeof, key_size = TotalSize}, Filter0
) -> ) ->
NDim = length(DimSizeof), NDim = length(DimSizeof),
%% Transform "symbolic" constraints to ranges: %% Transform "symbolic" constraints to ranges:
@ -568,14 +594,14 @@ do_vector_to_key([Action | Actions], Scanner, Coord, Vector, Acc0) ->
Acc = Acc0 bor extract(Coord, Action), Acc = Acc0 bor extract(Coord, Action),
do_vector_to_key(Actions, Scanner, Coord, Vector, Acc). do_vector_to_key(Actions, Scanner, Coord, Vector, Acc).
-spec extract(_Source :: scalar(), scan_action()) -> integer(). -spec extract(_Source :: coord(), scan_action()) -> integer().
extract(Src, #scan_action{src_bitmask = SrcBitmask, src_offset = SrcOffset, dst_offset = DstOffset}) -> extract(Src, #scan_action{vec_coord_bitmask = SrcBitmask, vec_coord_offset = SrcOffset, scalar_offset = DstOffset}) ->
((Src bsr SrcOffset) band SrcBitmask) bsl DstOffset. ((Src bsr SrcOffset) band SrcBitmask) bsl DstOffset.
%% extract^-1 %% extract^-1
-spec extract_inv(_Dest :: scalar(), scan_action()) -> integer(). -spec extract_inv(_Dest :: coord(), scan_action()) -> integer().
extract_inv(Dest, #scan_action{ extract_inv(Dest, #scan_action{
src_bitmask = SrcBitmask, src_offset = SrcOffset, dst_offset = DestOffset vec_coord_bitmask = SrcBitmask, vec_coord_offset = SrcOffset, scalar_offset = DestOffset
}) -> }) ->
((Dest bsr DestOffset) band SrcBitmask) bsl SrcOffset. ((Dest bsr DestOffset) band SrcBitmask) bsl SrcOffset.
@ -593,8 +619,8 @@ make_keymapper0_test() ->
?assertEqual( ?assertEqual(
#keymapper{ #keymapper{
schema = Schema, schema = Schema,
scanner = [], vec_scanner = [],
size = 0, key_size = 0,
dim_sizeof = [] dim_sizeof = []
}, },
make_keymapper(Schema) make_keymapper(Schema)
@ -605,11 +631,11 @@ make_keymapper1_test() ->
?assertEqual( ?assertEqual(
#keymapper{ #keymapper{
schema = Schema, schema = Schema,
scanner = [ vec_scanner = [
[#scan_action{src_bitmask = 2#111, src_offset = 0, dst_offset = 0}], [#scan_action{vec_coord_bitmask = 2#111, vec_coord_offset = 0, scalar_offset = 0}],
[#scan_action{src_bitmask = 2#11111, src_offset = 0, dst_offset = 3}] [#scan_action{vec_coord_bitmask = 2#11111, vec_coord_offset = 0, scalar_offset = 3}]
], ],
size = 8, key_size = 8,
dim_sizeof = [3, 5] dim_sizeof = [3, 5]
}, },
make_keymapper(Schema) make_keymapper(Schema)
@ -620,14 +646,14 @@ make_keymapper2_test() ->
?assertEqual( ?assertEqual(
#keymapper{ #keymapper{
schema = Schema, schema = Schema,
scanner = [ vec_scanner = [
[ [
#scan_action{src_bitmask = 2#11111, src_offset = 3, dst_offset = 8}, #scan_action{vec_coord_bitmask = 2#11111, vec_coord_offset = 3, scalar_offset = 8},
#scan_action{src_bitmask = 2#111, src_offset = 0, dst_offset = 0} #scan_action{vec_coord_bitmask = 2#111, vec_coord_offset = 0, scalar_offset = 0}
], ],
[#scan_action{src_bitmask = 2#11111, src_offset = 0, dst_offset = 3}] [#scan_action{vec_coord_bitmask = 2#11111, vec_coord_offset = 0, scalar_offset = 3}]
], ],
size = 13, key_size = 13,
dim_sizeof = [8, 5] dim_sizeof = [8, 5]
}, },
make_keymapper(Schema) make_keymapper(Schema)