yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

test: make assembly proptests waste less memory

This commit is contained in:
Andrew Mayorov 2023-02-28 16:55:23 +03:00 committed by Ilya Averyanov
parent 8e6f960c09
commit 788e76ed2d
1 changed files with 39 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl
View File

@ -28,11 +28,11 @@ prop_coverage() ->
{filesize_t(), segsizes_t()}, {filesize_t(), segsizes_t()},
?FORALL( ?FORALL(
Fragments, Fragments,
noshrink(fragments_t(Filesize, Segsizes)), noshrink(segments_t(Filesize, Segsizes)),
?TIMEOUT( ?TIMEOUT(
?COVERAGE_TIMEOUT, ?COVERAGE_TIMEOUT,
begin begin
ASM1 = append_fragments(mk_assembly(Filesize), Fragments), ASM1 = append_segments(mk_assembly(Filesize), Fragments),
{Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]),
measure( measure(
#{"Fragments" => length(Fragments), "Time" => Time}, #{"Fragments" => length(Fragments), "Time" => Time},
@ -58,11 +58,11 @@ prop_coverage_likely_incomplete() ->
{filesize_t(), segsizes_t(), filesize_t()}, {filesize_t(), segsizes_t(), filesize_t()},
?FORALL( ?FORALL(
Fragments, Fragments,
noshrink(fragments_t(Filesize, Segsizes, Hole)), noshrink(segments_t(Filesize, Segsizes, Hole)),
?TIMEOUT( ?TIMEOUT(
?COVERAGE_TIMEOUT, ?COVERAGE_TIMEOUT,
begin begin
ASM1 = append_fragments(mk_assembly(Filesize), Fragments), ASM1 = append_segments(mk_assembly(Filesize), Fragments),
{Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]),
measure( measure(
#{"Fragments" => length(Fragments), "Time" => Time}, #{"Fragments" => length(Fragments), "Time" => Time},
@ -85,17 +85,19 @@ prop_coverage_complete() ->
{Filesize, Segsizes}, {Filesize, Segsizes},
{filesize_t(), ?SUCHTHAT([BaseSegsize | _], segsizes_t(), BaseSegsize > 0)}, {filesize_t(), ?SUCHTHAT([BaseSegsize | _], segsizes_t(), BaseSegsize > 0)},
?FORALL( ?FORALL(
{Fragments, MaxCoverage}, {Fragments, RemoteNode},
noshrink({fragments_t(Filesize, Segsizes), coverage_t(Filesize, Segsizes)}), noshrink({segments_t(Filesize, Segsizes), remote_node_t()}),
begin begin
% Ensure that we have complete coverage % Ensure that we have complete coverage
ASM1 = append_fragments(mk_assembly(Filesize), Fragments ++ MaxCoverage), ASM1 = mk_assembly(Filesize),
{Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), ASM2 = append_coverage(ASM1, RemoteNode, Filesize, Segsizes),
ASM3 = append_segments(ASM2, Fragments),
{Time, ASM4} = timer:tc(emqx_ft_assembly, update, [ASM3]),
measure( measure(
#{"CoverageMax" => length(MaxCoverage), "Time" => Time}, #{"CoverageMax" => nsegs(Filesize, Segsizes), "Time" => Time},
case emqx_ft_assembly:status(ASM2) of case emqx_ft_assembly:status(ASM4) of
complete -> complete ->
Coverage = emqx_ft_assembly:coverage(ASM2), Coverage = emqx_ft_assembly:coverage(ASM4),
measure( measure(
#{"Coverage" => length(Coverage)}, #{"Coverage" => length(Coverage)},
is_coverage_complete(Coverage) is_coverage_complete(Coverage)
@ -127,15 +129,26 @@ is_coverage_complete(
mk_assembly(Filesize) -> mk_assembly(Filesize) ->
emqx_ft_assembly:append(emqx_ft_assembly:new(Filesize), node(), mk_filemeta(Filesize)). emqx_ft_assembly:append(emqx_ft_assembly:new(Filesize), node(), mk_filemeta(Filesize)).
append_fragments(ASMIn, Fragments) -> append_segments(ASMIn, Fragments) ->
lists:foldl( lists:foldl(
fun({Node, Frag}, ASM) -> fun({Node, {Offset, Size}}, ASM) ->
emqx_ft_assembly:append(ASM, Node, Frag) emqx_ft_assembly:append(ASM, Node, mk_segment(Offset, Size))
end, end,
ASMIn, ASMIn,
Fragments Fragments
). ).
append_coverage(ASM, Node, Filesize, Segsizes = [BaseSegsize | _]) ->
append_coverage(ASM, Node, Filesize, BaseSegsize, 0, nsegs(Filesize, Segsizes)).
append_coverage(ASM, Node, Filesize, Segsize, I, NSegs) when I < NSegs ->
Offset = I * Segsize,
Size = min(Segsize, Filesize - Offset),
ASMNext = emqx_ft_assembly:append(ASM, Node, mk_segment(Offset, Size)),
append_coverage(ASMNext, Node, Filesize, Segsize, I + 1, NSegs);
append_coverage(ASM, _Node, _Filesize, _Segsize, _, _NSegs) ->
ASM.
mk_filemeta(Filesize) -> mk_filemeta(Filesize) ->
#{ #{
path => "MANIFEST.json", path => "MANIFEST.json",
@ -148,39 +161,33 @@ mk_segment(Offset, Size) ->
fragment => {segment, #{offset => Offset, size => Size}} fragment => {segment, #{offset => Offset, size => Size}}
}. }.
fragments_t(Filesize, Segsizes = [BaseSegsize | _]) -> nsegs(Filesize, [BaseSegsize | _]) ->
NSegs = Filesize / max(1, BaseSegsize), Filesize div max(1, BaseSegsize) + 1.
scaled(1 + NSegs, list({node_t(), fragment_t(Filesize, Segsizes)})).
fragments_t(Filesize, Segsizes = [BaseSegsize | _], Hole) -> segments_t(Filesize, Segsizes) ->
NSegs = Filesize / max(1, BaseSegsize), scaled(nsegs(Filesize, Segsizes), list({node_t(), segment_t(Filesize, Segsizes)})).
scaled(1 + NSegs, list({node_t(), fragment_t(Filesize, Segsizes, Hole)})).
fragment_t(Filesize, Segsizes, Hole) -> segments_t(Filesize, Segsizes, Hole) ->
scaled(nsegs(Filesize, Segsizes), list({node_t(), segment_t(Filesize, Segsizes, Hole)})).
segment_t(Filesize, Segsizes, Hole) ->
?SUCHTHATMAYBE( ?SUCHTHATMAYBE(
#{fragment := {segment, #{offset := Offset, size := Size}}}, {Offset, Size},
fragment_t(Filesize, Segsizes), segment_t(Filesize, Segsizes),
(Hole rem Filesize) =< Offset orelse (Hole rem Filesize) > (Offset + Size) (Hole rem Filesize) =< Offset orelse (Hole rem Filesize) > (Offset + Size)
). ).
fragment_t(Filesize, Segsizes) -> segment_t(Filesize, Segsizes) ->
?LET( ?LET(
Segsize, Segsize,
oneof(Segsizes), oneof(Segsizes),
?LET( ?LET(
Index, Index,
range(0, Filesize div max(1, Segsize)), range(0, Filesize div max(1, Segsize)),
mk_segment(Index * Segsize, min(Segsize, Filesize - (Index * Segsize))) {Index * Segsize, min(Segsize, Filesize - (Index * Segsize))}
) )
). ).
coverage_t(Filesize, [Segsize | _]) ->
NSegs = Filesize div max(1, Segsize),
[
{remote_node_t(), mk_segment(I * Segsize, min(Segsize, Filesize - (I * Segsize)))}
|| I <- lists:seq(0, NSegs)
].
filesize_t() -> filesize_t() ->
scaled(4000, non_neg_integer()). scaled(4000, non_neg_integer()).