From ff327609db88c9457eed6e74d9d47a70c1bb04b7 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 27 Jun 2023 17:11:23 +0200 Subject: [PATCH 1/2] test(ft): generate segments manually in proptests In order to avoid spamming proptest logs with HUGE type instances, and make proptests themselves simpler. Proper now should refuse shrinking altogether, instances that cause prop function to timeout do not shrink well for some reason, making proptests stuck. --- apps/emqx/test/emqx_proper_types.erl | 39 +++++- .../test/props/prop_emqx_ft_assembly.erl | 120 ++++++++++-------- 2 files changed, 104 insertions(+), 55 deletions(-) diff --git a/apps/emqx/test/emqx_proper_types.erl b/apps/emqx/test/emqx_proper_types.erl index c1b41c292..95cf29bee 100644 --- a/apps/emqx/test/emqx_proper_types.erl +++ b/apps/emqx/test/emqx_proper_types.erl @@ -52,12 +52,19 @@ %% Generic Types -export([ - scaled/2 + scaled/2, + fixedmap/1 ]). %% Iterators -export([nof/1]). +%% Utilities +-export([ + generate/2, + typegen/0 +]). + -type proptype() :: proper_types:raw_type(). %%-------------------------------------------------------------------- @@ -679,6 +686,36 @@ limited_list(N, T) -> scaled(F, T) when F > 0 -> ?SIZED(S, resize(round(S * F), T)). +-spec fixedmap(#{_Key => proptype()}) -> proptype(). +fixedmap(M) -> + ?LET(PList, maps:to_list(M), maps:from_list(PList)). + +%%-------------------------------------------------------------------- +%% Utilities +%%-------------------------------------------------------------------- + +-type typegen() :: {typegen, proper_gen:size(), proper_gen:seed()}. + +-spec typegen() -> proptype(). +typegen() -> + Seed = {non_neg_integer(), non_neg_integer(), non_neg_integer()}, + {typegen, ?SIZED(S, S), Seed}. + +-spec generate(proptype(), typegen()) -> _Instance. +generate(T, {typegen, Size, Seed}) -> + % NOTE + % We need to run it in a separate process so that it won't erase + % any proper state in the current process allocated by the property + % being evaluated. + {Pid, MRef} = erlang:spawn_monitor( + fun() -> exit(proper_gen:pick(T, Size, Seed)) end + ), + receive + {'DOWN', MRef, process, Pid, Result} -> + {ok, Instance} = Result, + Instance + end. + %%-------------------------------------------------------------------- %% Internal funcs %%-------------------------------------------------------------------- diff --git a/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl b/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl index db2f0c0b2..dd640c513 100644 --- a/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl +++ b/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl @@ -18,80 +18,92 @@ -include_lib("proper/include/proper.hrl"). --import(emqx_proper_types, [scaled/2]). +-import(emqx_proper_types, [scaled/2, fixedmap/1, typegen/0, generate/2]). -define(COVERAGE_TIMEOUT, 5000). prop_coverage() -> ?FORALL( - {Filesize, Segsizes}, - {filesize_t(), segsizes_t()}, - ?FORALL( - Fragments, - noshrink(segments_t(Filesize, Segsizes)), - ?TIMEOUT( - ?COVERAGE_TIMEOUT, - begin - ASM1 = append_segments(mk_assembly(Filesize), Fragments), - {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), - measure( - #{"Fragments" => length(Fragments), "Time" => Time}, - case emqx_ft_assembly:status(ASM2) of - complete -> - Coverage = emqx_ft_assembly:coverage(ASM2), - measure( - #{"CoverageLength" => length(Coverage)}, - is_coverage_complete(Coverage) - ); - {incomplete, {missing, {segment, _, _}}} -> - measure("CoverageLength", 0, true) - end - ) - end - ) + #{filesize := Filesize, segsizes := Segsizes, typegen := TypeGen}, + noshrink( + fixedmap(#{ + filesize => filesize_t(), + segsizes => segsizes_t(), + typegen => typegen() + }) + ), + ?TIMEOUT( + ?COVERAGE_TIMEOUT, + begin + Segments = generate(segments_t(Filesize, Segsizes), TypeGen), + ASM1 = append_segments(mk_assembly(Filesize), Segments), + {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), + measure( + #{"Segments" => length(Segments), "Time" => Time}, + case emqx_ft_assembly:status(ASM2) of + complete -> + Coverage = emqx_ft_assembly:coverage(ASM2), + measure( + #{"CoverageLength" => length(Coverage)}, + is_coverage_complete(Coverage) + ); + {incomplete, {missing, {segment, _, _}}} -> + measure("CoverageLength", 0, true) + end + ) + end ) ). prop_coverage_likely_incomplete() -> ?FORALL( - {Filesize, Segsizes, Hole}, - {filesize_t(), segsizes_t(), filesize_t()}, - ?FORALL( - Fragments, - noshrink(segments_t(Filesize, Segsizes, (Hole rem max(Filesize, 1)))), - ?TIMEOUT( - ?COVERAGE_TIMEOUT, - begin - ASM1 = append_segments(mk_assembly(Filesize), Fragments), - {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), - measure( - #{"Fragments" => length(Fragments), "Time" => Time}, - case emqx_ft_assembly:status(ASM2) of - complete -> - % NOTE: this is still possible due to the nature of `SUCHTHATMAYBE` - IsComplete = emqx_ft_assembly:coverage(ASM2), - collect(complete, is_coverage_complete(IsComplete)); - {incomplete, {missing, {segment, _, _}}} -> - collect(incomplete, true) - end - ) + #{filesize := Filesize, segsizes := Segsizes, hole := HoleIn, typegen := TypeGen}, + noshrink( + fixedmap(#{ + filesize => filesize_t(), + segsizes => segsizes_t(), + hole => filesize_t(), + typegen => typegen() + }) + ), + ?TIMEOUT(?COVERAGE_TIMEOUT, begin + Hole = HoleIn rem max(Filesize, 1), + Segments = generate(segments_t(Filesize, Segsizes, Hole), TypeGen), + ASM1 = append_segments(mk_assembly(Filesize), Segments), + {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), + measure( + #{"Segments" => length(Segments), "Time" => Time}, + case emqx_ft_assembly:status(ASM2) of + complete -> + % NOTE: this is still possible due to the nature of `SUCHTHATMAYBE` + IsComplete = emqx_ft_assembly:coverage(ASM2), + collect(complete, is_coverage_complete(IsComplete)); + {incomplete, {missing, {segment, _, _}}} -> + collect(incomplete, true) end ) - ) + end) ). prop_coverage_complete() -> ?FORALL( - {Filesize, Segsizes}, - {filesize_t(), ?SUCHTHAT([BaseSegsize | _], segsizes_t(), BaseSegsize > 0)}, - ?FORALL( - {Fragments, RemoteNode}, - noshrink({segments_t(Filesize, Segsizes), remote_node_t()}), + #{filesize := Filesize, segsizes := Segsizes, node := RemoteNode, typegen := TypeGen}, + noshrink( + fixedmap(#{ + filesize => filesize_t(), + segsizes => ?SUCHTHAT([BaseSegsize | _], segsizes_t(), BaseSegsize > 0), + node => remote_node_t(), + typegen => typegen() + }) + ), + ?TIMEOUT( + ?COVERAGE_TIMEOUT, begin % Ensure that we have complete coverage + Segments = generate(segments_t(Filesize, Segsizes), TypeGen), ASM1 = mk_assembly(Filesize), ASM2 = append_coverage(ASM1, RemoteNode, Filesize, Segsizes), - ASM3 = append_segments(ASM2, Fragments), + ASM3 = append_segments(ASM2, Segments), {Time, ASM4} = timer:tc(emqx_ft_assembly, update, [ASM3]), measure( #{"CoverageMax" => nsegs(Filesize, Segsizes), "Time" => Time}, From 7d171de02bf1ef7d0b436dd1a84a5b1ddb077eca Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 27 Jun 2023 17:17:56 +0200 Subject: [PATCH 2/2] test(ft): make proptests even less aggressive --- apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl b/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl index dd640c513..9fa3d2279 100644 --- a/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl +++ b/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl @@ -20,7 +20,7 @@ -import(emqx_proper_types, [scaled/2, fixedmap/1, typegen/0, generate/2]). --define(COVERAGE_TIMEOUT, 5000). +-define(COVERAGE_TIMEOUT, 10000). prop_coverage() -> ?FORALL( @@ -201,7 +201,7 @@ segment_t(Filesize, Segsizes) -> ). filesize_t() -> - scaled(2500, non_neg_integer()). + scaled(2000, non_neg_integer()). segsizes_t() -> ?LET(