From ec1871ffdee91d117dfecfecd0ba30ddacbe7c86 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 31 Mar 2023 16:18:03 -0300 Subject: [PATCH 1/3] test(janitor): catch each callback invocation --- apps/emqx/test/emqx_test_janitor.erl | 2 +- .../test/emqx_bridge_impl_kafka_consumer_SUITE.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/test/emqx_test_janitor.erl b/apps/emqx/test/emqx_test_janitor.erl index c9b297dc7..c3f82a3e1 100644 --- a/apps/emqx/test/emqx_test_janitor.erl +++ b/apps/emqx/test/emqx_test_janitor.erl @@ -65,7 +65,7 @@ terminate(_Reason, #{callbacks := Callbacks}) -> handle_call({push, Callback}, _From, State = #{callbacks := Callbacks}) -> {reply, ok, State#{callbacks := [Callback | Callbacks]}}; handle_call(terminate, _From, State = #{callbacks := Callbacks}) -> - lists:foreach(fun(Fun) -> Fun() end, Callbacks), + lists:foreach(fun(Fun) -> catch Fun() end, Callbacks), {stop, normal, ok, State}; handle_call(_Req, _From, State) -> {reply, error, State}. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl index 02a4c3c3b..8664f9745 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl @@ -388,7 +388,7 @@ end_per_testcase(_Testcase, Config) -> maps:values(ProducersMapping) ), ok = wolff:stop_and_delete_supervised_client(KafkaProducerClientId), - emqx_common_test_helpers:call_janitor(), + emqx_common_test_helpers:call_janitor(30_000), ok = snabbkaffe:stop(), ok end. From 0e5a22f500911080cebbcc011c48c0b1f2804e49 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 31 Mar 2023 16:27:06 -0300 Subject: [PATCH 2/3] test(cassandra): cap test container memory usage Default memory usage is excessive. --- .ci/docker-compose-file/cassandra/cassandra.yaml | 4 ++-- .ci/docker-compose-file/docker-compose-cassandra.yaml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.ci/docker-compose-file/cassandra/cassandra.yaml b/.ci/docker-compose-file/cassandra/cassandra.yaml index 968efe5f6..bc1bf3357 100644 --- a/.ci/docker-compose-file/cassandra/cassandra.yaml +++ b/.ci/docker-compose-file/cassandra/cassandra.yaml @@ -469,8 +469,8 @@ concurrent_materialized_view_writes: 32 # accepting writes when the limit is exceeded until a flush completes, # and will trigger a flush based on memtable_cleanup_threshold # If omitted, Cassandra will set both to 1/4 the size of the heap. -# memtable_heap_space_in_mb: 2048 -# memtable_offheap_space_in_mb: 2048 +memtable_heap_space_in_mb: 2048 +memtable_offheap_space_in_mb: 2048 # memtable_cleanup_threshold is deprecated. The default calculation # is the only reasonable choice. See the comments on memtable_flush_writers diff --git a/.ci/docker-compose-file/docker-compose-cassandra.yaml b/.ci/docker-compose-file/docker-compose-cassandra.yaml index a54f621c1..f7143f471 100644 --- a/.ci/docker-compose-file/docker-compose-cassandra.yaml +++ b/.ci/docker-compose-file/docker-compose-cassandra.yaml @@ -12,6 +12,8 @@ services: environment: CASSANDRA_BROADCAST_ADDRESS: "1.2.3.4" CASSANDRA_RPC_ADDRESS: "0.0.0.0" + HEAP_NEWSIZE: "128M" + MAX_HEAP_SIZE: "2048M" volumes: - ./certs:/certs #ports: From 8b5a717a1f7617a25e2d1e6a132063441f7850a5 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 31 Mar 2023 11:02:15 -0300 Subject: [PATCH 3/3] test(peer): increase init and startup timeout for peer nodes Attempt to stabilize tests that use cluster nodes. --- apps/emqx/test/emqx_common_test_helpers.erl | 6 +++--- .../emqx_bridge_impl_kafka_consumer_SUITE.erl | 20 ++++++++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 9a4461fac..077ebe138 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -660,6 +660,7 @@ start_slave(Name, Opts) when is_list(Opts) -> start_slave(Name, Opts) when is_map(Opts) -> SlaveMod = maps:get(peer_mod, Opts, ct_slave), Node = node_name(Name), + put_peer_mod(Node, SlaveMod), DoStart = fun() -> case SlaveMod of @@ -669,8 +670,8 @@ start_slave(Name, Opts) when is_map(Opts) -> [ {kill_if_fail, true}, {monitor_master, true}, - {init_timeout, 10000}, - {startup_timeout, 10000}, + {init_timeout, 20_000}, + {startup_timeout, 20_000}, {erl_flags, erl_flags()} ] ); @@ -687,7 +688,6 @@ start_slave(Name, Opts) when is_map(Opts) -> throw(Other) end, pong = net_adm:ping(Node), - put_peer_mod(Node, SlaveMod), setup_node(Node, Opts), ok = snabbkaffe:forward_trace(Node), Node. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl index 8664f9745..4019a9c42 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl @@ -388,7 +388,9 @@ end_per_testcase(_Testcase, Config) -> maps:values(ProducersMapping) ), ok = wolff:stop_and_delete_supervised_client(KafkaProducerClientId), - emqx_common_test_helpers:call_janitor(30_000), + %% in CI, apparently this needs more time since the + %% machines struggle with all the containers running... + emqx_common_test_helpers:call_janitor(60_000), ok = snabbkaffe:stop(), ok end. @@ -1664,7 +1666,7 @@ t_cluster_group(Config) -> || {Name, Opts} <- Cluster ], on_exit(fun() -> - lists:foreach( + emqx_misc:pmap( fun(N) -> ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) @@ -1875,7 +1877,7 @@ t_cluster_node_down(Config) -> Cluster ), on_exit(fun() -> - lists:foreach( + emqx_misc:pmap( fun(N) -> ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) @@ -1894,10 +1896,14 @@ t_cluster_node_down(Config) -> {ok, _} = snabbkaffe:receive_events(SRef0), lists:foreach( fun(N) -> - ?assertMatch( - {ok, _}, - erpc:call(N, emqx_bridge, lookup, [BridgeId]), - #{node => N} + ?retry( + _Sleep1 = 100, + _Attempts1 = 50, + ?assertMatch( + {ok, _}, + erpc:call(N, emqx_bridge, lookup, [BridgeId]), + #{node => N} + ) ) end, Nodes