diff --git a/apps/emqx/src/emqx_crl_cache.erl b/apps/emqx/src/emqx_crl_cache.erl index 0ca779181..734ce26a8 100644 --- a/apps/emqx/src/emqx_crl_cache.erl +++ b/apps/emqx/src/emqx_crl_cache.erl @@ -24,7 +24,8 @@ register_der_crls/2, refresh/1, evict/1, - update_config/1 + update_config/1, + info/0 ]). %% gen_server callbacks @@ -102,6 +103,11 @@ update_config(Conf) -> register_der_crls(URL, CRLs) when is_list(CRLs) -> gen_server:cast(?MODULE, {register_der_crls, URL, CRLs}). +-spec info() -> #{atom() => _}. +info() -> + [state | State] = tuple_to_list(sys:get_state(?MODULE)), + maps:from_list(lists:zip(record_info(fields, state), State)). + %%-------------------------------------------------------------------- %% gen_server behaviour %%-------------------------------------------------------------------- diff --git a/apps/emqx/test/emqx_SUITE.erl b/apps/emqx/test/emqx_SUITE.erl index d2e26232f..779038045 100644 --- a/apps/emqx/test/emqx_SUITE.erl +++ b/apps/emqx/test/emqx_SUITE.erl @@ -26,12 +26,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(?config(apps, Config)). t_emqx_pubsub_api(_) -> true = emqx:is_running(node()), diff --git a/apps/emqx/test/emqx_access_control_SUITE.erl b/apps/emqx/test/emqx_access_control_SUITE.erl index 8f7a1fa6c..188a46e5d 100644 --- a/apps/emqx/test/emqx_access_control_SUITE.erl +++ b/apps/emqx/test/emqx_access_control_SUITE.erl @@ -26,12 +26,14 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules([broker]), - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start( + [{emqx, #{override_env => [{boot_modules, [broker]}]}}], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(proplists:get_value(apps, Config)). init_per_testcase(_, Config) -> Config. diff --git a/apps/emqx/test/emqx_alarm_SUITE.erl b/apps/emqx/test/emqx_alarm_SUITE.erl index 49dd4c240..385e7ca51 100644 --- a/apps/emqx/test/emqx_alarm_SUITE.erl +++ b/apps/emqx/test/emqx_alarm_SUITE.erl @@ -19,29 +19,25 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). -init_per_testcase(t_size_limit, Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - {ok, _} = emqx:update_config([alarm], #{ - <<"size_limit">> => 2 - }), - Config; -init_per_testcase(_, Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - {ok, _} = emqx:update_config([alarm], #{ - <<"validity_period">> => <<"1s">> - }), - Config. +init_per_testcase(t_size_limit = TC, Config) -> + Apps = emqx_cth_suite:start( + [{emqx, "alarm.size_limit = 2"}], + #{work_dir => emqx_cth_suite:work_dir(TC, Config)} + ), + [{apps, Apps} | Config]; +init_per_testcase(TC, Config) -> + Apps = emqx_cth_suite:start( + [{emqx, "alarm.validity_period = \"1s\""}], + #{work_dir => emqx_cth_suite:work_dir(TC, Config)} + ), + [{apps, Apps} | Config]. -end_per_testcase(_, _Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_testcase(_, Config) -> + emqx_cth_suite:stop(proplists:get_value(apps, Config)). t_alarm(_) -> ok = emqx_alarm:activate(unknown_alarm), diff --git a/apps/emqx/test/emqx_authz_cache_SUITE.erl b/apps/emqx/test/emqx_authz_cache_SUITE.erl index 09d1e1522..53e38c8f1 100644 --- a/apps/emqx/test/emqx_authz_cache_SUITE.erl +++ b/apps/emqx/test/emqx_authz_cache_SUITE.erl @@ -24,12 +24,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(proplists:get_value(apps, Config)). %%-------------------------------------------------------------------- %% Test cases diff --git a/apps/emqx/test/emqx_banned_SUITE.erl b/apps/emqx/test/emqx_banned_SUITE.erl index 47134c666..8c86e17f6 100644 --- a/apps/emqx/test/emqx_banned_SUITE.erl +++ b/apps/emqx/test/emqx_banned_SUITE.erl @@ -26,15 +26,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:start_apps([]), - ok = ekka:start(), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - ekka:stop(), - mria:stop(), - mria_mnesia:delete_schema(), - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(proplists:get_value(apps, Config)). t_add_delete(_) -> Banned = #banned{ diff --git a/apps/emqx/test/emqx_boot_SUITE.erl b/apps/emqx/test/emqx_boot_SUITE.erl index 006888d9c..323bc7591 100644 --- a/apps/emqx/test/emqx_boot_SUITE.erl +++ b/apps/emqx/test/emqx_boot_SUITE.erl @@ -23,6 +23,13 @@ all() -> emqx_common_test_helpers:all(?MODULE). +init_per_suite(Config) -> + ok = application:load(emqx), + Config. + +end_per_suite(_) -> + ok = application:unload(emqx). + t_is_enabled(_) -> try ok = application:set_env(emqx, boot_modules, all), diff --git a/apps/emqx/test/emqx_bpapi_SUITE.erl b/apps/emqx/test/emqx_bpapi_SUITE.erl index 5e29071a2..d9a0a1b20 100644 --- a/apps/emqx/test/emqx_bpapi_SUITE.erl +++ b/apps/emqx/test/emqx_bpapi_SUITE.erl @@ -26,16 +26,13 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:start_apps([emqx]), + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), [mnesia:dirty_write(Rec) || Rec <- fake_records()], - Config. + [{apps, Apps} | Config]. -end_per_suite(_Config) -> +end_per_suite(Config) -> meck:unload(), - [mnesia:dirty_delete({?TAB, Key}) || #?TAB{key = Key} <- fake_records()], - emqx_bpapi:announce(emqx), - emqx_common_test_helpers:stop_apps([emqx]), - ok. + emqx_cth_suite:stop(?config(apps, Config)). t_max_supported_version(_Config) -> ?assertMatch(3, emqx_bpapi:supported_version('fake-node2@localhost', api2)), diff --git a/apps/emqx/test/emqx_channel_delayed_puback_SUITE.erl b/apps/emqx/test/emqx_channel_delayed_puback_SUITE.erl index 4f2938b24..da8f5d7a5 100644 --- a/apps/emqx/test/emqx_channel_delayed_puback_SUITE.erl +++ b/apps/emqx/test/emqx_channel_delayed_puback_SUITE.erl @@ -31,12 +31,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(?config(apps, Config)). init_per_testcase(Case, Config) -> ?MODULE:Case({init, Config}). diff --git a/apps/emqx/test/emqx_client_SUITE.erl b/apps/emqx/test/emqx_client_SUITE.erl index c9923365e..7c2c18126 100644 --- a/apps/emqx/test/emqx_client_SUITE.erl +++ b/apps/emqx/test/emqx_client_SUITE.erl @@ -83,14 +83,14 @@ groups() -> ]. init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - emqx_config:put_listener_conf(ssl, default, [ssl_options, verify], verify_peer), - emqx_listeners:restart_listener('ssl:default'), - Config. + Apps = emqx_cth_suite:start( + [{emqx, "listeners.ssl.default.ssl_options.verify = verify_peer"}], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(?config(apps, Config)). init_per_testcase(_Case, Config) -> Config. diff --git a/apps/emqx/test/emqx_cm_SUITE.erl b/apps/emqx/test/emqx_cm_SUITE.erl index 9bfb4d5e7..7a7a989cd 100644 --- a/apps/emqx/test/emqx_cm_SUITE.erl +++ b/apps/emqx/test/emqx_cm_SUITE.erl @@ -19,7 +19,6 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_cm.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -54,12 +53,11 @@ suite() -> [{timetrap, {minutes, 2}}]. all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(proplists:get_value(apps, Config)). %%-------------------------------------------------------------------- %% TODO: Add more test cases diff --git a/apps/emqx/test/emqx_cm_locker_SUITE.erl b/apps/emqx/test/emqx_cm_locker_SUITE.erl index 3dfb6e5ad..d79fc6ef7 100644 --- a/apps/emqx/test/emqx_cm_locker_SUITE.erl +++ b/apps/emqx/test/emqx_cm_locker_SUITE.erl @@ -24,12 +24,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(proplists:get_value(apps, Config)). t_start_link(_) -> emqx_cm_locker:start_link(). diff --git a/apps/emqx/test/emqx_cm_registry_SUITE.erl b/apps/emqx/test/emqx_cm_registry_SUITE.erl index ee764a58f..c8c32a03d 100644 --- a/apps/emqx/test/emqx_cm_registry_SUITE.erl +++ b/apps/emqx/test/emqx_cm_registry_SUITE.erl @@ -28,12 +28,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(proplists:get_value(apps, Config)). init_per_testcase(_TestCase, Config) -> Config. diff --git a/apps/emqx/test/emqx_config_SUITE.erl b/apps/emqx/test/emqx_config_SUITE.erl index 3de081df8..6a49507a6 100644 --- a/apps/emqx/test/emqx_config_SUITE.erl +++ b/apps/emqx/test/emqx_config_SUITE.erl @@ -25,12 +25,21 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - Config. + WorkDir = emqx_cth_suite:work_dir(Config), + Apps = emqx_cth_suite:start( + [ + {emqx, #{ + override_env => [ + {cluster_override_conf_file, filename:join(WorkDir, "cluster_override.conf")} + ] + }} + ], + #{work_dir => WorkDir} + ), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(?config(apps, Config)). init_per_testcase(TestCase, Config) -> try diff --git a/apps/emqx/test/emqx_config_handler_SUITE.erl b/apps/emqx/test/emqx_config_handler_SUITE.erl index bb91bcbe4..b4f4085fe 100644 --- a/apps/emqx/test/emqx_config_handler_SUITE.erl +++ b/apps/emqx/test/emqx_config_handler_SUITE.erl @@ -30,12 +30,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(?config(apps, Config)). init_per_testcase(_Case, Config) -> _ = file:delete(?CLUSTER_CONF), diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index 83f5bce0f..98d4e3102 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -57,10 +57,10 @@ init_per_suite(Config) -> ok = meck:expect(emqx_alarm, deactivate, fun(_) -> ok end), ok = meck:expect(emqx_alarm, deactivate, fun(_, _) -> ok end), - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> +end_per_suite(Config) -> ok = meck:unload(emqx_transport), catch meck:unload(emqx_channel), ok = meck:unload(emqx_cm), @@ -68,8 +68,8 @@ end_per_suite(_Config) -> ok = meck:unload(emqx_metrics), ok = meck:unload(emqx_hooks), ok = meck:unload(emqx_alarm), - emqx_common_test_helpers:stop_apps([]), - ok. + + emqx_cth_suite:stop(proplists:get_value(apps, Config)). init_per_testcase(TestCase, Config) when TestCase =/= t_ws_pingreq_before_connected diff --git a/apps/emqx/test/emqx_crl_cache_SUITE.erl b/apps/emqx/test/emqx_crl_cache_SUITE.erl index 806a120aa..76bf5cf44 100644 --- a/apps/emqx/test/emqx_crl_cache_SUITE.erl +++ b/apps/emqx/test/emqx_crl_cache_SUITE.erl @@ -7,7 +7,8 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("eunit/include/eunit.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("emqx/include/asserts.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -34,14 +35,9 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - application:load(emqx), - {ok, _} = application:ensure_all_started(ssl), - emqx_config:save_schema_mod_and_names(emqx_schema), - emqx_common_test_helpers:boot_modules(all), Config. end_per_suite(_Config) -> - emqx_config:erase_all(), ok. init_per_testcase(TestCase, Config) when @@ -50,76 +46,78 @@ init_per_testcase(TestCase, Config) when TestCase =:= t_revoked -> ct:timetrap({seconds, 30}), - DataDir = ?config(data_dir, Config), - CRLFile = filename:join([DataDir, "intermediate-revoked.crl.pem"]), - {ok, CRLPem} = file:read_file(CRLFile), - [{'CertificateList', CRLDer, not_encrypted}] = public_key:pem_decode(CRLPem), ok = snabbkaffe:start_trace(), + DataDir = ?config(data_dir, Config), + {CRLPem, CRLDer} = read_crl(filename:join(DataDir, "intermediate-revoked.crl.pem")), ServerPid = start_crl_server(CRLPem), IsCached = lists:member(TestCase, [t_filled_cache, t_revoked]), - ok = setup_crl_options(Config, #{is_cached => IsCached}), + Apps = start_emqx_with_crl_cache(#{is_cached => IsCached}, TestCase, Config), [ {crl_pem, CRLPem}, {crl_der, CRLDer}, - {http_server, ServerPid} + {http_server, ServerPid}, + {tc_apps, Apps} | Config ]; -init_per_testcase(t_revoke_then_refresh, Config) -> +init_per_testcase(t_revoke_then_refresh = TestCase, Config) -> ct:timetrap({seconds, 120}), - DataDir = ?config(data_dir, Config), - CRLFileNotRevoked = filename:join([DataDir, "intermediate-not-revoked.crl.pem"]), - {ok, CRLPemNotRevoked} = file:read_file(CRLFileNotRevoked), - [{'CertificateList', CRLDerNotRevoked, not_encrypted}] = public_key:pem_decode( - CRLPemNotRevoked - ), - CRLFileRevoked = filename:join([DataDir, "intermediate-revoked.crl.pem"]), - {ok, CRLPemRevoked} = file:read_file(CRLFileRevoked), - [{'CertificateList', CRLDerRevoked, not_encrypted}] = public_key:pem_decode(CRLPemRevoked), ok = snabbkaffe:start_trace(), + DataDir = ?config(data_dir, Config), + {CRLPemNotRevoked, CRLDerNotRevoked} = + read_crl(filename:join(DataDir, "intermediate-not-revoked.crl.pem")), + {CRLPemRevoked, CRLDerRevoked} = + read_crl(filename:join(DataDir, "intermediate-revoked.crl.pem")), ServerPid = start_crl_server(CRLPemNotRevoked), - ExtraVars = #{refresh_interval => <<"10s">>}, - ok = setup_crl_options(Config, #{is_cached => true, extra_vars => ExtraVars}), + Apps = start_emqx_with_crl_cache( + #{is_cached => true, overrides => #{crl_cache => #{refresh_interval => <<"10s">>}}}, + TestCase, + Config + ), [ {crl_pem_not_revoked, CRLPemNotRevoked}, {crl_der_not_revoked, CRLDerNotRevoked}, {crl_pem_revoked, CRLPemRevoked}, {crl_der_revoked, CRLDerRevoked}, - {http_server, ServerPid} + {http_server, ServerPid}, + {tc_apps, Apps} | Config ]; -init_per_testcase(t_cache_overflow, Config) -> +init_per_testcase(t_cache_overflow = TestCase, Config) -> ct:timetrap({seconds, 120}), - DataDir = ?config(data_dir, Config), - CRLFileRevoked = filename:join([DataDir, "intermediate-revoked.crl.pem"]), - {ok, CRLPemRevoked} = file:read_file(CRLFileRevoked), ok = snabbkaffe:start_trace(), + DataDir = ?config(data_dir, Config), + {CRLPemRevoked, _} = read_crl(filename:join(DataDir, "intermediate-revoked.crl.pem")), ServerPid = start_crl_server(CRLPemRevoked), - ExtraVars = #{cache_capacity => <<"2">>}, - ok = setup_crl_options(Config, #{is_cached => false, extra_vars => ExtraVars}), + Apps = start_emqx_with_crl_cache( + #{is_cached => false, overrides => #{crl_cache => #{capacity => 2}}}, + TestCase, + Config + ), [ - {http_server, ServerPid} + {http_server, ServerPid}, + {tc_apps, Apps} | Config ]; -init_per_testcase(t_not_cached_and_unreachable, Config) -> +init_per_testcase(TestCase, Config) when + TestCase =:= t_not_cached_and_unreachable; + TestCase =:= t_update_config +-> ct:timetrap({seconds, 30}), - DataDir = ?config(data_dir, Config), - CRLFile = filename:join([DataDir, "intermediate-revoked.crl.pem"]), - {ok, CRLPem} = file:read_file(CRLFile), - [{'CertificateList', CRLDer, not_encrypted}] = public_key:pem_decode(CRLPem), ok = snabbkaffe:start_trace(), - application:stop(cowboy), - ok = setup_crl_options(Config, #{is_cached => false}), + DataDir = ?config(data_dir, Config), + {CRLPem, CRLDer} = read_crl(filename:join(DataDir, "intermediate-revoked.crl.pem")), + Apps = start_emqx_with_crl_cache(#{is_cached => false}, TestCase, Config), [ {crl_pem, CRLPem}, - {crl_der, CRLDer} + {crl_der, CRLDer}, + {tc_apps, Apps} | Config ]; -init_per_testcase(t_refresh_config, Config) -> +init_per_testcase(t_refresh_config = TestCase, Config) -> ct:timetrap({seconds, 30}), + ok = snabbkaffe:start_trace(), DataDir = ?config(data_dir, Config), - CRLFile = filename:join([DataDir, "intermediate-revoked.crl.pem"]), - {ok, CRLPem} = file:read_file(CRLFile), - [{'CertificateList', CRLDer, not_encrypted}] = public_key:pem_decode(CRLPem), + {CRLPem, CRLDer} = read_crl(filename:join(DataDir, "intermediate-revoked.crl.pem")), TestPid = self(), ok = meck:new(emqx_crl_cache, [non_strict, passthrough, no_history, no_link]), meck:expect( @@ -131,42 +129,49 @@ init_per_testcase(t_refresh_config, Config) -> {ok, {{"HTTP/1.0", 200, "OK"}, [], CRLPem}} end ), - ok = snabbkaffe:start_trace(), - ok = setup_crl_options(Config, #{is_cached => false}), + Apps = start_emqx_with_crl_cache(#{is_cached => false}, TestCase, Config), [ {crl_pem, CRLPem}, - {crl_der, CRLDer} + {crl_der, CRLDer}, + {tc_apps, Apps} | Config ]; init_per_testcase(TestCase, Config) when TestCase =:= t_update_listener; TestCase =:= t_validations -> + ct:timetrap({seconds, 30}), + ok = snabbkaffe:start_trace(), %% when running emqx standalone tests, we can't use those %% features. - case does_module_exist(emqx_mgmt_api_test_util) of + case does_module_exist(emqx_management) of true -> - ct:timetrap({seconds, 30}), DataDir = ?config(data_dir, Config), - PrivDir = ?config(priv_dir, Config), CRLFile = filename:join([DataDir, "intermediate-revoked.crl.pem"]), {ok, CRLPem} = file:read_file(CRLFile), - ok = snabbkaffe:start_trace(), ServerPid = start_crl_server(CRLPem), - ConfFilePath = filename:join([DataDir, "emqx_just_verify.conf"]), - emqx_mgmt_api_test_util:init_suite( - [emqx_conf], - fun emqx_mgmt_api_test_util:set_special_configs/1, - #{ - extra_mustache_vars => #{ - test_data_dir => DataDir, - test_priv_dir => PrivDir - }, - conf_file_path => ConfFilePath + ListenerConf = #{ + enable => true, + ssl_options => #{ + keyfile => filename:join(DataDir, "server.key.pem"), + certfile => filename:join(DataDir, "server.cert.pem"), + cacertfile => filename:join(DataDir, "ca-chain.cert.pem"), + verify => verify_peer, + enable_crl_check => false } + }, + Apps = emqx_cth_suite:start( + [ + {emqx_conf, #{config => #{listeners => #{ssl => #{default => ListenerConf}}}}}, + emqx, + emqx_management, + {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} + ], + #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)} ), [ - {http_server, ServerPid} + {http_server, ServerPid}, + {tc_apps, Apps} | Config ]; false -> @@ -174,10 +179,9 @@ init_per_testcase(TestCase, Config) when end; init_per_testcase(_TestCase, Config) -> ct:timetrap({seconds, 30}), + ok = snabbkaffe:start_trace(), DataDir = ?config(data_dir, Config), - CRLFile = filename:join([DataDir, "intermediate-revoked.crl.pem"]), - {ok, CRLPem} = file:read_file(CRLFile), - [{'CertificateList', CRLDer, not_encrypted}] = public_key:pem_decode(CRLPem), + {CRLPem, CRLDer} = read_crl(filename:join(DataDir, "intermediate-revoked.crl.pem")), TestPid = self(), ok = meck:new(emqx_crl_cache, [non_strict, passthrough, no_history, no_link]), meck:expect( @@ -189,53 +193,17 @@ init_per_testcase(_TestCase, Config) -> {ok, {{"HTTP/1.0", 200, 'OK'}, [], CRLPem}} end ), - ok = snabbkaffe:start_trace(), [ {crl_pem, CRLPem}, {crl_der, CRLDer} | Config ]. -end_per_testcase(TestCase, Config) when - TestCase =:= t_cache; - TestCase =:= t_filled_cache; - TestCase =:= t_revoked --> - ServerPid = ?config(http_server, Config), - emqx_crl_cache_http_server:stop(ServerPid), - emqx_common_test_helpers:stop_apps([]), - clear_listeners(), - application:stop(cowboy), - clear_crl_cache(), - ok = snabbkaffe:stop(), - ok; -end_per_testcase(TestCase, Config) when - TestCase =:= t_revoke_then_refresh; - TestCase =:= t_cache_overflow --> - ServerPid = ?config(http_server, Config), - emqx_crl_cache_http_server:stop(ServerPid), - emqx_common_test_helpers:stop_apps([]), - clear_listeners(), - clear_crl_cache(), - application:stop(cowboy), - ok = snabbkaffe:stop(), - ok; -end_per_testcase(t_not_cached_and_unreachable, _Config) -> - emqx_common_test_helpers:stop_apps([]), - clear_listeners(), - clear_crl_cache(), - ok = snabbkaffe:stop(), - ok; -end_per_testcase(t_refresh_config, _Config) -> - meck:unload([emqx_crl_cache]), - clear_crl_cache(), - emqx_common_test_helpers:stop_apps([]), - clear_listeners(), - clear_crl_cache(), - application:stop(cowboy), - ok = snabbkaffe:stop(), - ok; +read_crl(Filename) -> + {ok, PEM} = file:read_file(Filename), + [{'CertificateList', DER, not_encrypted}] = public_key:pem_decode(PEM), + {PEM, DER}. + end_per_testcase(TestCase, Config) when TestCase =:= t_update_listener; TestCase =:= t_validations @@ -245,18 +213,20 @@ end_per_testcase(TestCase, Config) when true -> ok; false -> - ServerPid = ?config(http_server, Config), - emqx_crl_cache_http_server:stop(ServerPid), - emqx_mgmt_api_test_util:end_suite([emqx_conf]), - clear_listeners(), - ok = snabbkaffe:stop(), - clear_crl_cache(), - ok + end_per_testcase(common, Config) end; -end_per_testcase(_TestCase, _Config) -> - meck:unload([emqx_crl_cache]), - clear_crl_cache(), +end_per_testcase(_TestCase, Config) -> ok = snabbkaffe:stop(), + clear_crl_cache(), + _ = emqx_maybe:apply( + fun emqx_crl_cache_http_server:stop/1, + proplists:get_value(http_server, Config) + ), + _ = emqx_maybe:apply( + fun emqx_cth_suite:stop/1, + proplists:get_value(tc_apps, Config) + ), + catch meck:unload([emqx_crl_cache]), ok. %%-------------------------------------------------------------------- @@ -278,11 +248,6 @@ does_module_exist(Mod) -> end end. -clear_listeners() -> - emqx_config:put([listeners], #{}), - emqx_config:put_raw([<<"listeners">>], #{}), - ok. - assert_http_get(URL) -> receive {http_get, URL} -> @@ -341,61 +306,41 @@ clear_crl_cache() -> ensure_ssl_manager_alive(), ok. -force_cacertfile(Cacertfile) -> - {SSLListeners0, OtherListeners} = lists:partition( - fun(#{proto := Proto}) -> Proto =:= ssl end, - emqx:get_env(listeners) - ), - SSLListeners = - lists:map( - fun(Listener = #{opts := Opts0}) -> - SSLOpts0 = proplists:get_value(ssl_options, Opts0), - %% it injects some garbage... - SSLOpts1 = lists:keydelete(cacertfile, 1, lists:keydelete(cacertfile, 1, SSLOpts0)), - SSLOpts2 = [{cacertfile, Cacertfile} | SSLOpts1], - Opts1 = lists:keyreplace(ssl_options, 1, Opts0, {ssl_options, SSLOpts2}), - Listener#{opts => Opts1} - end, - SSLListeners0 - ), - application:set_env(emqx, listeners, SSLListeners ++ OtherListeners), - ok. - -setup_crl_options(Config, #{is_cached := IsCached} = Opts) -> +start_emqx_with_crl_cache(#{is_cached := IsCached} = Opts, TC, Config) -> DataDir = ?config(data_dir, Config), - ConfFilePath = filename:join([DataDir, "emqx.conf"]), - Defaults = #{ - refresh_interval => <<"11m">>, - cache_capacity => <<"100">>, - test_data_dir => DataDir - }, - ExtraVars0 = maps:get(extra_vars, Opts, #{}), - ExtraVars = maps:merge(Defaults, ExtraVars0), - emqx_common_test_helpers:start_apps( - [], - fun(_) -> ok end, - #{ - extra_mustache_vars => ExtraVars, - conf_file_path => ConfFilePath + Overrides = maps:get(overrides, Opts, #{}), + ListenerConf = #{ + enable => true, + ssl_options => #{ + keyfile => filename:join(DataDir, "server.key.pem"), + certfile => filename:join(DataDir, "server.cert.pem"), + cacertfile => filename:join(DataDir, "ca-chain.cert.pem"), + verify => verify_peer, + enable_crl_check => true } + }, + Conf = #{ + listeners => #{ssl => #{default => ListenerConf}}, + crl_cache => #{ + refresh_interval => <<"11m">>, + http_timeout => <<"17s">>, + capacity => 100 + } + }, + Apps = emqx_cth_suite:start( + [{emqx, #{config => emqx_utils_maps:deep_merge(Conf, Overrides)}}], + #{work_dir => emqx_cth_suite:work_dir(TC, Config)} ), case IsCached of true -> %% wait the cache to be filled emqx_crl_cache:refresh(?DEFAULT_URL), - receive - {http_get, <>} -> ok - after 1_000 -> - ct:pal("mailbox: ~p", [process_info(self(), messages)]), - error(crl_cache_not_filled) - end; + ?assertReceive({http_get, <>}); false -> %% ensure cache is empty - clear_crl_cache(), ok end, - drain_msgs(), - ok. + Apps. start_crl_server(CRLPem) -> application:ensure_all_started(cowboy), @@ -494,31 +439,21 @@ t_init_empty_urls(_Config) -> ok. t_update_config(_Config) -> - emqx_config:save_schema_mod_and_names(emqx_schema), - emqx_config_handler:start_link(), - {ok, Pid} = emqx_crl_cache:start_link(), Conf = #{ - refresh_interval => <<"5m">>, - http_timeout => <<"10m">>, - capacity => 123 + <<"refresh_interval">> => <<"5m">>, + <<"http_timeout">> => <<"10m">>, + <<"capacity">> => 123 }, ?assertMatch({ok, _}, emqx:update_config([<<"crl_cache">>], Conf)), - State = sys:get_state(Pid), + State = emqx_crl_cache:info(), ?assertEqual( #{ refresh_interval => timer:minutes(5), http_timeout => timer:minutes(10), - capacity => 123 + cache_capacity => 123 }, - #{ - refresh_interval => element(3, State), - http_timeout => element(4, State), - capacity => element(7, State) - } - ), - emqx_config:erase(<<"crl_cache">>), - emqx_config_handler:stop(), - ok. + maps:with([refresh_interval, http_timeout, cache_capacity], State) + ). t_manual_refresh(Config) -> CRLDer = ?config(crl_der, Config), diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx.conf b/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx.conf deleted file mode 100644 index f34ab1456..000000000 --- a/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx.conf +++ /dev/null @@ -1,12 +0,0 @@ -crl_cache.refresh_interval = {{ refresh_interval }} -crl_cache.http_timeout = 17s -crl_cache.capacity = {{ cache_capacity }} -listeners.ssl.default { - ssl_options { - keyfile = "{{ test_data_dir }}/server.key.pem" - certfile = "{{ test_data_dir }}/server.cert.pem" - cacertfile = "{{ test_data_dir }}/ca-chain.cert.pem" - verify = verify_peer - enable_crl_check = true - } -} diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_just_verify.conf b/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_just_verify.conf deleted file mode 100644 index 8b9549823..000000000 --- a/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_just_verify.conf +++ /dev/null @@ -1,12 +0,0 @@ -node.name = test@127.0.0.1 -node.cookie = emqxsecretcookie -node.data_dir = "{{ test_priv_dir }}" -listeners.ssl.default { - ssl_options { - keyfile = "{{ test_data_dir }}/server.key.pem" - certfile = "{{ test_data_dir }}/server.cert.pem" - cacertfile = "{{ test_data_dir }}/ca-chain.cert.pem" - verify = verify_peer - enable_crl_check = false - } -} diff --git a/apps/emqx/test/emqx_exclusive_sub_SUITE.erl b/apps/emqx/test/emqx_exclusive_sub_SUITE.erl index 1caf4cc9b..a122c55e8 100644 --- a/apps/emqx/test/emqx_exclusive_sub_SUITE.erl +++ b/apps/emqx/test/emqx_exclusive_sub_SUITE.erl @@ -34,20 +34,14 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - ok = ekka:start(), - OldConf = emqx:get_config([zones], #{}), - emqx_config:put_zone_conf(default, [mqtt, exclusive_subscription], true), - timer:sleep(50), - [{old_conf, OldConf} | Config]. + Apps = emqx_cth_suite:start( + [{emqx, "mqtt.exclusive_subscription = true"}], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + [{apps, Apps} | Config]. end_per_suite(Config) -> - emqx_config:put([zones], proplists:get_value(old_conf, Config)), - ekka:stop(), - mria:stop(), - mria_mnesia:delete_schema(), - emqx_common_test_helpers:stop_apps([]). + emqx_cth_suite:stop(proplists:get_value(apps, Config)). end_per_testcase(_TestCase, _Config) -> emqx_exclusive_subscription:clear(). diff --git a/apps/emqx/test/emqx_listeners_update_SUITE.erl b/apps/emqx/test/emqx_listeners_update_SUITE.erl index c1fdff2e2..8ae9775a5 100644 --- a/apps/emqx/test/emqx_listeners_update_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_update_SUITE.erl @@ -30,12 +30,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(proplists:get_value(apps, Config)). init_per_testcase(_TestCase, Config) -> Init = emqx:get_raw_config(?LISTENERS), diff --git a/apps/emqx/test/emqx_mqtt_SUITE.erl b/apps/emqx/test/emqx_mqtt_SUITE.erl index f03c7af83..591a08e9a 100644 --- a/apps/emqx/test/emqx_mqtt_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_SUITE.erl @@ -19,7 +19,6 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -39,12 +38,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(proplists:get_value(apps, Config)). init_per_testcase(TestCase, Config) -> case erlang:function_exported(?MODULE, TestCase, 2) of diff --git a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl index e97684b74..8be5564b2 100644 --- a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl @@ -26,12 +26,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]), - ok. +end_per_suite(Config) -> + emqx_cth_suite:stop(proplists:get_value(apps, Config)). t_check_pub(_) -> OldConf = emqx:get_config([zones], #{}), diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl index a2a2e5244..ff248a16a 100644 --- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl @@ -19,7 +19,6 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -59,31 +58,17 @@ groups() -> ]. init_per_group(tcp, Config) -> - emqx_common_test_helpers:start_apps([]), - [{port, 1883}, {conn_fun, connect} | Config]; + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{port, 1883}, {conn_fun, connect}, {group_apps, Apps} | Config]; init_per_group(quic, Config) -> - UdpPort = 1884, - emqx_common_test_helpers:start_apps([]), - emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort), - [{port, UdpPort}, {conn_fun, quic_connect} | Config]; -init_per_group(_, Config) -> - emqx_common_test_helpers:stop_apps([]), - Config. + Apps = emqx_cth_suite:start( + [{emqx, "listeners.quic.test { enable = true, bind = 1884 }"}], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + [{port, 1884}, {conn_fun, quic_connect}, {group_apps, Apps} | Config]. -end_per_group(quic, _Config) -> - emqx_config:put([listeners, quic], #{}), - ok; -end_per_group(_Group, _Config) -> - ok. - -init_per_suite(Config) -> - %% Start Apps - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - Config. - -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_group(_Group, Config) -> + emqx_cth_suite:stop(?config(group_apps, Config)). init_per_testcase(TestCase, Config) -> case erlang:function_exported(?MODULE, TestCase, 2) of diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index aa5d78121..fce74785c 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -44,14 +44,33 @@ groups() -> ]. init_per_suite(Config) -> - application:load(emqx), - emqx_config:save_schema_mod_and_names(emqx_schema), - emqx_common_test_helpers:boot_modules(all), Config. end_per_suite(_Config) -> ok. +init_per_group(openssl, Config) -> + DataDir = ?config(data_dir, Config), + ListenerConf = #{ + bind => <<"0.0.0.0:8883">>, + max_connections => 512000, + ssl_options => #{ + keyfile => filename(DataDir, "server.key"), + certfile => filename(DataDir, "server.pem"), + cacertfile => filename(DataDir, "ca.pem"), + ocsp => #{ + enable_ocsp_stapling => true, + issuer_pem => filename(DataDir, "ocsp-issuer.pem"), + responder_url => <<"http://127.0.0.1:9877">> + } + } + }, + Conf = #{listeners => #{ssl => #{default => ListenerConf}}}, + Apps = emqx_cth_suite:start( + [{emqx, #{config => Conf}}], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + [{group_apps, Apps} | Config]; init_per_group(tls12, Config) -> [{tls_vsn, "-tls1_2"} | Config]; init_per_group(tls13, Config) -> @@ -63,24 +82,14 @@ init_per_group(without_status_request, Config) -> init_per_group(_Group, Config) -> Config. +end_per_group(openssl, Config) -> + emqx_cth_suite:stop(?config(group_apps, Config)); end_per_group(_Group, _Config) -> ok. init_per_testcase(t_openssl_client, Config) -> ct:timetrap({seconds, 30}), - DataDir = ?config(data_dir, Config), - Handler = fun(_) -> ok end, {OCSPResponderPort, OCSPOSPid} = setup_openssl_ocsp(Config), - ConfFilePath = filename:join([DataDir, "openssl_listeners.conf"]), - emqx_common_test_helpers:start_apps( - [], - Handler, - #{ - extra_mustache_vars => #{test_data_dir => DataDir}, - conf_file_path => ConfFilePath - } - ), - ct:sleep(1_000), [ {ocsp_responder_port, OCSPResponderPort}, {ocsp_responder_os_pid, OCSPOSPid} @@ -107,15 +116,25 @@ init_per_testcase(TestCase, Config) when {ok, {{"HTTP/1.0", 200, 'OK'}, [], <<"ocsp response">>}} end ), - emqx_mgmt_api_test_util:init_suite([emqx_conf]), + Apps = emqx_cth_suite:start( + [ + emqx_conf, + emqx, + emqx_management, + {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} + ], + #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)} + ), + _ = emqx_common_test_http:create_default_app(), snabbkaffe:start_trace(), - Config; + [{tc_apps, Apps} | Config]; false -> [{skip_does_not_apply, true} | Config] end; -init_per_testcase(t_ocsp_responder_error_responses, Config) -> +init_per_testcase(TC, Config) -> ct:timetrap({seconds, 30}), TestPid = self(), + DataDir = ?config(data_dir, Config), ok = meck:new(emqx_ocsp_cache, [non_strict, passthrough, no_history, no_link]), meck:expect( emqx_ocsp_cache, @@ -123,90 +142,44 @@ init_per_testcase(t_ocsp_responder_error_responses, Config) -> fun(URL, _HTTPTimeout) -> ct:pal("ocsp http request ~p", [URL]), TestPid ! {http_get, URL}, - persistent_term:get({?MODULE, http_response}) + persistent_term:get( + {?MODULE, http_response}, + {ok, {{"HTTP/1.0", 200, 'OK'}, [], <<"ocsp response">>}} + ) end ), - DataDir = ?config(data_dir, Config), - Type = ssl, - Name = test_ocsp, - ListenerOpts = #{ - ssl_options => - #{ - certfile => filename:join(DataDir, "server.pem"), - ocsp => #{ - enable_ocsp_stapling => true, - responder_url => <<"http://localhost:9877/">>, - issuer_pem => filename:join(DataDir, "ocsp-issuer.pem"), - refresh_http_timeout => <<"15s">>, - refresh_interval => <<"1s">> - } - } - }, - Conf = #{listeners => #{Type => #{Name => ListenerOpts}}}, - ConfBin = emqx_utils_maps:binary_key_map(Conf), - CheckedConf = hocon_tconf:check_plain(emqx_schema, ConfBin, #{ - required => false, atom_keys => false - }), - Conf2 = emqx_utils_maps:unsafe_atom_key_map(CheckedConf), - ListenerOpts2 = emqx_utils_maps:deep_get([listeners, Type, Name], Conf2), - emqx_config:put_listener_conf(Type, Name, [], ListenerOpts2), - snabbkaffe:start_trace(), - _Heir = spawn_dummy_heir(), - {ok, CachePid} = emqx_ocsp_cache:start_link(), - [ - {cache_pid, CachePid} - | Config - ]; -init_per_testcase(_TestCase, Config) -> - ct:timetrap({seconds, 10}), - TestPid = self(), - ok = meck:new(emqx_ocsp_cache, [non_strict, passthrough, no_history, no_link]), - meck:expect( - emqx_ocsp_cache, - http_get, - fun(URL, _HTTPTimeout) -> - TestPid ! {http_get, URL}, - {ok, {{"HTTP/1.0", 200, 'OK'}, [], <<"ocsp response">>}} - end - ), - snabbkaffe:start_trace(), - _Heir = spawn_dummy_heir(), - {ok, CachePid} = emqx_ocsp_cache:start_link(), - DataDir = ?config(data_dir, Config), - Type = ssl, - Name = test_ocsp, ResponderURL = <<"http://localhost:9877/">>, - ListenerOpts = #{ - ssl_options => - #{ - certfile => filename:join(DataDir, "server.pem"), - ocsp => #{ - enable_ocsp_stapling => true, - responder_url => ResponderURL, - issuer_pem => filename:join(DataDir, "ocsp-issuer.pem"), - refresh_http_timeout => <<"15s">>, - refresh_interval => <<"1s">> - } + ListenerConf = #{ + enable => false, + bind => 0, + ssl_options => #{ + certfile => filename(DataDir, "server.pem"), + ocsp => #{ + enable_ocsp_stapling => true, + responder_url => ResponderURL, + issuer_pem => filename(DataDir, "ocsp-issuer.pem"), + refresh_http_timeout => <<"15s">>, + refresh_interval => <<"1s">> } + } }, - Conf = #{listeners => #{Type => #{Name => ListenerOpts}}}, - ConfBin = emqx_utils_maps:binary_key_map(Conf), - CheckedConf = hocon_tconf:check_plain(emqx_schema, ConfBin, #{ - required => false, atom_keys => false - }), - Conf2 = emqx_utils_maps:unsafe_atom_key_map(CheckedConf), - ListenerOpts2 = emqx_utils_maps:deep_get([listeners, Type, Name], Conf2), - emqx_config:put_listener_conf(Type, Name, [], ListenerOpts2), + Conf = #{listeners => #{ssl => #{test_ocsp => ListenerConf}}}, + Apps = emqx_cth_suite:start( + [{emqx, #{config => Conf}}], + #{work_dir => emqx_cth_suite:work_dir(TC, Config)} + ), + snabbkaffe:start_trace(), [ - {cache_pid, CachePid}, - {responder_url, ResponderURL} + {responder_url, ResponderURL}, + {tc_apps, Apps} | Config ]. +filename(Dir, Name) -> + unicode:characters_to_binary(filename:join(Dir, Name)). + end_per_testcase(t_openssl_client, Config) -> - OCSPResponderOSPid = ?config(ocsp_responder_os_pid, Config), - catch kill_pid(OCSPResponderOSPid), - emqx_common_test_helpers:stop_apps([]), + catch kill_pid(?config(ocsp_responder_os_pid, Config)), ok; end_per_testcase(TestCase, Config) when TestCase =:= t_update_listener; @@ -217,19 +190,12 @@ end_per_testcase(TestCase, Config) when true -> ok; false -> - emqx_mgmt_api_test_util:end_suite([emqx_conf]), - meck:unload([emqx_ocsp_cache]), - ok + end_per_testcase(common, Config) end; -end_per_testcase(t_ocsp_responder_error_responses, Config) -> - CachePid = ?config(cache_pid, Config), - catch gen_server:stop(CachePid), - meck:unload([emqx_ocsp_cache]), - persistent_term:erase({?MODULE, http_response}), - ok; end_per_testcase(_TestCase, Config) -> - CachePid = ?config(cache_pid, Config), - catch gen_server:stop(CachePid), + snabbkaffe:stop(), + emqx_cth_suite:stop(?config(tc_apps, Config)), + persistent_term:erase({?MODULE, http_response}), meck:unload([emqx_ocsp_cache]), ok. @@ -237,24 +203,6 @@ end_per_testcase(_TestCase, Config) -> %% Helper functions %%-------------------------------------------------------------------- -%% The real cache makes `emqx_kernel_sup' the heir to its ETS table. -%% In some tests, we don't start the full supervision tree, so we need -%% this dummy process. -spawn_dummy_heir() -> - {_, {ok, _}} = - ?wait_async_action( - spawn_link(fun() -> - true = register(emqx_kernel_sup, self()), - ?tp(heir_name_registered, #{}), - receive - stop -> ok - end - end), - #{?snk_kind := heir_name_registered}, - 1_000 - ), - ok. - does_module_exist(Mod) -> case erlang:module_loaded(Mod) of true -> @@ -416,11 +364,6 @@ do_ensure_port_open(Port, N) when N > 0 -> do_ensure_port_open(Port, N - 1) end. -get_sni_fun(ListenerID) -> - #{opts := Opts} = emqx_listeners:find_by_id(ListenerID), - SSLOpts = proplists:get_value(ssl_options, Opts), - proplists:get_value(sni_fun, SSLOpts). - openssl_version() -> Res0 = string:trim(os:cmd("openssl version"), trailing), [_, Res] = string:split(Res0, " "), @@ -516,9 +459,7 @@ t_request_ocsp_response(_Config) -> end ). -t_request_ocsp_response_restart_cache(Config) -> - process_flag(trap_exit, true), - CachePid = ?config(cache_pid, Config), +t_request_ocsp_response_restart_cache(_Config) -> ListenerID = <<"ssl:test_ocsp">>, ?check_trace( begin @@ -526,6 +467,7 @@ t_request_ocsp_response_restart_cache(Config) -> {ok, _} = emqx_ocsp_cache:fetch_response(ListenerID), ?wait_async_action( begin + CachePid = whereis(emqx_ocsp_cache), Ref = monitor(process, CachePid), exit(CachePid, kill), receive @@ -533,9 +475,7 @@ t_request_ocsp_response_restart_cache(Config) -> ok after 1_000 -> error(cache_not_killed) - end, - {ok, _} = emqx_ocsp_cache:start_link(), - ok + end end, #{?snk_kind := ocsp_cache_init} ), diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE_data/openssl_listeners.conf b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/openssl_listeners.conf deleted file mode 100644 index d26e12acf..000000000 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE_data/openssl_listeners.conf +++ /dev/null @@ -1,14 +0,0 @@ -listeners.ssl.default { - bind = "0.0.0.0:8883" - max_connections = 512000 - ssl_options { - keyfile = "{{ test_data_dir }}/server.key" - certfile = "{{ test_data_dir }}/server.pem" - cacertfile = "{{ test_data_dir }}/ca.pem" - ocsp { - enable_ocsp_stapling = true - issuer_pem = "{{ test_data_dir }}/ocsp-issuer.pem" - responder_url = "http://127.0.0.1:9877" - } - } -} diff --git a/apps/emqx/test/emqx_olp_SUITE.erl b/apps/emqx/test/emqx_olp_SUITE.erl index cd8db7a8f..2c418ebdf 100644 --- a/apps/emqx/test/emqx_olp_SUITE.erl +++ b/apps/emqx/test/emqx_olp_SUITE.erl @@ -26,18 +26,15 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), OldSch = erlang:system_flag(schedulers_online, 1), - [{old_sch, OldSch} | Config]. + [{apps, Apps}, {old_sch, OldSch} | Config]. end_per_suite(Config) -> erlang:system_flag(schedulers_online, ?config(old_sch, Config)), - emqx_common_test_helpers:stop_apps([]). + emqx_cth_suite:stop(?config(apps, Config)). init_per_testcase(_, Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), emqx_olp:enable(), case wait_for(fun() -> lc_sup:whereis_runq_flagman() end, 10) of true -> ok; diff --git a/apps/emqx/test/emqx_os_mon_SUITE.erl b/apps/emqx/test/emqx_os_mon_SUITE.erl index 1833be48e..2d7558392 100644 --- a/apps/emqx/test/emqx_os_mon_SUITE.erl +++ b/apps/emqx/test/emqx_os_mon_SUITE.erl @@ -24,12 +24,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(proplists:get_value(apps, Config)). init_per_testcase(t_cpu_check_alarm, Config) -> SysMon = emqx_config:get([sysmon, os], #{}), diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index b2205a659..da1ee9191 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -144,19 +144,35 @@ groups() -> ]. init_per_suite(Config) -> - emqx_common_test_helpers:start_apps([]), - UdpPort = 14567, - start_emqx_quic(UdpPort), - %% Turn off force_shutdown policy. - ShutdownPolicy = emqx_config:get_zone_conf(default, [force_shutdown]), - ct:pal("force shutdown config: ~p", [ShutdownPolicy]), - emqx_config:put_zone_conf(default, [force_shutdown], ShutdownPolicy#{enable := false}), - [{shutdown_policy, ShutdownPolicy}, {port, UdpPort}, {pub_qos, 0}, {sub_qos, 0} | Config]. + Apps = start_emqx(Config), + [{port, 14567}, {pub_qos, 0}, {sub_qos, 0}, {apps, Apps} | Config]. end_per_suite(Config) -> - emqx_config:put_zone_conf(default, [force_shutdown], ?config(shutdown_policy, Config)), - emqx_common_test_helpers:stop_apps([]), - ok. + emqx_cth_suite:stop(?config(apps, Config)). + +start_emqx(Config) -> + emqx_cth_suite:start( + [mk_emqx_spec()], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ). + +stop_emqx(Config) -> + emqx_cth_suite:stop(?config(apps, Config)). + +restart_emqx(Config) -> + ok = stop_emqx(Config), + emqx_cth_suite:start( + [mk_emqx_spec()], + #{work_dir => emqx_cth_suite:work_dir(Config), boot_type => restart} + ). + +mk_emqx_spec() -> + {emqx, + %% Turn off force_shutdown policy. + "force_shutdown.enable = false" + "\n listeners.quic.default {" + "\n enable = true, bind = 14567, acceptors = 16, idle_timeout_ms = 15000" + "\n }"}. init_per_group(pub_qos0, Config) -> [{pub_qos, 0} | Config]; @@ -190,11 +206,6 @@ init_per_group(_, Config) -> end_per_group(_, Config) -> Config. -init_per_testcase(_, Config) -> - emqx_common_test_helpers:start_apps([]), - start_emqx_quic(?config(port, Config)), - Config. - t_quic_sock(Config) -> Port = 4567, SslOpts = [ @@ -1582,9 +1593,13 @@ t_multi_streams_remote_shutdown(Config) -> {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), - ok = stop_emqx(), - %% Client should be closed - assert_client_die(C, 100, 200). + ok = stop_emqx(Config), + try + %% Client should be closed + assert_client_die(C, 100, 200) + after + restart_emqx(Config) + end. t_multi_streams_remote_shutdown_with_reconnect(Config) -> erlang:process_flag(trap_exit, true), @@ -1636,10 +1651,8 @@ t_multi_streams_remote_shutdown_with_reconnect(Config) -> {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), - ok = stop_emqx(), + _Apps = restart_emqx(Config), - timer:sleep(200), - start_emqx_quic(?config(port, Config)), ?assert(is_list(emqtt:info(C))), emqtt:stop(C). @@ -2028,16 +2041,6 @@ calc_pkt_id(1, Id) -> calc_pkt_id(2, Id) -> Id. --spec start_emqx_quic(inet:port_number()) -> ok. -start_emqx_quic(UdpPort) -> - emqx_common_test_helpers:start_apps([]), - application:ensure_all_started(quicer), - emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort). - --spec stop_emqx() -> ok. -stop_emqx() -> - emqx_common_test_helpers:stop_apps([]). - %% select a random port picked by OS -spec select_port() -> inet:port_number(). select_port() -> diff --git a/apps/emqx/test/emqx_ratelimiter_SUITE.erl b/apps/emqx/test/emqx_ratelimiter_SUITE.erl index f414c3759..29f268f7c 100644 --- a/apps/emqx/test/emqx_ratelimiter_SUITE.erl +++ b/apps/emqx/test/emqx_ratelimiter_SUITE.erl @@ -47,23 +47,23 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - load_conf(), - emqx_common_test_helpers:start_apps([?APP]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + ok = load_conf(), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([?APP]). +end_per_suite(Config) -> + emqx_cth_suite:stop(?config(apps, Config)). init_per_testcase(_TestCase, Config) -> - emqx_config:erase(limiter), - load_conf(), + ok = emqx_config:erase(limiter), + ok = load_conf(), Config. end_per_testcase(_TestCase, Config) -> Config. load_conf() -> - ok = emqx_common_test_helpers:load_config(emqx_limiter_schema, ?BASE_CONF). + emqx_common_test_helpers:load_config(emqx_limiter_schema, ?BASE_CONF). init_config() -> emqx_config:init_load(emqx_limiter_schema, ?BASE_CONF). diff --git a/apps/emqx/test/emqx_request_responser_SUITE.erl b/apps/emqx/test/emqx_request_responser_SUITE.erl index 2685b4ac3..299ebf854 100644 --- a/apps/emqx/test/emqx_request_responser_SUITE.erl +++ b/apps/emqx/test/emqx_request_responser_SUITE.erl @@ -22,12 +22,11 @@ -include_lib("common_test/include/ct.hrl"). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(?config(apps, Config)). all() -> emqx_common_test_helpers:all(?MODULE). diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 8af3b2190..a5d2567d0 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -49,31 +49,11 @@ all() -> emqx_common_test_helpers:all(?SUITE). init_per_suite(Config) -> - DistPid = - case net_kernel:nodename() of - ignored -> - %% calling `net_kernel:start' without `epmd' - %% running will result in a failure. - emqx_common_test_helpers:start_epmd(), - {ok, Pid} = net_kernel:start(['master@127.0.0.1', longnames]), - ct:pal("start epmd, node name: ~p", [node()]), - Pid; - _ -> - undefined - end, - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - [{dist_pid, DistPid} | Config]. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. end_per_suite(Config) -> - DistPid = ?config(dist_pid, Config), - case DistPid of - Pid when is_pid(Pid) -> - net_kernel:stop(); - _ -> - ok - end, - emqx_common_test_helpers:stop_apps([]). + emqx_cth_suite:stop(?config(apps, Config)). init_per_testcase(Case, Config) -> try diff --git a/apps/emqx/test/emqx_sup_SUITE.erl b/apps/emqx/test/emqx_sup_SUITE.erl index d2780469e..7a2461c20 100644 --- a/apps/emqx/test/emqx_sup_SUITE.erl +++ b/apps/emqx/test/emqx_sup_SUITE.erl @@ -24,12 +24,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(proplists:get_value(apps, Config)). t_child(_) -> ?assertMatch({error, _}, emqx_sup:start_child(undef, worker)),