feat(cthsuite): add function to determine workdir of testrun

In a deterministic fashion, to lift the burden of undestanding where
the testrun's data should go from the test writer.

apps/emqx/integration_test/emqx_ds_SUITE.erl

@@ -22,7 +22,7 @@ all() ->
 init_per_suite(Config) ->
     TCApps = emqx_cth_suite:start(
         app_specs(),
-        #{work_dir => ?config(priv_dir, Config)}
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
     ),
     [{tc_apps, TCApps} | Config].
 
@@ -31,9 +31,9 @@ end_per_suite(Config) ->
     emqx_cth_suite:stop(TCApps),
     ok.
 
-init_per_testcase(t_session_subscription_idempotency, Config) ->
+init_per_testcase(t_session_subscription_idempotency = TC, Config) ->
     Cluster = cluster(#{n => 1}),
-    ClusterOpts = #{work_dir => ?config(priv_dir, Config)},
+    ClusterOpts = #{work_dir => emqx_cth_suite:work_dir(TC, Config)},
     NodeSpecs = emqx_cth_cluster:mk_nodespecs(Cluster, ClusterOpts),
     Nodes = emqx_cth_cluster:start(Cluster, ClusterOpts),
     [

apps/emqx/test/emqx_cth_suite.erl

@@ -22,6 +22,9 @@
 -export([start/2]).
 -export([stop/1]).
 
+-export([work_dir/1]).
+-export([work_dir/2]).
+
 -export([load_apps/1]).
 -export([start_apps/2]).
 -export([start_app/2]).
@@ -98,16 +101,11 @@ when
     SuiteOpts :: #{
         %% Working directory
         %% Everything a test produces should go here. If this directory is not empty,
-        %% function will raise an error.
+        %% function will raise an error. Most of the time, the result of `work_dir/1`
+        %% or `work_dir/2` (if used in a testcase) should be fine here.
         work_dir := file:name()
     }.
-start(Apps, SuiteOpts0 = #{work_dir := WorkDir0}) ->
+start(Apps, SuiteOpts = #{work_dir := WorkDir}) ->
-    %% when running CT on the whole app, it seems like `priv_dir` is the same on all
-    %% suites and leads to the "clean slate" verificatin to fail.
-    WorkDir = binary_to_list(
-        filename:join([WorkDir0, emqx_guid:to_hexstr(emqx_guid:gen())])
-    ),
-    SuiteOpts = SuiteOpts0#{work_dir := WorkDir},
     % 1. Prepare appspec instructions
     AppSpecs = [mk_appspec(App, SuiteOpts) || App <- Apps],
     % 2. Load every app so that stuff scanning attributes of loaded modules works
@@ -339,6 +337,45 @@ default_config(App, SuiteOpts) ->
 
 %%
 
+%% @doc Determine the unique work directory for the current test run.
+%% Takes into account name of the test suite, and all test groups the current run
+%% is part of.
+-spec work_dir(CTConfig :: proplists:proplist()) ->
+    file:filename_all().
+work_dir(CTConfig) ->
+    % Directory specific to the current test run.
+    [PrivDir] = proplists:get_all_values(priv_dir, CTConfig),
+    % Directory specific to the currently executing test suite.
+    [DataDir] = proplists:get_all_values(data_dir, CTConfig),
+    % NOTE: Contains the name of the current test group, if executed as part of a group.
+    GroupProps = proplists:get_value(tc_group_properties, CTConfig, []),
+    % NOTE: Contains names of outer test groups, if any.
+    GroupPathOuter = proplists:get_value(tc_group_path, CTConfig, []),
+    SuiteDir = filename:basename(DataDir),
+    GroupPath = lists:append([GroupProps | GroupPathOuter]),
+    GroupLevels = [atom_to_list(Name) || {name, Name} <- GroupPath],
+    WorkDir1 = filename:join(PrivDir, SuiteDir),
+    WorkDir2 =
+        case GroupLevels of
+            [] ->
+                WorkDir1;
+            [_ | _] ->
+                GroupDir = string:join(lists:reverse(GroupLevels), "."),
+                filename:join(WorkDir1, GroupDir)
+        end,
+    WorkDir2.
+
+%% @doc Determine the unique work directory for the current testcase run.
+%% Be careful when testcase runs under no groups, and its name matches the name of a
+%% previously executed test group, it's best to avoid such naming.
+-spec work_dir(TestCaseName :: atom(), CTConfig :: proplists:proplist()) ->
+    file:filename_all().
+work_dir(TCName, CTConfig) ->
+    WorkDir = work_dir(CTConfig),
+    filename:join(WorkDir, TCName).
+
+%%
+
 start_ekka() ->
     ok = emqx_common_test_helpers:start_ekka(),
     {ok, [mnesia, ekka]}.

apps/emqx/test/emqx_flapping_SUITE.erl

@@ -35,7 +35,7 @@ init_per_suite(Config) ->
                 "\n ban_time = 2s"
                 "\n }"}
         ],
-        #{work_dir => ?config(priv_dir, Config)}
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
     ),
     [{suite_apps, Apps} | Config].
 

apps/emqx/test/emqx_persistent_messages_SUITE.erl

@@ -33,10 +33,9 @@ init_per_suite(Config) ->
     %% TODO: remove after other suites start to use `emx_cth_suite'
     application:stop(emqx),
     application:stop(emqx_durable_storage),
-    WorkDir = ?config(priv_dir, Config),
     TCApps = emqx_cth_suite:start(
         app_specs(),
-        #{work_dir => WorkDir}
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
     ),
     [{tc_apps, TCApps} | Config].
 

apps/emqx_authz/test/emqx_authz_file_SUITE.erl

@@ -44,7 +44,7 @@ init_per_testcase(TestCase, Config) ->
             {emqx_conf, "authorization.no_match = deny, authorization.cache.enable = false"},
             emqx_authz
         ],
-        #{work_dir => filename:join(?config(priv_dir, Config), TestCase)}
+        #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)}
     ),
     [{tc_apps, Apps} | Config].
 

apps/emqx_authz/test/emqx_authz_rich_actions_SUITE.erl

@@ -37,7 +37,7 @@ init_per_testcase(TestCase, Config) ->
             {emqx_conf, "authorization.no_match = deny, authorization.cache.enable = false"},
             emqx_authz
         ],
-        #{work_dir => filename:join(?config(priv_dir, Config), TestCase)}
+        #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)}
     ),
     [{tc_apps, Apps} | Config].
 

apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl

@@ -116,13 +116,13 @@ end_per_suite(_Config) ->
     ok.
 
 init_per_group(cluster = Name, Config) ->
-    Nodes = [NodePrimary | _] = mk_cluster(Name, Config),
+    Nodes = [NodePrimary | _] = mk_cluster(Config),
     init_api([{group, Name}, {cluster_nodes, Nodes}, {node, NodePrimary} | Config]);
 init_per_group(cluster_later_join = Name, Config) ->
-    Nodes = [NodePrimary | _] = mk_cluster(Name, Config, #{join_to => undefined}),
+    Nodes = [NodePrimary | _] = mk_cluster(Config, #{join_to => undefined}),
     init_api([{group, Name}, {cluster_nodes, Nodes}, {node, NodePrimary} | Config]);
-init_per_group(Name, Config) ->
+init_per_group(_Name, Config) ->
-    WorkDir = filename:join(?config(priv_dir, Config), Name),
+    WorkDir = emqx_cth_suite:work_dir(Config),
     Apps = emqx_cth_suite:start(?APPSPECS ++ [?APPSPEC_DASHBOARD], #{work_dir => WorkDir}),
     init_api([{group, single}, {group_apps, Apps}, {node, node()} | Config]).
 
@@ -131,10 +131,10 @@ init_api(Config) ->
     {ok, App} = erpc:call(APINode, emqx_common_test_http, create_default_app, []),
     [{api, App} | Config].
 
-mk_cluster(Name, Config) ->
+mk_cluster(Config) ->
-    mk_cluster(Name, Config, #{}).
+    mk_cluster(Config, #{}).
 
-mk_cluster(Name, Config, Opts) ->
+mk_cluster(Config, Opts) ->
     Node1Apps = ?APPSPECS ++ [?APPSPEC_DASHBOARD],
     Node2Apps = ?APPSPECS,
     emqx_cth_cluster:start(
@@ -142,7 +142,7 @@ mk_cluster(Name, Config, Opts) ->
             {emqx_bridge_api_SUITE1, Opts#{role => core, apps => Node1Apps}},
             {emqx_bridge_api_SUITE2, Opts#{role => core, apps => Node2Apps}}
         ],
-        #{work_dir => filename:join(?config(priv_dir, Config), Name)}
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
     ).
 
 end_per_group(Group, Config) when

apps/emqx_ft/test/emqx_ft_SUITE.erl

@@ -76,7 +76,7 @@ init_per_suite(Config) ->
         [
             {emqx_ft, #{config => emqx_ft_test_helpers:config(Storage)}}
         ],
-        #{work_dir => ?config(priv_dir, Config)}
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
     ),
     [{suite_apps, Apps} | Config].
 

apps/emqx_ft/test/emqx_ft_conf_SUITE.erl

@@ -32,13 +32,12 @@ end_per_suite(_Config) ->
     ok.
 
 init_per_testcase(Case, Config) ->
-    WorkDir = filename:join(?config(priv_dir, Config), Case),
     Apps = emqx_cth_suite:start(
         [
             {emqx_conf, #{}},
             {emqx_ft, #{config => "file_transfer {}"}}
         ],
-        #{work_dir => WorkDir}
+        #{work_dir => emqx_cth_suite:work_dir(Case, Config)}
     ),
     [{suite_apps, Apps} | Config].
 

apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl

@@ -36,12 +36,11 @@ groups() ->
 
 init_per_suite(Config) ->
     Storage = emqx_ft_test_helpers:local_storage(Config),
-    WorkDir = ?config(priv_dir, Config),
     Apps = emqx_cth_suite:start(
         [
             {emqx_ft, #{config => emqx_ft_test_helpers:config(Storage)}}
         ],
-        #{work_dir => WorkDir}
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
     ),
     [{suite_apps, Apps} | Config].
 

apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl

@@ -28,7 +28,7 @@ all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
-    Apps = emqx_cth_suite:start([emqx], #{work_dir => ?config(priv_dir, Config)}),
+    Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}),
     [{suite_apps, Apps} | Config].
 
 end_per_suite(Config) ->