feat(cover): add covertool

rebar.config

@@ -29,7 +29,7 @@
 
 {cover_opts, [verbose]}.
 {cover_export_enabled, true}.
-{cover_excl_mods, [emqx_exproto_pb, emqx_exhook_pb]}.
+{cover_excl_mods, [emqx_exproto_pb, emqx_exhook_pb, emqx_cover]}.
 
 {provider_hooks, [{pre, [{release, {relup_helper, gen_appups}}]}]}.
 
@@ -40,6 +40,7 @@
 {deps,
     [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps
     , {redbug, "2.0.7"}
+    , {covertool, {git, "https://github.com/zmstone/covertool", {tag, "2.0.4.1"}}}
     , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.2"}}}
     , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.8"}}}
     , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.4"}}}

rebar.config.erl

@@ -253,6 +253,7 @@ relx_apps(ReleaseType) ->
     , {ekka, load}
     , {emqx_plugin_libs, load}
     , observer_cli
+    , {covertool, load}
     ]
     ++ [emqx_modules || not is_enterprise()]
     ++ [emqx_license || is_enterprise()]

scripts/rel_otp_apps.eterm

@@ -14,4 +14,5 @@
 , {mnesia, load}
 , xmerl
 , tools
+, covertool % this is not really a otp app, but we don't need to worry about relup of it
 ].

scripts/xref_check.eterm

@@ -5,6 +5,7 @@
    , excl_apps =>
          [ observer
          , redbug
+         , covertool
          ]
    , excl_mods =>
          [ hipe_unified_loader
@@ -14,6 +15,7 @@
          , basho_bench_driver_erldis
          , release_handler
          , cuttlefish_rebar_plugin
+         , emqx_cover
          ]
    , filters =>
          [{{coap_client,channel_apply,3},{coap_dtls_socket,close,1}},
@@ -239,6 +241,7 @@
    , analysis => undefined_functions
    , excl_apps =>
          [ observer
+         , covertool
          ]
    , excl_mods =>
          [ systools
@@ -246,6 +249,7 @@
          , release_handler
          , systools_relup
          , cuttlefish_rebar_plugin
+         , emqx_cover
          ]
    , filters =>
          [{'Elixir.Atom',to_string,1},

src/emqx_cover.erl

@@ -18,16 +18,68 @@
 %% It is used to collect coverage data when running blackbox test
 -module(emqx_cover).
 
+-include_lib("covertool/include/covertool.hrl").
+
+-ifdef(EMQX_ENTERPRISE).
+-define(OUTPUT_APPNAME, 'EMQX Enterprise').
+-else.
+-define(OUTPUT_APPNAME, 'EMQX').
+-endif.
+
 -export([start/0,
          start/1,
-         export_and_stop/1
+         abort/0,
+         export_and_stop/1,
+         lookup_source/1
         ]).
 
+%% This is a ETS table to keep a mapping of module name (atom) to
+%% .erl file path (relative path from project root)
+%% We needed this ETS table because the source file information
+%% is missing from the .beam metadata sicne we are using 'deterministic'
+%% compile flag.
+-define(SRC, emqx_cover_module_src).
+
+%% @doc Start cover.
+%% All emqx_ modules will be cover-compiled, this may cause
+%% some excessive RAM consumption and result in warning logs.
 start() ->
     start(#{}).
 
+%% @doc Start cover.
+%% All emqx_ modules will be cover-compiled, this may cause
+%% some excessive RAM consumption and result in warning logs.
+%% Supported options:
+%% - project_root: the directory to search for .erl source code
+%% - debug_secret_file: only applicable to EMQX Enterprise
 start(Opts) ->
-    ok = maybe_set_secret(),
+    ok = abort(),
+    %% spawn a ets table owner
+    %% this implementation is kept dead-simple
+    %% because there is no concurrency requirement
+    Parent = self(),
+    {Pid, Ref} =
+        erlang:spawn_monitor(
+          fun() ->
+                  true = register(?SRC, self()),
+                  _ = ets:new(?SRC, [named_table, public]),
+                  _ = Parent ! {started, self()},
+                  receive
+                      stop ->
+                          ok
+                  end
+          end),
+    receive
+        {started, Pid} ->
+            ok;
+        {'DOWN', Ref, process, Pid, Reason} ->
+            throw({failed_to_start, Reason})
+    after
+        1000 ->
+            throw({failed_to_start, timeout})
+    end,
+    Modules = modules(Opts),
+    ok = maybe_set_secret(Opts),
     case cover:start() of
         {ok, _Pid} ->
             ok;
@@ -36,28 +88,91 @@ start(Opts) ->
         Other ->
             throw(Other)
     end,
-    ok = cover_compile(Opts).
-
-export_and_stop(Path) ->
-    ok = cover:export(Path),
-    _ = cover:stop(),
+    ok = cover_compile(Modules),
+    io:format("cover-compiled ~p modules~n", [length(Modules)]),
+    ok = build_source_mapping(Opts, sets:from_list(Modules, [{version, 2}])),
+    CachedModulesCount = ets:info(?SRC, size),
+    io:format("source-cached ~p modules~n", [CachedModulesCount]),
     ok.
 
-maybe_set_secret() ->
-    case os:getenv("EMQX_DEBUG_SECRET_FILE") of
-        false ->
-            ok;
+%% @doc Abort cover data collection without exporting.
+abort() ->
+    _ = cover:stop(),
+    case whereis(?SRC) of
+        undefined -> ok;
+        Pid -> exit(Pid, kill)
+    end,
+    ok.
+
+%% @doc Export coverage report (xml) format.
+%% e.g. `emqx_cover:export_and_stop("/tmp/cover.xml").'
+export_and_stop(Path) when is_list(Path) ->
+    ProjectRoot = get_project_root(),
+    Config = #config{appname = ?OUTPUT_APPNAME,
+                     sources = [ProjectRoot],
+                     output = Path,
+                     lookup_source = fun ?MODULE:lookup_source/1
+                    },
+    covertool:generate_report(Config, cover:modules()).
+
+build_source_mapping(Opts, Modules) ->
+    Default = os_env("EMQX_PROJECT_ROOT"),
+    case maps:get(project_root, Opts, Default) of
+        "" ->
+            io:format(standard_error, "EMQX_PROJECT_ROOT is not set", []),
+            throw(emqx_project_root_undefined);
+        Dir ->
+            ok = put_project_root(Dir),
+            ok = do_build_source_mapping(Dir, Modules)
+    end.
+
+get_project_root() ->
+    [{_, Dir}] = ets:lookup(?SRC, {root, ?OUTPUT_APPNAME}),
+    Dir.
+
+put_project_root(Dir) ->
+    _ = ets:insert(?SRC, {{root, ?OUTPUT_APPNAME}, Dir}),
+    ok.
+
+do_build_source_mapping(Dir, Modules) ->
+    All = filelib:wildcard("**/*.erl", Dir),
+    lists:foreach(
+      fun(Path) ->
+              ModuleNameStr = filename:basename(Path, ".erl"),
+              Module = list_to_atom(ModuleNameStr),
+              case sets:is_element(Module, Modules) of
+                  true ->
+                      ets:insert(?SRC, {Module, Path});
+                  false ->
+                      ok
+              end
+      end, All),
+    ok.
+
+lookup_source(Module) ->
+    case ets:lookup(?SRC, Module) of
+        [{_, Path}] ->
+            Path;
+        [] ->
+            false
+    end.
+
+maybe_set_secret(Opts) ->
+    Default = os_env("EMQX_DEBUG_SECRET_FILE"),
+    case maps:get(debug_secret_file, Opts, Default) of
         "" ->
             ok;
         File ->
             ok = emqx:set_debug_secret(File)
     end.
 
-cover_compile(_Opts) ->
+modules(_Opts) ->
     %% TODO better filter based on Opts,
     %% e.g. we may want to see coverage info for ehttpc
     Filter = fun is_emqx_module/1,
-    Modules = find_modules(Filter),
+    find_modules(Filter).
+
+cover_compile(Modules) ->
     Results = cover:compile_beam(Modules),
     Errors = lists:filter(fun({ok, _}) -> false;
                              (_) -> true
@@ -66,14 +181,14 @@ cover_compile(_Opts) ->
         [] ->
             ok;
         _ ->
-            io:format(user, "failed_to_cover_compile:~n~p~n", [Errors]),
+            io:format("failed_to_cover_compile:~n~p~n", [Errors]),
             throw(failed_to_cover_compile)
     end.
 
 find_modules(Filter) ->
     All = code:all_loaded(),
-    F = fun({M, _BeamPath}) -> Filter(M) end,
-    lists:filter(F, All).
+    F = fun({M, _BeamPath}) -> Filter(M) andalso {true, M} end,
+    lists:filtermap(F, All).
 
 is_emqx_module(?MODULE) ->
     %% do not cover-compile self
@@ -85,3 +200,6 @@ is_emqx_module(Module) ->
         _ ->
             false
     end.
+
+os_env(Name) ->
+    os:getenv(Name, "").