Merge pull request #9446 from zmstone/1129-emqx-7958-add-cover

1129 emqx 7958 add cover

scripts/rel_otp_apps.eterm

@@ -13,4 +13,5 @@
 , runtime_tools
 , {mnesia, load}
 , xmerl
+, tools
 ].

src/emqx.appup.src

@@ -2,12 +2,14 @@
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
   [{"4.4.11",
-    [{load_module,emqx_relup,brutal_purge,soft_purge,[]},
+    [{add_module,emqx_cover},
+     {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
    {"4.4.10",
-    [{add_module,emqx_ocsp_cache},
+    [{add_module,emqx_cover},
+     {add_module,emqx_ocsp_cache},
      {add_module,emqx_crl_cache},
      {add_module,emqx_const_v1},
      {load_module,emqx_listeners,brutal_purge,soft_purge,[]},
@@ -24,7 +26,8 @@
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {apply, {application, set_env, [gen_rpc, insecure_auth_fallback_allowed, true]}}]},
    {"4.4.9",
-    [{add_module,emqx_ocsp_cache},
+    [{add_module,emqx_cover},
+     {add_module,emqx_ocsp_cache},
      {add_module,emqx_crl_cache},
      {add_module,emqx_const_v1},
      {add_module,emqx_secret},
@@ -48,7 +51,8 @@
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {apply, {application, set_env, [gen_rpc, insecure_auth_fallback_allowed, true]}}]},
    {"4.4.8",
-    [{add_module,emqx_ocsp_cache},
+    [{add_module,emqx_cover},
+     {add_module,emqx_ocsp_cache},
      {add_module,emqx_crl_cache},
      {add_module,emqx_const_v1},
      {add_module,emqx_secret},
@@ -73,7 +77,8 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {apply, {application, set_env, [gen_rpc, insecure_auth_fallback_allowed, true]}}]},
    {"4.4.7",
-    [{add_module,emqx_ocsp_cache},
+    [{add_module,emqx_cover},
+     {add_module,emqx_ocsp_cache},
      {add_module,emqx_crl_cache},
      {add_module,emqx_const_v1},
      {add_module,emqx_secret},
@@ -98,7 +103,8 @@
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {apply, {application, set_env, [gen_rpc, insecure_auth_fallback_allowed, true]}}]},
    {"4.4.6",
-    [{add_module,emqx_ocsp_cache},
+    [{add_module,emqx_cover},
+     {add_module,emqx_ocsp_cache},
      {add_module,emqx_crl_cache},
      {add_module,emqx_const_v1},
      {add_module,emqx_secret},
@@ -123,7 +129,8 @@
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {apply, {application, set_env, [gen_rpc, insecure_auth_fallback_allowed, true]}}]},
    {"4.4.5",
-    [{add_module,emqx_ocsp_cache},
+    [{add_module,emqx_cover},
+     {add_module,emqx_ocsp_cache},
      {add_module,emqx_crl_cache},
      {add_module,emqx_const_v1},
      {add_module,emqx_secret},
@@ -150,7 +157,8 @@
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {apply, {application, set_env, [gen_rpc, insecure_auth_fallback_allowed, true]}}]},
    {"4.4.4",
-    [{add_module,emqx_ocsp_cache},
+    [{add_module,emqx_cover},
+     {add_module,emqx_ocsp_cache},
      {add_module,emqx_crl_cache},
      {add_module,emqx_const_v1},
      {add_module,emqx_secret},
@@ -184,7 +192,8 @@
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {apply, {application, set_env, [gen_rpc, insecure_auth_fallback_allowed, true]}}]},
    {"4.4.3",
-    [{add_module,emqx_ocsp_cache},
+    [{add_module,emqx_cover},
+     {add_module,emqx_ocsp_cache},
      {add_module,emqx_crl_cache},
      {add_module,emqx_const_v1},
      {add_module,emqx_secret},
@@ -225,7 +234,8 @@
      {load_module,emqx_relup},
      {apply, {application, set_env, [gen_rpc, insecure_auth_fallback_allowed, true]}}]},
    {"4.4.2",
-    [{add_module,emqx_ocsp_cache},
+    [{add_module,emqx_cover},
+     {add_module,emqx_ocsp_cache},
      {add_module,emqx_crl_cache},
      {add_module,emqx_const_v1},
      {add_module,emqx_secret},
@@ -267,7 +277,8 @@
      {load_module,emqx_relup},
      {apply, {application, set_env, [gen_rpc, insecure_auth_fallback_allowed, true]}}]},
    {"4.4.1",
-    [{add_module,emqx_ocsp_cache},
+    [{add_module,emqx_cover},
+     {add_module,emqx_ocsp_cache},
      {add_module,emqx_crl_cache},
      {add_module,emqx_const_v1},
      {add_module,emqx_secret},
@@ -314,7 +325,8 @@
      {add_module,emqx_relup},
      {apply, {application, set_env, [gen_rpc, insecure_auth_fallback_allowed, true]}}]},
    {"4.4.0",
-    [{add_module,emqx_ocsp_cache},
+    [{add_module,emqx_cover},
+     {add_module,emqx_ocsp_cache},
      {add_module,emqx_crl_cache},
      {add_module,emqx_const_v1},
      {add_module,emqx_secret},
@@ -367,7 +379,8 @@
     [{load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
-     {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_cover}]},
    {"4.4.10",
     [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
      {load_module,emqx_kernel_sup,brutal_purge,soft_purge,[]},
@@ -381,6 +394,7 @@
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_cover},
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.9",
@@ -402,6 +416,7 @@
      {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_router,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_cover},
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.8",
@@ -424,6 +439,7 @@
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_cover},
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.7",
@@ -446,6 +462,7 @@
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_cover},
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.6",
@@ -468,6 +485,7 @@
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_cover},
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.5",
@@ -492,6 +510,7 @@
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_cover},
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.4",
@@ -523,6 +542,7 @@
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
      {load_module,emqx_metrics,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_cover},
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.3",
@@ -560,6 +580,7 @@
      {load_module,emqx_access_control,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_relup},
+     {delete_module,emqx_cover},
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.2",
@@ -598,6 +619,7 @@
      {load_module,emqx_hooks,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_relup},
+     {delete_module,emqx_cover},
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.1",
@@ -641,6 +663,7 @@
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module,emqx_connection,brutal_purge,soft_purge,[]},
      {delete_module,emqx_relup},
+     {delete_module,emqx_cover},
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.0",
@@ -685,6 +708,7 @@
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]},
      {delete_module,emqx_relup},
+     {delete_module,emqx_cover},
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {<<".*">>,[]}]}.

src/emqx_cover.erl

@@ -0,0 +1,87 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% @doc This module is NOT used in production.
+%% It is used to collect coverage data when running blackbox test
+-module(emqx_cover).
+
+-export([start/0,
+         start/1,
+         export_and_stop/1
+        ]).
+
+start() ->
+    start(#{}).
+
+start(Opts) ->
+    ok = maybe_set_secret(),
+    case cover:start() of
+        {ok, _Pid} ->
+            ok;
+        {error, {already_started, _Pid}} ->
+            ok;
+        Other ->
+            throw(Other)
+    end,
+    ok = cover_compile(Opts).
+
+export_and_stop(Path) ->
+    ok = cover:export(Path),
+    _ = cover:stop(),
+    ok.
+
+maybe_set_secret() ->
+    case os:getenv("EMQX_DEBUG_SECRET_FILE") of
+        false ->
+            ok;
+        "" ->
+            ok;
+        File ->
+            ok = emqx:set_debug_secret(File)
+    end.
+
+cover_compile(_Opts) ->
+    %% TODO better filter based on Opts,
+    %% e.g. we may want to see coverage info for ehttpc
+    Filter = fun is_emqx_module/1,
+    Modules = find_modules(Filter),
+    Results = cover:compile_beam(Modules),
+    Errors = lists:filter(fun({ok, _}) -> false;
+                             (_) -> true
+                          end, Results),
+    case Errors of
+        [] ->
+            ok;
+        _ ->
+            io:format(user, "failed_to_cover_compile:~n~p~n", [Errors]),
+            throw(failed_to_cover_compile)
+    end.
+
+find_modules(Filter) ->
+    All = code:all_loaded(),
+    F = fun({M, _BeamPath}) -> Filter(M) end,
+    lists:filter(F, All).
+
+is_emqx_module(?MODULE) ->
+    %% do not cover-compile self
+    false;
+is_emqx_module(Module) ->
+    case erlang:atom_to_binary(Module, utf8) of
+        <<"emqx", _/binary>> ->
+            true;
+        _ ->
+            false
+    end.