Merge pull request #8742 from thalesmg/merge-v44-into-v45

Merge v44 into v45

CHANGES-4.4.md

@@ -1,12 +1,16 @@
 # EMQX 4.4 Changes
 
-## v4.4.7
 
-### Enhancements (synced from v4.3.18)
+## v4.4.8
+
+### Enhancements (synced from v4.3.19)
+* Support HTTP API `/trace/:name/detail`.
+
 
 ### Bug fixes
 - Fix: Check if emqx_mod_trace is enabled when the trace file is not found.
 
+
 ## v4.4.5
 
 ### Enhancements (synced from v4.3.16)

apps/emqx_management/src/emqx_management.app.src

@@ -1,6 +1,6 @@
 {application, emqx_management,
  [{description, "EMQ X Management API and CLI"},
-  {vsn, "4.4.6"}, % strict semver, bump manually!
+  {vsn, "4.4.7"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_management_sup]},
   {applications, [kernel,stdlib,emqx_plugin_libs,minirest]},

apps/emqx_management/src/emqx_mgmt_data_backup.erl

@@ -939,6 +939,8 @@ is_version_supported2("4.3") ->
     true;
 is_version_supported2("4.4") ->
     true;
+is_version_supported2("4.5") ->
+    true;
 is_version_supported2(Version) ->
     case re:run(Version, "^4.[02].\\d+$", [{capture, none}]) of
         match ->

apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl

@@ -19,6 +19,7 @@
 
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("kernel/include/file.hrl").
 
 -logger_header("[Tracer]").
 
@@ -42,6 +43,7 @@
         , filename/2
         , trace_dir/0
         , trace_file/1
+        , trace_file_detail/1
         , delete_files_after_send/2
         , is_enable/0
         ]).
@@ -186,6 +188,14 @@ trace_file(File) ->
             {error, Node, Reason}
     end.
 
+trace_file_detail(File) ->
+    FileName = filename:join(trace_dir(), File),
+    Node = atom_to_binary(node()),
+    case file:read_file_info(FileName, [{'time', 'posix'}]) of
+        {ok, #file_info{size = Size, mtime = Mtime}} -> {ok, Node, #{size => Size, mtime => Mtime}};
+        {error, Reason} -> {error, Node, Reason}
+    end.
+
 delete_files_after_send(TraceLog, Zips) ->
     gen_server:cast(?MODULE, {delete_tag, self(), [TraceLog | Zips]}).
 

apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl

@@ -24,6 +24,7 @@
         , update_trace/2
         , delete_trace/2
         , clear_traces/2
+        , trace_file_detail/2
         , download_zip_log/2
         , stream_log_file/2
 ]).
@@ -105,6 +106,15 @@ download_zip_log(#{name := Name}, _Param) ->
             {error, Reason}
     end.
 
+trace_file_detail(#{name := Name}, _Param) ->
+    case emqx_trace:get_trace_filename(Name) of
+        {ok, TraceLog} ->
+            TraceFiles = collect_trace_file_detail(TraceLog),
+            {ok, group_trace_file_detail(TraceLog, TraceFiles)};
+        {error, Reason} ->
+            {error, Reason}
+    end.
+
 group_trace_file(ZipDir, TraceLog, TraceFiles) ->
     lists:foldl(fun(Res, Acc) ->
         case Res of
@@ -124,9 +134,23 @@ group_trace_file(ZipDir, TraceLog, TraceFiles) ->
         end
                 end, [], TraceFiles).
 
+group_trace_file_detail(TraceLog, TraceFiles) ->
+    lists:foldl(fun(Res, Acc) ->
+        case Res of
+            {ok, Node, Info} ->
+                [Info#{node => Node} | Acc];
+            {error, Node, Reason} ->
+                ?LOG(error, "read trace file detail failed:~p", [{Node, TraceLog, Reason}]),
+                Acc
+        end
+                end, [], TraceFiles).
+
 collect_trace_file(TraceLog) ->
     cluster_call(emqx_trace, trace_file, [TraceLog], 60000).
 
+collect_trace_file_detail(TraceLog) ->
+    cluster_call(emqx_trace, trace_file_detail, [TraceLog], 25000).
+
 cluster_call(Mod, Fun, Args, Timeout) ->
     Nodes = ekka_mnesia:running_nodes(),
     {GoodRes, BadNodes} = rpc:multicall(Nodes, Mod, Fun, Args, Timeout),

apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl

@@ -300,7 +300,7 @@ t_trace_file(_Config) ->
 
 t_download_log(_Config) ->
     ClientId = <<"client-test">>,
-    Now = erlang:system_time(second),
+    Now = erlang:system_time(second) - 2,
     Start = to_rfc3339(Now),
     Name = <<"test_client_id">>,
     ok = emqx_trace:create([{<<"name">>, Name},
@@ -314,6 +314,27 @@ t_download_log(_Config) ->
     ok = emqtt:disconnect(Client),
     ok.
 
+t_trace_file_detail(_Config) ->
+    ClientId = <<"client-test1">>,
+    Now = erlang:system_time(second) - 10,
+    Start = to_rfc3339(Now),
+    Name = <<"test_client_id1">>,
+    ok = emqx_trace:create([{<<"name">>, Name},
+        {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]),
+    {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]),
+    {ok, _} = emqtt:connect(Client),
+    [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 10)],
+    ct:sleep(200),
+    ok = emqx_trace_handler_SUITE:filesync(Name, clientid),
+    {ok, [#{mtime := Mtime, node := Node, size := Size} = Detail]}
+        = emqx_trace_api:trace_file_detail(#{name => Name}, []),
+    ct:pal("~p detail:~p~n", [{Name, Now}, Detail]),
+    ?assertEqual(atom_to_binary(node()), Node),
+    ?assert(Size >= 0),
+    ?assert(Mtime >= Now),
+    ok = emqtt:disconnect(Client),
+    ok.
+
 t_find_closed_time(_Config) ->
     DefaultMs = 60 * 15000,
     Now = erlang:system_time(second),

apps/emqx_rule_engine/src/emqx_rule_engine.app.src

@@ -1,6 +1,6 @@
 {application, emqx_rule_engine,
  [{description, "EMQ X Rule Engine"},
-  {vsn, "4.4.7"}, % strict semver, bump manually!
+  {vsn, "4.4.8"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_rule_engine_sup, emqx_rule_registry]},
   {applications, [kernel,stdlib,rulesql,getopt]},

apps/emqx_rule_engine/src/emqx_rule_engine.appup.src

@@ -1,7 +1,8 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.4.6",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+  [{"4.4.7",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+   {"4.4.6",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.4.5",
     [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
@@ -72,7 +73,8 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.4.6",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+  [{"4.4.7",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+   {"4.4.6",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.4.5",
     [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},

bin/install_upgrade.escript

@@ -437,6 +437,8 @@ validate_target_version(TargetVersion, TargetNode) ->
     CurrentVersion = current_release_version(TargetNode),
     case {get_major_minor_vsn(CurrentVersion), get_major_minor_vsn(TargetVersion)} of
         {{Major, Minor}, {Major, Minor}} -> ok;
+        {{<<"4">>, <<"5">>}, {<<"4">>, <<"4">>}} -> ok;
+        {{<<"4">>, <<"4">>}, {<<"4">>, <<"5">>}} -> ok;
         _ ->
             ?INFO("Cannot upgrade/downgrade to ~s from ~s~n"
                   "We only support relup between patch versions",

include/emqx_release.hrl

@@ -29,7 +29,7 @@
 
 -ifndef(EMQX_ENTERPRISE).
 
--define(EMQX_RELEASE, {opensource, "4.4.7-beta.1"}).
+-define(EMQX_RELEASE, {opensource, "4.4.8"}).
 
 -else.
 

lib-ce/emqx_modules/src/emqx_mod_trace_api.erl

@@ -22,6 +22,7 @@
         , disable_trace/2
         , delete_trace/2
         , clear_traces/2
+        , trace_file_detail/2
         , download_zip_log/2
         , stream_log_file/2
 ]).
@@ -58,6 +59,12 @@
             func   => disable_trace,
             descr  => "stop trace"}).
 
+-rest_api(#{name   => trace_file_detail,
+            method => 'GET',
+            path   => "/trace/:bin:name/detail",
+            func   => trace_file_detail,
+            descr  => "view trace file's detail"}).
+
 -rest_api(#{name   => download_zip_log,
             method => 'GET',
             path   => "/trace/:bin:name/download",
@@ -101,6 +108,12 @@ disable_trace(#{name := Name}, Params) ->
         false -> return(?NOT_STARTED)
     end.
 
+trace_file_detail(Path, Params) ->
+    case emqx_trace_api:trace_file_detail(Path, Params) of
+        {ok, Detail} -> return({ok, Detail});
+        {error, Reason} ->  return({error, 'NOT_FOUND', Reason})
+    end.
+
 download_zip_log(Path, Params) ->
     case emqx_trace_api:download_zip_log(Path, Params) of
         {ok, File} -> minirest:return_file(File);

lib-ce/emqx_modules/src/emqx_modules.app.src

@@ -1,6 +1,6 @@
 {application, emqx_modules,
  [{description, "EMQ X Module Management"},
-  {vsn, "4.4.5"},
+  {vsn, "4.4.6"},
   {modules, []},
   {applications, [kernel,stdlib]},
   {mod, {emqx_modules_app, []}},

lib-ce/emqx_modules/src/emqx_modules.appup.src

@@ -1,15 +1,22 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.4.4",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
-   {"4.4.3",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
+  [{"4.4.5",[{load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]},
+   {"4.4.4",
+    [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]},
+   {"4.4.3",
+     [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},
+      {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]},
    {"4.4.2",
     [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_modules,brutal_purge,soft_purge,[]}]},
    {"4.4.1",
     [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},
      {load_module,emqx_modules,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]},
    {"4.4.0",
     [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},
@@ -20,15 +27,22 @@
      {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.4.4",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
-   {"4.4.3",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
+  [{"4.4.5",[{load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]},
+   {"4.4.4",
+    [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]},
+   {"4.4.3",
+    [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]},
    {"4.4.2",
     [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_modules,brutal_purge,soft_purge,[]}]},
    {"4.4.1",
     [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},
      {load_module,emqx_modules,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]},
    {"4.4.0",
     [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},

lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl

@@ -141,6 +141,13 @@ t_stream_log(_Config) ->
     #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta1, <<"items">> := Bin1}} = json(Binary1),
     ?assertEqual(#{<<"position">> => 30, <<"bytes">> => 10}, Meta1),
     ?assertEqual(10, byte_size(Bin1)),
+
+    {ok, Detail} = request_api(get, api_path("trace/test_stream_log/detail"), Header),
+    #{<<"data">> := [#{<<"size">> := Size, <<"node">> := Node, <<"mtime">> := Mtime}],
+        <<"code">> := 0} = json(Detail),
+    ?assertEqual(atom_to_binary(node()), Node),
+    ?assert(Size > 0),
+    ?assert(Mtime >= Now),
     unload(),
     ok.
 

src/emqx.app.src

@@ -6,7 +6,7 @@
   %% the emqx `release' version, which in turn is comprised of several
   %% apps, one of which is this.  See `emqx_release.hrl' for more
   %% info.
-  {vsn, "4.4.7"}, % strict semver, bump manually!
+  {vsn, "4.4.8"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
   {applications, [ kernel

src/emqx.appup.src

@@ -1,7 +1,10 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.4.6",
+  [{"4.4.7",
+    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_relup,brutal_purge,soft_purge,[]}]},
+   {"4.4.6",
     [{load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_relup,brutal_purge,soft_purge,[]}]},
    {"4.4.5",
@@ -162,7 +165,10 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.4.6",
+  [{"4.4.7",
+    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_relup,brutal_purge,soft_purge,[]}]},
+   {"4.4.6",
     [{load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_relup,brutal_purge,soft_purge,[]}]},
    {"4.4.5",