diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 20018b2d5..b0b316321 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -229,7 +229,7 @@ roots(low) -> {"trace", sc( ref("trace"), - #{} + #{importance => ?IMPORTANCE_HIDDEN} )}, {"crl_cache", sc( @@ -1853,6 +1853,8 @@ fields("trace") -> {"payload_encode", sc(hoconsc:enum([hex, text, hidden]), #{ default => text, + deprecated => {since, "5.0.22"}, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(fields_trace_payload_encode) })} ]. diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index ea6736038..a185d1910 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -59,7 +59,8 @@ -ifdef(TEST). -export([ log_file/2, - find_closest_time/2 + find_closest_time/2, + migrate_trace/0 ]). -endif. @@ -230,6 +231,7 @@ init([]) -> {attributes, record_info(fields, ?TRACE)} ]), ok = mria:wait_for_tables([?TRACE]), + migrate_trace(), {ok, _} = mnesia:subscribe({table, ?TRACE, simple}), ok = filelib:ensure_dir(filename:join([trace_dir(), dummy])), ok = filelib:ensure_dir(filename:join([zip_dir(), dummy])), @@ -358,9 +360,10 @@ start_trace(Trace) -> name = Name, type = Type, filter = Filter, - start_at = Start + start_at = Start, + payload_encode = PayloadEncode } = Trace, - Who = #{name => Name, type => Type, filter => Filter}, + Who = #{name => Name, type => Type, filter => Filter, payload_encode => PayloadEncode}, emqx_trace_handler:install(Who, debug, log_file(Name, Start)). stop_trace(Finished, Started) -> @@ -490,6 +493,8 @@ to_trace(#{type := ip_address, ip_address := Filter} = Trace, Rec) -> end; to_trace(#{type := Type}, _Rec) -> {error, io_lib:format("required ~s field", [Type])}; +to_trace(#{payload_encode := PayloadEncode} = Trace, Rec) -> + to_trace(maps:remove(payload_encode, Trace), Rec#?TRACE{payload_encode = PayloadEncode}); to_trace(#{start_at := StartAt} = Trace, Rec) -> {ok, Sec} = to_system_second(StartAt), to_trace(maps:remove(start_at, Trace), Rec#?TRACE{start_at = Sec}); @@ -573,3 +578,30 @@ filter_cli_handler(Names) -> now_second() -> os:system_time(second). + +migrate_trace() -> + Fields = record_info(fields, ?TRACE), + case mnesia:table_info(emqx_trace, attributes) =:= Fields of + true -> + ok; + false -> + TransFun = fun(Trace) -> + case Trace of + {?TRACE, Name, Type, Filter, Enable, StartAt, EndAt} -> + #?TRACE{ + name = Name, + type = Type, + filter = Filter, + enable = Enable, + start_at = StartAt, + end_at = EndAt, + payload_encode = text, + extra = #{} + }; + #?TRACE{} -> + Trace + end + end, + {atomic, ok} = mnesia:transform_table(?TRACE, TransFun, Fields, ?TRACE), + ok + end. diff --git a/apps/emqx/src/emqx_trace/emqx_trace.hrl b/apps/emqx/src/emqx_trace/emqx_trace.hrl index 096e786dd..62028bcc0 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.hrl +++ b/apps/emqx/src/emqx_trace/emqx_trace.hrl @@ -24,6 +24,8 @@ filter :: emqx_types:topic() | emqx_types:clientid() | emqx_trace:ip_address() | undefined | '_', enable = true :: boolean() | '_', + payload_encode = text :: hex | text | hidden | '_', + extra = #{} :: map() | '_', start_at :: integer() | undefined | '_', end_at :: integer() | undefined | '_' }). diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl index 9c2d2358e..7f586f7a4 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl @@ -160,14 +160,14 @@ filters(#{type := topic, filter := Filter, name := Name}) -> filters(#{type := ip_address, filter := Filter, name := Name}) -> [{ip_address, {fun ?MODULE:filter_ip_address/2, {ensure_list(Filter), Name}}}]. -formatter(#{type := _Type}) -> +formatter(#{type := _Type, payload_encode := PayloadEncode}) -> {emqx_trace_formatter, #{ %% template is for ?SLOG message not ?TRACE. template => [time, " [", level, "] ", msg, "\n"], single_line => true, max_size => unlimited, depth => unlimited, - payload_encode => payload_encode() + payload_encode => PayloadEncode }}. filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) -> diff --git a/apps/emqx/test/emqx_trace_SUITE.erl b/apps/emqx/test/emqx_trace_SUITE.erl index c66808132..3594f0651 100644 --- a/apps/emqx/test/emqx_trace_SUITE.erl +++ b/apps/emqx/test/emqx_trace_SUITE.erl @@ -24,7 +24,9 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). --record(emqx_trace, {name, type, filter, enable = true, start_at, end_at}). +-record(emqx_trace, { + name, type, filter, enable = true, payload_encode = text, extra = #{}, start_at, end_at +}). %%-------------------------------------------------------------------- %% Setups @@ -97,7 +99,9 @@ t_base_create_delete(_Config) -> type => clientid, name => <<"name1">>, start_at => Now, - end_at => Now + 30 * 60 + end_at => Now + 30 * 60, + payload_encode => text, + extra => #{} } ], ?assertEqual(ExpectFormat, emqx_trace:format([TraceRec])), @@ -385,6 +389,48 @@ t_find_closed_time(_Config) -> ?assertEqual(1000, emqx_trace:find_closest_time(Traces, Now)), ok. +t_migrate_trace(_Config) -> + build_new_trace_data(), + build_old_trace_data(), + reload(), + Traces = emqx_trace:format(emqx_trace:list()), + ?assertEqual(2, erlang:length(Traces)), + lists:foreach( + fun(#{name := Name, enable := Enable}) -> + ?assertEqual(true, Enable, Name) + end, + Traces + ), + LoggerIds = logger:get_handler_ids(), + lists:foreach( + fun(Id) -> + ?assertEqual(true, lists:member(Id, LoggerIds), LoggerIds) + end, + [ + trace_topic_test_topic_migrate_new, + trace_topic_test_topic_migrate_old + ] + ), + ok. + +build_new_trace_data() -> + Now = erlang:system_time(second), + {ok, _} = emqx_trace:create([ + {<<"name">>, <<"test_topic_migrate_new">>}, + {<<"type">>, topic}, + {<<"topic">>, <<"/test/migrate/new">>}, + {<<"start_at">>, Now - 10} + ]). + +build_old_trace_data() -> + Now = erlang:system_time(second), + OldAttrs = [name, type, filter, enable, start_at, end_at], + {atomic, ok} = mnesia:transform_table(emqx_trace, ignore, OldAttrs, emqx_trace), + OldTrace = + {emqx_trace, <<"test_topic_migrate_old">>, topic, <<"topic">>, true, Now - 10, Now + 100}, + ok = mnesia:dirty_write(OldTrace), + ok. + reload() -> catch ok = gen_server:stop(emqx_trace), {ok, _Pid} = emqx_trace:start_link(). diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index b93839b0b..73c767a52 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -94,7 +94,8 @@ schema("/trace") -> 409 => emqx_dashboard_swagger:error_codes( [ 'ALREADY_EXISTS', - 'DUPLICATE_CONDITION' + 'DUPLICATE_CONDITION', + 'BAD_TYPE' ], <<"trace already exists">> ) @@ -265,6 +266,19 @@ fields(trace) -> example => running } )}, + {payload_encode, + hoconsc:mk(hoconsc:enum([hex, text, hidden]), #{ + desc => + "" + "Determine the format of the payload format in the trace file.
\n" + "`text`: Text-based protocol or plain text protocol.\n" + " It is recommended when payload is JSON encoded.
\n" + "`hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.
\n" + "`hidden`: payload is obfuscated as `******`" + "", + default => text, + required => false + })}, {start_at, hoconsc:mk( emqx_datetime:epoch_second(), @@ -421,6 +435,11 @@ trace(post, #{body := Param}) -> code => 'DUPLICATE_CONDITION', message => ?TO_BIN([Name, " Duplication Condition"]) }}; + {error, {bad_type, _}} -> + {409, #{ + code => 'BAD_TYPE', + message => <<"Rolling upgrade in progress, create failed">> + }}; {error, Reason} -> {400, #{ code => 'INVALID_PARAMS', diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index 162d07aaa..7922bbb40 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -174,6 +174,13 @@ t_create_failed(_Config) -> {error, {"HTTP/1.1", 409, _}}, request_api(post, api_path("trace"), [GoodName2 | Trace]) ), + %% new name but bad payload-encode + GoodName3 = {<<"name">>, <<"test-name-2">>}, + PayloadEncode = {<<"payload_encode">>, <<"bad">>}, + ?assertMatch( + {error, {"HTTP/1.1", 400, _}}, + request_api(post, api_path("trace"), [GoodName3, PayloadEncode | Trace]) + ), unload(), emqx_trace:clear(),