From f65168982b27ba41ab655980344968e79a940c8e Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 28 May 2024 06:02:31 +0200 Subject: [PATCH 1/4] fix(oracle action trace): parameters should not be rendered as IO Data This forces the parameters to the database statement to be rendered as a JSON array in JSON traces instead of being rendered as a string when the parameters are interpreted as IO data. Fixes: https://emqx.atlassian.net/browse/EMQX-12433 --- apps/emqx/src/emqx_logger_jsonfmt.erl | 32 +++++++++++++++++++++++++++ apps/emqx_oracle/src/emqx_oracle.erl | 12 +++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl index 45697c1df..33763418c 100644 --- a/apps/emqx/src/emqx_logger_jsonfmt.erl +++ b/apps/emqx/src/emqx_logger_jsonfmt.erl @@ -270,6 +270,8 @@ json(L, Config) when is_list(L) -> end; json(Map, Config) when is_map(Map) -> best_effort_json_obj(Map, Config); +json({'$array$', List}, Config) when is_list(List) -> + [json(I, Config) || I <- List]; json(Term, Config) -> do_format_msg("~p", [Term], Config). @@ -448,6 +450,36 @@ best_effort_json_test() -> <<"[\n {\n \"key\" : [\n \n ]\n }\n]">>, best_effort_json([#{key => []}]) ), + %% List is IO Data + ?assertMatch( + #{<<"what">> => <<"hej\n">>}, + emqx_utils_json:decode(emqx_logger_jsonfmt:best_effort_json(#{what => [<<"hej">>, 10]})) + ), + %% Force list to be interpreted as an array + ?assertMatch( + #{<<"what">> => [<<"hej">>, 10]}, + emqx_utils_json:decode( + emqx_logger_jsonfmt:best_effort_json(#{what => {'$array$', [<<"hej">>, 10]}}) + ) + ), + %% IO Data inside an array + ?assertMatch( + #{<<"what">> => [<<"hej">>, 10, <<"hej\n">>]}, + emqx_utils_json:decode( + emqx_logger_jsonfmt:best_effort_json(#{ + what => {'$array$', [<<"hej">>, 10, [<<"hej">>, 10]]} + }) + ) + ), + %% Array inside an array + ?assertMatch( + #{<<"what">> => [<<"hej">>, 10, [<<"hej">>, 10]]}, + emqx_utils_json:decode( + emqx_logger_jsonfmt:best_effort_json(#{ + what => {'$array$', [<<"hej">>, 10, {'$array$', [<<"hej">>, 10]}]} + }) + ) + ), ok. config() -> diff --git a/apps/emqx_oracle/src/emqx_oracle.erl b/apps/emqx_oracle/src/emqx_oracle.erl index e90665cc4..5b25e049a 100644 --- a/apps/emqx_oracle/src/emqx_oracle.erl +++ b/apps/emqx_oracle/src/emqx_oracle.erl @@ -8,6 +8,7 @@ -include_lib("emqx_resource/include/emqx_resource.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_trace.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -define(UNHEALTHY_TARGET_MSG, @@ -288,7 +289,7 @@ on_sql_query(InstId, ChannelID, PoolName, Type, ApplyMode, NameOrSQL, Data) -> type => Type, apply_mode => ApplyMode, name_or_sql => NameOrSQL, - data => Data + data => #emqx_trace_format_func_data{function = fun trace_format_data/1, data = Data} }), case ecpool:pick_and_do(PoolName, {?MODULE, Type, [NameOrSQL, Data]}, ApplyMode) of {error, Reason} = Result -> @@ -317,6 +318,15 @@ on_sql_query(InstId, ChannelID, PoolName, Type, ApplyMode, NameOrSQL, Data) -> Result end. +trace_format_data(Data0) -> + %% In batch request, we get a two level list + {'$array$', lists:map(fun insert_array_marker_if_list/1, Data0)}. + +insert_array_marker_if_list(List) when is_list(List) -> + {'$array$', List}; +insert_array_marker_if_list(Item) -> + Item. + on_get_status(_InstId, #{pool_name := Pool} = _State) -> case emqx_resource_pool:health_check_workers(Pool, fun ?MODULE:do_get_status/1) of true -> From e55bda6fe6ad884a771e9e6282ab776c56e6044f Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 28 May 2024 10:56:59 +0200 Subject: [PATCH 2/4] chore: bump app versions Bump app version as required to pass the check which is done by the following script: ./scripts/apps-version-check.sh --- apps/emqx_oracle/src/emqx_oracle.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_oracle/src/emqx_oracle.app.src b/apps/emqx_oracle/src/emqx_oracle.app.src index ce05bdfae..09aa4e589 100644 --- a/apps/emqx_oracle/src/emqx_oracle.app.src +++ b/apps/emqx_oracle/src/emqx_oracle.app.src @@ -1,6 +1,6 @@ {application, emqx_oracle, [ {description, "EMQX Enterprise Oracle Database Connector"}, - {vsn, "0.2.0"}, + {vsn, "0.2.1"}, {registered, []}, {applications, [ kernel, From e3cb63f59a41b17c17197ca3113743541a373f9f Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 28 May 2024 11:08:07 +0200 Subject: [PATCH 3/4] docs: add change log entry --- changes/ee/fix-13136.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ee/fix-13136.en.md diff --git a/changes/ee/fix-13136.en.md b/changes/ee/fix-13136.en.md new file mode 100644 index 000000000..8593c6fdb --- /dev/null +++ b/changes/ee/fix-13136.en.md @@ -0,0 +1 @@ +The template-rendered traces for Oracle actions have been enhanced for better readability. From 5e3de1a4e3f204a45e4df7f4355307796a51087d Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 28 May 2024 17:15:32 +0200 Subject: [PATCH 4/4] fix(emqx_logger_jsonfmt:best_effort_json_test): test cases --- apps/emqx/src/emqx_logger_jsonfmt.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl index 33763418c..d8dcc77dc 100644 --- a/apps/emqx/src/emqx_logger_jsonfmt.erl +++ b/apps/emqx/src/emqx_logger_jsonfmt.erl @@ -452,19 +452,19 @@ best_effort_json_test() -> ), %% List is IO Data ?assertMatch( - #{<<"what">> => <<"hej\n">>}, + #{<<"what">> := <<"hej\n">>}, emqx_utils_json:decode(emqx_logger_jsonfmt:best_effort_json(#{what => [<<"hej">>, 10]})) ), %% Force list to be interpreted as an array ?assertMatch( - #{<<"what">> => [<<"hej">>, 10]}, + #{<<"what">> := [<<"hej">>, 10]}, emqx_utils_json:decode( emqx_logger_jsonfmt:best_effort_json(#{what => {'$array$', [<<"hej">>, 10]}}) ) ), %% IO Data inside an array ?assertMatch( - #{<<"what">> => [<<"hej">>, 10, <<"hej\n">>]}, + #{<<"what">> := [<<"hej">>, 10, <<"hej\n">>]}, emqx_utils_json:decode( emqx_logger_jsonfmt:best_effort_json(#{ what => {'$array$', [<<"hej">>, 10, [<<"hej">>, 10]]} @@ -473,7 +473,7 @@ best_effort_json_test() -> ), %% Array inside an array ?assertMatch( - #{<<"what">> => [<<"hej">>, 10, [<<"hej">>, 10]]}, + #{<<"what">> := [<<"hej">>, 10, [<<"hej">>, 10]]}, emqx_utils_json:decode( emqx_logger_jsonfmt:best_effort_json(#{ what => {'$array$', [<<"hej">>, 10, {'$array$', [<<"hej">>, 10]}]}