From dd9938114c7e21c85a1ba06593bad20ca396df6e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 27 Sep 2023 20:10:30 +0800 Subject: [PATCH 1/6] chore(audit): add from field --- apps/emqx/include/logger.hrl | 6 ++++-- apps/emqx/src/emqx_trace/emqx_trace.erl | 9 +++++++-- apps/emqx_ctl/src/emqx_ctl.erl | 2 +- apps/emqx_dashboard/src/emqx_dashboard_audit.erl | 2 +- apps/emqx_machine/src/emqx_machine_boot.erl | 2 +- apps/emqx_machine/src/emqx_machine_terminator.erl | 4 ++-- apps/emqx_machine/src/emqx_restricted_shell.erl | 2 +- 7 files changed, 17 insertions(+), 10 deletions(-) diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index 9bff4b293..7ec18420c 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -61,7 +61,9 @@ ) end). --define(AUDIT(_Level_, _Msg_, _Meta_), begin +-define(AUDIT(_Level_, _From_, _Meta_), ?AUDIT(_Level_, _From_, undefined, _Meta_)). + +-define(AUDIT(_Level_, _From_, _Msg_, _Meta_), begin case emqx_config:get([log, audit], #{enable => false}) of #{enable := false} -> ok; @@ -72,7 +74,7 @@ end). _Level_, [{emqx_audit, fun(L, _) -> L end, undefined, undefined}], {report, _Msg_}, - _Meta_ + _Meta_#{from => _From_} ); gt -> ok diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index 99bbcc5f9..37b9d4ab5 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -87,8 +87,13 @@ log(List, Msg, Meta) -> log(debug, List, Msg, Meta). log(Level, List, Msg, Meta) -> - Log = #{level => Level, meta => enrich_meta(Meta), msg => Msg}, - log_filter(List, Log). + Log = #{level => Level, meta => enrich_meta(Meta)}, + Log1 = + case Msg of + undefined -> Log; + _ -> maps:put(msg, Msg, Log) + end, + log_filter(List, Log1). enrich_meta(Meta) -> case logger:get_process_metadata() of diff --git a/apps/emqx_ctl/src/emqx_ctl.erl b/apps/emqx_ctl/src/emqx_ctl.erl index 8b3eb8058..ad51de837 100644 --- a/apps/emqx_ctl/src/emqx_ctl.erl +++ b/apps/emqx_ctl/src/emqx_ctl.erl @@ -145,7 +145,7 @@ run_command(Cmd, Args) when is_atom(Cmd) -> audit_log( audit_level(Result, Duration), - "from_cli", + cli, #{duration_ms => Duration, cmd => Cmd, args => Args, node => node()} ), Result. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl index 5fdc68f70..443c62e7c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl @@ -30,7 +30,7 @@ log(Meta0) -> Meta = emqx_utils:redact(Meta2), ?AUDIT( Level, - "from_api", + rest_api, Meta#{ from => from(maps:get(auth_type, Meta0, "")), username => binary_to_list(Username), diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index 610931ce0..05caf8742 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -47,7 +47,7 @@ post_boot() -> ok = ensure_apps_started(), ok = print_vsn(), ok = start_autocluster(), - ?AUDIT(alert, "from_cli", #{time => logger:timestamp(), event => "emqx_start"}), + ?AUDIT(alert, cli, #{time => logger:timestamp(), event => "emqx_start"}), ignore. -ifdef(TEST). diff --git a/apps/emqx_machine/src/emqx_machine_terminator.erl b/apps/emqx_machine/src/emqx_machine_terminator.erl index 54452ef41..fe857ff36 100644 --- a/apps/emqx_machine/src/emqx_machine_terminator.erl +++ b/apps/emqx_machine/src/emqx_machine_terminator.erl @@ -67,8 +67,8 @@ graceful() -> %% @doc Shutdown the Erlang VM and wait indefinitely. graceful_wait() -> - ?AUDIT(alert, "from_cli", #{ - time => logger:timestamp(), msg => "run_emqx_stop_to_grace_shutdown" + ?AUDIT(alert, cli, run_emqx_stop_to_grace_shutdown, #{ + time => logger:timestamp() }), ok = graceful(), exit_loop(). diff --git a/apps/emqx_machine/src/emqx_restricted_shell.erl b/apps/emqx_machine/src/emqx_restricted_shell.erl index 115fa478f..77806aa85 100644 --- a/apps/emqx_machine/src/emqx_restricted_shell.erl +++ b/apps/emqx_machine/src/emqx_restricted_shell.erl @@ -112,7 +112,7 @@ max_heap_size_warning(MF, Args) -> log(_, {?MODULE, prompt_func}, [[{history, _}]]) -> ok; log(IsAllow, MF, Args) -> - ?AUDIT(warning, "from_remote_console", #{ + ?AUDIT(warning, remote_console, #{ time => logger:timestamp(), function => MF, args => pp_args(Args), From 71acf121ba1fa481f0bbab765d708b3e9c4d943b Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 27 Sep 2023 20:21:10 +0800 Subject: [PATCH 2/6] chore(audit): distinguish requests from rest_api or dashboard --- apps/emqx_dashboard/src/emqx_dashboard_audit.erl | 11 ++++------- apps/emqx_machine/src/emqx_restricted_shell.erl | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl index 443c62e7c..72b94158d 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl @@ -25,22 +25,19 @@ log(Meta0) -> Duration = erlang:convert_time_unit(ReqEnd - ReqStart, native, millisecond), Level = level(Method, Code, Duration), Username = maps:get(username, Meta0, <<"">>), + From = from(maps:get(auth_type, Meta0, "")), Meta1 = maps:without([req_start, req_end], Meta0), Meta2 = Meta1#{time => logger:timestamp(), duration_ms => Duration}, Meta = emqx_utils:redact(Meta2), ?AUDIT( Level, - rest_api, - Meta#{ - from => from(maps:get(auth_type, Meta0, "")), - username => binary_to_list(Username), - node => node() - } + From, + Meta#{username => binary_to_list(Username), node => node()} ), ok. from(jwt_token) -> "dashboard"; -from(api_key) -> "aip_key"; +from(api_key) -> "rest_api"; from(_) -> "unauthorized". level(get, _Code, _) -> debug; diff --git a/apps/emqx_machine/src/emqx_restricted_shell.erl b/apps/emqx_machine/src/emqx_restricted_shell.erl index 77806aa85..07f7507eb 100644 --- a/apps/emqx_machine/src/emqx_restricted_shell.erl +++ b/apps/emqx_machine/src/emqx_restricted_shell.erl @@ -112,7 +112,7 @@ max_heap_size_warning(MF, Args) -> log(_, {?MODULE, prompt_func}, [[{history, _}]]) -> ok; log(IsAllow, MF, Args) -> - ?AUDIT(warning, remote_console, #{ + ?AUDIT(warning, shell, #{ time => logger:timestamp(), function => MF, args => pp_args(Args), From bdf24d0ec80e35cc53cc0f97e8d53910f514bccc Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 27 Sep 2023 20:32:46 +0800 Subject: [PATCH 3/6] chore: remove the `unauthorized` type for `from` field --- apps/emqx_dashboard/src/emqx_dashboard_audit.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl index 72b94158d..cb5c0f42b 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl @@ -37,8 +37,7 @@ log(Meta0) -> ok. from(jwt_token) -> "dashboard"; -from(api_key) -> "rest_api"; -from(_) -> "unauthorized". +from(_) -> "rest_api". level(get, _Code, _) -> debug; level(_, Code, _) when Code >= 200 andalso Code < 300 -> info; From b52e4ac99d3448b206229a95409f04ca707b1fe0 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 27 Sep 2023 20:56:45 +0800 Subject: [PATCH 4/6] chore: update apps/emqx_machine/src/emqx_restricted_shell.erl Co-authored-by: Zaiming (Stone) Shi --- apps/emqx_machine/src/emqx_restricted_shell.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_machine/src/emqx_restricted_shell.erl b/apps/emqx_machine/src/emqx_restricted_shell.erl index 07f7507eb..a582a3cb8 100644 --- a/apps/emqx_machine/src/emqx_restricted_shell.erl +++ b/apps/emqx_machine/src/emqx_restricted_shell.erl @@ -112,7 +112,7 @@ max_heap_size_warning(MF, Args) -> log(_, {?MODULE, prompt_func}, [[{history, _}]]) -> ok; log(IsAllow, MF, Args) -> - ?AUDIT(warning, shell, #{ + ?AUDIT(warning, erlang_console, #{ time => logger:timestamp(), function => MF, args => pp_args(Args), From 859b122cdd4971b6753ebf18e61278c5b874b5fa Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 27 Sep 2023 20:55:51 +0800 Subject: [PATCH 5/6] chore: format codes --- apps/emqx/src/emqx_trace/emqx_trace.erl | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index 37b9d4ab5..99bbcc5f9 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -87,13 +87,8 @@ log(List, Msg, Meta) -> log(debug, List, Msg, Meta). log(Level, List, Msg, Meta) -> - Log = #{level => Level, meta => enrich_meta(Meta)}, - Log1 = - case Msg of - undefined -> Log; - _ -> maps:put(msg, Msg, Log) - end, - log_filter(List, Log1). + Log = #{level => Level, meta => enrich_meta(Meta), msg => Msg}, + log_filter(List, Log). enrich_meta(Meta) -> case logger:get_process_metadata() of From 6d2adfc2599fb44ecb63af6445302da943a553e7 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 28 Sep 2023 11:04:42 +0800 Subject: [PATCH 6/6] chore(audit): emit the message field --- apps/emqx/include/logger.hrl | 6 ++---- apps/emqx/src/emqx_logger_jsonfmt.erl | 2 ++ apps/emqx_machine/src/emqx_machine_terminator.erl | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index 7ec18420c..d803f67be 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -61,9 +61,7 @@ ) end). --define(AUDIT(_Level_, _From_, _Meta_), ?AUDIT(_Level_, _From_, undefined, _Meta_)). - --define(AUDIT(_Level_, _From_, _Msg_, _Meta_), begin +-define(AUDIT(_Level_, _From_, _Meta_), begin case emqx_config:get([log, audit], #{enable => false}) of #{enable := false} -> ok; @@ -73,7 +71,7 @@ end). emqx_trace:log( _Level_, [{emqx_audit, fun(L, _) -> L end, undefined, undefined}], - {report, _Msg_}, + _Msg = undefined, _Meta_#{from => _From_} ); gt -> diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl index 0f04ee28c..5df4157f4 100644 --- a/apps/emqx/src/emqx_logger_jsonfmt.erl +++ b/apps/emqx/src/emqx_logger_jsonfmt.erl @@ -93,6 +93,8 @@ format(Msg, Meta, Config) -> end, emqx_utils_json:encode(json_obj_root(Data, Config)). +maybe_format_msg(undefined, _Meta, _Config) -> + #{}; maybe_format_msg({report, Report} = Msg, #{report_cb := Cb} = Meta, Config) -> case is_map(Report) andalso Cb =:= ?DEFAULT_FORMATTER of true -> diff --git a/apps/emqx_machine/src/emqx_machine_terminator.erl b/apps/emqx_machine/src/emqx_machine_terminator.erl index fe857ff36..4757507b5 100644 --- a/apps/emqx_machine/src/emqx_machine_terminator.erl +++ b/apps/emqx_machine/src/emqx_machine_terminator.erl @@ -67,8 +67,9 @@ graceful() -> %% @doc Shutdown the Erlang VM and wait indefinitely. graceful_wait() -> - ?AUDIT(alert, cli, run_emqx_stop_to_grace_shutdown, #{ - time => logger:timestamp() + ?AUDIT(alert, cli, #{ + time => logger:timestamp(), + event => emqx_gracefully_stop }), ok = graceful(), exit_loop().