From 7b137c0aafaedf0a36b1065c0808843e6fa26b6f Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 24 May 2024 14:08:35 +0200 Subject: [PATCH 1/3] fix(monitor): reflect durable subscriptions in historical metrics Essentially in the same manner as they are currently reflected in the current metrics. --- .../src/emqx_dashboard_monitor.erl | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl index a11d875f3..5b55a4563 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl @@ -222,8 +222,11 @@ do_sample(Node, Time) -> Res end. -do_sample([], _Time, Res) -> - Res; +do_sample([], _Time, Samples) -> + maps:map( + fun(_TS, Sample) -> adjust_synthetic_cluster_metrics(Sample) end, + Samples + ); do_sample([Node | Nodes], Time, Res) -> case do_sample(Node, Time) of {badrpc, Reason} -> @@ -237,22 +240,28 @@ match_spec(infinity) -> match_spec(Time) -> [{{'_', '$1', '_'}, [{'>=', '$1', Time}], ['$_']}]. -merge_cluster_samplers(Node, Cluster) -> - maps:fold(fun merge_cluster_samplers/3, Cluster, Node). +merge_cluster_samplers(NodeSamples, Cluster) -> + maps:fold(fun merge_cluster_samplers/3, Cluster, NodeSamples). -merge_cluster_samplers(TS, NodeData, Cluster) -> +merge_cluster_samplers(TS, NodeSample0, Cluster) -> + NodeSample = adjust_individual_node_metrics(NodeSample0), case maps:get(TS, Cluster, undefined) of undefined -> - Cluster#{TS => NodeData}; - ClusterData -> - Cluster#{TS => merge_cluster_sampler_map(NodeData, ClusterData)} + Cluster#{TS => NodeSample}; + ClusterSample -> + Cluster#{TS => merge_cluster_sampler_map(NodeSample, ClusterSample)} end. merge_cluster_sampler_map(M1, M2) -> Fun = fun - (topics, Map) -> - Map#{topics => maps:get(topics, M1)}; + (Key, Map) when + %% cluster-synced values + Key =:= topics; + Key =:= subscriptions_durable; + Key =:= disconnected_durable_sessions + -> + Map#{Key => maps:get(Key, M1)}; (Key, Map) -> Map#{Key => maps:get(Key, M1, 0) + maps:get(Key, M2, 0)} end, From fd9655bc35d442c019778d2bdb7b07b50a6ba331 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 24 May 2024 14:32:02 +0200 Subject: [PATCH 2/3] fix(monitor): unify metric names for consistency and simplicity --- apps/emqx_dashboard/include/emqx_dashboard.hrl | 9 +++++++-- .../emqx_dashboard/src/emqx_dashboard_monitor.erl | 12 +++--------- .../src/emqx_dashboard_monitor_api.erl | 11 +++-------- .../test/emqx_dashboard_monitor_SUITE.erl | 15 +++++++++------ 4 files changed, 22 insertions(+), 25 deletions(-) diff --git a/apps/emqx_dashboard/include/emqx_dashboard.hrl b/apps/emqx_dashboard/include/emqx_dashboard.hrl index c2d3479cf..b2e2c1bae 100644 --- a/apps/emqx_dashboard/include/emqx_dashboard.hrl +++ b/apps/emqx_dashboard/include/emqx_dashboard.hrl @@ -73,14 +73,14 @@ -define(GAUGE_SAMPLER_LIST, [ disconnected_durable_sessions, - durable_subscriptions, + subscriptions_durable, subscriptions, topics, connections, live_connections ]). --define(SAMPLER_LIST, ?GAUGE_SAMPLER_LIST ++ ?DELTA_SAMPLER_LIST). +-define(SAMPLER_LIST, (?GAUGE_SAMPLER_LIST ++ ?DELTA_SAMPLER_LIST)). -define(DELTA_SAMPLER_RATE_MAP, #{ received => received_msg_rate, @@ -102,6 +102,11 @@ ] ++ ?LICENSE_QUOTA ). +-define(CLUSTERONLY_SAMPLER_LIST, [ + subscriptions_durable, + disconnected_durable_sessions +]). + -if(?EMQX_RELEASE_EDITION == ee). -define(LICENSE_QUOTA, [license_quota]). -else. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl index 5b55a4563..a82be6a11 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl @@ -118,8 +118,7 @@ current_rate(all) -> current_rate_cluster(); current_rate(Node) when Node == node() -> try - {ok, Rate} = do_call(current_rate), - {ok, adjust_individual_node_metrics(Rate)} + do_call(current_rate) catch _E:R -> ?SLOG(warning, #{msg => "dashboard_monitor_error", reason => R}), @@ -243,8 +242,7 @@ match_spec(Time) -> merge_cluster_samplers(NodeSamples, Cluster) -> maps:fold(fun merge_cluster_samplers/3, Cluster, NodeSamples). -merge_cluster_samplers(TS, NodeSample0, Cluster) -> - NodeSample = adjust_individual_node_metrics(NodeSample0), +merge_cluster_samplers(TS, NodeSample, Cluster) -> case maps:get(TS, Cluster, undefined) of undefined -> Cluster#{TS => NodeSample}; @@ -292,10 +290,6 @@ merge_cluster_rate(Node, Cluster) -> end, maps:fold(Fun, Cluster, Node). -adjust_individual_node_metrics(Metrics0) -> - %% ensure renamed - emqx_utils_maps:rename(durable_subscriptions, subscriptions_durable, Metrics0). - adjust_synthetic_cluster_metrics(Metrics0) -> DSSubs = maps:get(subscriptions_durable, Metrics0, 0), RamSubs = maps:get(subscriptions, Metrics0, 0), @@ -454,7 +448,7 @@ stats(connections) -> emqx_stats:getstat('connections.count'); stats(disconnected_durable_sessions) -> emqx_persistent_session_bookkeeper:get_disconnected_session_count(); -stats(durable_subscriptions) -> +stats(subscriptions_durable) -> emqx_stats:getstat('durable_subscriptions.count'); stats(live_connections) -> emqx_stats:getstat('live_connections.count'); diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index 2560f8f2a..fafa821e3 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -202,11 +202,10 @@ swagger_desc(persisted) -> swagger_desc_format("Messages saved to the durable storage "); swagger_desc(disconnected_durable_sessions) -> <<"Disconnected durable sessions at the time of sampling.", ?APPROXIMATE_DESC>>; -swagger_desc(durable_subscriptions) -> +swagger_desc(subscriptions_durable) -> <<"Subscriptions from durable sessions at the time of sampling.", ?APPROXIMATE_DESC>>; swagger_desc(subscriptions) -> - <<"Subscriptions at the time of sampling (not considering durable sessions).", - ?APPROXIMATE_DESC>>; + <<"Subscriptions at the time of sampling.", ?APPROXIMATE_DESC>>; swagger_desc(topics) -> <<"Count topics at the time of sampling.", ?APPROXIMATE_DESC>>; swagger_desc(connections) -> @@ -252,8 +251,4 @@ swagger_desc_format(Format, Type) -> maybe_reject_cluster_only_metrics(<<"all">>, Rates) -> Rates; maybe_reject_cluster_only_metrics(_Node, Rates) -> - ClusterOnlyMetrics = [ - subscriptions_durable, - disconnected_durable_sessions - ], - maps:without(ClusterOnlyMetrics, Rates). + maps:without(?CLUSTERONLY_SAMPLER_LIST, Rates). diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl index edf7c9bbf..3c297d4a1 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl @@ -224,13 +224,16 @@ t_monitor_current_api(_) -> ], ?assert(maps:is_key(<<"subscriptions_durable">>, Rate)), ?assert(maps:is_key(<<"disconnected_durable_sessions">>, Rate)), - ClusterOnlyMetrics = [durable_subscriptions, disconnected_durable_sessions], {ok, NodeRate} = request(["monitor_current", "nodes", node()]), - [ - ?assert(maps:is_key(atom_to_binary(Key, utf8), NodeRate), #{key => Key, rates => NodeRate}) - || Key <- maps:values(?DELTA_SAMPLER_RATE_MAP) ++ ?GAUGE_SAMPLER_LIST, - not lists:member(Key, ClusterOnlyMetrics) - ], + ExpectedKeys = lists:map( + fun atom_to_binary/1, + (?GAUGE_SAMPLER_LIST ++ maps:values(?DELTA_SAMPLER_RATE_MAP)) -- ?CLUSTERONLY_SAMPLER_LIST + ), + ?assertEqual( + [], + ExpectedKeys -- maps:keys(NodeRate), + NodeRate + ), ?assertNot(maps:is_key(<<"subscriptions_durable">>, NodeRate)), ?assertNot(maps:is_key(<<"subscriptions_ram">>, NodeRate)), ?assertNot(maps:is_key(<<"disconnected_durable_sessions">>, NodeRate)), From c30b881239b4968f5667574007aa78ec238d2a96 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 24 May 2024 14:56:10 +0200 Subject: [PATCH 3/3] test(monitor): verify historical and current metrics are consistent --- .../test/emqx_dashboard_monitor_SUITE.erl | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl index 3c297d4a1..8f561deca 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl @@ -429,6 +429,21 @@ t_persistent_session_stats(Config) -> ?ON(N1, request(["monitor_current"])) ) end), + %% Verify that historical metrics are in line with the current ones. + ?assertMatch( + {ok, [ + #{ + <<"time_stamp">> := _, + <<"connections">> := 3, + <<"disconnected_durable_sessions">> := 1, + <<"topics">> := 8, + <<"subscriptions">> := 8, + <<"subscriptions_ram">> := 4, + <<"subscriptions_durable">> := 4 + } + ]}, + ?ON(N1, request(["monitor"], "latest=1")) + ), {ok, {ok, _}} = ?wait_async_action( emqtt:disconnect(PSClient2),