diff --git a/Makefile b/Makefile index 739195926..47e665de2 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ endif # Dashboard version # from https://github.com/emqx/emqx-dashboard5 export EMQX_DASHBOARD_VERSION ?= v1.6.1 -export EMQX_EE_DASHBOARD_VERSION ?= e1.5.0-beta.3 +export EMQX_EE_DASHBOARD_VERSION ?= e1.5.0-beta.8 PROFILE ?= emqx REL_PROFILES := emqx emqx-enterprise diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 41f9f2c35..987c19535 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -59,6 +59,7 @@ {emqx_persistent_session_ds,1}. {emqx_plugins,1}. {emqx_prometheus,1}. +{emqx_prometheus,2}. {emqx_resource,1}. {emqx_retainer,1}. {emqx_retainer,2}. diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 8b173d493..bfd981854 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -28,7 +28,7 @@ {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.11.1"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.18.1"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.18.3"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.4"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, diff --git a/apps/emqx/src/emqx_stats.erl b/apps/emqx/src/emqx_stats.erl index dfd3115f0..9685823ff 100644 --- a/apps/emqx/src/emqx_stats.erl +++ b/apps/emqx/src/emqx_stats.erl @@ -166,6 +166,8 @@ names() -> emqx_live_connections_max, emqx_sessions_count, emqx_sessions_max, + emqx_channels_count, + emqx_channels_max, emqx_topics_count, emqx_topics_max, emqx_suboptions_count, diff --git a/apps/emqx/test/emqx_persistent_messages_SUITE.erl b/apps/emqx/test/emqx_persistent_messages_SUITE.erl index 8a63d46be..a6ad9181a 100644 --- a/apps/emqx/test/emqx_persistent_messages_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_messages_SUITE.erl @@ -476,7 +476,6 @@ t_metrics_not_dropped(_Config) -> {ok, _, [?RC_GRANTED_QOS_1]} = emqtt:subscribe(Sub, <<"t/+">>, ?QOS_1), emqtt:publish(Pub, <<"t/ps">>, <<"payload">>, ?QOS_1), ?assertMatch([_], receive_messages(1)), - DroppedAfter = emqx_metrics:val('messages.dropped'), DroppedNoSubAfter = emqx_metrics:val('messages.dropped.no_subscribers'), diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index e27748610..f48a44df2 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -103,33 +103,37 @@ load() -> Bridges = emqx:get_config([?ROOT_KEY], #{}), - lists:foreach( + emqx_utils:pforeach( fun({Type, NamedConf}) -> - lists:foreach( + emqx_utils:pforeach( fun({Name, Conf}) -> %% fetch opts for `emqx_resource_buffer_worker` ResOpts = emqx_resource:fetch_creation_opts(Conf), safe_load_bridge(Type, Name, Conf, ResOpts) end, - maps:to_list(NamedConf) + maps:to_list(NamedConf), + infinity ) end, - maps:to_list(Bridges) + maps:to_list(Bridges), + infinity ). unload() -> unload_hook(), Bridges = emqx:get_config([?ROOT_KEY], #{}), - lists:foreach( + emqx_utils:pforeach( fun({Type, NamedConf}) -> - lists:foreach( + emqx_utils:pforeach( fun({Name, _Conf}) -> _ = emqx_bridge_resource:stop(Type, Name) end, - maps:to_list(NamedConf) + maps:to_list(NamedConf), + infinity ) end, - maps:to_list(Bridges) + maps:to_list(Bridges), + infinity ). safe_load_bridge(Type, Name, Conf, Opts) -> @@ -284,15 +288,15 @@ pre_config_update([?ROOT_KEY], NewConf, _RawConf) -> post_config_update([?ROOT_KEY], _Req, NewConf, OldConf, _AppEnv) -> #{added := Added, removed := Removed, changed := Updated} = diff_confs(NewConf, OldConf), - %% The config update will be failed if any task in `perform_bridge_changes` failed. Result = perform_bridge_changes([ - #{action => fun emqx_bridge_resource:remove/4, data => Removed}, + #{action => fun emqx_bridge_resource:remove/4, action_name => remove, data => Removed}, #{ action => fun emqx_bridge_resource:create/4, + action_name => create, data => Added, on_exception_fn => fun emqx_bridge_resource:remove/4 }, - #{action => fun emqx_bridge_resource:update/4, data => Updated} + #{action => fun emqx_bridge_resource:update/4, action_name => update, data => Updated} ]), ok = unload_hook(), ok = load_hook(NewConf), @@ -534,28 +538,21 @@ convert_certs(BridgesConf) -> ). perform_bridge_changes(Tasks) -> - perform_bridge_changes(Tasks, ok). + perform_bridge_changes(Tasks, []). -perform_bridge_changes([], Result) -> - Result; -perform_bridge_changes([#{action := Action, data := MapConfs} = Task | Tasks], Result0) -> +perform_bridge_changes([], Errors) -> + case Errors of + [] -> ok; + _ -> {error, Errors} + end; +perform_bridge_changes([#{action := Action, data := MapConfs} = Task | Tasks], Errors0) -> OnException = maps:get(on_exception_fn, Task, fun(_Type, _Name, _Conf, _Opts) -> ok end), - Result = maps:fold( - fun - ({_Type, _Name}, _Conf, {error, Reason}) -> - {error, Reason}; - %% for emqx_bridge_resource:update/4 - ({Type, Name}, {OldConf, Conf}, _) -> - ResOpts = emqx_resource:fetch_creation_opts(Conf), - case Action(Type, Name, {OldConf, Conf}, ResOpts) of - {error, Reason} -> {error, Reason}; - Return -> Return - end; - ({Type, Name}, Conf, _) -> - ResOpts = emqx_resource:fetch_creation_opts(Conf), - try Action(Type, Name, Conf, ResOpts) of - {error, Reason} -> {error, Reason}; - Return -> Return + Results = emqx_utils:pmap( + fun({{Type, Name}, Conf}) -> + ResOpts = creation_opts(Conf), + Res = + try + Action(Type, Name, Conf, ResOpts) catch Kind:Error:Stacktrace -> ?SLOG(error, #{ @@ -567,13 +564,34 @@ perform_bridge_changes([#{action := Action, data := MapConfs} = Task | Tasks], R stacktrace => Stacktrace }), OnException(Type, Name, Conf, ResOpts), - erlang:raise(Kind, Error, Stacktrace) - end + {error, Error} + end, + {{Type, Name}, Res} end, - Result0, - MapConfs + maps:to_list(MapConfs), + infinity ), - perform_bridge_changes(Tasks, Result). + Errs = lists:filter( + fun + ({_TypeName, {error, _}}) -> true; + (_) -> false + end, + Results + ), + Errors = + case Errs of + [] -> + Errors0; + _ -> + #{action_name := ActionName} = Task, + [#{action => ActionName, errors => Errs} | Errors0] + end, + perform_bridge_changes(Tasks, Errors). + +creation_opts({_OldConf, Conf}) -> + emqx_resource:fetch_creation_opts(Conf); +creation_opts(Conf) -> + emqx_resource:fetch_creation_opts(Conf). diff_confs(NewConfs, OldConfs) -> emqx_utils_maps:diff_maps( diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index f53503b86..e0b921a1c 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -1132,15 +1132,14 @@ maybe_unwrap({error, not_implemented}) -> maybe_unwrap(RpcMulticallResult) -> emqx_rpc:unwrap_erpc(RpcMulticallResult). -supported_versions(start_bridge_to_node) -> bpapi_version_range(2, latest); -supported_versions(start_bridges_to_all_nodes) -> bpapi_version_range(2, latest); -supported_versions(get_metrics_from_all_nodes) -> bpapi_version_range(4, latest); -supported_versions(_Call) -> bpapi_version_range(1, latest). +supported_versions(start_bridge_to_node) -> bpapi_version_range(2, 6); +supported_versions(start_bridges_to_all_nodes) -> bpapi_version_range(2, 6); +supported_versions(get_metrics_from_all_nodes) -> bpapi_version_range(4, 6); +supported_versions(_Call) -> bpapi_version_range(1, 6). %% [From, To] (inclusive on both ends) -bpapi_version_range(From, latest) -> - ThisNodeVsn = emqx_bpapi:supported_version(node(), ?BPAPI_NAME), - lists:seq(From, ThisNodeVsn). +bpapi_version_range(From, To) -> + lists:seq(From, To). redact(Term) -> emqx_utils:redact(Term). diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index b69882080..7ee384b84 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -182,17 +182,20 @@ load() -> load_bridges(RootName) -> Bridges = emqx:get_config([RootName], #{}), - lists:foreach( + _ = emqx_utils:pmap( fun({Type, Bridge}) -> - lists:foreach( + emqx_utils:pmap( fun({Name, BridgeConf}) -> install_bridge_v2(RootName, Type, Name, BridgeConf) end, - maps:to_list(Bridge) + maps:to_list(Bridge), + infinity ) end, - maps:to_list(Bridges) - ). + maps:to_list(Bridges), + infinity + ), + ok. unload() -> unload_bridges(?ROOT_KEY_ACTIONS), @@ -204,17 +207,20 @@ unload() -> unload_bridges(ConfRooKey) -> Bridges = emqx:get_config([ConfRooKey], #{}), - lists:foreach( + _ = emqx_utils:pmap( fun({Type, Bridge}) -> - lists:foreach( + emqx_utils:pmap( fun({Name, BridgeConf}) -> uninstall_bridge_v2(ConfRooKey, Type, Name, BridgeConf) end, - maps:to_list(Bridge) + maps:to_list(Bridge), + infinity ) end, - maps:to_list(Bridges) - ). + maps:to_list(Bridges), + infinity + ), + ok. %%==================================================================== %% CRUD API @@ -641,8 +647,8 @@ reset_metrics(ConfRootKey, Type, Name) -> reset_metrics_helper(_ConfRootKey, _Type, _Name, #{enable := false}) -> ok; reset_metrics_helper(ConfRootKey, BridgeV2Type, BridgeName, #{connector := ConnectorName}) -> - BridgeV2Id = id_with_root_name(ConfRootKey, BridgeV2Type, BridgeName, ConnectorName), - ok = emqx_metrics_worker:reset_metrics(?RES_METRICS, BridgeV2Id); + ResourceId = id_with_root_name(ConfRootKey, BridgeV2Type, BridgeName, ConnectorName), + emqx_resource:reset_metrics(ResourceId); reset_metrics_helper(_, _, _, _) -> {error, not_found}. @@ -1059,7 +1065,6 @@ post_config_update([ConfRootKey], _Req, NewConf, OldConf, _AppEnv) when -> #{added := Added, removed := Removed, changed := Updated} = diff_confs(NewConf, OldConf), - %% The config update will be failed if any task in `perform_bridge_changes` failed. RemoveFun = fun(Type, Name, Conf) -> uninstall_bridge_v2(ConfRootKey, Type, Name, Conf) end, @@ -1071,13 +1076,14 @@ post_config_update([ConfRootKey], _Req, NewConf, OldConf, _AppEnv) when install_bridge_v2(ConfRootKey, Type, Name, Conf) end, Result = perform_bridge_changes([ - #{action => RemoveFun, data => Removed}, + #{action => RemoveFun, action_name => remove, data => Removed}, #{ action => CreateFun, + action_name => create, data => Added, on_exception_fn => fun emqx_bridge_resource:remove/4 }, - #{action => UpdateFun, data => Updated} + #{action => UpdateFun, action_name => update, data => Updated} ]), reload_message_publish_hook(NewConf), ?tp(bridge_post_config_update_done, #{}), @@ -1141,26 +1147,20 @@ do_flatten_confs(Type, Conf0) -> [{{Type, Name}, Conf} || {Name, Conf} <- maps:to_list(Conf0)]. perform_bridge_changes(Tasks) -> - perform_bridge_changes(Tasks, ok). + perform_bridge_changes(Tasks, []). -perform_bridge_changes([], Result) -> - Result; -perform_bridge_changes([#{action := Action, data := MapConfs} = Task | Tasks], Result0) -> +perform_bridge_changes([], Errors) -> + case Errors of + [] -> ok; + _ -> {error, Errors} + end; +perform_bridge_changes([#{action := Action, data := MapConfs} = Task | Tasks], Errors0) -> OnException = maps:get(on_exception_fn, Task, fun(_Type, _Name, _Conf, _Opts) -> ok end), - Result = maps:fold( - fun - ({_Type, _Name}, _Conf, {error, Reason}) -> - {error, Reason}; - %% for update - ({Type, Name}, {OldConf, Conf}, _) -> - case Action(Type, Name, {OldConf, Conf}) of - {error, Reason} -> {error, Reason}; - Return -> Return - end; - ({Type, Name}, Conf, _) -> - try Action(Type, Name, Conf) of - {error, Reason} -> {error, Reason}; - Return -> Return + Results = emqx_utils:pmap( + fun({{Type, Name}, Conf}) -> + Res = + try + Action(Type, Name, Conf) catch Kind:Error:Stacktrace -> ?SLOG(error, #{ @@ -1172,13 +1172,29 @@ perform_bridge_changes([#{action := Action, data := MapConfs} = Task | Tasks], R stacktrace => Stacktrace }), OnException(Type, Name, Conf), - erlang:raise(Kind, Error, Stacktrace) - end + {error, Error} + end, + {{Type, Name}, Res} end, - Result0, - MapConfs + maps:to_list(MapConfs), + infinity ), - perform_bridge_changes(Tasks, Result). + Errs = lists:filter( + fun + ({_TypeName, {error, _}}) -> true; + (_) -> false + end, + Results + ), + Errors = + case Errs of + [] -> + Errors0; + _ -> + #{action_name := ActionName} = Task, + [#{action => ActionName, errors => Errs} | Errors0] + end, + perform_bridge_changes(Tasks, Errors). fill_defaults(Type, RawConf, TopLevelConf, SchemaModule) -> PackedConf = pack_bridge_conf(Type, RawConf, TopLevelConf), diff --git a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl index e8a500e85..6dcd24355 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl @@ -1052,12 +1052,11 @@ do_bpapi_call_vsn(Version, Call, Args) -> is_supported_version(Version, Call) -> lists:member(Version, supported_versions(Call)). -supported_versions(_Call) -> bpapi_version_range(6, latest). +supported_versions(_Call) -> bpapi_version_range(6, 6). %% [From, To] (inclusive on both ends) -bpapi_version_range(From, latest) -> - ThisNodeVsn = emqx_bpapi:supported_version(node(), ?BPAPI_NAME), - lists:seq(From, ThisNodeVsn). +bpapi_version_range(From, To) -> + lists:seq(From, To). maybe_unwrap({error, not_implemented}) -> {error, not_implemented}; @@ -1178,6 +1177,9 @@ format_resource( ) ). +%% FIXME: +%% missing metrics: +%% 'retried.success' and 'retried.failed' format_metrics(#{ counters := #{ 'dropped' := Dropped, diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl index f3b8a29d7..ba631f71a 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl @@ -606,12 +606,22 @@ t_load_no_matching_connector(_Config) -> }, ?assertMatch( {error, - {post_config_update, _HandlerMod, #{ - bridge_name := my_test_bridge_update, - connector_name := <<"unknown">>, - bridge_type := _, - reason := <<"connector_not_found_or_wrong_type">> - }}}, + {post_config_update, _HandlerMod, [ + #{ + errors := [ + { + {_, my_test_bridge_update}, + {error, #{ + bridge_name := my_test_bridge_update, + connector_name := <<"unknown">>, + bridge_type := _, + reason := <<"connector_not_found_or_wrong_type">> + }} + } + ], + action := update + } + ]}}, update_root_config(RootConf0) ), @@ -623,12 +633,22 @@ t_load_no_matching_connector(_Config) -> }, ?assertMatch( {error, - {post_config_update, _HandlerMod, #{ - bridge_name := my_test_bridge_new, - connector_name := <<"unknown">>, - bridge_type := _, - reason := <<"connector_not_found_or_wrong_type">> - }}}, + {post_config_update, _HandlerMod, [ + #{ + errors := [ + { + {_, my_test_bridge_new}, + {error, #{ + bridge_name := my_test_bridge_new, + connector_name := <<"unknown">>, + bridge_type := _, + reason := <<"connector_not_found_or_wrong_type">> + }} + } + ], + action := create + } + ]}}, update_root_config(RootConf1) ), diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index fc9c9573f..4f98baebf 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -286,6 +286,10 @@ init_mocks() -> ok = meck:expect(?CONNECTOR_IMPL, on_get_channels, fun(ResId) -> emqx_bridge_v2:get_channels_for_connector(ResId) end), + meck:expect(?CONNECTOR_IMPL, on_query_async, fun(_ResId, _Req, ReplyFunAndArgs, _ConnState) -> + emqx_resource:apply_reply_fun(ReplyFunAndArgs, ok), + {ok, self()} + end), ok. clear_resources() -> @@ -378,6 +382,9 @@ enable_path(Enable, BridgeID) -> publish_message(Topic, Body, Config) -> Node = ?config(node, Config), + publish_message(Topic, Body, Node, Config). + +publish_message(Topic, Body, Node, _Config) -> erpc:call(Node, emqx, publish, [emqx_message:make(Topic, Body)]). update_config(Path, Value, Config) -> @@ -524,6 +531,17 @@ get_common_values(Kind, FnName) -> } end. +maybe_get_other_node(Config) -> + %% In the single node test group, this simply returns the lone node. Otherwise, it'll + %% return a node that's not the primary one that receives API calls. + PrimaryNode = ?config(node, Config), + case proplists:get_value(cluster_nodes, Config, []) -- [PrimaryNode] of + [] -> + PrimaryNode; + [OtherNode | _] -> + OtherNode + end. + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ @@ -1385,7 +1403,8 @@ t_reset_metrics(Config) -> ActionID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ActionName), Body = <<"my msg">>, - _ = publish_message(?MQTT_LOCAL_TOPIC, Body, Config), + OtherNode = maybe_get_other_node(Config), + _ = publish_message(?MQTT_LOCAL_TOPIC, Body, OtherNode, Config), ?retry( _Sleep0 = 200, _Retries0 = 20, @@ -1400,16 +1419,30 @@ t_reset_metrics(Config) -> {ok, 204, <<>>} = request(put, uri([?ACTIONS_ROOT, ActionID, "metrics", "reset"]), Config), - ?retry( + Res = ?retry( _Sleep0 = 200, _Retries0 = 20, - ?assertMatch( - {ok, 200, #{ - <<"metrics">> := #{<<"matched">> := 0}, - <<"node_metrics">> := [#{<<"metrics">> := #{}} | _] - }}, - request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) - ) + begin + Res0 = request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config), + ?assertMatch( + {ok, 200, #{ + <<"metrics">> := #{<<"matched">> := 0}, + <<"node_metrics">> := [#{<<"metrics">> := #{}} | _] + }}, + Res0 + ), + Res0 + end + ), + {ok, 200, #{<<"node_metrics">> := NodeMetrics}} = Res, + ?assert( + lists:all( + fun(#{<<"metrics">> := #{<<"matched">> := Matched}}) -> + Matched == 0 + end, + NodeMetrics + ), + #{node_metrics => NodeMetrics} ), ok. diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es.app.src b/apps/emqx_bridge_es/src/emqx_bridge_es.app.src index 9e98cd33e..34b07d305 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es.app.src +++ b/apps/emqx_bridge_es/src/emqx_bridge_es.app.src @@ -10,8 +10,7 @@ {applications, [ kernel, stdlib, - emqx_resource, - emqx_connector + emqx_resource ]}, {env, []}, {licenses, ["Business Source License 1.1"]}, diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl index 8b68af10f..256645c31 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl @@ -34,6 +34,7 @@ ]). -export([render_template/2]). +-export([convert_server/2]). %% emqx_connector_resource behaviour callbacks -export([connector_config/2]). @@ -92,7 +93,7 @@ connector_example_values() -> <<"username">> => <<"root">>, <<"password">> => <<"******">> }, - base_url => <<"http://127.0.0.1:9200/">>, + server => <<"127.0.0.1:9200">>, connect_timeout => <<"15s">>, pool_type => <<"random">>, pool_size => 8, @@ -116,14 +117,7 @@ fields(config) -> fields("connection_fields"); fields("connection_fields") -> [ - {base_url, - ?HOCON( - emqx_schema:url(), - #{ - required => true, - desc => ?DESC(emqx_bridge_es, "config_base_url") - } - )}, + {server, server()}, {authentication, ?HOCON( ?UNION([?R_REF(auth_basic)]), @@ -158,30 +152,36 @@ desc(auth_basic) -> "Basic Authentication"; desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for Elastic Search using `", string:to_upper(Method), "` method."]; +desc("server") -> + ?DESC("server"); desc(_) -> undefined. +server() -> + Meta = #{ + required => true, + default => <<"127.0.0.1:9200">>, + desc => ?DESC("server"), + converter => fun ?MODULE:convert_server/2 + }, + emqx_schema:servers_sc(Meta, #{default_port => 9200}). + +convert_server(<<"http://", Server/binary>>, HoconOpts) -> + convert_server(Server, HoconOpts); +convert_server(<<"https://", Server/binary>>, HoconOpts) -> + convert_server(Server, HoconOpts); +convert_server(Server0, HoconOpts) -> + Server = string:trim(Server0, trailing, "/"), + emqx_schema:convert_servers(Server, HoconOpts). + connector_config(Conf, #{name := Name, parse_confs := ParseConfs}) -> - #{ - base_url := BaseUrl, - authentication := - #{ - username := Username, - password := Password0 - } - } = Conf, - - Password = emqx_secret:unwrap(Password0), - Base64 = base64:encode(<>), - BasicToken = <<"Basic ", Base64/binary>>, - WebhookConfig = Conf#{ method => <<"post">>, - url => BaseUrl, + url => base_url(Conf), headers => [ {<<"Content-type">>, <<"application/json">>}, - {<<"Authorization">>, BasicToken} + {<<"Authorization">>, basic_token(Conf)} ] }, ParseConfs( @@ -190,6 +190,19 @@ connector_config(Conf, #{name := Name, parse_confs := ParseConfs}) -> WebhookConfig ). +basic_token(#{ + authentication := + #{ + username := Username, + password := Password0 + } +}) -> + Password = emqx_secret:unwrap(Password0), + Base64 = base64:encode(<>), + <<"Basic ", Base64/binary>>. + +base_url(#{ssl := #{enable := true}, server := Server}) -> "https://" ++ Server; +base_url(#{server := Server}) -> "http://" ++ Server. %%------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------- @@ -316,6 +329,10 @@ on_get_channel_status(_InstanceId, ChannelId, #{channels := Channels}) -> {error, not_exists} end. +render_template([<<"update_without_doc_template">>], Msg) -> + emqx_utils_json:encode(#{<<"doc">> => Msg}); +render_template([<<"create_without_doc_template">>], Msg) -> + emqx_utils_json:encode(#{<<"doc">> => Msg, <<"doc_as_upsert">> => true}); render_template(Template, Msg) -> % Ignoring errors here, undefined bindings will be replaced with empty string. Opts = #{var_trans => fun to_string/2}, @@ -395,6 +412,11 @@ get_body_template(#{action := update, doc := Doc} = Template) -> false -> <<"{\"doc\":", Doc/binary, "}">>; true -> <<"{\"doc\":", Doc/binary, ",\"doc_as_upsert\": true}">> end; +get_body_template(#{action := update} = Template) -> + case maps:get(doc_as_upsert, Template, false) of + false -> <<"update_without_doc_template">>; + true -> <<"create_without_doc_template">> + end; get_body_template(#{doc := Doc}) -> Doc; get_body_template(_) -> diff --git a/apps/emqx_bridge_es/test/emqx_bridge_es_SUITE.erl b/apps/emqx_bridge_es/test/emqx_bridge_es_SUITE.erl index a9ff70957..530eb77b2 100644 --- a/apps/emqx_bridge_es/test/emqx_bridge_es_SUITE.erl +++ b/apps/emqx_bridge_es/test/emqx_bridge_es_SUITE.erl @@ -103,13 +103,13 @@ end_per_testcase(_TestCase, _Config) -> %% Helper fns %%------------------------------------------------------------------------------------- -check_send_message_with_action(Topic, ActionName, ConnectorName) -> +check_send_message_with_action(Topic, ActionName, ConnectorName, Expect) -> send_message(Topic), %% ###################################### %% Check if message is sent to es %% ###################################### timer:sleep(500), - check_action_metrics(ActionName, ConnectorName). + check_action_metrics(ActionName, ConnectorName, Expect). send_message(Topic) -> Now = emqx_utils_calendar:now_to_rfc3339(microsecond), @@ -123,7 +123,7 @@ send_message(Topic) -> ok = emqtt:publish(Client, Topic, Payload, [{qos, 0}]), ok. -check_action_metrics(ActionName, ConnectorName) -> +check_action_metrics(ActionName, ConnectorName, Expect) -> ActionId = emqx_bridge_v2:id(?TYPE, ActionName, ConnectorName), Metrics = #{ @@ -134,13 +134,7 @@ check_action_metrics(ActionName, ConnectorName) -> dropped => emqx_resource_metrics:dropped_get(ActionId) }, ?assertEqual( - #{ - match => 1, - success => 1, - dropped => 0, - failed => 0, - queuing => 0 - }, + Expect, Metrics, {ActionName, ConnectorName, ActionId} ). @@ -169,11 +163,10 @@ action(ConnectorName) -> } }. -base_url(Config) -> +server(Config) -> Host = ?config(es_host, Config), Port = ?config(es_port, Config), iolist_to_binary([ - "https://", Host, ":", integer_to_binary(Port) @@ -185,7 +178,7 @@ connector_config(Config) -> connector_config(Overrides, Config) -> Defaults = #{ - <<"base_url">> => base_url(Config), + <<"server">> => server(Config), <<"enable">> => true, <<"authentication">> => #{ <<"password">> => <<"emqx123">>, @@ -249,7 +242,7 @@ t_create_remove_list(Config) -> ok. %% Test sending a message to a bridge V2 -t_send_message(Config) -> +t_create_message(Config) -> ConnectorConfig = connector_config(Config), {ok, _} = emqx_connector:create(?TYPE, test_connector2, ConnectorConfig), ActionConfig = action(<<"test_connector2">>), @@ -262,7 +255,8 @@ t_send_message(Config) -> }, {ok, _} = emqx_rule_engine:create_rule(Rule), %% Use the action to send a message - check_send_message_with_action(<<"es/1">>, test_action_1, test_connector2), + Expect = #{match => 1, success => 1, dropped => 0, failed => 0, queuing => 0}, + check_send_message_with_action(<<"es/1">>, test_action_1, test_connector2, Expect), %% Create a few more bridges with the same connector and test them ActionNames1 = lists:foldl( @@ -279,7 +273,7 @@ t_send_message(Config) -> }, {ok, _} = emqx_rule_engine:create_rule(Rule1), Topic = <<"es/", Seq/binary>>, - check_send_message_with_action(Topic, ActionName, test_connector2), + check_send_message_with_action(Topic, ActionName, test_connector2, Expect), [ActionName | Acc] end, [], @@ -294,6 +288,74 @@ t_send_message(Config) -> ActionNames ), emqx_connector:remove(?TYPE, test_connector2), + lists:foreach( + fun(#{id := Id}) -> + emqx_rule_engine:delete_rule(Id) + end, + emqx_rule_engine:get_rules() + ), + ok. + +t_update_message(Config) -> + ConnectorConfig = connector_config(Config), + {ok, _} = emqx_connector:create(?TYPE, update_connector, ConnectorConfig), + ActionConfig0 = action(<<"update_connector">>), + DocId = emqx_guid:to_hexstr(emqx_guid:gen()), + ActionConfig1 = ActionConfig0#{ + <<"parameters">> => #{ + <<"index">> => <<"${payload.index}">>, + <<"id">> => DocId, + <<"max_retries">> => 0, + <<"action">> => <<"update">>, + <<"doc">> => <<"${payload.doc}">> + } + }, + {ok, _} = emqx_bridge_v2:create(?TYPE, update_action, ActionConfig1), + Rule = #{ + id => <<"rule:t_es_1">>, + sql => <<"SELECT\n *\nFROM\n \"es/#\"">>, + actions => [<<"elasticsearch:update_action">>], + description => <<"sink doc to elasticsearch">> + }, + {ok, _} = emqx_rule_engine:create_rule(Rule), + %% failed to update a nonexistent doc + Expect0 = #{match => 1, success => 0, dropped => 0, failed => 1, queuing => 0}, + check_send_message_with_action(<<"es/1">>, update_action, update_connector, Expect0), + %% doc_as_upsert to insert a new doc + ActionConfig2 = ActionConfig1#{ + <<"parameters">> => #{ + <<"index">> => <<"${payload.index}">>, + <<"id">> => DocId, + <<"action">> => <<"update">>, + <<"doc">> => <<"${payload.doc}">>, + <<"doc_as_upsert">> => true, + <<"max_retries">> => 0 + } + }, + {ok, _} = emqx_bridge_v2:create(?TYPE, update_action, ActionConfig2), + Expect1 = #{match => 1, success => 1, dropped => 0, failed => 0, queuing => 0}, + check_send_message_with_action(<<"es/1">>, update_action, update_connector, Expect1), + %% update without doc, use msg as default + ActionConfig3 = ActionConfig1#{ + <<"parameters">> => #{ + <<"index">> => <<"${payload.index}">>, + <<"id">> => DocId, + <<"action">> => <<"update">>, + <<"max_retries">> => 0 + } + }, + {ok, _} = emqx_bridge_v2:create(?TYPE, update_action, ActionConfig3), + Expect2 = #{match => 1, success => 1, dropped => 0, failed => 0, queuing => 0}, + check_send_message_with_action(<<"es/1">>, update_action, update_connector, Expect2), + %% Clean + ok = emqx_bridge_v2:remove(?TYPE, update_action), + emqx_connector:remove(?TYPE, update_connector), + lists:foreach( + fun(#{id := Id}) -> + emqx_rule_engine:delete_rule(Id) + end, + emqx_rule_engine:get_rules() + ), ok. %% Test that we can get the status of the bridge V2 @@ -314,7 +376,7 @@ t_bad_url(Config) -> ActionName = <<"test_action">>, ActionConfig = action(<<"test_connector">>), ConnectorConfig0 = connector_config(Config), - ConnectorConfig = ConnectorConfig0#{<<"base_url">> := <<"bad_host:9092">>}, + ConnectorConfig = ConnectorConfig0#{<<"server">> := <<"bad_host:9092">>}, ?assertMatch({ok, _}, create_connector(ConnectorName, ConnectorConfig)), ?assertMatch({ok, _}, create_action(ActionName, ActionConfig)), ?assertMatch( diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http.app.src b/apps/emqx_bridge_http/src/emqx_bridge_http.app.src index c1c961e5b..0c355691b 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http.app.src +++ b/apps/emqx_bridge_http/src/emqx_bridge_http.app.src @@ -2,7 +2,7 @@ {description, "EMQX HTTP Bridge and Connector Application"}, {vsn, "0.2.2"}, {registered, []}, - {applications, [kernel, stdlib, emqx_connector, emqx_resource, ehttpc]}, + {applications, [kernel, stdlib, emqx_resource, ehttpc]}, {env, [{emqx_action_info_modules, [emqx_bridge_http_action_info]}]}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl index 478486e5b..809f47e49 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl @@ -865,7 +865,8 @@ convert_server(<<"http://", Server/binary>>, HoconOpts) -> convert_server(Server, HoconOpts); convert_server(<<"https://", Server/binary>>, HoconOpts) -> convert_server(Server, HoconOpts); -convert_server(Server, HoconOpts) -> +convert_server(Server0, HoconOpts) -> + Server = string:trim(Server0, trailing, "/"), emqx_schema:convert_servers(Server, HoconOpts). str(A) when is_atom(A) -> diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src index d7044b063..4fd96d5e7 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src @@ -10,9 +10,7 @@ {applications, [ kernel, stdlib, - emqx_resource, - %% for module emqx_connector_http - emqx_connector + emqx_resource ]}, {env, []}, {licenses, ["Business Source License 1.1"]}, diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src index 198c5f8e8..aaaae4cf4 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src @@ -5,7 +5,6 @@ {applications, [ kernel, stdlib, - emqx_connector, emqx_resource, emqx_mongodb ]}, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl index 369238ecf..08127ebcd 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl @@ -18,6 +18,7 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). %% management APIs -export([ @@ -284,9 +285,10 @@ maybe_on_message_received(Msg, {Mod, Func, Args}) -> maybe_on_message_received(_Msg, undefined) -> ok. -maybe_publish_local(Msg, Local = #{}, Props) -> +maybe_publish_local(Msg, Local = #{topic := Topic}, Props) when Topic =/= undefined -> + ?tp(mqtt_ingress_publish_local, #{msg => Msg, local => Local}), emqx_broker:publish(to_broker_msg(Msg, Local, Props)); -maybe_publish_local(_Msg, undefined, _Props) -> +maybe_publish_local(_Msg, _Local, _Props) -> ok. %% diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl index 05b2d6d3a..e49aa3da3 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -48,13 +48,14 @@ fields(action) -> )}; fields("mqtt_publisher_action") -> emqx_bridge_v2_schema:make_producer_action_schema( - hoconsc:mk( - hoconsc:ref(?MODULE, action_parameters), + mk( + ref(?MODULE, action_parameters), #{ required => true, desc => ?DESC("action_parameters") } - ) + ), + #{resource_opts_ref => ref(?MODULE, action_resource_opts)} ); fields(action_parameters) -> [ @@ -86,7 +87,8 @@ fields("mqtt_subscriber_source") -> required => true, desc => ?DESC("source_parameters") } - ) + ), + #{resource_opts_ref => ref(?MODULE, source_resource_opts)} ); fields(ingress_parameters) -> [ @@ -187,7 +189,7 @@ source_examples(Method) -> #{ parameters => #{ topic => <<"remote/topic">>, - qos => 2 + qos => 1 } } ) diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl index 3e5471d55..62e0e4f51 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl @@ -238,6 +238,9 @@ t_receive_via_rule(Config) -> end, fun(Trace) -> ?assertEqual([], ?of_kind("action_references_nonexistent_bridges", Trace)), + %% We don't have the hidden, legacy `local' config set, so we shouldn't + %% attempt to publish directly. + ?assertEqual([], ?of_kind(mqtt_ingress_publish_local, Trace)), ok end ), diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src index 5c2651b89..be5f4b417 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src @@ -5,7 +5,6 @@ {applications, [ kernel, stdlib, - emqx_connector, emqx_resource, emqx_mysql ]}, diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src index a2e006443..8e737c4a3 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src @@ -5,7 +5,6 @@ {applications, [ kernel, stdlib, - emqx_connector, emqx_resource, emqx_redis ]}, diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl index 187ca1c64..3fce41ec3 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl @@ -67,11 +67,17 @@ on_start( {tcp_options, [{mode, binary}, {reuseaddr, true}, {nodelay, true}]} ], MFArgs = {?MODULE, start_link, [maps:with([handshake_timeout], Config)]}, - ok = emqx_resource:allocate_resource(InstanceId, listen_on, ListenOn), + %% Since the esockd only supports atomic name and we don't want to introduce a new atom per each instance + %% when the port is same for two instance/connector, them will reference to a same esockd listener + %% to prevent the failed one dealloctes the listener which created by a earlier instance + %% we need record only when the listen is successed case esockd:open(?MODULE, ListenOn, Options, MFArgs) of {ok, _} -> + ok = emqx_resource:allocate_resource(InstanceId, listen_on, ListenOn), {ok, #{listen_on => ListenOn}}; + {error, {already_started, _}} -> + {error, eaddrinuse}; Error -> Error end. @@ -83,7 +89,12 @@ on_stop(InstanceId, _State) -> }), case emqx_resource:get_allocated_resources(InstanceId) of #{listen_on := ListenOn} -> - esockd:close(?MODULE, ListenOn); + case esockd:close(?MODULE, ListenOn) of + {error, not_found} -> + ok; + Result -> + Result + end; _ -> ok end. diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index d88a26c1d..04f19b95f 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1106,6 +1106,8 @@ tr_prometheus_collectors(Conf) -> prometheus_summary, %% emqx collectors emqx_prometheus, + {'/prometheus/auth', emqx_prometheus_auth}, + {'/prometheus/data_integration', emqx_prometheus_data_integration}, emqx_prometheus_mria %% builtin vm collectors | prometheus_collectors(Conf) diff --git a/apps/emqx_connector/src/emqx_connector.erl b/apps/emqx_connector/src/emqx_connector.erl index 92cf9439e..f4fee3eac 100644 --- a/apps/emqx_connector/src/emqx_connector.erl +++ b/apps/emqx_connector/src/emqx_connector.erl @@ -54,30 +54,34 @@ load() -> Connectors = emqx:get_config([?ROOT_KEY], #{}), - lists:foreach( + emqx_utils:pforeach( fun({Type, NamedConf}) -> - lists:foreach( + emqx_utils:pforeach( fun({Name, Conf}) -> safe_load_connector(Type, Name, Conf) end, - maps:to_list(NamedConf) + maps:to_list(NamedConf), + infinity ) end, - maps:to_list(Connectors) + maps:to_list(Connectors), + infinity ). unload() -> Connectors = emqx:get_config([?ROOT_KEY], #{}), - lists:foreach( + emqx_utils:pforeach( fun({Type, NamedConf}) -> - lists:foreach( + emqx_utils:pforeach( fun({Name, _Conf}) -> _ = emqx_connector_resource:stop(Type, Name) end, - maps:to_list(NamedConf) + maps:to_list(NamedConf), + infinity ) end, - maps:to_list(Connectors) + maps:to_list(Connectors), + infinity ). safe_load_connector(Type, Name, Conf) -> @@ -169,16 +173,16 @@ post_config_update([?ROOT_KEY, Type, Name], _Req, NewConf, OldConf, _AppEnvs) -> ?tp(connector_post_config_update_done, #{}), ok. -%% The config update will be failed if any task in `perform_connector_changes` failed. perform_connector_changes(Removed, Added, Updated) -> Result = perform_connector_changes([ - #{action => fun emqx_connector_resource:remove/4, data => Removed}, + #{action => fun emqx_connector_resource:remove/4, action_name => remove, data => Removed}, #{ action => fun emqx_connector_resource:create/4, + action_name => create, data => Added, on_exception_fn => fun emqx_connector_resource:remove/4 }, - #{action => fun emqx_connector_resource:update/4, data => Updated} + #{action => fun emqx_connector_resource:update/4, action_name => update, data => Updated} ]), ?tp(connector_post_config_update_done, #{}), Result. @@ -351,28 +355,21 @@ convert_certs(ConnectorsConf) -> ). perform_connector_changes(Tasks) -> - perform_connector_changes(Tasks, ok). + perform_connector_changes(Tasks, []). -perform_connector_changes([], Result) -> - Result; -perform_connector_changes([#{action := Action, data := MapConfs} = Task | Tasks], Result0) -> +perform_connector_changes([], Errors) -> + case Errors of + [] -> ok; + _ -> {error, Errors} + end; +perform_connector_changes([#{action := Action, data := MapConfs} = Task | Tasks], Errors0) -> OnException = maps:get(on_exception_fn, Task, fun(_Type, _Name, _Conf, _Opts) -> ok end), - Result = maps:fold( - fun - ({_Type, _Name}, _Conf, {error, Reason}) -> - {error, Reason}; - %% for emqx_connector_resource:update/4 - ({Type, Name}, {OldConf, Conf}, _) -> - ResOpts = emqx_resource:fetch_creation_opts(Conf), - case Action(Type, Name, {OldConf, Conf}, ResOpts) of - {error, Reason} -> {error, Reason}; - Return -> Return - end; - ({Type, Name}, Conf, _) -> - ResOpts = emqx_resource:fetch_creation_opts(Conf), - try Action(Type, Name, Conf, ResOpts) of - {error, Reason} -> {error, Reason}; - Return -> Return + Results = emqx_utils:pmap( + fun({{Type, Name}, Conf}) -> + ResOpts = creation_opts(Conf), + Res = + try + Action(Type, Name, Conf, ResOpts) catch Kind:Error:Stacktrace -> ?SLOG(error, #{ @@ -384,13 +381,34 @@ perform_connector_changes([#{action := Action, data := MapConfs} = Task | Tasks] stacktrace => Stacktrace }), OnException(Type, Name, Conf, ResOpts), - erlang:raise(Kind, Error, Stacktrace) - end + {error, Error} + end, + {{Type, Name}, Res} end, - Result0, - MapConfs + maps:to_list(MapConfs), + infinity ), - perform_connector_changes(Tasks, Result). + Errs = lists:filter( + fun + ({_TypeName, {error, _}}) -> true; + (_) -> false + end, + Results + ), + Errors = + case Errs of + [] -> + Errors0; + _ -> + #{action_name := ActionName} = Task, + [#{action => ActionName, errors => Errs} | Errors0] + end, + perform_connector_changes(Tasks, Errors). + +creation_opts({_OldConf, Conf}) -> + emqx_resource:fetch_creation_opts(Conf); +creation_opts(Conf) -> + emqx_resource:fetch_creation_opts(Conf). diff_confs(NewConfs, OldConfs) -> emqx_utils_maps:diff_maps( diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index f85109080..74c167b32 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -107,7 +107,7 @@ parse_connector_id(ConnectorId) -> {atom(), atom() | binary()}. parse_connector_id(<<"connector:", ConnectorId/binary>>, Opts) -> parse_connector_id(ConnectorId, Opts); -parse_connector_id(<>, Opts) -> +parse_connector_id(<>, Opts) -> parse_connector_id(ConnectorId, Opts); parse_connector_id(ConnectorId, Opts) -> emqx_resource:parse_resource_id(ConnectorId, Opts). @@ -229,7 +229,10 @@ create_dry_run(Type, Conf0, Callback) -> TypeBin = bin(Type), TypeAtom = safe_atom(Type), %% We use a fixed name here to avoid creating an atom - TmpName = iolist_to_binary([?TEST_ID_PREFIX, TypeBin, ":", <<"probedryrun">>]), + %% to avoid potential race condition, the resource id should be unique + Prefix = emqx_resource_manager:make_test_id(), + TmpName = + iolist_to_binary([Prefix, TypeBin, ":", <<"probedryrun">>]), TmpPath = emqx_utils:safe_filename(TmpName), Conf1 = maps:without([<<"name">>], Conf0), RawConf = #{<<"connectors">> => #{TypeBin => #{<<"temp_name">> => Conf1}}}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl index ad95e8678..4891b5293 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor.erl @@ -40,11 +40,14 @@ -export([ samplers/0, samplers/2, - current_rate/0, current_rate/1, granularity_adapter/1 ]). +-ifdef(TEST). +-export([current_rate_cluster/0]). +-endif. + %% for rpc -export([do_sample/2]). @@ -112,8 +115,33 @@ granularity_adapter(List) when length(List) > 1000 -> granularity_adapter(List) -> List. +current_rate(all) -> + current_rate_cluster(); +current_rate(Node) when Node == node() -> + try + {ok, Rate} = do_call(current_rate), + {ok, Rate} + catch + _E:R -> + ?SLOG(warning, #{msg => "dashboard_monitor_error", reason => R}), + %% Rate map 0, ensure api will not crash. + %% When joining cluster, dashboard monitor restart. + Rate0 = [ + {Key, 0} + || Key <- ?GAUGE_SAMPLER_LIST ++ maps:values(?DELTA_SAMPLER_RATE_MAP) + ], + {ok, maps:merge(maps:from_list(Rate0), non_rate_value())} + end; +current_rate(Node) -> + case emqx_dashboard_proto_v1:current_rate(Node) of + {badrpc, Reason} -> + {badrpc, {Node, Reason}}; + {ok, Rate} -> + {ok, Rate} + end. + %% Get the current rate. Not the current sampler data. -current_rate() -> +current_rate_cluster() -> Fun = fun (Node, Cluster) when is_map(Cluster) -> @@ -133,31 +161,6 @@ current_rate() -> {ok, Rate} end. -current_rate(all) -> - current_rate(); -current_rate(Node) when Node == node() -> - try - {ok, Rate} = do_call(current_rate), - {ok, Rate} - catch - _E:R -> - ?SLOG(warning, #{msg => "dashboard_monitor_error", reason => R}), - %% Rate map 0, ensure api will not crash. - %% When joining cluster, dashboard monitor restart. - Rate0 = [ - {Key, 0} - || Key <- ?GAUGE_SAMPLER_LIST ++ maps:values(?DELTA_SAMPLER_RATE_MAP) - ], - {ok, maps:from_list(Rate0)} - end; -current_rate(Node) -> - case emqx_dashboard_proto_v1:current_rate(Node) of - {badrpc, Reason} -> - {badrpc, {Node, Reason}}; - {ok, Rate} -> - {ok, Rate} - end. - %% ------------------------------------------------------------------------------------------------- %% gen_server functions @@ -173,7 +176,9 @@ handle_call(current_rate, _From, State = #state{last = Last}) -> NowTime = erlang:system_time(millisecond), NowSamplers = sample(NowTime), Rate = cal_rate(NowSamplers, Last), - {reply, {ok, Rate}, State}; + NonRateValue = non_rate_value(), + Samples = maps:merge(Rate, NonRateValue), + {reply, {ok, Samples}, State}; handle_call(_Request, _From, State = #state{}) -> {reply, ok, State}. @@ -256,8 +261,16 @@ merge_cluster_sampler_map(M1, M2) -> merge_cluster_rate(Node, Cluster) -> Fun = fun - (topics, Value, NCluster) -> - NCluster#{topics => Value}; + %% cluster-synced values + (topics, V, NCluster) -> + NCluster#{topics => V}; + (retained_msg_count, V, NCluster) -> + NCluster#{retained_msg_count => V}; + (license_quota, V, NCluster) -> + NCluster#{license_quota => V}; + %% for cluster sample, ignore node_uptime + (node_uptime, _V, NCluster) -> + NCluster; (Key, Value, NCluster) -> ClusterValue = maps:get(Key, NCluster, 0), NCluster#{Key => Value + ClusterValue} @@ -409,3 +422,26 @@ stats(received_bytes) -> emqx_metrics:val('bytes.received'); stats(sent) -> emqx_metrics:val('messages.sent'); stats(sent_bytes) -> emqx_metrics:val('bytes.sent'); stats(dropped) -> emqx_metrics:val('messages.dropped'). + +%% ------------------------------------------------------------------------------------------------- +%% Retained && License Quota + +%% the non rate values should be same on all nodes +non_rate_value() -> + (license_quota())#{ + retained_msg_count => emqx_retainer:retained_count(), + node_uptime => emqx_sys:uptime() + }. + +-if(?EMQX_RELEASE_EDITION == ee). +license_quota() -> + case emqx_license_checker:limits() of + {ok, #{max_connections := Quota}} -> + #{license_quota => Quota}; + {error, no_license} -> + #{license_quota => 0} + end. +-else. +license_quota() -> + #{}. +-endif. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index a152531f1..d7e3c094c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -1,5 +1,17 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_dashboard_monitor_api). @@ -133,13 +145,15 @@ dashboard_samplers_fun(Latest) -> end end. -monitor_current(get, #{bindings := []}) -> - emqx_utils_api:with_node_or_cluster(erlang:node(), fun emqx_dashboard_monitor:current_rate/1); monitor_current(get, #{bindings := Bindings}) -> RawNode = maps:get(node, Bindings, <<"all">>), emqx_utils_api:with_node_or_cluster(RawNode, fun current_rate/1). +-spec current_rate(atom()) -> + {error, term()} + | {ok, Result :: map()}. current_rate(Node) -> + %% Node :: 'all' or `NodeName` case emqx_dashboard_monitor:current_rate(Node) of {badrpc, _} = BadRpc -> {error, BadRpc}; diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl index 71e559647..8b02ae20f 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl @@ -31,10 +31,13 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + meck:new(emqx_retainer, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_retainer, retained_count, fun() -> 0 end), emqx_mgmt_api_test_util:init_suite([]), Config. end_per_suite(_Config) -> + meck:unload([emqx_retainer]), emqx_mgmt_api_test_util:end_suite([]). t_monitor_samplers_all(_Config) -> @@ -198,5 +201,5 @@ waiting_emqx_stats_and_monitor_update(WaitKey) -> end, meck:unload([emqx_stats]), %% manually call monitor update - _ = emqx_dashboard_monitor:current_rate(), + _ = emqx_dashboard_monitor:current_rate_cluster(), ok. diff --git a/apps/emqx_license/src/emqx_license.app.src b/apps/emqx_license/src/emqx_license.app.src index 8d11c6522..7b80af601 100644 --- a/apps/emqx_license/src/emqx_license.app.src +++ b/apps/emqx_license/src/emqx_license.app.src @@ -1,6 +1,6 @@ {application, emqx_license, [ {description, "EMQX License"}, - {vsn, "5.0.14"}, + {vsn, "5.0.15"}, {modules, []}, {registered, [emqx_license_sup]}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_license/src/emqx_license_checker.erl b/apps/emqx_license/src/emqx_license_checker.erl index 198814fb9..88bc79f90 100644 --- a/apps/emqx_license/src/emqx_license_checker.erl +++ b/apps/emqx_license/src/emqx_license_checker.erl @@ -30,6 +30,7 @@ start_link/2, update/1, dump/0, + expiry_epoch/0, purge/0, limits/0, print_warnings/1 @@ -67,6 +68,10 @@ update(License) -> dump() -> gen_server:call(?MODULE, dump, infinity). +-spec expiry_epoch() -> integer(). +expiry_epoch() -> + gen_server:call(?MODULE, expiry_epoch, infinity). + -spec limits() -> {ok, limits()} | {error, any()}. limits() -> try ets:lookup(?LICENSE_TAB, limits) of @@ -111,6 +116,9 @@ handle_call({update, License}, _From, #{license := Old} = State) -> {reply, check_license(License), State1#{license => License}}; handle_call(dump, _From, #{license := License} = State) -> {reply, emqx_license_parser:dump(License), State}; +handle_call(expiry_epoch, _From, #{license := License} = State) -> + ExpiryEpoch = date_to_expiry_epoch(emqx_license_parser:expiry_date(License)), + {reply, ExpiryEpoch, State}; handle_call(purge, _From, State) -> _ = ets:delete_all_objects(?LICENSE_TAB), {reply, ok, State}; @@ -234,6 +242,12 @@ small_customer_overdue(_CType, _DaysLeft) -> false. non_official_license_overdue(?OFFICIAL, _) -> false; non_official_license_overdue(_, DaysLeft) -> DaysLeft < 0. +%% 62167219200 =:= calendar:datetime_to_gregorian_seconds({{1970, 1, 1}, {0, 0, 0}}). +-define(EPOCH_START, 62167219200). +-spec date_to_expiry_epoch(calendar:date()) -> Seconds :: non_neg_integer(). +date_to_expiry_epoch({Y, M, D}) -> + calendar:datetime_to_gregorian_seconds({{Y, M, D}, {0, 0, 0}}) - ?EPOCH_START. + apply_limits(Limits) -> ets:insert(?LICENSE_TAB, {limits, Limits}). diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index fb2c94758..8783dd10a 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -179,6 +179,11 @@ app_deps(App, RebootApps) -> %% `emqx_bridge' is special in that it needs all the bridges apps to %% be started before it, so that, when it loads the bridges from %% configuration, the bridge app and its dependencies need to be up. +%% +%% `emqx_connector' also needs to start all connector dependencies for the same reason. +%% Since standalone apps like `emqx_mongodb' are already dependencies of `emqx_bridge_*' +%% apps, we may apply the same tactic for `emqx_connector' and inject individual bridges +%% as its dependencies. inject_bridge_deps(RebootAppDeps) -> BridgeApps = [ App @@ -189,6 +194,8 @@ inject_bridge_deps(RebootAppDeps) -> fun ({emqx_bridge, Deps0}) when is_list(Deps0) -> {emqx_bridge, Deps0 ++ BridgeApps}; + ({emqx_connector, Deps0}) when is_list(Deps0) -> + {emqx_connector, Deps0 ++ BridgeApps}; (App) -> App end, diff --git a/apps/emqx_management/src/emqx_mgmt_cluster.erl b/apps/emqx_management/src/emqx_mgmt_cluster.erl index 828567776..be9685f1b 100644 --- a/apps/emqx_management/src/emqx_mgmt_cluster.erl +++ b/apps/emqx_management/src/emqx_mgmt_cluster.erl @@ -69,7 +69,8 @@ handle_call({invite_async, Node, JoinTo}, _From, State) -> undefined -> Caller = self(), Task = spawn_link_invite_worker(Node, JoinTo, Caller), - {reply, ok, State#{Node => Task}}; + State1 = remove_finished_task(Node, State), + {reply, ok, State1#{Node => Task}}; WorkerPid -> {reply, {error, {already_started, WorkerPid}}, State} end; @@ -157,6 +158,11 @@ find_node_name_via_worker_pid(WorkerPid, {Key, Task, I}) -> find_node_name_via_worker_pid(WorkerPid, maps:next(I)) end. +remove_finished_task(Node, State = #{history := History}) -> + State#{history => maps:remove(Node, History)}; +remove_finished_task(_Node, State) -> + State. + state_to_invitation_status(State) -> History = maps:get(history, State, #{}), {Succ, Failed} = lists:foldl( diff --git a/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl index b2658f8fa..d1438b3ee 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl @@ -190,7 +190,7 @@ t_cluster_invite_async(Config) -> lists:sort(Core1Resp) ), - %% force leave the core2 and replicant + %% force leave the core2 {204} = rpc:call( Core1, emqx_mgmt_api_cluster, @@ -260,7 +260,41 @@ t_cluster_invite_async(Config) -> } ], lists:sort(Core1Resp3) - ). + ), + + %% force leave the core2 + {204} = rpc:call( + Core1, + emqx_mgmt_api_cluster, + force_leave, + [delete, #{bindings => #{node => atom_to_binary(Core2)}}] + ), + %% invite core2 again + ?assertMatch( + {200}, + Invite(Core2) + ), + + %% assert: core2 is in_progress status + {200, InvitationStatus1} = rpc:call(Core1, emqx_mgmt_api_cluster, get_invitation_status, [ + get, #{} + ]), + ?assertMatch( + #{succeed := [], in_progress := [#{node := Core2}], failed := []}, + InvitationStatus1 + ), + + %% waiting the async invitation_succeed + ?assertMatch({succeed, _}, waiting_the_async_invitation_succeed(Core1, Core2)), + + {200, InvitationStatus2} = rpc:call(Core1, emqx_mgmt_api_cluster, get_invitation_status, [ + get, #{} + ]), + ?assertMatch( + #{succeed := [#{node := Core2}], in_progress := [], failed := []}, + InvitationStatus2 + ), + ok. cluster(Config) -> NodeSpec = #{apps => ?APPS}, diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 6e520ba58..c555d3e16 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -392,7 +392,7 @@ t_create_webhook_v1_bridges_api({'init', Config}) -> lists:foreach( fun(App) -> _ = application:stop(App), - {ok, [App]} = application:ensure_all_started(App) + {ok, _} = application:ensure_all_started(App) end, [emqx_connector, emqx_bridge] ), diff --git a/apps/emqx_mongodb/src/emqx_mongodb.app.src b/apps/emqx_mongodb/src/emqx_mongodb.app.src index 8279da934..e171d3dd2 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.app.src +++ b/apps/emqx_mongodb/src/emqx_mongodb.app.src @@ -1,12 +1,11 @@ {application, emqx_mongodb, [ {description, "EMQX MongoDB Connector"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, stdlib, mongodb, - emqx_connector, emqx_resource ]}, {env, []}, diff --git a/apps/emqx_mysql/src/emqx_mysql.app.src b/apps/emqx_mysql/src/emqx_mysql.app.src index 9ae3234cc..bc6f6f6fb 100644 --- a/apps/emqx_mysql/src/emqx_mysql.app.src +++ b/apps/emqx_mysql/src/emqx_mysql.app.src @@ -1,12 +1,11 @@ {application, emqx_mysql, [ {description, "EMQX MySQL Database Connector"}, - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {registered, []}, {applications, [ kernel, stdlib, mysql, - emqx_connector, emqx_resource ]}, {env, []}, diff --git a/apps/emqx_postgresql/src/emqx_postgresql.app.src b/apps/emqx_postgresql/src/emqx_postgresql.app.src index 9c31b49c6..14877af68 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.app.src +++ b/apps/emqx_postgresql/src/emqx_postgresql.app.src @@ -1,12 +1,11 @@ {application, emqx_postgresql, [ {description, "EMQX PostgreSQL Database Connector"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [ kernel, stdlib, epgsql, - emqx_connector, emqx_resource ]}, {env, []}, diff --git a/apps/emqx_prometheus/include/emqx_prometheus.hrl b/apps/emqx_prometheus/include/emqx_prometheus.hrl index 8d552f025..b36f647da 100644 --- a/apps/emqx_prometheus/include/emqx_prometheus.hrl +++ b/apps/emqx_prometheus/include/emqx_prometheus.hrl @@ -16,3 +16,30 @@ -define(APP, emqx_prometheus). -define(PROMETHEUS, [prometheus]). + +-define(PROMETHEUS_DEFAULT_REGISTRY, default). +-define(PROMETHEUS_AUTH_REGISTRY, '/prometheus/auth'). +-define(PROMETHEUS_AUTH_COLLECTOR, emqx_prometheus_auth). +-define(PROMETHEUS_DATA_INTEGRATION_REGISTRY, '/prometheus/data_integration'). +-define(PROMETHEUS_DATA_INTEGRATION_COLLECTOR, emqx_prometheus_data_integration). + +-define(PROMETHEUS_ALL_REGISTRYS, [ + ?PROMETHEUS_DEFAULT_REGISTRY, + ?PROMETHEUS_AUTH_REGISTRY, + ?PROMETHEUS_DATA_INTEGRATION_REGISTRY +]). + +-define(PROM_DATA_MODE__NODE, node). +-define(PROM_DATA_MODE__ALL_NODES_AGGREGATED, all_nodes_aggregated). +-define(PROM_DATA_MODE__ALL_NODES_UNAGGREGATED, all_nodes_unaggregated). + +-define(PROM_DATA_MODES, [ + ?PROM_DATA_MODE__NODE, + ?PROM_DATA_MODE__ALL_NODES_AGGREGATED, + ?PROM_DATA_MODE__ALL_NODES_UNAGGREGATED +]). + +-define(PROM_DATA_MODE_KEY__, prom_data_mode). + +-define(PUT_PROM_DATA_MODE(MODE__), erlang:put(?PROM_DATA_MODE_KEY__, MODE__)). +-define(GET_PROM_DATA_MODE(), erlang:get(?PROM_DATA_MODE_KEY__)). diff --git a/apps/emqx_prometheus/rebar.config b/apps/emqx_prometheus/rebar.config index 12aa9060b..578881d71 100644 --- a/apps/emqx_prometheus/rebar.config +++ b/apps/emqx_prometheus/rebar.config @@ -3,7 +3,9 @@ {deps, [ {emqx, {path, "../emqx"}}, {emqx_utils, {path, "../emqx_utils"}}, - {prometheus, {git, "https://github.com/emqx/prometheus.erl", {tag, "v4.10.0.1"}}} + {emqx_auth, {path, "../emqx_auth"}}, + {emqx_resource, {path, "../emqx_resource"}}, + {prometheus, {git, "https://github.com/emqx/prometheus.erl", {tag, "v4.10.0.2"}}} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index fe0c42566..9e9952d6c 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -5,7 +5,7 @@ {vsn, "5.0.19"}, {modules, []}, {registered, [emqx_prometheus_sup]}, - {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, + {applications, [kernel, stdlib, prometheus, emqx, emqx_auth, emqx_resource, emqx_management]}, {mod, {emqx_prometheus_app, []}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index d513e2c37..ffa113a23 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -22,8 +22,19 @@ %% be used by the prometheus application -behaviour(prometheus_collector). +-behaviour(emqx_prometheus_cluster). +-export([ + fetch_from_local_node/1, + fetch_cluster_consistented_data/0, + aggre_or_zip_init_acc/0, + logic_sum_metrics/0 +]). + +-export([zip_json_prom_stats_metrics/3]). + -include("emqx_prometheus.hrl"). +-include_lib("public_key/include/public_key.hrl"). -include_lib("prometheus/include/prometheus_model.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -32,7 +43,8 @@ [ create_mf/5, gauge_metric/1, - counter_metric/1 + gauge_metrics/1, + counter_metrics/1 ] ). @@ -65,6 +77,13 @@ do_stop/0 ]). +%%-------------------------------------------------------------------- +%% Macros +%%-------------------------------------------------------------------- + +-define(MG(K, MAP), maps:get(K, MAP)). +-define(MG0(K, MAP), maps:get(K, MAP, 0)). + -define(C(K, L), proplists:get_value(K, L, 0)). -define(TIMER_MSG, '#interval'). @@ -119,7 +138,7 @@ handle_info(_Msg, State) -> {noreply, State}. push_to_push_gateway(Url, Headers) when is_list(Headers) -> - Data = prometheus_text_format:format(), + Data = prometheus_text_format:format(?PROMETHEUS_DEFAULT_REGISTRY), case httpc:request(post, {Url, Headers, "text/plain", Data}, ?HTTP_OPTIONS, []) of {ok, {{"HTTP/1.1", 200, _}, _RespHeaders, _RespBody}} -> ok; @@ -166,63 +185,106 @@ join_url(Url, JobName0) -> }), lists:concat([Url, "/metrics/job/", unicode:characters_to_list(JobName1)]). -deregister_cleanup(_Registry) -> +deregister_cleanup(?PROMETHEUS_DEFAULT_REGISTRY) -> ok. -collect_mf(_Registry, Callback) -> - Metrics = emqx_metrics:all(), - Stats = emqx_stats:getstats(), - VMData = emqx_vm_data(), - ClusterData = emqx_cluster_data(), - _ = [add_collect_family(Name, Stats, Callback, gauge) || Name <- emqx_stats:names()], - _ = [add_collect_family(Name, VMData, Callback, gauge) || Name <- emqx_vm()], - _ = [add_collect_family(Name, ClusterData, Callback, gauge) || Name <- emqx_cluster()], - _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_packets()], - _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_messages()], - _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_delivery()], - _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_client()], - _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_session()], - _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_olp()], - _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_acl()], - _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_authn()], +collect_mf(?PROMETHEUS_DEFAULT_REGISTRY, Callback) -> + RawData = emqx_prometheus_cluster:raw_data(?MODULE, ?GET_PROM_DATA_MODE()), + %% TODO: license expiry epoch and cert expiry epoch should be cached + ok = add_collect_family(Callback, stats_metric_meta(), ?MG(stats_data, RawData)), + ok = add_collect_family( + Callback, + stats_metric_cluster_consistened_meta(), + ?MG(stats_data_cluster_consistented, RawData) + ), + ok = add_collect_family(Callback, vm_metric_meta(), ?MG(vm_data, RawData)), + ok = add_collect_family(Callback, cluster_metric_meta(), ?MG(cluster_data, RawData)), + + ok = add_collect_family(Callback, emqx_packet_metric_meta(), ?MG(emqx_packet_data, RawData)), + ok = add_collect_family(Callback, message_metric_meta(), ?MG(emqx_message_data, RawData)), + ok = add_collect_family(Callback, delivery_metric_meta(), ?MG(emqx_delivery_data, RawData)), + ok = add_collect_family(Callback, client_metric_meta(), ?MG(emqx_client_data, RawData)), + ok = add_collect_family(Callback, session_metric_meta(), ?MG(emqx_session_data, RawData)), + ok = add_collect_family(Callback, olp_metric_meta(), ?MG(emqx_olp_data, RawData)), + ok = add_collect_family(Callback, acl_metric_meta(), ?MG(emqx_acl_data, RawData)), + ok = add_collect_family(Callback, authn_metric_meta(), ?MG(emqx_authn_data, RawData)), + + ok = add_collect_family(Callback, cert_metric_meta(), ?MG(cert_data, RawData)), + ok = maybe_license_add_collect_family(Callback, RawData), + ok; +collect_mf(_Registry, _Callback) -> ok. %% @private collect(<<"json">>) -> - Metrics = emqx_metrics:all(), - Stats = emqx_stats:getstats(), - VMData = emqx_vm_data(), - #{ - stats => maps:from_list([collect_stats(Name, Stats) || Name <- emqx_stats:names()]), - metrics => maps:from_list([collect_stats(Name, VMData) || Name <- emqx_vm()]), - packets => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_packets()]), - messages => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_messages()]), - delivery => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_delivery()]), - client => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_client()]), - session => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_session()]) + RawData = emqx_prometheus_cluster:raw_data(?MODULE, ?GET_PROM_DATA_MODE()), + (maybe_license_collect_json_data(RawData))#{ + stats => collect_stats_json_data(RawData), + metrics => collect_vm_json_data(?MG(vm_data, RawData)), + packets => collect_json_data(?MG(emqx_packet_data, RawData)), + messages => collect_json_data(?MG(emqx_message_data, RawData)), + delivery => collect_json_data(?MG(emqx_delivery_data, RawData)), + client => collect_json_data(?MG(emqx_client_data, RawData)), + session => collect_json_data(?MG(emqx_session_data, RawData)), + cluster => collect_json_data(?MG(cluster_data, RawData)), + olp => collect_json_data(?MG(emqx_olp_data, RawData)), + acl => collect_json_data(?MG(emqx_acl_data, RawData)), + authn => collect_json_data(?MG(emqx_authn_data, RawData)), + certs => collect_cert_json_data(?MG(cert_data, RawData)) }; collect(<<"prometheus">>) -> - prometheus_text_format:format(). - -%% @private -collect_stats(Name, Stats) -> - R = collect_metrics(Name, Stats), - case R#'Metric'.gauge of - undefined -> - {_, Val} = R#'Metric'.counter, - {Name, Val}; - {_, Val} -> - {Name, Val} - end. + prometheus_text_format:format(?PROMETHEUS_DEFAULT_REGISTRY). collect_metrics(Name, Metrics) -> emqx_collect(Name, Metrics). -add_collect_family(Name, Data, Callback, Type) -> - Callback(create_schema(Name, <<"">>, Data, Type)). +add_collect_family(Callback, MetricWithType, Data) -> + _ = [add_collect_family(Name, Data, Callback, Type) || {Name, Type, _} <- MetricWithType], + ok. -create_schema(Name, Help, Data, Type) -> - create_mf(Name, Help, Type, ?MODULE, Data). +add_collect_family(Name, Data, Callback, Type) -> + Callback(create_mf(Name, _Help = <<"">>, Type, ?MODULE, Data)). + +%% behaviour +fetch_from_local_node(Mode) -> + {node(self()), #{ + stats_data => stats_data(Mode), + vm_data => vm_data(Mode), + cluster_data => cluster_data(Mode), + %% Metrics + emqx_packet_data => emqx_metric_data(emqx_packet_metric_meta(), Mode), + emqx_message_data => emqx_metric_data(message_metric_meta(), Mode), + emqx_delivery_data => emqx_metric_data(delivery_metric_meta(), Mode), + emqx_client_data => emqx_metric_data(client_metric_meta(), Mode), + emqx_session_data => emqx_metric_data(session_metric_meta(), Mode), + emqx_olp_data => emqx_metric_data(olp_metric_meta(), Mode), + emqx_acl_data => emqx_metric_data(acl_metric_meta(), Mode), + emqx_authn_data => emqx_metric_data(authn_metric_meta(), Mode) + }}. + +fetch_cluster_consistented_data() -> + (maybe_license_fetch_data())#{ + stats_data_cluster_consistented => stats_data_cluster_consistented(), + cert_data => cert_data() + }. + +aggre_or_zip_init_acc() -> + #{ + stats_data => maps:from_keys(metrics_name(stats_metric_meta()), []), + vm_data => maps:from_keys(metrics_name(vm_metric_meta()), []), + cluster_data => maps:from_keys(metrics_name(cluster_metric_meta()), []), + emqx_packet_data => maps:from_keys(metrics_name(emqx_packet_metric_meta()), []), + emqx_message_data => maps:from_keys(metrics_name(message_metric_meta()), []), + emqx_delivery_data => maps:from_keys(metrics_name(delivery_metric_meta()), []), + emqx_client_data => maps:from_keys(metrics_name(client_metric_meta()), []), + emqx_session_data => maps:from_keys(metrics_name(session_metric_meta()), []), + emqx_olp_data => maps:from_keys(metrics_name(olp_metric_meta()), []), + emqx_acl_data => maps:from_keys(metrics_name(acl_metric_meta()), []), + emqx_authn_data => maps:from_keys(metrics_name(authn_metric_meta()), []) + }. + +logic_sum_metrics() -> + []. %%-------------------------------------------------------------------- %% Collector @@ -230,463 +292,678 @@ create_schema(Name, Help, Data, Type) -> %%-------------------------------------------------------------------- %% Stats - %% connections -emqx_collect(emqx_connections_count, Stats) -> - gauge_metric(?C('connections.count', Stats)); -emqx_collect(emqx_connections_max, Stats) -> - gauge_metric(?C('connections.max', Stats)); -emqx_collect(emqx_live_connections_count, Stats) -> - gauge_metric(?C('live_connections.count', Stats)); -emqx_collect(emqx_live_connections_max, Stats) -> - gauge_metric(?C('live_connections.max', Stats)); +emqx_collect(K = emqx_connections_count, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_connections_max, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_live_connections_count, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_live_connections_max, D) -> gauge_metrics(?MG(K, D)); %% sessions -emqx_collect(emqx_sessions_count, Stats) -> - gauge_metric(?C('sessions.count', Stats)); -emqx_collect(emqx_sessions_max, Stats) -> - gauge_metric(?C('sessions.max', Stats)); -emqx_collect(emqx_channels_count, Stats) -> - gauge_metric(?C('channels.count', Stats)); -emqx_collect(emqx_channels_max, Stats) -> - gauge_metric(?C('channels.max', Stats)); +emqx_collect(K = emqx_sessions_count, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_sessions_max, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_channels_count, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_channels_max, D) -> gauge_metrics(?MG(K, D)); %% pub/sub stats -emqx_collect(emqx_topics_count, Stats) -> - gauge_metric(?C('topics.count', Stats)); -emqx_collect(emqx_topics_max, Stats) -> - gauge_metric(?C('topics.max', Stats)); -emqx_collect(emqx_suboptions_count, Stats) -> - gauge_metric(?C('suboptions.count', Stats)); -emqx_collect(emqx_suboptions_max, Stats) -> - gauge_metric(?C('suboptions.max', Stats)); -emqx_collect(emqx_subscribers_count, Stats) -> - gauge_metric(?C('subscribers.count', Stats)); -emqx_collect(emqx_subscribers_max, Stats) -> - gauge_metric(?C('subscribers.max', Stats)); -emqx_collect(emqx_subscriptions_count, Stats) -> - gauge_metric(?C('subscriptions.count', Stats)); -emqx_collect(emqx_subscriptions_max, Stats) -> - gauge_metric(?C('subscriptions.max', Stats)); -emqx_collect(emqx_subscriptions_shared_count, Stats) -> - gauge_metric(?C('subscriptions.shared.count', Stats)); -emqx_collect(emqx_subscriptions_shared_max, Stats) -> - gauge_metric(?C('subscriptions.shared.max', Stats)); +emqx_collect(K = emqx_topics_count, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_topics_max, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_suboptions_count, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_suboptions_max, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_subscribers_count, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_subscribers_max, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_subscriptions_count, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_subscriptions_max, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_subscriptions_shared_count, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_subscriptions_shared_max, D) -> gauge_metrics(?MG(K, D)); %% retained -emqx_collect(emqx_retained_count, Stats) -> - gauge_metric(?C('retained.count', Stats)); -emqx_collect(emqx_retained_max, Stats) -> - gauge_metric(?C('retained.max', Stats)); +emqx_collect(K = emqx_retained_count, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_retained_max, D) -> gauge_metrics(?MG(K, D)); %% delayed -emqx_collect(emqx_delayed_count, Stats) -> - gauge_metric(?C('delayed.count', Stats)); -emqx_collect(emqx_delayed_max, Stats) -> - gauge_metric(?C('delayed.max', Stats)); -%%-------------------------------------------------------------------- -%% Metrics - packets & bytes - -%% bytes -emqx_collect(emqx_bytes_received, Metrics) -> - counter_metric(?C('bytes.received', Metrics)); -emqx_collect(emqx_bytes_sent, Metrics) -> - counter_metric(?C('bytes.sent', Metrics)); -%% received.sent -emqx_collect(emqx_packets_received, Metrics) -> - counter_metric(?C('packets.received', Metrics)); -emqx_collect(emqx_packets_sent, Metrics) -> - counter_metric(?C('packets.sent', Metrics)); -%% connect -emqx_collect(emqx_packets_connect, Metrics) -> - counter_metric(?C('packets.connect.received', Metrics)); -emqx_collect(emqx_packets_connack_sent, Metrics) -> - counter_metric(?C('packets.connack.sent', Metrics)); -emqx_collect(emqx_packets_connack_error, Metrics) -> - counter_metric(?C('packets.connack.error', Metrics)); -emqx_collect(emqx_packets_connack_auth_error, Metrics) -> - counter_metric(?C('packets.connack.auth_error', Metrics)); -%% sub.unsub -emqx_collect(emqx_packets_subscribe_received, Metrics) -> - counter_metric(?C('packets.subscribe.received', Metrics)); -emqx_collect(emqx_packets_subscribe_auth_error, Metrics) -> - counter_metric(?C('packets.subscribe.auth_error', Metrics)); -emqx_collect(emqx_packets_subscribe_error, Metrics) -> - counter_metric(?C('packets.subscribe.error', Metrics)); -emqx_collect(emqx_packets_suback_sent, Metrics) -> - counter_metric(?C('packets.suback.sent', Metrics)); -emqx_collect(emqx_packets_unsubscribe_received, Metrics) -> - counter_metric(?C('packets.unsubscribe.received', Metrics)); -emqx_collect(emqx_packets_unsubscribe_error, Metrics) -> - counter_metric(?C('packets.unsubscribe.error', Metrics)); -emqx_collect(emqx_packets_unsuback_sent, Metrics) -> - counter_metric(?C('packets.unsuback.sent', Metrics)); -%% publish.puback -emqx_collect(emqx_packets_publish_received, Metrics) -> - counter_metric(?C('packets.publish.received', Metrics)); -emqx_collect(emqx_packets_publish_sent, Metrics) -> - counter_metric(?C('packets.publish.sent', Metrics)); -emqx_collect(emqx_packets_publish_inuse, Metrics) -> - counter_metric(?C('packets.publish.inuse', Metrics)); -emqx_collect(emqx_packets_publish_error, Metrics) -> - counter_metric(?C('packets.publish.error', Metrics)); -emqx_collect(emqx_packets_publish_auth_error, Metrics) -> - counter_metric(?C('packets.publish.auth_error', Metrics)); -emqx_collect(emqx_packets_publish_dropped, Metrics) -> - counter_metric(?C('packets.publish.dropped', Metrics)); -%% puback -emqx_collect(emqx_packets_puback_received, Metrics) -> - counter_metric(?C('packets.puback.received', Metrics)); -emqx_collect(emqx_packets_puback_sent, Metrics) -> - counter_metric(?C('packets.puback.sent', Metrics)); -emqx_collect(emqx_packets_puback_inuse, Metrics) -> - counter_metric(?C('packets.puback.inuse', Metrics)); -emqx_collect(emqx_packets_puback_missed, Metrics) -> - counter_metric(?C('packets.puback.missed', Metrics)); -%% pubrec -emqx_collect(emqx_packets_pubrec_received, Metrics) -> - counter_metric(?C('packets.pubrec.received', Metrics)); -emqx_collect(emqx_packets_pubrec_sent, Metrics) -> - counter_metric(?C('packets.pubrec.sent', Metrics)); -emqx_collect(emqx_packets_pubrec_inuse, Metrics) -> - counter_metric(?C('packets.pubrec.inuse', Metrics)); -emqx_collect(emqx_packets_pubrec_missed, Metrics) -> - counter_metric(?C('packets.pubrec.missed', Metrics)); -%% pubrel -emqx_collect(emqx_packets_pubrel_received, Metrics) -> - counter_metric(?C('packets.pubrel.received', Metrics)); -emqx_collect(emqx_packets_pubrel_sent, Metrics) -> - counter_metric(?C('packets.pubrel.sent', Metrics)); -emqx_collect(emqx_packets_pubrel_missed, Metrics) -> - counter_metric(?C('packets.pubrel.missed', Metrics)); -%% pubcomp -emqx_collect(emqx_packets_pubcomp_received, Metrics) -> - counter_metric(?C('packets.pubcomp.received', Metrics)); -emqx_collect(emqx_packets_pubcomp_sent, Metrics) -> - counter_metric(?C('packets.pubcomp.sent', Metrics)); -emqx_collect(emqx_packets_pubcomp_inuse, Metrics) -> - counter_metric(?C('packets.pubcomp.inuse', Metrics)); -emqx_collect(emqx_packets_pubcomp_missed, Metrics) -> - counter_metric(?C('packets.pubcomp.missed', Metrics)); -%% pingreq -emqx_collect(emqx_packets_pingreq_received, Metrics) -> - counter_metric(?C('packets.pingreq.received', Metrics)); -emqx_collect(emqx_packets_pingresp_sent, Metrics) -> - counter_metric(?C('packets.pingresp.sent', Metrics)); -%% disconnect -emqx_collect(emqx_packets_disconnect_received, Metrics) -> - counter_metric(?C('packets.disconnect.received', Metrics)); -emqx_collect(emqx_packets_disconnect_sent, Metrics) -> - counter_metric(?C('packets.disconnect.sent', Metrics)); -%% auth -emqx_collect(emqx_packets_auth_received, Metrics) -> - counter_metric(?C('packets.auth.received', Metrics)); -emqx_collect(emqx_packets_auth_sent, Metrics) -> - counter_metric(?C('packets.auth.sent', Metrics)); -%%-------------------------------------------------------------------- -%% Metrics - messages - -%% messages -emqx_collect(emqx_messages_received, Metrics) -> - counter_metric(?C('messages.received', Metrics)); -emqx_collect(emqx_messages_sent, Metrics) -> - counter_metric(?C('messages.sent', Metrics)); -emqx_collect(emqx_messages_qos0_received, Metrics) -> - counter_metric(?C('messages.qos0.received', Metrics)); -emqx_collect(emqx_messages_qos0_sent, Metrics) -> - counter_metric(?C('messages.qos0.sent', Metrics)); -emqx_collect(emqx_messages_qos1_received, Metrics) -> - counter_metric(?C('messages.qos1.received', Metrics)); -emqx_collect(emqx_messages_qos1_sent, Metrics) -> - counter_metric(?C('messages.qos1.sent', Metrics)); -emqx_collect(emqx_messages_qos2_received, Metrics) -> - counter_metric(?C('messages.qos2.received', Metrics)); -emqx_collect(emqx_messages_qos2_sent, Metrics) -> - counter_metric(?C('messages.qos2.sent', Metrics)); -emqx_collect(emqx_messages_publish, Metrics) -> - counter_metric(?C('messages.publish', Metrics)); -emqx_collect(emqx_messages_dropped, Metrics) -> - counter_metric(?C('messages.dropped', Metrics)); -emqx_collect(emqx_messages_dropped_expired, Metrics) -> - counter_metric(?C('messages.dropped.await_pubrel_timeout', Metrics)); -emqx_collect(emqx_messages_dropped_no_subscribers, Metrics) -> - counter_metric(?C('messages.dropped.no_subscribers', Metrics)); -emqx_collect(emqx_messages_forward, Metrics) -> - counter_metric(?C('messages.forward', Metrics)); -emqx_collect(emqx_messages_retained, Metrics) -> - counter_metric(?C('messages.retained', Metrics)); -emqx_collect(emqx_messages_delayed, Stats) -> - counter_metric(?C('messages.delayed', Stats)); -emqx_collect(emqx_messages_delivered, Stats) -> - counter_metric(?C('messages.delivered', Stats)); -emqx_collect(emqx_messages_acked, Stats) -> - counter_metric(?C('messages.acked', Stats)); -%%-------------------------------------------------------------------- -%% Metrics - delivery - -emqx_collect(emqx_delivery_dropped, Stats) -> - counter_metric(?C('delivery.dropped', Stats)); -emqx_collect(emqx_delivery_dropped_no_local, Stats) -> - counter_metric(?C('delivery.dropped.no_local', Stats)); -emqx_collect(emqx_delivery_dropped_too_large, Stats) -> - counter_metric(?C('delivery.dropped.too_large', Stats)); -emqx_collect(emqx_delivery_dropped_qos0_msg, Stats) -> - counter_metric(?C('delivery.dropped.qos0_msg', Stats)); -emqx_collect(emqx_delivery_dropped_queue_full, Stats) -> - counter_metric(?C('delivery.dropped.queue_full', Stats)); -emqx_collect(emqx_delivery_dropped_expired, Stats) -> - counter_metric(?C('delivery.dropped.expired', Stats)); -%%-------------------------------------------------------------------- -%% Metrics - client -emqx_collect(emqx_client_connect, Stats) -> - counter_metric(?C('client.connect', Stats)); -emqx_collect(emqx_client_connack, Stats) -> - counter_metric(?C('client.connack', Stats)); -emqx_collect(emqx_client_connected, Stats) -> - counter_metric(?C('client.connected', Stats)); -emqx_collect(emqx_client_authenticate, Stats) -> - counter_metric(?C('client.authenticate', Stats)); -emqx_collect(emqx_client_auth_anonymous, Stats) -> - counter_metric(?C('client.auth.anonymous', Stats)); -emqx_collect(emqx_client_authorize, Stats) -> - counter_metric(?C('client.authorize', Stats)); -emqx_collect(emqx_client_subscribe, Stats) -> - counter_metric(?C('client.subscribe', Stats)); -emqx_collect(emqx_client_unsubscribe, Stats) -> - counter_metric(?C('client.unsubscribe', Stats)); -emqx_collect(emqx_client_disconnected, Stats) -> - counter_metric(?C('client.disconnected', Stats)); -%%-------------------------------------------------------------------- -%% Metrics - session - -emqx_collect(emqx_session_created, Stats) -> - counter_metric(?C('session.created', Stats)); -emqx_collect(emqx_session_resumed, Stats) -> - counter_metric(?C('session.resumed', Stats)); -emqx_collect(emqx_session_takenover, Stats) -> - counter_metric(?C('session.takenover', Stats)); -emqx_collect(emqx_session_discarded, Stats) -> - counter_metric(?C('session.discarded', Stats)); -emqx_collect(emqx_session_terminated, Stats) -> - counter_metric(?C('session.terminated', Stats)); -%%-------------------------------------------------------------------- - -%% Metrics - overload protection -emqx_collect(emqx_overload_protection_delay_ok, Stats) -> - counter_metric(?C('overload_protection.delay.ok', Stats)); -emqx_collect(emqx_overload_protection_delay_timeout, Stats) -> - counter_metric(?C('overload_protection.delay.timeout', Stats)); -emqx_collect(emqx_overload_protection_hibernation, Stats) -> - counter_metric(?C('overload_protection.hibernation', Stats)); -emqx_collect(emqx_overload_protection_gc, Stats) -> - counter_metric(?C('overload_protection.gc', Stats)); -emqx_collect(emqx_overload_protection_new_conn, Stats) -> - counter_metric(?C('overload_protection.new_conn', Stats)); -%%-------------------------------------------------------------------- -%% Metrics - acl -emqx_collect(emqx_authorization_allow, Stats) -> - counter_metric(?C('authorization.allow', Stats)); -emqx_collect(emqx_authorization_deny, Stats) -> - counter_metric(?C('authorization.deny', Stats)); -emqx_collect(emqx_authorization_cache_hit, Stats) -> - counter_metric(?C('authorization.cache_hit', Stats)); -emqx_collect(emqx_authorization_cache_miss, Stats) -> - counter_metric(?C('authorization.cache_miss', Stats)); -emqx_collect(emqx_authorization_superuser, Stats) -> - counter_metric(?C('authorization.superuser', Stats)); -emqx_collect(emqx_authorization_nomatch, Stats) -> - counter_metric(?C('authorization.nomatch', Stats)); -emqx_collect(emqx_authorization_matched_allow, Stats) -> - counter_metric(?C('authorization.matched_allow', Stats)); -emqx_collect(emqx_authorization_matched_deny, Stats) -> - counter_metric(?C('authorization.matched_deny', Stats)); -%%-------------------------------------------------------------------- -%% Metrics - authn -emqx_collect(emqx_authentication_success, Stats) -> - counter_metric(?C('authentication.success', Stats)); -emqx_collect(emqx_authentication_success_anonymous, Stats) -> - counter_metric(?C('authentication.success.anonymous', Stats)); -emqx_collect(emqx_authentication_failure, Stats) -> - counter_metric(?C('authentication.failure', Stats)); +emqx_collect(K = emqx_delayed_count, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_delayed_max, D) -> gauge_metrics(?MG(K, D)); %%-------------------------------------------------------------------- %% VM - -emqx_collect(emqx_vm_cpu_use, VMData) -> - gauge_metric(?C(cpu_use, VMData)); -emqx_collect(emqx_vm_cpu_idle, VMData) -> - gauge_metric(?C(cpu_idle, VMData)); -emqx_collect(emqx_vm_run_queue, VMData) -> - gauge_metric(?C(run_queue, VMData)); -emqx_collect(emqx_vm_process_messages_in_queues, VMData) -> - gauge_metric(?C(process_total_messages, VMData)); -emqx_collect(emqx_vm_total_memory, VMData) -> - gauge_metric(?C(total_memory, VMData)); -emqx_collect(emqx_vm_used_memory, VMData) -> - gauge_metric(?C(used_memory, VMData)); -emqx_collect(emqx_cluster_nodes_running, ClusterData) -> - gauge_metric(?C(nodes_running, ClusterData)); -emqx_collect(emqx_cluster_nodes_stopped, ClusterData) -> - gauge_metric(?C(nodes_stopped, ClusterData)). +emqx_collect(K = emqx_vm_cpu_use, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_vm_cpu_idle, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_vm_run_queue, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_vm_process_messages_in_queues, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_vm_total_memory, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_vm_used_memory, D) -> gauge_metrics(?MG(K, D)); +%%-------------------------------------------------------------------- +%% Cluster Info +emqx_collect(K = emqx_cluster_nodes_running, D) -> gauge_metrics(?MG(K, D)); +emqx_collect(K = emqx_cluster_nodes_stopped, D) -> gauge_metrics(?MG(K, D)); +%%-------------------------------------------------------------------- +%% Metrics - packets & bytes +%% bytes +emqx_collect(K = emqx_bytes_received, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_bytes_sent, D) -> counter_metrics(?MG(K, D)); +%% received.sent +emqx_collect(K = emqx_packets_received, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_sent, D) -> counter_metrics(?MG(K, D)); +%% connect +emqx_collect(K = emqx_packets_connect, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_connack_sent, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_connack_error, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_connack_auth_error, D) -> counter_metrics(?MG(K, D)); +%% sub.unsub +emqx_collect(K = emqx_packets_subscribe_received, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_subscribe_auth_error, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_subscribe_error, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_suback_sent, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_unsubscribe_received, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_unsubscribe_error, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_unsuback_sent, D) -> counter_metrics(?MG(K, D)); +%% publish.puback +emqx_collect(K = emqx_packets_publish_received, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_publish_sent, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_publish_inuse, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_publish_error, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_publish_auth_error, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_publish_dropped, D) -> counter_metrics(?MG(K, D)); +%% puback +emqx_collect(K = emqx_packets_puback_received, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_puback_sent, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_puback_inuse, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_puback_missed, D) -> counter_metrics(?MG(K, D)); +%% pubrec +emqx_collect(K = emqx_packets_pubrec_received, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_pubrec_sent, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_pubrec_inuse, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_pubrec_missed, D) -> counter_metrics(?MG(K, D)); +%% pubrel +emqx_collect(K = emqx_packets_pubrel_received, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_pubrel_sent, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_pubrel_missed, D) -> counter_metrics(?MG(K, D)); +%% pubcomp +emqx_collect(K = emqx_packets_pubcomp_received, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_pubcomp_sent, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_pubcomp_inuse, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_pubcomp_missed, D) -> counter_metrics(?MG(K, D)); +%% pingreq +emqx_collect(K = emqx_packets_pingreq_received, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_pingresp_sent, D) -> counter_metrics(?MG(K, D)); +%% disconnect +emqx_collect(K = emqx_packets_disconnect_received, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_disconnect_sent, D) -> counter_metrics(?MG(K, D)); +%% auth +emqx_collect(K = emqx_packets_auth_received, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_packets_auth_sent, D) -> counter_metrics(?MG(K, D)); +%%-------------------------------------------------------------------- +%% Metrics - messages +%% messages +emqx_collect(K = emqx_messages_received, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_messages_sent, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_messages_qos0_received, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_messages_qos0_sent, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_messages_qos1_received, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_messages_qos1_sent, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_messages_qos2_received, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_messages_qos2_sent, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_messages_publish, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_messages_dropped, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_messages_dropped_expired, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_messages_dropped_no_subscribers, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_messages_forward, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_messages_retained, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_messages_delayed, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_messages_delivered, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_messages_acked, D) -> counter_metrics(?MG(K, D)); +%%-------------------------------------------------------------------- +%% Metrics - delivery +emqx_collect(K = emqx_delivery_dropped, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_delivery_dropped_no_local, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_delivery_dropped_too_large, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_delivery_dropped_qos0_msg, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_delivery_dropped_queue_full, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_delivery_dropped_expired, D) -> counter_metrics(?MG(K, D)); +%%-------------------------------------------------------------------- +%% Metrics - client +emqx_collect(K = emqx_client_connect, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_client_connack, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_client_connected, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_client_authenticate, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_client_auth_anonymous, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_client_authorize, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_client_subscribe, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_client_unsubscribe, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_client_disconnected, D) -> counter_metrics(?MG(K, D)); +%%-------------------------------------------------------------------- +%% Metrics - session +emqx_collect(K = emqx_session_created, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_session_resumed, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_session_takenover, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_session_discarded, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_session_terminated, D) -> counter_metrics(?MG(K, D)); +%%-------------------------------------------------------------------- +%% Metrics - overload protection +emqx_collect(K = emqx_overload_protection_delay_ok, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_overload_protection_delay_timeout, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_overload_protection_hibernation, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_overload_protection_gc, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_overload_protection_new_conn, D) -> counter_metrics(?MG(K, D)); +%%-------------------------------------------------------------------- +%% Metrics - acl +emqx_collect(K = emqx_authorization_allow, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_authorization_deny, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_authorization_cache_hit, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_authorization_cache_miss, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_authorization_superuser, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_authorization_nomatch, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_authorization_matched_allow, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_authorization_matched_deny, D) -> counter_metrics(?MG(K, D)); +%%-------------------------------------------------------------------- +%% Metrics - authn +emqx_collect(K = emqx_authentication_success, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_authentication_success_anonymous, D) -> counter_metrics(?MG(K, D)); +emqx_collect(K = emqx_authentication_failure, D) -> counter_metrics(?MG(K, D)); +%%-------------------------------------------------------------------- +%% License +emqx_collect(K = emqx_license_expiry_at, D) -> gauge_metric(?MG(K, D)); +%%-------------------------------------------------------------------- +%% Certs +emqx_collect(K = emqx_cert_expiry_at, D) -> gauge_metrics(?MG(K, D)). %%-------------------------------------------------------------------- %% Indicators %%-------------------------------------------------------------------- -emqx_metrics_packets() -> +%%======================================== +%% Stats +%%======================================== + +stats_metric_meta() -> [ - emqx_bytes_received, - emqx_bytes_sent, - emqx_packets_received, - emqx_packets_sent, - emqx_packets_connect, - emqx_packets_connack_sent, - emqx_packets_connack_error, - emqx_packets_connack_auth_error, - emqx_packets_publish_received, - emqx_packets_publish_sent, - emqx_packets_publish_inuse, - emqx_packets_publish_error, - emqx_packets_publish_auth_error, - emqx_packets_publish_dropped, - emqx_packets_puback_received, - emqx_packets_puback_sent, - emqx_packets_puback_inuse, - emqx_packets_puback_missed, - emqx_packets_pubrec_received, - emqx_packets_pubrec_sent, - emqx_packets_pubrec_inuse, - emqx_packets_pubrec_missed, - emqx_packets_pubrel_received, - emqx_packets_pubrel_sent, - emqx_packets_pubrel_missed, - emqx_packets_pubcomp_received, - emqx_packets_pubcomp_sent, - emqx_packets_pubcomp_inuse, - emqx_packets_pubcomp_missed, - emqx_packets_subscribe_received, - emqx_packets_subscribe_error, - emqx_packets_subscribe_auth_error, - emqx_packets_suback_sent, - emqx_packets_unsubscribe_received, - emqx_packets_unsubscribe_error, - emqx_packets_unsuback_sent, - emqx_packets_pingreq_received, - emqx_packets_pingresp_sent, - emqx_packets_disconnect_received, - emqx_packets_disconnect_sent, - emqx_packets_auth_received, - emqx_packets_auth_sent + %% connections + {emqx_connections_count, gauge, 'connections.count'}, + {emqx_connections_max, gauge, 'connections.max'}, + {emqx_live_connections_count, gauge, 'live_connections.count'}, + {emqx_live_connections_max, gauge, 'live_connections.max'}, + %% sessions + {emqx_sessions_count, gauge, 'sessions.count'}, + {emqx_sessions_max, gauge, 'sessions.max'}, + {emqx_channels_count, gauge, 'channels.count'}, + {emqx_channels_max, gauge, 'channels.max'}, + %% pub/sub stats + {emqx_suboptions_count, gauge, 'suboptions.count'}, + {emqx_suboptions_max, gauge, 'suboptions.max'}, + {emqx_subscribers_count, gauge, 'subscribers.count'}, + {emqx_subscribers_max, gauge, 'subscribers.max'}, + {emqx_subscriptions_count, gauge, 'subscriptions.count'}, + {emqx_subscriptions_max, gauge, 'subscriptions.max'}, + {emqx_subscriptions_shared_count, gauge, 'subscriptions.shared.count'}, + {emqx_subscriptions_shared_max, gauge, 'subscriptions.shared.max'}, + %% delayed + {emqx_delayed_count, gauge, 'delayed.count'}, + {emqx_delayed_max, gauge, 'delayed.max'} ]. -emqx_metrics_olp() -> - case emqx_config_zones:is_olp_enabled() of - true -> - [ - emqx_overload_protection_delay_ok, - emqx_overload_protection_delay_timeout, - emqx_overload_protection_hibernation, - emqx_overload_protection_gc, - emqx_overload_protection_new_conn - ]; - false -> - [] - end. - -emqx_metrics_acl() -> +stats_metric_cluster_consistened_meta() -> [ - emqx_authorization_allow, - emqx_authorization_deny, - emqx_authorization_cache_hit, - emqx_authorization_cache_miss, - emqx_authorization_superuser, - emqx_authorization_nomatch, - emqx_authorization_matched_allow, - emqx_authorization_matched_deny + %% topics + {emqx_topics_max, gauge, 'topics.max'}, + {emqx_topics_count, gauge, 'topics.count'}, + %% retained + {emqx_retained_count, gauge, 'retained.count'}, + {emqx_retained_max, gauge, 'retained.max'} ]. -emqx_metrics_authn() -> +stats_data(Mode) -> + Stats = emqx_stats:getstats(), + lists:foldl( + fun({Name, _Type, MetricKAtom}, AccIn) -> + AccIn#{Name => [{with_node_label(Mode, []), ?C(MetricKAtom, Stats)}]} + end, + #{}, + stats_metric_meta() + ). + +stats_data_cluster_consistented() -> + Stats = emqx_stats:getstats(), + lists:foldl( + fun({Name, _Type, MetricKAtom}, AccIn) -> + AccIn#{Name => [{[], ?C(MetricKAtom, Stats)}]} + end, + #{}, + stats_metric_cluster_consistened_meta() + ). + +%%======================================== +%% Erlang VM +%%======================================== + +vm_metric_meta() -> [ - emqx_authentication_success, - emqx_authentication_success_anonymous, - emqx_authentication_failure + {emqx_vm_cpu_use, gauge, 'cpu_use'}, + {emqx_vm_cpu_idle, gauge, 'cpu_idle'}, + {emqx_vm_run_queue, gauge, 'run_queue'}, + {emqx_vm_process_messages_in_queues, gauge, 'process_total_messages'}, + {emqx_vm_total_memory, gauge, 'total_memory'}, + {emqx_vm_used_memory, gauge, 'used_memory'} ]. -emqx_metrics_messages() -> +vm_data(Mode) -> + VmStats = emqx_mgmt:vm_stats(), + lists:foldl( + fun({Name, _Type, MetricKAtom}, AccIn) -> + Labels = + case Mode of + node -> + []; + _ -> + [{node, node(self())}] + end, + AccIn#{Name => [{Labels, ?C(MetricKAtom, VmStats)}]} + end, + #{}, + vm_metric_meta() + ). + +%%======================================== +%% Cluster +%%======================================== + +cluster_metric_meta() -> [ - emqx_messages_received, - emqx_messages_sent, - emqx_messages_qos0_received, - emqx_messages_qos0_sent, - emqx_messages_qos1_received, - emqx_messages_qos1_sent, - emqx_messages_qos2_received, - emqx_messages_qos2_sent, - emqx_messages_publish, - emqx_messages_dropped, - emqx_messages_dropped_expired, - emqx_messages_dropped_no_subscribers, - emqx_messages_forward, - emqx_messages_retained, - emqx_messages_delayed, - emqx_messages_delivered, - emqx_messages_acked + {emqx_cluster_nodes_running, gauge, undefined}, + {emqx_cluster_nodes_stopped, gauge, undefined} ]. -emqx_metrics_delivery() -> - [ - emqx_delivery_dropped, - emqx_delivery_dropped_no_local, - emqx_delivery_dropped_too_large, - emqx_delivery_dropped_qos0_msg, - emqx_delivery_dropped_queue_full, - emqx_delivery_dropped_expired - ]. - -emqx_metrics_client() -> - [ - emqx_client_connect, - emqx_client_connack, - emqx_client_connected, - emqx_client_authenticate, - emqx_client_auth_anonymous, - emqx_client_authorize, - emqx_client_subscribe, - emqx_client_unsubscribe, - emqx_client_disconnected - ]. - -emqx_metrics_session() -> - [ - emqx_session_created, - emqx_session_resumed, - emqx_session_takenover, - emqx_session_discarded, - emqx_session_terminated - ]. - -emqx_vm() -> - [ - emqx_vm_cpu_use, - emqx_vm_cpu_idle, - emqx_vm_run_queue, - emqx_vm_process_messages_in_queues, - emqx_vm_total_memory, - emqx_vm_used_memory - ]. - -emqx_vm_data() -> - emqx_mgmt:vm_stats(). - -emqx_cluster() -> - [ - emqx_cluster_nodes_running, - emqx_cluster_nodes_stopped - ]. - -emqx_cluster_data() -> +cluster_data(Mode) -> Running = emqx:cluster_nodes(running), Stopped = emqx:cluster_nodes(stopped), + #{ + emqx_cluster_nodes_running => [{with_node_label(Mode, []), length(Running)}], + emqx_cluster_nodes_stopped => [{with_node_label(Mode, []), length(Stopped)}] + }. + +%%======================================== +%% Metrics +%%======================================== + +emqx_metric_data(MetricNameTypeKeyL, Mode) -> + Metrics = emqx_metrics:all(), + lists:foldl( + fun({Name, _Type, MetricKAtom}, AccIn) -> + AccIn#{Name => [{with_node_label(Mode, []), ?C(MetricKAtom, Metrics)}]} + end, + #{}, + MetricNameTypeKeyL + ). + +%%========== +%% Bytes && Packets +emqx_packet_metric_meta() -> [ - {nodes_running, length(Running)}, - {nodes_stopped, length(Stopped)} + {emqx_bytes_received, counter, 'bytes.received'}, + {emqx_bytes_sent, counter, 'bytes.sent'}, + %% received.sent + {emqx_packets_received, counter, 'packets.received'}, + {emqx_packets_sent, counter, 'packets.sent'}, + %% connect + {emqx_packets_connect, counter, 'packets.connect.received'}, + {emqx_packets_connack_sent, counter, 'packets.connack.sent'}, + {emqx_packets_connack_error, counter, 'packets.connack.error'}, + {emqx_packets_connack_auth_error, counter, 'packets.connack.auth_error'}, + %% sub.unsub + {emqx_packets_subscribe_received, counter, 'packets.subscribe.received'}, + {emqx_packets_subscribe_auth_error, counter, 'packets.subscribe.auth_error'}, + {emqx_packets_subscribe_error, counter, 'packets.subscribe.error'}, + {emqx_packets_suback_sent, counter, 'packets.suback.sent'}, + {emqx_packets_unsubscribe_received, counter, 'packets.unsubscribe.received'}, + {emqx_packets_unsubscribe_error, counter, 'packets.unsubscribe.error'}, + {emqx_packets_unsuback_sent, counter, 'packets.unsuback.sent'}, + %% publish.puback + {emqx_packets_publish_received, counter, 'packets.publish.received'}, + {emqx_packets_publish_sent, counter, 'packets.publish.sent'}, + {emqx_packets_publish_inuse, counter, 'packets.publish.inuse'}, + {emqx_packets_publish_error, counter, 'packets.publish.error'}, + {emqx_packets_publish_auth_error, counter, 'packets.publish.auth_error'}, + {emqx_packets_publish_dropped, counter, 'packets.publish.dropped'}, + %% puback + {emqx_packets_puback_received, counter, 'packets.puback.received'}, + {emqx_packets_puback_sent, counter, 'packets.puback.sent'}, + {emqx_packets_puback_inuse, counter, 'packets.puback.inuse'}, + {emqx_packets_puback_missed, counter, 'packets.puback.missed'}, + %% pubrec + {emqx_packets_pubrec_received, counter, 'packets.pubrec.received'}, + {emqx_packets_pubrec_sent, counter, 'packets.pubrec.sent'}, + {emqx_packets_pubrec_inuse, counter, 'packets.pubrec.inuse'}, + {emqx_packets_pubrec_missed, counter, 'packets.pubrec.missed'}, + %% pubrel + {emqx_packets_pubrel_received, counter, 'packets.pubrel.received'}, + {emqx_packets_pubrel_sent, counter, 'packets.pubrel.sent'}, + {emqx_packets_pubrel_missed, counter, 'packets.pubrel.missed'}, + %% pubcomp + {emqx_packets_pubcomp_received, counter, 'packets.pubcomp.received'}, + {emqx_packets_pubcomp_sent, counter, 'packets.pubcomp.sent'}, + {emqx_packets_pubcomp_inuse, counter, 'packets.pubcomp.inuse'}, + {emqx_packets_pubcomp_missed, counter, 'packets.pubcomp.missed'}, + %% pingreq + {emqx_packets_pingreq_received, counter, 'packets.pingreq.received'}, + {emqx_packets_pingresp_sent, counter, 'packets.pingresp.sent'}, + %% disconnect + {emqx_packets_disconnect_received, counter, 'packets.disconnect.received'}, + {emqx_packets_disconnect_sent, counter, 'packets.disconnect.sent'}, + %% auth + {emqx_packets_auth_received, counter, 'packets.auth.received'}, + {emqx_packets_auth_sent, counter, 'packets.auth.sent'} ]. +%%========== +%% Messages +message_metric_meta() -> + [ + {emqx_messages_received, counter, 'messages.received'}, + {emqx_messages_sent, counter, 'messages.sent'}, + {emqx_messages_qos0_received, counter, 'messages.qos0.received'}, + {emqx_messages_qos0_sent, counter, 'messages.qos0.sent'}, + {emqx_messages_qos1_received, counter, 'messages.qos1.received'}, + {emqx_messages_qos1_sent, counter, 'messages.qos1.sent'}, + {emqx_messages_qos2_received, counter, 'messages.qos2.received'}, + {emqx_messages_qos2_sent, counter, 'messages.qos2.sent'}, + {emqx_messages_publish, counter, 'messages.publish'}, + {emqx_messages_dropped, counter, 'messages.dropped'}, + {emqx_messages_dropped_expired, counter, 'messages.dropped.await_pubrel_timeout'}, + {emqx_messages_dropped_no_subscribers, counter, 'messages.dropped.no_subscribers'}, + {emqx_messages_forward, counter, 'messages.forward'}, + {emqx_messages_retained, counter, 'messages.retained'}, + {emqx_messages_delayed, counter, 'messages.delayed'}, + {emqx_messages_delivered, counter, 'messages.delivered'}, + {emqx_messages_acked, counter, 'messages.acked'} + ]. + +%%========== +%% Delivery +delivery_metric_meta() -> + [ + {emqx_delivery_dropped, counter, 'delivery.dropped'}, + {emqx_delivery_dropped_no_local, counter, 'delivery.dropped.no_local'}, + {emqx_delivery_dropped_too_large, counter, 'delivery.dropped.too_large'}, + {emqx_delivery_dropped_qos0_msg, counter, 'delivery.dropped.qos0_msg'}, + {emqx_delivery_dropped_queue_full, counter, 'delivery.dropped.queue_full'}, + {emqx_delivery_dropped_expired, counter, 'delivery.dropped.expired'} + ]. + +%%========== +%% Client +client_metric_meta() -> + [ + {emqx_client_connect, counter, 'client.connect'}, + {emqx_client_connack, counter, 'client.connack'}, + {emqx_client_connected, counter, 'client.connected'}, + {emqx_client_authenticate, counter, 'client.authenticate'}, + {emqx_client_auth_anonymous, counter, 'client.auth.anonymous'}, + {emqx_client_authorize, counter, 'client.authorize'}, + {emqx_client_subscribe, counter, 'client.subscribe'}, + {emqx_client_unsubscribe, counter, 'client.unsubscribe'}, + {emqx_client_disconnected, counter, 'client.disconnected'} + ]. + +%%========== +%% Metrics - session +session_metric_meta() -> + [ + {emqx_session_created, counter, 'session.created'}, + {emqx_session_resumed, counter, 'session.resumed'}, + {emqx_session_takenover, counter, 'session.takenover'}, + {emqx_session_discarded, counter, 'session.discarded'}, + {emqx_session_terminated, counter, 'session.terminated'} + ]. + +%%========== +%% Metrics - acl +acl_metric_meta() -> + [ + {emqx_authorization_allow, counter, 'authorization.allow'}, + {emqx_authorization_deny, counter, 'authorization.deny'}, + {emqx_authorization_cache_hit, counter, 'authorization.cache_hit'}, + {emqx_authorization_cache_miss, counter, 'authorization.cache_miss'}, + {emqx_authorization_superuser, counter, 'authorization.superuser'}, + {emqx_authorization_nomatch, counter, 'authorization.nomatch'}, + {emqx_authorization_matched_allow, counter, 'authorization.matched_allow'}, + {emqx_authorization_matched_deny, counter, 'authorization.matched_deny'} + ]. + +%%========== +%% Metrics - authn +authn_metric_meta() -> + [ + {emqx_authentication_success, counter, 'authentication.success'}, + {emqx_authentication_success_anonymous, counter, 'authentication.success.anonymous'}, + {emqx_authentication_failure, counter, 'authentication.failure'} + ]. + +%%========== +%% Overload Protection +olp_metric_meta() -> + emqx_metrics_olp_meta(emqx_config_zones:is_olp_enabled()). + +emqx_metrics_olp_meta(true) -> + [ + {emqx_overload_protection_delay_ok, counter, 'overload_protection.delay.ok'}, + {emqx_overload_protection_delay_timeout, counter, 'overload_protection.delay.timeout'}, + {emqx_overload_protection_hibernation, counter, 'overload_protection.hibernation'}, + {emqx_overload_protection_gc, counter, 'overload_protection.gc'}, + {emqx_overload_protection_new_conn, counter, 'overload_protection.new_conn'} + ]; +emqx_metrics_olp_meta(false) -> + []. + +%%======================================== +%% License +%%======================================== + +-if(?EMQX_RELEASE_EDITION == ee). + +maybe_license_add_collect_family(Callback, RawData) -> + ok = add_collect_family(Callback, license_metric_meta(), ?MG(license_data, RawData)), + ok. + +maybe_license_fetch_data() -> + #{license_data => license_data()}. + +maybe_license_collect_json_data(RawData) -> + #{license => ?MG(license_data, RawData)}. + +%% license +license_metric_meta() -> + [ + {emqx_license_expiry_at, gauge, undefined} + ]. + +license_data() -> + #{emqx_license_expiry_at => emqx_license_checker:expiry_epoch()}. + +-else. + +maybe_license_add_collect_family(_, _) -> + ok. + +maybe_license_fetch_data() -> + #{}. + +maybe_license_collect_json_data(_RawData) -> + #{}. + +-endif. + +%%======================================== +%% Certs +%%======================================== + +cert_metric_meta() -> + [ + {emqx_cert_expiry_at, gauge, undefined} + ]. + +-define(LISTENER_TYPES, [ssl, wss, quic]). + +-spec cert_data() -> + [_Point :: {[Label], Epoch}] +when + Label :: TypeLabel | NameLabel, + TypeLabel :: {listener_type, ssl | wss | quic}, + NameLabel :: {listener_name, atom()}, + Epoch :: non_neg_integer(). +cert_data() -> + cert_data(emqx_config:get([listeners], undefined)). + +cert_data(undefined) -> + []; +cert_data(AllListeners) -> + Points = lists:foldl( + fun(ListenerType, PointsAcc) -> + PointsAcc ++ + points_of_listeners(ListenerType, AllListeners) + end, + _PointsInitAcc = [], + ?LISTENER_TYPES + ), + #{ + emqx_cert_expiry_at => Points + }. + +points_of_listeners(Type, AllListeners) -> + do_points_of_listeners(Type, maps:get(Type, AllListeners, undefined)). + +-define(CERT_TYPES, [certfile]). + +-spec do_points_of_listeners(Type, TypeOfListeners) -> + [_Point :: {[{LabelKey, LabelValue}], Epoch}] +when + Type :: ssl | wss | quic, + TypeOfListeners :: #{ListenerName :: atom() => ListenerConf :: map()} | undefined, + LabelKey :: atom(), + LabelValue :: atom(), + Epoch :: non_neg_integer(). +do_points_of_listeners(_, undefined) -> + []; +do_points_of_listeners(ListenerType, TypeOfListeners) -> + lists:foldl( + fun(Name, PointsAcc) -> + lists:foldl( + fun(CertType, AccIn) -> + case + emqx_utils_maps:deep_get( + [Name, ssl_options, CertType], TypeOfListeners, undefined + ) + of + undefined -> AccIn; + Path -> [gen_point(ListenerType, Name, Path) | AccIn] + end + end, + [], + ?CERT_TYPES + ) ++ PointsAcc + end, + [], + maps:keys(TypeOfListeners) + ). + +gen_point(Type, Name, Path) -> + {[{listener_type, Type}, {listener_name, Name}], cert_expiry_at_from_path(Path)}. + +%% TODO: cert manager for more generic utils functions +cert_expiry_at_from_path(Path0) -> + Path = emqx_schema:naive_env_interpolation(Path0), + {ok, PemBin} = file:read_file(Path), + [CertEntry | _] = public_key:pem_decode(PemBin), + Cert = public_key:pem_entry_decode(CertEntry), + %% TODO: Not fully tested for all certs type + {'utcTime', NotAfterUtc} = + Cert#'Certificate'.'tbsCertificate'#'TBSCertificate'.validity#'Validity'.'notAfter', + utc_time_to_epoch(NotAfterUtc). + +utc_time_to_epoch(UtcTime) -> + date_to_expiry_epoch(utc_time_to_datetime(UtcTime)). + +utc_time_to_datetime(Str) -> + {ok, [Year, Month, Day, Hour, Minute, Second], _} = io_lib:fread( + "~2d~2d~2d~2d~2d~2dZ", Str + ), + %% Alwoys Assuming YY is in 2000 + {{2000 + Year, Month, Day}, {Hour, Minute, Second}}. + +%% 62167219200 =:= calendar:datetime_to_gregorian_seconds({{1970, 1, 1}, {0, 0, 0}}). +-define(EPOCH_START, 62167219200). +-spec date_to_expiry_epoch(calendar:datetime()) -> Seconds :: non_neg_integer(). +date_to_expiry_epoch(DateTime) -> + calendar:datetime_to_gregorian_seconds(DateTime) - ?EPOCH_START. + +%%-------------------------------------------------------------------- +%% Collect functions +%%-------------------------------------------------------------------- + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% merge / zip formatting funcs for type `application/json` + +collect_stats_json_data(RawData) -> + StatsData = ?MG(stats_data, RawData), + StatsClData = ?MG(stats_data_cluster_consistented, RawData), + D = maps:merge(StatsData, StatsClData), + collect_json_data(D). + +%% always return json array +collect_cert_json_data(Data) -> + collect_json_data_(Data). + +collect_vm_json_data(Data) -> + DataListPerNode = collect_json_data_(Data), + case {?GET_PROM_DATA_MODE(), DataListPerNode} of + {?PROM_DATA_MODE__NODE, [NData | _]} -> + NData; + {_, _} -> + DataListPerNode + end. + +collect_json_data(Data0) -> + DataListPerNode = collect_json_data_(Data0), + case {?GET_PROM_DATA_MODE(), DataListPerNode} of + %% all nodes results unaggregated, should be a list + {?PROM_DATA_MODE__ALL_NODES_UNAGGREGATED, _} -> + DataListPerNode; + %% only local node result [#{...}] + %% To guaranteed compatibility, return a json object, not array + {?PROM_DATA_MODE__NODE, [NData | _]} -> + NData; + %% All nodes results aggregated + %% return a json object, not array + {?PROM_DATA_MODE__ALL_NODES_AGGREGATED, [NData | _]} -> + NData; + %% olp maybe not enabled, with empty list to empty object + {_, []} -> + #{} + end. + +collect_json_data_(Data) -> + emqx_prometheus_cluster:collect_json_data(Data, fun zip_json_prom_stats_metrics/3). + +zip_json_prom_stats_metrics(Key, Points, [] = _AccIn) -> + lists:foldl( + fun({Lables, Metric}, AccIn2) -> + LablesKVMap = maps:from_list(Lables), + Point = LablesKVMap#{Key => Metric}, + [Point | AccIn2] + end, + [], + Points + ); +zip_json_prom_stats_metrics(Key, Points, AllResultedAcc) -> + ThisKeyResult = lists:foldl(emqx_prometheus_cluster:point_to_map_fun(Key), [], Points), + lists:zipwith(fun maps:merge/2, AllResultedAcc, ThisKeyResult). + +metrics_name(MetricsAll) -> + [Name || {Name, _, _} <- MetricsAll]. + +with_node_label(?PROM_DATA_MODE__NODE, Labels) -> + Labels; +with_node_label(?PROM_DATA_MODE__ALL_NODES_AGGREGATED, Labels) -> + Labels; +with_node_label(?PROM_DATA_MODE__ALL_NODES_UNAGGREGATED, Labels) -> + [{node, node(self())} | Labels]. + +%%-------------------------------------------------------------------- +%% bpapi + %% deprecated_since 5.0.10, remove this when 5.1.x do_start() -> emqx_prometheus_sup:start_child(?APP). diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index 44e0fac16..89bfa6e6a 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -18,20 +18,37 @@ -behaviour(minirest_api). +-include("emqx_prometheus.hrl"). -include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-import( + hoconsc, + [ + mk/2, + ref/1 + ] +). -export([ api_spec/0, paths/0, - schema/1 + schema/1, + fields/1 ]). -export([ setting/2, - stats/2 + stats/2, + auth/2, + data_integration/2 ]). +-export([lookup_from_local_nodes/3]). + -define(TAGS, [<<"Monitor">>]). +-define(IS_TRUE(Val), ((Val =:= true) orelse (Val =:= <<"true">>))). +-define(IS_FALSE(Val), ((Val =:= false) orelse (Val =:= <<"false">>))). api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). @@ -39,7 +56,9 @@ api_spec() -> paths() -> [ "/prometheus", - "/prometheus/stats" + "/prometheus/auth", + "/prometheus/stats", + "/prometheus/data_integration" ]. schema("/prometheus") -> @@ -61,6 +80,19 @@ schema("/prometheus") -> #{200 => prometheus_setting_response()} } }; +schema("/prometheus/auth") -> + #{ + 'operationId' => auth, + get => + #{ + description => ?DESC(get_prom_auth_data), + tags => ?TAGS, + parameters => [ref(mode)], + security => security(), + responses => + #{200 => prometheus_data_schema()} + } + }; schema("/prometheus/stats") -> #{ 'operationId' => stats, @@ -68,6 +100,20 @@ schema("/prometheus/stats") -> #{ description => ?DESC(get_prom_data), tags => ?TAGS, + parameters => [ref(mode)], + security => security(), + responses => + #{200 => prometheus_data_schema()} + } + }; +schema("/prometheus/data_integration") -> + #{ + 'operationId' => data_integration, + get => + #{ + description => ?DESC(get_prom_data_integration_data), + tags => ?TAGS, + parameters => [ref(mode)], security => security(), responses => #{200 => prometheus_data_schema()} @@ -79,6 +125,41 @@ security() -> true -> [#{'basicAuth' => []}, #{'bearerAuth' => []}]; false -> [] end. + +%% erlfmt-ignore +fields(mode) -> + [ + {mode, + mk( + hoconsc:enum(?PROM_DATA_MODES), + #{ + default => node, + desc => <<" +Metrics format mode. + +`node`: +Return metrics from local node. And it is the default behaviour if `mode` not specified. + +`all_nodes_aggregated`: +Return metrics for all nodes. +And if possible, calculate the arithmetic sum or logical sum of the indicators of all nodes. + +`all_nodes_unaggregated`: +Return metrics from all nodes, and the metrics are not aggregated. +The node name will be included in the returned results to +indicate that certain metrics were returned on a certain node. +">>, + in => query, + required => false, + example => node + } + )} + ]. + +%% bpapi +lookup_from_local_nodes(M, F, A) -> + erlang:apply(M, F, A). + %%-------------------------------------------------------------------- %% API Handler funcs %%-------------------------------------------------------------------- @@ -100,24 +181,60 @@ setting(put, #{body := Body}) -> {500, 'INTERNAL_ERROR', Message} end. -stats(get, #{headers := Headers}) -> - Type = - case maps:get(<<"accept">>, Headers, <<"text/plain">>) of - <<"application/json">> -> <<"json">>; - _ -> <<"prometheus">> - end, - Data = emqx_prometheus:collect(Type), - case Type of - <<"json">> -> - {200, Data}; - <<"prometheus">> -> - {200, #{<<"content-type">> => <<"text/plain">>}, Data} - end. +stats(get, #{headers := Headers, query_string := Qs}) -> + collect(emqx_prometheus, collect_opts(Headers, Qs)). + +auth(get, #{headers := Headers, query_string := Qs}) -> + collect(emqx_prometheus_auth, collect_opts(Headers, Qs)). + +data_integration(get, #{headers := Headers, query_string := Qs}) -> + collect(emqx_prometheus_data_integration, collect_opts(Headers, Qs)). %%-------------------------------------------------------------------- %% Internal funcs %%-------------------------------------------------------------------- +collect(Module, #{type := Type, mode := Mode}) -> + %% `Mode` is used to control the format of the returned data + %% It will used in callback `Module:collect_mf/1` to fetch data from node or cluster + %% And use this mode parameter to determine the formatting method of the returned information. + %% Since the arity of the callback function has been fixed. + %% so it is placed in the process dictionary of the current process. + ?PUT_PROM_DATA_MODE(Mode), + Data = + case erlang:function_exported(Module, collect, 1) of + true -> + erlang:apply(Module, collect, [Type]); + false -> + ?SLOG(error, #{ + msg => "prometheus callback module not found, empty data responded", + module_name => Module + }), + <<>> + end, + gen_response(Type, Data). + +collect_opts(Headers, Qs) -> + #{type => response_type(Headers), mode => mode(Qs)}. + +response_type(#{<<"accept">> := <<"application/json">>}) -> + <<"json">>; +response_type(_) -> + <<"prometheus">>. + +mode(#{<<"mode">> := Mode}) -> + case lists:member(Mode, ?PROM_DATA_MODES) of + true -> Mode; + false -> ?PROM_DATA_MODE__NODE + end; +mode(_) -> + ?PROM_DATA_MODE__NODE. + +gen_response(<<"json">>, Data) -> + {200, Data}; +gen_response(<<"prometheus">>, Data) -> + {200, #{<<"content-type">> => <<"text/plain">>}, Data}. + prometheus_setting_request() -> [{prometheus, #{type := Setting}}] = emqx_prometheus_schema:roots(), emqx_dashboard_swagger:schema_with_examples( @@ -181,7 +298,7 @@ recommend_setting_example() -> prometheus_data_schema() -> #{ description => - <<"Get Prometheus Data. Note that support for JSON output is deprecated and will be removed in v5.2.">>, + <<"Get Prometheus Data.">>, content => [ {'text/plain', #{schema => #{type => string}}}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus_auth.erl b/apps/emqx_prometheus/src/emqx_prometheus_auth.erl new file mode 100644 index 000000000..f9086cfb8 --- /dev/null +++ b/apps/emqx_prometheus/src/emqx_prometheus_auth.erl @@ -0,0 +1,498 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_prometheus_auth). + +-export([ + deregister_cleanup/1, + collect_mf/2, + collect_metrics/2 +]). + +-export([collect/1]). + +%% for bpapi +-behaviour(emqx_prometheus_cluster). +-export([ + fetch_from_local_node/1, + fetch_cluster_consistented_data/0, + aggre_or_zip_init_acc/0, + logic_sum_metrics/0 +]). + +%% %% @private +-export([ + zip_json_auth_metrics/3 +]). + +-include("emqx_prometheus.hrl"). +-include_lib("emqx_auth/include/emqx_authn_chains.hrl"). +-include_lib("prometheus/include/prometheus.hrl"). + +-import( + prometheus_model_helpers, + [ + create_mf/5, + gauge_metric/1, + gauge_metrics/1, + counter_metrics/1 + ] +). + +-type authn_metric_name() :: + emqx_authn_enable + | emqx_authn_status + | emqx_authn_nomatch + | emqx_authn_total + | emqx_authn_success + | emqx_authn_failed. + +-type authz_metric_name() :: + emqx_authz_enable + | emqx_authz_status + | emqx_authz_nomatch + | emqx_authz_total + | emqx_authz_allow + | emqx_authz_deny. + +%% Please don't remove this attribute, prometheus uses it to +%% automatically register collectors. +-behaviour(prometheus_collector). + +%%-------------------------------------------------------------------- +%% Macros +%%-------------------------------------------------------------------- + +-define(METRIC_NAME_PREFIX, "emqx_auth_"). + +-define(MG(K, MAP), maps:get(K, MAP)). +-define(MG0(K, MAP), maps:get(K, MAP, 0)). +-define(PG0(K, PROPLISTS), proplists:get_value(K, PROPLISTS, 0)). + +%%-------------------------------------------------------------------- +%% Collector API +%%-------------------------------------------------------------------- + +%% @private +deregister_cleanup(_) -> ok. + +%% @private +-spec collect_mf(_Registry, Callback) -> ok when + _Registry :: prometheus_registry:registry(), + Callback :: prometheus_collector:collect_mf_callback(). +%% erlfmt-ignore +collect_mf(?PROMETHEUS_AUTH_REGISTRY, Callback) -> + RawData = emqx_prometheus_cluster:raw_data(?MODULE, ?GET_PROM_DATA_MODE()), + ok = add_collect_family(Callback, authn_metric_meta(), ?MG(authn_data, RawData)), + ok = add_collect_family(Callback, authn_users_count_metric_meta(), ?MG(authn_users_count_data, RawData)), + ok = add_collect_family(Callback, authz_metric_meta(), ?MG(authz_data, RawData)), + ok = add_collect_family(Callback, authz_rules_count_metric_meta(), ?MG(authz_rules_count_data, RawData)), + ok = add_collect_family(Callback, banned_count_metric_meta(), ?MG(banned_count_data, RawData)), + ok; +collect_mf(_, _) -> + ok. + +%% @private +collect(<<"json">>) -> + RawData = emqx_prometheus_cluster:raw_data(?MODULE, ?GET_PROM_DATA_MODE()), + #{ + emqx_authn => collect_json_data(?MG(authn_data, RawData)), + emqx_authz => collect_json_data(?MG(authz_data, RawData)), + emqx_banned => collect_banned_data() + }; +collect(<<"prometheus">>) -> + prometheus_text_format:format(?PROMETHEUS_AUTH_REGISTRY). + +add_collect_family(Callback, MetricWithType, Data) -> + _ = [add_collect_family(Name, Data, Callback, Type) || {Name, Type} <- MetricWithType], + ok. + +add_collect_family(Name, Data, Callback, Type) -> + Callback(create_mf(Name, _Help = <<"">>, Type, ?MODULE, Data)). + +collect_metrics(Name, Metrics) -> + collect_auth(Name, Metrics). + +%% behaviour +fetch_from_local_node(Mode) -> + {node(self()), #{ + authn_data => authn_data(Mode), + authz_data => authz_data(Mode) + }}. + +fetch_cluster_consistented_data() -> + #{ + authn_users_count_data => authn_users_count_data(), + authz_rules_count_data => authz_rules_count_data(), + banned_count_data => banned_count_data() + }. + +aggre_or_zip_init_acc() -> + #{ + authn_data => maps:from_keys(authn_metric(names), []), + authz_data => maps:from_keys(authz_metric(names), []) + }. + +logic_sum_metrics() -> + [ + emqx_authn_enable, + emqx_authn_status, + emqx_authz_enable, + emqx_authz_status + ]. + +%%-------------------------------------------------------------------- +%% Collector +%%-------------------------------------------------------------------- + +%%==================== +%% Authn overview +collect_auth(K = emqx_authn_enable, Data) -> + gauge_metrics(?MG(K, Data)); +collect_auth(K = emqx_authn_status, Data) -> + gauge_metrics(?MG(K, Data)); +collect_auth(K = emqx_authn_nomatch, Data) -> + counter_metrics(?MG(K, Data)); +collect_auth(K = emqx_authn_total, Data) -> + counter_metrics(?MG(K, Data)); +collect_auth(K = emqx_authn_success, Data) -> + counter_metrics(?MG(K, Data)); +collect_auth(K = emqx_authn_failed, Data) -> + counter_metrics(?MG(K, Data)); +%%==================== +%% Authn users count +%% Only provided for `password_based:built_in_database` and `scram:built_in_database` +collect_auth(K = emqx_authn_users_count, Data) -> + gauge_metrics(?MG(K, Data)); +%%==================== +%% Authz overview +collect_auth(K = emqx_authz_enable, Data) -> + gauge_metrics(?MG(K, Data)); +collect_auth(K = emqx_authz_status, Data) -> + gauge_metrics(?MG(K, Data)); +collect_auth(K = emqx_authz_nomatch, Data) -> + counter_metrics(?MG(K, Data)); +collect_auth(K = emqx_authz_total, Data) -> + counter_metrics(?MG(K, Data)); +collect_auth(K = emqx_authz_allow, Data) -> + counter_metrics(?MG(K, Data)); +collect_auth(K = emqx_authz_deny, Data) -> + counter_metrics(?MG(K, Data)); +%%==================== +%% Authz rules count +%% Only provided for `file` and `built_in_database` +collect_auth(K = emqx_authz_rules_count, Data) -> + gauge_metrics(?MG(K, Data)); +%%==================== +%% Banned +collect_auth(emqx_banned_count, Data) -> + gauge_metric(Data). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +%%======================================== +%% AuthN (Authentication) +%%======================================== + +%%==================== +%% Authn overview + +authn_metric_meta() -> + [ + {emqx_authn_enable, gauge}, + {emqx_authn_status, gauge}, + {emqx_authn_nomatch, counter}, + {emqx_authn_total, counter}, + {emqx_authn_success, counter}, + {emqx_authn_failed, counter} + ]. + +authn_metric(names) -> + emqx_prometheus_cluster:metric_names(authn_metric_meta()). + +-spec authn_data(atom()) -> #{Key => [Point]} when + Key :: authn_metric_name(), + Point :: {[Label], Metric}, + Label :: IdLabel, + IdLabel :: {id, AuthnName :: binary()}, + Metric :: number(). +authn_data(Mode) -> + Authns = emqx_config:get([authentication]), + lists:foldl( + fun(Key, AccIn) -> + AccIn#{Key => authn_backend_to_points(Mode, Key, Authns)} + end, + #{}, + authn_metric(names) + ). + +-spec authn_backend_to_points(atom(), Key, list(Authn)) -> list(Point) when + Key :: authn_metric_name(), + Authn :: map(), + Point :: {[Label], Metric}, + Label :: IdLabel, + IdLabel :: {id, AuthnName :: binary()}, + Metric :: number(). +authn_backend_to_points(Mode, Key, Authns) -> + do_authn_backend_to_points(Mode, Key, Authns, []). + +do_authn_backend_to_points(_Mode, _K, [], AccIn) -> + lists:reverse(AccIn); +do_authn_backend_to_points(Mode, K, [Authn | Rest], AccIn) -> + Id = authenticator_id(Authn), + Point = { + with_node_label(Mode, [{id, Id}]), + do_metric(K, Authn, lookup_authn_metrics_local(Id)) + }, + do_authn_backend_to_points(Mode, K, Rest, [Point | AccIn]). + +lookup_authn_metrics_local(Id) -> + case emqx_authn_api:lookup_from_local_node(?GLOBAL, Id) of + {ok, {_Node, Status, #{counters := Counters}, _ResourceMetrics}} -> + #{ + emqx_authn_status => emqx_prometheus_cluster:status_to_number(Status), + emqx_authn_nomatch => ?MG0(nomatch, Counters), + emqx_authn_total => ?MG0(total, Counters), + emqx_authn_success => ?MG0(success, Counters), + emqx_authn_failed => ?MG0(failed, Counters) + }; + {error, _Reason} -> + maps:from_keys(authn_metric(names) -- [emqx_authn_enable], 0) + end. + +%%==================== +%% Authn users count + +authn_users_count_metric_meta() -> + [ + {emqx_authn_users_count, gauge} + ]. + +-define(AUTHN_MNESIA, emqx_authn_mnesia). +-define(AUTHN_SCRAM_MNESIA, emqx_authn_scram_mnesia). + +authn_users_count_data() -> + Samples = lists:foldl( + fun + (#{backend := built_in_database, mechanism := password_based} = Authn, AccIn) -> + [auth_data_sample_point(authn, Authn, ?AUTHN_MNESIA) | AccIn]; + (#{backend := built_in_database, mechanism := scram} = Authn, AccIn) -> + [auth_data_sample_point(authn, Authn, ?AUTHN_SCRAM_MNESIA) | AccIn]; + (_, AccIn) -> + AccIn + end, + [], + emqx_config:get([authentication]) + ), + #{emqx_authn_users_count => Samples}. + +%%======================================== +%% AuthZ (Authorization) +%%======================================== + +%%==================== +%% Authz overview + +authz_metric_meta() -> + [ + {emqx_authz_enable, gauge}, + {emqx_authz_status, gauge}, + {emqx_authz_nomatch, counter}, + {emqx_authz_total, counter}, + {emqx_authz_allow, counter}, + {emqx_authz_deny, counter} + ]. + +authz_metric(names) -> + emqx_prometheus_cluster:metric_names(authz_metric_meta()). + +-spec authz_data(atom()) -> #{Key => [Point]} when + Key :: authz_metric_name(), + Point :: {[Label], Metric}, + Label :: TypeLabel, + TypeLabel :: {type, AuthZType :: binary()}, + Metric :: number(). +authz_data(Mode) -> + Authzs = emqx_config:get([authorization, sources]), + lists:foldl( + fun(Key, AccIn) -> + AccIn#{Key => authz_backend_to_points(Mode, Key, Authzs)} + end, + #{}, + authz_metric(names) + ). + +-spec authz_backend_to_points(atom(), Key, list(Authz)) -> list(Point) when + Key :: authz_metric_name(), + Authz :: map(), + Point :: {[Label], Metric}, + Label :: TypeLabel, + TypeLabel :: {type, AuthZType :: binary()}, + Metric :: number(). +authz_backend_to_points(Mode, Key, Authzs) -> + do_authz_backend_to_points(Mode, Key, Authzs, []). + +do_authz_backend_to_points(_Mode, _K, [], AccIn) -> + lists:reverse(AccIn); +do_authz_backend_to_points(Mode, K, [Authz | Rest], AccIn) -> + Type = maps:get(type, Authz), + Point = { + with_node_label(Mode, [{type, Type}]), + do_metric(K, Authz, lookup_authz_metrics_local(Type)) + }, + do_authz_backend_to_points(Mode, K, Rest, [Point | AccIn]). + +lookup_authz_metrics_local(Type) -> + case emqx_authz_api_sources:lookup_from_local_node(Type) of + {ok, {_Node, Status, #{counters := Counters}, _ResourceMetrics}} -> + #{ + emqx_authz_status => emqx_prometheus_cluster:status_to_number(Status), + emqx_authz_nomatch => ?MG0(nomatch, Counters), + emqx_authz_total => ?MG0(total, Counters), + emqx_authz_allow => ?MG0(allow, Counters), + emqx_authz_deny => ?MG0(deny, Counters) + }; + {error, _Reason} -> + maps:from_keys(authz_metric(names) -- [emqx_authz_enable], 0) + end. + +%%==================== +%% Authz rules count + +authz_rules_count_metric_meta() -> + [ + {emqx_authz_rules_count, gauge} + ]. + +-define(ACL_TABLE, emqx_acl). + +authz_rules_count_data() -> + Samples = lists:foldl( + fun + (#{type := built_in_database} = Authz, AccIn) -> + [auth_data_sample_point(authz, Authz, ?ACL_TABLE) | AccIn]; + (#{type := file}, AccIn) -> + #{annotations := #{rules := Rules}} = emqx_authz:lookup(file), + Size = erlang:length(Rules), + [{[{type, file}], Size} | AccIn]; + (_, AccIn) -> + AccIn + end, + [], + emqx_config:get([authorization, sources]) + ), + #{emqx_authz_rules_count => Samples}. + +%%======================================== +%% Banned +%%======================================== + +%%==================== +%% Banned count + +banned_count_metric_meta() -> + [ + {emqx_banned_count, gauge} + ]. +-define(BANNED_TABLE, + emqx_banned +). +banned_count_data() -> + mnesia_size(?BANNED_TABLE). + +%%-------------------------------------------------------------------- +%% Collect functions +%%-------------------------------------------------------------------- + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% merge / zip formatting funcs for type `application/json` + +collect_json_data(Data) -> + emqx_prometheus_cluster:collect_json_data(Data, fun zip_json_auth_metrics/3). + +collect_banned_data() -> + #{emqx_banned_count => banned_count_data()}. + +%% for initialized empty AccIn +%% The following fields will be put into Result +%% For Authn: +%% `id`, `emqx_authn_users_count` +%% For Authz: +%% `type`, `emqx_authz_rules_count`n +zip_json_auth_metrics(Key, Points, [] = _AccIn) -> + lists:foldl( + fun({Lables, Metric}, AccIn2) -> + LablesKVMap = maps:from_list(Lables), + Point = (maps:merge(LablesKVMap, users_or_rule_count(LablesKVMap)))#{Key => Metric}, + [Point | AccIn2] + end, + [], + Points + ); +zip_json_auth_metrics(Key, Points, AllResultedAcc) -> + ThisKeyResult = lists:foldl(emqx_prometheus_cluster:point_to_map_fun(Key), [], Points), + lists:zipwith(fun maps:merge/2, AllResultedAcc, ThisKeyResult). + +users_or_rule_count(#{id := Id}) -> + #{emqx_authn_users_count := Points} = authn_users_count_data(), + case lists:keyfind([{id, Id}], 1, Points) of + {_, Metric} -> + #{emqx_authn_users_count => Metric}; + false -> + #{} + end; +users_or_rule_count(#{type := Type}) -> + #{emqx_authz_rules_count := Points} = authz_rules_count_data(), + case lists:keyfind([{type, Type}], 1, Points) of + {_, Metric} -> + #{emqx_authz_rules_count => Metric}; + false -> + #{} + end; +users_or_rule_count(_) -> + #{}. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Helper funcs + +authenticator_id(Authn) -> + emqx_authn_chains:authenticator_id(Authn). + +auth_data_sample_point(authn, Authn, Tab) -> + Size = mnesia_size(Tab), + Id = authenticator_id(Authn), + {[{id, Id}], Size}; +auth_data_sample_point(authz, #{type := Type} = _Authz, Tab) -> + Size = mnesia_size(Tab), + {[{type, Type}], Size}. + +mnesia_size(Tab) -> + mnesia:table_info(Tab, size). + +do_metric(emqx_authn_enable, #{enable := B}, _) -> + emqx_prometheus_cluster:boolean_to_number(B); +do_metric(emqx_authz_enable, #{enable := B}, _) -> + emqx_prometheus_cluster:boolean_to_number(B); +do_metric(K, _, Metrics) -> + ?MG0(K, Metrics). + +with_node_label(?PROM_DATA_MODE__NODE, Labels) -> + Labels; +with_node_label(?PROM_DATA_MODE__ALL_NODES_AGGREGATED, Labels) -> + Labels; +with_node_label(?PROM_DATA_MODE__ALL_NODES_UNAGGREGATED, Labels) -> + [{node, node(self())} | Labels]. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_cluster.erl b/apps/emqx_prometheus/src/emqx_prometheus_cluster.erl new file mode 100644 index 000000000..c4f1dc3b8 --- /dev/null +++ b/apps/emqx_prometheus/src/emqx_prometheus_cluster.erl @@ -0,0 +1,205 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_prometheus_cluster). + +-include("emqx_prometheus.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). + +-export([ + raw_data/2, + + collect_json_data/2, + + aggre_cluster/3, + %% with_node_name_label/2, + + point_to_map_fun/1, + + boolean_to_number/1, + status_to_number/1, + metric_names/1 +]). + +-callback fetch_cluster_consistented_data() -> map(). + +-callback fetch_from_local_node(atom()) -> {node(), map()}. + +-callback aggre_or_zip_init_acc() -> map(). + +-callback logic_sum_metrics() -> list(). + +-define(MG(K, MAP), maps:get(K, MAP)). +-define(PG0(K, PROPLISTS), proplists:get_value(K, PROPLISTS, 0)). + +raw_data(Module, undefined) -> + %% TODO: for push gateway, the format mode should be configurable + raw_data(Module, ?PROM_DATA_MODE__NODE); +raw_data(Module, ?PROM_DATA_MODE__ALL_NODES_AGGREGATED = Mode) -> + AllNodesMetrics = aggre_cluster(Module, Mode), + Cluster = Module:fetch_cluster_consistented_data(), + maps:merge(AllNodesMetrics, Cluster); +raw_data(Module, ?PROM_DATA_MODE__ALL_NODES_UNAGGREGATED = Mode) -> + AllNodesMetrics = zip_cluster_data(Module, Mode), + Cluster = Module:fetch_cluster_consistented_data(), + maps:merge(AllNodesMetrics, Cluster); +raw_data(Module, ?PROM_DATA_MODE__NODE = Mode) -> + {_Node, LocalNodeMetrics} = Module:fetch_from_local_node(Mode), + Cluster = Module:fetch_cluster_consistented_data(), + maps:merge(LocalNodeMetrics, Cluster). + +fetch_data_from_all_nodes(Module, Mode) -> + Nodes = mria:running_nodes(), + _ResL = emqx_prometheus_proto_v2:raw_prom_data( + Nodes, Module, fetch_from_local_node, [Mode] + ). + +collect_json_data(Data, Func) when is_function(Func, 3) -> + maps:fold( + fun(K, V, Acc) -> + Func(K, V, Acc) + end, + [], + Data + ); +collect_json_data(_, _) -> + error(badarg). + +aggre_cluster(Module, Mode) -> + do_aggre_cluster( + Module:logic_sum_metrics(), + fetch_data_from_all_nodes(Module, Mode), + Module:aggre_or_zip_init_acc() + ). + +aggre_cluster(LogicSumKs, ResL, Init) -> + do_aggre_cluster(LogicSumKs, ResL, Init). + +do_aggre_cluster(_LogicSumKs, [], AccIn) -> + AccIn; +do_aggre_cluster(LogicSumKs, [{ok, {_NodeName, NodeMetric}} | Rest], AccIn) -> + do_aggre_cluster( + LogicSumKs, + Rest, + maps:fold( + fun(K, V, AccIn0) -> + AccIn0#{K => aggre_metric(LogicSumKs, V, ?MG(K, AccIn0))} + end, + AccIn, + NodeMetric + ) + ); +do_aggre_cluster(LogicSumKs, [{_, _} | Rest], AccIn) -> + do_aggre_cluster(LogicSumKs, Rest, AccIn). + +aggre_metric(LogicSumKs, NodeMetrics, AccIn0) -> + lists:foldl( + fun(K, AccIn) -> + NAccL = do_aggre_metric( + K, LogicSumKs, ?MG(K, NodeMetrics), ?MG(K, AccIn) + ), + AccIn#{K => NAccL} + end, + AccIn0, + maps:keys(NodeMetrics) + ). + +do_aggre_metric(K, LogicSumKs, NodeMetrics, AccL) -> + lists:foldl( + fun(Point = {_Labels, _Metric}, AccIn) -> + sum(K, LogicSumKs, Point, AccIn) + end, + AccL, + NodeMetrics + ). + +sum(K, LogicSumKs, {Labels, Metric} = Point, MetricAccL) -> + case lists:keytake(Labels, 1, MetricAccL) of + {value, {Labels, MetricAcc}, NMetricAccL} -> + NPoint = {Labels, do_sum(K, LogicSumKs, Metric, MetricAcc)}, + [NPoint | NMetricAccL]; + false -> + [Point | MetricAccL] + end. + +do_sum(K, LogicSumKs, Metric, MetricAcc) -> + case lists:member(K, LogicSumKs) of + true -> + logic_sum(Metric, MetricAcc); + false -> + Metric + MetricAcc + end. + +zip_cluster_data(Module, Mode) -> + zip_cluster( + fetch_data_from_all_nodes(Module, Mode), + Module:aggre_or_zip_init_acc() + ). + +zip_cluster([], AccIn) -> + AccIn; +zip_cluster([{ok, {_NodeName, NodeMetric}} | Rest], AccIn) -> + zip_cluster( + Rest, + maps:fold( + fun(K, V, AccIn0) -> + AccIn0#{ + K => do_zip_cluster(V, ?MG(K, AccIn0)) + } + end, + AccIn, + NodeMetric + ) + ); +zip_cluster([{_, _} | Rest], AccIn) -> + zip_cluster(Rest, AccIn). + +do_zip_cluster(NodeMetrics, AccIn0) -> + lists:foldl( + fun(K, AccIn) -> + AccMetricL = ?MG(K, AccIn), + NAccL = ?MG(K, NodeMetrics) ++ AccMetricL, + AccIn#{K => NAccL} + end, + AccIn0, + maps:keys(NodeMetrics) + ). + +point_to_map_fun(Key) -> + fun({Lables, Metric}, AccIn2) -> + LablesKVMap = maps:from_list(Lables), + [maps:merge(LablesKVMap, #{Key => Metric}) | AccIn2] + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +logic_sum(N1, N2) when + (N1 > 0 andalso N2 > 0) +-> + 1; +logic_sum(_, _) -> + 0. + +boolean_to_number(true) -> 1; +boolean_to_number(false) -> 0. + +status_to_number(?status_connected) -> 1; +status_to_number(?status_connecting) -> 0; +status_to_number(?status_disconnected) -> 0; +status_to_number(?rm_status_stopped) -> 0; +status_to_number(_) -> 0. + +metric_names(MetricWithType) when is_list(MetricWithType) -> + [Name || {Name, _Type} <- MetricWithType]. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_config.erl b/apps/emqx_prometheus/src/emqx_prometheus_config.erl index a24b52537..f5140938c 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_config.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_config.erl @@ -25,6 +25,10 @@ -export([conf/0, is_push_gateway_server_enabled/1]). -export([to_recommend_type/1]). +-ifdef(TEST). +-export([all_collectors/0]). +-endif. + update(Config) -> case emqx_conf:update( @@ -101,7 +105,7 @@ post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) -> ok. update_prometheus(AppEnvs) -> - PrevCollectors = prometheus_registry:collectors(default), + PrevCollectors = all_collectors(), CurCollectors = proplists:get_value(collectors, proplists:get_value(prometheus, AppEnvs)), lists:foreach( fun prometheus_registry:deregister_collector/1, @@ -113,6 +117,15 @@ update_prometheus(AppEnvs) -> ), application:set_env(AppEnvs). +all_collectors() -> + lists:foldl( + fun(Registry, AccIn) -> + prometheus_registry:collectors(Registry) ++ AccIn + end, + _InitAcc = [], + ?PROMETHEUS_ALL_REGISTRYS + ). + update_push_gateway(Prometheus) -> case is_push_gateway_server_enabled(Prometheus) of true -> diff --git a/apps/emqx_prometheus/src/emqx_prometheus_data_integration.erl b/apps/emqx_prometheus/src/emqx_prometheus_data_integration.erl new file mode 100644 index 000000000..f3bc98f50 --- /dev/null +++ b/apps/emqx_prometheus/src/emqx_prometheus_data_integration.erl @@ -0,0 +1,543 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_prometheus_data_integration). + +-export([ + deregister_cleanup/1, + collect_mf/2, + collect_metrics/2 +]). + +-export([collect/1]). + +-export([ + zip_json_data_integration_metrics/3 +]). + +%% for bpapi +-behaviour(emqx_prometheus_cluster). +-export([ + fetch_from_local_node/1, + fetch_cluster_consistented_data/0, + aggre_or_zip_init_acc/0, + logic_sum_metrics/0 +]). + +-export([add_collect_family/4]). + +-include("emqx_prometheus.hrl"). +-include_lib("prometheus/include/prometheus.hrl"). + +-import( + prometheus_model_helpers, + [ + create_mf/5, + gauge_metric/1, + gauge_metrics/1, + counter_metrics/1 + ] +). + +%% Please don't remove this attribute, prometheus uses it to +%% automatically register collectors. +-behaviour(prometheus_collector). + +%%-------------------------------------------------------------------- +%% Macros +%%-------------------------------------------------------------------- + +-define(METRIC_NAME_PREFIX, "emqx_data_integration_"). + +-define(MG(K, MAP), maps:get(K, MAP)). +-define(MG0(K, MAP), maps:get(K, MAP, 0)). + +%%-------------------------------------------------------------------- +%% Callback for emqx_prometheus_cluster +%%-------------------------------------------------------------------- + +-define(ROOT_KEY_ACTIONS, actions). + +fetch_from_local_node(Mode) -> + Rules = emqx_rule_engine:get_rules(), + Bridges = emqx_bridge_v2:list(?ROOT_KEY_ACTIONS), + Connectors = emqx_connector:list(), + {node(self()), #{ + rule_metric_data => rule_metric_data(Mode, Rules), + action_metric_data => action_metric_data(Mode, Bridges), + connector_metric_data => connector_metric_data(Mode, Connectors) + }}. + +fetch_cluster_consistented_data() -> + Rules = emqx_rule_engine:get_rules(), + Connectors = emqx_connector:list(), + (maybe_collect_schema_registry())#{ + rules_ov_data => rules_ov_data(Rules), + connectors_ov_data => connectors_ov_data(Connectors) + }. + +aggre_or_zip_init_acc() -> + #{ + rule_metric_data => maps:from_keys(rule_metric(names), []), + action_metric_data => maps:from_keys(action_metric(names), []), + connector_metric_data => maps:from_keys(connectr_metric(names), []) + }. + +logic_sum_metrics() -> + [ + emqx_rule_enable, + emqx_connector_enable, + emqx_connector_status + ]. + +%%-------------------------------------------------------------------- +%% Collector API +%%-------------------------------------------------------------------- + +%% @private +deregister_cleanup(_) -> ok. + +%% @private +-spec collect_mf(_Registry, Callback) -> ok when + _Registry :: prometheus_registry:registry(), + Callback :: prometheus_collector:collect_mf_callback(). +collect_mf(?PROMETHEUS_DATA_INTEGRATION_REGISTRY, Callback) -> + RawData = emqx_prometheus_cluster:raw_data(?MODULE, ?GET_PROM_DATA_MODE()), + + %% Data Integration Overview + ok = add_collect_family(Callback, rules_ov_metric_meta(), ?MG(rules_ov_data, RawData)), + ok = add_collect_family( + Callback, connectors_ov_metric_meta(), ?MG(connectors_ov_data, RawData) + ), + ok = maybe_collect_family_schema_registry(Callback), + + %% Rule Metric + RuleMetricDs = ?MG(rule_metric_data, RawData), + ok = add_collect_family(Callback, rule_metric_meta(), RuleMetricDs), + + %% Action Metric + ActionMetricDs = ?MG(action_metric_data, RawData), + ok = add_collect_family(Callback, action_metric_meta(), ActionMetricDs), + + %% Connector Metric + ConnectorMetricDs = ?MG(connector_metric_data, RawData), + ok = add_collect_family(Callback, connector_metric_meta(), ConnectorMetricDs), + + ok; +collect_mf(_, _) -> + ok. + +%% @private +collect(<<"json">>) -> + RawData = emqx_prometheus_cluster:raw_data(?MODULE, ?GET_PROM_DATA_MODE()), + Rules = emqx_rule_engine:get_rules(), + Bridges = emqx_bridge:list(), + #{ + data_integration_overview => collect_data_integration_overview(Rules, Bridges), + rules => collect_json_data(?MG(rule_metric_data, RawData)), + actions => collect_json_data(?MG(action_metric_data, RawData)), + connectors => collect_json_data(?MG(connector_metric_data, RawData)) + }; +collect(<<"prometheus">>) -> + prometheus_text_format:format(?PROMETHEUS_DATA_INTEGRATION_REGISTRY). + +%%==================== +%% API Helpers + +add_collect_family(Callback, MetricWithType, Data) -> + _ = [add_collect_family(Name, Data, Callback, Type) || {Name, Type} <- MetricWithType], + ok. + +add_collect_family(Name, Data, Callback, Type) -> + %% TODO: help document from Name + Callback(create_mf(Name, _Help = <<"">>, Type, ?MODULE, Data)). + +collect_metrics(Name, Metrics) -> + collect_di(Name, Metrics). + +%%-------------------------------------------------------------------- +%% Collector +%%-------------------------------------------------------------------- + +%%======================================== +%% Data Integration Overview +%%======================================== + +%%==================== +%% All Rules +%% Rules +collect_di(K = emqx_rules_count, Data) -> gauge_metric(?MG(K, Data)); +%%==================== +%% Schema Registry +collect_di(K = emqx_schema_registrys_count, Data) -> gauge_metric(?MG(K, Data)); +%%==================== +%% Connectors +collect_di(K = emqx_connectors_count, Data) -> gauge_metric(?MG(K, Data)); +%%======================================== +%% Data Integration Metric for: Rule && Action && Connector +%%======================================== + +%%==================== +%% Rule Metric +collect_di(K = emqx_rule_enable, Data) -> gauge_metrics(?MG(K, Data)); +collect_di(K = emqx_rule_matched, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_rule_failed, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_rule_passed, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_rule_failed_exception, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_rule_failed_no_result, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_rule_actions_total, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_rule_actions_success, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_rule_actions_failed, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_rule_actions_failed_out_of_service, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_rule_actions_failed_unknown, Data) -> counter_metrics(?MG(K, Data)); +%%==================== +%% Action Metric +collect_di(K = emqx_action_matched, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_action_dropped, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_action_success, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_action_failed, Data) -> counter_metrics(?MG(K, Data)); +%% inflight type: gauge +collect_di(K = emqx_action_inflight, Data) -> gauge_metrics(?MG(K, Data)); +collect_di(K = emqx_action_received, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_action_late_reply, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_action_retried, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_action_retried_success, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_action_retried_failed, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_action_dropped_resource_stopped, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_action_dropped_resource_not_found, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_action_dropped_queue_full, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_action_dropped_other, Data) -> counter_metrics(?MG(K, Data)); +collect_di(K = emqx_action_dropped_expired, Data) -> counter_metrics(?MG(K, Data)); +%% queuing type: gauge +collect_di(K = emqx_action_queuing, Data) -> gauge_metrics(?MG(K, Data)); +%%==================== +%% Connector Metric +collect_di(K = emqx_connector_enable, Data) -> gauge_metrics(?MG(K, Data)); +collect_di(K = emqx_connector_status, Data) -> gauge_metrics(?MG(K, Data)). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +%%======================================== +%% Data Integration Overview +%%======================================== + +%%==================== +%% All Rules + +rules_ov_metric_meta() -> + [ + {emqx_rules_count, gauge} + ]. + +rules_ov_metric(names) -> + emqx_prometheus_cluster:metric_names(rules_ov_metric_meta()). + +-define(RULE_TAB, emqx_rule_engine). +rules_ov_data(_Rules) -> + #{ + emqx_rules_count => ets:info(?RULE_TAB, size) + }. + +%%==================== +%% Schema Registry + +-if(?EMQX_RELEASE_EDITION == ee). + +maybe_collect_family_schema_registry(Callback) -> + ok = add_collect_family(Callback, schema_registry_metric_meta(), schema_registry_data()), + ok. + +schema_registry_metric_meta() -> + [ + {emqx_schema_registrys_count, gauge} + ]. + +schema_registry_data() -> + #{ + emqx_schema_registrys_count => erlang:map_size(emqx_schema_registry:list_schemas()) + }. + +maybe_collect_schema_registry() -> + schema_registry_data(). + +-else. + +maybe_collect_family_schema_registry(_) -> + ok. + +maybe_collect_schema_registry() -> + #{}. + +-endif. + +%%==================== +%% Connectors + +connectors_ov_metric_meta() -> + [ + {emqx_connectors_count, gauge} + ]. + +connectors_ov_metric(names) -> + emqx_prometheus_cluster:metric_names(connectors_ov_metric_meta()). + +connectors_ov_data(Connectors) -> + #{ + %% Both Bridge V1 and V2 + emqx_connectors_count => erlang:length(Connectors) + }. + +%%======================================== +%% Data Integration Metric for: Rule && Action && Connector +%%======================================== + +%%==================== +%% Rule Metric +%% With rule_id as label key: `rule_id` + +rule_metric_meta() -> + [ + {emqx_rule_enable, gauge}, + {emqx_rule_matched, counter}, + {emqx_rule_failed, counter}, + {emqx_rule_passed, counter}, + {emqx_rule_failed_exception, counter}, + {emqx_rule_failed_no_result, counter}, + {emqx_rule_actions_total, counter}, + {emqx_rule_actions_success, counter}, + {emqx_rule_actions_failed, counter}, + {emqx_rule_actions_failed_out_of_service, counter}, + {emqx_rule_actions_failed_unknown, counter} + ]. + +rule_metric(names) -> + emqx_prometheus_cluster:metric_names(rule_metric_meta()). + +rule_metric_data(Mode, Rules) -> + lists:foldl( + fun(#{id := Id} = Rule, AccIn) -> + merge_acc_with_rules(Mode, Id, get_metric(Rule), AccIn) + end, + maps:from_keys(rule_metric(names), []), + Rules + ). + +merge_acc_with_rules(Mode, Id, RuleMetrics, PointsAcc) -> + maps:fold( + fun(K, V, AccIn) -> + AccIn#{K => [rule_point(Mode, Id, V) | ?MG(K, AccIn)]} + end, + PointsAcc, + RuleMetrics + ). + +rule_point(Mode, Id, V) -> + {with_node_label(Mode, [{id, Id}]), V}. + +get_metric(#{id := Id, enable := Bool} = _Rule) -> + case emqx_metrics_worker:get_metrics(rule_metrics, Id) of + #{counters := Counters} -> + #{ + emqx_rule_enable => emqx_prometheus_cluster:boolean_to_number(Bool), + emqx_rule_matched => ?MG(matched, Counters), + emqx_rule_failed => ?MG(failed, Counters), + emqx_rule_passed => ?MG(passed, Counters), + emqx_rule_failed_exception => ?MG('failed.exception', Counters), + emqx_rule_failed_no_result => ?MG('failed.no_result', Counters), + emqx_rule_actions_total => ?MG('actions.total', Counters), + emqx_rule_actions_success => ?MG('actions.success', Counters), + emqx_rule_actions_failed => ?MG('actions.failed', Counters), + emqx_rule_actions_failed_out_of_service => ?MG( + 'actions.failed.out_of_service', Counters + ), + emqx_rule_actions_failed_unknown => ?MG('actions.failed.unknown', Counters) + } + end. + +%%==================== +%% Action Metric +%% With action_id: `{type}:{name}` as label key: `action_id` + +action_metric_meta() -> + [ + {emqx_action_matched, counter}, + {emqx_action_dropped, counter}, + {emqx_action_success, counter}, + {emqx_action_failed, counter}, + {emqx_action_inflight, gauge}, + {emqx_action_received, counter}, + {emqx_action_late_reply, counter}, + {emqx_action_retried, counter}, + {emqx_action_retried_success, counter}, + {emqx_action_retried_failed, counter}, + {emqx_action_dropped_resource_stopped, counter}, + {emqx_action_dropped_resource_not_found, counter}, + {emqx_action_dropped_queue_full, counter}, + {emqx_action_dropped_other, counter}, + {emqx_action_dropped_expired, counter}, + {emqx_action_queuing, gauge} + ]. + +action_metric(names) -> + emqx_prometheus_cluster:metric_names(action_metric_meta()). + +action_metric_data(Mode, Bridges) -> + lists:foldl( + fun(#{type := Type, name := Name} = _Bridge, AccIn) -> + Id = emqx_bridge_resource:bridge_id(Type, Name), + merge_acc_with_bridges(Mode, Id, get_bridge_metric(Type, Name), AccIn) + end, + maps:from_keys(action_metric(names), []), + Bridges + ). + +merge_acc_with_bridges(Mode, Id, BridgeMetrics, PointsAcc) -> + maps:fold( + fun(K, V, AccIn) -> + AccIn#{K => [action_point(Mode, Id, V) | ?MG(K, AccIn)]} + end, + PointsAcc, + BridgeMetrics + ). + +action_point(Mode, Id, V) -> + {with_node_label(Mode, [{id, Id}]), V}. + +get_bridge_metric(Type, Name) -> + #{counters := Counters, gauges := Gauges} = emqx_bridge_v2:get_metrics(Type, Name), + #{ + emqx_action_matched => ?MG0(matched, Counters), + emqx_action_dropped => ?MG0(dropped, Counters), + emqx_action_success => ?MG0(success, Counters), + emqx_action_failed => ?MG0(failed, Counters), + emqx_action_inflight => ?MG0(inflight, Gauges), + emqx_action_received => ?MG0(received, Counters), + emqx_action_late_reply => ?MG0(late_reply, Counters), + emqx_action_retried => ?MG0(retried, Counters), + emqx_action_retried_success => ?MG0('retried.success', Counters), + emqx_action_retried_failed => ?MG0('retried.failed', Counters), + emqx_action_dropped_resource_stopped => ?MG0('dropped.resource_stopped', Counters), + emqx_action_dropped_resource_not_found => ?MG0('dropped.resource_not_found', Counters), + emqx_action_dropped_queue_full => ?MG0('dropped.queue_full', Counters), + emqx_action_dropped_other => ?MG0('dropped.other', Counters), + emqx_action_dropped_expired => ?MG0('dropped.expired', Counters), + emqx_action_queuing => ?MG0(queuing, Gauges) + }. + +%%==================== +%% Connector Metric +%% With connector_id: `{type}:{name}` as label key: `connector_id` + +connector_metric_meta() -> + [ + {emqx_connector_enable, gauge}, + {emqx_connector_status, gauge} + ]. + +connectr_metric(names) -> + emqx_prometheus_cluster:metric_names(connector_metric_meta()). + +connector_metric_data(Mode, Connectors) -> + lists:foldl( + fun(#{type := Type, name := Name} = Connector, AccIn) -> + Id = emqx_connector_resource:connector_id(Type, Name), + merge_acc_with_connectors(Mode, Id, get_connector_status(Connector), AccIn) + end, + maps:from_keys(connectr_metric(names), []), + Connectors + ). + +merge_acc_with_connectors(Mode, Id, ConnectorMetrics, PointsAcc) -> + maps:fold( + fun(K, V, AccIn) -> + AccIn#{K => [connector_point(Mode, Id, V) | ?MG(K, AccIn)]} + end, + PointsAcc, + ConnectorMetrics + ). + +connector_point(Mode, Id, V) -> + {with_node_label(Mode, [{id, Id}]), V}. + +get_connector_status(#{resource_data := ResourceData} = _Connector) -> + Enabled = emqx_utils_maps:deep_get([config, enable], ResourceData), + Status = ?MG(status, ResourceData), + #{ + emqx_connector_enable => emqx_prometheus_cluster:boolean_to_number(Enabled), + emqx_connector_status => emqx_prometheus_cluster:status_to_number(Status) + }. + +%%-------------------------------------------------------------------- +%% Collect functions +%%-------------------------------------------------------------------- + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% merge / zip formatting funcs for type `application/json` +collect_data_integration_overview(Rules, Bridges) -> + RulesD = rules_ov_data(Rules), + ConnectorsD = connectors_ov_data(Bridges), + + M1 = lists:foldl( + fun(K, AccIn) -> AccIn#{K => ?MG(K, RulesD)} end, + #{}, + rules_ov_metric(names) + ), + M2 = lists:foldl( + fun(K, AccIn) -> AccIn#{K => ?MG(K, ConnectorsD)} end, + #{}, + connectors_ov_metric(names) + ), + M3 = maybe_collect_schema_registry(), + + lists:foldl(fun(M, AccIn) -> maps:merge(M, AccIn) end, #{}, [M1, M2, M3]). + +collect_json_data(Data) -> + emqx_prometheus_cluster:collect_json_data(Data, fun zip_json_data_integration_metrics/3). + +%% for initialized empty AccIn +%% The following fields will be put into Result +%% For Rules: +%% `id` => [RULE_ID] +%% For Actions +%% `id` => [ACTION_ID] +%% FOR Connectors +%% `id` => [CONNECTOR_ID] %% CONNECTOR_ID = BRIDGE_ID +%% formatted with {type}:{name} +zip_json_data_integration_metrics(Key, Points, [] = _AccIn) -> + lists:foldl( + fun({Lables, Metric}, AccIn2) -> + LablesKVMap = maps:from_list(Lables), + Point = LablesKVMap#{Key => Metric}, + [Point | AccIn2] + end, + [], + Points + ); +zip_json_data_integration_metrics(Key, Points, AllResultedAcc) -> + ThisKeyResult = lists:foldl(emqx_prometheus_cluster:point_to_map_fun(Key), [], Points), + lists:zipwith(fun maps:merge/2, AllResultedAcc, ThisKeyResult). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Helper funcs + +with_node_label(?PROM_DATA_MODE__NODE, Labels) -> + Labels; +with_node_label(?PROM_DATA_MODE__ALL_NODES_AGGREGATED, Labels) -> + Labels; +with_node_label(?PROM_DATA_MODE__ALL_NODES_UNAGGREGATED, Labels) -> + [{node, node(self())} | Labels]. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl index ea8a2ebaa..1c7eb73e4 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl @@ -67,6 +67,9 @@ init([]) -> Children = case emqx_prometheus_config:is_push_gateway_server_enabled(Conf) of false -> []; + %% TODO: add push gateway for endpoints + %% `/prometheus/auth` + %% `/prometheus/data_integration` true -> [?CHILD(emqx_prometheus, Conf)] end, {ok, {{one_for_one, 10, 3600}, Children}}. diff --git a/apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v2.erl b/apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v2.erl new file mode 100644 index 000000000..e770dc0ab --- /dev/null +++ b/apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v2.erl @@ -0,0 +1,53 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_prometheus_proto_v2). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + start/1, + stop/1, + + raw_prom_data/4 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.5.0". + +-spec start([node()]) -> emqx_rpc:multicall_result(). +start(Nodes) -> + rpc:multicall(Nodes, emqx_prometheus, do_start, [], 5000). + +-spec stop([node()]) -> emqx_rpc:multicall_result(). +stop(Nodes) -> + rpc:multicall(Nodes, emqx_prometheus, do_stop, [], 5000). + +-type key() :: atom(). +-type arg() :: list(term()). + +-spec raw_prom_data([node()], key(), key(), arg()) -> emqx_rpc:erpc_multicall(term()). +raw_prom_data(Nodes, M, F, A) -> + erpc:multicall( + Nodes, + emqx_prometheus_api, + lookup_from_local_nodes, + [M, F, A], + 5000 + ). diff --git a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl index 496919b10..11ca49f89 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl @@ -103,13 +103,16 @@ init_group() -> ok = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], infinity), meck:new(emqx_alarm, [non_strict, passthrough, no_link]), meck:expect(emqx_alarm, activate, 3, ok), - meck:expect(emqx_alarm, deactivate, 3, ok). + meck:expect(emqx_alarm, deactivate, 3, ok), + meck:new(emqx_license_checker, [non_strict, passthrough, no_link]), + meck:expect(emqx_license_checker, expiry_epoch, fun() -> 1859673600 end). end_group() -> ekka:stop(), mria:stop(), mria_mnesia:delete_schema(), meck:unload(emqx_alarm), + meck:unload(emqx_license_checker), emqx_common_test_helpers:stop_apps([emqx_prometheus]). end_per_group(_Group, Config) -> diff --git a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl index cc20e60c7..6092a5d54 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl @@ -128,8 +128,8 @@ t_legacy_prometheus_api(_) -> Conf2 = emqx_utils_json:decode(Response2, [return_maps]), ?assertEqual(NewConf, Conf2), - EnvCollectors = application:get_env(prometheus, collectors, []), - PromCollectors = prometheus_registry:collectors(default), + EnvCollectors = env_collectors(), + PromCollectors = all_collectors(), ?assertEqual(lists:sort(EnvCollectors), lists:sort(PromCollectors)), ?assert(lists:member(prometheus_vm_statistics_collector, EnvCollectors), EnvCollectors), @@ -221,8 +221,8 @@ t_prometheus_api(_) -> Conf2 = emqx_utils_json:decode(Response2, [return_maps]), ?assertMatch(NewConf, Conf2), - EnvCollectors = application:get_env(prometheus, collectors, []), - PromCollectors = prometheus_registry:collectors(default), + EnvCollectors = env_collectors(), + PromCollectors = all_collectors(), ?assertEqual(lists:sort(EnvCollectors), lists:sort(PromCollectors)), ?assert(lists:member(prometheus_vm_statistics_collector, EnvCollectors), EnvCollectors), @@ -308,3 +308,16 @@ request_stats(JsonAuth, Auth) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Internal Functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +env_collectors() -> + do_env_collectors(application:get_env(prometheus, collectors, []), []). + +do_env_collectors([], Acc) -> + lists:reverse(Acc); +do_env_collectors([{_Registry, Collector} | Rest], Acc) when is_atom(Collector) -> + do_env_collectors(Rest, [Collector | Acc]); +do_env_collectors([Collector | Rest], Acc) when is_atom(Collector) -> + do_env_collectors(Rest, [Collector | Acc]). + +all_collectors() -> + emqx_prometheus_config:all_collectors(). diff --git a/apps/emqx_redis/src/emqx_redis.app.src b/apps/emqx_redis/src/emqx_redis.app.src index 1f8c5fbc3..660c490e6 100644 --- a/apps/emqx_redis/src/emqx_redis.app.src +++ b/apps/emqx_redis/src/emqx_redis.app.src @@ -1,13 +1,12 @@ {application, emqx_redis, [ {description, "EMQX Redis Database Connector"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, stdlib, eredis, eredis_cluster, - emqx_connector, emqx_resource ]}, {env, []}, diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 4fd566f26..b16520d3d 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -50,7 +50,8 @@ ]). -export([ - set_resource_status_connecting/1 + set_resource_status_connecting/1, + make_test_id/0 ]). % Server diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl index bb232f9e4..6d5eee477 100644 --- a/apps/emqx_retainer/src/emqx_retainer_api.erl +++ b/apps/emqx_retainer/src/emqx_retainer_api.erl @@ -44,7 +44,11 @@ api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). paths() -> - [?PREFIX, ?PREFIX ++ "/messages", ?PREFIX ++ "/message/:topic"]. + [ + ?PREFIX, + ?PREFIX ++ "/messages", + ?PREFIX ++ "/message/:topic" + ]. schema(?PREFIX) -> #{ diff --git a/apps/emqx_utils/src/emqx_utils.erl b/apps/emqx_utils/src/emqx_utils.erl index 8d7c622a4..0eeef2e5e 100644 --- a/apps/emqx_utils/src/emqx_utils.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -51,6 +51,8 @@ gen_id/0, gen_id/1, explain_posix/1, + pforeach/2, + pforeach/3, pmap/2, pmap/3, readable_error_msg/1, @@ -423,6 +425,15 @@ explain_posix(estale) -> "Stale remote file handle"; explain_posix(exdev) -> "Cross-domain link"; explain_posix(NotPosix) -> NotPosix. +-spec pforeach(fun((A) -> term()), list(A)) -> ok. +pforeach(Fun, List) when is_function(Fun, 1), is_list(List) -> + pforeach(Fun, List, ?DEFAULT_PMAP_TIMEOUT). + +-spec pforeach(fun((A) -> term()), list(A), timeout()) -> ok. +pforeach(Fun, List, Timeout) -> + _ = pmap(Fun, List, Timeout), + ok. + %% @doc Like lists:map/2, only the callback function is evaluated %% concurrently. -spec pmap(fun((A) -> B), list(A)) -> list(B). @@ -431,7 +442,9 @@ pmap(Fun, List) when is_function(Fun, 1), is_list(List) -> -spec pmap(fun((A) -> B), list(A), timeout()) -> list(B). pmap(Fun, List, Timeout) when - is_function(Fun, 1), is_list(List), is_integer(Timeout), Timeout >= 0 + is_function(Fun, 1), + is_list(List), + (is_integer(Timeout) andalso Timeout >= 0 orelse Timeout =:= infinity) -> nolink_apply(fun() -> do_parallel_map(Fun, List) end, Timeout). diff --git a/changes/ce/feat-12299.en.md b/changes/ce/feat-12299.en.md new file mode 100644 index 000000000..1721970e4 --- /dev/null +++ b/changes/ce/feat-12299.en.md @@ -0,0 +1,15 @@ +Expose more metrics to improve observability: + +Montior API: + - Add `retained_msg_count` field to `/api/v5/monitor_current`. + - Add `retained_msg_count` and `node_uptime` fields to `/api/v5/monitor_current/nodes/{node}`. + +Prometheus API: + - Add `emqx_cert_expiry_at` to `/api/v5/prometheus/stats` to display TLS listener certificate expiration time. + - Add `/api/v5/prometheus/auth` endpoint to provide metrics such as execution count and running status for all authenticatiors and authorizators. + - Add `/api/v5/prometheus/data_integration` endpoint to provide metrics such as execution count and status for all rules, actions, and connectors. + +Limitations: + Prometheus push gateway only supports content in `/api/v5/prometheus/stats?mode=node` for now. + +For more API details and metric type information. Please see also in swagger api docs. diff --git a/changes/ce/fix-12359.en.md b/changes/ce/fix-12359.en.md new file mode 100644 index 000000000..a4b2a725c --- /dev/null +++ b/changes/ce/fix-12359.en.md @@ -0,0 +1 @@ +Fixed an issue that could lead to error messages when restarting a node configured with some types of data bridges. Said bridges could also start in a failed state, requiring manual restart. diff --git a/changes/ce/perf-12354.en.md b/changes/ce/perf-12354.en.md new file mode 100644 index 000000000..ac0be69a0 --- /dev/null +++ b/changes/ce/perf-12354.en.md @@ -0,0 +1,3 @@ +Apply post config bridge changes in parallel. +This can greatly improve the performance when multiple bridges are being changed, +e.g. when a backup file is being imported. diff --git a/changes/ee/feat-12299.en.md b/changes/ee/feat-12299.en.md new file mode 100644 index 000000000..629928b90 --- /dev/null +++ b/changes/ee/feat-12299.en.md @@ -0,0 +1,17 @@ +# Expose more metrics to improve observability: + +Montior API: + - Add `retained_msg_count` field to `/api/v5/monitor_current`. + - Add `license_quota` field to `/api/v5/monitor_current` + - Add `retained_msg_count` and `node_uptime` fields to `/api/v5/monitor_current/nodes/{node}`. + - Add `retained_msg_count`, `license_quota` and `node_uptime` fields to `/api/v5/monitor_current/nodes/{node}`. + +Prometheus API: + - Add `emqx_cert_expiry_at` and `emqx_license_expiry_at` to `/api/v5/prometheus/stats` to display TLS listener certificate expiration time and license expiration time. + - Add `/api/v5/prometheus/auth` endpoint to provide metrics such as execution count and running status for all authenticatiors and authorizators. + - Add `/api/v5/prometheus/data_integration` endpoint to provide metrics such as execution count and status for all rules, actions, and connectors. + +Limitations: + Prometheus push gateway only supports the content in `/api/v5/prometheus/stats?mode=node` + +For more API details and metric type information. Please see also in swagger api docs. diff --git a/mix.exs b/mix.exs index 7389cb0f9..bb79c3204 100644 --- a/mix.exs +++ b/mix.exs @@ -55,7 +55,7 @@ defmodule EMQXUmbrella.MixProject do {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true}, {:esockd, github: "emqx/esockd", tag: "5.11.1", override: true}, {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-2", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.18.1", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.18.3", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "3.3.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.15", override: true}, diff --git a/rebar.config b/rebar.config index a1f17c321..7a2e7ff8c 100644 --- a/rebar.config +++ b/rebar.config @@ -83,7 +83,7 @@ {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.11.1"}}}, {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.8.0-emqx-2"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.18.1"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.18.3"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}}, {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.12"}}}, {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.15"}}}, diff --git a/rel/i18n/emqx_bridge_es.hocon b/rel/i18n/emqx_bridge_es.hocon index 8ad11f05b..1cff9dbb9 100644 --- a/rel/i18n/emqx_bridge_es.hocon +++ b/rel/i18n/emqx_bridge_es.hocon @@ -35,11 +35,6 @@ config_auth_basic_password.desc: config_auth_basic_password.label: """HTTP Basic Auth Password""" -config_base_url.desc: -"""The base URL of the external ElasticSearch service's REST interface.""" -config_base_url.label: -"""ElasticSearch REST Service Base URL""" - config_target.desc: """Name of the data stream, index, or index alias to perform bulk actions on""" diff --git a/rel/i18n/emqx_bridge_es_connector.hocon b/rel/i18n/emqx_bridge_es_connector.hocon index ddd53e0fc..21406b840 100644 --- a/rel/i18n/emqx_bridge_es_connector.hocon +++ b/rel/i18n/emqx_bridge_es_connector.hocon @@ -1,5 +1,13 @@ emqx_bridge_es_connector { +server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to. +A host entry has the following form: `Host[:Port]`. +The Elasticsearch default port 9200 is used if `[:Port]` is not specified.""" + +server.label: +"""Server Host""" + config_authentication.desc: """Authentication configuration""" diff --git a/rel/i18n/emqx_prometheus_api.hocon b/rel/i18n/emqx_prometheus_api.hocon index 0d9b5dc5f..0c48e3add 100644 --- a/rel/i18n/emqx_prometheus_api.hocon +++ b/rel/i18n/emqx_prometheus_api.hocon @@ -15,4 +15,14 @@ get_prom_data.desc: get_prom_data.label: """Prometheus Metrics""" +get_prom_auth_data.desc: +"""Get Prometheus Metrics for AuthN, AuthZ and Banned""" +get_prom_auth_data.label: +"""Prometheus Metrics for Auth""" + +get_prom_data_integration_data.desc: +"""Get Prometheus Metrics for Data Integration""" +get_prom_data_integration_data.label: +"""Prometheus Metrics for Data Integration""" + }