chore: add refactor prometheus refactor changelog

This commit is contained in:
zhongwencool 2023-11-06 09:42:04 +08:00
parent b343653769
commit c4eb9f86e7
9 changed files with 169 additions and 22 deletions

View File

@ -59,6 +59,12 @@
-export([collect/1]). -export([collect/1]).
-export([
%% For bpapi, deprecated_since 5.0.10, remove this when 5.1.x
do_start/0,
do_stop/0
]).
-define(C(K, L), proplists:get_value(K, L, 0)). -define(C(K, L), proplists:get_value(K, L, 0)).
-define(TIMER_MSG, '#interval'). -define(TIMER_MSG, '#interval').
@ -112,7 +118,6 @@ handle_info({update, Conf}, State = #{timer := Timer}) ->
handle_info(_Msg, State) -> handle_info(_Msg, State) ->
{noreply, State}. {noreply, State}.
push_to_push_gateway(Url, Headers) when is_list(Headers) -> push_to_push_gateway(Url, Headers) when is_list(Headers) ->
Data = prometheus_text_format:format(), Data = prometheus_text_format:format(),
case httpc:request(post, {Url, Headers, "text/plain", Data}, ?HTTP_OPTIONS, []) of case httpc:request(post, {Url, Headers, "text/plain", Data}, ?HTTP_OPTIONS, []) of
@ -678,3 +683,11 @@ emqx_cluster_data() ->
{nodes_running, length(Running)}, {nodes_running, length(Running)},
{nodes_stopped, length(Stopped)} {nodes_stopped, length(Stopped)}
]. ].
%% deprecated_since 5.0.10, remove this when 5.1.x
do_start() ->
emqx_prometheus_sup:start_child(?APP).
%% deprecated_since 5.0.10, remove this when 5.1.x
do_stop() ->
emqx_prometheus_sup:stop_child(?APP).

View File

@ -20,7 +20,7 @@
-include("emqx_prometheus.hrl"). -include("emqx_prometheus.hrl").
-export([add_handler/0, remove_handler/0]). -export([add_handler/0, remove_handler/0]).
-export([post_config_update/5]). -export([pre_config_update/3, post_config_update/5]).
-export([update/1]). -export([update/1]).
-export([conf/0, is_push_gateway_server_enabled/1]). -export([conf/0, is_push_gateway_server_enabled/1]).
@ -46,6 +46,54 @@ remove_handler() ->
ok = emqx_config_handler:remove_handler(?PROMETHEUS), ok = emqx_config_handler:remove_handler(?PROMETHEUS),
ok. ok.
%% when we import the config with the old version
%% we need to respect it, and convert to new schema.
pre_config_update(?PROMETHEUS, MergeConf, OriginConf) ->
OriginType = emqx_prometheus_schema:is_recommend_type(OriginConf),
MergeType = emqx_prometheus_schema:is_recommend_type(MergeConf),
{ok,
case {OriginType, MergeType} of
{true, false} -> to_recommend_type(MergeConf);
_ -> MergeConf
end}.
to_recommend_type(Conf) ->
#{
<<"push_gateway">> => to_push_gateway(Conf),
<<"collectors">> => to_collectors(Conf)
}.
to_push_gateway(Conf) ->
Init = maps:with([<<"interval">>, <<"headers">>, <<"job_name">>], Conf),
case maps:get(<<"push_gateway_server">>, Conf, "") of
"" ->
Init#{<<"url">> => <<"">>};
Url ->
case maps:get(<<"enable">>, Conf, false) of
false -> Init#{<<"url">> => <<"">>};
true -> Init#{<<"url">> => Url}
end
end.
to_collectors(Conf) ->
lists:foldl(
fun({From, To}, Acc) ->
case maps:find(From, Conf) of
{ok, Value} -> Acc#{To => Value};
error -> Acc
end
end,
#{},
[
{<<"vm_dist_collector">>, <<"vm_dist">>},
{<<"mnesia_collector">>, <<"mnesia">>},
{<<"vm_statistics_collector">>, <<"vm_statistics">>},
{<<"vm_system_info_collector">>, <<"vm_system_info">>},
{<<"vm_memory_collector">>, <<"vm_memory">>},
{<<"vm_msacc_collector">>, <<"vm_msacc">>}
]
).
post_config_update(?PROMETHEUS, _Req, New, Old, AppEnvs) -> post_config_update(?PROMETHEUS, _Req, New, Old, AppEnvs) ->
update_prometheus(AppEnvs), update_prometheus(AppEnvs),
_ = update_push_gateway(New), _ = update_push_gateway(New),

View File

@ -27,7 +27,8 @@
desc/1, desc/1,
translation/1, translation/1,
convert_headers/2, convert_headers/2,
validate_url/1 validate_url/1,
is_recommend_type/1
]). ]).
namespace() -> prometheus. namespace() -> prometheus.
@ -63,7 +64,7 @@ fields(recommend_setting) ->
} }
)}, )},
{collectors, {collectors,
?HOCON(?R_REF(collector), #{ ?HOCON(?R_REF(collectors), #{
required => false, required => false,
importance => ?IMPORTANCE_LOW, importance => ?IMPORTANCE_LOW,
desc => ?DESC(collectors) desc => ?DESC(collectors)
@ -110,7 +111,7 @@ fields(push_gateway) ->
} }
)} )}
]; ];
fields(collector) -> fields(collectors) ->
[ [
{vm_dist, {vm_dist,
?HOCON( ?HOCON(
@ -295,13 +296,35 @@ setting_union_schema() ->
RecommendSetting = ?R_REF(recommend_setting), RecommendSetting = ?R_REF(recommend_setting),
LegacySetting = ?R_REF(legacy_deprecated_setting), LegacySetting = ?R_REF(legacy_deprecated_setting),
fun fun
(all_union_members) -> [RecommendSetting, LegacySetting]; (all_union_members) ->
({value, #{<<"enable">> := _}}) -> [LegacySetting]; [RecommendSetting, LegacySetting];
%% all other cases treat as new config, include init empty config. ({value, Setting}) ->
({value, _}) -> [RecommendSetting] case is_recommend_type(Setting) of
true -> [RecommendSetting];
false -> [LegacySetting]
end
end.
%% For it to be considered as new schema,
%% all keys must be included in the new configuration.
is_recommend_type(Setting) ->
case maps:keys(Setting) of
[] ->
true;
Keys ->
NewKeys = fields(recommend_setting),
Fun = fun(Key0) ->
Key = binary_to_existing_atom(Key0),
lists:keymember(Key, 1, NewKeys)
end,
lists:all(Fun, Keys)
end. end.
desc(prometheus) -> ?DESC(prometheus); desc(prometheus) -> ?DESC(prometheus);
desc(collectors) -> ?DESC(collectors);
desc(legacy_deprecated_setting) -> ?DESC(legacy_deprecated_setting);
desc(recommend_setting) -> ?DESC(recommend_setting);
desc(push_gateway) -> ?DESC(push_gateway);
desc(_) -> undefined. desc(_) -> undefined.
convert_headers(undefined, _) -> convert_headers(undefined, _) ->

View File

@ -20,6 +20,7 @@
-export([ -export([
start_link/0, start_link/0,
start_child/1,
start_child/2, start_child/2,
update_child/2, update_child/2,
stop_child/1 stop_child/1
@ -40,6 +41,10 @@
start_link() -> start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []). supervisor:start_link({local, ?MODULE}, ?MODULE, []).
-spec start_child(atom()) -> ok.
start_child(Mod) when is_atom(Mod) ->
start_child(Mod, emqx_prometheus_config:conf()).
-spec start_child(atom(), map()) -> ok. -spec start_child(atom(), map()) -> ok.
start_child(Mod, Conf) when is_atom(Mod) -> start_child(Mod, Conf) when is_atom(Mod) ->
assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Conf))). assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Conf))).

View File

@ -0,0 +1,41 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_prometheus_proto_v1).
-behaviour(emqx_bpapi).
-export([
introduced_in/0,
deprecated_since/0,
start/1,
stop/1
]).
-include_lib("emqx/include/bpapi.hrl").
deprecated_since() -> "5.0.10".
introduced_in() ->
"5.0.0".
-spec start([node()]) -> emqx_rpc:multicall_result().
start(Nodes) ->
rpc:multicall(Nodes, emqx_prometheus, do_start, [], 5000).
-spec stop([node()]) -> emqx_rpc:multicall_result().
stop(Nodes) ->
rpc:multicall(Nodes, emqx_prometheus, do_stop, [], 5000).

View File

@ -219,5 +219,5 @@ init(Req0, Opts) ->
Headers Headers
), ),
RespHeader = #{<<"content-type">> => <<"text/plain; charset=utf-8">>}, RespHeader = #{<<"content-type">> => <<"text/plain; charset=utf-8">>},
Req = -cowboy_req:reply(200, RespHeader, <<"OK">>, Req0), Req = cowboy_req:reply(200, RespHeader, <<"OK">>, Req0),
{ok, Req, Opts}. {ok, Req, Opts}.

View File

@ -0,0 +1,4 @@
Modified the Prometheus API and configuration to:
- Restructure configuration sections to group related settings, improving readability and maintainability
- Introduced `enable_basic_auth` configuration for basic authentication on the scrape API endpoint, enhancing security
- Maintained backwards compatibility while refactoring code, avoiding breaking changes

View File

@ -17,7 +17,7 @@ Default value is: <code>${name}/instance/${name}~${host}</code>"""
prometheus.desc: prometheus.desc:
"""EMQX's Prometheus scraping endpoint is enabled by default without authentication. """EMQX's Prometheus scraping endpoint is enabled by default without authentication.
You can inspect it with a `curl` command like this: `curl -f "127.0.0.1:18083/api/v5/prometheus/stats"`<br/>""" You can inspect it with a `curl` command like this: `curl -f "127.0.0.1:18083/api/v5/prometheus/stats"`"""
prometheus.label: prometheus.label:
"""Prometheus""" """Prometheus"""
@ -25,11 +25,17 @@ prometheus.label:
push_gateway.desc: push_gateway.desc:
"""Push Gateway is optional, should not be configured if prometheus is to scrape EMQX.""" """Push Gateway is optional, should not be configured if prometheus is to scrape EMQX."""
enable_basic_auth.desc:
"""Enable or disable basic authentication for prometheus scrape api, not for Push Gateway"""
collectors.desc: collectors.desc:
"""The internal advanced metrics of the virtual machine are initially disabled """The internal advanced metrics of the virtual machine are initially disabled
and are usually only enabled during performance testing. and are usually only enabled during performance testing.
Enabling them will increase the CPU load.""" Enabling them will increase the CPU load."""
recommend_setting.desc:
"""Recommended setting"""
push_gateway_url.desc: push_gateway_url.desc:
"""URL of Pushgateway server. Pushgateway is optional, should not be configured if prometheus is to scrape EMQX. """URL of Pushgateway server. Pushgateway is optional, should not be configured if prometheus is to scrape EMQX.
Set url to "" to disable push gateway""" Set url to "" to disable push gateway"""
@ -54,37 +60,43 @@ vm_statistics_collector.desc:
vm_system_info_collector.desc: vm_system_info_collector.desc:
"""Enable or disable VM system info collector.""" """Enable or disable VM system info collector."""
legacy_deprecated_setting.desc:
"""Deprecated"""
legacy_enable.desc: legacy_enable.desc:
"""Deprecated, use prometheus.push_gateway.url instead""" """Deprecated, use `prometheus.push_gateway.url` instead"""
legacy_headers.desc: legacy_headers.desc:
"""Deprecated, use prometheus.push_gateway.headers instead""" """Deprecated, use `prometheus.push_gateway.headers` instead"""
legacy_interval.desc: legacy_interval.desc:
"""Deprecated, use prometheus.push_gateway.interval instead""" """Deprecated, use `prometheus.push_gateway.interval` instead"""
legacy_job_name.desc: legacy_job_name.desc:
"""Deprecated, use prometheus.push_gateway.job_name instead""" """Deprecated, use `prometheus.push_gateway.job_name` instead"""
legacy_push_gateway_server.desc: legacy_push_gateway_server.desc:
"""Deprecated, use prometheus.push_gateway.url instead""" """Deprecated, use `prometheus.push_gateway.url` instead"""
legacy_mnesia_collector.desc: legacy_mnesia_collector.desc:
"""Deprecated, use prometheus.collectors.mnesia instead""" """Deprecated, use `prometheus.collectors.mnesia` instead"""
legacy_vm_dist_collector.desc: legacy_vm_dist_collector.desc:
"""Deprecated, use prometheus.collectors.vm_dist instead""" """Deprecated, use `prometheus.collectors.vm_dist` instead"""
legacy_vm_memory_collector.desc: legacy_vm_memory_collector.desc:
"""Deprecated, use prometheus.collectors.vm_memory instead""" """Deprecated, use `prometheus.collectors.vm_memory` instead"""
legacy_vm_msacc_collector.desc: legacy_vm_msacc_collector.desc:
"""Deprecated, use prometheus.collectors.vm_msacc instead""" """Deprecated, use `prometheus.collectors.vm_msacc` instead"""
legacy_vm_statistics_collector.desc: legacy_vm_statistics_collector.desc:
"""Deprecated, use prometheus.collectors.vm_statistics instead""" """Deprecated, use `prometheus.collectors.vm_statistics` instead"""
legacy_vm_system_info_collector.desc: legacy_vm_system_info_collector.desc:
"""Deprecated, use prometheus.collectors.vm_system_info instead""" """Deprecated, use `prometheus.collectors.vm_system_info` instead"""
legacy_deprecated_setting.desc:
"""Deprecated"""
} }

View File

@ -295,3 +295,4 @@ dnstream
upstream upstream
priv priv
Syskeeper Syskeeper
msacc