From 03843c6071adfa6cc6af52d7cc8aef9c556658dc Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 13 Nov 2023 17:17:01 +0700 Subject: [PATCH 01/20] feat(router): switch to v2 routing store by default --- apps/emqx/src/emqx_schema.erl | 2 +- rel/i18n/emqx_schema.hocon | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 3848e77b4..1eadd8c61 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1388,7 +1388,7 @@ fields("broker_routing") -> sc( hoconsc:enum([v1, v2]), #{ - default => v1, + default => v2, 'readOnly' => true, desc => ?DESC(broker_routing_storage_schema) } diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index e1d086197..3959c8474 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1530,7 +1530,7 @@ sys_event_messages.desc: broker_routing_storage_schema.desc: """Routing storage schema. -Set v1 to leave the default. +Set v1 to use the former schema. v2 is introduced in 5.2. It enables routing through 2 separate tables, one for topic filter and one for regular topic subscriptions. This schema should increase both subscription and routing performance at the cost of slight increase in memory consumption per subscription. NOTE: Schema v2 is still experimental. NOTE: Full non-rolling cluster restart is needed after altering this option for it to take any effect.""" From dc3e818e84e441d53a2dd34f13832b0307880632 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 13 Nov 2023 17:44:24 +0700 Subject: [PATCH 02/20] chore: add changelog entry Co-Authored-By: Zaiming (Stone) Shi --- changes/ce/feat-11935.en.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/ce/feat-11935.en.md diff --git a/changes/ce/feat-11935.en.md b/changes/ce/feat-11935.en.md new file mode 100644 index 000000000..3cc3a45d5 --- /dev/null +++ b/changes/ce/feat-11935.en.md @@ -0,0 +1,5 @@ +Switch to the new `v2` routing store schema by default. New schema improves both subscription and routing performance, especially so for scenarios with concurrent subscriptions to topic filters sharing common wildcard prefixes, at the cost of slightly increased memory usage. This schema also eliminates the need for a separate index, thus inconsistencies in the routing state rarely encountered in previous versions should no longer be possible. + +If a cluster is rolling upgraded from older version, the cluster will continue to use `v1` store until a full cluster (non-rolling) restart happens. + +The former schema can still be forced by setting `broker.routing.storage_schema` configuration option to `v1` and conducting full non-rolling cluster restart as well. From 6812ee9d0f0732ca96fbbc6c48db167f01ad2d4d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 14 Nov 2023 15:54:18 +0700 Subject: [PATCH 03/20] fix(mgmt): hide route selection behind router interface Also introduce a generic _stream_ concept, mostly to deal with iterating over 2 ETS tables at once with `ets:match_object/3`. --- apps/emqx/src/emqx_router.erl | 82 ++++++++++- apps/emqx/src/emqx_router_helper.erl | 7 +- apps/emqx/src/emqx_trie_search.erl | 8 +- apps/emqx_management/src/emqx_mgmt_api.erl | 9 ++ .../src/emqx_mgmt_api_topics.erl | 104 ++++++++------ apps/emqx_utils/src/emqx_utils_stream.erl | 131 ++++++++++++++++++ .../test/emqx_utils_stream_tests.erl | 75 ++++++++++ 7 files changed, 369 insertions(+), 47 deletions(-) create mode 100644 apps/emqx_utils/src/emqx_utils_stream.erl create mode 100644 apps/emqx_utils/test/emqx_utils_stream_tests.erl diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index fd988eda1..892a4e5ba 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -52,6 +52,9 @@ lookup_routes/1 ]). +%% Topics API +-export([select/3]). + -export([print_routes/1]). -export([ @@ -59,7 +62,10 @@ foldr_routes/2 ]). --export([topics/0]). +-export([ + topics/0, + stats/1 +]). %% Exported for tests -export([has_route/2]). @@ -219,6 +225,19 @@ mria_delete_route(v2, Topic, Dest) -> mria_delete_route(v1, Topic, Dest) -> mria_delete_route_v1(Topic, Dest). +-spec select(Spec, _Limit :: pos_integer(), Continuation) -> + {[emqx_types:route()], Continuation} | '$end_of_table' +when + Spec :: {_TopicPat, _DestPat}, + Continuation :: term() | '$end_of_table'. +select(MatchSpec, Limit, Cont) -> + select(get_schema_vsn(), MatchSpec, Limit, Cont). + +select(v2, MatchSpec, Limit, Cont) -> + select_v2(MatchSpec, Limit, Cont); +select(v1, MatchSpec, Limit, Cont) -> + select_v1(MatchSpec, Limit, Cont). + -spec topics() -> list(emqx_types:topic()). topics() -> topics(get_schema_vsn()). @@ -228,6 +247,15 @@ topics(v2) -> topics(v1) -> list_topics_v1(). +-spec stats(n_routes) -> non_neg_integer(). +stats(Item) -> + stats(get_schema_vsn(), Item). + +stats(v2, Item) -> + get_stats_v2(Item); +stats(v1, Item) -> + get_stats_v1(Item). + %% @doc Print routes to a topic -spec print_routes(emqx_types:topic()) -> ok. print_routes(Topic) -> @@ -345,9 +373,17 @@ cleanup_routes_v1(Node) -> ] end). +select_v1({MTopic, MDest}, Limit, undefined) -> + ets:match_object(?ROUTE_TAB, #route{topic = MTopic, dest = MDest}, Limit); +select_v1(_Spec, _Limit, Cont) -> + ets:select(Cont). + list_topics_v1() -> list_route_tab_topics(). +get_stats_v1(n_routes) -> + emqx_maybe:define(ets:info(?ROUTE_TAB, size), 0). + list_route_tab_topics() -> mnesia:dirty_all_keys(?ROUTE_TAB). @@ -436,11 +472,52 @@ get_dest_node({_, Node}) -> get_dest_node(Node) -> Node. +select_v2(Spec, Limit, undefined) -> + Stream = mk_route_stream(Spec), + select_next(Limit, Stream); +select_v2(_Spec, Limit, Stream) -> + select_next(Limit, Stream). + +select_next(N, Stream) -> + case emqx_utils_stream:take(N, Stream) of + {Routes, SRest} -> + {Routes, SRest}; + Routes -> + {Routes, '$end_of_table'} + end. + +mk_route_stream(Spec) -> + emqx_utils_stream:chain( + mk_route_stream(route, Spec), + mk_route_stream(filter, Spec) + ). + +mk_route_stream(route, Spec) -> + emqx_utils_stream:ets(fun(Cont) -> select_v1(Spec, 1, Cont) end); +mk_route_stream(filter, {MTopic, MDest}) -> + emqx_utils_stream:map( + fun routeidx_to_route/1, + emqx_utils_stream:ets( + fun + (undefined) -> + MatchSpec = #routeidx{entry = emqx_trie_search:make_pat(MTopic, MDest)}, + ets:match_object(?ROUTE_TAB_FILTERS, MatchSpec, 1); + (Cont) -> + ets:match_object(Cont) + end + ) + ). + list_topics_v2() -> Pat = #routeidx{entry = '$1'}, Filters = [emqx_topic_index:get_topic(K) || [K] <- ets:match(?ROUTE_TAB_FILTERS, Pat)], list_route_tab_topics() ++ Filters. +get_stats_v2(n_routes) -> + NTopics = emqx_maybe:define(ets:info(?ROUTE_TAB, size), 0), + NWildcards = emqx_maybe:define(ets:info(?ROUTE_TAB_FILTERS, size), 0), + NTopics + NWildcards. + fold_routes_v2(FunName, FoldFun, AccIn) -> FilterFoldFun = mk_filtertab_fold_fun(FoldFun), Acc = ets:FunName(FoldFun, AccIn, ?ROUTE_TAB), @@ -449,6 +526,9 @@ fold_routes_v2(FunName, FoldFun, AccIn) -> mk_filtertab_fold_fun(FoldFun) -> fun(#routeidx{entry = K}, Acc) -> FoldFun(match_to_route(K), Acc) end. +routeidx_to_route(#routeidx{entry = M}) -> + match_to_route(M). + match_to_route(M) -> #route{topic = emqx_topic_index:get_topic(M), dest = emqx_topic_index:get_id(M)}. diff --git a/apps/emqx/src/emqx_router_helper.erl b/apps/emqx/src/emqx_router_helper.erl index b9cdbae4b..c43192d4e 100644 --- a/apps/emqx/src/emqx_router_helper.erl +++ b/apps/emqx/src/emqx_router_helper.erl @@ -190,12 +190,7 @@ code_change(_OldVsn, State, _Extra) -> %%-------------------------------------------------------------------- stats_fun() -> - case ets:info(?ROUTE_TAB, size) of - undefined -> - ok; - Size -> - emqx_stats:setstat('topics.count', 'topics.max', Size) - end. + emqx_stats:setstat('topics.count', 'topics.max', emqx_router:stats(n_routes)). cleanup_routes(Node) -> emqx_router:cleanup_routes(Node). diff --git a/apps/emqx/src/emqx_trie_search.erl b/apps/emqx/src/emqx_trie_search.erl index c8c088b58..da37f2b21 100644 --- a/apps/emqx/src/emqx_trie_search.erl +++ b/apps/emqx/src/emqx_trie_search.erl @@ -98,7 +98,7 @@ -module(emqx_trie_search). --export([make_key/2, filter/1]). +-export([make_key/2, make_pat/2, filter/1]). -export([match/2, matches/3, get_id/1, get_topic/1]). -export_type([key/1, word/0, words/0, nextf/0, opts/0]). @@ -127,6 +127,12 @@ make_key(Topic, ID) when is_binary(Topic) -> make_key(Words, ID) when is_list(Words) -> {Words, {ID}}. +-spec make_pat(emqx_types:topic() | words() | '_', _ID | '_') -> _Pat. +make_pat(Pattern = '_', ID) -> + {Pattern, {ID}}; +make_pat(Topic, ID) -> + make_key(Topic, ID). + %% @doc Parse a topic filter into a list of words. Returns `false` if it's not a filter. -spec filter(emqx_types:topic()) -> words() | false. filter(Topic) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index dffa39ae5..b14d1d316 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -35,6 +35,13 @@ b2i/1 ]). +-export([ + parse_pager_params/1, + parse_qstring/2, + init_query_result/0, + accumulate_query_rows/4 +]). + -ifdef(TEST). -export([paginate_test_format/1]). -endif. @@ -444,6 +451,8 @@ accumulate_query_rows( count => Count + length(SubRows), rows => [{Node, SubRows} | RowsAcc] }}; + NCursor when NCursor >= PageEnd + Limit -> + {enough, ResultAcc#{cursor => NCursor}}; NCursor when NCursor >= PageEnd -> SubRows = lists:sublist(Rows, Limit - Count), {enough, ResultAcc#{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_topics.erl b/apps/emqx_management/src/emqx_mgmt_api_topics.erl index 31c70573f..c1d5f8e74 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_topics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_topics.erl @@ -18,7 +18,6 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx/include/emqx_router.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -37,8 +36,6 @@ topic/2 ]). --export([qs2ms/2, format/1]). - -define(TOPIC_NOT_FOUND, 'TOPIC_NOT_FOUND'). -define(TOPICS_QUERY_SCHEMA, [{<<"topic">>, binary}, {<<"node">>, atom}]). @@ -110,23 +107,15 @@ topic(get, #{bindings := Bindings}) -> %%%============================================================================================== %% api apply do_list(Params) -> - case - emqx_mgmt_api:node_query( - node(), - ?ROUTE_TAB, - Params, - ?TOPICS_QUERY_SCHEMA, - fun ?MODULE:qs2ms/2, - fun ?MODULE:format/1 - ) - of - {error, page_limit_invalid} -> - {400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}}; - {error, Node, Error} -> - Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error])), - {500, #{code => <<"NODE_DOWN">>, message => Message}}; - Response -> - {200, Response} + try + Pager = parse_pager_params(Params), + {_, Query} = emqx_mgmt_api:parse_qstring(Params, ?TOPICS_QUERY_SCHEMA), + QState = Pager#{continuation => undefined}, + QResult = eval_topic_query(qs2ms(Query), QState), + {200, format_list_response(Pager, QResult)} + catch + throw:{error, page_limit_invalid} -> + {400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}} end. lookup(#{topic := Topic}) -> @@ -140,26 +129,63 @@ lookup(#{topic := Topic}) -> %%%============================================================================================== %% internal --spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter(). -qs2ms(_Tab, {Qs, _}) -> - #{ - match_spec => gen_match_spec(Qs, [{{route, '_', '_'}, [], ['$_']}]), - fuzzy_fun => undefined - }. -gen_match_spec([], Res) -> - Res; -gen_match_spec([{topic, '=:=', T0} | Qs], [{{route, _, Node}, [], ['$_']}]) when is_atom(Node) -> - {T, D} = - case emqx_topic:parse(T0) of - {#share{group = Group, topic = Topic}, _SubOpts} -> - {Topic, {Group, Node}}; - {T1, _SubOpts} -> - {T1, Node} - end, - gen_match_spec(Qs, [{{route, T, D}, [], ['$_']}]); -gen_match_spec([{node, '=:=', N} | Qs], [{{route, T, _}, [], ['$_']}]) -> - gen_match_spec(Qs, [{{route, T, N}, [], ['$_']}]). +parse_pager_params(Params) -> + try emqx_mgmt_api:parse_pager_params(Params) of + Pager = #{} -> + Pager; + false -> + throw({error, page_limit_invalid}) + catch + error:badarg -> + throw({error, page_limit_invalid}) + end. + +-spec qs2ms({list(), list()}) -> tuple(). +qs2ms({Qs, _}) -> + lists:foldl(fun gen_match_spec/2, {'_', '_'}, Qs). + +gen_match_spec({topic, '=:=', QTopic}, {_MTopic, MNode}) when is_atom(MNode) -> + case emqx_topic:parse(QTopic) of + {#share{group = Group, topic = Topic}, _SubOpts} -> + {Topic, {Group, MNode}}; + {Topic, _SubOpts} -> + {Topic, MNode} + end; +gen_match_spec({node, '=:=', QNode}, {MTopic, _MDest}) -> + {MTopic, QNode}. + +eval_topic_query(MS, QState) -> + finalize_query(eval_topic_query(MS, QState, emqx_mgmt_api:init_query_result())). + +eval_topic_query(MS, QState, QResult) -> + QPage = eval_topic_query_page(MS, QState), + case QPage of + {Rows, '$end_of_table'} -> + {_, NQResult} = emqx_mgmt_api:accumulate_query_rows(node(), Rows, QState, QResult), + NQResult#{complete => true}; + {Rows, NCont} -> + {_, NQResult} = emqx_mgmt_api:accumulate_query_rows(node(), Rows, QState, QResult), + eval_topic_query(MS, QState#{continuation := NCont}, NQResult); + '$end_of_table' -> + QResult#{complete => true} + end. + +eval_topic_query_page(MS, #{limit := Limit, continuation := Cont}) -> + emqx_router:select(MS, Limit, Cont). + +finalize_query(QResult = #{overflow := Overflow, complete := Complete}) -> + HasNext = Overflow orelse not Complete, + QResult#{hasnext => HasNext}. + +format_list_response(Meta, _QResult = #{hasnext := HasNext, rows := RowsAcc, cursor := Cursor}) -> + #{ + meta => Meta#{hasnext => HasNext, count => Cursor}, + data => lists:flatmap( + fun({_Node, Rows}) -> [format(R) || R <- Rows] end, + RowsAcc + ) + }. format(#route{topic = Topic, dest = {Group, Node}}) -> #{topic => ?SHARE(Group, Topic), node => Node}; diff --git a/apps/emqx_utils/src/emqx_utils_stream.erl b/apps/emqx_utils/src/emqx_utils_stream.erl new file mode 100644 index 000000000..e7374c861 --- /dev/null +++ b/apps/emqx_utils/src/emqx_utils_stream.erl @@ -0,0 +1,131 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_utils_stream). + +%% Constructors / Combinators +-export([ + empty/0, + list/1, + map/2, + chain/2 +]). + +%% Evaluating +-export([ + next/1, + take/2, + consume/1 +]). + +%% Streams from ETS tables +-export([ + ets/1 +]). + +-export_type([stream/1]). + +%% @doc A stream is essentially a lazy list. +-type stream(T) :: fun(() -> next(T) | []). +-type next(T) :: nonempty_improper_list(T, stream(T)). + +-dialyzer(no_improper_lists). + +%% + +-spec empty() -> stream(none()). +empty() -> + fun() -> [] end. + +-spec list([T]) -> stream(T). +list([]) -> + empty(); +list([X | Rest]) -> + fun() -> [X | list(Rest)] end. + +-spec map(fun((X) -> Y), stream(X)) -> stream(Y). +map(F, S) -> + fun() -> + case next(S) of + [X | Rest] -> + [F(X) | map(F, Rest)]; + [] -> + [] + end + end. + +-spec chain(stream(X), stream(Y)) -> stream(X | Y). +chain(SFirst, SThen) -> + fun() -> + case next(SFirst) of + [X | SRest] -> + [X | chain(SRest, SThen)]; + [] -> + next(SThen) + end + end. + +%% + +-spec next(stream(T)) -> next(T) | []. +next(S) -> + S(). + +-spec take(non_neg_integer(), stream(T)) -> {[T], stream(T)} | [T]. +take(N, S) -> + take(N, S, []). + +take(0, S, Acc) -> + {lists:reverse(Acc), S}; +take(N, S, Acc) -> + case next(S) of + [X | SRest] -> + take(N - 1, SRest, [X | Acc]); + [] -> + lists:reverse(Acc) + end. + +-spec consume(stream(T)) -> [T]. +consume(S) -> + case next(S) of + [X | SRest] -> + [X | consume(SRest)]; + [] -> + [] + end. + +%% + +-type select_result(Record, Cont) :: + {[Record], Cont} + | {[Record], '$end_of_table'} + | '$end_of_table'. + +-spec ets(fun((Cont) -> select_result(Record, Cont))) -> stream(Record). +ets(ContF) -> + ets(undefined, ContF). + +ets(Cont, ContF) -> + fun() -> + case ContF(Cont) of + {Records, '$end_of_table'} -> + next(list(Records)); + {Records, NCont} -> + next(chain(list(Records), ets(NCont, ContF))); + '$end_of_table' -> + [] + end + end. diff --git a/apps/emqx_utils/test/emqx_utils_stream_tests.erl b/apps/emqx_utils/test/emqx_utils_stream_tests.erl new file mode 100644 index 000000000..0f98bae21 --- /dev/null +++ b/apps/emqx_utils/test/emqx_utils_stream_tests.erl @@ -0,0 +1,75 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_utils_stream_tests). + +-include_lib("eunit/include/eunit.hrl"). + +empty_test() -> + S = emqx_utils_stream:empty(), + ?assertEqual([], emqx_utils_stream:next(S)). + +empty_consume_test() -> + S = emqx_utils_stream:empty(), + ?assertEqual([], emqx_utils_stream:consume(S)). + +chain_empties_test() -> + S = emqx_utils_stream:chain( + emqx_utils_stream:empty(), + emqx_utils_stream:empty() + ), + ?assertEqual([], emqx_utils_stream:next(S)). + +chain_list_test() -> + S = emqx_utils_stream:chain( + emqx_utils_stream:list([1, 2, 3]), + emqx_utils_stream:list([4, 5, 6]) + ), + ?assertEqual( + [1, 2, 3, 4, 5, 6], + emqx_utils_stream:consume(S) + ). + +chain_take_test() -> + S = emqx_utils_stream:chain( + emqx_utils_stream:list([1, 2, 3]), + emqx_utils_stream:list([4, 5, 6, 7, 8]) + ), + ?assertMatch( + {[1, 2, 3, 4, 5], _SRest}, + emqx_utils_stream:take(5, S) + ), + {_, SRest} = emqx_utils_stream:take(5, S), + ?assertEqual( + [6, 7, 8], + emqx_utils_stream:take(5, SRest) + ). + +chain_list_map_test() -> + S = emqx_utils_stream:map( + fun integer_to_list/1, + emqx_utils_stream:chain( + emqx_utils_stream:list([1, 2, 3]), + emqx_utils_stream:chain( + emqx_utils_stream:empty(), + emqx_utils_stream:list([4, 5, 6]) + ) + ) + ), + ?assertEqual( + ["1", "2", "3", "4", "5", "6"], + emqx_utils_stream:consume(S) + ). From 8919b08207a5f2b53931a08d634285442646f04a Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 15 Nov 2023 17:20:40 +0700 Subject: [PATCH 04/20] fix(utils): rename `emqx_utils_stream:take/2` to `consume/2` Which is more neutral and harder to confuse with a destructive `take` in collections. --- apps/emqx/src/emqx_router.erl | 2 +- apps/emqx_utils/src/emqx_utils_stream.erl | 47 ++++++++++++------- .../test/emqx_utils_stream_tests.erl | 6 +-- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index 892a4e5ba..1aebb1b21 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -479,7 +479,7 @@ select_v2(_Spec, Limit, Stream) -> select_next(Limit, Stream). select_next(N, Stream) -> - case emqx_utils_stream:take(N, Stream) of + case emqx_utils_stream:consume(N, Stream) of {Routes, SRest} -> {Routes, SRest}; Routes -> diff --git a/apps/emqx_utils/src/emqx_utils_stream.erl b/apps/emqx_utils/src/emqx_utils_stream.erl index e7374c861..79ce5ce7b 100644 --- a/apps/emqx_utils/src/emqx_utils_stream.erl +++ b/apps/emqx_utils/src/emqx_utils_stream.erl @@ -27,8 +27,8 @@ %% Evaluating -export([ next/1, - take/2, - consume/1 + consume/1, + consume/2 ]). %% Streams from ETS tables @@ -46,16 +46,20 @@ %% +%% @doc Make a stream that produces no values. -spec empty() -> stream(none()). empty() -> fun() -> [] end. +%% @doc Make a stream out of the given list. +%% Essentially it's an opposite of `consume/1`, i.e. `L = consume(list(L))`. -spec list([T]) -> stream(T). list([]) -> empty(); list([X | Rest]) -> fun() -> [X | list(Rest)] end. +%% @doc Make a stream by applying a function to each element of the underlying stream. -spec map(fun((X) -> Y), stream(X)) -> stream(Y). map(F, S) -> fun() -> @@ -67,6 +71,8 @@ map(F, S) -> end end. +%% @doc Make a stream by chaining (concatenating) two streams. +%% The second stream begins to produce values only after the first one is exhausted. -spec chain(stream(X), stream(Y)) -> stream(X | Y). chain(SFirst, SThen) -> fun() -> @@ -80,24 +86,12 @@ chain(SFirst, SThen) -> %% +%% @doc Produce the next value from the stream. -spec next(stream(T)) -> next(T) | []. next(S) -> S(). --spec take(non_neg_integer(), stream(T)) -> {[T], stream(T)} | [T]. -take(N, S) -> - take(N, S, []). - -take(0, S, Acc) -> - {lists:reverse(Acc), S}; -take(N, S, Acc) -> - case next(S) of - [X | SRest] -> - take(N - 1, SRest, [X | Acc]); - [] -> - lists:reverse(Acc) - end. - +%% @doc Consume the stream and return a list of all produced values. -spec consume(stream(T)) -> [T]. consume(S) -> case next(S) of @@ -107,6 +101,22 @@ consume(S) -> [] end. +%% @doc Consume N values from the stream and return a list of them and the rest of the stream. +%% If the stream is exhausted before N values are produced, return just a list of these values. +-spec consume(non_neg_integer(), stream(T)) -> {[T], stream(T)} | [T]. +consume(N, S) -> + consume(N, S, []). + +consume(0, S, Acc) -> + {lists:reverse(Acc), S}; +consume(N, S, Acc) -> + case next(S) of + [X | SRest] -> + consume(N - 1, SRest, [X | Acc]); + [] -> + lists:reverse(Acc) + end. + %% -type select_result(Record, Cont) :: @@ -114,6 +124,11 @@ consume(S) -> | {[Record], '$end_of_table'} | '$end_of_table'. +%% @doc Make a stream out of an ETS table, where the ETS table is scanned through in chunks, +%% with the given continuation function. The function is assumed to return a result of a call to: +%% * `ets:select/1` / `ets:select/3` +%% * `ets:match/1` / `ets:match/3` +%% * `ets:match_object/1` / `ets:match_object/3` -spec ets(fun((Cont) -> select_result(Record, Cont))) -> stream(Record). ets(ContF) -> ets(undefined, ContF). diff --git a/apps/emqx_utils/test/emqx_utils_stream_tests.erl b/apps/emqx_utils/test/emqx_utils_stream_tests.erl index 0f98bae21..4a48ae45d 100644 --- a/apps/emqx_utils/test/emqx_utils_stream_tests.erl +++ b/apps/emqx_utils/test/emqx_utils_stream_tests.erl @@ -50,12 +50,12 @@ chain_take_test() -> ), ?assertMatch( {[1, 2, 3, 4, 5], _SRest}, - emqx_utils_stream:take(5, S) + emqx_utils_stream:consume(5, S) ), - {_, SRest} = emqx_utils_stream:take(5, S), + {_, SRest} = emqx_utils_stream:consume(5, S), ?assertEqual( [6, 7, 8], - emqx_utils_stream:take(5, SRest) + emqx_utils_stream:consume(5, SRest) ). chain_list_map_test() -> From 893e90b3724107af91d98f5a3aa917fac11652e4 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 15 Nov 2023 22:37:00 +0700 Subject: [PATCH 05/20] fix(kafka): use safe publish in consumer Routing with v2 schema is actually more strict with respect to input to `emqx_router` module routines. This causes Kafka consumer bridge to crash when it tries to publish a message to a topic that looks like a topic filter. --- apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl index 89cb9a78f..24ea4d300 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl @@ -251,7 +251,7 @@ do_handle_message(Message, State) -> Payload = render(FullMessage, PayloadTemplate), MQTTTopic = render(FullMessage, MQTTTopicTemplate), MQTTMessage = emqx_message:make(ResourceId, MQTTQoS, MQTTTopic, Payload), - _ = emqx:publish(MQTTMessage), + _ = emqx_broker:safe_publish(MQTTMessage), emqx_hooks:run(Hookpoint, [FullMessage]), emqx_resource_metrics:received_inc(ResourceId), %% note: just `ack' does not commit the offset to the From 656d575e2de333d7f13c0cede459c328bbcdf832 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 15 Nov 2023 22:28:52 +0700 Subject: [PATCH 06/20] ci(router): fix flaky testcase --- apps/emqx/test/emqx_routing_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_routing_SUITE.erl b/apps/emqx/test/emqx_routing_SUITE.erl index 6966ac56a..a54e1b4dd 100644 --- a/apps/emqx/test/emqx_routing_SUITE.erl +++ b/apps/emqx/test/emqx_routing_SUITE.erl @@ -100,7 +100,7 @@ mk_config_listeners(N) -> t_cluster_routing(Config) -> Cluster = ?config(cluster, Config), - Clients = [C1, C2, C3] = [start_client(N) || N <- Cluster], + Clients = [C1, C2, C3] = lists:sort([start_client(N) || N <- Cluster]), Commands = [ {fun publish/3, [C1, <<"a/b/c">>, <<"wontsee">>]}, {fun publish/3, [C2, <<"a/b/d">>, <<"wontsee">>]}, From c376a5db2910f9a2c10413198d6ed0344c880834 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sun, 5 Nov 2023 00:42:30 +0800 Subject: [PATCH 07/20] refactor: prometheus config --- apps/emqx_conf/src/emqx_conf_schema.erl | 63 +++--- apps/emqx_prometheus/src/emqx_prometheus.erl | 90 ++++---- .../src/emqx_prometheus_api.erl | 78 +++++-- .../src/emqx_prometheus_config.erl | 22 +- .../src/emqx_prometheus_schema.erl | 209 ++++++++++++++++-- .../src/emqx_prometheus_sup.erl | 20 +- .../src/proto/emqx_prometheus_proto_v1.erl | 41 ---- .../test/emqx_prometheus_SUITE.erl | 144 ++++++++++-- .../test/emqx_prometheus_api_SUITE.erl | 162 ++++++++++++-- rel/i18n/emqx_prometheus_api.hocon | 4 +- rel/i18n/emqx_prometheus_schema.hocon | 56 ++++- 11 files changed, 667 insertions(+), 222 deletions(-) delete mode 100644 apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 3a2b5d972..45219c3f6 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1190,37 +1190,44 @@ tr_prometheus_collectors(Conf) -> emqx_prometheus, emqx_prometheus_mria %% builtin vm collectors - | tr_vm_dist_collector(Conf) ++ - tr_mnesia_collector(Conf) ++ - tr_vm_statistics_collector(Conf) ++ - tr_vm_system_info_collector(Conf) ++ - tr_vm_memory_collector(Conf) ++ - tr_vm_msacc_collector(Conf) + | prometheus_collectors(Conf) ]. -tr_vm_dist_collector(Conf) -> - Enabled = conf_get("prometheus.vm_dist_collector", Conf, disabled), - collector_enabled(Enabled, prometheus_vm_dist_collector). +prometheus_collectors(Conf) -> + case conf_get("prometheus.enable_basic_auth", Conf, undefined) of + %% legacy + undefined -> + tr_collector("prometheus.vm_dist_collector", prometheus_vm_dist_collector, Conf) ++ + tr_collector("prometheus.mnesia_collector", prometheus_mnesia_collector, Conf) ++ + tr_collector( + "prometheus.vm_statistics_collector", prometheus_vm_statistics_collector, Conf + ) ++ + tr_collector( + "prometheus.vm_system_info_collector", prometheus_vm_system_info_collector, Conf + ) ++ + tr_collector("prometheus.vm_memory_collector", prometheus_vm_memory_collector, Conf) ++ + tr_collector("prometheus.vm_msacc_collector", prometheus_vm_msacc_collector, Conf); + %% new + _ -> + tr_collector("prometheus.collectors.vm_dist", prometheus_vm_dist_collector, Conf) ++ + tr_collector("prometheus.collectors.mnesia", prometheus_mnesia_collector, Conf) ++ + tr_collector( + "prometheus.collectors.vm_statistics", prometheus_vm_statistics_collector, Conf + ) ++ + tr_collector( + "prometheus.collectors.vm_system_info", + prometheus_vm_system_info_collector, + Conf + ) ++ + tr_collector( + "prometheus.collectors.vm_memory", prometheus_vm_memory_collector, Conf + ) ++ + tr_collector("prometheus.collectors.vm_msacc", prometheus_vm_msacc_collector, Conf) + end. -tr_mnesia_collector(Conf) -> - Enabled = conf_get("prometheus.mnesia_collector", Conf, disabled), - collector_enabled(Enabled, prometheus_mnesia_collector). - -tr_vm_statistics_collector(Conf) -> - Enabled = conf_get("prometheus.vm_statistics_collector", Conf, disabled), - collector_enabled(Enabled, prometheus_vm_statistics_collector). - -tr_vm_system_info_collector(Conf) -> - Enabled = conf_get("prometheus.vm_system_info_collector", Conf, disabled), - collector_enabled(Enabled, prometheus_vm_system_info_collector). - -tr_vm_memory_collector(Conf) -> - Enabled = conf_get("prometheus.vm_memory_collector", Conf, disabled), - collector_enabled(Enabled, prometheus_vm_memory_collector). - -tr_vm_msacc_collector(Conf) -> - Enabled = conf_get("prometheus.vm_msacc_collector", Conf, disabled), - collector_enabled(Enabled, prometheus_vm_msacc_collector). +tr_collector(Key, Collect, Conf) -> + Enabled = conf_get(Key, Conf, disabled), + collector_enabled(Enabled, Collect). collector_enabled(enabled, Collector) -> [Collector]; collector_enabled(disabled, _) -> []. diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index a242931c4..29d531140 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -42,6 +42,7 @@ %% gen_server callbacks -export([ init/1, + handle_continue/2, handle_call/3, handle_cast/2, handle_info/2, @@ -58,12 +59,6 @@ -export([collect/1]). --export([ - %% For bpapi, deprecated_since 5.0.10, remove this when 5.1.x - do_start/0, - do_stop/0 -]). - -define(C(K, L), proplists:get_value(K, L, 0)). -define(TIMER_MSG, '#interval'). @@ -74,8 +69,8 @@ %% APIs %%-------------------------------------------------------------------- -start_link([]) -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). +start_link(Conf) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, Conf, []). info() -> gen_server:call(?MODULE, info). @@ -84,49 +79,42 @@ info() -> %% gen_server callbacks %%-------------------------------------------------------------------- -init([]) -> - #{interval := Interval} = opts(), - {ok, #{timer => ensure_timer(Interval), ok => 0, failed => 0}}. +init(Conf) -> + {ok, #{}, {continue, Conf}}. -handle_call(info, _From, State = #{timer := Timer}) -> - {reply, State#{opts => opts(), next_push_ms => erlang:read_timer(Timer)}, State}; +handle_continue(Conf, State) -> + Opts = #{interval := Interval} = opts(Conf), + {noreply, State#{ + timer => ensure_timer(Interval), + opts => Opts, + ok => 0, + failed => 0 + }}. + +handle_call(info, _From, State = #{timer := Timer, opts := Opts}) -> + {reply, State#{opts => Opts, next_push_ms => erlang:read_timer(Timer)}, State}; handle_call(_Msg, _From, State) -> {reply, ok, State}. handle_cast(_Msg, State) -> {noreply, State}. -handle_info({timeout, Timer, ?TIMER_MSG}, State = #{timer := Timer}) -> - #{ - interval := Interval, - headers := Headers, - job_name := JobName, - push_gateway_server := Server - } = opts(), - PushRes = push_to_push_gateway(Server, Headers, JobName), +handle_info({timeout, Timer, ?TIMER_MSG}, State = #{timer := Timer, opts := Opts}) -> + #{interval := Interval, headers := Headers, url := Server} = Opts, + PushRes = push_to_push_gateway(Server, Headers), NewTimer = ensure_timer(Interval), NewState = maps:update_with(PushRes, fun(C) -> C + 1 end, 1, State#{timer => NewTimer}), %% Data is too big, hibernate for saving memory and stop system monitor warning. {noreply, NewState, hibernate}; +handle_info({update, Conf}, State = #{timer := Timer}) -> + emqx_utils:cancel_timer(Timer), + handle_continue(Conf, State); handle_info(_Msg, State) -> {noreply, State}. -push_to_push_gateway(Uri, Headers, JobName) when is_list(Headers) -> - [Name, Ip] = string:tokens(atom_to_list(node()), "@"), - % NOTE: allowing errors here to keep rough backward compatibility - {JobName1, Errors} = emqx_template:render( - emqx_template:parse(JobName), - #{<<"name">> => Name, <<"host">> => Ip} - ), - _ = - Errors == [] orelse - ?SLOG(warning, #{ - msg => "prometheus_job_name_template_invalid", - errors => Errors, - template => JobName - }), + +push_to_push_gateway(Url, Headers) when is_list(Headers) -> Data = prometheus_text_format:format(), - Url = lists:concat([Uri, "/metrics/job/", unicode:characters_to_list(JobName1)]), case httpc:request(post, {Url, Headers, "text/plain", Data}, ?HTTP_OPTIONS, []) of {ok, {{"HTTP/1.1", 200, _}, _RespHeaders, _RespBody}} -> ok; @@ -152,8 +140,26 @@ ensure_timer(Interval) -> %%-------------------------------------------------------------------- %% prometheus callbacks %%-------------------------------------------------------------------- -opts() -> - emqx_conf:get(?PROMETHEUS). +opts(#{interval := Interval, headers := Headers, job_name := JobName, push_gateway_server := Url}) -> + #{interval => Interval, headers => Headers, url => join_url(Url, JobName)}; +opts(#{push_gateway := #{url := Url, job_name := JobName} = PushGateway}) -> + maps:put(url, join_url(Url, JobName), PushGateway). + +join_url(Url, JobName0) -> + [Name, Ip] = string:tokens(atom_to_list(node()), "@"), + % NOTE: allowing errors here to keep rough backward compatibility + {JobName1, Errors} = emqx_template:render( + emqx_template:parse(JobName0), + #{<<"name">> => Name, <<"host">> => Ip} + ), + _ = + Errors == [] orelse + ?SLOG(warning, #{ + msg => "prometheus_job_name_template_invalid", + errors => Errors, + template => JobName0 + }), + lists:concat([Url, "/metrics/job/", unicode:characters_to_list(JobName1)]). deregister_cleanup(_Registry) -> ok. @@ -672,11 +678,3 @@ emqx_cluster_data() -> {nodes_running, length(Running)}, {nodes_stopped, length(Stopped)} ]. - -%% deprecated_since 5.0.10, remove this when 5.1.x -do_start() -> - emqx_prometheus_sup:start_child(?APP). - -%% deprecated_since 5.0.10, remove this when 5.1.x -do_stop() -> - emqx_prometheus_sup:stop_child(?APP). diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index 987386b61..3072ab5a7 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -20,8 +20,6 @@ -include_lib("hocon/include/hoconsc.hrl"). --import(hoconsc, [ref/2]). - -export([ api_spec/0, paths/0, @@ -29,11 +27,10 @@ ]). -export([ - prometheus/2, + setting/2, stats/2 ]). --define(SCHEMA_MODULE, emqx_prometheus_schema). -define(TAGS, [<<"Monitor">>]). api_spec() -> @@ -47,21 +44,21 @@ paths() -> schema("/prometheus") -> #{ - 'operationId' => prometheus, + 'operationId' => setting, get => #{ description => ?DESC(get_prom_conf_info), tags => ?TAGS, responses => - #{200 => prometheus_config_schema()} + #{200 => prometheus_setting_schema()} }, put => #{ description => ?DESC(update_prom_conf_info), tags => ?TAGS, - 'requestBody' => prometheus_config_schema(), + 'requestBody' => prometheus_setting_schema(), responses => - #{200 => prometheus_config_schema()} + #{200 => prometheus_setting_schema()} } }; schema("/prometheus/stats") -> @@ -81,9 +78,9 @@ schema("/prometheus/stats") -> %% API Handler funcs %%-------------------------------------------------------------------- -prometheus(get, _Params) -> +setting(get, _Params) -> {200, emqx:get_raw_config([<<"prometheus">>], #{})}; -prometheus(put, #{body := Body}) -> +setting(put, #{body := Body}) -> case emqx_prometheus_config:update(Body) of {ok, NewConfig} -> {200, NewConfig}; @@ -110,20 +107,57 @@ stats(get, #{headers := Headers}) -> %% Internal funcs %%-------------------------------------------------------------------- -prometheus_config_schema() -> - emqx_dashboard_swagger:schema_with_example( - ref(?SCHEMA_MODULE, "prometheus"), - prometheus_config_example() +prometheus_setting_schema() -> + [{prometheus, #{type := Setting}}] = emqx_prometheus_schema:roots(), + emqx_dashboard_swagger:schema_with_examples( + Setting, + [ + recommend_setting_example(), + legacy_setting_example() + ] ). -prometheus_config_example() -> - #{ - enable => true, - interval => "15s", - push_gateway_server => <<"http://127.0.0.1:9091">>, - headers => #{'header-name' => 'header-value'}, - job_name => <<"${name}/instance/${name}~${host}">> - }. +legacy_setting_example() -> + Summary = <<"legacy_deprecated_setting">>, + {Summary, #{ + summary => Summary, + value => #{ + enable => true, + interval => <<"15s">>, + push_gateway_server => <<"http://127.0.0.1:9091">>, + headers => #{<<"Authorization">> => <<"Basic YWRtaW46Y2JraG55eWd5QDE=">>}, + job_name => <<"${name}/instance/${name}~${host}">>, + vm_dist_collector => <<"disabled">>, + vm_memory_collector => <<"disabled">>, + vm_msacc_collector => <<"disabled">>, + mnesia_collector => <<"disabled">>, + vm_statistics_collector => <<"disabled">>, + vm_system_info_collector => <<"disabled">> + } + }}. + +recommend_setting_example() -> + Summary = <<"recommend_setting">>, + {Summary, #{ + summary => Summary, + value => #{ + enable_basic_auth => false, + push_gateway => #{ + interval => <<"15s">>, + url => <<"http://127.0.0.1:9091">>, + headers => #{<<"Authorization">> => <<"Basic YWRtaW46Y2JraG55eWd5QDE=">>}, + job_name => <<"${name}/instance/${name}~${host}">> + }, + collectors => #{ + vm_dist => <<"disabled">>, + vm_memory => <<"disabled">>, + vm_msacc => <<"disabled">>, + mnesia => <<"disabled">>, + vm_statistics => <<"disabled">>, + vm_system_info => <<"disabled">> + } + } + }}. prometheus_data_schema() -> #{ diff --git a/apps/emqx_prometheus/src/emqx_prometheus_config.erl b/apps/emqx_prometheus/src/emqx_prometheus_config.erl index 00dad47f9..7cc5c7f8b 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_config.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_config.erl @@ -22,6 +22,7 @@ -export([add_handler/0, remove_handler/0]). -export([post_config_update/5]). -export([update/1]). +-export([conf/0, is_push_gateway_server_enabled/1]). update(Config) -> case @@ -64,7 +65,20 @@ update_prometheus(AppEnvs) -> ), application:set_env(AppEnvs). -update_push_gateway(#{enable := true}) -> - emqx_prometheus_sup:start_child(?APP); -update_push_gateway(#{enable := false}) -> - emqx_prometheus_sup:stop_child(?APP). +update_push_gateway(Prometheus) -> + case is_push_gateway_server_enabled(Prometheus) of + true -> + case erlang:whereis(?APP) of + undefined -> emqx_prometheus_sup:start_child(?APP, Prometheus); + Pid -> emqx_prometheus_sup:update_child(Pid, Prometheus) + end; + false -> + emqx_prometheus_sup:stop_child(?APP) + end. + +conf() -> + emqx_config:get(?PROMETHEUS). + +is_push_gateway_server_enabled(#{enable := true, push_gateway_server := Url}) -> Url =/= ""; +is_push_gateway_server_enabled(#{push_gateway := #{url := Url}}) -> Url =/= ""; +is_push_gateway_server_enabled(_) -> false. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 3aaf4292f..1e67b06ca 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -27,23 +27,58 @@ desc/1, translation/1, convert_headers/2, - validate_push_gateway_server/1 + validate_url/1 ]). -namespace() -> "prometheus". +namespace() -> prometheus. -roots() -> [{"prometheus", ?HOCON(?R_REF("prometheus"), #{translate_to => ["prometheus"]})}]. - -fields("prometheus") -> +roots() -> [ - {push_gateway_server, + {prometheus, + ?HOCON( + ?UNION(setting_union_schema()), + #{translate_to => ["prometheus"], default => #{}} + )} + ]. + +fields(recommend_setting) -> + [ + {enable_basic_auth, + ?HOCON( + boolean(), + #{ + default => false, + required => true, + importance => ?IMPORTANCE_HIGH, + desc => ?DESC(enable_basic_auth) + } + )}, + {push_gateway, + ?HOCON( + ?R_REF(push_gateway), + #{ + required => false, + importance => ?IMPORTANCE_MEDIUM, + desc => ?DESC(push_gateway) + } + )}, + {collectors, + ?HOCON(?R_REF(collector), #{ + required => false, + importance => ?IMPORTANCE_LOW, + desc => ?DESC(collectors) + })} + ]; +fields(push_gateway) -> + [ + {url, ?HOCON( string(), #{ - default => <<"http://127.0.0.1:9091">>, - required => true, - validator => fun ?MODULE:validate_push_gateway_server/1, - desc => ?DESC(push_gateway_server) + required => false, + default => <<"">>, + validator => fun ?MODULE:validate_url/1, + desc => ?DESC(push_gateway_url) } )}, {interval, @@ -51,7 +86,7 @@ fields("prometheus") -> emqx_schema:timeout_duration_ms(), #{ default => <<"15s">>, - required => true, + required => false, desc => ?DESC(interval) } )}, @@ -70,9 +105,113 @@ fields("prometheus") -> binary(), #{ default => <<"${name}/instance/${name}~${host}">>, - required => true, + required => false, desc => ?DESC(job_name) } + )} + ]; +fields(collector) -> + [ + {vm_dist, + ?HOCON( + hoconsc:enum([disabled, enabled]), + #{ + default => disabled, + required => true, + desc => ?DESC(vm_dist_collector) + } + )}, + %% Mnesia metrics mainly using mnesia:system_info/1 + {mnesia, + ?HOCON( + hoconsc:enum([enabled, disabled]), + #{ + default => disabled, + required => true, + desc => ?DESC(mnesia_collector) + } + )}, + %% Collects Erlang VM metrics using erlang:statistics/1. + {vm_statistics, + ?HOCON( + hoconsc:enum([enabled, disabled]), + #{ + default => disabled, + required => true, + desc => ?DESC(vm_statistics_collector) + } + )}, + %% Collects Erlang VM metrics using erlang:system_info/1. + {vm_system_info, + ?HOCON( + hoconsc:enum([enabled, disabled]), + #{ + default => disabled, + required => true, + desc => ?DESC(vm_system_info_collector) + } + )}, + %% Collects information about memory dynamically allocated by the Erlang VM using erlang:memory/0, + %% it also provides basic (D)ETS statistics. + {vm_memory, + ?HOCON( + hoconsc:enum([enabled, disabled]), + #{ + default => disabled, + required => true, + desc => ?DESC(vm_memory_collector) + } + )}, + %% Collects microstate accounting metrics using erlang:statistics(microstate_accounting). + {vm_msacc, + ?HOCON( + hoconsc:enum([enabled, disabled]), + #{ + default => disabled, + required => true, + desc => ?DESC(vm_msacc_collector) + } + )} + ]; +fields(legacy_deprecated_setting) -> + [ + {push_gateway_server, + ?HOCON( + string(), + #{ + default => <<"http://127.0.0.1:9091">>, + required => true, + validator => fun ?MODULE:validate_url/1, + desc => ?DESC(legacy_push_gateway_server) + } + )}, + {interval, + ?HOCON( + emqx_schema:timeout_duration_ms(), + #{ + default => <<"15s">>, + required => true, + desc => ?DESC(legacy_interval) + } + )}, + {headers, + ?HOCON( + list({string(), string()}), + #{ + default => #{}, + required => false, + converter => fun ?MODULE:convert_headers/2, + desc => ?DESC(legacy_headers) + } + )}, + {job_name, + ?HOCON( + binary(), + #{ + default => <<"${name}/instance/${name}~${host}">>, + required => true, + desc => ?DESC(legacy_job_name) + } )}, {enable, @@ -81,7 +220,7 @@ fields("prometheus") -> #{ default => false, required => true, - desc => ?DESC(enable) + desc => ?DESC(legacy_enable) } )}, {vm_dist_collector, @@ -91,7 +230,7 @@ fields("prometheus") -> default => disabled, required => true, importance => ?IMPORTANCE_LOW, - desc => ?DESC(vm_dist_collector) + desc => ?DESC(legacy_vm_dist_collector) } )}, %% Mnesia metrics mainly using mnesia:system_info/1 @@ -102,7 +241,7 @@ fields("prometheus") -> default => disabled, required => true, importance => ?IMPORTANCE_LOW, - desc => ?DESC(mnesia_collector) + desc => ?DESC(legacy_mnesia_collector) } )}, %% Collects Erlang VM metrics using erlang:statistics/1. @@ -113,7 +252,7 @@ fields("prometheus") -> default => disabled, required => true, importance => ?IMPORTANCE_LOW, - desc => ?DESC(vm_statistics_collector) + desc => ?DESC(legacy_vm_statistics_collector) } )}, %% Collects Erlang VM metrics using erlang:system_info/1. @@ -124,7 +263,7 @@ fields("prometheus") -> default => disabled, required => true, importance => ?IMPORTANCE_LOW, - desc => ?DESC(vm_system_info_collector) + desc => ?DESC(legacy_vm_system_info_collector) } )}, %% Collects information about memory dynamically allocated by the Erlang VM using erlang:memory/0, @@ -136,7 +275,7 @@ fields("prometheus") -> default => disabled, required => true, importance => ?IMPORTANCE_LOW, - desc => ?DESC(vm_memory_collector) + desc => ?DESC(legacy_vm_memory_collector) } )}, %% Collects microstate accounting metrics using erlang:statistics(microstate_accounting). @@ -147,14 +286,26 @@ fields("prometheus") -> default => disabled, required => true, importance => ?IMPORTANCE_LOW, - desc => ?DESC(vm_msacc_collector) + desc => ?DESC(legacy_vm_msacc_collector) } )} ]. -desc("prometheus") -> ?DESC(prometheus); +setting_union_schema() -> + RecommendSetting = ?R_REF(recommend_setting), + LegacySetting = ?R_REF(legacy_deprecated_setting), + fun + (all_union_members) -> [RecommendSetting, LegacySetting]; + ({value, #{<<"enable">> := _}}) -> [LegacySetting]; + %% all other cases treat as new config, include init empty config. + ({value, _}) -> [RecommendSetting] + end. + +desc(prometheus) -> ?DESC(prometheus); desc(_) -> undefined. +convert_headers(undefined, _) -> + undefined; convert_headers(Headers, #{make_serializable := true}) -> Headers; convert_headers(<<>>, _Opts) -> @@ -170,10 +321,22 @@ convert_headers(Headers, _Opts) when is_map(Headers) -> convert_headers(Headers, _Opts) when is_list(Headers) -> Headers. -validate_push_gateway_server(Url) -> +validate_url(Url) -> case uri_string:parse(Url) of - #{scheme := S} when S =:= "https" orelse S =:= "http" -> ok; - _ -> {error, "Invalid url"} + #{scheme := S} when + S =:= "https"; + S =:= "http"; + S =:= <<"https">>; + S =:= <<"http">> + -> + ok; + %% default is "" + #{path := []} -> + ok; + #{path := <<>>} -> + ok; + _ -> + {error, "Invalid url"} end. %% for CI test, CI don't load the whole emqx_conf_schema. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl index c284328ff..85281654f 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl @@ -20,7 +20,8 @@ -export([ start_link/0, - start_child/1, + start_child/2, + update_child/2, stop_child/1 ]). @@ -39,11 +40,13 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). --spec start_child(supervisor:child_spec() | atom()) -> ok. -start_child(ChildSpec) when is_map(ChildSpec) -> - assert_started(supervisor:start_child(?MODULE, ChildSpec)); -start_child(Mod) when is_atom(Mod) -> - assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, []))). +-spec start_child(atom(), map()) -> ok. +start_child(Mod, Conf) when is_atom(Mod) -> + assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Conf))). + +update_child(Pid, Conf) -> + erlang:send(Pid, {update, Conf}), + ok. -spec stop_child(any()) -> ok | {error, term()}. stop_child(ChildId) -> @@ -54,10 +57,11 @@ stop_child(ChildId) -> end. init([]) -> + Conf = emqx_prometheus_config:conf(), Children = - case emqx_conf:get([prometheus, enable], false) of + case emqx_prometheus_config:is_push_gateway_server_enabled(Conf) of false -> []; - true -> [?CHILD(emqx_prometheus, [])] + true -> [?CHILD(emqx_prometheus, Conf)] end, {ok, {{one_for_one, 10, 3600}, Children}}. diff --git a/apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl b/apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl deleted file mode 100644 index b3ba1b6ce..000000000 --- a/apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl +++ /dev/null @@ -1,41 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_prometheus_proto_v1). - --behaviour(emqx_bpapi). - --export([ - introduced_in/0, - deprecated_since/0, - start/1, - stop/1 -]). - --include_lib("emqx/include/bpapi.hrl"). - -deprecated_since() -> "5.0.10". - -introduced_in() -> - "5.0.0". - --spec start([node()]) -> emqx_rpc:multicall_result(). -start(Nodes) -> - rpc:multicall(Nodes, emqx_prometheus, do_start, [], 5000). - --spec stop([node()]) -> emqx_rpc:multicall_result(). -stop(Nodes) -> - rpc:multicall(Nodes, emqx_prometheus, do_stop, [], 5000). diff --git a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl index 3f9e743f3..4ac935604 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl @@ -22,7 +22,7 @@ -compile(export_all). -define(CLUSTER_RPC_SHARD, emqx_cluster_rpc_shard). --define(CONF_DEFAULT, << +-define(LEGACY_CONF_DEFAULT, << "\n" "prometheus {\n" " push_gateway_server = \"http://127.0.0.1:9091\"\n" @@ -38,45 +38,120 @@ " vm_msacc_collector = disabled\n" "}\n" >>). +-define(CONF_DEFAULT, #{ + <<"prometheus">> => + #{ + <<"enable_basic_auth">> => false, + <<"collectors">> => + #{ + <<"mnesia">> => <<"disabled">>, + <<"vm_dist">> => <<"disabled">>, + <<"vm_memory">> => <<"disabled">>, + <<"vm_msacc">> => <<"disabled">>, + <<"vm_statistics">> => <<"disabled">>, + <<"vm_system_info">> => <<"disabled">> + }, + <<"push_gateway">> => + #{ + <<"headers">> => #{<<"Authorization">> => <<"some-authz-tokens">>}, + <<"interval">> => <<"1s">>, + <<"job_name">> => <<"${name}~${host}">>, + <<"url">> => <<"http://127.0.0.1:9091">> + } + } +}). %%-------------------------------------------------------------------- %% Setups %%-------------------------------------------------------------------- +all() -> + [ + {group, new_config}, + {group, legacy_config} + ]. -all() -> emqx_common_test_helpers:all(?MODULE). +groups() -> + [ + {new_config, [sequence], common_tests()}, + {legacy_config, [sequence], common_tests()} + ]. -init_per_suite(Cfg) -> +suite() -> + [{timetrap, {seconds, 30}}]. + +common_tests() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_group(new_config, Config) -> + init_group(), + load_config(), + emqx_common_test_helpers:start_apps([emqx_prometheus]), + %% coverage olp metrics + {ok, _} = emqx:update_config([overload_protection, enable], true), + Config; +init_per_group(legacy_config, Config) -> + init_group(), + load_legacy_config(), + emqx_common_test_helpers:start_apps([emqx_prometheus]), + {ok, _} = emqx:update_config([overload_protection, enable], false), + Config. + +init_group() -> application:load(emqx_conf), ok = ekka:start(), ok = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], infinity), meck:new(emqx_alarm, [non_strict, passthrough, no_link]), meck:expect(emqx_alarm, activate, 3, ok), - meck:expect(emqx_alarm, deactivate, 3, ok), + meck:expect(emqx_alarm, deactivate, 3, ok). - load_config(), - emqx_common_test_helpers:start_apps([emqx_prometheus]), - Cfg. - -end_per_suite(_Cfg) -> +end_group() -> ekka:stop(), mria:stop(), mria_mnesia:delete_schema(), meck:unload(emqx_alarm), - emqx_common_test_helpers:stop_apps([emqx_prometheus]). +end_per_group(_Group, Config) -> + end_group(), + Config. + +init_per_testcase(t_assert_push, Config) -> + meck:new(httpc, [passthrough]), + Config; +init_per_testcase(t_push_gateway, Config) -> + start_mock_pushgateway(9091), + Config; +init_per_testcase(_Testcase, Config) -> + Config. + +end_per_testcase(t_push_gateway, Config) -> + stop_mock_pushgateway(), + Config; +end_per_testcase(t_assert_push, _Config) -> + meck:unload(httpc), + ok; +end_per_testcase(_Testcase, _Config) -> + ok. + load_config() -> ok = emqx_common_test_helpers:load_config(emqx_prometheus_schema, ?CONF_DEFAULT). +load_legacy_config() -> + ok = emqx_common_test_helpers:load_config(emqx_prometheus_schema, ?LEGACY_CONF_DEFAULT). + %%-------------------------------------------------------------------- %% Test cases %%-------------------------------------------------------------------- t_start_stop(_) -> App = emqx_prometheus, - ?assertMatch(ok, emqx_prometheus_sup:start_child(App)), + Conf = emqx_prometheus_config:conf(), + ?assertMatch(ok, emqx_prometheus_sup:start_child(App, Conf)), %% start twice return ok. - ?assertMatch(ok, emqx_prometheus_sup:start_child(App)), + ?assertMatch(ok, emqx_prometheus_sup:start_child(App, Conf)), + ok = gen_server:call(emqx_prometheus, dump, 1000), + ok = gen_server:cast(emqx_prometheus, dump), + dump = erlang:send(emqx_prometheus, dump), ?assertMatch(ok, emqx_prometheus_sup:stop_child(App)), %% stop twice return ok. ?assertMatch(ok, emqx_prometheus_sup:stop_child(App)), @@ -88,7 +163,6 @@ t_collector_no_crash_test(_) -> ok. t_assert_push(_) -> - meck:new(httpc, [passthrough]), Self = self(), AssertPush = fun(Method, Req = {Url, Headers, ContentType, _Data}, HttpOpts, Opts) -> ?assertEqual(post, Method), @@ -99,13 +173,51 @@ t_assert_push(_) -> meck:passthrough([Method, Req, HttpOpts, Opts]) end, meck:expect(httpc, request, AssertPush), - ?assertMatch(ok, emqx_prometheus_sup:start_child(emqx_prometheus)), + Conf = emqx_prometheus_config:conf(), + ?assertMatch(ok, emqx_prometheus_sup:start_child(emqx_prometheus, Conf)), receive pass -> ok after 2000 -> ct:fail(assert_push_request_failed) end. -t_only_for_coverage(_) -> - ?assertEqual("5.0.0", emqx_prometheus_proto_v1:introduced_in()), +t_push_gateway(_) -> + Conf = emqx_prometheus_config:conf(), + ?assertMatch(ok, emqx_prometheus_sup:stop_child(emqx_prometheus)), + ?assertMatch(ok, emqx_prometheus_sup:start_child(emqx_prometheus, Conf)), + ?assertMatch(#{ok := 0, failed := 0}, emqx_prometheus:info()), + timer:sleep(1100), + ?assertMatch(#{ok := 1, failed := 0}, emqx_prometheus:info()), + ok = emqx_prometheus_sup:update_child(emqx_prometheus, Conf), + ?assertMatch(#{ok := 0, failed := 0}, emqx_prometheus:info()), + ok. + +start_mock_pushgateway(Port) -> + application:ensure_all_started(cowboy), + Dispatch = cowboy_router:compile([{'_', [{'_', ?MODULE, []}]}]), + {ok, _} = cowboy:start_clear( + mock_pushgateway_listener, + [{port, Port}], + #{env => #{dispatch => Dispatch}} + ). + +stop_mock_pushgateway() -> + cowboy:stop_listener(mock_pushgateway_listener). + +init(Req0, Opts) -> + Method = cowboy_req:method(Req0), + Headers = cowboy_req:headers(Req0), + ?assertEqual(<<"POST">>, Method), + ?assertMatch( + #{ + <<"authorization">> := <<"some-authz-tokens">>, + <<"content-length">> := _, + <<"content-type">> := <<"text/plain">>, + <<"host">> := <<"127.0.0.1:9091">> + }, + Headers + ), + RespHeader = #{<<"content-type">> => <<"text/plain; charset=utf-8">>}, + Req = -cowboy_req:reply(200, RespHeader, <<"OK">>, Req0), + {ok, Req, Opts}. diff --git a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl index eb909baf5..26957ed23 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl @@ -28,40 +28,59 @@ %%-------------------------------------------------------------------- %% Setups %%-------------------------------------------------------------------- - all() -> - emqx_common_test_helpers:all(?MODULE). + [ + {group, new_config}, + {group, legacy_config} + ]. + +groups() -> + [ + {new_config, [sequence], [t_stats_api, t_prometheus_api]}, + {legacy_config, [sequence], [t_stats_api, t_legacy_prometheus_api]} + ]. init_per_suite(Config) -> - application:load(emqx_conf), - ok = ekka:start(), - ok = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], infinity), - - meck:new(mria_rlog, [non_strict, passthrough, no_link]), - - emqx_prometheus_SUITE:load_config(), - emqx_mgmt_api_test_util:init_suite([emqx_prometheus]), - + emqx_prometheus_SUITE:init_group(), + emqx_mgmt_api_test_util:init_suite([emqx_conf]), Config. - end_per_suite(Config) -> - ekka:stop(), - mria:stop(), - mria_mnesia:delete_schema(), - - meck:unload(mria_rlog), - - emqx_mgmt_api_test_util:end_suite([emqx_prometheus]), + emqx_prometheus_SUITE:end_group(), + emqx_mgmt_api_test_util:end_suite([emqx_conf]), Config. -init_per_testcase(_, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(), +init_per_group(new_config, Config) -> + emqx_common_test_helpers:start_apps( + [emqx_prometheus], + fun(App) -> set_special_configs(App, new_config) end + ), + Config; +init_per_group(legacy_config, Config) -> + emqx_common_test_helpers:start_apps( + [emqx_prometheus], + fun(App) -> set_special_configs(App, legacy_config) end + ), Config. +end_per_group(_Group, Config) -> + _ = application:stop(emqx_prometheus), + Config. + +set_special_configs(emqx_dashboard, _) -> + emqx_dashboard_api_test_helpers:set_default_config(); +set_special_configs(emqx_prometheus, new_config) -> + emqx_prometheus_SUITE:load_config(), + ok; +set_special_configs(emqx_prometheus, legacy_config) -> + emqx_prometheus_SUITE:load_legacy_config(), + ok; +set_special_configs(_App, _) -> + ok. + %%-------------------------------------------------------------------- %% Cases %%-------------------------------------------------------------------- -t_prometheus_api(_) -> +t_legacy_prometheus_api(_) -> Path = emqx_mgmt_api_test_util:api_path(["prometheus"]), Auth = emqx_mgmt_api_test_util:auth_header_(), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, "", Auth), @@ -145,6 +164,105 @@ t_prometheus_api(_) -> ), ok. +t_prometheus_api(_) -> + Path = emqx_mgmt_api_test_util:api_path(["prometheus"]), + Auth = emqx_mgmt_api_test_util:auth_header_(), + {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, "", Auth), + + Conf = emqx_utils_json:decode(Response, [return_maps]), + ?assertMatch( + #{ + <<"push_gateway">> := #{}, + <<"collectors">> := _, + <<"enable_basic_auth">> := _ + }, + Conf + ), + #{ + <<"push_gateway">> := + #{<<"url">> := Url} = PushGateway, + <<"collectors">> := Collector + } = Conf, + Pid = erlang:whereis(emqx_prometheus), + ?assertEqual(Url =/= "", undefined =/= Pid, {Url, Pid}), + + NewConf = Conf#{ + <<"push_gateway">> => PushGateway#{ + <<"interval">> => <<"2s">>, + <<"headers">> => #{ + <<"test-str1">> => <<"test-value">>, + <<"test-str2">> => <<"42">> + } + }, + <<"collectors">> => Collector#{ + <<"vm_dist">> => <<"enabled">>, + <<"vm_system_info">> => <<"enabled">>, + <<"vm_memory">> => <<"enabled">>, + <<"vm_msacc">> => <<"enabled">>, + <<"mnesia">> => <<"enabled">>, + <<"vm_statistics">> => <<"enabled">> + } + }, + {ok, Response2} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf), + + Conf2 = emqx_utils_json:decode(Response2, [return_maps]), + ?assertMatch(NewConf, Conf2), + + EnvCollectors = application:get_env(prometheus, collectors, []), + PromCollectors = prometheus_registry:collectors(default), + ?assertEqual(lists:sort(EnvCollectors), lists:sort(PromCollectors)), + ?assert(lists:member(prometheus_vm_statistics_collector, EnvCollectors), EnvCollectors), + + lists:foreach( + fun({C, Enabled}) -> + ?assertEqual(Enabled, lists:member(C, EnvCollectors), EnvCollectors) + end, + [ + {prometheus_vm_dist_collector, true}, + {prometheus_vm_system_info_collector, true}, + {prometheus_vm_memory_collector, true}, + {prometheus_mnesia_collector, true}, + {prometheus_vm_msacc_collector, true}, + {prometheus_vm_statistics_collector, true} + ] + ), + + ?assertMatch( + #{ + <<"push_gateway">> := #{ + <<"headers">> := #{ + <<"test-str1">> := <<"test-value">>, + <<"test-str2">> := <<"42">> + } + } + }, + emqx_config:get_raw([prometheus]) + ), + ?assertMatch( + #{ + push_gateway := #{ + headers := [ + {"test-str2", "42"}, + {"test-str1", "test-value"} + ] + } + }, + emqx_config:get([prometheus]) + ), + + NewConf1 = Conf#{<<"push_gateway">> => PushGateway#{<<"url">> => <<"">>}}, + {ok, _Response3} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf1), + ?assertEqual(undefined, erlang:whereis(emqx_prometheus)), + + ConfWithoutScheme = Conf#{ + <<"push_gateway">> => PushGateway#{<<"url">> => <<"127.0.0.1:8081">>} + }, + ?assertMatch( + {error, {"HTTP/1.1", 400, _}}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, ConfWithoutScheme) + ), + ok. + t_stats_api(_) -> Path = emqx_mgmt_api_test_util:api_path(["prometheus", "stats"]), Auth = emqx_mgmt_api_test_util:auth_header_(), diff --git a/rel/i18n/emqx_prometheus_api.hocon b/rel/i18n/emqx_prometheus_api.hocon index c468ccc2f..0d9b5dc5f 100644 --- a/rel/i18n/emqx_prometheus_api.hocon +++ b/rel/i18n/emqx_prometheus_api.hocon @@ -11,8 +11,8 @@ update_prom_conf_info.label: """Update Prometheus config""" get_prom_data.desc: -"""Get Prometheus Data""" +"""Get Prometheus Metrics""" get_prom_data.label: -"""Get Prometheus Data""" +"""Prometheus Metrics""" } diff --git a/rel/i18n/emqx_prometheus_schema.hocon b/rel/i18n/emqx_prometheus_schema.hocon index d665343e9..d97f1884f 100644 --- a/rel/i18n/emqx_prometheus_schema.hocon +++ b/rel/i18n/emqx_prometheus_schema.hocon @@ -1,8 +1,5 @@ emqx_prometheus_schema { -enable.desc: -"""Turn Prometheus data pushing on or off""" - headers.desc: """An HTTP Headers when pushing to Push Gateway.
For example, { Authorization = "some-authz-tokens"}""" @@ -17,19 +14,25 @@ job_name.desc: For example, when the EMQX node name is emqx@127.0.0.1 then the name variable takes value emqx and the host variable takes value 127.0.0.1.
Default value is: ${name}/instance/${name}~${host}""" -mnesia_collector.desc: -"""Enable or disable Mnesia metrics collector""" - prometheus.desc: """EMQX's Prometheus scraping endpoint is enabled by default without authentication. -You can inspect it with a `curl` command like this: `curl -f "127.0.0.1:18083/api/v5/prometheus/stats"`
-The 'enable' flag is used to turn on and off for the push-gateway integration.""" +You can inspect it with a `curl` command like this: `curl -f "127.0.0.1:18083/api/v5/prometheus/stats"`
""" prometheus.label: """Prometheus""" -push_gateway_server.desc: -"""URL of Prometheus server. Pushgateway is optional, should not be configured if prometheus is to scrape EMQX.""" +push_gateway.desc: +"""Push Gateway is optional, should not be configured if prometheus is to scrape EMQX.""" + +collectors.desc: +"""Metrics collectors to be enabled.""" + +push_gateway_url.desc: +"""URL of Prometheus server. Pushgateway is optional, should not be configured if prometheus is to scrape EMQX. +Set url to "" to disable push gateway""" + +mnesia_collector.desc: +"""Enable or disable Mnesia metrics collector""" vm_dist_collector.desc: """Enable or disable VM distribution collector, collects information about the sockets and processes involved in the Erlang distribution mechanism.""" @@ -46,4 +49,37 @@ vm_statistics_collector.desc: vm_system_info_collector.desc: """Enable or disable VM system info collector.""" +legacy_enable.desc: +"""Deprecated, use prometheus.push_gateway.url instead""" + +legacy_headers.desc: +"""Deprecated, use prometheus.push_gateway.headers instead""" + +legacy_interval.desc: +"""Deprecated, use prometheus.push_gateway.interval instead""" + +legacy_job_name.desc: +"""Deprecated, use prometheus.push_gateway.job_name instead""" + +legacy_push_gateway_server.desc: +"""Deprecated, use prometheus.push_gateway.url instead""" + +legacy_mnesia_collector.desc: +"""Deprecated, use prometheus.collectors.mnesia instead""" + +legacy_vm_dist_collector.desc: +"""Deprecated, use prometheus.collectors.vm_dist instead""" + +legacy_vm_memory_collector.desc: +"""Deprecated, use prometheus.collectors.vm_memory instead""" + +legacy_vm_msacc_collector.desc: +"""Deprecated, use prometheus.collectors.vm_msacc instead""" + +legacy_vm_statistics_collector.desc: +"""Deprecated, use prometheus.collectors.vm_statistics instead""" + +legacy_vm_system_info_collector.desc: +"""Deprecated, use prometheus.collectors.vm_system_info instead""" + } From 0c2610fa61734cf9b6aa03403fe842bd9c57a076 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sun, 5 Nov 2023 10:51:22 +0800 Subject: [PATCH 08/20] feat: support prometheus.enable_basic_auth --- .../src/emqx_prometheus_api.erl | 7 +++- .../src/emqx_prometheus_config.erl | 11 +++++- .../test/emqx_prometheus_api_SUITE.erl | 37 +++++++++++++------ 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index 3072ab5a7..b08cd8388 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -68,12 +68,17 @@ schema("/prometheus/stats") -> #{ description => ?DESC(get_prom_data), tags => ?TAGS, - security => [], + security => security(), responses => #{200 => prometheus_data_schema()} } }. +security() -> + case emqx_config:get([prometheus, enable_basic_auth], false) of + true -> [#{'basicAuth' => []}, #{'bearerAuth' => []}]; + false -> [] + end. %%-------------------------------------------------------------------- %% API Handler funcs %%-------------------------------------------------------------------- diff --git a/apps/emqx_prometheus/src/emqx_prometheus_config.erl b/apps/emqx_prometheus/src/emqx_prometheus_config.erl index 7cc5c7f8b..a3c20a7af 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_config.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_config.erl @@ -46,9 +46,10 @@ remove_handler() -> ok = emqx_config_handler:remove_handler(?PROMETHEUS), ok. -post_config_update(?PROMETHEUS, _Req, New, _Old, AppEnvs) -> +post_config_update(?PROMETHEUS, _Req, New, Old, AppEnvs) -> update_prometheus(AppEnvs), - update_push_gateway(New); + update_push_gateway(New), + update_auth(New, Old); post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) -> ok. @@ -76,6 +77,12 @@ update_push_gateway(Prometheus) -> emqx_prometheus_sup:stop_child(?APP) end. +update_auth(#{enable_basic_auth := New}, #{enable_basic_auth := Old}) when New =/= Old -> + emqx_dashboard_listener:regenerate_minirest_dispatch(), + ok; +update_auth(_, _) -> + ok. + conf() -> emqx_config:get(?PROMETHEUS). diff --git a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl index 26957ed23..dc56841ad 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl @@ -36,8 +36,8 @@ all() -> groups() -> [ - {new_config, [sequence], [t_stats_api, t_prometheus_api]}, - {legacy_config, [sequence], [t_stats_api, t_legacy_prometheus_api]} + {new_config, [sequence], [t_stats_auth_api, t_stats_no_auth_api, t_prometheus_api]}, + {legacy_config, [sequence], [t_stats_no_auth_api, t_legacy_prometheus_api]} ]. init_per_suite(Config) -> @@ -263,21 +263,34 @@ t_prometheus_api(_) -> ), ok. -t_stats_api(_) -> - Path = emqx_mgmt_api_test_util:api_path(["prometheus", "stats"]), - Auth = emqx_mgmt_api_test_util:auth_header_(), - Headers = [{"accept", "application/json"}, Auth], - {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, "", Headers), +t_stats_no_auth_api(_) -> + %% undefined is legacy prometheus + case emqx:get_config([prometheus, enable_basic_auth], undefined) of + true -> + {ok, _} = emqx:update_config([prometheus, enable_basic_auth], false), + emqx_dashboard_listener:regenerate_minirest_dispatch(); + _ -> + ok + end, + emqx_dashboard_listener:regenerate_minirest_dispatch(), + Json = [{"accept", "application/json"}], + request_stats(Json, []). +t_stats_auth_api(_) -> + {ok, _} = emqx:update_config([prometheus, enable_basic_auth], true), + Auth = emqx_mgmt_api_test_util:auth_header_(), + JsonAuth = [{"accept", "application/json"}, Auth], + request_stats(JsonAuth, Auth), + ok. + +request_stats(JsonAuth, Auth) -> + Path = emqx_mgmt_api_test_util:api_path(["prometheus", "stats"]), + {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, "", JsonAuth), Data = emqx_utils_json:decode(Response, [return_maps]), ?assertMatch(#{<<"client">> := _, <<"delivery">> := _}, Data), - {ok, _} = emqx_mgmt_api_test_util:request_api(get, Path, "", Auth), - ok = meck:expect(mria_rlog, backend, fun() -> rlog end), - {ok, _} = emqx_mgmt_api_test_util:request_api(get, Path, "", Auth), - - ok. + {ok, _} = emqx_mgmt_api_test_util:request_api(get, Path, "", Auth). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Internal Functions From f04cafe57e79307e8eec7f35c0fa54955760e9c3 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sun, 5 Nov 2023 12:18:51 +0800 Subject: [PATCH 09/20] chore: update prometheus config desc --- rel/i18n/emqx_prometheus_schema.hocon | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/rel/i18n/emqx_prometheus_schema.hocon b/rel/i18n/emqx_prometheus_schema.hocon index d97f1884f..f5cb2ad8e 100644 --- a/rel/i18n/emqx_prometheus_schema.hocon +++ b/rel/i18n/emqx_prometheus_schema.hocon @@ -11,7 +11,8 @@ job_name.desc: """Job Name that is pushed to the Push Gateway. Available variables:
- ${name}: Name of EMQX node.
- ${host}: Host name of EMQX node.
-For example, when the EMQX node name is emqx@127.0.0.1 then the name variable takes value emqx and the host variable takes value 127.0.0.1.
+For example, when the EMQX node name is emqx@127.0.0.1 then the name +variable takes value emqx and the host variable takes value 127.0.0.1. Default value is: ${name}/instance/${name}~${host}""" prometheus.desc: @@ -25,20 +26,24 @@ push_gateway.desc: """Push Gateway is optional, should not be configured if prometheus is to scrape EMQX.""" collectors.desc: -"""Metrics collectors to be enabled.""" +"""The internal advanced metrics of the virtual machine are initially disabled +and are usually only enabled during performance testing. +Enabling them will increase the CPU load.""" push_gateway_url.desc: -"""URL of Prometheus server. Pushgateway is optional, should not be configured if prometheus is to scrape EMQX. +"""URL of Pushgateway server. Pushgateway is optional, should not be configured if prometheus is to scrape EMQX. Set url to "" to disable push gateway""" mnesia_collector.desc: -"""Enable or disable Mnesia metrics collector""" +"""Collects Mnesia metrics mainly using mnesia:system_info/1 """ vm_dist_collector.desc: -"""Enable or disable VM distribution collector, collects information about the sockets and processes involved in the Erlang distribution mechanism.""" +"""Enable or disable VM distribution collector, +collects information about the sockets and processes involved in the Erlang distribution mechanism.""" vm_memory_collector.desc: -"""Enable or disable VM memory metrics collector.""" +"""Collects information about memory dynamically allocated by the Erlang emulator using + erlang:memory/0 .""" vm_msacc_collector.desc: """Enable or disable VM microstate accounting metrics collector.""" From b343653769e2a71cfe6142572e2a16c4ff4c139d Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sun, 5 Nov 2023 12:30:49 +0800 Subject: [PATCH 10/20] chore: support list(tuple()) for schema_with_examples/3 --- apps/emqx_dashboard/src/emqx_dashboard_swagger.erl | 3 ++- apps/emqx_prometheus/src/emqx_prometheus_config.erl | 2 +- apps/emqx_prometheus/src/emqx_prometheus_sup.erl | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index c1379d4d6..6b3d5b4fa 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -185,7 +185,8 @@ fields(meta) -> schema_with_example(Type, Example) -> hoconsc:mk(Type, #{examples => #{<<"example">> => Example}}). --spec schema_with_examples(hocon_schema:type(), map()) -> hocon_schema:field_schema_map(). +-spec schema_with_examples(hocon_schema:type(), map() | list(tuple())) -> + hocon_schema:field_schema_map(). schema_with_examples(Type, Examples) -> hoconsc:mk(Type, #{examples => #{<<"examples">> => Examples}}). diff --git a/apps/emqx_prometheus/src/emqx_prometheus_config.erl b/apps/emqx_prometheus/src/emqx_prometheus_config.erl index a3c20a7af..27f654bea 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_config.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_config.erl @@ -48,7 +48,7 @@ remove_handler() -> post_config_update(?PROMETHEUS, _Req, New, Old, AppEnvs) -> update_prometheus(AppEnvs), - update_push_gateway(New), + _ = update_push_gateway(New), update_auth(New, Old); post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) -> ok. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl index 85281654f..e8404fc6b 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl @@ -44,6 +44,7 @@ start_link() -> start_child(Mod, Conf) when is_atom(Mod) -> assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Conf))). +-spec update_child(pid() | atom(), map()) -> ok. update_child(Pid, Conf) -> erlang:send(Pid, {update, Conf}), ok. From c4eb9f86e77f65fbeee211dbb687e07b22c8adf4 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 6 Nov 2023 09:42:04 +0800 Subject: [PATCH 11/20] chore: add refactor prometheus refactor changelog --- apps/emqx_prometheus/src/emqx_prometheus.erl | 15 +++++- .../src/emqx_prometheus_config.erl | 50 ++++++++++++++++++- .../src/emqx_prometheus_schema.erl | 37 +++++++++++--- .../src/emqx_prometheus_sup.erl | 5 ++ .../src/proto/emqx_prometheus_proto_v1.erl | 41 +++++++++++++++ .../test/emqx_prometheus_SUITE.erl | 2 +- changes/ce/feat-11884.en.md | 4 ++ rel/i18n/emqx_prometheus_schema.hocon | 36 ++++++++----- scripts/spellcheck/dicts/emqx.txt | 1 + 9 files changed, 169 insertions(+), 22 deletions(-) create mode 100644 apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl create mode 100644 changes/ce/feat-11884.en.md diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 29d531140..09ba157a0 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -59,6 +59,12 @@ -export([collect/1]). +-export([ + %% For bpapi, deprecated_since 5.0.10, remove this when 5.1.x + do_start/0, + do_stop/0 +]). + -define(C(K, L), proplists:get_value(K, L, 0)). -define(TIMER_MSG, '#interval'). @@ -112,7 +118,6 @@ handle_info({update, Conf}, State = #{timer := Timer}) -> handle_info(_Msg, State) -> {noreply, State}. - push_to_push_gateway(Url, Headers) when is_list(Headers) -> Data = prometheus_text_format:format(), case httpc:request(post, {Url, Headers, "text/plain", Data}, ?HTTP_OPTIONS, []) of @@ -678,3 +683,11 @@ emqx_cluster_data() -> {nodes_running, length(Running)}, {nodes_stopped, length(Stopped)} ]. + +%% deprecated_since 5.0.10, remove this when 5.1.x +do_start() -> + emqx_prometheus_sup:start_child(?APP). + +%% deprecated_since 5.0.10, remove this when 5.1.x +do_stop() -> + emqx_prometheus_sup:stop_child(?APP). diff --git a/apps/emqx_prometheus/src/emqx_prometheus_config.erl b/apps/emqx_prometheus/src/emqx_prometheus_config.erl index 27f654bea..dccf28af0 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_config.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_config.erl @@ -20,7 +20,7 @@ -include("emqx_prometheus.hrl"). -export([add_handler/0, remove_handler/0]). --export([post_config_update/5]). +-export([pre_config_update/3, post_config_update/5]). -export([update/1]). -export([conf/0, is_push_gateway_server_enabled/1]). @@ -46,6 +46,54 @@ remove_handler() -> ok = emqx_config_handler:remove_handler(?PROMETHEUS), ok. +%% when we import the config with the old version +%% we need to respect it, and convert to new schema. +pre_config_update(?PROMETHEUS, MergeConf, OriginConf) -> + OriginType = emqx_prometheus_schema:is_recommend_type(OriginConf), + MergeType = emqx_prometheus_schema:is_recommend_type(MergeConf), + {ok, + case {OriginType, MergeType} of + {true, false} -> to_recommend_type(MergeConf); + _ -> MergeConf + end}. + +to_recommend_type(Conf) -> + #{ + <<"push_gateway">> => to_push_gateway(Conf), + <<"collectors">> => to_collectors(Conf) + }. + +to_push_gateway(Conf) -> + Init = maps:with([<<"interval">>, <<"headers">>, <<"job_name">>], Conf), + case maps:get(<<"push_gateway_server">>, Conf, "") of + "" -> + Init#{<<"url">> => <<"">>}; + Url -> + case maps:get(<<"enable">>, Conf, false) of + false -> Init#{<<"url">> => <<"">>}; + true -> Init#{<<"url">> => Url} + end + end. + +to_collectors(Conf) -> + lists:foldl( + fun({From, To}, Acc) -> + case maps:find(From, Conf) of + {ok, Value} -> Acc#{To => Value}; + error -> Acc + end + end, + #{}, + [ + {<<"vm_dist_collector">>, <<"vm_dist">>}, + {<<"mnesia_collector">>, <<"mnesia">>}, + {<<"vm_statistics_collector">>, <<"vm_statistics">>}, + {<<"vm_system_info_collector">>, <<"vm_system_info">>}, + {<<"vm_memory_collector">>, <<"vm_memory">>}, + {<<"vm_msacc_collector">>, <<"vm_msacc">>} + ] + ). + post_config_update(?PROMETHEUS, _Req, New, Old, AppEnvs) -> update_prometheus(AppEnvs), _ = update_push_gateway(New), diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 1e67b06ca..533b26c9f 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -27,7 +27,8 @@ desc/1, translation/1, convert_headers/2, - validate_url/1 + validate_url/1, + is_recommend_type/1 ]). namespace() -> prometheus. @@ -63,7 +64,7 @@ fields(recommend_setting) -> } )}, {collectors, - ?HOCON(?R_REF(collector), #{ + ?HOCON(?R_REF(collectors), #{ required => false, importance => ?IMPORTANCE_LOW, desc => ?DESC(collectors) @@ -110,7 +111,7 @@ fields(push_gateway) -> } )} ]; -fields(collector) -> +fields(collectors) -> [ {vm_dist, ?HOCON( @@ -295,13 +296,35 @@ setting_union_schema() -> RecommendSetting = ?R_REF(recommend_setting), LegacySetting = ?R_REF(legacy_deprecated_setting), fun - (all_union_members) -> [RecommendSetting, LegacySetting]; - ({value, #{<<"enable">> := _}}) -> [LegacySetting]; - %% all other cases treat as new config, include init empty config. - ({value, _}) -> [RecommendSetting] + (all_union_members) -> + [RecommendSetting, LegacySetting]; + ({value, Setting}) -> + case is_recommend_type(Setting) of + true -> [RecommendSetting]; + false -> [LegacySetting] + end + end. + +%% For it to be considered as new schema, +%% all keys must be included in the new configuration. +is_recommend_type(Setting) -> + case maps:keys(Setting) of + [] -> + true; + Keys -> + NewKeys = fields(recommend_setting), + Fun = fun(Key0) -> + Key = binary_to_existing_atom(Key0), + lists:keymember(Key, 1, NewKeys) + end, + lists:all(Fun, Keys) end. desc(prometheus) -> ?DESC(prometheus); +desc(collectors) -> ?DESC(collectors); +desc(legacy_deprecated_setting) -> ?DESC(legacy_deprecated_setting); +desc(recommend_setting) -> ?DESC(recommend_setting); +desc(push_gateway) -> ?DESC(push_gateway); desc(_) -> undefined. convert_headers(undefined, _) -> diff --git a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl index e8404fc6b..ea8a2ebaa 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl @@ -20,6 +20,7 @@ -export([ start_link/0, + start_child/1, start_child/2, update_child/2, stop_child/1 @@ -40,6 +41,10 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). +-spec start_child(atom()) -> ok. +start_child(Mod) when is_atom(Mod) -> + start_child(Mod, emqx_prometheus_config:conf()). + -spec start_child(atom(), map()) -> ok. start_child(Mod, Conf) when is_atom(Mod) -> assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Conf))). diff --git a/apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl b/apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl new file mode 100644 index 000000000..b3ba1b6ce --- /dev/null +++ b/apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl @@ -0,0 +1,41 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_prometheus_proto_v1). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + deprecated_since/0, + start/1, + stop/1 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +deprecated_since() -> "5.0.10". + +introduced_in() -> + "5.0.0". + +-spec start([node()]) -> emqx_rpc:multicall_result(). +start(Nodes) -> + rpc:multicall(Nodes, emqx_prometheus, do_start, [], 5000). + +-spec stop([node()]) -> emqx_rpc:multicall_result(). +stop(Nodes) -> + rpc:multicall(Nodes, emqx_prometheus, do_stop, [], 5000). diff --git a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl index 4ac935604..311233f1b 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl @@ -219,5 +219,5 @@ init(Req0, Opts) -> Headers ), RespHeader = #{<<"content-type">> => <<"text/plain; charset=utf-8">>}, - Req = -cowboy_req:reply(200, RespHeader, <<"OK">>, Req0), + Req = cowboy_req:reply(200, RespHeader, <<"OK">>, Req0), {ok, Req, Opts}. diff --git a/changes/ce/feat-11884.en.md b/changes/ce/feat-11884.en.md new file mode 100644 index 000000000..66d2b0a8e --- /dev/null +++ b/changes/ce/feat-11884.en.md @@ -0,0 +1,4 @@ +Modified the Prometheus API and configuration to: +- Restructure configuration sections to group related settings, improving readability and maintainability +- Introduced `enable_basic_auth` configuration for basic authentication on the scrape API endpoint, enhancing security +- Maintained backwards compatibility while refactoring code, avoiding breaking changes diff --git a/rel/i18n/emqx_prometheus_schema.hocon b/rel/i18n/emqx_prometheus_schema.hocon index f5cb2ad8e..57d6b8065 100644 --- a/rel/i18n/emqx_prometheus_schema.hocon +++ b/rel/i18n/emqx_prometheus_schema.hocon @@ -17,7 +17,7 @@ Default value is: ${name}/instance/${name}~${host}""" prometheus.desc: """EMQX's Prometheus scraping endpoint is enabled by default without authentication. -You can inspect it with a `curl` command like this: `curl -f "127.0.0.1:18083/api/v5/prometheus/stats"`
""" +You can inspect it with a `curl` command like this: `curl -f "127.0.0.1:18083/api/v5/prometheus/stats"`""" prometheus.label: """Prometheus""" @@ -25,11 +25,17 @@ prometheus.label: push_gateway.desc: """Push Gateway is optional, should not be configured if prometheus is to scrape EMQX.""" +enable_basic_auth.desc: +"""Enable or disable basic authentication for prometheus scrape api, not for Push Gateway""" + collectors.desc: """The internal advanced metrics of the virtual machine are initially disabled and are usually only enabled during performance testing. Enabling them will increase the CPU load.""" +recommend_setting.desc: +"""Recommended setting""" + push_gateway_url.desc: """URL of Pushgateway server. Pushgateway is optional, should not be configured if prometheus is to scrape EMQX. Set url to "" to disable push gateway""" @@ -54,37 +60,43 @@ vm_statistics_collector.desc: vm_system_info_collector.desc: """Enable or disable VM system info collector.""" +legacy_deprecated_setting.desc: +"""Deprecated""" + legacy_enable.desc: -"""Deprecated, use prometheus.push_gateway.url instead""" +"""Deprecated, use `prometheus.push_gateway.url` instead""" legacy_headers.desc: -"""Deprecated, use prometheus.push_gateway.headers instead""" +"""Deprecated, use `prometheus.push_gateway.headers` instead""" legacy_interval.desc: -"""Deprecated, use prometheus.push_gateway.interval instead""" +"""Deprecated, use `prometheus.push_gateway.interval` instead""" legacy_job_name.desc: -"""Deprecated, use prometheus.push_gateway.job_name instead""" +"""Deprecated, use `prometheus.push_gateway.job_name` instead""" legacy_push_gateway_server.desc: -"""Deprecated, use prometheus.push_gateway.url instead""" +"""Deprecated, use `prometheus.push_gateway.url` instead""" legacy_mnesia_collector.desc: -"""Deprecated, use prometheus.collectors.mnesia instead""" +"""Deprecated, use `prometheus.collectors.mnesia` instead""" legacy_vm_dist_collector.desc: -"""Deprecated, use prometheus.collectors.vm_dist instead""" +"""Deprecated, use `prometheus.collectors.vm_dist` instead""" legacy_vm_memory_collector.desc: -"""Deprecated, use prometheus.collectors.vm_memory instead""" +"""Deprecated, use `prometheus.collectors.vm_memory` instead""" legacy_vm_msacc_collector.desc: -"""Deprecated, use prometheus.collectors.vm_msacc instead""" +"""Deprecated, use `prometheus.collectors.vm_msacc` instead""" legacy_vm_statistics_collector.desc: -"""Deprecated, use prometheus.collectors.vm_statistics instead""" +"""Deprecated, use `prometheus.collectors.vm_statistics` instead""" legacy_vm_system_info_collector.desc: -"""Deprecated, use prometheus.collectors.vm_system_info instead""" +"""Deprecated, use `prometheus.collectors.vm_system_info` instead""" + +legacy_deprecated_setting.desc: +"""Deprecated""" } diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index 5630404c3..bc05df68a 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -295,3 +295,4 @@ dnstream upstream priv Syskeeper +msacc From b2e168d5080c6adf97cfc84b0fa91d887f87996e Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 17 Nov 2023 08:42:55 +0800 Subject: [PATCH 12/20] feat: add prometheus.push_gateway.enable config --- .../src/emqx_prometheus_config.erl | 18 ++++++------ .../src/emqx_prometheus_schema.erl | 19 +++++++----- .../test/emqx_prometheus_SUITE.erl | 1 + .../test/emqx_prometheus_api_SUITE.erl | 6 ++-- rel/config/examples/prometheus.conf.example | 20 +++++++++++-- rel/i18n/emqx_prometheus_schema.hocon | 29 ++++++++++--------- 6 files changed, 56 insertions(+), 37 deletions(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus_config.erl b/apps/emqx_prometheus/src/emqx_prometheus_config.erl index dccf28af0..66d53f2db 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_config.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_config.erl @@ -64,15 +64,12 @@ to_recommend_type(Conf) -> }. to_push_gateway(Conf) -> - Init = maps:with([<<"interval">>, <<"headers">>, <<"job_name">>], Conf), + Init = maps:with([<<"interval">>, <<"headers">>, <<"job_name">>, <<"enable">>], Conf), case maps:get(<<"push_gateway_server">>, Conf, "") of "" -> - Init#{<<"url">> => <<"">>}; + Init#{<<"enable">> => false}; Url -> - case maps:get(<<"enable">>, Conf, false) of - false -> Init#{<<"url">> => <<"">>}; - true -> Init#{<<"url">> => Url} - end + Init#{<<"url">> => Url} end. to_collectors(Conf) -> @@ -134,6 +131,9 @@ update_auth(_, _) -> conf() -> emqx_config:get(?PROMETHEUS). -is_push_gateway_server_enabled(#{enable := true, push_gateway_server := Url}) -> Url =/= ""; -is_push_gateway_server_enabled(#{push_gateway := #{url := Url}}) -> Url =/= ""; -is_push_gateway_server_enabled(_) -> false. +is_push_gateway_server_enabled(#{enable := true, push_gateway_server := Url}) -> + Url =/= ""; +is_push_gateway_server_enabled(#{push_gateway := #{url := Url, enable := Enable}}) -> + Enable andalso Url =/= ""; +is_push_gateway_server_enabled(_) -> + false. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 533b26c9f..7df5951f0 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -72,12 +72,21 @@ fields(recommend_setting) -> ]; fields(push_gateway) -> [ + {enable, + ?HOCON( + boolean(), + #{ + default => false, + required => true, + desc => ?DESC(push_gateway_enable) + } + )}, {url, ?HOCON( string(), #{ required => false, - default => <<"">>, + default => <<"http://127.0.0.1:9091">>, validator => fun ?MODULE:validate_url/1, desc => ?DESC(push_gateway_url) } @@ -197,7 +206,7 @@ fields(legacy_deprecated_setting) -> )}, {headers, ?HOCON( - list({string(), string()}), + typerefl:alias("map", list({string(), string()}), #{}, [string(), string()]), #{ default => #{}, required => false, @@ -214,7 +223,6 @@ fields(legacy_deprecated_setting) -> desc => ?DESC(legacy_job_name) } )}, - {enable, ?HOCON( boolean(), @@ -353,11 +361,6 @@ validate_url(Url) -> S =:= <<"http">> -> ok; - %% default is "" - #{path := []} -> - ok; - #{path := <<>>} -> - ok; _ -> {error, "Invalid url"} end. diff --git a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl index 311233f1b..c50813938 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl @@ -53,6 +53,7 @@ }, <<"push_gateway">> => #{ + <<"enable">> => true, <<"headers">> => #{<<"Authorization">> => <<"some-authz-tokens">>}, <<"interval">> => <<"1s">>, <<"job_name">> => <<"${name}~${host}">>, diff --git a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl index dc56841ad..ad2b3d5ec 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl @@ -180,11 +180,11 @@ t_prometheus_api(_) -> ), #{ <<"push_gateway">> := - #{<<"url">> := Url} = PushGateway, + #{<<"url">> := Url, <<"enable">> := Enable} = PushGateway, <<"collectors">> := Collector } = Conf, Pid = erlang:whereis(emqx_prometheus), - ?assertEqual(Url =/= "", undefined =/= Pid, {Url, Pid}), + ?assertEqual(Enable, undefined =/= Pid, {Url, Pid}), NewConf = Conf#{ <<"push_gateway">> => PushGateway#{ @@ -250,7 +250,7 @@ t_prometheus_api(_) -> emqx_config:get([prometheus]) ), - NewConf1 = Conf#{<<"push_gateway">> => PushGateway#{<<"url">> => <<"">>}}, + NewConf1 = Conf#{<<"push_gateway">> => PushGateway#{<<"enable">> => false}}, {ok, _Response3} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf1), ?assertEqual(undefined, erlang:whereis(emqx_prometheus)), diff --git a/rel/config/examples/prometheus.conf.example b/rel/config/examples/prometheus.conf.example index 3b186e93f..049b11ee3 100644 --- a/rel/config/examples/prometheus.conf.example +++ b/rel/config/examples/prometheus.conf.example @@ -1,10 +1,24 @@ ## Prometheus ## EMQX's Prometheus scraping endpoint is enabled by default without authentication. -## And there is no way to turn it off. +## You can enable basic authentication by setting enable_basic_auth to true. ## You can inspect it with a curl command: curl -f "127.0.0.1:18083/api/v5/prometheus/stats" prometheus { - # turn off this expensive collector - vm_dist_collector = disabled + enable_basic_auth = false + push_gateway { + enable = false + url = "http://127.0.0.1:9091" + headers {Authorization = "Basic YWRtaW46Y2JraG55eWd5QDE="} + interval = 15s + job_name = "${name}/instance/${name}~${host}" + } + collectors { + mnesia = disabled + vm_dist = disabled + vm_memory = disabled + vm_msacc = disabled + vm_statistics = disabled + vm_system_info = enabled + } } diff --git a/rel/i18n/emqx_prometheus_schema.hocon b/rel/i18n/emqx_prometheus_schema.hocon index 57d6b8065..ef507b520 100644 --- a/rel/i18n/emqx_prometheus_schema.hocon +++ b/rel/i18n/emqx_prometheus_schema.hocon @@ -37,8 +37,9 @@ recommend_setting.desc: """Recommended setting""" push_gateway_url.desc: -"""URL of Pushgateway server. Pushgateway is optional, should not be configured if prometheus is to scrape EMQX. -Set url to "" to disable push gateway""" +"""URL of Pushgateway server. Pushgateway is optional, should not be configured if prometheus is to scrape EMQX.""" +push_gateway_enable.desc: +"""Enable or disable Pushgateway""" mnesia_collector.desc: """Collects Mnesia metrics mainly using mnesia:system_info/1 """ @@ -61,42 +62,42 @@ vm_system_info_collector.desc: """Enable or disable VM system info collector.""" legacy_deprecated_setting.desc: -"""Deprecated""" +"""Deprecated since 5.4.0""" legacy_enable.desc: -"""Deprecated, use `prometheus.push_gateway.url` instead""" +"""Deprecated since 5.4.0, use `prometheus.push_gateway.url` instead""" legacy_headers.desc: -"""Deprecated, use `prometheus.push_gateway.headers` instead""" +"""Deprecated since 5.4.0, use `prometheus.push_gateway.headers` instead""" legacy_interval.desc: -"""Deprecated, use `prometheus.push_gateway.interval` instead""" +"""Deprecated since 5.4.0, use `prometheus.push_gateway.interval` instead""" legacy_job_name.desc: -"""Deprecated, use `prometheus.push_gateway.job_name` instead""" +"""Deprecated since 5.4.0, use `prometheus.push_gateway.job_name` instead""" legacy_push_gateway_server.desc: -"""Deprecated, use `prometheus.push_gateway.url` instead""" +"""Deprecated since 5.4.0, use `prometheus.push_gateway.url` instead""" legacy_mnesia_collector.desc: -"""Deprecated, use `prometheus.collectors.mnesia` instead""" +"""Deprecated since 5.4.0, use `prometheus.collectors.mnesia` instead""" legacy_vm_dist_collector.desc: -"""Deprecated, use `prometheus.collectors.vm_dist` instead""" +"""Deprecated since 5.4.0, use `prometheus.collectors.vm_dist` instead""" legacy_vm_memory_collector.desc: -"""Deprecated, use `prometheus.collectors.vm_memory` instead""" +"""Deprecated since 5.4.0, use `prometheus.collectors.vm_memory` instead""" legacy_vm_msacc_collector.desc: -"""Deprecated, use `prometheus.collectors.vm_msacc` instead""" +"""Deprecated since 5.4.0, use `prometheus.collectors.vm_msacc` instead""" legacy_vm_statistics_collector.desc: -"""Deprecated, use `prometheus.collectors.vm_statistics` instead""" +"""Deprecated since 5.4.0, use `prometheus.collectors.vm_statistics` instead""" legacy_vm_system_info_collector.desc: """Deprecated, use `prometheus.collectors.vm_system_info` instead""" legacy_deprecated_setting.desc: -"""Deprecated""" +"""Deprecated since 5.4.0""" } From d7974d835faf63f7531bc84be4191a940c4c4b5e Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Nov 2023 15:49:01 +0700 Subject: [PATCH 13/20] fix(sharesub): anticipate messages w/o redispatch header For instance, `emqx_session_mem` will push almost all of the messages that are still in the state to `emqx_shared_sub:redispatch/1`. After this commit, the session will no longer crash during channel terminate. --- apps/emqx/src/emqx_shared_sub.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 89a785590..0a6538282 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -242,7 +242,9 @@ with_redispatch_to(Msg, Group, Topic) -> is_redispatch_needed(#message{qos = ?QOS_0}) -> false; is_redispatch_needed(#message{headers = #{redispatch_to := ?REDISPATCH_TO(_, _)}}) -> - true. + true; +is_redispatch_needed(#message{}) -> + false. %% @doc Redispatch shared deliveries to other members in the group. redispatch(Messages0) -> From 9716b051d9be5f3bb083998ab112d33493d42141 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 21 Nov 2023 11:24:23 +0800 Subject: [PATCH 14/20] fix(ocpp): fix bad webscoket schema for HTTP API --- apps/emqx_gateway/src/emqx_gateway_schema.erl | 12 ++++++++++-- apps/emqx_gateway_ocpp/src/emqx_ocpp_conf.erl | 2 +- apps/emqx_gateway_ocpp/src/emqx_ocpp_schemas.erl | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index c84cf086b..c0abb48ce 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -127,9 +127,15 @@ fields(ssl_listener) -> )} ]; fields(ws_listener) -> - ws_listener() ++ ws_opts(<<>>, <<>>); + emqx_gateway_schema:ws_listener() ++ + [{websocket, sc(ref(websocket), #{})}]; fields(wss_listener) -> - wss_listener() ++ ws_opts(<<>>, <<>>); + emqx_gateway_schema:wss_listener() ++ + [{websocket, sc(ref(websocket), #{})}]; +fields(websocket) -> + DefaultPath = <<>>, + SubProtocols = <<>>, + emqx_gateway_schema:ws_opts(DefaultPath, SubProtocols); fields(udp_listener) -> [ %% some special configs for udp listener @@ -193,6 +199,8 @@ desc(udp_opts) -> "Settings for UDP sockets."; desc(dtls_opts) -> "Settings for DTLS protocol."; +desc(websocket) -> + "Websocket options"; desc(_) -> undefined. diff --git a/apps/emqx_gateway_ocpp/src/emqx_ocpp_conf.erl b/apps/emqx_gateway_ocpp/src/emqx_ocpp_conf.erl index 1151e1dbb..00d1aca6c 100644 --- a/apps/emqx_gateway_ocpp/src/emqx_ocpp_conf.erl +++ b/apps/emqx_gateway_ocpp/src/emqx_ocpp_conf.erl @@ -63,7 +63,7 @@ max_mqueue_len() -> | dnstream_only | disable. message_format_checking() -> - conf(message_format_checking, all). + conf(message_format_checking, disable). uptopic(Action) -> Topic = upstream(topic), diff --git a/apps/emqx_gateway_ocpp/src/emqx_ocpp_schemas.erl b/apps/emqx_gateway_ocpp/src/emqx_ocpp_schemas.erl index e2bd00d0e..99003763f 100644 --- a/apps/emqx_gateway_ocpp/src/emqx_ocpp_schemas.erl +++ b/apps/emqx_gateway_ocpp/src/emqx_ocpp_schemas.erl @@ -32,7 +32,7 @@ load() -> disable -> ok; _ -> - case feedvar(emqx_config:get([gateway, ocpp, json_schema_dir])) of + case feedvar(emqx_config:get([gateway, ocpp, json_schema_dir], undefined)) of undefined -> ok; Dir -> From d9f95cdc56978151f31ba810a55dbb8746a8f195 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Thu, 9 Nov 2023 20:48:50 +0200 Subject: [PATCH 15/20] feat: integrate OpenTelemetry log handler --- apps/emqx_conf/src/emqx_conf_schema.erl | 3 +- apps/emqx_machine/src/emqx_machine_boot.erl | 4 +- .../src/emqx_opentelemetry.app.src | 10 +- apps/emqx_opentelemetry/src/emqx_otel_api.erl | 22 ++- apps/emqx_opentelemetry/src/emqx_otel_app.erl | 6 + .../src/emqx_otel_config.erl | 114 +++++++++++- .../{emqx_otel.erl => emqx_otel_metrics.erl} | 50 +++--- .../src/emqx_otel_schema.erl | 163 +++++++++++++++--- apps/emqx_opentelemetry/src/emqx_otel_sup.erl | 4 +- changes/ce/feat-11921.en.md | 2 + mix.exs | 10 +- rebar.config | 10 +- rel/i18n/emqx_otel_schema.hocon | 37 +++- 13 files changed, 363 insertions(+), 72 deletions(-) rename apps/emqx_opentelemetry/src/{emqx_otel.erl => emqx_otel_metrics.erl} (88%) create mode 100644 changes/ce/feat-11921.en.md diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 45219c3f6..3a278ca9b 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -77,7 +77,8 @@ %% Callback to upgrade config after loaded from config file but before validation. upgrade_raw_conf(RawConf) -> - emqx_connector_schema:transform_bridges_v1_to_connectors_and_bridges_v2(RawConf). + RawConf1 = emqx_connector_schema:transform_bridges_v1_to_connectors_and_bridges_v2(RawConf), + emqx_otel_schema:upgrade_legacy_metrics(RawConf1). %% root config should not have a namespace namespace() -> undefined. diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index 026a82cbf..7abac0862 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -69,9 +69,9 @@ stop_apps() -> ?SLOG(notice, #{msg => "stopping_emqx_apps"}), _ = emqx_alarm_handler:unload(), ok = emqx_conf_app:unset_config_loaded(), + lists:foreach(fun stop_one_app/1, lists:reverse(sorted_reboot_apps())), %% Mute otel deps application. - _ = emqx_otel:stop_otel(), - lists:foreach(fun stop_one_app/1, lists:reverse(sorted_reboot_apps())). + ok = emqx_otel_app:stop_deps(). %% Those port apps are terminated after the main apps %% Don't need to stop when reboot. diff --git a/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src b/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src index 3b429ea08..134ef2a44 100644 --- a/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src +++ b/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src @@ -1,9 +1,15 @@ {application, emqx_opentelemetry, [ {description, "OpenTelemetry for EMQX Broker"}, - {vsn, "0.1.3"}, + {vsn, "0.2.0"}, {registered, []}, {mod, {emqx_otel_app, []}}, - {applications, [kernel, stdlib, emqx]}, + {applications, [ + kernel, + stdlib, + emqx, + %% otel metrics depend on emqx_mgmt_cache + emqx_management + ]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_opentelemetry/src/emqx_otel_api.erl b/apps/emqx_opentelemetry/src/emqx_otel_api.erl index 7478859f8..d8c76ebcf 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_api.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_api.erl @@ -103,10 +103,24 @@ otel_config_schema() -> otel_config_example() -> #{ - enable => true, - exporter => - #{ + logs => #{ + enable => true, + exporter => #{ endpoint => "http://localhost:4317", - interval => "10s" + ssl_options => #{ + enable => false + } + }, + level => warning + }, + metrics => #{ + enable => true, + exporter => #{ + endpoint => "http://localhost:4317", + interval => "10s", + ssl_options => #{ + enable => false + } } + } }. diff --git a/apps/emqx_opentelemetry/src/emqx_otel_app.erl b/apps/emqx_opentelemetry/src/emqx_otel_app.erl index f028a000a..cf93d7753 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_app.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_app.erl @@ -19,11 +19,17 @@ -behaviour(application). -export([start/2, stop/1]). +-export([stop_deps/0]). start(_StartType, _StartArgs) -> emqx_otel_config:add_handler(), + ok = emqx_otel_config:add_otel_log_handler(), emqx_otel_sup:start_link(). stop(_State) -> emqx_otel_config:remove_handler(), + _ = emqx_otel_config:remove_otel_log_handler(), ok. + +stop_deps() -> + emqx_otel_config:stop_all_otel_apps(). diff --git a/apps/emqx_opentelemetry/src/emqx_otel_config.erl b/apps/emqx_opentelemetry/src/emqx_otel_config.erl index 45a38e134..11e97dcdd 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_config.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_config.erl @@ -19,9 +19,16 @@ -define(OPTL, [opentelemetry]). +-define(OTEL_EXPORTER, opentelemetry_exporter). +-define(OTEL_LOG_HANDLER, otel_log_handler). +-define(OTEL_LOG_HANDLER_ID, opentelemetry_handler). + -export([add_handler/0, remove_handler/0]). -export([post_config_update/5]). -export([update/1]). +-export([add_otel_log_handler/0, remove_otel_log_handler/0]). +-export([stop_all_otel_apps/0]). +-export([otel_exporter/1]). update(Config) -> case @@ -45,14 +52,109 @@ remove_handler() -> ok = emqx_config_handler:remove_handler(?OPTL), ok. +post_config_update(?OPTL, _Req, Old, Old, _AppEnvs) -> + ok; post_config_update(?OPTL, _Req, New, _Old, AppEnvs) -> application:set_env(AppEnvs), - ensure_otel(New); + MetricsRes = ensure_otel_metrics(New), + LogsRes = ensure_otel_logs(New), + _ = maybe_stop_all_otel_apps(New), + case {MetricsRes, LogsRes} of + {ok, ok} -> ok; + Other -> {error, Other} + end; post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) -> ok. -ensure_otel(#{enable := true} = Conf) -> - _ = emqx_otel:stop_otel(), - emqx_otel:start_otel(Conf); -ensure_otel(#{enable := false}) -> - emqx_otel:stop_otel(). +stop_all_otel_apps() -> + _ = application:stop(opentelemetry), + _ = application:stop(opentelemetry_experimental), + _ = application:stop(opentelemetry_experimental_api), + _ = application:stop(opentelemetry_exporter), + ok. + +add_otel_log_handler() -> + ensure_otel_logs(emqx:get_config(?OPTL)). + +remove_otel_log_handler() -> + remove_handler_if_present(?OTEL_LOG_HANDLER_ID). + +otel_exporter(ExporterConf) -> + #{ + endpoint := Endpoint, + protocol := Proto, + ssl_options := SSLOpts + } = ExporterConf, + {?OTEL_EXPORTER, #{ + endpoint => Endpoint, + protocol => Proto, + ssl_options => ssl_opts(Endpoint, SSLOpts) + }}. + +%% Internal functions + +ensure_otel_metrics(#{metrics := #{enable := true} = MetricsConf}) -> + _ = emqx_otel_metrics:stop_otel(), + emqx_otel_metrics:start_otel(MetricsConf); +ensure_otel_metrics(#{metrics := #{enable := false}}) -> + emqx_otel_metrics:stop_otel(); +ensure_otel_metrics(_) -> + ok. + +ensure_otel_logs(#{logs := #{enable := true} = LogsConf}) -> + ok = remove_handler_if_present(?OTEL_LOG_HANDLER_ID), + ok = ensure_log_apps(), + HandlerConf = tr_handler_conf(LogsConf), + %% NOTE: should primary logger level be updated if it's higher than otel log level? + logger:add_handler(?OTEL_LOG_HANDLER_ID, ?OTEL_LOG_HANDLER, HandlerConf); +ensure_otel_logs(#{logs := #{enable := false}}) -> + remove_handler_if_present(?OTEL_LOG_HANDLER_ID). + +remove_handler_if_present(HandlerId) -> + case logger:get_handler_config(HandlerId) of + {ok, _} -> + ok = logger:remove_handler(HandlerId); + _ -> + ok + end. + +ensure_log_apps() -> + {ok, _} = application:ensure_all_started(opentelemetry_exporter), + {ok, _} = application:ensure_all_started(opentelemetry_experimental), + ok. + +maybe_stop_all_otel_apps(#{metrics := #{enable := false}, logs := #{enable := false}}) -> + stop_all_otel_apps(); +maybe_stop_all_otel_apps(_) -> + ok. + +tr_handler_conf(Conf) -> + #{ + level := Level, + max_queue_size := MaxQueueSize, + exporting_timeout := ExportingTimeout, + scheduled_delay := ScheduledDelay, + exporter := ExporterConf + } = Conf, + #{ + level => Level, + config => #{ + max_queue_size => MaxQueueSize, + exporting_timeout_ms => ExportingTimeout, + scheduled_delay_ms => ScheduledDelay, + exporter => otel_exporter(ExporterConf) + } + }. + +ssl_opts(Endpoint, SSLOpts) -> + case is_ssl(Endpoint) of + true -> + emqx_tls_lib:to_client_opts(SSLOpts#{enable => true}); + false -> + [] + end. + +is_ssl(<<"https://", _/binary>> = _Endpoint) -> + true; +is_ssl(_Endpoint) -> + false. diff --git a/apps/emqx_opentelemetry/src/emqx_otel.erl b/apps/emqx_opentelemetry/src/emqx_otel_metrics.erl similarity index 88% rename from apps/emqx_opentelemetry/src/emqx_otel.erl rename to apps/emqx_opentelemetry/src/emqx_otel_metrics.erl index aa0c7d2d4..9ca1c5deb 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_metrics.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_otel). +-module(emqx_otel_metrics). -include_lib("emqx/include/logger.hrl"). -export([start_otel/1, stop_otel/0]). @@ -29,17 +29,19 @@ start_otel(Conf) -> assert_started(supervisor:start_child(?SUPERVISOR, Spec)). stop_otel() -> + Res = + case erlang:whereis(?SUPERVISOR) of + undefined -> + ok; + Pid -> + case supervisor:terminate_child(Pid, ?MODULE) of + ok -> supervisor:delete_child(Pid, ?MODULE); + {error, not_found} -> ok; + Error -> Error + end + end, ok = cleanup(), - case erlang:whereis(?SUPERVISOR) of - undefined -> - ok; - Pid -> - case supervisor:terminate_child(Pid, ?MODULE) of - ok -> supervisor:delete_child(Pid, ?MODULE); - {error, not_found} -> ok; - Error -> Error - end - end. + Res. start_link(Conf) -> gen_server:start_link({local, ?MODULE}, ?MODULE, Conf, []). @@ -71,32 +73,40 @@ setup(_Conf) -> ok. ensure_apps(Conf) -> - #{exporter := #{interval := ExporterInterval}} = Conf, + #{exporter := #{interval := ExporterInterval} = Exporter} = Conf, {ok, _} = application:ensure_all_started(opentelemetry_exporter), {ok, _} = application:ensure_all_started(opentelemetry), - _ = application:stop(opentelemetry_experimental), + {ok, _} = application:ensure_all_started(opentelemetry_experimental), + {ok, _} = application:ensure_all_started(opentelemetry_api_experimental), + + _ = opentelemetry_experimental:stop_default_metrics(), ok = application:set_env( opentelemetry_experimental, readers, [ #{ + id => emqx_otel_metric_reader, module => otel_metric_reader, config => #{ - exporter => {opentelemetry_exporter, #{}}, + exporter => emqx_otel_config:otel_exporter(Exporter), export_interval_ms => ExporterInterval } } ] ), - {ok, _} = application:ensure_all_started(opentelemetry_experimental), - {ok, _} = application:ensure_all_started(opentelemetry_api_experimental), + {ok, _} = opentelemetry_experimental:start_default_metrics(), ok. cleanup() -> - _ = application:stop(opentelemetry), - _ = application:stop(opentelemetry_experimental), - _ = application:stop(opentelemetry_experimental_api), - _ = application:stop(opentelemetry_exporter), + safe_stop_default_metrics(). + +safe_stop_default_metrics() -> + try + _ = opentelemetry_experimental:stop_default_metrics() + catch + %% noramal scenario, metrics supervisor is not started + exit:{noproc, _} -> ok + end, ok. create_metric_views() -> diff --git a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl index 7fae5071e..927bc9dfd 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl @@ -24,16 +24,48 @@ desc/1 ]). +-export([upgrade_legacy_metrics/1]). + +%% Compatibility with the previous schema that defined only metric fields +upgrade_legacy_metrics(RawConf) -> + case RawConf of + #{<<"opentelemetry">> := Otel} -> + LegacyMetricsFields = [<<"enable">>, <<"exporter">>], + Otel1 = maps:without(LegacyMetricsFields, Otel), + Metrics = maps:with(LegacyMetricsFields, Otel), + case Metrics =:= #{} of + true -> + RawConf; + false -> + RawConf#{<<"opentelemetry">> => Otel1#{<<"metrics">> => Metrics}} + end; + _ -> + RawConf + end. + namespace() -> opentelemetry. + roots() -> ["opentelemetry"]. fields("opentelemetry") -> [ - {exporter, + {metrics, ?HOCON( - ?R_REF("exporter"), - #{desc => ?DESC(exporter)} + ?R_REF("otel_metrics"), + #{ + desc => ?DESC(otel_metrics) + } )}, + {logs, + ?HOCON( + ?R_REF("otel_logs"), + #{ + desc => ?DESC(otel_logs) + } + )} + ]; +fields("otel_metrics") -> + [ {enable, ?HOCON( boolean(), @@ -42,41 +74,130 @@ fields("opentelemetry") -> required => true, desc => ?DESC(enable) } + )}, + {exporter, + ?HOCON( + ?R_REF("otel_metrics_exporter"), + #{desc => ?DESC(exporter)} )} ]; -fields("exporter") -> +fields("otel_logs") -> [ - {"protocol", + {level, ?HOCON( - %% http_protobuf is not support for metrics yet. - ?ENUM([grpc]), + emqx_conf_schema:log_level(), #{ - mapping => "opentelemetry_exporter.otlp_protocol", - desc => ?DESC(protocol), - default => grpc, + default => warning, + desc => ?DESC(otel_log_handler_level), + importance => ?IMPORTANCE_HIGH + } + )}, + {enable, + ?HOCON( + boolean(), + #{ + default => false, + desc => ?DESC(enable), + importance => ?IMPORTANCE_HIGH + } + )}, + {max_queue_size, + ?HOCON( + pos_integer(), + #{ + default => 2048, + desc => ?DESC(max_queue_size), importance => ?IMPORTANCE_HIDDEN } )}, - {"endpoint", + {exporting_timeout, + ?HOCON( + emqx_schema:timeout_duration_ms(), + #{ + default => <<"30s">>, + desc => ?DESC(exporting_timeout), + importance => ?IMPORTANCE_HIDDEN + } + )}, + {scheduled_delay, + ?HOCON( + emqx_schema:timeout_duration_ms(), + #{ + default => <<"1s">>, + desc => ?DESC(scheduled_delay), + importance => ?IMPORTANCE_HIDDEN + } + )}, + {exporter, + ?HOCON( + ?R_REF("otel_logs_exporter"), + #{ + desc => ?DESC(exporter), + importance => ?IMPORTANCE_HIGH + } + )} + ]; +fields("otel_metrics_exporter") -> + exporter_fields(metrics); +fields("otel_logs_exporter") -> + exporter_fields(logs); +fields("ssl_opts") -> + Schema = emqx_schema:client_ssl_opts_schema(#{}), + lists:keydelete("enable", 1, Schema). + +desc("opentelemetry") -> ?DESC(opentelemetry); +desc("exporter") -> ?DESC(exporter); +desc("otel_logs_exporter") -> ?DESC(exporter); +desc("otel_metrics_exporter") -> ?DESC(exporter); +desc("otel_logs") -> ?DESC(otel_logs); +desc("otel_metrics") -> ?DESC(otel_metrics); +desc("ssl_opts") -> ?DESC(exporter_ssl); +desc(_) -> undefined. + +exporter_fields(OtelSignal) -> + [ + {endpoint, ?HOCON( emqx_schema:url(), #{ - mapping => "opentelemetry_exporter.otlp_endpoint", - default => <<"http://localhost:4317">>, - desc => ?DESC(endpoint) + default => "http://localhost:4317", + desc => ?DESC(exporter_endpoint), + importance => ?IMPORTANCE_HIGH } )}, - {"interval", + {protocol, + ?HOCON( + %% http protobuf/json may be added in future + ?ENUM([grpc]), + #{ + default => grpc, + desc => ?DESC(exporter_protocol), + importance => ?IMPORTANCE_HIDDEN + } + )}, + {ssl_options, + ?HOCON( + ?R_REF("ssl_opts"), + #{ + desc => ?DESC(exporter_ssl), + importance => ?IMPORTANCE_LOW + } + )} + ] ++ exporter_extra_fields(OtelSignal). + +%% Let's keep it in exporter config for metrics, as it is different from +%% scheduled_delay_ms opt used for otel traces and logs +exporter_extra_fields(metrics) -> + [ + {interval, ?HOCON( emqx_schema:timeout_duration_ms(), #{ default => <<"10s">>, required => true, - desc => ?DESC(interval) + desc => ?DESC(scheduled_delay) } )} - ]. - -desc("opentelemetry") -> ?DESC(opentelemetry); -desc("exporter") -> ?DESC(exporter); -desc(_) -> undefined. + ]; +exporter_extra_fields(_OtelSignal) -> + []. diff --git a/apps/emqx_opentelemetry/src/emqx_otel_sup.erl b/apps/emqx_opentelemetry/src/emqx_otel_sup.erl index 19f22cb84..a823b2239 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_sup.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_sup.erl @@ -41,8 +41,8 @@ init([]) -> period => 512 }, Children = - case emqx_conf:get([opentelemetry]) of + case emqx_conf:get([opentelemetry, metrics]) of #{enable := false} -> []; - #{enable := true} = Conf -> [worker_spec(emqx_otel, Conf)] + #{enable := true} = Conf -> [worker_spec(emqx_otel_metrics, Conf)] end, {ok, {SupFlags, Children}}. diff --git a/changes/ce/feat-11921.en.md b/changes/ce/feat-11921.en.md new file mode 100644 index 000000000..c226c1e5b --- /dev/null +++ b/changes/ce/feat-11921.en.md @@ -0,0 +1,2 @@ +Introduced Open Telemetry Logs Handler that allows to format log events according to Open Telemetry log data model and +export them to the configured Open Telemetry collector or back-end. diff --git a/mix.exs b/mix.exs index d931b799d..91ebf7f9b 100644 --- a/mix.exs +++ b/mix.exs @@ -102,31 +102,31 @@ defmodule EMQXUmbrella.MixProject do {:opentelemetry_api, github: "emqx/opentelemetry-erlang", sparse: "apps/opentelemetry_api", - tag: "v1.3.2-emqx", + tag: "v1.4.2-emqx", override: true, runtime: false}, {:opentelemetry, github: "emqx/opentelemetry-erlang", sparse: "apps/opentelemetry", - tag: "v1.3.2-emqx", + tag: "v1.4.2-emqx", override: true, runtime: false}, {:opentelemetry_api_experimental, github: "emqx/opentelemetry-erlang", sparse: "apps/opentelemetry_api_experimental", - tag: "v1.3.2-emqx", + tag: "v1.4.2-emqx", override: true, runtime: false}, {:opentelemetry_experimental, github: "emqx/opentelemetry-erlang", sparse: "apps/opentelemetry_experimental", - tag: "v1.3.2-emqx", + tag: "v1.4.2-emqx", override: true, runtime: false}, {:opentelemetry_exporter, github: "emqx/opentelemetry-erlang", sparse: "apps/opentelemetry_exporter", - tag: "v1.3.2-emqx", + tag: "v1.4.2-emqx", override: true, runtime: false} ] ++ diff --git a/rebar.config b/rebar.config index f4273f6fb..3a887e941 100644 --- a/rebar.config +++ b/rebar.config @@ -85,13 +85,13 @@ , {jsone, {git, "https://github.com/emqx/jsone.git", {tag, "1.7.1"}}} , {uuid, {git, "https://github.com/okeuday/uuid.git", {tag, "v2.0.6"}}} %% trace - , {opentelemetry_api, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.3.2-emqx"}, "apps/opentelemetry_api"}} - , {opentelemetry, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.3.2-emqx"}, "apps/opentelemetry"}} + , {opentelemetry_api, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.2-emqx"}, "apps/opentelemetry_api"}} + , {opentelemetry, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.2-emqx"}, "apps/opentelemetry"}} %% log metrics - , {opentelemetry_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.3.2-emqx"}, "apps/opentelemetry_experimental"}} - , {opentelemetry_api_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.3.2-emqx"}, "apps/opentelemetry_api_experimental"}} + , {opentelemetry_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.2-emqx"}, "apps/opentelemetry_experimental"}} + , {opentelemetry_api_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.2-emqx"}, "apps/opentelemetry_api_experimental"}} %% export - , {opentelemetry_exporter, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.3.2-emqx"}, "apps/opentelemetry_exporter"}} + , {opentelemetry_exporter, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.2-emqx"}, "apps/opentelemetry_exporter"}} ]}. {xref_ignores, diff --git a/rel/i18n/emqx_otel_schema.hocon b/rel/i18n/emqx_otel_schema.hocon index f662598b9..9e59b2a76 100644 --- a/rel/i18n/emqx_otel_schema.hocon +++ b/rel/i18n/emqx_otel_schema.hocon @@ -1,15 +1,44 @@ emqx_otel_schema { opentelemetry.desc: "Open Telemetry Toolkit configuration" +opentelemetry.label: "Open Telemetry" + +otel_logs.desc: +"""Open Telemetry Logs configuration. If enabled, EMQX installs a log handler that formats events according to Open Telemetry log data model and +exports them to the configured Open Telemetry collector or backend.""" +otel_logs.label: "Open Telemetry Logs" + +otel_metrics.desc: "Open Telemetry Metrics configuration." +otel_metrics.label: "Open Telemetry Metrics" + +enable.desc: "Enable or disable Open Telemetry signal." +enable.label: "Enable." exporter.desc: "Open Telemetry Exporter" +exporter.label: "Exporter" -enable.desc: "Enable or disable open telemetry metrics" +max_queue_size.desc: +"""The maximum queue size. After the size is reached Open Telemetry signals are dropped.""" +max_queue_size.label: "Max Queue Size" -protocol.desc: "Open Telemetry Exporter Protocol" +exporting_timeout.desc: "The time Open Telemetry signal export can run before it is cancelled." +exporting_timeout.label: "Exporting Timeout" -endpoint.desc: "Open Telemetry Exporter Endpoint" +scheduled_delay.desc: "The delay interval between two consecutive exports of Open Telemetry signals." +scheduled_delay.label: "Scheduled Delay Interval" -interval.desc: "The interval of sending metrics to Open Telemetry Endpoint" +exporter_endpoint.desc: +"""The target URL to which the exporter is going to send Open Telemetry signal data.""" +exporter_endpoint.label: "Exporter Endpoint" + +exporter_protocol.desc: "The transport protocol of Open Telemetry Exporter" +exporter_protocol.label: "Exporter Protocol" + +exporter_ssl.desc: "SSL configuration for the Open Telemetry exporter" +exporter_ssl.label: "SSL Options" + +otel_log_handler_level.desc: +"""The log level of the Open Telemetry log handler.""" +otel_log_handler_level.label: "Log Level" } From 6acdc6e432fbc85169e17a3ee5ef389de9f05f08 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 27 Nov 2023 11:32:23 -0300 Subject: [PATCH 16/20] feat: add `/actions/:id/metrics/`, `/actions/:id/metrics/reset` APIs Fixes https://emqx.atlassian.net/browse/EMQX-11381 --- apps/emqx_bridge/src/emqx_bridge_v2_api.erl | 186 ++++++++++++++++- .../src/proto/emqx_bridge_proto_v5.erl | 14 +- .../test/emqx_bridge_v2_api_SUITE.erl | 192 +++++++++++++----- 3 files changed, 341 insertions(+), 51 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl index cb1f7cc62..13e84f84e 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl @@ -21,6 +21,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_utils/include/emqx_utils_api.hrl"). +-include_lib("emqx_bridge/include/emqx_bridge.hrl"). -import(hoconsc, [mk/2, array/1, enum/1]). -import(emqx_utils, [redact/1]). @@ -37,6 +38,8 @@ -export([ '/actions'/2, '/actions/:id'/2, + '/actions/:id/metrics'/2, + '/actions/:id/metrics/reset'/2, '/actions/:id/enable/:enable'/2, '/actions/:id/:operation'/2, '/nodes/:node/actions/:id/:operation'/2, @@ -44,8 +47,8 @@ '/action_types'/2 ]). -%% BpAPI --export([lookup_from_local_node/2]). +%% BpAPI / RPC Targets +-export([lookup_from_local_node/2, get_metrics_from_local_node/2]). -define(BRIDGE_NOT_FOUND(BRIDGE_TYPE, BRIDGE_NAME), ?NOT_FOUND( @@ -80,6 +83,10 @@ paths() -> "/actions/:id/enable/:enable", "/actions/:id/:operation", "/nodes/:node/actions/:id/:operation", + %% Caveat: metrics paths must come *after* `/:operation', otherwise minirest will + %% try to match the latter first, trying to interpret `metrics' as an operation... + "/actions/:id/metrics", + "/actions/:id/metrics/reset", "/actions_probe", "/action_types" ]. @@ -247,6 +254,34 @@ schema("/actions/:id") -> } } }; +schema("/actions/:id/metrics") -> + #{ + 'operationId' => '/actions/:id/metrics', + get => #{ + tags => [<<"actions">>], + summary => <<"Get action metrics">>, + description => ?DESC("desc_bridge_metrics"), + parameters => [param_path_id()], + responses => #{ + 200 => emqx_bridge_schema:metrics_fields(), + 404 => error_schema('NOT_FOUND', "Action not found") + } + } + }; +schema("/actions/:id/metrics/reset") -> + #{ + 'operationId' => '/actions/:id/metrics/reset', + put => #{ + tags => [<<"actions">>], + summary => <<"Reset action metrics">>, + description => ?DESC("desc_api6"), + parameters => [param_path_id()], + responses => #{ + 204 => <<"Reset success">>, + 404 => error_schema('NOT_FOUND', "Action not found") + } + } + }; schema("/actions/:id/enable/:enable") -> #{ 'operationId' => '/actions/:id/enable/:enable', @@ -429,6 +464,19 @@ schema("/action_types") -> end ). +'/actions/:id/metrics'(get, #{bindings := #{id := Id}}) -> + ?TRY_PARSE_ID(Id, get_metrics_from_all_nodes(BridgeType, BridgeName)). + +'/actions/:id/metrics/reset'(put, #{bindings := #{id := Id}}) -> + ?TRY_PARSE_ID( + Id, + begin + ActionType = emqx_bridge_v2:bridge_v2_type_to_connector_type(BridgeType), + ok = emqx_bridge_v2:reset_metrics(ActionType, BridgeName), + ?NO_CONTENT + end + ). + '/actions/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) -> ?TRY_PARSE_ID( Id, @@ -570,6 +618,18 @@ lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) -> ?INTERNAL_ERROR(Reason) end. +get_metrics_from_all_nodes(ActionType, ActionName) -> + Nodes = emqx:running_nodes(), + Result = maybe_unwrap( + emqx_bridge_proto_v5:v2_get_metrics_from_all_nodes(Nodes, ActionType, ActionName) + ), + case Result of + Metrics when is_list(Metrics) -> + {200, format_bridge_metrics(lists:zip(Nodes, Metrics))}; + {error, Reason} -> + ?INTERNAL_ERROR(Reason) + end. + operation_func(all, start) -> v2_start_bridge_to_all_nodes; operation_func(_Node, start) -> v2_start_bridge_to_node. @@ -720,12 +780,17 @@ aggregate_status(AllStatus) -> false -> inconsistent end. +%% RPC Target lookup_from_local_node(BridgeType, BridgeName) -> case emqx_bridge_v2:lookup(BridgeType, BridgeName) of {ok, Res} -> {ok, format_resource(Res, node())}; Error -> Error end. +%% RPC Target +get_metrics_from_local_node(ActionType, ActionName) -> + format_metrics(emqx_bridge_v2:get_metrics(ActionType, ActionName)). + %% resource format_resource( #{ @@ -751,6 +816,123 @@ format_resource( ) ). +format_metrics(#{ + counters := #{ + 'dropped' := Dropped, + 'dropped.other' := DroppedOther, + 'dropped.expired' := DroppedExpired, + 'dropped.queue_full' := DroppedQueueFull, + 'dropped.resource_not_found' := DroppedResourceNotFound, + 'dropped.resource_stopped' := DroppedResourceStopped, + 'matched' := Matched, + 'retried' := Retried, + 'late_reply' := LateReply, + 'failed' := SentFailed, + 'success' := SentSucc, + 'received' := Rcvd + }, + gauges := Gauges, + rate := #{ + matched := #{current := Rate, last5m := Rate5m, max := RateMax} + } +}) -> + Queued = maps:get('queuing', Gauges, 0), + SentInflight = maps:get('inflight', Gauges, 0), + ?METRICS( + Dropped, + DroppedOther, + DroppedExpired, + DroppedQueueFull, + DroppedResourceNotFound, + DroppedResourceStopped, + Matched, + Queued, + Retried, + LateReply, + SentFailed, + SentInflight, + SentSucc, + Rate, + Rate5m, + RateMax, + Rcvd + ); +format_metrics(_Metrics) -> + %% Empty metrics: can happen when a node joins another and a + %% bridge is not yet replicated to it, so the counters map is + %% empty. + empty_metrics(). + +empty_metrics() -> + ?METRICS( + _Dropped = 0, + _DroppedOther = 0, + _DroppedExpired = 0, + _DroppedQueueFull = 0, + _DroppedResourceNotFound = 0, + _DroppedResourceStopped = 0, + _Matched = 0, + _Queued = 0, + _Retried = 0, + _LateReply = 0, + _SentFailed = 0, + _SentInflight = 0, + _SentSucc = 0, + _Rate = 0, + _Rate5m = 0, + _RateMax = 0, + _Rcvd = 0 + ). + +format_bridge_metrics(Bridges) -> + NodeMetrics = lists:filtermap( + fun + ({Node, Metrics}) when is_map(Metrics) -> + {true, #{node => Node, metrics => Metrics}}; + ({Node, _}) -> + {true, #{node => Node, metrics => empty_metrics()}} + end, + Bridges + ), + #{ + metrics => aggregate_metrics(NodeMetrics), + node_metrics => NodeMetrics + }. + +aggregate_metrics(AllMetrics) -> + InitMetrics = ?EMPTY_METRICS, + lists:foldl(fun aggregate_metrics/2, InitMetrics, AllMetrics). + +aggregate_metrics( + #{ + metrics := ?metrics( + M1, M2, M3, M4, M5, M6, M7, M8, M9, M10, M11, M12, M13, M14, M15, M16, M17 + ) + }, + ?metrics( + N1, N2, N3, N4, N5, N6, N7, N8, N9, N10, N11, N12, N13, N14, N15, N16, N17 + ) +) -> + ?METRICS( + M1 + N1, + M2 + N2, + M3 + N3, + M4 + N4, + M5 + N5, + M6 + N6, + M7 + N7, + M8 + N8, + M9 + N9, + M10 + N10, + M11 + N11, + M12 + N12, + M13 + N13, + M14 + N14, + M15 + N15, + M16 + N16, + M17 + N17 + ). + format_bridge_status_and_error(Data) -> maps:fold(fun format_resource_data/3, #{}, maps:with([status, error], Data)). diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v5.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v5.erl index 1417615a7..75b99f0ec 100644 --- a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v5.erl +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v5.erl @@ -34,7 +34,8 @@ v2_start_bridge_to_node/3, v2_start_bridge_to_all_nodes/3, v2_list_bridges_on_nodes/1, - v2_lookup_from_all_nodes/3 + v2_lookup_from_all_nodes/3, + v2_get_metrics_from_all_nodes/3 ]). -include_lib("emqx/include/bpapi.hrl"). @@ -156,6 +157,17 @@ v2_lookup_from_all_nodes(Nodes, BridgeType, BridgeName) -> ?TIMEOUT ). +-spec v2_get_metrics_from_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(). +v2_get_metrics_from_all_nodes(Nodes, ActionType, ActionName) -> + erpc:multicall( + Nodes, + emqx_bridge_v2_api, + get_metrics_from_local_node, + [ActionType, ActionName], + ?TIMEOUT + ). + -spec v2_start_bridge_to_all_nodes([node()], key(), key()) -> emqx_rpc:erpc_multicall(). v2_start_bridge_to_all_nodes(Nodes, BridgeType, BridgeName) -> diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index cf58eefde..8758c325d 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -56,6 +56,7 @@ -define(CONNECTOR(Name), ?KAFKA_CONNECTOR(Name, ?KAFKA_BOOTSTRAP_HOST)). -define(CONNECTOR, ?CONNECTOR(?CONNECTOR_NAME)). +-define(MQTT_LOCAL_TOPIC, <<"mqtt/local/topic">>). -define(BRIDGE_NAME, (atom_to_binary(?FUNCTION_NAME))). -define(BRIDGE_TYPE_STR, "kafka_producer"). -define(BRIDGE_TYPE, <>). @@ -93,7 +94,7 @@ <<"required_acks">> => <<"all_isr">>, <<"topic">> => <<"kafka-topic">> }, - <<"local_topic">> => <<"mqtt/local/topic">>, + <<"local_topic">> => ?MQTT_LOCAL_TOPIC, <<"resource_opts">> => #{ <<"health_check_interval">> => <<"32s">> } @@ -105,48 +106,6 @@ ). -define(KAFKA_BRIDGE_UPDATE(Name), ?KAFKA_BRIDGE_UPDATE(Name, ?CONNECTOR_NAME)). -%% -define(BRIDGE_TYPE_MQTT, <<"mqtt">>). -%% -define(MQTT_BRIDGE(SERVER, NAME), ?BRIDGE(NAME, ?BRIDGE_TYPE_MQTT)#{ -%% <<"server">> => SERVER, -%% <<"username">> => <<"user1">>, -%% <<"password">> => <<"">>, -%% <<"proto_ver">> => <<"v5">>, -%% <<"egress">> => #{ -%% <<"remote">> => #{ -%% <<"topic">> => <<"emqx/${topic}">>, -%% <<"qos">> => <<"${qos}">>, -%% <<"retain">> => false -%% } -%% } -%% }). -%% -define(MQTT_BRIDGE(SERVER), ?MQTT_BRIDGE(SERVER, <<"mqtt_egress_test_bridge">>)). - -%% -define(BRIDGE_TYPE_HTTP, <<"kafka">>). -%% -define(HTTP_BRIDGE(URL, NAME), ?BRIDGE(NAME, ?BRIDGE_TYPE_HTTP)#{ -%% <<"url">> => URL, -%% <<"local_topic">> => <<"emqx_webhook/#">>, -%% <<"method">> => <<"post">>, -%% <<"body">> => <<"${payload}">>, -%% <<"headers">> => #{ -%% % NOTE -%% % The Pascal-Case is important here. -%% % The reason is kinda ridiculous: `emqx_bridge_resource:create_dry_run/2` converts -%% % bridge config keys into atoms, and the atom 'Content-Type' exists in the ERTS -%% % when this happens (while the 'content-type' does not). -%% <<"Content-Type">> => <<"application/json">> -%% } -%% }). -%% -define(HTTP_BRIDGE(URL), ?HTTP_BRIDGE(URL, ?BRIDGE_NAME)). - -%% -define(URL(PORT, PATH), -%% list_to_binary( -%% io_lib:format( -%% "http://localhost:~s/~s", -%% [integer_to_list(PORT), PATH] -%% ) -%% ) -%% ). - -define(APPSPECS, [ emqx_conf, emqx, @@ -166,7 +125,7 @@ all() -> [ {group, single}, - %{group, cluster_later_join}, + {group, cluster_later_join}, {group, cluster} ]. -else. @@ -182,7 +141,7 @@ groups() -> t_fix_broken_bridge_config ], ClusterLaterJoinOnlyTCs = [ - % t_cluster_later_join_metrics + t_cluster_later_join_metrics ], [ {single, [], AllTCs -- ClusterLaterJoinOnlyTCs}, @@ -202,9 +161,9 @@ end_per_suite(_Config) -> init_per_group(cluster = Name, Config) -> Nodes = [NodePrimary | _] = mk_cluster(Name, Config), init_api([{group, Name}, {cluster_nodes, Nodes}, {node, NodePrimary} | Config]); -%% init_per_group(cluster_later_join = Name, Config) -> -%% Nodes = [NodePrimary | _] = mk_cluster(Name, Config, #{join_to => undefined}), -%% init_api([{group, Name}, {cluster_nodes, Nodes}, {node, NodePrimary} | Config]); +init_per_group(cluster_later_join = Name, Config) -> + Nodes = [NodePrimary | _] = mk_cluster(Name, Config, #{join_to => undefined}), + init_api([{group, Name}, {cluster_nodes, Nodes}, {node, NodePrimary} | Config]); init_per_group(Name, Config) -> WorkDir = filename:join(?config(priv_dir, Config), Name), Apps = emqx_cth_suite:start(?APPSPECS ++ [?APPSPEC_DASHBOARD], #{work_dir => WorkDir}), @@ -1041,6 +1000,143 @@ t_bad_name(Config) -> ), ok. +t_metrics(Config) -> + {ok, 200, []} = request_json(get, uri([?ROOT]), Config), + + ActionName = ?BRIDGE_NAME, + ?assertMatch( + {ok, 201, _}, + request_json( + post, + uri([?ROOT]), + ?KAFKA_BRIDGE(?BRIDGE_NAME), + Config + ) + ), + + ActionID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ActionName), + + ?assertMatch( + {ok, 200, #{ + <<"metrics">> := #{<<"matched">> := 0}, + <<"node_metrics">> := [#{<<"metrics">> := #{<<"matched">> := 0}} | _] + }}, + request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + ), + + {ok, 200, Bridge} = request_json(get, uri([?ROOT, ActionID]), Config), + ?assertNot(maps:is_key(<<"metrics">>, Bridge)), + ?assertNot(maps:is_key(<<"node_metrics">>, Bridge)), + + Body = <<"my msg">>, + _ = publish_message(?MQTT_LOCAL_TOPIC, Body, Config), + + %% check for non-empty bridge metrics + ?retry( + _Sleep0 = 200, + _Retries0 = 20, + ?assertMatch( + {ok, 200, #{ + <<"metrics">> := #{<<"matched">> := 1}, + <<"node_metrics">> := [#{<<"metrics">> := #{<<"matched">> := 1}} | _] + }}, + request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + ) + ), + + %% check for absence of metrics when listing all bridges + {ok, 200, Bridges} = request_json(get, uri([?ROOT]), Config), + ?assertNotMatch( + [ + #{ + <<"metrics">> := #{}, + <<"node_metrics">> := [_ | _] + } + ], + Bridges + ), + ok. + +t_reset_metrics(Config) -> + %% assert there's no bridges at first + {ok, 200, []} = request_json(get, uri([?ROOT]), Config), + + ActionName = ?BRIDGE_NAME, + ?assertMatch( + {ok, 201, _}, + request_json( + post, + uri([?ROOT]), + ?KAFKA_BRIDGE(?BRIDGE_NAME), + Config + ) + ), + ActionID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ActionName), + + Body = <<"my msg">>, + _ = publish_message(?MQTT_LOCAL_TOPIC, Body, Config), + ?retry( + _Sleep0 = 200, + _Retries0 = 20, + ?assertMatch( + {ok, 200, #{ + <<"metrics">> := #{<<"matched">> := 1}, + <<"node_metrics">> := [#{<<"metrics">> := #{}} | _] + }}, + request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + ) + ), + + {ok, 204, <<>>} = request(put, uri([?ROOT, ActionID, "metrics", "reset"]), Config), + + ?retry( + _Sleep0 = 200, + _Retries0 = 20, + ?assertMatch( + {ok, 200, #{ + <<"metrics">> := #{<<"matched">> := 0}, + <<"node_metrics">> := [#{<<"metrics">> := #{}} | _] + }}, + request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + ) + ), + + ok. + +t_cluster_later_join_metrics(Config) -> + [PrimaryNode, OtherNode | _] = ?config(cluster_nodes, Config), + Name = ?BRIDGE_NAME, + ActionParams = ?KAFKA_BRIDGE(Name), + ActionID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), + ?check_trace( + begin + %% Create a bridge on only one of the nodes. + ?assertMatch({ok, 201, _}, request_json(post, uri([?ROOT]), ActionParams, Config)), + %% Pre-condition. + ?assertMatch( + {ok, 200, #{ + <<"metrics">> := #{<<"success">> := _}, + <<"node_metrics">> := [#{<<"metrics">> := #{}} | _] + }}, + request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + ), + %% Now join the other node join with the api node. + ok = erpc:call(OtherNode, ekka, join, [PrimaryNode]), + %% Check metrics; shouldn't crash even if the bridge is not + %% ready on the node that just joined the cluster. + ?assertMatch( + {ok, 200, #{ + <<"metrics">> := #{<<"success">> := _}, + <<"node_metrics">> := [#{<<"metrics">> := #{}}, #{<<"metrics">> := #{}} | _] + }}, + request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + ), + ok + end, + [] + ), + ok. + %%% helpers listen_on_random_port() -> SockOpts = [binary, {active, false}, {packet, raw}, {reuseaddr, true}, {backlog, 1000}], From 048f4724a9f5d853840bc49139e5c1d99bfbec25 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 27 Nov 2023 16:09:17 +0100 Subject: [PATCH 17/20] feat(emqx_connector): add field 'actions' in API response Also unify schemas, use emqx_connector_schema for the generic parts. --- .../src/schema/emqx_bridge_v2_schema.erl | 2 +- .../src/emqx_bridge_azure_event_hub.erl | 51 ++++++----- .../src/emqx_bridge_confluent_producer.erl | 51 ++++++----- ...emqx_bridge_gcp_pubsub_producer_schema.erl | 26 +++--- .../src/emqx_bridge_kafka.erl | 29 ++++++- .../src/emqx_bridge_matrix.erl | 14 +-- .../src/emqx_bridge_mongodb.erl | 41 ++++----- .../src/emqx_bridge_syskeeper_connector.erl | 28 +++--- .../src/emqx_bridge_syskeeper_proxy.erl | 31 +++---- .../src/emqx_bridge_timescale.erl | 16 ++-- .../emqx_connector/src/emqx_connector_api.erl | 12 ++- .../src/schema/emqx_connector_schema.erl | 85 ++++++++++++++++++- .../test/emqx_connector_api_SUITE.erl | 82 +++++++++++++++--- .../emqx_postgresql_connector_schema.erl | 31 ++++--- rel/i18n/emqx_connector_schema.hocon | 45 ++++++++++ 15 files changed, 401 insertions(+), 143 deletions(-) diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index 25619d99a..188a550fc 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -250,7 +250,7 @@ is_bad_schema(#{type := ?MAP(_, ?R_REF(Module, TypeName))}) -> false; _ -> {true, #{ - schema_modle => Module, + schema_module => Module, type_name => TypeName, missing_fields => MissingFileds }} diff --git a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl index 569725a34..cf733ddfd 100644 --- a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl +++ b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl @@ -31,8 +31,8 @@ -import(hoconsc, [mk/2, enum/1, ref/2]). --define(AEH_CONNECTOR_TYPE, azure_event_hub_producer). --define(AEH_CONNECTOR_TYPE_BIN, <<"azure_event_hub_producer">>). +-define(CONNECTOR_TYPE, azure_event_hub_producer). +-define(CONNECTOR_TYPE_BIN, <<"azure_event_hub_producer">>). %%------------------------------------------------------------------------------------------------- %% `hocon_schema' API @@ -42,18 +42,17 @@ namespace() -> "bridge_azure_event_hub". roots() -> ["config_producer"]. -fields("put_connector") -> +fields(Field) when + Field == "get_connector"; + Field == "put_connector"; + Field == "post_connector" +-> Fields = override( - emqx_bridge_kafka:fields("put_connector"), - connector_overrides() - ), - override_documentations(Fields); -fields("get_connector") -> - emqx_bridge_schema:status_fields() ++ - fields("post_connector"); -fields("post_connector") -> - Fields = override( - emqx_bridge_kafka:fields("post_connector"), + emqx_connector_schema:api_fields( + Field, + ?CONNECTOR_TYPE, + emqx_bridge_kafka:kafka_connector_config_fields() + ), connector_overrides() ), override_documentations(Fields); @@ -170,7 +169,7 @@ struct_names() -> bridge_v2_examples(Method) -> [ #{ - ?AEH_CONNECTOR_TYPE_BIN => #{ + ?CONNECTOR_TYPE_BIN => #{ summary => <<"Azure Event Hub Action">>, value => values({Method, bridge_v2}) } @@ -180,7 +179,7 @@ bridge_v2_examples(Method) -> connector_examples(Method) -> [ #{ - ?AEH_CONNECTOR_TYPE_BIN => #{ + ?CONNECTOR_TYPE_BIN => #{ summary => <<"Azure Event Hub Connector">>, value => values({Method, connector}) } @@ -197,6 +196,20 @@ conn_bridge_examples(Method) -> } ]. +values({get, connector}) -> + maps:merge( + #{ + status => <<"connected">>, + node_status => [ + #{ + node => <<"emqx@localhost">>, + status => <<"connected">> + } + ], + actions => [<<"my_action">>] + }, + values({post, connector}) + ); values({get, AEHType}) -> maps:merge( #{ @@ -217,7 +230,7 @@ values({post, bridge_v2}) -> enable => true, connector => <<"my_azure_event_hub_producer_connector">>, name => <<"my_azure_event_hub_producer_action">>, - type => ?AEH_CONNECTOR_TYPE_BIN + type => ?CONNECTOR_TYPE_BIN } ); values({post, connector}) -> @@ -225,7 +238,7 @@ values({post, connector}) -> values(common_config), #{ name => <<"my_azure_event_hub_producer_connector">>, - type => ?AEH_CONNECTOR_TYPE_BIN, + type => ?CONNECTOR_TYPE_BIN, ssl => #{ enable => true, server_name_indication => <<"auto">>, @@ -358,7 +371,7 @@ connector_overrides() -> } ), type => mk( - ?AEH_CONNECTOR_TYPE, + ?CONNECTOR_TYPE, #{ required => true, desc => ?DESC("connector_type") @@ -414,7 +427,7 @@ bridge_v2_overrides() -> }), ssl => mk(ref(ssl_client_opts), #{default => #{<<"enable">> => true}}), type => mk( - ?AEH_CONNECTOR_TYPE, + ?CONNECTOR_TYPE, #{ required => true, desc => ?DESC("bridge_v2_type") diff --git a/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl index 8742d7ccf..a43a8a285 100644 --- a/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl +++ b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl @@ -30,8 +30,8 @@ -import(hoconsc, [mk/2, enum/1, ref/2]). --define(CONFLUENT_CONNECTOR_TYPE, confluent_producer). --define(CONFLUENT_CONNECTOR_TYPE_BIN, <<"confluent_producer">>). +-define(CONNECTOR_TYPE, confluent_producer). +-define(CONNECTOR_TYPE_BIN, <<"confluent_producer">>). %%------------------------------------------------------------------------------------------------- %% `hocon_schema' API @@ -41,18 +41,17 @@ namespace() -> "confluent". roots() -> ["config_producer"]. -fields("put_connector") -> +fields(Field) when + Field == "get_connector"; + Field == "put_connector"; + Field == "post_connector" +-> Fields = override( - emqx_bridge_kafka:fields("put_connector"), - connector_overrides() - ), - override_documentations(Fields); -fields("get_connector") -> - emqx_bridge_schema:status_fields() ++ - fields("post_connector"); -fields("post_connector") -> - Fields = override( - emqx_bridge_kafka:fields("post_connector"), + emqx_connector_schema:api_fields( + Field, + ?CONNECTOR_TYPE, + emqx_bridge_kafka:kafka_connector_config_fields() + ), connector_overrides() ), override_documentations(Fields); @@ -155,7 +154,7 @@ struct_names() -> bridge_v2_examples(Method) -> [ #{ - ?CONFLUENT_CONNECTOR_TYPE_BIN => #{ + ?CONNECTOR_TYPE_BIN => #{ summary => <<"Confluent Action">>, value => values({Method, bridge_v2}) } @@ -165,13 +164,27 @@ bridge_v2_examples(Method) -> connector_examples(Method) -> [ #{ - ?CONFLUENT_CONNECTOR_TYPE_BIN => #{ + ?CONNECTOR_TYPE_BIN => #{ summary => <<"Confluent Connector">>, value => values({Method, connector}) } } ]. +values({get, connector}) -> + maps:merge( + #{ + status => <<"connected">>, + node_status => [ + #{ + node => <<"emqx@localhost">>, + status => <<"connected">> + } + ], + actions => [<<"my_action">>] + }, + values({post, connector}) + ); values({get, ConfluentType}) -> maps:merge( #{ @@ -192,7 +205,7 @@ values({post, bridge_v2}) -> enable => true, connector => <<"my_confluent_producer_connector">>, name => <<"my_confluent_producer_action">>, - type => ?CONFLUENT_CONNECTOR_TYPE_BIN + type => ?CONNECTOR_TYPE_BIN } ); values({post, connector}) -> @@ -200,7 +213,7 @@ values({post, connector}) -> values(common_config), #{ name => <<"my_confluent_producer_connector">>, - type => ?CONFLUENT_CONNECTOR_TYPE_BIN, + type => ?CONNECTOR_TYPE_BIN, ssl => #{ enable => true, server_name_indication => <<"auto">>, @@ -320,7 +333,7 @@ connector_overrides() -> } ), type => mk( - ?CONFLUENT_CONNECTOR_TYPE, + ?CONNECTOR_TYPE, #{ required => true, desc => ?DESC("connector_type") @@ -342,7 +355,7 @@ bridge_v2_overrides() -> } }), type => mk( - ?CONFLUENT_CONNECTOR_TYPE, + ?CONNECTOR_TYPE, #{ required => true, desc => ?DESC("bridge_v2_type") diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_producer_schema.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_producer_schema.erl index 0ee625824..a4c939d7a 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_producer_schema.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_producer_schema.erl @@ -24,6 +24,8 @@ connector_examples/1 ]). +-define(CONNECTOR_TYPE, gcp_pubsub_producer). + %%------------------------------------------------------------------------------------------------- %% `hocon_schema' API %%------------------------------------------------------------------------------------------------- @@ -68,8 +70,7 @@ fields(action_parameters) -> fields("config_connector") -> %% FIXME emqx_connector_schema:common_fields() ++ - emqx_bridge_gcp_pubsub:fields(connector_config) ++ - emqx_resource_schema:fields("resource_opts"); + connector_config_fields(); %%========================================= %% HTTP API fields: action %%========================================= @@ -82,12 +83,16 @@ fields("put_bridge_v2") -> %%========================================= %% HTTP API fields: connector %%========================================= -fields("get_connector") -> - emqx_bridge_schema:status_fields() ++ fields("post_connector"); -fields("post_connector") -> - [type_field(), name_field() | fields("put_connector")]; -fields("put_connector") -> - fields("config_connector"). +fields(Field) when + Field == "get_connector"; + Field == "put_connector"; + Field == "post_connector" +-> + emqx_connector_schema:api_fields(Field, ?CONNECTOR_TYPE, connector_config_fields()). + +connector_config_fields() -> + emqx_bridge_gcp_pubsub:fields(connector_config) ++ + emqx_resource_schema:fields("resource_opts"). desc("config_connector") -> ?DESC("config_connector"); @@ -177,7 +182,7 @@ action_example(put) -> connector_example(get) -> maps:merge( - connector_example(put), + connector_example(post), #{ status => <<"connected">>, node_status => [ @@ -185,7 +190,8 @@ connector_example(get) -> node => <<"emqx@localhost">>, status => <<"connected">> } - ] + ], + actions => [<<"my_action">>] } ); connector_example(post) -> diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl index 93515b5db..28050d368 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl @@ -33,10 +33,13 @@ ]). -export([ + kafka_connector_config_fields/0, kafka_producer_converter/2, producer_strategy_key_validator/1 ]). +-define(CONNECTOR_TYPE, kafka_producer). + %% ------------------------------------------------------------------------------------------------- %% api @@ -76,6 +79,20 @@ conn_bridge_examples(Method) -> } ]. +values({get, connector}) -> + maps:merge( + #{ + status => <<"connected">>, + node_status => [ + #{ + node => <<"emqx@localhost">>, + status => <<"connected">> + } + ], + actions => [<<"my_action">>] + }, + values({post, connector}) + ); values({get, KafkaType}) -> maps:merge( #{ @@ -247,6 +264,12 @@ namespace() -> "bridge_kafka". roots() -> ["config_consumer", "config_producer", "config_bridge_v2"]. +fields(Field) when + Field == "get_connector"; + Field == "put_connector"; + Field == "post_connector" +-> + emqx_connector_schema:api_fields(Field, ?CONNECTOR_TYPE, kafka_connector_config_fields()); fields("post_" ++ Type) -> [type_field(Type), name_field() | fields("config_" ++ Type)]; fields("put_" ++ Type) -> @@ -560,9 +583,11 @@ desc(Name) -> ?DESC(Name). connector_config_fields() -> + emqx_connector_schema:common_fields() ++ + kafka_connector_config_fields(). + +kafka_connector_config_fields() -> [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, - {description, emqx_schema:description_schema()}, {bootstrap_hosts, mk( binary(), diff --git a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.erl b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.erl index f74e18d3b..4f7a1a370 100644 --- a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.erl +++ b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.erl @@ -22,6 +22,8 @@ connector_examples/1 ]). +-define(CONNECTOR_TYPE, matrix). + %% ------------------------------------------------------------------------------------------------- %% api @@ -60,12 +62,12 @@ fields("get_bridge_v2") -> emqx_bridge_pgsql:fields(pgsql_action); fields("post_bridge_v2") -> emqx_bridge_pgsql:fields(pgsql_action); -fields("put_connector") -> - emqx_bridge_pgsql:fields("config_connector"); -fields("get_connector") -> - emqx_bridge_pgsql:fields("config_connector"); -fields("post_connector") -> - emqx_bridge_pgsql:fields("config_connector"); +fields(Field) when + Field == "get_connector"; + Field == "put_connector"; + Field == "post_connector" +-> + emqx_postgresql_connector_schema:fields({Field, ?CONNECTOR_TYPE}); fields(Method) -> emqx_bridge_pgsql:fields(Method). diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl index ac7aa6280..796a4a4d1 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl @@ -25,6 +25,8 @@ desc/1 ]). +-define(CONNECTOR_TYPE, mongodb). + %%================================================================================================= %% hocon_schema API %%================================================================================================= @@ -51,16 +53,18 @@ fields("config") -> ]; fields("config_connector") -> emqx_connector_schema:common_fields() ++ - [ - {parameters, - mk( - hoconsc:union([ - ref(emqx_mongodb, "connector_" ++ T) - || T <- ["single", "sharded", "rs"] - ]), - #{required => true, desc => ?DESC("mongodb_parameters")} - )} - ] ++ emqx_mongodb:fields(mongodb); + fields("connection_fields"); +fields("connection_fields") -> + [ + {parameters, + mk( + hoconsc:union([ + ref(emqx_mongodb, "connector_" ++ T) + || T <- ["single", "sharded", "rs"] + ]), + #{required => true, desc => ?DESC("mongodb_parameters")} + )} + ] ++ emqx_mongodb:fields(mongodb); fields("creation_opts") -> %% so far, mongodb connector does not support batching %% but we cannot delete this field due to compatibility reasons @@ -97,14 +101,12 @@ fields(mongodb_sharded) -> emqx_mongodb:fields(sharded) ++ fields("config"); fields(mongodb_single) -> emqx_mongodb:fields(single) ++ fields("config"); -fields("post_connector") -> - type_and_name_fields(mongodb) ++ - fields("config_connector"); -fields("put_connector") -> - fields("config_connector"); -fields("get_connector") -> - emqx_bridge_schema:status_fields() ++ - fields("post_connector"); +fields(Field) when + Field == "get_connector"; + Field == "put_connector"; + Field == "post_connector" +-> + emqx_connector_schema:api_fields(Field, ?CONNECTOR_TYPE, fields("connection_fields")); fields("get_bridge_v2") -> emqx_bridge_schema:status_fields() ++ fields("post_bridge_v2"); @@ -319,7 +321,8 @@ method_values(Type, get) -> node => <<"emqx@localhost">>, status => <<"connected">> } - ] + ], + actions => [<<"my_action">>] } ); method_values(_Type, put) -> diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl index c267ee521..49942065a 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl @@ -35,6 +35,7 @@ -import(hoconsc, [mk/2, enum/1, ref/2]). +-define(CONNECTOR_TYPE, syskeeper_forwarder). -define(SYSKEEPER_HOST_OPTIONS, #{ default_port => 9092 }). @@ -62,7 +63,8 @@ values(get) -> node => <<"emqx@localhost">>, status => <<"connected">> } - ] + ], + actions => [<<"my_action">>] }, values(post) ); @@ -89,9 +91,9 @@ roots() -> [{config, #{type => hoconsc:ref(?MODULE, config)}}]. fields(config) -> + emqx_connector_schema:common_fields() ++ fields("connection_fields"); +fields("connection_fields") -> [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, - {description, emqx_schema:description_schema()}, {server, server()}, {ack_mode, mk( @@ -110,12 +112,14 @@ fields(config) -> emqx_connector_schema_lib:pool_size(Other) end} ]; -fields("post") -> - [type_field(), name_field() | fields(config)]; -fields("put") -> - fields(config); -fields("get") -> - emqx_bridge_schema:status_fields() ++ fields("post"). +fields(Field) when + Field == "get"; + Field == "post"; + Field == "put" +-> + emqx_connector_schema:api_fields( + Field ++ "_connector", ?CONNECTOR_TYPE, fields("connection_fields") + ). desc(config) -> ?DESC("desc_config"); @@ -128,12 +132,6 @@ server() -> Meta = #{desc => ?DESC("server")}, emqx_schema:servers_sc(Meta, ?SYSKEEPER_HOST_OPTIONS). -type_field() -> - {type, mk(enum([syskeeper_forwarder]), #{required => true, desc => ?DESC("desc_type")})}. - -name_field() -> - {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. - %% ------------------------------------------------------------------------------------------------- %% `emqx_resource' API diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy.erl index 1968022c1..f930b0042 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy.erl @@ -22,6 +22,8 @@ desc/1 ]). +-define(CONNECTOR_TYPE, syskeeper_proxy). + -define(SYSKEEPER_HOST_OPTIONS, #{ default_port => 9092 }). @@ -47,7 +49,8 @@ values(get) -> node => <<"emqx@localhost">>, status => <<"connected">> } - ] + ], + actions => [<<"my_action">>] }, values(post) ); @@ -74,9 +77,9 @@ namespace() -> "connector_syskeeper_proxy". roots() -> []. fields(config) -> + emqx_connector_schema:common_fields() ++ fields("connection_fields"); +fields("connection_fields") -> [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, - {description, emqx_schema:description_schema()}, {listen, listen()}, {acceptors, mk( @@ -89,12 +92,14 @@ fields(config) -> #{desc => ?DESC(handshake_timeout), default => <<"10s">>} )} ]; -fields("post") -> - [type_field(), name_field() | fields(config)]; -fields("put") -> - fields(config); -fields("get") -> - emqx_bridge_schema:status_fields() ++ fields("post"). +fields(Field) when + Field == "get"; + Field == "post"; + Field == "put" +-> + emqx_connector_schema:api_fields( + Field ++ "_connector", ?CONNECTOR_TYPE, fields("connection_fields") + ). desc(config) -> ?DESC("desc_config"); @@ -106,11 +111,3 @@ desc(_) -> listen() -> Meta = #{desc => ?DESC("listen")}, emqx_schema:servers_sc(Meta, ?SYSKEEPER_HOST_OPTIONS). - -%% ------------------------------------------------------------------------------------------------- - -type_field() -> - {type, mk(enum([syskeeper_proxy]), #{required => true, desc => ?DESC("desc_type")})}. - -name_field() -> - {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.erl b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.erl index 796d9d9f6..5d6c5498d 100644 --- a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.erl +++ b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.erl @@ -22,6 +22,8 @@ connector_examples/1 ]). +-define(CONNECTOR_TYPE, timescale). + %% ------------------------------------------------------------------------------------------------- %% api @@ -44,7 +46,7 @@ roots() -> []. fields("post") -> emqx_bridge_pgsql:fields("post", timescale); fields("config_connector") -> - emqx_bridge_pgsql:fields("config_connector"); + emqx_postgresql_connector_schema:fields("config_connector"); fields(action) -> {timescale, hoconsc:mk( @@ -60,12 +62,12 @@ fields("get_bridge_v2") -> emqx_bridge_pgsql:fields(pgsql_action); fields("post_bridge_v2") -> emqx_bridge_pgsql:fields(pgsql_action); -fields("put_connector") -> - emqx_bridge_pgsql:fields("config_connector"); -fields("get_connector") -> - emqx_bridge_pgsql:fields("config_connector"); -fields("post_connector") -> - emqx_bridge_pgsql:fields("config_connector"); +fields(Field) when + Field == "get_connector"; + Field == "put_connector"; + Field == "post_connector" +-> + emqx_postgresql_connector_schema:fields({Field, ?CONNECTOR_TYPE}); fields(Method) -> emqx_bridge_pgsql:fields(Method). diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index f6e0c0f95..d09c67c8a 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -637,15 +637,25 @@ format_resource( ). format_resource_data(ResData) -> - maps:fold(fun format_resource_data/3, #{}, maps:with([status, error], ResData)). + maps:fold(fun format_resource_data/3, #{}, maps:with([status, error, added_channels], ResData)). format_resource_data(error, undefined, Result) -> Result; format_resource_data(error, Error, Result) -> Result#{status_reason => emqx_utils:readable_error_msg(Error)}; +format_resource_data(added_channels, Channels, Result) -> + Result#{actions => lists:map(fun format_action/1, maps:keys(Channels))}; format_resource_data(K, V, Result) -> Result#{K => V}. +format_action(Action) -> + case string:split(Action, ":", all) of + [_Prefix, _Type, Name | _] -> + Name; + _ -> + Action + end. + is_ok(ok) -> ok; is_ok(OkResult = {ok, _}) -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index a7de0cf52..d6f8608ae 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -33,7 +33,12 @@ -export([get_response/0, put_request/0, post_request/0]). -export([connector_type_to_bridge_types/1]). --export([common_fields/0]). +-export([ + api_fields/3, + common_fields/0, + status_and_actions_fields/0, + type_and_name_fields/1 +]). -export([resource_opts_fields/0, resource_opts_fields/1]). @@ -352,19 +357,87 @@ roots() -> end. fields(connectors) -> - [] ++ enterprise_fields_connectors(). + [] ++ enterprise_fields_connectors(); +fields("node_status") -> + [ + node_name(), + {"status", mk(status(), #{})}, + {"status_reason", + mk(binary(), #{ + required => false, + desc => ?DESC("desc_status_reason"), + example => <<"Connection refused">> + })} + ]. desc(connectors) -> ?DESC("desc_connectors"); +desc("node_status") -> + ?DESC("desc_node_status"); desc(_) -> undefined. +api_fields("get_connector", Type, Fields) -> + lists:append( + [ + type_and_name_fields(Type), + common_fields(), + status_and_actions_fields(), + Fields + ] + ); +api_fields("post_connector", Type, Fields) -> + lists:append( + [ + type_and_name_fields(Type), + common_fields(), + Fields + ] + ); +api_fields("put_connector", _Type, Fields) -> + lists:append( + [ + common_fields(), + Fields + ] + ). + common_fields() -> [ {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, {description, emqx_schema:description_schema()} ]. +type_and_name_fields(ConnectorType) -> + [ + {type, mk(ConnectorType, #{required => true, desc => ?DESC("desc_type")})}, + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})} + ]. + +status_and_actions_fields() -> + [ + {"status", mk(status(), #{desc => ?DESC("desc_status")})}, + {"status_reason", + mk(binary(), #{ + required => false, + desc => ?DESC("desc_status_reason"), + example => <<"Connection refused">> + })}, + {"node_status", + mk( + hoconsc:array(ref(?MODULE, "node_status")), + #{desc => ?DESC("desc_node_status")} + )}, + {"actions", + mk( + hoconsc:array(binary()), + #{ + desc => ?DESC("connector_actions"), + example => [<<"my_action">>] + } + )} + ]. + resource_opts_fields() -> resource_opts_fields(_Overrides = []). @@ -422,12 +495,18 @@ is_bad_schema(#{type := ?MAP(_, ?R_REF(Module, TypeName))}) -> false; _ -> {true, #{ - schema_modle => Module, + schema_module => Module, type_name => TypeName, missing_fields => MissingFileds }} end. +status() -> + hoconsc:enum([connected, disconnected, connecting, inconsistent]). + +node_name() -> + {"node", mk(binary(), #{desc => ?DESC("desc_node_name"), example => "emqx@127.0.0.1"})}. + common_field_names() -> [ enable, description diff --git a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl index bd8aa9ddf..0b4189396 100644 --- a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl @@ -175,7 +175,8 @@ groups() -> AllTCs = emqx_common_test_helpers:all(?MODULE), SingleOnlyTests = [ t_connectors_probe, - t_fail_delete_with_action + t_fail_delete_with_action, + t_actions_field ], ClusterLaterJoinOnlyTCs = [ % t_cluster_later_join_metrics @@ -256,15 +257,6 @@ end_per_testcase(TestCase, Config) -> ok. -define(CONNECTOR_IMPL, dummy_connector_impl). -init_mocks(t_fail_delete_with_action) -> - init_mocks(common), - meck:expect(?CONNECTOR_IMPL, on_add_channel, 4, {ok, connector_state}), - meck:expect(?CONNECTOR_IMPL, on_remove_channel, 3, {ok, connector_state}), - meck:expect(?CONNECTOR_IMPL, on_get_channel_status, 3, connected), - ok = meck:expect(?CONNECTOR_IMPL, on_get_channels, fun(ResId) -> - emqx_bridge_v2:get_channels_for_connector(ResId) - end), - ok; init_mocks(_TestCase) -> meck:new(emqx_connector_ee_schema, [passthrough, no_link]), meck:expect(emqx_connector_ee_schema, resource_type, 1, ?CONNECTOR_IMPL), @@ -289,17 +281,25 @@ init_mocks(_TestCase) -> (_, _) -> connected end ), + meck:expect(?CONNECTOR_IMPL, on_add_channel, 4, {ok, connector_state}), + meck:expect(?CONNECTOR_IMPL, on_remove_channel, 3, {ok, connector_state}), + meck:expect(?CONNECTOR_IMPL, on_get_channel_status, 3, connected), + meck:expect( + ?CONNECTOR_IMPL, + on_get_channels, + fun(ResId) -> + emqx_bridge_v2:get_channels_for_connector(ResId) + end + ), [?CONNECTOR_IMPL, emqx_connector_ee_schema]. -clear_resources(t_fail_delete_with_action) -> +clear_resources(_) -> lists:foreach( fun(#{type := Type, name := Name}) -> ok = emqx_bridge_v2:remove(Type, Name) end, emqx_bridge_v2:list() ), - clear_resources(common); -clear_resources(_) -> lists:foreach( fun(#{type := Type, name := Name}) -> ok = emqx_connector:remove(Type, Name) @@ -738,6 +738,62 @@ t_create_with_bad_name(Config) -> ?assertMatch(#{<<"kind">> := <<"validation_error">>}, Msg), ok. +t_actions_field(Config) -> + Name = ?CONNECTOR_NAME, + ?assertMatch( + {ok, 201, #{ + <<"type">> := ?CONNECTOR_TYPE, + <<"name">> := Name, + <<"enable">> := true, + <<"status">> := <<"connected">>, + <<"node_status">> := [_ | _], + <<"actions">> := [] + }}, + request_json( + post, + uri(["connectors"]), + ?KAFKA_CONNECTOR(Name), + Config + ) + ), + ConnectorID = emqx_connector_resource:connector_id(?CONNECTOR_TYPE, Name), + BridgeName = ?BRIDGE_NAME, + ?assertMatch( + {ok, 201, #{ + <<"type">> := ?BRIDGE_TYPE, + <<"name">> := BridgeName, + <<"enable">> := true, + <<"status">> := <<"connected">>, + <<"node_status">> := [_ | _], + <<"connector">> := Name, + <<"kafka">> := #{}, + <<"local_topic">> := _, + <<"resource_opts">> := _ + }}, + request_json( + post, + uri(["actions"]), + ?KAFKA_BRIDGE(?BRIDGE_NAME), + Config + ) + ), + ?assertMatch( + {ok, 200, #{ + <<"type">> := ?CONNECTOR_TYPE, + <<"name">> := Name, + <<"enable">> := true, + <<"status">> := <<"connected">>, + <<"node_status">> := [_ | _], + <<"actions">> := [BridgeName] + }}, + request_json( + get, + uri(["connectors", ConnectorID]), + Config + ) + ), + ok. + t_fail_delete_with_action(Config) -> Name = ?CONNECTOR_NAME, ?assertMatch( diff --git a/apps/emqx_postgresql/src/schema/emqx_postgresql_connector_schema.erl b/apps/emqx_postgresql/src/schema/emqx_postgresql_connector_schema.erl index 74591beee..94e07ba7a 100644 --- a/apps/emqx_postgresql/src/schema/emqx_postgresql_connector_schema.erl +++ b/apps/emqx_postgresql/src/schema/emqx_postgresql_connector_schema.erl @@ -35,6 +35,8 @@ values/1 ]). +-define(CONNECTOR_TYPE, pgsql). + roots() -> []. @@ -64,12 +66,18 @@ fields("get_bridge_v2") -> fields(pgsql_action); fields("post_bridge_v2") -> fields(pgsql_action); -fields("put_connector") -> - fields("config_connector"); -fields("get_connector") -> - fields("config_connector"); -fields("post_connector") -> - fields("config_connector"). +fields(Field) when + Field == "get_connector"; + Field == "put_connector"; + Field == "post_connector" +-> + fields({Field, ?CONNECTOR_TYPE}); +fields({Field, Type}) when + Field == "get_connector"; + Field == "put_connector"; + Field == "post_connector" +-> + emqx_connector_schema:api_fields(Field, Type, fields("connection_fields")). server() -> Meta = #{desc => ?DESC("server")}, @@ -94,7 +102,7 @@ connector_examples(Method) -> #{ <<"pgsql">> => #{ summary => <<"PostgreSQL Connector">>, - value => values({Method, pgsql}) + value => values({Method, <<"pgsql">>}) } } ]. @@ -109,20 +117,21 @@ values({get, PostgreSQLType}) -> node => <<"emqx@localhost">>, status => <<"connected">> } - ] + ], + actions => [<<"my_action">>] }, values({post, PostgreSQLType}) ); values({post, PostgreSQLType}) -> - values({put, PostgreSQLType}); -values({put, PostgreSQLType}) -> maps:merge( #{ - name => <<"my_action">>, + name => <<"my_", PostgreSQLType/binary, "_connector">>, type => PostgreSQLType }, values(common) ); +values({put, _PostgreSQLType}) -> + values(common); values(common) -> #{ <<"database">> => <<"emqx_data">>, diff --git a/rel/i18n/emqx_connector_schema.hocon b/rel/i18n/emqx_connector_schema.hocon index d3aa1c82b..16d153e12 100644 --- a/rel/i18n/emqx_connector_schema.hocon +++ b/rel/i18n/emqx_connector_schema.hocon @@ -10,9 +10,54 @@ connector_field.desc: connector_field.label: """Connector""" +desc_name.desc: +"""The name of the connector.""" + +desc_name.label: +"""Connector Name""" + +desc_type.desc: +"""The type of the connector.""" + +desc_type.label: +"""Connector Type""" + config_enable.desc: """Enable (true) or disable (false) this connector.""" config_enable.label: """Enable or Disable""" +desc_node_name.desc: +"""The node name.""" + +desc_node_name.label: +"""Node Name""" + +desc_node_status.desc: +"""Node status.""" + +desc_node_status.label: +"""Node Status""" + +desc_status.desc: +"""The status of the connector
+- connecting: the initial state before any health probes were made.
+- connected: when the connector passes the health probes.
+- disconnected: when the connector can not pass health probes.
+- inconsistent: When not all the nodes are at the same status.""" + +desc_status.label: +"""Connector Status""" + +desc_status_reason.desc: +"""This is the reason given in case a connector is failing to connect.""" + +desc_status_reason.label: +"""Failure reason""" + +connector_actions.desc: +"""List of actions added to this connector.""" + +connector_actions.label: +"""Actions""" } From 64ee29af817018dbdacbc79e513508967c364fa7 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Thu, 23 Nov 2023 23:10:49 +0200 Subject: [PATCH 18/20] feat(emqx_management): implement data backup API --- apps/emqx/priv/bpapi.versions | 1 + .../src/emqx_mgmt_api_data_backup.erl | 361 ++++++++++++++++++ .../src/emqx_mgmt_data_backup.erl | 211 ++++++++-- .../proto/emqx_mgmt_data_backup_proto_v1.erl | 51 +++ .../test/emqx_mgmt_api_data_backup_SUITE.erl | 355 +++++++++++++++++ .../emqx-export-bad-file.tar.gz | Bin 0 -> 2963 bytes .../emqx-export-bad-upload.tar.gz | Bin 0 -> 2616 bytes .../emqx-export-upload-ce.tar.gz | Bin 0 -> 2604 bytes .../emqx-export-upload-ee.tar.gz | Bin 0 -> 3073 bytes changes/ce/feat-12017.en.md | 1 + 10 files changed, 950 insertions(+), 30 deletions(-) create mode 100644 apps/emqx_management/src/emqx_mgmt_api_data_backup.erl create mode 100644 apps/emqx_management/src/proto/emqx_mgmt_data_backup_proto_v1.erl create mode 100644 apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE.erl create mode 100644 apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE_data/emqx-export-bad-file.tar.gz create mode 100644 apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE_data/emqx-export-bad-upload.tar.gz create mode 100644 apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE_data/emqx-export-upload-ce.tar.gz create mode 100644 apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE_data/emqx-export-upload-ee.tar.gz create mode 100644 changes/ce/feat-12017.en.md diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index ea4ce159d..e2c726fe5 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -39,6 +39,7 @@ {emqx_mgmt_api_plugins,2}. {emqx_mgmt_cluster,1}. {emqx_mgmt_cluster,2}. +{emqx_mgmt_data_backup,1}. {emqx_mgmt_trace,1}. {emqx_mgmt_trace,2}. {emqx_node_rebalance,1}. diff --git a/apps/emqx_management/src/emqx_mgmt_api_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_api_data_backup.erl new file mode 100644 index 000000000..4cd7d404e --- /dev/null +++ b/apps/emqx_management/src/emqx_mgmt_api_data_backup.erl @@ -0,0 +1,361 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mgmt_api_data_backup). + +-behaviour(minirest_api). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-export([api_spec/0, paths/0, schema/1, fields/1]). + +-export([ + data_export/2, + data_import/2, + data_files/2, + data_file_by_name/2 +]). + +-define(TAGS, [<<"Data Backup">>]). + +-define(BAD_REQUEST, 'BAD_REQUEST'). +-define(NOT_FOUND, 'NOT_FOUND'). + +-define(node_field(IsRequired), ?node_field(IsRequired, #{})). +-define(node_field(IsRequired, Meta), + {node, ?HOCON(binary(), Meta#{desc => "Node name", required => IsRequired})} +). +-define(filename_field(IsRequired), ?filename_field(IsRequired, #{})). +-define(filename_field(IsRequired, Meta), + {filename, + ?HOCON(binary(), Meta#{ + desc => "Data backup file name", + required => IsRequired + })} +). + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + [ + "/data/export", + "/data/import", + "/data/files", + "/data/files/:filename" + ]. + +schema("/data/export") -> + #{ + 'operationId' => data_export, + post => #{ + tags => ?TAGS, + desc => <<"Export a data backup file">>, + responses => #{ + 200 => + emqx_dashboard_swagger:schema_with_example( + ?R_REF(backup_file_info), + backup_file_info_example() + ) + } + } + }; +schema("/data/import") -> + #{ + 'operationId' => data_import, + post => #{ + tags => ?TAGS, + desc => <<"Import a data backup file">>, + 'requestBody' => emqx_dashboard_swagger:schema_with_example( + ?R_REF(import_request_body), + maps:with([node, filename], backup_file_info_example()) + ), + + responses => #{ + 204 => <<"No Content">>, + 400 => emqx_dashboard_swagger:error_codes( + [?BAD_REQUEST], <<"Backup file import failed">> + ) + } + } + }; +schema("/data/files") -> + #{ + 'operationId' => data_files, + post => #{ + tags => ?TAGS, + desc => <<"Upload a data backup file">>, + 'requestBody' => emqx_dashboard_swagger:file_schema(filename), + responses => #{ + 204 => <<"No Content">>, + 400 => emqx_dashboard_swagger:error_codes( + [?BAD_REQUEST], <<"Bad backup file">> + ) + } + }, + get => #{ + tags => ?TAGS, + desc => <<"List backup files">>, + parameters => [ + ?R_REF(emqx_dashboard_swagger, page), + ?R_REF(emqx_dashboard_swagger, limit) + ], + responses => #{ + 200 => + emqx_dashboard_swagger:schema_with_example( + ?R_REF(files_response), + files_response_example() + ) + } + } + }; +schema("/data/files/:filename") -> + #{ + 'operationId' => data_file_by_name, + get => #{ + tags => ?TAGS, + desc => <<"Download a data backup file">>, + parameters => [ + ?filename_field(true, #{in => path}), + ?node_field(false, #{in => query}) + ], + responses => #{ + 200 => ?HOCON(binary), + 400 => emqx_dashboard_swagger:error_codes( + [?BAD_REQUEST], <<"Bad request">> + ), + 404 => emqx_dashboard_swagger:error_codes( + [?NOT_FOUND], <<"Backup file not found">> + ) + } + }, + delete => #{ + tags => ?TAGS, + desc => <<"Delete a data backup file">>, + parameters => [ + ?filename_field(true, #{in => path}), + ?node_field(false, #{in => query}) + ], + responses => #{ + 204 => <<"No Content">>, + 400 => emqx_dashboard_swagger:error_codes( + [?BAD_REQUEST], <<"Bad request">> + ), + 404 => emqx_dashboard_swagger:error_codes( + [?NOT_FOUND], <<"Backup file not found">> + ) + } + } + }. + +fields(files_response) -> + [ + {data, ?ARRAY(?R_REF(backup_file_info))}, + {meta, ?R_REF(emqx_dashboard_swagger, meta)} + ]; +fields(backup_file_info) -> + [ + ?node_field(true), + ?filename_field(true), + {created_at, + ?HOCON(binary(), #{ + desc => "Data backup file creation date and time", + required => true + })} + ]; +fields(import_request_body) -> + [?node_field(false), ?filename_field(true)]; +fields(data_backup_file) -> + [ + ?filename_field(true), + {file, + ?HOCON(binary(), #{ + desc => "Data backup file content", + required => true + })} + ]. + +%%------------------------------------------------------------------------------ +%% HTTP API Callbacks +%%------------------------------------------------------------------------------ + +data_export(post, _Request) -> + case emqx_mgmt_data_backup:export() of + {ok, #{filename := FileName} = File} -> + {200, File#{filename => filename:basename(FileName)}}; + Error -> + Error + end. + +data_import(post, #{body := #{<<"filename">> := FileName} = Body}) -> + case safe_parse_node(Body) of + {error, Msg} -> + {400, #{code => 'BAD_REQUEST', message => Msg}}; + FileNode -> + CoreNode = core_node(FileNode), + response( + emqx_mgmt_data_backup_proto_v1:import_file(CoreNode, FileNode, FileName, infinity) + ) + end. + +core_node(FileNode) -> + case mria_rlog:role(FileNode) of + core -> + FileNode; + replicant -> + case mria_rlog:role() of + core -> + node(); + replicant -> + mria_membership:coordinator() + end + end. + +data_files(post, #{body := #{<<"filename">> := #{type := _} = File}}) -> + [{FileName, FileContent} | _] = maps:to_list(maps:without([type], File)), + case emqx_mgmt_data_backup:upload(FileName, FileContent) of + ok -> + {204}; + {error, Reason} -> + {400, #{code => 'BAD_REQUEST', message => emqx_mgmt_data_backup:format_error(Reason)}} + end; +data_files(get, #{query_string := PageParams}) -> + case emqx_mgmt_api:parse_pager_params(PageParams) of + false -> + {400, #{code => ?BAD_REQUEST, message => <<"page_limit_invalid">>}}; + #{page := Page, limit := Limit} = Pager -> + {200, #{data => list_backup_files(Page, Limit), meta => Pager}} + end. + +data_file_by_name(Method, #{bindings := #{filename := Filename}, query_string := QS}) -> + case safe_parse_node(QS) of + {error, Msg} -> + {400, #{code => 'BAD_REQUEST', message => Msg}}; + Node -> + case get_or_delete_file(Method, Filename, Node) of + {error, not_found} -> + {404, #{ + code => ?NOT_FOUND, message => emqx_mgmt_data_backup:format_error(not_found) + }}; + Other -> + response(Other) + end + end. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +get_or_delete_file(get, Filename, Node) -> + emqx_mgmt_data_backup_proto_v1:read_file(Node, Filename, infinity); +get_or_delete_file(delete, Filename, Node) -> + emqx_mgmt_data_backup_proto_v1:delete_file(Node, Filename, infinity). + +safe_parse_node(#{<<"node">> := NodeBin}) -> + NodesBin = [erlang:atom_to_binary(N, utf8) || N <- emqx:running_nodes()], + case lists:member(NodeBin, NodesBin) of + true -> erlang:binary_to_atom(NodeBin, utf8); + false -> {error, io_lib:format("Unknown node: ~s", [NodeBin])} + end; +safe_parse_node(_) -> + node(). + +response({ok, #{db_errors := DbErrs, config_errors := ConfErrs}}) -> + case DbErrs =:= #{} andalso ConfErrs =:= #{} of + true -> + {204}; + false -> + DbErrs1 = emqx_mgmt_data_backup:format_db_errors(DbErrs), + ConfErrs1 = emqx_mgmt_data_backup:format_conf_errors(ConfErrs), + Msg = unicode:characters_to_binary(io_lib:format("~s", [DbErrs1 ++ ConfErrs1])), + {400, #{code => ?BAD_REQUEST, message => Msg}} + end; +response({ok, Res}) -> + {200, Res}; +response(ok) -> + {204}; +response({error, Reason}) -> + {400, #{code => ?BAD_REQUEST, message => emqx_mgmt_data_backup:format_error(Reason)}}. + +list_backup_files(Page, Limit) -> + Start = Page * Limit - Limit + 1, + lists:sublist(list_backup_files(), Start, Limit). + +list_backup_files() -> + Nodes = emqx:running_nodes(), + Results = emqx_mgmt_data_backup_proto_v1:list_files(Nodes, 30_0000), + NodeResults = lists:zip(Nodes, Results), + {Successes, Failures} = + lists:partition( + fun({_Node, Result}) -> + case Result of + {ok, _} -> true; + _ -> false + end + end, + NodeResults + ), + case Failures of + [] -> + ok; + [_ | _] -> + ?SLOG(error, #{msg => "list_exported_backup_files_failed", node_errors => Failures}) + end, + FileList = [FileInfo || {_Node, {ok, FileInfos}} <- Successes, FileInfo <- FileInfos], + lists:sort( + fun(#{created_at_sec := T1, filename := F1}, #{created_at_sec := T2, filename := F2}) -> + case T1 =:= T2 of + true -> F1 >= F2; + false -> T1 > T2 + end + end, + FileList + ). + +backup_file_info_example() -> + #{ + created_at => <<"2023-11-23T19:13:19+02:00">>, + created_at_sec => 1700759599, + filename => <<"emqx-export-2023-11-23-19-13-19.043.tar.gz">>, + node => 'emqx@127.0.0.1', + size => 22740 + }. + +files_response_example() -> + #{ + data => [ + #{ + created_at => <<"2023-09-02T11:11:33+02:00">>, + created_at_sec => 1693645893, + filename => <<"emqx-export-2023-09-02-11-11-33.012.tar.gz">>, + node => 'emqx@127.0.0.1', + size => 22740 + }, + #{ + created_at => <<"2023-11-23T19:13:19+02:00">>, + created_at_sec => 1700759599, + filename => <<"emqx-export-2023-11-23-19-13-19.043.tar.gz">>, + node => 'emqx@127.0.0.1', + size => 22740 + } + ], + meta => #{ + page => 0, + limit => 20, + count => 300 + } + }. diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 9825b26cf..e0887d788 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -24,8 +24,21 @@ format_error/1 ]). +%% HTTP API +-export([ + upload/2, + maybe_copy_and_import/2, + read_file/1, + delete_file/1, + list_files/0, + format_conf_errors/1, + format_db_errors/1 +]). + -export([default_validate_mnesia_backup/1]). +-export_type([import_res/0]). + -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). @@ -80,17 +93,21 @@ end end() ). +-define(backup_path(_FileName_), filename:join(root_backup_dir(), _FileName_)). -type backup_file_info() :: #{ - filename => binary(), - size => non_neg_integer(), - created_at => binary(), - node => node(), + filename := binary(), + size := non_neg_integer(), + created_at := binary(), + created_at_sec := integer(), + node := node(), atom() => _ }. -type db_error_details() :: #{mria:table() => {error, _}}. -type config_error_details() :: #{emqx_utils_maps:config_path() => {error, _}}. +-type import_res() :: + {ok, #{db_errors => db_error_details(), config_errors => config_error_details()}} | {error, _}. %%------------------------------------------------------------------------------ %% APIs @@ -120,15 +137,11 @@ export(Opts) -> file:del_dir_r(BackupName) end. --spec import(file:filename_all()) -> - {ok, #{db_errors => db_error_details(), config_errors => config_error_details()}} - | {error, _}. +-spec import(file:filename_all()) -> import_res(). import(BackupFileName) -> import(BackupFileName, ?DEFAULT_OPTS). --spec import(file:filename_all(), map()) -> - {ok, #{db_errors => db_error_details(), config_errors => config_error_details()}} - | {error, _}. +-spec import(file:filename_all(), map()) -> import_res(). import(BackupFileName, Opts) -> case is_import_allowed() of true -> @@ -142,6 +155,74 @@ import(BackupFileName, Opts) -> {error, not_core_node} end. +-spec maybe_copy_and_import(node(), file:filename_all()) -> import_res(). +maybe_copy_and_import(FileNode, BackupFileName) when FileNode =:= node() -> + import(BackupFileName, #{}); +maybe_copy_and_import(FileNode, BackupFileName) -> + %% The file can be already present locally + case filelib:is_file(?backup_path(str(BackupFileName))) of + true -> + import(BackupFileName, #{}); + false -> + copy_and_import(FileNode, BackupFileName) + end. + +-spec read_file(file:filename_all()) -> + {ok, #{filename => file:filename_all(), file => binary()}} | {error, _}. +read_file(BackupFileName) -> + BackupFileNameStr = str(BackupFileName), + case validate_backup_name(BackupFileNameStr) of + ok -> + maybe_not_found(file:read_file(?backup_path(BackupFileName))); + Err -> + Err + end. + +-spec delete_file(file:filename_all()) -> ok | {error, _}. +delete_file(BackupFileName) -> + BackupFileNameStr = str(BackupFileName), + case validate_backup_name(BackupFileNameStr) of + ok -> + maybe_not_found(file:delete(?backup_path(BackupFileName))); + Err -> + Err + end. + +-spec upload(file:filename_all(), binary()) -> ok | {error, _}. +upload(BackupFileName, BackupFileContent) -> + BackupFileNameStr = str(BackupFileName), + FilePath = ?backup_path(BackupFileNameStr), + case filelib:is_file(FilePath) of + true -> + {error, {already_exists, BackupFileNameStr}}; + false -> + do_upload(BackupFileNameStr, BackupFileContent) + end. + +-spec list_files() -> [backup_file_info()]. +list_files() -> + Filter = + fun(File) -> + case file:read_file_info(File, [{time, posix}]) of + {ok, #file_info{size = Size, ctime = CTimeSec}} -> + BaseFilename = bin(filename:basename(File)), + Info = #{ + filename => BaseFilename, + size => Size, + created_at => emqx_utils_calendar:epoch_to_rfc3339(CTimeSec, second), + created_at_sec => CTimeSec, + node => node() + }, + {true, Info}; + _ -> + false + end + end, + lists:filtermap(Filter, backup_files()). + +backup_files() -> + filelib:wildcard(?backup_path("*" ++ ?TAR_SUFFIX)). + format_error(not_core_node) -> str( io_lib:format( @@ -170,13 +251,83 @@ format_error({unsupported_version, ImportVersion}) -> [str(ImportVersion), str(emqx_release:version())] ) ); +format_error({already_exists, BackupFileName}) -> + str(io_lib:format("Backup file \"~s\" already exists", [BackupFileName])); format_error(Reason) -> Reason. +format_conf_errors(Errors) -> + Opts = #{print_fun => fun io_lib:format/2}, + maps:values(maps:map(conf_error_formatter(Opts), Errors)). + +format_db_errors(Errors) -> + Opts = #{print_fun => fun io_lib:format/2}, + maps:values( + maps:map( + fun(Tab, Err) -> maybe_print_mnesia_import_err(Tab, Err, Opts) end, + Errors + ) + ). + %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ +copy_and_import(FileNode, BackupFileName) -> + case emqx_mgmt_data_backup_proto_v1:read_file(FileNode, BackupFileName, infinity) of + {ok, BackupFileContent} -> + case upload(BackupFileName, BackupFileContent) of + ok -> + import(BackupFileName, #{}); + Err -> + Err + end; + Err -> + Err + end. + +%% compatibility with import API that uses lookup_file/1 and returns `not_found` reason +maybe_not_found({error, enoent}) -> + {error, not_found}; +maybe_not_found(Other) -> + Other. + +do_upload(BackupFileNameStr, BackupFileContent) -> + FilePath = ?backup_path(BackupFileNameStr), + BackupDir = ?backup_path(filename:basename(BackupFileNameStr, ?TAR_SUFFIX)), + try + ok = validate_backup_name(BackupFileNameStr), + ok = file:write_file(FilePath, BackupFileContent), + ok = extract_backup(FilePath), + {ok, _} = validate_backup(BackupDir), + HoconFileName = filename:join(BackupDir, ?CLUSTER_HOCON_FILENAME), + case filelib:is_regular(HoconFileName) of + true -> + {ok, RawConf} = hocon:files([HoconFileName]), + RawConf1 = upgrade_raw_conf(emqx_conf:schema_module(), RawConf), + {ok, _} = validate_cluster_hocon(RawConf1), + ok; + false -> + %% cluster.hocon can be missing in the backup + ok + end, + ?SLOG(info, #{msg => "emqx_data_upload_success"}) + catch + error:{badmatch, {error, Reason}}:Stack -> + ?SLOG(error, #{msg => "emqx_data_upload_failed", reason => Reason, stacktrace => Stack}), + {error, Reason}; + Class:Reason:Stack -> + ?SLOG(error, #{ + msg => "emqx_data_upload_failed", + exception => Class, + reason => Reason, + stacktrace => Stack + }), + {error, Reason} + after + file:del_dir_r(BackupDir) + end. + prepare_new_backup(Opts) -> Ts = erlang:system_time(millisecond), {{Y, M, D}, {HH, MM, SS}} = local_datetime(Ts), @@ -186,7 +337,7 @@ prepare_new_backup(Opts) -> [Y, M, D, HH, MM, SS, Ts rem 1000] ) ), - BackupName = filename:join(root_backup_dir(), BackupBaseName), + BackupName = ?backup_path(BackupBaseName), BackupTarName = ?tar(BackupName), maybe_print("Exporting data to ~p...~n", [BackupTarName], Opts), {ok, TarDescriptor} = ?fmt_tar_err(erl_tar:open(BackupTarName, [write, compressed])), @@ -208,13 +359,13 @@ do_export(BackupName, TarDescriptor, Opts) -> ok = ?fmt_tar_err(erl_tar:close(TarDescriptor)), {ok, #file_info{ size = Size, - ctime = {{Y1, M1, D1}, {H1, MM1, S1}} - }} = file:read_file_info(BackupTarName), - CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y1, M1, D1, H1, MM1, S1]), + ctime = CTime + }} = file:read_file_info(BackupTarName, [{time, posix}]), {ok, #{ filename => bin(BackupTarName), size => Size, - created_at => bin(CreatedAt), + created_at => emqx_utils_calendar:epoch_to_rfc3339(CTime, second), + created_at_sec => CTime, node => node() }}. @@ -351,7 +502,7 @@ parse_version_no_patch(VersionBin) -> end. do_import(BackupFileName, Opts) -> - BackupDir = filename:join(root_backup_dir(), filename:basename(BackupFileName, ?TAR_SUFFIX)), + BackupDir = ?backup_path(filename:basename(BackupFileName, ?TAR_SUFFIX)), maybe_print("Importing data from ~p...~n", [BackupFileName], Opts), try ok = validate_backup_name(BackupFileName), @@ -619,7 +770,7 @@ validate_cluster_hocon(RawConf) -> do_import_conf(RawConf, Opts) -> GenConfErrs = filter_errors(maps:from_list(import_generic_conf(RawConf))), - maybe_print_errors(GenConfErrs, Opts), + maybe_print_conf_errors(GenConfErrs, Opts), Errors = lists:foldl( fun(Module, ErrorsAcc) -> @@ -634,7 +785,7 @@ do_import_conf(RawConf, Opts) -> GenConfErrs, sort_importer_modules(find_behaviours(emqx_config_backup)) ), - maybe_print_errors(Errors, Opts), + maybe_print_conf_errors(Errors, Opts), Errors. sort_importer_modules(Modules) -> @@ -677,17 +828,17 @@ maybe_print_changed(Changed, Opts) -> Changed ). -maybe_print_errors(Errors, Opts) -> - maps:foreach( - fun(Path, Err) -> - maybe_print( - "Failed to import the following config path: ~p, reason: ~p~n", - [pretty_path(Path), Err], - Opts - ) - end, - Errors - ). +maybe_print_conf_errors(Errors, Opts) -> + maps:foreach(conf_error_formatter(Opts), Errors). + +conf_error_formatter(Opts) -> + fun(Path, Err) -> + maybe_print( + "Failed to import the following config path: ~p, reason: ~p~n", + [pretty_path(Path), Err], + Opts + ) + end. filter_errors(Results) -> maps:filter( @@ -727,7 +878,7 @@ lookup_file(FileName) -> %% Only lookup by basename, don't allow to lookup by file path case FileName =:= filename:basename(FileName) of true -> - FilePath = filename:join(root_backup_dir(), FileName), + FilePath = ?backup_path(FileName), case filelib:is_file(FilePath) of true -> {ok, FilePath}; false -> {error, not_found} diff --git a/apps/emqx_management/src/proto/emqx_mgmt_data_backup_proto_v1.erl b/apps/emqx_management/src/proto/emqx_mgmt_data_backup_proto_v1.erl new file mode 100644 index 000000000..066961c00 --- /dev/null +++ b/apps/emqx_management/src/proto/emqx_mgmt_data_backup_proto_v1.erl @@ -0,0 +1,51 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mgmt_data_backup_proto_v1). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + import_file/4, + list_files/2, + read_file/3, + delete_file/3 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.4.0". + +-spec list_files([node()], timeout()) -> + emqx_rpc:erpc_multicall({non_neg_integer(), map()}). +list_files(Nodes, Timeout) -> + erpc:multicall(Nodes, emqx_mgmt_data_backup, list_files, [], Timeout). + +-spec import_file(node(), node(), binary(), timeout()) -> + emqx_mgmt_data_backup:import_res() | {badrpc, _}. +import_file(Node, FileNode, FileName, Timeout) -> + rpc:call(Node, emqx_mgmt_data_backup, maybe_copy_and_import, [FileNode, FileName], Timeout). + +-spec read_file(node(), binary(), timeout()) -> + {ok, binary()} | {error, _} | {bardrpc, _}. +read_file(Node, FileName, Timeout) -> + rpc:call(Node, emqx_mgmt_data_backup, read_file, [FileName], Timeout). + +-spec delete_file(node(), binary(), timeout()) -> ok | {error, _} | {bardrpc, _}. +delete_file(Node, FileName, Timeout) -> + rpc:call(Node, emqx_mgmt_data_backup, delete_file, [FileName], Timeout). diff --git a/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE.erl new file mode 100644 index 000000000..cef32ab92 --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE.erl @@ -0,0 +1,355 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mgmt_api_data_backup_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-define(NODE1_PORT, 18085). +-define(NODE2_PORT, 18086). +-define(NODE3_PORT, 18087). +-define(api_base_url(_Port_), ("http://127.0.0.1:" ++ (integer_to_list(_Port_)))). + +-define(UPLOAD_EE_BACKUP, "emqx-export-upload-ee.tar.gz"). +-define(UPLOAD_CE_BACKUP, "emqx-export-upload-ce.tar.gz"). +-define(BAD_UPLOAD_BACKUP, "emqx-export-bad-upload.tar.gz"). +-define(BAD_IMPORT_BACKUP, "emqx-export-bad-file.tar.gz"). +-define(backup_path(_Config_, _BackupName_), + filename:join(?config(data_dir, _Config_), _BackupName_) +). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Config. + +end_per_suite(_) -> + ok. + +init_per_testcase(TC, Config) when + TC =:= t_upload_ee_backup; + TC =:= t_import_ee_backup +-> + case emqx_release:edition() of + ee -> do_init_per_testcase(TC, Config); + ce -> Config + end; +init_per_testcase(TC, Config) -> + do_init_per_testcase(TC, Config). + +end_per_testcase(_TC, Config) -> + case ?config(cluster, Config) of + undefined -> ok; + Cluster -> emqx_cth_cluster:stop(Cluster) + end. + +t_export_backup(Config) -> + Auth = ?config(auth, Config), + export_test(?NODE1_PORT, Auth), + export_test(?NODE2_PORT, Auth), + export_test(?NODE3_PORT, Auth). + +t_delete_backup(Config) -> + test_file_op(delete, Config). + +t_get_backup(Config) -> + test_file_op(get, Config). + +t_list_backups(Config) -> + Auth = ?config(auth, Config), + + [{ok, _} = export_backup(?NODE1_PORT, Auth) || _ <- lists:seq(1, 10)], + [{ok, _} = export_backup(?NODE2_PORT, Auth) || _ <- lists:seq(1, 10)], + + {ok, RespBody} = list_backups(?NODE1_PORT, Auth, <<"1">>, <<"100">>), + #{<<"data">> := Data, <<"meta">> := _} = emqx_utils_json:decode(RespBody), + ?assertEqual(20, length(Data)), + + {ok, EmptyRespBody} = list_backups(?NODE2_PORT, Auth, <<"2">>, <<"100">>), + #{<<"data">> := EmptyData, <<"meta">> := _} = emqx_utils_json:decode(EmptyRespBody), + ?assertEqual(0, length(EmptyData)), + + {ok, RespBodyP1} = list_backups(?NODE3_PORT, Auth, <<"1">>, <<"10">>), + {ok, RespBodyP2} = list_backups(?NODE3_PORT, Auth, <<"2">>, <<"10">>), + {ok, RespBodyP3} = list_backups(?NODE3_PORT, Auth, <<"3">>, <<"10">>), + + #{<<"data">> := DataP1, <<"meta">> := _} = emqx_utils_json:decode(RespBodyP1), + ?assertEqual(10, length(DataP1)), + #{<<"data">> := DataP2, <<"meta">> := _} = emqx_utils_json:decode(RespBodyP2), + ?assertEqual(10, length(DataP2)), + #{<<"data">> := DataP3, <<"meta">> := _} = emqx_utils_json:decode(RespBodyP3), + ?assertEqual(0, length(DataP3)), + + ?assertEqual(Data, DataP1 ++ DataP2). + +t_upload_ce_backup(Config) -> + upload_backup_test(Config, ?UPLOAD_CE_BACKUP). + +t_upload_ee_backup(Config) -> + case emqx_release:edition() of + ee -> upload_backup_test(Config, ?UPLOAD_EE_BACKUP); + ce -> ok + end. + +t_import_ce_backup(Config) -> + import_backup_test(Config, ?UPLOAD_CE_BACKUP). + +t_import_ee_backup(Config) -> + case emqx_release:edition() of + ee -> import_backup_test(Config, ?UPLOAD_EE_BACKUP); + ce -> ok + end. + +do_init_per_testcase(TC, Config) -> + Cluster = [Core1, _Core2, Repl] = cluster(TC, Config), + Auth = auth_header(Core1), + ok = wait_for_auth_replication(Repl), + [{auth, Auth}, {cluster, Cluster} | Config]. + +test_file_op(Method, Config) -> + Auth = ?config(auth, Config), + + {ok, Node1Resp} = export_backup(?NODE1_PORT, Auth), + {ok, Node2Resp} = export_backup(?NODE2_PORT, Auth), + {ok, Node3Resp} = export_backup(?NODE3_PORT, Auth), + + ParsedResps = [emqx_utils_json:decode(R) || R <- [Node1Resp, Node2Resp, Node3Resp]], + + [Node1Parsed, Node2Parsed, Node3Parsed] = ParsedResps, + + %% node param is not set in Query, expect get/delete the backup on the local node + F1 = fun() -> + backup_file_op(Method, ?NODE1_PORT, Auth, maps:get(<<"filename">>, Node1Parsed), []) + end, + ?assertMatch({ok, _}, F1()), + assert_second_call(Method, F1()), + + %% Node 2 must get/delete the backup on Node 3 via rpc + F2 = fun() -> + backup_file_op( + Method, + ?NODE2_PORT, + Auth, + maps:get(<<"filename">>, Node3Parsed), + [{<<"node">>, maps:get(<<"node">>, Node3Parsed)}] + ) + end, + ?assertMatch({ok, _}, F2()), + assert_second_call(Method, F2()), + + %% The same as above but nodes are switched + F3 = fun() -> + backup_file_op( + Method, + ?NODE3_PORT, + Auth, + maps:get(<<"filename">>, Node2Parsed), + [{<<"node">>, maps:get(<<"node">>, Node2Parsed)}] + ) + end, + ?assertMatch({ok, _}, F3()), + assert_second_call(Method, F3()). + +export_test(NodeApiPort, Auth) -> + {ok, RespBody} = export_backup(NodeApiPort, Auth), + #{ + <<"created_at">> := _, + <<"created_at_sec">> := CreatedSec, + <<"filename">> := _, + <<"node">> := _, + <<"size">> := Size + } = emqx_utils_json:decode(RespBody), + ?assert(is_integer(Size)), + ?assert(is_integer(CreatedSec) andalso CreatedSec > 0). + +upload_backup_test(Config, BackupName) -> + Auth = ?config(auth, Config), + UploadFile = ?backup_path(Config, BackupName), + BadImportFile = ?backup_path(Config, ?BAD_IMPORT_BACKUP), + BadUploadFile = ?backup_path(Config, ?BAD_UPLOAD_BACKUP), + + ?assertEqual(ok, upload_backup(?NODE3_PORT, Auth, UploadFile)), + %% This file was specially forged to pass upload validation bat fail on import + ?assertEqual(ok, upload_backup(?NODE2_PORT, Auth, BadImportFile)), + ?assertEqual({error, bad_request}, upload_backup(?NODE1_PORT, Auth, BadUploadFile)). + +import_backup_test(Config, BackupName) -> + Auth = ?config(auth, Config), + UploadFile = ?backup_path(Config, BackupName), + BadImportFile = ?backup_path(Config, ?BAD_IMPORT_BACKUP), + + ?assertEqual(ok, upload_backup(?NODE3_PORT, Auth, UploadFile)), + + %% This file was specially forged to pass upload validation bat fail on import + ?assertEqual(ok, upload_backup(?NODE2_PORT, Auth, BadImportFile)), + + %% Replicant node must be able to import the file by doing rpc to a core node + ?assertMatch({ok, _}, import_backup(?NODE3_PORT, Auth, BackupName)), + + [N1, N2, N3] = ?config(cluster, Config), + + ?assertMatch({ok, _}, import_backup(?NODE3_PORT, Auth, BackupName)), + + ?assertMatch({ok, _}, import_backup(?NODE1_PORT, Auth, BackupName, N3)), + %% Now this node must also have the file locally + ?assertMatch({ok, _}, import_backup(?NODE1_PORT, Auth, BackupName, N1)), + + ?assertMatch({error, {_, 400, _}}, import_backup(?NODE2_PORT, Auth, ?BAD_IMPORT_BACKUP, N2)). + +assert_second_call(get, Res) -> + ?assertMatch({ok, _}, Res); +assert_second_call(delete, Res) -> + ?assertMatch({error, {_, 404, _}}, Res). + +export_backup(NodeApiPort, Auth) -> + Path = ["data", "export"], + request(post, NodeApiPort, Path, Auth). + +import_backup(NodeApiPort, Auth, BackupName) -> + import_backup(NodeApiPort, Auth, BackupName, undefined). + +import_backup(NodeApiPort, Auth, BackupName, Node) -> + Path = ["data", "import"], + Body = #{<<"filename">> => unicode:characters_to_binary(BackupName)}, + Body1 = + case Node of + undefined -> Body; + _ -> Body#{<<"node">> => Node} + end, + request(post, NodeApiPort, Path, Body1, Auth). + +list_backups(NodeApiPort, Auth, Page, Limit) -> + Path = ["data", "files"], + request(get, NodeApiPort, Path, [{<<"page">>, Page}, {<<"limit">>, Limit}], [], Auth). + +backup_file_op(Method, NodeApiPort, Auth, BackupName, QueryList) -> + Path = ["data", "files", BackupName], + request(Method, NodeApiPort, Path, QueryList, [], Auth). + +upload_backup(NodeApiPort, Auth, BackupFilePath) -> + Path = emqx_mgmt_api_test_util:api_path(?api_base_url(NodeApiPort), ["data", "files"]), + Res = emqx_mgmt_api_test_util:upload_request( + Path, + BackupFilePath, + "filename", + <<"application/octet-stream">>, + [], + Auth + ), + case Res of + {ok, {{"HTTP/1.1", 204, _}, _Headers, _}} -> + ok; + {ok, {{"HTTP/1.1", 400, _}, _Headers, _} = Resp} -> + ct:pal("Backup upload failed: ~p", [Resp]), + {error, bad_request}; + Err -> + Err + end. + +request(Method, NodePort, PathParts, Auth) -> + request(Method, NodePort, PathParts, [], [], Auth). + +request(Method, NodePort, PathParts, Body, Auth) -> + request(Method, NodePort, PathParts, [], Body, Auth). + +request(Method, NodePort, PathParts, QueryList, Body, Auth) -> + Path = emqx_mgmt_api_test_util:api_path(?api_base_url(NodePort), PathParts), + Query = unicode:characters_to_list(uri_string:compose_query(QueryList)), + emqx_mgmt_api_test_util:request_api(Method, Path, Query, Auth, Body). + +cluster(TC, Config) -> + Nodes = emqx_cth_cluster:start( + [ + {api_data_backup_core1, #{role => core, apps => apps_spec(18085, TC)}}, + {api_data_backup_core2, #{role => core, apps => apps_spec(18086, TC)}}, + {api_data_backup_replicant, #{role => replicant, apps => apps_spec(18087, TC)}} + ], + #{work_dir => emqx_cth_suite:work_dir(TC, Config)} + ), + Nodes. + +auth_header(Node) -> + {ok, API} = erpc:call(Node, emqx_common_test_http, create_default_app, []), + emqx_common_test_http:auth_header(API). + +wait_for_auth_replication(ReplNode) -> + wait_for_auth_replication(ReplNode, 100). + +wait_for_auth_replication(ReplNode, 0) -> + {error, {ReplNode, auth_not_ready}}; +wait_for_auth_replication(ReplNode, Retries) -> + try + {_Header, _Val} = erpc:call(ReplNode, emqx_common_test_http, default_auth_header, []), + ok + catch + _:_ -> + timer:sleep(1), + wait_for_auth_replication(ReplNode, Retries - 1) + end. + +apps_spec(APIPort, TC) -> + common_apps_spec() ++ + app_spec_dashboard(APIPort) ++ + upload_import_apps_spec(TC). + +common_apps_spec() -> + [ + emqx, + emqx_conf, + emqx_management + ]. + +app_spec_dashboard(APIPort) -> + [ + {emqx_dashboard, #{ + config => + #{ + dashboard => + #{ + listeners => + #{ + http => + #{bind => APIPort} + }, + default_username => "", + default_password => "" + } + } + }} + ]. + +upload_import_apps_spec(TC) when + TC =:= t_upload_ee_backup; + TC =:= t_import_ee_backup; + TC =:= t_upload_ce_backup; + TC =:= t_import_ce_backup +-> + [ + emqx_auth, + emqx_auth_http, + emqx_auth_jwt, + emqx_auth_mnesia, + emqx_rule_engine, + emqx_modules, + emqx_bridge + ]; +upload_import_apps_spec(_TC) -> + []. diff --git a/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE_data/emqx-export-bad-file.tar.gz b/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE_data/emqx-export-bad-file.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..ee56fcbe6037617457acab9c85f3affdffdfc4df GIT binary patch literal 2963 zcmV;E3vBcsiwFP!000001MORBj2uT5?mfMkbz;W^tVsFM8rzC3_UNA5mW*Nxh#=S? zg)8zdo$BtI>E7+>Zg=%~cWe(4AR!@sAtWxL{J~NL5}+V)C`vGgB1C@VA4((=AmS27 z;139ifWv#$-LpN@vmWnllKP^oyIxhjx?X+nRdo+-UR<4}tG*Y6vpO+n>$XGZrR{|R zs8-APuUaiD>3l^KLs7M z{^#EH!83D~XL#`D}Ts zZE*gttjqr@B)a^sU^%s_l&XUNcLO7%W8=n0&rQSQdHo~rJM$@P(v)(J*U&6VOd3E! zC&qHi*Fd%`jj3~mWofcBS5W6lrgRsY$OCGjI;X&Ysx_ud(&RN=+W)y#T^fDzv?Prl zjTXHskn=0IBs?2uySRFf=u zkZ7CIM4dQ{3ny&Xq^n%pICGmmf4bB04B|jbZb;oQU3@yA2DGO|1HJ>sszHgM&3i+_ zFtGJj7>xv{4D)Rh6f%oMlVAiC71Gr(AiS*uEig^Z4?Ld+VKT4-^?)?`g@;(^ftaS` ze8w?Z3LxXn;tF~O+~*09pI0moDhDkmVfekJ{mHLcKaOetrA^TO z&$_0N8aEkfLe_rL3N3dQhc~P8(POa*;yB(4?=n0^VoY50AU+*~Fot&#d?SHH;ANIj z**$4xvs?!?QQx>p-S&pny`@JTsIaMt!|4nh&L~#7E2?5RHY|TMpph*M=xx2~k|8Z^ z_M|?X|FJ!d8MxOX`G(sV#{UJ#>xush1r_iA1pn_mc9Qg;|1%i%`7G%lUdOQdeZYF< z-YhQR4o>m5TlWU|gb85QC?AdoIiz@)g=N~t1Py?5qr^BP4m7dfW*Tb&`(u^c!GByo zaxL*63*y+YcNot9zQ5HrhVg%?pk(fU3Pr*HyMdkL|3AP2j_zMK==c3~f_~pmG5pvY zk=c6&+DGjBy3?_b_;$qN2kCnNAkc0ON|KN#wodi~ z*n8~Wb3cmN`WxaZ){ zUwPul<>|*h{I}QNn)~%D-~0YQpFVx|z&D?HZ~n1Aeg27~vBwD|erSRqpBY$Jg6F_< zC_djw%%<(y3}y|63S9sP3+b?}(?>rsu{XPL?=YPIakjSB#xVY`C~80d7mMQlXIHS3 z{QvhF?838I{?Dx$R=)>GWxdgTK;KZa;r$encI*lK6*J=fX{5Bk_VJ~COlmv$HemE- zVLj&E%_ry1}1ui>oO=i1aEqfd07j*?QPW8?Z|21QM`uNK6x1V2m@x`nY$V$R^dn~TkW)6MTC7Z5LUi+v)b>;WJ`uFc% zeeRdPefF$+`pK_;>EvhkKlh*S{B`7q*H_j0w?1&>5cfp*XELf7N_&r6y#Lwc%}<13 z{9o>m{|ic074Ls{1v|?*_6Z#GMa}WITQ6bEi)jTfv%b~Ho^Mei0o%#J)@XzUJS!i4ad`olX-oBf(OcGwXD)wmDEb=NnIgGI4V_Y z)pEU1s~a_=rq)fprWXq}y;dxldby|!xFbw=h@s?+z<%YI%J8=BS zhu;0unB!l5ZQ~sOP}ko27kw&v(P^7V!c&oaw#XCBMl;liZ+CBwbo%Hg->Fk_-)}(j z;Ko#{8I=;MJE1queD~`cS63LUZXVTr`?mV1`c}UG`S=#w7<&Jc{r zHh2#pKKc0g|91f*>K9SJi28RS>fhB}ed`#= z|AqssNQ3SA3Gk5L|5l2nO6vFD1>pZmxghv|SCGTd0xj7f@$Y~y+F!y>NDnoy#98I8Fn&+fUO)`-(U_OzPv6=XlC!YooKS7v0q zx7K`*B|k)Ft$N*F?Qk9OZMTMA0J4%dqLA0gE%LsD(#uiTvC{?t9}{;!`a%Y3A%8_3 zCaW5=FQcqj;F2(OAfYz7LRrT65aPJFBX+~}0<7tJT9br^1u`Cbfrxpn0DMI}Uoc~F z4>L2>nThiG-$9*{Wgg$-kFJj$o81O1xW|skv)G^GwognmIDSGtbxOVxC1Ae6)3}Y} zCpstQuxO?@>nYD>nTs9dUKWQ=);x!iEG{u)3q^Wi=Fk!qKE_GnI znc+v49?bx2>G)xRPC=XB(jA*w@HD7a2oaQ_fujHy5!*_#41>$Yg)a10ge9XFXXHL{ zXECD%B9^AjShufy6GC;|u9?EPn_`1W*PD^~V$aCQpyP&FY&^YXp@rC5wr58!xL0_m zXQEAzvi^nq$4uZy-6Mf{bRpmA%&aX1*wRA&=);dZJbz>%zu1}U(UWZFg?u;Eg6h2M zJ@v3esyfbDTxf7RGtsK>N}aw&aaur-kBKYhJpTJJOe2LZ<~lh&u+0Wg8j{`$)h!4b zV*ZFT)5N-wa{)-uN5K@mst-_YQF7L$9BQEFjrCH+~{lq4a70_Y}}JPepDK} zviT*5ne&~vMib5t9=bAcG+`|BJw^VfB2LdhWZ8D>j@__AZ3PYsV7KXkL_SFej|YBEZ-?9us1Kg)W?yX4pv?~?=zLh) zHM&^=g-iz1$9-5S)gp#8>1j=6fkR%#Gec_Vr8qO0D#jczfy7Vov&)PO0lZyrLpplU zd`g3)LMy-ypAs=0k;#Dg_BqH}Sfl#+AXLfxf@&gR$T7MP^ALcn61Bv{6L^;|mN1(S0~yKg?w z>VUwIc``W+V#V}WU=QR6=W2%IvA!UvYku-&&*ghLHi}qBr^iw>v*inG5!N!qS|Z|+ zZ>u+M0s;a80s;a80s;a80s;a80s;a80s;a80s;a80s;a80s;a80s;a80s@=I{{R`P JCRPA=0082F;$i>* literal 0 HcmV?d00001 diff --git a/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE_data/emqx-export-bad-upload.tar.gz b/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE_data/emqx-export-bad-upload.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..569a58a2c1d0a2db8fba26f4de41c686d0e0fae0 GIT binary patch literal 2616 zcmV-83di*yiwFP!000001MORBj2uT5?wvinnYClb1guE;(HgH6Tg>iG&-5H?$;h^V zh{FacT#U8uTXh zHj@9J5*@t$7vA-ubF+q{Ira|pfeT!b<>BjJs;;kpu?p*7k&3YXg&pX7L%Q47UpIZz zvBfi@MzdF_$AUtxJX@NTa@qYJUP`-6L-v2urpzQe6b3en{{^XBsLB=C|K&QpIJ<+v|hle`P5COHk-r|Eeq%Q~cix8p(`~X&<{V1&&9k1-mzTh7!7!+a>yf6S zIj%|B5|*cQlW8FX^^GJ(#j73R)g0Su#~WwS*%jUc5x6#@X_PUg>F5;uf;muDJkzHr zoNSUM2MTRn7_SqHapt&b>vWZC8)J6U(6}VE641~ zktg2dbbut+0NWi1(+g;rAsHepE$ye{r^IO!_)j!zdwpz#|CJ>Emnv!gzbCYt`2W%~ zfd8Yz@E_c2F#dz!Zo&T^gZtngL3zb+pt0AoB3R#Fte^Or@#7HdU%3UW|5?`+Vs?{} zCWsD_mT%ZIXx@y(t;cHP#Il?fK4f?b*jTviKuS6azRQOQ0g=EW@H$JV?Ebj28Lors zsBcWCc6-a}-qNEERG3spb2|-PRFM_NG|BFSrET{Ot7qpxB|Kgv)zb_{7|JV?>>JJCQ>fJ5T;U13mrd@Xi zgokke*+34*gDj#w%)&BlV}W`Azd>P)5evFFXflPhAP``a+aZ)(KmKGCN*2;%!~W4` z@ZWW}*T+WqFBkjo{~-Ubr1-xVw43<<2Ux(&!J+xT>ki5PT{njCqjw^*_ZZp-><7Bj zae%mHfZ~VoDY*Ml48jbUOllL=qR~wdXn{>^VhkYAZx(72pBJ_d_5;v+^nnXM3eo$y z|85<$aQ|pC_>c3oojx|a|0ziQ_+P13Q~cix+D-ibdkuEs`6T{l z*WIc=0>ngjdJO0@HJRQ|B4~$^z+KS;+@C^7^Gp|C+J~UFi*ExmH%seLb|0UhP3HWs zzVVYNYCLgc>rum6_s{jILqs3%K1~z3QZt`@a^=NW-ZS@sv+9Eny!3;=T>oO@%m2z7 zpFX*A^6eK_UVb?l1d<#-+#ZX=`pjaix?<85%3DusR4x4eSO5Oq>o5HBx6hxK&OH6q zFU^1U;0yox&R<7f9r$CkMc-_9}_`CU)X=z^!?8*-TVZ!5&oC^^Z%k$tfcRM z_k?y6|DXQ_AV@8V|Mw-_sy_@QYovERV98kg&gYOz7+b-XL1egHZ+iam=pVZ;4Q>JT zzY+M)_%NrKYF_sJ(+$f}iIsShfQ++cy;_!Ntx9ULG_Mv20**urwQ9Lutkt!eR+H+w zT2o8Knp!K#x>_!&C2hW3oG$>kT0kYDRH(~kwW`-heO@nB)vBhGQdN^Hby6(X3Tm{X z`?Zg6l{SI@DzRKj`}_V;!Dt$)!cgEyU~ zj?g<9@MlWA&}=k)g}7$-zDT8yee#_uB@g}vI1g`4rIb)9pt@6Pdr2egd$;_y1+7(#`)X1*wGhKZW%BKYK%%EukZ8B>Wk`rK~8f z!GBRi+^4nfV8QS z7cpm5T!%$JLSU_W-CXT(9bu-cU<3eBbQ=P)PIfVi0|%v7gK^`CH3IP^9D8s(4y_Qo zg7K17g_&28S1NLj?^{rinp~kM;&($7qJ;x#*~_-GVuuV6!yGh;uNffYU%vnrb6Ot6 zi*V9l+QN~lEfcIVZAB+OQ3<1+d?4qag9ayx{5wAQ!;2FqW_ExU>=P%%863|^(21vM?f1Y4~n25;`gZ^Am5+^h_iw2?nGS^*C%0pq`VLt%{8v0kJcU{w*C-I z*VdFRc7~~QJkYjN>cDXm<_8WQ%m8btc)6%j(B`&O%VY*T4Qk~>7Nse5@N*jZO0*1v z>xPvs^*4kiqnGnyAG@=d&;nUY(`2kWR=x?LId0oTW!z1n!^GR=MZVa1F)}*rn1#;M zTNio=y=7;1oC~FeLMRQ+C(uEd zXU~Kq$>EFTFrLjVLEfC}ge{tIhHx;RLBR1tp6fVP6c=i7UZ@S@R|L^C0DQ%9ebzOA z55)2f1%i*Z45$wte1%v&hKQnR*DbSQ_{s_#7{G7ih6J%mg~$i-PHp=<4oDB5?}{(C zsMqF)5)3}9ZEIbTM?uJ7`gkmpV?AP86Q9>$*h;++x4;os?2aDW#NBN-3q3 aQc5YMlu}A5rIb>-o9KVw#`iq{Pyhg=NHdiH literal 0 HcmV?d00001 diff --git a/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE_data/emqx-export-upload-ce.tar.gz b/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE_data/emqx-export-upload-ce.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..1c74cb4f84b67f681a3944b79cbac51f4f2ccc45 GIT binary patch literal 2604 zcmV+{3e)u;iwFP!000001MORDjNDcg_Rc=ColTlF6rxJ~VNEt_B4&5To|&E5NI^*i zR6G)p%B#vQ?zMeq>|KxT_= zDqiAI_ya@QpED2(HXQwfkn#?DjC<=^vb$XM2 z!{q;`LEPk zJR@o}dxd%|XynSX#aStr-S6q8w97R3{5NdMOtMpHU@Q0!K!NAKRH>x+zZZ1V{YRnV zvXyO)!SR27L;RPZ(YOBB7AjA!jAIGFyOuk};|Z!o-uRaPV`hs*rj5j38u= z1y)COVS;G}Z4h0UjN;0IQI8rKT^O@TgT@4$=eJ$pO*tM+gX#*SKC-hOX(*cGnv^YL zdrCK%7BbM@NMcsJ+Y#Q)v8{GoIg0_W@DYf>tr1P5j42IAr#Ke)KxuiVPf@swb}_rO{e=;5{bY>4lB zrrPv_m7vQ|Zt7r=86y0CP-7tHt>3P)F+PE8DlP+^J6Pn|!n6MgS)SeVx&)&HH%xAuG%$^*1;!REm zNOBKQ??6~yfMSMZh_Jl8pMjqer{VZdG;5n<48#BV0r)SMQvBZw+D-g_=^4QP(T(sQ z;%YGdgWzt#|DFZ+!9Rj>%Wy#1Yg!Sk?=RL*e9icAi1n}B0@nYm8w#;?laU694w9yC z*fY4i8HukRyNwgea$0=Ka22qzaoK^IbQEHjPZ1I#flc6jmeJV#abq*w2G!Btm`?5X zmd(AT#~he2sgBF(3|vkIRqhQ`G|PnT4;D1CjRn1}*Icxuoyeb?z<(TDVH%#hfWOr- zhT*>qvL5`G<)Tzc@qaI9H}U_)KSO+9Oyd8s4dSXl9cRC4|JlTj*JOm7?Zk2Zn-uDi`K zhT(rn?tlJQfdr7^|6b5;;{P9D12YFVtp8nigZkffV+cQbCn9^#Li>RIK-V1yh-(HY zei*NUXFrNTm;uhDHc>4aJqZFUaEMLJ0R+a)LQmrJ#`eK}0D6x;aN$QGdO!Eyt)urB zf|!_$IReYvG4Ozo7e^?S=hcF)mq@KbE2RZwPM1n$?fxS_f9(Qg(*OS zg|^>T>0_T5-=9;sf3ykwM|W*&jG@ne3Q|A*mrLpU-#ww-#Q(on;S`=v;(vBMuKF`T zOmwH`fWD@{PMTYpO?-&{nalmeD>fA z|M|{eM}ByHRjPgKgU65ZOoSg3fub+$KW+B;&#gWA31}Gpm;38~S(4=R{qLU8ZsPy* zzW@ZOCh`BiL|pZ!fn<;LnGdXFEdI>rkV_bAp=OX7Zr3M0|9JF|-IoTpfcoDE;%9uL zpqOf0_Wjd!%TbAyc$0vPvt_+fmT0v?swHVbEf548i59Ara!sz*w5nE>YPwogi*i-1 z7E8KXE~-Usp)4;HfLke`5m74CN@ca8S4nL_FILowrjuetE6vx4T&@<>Xh-+!9N#Jp z$A6XBHU#E2Xej=d6ZLcIj5lrHv~CAvANlaRe;OkDt8Z)_+27ZVxBg9^4Bm8_IzsPcz@I7d zMx);F72=xR=S3=g?33?QDS7ZWz*v1v^{tze2b-HibKl<6Q@wc_ zjQ^Siut>cf`U$|0@Bf#i^7{K~ zV%@5^4vT(>z?!w1x!U12!fIE+EC8bDi3rF#*(E-AFnTqZI!;<6kYmE>2T#aB3-K$M zFj-ZYc@=p@nR9&Kf`-)K21OCSAEFQ~oQPSs9S=2aM`;jWGl0jxdI2ovG(E@_p})Xm z;S3X=>f{4C{~R|mI5D#Wv}m6=A8NHkr``DeO zgcYb*8YW}ix$;8@>Udleo$)k<0Tb8Di+r>5VzlURU={{XZ(kT8jFz1_k&E^kpL9Mr z1hME}%za!3d{iCc>C=n3PA9*<6%b2{x#>q9ePr(VVs5E3+hZp>&WpKjsRinM>OFec zBc_f%3p))SCm-wzSL*Z)if#c$E(ER+^I+=&C_@Nc%677>O%xy zAy&^qMA5WsmRUD^r3E(&5V!G#1UX5Cj0bs6ZTmbANDrUyiZ3^**XB1Qn0#2<*195( zf{=mxcn&MYM#PdP?$($H67mY(GsKo&3WbrY7_%S*qCCa#UGgFn@Mf(I?chRjDfJ?S zriT;A8NTnHo}06sS=Q7Hs2!P~>xM-K!eYN`os?2aDW#NBN-3q3Qc5YMlu}A5rIb>- O+vtC;Iet0-PyhhGX&s0F literal 0 HcmV?d00001 diff --git a/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE_data/emqx-export-upload-ee.tar.gz b/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE_data/emqx-export-upload-ee.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..5026c15545390f2049130d49d3a4e66c75d1915e GIT binary patch literal 3073 zcmV+c4F2;UiwFP!000001MORDj3iYRu6gdzv$)HO5)yt?dS}C~&ZDZ|v&kA*1){KE z1RsPMi(6HaL#cot+m#olR!z-h1x3=bm%!ch0Svq1O850^M{SFIZ^1rbF}vN|%H^ zB?+inE#v=kwJat7k4TV9<)S1@aqcNWws=mY zGzGTLVve*P_+`UcsnG_Z60FPAFS4F3;;F6)0BR9wHc%QYDO zSGL7}!2I$6{#OenDZ~GRpt13ZN$q17X5sN}^`m#)`l;r$F62z7p)@JcsRs>}Xsx!Z zKwnFkkQWOrVY;+fkQYn3@HUK)TjWA@QG$PRb3zq_>Bm&z=;t<7Vf=}6f-rsqmpY*f z)4tZE7SV;7D6ULc^+a=A7bb0D(Zm4j3py_J%{m^aL3PEa0e0s+Vkw&A8q{CI_N;FB zTG)Z|#!^}_w&RS=vCU3$at;GtVJi@UNh6v@eP6L0o#I+x17*WA0*Zq(7Flzk(b0ve zIx&6LIc3;7-DJ`xecN)`)3v6f5ffIj18N6J_F0c=u%5yRjv>`#@8EQM= zJ3*7-xS@kUze%_eG>@V{x*2$ct#zyop{cl@<5Dk(Hg>G;k;Y)>2|sY4n@pQ0Arf!`<5@#t*C&N7Fd0-wd6PP|J3AKl z_C9eS!k{{Cr#o~z<2dDQe2Qio;C{ZLv0ZHFo&DmXE$v1AyaN2kwH05(bI1F4y2dE{ zFThwI{uc^GDSQ8OAhe(O|J5qxdm= z;_rTxCh?nKOllL=q|uuokOGI;z#Kqe-5hX|fHigx_CwHn;>HU<3eo%N|Lz>UzrbT+ zCSeFHb0@(90X`h3v`|qcT`!S(l~zm3*g01!m9^`S|NQ00Pd+sJ$VdM6`a5#Je))Uf z|L0TZ&L8{c(;vt`@~6)~J{M-3Kw$SLsP}yh=L+!bIVQ#DYl&_dw&8=-V5^`4Bv@z% z9hE-#iK)Xmg@;F1fd6Q&U9B&4Yz21pFO z*>k|aR@3GE41#u;3ET~xCxLWN5FU@!?bgg;uDWQ@4a#QEX;hVd|Equh?v-bM`P*mC%jcf>>X(*3d-U1= zeCMxYKfJUl*T40_lP6dv!XMLoVjvtoUFrLut9$dqX%zmK2kZZWEX&#V-vgoj#Q$f0 z0SHn{_8bDeKj;OlwtAE6K~ML=bRfD%GmxdZAX=YFbUM>uOCc7HVp( zSkl#UQ7vlA<-)QAy;TW?h_X~KmDQ?VBlTsySXHZ6fxhAQRM-S^9UoDNs zf0fua1m-SiB>tCE^?#+B;s1fqe&YXY?*aTjk;eboL|hFbe{$^2egl~9Z8Kf{^KZZ7 zYKMIgBtDk=erFtSgHE5}UpfsPp?8M&FBDnB zYFL3nTqAy8q|ygJ`977Bn|=d5ckE20oDwOZxHD?w*6)6O=i&;(#Vw(@Zy)HXzH%Cl z|C$M~NWDG!6MzxF|6i4ggZN*m%9RZN4}yq>ZxVcQD<_IAQg0DN2)(UQk9CP6p0V&z z6d@XHssWUU=X%cyvV;swM9eOVYP(*KMRBsI<#-*%0Z3r@3+n|`3;zg#A!U*okQlo8 zoJ7BixJLlTaGqismJtA3;g0hO9-lTWIHQW%0R>@HmZS^4@oxO2IhN}I>~jrlM@e+V zOx!j+Cn^_-*`^AhBWS0DD%y_SQ`{m57J=4`7q2Z>?mXW&4mG}MG@1d^tTzJd0#5f? ztVKKcCmC~FixKK{^NHcWfJp(d+>j}5_i-_uLyp8i&kQ|tY7V3r#iMmFLKE}^B!Jm4 zw<7sN*x4%++CUvWV(X3-^Q^Zicseu?@jJE_4|#OG%|BOYEwm3YO~vrNcpRA(J|Fkv zw9!uG-tqyYaSgf>u@`Lxw07E_Mis{mAh5w_X3$ii_Rw0%&B$e+&%mGETsMao2eht6 z!eD#=2Dl%v1fsSMeQMh<0}!L=Bp@mYgVmy1lh}rD0Y`i-!%;oQZI!sGDa34G9nrMJ zEvy4zir<7wL}gyY8=T@ge)K0fKAaw7lWQ5?`xLB)U?_Sw$aG(2x`WV%xaz31MqtOG z`utlFS_?9HE($>3cnJH71%A#0W)v&O4T8TZrI6@Y8V%d=a3-7%3*(x5%-Mi0Zfrla z7g#LRFlDLkJRc9;lvWB{hrj8DL3--c!XD6yed?6Bfcl&^T%zmV=`-Tlv*H$SfW4vS zvKyz*bk8gzYl_|P0!LbSbnrvr zya@uL{sHkr#z!mw)h3r68|q=PXM~=ntdY>>VJ2gMwr&O-xEai~GhwUU?Zi1ANZTuM z5V)!B^GN3w@Y^cBVpl0hbK9zE_)T~kHj9CfQinLFQLIF67|t6qJuo;CTt?UD#eweb zYDx;^J_gh^No3O_Vxa z$1$UPP)qiO(lBog&#rMozD$~pd7_d>*??;^#BFjTVbR3u#a_vlDCkBM6JSn-=de;@ zMJ(=v;S~fVj>QgtP+UsAXhPdV1@g@xaL+C+ z+0LTh)|zmO@`HR|sS(%Cmnppa>+9IksVXH=c&d&QU^V6{xFk!YNgd8EOUs4)wjmeW z1|o!D(pF--4&i~sp5RNBip7Z0cc63%2BXCWH=3l=pZ(F18bkn#i$W<;FZ;GWv_H^X zrA4VrOylA}5q_0Ej68(c2^D_Buz3;1D Date: Thu, 23 Nov 2023 23:11:40 +0200 Subject: [PATCH 19/20] fix(emqx_mgmt_cli): remove duplicated word in the log message --- apps/emqx_management/src/emqx_mgmt_cli.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 866efb267..6753753ad 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -775,7 +775,7 @@ data(["import", Filename]) -> emqx_ctl:print("Data has been imported successfully.~n"); {ok, _} -> emqx_ctl:print( - "Data has been imported, but some errors occurred, see the the log above.~n" + "Data has been imported, but some errors occurred, see the log above.~n" ); {error, Reason} -> Reason1 = emqx_mgmt_data_backup:format_error(Reason), From 70978df5bb00b2440492a2a36fd496c473b7c24e Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 27 Nov 2023 16:32:15 +0100 Subject: [PATCH 20/20] fix: use parse_id --- apps/emqx_connector/src/emqx_connector_api.erl | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index d09c67c8a..b31fb4f74 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -648,13 +648,8 @@ format_resource_data(added_channels, Channels, Result) -> format_resource_data(K, V, Result) -> Result#{K => V}. -format_action(Action) -> - case string:split(Action, ":", all) of - [_Prefix, _Type, Name | _] -> - Name; - _ -> - Action - end. +format_action(ActionId) -> + element(2, emqx_bridge_v2:parse_id(ActionId)). is_ok(ok) -> ok;