From 9e2987034bc6aca851eff5474668c2667a786ca5 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 2 Aug 2022 10:36:43 +0800 Subject: [PATCH 01/73] chore(exproto): start idle timer for udp clients --- .../emqx_exproto/src/emqx_exproto_channel.erl | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl index 80e5f4045..765dcf53e 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -76,7 +76,8 @@ -define(TIMER_TABLE, #{ alive_timer => keepalive, - force_timer => force_close + force_timer => force_close, + idle_timer => force_close_idle }). -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]). @@ -94,6 +95,8 @@ awaiting_rel_max ]). +-define(DEFAULT_IDLE_TIMEOUT, 30000). + %%-------------------------------------------------------------------- %% Info, Attrs and Caps %%-------------------------------------------------------------------- @@ -149,8 +152,12 @@ init(ConnInfo = #{socktype := Socktype, GRpcChann = proplists:get_value(handler, Options), NConnInfo = default_conninfo(ConnInfo), ClientInfo = default_clientinfo(ConnInfo), + + IdleTimeout = proplists:get_value(idle_timeout, Options, ?DEFAULT_IDLE_TIMEOUT), + + NConnInfo1 = NConnInfo#{idle_timeout => IdleTimeout}, Channel = #channel{gcli = #{channel => GRpcChann}, - conninfo = NConnInfo, + conninfo = NConnInfo1, clientinfo = ClientInfo, conn_state = accepted, timers = #{} @@ -165,7 +172,8 @@ init(ConnInfo = #{socktype := Socktype, #{socktype => socktype(Socktype), peername => address(Peername), sockname => address(Sockname)})}, - try_dispatch(on_socket_created, wrap(Req), Channel) + start_idle_checking_timer( + try_dispatch(on_socket_created, wrap(Req), Channel)) end. register_the_anonymous_client(ClientInfo, ConnInfo) -> @@ -184,6 +192,12 @@ register_the_anonymous_client(ClientInfo, ConnInfo) -> unregister_the_anonymous_client(ClientId) -> emqx_cm:unregister_channel(ClientId). +start_idle_checking_timer(Channel = #channel{conninfo = #{socktype := udp}}) -> + ensure_timer(idle_timer, Channel); + +start_idle_checking_timer(Channel) -> + Channel. + %% @private peercert(NoSsl, ConnInfo) when NoSsl == nossl; NoSsl == undefined -> @@ -267,6 +281,9 @@ handle_timeout(_TRef, {keepalive, StatVal}, handle_timeout(_TRef, force_close, Channel = #channel{closed_reason = Reason}) -> {shutdown, Reason, Channel}; +handle_timeout(_TRef, force_close_idle, Channel) -> + {shutdown, idle_timeout, Channel}; + handle_timeout(_TRef, Msg, Channel) -> ?WARN("Unexpected timeout: ~p", [Msg]), {ok, Channel}. @@ -328,7 +345,8 @@ handle_call({start_timer, keepalive, Interval}, NConnInfo = ConnInfo#{keepalive => Interval}, NClientInfo = ClientInfo#{keepalive => Interval}, NChannel = Channel#channel{conninfo = NConnInfo, clientinfo = NClientInfo}, - {reply, ok, [{event, updated}], ensure_keepalive(NChannel)}; + {reply, ok, [{event, updated}], + ensure_keepalive(cancel_timer(idle_timer, NChannel))}; handle_call({subscribe, TopicFilter, Qos}, Channel = #channel{ @@ -561,6 +579,12 @@ reset_timer(Name, Channel) -> clean_timer(Name, Channel = #channel{timers = Timers}) -> Channel#channel{timers = maps:remove(Name, Timers)}. +cancel_timer(Name, Channel = #channel{timers = Timers}) -> + emqx_misc:cancel_timer(maps:get(Name, Timers, undefined)), + clean_timer(Name, Channel). + +interval(idle_timer, #channel{conninfo = #{idle_timeout := IdleTimeout}}) -> + IdleTimeout; interval(force_timer, _) -> 15000; interval(alive_timer, #channel{keepalive = Keepalive}) -> From ec5c0816f70258822b0d4e3517e3e9e5138ab168 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 3 Aug 2022 13:59:49 +0800 Subject: [PATCH 02/73] chore: update app.src & appup.src --- CHANGES-4.3.md | 6 ++++++ apps/emqx_exproto/src/emqx_exproto.app.src | 2 +- apps/emqx_exproto/src/emqx_exproto.appup.src | 8 ++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index db82c8eb1..d67f8e838 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -10,6 +10,12 @@ File format: - One list item per change topic Change log ends with a list of github PRs +## v4.3.19 + +### Bug fixes + +- Add a idle timer for ExProto UDP client to avoid client leaking [#8628] + ## v4.3.18 ### Enhancements diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src index 76bc9fa85..a267d1daf 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_exproto, [{description, "EMQ X Extension for Protocol"}, - {vsn, "4.3.9"}, %% 4.3.3 is used by ee + {vsn, "4.3.10"}, %% 4.3.3 is used by ee {modules, []}, {registered, []}, {mod, {emqx_exproto_app, []}}, diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index d6406b239..8e0380089 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -1,7 +1,9 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{<<"4\\.3\\.[2-8]">>, + [{"4.3.9", + [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[2-8]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-1]">>, @@ -10,7 +12,9 @@ {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{<<"4\\.3\\.[2-8]">>, + [{"4.3.9", + [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[2-8]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-1]">>, From 616a2235f28d11647256873193db70e7e3662383 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 4 Aug 2022 19:09:03 +0200 Subject: [PATCH 03/73] fix(lwm2m): improve error logging for bad lwm2m requests --- apps/emqx_lwm2m/src/emqx_lwm2m.app.src | 2 +- apps/emqx_lwm2m/src/emqx_lwm2m.appup.src | 74 +++++++++++++++---- .../emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl | 6 +- apps/emqx_lwm2m/src/emqx_lwm2m_json.erl | 4 +- apps/emqx_lwm2m/src/emqx_lwm2m_message.erl | 4 +- apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl | 11 ++- .../src/emqx_lwm2m_xml_object_db.erl | 7 +- apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl | 54 ++++++++++++++ 8 files changed, 133 insertions(+), 29 deletions(-) diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src index 18c85faad..6b0e6bd92 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src @@ -1,6 +1,6 @@ {application,emqx_lwm2m, [{description,"EMQ X LwM2M Gateway"}, - {vsn, "4.3.7"}, % strict semver, bump manually! + {vsn, "4.3.8"}, % strict semver, bump manually! {modules,[]}, {registered,[emqx_lwm2m_sup]}, {applications,[kernel,stdlib,lwm2m_coap]}, diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src index 1d2dd58fc..793feecd1 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src @@ -1,29 +1,75 @@ %% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{<<"4\\.3\\.[0-1]">>, - [{restart_application,emqx_lwm2m}]}, + [{"4.3.7", + [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[0-1]">>,[{restart_application,emqx_lwm2m}]}, {"4.3.2", - [{load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]}, + [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]}, {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]}, {load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[3-4]">>, - [{load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]}, + [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]}, {load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [ %% There are only changes to the schema file, so we don't need any - %% commands here - ]}], - [{<<"4\\.3\\.[0-1]">>, - [{restart_application,emqx_lwm2m}]}, + [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]}]}], + [{"4.3.7", + [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[0-1]">>,[{restart_application,emqx_lwm2m}]}, {"4.3.2", - [{load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]}, + [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]}, {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]}, {load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[3-4]">>, - [{load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]}, + [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]}, {load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]}, - {"4.3.6", []}]}. + [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]}, + {"4.3.6", + [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]}, + {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]}]}]}. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl index b0c881f8b..3203cf070 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl @@ -106,9 +106,11 @@ coap_read_to_mqtt({ok, SuccessCode}, CoapPayload, Format, Ref) -> Result = coap_content_to_mqtt_payload(CoapPayload, Format, Ref), make_response(SuccessCode, Ref, Format, Result) catch - error:not_implemented -> make_response(not_implemented, Ref); + throw : {bad_request, Reason} -> + ?LOG(error, "bad_request, reason=~p, payload=~p", [Reason, CoapPayload]), + make_response(bad_request, Ref); C:R:Stack -> - ?LOG(error, "~p, bad payload format: ~p, stacktrace: ~p", [{C, R}, CoapPayload, Stack]), + ?LOG(error, "bad_request, error=~p, stacktrace=~p~npayload=~p", [{C, R}, Stack, CoapPayload]), make_response(bad_request, Ref) end. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_json.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_json.erl index 2a10f11f4..d2356424e 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_json.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_json.erl @@ -29,7 +29,7 @@ tlv_to_json(BaseName, TlvData) -> DecodedTlv = emqx_lwm2m_tlv:parse(TlvData), ObjectId = object_id(BaseName), - ObjDefinition = emqx_lwm2m_xml_object:get_obj_def(ObjectId, true), + ObjDefinition = emqx_lwm2m_xml_object:get_obj_def_assertive(ObjectId, true), case DecodedTlv of [#{tlv_resource_with_value:=Id, value:=Value}] -> TrueBaseName = basename(BaseName, undefined, undefined, Id, 3), @@ -315,7 +315,7 @@ encode_int(Int) -> binary:encode_unsigned(Int). text_to_json(BaseName, Text) -> {ObjectId, ResourceId} = object_resource_id(BaseName), - ObjDefinition = emqx_lwm2m_xml_object:get_obj_def(ObjectId, true), + ObjDefinition = emqx_lwm2m_xml_object:get_obj_def_assertive(ObjectId, true), {K, V} = text_value(Text, ResourceId, ObjDefinition), #{bn=>BaseName, e=>[#{K=>V}]}. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl index 9a05e1663..0fb6c0e9c 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl @@ -33,7 +33,7 @@ tlv_to_json(BaseName, TlvData) -> DecodedTlv = emqx_lwm2m_tlv:parse(TlvData), ObjectId = object_id(BaseName), - ObjDefinition = emqx_lwm2m_xml_object:get_obj_def(ObjectId, true), + ObjDefinition = emqx_lwm2m_xml_object:get_obj_def_assertive(ObjectId, true), case DecodedTlv of [#{tlv_resource_with_value:=Id, value:=Value}] -> TrueBaseName = basename(BaseName, undefined, undefined, Id, 3), @@ -289,7 +289,7 @@ path([H|T], Acc) -> text_to_json(BaseName, Text) -> {ObjectId, ResourceId} = object_resource_id(BaseName), - ObjDefinition = emqx_lwm2m_xml_object:get_obj_def(ObjectId, true), + ObjDefinition = emqx_lwm2m_xml_object:get_obj_def_assertive(ObjectId, true), Val = text_value(Text, ResourceId, ObjDefinition), [#{path => BaseName, value => Val}]. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl index 6ac29b3db..9046f6425 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl @@ -20,6 +20,7 @@ -include_lib("xmerl/include/xmerl.hrl"). -export([ get_obj_def/2 + , get_obj_def_assertive/2 , get_object_id/1 , get_object_name/1 , get_object_and_resource_id/2 @@ -31,15 +32,19 @@ -define(LOG(Level, Format, Args), logger:Level("LWM2M-OBJ: " ++ Format, Args)). -% This module is for future use. Disabled now. +get_obj_def_assertive(ObjectId, IsInt) -> + case get_obj_def(ObjectId, IsInt) of + {error, no_xml_definition} -> + erlang:throw({bad_request, {unknown_object_id, ObjectId}}); + Xml -> + Xml + end. get_obj_def(ObjectIdInt, true) -> emqx_lwm2m_xml_object_db:find_objectid(ObjectIdInt); get_obj_def(ObjectNameStr, false) -> emqx_lwm2m_xml_object_db:find_name(ObjectNameStr). - - get_object_id(ObjDefinition) -> [#xmlText{value=ObjectId}] = xmerl_xpath:string("ObjectID/text()", ObjDefinition), ObjectId. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl index 0f5fd7df9..df16d6f7c 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl @@ -69,12 +69,9 @@ find_name(Name) -> end, case ets:lookup(?LWM2M_OBJECT_NAME_TO_ID_TAB, NameBinary) of [] -> - undefined; + {error, no_xml_definition}; [{NameBinary, ObjectId}] -> - case ets:lookup(?LWM2M_OBJECT_DEF_TAB, ObjectId) of - [] -> undefined; - [{ObjectId, Xml}] -> Xml - end + find_objectid(ObjectId) end. stop() -> diff --git a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl index 27cde6d08..31ee16745 100644 --- a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl @@ -689,6 +689,60 @@ case10_read(Config) -> }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). +case10_read_bad_request(Config) -> + UdpSock = ?config(sock, Config), + Epn = "urn:oma:lwm2m:oma:3", + MsgId1 = 15, + RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), + emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), + timer:sleep(200), + % step 1, device register ... + test_send_coap_request( UdpSock, + post, + sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), + #coap_content{content_format = <<"text/plain">>, + payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, + [], + MsgId1), + #coap_message{method = Method1} = test_recv_coap_response(UdpSock), + ?assertEqual({ok,created}, Method1), + test_recv_mqtt_response(RespTopic), + + % step2, send a READ command to device + CmdId = 206, + CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, + Command = #{ + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"path">> => <<"/3333/0/0">> + } + }, + CommandJson = emqx_json:encode(Command), + ?LOGT("CommandJson=~p", [CommandJson]), + test_mqtt_broker:publish(CommandTopic, CommandJson, 0), + timer:sleep(50), + Request2 = test_recv_coap_request(UdpSock), + #coap_message{method = Method2, payload=Payload2} = Request2, + ?LOGT("LwM2M client got ~p", [Request2]), + ?assertEqual(get, Method2), + ?assertEqual(<<>>, Payload2), + timer:sleep(50), + + test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request2, true), + timer:sleep(100), + + ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"code">> => <<"4.00">>, + <<"codeMsg">> => <<"bad_request">>, + <<"reqPath">> => <<"/3333/0/0">> + } + }), + ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + + case10_read_separate_ack(Config) -> UdpSock = ?config(sock, Config), Epn = "urn:oma:lwm2m:oma:3", From 1d9ee25c920516e217d241f6444bdf15c3665275 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 4 Aug 2022 10:23:26 +0800 Subject: [PATCH 04/73] fix: support custom count function --- apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl | 12 ++++++++++-- apps/emqx_management/src/emqx_mgmt_api.erl | 13 +++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl index 331d14a92..b09f99466 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl @@ -132,7 +132,11 @@ list_clientid(_Bindings, Params) -> SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end, - return({ok, emqx_mgmt_api:node_query(node(), Params, ?CLIENTID_SCHEMA, ?query_clientid, SortFun)}). + CountFun = fun() -> + MatchSpec = [{{?TABLE, {clientid, '_'}, '$1', '$2'}, [], [true]}], + ets:select_count(?TABLE, MatchSpec) + end, + return({ok, emqx_mgmt_api:node_query(node(), Params, ?CLIENTID_SCHEMA, ?query_clientid, SortFun, CountFun)}). lookup_clientid(#{clientid := Clientid}, _Params) -> return({ok, format(emqx_auth_mnesia_cli:lookup_user({clientid, urldecode(Clientid)}))}). @@ -182,7 +186,11 @@ delete_clientid(#{clientid := Clientid}, _) -> list_username(_Bindings, Params) -> SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end, - return({ok, emqx_mgmt_api:node_query(node(), Params, ?USERNAME_SCHEMA, ?query_username, SortFun)}). + CountFun = fun() -> + MatchSpec = [{{?TABLE, {username, '_'}, '$1', '$2'}, [], [true]}], + ets:select_count(?TABLE, MatchSpec) + end, + return({ok, emqx_mgmt_api:node_query(node(), Params, ?USERNAME_SCHEMA, ?query_username, SortFun, CountFun)}). lookup_username(#{username := Username}, _Params) -> return({ok, format(emqx_auth_mnesia_cli:lookup_user({username, urldecode(Username)}))}). diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index 66a5c1f9d..482dcac0e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -24,6 +24,7 @@ -export([ params2qs/2 , node_query/4 , node_query/5 + , node_query/6 , cluster_query/3 , traverse_table/5 , select_table/5 @@ -59,6 +60,11 @@ query_handle([Table]) when is_atom(Table) -> query_handle(Tables) -> qlc:append([qlc:q([E || E <- ets:table(T)]) || T <- Tables]). +count_size(Table, undefined) -> + count(Table); +count_size(_Table, CountFun) -> + CountFun(). + count(Table) when is_atom(Table) -> ets:info(Table, size); count([Table]) when is_atom(Table) -> @@ -83,9 +89,12 @@ limit(Params) -> %%-------------------------------------------------------------------- node_query(Node, Params, {Tab, QsSchema}, QueryFun) -> - node_query(Node, Params, {Tab, QsSchema}, QueryFun, undefined). + node_query(Node, Params, {Tab, QsSchema}, QueryFun, undefined, undefined). node_query(Node, Params, {Tab, QsSchema}, QueryFun, SortFun) -> + node_query(Node, Params, {Tab, QsSchema}, QueryFun, SortFun, undefined). + +node_query(Node, Params, {Tab, QsSchema}, QueryFun, SortFun, CountFun) -> {CodCnt, Qs} = params2qs(Params, QsSchema), Limit = limit(Params), Page = page(Params), @@ -95,7 +104,7 @@ node_query(Node, Params, {Tab, QsSchema}, QueryFun, SortFun) -> {_, Rows} = do_query(Node, Qs, QueryFun, Start, Limit+1), Meta = #{page => Page, limit => Limit}, NMeta = case CodCnt =:= 0 of - true -> Meta#{count => count(Tab), hasnext => length(Rows) > Limit}; + true -> Meta#{count => count_size(Tab, CountFun), hasnext => length(Rows) > Limit}; _ -> Meta#{count => -1, hasnext => length(Rows) > Limit} end, Data0 = lists:sublist(Rows, Limit), From f97820f9138c3d5e06d1fac4c7aabd5a95061fca Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 5 Aug 2022 11:07:16 +0800 Subject: [PATCH 05/73] test: add auth_mnesia count test --- .../test/emqx_auth_mnesia_SUITE.erl | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl index 1dd979379..508ff93b3 100644 --- a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl @@ -272,7 +272,8 @@ t_clientid_rest_api(_Config) -> clean_all_users(), {ok, Result1} = request_http_rest_list(["auth_clientid"]), - [] = get_http_data(Result1), + ?assertMatch(#{<<"data">> := [], <<"meta">> := #{<<"count">> := 0}}, + emqx_json:decode(Result1, [return_maps])), Params1 = #{<<"clientid">> => ?CLIENTID, <<"password">> => ?PASSWORD}, {ok, _} = request_http_rest_add(["auth_clientid"], Params1), @@ -295,8 +296,27 @@ t_clientid_rest_api(_Config) -> }, get_http_data(Result3)), {ok, Result4} = request_http_rest_list(["auth_clientid"]), + #{<<"data">> := Data4, <<"meta">> := #{<<"count">> := Count4}} + = emqx_json:decode(Result4, [return_maps]), - ?assertEqual(3, length(get_http_data(Result4))), + ?assertEqual(3, Count4), + ?assertEqual(3, length(Data4)), + + UserNameParams = [ #{<<"username">> => ?USERNAME, <<"password">> => ?PASSWORD} + , #{<<"username">> => <<"username1">>, <<"password">> => ?PASSWORD} + , #{<<"username">> => <<"username2">>, <<"password">> => ?PASSWORD} + ], + {ok, _} = request_http_rest_add(["auth_username"], UserNameParams), + + {ok, Result41} = request_http_rest_list(["auth_clientid"]), + %% the count clientid is not affected by username count. + ?assertEqual(Result4, Result41), + + {ok, Result42} = request_http_rest_list(["auth_username"]), + #{<<"data">> := Data42, <<"meta">> := #{<<"count">> := Count42}} + = emqx_json:decode(Result42, [return_maps]), + ?assertEqual(3, Count42), + ?assertEqual(3, length(Data42)), {ok, Result5} = request_http_rest_list(["auth_clientid?_like_clientid=id"]), ?assertEqual(2, length(get_http_data(Result5))), From 43e1087e131960942d9a1144aec2890b4cc3b8fe Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 5 Aug 2022 11:34:04 +0800 Subject: [PATCH 06/73] test: add auth_mnesia count test --- CHANGES-4.3.md | 7 ++++++- apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src | 2 +- apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src | 7 +++++-- apps/emqx_management/src/emqx_management.app.src | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index a9d89473e..6d1e0a4dc 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -8,7 +8,12 @@ File format: - Use weight-2 heading for releases - One list item per change topic - Change log ends with a list of github PRs + Change log ends with a list of GitHub PRs + +## v4.3.19 + +### Bug fixes +- Fix GET `/auth_clientid`& `/auth_username`'s count. [#8655](https://github.com/emqx/emqx/pull/8655) ## v4.3.18 diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src index 592535f46..54d1317d7 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mnesia, [{description, "EMQ X Authentication with Mnesia"}, - {vsn, "4.3.7"}, % strict semver, bump manually + {vsn, "4.3.8"}, % strict semver, bump manually {modules, []}, {registered, []}, {applications, [kernel,stdlib,mnesia]}, diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src index a849002d6..8fa8384d4 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src @@ -1,7 +1,9 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{<<"4\\.3\\.[5-6]">>, + [{<<"4\\.3\\.7">>, + [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[5-6]">>, [{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, @@ -28,7 +30,8 @@ {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{<<"4\\.3\\.[5-6]">>, + [{"4.3.7",[{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[5-6]">>, [{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 937ec9db6..6a5ae3d07 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.15"}, % strict semver, bump manually! + {vsn, "4.3.16"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, From 5ff864af156de67250b767b2202be7e0075eab95 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 5 Aug 2022 15:48:24 +0200 Subject: [PATCH 07/73] docs: update CHANGES-4.3.md --- CHANGES-4.3.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index a9d89473e..0c4e1dd21 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -10,6 +10,12 @@ File format: - One list item per change topic Change log ends with a list of github PRs +## v4.3.19 + +### Enhancements + +- Improve error message for LwM2M plugin when object ID is not valid [#8654](https://github.com/emqx/emqx/pull/8654). + ## v4.3.18 ### Enhancements From f2a248d83ff6d0e61b7f8d1f637bb92054bed3ee Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 8 Aug 2022 09:35:26 +0800 Subject: [PATCH 08/73] chore: update CHANGES-4.3.md Co-authored-by: Thales Macedo Garitezi --- CHANGES-4.3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 6d1e0a4dc..d711c589b 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -13,7 +13,7 @@ File format: ## v4.3.19 ### Bug fixes -- Fix GET `/auth_clientid`& `/auth_username`'s count. [#8655](https://github.com/emqx/emqx/pull/8655) +- Fix GET `/auth_clientid` and `/auth_username` counts. [#8655](https://github.com/emqx/emqx/pull/8655) ## v4.3.18 From 4869c94e974eb44fea44f4b6aca9eaaa96fb349f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 8 Aug 2022 10:25:01 +0800 Subject: [PATCH 09/73] chore: replace / with _ in match_spec --- apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl | 4 ++-- apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl index b09f99466..6e435ee11 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl @@ -133,7 +133,7 @@ list_clientid(_Bindings, Params) -> SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end, CountFun = fun() -> - MatchSpec = [{{?TABLE, {clientid, '_'}, '$1', '$2'}, [], [true]}], + MatchSpec = [{{?TABLE, {clientid, '_'}, '_', '_'}, [], [true]}], ets:select_count(?TABLE, MatchSpec) end, return({ok, emqx_mgmt_api:node_query(node(), Params, ?CLIENTID_SCHEMA, ?query_clientid, SortFun, CountFun)}). @@ -187,7 +187,7 @@ delete_clientid(#{clientid := Clientid}, _) -> list_username(_Bindings, Params) -> SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end, CountFun = fun() -> - MatchSpec = [{{?TABLE, {username, '_'}, '$1', '$2'}, [], [true]}], + MatchSpec = [{{?TABLE, {username, '_'}, '_', '_'}, [], [true]}], ets:select_count(?TABLE, MatchSpec) end, return({ok, emqx_mgmt_api:node_query(node(), Params, ?USERNAME_SCHEMA, ?query_username, SortFun, CountFun)}). diff --git a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl index 508ff93b3..0253110aa 100644 --- a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl @@ -300,10 +300,10 @@ t_clientid_rest_api(_Config) -> = emqx_json:decode(Result4, [return_maps]), ?assertEqual(3, Count4), - ?assertEqual(3, length(Data4)), + ?assertEqual([<<"client2">>, <<"clientid1">>, ?CLIENTID], + lists:sort(lists:map(fun(#{<<"clientid">> := C}) -> C end, Data4))), - UserNameParams = [ #{<<"username">> => ?USERNAME, <<"password">> => ?PASSWORD} - , #{<<"username">> => <<"username1">>, <<"password">> => ?PASSWORD} + UserNameParams = [#{<<"username">> => <<"username1">>, <<"password">> => ?PASSWORD} , #{<<"username">> => <<"username2">>, <<"password">> => ?PASSWORD} ], {ok, _} = request_http_rest_add(["auth_username"], UserNameParams), @@ -315,8 +315,9 @@ t_clientid_rest_api(_Config) -> {ok, Result42} = request_http_rest_list(["auth_username"]), #{<<"data">> := Data42, <<"meta">> := #{<<"count">> := Count42}} = emqx_json:decode(Result42, [return_maps]), - ?assertEqual(3, Count42), - ?assertEqual(3, length(Data42)), + ?assertEqual(2, Count42), + ?assertEqual([<<"username1">>, <<"username2">>], + lists:sort(lists:map(fun(#{<<"username">> := U}) -> U end, Data42))), {ok, Result5} = request_http_rest_list(["auth_clientid?_like_clientid=id"]), ?assertEqual(2, length(get_http_data(Result5))), From 8dd966e8ef18113f9327d5727f3554721f692ee9 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 8 Aug 2022 16:22:10 +0800 Subject: [PATCH 10/73] fix: enable emqx_mod_module if default_plugins --- src/emqx.app.src | 2 +- src/emqx.appup.src | 16 ++++++++++++++-- src/emqx_plugins.erl | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/emqx.app.src b/src/emqx.app.src index 14a6f4fa4..14c63c75e 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.3.19"}, % strict semver, bump manually! + {vsn, "4.3.20"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel diff --git a/src/emqx.appup.src b/src/emqx.appup.src index c86661891..adc7be0c3 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,13 +1,19 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.18",[{load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + [{"4.3.19", + [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, + {"4.3.18", + [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.3.17", [{load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {update,emqx_broker_sup,supervisor}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, {"4.3.16", [{load_module,emqx_session,brutal_purge,soft_purge,[]}, @@ -678,13 +684,19 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.18",[{load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + [{"4.3.19", + [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, + {"4.3.18", + [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.3.17", [{load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {update,emqx_broker_sup,supervisor}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, {"4.3.16", [{load_module,emqx_session,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index 3177f05e7..caf47b0c0 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -219,7 +219,7 @@ ensure_file(File) -> false -> DefaultPlugins = [ {emqx_management, true} , {emqx_dashboard, true} - , {emqx_modules, false} + , {emqx_modules, true} , {emqx_recon, true} , {emqx_retainer, true} , {emqx_telemetry, true} From 50bb5a4f5ac8f4bc9c8197cce54b67779553df1b Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 9 Aug 2022 09:36:33 +0800 Subject: [PATCH 11/73] fix: emqx version is 4.3.19 --- src/emqx.app.src | 2 +- src/emqx.appup.src | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/emqx.app.src b/src/emqx.app.src index 14c63c75e..14a6f4fa4 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.3.20"}, % strict semver, bump manually! + {vsn, "4.3.19"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel diff --git a/src/emqx.appup.src b/src/emqx.appup.src index adc7be0c3..70d18ff48 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,9 +1,7 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.19", - [{load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, + [ {"4.3.18", [{load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, @@ -684,9 +682,7 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.19", - [{load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, + [ {"4.3.18", [{load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, From 25228ed248db60f5d069c02ef5e2ef3cc9ad0c44 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 9 Aug 2022 09:37:05 +0800 Subject: [PATCH 12/73] test: fix compile warning --- apps/emqx_sn/examples/simple_example2.erl | 1 + apps/emqx_sn/examples/simple_example3.erl | 1 + apps/emqx_sn/examples/simple_example4.erl | 2 +- test/emqx_broker_sup_SUITE.erl | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx_sn/examples/simple_example2.erl b/apps/emqx_sn/examples/simple_example2.erl index b9ada6d22..9af9262e9 100644 --- a/apps/emqx_sn/examples/simple_example2.erl +++ b/apps/emqx_sn/examples/simple_example2.erl @@ -6,6 +6,7 @@ -define(PORT, 1884). -export([start/0]). +-export([gen_register_packet/2]). start() -> io:format("start to connect ~p:~p~n", [?HOST, ?PORT]), diff --git a/apps/emqx_sn/examples/simple_example3.erl b/apps/emqx_sn/examples/simple_example3.erl index 40f0bf572..6bc41f1e8 100644 --- a/apps/emqx_sn/examples/simple_example3.erl +++ b/apps/emqx_sn/examples/simple_example3.erl @@ -6,6 +6,7 @@ -define(PORT, 1884). -export([start/0]). +-export([gen_register_packet/2]). start() -> io:format("start to connect ~p:~p~n", [?HOST, ?PORT]), diff --git a/apps/emqx_sn/examples/simple_example4.erl b/apps/emqx_sn/examples/simple_example4.erl index 6beb5835c..1b4809626 100644 --- a/apps/emqx_sn/examples/simple_example4.erl +++ b/apps/emqx_sn/examples/simple_example4.erl @@ -5,7 +5,7 @@ -define(HOST, {127,0,0,1}). -define(PORT, 1884). --export([start/0]). +-export([start/1]). start(LoopTimes) -> io:format("start to connect ~p:~p~n", [?HOST, ?PORT]), diff --git a/test/emqx_broker_sup_SUITE.erl b/test/emqx_broker_sup_SUITE.erl index 04881f308..20bcfbbf6 100644 --- a/test/emqx_broker_sup_SUITE.erl +++ b/test/emqx_broker_sup_SUITE.erl @@ -61,5 +61,5 @@ t_restart_shared_sub(Config) when is_list(Config) -> after 2000 -> false end); -t_restart_shared_sub({'end', Config}) -> +t_restart_shared_sub({'end', _Config}) -> emqx:unsubscribe(<<"$share/grpa/t/a">>). From e9a5c4295c725b26b9858c718b8f10b20ac6a5e5 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 9 Aug 2022 10:29:17 +0800 Subject: [PATCH 13/73] fix: bad NeedToLoad plugins --- src/emqx_plugins.erl | 2 +- test/emqx_plugins_SUITE.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index caf47b0c0..764778b74 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -265,7 +265,7 @@ load_plugins(Names, Persistent) -> [] -> ok; NotFound -> ?LOG(alert, "cannot_find_plugins: ~p", [NotFound]) end, - NeedToLoad = Names -- NotFound -- names(started_app), + NeedToLoad = (Names -- NotFound) -- names(started_app), lists:foreach(fun(Name) -> Plugin = find_plugin(Name, Plugins), load_plugin(Plugin#plugin.name, Persistent) diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl index cd4b68d86..66a88a047 100644 --- a/test/emqx_plugins_SUITE.erl +++ b/test/emqx_plugins_SUITE.erl @@ -100,7 +100,7 @@ t_ensure_default_loaded_plugins_file(Config) -> [ {emqx_bridge_mqtt, false} , {emqx_dashboard, true} , {emqx_management, true} - , {emqx_modules, false} + , {emqx_modules, true} , {emqx_recon, true} , {emqx_retainer, true} , {emqx_rule_engine, true} From c93dbf7624402c66f71d9162979df5174cdcd099 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 9 Aug 2022 13:21:39 +0200 Subject: [PATCH 14/73] chore: add tzdata to alpine docker image --- deploy/docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 346d33630..40d03f86a 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -16,6 +16,7 @@ RUN apk add --no-cache \ libc-dev \ libstdc++ \ bash \ + tzdata \ jq COPY . /emqx @@ -48,7 +49,7 @@ COPY deploy/docker/docker-entrypoint.sh /usr/bin/ COPY --from=builder /emqx/_build/$EMQX_NAME/rel/emqx /opt/emqx RUN ln -s /opt/emqx/bin/* /usr/local/bin/ -RUN apk add --no-cache curl ncurses-libs openssl sudo libstdc++ bash +RUN apk add --no-cache curl ncurses-libs openssl sudo libstdc++ bash tzdata WORKDIR /opt/emqx From 20d6a53fa188d55fb52b3bc333c10420d5f24652 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 9 Aug 2022 13:23:22 +0200 Subject: [PATCH 15/73] chore: add changelog for tzdata addition in alpine image --- CHANGES-4.3.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 5f387222d..8e982cfa6 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -20,6 +20,7 @@ File format: ### Enhancements - Improve error message for LwM2M plugin when object ID is not valid [#8654](https://github.com/emqx/emqx/pull/8654). +- Add tzdata apk package to alpine docker image. [#8671](https://github.com/emqx/emqx/pull/8671) ## v4.3.18 From 6630fe6318cbf4ddf486e6536550d198a7d3bbbb Mon Sep 17 00:00:00 2001 From: Rory Z Date: Wed, 10 Aug 2022 09:54:19 +0800 Subject: [PATCH 16/73] ci: cancel latest tag for docker image only EMQX 5.0 image can update latest tag --- .github/workflows/build_packages.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 84e02bf89..19ed368a6 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -365,9 +365,9 @@ jobs: id: meta with: images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }} - ## only stable tag is latest + ## only 5.0 is latest flavor: | - latest=${{ contains(github.ref, 'tags') && !contains(github.ref_name, 'rc') && !contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha') }} + latest=false tags: | type=ref,event=branch type=ref,event=pr From dd5eea5ada7fd1ec5603ca9fd73f90a7426523bd Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 10 Aug 2022 14:34:35 +0200 Subject: [PATCH 17/73] build: bump relup helper to 2.1.0 --- rebar.config.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.erl b/rebar.config.erl index db761e367..78e1ef94e 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -88,7 +88,7 @@ project_app_dirs() -> ["apps/*", alternative_lib_dir() ++ "/*", "."]. plugins(HasElixir) -> - [ {relup_helper,{git,"https://github.com/emqx/relup_helper", {tag, "2.0.0"}}} + [ {relup_helper,{git,"https://github.com/emqx/relup_helper", {tag, "2.1.0"}}} , {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}} %% emqx main project does not require port-compiler %% pin at root level for deterministic From 6a0d2c9d46a2a0708db1b31ea70434ed532754a2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 11 Aug 2022 08:25:37 +0200 Subject: [PATCH 18/73] fix: add the 'die' function in bin/emqx --- bin/emqx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/emqx b/bin/emqx index 20cf266b0..7a31f32ce 100755 --- a/bin/emqx +++ b/bin/emqx @@ -36,6 +36,13 @@ ERTS_LIB_DIR="$ERTS_DIR/../lib" # Echo to stderr on errors echoerr() { echo "$*" 1>&2; } +die() { + set +x + echoerr "ERROR: $1" + errno=${2:-1} + exit "$errno" +} + assert_node_alive() { if ! relx_nodetool "ping" > /dev/null; then die "node_is_not_running!" 1 From 3a917605f585582e8f6cd2e9f61778b11c93f52d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 11 Aug 2022 13:15:45 +0200 Subject: [PATCH 19/73] fix(bin/emqx): avoid creating crash dump file when checking erlang --- bin/emqx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/emqx b/bin/emqx index 7a31f32ce..13371f9da 100755 --- a/bin/emqx +++ b/bin/emqx @@ -52,7 +52,8 @@ assert_node_alive() { check_erlang_start() { # Fix bin permission find "$BINDIR" ! -executable -exec chmod a+x {} \; - "$BINDIR/$PROGNAME" -boot "$REL_DIR/start_clean" -eval "crypto:start(),halt()" + # set ERL_CRASH_DUMP_BYTES to zero so it will not write a crash dump file + env ERL_CRASH_DUMP_BYTES=0 "$BINDIR/$PROGNAME" -boot "$REL_DIR/start_clean" -eval "crypto:start(),halt()" } if ! check_erlang_start >/dev/null 2>&1; then From 4531703d1430ba159d8de96861761a8e002b53aa Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 11 Aug 2022 13:17:56 +0200 Subject: [PATCH 20/73] refactor(bin/emqx): move file permission fix out from check_erlang_start --- bin/emqx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/emqx b/bin/emqx index 13371f9da..f129a8447 100755 --- a/bin/emqx +++ b/bin/emqx @@ -33,6 +33,10 @@ export PROGNAME="erl" DYNLIBS_DIR="$RUNNER_ROOT_DIR/dynlibs" ERTS_LIB_DIR="$ERTS_DIR/../lib" +# Fix bin permission for all erts bin files +# the 'x' attributes may get lost if the files are extracted from a relup package +find "$BINDIR" -exec chmod a+x {} \; + # Echo to stderr on errors echoerr() { echo "$*" 1>&2; } @@ -50,8 +54,6 @@ assert_node_alive() { } check_erlang_start() { - # Fix bin permission - find "$BINDIR" ! -executable -exec chmod a+x {} \; # set ERL_CRASH_DUMP_BYTES to zero so it will not write a crash dump file env ERL_CRASH_DUMP_BYTES=0 "$BINDIR/$PROGNAME" -boot "$REL_DIR/start_clean" -eval "crypto:start(),halt()" } From 2ef8a985b34adc085bff8ce5fb4399b0afbad764 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 12 Aug 2022 10:41:05 +0800 Subject: [PATCH 21/73] chore: bump vsn to 4.3.18 --- include/emqx_release.hrl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 77a02177b..305815de1 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,11 +29,10 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.18-beta.1"}). +-define(EMQX_RELEASE, {opensource, "4.3.18"}). -else. - -endif. -endif. From 9064b5acb8248599323b7f7fa8fc4e789fb700a3 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 28 Jul 2022 15:42:57 +0300 Subject: [PATCH 22/73] chore(ci): make apps-version-check.sh accept new apps --- scripts/apps-version-check.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index ffef46c87..2cb05d3ed 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -19,6 +19,14 @@ while read -r app; do app_path="." fi src_file="$app_path/src/$(basename "$app").app.src" + + old_app_exists=0 + git show "$latest_release":"$src_file" >/dev/null 2>&1 || old_app_exists="$?" + if [ "$old_app_exists" != "0" ]; then + echo "$app is new, skipping version check" + continue + fi + old_app_version="$(git show "$latest_release":"$src_file" | grep vsn | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"')" now_app_version=$(grep -E 'vsn' "$src_file" | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') if [ "$old_app_version" = "$now_app_version" ]; then @@ -48,7 +56,7 @@ while read -r app; do now_app_version_semver=($(parse_semver "$now_app_version")) if [ "${old_app_version_semver[0]}" = "${now_app_version_semver[0]}" ] && \ [ "${old_app_version_semver[1]}" = "${now_app_version_semver[1]}" ] && \ - [ "$(( "${old_app_version_semver[2]}" + 1 ))" = "${now_app_version_semver[2]}" ]; then + [ "$(( old_app_version_semver[2] + 1 ))" = "${now_app_version_semver[2]}" ]; then true else echo "$src_file: non-strict semver version bump from $old_app_version to $now_app_version" From a19fbe214f2971e9971185e705cbff7296101087 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 28 Apr 2022 15:58:17 +0300 Subject: [PATCH 23/73] feat(node_rebalance): implement node evacuation and rebalancing --- CHANGES-4.3.md | 2 + apps/emqx_eviction_agent/.gitignore | 19 + apps/emqx_eviction_agent/README.md | 9 + .../etc/emqx_eviction_agent.conf | 3 + .../priv/emqx_eviction_agent.schema | 0 apps/emqx_eviction_agent/rebar.config | 2 + .../src/emqx_eviction_agent.app.src | 18 + .../src/emqx_eviction_agent.erl | 291 ++++++++++++ .../src/emqx_eviction_agent_api.erl | 36 ++ .../src/emqx_eviction_agent_app.erl | 36 ++ .../src/emqx_eviction_agent_channel.erl | 299 +++++++++++++ .../src/emqx_eviction_agent_cli.erl | 42 ++ .../src/emqx_eviction_agent_conn_sup.erl | 33 ++ .../src/emqx_eviction_agent_sup.erl | 43 ++ .../test/emqx_eviction_agent_SUITE.erl | 232 ++++++++++ .../test/emqx_eviction_agent_api_SUITE.erl | 64 +++ .../emqx_eviction_agent_channel_SUITE.erl | 155 +++++++ .../test/emqx_eviction_agent_cli_SUITE.erl | 47 ++ .../test/emqx_eviction_agent_test_helpers.erl | 55 +++ .../test/emqx_mgmt_api_SUITE.erl | 56 +-- .../test/emqx_mgmt_api_test_helpers.erl | 69 +++ apps/emqx_node_rebalance/.gitignore | 19 + apps/emqx_node_rebalance/README.md | 9 + .../etc/emqx_node_rebalance.conf | 3 + .../include/emqx_node_rebalance.hrl | 31 ++ .../priv/emqx_node_rebalance.schema | 0 apps/emqx_node_rebalance/rebar.config | 2 + .../src/emqx_node_rebalance.app.src | 19 + .../src/emqx_node_rebalance.erl | 414 ++++++++++++++++++ .../src/emqx_node_rebalance_agent.erl | 127 ++++++ .../src/emqx_node_rebalance_api.erl | 243 ++++++++++ .../src/emqx_node_rebalance_app.erl | 34 ++ .../src/emqx_node_rebalance_cli.erl | 265 +++++++++++ .../src/emqx_node_rebalance_evacuation.erl | 298 +++++++++++++ ...emqx_node_rebalance_evacuation_persist.erl | 109 +++++ .../src/emqx_node_rebalance_status.erl | 225 ++++++++++ .../src/emqx_node_rebalance_sup.erl | 44 ++ .../test/emqx_node_rebalance_SUITE.erl | 183 ++++++++ .../test/emqx_node_rebalance_agent_SUITE.erl | 163 +++++++ .../test/emqx_node_rebalance_api_SUITE.erl | 321 ++++++++++++++ .../test/emqx_node_rebalance_cli_SUITE.erl | 199 +++++++++ .../emqx_node_rebalance_evacuation_SUITE.erl | 194 ++++++++ ...ode_rebalance_evacuation_persist_SUITE.erl | 107 +++++ .../src/emqx_rule_engine.app.src | 2 +- .../src/emqx_rule_engine.appup.src | 6 +- data/loaded_plugins.tmpl | 2 + .../emqx_dashboard/src/emqx_dashboard.app.src | 2 +- rebar.config.erl | 4 + src/emqx.app.src | 2 +- src/emqx.appup.src | 46 +- src/emqx_channel.erl | 20 +- src/emqx_cm.erl | 46 +- src/emqx_plugins.erl | 2 + test/emqx_node_helpers.erl | 83 ++++ test/emqx_plugins_SUITE.erl | 2 + test/emqx_shared_sub_SUITE.erl | 61 +-- 56 files changed, 4668 insertions(+), 130 deletions(-) create mode 100644 apps/emqx_eviction_agent/.gitignore create mode 100644 apps/emqx_eviction_agent/README.md create mode 100644 apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf create mode 100644 apps/emqx_eviction_agent/priv/emqx_eviction_agent.schema create mode 100644 apps/emqx_eviction_agent/rebar.config create mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src create mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent.erl create mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl create mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl create mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl create mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl create mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl create mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl create mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl create mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl create mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl create mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl create mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl create mode 100644 apps/emqx_management/test/emqx_mgmt_api_test_helpers.erl create mode 100644 apps/emqx_node_rebalance/.gitignore create mode 100644 apps/emqx_node_rebalance/README.md create mode 100644 apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf create mode 100644 apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl create mode 100644 apps/emqx_node_rebalance/priv/emqx_node_rebalance.schema create mode 100644 apps/emqx_node_rebalance/rebar.config create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance.erl create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl create mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl create mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl create mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl create mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl create mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl create mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl create mode 100644 test/emqx_node_helpers.erl diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 1345ddc65..10304fc70 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -21,6 +21,7 @@ File format: - Improve error message for LwM2M plugin when object ID is not valid [#8654](https://github.com/emqx/emqx/pull/8654). - Add tzdata apk package to alpine docker image. [#8671](https://github.com/emqx/emqx/pull/8671) +- Add node evacuation and cluster rebalancing features [#8597] ## v4.3.19 @@ -55,6 +56,7 @@ File format: - HTTP API(GET /rules/) support for pagination and fuzzy filtering. [#8450] - Add check_conf cli to check config format. [#8486] - Optimize performance of shared subscription +- Make possible to debug-print SSL handshake procedure by setting listener config `log_level=debug` [#8553](https://github.com/emqx/emqx/pull/8553) ## v4.3.16 diff --git a/apps/emqx_eviction_agent/.gitignore b/apps/emqx_eviction_agent/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_eviction_agent/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_eviction_agent/README.md b/apps/emqx_eviction_agent/README.md new file mode 100644 index 000000000..f9b8037bf --- /dev/null +++ b/apps/emqx_eviction_agent/README.md @@ -0,0 +1,9 @@ +emqx_eviction_agent +===== + +An OTP library + +Build +----- + + $ rebar3 compile diff --git a/apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf b/apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf new file mode 100644 index 000000000..011b7fb0f --- /dev/null +++ b/apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf @@ -0,0 +1,3 @@ +##-------------------------------------------------------------------- +## EMQX Eviction Agent Plugin +##-------------------------------------------------------------------- diff --git a/apps/emqx_eviction_agent/priv/emqx_eviction_agent.schema b/apps/emqx_eviction_agent/priv/emqx_eviction_agent.schema new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_eviction_agent/rebar.config b/apps/emqx_eviction_agent/rebar.config new file mode 100644 index 000000000..2656fd554 --- /dev/null +++ b/apps/emqx_eviction_agent/rebar.config @@ -0,0 +1,2 @@ +{erl_opts, [debug_info]}. +{deps, []}. diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src b/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src new file mode 100644 index 000000000..ffa28af61 --- /dev/null +++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src @@ -0,0 +1,18 @@ +{application, emqx_eviction_agent, + [{description, "EMQX Eviction Agent"}, + {vsn, "4.3.0"}, + {registered, [emqx_eviction_agent_sup, + emqx_eviction_agent, + emqx_eviction_agent_conn_sup]}, + {applications, + [kernel, + stdlib + ]}, + {mod, {emqx_eviction_agent_app,[]}}, + {env,[]}, + {modules, []}, + {maintainers, ["EMQX Team "]}, + {links, [{"Homepage", "https://emqx.io/"}, + {"Github", "https://github.com/emqx"} + ]} + ]}. diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent.erl new file mode 100644 index 000000000..a8cf442cf --- /dev/null +++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent.erl @@ -0,0 +1,291 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). + +-include_lib("stdlib/include/qlc.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-export([start_link/0, + enable/2, + disable/1, + status/0, + connection_count/0, + session_count/0, + session_count/1, + evict_connections/1, + evict_sessions/2, + evict_sessions/3, + evict_session_channel/3 + ]). + +-behaviour(gen_server). + +-export([init/1, + handle_call/3, + handle_info/2, + handle_cast/2, + code_change/3 + ]). + +-export([on_connect/2, + on_connack/3]). + +-export([hook/0, + unhook/0]). + +-export_type([server_reference/0]). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +-type server_reference() :: binary() | undefined. +-type status() :: {enabled, conn_stats()} | disabled. +-type conn_stats() :: #{connections := non_neg_integer(), + sessions := non_neg_integer()}. +-type kind() :: atom(). + +-spec start_link() -> startlink_ret(). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec enable(kind(), server_reference()) -> ok_or_error(eviction_agent_busy). +enable(Kind, ServerReference) -> + gen_server:call(?MODULE, {enable, Kind, ServerReference}). + +-spec disable(kind()) -> ok. +disable(Kind) -> + gen_server:call(?MODULE, {disable, Kind}). + +-spec status() -> status(). +status() -> + case enable_status() of + {enabled, _Kind, _ServerReference} -> + {enabled, stats()}; + disabled -> + disabled + end. + +-spec evict_connections(pos_integer()) -> ok_or_error(disabled). +evict_connections(N) -> + case enable_status() of + {enabled, _Kind, ServerReference} -> + ok = do_evict_connections(N, ServerReference); + disabled -> + {error, disabled} + end. + +-spec evict_sessions(pos_integer(), node() | [node()]) -> ok_or_error(disabled). +evict_sessions(N, Node) when is_atom(Node) -> + evict_sessions(N, [Node]); +evict_sessions(N, Nodes) when is_list(Nodes) andalso length(Nodes) > 0 -> + evict_sessions(N, Nodes, any). + +-spec evict_sessions(pos_integer(), node() | [node()], atom()) -> ok_or_error(disabled). +evict_sessions(N, Node, ConnState) when is_atom(Node) -> + evict_sessions(N, [Node], ConnState); +evict_sessions(N, Nodes, ConnState) + when is_list(Nodes) andalso length(Nodes) > 0 -> + case enable_status() of + {enabled, _Kind, _ServerReference} -> + ok = do_evict_sessions(N, Nodes, ConnState); + disabled -> + {error, disabled} + end. + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([]) -> + _ = persistent_term:erase(?MODULE), + {ok, #{}}. + +%% enable +handle_call({enable, Kind, ServerReference}, _From, St) -> + Reply = case enable_status() of + disabled -> + ok = persistent_term:put(?MODULE, {enabled, Kind, ServerReference}); + {enabled, Kind, _ServerReference} -> + ok = persistent_term:put(?MODULE, {enabled, Kind, ServerReference}); + {enabled, _OtherKind, _ServerReference} -> + {error, eviction_agent_busy} + end, + {reply, Reply, St}; + +%% disable +handle_call({disable, Kind}, _From, St) -> + Reply = case enable_status() of + disabled -> + {error, disabled}; + {enabled, Kind, _ServerReference} -> + _ = persistent_term:erase(?MODULE), + ok; + {enabled, _OtherKind, _ServerReference} -> + {error, eviction_agent_busy} + end, + {reply, Reply, St}. + +handle_info(Msg, St) -> + ?LOG(warning, "Unknown Msg: ~p, State: ~p", [Msg, St]), + {noreply, St}. + +handle_cast(Msg, St) -> + ?LOG(warning, "Unknown cast Msg: ~p, State: ~p", [Msg, St]), + {noreply, St}. + +code_change(_Vsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Hook callbacks +%%-------------------------------------------------------------------- + +on_connect(_ConnInfo, _Props) -> + case enable_status() of + {enabled, _Kind, _ServerReference} -> + {stop, {error, ?RC_USE_ANOTHER_SERVER}}; + disabled -> + ignore + end. + +on_connack(#{proto_name := <<"MQTT">>, proto_ver := ?MQTT_PROTO_V5}, + use_another_server, + Props) -> + case enable_status() of + {enabled, _Kind, ServerReference} -> + {ok, Props#{'Server-Reference' => ServerReference}}; + disabled -> + {ok, Props} + end; +on_connack(_ClientInfo, _Reason, Props) -> + {ok, Props}. + +%%-------------------------------------------------------------------- +%% Hook funcs +%%-------------------------------------------------------------------- + +hook() -> + ?tp(debug, eviction_agent_hook, #{}), + ok = emqx_hooks:put('client.connack', {?MODULE, on_connack, []}), + ok = emqx_hooks:put('client.connect', {?MODULE, on_connect, []}). + +unhook() -> + ?tp(debug, eviction_agent_unhook, #{}), + ok = emqx_hooks:del('client.connect', {?MODULE, on_connect}), + ok = emqx_hooks:del('client.connack', {?MODULE, on_connack}). + +enable_status() -> + persistent_term:get(?MODULE, disabled). + +% connection management +stats() -> + #{ + connections => connection_count(), + sessions => session_count() + }. + +connection_table() -> + emqx_cm:live_connection_table(). + +connection_count() -> + table_count(connection_table()). + +channel_with_session_table(any) -> + qlc:q([{ClientId, ConnInfo, ClientInfo} + || {ClientId, _, ConnInfo, ClientInfo} <- emqx_cm:channel_with_session_table()]); +channel_with_session_table(RequiredConnState) -> + qlc:q([{ClientId, ConnInfo, ClientInfo} + || {ClientId, ConnState, ConnInfo, ClientInfo} <- emqx_cm:channel_with_session_table(), + RequiredConnState =:= ConnState]). + +session_count() -> + session_count(any). + +session_count(ConnState) -> + table_count(channel_with_session_table(ConnState)). + +table_count(QH) -> + qlc:fold(fun(_, Acc) -> Acc + 1 end, 0, QH). + +take_connections(N) -> + ChanQH = qlc:q([ChanPid || {_ClientId, ChanPid} <- connection_table()]), + ChanPidCursor = qlc:cursor(ChanQH), + ChanPids = qlc:next_answers(ChanPidCursor, N), + ok = qlc:delete_cursor(ChanPidCursor), + ChanPids. + +take_channel_with_sessions(N, ConnState) -> + ChanPidCursor = qlc:cursor(channel_with_session_table(ConnState)), + Channels = qlc:next_answers(ChanPidCursor, N), + ok = qlc:delete_cursor(ChanPidCursor), + Channels. + +do_evict_connections(N, ServerReference) when N > 0 -> + ChanPids = take_connections(N), + ok = lists:foreach( + fun(ChanPid) -> + disconnect_channel(ChanPid, ServerReference) + end, + ChanPids). + +do_evict_sessions(N, Nodes, ConnState) when N > 0 -> + Channels = take_channel_with_sessions(N, ConnState), + ok = lists:foreach( + fun({ClientId, ConnInfo, ClientInfo}) -> + evict_session_channel(Nodes, ClientId, ConnInfo, ClientInfo) + end, + Channels). + +evict_session_channel(Nodes, ClientId, ConnInfo, ClientInfo) -> + Node = select_random(Nodes), + ?LOG(info, "Evicting client=~p to node=~p, conninfo=~p, clientinfo=~p", + [ClientId, Node, ConnInfo, ClientInfo]), + case rpc:call(Node, ?MODULE, evict_session_channel, [ClientId, ConnInfo, ClientInfo]) of + {badrpc, Reason} -> + ?LOG(error, "RPC error while evicting client=~p to node=~p: ~p", + [ClientId, Node, Reason]), + {error, Reason}; + {error, Reason} = Error -> + ?LOG(error, "Error evicting client=~p to node=~p: ~p", + [ClientId, Node, Reason]), + Error; + Res -> Res + end. + +evict_session_channel(ClientId, ConnInfo, ClientInfo) -> + ?LOG(info, "Taking up client=~p, conninfo=~p, clientinfo=~p", + [ClientId, ConnInfo, ClientInfo]), + Result = emqx_eviction_agent_channel:start_supervised( + #{conninfo => ConnInfo, + clientinfo => ClientInfo}), + ?LOG(info, "Taking up client=~p, result=~p", + [ClientId, Result]), + Result. + +disconnect_channel(ChanPid, ServerReference) -> + ChanPid ! {disconnect, + ?RC_USE_ANOTHER_SERVER, + use_another_server, + #{'Server-Reference' => ServerReference}}. + +select_random(List) when length(List) > 0 -> + lists:nth(rand:uniform(length(List)) , List). diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl new file mode 100644 index 000000000..426f5978d --- /dev/null +++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl @@ -0,0 +1,36 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_api). + +-include_lib("emqx/include/logger.hrl"). + +-rest_api(#{name => node_eviction_status, + method => 'GET', + path => "/node_eviction/status", + func => status, + descr => "Get node eviction status"}). + +-export([status/2]). + +status(_Bindings, _Params) -> + case emqx_eviction_agent:status() of + disabled -> + {ok, #{status => disabled}}; + {enabled, Stats} -> + {ok, #{status => enabled, + stats => Stats}} + end. diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl new file mode 100644 index 000000000..c13fcfb0d --- /dev/null +++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl @@ -0,0 +1,36 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_app). + +-behaviour(application). + +-emqx_plugin(?MODULE). + +-export([ start/2 + , stop/1 + ]). + +start(_Type, _Args) -> + Env = application:get_all_env(emqx_eviction_agent), + ok = emqx_eviction_agent:hook(), + {ok, Sup} = emqx_eviction_agent_sup:start_link(Env), + ok = emqx_eviction_agent_cli:load(), + {ok, Sup}. + +stop(_State) -> + ok = emqx_eviction_agent:unhook(), + ok = emqx_eviction_agent_cli:unload(). diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl new file mode 100644 index 000000000..1f6fad00f --- /dev/null +++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl @@ -0,0 +1,299 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% MQTT Channel +-module(emqx_eviction_agent_channel). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-logger_header("[Evicted Channel]"). + +-export([start_link/1, + start_supervised/1, + call/2, + call/3, + cast/2, + stop/1 + ]). + +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 + ]). + +-type opts() :: #{conninfo := emqx_types:conninfo(), + clientinfo := emqx_types:clientinfo()}. + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec start_supervised(opts()) -> startlink_ret(). +start_supervised(#{clientinfo := #{clientid := ClientId}} = Opts) -> + RandomId = integer_to_binary(erlang:unique_integer([positive])), + Id = <>, + ChildSpec = #{id => Id, + start => {?MODULE, start_link, [Opts]}, + restart => temporary, + shutdown => 5000, + type => worker, + modules => [?MODULE] + }, + supervisor:start_child( + emqx_eviction_agent_conn_sup, + ChildSpec). + +-spec start_link(opts()) -> startlink_ret(). +start_link(Opts) -> + gen_server:start_link(?MODULE, [Opts], []). + +-spec cast(pid(), term()) -> ok. +cast(Pid, Req) -> + gen_server:cast(Pid, Req). + +-spec call(pid(), term()) -> term(). +call(Pid, Req) -> + call(Pid, Req, infinity). + +-spec call(pid(), term(), timeout()) -> term(). +call(Pid, Req, Timeout) -> + gen_server:call(Pid, Req, Timeout). + +-spec stop(pid()) -> ok. +stop(Pid) -> + gen_server:stop(Pid). + +%%-------------------------------------------------------------------- +%% gen_server API +%%-------------------------------------------------------------------- + +init([#{conninfo := OldConnInfo, clientinfo := #{clientid := ClientId} = OldClientInfo}]) -> + process_flag(trap_exit, true), + ClientInfo = clientinfo(OldClientInfo), + ConnInfo = conninfo(OldConnInfo), + case open_session(ConnInfo, ClientInfo) of + {ok, Channel0} -> + case set_expiry_timer(Channel0) of + {ok, Channel1} -> + ?LOG( + info, + "Channel initialized for client=~p on node=~p", + [ClientId, node()]), + {ok, Channel1, hibernate}; + {error, Reason} -> + {stop, Reason} + end; + {error, Reason} -> + {stop, Reason} + end. + +handle_call(kick, _From, Channel) -> + {stop, kicked, ok, Channel}; + +handle_call(discard, _From, Channel) -> + {stop, discarded, ok, Channel}; + +handle_call({takeover, 'begin'}, _From, #{session := Session} = Channel) -> + {reply, Session, Channel#{takeover => true}}; + +handle_call({takeover, 'end'}, _From, #{session := Session, + clientinfo := #{clientid := ClientId}, + pendings := Pendings} = Channel) -> + ok = emqx_session:takeover(Session), + %% TODO: Should not drain deliver here (side effect) + Delivers = emqx_misc:drain_deliver(), + AllPendings = lists:append(Delivers, Pendings), + ?tp(debug, + emqx_channel_takeover_end, + #{clientid => ClientId}), + {stop, normal, AllPendings, Channel}; + +handle_call(list_acl_cache, _From, Channel) -> + {reply, [], Channel}; + +handle_call({quota, _Policy}, _From, Channel) -> + {reply, ok, Channel}; + +handle_call(Req, _From, Channel) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, Channel}. + +handle_info(Deliver = {deliver, _Topic, _Msg}, Channel) -> + Delivers = [Deliver | emqx_misc:drain_deliver()], + {noreply, handle_deliver(Delivers, Channel)}; + +handle_info(expire_session, Channel) -> + {stop, expired, Channel}; + +handle_info(Info, Channel) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, Channel}. + +handle_cast(Msg, Channel) -> + ?LOG(error, "Unexpected cast: ~p", [Msg]), + {noreply, Channel}. + +terminate(Reason, #{clientinfo := ClientInfo, session := Session} = Channel) -> + ok = cancel_expiry_timer(Channel), + emqx_session:terminate(ClientInfo, Reason, Session). + +code_change(_OldVsn, Channel, _Extra) -> + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +handle_deliver(Delivers, + #{takeover := true, + pendings := Pendings, + session := Session, + clientinfo := #{clientid := ClientId} = ClientInfo} = Channel) -> + %% NOTE: Order is important here. While the takeover is in + %% progress, the session cannot enqueue messages, since it already + %% passed on the queue to the new connection in the session state. + NPendings = lists:append( + Pendings, + ignore_local(ClientInfo, maybe_nack(Delivers), ClientId, Session)), + Channel#{pendings => NPendings}; + +handle_deliver(Delivers, + #{takeover := false, + session := Session, + clientinfo := #{clientid := ClientId} = ClientInfo} = Channel) -> + NSession = emqx_session:enqueue( + ClientInfo, + ignore_local(ClientInfo, maybe_nack(Delivers), ClientId, Session), + Session), + Channel#{session => NSession}. + +cancel_expiry_timer(#{expiry_timer := TRef}) when is_reference(TRef) -> + _ = erlang:cancel_timer(TRef), + ok; +cancel_expiry_timer(_) -> + ok. + +set_expiry_timer(#{conninfo := ConnInfo} = Channel) -> + case maps:get(expiry_interval, ConnInfo) of + ?UINT_MAX -> {ok, Channel}; + I when I > 0 -> + Timer = erlang:send_after(timer:seconds(I), self(), expire_session), + {ok, Channel#{expiry_timer => Timer}}; + _ -> + {error, should_be_expired} + end. + +open_session(ConnInfo, #{clientid := ClientId} = ClientInfo) -> + Channel = channel(ConnInfo, ClientInfo), + case emqx_cm:open_session(false, ClientInfo, ConnInfo) of + {ok, #{present := false}} -> + ?LOG(info, "No session for clientid=~p", [ClientId]), + {error, no_session}; + {ok, #{session := Session, present := true, pendings := Pendings0}} -> + ?LOG(info, "Session opened for client=~p on node=~p", [ClientId, node()]), + Pendings1 = lists:usort(lists:append(Pendings0, emqx_misc:drain_deliver())), + NSession = emqx_session:enqueue( + ClientInfo, + ignore_local( + ClientInfo, + maybe_nack(Pendings1), + ClientId, + Session), + Session), + NChannel = Channel#{session => NSession}, + ok = emqx_cm:insert_channel_info(ClientId, info(NChannel), []), + ?LOG(info, "Channel info updated for client=~p on node=~p", [ClientId, node()]), + {ok, NChannel}; + {error, Reason} = Error -> + ?LOG(error, "Failed to open session due to ~p", [Reason]), + Error + end. + +conninfo(OldConnInfo) -> + DisconnectedAt = maps:get(disconnected_at, OldConnInfo, erlang:system_time(millisecond)), + ConnInfo0 = maps:with( + [socktype, + sockname, + peername, + peercert, + clientid, + clean_start, + receive_maximum, + expiry_interval], + OldConnInfo), + ConnInfo0#{ + conn_mod => ?MODULE, + connected => false, + disconnected_at => DisconnectedAt + }. + +clientinfo(OldClientInfo) -> + maps:with( + [zone, + protocol, + peerhost, + sockport, + clientid, + username, + is_bridge, + is_superuser, + mountpoint], + OldClientInfo). + +channel(ConnInfo, ClientInfo) -> + #{conninfo => ConnInfo, + clientinfo => ClientInfo, + expiry_timer => undefined, + takeover => false, + resuming => false, + pendings => [] + }. + +info(Channel) -> + #{conninfo => maps:get(conninfo, Channel, undefined), + clientinfo => maps:get(clientinfo, Channel, undefined), + session => maps:get(session, Channel, undefined), + conn_state => disconnected + }. + +ignore_local(ClientInfo, Delivers, Subscriber, Session) -> + Subs = emqx_session:info(subscriptions, Session), + lists:dropwhile(fun({deliver, Topic, #message{from = Publisher} = Msg}) -> + case maps:find(Topic, Subs) of + {ok, #{nl := 1}} when Subscriber =:= Publisher -> + ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, no_local]), + ok = emqx_metrics:inc('delivery.dropped'), + ok = emqx_metrics:inc('delivery.dropped.no_local'), + true; + _ -> + false + end + end, Delivers). + +maybe_nack(Delivers) -> + lists:filter(fun not_nacked/1, Delivers). + +not_nacked({deliver, _Topic, Msg}) -> + not (emqx_shared_sub:is_ack_required(Msg) + andalso (ok == emqx_shared_sub:nack_no_connection(Msg))). diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl new file mode 100644 index 000000000..632f8e480 --- /dev/null +++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl @@ -0,0 +1,42 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_cli). + +%% APIs +-export([ load/0 + , unload/0 + , cli/1 + ]). + +load() -> + emqx_ctl:register_command(eviction, {?MODULE, cli}, []). + +unload() -> + emqx_ctl:unregister_command(eviction). + +cli(["status"]) -> + case emqx_eviction_agent:status() of + disabled -> + emqx_ctl:print("Eviction status: disabled~n"); + {enabled, _Stats} -> + emqx_ctl:print("Eviction status: enabled~n") + end; + +cli(_) -> + emqx_ctl:usage( + [{"eviction status", + "Get current node eviction status"}]). diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl new file mode 100644 index 000000000..2b11ce9d1 --- /dev/null +++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl @@ -0,0 +1,33 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_conn_sup). + +-behaviour(supervisor). + +-export([start_link/1]). + +-export([init/1]). + +start_link(Env) -> + supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]). + +init([_Env]) -> + Childs = [], + {ok, { + {one_for_one, 10, 3600}, + Childs} + }. diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl new file mode 100644 index 000000000..9e28c2251 --- /dev/null +++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl @@ -0,0 +1,43 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_sup). + +-behaviour(supervisor). + +-export([start_link/1]). + +-export([init/1]). + +start_link(Env) -> + supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]). + +init([_Env]) -> + Childs = [child_spec(worker, emqx_eviction_agent, []), + child_spec(supervisor, emqx_eviction_agent_conn_sup, [#{}])], + {ok, { + {one_for_one, 10, 3600}, + Childs} + }. + +child_spec(Type, Mod, Args) -> + #{id => Mod, + start => {Mod, start_link, Args}, + restart => permanent, + shutdown => 5000, + type => Type, + modules => [Mod] + }. diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl new file mode 100644 index 000000000..b77c0dee8 --- /dev/null +++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl @@ -0,0 +1,232 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-import(emqx_eviction_agent_test_helpers, + [emqtt_connect/0, emqtt_connect/2]). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_eviction_agent]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_eviction_agent]). + +init_per_testcase(t_explicit_session_takeover, Config) -> + _ = emqx_eviction_agent:disable(test_eviction), + Node = emqx_node_helpers:start_slave( + evacuate1, + #{start_apps => [emqx, emqx_eviction_agent]}), + [{evacuate_node, Node} | Config]; +init_per_testcase(_TestCase, Config) -> + _ = emqx_eviction_agent:disable(test_eviction), + Config. + +end_per_testcase(t_explicit_session_takeover, Config) -> + _ = emqx_node_helpers:stop_slave(?config(evacuate_node, Config)), + _ = emqx_eviction_agent:disable(test_eviction); +end_per_testcase(_TestCase, _Config) -> + _ = emqx_eviction_agent:disable(test_eviction). + +t_enable_disable(_Config) -> + erlang:process_flag(trap_exit, true), + + ?assertMatch( + disabled, + emqx_eviction_agent:status()), + + {ok, C0} = emqtt_connect(), + ok = emqtt:disconnect(C0), + + ok = emqx_eviction_agent:enable(test_eviction, undefined), + + ?assertMatch( + {error, eviction_agent_busy}, + emqx_eviction_agent:enable(bar, undefined)), + + ?assertMatch( + ok, + emqx_eviction_agent:enable(test_eviction, <<"srv">>)), + + ?assertMatch( + {enabled, #{}}, + emqx_eviction_agent:status()), + + ?assertMatch( + {error, {use_another_server, #{}}}, + emqtt_connect()), + + ?assertMatch( + {error, eviction_agent_busy}, + emqx_eviction_agent:disable(bar)), + + ?assertMatch( + ok, + emqx_eviction_agent:disable(test_eviction)), + + ?assertMatch( + {error, disabled}, + emqx_eviction_agent:disable(test_eviction)), + + ?assertMatch( + disabled, + emqx_eviction_agent:status()), + + {ok, C1} = emqtt_connect(), + ok = emqtt:disconnect(C1). + + +t_evict_connections_status(_Config) -> + erlang:process_flag(trap_exit, true), + + {ok, _C} = emqtt_connect(), + + {error, disabled} = emqx_eviction_agent:evict_connections(1), + + ok = emqx_eviction_agent:enable(test_eviction, undefined), + + ?assertMatch( + {enabled, #{connections := 1, sessions := _}}, + emqx_eviction_agent:status()), + + ok = emqx_eviction_agent:evict_connections(1), + + ct:sleep(100), + + ?assertMatch( + {enabled, #{connections := 0, sessions := _}}, + emqx_eviction_agent:status()), + + ok = emqx_eviction_agent:disable(test_eviction). + + +t_explicit_session_takeover(Config) -> + erlang:process_flag(trap_exit, true), + + {ok, C0} = emqtt_connect(<<"client_with_session">>, false), + {ok, _, _} = emqtt:subscribe(C0, <<"t1">>), + + ok = emqx_eviction_agent:enable(test_eviction, undefined), + + ?assertEqual( + 1, + emqx_eviction_agent:connection_count()), + + ok = emqx_eviction_agent:evict_connections(1), + + receive + {'EXIT', C0, {disconnected, ?RC_USE_ANOTHER_SERVER, _}} -> ok + after 1000 -> + ?assert(false, "Connection not evicted") + end, + + ?assertEqual( + 0, + emqx_eviction_agent:connection_count()), + + ?assertEqual( + 1, + emqx_eviction_agent:session_count()), + + %% First, evacuate to the same node + + ?check_trace( + ?wait_async_action( + emqx_eviction_agent:evict_sessions(1, node()), + #{?snk_kind := emqx_channel_takeover_end}, + 1000), + fun(_Result, Trace) -> + ?assertMatch( + [#{clientid := <<"client_with_session">>} | _ ], + ?of_kind(emqx_channel_takeover_end, Trace)) + end), + + ok = emqx_eviction_agent:disable(test_eviction), + ok = connect_and_publish(<<"t1">>, <<"MessageToEvictedSession1">>), + ok = emqx_eviction_agent:enable(test_eviction, undefined), + + %% Evacuate to another node + + TargetNodeForEvacuation = ?config(evacuate_node, Config), + ?check_trace( + ?wait_async_action( + emqx_eviction_agent:evict_sessions(1, TargetNodeForEvacuation), + #{?snk_kind := emqx_channel_takeover_end}, + 1000), + fun(_Result, Trace) -> + ?assertMatch( + [#{clientid := <<"client_with_session">>} | _ ], + ?of_kind(emqx_channel_takeover_end, Trace)) + end), + + ?assertEqual( + 0, + emqx_eviction_agent:session_count()), + + ?assertEqual( + 1, + rpc:call(TargetNodeForEvacuation, emqx_eviction_agent, session_count, [])), + + ok = emqx_eviction_agent:disable(test_eviction), + + ct:pal("evicted chann info: ~p", [emqx_cm:get_chan_info(<<"client_with_session">>)]), + + ok = connect_and_publish(<<"t1">>, <<"MessageToEvictedSession2">>), + ct:sleep(100), + + {ok, C2} = emqtt_connect(<<"client_with_session">>, false), + + ok = assert_receive_publish( + [#{payload => <<"MessageToEvictedSession1">>, topic => <<"t1">>}, + #{payload => <<"MessageToEvictedSession2">>, topic => <<"t1">>}]), + ok = emqtt:disconnect(C2). + +t_disable_on_restart(_Config) -> + ok = emqx_eviction_agent:enable(test_eviction, undefined), + + ok = supervisor:terminate_child(emqx_eviction_agent_sup, emqx_eviction_agent), + {ok, _} = supervisor:restart_child(emqx_eviction_agent_sup, emqx_eviction_agent), + + ?assertEqual( + disabled, + emqx_eviction_agent:status()). + +assert_receive_publish([]) -> ok; +assert_receive_publish([#{payload := Msg, topic := Topic} | Rest]) -> + receive + {publish, #{payload := Msg, + topic := Topic}} -> + assert_receive_publish(Rest) + after 1000 -> + ?assert(false, "Message `" ++ binary_to_list(Msg) ++ "` is lost") + end. + +connect_and_publish(Topic, Message) -> + {ok, C} = emqtt_connect(), + emqtt:publish(C, Topic, Message), + ok = emqtt:disconnect(C). diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl new file mode 100644 index 000000000..f9585c0e6 --- /dev/null +++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl @@ -0,0 +1,64 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-import(emqx_mgmt_api_test_helpers, + [request_api/3, + auth_header_/0, + api_path/1]). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_management]), + Config. + +end_per_suite(Config) -> + emqx_ct_helpers:stop_apps([emqx_management, emqx_eviction_agent]), + Config. + +t_status(_Config) -> + ?assertMatch( + {ok, #{<<"status">> := <<"disabled">>}}, + api_get(["node_eviction", "status"])), + + ok = emqx_eviction_agent:enable(apitest, undefined), + + ?assertMatch( + {ok, #{<<"status">> := <<"enabled">>, + <<"stats">> := #{}}}, + api_get(["node_eviction", "status"])), + + ok = emqx_eviction_agent:disable(apitest), + + ?assertMatch( + {ok, #{<<"status">> := <<"disabled">>}}, + api_get(["node_eviction", "status"])). + +api_get(Path) -> + case request_api(get, api_path(Path), auth_header_()) of + {ok, ResponseBody} -> + {ok, jiffy:decode(list_to_binary(ResponseBody), [return_maps])}; + {error, _} = Error -> Error + end. diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl new file mode 100644 index 000000000..cbbf1be5a --- /dev/null +++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl @@ -0,0 +1,155 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_channel_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-define(CLIENT_ID, <<"client_with_session">>). + +-import(emqx_eviction_agent_test_helpers, + [emqtt_connect/2]). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_eviction_agent]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_eviction_agent]). + +t_start_no_session(_Config) -> + Opts = #{clientinfo => #{clientid => ?CLIENT_ID, + zone => internal}, + conninfo => #{clientid => ?CLIENT_ID, + receive_maximum => 32}}, + ?assertMatch( + {error, {no_session, _}}, + emqx_eviction_agent_channel:start_supervised(Opts)). + +t_start_no_expire(_Config) -> + erlang:process_flag(trap_exit, true), + + _ = emqtt_connect(?CLIENT_ID, false), + + Opts = #{clientinfo => #{clientid => ?CLIENT_ID, + zone => internal}, + conninfo => #{clientid => ?CLIENT_ID, + receive_maximum => 32, + expiry_interval => 0}}, + ?assertMatch( + {error, {should_be_expired, _}}, + emqx_eviction_agent_channel:start_supervised(Opts)). + +t_start_infinite_expire(_Config) -> + erlang:process_flag(trap_exit, true), + + _ = emqtt_connect(?CLIENT_ID, false), + + Opts = #{clientinfo => #{clientid => ?CLIENT_ID, + zone => internal}, + conninfo => #{clientid => ?CLIENT_ID, + receive_maximum => 32, + expiry_interval => ?UINT_MAX}}, + ?assertMatch( + {ok, _}, + emqx_eviction_agent_channel:start_supervised(Opts)). + +t_kick(_Config) -> + erlang:process_flag(trap_exit, true), + + _ = emqtt_connect(?CLIENT_ID, false), + Opts = evict_session_opts(?CLIENT_ID), + + {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts), + + ?assertEqual( + ok, + emqx_eviction_agent_channel:call(Pid, kick)). + +t_discard(_Config) -> + erlang:process_flag(trap_exit, true), + + _ = emqtt_connect(?CLIENT_ID, false), + Opts = evict_session_opts(?CLIENT_ID), + + {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts), + + ?assertEqual( + ok, + emqx_eviction_agent_channel:call(Pid, discard)). + +t_stop(_Config) -> + erlang:process_flag(trap_exit, true), + + _ = emqtt_connect(?CLIENT_ID, false), + Opts = evict_session_opts(?CLIENT_ID), + + {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts), + + ?assertEqual( + ok, + emqx_eviction_agent_channel:stop(Pid)). + + +t_ignored_calls(_Config) -> + erlang:process_flag(trap_exit, true), + + _ = emqtt_connect(?CLIENT_ID, false), + Opts = evict_session_opts(?CLIENT_ID), + + {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts), + + ok = emqx_eviction_agent_channel:cast(Pid, unknown), + Pid ! unknown, + + ?assertEqual( + [], + emqx_eviction_agent_channel:call(Pid, list_acl_cache)), + + ?assertEqual( + ok, + emqx_eviction_agent_channel:call(Pid, {quota, quota})), + + ?assertEqual( + ignored, + emqx_eviction_agent_channel:call(Pid, unknown)). + +t_expire(_Config) -> + erlang:process_flag(trap_exit, true), + + _ = emqtt_connect(?CLIENT_ID, false), + #{conninfo := ConnInfo} = Opts0 = evict_session_opts(?CLIENT_ID), + Opts1 = Opts0#{conninfo => ConnInfo#{expiry_interval => 1}}, + + {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts1), + + ct:sleep(1500), + + ?assertNot(is_process_alive(Pid)). + +evict_session_opts(ClientId) -> + maps:with( + [conninfo, clientinfo], + emqx_cm:get_chan_info(ClientId)). diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl new file mode 100644 index 000000000..9c05cdb21 --- /dev/null +++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl @@ -0,0 +1,47 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_cli_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_eviction_agent]), + Config. + +end_per_suite(Config) -> + _ = emqx_eviction_agent:disable(foo), + emqx_ct_helpers:stop_apps([emqx_eviction_agent]), + Config. + +t_status(_Config) -> + %% usage + ok = emqx_eviction_agent_cli:cli(["foobar"]), + + %% status + ok = emqx_eviction_agent_cli:cli(["status"]), + + ok = emqx_eviction_agent:enable(foo, undefined), + + %% status + ok = emqx_eviction_agent_cli:cli(["status"]). diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl new file mode 100644 index 000000000..68b86f059 --- /dev/null +++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl @@ -0,0 +1,55 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_eviction_agent_test_helpers). + +-export([emqtt_connect/0, + emqtt_connect/2, + emqtt_connect_many/1, + emqtt_try_connect/0]). + +emqtt_connect() -> + emqtt_connect(<<"client1">>, true). + +emqtt_connect(ClientId, CleanStart) -> + {ok, C} = emqtt:start_link( + [{clientid, ClientId}, + {clean_start, CleanStart}, + {proto_ver, v5}, + {properties, #{'Session-Expiry-Interval' => 600}} + ]), + case emqtt:connect(C) of + {ok, _} -> {ok, C}; + {error, _} = Error -> Error + end. + +emqtt_connect_many(Count) -> + lists:map( + fun(N) -> + NBin = integer_to_binary(N), + ClientId = <<"client-", NBin/binary>>, + {ok, C} = emqtt_connect(ClientId, false), + C + end, + lists:seq(1, Count)). + +emqtt_try_connect() -> + case emqtt_connect() of + {ok, C} -> + emqtt:disconnect(C), + ok; + {error, _} = Error -> Error + end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index cfed74c16..fda5c2adf 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -25,13 +25,12 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx_management/include/emqx_mgmt.hrl"). --define(CONTENT_TYPE, "application/x-www-form-urlencoded"). - --define(HOST, "http://127.0.0.1:8081/"). - --define(API_VERSION, "v4"). - --define(BASE_PATH, "api"). +-import(emqx_mgmt_api_test_helpers, + [request_api/3, + request_api/4, + request_api/5, + auth_header_/0, + api_path/1]). all() -> emqx_ct:all(?MODULE). @@ -657,49 +656,6 @@ t_data_import_content(_) -> application:stop(emqx_rule_engine), application:stop(emqx_dashboard). -request_api(Method, Url, Auth) -> - request_api(Method, Url, [], Auth, []). - -request_api(Method, Url, QueryParams, Auth) -> - request_api(Method, Url, QueryParams, Auth, []). - -request_api(Method, Url, QueryParams, Auth, []) -> - NewUrl = case QueryParams of - "" -> Url; - _ -> Url ++ "?" ++ QueryParams - end, - do_request_api(Method, {NewUrl, [Auth]}); -request_api(Method, Url, QueryParams, Auth, Body) -> - NewUrl = case QueryParams of - "" -> Url; - _ -> Url ++ "?" ++ QueryParams - end, - do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). - -do_request_api(Method, Request)-> - ct:pal("Method: ~p, Request: ~p", [Method, Request]), - case httpc:request(Method, Request, [], []) of - {error, socket_closed_remotely} -> - {error, socket_closed_remotely}; - {ok, {{"HTTP/1.1", Code, _}, _, Return} } - when Code =:= 200 orelse Code =:= 201 -> - {ok, Return}; - {ok, {Reason, _, _}} -> - {error, Reason} - end. - -auth_header_() -> - AppId = <<"admin">>, - AppSecret = <<"public">>, - auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)). - -auth_header_(User, Pass) -> - Encoded = base64:encode_to_string(lists:append([User,":",Pass])), - {"Authorization","Basic " ++ Encoded}. - -api_path(Parts)-> - ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts). - filter(List, Key, Value) -> lists:filter(fun(Item) -> maps:get(Key, Item) == Value diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_helpers.erl b/apps/emqx_management/test/emqx_mgmt_api_test_helpers.erl new file mode 100644 index 000000000..a943ca760 --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_api_test_helpers.erl @@ -0,0 +1,69 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mgmt_api_test_helpers). + +-compile(export_all). +-compile(nowarn_export_all). + +-define(HOST, "http://127.0.0.1:8081/"). + +-define(API_VERSION, "v4"). + +-define(BASE_PATH, "api"). + +request_api(Method, Url, Auth) -> + request_api(Method, Url, [], Auth, []). + +request_api(Method, Url, QueryParams, Auth) -> + request_api(Method, Url, QueryParams, Auth, []). + +request_api(Method, Url, QueryParams, Auth, []) -> + NewUrl = case QueryParams of + "" -> Url; + _ -> Url ++ "?" ++ QueryParams + end, + do_request_api(Method, {NewUrl, [Auth]}); +request_api(Method, Url, QueryParams, Auth, Body) -> + NewUrl = case QueryParams of + "" -> Url; + _ -> Url ++ "?" ++ QueryParams + end, + do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + +do_request_api(Method, Request)-> + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], []) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {ok, {{"HTTP/1.1", Code, _}, _, Return} } + when Code =:= 200 orelse Code =:= 201 -> + {ok, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +auth_header_() -> + AppId = <<"admin">>, + AppSecret = <<"public">>, + auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)). + +auth_header_(User, Pass) -> + Encoded = base64:encode_to_string(lists:append([User,":",Pass])), + {"Authorization","Basic " ++ Encoded}. + +api_path(Parts)-> + ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts). diff --git a/apps/emqx_node_rebalance/.gitignore b/apps/emqx_node_rebalance/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_node_rebalance/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_node_rebalance/README.md b/apps/emqx_node_rebalance/README.md new file mode 100644 index 000000000..2e56f62cd --- /dev/null +++ b/apps/emqx_node_rebalance/README.md @@ -0,0 +1,9 @@ +emqx_node_rebalance +===== + +An OTP library + +Build +----- + + $ rebar3 compile diff --git a/apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf b/apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf new file mode 100644 index 000000000..8ace22435 --- /dev/null +++ b/apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf @@ -0,0 +1,3 @@ +##-------------------------------------------------------------------- +## EMQX Node Rebalance Plugin +##-------------------------------------------------------------------- diff --git a/apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl b/apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl new file mode 100644 index 000000000..1903b87cc --- /dev/null +++ b/apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl @@ -0,0 +1,31 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-define(DEFAULT_CONN_EVICT_RATE, 500). +-define(DEFAULT_SESS_EVICT_RATE, 500). + +-define(DEFAULT_WAIT_HEALTH_CHECK, 60). %% sec +-define(DEFAULT_WAIT_TAKEOVER, 60). %% sec + +-define(DEFAULT_ABS_CONN_THRESHOLD, 1000). +-define(DEFAULT_ABS_SESS_THRESHOLD, 1000). + +-define(DEFAULT_REL_CONN_THRESHOLD, 1.1). +-define(DEFAULT_REL_SESS_THRESHOLD, 1.1). + +-define(EVICT_INTERVAL, 1000). + +-define(EVACUATION_FILENAME, <<".evacuation">>). diff --git a/apps/emqx_node_rebalance/priv/emqx_node_rebalance.schema b/apps/emqx_node_rebalance/priv/emqx_node_rebalance.schema new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_node_rebalance/rebar.config b/apps/emqx_node_rebalance/rebar.config new file mode 100644 index 000000000..2656fd554 --- /dev/null +++ b/apps/emqx_node_rebalance/rebar.config @@ -0,0 +1,2 @@ +{erl_opts, [debug_info]}. +{deps, []}. diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src b/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src new file mode 100644 index 000000000..761cd1d6f --- /dev/null +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src @@ -0,0 +1,19 @@ +{application, emqx_node_rebalance, + [{description, "EMQX Node Rebalance"}, + {vsn, "4.3.0"}, + {registered, [emqx_node_rebalance_sup, + emqx_node_rebalance, + emqx_node_rebalance_agent, + emqx_node_rebalance_evacuation]}, + {applications, + [kernel, + stdlib + ]}, + {mod, {emqx_node_rebalance_app,[]}}, + {env,[]}, + {modules, []}, + {maintainers, ["EMQX Team "]}, + {links, [{"Homepage", "https://emqx.io/"}, + {"Github", "https://github.com/emqx"} + ]} + ]}. diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl new file mode 100644 index 000000000..1edbbbe3b --- /dev/null +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl @@ -0,0 +1,414 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance). + +-include("emqx_node_rebalance.hrl"). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-export([start/1, + status/0, + status/1, + stop/0 + ]). + +-export([start_link/0]). + +-behavior(gen_statem). + +-export([init/1, + callback_mode/0, + handle_event/4, + code_change/4 + ]). + +-export([is_node_available/0, + available_nodes/1, + connection_count/0, + session_count/0, + disconnected_session_count/0]). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +-type start_opts() :: #{conn_evict_rate => pos_integer(), + sess_evict_rate => pos_integer(), + wait_health_check => pos_integer(), + wait_takeover => pos_integer(), + abs_conn_threshold => pos_integer(), + rel_conn_threshold => number(), + abs_sess_threshold => pos_integer(), + rel_sess_threshold => number(), + nodes => [node()] + }. +-type start_error() :: already_started | [{node(), term()}]. + + +-spec start(start_opts()) -> ok_or_error(start_error()). +start(StartOpts) -> + Opts = maps:merge(default_opts(), StartOpts), + gen_statem:call(?MODULE, {start, Opts}). + +-spec stop() -> ok_or_error(not_started). +stop() -> + gen_statem:call(?MODULE, stop). + +-spec status() -> disabled | {enabled, map()}. +status() -> + gen_statem:call(?MODULE, status). + +-spec status(pid()) -> disabled | {enabled, map()}. +status(Pid) -> + gen_statem:call(Pid, status). + +-spec start_link() -> startlink_ret(). +start_link() -> + gen_statem:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec available_nodes(list(node())) -> list(node()). +available_nodes(Nodes) when is_list(Nodes) -> + {Available, _} = rpc:multicall(Nodes, ?MODULE, is_node_available, []), + lists:filter(fun is_atom/1, Available). + +%%-------------------------------------------------------------------- +%% gen_statem callbacks +%%-------------------------------------------------------------------- + +callback_mode() -> handle_event_function. + +%% states: disabled, wait_health_check, evicting_conns, wait_takeover, evicting_sessions + +init([]) -> + ?tp(debug, emqx_node_rebalance_started, #{}), + {ok, disabled, #{}}. + +%% start +handle_event({call, From}, + {start, #{wait_health_check := WaitHealthCheck} = Opts}, + disabled, + #{} = Data) -> + case enable_rebalance(Data#{opts => Opts}) of + {ok, NewData} -> + ?LOG(warning, "Node rebalance enabled: ~p", [Opts]), + {next_state, + wait_health_check, + NewData, + [{state_timeout, seconds(WaitHealthCheck), evict_conns}, + {reply, From, ok}]}; + {error, Reason} -> + ?LOG(warning, "Node rebalance enabling failed: ~p", [Reason]), + {keep_state_and_data, + [{reply, From, {error, Reason}}]} + end; +handle_event({call, From}, {start, _Opts}, _State, #{}) -> + {keep_state_and_data, + [{reply, From, {error, already_started}}]}; + +%% stop +handle_event({call, From}, stop, disabled, #{}) -> + {keep_state_and_data, + [{reply, From, {error, not_started}}]}; +handle_event({call, From}, stop, _State, Data) -> + ok = disable_rebalance(Data), + ?LOG(warning, "Node rebalance stopped"), + {next_state, + disabled, + deinit(Data), + [{reply, From, ok}]}; + +%% status +handle_event({call, From}, status, disabled, #{}) -> + {keep_state_and_data, + [{reply, From, disabled}]}; +handle_event({call, From}, status, State, Data) -> + Stats = get_stats(State, Data), + {keep_state_and_data, + [{reply, From, {enabled, Stats#{state => State, + coordinator_node => node()}}}]}; + +%% conn eviction +handle_event(state_timeout, + evict_conns, + wait_health_check, + Data) -> + ?LOG(warning, "Node rebalance wait_health_check over"), + {next_state, + evicting_conns, + Data, + [{state_timeout, 0, evict_conns}]}; + +handle_event(state_timeout, + evict_conns, + evicting_conns, + #{opts := #{wait_takeover := WaitTakeover, + evict_interval := EvictInterval}} = Data) -> + case evict_conns(Data) of + ok -> + ?LOG(warning, "Node rebalance evict_conns over"), + {next_state, + wait_takeover, + Data, + [{state_timeout, seconds(WaitTakeover), evict_sessions}]}; + {continue, NewData} -> + {keep_state, + NewData, + [{state_timeout, EvictInterval, evict_conns}]} + end; + +handle_event(state_timeout, + evict_sessions, + wait_takeover, + Data) -> + ?LOG(warning, "Node rebalance wait_takeover over"), + {next_state, + evicting_sessions, + Data, + [{state_timeout, 0, evict_sessions}]}; + +handle_event(state_timeout, + evict_sessions, + evicting_sessions, + #{opts := #{evict_interval := EvictInterval}} = Data) -> + case evict_sessions(Data) of + ok -> + ?tp(debug, emqx_node_rebalance_evict_sess_over, #{}), + ?LOG(warning, "Node rebalance evict_sess over"), + ok = disable_rebalance(Data), + ?LOG(warning, "Rebalance finished successfully"), + {next_state, + disabled, + deinit(Data)}; + {continue, NewData} -> + {keep_state, + NewData, + [{state_timeout, EvictInterval, evict_sessions}]} + end; + +handle_event({call, From}, Msg, State, Data) -> + ?LOG(warning, "Unknown call: ~p, State: ~p, Data: ~p", [Msg, State, Data]), + {keep_state_and_data, + [{reply, From, ignored}]}; + +handle_event(info, Msg, State, Data) -> + ?LOG(warning, "Unknown Msg: ~p, State: ~p, Data: ~p", [Msg, State, Data]), + keep_state_and_data; + +handle_event(cast, Msg, State, Data) -> + ?LOG(warning, "Unknown cast Msg: ~p, State: ~p, Data: ~p", [Msg, State, Data]), + keep_state_and_data. + +code_change(_Vsn, State, Data, _Extra) -> + {ok, State, Data}. + +%%-------------------------------------------------------------------- +%% internal funs +%%-------------------------------------------------------------------- + +enable_rebalance(#{opts := Opts} = Data) -> + Nodes = maps:get(nodes, Opts), + ConnCounts = multicall(Nodes, {?MODULE, connection_count, []}), + SessCounts = multicall(Nodes, {?MODULE, session_count, []}), + {_, Counts} = lists:unzip(ConnCounts), + Avg = avg(Counts), + {DonorCounts, + RecipientCounts} = lists:partition( + fun({_Node, Count}) -> + Count >= Avg + end, + ConnCounts), + ?LOG(warning, "Enabling rebalance: ConnCounts=~p, DonorCounts=~p, RecipientCounts=~p", + [ConnCounts, DonorCounts, RecipientCounts]), + {DonorNodes, _} = lists:unzip(DonorCounts), + {RecipientNodes, _} = lists:unzip(RecipientCounts), + case need_rebalance(DonorNodes, RecipientNodes, ConnCounts, SessCounts, Opts) of + false -> {error, nothing_to_balance}; + true -> + _ = multicall(DonorNodes, {emqx_node_rebalance_agent, enable, [self()]}), + {ok, Data#{donors => DonorNodes, + recipients => RecipientNodes, + initial_conn_counts => maps:from_list(ConnCounts), + initial_sess_counts => maps:from_list(SessCounts)}} + end. + +disable_rebalance(#{donors := DonorNodes}) -> + _ = multicall(DonorNodes, {emqx_node_rebalance_agent, disable, [self()]}), + ok. + +evict_conns(#{donors := DonorNodes, recipients := RecipientNodes, opts := Opts} = Data) -> + DonorNodeCounts = multicall(DonorNodes, {?MODULE, connection_count, []}), + {_, DonorCounts} = lists:unzip(DonorNodeCounts), + RecipientNodeCounts = multicall(RecipientNodes, {?MODULE, connection_count, []}), + {_, RecipientCounts} = lists:unzip(RecipientNodeCounts), + + DonorAvg = avg(DonorCounts), + RecipientAvg = avg(RecipientCounts), + Thresholds = thresholds(conn, Opts), + NewData = Data#{donor_conn_avg => DonorAvg, + recipient_conn_avg => RecipientAvg, + donor_conn_counts => maps:from_list(DonorNodeCounts), + recipient_conn_counts => maps:from_list(RecipientNodeCounts)}, + case within_thresholds(DonorAvg, RecipientAvg, Thresholds) of + true -> ok; + false -> + ConnEvictRate = maps:get(conn_evict_rate, Opts), + NodesToEvict = nodes_to_evict(RecipientAvg, DonorNodeCounts), + ?LOG(warning, "Node rebalance, evict_conns, nodes=~p, counts=~p", + [NodesToEvict, ConnEvictRate]), + _ = multicall(NodesToEvict, {emqx_eviction_agent, evict_connections, [ConnEvictRate]}), + {continue, NewData} + end. + +evict_sessions(#{donors := DonorNodes, recipients := RecipientNodes, opts := Opts} = Data) -> + DonorNodeCounts = multicall(DonorNodes, {?MODULE, disconnected_session_count, []}), + {_, DonorCounts} = lists:unzip(DonorNodeCounts), + RecipientNodeCounts = multicall(RecipientNodes, {?MODULE, disconnected_session_count, []}), + {_, RecipientCounts} = lists:unzip(RecipientNodeCounts), + + DonorAvg = avg(DonorCounts), + RecipientAvg = avg(RecipientCounts), + Thresholds = thresholds(sess, Opts), + NewData = Data#{donor_sess_avg => DonorAvg, + recipient_sess_avg => RecipientAvg, + donor_sess_counts => maps:from_list(DonorNodeCounts), + recipient_sess_counts => maps:from_list(RecipientNodeCounts)}, + case within_thresholds(DonorAvg, RecipientAvg, Thresholds) of + true -> ok; + false -> + SessEvictRate = maps:get(sess_evict_rate, Opts), + NodesToEvict = nodes_to_evict(RecipientAvg, DonorNodeCounts), + ?LOG(warning, "Node rebalance, evict_sessions, nodes=~p, counts=~p", + [NodesToEvict, SessEvictRate]), + _ = multicall(NodesToEvict, + {emqx_eviction_agent, + evict_sessions, + [SessEvictRate, RecipientNodes, disconnected]}), + {continue, NewData} + end. + +need_rebalance([] = _DonorNodes, _RecipientNodes, _ConnCounts, _SessCounts, _Opts) -> false; +need_rebalance(_DonorNodes, [] = _RecipientNodes, _ConnCounts, _SessCounts, _Opts) -> false; +need_rebalance(DonorNodes, RecipientNodes, ConnCounts, SessCounts, Opts) -> + DonorConnAvg = avg_for_nodes(DonorNodes, ConnCounts), + RecipientConnAvg = avg_for_nodes(RecipientNodes, ConnCounts), + DonorSessAvg = avg_for_nodes(DonorNodes, SessCounts), + RecipientSessAvg = avg_for_nodes(RecipientNodes, SessCounts), + Result = (not within_thresholds(DonorConnAvg, RecipientConnAvg, thresholds(conn, Opts))) + orelse (not within_thresholds(DonorSessAvg, RecipientSessAvg, thresholds(sess, Opts))), + ?tp(debug, emqx_node_rebalance_need_rebalance, + #{donors => DonorNodes, + recipients => RecipientNodes, + conn_counts => ConnCounts, + sess_counts => SessCounts, + opts => Opts, + result => Result + }), + Result. + +avg_for_nodes(Nodes, Counts) -> + avg(maps:values(maps:with(Nodes, maps:from_list(Counts)))). + +within_thresholds(Value, GoalValue, {AbsThres, RelThres}) -> + (Value =< GoalValue + AbsThres) orelse (Value =< GoalValue * RelThres). + +thresholds(conn, #{abs_conn_threshold := Abs, rel_conn_threshold := Rel}) -> + {Abs, Rel}; +thresholds(sess, #{abs_sess_threshold := Abs, rel_sess_threshold := Rel}) -> + {Abs, Rel}. + +nodes_to_evict(Goal, NodeCounts) -> + {Nodes, _} = lists:unzip( + lists:filter( + fun({_Node, Count}) -> + Count > Goal + end, + NodeCounts)), + Nodes. + +get_stats(disabled, _Data) -> #{}; +get_stats(_State, Data) -> Data. + +avg(List) when length(List) >= 1 -> + lists:sum(List) / length(List). + +multicall(Nodes, {M, F, A}) -> + case rpc:multicall(Nodes, M, F, A) of + {Results, []} -> + case lists:partition(fun is_ok/1, lists:zip(Nodes, Results)) of + {OkResults, []} -> + [{Node, ok_result(Result)} || {Node, Result} <- OkResults]; + {_, BadResults} -> + error({bad_nodes, BadResults}) + end; + {_, [_BadNode | _] = BadNodes} -> + error({bad_nodes, BadNodes}) + end. + +is_ok({_Node, {ok, _}}) -> true; +is_ok({_Node, ok}) -> true; +is_ok(_) -> false. + +ok_result({ok, Result}) -> Result; +ok_result(ok) -> ok. + +connection_count() -> + {ok, emqx_eviction_agent:connection_count()}. + +session_count() -> + {ok, emqx_eviction_agent:session_count()}. + +disconnected_session_count() -> + {ok, emqx_eviction_agent:session_count(disconnected)}. + +default_opts() -> + #{ + conn_evict_rate => ?DEFAULT_CONN_EVICT_RATE, + abs_conn_threshold => ?DEFAULT_ABS_CONN_THRESHOLD, + rel_conn_threshold => ?DEFAULT_REL_CONN_THRESHOLD, + + sess_evict_rate => ?DEFAULT_SESS_EVICT_RATE, + abs_sess_threshold => ?DEFAULT_ABS_SESS_THRESHOLD, + rel_sess_threshold => ?DEFAULT_REL_SESS_THRESHOLD, + + wait_health_check => ?DEFAULT_WAIT_HEALTH_CHECK, + wait_takeover => ?DEFAULT_WAIT_TAKEOVER, + + evict_interval => ?EVICT_INTERVAL, + + nodes => all_nodes() + }. + + +deinit(Data) -> + Keys = [recipient_conn_avg, recipient_sess_avg, donor_conn_avg, donor_sess_avg, + recipient_conn_counts, recipient_sess_counts, donor_conn_counts, donor_sess_counts, + initial_conn_counts, initial_sess_counts, + opts], + maps:without(Keys, Data). + +is_node_available() -> + true = is_pid(whereis(emqx_node_rebalance_agent)), + disabled = emqx_eviction_agent:status(), + node(). + +all_nodes() -> + ekka_mnesia:cluster_nodes(all). + +seconds(Sec) -> + round(timer:seconds(Sec)). diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl new file mode 100644 index 000000000..ade043d6b --- /dev/null +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl @@ -0,0 +1,127 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_agent). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). + +-include_lib("stdlib/include/qlc.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-export([start_link/0, + enable/1, + disable/1, + status/0 + ]). + +-export([init/1, + handle_call/3, + handle_info/2, + handle_cast/2, + code_change/3 + ]). + +-define(ENABLE_KIND, emqx_node_rebalance). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +-type status() :: {enabled, pid()} | disabled. + +-spec start_link() -> startlink_ret(). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec enable(pid()) -> ok_or_error(already_enabled | eviction_agent_busy). +enable(CoordinatorPid) -> + gen_server:call(?MODULE, {enable, CoordinatorPid}). + +-spec disable(pid()) -> ok_or_error(already_disabled | invalid_coordinator). +disable(CoordinatorPid) -> + gen_server:call(?MODULE, {disable, CoordinatorPid}). + +-spec status() -> status(). +status() -> + gen_server:call(?MODULE, status). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([]) -> + {ok, #{}}. + +handle_call({enable, CoordinatorPid}, _From, St) -> + case St of + #{coordinator_pid := _Pid} -> + {reply, {error, already_enabled}, St}; + _ -> + true = link(CoordinatorPid), + EvictionAgentPid = whereis(emqx_eviction_agent), + true = link(EvictionAgentPid), + case emqx_eviction_agent:enable(?ENABLE_KIND, undefined) of + ok -> + {reply, ok, #{coordinator_pid => CoordinatorPid, + eviction_agent_pid => EvictionAgentPid}}; + {error, eviction_agent_busy} -> + true = unlink(EvictionAgentPid), + true = unlink(CoordinatorPid), + {reply, {error, eviction_agent_busy}, St} + end + end; + +handle_call({disable, CoordinatorPid}, _From, St) -> + case St of + #{coordinator_pid := CoordinatorPid, + eviction_agent_pid := EvictionAgentPid} -> + _ = emqx_eviction_agent:disable(?ENABLE_KIND), + true = unlink(EvictionAgentPid), + true = unlink(CoordinatorPid), + NewSt = maps:without( + [coordinator_pid, eviction_agent_pid], + St), + {reply, ok, NewSt}; + #{coordinator_pid := _CoordinatorPid} -> + {reply, {error, invalid_coordinator}, St}; + #{} -> + {reply, {error, already_disabled}, St} + end; + +handle_call(status, _From, St) -> + case St of + #{coordinator_pid := Pid} -> + {reply, {enabled, Pid}, St}; + _ -> + {reply, disabled, St} + end; + +handle_call(Msg, _From, St) -> + ?LOG(warning, "Unknown call: ~p, State: ~p", [Msg, St]), + {reply, ignored, St}. + +handle_info(Msg, St) -> + ?LOG(warning, "Unknown Msg: ~p, State: ~p", [Msg, St]), + {noreply, St}. + +handle_cast(Msg, St) -> + ?LOG(warning, "Unknown cast Msg: ~p, State: ~p", [Msg, St]), + {noreply, St}. + +code_change(_Vsn, State, _Extra) -> + {ok, State}. diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl new file mode 100644 index 000000000..6ba221cd8 --- /dev/null +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl @@ -0,0 +1,243 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_api). + +-import(minirest, [return/1]). + +-rest_api(#{name => load_rebalance_status, + method => 'GET', + path => "/load_rebalance/status", + func => status, + descr => "Get load rebalance status"}). + +-rest_api(#{name => load_rebalance_global_status, + method => 'GET', + path => "/load_rebalance/global_status", + func => global_status, + descr => "Get status of all rebalance/evacuation processes across the cluster"}). + +-rest_api(#{name => load_rebalance_availability_check, + method => 'GET', + path => "/load_rebalance/availability_check", + func => availability_check, + descr => "Node rebalance availability check"}). + +-rest_api(#{name => load_rebalance_start, + method => 'POST', + path => "/load_rebalance/:bin:node/start", + func => rebalance_start, + descr => "Start rebalancing with the node as coordinator"}). + +-rest_api(#{name => load_rebalance_stop, + method => 'POST', + path => "/load_rebalance/:bin:node/stop", + func => rebalance_stop, + descr => "Stop rebalancing coordinated by the node"}). + +-rest_api(#{name => load_rebalance_evacuation_start, + method => 'POST', + path => "/load_rebalance/:bin:node/evacuation/start", + func => rebalance_evacuation_start, + descr => "Start evacuation on a node "}). + +-rest_api(#{name => load_rebalance_evacuation_stop, + method => 'POST', + path => "/load_rebalance/:bin:node/evacuation/stop", + func => rebalance_evacuation_stop, + descr => "Stop evacuation on the node"}). + +-export([status/2, + availability_check/2, + global_status/2, + + rebalance_evacuation_start/2, + rebalance_evacuation_stop/2, + rebalance_start/2, + rebalance_stop/2 + ]). + +status(_Bindings, _Params) -> + case emqx_node_rebalance_status:local_status() of + disabled -> + {ok, #{status => disabled}}; + {rebalance, Stats} -> + {ok, format_status(rebalance, Stats)}; + {evacuation, Stats} -> + {ok, format_status(evacuation, Stats)} + end. + +global_status(_Bindings, _Params) -> + #{evacuations := Evacuations, + rebalances := Rebalances} = emqx_node_rebalance_status:global_status(), + {ok, #{evacuations => maps:from_list(Evacuations), + rebalances => maps:from_list(Rebalances)}}. + +availability_check(_Bindings, _Params) -> + case emqx_eviction_agent:status() of + disabled -> + {200, #{}}; + {enabled, _Stats} -> + {503, #{}} + end. + +rebalance_evacuation_start(#{node := NodeBin}, Params) -> + validated( + fun() -> + {Node, Opts} = validate_evacuation(NodeBin, params(Params)), + rpc(Node, emqx_node_rebalance_evacuation, start, [Opts]) + end). + +rebalance_evacuation_stop(#{node := NodeBin}, _Params) -> + validated( + fun() -> + Node = parse_node(NodeBin), + rpc(Node, emqx_node_rebalance_evacuation, stop, []) + end). + +rebalance_start(#{node := NodeBin}, Params) -> + validated( + fun() -> + {Node, Opts} = validate_rebalance(NodeBin, params(Params)), + rpc(Node, emqx_node_rebalance, start, [Opts]) + end). + +rebalance_stop(#{node := NodeBin}, _Params) -> + validated( + fun() -> + Node = parse_node(NodeBin), + rpc(Node, emqx_node_rebalance, stop, []) + end). + +rpc(Node, M, F, A) -> + case rpc:call(Node, M, F, A) of + ok -> return({ok, []}); + {error, Error} -> + return({error, 400, io_lib:format("~p", [Error])}); + {badrpc, _} -> + return({error, 400, io_lib:format("Error communicating with node ~p", [Node])}); + Unknown -> + return({error, 400, io_lib:format("Unrecognized rpc result from node ~p: ~p", + [Node, Unknown])}) + end. + +format_status(Process, Stats) -> + Stats#{process => Process, status => enabled}. + +validate_evacuation(Node, Params) -> + NodeToEvacuate = parse_node(Node), + OptList = lists:map( + fun validate_evacuation_param/1, + Params), + {NodeToEvacuate, maps:from_list(OptList)}. + +validate_rebalance(Node, Params) -> + CoordinatorNode = parse_node(Node), + OptList = lists:map( + fun validate_rebalance_param/1, + Params), + {CoordinatorNode, maps:from_list(OptList)}. + +validate_evacuation_param({<<"conn_evict_rate">>, Value}) -> + validate_pos_int(conn_evict_rate, Value); +validate_evacuation_param({<<"sess_evict_rate">>, Value}) -> + validate_pos_int(sess_evict_rate, Value); +validate_evacuation_param({<<"redirect_to">>, Value}) -> + validate_binary(server_reference, Value); +validate_evacuation_param({<<"wait_takeover">>, Value}) -> + validate_pos_int(wait_takeover, Value); +validate_evacuation_param({<<"migrate_to">>, Value}) -> + validate_nodes(migrate_to, Value); +validate_evacuation_param(Value) -> + validation_error(io_lib:format("Unknown evacuation param: ~p", [Value])). + +validate_rebalance_param({<<"wait_health_check">>, Value}) -> + validate_pos_int(wait_health_check, Value); +validate_rebalance_param({<<"conn_evict_rate">>, Value}) -> + validate_pos_int(conn_evict_rate, Value); +validate_rebalance_param({<<"sess_evict_rate">>, Value}) -> + validate_pos_int(sess_evict_rate, Value); +validate_rebalance_param({<<"abs_conn_threshold">>, Value}) -> + validate_pos_int(abs_conn_threshold, Value); +validate_rebalance_param({<<"rel_conn_threshold">>, Value}) -> + validate_fraction(rel_conn_threshold, Value); +validate_rebalance_param({<<"abs_sess_threshold">>, Value}) -> + validate_pos_int(abs_sess_threshold, Value); +validate_rebalance_param({<<"rel_sess_threshold">>, Value}) -> + validate_fraction(rel_sess_threshold, Value); +validate_rebalance_param({<<"wait_takeover">>, Value}) -> + validate_pos_int(wait_takeover, Value); +validate_rebalance_param({<<"nodes">>, Value}) -> + validate_nodes(nodes, Value); +validate_rebalance_param(Value) -> + validation_error(io_lib:format("Unknown rebalance param: ~p", [Value])). + +validate_binary(Name, Value) when is_binary(Value) -> + {Name, Value}; +validate_binary(Name, _Value) -> + validation_error("invalid string in " ++ atom_to_list(Name)). + +validate_pos_int(Name, Value) -> + case is_integer(Value) andalso Value > 0 of + true -> {Name, Value}; + false -> + validation_error("invalid " ++ atom_to_list(Name) ++ " value") + end. + +validate_fraction(Name, Value) -> + case is_number(Value) andalso Value > 1.0 of + true -> {Name, Value}; + false -> + validation_error("invalid " ++ atom_to_list(Name) ++ " value") + end. + +validate_nodes(Name, NodeList) when is_list(NodeList) -> + Nodes = lists:map( + fun parse_node/1, + NodeList), + case emqx_node_rebalance_evacuation:available_nodes(Nodes) of + [] -> + validation_error(io_lib:format("no available nodes list in ~p: ~p", [Name, Nodes])); + Nodes -> + {Name, Nodes}; + OtherNodes -> + validation_error( + io_lib:format("unavailable nodes in ~p: ~p", + [Name, Nodes -- OtherNodes])) + end; +validate_nodes(Name, Nodes) -> + validation_error(io_lib:format("invalid node list in ~p: ~p", [Name, Nodes])). + +validated(Fun) -> + try + Fun() + catch throw:{validation_error, Error} -> + return({error, 400, iolist_to_binary(Error)}) + end. + +validation_error(Error) -> + throw({validation_error, Error}). + +parse_node(Bin) when is_binary(Bin) -> + try + binary_to_existing_atom(Bin) + catch + error:badarg -> + validation_error("invalid node: " ++ [Bin]) + end. + +params([{}]) -> []; +params(Params) -> Params. diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl new file mode 100644 index 000000000..3cba331de --- /dev/null +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl @@ -0,0 +1,34 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_app). + +-behaviour(application). + +-emqx_plugin(?MODULE). + +-export([ start/2 + , stop/1 + ]). + +start(_Type, _Args) -> + Env = application:get_all_env(emqx_node_rebalance), + {ok, Sup} = emqx_node_rebalance_sup:start_link(Env), + ok = emqx_node_rebalance_cli:load(), + {ok, Sup}. + +stop(_State) -> + emqx_node_rebalance_cli:unload(). diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl new file mode 100644 index 000000000..10d0912c4 --- /dev/null +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl @@ -0,0 +1,265 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_cli). + +%% APIs +-export([ load/0 + , unload/0 + , cli/1 + ]). + +load() -> + emqx_ctl:register_command(rebalance, {?MODULE, cli}, []). + +unload() -> + emqx_ctl:unregister_command(rebalance). + +cli(["start" | StartArgs]) -> + case start_args(StartArgs) of + {evacuation, Opts} -> + case emqx_node_rebalance_evacuation:status() of + disabled -> + ok = emqx_node_rebalance_evacuation:start(Opts), + emqx_ctl:print("Rebalance(evacuation) started~n"), + true; + {enabled, _} -> + emqx_ctl:print("Rebalance is already enabled~n"), + false + end; + {rebalance, Opts} -> + case emqx_node_rebalance:start(Opts) of + ok -> + emqx_ctl:print("Rebalance started~n"), + true; + {error, Reason} -> + emqx_ctl:print("Rebalance start error: ~p~n", [Reason]), + false + end; + {error, Error} -> + emqx_ctl:print("Rebalance start error: ~s~n", [Error]), + false + end; +cli(["node-status", NodeStr]) -> + Node = list_to_atom(NodeStr), + node_status(emqx_node_rebalance_status:local_status(Node)); +cli(["node-status"]) -> + node_status(emqx_node_rebalance_status:local_status()); +cli(["status"]) -> + #{evacuations := Evacuations, + rebalances := Rebalances} = emqx_node_rebalance_status:global_status(), + lists:foreach( + fun({Node, Status}) -> + emqx_ctl:print("--------------------------------------------------------------------~n"), + emqx_ctl:print("Node ~p: evacuation~n~s", + [Node, emqx_node_rebalance_status:format_local_status(Status)]) + end, + Evacuations), + lists:foreach( + fun({Node, Status}) -> + emqx_ctl:print("--------------------------------------------------------------------~n"), + emqx_ctl:print("Node ~p: rebalance coordinator~n~s", + [Node, emqx_node_rebalance_status:format_coordinator_status(Status)]) + end, + Rebalances); +cli(["stop"]) -> + case emqx_node_rebalance_evacuation:status() of + {enabled, _} -> + ok = emqx_node_rebalance_evacuation:stop(), + emqx_ctl:print("Rebalance(evacuation) stopped~n"), + true; + disabled -> + case emqx_node_rebalance:status() of + {enabled, _} -> + ok = emqx_node_rebalance:stop(), + emqx_ctl:print("Rebalance stopped~n"), + true; + disabled -> + emqx_ctl:print("Rebalance is already disabled~n"), + false + end + end; +cli(_) -> + emqx_ctl:usage( + [{"rebalance start --evacuation \\\n" + " [--redirect-to \"Host1:Port1 Host2:Port2 ...\"] \\\n" + " [--conn-evict-rate CountPerSec] \\\n" + " [--migrate-to \"node1@host1 node2@host2 ...\"] \\\n" + " [--wait-takeover Secs] \\\n" + " [--sess-evict-rate CountPerSec]", + "Start current node evacuation with optional server redirect to the specified servers"}, + + {"rebalance start \\\n" + " [--nodes \"node1@host1 node2@host2\"] \\\n" + " [--wait-health-check Secs] \\\n" + " [--conn-evict-rate ConnPerSec] \\\n" + " [--abs-conn-threshold Count] \\\n" + " [--rel-conn-threshold Fraction] \\\n" + " [--conn-evict-rate ConnPerSec] \\\n" + " [--wait-takeover Secs] \\\n" + " [--sess-evict-rate CountPerSec] \\\n" + " [--abs-sess-threshold Count] \\\n" + " [--rel-sess-threshold Fraction]", + "Start current node evacuation with optional server redirect to the specified servers"}, + + {"rebalance node-status", + "Get current node rebalance status"}, + + {"rebalance node-status \"node1@host1\"", + "Get remote node rebalance status"}, + + {"rebalance status", + "Get statuses of all current rebalance/evacuation processes across the cluster"}, + + {"rebalance stop", + "Stop node rebalance"}]). + +node_status(NodeStatus) -> + case NodeStatus of + {Process, Status} when Process =:= evacuation orelse Process =:= rebalance -> + emqx_ctl:print("Rebalance type: ~p~n~s~n", + [Process, emqx_node_rebalance_status:format_local_status(Status)]); + disabled -> + emqx_ctl:print("Rebalance disabled~n"); + Other -> + emqx_ctl:print("Error detecting rebalance status: ~p~n", [Other]) + end. + +start_args(Args) -> + case collect_args(Args, #{}) of + {ok, #{"--evacuation" := true} = Collected} -> + case validate_evacuation(maps:to_list(Collected), #{}) of + {ok, Validated} -> + {evacuation, Validated}; + {error, _} = Error -> Error + end; + {ok, #{} = Collected} -> + case validate_rebalance(maps:to_list(Collected), #{}) of + {ok, Validated} -> + {rebalance, Validated}; + {error, _} = Error -> Error + end; + {error, _} = Error -> Error + end. + +collect_args([], Map) -> {ok, Map}; + +%% evacuation +collect_args(["--evacuation" | Args], Map) -> + collect_args(Args, Map#{"--evacuation" => true}); +collect_args(["--redirect-to", ServerReference | Args], Map) -> + collect_args(Args, Map#{"--redirect-to" => ServerReference}); +collect_args(["--migrate-to", MigrateTo | Args], Map) -> + collect_args(Args, Map#{"--migrate-to" => MigrateTo}); +%% rebalance +collect_args(["--nodes", Nodes | Args], Map) -> + collect_args(Args, Map#{"--nodes" => Nodes}); +collect_args(["--wait-health-check", WaitHealthCheck | Args], Map) -> + collect_args(Args, Map#{"--wait-health-check" => WaitHealthCheck}); +collect_args(["--abs-conn-threshold", AbsConnThres | Args], Map) -> + collect_args(Args, Map#{"--abs-conn-threshold" => AbsConnThres}); +collect_args(["--rel-conn-threshold", RelConnThres | Args], Map) -> + collect_args(Args, Map#{"--rel-conn-threshold" => RelConnThres}); +collect_args(["--abs-sess-threshold", AbsSessThres | Args], Map) -> + collect_args(Args, Map#{"--abs-sess-threshold" => AbsSessThres}); +collect_args(["--rel-sess-threshold", RelSessThres | Args], Map) -> + collect_args(Args, Map#{"--rel-sess-threshold" => RelSessThres}); +%% common +collect_args(["--conn-evict-rate", ConnEvictRate | Args], Map) -> + collect_args(Args, Map#{"--conn-evict-rate" => ConnEvictRate}); +collect_args(["--wait-takeover", WaitTakeover | Args], Map) -> + collect_args(Args, Map#{"--wait-takeover" => WaitTakeover}); +collect_args(["--sess-evict-rate", SessEvictRate | Args], Map) -> + collect_args(Args, Map#{"--sess-evict-rate" => SessEvictRate}); +%% fallback +collect_args(Args, _Map) -> + {error, io_lib:format("unknown arguments: ~p", [Args])}. + +validate_evacuation([], Map) -> + {ok, Map}; +validate_evacuation([{"--evacuation", _} | Rest], Map) -> + validate_evacuation(Rest, Map); +validate_evacuation([{"--redirect-to", ServerReference} | Rest], Map) -> + validate_evacuation(Rest, Map#{server_reference => list_to_binary(ServerReference)}); +validate_evacuation([{"--conn-evict-rate", _} | _] = Opts, Map) -> + validate_pos_int(conn_evict_rate, Opts, Map, fun validate_evacuation/2); +validate_evacuation([{"--sess-evict-rate", _} | _] = Opts, Map) -> + validate_pos_int(sess_evict_rate, Opts, Map, fun validate_evacuation/2); +validate_evacuation([{"--wait-takeover", _} | _] = Opts, Map) -> + validate_pos_int(wait_takeover, Opts, Map, fun validate_evacuation/2); +validate_evacuation([{"--migrate-to", MigrateTo} | Rest], Map) -> + Nodes = lists:map(fun list_to_atom/1, string:tokens(MigrateTo, ", ")), + case emqx_node_rebalance_evacuation:available_nodes(Nodes) of + [] -> + {error, "invalid --migrate-to, no nodes"}; + Nodes -> + validate_evacuation(Rest, Map#{migrate_to => Nodes}); + OtherNodes -> + {error, + io_lib:format("invalid --migrate-to, unavailable nodes: ~p", + [Nodes -- OtherNodes])} + end; +validate_evacuation(Rest, _Map) -> + {error, io_lib:format("unknown evacuation arguments: ~p", [Rest])}. + +validate_rebalance([], Map) -> + {ok, Map}; +validate_rebalance([{"--wait-health-check", _} | _] = Opts, Map) -> + validate_pos_int(wait_health_check, Opts, Map, fun validate_rebalance/2); +validate_rebalance([{"--conn-evict-rate", _} | _] = Opts, Map) -> + validate_pos_int(conn_evict_rate, Opts, Map, fun validate_rebalance/2); +validate_rebalance([{"--sess-evict-rate", _} | _] = Opts, Map) -> + validate_pos_int(sess_evict_rate, Opts, Map, fun validate_rebalance/2); +validate_rebalance([{"--abs-conn-threshold", _} | _] = Opts, Map) -> + validate_pos_int(abs_conn_threshold, Opts, Map, fun validate_rebalance/2); +validate_rebalance([{"--rel-conn-threshold", _} | _] = Opts, Map) -> + validate_fraction(rel_conn_threshold, Opts, Map, fun validate_rebalance/2); +validate_rebalance([{"--abs-sess-threshold", _} | _] = Opts, Map) -> + validate_pos_int(abs_sess_threshold, Opts, Map, fun validate_rebalance/2); +validate_rebalance([{"--rel-sess-threshold", _} | _] = Opts, Map) -> + validate_fraction(rel_sess_threshold, Opts, Map, fun validate_rebalance/2); +validate_rebalance([{"--wait-takeover", _} | _] = Opts, Map) -> + validate_pos_int(wait_takeover, Opts, Map, fun validate_rebalance/2); +validate_rebalance([{"--nodes", NodeStr} | Rest], Map) -> + Nodes = lists:map(fun list_to_atom/1, string:tokens(NodeStr, ", ")), + case emqx_node_rebalance:available_nodes(Nodes) of + [] -> + {error, "invalid --nodes, no nodes"}; + Nodes -> + validate_rebalance(Rest, Map#{nodes => Nodes}); + OtherNodes -> + {error, + io_lib:format("invalid --nodes, unavailable nodes: ~p", + [Nodes -- OtherNodes])} + end; +validate_rebalance(Rest, _Map) -> + {error, io_lib:format("unknown rebalance arguments: ~p", [Rest])}. + +validate_fraction(Name, [{OptionName, Value} | Rest], Map, Next) -> + case string:to_float(Value) of + {Num, ""} when Num > 1.0 -> + Next(Rest, Map#{Name => Num}); + _ -> + {error, "invalid " ++ OptionName ++ " value"} + end. + +validate_pos_int(Name, [{OptionName, Value} | Rest], Map, Next) -> + case string:to_integer(Value) of + {Int, ""} when Int > 0 -> + Next(Rest, Map#{Name => Int}); + _ -> + {error, "invalid " ++ OptionName ++ " value"} + end. diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl new file mode 100644 index 000000000..5dc99d736 --- /dev/null +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl @@ -0,0 +1,298 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_evacuation). + +-include("emqx_node_rebalance.hrl"). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-export([start/1, + status/0, + stop/0 + ]). + +-export([start_link/0]). + +-behavior(gen_statem). + +-export([init/1, + callback_mode/0, + handle_event/4, + code_change/4 + ]). + +-export([is_node_available/0, + available_nodes/1]). + +-ifdef(TEST). +-export([migrate_to/1]). +-endif. + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +-define(EVICT_INTERVAL_NO_NODES, 30000). + +-type migrate_to() :: [node()] | undefined. + +-type start_opts() :: #{server_reference => emqx_eviction_agent:server_reference(), + conn_evict_rate => pos_integer(), + sess_evict_rate => pos_integer(), + wait_takeover => pos_integer(), + migrate_to => migrate_to() + }. +-type start_error() :: already_started | eviction_agent_busy. +-type stats() :: #{ + initial_conns := non_neg_integer(), + initial_sessions := non_neg_integer(), + current_conns := non_neg_integer(), + current_sessions := non_neg_integer(), + conn_evict_rate := pos_integer(), + sess_evict_rate := pos_integer(), + server_reference := emqx_eviction_agent:server_reference(), + migrate_to := migrate_to() + }. +-type status() :: {started, stats()} | stopped. + +-spec start(start_opts()) -> ok_or_error(start_error()). +start(StartOpts) -> + Opts = maps:merge(default_opts(), StartOpts), + gen_statem:call(?MODULE, {start, Opts}). + +-spec stop() -> ok_or_error(not_started). +stop() -> + gen_statem:call(?MODULE, stop). + +-spec status() -> status(). +status() -> + gen_statem:call(?MODULE, status). + +-spec start_link() -> startlink_ret(). +start_link() -> + gen_statem:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec available_nodes(list(node())) -> list(node()). +available_nodes(Nodes) when is_list(Nodes) -> + {Available, _} = rpc:multicall(Nodes, ?MODULE, is_node_available, []), + lists:filter(fun is_atom/1, Available). + +%%-------------------------------------------------------------------- +%% gen_statem callbacks +%%-------------------------------------------------------------------- + +callback_mode() -> handle_event_function. + +%% states: disabled, evicting_conns, waiting_takeover, evicting_sessions, prohibiting + +init([]) -> + case emqx_node_rebalance_evacuation_persist:read(default_opts()) of + {ok, #{server_reference := ServerReference} = Opts} -> + ?LOG(warning, "Restoring evacuation state: ~p", [Opts]), + case emqx_eviction_agent:enable(?MODULE, ServerReference) of + ok -> + Data = init_data(#{}, Opts), + ok = warn_enabled(), + {ok, evicting_conns, Data, [{state_timeout, 0, evict_conns}]}; + {error, eviction_agent_busy} -> + emqx_node_rebalance_evacuation_persist:clear(), + {ok, disabled, #{}} + end; + none -> + {ok, disabled, #{}} + end. + +%% start +handle_event({call, From}, + {start, #{server_reference := ServerReference} = Opts}, + disabled, + #{} = Data) -> + case emqx_eviction_agent:enable(?MODULE, ServerReference) of + ok -> + NewData = init_data(Data, Opts), + ok = emqx_node_rebalance_evacuation_persist:save(Opts), + ?LOG(warning, "Node evacuation started"), + {next_state, + evicting_conns, + NewData, + [{state_timeout, 0, evict_conns}, + {reply, From, ok}]}; + {error, eviction_agent_busy} -> + {keep_state_and_data, + [{reply, From, {error, eviction_agent_busy}}]} + end; +handle_event({call, From}, {start, _Opts}, _State, #{}) -> + {keep_state_and_data, + [{reply, From, {error, already_started}}]}; + +%% stop +handle_event({call, From}, stop, disabled, #{}) -> + {keep_state_and_data, + [{reply, From, {error, not_started}}]}; +handle_event({call, From}, stop, _State, Data) -> + ok = emqx_node_rebalance_evacuation_persist:clear(), + _ = emqx_eviction_agent:disable(?MODULE), + ?LOG(warning, "Node evacuation stopped"), + {next_state, + disabled, + deinit(Data), + [{reply, From, ok}]}; + +%% status +handle_event({call, From}, status, disabled, #{}) -> + {keep_state_and_data, + [{reply, From, disabled}]}; +handle_event({call, From}, status, State, #{migrate_to := MigrateTo} = Data) -> + Stats = maps:with( + [initial_conns, current_conns, + initial_sessions, current_sessions, + server_reference, conn_evict_rate, sess_evict_rate], + Data), + {keep_state_and_data, + [{reply, From, {enabled, Stats#{state => State, migrate_to => migrate_to(MigrateTo)}}}]}; + +%% conn eviction +handle_event(state_timeout, + evict_conns, + evicting_conns, + #{conn_evict_rate := ConnEvictRate, + wait_takeover := WaitTakeover} = Data) -> + case emqx_eviction_agent:status() of + {enabled, #{connections := Conns}} when Conns > 0 -> + ok = emqx_eviction_agent:evict_connections(ConnEvictRate), + ?tp(debug, node_evacuation_evict_conn, #{conn_evict_rate => ConnEvictRate}), + ?LOG(warning, "Node evacuation evict_conns, count=~p, conn_evict_rate=~p", + [Conns, ConnEvictRate]), + NewData = Data#{current_conns => Conns}, + {keep_state, + NewData, + [{state_timeout, ?EVICT_INTERVAL, evict_conns}]}; + {enabled, #{connections := 0}} -> + NewData = Data#{current_conns => 0}, + ?LOG(warning, "Node evacuation evict_conns over"), + {next_state, + waiting_takeover, + NewData, + [{state_timeout, timer:seconds(WaitTakeover), evict_sessions}]} + end; + +handle_event(state_timeout, + evict_sessions, + waiting_takeover, + Data) -> + ?LOG(warning, "Node evacuation wait_takeover over"), + {next_state, + evicting_sessions, + Data, + [{state_timeout, 0, evict_sessions}]}; + +%% session eviction +handle_event(state_timeout, + evict_sessions, + evicting_sessions, + #{sess_evict_rate := SessEvictRate, + migrate_to := MigrateTo, + current_sessions := CurrSessCount} = Data) -> + case emqx_eviction_agent:status() of + {enabled, #{sessions := SessCount}} when SessCount > 0 -> + case migrate_to(MigrateTo) of + [] -> + ?LOG(warning, + "No nodes are available to evacuate sessions, session_count=~p", + [CurrSessCount]), + {keep_state_and_data, + [{state_timeout, ?EVICT_INTERVAL_NO_NODES, evict_sessions}]}; + Nodes -> + ok = emqx_eviction_agent:evict_sessions(SessEvictRate, Nodes), + ?LOG(warning, "Node evacuation evict_sessions, count=~p, sess_evict_rate=~p," + "target_nodes=~p", [SessCount, SessEvictRate, Nodes]), + NewData = Data#{current_sessions => SessCount}, + {keep_state, + NewData, + [{state_timeout, ?EVICT_INTERVAL, evict_sessions}]} + end; + {enabled, #{sessions := 0}} -> + ?tp(debug, node_evacuation_evict_sess_over, #{}), + ?LOG(warning, "Node evacuation evict_sessions over"), + NewData = Data#{current_sessions => 0}, + {next_state, + prohibiting, + NewData} + end; + +handle_event({call, From}, Msg, State, Data) -> + ?LOG(warning, "Unknown call: ~p, State: ~p, Data: ~p", [Msg, State, Data]), + {keep_state_and_data, + [{reply, From, ignored}]}; + +handle_event(info, Msg, State, Data) -> + ?LOG(warning, "Unknown Msg: ~p, State: ~p, Data: ~p", [Msg, State, Data]), + keep_state_and_data; + +handle_event(cast, Msg, State, Data) -> + ?LOG(warning, "Unknown cast Msg: ~p, State: ~p, Data: ~p", [Msg, State, Data]), + keep_state_and_data. + +code_change(_Vsn, State, Data, _Extra) -> + {ok, State, Data}. + +%%-------------------------------------------------------------------- +%% internal funs +%%-------------------------------------------------------------------- + +default_opts() -> + #{ + server_reference => undefined, + conn_evict_rate => ?DEFAULT_CONN_EVICT_RATE, + sess_evict_rate => ?DEFAULT_SESS_EVICT_RATE, + wait_takeover => ?DEFAULT_WAIT_TAKEOVER, + migrate_to => undefined + }. + +init_data(Data0, Opts) -> + Data1 = maps:merge(Data0, Opts), + {enabled, #{connections := ConnCount, sessions := SessCount}} = emqx_eviction_agent:status(), + Data1#{ + initial_conns => ConnCount, + current_conns => ConnCount, + initial_sessions => SessCount, + current_sessions => SessCount + }. + +deinit(Data) -> + Keys = [initial_conns, current_conns, initial_sessions, current_sessions] + ++ maps:keys(default_opts()), + maps:without(Keys, Data). + +warn_enabled() -> + Msg = "Node evacuation is enabled. The node will not receive connections.", + ?LOG(warning, Msg), + io:format(standard_error, "~s~n", [Msg]). + +migrate_to(undefined) -> + migrate_to(all_nodes()); +migrate_to(Nodes) when is_list(Nodes) -> + available_nodes(Nodes). + +is_node_available() -> + disabled = emqx_eviction_agent:status(), + node(). + +all_nodes() -> + ekka_mnesia:cluster_nodes(all) -- [node()]. diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl new file mode 100644 index 000000000..06d8800da --- /dev/null +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl @@ -0,0 +1,109 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_evacuation_persist). + +-export([save/1, + clear/0, + read/1]). + + +-ifdef(TEST). +-export([evacuation_filepath/0]). +-endif. + +-include("emqx_node_rebalance.hrl"). +-include_lib("emqx/include/types.hrl"). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +%% do not persist `migrate_to`: +%% * after restart there is nothing to migrate +%% * this value may be invalid after node was offline +-type start_opts() :: #{server_reference => emqx_eviction_agent:server_reference(), + conn_evict_rate => pos_integer(), + sess_evict_rate => pos_integer(), + wait_takeover => pos_integer() + }. + +-spec save(start_opts()) -> ok_or_error(term()). +save(#{server_reference := ServerReference, + conn_evict_rate := ConnEvictRate, + sess_evict_rate := SessEvictRate, + wait_takeover := WaitTakeover} = Data) + when (is_binary(ServerReference) orelse ServerReference =:= undefined) andalso + is_integer(ConnEvictRate) andalso ConnEvictRate > 0 andalso + is_integer(SessEvictRate) andalso SessEvictRate > 0 andalso + is_integer(WaitTakeover) andalso WaitTakeover >= 0 -> + Filepath = evacuation_filepath(), + case filelib:ensure_dir(Filepath) of + ok -> + JsonData = emqx_json:encode( + prepare_for_encode(maps:with(persist_keys(), Data)), + [pretty]), + file:write_file(Filepath, JsonData); + {error, _} = Error -> Error + end. + +-spec clear() -> ok. +clear() -> + file:delete(evacuation_filepath()). + +-spec read(start_opts()) -> {ok, start_opts()} | none. +read(DefaultOpts) -> + case file:read_file(evacuation_filepath()) of + {ok, Data} -> + case emqx_json:safe_decode(Data, [return_maps]) of + {ok, Map} when is_map(Map) -> + {ok, map_to_opts(DefaultOpts, Map)}; + _NotAMap -> + {ok, DefaultOpts} + end; + {error, _} -> + none + end. + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +persist_keys() -> + [server_reference, + conn_evict_rate, + sess_evict_rate, + wait_takeover]. + +prepare_for_encode(#{server_reference := undefined} = Data) -> + Data#{server_reference => null}; +prepare_for_encode(Data) -> Data. + +format_after_decode(#{server_reference := null} = Data) -> + Data#{server_reference => undefined}; +format_after_decode(Data) -> Data. + +map_to_opts(DefaultOpts, Map) -> + format_after_decode( + map_to_opts( + maps:to_list(DefaultOpts), Map, #{})). + +map_to_opts([], _Map, Opts) -> Opts; +map_to_opts([{Key, DefaultVal} | Rest], Map, Opts) -> + map_to_opts(Rest, Map, Opts#{Key => maps:get(atom_to_binary(Key), Map, DefaultVal)}). + +evacuation_filepath() -> + filename:join([emqx:get_env(data_dir), ?EVACUATION_FILENAME]). diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl new file mode 100644 index 000000000..4002d7c13 --- /dev/null +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl @@ -0,0 +1,225 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_status). + +-export([local_status/0, + local_status/1, + global_status/0, + format_local_status/1, + format_coordinator_status/1]). + +%% For RPC +-export([evacuation_status/0, + rebalance_status/0]). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +-spec local_status() -> disabled | {evacuation, map()} | {rebalance, map()}. +local_status() -> + case emqx_node_rebalance_evacuation:status() of + {enabled, Status} -> + {evacuation, evacuation(Status)}; + disabled -> + case emqx_node_rebalance_agent:status() of + {enabled, CoordinatorPid} -> + case emqx_node_rebalance:status(CoordinatorPid) of + {enabled, Status} -> + local_rebalance(Status, node()); + disabled -> + disabled + end; + disabled -> + disabled + end + end. + +-spec local_status(node()) -> disabled | {evacuation, map()} | {rebalance, map()}. +local_status(Node) -> + rpc:call(Node, ?MODULE, ?FUNCTION_NAME, []). + +-spec format_local_status(map()) -> iodata(). +format_local_status(Status) -> + format_status(Status, local_status_field_format_order()). + +-spec global_status() -> #{rebalances := [{node(), map()}], evacuations := [{node(), map()}]}. +global_status() -> + Nodes = ekka_mnesia:cluster_nodes(all), + {RebalanceResults, _} = rpc:multicall(Nodes, ?MODULE, rebalance_status, []), + Rebalances = [{Node, coordinator_rebalance(Status)} || {Node, {enabled, Status}} <- RebalanceResults], + {EvacuatioResults, _} = rpc:multicall(Nodes, ?MODULE, evacuation_status, []), + Evacuations = [{Node, evacuation(Status)} || {Node, {enabled, Status}} <- EvacuatioResults], + #{rebalances => Rebalances, evacuations => Evacuations}. + +-spec format_coordinator_status(map()) -> iodata(). +format_coordinator_status(Status) -> + format_status(Status, coordinator_status_field_format_order()). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +evacuation(Status) -> + #{ + state => maps:get(state, Status), + connection_eviction_rate => maps:get(conn_evict_rate, Status), + session_eviction_rate => maps:get(sess_evict_rate, Status), + connection_goal => 0, + session_goal => 0, + session_recipients => maps:get(migrate_to, Status), + stats => #{ + initial_connected => maps:get(initial_conns, Status), + current_connected => maps:get(current_conns, Status), + initial_sessions => maps:get(initial_sessions, Status), + current_sessions => maps:get(current_sessions, Status) + } + }. + +local_rebalance(#{donors := Donors} = Stats, Node) -> + case lists:member(Node, Donors) of + true -> {rebalance, donor_rebalance(Stats, Node)}; + false -> disabled + end. + +donor_rebalance(Status, Node) -> + Opts = maps:get(opts, Status), + InitialConnCounts = maps:get(initial_conn_counts, Status), + InitialSessCounts = maps:get(initial_sess_counts, Status), + + CurrentStats = #{ + initial_connected => maps:get(Node, InitialConnCounts), + initial_sessions => maps:get(Node, InitialSessCounts), + current_connected => emqx_eviction_agent:connection_count(), + current_sessions => emqx_eviction_agent:session_count(), + current_disconnected_sessions => emqx_eviction_agent:session_count( + disconnected) + }, + maps:from_list( + [ + {state, maps:get(state, Status)}, + {coordinator_node, maps:get(coordinator_node, Status)}, + {connection_eviction_rate, maps:get(conn_evict_rate, Opts)}, + {session_eviction_rate, maps:get(sess_evict_rate, Opts)}, + {recipients, maps:get(recipients, Status)}, + {stats, CurrentStats} + ] ++ + [{connection_goal, maps:get(recipient_conn_avg, Status)} + || maps:is_key(recipient_conn_avg, Status) + ] ++ + [{disconnected_session_goal, maps:get(recipient_sess_avg, Status)} + || maps:is_key(recipient_sess_avg, Status) + ]). + +coordinator_rebalance(Status) -> + Opts = maps:get(opts, Status), + maps:from_list( + [ + {state, maps:get(state, Status)}, + {coordinator_node, maps:get(coordinator_node, Status)}, + {connection_eviction_rate, maps:get(conn_evict_rate, Opts)}, + {session_eviction_rate, maps:get(sess_evict_rate, Opts)}, + {recipients, maps:get(recipients, Status)}, + {donors, maps:get(donors, Status)} + ] ++ + [{connection_goal, maps:get(recipient_conn_avg, Status)} + || maps:is_key(recipient_conn_avg, Status) + ] ++ + [{disconnected_session_goal, maps:get(recipient_sess_avg, Status)} + || maps:is_key(recipient_sess_avg, Status) + ] ++ + [{donor_conn_avg, maps:get(donor_conn_avg, Status)} + || maps:is_key(donor_conn_avg, Status) + ] ++ + [{donor_sess_avg, maps:get(donor_sess_avg, Status)} + || maps:is_key(donor_sess_avg, Status) + ]). + +local_status_field_format_order() -> + [state, + coordinator_node, + connection_eviction_rate, + session_eviction_rate, + connection_goal, + session_goal, + disconnected_session_goal, + session_recipients, + recipients, + stats]. + +coordinator_status_field_format_order() -> + [state, + coordinator_node, + donors, + recipients, + connection_eviction_rate, + session_eviction_rate, + connection_goal, + disconnected_session_goal, + donor_conn_avg, + donor_sess_avg]. + +format_status(Status, FieldOrder) -> + Fields = lists:flatmap( + fun(FieldName) -> + maps:to_list(maps:with([FieldName], Status)) + end, + FieldOrder), + lists:map( + fun format_local_status_field/1, + Fields). + +format_local_status_field({state, State}) -> + io_lib:format("Rebalance state: ~p~n", [State]); +format_local_status_field({coordinator_node, Node}) -> + io_lib:format("Coordinator node: ~p~n", [Node]); +format_local_status_field({connection_eviction_rate, ConnEvictRate}) -> + io_lib:format("Connection eviction rate: ~p connections/second~n", [ConnEvictRate]); +format_local_status_field({session_eviction_rate, SessEvictRate}) -> + io_lib:format("Session eviction rate: ~p sessions/second~n", [SessEvictRate]); +format_local_status_field({connection_goal, ConnGoal}) -> + io_lib:format("Connection goal: ~p~n", [ConnGoal]); +format_local_status_field({session_goal, SessGoal}) -> + io_lib:format("Session goal: ~p~n", [SessGoal]); +format_local_status_field({disconnected_session_goal, DisconnSessGoal}) -> + io_lib:format("Disconnected session goal: ~p~n", [DisconnSessGoal]); +format_local_status_field({session_recipients, SessionRecipients}) -> + io_lib:format("Session recipient nodes: ~p~n", [SessionRecipients]); +format_local_status_field({recipients, Recipients}) -> + io_lib:format("Recipient nodes: ~p~n", [Recipients]); +format_local_status_field({donors, Donors}) -> + io_lib:format("Donor nodes: ~p~n", [Donors]); +format_local_status_field({donor_conn_avg, DonorConnAvg}) -> + io_lib:format("Current average donor node connection count: ~p~n", [DonorConnAvg]); +format_local_status_field({donor_sess_avg, DonorSessAvg}) -> + io_lib:format("Current average donor node disconnected session count: ~p~n", [DonorSessAvg]); +format_local_status_field({stats, Stats}) -> + format_local_stats(Stats). + +format_local_stats(Stats) -> + ["Channel statistics:\n" | + lists:map( + fun({Name, Value}) -> + io_lib:format(" ~p: ~p~n", [Name, Value]) + end, + maps:to_list(Stats))]. + +evacuation_status() -> + {node(), emqx_node_rebalance_evacuation:status()}. + +rebalance_status() -> + {node(), emqx_node_rebalance:status()}. diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl new file mode 100644 index 000000000..e677eb22e --- /dev/null +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl @@ -0,0 +1,44 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_sup). + +-behaviour(supervisor). + +-export([start_link/1]). + +-export([init/1]). + +start_link(Env) -> + supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]). + +init([_Env]) -> + Childs = [child_spec(emqx_node_rebalance_evacuation, []), + child_spec(emqx_node_rebalance_agent, []), + child_spec(emqx_node_rebalance, [])], + {ok, { + {one_for_one, 10, 3600}, + Childs} + }. + +child_spec(Mod, Args) -> + #{id => Mod, + start => {Mod, start_link, Args}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [Mod] + }. diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl new file mode 100644 index 000000000..c5c27aec0 --- /dev/null +++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl @@ -0,0 +1,183 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-import(emqx_eviction_agent_test_helpers, + [emqtt_connect_many/1]). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]), + Config. + +end_per_suite(Config) -> + emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]), + Config. + +init_per_testcase(_Case, Config) -> + _ = emqx_node_rebalance:stop(), + Node = emqx_node_helpers:start_slave( + recipient1, + #{start_apps => [emqx, emqx_eviction_agent, emqx_node_rebalance]}), + [{recipient_node, Node} | Config]. + +end_per_testcase(_Case, Config) -> + _ = emqx_node_helpers:stop_slave(?config(recipient_node, Config)), + _ = emqx_node_rebalance:stop(). + +t_rebalance(Config) -> + process_flag(trap_exit, true), + RecipientNode = ?config(recipient_node, Config), + + Nodes = [node(), RecipientNode], + + _Conns = emqtt_connect_many(500), + + Opts = #{conn_evict_rate => 10, + sess_evict_rate => 10, + evict_interval => 10, + abs_conn_threshold => 50, + abs_sess_threshold => 50, + rel_conn_threshold => 1.0, + rel_sess_threshold => 1.0, + wait_health_check => 0.01, + wait_takeover => 0.01, + nodes => Nodes + }, + + ?check_trace( + ?wait_async_action( + emqx_node_rebalance:start(Opts), + #{?snk_kind := emqx_node_rebalance_evict_sess_over}, + 10000), + fun({ok, _}, Trace) -> + ?assertMatch( + [_ | _], + ?of_kind(emqx_node_rebalance_evict_sess_over, Trace)) + end), + + DonorConnCount = emqx_eviction_agent:connection_count(), + DonorSessCount = emqx_eviction_agent:session_count(), + DonorDSessCount = emqx_eviction_agent:session_count(disconnected), + + RecipientConnCount = rpc:call(RecipientNode, emqx_eviction_agent, connection_count, []), + RecipientSessCount = rpc:call(RecipientNode, emqx_eviction_agent, session_count, []), + RecipientDSessCount = rpc:call(RecipientNode, emqx_eviction_agent, session_count, [disconnected]), + + ct:pal("Donor: conn=~p, sess=~p, dsess=~p", + [DonorConnCount, DonorSessCount, DonorDSessCount]), + ct:pal("Recipient: conn=~p, sess=~p, dsess=~p", + [RecipientConnCount, RecipientSessCount, RecipientDSessCount]), + + ?assert(DonorConnCount - 50 =< RecipientConnCount), + ?assert(DonorDSessCount - 50 =< RecipientDSessCount). + +t_rebalance_node_crash(Config) -> + process_flag(trap_exit, true), + RecipientNode = ?config(recipient_node, Config), + + Nodes = [node(), RecipientNode], + + _Conns = emqtt_connect_many(50), + + Opts = #{conn_evict_rate => 10, + sess_evict_rate => 10, + evict_interval => 10, + abs_conn_threshold => 50, + abs_sess_threshold => 50, + rel_conn_threshold => 1.0, + rel_sess_threshold => 1.0, + wait_health_check => 0.01, + wait_takeover => 0.01, + nodes => Nodes + }, + + ok = emqx_node_rebalance:start(Opts), + + ?check_trace( + ?wait_async_action( + emqx_node_helpers:stop_slave(?config(recipient_node, Config)), + #{?snk_kind := emqx_node_rebalance_started}, + 1000), + fun(_Result, _Trace) -> ok end), + + ?assertEqual( + disabled, + emqx_node_rebalance:status()). + +t_no_need_to_rebalance(_Config) -> + process_flag(trap_exit, true), + + ?assertEqual( + {error, nothing_to_balance}, + emqx_node_rebalance:start(#{})), + + _Conns = emqtt_connect_many(50), + + ?assertEqual( + {error, nothing_to_balance}, + emqx_node_rebalance:start(#{})). + +t_unknown_mesages(Config) -> + process_flag(trap_exit, true), + RecipientNode = ?config(recipient_node, Config), + + Nodes = [node(), RecipientNode], + + _Conns = emqtt_connect_many(500), + + Opts = #{wait_health_check => 100, + abs_conn_threshold => 50, + nodes => Nodes + }, + + Pid = whereis(emqx_node_rebalance), + + Pid ! unknown, + ok = gen_server:cast(Pid, unknown), + ?assertEqual( + ignored, + gen_server:call(Pid, unknown)), + + ok = emqx_node_rebalance:start(Opts), + + Pid ! unknown, + ok = gen_server:cast(Pid, unknown), + ?assertEqual( + ignored, + gen_server:call(Pid, unknown)). + +t_available_nodes(Config) -> + rpc:call(?config(recipient_node, Config), + emqx_eviction_agent, + enable, + [test_rebalance, undefined]), + ?assertEqual( + [node()], + emqx_node_rebalance:available_nodes( + [node(), ?config(recipient_node, Config)])). diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl new file mode 100644 index 000000000..30edd51fe --- /dev/null +++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl @@ -0,0 +1,163 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_agent_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]), + Config. + +end_per_suite(Config) -> + emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]), + Config. + +init_per_testcase(_Case, Config) -> + _ = emqx_node_rebalance_evacuation:stop(), + Node = emqx_node_helpers:start_slave( + evacuate1, + #{start_apps => [emqx, emqx_eviction_agent, emqx_node_rebalance]}), + [{evacuate_node, Node} | Config]. + +end_per_testcase(_Case, Config) -> + _ = emqx_node_helpers:stop_slave(?config(evacuate_node, Config)), + _ = emqx_node_rebalance_evacuation:stop(). + +t_enable_disable(_Config) -> + ?assertEqual( + disabled, + emqx_node_rebalance_agent:status()), + + ?assertEqual( + ok, + emqx_node_rebalance_agent:enable(self())), + + ?assertEqual( + {error, already_enabled}, + emqx_node_rebalance_agent:enable(self())), + + ?assertEqual( + {enabled, self()}, + emqx_node_rebalance_agent:status()), + + ?assertEqual( + {error, invalid_coordinator}, + emqx_node_rebalance_agent:disable(spawn_link(fun() -> ok end))), + + ?assertEqual( + ok, + emqx_node_rebalance_agent:disable(self())), + + ?assertEqual( + {error, already_disabled}, + emqx_node_rebalance_agent:disable(self())), + + ?assertEqual( + disabled, + emqx_node_rebalance_agent:status()). + +t_enable_egent_busy(_Config) -> + ok = emqx_eviction_agent:enable(rebalance_test, undefined), + + ?assertEqual( + {error, eviction_agent_busy}, + emqx_node_rebalance_agent:enable(self())), + + ok = emqx_eviction_agent:disable(rebalance_test). + +% The following tests verify that emqx_node_rebalance_agent correctly links +% coordinator process with emqx_eviction_agent-s. + +t_rebalance_agent_coordinator_fail(Config) -> + process_flag(trap_exit, true), + + Node = ?config(evacuate_node, Config), + + + CoordinatorPid = spawn_link( + fun() -> + receive + done -> ok + end + end), + + ?assertEqual( + disabled, + rpc:call(Node, emqx_eviction_agent, status, [])), + + ?assertEqual( + ok, + rpc:call(Node, emqx_node_rebalance_agent, enable, [CoordinatorPid])), + + ?assertMatch( + {enabled, _}, + rpc:call(Node, emqx_eviction_agent, status, [])), + + EvictionAgentPid = rpc:call(Node, erlang, whereis, [emqx_eviction_agent]), + true = link(EvictionAgentPid), + + true = exit(CoordinatorPid, kill), + + receive + {'EXIT', EvictionAgentPid, _} -> true + after + 1000 -> ?assert(false, "emqx_eviction_agent did not exit") + end. + +t_rebalance_agent_fail(Config) -> + process_flag(trap_exit, true), + + Node = ?config(evacuate_node, Config), + + CoordinatorPid = spawn_link( + fun() -> + receive + done -> ok + end + end), + + ?assertEqual( + ok, + rpc:call(Node, emqx_node_rebalance_agent, enable, [CoordinatorPid])), + + EvictionAgentPid = rpc:call(Node, erlang, whereis, [emqx_eviction_agent]), + true = exit(EvictionAgentPid, kill), + + receive + {'EXIT', CoordinatorPid, _} -> true + after + 1000 -> ?assert(false, "emqx_eviction_agent did not exit") + end. + +t_unknown_messages(_Config) -> + Pid = whereis(emqx_node_rebalance_agent), + + ok = gen_server:cast(Pid, unknown), + + Pid ! unknown, + + ignored = gen_server:call(Pid, unknown). diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl new file mode 100644 index 000000000..a15b8e8ab --- /dev/null +++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl @@ -0,0 +1,321 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-import(emqx_mgmt_api_test_helpers, + [request_api/3, + request_api/5, + auth_header_/0, + api_path/1]). + +-import(emqx_eviction_agent_test_helpers, + [emqtt_connect_many/1]). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance, emqx_management]), + Config. + +end_per_suite(Config) -> + emqx_ct_helpers:stop_apps([emqx_management, emqx_node_rebalance, emqx_eviction_agent]), + Config. + +init_per_testcase(Case, Config) + when Case =:= t_start_evacuation_validation + orelse Case =:= t_start_rebalance_validation + orelse Case =:= t_start_stop_rebalance -> + _ = emqx_node_rebalance:stop(), + _ = emqx_node_rebalance_evacuation:stop(), + Node = emqx_node_helpers:start_slave( + recipient1, + #{start_apps => [emqx, emqx_eviction_agent, emqx_node_rebalance]}), + [{recipient_node, Node} | Config]; +init_per_testcase(_Case, Config) -> + _ = emqx_node_rebalance:stop(), + _ = emqx_node_rebalance_evacuation:stop(), + Config. + +end_per_testcase(Case, Config) + when Case =:= t_start_evacuation_validation + orelse Case =:= t_start_rebalance_validation + orelse Case =:= t_start_stop_rebalance -> + _ = emqx_node_helpers:stop_slave(?config(recipient_node, Config)), + _ = emqx_node_rebalance:stop(), + _ = emqx_node_rebalance_evacuation:stop(); +end_per_testcase(_Case, _Config) -> + _ = emqx_node_rebalance:stop(), + _ = emqx_node_rebalance_evacuation:stop(). + +t_start_evacuation_validation(Config) -> + BadOpts = [#{conn_evict_rate => <<"conn">>}, + #{sess_evict_rate => <<"sess">>}, + #{redirect_to => 123}, + #{wait_takeover => <<"wait">>}, + #{migrate_to => []}, + #{migrate_to => <<"migrate_to">>}, + #{migrate_to => [<<"bad_node">>]}, + #{migrate_to => [<<"bad_node">>, atom_to_binary(node())]}, + #{unknown => <<"Value">>} + ], + lists:foreach( + fun(Opts) -> + ?assertMatch( + {ok, #{}}, + api_post(["load_rebalance", atom_to_list(node()), "evacuation", "start"], + Opts)), + + ?assertMatch( + {ok, #{<<"status">> := <<"disabled">>}}, + api_get(["load_rebalance", "status"])) + + end, + BadOpts), + ?assertMatch( + {ok, #{}}, + api_post(["load_rebalance", "bad@node", "evacuation", "start"], + #{})), + + ?assertMatch( + {ok, #{<<"status">> := <<"disabled">>}}, + api_get(["load_rebalance", "status"])), + + ?assertMatch( + {ok, #{}}, + api_post(["load_rebalance", atom_to_list(node()), "evacuation", "start"], + #{conn_evict_rate => 10, + sess_evict_rate => 10, + wait_takeover => 10, + redirect_to => <<"srv">>, + migrate_to => [atom_to_binary(?config(recipient_node, Config))]})), + + ?assertMatch( + {ok, #{<<"status">> := <<"enabled">>}}, + api_get(["load_rebalance", "status"])). + + +t_start_rebalance_validation(Config) -> + BadOpts = [#{conn_evict_rate => <<"conn">>}, + #{sess_evict_rate => <<"sess">>}, + #{abs_conn_threshold => <<"act">>}, + #{rel_conn_threshold => <<"rct">>}, + #{abs_sess_threshold => <<"act">>}, + #{rel_sess_threshold => <<"rct">>}, + #{wait_takeover => <<"wait">>}, + #{wait_health_check => <<"wait">>}, + #{nodes => <<"nodes">>}, + #{nodes => []}, + #{nodes => [<<"bad_node">>]}, + #{nodes => [<<"bad_node">>, atom_to_binary(node())]}, + #{unknown => <<"Value">>} + ], + lists:foreach( + fun(Opts) -> + ?assertMatch( + {ok, #{}}, + api_post(["load_rebalance", atom_to_list(node()), "start"], + Opts)), + + ?assertMatch( + {ok, #{<<"status">> := <<"disabled">>}}, + api_get(["load_rebalance", "status"])) + + end, + BadOpts), + ?assertMatch( + {ok, #{}}, + api_post(["load_rebalance", "bad@node", "start"], + #{})), + + ?assertMatch( + {ok, #{<<"status">> := <<"disabled">>}}, + api_get(["load_rebalance", "status"])), + + _Conns = emqtt_connect_many(50), + + ?assertMatch( + {ok, #{}}, + api_post(["load_rebalance", atom_to_list(node()), "start"], + #{conn_evict_rate => 10, + sess_evict_rate => 10, + wait_takeover => 10, + wait_health_check => 10, + abs_conn_threshold => 10, + rel_conn_threshold => 1.001, + abs_sess_threshold => 10, + rel_sess_threshold => 1.001, + nodes => [atom_to_binary(?config(recipient_node, Config)), + atom_to_binary(node())]})), + + ?assertMatch( + {ok, #{<<"status">> := <<"enabled">>}}, + api_get(["load_rebalance", "status"])). + +t_start_stop_evacuation(_Config) -> + + ?assertMatch( + {ok, #{<<"status">> := <<"disabled">>}}, + api_get(["load_rebalance", "status"])), + + ?assertMatch( + {ok, #{}}, + api_post(["load_rebalance", atom_to_list(node()), "evacuation", "start"], + #{conn_evict_rate => 10, + sess_evict_rate => 20})), + + ?assertMatch( + {ok, #{<<"state">> := _, + <<"process">> := <<"evacuation">>, + <<"connection_eviction_rate">> := 10, + <<"session_eviction_rate">> := 20, + <<"connection_goal">> := 0, + <<"session_goal">> := 0, + <<"stats">> := #{ + <<"initial_connected">> := _, + <<"current_connected">> := _, + <<"initial_sessions">> := _, + <<"current_sessions">> := _ + }}}, + api_get(["load_rebalance", "status"])), + + ?assertMatch( + {ok, #{<<"rebalances">> := #{}, + <<"evacuations">> := + #{<<"test@127.0.0.1">> := #{<<"state">> := _, + <<"connection_eviction_rate">> := 10, + <<"session_eviction_rate">> := 20, + <<"connection_goal">> := 0, + <<"session_goal">> := 0, + <<"stats">> := #{ + <<"initial_connected">> := _, + <<"current_connected">> := _, + <<"initial_sessions">> := _, + <<"current_sessions">> := _ + } + } + }}}, + api_get(["load_rebalance", "global_status"])), + + ?assertMatch( + {ok, #{}}, + api_post(["load_rebalance", atom_to_list(node()), "evacuation", "stop"], + #{})), + + ?assertMatch( + {ok, #{<<"status">> := <<"disabled">>}}, + api_get(["load_rebalance", "status"])), + + ?assertMatch( + {ok, #{<<"evacuations">> := #{}, <<"rebalances">> := #{}}}, + api_get(["load_rebalance", "global_status"])). + +t_start_stop_rebalance(Config) -> + + ?assertMatch( + {ok, #{<<"status">> := <<"disabled">>}}, + api_get(["load_rebalance", "status"])), + + _Conns = emqtt_connect_many(100), + + ?assertMatch( + {ok, #{}}, + api_post(["load_rebalance", atom_to_list(node()), "start"], + #{conn_evict_rate => 10, + sess_evict_rate => 20, + abs_conn_threshold => 10})), + + ?assertMatch( + {ok, #{<<"state">> := _, + <<"process">> := <<"rebalance">>, + <<"coordinator_node">> := _, + <<"connection_eviction_rate">> := 10, + <<"session_eviction_rate">> := 20, + <<"stats">> := #{ + <<"initial_connected">> := _, + <<"current_connected">> := _, + <<"initial_sessions">> := _, + <<"current_sessions">> := _ + }}}, + api_get(["load_rebalance", "status"])), + + DonorNode = atom_to_binary(node()), + RecipientNode = atom_to_binary(?config(recipient_node, Config)), + + ?assertMatch( + {ok, #{<<"evacuations">> := #{}, + <<"rebalances">> := + #{<<"test@127.0.0.1">> := #{<<"state">> := _, + <<"coordinator_node">> := _, + <<"connection_eviction_rate">> := 10, + <<"session_eviction_rate">> := 20, + <<"donors">> := [DonorNode], + <<"recipients">> := [RecipientNode] + } + }}}, + api_get(["load_rebalance", "global_status"])), + + ?assertMatch( + {ok, #{}}, + api_post(["load_rebalance", atom_to_list(node()), "stop"], + #{})), + + ?assertMatch( + {ok, #{<<"status">> := <<"disabled">>}}, + api_get(["load_rebalance", "status"])), + + ?assertMatch( + {ok, #{<<"evacuations">> := #{}, <<"rebalances">> := #{}}}, + api_get(["load_rebalance", "global_status"])). + +t_availability_check(_Config) -> + + ?assertMatch( + {ok, #{}}, + api_get(["load_rebalance", "availability_check"])), + + ok = emqx_node_rebalance_evacuation:start(#{}), + + ?assertMatch( + {error, {_, 503, _}}, + api_get(["load_rebalance", "availability_check"])), + + ok = emqx_node_rebalance_evacuation:stop(), + + ?assertMatch( + {ok, #{}}, + api_get(["load_rebalance", "availability_check"])). + +api_get(Path) -> + case request_api(get, api_path(Path), auth_header_()) of + {ok, ResponseBody} -> + {ok, jiffy:decode(list_to_binary(ResponseBody), [return_maps])}; + {error, _} = Error -> Error + end. + +api_post(Path, Data) -> + case request_api(post, api_path(Path), [], auth_header_(), Data) of + {ok, ResponseBody} -> + {ok, jiffy:decode(list_to_binary(ResponseBody), [return_maps])}; + {error, _} = Error -> Error + end. diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl new file mode 100644 index 000000000..943746298 --- /dev/null +++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl @@ -0,0 +1,199 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_cli_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-import(emqx_eviction_agent_test_helpers, + [emqtt_connect_many/1]). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]), + Config. + +end_per_suite(Config) -> + emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]), + Config. + + +init_per_testcase(t_rebalance, Config) -> + _ = emqx_node_rebalance_evacuation:stop(), + Node = emqx_node_helpers:start_slave( + evacuate1, + #{start_apps => [emqx, emqx_eviction_agent, emqx_node_rebalance]}), + [{evacuate_node, Node} | Config]; +init_per_testcase(_Case, Config) -> + _ = emqx_node_rebalance_evacuation:stop(), + _ = emqx_node_rebalance:stop(), + Config. + +end_per_testcase(t_rebalance, Config) -> + _ = emqx_node_rebalance_evacuation:stop(), + _ = emqx_node_rebalance:stop(), + _ = emqx_node_helpers:stop_slave(?config(evacuate_node, Config)); +end_per_testcase(_Case, _Config) -> + _ = emqx_node_rebalance_evacuation:stop(), + _ = emqx_node_rebalance:stop(). + +t_evacuation(_Config) -> + %% usage + ok = emqx_node_rebalance_cli:cli(["foobar"]), + + %% status + ok = emqx_node_rebalance_cli:cli(["status"]), + ok = emqx_node_rebalance_cli:cli(["node-status"]), + ok = emqx_node_rebalance_cli:cli(["node-status", atom_to_list(node())]), + + %% start with invalid args + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--evacuation", "--foo-bar"])), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--evacuation", "--conn-evict-rate", "foobar"])), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--evacuation", "--sess-evict-rate", "foobar"])), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--evacuation", "--wait-takeover", "foobar"])), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--evacuation", + "--migrate-to", "nonexistent@node"])), + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--evacuation", + "--migrate-to", ""])), + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--evacuation", + "--unknown-arg"])), + ?assert( + emqx_node_rebalance_cli:cli(["start", "--evacuation", + "--conn-evict-rate", "10", + "--sess-evict-rate", "10", + "--wait-takeover", "10", + "--migrate-to", atom_to_list(node()), + "--redirect-to", "srv"])), + + %% status + ok = emqx_node_rebalance_cli:cli(["status"]), + ok = emqx_node_rebalance_cli:cli(["node-status"]), + ok = emqx_node_rebalance_cli:cli(["node-status", atom_to_list(node())]), + + ?assertMatch( + {enabled, #{}}, + emqx_node_rebalance_evacuation:status()), + + %% already enabled + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--evacuation", + "--conn-evict-rate", "10", + "--redirect-to", "srv"])), + + %% stop + true = emqx_node_rebalance_cli:cli(["stop"]), + + false = emqx_node_rebalance_cli:cli(["stop"]), + + ?assertEqual( + disabled, + emqx_node_rebalance_evacuation:status()). + +t_rebalance(Config) -> + %% start with invalid args + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--foo-bar"])), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--conn-evict-rate", "foobar"])), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--abs-conn-threshold", "foobar"])), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--rel-conn-threshold", "foobar"])), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--sess-evict-rate", "foobar"])), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--abs-sess-threshold", "foobar"])), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--rel-sess-threshold", "foobar"])), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--wait-takeover", "foobar"])), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", "--wait-health-check", "foobar"])), + + ?assertNot( + emqx_node_rebalance_cli:cli(["start", + "--nodes", "nonexistent@node"])), + ?assertNot( + emqx_node_rebalance_cli:cli(["start", + "--nodes", ""])), + ?assertNot( + emqx_node_rebalance_cli:cli(["start", + "--nodes", atom_to_list(?config(evacuate_node, Config))])), + ?assertNot( + emqx_node_rebalance_cli:cli(["start", + "--unknown-arg"])), + + _ = emqtt_connect_many(20), + + ?assert( + emqx_node_rebalance_cli:cli(["start", + "--conn-evict-rate", "10", + "--abs-conn-threshold", "10", + "--rel-conn-threshold", "1.1", + "--sess-evict-rate", "10", + "--abs-sess-threshold", "10", + "--rel-sess-threshold", "1.1", + "--wait-takeover", "10", + "--nodes", atom_to_list(node()) ++ "," + ++ atom_to_list(?config(evacuate_node, Config)) + ])), + + %% status + ok = emqx_node_rebalance_cli:cli(["status"]), + ok = emqx_node_rebalance_cli:cli(["node-status"]), + ok = emqx_node_rebalance_cli:cli(["node-status", atom_to_list(node())]), + + ?assertMatch( + {enabled, #{}}, + emqx_node_rebalance:status()), + + %% already enabled + ?assertNot( + emqx_node_rebalance_cli:cli(["start"])), + + %% stop + true = emqx_node_rebalance_cli:cli(["stop"]), + + false = emqx_node_rebalance_cli:cli(["stop"]), + + ?assertEqual( + disabled, + emqx_node_rebalance:status()). diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl new file mode 100644 index 000000000..23f35f61e --- /dev/null +++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl @@ -0,0 +1,194 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_evacuation_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-import(emqx_eviction_agent_test_helpers, + [emqtt_connect/0, emqtt_connect/2, emqtt_try_connect/0]). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]), + Config. + +end_per_suite(Config) -> + emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]), + Config. + +init_per_testcase(_Case, Config) -> + _ = emqx_node_rebalance_evacuation:stop(), + Node = emqx_node_helpers:start_slave( + evacuate1, + #{start_apps => [emqx, emqx_eviction_agent]}), + [{evacuate_node, Node} | Config]. + +end_per_testcase(_Case, Config) -> + _ = emqx_node_helpers:stop_slave(?config(evacuate_node, Config)), + _ = emqx_node_rebalance_evacuation:stop(). + +t_agent_busy(Config) -> + + ok = emqx_eviction_agent:enable(other_rebalance, undefined), + + ?assertEqual( + {error, eviction_agent_busy}, + emqx_node_rebalance_evacuation:start(opts(Config))), + + emqx_eviction_agent:disable(other_rebalance). + + +t_already_started(Config) -> + ok = emqx_node_rebalance_evacuation:start(opts(Config)), + + ?assertEqual( + {error, already_started}, + emqx_node_rebalance_evacuation:start(opts(Config))), + + ok = emqx_node_rebalance_evacuation:stop(). + +t_not_started(_Config) -> + ?assertEqual( + {error, not_started}, + emqx_node_rebalance_evacuation:stop()). + +t_start(Config) -> + process_flag(trap_exit, true), + + ok = emqx_node_rebalance_evacuation:start(opts(Config)), + ?assertMatch( + {error, {use_another_server, #{}}}, + emqtt_try_connect()), + ok = emqx_node_rebalance_evacuation:stop(). + +t_persistence(Config) -> + process_flag(trap_exit, true), + + ok = emqx_node_rebalance_evacuation:start(opts(Config)), + + ?assertMatch( + {error, {use_another_server, #{}}}, + emqtt_try_connect()), + + ok = supervisor:terminate_child(emqx_node_rebalance_sup, emqx_node_rebalance_evacuation), + {ok, _} = supervisor:restart_child(emqx_node_rebalance_sup, emqx_node_rebalance_evacuation), + + ?assertMatch( + {error, {use_another_server, #{}}}, + emqtt_try_connect()), + ?assertMatch( + {enabled, #{conn_evict_rate := 10}}, + emqx_node_rebalance_evacuation:status()), + + ok = emqx_node_rebalance_evacuation:stop(). + +t_conn_evicted(Config) -> + process_flag(trap_exit, true), + + {ok, C} = emqtt_connect(), + + ?check_trace( + ?wait_async_action( + emqx_node_rebalance_evacuation:start(opts(Config)), + #{?snk_kind := node_evacuation_evict_conn}, + 1000), + fun(_Result, _Trace) -> ok end), + + ct:sleep(100), + + ?assertMatch( + {error, {use_another_server, #{}}}, + emqtt_try_connect()), + + ?assertNot( + is_process_alive(C)). + +t_migrate_to(Config) -> + ?assertEqual( + [?config(evacuate_node, Config)], + emqx_node_rebalance_evacuation:migrate_to(undefined)), + + ?assertEqual( + [], + emqx_node_rebalance_evacuation:migrate_to(['unknown@node'])), + + rpc:call(?config(evacuate_node, Config), + emqx_eviction_agent, + enable, + [test_rebalance, undefined]), + + ?assertEqual( + [], + emqx_node_rebalance_evacuation:migrate_to(undefined)). + +t_session_evicted(Config) -> + process_flag(trap_exit, true), + + {ok, C} = emqtt_connect(<<"client_with_sess">>, false), + + ?check_trace( + ?wait_async_action( + emqx_node_rebalance_evacuation:start(opts(Config)), + #{?snk_kind := node_evacuation_evict_sess_over}, + 5000), + fun(_Result, Trace) -> + ?assertMatch( + [_ | _], + ?of_kind(node_evacuation_evict_sess_over, Trace)) + end), + + receive + {'EXIT', C, {disconnected, ?RC_USE_ANOTHER_SERVER, _}} -> ok + after 1000 -> + ?assert(false, "Connection not evicted") + end, + + [ChannelPid] = emqx_cm_registry:lookup_channels(<<"client_with_sess">>), + + ?assertEqual( + ?config(evacuate_node, Config), + node(ChannelPid)). + +t_unknown_messages(Config) -> + ok = emqx_node_rebalance_evacuation:start(opts(Config)), + + whereis(emqx_node_rebalance_evacuation) ! unknown, + + gen_server:cast(emqx_node_rebalance_evacuation, unknown), + + ?assertEqual( + ignored, + gen_server:call(emqx_node_rebalance_evacuation, unknown)), + + ok = emqx_node_rebalance_evacuation:stop(). + +opts(Config) -> + #{ + server_reference => <<"srv">>, + conn_evict_rate => 10, + sess_evict_rate => 10, + wait_takeover => 1, + migrate_to => [?config(evacuate_node, Config)] + }. diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl new file mode 100644 index 000000000..ba1a12775 --- /dev/null +++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl @@ -0,0 +1,107 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_evacuation_persist_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]), + Config. + +end_per_suite(Config) -> + emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]), + Config. + +init_per_testcase(_Case, Config) -> + _ = emqx_node_rebalance_evacuation_persist:clear(), + Config. + +end_per_testcase(_Case, _Config) -> + _ = emqx_node_rebalance_evacuation_persist:clear(). + +t_save_read(_Config) -> + DefaultOpts = #{server_reference => <<"default_ref">>, + conn_evict_rate => 2001, + sess_evict_rate => 2002, + wait_takeover => 2003 + }, + + Opts0 = #{server_reference => <<"ref">>, + conn_evict_rate => 1001, + sess_evict_rate => 1002, + wait_takeover => 1003 + }, + ok = emqx_node_rebalance_evacuation_persist:save(Opts0), + + {ok, ReadOpts0} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts), + ?assertEqual(Opts0, ReadOpts0), + + Opts1 = Opts0#{server_reference => undefined}, + ok = emqx_node_rebalance_evacuation_persist:save(Opts1), + + {ok, ReadOpts1} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts), + ?assertEqual(Opts1, ReadOpts1). + +t_read_default(_Config) -> + ok = write_evacuation_file(<<"{}">>), + + DefaultOpts = #{server_reference => <<"ref">>, + conn_evict_rate => 1001, + sess_evict_rate => 1002, + wait_takeover => 1003 + }, + + {ok, ReadOpts} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts), + ?assertEqual(DefaultOpts, ReadOpts). + +t_read_bad_data(_Config) -> + ok = write_evacuation_file(<<"{bad json">>), + + DefaultOpts = #{server_reference => <<"ref">>, + conn_evict_rate => 1001, + sess_evict_rate => 1002, + wait_takeover => 1003 + }, + + {ok, ReadOpts} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts), + ?assertEqual(DefaultOpts, ReadOpts). + +t_clear(_Config) -> + ok = write_evacuation_file(<<"{}">>), + + ?assertMatch( + {ok, _}, + emqx_node_rebalance_evacuation_persist:read(#{})), + + ok = emqx_node_rebalance_evacuation_persist:clear(), + + ?assertEqual( + none, + emqx_node_rebalance_evacuation_persist:read(#{})). + +write_evacuation_file(Json) -> + ok = filelib:ensure_dir(emqx_node_rebalance_evacuation_persist:evacuation_filepath()), + ok = file:write_file( + emqx_node_rebalance_evacuation_persist:evacuation_filepath(), + Json). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 259efa08d..90620e2df 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.13"}, % strict semver, bump manually! + {vsn, "4.3.14"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index ba39c4ec9..1c5fb0e7b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + [{"4.3.13",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.11", [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, @@ -165,7 +166,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + [{"4.3.13",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.11", [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, diff --git a/data/loaded_plugins.tmpl b/data/loaded_plugins.tmpl index d0dac7fe1..62b3205cd 100644 --- a/data/loaded_plugins.tmpl +++ b/data/loaded_plugins.tmpl @@ -6,3 +6,5 @@ {emqx_telemetry, {{enable_plugin_emqx_telemetry}}}. {emqx_rule_engine, {{enable_plugin_emqx_rule_engine}}}. {emqx_bridge_mqtt, {{enable_plugin_emqx_bridge_mqtt}}}. +{emqx_eviction_agent, true}. +{emqx_node_rebalance, true}. diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 1281145f0..3c9a4e3fa 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.3.14"}, % strict semver, bump manually! + {vsn, "4.3.15"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, diff --git a/rebar.config.erl b/rebar.config.erl index 78e1ef94e..04ffaefbd 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -51,6 +51,8 @@ overrides() -> [ {add, [ {extra_src_dirs, [{"etc", [{recursive,true}]}]} , {erl_opts, [{compile_info, [{emqx_vsn, get_vsn()}]}]} ]} + + , {add, relx, [{erl_opts, [{d, 'RLX_LOG', rlx_log}]}]} , {add, snabbkaffe, [{erl_opts, common_compile_opts()}]} ] ++ community_plugin_overrides(). @@ -295,6 +297,8 @@ relx_plugin_apps(ReleaseType) -> , emqx_recon , emqx_rule_engine , emqx_sasl + , emqx_eviction_agent + , emqx_node_rebalance ] ++ [emqx_telemetry || not is_enterprise()] ++ relx_plugin_apps_per_rel(ReleaseType) diff --git a/src/emqx.app.src b/src/emqx.app.src index 14a6f4fa4..14c63c75e 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.3.19"}, % strict semver, bump manually! + {vsn, "4.3.20"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 70d18ff48..b2be51df5 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,12 +1,19 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [ + [{"4.3.19", + [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}]}, {"4.3.18", - [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + [{load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.3.17", - [{load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]}, + [{load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {update,emqx_broker_sup,supervisor}, @@ -14,7 +21,8 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, {"4.3.16", - [{load_module,emqx_session,brutal_purge,soft_purge,[]}, + [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {update,emqx_broker_sup,supervisor}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, @@ -30,7 +38,8 @@ {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}]}, {"4.3.15", - [{add_module,emqx_calendar}, + [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, {apply,{emqx_exclusive_subscription,on_add_module,[]}}, @@ -55,7 +64,8 @@ {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{add_module,emqx_calendar}, + [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, {apply,{emqx_exclusive_subscription,on_add_module,[]}}, @@ -682,12 +692,19 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [ + [{"4.3.19", + [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}]}, {"4.3.18", - [{load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.3.17", - [{load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]}, + [{load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {update,emqx_broker_sup,supervisor}, @@ -695,7 +712,8 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, {"4.3.16", - [{load_module,emqx_session,brutal_purge,soft_purge,[]}, + [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {update,emqx_broker_sup,supervisor}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, @@ -711,7 +729,8 @@ {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}]}, {"4.3.15", - [{delete_module,emqx_calendar}, + [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, @@ -735,7 +754,8 @@ {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{delete_module,emqx_calendar}, + [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index bf292f10b..852c171fe 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -22,6 +22,8 @@ -include("logger.hrl"). -include("types.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + -logger_header("[Channel]"). -ifdef(TEST). @@ -850,11 +852,14 @@ handle_out(disconnect, ReasonCode, Channel) when is_integer(ReasonCode) -> ReasonName = disconnect_reason(ReasonCode), handle_out(disconnect, {ReasonCode, ReasonName}, Channel); -handle_out(disconnect, {ReasonCode, ReasonName}, Channel = ?IS_MQTT_V5) -> - Packet = ?DISCONNECT_PACKET(ReasonCode), +handle_out(disconnect, {ReasonCode, ReasonName}, Channel) -> + handle_out(disconnect, {ReasonCode, ReasonName, #{}}, Channel); + +handle_out(disconnect, {ReasonCode, ReasonName, Props}, Channel = ?IS_MQTT_V5) -> + Packet = ?DISCONNECT_PACKET(ReasonCode, Props), {ok, [{outgoing, Packet}, {close, ReasonName}], Channel}; -handle_out(disconnect, {_ReasonCode, ReasonName}, Channel) -> +handle_out(disconnect, {_ReasonCode, ReasonName, _Props}, Channel) -> {ok, {close, ReasonName}, Channel}; handle_out(auth, {ReasonCode, Properties}, Channel) -> @@ -954,11 +959,15 @@ handle_call({takeover, 'begin'}, Channel = #channel{session = Session}) -> reply(Session, Channel#channel{takeover = true}); handle_call({takeover, 'end'}, Channel = #channel{session = Session, - pendings = Pendings}) -> + pendings = Pendings, + conninfo = #{clientid := ClientId}}) -> ok = emqx_session:takeover(Session), %% TODO: Should not drain deliver here (side effect) Delivers = emqx_misc:drain_deliver(), AllPendings = lists:append(Delivers, Pendings), + ?tp(debug, + emqx_channel_takeover_end, + #{clientid => ClientId}), disconnect_and_shutdown(takeovered, AllPendings, Channel); handle_call(list_acl_cache, Channel) -> @@ -1019,6 +1028,9 @@ handle_info(clean_acl_cache, Channel) -> ok = emqx_acl_cache:empty_acl_cache(), {ok, Channel}; +handle_info({disconnect, ReasonCode, ReasonName, Props}, Channel) -> + handle_out(disconnect, {ReasonCode, ReasonName, Props}, Channel); + handle_info(Info, Channel) -> ?LOG(error, "Unexpected info: ~p", [Info]), {ok, Channel}. diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index 09a2ac9a0..09169b160 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -22,6 +22,8 @@ -include("emqx.hrl"). -include("logger.hrl"). -include("types.hrl"). +-include_lib("stdlib/include/qlc.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -logger_header("[CM]"). @@ -60,7 +62,9 @@ , lookup_channels/2 ]). --export([all_channels/0]). +-export([all_channels/0, + channel_with_session_table/0, + live_connection_table/0]). %% gen_server callbacks -export([ init/1 @@ -149,8 +153,11 @@ connection_closed(ClientId) -> connection_closed(ClientId, self()). -spec(connection_closed(emqx_types:clientid(), chan_pid()) -> true). -connection_closed(ClientId, ChanPid) -> - ets:delete_object(?CHAN_CONN_TAB, {ClientId, ChanPid}). +connection_closed(_ClientId, _ChanPid) -> + %% We can't clean CHAN_CONN_TAB because records for dead connections + %% are required for `get_chann_conn_mod/1` function, and `get_chann_conn_mod/1` + %% is used for takeover. + true. %% @doc Get info of a channel. -spec(get_chan_info(emqx_types:clientid()) -> maybe(emqx_types:infos())). @@ -425,6 +432,38 @@ all_channels() -> Pat = [{{'_', '$1'}, [], ['$1']}], ets:select(?CHAN_TAB, Pat). +%% @doc Get clientinfo for all clients with sessions +channel_with_session_table() -> + Ms = ets:fun2ms( + fun({{ClientId, _ChanPid}, + Info, + _Stats}) -> + {ClientId, Info} + end), + Table = ets:table(?CHAN_INFO_TAB, [{traverse, {select, Ms}}]), + qlc:q([ {ClientId, ConnState, ConnInfo, ClientInfo} + || {ClientId, + #{conn_state := ConnState, + clientinfo := ClientInfo, + conninfo := #{clean_start := false} = ConnInfo}} <- Table + ]). + +%% @doc Get all local connection query handle +live_connection_table() -> + Ms = ets:fun2ms( + fun({{ClientId, ChanPid}, _}) -> + {ClientId, ChanPid} + end), + Table = ets:table(?CHAN_CONN_TAB, [{traverse, {select, Ms}}]), + qlc:q([{ClientId, ChanPid} || {ClientId, ChanPid} <- Table, is_channel_connected(ClientId, ChanPid)]). + +is_channel_connected(ClientId, ChanPid) when node(ChanPid) =:= node() -> + case get_chan_info(ClientId, ChanPid) of + #{conn_state := disconnected} -> false; + _ -> true + end; +is_channel_connected(_ClientId, _ChanPid) -> false. + %% @doc Lookup channels. -spec(lookup_channels(emqx_types:clientid()) -> list(chan_pid())). lookup_channels(ClientId) -> @@ -523,4 +562,3 @@ get_chann_conn_mod(ClientId, ChanPid) when node(ChanPid) == node() -> end; get_chann_conn_mod(ClientId, ChanPid) -> rpc_call(node(ChanPid), get_chann_conn_mod, [ClientId, ChanPid], ?T_GET_INFO). - diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index 764778b74..4088988c2 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -225,6 +225,8 @@ ensure_file(File) -> , {emqx_telemetry, true} , {emqx_rule_engine, true} , {emqx_bridge_mqtt, false} + , {emqx_eviction_agent, true} + , {emqx_node_rebalance, true} ], write_loaded(DefaultPlugins); true -> diff --git a/test/emqx_node_helpers.erl b/test/emqx_node_helpers.erl new file mode 100644 index 000000000..16d4be000 --- /dev/null +++ b/test/emqx_node_helpers.erl @@ -0,0 +1,83 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_helpers). + +-include_lib("eunit/include/eunit.hrl"). + +-define(SLAVE_START_APPS, [emqx]). + +-export([start_slave/1, + start_slave/2, + stop_slave/1]). + +start_slave(Name) -> + start_slave(Name, #{}). + +start_slave(Name, Opts) -> + {ok, Node} = ct_slave:start(list_to_atom(atom_to_list(Name) ++ "@" ++ host()), + [{kill_if_fail, true}, + {monitor_master, true}, + {init_timeout, 10000}, + {startup_timeout, 10000}, + {erl_flags, ebin_path()}]), + + pong = net_adm:ping(Node), + setup_node(Node, Opts), + Node. + +stop_slave(Node) -> + rpc:call(Node, ekka, leave, []), + ct_slave:stop(Node). + +host() -> + [_, Host] = string:tokens(atom_to_list(node()), "@"), Host. + +ebin_path() -> + string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " "). + +is_lib(Path) -> + string:prefix(Path, code:lib_dir()) =:= nomatch. + +setup_node(Node, #{} = Opts) -> + Listeners = maps:get(listeners, Opts, []), + StartApps = maps:get(start_apps, Opts, ?SLAVE_START_APPS), + DefaultEnvHandler = + fun(emqx) -> + application:set_env( + emqx, + listeners, + Listeners), + application:set_env(gen_rpc, port_discovery, stateless), + ok; + (_) -> + ok + end, + EnvHandler = maps:get(env_handler, Opts, DefaultEnvHandler), + + [ok = rpc:call(Node, application, load, [App]) || App <- [gen_rpc, emqx]], + ok = rpc:call(Node, emqx_ct_helpers, start_apps, [StartApps, EnvHandler]), + + rpc:call(Node, ekka, join, [node()]), + + %% Sanity check. Assert that `gen_rpc' is set up correctly: + ?assertEqual( Node + , gen_rpc:call(Node, erlang, node, []) + ), + ?assertEqual( node() + , gen_rpc:call(Node, gen_rpc, call, [node(), erlang, node, []]) + ), + ok. diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl index 66a88a047..b95030889 100644 --- a/test/emqx_plugins_SUITE.erl +++ b/test/emqx_plugins_SUITE.erl @@ -99,8 +99,10 @@ t_ensure_default_loaded_plugins_file(Config) -> ?assertEqual( [ {emqx_bridge_mqtt, false} , {emqx_dashboard, true} + , {emqx_eviction_agent, true} , {emqx_management, true} , {emqx_modules, true} + , {emqx_node_rebalance, true} , {emqx_recon, true} , {emqx_retainer, true} , {emqx_rule_engine, true} diff --git a/test/emqx_shared_sub_SUITE.erl b/test/emqx_shared_sub_SUITE.erl index 02c4c6598..0eb63dbf0 100644 --- a/test/emqx_shared_sub_SUITE.erl +++ b/test/emqx_shared_sub_SUITE.erl @@ -380,7 +380,7 @@ t_local(_) -> emqtt:stop(ConnPid1), emqtt:stop(ConnPid2), - stop_slave(Node), + emqx_node_helpers:stop_slave(Node), ?assertEqual(local, emqx_shared_sub:strategy(<<"local_group">>)), ?assertEqual(local, RemoteLocalGroupStrategy), @@ -415,7 +415,7 @@ t_local_fallback(_) -> {true, UsedSubPid2} = last_message(<<"hello2">>, [ConnPid1]), emqtt:stop(ConnPid1), - stop_slave(Node), + emqx_node_helpers:stop_slave(Node), ?assertEqual(UsedSubPid1, UsedSubPid2), ok. @@ -536,55 +536,8 @@ recv_msgs(Count, Msgs) -> end. start_slave(Name, Port) -> - {ok, Node} = ct_slave:start(list_to_atom(atom_to_list(Name) ++ "@" ++ host()), - [{kill_if_fail, true}, - {monitor_master, true}, - {init_timeout, 10000}, - {startup_timeout, 10000}, - {erl_flags, ebin_path()}]), - - pong = net_adm:ping(Node), - ok = setup_node(Node, Port), - Node. - -stop_slave(Node) -> - rpc:call(Node, ekka, leave, []), - ct_slave:stop(Node). - -host() -> - [_, Host] = string:tokens(atom_to_list(node()), "@"), Host. - -ebin_path() -> - string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " "). - -is_lib(Path) -> - string:prefix(Path, code:lib_dir()) =:= nomatch. - -setup_node(Node, Port) -> - EnvHandler = - fun(emqx) -> - application:set_env( - emqx, - listeners, - [#{listen_on => {{127,0,0,1},Port}, - name => "internal", - opts => [{zone,internal}], - proto => tcp}]), - application:set_env(gen_rpc, port_discovery, stateless), - ok; - (_) -> - ok - end, - - [ok = rpc:call(Node, application, load, [App]) || App <- [gen_rpc, emqx]], - ok = rpc:call(Node, emqx_ct_helpers, start_apps, [[emqx], EnvHandler]), - rpc:call(Node, ekka, join, [node()]), - - %% Sanity check. Assert that `gen_rpc' is set up correctly: - ?assertEqual( Node - , gen_rpc:call(Node, erlang, node, []) - ), - ?assertEqual( node() - , gen_rpc:call(Node, gen_rpc, call, [node(), erlang, node, []]) - ), - ok. + Listeners = [#{listen_on => {{127,0,0,1}, Port}, + name => "internal", + opts => [{zone,internal}], + proto => tcp}], + emqx_node_helpers:start_slave(Name, #{listeners => Listeners}). From 9435b6aa8227f7744630c0bfdb704f05687be655 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 13 Aug 2022 14:07:55 +0200 Subject: [PATCH 24/73] docs: Update CHANGES-4.3.md --- CHANGES-4.3.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 10304fc70..ab74b877a 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -56,7 +56,6 @@ File format: - HTTP API(GET /rules/) support for pagination and fuzzy filtering. [#8450] - Add check_conf cli to check config format. [#8486] - Optimize performance of shared subscription -- Make possible to debug-print SSL handshake procedure by setting listener config `log_level=debug` [#8553](https://github.com/emqx/emqx/pull/8553) ## v4.3.16 From 698b1b6df67e0b44629aae5f93cedd8983714c99 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 11 Aug 2022 16:45:03 +0800 Subject: [PATCH 25/73] fix: different default plugins start between ee and ce --- src/emqx_plugins.erl | 50 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index 4088988c2..1c8849745 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -217,22 +217,50 @@ load_plugin_conf(AppName, PluginDir) -> ensure_file(File) -> case filelib:is_file(File) of false -> - DefaultPlugins = [ {emqx_management, true} - , {emqx_dashboard, true} - , {emqx_modules, true} - , {emqx_recon, true} - , {emqx_retainer, true} - , {emqx_telemetry, true} - , {emqx_rule_engine, true} - , {emqx_bridge_mqtt, false} - , {emqx_eviction_agent, true} - , {emqx_node_rebalance, true} - ], + DefaultPlugins = default_plugins(), write_loaded(DefaultPlugins); true -> ok end. +-ifndef(EMQX_ENTERPRISE). +%% default plugins see rebar.config.erl +default_plugins() -> + [ + {emqx_management, true}, + {emqx_dashboard, true}, + %% emqx_modules is not a plugin, but a normal application starting when boots. + {emqx_modules, false}, + {emqx_retainer, true}, + {emqx_recon, true}, + {emqx_telemetry, true}, + {emqx_rule_engine, true}, + {emqx_bridge_mqtt, false}, + {emqx_eviction_agent, true}, + {emqx_node_rebalance, true} + ]. + +-else. + +default_plugins() -> + [ + {emqx_management, true}, + {emqx_dashboard, true}, + %% enterprise version of emqx_modules is a plugin + {emqx_modules, true}, + %% retainer is managed by emqx_modules. + %% default is true in data/load_modules. **NOT HERE** + {emqx_retainer, false}, + {emqx_recon, true}, + {emqx_telemetry, true}, + {emqx_rule_engine, true}, + {emqx_bridge_mqtt, false}, + {emqx_eviction_agent, true}, + {emqx_node_rebalance, true} + ]. + +-endif. + with_loaded_file(File, SuccFun) -> case read_loaded(File) of {ok, Names0} -> From e6a98524b5755050f72f42e70d696b50edbb8360 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 11 Aug 2022 23:20:41 +0800 Subject: [PATCH 26/73] chore: improve retainer ct init between ce and ee --- .../test/emqx_retainer_SUITE.erl | 6 +-- .../test/emqx_retainer_cli_SUITE.erl | 5 +- .../test/emqx_retainer_ct_helper.erl | 46 ++++++++++++++++++ .../test/mqtt_protocol_v5_SUITE.erl | 7 +-- test/emqx_plugins_SUITE.erl | 47 ++++++++++++++----- 5 files changed, 90 insertions(+), 21 deletions(-) create mode 100644 apps/emqx_retainer/test/emqx_retainer_ct_helper.erl diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl index 6f29eaedf..0e38548bf 100644 --- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl @@ -31,11 +31,12 @@ all() -> emqx_ct:all(?MODULE). %%-------------------------------------------------------------------- init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_retainer]), + emqx_retainer_ct_helper:ensure_start(), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_retainer]). + emqx_retainer_ct_helper:ensure_stop(), + ok. init_per_testcase(TestCase, Config) -> emqx_retainer:clean(<<"#">>), @@ -207,4 +208,3 @@ receive_messages(Count, Msgs) -> after 2000 -> Msgs end. - diff --git a/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl index 452a4f3d9..995e3508c 100644 --- a/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl @@ -24,11 +24,12 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_retainer]), + emqx_retainer_ct_helper:ensure_start(), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_retainer]). + emqx_retainer_ct_helper:ensure_stop(), + ok. init_per_testcase(_TestCase, Config) -> Config. diff --git a/apps/emqx_retainer/test/emqx_retainer_ct_helper.erl b/apps/emqx_retainer/test/emqx_retainer_ct_helper.erl new file mode 100644 index 000000000..fd12a4e01 --- /dev/null +++ b/apps/emqx_retainer/test/emqx_retainer_ct_helper.erl @@ -0,0 +1,46 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_retainer_ct_helper). + + +%% API +-export([ensure_start/0, ensure_stop/0]). +-ifdef(EMQX_ENTERPRISE). +ensure_start() -> + application:stop(emqx_modules), + init_conf(), + emqx_ct_helpers:start_apps([emqx_retainer]), + ok. + +-else. + +ensure_start() -> + init_conf(), + emqx_ct_helpers:start_apps([emqx_retainer]), + ok. + +-endif. + +ensure_stop() -> + emqx_ct_helpers:stop_apps([emqx_retainer]). + + +init_conf() -> + application:set_env(emqx_retainer, expiry_interval, 0), + application:set_env(emqx_retainer, max_payload_size, 1024000), + application:set_env(emqx_retainer, max_retained_messages, 0), + application:set_env(emqx_retainer, storage_type, ram), + ok. diff --git a/apps/emqx_retainer/test/mqtt_protocol_v5_SUITE.erl b/apps/emqx_retainer/test/mqtt_protocol_v5_SUITE.erl index ef0b111e9..17a31e04b 100644 --- a/apps/emqx_retainer/test/mqtt_protocol_v5_SUITE.erl +++ b/apps/emqx_retainer/test/mqtt_protocol_v5_SUITE.erl @@ -27,12 +27,13 @@ init_per_suite(Config) -> %% Meck emqtt ok = meck:new(emqtt, [non_strict, passthrough, no_history, no_link]), %% Start Apps - emqx_ct_helpers:start_apps([emqx_retainer]), + emqx_retainer_ct_helper:ensure_start(), Config. end_per_suite(_Config) -> ok = meck:unload(emqtt), - emqx_ct_helpers:stop_apps([emqx_retainer]). + emqx_retainer_ct_helper:ensure_stop(). + %%-------------------------------------------------------------------- %% Helpers @@ -107,7 +108,7 @@ t_publish_message_expiry_interval(_) -> Msgs = receive_messages(4), ?assertEqual(2, length(Msgs)), %% [MQTT-3.3.2-5] - L = lists:map(fun(Msg) -> MessageExpiryInterval = maps:get('Message-Expiry-Interval', maps:get(properties, Msg)), MessageExpiryInterval < 10 end, Msgs), + L = lists:map(fun(Msg) -> MessageExpiryInterval = maps:get('Message-Expiry-Interval', maps:get(properties, Msg)), MessageExpiryInterval < 10 end, Msgs), ?assertEqual(2, length(L)), %% [MQTT-3.3.2-6] ok = emqtt:disconnect(Client1), diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl index b95030889..aa548d766 100644 --- a/test/emqx_plugins_SUITE.erl +++ b/test/emqx_plugins_SUITE.erl @@ -89,6 +89,38 @@ t_load(_) -> ?assertEqual(ignore, emqx_plugins:load()), ?assertEqual(ignore, emqx_plugins:unload()). +-ifndef(EMQX_ENTERPRISE). +default_plugins() -> + [ + {emqx_bridge_mqtt, false}, + {emqx_dashboard, true}, + {emqx_eviction_agent, true}, + {emqx_management, true}, + {emqx_modules, false}, + {emqx_node_rebalance, true}, + {emqx_recon, true}, + {emqx_retainer, true}, + {emqx_rule_engine, true}, + {emqx_telemetry, true} + ]. + +-else. + +default_plugins() -> + [ + {emqx_bridge_mqtt, false}, + {emqx_dashboard, true}, + {emqx_eviction_agent, true}, + {emqx_management, true}, + {emqx_modules, true}, + {emqx_node_rebalance, true}, + {emqx_recon, true}, + {emqx_retainer, false}, + {emqx_rule_engine, true}, + {emqx_telemetry, true} + ]. + +-endif. t_ensure_default_loaded_plugins_file(Config) -> %% this will trigger it to write the default plugins to the %% inexistent file; but it won't truly load them in this test @@ -96,19 +128,8 @@ t_ensure_default_loaded_plugins_file(Config) -> TmpFilepath = ?config(tmp_filepath, Config), ok = emqx_plugins:load(), {ok, Contents} = file:consult(TmpFilepath), - ?assertEqual( - [ {emqx_bridge_mqtt, false} - , {emqx_dashboard, true} - , {emqx_eviction_agent, true} - , {emqx_management, true} - , {emqx_modules, true} - , {emqx_node_rebalance, true} - , {emqx_recon, true} - , {emqx_retainer, true} - , {emqx_rule_engine, true} - , {emqx_telemetry, true} - ], - lists:sort(Contents)), + DefaultPlugins = default_plugins(), + ?assertEqual(DefaultPlugins, lists:sort(Contents)), ok. t_init_config(_) -> From b02a3b94293b2c9ec063bc3cd915958bb2d25122 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 1 Aug 2022 17:14:42 +0800 Subject: [PATCH 27/73] fix(channel): Adjust the timing of the `client.connected` event --- src/emqx_channel.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 852c171fe..f422bfcf9 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -293,7 +293,7 @@ handle_in(?CONNECT_PACKET(ConnPkt), Channel) -> }, case enhanced_auth(?CONNECT_PACKET(NConnPkt), NChannel1) of {ok, Properties, NChannel2} -> - process_connect(Properties, ensure_connected(NChannel2)); + process_connect(Properties, NChannel2); {continue, Properties, NChannel2} -> handle_out(auth, {?RC_CONTINUE_AUTHENTICATION, Properties}, NChannel2); {error, ReasonCode, NChannel2} -> @@ -309,7 +309,7 @@ handle_in(Packet = ?AUTH_PACKET(?RC_CONTINUE_AUTHENTICATION, _Properties), {ok, NProperties, NChannel} -> case ConnState of connecting -> - process_connect(NProperties, ensure_connected(NChannel)); + process_connect(NProperties, NChannel); connected -> handle_out(auth, {?RC_SUCCESS, NProperties}, NChannel); _ -> @@ -499,14 +499,14 @@ process_connect(AckProps, Channel = #channel{conninfo = ConnInfo, case emqx_cm:open_session(CleanStart, ClientInfo, ConnInfo) of {ok, #{session := Session, present := false}} -> NChannel = Channel#channel{session = Session}, - handle_out(connack, {?RC_SUCCESS, sp(false), AckProps}, NChannel); + handle_out(connack, {?RC_SUCCESS, sp(false), AckProps}, ensure_connected(NChannel)); {ok, #{session := Session, present := true, pendings := Pendings}} -> Pendings1 = lists:usort(lists:append(Pendings, emqx_misc:drain_deliver())), NChannel = Channel#channel{session = Session, resuming = true, pendings = Pendings1 }, - handle_out(connack, {?RC_SUCCESS, sp(true), AckProps}, NChannel); + handle_out(connack, {?RC_SUCCESS, sp(true), AckProps}, ensure_connected(NChannel)); {error, client_id_unavailable} -> handle_out(connack, ?RC_CLIENT_IDENTIFIER_NOT_VALID, Channel); {error, Reason} -> From 93a97e0df2f826439955fa9679b44c1ca293b420 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 16 Aug 2022 00:56:50 +0200 Subject: [PATCH 28/73] chore: make a no-op change in emqx_retainer_sup Just to keep the appup in sync with enterprise edition --- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- .../emqx_retainer/src/emqx_retainer.appup.src | 19 +++++++++++++------ apps/emqx_retainer/src/emqx_retainer_sup.erl | 5 ++++- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index 05920e985..80ec02c73 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -1,6 +1,6 @@ {application, emqx_retainer, [{description, "EMQ X Retainer"}, - {vsn, "4.3.3"}, % strict semver, bump manually! + {vsn, "4.3.4"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel,stdlib]}, diff --git a/apps/emqx_retainer/src/emqx_retainer.appup.src b/apps/emqx_retainer/src/emqx_retainer.appup.src index 45ec6420c..18f047119 100644 --- a/apps/emqx_retainer/src/emqx_retainer.appup.src +++ b/apps/emqx_retainer/src/emqx_retainer.appup.src @@ -1,14 +1,21 @@ %% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.2", - [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]}, + [{"4.3.3",[{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}]}, + {"4.3.2", + [{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-1]">>, - [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}, + [{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_retainer,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.2", - [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]}, + [{"4.3.3",[{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}]}, + {"4.3.2", + [{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-1]">>, - [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}, + [{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_retainer,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. diff --git a/apps/emqx_retainer/src/emqx_retainer_sup.erl b/apps/emqx_retainer/src/emqx_retainer_sup.erl index 22af32504..d3a979788 100644 --- a/apps/emqx_retainer/src/emqx_retainer_sup.erl +++ b/apps/emqx_retainer/src/emqx_retainer_sup.erl @@ -32,5 +32,8 @@ init([Env]) -> restart => permanent, shutdown => 5000, type => worker, - modules => [emqx_retainer]}]}}. + modules => [emqx_retainer]} || not is_managed_by_modules()]}}. +is_managed_by_modules() -> + %% always flase for opensource edition + false. From 303249e2e8a8a5d70a7c0fb604690450a7c9dd54 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 16 Aug 2022 07:40:16 +0200 Subject: [PATCH 29/73] chore: move rebalance and eviction code to ee --- apps/emqx_eviction_agent/.gitignore | 19 - apps/emqx_eviction_agent/README.md | 9 - .../etc/emqx_eviction_agent.conf | 3 - .../priv/emqx_eviction_agent.schema | 0 apps/emqx_eviction_agent/rebar.config | 2 - .../src/emqx_eviction_agent.app.src | 18 - .../src/emqx_eviction_agent.erl | 291 ------------ .../src/emqx_eviction_agent_api.erl | 36 -- .../src/emqx_eviction_agent_app.erl | 36 -- .../src/emqx_eviction_agent_channel.erl | 299 ------------- .../src/emqx_eviction_agent_cli.erl | 42 -- .../src/emqx_eviction_agent_conn_sup.erl | 33 -- .../src/emqx_eviction_agent_sup.erl | 43 -- .../test/emqx_eviction_agent_SUITE.erl | 232 ---------- .../test/emqx_eviction_agent_api_SUITE.erl | 64 --- .../emqx_eviction_agent_channel_SUITE.erl | 155 ------- .../test/emqx_eviction_agent_cli_SUITE.erl | 47 -- .../test/emqx_eviction_agent_test_helpers.erl | 55 --- apps/emqx_node_rebalance/.gitignore | 19 - apps/emqx_node_rebalance/README.md | 9 - .../etc/emqx_node_rebalance.conf | 3 - .../include/emqx_node_rebalance.hrl | 31 -- .../priv/emqx_node_rebalance.schema | 0 apps/emqx_node_rebalance/rebar.config | 2 - .../src/emqx_node_rebalance.app.src | 19 - .../src/emqx_node_rebalance.erl | 414 ------------------ .../src/emqx_node_rebalance_agent.erl | 127 ------ .../src/emqx_node_rebalance_api.erl | 243 ---------- .../src/emqx_node_rebalance_app.erl | 34 -- .../src/emqx_node_rebalance_cli.erl | 265 ----------- .../src/emqx_node_rebalance_evacuation.erl | 298 ------------- ...emqx_node_rebalance_evacuation_persist.erl | 109 ----- .../src/emqx_node_rebalance_status.erl | 225 ---------- .../src/emqx_node_rebalance_sup.erl | 44 -- .../test/emqx_node_rebalance_SUITE.erl | 183 -------- .../test/emqx_node_rebalance_agent_SUITE.erl | 163 ------- .../test/emqx_node_rebalance_api_SUITE.erl | 321 -------------- .../test/emqx_node_rebalance_cli_SUITE.erl | 199 --------- .../emqx_node_rebalance_evacuation_SUITE.erl | 194 -------- ...ode_rebalance_evacuation_persist_SUITE.erl | 107 ----- data/loaded_plugins.tmpl | 2 - rebar.config.erl | 2 - src/emqx_plugins.erl | 4 +- test/emqx_plugins_SUITE.erl | 2 - 44 files changed, 1 insertion(+), 4402 deletions(-) delete mode 100644 apps/emqx_eviction_agent/.gitignore delete mode 100644 apps/emqx_eviction_agent/README.md delete mode 100644 apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf delete mode 100644 apps/emqx_eviction_agent/priv/emqx_eviction_agent.schema delete mode 100644 apps/emqx_eviction_agent/rebar.config delete mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src delete mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent.erl delete mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl delete mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl delete mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl delete mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl delete mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl delete mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl delete mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl delete mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl delete mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl delete mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl delete mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl delete mode 100644 apps/emqx_node_rebalance/.gitignore delete mode 100644 apps/emqx_node_rebalance/README.md delete mode 100644 apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf delete mode 100644 apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl delete mode 100644 apps/emqx_node_rebalance/priv/emqx_node_rebalance.schema delete mode 100644 apps/emqx_node_rebalance/rebar.config delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance.erl delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl delete mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl delete mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl delete mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl delete mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl delete mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl delete mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl diff --git a/apps/emqx_eviction_agent/.gitignore b/apps/emqx_eviction_agent/.gitignore deleted file mode 100644 index f1c455451..000000000 --- a/apps/emqx_eviction_agent/.gitignore +++ /dev/null @@ -1,19 +0,0 @@ -.rebar3 -_* -.eunit -*.o -*.beam -*.plt -*.swp -*.swo -.erlang.cookie -ebin -log -erl_crash.dump -.rebar -logs -_build -.idea -*.iml -rebar3.crashdump -*~ diff --git a/apps/emqx_eviction_agent/README.md b/apps/emqx_eviction_agent/README.md deleted file mode 100644 index f9b8037bf..000000000 --- a/apps/emqx_eviction_agent/README.md +++ /dev/null @@ -1,9 +0,0 @@ -emqx_eviction_agent -===== - -An OTP library - -Build ------ - - $ rebar3 compile diff --git a/apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf b/apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf deleted file mode 100644 index 011b7fb0f..000000000 --- a/apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf +++ /dev/null @@ -1,3 +0,0 @@ -##-------------------------------------------------------------------- -## EMQX Eviction Agent Plugin -##-------------------------------------------------------------------- diff --git a/apps/emqx_eviction_agent/priv/emqx_eviction_agent.schema b/apps/emqx_eviction_agent/priv/emqx_eviction_agent.schema deleted file mode 100644 index e69de29bb..000000000 diff --git a/apps/emqx_eviction_agent/rebar.config b/apps/emqx_eviction_agent/rebar.config deleted file mode 100644 index 2656fd554..000000000 --- a/apps/emqx_eviction_agent/rebar.config +++ /dev/null @@ -1,2 +0,0 @@ -{erl_opts, [debug_info]}. -{deps, []}. diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src b/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src deleted file mode 100644 index ffa28af61..000000000 --- a/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src +++ /dev/null @@ -1,18 +0,0 @@ -{application, emqx_eviction_agent, - [{description, "EMQX Eviction Agent"}, - {vsn, "4.3.0"}, - {registered, [emqx_eviction_agent_sup, - emqx_eviction_agent, - emqx_eviction_agent_conn_sup]}, - {applications, - [kernel, - stdlib - ]}, - {mod, {emqx_eviction_agent_app,[]}}, - {env,[]}, - {modules, []}, - {maintainers, ["EMQX Team "]}, - {links, [{"Homepage", "https://emqx.io/"}, - {"Github", "https://github.com/emqx"} - ]} - ]}. diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent.erl deleted file mode 100644 index a8cf442cf..000000000 --- a/apps/emqx_eviction_agent/src/emqx_eviction_agent.erl +++ /dev/null @@ -1,291 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_eviction_agent). - --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/types.hrl"). - --include_lib("stdlib/include/qlc.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). - --export([start_link/0, - enable/2, - disable/1, - status/0, - connection_count/0, - session_count/0, - session_count/1, - evict_connections/1, - evict_sessions/2, - evict_sessions/3, - evict_session_channel/3 - ]). - --behaviour(gen_server). - --export([init/1, - handle_call/3, - handle_info/2, - handle_cast/2, - code_change/3 - ]). - --export([on_connect/2, - on_connack/3]). - --export([hook/0, - unhook/0]). - --export_type([server_reference/0]). - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - --type server_reference() :: binary() | undefined. --type status() :: {enabled, conn_stats()} | disabled. --type conn_stats() :: #{connections := non_neg_integer(), - sessions := non_neg_integer()}. --type kind() :: atom(). - --spec start_link() -> startlink_ret(). -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - --spec enable(kind(), server_reference()) -> ok_or_error(eviction_agent_busy). -enable(Kind, ServerReference) -> - gen_server:call(?MODULE, {enable, Kind, ServerReference}). - --spec disable(kind()) -> ok. -disable(Kind) -> - gen_server:call(?MODULE, {disable, Kind}). - --spec status() -> status(). -status() -> - case enable_status() of - {enabled, _Kind, _ServerReference} -> - {enabled, stats()}; - disabled -> - disabled - end. - --spec evict_connections(pos_integer()) -> ok_or_error(disabled). -evict_connections(N) -> - case enable_status() of - {enabled, _Kind, ServerReference} -> - ok = do_evict_connections(N, ServerReference); - disabled -> - {error, disabled} - end. - --spec evict_sessions(pos_integer(), node() | [node()]) -> ok_or_error(disabled). -evict_sessions(N, Node) when is_atom(Node) -> - evict_sessions(N, [Node]); -evict_sessions(N, Nodes) when is_list(Nodes) andalso length(Nodes) > 0 -> - evict_sessions(N, Nodes, any). - --spec evict_sessions(pos_integer(), node() | [node()], atom()) -> ok_or_error(disabled). -evict_sessions(N, Node, ConnState) when is_atom(Node) -> - evict_sessions(N, [Node], ConnState); -evict_sessions(N, Nodes, ConnState) - when is_list(Nodes) andalso length(Nodes) > 0 -> - case enable_status() of - {enabled, _Kind, _ServerReference} -> - ok = do_evict_sessions(N, Nodes, ConnState); - disabled -> - {error, disabled} - end. - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([]) -> - _ = persistent_term:erase(?MODULE), - {ok, #{}}. - -%% enable -handle_call({enable, Kind, ServerReference}, _From, St) -> - Reply = case enable_status() of - disabled -> - ok = persistent_term:put(?MODULE, {enabled, Kind, ServerReference}); - {enabled, Kind, _ServerReference} -> - ok = persistent_term:put(?MODULE, {enabled, Kind, ServerReference}); - {enabled, _OtherKind, _ServerReference} -> - {error, eviction_agent_busy} - end, - {reply, Reply, St}; - -%% disable -handle_call({disable, Kind}, _From, St) -> - Reply = case enable_status() of - disabled -> - {error, disabled}; - {enabled, Kind, _ServerReference} -> - _ = persistent_term:erase(?MODULE), - ok; - {enabled, _OtherKind, _ServerReference} -> - {error, eviction_agent_busy} - end, - {reply, Reply, St}. - -handle_info(Msg, St) -> - ?LOG(warning, "Unknown Msg: ~p, State: ~p", [Msg, St]), - {noreply, St}. - -handle_cast(Msg, St) -> - ?LOG(warning, "Unknown cast Msg: ~p, State: ~p", [Msg, St]), - {noreply, St}. - -code_change(_Vsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Hook callbacks -%%-------------------------------------------------------------------- - -on_connect(_ConnInfo, _Props) -> - case enable_status() of - {enabled, _Kind, _ServerReference} -> - {stop, {error, ?RC_USE_ANOTHER_SERVER}}; - disabled -> - ignore - end. - -on_connack(#{proto_name := <<"MQTT">>, proto_ver := ?MQTT_PROTO_V5}, - use_another_server, - Props) -> - case enable_status() of - {enabled, _Kind, ServerReference} -> - {ok, Props#{'Server-Reference' => ServerReference}}; - disabled -> - {ok, Props} - end; -on_connack(_ClientInfo, _Reason, Props) -> - {ok, Props}. - -%%-------------------------------------------------------------------- -%% Hook funcs -%%-------------------------------------------------------------------- - -hook() -> - ?tp(debug, eviction_agent_hook, #{}), - ok = emqx_hooks:put('client.connack', {?MODULE, on_connack, []}), - ok = emqx_hooks:put('client.connect', {?MODULE, on_connect, []}). - -unhook() -> - ?tp(debug, eviction_agent_unhook, #{}), - ok = emqx_hooks:del('client.connect', {?MODULE, on_connect}), - ok = emqx_hooks:del('client.connack', {?MODULE, on_connack}). - -enable_status() -> - persistent_term:get(?MODULE, disabled). - -% connection management -stats() -> - #{ - connections => connection_count(), - sessions => session_count() - }. - -connection_table() -> - emqx_cm:live_connection_table(). - -connection_count() -> - table_count(connection_table()). - -channel_with_session_table(any) -> - qlc:q([{ClientId, ConnInfo, ClientInfo} - || {ClientId, _, ConnInfo, ClientInfo} <- emqx_cm:channel_with_session_table()]); -channel_with_session_table(RequiredConnState) -> - qlc:q([{ClientId, ConnInfo, ClientInfo} - || {ClientId, ConnState, ConnInfo, ClientInfo} <- emqx_cm:channel_with_session_table(), - RequiredConnState =:= ConnState]). - -session_count() -> - session_count(any). - -session_count(ConnState) -> - table_count(channel_with_session_table(ConnState)). - -table_count(QH) -> - qlc:fold(fun(_, Acc) -> Acc + 1 end, 0, QH). - -take_connections(N) -> - ChanQH = qlc:q([ChanPid || {_ClientId, ChanPid} <- connection_table()]), - ChanPidCursor = qlc:cursor(ChanQH), - ChanPids = qlc:next_answers(ChanPidCursor, N), - ok = qlc:delete_cursor(ChanPidCursor), - ChanPids. - -take_channel_with_sessions(N, ConnState) -> - ChanPidCursor = qlc:cursor(channel_with_session_table(ConnState)), - Channels = qlc:next_answers(ChanPidCursor, N), - ok = qlc:delete_cursor(ChanPidCursor), - Channels. - -do_evict_connections(N, ServerReference) when N > 0 -> - ChanPids = take_connections(N), - ok = lists:foreach( - fun(ChanPid) -> - disconnect_channel(ChanPid, ServerReference) - end, - ChanPids). - -do_evict_sessions(N, Nodes, ConnState) when N > 0 -> - Channels = take_channel_with_sessions(N, ConnState), - ok = lists:foreach( - fun({ClientId, ConnInfo, ClientInfo}) -> - evict_session_channel(Nodes, ClientId, ConnInfo, ClientInfo) - end, - Channels). - -evict_session_channel(Nodes, ClientId, ConnInfo, ClientInfo) -> - Node = select_random(Nodes), - ?LOG(info, "Evicting client=~p to node=~p, conninfo=~p, clientinfo=~p", - [ClientId, Node, ConnInfo, ClientInfo]), - case rpc:call(Node, ?MODULE, evict_session_channel, [ClientId, ConnInfo, ClientInfo]) of - {badrpc, Reason} -> - ?LOG(error, "RPC error while evicting client=~p to node=~p: ~p", - [ClientId, Node, Reason]), - {error, Reason}; - {error, Reason} = Error -> - ?LOG(error, "Error evicting client=~p to node=~p: ~p", - [ClientId, Node, Reason]), - Error; - Res -> Res - end. - -evict_session_channel(ClientId, ConnInfo, ClientInfo) -> - ?LOG(info, "Taking up client=~p, conninfo=~p, clientinfo=~p", - [ClientId, ConnInfo, ClientInfo]), - Result = emqx_eviction_agent_channel:start_supervised( - #{conninfo => ConnInfo, - clientinfo => ClientInfo}), - ?LOG(info, "Taking up client=~p, result=~p", - [ClientId, Result]), - Result. - -disconnect_channel(ChanPid, ServerReference) -> - ChanPid ! {disconnect, - ?RC_USE_ANOTHER_SERVER, - use_another_server, - #{'Server-Reference' => ServerReference}}. - -select_random(List) when length(List) > 0 -> - lists:nth(rand:uniform(length(List)) , List). diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl deleted file mode 100644 index 426f5978d..000000000 --- a/apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl +++ /dev/null @@ -1,36 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_eviction_agent_api). - --include_lib("emqx/include/logger.hrl"). - --rest_api(#{name => node_eviction_status, - method => 'GET', - path => "/node_eviction/status", - func => status, - descr => "Get node eviction status"}). - --export([status/2]). - -status(_Bindings, _Params) -> - case emqx_eviction_agent:status() of - disabled -> - {ok, #{status => disabled}}; - {enabled, Stats} -> - {ok, #{status => enabled, - stats => Stats}} - end. diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl deleted file mode 100644 index c13fcfb0d..000000000 --- a/apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl +++ /dev/null @@ -1,36 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_eviction_agent_app). - --behaviour(application). - --emqx_plugin(?MODULE). - --export([ start/2 - , stop/1 - ]). - -start(_Type, _Args) -> - Env = application:get_all_env(emqx_eviction_agent), - ok = emqx_eviction_agent:hook(), - {ok, Sup} = emqx_eviction_agent_sup:start_link(Env), - ok = emqx_eviction_agent_cli:load(), - {ok, Sup}. - -stop(_State) -> - ok = emqx_eviction_agent:unhook(), - ok = emqx_eviction_agent_cli:unload(). diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl deleted file mode 100644 index 1f6fad00f..000000000 --- a/apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl +++ /dev/null @@ -1,299 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% MQTT Channel --module(emqx_eviction_agent_channel). - --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/types.hrl"). - --include_lib("snabbkaffe/include/snabbkaffe.hrl"). - --logger_header("[Evicted Channel]"). - --export([start_link/1, - start_supervised/1, - call/2, - call/3, - cast/2, - stop/1 - ]). - --export([init/1, - handle_call/3, - handle_cast/2, - handle_info/2, - terminate/2, - code_change/3 - ]). - --type opts() :: #{conninfo := emqx_types:conninfo(), - clientinfo := emqx_types:clientinfo()}. - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - --spec start_supervised(opts()) -> startlink_ret(). -start_supervised(#{clientinfo := #{clientid := ClientId}} = Opts) -> - RandomId = integer_to_binary(erlang:unique_integer([positive])), - Id = <>, - ChildSpec = #{id => Id, - start => {?MODULE, start_link, [Opts]}, - restart => temporary, - shutdown => 5000, - type => worker, - modules => [?MODULE] - }, - supervisor:start_child( - emqx_eviction_agent_conn_sup, - ChildSpec). - --spec start_link(opts()) -> startlink_ret(). -start_link(Opts) -> - gen_server:start_link(?MODULE, [Opts], []). - --spec cast(pid(), term()) -> ok. -cast(Pid, Req) -> - gen_server:cast(Pid, Req). - --spec call(pid(), term()) -> term(). -call(Pid, Req) -> - call(Pid, Req, infinity). - --spec call(pid(), term(), timeout()) -> term(). -call(Pid, Req, Timeout) -> - gen_server:call(Pid, Req, Timeout). - --spec stop(pid()) -> ok. -stop(Pid) -> - gen_server:stop(Pid). - -%%-------------------------------------------------------------------- -%% gen_server API -%%-------------------------------------------------------------------- - -init([#{conninfo := OldConnInfo, clientinfo := #{clientid := ClientId} = OldClientInfo}]) -> - process_flag(trap_exit, true), - ClientInfo = clientinfo(OldClientInfo), - ConnInfo = conninfo(OldConnInfo), - case open_session(ConnInfo, ClientInfo) of - {ok, Channel0} -> - case set_expiry_timer(Channel0) of - {ok, Channel1} -> - ?LOG( - info, - "Channel initialized for client=~p on node=~p", - [ClientId, node()]), - {ok, Channel1, hibernate}; - {error, Reason} -> - {stop, Reason} - end; - {error, Reason} -> - {stop, Reason} - end. - -handle_call(kick, _From, Channel) -> - {stop, kicked, ok, Channel}; - -handle_call(discard, _From, Channel) -> - {stop, discarded, ok, Channel}; - -handle_call({takeover, 'begin'}, _From, #{session := Session} = Channel) -> - {reply, Session, Channel#{takeover => true}}; - -handle_call({takeover, 'end'}, _From, #{session := Session, - clientinfo := #{clientid := ClientId}, - pendings := Pendings} = Channel) -> - ok = emqx_session:takeover(Session), - %% TODO: Should not drain deliver here (side effect) - Delivers = emqx_misc:drain_deliver(), - AllPendings = lists:append(Delivers, Pendings), - ?tp(debug, - emqx_channel_takeover_end, - #{clientid => ClientId}), - {stop, normal, AllPendings, Channel}; - -handle_call(list_acl_cache, _From, Channel) -> - {reply, [], Channel}; - -handle_call({quota, _Policy}, _From, Channel) -> - {reply, ok, Channel}; - -handle_call(Req, _From, Channel) -> - ?LOG(error, "Unexpected call: ~p", [Req]), - {reply, ignored, Channel}. - -handle_info(Deliver = {deliver, _Topic, _Msg}, Channel) -> - Delivers = [Deliver | emqx_misc:drain_deliver()], - {noreply, handle_deliver(Delivers, Channel)}; - -handle_info(expire_session, Channel) -> - {stop, expired, Channel}; - -handle_info(Info, Channel) -> - ?LOG(error, "Unexpected info: ~p", [Info]), - {noreply, Channel}. - -handle_cast(Msg, Channel) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), - {noreply, Channel}. - -terminate(Reason, #{clientinfo := ClientInfo, session := Session} = Channel) -> - ok = cancel_expiry_timer(Channel), - emqx_session:terminate(ClientInfo, Reason, Session). - -code_change(_OldVsn, Channel, _Extra) -> - {ok, Channel}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -handle_deliver(Delivers, - #{takeover := true, - pendings := Pendings, - session := Session, - clientinfo := #{clientid := ClientId} = ClientInfo} = Channel) -> - %% NOTE: Order is important here. While the takeover is in - %% progress, the session cannot enqueue messages, since it already - %% passed on the queue to the new connection in the session state. - NPendings = lists:append( - Pendings, - ignore_local(ClientInfo, maybe_nack(Delivers), ClientId, Session)), - Channel#{pendings => NPendings}; - -handle_deliver(Delivers, - #{takeover := false, - session := Session, - clientinfo := #{clientid := ClientId} = ClientInfo} = Channel) -> - NSession = emqx_session:enqueue( - ClientInfo, - ignore_local(ClientInfo, maybe_nack(Delivers), ClientId, Session), - Session), - Channel#{session => NSession}. - -cancel_expiry_timer(#{expiry_timer := TRef}) when is_reference(TRef) -> - _ = erlang:cancel_timer(TRef), - ok; -cancel_expiry_timer(_) -> - ok. - -set_expiry_timer(#{conninfo := ConnInfo} = Channel) -> - case maps:get(expiry_interval, ConnInfo) of - ?UINT_MAX -> {ok, Channel}; - I when I > 0 -> - Timer = erlang:send_after(timer:seconds(I), self(), expire_session), - {ok, Channel#{expiry_timer => Timer}}; - _ -> - {error, should_be_expired} - end. - -open_session(ConnInfo, #{clientid := ClientId} = ClientInfo) -> - Channel = channel(ConnInfo, ClientInfo), - case emqx_cm:open_session(false, ClientInfo, ConnInfo) of - {ok, #{present := false}} -> - ?LOG(info, "No session for clientid=~p", [ClientId]), - {error, no_session}; - {ok, #{session := Session, present := true, pendings := Pendings0}} -> - ?LOG(info, "Session opened for client=~p on node=~p", [ClientId, node()]), - Pendings1 = lists:usort(lists:append(Pendings0, emqx_misc:drain_deliver())), - NSession = emqx_session:enqueue( - ClientInfo, - ignore_local( - ClientInfo, - maybe_nack(Pendings1), - ClientId, - Session), - Session), - NChannel = Channel#{session => NSession}, - ok = emqx_cm:insert_channel_info(ClientId, info(NChannel), []), - ?LOG(info, "Channel info updated for client=~p on node=~p", [ClientId, node()]), - {ok, NChannel}; - {error, Reason} = Error -> - ?LOG(error, "Failed to open session due to ~p", [Reason]), - Error - end. - -conninfo(OldConnInfo) -> - DisconnectedAt = maps:get(disconnected_at, OldConnInfo, erlang:system_time(millisecond)), - ConnInfo0 = maps:with( - [socktype, - sockname, - peername, - peercert, - clientid, - clean_start, - receive_maximum, - expiry_interval], - OldConnInfo), - ConnInfo0#{ - conn_mod => ?MODULE, - connected => false, - disconnected_at => DisconnectedAt - }. - -clientinfo(OldClientInfo) -> - maps:with( - [zone, - protocol, - peerhost, - sockport, - clientid, - username, - is_bridge, - is_superuser, - mountpoint], - OldClientInfo). - -channel(ConnInfo, ClientInfo) -> - #{conninfo => ConnInfo, - clientinfo => ClientInfo, - expiry_timer => undefined, - takeover => false, - resuming => false, - pendings => [] - }. - -info(Channel) -> - #{conninfo => maps:get(conninfo, Channel, undefined), - clientinfo => maps:get(clientinfo, Channel, undefined), - session => maps:get(session, Channel, undefined), - conn_state => disconnected - }. - -ignore_local(ClientInfo, Delivers, Subscriber, Session) -> - Subs = emqx_session:info(subscriptions, Session), - lists:dropwhile(fun({deliver, Topic, #message{from = Publisher} = Msg}) -> - case maps:find(Topic, Subs) of - {ok, #{nl := 1}} when Subscriber =:= Publisher -> - ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, no_local]), - ok = emqx_metrics:inc('delivery.dropped'), - ok = emqx_metrics:inc('delivery.dropped.no_local'), - true; - _ -> - false - end - end, Delivers). - -maybe_nack(Delivers) -> - lists:filter(fun not_nacked/1, Delivers). - -not_nacked({deliver, _Topic, Msg}) -> - not (emqx_shared_sub:is_ack_required(Msg) - andalso (ok == emqx_shared_sub:nack_no_connection(Msg))). diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl deleted file mode 100644 index 632f8e480..000000000 --- a/apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl +++ /dev/null @@ -1,42 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_eviction_agent_cli). - -%% APIs --export([ load/0 - , unload/0 - , cli/1 - ]). - -load() -> - emqx_ctl:register_command(eviction, {?MODULE, cli}, []). - -unload() -> - emqx_ctl:unregister_command(eviction). - -cli(["status"]) -> - case emqx_eviction_agent:status() of - disabled -> - emqx_ctl:print("Eviction status: disabled~n"); - {enabled, _Stats} -> - emqx_ctl:print("Eviction status: enabled~n") - end; - -cli(_) -> - emqx_ctl:usage( - [{"eviction status", - "Get current node eviction status"}]). diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl deleted file mode 100644 index 2b11ce9d1..000000000 --- a/apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl +++ /dev/null @@ -1,33 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_eviction_agent_conn_sup). - --behaviour(supervisor). - --export([start_link/1]). - --export([init/1]). - -start_link(Env) -> - supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]). - -init([_Env]) -> - Childs = [], - {ok, { - {one_for_one, 10, 3600}, - Childs} - }. diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl deleted file mode 100644 index 9e28c2251..000000000 --- a/apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl +++ /dev/null @@ -1,43 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_eviction_agent_sup). - --behaviour(supervisor). - --export([start_link/1]). - --export([init/1]). - -start_link(Env) -> - supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]). - -init([_Env]) -> - Childs = [child_spec(worker, emqx_eviction_agent, []), - child_spec(supervisor, emqx_eviction_agent_conn_sup, [#{}])], - {ok, { - {one_for_one, 10, 3600}, - Childs} - }. - -child_spec(Type, Mod, Args) -> - #{id => Mod, - start => {Mod, start_link, Args}, - restart => permanent, - shutdown => 5000, - type => Type, - modules => [Mod] - }. diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl deleted file mode 100644 index b77c0dee8..000000000 --- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl +++ /dev/null @@ -1,232 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_eviction_agent_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). - --import(emqx_eviction_agent_test_helpers, - [emqtt_connect/0, emqtt_connect/2]). - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_eviction_agent]), - Config. - -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_eviction_agent]). - -init_per_testcase(t_explicit_session_takeover, Config) -> - _ = emqx_eviction_agent:disable(test_eviction), - Node = emqx_node_helpers:start_slave( - evacuate1, - #{start_apps => [emqx, emqx_eviction_agent]}), - [{evacuate_node, Node} | Config]; -init_per_testcase(_TestCase, Config) -> - _ = emqx_eviction_agent:disable(test_eviction), - Config. - -end_per_testcase(t_explicit_session_takeover, Config) -> - _ = emqx_node_helpers:stop_slave(?config(evacuate_node, Config)), - _ = emqx_eviction_agent:disable(test_eviction); -end_per_testcase(_TestCase, _Config) -> - _ = emqx_eviction_agent:disable(test_eviction). - -t_enable_disable(_Config) -> - erlang:process_flag(trap_exit, true), - - ?assertMatch( - disabled, - emqx_eviction_agent:status()), - - {ok, C0} = emqtt_connect(), - ok = emqtt:disconnect(C0), - - ok = emqx_eviction_agent:enable(test_eviction, undefined), - - ?assertMatch( - {error, eviction_agent_busy}, - emqx_eviction_agent:enable(bar, undefined)), - - ?assertMatch( - ok, - emqx_eviction_agent:enable(test_eviction, <<"srv">>)), - - ?assertMatch( - {enabled, #{}}, - emqx_eviction_agent:status()), - - ?assertMatch( - {error, {use_another_server, #{}}}, - emqtt_connect()), - - ?assertMatch( - {error, eviction_agent_busy}, - emqx_eviction_agent:disable(bar)), - - ?assertMatch( - ok, - emqx_eviction_agent:disable(test_eviction)), - - ?assertMatch( - {error, disabled}, - emqx_eviction_agent:disable(test_eviction)), - - ?assertMatch( - disabled, - emqx_eviction_agent:status()), - - {ok, C1} = emqtt_connect(), - ok = emqtt:disconnect(C1). - - -t_evict_connections_status(_Config) -> - erlang:process_flag(trap_exit, true), - - {ok, _C} = emqtt_connect(), - - {error, disabled} = emqx_eviction_agent:evict_connections(1), - - ok = emqx_eviction_agent:enable(test_eviction, undefined), - - ?assertMatch( - {enabled, #{connections := 1, sessions := _}}, - emqx_eviction_agent:status()), - - ok = emqx_eviction_agent:evict_connections(1), - - ct:sleep(100), - - ?assertMatch( - {enabled, #{connections := 0, sessions := _}}, - emqx_eviction_agent:status()), - - ok = emqx_eviction_agent:disable(test_eviction). - - -t_explicit_session_takeover(Config) -> - erlang:process_flag(trap_exit, true), - - {ok, C0} = emqtt_connect(<<"client_with_session">>, false), - {ok, _, _} = emqtt:subscribe(C0, <<"t1">>), - - ok = emqx_eviction_agent:enable(test_eviction, undefined), - - ?assertEqual( - 1, - emqx_eviction_agent:connection_count()), - - ok = emqx_eviction_agent:evict_connections(1), - - receive - {'EXIT', C0, {disconnected, ?RC_USE_ANOTHER_SERVER, _}} -> ok - after 1000 -> - ?assert(false, "Connection not evicted") - end, - - ?assertEqual( - 0, - emqx_eviction_agent:connection_count()), - - ?assertEqual( - 1, - emqx_eviction_agent:session_count()), - - %% First, evacuate to the same node - - ?check_trace( - ?wait_async_action( - emqx_eviction_agent:evict_sessions(1, node()), - #{?snk_kind := emqx_channel_takeover_end}, - 1000), - fun(_Result, Trace) -> - ?assertMatch( - [#{clientid := <<"client_with_session">>} | _ ], - ?of_kind(emqx_channel_takeover_end, Trace)) - end), - - ok = emqx_eviction_agent:disable(test_eviction), - ok = connect_and_publish(<<"t1">>, <<"MessageToEvictedSession1">>), - ok = emqx_eviction_agent:enable(test_eviction, undefined), - - %% Evacuate to another node - - TargetNodeForEvacuation = ?config(evacuate_node, Config), - ?check_trace( - ?wait_async_action( - emqx_eviction_agent:evict_sessions(1, TargetNodeForEvacuation), - #{?snk_kind := emqx_channel_takeover_end}, - 1000), - fun(_Result, Trace) -> - ?assertMatch( - [#{clientid := <<"client_with_session">>} | _ ], - ?of_kind(emqx_channel_takeover_end, Trace)) - end), - - ?assertEqual( - 0, - emqx_eviction_agent:session_count()), - - ?assertEqual( - 1, - rpc:call(TargetNodeForEvacuation, emqx_eviction_agent, session_count, [])), - - ok = emqx_eviction_agent:disable(test_eviction), - - ct:pal("evicted chann info: ~p", [emqx_cm:get_chan_info(<<"client_with_session">>)]), - - ok = connect_and_publish(<<"t1">>, <<"MessageToEvictedSession2">>), - ct:sleep(100), - - {ok, C2} = emqtt_connect(<<"client_with_session">>, false), - - ok = assert_receive_publish( - [#{payload => <<"MessageToEvictedSession1">>, topic => <<"t1">>}, - #{payload => <<"MessageToEvictedSession2">>, topic => <<"t1">>}]), - ok = emqtt:disconnect(C2). - -t_disable_on_restart(_Config) -> - ok = emqx_eviction_agent:enable(test_eviction, undefined), - - ok = supervisor:terminate_child(emqx_eviction_agent_sup, emqx_eviction_agent), - {ok, _} = supervisor:restart_child(emqx_eviction_agent_sup, emqx_eviction_agent), - - ?assertEqual( - disabled, - emqx_eviction_agent:status()). - -assert_receive_publish([]) -> ok; -assert_receive_publish([#{payload := Msg, topic := Topic} | Rest]) -> - receive - {publish, #{payload := Msg, - topic := Topic}} -> - assert_receive_publish(Rest) - after 1000 -> - ?assert(false, "Message `" ++ binary_to_list(Msg) ++ "` is lost") - end. - -connect_and_publish(Topic, Message) -> - {ok, C} = emqtt_connect(), - emqtt:publish(C, Topic, Message), - ok = emqtt:disconnect(C). diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl deleted file mode 100644 index f9585c0e6..000000000 --- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl +++ /dev/null @@ -1,64 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_eviction_agent_api_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - --import(emqx_mgmt_api_test_helpers, - [request_api/3, - auth_header_/0, - api_path/1]). - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_management]), - Config. - -end_per_suite(Config) -> - emqx_ct_helpers:stop_apps([emqx_management, emqx_eviction_agent]), - Config. - -t_status(_Config) -> - ?assertMatch( - {ok, #{<<"status">> := <<"disabled">>}}, - api_get(["node_eviction", "status"])), - - ok = emqx_eviction_agent:enable(apitest, undefined), - - ?assertMatch( - {ok, #{<<"status">> := <<"enabled">>, - <<"stats">> := #{}}}, - api_get(["node_eviction", "status"])), - - ok = emqx_eviction_agent:disable(apitest), - - ?assertMatch( - {ok, #{<<"status">> := <<"disabled">>}}, - api_get(["node_eviction", "status"])). - -api_get(Path) -> - case request_api(get, api_path(Path), auth_header_()) of - {ok, ResponseBody} -> - {ok, jiffy:decode(list_to_binary(ResponseBody), [return_maps])}; - {error, _} = Error -> Error - end. diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl deleted file mode 100644 index cbbf1be5a..000000000 --- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl +++ /dev/null @@ -1,155 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_eviction_agent_channel_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). - --define(CLIENT_ID, <<"client_with_session">>). - --import(emqx_eviction_agent_test_helpers, - [emqtt_connect/2]). - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_eviction_agent]), - Config. - -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_eviction_agent]). - -t_start_no_session(_Config) -> - Opts = #{clientinfo => #{clientid => ?CLIENT_ID, - zone => internal}, - conninfo => #{clientid => ?CLIENT_ID, - receive_maximum => 32}}, - ?assertMatch( - {error, {no_session, _}}, - emqx_eviction_agent_channel:start_supervised(Opts)). - -t_start_no_expire(_Config) -> - erlang:process_flag(trap_exit, true), - - _ = emqtt_connect(?CLIENT_ID, false), - - Opts = #{clientinfo => #{clientid => ?CLIENT_ID, - zone => internal}, - conninfo => #{clientid => ?CLIENT_ID, - receive_maximum => 32, - expiry_interval => 0}}, - ?assertMatch( - {error, {should_be_expired, _}}, - emqx_eviction_agent_channel:start_supervised(Opts)). - -t_start_infinite_expire(_Config) -> - erlang:process_flag(trap_exit, true), - - _ = emqtt_connect(?CLIENT_ID, false), - - Opts = #{clientinfo => #{clientid => ?CLIENT_ID, - zone => internal}, - conninfo => #{clientid => ?CLIENT_ID, - receive_maximum => 32, - expiry_interval => ?UINT_MAX}}, - ?assertMatch( - {ok, _}, - emqx_eviction_agent_channel:start_supervised(Opts)). - -t_kick(_Config) -> - erlang:process_flag(trap_exit, true), - - _ = emqtt_connect(?CLIENT_ID, false), - Opts = evict_session_opts(?CLIENT_ID), - - {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts), - - ?assertEqual( - ok, - emqx_eviction_agent_channel:call(Pid, kick)). - -t_discard(_Config) -> - erlang:process_flag(trap_exit, true), - - _ = emqtt_connect(?CLIENT_ID, false), - Opts = evict_session_opts(?CLIENT_ID), - - {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts), - - ?assertEqual( - ok, - emqx_eviction_agent_channel:call(Pid, discard)). - -t_stop(_Config) -> - erlang:process_flag(trap_exit, true), - - _ = emqtt_connect(?CLIENT_ID, false), - Opts = evict_session_opts(?CLIENT_ID), - - {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts), - - ?assertEqual( - ok, - emqx_eviction_agent_channel:stop(Pid)). - - -t_ignored_calls(_Config) -> - erlang:process_flag(trap_exit, true), - - _ = emqtt_connect(?CLIENT_ID, false), - Opts = evict_session_opts(?CLIENT_ID), - - {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts), - - ok = emqx_eviction_agent_channel:cast(Pid, unknown), - Pid ! unknown, - - ?assertEqual( - [], - emqx_eviction_agent_channel:call(Pid, list_acl_cache)), - - ?assertEqual( - ok, - emqx_eviction_agent_channel:call(Pid, {quota, quota})), - - ?assertEqual( - ignored, - emqx_eviction_agent_channel:call(Pid, unknown)). - -t_expire(_Config) -> - erlang:process_flag(trap_exit, true), - - _ = emqtt_connect(?CLIENT_ID, false), - #{conninfo := ConnInfo} = Opts0 = evict_session_opts(?CLIENT_ID), - Opts1 = Opts0#{conninfo => ConnInfo#{expiry_interval => 1}}, - - {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts1), - - ct:sleep(1500), - - ?assertNot(is_process_alive(Pid)). - -evict_session_opts(ClientId) -> - maps:with( - [conninfo, clientinfo], - emqx_cm:get_chan_info(ClientId)). diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl deleted file mode 100644 index 9c05cdb21..000000000 --- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl +++ /dev/null @@ -1,47 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_eviction_agent_cli_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_eviction_agent]), - Config. - -end_per_suite(Config) -> - _ = emqx_eviction_agent:disable(foo), - emqx_ct_helpers:stop_apps([emqx_eviction_agent]), - Config. - -t_status(_Config) -> - %% usage - ok = emqx_eviction_agent_cli:cli(["foobar"]), - - %% status - ok = emqx_eviction_agent_cli:cli(["status"]), - - ok = emqx_eviction_agent:enable(foo, undefined), - - %% status - ok = emqx_eviction_agent_cli:cli(["status"]). diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl deleted file mode 100644 index 68b86f059..000000000 --- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl +++ /dev/null @@ -1,55 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_eviction_agent_test_helpers). - --export([emqtt_connect/0, - emqtt_connect/2, - emqtt_connect_many/1, - emqtt_try_connect/0]). - -emqtt_connect() -> - emqtt_connect(<<"client1">>, true). - -emqtt_connect(ClientId, CleanStart) -> - {ok, C} = emqtt:start_link( - [{clientid, ClientId}, - {clean_start, CleanStart}, - {proto_ver, v5}, - {properties, #{'Session-Expiry-Interval' => 600}} - ]), - case emqtt:connect(C) of - {ok, _} -> {ok, C}; - {error, _} = Error -> Error - end. - -emqtt_connect_many(Count) -> - lists:map( - fun(N) -> - NBin = integer_to_binary(N), - ClientId = <<"client-", NBin/binary>>, - {ok, C} = emqtt_connect(ClientId, false), - C - end, - lists:seq(1, Count)). - -emqtt_try_connect() -> - case emqtt_connect() of - {ok, C} -> - emqtt:disconnect(C), - ok; - {error, _} = Error -> Error - end. diff --git a/apps/emqx_node_rebalance/.gitignore b/apps/emqx_node_rebalance/.gitignore deleted file mode 100644 index f1c455451..000000000 --- a/apps/emqx_node_rebalance/.gitignore +++ /dev/null @@ -1,19 +0,0 @@ -.rebar3 -_* -.eunit -*.o -*.beam -*.plt -*.swp -*.swo -.erlang.cookie -ebin -log -erl_crash.dump -.rebar -logs -_build -.idea -*.iml -rebar3.crashdump -*~ diff --git a/apps/emqx_node_rebalance/README.md b/apps/emqx_node_rebalance/README.md deleted file mode 100644 index 2e56f62cd..000000000 --- a/apps/emqx_node_rebalance/README.md +++ /dev/null @@ -1,9 +0,0 @@ -emqx_node_rebalance -===== - -An OTP library - -Build ------ - - $ rebar3 compile diff --git a/apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf b/apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf deleted file mode 100644 index 8ace22435..000000000 --- a/apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf +++ /dev/null @@ -1,3 +0,0 @@ -##-------------------------------------------------------------------- -## EMQX Node Rebalance Plugin -##-------------------------------------------------------------------- diff --git a/apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl b/apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl deleted file mode 100644 index 1903b87cc..000000000 --- a/apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl +++ /dev/null @@ -1,31 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --define(DEFAULT_CONN_EVICT_RATE, 500). --define(DEFAULT_SESS_EVICT_RATE, 500). - --define(DEFAULT_WAIT_HEALTH_CHECK, 60). %% sec --define(DEFAULT_WAIT_TAKEOVER, 60). %% sec - --define(DEFAULT_ABS_CONN_THRESHOLD, 1000). --define(DEFAULT_ABS_SESS_THRESHOLD, 1000). - --define(DEFAULT_REL_CONN_THRESHOLD, 1.1). --define(DEFAULT_REL_SESS_THRESHOLD, 1.1). - --define(EVICT_INTERVAL, 1000). - --define(EVACUATION_FILENAME, <<".evacuation">>). diff --git a/apps/emqx_node_rebalance/priv/emqx_node_rebalance.schema b/apps/emqx_node_rebalance/priv/emqx_node_rebalance.schema deleted file mode 100644 index e69de29bb..000000000 diff --git a/apps/emqx_node_rebalance/rebar.config b/apps/emqx_node_rebalance/rebar.config deleted file mode 100644 index 2656fd554..000000000 --- a/apps/emqx_node_rebalance/rebar.config +++ /dev/null @@ -1,2 +0,0 @@ -{erl_opts, [debug_info]}. -{deps, []}. diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src b/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src deleted file mode 100644 index 761cd1d6f..000000000 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src +++ /dev/null @@ -1,19 +0,0 @@ -{application, emqx_node_rebalance, - [{description, "EMQX Node Rebalance"}, - {vsn, "4.3.0"}, - {registered, [emqx_node_rebalance_sup, - emqx_node_rebalance, - emqx_node_rebalance_agent, - emqx_node_rebalance_evacuation]}, - {applications, - [kernel, - stdlib - ]}, - {mod, {emqx_node_rebalance_app,[]}}, - {env,[]}, - {modules, []}, - {maintainers, ["EMQX Team "]}, - {links, [{"Homepage", "https://emqx.io/"}, - {"Github", "https://github.com/emqx"} - ]} - ]}. diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl deleted file mode 100644 index 1edbbbe3b..000000000 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl +++ /dev/null @@ -1,414 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_node_rebalance). - --include("emqx_node_rebalance.hrl"). - --include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/types.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). - --export([start/1, - status/0, - status/1, - stop/0 - ]). - --export([start_link/0]). - --behavior(gen_statem). - --export([init/1, - callback_mode/0, - handle_event/4, - code_change/4 - ]). - --export([is_node_available/0, - available_nodes/1, - connection_count/0, - session_count/0, - disconnected_session_count/0]). - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - --type start_opts() :: #{conn_evict_rate => pos_integer(), - sess_evict_rate => pos_integer(), - wait_health_check => pos_integer(), - wait_takeover => pos_integer(), - abs_conn_threshold => pos_integer(), - rel_conn_threshold => number(), - abs_sess_threshold => pos_integer(), - rel_sess_threshold => number(), - nodes => [node()] - }. --type start_error() :: already_started | [{node(), term()}]. - - --spec start(start_opts()) -> ok_or_error(start_error()). -start(StartOpts) -> - Opts = maps:merge(default_opts(), StartOpts), - gen_statem:call(?MODULE, {start, Opts}). - --spec stop() -> ok_or_error(not_started). -stop() -> - gen_statem:call(?MODULE, stop). - --spec status() -> disabled | {enabled, map()}. -status() -> - gen_statem:call(?MODULE, status). - --spec status(pid()) -> disabled | {enabled, map()}. -status(Pid) -> - gen_statem:call(Pid, status). - --spec start_link() -> startlink_ret(). -start_link() -> - gen_statem:start_link({local, ?MODULE}, ?MODULE, [], []). - --spec available_nodes(list(node())) -> list(node()). -available_nodes(Nodes) when is_list(Nodes) -> - {Available, _} = rpc:multicall(Nodes, ?MODULE, is_node_available, []), - lists:filter(fun is_atom/1, Available). - -%%-------------------------------------------------------------------- -%% gen_statem callbacks -%%-------------------------------------------------------------------- - -callback_mode() -> handle_event_function. - -%% states: disabled, wait_health_check, evicting_conns, wait_takeover, evicting_sessions - -init([]) -> - ?tp(debug, emqx_node_rebalance_started, #{}), - {ok, disabled, #{}}. - -%% start -handle_event({call, From}, - {start, #{wait_health_check := WaitHealthCheck} = Opts}, - disabled, - #{} = Data) -> - case enable_rebalance(Data#{opts => Opts}) of - {ok, NewData} -> - ?LOG(warning, "Node rebalance enabled: ~p", [Opts]), - {next_state, - wait_health_check, - NewData, - [{state_timeout, seconds(WaitHealthCheck), evict_conns}, - {reply, From, ok}]}; - {error, Reason} -> - ?LOG(warning, "Node rebalance enabling failed: ~p", [Reason]), - {keep_state_and_data, - [{reply, From, {error, Reason}}]} - end; -handle_event({call, From}, {start, _Opts}, _State, #{}) -> - {keep_state_and_data, - [{reply, From, {error, already_started}}]}; - -%% stop -handle_event({call, From}, stop, disabled, #{}) -> - {keep_state_and_data, - [{reply, From, {error, not_started}}]}; -handle_event({call, From}, stop, _State, Data) -> - ok = disable_rebalance(Data), - ?LOG(warning, "Node rebalance stopped"), - {next_state, - disabled, - deinit(Data), - [{reply, From, ok}]}; - -%% status -handle_event({call, From}, status, disabled, #{}) -> - {keep_state_and_data, - [{reply, From, disabled}]}; -handle_event({call, From}, status, State, Data) -> - Stats = get_stats(State, Data), - {keep_state_and_data, - [{reply, From, {enabled, Stats#{state => State, - coordinator_node => node()}}}]}; - -%% conn eviction -handle_event(state_timeout, - evict_conns, - wait_health_check, - Data) -> - ?LOG(warning, "Node rebalance wait_health_check over"), - {next_state, - evicting_conns, - Data, - [{state_timeout, 0, evict_conns}]}; - -handle_event(state_timeout, - evict_conns, - evicting_conns, - #{opts := #{wait_takeover := WaitTakeover, - evict_interval := EvictInterval}} = Data) -> - case evict_conns(Data) of - ok -> - ?LOG(warning, "Node rebalance evict_conns over"), - {next_state, - wait_takeover, - Data, - [{state_timeout, seconds(WaitTakeover), evict_sessions}]}; - {continue, NewData} -> - {keep_state, - NewData, - [{state_timeout, EvictInterval, evict_conns}]} - end; - -handle_event(state_timeout, - evict_sessions, - wait_takeover, - Data) -> - ?LOG(warning, "Node rebalance wait_takeover over"), - {next_state, - evicting_sessions, - Data, - [{state_timeout, 0, evict_sessions}]}; - -handle_event(state_timeout, - evict_sessions, - evicting_sessions, - #{opts := #{evict_interval := EvictInterval}} = Data) -> - case evict_sessions(Data) of - ok -> - ?tp(debug, emqx_node_rebalance_evict_sess_over, #{}), - ?LOG(warning, "Node rebalance evict_sess over"), - ok = disable_rebalance(Data), - ?LOG(warning, "Rebalance finished successfully"), - {next_state, - disabled, - deinit(Data)}; - {continue, NewData} -> - {keep_state, - NewData, - [{state_timeout, EvictInterval, evict_sessions}]} - end; - -handle_event({call, From}, Msg, State, Data) -> - ?LOG(warning, "Unknown call: ~p, State: ~p, Data: ~p", [Msg, State, Data]), - {keep_state_and_data, - [{reply, From, ignored}]}; - -handle_event(info, Msg, State, Data) -> - ?LOG(warning, "Unknown Msg: ~p, State: ~p, Data: ~p", [Msg, State, Data]), - keep_state_and_data; - -handle_event(cast, Msg, State, Data) -> - ?LOG(warning, "Unknown cast Msg: ~p, State: ~p, Data: ~p", [Msg, State, Data]), - keep_state_and_data. - -code_change(_Vsn, State, Data, _Extra) -> - {ok, State, Data}. - -%%-------------------------------------------------------------------- -%% internal funs -%%-------------------------------------------------------------------- - -enable_rebalance(#{opts := Opts} = Data) -> - Nodes = maps:get(nodes, Opts), - ConnCounts = multicall(Nodes, {?MODULE, connection_count, []}), - SessCounts = multicall(Nodes, {?MODULE, session_count, []}), - {_, Counts} = lists:unzip(ConnCounts), - Avg = avg(Counts), - {DonorCounts, - RecipientCounts} = lists:partition( - fun({_Node, Count}) -> - Count >= Avg - end, - ConnCounts), - ?LOG(warning, "Enabling rebalance: ConnCounts=~p, DonorCounts=~p, RecipientCounts=~p", - [ConnCounts, DonorCounts, RecipientCounts]), - {DonorNodes, _} = lists:unzip(DonorCounts), - {RecipientNodes, _} = lists:unzip(RecipientCounts), - case need_rebalance(DonorNodes, RecipientNodes, ConnCounts, SessCounts, Opts) of - false -> {error, nothing_to_balance}; - true -> - _ = multicall(DonorNodes, {emqx_node_rebalance_agent, enable, [self()]}), - {ok, Data#{donors => DonorNodes, - recipients => RecipientNodes, - initial_conn_counts => maps:from_list(ConnCounts), - initial_sess_counts => maps:from_list(SessCounts)}} - end. - -disable_rebalance(#{donors := DonorNodes}) -> - _ = multicall(DonorNodes, {emqx_node_rebalance_agent, disable, [self()]}), - ok. - -evict_conns(#{donors := DonorNodes, recipients := RecipientNodes, opts := Opts} = Data) -> - DonorNodeCounts = multicall(DonorNodes, {?MODULE, connection_count, []}), - {_, DonorCounts} = lists:unzip(DonorNodeCounts), - RecipientNodeCounts = multicall(RecipientNodes, {?MODULE, connection_count, []}), - {_, RecipientCounts} = lists:unzip(RecipientNodeCounts), - - DonorAvg = avg(DonorCounts), - RecipientAvg = avg(RecipientCounts), - Thresholds = thresholds(conn, Opts), - NewData = Data#{donor_conn_avg => DonorAvg, - recipient_conn_avg => RecipientAvg, - donor_conn_counts => maps:from_list(DonorNodeCounts), - recipient_conn_counts => maps:from_list(RecipientNodeCounts)}, - case within_thresholds(DonorAvg, RecipientAvg, Thresholds) of - true -> ok; - false -> - ConnEvictRate = maps:get(conn_evict_rate, Opts), - NodesToEvict = nodes_to_evict(RecipientAvg, DonorNodeCounts), - ?LOG(warning, "Node rebalance, evict_conns, nodes=~p, counts=~p", - [NodesToEvict, ConnEvictRate]), - _ = multicall(NodesToEvict, {emqx_eviction_agent, evict_connections, [ConnEvictRate]}), - {continue, NewData} - end. - -evict_sessions(#{donors := DonorNodes, recipients := RecipientNodes, opts := Opts} = Data) -> - DonorNodeCounts = multicall(DonorNodes, {?MODULE, disconnected_session_count, []}), - {_, DonorCounts} = lists:unzip(DonorNodeCounts), - RecipientNodeCounts = multicall(RecipientNodes, {?MODULE, disconnected_session_count, []}), - {_, RecipientCounts} = lists:unzip(RecipientNodeCounts), - - DonorAvg = avg(DonorCounts), - RecipientAvg = avg(RecipientCounts), - Thresholds = thresholds(sess, Opts), - NewData = Data#{donor_sess_avg => DonorAvg, - recipient_sess_avg => RecipientAvg, - donor_sess_counts => maps:from_list(DonorNodeCounts), - recipient_sess_counts => maps:from_list(RecipientNodeCounts)}, - case within_thresholds(DonorAvg, RecipientAvg, Thresholds) of - true -> ok; - false -> - SessEvictRate = maps:get(sess_evict_rate, Opts), - NodesToEvict = nodes_to_evict(RecipientAvg, DonorNodeCounts), - ?LOG(warning, "Node rebalance, evict_sessions, nodes=~p, counts=~p", - [NodesToEvict, SessEvictRate]), - _ = multicall(NodesToEvict, - {emqx_eviction_agent, - evict_sessions, - [SessEvictRate, RecipientNodes, disconnected]}), - {continue, NewData} - end. - -need_rebalance([] = _DonorNodes, _RecipientNodes, _ConnCounts, _SessCounts, _Opts) -> false; -need_rebalance(_DonorNodes, [] = _RecipientNodes, _ConnCounts, _SessCounts, _Opts) -> false; -need_rebalance(DonorNodes, RecipientNodes, ConnCounts, SessCounts, Opts) -> - DonorConnAvg = avg_for_nodes(DonorNodes, ConnCounts), - RecipientConnAvg = avg_for_nodes(RecipientNodes, ConnCounts), - DonorSessAvg = avg_for_nodes(DonorNodes, SessCounts), - RecipientSessAvg = avg_for_nodes(RecipientNodes, SessCounts), - Result = (not within_thresholds(DonorConnAvg, RecipientConnAvg, thresholds(conn, Opts))) - orelse (not within_thresholds(DonorSessAvg, RecipientSessAvg, thresholds(sess, Opts))), - ?tp(debug, emqx_node_rebalance_need_rebalance, - #{donors => DonorNodes, - recipients => RecipientNodes, - conn_counts => ConnCounts, - sess_counts => SessCounts, - opts => Opts, - result => Result - }), - Result. - -avg_for_nodes(Nodes, Counts) -> - avg(maps:values(maps:with(Nodes, maps:from_list(Counts)))). - -within_thresholds(Value, GoalValue, {AbsThres, RelThres}) -> - (Value =< GoalValue + AbsThres) orelse (Value =< GoalValue * RelThres). - -thresholds(conn, #{abs_conn_threshold := Abs, rel_conn_threshold := Rel}) -> - {Abs, Rel}; -thresholds(sess, #{abs_sess_threshold := Abs, rel_sess_threshold := Rel}) -> - {Abs, Rel}. - -nodes_to_evict(Goal, NodeCounts) -> - {Nodes, _} = lists:unzip( - lists:filter( - fun({_Node, Count}) -> - Count > Goal - end, - NodeCounts)), - Nodes. - -get_stats(disabled, _Data) -> #{}; -get_stats(_State, Data) -> Data. - -avg(List) when length(List) >= 1 -> - lists:sum(List) / length(List). - -multicall(Nodes, {M, F, A}) -> - case rpc:multicall(Nodes, M, F, A) of - {Results, []} -> - case lists:partition(fun is_ok/1, lists:zip(Nodes, Results)) of - {OkResults, []} -> - [{Node, ok_result(Result)} || {Node, Result} <- OkResults]; - {_, BadResults} -> - error({bad_nodes, BadResults}) - end; - {_, [_BadNode | _] = BadNodes} -> - error({bad_nodes, BadNodes}) - end. - -is_ok({_Node, {ok, _}}) -> true; -is_ok({_Node, ok}) -> true; -is_ok(_) -> false. - -ok_result({ok, Result}) -> Result; -ok_result(ok) -> ok. - -connection_count() -> - {ok, emqx_eviction_agent:connection_count()}. - -session_count() -> - {ok, emqx_eviction_agent:session_count()}. - -disconnected_session_count() -> - {ok, emqx_eviction_agent:session_count(disconnected)}. - -default_opts() -> - #{ - conn_evict_rate => ?DEFAULT_CONN_EVICT_RATE, - abs_conn_threshold => ?DEFAULT_ABS_CONN_THRESHOLD, - rel_conn_threshold => ?DEFAULT_REL_CONN_THRESHOLD, - - sess_evict_rate => ?DEFAULT_SESS_EVICT_RATE, - abs_sess_threshold => ?DEFAULT_ABS_SESS_THRESHOLD, - rel_sess_threshold => ?DEFAULT_REL_SESS_THRESHOLD, - - wait_health_check => ?DEFAULT_WAIT_HEALTH_CHECK, - wait_takeover => ?DEFAULT_WAIT_TAKEOVER, - - evict_interval => ?EVICT_INTERVAL, - - nodes => all_nodes() - }. - - -deinit(Data) -> - Keys = [recipient_conn_avg, recipient_sess_avg, donor_conn_avg, donor_sess_avg, - recipient_conn_counts, recipient_sess_counts, donor_conn_counts, donor_sess_counts, - initial_conn_counts, initial_sess_counts, - opts], - maps:without(Keys, Data). - -is_node_available() -> - true = is_pid(whereis(emqx_node_rebalance_agent)), - disabled = emqx_eviction_agent:status(), - node(). - -all_nodes() -> - ekka_mnesia:cluster_nodes(all). - -seconds(Sec) -> - round(timer:seconds(Sec)). diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl deleted file mode 100644 index ade043d6b..000000000 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl +++ /dev/null @@ -1,127 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_node_rebalance_agent). - --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/types.hrl"). - --include_lib("stdlib/include/qlc.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). - --export([start_link/0, - enable/1, - disable/1, - status/0 - ]). - --export([init/1, - handle_call/3, - handle_info/2, - handle_cast/2, - code_change/3 - ]). - --define(ENABLE_KIND, emqx_node_rebalance). - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - --type status() :: {enabled, pid()} | disabled. - --spec start_link() -> startlink_ret(). -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - --spec enable(pid()) -> ok_or_error(already_enabled | eviction_agent_busy). -enable(CoordinatorPid) -> - gen_server:call(?MODULE, {enable, CoordinatorPid}). - --spec disable(pid()) -> ok_or_error(already_disabled | invalid_coordinator). -disable(CoordinatorPid) -> - gen_server:call(?MODULE, {disable, CoordinatorPid}). - --spec status() -> status(). -status() -> - gen_server:call(?MODULE, status). - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([]) -> - {ok, #{}}. - -handle_call({enable, CoordinatorPid}, _From, St) -> - case St of - #{coordinator_pid := _Pid} -> - {reply, {error, already_enabled}, St}; - _ -> - true = link(CoordinatorPid), - EvictionAgentPid = whereis(emqx_eviction_agent), - true = link(EvictionAgentPid), - case emqx_eviction_agent:enable(?ENABLE_KIND, undefined) of - ok -> - {reply, ok, #{coordinator_pid => CoordinatorPid, - eviction_agent_pid => EvictionAgentPid}}; - {error, eviction_agent_busy} -> - true = unlink(EvictionAgentPid), - true = unlink(CoordinatorPid), - {reply, {error, eviction_agent_busy}, St} - end - end; - -handle_call({disable, CoordinatorPid}, _From, St) -> - case St of - #{coordinator_pid := CoordinatorPid, - eviction_agent_pid := EvictionAgentPid} -> - _ = emqx_eviction_agent:disable(?ENABLE_KIND), - true = unlink(EvictionAgentPid), - true = unlink(CoordinatorPid), - NewSt = maps:without( - [coordinator_pid, eviction_agent_pid], - St), - {reply, ok, NewSt}; - #{coordinator_pid := _CoordinatorPid} -> - {reply, {error, invalid_coordinator}, St}; - #{} -> - {reply, {error, already_disabled}, St} - end; - -handle_call(status, _From, St) -> - case St of - #{coordinator_pid := Pid} -> - {reply, {enabled, Pid}, St}; - _ -> - {reply, disabled, St} - end; - -handle_call(Msg, _From, St) -> - ?LOG(warning, "Unknown call: ~p, State: ~p", [Msg, St]), - {reply, ignored, St}. - -handle_info(Msg, St) -> - ?LOG(warning, "Unknown Msg: ~p, State: ~p", [Msg, St]), - {noreply, St}. - -handle_cast(Msg, St) -> - ?LOG(warning, "Unknown cast Msg: ~p, State: ~p", [Msg, St]), - {noreply, St}. - -code_change(_Vsn, State, _Extra) -> - {ok, State}. diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl deleted file mode 100644 index 6ba221cd8..000000000 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl +++ /dev/null @@ -1,243 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_node_rebalance_api). - --import(minirest, [return/1]). - --rest_api(#{name => load_rebalance_status, - method => 'GET', - path => "/load_rebalance/status", - func => status, - descr => "Get load rebalance status"}). - --rest_api(#{name => load_rebalance_global_status, - method => 'GET', - path => "/load_rebalance/global_status", - func => global_status, - descr => "Get status of all rebalance/evacuation processes across the cluster"}). - --rest_api(#{name => load_rebalance_availability_check, - method => 'GET', - path => "/load_rebalance/availability_check", - func => availability_check, - descr => "Node rebalance availability check"}). - --rest_api(#{name => load_rebalance_start, - method => 'POST', - path => "/load_rebalance/:bin:node/start", - func => rebalance_start, - descr => "Start rebalancing with the node as coordinator"}). - --rest_api(#{name => load_rebalance_stop, - method => 'POST', - path => "/load_rebalance/:bin:node/stop", - func => rebalance_stop, - descr => "Stop rebalancing coordinated by the node"}). - --rest_api(#{name => load_rebalance_evacuation_start, - method => 'POST', - path => "/load_rebalance/:bin:node/evacuation/start", - func => rebalance_evacuation_start, - descr => "Start evacuation on a node "}). - --rest_api(#{name => load_rebalance_evacuation_stop, - method => 'POST', - path => "/load_rebalance/:bin:node/evacuation/stop", - func => rebalance_evacuation_stop, - descr => "Stop evacuation on the node"}). - --export([status/2, - availability_check/2, - global_status/2, - - rebalance_evacuation_start/2, - rebalance_evacuation_stop/2, - rebalance_start/2, - rebalance_stop/2 - ]). - -status(_Bindings, _Params) -> - case emqx_node_rebalance_status:local_status() of - disabled -> - {ok, #{status => disabled}}; - {rebalance, Stats} -> - {ok, format_status(rebalance, Stats)}; - {evacuation, Stats} -> - {ok, format_status(evacuation, Stats)} - end. - -global_status(_Bindings, _Params) -> - #{evacuations := Evacuations, - rebalances := Rebalances} = emqx_node_rebalance_status:global_status(), - {ok, #{evacuations => maps:from_list(Evacuations), - rebalances => maps:from_list(Rebalances)}}. - -availability_check(_Bindings, _Params) -> - case emqx_eviction_agent:status() of - disabled -> - {200, #{}}; - {enabled, _Stats} -> - {503, #{}} - end. - -rebalance_evacuation_start(#{node := NodeBin}, Params) -> - validated( - fun() -> - {Node, Opts} = validate_evacuation(NodeBin, params(Params)), - rpc(Node, emqx_node_rebalance_evacuation, start, [Opts]) - end). - -rebalance_evacuation_stop(#{node := NodeBin}, _Params) -> - validated( - fun() -> - Node = parse_node(NodeBin), - rpc(Node, emqx_node_rebalance_evacuation, stop, []) - end). - -rebalance_start(#{node := NodeBin}, Params) -> - validated( - fun() -> - {Node, Opts} = validate_rebalance(NodeBin, params(Params)), - rpc(Node, emqx_node_rebalance, start, [Opts]) - end). - -rebalance_stop(#{node := NodeBin}, _Params) -> - validated( - fun() -> - Node = parse_node(NodeBin), - rpc(Node, emqx_node_rebalance, stop, []) - end). - -rpc(Node, M, F, A) -> - case rpc:call(Node, M, F, A) of - ok -> return({ok, []}); - {error, Error} -> - return({error, 400, io_lib:format("~p", [Error])}); - {badrpc, _} -> - return({error, 400, io_lib:format("Error communicating with node ~p", [Node])}); - Unknown -> - return({error, 400, io_lib:format("Unrecognized rpc result from node ~p: ~p", - [Node, Unknown])}) - end. - -format_status(Process, Stats) -> - Stats#{process => Process, status => enabled}. - -validate_evacuation(Node, Params) -> - NodeToEvacuate = parse_node(Node), - OptList = lists:map( - fun validate_evacuation_param/1, - Params), - {NodeToEvacuate, maps:from_list(OptList)}. - -validate_rebalance(Node, Params) -> - CoordinatorNode = parse_node(Node), - OptList = lists:map( - fun validate_rebalance_param/1, - Params), - {CoordinatorNode, maps:from_list(OptList)}. - -validate_evacuation_param({<<"conn_evict_rate">>, Value}) -> - validate_pos_int(conn_evict_rate, Value); -validate_evacuation_param({<<"sess_evict_rate">>, Value}) -> - validate_pos_int(sess_evict_rate, Value); -validate_evacuation_param({<<"redirect_to">>, Value}) -> - validate_binary(server_reference, Value); -validate_evacuation_param({<<"wait_takeover">>, Value}) -> - validate_pos_int(wait_takeover, Value); -validate_evacuation_param({<<"migrate_to">>, Value}) -> - validate_nodes(migrate_to, Value); -validate_evacuation_param(Value) -> - validation_error(io_lib:format("Unknown evacuation param: ~p", [Value])). - -validate_rebalance_param({<<"wait_health_check">>, Value}) -> - validate_pos_int(wait_health_check, Value); -validate_rebalance_param({<<"conn_evict_rate">>, Value}) -> - validate_pos_int(conn_evict_rate, Value); -validate_rebalance_param({<<"sess_evict_rate">>, Value}) -> - validate_pos_int(sess_evict_rate, Value); -validate_rebalance_param({<<"abs_conn_threshold">>, Value}) -> - validate_pos_int(abs_conn_threshold, Value); -validate_rebalance_param({<<"rel_conn_threshold">>, Value}) -> - validate_fraction(rel_conn_threshold, Value); -validate_rebalance_param({<<"abs_sess_threshold">>, Value}) -> - validate_pos_int(abs_sess_threshold, Value); -validate_rebalance_param({<<"rel_sess_threshold">>, Value}) -> - validate_fraction(rel_sess_threshold, Value); -validate_rebalance_param({<<"wait_takeover">>, Value}) -> - validate_pos_int(wait_takeover, Value); -validate_rebalance_param({<<"nodes">>, Value}) -> - validate_nodes(nodes, Value); -validate_rebalance_param(Value) -> - validation_error(io_lib:format("Unknown rebalance param: ~p", [Value])). - -validate_binary(Name, Value) when is_binary(Value) -> - {Name, Value}; -validate_binary(Name, _Value) -> - validation_error("invalid string in " ++ atom_to_list(Name)). - -validate_pos_int(Name, Value) -> - case is_integer(Value) andalso Value > 0 of - true -> {Name, Value}; - false -> - validation_error("invalid " ++ atom_to_list(Name) ++ " value") - end. - -validate_fraction(Name, Value) -> - case is_number(Value) andalso Value > 1.0 of - true -> {Name, Value}; - false -> - validation_error("invalid " ++ atom_to_list(Name) ++ " value") - end. - -validate_nodes(Name, NodeList) when is_list(NodeList) -> - Nodes = lists:map( - fun parse_node/1, - NodeList), - case emqx_node_rebalance_evacuation:available_nodes(Nodes) of - [] -> - validation_error(io_lib:format("no available nodes list in ~p: ~p", [Name, Nodes])); - Nodes -> - {Name, Nodes}; - OtherNodes -> - validation_error( - io_lib:format("unavailable nodes in ~p: ~p", - [Name, Nodes -- OtherNodes])) - end; -validate_nodes(Name, Nodes) -> - validation_error(io_lib:format("invalid node list in ~p: ~p", [Name, Nodes])). - -validated(Fun) -> - try - Fun() - catch throw:{validation_error, Error} -> - return({error, 400, iolist_to_binary(Error)}) - end. - -validation_error(Error) -> - throw({validation_error, Error}). - -parse_node(Bin) when is_binary(Bin) -> - try - binary_to_existing_atom(Bin) - catch - error:badarg -> - validation_error("invalid node: " ++ [Bin]) - end. - -params([{}]) -> []; -params(Params) -> Params. diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl deleted file mode 100644 index 3cba331de..000000000 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl +++ /dev/null @@ -1,34 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_node_rebalance_app). - --behaviour(application). - --emqx_plugin(?MODULE). - --export([ start/2 - , stop/1 - ]). - -start(_Type, _Args) -> - Env = application:get_all_env(emqx_node_rebalance), - {ok, Sup} = emqx_node_rebalance_sup:start_link(Env), - ok = emqx_node_rebalance_cli:load(), - {ok, Sup}. - -stop(_State) -> - emqx_node_rebalance_cli:unload(). diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl deleted file mode 100644 index 10d0912c4..000000000 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl +++ /dev/null @@ -1,265 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_node_rebalance_cli). - -%% APIs --export([ load/0 - , unload/0 - , cli/1 - ]). - -load() -> - emqx_ctl:register_command(rebalance, {?MODULE, cli}, []). - -unload() -> - emqx_ctl:unregister_command(rebalance). - -cli(["start" | StartArgs]) -> - case start_args(StartArgs) of - {evacuation, Opts} -> - case emqx_node_rebalance_evacuation:status() of - disabled -> - ok = emqx_node_rebalance_evacuation:start(Opts), - emqx_ctl:print("Rebalance(evacuation) started~n"), - true; - {enabled, _} -> - emqx_ctl:print("Rebalance is already enabled~n"), - false - end; - {rebalance, Opts} -> - case emqx_node_rebalance:start(Opts) of - ok -> - emqx_ctl:print("Rebalance started~n"), - true; - {error, Reason} -> - emqx_ctl:print("Rebalance start error: ~p~n", [Reason]), - false - end; - {error, Error} -> - emqx_ctl:print("Rebalance start error: ~s~n", [Error]), - false - end; -cli(["node-status", NodeStr]) -> - Node = list_to_atom(NodeStr), - node_status(emqx_node_rebalance_status:local_status(Node)); -cli(["node-status"]) -> - node_status(emqx_node_rebalance_status:local_status()); -cli(["status"]) -> - #{evacuations := Evacuations, - rebalances := Rebalances} = emqx_node_rebalance_status:global_status(), - lists:foreach( - fun({Node, Status}) -> - emqx_ctl:print("--------------------------------------------------------------------~n"), - emqx_ctl:print("Node ~p: evacuation~n~s", - [Node, emqx_node_rebalance_status:format_local_status(Status)]) - end, - Evacuations), - lists:foreach( - fun({Node, Status}) -> - emqx_ctl:print("--------------------------------------------------------------------~n"), - emqx_ctl:print("Node ~p: rebalance coordinator~n~s", - [Node, emqx_node_rebalance_status:format_coordinator_status(Status)]) - end, - Rebalances); -cli(["stop"]) -> - case emqx_node_rebalance_evacuation:status() of - {enabled, _} -> - ok = emqx_node_rebalance_evacuation:stop(), - emqx_ctl:print("Rebalance(evacuation) stopped~n"), - true; - disabled -> - case emqx_node_rebalance:status() of - {enabled, _} -> - ok = emqx_node_rebalance:stop(), - emqx_ctl:print("Rebalance stopped~n"), - true; - disabled -> - emqx_ctl:print("Rebalance is already disabled~n"), - false - end - end; -cli(_) -> - emqx_ctl:usage( - [{"rebalance start --evacuation \\\n" - " [--redirect-to \"Host1:Port1 Host2:Port2 ...\"] \\\n" - " [--conn-evict-rate CountPerSec] \\\n" - " [--migrate-to \"node1@host1 node2@host2 ...\"] \\\n" - " [--wait-takeover Secs] \\\n" - " [--sess-evict-rate CountPerSec]", - "Start current node evacuation with optional server redirect to the specified servers"}, - - {"rebalance start \\\n" - " [--nodes \"node1@host1 node2@host2\"] \\\n" - " [--wait-health-check Secs] \\\n" - " [--conn-evict-rate ConnPerSec] \\\n" - " [--abs-conn-threshold Count] \\\n" - " [--rel-conn-threshold Fraction] \\\n" - " [--conn-evict-rate ConnPerSec] \\\n" - " [--wait-takeover Secs] \\\n" - " [--sess-evict-rate CountPerSec] \\\n" - " [--abs-sess-threshold Count] \\\n" - " [--rel-sess-threshold Fraction]", - "Start current node evacuation with optional server redirect to the specified servers"}, - - {"rebalance node-status", - "Get current node rebalance status"}, - - {"rebalance node-status \"node1@host1\"", - "Get remote node rebalance status"}, - - {"rebalance status", - "Get statuses of all current rebalance/evacuation processes across the cluster"}, - - {"rebalance stop", - "Stop node rebalance"}]). - -node_status(NodeStatus) -> - case NodeStatus of - {Process, Status} when Process =:= evacuation orelse Process =:= rebalance -> - emqx_ctl:print("Rebalance type: ~p~n~s~n", - [Process, emqx_node_rebalance_status:format_local_status(Status)]); - disabled -> - emqx_ctl:print("Rebalance disabled~n"); - Other -> - emqx_ctl:print("Error detecting rebalance status: ~p~n", [Other]) - end. - -start_args(Args) -> - case collect_args(Args, #{}) of - {ok, #{"--evacuation" := true} = Collected} -> - case validate_evacuation(maps:to_list(Collected), #{}) of - {ok, Validated} -> - {evacuation, Validated}; - {error, _} = Error -> Error - end; - {ok, #{} = Collected} -> - case validate_rebalance(maps:to_list(Collected), #{}) of - {ok, Validated} -> - {rebalance, Validated}; - {error, _} = Error -> Error - end; - {error, _} = Error -> Error - end. - -collect_args([], Map) -> {ok, Map}; - -%% evacuation -collect_args(["--evacuation" | Args], Map) -> - collect_args(Args, Map#{"--evacuation" => true}); -collect_args(["--redirect-to", ServerReference | Args], Map) -> - collect_args(Args, Map#{"--redirect-to" => ServerReference}); -collect_args(["--migrate-to", MigrateTo | Args], Map) -> - collect_args(Args, Map#{"--migrate-to" => MigrateTo}); -%% rebalance -collect_args(["--nodes", Nodes | Args], Map) -> - collect_args(Args, Map#{"--nodes" => Nodes}); -collect_args(["--wait-health-check", WaitHealthCheck | Args], Map) -> - collect_args(Args, Map#{"--wait-health-check" => WaitHealthCheck}); -collect_args(["--abs-conn-threshold", AbsConnThres | Args], Map) -> - collect_args(Args, Map#{"--abs-conn-threshold" => AbsConnThres}); -collect_args(["--rel-conn-threshold", RelConnThres | Args], Map) -> - collect_args(Args, Map#{"--rel-conn-threshold" => RelConnThres}); -collect_args(["--abs-sess-threshold", AbsSessThres | Args], Map) -> - collect_args(Args, Map#{"--abs-sess-threshold" => AbsSessThres}); -collect_args(["--rel-sess-threshold", RelSessThres | Args], Map) -> - collect_args(Args, Map#{"--rel-sess-threshold" => RelSessThres}); -%% common -collect_args(["--conn-evict-rate", ConnEvictRate | Args], Map) -> - collect_args(Args, Map#{"--conn-evict-rate" => ConnEvictRate}); -collect_args(["--wait-takeover", WaitTakeover | Args], Map) -> - collect_args(Args, Map#{"--wait-takeover" => WaitTakeover}); -collect_args(["--sess-evict-rate", SessEvictRate | Args], Map) -> - collect_args(Args, Map#{"--sess-evict-rate" => SessEvictRate}); -%% fallback -collect_args(Args, _Map) -> - {error, io_lib:format("unknown arguments: ~p", [Args])}. - -validate_evacuation([], Map) -> - {ok, Map}; -validate_evacuation([{"--evacuation", _} | Rest], Map) -> - validate_evacuation(Rest, Map); -validate_evacuation([{"--redirect-to", ServerReference} | Rest], Map) -> - validate_evacuation(Rest, Map#{server_reference => list_to_binary(ServerReference)}); -validate_evacuation([{"--conn-evict-rate", _} | _] = Opts, Map) -> - validate_pos_int(conn_evict_rate, Opts, Map, fun validate_evacuation/2); -validate_evacuation([{"--sess-evict-rate", _} | _] = Opts, Map) -> - validate_pos_int(sess_evict_rate, Opts, Map, fun validate_evacuation/2); -validate_evacuation([{"--wait-takeover", _} | _] = Opts, Map) -> - validate_pos_int(wait_takeover, Opts, Map, fun validate_evacuation/2); -validate_evacuation([{"--migrate-to", MigrateTo} | Rest], Map) -> - Nodes = lists:map(fun list_to_atom/1, string:tokens(MigrateTo, ", ")), - case emqx_node_rebalance_evacuation:available_nodes(Nodes) of - [] -> - {error, "invalid --migrate-to, no nodes"}; - Nodes -> - validate_evacuation(Rest, Map#{migrate_to => Nodes}); - OtherNodes -> - {error, - io_lib:format("invalid --migrate-to, unavailable nodes: ~p", - [Nodes -- OtherNodes])} - end; -validate_evacuation(Rest, _Map) -> - {error, io_lib:format("unknown evacuation arguments: ~p", [Rest])}. - -validate_rebalance([], Map) -> - {ok, Map}; -validate_rebalance([{"--wait-health-check", _} | _] = Opts, Map) -> - validate_pos_int(wait_health_check, Opts, Map, fun validate_rebalance/2); -validate_rebalance([{"--conn-evict-rate", _} | _] = Opts, Map) -> - validate_pos_int(conn_evict_rate, Opts, Map, fun validate_rebalance/2); -validate_rebalance([{"--sess-evict-rate", _} | _] = Opts, Map) -> - validate_pos_int(sess_evict_rate, Opts, Map, fun validate_rebalance/2); -validate_rebalance([{"--abs-conn-threshold", _} | _] = Opts, Map) -> - validate_pos_int(abs_conn_threshold, Opts, Map, fun validate_rebalance/2); -validate_rebalance([{"--rel-conn-threshold", _} | _] = Opts, Map) -> - validate_fraction(rel_conn_threshold, Opts, Map, fun validate_rebalance/2); -validate_rebalance([{"--abs-sess-threshold", _} | _] = Opts, Map) -> - validate_pos_int(abs_sess_threshold, Opts, Map, fun validate_rebalance/2); -validate_rebalance([{"--rel-sess-threshold", _} | _] = Opts, Map) -> - validate_fraction(rel_sess_threshold, Opts, Map, fun validate_rebalance/2); -validate_rebalance([{"--wait-takeover", _} | _] = Opts, Map) -> - validate_pos_int(wait_takeover, Opts, Map, fun validate_rebalance/2); -validate_rebalance([{"--nodes", NodeStr} | Rest], Map) -> - Nodes = lists:map(fun list_to_atom/1, string:tokens(NodeStr, ", ")), - case emqx_node_rebalance:available_nodes(Nodes) of - [] -> - {error, "invalid --nodes, no nodes"}; - Nodes -> - validate_rebalance(Rest, Map#{nodes => Nodes}); - OtherNodes -> - {error, - io_lib:format("invalid --nodes, unavailable nodes: ~p", - [Nodes -- OtherNodes])} - end; -validate_rebalance(Rest, _Map) -> - {error, io_lib:format("unknown rebalance arguments: ~p", [Rest])}. - -validate_fraction(Name, [{OptionName, Value} | Rest], Map, Next) -> - case string:to_float(Value) of - {Num, ""} when Num > 1.0 -> - Next(Rest, Map#{Name => Num}); - _ -> - {error, "invalid " ++ OptionName ++ " value"} - end. - -validate_pos_int(Name, [{OptionName, Value} | Rest], Map, Next) -> - case string:to_integer(Value) of - {Int, ""} when Int > 0 -> - Next(Rest, Map#{Name => Int}); - _ -> - {error, "invalid " ++ OptionName ++ " value"} - end. diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl deleted file mode 100644 index 5dc99d736..000000000 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl +++ /dev/null @@ -1,298 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_node_rebalance_evacuation). - --include("emqx_node_rebalance.hrl"). - --include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/types.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). - --export([start/1, - status/0, - stop/0 - ]). - --export([start_link/0]). - --behavior(gen_statem). - --export([init/1, - callback_mode/0, - handle_event/4, - code_change/4 - ]). - --export([is_node_available/0, - available_nodes/1]). - --ifdef(TEST). --export([migrate_to/1]). --endif. - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - --define(EVICT_INTERVAL_NO_NODES, 30000). - --type migrate_to() :: [node()] | undefined. - --type start_opts() :: #{server_reference => emqx_eviction_agent:server_reference(), - conn_evict_rate => pos_integer(), - sess_evict_rate => pos_integer(), - wait_takeover => pos_integer(), - migrate_to => migrate_to() - }. --type start_error() :: already_started | eviction_agent_busy. --type stats() :: #{ - initial_conns := non_neg_integer(), - initial_sessions := non_neg_integer(), - current_conns := non_neg_integer(), - current_sessions := non_neg_integer(), - conn_evict_rate := pos_integer(), - sess_evict_rate := pos_integer(), - server_reference := emqx_eviction_agent:server_reference(), - migrate_to := migrate_to() - }. --type status() :: {started, stats()} | stopped. - --spec start(start_opts()) -> ok_or_error(start_error()). -start(StartOpts) -> - Opts = maps:merge(default_opts(), StartOpts), - gen_statem:call(?MODULE, {start, Opts}). - --spec stop() -> ok_or_error(not_started). -stop() -> - gen_statem:call(?MODULE, stop). - --spec status() -> status(). -status() -> - gen_statem:call(?MODULE, status). - --spec start_link() -> startlink_ret(). -start_link() -> - gen_statem:start_link({local, ?MODULE}, ?MODULE, [], []). - --spec available_nodes(list(node())) -> list(node()). -available_nodes(Nodes) when is_list(Nodes) -> - {Available, _} = rpc:multicall(Nodes, ?MODULE, is_node_available, []), - lists:filter(fun is_atom/1, Available). - -%%-------------------------------------------------------------------- -%% gen_statem callbacks -%%-------------------------------------------------------------------- - -callback_mode() -> handle_event_function. - -%% states: disabled, evicting_conns, waiting_takeover, evicting_sessions, prohibiting - -init([]) -> - case emqx_node_rebalance_evacuation_persist:read(default_opts()) of - {ok, #{server_reference := ServerReference} = Opts} -> - ?LOG(warning, "Restoring evacuation state: ~p", [Opts]), - case emqx_eviction_agent:enable(?MODULE, ServerReference) of - ok -> - Data = init_data(#{}, Opts), - ok = warn_enabled(), - {ok, evicting_conns, Data, [{state_timeout, 0, evict_conns}]}; - {error, eviction_agent_busy} -> - emqx_node_rebalance_evacuation_persist:clear(), - {ok, disabled, #{}} - end; - none -> - {ok, disabled, #{}} - end. - -%% start -handle_event({call, From}, - {start, #{server_reference := ServerReference} = Opts}, - disabled, - #{} = Data) -> - case emqx_eviction_agent:enable(?MODULE, ServerReference) of - ok -> - NewData = init_data(Data, Opts), - ok = emqx_node_rebalance_evacuation_persist:save(Opts), - ?LOG(warning, "Node evacuation started"), - {next_state, - evicting_conns, - NewData, - [{state_timeout, 0, evict_conns}, - {reply, From, ok}]}; - {error, eviction_agent_busy} -> - {keep_state_and_data, - [{reply, From, {error, eviction_agent_busy}}]} - end; -handle_event({call, From}, {start, _Opts}, _State, #{}) -> - {keep_state_and_data, - [{reply, From, {error, already_started}}]}; - -%% stop -handle_event({call, From}, stop, disabled, #{}) -> - {keep_state_and_data, - [{reply, From, {error, not_started}}]}; -handle_event({call, From}, stop, _State, Data) -> - ok = emqx_node_rebalance_evacuation_persist:clear(), - _ = emqx_eviction_agent:disable(?MODULE), - ?LOG(warning, "Node evacuation stopped"), - {next_state, - disabled, - deinit(Data), - [{reply, From, ok}]}; - -%% status -handle_event({call, From}, status, disabled, #{}) -> - {keep_state_and_data, - [{reply, From, disabled}]}; -handle_event({call, From}, status, State, #{migrate_to := MigrateTo} = Data) -> - Stats = maps:with( - [initial_conns, current_conns, - initial_sessions, current_sessions, - server_reference, conn_evict_rate, sess_evict_rate], - Data), - {keep_state_and_data, - [{reply, From, {enabled, Stats#{state => State, migrate_to => migrate_to(MigrateTo)}}}]}; - -%% conn eviction -handle_event(state_timeout, - evict_conns, - evicting_conns, - #{conn_evict_rate := ConnEvictRate, - wait_takeover := WaitTakeover} = Data) -> - case emqx_eviction_agent:status() of - {enabled, #{connections := Conns}} when Conns > 0 -> - ok = emqx_eviction_agent:evict_connections(ConnEvictRate), - ?tp(debug, node_evacuation_evict_conn, #{conn_evict_rate => ConnEvictRate}), - ?LOG(warning, "Node evacuation evict_conns, count=~p, conn_evict_rate=~p", - [Conns, ConnEvictRate]), - NewData = Data#{current_conns => Conns}, - {keep_state, - NewData, - [{state_timeout, ?EVICT_INTERVAL, evict_conns}]}; - {enabled, #{connections := 0}} -> - NewData = Data#{current_conns => 0}, - ?LOG(warning, "Node evacuation evict_conns over"), - {next_state, - waiting_takeover, - NewData, - [{state_timeout, timer:seconds(WaitTakeover), evict_sessions}]} - end; - -handle_event(state_timeout, - evict_sessions, - waiting_takeover, - Data) -> - ?LOG(warning, "Node evacuation wait_takeover over"), - {next_state, - evicting_sessions, - Data, - [{state_timeout, 0, evict_sessions}]}; - -%% session eviction -handle_event(state_timeout, - evict_sessions, - evicting_sessions, - #{sess_evict_rate := SessEvictRate, - migrate_to := MigrateTo, - current_sessions := CurrSessCount} = Data) -> - case emqx_eviction_agent:status() of - {enabled, #{sessions := SessCount}} when SessCount > 0 -> - case migrate_to(MigrateTo) of - [] -> - ?LOG(warning, - "No nodes are available to evacuate sessions, session_count=~p", - [CurrSessCount]), - {keep_state_and_data, - [{state_timeout, ?EVICT_INTERVAL_NO_NODES, evict_sessions}]}; - Nodes -> - ok = emqx_eviction_agent:evict_sessions(SessEvictRate, Nodes), - ?LOG(warning, "Node evacuation evict_sessions, count=~p, sess_evict_rate=~p," - "target_nodes=~p", [SessCount, SessEvictRate, Nodes]), - NewData = Data#{current_sessions => SessCount}, - {keep_state, - NewData, - [{state_timeout, ?EVICT_INTERVAL, evict_sessions}]} - end; - {enabled, #{sessions := 0}} -> - ?tp(debug, node_evacuation_evict_sess_over, #{}), - ?LOG(warning, "Node evacuation evict_sessions over"), - NewData = Data#{current_sessions => 0}, - {next_state, - prohibiting, - NewData} - end; - -handle_event({call, From}, Msg, State, Data) -> - ?LOG(warning, "Unknown call: ~p, State: ~p, Data: ~p", [Msg, State, Data]), - {keep_state_and_data, - [{reply, From, ignored}]}; - -handle_event(info, Msg, State, Data) -> - ?LOG(warning, "Unknown Msg: ~p, State: ~p, Data: ~p", [Msg, State, Data]), - keep_state_and_data; - -handle_event(cast, Msg, State, Data) -> - ?LOG(warning, "Unknown cast Msg: ~p, State: ~p, Data: ~p", [Msg, State, Data]), - keep_state_and_data. - -code_change(_Vsn, State, Data, _Extra) -> - {ok, State, Data}. - -%%-------------------------------------------------------------------- -%% internal funs -%%-------------------------------------------------------------------- - -default_opts() -> - #{ - server_reference => undefined, - conn_evict_rate => ?DEFAULT_CONN_EVICT_RATE, - sess_evict_rate => ?DEFAULT_SESS_EVICT_RATE, - wait_takeover => ?DEFAULT_WAIT_TAKEOVER, - migrate_to => undefined - }. - -init_data(Data0, Opts) -> - Data1 = maps:merge(Data0, Opts), - {enabled, #{connections := ConnCount, sessions := SessCount}} = emqx_eviction_agent:status(), - Data1#{ - initial_conns => ConnCount, - current_conns => ConnCount, - initial_sessions => SessCount, - current_sessions => SessCount - }. - -deinit(Data) -> - Keys = [initial_conns, current_conns, initial_sessions, current_sessions] - ++ maps:keys(default_opts()), - maps:without(Keys, Data). - -warn_enabled() -> - Msg = "Node evacuation is enabled. The node will not receive connections.", - ?LOG(warning, Msg), - io:format(standard_error, "~s~n", [Msg]). - -migrate_to(undefined) -> - migrate_to(all_nodes()); -migrate_to(Nodes) when is_list(Nodes) -> - available_nodes(Nodes). - -is_node_available() -> - disabled = emqx_eviction_agent:status(), - node(). - -all_nodes() -> - ekka_mnesia:cluster_nodes(all) -- [node()]. diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl deleted file mode 100644 index 06d8800da..000000000 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl +++ /dev/null @@ -1,109 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_node_rebalance_evacuation_persist). - --export([save/1, - clear/0, - read/1]). - - --ifdef(TEST). --export([evacuation_filepath/0]). --endif. - --include("emqx_node_rebalance.hrl"). --include_lib("emqx/include/types.hrl"). - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -%% do not persist `migrate_to`: -%% * after restart there is nothing to migrate -%% * this value may be invalid after node was offline --type start_opts() :: #{server_reference => emqx_eviction_agent:server_reference(), - conn_evict_rate => pos_integer(), - sess_evict_rate => pos_integer(), - wait_takeover => pos_integer() - }. - --spec save(start_opts()) -> ok_or_error(term()). -save(#{server_reference := ServerReference, - conn_evict_rate := ConnEvictRate, - sess_evict_rate := SessEvictRate, - wait_takeover := WaitTakeover} = Data) - when (is_binary(ServerReference) orelse ServerReference =:= undefined) andalso - is_integer(ConnEvictRate) andalso ConnEvictRate > 0 andalso - is_integer(SessEvictRate) andalso SessEvictRate > 0 andalso - is_integer(WaitTakeover) andalso WaitTakeover >= 0 -> - Filepath = evacuation_filepath(), - case filelib:ensure_dir(Filepath) of - ok -> - JsonData = emqx_json:encode( - prepare_for_encode(maps:with(persist_keys(), Data)), - [pretty]), - file:write_file(Filepath, JsonData); - {error, _} = Error -> Error - end. - --spec clear() -> ok. -clear() -> - file:delete(evacuation_filepath()). - --spec read(start_opts()) -> {ok, start_opts()} | none. -read(DefaultOpts) -> - case file:read_file(evacuation_filepath()) of - {ok, Data} -> - case emqx_json:safe_decode(Data, [return_maps]) of - {ok, Map} when is_map(Map) -> - {ok, map_to_opts(DefaultOpts, Map)}; - _NotAMap -> - {ok, DefaultOpts} - end; - {error, _} -> - none - end. - -%%-------------------------------------------------------------------- -%% Internal funcs -%%-------------------------------------------------------------------- - -persist_keys() -> - [server_reference, - conn_evict_rate, - sess_evict_rate, - wait_takeover]. - -prepare_for_encode(#{server_reference := undefined} = Data) -> - Data#{server_reference => null}; -prepare_for_encode(Data) -> Data. - -format_after_decode(#{server_reference := null} = Data) -> - Data#{server_reference => undefined}; -format_after_decode(Data) -> Data. - -map_to_opts(DefaultOpts, Map) -> - format_after_decode( - map_to_opts( - maps:to_list(DefaultOpts), Map, #{})). - -map_to_opts([], _Map, Opts) -> Opts; -map_to_opts([{Key, DefaultVal} | Rest], Map, Opts) -> - map_to_opts(Rest, Map, Opts#{Key => maps:get(atom_to_binary(Key), Map, DefaultVal)}). - -evacuation_filepath() -> - filename:join([emqx:get_env(data_dir), ?EVACUATION_FILENAME]). diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl deleted file mode 100644 index 4002d7c13..000000000 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl +++ /dev/null @@ -1,225 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_node_rebalance_status). - --export([local_status/0, - local_status/1, - global_status/0, - format_local_status/1, - format_coordinator_status/1]). - -%% For RPC --export([evacuation_status/0, - rebalance_status/0]). - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - --spec local_status() -> disabled | {evacuation, map()} | {rebalance, map()}. -local_status() -> - case emqx_node_rebalance_evacuation:status() of - {enabled, Status} -> - {evacuation, evacuation(Status)}; - disabled -> - case emqx_node_rebalance_agent:status() of - {enabled, CoordinatorPid} -> - case emqx_node_rebalance:status(CoordinatorPid) of - {enabled, Status} -> - local_rebalance(Status, node()); - disabled -> - disabled - end; - disabled -> - disabled - end - end. - --spec local_status(node()) -> disabled | {evacuation, map()} | {rebalance, map()}. -local_status(Node) -> - rpc:call(Node, ?MODULE, ?FUNCTION_NAME, []). - --spec format_local_status(map()) -> iodata(). -format_local_status(Status) -> - format_status(Status, local_status_field_format_order()). - --spec global_status() -> #{rebalances := [{node(), map()}], evacuations := [{node(), map()}]}. -global_status() -> - Nodes = ekka_mnesia:cluster_nodes(all), - {RebalanceResults, _} = rpc:multicall(Nodes, ?MODULE, rebalance_status, []), - Rebalances = [{Node, coordinator_rebalance(Status)} || {Node, {enabled, Status}} <- RebalanceResults], - {EvacuatioResults, _} = rpc:multicall(Nodes, ?MODULE, evacuation_status, []), - Evacuations = [{Node, evacuation(Status)} || {Node, {enabled, Status}} <- EvacuatioResults], - #{rebalances => Rebalances, evacuations => Evacuations}. - --spec format_coordinator_status(map()) -> iodata(). -format_coordinator_status(Status) -> - format_status(Status, coordinator_status_field_format_order()). - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -evacuation(Status) -> - #{ - state => maps:get(state, Status), - connection_eviction_rate => maps:get(conn_evict_rate, Status), - session_eviction_rate => maps:get(sess_evict_rate, Status), - connection_goal => 0, - session_goal => 0, - session_recipients => maps:get(migrate_to, Status), - stats => #{ - initial_connected => maps:get(initial_conns, Status), - current_connected => maps:get(current_conns, Status), - initial_sessions => maps:get(initial_sessions, Status), - current_sessions => maps:get(current_sessions, Status) - } - }. - -local_rebalance(#{donors := Donors} = Stats, Node) -> - case lists:member(Node, Donors) of - true -> {rebalance, donor_rebalance(Stats, Node)}; - false -> disabled - end. - -donor_rebalance(Status, Node) -> - Opts = maps:get(opts, Status), - InitialConnCounts = maps:get(initial_conn_counts, Status), - InitialSessCounts = maps:get(initial_sess_counts, Status), - - CurrentStats = #{ - initial_connected => maps:get(Node, InitialConnCounts), - initial_sessions => maps:get(Node, InitialSessCounts), - current_connected => emqx_eviction_agent:connection_count(), - current_sessions => emqx_eviction_agent:session_count(), - current_disconnected_sessions => emqx_eviction_agent:session_count( - disconnected) - }, - maps:from_list( - [ - {state, maps:get(state, Status)}, - {coordinator_node, maps:get(coordinator_node, Status)}, - {connection_eviction_rate, maps:get(conn_evict_rate, Opts)}, - {session_eviction_rate, maps:get(sess_evict_rate, Opts)}, - {recipients, maps:get(recipients, Status)}, - {stats, CurrentStats} - ] ++ - [{connection_goal, maps:get(recipient_conn_avg, Status)} - || maps:is_key(recipient_conn_avg, Status) - ] ++ - [{disconnected_session_goal, maps:get(recipient_sess_avg, Status)} - || maps:is_key(recipient_sess_avg, Status) - ]). - -coordinator_rebalance(Status) -> - Opts = maps:get(opts, Status), - maps:from_list( - [ - {state, maps:get(state, Status)}, - {coordinator_node, maps:get(coordinator_node, Status)}, - {connection_eviction_rate, maps:get(conn_evict_rate, Opts)}, - {session_eviction_rate, maps:get(sess_evict_rate, Opts)}, - {recipients, maps:get(recipients, Status)}, - {donors, maps:get(donors, Status)} - ] ++ - [{connection_goal, maps:get(recipient_conn_avg, Status)} - || maps:is_key(recipient_conn_avg, Status) - ] ++ - [{disconnected_session_goal, maps:get(recipient_sess_avg, Status)} - || maps:is_key(recipient_sess_avg, Status) - ] ++ - [{donor_conn_avg, maps:get(donor_conn_avg, Status)} - || maps:is_key(donor_conn_avg, Status) - ] ++ - [{donor_sess_avg, maps:get(donor_sess_avg, Status)} - || maps:is_key(donor_sess_avg, Status) - ]). - -local_status_field_format_order() -> - [state, - coordinator_node, - connection_eviction_rate, - session_eviction_rate, - connection_goal, - session_goal, - disconnected_session_goal, - session_recipients, - recipients, - stats]. - -coordinator_status_field_format_order() -> - [state, - coordinator_node, - donors, - recipients, - connection_eviction_rate, - session_eviction_rate, - connection_goal, - disconnected_session_goal, - donor_conn_avg, - donor_sess_avg]. - -format_status(Status, FieldOrder) -> - Fields = lists:flatmap( - fun(FieldName) -> - maps:to_list(maps:with([FieldName], Status)) - end, - FieldOrder), - lists:map( - fun format_local_status_field/1, - Fields). - -format_local_status_field({state, State}) -> - io_lib:format("Rebalance state: ~p~n", [State]); -format_local_status_field({coordinator_node, Node}) -> - io_lib:format("Coordinator node: ~p~n", [Node]); -format_local_status_field({connection_eviction_rate, ConnEvictRate}) -> - io_lib:format("Connection eviction rate: ~p connections/second~n", [ConnEvictRate]); -format_local_status_field({session_eviction_rate, SessEvictRate}) -> - io_lib:format("Session eviction rate: ~p sessions/second~n", [SessEvictRate]); -format_local_status_field({connection_goal, ConnGoal}) -> - io_lib:format("Connection goal: ~p~n", [ConnGoal]); -format_local_status_field({session_goal, SessGoal}) -> - io_lib:format("Session goal: ~p~n", [SessGoal]); -format_local_status_field({disconnected_session_goal, DisconnSessGoal}) -> - io_lib:format("Disconnected session goal: ~p~n", [DisconnSessGoal]); -format_local_status_field({session_recipients, SessionRecipients}) -> - io_lib:format("Session recipient nodes: ~p~n", [SessionRecipients]); -format_local_status_field({recipients, Recipients}) -> - io_lib:format("Recipient nodes: ~p~n", [Recipients]); -format_local_status_field({donors, Donors}) -> - io_lib:format("Donor nodes: ~p~n", [Donors]); -format_local_status_field({donor_conn_avg, DonorConnAvg}) -> - io_lib:format("Current average donor node connection count: ~p~n", [DonorConnAvg]); -format_local_status_field({donor_sess_avg, DonorSessAvg}) -> - io_lib:format("Current average donor node disconnected session count: ~p~n", [DonorSessAvg]); -format_local_status_field({stats, Stats}) -> - format_local_stats(Stats). - -format_local_stats(Stats) -> - ["Channel statistics:\n" | - lists:map( - fun({Name, Value}) -> - io_lib:format(" ~p: ~p~n", [Name, Value]) - end, - maps:to_list(Stats))]. - -evacuation_status() -> - {node(), emqx_node_rebalance_evacuation:status()}. - -rebalance_status() -> - {node(), emqx_node_rebalance:status()}. diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl deleted file mode 100644 index e677eb22e..000000000 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl +++ /dev/null @@ -1,44 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_node_rebalance_sup). - --behaviour(supervisor). - --export([start_link/1]). - --export([init/1]). - -start_link(Env) -> - supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]). - -init([_Env]) -> - Childs = [child_spec(emqx_node_rebalance_evacuation, []), - child_spec(emqx_node_rebalance_agent, []), - child_spec(emqx_node_rebalance, [])], - {ok, { - {one_for_one, 10, 3600}, - Childs} - }. - -child_spec(Mod, Args) -> - #{id => Mod, - start => {Mod, start_link, Args}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [Mod] - }. diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl deleted file mode 100644 index c5c27aec0..000000000 --- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl +++ /dev/null @@ -1,183 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_node_rebalance_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). - --import(emqx_eviction_agent_test_helpers, - [emqtt_connect_many/1]). - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]), - Config. - -end_per_suite(Config) -> - emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]), - Config. - -init_per_testcase(_Case, Config) -> - _ = emqx_node_rebalance:stop(), - Node = emqx_node_helpers:start_slave( - recipient1, - #{start_apps => [emqx, emqx_eviction_agent, emqx_node_rebalance]}), - [{recipient_node, Node} | Config]. - -end_per_testcase(_Case, Config) -> - _ = emqx_node_helpers:stop_slave(?config(recipient_node, Config)), - _ = emqx_node_rebalance:stop(). - -t_rebalance(Config) -> - process_flag(trap_exit, true), - RecipientNode = ?config(recipient_node, Config), - - Nodes = [node(), RecipientNode], - - _Conns = emqtt_connect_many(500), - - Opts = #{conn_evict_rate => 10, - sess_evict_rate => 10, - evict_interval => 10, - abs_conn_threshold => 50, - abs_sess_threshold => 50, - rel_conn_threshold => 1.0, - rel_sess_threshold => 1.0, - wait_health_check => 0.01, - wait_takeover => 0.01, - nodes => Nodes - }, - - ?check_trace( - ?wait_async_action( - emqx_node_rebalance:start(Opts), - #{?snk_kind := emqx_node_rebalance_evict_sess_over}, - 10000), - fun({ok, _}, Trace) -> - ?assertMatch( - [_ | _], - ?of_kind(emqx_node_rebalance_evict_sess_over, Trace)) - end), - - DonorConnCount = emqx_eviction_agent:connection_count(), - DonorSessCount = emqx_eviction_agent:session_count(), - DonorDSessCount = emqx_eviction_agent:session_count(disconnected), - - RecipientConnCount = rpc:call(RecipientNode, emqx_eviction_agent, connection_count, []), - RecipientSessCount = rpc:call(RecipientNode, emqx_eviction_agent, session_count, []), - RecipientDSessCount = rpc:call(RecipientNode, emqx_eviction_agent, session_count, [disconnected]), - - ct:pal("Donor: conn=~p, sess=~p, dsess=~p", - [DonorConnCount, DonorSessCount, DonorDSessCount]), - ct:pal("Recipient: conn=~p, sess=~p, dsess=~p", - [RecipientConnCount, RecipientSessCount, RecipientDSessCount]), - - ?assert(DonorConnCount - 50 =< RecipientConnCount), - ?assert(DonorDSessCount - 50 =< RecipientDSessCount). - -t_rebalance_node_crash(Config) -> - process_flag(trap_exit, true), - RecipientNode = ?config(recipient_node, Config), - - Nodes = [node(), RecipientNode], - - _Conns = emqtt_connect_many(50), - - Opts = #{conn_evict_rate => 10, - sess_evict_rate => 10, - evict_interval => 10, - abs_conn_threshold => 50, - abs_sess_threshold => 50, - rel_conn_threshold => 1.0, - rel_sess_threshold => 1.0, - wait_health_check => 0.01, - wait_takeover => 0.01, - nodes => Nodes - }, - - ok = emqx_node_rebalance:start(Opts), - - ?check_trace( - ?wait_async_action( - emqx_node_helpers:stop_slave(?config(recipient_node, Config)), - #{?snk_kind := emqx_node_rebalance_started}, - 1000), - fun(_Result, _Trace) -> ok end), - - ?assertEqual( - disabled, - emqx_node_rebalance:status()). - -t_no_need_to_rebalance(_Config) -> - process_flag(trap_exit, true), - - ?assertEqual( - {error, nothing_to_balance}, - emqx_node_rebalance:start(#{})), - - _Conns = emqtt_connect_many(50), - - ?assertEqual( - {error, nothing_to_balance}, - emqx_node_rebalance:start(#{})). - -t_unknown_mesages(Config) -> - process_flag(trap_exit, true), - RecipientNode = ?config(recipient_node, Config), - - Nodes = [node(), RecipientNode], - - _Conns = emqtt_connect_many(500), - - Opts = #{wait_health_check => 100, - abs_conn_threshold => 50, - nodes => Nodes - }, - - Pid = whereis(emqx_node_rebalance), - - Pid ! unknown, - ok = gen_server:cast(Pid, unknown), - ?assertEqual( - ignored, - gen_server:call(Pid, unknown)), - - ok = emqx_node_rebalance:start(Opts), - - Pid ! unknown, - ok = gen_server:cast(Pid, unknown), - ?assertEqual( - ignored, - gen_server:call(Pid, unknown)). - -t_available_nodes(Config) -> - rpc:call(?config(recipient_node, Config), - emqx_eviction_agent, - enable, - [test_rebalance, undefined]), - ?assertEqual( - [node()], - emqx_node_rebalance:available_nodes( - [node(), ?config(recipient_node, Config)])). diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl deleted file mode 100644 index 30edd51fe..000000000 --- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl +++ /dev/null @@ -1,163 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_node_rebalance_agent_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]), - Config. - -end_per_suite(Config) -> - emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]), - Config. - -init_per_testcase(_Case, Config) -> - _ = emqx_node_rebalance_evacuation:stop(), - Node = emqx_node_helpers:start_slave( - evacuate1, - #{start_apps => [emqx, emqx_eviction_agent, emqx_node_rebalance]}), - [{evacuate_node, Node} | Config]. - -end_per_testcase(_Case, Config) -> - _ = emqx_node_helpers:stop_slave(?config(evacuate_node, Config)), - _ = emqx_node_rebalance_evacuation:stop(). - -t_enable_disable(_Config) -> - ?assertEqual( - disabled, - emqx_node_rebalance_agent:status()), - - ?assertEqual( - ok, - emqx_node_rebalance_agent:enable(self())), - - ?assertEqual( - {error, already_enabled}, - emqx_node_rebalance_agent:enable(self())), - - ?assertEqual( - {enabled, self()}, - emqx_node_rebalance_agent:status()), - - ?assertEqual( - {error, invalid_coordinator}, - emqx_node_rebalance_agent:disable(spawn_link(fun() -> ok end))), - - ?assertEqual( - ok, - emqx_node_rebalance_agent:disable(self())), - - ?assertEqual( - {error, already_disabled}, - emqx_node_rebalance_agent:disable(self())), - - ?assertEqual( - disabled, - emqx_node_rebalance_agent:status()). - -t_enable_egent_busy(_Config) -> - ok = emqx_eviction_agent:enable(rebalance_test, undefined), - - ?assertEqual( - {error, eviction_agent_busy}, - emqx_node_rebalance_agent:enable(self())), - - ok = emqx_eviction_agent:disable(rebalance_test). - -% The following tests verify that emqx_node_rebalance_agent correctly links -% coordinator process with emqx_eviction_agent-s. - -t_rebalance_agent_coordinator_fail(Config) -> - process_flag(trap_exit, true), - - Node = ?config(evacuate_node, Config), - - - CoordinatorPid = spawn_link( - fun() -> - receive - done -> ok - end - end), - - ?assertEqual( - disabled, - rpc:call(Node, emqx_eviction_agent, status, [])), - - ?assertEqual( - ok, - rpc:call(Node, emqx_node_rebalance_agent, enable, [CoordinatorPid])), - - ?assertMatch( - {enabled, _}, - rpc:call(Node, emqx_eviction_agent, status, [])), - - EvictionAgentPid = rpc:call(Node, erlang, whereis, [emqx_eviction_agent]), - true = link(EvictionAgentPid), - - true = exit(CoordinatorPid, kill), - - receive - {'EXIT', EvictionAgentPid, _} -> true - after - 1000 -> ?assert(false, "emqx_eviction_agent did not exit") - end. - -t_rebalance_agent_fail(Config) -> - process_flag(trap_exit, true), - - Node = ?config(evacuate_node, Config), - - CoordinatorPid = spawn_link( - fun() -> - receive - done -> ok - end - end), - - ?assertEqual( - ok, - rpc:call(Node, emqx_node_rebalance_agent, enable, [CoordinatorPid])), - - EvictionAgentPid = rpc:call(Node, erlang, whereis, [emqx_eviction_agent]), - true = exit(EvictionAgentPid, kill), - - receive - {'EXIT', CoordinatorPid, _} -> true - after - 1000 -> ?assert(false, "emqx_eviction_agent did not exit") - end. - -t_unknown_messages(_Config) -> - Pid = whereis(emqx_node_rebalance_agent), - - ok = gen_server:cast(Pid, unknown), - - Pid ! unknown, - - ignored = gen_server:call(Pid, unknown). diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl deleted file mode 100644 index a15b8e8ab..000000000 --- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl +++ /dev/null @@ -1,321 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_node_rebalance_api_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - --import(emqx_mgmt_api_test_helpers, - [request_api/3, - request_api/5, - auth_header_/0, - api_path/1]). - --import(emqx_eviction_agent_test_helpers, - [emqtt_connect_many/1]). - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance, emqx_management]), - Config. - -end_per_suite(Config) -> - emqx_ct_helpers:stop_apps([emqx_management, emqx_node_rebalance, emqx_eviction_agent]), - Config. - -init_per_testcase(Case, Config) - when Case =:= t_start_evacuation_validation - orelse Case =:= t_start_rebalance_validation - orelse Case =:= t_start_stop_rebalance -> - _ = emqx_node_rebalance:stop(), - _ = emqx_node_rebalance_evacuation:stop(), - Node = emqx_node_helpers:start_slave( - recipient1, - #{start_apps => [emqx, emqx_eviction_agent, emqx_node_rebalance]}), - [{recipient_node, Node} | Config]; -init_per_testcase(_Case, Config) -> - _ = emqx_node_rebalance:stop(), - _ = emqx_node_rebalance_evacuation:stop(), - Config. - -end_per_testcase(Case, Config) - when Case =:= t_start_evacuation_validation - orelse Case =:= t_start_rebalance_validation - orelse Case =:= t_start_stop_rebalance -> - _ = emqx_node_helpers:stop_slave(?config(recipient_node, Config)), - _ = emqx_node_rebalance:stop(), - _ = emqx_node_rebalance_evacuation:stop(); -end_per_testcase(_Case, _Config) -> - _ = emqx_node_rebalance:stop(), - _ = emqx_node_rebalance_evacuation:stop(). - -t_start_evacuation_validation(Config) -> - BadOpts = [#{conn_evict_rate => <<"conn">>}, - #{sess_evict_rate => <<"sess">>}, - #{redirect_to => 123}, - #{wait_takeover => <<"wait">>}, - #{migrate_to => []}, - #{migrate_to => <<"migrate_to">>}, - #{migrate_to => [<<"bad_node">>]}, - #{migrate_to => [<<"bad_node">>, atom_to_binary(node())]}, - #{unknown => <<"Value">>} - ], - lists:foreach( - fun(Opts) -> - ?assertMatch( - {ok, #{}}, - api_post(["load_rebalance", atom_to_list(node()), "evacuation", "start"], - Opts)), - - ?assertMatch( - {ok, #{<<"status">> := <<"disabled">>}}, - api_get(["load_rebalance", "status"])) - - end, - BadOpts), - ?assertMatch( - {ok, #{}}, - api_post(["load_rebalance", "bad@node", "evacuation", "start"], - #{})), - - ?assertMatch( - {ok, #{<<"status">> := <<"disabled">>}}, - api_get(["load_rebalance", "status"])), - - ?assertMatch( - {ok, #{}}, - api_post(["load_rebalance", atom_to_list(node()), "evacuation", "start"], - #{conn_evict_rate => 10, - sess_evict_rate => 10, - wait_takeover => 10, - redirect_to => <<"srv">>, - migrate_to => [atom_to_binary(?config(recipient_node, Config))]})), - - ?assertMatch( - {ok, #{<<"status">> := <<"enabled">>}}, - api_get(["load_rebalance", "status"])). - - -t_start_rebalance_validation(Config) -> - BadOpts = [#{conn_evict_rate => <<"conn">>}, - #{sess_evict_rate => <<"sess">>}, - #{abs_conn_threshold => <<"act">>}, - #{rel_conn_threshold => <<"rct">>}, - #{abs_sess_threshold => <<"act">>}, - #{rel_sess_threshold => <<"rct">>}, - #{wait_takeover => <<"wait">>}, - #{wait_health_check => <<"wait">>}, - #{nodes => <<"nodes">>}, - #{nodes => []}, - #{nodes => [<<"bad_node">>]}, - #{nodes => [<<"bad_node">>, atom_to_binary(node())]}, - #{unknown => <<"Value">>} - ], - lists:foreach( - fun(Opts) -> - ?assertMatch( - {ok, #{}}, - api_post(["load_rebalance", atom_to_list(node()), "start"], - Opts)), - - ?assertMatch( - {ok, #{<<"status">> := <<"disabled">>}}, - api_get(["load_rebalance", "status"])) - - end, - BadOpts), - ?assertMatch( - {ok, #{}}, - api_post(["load_rebalance", "bad@node", "start"], - #{})), - - ?assertMatch( - {ok, #{<<"status">> := <<"disabled">>}}, - api_get(["load_rebalance", "status"])), - - _Conns = emqtt_connect_many(50), - - ?assertMatch( - {ok, #{}}, - api_post(["load_rebalance", atom_to_list(node()), "start"], - #{conn_evict_rate => 10, - sess_evict_rate => 10, - wait_takeover => 10, - wait_health_check => 10, - abs_conn_threshold => 10, - rel_conn_threshold => 1.001, - abs_sess_threshold => 10, - rel_sess_threshold => 1.001, - nodes => [atom_to_binary(?config(recipient_node, Config)), - atom_to_binary(node())]})), - - ?assertMatch( - {ok, #{<<"status">> := <<"enabled">>}}, - api_get(["load_rebalance", "status"])). - -t_start_stop_evacuation(_Config) -> - - ?assertMatch( - {ok, #{<<"status">> := <<"disabled">>}}, - api_get(["load_rebalance", "status"])), - - ?assertMatch( - {ok, #{}}, - api_post(["load_rebalance", atom_to_list(node()), "evacuation", "start"], - #{conn_evict_rate => 10, - sess_evict_rate => 20})), - - ?assertMatch( - {ok, #{<<"state">> := _, - <<"process">> := <<"evacuation">>, - <<"connection_eviction_rate">> := 10, - <<"session_eviction_rate">> := 20, - <<"connection_goal">> := 0, - <<"session_goal">> := 0, - <<"stats">> := #{ - <<"initial_connected">> := _, - <<"current_connected">> := _, - <<"initial_sessions">> := _, - <<"current_sessions">> := _ - }}}, - api_get(["load_rebalance", "status"])), - - ?assertMatch( - {ok, #{<<"rebalances">> := #{}, - <<"evacuations">> := - #{<<"test@127.0.0.1">> := #{<<"state">> := _, - <<"connection_eviction_rate">> := 10, - <<"session_eviction_rate">> := 20, - <<"connection_goal">> := 0, - <<"session_goal">> := 0, - <<"stats">> := #{ - <<"initial_connected">> := _, - <<"current_connected">> := _, - <<"initial_sessions">> := _, - <<"current_sessions">> := _ - } - } - }}}, - api_get(["load_rebalance", "global_status"])), - - ?assertMatch( - {ok, #{}}, - api_post(["load_rebalance", atom_to_list(node()), "evacuation", "stop"], - #{})), - - ?assertMatch( - {ok, #{<<"status">> := <<"disabled">>}}, - api_get(["load_rebalance", "status"])), - - ?assertMatch( - {ok, #{<<"evacuations">> := #{}, <<"rebalances">> := #{}}}, - api_get(["load_rebalance", "global_status"])). - -t_start_stop_rebalance(Config) -> - - ?assertMatch( - {ok, #{<<"status">> := <<"disabled">>}}, - api_get(["load_rebalance", "status"])), - - _Conns = emqtt_connect_many(100), - - ?assertMatch( - {ok, #{}}, - api_post(["load_rebalance", atom_to_list(node()), "start"], - #{conn_evict_rate => 10, - sess_evict_rate => 20, - abs_conn_threshold => 10})), - - ?assertMatch( - {ok, #{<<"state">> := _, - <<"process">> := <<"rebalance">>, - <<"coordinator_node">> := _, - <<"connection_eviction_rate">> := 10, - <<"session_eviction_rate">> := 20, - <<"stats">> := #{ - <<"initial_connected">> := _, - <<"current_connected">> := _, - <<"initial_sessions">> := _, - <<"current_sessions">> := _ - }}}, - api_get(["load_rebalance", "status"])), - - DonorNode = atom_to_binary(node()), - RecipientNode = atom_to_binary(?config(recipient_node, Config)), - - ?assertMatch( - {ok, #{<<"evacuations">> := #{}, - <<"rebalances">> := - #{<<"test@127.0.0.1">> := #{<<"state">> := _, - <<"coordinator_node">> := _, - <<"connection_eviction_rate">> := 10, - <<"session_eviction_rate">> := 20, - <<"donors">> := [DonorNode], - <<"recipients">> := [RecipientNode] - } - }}}, - api_get(["load_rebalance", "global_status"])), - - ?assertMatch( - {ok, #{}}, - api_post(["load_rebalance", atom_to_list(node()), "stop"], - #{})), - - ?assertMatch( - {ok, #{<<"status">> := <<"disabled">>}}, - api_get(["load_rebalance", "status"])), - - ?assertMatch( - {ok, #{<<"evacuations">> := #{}, <<"rebalances">> := #{}}}, - api_get(["load_rebalance", "global_status"])). - -t_availability_check(_Config) -> - - ?assertMatch( - {ok, #{}}, - api_get(["load_rebalance", "availability_check"])), - - ok = emqx_node_rebalance_evacuation:start(#{}), - - ?assertMatch( - {error, {_, 503, _}}, - api_get(["load_rebalance", "availability_check"])), - - ok = emqx_node_rebalance_evacuation:stop(), - - ?assertMatch( - {ok, #{}}, - api_get(["load_rebalance", "availability_check"])). - -api_get(Path) -> - case request_api(get, api_path(Path), auth_header_()) of - {ok, ResponseBody} -> - {ok, jiffy:decode(list_to_binary(ResponseBody), [return_maps])}; - {error, _} = Error -> Error - end. - -api_post(Path, Data) -> - case request_api(post, api_path(Path), [], auth_header_(), Data) of - {ok, ResponseBody} -> - {ok, jiffy:decode(list_to_binary(ResponseBody), [return_maps])}; - {error, _} = Error -> Error - end. diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl deleted file mode 100644 index 943746298..000000000 --- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl +++ /dev/null @@ -1,199 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_node_rebalance_cli_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - --import(emqx_eviction_agent_test_helpers, - [emqtt_connect_many/1]). - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]), - Config. - -end_per_suite(Config) -> - emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]), - Config. - - -init_per_testcase(t_rebalance, Config) -> - _ = emqx_node_rebalance_evacuation:stop(), - Node = emqx_node_helpers:start_slave( - evacuate1, - #{start_apps => [emqx, emqx_eviction_agent, emqx_node_rebalance]}), - [{evacuate_node, Node} | Config]; -init_per_testcase(_Case, Config) -> - _ = emqx_node_rebalance_evacuation:stop(), - _ = emqx_node_rebalance:stop(), - Config. - -end_per_testcase(t_rebalance, Config) -> - _ = emqx_node_rebalance_evacuation:stop(), - _ = emqx_node_rebalance:stop(), - _ = emqx_node_helpers:stop_slave(?config(evacuate_node, Config)); -end_per_testcase(_Case, _Config) -> - _ = emqx_node_rebalance_evacuation:stop(), - _ = emqx_node_rebalance:stop(). - -t_evacuation(_Config) -> - %% usage - ok = emqx_node_rebalance_cli:cli(["foobar"]), - - %% status - ok = emqx_node_rebalance_cli:cli(["status"]), - ok = emqx_node_rebalance_cli:cli(["node-status"]), - ok = emqx_node_rebalance_cli:cli(["node-status", atom_to_list(node())]), - - %% start with invalid args - ?assertNot( - emqx_node_rebalance_cli:cli(["start", "--evacuation", "--foo-bar"])), - - ?assertNot( - emqx_node_rebalance_cli:cli(["start", "--evacuation", "--conn-evict-rate", "foobar"])), - - ?assertNot( - emqx_node_rebalance_cli:cli(["start", "--evacuation", "--sess-evict-rate", "foobar"])), - - ?assertNot( - emqx_node_rebalance_cli:cli(["start", "--evacuation", "--wait-takeover", "foobar"])), - - ?assertNot( - emqx_node_rebalance_cli:cli(["start", "--evacuation", - "--migrate-to", "nonexistent@node"])), - ?assertNot( - emqx_node_rebalance_cli:cli(["start", "--evacuation", - "--migrate-to", ""])), - ?assertNot( - emqx_node_rebalance_cli:cli(["start", "--evacuation", - "--unknown-arg"])), - ?assert( - emqx_node_rebalance_cli:cli(["start", "--evacuation", - "--conn-evict-rate", "10", - "--sess-evict-rate", "10", - "--wait-takeover", "10", - "--migrate-to", atom_to_list(node()), - "--redirect-to", "srv"])), - - %% status - ok = emqx_node_rebalance_cli:cli(["status"]), - ok = emqx_node_rebalance_cli:cli(["node-status"]), - ok = emqx_node_rebalance_cli:cli(["node-status", atom_to_list(node())]), - - ?assertMatch( - {enabled, #{}}, - emqx_node_rebalance_evacuation:status()), - - %% already enabled - ?assertNot( - emqx_node_rebalance_cli:cli(["start", "--evacuation", - "--conn-evict-rate", "10", - "--redirect-to", "srv"])), - - %% stop - true = emqx_node_rebalance_cli:cli(["stop"]), - - false = emqx_node_rebalance_cli:cli(["stop"]), - - ?assertEqual( - disabled, - emqx_node_rebalance_evacuation:status()). - -t_rebalance(Config) -> - %% start with invalid args - ?assertNot( - emqx_node_rebalance_cli:cli(["start", "--foo-bar"])), - - ?assertNot( - emqx_node_rebalance_cli:cli(["start", "--conn-evict-rate", "foobar"])), - - ?assertNot( - emqx_node_rebalance_cli:cli(["start", "--abs-conn-threshold", "foobar"])), - - ?assertNot( - emqx_node_rebalance_cli:cli(["start", "--rel-conn-threshold", "foobar"])), - - ?assertNot( - emqx_node_rebalance_cli:cli(["start", "--sess-evict-rate", "foobar"])), - - ?assertNot( - emqx_node_rebalance_cli:cli(["start", "--abs-sess-threshold", "foobar"])), - - ?assertNot( - emqx_node_rebalance_cli:cli(["start", "--rel-sess-threshold", "foobar"])), - - ?assertNot( - emqx_node_rebalance_cli:cli(["start", "--wait-takeover", "foobar"])), - - ?assertNot( - emqx_node_rebalance_cli:cli(["start", "--wait-health-check", "foobar"])), - - ?assertNot( - emqx_node_rebalance_cli:cli(["start", - "--nodes", "nonexistent@node"])), - ?assertNot( - emqx_node_rebalance_cli:cli(["start", - "--nodes", ""])), - ?assertNot( - emqx_node_rebalance_cli:cli(["start", - "--nodes", atom_to_list(?config(evacuate_node, Config))])), - ?assertNot( - emqx_node_rebalance_cli:cli(["start", - "--unknown-arg"])), - - _ = emqtt_connect_many(20), - - ?assert( - emqx_node_rebalance_cli:cli(["start", - "--conn-evict-rate", "10", - "--abs-conn-threshold", "10", - "--rel-conn-threshold", "1.1", - "--sess-evict-rate", "10", - "--abs-sess-threshold", "10", - "--rel-sess-threshold", "1.1", - "--wait-takeover", "10", - "--nodes", atom_to_list(node()) ++ "," - ++ atom_to_list(?config(evacuate_node, Config)) - ])), - - %% status - ok = emqx_node_rebalance_cli:cli(["status"]), - ok = emqx_node_rebalance_cli:cli(["node-status"]), - ok = emqx_node_rebalance_cli:cli(["node-status", atom_to_list(node())]), - - ?assertMatch( - {enabled, #{}}, - emqx_node_rebalance:status()), - - %% already enabled - ?assertNot( - emqx_node_rebalance_cli:cli(["start"])), - - %% stop - true = emqx_node_rebalance_cli:cli(["stop"]), - - false = emqx_node_rebalance_cli:cli(["stop"]), - - ?assertEqual( - disabled, - emqx_node_rebalance:status()). diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl deleted file mode 100644 index 23f35f61e..000000000 --- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl +++ /dev/null @@ -1,194 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_node_rebalance_evacuation_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). - --import(emqx_eviction_agent_test_helpers, - [emqtt_connect/0, emqtt_connect/2, emqtt_try_connect/0]). - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]), - Config. - -end_per_suite(Config) -> - emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]), - Config. - -init_per_testcase(_Case, Config) -> - _ = emqx_node_rebalance_evacuation:stop(), - Node = emqx_node_helpers:start_slave( - evacuate1, - #{start_apps => [emqx, emqx_eviction_agent]}), - [{evacuate_node, Node} | Config]. - -end_per_testcase(_Case, Config) -> - _ = emqx_node_helpers:stop_slave(?config(evacuate_node, Config)), - _ = emqx_node_rebalance_evacuation:stop(). - -t_agent_busy(Config) -> - - ok = emqx_eviction_agent:enable(other_rebalance, undefined), - - ?assertEqual( - {error, eviction_agent_busy}, - emqx_node_rebalance_evacuation:start(opts(Config))), - - emqx_eviction_agent:disable(other_rebalance). - - -t_already_started(Config) -> - ok = emqx_node_rebalance_evacuation:start(opts(Config)), - - ?assertEqual( - {error, already_started}, - emqx_node_rebalance_evacuation:start(opts(Config))), - - ok = emqx_node_rebalance_evacuation:stop(). - -t_not_started(_Config) -> - ?assertEqual( - {error, not_started}, - emqx_node_rebalance_evacuation:stop()). - -t_start(Config) -> - process_flag(trap_exit, true), - - ok = emqx_node_rebalance_evacuation:start(opts(Config)), - ?assertMatch( - {error, {use_another_server, #{}}}, - emqtt_try_connect()), - ok = emqx_node_rebalance_evacuation:stop(). - -t_persistence(Config) -> - process_flag(trap_exit, true), - - ok = emqx_node_rebalance_evacuation:start(opts(Config)), - - ?assertMatch( - {error, {use_another_server, #{}}}, - emqtt_try_connect()), - - ok = supervisor:terminate_child(emqx_node_rebalance_sup, emqx_node_rebalance_evacuation), - {ok, _} = supervisor:restart_child(emqx_node_rebalance_sup, emqx_node_rebalance_evacuation), - - ?assertMatch( - {error, {use_another_server, #{}}}, - emqtt_try_connect()), - ?assertMatch( - {enabled, #{conn_evict_rate := 10}}, - emqx_node_rebalance_evacuation:status()), - - ok = emqx_node_rebalance_evacuation:stop(). - -t_conn_evicted(Config) -> - process_flag(trap_exit, true), - - {ok, C} = emqtt_connect(), - - ?check_trace( - ?wait_async_action( - emqx_node_rebalance_evacuation:start(opts(Config)), - #{?snk_kind := node_evacuation_evict_conn}, - 1000), - fun(_Result, _Trace) -> ok end), - - ct:sleep(100), - - ?assertMatch( - {error, {use_another_server, #{}}}, - emqtt_try_connect()), - - ?assertNot( - is_process_alive(C)). - -t_migrate_to(Config) -> - ?assertEqual( - [?config(evacuate_node, Config)], - emqx_node_rebalance_evacuation:migrate_to(undefined)), - - ?assertEqual( - [], - emqx_node_rebalance_evacuation:migrate_to(['unknown@node'])), - - rpc:call(?config(evacuate_node, Config), - emqx_eviction_agent, - enable, - [test_rebalance, undefined]), - - ?assertEqual( - [], - emqx_node_rebalance_evacuation:migrate_to(undefined)). - -t_session_evicted(Config) -> - process_flag(trap_exit, true), - - {ok, C} = emqtt_connect(<<"client_with_sess">>, false), - - ?check_trace( - ?wait_async_action( - emqx_node_rebalance_evacuation:start(opts(Config)), - #{?snk_kind := node_evacuation_evict_sess_over}, - 5000), - fun(_Result, Trace) -> - ?assertMatch( - [_ | _], - ?of_kind(node_evacuation_evict_sess_over, Trace)) - end), - - receive - {'EXIT', C, {disconnected, ?RC_USE_ANOTHER_SERVER, _}} -> ok - after 1000 -> - ?assert(false, "Connection not evicted") - end, - - [ChannelPid] = emqx_cm_registry:lookup_channels(<<"client_with_sess">>), - - ?assertEqual( - ?config(evacuate_node, Config), - node(ChannelPid)). - -t_unknown_messages(Config) -> - ok = emqx_node_rebalance_evacuation:start(opts(Config)), - - whereis(emqx_node_rebalance_evacuation) ! unknown, - - gen_server:cast(emqx_node_rebalance_evacuation, unknown), - - ?assertEqual( - ignored, - gen_server:call(emqx_node_rebalance_evacuation, unknown)), - - ok = emqx_node_rebalance_evacuation:stop(). - -opts(Config) -> - #{ - server_reference => <<"srv">>, - conn_evict_rate => 10, - sess_evict_rate => 10, - wait_takeover => 1, - migrate_to => [?config(evacuate_node, Config)] - }. diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl deleted file mode 100644 index ba1a12775..000000000 --- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl +++ /dev/null @@ -1,107 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_node_rebalance_evacuation_persist_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]), - Config. - -end_per_suite(Config) -> - emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]), - Config. - -init_per_testcase(_Case, Config) -> - _ = emqx_node_rebalance_evacuation_persist:clear(), - Config. - -end_per_testcase(_Case, _Config) -> - _ = emqx_node_rebalance_evacuation_persist:clear(). - -t_save_read(_Config) -> - DefaultOpts = #{server_reference => <<"default_ref">>, - conn_evict_rate => 2001, - sess_evict_rate => 2002, - wait_takeover => 2003 - }, - - Opts0 = #{server_reference => <<"ref">>, - conn_evict_rate => 1001, - sess_evict_rate => 1002, - wait_takeover => 1003 - }, - ok = emqx_node_rebalance_evacuation_persist:save(Opts0), - - {ok, ReadOpts0} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts), - ?assertEqual(Opts0, ReadOpts0), - - Opts1 = Opts0#{server_reference => undefined}, - ok = emqx_node_rebalance_evacuation_persist:save(Opts1), - - {ok, ReadOpts1} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts), - ?assertEqual(Opts1, ReadOpts1). - -t_read_default(_Config) -> - ok = write_evacuation_file(<<"{}">>), - - DefaultOpts = #{server_reference => <<"ref">>, - conn_evict_rate => 1001, - sess_evict_rate => 1002, - wait_takeover => 1003 - }, - - {ok, ReadOpts} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts), - ?assertEqual(DefaultOpts, ReadOpts). - -t_read_bad_data(_Config) -> - ok = write_evacuation_file(<<"{bad json">>), - - DefaultOpts = #{server_reference => <<"ref">>, - conn_evict_rate => 1001, - sess_evict_rate => 1002, - wait_takeover => 1003 - }, - - {ok, ReadOpts} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts), - ?assertEqual(DefaultOpts, ReadOpts). - -t_clear(_Config) -> - ok = write_evacuation_file(<<"{}">>), - - ?assertMatch( - {ok, _}, - emqx_node_rebalance_evacuation_persist:read(#{})), - - ok = emqx_node_rebalance_evacuation_persist:clear(), - - ?assertEqual( - none, - emqx_node_rebalance_evacuation_persist:read(#{})). - -write_evacuation_file(Json) -> - ok = filelib:ensure_dir(emqx_node_rebalance_evacuation_persist:evacuation_filepath()), - ok = file:write_file( - emqx_node_rebalance_evacuation_persist:evacuation_filepath(), - Json). diff --git a/data/loaded_plugins.tmpl b/data/loaded_plugins.tmpl index 62b3205cd..d0dac7fe1 100644 --- a/data/loaded_plugins.tmpl +++ b/data/loaded_plugins.tmpl @@ -6,5 +6,3 @@ {emqx_telemetry, {{enable_plugin_emqx_telemetry}}}. {emqx_rule_engine, {{enable_plugin_emqx_rule_engine}}}. {emqx_bridge_mqtt, {{enable_plugin_emqx_bridge_mqtt}}}. -{emqx_eviction_agent, true}. -{emqx_node_rebalance, true}. diff --git a/rebar.config.erl b/rebar.config.erl index 04ffaefbd..841399a73 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -297,8 +297,6 @@ relx_plugin_apps(ReleaseType) -> , emqx_recon , emqx_rule_engine , emqx_sasl - , emqx_eviction_agent - , emqx_node_rebalance ] ++ [emqx_telemetry || not is_enterprise()] ++ relx_plugin_apps_per_rel(ReleaseType) diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index 1c8849745..8b5b7ffb2 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -254,9 +254,7 @@ default_plugins() -> {emqx_recon, true}, {emqx_telemetry, true}, {emqx_rule_engine, true}, - {emqx_bridge_mqtt, false}, - {emqx_eviction_agent, true}, - {emqx_node_rebalance, true} + {emqx_bridge_mqtt, false} ]. -endif. diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl index aa548d766..eef5152b0 100644 --- a/test/emqx_plugins_SUITE.erl +++ b/test/emqx_plugins_SUITE.erl @@ -110,10 +110,8 @@ default_plugins() -> [ {emqx_bridge_mqtt, false}, {emqx_dashboard, true}, - {emqx_eviction_agent, true}, {emqx_management, true}, {emqx_modules, true}, - {emqx_node_rebalance, true}, {emqx_recon, true}, {emqx_retainer, false}, {emqx_rule_engine, true}, From 0287d6c755532df8c01fe1717e5bfec65dc36bb9 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 16 Aug 2022 10:30:05 +0200 Subject: [PATCH 30/73] docs: fix comment typo in emqx_retainer_sup.erl Co-authored-by: ieQu1 <99872536+ieQu1@users.noreply.github.com> --- apps/emqx_retainer/src/emqx_retainer_sup.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_retainer/src/emqx_retainer_sup.erl b/apps/emqx_retainer/src/emqx_retainer_sup.erl index d3a979788..2028affb6 100644 --- a/apps/emqx_retainer/src/emqx_retainer_sup.erl +++ b/apps/emqx_retainer/src/emqx_retainer_sup.erl @@ -35,5 +35,5 @@ init([Env]) -> modules => [emqx_retainer]} || not is_managed_by_modules()]}}. is_managed_by_modules() -> - %% always flase for opensource edition + %% always false for opensource edition false. From 4055b2025981857b5ac00728375e265015128ccb Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 16 Aug 2022 21:32:24 +0800 Subject: [PATCH 31/73] fix: sql compare to undefined values --- .../src/emqx_rule_runtime.erl | 14 +- .../test/emqx_rule_engine_SUITE.erl | 208 ++++++++++++++++++ 2 files changed, 216 insertions(+), 6 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 88b2c9143..145e30eec 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -216,10 +216,8 @@ match_conditions({}, _Data) -> true. %% comparing numbers against strings -compare(Op, undefined, undefined) -> - do_compare(Op, undefined, undefined); -compare(_Op, L, R) when L == undefined; R == undefined -> - false; +compare(Op, L, R) when L == undefined; R == undefined -> + do_compare(Op, L, R); compare(Op, L, R) when is_number(L), is_binary(R) -> do_compare(Op, L, number(R)); compare(Op, L, R) when is_binary(L), is_number(R) -> @@ -232,10 +230,14 @@ compare(Op, L, R) -> do_compare(Op, L, R). do_compare('=', L, R) -> L == R; +do_compare('>', L, R) when L == undefined; R == undefined -> false; do_compare('>', L, R) -> L > R; +do_compare('<', L, R) when L == undefined; R == undefined -> false; do_compare('<', L, R) -> L < R; -do_compare('<=', L, R) -> L =< R; -do_compare('>=', L, R) -> L >= R; +do_compare('<=', L, R) -> + do_compare('=', L, R) orelse do_compare('<', L, R); +do_compare('>=', L, R) -> + do_compare('=', L, R) orelse do_compare('>', L, R); do_compare('<>', L, R) -> L /= R; do_compare('!=', L, R) -> L /= R; do_compare('=~', T, F) -> emqx_topic:match(T, F). diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 9031c3df8..e1851b76c 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -126,6 +126,10 @@ groups() -> t_sqlparse_array_range_1, t_sqlparse_array_range_2, t_sqlparse_true_false, + t_sqlparse_compare_null_null, + t_sqlparse_compare_null_notnull, + t_sqlparse_compare_notnull_null, + t_sqlparse_compare, t_sqlparse_new_map, t_sqlparse_invalid_json ]}, @@ -2486,6 +2490,210 @@ t_sqlparse_true_false(_Config) -> <<"c">> := [true] }, Res00). +-define(TEST_SQL(SQL), + emqx_rule_sqltester:test( + #{<<"rawsql">> => SQL, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})). +t_sqlparse_compare_null_null(_Config) -> + %% test undefined == undefined + Sql00 = "select " + " a = b as c " + "from \"t/#\" ", + {ok, Res00} = ?TEST_SQL(Sql00), + ?assertMatch(#{<<"c">> := true + }, Res00), + + %% test undefined != undefined + Sql01 = "select " + " a != b as c " + "from \"t/#\" ", + {ok, Res01} = ?TEST_SQL(Sql01), + ?assertMatch(#{<<"c">> := false + }, Res01), + + %% test undefined > undefined + Sql02 = "select " + " a > b as c " + "from \"t/#\" ", + {ok, Res02} = ?TEST_SQL(Sql02), + ?assertMatch(#{<<"c">> := false + }, Res02), + + %% test undefined < undefined + Sql03 = "select " + " a < b as c " + "from \"t/#\" ", + {ok, Res03} = ?TEST_SQL(Sql03), + ?assertMatch(#{<<"c">> := false + }, Res03), + + %% test undefined <= undefined + Sql04 = "select " + " a <= b as c " + "from \"t/#\" ", + {ok, Res04} = ?TEST_SQL(Sql04), + ?assertMatch(#{<<"c">> := true + }, Res04), + + %% test undefined >= undefined + Sql05 = "select " + " a >= b as c " + "from \"t/#\" ", + {ok, Res05} = ?TEST_SQL(Sql05), + ?assertMatch(#{<<"c">> := true + }, Res05). + +t_sqlparse_compare_null_notnull(_Config) -> + %% test undefined == b + Sql00 = "select " + " 'b' as b, a = b as c " + "from \"t/#\" ", + {ok, Res00} = ?TEST_SQL(Sql00), + ?assertMatch(#{<<"c">> := false + }, Res00), + + %% test undefined != b + Sql01 = "select " + " 'b' as b, a != b as c " + "from \"t/#\" ", + {ok, Res01} = ?TEST_SQL(Sql01), + ?assertMatch(#{<<"c">> := true + }, Res01), + + %% test undefined > b + Sql02 = "select " + " 'b' as b, a > b as c " + "from \"t/#\" ", + {ok, Res02} = ?TEST_SQL(Sql02), + ?assertMatch(#{<<"c">> := false + }, Res02), + + %% test undefined < b + Sql03 = "select " + " 'b' as b, a < b as c " + "from \"t/#\" ", + {ok, Res03} = ?TEST_SQL(Sql03), + ?assertMatch(#{<<"c">> := false + }, Res03), + + %% test undefined <= b + Sql04 = "select " + " 'b' as b, a <= b as c " + "from \"t/#\" ", + {ok, Res04} = ?TEST_SQL(Sql04), + ?assertMatch(#{<<"c">> := false + }, Res04), + + %% test undefined >= b + Sql05 = "select " + " 'b' as b, a >= b as c " + "from \"t/#\" ", + {ok, Res05} = ?TEST_SQL(Sql05), + ?assertMatch(#{<<"c">> := false + }, Res05). + +t_sqlparse_compare_notnull_null(_Config) -> + %% test 'a' == undefined + Sql00 = "select " + " 'a' as a, a = b as c " + "from \"t/#\" ", + {ok, Res00} = ?TEST_SQL(Sql00), + ?assertMatch(#{<<"c">> := false + }, Res00), + + %% test 'a' != undefined + Sql01 = "select " + " 'a' as a, a != b as c " + "from \"t/#\" ", + {ok, Res01} = ?TEST_SQL(Sql01), + ?assertMatch(#{<<"c">> := true + }, Res01), + + %% test 'a' > undefined + Sql02 = "select " + " 'a' as a, a > b as c " + "from \"t/#\" ", + {ok, Res02} = ?TEST_SQL(Sql02), + ?assertMatch(#{<<"c">> := false + }, Res02), + + %% test 'a' < undefined + Sql03 = "select " + " 'a' as a, a < b as c " + "from \"t/#\" ", + {ok, Res03} = ?TEST_SQL(Sql03), + ?assertMatch(#{<<"c">> := false + }, Res03), + + %% test 'a' <= undefined + Sql04 = "select " + " 'a' as a, a <= b as c " + "from \"t/#\" ", + {ok, Res04} = ?TEST_SQL(Sql04), + ?assertMatch(#{<<"c">> := false + }, Res04), + + %% test 'a' >= undefined + Sql05 = "select " + " 'a' as a, a >= b as c " + "from \"t/#\" ", + {ok, Res05} = ?TEST_SQL(Sql05), + ?assertMatch(#{<<"c">> := false + }, Res05). + +t_sqlparse_compare(_Config) -> + Sql00 = "select " + " 'a' as a, 'a' as b, a = b as c " + "from \"t/#\" ", + {ok, Res00} = ?TEST_SQL(Sql00), + ?assertMatch(#{<<"c">> := true + }, Res00), + + Sql01 = "select " + " is_null(a) as c " + "from \"t/#\" ", + {ok, Res01} = ?TEST_SQL(Sql01), + ?assertMatch(#{<<"c">> := true + }, Res01), + + Sql02 = "select " + " 1 as a, 2 as b, a < b as c " + "from \"t/#\" ", + {ok, Res02} = ?TEST_SQL(Sql02), + ?assertMatch(#{<<"c">> := true + }, Res02), + + Sql03 = "select " + " 1 as a, 2 as b, a > b as c " + "from \"t/#\" ", + {ok, Res03} = ?TEST_SQL(Sql03), + ?assertMatch(#{<<"c">> := false + }, Res03), + + Sql04 = "select " + " 1 as a, 2 as b, a = b as c " + "from \"t/#\" ", + {ok, Res04} = ?TEST_SQL(Sql04), + ?assertMatch(#{<<"c">> := false + }, Res04), + + %% test 'a' >= undefined + Sql05 = "select " + " 1 as a, 2 as b, a >= b as c " + "from \"t/#\" ", + {ok, Res05} = ?TEST_SQL(Sql05), + ?assertMatch(#{<<"c">> := false + }, Res05), + + %% test 'a' >= undefined + Sql06 = "select " + " 1 as a, 2 as b, a <= b as c " + "from \"t/#\" ", + {ok, Res06} = ?TEST_SQL(Sql06), + ?assertMatch(#{<<"c">> := true + }, Res06). + t_sqlparse_new_map(_Config) -> %% construct a range without 'as' Sql00 = "select " From 582ead1d77cba4bda669b931c22c63e2ca9be4a4 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 16 Aug 2022 21:38:57 +0800 Subject: [PATCH 32/73] fix: update appup for rule engine --- .../src/emqx_rule_engine.appup.src | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 1c5fb0e7b..792f99310 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,10 +1,17 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.13",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, - {"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + [{"4.3.13",[ + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + ]}, + {"4.3.12",[ + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + ]}, {"4.3.11", [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", @@ -166,11 +173,18 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.13",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, - {"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + [{"4.3.13",[ + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + ]}, + {"4.3.12",[ + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + ]}, {"4.3.11", [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, From 67ec6e0e66fe7e41f2176b3ae8a5c967fe10d44e Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 16 Aug 2022 10:24:05 +0800 Subject: [PATCH 33/73] fix: log RuleId for take action failed --- .../emqx_rule_engine/include/rule_actions.hrl | 19 ++++++++ .../src/emqx_rule_actions.erl | 18 +++++--- .../src/emqx_rule_runtime.erl | 45 +++++++++---------- apps/emqx_rule_engine/src/emqx_rule_utils.erl | 34 ++++++++++++++ .../src/emqx_web_hook_actions.erl | 10 +++-- 5 files changed, 93 insertions(+), 33 deletions(-) diff --git a/apps/emqx_rule_engine/include/rule_actions.hrl b/apps/emqx_rule_engine/include/rule_actions.hrl index e432c4399..4571849d3 100644 --- a/apps/emqx_rule_engine/include/rule_actions.hrl +++ b/apps/emqx_rule_engine/include/rule_actions.hrl @@ -1,3 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -compile({parse_transform, emqx_rule_actions_trans}). -type selected_data() :: map(). @@ -6,6 +22,9 @@ -define(BINDING_KEYS, '__bindings__'). +-define(LOG_RULE_ACTION(Level, Metadata, Fmt, Args), + emqx_rule_utils:log_action(Level, Metadata, Fmt, Args)). + -define(bound_v(Key, ENVS0), maps:get(Key, maps:get(?BINDING_KEYS, ENVS0, #{}))). diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index d665a0c96..557ddf423 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -171,9 +171,13 @@ on_action_create_republish(Id, Params = #{ on_action_republish(_Selected, Envs = #{ topic := Topic, headers := #{republish_by := ActId}, - ?BINDING_KEYS := #{'Id' := ActId} + ?BINDING_KEYS := #{'Id' := ActId}, + metadata := Metadata }) -> - ?LOG(error, "[republish] recursively republish detected, msg topic: ~p, target topic: ~p", + ?LOG_RULE_ACTION( + error, + Metadata, + "[republish] recursively republish detected, msg topic: ~p, target topic: ~p", [Topic, ?bound_v('TargetTopic', Envs)]), emqx_rule_metrics:inc_actions_error(?bound_v('Id', Envs)), {badact, recursively_republish}; @@ -186,8 +190,9 @@ on_action_republish(Selected, _Envs = #{ 'TargetQoS' := TargetQoS, 'TopicTks' := TopicTks, 'PayloadTks' := PayloadTks - } = Bindings}) -> - ?LOG(debug, "[republish] republish to: ~p, Selected: ~p", [TargetTopic, Selected]), + } = Bindings, + metadata := Metadata}) -> + ?LOG_RULE_ACTION(debug, Metadata, "[republish] republish to: ~p, Selected: ~p", [TargetTopic, Selected]), TargetRetain = maps:get('TargetRetain', Bindings, false), Message = #message{ @@ -210,8 +215,9 @@ on_action_republish(Selected, _Envs = #{ 'TargetQoS' := TargetQoS, 'TopicTks' := TopicTks, 'PayloadTks' := PayloadTks - } = Bindings}) -> - ?LOG(debug, "[republish] republish to: ~p, Selected: ~p", [TargetTopic, Selected]), + } = Bindings, + metadata := Metadata}) -> + ?LOG_RULE_ACTION(debug, Metadata, "[republish] republish to: ~p, Selected: ~p", [TargetTopic, Selected]), TargetRetain = maps:get('TargetRetain', Bindings, false), Message = #message{ diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 88b2c9143..22fac23c0 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -17,6 +17,7 @@ -module(emqx_rule_runtime). -include("rule_engine.hrl"). +-include("rule_actions.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -54,36 +55,37 @@ apply_rules([Rule|More], Input) -> apply_rule(Rule, Input), apply_rules(More, Input). -apply_rule(Rule = #rule{id = RuleID}, Input) -> +apply_rule(Rule = #rule{id = RuleId}, Input) -> clear_rule_payload(), - ok = emqx_rule_metrics:inc_rules_matched(RuleID), - try do_apply_rule(Rule, add_metadata(Input, #{rule_id => RuleID})) + ok = emqx_rule_metrics:inc_rules_matched(RuleId), + %% Add metadata here caused we need support `metadata` and `rule_id` in SQL + try do_apply_rule(Rule, emqx_rule_utils:add_metadata(Input, #{rule_id => RuleId})) catch %% ignore the errors if select or match failed _:Reason = {select_and_transform_error, Error} -> - emqx_rule_metrics:inc_rules_exception(RuleID), + emqx_rule_metrics:inc_rules_exception(RuleId), ?LOG(warning, "SELECT clause exception for ~s failed: ~p", - [RuleID, Error]), + [RuleId, Error]), {error, Reason}; _:Reason = {match_conditions_error, Error} -> - emqx_rule_metrics:inc_rules_exception(RuleID), + emqx_rule_metrics:inc_rules_exception(RuleId), ?LOG(warning, "WHERE clause exception for ~s failed: ~p", - [RuleID, Error]), + [RuleId, Error]), {error, Reason}; _:Reason = {select_and_collect_error, Error} -> - emqx_rule_metrics:inc_rules_exception(RuleID), + emqx_rule_metrics:inc_rules_exception(RuleId), ?LOG(warning, "FOREACH clause exception for ~s failed: ~p", - [RuleID, Error]), + [RuleId, Error]), {error, Reason}; _:Reason = {match_incase_error, Error} -> - emqx_rule_metrics:inc_rules_exception(RuleID), + emqx_rule_metrics:inc_rules_exception(RuleId), ?LOG(warning, "INCASE clause exception for ~s failed: ~p", - [RuleID, Error]), + [RuleId, Error]), {error, Reason}; _:Error:StkTrace -> - emqx_rule_metrics:inc_rules_exception(RuleID), + emqx_rule_metrics:inc_rules_exception(RuleId), ?LOG(error, "Apply rule ~s failed: ~p. Stacktrace:~n~p", - [RuleID, Error, StkTrace]), + [RuleId, Error, StkTrace]), {error, {Error, StkTrace}} end. @@ -245,9 +247,10 @@ number(Bin) -> catch error:badarg -> binary_to_float(Bin) end. -%% Step3 -> Take actions +%% %% Step3 -> Take actions +%% fallback actions already have `rule_id` in `metadata` take_actions(Actions, Selected, Envs, OnFailed) -> - [take_action(ActInst, Selected, Envs, OnFailed, ?ActionMaxRetry) + [take_action(ActInst, Selected, emqx_rule_utils:add_metadata(Envs, ActInst), OnFailed, ?ActionMaxRetry) || ActInst <- Actions]. take_action(#action_instance{id = Id, name = ActName, fallbacks = Fallbacks} = ActInst, @@ -312,12 +315,12 @@ wait_action_on(Id, RetryN) -> end end. -handle_action_failure(continue, Id, Fallbacks, Selected, Envs, Reason) -> - ?LOG(error, "Take action ~p failed, continue next action, reason: ~0p", [Id, Reason]), +handle_action_failure(continue, _Id, Fallbacks, Selected, Envs = #{metadata := Metadata}, Reason) -> + ?LOG_RULE_ACTION(error, Metadata, "Continue next action, reason: ~0p", [Reason]), _ = take_actions(Fallbacks, Selected, Envs, continue), failed; -handle_action_failure(stop, Id, Fallbacks, Selected, Envs, Reason) -> - ?LOG(error, "Take action ~p failed, skip all actions, reason: ~0p", [Id, Reason]), +handle_action_failure(stop, Id, Fallbacks, Selected, Envs = #{metadata := Metadata}, Reason) -> + ?LOG_RULE_ACTION(error, Metadata, "Skip all actions, reason: ~0p", [Reason]), _ = take_actions(Fallbacks, Selected, Envs, continue), error({take_action_failed, {Id, Reason}}). @@ -429,10 +432,6 @@ do_apply_func(Name, Args, Input) -> Result -> Result end. -add_metadata(Input, Metadata) when is_map(Input), is_map(Metadata) -> - NewMetadata = maps:merge(maps:get(metadata, Input, #{}), Metadata), - Input#{metadata => NewMetadata}. - %%------------------------------------------------------------------------------ %% Internal Functions %%------------------------------------------------------------------------------ diff --git a/apps/emqx_rule_engine/src/emqx_rule_utils.erl b/apps/emqx_rule_engine/src/emqx_rule_utils.erl index 137e22128..8bd46a958 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_utils.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_utils.erl @@ -16,6 +16,9 @@ -module(emqx_rule_utils). +-include("rule_engine.hrl"). +-include_lib("emqx/include/logger.hrl"). + -export([ replace_var/2 ]). @@ -59,6 +62,10 @@ , can_topic_match_oneof/2 ]). +-export([ add_metadata/2 + , log_action/4 + ]). + -compile({no_auto_import, [ float/1 ]}). @@ -371,3 +378,30 @@ can_topic_match_oneof(Topic, Filters) -> lists:any(fun(Fltr) -> emqx_topic:match(Topic, Fltr) end, Filters). + +add_metadata(Envs, Metadata) when is_map(Envs), is_map(Metadata) -> + NMetadata = maps:merge(maps:get(metadata, Envs, #{}), Metadata), + Envs#{metadata => NMetadata}; +add_metadata(Envs, Action) when is_map(Envs), is_record(Action, action_instance)-> + Metadata = gen_metadata_from_action(Action), + NMetadata = maps:merge(maps:get(metadata, Envs, #{}), Metadata), + Envs#{metadata => NMetadata}. + +gen_metadata_from_action(#action_instance{name = Name, args = undefined}) -> + #{action_name => Name, resource_id => undefined}; +gen_metadata_from_action(#action_instance{name = Name, args = Args}) + when is_map(Args) -> + #{action_name => Name, resource_id => maps:get(<<"$resource">>, Args, undefined)}; +gen_metadata_from_action(#action_instance{name = Name}) -> + #{action_name => Name, resource_id => undefined}. + +log_action(Level, Metadata, Fmt, Args) -> + ?LOG(Level, + "Rule: ~p; Action: ~p; Rusource: ~p. " ++ Fmt, + metadata_values(Metadata) ++ Args). + +metadata_values(Metadata) -> + RuleId = maps:get(rule_id, Metadata, undefined), + ActionName = maps:get(action_name, Metadata, undefined), + ResourceName = maps:get(resource_id, Metadata, undefined), + [RuleId, ActionName, ResourceName]. diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index 00094d46d..29550e1e9 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -259,25 +259,27 @@ on_action_data_to_webserver(Selected, _Envs = 'BodyTokens' := BodyTokens, 'Pool' := Pool, 'RequestTimeout' := RequestTimeout}, - clientid := ClientID}) -> + clientid := ClientID, + metadata := Metadata}) -> NBody = format_msg(BodyTokens, clear_user_property_header(Selected)), NPath = emqx_rule_utils:proc_tmpl(PathTokens, Selected), Req = create_req(Method, NPath, Headers, NBody), case ehttpc:request({Pool, ClientID}, Method, Req, RequestTimeout) of {ok, StatusCode, _} when StatusCode >= 200 andalso StatusCode < 300 -> + ?LOG_RULE_ACTION(debug, Metadata, "HTTP Request succeeded with path: ~p status code ~p", [NPath, StatusCode]), emqx_rule_metrics:inc_actions_success(Id); {ok, StatusCode, _, _} when StatusCode >= 200 andalso StatusCode < 300 -> emqx_rule_metrics:inc_actions_success(Id); {ok, StatusCode, _} -> - ?LOG(warning, "HTTP request failed with path: ~p status code: ~p", [NPath, StatusCode]), + ?LOG_RULE_ACTION(warning, Metadata, "HTTP request failed with path: ~p status code: ~p", [NPath, StatusCode]), emqx_rule_metrics:inc_actions_error(Id), {badact, StatusCode}; {ok, StatusCode, _, _} -> - ?LOG(warning, "HTTP request failed with path: ~p status code: ~p", [NPath, StatusCode]), + ?LOG_RULE_ACTION(warning, Metadata, "HTTP request failed with path: ~p status code: ~p", [NPath, StatusCode]), emqx_rule_metrics:inc_actions_error(Id), {badact, StatusCode}; {error, Reason} -> - ?LOG(error, "HTTP request failed path: ~p error: ~p", [NPath, Reason]), + ?LOG_RULE_ACTION(error, Metadata, "HTTP request failed path: ~p error: ~p", [NPath, Reason]), emqx_rule_metrics:inc_actions_error(Id), {badact, Reason} end. From 4a89dfe3622060ebfe31365c8b6681e62806693e Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 16 Aug 2022 10:53:39 +0800 Subject: [PATCH 34/73] chore: update CHANGES.md and appup.src This reverts commit 7af25a82e70845a631be0c8b83ba7f1838d68389. --- CHANGES-4.3.md | 13 +++---- .../src/emqx_rule_engine.appup.src | 34 +++++++++++++++---- apps/emqx_web_hook/src/emqx_web_hook.app.src | 2 +- .../emqx_web_hook/src/emqx_web_hook.appup.src | 18 +++------- 4 files changed, 37 insertions(+), 30 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index ab74b877a..babc8cb17 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -12,22 +12,17 @@ File format: ## v4.3.19 -### Bug fixes -- Fix GET `/auth_clientid` and `/auth_username` counts. [#8655](https://github.com/emqx/emqx/pull/8655) - -## v4.3.19 - ### Enhancements - Improve error message for LwM2M plugin when object ID is not valid [#8654](https://github.com/emqx/emqx/pull/8654). - Add tzdata apk package to alpine docker image. [#8671](https://github.com/emqx/emqx/pull/8671) -- Add node evacuation and cluster rebalancing features [#8597] - -## v4.3.19 +- Add node evacuation and cluster rebalancing features [#8597](https://github.com/emqx/emqx/pull/8597) +- Refine Rule Engine error log. RuleId will be logged when take action failed. [#8737](https://github.com/emqx/emqx/pull/8737) ### Bug fixes -- Add a idle timer for ExProto UDP client to avoid client leaking [#8628] +- Fix GET `/auth_clientid` and `/auth_username` counts. [#8655](https://github.com/emqx/emqx/pull/8655) +- Add a idle timer for ExProto UDP client to avoid client leaking [#8628](https://github.com/emqx/emqx/pull/8628) ## v4.3.18 diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 1c5fb0e7b..b113b2e22 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,10 +1,21 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.13",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, - {"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + [{"4.3.13", + [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {"4.3.12", + [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", @@ -166,10 +177,21 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.13",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, - {"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + [{"4.3.13", + [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {"4.3.12", + [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", diff --git a/apps/emqx_web_hook/src/emqx_web_hook.app.src b/apps/emqx_web_hook/src/emqx_web_hook.app.src index 43e28fe1a..efe41b3bb 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.app.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.app.src @@ -1,6 +1,6 @@ {application, emqx_web_hook, [{description, "EMQ X WebHook Plugin"}, - {vsn, "4.3.13"}, % strict semver, bump manually! + {vsn, "4.3.14"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_web_hook_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_web_hook/src/emqx_web_hook.appup.src b/apps/emqx_web_hook/src/emqx_web_hook.appup.src index fff4ec08c..a2f72f0e2 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.appup.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.appup.src @@ -1,12 +1,7 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{<<"4\\.3\\.[0-2]">>, - [{apply,{application,stop,[emqx_web_hook]}}, - {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, - {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[3-7]">>, + [{<<"4\\.3\\.[0-7]">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, @@ -26,15 +21,10 @@ {"4.3.11", [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]}, - {"4.3.12", + {<<"4\\.3\\.1[2-3]">>, [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{<<"4\\.3\\.[0-2]">>, - [{apply,{application,stop,[emqx_web_hook]}}, - {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, - {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[3-7]">>, + [{<<"4\\.3\\.[0-7]">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, @@ -54,6 +44,6 @@ {"4.3.11", [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]}, - {"4.3.12", + {<<"4\\.3\\.1[2-3]">>, [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. From 0502be6055f8841b5d91734110e945051b6314bc Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 17 Aug 2022 15:36:04 +0800 Subject: [PATCH 35/73] chore(typo): fix typo --- apps/emqx_rule_engine/src/emqx_rule_utils.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_utils.erl b/apps/emqx_rule_engine/src/emqx_rule_utils.erl index 8bd46a958..5c9472367 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_utils.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_utils.erl @@ -397,7 +397,7 @@ gen_metadata_from_action(#action_instance{name = Name}) -> log_action(Level, Metadata, Fmt, Args) -> ?LOG(Level, - "Rule: ~p; Action: ~p; Rusource: ~p. " ++ Fmt, + "Rule: ~p; Action: ~p; Resource: ~p. " ++ Fmt, metadata_values(Metadata) ++ Args). metadata_values(Metadata) -> From 768ab4eacdd60d2edb1e1c6ec1958ec2a91c54fb Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 17 Aug 2022 16:03:42 +0800 Subject: [PATCH 36/73] fix(bridge): mqtt bridge worker status idle --- .../src/emqx_bridge_mqtt.app.src | 2 +- .../src/emqx_bridge_mqtt.appup.src | 4 ++-- .../src/emqx_bridge_mqtt_actions.erl | 23 +++++++++++++++---- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src index 31c795ff5..578671f4e 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mqtt, [{description, "EMQ X Bridge to MQTT Broker"}, - {vsn, "4.3.5"}, % strict semver, bump manually! + {vsn, "4.3.6"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,replayq,emqtt]}, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src index 9e1eda7f4..a72a18658 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.4", + [{<<"4\\.3\\.[4-5]">>, [{load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]}, {"4.3.3", [{load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}, @@ -14,7 +14,7 @@ {load_module,emqx_bridge_worker,brutal_purge,soft_purge,[]}, {load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.4", + [{<<"4\\.3\\.[4-5]">>, [{load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]}, {"4.3.3", [{load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl index 941ad51d0..a2b88352e 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl @@ -433,7 +433,7 @@ test_resource_status(PoolName) -> try Status = [ receive {Pid, R} -> R - after 1000 -> %% get_worker_status/1 should be a quick operation + after 10000 -> %% get_worker_status/1 should be a quick operation throw({timeout, Pid}) end || Pid <- Pids], lists:any(fun(St) -> St =:= true end, Status) @@ -444,13 +444,28 @@ test_resource_status(PoolName) -> false end. +-define(RETRY_TIMES, 4). + get_worker_status(Worker) -> + get_worker_status(Worker, ?RETRY_TIMES). + +get_worker_status(_Worker, 0) -> + false; +get_worker_status(Worker, Times) -> case ecpool_worker:client(Worker) of {ok, Bridge} -> try emqx_bridge_worker:status(Bridge) of - connected -> true; - _ -> false - catch _Error:_Reason -> + connected -> + true; + idle -> + ?LOG(info, "MQTT Bridge get status idle. Should not ignore this."), + timer:sleep(100), + get_worker_status(Worker, Times - 1); + ErrorStatus -> + ?LOG(error, "MQTT Bridge get status ~p", [ErrorStatus]), + false + catch Error:Reason:ST -> + ?LOG(error, "MQTT Bridge get status error: ~p reason: ~p stacktrace: ~p", [Error, Reason, ST]), false end; {error, _} -> From 58e24c2fde0f08bdeadeb23e5533e5a3a2a8cb58 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Aug 2022 16:27:52 +0800 Subject: [PATCH 37/73] chore: bump eredis to 1.2.8 to fix reconnect port leak --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index daddc8b97..799631dc7 100644 --- a/rebar.config +++ b/rebar.config @@ -41,7 +41,7 @@ [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps , {redbug, "2.0.7"} , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.2.0"}}} - , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.2"}}} + , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.3"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} From 2a50daa98b4ac7d972d2e5a9ade53cfbd46d6c52 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 17 Aug 2022 11:45:53 +0200 Subject: [PATCH 38/73] chore: sync ee emqx.appup.src to ce main-v4.3-enterprise had some ee specific changes added to emqx.erl having emqx module added to reload even there is no change is OK so we can simply sync the file from ee to ce --- src/emqx.appup.src | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index b2be51df5..fe65271fc 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,17 +2,18 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.19", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.18", - [{load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.3.17", - [{load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, @@ -22,6 +23,7 @@ {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, {"4.3.16", [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {update,emqx_broker_sup,supervisor}, @@ -39,6 +41,7 @@ {load_module,emqx_topic,brutal_purge,soft_purge,[]}]}, {"4.3.15", [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, @@ -693,17 +696,18 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.19", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.18", - [{load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.3.17", - [{load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, @@ -713,6 +717,7 @@ {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, {"4.3.16", [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {update,emqx_broker_sup,supervisor}, @@ -730,6 +735,7 @@ {delete_module,emqx_exclusive_subscription}]}, {"4.3.15", [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, From 719f6cfb2c5ad43a38a04036187afcc2220276ca Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 16 Aug 2022 10:48:48 +0800 Subject: [PATCH 39/73] fix(jwt): change `request_jwks` to be called after initialization If called in `init/1`, the module-enabled API may have inconsistent state due to timeout --- apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl | 60 ++++++++++++-------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl index 08c60d8ed..ac07a8640 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl @@ -73,8 +73,9 @@ verify(JwsCompacted) when is_binary(JwsCompacted) -> init([Options]) -> ok = jose:json_module(jiffy), _ = ets:new(?TAB, [set, protected, named_table]), - {Static, Remote} = do_init_jwks(Options), - true = ets:insert(?TAB, [{static, Static}, {remote, Remote}]), + Static = do_init_jwks(Options), + to_request_jwks(Options), + true = ets:insert(?TAB, [{static, Static}, {remote, undefined}]), Intv = proplists:get_value(interval, Options, ?INTERVAL), {ok, reset_timer( #state{ @@ -83,29 +84,13 @@ init([Options]) -> %% @private do_init_jwks(Options) -> - K2J = fun(K, F) -> - case proplists:get_value(K, Options) of - undefined -> undefined; - V -> - try F(V) of - {error, Reason} -> - ?LOG(warning, "Build ~p JWK ~p failed: {error, ~p}~n", - [K, V, Reason]), - undefined; - J -> J - catch T:R -> - ?LOG(warning, "Build ~p JWK ~p failed: {~p, ~p}~n", - [K, V, T, R]), - undefined - end - end - end, - OctJwk = K2J(secret, fun(V) -> - jose_jwk:from_oct(list_to_binary(V)) - end), - PemJwk = K2J(pubkey, fun jose_jwk:from_pem_file/1), - Remote = K2J(jwks_addr, fun request_jwks/1), - {[J ||J <- [OctJwk, PemJwk], J /= undefined], Remote}. + OctJwk = key2jwt_value(secret, + fun(V) -> + jose_jwk:from_oct(list_to_binary(V)) + end, + Options), + PemJwk = key2jwt_value(pubkey, fun jose_jwk:from_pem_file/1, Options), + [J ||J <- [OctJwk, PemJwk], J /= undefined]. handle_call(_Req, _From, State) -> {reply, ok, State}. @@ -122,6 +107,11 @@ handle_info({timeout, _TRef, refresh}, State = #state{addr = Addr}) -> end, {noreply, reset_timer(NState)}; +handle_info({request_jwks, Options}, State) -> + Remote = key2jwt_value(jwks_addr, fun request_jwks/1, Options), + true = ets:insert(?TAB, {remote, Remote}), + {noreply, State}; + handle_info(_Info, State) -> {noreply, State}. @@ -249,3 +239,23 @@ do_check_claim([{K, F}|More], Claims) -> _ -> do_check_claim(More, Claims) end. + +to_request_jwks(Options) -> + erlang:send(self(), {request_jwks, Options}). + +key2jwt_value(Key, Func, Options) -> + case proplists:get_value(Key, Options) of + undefined -> undefined; + V -> + try Func(V) of + {error, Reason} -> + ?LOG(warning, "Build ~p JWK ~p failed: {error, ~p}~n", + [Key, V, Reason]), + undefined; + J -> J + catch T:R -> + ?LOG(warning, "Build ~p JWK ~p failed: {~p, ~p}~n", + [Key, V, T, R]), + undefined + end + end. From b9d26506a64149a989f0b46728011ef1810f164d Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 16 Aug 2022 10:55:24 +0800 Subject: [PATCH 40/73] chore(jwt): bump version && update appup --- apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src | 2 +- apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src index 32ca1fc14..72e6e749b 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_jwt, [{description, "EMQ X Authentication with JWT"}, - {vsn, "4.3.4"}, % strict semver, bump manually! + {vsn, "4.3.5"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_jwt_sup]}, {applications, [kernel,stdlib,jose]}, diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src index 2538187fd..2364f901e 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src @@ -1,9 +1,11 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.3",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, + [{"4.3.4",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, + {"4.3.3",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-2]">>,[{restart_application,emqx_auth_jwt}]}, {<<".*">>,[]}], - [{"4.3.3",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, + [{"4.3.4",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, + {"4.3.3",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-2]">>,[{restart_application,emqx_auth_jwt}]}, {<<".*">>,[]}]}. From 1bcc58e2987acb8ccadfe486bbc7dd8abc32b13d Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 17 Aug 2022 17:59:16 +0800 Subject: [PATCH 41/73] chore: update CHANGES-4.3.md --- CHANGES-4.3.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index babc8cb17..2eb3d93a1 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -18,6 +18,7 @@ File format: - Add tzdata apk package to alpine docker image. [#8671](https://github.com/emqx/emqx/pull/8671) - Add node evacuation and cluster rebalancing features [#8597](https://github.com/emqx/emqx/pull/8597) - Refine Rule Engine error log. RuleId will be logged when take action failed. [#8737](https://github.com/emqx/emqx/pull/8737) +- Improved jwt authentication module initialization process.[#8736](https://github.com/emqx/emqx/pull/8736) ### Bug fixes From e1615e86df2741141ca41d86018252da34643fd1 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 17 Aug 2022 18:14:29 +0800 Subject: [PATCH 42/73] chore: update the change log --- CHANGES-4.3.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index ab74b877a..6023772a3 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -13,7 +13,12 @@ File format: ## v4.3.19 ### Bug fixes + - Fix GET `/auth_clientid` and `/auth_username` counts. [#8655](https://github.com/emqx/emqx/pull/8655) +- Fix rule SQL compare to null values always returns false. [#8743](https://github.com/emqx/emqx/pull/8743) + Before this change, the following SQL failed to match on the WHERE clause (`clientid != foo` returns false): + `SELECT 'some_var' as clientid FROM "t" WHERE clientid != foo`. + The `foo` variable is a null value, so `clientid != foo` should be evaluated as true. ## v4.3.19 From 1d43c11e5b6ce39bced1048a6c49fa7ef3bc75be Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Aug 2022 20:27:41 +0800 Subject: [PATCH 43/73] chore: bump to v4.3.19-beta.1 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 305815de1..404532b87 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.18"}). +-define(EMQX_RELEASE, {opensource, "4.3.19-beta.1"}). -else. From 117348fb858d2ca368751fe44b60ace8c10d09a4 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Aug 2022 22:05:22 +0800 Subject: [PATCH 44/73] feat(rebalance): remove unannounced plugins from plugin lists --- src/emqx_plugins.erl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index 8b5b7ffb2..97ec9a88a 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -235,9 +235,7 @@ default_plugins() -> {emqx_recon, true}, {emqx_telemetry, true}, {emqx_rule_engine, true}, - {emqx_bridge_mqtt, false}, - {emqx_eviction_agent, true}, - {emqx_node_rebalance, true} + {emqx_bridge_mqtt, false} ]. -else. From a583c221f0abacda5dea9c9069462804584981dc Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Aug 2022 22:51:02 +0800 Subject: [PATCH 45/73] fix: list_listener crash with invaild json --- CHANGES-4.3.md | 3 ++- apps/emqx_management/src/emqx_mgmt_api_listeners.erl | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index babc8cb17..ea45e0d69 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -22,7 +22,8 @@ File format: ### Bug fixes - Fix GET `/auth_clientid` and `/auth_username` counts. [#8655](https://github.com/emqx/emqx/pull/8655) -- Add a idle timer for ExProto UDP client to avoid client leaking [#8628](https://github.com/emqx/emqx/pull/8628) +- Add an idle timer for ExProto UDP client to avoid client leaking [#8628](https://github.com/emqx/emqx/pull/8628) +- Fix GET `/listeners/` crashes when listener is not ready. [#8752](https://github.com/emqx/emqx/pull/8752) ## v4.3.18 diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index 461c91d84..10ad9f4d4 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -72,5 +72,4 @@ format(Listeners) when is_list(Listeners) -> [ Info#{listen_on => list_to_binary(esockd:to_string(ListenOn))} || Info = #{listen_on := ListenOn} <- Listeners ]; -format({error, Reason}) -> [{error, Reason}]. - +format({error, Reason}) -> [{error, iolist_to_binary(io_lib:format("~p", [Reason]))}]. From 8a78c8a2f548d40d9478e1958c3135bafa725943 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Aug 2022 23:37:50 +0800 Subject: [PATCH 46/73] chore: add emqx_app to 4.3.19 --- src/emqx.appup.src | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index fe65271fc..5b6b88e61 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,7 +2,8 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.19", - [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.18", @@ -696,7 +697,8 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.19", - [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.18", From 32a0aebf779125294393233bc99afc59d620dc6a Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Aug 2022 23:52:19 +0800 Subject: [PATCH 47/73] chore: bump to v4.4.8-beta.1 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 261b38af0..66476d893 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.8"}). +-define(EMQX_RELEASE, {opensource, "4.4.8-beta.1"}). -else. From 10cabddb0793599ba4457cbfea0ffb2342e1b4ce Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 17 Aug 2022 16:26:36 +0200 Subject: [PATCH 48/73] ci: update OTP version --- .ci/build_packages/Dockerfile | 2 +- .ci/docker-compose-file/docker-compose.yaml | 2 +- .github/workflows/apps_version_check.yaml | 2 +- .github/workflows/build_packages.yaml | 4 ++-- .github/workflows/build_slim_packages.yaml | 4 ++-- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/release.yaml | 2 +- .github/workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 8 ++++---- .github/workflows/run_test_cases.yaml | 2 +- .tool-versions | 2 +- Makefile | 2 +- Windows.md | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile index 3c7e401ae..b56cb5220 100644 --- a/.ci/build_packages/Dockerfile +++ b/.ci/build_packages/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 +ARG BUILD_FROM=emqx/build-env:erl23.3.4.9-3-ubuntu20.04 FROM ${BUILD_FROM} ARG EMQX_NAME=emqx diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 6b5c1b602..918be18fd 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + image: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 env_file: - conf.env environment: diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index cb2c7a26b..86fe60204 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -9,7 +9,7 @@ jobs: strategy: matrix: erl_otp: - - erl23.2.7.2-emqx-3 + - erl23.3.4.9-3 os: - ubuntu20.04 diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 19ed368a6..2571746bd 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -18,7 +18,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} @@ -125,7 +125,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} erl_otp: - - 23.2.7.2-emqx-3 + - 23.3.4.9-3 exclude: - profile: emqx-edge macos: diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 713ccc605..17e99c831 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -11,7 +11,7 @@ jobs: strategy: matrix: erl_otp: - - erl23.2.7.2-emqx-3 + - erl23.3.4.9-3 os: - ubuntu20.04 - centos7 @@ -55,7 +55,7 @@ jobs: strategy: matrix: erl_otp: - - 23.2.7.2-emqx-3 + - 23.3.4.9-3 macos: - macos-11 runs-on: ${{ matrix.macos }} diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index e58afcc1a..07ec47754 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8e6b02c21..bdbd7e982 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,7 +7,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index 0f001ac04..bbaed018f 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index ef76e8108..601ab304f 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v1 - uses: erlef/setup-beam@v1 with: - otp-version: "23.2" + otp-version: "23.3.4.9" - name: make docker run: | if make emqx-ee --dry-run > /dev/null 2>&1; then @@ -67,7 +67,7 @@ jobs: - uses: actions/checkout@v1 - uses: erlef/setup-beam@v1 with: - otp-version: "23.2" + otp-version: "23.3.4.9" - name: prepare run: | if make emqx-ee --dry-run > /dev/null 2>&1; then @@ -176,7 +176,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -222,7 +222,7 @@ jobs: relup_test_build: needs: relup_test_plan runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 defaults: run: shell: bash diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 6110c587c..b7869c74c 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,7 +10,7 @@ on: jobs: run_proper_test: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.tool-versions b/.tool-versions index 0b6392b95..757309f18 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -erlang 23.2.7.2-emqx-3 +erlang 23.3.4.9-3 diff --git a/Makefile b/Makefile index 0fb5c63e5..f7f46c48f 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true -export EMQX_DEFAULT_BUILDER = emqx/build-env:erl23.2.7.2-emqx-3-alpine +export EMQX_DEFAULT_BUILDER = emqx/build-env:erl23.3.4.9-3-alpine export EMQX_DEFAULT_RUNNER = alpine:3.12 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export DOCKERFILE := deploy/docker/Dockerfile diff --git a/Windows.md b/Windows.md index 6b6eff60a..ff49365e0 100644 --- a/Windows.md +++ b/Windows.md @@ -29,7 +29,7 @@ The second path is for CMD to setup environment variables. ### Erlang/OTP -Install Erlang/OTP 23.2 from https://www.erlang.org/downloads +Install Erlang/OTP 23.3 from https://www.erlang.org/downloads You may need to edit the `Path` environment variable to allow running Erlang commands such as `erl` from CMD. From a645a8870099b7917a9caced1d17e260058cab6f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 17 Aug 2022 17:01:23 +0200 Subject: [PATCH 49/73] ci: skip relup test from 4.3.13 --- .ci/build_packages/tests.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index d4620e2d3..93844b3e2 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -139,6 +139,10 @@ relup_test(){ find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.zip" | while read -r pkg; do + if [[ "${pkg}" == *4.3.13* ]]; then + echo "skipping upgrade test from 4.3.13 because this release had crypto linked with openssl 1.1.1n, it was in later version rolled back (to default 1.1.1k)." + continue + fi packagename=$(basename "${pkg}") unzip -q "$packagename" ./emqx/bin/emqx start || ( tail emqx/log/emqx.log.1 && exit 1 ) From f1ad4cb8cab7f9c8aec6ed852856f1eefe02c821 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 17 Aug 2022 23:40:48 +0200 Subject: [PATCH 50/73] chore: fix emqx_misc ipv6 probe The new OTP version exports gen_tcp:ipv6_probe/0 which always return true. So we no longer need the old hacky way to do it. --- src/emqx.appup.src | 24 ++++++++++++++++-------- src/emqx_misc.erl | 13 +++---------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 5b6b88e61..27c7d3e1a 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,17 +2,20 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.19", - [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.18", - [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.3.17", - [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]}, @@ -23,7 +26,8 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, {"4.3.16", - [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, @@ -697,17 +701,20 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.19", - [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.18", - [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.3.17", - [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]}, @@ -718,7 +725,8 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, {"4.3.16", - [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_misc.erl b/src/emqx_misc.erl index 36a3386e3..813ab84e8 100644 --- a/src/emqx_misc.erl +++ b/src/emqx_misc.erl @@ -84,16 +84,9 @@ maybe_parse_ip(Host) -> %% @doc Add `ipv6_probe' socket option if it's supported. ipv6_probe(Opts) -> - case persistent_term:get({?MODULE, ipv6_probe_supported}, unknown) of - unknown -> - %% e.g. 23.2.7.1-emqx-2-x86_64-unknown-linux-gnu-64 - OtpVsn = emqx_vm:get_otp_version(), - Bool = (match =:= re:run(OtpVsn, "emqx", [{capture, none}])), - _ = persistent_term:put({?MODULE, ipv6_probe_supported}, Bool), - ipv6_probe(Bool, Opts); - Bool -> - ipv6_probe(Bool, Opts) - end. + Bool = try gen_tcp:ipv6_probe() + catch _ : _ -> false end, + ipv6_probe(Bool, Opts). ipv6_probe(false, Opts) -> Opts; ipv6_probe(true, Opts) -> [{ipv6_probe, true} | Opts]. From 413612a69de1a8ad8bdcee773775c388b103269c Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 18 Aug 2022 09:18:41 +0800 Subject: [PATCH 51/73] fix: duplicate appup instructions --- .../src/emqx_rule_engine.appup.src | 1 - .../test/emqx_rule_engine_SUITE.erl | 28 ++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index ff2ddc4c9..f50c1a368 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -190,7 +190,6 @@ {"4.3.11", [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index e1851b76c..d1e6b26a0 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -126,6 +126,7 @@ groups() -> t_sqlparse_array_range_1, t_sqlparse_array_range_2, t_sqlparse_true_false, + t_sqlparse_compare_undefined, t_sqlparse_compare_null_null, t_sqlparse_compare_null_notnull, t_sqlparse_compare_notnull_null, @@ -2493,8 +2494,33 @@ t_sqlparse_true_false(_Config) -> -define(TEST_SQL(SQL), emqx_rule_sqltester:test( #{<<"rawsql">> => SQL, - <<"ctx">> => #{<<"payload">> => <<"">>, + <<"ctx">> => #{<<"payload">> => <<"{}">>, <<"topic">> => <<"t/a">>}})). + +t_sqlparse_compare_undefined(_Config) -> + Sql00 = "select " + " * " + "from \"t/#\" " + "where dev != undefined ", + %% no match + ?assertMatch({error, nomatch}, ?TEST_SQL(Sql00)), + + Sql01 = "select " + " 'd' as dev " + "from \"t/#\" " + "where dev != undefined ", + {ok, Res01} = ?TEST_SQL(Sql01), + %% pass + ?assertMatch(#{}, Res01), + + Sql02 = "select " + " * " + "from \"t/#\" " + "where dev != 'undefined' ", + {ok, Res02} = ?TEST_SQL(Sql02), + %% pass + ?assertMatch(#{}, Res02). + t_sqlparse_compare_null_null(_Config) -> %% test undefined == undefined Sql00 = "select " From da05a2ad305f8839999172e58e454eaf2b9d39ff Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 18 Aug 2022 07:30:19 +0200 Subject: [PATCH 52/73] test: delete unknown plugin --- test/emqx_plugins_SUITE.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl index eef5152b0..f3c37afea 100644 --- a/test/emqx_plugins_SUITE.erl +++ b/test/emqx_plugins_SUITE.erl @@ -94,10 +94,8 @@ default_plugins() -> [ {emqx_bridge_mqtt, false}, {emqx_dashboard, true}, - {emqx_eviction_agent, true}, {emqx_management, true}, {emqx_modules, false}, - {emqx_node_rebalance, true}, {emqx_recon, true}, {emqx_retainer, true}, {emqx_rule_engine, true}, From b4ea2aefa6e6841c873fbb5354eca2ded8cea7fa Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 18 Aug 2022 07:35:40 +0200 Subject: [PATCH 53/73] test: export ct callbacks --- .../test/emqx_auth_mnesia_data_export_import_SUITE.erl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl index 021cf735a..da27fa2ee 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl @@ -70,4 +70,10 @@ remove_all_users_and_acl() -> mnesia:delete_table(emqx_user), mnesia:delete_table(emqx_acl). +-else. + +%% opensource edition + +all() -> []. + -endif. From 58db1eb5a9379dfa5a18f2c29eb4e7f74ca8a337 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 15 Aug 2022 14:41:47 +0800 Subject: [PATCH 54/73] fix(exhook): avoid emqx_exhook_mgnr to force killed due to exceed supervior shutdown timeout --- apps/emqx_exhook/src/emqx_exhook_mngr.erl | 15 +++++++++++++-- apps/emqx_exhook/src/emqx_exhook_server.erl | 6 ++++-- apps/emqx_exproto/src/emqx_exproto_gcli.erl | 9 ++++++++- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook_mngr.erl b/apps/emqx_exhook/src/emqx_exhook_mngr.erl index bd8b88e0a..f4296a676 100644 --- a/apps/emqx_exhook/src/emqx_exhook_mngr.erl +++ b/apps/emqx_exhook/src/emqx_exhook_mngr.erl @@ -192,6 +192,17 @@ handle_info({timeout, _Ref, {reload, Name}}, State) -> {noreply, NState}; {error, not_found} -> {noreply, NState}; + {error, {already_started, Pid}} -> + ?LOG(warning, "Server ~s already started on ~p, try to restart it", [Name, Pid]), + case server(Name) of + undefined -> + %% force close grpc client pool + grpc_client_sup:stop_channel_pool(Name); + ServerState -> + emqx_exhook_server:unload(ServerState) + end, + %% try again immediately + handle_info({timeout, _Ref, {reload, Name}}, State); {error, Reason} -> ?LOG(warning, "Failed to reload exhook callback server \"~s\", " "Reason: ~0p", [Name, Reason]), @@ -202,11 +213,11 @@ handle_info(_Info, State) -> {noreply, State}. terminate(_Reason, State = #state{running = Running}) -> + _ = unload_exhooks(), _ = maps:fold(fun(Name, _, AccIn) -> {ok, NAccIn} = do_unload_server(Name, AccIn), NAccIn end, State, Running), - _ = unload_exhooks(), ok. %% in the emqx_exhook:v4.3.5, we have added one new field in the state last: @@ -318,7 +329,7 @@ get_request_failed_action() -> save(Name, ServerState) -> Saved = persistent_term:get(?APP, []), - persistent_term:put(?APP, lists:reverse([Name | Saved])), + persistent_term:put(?APP, lists:usort(lists:reverse([Name | Saved]))), persistent_term:put({?APP, Name}, ServerState). unsave(Name) -> diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index 93568783a..5ce602535 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -146,13 +146,15 @@ format_http_uri(Scheme, Host0, Port) -> -spec unload(server()) -> ok. unload(#server{name = Name, options = ReqOpts, hookspec = HookSpecs}) -> - _ = do_deinit(Name, ReqOpts), _ = may_unload_hooks(HookSpecs), + _ = do_deinit(Name, ReqOpts), _ = emqx_exhook_sup:stop_grpc_client_channel(Name), ok. do_deinit(Name, ReqOpts) -> - _ = do_call(Name, 'on_provider_unloaded', #{}, ReqOpts), + %% Using shorter timeout to deinit grpc server to avoid emqx_exhook_mngr + %% force killed by upper supervisor + _ = do_call(Name, 'on_provider_unloaded', #{}, ReqOpts#{timeout => 3000}), ok. do_init(ChannName, ReqOpts) -> diff --git a/apps/emqx_exproto/src/emqx_exproto_gcli.erl b/apps/emqx_exproto/src/emqx_exproto_gcli.erl index b7ef20c13..f7fa6ba23 100644 --- a/apps/emqx_exproto/src/emqx_exproto_gcli.erl +++ b/apps/emqx_exproto/src/emqx_exproto_gcli.erl @@ -54,7 +54,13 @@ start_link(Pool, Id) -> ?MODULE, [Pool, Id], []). async_call(FunName, Req = #{conn := Conn}, Options) -> - cast(pick(Conn), {rpc, FunName, Req, Options, self()}). + case pick(Conn) of + false -> + ?LOG(error, "No available grpc client for ~s: ~p", + [FunName, Req]); + Pid when is_pid(Pid) -> + cast(Pid, {rpc, FunName, Req, Options, self()}) + end. %%-------------------------------------------------------------------- %% cast, pick @@ -65,6 +71,7 @@ async_call(FunName, Req = #{conn := Conn}, Options) -> cast(Deliver, Msg) -> gen_server:cast(Deliver, Msg). +-spec pick(term()) -> pid() | false. pick(Conn) -> gproc_pool:pick_worker(exproto_gcli_pool, Conn). From 515fd014d3b68d82991b89836ac9aca390bf4719 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 15 Aug 2022 13:59:16 +0800 Subject: [PATCH 55/73] fix(exproto): fix undefined clientid in client.connect hook --- apps/emqx_exproto/src/emqx_exproto_channel.erl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl index 765dcf53e..30f283d9d 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -151,7 +151,7 @@ init(ConnInfo = #{socktype := Socktype, peercert := Peercert}, Options) -> GRpcChann = proplists:get_value(handler, Options), NConnInfo = default_conninfo(ConnInfo), - ClientInfo = default_clientinfo(ConnInfo), + ClientInfo = default_clientinfo(NConnInfo), IdleTimeout = proplists:get_value(idle_timeout, Options, ?DEFAULT_IDLE_TIMEOUT), @@ -633,9 +633,10 @@ enrich_clientinfo(InClientInfo = #{proto_name := ProtoName}, ClientInfo) -> NClientInfo = maps:merge(ClientInfo, maps:with(Ks, InClientInfo)), NClientInfo#{protocol => ProtoName}. -default_conninfo(ConnInfo) -> +default_conninfo(ConnInfo = + #{peername := {PeerHost, PeerPort}}) -> ConnInfo#{clean_start => true, - clientid => undefined, + clientid => anonymous_clientid(PeerHost, PeerPort), username => undefined, conn_props => #{}, connected => true, @@ -646,13 +647,15 @@ default_conninfo(ConnInfo) -> receive_maximum => 0, expiry_interval => 0}. -default_clientinfo(#{peername := {PeerHost, PeerPort}, - sockname := {_, SockPort}}) -> +default_clientinfo(#{peername := {PeerHost, _}, + sockname := {_, SockPort}, + clientid := ClientId + }) -> #{zone => external, protocol => exproto, peerhost => PeerHost, sockport => SockPort, - clientid => anonymous_clientid(PeerHost, PeerPort), + clientid => ClientId, username => undefined, is_bridge => false, is_superuser => false, From 7d3ea85ef3a0762d897080bc1a10def722124057 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 9 Aug 2022 11:18:45 +0800 Subject: [PATCH 56/73] fix(exproto): produce disconnected event if kicked --- apps/emqx_exproto/src/emqx_exproto_channel.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl index 30f283d9d..50dd203b6 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -381,7 +381,7 @@ handle_call({publish, Topic, Qos, Payload}, end; handle_call(kick, Channel) -> - {shutdown, kicked, ok, Channel}; + {reply, ok, [{event, disconnected}, {close, kicked}], Channel}; handle_call(discard, Channel) -> {shutdown, discarded, ok, Channel}; From 8186e9e47a62d9350d054cd78d46a778c6c46225 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 9 Aug 2022 11:04:51 +0800 Subject: [PATCH 57/73] chore: close keepalive timeout channel --- apps/emqx_exproto/src/emqx_exproto_channel.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl index 50dd203b6..cf6229be3 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -275,7 +275,9 @@ handle_timeout(_TRef, {keepalive, StatVal}, {error, timeout} -> Req = #{type => 'KEEPALIVE'}, NChannel = clean_timer(alive_timer, Channel), - {ok, try_dispatch(on_timer_timeout, wrap(Req), NChannel)} + %% close connection if keepalive timeout + Replies = [{event, disconnected}, {close, normal}], + {ok, Replies, try_dispatch(on_timer_timeout, wrap(Req), NChannel)} end; handle_timeout(_TRef, force_close, Channel = #channel{closed_reason = Reason}) -> From 5ba048f78729762a0909a33e12ed1917cbe8241e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 15 Aug 2022 15:01:43 +0800 Subject: [PATCH 58/73] chore: upgrade grpc-erl to 0.6.7 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 799631dc7..074949bbf 100644 --- a/rebar.config +++ b/rebar.config @@ -62,7 +62,7 @@ , {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.1"}}} , {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.13"}}} , {epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}} - , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.6"}}} + , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} ]}. {xref_ignores, From 5e505fa41c997d0d8ddf07f05218ba8aefbc4d41 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 15 Aug 2022 15:23:47 +0800 Subject: [PATCH 59/73] chore: update appup.src --- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- apps/emqx_exhook/src/emqx_exhook.appup.src | 11 +++++++++-- apps/emqx_exproto/src/emqx_exproto.appup.src | 6 ++++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index cf39d9e42..d0067742a 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [{description, "EMQ X Extension for Hook"}, - {vsn, "4.3.6"}, + {vsn, "4.3.7"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index f286255f0..d5a848fb1 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -1,8 +1,10 @@ %% -*- mode: erlang -*- {VSN, [ - {"4.3.5", [ - {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} + {<<"4\\.3\\.[5-6]">>, [ + {load_module, emqx_exhook_server, brutal_purge, soft_purge, []}, + {load_module, emqx_exhook_mngr, brutal_purge, soft_purge, []} + ]}, {"4.3.4", [ {load_module, emqx_exhook_sup, brutal_purge, soft_purge, []}, @@ -18,6 +20,11 @@ {<<".*">>, []} ], [ + {<<"4\\.3\\.[5-6]">>, [ + {load_module, emqx_exhook_server, brutal_purge, soft_purge, []}, + {load_module, emqx_exhook_mngr, brutal_purge, soft_purge, []} + + ]}, {"4.3.5", [ {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} ]}, diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index 8e0380089..d7414e53e 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -2,7 +2,8 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.9", - [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[2-8]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, @@ -13,7 +14,8 @@ {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.9", - [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[2-8]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, From a66dc7c02b98e7fcfba2bffccdc71c08683c287a Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 15 Aug 2022 15:14:19 +0800 Subject: [PATCH 60/73] chore: update changes-4.3 --- CHANGES-4.3.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 8a031efa8..2c7fce55f 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -18,6 +18,8 @@ File format: - Add tzdata apk package to alpine docker image. [#8671](https://github.com/emqx/emqx/pull/8671) - Add node evacuation and cluster rebalancing features [#8597](https://github.com/emqx/emqx/pull/8597) - Refine Rule Engine error log. RuleId will be logged when take action failed. [#8737](https://github.com/emqx/emqx/pull/8737) +- Close ExProto client process immediately if it's keepalive timeouted. [#8725](https://github.com/emqx/emqx/pull/8725) +- Upgrade grpc-erl driver to 0.6.7 to support batch operation in sending stream. [#8725](https://github.com/emqx/emqx/pull/8725) - Improved jwt authentication module initialization process.[#8736](https://github.com/emqx/emqx/pull/8736) ### Bug fixes @@ -28,8 +30,10 @@ File format: The `foo` variable is a null value, so `clientid != foo` should be evaluated as true. - Fix GET `/auth_clientid` and `/auth_username` counts. [#8655](https://github.com/emqx/emqx/pull/8655) - Add an idle timer for ExProto UDP client to avoid client leaking [#8628](https://github.com/emqx/emqx/pull/8628) +- Fix ExHook can't be un-hooked if the grpc service stop first. [#8725](//github.com/emqx/emqx/pull/8725) - Fix GET `/listeners/` crashes when listener is not ready. [#8752](https://github.com/emqx/emqx/pull/8752) + ## v4.3.18 ### Enhancements From f4ad7acd06d75519db90b59fbf469763fcef0497 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 17 Aug 2022 18:32:44 +0800 Subject: [PATCH 61/73] chore: update appup.src --- apps/emqx_exhook/src/emqx_exhook.appup.src | 70 ++++++++------------ apps/emqx_exproto/src/emqx_exproto.appup.src | 6 +- 2 files changed, 32 insertions(+), 44 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index d5a848fb1..eada3146e 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -1,44 +1,30 @@ %% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [ - {<<"4\\.3\\.[5-6]">>, [ - {load_module, emqx_exhook_server, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook_mngr, brutal_purge, soft_purge, []} - - ]}, - {"4.3.4", [ - {load_module, emqx_exhook_sup, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook_server, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook_handler, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook, brutal_purge, soft_purge, []}, - {update, emqx_exhook_mngr, {advanced, ["4.3.4"]}} - ]}, - {<<"4\\.3\\.[0-3]">>, [ - {restart_application, emqx_exhook} - ]}, - {<<".*">>, []} - ], - [ - {<<"4\\.3\\.[5-6]">>, [ - {load_module, emqx_exhook_server, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook_mngr, brutal_purge, soft_purge, []} - - ]}, - {"4.3.5", [ - {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} - ]}, - {"4.3.4", [ - {load_module, emqx_exhook_sup, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook_server, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook_handler, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook, brutal_purge, soft_purge, []}, - {update, emqx_exhook_mngr, {advanced, ["4.3.4"]}} - ]}, - {<<"4\\.3\\.[0-3]">>, [ - {restart_application, emqx_exhook} - ]}, - {<<".*">>, []} - ] -}. + [{<<"4\\.3\\.[5-6]">>, + [{load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}]}, + {"4.3.4", + [{load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook,brutal_purge,soft_purge,[]}, + {update,emqx_exhook_mngr,{advanced,["4.3.4"]}}]}, + {<<"4\\.3\\.[0-3]">>,[{restart_application,emqx_exhook}]}, + {<<".*">>,[]}], + [{<<"4\\.3\\.[5-6]">>, + [{load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}]}, + {"4.3.5", + [{load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}]}, + {"4.3.4", + [{load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook,brutal_purge,soft_purge,[]}, + {update,emqx_exhook_mngr,{advanced,["4.3.4"]}}]}, + {<<"4\\.3\\.[0-3]">>,[{restart_application,emqx_exhook}]}, + {<<".*">>,[]}]}. diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index d7414e53e..20939ae63 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -5,7 +5,8 @@ [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[2-8]">>, - [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, + [{load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-1]">>, [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, @@ -17,7 +18,8 @@ [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[2-8]">>, - [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, + [{load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-1]">>, [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, From e8d495b17b00eb0050e70bfff402dd3a543a4b72 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 19 Aug 2022 16:35:20 +0800 Subject: [PATCH 62/73] chore: bump to v4.3.19-beta.2 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 404532b87..ceafd87e4 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.19-beta.1"}). +-define(EMQX_RELEASE, {opensource, "4.3.19-beta.2"}). -else. From 50f1d241dc47bd332e7aa6d3389ea9e8557cf672 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 19 Aug 2022 16:54:23 +0800 Subject: [PATCH 63/73] chore: bump dashboard to v4.3.10/v4.3.24 --- scripts/get-dashboard.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index bd596a8d2..f9a85a108 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -8,8 +8,8 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" case "${PKG_VSN}" in 4.3*) - EMQX_CE_DASHBOARD_VERSION='v4.3.9' - EMQX_EE_DASHBOARD_VERSION='v4.3.23' + EMQX_CE_DASHBOARD_VERSION='v4.3.10' + EMQX_EE_DASHBOARD_VERSION='v4.3.24' ;; *) echo "Unsupported version $PKG_VSN" >&2 From 8f40d7befdb787316e3f4cd3a63b96993fcc1b57 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 19 Aug 2022 16:56:20 +0800 Subject: [PATCH 64/73] chore: bump dashboard to v4.4.5/v4.4.15 --- scripts/get-dashboard.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index b4ddc5679..9361a2942 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -13,8 +13,8 @@ case "${PKG_VSN}" in ;; 4.4*) # keep the above 4.3 untouched, otherwise conflicts! - EMQX_CE_DASHBOARD_VERSION='v4.4.4' - EMQX_EE_DASHBOARD_VERSION='v4.4.14' + EMQX_CE_DASHBOARD_VERSION='v4.4.5' + EMQX_EE_DASHBOARD_VERSION='v4.4.15' ;; *) echo "Unsupported version $PKG_VSN" >&2 From efdde6e1078dba033e8f28fad981fc9bc6c41376 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 19 Aug 2022 18:13:59 +0800 Subject: [PATCH 65/73] test(exproto): fix timeout cases --- apps/emqx_exproto/test/emqx_exproto_SUITE.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl index a81cbe253..d4af25d54 100644 --- a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl @@ -213,11 +213,13 @@ t_keepalive_timeout(Cfg) -> send(Sock, ConnBin), {ok, ConnAckBin} = recv(Sock, 5000), - DisconnectBin = frame_disconnect(), - {ok, DisconnectBin} = recv(Sock, 10000), + %% Timed out connections are closed immediately, + %% so there may not be a disconnect message here + %%DisconnectBin = frame_disconnect(), + %%{ok, DisconnectBin} = recv(Sock, 10000), SockType =/= udp andalso begin - {error, closed} = recv(Sock, 5000) + {error, closed} = recv(Sock, 10000) end, ok. t_hook_connected_disconnected(Cfg) -> From 7ad7658bbcdb1e32fcf5095935a12657067ef1ca Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 22 Aug 2022 15:29:08 +0800 Subject: [PATCH 66/73] fix: remove emqx_telemetry from ee's default plugins --- src/emqx_plugins.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index 97ec9a88a..39fe41904 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -250,7 +250,8 @@ default_plugins() -> %% default is true in data/load_modules. **NOT HERE** {emqx_retainer, false}, {emqx_recon, true}, - {emqx_telemetry, true}, + %% emqx_telemetry is not exist in enterprise. + %% {emqx_telemetry, false}, {emqx_rule_engine, true}, {emqx_bridge_mqtt, false} ]. From 6f404052423752ebdff8f35aa56bad6fe5b8b822 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 18 Aug 2022 16:34:29 -0300 Subject: [PATCH 67/73] ci: add relup version db --- .github/workflows/apps_version_check.yaml | 4 + .github/workflows/build_slim_packages.yaml | 1 - build | 14 +- data/relup_bases.eterm | 32 +++ scripts/relup-base-packages.sh | 8 +- scripts/relup-base-vsns.escript | 269 +++++++++++++++++++++ scripts/relup-base-vsns.sh | 33 +-- 7 files changed, 329 insertions(+), 32 deletions(-) create mode 100644 data/relup_bases.eterm create mode 100755 scripts/relup-base-vsns.escript diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index d27c2b3d8..bf3544708 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -21,6 +21,10 @@ jobs: fetch-depth: 0 # need full history - name: fix-git-unsafe-repository run: git config --global --add safe.directory /__w/emqx/emqx + - name: Check relup version DB + run: | + PKG_VSN=$(./pkg-vsn.sh) + ./scripts/relup-base-vsns.escript check-vsn-db $PKG_VSN ./data/relup_bases.eterm - name: Check relup (ce) if: endsWith(github.repository, 'emqx') run: ./scripts/update-appup.sh emqx --check diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 970fc981f..d940760db 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -17,7 +17,6 @@ jobs: fail-fast: false matrix: erl_otp: - - 23.3.4.9-3 - 24.1.5-3 os: - ubuntu20.04 diff --git a/build b/build index 979af4510..d3c9916e1 100755 --- a/build +++ b/build @@ -66,15 +66,21 @@ make_rel() { ./rebar3 as "$PROFILE" tar } +relup_db() { + ./scripts/relup-base-vsns.escript "$@" ./data/relup_bases.eterm +} + ## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup make_relup() { local lib_dir="_build/$PROFILE/rel/emqx/lib" local releases_dir="_build/$PROFILE/rel/emqx/releases" local name_pattern - name_pattern="${PROFILE}-$(./scripts/pkg-full-vsn.sh 'vsn_matcher')" mkdir -p "$lib_dir" "$releases_dir" '_upgrade_base' local releases=() if [ -d "$releases_dir" ]; then + for BASE_VSN in $(relup_db base-vsns "$PKG_VSN"); do + OTP_BASE=$(relup_db otp-vsn-for "$PKG_VSN") + name_pattern="${PROFILE}-$(env OTP_VSN="$OTP_BASE" PKG_VSN="$BASE_VSN" ./scripts/pkg-full-vsn.sh 'vsn_exact')" while read -r zip; do local base_vsn base_vsn="$(echo "$zip" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-[0-9a-f]{8})?" | head -1)" @@ -89,6 +95,7 @@ make_relup() { fi releases+=( "$base_vsn" ) done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.zip" -type f) + done fi if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" @@ -181,6 +188,11 @@ make_zip() { esac ;; esac + # shellcheck disable=SC2207 + bases=($(relup_db base-vsns "$PKG_VSN")) + if [[ "${#bases[@]}" -eq 0 ]]; then + has_relup='no' + fi if [ "$has_relup" = 'yes' ]; then ./scripts/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup" fi diff --git a/data/relup_bases.eterm b/data/relup_bases.eterm new file mode 100644 index 000000000..1582cf75e --- /dev/null +++ b/data/relup_bases.eterm @@ -0,0 +1,32 @@ +%% -*- mode: erlang; -*- + +#{<<"4.4.0">> => #{from_versions => [],otp => <<"24.1.5-3">>}, + <<"4.4.1">> => #{from_versions => [<<"4.4.0">>],otp => <<"24.1.5-3">>}, + <<"4.4.2">> => + #{from_versions => [<<"4.4.0">>,<<"4.4.1">>],otp => <<"24.1.5-3">>}, + <<"4.4.3">> => + #{from_versions => [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>], + otp => <<"24.1.5-3">>}, + <<"4.4.4">> => + #{from_versions => [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>], + otp => <<"24.1.5-3">>}, + <<"4.4.5">> => + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>], + otp => <<"24.1.5-3">>}, + <<"4.4.6">> => + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, + <<"4.4.5">>], + otp => <<"24.1.5-3">>}, + <<"4.4.7">> => + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, + <<"4.4.5">>,<<"4.4.6">>], + otp => <<"24.1.5-3">>}, + <<"4.4.8">> => + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, + <<"4.4.5">>,<<"4.4.6">>,<<"4.4.7">>], + otp => <<"24.1.5-3">>}, + <<"4.5.0">> => #{from_versions => [<<"4.4.8">>],otp => <<"24.3.4.2-1">>}}. diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 9505a68f0..bc1d8977e 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -63,8 +63,12 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." mkdir -p _upgrade_base pushd _upgrade_base +otp_vsn_for() { + ../scripts/relup-base-vsns.escript otp-vsn-for "${1#[e|v]}" ../data/relup_bases.eterm +} + for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do - filename="$PROFILE-${tag#[e|v]}-otp$OTP_VSN-$SYSTEM-$ARCH.zip" + filename="$PROFILE-${tag#[e|v]}-otp$(otp_vsn_for "$tag")-$SYSTEM-$ARCH.zip" url="https://packages.emqx.io/$DIR/$tag/$filename" if [ ! -f "$filename" ] && curl -L -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then echo "downloading base package from ${url} ..." @@ -77,6 +81,8 @@ for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do ## https://askubuntu.com/questions/1202208/checking-sha256-checksum echo "${SUMSTR} ${filename}" | $SHASUM -c || exit 1 fi + else + echo "file $filename already downloaded or doesn't exist in the archives; skipping it" fi done diff --git a/scripts/relup-base-vsns.escript b/scripts/relup-base-vsns.escript new file mode 100755 index 000000000..1109b2555 --- /dev/null +++ b/scripts/relup-base-vsns.escript @@ -0,0 +1,269 @@ +#!/usr/bin/env escript +%% -*- mode: erlang; -*- + +-mode(compile). + +-define(RED, "\e[31m"). +-define(RESET, "\e[39m"). + +usage() -> +"A script to manage the released versions of EMQX for relup and hot +upgrade/downgrade. + +We store a \"database\" of released versions as an `eterm' file, which +is a mapping from a given version `Vsn' to its OTP version and a list +of previous versions from which one can upgrade to `Vsn' (the +\"from_versions\" list). That allow us to more easily/explicitly keep +track of allowed version upgrades/downgrades, as well as OTP changes +between releases + +In the examples below, `VERSION_DB_PATH' represents the path to the +`eterm' file containing the version database to be used. + +Usage: + + * List the previous base versions from which `TO_VSN' may be + upgraded to. Used to list versions for which relup files are to + be made. + + relup-base-vsns.escript base-vsns TO_VSN VERSION_DB_PATH + + + * Show the OTP version with which `Vsn' was built. + + relup-base-vsns.escript otp-vsn-for VSN VERSION_DB_PATH + + + * Automatically inserts a new version into the database. Previous + versions with the same Major and Minor numbers as `Vsn' are + considered to be upgradeable from, and versions with higher Major + and Minor numbers will automatically include `Vsn' in their + \"from_versions\" list. + + For example, if inserting 4.4.8 when 4.5.0 and 4.5.1 exists, + versions `BASE_FROM_VSN'...4.4.7 will be considered 4.4.8's + \"from_versions\", and 4.4.8 will be included into 4.5.0 and + 4.5.1's from versions. + + relup-base-vsns.escript insert-new-vsn NEW_VSN BASE_FROM_VSN OTP_VSN VERSION_DB_PATH + + * Check if the version database is consistent considering `VSN'. + + relup-base-vsns.escript check-vsn-db VSN VERSION_DB_PATH +". + +main(["base-vsns", To0, VsnDB]) -> + {ok, [VsnMap]} = file:consult(VsnDB), + To = strip_pre_release(To0), + #{from_versions := Froms} = fetch_version(To, VsnMap), + AvailableVersionsIndex = available_versions_index(), + lists:foreach( + fun(From) -> + io:format(user, "~s~n", [From]) + end, + filter_froms(Froms, AvailableVersionsIndex)), + halt(0); +main(["otp-vsn-for", Vsn0, VsnDB]) -> + {ok, [VsnMap]} = file:consult(VsnDB), + Vsn = strip_pre_release(Vsn0), + #{otp := OtpVsn} = fetch_version(Vsn, VsnMap), + io:format(user, "~s~n", [OtpVsn]), + halt(0); +main(["insert-new-vsn", NewVsn0, BaseFromVsn0, OtpVsn0, VsnDB]) -> + {ok, [VsnMap]} = file:consult(VsnDB), + NewVsn = strip_pre_release(NewVsn0), + validate_version(NewVsn), + BaseFromVsn = strip_pre_release(BaseFromVsn0), + validate_version(BaseFromVsn), + OtpVsn = list_to_binary(OtpVsn0), + case VsnMap of + #{NewVsn := _} -> + print_warning("Version ~s already in DB!~n", [NewVsn]), + halt(1); + #{BaseFromVsn := _} -> + ok; + _ -> + print_warning("Version ~s not found in DB!~n", [BaseFromVsn]), + halt(1) + end, + NewVsnMap = insert_new_vsn(VsnMap, NewVsn, OtpVsn, BaseFromVsn), + file:write_file(VsnDB, io_lib:format("%% -*- mode: erlang; -*-\n\n~p.~n", [NewVsnMap])), + halt(0); +main(["check-vsn-db", NewVsn0, VsnDB]) -> + {ok, [VsnMap]} = file:consult(VsnDB), + NewVsn = strip_pre_release(NewVsn0), + case check_all_vsns_schema(VsnMap) of + [] -> ok; + Problems -> + print_warning("Invalid Version DB ~s!~n", [VsnDB]), + print_warning("Problems found:~n"), + lists:foreach( + fun(Problem) -> + print_warning(" ~p~n", [Problem]) + end, Problems), + halt(1) + end, + case VsnMap of + #{NewVsn := _} -> + io:format(user, "ok~n", []), + halt(0); + _ -> + Candidates = find_insertion_candidates(NewVsn, VsnMap), + print_warning("Version ~s not found in the version DB!~n", [NewVsn]), + [] =/= Candidates + andalso print_warning("Candidates for to insert this version into:~n"), + lists:foreach( + fun(Vsn) -> + io:format(user, " ~s~n", [Vsn]) + end, Candidates), + print_warning( + "To insert this version automatically, run:~n" + "./scripts/relup-base-vsns insert-new-vsn NEW-VSN BASE-FROM-VSN NEW-OTP-VSN ~s~n" + "And commit the results. Be sure to revise the changes.~n" + "Otherwise, edit the file manually~n", + [VsnDB]), + halt(1) + end; +main(_) -> + io:format(user, usage(), []), + halt(1). + +strip_pre_release(Vsn0) -> + case re:run(Vsn0, "[0-9]+\\.[0-9]+\\.[0-9]+", [{capture, all, binary}]) of + {match, [Vsn]} -> + Vsn; + _ -> + print_warning("Invalid Version: ~s ~n", [Vsn0]), + halt(1) + end. + +fetch_version(Vsn, VsnMap) -> + case VsnMap of + #{Vsn := VsnData} -> + VsnData; + _ -> + print_warning("Version not found in releases: ~s ~n", [Vsn]), + halt(1) + end. + +filter_froms(Froms0, AvailableVersionsIndex) -> + Froms1 = + case os:getenv("SYSTEM") of + %% debian11 is introduced since v4.4.2 and e4.4.2 + %% exclude tags before them + "debian11" -> + lists:filter( + fun(Vsn) -> + not lists:member(Vsn, [<<"4.4.0">>, <<"4.4.1">>]) + end, Froms0); + _ -> + Froms0 + end, + lists:filter( + fun(V) -> maps:get(V, AvailableVersionsIndex, false) end, + Froms1). + +%% assumes that's X.Y.Z, without pre-releases +parse_vsn(VsnBin) -> + {match, [Major0, Minor0, Patch0]} = re:run(VsnBin, "([0-9]+)\\.([0-9]+)\\.([0-9]+)", + [{capture, all_but_first, binary}]), + [Major, Minor, Patch] = lists:map(fun binary_to_integer/1, [Major0, Minor0, Patch0]), + {Major, Minor, Patch}. + +parsed_vsn_to_bin({Major, Minor, Patch}) -> + iolist_to_binary(io_lib:format("~b.~b.~b", [Major, Minor, Patch])). + +find_insertion_candidates(NewVsn, VsnMap) -> + ParsedNewVsn = parse_vsn(NewVsn), + [Vsn + || Vsn <- maps:keys(VsnMap), + ParsedVsn <- [parse_vsn(Vsn)], + ParsedVsn > ParsedNewVsn]. + +check_all_vsns_schema(VsnMap) -> + maps:fold( + fun(Vsn, Val, Acc) -> + Problems = + [{Vsn, should_be_binary} || not is_binary(Vsn)] ++ + [{Vsn, must_have_map_value} || not is_map(Val)] ++ + [{Vsn, {must_contain_keys, [otp, from_versions]}} + || case Val of + #{otp := _, from_versions := _} -> + false; + _ -> + true + end] ++ + [{Vsn, otp_version_must_be_binary} + || case Val of + #{otp := Otp} when is_binary(Otp) -> + false; + _ -> + true + end] ++ + [{Vsn, versions_must_be_list_of_binaries} + || case Val of + #{from_versions := Froms} when is_list(Froms) -> + not lists:all(fun is_binary/1, Froms); + _ -> + true + end], + Problems ++ Acc + end, + [], + VsnMap). + +insert_new_vsn(VsnMap0, NewVsn, OtpVsn, BaseFromVsn) -> + ParsedNewVsn = parse_vsn(NewVsn), + ParsedBaseFromVsn = parse_vsn(BaseFromVsn), + %% candidates to insert this version into (they are "future" versions) + Candidates = find_insertion_candidates(NewVsn, VsnMap0), + %% Past versions we can upgrade from + Froms = [Vsn || Vsn <- maps:keys(VsnMap0), + ParsedVsn <- [parse_vsn(Vsn)], + ParsedVsn >= ParsedBaseFromVsn, + ParsedVsn < ParsedNewVsn], + VsnMap1 = + lists:foldl( + fun(FutureVsn, Acc) -> + FutureData0 = #{from_versions := Froms0} = maps:get(FutureVsn, Acc), + FutureData = FutureData0#{from_versions => lists:usort(Froms0 ++ [NewVsn])}, + Acc#{FutureVsn => FutureData} + end, + VsnMap0, + Candidates), + VsnMap1#{NewVsn => #{otp => OtpVsn, from_versions => Froms}}. + +validate_version(Vsn) -> + ParsedVsn = parse_vsn(Vsn), + VsnBack = parsed_vsn_to_bin(ParsedVsn), + case VsnBack =:= Vsn of + true -> ok; + false -> + print_warning("Invalid version ~p !~n", [Vsn]), + print_warning("Versions MUST be of the form X.Y.Z " + "and not prefixed by `e` or `v`~n"), + halt(1) + end. + +available_versions_index() -> + Output = os:cmd("git tag -l"), + AllVersions = + lists:filtermap( + fun(Line) -> + case re:run(Line, "^[ve]([0-9]+)\\.([0-9]+)\\.([0-9]+)$", + [{capture, all_but_first, binary}]) of + {match, [Major, Minor, Patch]} -> + Vsn = iolist_to_binary(io_lib:format("~s.~s.~s", [Major, Minor, Patch])), + {true, Vsn}; + _ -> false + end + end, string:split(Output, "\n", all)), + %% FIXME: `maps:from_keys' is available only in OTP 24, but we + %% still build with 23. Switch to that once we drop OTP 23. + maps:from_list([{Vsn, true} || Vsn <- AllVersions]). + +print_warning(Msg) -> + print_warning(Msg, []). + +print_warning(Msg, Args) -> + io:format(user, ?RED ++ Msg ++ ?RESET, Args). diff --git a/scripts/relup-base-vsns.sh b/scripts/relup-base-vsns.sh index 91c26823e..03809dab8 100755 --- a/scripts/relup-base-vsns.sh +++ b/scripts/relup-base-vsns.sh @@ -41,14 +41,6 @@ if [ "${#CUR_SEMVER[@]}" -lt 3 ]; then usage fi -## when the current version has no suffix such as -abcdef00 -## it is a formal release -if [ "${#CUR_SEMVER[@]}" -eq 3 ]; then - IS_RELEASE=true -else - IS_RELEASE=false -fi - case "${EDITION}" in *enterprise*) GIT_TAG_PREFIX="e" @@ -61,28 +53,11 @@ esac # must not be empty for MacOS (bash 3.x) TAGS=( 'dummy' ) TAGS_EXCLUDE=( 'dummy' ) -while read -r git_tag; do - # shellcheck disable=SC2207 - semver=($(parse_semver "$git_tag")) - if [ "${#semver[@]}" -eq 3 ] && [ "${semver[2]}" -le "${CUR_SEMVER[2]}" ]; then - if [ ${IS_RELEASE} = true ] && [ "${semver[2]}" -eq "${CUR_SEMVER[2]}" ] ; then - # do nothing - # exact match, do not print current version - # because current version is not an upgrade base - true - else - TAGS+=( "$git_tag" ) - fi - fi -done < <(git tag -l "${GIT_TAG_PREFIX}${CUR_SEMVER[0]}.${CUR_SEMVER[1]}.*") -# debian11 is introduced since v4.4.2 and e4.4.2 -# exclude tags before them -SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" -if [ "$SYSTEM" = 'debian11' ]; then - TAGS_EXCLUDE+=( 'v4.4.0' 'v4.4.1' ) - TAGS_EXCLUDE+=( 'e4.4.0' 'e4.4.1' ) -fi +while read -r vsn; do + # shellcheck disable=SC2207 + TAGS+=($(git tag -l "${GIT_TAG_PREFIX}${vsn}")) +done < <(./scripts/relup-base-vsns.escript base-vsns "$CUR" ./data/relup_bases.eterm) for tag_to_del in "${TAGS_EXCLUDE[@]}"; do TAGS=( "${TAGS[@]/$tag_to_del}" ) From 54a6674b088f95ef2f89990324f0b696bad09f54 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 22 Aug 2022 17:33:24 -0300 Subject: [PATCH 68/73] style: fix indentation --- build | 40 +++++++++++++++++----------------- scripts/relup-base-packages.sh | 4 ++-- scripts/relup-base-vsns.sh | 4 ++-- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/build b/build index d3c9916e1..fb2ecf8bb 100755 --- a/build +++ b/build @@ -67,7 +67,7 @@ make_rel() { } relup_db() { - ./scripts/relup-base-vsns.escript "$@" ./data/relup_bases.eterm + ./scripts/relup-base-vsns.escript "$@" ./data/relup_bases.eterm } ## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup @@ -78,24 +78,24 @@ make_relup() { mkdir -p "$lib_dir" "$releases_dir" '_upgrade_base' local releases=() if [ -d "$releases_dir" ]; then - for BASE_VSN in $(relup_db base-vsns "$PKG_VSN"); do - OTP_BASE=$(relup_db otp-vsn-for "$PKG_VSN") - name_pattern="${PROFILE}-$(env OTP_VSN="$OTP_BASE" PKG_VSN="$BASE_VSN" ./scripts/pkg-full-vsn.sh 'vsn_exact')" - while read -r zip; do - local base_vsn - base_vsn="$(echo "$zip" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-[0-9a-f]{8})?" | head -1)" - if [ ! -d "$releases_dir/$base_vsn" ]; then - local tmp_dir - tmp_dir="$(mktemp -d -t emqx.XXXXXXX)" - unzip -q "$zip" "emqx/releases/*" -d "$tmp_dir" - unzip -q "$zip" "emqx/lib/*" -d "$tmp_dir" - cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir" || true - cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir" || true - rm -rf "$tmp_dir" - fi - releases+=( "$base_vsn" ) - done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.zip" -type f) - done + for BASE_VSN in $(relup_db base-vsns "$PKG_VSN"); do + OTP_BASE=$(relup_db otp-vsn-for "$PKG_VSN") + name_pattern="${PROFILE}-$(env OTP_VSN="$OTP_BASE" PKG_VSN="$BASE_VSN" ./scripts/pkg-full-vsn.sh 'vsn_exact')" + while read -r zip; do + local base_vsn + base_vsn="$(echo "$zip" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-[0-9a-f]{8})?" | head -1)" + if [ ! -d "$releases_dir/$base_vsn" ]; then + local tmp_dir + tmp_dir="$(mktemp -d -t emqx.XXXXXXX)" + unzip -q "$zip" "emqx/releases/*" -d "$tmp_dir" + unzip -q "$zip" "emqx/lib/*" -d "$tmp_dir" + cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir" || true + cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir" || true + rm -rf "$tmp_dir" + fi + releases+=( "$base_vsn" ) + done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.zip" -type f) + done fi if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" @@ -191,7 +191,7 @@ make_zip() { # shellcheck disable=SC2207 bases=($(relup_db base-vsns "$PKG_VSN")) if [[ "${#bases[@]}" -eq 0 ]]; then - has_relup='no' + has_relup='no' fi if [ "$has_relup" = 'yes' ]; then ./scripts/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup" diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index bc1d8977e..416030219 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -64,7 +64,7 @@ mkdir -p _upgrade_base pushd _upgrade_base otp_vsn_for() { - ../scripts/relup-base-vsns.escript otp-vsn-for "${1#[e|v]}" ../data/relup_bases.eterm + ../scripts/relup-base-vsns.escript otp-vsn-for "${1#[e|v]}" ../data/relup_bases.eterm } for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do @@ -82,7 +82,7 @@ for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do echo "${SUMSTR} ${filename}" | $SHASUM -c || exit 1 fi else - echo "file $filename already downloaded or doesn't exist in the archives; skipping it" + echo "file $filename already downloaded or doesn't exist in the archives; skipping it" fi done diff --git a/scripts/relup-base-vsns.sh b/scripts/relup-base-vsns.sh index 03809dab8..c5ccb462f 100755 --- a/scripts/relup-base-vsns.sh +++ b/scripts/relup-base-vsns.sh @@ -55,8 +55,8 @@ TAGS=( 'dummy' ) TAGS_EXCLUDE=( 'dummy' ) while read -r vsn; do - # shellcheck disable=SC2207 - TAGS+=($(git tag -l "${GIT_TAG_PREFIX}${vsn}")) + # shellcheck disable=SC2207 + TAGS+=($(git tag -l "${GIT_TAG_PREFIX}${vsn}")) done < <(./scripts/relup-base-vsns.escript base-vsns "$CUR" ./data/relup_bases.eterm) for tag_to_del in "${TAGS_EXCLUDE[@]}"; do From e7d42dce491c16cd3a6067b18d4a733f15245530 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 22 Aug 2022 17:39:37 -0300 Subject: [PATCH 69/73] refactor: remove redundant find usage --- build | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/build b/build index fb2ecf8bb..bed17f407 100755 --- a/build +++ b/build @@ -74,27 +74,23 @@ relup_db() { make_relup() { local lib_dir="_build/$PROFILE/rel/emqx/lib" local releases_dir="_build/$PROFILE/rel/emqx/releases" - local name_pattern + local zip_file mkdir -p "$lib_dir" "$releases_dir" '_upgrade_base' local releases=() if [ -d "$releases_dir" ]; then for BASE_VSN in $(relup_db base-vsns "$PKG_VSN"); do OTP_BASE=$(relup_db otp-vsn-for "$PKG_VSN") - name_pattern="${PROFILE}-$(env OTP_VSN="$OTP_BASE" PKG_VSN="$BASE_VSN" ./scripts/pkg-full-vsn.sh 'vsn_exact')" - while read -r zip; do - local base_vsn - base_vsn="$(echo "$zip" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-[0-9a-f]{8})?" | head -1)" - if [ ! -d "$releases_dir/$base_vsn" ]; then - local tmp_dir - tmp_dir="$(mktemp -d -t emqx.XXXXXXX)" - unzip -q "$zip" "emqx/releases/*" -d "$tmp_dir" - unzip -q "$zip" "emqx/lib/*" -d "$tmp_dir" - cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir" || true - cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir" || true - rm -rf "$tmp_dir" - fi - releases+=( "$base_vsn" ) - done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.zip" -type f) + zip_file="_upgrade_base/${PROFILE}-$(env OTP_VSN="$OTP_BASE" PKG_VSN="$BASE_VSN" ./scripts/pkg-full-vsn.sh 'vsn_exact').zip" + if [ ! -d "$releases_dir/$BASE_VSN" ]; then + local tmp_dir + tmp_dir="$(mktemp -d -t emqx.XXXXXXX)" + unzip -q "$zip_file" "emqx/releases/*" -d "$tmp_dir" + unzip -q "$zip_file" "emqx/lib/*" -d "$tmp_dir" + cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir" || true + cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir" || true + rm -rf "$tmp_dir" + fi + releases+=( "$BASE_VSN" ) done fi if [ ${#releases[@]} -eq 0 ]; then From 4a1d05b266f2ab931b981bf8de9de08b0681aeab Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 23 Aug 2022 21:39:35 +0800 Subject: [PATCH 70/73] chore: bump to e4.3.19.beta.3 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index ceafd87e4..609241e68 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.19-beta.2"}). +-define(EMQX_RELEASE, {opensource, "4.3.19-beta.3"}). -else. From c52286a20ae35b28a67b7c28a984f511639c13c9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 23 Aug 2022 10:48:10 -0300 Subject: [PATCH 71/73] refactor: store db as list and rename file to `relup-paths.eterm` --- .github/workflows/apps_version_check.yaml | 2 +- build | 2 +- data/relup-paths.eterm | 32 +++++++++++++++++++++++ data/relup_bases.eterm | 32 ----------------------- scripts/relup-base-packages.sh | 2 +- scripts/relup-base-vsns.escript | 19 ++++++++++---- scripts/relup-base-vsns.sh | 2 +- 7 files changed, 50 insertions(+), 41 deletions(-) create mode 100644 data/relup-paths.eterm delete mode 100644 data/relup_bases.eterm diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index bf3544708..a86b06967 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -24,7 +24,7 @@ jobs: - name: Check relup version DB run: | PKG_VSN=$(./pkg-vsn.sh) - ./scripts/relup-base-vsns.escript check-vsn-db $PKG_VSN ./data/relup_bases.eterm + ./scripts/relup-base-vsns.escript check-vsn-db $PKG_VSN ./data/relup-paths.eterm - name: Check relup (ce) if: endsWith(github.repository, 'emqx') run: ./scripts/update-appup.sh emqx --check diff --git a/build b/build index bed17f407..732240194 100755 --- a/build +++ b/build @@ -67,7 +67,7 @@ make_rel() { } relup_db() { - ./scripts/relup-base-vsns.escript "$@" ./data/relup_bases.eterm + ./scripts/relup-base-vsns.escript "$@" ./data/relup-paths.eterm } ## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup diff --git a/data/relup-paths.eterm b/data/relup-paths.eterm new file mode 100644 index 000000000..ee4840f11 --- /dev/null +++ b/data/relup-paths.eterm @@ -0,0 +1,32 @@ +%% -*- mode: erlang; -*- + +[{<<"4.4.0">>,#{from_versions => [],otp => <<"24.1.5-3">>}}, + {<<"4.4.1">>,#{from_versions => [<<"4.4.0">>],otp => <<"24.1.5-3">>}}, + {<<"4.4.2">>, + #{from_versions => [<<"4.4.0">>,<<"4.4.1">>],otp => <<"24.1.5-3">>}}, + {<<"4.4.3">>, + #{from_versions => [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>], + otp => <<"24.1.5-3">>}}, + {<<"4.4.4">>, + #{from_versions => [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>], + otp => <<"24.1.5-3">>}}, + {<<"4.4.5">>, + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>], + otp => <<"24.1.5-3">>}}, + {<<"4.4.6">>, + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, + <<"4.4.5">>], + otp => <<"24.1.5-3">>}}, + {<<"4.4.7">>, + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, + <<"4.4.5">>,<<"4.4.6">>], + otp => <<"24.1.5-3">>}}, + {<<"4.4.8">>, + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, + <<"4.4.5">>,<<"4.4.6">>,<<"4.4.7">>], + otp => <<"24.1.5-3">>}}, + {<<"4.5.0">>,#{from_versions => [<<"4.4.8">>],otp => <<"24.3.4.2-1">>}}]. diff --git a/data/relup_bases.eterm b/data/relup_bases.eterm deleted file mode 100644 index 1582cf75e..000000000 --- a/data/relup_bases.eterm +++ /dev/null @@ -1,32 +0,0 @@ -%% -*- mode: erlang; -*- - -#{<<"4.4.0">> => #{from_versions => [],otp => <<"24.1.5-3">>}, - <<"4.4.1">> => #{from_versions => [<<"4.4.0">>],otp => <<"24.1.5-3">>}, - <<"4.4.2">> => - #{from_versions => [<<"4.4.0">>,<<"4.4.1">>],otp => <<"24.1.5-3">>}, - <<"4.4.3">> => - #{from_versions => [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>], - otp => <<"24.1.5-3">>}, - <<"4.4.4">> => - #{from_versions => [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>], - otp => <<"24.1.5-3">>}, - <<"4.4.5">> => - #{from_versions => - [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>], - otp => <<"24.1.5-3">>}, - <<"4.4.6">> => - #{from_versions => - [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, - <<"4.4.5">>], - otp => <<"24.1.5-3">>}, - <<"4.4.7">> => - #{from_versions => - [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, - <<"4.4.5">>,<<"4.4.6">>], - otp => <<"24.1.5-3">>}, - <<"4.4.8">> => - #{from_versions => - [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, - <<"4.4.5">>,<<"4.4.6">>,<<"4.4.7">>], - otp => <<"24.1.5-3">>}, - <<"4.5.0">> => #{from_versions => [<<"4.4.8">>],otp => <<"24.3.4.2-1">>}}. diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 416030219..60785adc8 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -64,7 +64,7 @@ mkdir -p _upgrade_base pushd _upgrade_base otp_vsn_for() { - ../scripts/relup-base-vsns.escript otp-vsn-for "${1#[e|v]}" ../data/relup_bases.eterm + ../scripts/relup-base-vsns.escript otp-vsn-for "${1#[e|v]}" ../data/relup-paths.eterm } for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do diff --git a/scripts/relup-base-vsns.escript b/scripts/relup-base-vsns.escript index 1109b2555..5b04c7dba 100755 --- a/scripts/relup-base-vsns.escript +++ b/scripts/relup-base-vsns.escript @@ -53,7 +53,7 @@ Usage: ". main(["base-vsns", To0, VsnDB]) -> - {ok, [VsnMap]} = file:consult(VsnDB), + VsnMap = read_db(VsnDB), To = strip_pre_release(To0), #{from_versions := Froms} = fetch_version(To, VsnMap), AvailableVersionsIndex = available_versions_index(), @@ -64,13 +64,13 @@ main(["base-vsns", To0, VsnDB]) -> filter_froms(Froms, AvailableVersionsIndex)), halt(0); main(["otp-vsn-for", Vsn0, VsnDB]) -> - {ok, [VsnMap]} = file:consult(VsnDB), + VsnMap = read_db(VsnDB), Vsn = strip_pre_release(Vsn0), #{otp := OtpVsn} = fetch_version(Vsn, VsnMap), io:format(user, "~s~n", [OtpVsn]), halt(0); main(["insert-new-vsn", NewVsn0, BaseFromVsn0, OtpVsn0, VsnDB]) -> - {ok, [VsnMap]} = file:consult(VsnDB), + VsnMap = read_db(VsnDB), NewVsn = strip_pre_release(NewVsn0), validate_version(NewVsn), BaseFromVsn = strip_pre_release(BaseFromVsn0), @@ -87,10 +87,15 @@ main(["insert-new-vsn", NewVsn0, BaseFromVsn0, OtpVsn0, VsnDB]) -> halt(1) end, NewVsnMap = insert_new_vsn(VsnMap, NewVsn, OtpVsn, BaseFromVsn), - file:write_file(VsnDB, io_lib:format("%% -*- mode: erlang; -*-\n\n~p.~n", [NewVsnMap])), + NewVsnList = + lists:sort( + fun({Vsn1, _}, {Vsn2, _}) -> + Vsn1 < Vsn2 + end, maps:to_list(NewVsnMap)), + file:write_file(VsnDB, io_lib:format("%% -*- mode: erlang; -*-\n\n~p.~n", [NewVsnList])), halt(0); main(["check-vsn-db", NewVsn0, VsnDB]) -> - {ok, [VsnMap]} = file:consult(VsnDB), + VsnMap = read_db(VsnDB), NewVsn = strip_pre_release(NewVsn0), case check_all_vsns_schema(VsnMap) of [] -> ok; @@ -262,6 +267,10 @@ available_versions_index() -> %% still build with 23. Switch to that once we drop OTP 23. maps:from_list([{Vsn, true} || Vsn <- AllVersions]). +read_db(VsnDB) -> + {ok, [VsnList]} = file:consult(VsnDB), + maps:from_list(VsnList). + print_warning(Msg) -> print_warning(Msg, []). diff --git a/scripts/relup-base-vsns.sh b/scripts/relup-base-vsns.sh index c5ccb462f..15b97921c 100755 --- a/scripts/relup-base-vsns.sh +++ b/scripts/relup-base-vsns.sh @@ -57,7 +57,7 @@ TAGS_EXCLUDE=( 'dummy' ) while read -r vsn; do # shellcheck disable=SC2207 TAGS+=($(git tag -l "${GIT_TAG_PREFIX}${vsn}")) -done < <(./scripts/relup-base-vsns.escript base-vsns "$CUR" ./data/relup_bases.eterm) +done < <(./scripts/relup-base-vsns.escript base-vsns "$CUR" ./data/relup-paths.eterm) for tag_to_del in "${TAGS_EXCLUDE[@]}"; do TAGS=( "${TAGS[@]/$tag_to_del}" ) From 81b0844f0332e6a4e314530375253c4a873a90d8 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 23 Aug 2022 17:49:31 -0300 Subject: [PATCH 72/73] fix: compare parsed versions --- scripts/relup-base-vsns.escript | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/relup-base-vsns.escript b/scripts/relup-base-vsns.escript index 5b04c7dba..cf1164ed9 100755 --- a/scripts/relup-base-vsns.escript +++ b/scripts/relup-base-vsns.escript @@ -90,7 +90,7 @@ main(["insert-new-vsn", NewVsn0, BaseFromVsn0, OtpVsn0, VsnDB]) -> NewVsnList = lists:sort( fun({Vsn1, _}, {Vsn2, _}) -> - Vsn1 < Vsn2 + parse_vsn(Vsn1) < parse_vsn(Vsn2) end, maps:to_list(NewVsnMap)), file:write_file(VsnDB, io_lib:format("%% -*- mode: erlang; -*-\n\n~p.~n", [NewVsnList])), halt(0); From c30ba73af1e0a52eaf1cd8d8b0de94ee9daf39cc Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 23 Aug 2022 17:55:14 -0300 Subject: [PATCH 73/73] refactor: store entries as multiple tuples --- data/relup-paths.eterm | 60 ++++++++++++++++----------------- scripts/relup-base-vsns.escript | 11 ++++-- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/data/relup-paths.eterm b/data/relup-paths.eterm index ee4840f11..02d171e8f 100644 --- a/data/relup-paths.eterm +++ b/data/relup-paths.eterm @@ -1,32 +1,32 @@ %% -*- mode: erlang; -*- -[{<<"4.4.0">>,#{from_versions => [],otp => <<"24.1.5-3">>}}, - {<<"4.4.1">>,#{from_versions => [<<"4.4.0">>],otp => <<"24.1.5-3">>}}, - {<<"4.4.2">>, - #{from_versions => [<<"4.4.0">>,<<"4.4.1">>],otp => <<"24.1.5-3">>}}, - {<<"4.4.3">>, - #{from_versions => [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>], - otp => <<"24.1.5-3">>}}, - {<<"4.4.4">>, - #{from_versions => [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>], - otp => <<"24.1.5-3">>}}, - {<<"4.4.5">>, - #{from_versions => - [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>], - otp => <<"24.1.5-3">>}}, - {<<"4.4.6">>, - #{from_versions => - [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, - <<"4.4.5">>], - otp => <<"24.1.5-3">>}}, - {<<"4.4.7">>, - #{from_versions => - [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, - <<"4.4.5">>,<<"4.4.6">>], - otp => <<"24.1.5-3">>}}, - {<<"4.4.8">>, - #{from_versions => - [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, - <<"4.4.5">>,<<"4.4.6">>,<<"4.4.7">>], - otp => <<"24.1.5-3">>}}, - {<<"4.5.0">>,#{from_versions => [<<"4.4.8">>],otp => <<"24.3.4.2-1">>}}]. +{<<"4.4.0">>,#{from_versions => [],otp => <<"24.1.5-3">>}}. +{<<"4.4.1">>,#{from_versions => [<<"4.4.0">>],otp => <<"24.1.5-3">>}}. +{<<"4.4.2">>, + #{from_versions => [<<"4.4.0">>,<<"4.4.1">>],otp => <<"24.1.5-3">>}}. +{<<"4.4.3">>, + #{from_versions => [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>], + otp => <<"24.1.5-3">>}}. +{<<"4.4.4">>, + #{from_versions => [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>], + otp => <<"24.1.5-3">>}}. +{<<"4.4.5">>, + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>], + otp => <<"24.1.5-3">>}}. +{<<"4.4.6">>, + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, + <<"4.4.5">>], + otp => <<"24.1.5-3">>}}. +{<<"4.4.7">>, + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, + <<"4.4.5">>,<<"4.4.6">>], + otp => <<"24.1.5-3">>}}. +{<<"4.4.8">>, + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, + <<"4.4.5">>,<<"4.4.6">>,<<"4.4.7">>], + otp => <<"24.1.5-3">>}}. +{<<"4.5.0">>,#{from_versions => [<<"4.4.8">>],otp => <<"24.3.4.2-1">>}}. diff --git a/scripts/relup-base-vsns.escript b/scripts/relup-base-vsns.escript index cf1164ed9..94cc4bd4f 100755 --- a/scripts/relup-base-vsns.escript +++ b/scripts/relup-base-vsns.escript @@ -92,7 +92,14 @@ main(["insert-new-vsn", NewVsn0, BaseFromVsn0, OtpVsn0, VsnDB]) -> fun({Vsn1, _}, {Vsn2, _}) -> parse_vsn(Vsn1) < parse_vsn(Vsn2) end, maps:to_list(NewVsnMap)), - file:write_file(VsnDB, io_lib:format("%% -*- mode: erlang; -*-\n\n~p.~n", [NewVsnList])), + {ok, FD} = file:open(VsnDB, [write]), + io:format(FD, "%% -*- mode: erlang; -*-\n\n", []), + lists:foreach( + fun(Entry) -> + io:format(FD, "~p.~n", [Entry]) + end, + NewVsnList), + file:close(FD), halt(0); main(["check-vsn-db", NewVsn0, VsnDB]) -> VsnMap = read_db(VsnDB), @@ -268,7 +275,7 @@ available_versions_index() -> maps:from_list([{Vsn, true} || Vsn <- AllVersions]). read_db(VsnDB) -> - {ok, [VsnList]} = file:consult(VsnDB), + {ok, VsnList} = file:consult(VsnDB), maps:from_list(VsnList). print_warning(Msg) ->