From 6baf2dbd95c15c4a4f5e1108b3fa7ef53c3b7ba3 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 5 Oct 2021 09:21:32 +0200 Subject: [PATCH] feat(olp): backoff new conn --- apps/emqx/etc/emqx.conf | 10 ++++-- apps/emqx/src/emqx_connection.erl | 7 ++-- apps/emqx/src/emqx_listeners.erl | 4 ++- apps/emqx/src/emqx_metrics.erl | 4 +-- apps/emqx/src/emqx_olp.erl | 46 +++++++++++++++++++------- apps/emqx/src/emqx_quic_connection.erl | 2 +- apps/emqx/src/emqx_schema.erl | 6 +++- 7 files changed, 57 insertions(+), 22 deletions(-) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index c747052f0..2b70d6dda 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -850,14 +850,20 @@ overload_protection { ## Backoff GC enabled ## @doc overload_protection.backoff_gc ## ValueType: Boolean - ## Default: true - backoff_gc = true + ## Default: false + backoff_gc = false ## Backoff hibernation enabled ## @doc overload_protection.backoff_hibernation ## ValueType: Boolean ## Default: true backoff_hibernation = true + + ## Backoff hibernation enabled + ## @doc overload_protection.backoff_hibernation + ## ValueType: Boolean + ## Default: true + backoff_new_conn = true } force_gc { diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 57c68510f..2b81f3526 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -317,15 +317,16 @@ exit_on_sock_error(Reason) -> %%-------------------------------------------------------------------- %% Recv Loop -recvloop(Parent, State = #state{idle_timeout = IdleTimeout}) -> +recvloop(Parent, State = #state{ idle_timeout = IdleTimeout + , zone = Zone + }) -> receive Msg -> handle_recv(Msg, Parent, State) after IdleTimeout + 100 -> - case emqx_olp:is_overloaded() of + case emqx_olp:backoff_hibernation(Zone) of true -> - emqx_metrics:inc('olp.hbn'), recvloop(Parent, State); false -> hibernate(Parent, cancel_stats_timer(State)) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 5c4776207..b9367d245 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -289,7 +289,9 @@ esockd_opts(Type, Opts0) -> infinity -> Opts1; Rate -> Opts1#{max_conn_rate => Rate} end, - Opts3 = Opts2#{access_rules => esockd_access_rules(maps:get(access_rules, Opts0, []))}, + Opts3 = Opts2#{ access_rules => esockd_access_rules(maps:get(access_rules, Opts0, [])) + , tune_fun => {emqx_olp, backoff_new_conn, [zone(Opts0)]} + }, maps:to_list(case Type of tcp -> Opts3#{tcp_options => tcp_opts(Opts0)}; ssl -> Opts3#{ssl_options => ssl_opts(Opts0), tcp_options => tcp_opts(Opts0)} diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index 8b79f9ff2..c8cef1308 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -190,7 +190,7 @@ {counter, 'olp.delay.timeout'}, {counter, 'olp.hbn'}, {counter, 'olp.gc'}, - {counter, 'olp.close.quic'} + {counter, 'olp.new_conn'} ]). -record(state, {next_idx = 1}). @@ -585,7 +585,7 @@ reserved_idx('olp.delay.ok') -> 300; reserved_idx('olp.delay.timeout') -> 301; reserved_idx('olp.hbn') -> 302; reserved_idx('olp.gc') -> 303; -reserved_idx('olp.close.quic') -> 304; +reserved_idx('olp.new_conn') -> 304; reserved_idx(_) -> undefined. diff --git a/apps/emqx/src/emqx_olp.erl b/apps/emqx/src/emqx_olp.erl index 2e3efe080..aa4966e36 100644 --- a/apps/emqx/src/emqx_olp.erl +++ b/apps/emqx/src/emqx_olp.erl @@ -19,6 +19,7 @@ , backoff/1 , backoff_gc/1 , backoff_hibernation/1 + , backoff_new_conn/1 ]). @@ -28,15 +29,16 @@ , off/0 ]). +-define(overload_protection, overload_protection). + -spec is_overloaded() -> boolean(). is_overloaded() -> load_ctl:is_overloaded(). -spec backoff(Zone :: atom()) -> ok | false | timeout. backoff(Zone) -> - case emqx_config:get_zone_conf(Zone, [overload_protection, enable], false) of - true -> - Delay = emqx_config:get_zone_conf(Zone, [overload_protection, backoff_delay], 1), + case emqx_config:get_zone_conf(Zone, [?overload_protection]) of + #{enable := true, backoff_delay := Delay} -> case load_ctl:maydelay(Delay) of false -> false; ok -> @@ -46,21 +48,26 @@ backoff(Zone) -> emqx_metrics:inc('olp.delay.timeout'), timeout end; - false -> + _ -> ok end. --spec backoff_gc(Zone :: atom()) -> ok | timeout. +-spec backoff_gc(Zone :: atom()) -> boolean(). backoff_gc(Zone) -> - load_ctl:is_overloaded() - andalso emqx_config:get_zone_conf(Zone, [overload_protection, enable], false) - andalso emqx_config:get_zone_conf(Zone, [overload_protection, backoff_gc], false). + do_check(Zone, ?FUNCTION_NAME, 'olp.gc'). --spec backoff_hibernation(Zone :: atom()) -> ok | timeout. +-spec backoff_hibernation(Zone :: atom()) -> boolean(). backoff_hibernation(Zone) -> - load_ctl:is_overloaded() - andalso emqx_config:get_zone_conf(Zone, [overload_protection, enable], false) - andalso emqx_config:get_zone_conf(Zone, [overload_protection, backoff_hibernation], false). + do_check(Zone, ?FUNCTION_NAME, 'olp.hbn'). + +-spec backoff_new_conn(Zone :: atom()) -> ok | {error, overloaded}. +backoff_new_conn(Zone) -> + case do_check(Zone, ?FUNCTION_NAME, 'olp.new_conn') of + true -> + {error, overloaded}; + false -> + ok + end. -spec status() -> any(). status() -> @@ -74,6 +81,21 @@ off() -> on() -> load_ctl:restart_runq_flagman(). +%%% Internals +do_check(Zone, Key, CntName) -> + case load_ctl:is_overloaded() of + true -> + case emqx_config:get_zone_conf(Zone, [?overload_protection]) of + #{enable := true, Key := true} -> + emqx_metrics:inc(CntName), + true; + _ -> + false + end; + false -> false + end. + + %%%_* Emacs ==================================================================== %%% Local Variables: %%% allout-layout: t diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index aaf7321dd..cc195419c 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -46,7 +46,7 @@ new_conn(Conn, S) -> {error, stream_accept_error} end; true -> - emqx_metrics:inc('olp.close.quic'), + emqx_metrics:inc('olp.new_conn'), {error, overloaded} end. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index cd9c5e265..bbb9fd67c 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -406,12 +406,16 @@ fields("overload_protection") -> })} , {"backoff_gc", sc(boolean(), - #{ default => true + #{ default => false })} , {"backoff_hibernation", sc(boolean(), #{ default => true })} + , {"backoff_new_conn", + sc(boolean(), + #{ default => true + })} ]; fields("conn_congestion") ->