fix: rename queue_max_bytes -> max_queue_bytes

This commit is contained in:
Shawn 2022-08-30 17:13:49 +08:00
parent c4106c0d77
commit 9e50866cd0
7 changed files with 42 additions and 29 deletions

View File

@ -173,7 +173,7 @@ get_metrics(Name, Id) ->
inc(Name, Id, Metric) -> inc(Name, Id, Metric) ->
inc(Name, Id, Metric, 1). inc(Name, Id, Metric, 1).
-spec inc(handler_name(), metric_id(), atom(), pos_integer()) -> ok. -spec inc(handler_name(), metric_id(), atom(), integer()) -> ok.
inc(Name, Id, Metric, Val) -> inc(Name, Id, Metric, Val) ->
counters:add(get_ref(Name, Id), idx_metric(Name, Id, Metric), Val). counters:add(get_ref(Name, Id), idx_metric(Name, Id, Metric), Val).

View File

@ -143,7 +143,7 @@ emqx_resource_schema {
} }
} }
queue_max_bytes { max_queue_bytes {
desc { desc {
en: """Maximum queue storage.""" en: """Maximum queue storage."""
zh: """消息队列的最大长度。""" zh: """消息队列的最大长度。"""

View File

@ -68,7 +68,7 @@
batch_size => pos_integer(), batch_size => pos_integer(),
batch_time => pos_integer(), batch_time => pos_integer(),
enable_queue => boolean(), enable_queue => boolean(),
queue_max_bytes => pos_integer(), max_queue_bytes => pos_integer(),
query_mode => query_mode(), query_mode => query_mode(),
resume_interval => pos_integer(), resume_interval => pos_integer(),
async_inflight_window => pos_integer() async_inflight_window => pos_integer()
@ -81,8 +81,11 @@
-define(WORKER_POOL_SIZE, 16). -define(WORKER_POOL_SIZE, 16).
-define(DEFAULT_QUEUE_SIZE, 1024 * 1024 * 1024). -define(DEFAULT_QUEUE_SEG_SIZE, 10 * 1024 * 1024).
-define(DEFAULT_QUEUE_SIZE_RAW, <<"1GB">>). -define(DEFAULT_QUEUE_SEG_SIZE_RAW, <<"10MB">>).
-define(DEFAULT_QUEUE_SIZE, 100 * 1024 * 1024 * 1024).
-define(DEFAULT_QUEUE_SIZE_RAW, <<"100GB">>).
%% count %% count
-define(DEFAULT_BATCH_SIZE, 100). -define(DEFAULT_BATCH_SIZE, 100).

View File

@ -577,8 +577,12 @@ maybe_alarm(_Status, ResId) ->
). ).
maybe_resume_resource_workers(connected) -> maybe_resume_resource_workers(connected) ->
{_, Pid, _, _} = supervisor:which_children(emqx_resource_worker_sup), lists:foreach(
emqx_resource_worker:resume(Pid); fun({_, Pid, _, _}) ->
emqx_resource_worker:resume(Pid)
end,
supervisor:which_children(emqx_resource_worker_sup)
);
maybe_resume_resource_workers(_) -> maybe_resume_resource_workers(_) ->
ok. ok.

View File

@ -123,13 +123,15 @@ init({Id, Index, Opts}) ->
true -> true ->
replayq:open(#{ replayq:open(#{
dir => disk_queue_dir(Id, Index), dir => disk_queue_dir(Id, Index),
seg_bytes => maps:get(queue_max_bytes, Opts, ?DEFAULT_QUEUE_SIZE), seg_bytes => maps:get(queue_seg_bytes, Opts, ?DEFAULT_QUEUE_SEG_SIZE),
max_total_bytes => maps:get(max_queue_bytes, Opts, ?DEFAULT_QUEUE_SIZE),
sizer => fun ?MODULE:estimate_size/1, sizer => fun ?MODULE:estimate_size/1,
marshaller => fun ?MODULE:queue_item_marshaller/1 marshaller => fun ?MODULE:queue_item_marshaller/1
}); });
false -> false ->
undefined undefined
end, end,
emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued', replayq:count(Queue)),
ok = inflight_new(Name), ok = inflight_new(Name),
St = #{ St = #{
id => Id, id => Id,
@ -323,23 +325,27 @@ flush(
end. end.
maybe_append_queue(Id, undefined, _Items) -> maybe_append_queue(Id, undefined, _Items) ->
emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_not_enabled'), emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_not_enabled'),
undefined; undefined;
maybe_append_queue(Id, Q, Items) -> maybe_append_queue(Id, Q, Items) ->
case replayq:overflow(Q) of Q2 =
Overflow when Overflow =< 0 -> case replayq:overflow(Q) of
emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued'), Overflow when Overflow =< 0 ->
replayq:append(Q, Items); Q;
Overflow -> Overflow ->
PopOpts = #{bytes_limit => Overflow, count_limit => 999999999}, PopOpts = #{bytes_limit => Overflow, count_limit => 999999999},
{Q1, QAckRef, Items} = replayq:pop(Q, PopOpts), {Q1, QAckRef, Items2} = replayq:pop(Q, PopOpts),
ok = replayq:ack(Q1, QAckRef), ok = replayq:ack(Q1, QAckRef),
Dropped = length(Items), Dropped = length(Items2),
emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued', -Dropped), emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued', -Dropped),
emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_full'), emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
?SLOG(error, #{msg => drop_query, reason => queue_full, dropped => Dropped}), emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_full'),
Q1 ?SLOG(error, #{msg => drop_query, reason => queue_full, dropped => Dropped}),
end. Q1
end,
emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued'),
replayq:append(Q2, Items).
batch_reply_caller(Id, BatchResult, Batch) -> batch_reply_caller(Id, BatchResult, Batch) ->
lists:foldl( lists:foldl(

View File

@ -53,7 +53,7 @@ fields("creation_opts") ->
{batch_size, fun batch_size/1}, {batch_size, fun batch_size/1},
{batch_time, fun batch_time/1}, {batch_time, fun batch_time/1},
{enable_queue, fun enable_queue/1}, {enable_queue, fun enable_queue/1},
{max_queue_bytes, fun queue_max_bytes/1} {max_queue_bytes, fun max_queue_bytes/1}
]. ].
worker_pool_size(type) -> pos_integer(); worker_pool_size(type) -> pos_integer();
@ -110,11 +110,11 @@ batch_time(default) -> ?DEFAULT_BATCH_TIME_RAW;
batch_time(required) -> false; batch_time(required) -> false;
batch_time(_) -> undefined. batch_time(_) -> undefined.
queue_max_bytes(type) -> emqx_schema:bytesize(); max_queue_bytes(type) -> emqx_schema:bytesize();
queue_max_bytes(desc) -> ?DESC("queue_max_bytes"); max_queue_bytes(desc) -> ?DESC("max_queue_bytes");
queue_max_bytes(default) -> ?DEFAULT_QUEUE_SIZE_RAW; max_queue_bytes(default) -> ?DEFAULT_QUEUE_SIZE_RAW;
queue_max_bytes(required) -> false; max_queue_bytes(required) -> false;
queue_max_bytes(_) -> undefined. max_queue_bytes(_) -> undefined.
desc("creation_opts") -> desc("creation_opts") ->
?DESC("creation_opts"). ?DESC("creation_opts").

View File

@ -522,6 +522,6 @@ inc_action_metrics(R, RuleId) ->
is_ok_result(ok) -> is_ok_result(ok) ->
true; true;
is_ok_result(R) when is_tuple(R) -> is_ok_result(R) when is_tuple(R) ->
ok = erlang:element(1, R); ok == erlang:element(1, R);
is_ok_result(ok) -> is_ok_result(ok) ->
false. false.