fix: rename queue_max_bytes -> max_queue_bytes
This commit is contained in:
parent
c4106c0d77
commit
9e50866cd0
|
@ -173,7 +173,7 @@ get_metrics(Name, Id) ->
|
||||||
inc(Name, Id, Metric) ->
|
inc(Name, Id, Metric) ->
|
||||||
inc(Name, Id, Metric, 1).
|
inc(Name, Id, Metric, 1).
|
||||||
|
|
||||||
-spec inc(handler_name(), metric_id(), atom(), pos_integer()) -> ok.
|
-spec inc(handler_name(), metric_id(), atom(), integer()) -> ok.
|
||||||
inc(Name, Id, Metric, Val) ->
|
inc(Name, Id, Metric, Val) ->
|
||||||
counters:add(get_ref(Name, Id), idx_metric(Name, Id, Metric), Val).
|
counters:add(get_ref(Name, Id), idx_metric(Name, Id, Metric), Val).
|
||||||
|
|
||||||
|
|
|
@ -143,7 +143,7 @@ emqx_resource_schema {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
queue_max_bytes {
|
max_queue_bytes {
|
||||||
desc {
|
desc {
|
||||||
en: """Maximum queue storage."""
|
en: """Maximum queue storage."""
|
||||||
zh: """消息队列的最大长度。"""
|
zh: """消息队列的最大长度。"""
|
||||||
|
|
|
@ -68,7 +68,7 @@
|
||||||
batch_size => pos_integer(),
|
batch_size => pos_integer(),
|
||||||
batch_time => pos_integer(),
|
batch_time => pos_integer(),
|
||||||
enable_queue => boolean(),
|
enable_queue => boolean(),
|
||||||
queue_max_bytes => pos_integer(),
|
max_queue_bytes => pos_integer(),
|
||||||
query_mode => query_mode(),
|
query_mode => query_mode(),
|
||||||
resume_interval => pos_integer(),
|
resume_interval => pos_integer(),
|
||||||
async_inflight_window => pos_integer()
|
async_inflight_window => pos_integer()
|
||||||
|
@ -81,8 +81,11 @@
|
||||||
|
|
||||||
-define(WORKER_POOL_SIZE, 16).
|
-define(WORKER_POOL_SIZE, 16).
|
||||||
|
|
||||||
-define(DEFAULT_QUEUE_SIZE, 1024 * 1024 * 1024).
|
-define(DEFAULT_QUEUE_SEG_SIZE, 10 * 1024 * 1024).
|
||||||
-define(DEFAULT_QUEUE_SIZE_RAW, <<"1GB">>).
|
-define(DEFAULT_QUEUE_SEG_SIZE_RAW, <<"10MB">>).
|
||||||
|
|
||||||
|
-define(DEFAULT_QUEUE_SIZE, 100 * 1024 * 1024 * 1024).
|
||||||
|
-define(DEFAULT_QUEUE_SIZE_RAW, <<"100GB">>).
|
||||||
|
|
||||||
%% count
|
%% count
|
||||||
-define(DEFAULT_BATCH_SIZE, 100).
|
-define(DEFAULT_BATCH_SIZE, 100).
|
||||||
|
|
|
@ -577,8 +577,12 @@ maybe_alarm(_Status, ResId) ->
|
||||||
).
|
).
|
||||||
|
|
||||||
maybe_resume_resource_workers(connected) ->
|
maybe_resume_resource_workers(connected) ->
|
||||||
{_, Pid, _, _} = supervisor:which_children(emqx_resource_worker_sup),
|
lists:foreach(
|
||||||
emqx_resource_worker:resume(Pid);
|
fun({_, Pid, _, _}) ->
|
||||||
|
emqx_resource_worker:resume(Pid)
|
||||||
|
end,
|
||||||
|
supervisor:which_children(emqx_resource_worker_sup)
|
||||||
|
);
|
||||||
maybe_resume_resource_workers(_) ->
|
maybe_resume_resource_workers(_) ->
|
||||||
ok.
|
ok.
|
||||||
|
|
||||||
|
|
|
@ -123,13 +123,15 @@ init({Id, Index, Opts}) ->
|
||||||
true ->
|
true ->
|
||||||
replayq:open(#{
|
replayq:open(#{
|
||||||
dir => disk_queue_dir(Id, Index),
|
dir => disk_queue_dir(Id, Index),
|
||||||
seg_bytes => maps:get(queue_max_bytes, Opts, ?DEFAULT_QUEUE_SIZE),
|
seg_bytes => maps:get(queue_seg_bytes, Opts, ?DEFAULT_QUEUE_SEG_SIZE),
|
||||||
|
max_total_bytes => maps:get(max_queue_bytes, Opts, ?DEFAULT_QUEUE_SIZE),
|
||||||
sizer => fun ?MODULE:estimate_size/1,
|
sizer => fun ?MODULE:estimate_size/1,
|
||||||
marshaller => fun ?MODULE:queue_item_marshaller/1
|
marshaller => fun ?MODULE:queue_item_marshaller/1
|
||||||
});
|
});
|
||||||
false ->
|
false ->
|
||||||
undefined
|
undefined
|
||||||
end,
|
end,
|
||||||
|
emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued', replayq:count(Queue)),
|
||||||
ok = inflight_new(Name),
|
ok = inflight_new(Name),
|
||||||
St = #{
|
St = #{
|
||||||
id => Id,
|
id => Id,
|
||||||
|
@ -323,23 +325,27 @@ flush(
|
||||||
end.
|
end.
|
||||||
|
|
||||||
maybe_append_queue(Id, undefined, _Items) ->
|
maybe_append_queue(Id, undefined, _Items) ->
|
||||||
|
emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
|
||||||
emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_not_enabled'),
|
emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_not_enabled'),
|
||||||
undefined;
|
undefined;
|
||||||
maybe_append_queue(Id, Q, Items) ->
|
maybe_append_queue(Id, Q, Items) ->
|
||||||
case replayq:overflow(Q) of
|
Q2 =
|
||||||
Overflow when Overflow =< 0 ->
|
case replayq:overflow(Q) of
|
||||||
emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued'),
|
Overflow when Overflow =< 0 ->
|
||||||
replayq:append(Q, Items);
|
Q;
|
||||||
Overflow ->
|
Overflow ->
|
||||||
PopOpts = #{bytes_limit => Overflow, count_limit => 999999999},
|
PopOpts = #{bytes_limit => Overflow, count_limit => 999999999},
|
||||||
{Q1, QAckRef, Items} = replayq:pop(Q, PopOpts),
|
{Q1, QAckRef, Items2} = replayq:pop(Q, PopOpts),
|
||||||
ok = replayq:ack(Q1, QAckRef),
|
ok = replayq:ack(Q1, QAckRef),
|
||||||
Dropped = length(Items),
|
Dropped = length(Items2),
|
||||||
emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued', -Dropped),
|
emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued', -Dropped),
|
||||||
emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_full'),
|
emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped'),
|
||||||
?SLOG(error, #{msg => drop_query, reason => queue_full, dropped => Dropped}),
|
emqx_metrics_worker:inc(?RES_METRICS, Id, 'dropped.queue_full'),
|
||||||
Q1
|
?SLOG(error, #{msg => drop_query, reason => queue_full, dropped => Dropped}),
|
||||||
end.
|
Q1
|
||||||
|
end,
|
||||||
|
emqx_metrics_worker:inc(?RES_METRICS, Id, 'queued'),
|
||||||
|
replayq:append(Q2, Items).
|
||||||
|
|
||||||
batch_reply_caller(Id, BatchResult, Batch) ->
|
batch_reply_caller(Id, BatchResult, Batch) ->
|
||||||
lists:foldl(
|
lists:foldl(
|
||||||
|
|
|
@ -53,7 +53,7 @@ fields("creation_opts") ->
|
||||||
{batch_size, fun batch_size/1},
|
{batch_size, fun batch_size/1},
|
||||||
{batch_time, fun batch_time/1},
|
{batch_time, fun batch_time/1},
|
||||||
{enable_queue, fun enable_queue/1},
|
{enable_queue, fun enable_queue/1},
|
||||||
{max_queue_bytes, fun queue_max_bytes/1}
|
{max_queue_bytes, fun max_queue_bytes/1}
|
||||||
].
|
].
|
||||||
|
|
||||||
worker_pool_size(type) -> pos_integer();
|
worker_pool_size(type) -> pos_integer();
|
||||||
|
@ -110,11 +110,11 @@ batch_time(default) -> ?DEFAULT_BATCH_TIME_RAW;
|
||||||
batch_time(required) -> false;
|
batch_time(required) -> false;
|
||||||
batch_time(_) -> undefined.
|
batch_time(_) -> undefined.
|
||||||
|
|
||||||
queue_max_bytes(type) -> emqx_schema:bytesize();
|
max_queue_bytes(type) -> emqx_schema:bytesize();
|
||||||
queue_max_bytes(desc) -> ?DESC("queue_max_bytes");
|
max_queue_bytes(desc) -> ?DESC("max_queue_bytes");
|
||||||
queue_max_bytes(default) -> ?DEFAULT_QUEUE_SIZE_RAW;
|
max_queue_bytes(default) -> ?DEFAULT_QUEUE_SIZE_RAW;
|
||||||
queue_max_bytes(required) -> false;
|
max_queue_bytes(required) -> false;
|
||||||
queue_max_bytes(_) -> undefined.
|
max_queue_bytes(_) -> undefined.
|
||||||
|
|
||||||
desc("creation_opts") ->
|
desc("creation_opts") ->
|
||||||
?DESC("creation_opts").
|
?DESC("creation_opts").
|
||||||
|
|
|
@ -522,6 +522,6 @@ inc_action_metrics(R, RuleId) ->
|
||||||
is_ok_result(ok) ->
|
is_ok_result(ok) ->
|
||||||
true;
|
true;
|
||||||
is_ok_result(R) when is_tuple(R) ->
|
is_ok_result(R) when is_tuple(R) ->
|
||||||
ok = erlang:element(1, R);
|
ok == erlang:element(1, R);
|
||||||
is_ok_result(ok) ->
|
is_ok_result(ok) ->
|
||||||
false.
|
false.
|
||||||
|
|
Loading…
Reference in New Issue