feat(ds): Add a benchmarking tool for storage efficiency analysis
This commit is contained in:
ieQu1
parent
afeb2ab8aa
commit
23dafbb03b
|
|
@ -0,0 +1,223 @@
|
|||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
%% @doc This script can be loaded to a running EMQX EE node. It will
|
||||
%% create a number of DS databases with different options and fill
|
||||
%% them with data of given size.
|
||||
%%
|
||||
%% Then it will measure size of the database directories and create
|
||||
%% a "storage (in)efficiency" report.
|
||||
-module(storage_efficiency).
|
||||
|
||||
-include_lib("emqx_utils/include/emqx_message.hrl").
|
||||
|
||||
%% API:
|
||||
-export([run/0, run/1]).
|
||||
|
||||
%%================================================================================
|
||||
%% API functions
|
||||
%%================================================================================
|
||||
|
||||
run() ->
|
||||
run(#{}).
|
||||
|
||||
run(Custom) ->
|
||||
RunConf = maps:merge(
|
||||
#{
|
||||
%% Sleep between batches:
|
||||
sleep => 1_000,
|
||||
%% Don't run test, only plot data:
|
||||
dry_run => false,
|
||||
%% Payload size multiplier:
|
||||
size => 10,
|
||||
%% Number of batches:
|
||||
batches => 100,
|
||||
%% Add generation every N batches:
|
||||
add_generation => 10
|
||||
},
|
||||
Custom
|
||||
),
|
||||
lists:foreach(
|
||||
fun(DBConf) ->
|
||||
run(DBConf, RunConf)
|
||||
end,
|
||||
configs()
|
||||
).
|
||||
|
||||
%% erlfmt-ignore
|
||||
gnuplot_script(Filename) ->
|
||||
"set terminal qt\n"
|
||||
%% "set logscale y 10\n"
|
||||
"set title \"" ++ filename:basename(Filename, ".dat") ++ "\"\n"
|
||||
"set key autotitle columnheader\n"
|
||||
"plot for [n=2:*] \"" ++ Filename ++ "\" using 1:n with linespoints".
|
||||
|
||||
%%================================================================================
|
||||
%% Internal functions
|
||||
%%================================================================================
|
||||
|
||||
configs() ->
|
||||
[
|
||||
{'benchmark-skipstream-asn1',
|
||||
db_conf({emqx_ds_storage_skipstream_lts, #{serialization_schema => asn1}})},
|
||||
{'benchmark-skipstream-v1',
|
||||
db_conf({emqx_ds_storage_skipstream_lts, #{serialization_schema => v1}})},
|
||||
{'benchmark-bitfield', db_conf({emqx_ds_storage_bitfield_lts, #{}})}
|
||||
].
|
||||
|
||||
db_conf(Storage) ->
|
||||
#{
|
||||
backend => builtin_local,
|
||||
%% n_sites => 1,
|
||||
n_shards => 1,
|
||||
%% replication_factor => 1,
|
||||
%% replication_options => #{},
|
||||
storage => Storage
|
||||
}.
|
||||
|
||||
-record(s, {
|
||||
data_size = 0,
|
||||
payload_size = 0,
|
||||
n_messages = 0,
|
||||
datapoints = #{},
|
||||
x_axis = []
|
||||
}).
|
||||
|
||||
run({DB, Config}, RunConf) ->
|
||||
#{
|
||||
batches := NBatches,
|
||||
size := PSMultiplier,
|
||||
add_generation := AddGeneration,
|
||||
sleep := Sleep,
|
||||
dry_run := DryRun
|
||||
} = RunConf,
|
||||
{ok, _} = application:ensure_all_started(emqx_ds_backends),
|
||||
Dir = dir(DB),
|
||||
Filename = atom_to_list(DB) ++ ".dat",
|
||||
DryRun orelse
|
||||
begin
|
||||
io:format(user, "Running benchmark for ~p in ~p~n", [DB, Dir]),
|
||||
%% Ensure safe directory:
|
||||
{match, _} = re:run(Dir, filename:join("data", DB)),
|
||||
%% Ensure clean state:
|
||||
ok = emqx_ds:open_db(DB, Config),
|
||||
ok = emqx_ds:drop_db(DB),
|
||||
ok = file:del_dir_r(Dir),
|
||||
%% Open a fresh DB:
|
||||
ok = emqx_ds:open_db(DB, Config),
|
||||
S = lists:foldl(
|
||||
fun(Batch, Acc0) ->
|
||||
Size = PSMultiplier * Batch,
|
||||
io:format(user, "Storing batch with payload size ~p~n", [Size]),
|
||||
Acc1 = store_batch(DB, Size, Acc0),
|
||||
%% Sleep so all data is hopefully flushed:
|
||||
timer:sleep(Sleep),
|
||||
(Batch div AddGeneration) =:= 0 andalso
|
||||
emqx_ds:add_generation(DB),
|
||||
collect_datapoint(DB, Acc1)
|
||||
end,
|
||||
collect_datapoint(DB, #s{}),
|
||||
lists:seq(1, NBatches)
|
||||
),
|
||||
{ok, FD} = file:open(Filename, [write]),
|
||||
io:put_chars(FD, print(S)),
|
||||
file:close(FD)
|
||||
end,
|
||||
os:cmd("echo '" ++ gnuplot_script(Filename) ++ "' | gnuplot --persist -"),
|
||||
ok.
|
||||
|
||||
collect_datapoint(
|
||||
DB, S0 = #s{n_messages = N, data_size = DS, payload_size = PS, datapoints = DP0, x_axis = X}
|
||||
) ->
|
||||
NewData = [{"$_n", N}, {"$data", DS}, {"$payloads", PS} | dirsize(DB)],
|
||||
DP = lists:foldl(
|
||||
fun({Key, Val}, Acc) ->
|
||||
maps:update_with(
|
||||
Key,
|
||||
fun(M) -> M#{N => Val} end,
|
||||
#{},
|
||||
Acc
|
||||
)
|
||||
end,
|
||||
DP0,
|
||||
NewData
|
||||
),
|
||||
S0#s{
|
||||
datapoints = DP,
|
||||
x_axis = [N | X]
|
||||
}.
|
||||
|
||||
print(#s{x_axis = XX, datapoints = DP}) ->
|
||||
Cols = lists:sort(maps:keys(DP)),
|
||||
Lines = [
|
||||
%% Print header:
|
||||
Cols
|
||||
%% Scan through rows:
|
||||
| [
|
||||
%% Scan throgh columns:
|
||||
[integer_to_binary(maps:get(X, maps:get(Col, DP), 0)) || Col <- Cols]
|
||||
|| X <- lists:reverse(XX)
|
||||
]
|
||||
],
|
||||
lists:join(
|
||||
"\n",
|
||||
[lists:join(" ", Line) || Line <- Lines]
|
||||
).
|
||||
|
||||
dirsize(DB) ->
|
||||
RawOutput = os:cmd("cd " ++ dir(DB) ++ "; du -b --max-depth 1 ."),
|
||||
[
|
||||
begin
|
||||
[Sz, Dir] = string:lexemes(L, "\t"),
|
||||
{Dir, list_to_integer(Sz)}
|
||||
end
|
||||
|| L <- string:lexemes(RawOutput, "\n")
|
||||
].
|
||||
|
||||
dir(DB) ->
|
||||
filename:join(emqx_ds_storage_layer:base_dir(), DB).
|
||||
|
||||
store_batch(DB, PayloadSize, S0 = #s{n_messages = N, data_size = DS, payload_size = PS}) ->
|
||||
From = rand:bytes(16),
|
||||
BatchSize = 50,
|
||||
Batch = [
|
||||
#message{
|
||||
id = emqx_guid:gen(),
|
||||
timestamp = emqx_message:timestamp_now(),
|
||||
payload = rand:bytes(PayloadSize),
|
||||
from = From,
|
||||
topic = emqx_topic:join([
|
||||
<<"blah">>,
|
||||
<<"blah">>,
|
||||
'',
|
||||
<<"blah">>,
|
||||
From,
|
||||
<<"bazzzzzzzzzzzzzzzzzzzzzzz">>,
|
||||
integer_to_binary(I)
|
||||
])
|
||||
}
|
||||
|| I <- lists:seq(1, BatchSize)
|
||||
],
|
||||
ok = emqx_ds:store_batch(DB, Batch, #{sync => true}),
|
||||
S0#s{
|
||||
n_messages = N + length(Batch),
|
||||
data_size = DS + lists:sum(lists:map(fun msg_size/1, Batch)),
|
||||
payload_size = PS + length(Batch) * PayloadSize
|
||||
}.
|
||||
|
||||
%% We consider MQTT wire encoding to be "close to the ideal".
|
||||
msg_size(Msg = #message{}) ->
|
||||
iolist_size(emqx_frame:serialize(emqx_message:to_packet(undefined, Msg))).
|
||||
Loading…
Reference in New Issue