yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: convert rotation struct to roation_count

This commit is contained in:
Zhongwen Deng 2023-05-05 16:29:43 +08:00
parent bf87aebbba
commit 14952658d5
3 changed files with 123 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl
View File

@ -129,7 +129,7 @@ assert_upgraded1(Map) ->
?assert(maps:is_key(<<"ssl">>, Map)).
check(Conf) when is_map(Conf) ->
hocon_tconf:check_plain(emqx_bridge_schema, Conf).
hocon_tconf:check_plain(emqx_bridge_schema, Conf, #{required => false}).
%% erlfmt-ignore
%% this is config generated from v5.0.11

10
apps/emqx_conf/src/emqx_conf_schema.erl
View File

@ -1342,11 +1342,17 @@ node_array() ->
hoconsc:union([emqx_schema:comma_separated_atoms(), hoconsc:array(atom())]).
ensure_file_handlers(Conf, _Opts) ->
FileFields = lists:map(fun({F, _}) -> list_to_binary(F) end, fields("log_file_handler")),
FileFields = lists:flatmap(
fun({F, Schema}) ->
Alias = [atom_to_binary(A) || A <- maps:get(aliases, Schema, [])],
[list_to_binary(F) | Alias]
end,
fields("log_file_handler")
),
HandlersWithoutName = maps:with(FileFields, Conf),
HandlersWithName = maps:without(FileFields, Conf),
emqx_utils_maps:deep_merge(#{<<"default">> => HandlersWithoutName}, HandlersWithName).
convert_rotation(undefined, _Opts) -> undefined;
convert_rotation(#{} = Rotation, _Opts) -> maps:get(count, Rotation, 10);
convert_rotation(#{} = Rotation, _Opts) -> maps:get(<<"count">>, Rotation, 10);
convert_rotation(Count, _Opts) when is_integer(Count) -> Count.

163
apps/emqx_conf/test/emqx_conf_schema_tests.erl
View File

@ -50,7 +50,7 @@ array_nodes_test() ->
%% erlfmt-ignore
-define(OUTDATED_LOG_CONF,
"""
console_handler {
log.console_handler {
burst_limit {
enable = true
max_count = 10000
@ -72,9 +72,9 @@ console_handler {
single_line = true
supervisor_reports = error
sync_mode_qlen = 100
time_offset = system
time_offset = \"+02:00\"
}
file_handlers {
log.file_handlers {
default {
burst_limit {
enable = true
@ -84,19 +84,19 @@ file_handlers {
chars_limit = unlimited
drop_mode_qlen = 3000
enable = false
file = \"log/emqx.log\"
file = \"log/my-emqx.log\"
flush_qlen = 8000
formatter = text
level = warning
level = debug
max_depth = 100
max_size = 52428800
max_size = \"1024MB\"
overload_kill {
enable = true
mem_size = 31457280
qlen = 20000
restart_after = 5000
}
rotation {count = 10, enable = true}
rotation {count = 20, enable = true}
single_line = true
supervisor_reports = error
sync_mode_qlen = 100
@ -105,45 +105,98 @@ file_handlers {
}
"""
).
-define(FORMATTER(TimeOffset),
{emqx_logger_textfmt, #{
chars_limit => unlimited,
depth => 100,
single_line => true,
template => [time, " [", level, "] ", msg, "\n"],
time_offset => TimeOffset
}}
).
-define(FILTERS, [{drop_progress_reports, {fun logger_filters:progress/2, stop}}]).
-define(LOG_CONFIG, #{
burst_limit_enable => true,
burst_limit_max_count => 10000,
burst_limit_window_time => 1000,
drop_mode_qlen => 3000,
flush_qlen => 8000,
overload_kill_enable => true,
overload_kill_mem_size => 31457280,
overload_kill_qlen => 20000,
overload_kill_restart_after => 5000,
sync_mode_qlen => 100
}).
outdated_log_test() ->
validate_log(?OUTDATED_LOG_CONF).
validate_log(Conf) ->
BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]),
Conf0 = <<BaseConf/binary, ?OUTDATED_LOG_CONF>>,
Conf0 = <<BaseConf/binary, (list_to_binary(Conf))/binary>>,
{ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}),
ConfList = hocon_tconf:generate(emqx_conf_schema, ConfMap0),
ct:pal("fff:~p", [ConfList]),
Log = proplists:get_value(log, ConfList),
Console = proplists:get_value(console, Log),
File = proplists:get_value(file, Log),
?assertEqual(1, Console, {Console, File}),
ok.
Kernel = proplists:get_value(kernel, ConfList),
-define(NEW_LOG_CONF,
""
"\n"
"console {\n"
" enable = true\n"
" formatter = text\n"
" level = warning\n"
" time_offset = system\n"
"}\n"
"file {\n"
" enable = true\n"
" file = \"log/emqx.log\"\n"
" formatter = text\n"
" level = warning\n"
" rotation {count = 10, enable = true}\n"
" time_offset = \"+01:00\"\n"
" }\n"
"file_handlers.default {\n"
" enable = false,\n"
" file = \"log/file_handlres_emqx.log\"\n"
" }\n"
"}\n"
" "
""
?assertEqual(silent, proplists:get_value(error_logger, Kernel)),
?assertEqual(debug, proplists:get_value(logger_level, Kernel)),
Loggers = proplists:get_value(logger, Kernel),
FileHandler = lists:keyfind(logger_disk_log_h, 3, Loggers),
?assertEqual(
{handler, default, logger_disk_log_h, #{
config => ?LOG_CONFIG#{
type => wrap,
file => "log/my-emqx.log",
max_no_bytes => 1073741824,
max_no_files => 20
},
filesync_repeat_interval => no_repeat,
filters => ?FILTERS,
formatter => ?FORMATTER("+01:00"),
level => debug
}},
FileHandler
),
ConsoleHandler = lists:keyfind(logger_std_h, 3, Loggers),
?assertEqual(
{handler, console, logger_std_h, #{
config => ?LOG_CONFIG#{type => standard_io},
filters => ?FILTERS,
formatter => ?FORMATTER("+02:00"),
level => warning
}},
ConsoleHandler
).
%% erlfmt-ignore
-define(KERNEL_LOG_CONF,
"""
log.console {
enable = true
formatter = text
level = warning
time_offset = \"+02:00\"
}
log.file {
enable = false
file = \"log/emqx.log\"
formatter = text
level = debug
rotation_count = 20
rotation_size = \"1024MB\"
time_offset = \"+01:00\"
}
log.file_handlers.default {
enable = true
file = \"log/my-emqx.log\"
}
"""
).
log_test() ->
validate_log(?KERNEL_LOG_CONF).
%% erlfmt-ignore
-define(BASE_AUTHN_ARRAY,
"""
@ -181,38 +234,50 @@ authn_validations_test() ->
OKHttps = to_bin(?BASE_AUTHN_ARRAY, [post, true, <<"https://127.0.0.1:8080">>]),
Conf0 = <<BaseConf/binary, OKHttps/binary>>,
{ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}),
?assert(is_list(hocon_tconf:generate(emqx_conf_schema, ConfMap0))),
{_, Res0} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap0, #{format => richmap}),
Headers0 = authentication_headers(Res0),
?assertEqual(<<"application/json">>, maps:get(<<"content-type">>, Headers0)),
%% accept from converter
?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers0)),
OKHttp = to_bin(?BASE_AUTHN_ARRAY, [post, false, <<"http://127.0.0.1:8080">>]),
Conf1 = <<BaseConf/binary, OKHttp/binary>>,
{ok, ConfMap1} = hocon:binary(Conf1, #{format => richmap}),
?assert(is_list(hocon_tconf:generate(emqx_conf_schema, ConfMap1))),
{_, Res1} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap1, #{format => richmap}),
Headers1 = authentication_headers(Res1),
?assertEqual(<<"application/json">>, maps:get(<<"content-type">>, Headers1), Headers1),
?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers1), Headers1),
DisableSSLWithHttps = to_bin(?BASE_AUTHN_ARRAY, [post, false, <<"https://127.0.0.1:8080">>]),
Conf2 = <<BaseConf/binary, DisableSSLWithHttps/binary>>,
{ok, ConfMap2} = hocon:binary(Conf2, #{format => richmap}),
?assertThrow(
?ERROR(check_http_ssl_opts),
hocon_tconf:generate(emqx_conf_schema, ConfMap2)
hocon_tconf:map_translate(emqx_conf_schema, ConfMap2, #{format => richmap})
),
BadHeader = to_bin(?BASE_AUTHN_ARRAY, [get, true, <<"https://127.0.0.1:8080">>]),
Conf3 = <<BaseConf/binary, BadHeader/binary>>,
{ok, ConfMap3} = hocon:binary(Conf3, #{format => richmap}),
?assertThrow(
?ERROR(check_http_headers),
hocon_tconf:generate(emqx_conf_schema, ConfMap3)
),
{_, Res3} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap3, #{format => richmap}),
Headers3 = authentication_headers(Res3),
%% remove the content-type header when get method
?assertEqual(false, maps:is_key(<<"content-type">>, Headers3), Headers3),
?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers3), Headers3),
BadHeaderWithTuple = binary:replace(BadHeader, [<<"[">>, <<"]">>], <<"">>, [global]),
Conf4 = <<BaseConf/binary, BadHeaderWithTuple/binary>>,
{ok, ConfMap4} = hocon:binary(Conf4, #{format => richmap}),
?assertThrow(
?ERROR(check_http_headers),
hocon_tconf:generate(emqx_conf_schema, ConfMap4)
),
{_, Res4} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap4, #{}),
Headers4 = authentication_headers(Res4),
?assertEqual(false, maps:is_key(<<"content-type">>, Headers4), Headers4),
?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers4), Headers4),
ok.
authentication_headers(Conf) ->
[#{<<"headers">> := Headers}] = hocon_maps:get("authentication", Conf),
Headers.
doc_gen_test() ->
%% the json file too large to encode.
{