From 14952658d5f910ecf2b271149954e0f503c30452 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 5 May 2023 16:29:43 +0800 Subject: [PATCH] chore: convert rotation struct to roation_count --- .../emqx_bridge_compatible_config_tests.erl | 2 +- apps/emqx_conf/src/emqx_conf_schema.erl | 10 +- .../emqx_conf/test/emqx_conf_schema_tests.erl | 163 ++++++++++++------ 3 files changed, 123 insertions(+), 52 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl b/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl index acafb84ca..08bee15f3 100644 --- a/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl +++ b/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl @@ -129,7 +129,7 @@ assert_upgraded1(Map) -> ?assert(maps:is_key(<<"ssl">>, Map)). check(Conf) when is_map(Conf) -> - hocon_tconf:check_plain(emqx_bridge_schema, Conf). + hocon_tconf:check_plain(emqx_bridge_schema, Conf, #{required => false}). %% erlfmt-ignore %% this is config generated from v5.0.11 diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 682e8f59d..0e41d598c 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1342,11 +1342,17 @@ node_array() -> hoconsc:union([emqx_schema:comma_separated_atoms(), hoconsc:array(atom())]). ensure_file_handlers(Conf, _Opts) -> - FileFields = lists:map(fun({F, _}) -> list_to_binary(F) end, fields("log_file_handler")), + FileFields = lists:flatmap( + fun({F, Schema}) -> + Alias = [atom_to_binary(A) || A <- maps:get(aliases, Schema, [])], + [list_to_binary(F) | Alias] + end, + fields("log_file_handler") + ), HandlersWithoutName = maps:with(FileFields, Conf), HandlersWithName = maps:without(FileFields, Conf), emqx_utils_maps:deep_merge(#{<<"default">> => HandlersWithoutName}, HandlersWithName). convert_rotation(undefined, _Opts) -> undefined; -convert_rotation(#{} = Rotation, _Opts) -> maps:get(count, Rotation, 10); +convert_rotation(#{} = Rotation, _Opts) -> maps:get(<<"count">>, Rotation, 10); convert_rotation(Count, _Opts) when is_integer(Count) -> Count. diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 0e88da318..b0eea9a07 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -50,7 +50,7 @@ array_nodes_test() -> %% erlfmt-ignore -define(OUTDATED_LOG_CONF, """ -console_handler { +log.console_handler { burst_limit { enable = true max_count = 10000 @@ -72,9 +72,9 @@ console_handler { single_line = true supervisor_reports = error sync_mode_qlen = 100 - time_offset = system + time_offset = \"+02:00\" } -file_handlers { +log.file_handlers { default { burst_limit { enable = true @@ -84,19 +84,19 @@ file_handlers { chars_limit = unlimited drop_mode_qlen = 3000 enable = false - file = \"log/emqx.log\" + file = \"log/my-emqx.log\" flush_qlen = 8000 formatter = text - level = warning + level = debug max_depth = 100 - max_size = 52428800 + max_size = \"1024MB\" overload_kill { enable = true mem_size = 31457280 qlen = 20000 restart_after = 5000 } - rotation {count = 10, enable = true} + rotation {count = 20, enable = true} single_line = true supervisor_reports = error sync_mode_qlen = 100 @@ -105,45 +105,98 @@ file_handlers { } """ ). +-define(FORMATTER(TimeOffset), + {emqx_logger_textfmt, #{ + chars_limit => unlimited, + depth => 100, + single_line => true, + template => [time, " [", level, "] ", msg, "\n"], + time_offset => TimeOffset + }} +). + +-define(FILTERS, [{drop_progress_reports, {fun logger_filters:progress/2, stop}}]). +-define(LOG_CONFIG, #{ + burst_limit_enable => true, + burst_limit_max_count => 10000, + burst_limit_window_time => 1000, + drop_mode_qlen => 3000, + flush_qlen => 8000, + overload_kill_enable => true, + overload_kill_mem_size => 31457280, + overload_kill_qlen => 20000, + overload_kill_restart_after => 5000, + sync_mode_qlen => 100 +}). outdated_log_test() -> + validate_log(?OUTDATED_LOG_CONF). + +validate_log(Conf) -> BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), - Conf0 = <>, + Conf0 = <>, {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), ConfList = hocon_tconf:generate(emqx_conf_schema, ConfMap0), - ct:pal("fff:~p", [ConfList]), - Log = proplists:get_value(log, ConfList), - Console = proplists:get_value(console, Log), - File = proplists:get_value(file, Log), - ?assertEqual(1, Console, {Console, File}), - ok. + Kernel = proplists:get_value(kernel, ConfList), --define(NEW_LOG_CONF, - "" - "\n" - "console {\n" - " enable = true\n" - " formatter = text\n" - " level = warning\n" - " time_offset = system\n" - "}\n" - "file {\n" - " enable = true\n" - " file = \"log/emqx.log\"\n" - " formatter = text\n" - " level = warning\n" - " rotation {count = 10, enable = true}\n" - " time_offset = \"+01:00\"\n" - " }\n" - "file_handlers.default {\n" - " enable = false,\n" - " file = \"log/file_handlres_emqx.log\"\n" - " }\n" - "}\n" - " " - "" + ?assertEqual(silent, proplists:get_value(error_logger, Kernel)), + ?assertEqual(debug, proplists:get_value(logger_level, Kernel)), + Loggers = proplists:get_value(logger, Kernel), + FileHandler = lists:keyfind(logger_disk_log_h, 3, Loggers), + ?assertEqual( + {handler, default, logger_disk_log_h, #{ + config => ?LOG_CONFIG#{ + type => wrap, + file => "log/my-emqx.log", + max_no_bytes => 1073741824, + max_no_files => 20 + }, + filesync_repeat_interval => no_repeat, + filters => ?FILTERS, + formatter => ?FORMATTER("+01:00"), + level => debug + }}, + FileHandler + ), + ConsoleHandler = lists:keyfind(logger_std_h, 3, Loggers), + ?assertEqual( + {handler, console, logger_std_h, #{ + config => ?LOG_CONFIG#{type => standard_io}, + filters => ?FILTERS, + formatter => ?FORMATTER("+02:00"), + level => warning + }}, + ConsoleHandler + ). + +%% erlfmt-ignore +-define(KERNEL_LOG_CONF, + """ + log.console { + enable = true + formatter = text + level = warning + time_offset = \"+02:00\" + } + log.file { + enable = false + file = \"log/emqx.log\" + formatter = text + level = debug + rotation_count = 20 + rotation_size = \"1024MB\" + time_offset = \"+01:00\" + } + log.file_handlers.default { + enable = true + file = \"log/my-emqx.log\" + } + """ ). +log_test() -> + validate_log(?KERNEL_LOG_CONF). + %% erlfmt-ignore -define(BASE_AUTHN_ARRAY, """ @@ -181,38 +234,50 @@ authn_validations_test() -> OKHttps = to_bin(?BASE_AUTHN_ARRAY, [post, true, <<"https://127.0.0.1:8080">>]), Conf0 = <>, {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), - ?assert(is_list(hocon_tconf:generate(emqx_conf_schema, ConfMap0))), + {_, Res0} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap0, #{format => richmap}), + Headers0 = authentication_headers(Res0), + ?assertEqual(<<"application/json">>, maps:get(<<"content-type">>, Headers0)), + %% accept from converter + ?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers0)), OKHttp = to_bin(?BASE_AUTHN_ARRAY, [post, false, <<"http://127.0.0.1:8080">>]), Conf1 = <>, {ok, ConfMap1} = hocon:binary(Conf1, #{format => richmap}), - ?assert(is_list(hocon_tconf:generate(emqx_conf_schema, ConfMap1))), + {_, Res1} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap1, #{format => richmap}), + Headers1 = authentication_headers(Res1), + ?assertEqual(<<"application/json">>, maps:get(<<"content-type">>, Headers1), Headers1), + ?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers1), Headers1), DisableSSLWithHttps = to_bin(?BASE_AUTHN_ARRAY, [post, false, <<"https://127.0.0.1:8080">>]), Conf2 = <>, {ok, ConfMap2} = hocon:binary(Conf2, #{format => richmap}), ?assertThrow( ?ERROR(check_http_ssl_opts), - hocon_tconf:generate(emqx_conf_schema, ConfMap2) + hocon_tconf:map_translate(emqx_conf_schema, ConfMap2, #{format => richmap}) ), BadHeader = to_bin(?BASE_AUTHN_ARRAY, [get, true, <<"https://127.0.0.1:8080">>]), Conf3 = <>, {ok, ConfMap3} = hocon:binary(Conf3, #{format => richmap}), - ?assertThrow( - ?ERROR(check_http_headers), - hocon_tconf:generate(emqx_conf_schema, ConfMap3) - ), + {_, Res3} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap3, #{format => richmap}), + Headers3 = authentication_headers(Res3), + %% remove the content-type header when get method + ?assertEqual(false, maps:is_key(<<"content-type">>, Headers3), Headers3), + ?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers3), Headers3), BadHeaderWithTuple = binary:replace(BadHeader, [<<"[">>, <<"]">>], <<"">>, [global]), Conf4 = <>, {ok, ConfMap4} = hocon:binary(Conf4, #{format => richmap}), - ?assertThrow( - ?ERROR(check_http_headers), - hocon_tconf:generate(emqx_conf_schema, ConfMap4) - ), + {_, Res4} = hocon_tconf:map_translate(emqx_conf_schema, ConfMap4, #{}), + Headers4 = authentication_headers(Res4), + ?assertEqual(false, maps:is_key(<<"content-type">>, Headers4), Headers4), + ?assertEqual(<<"application/json">>, maps:get(<<"accept">>, Headers4), Headers4), ok. +authentication_headers(Conf) -> + [#{<<"headers">> := Headers}] = hocon_maps:get("authentication", Conf), + Headers. + doc_gen_test() -> %% the json file too large to encode. {