feat: add a backup copies for cluster.hocon
This commit is contained in:
parent
332daabcc5
commit
5146de5b1c
|
@ -91,7 +91,7 @@
|
||||||
-export([ensure_atom_conf_path/2]).
|
-export([ensure_atom_conf_path/2]).
|
||||||
|
|
||||||
-ifdef(TEST).
|
-ifdef(TEST).
|
||||||
-export([erase_all/0]).
|
-export([erase_all/0, backup_and_write/2]).
|
||||||
-endif.
|
-endif.
|
||||||
|
|
||||||
-include("logger.hrl").
|
-include("logger.hrl").
|
||||||
|
@ -105,6 +105,7 @@
|
||||||
-define(LISTENER_CONF_PATH(TYPE, LISTENER, PATH), [listeners, TYPE, LISTENER | PATH]).
|
-define(LISTENER_CONF_PATH(TYPE, LISTENER, PATH), [listeners, TYPE, LISTENER | PATH]).
|
||||||
|
|
||||||
-define(CONFIG_NOT_FOUND_MAGIC, '$0tFound').
|
-define(CONFIG_NOT_FOUND_MAGIC, '$0tFound').
|
||||||
|
-define(MAX_KEEP_BACKUP_CONFIGS, 10).
|
||||||
|
|
||||||
-export_type([
|
-export_type([
|
||||||
update_request/0,
|
update_request/0,
|
||||||
|
@ -601,44 +602,95 @@ save_to_config_map(Conf, RawConf) ->
|
||||||
-spec save_to_override_conf(boolean(), raw_config(), update_opts()) -> ok | {error, term()}.
|
-spec save_to_override_conf(boolean(), raw_config(), update_opts()) -> ok | {error, term()}.
|
||||||
save_to_override_conf(_, undefined, _) ->
|
save_to_override_conf(_, undefined, _) ->
|
||||||
ok;
|
ok;
|
||||||
%% TODO: Remove deprecated override conf file when 5.1
|
|
||||||
save_to_override_conf(true, RawConf, Opts) ->
|
save_to_override_conf(true, RawConf, Opts) ->
|
||||||
case deprecated_conf_file(Opts) of
|
case deprecated_conf_file(Opts) of
|
||||||
undefined ->
|
undefined ->
|
||||||
ok;
|
ok;
|
||||||
FileName ->
|
FileName ->
|
||||||
ok = filelib:ensure_dir(FileName),
|
backup_and_write(FileName, hocon_pp:do(RawConf, #{}))
|
||||||
case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of
|
|
||||||
ok ->
|
|
||||||
ok;
|
|
||||||
{error, Reason} ->
|
|
||||||
?SLOG(error, #{
|
|
||||||
msg => "failed_to_write_override_file",
|
|
||||||
filename => FileName,
|
|
||||||
reason => Reason
|
|
||||||
}),
|
|
||||||
{error, Reason}
|
|
||||||
end
|
|
||||||
end;
|
end;
|
||||||
save_to_override_conf(false, RawConf, _Opts) ->
|
save_to_override_conf(false, RawConf, _Opts) ->
|
||||||
case cluster_hocon_file() of
|
case cluster_hocon_file() of
|
||||||
undefined ->
|
undefined ->
|
||||||
ok;
|
ok;
|
||||||
FileName ->
|
FileName ->
|
||||||
ok = filelib:ensure_dir(FileName),
|
backup_and_write(FileName, hocon_pp:do(RawConf, #{}))
|
||||||
case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of
|
end.
|
||||||
|
|
||||||
|
%% @priv This is the same human-readable timestamp format as
|
||||||
|
%% hocon-cli generated app.<time>.config file name.
|
||||||
|
now_time() ->
|
||||||
|
Ts = os:system_time(millisecond),
|
||||||
|
{{Y, M, D}, {HH, MM, SS}} = calendar:system_time_to_local_time(Ts, millisecond),
|
||||||
|
Res = io_lib:format(
|
||||||
|
"~0p.~2..0b.~2..0b.~2..0b.~2..0b.~2..0b.~3..0b",
|
||||||
|
[Y, M, D, HH, MM, SS, Ts rem 1000]
|
||||||
|
),
|
||||||
|
lists:flatten(Res).
|
||||||
|
|
||||||
|
%% @private Backup the current config to a file with a timestamp suffix and
|
||||||
|
%% then save the new config to the config file.
|
||||||
|
backup_and_write(Path, Content) ->
|
||||||
|
%% this may fail, but we don't care
|
||||||
|
%% e.g. read-only file system
|
||||||
|
_ = filelib:ensure_dir(Path),
|
||||||
|
TmpFile = Path ++ ".tmp",
|
||||||
|
case file:write_file(TmpFile, Content) of
|
||||||
ok ->
|
ok ->
|
||||||
ok;
|
backup_and_replace(Path, TmpFile);
|
||||||
{error, Reason} ->
|
{error, Reason} ->
|
||||||
?SLOG(error, #{
|
?SLOG(error, #{
|
||||||
msg => "failed_to_save_conf_file",
|
msg => "failed_to_save_conf_file",
|
||||||
filename => FileName,
|
hint =>
|
||||||
|
"The updated cluster config is note saved on this node, please check the file system.",
|
||||||
|
filename => TmpFile,
|
||||||
reason => Reason
|
reason => Reason
|
||||||
}),
|
}),
|
||||||
{error, Reason}
|
%% e.g. read-only, it's not the end of the world
|
||||||
end
|
ok
|
||||||
end.
|
end.
|
||||||
|
|
||||||
|
backup_and_replace(Path, TmpPath) ->
|
||||||
|
Backup = Path ++ "." ++ now_time() ++ ".bak",
|
||||||
|
case file:rename(Path, Backup) of
|
||||||
|
ok ->
|
||||||
|
ok = file:rename(TmpPath, Path),
|
||||||
|
ok = prune_backup_files(Path);
|
||||||
|
{error, enoent} ->
|
||||||
|
%% not created yet
|
||||||
|
ok = file:rename(TmpPath, Path);
|
||||||
|
{error, Reason} ->
|
||||||
|
?SLOG(warning, #{
|
||||||
|
msg => "failed_to_backup_conf_file",
|
||||||
|
filename => Backup,
|
||||||
|
reason => Reason
|
||||||
|
}),
|
||||||
|
ok
|
||||||
|
end.
|
||||||
|
|
||||||
|
prune_backup_files(Path) ->
|
||||||
|
Files0 = filelib:wildcard(Path ++ ".*"),
|
||||||
|
Re = "\\.[0-9]{4}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{3}\\.bak$",
|
||||||
|
Files = lists:filter(fun(F) -> re:run(F, Re) =/= nomatch end, Files0),
|
||||||
|
Sorted = lists:reverse(lists:sort(Files)),
|
||||||
|
{_Keeps, Deletes} = lists:split(min(?MAX_KEEP_BACKUP_CONFIGS, length(Sorted)), Sorted),
|
||||||
|
lists:foreach(
|
||||||
|
fun(F) ->
|
||||||
|
case file:delete(F) of
|
||||||
|
ok ->
|
||||||
|
ok;
|
||||||
|
{error, Reason} ->
|
||||||
|
?SLOG(warning, #{
|
||||||
|
msg => "failed_to_delete_backup_conf_file",
|
||||||
|
filename => F,
|
||||||
|
reason => Reason
|
||||||
|
}),
|
||||||
|
ok
|
||||||
|
end
|
||||||
|
end,
|
||||||
|
Deletes
|
||||||
|
).
|
||||||
|
|
||||||
add_handlers() ->
|
add_handlers() ->
|
||||||
ok = emqx_config_logger:add_handler(),
|
ok = emqx_config_logger:add_handler(),
|
||||||
emqx_sys_mon:add_handler(),
|
emqx_sys_mon:add_handler(),
|
||||||
|
|
|
@ -31,7 +31,24 @@ init_per_suite(Config) ->
|
||||||
end_per_suite(_Config) ->
|
end_per_suite(_Config) ->
|
||||||
emqx_common_test_helpers:stop_apps([]).
|
emqx_common_test_helpers:stop_apps([]).
|
||||||
|
|
||||||
t_fill_default_values(_) ->
|
init_per_testcase(TestCase, Config) ->
|
||||||
|
try
|
||||||
|
?MODULE:TestCase({init, Config})
|
||||||
|
catch
|
||||||
|
error:function_clause ->
|
||||||
|
ok
|
||||||
|
end,
|
||||||
|
Config.
|
||||||
|
|
||||||
|
end_per_testcase(TestCase, Config) ->
|
||||||
|
try
|
||||||
|
?MODULE:TestCase({'end', Config})
|
||||||
|
catch
|
||||||
|
error:function_clause ->
|
||||||
|
ok
|
||||||
|
end.
|
||||||
|
|
||||||
|
t_fill_default_values(C) when is_list(C) ->
|
||||||
Conf = #{
|
Conf = #{
|
||||||
<<"broker">> => #{
|
<<"broker">> => #{
|
||||||
<<"perf">> => #{},
|
<<"perf">> => #{},
|
||||||
|
@ -60,7 +77,7 @@ t_fill_default_values(_) ->
|
||||||
_ = emqx_utils_json:encode(WithDefaults),
|
_ = emqx_utils_json:encode(WithDefaults),
|
||||||
ok.
|
ok.
|
||||||
|
|
||||||
t_init_load(_Config) ->
|
t_init_load(C) when is_list(C) ->
|
||||||
ConfFile = "./test_emqx.conf",
|
ConfFile = "./test_emqx.conf",
|
||||||
ok = file:write_file(ConfFile, <<"">>),
|
ok = file:write_file(ConfFile, <<"">>),
|
||||||
ExpectRootNames = lists:sort(hocon_schema:root_names(emqx_schema)),
|
ExpectRootNames = lists:sort(hocon_schema:root_names(emqx_schema)),
|
||||||
|
@ -79,7 +96,7 @@ t_init_load(_Config) ->
|
||||||
?assertMatch({ok, #{raw_config := 128}}, emqx:update_config([mqtt, max_topic_levels], 128)),
|
?assertMatch({ok, #{raw_config := 128}}, emqx:update_config([mqtt, max_topic_levels], 128)),
|
||||||
ok = file:delete(DeprecatedFile).
|
ok = file:delete(DeprecatedFile).
|
||||||
|
|
||||||
t_unknown_rook_keys(_) ->
|
t_unknown_rook_keys(C) when is_list(C) ->
|
||||||
?check_trace(
|
?check_trace(
|
||||||
#{timetrap => 1000},
|
#{timetrap => 1000},
|
||||||
begin
|
begin
|
||||||
|
@ -96,3 +113,46 @@ t_unknown_rook_keys(_) ->
|
||||||
end
|
end
|
||||||
),
|
),
|
||||||
ok.
|
ok.
|
||||||
|
|
||||||
|
t_cluster_hocon_backup({init, C}) ->
|
||||||
|
C;
|
||||||
|
t_cluster_hocon_backup({'end', _C}) ->
|
||||||
|
File = "backup-test.hocon",
|
||||||
|
Files = [File | filelib:wildcard(File ++ ".*.bak")],
|
||||||
|
lists:foreach(fun file:delete/1, Files);
|
||||||
|
t_cluster_hocon_backup(C) when is_list(C) ->
|
||||||
|
Write = fun(Path, Content) ->
|
||||||
|
%% avoid name clash
|
||||||
|
timer:sleep(1),
|
||||||
|
emqx_config:backup_and_write(Path, Content)
|
||||||
|
end,
|
||||||
|
File = "backup-test.hocon",
|
||||||
|
%% write 12 times, 10 backups should be kept
|
||||||
|
%% the latest one is File itself without suffix
|
||||||
|
%% the oldest one is expected to be deleted
|
||||||
|
N = 12,
|
||||||
|
Inputs = lists:seq(1, N),
|
||||||
|
Backups = lists:seq(N - 10, N - 1),
|
||||||
|
InputContents = [integer_to_binary(I) || I <- Inputs],
|
||||||
|
BackupContents = [integer_to_binary(I) || I <- Backups],
|
||||||
|
lists:foreach(
|
||||||
|
fun(Content) ->
|
||||||
|
Write(File, Content)
|
||||||
|
end,
|
||||||
|
InputContents
|
||||||
|
),
|
||||||
|
LatestContent = integer_to_binary(N),
|
||||||
|
?assertEqual({ok, LatestContent}, file:read_file(File)),
|
||||||
|
Re = "\\.[0-9]{4}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{3}\\.bak$",
|
||||||
|
Files = filelib:wildcard(File ++ ".*.bak"),
|
||||||
|
?assert(lists:all(fun(F) -> re:run(F, Re) =/= nomatch end, Files)),
|
||||||
|
%% keep only the latest 10
|
||||||
|
?assertEqual(10, length(Files)),
|
||||||
|
FilesSorted = lists:zip(lists:sort(Files), BackupContents),
|
||||||
|
lists:foreach(
|
||||||
|
fun({BackupFile, ExpectedContent}) ->
|
||||||
|
?assertEqual({ok, ExpectedContent}, file:read_file(BackupFile))
|
||||||
|
end,
|
||||||
|
FilesSorted
|
||||||
|
),
|
||||||
|
ok.
|
||||||
|
|
Loading…
Reference in New Issue