From fde506838abd31c7223c2b664337457c760b185f Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 4 Jul 2023 14:52:05 +0300 Subject: [PATCH] fix(ft-api): implement import config behaviour --- apps/emqx_ft/src/emqx_ft_conf.erl | 28 +++++++++++ apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 63 ++++++++++++++++++++---- apps/emqx_s3/src/emqx_s3.app.src | 2 +- 3 files changed, 83 insertions(+), 10 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 0907ffd09..f936b3056 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -19,6 +19,7 @@ -module(emqx_ft_conf). -behaviour(emqx_config_handler). +-behaviour(emqx_config_backup). -include_lib("emqx/include/logger.hrl"). @@ -45,6 +46,11 @@ post_config_update/5 ]). +%% callbacks for emqx_config_backup +-export([ + import_config/1 +]). + -type update_request() :: emqx_config:config(). -type milliseconds() :: non_neg_integer(). @@ -110,6 +116,24 @@ get() -> update(Config) -> emqx_conf:update([file_transfer], Config, #{override_to => cluster}). +%%---------------------------------------------------------------------------------------- +%% Data backup +%%---------------------------------------------------------------------------------------- + +import_config(#{<<"file_transfer">> := FTConf}) -> + OldFTConf = emqx:get_raw_config([file_transfer], #{}), + NewFTConf = maps:merge(OldFTConf, FTConf), + case emqx_conf:update([file_transfer], NewFTConf, #{override_to => cluster}) of + {ok, #{raw_config := NewRawConf}} -> + Changed = maps:get(changed, emqx_utils_maps:diff_maps(NewRawConf, FTConf)), + ChangedPaths = [[file_transfer, K] || K <- maps:keys(Changed)], + {ok, #{root_key => file_transfer, changed => ChangedPaths}}; + Error -> + {error, #{root_key => file_transfer, reason => Error}} + end; +import_config(_) -> + {ok, #{root_key => file_transfer, changed => []}}. + %%-------------------------------------------------------------------- %% emqx_config_handler callbacks %%-------------------------------------------------------------------- @@ -146,6 +170,10 @@ post_config_update([file_transfer | _], _Req, NewConfig, OldConfig, _AppEnvs) -> {error, Reason} end. +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + propagate_config_update(Fun, ConfKey, NewConfig, OldConfig) -> NewSubConf = emqx_utils_maps:deep_get(ConfKey, NewConfig, undefined), OldSubConf = emqx_utils_maps:deep_get(ConfKey, OldConfig, undefined), diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index fc4391cde..7316847aa 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -238,23 +238,18 @@ t_persist_ssl_certfiles(Config) -> [], list_ssl_certfiles(Config) ), - S3Config = #{ - <<"bucket">> => <<"emqx">>, - <<"host">> => <<"https://localhost">>, - <<"port">> => 9000 - }, ?assertMatch( {error, {pre_config_update, _, {bad_ssl_config, #{}}}}, emqx_ft_conf:update( mk_storage(true, #{ - <<"s3">> => S3Config#{ + <<"s3">> => mk_s3_config(#{ <<"transport_options">> => #{ <<"ssl">> => #{ <<"certfile">> => <<"cert.pem">>, <<"keyfile">> => <<"key.pem">> } } - } + }) }) ) ), @@ -262,14 +257,14 @@ t_persist_ssl_certfiles(Config) -> {ok, _}, emqx_ft_conf:update( mk_storage(false, #{ - <<"s3">> => S3Config#{ + <<"s3">> => mk_s3_config(#{ <<"transport_options">> => #{ <<"ssl">> => #{ <<"certfile">> => emqx_ft_test_helpers:pem_privkey(), <<"keyfile">> => emqx_ft_test_helpers:pem_privkey() } } - } + }) }) ) ), @@ -299,6 +294,48 @@ t_persist_ssl_certfiles(Config) -> emqx_ft_conf:update(mk_storage(true)) ). +t_import(_Config) -> + {ok, _} = + emqx_ft_conf:update( + mk_storage(true, #{ + <<"s3">> => mk_s3_config(#{ + <<"transport_options">> => #{ + <<"ssl">> => #{ + <<"certfile">> => emqx_ft_test_helpers:pem_privkey(), + <<"keyfile">> => emqx_ft_test_helpers:pem_privkey() + } + } + }) + }) + ), + + {ok, #{filename := BackupFile}} = emqx_mgmt_data_backup:export(), + {ok, FileNames} = erl_tar:table(BackupFile, [compressed]), + [HoconFileName] = lists:filter( + fun(N) -> filename:basename(N) =:= "cluster.hocon" end, FileNames + ), + {ok, [{_, HoconConfig}]} = erl_tar:extract(BackupFile, [ + memory, compressed, {files, [HoconFileName]} + ]), + {ok, BackupConfig} = hocon:binary(HoconConfig), + FTBackupConfig = maps:with([<<"file_transfer">>], BackupConfig), + + {ok, _} = emqx_ft_conf:update(mk_storage(true)), + + ?assertMatch( + {ok, _}, + emqx_ft_conf:import_config(FTBackupConfig) + ), + + ?assertMatch( + #{local := #{exporter := #{s3 := #{enable := true}}}}, + emqx_ft_conf:storage() + ). + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + mk_storage(Enabled) -> mk_storage(Enabled, #{<<"local">> => #{}}). @@ -312,6 +349,14 @@ mk_storage(Enabled, Exporter) -> } }. +mk_s3_config(S3Config) -> + BaseS3Config = #{ + <<"bucket">> => <<"emqx">>, + <<"host">> => <<"https://localhost">>, + <<"port">> => 9000 + }, + maps:merge(BaseS3Config, S3Config). + gen_clientid() -> emqx_base62:encode(emqx_guid:gen()). diff --git a/apps/emqx_s3/src/emqx_s3.app.src b/apps/emqx_s3/src/emqx_s3.app.src index 0599d7923..6dee7ed0a 100644 --- a/apps/emqx_s3/src/emqx_s3.app.src +++ b/apps/emqx_s3/src/emqx_s3.app.src @@ -1,6 +1,6 @@ {application, emqx_s3, [ {description, "EMQX S3"}, - {vsn, "5.0.8"}, + {vsn, "5.0.9"}, {modules, []}, {registered, [emqx_s3_sup]}, {applications, [