From 319ec50c0d394f87b97dce0e6a5e1f609da4dc71 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 3 Apr 2024 11:57:01 +0800 Subject: [PATCH 1/4] fix: source bridges missing after restore the backup files --- apps/emqx/src/bhvrs/emqx_config_backup.erl | 15 ++++-- apps/emqx/src/emqx.app.src | 2 +- apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- apps/emqx_bridge/src/emqx_bridge_v2.erl | 21 ++++++++- .../src/emqx_mgmt_data_backup.erl | 47 +++++++++++++------ changes/ce/fix-12826.en.md | 18 +++++++ 6 files changed, 83 insertions(+), 22 deletions(-) create mode 100644 changes/ce/fix-12826.en.md diff --git a/apps/emqx/src/bhvrs/emqx_config_backup.erl b/apps/emqx/src/bhvrs/emqx_config_backup.erl index e4818a871..1ec08c23b 100644 --- a/apps/emqx/src/bhvrs/emqx_config_backup.erl +++ b/apps/emqx/src/bhvrs/emqx_config_backup.erl @@ -16,9 +16,14 @@ -module(emqx_config_backup). +-type ok_result() :: #{ + root_key => emqx_utils_maps:config_key(), + changed => [emqx_utils_maps:config_key_path()] +}. + +-type error_result() :: #{root_key => emqx_utils_maps:config_key(), reason => term()}. + -callback import_config(RawConf :: map()) -> - {ok, #{ - root_key => emqx_utils_maps:config_key(), - changed => [emqx_utils_maps:config_key_path()] - }} - | {error, #{root_key => emqx_utils_maps:config_key(), reason => term()}}. + {ok, ok_result()} + | {error, error_result()} + | {results, {[ok_result()], [error_result()]}}. diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 1d8c55fe9..462b7e74b 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -2,7 +2,7 @@ {application, emqx, [ {id, "emqx"}, {description, "EMQX Core"}, - {vsn, "5.2.0"}, + {vsn, "5.2.1"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index 9ef567f23..57dbc26ba 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.34"}, + {vsn, "0.1.35"}, {registered, [emqx_bridge_sup]}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index e834dc42e..10d597d36 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -1030,7 +1030,26 @@ bridge_v2_type_to_connector_type(Type) -> import_config(RawConf) -> %% actions structure - emqx_bridge:import_config(RawConf, <<"actions">>, ?ROOT_KEY_ACTIONS, config_key_path()). + ActionRes = emqx_bridge:import_config( + RawConf, <<"actions">>, ?ROOT_KEY_ACTIONS, config_key_path() + ), + SourceRes = emqx_bridge:import_config( + RawConf, <<"sources">>, ?ROOT_KEY_SOURCES, config_key_path_sources() + ), + combine_import_results([ActionRes, SourceRes]). + +combine_import_results(Results0) -> + Results = lists:foldr( + fun + ({ok, OkRes}, {OkAcc, ErrAcc}) -> + {[OkRes | OkAcc], ErrAcc}; + ({error, ErrRes}, {OkAcc, ErrAcc}) -> + {OkAcc, [ErrRes | ErrAcc]} + end, + {[], []}, + Results0 + ), + {results, Results}. %%==================================================================== %% Config Update Handler API diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 2aaa014a8..03eb7ac06 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -773,23 +773,42 @@ validate_cluster_hocon(RawConf) -> do_import_conf(RawConf, Opts) -> GenConfErrs = filter_errors(maps:from_list(import_generic_conf(RawConf))), maybe_print_conf_errors(GenConfErrs, Opts), - Errors = - lists:foldl( - fun(Module, ErrorsAcc) -> - case Module:import_config(RawConf) of - {ok, #{changed := Changed}} -> - maybe_print_changed(Changed, Opts), - ErrorsAcc; - {error, #{root_key := RootKey, reason := Reason}} -> - ErrorsAcc#{[RootKey] => Reason} - end - end, - GenConfErrs, - sort_importer_modules(find_behaviours(emqx_config_backup)) - ), + Modules = sort_importer_modules(find_behaviours(emqx_config_backup)), + Errors = lists:foldl(print_ok_results_collect_errors(RawConf, Opts), GenConfErrs, Modules), maybe_print_conf_errors(Errors, Opts), Errors. +print_ok_results_collect_errors(RawConf, Opts) -> + fun(Module, Errors) -> + case Module:import_config(RawConf) of + {results, {OkResults, ErrResults}} -> + print_ok_results(OkResults, Opts), + collect_errors(ErrResults, Errors); + {ok, OkResult} -> + print_ok_results([OkResult], Opts), + Errors; + {error, ErrResult} -> + collect_errors([ErrResult], Errors) + end + end. + +print_ok_results(Results, Opts) -> + lists:foreach( + fun(#{changed := Changed}) -> + maybe_print_changed(Changed, Opts) + end, + Results + ). + +collect_errors(Results, Errors) -> + lists:foldr( + fun(#{root_key := RootKey, reason := Reason}, Acc) -> + Acc#{[RootKey] => Reason} + end, + Errors, + Results + ). + sort_importer_modules(Modules) -> lists:sort( fun(M1, M2) -> order(M1, ?IMPORT_ORDER) =< order(M2, ?IMPORT_ORDER) end, diff --git a/changes/ce/fix-12826.en.md b/changes/ce/fix-12826.en.md new file mode 100644 index 000000000..51255059d --- /dev/null +++ b/changes/ce/fix-12826.en.md @@ -0,0 +1,18 @@ +Cannot import `sources` from backup files. + +Before the fix, the following configs in backup files cannot be imported: + +``` +sources { + mqtt { + source_c384b174 { + connector = source_connector_c8287217 + enable = true + parameters { + qos = 0 + topic = "t/#" + } + } + } +} +``` From 9d1a69aaa9ca77fce31d913847a70d763d3bc15e Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 3 Apr 2024 18:39:32 +0800 Subject: [PATCH 2/4] fix: cannot import retained messages --- apps/emqx_bridge/src/emqx_bridge_v2.erl | 4 ++-- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- .../src/emqx_retainer_mnesia.erl | 9 +++++++++ changes/ce/fix-12826.en.md | 20 ++++--------------- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index 10d597d36..e6feac7bd 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -1036,9 +1036,9 @@ import_config(RawConf) -> SourceRes = emqx_bridge:import_config( RawConf, <<"sources">>, ?ROOT_KEY_SOURCES, config_key_path_sources() ), - combine_import_results([ActionRes, SourceRes]). + group_import_results([ActionRes, SourceRes]). -combine_import_results(Results0) -> +group_import_results(Results0) -> Results = lists:foldr( fun ({ok, OkRes}, {OkAcc, ErrAcc}) -> diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index 248cc9310..4a8b3cdc3 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -2,7 +2,7 @@ {application, emqx_retainer, [ {description, "EMQX Retainer"}, % strict semver, bump manually! - {vsn, "5.0.21"}, + {vsn, "5.0.22"}, {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, diff --git a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl index bdc1f2c67..7e2a73a09 100644 --- a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl +++ b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl @@ -17,6 +17,7 @@ -module(emqx_retainer_mnesia). -behaviour(emqx_retainer). +-behaviour(emqx_db_backup). -include("emqx_retainer.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -54,6 +55,8 @@ -export([populate_index_meta/0]). -export([reindex/3]). +-export([backup_tables/0]). + -record(retained_message, {topic, msg, expiry_time}). -record(retained_index, {key, expiry_time}). -record(retained_index_meta, {key, read_indices, write_indices, reindexing, extra}). @@ -73,6 +76,12 @@ topics() -> [emqx_topic:join(I) || I <- mnesia:dirty_all_keys(?TAB_MESSAGE)]. +%%-------------------------------------------------------------------- +%% Data backup +%%-------------------------------------------------------------------- +backup_tables() -> + [?TAB_MESSAGE]. + %%-------------------------------------------------------------------- %% emqx_retainer callbacks %%-------------------------------------------------------------------- diff --git a/changes/ce/fix-12826.en.md b/changes/ce/fix-12826.en.md index 51255059d..28829cf87 100644 --- a/changes/ce/fix-12826.en.md +++ b/changes/ce/fix-12826.en.md @@ -1,18 +1,6 @@ -Cannot import `sources` from backup files. +Fixed an issue that prevented importing source data integrations and retained messages. -Before the fix, the following configs in backup files cannot be imported: +Before the fix: -``` -sources { - mqtt { - source_c384b174 { - connector = source_connector_c8287217 - enable = true - parameters { - qos = 0 - topic = "t/#" - } - } - } -} -``` +- source data integrations are ignored from the backup file +- importing the `mnesia` table for retained messages are not supported From 1c81c79a2cb5f6713e250a8b1f795845e01fef54 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sun, 7 Apr 2024 17:24:26 +0800 Subject: [PATCH 3/4] chore: add testcase for importing retained msgs and sources --- .../test/emqx_mgmt_data_backup_SUITE.erl | 23 ++++++++++++++++++ ...emqx-export-4.4.24-retainer-mqttsub.tar.gz | Bin 0 -> 2352 bytes 2 files changed, 23 insertions(+) create mode 100644 apps/emqx_management/test/emqx_mgmt_data_backup_SUITE_data/emqx-export-4.4.24-retainer-mqttsub.tar.gz diff --git a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl index 36a838743..e1d0a2512 100644 --- a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl @@ -18,6 +18,7 @@ -compile(export_all). -compile(nowarn_export_all). +-include_lib("emqx_utils/include/emqx_message.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -86,6 +87,28 @@ t_empty_export_import(_Config) -> ?assertEqual(Exp, emqx_mgmt_data_backup:import(FileName)), ?assertEqual(ExpRawConf, emqx:get_raw_config([])). +t_cluster_hocon_import_mqtt_subscribers_retainer_messages(Config) -> + FNameEmqx44 = "emqx-export-4.4.24-retainer-mqttsub.tar.gz", + BackupFile = filename:join(?config(data_dir, Config), FNameEmqx44), + Exp = {ok, #{db_errors => #{}, config_errors => #{}}}, + ?assertEqual(Exp, emqx_mgmt_data_backup:import(BackupFile)), + RawConfAfterImport = emqx:get_raw_config([]), + %% verify that MQTT sources are imported + ?assertMatch( + #{<<"sources">> := #{<<"mqtt">> := Sources}} when map_size(Sources) > 0, + RawConfAfterImport + ), + %% verify that retainer messages are imported + ?assertMatch( + {ok, [#message{payload = <<"test-payload">>}]}, + emqx_retainer:read_message(<<"test-retained-message/1">>) + ), + %% Export and import again + {ok, #{filename := FileName}} = emqx_mgmt_data_backup:export(), + ?assertEqual(Exp, emqx_mgmt_data_backup:import(FileName)), + ?assertEqual(RawConfAfterImport, emqx:get_raw_config([])), + ok. + t_cluster_hocon_export_import(Config) -> RawConfBeforeImport = emqx:get_raw_config([]), BootstrapFile = filename:join(?config(data_dir, Config), ?BOOTSTRAP_BACKUP), diff --git a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE_data/emqx-export-4.4.24-retainer-mqttsub.tar.gz b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE_data/emqx-export-4.4.24-retainer-mqttsub.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..67133f19d7e2958933f5fef09d07ccbfd64883e5 GIT binary patch literal 2352 zcmV-03D5Q)iwFR|V-jWn1MONtY$Hh(ZO4wCbu!Gb(r8bhmS>ikLAKLvw{3Q$MYBNI zIcx;GGl+rN&?d;Ify!a07`O7fv880)(_e;=oFrR^kF2k>J7s zB(Ci8tGc>lcbqt$buw(|A1Utszv}<`|5v^GRW|PL?pN@>=Mul7RkdnOs}SrX%fY15 z-}QYuXs=5jlN3d1H5xMiY&Q9)Qq#guL9(hf6-{fZ&AKKlYO7UOB)Rc%4WEVv)JFt{ zVq1g#;R)@aS*LS?T=)@xenJSJBL9XBLBgcka}C#d3L6lQrluX6|31TRsS1 z$};td<#cb}8r`an^*F9Go#;)=^24bA)+_}Biiv~zSZH9c@?$yTjtvC3N2Lb@{t?{8 z`_N*RDZ{o{noMCTd<)EP-bFhIHN0D^I~R>Ybnco9+l-NSYl9A z%+krWU~h6-E(9`SeFxm1GGB+~l8I=q?IL29M&&qt<%gx~_r7&+{aR^jR6W{L;7duZ zwW+Af)R7y0xT7eEZhLDkkt@xNAQ2EI4Z&> z?O?-q3FY9-c5#8yA$IF=1D(nAcswKq^ER=}F4p_5$t@t8MvxvY8{H$gW9@@9)lkS;ChJCz$GT%5h9-J+B&r!v6gDYBd+i2doZ4tYO!9(V%T^r9g&FIumYm3i90e9q6iHRN5do|2w+qI@o=<}v4b5`gy!fj*GK!h z=wRx7OeyN}Jx>WE4p9b%E^_H+S2x?y7k$)?>%CHM%-RL4#ihRt0Ix9ZTza8`tKg||CynncAz zR15G)8$W@Zbm;_x%>nKP3;Oi~@iHn+G}(1&D29iGC=7Hc4WAPYIHrVh{+JN6oWsm@ z0d0H>3o{TU&ZE^E-vD{#>!EyjW1{nD?71^=3{fKyEP`B2Hrx`QCT3ytMElYkQ3pAm zEFI$Z<;~ac-;awc-<56GfMkXk8I5CFjYW6uWQBEa8^a*b()M(;9;bjA9nPoZyCsU~OQwJ;?{*1W?OyHkq1A zn9qr!KI`K70?=vBf14ZkUwP`opEUFQrzrJS;{2yJlzMvpI}gI9C6Ec@BJ846z*M8! zWV`6I9$rdkOs84@eFswuJ^eM{-1mPqwWet4{oi@e9QkKWb>w+-+n9m>s;V@T_kXRr zmg4_;P-ZEcGahWN!sFfcH(z_@d%b*6wB4@W!^p%WD6;dV z9M^&4;T^~Axzz7v+me+3Sz9W+daW(3?S2-FMGf)JG-MKEU~R7OHm+k2$2u1I!3hqYm>`KM zLIxDLlpk-w2so+-#PTtSR{Ch$h38?g%nyseTDBY$?=w9)>hwMKaETY^tj_h}=zG|7 zm0$zR=ttZtBHxD!c;JU4F9^>I$g}hvJPaWakan-LYWoy{xk$y}6zH#jpSQyBD05%zGbx z#HfIhQHD+hQ71sJCA5U3?d@kfhf{ibniu~gVRY4T8XGh4zYd_E!2gEUYNYso9&|SN z|NYN_=D#NK|LW8kG`Sf}Zxinfs~p7v{Vu`|aV2?gcyy=82Fv{3Bgpn?H&_*SeZJMl z3kUyS{ro`${$KphoZ$a{nCjdKDPHQszZ|>oW)Fonad$w`cCQ!|L@UO>BIU0>{+D(4 zC;z@9NyRsp|AHj3krv8VcWhp750@rXiE*T;rGQ!XRUOiYdGk z-xWJ~>HQzP{mXYUOoN1&n#^o}a#Q`8qMor=+h(+;~Kok}nh-mg46NguSB;xR=1;pXnNpTS4VC`NI`yR5r9;((V zTBlxnNo#6T!l9mWo?nIK>qi@@Ss8OQqfT}G4|ji9V6LydEoD%Kv9qeG3dhZXClKVQ z=lNiTe_dj@c93HmHoRj@_+rhWCkVz3t9T{)vV&<}R-3if7lP;FuUSwo8NQ%+h!os= z9;c~Jt@FIDahbT!hrq3U@MJu2DgCd~y!BrjIgtGMY|J|UE1Ht1|LU5er1jr<(Am^~ zdl}ID+hqN>IyQqQtG;o`#kvK^$4(_GyX7Do^xR;nJ!GJcsOtnvaJ*k0IKE{sytw-y z|Cy+``%ZeYPbsE(@!z96bK97W|H<$FYOO|k{y!Hw8~nF_3YvdE{r(qb(6swsU|-Br z{&V1&4q`H#cD&CssbvP2Xb*)RvrFJ$8Snc9EqojJ_l`G5hrp7dRqUV3!M%AKY0f<|2>KSpFT2!CO3kk@(a2na7eeDj(c=R$OVN^ z3R^<-d)e?S!5l@lAFNqarvm_Z)@tGV!^%It5*-tMJntQFN}oiU8~+V^ZW}Y}|620< zzgkPJrTBjybT;_^2;MU99`pMT!=C#455vBgXg}^=0_>N Date: Sun, 7 Apr 2024 18:23:59 +0800 Subject: [PATCH 4/4] ci: run emqx_management both with ee and ce profile --- .../test/emqx_mgmt_data_backup_SUITE.erl | 43 +++++++++++-------- scripts/find-apps.sh | 4 ++ 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl index e1d0a2512..fee392479 100644 --- a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl @@ -88,25 +88,30 @@ t_empty_export_import(_Config) -> ?assertEqual(ExpRawConf, emqx:get_raw_config([])). t_cluster_hocon_import_mqtt_subscribers_retainer_messages(Config) -> - FNameEmqx44 = "emqx-export-4.4.24-retainer-mqttsub.tar.gz", - BackupFile = filename:join(?config(data_dir, Config), FNameEmqx44), - Exp = {ok, #{db_errors => #{}, config_errors => #{}}}, - ?assertEqual(Exp, emqx_mgmt_data_backup:import(BackupFile)), - RawConfAfterImport = emqx:get_raw_config([]), - %% verify that MQTT sources are imported - ?assertMatch( - #{<<"sources">> := #{<<"mqtt">> := Sources}} when map_size(Sources) > 0, - RawConfAfterImport - ), - %% verify that retainer messages are imported - ?assertMatch( - {ok, [#message{payload = <<"test-payload">>}]}, - emqx_retainer:read_message(<<"test-retained-message/1">>) - ), - %% Export and import again - {ok, #{filename := FileName}} = emqx_mgmt_data_backup:export(), - ?assertEqual(Exp, emqx_mgmt_data_backup:import(FileName)), - ?assertEqual(RawConfAfterImport, emqx:get_raw_config([])), + case emqx_release:edition() of + ce -> + ok; + ee -> + FNameEmqx44 = "emqx-export-4.4.24-retainer-mqttsub.tar.gz", + BackupFile = filename:join(?config(data_dir, Config), FNameEmqx44), + Exp = {ok, #{db_errors => #{}, config_errors => #{}}}, + ?assertEqual(Exp, emqx_mgmt_data_backup:import(BackupFile)), + RawConfAfterImport = emqx:get_raw_config([]), + %% verify that MQTT sources are imported + ?assertMatch( + #{<<"sources">> := #{<<"mqtt">> := Sources}} when map_size(Sources) > 0, + RawConfAfterImport + ), + %% verify that retainer messages are imported + ?assertMatch( + {ok, [#message{payload = <<"test-payload">>}]}, + emqx_retainer:read_message(<<"test-retained-message/1">>) + ), + %% Export and import again + {ok, #{filename := FileName}} = emqx_mgmt_data_backup:export(), + ?assertEqual(Exp, emqx_mgmt_data_backup:import(FileName)), + ?assertEqual(RawConfAfterImport, emqx:get_raw_config([])) + end, ok. t_cluster_hocon_export_import(Config) -> diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index 89f0a66e5..908f22d9c 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -101,6 +101,10 @@ matrix() { entries+=("$(format_app_entry "$app" 1 emqx "$runner")") entries+=("$(format_app_entry "$app" 1 emqx-enterprise "$runner")") ;; + apps/emqx_management) + entries+=("$(format_app_entry "$app" 1 emqx "$runner")") + entries+=("$(format_app_entry "$app" 1 emqx-enterprise "$runner")") + ;; apps/*) if [[ -f "${app}/BSL.txt" ]]; then profile='emqx-enterprise'