diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 6e467a8ba..4aa66b0fc 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -237,10 +237,12 @@ import_resource(#{<<"id">> := Id, config => Config, created_at => NCreatedAt, description => Desc}). + import_resources_and_rules(Resources, Rules, FromVersion) when FromVersion =:= "4.0" orelse FromVersion =:= "4.1" orelse - FromVersion =:= "4.2" -> + FromVersion =:= "4.2" orelse + FromVersion =:= "4.3" -> Configs = lists:foldl(fun compatible_version/2 , [], Resources), lists:foreach(fun(#{<<"actions">> := Actions} = Rule) -> NActions = apply_new_config(Actions, Configs), @@ -305,6 +307,17 @@ compatible_version(#{<<"id">> := ID, {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}), NHeaders = maps:put(<<"content-type">>, ContentType, covert_empty_headers(Headers)), [{ID, #{headers => NHeaders, method => Method}} | Acc]; + +compatible_version(#{<<"id">> := ID, + <<"type">> := Type, + <<"config">> := Config} = Resource, Acc) + when Type =:= <<"backend_mongo_single">> + orelse Type =:= <<"backend_mongo_sharded">> + orelse Type =:= <<"backend_mongo_rs">> -> + NewConfig = maps:merge(#{<<"srv_record">> => false}, Config), + {ok, _Resource} = import_resource(Resource#{<<"config">> := NewConfig}), + [{ID, NewConfig} | Acc]; + % normal version compatible_version(Resource, Acc) -> {ok, _Resource} = import_resource(Resource), @@ -511,16 +524,39 @@ import_modules(Modules) -> undefined -> ok; _ -> - lists:foreach(fun(#{<<"id">> := Id, - <<"type">> := Type, - <<"config">> := Config, - <<"enabled">> := Enabled, - <<"created_at">> := CreatedAt, - <<"description">> := Description}) -> - _ = emqx_modules:import_module({Id, any_to_atom(Type), Config, Enabled, CreatedAt, Description}) - end, Modules) + NModules = migrate_modules(Modules), + lists:foreach(fun(#{<<"id">> := Id, + <<"type">> := Type, + <<"config">> := Config, + <<"enabled">> := Enabled, + <<"created_at">> := CreatedAt, + <<"description">> := Description}) -> + _ = emqx_modules:import_module({Id, any_to_atom(Type), Config, Enabled, CreatedAt, Description}) + end, NModules) end. +migrate_modules(Modules) -> + migrate_modules(Modules, []). + +migrate_modules([], Acc) -> + lists:reverse(Acc); +migrate_modules([#{<<"type">> := <<"mongo_authentication">>, + <<"config">> := Config} = Module | More], Acc) -> + WMode = case maps:get(<<"w_mode">>, Config, <<"unsafe">>) of + <<"undef">> -> <<"unsafe">>; + Other -> Other + end, + RMode = case maps:get(<<"r_mode">>, Config, <<"master">>) of + <<"undef">> -> <<"master">>; + <<"slave-ok">> -> <<"slave_ok">>; + Other0 -> Other0 + end, + NConfig = Config#{<<"srv_record">> => false, + <<"w_mode">> => WMode, + <<"r_mode">> => RMode}, + migrate_modules(More, [Module#{<<"config">> => NConfig} | Acc]); +migrate_modules([Module | More], Acc) -> + migrate_modules(More, [Module | Acc]). import_schemas(Schemas) -> case ets:info(emqx_schema) of diff --git a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl new file mode 100644 index 000000000..3a697a17d --- /dev/null +++ b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl @@ -0,0 +1,68 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mongo_auth_module_migration_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-ifdef(EMQX_ENTERPRISE). +-include_lib("emqx_modules/include/emqx_modules.hrl"). +-endif. + +all() -> + emqx_ct:all(?MODULE). + +-ifdef(EMQX_ENTERPRISE). + +init_per_suite(Config) -> + application:load(emqx_modules_spec), + emqx_ct_helpers:start_apps([emqx_management, emqx_modules]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_modules, emqx_management]), + application:unload(emqx_modules_spec), + ok. + +t_import_4_2(Config) -> + ?assertMatch(ok, import("e4.2.8.json", Config)), + timer:sleep(100), + + MongoAuthNModule = emqx_modules_registry:find_module_by_type(mongo_authentication), + ?assertNotEqual(not_found, MongoAuthNModule), + ?assertMatch(#module{config = #{<<"srv_record">> := _}}, MongoAuthNModule), + delete_modules(). + +t_import_4_3(Config) -> + ?assertMatch(ok, import("e4.3.5.json", Config)), + timer:sleep(100), + + MongoAuthNModule = emqx_modules_registry:find_module_by_type(mongo_authentication), + ?assertNotEqual(not_found, MongoAuthNModule), + ?assertMatch(#module{config = #{<<"srv_record">> := _}}, MongoAuthNModule), + delete_modules(). + +import(File, Config) -> + Filename = filename:join(proplists:get_value(data_dir, Config), File), + emqx_mgmt_data_backup:import(Filename, "{}"). + +delete_modules() -> + [emqx_modules_registry:remove_module(Mod) || Mod <- emqx_modules_registry:get_modules()]. + +-endif. diff --git a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.2.8.json b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.2.8.json new file mode 100644 index 000000000..0ea956a93 --- /dev/null +++ b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.2.8.json @@ -0,0 +1 @@ +{"version":"4.2","date":"2021-11-15 01:52:40","modules":[{"id":"module:79002e0f","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1636941076704,"description":""},{"id":"module:34834081","type":"presence","config":{"qos":0},"enabled":true,"created_at":1636941076704,"description":""},{"id":"module:f6eb69d1","type":"recon","config":{},"enabled":true,"created_at":1636941076704,"description":""},{"id":"module:7ae737b2","type":"mongo_authentication","config":{"w_mode":"undef","verify":false,"type":"single","super_query_selector":"","super_query_field":"","super_query_collection":"","ssl":false,"server":"127.0.0.1:27017","r_mode":"undef","pool_size":8,"password":"public","login":"admin","keyfile":{"filename":"","file":""},"database":"mqtt","certfile":{"filename":"","file":""},"cacertfile":{"filename":"","file":""},"auth_source":"admin","auth_query_selector":"username=%u","auth_query_password_hash":"sha256","auth_query_password_field":"password","auth_query_collection":"mqtt_user","acl_query_selectors":[],"acl_query_collection":"mqtt_acl"},"enabled":false,"created_at":1636941148794,"description":""},{"id":"module:e8c63201","type":"internal_acl","config":{"acl_rule_file":"etc/acl.conf"},"enabled":true,"created_at":1636941076704,"description":""}],"rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"qP5m2iS9qnn51gHoGLbaiMo/GwE=","tags":"administrator"}],"auth_mnesia":[],"acl_mnesia":[],"schemas":[],"configs":[],"listeners_state":[]} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.3.5.json b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.3.5.json new file mode 100644 index 000000000..48828a9c0 --- /dev/null +++ b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.3.5.json @@ -0,0 +1 @@ +{"version":"4.3","rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"/mWV4UgV0xmVUZX4qdIXQvxXZB0=","tags":"administrator"}],"auth_mnesia":[],"acl_mnesia":[],"modules":[{"id":"module:5881add2","type":"mongo_authentication","config":{"w_mode":"undef","verify":false,"type":"single","super_query_selector":"","super_query_field":"","super_query_collection":"","ssl":false,"server":"127.0.0.1:27017","r_mode":"undef","pool_size":8,"password":"public","login":"admin","keyfile":{"filename":"","file":""},"database":"mqtt","certfile":{"filename":"","file":""},"cacertfile":{"filename":"","file":""},"auth_source":"admin","auth_query_selector":"username=%u","auth_query_password_hash":"sha256","auth_query_password_field":"password","auth_query_collection":"mqtt_user","acl_query_selectors":[],"acl_query_collection":"mqtt_acl"},"enabled":false,"created_at":1636942609573,"description":""},{"id":"module:2adb6480","type":"presence","config":{"qos":0},"enabled":true,"created_at":1636942586725,"description":""},{"id":"module:24fabe8a","type":"internal_acl","config":{"acl_rule_file":"etc/acl.conf"},"enabled":true,"created_at":1636942586725,"description":""},{"id":"module:22c70ab8","type":"recon","config":{},"enabled":true,"created_at":1636942586725,"description":""},{"id":"module:a59f9a4a","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1636942586725,"description":""}],"schemas":[],"configs":[],"listeners_state":[],"date":"2021-11-15 10:16:56"} \ No newline at end of file