Merge pull request #6170 from tigercl/feat/mongo-migration
feat(migration): improve modules migration and add test cases
This commit is contained in:
tigercl
GitHub
commit
f335edaf6a
|
|
@ -237,10 +237,12 @@ import_resource(#{<<"id">> := Id,
|
|||
config => Config,
|
||||
created_at => NCreatedAt,
|
||||
description => Desc}).
|
||||
|
||||
import_resources_and_rules(Resources, Rules, FromVersion)
|
||||
when FromVersion =:= "4.0" orelse
|
||||
FromVersion =:= "4.1" orelse
|
||||
FromVersion =:= "4.2" ->
|
||||
FromVersion =:= "4.2" orelse
|
||||
FromVersion =:= "4.3" ->
|
||||
Configs = lists:foldl(fun compatible_version/2 , [], Resources),
|
||||
lists:foreach(fun(#{<<"actions">> := Actions} = Rule) ->
|
||||
NActions = apply_new_config(Actions, Configs),
|
||||
|
|
@ -305,6 +307,17 @@ compatible_version(#{<<"id">> := ID,
|
|||
{ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}),
|
||||
NHeaders = maps:put(<<"content-type">>, ContentType, covert_empty_headers(Headers)),
|
||||
[{ID, #{headers => NHeaders, method => Method}} | Acc];
|
||||
|
||||
compatible_version(#{<<"id">> := ID,
|
||||
<<"type">> := Type,
|
||||
<<"config">> := Config} = Resource, Acc)
|
||||
when Type =:= <<"backend_mongo_single">>
|
||||
orelse Type =:= <<"backend_mongo_sharded">>
|
||||
orelse Type =:= <<"backend_mongo_rs">> ->
|
||||
NewConfig = maps:merge(#{<<"srv_record">> => false}, Config),
|
||||
{ok, _Resource} = import_resource(Resource#{<<"config">> := NewConfig}),
|
||||
[{ID, NewConfig} | Acc];
|
||||
|
||||
% normal version
|
||||
compatible_version(Resource, Acc) ->
|
||||
{ok, _Resource} = import_resource(Resource),
|
||||
|
|
@ -511,16 +524,39 @@ import_modules(Modules) ->
|
|||
undefined ->
|
||||
ok;
|
||||
_ ->
|
||||
lists:foreach(fun(#{<<"id">> := Id,
|
||||
<<"type">> := Type,
|
||||
<<"config">> := Config,
|
||||
<<"enabled">> := Enabled,
|
||||
<<"created_at">> := CreatedAt,
|
||||
<<"description">> := Description}) ->
|
||||
_ = emqx_modules:import_module({Id, any_to_atom(Type), Config, Enabled, CreatedAt, Description})
|
||||
end, Modules)
|
||||
NModules = migrate_modules(Modules),
|
||||
lists:foreach(fun(#{<<"id">> := Id,
|
||||
<<"type">> := Type,
|
||||
<<"config">> := Config,
|
||||
<<"enabled">> := Enabled,
|
||||
<<"created_at">> := CreatedAt,
|
||||
<<"description">> := Description}) ->
|
||||
_ = emqx_modules:import_module({Id, any_to_atom(Type), Config, Enabled, CreatedAt, Description})
|
||||
end, NModules)
|
||||
end.
|
||||
|
||||
migrate_modules(Modules) ->
|
||||
migrate_modules(Modules, []).
|
||||
|
||||
migrate_modules([], Acc) ->
|
||||
lists:reverse(Acc);
|
||||
migrate_modules([#{<<"type">> := <<"mongo_authentication">>,
|
||||
<<"config">> := Config} = Module | More], Acc) ->
|
||||
WMode = case maps:get(<<"w_mode">>, Config, <<"unsafe">>) of
|
||||
<<"undef">> -> <<"unsafe">>;
|
||||
Other -> Other
|
||||
end,
|
||||
RMode = case maps:get(<<"r_mode">>, Config, <<"master">>) of
|
||||
<<"undef">> -> <<"master">>;
|
||||
<<"slave-ok">> -> <<"slave_ok">>;
|
||||
Other0 -> Other0
|
||||
end,
|
||||
NConfig = Config#{<<"srv_record">> => false,
|
||||
<<"w_mode">> => WMode,
|
||||
<<"r_mode">> => RMode},
|
||||
migrate_modules(More, [Module#{<<"config">> => NConfig} | Acc]);
|
||||
migrate_modules([Module | More], Acc) ->
|
||||
migrate_modules(More, [Module | Acc]).
|
||||
|
||||
import_schemas(Schemas) ->
|
||||
case ets:info(emqx_schema) of
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
-module(emqx_mongo_auth_module_migration_SUITE).
|
||||
|
||||
-compile(export_all).
|
||||
-compile(nowarn_export_all).
|
||||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
|
||||
-ifdef(EMQX_ENTERPRISE).
|
||||
-include_lib("emqx_modules/include/emqx_modules.hrl").
|
||||
-endif.
|
||||
|
||||
all() ->
|
||||
emqx_ct:all(?MODULE).
|
||||
|
||||
-ifdef(EMQX_ENTERPRISE).
|
||||
|
||||
init_per_suite(Config) ->
|
||||
application:load(emqx_modules_spec),
|
||||
emqx_ct_helpers:start_apps([emqx_management, emqx_modules]),
|
||||
Config.
|
||||
|
||||
end_per_suite(_Config) ->
|
||||
emqx_ct_helpers:stop_apps([emqx_modules, emqx_management]),
|
||||
application:unload(emqx_modules_spec),
|
||||
ok.
|
||||
|
||||
t_import_4_2(Config) ->
|
||||
?assertMatch(ok, import("e4.2.8.json", Config)),
|
||||
timer:sleep(100),
|
||||
|
||||
MongoAuthNModule = emqx_modules_registry:find_module_by_type(mongo_authentication),
|
||||
?assertNotEqual(not_found, MongoAuthNModule),
|
||||
?assertMatch(#module{config = #{<<"srv_record">> := _}}, MongoAuthNModule),
|
||||
delete_modules().
|
||||
|
||||
t_import_4_3(Config) ->
|
||||
?assertMatch(ok, import("e4.3.5.json", Config)),
|
||||
timer:sleep(100),
|
||||
|
||||
MongoAuthNModule = emqx_modules_registry:find_module_by_type(mongo_authentication),
|
||||
?assertNotEqual(not_found, MongoAuthNModule),
|
||||
?assertMatch(#module{config = #{<<"srv_record">> := _}}, MongoAuthNModule),
|
||||
delete_modules().
|
||||
|
||||
import(File, Config) ->
|
||||
Filename = filename:join(proplists:get_value(data_dir, Config), File),
|
||||
emqx_mgmt_data_backup:import(Filename, "{}").
|
||||
|
||||
delete_modules() ->
|
||||
[emqx_modules_registry:remove_module(Mod) || Mod <- emqx_modules_registry:get_modules()].
|
||||
|
||||
-endif.
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"version":"4.2","date":"2021-11-15 01:52:40","modules":[{"id":"module:79002e0f","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1636941076704,"description":""},{"id":"module:34834081","type":"presence","config":{"qos":0},"enabled":true,"created_at":1636941076704,"description":""},{"id":"module:f6eb69d1","type":"recon","config":{},"enabled":true,"created_at":1636941076704,"description":""},{"id":"module:7ae737b2","type":"mongo_authentication","config":{"w_mode":"undef","verify":false,"type":"single","super_query_selector":"","super_query_field":"","super_query_collection":"","ssl":false,"server":"127.0.0.1:27017","r_mode":"undef","pool_size":8,"password":"public","login":"admin","keyfile":{"filename":"","file":""},"database":"mqtt","certfile":{"filename":"","file":""},"cacertfile":{"filename":"","file":""},"auth_source":"admin","auth_query_selector":"username=%u","auth_query_password_hash":"sha256","auth_query_password_field":"password","auth_query_collection":"mqtt_user","acl_query_selectors":[],"acl_query_collection":"mqtt_acl"},"enabled":false,"created_at":1636941148794,"description":""},{"id":"module:e8c63201","type":"internal_acl","config":{"acl_rule_file":"etc/acl.conf"},"enabled":true,"created_at":1636941076704,"description":""}],"rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"qP5m2iS9qnn51gHoGLbaiMo/GwE=","tags":"administrator"}],"auth_mnesia":[],"acl_mnesia":[],"schemas":[],"configs":[],"listeners_state":[]}
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"version":"4.3","rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"/mWV4UgV0xmVUZX4qdIXQvxXZB0=","tags":"administrator"}],"auth_mnesia":[],"acl_mnesia":[],"modules":[{"id":"module:5881add2","type":"mongo_authentication","config":{"w_mode":"undef","verify":false,"type":"single","super_query_selector":"","super_query_field":"","super_query_collection":"","ssl":false,"server":"127.0.0.1:27017","r_mode":"undef","pool_size":8,"password":"public","login":"admin","keyfile":{"filename":"","file":""},"database":"mqtt","certfile":{"filename":"","file":""},"cacertfile":{"filename":"","file":""},"auth_source":"admin","auth_query_selector":"username=%u","auth_query_password_hash":"sha256","auth_query_password_field":"password","auth_query_collection":"mqtt_user","acl_query_selectors":[],"acl_query_collection":"mqtt_acl"},"enabled":false,"created_at":1636942609573,"description":""},{"id":"module:2adb6480","type":"presence","config":{"qos":0},"enabled":true,"created_at":1636942586725,"description":""},{"id":"module:24fabe8a","type":"internal_acl","config":{"acl_rule_file":"etc/acl.conf"},"enabled":true,"created_at":1636942586725,"description":""},{"id":"module:22c70ab8","type":"recon","config":{},"enabled":true,"created_at":1636942586725,"description":""},{"id":"module:a59f9a4a","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1636942586725,"description":""}],"schemas":[],"configs":[],"listeners_state":[],"date":"2021-11-15 10:16:56"}
|
||||
Loading…
Reference in New Issue