Merge pull request #8031 from HJianBo/fix-modules-import-auth-mnesia

Fix modules import auth mnesia

apps/emqx_management/src/emqx_mgmt_data_backup.erl

@@ -657,26 +657,38 @@ import(Filename, OverridesJson) ->
 -endif.
 
 do_import_data(Data, Version) ->
-    do_import_extra_data(Data, Version),
     import_resources_and_rules(maps:get(<<"resources">>, Data, []), maps:get(<<"rules">>, Data, []), Version),
     import_blacklist(maps:get(<<"blacklist">>, Data, [])),
     import_applications(maps:get(<<"apps">>, Data, [])),
     import_users(maps:get(<<"users">>, Data, [])),
+    %% Import modules first to ensure the data of auth_mnesia module can be imported.
+    %% XXX: In opensource version, can't import if the emqx_auth_mnesia plug-in is not started??
+    do_import_enterprise_modules(Data, Version),
     import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])),
     import_auth_username(maps:get(<<"auth_username">>, Data, [])),
     import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, [])),
-    import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])).
+    import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])),
+    %% always do extra import at last, to make sure resources are initiated before
+    %% creating the schemas
+    do_import_extra_data(Data, Version).
 
 -ifdef(EMQX_ENTERPRISE).
 do_import_extra_data(Data, _Version) ->
     _ = import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])),
-    _ = import_modules(maps:get(<<"modules">>, Data, [])),
     _ = import_schemas(maps:get(<<"schemas">>, Data, [])),
     ok.
 -else.
 do_import_extra_data(_Data, _Version) -> ok.
 -endif.
 
+-ifdef(EMQX_ENTERPRISE).
+do_import_enterprise_modules(Data, _Version) ->
+    _ = import_modules(maps:get(<<"modules">>, Data, [])),
+    ok.
+-else.
+do_import_enterprise_modules(_Data, _Version) -> ok.
+-endif.
+
 covert_empty_headers([]) -> #{};
 covert_empty_headers(Other) -> Other.
 

apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl

@@ -0,0 +1,73 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_auth_mnesia_data_export_import_SUITE).
+
+-compile([export_all, nowarn_export_all]).
+
+-ifdef(EMQX_ENTERPRISE).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("emqx_modules/include/emqx_modules.hrl").
+
+%%--------------------------------------------------------------------
+%% Setups
+%%--------------------------------------------------------------------
+all() ->
+    emqx_ct:all(?MODULE).
+
+init_per_suite(Cfg) ->
+    _ = application:load(emqx_modules_spec),
+    emqx_ct_helpers:start_apps([emqx_rule_engine, emqx_modules,
+                                emqx_management, emqx_dashboard]),
+    Cfg.
+
+end_per_suite(Cfg) ->
+    emqx_ct_helpers:stop_apps([emqx_dashboard, emqx_management,
+                               emqx_modules, emqx_rule_engine]),
+    Cfg.
+
+get_data_path() ->
+    emqx_ct_helpers:deps_path(emqx_management, "test/emqx_auth_mnesia_data_export_import_SUITE_data/").
+
+import(FilePath, _Version) ->
+    ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>),
+    [_] = lists:filter(
+            fun(#module{type = mnesia_authentication}) -> true;
+               (_) -> false
+            end, emqx_modules_registry:get_modules()),
+    ?assertNotEqual(0, ets:info(emqx_user, size)),
+    ?assertNotEqual(0, ets:info(emqx_acl, size)).
+
+%%--------------------------------------------------------------------
+%% Cases
+%%--------------------------------------------------------------------
+
+t_importee427(_) ->
+    import("ee427.json", ee427),
+    {ok, _} = emqx_mgmt_data_backup:export(),
+    remove_all_users_and_acl().
+
+t_importee430(_) ->
+    import("ee435.json", ee435),
+    {ok, _} = emqx_mgmt_data_backup:export(),
+    remove_all_users_and_acl().
+
+remove_all_users_and_acl() ->
+    mnesia:delete_table(emqx_user),
+    mnesia:delete_table(emqx_acl).
+
+-endif.

apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json

@@ -0,0 +1,88 @@
+{
+  "version": "4.2",
+  "date": "2022-05-24 12:09:56",
+  "modules": [
+    {
+      "id": "module:842a5c57",
+      "type": "mnesia_authentication",
+      "config": {
+        "password_hash": "sha256"
+      },
+      "enabled": true,
+      "created_at": 1653365372585,
+      "description": ""
+    },
+    {
+      "id": "module:e3d38e5a",
+      "type": "retainer",
+      "config": {
+        "storage_type": "ram",
+        "max_retained_messages": 0,
+        "max_payload_size": "1MB",
+        "expiry_interval": 0
+      },
+      "enabled": true,
+      "created_at": 1652436434273,
+      "description": ""
+    },
+    {
+      "id": "module:4f911150",
+      "type": "presence",
+      "config": {
+        "qos": 0
+      },
+      "enabled": true,
+      "created_at": 1652436434273,
+      "description": ""
+    },
+    {
+      "id": "module:db11d08f",
+      "type": "recon",
+      "config": {},
+      "enabled": true,
+      "created_at": 1652436434273,
+      "description": ""
+    }
+  ],
+  "rules": [],
+  "resources": [],
+  "blacklist": [],
+  "apps": [
+    {
+      "id": "admin",
+      "secret": "public",
+      "name": "Default",
+      "desc": "Application user",
+      "status": true,
+      "expired": "undefined"
+    }
+  ],
+  "users": [
+    {
+      "username": "admin",
+      "password": "oFu0ZiOAYJmB1DyOzoLwEervBK0=",
+      "tags": "administrator"
+    }
+  ],
+  "auth_mnesia": [
+    {
+      "login": "usera",
+      "type": "clientid",
+      "password": "WVlGsjBiNGJkOWRkN2QyZmYyOWViYjI4MzRiZjQyMDgzNDhkYzJjZmZlOGVjMjUzOGU5NDkwYmYyYjY5N2Q3NjUyMDU=",
+      "created_at": 1653365382492
+    }
+  ],
+  "acl_mnesia": [
+    {
+      "type": "clientid",
+      "type_value": "clientida",
+      "topic": "t/a",
+      "action": "pubsub",
+      "access": "allow",
+      "created_at": 1653365390351
+    }
+  ],
+  "schemas": [],
+  "configs": [],
+  "listeners_state": []
+}

apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json

@@ -0,0 +1,96 @@
+{
+  "version": "4.3",
+  "rules": [],
+  "resources": [],
+  "blacklist": [],
+  "apps": [
+    {
+      "id": "admin",
+      "secret": "public",
+      "name": "Default",
+      "desc": "Application user",
+      "status": true,
+      "expired": "undefined"
+    }
+  ],
+  "users": [
+    {
+      "username": "admin",
+      "password": "02BzoSYaTxkscy2MDtU92EbX7b4=",
+      "tags": "administrator"
+    }
+  ],
+  "auth_mnesia": [
+    {
+      "login": "usera",
+      "type": "clientid",
+      "password": "joYZ7GY2NzcxNTQwMzY4OTRjNWUyMTdmNDlkNmE5Yzc5MDJiNjA5OWRkMWRkZjc5N2E5OGI4YWFlYTdlOWNiMjU5OWE=",
+      "created_at": 1653360665243
+    }
+  ],
+  "acl_mnesia": [
+    {
+      "type": "clientid",
+      "type_value": "clientida",
+      "topic": "t/a",
+      "action": "pub",
+      "access": "allow",
+      "created_at": 1653360687955
+    },
+    {
+      "type": "clientid",
+      "type_value": "clientida",
+      "topic": "t/a",
+      "action": "sub",
+      "access": "allow",
+      "created_at": 1653360687955
+    }
+  ],
+  "modules": [
+    {
+      "id": "module:fcda7532",
+      "type": "mnesia_authentication",
+      "config": {
+        "password_hash": "sha256"
+      },
+      "enabled": true,
+      "created_at": 1653360656060,
+      "description": ""
+    },
+    {
+      "id": "module:db849123",
+      "type": "retainer",
+      "config": {
+        "storage_type": "ram",
+        "max_retained_messages": 0,
+        "max_payload_size": "1MB",
+        "expiry_interval": 0
+      },
+      "enabled": true,
+      "created_at": 1653360591111,
+      "description": ""
+    },
+    {
+      "id": "module:55987aaa",
+      "type": "presence",
+      "config": {
+        "qos": 0
+      },
+      "enabled": true,
+      "created_at": 1653360591111,
+      "description": ""
+    },
+    {
+      "id": "module:78cae4f9",
+      "type": "recon",
+      "config": {},
+      "enabled": true,
+      "created_at": 1653360591111,
+      "description": ""
+    }
+  ],
+  "schemas": [],
+  "configs": [],
+  "listeners_state": [],
+  "date": "2022-05-24 10:51:39"
+}