From 24f0592e94f230cb4eb3f523cc54b98644140162 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 23 May 2022 16:18:56 -0300 Subject: [PATCH 1/8] fix(node_dump): define `RUNNER_ROOT_DIR` before sourcing vars Port of https://github.com/emqx/emqx/pull/7733 `emqx_vars` requires `RUNNER_ROOT_DIR` to be defined before being sourced. --- CHANGES-4.3.md | 1 + bin/node_dump | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index e387849e4..da2df758d 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -38,6 +38,7 @@ File format: startup. [#7876] [ekka-158](https://github.com/emqx/ekka/pull/158) * Add regular expression check ^[0-9A-Za-z_\-]+$ for node name [#7979] +* Fix `node_dump` variable sourcing. [#8026] ## v4.3.14 diff --git a/bin/node_dump b/bin/node_dump index 83b41332b..79b469112 100755 --- a/bin/node_dump +++ b/bin/node_dump @@ -1,13 +1,13 @@ #!/bin/sh set -eu -ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" -echo "Running node dump in ${ROOT_DIR}" +RUNNER_ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" +echo "Running node dump in ${RUNNER_ROOT_DIR}" # shellcheck disable=SC1090 -. "$ROOT_DIR"/releases/emqx_vars +. "$RUNNER_ROOT_DIR"/releases/emqx_vars -cd "${ROOT_DIR}" +cd "${RUNNER_ROOT_DIR}" DUMP="$RUNNER_LOG_DIR/node_dump_$(date +"%Y%m%d_%H%M%S").tar.gz" CONF_DUMP="$RUNNER_LOG_DIR/conf.dump" From 6e7cbc1b9b1e235efff17164230b8d924ec84175 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 24 May 2022 09:33:17 +0800 Subject: [PATCH 2/8] chore: move epgsql deps to the top-level rebar.config --- apps/emqx_auth_pgsql/rebar.config | 2 +- rebar.config | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_auth_pgsql/rebar.config b/apps/emqx_auth_pgsql/rebar.config index 7a6aaf411..c351ed13e 100644 --- a/apps/emqx_auth_pgsql/rebar.config +++ b/apps/emqx_auth_pgsql/rebar.config @@ -1,5 +1,5 @@ {deps, - [{epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}} + [ ]}. {erl_opts, [warn_unused_vars, diff --git a/rebar.config b/rebar.config index 8ed5933fc..6688e34a1 100644 --- a/rebar.config +++ b/rebar.config @@ -60,6 +60,7 @@ , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.15.0"}}} , {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.2.1"}}} , {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.13"}}} + , {epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}} ]}. {xref_ignores, From 8fe933e88536c43104bede3188ee480e268f0fa7 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 24 May 2022 11:51:10 +0800 Subject: [PATCH 3/8] fix: ensure auth_mnesia started first --- .../src/emqx_mgmt_data_backup.erl | 12 +++- ...x_auth_mnesia_data_export_import_SUITE.erl | 69 +++++++++++++++++++ .../ee435.json | 1 + 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl create mode 100644 apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 06dc0365a..bc9639e6a 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -662,6 +662,9 @@ do_import_data(Data, Version) -> import_blacklist(maps:get(<<"blacklist">>, Data, [])), import_applications(maps:get(<<"apps">>, Data, [])), import_users(maps:get(<<"users">>, Data, [])), + %% Import modules first to ensure the data of auth_mnesia module can be imported. + %% XXX: In opensource version, can't import if the emqx_auth_mnesia plug-in is not started?? + do_import_enterprise_modules(Data, Version), import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), import_auth_username(maps:get(<<"auth_username">>, Data, [])), import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, [])), @@ -670,13 +673,20 @@ do_import_data(Data, Version) -> -ifdef(EMQX_ENTERPRISE). do_import_extra_data(Data, _Version) -> _ = import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])), - _ = import_modules(maps:get(<<"modules">>, Data, [])), _ = import_schemas(maps:get(<<"schemas">>, Data, [])), ok. -else. do_import_extra_data(_Data, _Version) -> ok. -endif. +-ifdef(EMQX_ENTERPRISE). +do_import_enterprise_modules(Data, _Version) -> + _ = import_modules(maps:get(<<"modules">>, Data, [])), + ok. +-else. +do_import_enterprise_modules(_Data, _Version) -> ok. +-endif. + covert_empty_headers([]) -> #{}; covert_empty_headers(Other) -> Other. diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl new file mode 100644 index 000000000..abe496434 --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl @@ -0,0 +1,69 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_auth_mnesia_data_export_import_SUITE). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx_modules/include/emqx_modules.hrl"). +-compile([export_all, nowarn_export_all]). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Cfg) -> + _ = application:load(emqx_modules_spec), + emqx_ct_helpers:start_apps([emqx_rule_engine, emqx_modules, + emqx_management, emqx_dashboard]), + Cfg. + +end_per_suite(Cfg) -> + emqx_ct_helpers:stop_apps([emqx_dashboard, emqx_management, + emqx_modules, emqx_rule_engine]), + Cfg. + +get_data_path() -> + emqx_ct_helpers:deps_path(emqx_management, "test/emqx_auth_mnesia_data_export_import_SUITE_data/"). + +import(FilePath, _Version) -> + dbg:tracer(),dbg:p(all,call), + dbg:tpl(emqx_mgmt_data_backup,import_auth_mnesia,x), + dbg:tp(emqx_modules_registry,get_modules,x), + dbg:tp(emqx_module_internal_acl,on_module_create,x), + ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>), + [_] = lists:filter( + fun(#module{type = mnesia_authentication}) -> true; + (_) -> false + end, emqx_modules_registry:get_modules()), + ?assertNotEqual(0, ets:info(emqx_user, size)), + ?assertNotEqual(0, ets:info(emqx_acl, size)). + +%%-------------------------------------------------------------------- +%% Cases +%%-------------------------------------------------------------------- +-ifdef(EMQX_ENTERPRISE). + +t_importee430(_) -> + import("ee435.json", ee435), + {ok, _} = emqx_mgmt_data_backup:export(), + remove_all_users_and_acl(). + +-endif. + +remove_all_users_and_acl() -> + mnesia:delete_table(emqx_user), + mnesia:delete_table(emqx_acl). diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json new file mode 100644 index 000000000..fd2f51ad6 --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json @@ -0,0 +1 @@ +{"version":"4.3","rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"02BzoSYaTxkscy2MDtU92EbX7b4=","tags":"administrator"}],"auth_mnesia":[{"login":"usera","type":"clientid","password":"joYZ7GY2NzcxNTQwMzY4OTRjNWUyMTdmNDlkNmE5Yzc5MDJiNjA5OWRkMWRkZjc5N2E5OGI4YWFlYTdlOWNiMjU5OWE=","created_at":1653360665243}],"acl_mnesia":[{"type":"clientid","type_value":"clientida","topic":"t/a","action":"pub","access":"allow","created_at":1653360687955},{"type":"clientid","type_value":"clientida","topic":"t/a","action":"sub","access":"allow","created_at":1653360687955}],"modules":[{"id":"module:fcda7532","type":"mnesia_authentication","config":{"password_hash":"sha256"},"enabled":true,"created_at":1653360656060,"description":""},{"id":"module:16f3f067","type":"internal_acl","config":{"acl_rule_file":"etc/acl.conf"},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:db849123","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:55987aaa","type":"presence","config":{"qos":0},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:78cae4f9","type":"recon","config":{},"enabled":true,"created_at":1653360591111,"description":""}],"schemas":[],"configs":[],"listeners_state":[],"date":"2022-05-24 10:51:39"} \ No newline at end of file From 1531b34f8af2e81f4ff7235f3d3a13152958c27f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 24 May 2022 12:16:03 +0800 Subject: [PATCH 4/8] test: add cases for importing e427 --- .../test/emqx_auth_mnesia_data_export_import_SUITE.erl | 9 +++++---- .../ee427.json | 1 + .../ee435.json | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl index abe496434..e82e875e9 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl @@ -40,10 +40,6 @@ get_data_path() -> emqx_ct_helpers:deps_path(emqx_management, "test/emqx_auth_mnesia_data_export_import_SUITE_data/"). import(FilePath, _Version) -> - dbg:tracer(),dbg:p(all,call), - dbg:tpl(emqx_mgmt_data_backup,import_auth_mnesia,x), - dbg:tp(emqx_modules_registry,get_modules,x), - dbg:tp(emqx_module_internal_acl,on_module_create,x), ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>), [_] = lists:filter( fun(#module{type = mnesia_authentication}) -> true; @@ -57,6 +53,11 @@ import(FilePath, _Version) -> %%-------------------------------------------------------------------- -ifdef(EMQX_ENTERPRISE). +t_importee427(_) -> + import("ee427.json", ee427), + {ok, _} = emqx_mgmt_data_backup:export(), + remove_all_users_and_acl(). + t_importee430(_) -> import("ee435.json", ee435), {ok, _} = emqx_mgmt_data_backup:export(), diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json new file mode 100644 index 000000000..763bf2c7b --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json @@ -0,0 +1 @@ +{"version":"4.2","date":"2022-05-24 12:09:56","modules":[{"id":"module:842a5c57","type":"mnesia_authentication","config":{"password_hash":"sha256"},"enabled":true,"created_at":1653365372585,"description":""},{"id":"module:e3d38e5a","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1652436434273,"description":""},{"id":"module:4f911150","type":"presence","config":{"qos":0},"enabled":true,"created_at":1652436434273,"description":""},{"id":"module:db11d08f","type":"recon","config":{},"enabled":true,"created_at":1652436434273,"description":""}],"rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"oFu0ZiOAYJmB1DyOzoLwEervBK0=","tags":"administrator"}],"auth_mnesia":[{"login":"usera","type":"clientid","password":"WVlGsjBiNGJkOWRkN2QyZmYyOWViYjI4MzRiZjQyMDgzNDhkYzJjZmZlOGVjMjUzOGU5NDkwYmYyYjY5N2Q3NjUyMDU=","created_at":1653365382492}],"acl_mnesia":[{"type":"clientid","type_value":"clientida","topic":"t/a","action":"pubsub","access":"allow","created_at":1653365390351}],"schemas":[],"configs":[],"listeners_state":[]} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json index fd2f51ad6..b457ff6b8 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json @@ -1 +1 @@ -{"version":"4.3","rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"02BzoSYaTxkscy2MDtU92EbX7b4=","tags":"administrator"}],"auth_mnesia":[{"login":"usera","type":"clientid","password":"joYZ7GY2NzcxNTQwMzY4OTRjNWUyMTdmNDlkNmE5Yzc5MDJiNjA5OWRkMWRkZjc5N2E5OGI4YWFlYTdlOWNiMjU5OWE=","created_at":1653360665243}],"acl_mnesia":[{"type":"clientid","type_value":"clientida","topic":"t/a","action":"pub","access":"allow","created_at":1653360687955},{"type":"clientid","type_value":"clientida","topic":"t/a","action":"sub","access":"allow","created_at":1653360687955}],"modules":[{"id":"module:fcda7532","type":"mnesia_authentication","config":{"password_hash":"sha256"},"enabled":true,"created_at":1653360656060,"description":""},{"id":"module:16f3f067","type":"internal_acl","config":{"acl_rule_file":"etc/acl.conf"},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:db849123","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:55987aaa","type":"presence","config":{"qos":0},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:78cae4f9","type":"recon","config":{},"enabled":true,"created_at":1653360591111,"description":""}],"schemas":[],"configs":[],"listeners_state":[],"date":"2022-05-24 10:51:39"} \ No newline at end of file +{"version":"4.3","rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"02BzoSYaTxkscy2MDtU92EbX7b4=","tags":"administrator"}],"auth_mnesia":[{"login":"usera","type":"clientid","password":"joYZ7GY2NzcxNTQwMzY4OTRjNWUyMTdmNDlkNmE5Yzc5MDJiNjA5OWRkMWRkZjc5N2E5OGI4YWFlYTdlOWNiMjU5OWE=","created_at":1653360665243}],"acl_mnesia":[{"type":"clientid","type_value":"clientida","topic":"t/a","action":"pub","access":"allow","created_at":1653360687955},{"type":"clientid","type_value":"clientida","topic":"t/a","action":"sub","access":"allow","created_at":1653360687955}],"modules":[{"id":"module:fcda7532","type":"mnesia_authentication","config":{"password_hash":"sha256"},"enabled":true,"created_at":1653360656060,"description":""},{"id":"module:db849123","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:55987aaa","type":"presence","config":{"qos":0},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:78cae4f9","type":"recon","config":{},"enabled":true,"created_at":1653360591111,"description":""}],"schemas":[],"configs":[],"listeners_state":[],"date":"2022-05-24 10:51:39"} From 0eef2977471120cedcbec33863a86d8c8787ed62 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 24 May 2022 12:27:10 +0800 Subject: [PATCH 5/8] chore: fix dialyzer error --- .../emqx_auth_mnesia_data_export_import_SUITE.erl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl index e82e875e9..021cf735a 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl @@ -15,9 +15,13 @@ %%-------------------------------------------------------------------- -module(emqx_auth_mnesia_data_export_import_SUITE). + +-compile([export_all, nowarn_export_all]). + +-ifdef(EMQX_ENTERPRISE). + -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx_modules/include/emqx_modules.hrl"). --compile([export_all, nowarn_export_all]). %%-------------------------------------------------------------------- %% Setups @@ -51,7 +55,6 @@ import(FilePath, _Version) -> %%-------------------------------------------------------------------- %% Cases %%-------------------------------------------------------------------- --ifdef(EMQX_ENTERPRISE). t_importee427(_) -> import("ee427.json", ee427), @@ -63,8 +66,8 @@ t_importee430(_) -> {ok, _} = emqx_mgmt_data_backup:export(), remove_all_users_and_acl(). --endif. - remove_all_users_and_acl() -> mnesia:delete_table(emqx_user), mnesia:delete_table(emqx_acl). + +-endif. From 355f859a9b18714d37d7ed30c24bac7369d3ccb6 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 24 May 2022 13:50:27 +0800 Subject: [PATCH 6/8] test: format data-export files --- .../ee427.json | 89 ++++++++++++++++- .../ee435.json | 97 ++++++++++++++++++- 2 files changed, 184 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json index 763bf2c7b..43d58538a 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json @@ -1 +1,88 @@ -{"version":"4.2","date":"2022-05-24 12:09:56","modules":[{"id":"module:842a5c57","type":"mnesia_authentication","config":{"password_hash":"sha256"},"enabled":true,"created_at":1653365372585,"description":""},{"id":"module:e3d38e5a","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1652436434273,"description":""},{"id":"module:4f911150","type":"presence","config":{"qos":0},"enabled":true,"created_at":1652436434273,"description":""},{"id":"module:db11d08f","type":"recon","config":{},"enabled":true,"created_at":1652436434273,"description":""}],"rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"oFu0ZiOAYJmB1DyOzoLwEervBK0=","tags":"administrator"}],"auth_mnesia":[{"login":"usera","type":"clientid","password":"WVlGsjBiNGJkOWRkN2QyZmYyOWViYjI4MzRiZjQyMDgzNDhkYzJjZmZlOGVjMjUzOGU5NDkwYmYyYjY5N2Q3NjUyMDU=","created_at":1653365382492}],"acl_mnesia":[{"type":"clientid","type_value":"clientida","topic":"t/a","action":"pubsub","access":"allow","created_at":1653365390351}],"schemas":[],"configs":[],"listeners_state":[]} +{ + "version": "4.2", + "date": "2022-05-24 12:09:56", + "modules": [ + { + "id": "module:842a5c57", + "type": "mnesia_authentication", + "config": { + "password_hash": "sha256" + }, + "enabled": true, + "created_at": 1653365372585, + "description": "" + }, + { + "id": "module:e3d38e5a", + "type": "retainer", + "config": { + "storage_type": "ram", + "max_retained_messages": 0, + "max_payload_size": "1MB", + "expiry_interval": 0 + }, + "enabled": true, + "created_at": 1652436434273, + "description": "" + }, + { + "id": "module:4f911150", + "type": "presence", + "config": { + "qos": 0 + }, + "enabled": true, + "created_at": 1652436434273, + "description": "" + }, + { + "id": "module:db11d08f", + "type": "recon", + "config": {}, + "enabled": true, + "created_at": 1652436434273, + "description": "" + } + ], + "rules": [], + "resources": [], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "oFu0ZiOAYJmB1DyOzoLwEervBK0=", + "tags": "administrator" + } + ], + "auth_mnesia": [ + { + "login": "usera", + "type": "clientid", + "password": "WVlGsjBiNGJkOWRkN2QyZmYyOWViYjI4MzRiZjQyMDgzNDhkYzJjZmZlOGVjMjUzOGU5NDkwYmYyYjY5N2Q3NjUyMDU=", + "created_at": 1653365382492 + } + ], + "acl_mnesia": [ + { + "type": "clientid", + "type_value": "clientida", + "topic": "t/a", + "action": "pubsub", + "access": "allow", + "created_at": 1653365390351 + } + ], + "schemas": [], + "configs": [], + "listeners_state": [] +} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json index b457ff6b8..e6d9cbb3b 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json @@ -1 +1,96 @@ -{"version":"4.3","rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"02BzoSYaTxkscy2MDtU92EbX7b4=","tags":"administrator"}],"auth_mnesia":[{"login":"usera","type":"clientid","password":"joYZ7GY2NzcxNTQwMzY4OTRjNWUyMTdmNDlkNmE5Yzc5MDJiNjA5OWRkMWRkZjc5N2E5OGI4YWFlYTdlOWNiMjU5OWE=","created_at":1653360665243}],"acl_mnesia":[{"type":"clientid","type_value":"clientida","topic":"t/a","action":"pub","access":"allow","created_at":1653360687955},{"type":"clientid","type_value":"clientida","topic":"t/a","action":"sub","access":"allow","created_at":1653360687955}],"modules":[{"id":"module:fcda7532","type":"mnesia_authentication","config":{"password_hash":"sha256"},"enabled":true,"created_at":1653360656060,"description":""},{"id":"module:db849123","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:55987aaa","type":"presence","config":{"qos":0},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:78cae4f9","type":"recon","config":{},"enabled":true,"created_at":1653360591111,"description":""}],"schemas":[],"configs":[],"listeners_state":[],"date":"2022-05-24 10:51:39"} +{ + "version": "4.3", + "rules": [], + "resources": [], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "02BzoSYaTxkscy2MDtU92EbX7b4=", + "tags": "administrator" + } + ], + "auth_mnesia": [ + { + "login": "usera", + "type": "clientid", + "password": "joYZ7GY2NzcxNTQwMzY4OTRjNWUyMTdmNDlkNmE5Yzc5MDJiNjA5OWRkMWRkZjc5N2E5OGI4YWFlYTdlOWNiMjU5OWE=", + "created_at": 1653360665243 + } + ], + "acl_mnesia": [ + { + "type": "clientid", + "type_value": "clientida", + "topic": "t/a", + "action": "pub", + "access": "allow", + "created_at": 1653360687955 + }, + { + "type": "clientid", + "type_value": "clientida", + "topic": "t/a", + "action": "sub", + "access": "allow", + "created_at": 1653360687955 + } + ], + "modules": [ + { + "id": "module:fcda7532", + "type": "mnesia_authentication", + "config": { + "password_hash": "sha256" + }, + "enabled": true, + "created_at": 1653360656060, + "description": "" + }, + { + "id": "module:db849123", + "type": "retainer", + "config": { + "storage_type": "ram", + "max_retained_messages": 0, + "max_payload_size": "1MB", + "expiry_interval": 0 + }, + "enabled": true, + "created_at": 1653360591111, + "description": "" + }, + { + "id": "module:55987aaa", + "type": "presence", + "config": { + "qos": 0 + }, + "enabled": true, + "created_at": 1653360591111, + "description": "" + }, + { + "id": "module:78cae4f9", + "type": "recon", + "config": {}, + "enabled": true, + "created_at": 1653360591111, + "description": "" + } + ], + "schemas": [], + "configs": [], + "listeners_state": [], + "date": "2022-05-24 10:51:39" +} From d9252dc6727908ccdf4c0c5192ab2e31c4be4663 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sat, 16 Apr 2022 01:17:27 +0800 Subject: [PATCH 7/8] fix(schema): init resources before restoring schema registry --- apps/emqx_management/src/emqx_mgmt_data_backup.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index bc9639e6a..cb94b7cf9 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -657,7 +657,6 @@ import(Filename, OverridesJson) -> -endif. do_import_data(Data, Version) -> - do_import_extra_data(Data, Version), import_resources_and_rules(maps:get(<<"resources">>, Data, []), maps:get(<<"rules">>, Data, []), Version), import_blacklist(maps:get(<<"blacklist">>, Data, [])), import_applications(maps:get(<<"apps">>, Data, [])), @@ -668,7 +667,10 @@ do_import_data(Data, Version) -> import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), import_auth_username(maps:get(<<"auth_username">>, Data, [])), import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, [])), - import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])). + import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])), + %% always do extra import at last, to make sure resources are initiated before + %% creating the schemas + do_import_extra_data(Data, Version). -ifdef(EMQX_ENTERPRISE). do_import_extra_data(Data, _Version) -> From c741b9dfbe1e77876c6d2512890baa5b540bb70d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 24 May 2022 09:45:58 +0200 Subject: [PATCH 8/8] fix(bin/emqx): fix license option to allow space in path --- bin/emqx | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/bin/emqx b/bin/emqx index 756d2f705..7eed9ab3c 100755 --- a/bin/emqx +++ b/bin/emqx @@ -338,14 +338,12 @@ generate_config() { ## changing the config 'log.rotation.size' rm -rf "${RUNNER_LOG_DIR}"/*.siz - EMQX_LICENSE_CONF_OPTION="" - if [ "${EMQX_LICENSE_CONF:-}" != "" ]; then - EMQX_LICENSE_CONF_OPTION="-i ${EMQX_LICENSE_CONF}" - fi - set +e - # shellcheck disable=SC2086 - CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema $EMQX_LICENSE_CONF_OPTION -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" + if [ "${EMQX_LICENSE_CONF:-}" = "" ]; then + CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" + else + CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema -i "${EMQX_LICENSE_CONF}" -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" + fi # shellcheck disable=SC2181 RESULT=$? set -e