diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 0a61ca487..3086e7934 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -38,6 +38,7 @@ File format: startup. [#7876] [ekka-158](https://github.com/emqx/ekka/pull/158) * Add regular expression check ^[0-9A-Za-z_\-]+$ for node name [#7979] +* Fix `node_dump` variable sourcing. [#8026] ## v4.3.14 diff --git a/apps/emqx_auth_pgsql/rebar.config b/apps/emqx_auth_pgsql/rebar.config index 7a6aaf411..c351ed13e 100644 --- a/apps/emqx_auth_pgsql/rebar.config +++ b/apps/emqx_auth_pgsql/rebar.config @@ -1,5 +1,5 @@ {deps, - [{epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}} + [ ]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 47b590530..7b31dd8be 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -869,26 +869,38 @@ check_json(MaybeJson) -> end. do_import_data(Data, Version) -> - do_import_extra_data(Data, Version), import_resources_and_rules(maps:get(<<"resources">>, Data, []), maps:get(<<"rules">>, Data, []), Version), import_blacklist(maps:get(<<"blacklist">>, Data, [])), import_applications(maps:get(<<"apps">>, Data, [])), import_users(maps:get(<<"users">>, Data, [])), + %% Import modules first to ensure the data of auth_mnesia module can be imported. + %% XXX: In opensource version, can't import if the emqx_auth_mnesia plug-in is not started?? + do_import_enterprise_modules(Data, Version), import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), import_auth_username(maps:get(<<"auth_username">>, Data, [])), import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, [])), - import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])). + import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])), + %% always do extra import at last, to make sure resources are initiated before + %% creating the schemas + do_import_extra_data(Data, Version). -ifdef(EMQX_ENTERPRISE). do_import_extra_data(Data, _Version) -> _ = import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])), - _ = import_modules(maps:get(<<"modules">>, Data, [])), _ = import_schemas(maps:get(<<"schemas">>, Data, [])), ok. -else. do_import_extra_data(_Data, _Version) -> ok. -endif. +-ifdef(EMQX_ENTERPRISE). +do_import_enterprise_modules(Data, _Version) -> + _ = import_modules(maps:get(<<"modules">>, Data, [])), + ok. +-else. +do_import_enterprise_modules(_Data, _Version) -> ok. +-endif. + covert_empty_headers([]) -> #{}; covert_empty_headers(Other) -> Other. diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl new file mode 100644 index 000000000..021cf735a --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl @@ -0,0 +1,73 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_auth_mnesia_data_export_import_SUITE). + +-compile([export_all, nowarn_export_all]). + +-ifdef(EMQX_ENTERPRISE). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx_modules/include/emqx_modules.hrl"). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Cfg) -> + _ = application:load(emqx_modules_spec), + emqx_ct_helpers:start_apps([emqx_rule_engine, emqx_modules, + emqx_management, emqx_dashboard]), + Cfg. + +end_per_suite(Cfg) -> + emqx_ct_helpers:stop_apps([emqx_dashboard, emqx_management, + emqx_modules, emqx_rule_engine]), + Cfg. + +get_data_path() -> + emqx_ct_helpers:deps_path(emqx_management, "test/emqx_auth_mnesia_data_export_import_SUITE_data/"). + +import(FilePath, _Version) -> + ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>), + [_] = lists:filter( + fun(#module{type = mnesia_authentication}) -> true; + (_) -> false + end, emqx_modules_registry:get_modules()), + ?assertNotEqual(0, ets:info(emqx_user, size)), + ?assertNotEqual(0, ets:info(emqx_acl, size)). + +%%-------------------------------------------------------------------- +%% Cases +%%-------------------------------------------------------------------- + +t_importee427(_) -> + import("ee427.json", ee427), + {ok, _} = emqx_mgmt_data_backup:export(), + remove_all_users_and_acl(). + +t_importee430(_) -> + import("ee435.json", ee435), + {ok, _} = emqx_mgmt_data_backup:export(), + remove_all_users_and_acl(). + +remove_all_users_and_acl() -> + mnesia:delete_table(emqx_user), + mnesia:delete_table(emqx_acl). + +-endif. diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json new file mode 100644 index 000000000..43d58538a --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json @@ -0,0 +1,88 @@ +{ + "version": "4.2", + "date": "2022-05-24 12:09:56", + "modules": [ + { + "id": "module:842a5c57", + "type": "mnesia_authentication", + "config": { + "password_hash": "sha256" + }, + "enabled": true, + "created_at": 1653365372585, + "description": "" + }, + { + "id": "module:e3d38e5a", + "type": "retainer", + "config": { + "storage_type": "ram", + "max_retained_messages": 0, + "max_payload_size": "1MB", + "expiry_interval": 0 + }, + "enabled": true, + "created_at": 1652436434273, + "description": "" + }, + { + "id": "module:4f911150", + "type": "presence", + "config": { + "qos": 0 + }, + "enabled": true, + "created_at": 1652436434273, + "description": "" + }, + { + "id": "module:db11d08f", + "type": "recon", + "config": {}, + "enabled": true, + "created_at": 1652436434273, + "description": "" + } + ], + "rules": [], + "resources": [], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "oFu0ZiOAYJmB1DyOzoLwEervBK0=", + "tags": "administrator" + } + ], + "auth_mnesia": [ + { + "login": "usera", + "type": "clientid", + "password": "WVlGsjBiNGJkOWRkN2QyZmYyOWViYjI4MzRiZjQyMDgzNDhkYzJjZmZlOGVjMjUzOGU5NDkwYmYyYjY5N2Q3NjUyMDU=", + "created_at": 1653365382492 + } + ], + "acl_mnesia": [ + { + "type": "clientid", + "type_value": "clientida", + "topic": "t/a", + "action": "pubsub", + "access": "allow", + "created_at": 1653365390351 + } + ], + "schemas": [], + "configs": [], + "listeners_state": [] +} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json new file mode 100644 index 000000000..e6d9cbb3b --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json @@ -0,0 +1,96 @@ +{ + "version": "4.3", + "rules": [], + "resources": [], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "02BzoSYaTxkscy2MDtU92EbX7b4=", + "tags": "administrator" + } + ], + "auth_mnesia": [ + { + "login": "usera", + "type": "clientid", + "password": "joYZ7GY2NzcxNTQwMzY4OTRjNWUyMTdmNDlkNmE5Yzc5MDJiNjA5OWRkMWRkZjc5N2E5OGI4YWFlYTdlOWNiMjU5OWE=", + "created_at": 1653360665243 + } + ], + "acl_mnesia": [ + { + "type": "clientid", + "type_value": "clientida", + "topic": "t/a", + "action": "pub", + "access": "allow", + "created_at": 1653360687955 + }, + { + "type": "clientid", + "type_value": "clientida", + "topic": "t/a", + "action": "sub", + "access": "allow", + "created_at": 1653360687955 + } + ], + "modules": [ + { + "id": "module:fcda7532", + "type": "mnesia_authentication", + "config": { + "password_hash": "sha256" + }, + "enabled": true, + "created_at": 1653360656060, + "description": "" + }, + { + "id": "module:db849123", + "type": "retainer", + "config": { + "storage_type": "ram", + "max_retained_messages": 0, + "max_payload_size": "1MB", + "expiry_interval": 0 + }, + "enabled": true, + "created_at": 1653360591111, + "description": "" + }, + { + "id": "module:55987aaa", + "type": "presence", + "config": { + "qos": 0 + }, + "enabled": true, + "created_at": 1653360591111, + "description": "" + }, + { + "id": "module:78cae4f9", + "type": "recon", + "config": {}, + "enabled": true, + "created_at": 1653360591111, + "description": "" + } + ], + "schemas": [], + "configs": [], + "listeners_state": [], + "date": "2022-05-24 10:51:39" +} diff --git a/bin/emqx b/bin/emqx index 4232d8162..2394484b1 100755 --- a/bin/emqx +++ b/bin/emqx @@ -343,14 +343,12 @@ generate_config() { ## changing the config 'log.rotation.size' rm -rf "${RUNNER_LOG_DIR}"/*.siz - EMQX_LICENSE_CONF_OPTION="" - if [ "${EMQX_LICENSE_CONF:-}" != "" ]; then - EMQX_LICENSE_CONF_OPTION="-i ${EMQX_LICENSE_CONF}" - fi - set +e - # shellcheck disable=SC2086 - CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema $EMQX_LICENSE_CONF_OPTION -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" + if [ "${EMQX_LICENSE_CONF:-}" = "" ]; then + CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" + else + CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema -i "${EMQX_LICENSE_CONF}" -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" + fi # shellcheck disable=SC2181 RESULT=$? set -e diff --git a/bin/node_dump b/bin/node_dump index 83b41332b..79b469112 100755 --- a/bin/node_dump +++ b/bin/node_dump @@ -1,13 +1,13 @@ #!/bin/sh set -eu -ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" -echo "Running node dump in ${ROOT_DIR}" +RUNNER_ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" +echo "Running node dump in ${RUNNER_ROOT_DIR}" # shellcheck disable=SC1090 -. "$ROOT_DIR"/releases/emqx_vars +. "$RUNNER_ROOT_DIR"/releases/emqx_vars -cd "${ROOT_DIR}" +cd "${RUNNER_ROOT_DIR}" DUMP="$RUNNER_LOG_DIR/node_dump_$(date +"%Y%m%d_%H%M%S").tar.gz" CONF_DUMP="$RUNNER_LOG_DIR/conf.dump" diff --git a/rebar.config b/rebar.config index 7835a0e2f..9862aa424 100644 --- a/rebar.config +++ b/rebar.config @@ -60,6 +60,7 @@ , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.15.0"}}} , {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.2.1"}}} , {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.13"}}} + , {epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}} ]}. {xref_ignores,