Merge remote-tracking branch 'ce/main-v4.3' into merge-main-v4.3-into-v4.4

CHANGES-4.3.md

@@ -38,6 +38,7 @@ File format:
   startup. [#7876]
   [ekka-158](https://github.com/emqx/ekka/pull/158)
 * Add regular expression check ^[0-9A-Za-z_\-]+$ for node name [#7979]
+* Fix `node_dump` variable sourcing. [#8026]
 
 ## v4.3.14
 

apps/emqx_auth_pgsql/rebar.config

@@ -1,5 +1,5 @@
 {deps,
- [{epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}}
+ [
  ]}.
 
 {erl_opts, [warn_unused_vars,

apps/emqx_management/src/emqx_mgmt_data_backup.erl

@@ -869,26 +869,38 @@ check_json(MaybeJson) ->
     end.
 
 do_import_data(Data, Version) ->
-    do_import_extra_data(Data, Version),
     import_resources_and_rules(maps:get(<<"resources">>, Data, []), maps:get(<<"rules">>, Data, []), Version),
     import_blacklist(maps:get(<<"blacklist">>, Data, [])),
     import_applications(maps:get(<<"apps">>, Data, [])),
     import_users(maps:get(<<"users">>, Data, [])),
+    %% Import modules first to ensure the data of auth_mnesia module can be imported.
+    %% XXX: In opensource version, can't import if the emqx_auth_mnesia plug-in is not started??
+    do_import_enterprise_modules(Data, Version),
     import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])),
     import_auth_username(maps:get(<<"auth_username">>, Data, [])),
     import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, [])),
-    import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])).
+    import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])),
+    %% always do extra import at last, to make sure resources are initiated before
+    %% creating the schemas
+    do_import_extra_data(Data, Version).
 
 -ifdef(EMQX_ENTERPRISE).
 do_import_extra_data(Data, _Version) ->
     _ = import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])),
-    _ = import_modules(maps:get(<<"modules">>, Data, [])),
     _ = import_schemas(maps:get(<<"schemas">>, Data, [])),
     ok.
 -else.
 do_import_extra_data(_Data, _Version) -> ok.
 -endif.
 
+-ifdef(EMQX_ENTERPRISE).
+do_import_enterprise_modules(Data, _Version) ->
+    _ = import_modules(maps:get(<<"modules">>, Data, [])),
+    ok.
+-else.
+do_import_enterprise_modules(_Data, _Version) -> ok.
+-endif.
+
 covert_empty_headers([]) -> #{};
 covert_empty_headers(Other) -> Other.
 

apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl

@@ -0,0 +1,73 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_auth_mnesia_data_export_import_SUITE).
+
+-compile([export_all, nowarn_export_all]).
+
+-ifdef(EMQX_ENTERPRISE).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("emqx_modules/include/emqx_modules.hrl").
+
+%%--------------------------------------------------------------------
+%% Setups
+%%--------------------------------------------------------------------
+all() ->
+    emqx_ct:all(?MODULE).
+
+init_per_suite(Cfg) ->
+    _ = application:load(emqx_modules_spec),
+    emqx_ct_helpers:start_apps([emqx_rule_engine, emqx_modules,
+                                emqx_management, emqx_dashboard]),
+    Cfg.
+
+end_per_suite(Cfg) ->
+    emqx_ct_helpers:stop_apps([emqx_dashboard, emqx_management,
+                               emqx_modules, emqx_rule_engine]),
+    Cfg.
+
+get_data_path() ->
+    emqx_ct_helpers:deps_path(emqx_management, "test/emqx_auth_mnesia_data_export_import_SUITE_data/").
+
+import(FilePath, _Version) ->
+    ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>),
+    [_] = lists:filter(
+            fun(#module{type = mnesia_authentication}) -> true;
+               (_) -> false
+            end, emqx_modules_registry:get_modules()),
+    ?assertNotEqual(0, ets:info(emqx_user, size)),
+    ?assertNotEqual(0, ets:info(emqx_acl, size)).
+
+%%--------------------------------------------------------------------
+%% Cases
+%%--------------------------------------------------------------------
+
+t_importee427(_) ->
+    import("ee427.json", ee427),
+    {ok, _} = emqx_mgmt_data_backup:export(),
+    remove_all_users_and_acl().
+
+t_importee430(_) ->
+    import("ee435.json", ee435),
+    {ok, _} = emqx_mgmt_data_backup:export(),
+    remove_all_users_and_acl().
+
+remove_all_users_and_acl() ->
+    mnesia:delete_table(emqx_user),
+    mnesia:delete_table(emqx_acl).
+
+-endif.

apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json

@@ -0,0 +1,88 @@
+{
+  "version": "4.2",
+  "date": "2022-05-24 12:09:56",
+  "modules": [
+    {
+      "id": "module:842a5c57",
+      "type": "mnesia_authentication",
+      "config": {
+        "password_hash": "sha256"
+      },
+      "enabled": true,
+      "created_at": 1653365372585,
+      "description": ""
+    },
+    {
+      "id": "module:e3d38e5a",
+      "type": "retainer",
+      "config": {
+        "storage_type": "ram",
+        "max_retained_messages": 0,
+        "max_payload_size": "1MB",
+        "expiry_interval": 0
+      },
+      "enabled": true,
+      "created_at": 1652436434273,
+      "description": ""
+    },
+    {
+      "id": "module:4f911150",
+      "type": "presence",
+      "config": {
+        "qos": 0
+      },
+      "enabled": true,
+      "created_at": 1652436434273,
+      "description": ""
+    },
+    {
+      "id": "module:db11d08f",
+      "type": "recon",
+      "config": {},
+      "enabled": true,
+      "created_at": 1652436434273,
+      "description": ""
+    }
+  ],
+  "rules": [],
+  "resources": [],
+  "blacklist": [],
+  "apps": [
+    {
+      "id": "admin",
+      "secret": "public",
+      "name": "Default",
+      "desc": "Application user",
+      "status": true,
+      "expired": "undefined"
+    }
+  ],
+  "users": [
+    {
+      "username": "admin",
+      "password": "oFu0ZiOAYJmB1DyOzoLwEervBK0=",
+      "tags": "administrator"
+    }
+  ],
+  "auth_mnesia": [
+    {
+      "login": "usera",
+      "type": "clientid",
+      "password": "WVlGsjBiNGJkOWRkN2QyZmYyOWViYjI4MzRiZjQyMDgzNDhkYzJjZmZlOGVjMjUzOGU5NDkwYmYyYjY5N2Q3NjUyMDU=",
+      "created_at": 1653365382492
+    }
+  ],
+  "acl_mnesia": [
+    {
+      "type": "clientid",
+      "type_value": "clientida",
+      "topic": "t/a",
+      "action": "pubsub",
+      "access": "allow",
+      "created_at": 1653365390351
+    }
+  ],
+  "schemas": [],
+  "configs": [],
+  "listeners_state": []
+}

apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json

@@ -0,0 +1,96 @@
+{
+  "version": "4.3",
+  "rules": [],
+  "resources": [],
+  "blacklist": [],
+  "apps": [
+    {
+      "id": "admin",
+      "secret": "public",
+      "name": "Default",
+      "desc": "Application user",
+      "status": true,
+      "expired": "undefined"
+    }
+  ],
+  "users": [
+    {
+      "username": "admin",
+      "password": "02BzoSYaTxkscy2MDtU92EbX7b4=",
+      "tags": "administrator"
+    }
+  ],
+  "auth_mnesia": [
+    {
+      "login": "usera",
+      "type": "clientid",
+      "password": "joYZ7GY2NzcxNTQwMzY4OTRjNWUyMTdmNDlkNmE5Yzc5MDJiNjA5OWRkMWRkZjc5N2E5OGI4YWFlYTdlOWNiMjU5OWE=",
+      "created_at": 1653360665243
+    }
+  ],
+  "acl_mnesia": [
+    {
+      "type": "clientid",
+      "type_value": "clientida",
+      "topic": "t/a",
+      "action": "pub",
+      "access": "allow",
+      "created_at": 1653360687955
+    },
+    {
+      "type": "clientid",
+      "type_value": "clientida",
+      "topic": "t/a",
+      "action": "sub",
+      "access": "allow",
+      "created_at": 1653360687955
+    }
+  ],
+  "modules": [
+    {
+      "id": "module:fcda7532",
+      "type": "mnesia_authentication",
+      "config": {
+        "password_hash": "sha256"
+      },
+      "enabled": true,
+      "created_at": 1653360656060,
+      "description": ""
+    },
+    {
+      "id": "module:db849123",
+      "type": "retainer",
+      "config": {
+        "storage_type": "ram",
+        "max_retained_messages": 0,
+        "max_payload_size": "1MB",
+        "expiry_interval": 0
+      },
+      "enabled": true,
+      "created_at": 1653360591111,
+      "description": ""
+    },
+    {
+      "id": "module:55987aaa",
+      "type": "presence",
+      "config": {
+        "qos": 0
+      },
+      "enabled": true,
+      "created_at": 1653360591111,
+      "description": ""
+    },
+    {
+      "id": "module:78cae4f9",
+      "type": "recon",
+      "config": {},
+      "enabled": true,
+      "created_at": 1653360591111,
+      "description": ""
+    }
+  ],
+  "schemas": [],
+  "configs": [],
+  "listeners_state": [],
+  "date": "2022-05-24 10:51:39"
+}

bin/emqx

@@ -343,14 +343,12 @@ generate_config() {
     ## changing the config 'log.rotation.size'
     rm -rf "${RUNNER_LOG_DIR}"/*.siz
 
-    EMQX_LICENSE_CONF_OPTION=""
-    if [ "${EMQX_LICENSE_CONF:-}" != "" ]; then
-        EMQX_LICENSE_CONF_OPTION="-i ${EMQX_LICENSE_CONF}"
-    fi
-
     set +e
-    # shellcheck disable=SC2086
+    if [ "${EMQX_LICENSE_CONF:-}" = "" ]; then
-    CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema $EMQX_LICENSE_CONF_OPTION -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)"
+        CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)"
+    else
+        CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema -i "${EMQX_LICENSE_CONF}" -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)"
+    fi
     # shellcheck disable=SC2181
     RESULT=$?
     set -e

bin/node_dump

@@ -1,13 +1,13 @@
 #!/bin/sh
 set -eu
 
-ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)"
+RUNNER_ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)"
-echo "Running node dump in ${ROOT_DIR}"
+echo "Running node dump in ${RUNNER_ROOT_DIR}"
 
 # shellcheck disable=SC1090
-. "$ROOT_DIR"/releases/emqx_vars
+. "$RUNNER_ROOT_DIR"/releases/emqx_vars
 
-cd "${ROOT_DIR}"
+cd "${RUNNER_ROOT_DIR}"
 
 DUMP="$RUNNER_LOG_DIR/node_dump_$(date +"%Y%m%d_%H%M%S").tar.gz"
 CONF_DUMP="$RUNNER_LOG_DIR/conf.dump"

rebar.config

@@ -60,6 +60,7 @@
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.15.0"}}}
     , {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.2.1"}}}
     , {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.13"}}}
+    , {epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}}
     ]}.
 
 {xref_ignores,