test(mgmt): add data import tests (#4481)

apps/emqx_bridge_mqtt/test/emqx_bridge_worker_SUITE.erl

@@ -60,12 +60,12 @@ init_per_suite(Config) ->
         _ -> ok
     end,
     ok = application:set_env(gen_rpc, tcp_client_num, 1),
-    emqx_ct_helpers:start_apps([emqx_bridge_mqtt]),
+    emqx_ct_helpers:start_apps([emqx_modules, emqx_bridge_mqtt]),
     emqx_logger:set_log_level(error),
     [{log_level, error} | Config].
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([emqx_bridge_mqtt]).
+    emqx_ct_helpers:stop_apps([emqx_bridge_mqtt, emqx_modules]).
 
 init_per_testcase(_TestCase, Config) ->
     ok = snabbkaffe:start_trace(),

apps/emqx_management/src/emqx_mgmt_data_backup.erl

@@ -240,6 +240,7 @@ import_resources_and_rules(Resources, Rules, _FromVersion) ->
     import_resources(Resources),
     import_rules(Rules).
 -else.
+
 import_resources_and_rules(Resources, Rules, FromVersion)
   when FromVersion =:= "4.0" orelse
        FromVersion =:= "4.1" orelse
@@ -249,7 +250,6 @@ import_resources_and_rules(Resources, Rules, FromVersion)
                       NActions = apply_new_config(Actions, Configs),
                       import_rule(Rule#{<<"actions">> := NActions})
                   end, Rules);
-
 import_resources_and_rules(Resources, Rules, _FromVersion) ->
     import_resources(Resources),
     import_rules(Rules).
@@ -264,42 +264,39 @@ compatible_version(#{<<"id">> := ID,
                                        <<"pool_size">> := PoolSize,
                                        <<"request_timeout">> := RequestTimeout,
                                        <<"url">> := URL}} = Resource, Acc) ->
-                    CovertFun = fun(Int) ->
+    CovertFun = fun(Int) ->
-                        list_to_binary(integer_to_list(Int) ++ "s")
+        list_to_binary(integer_to_list(Int) ++ "s")
-                    end,
+    end,
-                    Cfg = make_new_config(#{<<"pool_size">> => PoolSize,
+    Cfg = make_new_config(#{<<"pool_size">> => PoolSize,
-                                            <<"connect_timeout">> => CovertFun(ConnectTimeout),
+                            <<"connect_timeout">> => CovertFun(ConnectTimeout),
-                                            <<"request_timeout">> => CovertFun(RequestTimeout),
+                            <<"request_timeout">> => CovertFun(RequestTimeout),
-                                            <<"url">> => URL}),
+                            <<"url">> => URL}),
-                    {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}),
+    {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}),
-                    NHeaders = maps:put(<<"content-type">>, ContentType, covert_empty_headers(Headers)),
+    NHeaders = maps:put(<<"content-type">>, ContentType, covert_empty_headers(Headers)),
-                    [{ID, #{headers => NHeaders, method => Method}} | Acc];
+    [{ID, #{headers => NHeaders, method => Method}} | Acc];
 % 4.2.0
 compatible_version(#{<<"id">> := ID,
-                     <<"type">> := <<"web_hook">>,
+                    <<"type">> := <<"web_hook">>,
-                     <<"config">> := #{<<"headers">> := Headers,
+                    <<"config">> := #{<<"headers">> := Headers,
-                                       <<"method">> := Method,%% 4.2.0 Different here
+                                    <<"method">> := Method,%% 4.2.0 Different here
-                                       <<"url">> := URL}} = Resource, Acc) ->
+                                    <<"url">> := URL}} = Resource, Acc) ->
-                    Cfg = make_new_config(#{<<"url">> => URL}),
+    Cfg = make_new_config(#{<<"url">> => URL}),
-                    {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}),
+    {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}),
-                    NHeaders = maps:put(<<"content-type">>, <<"application/json">> , covert_empty_headers(Headers)),
+    NHeaders = maps:put(<<"content-type">>, <<"application/json">> , covert_empty_headers(Headers)),
-                    [{ID, #{headers => NHeaders, method => Method}} | Acc];
+    [{ID, #{headers => NHeaders, method => Method}} | Acc];
 
 %% bridge mqtt
 %% 4.2.0 - 4.2.5 bridge_mqtt, ssl enabled from on/off to true/false
 compatible_version(#{<<"type">> := <<"bridge_mqtt">>,
                      <<"id">> := ID, %% begin 4.2.0.
                      <<"config">> := #{<<"ssl">> := Ssl} = Config} = Resource, Acc) ->
-                     F = fun(B) ->
+    NewConfig = Config#{<<"ssl">> := flag_to_boolean(Ssl),
-                         case B of
+                        <<"pool_size">> => case maps:get(<<"pool_size">>, Config, undefined) of %% 4.0.x, compatible `pool_size`
-                               <<"on">> -> true;
+                                                undefined -> 8;
-                               <<"off">> -> false;
+                                                PoolSize -> PoolSize
-                               Other -> Other
+                                            end},
-                         end
+    {ok, _Resource} = import_resource(Resource#{<<"config">> := NewConfig}),
-                     end,
+    [{ID, NewConfig} | Acc];
-                    NewConfig = Config#{<<"ssl">> := F(Ssl)},
-                    {ok, _Resource} = import_resource(Resource#{<<"config">> := NewConfig}),
-                    [{ID, NewConfig} | Acc];
 
 % 4.2.3, add :content_type
 compatible_version(#{<<"id">> := ID,
@@ -308,14 +305,14 @@ compatible_version(#{<<"id">> := ID,
                                       <<"content_type">> := ContentType,%% 4.2.3 Different here
                                       <<"method">> := Method,
                                       <<"url">> := URL}} = Resource, Acc) ->
-                    Cfg = make_new_config(#{<<"url">> => URL}),
+    Cfg = make_new_config(#{<<"url">> => URL}),
-                    {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}),
+    {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}),
-                    NHeaders = maps:put(<<"content-type">>, ContentType, covert_empty_headers(Headers)),
+    NHeaders = maps:put(<<"content-type">>, ContentType, covert_empty_headers(Headers)),
-                    [{ID, #{headers => NHeaders, method => Method}} | Acc];
+    [{ID, #{headers => NHeaders, method => Method}} | Acc];
 % normal version
 compatible_version(Resource, Acc) ->
-                    {ok, _Resource} = import_resource(Resource),
+    {ok, _Resource} = import_resource(Resource),
-                    Acc.
+    Acc.
 
 make_new_config(Cfg) ->
     Config = #{<<"pool_size">> => 8,
@@ -667,19 +664,23 @@ do_import_extra_data(_Data, _Version) -> ok.
 -endif.
 
 -ifndef(EMQX_ENTERPRISE).
-covert_empty_headers(Headers) ->
+covert_empty_headers([]) -> #{};
-    case Headers of
+covert_empty_headers(Other) -> Other.
-        [] -> #{};
+
-        Other -> Other
+flag_to_boolean(<<"on">>) -> true;
-    end.
+flag_to_boolean(<<"off">>) -> false;
+flag_to_boolean(Other) -> Other.
 -endif.
 
 read_global_auth_type(Data, Version) when Version =:= "4.0" orelse
                                           Version =:= "4.1" orelse
                                           Version =:= "4.2" ->
+    ct:print("|>=> :~p~n", [Data]),
     case Data of
-        #{<<"auth.mnesia.as">> := <<"username">>} -> application:set_env(emqx_auth_mnesia, as, username);
+        #{<<"auth.mnesia.as">> := <<"username">>} ->
-        #{<<"auth.mnesia.as">> := <<"clientid">>} -> application:set_env(emqx_auth_mnesia, as, clientid);
+            application:set_env(emqx_auth_mnesia, as, username);
+        #{<<"auth.mnesia.as">> := <<"clientid">>} ->
+            application:set_env(emqx_auth_mnesia, as, clientid);
         _ ->
             logger:error("While importing data from EMQX versions prior to 4.3 "
                          "it is necessary to specify the value of \"auth.mnesia.as\" parameter "

apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl

@@ -0,0 +1,118 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_bridge_mqtt_data_export_import_SUITE).
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("emqx_rule_engine/include/rule_engine.hrl").
+-compile([export_all, nowarn_export_all]).
+%%--------------------------------------------------------------------
+%% Setups
+%%--------------------------------------------------------------------
+all() ->
+    emqx_ct:all(?MODULE).
+
+init_per_suite(Cfg) ->
+    ok = ekka_mnesia:start(),
+    ok = emqx_rule_registry:mnesia(boot),
+    ok = emqx_rule_engine:load_providers(),
+    emqx_ct_helpers:start_apps([emqx_web_hook,
+                                emqx_bridge_mqtt,
+                                emqx_rule_engine,
+                                emqx_modules,
+                                emqx_management,
+                                emqx_dashboard]),
+    Cfg.
+
+end_per_suite(Cfg) ->
+    emqx_ct_helpers:stop_apps([emqx_dashboard,
+                               emqx_management,
+                               emqx_modules,
+                               emqx_rule_engine,
+                               emqx_bridge_mqtt,
+                               emqx_web_hook]),
+    Cfg.
+
+get_data_path() ->
+    emqx_ct_helpers:deps_path(emqx_management, "test/emqx_bridge_mqtt_data_export_import_SUITE_data/").
+%%--------------------------------------------------------------------
+%% Cases
+%%--------------------------------------------------------------------
+
+handle_config(Config, 420, brigde) ->
+    ?assertEqual(<<"off">>, maps:get(<<"ssl">>, Config));
+
+handle_config(Config, 430, brigde) ->
+    ?assertEqual(false, maps:get(<<"ssl">>, Config));
+
+handle_config(Config, 420, rpc) ->
+    handle_config(Config, 430, rpc);
+
+handle_config(Config, 409, rpc) ->
+    handle_config(Config, 420, rpc);
+
+handle_config(Config, 415, rpc) ->
+    handle_config(Config, 420, rpc);
+
+handle_config(Config, 409, brigde) ->
+    handle_config(Config, 420, brigde);
+
+handle_config(Config, 415, brigde) ->
+    handle_config(Config, 420, brigde);
+
+handle_config(Config, 430, rpc) ->
+    ?assertEqual(<<"emqx@127.0.0.1">>, maps:get(<<"address">>, Config)),
+    ?assertEqual(32, maps:get(<<"batch_size">>, Config)),
+    ?assertEqual(<<"off">>, maps:get(<<"disk_cache">>, Config)),
+    ?assertEqual(<<"bridge/emqx/${node}/">>, maps:get(<<"mountpoint">>, Config)),
+    ?assertEqual(<<"30s">>, maps:get(<<"reconnect_interval">>, Config)),
+    ?assertEqual(8, maps:get(<<"pool_size">>, Config));
+
+handle_config(_, _, _) -> ok.
+
+remove_resource(Id) ->
+    emqx_rule_registry:remove_resource(Id),
+    emqx_rule_registry:remove_resource_params(Id).
+
+import(FilePath, Version) ->
+    Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}),
+    ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, Overrides),
+    lists:foreach(fun(#resource{id = Id, config = Config} = _Resource) ->
+        case Id of
+            <<"brigde">> ->
+            handle_config(Config, Version, brigde),
+            remove_resource(Id);
+            <<"rpc">> ->
+            handle_config(Config, Version, rpc),
+            remove_resource(Id);
+            _ -> ok
+        end
+    end, emqx_rule_registry:get_resources()).
+
+t_import420(_) ->
+    import("420.json", 420),
+    {ok, _} = emqx_mgmt_data_backup:export().
+
+t_import430(_) ->
+    import("430.json", 430),
+    {ok, _} = emqx_mgmt_data_backup:export().
+
+t_import409(_) ->
+    import("409.json", 409),
+    {ok, _} = emqx_mgmt_data_backup:export().
+
+t_import415(_) ->
+    import("415.json", 415),
+    {ok, _} = emqx_mgmt_data_backup:export().

apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/409.json

@@ -0,0 +1,68 @@
+{
+    "version": "4.0",
+    "users": [
+        {
+            "username": "admin",
+            "tags": "administrator",
+            "password": "m/GtjNgri9GILklefVJH8BeTnNE="
+        }
+    ],
+    "schemas": [],
+    "rules": [],
+    "resources": [
+        {
+            "type": "bridge_mqtt",
+            "id": "bridge",
+            "description": "bridge",
+            "created_at": null,
+            "config": {
+                "username": "user",
+                "ssl": "off",
+                "retry_interval": "20s",
+                "reconnect_interval": "30s",
+                "proto_ver": "mqttv4",
+                "password": "passwd",
+                "mountpoint": "bridge/emqx/${node}/",
+                "keyfile": "etc/certs/client-key.pem",
+                "keepalive": "60s",
+                "disk_cache": "off",
+                "clientid": "bridge_aws",
+                "ciphers": "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384",
+                "certfile": "etc/certs/client-cert.pem",
+                "cacertfile": "etc/certs/cacert.pem",
+                "bridge_mode": true,
+                "address": "127.0.0.1:1883"
+            }
+        },
+        {
+            "type": "bridge_rpc",
+            "id": "rpc",
+            "description": "rpc",
+            "created_at": null,
+            "config": {
+                "reconnect_interval": "30s",
+                "pool_size": 8,
+                "mountpoint": "bridge/emqx/${node}/",
+                "disk_cache": "off",
+                "batch_size": 32,
+                "address": "emqx@127.0.0.1"
+            }
+        }
+    ],
+    "date": "2021-03-30 13:38:39",
+    "blacklist": [],
+    "auth_username": [],
+    "auth_mnesia": [],
+    "auth_clientid": [],
+    "apps": [
+        {
+            "status": true,
+            "secret": "public",
+            "name": "Default",
+            "id": "admin",
+            "expired": "undefined",
+            "desc": "Application user"
+        }
+    ],
+    "acl_mnesia": []
+}

apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/415.json

@@ -0,0 +1,70 @@
+{
+    "version": "4.1",
+    "users": [
+        {
+            "username": "admin",
+            "tags": "administrator",
+            "password": "3W+nHOkCZLFspENkIvHKzCbHxHI="
+        }
+    ],
+    "schemas": [],
+    "rules": [],
+    "resources": [
+        {
+            "type": "bridge_rpc",
+            "id": "rpc",
+            "description": "rpc",
+            "created_at": null,
+            "config": {
+                "reconnect_interval": "30s",
+                "pool_size": 8,
+                "mountpoint": "bridge/emqx/${node}/",
+                "disk_cache": "off",
+                "batch_size": 32,
+                "address": "emqx@127.0.0.1"
+            }
+        },
+        {
+            "type": "bridge_mqtt",
+            "id": "bridge",
+            "description": "bridge",
+            "created_at": null,
+            "config": {
+                "username": "user",
+                "ssl": "off",
+                "retry_interval": "20s",
+                "reconnect_interval": "30s",
+                "proto_ver": "mqttv4",
+                "pool_size": 8,
+                "password": "passwd",
+                "mountpoint": "bridge/emqx/${node}/",
+                "keyfile": "etc/certs/client-key.pem",
+                "keepalive": "60s",
+                "disk_cache": "off",
+                "clientid": "client",
+                "ciphers": "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA",
+                "certfile": "etc/certs/client-cert.pem",
+                "cacertfile": "etc/certs/cacert.pem",
+                "bridge_mode": true,
+                "append": true,
+                "address": "127.0.0.1:1883"
+            }
+        }
+    ],
+    "date": "2021-03-30 13:52:53",
+    "blacklist": [],
+    "auth_username": [],
+    "auth_mnesia": [],
+    "auth_clientid": [],
+    "apps": [
+        {
+            "status": true,
+            "secret": "public",
+            "name": "Default",
+            "id": "admin",
+            "expired": "undefined",
+            "desc": "Application user"
+        }
+    ],
+    "acl_mnesia": []
+}

apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/420.json

@@ -0,0 +1,70 @@
+{
+    "version": "4.2",
+    "date": "2021-03-23 23:22:30",
+    "rules": [],
+    "resources": [
+        {
+            "id": "rpc",
+            "type": "bridge_rpc",
+            "config": {
+                "address": "emqx@127.0.0.1",
+                "batch_size": 32,
+                "disk_cache": "off",
+                "mountpoint": "bridge/emqx/${node}/",
+                "pool_size": 8,
+                "reconnect_interval": "30s"
+            },
+            "created_at": null,
+            "description": "rpc"
+        },
+        {
+            "id": "bridge",
+            "type": "bridge_mqtt",
+            "config": {
+                "address": "127.0.0.1:1883",
+                "append": true,
+                "bridge_mode": false,
+                "cacertfile": "etc/certs/cacert.pem",
+                "certfile": "etc/certs/client-cert.pem",
+                "ciphers": "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA",
+                "clientid": "client",
+                "disk_cache": "off",
+                "keepalive": "60s",
+                "keyfile": "etc/certs/client-key.pem",
+                "mountpoint": "bridge/aws/${node}/",
+                "password": "",
+                "pool_size": 8,
+                "proto_ver": "mqttv4",
+                "reconnect_interval": "30s",
+                "retry_interval": "20s",
+                "ssl": "off",
+                "username": ""
+            },
+            "created_at": null,
+            "description": "bridge"
+        }
+    ],
+    "blacklist": [],
+    "apps": [
+        {
+            "id": "admin",
+            "secret": "public",
+            "name": "Default",
+            "desc": "Application user",
+            "status": true,
+            "expired": "undefined"
+        }
+    ],
+    "users": [
+        {
+            "username": "admin",
+            "password": "mOTRLWt85F7GW+CSiBuRUZiRANw=",
+            "tags": "administrator"
+        }
+    ],
+    "auth_clientid": [],
+    "auth_username": [],
+    "auth_mnesia": [],
+    "acl_mnesia": [],
+    "schemas": []
+}

apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/430.json

@@ -0,0 +1,76 @@
+{
+    "version": "4.3",
+    "rules": [],
+    "resources": [
+        {
+            "id": "brigde",
+            "type": "bridge_mqtt",
+            "config": {
+                "address": "127.0.0.1:1883",
+                "append": true,
+                "bridge_mode": false,
+                "cacertfile": {
+                    "filename": "etc/certs/cacert.pem",
+                    "file": ""
+                },
+                "certfile": {
+                    "filename": "etc/certs/client-cert.pem",
+                    "file": ""
+                },
+                "ciphers": "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA",
+                "clientid": "client",
+                "disk_cache": "off",
+                "keepalive": "60s",
+                "keyfile": {
+                    "filename": "etc/certs/client-key.pem",
+                    "file": ""
+                },
+                "mountpoint": "bridge/aws/${node}/",
+                "password": "",
+                "pool_size": 8,
+                "proto_ver": "mqttv4",
+                "reconnect_interval": "30s",
+                "retry_interval": "20s",
+                "ssl": false,
+                "username": ""
+            },
+            "created_at": 1616635569139,
+            "description": "brigde"
+        },
+        {
+            "id": "rpc",
+            "type": "bridge_rpc",
+            "config": {
+                "address": "emqx@127.0.0.1",
+                "batch_size": 32,
+                "disk_cache": "off",
+                "mountpoint": "bridge/emqx/${node}/",
+                "pool_size": 8,
+                "reconnect_interval": "30s"
+            },
+            "created_at": 1616635583552,
+            "description": "rpc"
+        }
+    ],
+    "blacklist": [],
+    "apps": [
+        {
+            "id": "admin",
+            "secret": "public",
+            "name": "Default",
+            "desc": "Application user",
+            "status": true,
+            "expired": "undefined"
+        }
+    ],
+    "users": [
+        {
+            "username": "admin",
+            "password": "q8v7hISIMz+iKn/ZuAaogvAxKbA=",
+            "tags": "administrator"
+        }
+    ],
+    "auth_mnesia": [],
+    "acl_mnesia": [],
+    "date": "2021-03-25 09:26:34"
+}

apps/emqx_management/test/emqx_mgmt_api_SUITE.erl

@@ -68,7 +68,7 @@ end_per_suite(_Config) ->
 
 init_per_testcase(data, Config) ->
     ok = emqx_dashboard_admin:mnesia(boot),
-    application:ensure_all_started(emqx_dahboard),
+    application:ensure_all_started(emqx_dashboard),
     ok = emqx_rule_registry:mnesia(boot),
     application:ensure_all_started(emqx_rule_engine),
     Config;

apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl

@@ -0,0 +1,131 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_webhook_data_export_import_SUITE).
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("emqx_rule_engine/include/rule_engine.hrl").
+-compile([export_all, nowarn_export_all]).
+%%--------------------------------------------------------------------
+%% Setups
+%%--------------------------------------------------------------------
+all() ->
+    emqx_ct:all(?MODULE).
+
+init_per_suite(Cfg) ->
+    emqx_ct_helpers:start_apps([emqx_web_hook,
+                                emqx_bridge_mqtt,
+                                emqx_rule_engine,
+                                emqx_modules,
+                                emqx_management,
+                                emqx_dashboard]),
+    ok = ekka_mnesia:start(),
+    ok = emqx_rule_registry:mnesia(boot),
+    ok = emqx_rule_engine:load_providers(),
+    Cfg.
+
+end_per_suite(Cfg) ->
+    emqx_ct_helpers:stop_apps([emqx_dashboard,
+                               emqx_management,
+                               emqx_modules,
+                               emqx_rule_engine,
+                               emqx_bridge_mqtt,
+                               emqx_web_hook]),
+    Cfg.
+
+get_data_path() ->
+    emqx_ct_helpers:deps_path(emqx_management, "test/emqx_webhook_data_export_import_SUITE_data/").
+%%--------------------------------------------------------------------
+%% Cases
+%%--------------------------------------------------------------------
+
+handle_config(Config, 409) ->
+    handle_config(Config, 422);
+
+handle_config(Config, 415) ->
+    handle_config(Config, 422);
+
+handle_config(Config, 422) ->
+    ?assertEqual(<<"http://www.emqx.io">>, maps:get(<<"url">>, Config)),
+    ?assertEqual(<<"POST">>, maps:get(<<"method">>, Config)),
+    ?assertEqual(#{"k" => "v"}, maps:get(<<"headers">>, Config));
+
+handle_config(Config, 423) ->
+    ?assertEqual(<<"http://www.emqx.io">>, maps:get(<<"url">>, Config)),
+    ?assertEqual(<<"POST">>, maps:get(<<"method">>, Config)),
+    ?assertEqual("application/json", maps:get(<<"content_type">>, Config)),
+    ?assertEqual(#{"k" => "v"}, maps:get(<<"headers">>, Config));
+
+handle_config(Config, 425) ->
+    ?assertEqual(<<"http://www.emqx.io">>, maps:get(<<"url">>, Config)),
+    ?assertEqual(<<"POST">>, maps:get(<<"method">>, Config)),
+    ?assertEqual(#{"k" => "v"}, maps:get(<<"headers">>, Config)),
+    ?assertEqual(5, maps:get(<<"connect_timeout">>, Config)),
+    ?assertEqual(5, maps:get(<<"request_timeout">>, Config)),
+    ?assertEqual(8, maps:get(<<"pool_size">>, Config));
+
+handle_config(Config, 430) ->
+    ?assertEqual(<<"http://www.emqx.io">>, maps:get(<<"url">>, Config)),
+    ?assertEqual(<<"POST">>, maps:get(<<"method">>, Config)),
+    ?assertEqual(#{"k" => "v"}, maps:get(<<"headers">>, Config)),
+    ?assertEqual("5s", maps:get(<<"connect_timeout">>, Config)),
+    ?assertEqual("5s", maps:get(<<"request_timeout">>, Config)),
+    ?assertEqual(false, maps:get(<<"verify">>, Config)),
+    ?assertEqual(true, is_map(maps:get(<<"cacertfile">>, Config))),
+    ?assertEqual(true, is_map(maps:get(<<"certfile">>, Config))),
+    ?assertEqual(true, is_map(maps:get(<<"keyfile">>, Config))),
+    ?assertEqual(8, maps:get(<<"pool_size">>, Config));
+
+handle_config(_, _) -> ok.
+
+remove_resource(Id) ->
+    emqx_rule_registry:remove_resource(Id),
+    emqx_rule_registry:remove_resource_params(Id).
+
+import(FilePath, Version) ->
+    Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}),
+    ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, Overrides),
+    lists:foreach(fun(#resource{id = Id, config = Config} = _Resource) ->
+        case Id of
+            "webhook" ->
+            handle_config(Config, Version),
+            remove_resource(Id);
+            _ -> ok
+        end
+    end, emqx_rule_registry:get_resources()).
+
+t_import422(_) ->
+    import("422.json", 422),
+    {ok, _} = emqx_mgmt_data_backup:export().
+
+t_import423(_) ->
+    import("423.json", 423),
+    {ok, _} = emqx_mgmt_data_backup:export().
+
+t_import425(_) ->
+    import("425.json", 425),
+    {ok, _} = emqx_mgmt_data_backup:export().
+
+t_import430(_) ->
+    import("430.json", 430),
+    {ok, _} = emqx_mgmt_data_backup:export().
+
+t_import409(_) ->
+    import("409.json", 409),
+    {ok, _} = emqx_mgmt_data_backup:export().
+
+t_import415(_) ->
+    import("415.json", 415),
+    {ok, _} = emqx_mgmt_data_backup:export().

apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/409.json

@@ -0,0 +1,43 @@
+{
+    "version": "4.0",
+    "users": [
+        {
+            "username": "admin",
+            "tags": "administrator",
+            "password": "m/GtjNgri9GILklefVJH8BeTnNE="
+        }
+    ],
+    "schemas": [],
+    "rules": [],
+    "resources": [
+        {
+            "type": "web_hook",
+            "id": "webhook",
+            "description": "webhook",
+            "created_at": null,
+            "config": {
+                "headers": {
+                    "k": "v"
+                },
+                "method": "POST",
+                "url": "http://www.emqx.io"
+            }
+        }
+    ],
+    "date": "2021-03-30 13:38:39",
+    "blacklist": [],
+    "auth_username": [],
+    "auth_mnesia": [],
+    "auth_clientid": [],
+    "apps": [
+        {
+            "status": true,
+            "secret": "public",
+            "name": "Default",
+            "id": "admin",
+            "expired": "undefined",
+            "desc": "Application user"
+        }
+    ],
+    "acl_mnesia": []
+}

apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/415.json

@@ -0,0 +1,42 @@
+{
+    "version": "4.1",
+    "users": [
+        {
+            "username": "admin",
+            "tags": "administrator",
+            "password": "3W+nHOkCZLFspENkIvHKzCbHxHI="
+        }
+    ],
+    "schemas": [],
+    "rules": [],
+    "resources": [
+        {
+            "type": "web_hook",
+            "id": "webhook",
+            "description": "webhook",
+            "created_at": null,
+            "config": {
+                "url": "http://www.emqx.io",
+                "method": "POST",
+                "headers": {
+                    "k": "v"
+                }
+            }
+        }
+    ],
+    "date": "2021-03-30 13:52:53",
+    "blacklist": [],
+    "auth_username": [],
+    "auth_clientid": [],
+    "apps": [
+        {
+            "status": true,
+            "secret": "public",
+            "name": "Default",
+            "id": "admin",
+            "expired": "undefined",
+            "desc": "Application user"
+        }
+    ],
+    "acl_mnesia": []
+}

apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/422.json

@@ -0,0 +1,43 @@
+{
+    "version": "4.2",
+    "date": "2021-03-23 23:22:30",
+    "rules": [],
+    "resources": [
+        {
+            "id": "webhook",
+            "type": "web_hook",
+            "config": {
+                "headers": {
+                    "k": "v"
+                },
+                "method": "POST",
+                "url": "http://www.emqx.io"
+            },
+            "created_at": null,
+            "description": "webhook"
+        }
+    ],
+    "blacklist": [],
+    "apps": [
+        {
+            "id": "admin",
+            "secret": "public",
+            "name": "Default",
+            "desc": "Application user",
+            "status": true,
+            "expired": "undefined"
+        }
+    ],
+    "users": [
+        {
+            "username": "admin",
+            "password": "mOTRLWt85F7GW+CSiBuRUZiRANw=",
+            "tags": "administrator"
+        }
+    ],
+    "auth_clientid": [],
+    "auth_username": [],
+    "auth_mnesia": [],
+    "acl_mnesia": [],
+    "schemas": []
+}

apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/423.json

@@ -0,0 +1,44 @@
+{
+    "version": "4.2",
+    "date": "2021-03-23 23:29:24",
+    "rules": [],
+    "resources": [
+        {
+            "id": "webhook",
+            "type": "web_hook",
+            "config": {
+                "headers": {
+                    "k": "v"
+                },
+                "method": "POST",
+                "url": "http://www.emqx.io"
+            },
+            "content_type": "application/json",
+            "created_at": null,
+            "description": "webhook"
+        }
+    ],
+    "blacklist": [],
+    "apps": [
+        {
+            "id": "admin",
+            "secret": "public",
+            "name": "Default",
+            "desc": "Application user",
+            "status": true,
+            "expired": "undefined"
+        }
+    ],
+    "users": [
+        {
+            "username": "admin",
+            "password": "mOTRLWt85F7GW+CSiBuRUZiRANw=",
+            "tags": "administrator"
+        }
+    ],
+    "auth_clientid": [],
+    "auth_username": [],
+    "auth_mnesia": [],
+    "acl_mnesia": [],
+    "schemas": []
+}

apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/425.json

@@ -0,0 +1,47 @@
+{
+    "version": "4.2",
+    "date": "2021-03-24 14:51:22",
+    "rules": [],
+    "resources": [
+        {
+            "id": "webhook",
+            "type": "web_hook",
+            "config": {
+                "headers": {
+                    "k": "v"
+                },
+                "method": "POST",
+                "url": "http://www.emqx.io"
+            },
+            "content_type": "application/json",
+            "connect_timeout" : 5,
+            "request_timeout" : 5,
+            "pool_size" : 8,
+            "created_at": null,
+            "description": "webhook"
+        }
+    ],
+    "blacklist": [],
+    "apps": [
+        {
+            "id": "admin",
+            "secret": "public",
+            "name": "Default",
+            "desc": "Application user",
+            "status": true,
+            "expired": "undefined"
+        }
+    ],
+    "users": [
+        {
+            "username": "admin",
+            "password": "mOTRLWt85F7GW+CSiBuRUZiRANw=",
+            "tags": "administrator"
+        }
+    ],
+    "auth_clientid": [],
+    "auth_username": [],
+    "auth_mnesia": [],
+    "acl_mnesia": [],
+    "schemas": []
+}

apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/430.json

@@ -0,0 +1,52 @@
+{
+    "version": "4.3",
+    "rules": [],
+    "resources": [
+        {
+            "id": "webhook",
+            "type": "web_hook",
+            "config": {
+                "cacertfile": {
+                    "filename": "",
+                    "file": ""
+                },
+                "certfile": {
+                    "filename": "",
+                    "file": ""
+                },
+                "keyfile": {
+                    "filename": "",
+                    "file": ""
+                },
+                "connect_timeout": "5s",
+                "pool_size": 8,
+                "request_timeout": "5s",
+                "url": "http://www.emqx.io",
+                "verify": false
+            },
+            "created_at": 1616581851001,
+            "description": "webhook"
+        }
+    ],
+    "blacklist": [],
+    "apps": [
+        {
+            "id": "admin",
+            "secret": "public",
+            "name": "Default",
+            "desc": "Application user",
+            "status": true,
+            "expired": "undefined"
+        }
+    ],
+    "users": [
+        {
+            "username": "admin",
+            "password": "q8v7hISIMz+iKn/ZuAaogvAxKbA=",
+            "tags": "administrator"
+        }
+    ],
+    "auth_mnesia": [],
+    "acl_mnesia": [],
+    "date": "2021-03-24 18:31:21"
+}

apps/emqx_web_hook/test/emqx_web_hook_SUITE.erl

@@ -44,16 +44,21 @@ groups() ->
      {ipv6https, [sequence], Cases}].
 
 init_per_group(Name, Config) ->
+    application:ensure_all_started(emqx_management),
     set_special_cfgs(),
     case Name of
         http ->
-            emqx_ct_helpers:start_apps([emqx_web_hook], fun set_special_configs_http/1);
+            emqx_ct_helpers:start_apps([emqx_web_hook, emqx_modules, emqx_management,
+                                        emqx_rule_engine], fun set_special_configs_http/1);
         https ->
-            emqx_ct_helpers:start_apps([emqx_web_hook], fun set_special_configs_https/1);
+            emqx_ct_helpers:start_apps([emqx_web_hook, emqx_modules, emqx_management,
+                                        emqx_rule_engine], fun set_special_configs_https/1);
         ipv6http ->
-            emqx_ct_helpers:start_apps([emqx_web_hook], fun set_special_configs_ipv6_http/1);
+            emqx_ct_helpers:start_apps([emqx_web_hook, emqx_management,
+                                        emqx_rule_engine], fun set_special_configs_ipv6_http/1);
         ipv6https ->
-            emqx_ct_helpers:start_apps([emqx_web_hook], fun set_special_configs_ipv6_https/1)
+            emqx_ct_helpers:start_apps([emqx_web_hook, emqx_management,
+                                        emqx_rule_engine], fun set_special_configs_ipv6_https/1)
     end,
     Config.