From e949cdca98a2963498cd139627651c61af52c3ab Mon Sep 17 00:00:00 2001 From: Rory Z Date: Fri, 25 Jun 2021 10:58:16 +0800 Subject: [PATCH] chore: delete import and export feature --- apps/emqx_authentication/rebar.config | 3 +- apps/emqx_connector/rebar.config | 16 +- .../src/emqx_mgmt_api_data.erl | 186 ----- apps/emqx_management/src/emqx_mgmt_cli.erl | 31 - .../src/emqx_mgmt_data_backup.erl | 739 ------------------ .../test/emqx_auth_mnesia_migration_SUITE.erl | 176 ----- .../make_data.sh | 134 ---- .../v4.0.11-no-auth.json | 23 - .../v4.0.11.json | 28 - .../v4.0.13.json | 37 - .../v4.1.5.json | 58 -- .../v4.2.10-no-auth.json | 29 - .../v4.2.10.json | 53 -- .../v4.2.11.json | 58 -- rebar.config | 1 - 15 files changed, 16 insertions(+), 1556 deletions(-) delete mode 100644 apps/emqx_management/src/emqx_mgmt_api_data.erl delete mode 100644 apps/emqx_management/src/emqx_mgmt_data_backup.erl delete mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl delete mode 100755 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh delete mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11-no-auth.json delete mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11.json delete mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.13.json delete mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json delete mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json delete mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10.json delete mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.11.json diff --git a/apps/emqx_authentication/rebar.config b/apps/emqx_authentication/rebar.config index 89bd1f54a..73696b033 100644 --- a/apps/emqx_authentication/rebar.config +++ b/apps/emqx_authentication/rebar.config @@ -1,5 +1,4 @@ -{deps, [{jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}} - ]}. +{deps, []}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index 53b5c63e8..633b427cf 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -4,7 +4,21 @@ ]}. {deps, [ - {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}} + {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}}, + {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}}, + {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}}, + {epgsql, {git, "https://github.com/epgsql/epgsql", {tag, "4.4.0"}}}, + %% NOTE: mind poolboy version when updating mongodb-erlang version + {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.7"}}}, + %% NOTE: mind poolboy version when updating eredis_cluster version + {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.6.6"}}}, + %% mongodb-erlang uses a special fork https://github.com/comtihon/poolboy.git + %% (which has overflow_ttl feature added). + %% However, it references `{branch, "master}` (commit 9c06a9a on 2021-04-07). + %% By accident, We have always been using the upstream fork due to + %% eredis_cluster's dependency getting resolved earlier. + %% Here we pin 1.5.2 to avoid surprises in the future. + {poolboy, {git, "https://github.com/emqx/poolboy.git", {tag, "1.5.2"}}} ]}. {shell, [ diff --git a/apps/emqx_management/src/emqx_mgmt_api_data.erl b/apps/emqx_management/src/emqx_mgmt_api_data.erl deleted file mode 100644 index e389a1313..000000000 --- a/apps/emqx_management/src/emqx_mgmt_api_data.erl +++ /dev/null @@ -1,186 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_mgmt_api_data). - --include_lib("emqx/include/emqx.hrl"). - --include_lib("kernel/include/file.hrl"). - --include("emqx_mgmt.hrl"). - --rest_api(#{name => export, - method => 'POST', - path => "/data/export", - func => export, - descr => "Export data"}). - --rest_api(#{name => list_exported, - method => 'GET', - path => "/data/export", - func => list_exported, - descr => "List exported file"}). - --rest_api(#{name => import, - method => 'POST', - path => "/data/import", - func => import, - descr => "Import data"}). - --rest_api(#{name => download, - method => 'GET', - path => "/data/file/:filename", - func => download, - descr => "Download data file to local"}). - --rest_api(#{name => upload, - method => 'POST', - path => "/data/file", - func => upload, - descr => "Upload data file from local"}). - --rest_api(#{name => delete, - method => 'DELETE', - path => "/data/file/:filename", - func => delete, - descr => "Delete data file"}). - --export([ export/2 - , list_exported/2 - , import/2 - , download/2 - , upload/2 - , delete/2 - ]). - --export([ get_list_exported/0 - , do_import/1 - ]). - -export(_Bindings, _Params) -> - case emqx_mgmt_data_backup:export() of - {ok, File = #{filename := Filename}} -> - minirest:return({ok, File#{filename => filename:basename(Filename)}}); - Return -> minirest:return(Return) - end. - -list_exported(_Bindings, _Params) -> - List = [ rpc:call(Node, ?MODULE, get_list_exported, []) || Node <- ekka_mnesia:running_nodes() ], - NList = lists:map(fun({_, FileInfo}) -> FileInfo end, lists:keysort(1, lists:append(List))), - minirest:return({ok, NList}). - -get_list_exported() -> - Dir = emqx:get_env(data_dir), - {ok, Files} = file:list_dir_all(Dir), - lists:foldl( - fun(File, Acc) -> - case filename:extension(File) =:= ".json" of - true -> - FullFile = filename:join([Dir, File]), - case file:read_file_info(FullFile) of - {ok, #file_info{size = Size, ctime = CTime = {{Y, M, D}, {H, MM, S}}}} -> - CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y, M, D, H, MM, S]), - Seconds = calendar:datetime_to_gregorian_seconds(CTime), - [{Seconds, [{filename, list_to_binary(File)}, - {size, Size}, - {created_at, list_to_binary(CreatedAt)}, - {node, node()} - ]} | Acc]; - {error, Reason} -> - logger:error("Read file info of ~s failed with: ~p", [File, Reason]), - Acc - end; - false -> Acc - end - end, [], Files). - -import(_Bindings, Params) -> - case proplists:get_value(<<"filename">>, Params) of - undefined -> - Result = import_content(Params), - minirest:return(Result); - Filename -> - case proplists:get_value(<<"node">>, Params) of - undefined -> - Result = do_import(Filename), - minirest:return(Result); - Node -> - case lists:member(Node, - [ erlang:atom_to_binary(N, utf8) || N <- ekka_mnesia:running_nodes() ] - ) of - true -> minirest:return(rpc:call(erlang:binary_to_atom(Node, utf8), ?MODULE, do_import, [Filename])); - false -> minirest:return({error, no_existent_node}) - end - end - end. - -do_import(Filename) -> - FullFilename = fullname(Filename), - emqx_mgmt_data_backup:import(FullFilename, "{}"). - -download(#{filename := Filename}, _Params) -> - FullFilename = fullname(Filename), - case file:read_file(FullFilename) of - {ok, Bin} -> - {ok, #{filename => list_to_binary(Filename), - file => Bin}}; - {error, Reason} -> - minirest:return({error, Reason}) - end. - -upload(Bindings, Params) -> - do_upload(Bindings, maps:from_list(Params)). - -do_upload(_Bindings, #{<<"filename">> := Filename, - <<"file">> := Bin}) -> - FullFilename = fullname(Filename), - case file:write_file(FullFilename, Bin) of - ok -> - minirest:return({ok, [{node, node()}]}); - {error, Reason} -> - minirest:return({error, Reason}) - end; -do_upload(Bindings, Params = #{<<"file">> := _}) -> - do_upload(Bindings, Params#{<<"filename">> => tmp_filename()}); -do_upload(_Bindings, _Params) -> - minirest:return({error, missing_required_params}). - -delete(#{filename := Filename}, _Params) -> - FullFilename = fullname(Filename), - case file:delete(FullFilename) of - ok -> - minirest:return(); - {error, Reason} -> - minirest:return({error, Reason}) - end. - -import_content(Content) -> - File = dump_to_tmp_file(Content), - do_import(File). - -dump_to_tmp_file(Content) -> - Bin = emqx_json:encode(Content), - Filename = tmp_filename(), - ok = file:write_file(fullname(Filename), Bin), - Filename. - -fullname(Name) -> - filename:join(emqx:get_env(data_dir), Name). - -tmp_filename() -> - Seconds = erlang:system_time(second), - {{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds), - io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]). diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 77fe96182..e70def3ee 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -38,7 +38,6 @@ , trace/1 , log/1 , mgmt/1 - , data/1 , acl/1 ]). @@ -544,36 +543,6 @@ stop_listener(#{listen_on := ListenOn} = Listener, _Input) -> [ID, ListenOnStr, Reason]) end. -%%-------------------------------------------------------------------- -%% @doc data Command - -data(["export"]) -> - case emqx_mgmt_data_backup:export() of - {ok, #{filename := Filename}} -> - emqx_ctl:print("The emqx data has been successfully exported to ~s.~n", [Filename]); - {error, Reason} -> - emqx_ctl:print("The emqx data export failed due to ~p.~n", [Reason]) - end; - -data(["import", Filename]) -> - data(["import", Filename, "--env", "{}"]); -data(["import", Filename, "--env", Env]) -> - case emqx_mgmt_data_backup:import(Filename, Env) of - ok -> - emqx_ctl:print("The emqx data has been imported successfully.~n"); - {error, import_failed} -> - emqx_ctl:print("The emqx data import failed.~n"); - {error, unsupported_version} -> - emqx_ctl:print("The emqx data import failed: Unsupported version.~n"); - {error, Reason} -> - emqx_ctl:print("The emqx data import failed: ~0p while reading ~s.~n", [Reason, Filename]) - end; - -data(_) -> - emqx_ctl:usage([{"data import [--env '']", - "Import data from the specified file, possibly with overrides"}, - {"data export", "Export data"}]). - %%-------------------------------------------------------------------- %% @doc acl Command diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl deleted file mode 100644 index 71a92686e..000000000 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ /dev/null @@ -1,739 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_mgmt_data_backup). - --include("emqx_mgmt.hrl"). --include_lib("emqx_rule_engine/include/rule_engine.hrl"). --include_lib("emqx/include/emqx.hrl"). --include_lib("kernel/include/file.hrl"). - --ifdef(EMQX_ENTERPRISE). --export([ export_modules/0 - , export_schemas/0 - , export_confs/0 - , import_modules/1 - , import_schemas/1 - , import_confs/2 - ]). --endif. - --export([ export_rules/0 - , export_resources/0 - , export_blacklist/0 - , export_applications/0 - , export_users/0 - , export_auth_mnesia/0 - , export_acl_mnesia/0 - , import_resources_and_rules/3 - , import_rules/1 - , import_resources/1 - , import_blacklist/1 - , import_applications/1 - , import_users/1 - , import_auth_clientid/1 %% BACKW: 4.1.x - , import_auth_username/1 %% BACKW: 4.1.x - , import_auth_mnesia/2 - , import_acl_mnesia/2 - , to_version/1 - ]). - --export([ export/0 - , import/2 - ]). - -%%-------------------------------------------------------------------- -%% Data Export and Import -%%-------------------------------------------------------------------- - -export_rules() -> - lists:map(fun(#rule{id = RuleId, - rawsql = RawSQL, - actions = Actions, - enabled = Enabled, - description = Desc}) -> - [{id, RuleId}, - {rawsql, RawSQL}, - {actions, actions_to_prop_list(Actions)}, - {enabled, Enabled}, - {description, Desc}] - end, emqx_rule_registry:get_rules()). - -export_resources() -> - lists:map(fun(#resource{id = Id, - type = Type, - config = Config, - created_at = CreatedAt, - description = Desc}) -> - NCreatedAt = case CreatedAt of - undefined -> null; - _ -> CreatedAt - end, - [{id, Id}, - {type, Type}, - {config, maps:to_list(Config)}, - {created_at, NCreatedAt}, - {description, Desc}] - end, emqx_rule_registry:get_resources()). - -export_blacklist() -> - lists:map(fun(#banned{who = Who, by = By, reason = Reason, at = At, until = Until}) -> - NWho = case Who of - {peerhost, Peerhost} -> {peerhost, inet:ntoa(Peerhost)}; - _ -> Who - end, - [{who, [NWho]}, {by, By}, {reason, Reason}, {at, At}, {until, Until}] - end, ets:tab2list(emqx_banned)). - -export_applications() -> - lists:map(fun({_, AppID, AppSecret, Name, Desc, Status, Expired}) -> - [{id, AppID}, {secret, AppSecret}, {name, Name}, {desc, Desc}, {status, Status}, {expired, Expired}] - end, ets:tab2list(mqtt_app)). - -export_users() -> - lists:map(fun({_, Username, Password, Tags}) -> - [{username, Username}, {password, base64:encode(Password)}, {tags, Tags}] - end, ets:tab2list(mqtt_admin)). - -export_auth_mnesia() -> - case ets:info(emqx_user) of - undefined -> []; - _ -> - lists:map(fun({_, {Type, Login}, Password, CreatedAt}) -> - [{login, Login}, {type, Type}, {password, base64:encode(Password)}, {created_at, CreatedAt}] - end, ets:tab2list(emqx_user)) - end. - -export_acl_mnesia() -> - case ets:info(emqx_acl) of - undefined -> []; - _ -> - lists:map(fun({_, Filter, Action, Access, CreatedAt}) -> - Filter1 = case Filter of - {{Type, TypeValue}, Topic} -> - [{type, Type}, {type_value, TypeValue}, {topic, Topic}]; - {Type, Topic} -> - [{type, Type}, {topic, Topic}] - end, - Filter1 ++ [{action, Action}, {access, Access}, {created_at, CreatedAt}] - end, ets:tab2list(emqx_acl)) - end. - --ifdef(EMQX_ENTERPRISE). -export_modules() -> - case ets:info(emqx_modules) of - undefined -> []; - _ -> - lists:map(fun({_, Id, Type, Config, Enabled, CreatedAt, Description}) -> - [{id, Id}, - {type, Type}, - {config, Config}, - {enabled, Enabled}, - {created_at, CreatedAt}, - {description, Description} - ] - end, ets:tab2list(emqx_modules)) - end. - -export_schemas() -> - case ets:info(emqx_schema) of - undefined -> []; - _ -> - [emqx_schema_api:format_schema(Schema) || Schema <- emqx_schema_registry:get_all_schemas()] - end. - -export_confs() -> - case ets:info(emqx_conf_info) of - undefined -> {[], []}; - _ -> - {lists:map(fun({_, Key, Confs}) -> - case Key of - {_Zone, Name} -> - [{zone, list_to_binary(Name)}, - {confs, confs_to_binary(Confs)}]; - {_Listener, Type, Name} -> - [{type, list_to_binary(Type)}, - {name, list_to_binary(Name)}, - {confs, confs_to_binary(Confs)}]; - Name -> - [{name, list_to_binary(Name)}, - {confs, confs_to_binary(Confs)}] - end - end, ets:tab2list(emqx_conf_b)), - lists:map(fun({_, {_Listener, Type, Name}, Status}) -> - [{type, list_to_binary(Type)}, - {name, list_to_binary(Name)}, - {status, Status}] - end, ets:tab2list(emqx_listeners_state))} - end. - -confs_to_binary(Confs) -> - [{list_to_binary(Key), list_to_binary(Val)} || {Key, Val} <-Confs]. - --endif. - -import_rule(#{<<"id">> := RuleId, - <<"rawsql">> := RawSQL, - <<"actions">> := Actions, - <<"enabled">> := Enabled, - <<"description">> := Desc}) -> - Rule = #{id => RuleId, - rawsql => RawSQL, - actions => map_to_actions(Actions), - enabled => Enabled, - description => Desc}, - try emqx_rule_engine:create_rule(Rule) - catch throw:{resource_not_initialized, _ResId} -> - emqx_rule_engine:create_rule(Rule#{enabled => false}) - end. - -map_to_actions(Maps) -> - [map_to_action(M) || M <- Maps]. - -map_to_action(Map = #{<<"id">> := ActionInstId, <<"name">> := Name, <<"args">> := Args}) -> - #{id => ActionInstId, - name => any_to_atom(Name), - args => Args, - fallbacks => map_to_actions(maps:get(<<"fallbacks">>, Map, []))}. - - -import_rules(Rules) -> - lists:foreach(fun(Rule) -> - import_rule(Rule) - end, Rules). - -import_resources(Reources) -> - lists:foreach(fun(Resource) -> - import_resource(Resource) - end, Reources). - -import_resource(#{<<"id">> := Id, - <<"type">> := Type, - <<"config">> := Config, - <<"created_at">> := CreatedAt, - <<"description">> := Desc}) -> - NCreatedAt = case CreatedAt of - null -> undefined; - _ -> CreatedAt - end, - emqx_rule_engine:create_resource(#{id => Id, - type => any_to_atom(Type), - config => Config, - created_at => NCreatedAt, - description => Desc}). -import_resources_and_rules(Resources, Rules, FromVersion) - when FromVersion =:= "4.0" orelse - FromVersion =:= "4.1" orelse - FromVersion =:= "4.2" -> - Configs = lists:foldl(fun compatible_version/2 , [], Resources), - lists:foreach(fun(#{<<"actions">> := Actions} = Rule) -> - NActions = apply_new_config(Actions, Configs), - import_rule(Rule#{<<"actions">> := NActions}) - end, Rules); -import_resources_and_rules(Resources, Rules, _FromVersion) -> - import_resources(Resources), - import_rules(Rules). - -%% 4.2.5 + -compatible_version(#{<<"id">> := ID, - <<"type">> := <<"web_hook">>, - <<"config">> := #{<<"connect_timeout">> := ConnectTimeout, - <<"content_type">> := ContentType, - <<"headers">> := Headers, - <<"method">> := Method, - <<"pool_size">> := PoolSize, - <<"request_timeout">> := RequestTimeout, - <<"url">> := URL}} = Resource, Acc) -> - CovertFun = fun(Int) -> - list_to_binary(integer_to_list(Int) ++ "s") - end, - Cfg = make_new_config(#{<<"pool_size">> => PoolSize, - <<"connect_timeout">> => CovertFun(ConnectTimeout), - <<"request_timeout">> => CovertFun(RequestTimeout), - <<"url">> => URL}), - {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}), - NHeaders = maps:put(<<"content-type">>, ContentType, covert_empty_headers(Headers)), - [{ID, #{headers => NHeaders, method => Method}} | Acc]; -% 4.2.0 -compatible_version(#{<<"id">> := ID, - <<"type">> := <<"web_hook">>, - <<"config">> := #{<<"headers">> := Headers, - <<"method">> := Method,%% 4.2.0 Different here - <<"url">> := URL}} = Resource, Acc) -> - Cfg = make_new_config(#{<<"url">> => URL}), - {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}), - NHeaders = maps:put(<<"content-type">>, <<"application/json">> , covert_empty_headers(Headers)), - [{ID, #{headers => NHeaders, method => Method}} | Acc]; - -%% bridge mqtt -%% 4.2.0 - 4.2.5 bridge_mqtt, ssl enabled from on/off to true/false -compatible_version(#{<<"type">> := <<"bridge_mqtt">>, - <<"id">> := ID, %% begin 4.2.0. - <<"config">> := #{<<"ssl">> := Ssl} = Config} = Resource, Acc) -> - NewConfig = Config#{<<"ssl">> := flag_to_boolean(Ssl), - <<"pool_size">> => case maps:get(<<"pool_size">>, Config, undefined) of %% 4.0.x, compatible `pool_size` - undefined -> 8; - PoolSize -> PoolSize - end}, - {ok, _Resource} = import_resource(Resource#{<<"config">> := NewConfig}), - [{ID, NewConfig} | Acc]; - -% 4.2.3, add :content_type -compatible_version(#{<<"id">> := ID, - <<"type">> := <<"web_hook">>, - <<"config">> := #{<<"headers">> := Headers, - <<"content_type">> := ContentType,%% 4.2.3 Different here - <<"method">> := Method, - <<"url">> := URL}} = Resource, Acc) -> - Cfg = make_new_config(#{<<"url">> => URL}), - {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}), - NHeaders = maps:put(<<"content-type">>, ContentType, covert_empty_headers(Headers)), - [{ID, #{headers => NHeaders, method => Method}} | Acc]; -% normal version -compatible_version(Resource, Acc) -> - {ok, _Resource} = import_resource(Resource), - Acc. - -make_new_config(Cfg) -> - Config = #{<<"pool_size">> => 8, - <<"connect_timeout">> => <<"5s">>, - <<"request_timeout">> => <<"5s">>, - <<"cacertfile">> => <<>>, - <<"certfile">> => <<>>, - <<"keyfile">> => <<>>, - <<"verify">> => false}, - maps:merge(Cfg, Config). - -apply_new_config(Actions, Configs) -> - apply_new_config(Actions, Configs, []). - -apply_new_config([], _Configs, Acc) -> - Acc; -apply_new_config(Actions, [], []) -> - Actions; -apply_new_config([Action = #{<<"name">> := <<"data_to_webserver">>, - <<"args">> := #{<<"$resource">> := ID, - <<"path">> := Path, - <<"payload_tmpl">> := PayloadTmpl}} | More], Configs, Acc) -> - case proplists:get_value(ID, Configs, undefined) of - undefined -> - apply_new_config(More, Configs, [Action | Acc]); - #{headers := Headers, method := Method} -> - Args = #{<<"$resource">> => ID, - <<"body">> => PayloadTmpl, - <<"headers">> => Headers, - <<"method">> => Method, - <<"path">> => Path}, - apply_new_config(More, Configs, [Action#{<<"args">> := Args} | Acc]) - end; - -apply_new_config([Action = #{<<"args">> := #{<<"$resource">> := ResourceId, - <<"forward_topic">> := ForwardTopic, - <<"payload_tmpl">> := PayloadTmpl}, - <<"fallbacks">> := _Fallbacks, - <<"id">> := _Id, - <<"name">> := <<"data_to_mqtt_broker">>} | More], Configs, Acc) -> - Args = #{<<"$resource">> => ResourceId, - <<"payload_tmpl">> => PayloadTmpl, - <<"forward_topic">> => ForwardTopic}, - apply_new_config(More, Configs, [Action#{<<"args">> := Args} | Acc]). - - -actions_to_prop_list(Actions) -> - [action_to_prop_list(Act) || Act <- Actions]. - -action_to_prop_list({action_instance, ActionInstId, Name, FallbackActions, Args}) -> - [{id, ActionInstId}, - {name, Name}, - {fallbacks, actions_to_prop_list(FallbackActions)}, - {args, Args}]. - -import_blacklist(Blacklist) -> - lists:foreach(fun(#{<<"who">> := Who, - <<"by">> := By, - <<"reason">> := Reason, - <<"at">> := At, - <<"until">> := Until}) -> - NWho = case Who of - #{<<"peerhost">> := Peerhost} -> - {ok, NPeerhost} = inet:parse_address(Peerhost), - {peerhost, NPeerhost}; - #{<<"clientid">> := ClientId} -> {clientid, ClientId}; - #{<<"username">> := Username} -> {username, Username} - end, - emqx_banned:create(#banned{who = NWho, by = By, reason = Reason, at = At, until = Until}) - end, Blacklist). - -import_applications(Apps) -> - lists:foreach(fun(#{<<"id">> := AppID, - <<"secret">> := AppSecret, - <<"name">> := Name, - <<"desc">> := Desc, - <<"status">> := Status, - <<"expired">> := Expired}) -> - NExpired = case is_integer(Expired) of - true -> Expired; - false -> undefined - end, - emqx_mgmt_auth:force_add_app(AppID, Name, AppSecret, Desc, Status, NExpired) - end, Apps). - -import_users(Users) -> - lists:foreach(fun(#{<<"username">> := Username, - <<"password">> := Password, - <<"tags">> := Tags}) -> - NPassword = base64:decode(Password), - emqx_dashboard_admin:force_add_user(Username, NPassword, Tags) - end, Users). - -import_auth_clientid(Lists) -> - case ets:info(emqx_user) of - undefined -> ok; - _ -> - lists:foreach(fun(#{<<"clientid">> := Clientid, <<"password">> := Password}) -> - mnesia:dirty_write({emqx_user, {clientid, Clientid} - , base64:decode(Password) - , erlang:system_time(millisecond)}) - end, Lists) - end. - -import_auth_username(Lists) -> - case ets:info(emqx_user) of - undefined -> ok; - _ -> - lists:foreach(fun(#{<<"username">> := Username, <<"password">> := Password}) -> - mnesia:dirty_write({emqx_user, {username, Username}, base64:decode(Password), erlang:system_time(millisecond)}) - end, Lists) - end. - --ifdef(EMQX_ENTERPRISE). -import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse - FromVersion =:= "4.1" -> - do_import_auth_mnesia_by_old_data(Auths); -import_auth_mnesia(Auths, _) -> - do_import_auth_mnesia(Auths). - -import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse - FromVersion =:= "4.1" -> - do_import_acl_mnesia_by_old_data(Acls); - -import_acl_mnesia(Acls, _) -> - do_import_acl_mnesia(Acls). --else. -import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.3" -> - do_import_auth_mnesia(Auths); -import_auth_mnesia(Auths, _FromVersion) -> - do_import_auth_mnesia_by_old_data(Auths). - -import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.3" -> - do_import_acl_mnesia(Acls); -import_acl_mnesia(Acls, _FromVersion) -> - do_import_acl_mnesia_by_old_data(Acls). - --endif. - -do_import_auth_mnesia_by_old_data(Auths) -> - case ets:info(emqx_user) of - undefined -> ok; - _ -> - CreatedAt = erlang:system_time(millisecond), - lists:foreach(fun(#{<<"login">> := Login, - <<"password">> := Password}) -> - mnesia:dirty_write({emqx_user, {get_old_type(), Login}, base64:decode(Password), CreatedAt}) - end, Auths) - end. - - -do_import_auth_mnesia(Auths) -> - case ets:info(emqx_user) of - undefined -> ok; - _ -> - lists:foreach(fun(#{<<"login">> := Login, - <<"type">> := Type, - <<"password">> := Password } = Map) -> - CreatedAt = maps:get(<<"created_at">>, Map, erlang:system_time(millisecond)), - mnesia:dirty_write({emqx_user, {any_to_atom(Type), Login}, base64:decode(Password), CreatedAt}) - end, Auths) - end. - -do_import_acl_mnesia_by_old_data(Acls) -> - case ets:info(emqx_acl) of - undefined -> ok; - _ -> - CreatedAt = erlang:system_time(millisecond), - lists:foreach(fun(#{<<"login">> := Login, - <<"topic">> := Topic, - <<"allow">> := Allow, - <<"action">> := Action}) -> - Allow1 = case any_to_atom(Allow) of - true -> allow; - false -> deny - end, - mnesia:dirty_write({emqx_acl, {{get_old_type(), Login}, Topic}, any_to_atom(Action), Allow1, CreatedAt}) - end, Acls) - end. -do_import_acl_mnesia(Acls) -> - case ets:info(emqx_acl) of - undefined -> ok; - _ -> - lists:foreach(fun(Map = #{<<"action">> := Action, - <<"access">> := Access}) -> - Topic = maps:get(<<"topic">>, Map), - Login = case maps:get(<<"type_value">>, Map, undefined) of - undefined -> - all; - Value -> - {any_to_atom(maps:get(<<"type">>, Map)), Value} - end, - emqx_acl_mnesia_cli:add_acl(Login, Topic, any_to_atom(Action), any_to_atom(Access)) - end, Acls) - end. - --ifdef(EMQX_ENTERPRISE). --dialyzer({nowarn_function, [import_modules/1]}). -import_modules(Modules) -> - case ets:info(emqx_modules) of - undefined -> - ok; - _ -> - lists:foreach(fun(#{<<"id">> := Id, - <<"type">> := Type, - <<"config">> := Config, - <<"enabled">> := Enabled, - <<"created_at">> := CreatedAt, - <<"description">> := Description}) -> - _ = emqx_modules:import_module({Id, any_to_atom(Type), Config, Enabled, CreatedAt, Description}) - end, Modules) - end. - - -import_schemas(Schemas) -> - case ets:info(emqx_schema) of - undefined -> ok; - _ -> [emqx_schema_registry:add_schema(emqx_schema_api:make_schema_params(Schema)) || Schema <- Schemas] - end. - -import_confs(Configs, ListenersState) -> - case ets:info(emqx_conf_info) of - undefined -> ok; - _ -> - emqx_conf:import_confs(Configs, ListenersState) - end. - --endif. - -any_to_atom(L) when is_list(L) -> list_to_atom(L); -any_to_atom(B) when is_binary(B) -> binary_to_atom(B, utf8); -any_to_atom(A) when is_atom(A) -> A. - -to_version(Version) when is_integer(Version) -> - integer_to_list(Version); -to_version(Version) when is_binary(Version) -> - binary_to_list(Version); -to_version(Version) when is_list(Version) -> - Version. - -export() -> - Seconds = erlang:system_time(second), - Data = do_export_data() ++ [{date, erlang:list_to_binary(emqx_mgmt_util:strftime(Seconds))}], - {{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds), - Filename = io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]), - NFilename = filename:join([emqx:get_env(data_dir), Filename]), - ok = filelib:ensure_dir(NFilename), - case file:write_file(NFilename, emqx_json:encode(Data)) of - ok -> - case file:read_file_info(NFilename) of - {ok, #file_info{size = Size, ctime = {{Y1, M1, D1}, {H1, MM1, S1}}}} -> - CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y1, M1, D1, H1, MM1, S1]), - {ok, #{filename => list_to_binary(NFilename), - size => Size, - created_at => list_to_binary(CreatedAt), - node => node() - }}; - Error -> Error - end; - Error -> Error - end. - -do_export_data() -> - Version = string:sub_string(emqx_sys:version(), 1, 3), - [{version, erlang:list_to_binary(Version)}, - {rules, export_rules()}, - {resources, export_resources()}, - {blacklist, export_blacklist()}, - {apps, export_applications()}, - {users, export_users()}, - {auth_mnesia, export_auth_mnesia()}, - {acl_mnesia, export_acl_mnesia()} - ] ++ do_export_extra_data(). - --ifdef(EMQX_ENTERPRISE). -do_export_extra_data() -> - {Configs, State} = export_confs(), - [{modules, export_modules()}, - {schemas, export_schemas()}, - {configs, Configs}, - {listeners_state, State} - ]. --else. -do_export_extra_data() -> []. --endif. - --ifdef(EMQX_ENTERPRISE). -import(Filename, OverridesJson) -> - case file:read_file(Filename) of - {ok, Json} -> - Imported = emqx_json:decode(Json, [return_maps]), - Overrides = emqx_json:decode(OverridesJson, [return_maps]), - Data = maps:merge(Imported, Overrides), - Version = to_version(maps:get(<<"version">>, Data)), - read_global_auth_type(Data), - try - do_import_data(Data, Version), - logger:debug("The emqx data has been imported successfully"), - ok - catch Class:Reason:Stack -> - logger:error("The emqx data import failed: ~0p", [{Class, Reason, Stack}]), - {error, import_failed} - end; - Error -> Error - end. --else. -import(Filename, OverridesJson) -> - case file:read_file(Filename) of - {ok, Json} -> - Imported = emqx_json:decode(Json, [return_maps]), - Overrides = emqx_json:decode(OverridesJson, [return_maps]), - Data = maps:merge(Imported, Overrides), - Version = to_version(maps:get(<<"version">>, Data)), - read_global_auth_type(Data), - case is_version_supported(Data, Version) of - true -> - try - do_import_data(Data, Version), - logger:debug("The emqx data has been imported successfully"), - ok - catch Class:Reason:Stack -> - logger:error("The emqx data import failed: ~0p", [{Class, Reason, Stack}]), - {error, import_failed} - end; - false -> - logger:error("Unsupported version: ~p", [Version]), - {error, unsupported_version, Version} - end; - Error -> Error - end. --endif. - -do_import_data(Data, Version) -> - do_import_extra_data(Data, Version), - import_resources_and_rules(maps:get(<<"resources">>, Data, []), maps:get(<<"rules">>, Data, []), Version), - import_blacklist(maps:get(<<"blacklist">>, Data, [])), - import_applications(maps:get(<<"apps">>, Data, [])), - import_users(maps:get(<<"users">>, Data, [])), - import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), - import_auth_username(maps:get(<<"auth_username">>, Data, [])), - import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version), - import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version). - --ifdef(EMQX_ENTERPRISE). -do_import_extra_data(Data, _Version) -> - _ = import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])), - _ = import_modules(maps:get(<<"modules">>, Data, [])), - _ = import_schemas(maps:get(<<"schemas">>, Data, [])), - ok. --else. -do_import_extra_data(_Data, _Version) -> ok. --endif. - -covert_empty_headers([]) -> #{}; -covert_empty_headers(Other) -> Other. - -flag_to_boolean(<<"on">>) -> true; -flag_to_boolean(<<"off">>) -> false; -flag_to_boolean(Other) -> Other. - --ifndef(EMQX_ENTERPRISE). -is_version_supported(Data, Version) -> - case { maps:get(<<"auth_clientid">>, Data, []) - , maps:get(<<"auth_username">>, Data, []) - , maps:get(<<"auth_mnesia">>, Data, [])} of - {[], [], []} -> lists:member(Version, ?VERSIONS); - _ -> is_version_supported2(Version) - end. - -is_version_supported2("4.1") -> - true; -is_version_supported2("4.3") -> - true; -is_version_supported2(Version) -> - case re:run(Version, "^4.[02].\\d+$", [{capture, none}]) of - match -> - try lists:map(fun erlang:list_to_integer/1, string:tokens(Version, ".")) of - [4, 2, N] -> N >= 11; - [4, 0, N] -> N >= 13; - _ -> false - catch - _ : _ -> false - end; - nomatch -> - false - end. --endif. - -read_global_auth_type(Data) -> - case {maps:get(<<"auth_mnesia">>, Data, []), maps:get(<<"acl_mnesia">>, Data, [])} of - {[], []} -> - %% Auth mnesia plugin is not used: - ok; - _ -> - do_read_global_auth_type(Data) - end. - --ifdef(EMQX_ENTERPRISE). -do_read_global_auth_type(Data) -> - case Data of - #{<<"auth.mnesia.as">> := <<"username">>} -> - application:set_env(emqx_auth_mnesia, as, username); - #{<<"auth.mnesia.as">> := <<"clientid">>} -> - application:set_env(emqx_auth_mnesia, as, clientid); - _ -> - ok - end. - --else. -do_read_global_auth_type(Data) -> - case Data of - #{<<"auth.mnesia.as">> := <<"username">>} -> - application:set_env(emqx_auth_mnesia, as, username); - #{<<"auth.mnesia.as">> := <<"clientid">>} -> - application:set_env(emqx_auth_mnesia, as, clientid); - _ -> - logger:error("While importing data from EMQX versions prior to 4.3 " - "it is necessary to specify the value of \"auth.mnesia.as\" parameter " - "as it was configured in etc/plugins/emqx_auth_mnesia.conf.\n" - "Use the following command to import data:\n" - " $ emqx_ctl data import --env '{\"auth.mnesia.as\":\"username\"}'\n" - "or\n" - " $ emqx_ctl data import --env '{\"auth.mnesia.as\":\"clientid\"}'", - []), - error(import_failed) - end. --endif. - -get_old_type() -> - {ok, Type} = application:get_env(emqx_auth_mnesia, as), - Type. diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl deleted file mode 100644 index 7c249b66b..000000000 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl +++ /dev/null @@ -1,176 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_auth_mnesia_migration_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). - --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx_auth_mnesia/include/emqx_auth_mnesia.hrl"). - -matrix() -> - [{ImportAs, Version} || ImportAs <- [clientid, username] - , Version <- ["v4.2.10", "v4.1.5"]]. - -all() -> - [t_import_4_0, t_import_4_1, t_import_4_2]. - -groups() -> - [{username, [], cases()}, {clientid, [], cases()}]. - -cases() -> - [t_import]. - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_management, emqx_dashboard, emqx_auth_mnesia]), - application:set_env(ekka, strict_mode, true), - ekka_mnesia:start(), - emqx_mgmt_auth:mnesia(boot), - Config. - -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_dashboard, emqx_auth_mnesia]), - ekka_mnesia:ensure_stopped(). - -init_per_testcase(_, Config) -> - Config. - -end_per_testcase(_, _Config) -> - {atomic,ok} = mnesia:clear_table(emqx_acl), - {atomic,ok} = mnesia:clear_table(emqx_user), - ok. --ifdef(EMQX_ENTERPRISE). -t_import_4_0(Config) -> - Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), - ?assertMatch(ok, do_import("e4.0.10.json", Config, Overrides)), - timer:sleep(100), - ct:pal("---~p~n", [ets:tab2list(emqx_user)]), - test_import(username, {<<"emqx_username">>, <<"public">>}), - test_import(clientid, {<<"emqx_c">>, <<"public">>}), - - Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}), - ?assertMatch(ok, do_import("e4.0.10.json", Config, Overrides1)), - timer:sleep(100), - test_import(username, {<<"emqx_c">>, <<"public">>}), - test_import(username, {<<"emqx_username">>, <<"public">>}). -t_import_4_1(Config) -> - Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), - ?assertMatch(ok, do_import("e4.1.1.json", Config, Overrides)), - timer:sleep(100), - test_import(clientid, {<<"emqx_c">>, <<"public">>}), - test_import(clientid, {<<"emqx_c">>, <<"public">>}), - - Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}), - ?assertMatch(ok, do_import("e4.1.1.json", Config, Overrides1)), - timer:sleep(100), - test_import(username, {<<"emqx_c">>, <<"public">>}), - test_import(clientid, {<<"emqx_clientid">>, <<"public">>}). - -t_import_4_2(Config) -> - ?assertMatch(ok, do_import("e4.2.9.json", Config, "{}")), - timer:sleep(100), - test_import(username, {<<"emqx_c">>, <<"public">>}), - test_import(clientid, {<<"emqx_clientid">>, <<"public">>}). - --else. -t_import_4_0(Config) -> - ?assertMatch(ok, do_import("v4.0.11-no-auth.json", Config)), - timer:sleep(100), - ?assertMatch(0, ets:info(emqx_user, size)), - - ?assertMatch({error, unsupported_version, "4.0"}, do_import("v4.0.11.json", Config)), - - ?assertMatch(ok, do_import("v4.0.13.json", Config)), - timer:sleep(100), - test_import(clientid, {<<"client_for_test">>, <<"public">>}), - test_import(username, {<<"user_for_test">>, <<"public">>}). - -t_import_4_1(Config) -> - Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), - ?assertMatch(ok, do_import("v4.1.5.json", Config, Overrides)), - timer:sleep(100), - test_import(clientid, {<<"user_mnesia">>, <<"public">>}), - test_import(clientid, {<<"client_for_test">>, <<"public">>}), - test_import(username, {<<"user_for_test">>, <<"public">>}), - - Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}), - ?assertMatch(ok, do_import("v4.1.5.json", Config, Overrides1)), - timer:sleep(100), - test_import(username, {<<"user_mnesia">>, <<"public">>}), - test_import(clientid, {<<"client_for_test">>, <<"public">>}), - test_import(username, {<<"user_for_test">>, <<"public">>}). - -t_import_4_2(Config) -> - ?assertMatch(ok, do_import("v4.2.10-no-auth.json", Config)), - timer:sleep(100), - ?assertMatch(0, ets:info(emqx_user, size)), - - Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), - ?assertMatch({error, unsupported_version, "4.2"}, do_import("v4.2.10.json", Config, Overrides)), - - Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), - ?assertMatch(ok, do_import("v4.2.11.json", Config, Overrides1)), - timer:sleep(100), - test_import(clientid, {<<"user_mnesia">>, <<"public">>}), - test_import(clientid, {<<"client_for_test">>, <<"public">>}), - test_import(username, {<<"user_for_test">>, <<"public">>}), - - Overrides2 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}), - ?assertMatch(ok, do_import("v4.2.11.json", Config, Overrides2)), - timer:sleep(100), - test_import(username, {<<"user_mnesia">>, <<"public">>}), - test_import(clientid, {<<"client_for_test">>, <<"public">>}), - test_import(username, {<<"user_for_test">>, <<"public">>}), - - ?assertMatch([#emqx_acl{ - filter = {{Type,<<"emqx_c">>}, <<"Topic/A">>}, - action = pub, - access = allow - }, - #emqx_acl{ - filter = {{Type,<<"emqx_c">>}, <<"Topic/A">>}, - action = sub, - access = allow - }], - lists:sort(ets:tab2list(emqx_acl))). --endif. - -do_import(File, Config) -> - do_import(File, Config, "{}"). - -do_import(File, Config, Overrides) -> - mnesia:clear_table(emqx_acl), - mnesia:clear_table(emqx_user), - Filename = filename:join(proplists:get_value(data_dir, Config), File), - emqx_mgmt_data_backup:import(Filename, Overrides). - -test_import(username, {Username, Password}) -> - [#emqx_user{password = _}] = ets:lookup(emqx_user, {username, Username}), - Req = #{clientid => <<"anyname">>, - username => Username, - password => Password}, - ?assertMatch({stop, #{auth_result := success}}, - emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})); -test_import(clientid, {ClientID, Password}) -> - [#emqx_user{password = _}] = ets:lookup(emqx_user, {clientid, ClientID}), - Req = #{clientid => ClientID, - password => Password}, - ?assertMatch({stop, #{auth_result := success}}, - emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})). diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh deleted file mode 100755 index e31729784..000000000 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash -set -eux pipefail -# Helper script for creating data export files - -container() { - version="${1}" - if [ -z ${2+x} ]; then - ee="" - else - ee="-ee" - fi - container="emqx/emqx${ee}:${version}" - docker rm -f emqx || true - docker run "$container" true # Make sure the image is cached locally - docker run --rm -e EMQX_LOADED_PLUGINS="emqx_auth_mnesia emqx_auth_clientid emqx_management" \ - --name emqx -p 8081:8081 "$container" emqx foreground & - sleep 7 -} - -create_acls () { - url="${1}" - curl -f -v "http://localhost:8081/$url" -u admin:public -d@- < "$filename.json" - cat "${filename}.json" -} - - -collect_4_2_no_mnesia_auth () { - container "4.2.10" - - # Add clientid - docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p - - export_data "v4.2.10-no-auth" -} - -collect_4_2 () { - container "4.2.10" - create_acls "api/v4/mqtt_acl" - create_user mqtt_user - - # Add clientid - docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p - - export_data "v4.2.10" -} - - -collect_e4_2 () { - container "4.2.5" "ee" - # Add ACLs: - docker exec emqx emqx_ctl acl add username emqx_c Topic/A pubsub allow - # Create users - docker exec emqx emqx_ctl user add emqx_c emqx_p - - # Add clientid - docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p - - export_data "e4.2.5" -} - -collect_e4_1 () { - container "4.1.1" "ee" - # Add ACLs: - create_acls "api/v4/emqx_acl" - # Create users - create_user "auth_user" - - # Add clientid - docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p - - export_data "e4.1.1" -} - -collect_4_1 () { - container "v4.1.5" - create_acls "api/v4/emqx_acl" - create_user auth_user - - # Add clientid - docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p - - export_data "v4.1.5" -} - -collect_4_0 () { - container "v4.0.11" - - # Add clientid - docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p - - export_data "v4.0.11" -} - -collect_4_0 -collect_4_1 -collect_4_2 -collect_4_2_no_mnesia_auth -collect_e4_2 -collect_e4_1 - -docker kill emqx diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11-no-auth.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11-no-auth.json deleted file mode 100644 index 8dad197dd..000000000 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11-no-auth.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "version": "4.0", - "users": [], - "schemas": [], - "rules": [], - "resources": [], - "date": "2021-04-10 11:45:26", - "blacklist": [], - "auth_username": [], - "auth_mnesia": [], - "auth_clientid": [], - "apps": [ - { - "status": true, - "secret": "public", - "name": "Default", - "id": "admin", - "expired": "undefined", - "desc": "Application user" - } - ], - "acl_mnesia": [] -} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11.json deleted file mode 100644 index a701d0944..000000000 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "version": "4.0", - "users": [], - "schemas": [], - "rules": [], - "resources": [], - "date": "2021-04-10 11:45:26", - "blacklist": [], - "auth_username": [], - "auth_mnesia": [], - "auth_clientid": [ - { - "password": "9Sv2tzJlNDlmNWZhYWQ5Yzc4MWUwNmFhZWI4NjFlMDM2OWEzYmE1OTkxOTBhOGQ4N2Y3MzExY2ZiZmIxNTFkMTdkZmY=", - "clientid": "emqx_clientid" - } - ], - "apps": [ - { - "status": true, - "secret": "public", - "name": "Default", - "id": "admin", - "expired": "undefined", - "desc": "Application user" - } - ], - "acl_mnesia": [] -} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.13.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.13.json deleted file mode 100644 index 9c602a0e3..000000000 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.13.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "version":"4.0.13", - "users":[ - { - "username":"admin", - "tags":"administrator", - "password":"p6C65OF0BQhvPmCziM2yRa8JN5o=" - } - ], - "schemas":[], - "rules":[], - "resources":[], - "date":"2021-04-16 11:20:00", - "blacklist":[], - "auth_username":[ - { - "username":"user_for_test", - "password":"ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=" - } - ], - "auth_clientid":[ - { - "password":"JBgSnzIxOWNiMDU1ZWFiNDAwMjVhOTQzZThlZjkxN2JlZWE4MGE4YzlmM2I5MjQ4OGI1NjllY2Q4NGQ4NjhjYzQ1NDM=", - "clientid":"client_for_test" - } - ], - "apps":[ - { - "status":true, - "secret":"public", - "name":"Default", - "id":"admin", - "expired":"undefined", - "desc":"Application user" - } - ] -} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json deleted file mode 100644 index 3d3b9aa7e..000000000 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "version": "4.1", - "users": [ - { - "username": "admin", - "tags": "administrator", - "password": "R0TpDmJtE/d5rIXAm6YY61RI0mg=" - } - ], - "schemas": [], - "rules": [], - "resources": [], - "date": "2021-04-07 14:28:58", - "blacklist": [], - "auth_username": [ - { - "username":"user_for_test", - "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=" - } - ], - "auth_mnesia": [ - { - "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=", - "login": "user_mnesia", - "is_superuser": true - } - ], - "auth_clientid": [ - { - "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=", - "clientid": "client_for_test" - } - ], - "apps": [ - { - "status": true, - "secret": "public", - "name": "Default", - "id": "admin", - "expired": "undefined", - "desc": "Application user" - } - ], - "acl_mnesia": [ - { - "topic": "Topic/A", - "login": "emqx_c", - "allow": true, - "action": "sub" - }, - { - "topic": "Topic/A", - "login": "emqx_c", - "allow": true, - "action": "pub" - } - ] -} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json deleted file mode 100644 index 446a46f07..000000000 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "version": "4.2", - "date": "2021-04-12 10:41:10", - "rules": [], - "resources": [], - "blacklist": [], - "apps": [ - { - "id": "admin", - "secret": "public", - "name": "Default", - "desc": "Application user", - "status": true, - "expired": "undefined" - } - ], - "users": [ - { - "username": "admin", - "password": "e5M8oWEwQVqjdqceQIthC+3cPoY=", - "tags": "administrator" - } - ], - "auth_clientid": [], - "auth_username": [], - "auth_mnesia": [], - "acl_mnesia": [], - "schemas": [] -} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10.json deleted file mode 100644 index 1ccc6ce9d..000000000 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "version": "4.2", - "date": "2021-04-12 10:40:58", - "rules": [], - "resources": [], - "blacklist": [], - "apps": [ - { - "id": "admin", - "secret": "public", - "name": "Default", - "desc": "Application user", - "status": true, - "expired": "undefined" - } - ], - "users": [ - { - "username": "admin", - "password": "8Vd7+gVg2J3nE1Xjyxqd59sA5mo=", - "tags": "administrator" - } - ], - "auth_clientid": [ - { - "clientid": "emqx_clientid", - "password": "UNb0e2RhNDc3NWIyNjg5Yjg4ZDExOTVhNWFkY2MzNGFmNzY2OTNmNmRlYzE4Y2ZiZjRjNzIyMWZlZTljZmEyZDE5Yzc=" - } - ], - "auth_username": [], - "auth_mnesia": [ - { - "login": "emqx_c", - "password": "ceb5e917f7930ae8f0dc3ceb496a428f7e644736eebca36a2b8f6bbac756171a", - "is_superuser": true - } - ], - "acl_mnesia": [ - { - "login": "emqx_c", - "topic": "Topic/A", - "action": "sub", - "allow": true - }, - { - "login": "emqx_c", - "topic": "Topic/A", - "action": "pub", - "allow": true - } - ], - "schemas": [] -} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.11.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.11.json deleted file mode 100644 index 52a91ac68..000000000 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.11.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "version": "4.2.11", - "date": "2021-04-12 10:40:58", - "rules": [], - "resources": [], - "blacklist": [], - "apps": [ - { - "id": "admin", - "secret": "public", - "name": "Default", - "desc": "Application user", - "status": true, - "expired": "undefined" - } - ], - "users": [ - { - "username": "test", - "password": "8Vd7+gVg2J3nE1Xjyxqd59sA5mo=", - "tags": "administrator" - } - ], - "auth_clientid": [ - { - "clientid": "client_for_test", - "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=" - } - ], - "auth_username": [ - { - "username": "user_for_test", - "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=" - } - ], - "auth_mnesia": [ - { - "login": "user_mnesia", - "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=", - "is_superuser": true - } - ], - "acl_mnesia": [ - { - "login": "emqx_c", - "topic": "Topic/A", - "action": "sub", - "allow": true - }, - { - "login": "emqx_c", - "topic": "Topic/A", - "action": "pub", - "allow": true - } - ], - "schemas": [] -} diff --git a/rebar.config b/rebar.config index e12ca02ba..4c086c312 100644 --- a/rebar.config +++ b/rebar.config @@ -35,7 +35,6 @@ {deps, [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.6"}}} - , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.6.7"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}}