yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(management): implement the auth-mnesia import/export 

All codes picks from emqx-management:dev/e4.2.3
This commit is contained in:
JianBo He 2020-12-11 18:13:13 +08:00 committed by JianBo He
parent 826419df5f
commit 5d98944c94
5 changed files with 115 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

171
apps/emqx_management/src/emqx_mgmt.erl
View File

@ -124,21 +124,17 @@
, export_blacklist/0
, export_applications/0
, export_users/0
, export_auth_clientid/0
, export_auth_username/0
, export_auth_mnesia/0
, export_acl_mnesia/0
, export_schemas/0
, import_rules/1
, import_resources/1
, import_blacklist/1
, import_applications/1
, import_users/1
, import_auth_clientid/1
, import_auth_username/1
, import_auth_mnesia/1
, import_acl_mnesia/1
, import_schemas/1
, import_auth_clientid/1 %% BACKW: 4.1.x
, import_auth_username/1 %% BACKW: 4.1.x
, import_auth_mnesia/2
, import_acl_mnesia/2
, to_version/1
]).
@ -612,78 +608,59 @@ export_rules() ->
end, emqx_rule_registry:get_rules()).
export_resources() ->
lists:foldl(fun({_, Id, Type, Config, CreatedAt, Desc}, Acc) ->
lists:map(fun({_, Id, Type, Config, CreatedAt, Desc}) ->
NCreatedAt = case CreatedAt of
undefined -> null;
_ -> CreatedAt
end,
[[{id, Id},
[{id, Id},
{type, Type},
{config, maps:to_list(Config)},
{created_at, NCreatedAt},
{description, Desc}] | Acc]
end, [], emqx_rule_registry:get_resources()).
{description, Desc}]
end, emqx_rule_registry:get_resources()).
export_blacklist() ->
lists:foldl(fun(#banned{who = Who, by = By, reason = Reason, at = At, until = Until}, Acc) ->
lists:map(fun(#banned{who = Who, by = By, reason = Reason, at = At, until = Until}) ->
NWho = case Who of
{peerhost, Peerhost} -> {peerhost, inet:ntoa(Peerhost)};
_ -> Who
end,
[[{who, [NWho]}, {by, By}, {reason, Reason}, {at, At}, {until, Until}] | Acc]
end, [], ets:tab2list(emqx_banned)).
[{who, [NWho]}, {by, By}, {reason, Reason}, {at, At}, {until, Until}]
end, ets:tab2list(emqx_banned)).
export_applications() ->
lists:foldl(fun({_, AppID, AppSecret, Name, Desc, Status, Expired}, Acc) ->
[[{id, AppID}, {secret, AppSecret}, {name, Name}, {desc, Desc}, {status, Status}, {expired, Expired}] | Acc]
end, [], ets:tab2list(mqtt_app)).
lists:map(fun({_, AppID, AppSecret, Name, Desc, Status, Expired}) ->
[{id, AppID}, {secret, AppSecret}, {name, Name}, {desc, Desc}, {status, Status}, {expired, Expired}]
end, ets:tab2list(mqtt_app)).
export_users() ->
lists:foldl(fun({_, Username, Password, Tags}, Acc) ->
[[{username, Username}, {password, base64:encode(Password)}, {tags, Tags}] | Acc]
end, [], ets:tab2list(mqtt_admin)).
export_auth_clientid() ->
case ets:info(emqx_auth_clientid) of
undefined -> [];
_ ->
lists:foldl(fun({_, ClientId, Password}, Acc) ->
[[{clientid, ClientId}, {password, Password}] | Acc]
end, [], ets:tab2list(emqx_auth_clientid))
end.
export_auth_username() ->
case ets:info(emqx_auth_username) of
undefined -> [];
_ ->
lists:foldl(fun({_, Username, Password}, Acc) ->
[[{username, Username}, {password, Password}] | Acc]
end, [], ets:tab2list(emqx_auth_username))
end.
lists:map(fun({_, Username, Password, Tags}) ->
[{username, Username}, {password, base64:encode(Password)}, {tags, Tags}]
end, ets:tab2list(mqtt_admin)).
export_auth_mnesia() ->
case ets:info(emqx_user) of
undefined -> [];
_ ->
lists:foldl(fun({_, Login, Password, IsSuperuser}, Acc) ->
[[{login, Login}, {password, Password}, {is_superuser, IsSuperuser}] | Acc]
end, [], ets:tab2list(emqx_user))
lists:map(fun({_, {Type, Login}, Password, CreatedAt}) ->
[{login, Login}, {type, Type}, {password, base64:encode(Password)}, {created_at, CreatedAt}]
end, ets:tab2list(emqx_user))
end.
export_acl_mnesia() ->
case ets:info(emqx_acl) of
undefined -> [];
_ ->
lists:foldl(fun({_, Login, Topic, Action, Allow}, Acc) ->
[[{login, Login}, {topic, Topic}, {action, Action}, {allow, Allow}] | Acc]
end, [], ets:tab2list(emqx_acl))
end.
export_schemas() ->
case ets:info(emqx_schema) of
undefined -> [];
_ ->
[emqx_schema_api:format_schema(Schema) || Schema <- emqx_schema_registry:get_all_schemas()]
lists:map(fun({_, Filter, Action, Access, CreatedAt}) ->
Filter1 = case Filter of
{{Type, TypeValue}, Topic} ->
[{type, Type}, {type_value, TypeValue}, {topic, Topic}];
{Type, Topic} ->
[{type, Type}, {topic, Topic}]
end,
Filter1 ++ [{action, Action}, {access, Access}, {created_at, CreatedAt}]
end, ets:tab2list(emqx_acl))
end.
import_rules(Rules) ->
@ -761,44 +738,80 @@ import_users(Users) ->
end, Users).
import_auth_clientid(Lists) ->
case ets:info(emqx_auth_clientid) of
undefined -> ok;
_ ->
[ mnesia:dirty_write({emqx_auth_clientid, ClientId, Password}) || #{<<"clientid">> := ClientId,
<<"password">> := Password} <- Lists ]
end.
import_auth_username(Lists) ->
case ets:info(emqx_auth_username) of
undefined -> ok;
_ ->
[ mnesia:dirty_write({emqx_auth_username, Username, Password}) || #{<<"username">> := Username,
<<"password">> := Password} <- Lists ]
end.
import_auth_mnesia(Auths) ->
case ets:info(emqx_user) of
undefined -> ok;
_ ->
[ mnesia:dirty_write({emqx_user, Login, Password, IsSuperuser}) || #{<<"login">> := Login,
<<"password">> := Password,
<<"is_superuser">> := IsSuperuser} <- Auths ]
[ mnesia:dirty_write({emqx_user, {clientid, Clientid}, base64:decode(Password), erlang:system_time(millisecond)})
|| #{<<"clientid">> := Clientid, <<"password">> := Password} <- Lists ]
end.
import_acl_mnesia(Acls) ->
import_auth_username(Lists) ->
case ets:info(emqx_user) of
undefined -> ok;
_ ->
[ mnesia:dirty_write({emqx_user, {username, Username}, base64:decode(Password), erlang:system_time(millisecond)})
|| #{<<"username">> := Username, <<"password">> := Password} <- Lists ]
end.
import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse
FromVersion =:= "4.1" ->
case ets:info(emqx_user) of
undefined -> ok;
_ ->
CreatedAt = erlang:system_time(millisecond),
[ begin
mnesia:dirty_write({emqx_user, {username, Login}, base64:decode(Password), CreatedAt})
end
|| #{<<"login">> := Login,
<<"password">> := Password} <- Auths ]
end;
import_auth_mnesia(Auths, _) ->
case ets:info(emqx_user) of
undefined -> ok;
_ ->
[ mnesia:dirty_write({emqx_user, {any_to_atom(Type), Login}, base64:decode(Password), CreatedAt})
|| #{<<"login">> := Login,
<<"type">> := Type,
<<"password">> := Password,
<<"created_at">> := CreatedAt } <- Auths ]
end.
import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse
FromVersion =:= "4.1" ->
case ets:info(emqx_acl) of
undefined -> ok;
_ ->
[ mnesia:dirty_write({emqx_acl ,Login, Topic, Action, Allow}) || #{<<"login">> := Login,
CreatedAt = erlang:system_time(millisecond),
[begin
Allow1 = case any_to_atom(Allow) of
true -> allow;
false -> deny
end,
mnesia:dirty_write({emqx_acl, {{username, Login}, Topic}, any_to_atom(Action), Allow1, CreatedAt})
end || #{<<"login">> := Login,
<<"topic">> := Topic,
<<"action">> := Action,
<<"allow">> := Allow} <- Acls ]
end.
<<"allow">> := Allow,
<<"action">> := Action} <- Acls]
end;
import_schemas(Schemas) ->
case ets:info(emqx_schema) of
import_acl_mnesia(Acls, _) ->
case ets:info(emqx_acl) of
undefined -> ok;
_ -> [emqx_schema_registry:add_schema(emqx_schema_api:make_schema_params(Schema)) || Schema <- Schemas]
_ ->
[ begin
Filter = case maps:get(<<"type_value">>, Map, undefined) of
undefined ->
{any_to_atom(maps:get(<<"type">>, Map)), maps:get(<<"topic">>, Map)};
Value ->
{{any_to_atom(maps:get(<<"type">>, Map)), Value}, maps:get(<<"topic">>, Map)}
end,
mnesia:dirty_write({emqx_acl ,Filter, any_to_atom(Action), any_to_atom(Access), CreatedAt})
end
|| Map = #{<<"action">> := Action,
<<"access">> := Access,
<<"created_at">> := CreatedAt} <- Acls ]
end.
any_to_atom(L) when is_list(L) -> list_to_atom(L);

6
apps/emqx_management/src/emqx_mgmt_api_apps.erl
View File

@ -70,13 +70,13 @@ add_app(_Bindings, Params) ->
Expired = get_value(<<"expired">>, Params),
case emqx_mgmt_auth:add_app(AppId, Name, Secret, Desc, Status, Expired) of
{ok, AppSecret} -> return({ok, #{secret => AppSecret}});
{error, Reason} -> return({error, ?ERROR2, Reason})
{error, Reason} -> return({error, Reason})
end.
del_app(#{appid := AppId}, _Params) ->
case emqx_mgmt_auth:del_app(AppId) of
ok -> return();
{error, Reason} -> return({error, ?ERROR2, Reason})
{error, Reason} -> return({error, Reason})
end.
list_apps(_Bindings, _Params) ->
@ -102,7 +102,7 @@ update_app(#{appid := AppId}, Params) ->
Expired = get_value(<<"expired">>, Params),
case emqx_mgmt_auth:update_app(AppId, Name, Desc, Status, Expired) of
ok -> return();
{error, Reason} -> return({error, ?ERROR2, Reason})
{error, Reason} -> return({error, Reason})
end.
format({AppId, _AppSecret, Name, Desc, Status, Expired}) ->

14
apps/emqx_management/src/emqx_mgmt_api_data.erl
View File

@ -80,11 +80,8 @@ export(_Bindings, _Params) ->
Blacklist = emqx_mgmt:export_blacklist(),
Apps = emqx_mgmt:export_applications(),
Users = emqx_mgmt:export_users(),
AuthClientid = emqx_mgmt:export_auth_clientid(),
AuthUsername = emqx_mgmt:export_auth_username(),
AuthMnesia = emqx_mgmt:export_auth_mnesia(),
AclMnesia = emqx_mgmt:export_acl_mnesia(),
Schemas = emqx_mgmt:export_schemas(),
Seconds = erlang:system_time(second),
{{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds),
Filename = io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]),
@ -97,11 +94,8 @@ export(_Bindings, _Params) ->
{blacklist, Blacklist},
{apps, Apps},
{users, Users},
{auth_clientid, AuthClientid},
{auth_username, AuthUsername},
{auth_mnesia, AuthMnesia},
{acl_mnesia, AclMnesia},
{schemas, Schemas}
{acl_mnesia, AclMnesia}
],
Bin = emqx_json:encode(Data),
@ -180,18 +174,18 @@ do_import(Filename) ->
case lists:member(Version, ?VERSIONS) of
true ->
try
emqx_mgmt:import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])),
%emqx_mgmt:import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])),
emqx_mgmt:import_resources(maps:get(<<"resources">>, Data, [])),
emqx_mgmt:import_rules(maps:get(<<"rules">>, Data, [])),
emqx_mgmt:import_blacklist(maps:get(<<"blacklist">>, Data, [])),
emqx_mgmt:import_applications(maps:get(<<"apps">>, Data, [])),
emqx_mgmt:import_users(maps:get(<<"users">>, Data, [])),
emqx_mgmt:import_modules(maps:get(<<"modules">>, Data, [])),
%emqx_mgmt:import_modules(maps:get(<<"modules">>, Data, [])),
_ = emqx_mgmt:import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])),
_ = emqx_mgmt:import_auth_username(maps:get(<<"auth_username">>, Data, [])),
_ = emqx_mgmt:import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version),
_ = emqx_mgmt:import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version),
_ = emqx_mgmt:import_schemas(maps:get(<<"schemas">>, Data, [])),
%_ = emqx_mgmt:import_schemas(maps:get(<<"schemas">>, Data, [])),
logger:debug("The emqx data has been imported successfully"),
ok
catch Class:Reason:Stack ->

13
apps/emqx_management/src/emqx_mgmt_cli.erl
View File

@ -563,11 +563,8 @@ data(["export"]) ->
Blacklist = emqx_mgmt:export_blacklist(),
Apps = emqx_mgmt:export_applications(),
Users = emqx_mgmt:export_users(),
AuthClientID = emqx_mgmt:export_auth_clientid(),
AuthUsername = emqx_mgmt:export_auth_username(),
AuthMnesia = emqx_mgmt:export_auth_mnesia(),
AclMnesia = emqx_mgmt:export_acl_mnesia(),
Schemas = emqx_mgmt:export_schemas(),
Seconds = erlang:system_time(second),
{{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds),
Filename = io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]),
@ -580,11 +577,8 @@ data(["export"]) ->
{blacklist, Blacklist},
{apps, Apps},
{users, Users},
{auth_clientid, AuthClientID},
{auth_username, AuthUsername},
{auth_mnesia, AuthMnesia},
{acl_mnesia, AclMnesia},
{schemas, Schemas}],
{acl_mnesia, AclMnesia}],
ok = filelib:ensure_dir(NFilename),
case file:write_file(NFilename, emqx_json:encode(Data)) of
ok ->
@ -608,9 +602,8 @@ data(["import", Filename]) ->
emqx_mgmt:import_users(maps:get(<<"users">>, Data, [])),
_ = emqx_mgmt:import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])),
_ = emqx_mgmt:import_auth_username(maps:get(<<"auth_username">>, Data, [])),
_ = emqx_mgmt:import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, [])),
_ = emqx_mgmt:import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])),
_ = emqx_mgmt:import_schemas(maps:get(<<"schemas">>, Data, [])),
_ = emqx_mgmt:import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version),
_ = emqx_mgmt:import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version),
emqx_ctl:print("The emqx data has been imported successfully.~n")
catch Class:Reason:Stack ->
emqx_ctl:print("The emqx data import failed due: ~0p~n", [{Class,Reason,Stack}])

10
apps/emqx_management/src/emqx_mgmt_http.erl
View File

@ -73,7 +73,8 @@ ranch_opts(Port, Options0) ->
socket_opts => [{port, Port} | Options]},
Res.
stop_listener({Proto, _Port, _}) ->
stop_listener({Proto, Port, _}) ->
io:format("Stop http:management listener on ~s successfully.~n",[format(Port)]),
minirest:stop_http(listener_name(Proto)).
listeners() ->
@ -123,3 +124,10 @@ filter(#{app := App}) ->
false -> false;
Plugin -> Plugin#plugin.active
end.
format(Port) when is_integer(Port) ->
io_lib:format("0.0.0.0:~w", [Port]);
format({Addr, Port}) when is_list(Addr) ->
io_lib:format("~s:~w", [Addr, Port]);
format({Addr, Port}) when is_tuple(Addr) ->
io_lib:format("~s:~w", [inet:ntoa(Addr), Port]).