yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #11766 from lafirest/feat/rbac 

feat(api_key): add RBAC feature for the API key
This commit is contained in:
lafirest 2023-10-20 09:24:41 +08:00 committed by GitHub
parent 5c01b3dac1 467a2bc9b1
commit 74442b0d31
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 552 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
apps/emqx/src/bhvrs/emqx_db_backup.erl
View File

@ -16,4 +16,21 @@
-module(emqx_db_backup).
-type traverse_break_reason() :: over | migrate.
-callback backup_tables() -> [mria:table()].
%% validate the backup
%% return `ok` to traverse the next item
%% return `{ok, over}` to finish the traverse
%% return `{ok, migrate}` to call the migration callback
-callback validate_mnesia_backup(tuple()) ->
ok
| {ok, traverse_break_reason()}
| {error, term()}.
-callback migrate_mnesia_backup(tuple()) -> {ok, tuple()} | {error, term()}.
-optional_callbacks([validate_mnesia_backup/1, migrate_mnesia_backup/1]).
-export_type([traverse_break_reason/0]).

10
apps/emqx/test/emqx_common_test_http.erl
View File

@ -33,6 +33,9 @@
-define(DEFAULT_APP_ID, <<"default_appid">>).
-define(DEFAULT_APP_SECRET, <<"default_app_secret">>).
%% from emqx_dashboard/include/emqx_dashboard_rbac.hrl
-define(ROLE_API_SUPERUSER, <<"api_administrator">>).
request_api(Method, Url, Auth) ->
request_api(Method, Url, [], Auth, []).
@ -90,7 +93,12 @@ create_default_app() ->
Now = erlang:system_time(second),
ExpiredAt = Now + timer:minutes(10),
emqx_mgmt_auth:create(
?DEFAULT_APP_ID, ?DEFAULT_APP_SECRET, true, ExpiredAt, <<"default app key for test">>
?DEFAULT_APP_ID,
?DEFAULT_APP_SECRET,
true,
ExpiredAt,
<<"default app key for test">>,
?ROLE_API_SUPERUSER
).
delete_default_app() ->

11
apps/emqx_dashboard/include/emqx_dashboard.hrl
View File

@ -13,16 +13,9 @@
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-define(ADMIN, emqx_admin).
-include("emqx_dashboard_rbac.hrl").
%% TODO:
%% The predefined roles of the preliminary RBAC implementation,
%% these may be removed when developing the full RBAC feature.
%% In full RBAC feature, the role may be customised created and deleted,
%% a predefined configuration would replace these macros.
-define(ROLE_VIEWER, <<"viewer">>).
-define(ROLE_SUPERUSER, <<"administrator">>).
-define(ROLE_DEFAULT, ?ROLE_SUPERUSER).
-define(ADMIN, emqx_admin).
-define(BACKEND_LOCAL, local).
-define(SSO_USERNAME(Backend, Name), {Backend, Name}).

33
apps/emqx_dashboard/include/emqx_dashboard_rbac.hrl Normal file
View File

@ -0,0 +1,33 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-ifndef(EMQX_DASHBOARD_RBAC).
-define(EMQX_DASHBOARD_RBAC, true).
%% TODO:
%% The predefined roles of the preliminary RBAC implementation,
%% these may be removed when developing the full RBAC feature.
%% In full RBAC feature, the role may be customised created and deleted,
%% a predefined configuration would replace these macros.
-define(ROLE_VIEWER, <<"viewer">>).
-define(ROLE_SUPERUSER, <<"administrator">>).
-define(ROLE_DEFAULT, ?ROLE_SUPERUSER).
-define(ROLE_API_VIEWER, <<"api_viewer">>).
-define(ROLE_API_SUPERUSER, <<"api_administrator">>).
-define(ROLE_API_PUBLISHER, <<"api_publisher">>).
-define(ROLE_API_DEFAULT, ?ROLE_API_SUPERUSER).
-endif.

7
apps/emqx_dashboard/src/emqx_dashboard.erl
View File

@ -212,7 +212,7 @@ listener_name(Protocol) ->
-if(?EMQX_RELEASE_EDITION =/= ee).
%% dialyzer complains about the `unauthorized_role' clause...
-dialyzer({no_match, [authorize/1]}).
-dialyzer({no_match, [authorize/1, api_key_authorize/3]}).
-endif.
authorize(Req) ->
@ -251,7 +251,7 @@ listeners() ->
api_key_authorize(Req, Key, Secret) ->
Path = cowboy_req:path(Req),
case emqx_mgmt_auth:authorize(Path, Key, Secret) of
case emqx_mgmt_auth:authorize(Path, Req, Key, Secret) of
ok ->
{ok, #{auth_type => api_key, api_key => Key}};
{error, <<"not_allowed">>} ->
@ -259,6 +259,9 @@ api_key_authorize(Req, Key, Secret) ->
?BAD_API_KEY_OR_SECRET,
<<"Not allowed, Check api_key/api_secret">>
);
{error, unauthorized_role} ->
{403, 'UNAUTHORIZED_ROLE',
<<"This API Key don't have permission to access this resource">>};
{error, _} ->
return_unauthorized(
?BAD_API_KEY_OR_SECRET,

6
apps/emqx_dashboard/src/emqx_dashboard_admin.erl
View File

@ -423,7 +423,7 @@ ensure_role(Role) when is_binary(Role) ->
-if(?EMQX_RELEASE_EDITION == ee).
legal_role(Role) ->
emqx_dashboard_rbac:valid_role(Role).
emqx_dashboard_rbac:valid_dashboard_role(Role).
role(Data) ->
emqx_dashboard_rbac:role(Data).
@ -454,8 +454,10 @@ lookup_user(Backend, Username) when is_atom(Backend) ->
-dialyzer({no_match, [add_user/4, update_user/3]}).
legal_role(?ROLE_DEFAULT) ->
ok;
legal_role(_) ->
ok.
{error, <<"Role does not exist">>}.
role(_) ->
?ROLE_DEFAULT.

2
apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_dashboard_rbac, [
{description, "EMQX Dashboard RBAC"},
{vsn, "0.1.0"},
{vsn, "0.1.1"},
{registered, []},
{applications, [
kernel,

54
apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl
View File

@ -6,7 +6,12 @@
-include_lib("emqx_dashboard/include/emqx_dashboard.hrl").
-export([check_rbac/2, role/1, valid_role/1]).
-export([
check_rbac/2,
role/1,
valid_dashboard_role/1,
valid_api_role/1
]).
-dialyzer({nowarn_function, role/1}).
%%=====================================================================
@ -31,25 +36,44 @@ role(#?ADMIN{role = Role}) ->
role([]) ->
?ROLE_SUPERUSER;
role(#{role := Role}) ->
Role;
role(Role) when is_binary(Role) ->
Role.
valid_role(Role) ->
case lists:member(Role, role_list()) of
valid_dashboard_role(Role) ->
valid_role(dashboard, Role).
valid_api_role(Role) ->
valid_role(api, Role).
%% ===================================================================
check_rbac(?ROLE_SUPERUSER, _, _) ->
true;
check_rbac(?ROLE_API_SUPERUSER, _, _) ->
true;
check_rbac(?ROLE_VIEWER, <<"GET">>, _) ->
true;
check_rbac(?ROLE_API_VIEWER, <<"GET">>, _) ->
true;
%% this API is a special case
check_rbac(?ROLE_VIEWER, <<"POST">>, <<"/logout">>) ->
true;
check_rbac(?ROLE_API_PUBLISHER, <<"POST">>, <<"/publish">>) ->
true;
check_rbac(?ROLE_API_PUBLISHER, <<"POST">>, <<"/publish/bulk">>) ->
true;
check_rbac(_, _, _) ->
false.
valid_role(Type, Role) ->
case lists:member(Role, role_list(Type)) of
true ->
ok;
_ ->
{error, <<"Role does not exist">>}
end.
%% ===================================================================
check_rbac(?ROLE_SUPERUSER, _, _) ->
true;
check_rbac(?ROLE_VIEWER, <<"GET">>, _) ->
true;
%% this API is a special case
check_rbac(?ROLE_VIEWER, <<"POST">>, <<"/logout">>) ->
true;
check_rbac(_, _, _) ->
false.
role_list() ->
[?ROLE_VIEWER, ?ROLE_SUPERUSER].
role_list(dashboard) ->
[?ROLE_VIEWER, ?ROLE_SUPERUSER];
role_list(api) ->
[?ROLE_API_VIEWER, ?ROLE_API_PUBLISHER, ?ROLE_API_SUPERUSER].

40
apps/emqx_management/src/emqx_mgmt_api_api_keys.erl
View File

@ -19,6 +19,7 @@
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("emqx_dashboard/include/emqx_dashboard_rbac.hrl").
-export([api_spec/0, fields/1, paths/0, schema/1, namespace/0]).
-export([api_key/2, api_key_by_name/2]).
@ -150,7 +151,7 @@ fields(app) ->
)},
{enable, hoconsc:mk(boolean(), #{desc => "Enable/Disable", required => false})},
{expired, hoconsc:mk(boolean(), #{desc => "Expired", required => false})}
];
] ++ app_extend_fields();
fields(name) ->
[
{name,
@ -192,7 +193,8 @@ api_key(post, #{body := App}) ->
} = App,
ExpiredAt = ensure_expired_at(App),
Desc = unicode:characters_to_binary(Desc0, unicode),
case emqx_mgmt_auth:create(Name, Enable, ExpiredAt, Desc) of
Role = maps:get(<<"role">>, App, ?ROLE_API_DEFAULT),
case emqx_mgmt_auth:create(Name, Enable, ExpiredAt, Desc, Role) of
{ok, NewApp} ->
{200, emqx_mgmt_auth:format(NewApp)};
{error, Reason} ->
@ -218,10 +220,38 @@ api_key_by_name(put, #{bindings := #{name := Name}, body := Body}) ->
Enable = maps:get(<<"enable">>, Body, undefined),
ExpiredAt = ensure_expired_at(Body),
Desc = maps:get(<<"desc">>, Body, undefined),
case emqx_mgmt_auth:update(Name, Enable, ExpiredAt, Desc) of
{ok, App} -> {200, emqx_mgmt_auth:format(App)};
{error, not_found} -> {404, ?NOT_FOUND_RESPONSE}
Role = maps:get(<<"role">>, Body, ?ROLE_API_DEFAULT),
case emqx_mgmt_auth:update(Name, Enable, ExpiredAt, Desc, Role) of
{ok, App} ->
{200, emqx_mgmt_auth:format(App)};
{error, not_found} ->
{404, ?NOT_FOUND_RESPONSE};
{error, Reason} ->
{400, #{
code => 'BAD_REQUEST',
message => iolist_to_binary(io_lib:format("~p", [Reason]))
}}
end.
ensure_expired_at(#{<<"expired_at">> := ExpiredAt}) when is_integer(ExpiredAt) -> ExpiredAt;
ensure_expired_at(_) -> infinity.
-if(?EMQX_RELEASE_EDITION == ee).
app_extend_fields() ->
[
{role,
hoconsc:mk(binary(), #{
desc => ?DESC(role),
default => ?ROLE_API_DEFAULT,
example => ?ROLE_API_DEFAULT,
validator => fun emqx_dashboard_rbac:valid_api_role/1
})}
].
-else.
app_extend_fields() ->
[].
-endif.

195
apps/emqx_management/src/emqx_mgmt_auth.erl
View File

@ -16,6 +16,7 @@
-module(emqx_mgmt_auth).
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx_dashboard/include/emqx_dashboard_rbac.hrl").
-behaviour(emqx_db_backup).
@ -25,33 +26,34 @@
-behaviour(emqx_config_handler).
-export([
create/4,
create/5,
read/1,
update/4,
update/5,
delete/1,
list/0,
init_bootstrap_file/0,
format/1
]).
-export([authorize/3]).
-export([authorize/4]).
-export([post_config_update/5]).
-export([backup_tables/0]).
-export([backup_tables/0, validate_mnesia_backup/1, migrate_mnesia_backup/1]).
%% Internal exports (RPC)
-export([
do_update/4,
do_update/5,
do_delete/1,
do_create_app/3,
do_force_create_app/3
]).
-ifdef(TEST).
-export([create/5]).
-export([create/6]).
-endif.
-define(APP, emqx_app).
-type api_user_role() :: binary().
-record(?APP, {
name = <<>> :: binary() | '_',
@ -60,17 +62,21 @@
enable = true :: boolean() | '_',
desc = <<>> :: binary() | '_',
expired_at = 0 :: integer() | undefined | infinity | '_',
created_at = 0 :: integer() | '_'
created_at = 0 :: integer() | '_',
role = ?ROLE_DEFAULT :: api_user_role() | '_',
extra = #{} :: map() | '_'
}).
mnesia(boot) ->
Fields = record_info(fields, ?APP),
ok = mria:create_table(?APP, [
{type, set},
{rlog_shard, ?COMMON_SHARD},
{storage, disc_copies},
{record_name, ?APP},
{attributes, record_info(fields, ?APP)}
]).
{attributes, Fields}
]),
maybe_migrate_table(Fields).
%%--------------------------------------------------------------------
%% Data backup
@ -78,6 +84,35 @@ mnesia(boot) ->
backup_tables() -> [?APP].
validate_mnesia_backup({schema, _Tab, CreateList} = Schema) ->
case emqx_mgmt_data_backup:default_validate_mnesia_backup(Schema) of
ok ->
{ok, over};
_ ->
case proplists:get_value(attributes, CreateList) of
[name, api_key, api_secret_hash, enable, desc, expired_at, created_at] ->
{ok, migrate};
Fields ->
{error, {unknow_fields, Fields}}
end
end;
validate_mnesia_backup(_Other) ->
ok.
migrate_mnesia_backup({schema, Tab, CreateList}) ->
case proplists:get_value(attributes, CreateList) of
[name, api_key, api_secret_hash, enable, desc, expired_at, created_at] = Fields ->
NewFields = Fields ++ [role, extra],
CreateList2 = lists:keyreplace(
attributes, 1, CreateList, {attributes, NewFields}
),
{ok, {schema, Tab, CreateList2}};
Fields ->
{error, {unknow_fields, Fields}}
end;
migrate_mnesia_backup(Data) ->
{ok, do_table_migrate(Data)}.
post_config_update([api_key], _Req, NewConf, _OldConf, _AppEnvs) ->
#{bootstrap_file := File} = NewConf,
case init_bootstrap_file(File) of
@ -95,13 +130,13 @@ init_bootstrap_file() ->
?SLOG(debug, #{msg => "init_bootstrap_api_keys_from_file", file => File}),
init_bootstrap_file(File).
create(Name, Enable, ExpiredAt, Desc) ->
create(Name, Enable, ExpiredAt, Desc, Role) ->
ApiSecret = generate_api_secret(),
create(Name, ApiSecret, Enable, ExpiredAt, Desc).
create(Name, ApiSecret, Enable, ExpiredAt, Desc, Role).
create(Name, ApiSecret, Enable, ExpiredAt, Desc) ->
create(Name, ApiSecret, Enable, ExpiredAt, Desc, Role) ->
case mnesia:table_info(?APP, size) < 100 of
true -> create_app(Name, ApiSecret, Enable, ExpiredAt, Desc);
true -> create_app(Name, ApiSecret, Enable, ExpiredAt, Desc, Role);
false -> {error, "Maximum ApiKey"}
end.
@ -111,10 +146,15 @@ read(Name) ->
[] -> {error, not_found}
end.
update(Name, Enable, ExpiredAt, Desc) ->
trans(fun ?MODULE:do_update/4, [Name, Enable, ExpiredAt, Desc]).
update(Name, Enable, ExpiredAt, Desc, Role) ->
case valid_role(Role) of
ok ->
trans(fun ?MODULE:do_update/5, [Name, Enable, ExpiredAt, Desc, Role]);
Error ->
Error
end.
do_update(Name, Enable, ExpiredAt, Desc) ->
do_update(Name, Enable, ExpiredAt, Desc, Role) ->
case mnesia:read(?APP, Name, write) of
[] ->
mnesia:abort(not_found);
@ -123,7 +163,8 @@ do_update(Name, Enable, ExpiredAt, Desc) ->
App0#?APP{
expired_at = ExpiredAt,
enable = ensure_not_undefined(Enable, Enable0),
desc = ensure_not_undefined(Desc, Desc0)
desc = ensure_not_undefined(Desc, Desc0),
role = Role
},
ok = mnesia:write(App),
to_map(App)
@ -138,37 +179,37 @@ do_delete(Name) ->
[_App] -> mnesia:delete({?APP, Name})
end.
format(App = #{expired_at := ExpiredAt0, created_at := CreateAt}) ->
ExpiredAt =
case ExpiredAt0 of
infinity -> <<"infinity">>;
_ -> emqx_utils_calendar:epoch_to_rfc3339(ExpiredAt0, second)
end,
App#{
expired_at => ExpiredAt,
created_at => emqx_utils_calendar:epoch_to_rfc3339(CreateAt, second)
}.
format(App = #{expired_at := ExpiredAt, created_at := CreateAt}) ->
format_app_extend(App#{
expired_at => format_epoch(ExpiredAt),
created_at => format_epoch(CreateAt)
}).
format_epoch(infinity) ->
<<"infinity">>;
format_epoch(Epoch) ->
emqx_utils_calendar:epoch_to_rfc3339(Epoch, second).
list() ->
to_map(ets:match_object(?APP, #?APP{_ = '_'})).
authorize(<<"/api/v5/users", _/binary>>, _ApiKey, _ApiSecret) ->
authorize(<<"/api/v5/users", _/binary>>, _Req, _ApiKey, _ApiSecret) ->
{error, <<"not_allowed">>};
authorize(<<"/api/v5/api_key", _/binary>>, _ApiKey, _ApiSecret) ->
authorize(<<"/api/v5/api_key", _/binary>>, _Req, _ApiKey, _ApiSecret) ->
{error, <<"not_allowed">>};
authorize(<<"/api/v5/logout", _/binary>>, _ApiKey, _ApiSecret) ->
authorize(<<"/api/v5/logout", _/binary>>, _Req, _ApiKey, _ApiSecret) ->
{error, <<"not_allowed">>};
authorize(_Path, ApiKey, ApiSecret) ->
authorize(_Path, Req, ApiKey, ApiSecret) ->
Now = erlang:system_time(second),
case find_by_api_key(ApiKey) of
{ok, true, ExpiredAt, SecretHash} when ExpiredAt >= Now ->
{ok, true, ExpiredAt, SecretHash, Role} when ExpiredAt >= Now ->
case emqx_dashboard_admin:verify_hash(ApiSecret, SecretHash) of
ok -> ok;
ok -> check_rbac(Req, Role);
error -> {error, "secret_error"}
end;
{ok, true, _ExpiredAt, _SecretHash} ->
{ok, true, _ExpiredAt, _SecretHash, _Role} ->
{error, "secret_expired"};
{ok, false, _ExpiredAt, _SecretHash} ->
{ok, false, _ExpiredAt, _SecretHash, _Role} ->
{error, "secret_disable"};
{error, Reason} ->
{error, Reason}
@ -177,8 +218,12 @@ authorize(_Path, ApiKey, ApiSecret) ->
find_by_api_key(ApiKey) ->
Fun = fun() -> mnesia:match_object(#?APP{api_key = ApiKey, _ = '_'}) end,
case mria:ro_transaction(?COMMON_SHARD, Fun) of
{atomic, [#?APP{api_secret_hash = SecretHash, enable = Enable, expired_at = ExpiredAt}]} ->
{ok, Enable, ExpiredAt, SecretHash};
{atomic, [
#?APP{
api_secret_hash = SecretHash, enable = Enable, expired_at = ExpiredAt, role = Role
}
]} ->
{ok, Enable, ExpiredAt, SecretHash, Role};
_ ->
{error, "not_found"}
end.
@ -188,7 +233,9 @@ ensure_not_undefined(New, _Old) -> New.
to_map(Apps) when is_list(Apps) ->
[to_map(App) || App <- Apps];
to_map(#?APP{name = N, api_key = K, enable = E, expired_at = ET, created_at = CT, desc = D}) ->
to_map(#?APP{
name = N, api_key = K, enable = E, expired_at = ET, created_at = CT, desc = D, role = Role
}) ->
#{
name => N,
api_key => K,
@ -196,13 +243,14 @@ to_map(#?APP{name = N, api_key = K, enable = E, expired_at = ET, created_at = CT
expired_at => ET,
created_at => CT,
desc => D,
expired => is_expired(ET)
expired => is_expired(ET),
role => Role
}.
is_expired(undefined) -> false;
is_expired(ExpiredTime) -> ExpiredTime < erlang:system_time(second).
create_app(Name, ApiSecret, Enable, ExpiredAt, Desc) ->
create_app(Name, ApiSecret, Enable, ExpiredAt, Desc, Role) ->
App =
#?APP{
name = Name,
@ -211,7 +259,8 @@ create_app(Name, ApiSecret, Enable, ExpiredAt, Desc) ->
desc = Desc,
created_at = erlang:system_time(second),
api_secret_hash = emqx_dashboard_admin:hash(ApiSecret),
api_key = list_to_binary(emqx_utils:gen_id(16))
api_key = list_to_binary(emqx_utils:gen_id(16)),
role = Role
},
case create_app(App) of
{ok, Res} ->
@ -220,8 +269,13 @@ create_app(Name, ApiSecret, Enable, ExpiredAt, Desc) ->
Error
end.
create_app(App = #?APP{api_key = ApiKey, name = Name}) ->
trans(fun ?MODULE:do_create_app/3, [App, ApiKey, Name]).
create_app(App = #?APP{api_key = ApiKey, name = Name, role = Role}) ->
case valid_role(Role) of
ok ->
trans(fun ?MODULE:do_create_app/3, [App, ApiKey, Name]);
Error ->
Error
end.
force_create_app(NamePrefix, App = #?APP{api_key = ApiKey}) ->
trans(fun ?MODULE:do_force_create_app/3, [App, ApiKey, NamePrefix]).
@ -340,3 +394,58 @@ add_bootstrap_file(File, Dev, MP, Line) ->
{error, Reason} ->
throw(#{file => File, line => Line, reason => Reason})
end.
-if(?EMQX_RELEASE_EDITION == ee).
check_rbac(Req, Role) ->
case emqx_dashboard_rbac:check_rbac(Req, Role) of
true ->
ok;
_ ->
{error, unauthorized_role}
end.
format_app_extend(App) ->
App.
valid_role(Role) ->
emqx_dashboard_rbac:valid_api_role(Role).
-else.
check_rbac(_Req, _Role) ->
ok.
format_app_extend(App) ->
maps:remove(role, App).
valid_role(?ROLE_API_DEFAULT) ->
ok;
valid_role(_) ->
{error, <<"Role does not exist">>}.
-endif.
maybe_migrate_table(Fields) ->
case mnesia:table_info(?APP, attributes) =:= Fields of
true ->
ok;
false ->
TransFun = fun do_table_migrate/1,
{atomic, ok} = mnesia:transform_table(?APP, TransFun, Fields, ?APP),
ok
end.
do_table_migrate({?APP, Name, Key, Hash, Enable, Desc, ExpiredAt, CreatedAt}) ->
#?APP{
name = Name,
api_key = Key,
api_secret_hash = Hash,
enable = Enable,
desc = Desc,
expired_at = ExpiredAt,
created_at = CreatedAt,
role = ?ROLE_API_DEFAULT,
extra = #{}
};
do_table_migrate(#?APP{} = App) ->
App.

116
apps/emqx_management/src/emqx_mgmt_data_backup.erl
View File

@ -24,6 +24,8 @@
format_error/1
]).
-export([default_validate_mnesia_backup/1]).
-ifdef(TEST).
-compile(export_all).
-compile(nowarn_export_all).
@ -223,7 +225,15 @@ export_cluster_hocon(TarDescriptor, BackupBaseName, Opts) ->
export_mnesia_tabs(TarDescriptor, BackupName, BackupBaseName, Opts) ->
maybe_print("Exporting built-in database...~n", [], Opts),
lists:foreach(
fun(Tab) -> export_mnesia_tab(TarDescriptor, Tab, BackupName, BackupBaseName, Opts) end,
fun(Mod) ->
Tabs = Mod:backup_tables(),
lists:foreach(
fun(Tab) ->
export_mnesia_tab(TarDescriptor, Tab, BackupName, BackupBaseName, Opts)
end,
Tabs
)
end,
tabs_to_backup()
).
@ -259,7 +269,7 @@ tabs_to_backup() ->
-endif.
mnesia_tabs_to_backup() ->
lists:flatten([M:backup_tables() || M <- find_behaviours(emqx_db_backup)]).
lists:flatten([M || M <- find_behaviours(emqx_db_backup)]).
mnesia_backup_name(Path, TabName) ->
filename:join([Path, ?BACKUP_MNESIA_DIR, atom_to_list(TabName)]).
@ -364,36 +374,42 @@ import_mnesia_tabs(BackupDir, Opts) ->
maybe_print("Importing built-in database...~n", [], Opts),
filter_errors(
lists:foldr(
fun(Tab, Acc) -> Acc#{Tab => import_mnesia_tab(BackupDir, Tab, Opts)} end,
fun(Mod, Acc) ->
Tabs = Mod:backup_tables(),
lists:foldr(
fun(Tab, InAcc) ->
InAcc#{Tab => import_mnesia_tab(BackupDir, Mod, Tab, Opts)}
end,
Acc,
Tabs
)
end,
#{},
tabs_to_backup()
)
).
import_mnesia_tab(BackupDir, TabName, Opts) ->
import_mnesia_tab(BackupDir, Mod, TabName, Opts) ->
MnesiaBackupFileName = mnesia_backup_name(BackupDir, TabName),
case filelib:is_regular(MnesiaBackupFileName) of
true ->
maybe_print("Importing ~p database table...~n", [TabName], Opts),
restore_mnesia_tab(BackupDir, MnesiaBackupFileName, TabName, Opts);
restore_mnesia_tab(BackupDir, MnesiaBackupFileName, Mod, TabName, Opts);
false ->
maybe_print("No backup file for ~p database table...~n", [TabName], Opts),
?SLOG(info, #{msg => "missing_mnesia_backup", table => TabName, backup => BackupDir}),
ok
end.
restore_mnesia_tab(BackupDir, MnesiaBackupFileName, TabName, Opts) ->
Validated =
catch mnesia:traverse_backup(
MnesiaBackupFileName, mnesia_backup, dummy, read_only, fun validate_mnesia_backup/2, 0
),
restore_mnesia_tab(BackupDir, MnesiaBackupFileName, Mod, TabName, Opts) ->
Validated = validate_mnesia_backup(MnesiaBackupFileName, Mod),
try
case Validated of
{ok, _} ->
{ok, #{backup_file := BackupFile}} ->
%% As we use keep_tables option, we don't need to modify 'copies' (nodes)
%% in a backup file before restoring it, as `mnsia:restore/2` will ignore
%% backed-up schema and keep the current table schema unchanged
Restored = mnesia:restore(MnesiaBackupFileName, [{default_op, keep_tables}]),
Restored = mnesia:restore(BackupFile, [{default_op, keep_tables}]),
case Restored of
{atomic, [TabName]} ->
ok;
@ -425,17 +441,81 @@ restore_mnesia_tab(BackupDir, MnesiaBackupFileName, TabName, Opts) ->
%% NOTE: if backup file is valid, we keep traversing it, though we only need to validate schema.
%% Looks like there is no clean way to abort traversal without triggering any error reporting,
%% `mnesia_bup:read_schema/2` is an option but its direct usage should also be avoided...
validate_mnesia_backup({schema, Tab, CreateList} = Schema, Acc) ->
validate_mnesia_backup(MnesiaBackupFileName, Mod) ->
Init = #{backup_file => MnesiaBackupFileName},
Validated =
catch mnesia:traverse_backup(
MnesiaBackupFileName,
mnesia_backup,
dummy,
read_only,
mnesia_backup_validator(Mod),
Init
),
case Validated of
ok ->
{ok, Init};
{error, {_, over}} ->
{ok, Init};
{error, {_, migrate}} ->
migrate_mnesia_backup(MnesiaBackupFileName, Mod, Init);
Error ->
Error
end.
%% if the module has validator callback, use it else use the default
mnesia_backup_validator(Mod) ->
Validator =
case erlang:function_exported(Mod, validate_mnesia_backup, 1) of
true ->
fun Mod:validate_mnesia_backup/1;
_ ->
fun default_validate_mnesia_backup/1
end,
fun(Schema, Acc) ->
case Validator(Schema) of
ok ->
{[Schema], Acc};
{ok, Break} ->
throw({error, Break});
Error ->
throw(Error)
end
end.
default_validate_mnesia_backup({schema, Tab, CreateList}) ->
ImportAttributes = proplists:get_value(attributes, CreateList),
Attributes = mnesia:table_info(Tab, attributes),
case ImportAttributes =/= Attributes of
case ImportAttributes == Attributes of
true ->
throw({error, different_table_schema});
ok;
false ->
{[Schema], Acc}
{error, different_table_schema}
end;
validate_mnesia_backup(Other, Acc) ->
{[Other], Acc}.
default_validate_mnesia_backup(_Other) ->
ok.
migrate_mnesia_backup(MnesiaBackupFileName, Mod, Acc) ->
case erlang:function_exported(Mod, migrate_mnesia_backup, 1) of
true ->
MigrateFile = MnesiaBackupFileName ++ ".migrate",
Migrator = fun(Schema, InAcc) ->
case Mod:migrate_mnesia_backup(Schema) of
{ok, NewSchema} ->
{[NewSchema], InAcc};
Error ->
throw(Error)
end
end,
catch mnesia:traverse_backup(
MnesiaBackupFileName,
MigrateFile,
Migrator,
Acc#{backup_file := MigrateFile}
);
_ ->
{error, no_migrator}
end.
extract_backup(BackupFileName) ->
BackupDir = root_backup_dir(),

146
apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl
View File

@ -19,12 +19,26 @@
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
-include_lib("emqx_dashboard/include/emqx_dashboard_rbac.hrl").
-if(?EMQX_RELEASE_EDITION == ee).
-define(EE_CASES, [
t_ee_create,
t_ee_update,
t_ee_authorize_viewer,
t_ee_authorize_admin,
t_ee_authorize_publisher
]).
-else.
-define(EE_CASES, []).
-endif.
all() -> [{group, parallel}, {group, sequence}].
suite() -> [{timetrap, {minutes, 1}}].
groups() ->
[
{parallel, [parallel], [t_create, t_update, t_delete, t_authorize, t_create_unexpired_app]},
{parallel, [parallel], ?EE_CASES},
{sequence, [], [t_bootstrap_file, t_create_failed]}
].
@ -41,37 +55,42 @@ t_bootstrap_file(_) ->
File = "./bootstrap_api_keys.txt",
ok = file:write_file(File, Bin),
update_file(File),
?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-1">>)),
?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-2">>)),
?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-1">>)),
?assertEqual(ok, auth_authorize(TestPath, <<"test-1">>, <<"secret-1">>)),
?assertEqual(ok, auth_authorize(TestPath, <<"test-2">>, <<"secret-2">>)),
?assertMatch({error, _}, auth_authorize(TestPath, <<"test-2">>, <<"secret-1">>)),
%% relaunch to check if the table is changed.
Bin1 = <<"test-1:new-secret-1\ntest-2:new-secret-2">>,
ok = file:write_file(File, Bin1),
update_file(File),
?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-1">>)),
?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-2">>)),
?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"new-secret-1">>)),
?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"new-secret-2">>)),
?assertMatch({error, _}, auth_authorize(TestPath, <<"test-1">>, <<"secret-1">>)),
?assertMatch({error, _}, auth_authorize(TestPath, <<"test-2">>, <<"secret-2">>)),
?assertEqual(ok, auth_authorize(TestPath, <<"test-1">>, <<"new-secret-1">>)),
?assertEqual(ok, auth_authorize(TestPath, <<"test-2">>, <<"new-secret-2">>)),
%% not error when bootstrap_file is empty
update_file(<<>>),
update_file("./bootstrap_apps_not_exist.txt"),
?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-1">>)),
?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-2">>)),
?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"new-secret-1">>)),
?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"new-secret-2">>)),
?assertMatch({error, _}, auth_authorize(TestPath, <<"test-1">>, <<"secret-1">>)),
?assertMatch({error, _}, auth_authorize(TestPath, <<"test-2">>, <<"secret-2">>)),
?assertEqual(ok, auth_authorize(TestPath, <<"test-1">>, <<"new-secret-1">>)),
?assertEqual(ok, auth_authorize(TestPath, <<"test-2">>, <<"new-secret-2">>)),
%% bad format
BadBin = <<"test-1:secret-11\ntest-2 secret-12">>,
ok = file:write_file(File, BadBin),
update_file(File),
?assertMatch({error, #{reason := "invalid_format"}}, emqx_mgmt_auth:init_bootstrap_file()),
?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-11">>)),
?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-12">>)),
?assertEqual(ok, auth_authorize(TestPath, <<"test-1">>, <<"secret-11">>)),
?assertMatch({error, _}, auth_authorize(TestPath, <<"test-2">>, <<"secret-12">>)),
update_file(<<>>),
ok.
auth_authorize(Path, Key, Secret) ->
FakePath = erlang:list_to_binary(emqx_dashboard_swagger:relative_uri("/fake")),
FakeReq = #{method => <<"GET">>, path => FakePath},
emqx_mgmt_auth:authorize(Path, FakeReq, Key, Secret).
update_file(File) ->
?assertMatch({ok, _}, emqx:update_config([<<"api_key">>], #{<<"bootstrap_file">> => File})).
@ -217,6 +236,102 @@ t_create_unexpired_app(_Config) ->
?assertMatch(#{<<"expired_at">> := <<"infinity">>}, Create2),
ok.
t_ee_create(_Config) ->
Name = <<"EMQX-EE-API-KEY-1">>,
{ok, Create} = create_app(Name, #{role => ?ROLE_API_VIEWER}),
?assertMatch(
#{
<<"api_key">> := _,
<<"api_secret">> := _,
<<"created_at">> := _,
<<"desc">> := _,
<<"enable">> := true,
<<"expired_at">> := _,
<<"name">> := Name,
<<"role">> := ?ROLE_API_VIEWER
},
Create
),
{ok, App} = read_app(Name),
?assertMatch(#{<<"name">> := Name, <<"role">> := ?ROLE_API_VIEWER}, App).
t_ee_update(_Config) ->
Name = <<"EMQX-EE-API-UPDATE-KEY">>,
{ok, _} = create_app(Name, #{role => ?ROLE_API_VIEWER}),
Change = #{
desc => <<"NoteVersion1"/utf8>>,
enable => false,
role => ?ROLE_API_SUPERUSER
},
{ok, Update1} = update_app(Name, Change),
?assertEqual(?ROLE_API_SUPERUSER, maps:get(<<"role">>, Update1)),
{ok, App} = read_app(Name),
?assertMatch(#{<<"name">> := Name, <<"role">> := ?ROLE_API_SUPERUSER}, App).
t_ee_authorize_viewer(_Config) ->
Name = <<"EMQX-EE-API-AUTHORIZE-KEY-VIEWER">>,
{ok, #{<<"api_key">> := ApiKey, <<"api_secret">> := ApiSecret}} = create_app(Name, #{
role => ?ROLE_API_VIEWER
}),
BasicHeader = emqx_common_test_http:auth_header(
binary_to_list(ApiKey),
binary_to_list(ApiSecret)
),
BanPath = emqx_mgmt_api_test_util:api_path(["banned"]),
?assertMatch({ok, _}, emqx_mgmt_api_test_util:request_api(get, BanPath, BasicHeader)),
?assertMatch(
{error, {_, 403, _}}, emqx_mgmt_api_test_util:request_api(delete, BanPath, BasicHeader)
).
t_ee_authorize_admin(_Config) ->
Name = <<"EMQX-EE-API-AUTHORIZE-KEY-ADMIN">>,
{ok, #{<<"api_key">> := ApiKey, <<"api_secret">> := ApiSecret}} = create_app(Name, #{
role => ?ROLE_API_SUPERUSER
}),
BasicHeader = emqx_common_test_http:auth_header(
binary_to_list(ApiKey),
binary_to_list(ApiSecret)
),
BanPath = emqx_mgmt_api_test_util:api_path(["banned"]),
?assertMatch({ok, _}, emqx_mgmt_api_test_util:request_api(get, BanPath, BasicHeader)),
?assertMatch(
{ok, _}, emqx_mgmt_api_test_util:request_api(delete, BanPath, BasicHeader)
).
t_ee_authorize_publisher(_Config) ->
Name = <<"EMQX-EE-API-AUTHORIZE-KEY-PUBLISHER">>,
{ok, #{<<"api_key">> := ApiKey, <<"api_secret">> := ApiSecret}} = create_app(Name, #{
role => ?ROLE_API_PUBLISHER
}),
BasicHeader = emqx_common_test_http:auth_header(
binary_to_list(ApiKey),
binary_to_list(ApiSecret)
),
BanPath = emqx_mgmt_api_test_util:api_path(["banned"]),
Publish = emqx_mgmt_api_test_util:api_path(["publish"]),
?assertMatch(
{error, {_, 403, _}}, emqx_mgmt_api_test_util:request_api(get, BanPath, BasicHeader)
),
?assertMatch(
{error, {_, 403, _}}, emqx_mgmt_api_test_util:request_api(delete, BanPath, BasicHeader)
),
?_assertMatch(
{ok, _},
emqx_mgmt_api_test_util:request_api(
post,
Publish,
[],
BasicHeader,
#{topic => <<"t/t_ee_authorize_publisher">>, payload => <<"hello">>}
)
).
list_app() ->
AuthHeader = emqx_dashboard_SUITE:auth_header_(),
Path = emqx_mgmt_api_test_util:api_path(["api_key"]),
@ -234,10 +349,13 @@ read_app(Name) ->
end.
create_app(Name) ->
create_app(Name, #{}).
create_app(Name, Extra) ->
AuthHeader = emqx_dashboard_SUITE:auth_header_(),
Path = emqx_mgmt_api_test_util:api_path(["api_key"]),
ExpiredAt = to_rfc3339(erlang:system_time(second) + 1000),
App = #{
App = Extra#{
name => Name,
expired_at => ExpiredAt,
desc => <<"Note"/utf8>>,

16
apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl
View File

@ -23,6 +23,7 @@
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-define(ROLE_SUPERUSER, <<"administrator">>).
-define(ROLE_API_SUPERUSER, <<"api_administrator">>).
-define(BOOTSTRAP_BACKUP, "emqx-export-test-bootstrap-ce.tar.gz").
all() ->
@ -56,7 +57,7 @@ init_per_testcase(TC = t_verify_imported_mnesia_tab_on_cluster, Config) ->
[{cluster, cluster(TC, Config)} | setup(TC, Config)];
init_per_testcase(t_mnesia_bad_tab_schema, Config) ->
meck:new(emqx_mgmt_data_backup, [passthrough]),
meck:expect(TC = emqx_mgmt_data_backup, mnesia_tabs_to_backup, 0, [data_backup_test]),
meck:expect(TC = emqx_mgmt_data_backup, mnesia_tabs_to_backup, 0, [?MODULE]),
setup(TC, Config);
init_per_testcase(TC, Config) ->
setup(TC, Config).
@ -99,7 +100,15 @@ t_cluster_hocon_export_import(Config) ->
?assertEqual(Exp, emqx_mgmt_data_backup:import(FileName)),
?assertEqual(RawConfAfterImport, emqx:get_raw_config([])),
%% lookup file inside <data_dir>/backup
?assertEqual(Exp, emqx_mgmt_data_backup:import(filename:basename(FileName))).
?assertEqual(Exp, emqx_mgmt_data_backup:import(filename:basename(FileName))),
%% backup data migration test
?assertMatch([_, _, _], ets:tab2list(emqx_app)),
?assertMatch(
{ok, #{name := <<"key_to_export2">>, role := ?ROLE_API_SUPERUSER}},
emqx_mgmt_auth:read(<<"key_to_export2">>)
),
ok.
t_ee_to_ce_backup(Config) ->
case emqx_release:edition() of
@ -329,6 +338,9 @@ t_verify_imported_mnesia_tab_on_cluster(Config) ->
timer:sleep(3000),
?assertEqual(AllUsers, lists:sort(rpc:call(ReplicantNode, mnesia, dirty_all_keys, [Tab]))).
backup_tables() ->
[data_backup_test].
t_mnesia_bad_tab_schema(_Config) ->
OldAttributes = [id, name, description],
ok = create_test_tab(OldAttributes),

8
changes/ee/feat-11766.en.md Normal file
View File

@ -0,0 +1,8 @@
Implemented a preliminary Role-Based Access Control for the REST API.
In this version, there are three predefined roles:
- Administrator: This role could access all resources.
- Viewer: This role can only view resources and data, corresponding to all GET requests in the REST API.
- Publisher: This role is special for MQTT messages publish, it can only access publish-related endpoints.

3
rel/i18n/emqx_mgmt_api_api_keys.hocon
View File

@ -30,4 +30,7 @@ format.desc:
format.label:
"""Unique and format by [a-zA-Z0-9-_]"""
role.desc:
"""Role for this API"""
}