Merge pull request #11766 from lafirest/feat/rbac

feat(api_key): add RBAC feature for the API key

apps/emqx/src/bhvrs/emqx_db_backup.erl

@@ -16,4 +16,21 @@
 
 -module(emqx_db_backup).
 
+-type traverse_break_reason() :: over | migrate.
+
 -callback backup_tables() -> [mria:table()].
+
+%% validate the backup
+%% return `ok` to traverse the next item
+%% return `{ok, over}` to finish the traverse
+%% return `{ok, migrate}` to call the migration callback
+-callback validate_mnesia_backup(tuple()) ->
+    ok
+    | {ok, traverse_break_reason()}
+    | {error, term()}.
+
+-callback migrate_mnesia_backup(tuple()) -> {ok, tuple()} | {error, term()}.
+
+-optional_callbacks([validate_mnesia_backup/1, migrate_mnesia_backup/1]).
+
+-export_type([traverse_break_reason/0]).

apps/emqx/test/emqx_common_test_http.erl

@@ -33,6 +33,9 @@
 -define(DEFAULT_APP_ID, <<"default_appid">>).
 -define(DEFAULT_APP_SECRET, <<"default_app_secret">>).
 
+%% from emqx_dashboard/include/emqx_dashboard_rbac.hrl
+-define(ROLE_API_SUPERUSER, <<"api_administrator">>).
+
 request_api(Method, Url, Auth) ->
     request_api(Method, Url, [], Auth, []).
 
@@ -90,7 +93,12 @@ create_default_app() ->
     Now = erlang:system_time(second),
     ExpiredAt = Now + timer:minutes(10),
     emqx_mgmt_auth:create(
-        ?DEFAULT_APP_ID, ?DEFAULT_APP_SECRET, true, ExpiredAt, <<"default app key for test">>
+        ?DEFAULT_APP_ID,
+        ?DEFAULT_APP_SECRET,
+        true,
+        ExpiredAt,
+        <<"default app key for test">>,
+        ?ROLE_API_SUPERUSER
     ).
 
 delete_default_app() ->

apps/emqx_dashboard/include/emqx_dashboard.hrl

@@ -13,16 +13,9 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 %%--------------------------------------------------------------------
--define(ADMIN, emqx_admin).
+-include("emqx_dashboard_rbac.hrl").
 
-%% TODO:
+-define(ADMIN, emqx_admin).
-%% The predefined roles of the preliminary RBAC implementation,
-%% these may be removed when developing the full RBAC feature.
-%% In full RBAC feature, the role may be customised created and deleted,
-%% a predefined configuration would replace these macros.
--define(ROLE_VIEWER, <<"viewer">>).
--define(ROLE_SUPERUSER, <<"administrator">>).
--define(ROLE_DEFAULT, ?ROLE_SUPERUSER).
 
 -define(BACKEND_LOCAL, local).
 -define(SSO_USERNAME(Backend, Name), {Backend, Name}).

apps/emqx_dashboard/include/emqx_dashboard_rbac.hrl

@@ -0,0 +1,33 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-ifndef(EMQX_DASHBOARD_RBAC).
+-define(EMQX_DASHBOARD_RBAC, true).
+
+%% TODO:
+%% The predefined roles of the preliminary RBAC implementation,
+%% these may be removed when developing the full RBAC feature.
+%% In full RBAC feature, the role may be customised created and deleted,
+%% a predefined configuration would replace these macros.
+-define(ROLE_VIEWER, <<"viewer">>).
+-define(ROLE_SUPERUSER, <<"administrator">>).
+-define(ROLE_DEFAULT, ?ROLE_SUPERUSER).
+
+-define(ROLE_API_VIEWER, <<"api_viewer">>).
+-define(ROLE_API_SUPERUSER, <<"api_administrator">>).
+-define(ROLE_API_PUBLISHER, <<"api_publisher">>).
+-define(ROLE_API_DEFAULT, ?ROLE_API_SUPERUSER).
+
+-endif.

apps/emqx_dashboard/src/emqx_dashboard.erl

@@ -212,7 +212,7 @@ listener_name(Protocol) ->
 
 -if(?EMQX_RELEASE_EDITION =/= ee).
 %% dialyzer complains about the `unauthorized_role' clause...
--dialyzer({no_match, [authorize/1]}).
+-dialyzer({no_match, [authorize/1, api_key_authorize/3]}).
 -endif.
 
 authorize(Req) ->
@@ -251,7 +251,7 @@ listeners() ->
 
 api_key_authorize(Req, Key, Secret) ->
     Path = cowboy_req:path(Req),
-    case emqx_mgmt_auth:authorize(Path, Key, Secret) of
+    case emqx_mgmt_auth:authorize(Path, Req, Key, Secret) of
         ok ->
             {ok, #{auth_type => api_key, api_key => Key}};
         {error, <<"not_allowed">>} ->
@@ -259,6 +259,9 @@ api_key_authorize(Req, Key, Secret) ->
                 ?BAD_API_KEY_OR_SECRET,
                 <<"Not allowed, Check api_key/api_secret">>
             );
+        {error, unauthorized_role} ->
+            {403, 'UNAUTHORIZED_ROLE',
+                <<"This API Key don't have permission to access this resource">>};
         {error, _} ->
             return_unauthorized(
                 ?BAD_API_KEY_OR_SECRET,

apps/emqx_dashboard/src/emqx_dashboard_admin.erl

@@ -423,7 +423,7 @@ ensure_role(Role) when is_binary(Role) ->
 
 -if(?EMQX_RELEASE_EDITION == ee).
 legal_role(Role) ->
-    emqx_dashboard_rbac:valid_role(Role).
+    emqx_dashboard_rbac:valid_dashboard_role(Role).
 
 role(Data) ->
     emqx_dashboard_rbac:role(Data).
@@ -454,8 +454,10 @@ lookup_user(Backend, Username) when is_atom(Backend) ->
 
 -dialyzer({no_match, [add_user/4, update_user/3]}).
 
+legal_role(?ROLE_DEFAULT) ->
+    ok;
 legal_role(_) ->
-    ok.
+    {error, <<"Role does not exist">>}.
 
 role(_) ->
     ?ROLE_DEFAULT.

apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src

@@ -1,6 +1,6 @@
 {application, emqx_dashboard_rbac, [
     {description, "EMQX Dashboard RBAC"},
-    {vsn, "0.1.0"},
+    {vsn, "0.1.1"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl

@@ -6,7 +6,12 @@
 
 -include_lib("emqx_dashboard/include/emqx_dashboard.hrl").
 
--export([check_rbac/2, role/1, valid_role/1]).
+-export([
+    check_rbac/2,
+    role/1,
+    valid_dashboard_role/1,
+    valid_api_role/1
+]).
 
 -dialyzer({nowarn_function, role/1}).
 %%=====================================================================
@@ -31,25 +36,44 @@ role(#?ADMIN{role = Role}) ->
 role([]) ->
     ?ROLE_SUPERUSER;
 role(#{role := Role}) ->
+    Role;
+role(Role) when is_binary(Role) ->
     Role.
 
-valid_role(Role) ->
+valid_dashboard_role(Role) ->
-    case lists:member(Role, role_list()) of
+    valid_role(dashboard, Role).
+
+valid_api_role(Role) ->
+    valid_role(api, Role).
+
+%% ===================================================================
+check_rbac(?ROLE_SUPERUSER, _, _) ->
+    true;
+check_rbac(?ROLE_API_SUPERUSER, _, _) ->
+    true;
+check_rbac(?ROLE_VIEWER, <<"GET">>, _) ->
+    true;
+check_rbac(?ROLE_API_VIEWER, <<"GET">>, _) ->
+    true;
+%% this API is a special case
+check_rbac(?ROLE_VIEWER, <<"POST">>, <<"/logout">>) ->
+    true;
+check_rbac(?ROLE_API_PUBLISHER, <<"POST">>, <<"/publish">>) ->
+    true;
+check_rbac(?ROLE_API_PUBLISHER, <<"POST">>, <<"/publish/bulk">>) ->
+    true;
+check_rbac(_, _, _) ->
+    false.
+
+valid_role(Type, Role) ->
+    case lists:member(Role, role_list(Type)) of
         true ->
             ok;
         _ ->
             {error, <<"Role does not exist">>}
     end.
-%% ===================================================================
-check_rbac(?ROLE_SUPERUSER, _, _) ->
-    true;
-check_rbac(?ROLE_VIEWER, <<"GET">>, _) ->
-    true;
-%% this API is a special case
-check_rbac(?ROLE_VIEWER, <<"POST">>, <<"/logout">>) ->
-    true;
-check_rbac(_, _, _) ->
-    false.
 
-role_list() ->
+role_list(dashboard) ->
-    [?ROLE_VIEWER, ?ROLE_SUPERUSER].
+    [?ROLE_VIEWER, ?ROLE_SUPERUSER];
+role_list(api) ->
+    [?ROLE_API_VIEWER, ?ROLE_API_PUBLISHER, ?ROLE_API_SUPERUSER].

apps/emqx_management/src/emqx_mgmt_api_api_keys.erl

@@ -19,6 +19,7 @@
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
+-include_lib("emqx_dashboard/include/emqx_dashboard_rbac.hrl").
 
 -export([api_spec/0, fields/1, paths/0, schema/1, namespace/0]).
 -export([api_key/2, api_key_by_name/2]).
@@ -150,7 +151,7 @@ fields(app) ->
             )},
         {enable, hoconsc:mk(boolean(), #{desc => "Enable/Disable", required => false})},
         {expired, hoconsc:mk(boolean(), #{desc => "Expired", required => false})}
-    ];
+    ] ++ app_extend_fields();
 fields(name) ->
     [
         {name,
@@ -192,7 +193,8 @@ api_key(post, #{body := App}) ->
     } = App,
     ExpiredAt = ensure_expired_at(App),
     Desc = unicode:characters_to_binary(Desc0, unicode),
-    case emqx_mgmt_auth:create(Name, Enable, ExpiredAt, Desc) of
+    Role = maps:get(<<"role">>, App, ?ROLE_API_DEFAULT),
+    case emqx_mgmt_auth:create(Name, Enable, ExpiredAt, Desc, Role) of
         {ok, NewApp} ->
             {200, emqx_mgmt_auth:format(NewApp)};
         {error, Reason} ->
@@ -218,10 +220,38 @@ api_key_by_name(put, #{bindings := #{name := Name}, body := Body}) ->
     Enable = maps:get(<<"enable">>, Body, undefined),
     ExpiredAt = ensure_expired_at(Body),
     Desc = maps:get(<<"desc">>, Body, undefined),
-    case emqx_mgmt_auth:update(Name, Enable, ExpiredAt, Desc) of
+    Role = maps:get(<<"role">>, Body, ?ROLE_API_DEFAULT),
-        {ok, App} -> {200, emqx_mgmt_auth:format(App)};
+    case emqx_mgmt_auth:update(Name, Enable, ExpiredAt, Desc, Role) of
-        {error, not_found} -> {404, ?NOT_FOUND_RESPONSE}
+        {ok, App} ->
+            {200, emqx_mgmt_auth:format(App)};
+        {error, not_found} ->
+            {404, ?NOT_FOUND_RESPONSE};
+        {error, Reason} ->
+            {400, #{
+                code => 'BAD_REQUEST',
+                message => iolist_to_binary(io_lib:format("~p", [Reason]))
+            }}
     end.
 
 ensure_expired_at(#{<<"expired_at">> := ExpiredAt}) when is_integer(ExpiredAt) -> ExpiredAt;
 ensure_expired_at(_) -> infinity.
+
+-if(?EMQX_RELEASE_EDITION == ee).
+
+app_extend_fields() ->
+    [
+        {role,
+            hoconsc:mk(binary(), #{
+                desc => ?DESC(role),
+                default => ?ROLE_API_DEFAULT,
+                example => ?ROLE_API_DEFAULT,
+                validator => fun emqx_dashboard_rbac:valid_api_role/1
+            })}
+    ].
+
+-else.
+
+app_extend_fields() ->
+    [].
+
+-endif.

apps/emqx_management/src/emqx_mgmt_auth.erl

@@ -16,6 +16,7 @@
 -module(emqx_mgmt_auth).
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("emqx_dashboard/include/emqx_dashboard_rbac.hrl").
 
 -behaviour(emqx_db_backup).
 
@@ -25,33 +26,34 @@
 -behaviour(emqx_config_handler).
 
 -export([
-    create/4,
+    create/5,
     read/1,
-    update/4,
+    update/5,
     delete/1,
     list/0,
     init_bootstrap_file/0,
     format/1
 ]).
 
--export([authorize/3]).
+-export([authorize/4]).
 -export([post_config_update/5]).
 
--export([backup_tables/0]).
+-export([backup_tables/0, validate_mnesia_backup/1, migrate_mnesia_backup/1]).
 
 %% Internal exports (RPC)
 -export([
-    do_update/4,
+    do_update/5,
     do_delete/1,
     do_create_app/3,
     do_force_create_app/3
 ]).
 
 -ifdef(TEST).
--export([create/5]).
+-export([create/6]).
 -endif.
 
 -define(APP, emqx_app).
+-type api_user_role() :: binary().
 
 -record(?APP, {
     name = <<>> :: binary() | '_',
@@ -60,17 +62,21 @@
     enable = true :: boolean() | '_',
     desc = <<>> :: binary() | '_',
     expired_at = 0 :: integer() | undefined | infinity | '_',
-    created_at = 0 :: integer() | '_'
+    created_at = 0 :: integer() | '_',
+    role = ?ROLE_DEFAULT :: api_user_role() | '_',
+    extra = #{} :: map() | '_'
 }).
 
 mnesia(boot) ->
+    Fields = record_info(fields, ?APP),
     ok = mria:create_table(?APP, [
         {type, set},
         {rlog_shard, ?COMMON_SHARD},
         {storage, disc_copies},
         {record_name, ?APP},
-        {attributes, record_info(fields, ?APP)}
+        {attributes, Fields}
-    ]).
+    ]),
+    maybe_migrate_table(Fields).
 
 %%--------------------------------------------------------------------
 %% Data backup
@@ -78,6 +84,35 @@ mnesia(boot) ->
 
 backup_tables() -> [?APP].
 
+validate_mnesia_backup({schema, _Tab, CreateList} = Schema) ->
+    case emqx_mgmt_data_backup:default_validate_mnesia_backup(Schema) of
+        ok ->
+            {ok, over};
+        _ ->
+            case proplists:get_value(attributes, CreateList) of
+                [name, api_key, api_secret_hash, enable, desc, expired_at, created_at] ->
+                    {ok, migrate};
+                Fields ->
+                    {error, {unknow_fields, Fields}}
+            end
+    end;
+validate_mnesia_backup(_Other) ->
+    ok.
+
+migrate_mnesia_backup({schema, Tab, CreateList}) ->
+    case proplists:get_value(attributes, CreateList) of
+        [name, api_key, api_secret_hash, enable, desc, expired_at, created_at] = Fields ->
+            NewFields = Fields ++ [role, extra],
+            CreateList2 = lists:keyreplace(
+                attributes, 1, CreateList, {attributes, NewFields}
+            ),
+            {ok, {schema, Tab, CreateList2}};
+        Fields ->
+            {error, {unknow_fields, Fields}}
+    end;
+migrate_mnesia_backup(Data) ->
+    {ok, do_table_migrate(Data)}.
+
 post_config_update([api_key], _Req, NewConf, _OldConf, _AppEnvs) ->
     #{bootstrap_file := File} = NewConf,
     case init_bootstrap_file(File) of
@@ -95,13 +130,13 @@ init_bootstrap_file() ->
     ?SLOG(debug, #{msg => "init_bootstrap_api_keys_from_file", file => File}),
     init_bootstrap_file(File).
 
-create(Name, Enable, ExpiredAt, Desc) ->
+create(Name, Enable, ExpiredAt, Desc, Role) ->
     ApiSecret = generate_api_secret(),
-    create(Name, ApiSecret, Enable, ExpiredAt, Desc).
+    create(Name, ApiSecret, Enable, ExpiredAt, Desc, Role).
 
-create(Name, ApiSecret, Enable, ExpiredAt, Desc) ->
+create(Name, ApiSecret, Enable, ExpiredAt, Desc, Role) ->
     case mnesia:table_info(?APP, size) < 100 of
-        true -> create_app(Name, ApiSecret, Enable, ExpiredAt, Desc);
+        true -> create_app(Name, ApiSecret, Enable, ExpiredAt, Desc, Role);
         false -> {error, "Maximum ApiKey"}
     end.
 
@@ -111,10 +146,15 @@ read(Name) ->
         [] -> {error, not_found}
     end.
 
-update(Name, Enable, ExpiredAt, Desc) ->
+update(Name, Enable, ExpiredAt, Desc, Role) ->
-    trans(fun ?MODULE:do_update/4, [Name, Enable, ExpiredAt, Desc]).
+    case valid_role(Role) of
+        ok ->
+            trans(fun ?MODULE:do_update/5, [Name, Enable, ExpiredAt, Desc, Role]);
+        Error ->
+            Error
+    end.
 
-do_update(Name, Enable, ExpiredAt, Desc) ->
+do_update(Name, Enable, ExpiredAt, Desc, Role) ->
     case mnesia:read(?APP, Name, write) of
         [] ->
             mnesia:abort(not_found);
@@ -123,7 +163,8 @@ do_update(Name, Enable, ExpiredAt, Desc) ->
                 App0#?APP{
                     expired_at = ExpiredAt,
                     enable = ensure_not_undefined(Enable, Enable0),
-                    desc = ensure_not_undefined(Desc, Desc0)
+                    desc = ensure_not_undefined(Desc, Desc0),
+                    role = Role
                 },
             ok = mnesia:write(App),
             to_map(App)
@@ -138,37 +179,37 @@ do_delete(Name) ->
         [_App] -> mnesia:delete({?APP, Name})
     end.
 
-format(App = #{expired_at := ExpiredAt0, created_at := CreateAt}) ->
+format(App = #{expired_at := ExpiredAt, created_at := CreateAt}) ->
-    ExpiredAt =
+    format_app_extend(App#{
-        case ExpiredAt0 of
+        expired_at => format_epoch(ExpiredAt),
-            infinity -> <<"infinity">>;
+        created_at => format_epoch(CreateAt)
-            _ -> emqx_utils_calendar:epoch_to_rfc3339(ExpiredAt0, second)
+    }).
-        end,
+
-    App#{
+format_epoch(infinity) ->
-        expired_at => ExpiredAt,
+    <<"infinity">>;
-        created_at => emqx_utils_calendar:epoch_to_rfc3339(CreateAt, second)
+format_epoch(Epoch) ->
-    }.
+    emqx_utils_calendar:epoch_to_rfc3339(Epoch, second).
 
 list() ->
     to_map(ets:match_object(?APP, #?APP{_ = '_'})).
 
-authorize(<<"/api/v5/users", _/binary>>, _ApiKey, _ApiSecret) ->
+authorize(<<"/api/v5/users", _/binary>>, _Req, _ApiKey, _ApiSecret) ->
     {error, <<"not_allowed">>};
-authorize(<<"/api/v5/api_key", _/binary>>, _ApiKey, _ApiSecret) ->
+authorize(<<"/api/v5/api_key", _/binary>>, _Req, _ApiKey, _ApiSecret) ->
     {error, <<"not_allowed">>};
-authorize(<<"/api/v5/logout", _/binary>>, _ApiKey, _ApiSecret) ->
+authorize(<<"/api/v5/logout", _/binary>>, _Req, _ApiKey, _ApiSecret) ->
     {error, <<"not_allowed">>};
-authorize(_Path, ApiKey, ApiSecret) ->
+authorize(_Path, Req, ApiKey, ApiSecret) ->
     Now = erlang:system_time(second),
     case find_by_api_key(ApiKey) of
-        {ok, true, ExpiredAt, SecretHash} when ExpiredAt >= Now ->
+        {ok, true, ExpiredAt, SecretHash, Role} when ExpiredAt >= Now ->
             case emqx_dashboard_admin:verify_hash(ApiSecret, SecretHash) of
-                ok -> ok;
+                ok -> check_rbac(Req, Role);
                 error -> {error, "secret_error"}
             end;
-        {ok, true, _ExpiredAt, _SecretHash} ->
+        {ok, true, _ExpiredAt, _SecretHash, _Role} ->
             {error, "secret_expired"};
-        {ok, false, _ExpiredAt, _SecretHash} ->
+        {ok, false, _ExpiredAt, _SecretHash, _Role} ->
             {error, "secret_disable"};
         {error, Reason} ->
             {error, Reason}
@@ -177,8 +218,12 @@ authorize(_Path, ApiKey, ApiSecret) ->
 find_by_api_key(ApiKey) ->
     Fun = fun() -> mnesia:match_object(#?APP{api_key = ApiKey, _ = '_'}) end,
     case mria:ro_transaction(?COMMON_SHARD, Fun) of
-        {atomic, [#?APP{api_secret_hash = SecretHash, enable = Enable, expired_at = ExpiredAt}]} ->
+        {atomic, [
-            {ok, Enable, ExpiredAt, SecretHash};
+            #?APP{
+                api_secret_hash = SecretHash, enable = Enable, expired_at = ExpiredAt, role = Role
+            }
+        ]} ->
+            {ok, Enable, ExpiredAt, SecretHash, Role};
         _ ->
             {error, "not_found"}
     end.
@@ -188,7 +233,9 @@ ensure_not_undefined(New, _Old) -> New.
 
 to_map(Apps) when is_list(Apps) ->
     [to_map(App) || App <- Apps];
-to_map(#?APP{name = N, api_key = K, enable = E, expired_at = ET, created_at = CT, desc = D}) ->
+to_map(#?APP{
+    name = N, api_key = K, enable = E, expired_at = ET, created_at = CT, desc = D, role = Role
+}) ->
     #{
         name => N,
         api_key => K,
@@ -196,13 +243,14 @@ to_map(#?APP{name = N, api_key = K, enable = E, expired_at = ET, created_at = CT
         expired_at => ET,
         created_at => CT,
         desc => D,
-        expired => is_expired(ET)
+        expired => is_expired(ET),
+        role => Role
     }.
 
 is_expired(undefined) -> false;
 is_expired(ExpiredTime) -> ExpiredTime < erlang:system_time(second).
 
-create_app(Name, ApiSecret, Enable, ExpiredAt, Desc) ->
+create_app(Name, ApiSecret, Enable, ExpiredAt, Desc, Role) ->
     App =
         #?APP{
             name = Name,
@@ -211,7 +259,8 @@ create_app(Name, ApiSecret, Enable, ExpiredAt, Desc) ->
             desc = Desc,
             created_at = erlang:system_time(second),
             api_secret_hash = emqx_dashboard_admin:hash(ApiSecret),
-            api_key = list_to_binary(emqx_utils:gen_id(16))
+            api_key = list_to_binary(emqx_utils:gen_id(16)),
+            role = Role
         },
     case create_app(App) of
         {ok, Res} ->
@@ -220,8 +269,13 @@ create_app(Name, ApiSecret, Enable, ExpiredAt, Desc) ->
             Error
     end.
 
-create_app(App = #?APP{api_key = ApiKey, name = Name}) ->
+create_app(App = #?APP{api_key = ApiKey, name = Name, role = Role}) ->
-    trans(fun ?MODULE:do_create_app/3, [App, ApiKey, Name]).
+    case valid_role(Role) of
+        ok ->
+            trans(fun ?MODULE:do_create_app/3, [App, ApiKey, Name]);
+        Error ->
+            Error
+    end.
 
 force_create_app(NamePrefix, App = #?APP{api_key = ApiKey}) ->
     trans(fun ?MODULE:do_force_create_app/3, [App, ApiKey, NamePrefix]).
@@ -340,3 +394,58 @@ add_bootstrap_file(File, Dev, MP, Line) ->
         {error, Reason} ->
             throw(#{file => File, line => Line, reason => Reason})
     end.
+
+-if(?EMQX_RELEASE_EDITION == ee).
+check_rbac(Req, Role) ->
+    case emqx_dashboard_rbac:check_rbac(Req, Role) of
+        true ->
+            ok;
+        _ ->
+            {error, unauthorized_role}
+    end.
+
+format_app_extend(App) ->
+    App.
+
+valid_role(Role) ->
+    emqx_dashboard_rbac:valid_api_role(Role).
+
+-else.
+
+check_rbac(_Req, _Role) ->
+    ok.
+
+format_app_extend(App) ->
+    maps:remove(role, App).
+
+valid_role(?ROLE_API_DEFAULT) ->
+    ok;
+valid_role(_) ->
+    {error, <<"Role does not exist">>}.
+
+-endif.
+
+maybe_migrate_table(Fields) ->
+    case mnesia:table_info(?APP, attributes) =:= Fields of
+        true ->
+            ok;
+        false ->
+            TransFun = fun do_table_migrate/1,
+            {atomic, ok} = mnesia:transform_table(?APP, TransFun, Fields, ?APP),
+            ok
+    end.
+
+do_table_migrate({?APP, Name, Key, Hash, Enable, Desc, ExpiredAt, CreatedAt}) ->
+    #?APP{
+        name = Name,
+        api_key = Key,
+        api_secret_hash = Hash,
+        enable = Enable,
+        desc = Desc,
+        expired_at = ExpiredAt,
+        created_at = CreatedAt,
+        role = ?ROLE_API_DEFAULT,
+        extra = #{}
+    };
+do_table_migrate(#?APP{} = App) ->
+    App.

apps/emqx_management/src/emqx_mgmt_data_backup.erl

@@ -24,6 +24,8 @@
     format_error/1
 ]).
 
+-export([default_validate_mnesia_backup/1]).
+
 -ifdef(TEST).
 -compile(export_all).
 -compile(nowarn_export_all).
@@ -223,7 +225,15 @@ export_cluster_hocon(TarDescriptor, BackupBaseName, Opts) ->
 export_mnesia_tabs(TarDescriptor, BackupName, BackupBaseName, Opts) ->
     maybe_print("Exporting built-in database...~n", [], Opts),
     lists:foreach(
-        fun(Tab) -> export_mnesia_tab(TarDescriptor, Tab, BackupName, BackupBaseName, Opts) end,
+        fun(Mod) ->
+            Tabs = Mod:backup_tables(),
+            lists:foreach(
+                fun(Tab) ->
+                    export_mnesia_tab(TarDescriptor, Tab, BackupName, BackupBaseName, Opts)
+                end,
+                Tabs
+            )
+        end,
         tabs_to_backup()
     ).
 
@@ -259,7 +269,7 @@ tabs_to_backup() ->
 -endif.
 
 mnesia_tabs_to_backup() ->
-    lists:flatten([M:backup_tables() || M <- find_behaviours(emqx_db_backup)]).
+    lists:flatten([M || M <- find_behaviours(emqx_db_backup)]).
 
 mnesia_backup_name(Path, TabName) ->
     filename:join([Path, ?BACKUP_MNESIA_DIR, atom_to_list(TabName)]).
@@ -364,36 +374,42 @@ import_mnesia_tabs(BackupDir, Opts) ->
     maybe_print("Importing built-in database...~n", [], Opts),
     filter_errors(
         lists:foldr(
-            fun(Tab, Acc) -> Acc#{Tab => import_mnesia_tab(BackupDir, Tab, Opts)} end,
+            fun(Mod, Acc) ->
+                Tabs = Mod:backup_tables(),
+                lists:foldr(
+                    fun(Tab, InAcc) ->
+                        InAcc#{Tab => import_mnesia_tab(BackupDir, Mod, Tab, Opts)}
+                    end,
+                    Acc,
+                    Tabs
+                )
+            end,
             #{},
             tabs_to_backup()
         )
     ).
 
-import_mnesia_tab(BackupDir, TabName, Opts) ->
+import_mnesia_tab(BackupDir, Mod, TabName, Opts) ->
     MnesiaBackupFileName = mnesia_backup_name(BackupDir, TabName),
     case filelib:is_regular(MnesiaBackupFileName) of
         true ->
             maybe_print("Importing ~p database table...~n", [TabName], Opts),
-            restore_mnesia_tab(BackupDir, MnesiaBackupFileName, TabName, Opts);
+            restore_mnesia_tab(BackupDir, MnesiaBackupFileName, Mod, TabName, Opts);
         false ->
             maybe_print("No backup file for ~p database table...~n", [TabName], Opts),
             ?SLOG(info, #{msg => "missing_mnesia_backup", table => TabName, backup => BackupDir}),
             ok
     end.
 
-restore_mnesia_tab(BackupDir, MnesiaBackupFileName, TabName, Opts) ->
+restore_mnesia_tab(BackupDir, MnesiaBackupFileName, Mod, TabName, Opts) ->
-    Validated =
+    Validated = validate_mnesia_backup(MnesiaBackupFileName, Mod),
-        catch mnesia:traverse_backup(
-            MnesiaBackupFileName, mnesia_backup, dummy, read_only, fun validate_mnesia_backup/2, 0
-        ),
     try
         case Validated of
-            {ok, _} ->
+            {ok, #{backup_file := BackupFile}} ->
                 %% As we use keep_tables option, we don't need to modify 'copies' (nodes)
                 %% in a backup file before restoring it,  as `mnsia:restore/2` will ignore
                 %% backed-up schema and keep the current table schema unchanged
-                Restored = mnesia:restore(MnesiaBackupFileName, [{default_op, keep_tables}]),
+                Restored = mnesia:restore(BackupFile, [{default_op, keep_tables}]),
                 case Restored of
                     {atomic, [TabName]} ->
                         ok;
@@ -425,17 +441,81 @@ restore_mnesia_tab(BackupDir, MnesiaBackupFileName, TabName, Opts) ->
 %% NOTE: if backup file is valid, we keep traversing it, though we only need to validate schema.
 %% Looks like there is no clean way to abort traversal without triggering any error reporting,
 %% `mnesia_bup:read_schema/2` is an option but its direct usage should also be avoided...
-validate_mnesia_backup({schema, Tab, CreateList} = Schema, Acc) ->
+validate_mnesia_backup(MnesiaBackupFileName, Mod) ->
+    Init = #{backup_file => MnesiaBackupFileName},
+    Validated =
+        catch mnesia:traverse_backup(
+            MnesiaBackupFileName,
+            mnesia_backup,
+            dummy,
+            read_only,
+            mnesia_backup_validator(Mod),
+            Init
+        ),
+    case Validated of
+        ok ->
+            {ok, Init};
+        {error, {_, over}} ->
+            {ok, Init};
+        {error, {_, migrate}} ->
+            migrate_mnesia_backup(MnesiaBackupFileName, Mod, Init);
+        Error ->
+            Error
+    end.
+
+%% if the module has validator callback, use it else use the default
+mnesia_backup_validator(Mod) ->
+    Validator =
+        case erlang:function_exported(Mod, validate_mnesia_backup, 1) of
+            true ->
+                fun Mod:validate_mnesia_backup/1;
+            _ ->
+                fun default_validate_mnesia_backup/1
+        end,
+    fun(Schema, Acc) ->
+        case Validator(Schema) of
+            ok ->
+                {[Schema], Acc};
+            {ok, Break} ->
+                throw({error, Break});
+            Error ->
+                throw(Error)
+        end
+    end.
+
+default_validate_mnesia_backup({schema, Tab, CreateList}) ->
     ImportAttributes = proplists:get_value(attributes, CreateList),
     Attributes = mnesia:table_info(Tab, attributes),
-    case ImportAttributes =/= Attributes of
+    case ImportAttributes == Attributes of
         true ->
-            throw({error, different_table_schema});
+            ok;
         false ->
-            {[Schema], Acc}
+            {error, different_table_schema}
     end;
-validate_mnesia_backup(Other, Acc) ->
+default_validate_mnesia_backup(_Other) ->
-    {[Other], Acc}.
+    ok.
+
+migrate_mnesia_backup(MnesiaBackupFileName, Mod, Acc) ->
+    case erlang:function_exported(Mod, migrate_mnesia_backup, 1) of
+        true ->
+            MigrateFile = MnesiaBackupFileName ++ ".migrate",
+            Migrator = fun(Schema, InAcc) ->
+                case Mod:migrate_mnesia_backup(Schema) of
+                    {ok, NewSchema} ->
+                        {[NewSchema], InAcc};
+                    Error ->
+                        throw(Error)
+                end
+            end,
+            catch mnesia:traverse_backup(
+                MnesiaBackupFileName,
+                MigrateFile,
+                Migrator,
+                Acc#{backup_file := MigrateFile}
+            );
+        _ ->
+            {error, no_migrator}
+    end.
 
 extract_backup(BackupFileName) ->
     BackupDir = root_backup_dir(),

apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl

@@ -19,12 +19,26 @@
 -compile(nowarn_export_all).
 
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("emqx_dashboard/include/emqx_dashboard_rbac.hrl").
+
+-if(?EMQX_RELEASE_EDITION == ee).
+-define(EE_CASES, [
+    t_ee_create,
+    t_ee_update,
+    t_ee_authorize_viewer,
+    t_ee_authorize_admin,
+    t_ee_authorize_publisher
+]).
+-else.
+-define(EE_CASES, []).
+-endif.
 
 all() -> [{group, parallel}, {group, sequence}].
 suite() -> [{timetrap, {minutes, 1}}].
 groups() ->
     [
         {parallel, [parallel], [t_create, t_update, t_delete, t_authorize, t_create_unexpired_app]},
+        {parallel, [parallel], ?EE_CASES},
         {sequence, [], [t_bootstrap_file, t_create_failed]}
     ].
 
@@ -41,37 +55,42 @@ t_bootstrap_file(_) ->
     File = "./bootstrap_api_keys.txt",
     ok = file:write_file(File, Bin),
     update_file(File),
-    ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-1">>)),
+    ?assertEqual(ok, auth_authorize(TestPath, <<"test-1">>, <<"secret-1">>)),
-    ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-2">>)),
+    ?assertEqual(ok, auth_authorize(TestPath, <<"test-2">>, <<"secret-2">>)),
-    ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-1">>)),
+    ?assertMatch({error, _}, auth_authorize(TestPath, <<"test-2">>, <<"secret-1">>)),
 
     %% relaunch to check if the table is changed.
     Bin1 = <<"test-1:new-secret-1\ntest-2:new-secret-2">>,
     ok = file:write_file(File, Bin1),
     update_file(File),
-    ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-1">>)),
+    ?assertMatch({error, _}, auth_authorize(TestPath, <<"test-1">>, <<"secret-1">>)),
-    ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-2">>)),
+    ?assertMatch({error, _}, auth_authorize(TestPath, <<"test-2">>, <<"secret-2">>)),
-    ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"new-secret-1">>)),
+    ?assertEqual(ok, auth_authorize(TestPath, <<"test-1">>, <<"new-secret-1">>)),
-    ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"new-secret-2">>)),
+    ?assertEqual(ok, auth_authorize(TestPath, <<"test-2">>, <<"new-secret-2">>)),
 
     %% not error when bootstrap_file is empty
     update_file(<<>>),
     update_file("./bootstrap_apps_not_exist.txt"),
-    ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-1">>)),
+    ?assertMatch({error, _}, auth_authorize(TestPath, <<"test-1">>, <<"secret-1">>)),
-    ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-2">>)),
+    ?assertMatch({error, _}, auth_authorize(TestPath, <<"test-2">>, <<"secret-2">>)),
-    ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"new-secret-1">>)),
+    ?assertEqual(ok, auth_authorize(TestPath, <<"test-1">>, <<"new-secret-1">>)),
-    ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"new-secret-2">>)),
+    ?assertEqual(ok, auth_authorize(TestPath, <<"test-2">>, <<"new-secret-2">>)),
 
     %% bad format
     BadBin = <<"test-1:secret-11\ntest-2 secret-12">>,
     ok = file:write_file(File, BadBin),
     update_file(File),
     ?assertMatch({error, #{reason := "invalid_format"}}, emqx_mgmt_auth:init_bootstrap_file()),
-    ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-11">>)),
+    ?assertEqual(ok, auth_authorize(TestPath, <<"test-1">>, <<"secret-11">>)),
-    ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-12">>)),
+    ?assertMatch({error, _}, auth_authorize(TestPath, <<"test-2">>, <<"secret-12">>)),
     update_file(<<>>),
     ok.
 
+auth_authorize(Path, Key, Secret) ->
+    FakePath = erlang:list_to_binary(emqx_dashboard_swagger:relative_uri("/fake")),
+    FakeReq = #{method => <<"GET">>, path => FakePath},
+    emqx_mgmt_auth:authorize(Path, FakeReq, Key, Secret).
+
 update_file(File) ->
     ?assertMatch({ok, _}, emqx:update_config([<<"api_key">>], #{<<"bootstrap_file">> => File})).
 
@@ -217,6 +236,102 @@ t_create_unexpired_app(_Config) ->
     ?assertMatch(#{<<"expired_at">> := <<"infinity">>}, Create2),
     ok.
 
+t_ee_create(_Config) ->
+    Name = <<"EMQX-EE-API-KEY-1">>,
+    {ok, Create} = create_app(Name, #{role => ?ROLE_API_VIEWER}),
+    ?assertMatch(
+        #{
+            <<"api_key">> := _,
+            <<"api_secret">> := _,
+            <<"created_at">> := _,
+            <<"desc">> := _,
+            <<"enable">> := true,
+            <<"expired_at">> := _,
+            <<"name">> := Name,
+            <<"role">> := ?ROLE_API_VIEWER
+        },
+        Create
+    ),
+
+    {ok, App} = read_app(Name),
+    ?assertMatch(#{<<"name">> := Name, <<"role">> := ?ROLE_API_VIEWER}, App).
+
+t_ee_update(_Config) ->
+    Name = <<"EMQX-EE-API-UPDATE-KEY">>,
+    {ok, _} = create_app(Name, #{role => ?ROLE_API_VIEWER}),
+
+    Change = #{
+        desc => <<"NoteVersion1"/utf8>>,
+        enable => false,
+        role => ?ROLE_API_SUPERUSER
+    },
+    {ok, Update1} = update_app(Name, Change),
+    ?assertEqual(?ROLE_API_SUPERUSER, maps:get(<<"role">>, Update1)),
+
+    {ok, App} = read_app(Name),
+    ?assertMatch(#{<<"name">> := Name, <<"role">> := ?ROLE_API_SUPERUSER}, App).
+
+t_ee_authorize_viewer(_Config) ->
+    Name = <<"EMQX-EE-API-AUTHORIZE-KEY-VIEWER">>,
+    {ok, #{<<"api_key">> := ApiKey, <<"api_secret">> := ApiSecret}} = create_app(Name, #{
+        role => ?ROLE_API_VIEWER
+    }),
+    BasicHeader = emqx_common_test_http:auth_header(
+        binary_to_list(ApiKey),
+        binary_to_list(ApiSecret)
+    ),
+
+    BanPath = emqx_mgmt_api_test_util:api_path(["banned"]),
+    ?assertMatch({ok, _}, emqx_mgmt_api_test_util:request_api(get, BanPath, BasicHeader)),
+    ?assertMatch(
+        {error, {_, 403, _}}, emqx_mgmt_api_test_util:request_api(delete, BanPath, BasicHeader)
+    ).
+
+t_ee_authorize_admin(_Config) ->
+    Name = <<"EMQX-EE-API-AUTHORIZE-KEY-ADMIN">>,
+    {ok, #{<<"api_key">> := ApiKey, <<"api_secret">> := ApiSecret}} = create_app(Name, #{
+        role => ?ROLE_API_SUPERUSER
+    }),
+    BasicHeader = emqx_common_test_http:auth_header(
+        binary_to_list(ApiKey),
+        binary_to_list(ApiSecret)
+    ),
+
+    BanPath = emqx_mgmt_api_test_util:api_path(["banned"]),
+    ?assertMatch({ok, _}, emqx_mgmt_api_test_util:request_api(get, BanPath, BasicHeader)),
+    ?assertMatch(
+        {ok, _}, emqx_mgmt_api_test_util:request_api(delete, BanPath, BasicHeader)
+    ).
+
+t_ee_authorize_publisher(_Config) ->
+    Name = <<"EMQX-EE-API-AUTHORIZE-KEY-PUBLISHER">>,
+    {ok, #{<<"api_key">> := ApiKey, <<"api_secret">> := ApiSecret}} = create_app(Name, #{
+        role => ?ROLE_API_PUBLISHER
+    }),
+    BasicHeader = emqx_common_test_http:auth_header(
+        binary_to_list(ApiKey),
+        binary_to_list(ApiSecret)
+    ),
+
+    BanPath = emqx_mgmt_api_test_util:api_path(["banned"]),
+    Publish = emqx_mgmt_api_test_util:api_path(["publish"]),
+    ?assertMatch(
+        {error, {_, 403, _}}, emqx_mgmt_api_test_util:request_api(get, BanPath, BasicHeader)
+    ),
+    ?assertMatch(
+        {error, {_, 403, _}}, emqx_mgmt_api_test_util:request_api(delete, BanPath, BasicHeader)
+    ),
+    ?_assertMatch(
+        {ok, _},
+        emqx_mgmt_api_test_util:request_api(
+            post,
+            Publish,
+            [],
+            BasicHeader,
+            #{topic => <<"t/t_ee_authorize_publisher">>, payload => <<"hello">>}
+        )
+    ).
+
 list_app() ->
     AuthHeader = emqx_dashboard_SUITE:auth_header_(),
     Path = emqx_mgmt_api_test_util:api_path(["api_key"]),
@@ -234,10 +349,13 @@ read_app(Name) ->
     end.
 
 create_app(Name) ->
+    create_app(Name, #{}).
+
+create_app(Name, Extra) ->
     AuthHeader = emqx_dashboard_SUITE:auth_header_(),
     Path = emqx_mgmt_api_test_util:api_path(["api_key"]),
     ExpiredAt = to_rfc3339(erlang:system_time(second) + 1000),
-    App = #{
+    App = Extra#{
         name => Name,
         expired_at => ExpiredAt,
         desc => <<"Note"/utf8>>,

apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl

@@ -23,6 +23,7 @@
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -define(ROLE_SUPERUSER, <<"administrator">>).
+-define(ROLE_API_SUPERUSER, <<"api_administrator">>).
 -define(BOOTSTRAP_BACKUP, "emqx-export-test-bootstrap-ce.tar.gz").
 
 all() ->
@@ -56,7 +57,7 @@ init_per_testcase(TC = t_verify_imported_mnesia_tab_on_cluster, Config) ->
     [{cluster, cluster(TC, Config)} | setup(TC, Config)];
 init_per_testcase(t_mnesia_bad_tab_schema, Config) ->
     meck:new(emqx_mgmt_data_backup, [passthrough]),
-    meck:expect(TC = emqx_mgmt_data_backup, mnesia_tabs_to_backup, 0, [data_backup_test]),
+    meck:expect(TC = emqx_mgmt_data_backup, mnesia_tabs_to_backup, 0, [?MODULE]),
     setup(TC, Config);
 init_per_testcase(TC, Config) ->
     setup(TC, Config).
@@ -99,7 +100,15 @@ t_cluster_hocon_export_import(Config) ->
     ?assertEqual(Exp, emqx_mgmt_data_backup:import(FileName)),
     ?assertEqual(RawConfAfterImport, emqx:get_raw_config([])),
     %% lookup file inside <data_dir>/backup
-    ?assertEqual(Exp, emqx_mgmt_data_backup:import(filename:basename(FileName))).
+    ?assertEqual(Exp, emqx_mgmt_data_backup:import(filename:basename(FileName))),
+
+    %% backup data migration test
+    ?assertMatch([_, _, _], ets:tab2list(emqx_app)),
+    ?assertMatch(
+        {ok, #{name := <<"key_to_export2">>, role := ?ROLE_API_SUPERUSER}},
+        emqx_mgmt_auth:read(<<"key_to_export2">>)
+    ),
+    ok.
 
 t_ee_to_ce_backup(Config) ->
     case emqx_release:edition() of
@@ -329,6 +338,9 @@ t_verify_imported_mnesia_tab_on_cluster(Config) ->
     timer:sleep(3000),
     ?assertEqual(AllUsers, lists:sort(rpc:call(ReplicantNode, mnesia, dirty_all_keys, [Tab]))).
 
+backup_tables() ->
+    [data_backup_test].
+
 t_mnesia_bad_tab_schema(_Config) ->
     OldAttributes = [id, name, description],
     ok = create_test_tab(OldAttributes),

changes/ee/feat-11766.en.md

@@ -0,0 +1,8 @@
+Implemented a preliminary Role-Based Access Control for the REST API.
+
+  In this version, there are three predefined roles:
+  - Administrator: This role could access all resources.
+
+  - Viewer: This role can only view resources and data, corresponding to all GET requests in the REST API.
+
+  - Publisher: This role is special for MQTT messages publish, it can only access publish-related endpoints.

rel/i18n/emqx_mgmt_api_api_keys.hocon

@@ -30,4 +30,7 @@ format.desc:
 format.label:
 """Unique and format by [a-zA-Z0-9-_]"""
 
+role.desc:
+"""Role for this API"""
+
 }