From a5ddc5390f6e1b24e5bb9b1525a1a0be02ccda59 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 9 May 2022 10:11:44 +0800 Subject: [PATCH 1/4] refactor(resource): add resource recreate fun with empty opts --- apps/emqx_resource/src/emqx_resource.erl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 647c1b949..05ad3ce46 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -56,7 +56,9 @@ create_dry_run/2, create_dry_run_local/2, %% this will do create_dry_run, stop the old instance and start a new one + recreate/3, recreate/4, + recreate_local/3, recreate_local/4, %% remove the config and stop the instance remove/1, @@ -200,11 +202,21 @@ create_dry_run(ResourceType, Config) -> create_dry_run_local(ResourceType, Config) -> emqx_resource_manager:create_dry_run(ResourceType, Config). +-spec recreate(instance_id(), resource_type(), resource_config()) -> + {ok, resource_data()} | {error, Reason :: term()}. +recreate(InstId, ResourceType, Config) -> + recreate(InstId, ResourceType, Config, #{}). + -spec recreate(instance_id(), resource_type(), resource_config(), create_opts()) -> {ok, resource_data()} | {error, Reason :: term()}. recreate(InstId, ResourceType, Config, Opts) -> wrap_rpc(emqx_resource_proto_v1:recreate(InstId, ResourceType, Config, Opts)). +-spec recreate_local(instance_id(), resource_type(), resource_config()) -> + {ok, resource_data()} | {error, Reason :: term()}. +recreate_local(InstId, ResourceType, Config) -> + recreate_local(InstId, ResourceType, Config, #{}). + -spec recreate_local(instance_id(), resource_type(), resource_config(), create_opts()) -> {ok, resource_data()} | {error, Reason :: term()}. recreate_local(InstId, ResourceType, Config, Opts) -> From 87af77ec35e13d3b3024dc9c5fd4ab1ca9419eec Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 9 May 2022 13:41:07 +0800 Subject: [PATCH 2/4] refactor: do not destory resource when update authn/authz resource --- apps/emqx_authn/src/emqx_authn_utils.erl | 23 +++++ .../src/simple_authn/emqx_authn_http.erl | 70 +++++++-------- .../src/simple_authn/emqx_authn_mongodb.erl | 69 +++++++-------- .../src/simple_authn/emqx_authn_mysql.erl | 61 ++++++------- .../src/simple_authn/emqx_authn_pgsql.erl | 56 ++++++------ .../src/simple_authn/emqx_authn_redis.erl | 87 +++++++++---------- apps/emqx_authz/src/emqx_authz.erl | 39 +++++---- .../emqx_authz/src/emqx_authz_api_sources.erl | 1 + apps/emqx_authz/src/emqx_authz_file.erl | 8 +- apps/emqx_authz/src/emqx_authz_http.erl | 17 +++- apps/emqx_authz/src/emqx_authz_jwt.erl | 8 +- apps/emqx_authz/src/emqx_authz_mnesia.erl | 7 +- apps/emqx_authz/src/emqx_authz_mongodb.erl | 26 +++--- apps/emqx_authz/src/emqx_authz_mysql.erl | 20 ++++- apps/emqx_authz/src/emqx_authz_postgresql.erl | 39 +++++---- apps/emqx_authz/src/emqx_authz_redis.erl | 23 +++-- apps/emqx_authz/src/emqx_authz_utils.erl | 32 ++++++- 17 files changed, 351 insertions(+), 235 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_utils.erl b/apps/emqx_authn/src/emqx_authn_utils.erl index 5f2e0100e..bcc6f4e0e 100644 --- a/apps/emqx_authn/src/emqx_authn_utils.erl +++ b/apps/emqx_authn/src/emqx_authn_utils.erl @@ -20,6 +20,8 @@ -include_lib("emqx_authn.hrl"). -export([ + create_resource/3, + update_resource/3, check_password_from_selected_map/3, parse_deep/1, parse_str/1, @@ -47,6 +49,27 @@ %% APIs %%------------------------------------------------------------------------------ +create_resource(ResourceId, Module, Config) -> + {ok, _Data} = emqx_resource:create_local( + ResourceId, + ?RESOURCE_GROUP, + Module, + Config, + #{} + ). + +update_resource(Module, Config, ResourceId) -> + %% recreate before maybe stop + %% resource will auto start during recreate + Result = emqx_resource:recreate_local(ResourceId, Module, Config), + case Config of + #{enable := true} -> + Result; + #{enable := false} -> + ok = emqx_resource:stop(ResourceId), + Result + end. + check_password_from_selected_map(_Algorithm, _Selected, undefined) -> {error, bad_username_or_password}; check_password_from_selected_map( diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index d98e9f940..583245f79 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -158,45 +158,24 @@ refs() -> create(_AuthenticatorID, Config) -> create(Config). -create( - #{ - method := Method, - url := RawUrl, - headers := Headers, - request_timeout := RequestTimeout - } = Config -) -> - {BaseUrl0, Path, Query} = parse_url(RawUrl), - {ok, BaseUrl} = emqx_http_lib:uri_parse(BaseUrl0), +create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), - State = #{ - method => Method, - path => Path, - headers => ensure_header_name_type(Headers), - base_path_templete => emqx_authn_utils:parse_str(Path), - base_query_template => emqx_authn_utils:parse_deep( - cow_qs:parse_qs(to_bin(Query)) - ), - body_template => emqx_authn_utils:parse_deep(maps:get(body, Config, #{})), - request_timeout => RequestTimeout, - resource_id => ResourceId - }, - {ok, _Data} = emqx_resource:create_local( + {Config, State} = parse_config(Config0), + {ok, _Data} = emqx_authn_utils:create_resource( ResourceId, - ?RESOURCE_GROUP, emqx_connector_http, - Config#{ - base_url => BaseUrl, - pool_type => random - }, - #{} + Config ), - {ok, State}. + {ok, State#{resource_id => ResourceId}}. -update(Config, State) -> - {ok, NewState} = create(Config), - ok = destroy(State), - {ok, NewState}. +update(Config0, #{resource_id := ResourceId} = _State) -> + {Config, NState} = parse_config(Config0), + case emqx_authn_utils:update_resource(emqx_connector_http, Config, ResourceId) of + {error, Reason} -> + error({load_config_error, Reason}); + {ok, _} -> + {ok, NState#{resource_id => ResourceId}} + end. authenticate(#{auth_method := _}, _) -> ignore; @@ -325,6 +304,29 @@ parse_url(Url) -> throw({invalid_url, Url}) end. +parse_config( + #{ + method := Method, + url := RawUrl, + headers := Headers, + request_timeout := RequestTimeout + } = Config +) -> + {BaseUrl0, Path, Query} = parse_url(RawUrl), + {ok, BaseUrl} = emqx_http_lib:uri_parse(BaseUrl0), + State = #{ + method => Method, + path => Path, + headers => ensure_header_name_type(Headers), + base_path_templete => emqx_authn_utils:parse_str(Path), + base_query_template => emqx_authn_utils:parse_deep( + cow_qs:parse_qs(to_bin(Query)) + ), + body_template => emqx_authn_utils:parse_deep(maps:get(body, Config, #{})), + request_timeout => RequestTimeout + }, + {Config#{base_url => BaseUrl, pool_type => random}, State}. + generate_request(Credential, #{ method := Method, headers := Headers0, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index e12fcee0d..040803a0d 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -126,39 +126,28 @@ refs() -> create(_AuthenticatorID, Config) -> create(Config). -create(#{filter := Filter} = Config) -> - FilterTemplate = emqx_authn_utils:parse_deep(Filter), - State = maps:with( - [ - collection, - password_hash_field, - salt_field, - is_superuser_field, - password_hash_algorithm, - salt_position - ], +create(Config0) -> + ResourceId = emqx_authn_utils:make_resource_id(?MODULE), + {Config, State} = parse_config(Config0), + {ok, _Data} = emqx_authn_utils:create_resource( + ResourceId, + emqx_connector_mongo, Config ), - #{password_hash_algorithm := Algorithm} = State, - ok = emqx_authn_password_hashing:init(Algorithm), - ResourceId = emqx_authn_utils:make_resource_id(?MODULE), - NState = State#{ - filter_template => FilterTemplate, - resource_id => ResourceId - }, - {ok, _Data} = emqx_resource:create_local( - ResourceId, - ?RESOURCE_GROUP, - emqx_connector_mongo, - Config, - #{} - ), - {ok, NState}. + {ok, State#{resource_id => ResourceId}}. -update(Config, State) -> - {ok, NewState} = create(Config), - ok = destroy(State), - {ok, NewState}. +update(Config0, #{resource_id := ResourceId} = _State) -> + {Config, NState} = parse_config(Config0), + case emqx_authn_utils:update_resource(emqx_connector_mongo, Config, ResourceId) of + {error, Reason} -> + error({load_config_error, Reason}); + {ok, _} -> + {ok, NState#{resource_id => ResourceId}} + end. + +destroy(#{resource_id := ResourceId}) -> + _ = emqx_resource:remove_local(ResourceId), + ok. authenticate(#{auth_method := _}, _) -> ignore; @@ -201,14 +190,26 @@ authenticate( end end. -destroy(#{resource_id := ResourceId}) -> - _ = emqx_resource:remove_local(ResourceId), - ok. - %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ +parse_config(#{filter := Filter} = Config) -> + FilterTemplate = emqx_authn_utils:parse_deep(Filter), + State = maps:with( + [ + collection, + password_hash_field, + salt_field, + is_superuser_field, + password_hash_algorithm, + salt_position + ], + Config + ), + ok = emqx_authn_password_hashing:init(maps:get(password_hash_algorithm, State)), + {Config, State#{filter_template => FilterTemplate}}. + check_password(undefined, _Selected, _State) -> {error, bad_username_or_password}; check_password( diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index f7cea29d1..d183748ff 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -83,35 +83,24 @@ refs() -> create(_AuthenticatorID, Config) -> create(Config). -create( - #{ - password_hash_algorithm := Algorithm, - query := Query0, - query_timeout := QueryTimeout - } = Config -) -> - ok = emqx_authn_password_hashing:init(Algorithm), - {PrepareSql, TmplToken} = emqx_authn_utils:parse_sql(Query0, '?'), +create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), - State = #{ - password_hash_algorithm => Algorithm, - tmpl_token => TmplToken, - query_timeout => QueryTimeout, - resource_id => ResourceId - }, - {ok, _Data} = emqx_resource:create_local( - ResourceId, - ?RESOURCE_GROUP, - emqx_connector_mysql, - Config#{prepare_statement => #{?PREPARE_KEY => PrepareSql}}, - #{} - ), - {ok, State}. + {Config, State} = parse_config(Config0), + {ok, _Data} = emqx_authn_utils:create_resource(ResourceId, emqx_connector_mysql, Config), + {ok, State#{resource_id => ResourceId}}. -update(Config, State) -> - {ok, NewState} = create(Config), - ok = destroy(State), - {ok, NewState}. +update(Config0, #{resource_id := ResourceId} = _State) -> + {Config, NState} = parse_config(Config0), + case emqx_authn_utils:update_resource(emqx_connector_mysql, Config, ResourceId) of + {error, Reason} -> + error({load_config_error, Reason}); + {ok, _} -> + {ok, NState#{resource_id => ResourceId}} + end. + +destroy(#{resource_id := ResourceId}) -> + _ = emqx_resource:remove_local(ResourceId), + ok. authenticate(#{auth_method := _}, _) -> ignore; @@ -152,6 +141,18 @@ authenticate( ignore end. -destroy(#{resource_id := ResourceId}) -> - _ = emqx_resource:remove_local(ResourceId), - ok. +parse_config( + #{ + password_hash_algorithm := Algorithm, + query := Query0, + query_timeout := QueryTimeout + } = Config +) -> + ok = emqx_authn_password_hashing:init(Algorithm), + {PrepareSql, TmplToken} = emqx_authn_utils:parse_sql(Query0, '?'), + State = #{ + password_hash_algorithm => Algorithm, + tmpl_token => TmplToken, + query_timeout => QueryTimeout + }, + {Config#{prepare_statement => #{?PREPARE_KEY => PrepareSql}}, State}. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index ad248c3c8..99f4fa43c 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -82,33 +82,28 @@ refs() -> create(_AuthenticatorID, Config) -> create(Config). -create( - #{ - query := Query0, - password_hash_algorithm := Algorithm - } = Config -) -> - ok = emqx_authn_password_hashing:init(Algorithm), - {Query, PlaceHolders} = emqx_authn_utils:parse_sql(Query0, '$n'), +create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), - State = #{ - placeholders => PlaceHolders, - password_hash_algorithm => Algorithm, - resource_id => ResourceId - }, - {ok, _Data} = emqx_resource:create_local( + {Config, State} = parse_config(Config0, ResourceId), + {ok, _Data} = emqx_authn_utils:create_resource( ResourceId, - ?RESOURCE_GROUP, emqx_connector_pgsql, - Config#{prepare_statement => #{ResourceId => Query}}, - #{} + Config ), - {ok, State}. + {ok, State#{resource_id => ResourceId}}. -update(Config, State) -> - {ok, NewState} = create(Config), - ok = destroy(State), - {ok, NewState}. +update(Config0, #{resource_id := ResourceId} = _State) -> + {Config, NState} = parse_config(Config0, ResourceId), + case emqx_authn_utils:update_resource(emqx_connector_pgsql, Config, ResourceId) of + {error, Reason} -> + error({load_config_error, Reason}); + {ok, _} -> + {ok, NState#{resource_id => ResourceId}} + end. + +destroy(#{resource_id := ResourceId}) -> + _ = emqx_resource:remove_local(ResourceId), + ok. authenticate(#{auth_method := _}, _) -> ignore; @@ -147,6 +142,17 @@ authenticate( ignore end. -destroy(#{resource_id := ResourceId}) -> - _ = emqx_resource:remove_local(ResourceId), - ok. +parse_config( + #{ + query := Query0, + password_hash_algorithm := Algorithm + } = Config, + ResourceId +) -> + ok = emqx_authn_password_hashing:init(Algorithm), + {Query, PlaceHolders} = emqx_authn_utils:parse_sql(Query0, '$n'), + State = #{ + placeholders => PlaceHolders, + password_hash_algorithm => Algorithm + }, + {Config#{prepare_statement => #{ResourceId => Query}}, State}. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index 44754f8eb..6c0839a1b 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -96,51 +96,33 @@ refs() -> create(_AuthenticatorID, Config) -> create(Config). -create( - #{ - cmd := Cmd, - password_hash_algorithm := Algorithm - } = Config -) -> - ok = emqx_authn_password_hashing:init(Algorithm), - try - NCmd = parse_cmd(Cmd), - ok = emqx_authn_utils:ensure_apps_started(Algorithm), - State = maps:with( - [password_hash_algorithm, salt_position], - Config - ), - ResourceId = emqx_authn_utils:make_resource_id(?MODULE), - NState = State#{ - cmd => NCmd, - resource_id => ResourceId - }, - {ok, _Data} = emqx_resource:create_local( - ResourceId, - ?RESOURCE_GROUP, - emqx_connector_redis, - Config, - #{} - ), - {ok, NState} - catch - error:{unsupported_cmd, _Cmd} -> - {error, {unsupported_cmd, Cmd}}; - error:missing_password_hash -> - {error, missing_password_hash}; - error:{unsupported_fields, Fields} -> - {error, {unsupported_fields, Fields}} +create(Config0) -> + ResourceId = emqx_authn_utils:make_resource_id(?MODULE), + case parse_config(Config0) of + {error, _} = Res -> + Res; + {Config, State} -> + {ok, _Data} = emqx_authn_utils:create_resource( + ResourceId, + emqx_connector_redis, + Config + ), + {ok, State#{resource_id => ResourceId}} end. -update(Config, State) -> - case create(Config) of - {ok, NewState} -> - ok = destroy(State), - {ok, NewState}; +update(Config0, #{resource_id := ResourceId} = _State) -> + {Config, NState} = parse_config(Config0), + case emqx_authn_utils:update_resource(emqx_connector_redis, Config, ResourceId) of {error, Reason} -> - {error, Reason} + error({load_config_error, Reason}); + {ok, _} -> + {ok, NState#{resource_id => ResourceId}} end. +destroy(#{resource_id := ResourceId}) -> + _ = emqx_resource:remove_local(ResourceId), + ok. + authenticate(#{auth_method := _}, _) -> ignore; authenticate( @@ -190,14 +172,31 @@ authenticate( ignore end. -destroy(#{resource_id := ResourceId}) -> - _ = emqx_resource:remove_local(ResourceId), - ok. - %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ +parse_config( + #{ + cmd := Cmd, + password_hash_algorithm := Algorithm + } = Config +) -> + try + NCmd = parse_cmd(Cmd), + ok = emqx_authn_password_hashing:init(Algorithm), + ok = emqx_authn_utils:ensure_apps_started(Algorithm), + State = maps:with([password_hash_algorithm, salt_position], Config), + {Config, State#{cmd => NCmd}} + catch + error:{unsupported_cmd, _Cmd} -> + {error, {unsupported_cmd, Cmd}}; + error:missing_password_hash -> + {error, missing_password_hash}; + error:{unsupported_fields, Fields} -> + {error, {unsupported_fields, Fields}} + end. + %% Only support HGET and HMGET parse_cmd(Cmd) -> case string:tokens(Cmd, " ") of diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 1ed0b7b02..e3e3bcaa5 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -63,16 +63,20 @@ %% Initialize authz backend. %% Populate the passed configuration map with necessary data, %% like `ResourceID`s --callback init(source()) -> source(). +-callback create(source()) -> source(). -%% Get authz text description. --callback description() -> string(). +%% Update authz backend. +%% Change configuration, or simply enable/disable +-callback update(source()) -> source(). %% Destroy authz backend. %% Make cleanup of all allocated data. %% An authz backend will not be used after `destroy`. -callback destroy(source()) -> ok. +%% Get authz text description. +-callback description() -> string(). + %% Authorize client action. -callback authorize( emqx_types:clientinfo(), @@ -81,6 +85,10 @@ source() ) -> match_result(). +-optional_callbacks([ + update/1 +]). + -spec register_metrics() -> ok. register_metrics() -> lists:foreach(fun emqx_metrics:ensure/1, ?METRICS). @@ -90,7 +98,7 @@ init() -> emqx_conf:add_handler(?CONF_KEY_PATH, ?MODULE), Sources = emqx_conf:get(?CONF_KEY_PATH, []), ok = check_dup_types(Sources), - NSources = init_sources(Sources), + NSources = create_sources(Sources), ok = emqx_hooks:add('client.authorize', {?MODULE, authorize, [NSources]}, -1). deinit() -> @@ -170,7 +178,7 @@ do_post_config_update({?CMD_MOVE, _Type, _Where} = Cmd, _Sources) -> InitedSources = lookup(), do_move(Cmd, InitedSources); do_post_config_update({?CMD_PREPEND, RawNewSource}, Sources) -> - InitedNewSource = init_source(get_source_by_type(type(RawNewSource), Sources)), + InitedNewSource = create_source(get_source_by_type(type(RawNewSource), Sources)), %% create metrics TypeName = type(RawNewSource), ok = emqx_metrics_worker:create_metrics( @@ -181,14 +189,13 @@ do_post_config_update({?CMD_PREPEND, RawNewSource}, Sources) -> ), [InitedNewSource] ++ lookup(); do_post_config_update({?CMD_APPEND, RawNewSource}, Sources) -> - InitedNewSource = init_source(get_source_by_type(type(RawNewSource), Sources)), + InitedNewSource = create_source(get_source_by_type(type(RawNewSource), Sources)), lookup() ++ [InitedNewSource]; do_post_config_update({{?CMD_REPLACE, Type}, RawNewSource}, Sources) -> OldSources = lookup(), {OldSource, Front, Rear} = take(Type, OldSources), NewSource = get_source_by_type(type(RawNewSource), Sources), - ok = ensure_resource_deleted(OldSource), - InitedSources = init_source(NewSource), + InitedSources = update_source(type(RawNewSource), OldSource, NewSource), Front ++ [InitedSources] ++ Rear; do_post_config_update({{?CMD_DELETE, Type}, _RawNewSource}, _Sources) -> OldInitedSources = lookup(), @@ -203,7 +210,7 @@ do_post_config_update({?CMD_REPLACE, _RawNewSources}, Sources) -> OldInitedSources = lookup(), lists:foreach(fun ensure_resource_deleted/1, OldInitedSources), lists:foreach(fun clear_certs/1, OldInitedSources), - init_sources(Sources). + create_sources(Sources). %% @doc do source move do_move({?CMD_MOVE, Type, ?CMD_MOVE_FRONT}, Sources) -> @@ -251,20 +258,22 @@ check_dup_types([Source | Sources], Checked) -> check_dup_types(Sources, [Type | Checked]) end. -init_sources(Sources) -> +create_sources(Sources) -> {_Enabled, Disabled} = lists:partition(fun(#{enable := Enable}) -> Enable end, Sources), case Disabled =/= [] of true -> ?SLOG(info, #{msg => "disabled_sources_ignored", sources => Disabled}); false -> ok end, ok = lists:foreach(fun init_metrics/1, Sources), - lists:map(fun init_source/1, Sources). + lists:map(fun create_source/1, Sources). -init_source(#{enable := false} = Source) -> - Source; -init_source(#{type := Type} = Source) -> +create_source(#{type := Type} = Source) -> Module = authz_module(Type), - Module:init(Source). + Module:create(Source). + +update_source(Type, OldSource, NewSource) -> + Module = authz_module(Type), + Module:update(maps:merge(OldSource, NewSource)). init_metrics(Source) -> TypeName = type(Source), diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index afbb7195f..f4696c0a6 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -317,6 +317,7 @@ lookup_from_local_node(Type) -> end; _ -> Metrics = emqx_metrics_worker:get_metrics(authz_metrics, Type), + %% for authz file/authz mnesia {ok, {NodeId, connected, Metrics, #{}}} catch _:Reason -> {error, {NodeId, list_to_binary(io_lib:format("~p", [Reason]))}} diff --git a/apps/emqx_authz/src/emqx_authz_file.erl b/apps/emqx_authz/src/emqx_authz_file.erl index 38ff8447a..fe3150048 100644 --- a/apps/emqx_authz/src/emqx_authz_file.erl +++ b/apps/emqx_authz/src/emqx_authz_file.erl @@ -29,7 +29,8 @@ %% APIs -export([ description/0, - init/1, + create/1, + update/1, destroy/1, authorize/4 ]). @@ -37,7 +38,7 @@ description() -> "AuthZ with static rules". -init(#{path := Path} = Source) -> +create(#{path := Path} = Source) -> Rules = case file:consult(Path) of {ok, Terms} -> @@ -55,6 +56,9 @@ init(#{path := Path} = Source) -> end, Source#{annotations => #{rules => Rules}}. +update(#{path := _Path} = Source) -> + create(Source). + destroy(_Source) -> ok. authorize(Client, PubSub, Topic, #{annotations := #{rules := Rules}}) -> diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index 3c17312ad..0f9c4dcce 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -26,7 +26,8 @@ %% AuthZ Callbacks -export([ description/0, - init/1, + create/1, + update/1, destroy/1, authorize/4, parse_url/1 @@ -50,10 +51,18 @@ description() -> "AuthZ with http". -init(Config) -> +create(Config) -> NConfig = parse_config(Config), - {ok, Id} = emqx_authz_utils:create_resource(emqx_connector_http, NConfig), - NConfig#{annotations => #{id => Id}}. + ResourceId = emqx_authn_utils:make_resource_id(?MODULE), + {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_connector_http, NConfig), + NConfig#{annotations => #{id => ResourceId}}. + +update(Config) -> + NConfig = parse_config(Config), + case emqx_authz_utils:update_resource(emqx_connector_http, NConfig) of + {error, Reason} -> error({load_config_error, Reason}); + {ok, Id} -> NConfig#{annotations => #{id => Id}} + end. destroy(#{annotations := #{id := Id}}) -> ok = emqx_resource:remove_local(Id). diff --git a/apps/emqx_authz/src/emqx_authz_jwt.erl b/apps/emqx_authz/src/emqx_authz_jwt.erl index 5f8e2f45d..132ffe891 100644 --- a/apps/emqx_authz/src/emqx_authz_jwt.erl +++ b/apps/emqx_authz/src/emqx_authz_jwt.erl @@ -28,7 +28,8 @@ %% APIs -export([ description/0, - init/1, + create/1, + update/1, destroy/1, authorize/4 ]). @@ -46,7 +47,10 @@ description() -> "AuthZ with JWT". -init(#{acl_claim_name := _AclClaimName} = Source) -> +create(#{acl_claim_name := _AclClaimName} = Source) -> + Source. + +update(#{acl_claim_name := _AclClaimName} = Source) -> Source. destroy(_Source) -> ok. diff --git a/apps/emqx_authz/src/emqx_authz_mnesia.erl b/apps/emqx_authz/src/emqx_authz_mnesia.erl index 267338493..4a5257ecb 100644 --- a/apps/emqx_authz/src/emqx_authz_mnesia.erl +++ b/apps/emqx_authz/src/emqx_authz_mnesia.erl @@ -46,7 +46,8 @@ %% AuthZ Callbacks -export([ description/0, - init/1, + create/1, + update/1, destroy/1, authorize/4 ]). @@ -88,7 +89,9 @@ mnesia(boot) -> description() -> "AuthZ with Mnesia". -init(Source) -> Source. +create(Source) -> Source. + +update(Source) -> Source. destroy(_Source) -> ok. diff --git a/apps/emqx_authz/src/emqx_authz_mongodb.erl b/apps/emqx_authz/src/emqx_authz_mongodb.erl index 7c79897e6..ac450e4cc 100644 --- a/apps/emqx_authz/src/emqx_authz_mongodb.erl +++ b/apps/emqx_authz/src/emqx_authz_mongodb.erl @@ -26,7 +26,8 @@ %% AuthZ Callbacks -export([ description/0, - init/1, + create/1, + update/1, destroy/1, authorize/4 ]). @@ -45,15 +46,20 @@ description() -> "AuthZ with MongoDB". -init(#{filter := Filter} = Source) -> - {ok, Id} = emqx_authz_utils:create_resource(emqx_connector_mongo, Source), - Source#{ - annotations => #{id => Id}, - filter_template => emqx_authz_utils:parse_deep( - Filter, - ?PLACEHOLDERS - ) - }. +create(#{filter := Filter} = Source) -> + ResourceId = emqx_authz_utils:make_resource_id(?MODULE), + {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_connector_mongo, Source), + FilterTemp = emqx_authz_utils:parse_deep(Filter, ?PLACEHOLDERS), + Source#{annotations => #{id => ResourceId}, filter_template => FilterTemp}. + +update(#{filter := Filter} = Source) -> + FilterTemp = emqx_authz_utils:parse_deep(Filter, ?PLACEHOLDERS), + case emqx_authz_utils:update_resource(emqx_connector_mongo, Source) of + {error, Reason} -> + error({load_config_error, Reason}); + {ok, Id} -> + Source#{annotations => #{id => Id}, filter_template => FilterTemp} + end. destroy(#{annotations := #{id := Id}}) -> ok = emqx_resource:remove_local(Id). diff --git a/apps/emqx_authz/src/emqx_authz_mysql.erl b/apps/emqx_authz/src/emqx_authz_mysql.erl index 54a85885c..df3e5c74d 100644 --- a/apps/emqx_authz/src/emqx_authz_mysql.erl +++ b/apps/emqx_authz/src/emqx_authz_mysql.erl @@ -28,7 +28,8 @@ %% AuthZ Callbacks -export([ description/0, - init/1, + create/1, + update/1, destroy/1, authorize/4 ]). @@ -49,11 +50,22 @@ description() -> "AuthZ with Mysql". -init(#{query := SQL} = Source0) -> +create(#{query := SQL} = Source0) -> + {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?PLACEHOLDERS), + ResourceId = emqx_authz_utils:make_resource_id(?MODULE), + Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, + {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_connector_mysql, Source), + Source#{annotations => #{id => ResourceId, tmpl_oken => TmplToken}}. + +update(#{query := SQL} = Source0) -> {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?PLACEHOLDERS), Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, - {ok, Id} = emqx_authz_utils:create_resource(emqx_connector_mysql, Source), - Source#{annotations => #{id => Id, tmpl_oken => TmplToken}}. + case emqx_authz_utils:update_resource(emqx_connector_mysql, Source) of + {error, Reason} -> + error({load_config_error, Reason}); + {ok, Id} -> + Source#{annotations => #{id => Id, tmpl_oken => TmplToken}} + end. destroy(#{annotations := #{id := Id}}) -> ok = emqx_resource:remove_local(Id). diff --git a/apps/emqx_authz/src/emqx_authz_postgresql.erl b/apps/emqx_authz/src/emqx_authz_postgresql.erl index 2543620b7..e4e8d13ba 100644 --- a/apps/emqx_authz/src/emqx_authz_postgresql.erl +++ b/apps/emqx_authz/src/emqx_authz_postgresql.erl @@ -26,7 +26,8 @@ %% AuthZ Callbacks -export([ description/0, - init/1, + create/1, + update/1, destroy/1, authorize/4 ]). @@ -47,27 +48,29 @@ description() -> "AuthZ with PostgreSQL". -init(#{query := SQL0} = Source) -> - {SQL, PlaceHolders} = emqx_authz_utils:parse_sql( - SQL0, - '$n', - ?PLACEHOLDERS - ), +create(#{query := SQL0} = Source) -> + {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?PLACEHOLDERS), ResourceID = emqx_authz_utils:make_resource_id(emqx_connector_pgsql), - {ok, _Data} = emqx_resource:create_local( + {ok, _Data} = emqx_authz_utils:create_resource( ResourceID, - ?RESOURCE_GROUP, emqx_connector_pgsql, - Source#{prepare_statement => #{ResourceID => SQL}}, - #{} + Source#{prepare_statement => #{ResourceID => SQL}} ), - Source#{ - annotations => - #{ - id => ResourceID, - placeholders => PlaceHolders - } - }. + Source#{annotations => #{id => ResourceID, placeholders => PlaceHolders}}. + +update(#{query := SQL0, annotations := #{id := ResourceID}} = Source) -> + {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?PLACEHOLDERS), + case + emqx_authz_utils:update_resource( + emqx_connector_pgsql, + Source#{prepare_statement => #{ResourceID => SQL}} + ) + of + {error, Reason} -> + error({load_config_error, Reason}); + {ok, Id} -> + Source#{annotations => #{id => Id, placeholders => PlaceHolders}} + end. destroy(#{annotations := #{id := Id}}) -> ok = emqx_resource:remove_local(Id). diff --git a/apps/emqx_authz/src/emqx_authz_redis.erl b/apps/emqx_authz/src/emqx_authz_redis.erl index 27d9d31bc..4298510ee 100644 --- a/apps/emqx_authz/src/emqx_authz_redis.erl +++ b/apps/emqx_authz/src/emqx_authz_redis.erl @@ -26,7 +26,8 @@ %% AuthZ Callbacks -export([ description/0, - init/1, + create/1, + update/1, destroy/1, authorize/4 ]). @@ -47,14 +48,22 @@ description() -> "AuthZ with Redis". -init(#{cmd := CmdStr} = Source) -> +create(#{cmd := CmdStr} = Source) -> + Cmd = tokens(CmdStr), + ResourceId = emqx_authz_utils:make_resource_id(?MODULE), + CmdTemplate = emqx_authz_utils:parse_deep(Cmd, ?PLACEHOLDERS), + {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_connector_redis, Source), + Source#{annotations => #{id => ResourceId}, cmd_template => CmdTemplate}. + +update(#{cmd := CmdStr} = Source) -> Cmd = tokens(CmdStr), CmdTemplate = emqx_authz_utils:parse_deep(Cmd, ?PLACEHOLDERS), - {ok, Id} = emqx_authz_utils:create_resource(emqx_connector_redis, Source), - Source#{ - annotations => #{id => Id}, - cmd_template => CmdTemplate - }. + case emqx_authz_utils:update_resource(emqx_connector_redis, Source) of + {error, Reason} -> + error({load_config_error, Reason}); + {ok, Id} -> + Source#{annotations => #{id => Id}, cmd_template => CmdTemplate} + end. destroy(#{annotations := #{id := Id}}) -> ok = emqx_resource:remove_local(Id). diff --git a/apps/emqx_authz/src/emqx_authz_utils.erl b/apps/emqx_authz/src/emqx_authz_utils.erl index 1d95c7936..a24df28cd 100644 --- a/apps/emqx_authz/src/emqx_authz_utils.erl +++ b/apps/emqx_authz/src/emqx_authz_utils.erl @@ -23,6 +23,8 @@ cleanup_resources/0, make_resource_id/1, create_resource/2, + create_resource/3, + update_resource/2, update_config/2, parse_deep/2, parse_str/2, @@ -37,15 +39,37 @@ %%------------------------------------------------------------------------------ create_resource(Module, Config) -> - ResourceID = make_resource_id(Module), + ResourceId = make_resource_id(Module), + create_resource(ResourceId, Module, Config). + +create_resource(ResourceId, Module, Config) -> {ok, _Data} = emqx_resource:create_local( - ResourceID, + ResourceId, ?RESOURCE_GROUP, Module, Config, #{} - ), - {ok, ResourceID}. + ). + +update_resource(Module, #{annotations := #{id := ResourceId}} = Source) -> + Result = + case + emqx_resource:recreate_local( + ResourceId, + Module, + Source + ) + of + {ok, _} -> {ok, ResourceId}; + {error, Reason} -> {error, Reason} + end, + case Source of + #{enable := true} -> + Result; + #{enable := false} -> + ok = emqx_resource:stop(ResourceId), + Result + end. cleanup_resources() -> lists:foreach( From 4fa84cce29672125ff1f545990c76addf14f9ef9 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 10 May 2022 19:08:27 +0800 Subject: [PATCH 3/4] test(authn): test unsupported redis CMD --- apps/emqx_authn/test/emqx_authn_redis_SUITE.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl index 26ca03ea4..58d1cbc0c 100644 --- a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl @@ -107,6 +107,9 @@ t_create_invalid(_Config) -> }, AuthConfig#{ cmd => <<"HMGET mqtt_user:${username} salt is_superuser">> + }, + AuthConfig#{ + cmd => <<"HGETALL mqtt_user:${username} salt is_superuser">> } ], lists:foreach( From c181efd0ae124f80f495ddaaa48d1a913b7a0921 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 12 May 2022 14:39:12 +0800 Subject: [PATCH 4/4] test(gateway): fix authz callback rename --- apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl index 7a5049d6c..eb67885a4 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl @@ -68,7 +68,7 @@ init_per_suite(Config) -> emqx_config:erase(gateway), init_gateway_conf(), meck:new(emqx_authz_file, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_authz_file, init, fun(S) -> S end), + meck:expect(emqx_authz_file, create, fun(S) -> S end), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authz, emqx_gateway]), application:ensure_all_started(cowboy), emqx_gateway_auth_ct:start(),