From 000020617c3bed6e3e8ff957c984981d8217b303 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 16 Feb 2022 09:30:25 +0800 Subject: [PATCH 1/5] refactor: authz_api_sources spec --- .../src/emqx_limiter/etc/emqx_limiter.conf | 2 +- apps/emqx_authz/src/emqx_authz_api_schema.erl | 664 +++++------------- .../emqx_authz/src/emqx_authz_api_sources.erl | 402 ++++------- 3 files changed, 314 insertions(+), 754 deletions(-) diff --git a/apps/emqx/src/emqx_limiter/etc/emqx_limiter.conf b/apps/emqx/src/emqx_limiter/etc/emqx_limiter.conf index 78cb35207..0d28fb106 100644 --- a/apps/emqx/src/emqx_limiter/etc/emqx_limiter.conf +++ b/apps/emqx/src/emqx_limiter/etc/emqx_limiter.conf @@ -1,5 +1,5 @@ ##-------------------------------------------------------------------- -## Emq X Rate Limiter +## EMQX Rate Limiter ##-------------------------------------------------------------------- limiter { diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index 2b6014aae..4c6850f7b 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -16,490 +16,184 @@ -module(emqx_authz_api_schema). --export([definitions/0]). +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx_connector/include/emqx_connector.hrl"). -definitions() -> - Sources = #{ - 'oneOf' => [ minirest:ref(<<"http">>) - , minirest:ref(<<"built-in-database">>) - , minirest:ref(<<"mongo_single">>) - , minirest:ref(<<"mongo_rs">>) - , minirest:ref(<<"mongo_sharded">>) - , minirest:ref(<<"mysql">>) - , minirest:ref(<<"postgresql">>) - , minirest:ref(<<"redis_single">>) - , minirest:ref(<<"redis_sentinel">>) - , minirest:ref(<<"redis_cluster">>) - , minirest:ref(<<"file">>) - ] - }, - SSL = #{ - type => object, - required => [enable], - properties => #{ - enable => #{type => boolean, example => true}, - cacertfile => #{type => string}, - keyfile => #{type => string}, - certfile => #{type => string}, - verify => #{type => boolean, example => false} - } - }, - HTTP = #{ - type => object, - required => [ type - , enable - , method - , headers - , request_timeout - , connect_timeout - , max_retries - , retry_interval - , pool_type - , pool_size - , enable_pipelining - , ssl - ], - properties => #{ - type => #{ - type => string, - enum => [<<"http">>], - example => <<"http">> - }, - enable => #{ - type => boolean, - example => true - }, - url => #{ - type => string, - example => <<"https://emqx.com">> - }, - method => #{ - type => string, - enum => [<<"get">>, <<"post">>], - example => <<"get">> - }, - headers => #{type => object}, - body => #{type => object}, - connect_timeout => #{type => string}, - max_retries => #{type => integer}, - retry_interval => #{type => string}, - pool_type => #{ - type => string, - enum => [<<"random">>, <<"hash">>], - example => <<"hash">> - }, - pool_size => #{type => integer}, - enable_pipelining => #{type => boolean}, - ssl => minirest:ref(<<"ssl">>) - } - }, - MongoSingle= #{ - type => object, - required => [ type - , enable - , collection - , selector - , mongo_type - , server - , pool_size - , username - , password - , auth_source - , database - , topology - , ssl - ], - properties => #{ - type => #{ - type => string, - enum => [<<"mongodb">>], - example => <<"mongodb">> - }, - enable => #{ - type => boolean, - example => true - }, - srv_record => #{type => boolean, example => false, default => false}, - collection => #{type => string}, - selector => #{type => object}, - mongo_type => #{type => string, - enum => [<<"single">>], - example => <<"single">>}, - server => #{type => string, example => <<"127.0.0.1:27017">>}, - pool_size => #{type => integer}, - username => #{type => string}, - password => #{type => string}, - auth_source => #{type => string}, - database => #{type => string}, - topology => #{type => object, - properties => #{ - pool_size => #{type => integer}, - max_overflow => #{type => integer}, - overflow_ttl => #{type => string}, - overflow_check_period => #{type => string}, - local_threshold_ms => #{type => integer}, - connect_timeout_ms => #{type => integer}, - socket_timeout_ms => #{type => integer}, - server_selection_timeout_ms => #{type => integer}, - wait_queue_timeout_ms => #{type => integer}, - heartbeat_frequency_ms => #{type => integer}, - min_heartbeat_frequency_ms => #{type => integer} - } - }, - ssl => minirest:ref(<<"ssl">>) - } - }, - MongoRs= #{ - type => object, - required => [ type - , enable - , collection - , selector - , mongo_type - , servers - , replica_set_name - , pool_size - , username - , password - , auth_source - , database - , topology - , ssl - ], - properties => #{ - type => #{ - type => string, - enum => [<<"mongodb">>], - example => <<"mongodb">> - }, - enable => #{ - type => boolean, - example => true - }, - srv_record => #{type => boolean, example => false, default => false}, - collection => #{type => string}, - selector => #{type => object}, - mongo_type => #{type => string, - enum => [<<"rs">>], - example => <<"rs">>}, - servers => #{type => string, example => <<"127.0.0.1:27017, 127.0.0.2:27017">>}, - replica_set_name => #{type => string}, - pool_size => #{type => integer}, - username => #{type => string}, - password => #{type => string}, - auth_source => #{type => string}, - database => #{type => string}, - topology => #{type => object, - properties => #{ - pool_size => #{type => integer}, - max_overflow => #{type => integer}, - overflow_ttl => #{type => string}, - overflow_check_period => #{type => string}, - local_threshold_ms => #{type => integer}, - connect_timeout_ms => #{type => integer}, - socket_timeout_ms => #{type => integer}, - server_selection_timeout_ms => #{type => integer}, - wait_queue_timeout_ms => #{type => integer}, - heartbeat_frequency_ms => #{type => integer}, - min_heartbeat_frequency_ms => #{type => integer} - } - }, - ssl => minirest:ref(<<"ssl">>) - } - }, - MongoSharded = #{ - type => object, - required => [ type - , enable - , collection - , selector - , mongo_type - , servers - , pool_size - , username - , password - , auth_source - , database - , topology - , ssl - ], - properties => #{ - type => #{ - type => string, - enum => [<<"mongodb">>], - example => <<"mongodb">> - }, - enable => #{ - type => boolean, - example => true - }, - srv_record => #{type => boolean, example => false, default => false}, - collection => #{type => string}, - selector => #{type => object}, - mongo_type => #{type => string, - enum => [<<"sharded">>], - example => <<"sharded">>}, - servers => #{type => string,example => <<"127.0.0.1:27017, 127.0.0.2:27017">>}, - pool_size => #{type => integer}, - username => #{type => string}, - password => #{type => string}, - auth_source => #{type => string}, - database => #{type => string}, - topology => #{type => object, - properties => #{ - pool_size => #{type => integer}, - max_overflow => #{type => integer}, - overflow_ttl => #{type => string}, - overflow_check_period => #{type => string}, - local_threshold_ms => #{type => integer}, - connect_timeout_ms => #{type => integer}, - socket_timeout_ms => #{type => integer}, - server_selection_timeout_ms => #{type => integer}, - wait_queue_timeout_ms => #{type => integer}, - heartbeat_frequency_ms => #{type => integer}, - min_heartbeat_frequency_ms => #{type => integer} - } - }, - ssl => minirest:ref(<<"ssl">>) - } - }, - Mysql = #{ - type => object, - required => [ type - , enable - , query - , server - , database - , pool_size - , username - , password - , auto_reconnect - , ssl - ], - properties => #{ - type => #{ - type => string, - enum => [<<"mysql">>], - example => <<"mysql">> - }, - enable => #{ - type => boolean, - example => true - }, - query => #{type => string}, - server => #{type => string, - example => <<"127.0.0.1:3306">> - }, - database => #{type => string}, - pool_size => #{type => integer}, - username => #{type => string}, - password => #{type => string}, - auto_reconnect => #{type => boolean, - example => true - }, - ssl => minirest:ref(<<"ssl">>) - } - }, - Pgsql = #{ - type => object, - required => [ type - , enable - , query - , server - , database - , pool_size - , username - , password - , auto_reconnect - , ssl - ], - properties => #{ - type => #{ - type => string, - enum => [<<"postgresql">>], - example => <<"postgresql">> - }, - enable => #{ - type => boolean, - example => true - }, - query => #{type => string}, - server => #{type => string, - example => <<"127.0.0.1:5432">> - }, - database => #{type => string}, - pool_size => #{type => integer}, - username => #{type => string}, - password => #{type => string}, - auto_reconnect => #{type => boolean, - example => true - }, - ssl => minirest:ref(<<"ssl">>) - } - }, - RedisSingle = #{ - type => object, - required => [ type - , enable - , cmd - , server - , redis_type - , pool_size - , auto_reconnect - , ssl - ], - properties => #{ - type => #{ - type => string, - enum => [<<"redis">>], - example => <<"redis">> - }, - enable => #{ - type => boolean, - example => true - }, - cmd => #{ - type => string, - example => <<"HGETALL mqtt_authz">> - }, - server => #{type => string, example => <<"127.0.0.1:3306">>}, - redis_type => #{type => string, - enum => [<<"single">>], - example => <<"single">>}, - pool_size => #{type => integer}, - auto_reconnect => #{type => boolean, example => true}, - password => #{type => string}, - database => #{type => integer}, - ssl => minirest:ref(<<"ssl">>) - } - }, - RedisSentinel= #{ - type => object, - required => [ type - , enable - , cmd - , servers - , redis_type - , sentinel - , pool_size - , auto_reconnect - , ssl - ], - properties => #{ - type => #{ - type => string, - enum => [<<"redis">>], - example => <<"redis">> - }, - enable => #{ - type => boolean, - example => true - }, - cmd => #{ - type => string, - example => <<"HGETALL mqtt_authz">> - }, - servers => #{type => string, example => <<"127.0.0.1:6379, 127.0.0.2:6379">>}, - redis_type => #{type => string, - enum => [<<"sentinel">>], - example => <<"sentinel">>}, - sentinel => #{type => string}, - pool_size => #{type => integer}, - auto_reconnect => #{type => boolean, example => true}, - password => #{type => string}, - database => #{type => integer}, - ssl => minirest:ref(<<"ssl">>) - } - }, - RedisCluster= #{ - type => object, - required => [ type - , enable - , cmd - , servers - , redis_type - , pool_size - , auto_reconnect - , ssl], - properties => #{ - type => #{ - type => string, - enum => [<<"redis">>], - example => <<"redis">> - }, - enable => #{ - type => boolean, - example => true - }, - cmd => #{ - type => string, - example => <<"HGETALL mqtt_authz">> - }, - servers => #{type => string, example => <<"127.0.0.1:6379, 127.0.0.2:6379">>}, - redis_type => #{type => string, - enum => [<<"cluster">>], - example => <<"cluster">>}, - pool_size => #{type => integer}, - auto_reconnect => #{type => boolean, example => true}, - password => #{type => string}, - database => #{type => integer}, - ssl => minirest:ref(<<"ssl">>) - } - }, - Mnesia = #{ - type => object, - required => [type, enable], - properties => #{ - type => #{ - type => string, - enum => [<<"redis">>], - example => <<"redis">> - }, - enable => #{ - type => boolean, - example => true - } - } - }, - File = #{ - type => object, - required => [type, enable, rules], - properties => #{ - type => #{ - type => string, - enum => [<<"redis">>], - example => <<"redis">> - }, - enable => #{ - type => boolean, - example => true - }, - rules => #{ - type => array, - items => #{ - type => string, - example => - <<"{allow,{username,\"^dashboard?\"},","subscribe,[\"$SYS/#\"]}.\n", - "{allow,{ipaddr,\"127.0.0.1\"},all,[\"$SYS/#\",\"#\"]}.">> - } - }, - path => #{ - type => string, - example => <<"/path/to/authorizaiton_rules.conf">> - } - } - }, - [ #{<<"sources">> => Sources} - , #{<<"ssl">> => SSL} - , #{<<"http">> => HTTP} - , #{<<"built-in-database">> => Mnesia} - , #{<<"mongo_single">> => MongoSingle} - , #{<<"mongo_rs">> => MongoRs} - , #{<<"mongo_sharded">> => MongoSharded} - , #{<<"mysql">> => Mysql} - , #{<<"postgresql">> => Pgsql} - , #{<<"redis_single">> => RedisSingle} - , #{<<"redis_sentinel">> => RedisSentinel} - , #{<<"redis_cluster">> => RedisCluster} - , #{<<"file">> => File} +-import(hoconsc, [mk/2, ref/1, ref/2, array/1, enum/1]). +-import(emqx_schema, [mk_duration/2]). + +-export([fields/1, authz_sources_types/1]). + +fields(http) -> + authz_common_fields(http) + ++ [ {url, fun url/1} + , {method, #{ type => enum([get, post]) + , default => get + , converter => fun to_bin/1}} + , {headers, fun headers/1} + , {body, fun body/1} + , {request_timeout, mk_duration("Request timeout", #{default => "30s"})}] + ++ maps:to_list(maps:without([ base_url + , pool_type], + maps:from_list(emqx_connector_http:fields(config)))); +fields('built-in-database') -> + authz_common_fields('built-in-database'); +fields(mongo_single) -> + authz_mongo_common_fields() + ++ emqx_connector_mongo:fields(single); +fields(mongo_rs) -> + authz_mongo_common_fields() + ++ emqx_connector_mongo:fields(rs); +fields(mongo_sharded) -> + authz_mongo_common_fields() + ++ emqx_connector_mongo:fields(sharded); +fields(mysql) -> + authz_common_fields(mysql) + ++ [ {query, #{type => binary()}}] + ++ emqx_connector_mysql:fields(config); +fields(postgresql) -> + authz_common_fields(postgresql) + ++ [ {query, #{type => binary()}}] + ++ proplists:delete(named_queries, emqx_connector_pgsql:fields(config)); +fields(redis_single) -> + authz_redis_common_fields() + ++ emqx_connector_redis:fields(single); +fields(redis_sentinel) -> + authz_redis_common_fields() + ++ emqx_connector_redis:fields(sentinel); +fields(redis_cluster) -> + authz_redis_common_fields() + ++ emqx_connector_redis:fields(cluster); +fields(file) -> + authz_common_fields(file) + ++ [ {rules, #{ type => binary() + , example => + <<"{allow,{username,\"^dashboard?\"},","subscribe,[\"$SYS/#\"]}.\n", + "{allow,{ipaddr,\"127.0.0.1\"},all,[\"$SYS/#\",\"#\"]}.">>}} + %% The path will be deprecated, `acl.conf` will be fixed in subdir of `data` + , {path, #{ type => binary() + , example => <<"acl.conf">>}}]; +fields(position) -> + [ { position + , mk( hoconsc:union([binary(), map()]) + , #{ desc => <<"Where to place the source">> + , required => true + , in => body + , example => #{<<"before">> => <<"file">>}})}]. + +%%------------------------------------------------------------------------------ +%% http type funcs + +url(type) -> binary(); +url(validator) -> [?NOT_EMPTY("the value of the field 'url' cannot be empty")]; +url(nullable) -> false; +url(_) -> undefined. + +headers(type) -> map(); +headers(converter) -> + fun(Headers) -> + maps:merge(default_headers(), transform_header_name(Headers)) + end; +headers(default) -> default_headers(); +headers(_) -> undefined. + +body(type) -> map(); +body(validator) -> [fun check_body/1]; +body(_) -> undefined. + +%% headers + +default_headers() -> + maps:put(<<"content-type">>, + <<"application/json">>, + default_headers_no_content_type()). + +default_headers_no_content_type() -> + #{ <<"accept">> => <<"application/json">> + , <<"cache-control">> => <<"no-cache">> + , <<"connection">> => <<"keep-alive">> + , <<"keep-alive">> => <<"timeout=5">> + }. + +transform_header_name(Headers) -> + maps:fold(fun(K0, V, Acc) -> + K = list_to_binary(string:to_lower(to_list(K0))), + maps:put(K, V, Acc) + end, #{}, Headers). + +%% body + +check_body(Body) -> + lists:all( + fun erlang:is_binary/1, + maps:values(Body)). + +%%------------------------------------------------------------------------------ +%% MonogDB type funcs + +authz_mongo_common_fields() -> + authz_common_fields(mongodb) ++ + [ {collection, fun collection/1} + , {selector, fun selector/1} ]. + +collection(type) -> binary(); +collection(_) -> undefined. + +selector(type) -> map(); +selector(_) -> undefined. + +%%------------------------------------------------------------------------------ +%% Redis type funcs + +authz_redis_common_fields() -> + authz_common_fields(redis) ++ + [ {cmd, #{ type => binary() + , example => <<"HGETALL mqtt_authz">>}}]. + +%%------------------------------------------------------------------------------ +%% Authz api type funcs + +authz_common_fields(Type) when is_atom(Type)-> + [ {enable, fun enable/1} + , {type, #{ type => enum([Type]) + , default => Type + , in => body + , converter => fun to_bin/1 + } + } + ]. + +enable(type) -> boolean(); +enable(default) -> true; +enable(desc) -> "Set to false to disable this auth provider"; +enable(_) -> undefined. + +%%------------------------------------------------------------------------------ +%% Internal funcs + +authz_sources_types(Type) -> + case Type of + simple -> [mongodb, redis]; + detailed -> [ mongo_single + , mongo_rs + , mongo_sharded + , redis_single + , redis_sentinel + , redis_cluster] + end + ++ + [ http + , 'built-in-database' + , mysql + , postgresql + , file]. + +to_list(A) when is_atom(A) -> + atom_to_list(A); +to_list(B) when is_binary(B) -> + binary_to_list(B). + +to_bin(List) when is_list(List) -> list_to_binary(List); +to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); +to_bin(X) -> X. diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 34881d248..4c0a8973b 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -18,9 +18,15 @@ -behaviour(minirest_api). +-include_lib("typerefl/include/types.hrl"). -include("emqx_authz.hrl"). -include_lib("emqx/include/logger.hrl"). +-import(hoconsc, [mk/1, mk/2, ref/1, ref/2, array/1, enum/1]). + +-define(BAD_REQUEST, 'BAD_REQUEST'). +-define(NOT_FOUND, 'NOT_FOUND'). + -define(EXAMPLE_REDIS, #{type=> redis, enable => true, @@ -44,290 +50,117 @@ -define(IS_TRUE(Val), ((Val =:= true) or (Val =:= <<"true">>))). +-define(API_SCHEMA_MODULE, emqx_authz_api_schema). + -export([ get_raw_sources/0 , get_raw_source/1 ]). -export([ api_spec/0 - , sources/2 + , paths/0 + , schema/1 + ]). + +-export([ sources/2 , source/2 , move_source/2 ]). api_spec() -> - {[ sources_api() - , source_api() - , move_source_api() - ], definitions()}. + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). -definitions() -> emqx_authz_api_schema:definitions(). +paths() -> + [ "/authorization/sources" + , "/authorization/sources/:type" + , "/authorization/sources/:type/move"]. -sources_api() -> - Metadata = #{ - get => #{ - description => "List authorization sources", - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => object, - required => [sources], - properties => #{sources => #{ - type => array, - items => minirest:ref(<<"sources">>) - } - } - }, - examples => #{ - sources => #{ - summary => <<"Sources">>, - value => jsx:encode(?EXAMPLE_RETURNED) - } - } - } - } - } - } - }, - post => #{ - description => "Add new source", - 'requestBody' => #{ - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"sources">>), - examples => #{ - redis => #{ - summary => <<"Redis">>, - value => jsx:encode(?EXAMPLE_REDIS) - }, - file => #{ - summary => <<"File">>, - value => jsx:encode(?EXAMPLE_FILE) - } - } - } - } - }, - responses => #{ - <<"204">> => #{description => <<"Created">>}, - <<"400">> => emqx_mgmt_util:bad_request() - } - }, - put => #{ - description => "Update all sources", - 'requestBody' => #{ - content => #{ - 'application/json' => #{ - schema => #{ - type => array, - items => minirest:ref(<<"sources">>) - }, - examples => #{ - redis => #{ - summary => <<"Redis">>, - value => jsx:encode(?EXAMPLE_REDIS) - }, - file => #{ - summary => <<"File">>, - value => jsx:encode(?EXAMPLE_FILE) - } - } - } - } - }, - responses => #{ - <<"204">> => #{description => <<"Created">>}, - <<"400">> => emqx_mgmt_util:bad_request() - } - } - }, - {"/authorization/sources", Metadata, sources}. +%%-------------------------------------------------------------------- +%% Schema for each URI +%%-------------------------------------------------------------------- -source_api() -> - Metadata = #{ - get => #{ - description => "List authorization sources", - parameters => [ - #{ - name => type, - in => path, - schema => #{ - type => string, - enum => [ <<"file">> - , <<"http">> - , <<"mongodb">> - , <<"mysql">> - , <<"postgresql">> - , <<"redis">> - , <<"built-in-database">> - ] - }, - required => true - } - ], - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"sources">>), - examples => #{ - redis => #{ - summary => <<"Redis">>, - value => jsx:encode(?EXAMPLE_REDIS) - }, - file => #{ - summary => <<"File">>, - value => jsx:encode(?EXAMPLE_FILE) - } - } - } - } - }, - <<"404">> => emqx_mgmt_util:bad_request(<<"Not Found">>) +schema("/authorization/sources") -> + #{ 'operationId' => sources + , get => + #{ description => <<"List all authorization sources">> + , responses => + #{ 200 => mk( array(hoconsc:union([ref(?API_SCHEMA_MODULE, Type) || Type <- authz_sources_types(detailed)])) + , #{desc => <<"Authorization source">>}) + } } - }, - put => #{ - description => "Update source", - parameters => [ - #{ - name => type, - in => path, - schema => #{ - type => string, - enum => [ <<"file">> - , <<"http">> - , <<"mongodb">> - , <<"mysql">> - , <<"postgresql">> - , <<"redis">> - , <<"built-in-database">> - ] - }, - required => true - } - ], - 'requestBody' => #{ - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"sources">>), - examples => #{ - redis => #{ - summary => <<"Redis">>, - value => jsx:encode(?EXAMPLE_REDIS) - }, - file => #{ - summary => <<"File">>, - value => jsx:encode(?EXAMPLE_FILE) - } - } - } - } - }, - responses => #{ - <<"204">> => #{description => <<"No Content">>}, - <<"404">> => emqx_mgmt_util:bad_request(<<"Not Found">>), - <<"400">> => emqx_mgmt_util:bad_request() + , post => + #{ description => <<"Add a new source">> + , 'requestBody' => mk( hoconsc:union([ref(?API_SCHEMA_MODULE, Type) || Type <- authz_sources_types(detailed)]) + , #{desc => <<"Source config">>}) + , responses => + #{ 204 => <<"Authorization source created successfully">> + , 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad Request">>) + } } - }, - delete => #{ - description => "Delete source", - parameters => [ - #{ - name => type, - in => path, - schema => #{ - type => string, - enum => [ <<"file">> - , <<"http">> - , <<"mongodb">> - , <<"mysql">> - , <<"postgresql">> - , <<"redis">> - , <<"built-in-database">> - ] - }, - required => true - } - ], - responses => #{ - <<"204">> => #{description => <<"Deleted">>}, - <<"400">> => emqx_mgmt_util:bad_request() + , put => + #{ description => <<"Update all sources">> + , 'requestBody' => mk( array(hoconsc:union([ref(?API_SCHEMA_MODULE, Type) || Type <- authz_sources_types(detailed)])) + , #{desc => <<"Sources">>}) + , responses => + #{ 204 => <<"Authorization source updated successfully">> + , 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad Request">>) + } } - } - }, - {"/authorization/sources/:type", Metadata, source}. + }; +schema("/authorization/sources/:type") -> + #{ 'operationId' => source + , get => + #{ description => <<"Get a authorization source">> + , parameters => parameters_field() + , responses => + #{ 200 => mk( hoconsc:union([ref(?API_SCHEMA_MODULE, Type) || Type <- authz_sources_types(detailed)]) + , #{desc => <<"Authorization source">>}) + , 404 => emqx_dashboard_swagger:error_codes([?NOT_FOUND], <<"Not Found">>) + } + } + , put => + #{ description => <<"Update source">> + , parameters => parameters_field() + , 'requestBody' => mk( hoconsc:union([ref(?API_SCHEMA_MODULE, Type) || Type <- authz_sources_types(detailed)])) + , responses => + #{ 204 => <<"Authorization source updated successfully">> + , 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad Request">>) + , 404 => emqx_dashboard_swagger:error_codes([?NOT_FOUND], <<"Not Found">>) + } + } + , delete => + #{ description => <<"Delete source">> + , parameters => parameters_field() + , responses => + #{ 204 => <<"Deleted successfully">> + , 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad Request">>) + } + } + }; +schema("/authorization/sources/:type/move") -> + #{ 'operationId' => move_source + , post => + #{ description => <<"Change the order of sources">> + , parameters => parameters_field() + , 'requestBody' => + emqx_dashboard_swagger:schema_with_examples( + ref(?API_SCHEMA_MODULE, position), + position_example()) + , responses => + #{ 204 => <<"No Content">> + , 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad Request">>) + , 404 => emqx_dashboard_swagger:error_codes([?NOT_FOUND], <<"Not Found">>) + } + } + }. -move_source_api() -> - Metadata = #{ - post => #{ - description => "Change the order of sources", - parameters => [ - #{ - name => type, - in => path, - schema => #{ - type => string, - enum => [ <<"file">> - , <<"http">> - , <<"mongodb">> - , <<"mysql">> - , <<"postgresql">> - , <<"redis">> - , <<"built-in-database">> - ] - }, - required => true - } - ], - 'requestBody' => #{ - content => #{ - 'application/json' => #{ - schema => #{ - type => object, - required => [position], - properties => #{ - position => #{ - 'oneOf' => [ - #{type => string, - enum => [<<"top">>, <<"bottom">>] - }, - #{type => object, - required => ['after'], - properties => #{ - 'after' => #{ - type => string - } - } - }, - #{type => object, - required => ['before'], - properties => #{ - 'before' => #{ - type => string - } - } - } - ] - } - } - } - } - } - }, - responses => #{ - <<"204">> => #{ - description => <<"No Content">> - }, - <<"404">> => emqx_mgmt_util:bad_request(<<"Not Found">>), - <<"400">> => emqx_mgmt_util:bad_request() - } - } - }, - {"/authorization/sources/:type/move", Metadata, move_source}. +%%-------------------------------------------------------------------- +%% Operation functions +%%-------------------------------------------------------------------- + +sources(Method, #{bindings := #{type := Type} = Bindings } = Req) + when is_atom(Type) -> + sources(Method, Req#{bindings => Bindings#{type => bin(Type)}}); sources(get, _) -> Sources = lists:foldl(fun (#{<<"type">> := <<"file">>, <<"enable">> := Enable, <<"path">> := Path}, AccIn) -> @@ -364,6 +197,9 @@ sources(put, #{body := Body}) when is_list(Body) -> end || Source <- Body], update_config(?CMD_REPLACE, NBody). +source(Method, #{bindings := #{type := Type} = Bindings } = Req) + when is_atom(Type) -> + source(Method, Req#{bindings => Bindings#{type => bin(Type)}}); source(get, #{bindings := #{type := Type}}) -> case get_raw_source(Type) of [] -> {404, #{message => <<"Not found ", Type/binary>>}}; @@ -390,6 +226,9 @@ source(put, #{bindings := #{type := <<"file">>}, body := #{<<"type">> := <<"file <<"enable">> => Enable, <<"path">> => Filename}) of {ok, _} -> {204}; + {error, {emqx_conf_schema, _}} -> + {400, #{code => <<"BAD_REQUEST">>, + message => <<"BAD_SCHEMA">>}}; {error, Reason} -> {400, #{code => <<"BAD_REQUEST">>, message => bin(Reason)}} @@ -399,17 +238,27 @@ source(put, #{bindings := #{type := Type}, body := Body}) when is_map(Body) -> source(delete, #{bindings := #{type := Type}}) -> update_config({?CMD_DELETE, Type}, #{}). +move_source(Method, #{bindings := #{type := Type} = Bindings } = Req) + when is_atom(Type) -> + move_source(Method, Req#{bindings => Bindings#{type => bin(Type)}}); move_source(post, #{bindings := #{type := Type}, body := #{<<"position">> := Position}}) -> case emqx_authz:move(Type, Position) of {ok, _} -> {204}; {error, not_found_source} -> {404, #{code => <<"NOT_FOUND">>, message => <<"source ", Type/binary, " not found">>}}; + {error, {emqx_conf_schema, _}} -> + {400, #{code => <<"BAD_REQUEST">>, + message => <<"BAD_SCHEMA">>}}; {error, Reason} -> {400, #{code => <<"BAD_REQUEST">>, message => bin(Reason)}} end. +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + get_raw_sources() -> RawSources = emqx:get_raw_config([authorization, sources], []), Schema = #{roots => emqx_authz_schema:fields("authorization"), fields => #{}}, @@ -449,6 +298,10 @@ update_config(Cmd, Sources) -> {error, {post_config_update, emqx_authz, Reason}} -> {400, #{code => <<"BAD_REQUEST">>, message => bin(Reason)}}; + %% TODO: The `Reason` may cann't be trans to json term. (i.e. ecpool start failed) + {error, {emqx_conf_schema, _}} -> + {400, #{code => <<"BAD_REQUEST">>, + message => <<"BAD_SCHEMA">>}}; {error, Reason} -> {400, #{code => <<"BAD_REQUEST">>, message => bin(Reason)}} @@ -489,8 +342,21 @@ do_write_file(Filename, Bytes) -> error(Reason) end. -bin(Term) -> - erlang:iolist_to_binary(io_lib:format("~p", [Term])). +bin(List) when is_list(List) -> list_to_binary(List); +bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); +bin(X) -> X. acl_conf_file() -> emqx_authz:acl_conf_file(). + +parameters_field() -> + [ {type, mk( enum(?API_SCHEMA_MODULE:authz_sources_types(simple)) + , #{in => path, desc => <<"Authorization type">>}) + } + ]. + +position_example() -> + #{<<"position">> => #{<<"before">> => <<"file">>}}. + +authz_sources_types(Type) -> + emqx_authz_api_schema:authz_sources_types(Type). From ff68e2a20f00478f26719432e1d2e29962d9a36e Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 21 Feb 2022 09:54:52 +0800 Subject: [PATCH 2/5] fix(http): connector and authn/authz http `keep-alive` default 30s , max 1000 --- apps/emqx_authn/src/simple_authn/emqx_authn_http.erl | 2 +- apps/emqx_authz/src/emqx_authz_api_schema.erl | 2 +- apps/emqx_authz/src/emqx_authz_schema.erl | 2 +- apps/emqx_connector/src/emqx_connector_http.erl | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index e04e938f6..9992dfa6c 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -223,7 +223,7 @@ default_headers_no_content_type() -> #{ <<"accept">> => <<"application/json">> , <<"cache-control">> => <<"no-cache">> , <<"connection">> => <<"keep-alive">> - , <<"keep-alive">> => <<"timeout=5">> + , <<"keep-alive">> => <<"timeout=30, max=1000">> }. transform_header_name(Headers) -> diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index 4c6850f7b..22cd5bb75 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -112,7 +112,7 @@ default_headers_no_content_type() -> #{ <<"accept">> => <<"application/json">> , <<"cache-control">> => <<"no-cache">> , <<"connection">> => <<"keep-alive">> - , <<"keep-alive">> => <<"timeout=5">> + , <<"keep-alive">> => <<"timeout=30, max=1000">> }. transform_header_name(Headers) -> diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index fb26c10e6..a010fde26 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -195,7 +195,7 @@ default_headers_no_content_type() -> #{ <<"accept">> => <<"application/json">> , <<"cache-control">> => <<"no-cache">> , <<"connection">> => <<"keep-alive">> - , <<"keep-alive">> => <<"timeout=5">> + , <<"keep-alive">> => <<"timeout=30, max=1000">> }. transform_header_name(Headers) -> diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index f8a9b9e93..ee59f75eb 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -159,7 +159,7 @@ on_start(InstId, #{base_url := #{scheme := Scheme, , {connect_timeout, ConnectTimeout} , {retry, MaxRetries} , {retry_timeout, RetryInterval} - , {keepalive, 5000} + , {keepalive, 30000} , {pool_type, PoolType} , {pool_size, PoolSize} , {transport, Transport} From eaa659f2a1f8d6db561838a4cd01c72dc61a0660 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 21 Feb 2022 11:37:30 +0800 Subject: [PATCH 3/5] refactor: authz_api_settings refactor with hoconsc --- .../src/emqx_authz_api_settings.erl | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz_api_settings.erl b/apps/emqx_authz/src/emqx_authz_api_settings.erl index 9b0ed2b34..b4b2ab5cc 100644 --- a/apps/emqx_authz/src/emqx_authz_api_settings.erl +++ b/apps/emqx_authz/src/emqx_authz_api_settings.erl @@ -18,35 +18,45 @@ -behaviour(minirest_api). +-import(hoconsc, [mk/1, ref/2]). + -export([ api_spec/0 - , settings/2 + , paths/0 + , schema/1 ]). +-export([settings/2]). + +-define(BAD_REQUEST, 'BAD_REQUEST'). + api_spec() -> - {[settings_api()], []}. + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). -authorization_settings() -> - maps:remove(<<"sources">>, emqx:get_raw_config([authorization], #{})). +paths() -> + ["/authorization/settings"]. -conf_schema() -> - emqx_mgmt_api_configs:gen_schema(authorization_settings()). +%%-------------------------------------------------------------------- +%% Schema for each URI +%%-------------------------------------------------------------------- -settings_api() -> - Metadata = #{ - get => #{ - description => "Get authorization settings", - responses => #{<<"200">> => emqx_mgmt_util:schema(conf_schema())} - }, - put => #{ - description => "Update authorization settings", - requestBody => emqx_mgmt_util:schema(conf_schema()), - responses => #{ - <<"200">> => emqx_mgmt_util:schema(conf_schema()), - <<"400">> => emqx_mgmt_util:bad_request() +schema("/authorization/settings") -> + #{ 'operationId' => settings + , get => + #{ description => <<"Get authorization settings">> + , responses => + #{200 => ref_authz_schema()} } - } - }, - {"/authorization/settings", Metadata, settings}. + , put => + #{ description => <<"Update authorization settings">> + , 'requestBody' => ref_authz_schema() + , responses => + #{ 200 => ref_authz_schema() + , 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad Request">>)} + } + }. + +ref_authz_schema() -> + proplists:delete(sources, emqx_conf_schema:fields("authorization")). settings(get, _Params) -> {200, authorization_settings()}; @@ -60,3 +70,6 @@ settings(put, #{body := #{<<"no_match">> := NoMatch, {ok, _} = emqx_authz_utils:update_config([authorization, cache], Cache), ok = emqx_authz_cache:drain_cache(), {200, authorization_settings()}. + +authorization_settings() -> + maps:remove(<<"sources">>, emqx:get_raw_config([authorization], #{})). From 4d5ee355bbfe25be53d7141f2369959efc24c26c Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 21 Feb 2022 18:09:04 +0800 Subject: [PATCH 4/5] fix(authz): authz http pool_type use default value `random` --- apps/emqx_authz/src/emqx_authz_http.erl | 2 ++ apps/emqx_authz/src/emqx_authz_schema.erl | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index 4c4ad8b8b..3c11660d7 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -105,6 +105,8 @@ parse_config(#{ url := URL ?PLACEHOLDERS) , headers => Headers , request_timeout => ReqTimeout + %% pool_type default value `random` + , pool_type => random }. parse_fullpath(RawURL) -> diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index a010fde26..0d8e80039 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -146,7 +146,9 @@ http_common_fields() -> [ {url, fun url/1} , {request_timeout, mk_duration("Request timeout", #{default => "30s"})} , {body, #{type => map(), nullable => true}} - ] ++ proplists:delete(base_url, connector_fields(http)). + ] ++ maps:to_list(maps:without([ base_url + , pool_type], + maps:from_list(connector_fields(http)))). mongo_common_fields() -> [ {collection, #{type => atom()}} From 9f9bf059b9ec1a5087b46733a56a5e172bf4b49c Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 21 Feb 2022 23:02:49 +0800 Subject: [PATCH 5/5] fix(schema): remove unuse to_bin function; simplify body schema. --- .../src/simple_authn/emqx_authn_http.erl | 11 +--------- apps/emqx_authz/src/emqx_authz_api_schema.erl | 22 ++----------------- .../emqx_authz/src/emqx_authz_api_sources.erl | 10 ++++----- .../src/emqx_dashboard_swagger.erl | 1 + 4 files changed, 8 insertions(+), 36 deletions(-) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 9992dfa6c..f8af51886 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -63,7 +63,7 @@ common_fields() -> [ {mechanism, emqx_authn_schema:mechanism('password-based')} , {backend, emqx_authn_schema:backend(http)} , {url, fun url/1} - , {body, fun body/1} + , {body, map([{fuzzy, term(), binary()}])} , {request_timeout, fun request_timeout/1} ] ++ emqx_authn_schema:common_fields() ++ maps:to_list(maps:without([ base_url @@ -96,10 +96,6 @@ headers_no_content_type(converter) -> headers_no_content_type(default) -> default_headers_no_content_type(); headers_no_content_type(_) -> undefined. -body(type) -> map(); -body(validator) -> [fun check_body/1]; -body(_) -> undefined. - request_timeout(type) -> emqx_schema:duration_ms(); request_timeout(default) -> <<"5s">>; request_timeout(_) -> undefined. @@ -209,11 +205,6 @@ destroy(#{resource_id := ResourceId}) -> parse_fullpath(RawURL) -> cow_http:parse_fullpath(to_bin(RawURL)). -check_body(Body) -> - lists:all( - fun erlang:is_binary/1, - maps:values(Body)). - default_headers() -> maps:put(<<"content-type">>, <<"application/json">>, diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index 22cd5bb75..9f21437d4 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -28,10 +28,9 @@ fields(http) -> authz_common_fields(http) ++ [ {url, fun url/1} , {method, #{ type => enum([get, post]) - , default => get - , converter => fun to_bin/1}} + , default => get}} , {headers, fun headers/1} - , {body, fun body/1} + , {body, map([{fuzzy, term(), binary()}])} , {request_timeout, mk_duration("Request timeout", #{default => "30s"})}] ++ maps:to_list(maps:without([ base_url , pool_type], @@ -97,12 +96,7 @@ headers(converter) -> headers(default) -> default_headers(); headers(_) -> undefined. -body(type) -> map(); -body(validator) -> [fun check_body/1]; -body(_) -> undefined. - %% headers - default_headers() -> maps:put(<<"content-type">>, <<"application/json">>, @@ -121,13 +115,6 @@ transform_header_name(Headers) -> maps:put(K, V, Acc) end, #{}, Headers). -%% body - -check_body(Body) -> - lists:all( - fun erlang:is_binary/1, - maps:values(Body)). - %%------------------------------------------------------------------------------ %% MonogDB type funcs @@ -159,7 +146,6 @@ authz_common_fields(Type) when is_atom(Type)-> , {type, #{ type => enum([Type]) , default => Type , in => body - , converter => fun to_bin/1 } } ]. @@ -193,7 +179,3 @@ to_list(A) when is_atom(A) -> atom_to_list(A); to_list(B) when is_binary(B) -> binary_to_list(B). - -to_bin(List) when is_list(List) -> list_to_binary(List); -to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); -to_bin(X) -> X. diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 4c0a8973b..4a58adaca 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -160,7 +160,7 @@ schema("/authorization/sources/:type/move") -> sources(Method, #{bindings := #{type := Type} = Bindings } = Req) when is_atom(Type) -> - sources(Method, Req#{bindings => Bindings#{type => bin(Type)}}); + sources(Method, Req#{bindings => Bindings#{type => atom_to_binary(Type, utf8)}}); sources(get, _) -> Sources = lists:foldl(fun (#{<<"type">> := <<"file">>, <<"enable">> := Enable, <<"path">> := Path}, AccIn) -> @@ -199,7 +199,7 @@ sources(put, #{body := Body}) when is_list(Body) -> source(Method, #{bindings := #{type := Type} = Bindings } = Req) when is_atom(Type) -> - source(Method, Req#{bindings => Bindings#{type => bin(Type)}}); + source(Method, Req#{bindings => Bindings#{type => atom_to_binary(Type, utf8)}}); source(get, #{bindings := #{type := Type}}) -> case get_raw_source(Type) of [] -> {404, #{message => <<"Not found ", Type/binary>>}}; @@ -240,7 +240,7 @@ source(delete, #{bindings := #{type := Type}}) -> move_source(Method, #{bindings := #{type := Type} = Bindings } = Req) when is_atom(Type) -> - move_source(Method, Req#{bindings => Bindings#{type => bin(Type)}}); + move_source(Method, Req#{bindings => Bindings#{type => atom_to_binary(Type, utf8)}}); move_source(post, #{bindings := #{type := Type}, body := #{<<"position">> := Position}}) -> case emqx_authz:move(Type, Position) of {ok, _} -> {204}; @@ -342,9 +342,7 @@ do_write_file(Filename, Bytes) -> error(Reason) end. -bin(List) when is_list(List) -> list_to_binary(List); -bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); -bin(X) -> X. +bin(Term) -> erlang:iolist_to_binary(io_lib:format("~p", [Term])). acl_conf_file() -> emqx_authz:acl_conf_file(). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 8ce92ecef..5744e7325 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -459,6 +459,7 @@ typename_to_spec("timeout()", _Mod) -> #{<<"oneOf">> => [#{type => string, examp typename_to_spec("bytesize()", _Mod) -> #{type => string, example => <<"32MB">>}; typename_to_spec("wordsize()", _Mod) -> #{type => string, example => <<"1024KB">>}; typename_to_spec("map()", _Mod) -> #{type => object, example => #{}}; +typename_to_spec("#{" ++ _, Mod) -> typename_to_spec("map()", Mod); typename_to_spec("qos()", _Mod) -> #{type => string, enum => [0, 1, 2], example => 0}; typename_to_spec("{binary(), binary()}", _Mod) -> #{type => object, example => #{}}; typename_to_spec("comma_separated_list()", _Mod) ->