From 7311132d499b248b5855aec9620bea1148432c8f Mon Sep 17 00:00:00 2001 From: zhouzb Date: Wed, 27 Oct 2021 09:22:17 +0800 Subject: [PATCH 001/179] fix(authn): fix handling of query result --- apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl | 4 ++-- apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index 98d515310..d8cc7618a 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -117,8 +117,8 @@ authenticate(#{password := Password} = Credential, Params = emqx_authn_utils:replace_placeholders(PlaceHolders, Credential), case emqx_resource:query(Unique, {sql, Query, Params, Timeout}) of {ok, _Columns, []} -> ignore; - {ok, Columns, Rows} -> - Selected = maps:from_list(lists:zip(Columns, Rows)), + {ok, Columns, [Row | _]} -> + Selected = maps:from_list(lists:zip(Columns, Row)), case emqx_authn_utils:check_password(Password, Selected, State) of ok -> {ok, emqx_authn_utils:is_superuser(Selected)}; diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index d1390697a..89c489d6d 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -106,10 +106,9 @@ authenticate(#{password := Password} = Credential, Params = emqx_authn_utils:replace_placeholders(PlaceHolders, Credential), case emqx_resource:query(Unique, {sql, Query, Params}) of {ok, _Columns, []} -> ignore; - {ok, Columns, Rows} -> + {ok, Columns, [Row | _]} -> NColumns = [Name || #column{name = Name} <- Columns], - NRows = [erlang:element(1, Row) || Row <- Rows], - Selected = maps:from_list(lists:zip(NColumns, NRows)), + Selected = maps:from_list(lists:zip(NColumns, erlang:tuple_to_list(Row))), case emqx_authn_utils:check_password(Password, Selected, State) of ok -> {ok, emqx_authn_utils:is_superuser(Selected)}; @@ -135,7 +134,7 @@ destroy(#{'_unique' := Unique}) -> parse_query(Query) -> case re:run(Query, ?RE_PLACEHOLDER, [global, {capture, all, binary}]) of {match, Captured} -> - PlaceHolders = [PlaceHolder || [PlaceHolder] <- Captured], + PlaceHolders = [PlaceHolder || ["\\" ++ PlaceHolder] <- Captured], Replacements = ["$" ++ integer_to_list(I) || I <- lists:seq(1, length(Captured))], NQuery = lists:foldl(fun({PlaceHolder, Replacement}, Query0) -> re:replace(Query0, PlaceHolder, Replacement, [{return, binary}]) From edc1581b4b9087313071f886752a75781bbedf21 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 27 Oct 2021 08:26:02 +0200 Subject: [PATCH 002/179] perf(pool): change emqx_retainer_pool type to hash --- apps/emqx_retainer/src/emqx_retainer_pool.erl | 2 +- apps/emqx_retainer/src/emqx_retainer_sup.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_retainer/src/emqx_retainer_pool.erl b/apps/emqx_retainer/src/emqx_retainer_pool.erl index 59ea1077a..6b48c0453 100644 --- a/apps/emqx_retainer/src/emqx_retainer_pool.erl +++ b/apps/emqx_retainer/src/emqx_retainer_pool.erl @@ -172,7 +172,7 @@ cast(Msg) -> %% @private worker() -> - gproc_pool:pick_worker(?POOL). + gproc_pool:pick_worker(?POOL, self()). run({M, F, A}) -> erlang:apply(M, F, A); diff --git a/apps/emqx_retainer/src/emqx_retainer_sup.erl b/apps/emqx_retainer/src/emqx_retainer_sup.erl index 3811ed8f2..0234c20e7 100644 --- a/apps/emqx_retainer/src/emqx_retainer_sup.erl +++ b/apps/emqx_retainer/src/emqx_retainer_sup.erl @@ -26,7 +26,7 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - PoolSpec = emqx_pool_sup:spec([emqx_retainer_pool, random, emqx_vm:schedulers(), + PoolSpec = emqx_pool_sup:spec([emqx_retainer_pool, hash, emqx_vm:schedulers(), {emqx_retainer_pool, start_link, []}]), {ok, {{one_for_one, 10, 3600}, [#{id => retainer, From 1c93331e34e531164e63622970eb03285f2266f4 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 27 Oct 2021 08:43:34 +0200 Subject: [PATCH 003/179] perf(pool): emqx_authn_http pool type hash --- apps/emqx_authn/src/simple_authn/emqx_authn_http.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index ceb4b30a8..66ea00af4 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -133,7 +133,7 @@ create(#{ method := Method case emqx_resource:create_local(Unique, emqx_connector_http, Config#{base_url => maps:remove(query, URIMap), - pool_type => random}) of + pool_type => hash}) of {ok, already_created} -> {ok, State}; {ok, _} -> From 34979c51d7976f6f221f14e63e93023dd4c67413 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 27 Oct 2021 08:45:13 +0200 Subject: [PATCH 004/179] perf(config): emqx_bridge default hash pool --- apps/emqx_bridge/etc/emqx_bridge.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge/etc/emqx_bridge.conf b/apps/emqx_bridge/etc/emqx_bridge.conf index f26172ef6..e1d2d4be7 100644 --- a/apps/emqx_bridge/etc/emqx_bridge.conf +++ b/apps/emqx_bridge/etc/emqx_bridge.conf @@ -51,7 +51,7 @@ # connect_timeout: "30s" # max_retries: 3 # retry_interval = "10s" -# pool_type = "random" +# pool_type = "hash" # pool_size = 4 # enable_pipelining = true # ssl { From 9c8cd6c43753ce0fa0e24688e7077de82460ef8c Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 27 Oct 2021 08:47:02 +0200 Subject: [PATCH 005/179] perf(pool): change emqx_connector default pool type to hash --- apps/emqx_connector/src/emqx_connector_http.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index c724ddb7a..61fcb1c67 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -105,7 +105,7 @@ retry_interval(default) -> <<"1s">>; retry_interval(_) -> undefined. pool_type(type) -> pool_type(); -pool_type(default) -> random; +pool_type(default) -> hash; pool_type(_) -> undefined. pool_size(type) -> non_neg_integer(); From 5e6dab435a921b200630d4c563a252b74ef12fe7 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 27 Oct 2021 08:48:41 +0200 Subject: [PATCH 006/179] perf(pool): authz http example pool_type to hash --- apps/emqx_authz/src/emqx_authz_api_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index b05476b03..3aa0cd677 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -86,7 +86,7 @@ definitions() -> pool_type => #{ type => string, enum => [<<"random">>, <<"hash">>], - example => <<"random">> + example => <<"hash">> }, pool_size => #{type => integer}, enable_pipelining => #{type => boolean}, From a712daaebc622cd13c60786f3a348311bf5e816e Mon Sep 17 00:00:00 2001 From: zhouzb Date: Wed, 27 Oct 2021 15:08:02 +0800 Subject: [PATCH 007/179] fix(authn): fix bad list comprehension --- apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 89c489d6d..57a8416df 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -134,7 +134,7 @@ destroy(#{'_unique' := Unique}) -> parse_query(Query) -> case re:run(Query, ?RE_PLACEHOLDER, [global, {capture, all, binary}]) of {match, Captured} -> - PlaceHolders = [PlaceHolder || ["\\" ++ PlaceHolder] <- Captured], + PlaceHolders = ["\\" ++ PlaceHolder || [PlaceHolder] <- Captured], Replacements = ["$" ++ integer_to_list(I) || I <- lists:seq(1, length(Captured))], NQuery = lists:foldl(fun({PlaceHolder, Replacement}, Query0) -> re:replace(Query0, PlaceHolder, Replacement, [{return, binary}]) From e62fde321cb6584bf0913970e0b4f3a18353d40c Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 28 Oct 2021 18:03:51 +0800 Subject: [PATCH 008/179] Emqx alarm (#5994) * chore(alarm): normalize_message outside emqx_alarm * chore(alarm): don't cache config in emqx_alarm; remove dirty_write/read; add desc/example to alarm; add more test * chore(alarm_api): alarm_api with hocon schema * fix: activted's nullable is true * fix(swagger): translate map to object * fix(cluster_rpc): debug failed cluster_rpc test * fix: Update schema description Co-authored-by: Zaiming (Stone) Shi Co-authored-by: Zaiming (Stone) Shi --- apps/emqx/include/emqx.hrl | 6 + apps/emqx/src/emqx_alarm.erl | 176 ++++++------------ apps/emqx/src/emqx_alarm_handler.erl | 15 +- apps/emqx/src/emqx_congestion.erl | 6 +- apps/emqx/src/emqx_os_mon.erl | 22 ++- apps/emqx/src/emqx_schema.erl | 34 +++- apps/emqx/src/emqx_sys_mon.erl | 8 +- apps/emqx/src/emqx_vm_mon.erl | 21 ++- apps/emqx/test/emqx_alarm_SUITE.erl | 74 +++++++- apps/emqx_conf/src/emqx_cluster_rpc.erl | 12 +- .../emqx_conf/test/emqx_cluster_rpc_SUITE.erl | 25 ++- .../src/emqx_dashboard_swagger.erl | 2 +- .../test/emqx_swagger_response_SUITE.erl | 2 +- .../src/emqx_mgmt_api_alarms.erl | 93 +++++---- 14 files changed, 291 insertions(+), 205 deletions(-) diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index cf419edc5..f7d3418ca 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -48,6 +48,12 @@ %% Queue topic -define(QUEUE, <<"$queue/">>). +%%-------------------------------------------------------------------- +%% alarms +%%-------------------------------------------------------------------- +-define(ACTIVATED_ALARM, emqx_activated_alarm). +-define(DEACTIVATED_ALARM, emqx_deactivated_alarm). + %%-------------------------------------------------------------------- %% Message and Delivery %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index 2585494eb..403308a68 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -17,7 +17,6 @@ -module(emqx_alarm). -behaviour(gen_server). --behaviour(emqx_config_handler). -include("emqx.hrl"). -include("logger.hrl"). @@ -27,22 +26,19 @@ -boot_mnesia({mnesia, [boot]}). --export([post_config_update/4]). - --export([ start_link/0 - , stop/0 +-export([start_link/0 ]). - --export([format/1]). - %% API -export([ activate/1 , activate/2 + , activate/3 , deactivate/1 , deactivate/2 + , deactivate/3 , delete_all_deactivated_alarms/0 , get_alarms/0 , get_alarms/1 + , format/1 ]). %% gen_server callbacks @@ -56,34 +52,19 @@ -record(activated_alarm, { name :: binary() | atom(), - details :: map() | list(), - message :: binary(), - activate_at :: integer() }). -record(deactivated_alarm, { activate_at :: integer(), - name :: binary() | atom(), - details :: map() | list(), - message :: binary(), - deactivate_at :: integer() | infinity }). --record(state, { - timer :: reference() - }). - --define(ACTIVATED_ALARM, emqx_activated_alarm). - --define(DEACTIVATED_ALARM, emqx_deactivated_alarm). - -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). @@ -114,20 +95,23 @@ mnesia(boot) -> start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). -stop() -> - gen_server:stop(?MODULE). - activate(Name) -> activate(Name, #{}). activate(Name, Details) -> - gen_server:call(?MODULE, {activate_alarm, Name, Details}). + activate(Name, Details, <<"">>). + +activate(Name, Details, Message) -> + gen_server:call(?MODULE, {activate_alarm, Name, Details, Message}). deactivate(Name) -> - gen_server:call(?MODULE, {deactivate_alarm, Name, no_details}). + deactivate(Name, no_details, <<"">>). deactivate(Name, Details) -> - gen_server:call(?MODULE, {deactivate_alarm, Name, Details}). + deactivate(Name, Details, <<"">>). + +deactivate(Name, Details, Message) -> + gen_server:call(?MODULE, {deactivate_alarm, Name, Details, Message}). delete_all_deactivated_alarms() -> gen_server:call(?MODULE, delete_all_deactivated_alarms). @@ -144,10 +128,6 @@ get_alarms(activated) -> get_alarms(deactivated) -> gen_server:call(?MODULE, {get_alarms, deactivated}). -post_config_update(_, #{validity_period := Period0}, _OldConf, _AppEnv) -> - ?MODULE ! {update_timer, Period0}, - ok. - format(#activated_alarm{name = Name, message = Message, activate_at = At, details = Details}) -> Now = erlang:system_time(microsecond), #{ @@ -159,7 +139,7 @@ format(#activated_alarm{name = Name, message = Message, activate_at = At, detail details => Details }; format(#deactivated_alarm{name = Name, message = Message, activate_at = At, details = Details, - deactivate_at = DAt}) -> + deactivate_at = DAt}) -> #{ node => node(), name => Name, @@ -168,9 +148,7 @@ format(#deactivated_alarm{name = Name, message = Message, activate_at = At, deta activate_at => to_rfc3339(At), deactivate_at => to_rfc3339(DAt), details => Details - }; -format(_) -> - {error, unknow_alarm}. + }. to_rfc3339(Timestamp) -> list_to_binary(calendar:system_time_to_rfc3339(Timestamp div 1000, [{unit, millisecond}])). @@ -180,85 +158,72 @@ to_rfc3339(Timestamp) -> %%-------------------------------------------------------------------- init([]) -> - _ = mria:wait_for_tables([?ACTIVATED_ALARM, ?DEACTIVATED_ALARM]), + ok = mria:wait_for_tables([?ACTIVATED_ALARM, ?DEACTIVATED_ALARM]), deactivate_all_alarms(), - ok = emqx_config_handler:add_handler([alarm], ?MODULE), - {ok, #state{timer = ensure_timer(undefined, get_validity_period())}}. + {ok, #{}, get_validity_period()}. -%% suppress dialyzer warning due to dirty read/write race condition. -%% TODO: change from dirty_read/write to transactional. -%% TODO: handle mnesia write errors. --dialyzer([{nowarn_function, [handle_call/3]}]). -handle_call({activate_alarm, Name, Details}, _From, State) -> - case mnesia:dirty_read(?ACTIVATED_ALARM, Name) of - [#activated_alarm{name = Name}] -> - {reply, {error, already_existed}, State}; - [] -> - Alarm = #activated_alarm{name = Name, - details = Details, - message = normalize_message(Name, Details), - activate_at = erlang:system_time(microsecond)}, - mria:dirty_write(?ACTIVATED_ALARM, Alarm), +handle_call({activate_alarm, Name, Details, Message}, _From, State) -> + Res = mria:transaction(mria:local_content_shard(), + fun create_activate_alarm/3, + [Name, Details, Message]), + case Res of + {atomic, Alarm} -> do_actions(activate, Alarm, emqx:get_config([alarm, actions])), - {reply, ok, State} + {reply, ok, State, get_validity_period()}; + {aborted, Reason} -> + {reply, Reason, State, get_validity_period()} end; -handle_call({deactivate_alarm, Name, Details}, _From, State) -> +handle_call({deactivate_alarm, Name, Details, Message}, _From, State) -> case mnesia:dirty_read(?ACTIVATED_ALARM, Name) of [] -> {reply, {error, not_found}, State}; [Alarm] -> - deactivate_alarm(Details, Alarm), - {reply, ok, State} + deactivate_alarm(Alarm, Details, Message), + {reply, ok, State, get_validity_period()} end; handle_call(delete_all_deactivated_alarms, _From, State) -> clear_table(?DEACTIVATED_ALARM), - {reply, ok, State}; + {reply, ok, State, get_validity_period()}; handle_call({get_alarms, all}, _From, State) -> {atomic, Alarms} = mria:ro_transaction( - ?COMMON_SHARD, + mria:local_content_shard(), fun() -> [normalize(Alarm) || Alarm <- ets:tab2list(?ACTIVATED_ALARM) ++ ets:tab2list(?DEACTIVATED_ALARM)] end), - {reply, Alarms, State}; + {reply, Alarms, State, get_validity_period()}; handle_call({get_alarms, activated}, _From, State) -> Alarms = [normalize(Alarm) || Alarm <- ets:tab2list(?ACTIVATED_ALARM)], - {reply, Alarms, State}; + {reply, Alarms, State, get_validity_period()}; handle_call({get_alarms, deactivated}, _From, State) -> Alarms = [normalize(Alarm) || Alarm <- ets:tab2list(?DEACTIVATED_ALARM)], - {reply, Alarms, State}; + {reply, Alarms, State, get_validity_period()}; -handle_call(Req, _From, State) -> - ?SLOG(error, #{msg => "unexpected_call", call => Req}), - {reply, ignored, State}. +handle_call(Req, From, State) -> + ?SLOG(error, #{msg => "unexpected_call", call_req => Req, from => From}), + {reply, ignored, State, get_validity_period()}. handle_cast(Msg, State) -> - ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), - {noreply, State}. + ?SLOG(error, #{msg => "unexpected_cast", cast_req => Msg}), + {noreply, State, get_validity_period()}. -handle_info({timeout, _TRef, delete_expired_deactivated_alarm}, - #state{timer = TRef} = State) -> +handle_info(timeout, State) -> Period = get_validity_period(), delete_expired_deactivated_alarms(erlang:system_time(microsecond) - Period * 1000), - {noreply, State#state{timer = ensure_timer(TRef, Period)}}; - -handle_info({update_timer, Period}, #state{timer = TRef} = State) -> - ?SLOG(warning, #{msg => "validity_timer_updated", period => Period}), - {noreply, State#state{timer = ensure_timer(TRef, Period)}}; + {noreply, State, Period}; handle_info(Info, State) -> - ?SLOG(error, #{msg => "unexpected_info", info => Info}), - {noreply, State}. + ?SLOG(error, #{msg => "unexpected_info", info_req => Info}), + {noreply, State, get_validity_period()}. terminate(_Reason, _State) -> - ok = emqx_config_handler:remove_handler([alarm]), ok. code_change(_OldVsn, State, _Extra) -> @@ -271,8 +236,21 @@ code_change(_OldVsn, State, _Extra) -> get_validity_period() -> emqx:get_config([alarm, validity_period]). -deactivate_alarm(Details, #activated_alarm{activate_at = ActivateAt, name = Name, - details = Details0, message = Msg0}) -> +create_activate_alarm(Name, Details, Message) -> + case mnesia:read(?ACTIVATED_ALARM, Name) of + [#activated_alarm{name = Name}] -> + mnesia:abort({error, already_existed}); + [] -> + Alarm = #activated_alarm{name = Name, + details = Details, + message = normalize_message(Name, iolist_to_binary(Message)), + activate_at = erlang:system_time(microsecond)}, + ok = mnesia:write(?ACTIVATED_ALARM, Alarm, write), + Alarm + end. + +deactivate_alarm(#activated_alarm{activate_at = ActivateAt, name = Name, + details = Details0, message = Msg0}, Details, Message) -> SizeLimit = emqx:get_config([alarm, size_limit]), case SizeLimit > 0 andalso (mnesia:table_info(?DEACTIVATED_ALARM, size) >= SizeLimit) of true -> @@ -286,7 +264,7 @@ deactivate_alarm(Details, #activated_alarm{activate_at = ActivateAt, name = Name HistoryAlarm = make_deactivated_alarm(ActivateAt, Name, Details0, Msg0, erlang:system_time(microsecond)), DeActAlarm = make_deactivated_alarm(ActivateAt, Name, Details, - normalize_message(Name, Details), + normalize_message(Name, iolist_to_binary(Message)), erlang:system_time(microsecond)), mria:dirty_write(?DEACTIVATED_ALARM, HistoryAlarm), mria:dirty_delete(?ACTIVATED_ALARM, Name), @@ -329,13 +307,6 @@ clear_table(TableName) -> ok end. -ensure_timer(OldTRef, Period) -> - _ = case is_reference(OldTRef) of - true -> erlang:cancel_timer(OldTRef); - false -> ok - end, - emqx_misc:start_timer(Period, delete_expired_deactivated_alarm). - delete_expired_deactivated_alarms(Checkpoint) -> delete_expired_deactivated_alarms(mnesia:dirty_first(?DEACTIVATED_ALARM), Checkpoint). @@ -368,16 +339,12 @@ do_actions(deactivate, Alarm = #deactivated_alarm{name = Name}, [log | More]) -> do_actions(deactivate, Alarm, More); do_actions(Operation, Alarm, [publish | More]) -> Topic = topic(Operation), - {ok, Payload} = encode_to_json(Alarm), + {ok, Payload} = emqx_json:safe_encode(normalize(Alarm)), Message = emqx_message:make(?MODULE, 0, Topic, Payload, #{sys => true}, #{properties => #{'Content-Type' => <<"application/json">>}}), - %% TODO log failed publishes _ = emqx_broker:safe_publish(Message), do_actions(Operation, Alarm, More). -encode_to_json(Alarm) -> - emqx_json:safe_encode(normalize(Alarm)). - topic(activate) -> emqx_topic:systop(<<"alarms/activate">>); topic(deactivate) -> @@ -405,25 +372,6 @@ normalize(#deactivated_alarm{activate_at = ActivateAt, deactivate_at => DeactivateAt, activated => false}. -normalize_message(Name, no_details) -> +normalize_message(Name, <<"">>) -> list_to_binary(io_lib:format("~p", [Name])); -normalize_message(runq_overload, #{node := Node, runq_length := Len}) -> - list_to_binary(io_lib:format("VM is overloaded on node: ~p: ~p", [Node, Len])); -normalize_message(high_system_memory_usage, #{high_watermark := HighWatermark}) -> - list_to_binary(io_lib:format("System memory usage is higher than ~p%", [HighWatermark])); -normalize_message(high_process_memory_usage, #{high_watermark := HighWatermark}) -> - list_to_binary(io_lib:format("Process memory usage is higher than ~p%", [HighWatermark])); -normalize_message(high_cpu_usage, #{usage := Usage}) -> - list_to_binary(io_lib:format("~ts cpu usage", [Usage])); -normalize_message(too_many_processes, #{usage := Usage}) -> - list_to_binary(io_lib:format("~ts process usage", [Usage])); -normalize_message(cluster_rpc_apply_failed, #{tnx_id := TnxId}) -> - list_to_binary(io_lib:format("cluster_rpc_apply_failed:~w", [TnxId])); -normalize_message(partition, #{occurred := Node}) -> - list_to_binary(io_lib:format("Partition occurs at node ~ts", [Node])); -normalize_message(<<"resource", _/binary>>, #{type := Type, id := ID}) -> - list_to_binary(io_lib:format("Resource ~ts(~ts) is down", [Type, ID])); -normalize_message(<<"conn_congestion/", Info/binary>>, _) -> - list_to_binary(io_lib:format("connection congested: ~ts", [Info])); -normalize_message(_Name, _UnknownDetails) -> - <<"Unknown alarm">>. +normalize_message(_Name, Message) -> Message. diff --git a/apps/emqx/src/emqx_alarm_handler.erl b/apps/emqx/src/emqx_alarm_handler.erl index 4cf699895..5290404b3 100644 --- a/apps/emqx/src/emqx_alarm_handler.erl +++ b/apps/emqx/src/emqx_alarm_handler.erl @@ -57,14 +57,18 @@ init(_) -> {ok, []}. handle_event({set_alarm, {system_memory_high_watermark, []}}, State) -> + HighWatermark = emqx_os_mon:get_sysmem_high_watermark(), + Message = to_bin("System memory usage is higher than ~p%", [HighWatermark]), emqx_alarm:activate(high_system_memory_usage, - #{high_watermark => emqx_os_mon:get_sysmem_high_watermark()}), + #{high_watermark => HighWatermark}, Message), {ok, State}; handle_event({set_alarm, {process_memory_high_watermark, Pid}}, State) -> + HighWatermark = emqx_os_mon:get_procmem_high_watermark(), + Message = to_bin("Process memory usage is higher than ~p%", [HighWatermark]), emqx_alarm:activate(high_process_memory_usage, #{pid => list_to_binary(pid_to_list(Pid)), - high_watermark => emqx_os_mon:get_procmem_high_watermark()}), + high_watermark => HighWatermark}, Message), {ok, State}; handle_event({clear_alarm, system_memory_high_watermark}, State) -> @@ -76,7 +80,9 @@ handle_event({clear_alarm, process_memory_high_watermark}, State) -> {ok, State}; handle_event({set_alarm, {?LC_ALARM_ID_RUNQ, Info}}, State) -> - emqx_alarm:activate(runq_overload, Info), + #{node := Node, runq_length := Len} = Info, + Message = to_bin("VM is overloaded on node: ~p: ~p", [Node, Len]), + emqx_alarm:activate(runq_overload, Info, Message), {ok, State}; handle_event({clear_alarm, ?LC_ALARM_ID_RUNQ}, State) -> @@ -96,3 +102,6 @@ terminate(swap, _State) -> {emqx_alarm_handler, []}; terminate(_, _) -> ok. + +to_bin(Format, Args) -> + io_lib:format(Format, Args). diff --git a/apps/emqx/src/emqx_congestion.erl b/apps/emqx/src/emqx_congestion.erl index 170c6bc69..783f4ee4a 100644 --- a/apps/emqx/src/emqx_congestion.erl +++ b/apps/emqx/src/emqx_congestion.erl @@ -78,13 +78,15 @@ cancel_alarm_congestion(Socket, Transport, Channel, Reason) -> do_alarm_congestion(Socket, Transport, Channel, Reason) -> ok = update_alarm_sent_at(Reason), AlarmDetails = tcp_congestion_alarm_details(Socket, Transport, Channel), - emqx_alarm:activate(?ALARM_CONN_CONGEST(Channel, Reason), AlarmDetails), + Message = io_lib:format("connection congested: ~ts", [AlarmDetails]), + emqx_alarm:activate(?ALARM_CONN_CONGEST(Channel, Reason), AlarmDetails, Message), ok. do_cancel_alarm_congestion(Socket, Transport, Channel, Reason) -> ok = remove_alarm_sent_at(Reason), AlarmDetails = tcp_congestion_alarm_details(Socket, Transport, Channel), - emqx_alarm:deactivate(?ALARM_CONN_CONGEST(Channel, Reason), AlarmDetails), + Message = io_lib:format("connection congested: ~ts", [AlarmDetails]), + emqx_alarm:deactivate(?ALARM_CONN_CONGEST(Channel, Reason), AlarmDetails, Message), ok. is_tcp_congested(Socket, Transport) -> diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index 24795c7ba..e0cfac7af 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -96,12 +96,26 @@ handle_info({timeout, _Timer, check}, State) -> _ = case emqx_vm:cpu_util() of %% TODO: should be improved? 0 -> ok; Busy when Busy >= CPUHighWatermark -> - emqx_alarm:activate(high_cpu_usage, #{usage => io_lib:format("~p%", [Busy]), - high_watermark => CPUHighWatermark, - low_watermark => CPULowWatermark}), + Usage = io_lib:format("~p%", [Busy]), + Message = [Usage, " cpu usage"], + emqx_alarm:activate(high_cpu_usage, + #{ + usage => Usage, + high_watermark => CPUHighWatermark, + low_watermark => CPULowWatermark + }, + Message), start_check_timer(); Busy when Busy =< CPULowWatermark -> - emqx_alarm:deactivate(high_cpu_usage), + Usage = io_lib:format("~p%", [Busy]), + Message = [Usage, " cpu usage"], + emqx_alarm:deactivate(high_cpu_usage, + #{ + usage => Usage, + high_watermark => CPUHighWatermark, + low_watermark => CPULowWatermark + }, + Message), start_check_timer(); _Busy -> start_check_timer() diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 99a7ed59b..e319dbe15 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -51,6 +51,7 @@ -export([ validate_heap_size/1 , parse_user_lookup_fun/1 + , validate_alarm_actions/1 ]). % workaround: prevent being recognized as unused functions @@ -889,17 +890,34 @@ fields("sysmon_os") -> fields("alarm") -> [ {"actions", sc(hoconsc:array(atom()), - #{ default => [log, publish] + #{ default => [log, publish], + validator => fun ?MODULE:validate_alarm_actions/1, + example => [log, publish], + desc => + """The actions triggered when the alarm is activated.<\br> +Currently supports two actions, 'log' and 'publish'. +'log' is to write the alarm to log (console or file). +'publish' is to publish the alarm as an MQTT message to the system topics: +$SYS/brokers/emqx@xx.xx.xx.x/alarms/activate and $SYS/brokers/emqx@xx.xx.xx.x/alarms/deactivate""" }) } , {"size_limit", - sc(integer(), - #{ default => 1000 + sc(range(1, 3000), + #{ default => 1000, + example => 1000, + desc => + """The maximum total number of deactivated alarms to keep as history.
+When this limit is exceeded, the oldest deactivated alarms are deleted to cap the total number. +""" }) } , {"validity_period", sc(duration(), - #{ default => "24h" + #{ default => "24h", + example => "24h", + desc => + """Retention time of deactivated alarms. Alarms are not deleted immediately when deactivated, but after the retention time. + """ }) } ]. @@ -1345,6 +1363,14 @@ validate_heap_size(Siz) -> true -> error(io_lib:format("force_shutdown_policy: heap-size ~ts is too large", [Siz])); false -> ok end. + +validate_alarm_actions(Actions) -> + UnSupported = lists:filter(fun(Action) -> Action =/= log andalso Action =/= publish end, Actions), + case UnSupported of + [] -> ok; + Error -> {error, Error} + end. + parse_user_lookup_fun(StrConf) -> [ModStr, FunStr] = string:tokens(str(StrConf), ":"), Mod = list_to_atom(ModStr), diff --git a/apps/emqx/src/emqx_sys_mon.erl b/apps/emqx/src/emqx_sys_mon.erl index 7d798060f..cdc4677f3 100644 --- a/apps/emqx/src/emqx_sys_mon.erl +++ b/apps/emqx/src/emqx_sys_mon.erl @@ -170,9 +170,11 @@ code_change(_OldVsn, State, _Extra) -> %%-------------------------------------------------------------------- handle_partition_event({partition, {occurred, Node}}) -> - emqx_alarm:activate(partition, #{occurred => Node}); -handle_partition_event({partition, {healed, _Node}}) -> - emqx_alarm:deactivate(partition). + Message = io_lib:format("Partition occurs at node ~ts", [Node]), + emqx_alarm:activate(partition, #{occurred => Node}, Message); +handle_partition_event({partition, {healed, Node}}) -> + Message = io_lib:format("Partition healed at node ~ts", [Node]), + emqx_alarm:deactivate(partition, no_details, Message). suppress(Key, SuccFun, State = #{events := Events}) -> case lists:member(Key, Events) of diff --git a/apps/emqx/src/emqx_vm_mon.erl b/apps/emqx/src/emqx_vm_mon.erl index 703aca52f..9a30e71f2 100644 --- a/apps/emqx/src/emqx_vm_mon.erl +++ b/apps/emqx/src/emqx_vm_mon.erl @@ -62,12 +62,23 @@ handle_info({timeout, _Timer, check}, State) -> ProcessCount = erlang:system_info(process_count), case ProcessCount / erlang:system_info(process_limit) of Percent when Percent >= ProcHighWatermark -> - emqx_alarm:activate(too_many_processes, #{ - usage => io_lib:format("~p%", [Percent*100]), - high_watermark => ProcHighWatermark, - low_watermark => ProcLowWatermark}); + Usage = io_lib:format("~p%", [Percent*100]), + Message = [Usage, " process usage"], + emqx_alarm:activate(too_many_processes, + #{ + usage => Usage, + high_watermark => ProcHighWatermark, + low_watermark => ProcLowWatermark}, + Message); Percent when Percent < ProcLowWatermark -> - emqx_alarm:deactivate(too_many_processes); + Usage = io_lib:format("~p%", [Percent*100]), + Message = [Usage, " process usage"], + emqx_alarm:deactivate(too_many_processes, + #{ + usage => Usage, + high_watermark => ProcHighWatermark, + low_watermark => ProcLowWatermark}, + Message); _Precent -> ok end, diff --git a/apps/emqx/test/emqx_alarm_SUITE.erl b/apps/emqx/test/emqx_alarm_SUITE.erl index 0a720ffc1..b542250b3 100644 --- a/apps/emqx/test/emqx_alarm_SUITE.erl +++ b/apps/emqx/test/emqx_alarm_SUITE.erl @@ -32,16 +32,12 @@ init_per_testcase(t_size_limit, Config) -> <<"size_limit">> => 2 }), Config; -init_per_testcase(t_validity_period, Config) -> +init_per_testcase(_, Config) -> emqx_common_test_helpers:boot_modules(all), emqx_common_test_helpers:start_apps([]), {ok, _} = emqx:update_config([alarm], #{ <<"validity_period">> => <<"1s">> }), - Config; -init_per_testcase(_, Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), Config. end_per_testcase(_, _Config) -> @@ -86,17 +82,77 @@ t_size_limit(_) -> ?assertEqual({error, not_found}, get_alarm(a, emqx_alarm:get_alarms(deactivated))), emqx_alarm:delete_all_deactivated_alarms(). -t_validity_period(_) -> - ok = emqx_alarm:activate(a), - ok = emqx_alarm:deactivate(a), +t_validity_period(_Config) -> + ok = emqx_alarm:activate(a, #{msg => "Request frequency is too high"}, <<"Reach Rate Limit">>), + ok = emqx_alarm:deactivate(a, #{msg => "Request frequency returns to normal"}), ?assertNotEqual({error, not_found}, get_alarm(a, emqx_alarm:get_alarms(deactivated))), + %% call with unknown msg + ?assertEqual(ignored, gen_server:call(emqx_alarm, unknown_alarm)), ct:sleep(3000), ?assertEqual({error, not_found}, get_alarm(a, emqx_alarm:get_alarms(deactivated))). +t_validity_period_1(_Config) -> + ok = emqx_alarm:activate(a, #{msg => "Request frequency is too high"}, <<"Reach Rate Limit">>), + ok = emqx_alarm:deactivate(a, #{msg => "Request frequency returns to normal"}), + ?assertNotEqual({error, not_found}, get_alarm(a, emqx_alarm:get_alarms(deactivated))), + %% info with unknown msg + erlang:send(emqx_alarm, unknown_alarm), + ct:sleep(3000), + ?assertEqual({error, not_found}, get_alarm(a, emqx_alarm:get_alarms(deactivated))). + +t_validity_period_2(_Config) -> + ok = emqx_alarm:activate(a, #{msg => "Request frequency is too high"}, <<"Reach Rate Limit">>), + ok = emqx_alarm:deactivate(a, #{msg => "Request frequency returns to normal"}), + ?assertNotEqual({error, not_found}, get_alarm(a, emqx_alarm:get_alarms(deactivated))), + %% cast with unknown msg + gen_server:cast(emqx_alarm, unknown_alarm), + ct:sleep(3000), + ?assertEqual({error, not_found}, get_alarm(a, emqx_alarm:get_alarms(deactivated))). + +-record(activated_alarm, { + name :: binary() | atom(), + details :: map() | list(), + message :: binary(), + activate_at :: integer() +}). + +-record(deactivated_alarm, { + activate_at :: integer(), + name :: binary() | atom(), + details :: map() | list(), + message :: binary(), + deactivate_at :: integer() | infinity +}). + +t_format(_Config) -> + Name = test_alarm, + Message = "test_msg", + At = erlang:system_time(microsecond), + Details = "test_details", + Node = node(), + Activate = #activated_alarm{name = Name, message = Message, activate_at = At, details = Details}, + #{ + node := Node, + name := Name, + message := Message, + duration := 0, + details := Details + } = emqx_alarm:format(Activate), + Deactivate = #deactivated_alarm{name = Name, message = Message, activate_at = At, details = Details, + deactivate_at = At}, + #{ + node := Node, + name := Name, + message := Message, + duration := 0, + details := Details + } = emqx_alarm:format(Deactivate), + ok. + + get_alarm(Name, [Alarm = #{name := Name} | _More]) -> Alarm; get_alarm(Name, [_Alarm | More]) -> get_alarm(Name, More); get_alarm(_Name, []) -> {error, not_found}. - diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 4187b35aa..153800414 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -320,21 +320,23 @@ apply_mfa(TnxId, {M, F, A}) -> end, Meta = #{tnx_id => TnxId, module => M, function => F, args => ?TO_BIN(A)}, IsSuccess = is_success(Res), - log_and_alarm(IsSuccess, Res, Meta), + log_and_alarm(IsSuccess, Res, Meta, TnxId), {IsSuccess, Res}. is_success(ok) -> true; is_success({ok, _}) -> true; is_success(_) -> false. -log_and_alarm(true, Res, Meta) -> +log_and_alarm(true, Res, Meta, TnxId) -> OkMeta = Meta#{msg => <<"succeeded to apply MFA">>, result => Res}, ?SLOG(debug, OkMeta), - emqx_alarm:deactivate(cluster_rpc_apply_failed, OkMeta#{result => ?TO_BIN(Res)}); -log_and_alarm(false, Res, Meta) -> + Message = ["cluster_rpc_apply_failed:", integer_to_binary(TnxId)], + emqx_alarm:deactivate(cluster_rpc_apply_failed, OkMeta#{result => ?TO_BIN(Res)}, Message); +log_and_alarm(false, Res, Meta, TnxId) -> NotOkMeta = Meta#{msg => <<"failed to apply MFA">>, result => Res}, ?SLOG(error, NotOkMeta), - emqx_alarm:activate(cluster_rpc_apply_failed, NotOkMeta#{result => ?TO_BIN(Res)}). + Message = ["cluster_rpc_apply_failed:", integer_to_binary(TnxId)], + emqx_alarm:activate(cluster_rpc_apply_failed, NotOkMeta#{result => ?TO_BIN(Res)}, Message). wait_for_all_nodes_commit(TnxId, Delay, Remain) -> case lagging_node(TnxId) of diff --git a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl index cb79151ce..4e689916a 100644 --- a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl +++ b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl @@ -43,8 +43,8 @@ init_per_suite(Config) -> ok = ekka:start(), ok = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], infinity), meck:new(emqx_alarm, [non_strict, passthrough, no_link]), - meck:expect(emqx_alarm, activate, 2, ok), - meck:expect(emqx_alarm, deactivate, 2, ok), + meck:expect(emqx_alarm, activate, 3, ok), + meck:expect(emqx_alarm, deactivate, 3, ok), Config. end_per_suite(_Config) -> @@ -122,17 +122,21 @@ t_commit_ok_apply_fail_on_other_node_then_recover(_Config) -> emqx_cluster_rpc:reset(), {atomic, []} = emqx_cluster_rpc:status(), Now = erlang:system_time(millisecond), + ct:pal("111:~p~n", [ets:tab2list(cluster_rpc_commit)]), {M, F, A} = {?MODULE, failed_on_other_recover_after_5_second, [erlang:whereis(?NODE1), Now]}, - {ok, _, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), - {ok, _, ok} = emqx_cluster_rpc:multicall(io, format, ["test"], 1, 1000), + {ok, 1, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), + ct:pal("222:~p~n", [ets:tab2list(cluster_rpc_commit)]), + {ok, 2, ok} = emqx_cluster_rpc:multicall(io, format, ["test"], 1, 1000), + ct:pal("333:~p~n", [ets:tab2list(cluster_rpc_commit)]), + ct:pal("444:~p~n", [emqx_cluster_rpc:status()]), {atomic, [Status|L]} = emqx_cluster_rpc:status(), ?assertEqual([], L), ?assertEqual({io, format, ["test"]}, maps:get(mfa, Status)), ?assertEqual(node(), maps:get(node, Status)), - sleep(2300), + ct:sleep(2300), {atomic, [Status1]} = emqx_cluster_rpc:status(), ?assertEqual(Status, Status1), - sleep(3600), + ct:sleep(3600), {atomic, NewStatus} = emqx_cluster_rpc:status(), ?assertEqual(3, length(NewStatus)), Pid = self(), @@ -161,7 +165,7 @@ t_del_stale_mfa(_Config) -> {ok, TnxId, ok} = emqx_cluster_rpc:multicall(M, F, A), TnxId end || _ <- Keys2], ?assertEqual(Keys2, Ids2), - sleep(1200), + ct:sleep(1200), [begin ?assertEqual({aborted, not_found}, emqx_cluster_rpc:query(I)) end || I <- lists:seq(1, 50)], @@ -177,7 +181,7 @@ t_skip_failed_commit(_Config) -> emqx_cluster_rpc:reset(), {atomic, []} = emqx_cluster_rpc:status(), {ok, 1, ok} = emqx_cluster_rpc:multicall(io, format, ["test~n"], all, 1000), - sleep(180), + ct:sleep(180), {atomic, List1} = emqx_cluster_rpc:status(), Node = node(), ?assertEqual([{Node, 1}, {{Node, ?NODE2}, 1}, {{Node, ?NODE3}, 1}], @@ -250,8 +254,3 @@ failed_on_other_recover_after_5_second(Pid, CreatedAt) -> false -> ok end end. - -sleep(Ms) -> - receive _ -> ok - after Ms -> timeout - end. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 1c8aeaf83..64514a7aa 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -362,7 +362,7 @@ typename_to_spec("timeout()", _Mod) -> #{<<"oneOf">> => [#{type => string, examp #{type => integer, example => 100}], example => infinity}; typename_to_spec("bytesize()", _Mod) -> #{type => string, example => <<"32MB">>}; typename_to_spec("wordsize()", _Mod) -> #{type => string, example => <<"1024KB">>}; -typename_to_spec("map()", _Mod) -> #{type => string, example => <<>>}; +typename_to_spec("map()", _Mod) -> #{type => object, example => #{}}; typename_to_spec("comma_separated_list()", _Mod) -> #{type => string, example => <<"item1,item2">>}; typename_to_spec("comma_separated_atoms()", _Mod) -> #{type => string, example => <<"item1,item2">>}; typename_to_spec("pool_type()", _Mod) -> #{type => string, enum => [random, hash], example => hash}; diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index 92f411da7..dff491225 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -172,7 +172,7 @@ t_complicated_type(_Config) -> [#{example => infinity, type => string}, #{example => 100, type => integer}]}}, {<<"bytesize">>, #{example => <<"32MB">>, type => string}}, {<<"wordsize">>, #{example => <<"1024KB">>, type => string}}, - {<<"maps">>, #{example => <<>>, type => string}}, + {<<"maps">>, #{example => #{}, type => object}}, {<<"comma_separated_list">>, #{example => <<"item1,item2">>, type => string}}, {<<"comma_separated_atoms">>, #{example => <<"item1,item2">>, type => string}}, {<<"log_level">>, diff --git a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl index c4e49a616..38d923742 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl @@ -18,65 +18,76 @@ -behaviour(minirest_api). --export([api_spec/0]). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-export([api_spec/0, paths/0, schema/1, fields/1]). -export([alarms/2]). %% internal export (for query) --export([ query/4 - ]). - -%% notice: from emqx_alarms --define(ACTIVATED_ALARM, emqx_activated_alarm). --define(DEACTIVATED_ALARM, emqx_deactivated_alarm). - --import(emqx_mgmt_util, [ object_array_schema/2 - , schema/1 - , properties/1 - ]). +-export([query/4]). api_spec() -> - {[alarms_api()], []}. + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). -properties() -> - properties([ - {node, string, <<"Alarm in node">>}, - {name, string, <<"Alarm name">>}, - {message, string, <<"Alarm readable information">>}, - {details, object}, - {duration, integer, <<"Alarms duration time; UNIX time stamp, millisecond">>}, - {activate_at, string, <<"Alarms activate time, RFC 3339">>}, - {deactivate_at, string, <<"Nullable, alarms deactivate time, RFC 3339">>} - ]). +paths() -> + ["/alarms"]. -alarms_api() -> - Metadata = #{ +schema("/alarms") -> + #{ + operationId => alarms, get => #{ description => <<"EMQ X alarms">>, - parameters => emqx_mgmt_util:page_params() ++ [#{ - name => activated, - in => query, - description => <<"All alarms, if not specified">>, - required => false, - schema => #{type => boolean, default => true} - }], + parameters => [ + hoconsc:ref(emqx_dashboard_swagger, page), + hoconsc:ref(emqx_dashboard_swagger, limit), + {activated, hoconsc:mk(boolean(), #{in => query, + desc => <<"All alarms, if not specified">>, + nullable => true})} + ], responses => #{ - <<"200">> => - object_array_schema(properties(), <<"List all alarms">>)}}, - delete => #{ + 200 => [ + {data, hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, alarm)), #{})}, + {meta, hoconsc:mk(hoconsc:ref(?MODULE, meta), #{})} + ] + } + }, + delete => #{ description => <<"Remove all deactivated alarms">>, responses => #{ - <<"200">> => - schema(<<"Remove all deactivated alarms ok">>)}}}, - {"/alarms", Metadata, alarms}. + 200 => <<"Remove all deactivated alarms ok">> + } + } + }. +fields(alarm) -> + [ + {node, hoconsc:mk(binary(), #{desc => <<"Alarm in node">>, example => atom_to_list(node())})}, + {name, hoconsc:mk(binary(), #{desc => <<"Alarm name">>, example => <<"high_system_memory_usage">>})}, + {message, hoconsc:mk(binary(), #{desc => <<"Alarm readable information">>, + example => <<"System memory usage is higher than 70%">>})}, + {details, hoconsc:mk(map(), #{desc => <<"Alarm details information">>, + example => #{<<"high_watermark">> => 70}})}, + {duration, hoconsc:mk(integer(), #{desc => <<"Alarms duration time; UNIX time stamp, millisecond">>, + example => 297056})}, + {activate_at, hoconsc:mk(binary(), #{desc => <<"Alarms activate time, RFC 3339">>, + example => <<"2021-10-25T11:52:52.548+08:00">>})}, + {deactivate_at, hoconsc:mk(binary(), #{desc => <<"Nullable, alarms deactivate time, RFC 3339">>, + example => <<"2021-10-31T10:52:52.548+08:00">>})} + ]; + +fields(meta) -> + emqx_dashboard_swagger:fields(page) ++ + emqx_dashboard_swagger:fields(limit) ++ + [{count, hoconsc:mk(integer(), #{example => 1})}]. %%%============================================================================================== %% parameters trans alarms(get, #{query_string := Qs}) -> Table = - case maps:get(<<"activated">>, Qs, <<"true">>) of - <<"true">> -> ?ACTIVATED_ALARM; - <<"false">> -> ?DEACTIVATED_ALARM + case maps:get(<<"activated">>, Qs, true) of + true -> ?ACTIVATED_ALARM; + false -> ?DEACTIVATED_ALARM end, Response = emqx_mgmt_api:cluster_query(Qs, Table, [], {?MODULE, query}), emqx_mgmt_util:generate_response(Response); From 966348db059ef72a8ac4028f0abdd54b15752984 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 29 Oct 2021 10:12:29 +0800 Subject: [PATCH 009/179] fix(authn): fix version switching error when updating multiple times --- apps/emqx/src/emqx_authentication.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 226d697d8..765437163 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -489,7 +489,7 @@ handle_call({update_authenticator, ChainName, AuthenticatorID, Config}, _From, S Unique = unique(ChainName, AuthenticatorID, Version), case Provider:update(Config#{'_unique' => Unique}, ST) of {ok, NewST} -> - NewAuthenticator = Authenticator#authenticator{state = switch_version(NewST), + NewAuthenticator = Authenticator#authenticator{state = switch_version(NewST#{version => Version}), enable = maps:get(enable, Config)}, NewAuthenticators = replace_authenticator(AuthenticatorID, NewAuthenticator, Authenticators), true = ets:insert(?CHAINS_TAB, Chain#chain{authenticators = NewAuthenticators}), From c64637ca39bf87264bfd8e1a65037ee9fee8b6c4 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 29 Oct 2021 14:18:25 +0800 Subject: [PATCH 010/179] test(authn): add test case of version checking --- apps/emqx/test/emqx_authentication_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index 62224a87f..8d7201a74 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -136,11 +136,11 @@ t_authenticator(Config) when is_list(Config) -> ID1 = <<"password-based:built-in-database">>, % CRUD of authencaticator - ?assertMatch({ok, #{id := ID1, state := #{mark := 1}}}, ?AUTHN:create_authenticator(ChainName, AuthenticatorConfig1)), + ?assertMatch({ok, #{id := ID1, state := #{mark := 1, version := <<"2">>}}}, ?AUTHN:create_authenticator(ChainName, AuthenticatorConfig1)), ?assertMatch({ok, #{id := ID1}}, ?AUTHN:lookup_authenticator(ChainName, ID1)), ?assertMatch({ok, [#{id := ID1}]}, ?AUTHN:list_authenticators(ChainName)), ?assertEqual({error, {already_exists, {authenticator, ID1}}}, ?AUTHN:create_authenticator(ChainName, AuthenticatorConfig1)), - ?assertMatch({ok, #{id := ID1, state := #{mark := 2}}}, ?AUTHN:update_authenticator(ChainName, ID1, AuthenticatorConfig1)), + ?assertMatch({ok, #{id := ID1, state := #{mark := 2, version := <<"1">>}}}, ?AUTHN:update_authenticator(ChainName, ID1, AuthenticatorConfig1)), ?assertEqual(ok, ?AUTHN:delete_authenticator(ChainName, ID1)), ?assertEqual({error, {not_found, {authenticator, ID1}}}, ?AUTHN:update_authenticator(ChainName, ID1, AuthenticatorConfig1)), ?assertMatch({ok, []}, ?AUTHN:list_authenticators(ChainName)), From d9cb0283f32ac75220e016c8567eb70bfcf80f27 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 29 Oct 2021 13:48:27 +0800 Subject: [PATCH 011/179] fix(alarm): duration unit in dashboard, microsecond => millisecond --- apps/emqx/src/emqx_alarm.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index 403308a68..d1c4dd748 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -130,6 +130,8 @@ get_alarms(deactivated) -> format(#activated_alarm{name = Name, message = Message, activate_at = At, details = Details}) -> Now = erlang:system_time(microsecond), + %% mnesia db stored microsecond for high frequency alarm + %% format for dashboard using millisecond #{ node => node(), name => Name, @@ -144,13 +146,14 @@ format(#deactivated_alarm{name = Name, message = Message, activate_at = At, deta node => node(), name => Name, message => Message, - duration => DAt - At, + duration => (DAt - At) div 1000, %% to millisecond activate_at => to_rfc3339(At), deactivate_at => to_rfc3339(DAt), details => Details }. to_rfc3339(Timestamp) -> + %% rfc3339 accuracy to millisecond list_to_binary(calendar:system_time_to_rfc3339(Timestamp div 1000, [{unit, millisecond}])). %%-------------------------------------------------------------------- From 800b4b32c762d5344e53d6edc118af20a4a6b0ed Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 1 Nov 2021 12:52:03 +0300 Subject: [PATCH 012/179] refactor(authn api): use config schemas for request validations (#5999) --- apps/emqx/src/emqx_authentication_config.erl | 1 - apps/emqx/src/emqx_listeners.erl | 11 +- apps/emqx_authn/src/emqx_authn_api.erl | 2412 +++++------------ apps/emqx_authn/src/emqx_authn_schema.erl | 8 +- apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 402 ++- apps/emqx_authn/test/emqx_authn_test_lib.erl | 27 +- .../emqx_dashboard/src/emqx_dashboard_api.erl | 2 +- .../src/emqx_dashboard_swagger.erl | 160 +- .../test/emqx_swagger_parameter_SUITE.erl | 43 +- .../test/emqx_swagger_requestBody_SUITE.erl | 55 +- apps/emqx_modules/src/emqx_rewrite_api.erl | 2 +- 11 files changed, 1228 insertions(+), 1895 deletions(-) diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index 24abb951c..2f7e55eba 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -268,4 +268,3 @@ dir(ChainName, ID) when is_binary(ID) -> binary:replace(iolist_to_binary([to_bin(ChainName), "-", ID]), <<":">>, <<"-">>); dir(ChainName, Config) when is_map(Config) -> dir(ChainName, authenticator_id(Config)). - diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 187a55fdd..0581937b1 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -352,10 +352,13 @@ listener_id(Type, ListenerName) -> list_to_atom(lists:append([str(Type), ":", str(ListenerName)])). parse_listener_id(Id) -> - [Type, Name] = string:split(str(Id), ":", leading), - case lists:member(Type, ?TYPES_STRING) of - true -> {list_to_existing_atom(Type), list_to_atom(Name)}; - false -> {error, {invalid_listener_id, Id}} + case string:split(str(Id), ":", leading) of + [Type, Name] -> + case lists:member(Type, ?TYPES_STRING) of + true -> {list_to_existing_atom(Type), list_to_atom(Name)}; + false -> {error, {invalid_listener_id, Id}} + end; + _ -> {error, {invalid_listener_id, Id}} end. zone(Opts) -> diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 1b02d37ae..c3978fb0d 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -18,1848 +18,582 @@ -behaviour(minirest_api). +-include_lib("typerefl/include/types.hrl"). -include("emqx_authn.hrl"). +-import(hoconsc, [mk/2, ref/1]). +-import(emqx_dashboard_swagger, [error_codes/2]). + +-define(BAD_REQUEST, 'BAD_REQUEST'). +-define(NOT_FOUND, 'NOT_FOUND'). +-define(CONFLICT, 'CONFLICT'). + +% Swagger + -export([ api_spec/0 - , authentication/2 - , authentication2/2 - , authentication3/2 - , authentication4/2 - , move/2 - , move2/2 - , import_users/2 - , import_users2/2 - , users/2 - , users2/2 - , users3/2 - , users4/2 + , paths/0 + , schema/1 ]). --define(EXAMPLE_1, #{mechanism => <<"password-based">>, - backend => <<"built-in-database">>, - user_id_type => <<"username">>, - password_hash_algorithm => #{ - name => <<"sha256">> - }}). +-export([ roots/0 + , fields/1 + ]). --define(EXAMPLE_2, #{mechanism => <<"password-based">>, - backend => <<"http">>, - method => <<"post">>, - url => <<"http://localhost:80/login">>, - headers => #{ - <<"content-type">> => <<"application/json">> - }, - body => #{ - <<"username">> => <<"${mqtt-username}">>, - <<"password">> => <<"${mqtt-password}">> - }}). +-export([ authenticators/2 + , authenticator/2 + , listener_authenticators/2 + , listener_authenticator/2 + , authenticator_move/2 + , listener_authenticator_move/2 + , authenticator_import_users/2 + , listener_authenticator_import_users/2 + , authenticator_users/2 + , authenticator_user/2 + , listener_authenticator_users/2 + , listener_authenticator_user/2 + ]). --define(EXAMPLE_3, #{mechanism => <<"jwt">>, - use_jwks => false, - algorithm => <<"hmac-based">>, - secret => <<"mysecret">>, - secret_base64_encoded => false, - verify_claims => #{ - <<"username">> => <<"${mqtt-username}">> - }}). - --define(EXAMPLE_4, #{mechanism => <<"password-based">>, - backend => <<"mongodb">>, - server => <<"127.0.0.1:27017">>, - database => example, - collection => users, - selector => #{ - username => <<"${mqtt-username}">> - }, - password_hash_field => <<"password_hash">>, - salt_field => <<"salt">>, - is_superuser_field => <<"is_superuser">>, - password_hash_algorithm => <<"sha256">>, - salt_position => <<"prefix">> - }). - --define(EXAMPLE_5, #{mechanism => <<"password-based">>, - backend => <<"redis">>, - server => <<"127.0.0.1:6379">>, - database => 0, - query => <<"HMGET ${mqtt-username} password_hash salt">>, - password_hash_algorithm => <<"sha256">>, - salt_position => <<"prefix">> - }). - --define(INSTANCE_EXAMPLE_1, maps:merge(?EXAMPLE_1, #{id => <<"password-based:built-in-database">>, - enable => true})). - --define(INSTANCE_EXAMPLE_2, maps:merge(?EXAMPLE_2, #{id => <<"password-based:http">>, - connect_timeout => "5s", - enable_pipelining => true, - headers => #{ - <<"accept">> => <<"application/json">>, - <<"cache-control">> => <<"no-cache">>, - <<"connection">> => <<"keepalive">>, - <<"content-type">> => <<"application/json">>, - <<"keep-alive">> => <<"timeout=5">> - }, - max_retries => 5, - pool_size => 8, - request_timeout => "5s", - retry_interval => "1s", - enable => true})). - --define(INSTANCE_EXAMPLE_3, maps:merge(?EXAMPLE_3, #{id => <<"jwt">>, - enable => true})). - --define(INSTANCE_EXAMPLE_4, maps:merge(?EXAMPLE_4, #{id => <<"password-based:mongodb">>, - mongo_type => <<"single">>, - pool_size => 8, - ssl => #{ - enable => false - }, - topology => #{ - max_overflow => 8, - pool_size => 8 - }, - enable => true})). - --define(INSTANCE_EXAMPLE_5, maps:merge(?EXAMPLE_5, #{id => <<"password-based:redis">>, - auto_reconnect => true, - redis_type => single, - pool_size => 8, - ssl => #{ - enable => false - }, - enable => true})). - --define(ERR_RESPONSE(Desc), #{description => Desc, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"Error">>), - examples => #{ - example1 => #{ - summary => <<"Not Found">>, - value => #{code => <<"NOT_FOUND">>, message => <<"Authenticator '67e4c9d3' does not exist">>} - }, - example2 => #{ - summary => <<"Conflict">>, - value => #{code => <<"ALREADY_EXISTS">>, message => <<"Name has be used">>} - }, - example3 => #{ - summary => <<"Bad Request 1">>, - value => #{code => <<"OUT_OF_RANGE">>, message => <<"Out of range">>} - } - }}}}). +-export([authenticator_examples/0]). api_spec() -> - {[ authentication_api() - , authentication_api2() - , move_api() - , authentication_api3() - , authentication_api4() - , move_api2() - , import_users_api() - , import_users_api2() - , users_api() - , users2_api() - , users3_api() - , users4_api() - ], definitions()}. + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). -authentication_api() -> - Metadata = #{ - post => create_authenticator_api_spec(), - get => list_authenticators_api_spec() - }, - {"/authentication", Metadata, authentication}. +paths() -> [ "/authentication" + , "/authentication/:id" + , "/authentication/:id/move" + , "/authentication/:id/import_users" + , "/authentication/:id/users" + , "/authentication/:id/users/:user_id" -authentication_api2() -> - Metadata = #{ - get => find_authenticator_api_spec(), - put => update_authenticator_api_spec(), - delete => delete_authenticator_api_spec() - }, - {"/authentication/:id", Metadata, authentication2}. + , "/listeners/:listener_id/authentication" + , "/listeners/:listener_id/authentication/:id" + , "/listeners/:listener_id/authentication/:id/move" + , "/listeners/:listener_id/authentication/:id/import_users" + , "/listeners/:listener_id/authentication/:id/users" + , "/listeners/:listener_id/authentication/:id/users/:user_id" + ]. -authentication_api3() -> - Metadata = #{ - post => create_authenticator_api_spec2(), - get => list_authenticators_api_spec2() - }, - {"/listeners/:listener_id/authentication", Metadata, authentication3}. +roots() -> [ request_user_create + , request_user_update + , request_move + , request_import_users + , response_user + ]. -authentication_api4() -> - Metadata = #{ - get => find_authenticator_api_spec2(), - put => update_authenticator_api_spec2(), - delete => delete_authenticator_api_spec2() - }, - {"/listeners/:listener_id/authentication/:id", Metadata, authentication4}. +fields(request_user_create) -> + [ + {user_id, binary()}, + {password, binary()}, + {is_superuser, mk(boolean(), #{default => false, nullable => true})} + ]; -move_api() -> - Metadata = #{ - post => move_authenticator_api_spec() - }, - {"/authentication/:id/move", Metadata, move}. +fields(request_user_update) -> + [ + {password, binary()}, + {is_superuser, mk(boolean(), #{default => false, nullable => true})} + ]; -move_api2() -> - Metadata = #{ - post => move_authenticator_api_spec2() - }, - {"/listeners/:listener_id/authentication/:id/move", Metadata, move2}. +fields(request_move) -> + [{position, binary()}]; -import_users_api() -> - Metadata = #{ - post => import_users_api_spec() - }, - {"/authentication/:id/import_users", Metadata, import_users}. +fields(request_import_users) -> + [{filename, binary()}]; -import_users_api2() -> - Metadata = #{ - post => import_users_api_spec2() - }, - {"/listeners/:listener_id/authentication/:id/import_users", Metadata, import_users2}. - -users_api() -> - Metadata = #{ - post => create_user_api_spec(), - get => list_users_api_spec() - }, - {"/authentication/:id/users", Metadata, users}. - -users2_api() -> - Metadata = #{ - put => update_user_api_spec(), - get => find_user_api_spec(), - delete => delete_user_api_spec() - }, - {"/authentication/:id/users/:user_id", Metadata, users2}. - -users3_api() -> - Metadata = #{ - post => create_user_api_spec2(), - get => list_users_api_spec2() - }, - {"/listeners/:listener_id/authentication/:id/users", Metadata, users3}. - -users4_api() -> - Metadata = #{ - put => update_user_api_spec2(), - get => find_user_api_spec2(), - delete => delete_user_api_spec2() - }, - {"/listeners/:listener_id/authentication/:id/users/:user_id", Metadata, users4}. - -create_authenticator_api_spec() -> - #{ - description => <<"Create a authenticator for global authentication">>, - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"AuthenticatorConfig">>), - examples => #{ - default => #{ - summary => <<"Default">>, - value => emqx_json:encode(?EXAMPLE_1) - }, - http => #{ - summary => <<"Authentication provided by HTTP Server">>, - value => emqx_json:encode(?EXAMPLE_2) - }, - jwt => #{ - summary => <<"JWT Authentication">>, - value => emqx_json:encode(?EXAMPLE_3) - }, - mongodb => #{ - summary => <<"Authentication with MongoDB">>, - value => emqx_json:encode(?EXAMPLE_4) - }, - redis => #{ - summary => <<"Authentication with Redis">>, - value => emqx_json:encode(?EXAMPLE_5) - } - } - } - } - }, - responses => #{ - <<"201">> => #{ - description => <<"Created">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"AuthenticatorInstance">>), - examples => #{ - example1 => #{ - summary => <<"Example 1">>, - value => emqx_json:encode(?INSTANCE_EXAMPLE_1) - }, - example2 => #{ - summary => <<"Example 2">>, - value => emqx_json:encode(?INSTANCE_EXAMPLE_2) - }, - example3 => #{ - summary => <<"Example 3">>, - value => emqx_json:encode(?INSTANCE_EXAMPLE_3) - }, - example4 => #{ - summary => <<"Example 4">>, - value => emqx_json:encode(?INSTANCE_EXAMPLE_4) - }, - example5 => #{ - summary => <<"Example 5">>, - value => emqx_json:encode(?INSTANCE_EXAMPLE_5) - } - } - } - } - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"409">> => ?ERR_RESPONSE(<<"Conflict">>) - } - }. - -create_authenticator_api_spec2() -> - Spec = create_authenticator_api_spec(), - Spec#{ - description => <<"Create a authenticator for listener">>, - parameters => [ - #{ - name => listener_id, - in => path, - schema => #{ - type => string - }, - required => true - } - ] - }. - -list_authenticators_api_spec() -> - #{ - description => <<"List authenticators for global authentication">>, - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => array, - items => minirest:ref(<<"AuthenticatorInstance">>) - }, - examples => #{ - example => #{ - summary => <<"Example">>, - value => emqx_json:encode([ ?INSTANCE_EXAMPLE_1 - , ?INSTANCE_EXAMPLE_2 - , ?INSTANCE_EXAMPLE_3 - , ?INSTANCE_EXAMPLE_4 - , ?INSTANCE_EXAMPLE_5 - ])}}}}}}}. - -list_authenticators_api_spec2() -> - Spec = list_authenticators_api_spec(), - Spec#{ - description => <<"List authenticators for listener">>, - parameters => [ - #{ - name => listener_id, - in => path, - schema => #{ - type => string - }, - required => true - } - ] - }. - -find_authenticator_api_spec() -> - #{ - description => <<"Get authenticator by id">>, - parameters => [ - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - } - ], - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"AuthenticatorInstance">>), - examples => #{ - example1 => #{ - summary => <<"Example 1">>, - value => emqx_json:encode(?INSTANCE_EXAMPLE_1) - }, - example2 => #{ - summary => <<"Example 2">>, - value => emqx_json:encode(?INSTANCE_EXAMPLE_2) - }, - example3 => #{ - summary => <<"Example 3">>, - value => emqx_json:encode(?INSTANCE_EXAMPLE_3) - }, - example4 => #{ - summary => <<"Example 4">>, - value => emqx_json:encode(?INSTANCE_EXAMPLE_4) - }, - example5 => #{ - summary => <<"Example 5">>, - value => emqx_json:encode(?INSTANCE_EXAMPLE_5) - } - } - } - } - }, - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - }. - -find_authenticator_api_spec2() -> - Spec = find_authenticator_api_spec(), - Spec#{ - parameters => [ - #{ - name => listener_id, - in => path, - description => <<"Listener id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - } - ] - }. - -update_authenticator_api_spec() -> - #{ - description => <<"Update authenticator">>, - parameters => [ - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - } - ], - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"AuthenticatorConfig">>), - examples => #{ - example1 => #{ - summary => <<"Example 1">>, - value => emqx_json:encode(?EXAMPLE_1) - }, - example2 => #{ - summary => <<"Example 2">>, - value => emqx_json:encode(?EXAMPLE_2) - }, - example3 => #{ - summary => <<"Example 3">>, - value => emqx_json:encode(?EXAMPLE_3) - }, - example4 => #{ - summary => <<"Example 4">>, - value => emqx_json:encode(?EXAMPLE_4) - }, - example5 => #{ - summary => <<"Example 5">>, - value => emqx_json:encode(?EXAMPLE_5) - } - } - } - } - }, - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"AuthenticatorInstance">>), - examples => #{ - example1 => #{ - summary => <<"Example 1">>, - value => emqx_json:encode(?INSTANCE_EXAMPLE_1) - }, - example2 => #{ - summary => <<"Example 2">>, - value => emqx_json:encode(?INSTANCE_EXAMPLE_2) - }, - example3 => #{ - summary => <<"Example 3">>, - value => emqx_json:encode(?INSTANCE_EXAMPLE_3) - }, - example4 => #{ - summary => <<"Example 4">>, - value => emqx_json:encode(?INSTANCE_EXAMPLE_4) - }, - example5 => #{ - summary => <<"Example 5">>, - value => emqx_json:encode(?INSTANCE_EXAMPLE_5) - } - } - } - } - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>), - <<"409">> => ?ERR_RESPONSE(<<"Conflict">>) - } - }. - -update_authenticator_api_spec2() -> - Spec = update_authenticator_api_spec(), - Spec#{ - parameters => [ - #{ - name => listener_id, - in => path, - description => <<"Listener id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - } - ] - }. - -delete_authenticator_api_spec() -> - #{ - description => <<"Delete authenticator">>, - parameters => [ - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - } - ], - responses => #{ - <<"204">> => #{ - description => <<"No Content">> - }, - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - }. - -delete_authenticator_api_spec2() -> - Spec = delete_authenticator_api_spec(), - Spec#{ - parameters => [ - #{ - name => listener_id, - in => path, - description => <<"Listener id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - } - ] - }. - -move_authenticator_api_spec() -> - #{ - description => <<"Move authenticator">>, - parameters => [ - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - } - ], - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => #{ - oneOf => [ - #{ - type => object, - required => [position], - properties => #{ - position => #{ - type => string, - enum => [<<"top">>, <<"bottom">>], - example => <<"top">> - } - } - }, - #{ - type => object, - required => [position], - properties => #{ - position => #{ - type => string, - description => <<"before:">>, - example => <<"before:password-based:mysql">> - } - } - } - ] - } - } - } - }, - responses => #{ - <<"204">> => #{ - description => <<"No Content">> - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - }. - -move_authenticator_api_spec2() -> - Spec = move_authenticator_api_spec(), - Spec#{ - parameters => [ - #{ - name => listener_id, - in => path, - description => <<"Listener id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - } - ] - }. - -import_users_api_spec() -> - #{ - description => <<"Import users from json/csv file">>, - parameters => [ - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - } - ], - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => #{ - type => object, - required => [filename], - properties => #{ - filename => #{ - type => string - } - } - } - } - } - }, - responses => #{ - <<"204">> => #{ - description => <<"No Content">> - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - }. - -import_users_api_spec2() -> - Spec = import_users_api_spec(), - Spec#{ - parameters => [ - #{ - name => listener_id, - in => path, - description => <<"Listener id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - } - ] - }. - -create_user_api_spec() -> - #{ - description => <<"Add user">>, - parameters => [ - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - } - ], - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => #{ - type => object, - required => [user_id, password], - properties => #{ - user_id => #{ - type => string - }, - password => #{ - type => string - }, - is_superuser => #{ - type => boolean, - default => false - } - } - } - } - } - }, - responses => #{ - <<"201">> => #{ - description => <<"Created">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => object, - properties => #{ - user_id => #{ - type => string - }, - is_superuser => #{ - type => boolean - } - } - } - } - } - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - }. - -create_user_api_spec2() -> - Spec = create_user_api_spec(), - Spec#{ - parameters => [ - #{ - name => listener_id, - in => path, - description => <<"Listener id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - } - ] - }. - -list_users_api_spec() -> - #{ - description => <<"List users">>, - parameters => [ - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => page, - in => query, - description => <<"Page Index">>, - schema => #{ - type => integer - }, - required => false - }, - #{ - name => limit, - in => query, - description => <<"Page limit">>, - schema => #{ - type => integer - }, - required => false - } - ], - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => array, - items => #{ - type => object, - properties => #{ - user_id => #{ - type => string - }, - is_superuser => #{ - type => boolean - } - } - } - } - } - } - }, - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - }. - -list_users_api_spec2() -> - Spec = list_users_api_spec(), - Spec#{ - parameters => [ - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => listener_id, - in => path, - description => <<"Listener id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => page, - in => query, - description => <<"Page Index">>, - schema => #{ - type => integer - }, - required => false - }, - #{ - name => limit, - in => query, - description => <<"Page limit">>, - schema => #{ - type => integer - }, - required => false - } - ] - }. - -update_user_api_spec() -> - #{ - description => <<"Update user">>, - parameters => [ - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => user_id, - in => path, - description => <<"User id">>, - schema => #{ - type => string - }, - required => true - } - ], - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => #{ - type => object, - properties => #{ - password => #{ - type => string - }, - is_superuser => #{ - type => boolean - } - } - } - } - } - }, - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => array, - items => #{ - type => object, - properties => #{ - user_id => #{ - type => string - }, - is_superuser => #{ - type => boolean - } - } - } - } - } - } - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - }. - -update_user_api_spec2() -> - Spec = update_user_api_spec(), - Spec#{ - parameters => [ - #{ - name => listener_id, - in => path, - description => <<"Listener id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => user_id, - in => path, - description => <<"User id">>, - schema => #{ - type => string - }, - required => true - } - ] - }. - -find_user_api_spec() -> - #{ - description => <<"Get user info">>, - parameters => [ - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => user_id, - in => path, - description => <<"User id">>, - schema => #{ - type => string - }, - required => true - } - ], - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => array, - items => #{ - type => object, - properties => #{ - user_id => #{ - type => string - }, - is_superuser => #{ - type => boolean - } - } - } - } - } - } - }, - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - }. - -find_user_api_spec2() -> - Spec = find_user_api_spec(), - Spec#{ - parameters => [ - #{ - name => listener_id, - in => path, - description => <<"Listener id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => user_id, - in => path, - description => <<"User id">>, - schema => #{ - type => string - }, - required => true - } - ] - }. - -delete_user_api_spec() -> - #{ - description => <<"Delete user">>, - parameters => [ - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => user_id, - in => path, - description => <<"User id">>, - schema => #{ - type => string - }, - required => true - } - ], - responses => #{ - <<"204">> => #{ - description => <<"No Content">> - }, - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - }. - -delete_user_api_spec2() -> - Spec = delete_user_api_spec(), - Spec#{ - parameters => [ - #{ - name => listener_id, - in => path, - description => <<"Listener id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => id, - in => path, - description => <<"Authenticator id">>, - schema => #{ - type => string - }, - required => true - }, - #{ - name => user_id, - in => path, - description => <<"User id">>, - schema => #{ - type => string - }, - required => true - } - ] - }. - - -definitions() -> - AuthenticatorConfigDef = #{ - allOf => [ - #{ - type => object, - properties => #{ - enable => #{ - type => boolean, - default => true, - example => true - } - } - }, - #{ - oneOf => [ minirest:ref(<<"PasswordBasedBuiltInDatabase">>) - , minirest:ref(<<"PasswordBasedMySQL">>) - , minirest:ref(<<"PasswordBasedPostgreSQL">>) - , minirest:ref(<<"PasswordBasedMongoDB">>) - , minirest:ref(<<"PasswordBasedRedis">>) - , minirest:ref(<<"PasswordBasedHTTPServer">>) - , minirest:ref(<<"JWT">>) - , minirest:ref(<<"SCRAMBuiltInDatabase">>) - ] - } - ] - }, - - AuthenticatorInstanceDef = #{ - allOf => [ - #{ - type => object, - properties => #{ - id => #{ - type => string - } - } - } - ] ++ maps:get(allOf, AuthenticatorConfigDef) - }, - - PasswordBasedBuiltInDatabaseDef = #{ - type => object, - required => [mechanism, backend], - properties => #{ - mechanism => #{ - type => string, - enum => [<<"password-based">>], - example => <<"password-based">> - }, - backend => #{ - type => string, - enum => [<<"built-in-database">>], - example => <<"built-in-database">> - }, - user_id_type => #{ - type => string, - enum => [<<"username">>, <<"clientid">>], - example => <<"username">> - }, - password_hash_algorithm => minirest:ref(<<"PasswordHashAlgorithm">>) - } - }, - - PasswordBasedMySQLDef = #{ - type => object, - required => [ mechanism - , backend - , server - , database - , username - , password - , query], - properties => #{ - mechanism => #{ - type => string, - enum => [<<"password-based">>], - example => <<"password-based">> - }, - backend => #{ - type => string, - enum => [<<"mysql">>], - example => <<"mysql">> - }, - server => #{ - type => string, - example => <<"localhost:3306">> - }, - database => #{ - type => string - }, - pool_size => #{ - type => integer, - default => 8 - }, - username => #{ - type => string - }, - password => #{ - type => string - }, - auto_reconnect => #{ - type => boolean, - default => true - }, - ssl => minirest:ref(<<"SSL">>), - password_hash_algorithm => #{ - type => string, - enum => [<<"plain">>, <<"md5">>, <<"sha">>, <<"sha256">>, <<"sha512">>, <<"bcrypt">>], - default => <<"sha256">> - }, - salt_position => #{ - type => string, - enum => [<<"prefix">>, <<"suffix">>], - default => <<"prefix">> - }, - query => #{ - type => string, - example => <<"SELECT password_hash FROM mqtt_user WHERE username = ${mqtt-username}">> - }, - query_timeout => #{ - type => string, - description => <<"Query timeout">>, - default => "5s" - } - } - }, - - PasswordBasedPostgreSQLDef = #{ - type => object, - required => [ mechanism - , backend - , server - , database - , username - , password - , query], - properties => #{ - mechanism => #{ - type => string, - enum => [<<"password-based">>], - example => <<"password-based">> - }, - backend => #{ - type => string, - enum => [<<"postgresql">>], - example => <<"postgresql">> - }, - server => #{ - type => string, - example => <<"localhost:5432">> - }, - database => #{ - type => string - }, - pool_size => #{ - type => integer, - default => 8 - }, - username => #{ - type => string - }, - password => #{ - type => string - }, - auto_reconnect => #{ - type => boolean, - default => true - }, - password_hash_algorithm => #{ - type => string, - enum => [<<"plain">>, <<"md5">>, <<"sha">>, <<"sha256">>, <<"sha512">>, <<"bcrypt">>], - default => <<"sha256">> - }, - salt_position => #{ - type => string, - enum => [<<"prefix">>, <<"suffix">>], - default => <<"prefix">> - }, - query => #{ - type => string, - example => <<"SELECT password_hash FROM mqtt_user WHERE username = ${mqtt-username}">> - } - } - }, - - PasswordBasedMongoDBDef = #{ - type => object, - required => [ mechanism - , backend - , server - , servers - , replica_set_name - , database - , username - , password - , collection - , selector - , password_hash_field - ], - properties => #{ - mechanism => #{ - type => string, - enum => [<<"password-based">>], - example => <<"password-based">> - }, - backend => #{ - type => string, - enum => [<<"mongodb">>], - example => <<"mongodb">> - }, - server => #{ - description => <<"Mutually exclusive with the 'servers' field, only valid in standalone mode">>, - type => string, - example => <<"127.0.0.1:27017">> - }, - servers => #{ - description => <<"Mutually exclusive with the 'server' field, only valid in replica set and sharded mode">>, - type => array, - items => #{ - type => string - }, - example => [<<"127.0.0.1:27017">>] - }, - replica_set_name => #{ - description => <<"Only valid in replica set mode">>, - type => string - }, - database => #{ - type => string - }, - username => #{ - type => string - }, - password => #{ - type => string - }, - auth_source => #{ - type => string, - default => <<"admin">> - }, - pool_size => #{ - type => integer, - default => 8 - }, - collection => #{ - type => string - }, - selector => #{ - type => object, - additionalProperties => true, - example => <<"{\"username\":\"${mqtt-username}\"}">> - }, - password_hash_field => #{ - type => string, - example => <<"password_hash">> - }, - salt_field => #{ - type => string, - example => <<"salt">> - }, - is_superuser_field => #{ - type => string, - example => <<"is_superuser">> - }, - password_hash_algorithm => #{ - type => string, - enum => [<<"plain">>, <<"md5">>, <<"sha">>, <<"sha256">>, <<"sha512">>, <<"bcrypt">>], - default => <<"sha256">>, - example => <<"sha256">> - }, - salt_position => #{ - description => <<"Only valid when the 'salt_field' field is specified">>, - type => string, - enum => [<<"prefix">>, <<"suffix">>], - default => <<"prefix">>, - example => <<"prefix">> - } - } - }, - - PasswordBasedRedisDef = #{ - type => object, - required => [ mechanism - , backend - , server - , servers - , password - , database - , query - ], - properties => #{ - mechanism => #{ - type => string, - enum => [<<"password-based">>], - example => <<"password-based">> - }, - backend => #{ - type => string, - enum => [<<"redis">>], - example => <<"redis">> - }, - server => #{ - description => <<"Mutually exclusive with the 'servers' field, only valid in standalone mode">>, - type => string, - example => <<"127.0.0.1:27017">> - }, - servers => #{ - description => <<"Mutually exclusive with the 'server' field, only valid in cluster and sentinel mode">>, - type => array, - items => #{ - type => string - }, - example => [<<"127.0.0.1:27017">>] - }, - sentinel => #{ - description => <<"Only valid in sentinel mode">>, - type => string - }, - password => #{ - type => string - }, - database => #{ - type => integer, - example => 0 - }, - query => #{ - type => string, - example => <<"HMGET ${mqtt-username} password_hash salt">> - }, - password_hash_algorithm => #{ - type => string, - enum => [<<"plain">>, <<"md5">>, <<"sha">>, <<"sha256">>, <<"sha512">>, <<"bcrypt">>], - default => <<"sha256">>, - example => <<"sha256">> - }, - salt_position => #{ - type => string, - enum => [<<"prefix">>, <<"suffix">>], - default => <<"prefix">>, - example => <<"prefix">> - }, - pool_size => #{ - type => integer, - default => 8 - }, - auto_reconnect => #{ - type => boolean, - default => true - } - } - }, - - PasswordBasedHTTPServerDef = #{ - type => object, - required => [ mechanism - , backend - , url - , body - ], - properties => #{ - mechanism => #{ - type => string, - enum => [<<"password-based">>], - example => <<"password-based">> - }, - backend => #{ - type => string, - enum => [<<"http">>], - example => <<"http">> - }, - method => #{ - type => string, - enum => [<<"get">>, <<"post">>], - default => <<"post">> - }, - url => #{ - type => string, - example => <<"http://localhost:80/login">> - }, - headers => #{ - type => object, - additionalProperties => #{ - type => string - } - }, - body => #{ - type => object - }, - connect_timeout => #{ - type => string, - default => <<"5s">> - }, - max_retries => #{ - type => integer, - default => 5 - }, - retry_interval => #{ - type => string, - default => <<"1s">> - }, - request_timout => #{ - type => integer, - default => 5000 - }, - pool_size => #{ - type => integer, - default => 8 - }, - enable_pipelining => #{ - type => boolean, - default => true - }, - ssl => minirest:ref(<<"SSL">>) - } - }, - - JWTDef = #{ - type => object, - required => [mechanism], - properties => #{ - mechanism => #{ - type => string, - enum => [<<"jwt">>], - example => <<"jwt">> - }, - use_jwks => #{ - type => boolean, - default => false, - example => false - }, - algorithm => #{ - type => string, - enum => [<<"hmac-based">>, <<"public-key">>], - default => <<"hmac-based">>, - example => <<"hmac-based">> - }, - secret => #{ - type => string - }, - secret_base64_encoded => #{ - type => boolean, - default => false - }, - certificate => #{ - type => string - }, - endpoint => #{ - type => string, - example => <<"http://localhost:80">> - }, - refresh_interval => #{ - type => integer, - default => 300, - example => 300 - }, - verify_claims => #{ - type => object, - additionalProperties => #{ - type => string - } - }, - ssl => minirest:ref(<<"SSL">>) - } - }, - - SCRAMBuiltInDatabaseDef = #{ - type => object, - required => [mechanism, backend], - properties => #{ - mechanism => #{ - type => string, - enum => [<<"scram">>], - example => <<"scram">> - }, - backend => #{ - type => string, - enum => [<<"built-in-database">>], - example => <<"built-in-database">> - }, - algorithm => #{ - type => string, - enum => [<<"sha256">>, <<"sha512">>], - default => <<"sha256">> - }, - iteration_count => #{ - type => integer, - default => 4096 - } - } - }, - - PasswordHashAlgorithmDef = #{ - type => object, - required => [name], - properties => #{ - name => #{ - type => string, - enum => [<<"plain">>, <<"md5">>, <<"sha">>, <<"sha256">>, <<"sha512">>, <<"bcrypt">>], - default => <<"sha256">> - }, - salt_rounds => #{ - type => integer, - description => <<"Only valid when the name field is set to bcrypt">>, - default => 10 - } - } - }, - - SSLDef = #{ - type => object, - properties => #{ - enable => #{ - type => boolean, - default => false - }, - certfile => #{ - type => string - }, - keyfile => #{ - type => string - }, - cacertfile => #{ - type => string - }, - verify => #{ - type => boolean, - default => true - }, - server_name_indication => #{ - type => object, - properties => #{ - enable => #{ - type => boolean, - default => false - }, - hostname => #{ - type => string - } - } - } - } - }, - - ErrorDef = #{ - type => object, - properties => #{ - code => #{ - type => string, - enum => [<<"NOT_FOUND">>], - example => <<"NOT_FOUND">> - }, - message => #{ - type => string - } - } - }, - - [ #{<<"AuthenticatorConfig">> => AuthenticatorConfigDef} - , #{<<"AuthenticatorInstance">> => AuthenticatorInstanceDef} - , #{<<"PasswordBasedBuiltInDatabase">> => PasswordBasedBuiltInDatabaseDef} - , #{<<"PasswordBasedMySQL">> => PasswordBasedMySQLDef} - , #{<<"PasswordBasedPostgreSQL">> => PasswordBasedPostgreSQLDef} - , #{<<"PasswordBasedMongoDB">> => PasswordBasedMongoDBDef} - , #{<<"PasswordBasedRedis">> => PasswordBasedRedisDef} - , #{<<"PasswordBasedHTTPServer">> => PasswordBasedHTTPServerDef} - , #{<<"JWT">> => JWTDef} - , #{<<"SCRAMBuiltInDatabase">> => SCRAMBuiltInDatabaseDef} - , #{<<"PasswordHashAlgorithm">> => PasswordHashAlgorithmDef} - , #{<<"SSL">> => SSLDef} - , #{<<"Error">> => ErrorDef} +fields(response_user) -> + [ + {user_id, binary()}, + {is_superuser, mk(boolean(), #{default => false, nullable => true})} ]. -authentication(post, #{body := Config}) -> +schema("/authentication") -> + #{ + operationId => authenticators, + get => #{ + tags => [<<"authentication">>, <<"global">>], + description => <<"List authenticators for global authentication">>, + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_example( + hoconsc:array(emqx_authn_schema:authenticator_type()), + authenticator_array_example()) + } + }, + post => #{ + tags => [<<"authentication">>, <<"global">>], + description => <<"Create authenticator for global authentication">>, + requestBody => emqx_dashboard_swagger:schema_with_examples( + emqx_authn_schema:authenticator_type(), + authenticator_examples()), + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_examples( + emqx_authn_schema:authenticator_type(), + authenticator_examples()), + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 409 => error_codes([?CONFLICT], <<"Conflict">>) + } + } + }; + +schema("/authentication/:id") -> + #{ + operationId => authenticator, + get => #{ + tags => [<<"authentication">>, <<"global">>], + description => <<"Get authenticator from global authentication chain">>, + parameters => [{id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}], + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_examples( + emqx_authn_schema:authenticator_type(), + authenticator_examples()), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + }, + put => #{ + tags => [<<"authentication">>, <<"global">>], + description => <<"Update authenticator from global authentication chain">>, + parameters => [{id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}], + requestBody => emqx_dashboard_swagger:schema_with_examples( + emqx_authn_schema:authenticator_type(), + authenticator_examples() + ), + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_examples( + emqx_authn_schema:authenticator_type(), + authenticator_examples()), + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>), + 409 => error_codes([?CONFLICT], <<"Conflict">>) + } + }, + delete => #{ + tags => [<<"authentication">>, <<"global">>], + description => <<"Delete authenticator from global authentication chain">>, + parameters => [{id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}], + responses => #{ + 200 => <<"Authenticator deleted">>, + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + } + }; + +schema("/listeners/:listener_id/authentication") -> + #{ + operationId => listener_authenticators, + get => #{ + tags => [<<"authentication">>, <<"listener">>], + description => <<"List authenticators for listener authentication">>, + parameters => [{listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}], + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_example( + hoconsc:array(emqx_authn_schema:authenticator_type()), + authenticator_array_example()) + } + }, + post => #{ + tags => [<<"authentication">>, <<"listener">>], + description => <<"Create authenticator for listener authentication">>, + parameters => [{listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}], + requestBody => emqx_dashboard_swagger:schema_with_examples( + emqx_authn_schema:authenticator_type(), + authenticator_examples() + ), + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_examples( + emqx_authn_schema:authenticator_type(), + authenticator_examples()), + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 409 => error_codes([?CONFLICT], <<"Conflict">>) + } + } + }; + +schema("/listeners/:listener_id/authentication/:id") -> + #{ + operationId => listener_authenticator, + get => #{ + tags => [<<"authentication">>, <<"listener">>], + description => <<"Get authenticator from listener authentication chain">>, + parameters => [ + {listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}, + {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})} + ], + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_examples( + emqx_authn_schema:authenticator_type(), + authenticator_examples()), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + }, + put => #{ + tags => [<<"authentication">>, <<"listener">>], + description => <<"Update authenticator from listener authentication chain">>, + parameters => [ + {listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}, + {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})} + ], + requestBody => emqx_dashboard_swagger:schema_with_examples( + emqx_authn_schema:authenticator_type(), + authenticator_examples()), + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_examples( + emqx_authn_schema:authenticator_type(), + authenticator_examples()), + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>), + 409 => error_codes([?CONFLICT], <<"Conflict">>) + } + }, + delete => #{ + tags => [<<"authentication">>, <<"listener">>], + description => <<"Delete authenticator from listener authentication chain">>, + parameters => [ + {listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}, + {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})} + ], + responses => #{ + 204 => <<"Authenticator deleted">>, + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + } + }; + + +schema("/authentication/:id/move") -> + #{ + operationId => authenticator_move, + post => #{ + tags => [<<"authentication">>, <<"global">>], + description => <<"Move authenticator in global authentication chain">>, + parameters => [{id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}], + requestBody => ref(request_move), + responses => #{ + 204 => <<"Authenticator moved">>, + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + } + }; + +schema("/listeners/:listener_id/authentication/:id/move") -> + #{ + operationId => listener_authenticator_move, + post => #{ + tags => [<<"authentication">>, <<"listener">>], + description => <<"Move authenticator in listener authentication chain">>, + parameters => [ + {listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}, + {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})} + ], + requestBody => ref(request_move), + responses => #{ + 204 => <<"Authenticator moved">>, + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + } + }; + +schema("/authentication/:id/import_users") -> + #{ + operationId => authenticator_import_users, + post => #{ + tags => [<<"authentication">>, <<"global">>], + description => <<"Import users into authenticator in global authentication chain">>, + parameters => [{id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}], + requestBody => ref(request_import_users), + responses => #{ + 204 => <<"Users imported">>, + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + } + }; + +schema("/listeners/:listener_id/authentication/:id/import_users") -> + #{ + operationId => listener_authenticator_import_users, + post => #{ + tags => [<<"authentication">>, <<"listener">>], + description => <<"Import users into authenticator in listener authentication chain">>, + parameters => [ + {listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}, + {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})} + ], + requestBody => ref(request_import_users), + responses => #{ + 204 => <<"Users imported">>, + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + } + }; + +schema("/authentication/:id/users") -> + #{ + operationId => authenticator_users, + post => #{ + tags => [<<"authentication">>, <<"global">>], + description => <<"Create users for authenticator in global authentication chain">>, + parameters => [{id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}], + requestBody => ref(request_user_create), + responses => #{ + 201 => ref(response_user), + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + }, + get => #{ + tags => [<<"authentication">>, <<"global">>], + description => <<"List users in authenticator in global authentication chain">>, + parameters => [ + {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}, + {page, mk(integer(), #{in => query, desc => <<"Page Index">>, nullable => true})}, + {limit, mk(integer(), #{in => query, desc => <<"Page Limit">>, nullable => true})} + ], + responses => #{ + 200 => mk(hoconsc:array(ref(response_user)), #{}), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + + } + }; + +schema("/listeners/:listener_id/authentication/:id/users") -> + #{ + operationId => listener_authenticator_users, + post => #{ + tags => [<<"authentication">>, <<"listener">>], + description => <<"Create users for authenticator in global authentication chain">>, + parameters => [ + {listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}, + {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})} + ], + requestBody => ref(request_user_create), + responses => #{ + 201 => ref(response_user), + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + }, + get => #{ + tags => [<<"authentication">>, <<"listener">>], + description => <<"List users in authenticator in listener authentication chain">>, + parameters => [ + {listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}, + {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}, + {page, mk(integer(), #{in => query, desc => <<"Page Index">>, nullable => true})}, + {limit, mk(integer(), #{in => query, desc => <<"Page Limit">>, nullable => true})} + ], + responses => #{ + 200 => mk(hoconsc:array(ref(response_user)), #{}), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + + } + }; + +schema("/authentication/:id/users/:user_id") -> + #{ + operationId => authenticator_user, + get => #{ + tags => [<<"authentication">>, <<"global">>], + description => <<"Get user from authenticator in global authentication chain">>, + parameters => [ + {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}, + {user_id, mk(binary(), #{in => path, desc => <<"User ID">>})} + ], + responses => #{ + 200 => ref(response_user), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + }, + put => #{ + tags => [<<"authentication">>, <<"global">>], + description => <<"Update user in authenticator in global authentication chain">>, + parameters => [ + {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}, + {user_id, mk(binary(), #{in => path, desc => <<"User ID">>})} + ], + requestBody => ref(request_user_update), + responses => #{ + 200 => mk(hoconsc:array(ref(response_user)), #{}), + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + }, + delete => #{ + tags => [<<"authentication">>, <<"global">>], + description => <<"Update user in authenticator in global authentication chain">>, + parameters => [ + {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}, + {user_id, mk(binary(), #{in => path, desc => <<"User ID">>})} + ], + responses => #{ + 204 => <<"User deleted">>, + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + } + }; + +schema("/listeners/:listener_id/authentication/:id/users/:user_id") -> + #{ + operationId => listener_authenticator_user, + get => #{ + tags => [<<"authentication">>, <<"listener">>], + description => <<"Get user from authenticator in listener authentication chain">>, + parameters => [ + {listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}, + {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}, + {user_id, mk(binary(), #{in => path, desc => <<"User ID">>})} + ], + responses => #{ + 200 => ref(response_user), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + }, + put => #{ + tags => [<<"authentication">>, <<"listener">>], + description => <<"Update user in authenticator in listener authentication chain">>, + parameters => [ + {listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}, + {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}, + {user_id, mk(binary(), #{in => path, desc => <<"User ID">>})} + ], + requestBody => ref(request_user_update), + responses => #{ + 200 => mk(hoconsc:array(ref(response_user)), #{}), + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + + }, + delete => #{ + tags => [<<"authentication">>, <<"listener">>], + description => <<"Update user in authenticator in listener authentication chain">>, + parameters => [ + {listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}, + {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}, + {user_id, mk(binary(), #{in => path, desc => <<"User ID">>})} + ], + responses => #{ + 204 => <<"User deleted">>, + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + } + }. + +authenticators(post, #{body := Config}) -> create_authenticator([authentication], ?GLOBAL, Config); -authentication(get, _Params) -> +authenticators(get, _Params) -> list_authenticators([authentication]). -authentication2(get, #{bindings := #{id := AuthenticatorID}}) -> +authenticator(get, #{bindings := #{id := AuthenticatorID}}) -> list_authenticator([authentication], AuthenticatorID); -authentication2(put, #{bindings := #{id := AuthenticatorID}, body := Config}) -> +authenticator(put, #{bindings := #{id := AuthenticatorID}, body := Config}) -> update_authenticator([authentication], ?GLOBAL, AuthenticatorID, Config); -authentication2(delete, #{bindings := #{id := AuthenticatorID}}) -> +authenticator(delete, #{bindings := #{id := AuthenticatorID}}) -> delete_authenticator([authentication], ?GLOBAL, AuthenticatorID). -authentication3(post, #{bindings := #{listener_id := ListenerID}, body := Config}) -> - case find_listener(ListenerID) of - {ok, {Type, Name}} -> - create_authenticator([listeners, Type, Name, authentication], ListenerID, Config); - {error, Reason} -> - serialize_error(Reason) - end; -authentication3(get, #{bindings := #{listener_id := ListenerID}}) -> - case find_listener(ListenerID) of - {ok, {Type, Name}} -> - list_authenticators([listeners, Type, Name, authentication]); - {error, Reason} -> - serialize_error(Reason) - end. +listener_authenticators(post, #{bindings := #{listener_id := ListenerID}, body := Config}) -> + with_listener(ListenerID, + fun(Type, Name) -> + create_authenticator([listeners, Type, Name, authentication], + ListenerID, + Config) + end); -authentication4(get, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}}) -> - case find_listener(ListenerID) of - {ok, {Type, Name}} -> - list_authenticator([listeners, Type, Name, authentication], AuthenticatorID); - {error, Reason} -> - serialize_error(Reason) - end; -authentication4(put, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := Config}) -> - case find_listener(ListenerID) of - {ok, {Type, Name}} -> - update_authenticator([listeners, Type, Name, authentication], ListenerID, AuthenticatorID, Config); - {error, Reason} -> - serialize_error(Reason) - end; -authentication4(delete, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}}) -> - case find_listener(ListenerID) of - {ok, {Type, Name}} -> - delete_authenticator([listeners, Type, Name, authentication], ListenerID, AuthenticatorID); - {error, Reason} -> - serialize_error(Reason) - end. +listener_authenticators(get, #{bindings := #{listener_id := ListenerID}}) -> + with_listener(ListenerID, + fun(Type, Name) -> + list_authenticators([listeners, Type, Name, authentication]) + end). -move(post, #{bindings := #{id := AuthenticatorID}, body := #{<<"position">> := Position}}) -> +listener_authenticator(get, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}}) -> + with_listener(ListenerID, + fun(Type, Name) -> + list_authenticator([listeners, Type, Name, authentication], + AuthenticatorID) + end); +listener_authenticator(put, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := Config}) -> + with_listener(ListenerID, + fun(Type, Name) -> + update_authenticator([listeners, Type, Name, authentication], + ListenerID, + AuthenticatorID, + Config) + end); +listener_authenticator(delete, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}}) -> + with_listener(ListenerID, + fun(Type, Name) -> + delete_authenticator([listeners, Type, Name, authentication], + ListenerID, + AuthenticatorID) + end). + +authenticator_move(post, #{bindings := #{id := AuthenticatorID}, body := #{<<"position">> := Position}}) -> move_authenitcator([authentication], ?GLOBAL, AuthenticatorID, Position); -move(post, #{bindings := #{id := _}, body := _}) -> +authenticator_move(post, #{bindings := #{id := _}, body := _}) -> serialize_error({missing_parameter, position}). -move2(post, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := #{<<"position">> := Position}}) -> - case find_listener(ListenerID) of - {ok, {Type, Name}} -> - move_authenitcator([listeners, Type, Name, authentication], ListenerID, AuthenticatorID, Position); - {error, Reason} -> - serialize_error(Reason) - end; -move2(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> +listener_authenticator_move(post, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := #{<<"position">> := Position}}) -> + with_listener(ListenerID, + fun(Type, Name) -> + move_authenitcator([listeners, Type, Name, authentication], + ListenerID, + AuthenticatorID, + Position) + end); +listener_authenticator_move(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> serialize_error({missing_parameter, position}). -import_users(post, #{bindings := #{id := AuthenticatorID}, body := #{<<"filename">> := Filename}}) -> +authenticator_import_users(post, #{bindings := #{id := AuthenticatorID}, body := #{<<"filename">> := Filename}}) -> case ?AUTHN:import_users(?GLOBAL, AuthenticatorID, Filename) of ok -> {204}; {error, Reason} -> serialize_error(Reason) end; -import_users(post, #{bindings := #{id := _}, body := _}) -> +authenticator_import_users(post, #{bindings := #{id := _}, body := _}) -> serialize_error({missing_parameter, filename}). -import_users2(post, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := #{<<"filename">> := Filename}}) -> - case ?AUTHN:import_users(ListenerID, AuthenticatorID, Filename) of +listener_authenticator_import_users(post, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := #{<<"filename">> := Filename}}) -> + ChainName = to_atom(ListenerID), + case ?AUTHN:import_users(ChainName, AuthenticatorID, Filename) of ok -> {204}; {error, Reason} -> serialize_error(Reason) end; -import_users2(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> +listener_authenticator_import_users(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> serialize_error({missing_parameter, filename}). -users(post, #{bindings := #{id := AuthenticatorID}, body := UserInfo}) -> +authenticator_users(post, #{bindings := #{id := AuthenticatorID}, body := UserInfo}) -> add_user(?GLOBAL, AuthenticatorID, UserInfo); -users(get, #{bindings := #{id := AuthenticatorID}, query_string := PageParams}) -> +authenticator_users(get, #{bindings := #{id := AuthenticatorID}, query_string := PageParams}) -> list_users(?GLOBAL, AuthenticatorID, PageParams). -users2(put, #{bindings := #{id := AuthenticatorID, +authenticator_user(put, #{bindings := #{id := AuthenticatorID, user_id := UserID}, body := UserInfo}) -> update_user(?GLOBAL, AuthenticatorID, UserID, UserInfo); -users2(get, #{bindings := #{id := AuthenticatorID, user_id := UserID}}) -> +authenticator_user(get, #{bindings := #{id := AuthenticatorID, user_id := UserID}}) -> find_user(?GLOBAL, AuthenticatorID, UserID); -users2(delete, #{bindings := #{id := AuthenticatorID, user_id := UserID}}) -> +authenticator_user(delete, #{bindings := #{id := AuthenticatorID, user_id := UserID}}) -> delete_user(?GLOBAL, AuthenticatorID, UserID). -users3(post, #{bindings := #{listener_id := ListenerID, +listener_authenticator_users(post, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := UserInfo}) -> add_user(ListenerID, AuthenticatorID, UserInfo); -users3(get, #{bindings := #{listener_id := ListenerID, - id := AuthenticatorID}, - query_string := PageParams}) -> +listener_authenticator_users(get, #{bindings := #{listener_id := ListenerID, + id := AuthenticatorID}, query_string := PageParams}) -> list_users(ListenerID, AuthenticatorID, PageParams). -users4(put, #{bindings := #{listener_id := ListenerID, +listener_authenticator_user(put, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID, user_id := UserID}, body := UserInfo}) -> update_user(ListenerID, AuthenticatorID, UserID, UserInfo); -users4(get, #{bindings := #{listener_id := ListenerID, +listener_authenticator_user(get, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID, user_id := UserID}}) -> find_user(ListenerID, AuthenticatorID, UserID); -users4(delete, #{bindings := #{listener_id := ListenerID, +listener_authenticator_user(delete, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID, user_id := UserID}}) -> delete_user(ListenerID, AuthenticatorID, UserID). @@ -1868,13 +602,25 @@ users4(delete, #{bindings := #{listener_id := ListenerID, %% Internal functions %%------------------------------------------------------------------------------ +with_listener(ListenerID, Fun) -> + case find_listener(ListenerID) of + {ok, {Type, Name}} -> + Fun(Type, Name); + {error, Reason} -> + serialize_error(Reason) + end. + find_listener(ListenerID) -> - {Type, Name} = emqx_listeners:parse_listener_id(ListenerID), - case emqx_config:find([listeners, Type, Name]) of - {not_found, _, _} -> + case emqx_listeners:parse_listener_id(ListenerID) of + {error, _} -> {error, {not_found, {listener, ListenerID}}}; - {ok, _} -> - {ok, {Type, Name}} + {Type, Name} -> + case emqx_config:find([listeners, Type, Name]) of + {not_found, _, _} -> + {error, {not_found, {listener, ListenerID}}}; + {ok, _} -> + {ok, {Type, Name}} + end end. create_authenticator(ConfKeyPath, ChainName, Config) -> @@ -1944,7 +690,7 @@ add_user(ChainName0, AuthenticatorID, #{<<"user_id">> := UserID, <<"password">> {ok, User} -> {201, User}; {error, Reason} -> - serialize_error(Reason) + serialize_error({user_error, Reason}) end; add_user(_, _, #{<<"user_id">> := _}) -> serialize_error({missing_parameter, password}); @@ -1961,7 +707,7 @@ update_user(ChainName0, AuthenticatorID, UserID, UserInfo) -> {ok, User} -> {200, User}; {error, Reason} -> - serialize_error(Reason) + serialize_error({user_error, Reason}) end end. @@ -1971,7 +717,7 @@ find_user(ChainName0, AuthenticatorID, UserID) -> {ok, User} -> {200, User}; {error, Reason} -> - serialize_error(Reason) + serialize_error({user_error, Reason}) end. delete_user(ChainName0, AuthenticatorID, UserID) -> @@ -1980,7 +726,7 @@ delete_user(ChainName0, AuthenticatorID, UserID) -> ok -> {204}; {error, Reason} -> - serialize_error(Reason) + serialize_error({user_error, Reason}) end. list_users(ChainName0, AuthenticatorID, PageParams) -> @@ -2024,6 +770,15 @@ convert_certs(#{<<"ssl">> := SSLOpts} = Config) -> convert_certs(Config) -> Config. +serialize_error({user_error, not_found}) -> + {404, #{code => <<"NOT_FOUND">>, + message => binfmt("User not found", [])}}; +serialize_error({user_error, already_exist}) -> + {409, #{code => <<"BAD_REQUEST">>, + message => binfmt("User already exists", [])}}; +serialize_error({user_error, Reason}) -> + {400, #{code => <<"BAD_REQUEST">>, + message => binfmt("User error: ~p", [Reason])}}; serialize_error({not_found, {authenticator, ID}}) -> {404, #{code => <<"NOT_FOUND">>, message => binfmt("Authenticator '~ts' does not exist", [ID]) }}; @@ -2035,7 +790,7 @@ serialize_error({not_found, {chain, ?GLOBAL}}) -> message => <<"Authenticator not found in the 'global' scope">>}}; serialize_error({not_found, {chain, Name}}) -> {400, #{code => <<"BAD_REQUEST">>, - message => binfmt("No authentication has been create for listener '~ts'", [Name])}}; + message => binfmt("No authentication has been created for listener ~p", [Name])}}; serialize_error({already_exists, {authenticator, ID}}) -> {409, #{code => <<"ALREADY_EXISTS">>, message => binfmt("Authenticator '~ts' already exist", [ID])}}; @@ -2079,9 +834,92 @@ parse_position(_) -> ensure_list(M) when is_map(M) -> [M]; ensure_list(L) when is_list(L) -> L. +% TODO: fix atom leak! to_atom(B) when is_binary(B) -> binary_to_atom(B); to_atom(A) when is_atom(A) -> A. binfmt(Fmt, Args) -> iolist_to_binary(io_lib:format(Fmt, Args)). + +authenticator_array_example() -> + [Config || #{value := Config} <- maps:values(authenticator_examples())]. + +authenticator_examples() -> + #{ + 'password-based:built-in-database' => #{ + summary => <<"Built-in password-based authentication">>, + value => #{ + mechanism => <<"password-based">>, + backend => <<"built-in-database">>, + user_id_type => <<"username">>, + password_hash_algorithm => #{ + name => <<"sha256">> + } + } + }, + 'password-based:http' => #{ + summary => <<"Password-based authentication througth external HTTP API">>, + value => #{ + mechanism => <<"password-based">>, + backend => <<"http">>, + method => <<"post">>, + url => <<"http://127.0.0.2:8080">>, + headers => #{ + <<"content-type">> => <<"application/json">> + }, + body => #{ + <<"username">> => <<"${mqtt-username}">>, + <<"password">> => <<"${mqtt-password}">> + }, + pool_size => 8, + connect_timeout => 5000, + request_timeout => 5000, + enable_pipelining => true, + ssl => #{enable => false} + } + }, + 'jwt' => #{ + summary => <<"JWT authentication">>, + value => #{ + mechanism => <<"jwt">>, + use_jwks => false, + algorithm => <<"hmac-based">>, + secret => <<"mysecret">>, + secret_base64_encoded => false, + verify_claims => #{ + <<"username">> => <<"${mqtt-username}">> + } + } + }, + 'password-based:mongodb' => #{ + summary => <<"Password-based authentication with MongoDB backend">>, + value => #{ + mechanism => <<"password-based">>, + backend => <<"mongodb">>, + server => <<"127.0.0.1:27017">>, + database => example, + collection => users, + selector => #{ + username => <<"${mqtt-username}">> + }, + password_hash_field => <<"password_hash">>, + salt_field => <<"salt">>, + is_superuser_field => <<"is_superuser">>, + password_hash_algorithm => <<"sha256">>, + salt_position => <<"prefix">> + } + }, + 'password-based:redis' => #{ + summary => <<"Password-based authentication with Redis backend">>, + value => #{ + mechanism => <<"password-based">>, + backend => <<"redis">>, + server => <<"127.0.0.1:6379">>, + database => 0, + query => <<"HMGET ${mqtt-username} password_hash salt">>, + password_hash_algorithm => <<"sha256">>, + salt_position => <<"prefix">> + } + } + }. diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index b36e88ebf..22f62f519 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -21,12 +21,11 @@ -export([ common_fields/0 , roots/0 , fields/1 + , authenticator_type/0 ]). %% only for doc generation -roots() -> [{authenticator_config, - #{type => hoconsc:union(config_refs([Module || {_AuthnType, Module} <- emqx_authn:providers()])) - }}]. +roots() -> [{authenticator_config, hoconsc:mk(authenticator_type())}]. fields(_) -> []. @@ -38,5 +37,8 @@ enable(type) -> boolean(); enable(default) -> true; enable(_) -> undefined. +authenticator_type() -> + hoconsc:union(config_refs([Module || {_AuthnType, Module} <- emqx_authn:providers()])). + config_refs(Modules) -> lists:append([Module:refs() || Module <- Modules]). diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index 32ba06c33..b2f9bb106 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -18,7 +18,7 @@ -compile(nowarn_export_all). -compile(export_all). --include("emqx_authz.hrl"). +-include("emqx_authn.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -27,12 +27,27 @@ -define(API_VERSION, "v5"). -define(BASE_PATH, "api"). +-define(TCP_DEFAULT, 'tcp:default'). + +-define( + assertAuthenticatorsMatch(Guard, Path), + (fun() -> + {ok, 200, Response} = request(get, uri(Path)), + ?assertMatch(Guard, jiffy:decode(Response, [return_maps])) + end)()). + all() -> emqx_common_test_helpers:all(?MODULE). groups() -> []. +init_per_testcase(_, Config) -> + delete_authenticators([authentication], ?GLOBAL), + delete_authenticators([listeners, tcp, default, authentication], ?TCP_DEFAULT), + {atomic, ok} = mria:clear_table(emqx_authn_mnesia), + Config. + init_per_suite(Config) -> ok = emqx_common_test_helpers:start_apps([emqx_authn, emqx_dashboard], fun set_special_configs/1), Config. @@ -55,10 +70,379 @@ set_special_configs(emqx_dashboard) -> set_special_configs(_App) -> ok. -t_create_http_authn(_) -> - {ok, 200, _} = request(post, uri(["authentication"]), - emqx_authn_test_lib:http_example()), - {ok, 200, _} = request(get, uri(["authentication"])). +%%------------------------------------------------------------------------------ +%% Tests +%%------------------------------------------------------------------------------ + +t_invalid_listener(_) -> + {ok, 404, _} = request(get, uri(["listeners", "invalid", "authentication"])), + {ok, 404, _} = request(get, uri(["listeners", "in:valid", "authentication"])). + +t_authenticators(_) -> + test_authenticators([]). + +t_authenticator(_) -> + test_authenticator([]). + +t_authenticator_users(_) -> + test_authenticator_users([]). + +t_authenticator_user(_) -> + test_authenticator_user([]). + +t_authenticator_move(_) -> + test_authenticator_move([]). + +t_authenticator_import_users(_) -> + test_authenticator_import_users([]). + +t_listener_authenticators(_) -> + test_authenticators(["listeners", ?TCP_DEFAULT]). + +t_listener_authenticator(_) -> + test_authenticator(["listeners", ?TCP_DEFAULT]). + +t_listener_authenticator_users(_) -> + test_authenticator_users(["listeners", ?TCP_DEFAULT]). + +t_listener_authenticator_user(_) -> + test_authenticator_user(["listeners", ?TCP_DEFAULT]). + +t_listener_authenticator_move(_) -> + test_authenticator_move(["listeners", ?TCP_DEFAULT]). + +t_listener_authenticator_import_users(_) -> + test_authenticator_import_users(["listeners", ?TCP_DEFAULT]). + +test_authenticators(PathPrefix) -> + + ValidConfig = emqx_authn_test_lib:http_example(), + {ok, 200, _} = request( + post, + uri(PathPrefix ++ ["authentication"]), + ValidConfig), + + InvalidConfig = ValidConfig#{method => <<"delete">>}, + {ok, 400, _} = request( + post, + uri(PathPrefix ++ ["authentication"]), + InvalidConfig), + + ?assertAuthenticatorsMatch( + [#{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}], + PathPrefix ++ ["authentication"]). + +test_authenticator(PathPrefix) -> + ValidConfig0 = emqx_authn_test_lib:http_example(), + + {ok, 200, _} = request( + post, + uri(PathPrefix ++ ["authentication"]), + ValidConfig0), + + {ok, 200, _} = request( + get, + uri(PathPrefix ++ ["authentication", "password-based:http"])), + + {ok, 404, _} = request( + get, + uri(PathPrefix ++ ["authentication", "password-based:redis"])), + + + {ok, 404, _} = request( + put, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database"]), + emqx_authn_test_lib:built_in_database_example()), + + InvalidConfig0 = ValidConfig0#{method => <<"delete">>}, + {ok, 400, _} = request( + put, + uri(PathPrefix ++ ["authentication", "password-based:http"]), + InvalidConfig0), + + ValidConfig1 = ValidConfig0#{pool_size => 9}, + {ok, 200, _} = request( + put, + uri(PathPrefix ++ ["authentication", "password-based:http"]), + ValidConfig1), + + {ok, 404, _} = request( + delete, + uri(PathPrefix ++ ["authentication", "password-based:redis"])), + + {ok, 204, _} = request( + delete, + uri(PathPrefix ++ ["authentication", "password-based:http"])), + + ?assertAuthenticatorsMatch([], PathPrefix ++ ["authentication"]). + +test_authenticator_users(PathPrefix) -> + {ok, 200, _} = request( + post, + uri(PathPrefix ++ ["authentication"]), + emqx_authn_test_lib:built_in_database_example()), + + InvalidUsers = [ + #{clientid => <<"u1">>, password => <<"p1">>}, + #{user_id => <<"u2">>}, + #{user_id => <<"u3">>, password => <<"p3">>, foobar => <<"foobar">>}], + + lists:foreach( + fun(User) -> + {ok, 400, _} = request( + post, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]), + User) + end, + InvalidUsers), + + + ValidUsers = [ + #{user_id => <<"u1">>, password => <<"p1">>}, + #{user_id => <<"u2">>, password => <<"p2">>, is_superuser => true}, + #{user_id => <<"u3">>, password => <<"p3">>}], + + lists:foreach( + fun(User) -> + {ok, 201, _} = request( + post, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]), + User) + end, + ValidUsers), + + {ok, 200, Page1Data} = + request( + get, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]) ++ "?page=1&limit=2"), + + Page1Users = response_data(Page1Data), + + {ok, 200, Page2Data} = + request( + get, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]) ++ "?page=2&limit=2"), + + Page2Users = response_data(Page2Data), + + ?assertEqual(2, length(Page1Users)), + ?assertEqual(1, length(Page2Users)), + + ?assertEqual( + [<<"u1">>, <<"u2">>, <<"u3">>], + lists:usort([ UserId || #{<<"user_id">> := UserId} <- Page1Users ++ Page2Users])). + +test_authenticator_user(PathPrefix) -> + {ok, 200, _} = request( + post, + uri(PathPrefix ++ ["authentication"]), + emqx_authn_test_lib:built_in_database_example()), + + User = #{user_id => <<"u1">>, password => <<"p1">>}, + {ok, 201, _} = request( + post, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]), + User), + + {ok, 404, _} = request( + get, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users", "u123"])), + + {ok, 409, _} = request( + post, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]), + User), + + {ok, 200, UserData} = request( + get, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users", "u1"])), + + FetchedUser = jiffy:decode(UserData, [return_maps]), + ?assertMatch(#{<<"user_id">> := <<"u1">>}, FetchedUser), + ?assertNotMatch(#{<<"password">> := _}, FetchedUser), + + ValidUserUpdates = [ + #{password => <<"p1">>}, + #{password => <<"p1">>, is_superuser => true}], + + lists:foreach( + fun(UserUpdate) -> + {ok, 200, _} = request( + put, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users", "u1"]), + UserUpdate) + end, + ValidUserUpdates), + + InvalidUserUpdates = [ + #{user_id => <<"u1">>, password => <<"p1">>}, + #{is_superuser => true}], + + lists:foreach( + fun(UserUpdate) -> + {ok, 400, _} = request( + put, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users", "u1"]), + UserUpdate) + end, + InvalidUserUpdates), + + {ok, 404, _} = request( + delete, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users", "u123"])), + + {ok, 204, _} = request( + delete, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users", "u1"])). + +test_authenticator_move(PathPrefix) -> + AuthenticatorConfs = [ + emqx_authn_test_lib:http_example(), + emqx_authn_test_lib:jwt_example(), + emqx_authn_test_lib:built_in_database_example() + ], + + lists:foreach( + fun(Conf) -> + {ok, 200, _} = request( + post, + uri(PathPrefix ++ ["authentication"]), + Conf) + end, + AuthenticatorConfs), + + ?assertAuthenticatorsMatch( + [ + #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}, + #{<<"mechanism">> := <<"jwt">>}, + #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>} + ], + PathPrefix ++ ["authentication"]), + + % Invalid moves + + {ok, 400, _} = request( + post, + uri(PathPrefix ++ ["authentication", "jwt", "move"]), + #{position => <<"up">>}), + + {ok, 400, _} = request( + post, + uri(PathPrefix ++ ["authentication", "jwt", "move"]), + #{}), + + {ok, 404, _} = request( + post, + uri(PathPrefix ++ ["authentication", "jwt", "move"]), + #{position => <<"before:invalid">>}), + + {ok, 404, _} = request( + post, + uri(PathPrefix ++ ["authentication", "jwt", "move"]), + #{position => <<"before:password-based:redis">>}), + + {ok, 404, _} = request( + post, + uri(PathPrefix ++ ["authentication", "jwt", "move"]), + #{position => <<"before:password-based:redis">>}), + + % Valid moves + + {ok, 204, _} = request( + post, + uri(PathPrefix ++ ["authentication", "jwt", "move"]), + #{position => <<"top">>}), + + ?assertAuthenticatorsMatch( + [ + #{<<"mechanism">> := <<"jwt">>}, + #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}, + #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>} + ], + PathPrefix ++ ["authentication"]), + + {ok, 204, _} = request( + post, + uri(PathPrefix ++ ["authentication", "jwt", "move"]), + #{position => <<"bottom">>}), + + ?assertAuthenticatorsMatch( + [ + #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}, + #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>}, + #{<<"mechanism">> := <<"jwt">>} + ], + PathPrefix ++ ["authentication"]), + + {ok, 204, _} = request( + post, + uri(PathPrefix ++ ["authentication", "jwt", "move"]), + #{position => <<"before:password-based:built-in-database">>}), + + ?assertAuthenticatorsMatch( + [ + #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}, + #{<<"mechanism">> := <<"jwt">>}, + #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>} + ], + PathPrefix ++ ["authentication"]). + +test_authenticator_import_users(PathPrefix) -> + {ok, 200, _} = request( + post, + uri(PathPrefix ++ ["authentication"]), + emqx_authn_test_lib:built_in_database_example()), + + {ok, 400, _} = request( + post, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "import_users"]), + #{}), + + {ok, 400, _} = request( + post, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "import_users"]), + #{filename => <<"/etc/passwd">>}), + + {ok, 400, _} = request( + post, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "import_users"]), + #{filename => <<"/not_exists.csv">>}), + + Dir = code:lib_dir(emqx_authn, test), + JSONFileName = filename:join([Dir, <<"data/user-credentials.json">>]), + CSVFileName = filename:join([Dir, <<"data/user-credentials.csv">>]), + + {ok, 204, _} = request( + post, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "import_users"]), + #{filename => JSONFileName}), + + {ok, 204, _} = request( + post, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "import_users"]), + #{filename => CSVFileName}). + +%%------------------------------------------------------------------------------ +%% Helpers +%%------------------------------------------------------------------------------ + +delete_authenticators(Path, Chain) -> + case emqx_authentication:list_authenticators(Chain) of + {error, _} -> ok; + {ok, Authenticators} -> + lists:foreach( + fun(#{id := ID}) -> + emqx:update_config( + Path, + {delete_authenticator, Chain, ID}, + #{rawconf_with_defaults => true}) + end, + Authenticators) + end. + +response_data(Response) -> + #{<<"data">> := Data} = jiffy:decode(Response, [return_maps]), + Data. request(Method, Url) -> request(Method, Url, []). @@ -83,10 +467,7 @@ request(Method, Url, Body) -> uri() -> uri([]). uri(Parts) when is_list(Parts) -> - NParts = [E || E <- Parts], - ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION | NParts]). - -get_sources(Result) -> jsx:decode(Result). + ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION | Parts]). auth_header() -> Username = <<"admin">>, @@ -94,6 +475,5 @@ auth_header() -> {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password), {"Authorization", "Bearer " ++ binary_to_list(Token)}. -to_json(Hocon) -> - {ok, Map} =hocon:binary(Hocon), +to_json(Map) -> jiffy:encode(Map). diff --git a/apps/emqx_authn/test/emqx_authn_test_lib.erl b/apps/emqx_authn/test/emqx_authn_test_lib.erl index e30854318..7ab07c3d0 100644 --- a/apps/emqx_authn/test/emqx_authn_test_lib.erl +++ b/apps/emqx_authn/test/emqx_authn_test_lib.erl @@ -19,20 +19,15 @@ -compile(nowarn_export_all). -compile(export_all). +authenticator_example(Id) -> + #{Id := #{value := Example}} = emqx_authn_api:authenticator_examples(), + Example. + http_example() -> -""" -{ - mechanism = \"password-based\" - backend = http - method = post - url = \"http://127.0.0.2:8080\" - headers = {\"content-type\" = \"application/json\"} - body = {username = \"${username}\", - password = \"${password}\"} - pool_size = 8 - connect_timeout = 5000 - request_timeout = 5000 - enable_pipelining = true - ssl = {enable = false} -} -""". + authenticator_example('password-based:http'). + +built_in_database_example() -> + authenticator_example('password-based:built-in-database'). + +jwt_example() -> + authenticator_example(jwt). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index 5cc4d2a16..2396af9eb 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -41,7 +41,7 @@ namespace() -> "dashboard". api_spec() -> - emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}). paths() -> ["/login", "/logout", "/users", "/users/:username", "/users/:username/change_pwd"]. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 64514a7aa..6f191bd5c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -5,16 +5,16 @@ %% API -export([spec/1, spec/2]). --export([translate_req/2]). -export([namespace/0, fields/1]). +-export([schema_with_example/2, schema_with_examples/2]). -export([error_codes/1, error_codes/2]). --define(MAX_ROW_LIMIT, 100). - -%% API -ifdef(TEST). --compile(export_all). --compile(nowarn_export_all). +-export([ + parse_spec_ref/2, + components/1, + filter_check_request/2, + filter_check_request_and_translate_body/2]). -endif. -define(METHODS, [get, post, put, head, delete, patch, options, trace]). @@ -22,33 +22,39 @@ -define(DEFAULT_FIELDS, [example, allowReserved, style, explode, maxLength, allowEmptyValue, deprecated, minimum, maximum]). --define(DEFAULT_FILTER, #{filter => fun ?MODULE:translate_req/2}). - -define(INIT_SCHEMA, #{fields => #{}, translations => #{}, validations => [], namespace => undefined}). -define(TO_REF(_N_, _F_), iolist_to_binary([to_bin(_N_), ".", to_bin(_F_)])). -define(TO_COMPONENTS_SCHEMA(_M_, _F_), iolist_to_binary([<<"#/components/schemas/">>, ?TO_REF(namespace(_M_), _F_)])). -define(TO_COMPONENTS_PARAM(_M_, _F_), iolist_to_binary([<<"#/components/parameters/">>, ?TO_REF(namespace(_M_), _F_)])). +-define(MAX_ROW_LIMIT, 100). + +-type(request() :: #{bindings => map(), query_string => map(), body => map()}). +-type(request_meta() :: #{module => module(), path => string(), method => atom()}). + +-type(filter_result() :: {ok, request()} | {400, 'BAD_REQUEST', binary()}). +-type(filter() :: fun((request(), request_meta()) -> filter_result())). + +-type(spec_opts() :: #{check_schema => boolean() | filter(), translate_body => boolean()}). + +-type(route_path() :: string() | binary()). +-type(route_methods() :: map()). +-type(route_handler() :: atom()). +-type(route_options() :: #{filter => filter() | undefined}). + +-type(api_spec_entry() :: {route_path(), route_methods(), route_handler(), route_options()}). +-type(api_spec_component() :: map()). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + %% @equiv spec(Module, #{check_schema => false}) --spec(spec(module()) -> - {list({Path, Specs, OperationId, Options}), list(Component)} when - Path :: string()|binary(), - Specs :: map(), - OperationId :: atom(), - Options :: #{filter => fun((map(), - #{module => module(), path => string(), method => atom()}) -> map())}, - Component :: map()). +-spec(spec(module()) -> {list(api_spec_entry()), list(api_spec_component())}). spec(Module) -> spec(Module, #{check_schema => false}). --spec(spec(module(), #{check_schema => boolean()}) -> - {list({Path, Specs, OperationId, Options}), list(Component)} when - Path :: string()|binary(), - Specs :: map(), - OperationId :: atom(), - Options :: #{filter => fun((map(), - #{module => module(), path => string(), method => atom()}) -> map())}, - Component :: map()). +-spec(spec(module(), spec_opts()) -> {list(api_spec_entry()), list(api_spec_component())}). spec(Module, Options) -> Paths = apply(Module, paths, []), {ApiSpec, AllRefs} = @@ -60,26 +66,10 @@ spec(Module, Options) -> end, {[], []}, Paths), {ApiSpec, components(lists:usort(AllRefs))}. --spec(translate_req(#{binding => list(), query_string => list(), body => map()}, - #{module => module(), path => string(), method => atom()}) -> - {ok, #{binding => list(), query_string => list(), body => map()}}| - {400, 'BAD_REQUEST', binary()}). -translate_req(Request, #{module := Module, path := Path, method := Method}) -> - #{Method := Spec} = apply(Module, schema, [Path]), - try - Params = maps:get(parameters, Spec, []), - Body = maps:get(requestBody, Spec, []), - {Bindings, QueryStr} = check_parameters(Request, Params, Module), - NewBody = check_requestBody(Request, Body, Module, hoconsc:is_schema(Body)), - {ok, Request#{bindings => Bindings, query_string => QueryStr, body => NewBody}} - catch throw:Error -> - {_, [{validation_error, ValidErr}]} = Error, - #{path := Key, reason := Reason} = ValidErr, - {400, 'BAD_REQUEST', iolist_to_binary(io_lib:format("~ts : ~p", [Key, Reason]))} - end. - +-spec(namespace() -> hocon_schema:name()). namespace() -> "public". +-spec(fields(hocon_schema:name()) -> hocon_schema:fields()). fields(page) -> Desc = <<"Page number of the results to fetch.">>, Meta = #{in => query, desc => Desc, default => 1, example => 1}, @@ -90,9 +80,19 @@ fields(limit) -> Meta = #{in => query, desc => Desc, default => ?MAX_ROW_LIMIT, example => 50}, [{limit, hoconsc:mk(range(1, ?MAX_ROW_LIMIT), Meta)}]. +-spec(schema_with_example(hocon_schema:type(), term()) -> hocon_schema:field_schema_map()). +schema_with_example(Type, Example) -> + hoconsc:mk(Type, #{examples => #{<<"example">> => Example}}). + +-spec(schema_with_examples(hocon_schema:type(), map()) -> hocon_schema:field_schema_map()). +schema_with_examples(Type, Examples) -> + hoconsc:mk(Type, #{examples => #{<<"examples">> => Examples}}). + +-spec(error_codes(list(atom())) -> hocon_schema:fields()). error_codes(Codes) -> error_codes(Codes, <<"Error code to troubleshoot problems.">>). +-spec(error_codes(nonempty_list(atom()), binary()) -> hocon_schema:fields()). error_codes(Codes = [_ | _], MsgExample) -> [ {code, hoconsc:mk(hoconsc:enum(Codes))}, @@ -102,9 +102,45 @@ error_codes(Codes = [_ | _], MsgExample) -> })} ]. -support_check_schema(#{check_schema := true}) -> ?DEFAULT_FILTER; -support_check_schema(#{check_schema := Func}) when is_function(Func, 2) -> #{filter => Func}; -support_check_schema(_) -> #{filter => undefined}. +%%------------------------------------------------------------------------------ +%% Private functions +%%------------------------------------------------------------------------------ + +filter_check_request_and_translate_body(Request, RequestMeta) -> + translate_req(Request, RequestMeta, fun check_and_translate/3). + +filter_check_request(Request, RequestMeta) -> + translate_req(Request, RequestMeta, fun check_only/3). + +translate_req(Request, #{module := Module, path := Path, method := Method}, CheckFun) -> + #{Method := Spec} = apply(Module, schema, [Path]), + try + Params = maps:get(parameters, Spec, []), + Body = maps:get(requestBody, Spec, []), + {Bindings, QueryStr} = check_parameters(Request, Params, Module), + NewBody = check_requestBody(Request, Body, Module, CheckFun, hoconsc:is_schema(Body)), + {ok, Request#{bindings => Bindings, query_string => QueryStr, body => NewBody}} + catch throw:Error -> + {_, [{validation_error, ValidErr}]} = Error, + #{path := Key, reason := Reason} = ValidErr, + {400, 'BAD_REQUEST', iolist_to_binary(io_lib:format("~ts : ~p", [Key, Reason]))} + end. + +check_and_translate(Schema, Map, Opts) -> + hocon_schema:check_plain(Schema, Map, Opts). + +check_only(Schema, Map, Opts) -> + _ = hocon_schema:check_plain(Schema, Map, Opts), + Map. + +support_check_schema(#{check_schema := true, translate_body := true}) -> + #{filter => fun filter_check_request_and_translate_body/2}; +support_check_schema(#{check_schema := true}) -> + #{filter => fun filter_check_request/2}; +support_check_schema(#{check_schema := Filter}) when is_function(Filter, 2) -> + #{filter => Filter}; +support_check_schema(_) -> + #{filter => undefined}. parse_spec_ref(Module, Path) -> Schema = @@ -143,10 +179,10 @@ check_parameter([{Name, Type} | Spec], Bindings, QueryStr, Module, BindingsAcc, query -> NewQueryStr = hocon_schema:check_plain(Schema, QueryStr, #{override_env => false}), NewQueryStrAcc = maps:merge(QueryStrAcc, NewQueryStr), - check_parameter(Spec, Bindings, QueryStr, Module, BindingsAcc, NewQueryStrAcc) + check_parameter(Spec, Bindings, QueryStr, Module,BindingsAcc, NewQueryStrAcc) end. -check_requestBody(#{body := Body}, Schema, Module, true) -> +check_requestBody(#{body := Body}, Schema, Module, CheckFun, true) -> Type0 = hocon_schema:field_schema(Schema, type), Type = case Type0 of @@ -154,7 +190,7 @@ check_requestBody(#{body := Body}, Schema, Module, true) -> _ -> Type0 end, NewSchema = ?INIT_SCHEMA#{roots => [{root, Type}]}, - #{<<"root">> := NewBody} = hocon_schema:check_plain(NewSchema, #{<<"root">> => Body}, #{override_env => false}), + #{<<"root">> := NewBody} = CheckFun(NewSchema, #{<<"root">> => Body}, #{override_env => false}), NewBody; %% TODO not support nest object check yet, please use ref! %% RequestBody = [ {per_page, mk(integer(), #{}}, @@ -163,10 +199,10 @@ check_requestBody(#{body := Body}, Schema, Module, true) -> %% {good_nest_2, mk(ref(?MODULE, good_ref), #{})} %% ]} %% ] -check_requestBody(#{body := Body}, Spec, _Module, false) -> +check_requestBody(#{body := Body}, Spec, _Module, CheckFun, false) -> lists:foldl(fun({Name, Type}, Acc) -> Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]}, - maps:merge(Acc, hocon_schema:check_plain(Schema, Body)) + maps:merge(Acc, CheckFun(Schema, Body, #{})) end, #{}, Spec). %% tags, description, summary, security, deprecated @@ -244,14 +280,15 @@ trans_desc(Spec, Hocon) -> requestBody([], _Module) -> {[], []}; requestBody(Schema, Module) -> - {Props, Refs} = + {{Props, Refs}, Examples} = case hoconsc:is_schema(Schema) of true -> HoconSchema = hocon_schema:field_schema(Schema, type), - hocon_schema_to_spec(HoconSchema, Module); - false -> parse_object(Schema, Module) + SchemaExamples = hocon_schema:field_schema(Schema, examples), + {hocon_schema_to_spec(HoconSchema, Module), SchemaExamples}; + false -> {parse_object(Schema, Module), undefined} end, - {#{<<"content">> => #{<<"application/json">> => #{<<"schema">> => Props}}}, + {#{<<"content">> => content(Props, Examples)}, Refs}. responses(Responses, Module) -> @@ -264,19 +301,20 @@ response(Status, ?REF(StructName), {Acc, RefsAcc, Module}) -> response(Status, ?R_REF(Module, StructName), {Acc, RefsAcc, Module}); response(Status, ?R_REF(_Mod, _Name) = RRef, {Acc, RefsAcc, Module}) -> {Spec, Refs} = hocon_schema_to_spec(RRef, Module), - Content = #{<<"application/json">> => #{<<"schema">> => Spec}}, + Content = content(Spec), {Acc#{integer_to_binary(Status) => #{<<"content">> => Content}}, Refs ++ RefsAcc, Module}; response(Status, Schema, {Acc, RefsAcc, Module}) -> case hoconsc:is_schema(Schema) of true -> Hocon = hocon_schema:field_schema(Schema, type), + Examples = hocon_schema:field_schema(Schema, examples), {Spec, Refs} = hocon_schema_to_spec(Hocon, Module), Init = trans_desc(#{}, Schema), - Content = #{<<"application/json">> => #{<<"schema">> => Spec}}, + Content = content(Spec, Examples), {Acc#{integer_to_binary(Status) => Init#{<<"content">> => Content}}, Refs ++ RefsAcc, Module}; false -> {Props, Refs} = parse_object(Schema, Module), - Content = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => Props}}}, + Content = #{<<"content">> => content(Props)}, {Acc#{integer_to_binary(Status) => Content}, Refs ++ RefsAcc, Module} end. @@ -467,3 +505,11 @@ parse_object(Other, Module) -> is_required(Hocon) -> hocon_schema:field_schema(Hocon, required) =:= true orelse hocon_schema:field_schema(Hocon, nullable) =:= false. + +content(ApiSpec) -> + content(ApiSpec, undefined). + +content(ApiSpec, undefined) -> + #{<<"application/json">> => #{<<"schema">> => ApiSpec}}; +content(ApiSpec, Examples) when is_map(Examples) -> + #{<<"application/json">> => Examples#{<<"schema">> => ApiSpec}}. diff --git a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl index 9c9958880..c938788e4 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl @@ -209,14 +209,32 @@ t_in_mix_trans_error(_Config) -> ok. t_api_spec(_Config) -> - {Spec, _Components} = emqx_dashboard_swagger:spec(?MODULE), - Filter = fun(V, S) -> lists:all(fun({_, _, _, #{filter := Filter}}) -> Filter =:= V end, S) end, - ?assertEqual(true, Filter(undefined, Spec)), - {Spec1, _Components1} = emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}), - ?assertEqual(true, Filter(fun emqx_dashboard_swagger:translate_req/2, Spec1)), - {Spec2, _Components2} = emqx_dashboard_swagger:spec(?MODULE, #{check_schema => fun emqx_dashboard_swagger:translate_req/2}), - ?assertEqual(true, Filter(fun emqx_dashboard_swagger:translate_req/2, Spec2)), - ok. + {Spec0, _} = emqx_dashboard_swagger:spec(?MODULE), + assert_all_filters_equal(Spec0, undefined), + + {Spec1, _} = emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false}), + assert_all_filters_equal(Spec1, undefined), + + CustomFilter = fun(Request, _RequestMeta) -> {ok, Request} end, + {Spec2, _} = emqx_dashboard_swagger:spec(?MODULE, #{check_schema => CustomFilter}), + assert_all_filters_equal(Spec2, CustomFilter), + + {Spec3, _} = emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}), + Path = "/test/in/:filter", + + Filter = filter(Spec3, Path), + Bindings = #{filter => <<"created">>}, + + ?assertMatch( + {ok, #{bindings := #{filter := created}}}, + trans_parameters(Path, Bindings, #{}, Filter)). + +assert_all_filters_equal(Spec, Filter) -> + lists:foreach( + fun({_, _, _, #{filter := F}}) -> + ?assertEqual(Filter, F) + end, + Spec). validate(Path, ExpectParams) -> {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path), @@ -226,10 +244,17 @@ validate(Path, ExpectParams) -> ?assertEqual([], Refs), Spec. +filter(ApiSpec, Path) -> + [Filter] = [F || {P, _, _, #{filter := F}} <- ApiSpec, P =:= Path], + Filter. + trans_parameters(Path, Bindings, QueryStr) -> + trans_parameters(Path, Bindings, QueryStr, fun emqx_dashboard_swagger:filter_check_request/2). + +trans_parameters(Path, Bindings, QueryStr, Filter) -> Meta = #{module => ?MODULE, method => post, path => Path}, Request = #{bindings => Bindings, query_string => QueryStr, body => #{}}, - emqx_dashboard_swagger:translate_req(Request, Meta). + Filter(Request, Meta). api_spec() -> emqx_dashboard_swagger:spec(?MODULE). diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index 7aa986d1d..768942776 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -10,7 +10,7 @@ t_ref_array_with_key/1, t_ref_array_without_key/1 ]). -export([ - t_object_trans/1, t_nest_object_trans/1, t_local_ref_trans/1, + t_object_trans/1, t_object_notrans/1, t_nest_object_trans/1, t_local_ref_trans/1, t_remote_ref_trans/1, t_nest_ref_trans/1, t_ref_array_with_key_trans/1, t_ref_array_without_key_trans/1, t_ref_trans_error/1, t_object_trans_error/1 @@ -32,7 +32,7 @@ groups() -> [ t_ref_array_with_key, t_ref_array_without_key, t_nest_ref]}, {validation, [parallel], [ - t_object_trans, t_local_ref_trans, t_remote_ref_trans, + t_object_trans, t_object_notrans, t_local_ref_trans, t_remote_ref_trans, t_ref_array_with_key_trans, t_ref_array_without_key_trans, t_nest_ref_trans, t_ref_trans_error, t_object_trans_error %% t_nest_object_trans, @@ -173,8 +173,29 @@ t_ref_array_without_key(_Config) -> ok. t_api_spec(_Config) -> - emqx_dashboard_swagger:spec(?MODULE), - ok. + {Spec0, _} = emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}), + Path = "/object", + Body = #{ + <<"per_page">> => 1, + <<"timeout">> => <<"infinity">>, + <<"inner_ref">> => #{ + <<"webhook-host">> => <<"127.0.0.1:80">>, + <<"log_dir">> => <<"var/log/test">>, + <<"tag">> => <<"god_tag">> + } + }, + + Filter0 = filter(Spec0, Path), + ?assertMatch( + {ok, #{body := ActualBody}}, + trans_requestBody(Path, Body, Filter0)), + + {Spec1, _} = emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}), + Filter1 = filter(Spec1, Path), + ?assertMatch( + {ok, #{body := #{<<"timeout">> := infinity}}}, + trans_requestBody(Path, Body, Filter1)). + t_object_trans(_Config) -> Path = "/object", @@ -205,6 +226,21 @@ t_object_trans(_Config) -> ?assertEqual(Expect, ActualBody), ok. +t_object_notrans(_Config) -> + Path = "/object", + Body = #{ + <<"per_page">> => 1, + <<"timeout">> => <<"infinity">>, + <<"inner_ref">> => #{ + <<"webhook-host">> => <<"127.0.0.1:80">>, + <<"log_dir">> => <<"var/log/test">>, + <<"tag">> => <<"god_tag">> + } + }, + {ok, #{body := ActualBody}} = trans_requestBody(Path, Body, fun emqx_dashboard_swagger:filter_check_request/2), + ?assertEqual(Body, ActualBody), + ok. + t_nest_object_trans(_Config) -> Path = "/nest/object", Body = #{ @@ -337,6 +373,7 @@ t_ref_array_with_key_trans(_Config) -> {ok, NewRequest} = trans_requestBody(Path, Body), ?assertEqual(Expect, NewRequest), ok. + t_ref_array_without_key_trans(_Config) -> Path = "/ref/array/without/key", Body = [#{ @@ -401,10 +438,18 @@ validate(Path, ExpectSpec, ExpectRefs) -> ?assertEqual(ExpectRefs, Refs), {Spec, emqx_dashboard_swagger:components(Refs)}. + +filter(ApiSpec, Path) -> + [Filter] = [F || {P, _, _, #{filter := F}} <- ApiSpec, P =:= Path], + Filter. + trans_requestBody(Path, Body) -> + trans_requestBody(Path, Body, fun emqx_dashboard_swagger:filter_check_request_and_translate_body/2). + +trans_requestBody(Path, Body, Filter) -> Meta = #{module => ?MODULE, method => post, path => Path}, Request = #{bindings => #{}, query_string => #{}, body => Body}, - emqx_dashboard_swagger:translate_req(Request, Meta). + Filter(Request, Meta). api_spec() -> emqx_dashboard_swagger:spec(?MODULE). paths() -> diff --git a/apps/emqx_modules/src/emqx_rewrite_api.erl b/apps/emqx_modules/src/emqx_rewrite_api.erl index 1fa5e9467..3f92cd11f 100644 --- a/apps/emqx_modules/src/emqx_rewrite_api.erl +++ b/apps/emqx_modules/src/emqx_rewrite_api.erl @@ -33,7 +33,7 @@ ]). api_spec() -> - emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}). paths() -> ["/mqtt/topic_rewrite"]. From 29fb9b33618e85c825bf10cd400add09fdf364fb Mon Sep 17 00:00:00 2001 From: zhouzb Date: Mon, 1 Nov 2021 18:49:13 +0800 Subject: [PATCH 013/179] fix(authn): fix bad type of hash --- apps/emqx_authn/src/emqx_authn_utils.erl | 6 +++++- apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_utils.erl b/apps/emqx_authn/src/emqx_authn_utils.erl index 4784c91c7..4aa7f550e 100644 --- a/apps/emqx_authn/src/emqx_authn_utils.erl +++ b/apps/emqx_authn/src/emqx_authn_utils.erl @@ -62,7 +62,7 @@ check_password(undefined, _Selected, _State) -> check_password(Password, #{<<"password_hash">> := Hash}, #{password_hash_algorithm := bcrypt}) -> - case {ok, Hash} =:= bcrypt:hashpw(Password, Hash) of + case {ok, to_list(Hash)} =:= bcrypt:hashpw(Password, Hash) of true -> ok; false -> {error, bad_username_or_password} end; @@ -100,3 +100,7 @@ convert_to_sql_param(undefined) -> null; convert_to_sql_param(V) -> bin(V). + +to_list(L) when is_list(L) -> L; +to_list(L) when is_binary(L) -> binary_to_list(L); +to_list(X) -> X. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index ce5d3d8ee..7ce3f46d0 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -205,7 +205,7 @@ check_password(Password, undefined -> {error, {cannot_find_password_hash_field, PasswordHashField}}; Hash -> - case {ok, Hash} =:= bcrypt:hashpw(Password, Hash) of + case {ok, to_list(Hash)} =:= bcrypt:hashpw(Password, Hash) of true -> ok; false -> {error, bad_username_or_password} end @@ -238,3 +238,7 @@ hash(Algorithm, Password, Salt, prefix) -> emqx_passwd:hash(Algorithm, <>); hash(Algorithm, Password, Salt, suffix) -> emqx_passwd:hash(Algorithm, <>). + +to_list(L) when is_list(L) -> L; +to_list(L) when is_binary(L) -> binary_to_list(L); +to_list(X) -> X. From 7ae6e04582f856c220d93b18e6665264010fa42e Mon Sep 17 00:00:00 2001 From: Tobias Lindahl Date: Wed, 27 Oct 2021 14:09:08 +0200 Subject: [PATCH 014/179] fix(persistent_sessions): channels can terminate without a session --- apps/emqx/src/emqx_channel.erl | 19 +++++++++++++------ apps/emqx/src/emqx_session.erl | 4 ++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 7df7cd42d..fc25490c4 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1179,20 +1179,27 @@ terminate(_, #channel{conn_state = idle}) -> ok; terminate(normal, Channel) -> run_terminate_hook(normal, Channel); terminate({shutdown, kicked}, Channel) -> - _ = emqx_persistent_session:persist(Channel#channel.clientinfo, - Channel#channel.conninfo, - Channel#channel.session), + persist_if_session(Channel), run_terminate_hook(kicked, Channel); terminate({shutdown, Reason}, Channel) when Reason =:= discarded; Reason =:= takeovered -> run_terminate_hook(Reason, Channel); terminate(Reason, Channel = #channel{will_msg = WillMsg}) -> (WillMsg =/= undefined) andalso publish_will_msg(WillMsg), - _ = emqx_persistent_session:persist(Channel#channel.clientinfo, - Channel#channel.conninfo, - Channel#channel.session), + persist_if_session(Channel), run_terminate_hook(Reason, Channel). +persist_if_session(#channel{session = Session} = Channel) -> + case emqx_session:is_session(Session) of + true -> + _ = emqx_persistent_session:persist(Channel#channel.clientinfo, + Channel#channel.conninfo, + Channel#channel.session), + ok; + false -> + ok + end. + run_terminate_hook(_Reason, #channel{session = undefined}) -> ok; run_terminate_hook(Reason, #channel{clientinfo = ClientInfo, session = Session}) -> emqx_session:terminate(ClientInfo, Reason, Session). diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index 03414fc60..ab80bd2be 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -58,6 +58,7 @@ -export([ info/1 , info/2 + , is_session/1 , stats/1 ]). @@ -202,6 +203,9 @@ init(Opts) -> %% Info, Stats %%-------------------------------------------------------------------- +is_session(#session{}) -> true; +is_session(_) -> false. + %% @doc Get infos of the session. -spec(info(session()) -> emqx_types:infos()). info(Session) -> From ec68d7fc589b2b7d2f5de8fdf963ebede9511077 Mon Sep 17 00:00:00 2001 From: Tobias Lindahl Date: Wed, 27 Oct 2021 14:12:03 +0200 Subject: [PATCH 015/179] test(persistent_sessions): stabilize flaky tests --- apps/emqx/src/emqx_cm.erl | 9 ++- .../test/emqx_persistent_session_SUITE.erl | 68 ++++++++++++------- 2 files changed, 51 insertions(+), 26 deletions(-) diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index 83f77050a..dbd8dfa63 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -58,7 +58,9 @@ , lookup_channels/2 ]). --export([all_channels/0]). +-export([ all_channels/0 + , all_client_ids/0 + ]). %% gen_server callbacks -export([ init/1 @@ -400,6 +402,11 @@ all_channels() -> Pat = [{{'_', '$1'}, [], ['$1']}], ets:select(?CHAN_TAB, Pat). +all_client_ids() -> + Pat = [{{'$1', '_'}, [], ['$1']}], + ets:select(?CHAN_TAB, Pat). + + %% @doc Lookup channels. -spec(lookup_channels(emqx_types:clientid()) -> list(chan_pid())). lookup_channels(ClientId) -> diff --git a/apps/emqx/test/emqx_persistent_session_SUITE.erl b/apps/emqx/test/emqx_persistent_session_SUITE.erl index ac51636f0..3e992ce2d 100644 --- a/apps/emqx/test/emqx_persistent_session_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_session_SUITE.erl @@ -113,6 +113,8 @@ init_per_group(snabbkaffe, Config) -> [ {kill_connection_process, true} | Config]; init_per_group(gc_tests, Config) -> %% We need to make sure the system does not interfere with this test group. + [maybe_kill_connection_process(ClientId, [{kill_connection_process, true}]) + || ClientId <- emqx_cm:all_client_ids()], emqx_common_test_helpers:stop_apps([]), SessionMsgEts = gc_tests_session_store, MsgEts = gc_tests_msg_store, @@ -230,32 +232,48 @@ receive_messages(Count, Msgs) -> maybe_kill_connection_process(ClientId, Config) -> case ?config(kill_connection_process, Config) of true -> - [ConnectionPid] = emqx_cm:lookup_channels(ClientId), - ?assert(is_pid(ConnectionPid)), - Ref = monitor(process, ConnectionPid), - ConnectionPid ! die_if_test, - receive {'DOWN', Ref, process, ConnectionPid, normal} -> ok - after 3000 -> error(process_did_not_die) + case emqx_cm:lookup_channels(ClientId) of + [] -> + ok; + [ConnectionPid] -> + ?assert(is_pid(ConnectionPid)), + Ref = monitor(process, ConnectionPid), + ConnectionPid ! die_if_test, + receive {'DOWN', Ref, process, ConnectionPid, normal} -> ok + after 3000 -> error(process_did_not_die) + end, + wait_for_cm_unregister(ClientId) end; false -> ok end. -snabbkaffe_sync_publish(Topic, Payloads, Config) -> +wait_for_cm_unregister(ClientId) -> + wait_for_cm_unregister(ClientId, 10). + +wait_for_cm_unregister(_ClientId, 0) -> + error(cm_did_not_unregister); +wait_for_cm_unregister(ClientId, N) -> + case emqx_cm:lookup_channels(ClientId) of + [] -> ok; + [_] -> timer:sleep(100), wait_for_cm_unregister(ClientId, N - 1) + end. + +snabbkaffe_sync_publish(Topic, Payloads) -> Fun = fun(Client, Payload) -> ?wait_async_action( {ok, _} = emqtt:publish(Client, Topic, Payload, 2) , #{?snk_kind := ps_persist_msg, payload := Payload} ) end, - do_publish(Payloads, Fun, Config). + do_publish(Payloads, Fun). -publish(Topic, Payloads, Config) -> +publish(Topic, Payloads) -> Fun = fun(Client, Payload) -> {ok, _} = emqtt:publish(Client, Topic, Payload, 2) end, - do_publish(Payloads, Fun, Config). + do_publish(Payloads, Fun). -do_publish(Payloads = [_|_], PublishFun, Config) -> +do_publish(Payloads = [_|_], PublishFun) -> %% Publish from another process to avoid connection confusion. {Pid, Ref} = spawn_monitor( @@ -272,8 +290,8 @@ do_publish(Payloads = [_|_], PublishFun, Config) -> {'DOWN', Ref, process, Pid, normal} -> ok; {'DOWN', Ref, process, Pid, What} -> error({failed_publish, What}) end; -do_publish(Payload, PublishFun, Config) -> - do_publish([Payload], PublishFun, Config). +do_publish(Payload, PublishFun) -> + do_publish([Payload], PublishFun). %%-------------------------------------------------------------------- %% Test Cases @@ -297,7 +315,7 @@ t_connect_session_expiry_interval(Config) -> maybe_kill_connection_process(ClientId, Config), - publish(Topic, Payload, Config), + publish(Topic, Payload), {ok, Client2} = emqtt:start_link([ {clientid, ClientId}, {proto_ver, v5}, @@ -424,7 +442,7 @@ t_process_dies_session_expires(Config) -> maybe_kill_connection_process(ClientId, Config), - ok = publish(Topic, [Payload], Config), + ok = publish(Topic, [Payload]), SessionId = case ?config(persistent_store_enabled, Config) of @@ -498,7 +516,7 @@ t_publish_while_client_is_gone(Config) -> ok = emqtt:disconnect(Client1), maybe_kill_connection_process(ClientId, Config), - ok = publish(Topic, [Payload1, Payload2], Config), + ok = publish(Topic, [Payload1, Payload2]), {ok, Client2} = emqtt:start_link([ {proto_ver, v5}, {clientid, ClientId}, @@ -544,7 +562,7 @@ t_clean_start_drops_subscriptions(Config) -> maybe_kill_connection_process(ClientId, Config), %% 2. - ok = publish(Topic, Payload1, Config), + ok = publish(Topic, Payload1), %% 3. {ok, Client2} = emqtt:start_link([ {proto_ver, v5}, @@ -556,7 +574,7 @@ t_clean_start_drops_subscriptions(Config) -> ?assertEqual(0, client_info(session_present, Client2)), {ok, _, [2]} = emqtt:subscribe(Client2, STopic, qos2), - ok = publish(Topic, Payload2, Config), + ok = publish(Topic, Payload2), [Msg1] = receive_messages(1), ?assertEqual({ok, iolist_to_binary(Payload2)}, maps:find(payload, Msg1)), @@ -571,7 +589,7 @@ t_clean_start_drops_subscriptions(Config) -> | Config]), {ok, _} = emqtt:ConnFun(Client3), - ok = publish(Topic, Payload3, Config), + ok = publish(Topic, Payload3), [Msg2] = receive_messages(1), ?assertEqual({ok, iolist_to_binary(Payload3)}, maps:find(payload, Msg2)), @@ -625,7 +643,7 @@ t_multiple_subscription_matches(Config) -> maybe_kill_connection_process(ClientId, Config), - publish(Topic, Payload, Config), + publish(Topic, Payload), {ok, Client2} = emqtt:start_link([ {clientid, ClientId}, {proto_ver, v5}, @@ -675,9 +693,9 @@ t_lost_messages_because_of_gc(Config) -> {ok, _, [2]} = emqtt:subscribe(Client1, STopic, qos2), emqtt:disconnect(Client1), maybe_kill_connection_process(ClientId, Config), - publish(Topic, Payload1, Config), + publish(Topic, Payload1), timer:sleep(2 * Retain), - publish(Topic, Payload2, Config), + publish(Topic, Payload2), emqx_persistent_session_gc:message_gc_worker(), {ok, Client2} = emqtt:start_link([ {clientid, ClientId}, {clean_start, false}, @@ -790,7 +808,7 @@ t_snabbkaffe_pending_messages(Config) -> ?check_trace( begin - snabbkaffe_sync_publish(Topic, Payloads, Config), + snabbkaffe_sync_publish(Topic, Payloads), {ok, Client2} = emqtt:start_link([{clean_start, false} | EmqttOpts]), {ok, _} = emqtt:ConnFun(Client2), Msgs = receive_messages(length(Payloads)), @@ -829,7 +847,7 @@ t_snabbkaffe_buffered_messages(Config) -> ok = emqtt:disconnect(Client1), maybe_kill_connection_process(ClientId, Config), - publish(Topic, Payloads1, Config), + publish(Topic, Payloads1), ?check_trace( begin @@ -838,7 +856,7 @@ t_snabbkaffe_buffered_messages(Config) -> #{ ?snk_kind := ps_resume_end }), spawn_link(fun() -> ?block_until(#{ ?snk_kind := ps_marker_pendings_msgs }, infinity, 5000), - publish(Topic, Payloads2, Config) + publish(Topic, Payloads2) end), {ok, Client2} = emqtt:start_link([{clean_start, false} | EmqttOpts]), {ok, _} = emqtt:ConnFun(Client2), From 1f13a6caad009b4ad5629638d35e01e3b6d6da70 Mon Sep 17 00:00:00 2001 From: Tobias Lindahl Date: Thu, 28 Oct 2021 09:56:36 +0200 Subject: [PATCH 016/179] chore(persistent_sessions): tune mnesia parameters for better dump behavior --- apps/emqx/etc/emqx_cloud/vm.args | 4 ++++ apps/emqx/etc/emqx_edge/vm.args | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/apps/emqx/etc/emqx_cloud/vm.args b/apps/emqx/etc/emqx_cloud/vm.args index 1e6b0b4cb..0ee4b1e15 100644 --- a/apps/emqx/etc/emqx_cloud/vm.args +++ b/apps/emqx/etc/emqx_cloud/vm.args @@ -116,3 +116,7 @@ ## patches dir -pa {{ platform_data_dir }}/patches + +## Mnesia thresholds +-mnesia dump_log_write_threshold 5000 +-mnesia dump_log_time_threshold 60000 diff --git a/apps/emqx/etc/emqx_edge/vm.args b/apps/emqx/etc/emqx_edge/vm.args index ef9749738..70ce81f9f 100644 --- a/apps/emqx/etc/emqx_edge/vm.args +++ b/apps/emqx/etc/emqx_edge/vm.args @@ -114,3 +114,7 @@ ## patches dir -pa {{ platform_data_dir }}/patches + +## Mnesia thresholds +-mnesia dump_log_write_threshold 5000 +-mnesia dump_log_time_threshold 60000 From 329dd4d780afb096f2476bce110085d72023d53f Mon Sep 17 00:00:00 2001 From: Tobias Lindahl Date: Mon, 1 Nov 2021 10:17:00 +0100 Subject: [PATCH 017/179] test(persistent_session): try to fix flaky snabbkaffe failure --- apps/emqx/test/emqx_persistent_session_SUITE.erl | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/apps/emqx/test/emqx_persistent_session_SUITE.erl b/apps/emqx/test/emqx_persistent_session_SUITE.erl index 3e992ce2d..fbf1696f0 100644 --- a/apps/emqx/test/emqx_persistent_session_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_session_SUITE.erl @@ -265,33 +265,37 @@ snabbkaffe_sync_publish(Topic, Payloads) -> , #{?snk_kind := ps_persist_msg, payload := Payload} ) end, - do_publish(Payloads, Fun). + do_publish(Payloads, Fun, true). publish(Topic, Payloads) -> Fun = fun(Client, Payload) -> {ok, _} = emqtt:publish(Client, Topic, Payload, 2) end, - do_publish(Payloads, Fun). + do_publish(Payloads, Fun, false). -do_publish(Payloads = [_|_], PublishFun) -> +do_publish(Payloads = [_|_], PublishFun, WaitForUnregister) -> %% Publish from another process to avoid connection confusion. {Pid, Ref} = spawn_monitor( fun() -> %% For convenience, always publish using tcp. %% The publish path is not what we are testing. + ClientID = <<"ps_SUITE_publisher">>, {ok, Client} = emqtt:start_link([ {proto_ver, v5} + , {clientid, ClientID} , {port, 1883} ]), {ok, _} = emqtt:connect(Client), lists:foreach(fun(Payload) -> PublishFun(Client, Payload) end, Payloads), - ok = emqtt:disconnect(Client) + ok = emqtt:disconnect(Client), + %% Snabbkaffe sometimes fails unless all processes are gone. + [wait_for_cm_unregister(ClientID) || WaitForUnregister] end), receive {'DOWN', Ref, process, Pid, normal} -> ok; {'DOWN', Ref, process, Pid, What} -> error({failed_publish, What}) end; -do_publish(Payload, PublishFun) -> - do_publish([Payload], PublishFun). +do_publish(Payload, PublishFun, WaitForUnregister) -> + do_publish([Payload], PublishFun, WaitForUnregister). %%-------------------------------------------------------------------- %% Test Cases From ce49a281ed40c53177e9b5a62949b8dc618ab2b5 Mon Sep 17 00:00:00 2001 From: Tobias Lindahl Date: Mon, 1 Nov 2021 13:50:16 +0100 Subject: [PATCH 018/179] fix(persistent_sessions): protect against looking up stale data --- apps/emqx/src/emqx_persistent_session.erl | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session.erl b/apps/emqx/src/emqx_persistent_session.erl index 71dac02c3..13c74b62e 100644 --- a/apps/emqx/src/emqx_persistent_session.erl +++ b/apps/emqx/src/emqx_persistent_session.erl @@ -179,12 +179,17 @@ timestamp_from_conninfo(ConnInfo) -> end. lookup(ClientID) when is_binary(ClientID) -> - case lookup_session_store(ClientID) of - none -> none; - {value, #session_store{session = S} = SS} -> - case persistent_session_status(SS) of - expired -> {expired, S}; - persistent -> {persistent, S} + case is_store_enabled() of + false -> + none; + true -> + case lookup_session_store(ClientID) of + none -> none; + {value, #session_store{session = S} = SS} -> + case persistent_session_status(SS) of + expired -> {expired, S}; + persistent -> {persistent, S} + end end end. From 796553b5ea725af554e5eaffa7cf12e290fec77f Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 1 Nov 2021 16:51:48 +0300 Subject: [PATCH 019/179] fix(authn api): eliminate possible atom leak --- apps/emqx/src/emqx_authentication.erl | 24 ++-- apps/emqx/test/emqx_authentication_SUITE.erl | 2 + apps/emqx_authn/src/emqx_authn_api.erl | 124 ++++++++++-------- .../test/emqx_swagger_requestBody_SUITE.erl | 2 +- 4 files changed, 89 insertions(+), 63 deletions(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 226d697d8..a317611f1 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -25,6 +25,8 @@ -include("emqx.hrl"). -include("logger.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). + %% The authentication entrypoint. -export([ authenticate/2 ]). @@ -45,6 +47,7 @@ , delete_chain/1 , lookup_chain/1 , list_chains/0 + , list_chain_names/0 , create_authenticator/2 , delete_authenticator/2 , update_authenticator/3 @@ -312,13 +315,24 @@ delete_chain(Name) -> -spec lookup_chain(chain_name()) -> {ok, chain()} | {error, term()}. lookup_chain(Name) -> - call({lookup_chain, Name}). + case ets:lookup(?CHAINS_TAB, Name) of + [] -> + {error, {not_found, {chain, Name}}}; + [Chain] -> + {ok, serialize_chain(Chain)} + end. -spec list_chains() -> {ok, [chain()]}. list_chains() -> Chains = ets:tab2list(?CHAINS_TAB), {ok, [serialize_chain(Chain) || Chain <- Chains]}. +-spec list_chain_names() -> {ok, [atom()]}. +list_chain_names() -> + Select = ets:fun2ms(fun(#chain{name = Name}) -> Name end), + ChainNames = ets:select(?CHAINS_TAB, Select), + {ok, ChainNames}. + -spec create_authenticator(chain_name(), config()) -> {ok, authenticator()} | {error, term()}. create_authenticator(ChainName, Config) -> call({create_authenticator, ChainName, Config}). @@ -432,14 +446,6 @@ handle_call({delete_chain, Name}, _From, State) -> reply(ok, maybe_unhook(State)) end; -handle_call({lookup_chain, Name}, _From, State) -> - case ets:lookup(?CHAINS_TAB, Name) of - [] -> - reply({error, {not_found, {chain, Name}}}, State); - [Chain] -> - reply({ok, serialize_chain(Chain)}, State) - end; - handle_call({create_authenticator, ChainName, Config}, _From, #{providers := Providers} = State) -> UpdateFun = fun(#chain{authenticators = Authenticators} = Chain) -> diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index 62224a87f..3ddd1f64e 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -108,10 +108,12 @@ t_chain(Config) when is_list(Config) -> % CRUD of authentication chain ChainName = 'test', ?assertMatch({ok, []}, ?AUTHN:list_chains()), + ?assertMatch({ok, []}, ?AUTHN:list_chain_names()), ?assertMatch({ok, #{name := ChainName, authenticators := []}}, ?AUTHN:create_chain(ChainName)), ?assertEqual({error, {already_exists, {chain, ChainName}}}, ?AUTHN:create_chain(ChainName)), ?assertMatch({ok, #{name := ChainName, authenticators := []}}, ?AUTHN:lookup_chain(ChainName)), ?assertMatch({ok, [#{name := ChainName}]}, ?AUTHN:list_chains()), + ?assertEqual({ok, [ChainName]}, ?AUTHN:list_chain_names()), ?assertEqual(ok, ?AUTHN:delete_chain(ChainName)), ?assertMatch({error, {not_found, {chain, ChainName}}}, ?AUTHN:lookup_chain(ChainName)), ok. diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index c3978fb0d..9721dad93 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -498,37 +498,37 @@ authenticator(delete, #{bindings := #{id := AuthenticatorID}}) -> listener_authenticators(post, #{bindings := #{listener_id := ListenerID}, body := Config}) -> with_listener(ListenerID, - fun(Type, Name) -> + fun(Type, Name, ChainName) -> create_authenticator([listeners, Type, Name, authentication], - ListenerID, + ChainName, Config) end); listener_authenticators(get, #{bindings := #{listener_id := ListenerID}}) -> with_listener(ListenerID, - fun(Type, Name) -> + fun(Type, Name, _) -> list_authenticators([listeners, Type, Name, authentication]) end). listener_authenticator(get, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}}) -> with_listener(ListenerID, - fun(Type, Name) -> + fun(Type, Name, _) -> list_authenticator([listeners, Type, Name, authentication], AuthenticatorID) end); listener_authenticator(put, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := Config}) -> with_listener(ListenerID, - fun(Type, Name) -> + fun(Type, Name, ChainName) -> update_authenticator([listeners, Type, Name, authentication], - ListenerID, + ChainName, AuthenticatorID, Config) end); listener_authenticator(delete, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}}) -> with_listener(ListenerID, - fun(Type, Name) -> + fun(Type, Name, ChainName) -> delete_authenticator([listeners, Type, Name, authentication], - ListenerID, + ChainName, AuthenticatorID) end). @@ -539,9 +539,9 @@ authenticator_move(post, #{bindings := #{id := _}, body := _}) -> listener_authenticator_move(post, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := #{<<"position">> := Position}}) -> with_listener(ListenerID, - fun(Type, Name) -> + fun(Type, Name, ChainName) -> move_authenitcator([listeners, Type, Name, authentication], - ListenerID, + ChainName, AuthenticatorID, Position) end); @@ -557,11 +557,13 @@ authenticator_import_users(post, #{bindings := #{id := _}, body := _}) -> serialize_error({missing_parameter, filename}). listener_authenticator_import_users(post, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := #{<<"filename">> := Filename}}) -> - ChainName = to_atom(ListenerID), - case ?AUTHN:import_users(ChainName, AuthenticatorID, Filename) of - ok -> {204}; - {error, Reason} -> serialize_error(Reason) - end; + with_chain(ListenerID, + fun(ChainName) -> + case ?AUTHN:import_users(ChainName, AuthenticatorID, Filename) of + ok -> {204}; + {error, Reason} -> serialize_error(Reason) + end + end); listener_authenticator_import_users(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> serialize_error({missing_parameter, filename}). @@ -580,23 +582,38 @@ authenticator_user(delete, #{bindings := #{id := AuthenticatorID, user_id := Use listener_authenticator_users(post, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := UserInfo}) -> - add_user(ListenerID, AuthenticatorID, UserInfo); + with_chain(ListenerID, + fun(ChainName) -> + add_user(ChainName, AuthenticatorID, UserInfo) + end); listener_authenticator_users(get, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, query_string := PageParams}) -> - list_users(ListenerID, AuthenticatorID, PageParams). + with_chain(ListenerID, + fun(ChainName) -> + list_users(ChainName, AuthenticatorID, PageParams) + end). listener_authenticator_user(put, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID, user_id := UserID}, body := UserInfo}) -> - update_user(ListenerID, AuthenticatorID, UserID, UserInfo); + with_chain(ListenerID, + fun(ChainName) -> + update_user(ChainName, AuthenticatorID, UserID, UserInfo) + end); listener_authenticator_user(get, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID, user_id := UserID}}) -> - find_user(ListenerID, AuthenticatorID, UserID); + with_chain(ListenerID, + fun(ChainName) -> + find_user(ChainName, AuthenticatorID, UserID) + end); listener_authenticator_user(delete, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID, user_id := UserID}}) -> - delete_user(ListenerID, AuthenticatorID, UserID). + with_chain(ListenerID, + fun(ChainName) -> + delete_user(ChainName, AuthenticatorID, UserID) + end). %%------------------------------------------------------------------------------ %% Internal functions @@ -604,27 +621,41 @@ listener_authenticator_user(delete, #{bindings := #{listener_id := ListenerID, with_listener(ListenerID, Fun) -> case find_listener(ListenerID) of - {ok, {Type, Name}} -> - Fun(Type, Name); + {ok, {BType, BName}} -> + Type = binary_to_existing_atom(BType), + Name = binary_to_existing_atom(BName), + ChainName = binary_to_atom(ListenerID), + Fun(Type, Name, ChainName); {error, Reason} -> serialize_error(Reason) end. find_listener(ListenerID) -> - case emqx_listeners:parse_listener_id(ListenerID) of - {error, _} -> - {error, {not_found, {listener, ListenerID}}}; - {Type, Name} -> - case emqx_config:find([listeners, Type, Name]) of - {not_found, _, _} -> - {error, {not_found, {listener, ListenerID}}}; + case binary:split(ListenerID, <<":">>) of + [BType, BName] -> + case emqx_config:find([listeners, BType, BName]) of {ok, _} -> - {ok, {Type, Name}} - end + {ok, {BType, BName}}; + {not_found, _, _} -> + {error, {not_found, {listener, ListenerID}}} + end; + _ -> + {error, {not_found, {listener, ListenerID}}} + end. + +with_chain(ListenerID, Fun) -> + {ok, ChainNames} = ?AUTHN:list_chain_names(), + ListenerChainName = + [ Name || Name <- ChainNames, atom_to_binary(Name) =:= ListenerID ], + case ListenerChainName of + [ChainName] -> + Fun(ChainName); + _ -> + serialize_error({not_found, {chain, ListenerID}}) end. create_authenticator(ConfKeyPath, ChainName, Config) -> - case update_config(ConfKeyPath, {create_authenticator, to_atom(ChainName), Config}) of + case update_config(ConfKeyPath, {create_authenticator, ChainName, Config}) of {ok, #{post_config_update := #{?AUTHN := #{id := ID}}, raw_config := AuthenticatorsConfig}} -> {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig), @@ -649,7 +680,7 @@ list_authenticator(ConfKeyPath, AuthenticatorID) -> end. update_authenticator(ConfKeyPath, ChainName, AuthenticatorID, Config) -> - case update_config(ConfKeyPath, {update_authenticator, to_atom(ChainName), AuthenticatorID, Config}) of + case update_config(ConfKeyPath, {update_authenticator, ChainName, AuthenticatorID, Config}) of {ok, #{post_config_update := #{?AUTHN := #{id := ID}}, raw_config := AuthenticatorsConfig}} -> {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig), @@ -658,8 +689,7 @@ update_authenticator(ConfKeyPath, ChainName, AuthenticatorID, Config) -> serialize_error(Reason) end. -delete_authenticator(ConfKeyPath, ChainName0, AuthenticatorID) -> - ChainName = to_atom(ChainName0), +delete_authenticator(ConfKeyPath, ChainName, AuthenticatorID) -> case update_config(ConfKeyPath, {delete_authenticator, ChainName, AuthenticatorID}) of {ok, _} -> {204}; @@ -667,8 +697,7 @@ delete_authenticator(ConfKeyPath, ChainName0, AuthenticatorID) -> serialize_error(Reason) end. -move_authenitcator(ConfKeyPath, ChainName0, AuthenticatorID, Position) -> - ChainName = to_atom(ChainName0), +move_authenitcator(ConfKeyPath, ChainName, AuthenticatorID, Position) -> case parse_position(Position) of {ok, NPosition} -> case update_config(ConfKeyPath, {move_authenticator, ChainName, AuthenticatorID, NPosition}) of @@ -681,8 +710,7 @@ move_authenitcator(ConfKeyPath, ChainName0, AuthenticatorID, Position) -> serialize_error(Reason) end. -add_user(ChainName0, AuthenticatorID, #{<<"user_id">> := UserID, <<"password">> := Password} = UserInfo) -> - ChainName = to_atom(ChainName0), +add_user(ChainName, AuthenticatorID, #{<<"user_id">> := UserID, <<"password">> := Password} = UserInfo) -> IsSuperuser = maps:get(<<"is_superuser">>, UserInfo, false), case ?AUTHN:add_user(ChainName, AuthenticatorID, #{ user_id => UserID , password => Password @@ -697,8 +725,7 @@ add_user(_, _, #{<<"user_id">> := _}) -> add_user(_, _, _) -> serialize_error({missing_parameter, user_id}). -update_user(ChainName0, AuthenticatorID, UserID, UserInfo) -> - ChainName = to_atom(ChainName0), +update_user(ChainName, AuthenticatorID, UserID, UserInfo) -> case maps:with([<<"password">>, <<"is_superuser">>], UserInfo) =:= #{} of true -> serialize_error({missing_parameter, password}); @@ -711,8 +738,7 @@ update_user(ChainName0, AuthenticatorID, UserID, UserInfo) -> end end. -find_user(ChainName0, AuthenticatorID, UserID) -> - ChainName = to_atom(ChainName0), +find_user(ChainName, AuthenticatorID, UserID) -> case ?AUTHN:lookup_user(ChainName, AuthenticatorID, UserID) of {ok, User} -> {200, User}; @@ -720,8 +746,7 @@ find_user(ChainName0, AuthenticatorID, UserID) -> serialize_error({user_error, Reason}) end. -delete_user(ChainName0, AuthenticatorID, UserID) -> - ChainName = to_atom(ChainName0), +delete_user(ChainName, AuthenticatorID, UserID) -> case ?AUTHN:delete_user(ChainName, AuthenticatorID, UserID) of ok -> {204}; @@ -729,8 +754,7 @@ delete_user(ChainName0, AuthenticatorID, UserID) -> serialize_error({user_error, Reason}) end. -list_users(ChainName0, AuthenticatorID, PageParams) -> - ChainName = to_atom(ChainName0), +list_users(ChainName, AuthenticatorID, PageParams) -> case ?AUTHN:list_users(ChainName, AuthenticatorID, PageParams) of {ok, Users} -> {200, Users}; @@ -834,12 +858,6 @@ parse_position(_) -> ensure_list(M) when is_map(M) -> [M]; ensure_list(L) when is_list(L) -> L. -% TODO: fix atom leak! -to_atom(B) when is_binary(B) -> - binary_to_atom(B); -to_atom(A) when is_atom(A) -> - A. - binfmt(Fmt, Args) -> iolist_to_binary(io_lib:format(Fmt, Args)). authenticator_array_example() -> diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index 768942776..149657ec0 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -187,7 +187,7 @@ t_api_spec(_Config) -> Filter0 = filter(Spec0, Path), ?assertMatch( - {ok, #{body := ActualBody}}, + {ok, #{body := #{<<"timeout">> := <<"infinity">>}}}, trans_requestBody(Path, Body, Filter0)), {Spec1, _} = emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}), From b7ed64918519de9229213e19fcf771d5ceaa932a Mon Sep 17 00:00:00 2001 From: Tobias Lindahl Date: Mon, 1 Nov 2021 14:56:10 +0100 Subject: [PATCH 020/179] test(persistent_session): wait in test to avoid race --- apps/emqx/test/emqx_persistent_session_SUITE.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx/test/emqx_persistent_session_SUITE.erl b/apps/emqx/test/emqx_persistent_session_SUITE.erl index fbf1696f0..d5637cad2 100644 --- a/apps/emqx/test/emqx_persistent_session_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_session_SUITE.erl @@ -378,6 +378,8 @@ t_cancel_on_disconnect(Config) -> {ok, _} = emqtt:ConnFun(Client1), ok = emqtt:disconnect(Client1, 0, #{'Session-Expiry-Interval' => 0}), + wait_for_cm_unregister(ClientId), + {ok, Client2} = emqtt:start_link([ {clientid, ClientId}, {proto_ver, v5}, {clean_start, false}, From 8385eff98ebe0a85ff145bfb826fe9e17990e9ee Mon Sep 17 00:00:00 2001 From: Tobias Lindahl Date: Tue, 2 Nov 2021 09:27:50 +0100 Subject: [PATCH 021/179] fix(persistent_sessions): we only need to persist if the session expired --- apps/emqx/src/emqx_channel.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index fc25490c4..7a5edbb48 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1179,14 +1179,13 @@ terminate(_, #channel{conn_state = idle}) -> ok; terminate(normal, Channel) -> run_terminate_hook(normal, Channel); terminate({shutdown, kicked}, Channel) -> - persist_if_session(Channel), run_terminate_hook(kicked, Channel); terminate({shutdown, Reason}, Channel) when Reason =:= discarded; Reason =:= takeovered -> run_terminate_hook(Reason, Channel); terminate(Reason, Channel = #channel{will_msg = WillMsg}) -> (WillMsg =/= undefined) andalso publish_will_msg(WillMsg), - persist_if_session(Channel), + (Reason =:= expired) andalso persist_if_session(Channel), run_terminate_hook(Reason, Channel). persist_if_session(#channel{session = Session} = Channel) -> From 89cd68d36f57d0817cbd8f9f5e43622f23db59fc Mon Sep 17 00:00:00 2001 From: Tobias Lindahl Date: Tue, 2 Nov 2021 09:31:53 +0100 Subject: [PATCH 022/179] refactor(persistent_sessions): fix coding style --- apps/emqx/src/emqx_cm.erl | 4 +++- apps/emqx/test/emqx_persistent_session_SUITE.erl | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index dbd8dfa63..890dce60a 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -58,6 +58,7 @@ , lookup_channels/2 ]). +%% Test/debug interface -export([ all_channels/0 , all_client_ids/0 ]). @@ -397,11 +398,12 @@ with_channel(ClientId, Fun) -> Pids -> Fun(lists:last(Pids)) end. -%% @doc Get all channels registed. +%% @doc Get all registed channel pids. Debugg/test interface all_channels() -> Pat = [{{'_', '$1'}, [], ['$1']}], ets:select(?CHAN_TAB, Pat). +%% @doc Get all registed clientIDs. Debugg/test interface all_client_ids() -> Pat = [{{'$1', '_'}, [], ['$1']}], ets:select(?CHAN_TAB, Pat). diff --git a/apps/emqx/test/emqx_persistent_session_SUITE.erl b/apps/emqx/test/emqx_persistent_session_SUITE.erl index d5637cad2..bcc8d5e69 100644 --- a/apps/emqx/test/emqx_persistent_session_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_session_SUITE.erl @@ -113,8 +113,9 @@ init_per_group(snabbkaffe, Config) -> [ {kill_connection_process, true} | Config]; init_per_group(gc_tests, Config) -> %% We need to make sure the system does not interfere with this test group. - [maybe_kill_connection_process(ClientId, [{kill_connection_process, true}]) - || ClientId <- emqx_cm:all_client_ids()], + lists:foreach(fun(ClientId) -> + maybe_kill_connection_process(ClientId, [{kill_connection_process, true}]) + end, emqx_cm:all_client_ids()), emqx_common_test_helpers:stop_apps([]), SessionMsgEts = gc_tests_session_store, MsgEts = gc_tests_msg_store, From a139a0d45345dc02259952ee0fc56496944d607b Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 3 Nov 2021 11:15:35 +0100 Subject: [PATCH 023/179] fix(config): pin hocon 0.20.6 fix translation error --- apps/emqx/rebar.config | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 61d5717d9..c27481a66 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -17,7 +17,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.20.5"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.20.6"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} diff --git a/rebar.config b/rebar.config index 410b0701b..87b45b695 100644 --- a/rebar.config +++ b/rebar.config @@ -63,7 +63,7 @@ , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.20.5"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.20.6"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}} From ca4bb100ec8b9a99b3595b939f7d45c8e258121c Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 4 Nov 2021 10:01:54 +0800 Subject: [PATCH 024/179] fix(authn): fix bad parsing for postgresql SQL --- .../src/simple_authn/emqx_authn_pgsql.erl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 57a8416df..fa7765542 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -36,6 +36,11 @@ , destroy/1 ]). +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-endif. + %%------------------------------------------------------------------------------ %% Hocon Schema %%------------------------------------------------------------------------------ @@ -48,7 +53,7 @@ fields(config) -> [ {mechanism, {enum, ['password-based']}} , {backend, {enum, [postgresql]}} , {password_hash_algorithm, fun password_hash_algorithm/1} - , {salt_position, {enum, [prefix, suffix]}} + , {salt_position, fun salt_position/1} , {query, fun query/1} ] ++ emqx_authn_schema:common_fields() ++ emqx_connector_schema_lib:relational_db_fields() @@ -58,6 +63,10 @@ password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt password_hash_algorithm(default) -> sha256; password_hash_algorithm(_) -> undefined. +salt_position(type) -> {enum, [prefix, suffix]}; +salt_position(default) -> prefix; +salt_position(_) -> undefined. + query(type) -> string(); query(_) -> undefined. @@ -134,10 +143,10 @@ destroy(#{'_unique' := Unique}) -> parse_query(Query) -> case re:run(Query, ?RE_PLACEHOLDER, [global, {capture, all, binary}]) of {match, Captured} -> - PlaceHolders = ["\\" ++ PlaceHolder || [PlaceHolder] <- Captured], + PlaceHolders = [PlaceHolder || [PlaceHolder] <- Captured], Replacements = ["$" ++ integer_to_list(I) || I <- lists:seq(1, length(Captured))], NQuery = lists:foldl(fun({PlaceHolder, Replacement}, Query0) -> - re:replace(Query0, PlaceHolder, Replacement, [{return, binary}]) + re:replace(Query0, "\\" ++ PlaceHolder, Replacement, [{return, binary}]) end, Query, lists:zip(PlaceHolders, Replacements)), {NQuery, PlaceHolders}; nomatch -> From 48ddd056b5270b8134b3c5f74e7acb91c97d556b Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 4 Nov 2021 10:03:34 +0800 Subject: [PATCH 025/179] test(authn): add test cases for authn --- .../test/emqx_authn_mysql_SUITE.erl | 90 ++++++++++++++++ .../test/emqx_authn_pgsql_SUITE.erl | 101 ++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl create mode 100644 apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl diff --git a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl new file mode 100644 index 000000000..844f45762 --- /dev/null +++ b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl @@ -0,0 +1,90 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_mysql_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include("emqx_authn.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +groups() -> + []. + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_authn]), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([emqx_authn]), + ok. + +init_per_testcase(t_authn, Config) -> + meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_resource, create_local, fun(_, _, _) -> {ok, undefined} end), + Config; +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(t_authn, _Config) -> + meck:unload(emqx_resource), + ok; +end_per_testcase(_, _Config) -> + ok. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_authn(_) -> + Password = <<"test">>, + Salt = <<"salt">>, + PasswordHash = emqx_authn_utils:hash(sha256, Password, Salt, prefix), + + Config = #{<<"mechanism">> => <<"password-based">>, + <<"backend">> => <<"mysql">>, + <<"server">> => <<"127.0.0.1:3306">>, + <<"database">> => <<"mqtt">>, + <<"query">> => <<"SELECT password_hash, salt FROM users where username = ${mqtt-username} LIMIT 1">> + }, + {ok, _} = update_config([authentication], {create_authenticator, ?GLOBAL, Config}), + + meck:expect(emqx_resource, query, + fun(_, {sql, _, [<<"good">>], _}) -> + {ok, [<<"password_hash">>, <<"salt">>], [[PasswordHash, Salt]]}; + (_, {sql, _, _, _}) -> + {error, this_is_a_fictitious_reason} + end), + + ClientInfo = #{zone => default, + listener => 'tcp:default', + protocol => mqtt, + username => <<"good">>, + password => Password}, + ?assertEqual({ok, #{is_superuser => false}}, emqx_access_control:authenticate(ClientInfo)), + + ClientInfo2 = ClientInfo#{username => <<"bad">>}, + ?assertEqual({error, not_authorized}, emqx_access_control:authenticate(ClientInfo2)), + + ?AUTHN:delete_chain(?GLOBAL). + +update_config(Path, ConfigRequest) -> + emqx:update_config(Path, ConfigRequest, #{rawconf_with_defaults => true}). + diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl new file mode 100644 index 000000000..574a11c74 --- /dev/null +++ b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl @@ -0,0 +1,101 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_pgsql_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include("emqx_authn.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("epgsql/include/epgsql.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +groups() -> + []. + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_authn]), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([emqx_authn]), + ok. + +init_per_testcase(t_authn, Config) -> + meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_resource, create_local, fun(_, _, _) -> {ok, undefined} end), + Config; +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(t_authn, _Config) -> + meck:unload(emqx_resource), + ok; +end_per_testcase(_, _Config) -> + ok. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_parse_query(_) -> + Query1 = <<"${mqtt-username}">>, + ?assertEqual({<<"$1">>, [<<"${mqtt-username}">>]}, emqx_authn_pgsql:parse_query(Query1)), + + Query2 = <<"${mqtt-username}, ${mqtt-clientid}">>, + ?assertEqual({<<"$1, $2">>, [<<"${mqtt-username}">>, <<"${mqtt-clientid}">>]}, emqx_authn_pgsql:parse_query(Query2)), + + Query3 = <<"nomatch">>, + ?assertEqual({<<"nomatch">>, []}, emqx_authn_pgsql:parse_query(Query3)). + +t_authn(_) -> + Password = <<"test">>, + Salt = <<"salt">>, + PasswordHash = emqx_authn_utils:hash(sha256, Password, Salt, prefix), + + Config = #{<<"mechanism">> => <<"password-based">>, + <<"backend">> => <<"postgresql">>, + <<"server">> => <<"127.0.0.1:5432">>, + <<"database">> => <<"mqtt">>, + <<"query">> => <<"SELECT password_hash, salt FROM users where username = ${mqtt-username} LIMIT 1">> + }, + {ok, _} = update_config([authentication], {create_authenticator, ?GLOBAL, Config}), + + meck:expect(emqx_resource, query, + fun(_, {sql, _, [<<"good">>]}) -> + {ok, [#column{name = <<"password_hash">>}, #column{name = <<"salt">>}], [{PasswordHash, Salt}]}; + (_, {sql, _, _}) -> + {error, this_is_a_fictitious_reason} + end), + + ClientInfo = #{zone => default, + listener => 'tcp:default', + protocol => mqtt, + username => <<"good">>, + password => Password}, + ?assertEqual({ok, #{is_superuser => false}}, emqx_access_control:authenticate(ClientInfo)), + + ClientInfo2 = ClientInfo#{username => <<"bad">>}, + ?assertEqual({error, not_authorized}, emqx_access_control:authenticate(ClientInfo2)), + + ?AUTHN:delete_chain(?GLOBAL). + +update_config(Path, ConfigRequest) -> + emqx:update_config(Path, ConfigRequest, #{rawconf_with_defaults => true}). + From 6f3cfbc10208407c3927a5e3b691e68b168dd604 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 4 Nov 2021 14:39:27 +0100 Subject: [PATCH 026/179] chore: add a script to find files without new line at EOF --- scripts/check-nl-at-eof.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100755 scripts/check-nl-at-eof.sh diff --git a/scripts/check-nl-at-eof.sh b/scripts/check-nl-at-eof.sh new file mode 100755 index 000000000..5d93d04ac --- /dev/null +++ b/scripts/check-nl-at-eof.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -euo pipefail + +cd -P -- "$(dirname -- "$0")/.." + +nl_at_eof() { + local file="$1" + if ! [ -f $file ]; then + return + fi + case "$file" in + *.png|*rebar3) + return + ;; + esac + local lastbyte + lastbyte="$(tail -c 1 "$file" 2>&1)" + if [ "$lastbyte" != '' ]; then + echo $file + fi +} + +while read -r file; do + nl_at_eof "$file" +done < <(git ls-files) From 56e2a9741f3413a3c0606c0574ad2094291b8a8d Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 4 Nov 2021 14:40:14 +0100 Subject: [PATCH 027/179] style: ensure newline at EOF for all files --- CONTRIBUTING.md | 2 +- apps/emqx/test/emqx_broker_helper_SUITE.erl | 6 +++--- apps/emqx/test/emqx_pqueue_SUITE.erl | 2 +- apps/emqx/test/emqx_sup_SUITE.erl | 2 +- apps/emqx_authn/test/emqx_authn_SUITE.erl | 2 +- apps/emqx_connector/src/emqx_connector_redis.erl | 2 +- apps/emqx_limiter/etc/emqx_limiter.conf | 2 +- apps/emqx_prometheus/grafana_template/EMQ.json | 2 +- apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json | 2 +- apps/emqx_prometheus/grafana_template/ErlangVM.json | 2 +- apps/emqx_resource/examples/log_tracer.conf | 2 +- apps/emqx_resource/src/emqx_resource_sup.erl | 2 +- apps/emqx_resource/src/emqx_resource_uitils.erl | 2 +- apps/emqx_retainer/.gitignore | 2 +- apps/emqx_rule_engine/src/emqx_rule_maps.erl | 2 +- deploy/charts/emqx/templates/ingress.yaml | 2 +- deploy/charts/emqx/templates/rbac.yaml | 6 +++--- 17 files changed, 21 insertions(+), 21 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5e73359e8..118e9a046 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,4 +79,4 @@ Just as in the **subject**, use the imperative, present tense: "change" not "cha The footer should contain any information about **Breaking Changes** and is also the place to reference GitHub issues that this commit **Closes**. -**Breaking Changes** should start with the word `BREAKING CHANGE:` with a space or two newlines. The rest of the commit message is then used for this. \ No newline at end of file +**Breaking Changes** should start with the word `BREAKING CHANGE:` with a space or two newlines. The rest of the commit message is then used for this. diff --git a/apps/emqx/test/emqx_broker_helper_SUITE.erl b/apps/emqx/test/emqx_broker_helper_SUITE.erl index 59b3847b0..29aca6e3b 100644 --- a/apps/emqx/test/emqx_broker_helper_SUITE.erl +++ b/apps/emqx/test/emqx_broker_helper_SUITE.erl @@ -41,7 +41,7 @@ t_lookup_subpid(_) -> emqx_broker_helper:register_sub(self(), <<"clientid">>), ct:sleep(10), ?assertEqual(self(), emqx_broker_helper:lookup_subpid(<<"clientid">>)). - + t_register_sub(_) -> ok = emqx_broker_helper:register_sub(self(), <<"clientid">>), ct:sleep(10), @@ -62,7 +62,7 @@ t_shard_seq(_) -> t_shards_num(_) -> ?assertEqual(emqx_vm:schedulers() * 32, emqx_broker_helper:shards_num()). - + t_get_sub_shard(_) -> ?assertEqual(0, emqx_broker_helper:get_sub_shard(self(), <<"topic">>)). @@ -72,4 +72,4 @@ t_terminate(_) -> t_uncovered_func(_) -> gen_server:call(emqx_broker_helper, test), gen_server:cast(emqx_broker_helper, test), - emqx_broker_helper ! test. \ No newline at end of file + emqx_broker_helper ! test. diff --git a/apps/emqx/test/emqx_pqueue_SUITE.erl b/apps/emqx/test/emqx_pqueue_SUITE.erl index a51c21473..76075381d 100644 --- a/apps/emqx/test/emqx_pqueue_SUITE.erl +++ b/apps/emqx/test/emqx_pqueue_SUITE.erl @@ -174,4 +174,4 @@ t_filter(_) -> t_highest(_) -> empty = ?PQ:highest(?PQ:new()), 0 = ?PQ:highest(?PQ:from_list([{0, a}, {0, b}])), - 2 = ?PQ:highest(?PQ:from_list([{0, a}, {0, b}, {1, c}, {2, d}, {2, e}])). \ No newline at end of file + 2 = ?PQ:highest(?PQ:from_list([{0, a}, {0, b}, {1, c}, {2, d}, {2, e}])). diff --git a/apps/emqx/test/emqx_sup_SUITE.erl b/apps/emqx/test/emqx_sup_SUITE.erl index 185e1d752..2f5600e38 100644 --- a/apps/emqx/test/emqx_sup_SUITE.erl +++ b/apps/emqx/test/emqx_sup_SUITE.erl @@ -36,4 +36,4 @@ t_child(_) -> ?assertMatch({error, not_found}, emqx_sup:stop_child(undef)), ?assertMatch({error, _}, emqx_sup:start_child(emqx_broker_sup, supervisor)), ?assertEqual(ok, emqx_sup:stop_child(emqx_broker_sup)), - ?assertMatch({ok, _}, emqx_sup:start_child(emqx_broker_sup, supervisor)). \ No newline at end of file + ?assertMatch({ok, _}, emqx_sup:start_child(emqx_broker_sup, supervisor)). diff --git a/apps/emqx_authn/test/emqx_authn_SUITE.erl b/apps/emqx_authn/test/emqx_authn_SUITE.erl index 1ee419bb0..d3704679f 100644 --- a/apps/emqx_authn/test/emqx_authn_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_SUITE.erl @@ -19,4 +19,4 @@ -compile(export_all). -compile(nowarn_export_all). -all() -> emqx_common_test_helpers:all(?MODULE). \ No newline at end of file +all() -> emqx_common_test_helpers:all(?MODULE). diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index aed06e724..ff6e4d82d 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -182,4 +182,4 @@ to_server(Server) -> case string:tokens(Server, ":") of [Host, Port] -> {ok, {Host, list_to_integer(Port)}}; _ -> {error, Server} - end. \ No newline at end of file + end. diff --git a/apps/emqx_limiter/etc/emqx_limiter.conf b/apps/emqx_limiter/etc/emqx_limiter.conf index 44bbb1740..7298931e3 100644 --- a/apps/emqx_limiter/etc/emqx_limiter.conf +++ b/apps/emqx_limiter/etc/emqx_limiter.conf @@ -47,4 +47,4 @@ emqx_limiter { per_client = "100/10s,10" } } -} \ No newline at end of file +} diff --git a/apps/emqx_prometheus/grafana_template/EMQ.json b/apps/emqx_prometheus/grafana_template/EMQ.json index 137e3a5a4..54d0a9d47 100644 --- a/apps/emqx_prometheus/grafana_template/EMQ.json +++ b/apps/emqx_prometheus/grafana_template/EMQ.json @@ -2099,4 +2099,4 @@ "title": "EMQ", "uid": "tjRlQw6Zk", "version": 29 -} \ No newline at end of file +} diff --git a/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json b/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json index 0b0e2036b..0ee3c6741 100644 --- a/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json +++ b/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json @@ -630,4 +630,4 @@ "title": "EMQ Dashboard", "uid": "5sreUw6Wz", "version": 11 -} \ No newline at end of file +} diff --git a/apps/emqx_prometheus/grafana_template/ErlangVM.json b/apps/emqx_prometheus/grafana_template/ErlangVM.json index 556d815b0..23088123c 100644 --- a/apps/emqx_prometheus/grafana_template/ErlangVM.json +++ b/apps/emqx_prometheus/grafana_template/ErlangVM.json @@ -1471,4 +1471,4 @@ "title": "ErlangVM", "uid": "stprQQ6Zk", "version": 13 -} \ No newline at end of file +} diff --git a/apps/emqx_resource/examples/log_tracer.conf b/apps/emqx_resource/examples/log_tracer.conf index 7b438ec1f..49a57a13f 100644 --- a/apps/emqx_resource/examples/log_tracer.conf +++ b/apps/emqx_resource/examples/log_tracer.conf @@ -8,4 +8,4 @@ "bulk": "10KB" "chars_limit": 1024 } -} \ No newline at end of file +} diff --git a/apps/emqx_resource/src/emqx_resource_sup.erl b/apps/emqx_resource/src/emqx_resource_sup.erl index 22984b940..69d1acd20 100644 --- a/apps/emqx_resource/src/emqx_resource_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_sup.erl @@ -55,4 +55,4 @@ ensure_pool_worker(Pool, Name, Slot) -> try gproc_pool:add_worker(Pool, Name, Slot) catch error:exists -> ok - end. \ No newline at end of file + end. diff --git a/apps/emqx_resource/src/emqx_resource_uitils.erl b/apps/emqx_resource/src/emqx_resource_uitils.erl index ab3f6dd1e..9f5f11a01 100644 --- a/apps/emqx_resource/src/emqx_resource_uitils.erl +++ b/apps/emqx_resource/src/emqx_resource_uitils.erl @@ -13,4 +13,4 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_resource_uitils). \ No newline at end of file +-module(emqx_resource_uitils). diff --git a/apps/emqx_retainer/.gitignore b/apps/emqx_retainer/.gitignore index d51b4e87f..a0c149280 100644 --- a/apps/emqx_retainer/.gitignore +++ b/apps/emqx_retainer/.gitignore @@ -23,4 +23,4 @@ logs/ rebar.lock test/ct.cover.spec etc/emqx_retainer.conf.rendered -.rebar3/ \ No newline at end of file +.rebar3/ diff --git a/apps/emqx_rule_engine/src/emqx_rule_maps.erl b/apps/emqx_rule_engine/src/emqx_rule_maps.erl index 4bb104f7f..c3f8fd3cf 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_maps.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_maps.erl @@ -206,4 +206,4 @@ unsafe_atom_key_map(BinKeyMap) when is_map(BinKeyMap) -> end, #{}, BinKeyMap); unsafe_atom_key_map(ListV) when is_list(ListV) -> [unsafe_atom_key_map(V) || V <- ListV]; -unsafe_atom_key_map(Val) -> Val. \ No newline at end of file +unsafe_atom_key_map(Val) -> Val. diff --git a/deploy/charts/emqx/templates/ingress.yaml b/deploy/charts/emqx/templates/ingress.yaml index 7fc66a86a..926023f61 100644 --- a/deploy/charts/emqx/templates/ingress.yaml +++ b/deploy/charts/emqx/templates/ingress.yaml @@ -95,4 +95,4 @@ spec: {{- toYaml .Values.ingress.mgmt.tls | nindent 4 }} {{- end }} --- -{{- end }} \ No newline at end of file +{{- end }} diff --git a/deploy/charts/emqx/templates/rbac.yaml b/deploy/charts/emqx/templates/rbac.yaml index 87cd18178..d2a0f35f4 100644 --- a/deploy/charts/emqx/templates/rbac.yaml +++ b/deploy/charts/emqx/templates/rbac.yaml @@ -17,8 +17,8 @@ rules: - apiGroups: - "" resources: - - endpoints - verbs: + - endpoints + verbs: - get - watch - list @@ -39,4 +39,4 @@ subjects: roleRef: kind: Role name: {{ include "emqx.fullname" . }} - apiGroup: rbac.authorization.k8s.io \ No newline at end of file + apiGroup: rbac.authorization.k8s.io From dc58e4a4418940ebd3c0217a84304ac9dc2a264b Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 4 Nov 2021 14:47:05 +0100 Subject: [PATCH 028/179] ci: check code style run elvis and nl-at-eof check --- .../{elvis_lint.yaml => code-style-check.yaml} | 8 ++++++-- scripts/check-nl-at-eof.sh | 12 +++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) rename .github/workflows/{elvis_lint.yaml => code-style-check.yaml} (74%) diff --git a/.github/workflows/elvis_lint.yaml b/.github/workflows/code-style-check.yaml similarity index 74% rename from .github/workflows/elvis_lint.yaml rename to .github/workflows/code-style-check.yaml index 1fdbeba87..93e363245 100644 --- a/.github/workflows/elvis_lint.yaml +++ b/.github/workflows/code-style-check.yaml @@ -1,4 +1,4 @@ -name: Elvis Linter +name: Code style check on: [pull_request] @@ -12,5 +12,9 @@ jobs: run: | echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials git config --global credential.helper store - - run: | + - name: Run elvis check + run: | ./scripts/elvis-check.sh $GITHUB_BASE_REF + - name: Check line-break at EOF + - run: | + ./scripts/check-nl-at-eof.sh diff --git a/scripts/check-nl-at-eof.sh b/scripts/check-nl-at-eof.sh index 5d93d04ac..f4f1ef04f 100755 --- a/scripts/check-nl-at-eof.sh +++ b/scripts/check-nl-at-eof.sh @@ -6,7 +6,7 @@ cd -P -- "$(dirname -- "$0")/.." nl_at_eof() { local file="$1" - if ! [ -f $file ]; then + if ! [ -f "$file" ]; then return fi case "$file" in @@ -17,10 +17,16 @@ nl_at_eof() { local lastbyte lastbyte="$(tail -c 1 "$file" 2>&1)" if [ "$lastbyte" != '' ]; then - echo $file + echo "$file" + return 1 fi } +n=0 while read -r file; do - nl_at_eof "$file" + if ! nl_at_eof "$file"; then + n=$(( n + 1 )) + fi done < <(git ls-files) + +exit $n From d1abb3081890169e391e78f3bcba0499964fdbcc Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 4 Nov 2021 15:22:33 +0100 Subject: [PATCH 029/179] ci: fix elvis check --- .github/workflows/code-style-check.yaml | 9 ++++++++- apps/emqx/test/emqx_pqueue_SUITE.erl | 3 ++- apps/emqx_connector/src/emqx_connector_redis.erl | 3 ++- apps/emqx_rule_engine/src/emqx_rule_maps.erl | 16 ++++++++-------- scripts/elvis-check.sh | 12 ++++-------- 5 files changed, 24 insertions(+), 19 deletions(-) diff --git a/.github/workflows/code-style-check.yaml b/.github/workflows/code-style-check.yaml index 93e363245..581dc4316 100644 --- a/.github/workflows/code-style-check.yaml +++ b/.github/workflows/code-style-check.yaml @@ -7,6 +7,8 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 + with: + fetch-depth: 1000 - name: Set git token if: endsWith(github.repository, 'enterprise') run: | @@ -14,7 +16,12 @@ jobs: git config --global credential.helper store - name: Run elvis check run: | - ./scripts/elvis-check.sh $GITHUB_BASE_REF + set -e + if [ -f EMQX_ENTERPRISE ]; then + ./scripts/elvis-check.sh $GITHUB_BASE_REF emqx-enterprise + else + ./scripts/elvis-check.sh $GITHUB_BASE_REF emqx + fi - name: Check line-break at EOF - run: | ./scripts/check-nl-at-eof.sh diff --git a/apps/emqx/test/emqx_pqueue_SUITE.erl b/apps/emqx/test/emqx_pqueue_SUITE.erl index 76075381d..797adb9ab 100644 --- a/apps/emqx/test/emqx_pqueue_SUITE.erl +++ b/apps/emqx/test/emqx_pqueue_SUITE.erl @@ -112,7 +112,8 @@ t_out(_) -> t_out_2(_) -> {empty, {pqueue, [{-1, {queue, [a], [], 1}}]}} = ?PQ:out(0, ?PQ:from_list([{1, a}])), {{value, a}, {queue, [], [], 0}} = ?PQ:out(1, ?PQ:from_list([{1, a}])), - {{value, a}, {pqueue, [{-1, {queue, [], [b], 1}}]}} = ?PQ:out(1, ?PQ:from_list([{1, a}, {1, b}])), + {{value, a}, {pqueue, [{-1, {queue, [], [b], 1}}]}} = + ?PQ:out(1, ?PQ:from_list([{1, a}, {1, b}])), {{value, a}, {queue, [b], [], 1}} = ?PQ:out(1, ?PQ:from_list([{1, a}, {0, b}])). t_out_p(_) -> diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index ff6e4d82d..670add693 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -100,7 +100,8 @@ on_start(InstId, #{redis_type := Type, Options = case maps:get(enable, SSL) of true -> [{ssl, true}, - {ssl_options, emqx_plugin_libs_ssl:save_files_return_opts(SSL, "connectors", InstId)} + {ssl_options, + emqx_plugin_libs_ssl:save_files_return_opts(SSL, "connectors", InstId)} ]; false -> [{ssl, false}] end ++ [{sentinel, maps:get(sentinel, Config, undefined)}], diff --git a/apps/emqx_rule_engine/src/emqx_rule_maps.erl b/apps/emqx_rule_engine/src/emqx_rule_maps.erl index c3f8fd3cf..fe4595c03 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_maps.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_maps.erl @@ -33,7 +33,7 @@ nested_get({var, Key}, Data, Default) -> nested_get({path, Path}, Data, Default) when is_list(Path) -> do_nested_get(Path, Data, Data, Default). -do_nested_get([Key|More], Data, OrgData, Default) -> +do_nested_get([Key | More], Data, OrgData, Default) -> case general_map_get(Key, Data, OrgData, undefined) of undefined -> Default; Val -> do_nested_get(More, Val, OrgData, Default) @@ -51,7 +51,7 @@ nested_put({var, Key}, Val, Map) -> nested_put({path, Path}, Val, Map) when is_list(Path) -> do_nested_put(Path, Val, Map, Map). -do_nested_put([Key|More], Val, Map, OrgData) -> +do_nested_put([Key | More], Val, Map, OrgData) -> SubMap = general_map_get(Key, Map, OrgData, undefined), general_map_put(Key, do_nested_put(More, Val, SubMap, OrgData), Map, OrgData); do_nested_put([], Val, _Map, _OrgData) -> @@ -131,13 +131,13 @@ setnth(tail, List, Val) when is_list(List) -> List ++ [Val]; setnth(tail, _List, Val) -> [Val]; setnth(I, List, _Val) when not is_integer(I) -> List; setnth(0, List, _Val) -> List; -setnth(I, List, _Val) when is_integer(I), I > 0 -> - do_setnth(I, List, _Val); -setnth(I, List, _Val) when is_integer(I), I < 0 -> - lists:reverse(do_setnth(-I, lists:reverse(List), _Val)). +setnth(I, List, Val) when is_integer(I), I > 0 -> + do_setnth(I, List, Val); +setnth(I, List, Val) when is_integer(I), I < 0 -> + lists:reverse(do_setnth(-I, lists:reverse(List), Val)). -do_setnth(1, [_|Rest], Val) -> [Val|Rest]; -do_setnth(I, [E|Rest], Val) -> [E|setnth(I-1, Rest, Val)]; +do_setnth(1, [_ | Rest], Val) -> [Val | Rest]; +do_setnth(I, [E | Rest], Val) -> [E | setnth(I-1, Rest, Val)]; do_setnth(_, [], _Val) -> []. getnth(0, _) -> diff --git a/scripts/elvis-check.sh b/scripts/elvis-check.sh index 5fe482865..ebcea98bc 100755 --- a/scripts/elvis-check.sh +++ b/scripts/elvis-check.sh @@ -5,16 +5,16 @@ set -euo pipefail -ELVIS_VERSION='1.0.0-emqx-2' +elvis_version='1.0.0-emqx-2' base="${1:-}" +repo="${2:-emqx}" +REPO="${GITHUB_REPOSITORY:-${repo}}" if [ "${base}" = "" ]; then echo "Usage $0 " exit 1 fi -elvis_version="${2:-$ELVIS_VERSION}" - echo "elvis -v: $elvis_version" echo "git diff base: $base" @@ -27,11 +27,7 @@ if [[ "$base" =~ [0-9a-f]{8,40} ]]; then # base is a commit sha1 compare_base="$base" else - if [[ $CI == true ]];then - remote="$(git remote -v | grep -E "github\.com(.|/)$GITHUB_REPOSITORY" | grep fetch | awk '{print $1}')" - else - remote="$(git remote -v | grep -E 'github\.com(.|/)emqx' | grep fetch | awk '{print $1}')" - fi + remote="$(git remote -v | grep -E "github\.com(:|/)emqx/$REPO((\.git)|(\s))" | grep fetch | awk '{print $1}')" git fetch "$remote" "$base" compare_base="$remote/$base" fi From 0a5a9bd7d07abe4279038c8915481b5bcd6a50de Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Thu, 4 Nov 2021 16:24:47 +0100 Subject: [PATCH 030/179] fix(emqx_connection): Add backpressure to TCP connections Fixes #5494 --- apps/emqx/src/emqx_connection.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index b01aad468..b5c3bb2ac 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -752,7 +752,7 @@ send(IoData, #state{transport = Transport, socket = Socket, channel = Channel}) ok = emqx_metrics:inc('bytes.sent', Oct), inc_counter(outgoing_bytes, Oct), emqx_congestion:maybe_alarm_conn_congestion(Socket, Transport, Channel), - case Transport:async_send(Socket, IoData, [nosuspend]) of + case Transport:async_send(Socket, IoData, []) of ok -> ok; Error = {error, _Reason} -> %% Send an inet_reply to postpone handling the error From ce8e52f4d02016498d7bb66be061508c57063c05 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 4 Nov 2021 18:12:26 +0100 Subject: [PATCH 031/179] ci: fix job syntax --- .../{code-style-check.yaml => code_style_check.yaml} | 2 +- scripts/elvis-check.sh | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) rename .github/workflows/{code-style-check.yaml => code_style_check.yaml} (98%) diff --git a/.github/workflows/code-style-check.yaml b/.github/workflows/code_style_check.yaml similarity index 98% rename from .github/workflows/code-style-check.yaml rename to .github/workflows/code_style_check.yaml index 581dc4316..f08060ec2 100644 --- a/.github/workflows/code-style-check.yaml +++ b/.github/workflows/code_style_check.yaml @@ -23,5 +23,5 @@ jobs: ./scripts/elvis-check.sh $GITHUB_BASE_REF emqx fi - name: Check line-break at EOF - - run: | + run: | ./scripts/check-nl-at-eof.sh diff --git a/scripts/elvis-check.sh b/scripts/elvis-check.sh index ebcea98bc..204ba3f7a 100755 --- a/scripts/elvis-check.sh +++ b/scripts/elvis-check.sh @@ -5,10 +5,11 @@ set -euo pipefail +set -x elvis_version='1.0.0-emqx-2' base="${1:-}" -repo="${2:-emqx}" +repo="${2:-emqx/emqx}" REPO="${GITHUB_REPOSITORY:-${repo}}" if [ "${base}" = "" ]; then echo "Usage $0 " @@ -27,7 +28,8 @@ if [[ "$base" =~ [0-9a-f]{8,40} ]]; then # base is a commit sha1 compare_base="$base" else - remote="$(git remote -v | grep -E "github\.com(:|/)emqx/$REPO((\.git)|(\s))" | grep fetch | awk '{print $1}')" + git remote -v + remote="$(git remote -v | grep -E "github\.com(:|/)$REPO((\.git)|(\s))" | grep fetch | awk '{print $1}')" git fetch "$remote" "$base" compare_base="$remote/$base" fi From 1e036bf74d6368335837e3a9cfba1d3b9ea9ccd5 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 5 Nov 2021 16:12:14 +0300 Subject: [PATCH 032/179] refactor(authn api): add more schema examples --- apps/emqx_authn/src/emqx_authn_api.erl | 199 ++++++++++++++++-- apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 25 ++- 2 files changed, 199 insertions(+), 25 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 9721dad93..ba6e44834 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -53,7 +53,14 @@ , listener_authenticator_user/2 ]). --export([authenticator_examples/0]). +-export([ authenticator_examples/0 + , request_move_examples/0 + , request_import_users_examples/0 + , request_user_create_examples/0 + , request_user_update_examples/0 + , response_user_examples/0 + , response_users_example/0 + ]). api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). @@ -78,6 +85,7 @@ roots() -> [ request_user_create , request_move , request_import_users , response_user + , response_users ]. fields(request_user_create) -> @@ -103,6 +111,16 @@ fields(response_user) -> [ {user_id, binary()}, {is_superuser, mk(boolean(), #{default => false, nullable => true})} + ]; + +fields(response_users) -> + paginated_list_type(ref(response_user)); + +fields(pagination_meta) -> + [ + {page, non_neg_integer()}, + {limit, non_neg_integer()}, + {count, non_neg_integer()} ]. schema("/authentication") -> @@ -264,7 +282,9 @@ schema("/authentication/:id/move") -> tags => [<<"authentication">>, <<"global">>], description => <<"Move authenticator in global authentication chain">>, parameters => [{id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}], - requestBody => ref(request_move), + requestBody => emqx_dashboard_swagger:schema_with_examples( + ref(request_move), + request_move_examples()), responses => #{ 204 => <<"Authenticator moved">>, 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), @@ -283,7 +303,9 @@ schema("/listeners/:listener_id/authentication/:id/move") -> {listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}, {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})} ], - requestBody => ref(request_move), + requestBody => emqx_dashboard_swagger:schema_with_examples( + ref(request_move), + request_move_examples()), responses => #{ 204 => <<"Authenticator moved">>, 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), @@ -299,7 +321,9 @@ schema("/authentication/:id/import_users") -> tags => [<<"authentication">>, <<"global">>], description => <<"Import users into authenticator in global authentication chain">>, parameters => [{id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}], - requestBody => ref(request_import_users), + requestBody => emqx_dashboard_swagger:schema_with_examples( + ref(request_import_users), + request_import_users_examples()), responses => #{ 204 => <<"Users imported">>, 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), @@ -318,7 +342,9 @@ schema("/listeners/:listener_id/authentication/:id/import_users") -> {listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}, {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})} ], - requestBody => ref(request_import_users), + requestBody => emqx_dashboard_swagger:schema_with_examples( + ref(request_import_users), + request_import_users_examples()), responses => #{ 204 => <<"Users imported">>, 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), @@ -334,9 +360,13 @@ schema("/authentication/:id/users") -> tags => [<<"authentication">>, <<"global">>], description => <<"Create users for authenticator in global authentication chain">>, parameters => [{id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}], - requestBody => ref(request_user_create), + requestBody => emqx_dashboard_swagger:schema_with_examples( + ref(request_user_create), + request_user_create_examples()), responses => #{ - 201 => ref(response_user), + 201 => emqx_dashboard_swagger:schema_with_examples( + ref(response_user), + response_user_examples()), 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), 404 => error_codes([?NOT_FOUND], <<"Not Found">>) } @@ -350,7 +380,9 @@ schema("/authentication/:id/users") -> {limit, mk(integer(), #{in => query, desc => <<"Page Limit">>, nullable => true})} ], responses => #{ - 200 => mk(hoconsc:array(ref(response_user)), #{}), + 200 => emqx_dashboard_swagger:schema_with_example( + ref(response_users), + response_users_example()), 404 => error_codes([?NOT_FOUND], <<"Not Found">>) } @@ -367,9 +399,13 @@ schema("/listeners/:listener_id/authentication/:id/users") -> {listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}, {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})} ], - requestBody => ref(request_user_create), + requestBody => emqx_dashboard_swagger:schema_with_examples( + ref(request_user_create), + request_user_create_examples()), responses => #{ - 201 => ref(response_user), + 201 => emqx_dashboard_swagger:schema_with_examples( + ref(response_user), + response_user_examples()), 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), 404 => error_codes([?NOT_FOUND], <<"Not Found">>) } @@ -384,7 +420,9 @@ schema("/listeners/:listener_id/authentication/:id/users") -> {limit, mk(integer(), #{in => query, desc => <<"Page Limit">>, nullable => true})} ], responses => #{ - 200 => mk(hoconsc:array(ref(response_user)), #{}), + 200 => emqx_dashboard_swagger:schema_with_example( + ref(response_users), + response_users_example()), 404 => error_codes([?NOT_FOUND], <<"Not Found">>) } @@ -402,7 +440,9 @@ schema("/authentication/:id/users/:user_id") -> {user_id, mk(binary(), #{in => path, desc => <<"User ID">>})} ], responses => #{ - 200 => ref(response_user), + 200 => emqx_dashboard_swagger:schema_with_examples( + ref(response_user), + response_user_examples()), 404 => error_codes([?NOT_FOUND], <<"Not Found">>) } }, @@ -413,9 +453,13 @@ schema("/authentication/:id/users/:user_id") -> {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}, {user_id, mk(binary(), #{in => path, desc => <<"User ID">>})} ], - requestBody => ref(request_user_update), + requestBody => emqx_dashboard_swagger:schema_with_examples( + ref(request_user_update), + request_user_update_examples()), responses => #{ - 200 => mk(hoconsc:array(ref(response_user)), #{}), + 200 => emqx_dashboard_swagger:schema_with_example( + ref(response_user), + response_user_examples()), 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), 404 => error_codes([?NOT_FOUND], <<"Not Found">>) } @@ -446,7 +490,9 @@ schema("/listeners/:listener_id/authentication/:id/users/:user_id") -> {user_id, mk(binary(), #{in => path, desc => <<"User ID">>})} ], responses => #{ - 200 => ref(response_user), + 200 => emqx_dashboard_swagger:schema_with_example( + ref(response_user), + response_user_examples()), 404 => error_codes([?NOT_FOUND], <<"Not Found">>) } }, @@ -458,9 +504,13 @@ schema("/listeners/:listener_id/authentication/:id/users/:user_id") -> {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}, {user_id, mk(binary(), #{in => path, desc => <<"User ID">>})} ], - requestBody => ref(request_user_update), + requestBody => emqx_dashboard_swagger:schema_with_example( + ref(request_user_update), + request_user_update_examples()), responses => #{ - 200 => mk(hoconsc:array(ref(response_user)), #{}), + 200 => emqx_dashboard_swagger:schema_with_example( + ref(response_user), + response_user_examples()), 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), 404 => error_codes([?NOT_FOUND], <<"Not Found">>) } @@ -860,6 +910,12 @@ ensure_list(L) when is_list(L) -> L. binfmt(Fmt, Args) -> iolist_to_binary(io_lib:format(Fmt, Args)). +paginated_list_type(Type) -> + [ + {data, hoconsc:array(Type)}, + {meta, ref(pagination_meta)} + ]. + authenticator_array_example() -> [Config || #{value := Config} <- maps:values(authenticator_examples())]. @@ -941,3 +997,112 @@ authenticator_examples() -> } } }. + +request_user_create_examples() -> + #{ + regular_user => #{ + summary => <<"Regular user">>, + value => #{ + user_id => <<"user1">>, + password => <<"secret">> + } + }, + super_user => #{ + summary => <<"Superuser">>, + value => #{ + user_id => <<"user2">>, + password => <<"secret">>, + is_superuser => true + } + } + }. + +request_user_update_examples() -> + #{ + regular_user => #{ + summary => <<"Update regular user">>, + value => #{ + password => <<"newsecret">> + } + }, + super_user => #{ + summary => <<"Update user and promote to superuser">>, + value => #{ + password => <<"newsecret">>, + is_superuser => true + } + } + }. + +request_move_examples() -> + #{ + move_to_top => #{ + summary => <<"Move authenticator to the beginning of the chain">>, + value => #{ + position => <<"top">> + } + }, + move_to_bottom => #{ + summary => <<"Move authenticator to the end of the chain">>, + value => #{ + position => <<"bottom">> + } + }, + 'move_before_password-based:built-in-database' => #{ + summary => <<"Move authenticator to the position preceding some other authenticator">>, + value => #{ + position => <<"before:password-based:built-in-database">> + } + } + }. + +request_import_users_examples() -> + #{ + import_csv => #{ + summary => <<"Import users from CSV file">>, + value => #{ + filename => <<"/path/to/user/data.csv">> + } + }, + import_json => #{ + summary => <<"Import users from JSON file">>, + value => #{ + filename => <<"/path/to/user/data.json">> + } + } + }. + +response_user_examples() -> + #{ + regular_user => #{ + summary => <<"Regular user">>, + value => #{ + user_id => <<"user1">> + } + }, + super_user => #{ + summary => <<"Superuser">>, + value => #{ + user_id => <<"user2">>, + is_superuser => true + } + } + }. + +response_users_example() -> + #{ + data => [ + #{ + user_id => <<"user1">> + }, + #{ + user_id => <<"user2">>, + is_superuser => true + } + ], + meta => #{ + page => 0, + limit => 20, + count => 300 + } + }. diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index b2f9bb106..19b53b25f 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -204,10 +204,13 @@ test_authenticator_users(PathPrefix) -> lists:foreach( fun(User) -> - {ok, 201, _} = request( + {ok, 201, UserData} = request( post, uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]), - User) + User), + CreatedUser = jiffy:decode(UserData, [return_maps]), + ?assertMatch(#{<<"user_id">> := _}, CreatedUser) + end, ValidUsers), @@ -216,14 +219,24 @@ test_authenticator_users(PathPrefix) -> get, uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]) ++ "?page=1&limit=2"), - Page1Users = response_data(Page1Data), + #{<<"data">> := Page1Users, + <<"meta">> := + #{<<"page">> := 1, + <<"limit">> := 2, + <<"count">> := 3}} = + jiffy:decode(Page1Data, [return_maps]), {ok, 200, Page2Data} = request( get, uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]) ++ "?page=2&limit=2"), - Page2Users = response_data(Page2Data), + #{<<"data">> := Page2Users, + <<"meta">> := + #{<<"page">> := 2, + <<"limit">> := 2, + <<"count">> := 3}} = + jiffy:decode(Page2Data, [return_maps]), ?assertEqual(2, length(Page1Users)), ?assertEqual(1, length(Page2Users)), @@ -440,10 +453,6 @@ delete_authenticators(Path, Chain) -> Authenticators) end. -response_data(Response) -> - #{<<"data">> := Data} = jiffy:decode(Response, [return_maps]), - Data. - request(Method, Url) -> request(Method, Url, []). From a84b84aac977addd30e73cb4de6718e91c9be887 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 5 Nov 2021 17:09:56 +0300 Subject: [PATCH 033/179] refactor(authn api): reformat for elvis compliance --- apps/emqx_authn/src/emqx_authn_api.erl | 129 ++++--- apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 349 ++++++++---------- 2 files changed, 228 insertions(+), 250 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index ba6e44834..68d9c0d1f 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -90,9 +90,8 @@ roots() -> [ request_user_create fields(request_user_create) -> [ - {user_id, binary()}, - {password, binary()}, - {is_superuser, mk(boolean(), #{default => false, nullable => true})} + {user_id, binary()} + | fields(request_user_update) ]; fields(request_user_update) -> @@ -125,7 +124,7 @@ fields(pagination_meta) -> schema("/authentication") -> #{ - operationId => authenticators, + 'operationId' => authenticators, get => #{ tags => [<<"authentication">>, <<"global">>], description => <<"List authenticators for global authentication">>, @@ -138,7 +137,7 @@ schema("/authentication") -> post => #{ tags => [<<"authentication">>, <<"global">>], description => <<"Create authenticator for global authentication">>, - requestBody => emqx_dashboard_swagger:schema_with_examples( + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( emqx_authn_schema:authenticator_type(), authenticator_examples()), responses => #{ @@ -153,7 +152,7 @@ schema("/authentication") -> schema("/authentication/:id") -> #{ - operationId => authenticator, + 'operationId' => authenticator, get => #{ tags => [<<"authentication">>, <<"global">>], description => <<"Get authenticator from global authentication chain">>, @@ -169,7 +168,7 @@ schema("/authentication/:id") -> tags => [<<"authentication">>, <<"global">>], description => <<"Update authenticator from global authentication chain">>, parameters => [{id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}], - requestBody => emqx_dashboard_swagger:schema_with_examples( + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( emqx_authn_schema:authenticator_type(), authenticator_examples() ), @@ -195,7 +194,7 @@ schema("/authentication/:id") -> schema("/listeners/:listener_id/authentication") -> #{ - operationId => listener_authenticators, + 'operationId' => listener_authenticators, get => #{ tags => [<<"authentication">>, <<"listener">>], description => <<"List authenticators for listener authentication">>, @@ -210,7 +209,7 @@ schema("/listeners/:listener_id/authentication") -> tags => [<<"authentication">>, <<"listener">>], description => <<"Create authenticator for listener authentication">>, parameters => [{listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}], - requestBody => emqx_dashboard_swagger:schema_with_examples( + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( emqx_authn_schema:authenticator_type(), authenticator_examples() ), @@ -226,7 +225,7 @@ schema("/listeners/:listener_id/authentication") -> schema("/listeners/:listener_id/authentication/:id") -> #{ - operationId => listener_authenticator, + 'operationId' => listener_authenticator, get => #{ tags => [<<"authentication">>, <<"listener">>], description => <<"Get authenticator from listener authentication chain">>, @@ -248,7 +247,7 @@ schema("/listeners/:listener_id/authentication/:id") -> {listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}, {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})} ], - requestBody => emqx_dashboard_swagger:schema_with_examples( + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( emqx_authn_schema:authenticator_type(), authenticator_examples()), responses => #{ @@ -277,12 +276,12 @@ schema("/listeners/:listener_id/authentication/:id") -> schema("/authentication/:id/move") -> #{ - operationId => authenticator_move, + 'operationId' => authenticator_move, post => #{ tags => [<<"authentication">>, <<"global">>], description => <<"Move authenticator in global authentication chain">>, parameters => [{id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}], - requestBody => emqx_dashboard_swagger:schema_with_examples( + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( ref(request_move), request_move_examples()), responses => #{ @@ -295,7 +294,7 @@ schema("/authentication/:id/move") -> schema("/listeners/:listener_id/authentication/:id/move") -> #{ - operationId => listener_authenticator_move, + 'operationId' => listener_authenticator_move, post => #{ tags => [<<"authentication">>, <<"listener">>], description => <<"Move authenticator in listener authentication chain">>, @@ -303,7 +302,7 @@ schema("/listeners/:listener_id/authentication/:id/move") -> {listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}, {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})} ], - requestBody => emqx_dashboard_swagger:schema_with_examples( + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( ref(request_move), request_move_examples()), responses => #{ @@ -316,12 +315,12 @@ schema("/listeners/:listener_id/authentication/:id/move") -> schema("/authentication/:id/import_users") -> #{ - operationId => authenticator_import_users, + 'operationId' => authenticator_import_users, post => #{ tags => [<<"authentication">>, <<"global">>], description => <<"Import users into authenticator in global authentication chain">>, parameters => [{id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}], - requestBody => emqx_dashboard_swagger:schema_with_examples( + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( ref(request_import_users), request_import_users_examples()), responses => #{ @@ -334,7 +333,7 @@ schema("/authentication/:id/import_users") -> schema("/listeners/:listener_id/authentication/:id/import_users") -> #{ - operationId => listener_authenticator_import_users, + 'operationId' => listener_authenticator_import_users, post => #{ tags => [<<"authentication">>, <<"listener">>], description => <<"Import users into authenticator in listener authentication chain">>, @@ -342,7 +341,7 @@ schema("/listeners/:listener_id/authentication/:id/import_users") -> {listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}, {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})} ], - requestBody => emqx_dashboard_swagger:schema_with_examples( + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( ref(request_import_users), request_import_users_examples()), responses => #{ @@ -355,12 +354,12 @@ schema("/listeners/:listener_id/authentication/:id/import_users") -> schema("/authentication/:id/users") -> #{ - operationId => authenticator_users, + 'operationId' => authenticator_users, post => #{ tags => [<<"authentication">>, <<"global">>], description => <<"Create users for authenticator in global authentication chain">>, parameters => [{id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}], - requestBody => emqx_dashboard_swagger:schema_with_examples( + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( ref(request_user_create), request_user_create_examples()), responses => #{ @@ -391,7 +390,7 @@ schema("/authentication/:id/users") -> schema("/listeners/:listener_id/authentication/:id/users") -> #{ - operationId => listener_authenticator_users, + 'operationId' => listener_authenticator_users, post => #{ tags => [<<"authentication">>, <<"listener">>], description => <<"Create users for authenticator in global authentication chain">>, @@ -399,7 +398,7 @@ schema("/listeners/:listener_id/authentication/:id/users") -> {listener_id, mk(binary(), #{in => path, desc => <<"Listener ID">>})}, {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})} ], - requestBody => emqx_dashboard_swagger:schema_with_examples( + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( ref(request_user_create), request_user_create_examples()), responses => #{ @@ -431,7 +430,7 @@ schema("/listeners/:listener_id/authentication/:id/users") -> schema("/authentication/:id/users/:user_id") -> #{ - operationId => authenticator_user, + 'operationId' => authenticator_user, get => #{ tags => [<<"authentication">>, <<"global">>], description => <<"Get user from authenticator in global authentication chain">>, @@ -453,7 +452,7 @@ schema("/authentication/:id/users/:user_id") -> {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}, {user_id, mk(binary(), #{in => path, desc => <<"User ID">>})} ], - requestBody => emqx_dashboard_swagger:schema_with_examples( + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( ref(request_user_update), request_user_update_examples()), responses => #{ @@ -480,7 +479,7 @@ schema("/authentication/:id/users/:user_id") -> schema("/listeners/:listener_id/authentication/:id/users/:user_id") -> #{ - operationId => listener_authenticator_user, + 'operationId' => listener_authenticator_user, get => #{ tags => [<<"authentication">>, <<"listener">>], description => <<"Get user from authenticator in listener authentication chain">>, @@ -504,7 +503,7 @@ schema("/listeners/:listener_id/authentication/:id/users/:user_id") -> {id, mk(binary(), #{in => path, desc => <<"Authenticator ID">>})}, {user_id, mk(binary(), #{in => path, desc => <<"User ID">>})} ], - requestBody => emqx_dashboard_swagger:schema_with_example( + 'requestBody' => emqx_dashboard_swagger:schema_with_example( ref(request_user_update), request_user_update_examples()), responses => #{ @@ -566,7 +565,9 @@ listener_authenticator(get, #{bindings := #{listener_id := ListenerID, id := Aut list_authenticator([listeners, Type, Name, authentication], AuthenticatorID) end); -listener_authenticator(put, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := Config}) -> +listener_authenticator(put, + #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, + body := Config}) -> with_listener(ListenerID, fun(Type, Name, ChainName) -> update_authenticator([listeners, Type, Name, authentication], @@ -574,7 +575,8 @@ listener_authenticator(put, #{bindings := #{listener_id := ListenerID, id := Aut AuthenticatorID, Config) end); -listener_authenticator(delete, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}}) -> +listener_authenticator(delete, + #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}}) -> with_listener(ListenerID, fun(Type, Name, ChainName) -> delete_authenticator([listeners, Type, Name, authentication], @@ -582,12 +584,16 @@ listener_authenticator(delete, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID) end). -authenticator_move(post, #{bindings := #{id := AuthenticatorID}, body := #{<<"position">> := Position}}) -> +authenticator_move(post, + #{bindings := #{id := AuthenticatorID}, + body := #{<<"position">> := Position}}) -> move_authenitcator([authentication], ?GLOBAL, AuthenticatorID, Position); authenticator_move(post, #{bindings := #{id := _}, body := _}) -> serialize_error({missing_parameter, position}). -listener_authenticator_move(post, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := #{<<"position">> := Position}}) -> +listener_authenticator_move(post, + #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, + body := #{<<"position">> := Position}}) -> with_listener(ListenerID, fun(Type, Name, ChainName) -> move_authenitcator([listeners, Type, Name, authentication], @@ -598,22 +604,28 @@ listener_authenticator_move(post, #{bindings := #{listener_id := ListenerID, id listener_authenticator_move(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> serialize_error({missing_parameter, position}). -authenticator_import_users(post, #{bindings := #{id := AuthenticatorID}, body := #{<<"filename">> := Filename}}) -> - case ?AUTHN:import_users(?GLOBAL, AuthenticatorID, Filename) of +authenticator_import_users(post, + #{bindings := #{id := AuthenticatorID}, + body := #{<<"filename">> := Filename}}) -> + case emqx_authentication:import_users(?GLOBAL, AuthenticatorID, Filename) of ok -> {204}; {error, Reason} -> serialize_error(Reason) end; authenticator_import_users(post, #{bindings := #{id := _}, body := _}) -> serialize_error({missing_parameter, filename}). -listener_authenticator_import_users(post, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := #{<<"filename">> := Filename}}) -> - with_chain(ListenerID, - fun(ChainName) -> - case ?AUTHN:import_users(ChainName, AuthenticatorID, Filename) of - ok -> {204}; - {error, Reason} -> serialize_error(Reason) - end - end); +listener_authenticator_import_users( + post, + #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, + body := #{<<"filename">> := Filename}}) -> + with_chain( + ListenerID, + fun(ChainName) -> + case emqx_authentication:import_users(ChainName, AuthenticatorID, Filename) of + ok -> {204}; + {error, Reason} -> serialize_error(Reason) + end + end); listener_authenticator_import_users(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> serialize_error({missing_parameter, filename}). @@ -694,7 +706,7 @@ find_listener(ListenerID) -> end. with_chain(ListenerID, Fun) -> - {ok, ChainNames} = ?AUTHN:list_chain_names(), + {ok, ChainNames} = emqx_authentication:list_chain_names(), ListenerChainName = [ Name || Name <- ChainNames, atom_to_binary(Name) =:= ListenerID ], case ListenerChainName of @@ -706,7 +718,7 @@ with_chain(ListenerID, Fun) -> create_authenticator(ConfKeyPath, ChainName, Config) -> case update_config(ConfKeyPath, {create_authenticator, ChainName, Config}) of - {ok, #{post_config_update := #{?AUTHN := #{id := ID}}, + {ok, #{post_config_update := #{emqx_authentication := #{id := ID}}, raw_config := AuthenticatorsConfig}} -> {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig), {200, maps:put(id, ID, convert_certs(fill_defaults(AuthenticatorConfig)))}; @@ -716,7 +728,10 @@ create_authenticator(ConfKeyPath, ChainName, Config) -> list_authenticators(ConfKeyPath) -> AuthenticatorsConfig = get_raw_config_with_defaults(ConfKeyPath), - NAuthenticators = [maps:put(id, ?AUTHN:authenticator_id(AuthenticatorConfig), convert_certs(AuthenticatorConfig)) + NAuthenticators = [ maps:put( + id, + emqx_authentication:authenticator_id(AuthenticatorConfig), + convert_certs(AuthenticatorConfig)) || AuthenticatorConfig <- AuthenticatorsConfig], {200, NAuthenticators}. @@ -731,7 +746,7 @@ list_authenticator(ConfKeyPath, AuthenticatorID) -> update_authenticator(ConfKeyPath, ChainName, AuthenticatorID, Config) -> case update_config(ConfKeyPath, {update_authenticator, ChainName, AuthenticatorID, Config}) of - {ok, #{post_config_update := #{?AUTHN := #{id := ID}}, + {ok, #{post_config_update := #{emqx_authentication := #{id := ID}}, raw_config := AuthenticatorsConfig}} -> {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig), {200, maps:put(id, ID, convert_certs(fill_defaults(AuthenticatorConfig)))}; @@ -750,7 +765,9 @@ delete_authenticator(ConfKeyPath, ChainName, AuthenticatorID) -> move_authenitcator(ConfKeyPath, ChainName, AuthenticatorID, Position) -> case parse_position(Position) of {ok, NPosition} -> - case update_config(ConfKeyPath, {move_authenticator, ChainName, AuthenticatorID, NPosition}) of + case update_config( + ConfKeyPath, + {move_authenticator, ChainName, AuthenticatorID, NPosition}) of {ok, _} -> {204}; {error, {_, _, Reason}} -> @@ -760,9 +777,11 @@ move_authenitcator(ConfKeyPath, ChainName, AuthenticatorID, Position) -> serialize_error(Reason) end. -add_user(ChainName, AuthenticatorID, #{<<"user_id">> := UserID, <<"password">> := Password} = UserInfo) -> +add_user(ChainName, + AuthenticatorID, + #{<<"user_id">> := UserID, <<"password">> := Password} = UserInfo) -> IsSuperuser = maps:get(<<"is_superuser">>, UserInfo, false), - case ?AUTHN:add_user(ChainName, AuthenticatorID, #{ user_id => UserID + case emqx_authentication:add_user(ChainName, AuthenticatorID, #{ user_id => UserID , password => Password , is_superuser => IsSuperuser}) of {ok, User} -> @@ -780,7 +799,7 @@ update_user(ChainName, AuthenticatorID, UserID, UserInfo) -> true -> serialize_error({missing_parameter, password}); false -> - case ?AUTHN:update_user(ChainName, AuthenticatorID, UserID, UserInfo) of + case emqx_authentication:update_user(ChainName, AuthenticatorID, UserID, UserInfo) of {ok, User} -> {200, User}; {error, Reason} -> @@ -789,7 +808,7 @@ update_user(ChainName, AuthenticatorID, UserID, UserInfo) -> end. find_user(ChainName, AuthenticatorID, UserID) -> - case ?AUTHN:lookup_user(ChainName, AuthenticatorID, UserID) of + case emqx_authentication:lookup_user(ChainName, AuthenticatorID, UserID) of {ok, User} -> {200, User}; {error, Reason} -> @@ -797,7 +816,7 @@ find_user(ChainName, AuthenticatorID, UserID) -> end. delete_user(ChainName, AuthenticatorID, UserID) -> - case ?AUTHN:delete_user(ChainName, AuthenticatorID, UserID) of + case emqx_authentication:delete_user(ChainName, AuthenticatorID, UserID) of ok -> {204}; {error, Reason} -> @@ -805,7 +824,7 @@ delete_user(ChainName, AuthenticatorID, UserID) -> end. list_users(ChainName, AuthenticatorID, PageParams) -> - case ?AUTHN:list_users(ChainName, AuthenticatorID, PageParams) of + case emqx_authentication:list_users(ChainName, AuthenticatorID, PageParams) of {ok, Users} -> {200, Users}; {error, Reason} -> @@ -821,7 +840,11 @@ get_raw_config_with_defaults(ConfKeyPath) -> ensure_list(fill_defaults(RawConfig)). find_config(AuthenticatorID, AuthenticatorsConfig) -> - case [AC || AC <- ensure_list(AuthenticatorsConfig), AuthenticatorID =:= ?AUTHN:authenticator_id(AC)] of + MatchingACs + = [AC + || AC <- ensure_list(AuthenticatorsConfig), + AuthenticatorID =:= emqx_authentication:authenticator_id(AC)], + case MatchingACs of [] -> {error, {not_found, {authenticator, AuthenticatorID}}}; [AuthenticatorConfig] -> {ok, AuthenticatorConfig} end. diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index 19b53b25f..654aa042a 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -49,7 +49,9 @@ init_per_testcase(_, Config) -> Config. init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_authn, emqx_dashboard], fun set_special_configs/1), + ok = emqx_common_test_helpers:start_apps( + [emqx_authn, emqx_dashboard], + fun set_special_configs/1), Config. end_per_suite(_Config) -> @@ -118,322 +120,275 @@ test_authenticators(PathPrefix) -> ValidConfig = emqx_authn_test_lib:http_example(), {ok, 200, _} = request( - post, - uri(PathPrefix ++ ["authentication"]), - ValidConfig), + post, + uri(PathPrefix ++ ["authentication"]), + ValidConfig), InvalidConfig = ValidConfig#{method => <<"delete">>}, {ok, 400, _} = request( - post, - uri(PathPrefix ++ ["authentication"]), - InvalidConfig), + post, + uri(PathPrefix ++ ["authentication"]), + InvalidConfig), ?assertAuthenticatorsMatch( - [#{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}], - PathPrefix ++ ["authentication"]). + [#{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}], + PathPrefix ++ ["authentication"]). test_authenticator(PathPrefix) -> ValidConfig0 = emqx_authn_test_lib:http_example(), {ok, 200, _} = request( - post, - uri(PathPrefix ++ ["authentication"]), - ValidConfig0), + post, + uri(PathPrefix ++ ["authentication"]), + ValidConfig0), {ok, 200, _} = request( - get, - uri(PathPrefix ++ ["authentication", "password-based:http"])), + get, + uri(PathPrefix ++ ["authentication", "password-based:http"])), {ok, 404, _} = request( - get, - uri(PathPrefix ++ ["authentication", "password-based:redis"])), + get, + uri(PathPrefix ++ ["authentication", "password-based:redis"])), {ok, 404, _} = request( - put, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database"]), - emqx_authn_test_lib:built_in_database_example()), + put, + uri(PathPrefix ++ ["authentication", "password-based:built-in-database"]), + emqx_authn_test_lib:built_in_database_example()), InvalidConfig0 = ValidConfig0#{method => <<"delete">>}, {ok, 400, _} = request( - put, - uri(PathPrefix ++ ["authentication", "password-based:http"]), - InvalidConfig0), + put, + uri(PathPrefix ++ ["authentication", "password-based:http"]), + InvalidConfig0), ValidConfig1 = ValidConfig0#{pool_size => 9}, {ok, 200, _} = request( - put, - uri(PathPrefix ++ ["authentication", "password-based:http"]), - ValidConfig1), + put, + uri(PathPrefix ++ ["authentication", "password-based:http"]), + ValidConfig1), {ok, 404, _} = request( - delete, - uri(PathPrefix ++ ["authentication", "password-based:redis"])), + delete, + uri(PathPrefix ++ ["authentication", "password-based:redis"])), {ok, 204, _} = request( - delete, - uri(PathPrefix ++ ["authentication", "password-based:http"])), + delete, + uri(PathPrefix ++ ["authentication", "password-based:http"])), ?assertAuthenticatorsMatch([], PathPrefix ++ ["authentication"]). test_authenticator_users(PathPrefix) -> + UsersUri = uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]), + {ok, 200, _} = request( - post, - uri(PathPrefix ++ ["authentication"]), - emqx_authn_test_lib:built_in_database_example()), + post, + uri(PathPrefix ++ ["authentication"]), + emqx_authn_test_lib:built_in_database_example()), InvalidUsers = [ - #{clientid => <<"u1">>, password => <<"p1">>}, - #{user_id => <<"u2">>}, - #{user_id => <<"u3">>, password => <<"p3">>, foobar => <<"foobar">>}], + #{clientid => <<"u1">>, password => <<"p1">>}, + #{user_id => <<"u2">>}, + #{user_id => <<"u3">>, password => <<"p3">>, foobar => <<"foobar">>}], lists:foreach( - fun(User) -> - {ok, 400, _} = request( - post, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]), - User) - end, - InvalidUsers), + fun(User) -> {ok, 400, _} = request(post, UsersUri, User) end, + InvalidUsers), ValidUsers = [ - #{user_id => <<"u1">>, password => <<"p1">>}, - #{user_id => <<"u2">>, password => <<"p2">>, is_superuser => true}, - #{user_id => <<"u3">>, password => <<"p3">>}], + #{user_id => <<"u1">>, password => <<"p1">>}, + #{user_id => <<"u2">>, password => <<"p2">>, is_superuser => true}, + #{user_id => <<"u3">>, password => <<"p3">>}], lists:foreach( - fun(User) -> - {ok, 201, UserData} = request( - post, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]), - User), - CreatedUser = jiffy:decode(UserData, [return_maps]), - ?assertMatch(#{<<"user_id">> := _}, CreatedUser) + fun(User) -> + {ok, 201, UserData} = request(post, UsersUri, User), + CreatedUser = jiffy:decode(UserData, [return_maps]), + ?assertMatch(#{<<"user_id">> := _}, CreatedUser) + end, + ValidUsers), - end, - ValidUsers), - - {ok, 200, Page1Data} = - request( - get, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]) ++ "?page=1&limit=2"), + {ok, 200, Page1Data} = request(get, UsersUri ++ "?page=1&limit=2"), #{<<"data">> := Page1Users, <<"meta">> := - #{<<"page">> := 1, - <<"limit">> := 2, - <<"count">> := 3}} = - jiffy:decode(Page1Data, [return_maps]), + #{<<"page">> := 1, + <<"limit">> := 2, + <<"count">> := 3}} = + jiffy:decode(Page1Data, [return_maps]), - {ok, 200, Page2Data} = - request( - get, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]) ++ "?page=2&limit=2"), + {ok, 200, Page2Data} = request(get, UsersUri ++ "?page=2&limit=2"), #{<<"data">> := Page2Users, <<"meta">> := - #{<<"page">> := 2, - <<"limit">> := 2, - <<"count">> := 3}} = - jiffy:decode(Page2Data, [return_maps]), + #{<<"page">> := 2, + <<"limit">> := 2, + <<"count">> := 3}} = jiffy:decode(Page2Data, [return_maps]), ?assertEqual(2, length(Page1Users)), ?assertEqual(1, length(Page2Users)), ?assertEqual( - [<<"u1">>, <<"u2">>, <<"u3">>], - lists:usort([ UserId || #{<<"user_id">> := UserId} <- Page1Users ++ Page2Users])). + [<<"u1">>, <<"u2">>, <<"u3">>], + lists:usort([ UserId || #{<<"user_id">> := UserId} <- Page1Users ++ Page2Users])). test_authenticator_user(PathPrefix) -> + UsersUri = uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]), + {ok, 200, _} = request( - post, - uri(PathPrefix ++ ["authentication"]), - emqx_authn_test_lib:built_in_database_example()), + post, + uri(PathPrefix ++ ["authentication"]), + emqx_authn_test_lib:built_in_database_example()), User = #{user_id => <<"u1">>, password => <<"p1">>}, - {ok, 201, _} = request( - post, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]), - User), + {ok, 201, _} = request(post, UsersUri, User), - {ok, 404, _} = request( - get, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users", "u123"])), + {ok, 404, _} = request(get, UsersUri ++ "/u123"), - {ok, 409, _} = request( - post, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]), - User), + {ok, 409, _} = request(post, UsersUri, User), - {ok, 200, UserData} = request( - get, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users", "u1"])), + {ok, 200, UserData} = request(get, UsersUri ++ "/u1"), FetchedUser = jiffy:decode(UserData, [return_maps]), ?assertMatch(#{<<"user_id">> := <<"u1">>}, FetchedUser), ?assertNotMatch(#{<<"password">> := _}, FetchedUser), ValidUserUpdates = [ - #{password => <<"p1">>}, - #{password => <<"p1">>, is_superuser => true}], + #{password => <<"p1">>}, + #{password => <<"p1">>, is_superuser => true}], lists:foreach( - fun(UserUpdate) -> - {ok, 200, _} = request( - put, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users", "u1"]), - UserUpdate) - end, - ValidUserUpdates), + fun(UserUpdate) -> {ok, 200, _} = request(put, UsersUri ++ "/u1", UserUpdate) end, + ValidUserUpdates), InvalidUserUpdates = [ - #{user_id => <<"u1">>, password => <<"p1">>}, - #{is_superuser => true}], + #{user_id => <<"u1">>, password => <<"p1">>}, + #{is_superuser => true}], lists:foreach( - fun(UserUpdate) -> - {ok, 400, _} = request( - put, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users", "u1"]), - UserUpdate) - end, - InvalidUserUpdates), + fun(UserUpdate) -> {ok, 400, _} = request(put, UsersUri ++ "/u1", UserUpdate) end, + InvalidUserUpdates), - {ok, 404, _} = request( - delete, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users", "u123"])), - - {ok, 204, _} = request( - delete, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users", "u1"])). + {ok, 404, _} = request(delete, UsersUri ++ "/u123"), + {ok, 204, _} = request(delete, UsersUri ++ "/u1"). test_authenticator_move(PathPrefix) -> AuthenticatorConfs = [ - emqx_authn_test_lib:http_example(), - emqx_authn_test_lib:jwt_example(), - emqx_authn_test_lib:built_in_database_example() - ], + emqx_authn_test_lib:http_example(), + emqx_authn_test_lib:jwt_example(), + emqx_authn_test_lib:built_in_database_example() + ], lists:foreach( - fun(Conf) -> - {ok, 200, _} = request( - post, - uri(PathPrefix ++ ["authentication"]), - Conf) - end, - AuthenticatorConfs), + fun(Conf) -> + {ok, 200, _} = request( + post, + uri(PathPrefix ++ ["authentication"]), + Conf) + end, + AuthenticatorConfs), ?assertAuthenticatorsMatch( - [ - #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}, - #{<<"mechanism">> := <<"jwt">>}, - #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>} - ], - PathPrefix ++ ["authentication"]), + [ + #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}, + #{<<"mechanism">> := <<"jwt">>}, + #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>} + ], + PathPrefix ++ ["authentication"]), % Invalid moves {ok, 400, _} = request( - post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), - #{position => <<"up">>}), + post, + uri(PathPrefix ++ ["authentication", "jwt", "move"]), + #{position => <<"up">>}), {ok, 400, _} = request( - post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), - #{}), + post, + uri(PathPrefix ++ ["authentication", "jwt", "move"]), + #{}), {ok, 404, _} = request( - post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), - #{position => <<"before:invalid">>}), + post, + uri(PathPrefix ++ ["authentication", "jwt", "move"]), + #{position => <<"before:invalid">>}), {ok, 404, _} = request( - post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), - #{position => <<"before:password-based:redis">>}), + post, + uri(PathPrefix ++ ["authentication", "jwt", "move"]), + #{position => <<"before:password-based:redis">>}), {ok, 404, _} = request( - post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), - #{position => <<"before:password-based:redis">>}), + post, + uri(PathPrefix ++ ["authentication", "jwt", "move"]), + #{position => <<"before:password-based:redis">>}), % Valid moves {ok, 204, _} = request( - post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), - #{position => <<"top">>}), + post, + uri(PathPrefix ++ ["authentication", "jwt", "move"]), + #{position => <<"top">>}), ?assertAuthenticatorsMatch( - [ - #{<<"mechanism">> := <<"jwt">>}, - #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}, - #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>} - ], - PathPrefix ++ ["authentication"]), + [ + #{<<"mechanism">> := <<"jwt">>}, + #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}, + #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>} + ], + PathPrefix ++ ["authentication"]), {ok, 204, _} = request( - post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), - #{position => <<"bottom">>}), + post, + uri(PathPrefix ++ ["authentication", "jwt", "move"]), + #{position => <<"bottom">>}), ?assertAuthenticatorsMatch( - [ - #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}, - #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>}, - #{<<"mechanism">> := <<"jwt">>} - ], - PathPrefix ++ ["authentication"]), + [ + #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}, + #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>}, + #{<<"mechanism">> := <<"jwt">>} + ], + PathPrefix ++ ["authentication"]), {ok, 204, _} = request( - post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), - #{position => <<"before:password-based:built-in-database">>}), + post, + uri(PathPrefix ++ ["authentication", "jwt", "move"]), + #{position => <<"before:password-based:built-in-database">>}), ?assertAuthenticatorsMatch( - [ - #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}, - #{<<"mechanism">> := <<"jwt">>}, - #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>} - ], - PathPrefix ++ ["authentication"]). + [ + #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}, + #{<<"mechanism">> := <<"jwt">>}, + #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>} + ], + PathPrefix ++ ["authentication"]). test_authenticator_import_users(PathPrefix) -> + ImportUri = uri( + PathPrefix ++ + ["authentication", "password-based:built-in-database", "import_users"]), + + {ok, 200, _} = request( - post, - uri(PathPrefix ++ ["authentication"]), - emqx_authn_test_lib:built_in_database_example()), + post, + uri(PathPrefix ++ ["authentication"]), + emqx_authn_test_lib:built_in_database_example()), - {ok, 400, _} = request( - post, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "import_users"]), - #{}), + {ok, 400, _} = request(post, ImportUri, #{}), - {ok, 400, _} = request( - post, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "import_users"]), - #{filename => <<"/etc/passwd">>}), + {ok, 400, _} = request(post, ImportUri, #{filename => <<"/etc/passwd">>}), - {ok, 400, _} = request( - post, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "import_users"]), - #{filename => <<"/not_exists.csv">>}), + {ok, 400, _} = request(post, ImportUri, #{filename => <<"/not_exists.csv">>}), Dir = code:lib_dir(emqx_authn, test), JSONFileName = filename:join([Dir, <<"data/user-credentials.json">>]), CSVFileName = filename:join([Dir, <<"data/user-credentials.csv">>]), - {ok, 204, _} = request( - post, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "import_users"]), - #{filename => JSONFileName}), + {ok, 204, _} = request(post, ImportUri, #{filename => JSONFileName}), - {ok, 204, _} = request( - post, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "import_users"]), - #{filename => CSVFileName}). + {ok, 204, _} = request(post, ImportUri, #{filename => CSVFileName}). %%------------------------------------------------------------------------------ %% Helpers From c60feaaad2d6d6ca64c962e4d17445eef03f8342 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 29 Oct 2021 11:30:57 -0300 Subject: [PATCH 034/179] test(fvt): extend functional verification tests to use replicant node This parameterizes the Functional Verification Tests (FVTs) that run in CI to use a replication log (RLOG) role of "replicant" for one of the nodes. With this addition, our FVTs may explore more scenarios with data replication. --- ...er-compose-emqx-cluster-rlog.override.yaml | 27 ++++++++++++++++ .../docker-compose-emqx-cluster.yaml | 28 +++++++---------- .ci/docker-compose-file/haproxy/haproxy.cfg | 1 - .ci/docker-compose-file/python/pytest.sh | 2 +- .ci/docker-compose-file/scripts/run-emqx.sh | 31 +++++++++++++++++++ .github/workflows/run_fvt_tests.yaml | 22 +++++-------- 6 files changed, 79 insertions(+), 32 deletions(-) create mode 100644 .ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml create mode 100755 .ci/docker-compose-file/scripts/run-emqx.sh diff --git a/.ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml b/.ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml new file mode 100644 index 000000000..3d8b86dd3 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml @@ -0,0 +1,27 @@ +x-default-emqx: &default-emqx + image: $TARGET:$EMQX_TAG + env_file: + - conf.cluster.env + healthcheck: + test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"] + interval: 5s + timeout: 25s + retries: 5 + +services: + emqx1: + <<: *default-emqx + environment: + - "EMQX_HOST=node1.emqx.io" + - "EMQX_CLUSTER__DB_BACKEND=rlog" + - "EMQX_CLUSTER__RLOG__ROLE=core" + - "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io]" + + emqx2: + <<: *default-emqx + environment: + - "EMQX_HOST=node2.emqx.io" + - "EMQX_CLUSTER__DB_BACKEND=rlog" + - "EMQX_CLUSTER__RLOG__ROLE=replicant" + - "EMQX_CLUSTER__RLOG__CORE_NODES=emqx@node1.emqx.io" + - "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io]" diff --git a/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml index 656905eb0..b2635ecfe 100644 --- a/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml +++ b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml @@ -1,5 +1,15 @@ version: '3.9' +x-default-emqx: &default-emqx + image: $TARGET:$EMQX_TAG + env_file: + - conf.cluster.env + healthcheck: + test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"] + interval: 5s + timeout: 25s + retries: 5 + services: haproxy: container_name: haproxy @@ -28,34 +38,20 @@ services: haproxy -f /usr/local/etc/haproxy/haproxy.cfg emqx1: + <<: *default-emqx container_name: node1.emqx.io - image: $TARGET:$EMQX_TAG - env_file: - - conf.cluster.env environment: - "EMQX_HOST=node1.emqx.io" - healthcheck: - test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"] - interval: 5s - timeout: 25s - retries: 5 networks: emqx_bridge: aliases: - node1.emqx.io emqx2: + <<: *default-emqx container_name: node2.emqx.io - image: $TARGET:$EMQX_TAG - env_file: - - conf.cluster.env environment: - "EMQX_HOST=node2.emqx.io" - healthcheck: - test: ["CMD", "/opt/emqx/bin/emqx", "ping"] - interval: 5s - timeout: 25s - retries: 5 networks: emqx_bridge: aliases: diff --git a/.ci/docker-compose-file/haproxy/haproxy.cfg b/.ci/docker-compose-file/haproxy/haproxy.cfg index b658789da..89c1d7d5d 100644 --- a/.ci/docker-compose-file/haproxy/haproxy.cfg +++ b/.ci/docker-compose-file/haproxy/haproxy.cfg @@ -54,7 +54,6 @@ backend emqx_dashboard_back server emqx-1 node1.emqx.io:18083 server emqx-2 node2.emqx.io:18083 - ##---------------------------------------------------------------- ## public ##---------------------------------------------------------------- diff --git a/.ci/docker-compose-file/python/pytest.sh b/.ci/docker-compose-file/python/pytest.sh index eacbecc3b..75f6441b5 100755 --- a/.ci/docker-compose-file/python/pytest.sh +++ b/.ci/docker-compose-file/python/pytest.sh @@ -1,7 +1,7 @@ #!/bin/sh ## This script is to run emqx cluster smoke tests (fvt) in github action -## This script is executed in pacho_client +## This script is executed in paho_client set -x set +e diff --git a/.ci/docker-compose-file/scripts/run-emqx.sh b/.ci/docker-compose-file/scripts/run-emqx.sh new file mode 100755 index 000000000..ebb07b8b6 --- /dev/null +++ b/.ci/docker-compose-file/scripts/run-emqx.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -euxo pipefail + +if [ "$EMQX_TEST_DB_BACKEND" = "rlog" ] +then + CLUSTER_OVERRIDES="-f .ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml" +else + CLUSTER_OVERRIDES="" +fi + +{ + echo "HOCON_ENV_OVERRIDE_PREFIX=EMQX_" + echo "EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s" + echo "EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10" +} >> .ci/docker-compose-file/conf.cluster.env + +is_cluster_up() { + docker exec -i node1.emqx.io \ + bash -c "emqx eval \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1 +} + +docker-compose \ + -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ + $CLUSTER_OVERRIDES \ + -f .ci/docker-compose-file/docker-compose-python.yaml \ + up -d + +while ! is_cluster_up; do + echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx"; + sleep 5; +done diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 509e84bab..e696ade29 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -69,8 +69,11 @@ jobs: fail-fast: false matrix: otp: - - 23.2.7.2-emqx-2 - - 24.1.1-emqx-1 + - 23.2.7.2-emqx-2 + - 24.1.1-emqx-1 + cluster_db_backend: + - "mnesia" + - "rlog" steps: - uses: actions/download-artifact@v2 @@ -91,18 +94,9 @@ jobs: timeout-minutes: 5 working-directory: source run: | - set -e -u -x - echo "HOCON_ENV_OVERRIDE_PREFIX=EMQX_" >> .ci/docker-compose-file/conf.cluster.env - echo "EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s" >> .ci/docker-compose-file/conf.cluster.env - echo "EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10" >> .ci/docker-compose-file/conf.cluster.env - docker-compose \ - -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ - -f .ci/docker-compose-file/docker-compose-python.yaml \ - up -d - while ! docker exec -i node1.emqx.io bash -c "emqx eval \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1; do - echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx"; - sleep 5; - done + set -x + export EMQX_TEST_DB_BACKEND="${{ matrix.cluster_db_backend }}" + ./.ci/docker-compose-file/scripts/run-emqx.sh - name: make paho tests run: | if ! docker exec -i python /scripts/pytest.sh; then From ee817cfa6f4d32bf52e423425b3148cb0950b64d Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 8 Nov 2021 13:34:12 +0100 Subject: [PATCH 035/179] fix(bin/emqx): mnesia dir name after node --- bin/emqx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/emqx b/bin/emqx index 23b991337..15f813eae 100755 --- a/bin/emqx +++ b/bin/emqx @@ -46,7 +46,6 @@ export EMU="beam" export PROGNAME="erl" export LD_LIBRARY_PATH="$ERTS_DIR/lib:$LD_LIBRARY_PATH" export ERTS_LIB_DIR="$ERTS_DIR/../lib" -MNESIA_DATA_DIR="$RUNNER_DATA_DIR/mnesia/$NAME" # Echo to stderr on errors echoerr() { echo "ERROR: $*" 1>&2; } @@ -374,6 +373,7 @@ fi # force to use 'emqx' short name [ -z "$NAME" ] && NAME='emqx' +MNESIA_DATA_DIR="$RUNNER_DATA_DIR/mnesia/$NAME" case "$NAME" in *@*) From b873b9271702a995cf6fa68001d8c414e09c8ea5 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 8 Nov 2021 23:23:17 +0800 Subject: [PATCH 036/179] fix(test): cluster_rpc retry interval incorrect (#6038) --- .../emqx_conf/test/emqx_cluster_rpc_SUITE.erl | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl index 4e689916a..89a44dfee 100644 --- a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl +++ b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl @@ -121,22 +121,16 @@ t_catch_up_status_handle_next_commit(_Config) -> t_commit_ok_apply_fail_on_other_node_then_recover(_Config) -> emqx_cluster_rpc:reset(), {atomic, []} = emqx_cluster_rpc:status(), - Now = erlang:system_time(millisecond), + ets:new(test, [named_table, public]), ct:pal("111:~p~n", [ets:tab2list(cluster_rpc_commit)]), - {M, F, A} = {?MODULE, failed_on_other_recover_after_5_second, [erlang:whereis(?NODE1), Now]}, + {M, F, A} = {?MODULE, failed_on_other_recover_after_retry, [erlang:whereis(?NODE1)]}, {ok, 1, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), ct:pal("222:~p~n", [ets:tab2list(cluster_rpc_commit)]), - {ok, 2, ok} = emqx_cluster_rpc:multicall(io, format, ["test"], 1, 1000), - ct:pal("333:~p~n", [ets:tab2list(cluster_rpc_commit)]), - ct:pal("444:~p~n", [emqx_cluster_rpc:status()]), - {atomic, [Status|L]} = emqx_cluster_rpc:status(), + ct:pal("333:~p~n", [emqx_cluster_rpc:status()]), + {atomic, [_Status|L]} = emqx_cluster_rpc:status(), ?assertEqual([], L), - ?assertEqual({io, format, ["test"]}, maps:get(mfa, Status)), - ?assertEqual(node(), maps:get(node, Status)), - ct:sleep(2300), - {atomic, [Status1]} = emqx_cluster_rpc:status(), - ?assertEqual(Status, Status1), - ct:sleep(3600), + {ok, 2, ok} = emqx_cluster_rpc:multicall(io, format, ["test"], 1, 1000), + ct:sleep(1000), {atomic, NewStatus} = emqx_cluster_rpc:status(), ?assertEqual(3, length(NewStatus)), Pid = self(), @@ -244,12 +238,12 @@ failed_on_node_by_odd(Pid) -> end end. -failed_on_other_recover_after_5_second(Pid, CreatedAt) -> - Now = erlang:system_time(millisecond), +failed_on_other_recover_after_retry(Pid) -> + Counter = ets:update_counter(test, counter, 1, {counter, 0}), case Pid =:= self() of true -> ok; false -> - case Now < CreatedAt + 5001 of + case Counter < 4 of true -> "MFA return not ok"; false -> ok end From fdc4bb06d32bd80f939be54aa20970c5b3964e04 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sun, 31 Oct 2021 10:10:15 +0100 Subject: [PATCH 037/179] build: copy dynamic libs for zip package --- bin/emqx | 29 ++++++++++++++++++++++++++++- build | 15 +++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/bin/emqx b/bin/emqx index 15f813eae..a97e13884 100755 --- a/bin/emqx +++ b/bin/emqx @@ -44,8 +44,8 @@ export ERTS_DIR="$ROOTDIR/erts-$ERTS_VSN" export BINDIR="$ERTS_DIR/bin" export EMU="beam" export PROGNAME="erl" -export LD_LIBRARY_PATH="$ERTS_DIR/lib:$LD_LIBRARY_PATH" export ERTS_LIB_DIR="$ERTS_DIR/../lib" +DYNLIBS_DIR="$RUNNER_ROOT_DIR/dynlibs" # Echo to stderr on errors echoerr() { echo "ERROR: $*" 1>&2; } @@ -62,6 +62,33 @@ assert_node_alive() { fi } + +# Echo to stderr on errors +echoerr() { echo "$*" 1>&2; } + +check_eralng_start() { + "$BINDIR/$PROGNAME" -noshell -boot "$REL_DIR/start_clean" -s crypto start -s init stop +} + +if ! check_eralng_start >/dev/null 2>&1; then + BUILT_ON="$(head -1 "${REL_DIR}/BUILT_ON")" + ## failed to start, might be due to missing libs, try to be portable + export LD_LIBRARY_PATH="$DYNLIBS_DIR:$LD_LIBRARY_PATH" + if ! check_eralng_start; then + ## it's hopeless + echoerr "FATAL: Unable to start Erlang (with libcrypto)." + echoerr "Please make sure it's running on the correct platform with all required dependencies." + echoerr "This EMQ X release is built for $BUILT_ON" + exit 1 + fi + echoerr "WARNING: There seem to be missing dynamic libs from the OS. Using libs from ${DYNLIBS_DIR}" +fi + +## backward compatible +if [ -d "$ERTS_DIR/lib" ]; then + export LD_LIBRARY_PATH="$ERTS_DIR/lib:$LD_LIBRARY_PATH" +fi + relx_usage() { command="$1" diff --git a/build b/build index d12cddc46..1e4fac8c8 100755 --- a/build +++ b/build @@ -120,6 +120,18 @@ make_relup() { ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}" } +cp_dyn_libs() { + local rel_dir="$1" + local target_dir="${rel_dir}/dynlibs" + if ! [ "$(uname -s)" = 'Linux' ]; then + return 0; + fi + mkdir -p "$target_dir" + while read -r so_file; do + cp -L "$so_file" "$target_dir/" + done < <(find "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 | xargs -0 ldd | grep -E '^\s+.*=>\s(/lib|/usr)' | awk '{print $3}') +} + ## make_zip turns .tar.gz into a .zip with a slightly different name. ## It assumes the .tar.gz has been built -- relies on Makefile dependency make_zip() { @@ -139,6 +151,9 @@ make_zip() { local zipball zipball="${pkgpath}/${PROFILE}-${SYSTEM}-${PKG_VSN}-${ARCH}.zip" tar zxf "${tarball}" -C "${tard}/emqx" + ## try to be portable for zip packages. + ## for DEB and RPM packages the dependencies are resoved by yum and apt + cp_dyn_libs "${tard}/emqx" (cd "${tard}" && zip -qr - emqx) > "${zipball}" } From f8fc67b313a10ee5eda16be2d97b67ac94cf7cb7 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Sun, 7 Nov 2021 16:54:38 -0300 Subject: [PATCH 038/179] fix(lag): target only replica if rlog core+replicant there seems to be race conditions related to some tests with sessions hitting the core and the replicant alternately and rlog. for intance, if there is some delay in this replication, a new connection made to the replica with a just-created session in the core may not have been replicated to the replicant, resulting in a test failure if it expects the session to be present. since such replication lags are inherent to the core-replicant topology, we can try to target only the replicant to avoid seeing this inconsistent view of the system during the tests. --- ...er-compose-emqx-cluster-rlog.override.yaml | 6 ++++++ .ci/docker-compose-file/python/pytest.sh | 14 ++++++++++--- .ci/docker-compose-file/scripts/run-emqx.sh | 20 +++++++++++++++++-- .github/workflows/run_fvt_tests.yaml | 5 ++++- 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml b/.ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml index 3d8b86dd3..8be146eb5 100644 --- a/.ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml +++ b/.ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml @@ -11,17 +11,23 @@ x-default-emqx: &default-emqx services: emqx1: <<: *default-emqx + container_name: node1.emqx.io environment: - "EMQX_HOST=node1.emqx.io" - "EMQX_CLUSTER__DB_BACKEND=rlog" - "EMQX_CLUSTER__RLOG__ROLE=core" - "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io]" + - "EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL=false" + - "EMQX_LISTENERS__WS__DEFAULT__PROXY_PROTOCOL=false" emqx2: <<: *default-emqx + container_name: node2.emqx.io environment: - "EMQX_HOST=node2.emqx.io" - "EMQX_CLUSTER__DB_BACKEND=rlog" - "EMQX_CLUSTER__RLOG__ROLE=replicant" - "EMQX_CLUSTER__RLOG__CORE_NODES=emqx@node1.emqx.io" - "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io]" + - "EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL=false" + - "EMQX_LISTENERS__WS__DEFAULT__PROXY_PROTOCOL=false" diff --git a/.ci/docker-compose-file/python/pytest.sh b/.ci/docker-compose-file/python/pytest.sh index 75f6441b5..4579691b3 100755 --- a/.ci/docker-compose-file/python/pytest.sh +++ b/.ci/docker-compose-file/python/pytest.sh @@ -6,16 +6,24 @@ set -x set +e -LB="haproxy" +EMQX_TEST_DB_BACKEND=$1 +if [ "$EMQX_TEST_DB_BACKEND" = "rlog" ] +then + # target only replica to avoid replication races + TARGET_HOST="node2.emqx.io" +else + # use loadbalancer + TARGET_HOST="haproxy" +fi apk update && apk add git curl git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing pip install pytest -pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$LB" +pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$TARGET_HOST" RESULT=$? -pytest -v /paho.mqtt.testing/interoperability/test_client --host "$LB" +pytest -v /paho.mqtt.testing/interoperability/test_client --host "$TARGET_HOST" RESULT=$(( RESULT + $? )) # pytest -v /paho.mqtt.testing/interoperability/test_cluster --host1 "node1.emqx.io" --host2 "node2.emqx.io" diff --git a/.ci/docker-compose-file/scripts/run-emqx.sh b/.ci/docker-compose-file/scripts/run-emqx.sh index ebb07b8b6..1465cb655 100755 --- a/.ci/docker-compose-file/scripts/run-emqx.sh +++ b/.ci/docker-compose-file/scripts/run-emqx.sh @@ -14,11 +14,27 @@ fi echo "EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10" } >> .ci/docker-compose-file/conf.cluster.env -is_cluster_up() { - docker exec -i node1.emqx.io \ +is_node_up() { + local node + node="$1" + docker exec -i "$node" \ bash -c "emqx eval \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1 } +is_node_listening() { + local node + node="$1" + docker exec -i "$node" \ + emqx eval "ok = case gen_tcp:connect(\"localhost\", 1883, []) of {ok, P} -> gen_tcp:close(P), ok; _ -> exit(1) end." > /dev/null 2>&1 +} + +is_cluster_up() { + is_node_up node1.emqx.io && \ + is_node_up node2.emqx.io && \ + is_node_listening node1.emqx.io && \ + is_node_listening node2.emqx.io +} + docker-compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ $CLUSTER_OVERRIDES \ diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index e696ade29..46ce95dab 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -99,10 +99,13 @@ jobs: ./.ci/docker-compose-file/scripts/run-emqx.sh - name: make paho tests run: | - if ! docker exec -i python /scripts/pytest.sh; then + if ! docker exec -i python /scripts/pytest.sh "${{ matrix.cluster_db_backend }}"; then echo "DUMP_CONTAINER_LOGS_BGN" + echo "============== haproxy ==============" docker logs haproxy + echo "============== node1 ==============" docker logs node1.emqx.io + echo "============== node2 ==============" docker logs node2.emqx.io echo "DUMP_CONTAINER_LOGS_END" exit 1 From 030e4857ec7ab31a028b3f60b7d011c43778d0fd Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 8 Nov 2021 15:36:54 -0300 Subject: [PATCH 039/179] docs(issue): mark solution as TODO and link related issue https://github.com/emqx/emqx/issues/6094 --- .ci/docker-compose-file/python/pytest.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/docker-compose-file/python/pytest.sh b/.ci/docker-compose-file/python/pytest.sh index 4579691b3..c079a65a4 100755 --- a/.ci/docker-compose-file/python/pytest.sh +++ b/.ci/docker-compose-file/python/pytest.sh @@ -9,7 +9,8 @@ set +e EMQX_TEST_DB_BACKEND=$1 if [ "$EMQX_TEST_DB_BACKEND" = "rlog" ] then - # target only replica to avoid replication races + # TODO: target only replica to avoid replication races + # see: https://github.com/emqx/emqx/issues/6094 TARGET_HOST="node2.emqx.io" else # use loadbalancer From eea789451ba46d4816f0dd23f4d2ef3453ee6e70 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 9 Nov 2021 15:41:28 +0800 Subject: [PATCH 040/179] feat: sha3_256 dashboard account's password (#6084) --- .../emqx_dashboard/include/emqx_dashboard.hrl | 10 ++-- .../src/emqx_dashboard_admin.erl | 15 ++--- .../src/emqx_dashboard_token.erl | 19 ++++--- .../test/emqx_dashboard_SUITE.erl | 55 +++++++++++++------ 4 files changed, 60 insertions(+), 39 deletions(-) diff --git a/apps/emqx_dashboard/include/emqx_dashboard.hrl b/apps/emqx_dashboard/include/emqx_dashboard.hrl index e7fb4557b..712be13a0 100644 --- a/apps/emqx_dashboard/include/emqx_dashboard.hrl +++ b/apps/emqx_dashboard/include/emqx_dashboard.hrl @@ -13,8 +13,9 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- +-define(ADMIN, emqx_admin). --record(emqx_admin, { +-record(?ADMIN, { username :: binary(), pwdhash :: binary(), tags :: list() | binary(), @@ -22,17 +23,16 @@ extra = [] :: term() %% not used so far, for future extension }). --define(ADMIN, emqx_admin). --record(emqx_admin_jwt, { +-define(ADMIN_JWT, emqx_admin_jwt). + +-record(?ADMIN_JWT, { token :: binary(), username :: binary(), exptime :: integer(), extra = [] :: term() %% not used so far, fur future extension }). --define(ADMIN_JWT, emqx_admin_jwt). - -define(EMPTY_KEY(Key), ((Key == undefined) orelse (Key == <<>>))). -define(DASHBOARD_SHARD, emqx_dashboard_shard). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index 9622e6ed8..68ddac651 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -167,7 +167,7 @@ check(_, undefined) -> check(Username, Password) -> case lookup_user(Username) of [#?ADMIN{pwdhash = <>}] -> - case Hash =:= md5_hash(Salt, Password) of + case Hash =:= sha3_hash(Salt, Password) of true -> ok; false -> {error, <<"BAD_USERNAME_OR_PASSWORD">>} end; @@ -201,16 +201,11 @@ destroy_token_by_username(Username, Token) -> %%-------------------------------------------------------------------- hash(Password) -> - SaltBin = salt(), - <>. + SaltBin = emqx_dashboard_token:salt(), + <>. -md5_hash(SaltBin, Password) -> - erlang:md5(<>). - -salt() -> - _ = emqx_misc:rand_seed(), - Salt = rand:uniform(16#ffffffff), - <>. +sha3_hash(SaltBin, Password) -> + crypto:hash('sha3_256', <>). add_default_user() -> add_default_user(binenv(default_username), binenv(default_password)). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_token.erl b/apps/emqx_dashboard/src/emqx_dashboard_token.erl index f8c023b46..ffd45241b 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_token.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_token.erl @@ -40,7 +40,7 @@ %% gen server part -behaviour(gen_server). --export([start_link/0]). +-export([start_link/0, salt/0]). -export([ init/1 , handle_call/3 @@ -75,6 +75,12 @@ destroy(Token) when is_binary(Token)-> destroy_by_username(Username) -> do_destroy_by_username(Username). +%% @doc create 4 bytes salt. +-spec(salt() -> binary()). +salt() -> + <> = crypto:strong_rand_bytes(2), + iolist_to_binary(io_lib:format("~4.16.0b", [X])). + mnesia(boot) -> ok = mria:create_table(?TAB, [ {type, set}, @@ -110,7 +116,9 @@ do_verify(Token)-> case ExpTime > erlang:system_time(millisecond) of true -> NewJWT = JWT#?ADMIN_JWT{exptime = jwt_expiration_time()}, - {atomic, Res} = mria:transaction(?DASHBOARD_SHARD, fun mnesia:write/1, [NewJWT]), + {atomic, Res} = mria:transaction(?DASHBOARD_SHARD, + fun mnesia:write/1, + [NewJWT]), Res; _ -> {error, token_timeout} @@ -145,7 +153,7 @@ lookup_by_username(Username) -> List. jwk(Username, Password, Salt) -> - Key = erlang:md5(<>), + Key = crypto:hash(md5, <>), #{ <<"kty">> => <<"oct">>, <<"k">> => jose_base64url:encode(Key) @@ -157,11 +165,6 @@ jwt_expiration_time() -> token_ttl() -> emqx_conf:get([emqx_dashboard, token_expired_time], ?EXPTIME). -salt() -> - _ = emqx_misc:rand_seed(), - Salt = rand:uniform(16#ffffffff), - <>. - format(Token, Username, ExpTime) -> #?ADMIN_JWT{ token = Token, diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 42ffd7d45..f876f7383 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -37,7 +37,18 @@ -define(BASE_PATH, "api"). --define(OVERVIEWS, ['alarms/activated', 'alarms/deactivated', banned, brokers, stats, metrics, listeners, clients, subscriptions, routes, plugins]). +-define(OVERVIEWS, ['alarms/activated', + 'alarms/deactivated', + banned, + brokers, + stats, + metrics, + listeners, + clients, + subscriptions, + routes, + plugins + ]). all() -> %% TODO: V5 API @@ -45,7 +56,8 @@ all() -> [t_cli, t_lookup_by_username_jwt, t_clean_expired_jwt]. init_per_suite(Config) -> - emqx_common_test_helpers:start_apps([emqx_management, emqx_dashboard],fun set_special_configs/1), + emqx_common_test_helpers:start_apps([emqx_management, emqx_dashboard], + fun set_special_configs/1), Config. end_per_suite(_Config) -> @@ -53,7 +65,8 @@ end_per_suite(_Config) -> mria:stop(). set_special_configs(emqx_management) -> - emqx_config:put([emqx_management], #{listeners => [#{protocol => http, port => 8081}], + Listeners = [#{protocol => http, port => 8081}], + emqx_config:put([emqx_management], #{listeners => Listeners, applications =>[#{id => "admin", secret => "public"}]}), ok; set_special_configs(_) -> @@ -62,27 +75,33 @@ set_special_configs(_) -> t_overview(_) -> mnesia:clear_table(?ADMIN), emqx_dashboard_admin:add_user(<<"admin">>, <<"public">>, <<"tag">>), - [?assert(request_dashboard(get, api_path(erlang:atom_to_list(Overview)), auth_header_()))|| Overview <- ?OVERVIEWS]. + [?assert(request_dashboard(get, api_path(erlang:atom_to_list(Overview)), + auth_header_()))|| Overview <- ?OVERVIEWS]. t_admins_add_delete(_) -> mnesia:clear_table(?ADMIN), - ok = emqx_dashboard_admin:add_user(<<"username">>, <<"password">>, <<"tag">>), - ok = emqx_dashboard_admin:add_user(<<"username1">>, <<"password1">>, <<"tag1">>), + Tag = <<"tag">>, + ok = emqx_dashboard_admin:add_user(<<"username">>, <<"password">>, Tag), + ok = emqx_dashboard_admin:add_user(<<"username1">>, <<"password1">>, Tag), Admins = emqx_dashboard_admin:all_users(), ?assertEqual(2, length(Admins)), ok = emqx_dashboard_admin:remove_user(<<"username1">>), Users = emqx_dashboard_admin:all_users(), ?assertEqual(1, length(Users)), - ok = emqx_dashboard_admin:change_password(<<"username">>, <<"password">>, <<"pwd">>), + ok = emqx_dashboard_admin:change_password(<<"username">>, + <<"password">>, + <<"pwd">>), timer:sleep(10), - ?assert(request_dashboard(get, api_path("brokers"), auth_header_("username", "pwd"))), + Header = auth_header_("username", "pwd"), + ?assert(request_dashboard(get, api_path("brokers"), Header)), ok = emqx_dashboard_admin:remove_user(<<"username">>), - ?assertNotEqual(true, request_dashboard(get, api_path("brokers"), auth_header_("username", "pwd"))). + ?assertNotEqual(true, request_dashboard(get, api_path("brokers"), Header)). t_rest_api(_Config) -> mnesia:clear_table(?ADMIN), - emqx_dashboard_admin:add_user(<<"admin">>, <<"public">>, <<"administrator">>), + Tag = <<"administrator">>, + emqx_dashboard_admin:add_user(<<"admin">>, <<"public">>, Tag), {ok, Res0} = http_get("users"), ?assertEqual([#{<<"username">> => <<"admin">>, @@ -93,11 +112,15 @@ t_rest_api(_Config) -> end, [AssertSuccess(R) || R <- [ http_put("users/admin", #{<<"tags">> => <<"a_new_tag">>}) - , http_post("users", #{<<"username">> => <<"usera">>, <<"password">> => <<"passwd">>}) - , http_post("auth", #{<<"username">> => <<"usera">>, <<"password">> => <<"passwd">>}) + , http_post("users", #{<<"username">> => <<"usera">>, + <<"password">> => <<"passwd">>}) + , http_post("auth", #{<<"username">> => <<"usera">>, + <<"password">> => <<"passwd">>}) , http_delete("users/usera") - , http_put("users/admin/change_pwd", #{<<"old_pwd">> => <<"public">>, <<"new_pwd">> => <<"newpwd">>}) - , http_post("auth", #{<<"username">> => <<"admin">>, <<"password">> => <<"newpwd">>}) + , http_put("users/admin/change_pwd", #{<<"old_pwd">> => <<"public">>, + <<"new_pwd">> => <<"newpwd">>}) + , http_post("auth", #{<<"username">> => <<"admin">>, + <<"password">> => <<"newpwd">>}) ]], ok. @@ -106,11 +129,11 @@ t_cli(_Config) -> emqx_dashboard_cli:admins(["add", "username", "password"]), [#?ADMIN{ username = <<"username">>, pwdhash = <>}] = emqx_dashboard_admin:lookup_user(<<"username">>), - ?assertEqual(Hash, erlang:md5(<>/binary>>)), + ?assertEqual(Hash, crypto:hash(sha3_256, <>/binary>>)), emqx_dashboard_cli:admins(["passwd", "username", "newpassword"]), [#?ADMIN{username = <<"username">>, pwdhash = <>}] = emqx_dashboard_admin:lookup_user(<<"username">>), - ?assertEqual(Hash1, erlang:md5(<>/binary>>)), + ?assertEqual(Hash1, crypto:hash(sha3_256, <>/binary>>)), emqx_dashboard_cli:admins(["del", "username"]), [] = emqx_dashboard_admin:lookup_user(<<"username">>), emqx_dashboard_cli:admins(["add", "admin1", "pass1"]), From b0dd944fa45c74b8f9916a37fe560d57323e2869 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 9 Nov 2021 11:33:32 +0100 Subject: [PATCH 041/179] chore(github-issue-template): https://askemq.com/ --- .github/ISSUE_TEMPLATE/support-needed.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/support-needed.md b/.github/ISSUE_TEMPLATE/support-needed.md index 49ba5a913..e50bdfcbf 100644 --- a/.github/ISSUE_TEMPLATE/support-needed.md +++ b/.github/ISSUE_TEMPLATE/support-needed.md @@ -9,6 +9,9 @@ labels: "Support, needs-triage" ### Subject of the support From 8ecdadee3f0624d345db4b96087b99b1f4aadd58 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 9 Nov 2021 21:08:55 +0100 Subject: [PATCH 042/179] chore(rebar.config): pin ehttpc 0.1.12 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 87b45b695..6ae9523ff 100644 --- a/rebar.config +++ b/rebar.config @@ -47,7 +47,7 @@ [ {lc, {git, "https://github.com/qzhuyan/lc.git", {tag, "0.1.2"}}} , {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}} - , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.9"}}} + , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.12"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.3"}}} From a27b452fe7ba65ddba2617aa575ede81aec538c7 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 9 Nov 2021 21:06:37 +0100 Subject: [PATCH 043/179] chore(bin/emqx): bash set -o --- bin/emqx | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/bin/emqx b/bin/emqx index a97e13884..f8c4d3f6c 100755 --- a/bin/emqx +++ b/bin/emqx @@ -2,8 +2,7 @@ # -*- tab-width:4;indent-tabs-mode:nil -*- # ex: ts=4 sw=4 et -set -e -set -o pipefail +set -euo pipefail DEBUG="${DEBUG:-0}" if [ "$DEBUG" -eq 1 ]; then @@ -171,11 +170,9 @@ if [ "$ES" -ne 0 ]; then exit $ES fi -if [ -z "$WITH_EPMD" ]; then - EPMD_ARG="-start_epmd false -epmd_module ekka_epmd -proto_dist ekka" -else - EPMD_ARG="-start_epmd true $PROTO_DIST_ARG" -fi +# EPMD_ARG="-start_epmd true $PROTO_DIST_ARG" +NO_EPMD="-start_epmd false -epmd_module ekka_epmd -proto_dist ekka" +EPMD_ARG="${EPMD_ARG:-${NO_EPMD}}" # Warn the user if ulimit -n is less than 1024 ULIMIT_F=$(ulimit -n) From 1480bb61585d9bb10a663bbc38f6b7f190d4d03b Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 9 Nov 2021 21:13:05 +0100 Subject: [PATCH 044/179] chore(bin/emqx): delete RELX_CONFIG_PATH this variable is to allow setting sys.config by user however this feature is now broken because the we always generate app.