From b41c41570a1f030e8854aac52eecea257acabb25 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 26 Nov 2021 15:14:53 +0800 Subject: [PATCH 01/72] fix(authz): placehodler regular escape --- apps/emqx/src/emqx_topic.erl | 1 - apps/emqx/test/emqx_topic_SUITE.erl | 5 +++-- apps/emqx_authz/README.md | 6 +++--- apps/emqx_authz/etc/emqx_authz.conf | 8 ++++---- apps/emqx_authz/src/emqx_authz.erl | 5 +++++ apps/emqx_authz/src/emqx_authz_http.erl | 14 +++++++------- apps/emqx_authz/src/emqx_authz_mongodb.erl | 6 +++--- apps/emqx_authz/src/emqx_authz_redis.erl | 9 +++++---- 8 files changed, 30 insertions(+), 24 deletions(-) diff --git a/apps/emqx/src/emqx_topic.erl b/apps/emqx/src/emqx_topic.erl index 00d26d147..5e0f2d43b 100644 --- a/apps/emqx/src/emqx_topic.erl +++ b/apps/emqx/src/emqx_topic.erl @@ -218,4 +218,3 @@ parse(TopicFilter = <<"$share/", Rest/binary>>, Options) -> end; parse(TopicFilter, Options) -> {TopicFilter, Options}. - diff --git a/apps/emqx/test/emqx_topic_SUITE.erl b/apps/emqx/test/emqx_topic_SUITE.erl index 76b95d9bc..e0d80c4a5 100644 --- a/apps/emqx/test/emqx_topic_SUITE.erl +++ b/apps/emqx/test/emqx_topic_SUITE.erl @@ -20,6 +20,7 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx_placeholder.hrl"). -import(emqx_topic, [ wildcard/1 @@ -183,9 +184,9 @@ t_feed_var(_) -> ?assertEqual(<<"$queue/client/clientId">>, feed_var(<<"$c">>, <<"clientId">>, <<"$queue/client/$c">>)), ?assertEqual(<<"username/test/client/x">>, - feed_var(<<"%u">>, <<"test">>, <<"username/%u/client/x">>)), + feed_var(?PH_USERNAME, <<"test">>, <<"username/", ?PH_USERNAME/binary, "/client/x">>)), ?assertEqual(<<"username/test/client/clientId">>, - feed_var(<<"%c">>, <<"clientId">>, <<"username/test/client/%c">>)). + feed_var(?PH_CLIENTID, <<"clientId">>, <<"username/test/client/", ?PH_CLIENTID/binary>>)). long_topic() -> iolist_to_binary([[integer_to_list(I), "/"] || I <- lists:seq(0, 66666)]). diff --git a/apps/emqx_authz/README.md b/apps/emqx_authz/README.md index a44297a55..bda09481a 100644 --- a/apps/emqx_authz/README.md +++ b/apps/emqx_authz/README.md @@ -23,7 +23,7 @@ authz:{ keyfile: "etc/certs/client-key.pem" } } - sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or clientid = '%c'" + sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = ${peerhost} or username = ${username} or clientid = ${clientid}" }, { type: postgresql @@ -36,7 +36,7 @@ authz:{ auto_reconnect: true ssl: {enable: false} } - sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'" + sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = ${peerhost} or username = ${username} or username = '$all' or clientid = ${clientid}" }, { type: redis @@ -48,7 +48,7 @@ authz:{ auto_reconnect: true ssl: {enable: false} } - cmd: "HGETALL mqtt_authz:%u" + cmd: "HGETALL mqtt_authz:${username}" }, { principal: {username: "^admin?"} diff --git a/apps/emqx_authz/etc/emqx_authz.conf b/apps/emqx_authz/etc/emqx_authz.conf index 3469aad3a..5bb6ab841 100644 --- a/apps/emqx_authz/etc/emqx_authz.conf +++ b/apps/emqx_authz/etc/emqx_authz.conf @@ -22,7 +22,7 @@ authorization { # certfile: "{{ platform_etc_dir }}/certs/client-cert.pem" # keyfile: "{{ platform_etc_dir }}/certs/client-key.pem" # } - # query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or clientid = '%c'" + # query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = ${peerhost} or username = ${username} or clientid = ${clientid}" # }, # { # type: postgresql @@ -33,7 +33,7 @@ authorization { # password: public # auto_reconnect: true # ssl: {enable: false} - # query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'" + # query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = ${peerhost} or username = ${username} or username = '$all' or clientid = ${clientid}" # }, # { # type: redis @@ -43,7 +43,7 @@ authorization { # password: public # auto_reconnect: true # ssl: {enable: false} - # cmd: "HGETALL mqtt_authz:%u" + # cmd: "HGETALL mqtt_authz:${username}" # }, # { # type: mongodb @@ -53,7 +53,7 @@ authorization { # database: mqtt # ssl: {enable: false} # collection: mqtt_authz - # selector: { "$or": [ { "username": "%u" }, { "clientid": "%c" } ] } + # selector: { "$or": [ { "username": "${username}" }, { "clientid": "${clientid}" } ] } # }, { type: built-in-database diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 4496e0299..a5efbe8f7 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -40,6 +40,8 @@ -export([acl_conf_file/0]). +-export([ph_to_re/1]). + -spec(register_metrics() -> ok). register_metrics() -> lists:foreach(fun emqx_metrics:ensure/1, ?AUTHZ_METRICS). @@ -386,3 +388,6 @@ type(Unknown) -> error({unknown_authz_source_type, Unknown}). % should never hap %% @doc where the acl.conf file is stored. acl_conf_file() -> filename:join([emqx:data_dir(), "authz", "acl.conf"]). + +ph_to_re(VarPH) -> + re:replace(VarPH, "[\\$\\{\\}]", "\\\\&", [global, {return, list}]). diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index 4c6af402c..4c17f90ec 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -87,19 +87,19 @@ replvar(Str0, PubSub, Topic, }) when is_list(Str0); is_binary(Str0) -> NTopic = emqx_http_lib:uri_encode(Topic), - Str1 = re:replace( Str0, ?PH_S_CLIENTID + Str1 = re:replace( Str0, emqx_authz:ph_to_re(?PH_S_CLIENTID) , Clientid, [global, {return, binary}]), - Str2 = re:replace( Str1, ?PH_S_USERNAME + Str2 = re:replace( Str1, emqx_authz:ph_to_re(?PH_S_USERNAME) , bin(Username), [global, {return, binary}]), - Str3 = re:replace( Str2, ?PH_S_HOST + Str3 = re:replace( Str2, emqx_authz:ph_to_re(?PH_S_HOST) , inet_parse:ntoa(IpAddress), [global, {return, binary}]), - Str4 = re:replace( Str3, ?PH_S_PROTONAME + Str4 = re:replace( Str3, emqx_authz:ph_to_re(?PH_S_PROTONAME) , bin(Protocol), [global, {return, binary}]), - Str5 = re:replace( Str4, ?PH_S_MOUNTPOINT + Str5 = re:replace( Str4, emqx_authz:ph_to_re(?PH_S_MOUNTPOINT) , Mountpoint, [global, {return, binary}]), - Str6 = re:replace( Str5, ?PH_S_TOPIC + Str6 = re:replace( Str5, emqx_authz:ph_to_re(?PH_S_TOPIC) , NTopic, [global, {return, binary}]), - Str7 = re:replace( Str6, ?PH_S_ACTION + Str7 = re:replace( Str6, emqx_authz:ph_to_re(?PH_S_ACTION) , bin(PubSub), [global, {return, binary}]), Str7. diff --git a/apps/emqx_authz/src/emqx_authz_mongodb.erl b/apps/emqx_authz/src/emqx_authz_mongodb.erl index ec34a266c..5b55c23b7 100644 --- a/apps/emqx_authz/src/emqx_authz_mongodb.erl +++ b/apps/emqx_authz/src/emqx_authz_mongodb.erl @@ -76,11 +76,11 @@ replvar(Selector, #{clientid := Clientid, end || M <- V], AccIn); InFun(K, V, AccIn) when is_binary(V) -> - V1 = re:replace( V, ?PH_S_CLIENTID + V1 = re:replace( V, emqx_authz:ph_to_re(?PH_S_CLIENTID) , bin(Clientid), [global, {return, binary}]), - V2 = re:replace( V1, ?PH_S_USERNAME + V2 = re:replace( V1, emqx_authz:ph_to_re(?PH_S_USERNAME) , bin(Username), [global, {return, binary}]), - V3 = re:replace( V2, ?PH_S_HOST + V3 = re:replace( V2, emqx_authz:ph_to_re(?PH_S_HOST) , inet_parse:ntoa(IpAddress), [global, {return, binary}]), maps:put(K, V3, AccIn); InFun(K, V, AccIn) -> maps:put(K, V, AccIn) diff --git a/apps/emqx_authz/src/emqx_authz_redis.erl b/apps/emqx_authz/src/emqx_authz_redis.erl index 50e8c9a7d..8fa1e94c3 100644 --- a/apps/emqx_authz/src/emqx_authz_redis.erl +++ b/apps/emqx_authz/src/emqx_authz_redis.erl @@ -71,8 +71,9 @@ replvar(Cmd, Client = #{username := Username}) -> replvar(Cmd, _) -> Cmd. -repl(S, _Var, undefined) -> +repl(S, _VarPH, undefined) -> S; -repl(S, Var, Val) -> - NVal = re:replace(Val, "&", "\\\\&", [global, {return, list}]), - re:replace(S, Var, NVal, [{return, list}]). +repl(S, VarPH, Val) -> + NVal = re:replace(Val, "&", "\\\\&", [global, {return, list}]), + NVarPH = emqx_authz:ph_to_re(VarPH), + re:replace(S, NVarPH, NVal, [{return, list}]). From 1d3558ebe0235c40c351a3e1055b2dfd71ab0603 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 26 Nov 2021 15:38:18 +0800 Subject: [PATCH 02/72] style: make elvis happy --- apps/emqx/src/emqx_topic.erl | 34 ++++++++++++------------ apps/emqx/test/emqx_topic_SUITE.erl | 6 +++-- apps/emqx_authz/src/emqx_authz.erl | 41 +++++++++++++++++++++-------- 3 files changed, 51 insertions(+), 30 deletions(-) diff --git a/apps/emqx/src/emqx_topic.erl b/apps/emqx/src/emqx_topic.erl index 5e0f2d43b..a58c179b7 100644 --- a/apps/emqx/src/emqx_topic.erl +++ b/apps/emqx/src/emqx_topic.erl @@ -54,11 +54,11 @@ wildcard(Topic) when is_binary(Topic) -> wildcard(words(Topic)); wildcard([]) -> false; -wildcard(['#'|_]) -> +wildcard(['#' | _]) -> true; -wildcard(['+'|_]) -> +wildcard(['+' | _]) -> true; -wildcard([_H|T]) -> +wildcard([_H | T]) -> wildcard(T). %% @doc Match Topic name with filter. @@ -73,17 +73,17 @@ match(Name, Filter) when is_binary(Name), is_binary(Filter) -> match(words(Name), words(Filter)); match([], []) -> true; -match([H|T1], [H|T2]) -> +match([H | T1], [H | T2]) -> match(T1, T2); -match([_H|T1], ['+'|T2]) -> +match([_H | T1], ['+' | T2]) -> match(T1, T2); match(_, ['#']) -> true; -match([_H1|_], [_H2|_]) -> +match([_H1 | _], [_H2 | _]) -> false; -match([_H1|_], []) -> +match([_H1 | _], []) -> false; -match([], [_H|_T2]) -> +match([], [_H | _T2]) -> false. %% @doc Validate topic name or filter @@ -110,13 +110,13 @@ validate2([]) -> true; validate2(['#']) -> % end with '#' true; -validate2(['#'|Words]) when length(Words) > 0 -> +validate2(['#' | Words]) when length(Words) > 0 -> error('topic_invalid_#'); -validate2([''|Words]) -> +validate2(['' | Words]) -> validate2(Words); -validate2(['+'|Words]) -> +validate2(['+' | Words]) -> validate2(Words); -validate2([W|Words]) -> +validate2([W | Words]) -> validate3(W) andalso validate2(Words). validate3(<<>>) -> @@ -164,7 +164,7 @@ word(<<"#">>) -> '#'; word(Bin) -> Bin. %% @doc '$SYS' Topic. --spec(systop(atom()|string()|binary()) -> topic()). +-spec(systop(atom() | string() | binary()) -> topic()). systop(Name) when is_atom(Name); is_list(Name) -> iolist_to_binary(lists:concat(["$SYS/brokers/", node(), "/", Name])); systop(Name) when is_binary(Name) -> @@ -175,10 +175,10 @@ feed_var(Var, Val, Topic) -> feed_var(Var, Val, words(Topic), []). feed_var(_Var, _Val, [], Acc) -> join(lists:reverse(Acc)); -feed_var(Var, Val, [Var|Words], Acc) -> - feed_var(Var, Val, Words, [Val|Acc]); -feed_var(Var, Val, [W|Words], Acc) -> - feed_var(Var, Val, Words, [W|Acc]). +feed_var(Var, Val, [Var | Words], Acc) -> + feed_var(Var, Val, Words, [Val | Acc]); +feed_var(Var, Val, [W | Words], Acc) -> + feed_var(Var, Val, Words, [W | Acc]). -spec(join(list(binary())) -> binary()). join([]) -> diff --git a/apps/emqx/test/emqx_topic_SUITE.erl b/apps/emqx/test/emqx_topic_SUITE.erl index e0d80c4a5..49108f83e 100644 --- a/apps/emqx/test/emqx_topic_SUITE.erl +++ b/apps/emqx/test/emqx_topic_SUITE.erl @@ -184,9 +184,11 @@ t_feed_var(_) -> ?assertEqual(<<"$queue/client/clientId">>, feed_var(<<"$c">>, <<"clientId">>, <<"$queue/client/$c">>)), ?assertEqual(<<"username/test/client/x">>, - feed_var(?PH_USERNAME, <<"test">>, <<"username/", ?PH_USERNAME/binary, "/client/x">>)), + feed_var( ?PH_USERNAME, <<"test">> + , <<"username/", ?PH_USERNAME/binary, "/client/x">>)), ?assertEqual(<<"username/test/client/clientId">>, - feed_var(?PH_CLIENTID, <<"clientId">>, <<"username/test/client/", ?PH_CLIENTID/binary>>)). + feed_var( ?PH_CLIENTID, <<"clientId">> + , <<"username/test/client/", ?PH_CLIENTID/binary>>)). long_topic() -> iolist_to_binary([[integer_to_list(I), "/"] || I <- lists:seq(0, 66666)]). diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index a5efbe8f7..2bcb7971d 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -66,11 +66,14 @@ move(Type, Cmd) -> move(Type, Cmd, #{}). move(Type, #{<<"before">> := Before}, Opts) -> - emqx:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))}, Opts); + emqx:update_config( ?CONF_KEY_PATH + , {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))}, Opts); move(Type, #{<<"after">> := After}, Opts) -> - emqx:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_AFTER(type(After))}, Opts); + emqx:update_config( ?CONF_KEY_PATH + , {?CMD_MOVE, type(Type), ?CMD_MOVE_AFTER(type(After))}, Opts); move(Type, Position, Opts) -> - emqx:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), Position}, Opts). + emqx:update_config( ?CONF_KEY_PATH + , {?CMD_MOVE, type(Type), Position}, Opts). update(Cmd, Sources) -> update(Cmd, Sources, #{}). @@ -157,7 +160,8 @@ do_post_update({{?CMD_REPLACE, Type}, Source}, _NewSources) when is_map(Source) {OldSource, Front, Rear} = take(Type, OldInitedSources), ok = ensure_resource_deleted(OldSource), InitedSources = init_sources(check_sources([Source])), - ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Front ++ InitedSources ++ Rear]}, -1), + ok = emqx_hooks:put( 'client.authorize' + , {?MODULE, authorize, [Front ++ InitedSources ++ Rear]}, -1), ok = emqx_authz_cache:drain_cache(); do_post_update({{?CMD_DELETE, Type}, _Source}, _NewSources) -> OldInitedSources = lookup(), @@ -269,7 +273,7 @@ init_source(#{type := DB, {error, Reason} -> error({load_config_error, Reason}); Id -> Source#{annotations => #{id => Id, - query => Mod:parse_query(SQL) + query => erlang:apply(Mod, parse_query, [SQL]) } } end. @@ -279,22 +283,36 @@ init_source(#{type := DB, %%-------------------------------------------------------------------- %% @doc Check AuthZ --spec(authorize(emqx_types:clientinfo(), emqx_types:all(), emqx_types:topic(), allow | deny, sources()) +-spec(authorize( emqx_types:clientinfo() + , emqx_types:all() + , emqx_types:topic() + , allow | deny + , sources()) -> {stop, allow} | {ok, deny}). authorize(#{username := Username, peerhost := IpAddress } = Client, PubSub, Topic, DefaultResult, Sources) -> case do_authorize(Client, PubSub, Topic, Sources) of {matched, allow} -> - ?SLOG(info, #{msg => "authorization_permission_allowed", username => Username, ipaddr => IpAddress, topic => Topic}), + ?SLOG(info, #{msg => "authorization_permission_allowed", + username => Username, + ipaddr => IpAddress, + topic => Topic}), emqx_metrics:inc(?AUTHZ_METRICS(allow)), {stop, allow}; {matched, deny} -> - ?SLOG(info, #{msg => "authorization_permission_denied", username => Username, ipaddr => IpAddress, topic => Topic}), + ?SLOG(info, #{msg => "authorization_permission_denied", + username => Username, + ipaddr => IpAddress, + topic => Topic}), emqx_metrics:inc(?AUTHZ_METRICS(deny)), {stop, deny}; nomatch -> - ?SLOG(info, #{msg => "authorization_failed_nomatch", username => Username, ipaddr => IpAddress, topic => Topic, reason => "no-match rule"}), + ?SLOG(info, #{msg => "authorization_failed_nomatch", + username => Username, + ipaddr => IpAddress, + topic => Topic, + reason => "no-match rule"}), {stop, DefaultResult} end. @@ -311,7 +329,7 @@ do_authorize(Client, PubSub, Topic, [#{type := file} = F | Tail]) -> do_authorize(Client, PubSub, Topic, [Connector = #{type := Type} | Tail] ) -> Mod = authz_module(Type), - case Mod:authorize(Client, PubSub, Topic, Connector) of + case erlang:apply(Mod, authorize, [Client, PubSub, Topic, Connector]) of nomatch -> do_authorize(Client, PubSub, Topic, Tail); Matched -> Matched end. @@ -383,7 +401,8 @@ type(postgresql) -> postgresql; type(<<"postgresql">>) -> postgresql; type('built-in-database') -> 'built-in-database'; type(<<"built-in-database">>) -> 'built-in-database'; -type(Unknown) -> error({unknown_authz_source_type, Unknown}). % should never happend if the input is type-checked by hocon schema +%% should never happend if the input is type-checked by hocon schema +type(Unknown) -> error({unknown_authz_source_type, Unknown}). %% @doc where the acl.conf file is stored. acl_conf_file() -> From 726e25d6aebae9c298742ac751222efb39ca973a Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 26 Nov 2021 16:12:49 +0300 Subject: [PATCH 03/72] chore(authn): add JWKS backend tests --- .../emqx_authn_jwks_connector.erl | 55 ++++++------ .../src/simple_authn/emqx_authn_jwt.erl | 23 ++--- apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl | 90 +++++++++++++++++-- 3 files changed, 128 insertions(+), 40 deletions(-) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl index d8ceb7f40..3fc4bac13 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl @@ -20,6 +20,8 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("jose/include/jose_jwk.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + -export([ start_link/1 , stop/1 @@ -66,9 +68,9 @@ init([Opts]) -> handle_call(get_cached_jwks, _From, #{jwks := Jwks} = State) -> {reply, {ok, Jwks}, State}; -handle_call({update, Opts}, _From, State) -> - State = handle_options(Opts), - {reply, ok, refresh_jwks(State)}; +handle_call({update, Opts}, _From, _State) -> + NewState = handle_options(Opts), + {reply, ok, refresh_jwks(NewState)}; handle_call(_Req, _From, State) -> {reply, ok, State}. @@ -91,25 +93,27 @@ handle_info({refresh_jwks, _TRef, refresh}, #{request_id := RequestID} = State) handle_info({http, {RequestID, Result}}, #{request_id := RequestID, endpoint := Endpoint} = State0) -> + ?tp(debug, jwks_endpoint_response, #{request_id => RequestID}), State1 = State0#{request_id := undefined}, - case Result of - {error, Reason} -> - ?SLOG(warning, #{msg => "failed_to_request_jwks_endpoint", - endpoint => Endpoint, - reason => Reason}), - State1; - {_StatusLine, _Headers, Body} -> - try - JWKS = jose_jwk:from(emqx_json:decode(Body, [return_maps])), - {_, JWKs} = JWKS#jose_jwk.keys, - State1#{jwks := JWKs} - catch _:_ -> - ?SLOG(warning, #{msg => "invalid_jwks_returned", - endpoint => Endpoint, - body => Body}), - State1 - end - end; + NewState = case Result of + {error, Reason} -> + ?SLOG(warning, #{msg => "failed_to_request_jwks_endpoint", + endpoint => Endpoint, + reason => Reason}), + State1; + {_StatusLine, _Headers, Body} -> + try + JWKS = jose_jwk:from(emqx_json:decode(Body, [return_maps])), + {_, JWKs} = JWKS#jose_jwk.keys, + State1#{jwks := JWKs} + catch _:_ -> + ?SLOG(warning, #{msg => "invalid_jwks_returned", + endpoint => Endpoint, + body => Body}), + State1 + end + end, + {noreply, NewState}; handle_info({http, {_, _}}, State) -> %% ignore @@ -147,17 +151,18 @@ refresh_jwks(#{endpoint := Endpoint, NState = case httpc:request(get, {Endpoint, [{"Accept", "application/json"}]}, HTTPOpts, [{body_format, binary}, {sync, false}, {receiver, self()}]) of {error, Reason} -> - ?SLOG(warning, #{msg => "failed_to_request_jwks_endpoint", - endpoint => Endpoint, - reason => Reason}), + ?tp(warning, jwks_endpoint_request_fail, #{endpoint => Endpoint, + http_opts => HTTPOpts, + reason => Reason}), State; {ok, RequestID} -> + ?tp(debug, jwks_endpoint_request_ok, #{request_id => RequestID}), State#{request_id := RequestID} end, ensure_expiry_timer(NState). ensure_expiry_timer(State = #{refresh_interval := Interval}) -> - State#{refresh_timer := emqx_misc:start_timer(timer:seconds(Interval), refresh_jwks)}. + State#{refresh_timer => emqx_misc:start_timer(timer:seconds(Interval), refresh_jwks)}. cancel_timer(State = #{refresh_timer := undefined}) -> State; diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 7ec7eac6d..916911ffa 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -157,7 +157,7 @@ update(#{use_jwks := false} = Config, _State) -> update(#{use_jwks := true} = Config, #{jwk := Connector} = State) when is_pid(Connector) -> - ok = emqx_authn_jwks_connector:update(Connector, Config), + ok = emqx_authn_jwks_connector:update(Connector, connector_opts(Config)), case maps:get(verify_cliams, Config, undefined) of undefined -> {ok, State}; @@ -208,7 +208,7 @@ create2(#{use_jwks := false, JWK = jose_jwk:from_oct(Secret), {ok, #{jwk => JWK, verify_claims => VerifyClaims}} - end; + end; create2(#{use_jwks := false, algorithm := 'public-key', @@ -219,13 +219,8 @@ create2(#{use_jwks := false, verify_claims => VerifyClaims}}; create2(#{use_jwks := true, - verify_claims := VerifyClaims, - ssl := #{enable := Enable} = SSL} = Config) -> - SSLOpts = case Enable of - true -> maps:without([enable], SSL); - false -> #{} - end, - case emqx_authn_jwks_connector:start_link(Config#{ssl_opts => SSLOpts}) of + verify_claims := VerifyClaims} = Config) -> + case emqx_authn_jwks_connector:start_link(connector_opts(Config)) of {ok, Connector} -> {ok, #{jwk => Connector, verify_claims => VerifyClaims}}; @@ -233,6 +228,14 @@ create2(#{use_jwks := true, {error, Reason} end. +connector_opts(#{ssl := #{enable := Enable} = SSL} = Config) -> + SSLOpts = case Enable of + true -> maps:without([enable], SSL); + false -> #{} + end, + Config#{ssl_opts => SSLOpts}. + + may_decode_secret(false, Secret) -> Secret; may_decode_secret(true, Secret) -> try base64:decode(Secret) @@ -260,7 +263,7 @@ verify(JWS, [JWK | More], VerifyClaims) -> Claims = emqx_json:decode(Payload, [return_maps]), case verify_claims(Claims, VerifyClaims) of ok -> - {ok, #{is_superuser => maps:get(<<"is_superuser">>, Claims, false)}}; + {ok, emqx_authn_utils:is_superuser(Claims)}; {error, Reason} -> {error, Reason} end; diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl index 54db0a3c5..fe730acb0 100644 --- a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -21,11 +21,16 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include("emqx_authn.hrl"). -define(AUTHN_ID, <<"mechanism:jwt">>). +-define(JWKS_PORT, 33333). +-define(JWKS_PATH, "/jwks.json"). + + all() -> emqx_common_test_helpers:all(?MODULE). @@ -37,7 +42,11 @@ end_per_suite(_) -> emqx_common_test_helpers:stop_apps([emqx_authn]), ok. -t_jwt_authenticator(_) -> +%%------------------------------------------------------------------------------ +%% Tests +%%------------------------------------------------------------------------------ + +t_jwt_authenticator_hmac_based(_) -> Secret = <<"abcdef">>, Config = #{mechanism => jwt, use_jwks => false, @@ -121,10 +130,9 @@ t_jwt_authenticator(_) -> ?assertEqual(ok, emqx_authn_jwt:destroy(State3)), ok. -t_jwt_authenticator2(_) -> - Dir = code:lib_dir(emqx_authn, test), - PublicKey = list_to_binary(filename:join([Dir, "data/public_key.pem"])), - PrivateKey = list_to_binary(filename:join([Dir, "data/private_key.pem"])), +t_jwt_authenticator_public_key(_) -> + PublicKey = test_rsa_key(public), + PrivateKey = test_rsa_key(private), Config = #{mechanism => jwt, use_jwks => false, algorithm => 'public-key', @@ -142,6 +150,78 @@ t_jwt_authenticator2(_) -> ?assertEqual(ok, emqx_authn_jwt:destroy(State)), ok. +t_jwks_renewal(_Config) -> + ok = emqx_authn_http_test_server:start(?JWKS_PORT, ?JWKS_PATH), + ok = emqx_authn_http_test_server:set_handler(fun jwks_handler/2), + + PrivateKey = test_rsa_key(private), + Payload = #{<<"username">> => <<"myuser">>}, + JWS = generate_jws('public-key', Payload, PrivateKey), + Credential = #{username => <<"myuser">>, + password => JWS}, + + BadConfig = #{mechanism => jwt, + algorithm => 'public-key', + ssl => #{enable => false}, + verify_claims => [], + + use_jwks => true, + endpoint => "http://127.0.0.1:" ++ integer_to_list(?JWKS_PORT + 1) ++ ?JWKS_PATH, + refresh_interval => 1000 + }, + + ok = snabbkaffe:start_trace(), + + {{ok, State0}, _} = ?wait_async_action( + emqx_authn_jwt:create(?AUTHN_ID, BadConfig), + #{?snk_kind := jwks_endpoint_response}, + 1000), + + ok = snabbkaffe:stop(), + + ?assertEqual(ignore, emqx_authn_jwt:authenticate(Credential, State0)), + ?assertEqual(ignore, emqx_authn_jwt:authenticate(Credential#{password => <<"badpassword">>}, State0)), + + GoodConfig = BadConfig#{endpoint => + "http://127.0.0.1:" ++ integer_to_list(?JWKS_PORT) ++ ?JWKS_PATH}, + + ok = snabbkaffe:start_trace(), + + {{ok, State1}, _} = ?wait_async_action( + emqx_authn_jwt:update(GoodConfig, State0), + #{?snk_kind := jwks_endpoint_response}, + 1000), + + ok = snabbkaffe:stop(), + + ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential, State1)), + ?assertEqual(ignore, emqx_authn_jwt:authenticate(Credential#{password => <<"badpassword">>}, State1)), + + ?assertEqual(ok, emqx_authn_jwt:destroy(State1)), + ok = emqx_authn_http_test_server:stop(). + +%%------------------------------------------------------------------------------ +%% Helpers +%%------------------------------------------------------------------------------ + +jwks_handler(Req0, State) -> + JWK = jose_jwk:from_pem_file(test_rsa_key(public)), + JWKS = jose_jwk_set:to_map([JWK], #{}), + Req = cowboy_req:reply( + 200, + #{<<"content-type">> => <<"application/json">>}, + jiffy:encode(JWKS), + Req0), + {ok, Req, State}. + +test_rsa_key(public) -> + Dir = code:lib_dir(emqx_authn, test), + list_to_binary(filename:join([Dir, "data/public_key.pem"])); + +test_rsa_key(private) -> + Dir = code:lib_dir(emqx_authn, test), + list_to_binary(filename:join([Dir, "data/private_key.pem"])). + generate_jws('hmac-based', Payload, Secret) -> JWK = jose_jwk:from_oct(Secret), Header = #{ <<"alg">> => <<"HS256">> From e711143724589b7205c3399fc87ef7958cba244c Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Wed, 17 Nov 2021 12:29:10 +0100 Subject: [PATCH 04/72] chore(mria): Update dependency to 0.1.3 This version introduces rocksdb backend --- rebar.config | 1 + 1 file changed, 1 insertion(+) diff --git a/rebar.config b/rebar.config index fd4b1aa1f..9e7aba87c 100644 --- a/rebar.config +++ b/rebar.config @@ -52,6 +52,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.3"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}} + , {mria, {git, "https://github.com/emqx/mria", {tag, "0.1.3"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.7"}}} From 4580c03ebcb32d8fceb3bb7eb6cd41b71e3781c5 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 29 Nov 2021 20:44:38 +0300 Subject: [PATCH 05/72] chore(authn): add SCRAM mechanism tests --- apps/emqx/src/emqx_frame.erl | 4 +- .../emqx_enhanced_authn_scram_mnesia.erl | 23 +- .../test/emqx_authn_mnesia_SUITE.erl | 2 +- .../test/emqx_authn_mqtt_test_client.erl | 115 ++++++ ...emqx_enhanced_authn_scram_mnesia_SUITE.erl | 373 ++++++++++++++++++ 5 files changed, 505 insertions(+), 12 deletions(-) create mode 100644 apps/emqx_authn/test/emqx_authn_mqtt_test_client.erl create mode 100644 apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl diff --git a/apps/emqx/src/emqx_frame.erl b/apps/emqx/src/emqx_frame.erl index 2fe1b6d1a..647d076d1 100644 --- a/apps/emqx/src/emqx_frame.erl +++ b/apps/emqx/src/emqx_frame.erl @@ -248,8 +248,8 @@ parse_packet(#mqtt_packet_header{type = ?CONNECT}, FrameBin, _Options) -> }, {ConnPacket1, Rest5} = parse_will_message(ConnPacket, Rest4), {Username, Rest6} = parse_utf8_string(Rest5, bool(UsernameFlag)), - {Passsword, <<>>} = parse_utf8_string(Rest6, bool(PasswordFlag)), - ConnPacket1#mqtt_packet_connect{username = Username, password = Passsword}; + {Password, <<>>} = parse_utf8_string(Rest6, bool(PasswordFlag)), + ConnPacket1#mqtt_packet_connect{username = Username, password = Password}; parse_packet(#mqtt_packet_header{type = ?CONNACK}, <>, #{version := Ver}) -> diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl index 5a477c7e0..f02655462 100644 --- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl +++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl @@ -137,10 +137,7 @@ authenticate(_Credential, _State) -> ignore. destroy(#{user_group := UserGroup}) -> - MatchSpec = ets:fun2ms( - fun(#user_info{user_id = {Group, _}} = User) when Group =:= UserGroup -> - User - end), + MatchSpec = group_match_spec(UserGroup), trans( fun() -> ok = lists:foreach(fun(UserInfo) -> @@ -205,16 +202,16 @@ lookup_user(UserID, #{user_group := UserGroup}) -> end. list_users(PageParams, #{user_group := UserGroup}) -> - MatchSpec = [{{user_info, {UserGroup, '_'}, '_', '_', '_', '_'}, [], ['$_']}], + MatchSpec = group_match_spec(UserGroup), {ok, emqx_mgmt_api:paginate(?TAB, MatchSpec, PageParams, ?FORMAT_FUN)}. %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ -ensure_auth_method('SCRAM-SHA-256', #{algorithm := sha256}) -> +ensure_auth_method(<<"SCRAM-SHA-256">>, #{algorithm := sha256}) -> true; -ensure_auth_method('SCRAM-SHA-512', #{algorithm := sha512}) -> +ensure_auth_method(<<"SCRAM-SHA-512">>, #{algorithm := sha512}) -> true; ensure_auth_method(_, _) -> false. @@ -228,8 +225,10 @@ check_client_first_message(Bin, _Cache, #{iteration_count := IterationCount} = S #{iteration_count => IterationCount, retrieve => RetrieveFun} ) of - {cotinue, ServerFirstMessage, Cache} -> - {cotinue, ServerFirstMessage, Cache}; + {continue, ServerFirstMessage, Cache} -> + {continue, ServerFirstMessage, Cache}; + ignore -> + ignore; {error, _Reason} -> {error, not_authorized} end. @@ -280,3 +279,9 @@ trans(Fun, Args) -> format_user_info(#user_info{user_id = {_, UserID}, is_superuser = IsSuperuser}) -> #{user_id => UserID, is_superuser => IsSuperuser}. + +group_match_spec(UserGroup) -> + ets:fun2ms( + fun(#user_info{user_id = {Group, _}} = User) when Group =:= UserGroup -> + User + end). diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index b5bca513c..b9eadbef6 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -37,7 +37,7 @@ end_per_suite(_) -> ok. init_per_testcase(_Case, Config) -> - mnesia:clear_table(emqx_authn_mnesia), + mria:clear_table(emqx_authn_mnesia), Config. end_per_testcase(_Case, Config) -> diff --git a/apps/emqx_authn/test/emqx_authn_mqtt_test_client.erl b/apps/emqx_authn/test/emqx_authn_mqtt_test_client.erl new file mode 100644 index 000000000..a217cb96d --- /dev/null +++ b/apps/emqx_authn/test/emqx_authn_mqtt_test_client.erl @@ -0,0 +1,115 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_mqtt_test_client). + +-behaviour(gen_server). + +-include_lib("emqx/include/emqx_mqtt.hrl"). + +%% API +-export([start_link/2, + stop/1]). + +-export([send/2]). + +%% gen_server callbacks + +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2]). + +-define(TIMEOUT, 1000). +-define(TCP_OPTIONS, [binary, {packet, raw}, {active, once}, + {nodelay, true}]). + +-define(PARSE_OPTIONS, + #{strict_mode => false, + max_size => ?MAX_PACKET_SIZE, + version => ?MQTT_PROTO_V5 + }). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +start_link(Host, Port) -> + gen_server:start_link(?MODULE, [Host, Port, self()], []). + +stop(Pid) -> + gen_server:call(Pid, stop). + +send(Pid, Packet) -> + gen_server:call(Pid, {send, Packet}). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([Host, Port, Owner]) -> + {ok, Socket} = gen_tcp:connect(Host, Port, ?TCP_OPTIONS, ?TIMEOUT), + {ok, #{owner => Owner, + socket => Socket, + parse_state => emqx_frame:initial_parse_state(?PARSE_OPTIONS) + }}. + +handle_info({tcp, _Sock, Data}, #{parse_state := PSt, + owner := Owner, + socket := Socket} = St) -> + {NewPSt, Packets} = process_incoming(PSt, Data, []), + ok = deliver(Owner, Packets), + ok = run_sock(Socket), + {noreply, St#{parse_state => NewPSt}}; + +handle_info({tcp_closed, _Sock}, St) -> + {stop, normal, St}. + +handle_call({send, Packet}, _From, #{socket := Socket} = St) -> + ok = gen_tcp:send(Socket, emqx_frame:serialize(Packet, ?MQTT_PROTO_V5)), + {reply, ok, St}; + +handle_call(stop, _From, #{socket := Socket} = St) -> + ok = gen_tcp:close(Socket), + {stop, normal, ok, St}. + +handle_cast(_, St) -> + {noreply, St}. + +terminate(_Reason, _St) -> + ok. + +%%-------------------------------------------------------------------- +%% internal functions +%%-------------------------------------------------------------------- + +process_incoming(PSt, Data, Packets) -> + case emqx_frame:parse(Data, PSt) of + {more, NewPSt} -> + {NewPSt, lists:reverse(Packets)}; + {ok, Packet, Rest, NewPSt} -> + process_incoming(NewPSt, Rest, [Packet | Packets]) + end. + +deliver(_Owner, []) -> ok; +deliver(Owner, [Packet | Packets]) -> + Owner ! {packet, Packet}, + deliver(Owner, Packets). + + +run_sock(Socket) -> + inet:setopts(Socket, [{active, once}]). diff --git a/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl new file mode 100644 index 000000000..3504b88cc --- /dev/null +++ b/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl @@ -0,0 +1,373 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_enhanced_authn_scram_mnesia_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include("emqx_authn.hrl"). + +-define(PATH, [authentication]). + +-define(USER_MAP, #{user_id := _, + is_superuser := _}). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_authn]), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_authn]). + +init_per_testcase(_Case, Config) -> + mria:clear_table(emqx_enhanced_authn_scram_mnesia), + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL), + Config. + +end_per_testcase(_Case, Config) -> + Config. + +%%------------------------------------------------------------------------------ +%% Tests +%%------------------------------------------------------------------------------ + +t_create(_Config) -> + ValidConfig = #{ + <<"mechanism">> => <<"scram">>, + <<"backend">> => <<"built-in-database">>, + <<"algorithm">> => <<"sha512">>, + <<"iteration_count">> => <<"4096">> + }, + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, ValidConfig}), + + {ok, [#{provider := emqx_enhanced_authn_scram_mnesia}]} + = emqx_authentication:list_authenticators(?GLOBAL). + +t_create_invalid(_Config) -> + InvalidConfig = #{ + <<"mechanism">> => <<"scram">>, + <<"backend">> => <<"built-in-database">>, + <<"algorithm">> => <<"sha271828">>, + <<"iteration_count">> => <<"4096">> + }, + + {error, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, InvalidConfig}), + + {ok, []} = emqx_authentication:list_authenticators(?GLOBAL). + +t_authenticate(_Config) -> + Algorithm = sha512, + Username = <<"u">>, + Password = <<"p">>, + + init_auth(Username, Password, Algorithm), + + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ClientFirstMessage = esasl_scram:client_first_message(Username), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFirstMessage + } + }), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{'Authentication-Data' := ServerFirstMessage}) = receive_packet(), + + {continue, ClientFinalMessage, ClientCache} = + esasl_scram:check_server_first_message( + ServerFirstMessage, + #{client_first_message => ClientFirstMessage, + password => Password, + algorithm => Algorithm} + ), + + AuthContinuePacket = ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFinalMessage}), + + ok = emqx_authn_mqtt_test_client:send(Pid, AuthContinuePacket), + + ?CONNACK_PACKET( + ?RC_SUCCESS, + _, + #{'Authentication-Data' := ServerFinalMessage}) = receive_packet(), + + ok = esasl_scram:check_server_final_message( + ServerFinalMessage, ClientCache#{algorithm => Algorithm} + ). + +t_authenticate_bad_username(_Config) -> + Algorithm = sha512, + Username = <<"u">>, + Password = <<"p">>, + + init_auth(Username, Password, Algorithm), + + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ClientFirstMessage = esasl_scram:client_first_message(<<"badusername">>), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFirstMessage + } + }), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + ?CONNACK_PACKET(?RC_NOT_AUTHORIZED) = receive_packet(). + +t_authenticate_bad_password(_Config) -> + Algorithm = sha512, + Username = <<"u">>, + Password = <<"p">>, + + init_auth(Username, Password, Algorithm), + + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ClientFirstMessage = esasl_scram:client_first_message(Username), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFirstMessage + } + }), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{'Authentication-Data' := ServerFirstMessage}) = receive_packet(), + + {continue, ClientFinalMessage, _ClientCache} = + esasl_scram:check_server_first_message( + ServerFirstMessage, + #{client_first_message => ClientFirstMessage, + password => <<"badpassword">>, + algorithm => Algorithm} + ), + + AuthContinuePacket = ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFinalMessage}), + + ok = emqx_authn_mqtt_test_client:send(Pid, AuthContinuePacket), + + ?CONNACK_PACKET(?RC_NOT_AUTHORIZED) = receive_packet(). + +t_destroy(_) -> + Config = config(), + OtherId = list_to_binary([<<"id-other">>]), + {ok, State0} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config), + {ok, StateOther} = emqx_enhanced_authn_scram_mnesia:create(OtherId, Config), + + User = #{user_id => <<"u">>, password => <<"p">>}, + + {ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(User, State0), + {ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(User, StateOther), + + {ok, _} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, State0), + {ok, _} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, StateOther), + + ok = emqx_enhanced_authn_scram_mnesia:destroy(State0), + + {ok, State1} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config), + {error,not_found} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, State1), + {ok, _} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, StateOther). + +t_add_user(_) -> + Config = config(), + {ok, State} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config), + + User = #{user_id => <<"u">>, password => <<"p">>}, + {ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(User, State), + {error, already_exist} = emqx_enhanced_authn_scram_mnesia:add_user(User, State). + +t_delete_user(_) -> + Config = config(), + {ok, State} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config), + + {error, not_found} = emqx_enhanced_authn_scram_mnesia:delete_user(<<"u">>, State), + User = #{user_id => <<"u">>, password => <<"p">>}, + {ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(User, State), + + ok = emqx_enhanced_authn_scram_mnesia:delete_user(<<"u">>, State), + {error, not_found} = emqx_enhanced_authn_scram_mnesia:delete_user(<<"u">>, State). + +t_update_user(_) -> + Config = config(), + {ok, State} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config), + + User = #{user_id => <<"u">>, password => <<"p">>}, + {ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(User, State), + {ok, #{is_superuser := false}} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, State), + + {ok, + #{user_id := <<"u">>, + is_superuser := true}} = emqx_enhanced_authn_scram_mnesia:update_user( + <<"u">>, + #{password => <<"p1">>, is_superuser => true}, + State), + + {ok, #{is_superuser := true}} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, State). + +t_list_users(_) -> + Config = config(), + {ok, State} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config), + + Users = [#{user_id => <<"u1">>, password => <<"p">>}, + #{user_id => <<"u2">>, password => <<"p">>}, + #{user_id => <<"u3">>, password => <<"p">>}], + + lists:foreach( + fun(U) -> {ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(U, State) end, + Users), + + {ok, + #{data := [?USER_MAP, ?USER_MAP], + meta := #{page := 1, limit := 2, count := 3}}} = emqx_enhanced_authn_scram_mnesia:list_users( + #{<<"page">> => 1, <<"limit">> => 2}, + State), + {ok, + #{data := [?USER_MAP], + meta := #{page := 2, limit := 2, count := 3}}} = emqx_enhanced_authn_scram_mnesia:list_users( + #{<<"page">> => 2, <<"limit">> => 2}, + State). + +t_is_superuser(_Config) -> + ok = test_is_superuser(#{is_superuser => false}, false), + ok = test_is_superuser(#{is_superuser => true}, true), + ok = test_is_superuser(#{}, false). + +test_is_superuser(UserInfo, ExpectedIsSuperuser) -> + Config = config(), + {ok, State} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config), + + Username = <<"u">>, + Password = <<"p">>, + + UserInfo0 = UserInfo#{user_id => Username, + password => Password}, + + {ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(UserInfo0, State), + + ClientFirstMessage = esasl_scram:client_first_message(Username), + + {continue, ServerFirstMessage, ServerCache} + = emqx_enhanced_authn_scram_mnesia:authenticate( + #{auth_method => <<"SCRAM-SHA-512">>, + auth_data => ClientFirstMessage, + auth_cache => #{} + }, + State), + + {continue, ClientFinalMessage, ClientCache} = + esasl_scram:check_server_first_message( + ServerFirstMessage, + #{client_first_message => ClientFirstMessage, + password => Password, + algorithm => sha512} + ), + + {ok, UserInfo1, ServerFinalMessage} + = emqx_enhanced_authn_scram_mnesia:authenticate( + #{auth_method => <<"SCRAM-SHA-512">>, + auth_data => ClientFinalMessage, + auth_cache => ServerCache + }, + State), + + ok = esasl_scram:check_server_final_message( + ServerFinalMessage, ClientCache#{algorithm => sha512} + ), + + ?assertMatch(#{is_superuser := ExpectedIsSuperuser}, UserInfo1), + + ok = emqx_enhanced_authn_scram_mnesia:destroy(State). + + +%%------------------------------------------------------------------------------ +%% Helpers +%%------------------------------------------------------------------------------ + +config() -> + #{ + mechanism => <<"scram">>, + backend => <<"built-in-database">>, + algorithm => sha512, + iteration_count => 4096 + }. + +raw_config(Algorithm) -> + #{ + <<"mechanism">> => <<"scram">>, + <<"backend">> => <<"built-in-database">>, + <<"algorithm">> => atom_to_binary(Algorithm), + <<"iteration_count">> => <<"4096">> + }. + +init_auth(Username, Password, Algorithm) -> + Config = raw_config(Algorithm), + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, Config}), + + {ok, [#{state := State}]} = emqx_authentication:list_authenticators(?GLOBAL), + + emqx_enhanced_authn_scram_mnesia:add_user( + #{user_id => Username, password => Password}, + State). + +receive_packet() -> + receive + {packet, Packet} -> + ct:pal("Delivered packet: ~p", [Packet]), + Packet + after 1000 -> + ct:fail("Deliver timeout") + end. From 390575eafb8a508a5c9e199164a35b1fb1bf30e7 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 22 Nov 2021 14:26:12 +0300 Subject: [PATCH 06/72] chore(authn): add MongoDB backend tests --- .ci/docker-compose-file/.env | 2 +- .ci/docker-compose-file/Makefile.local | 2 +- .../docker-compose-mongo-single-tcp.yaml | 4 +- .github/workflows/run_test_cases.yaml | 2 + apps/emqx_authn/src/emqx_authn_utils.erl | 2 + .../src/simple_authn/emqx_authn_mongodb.erl | 9 +- apps/emqx_authn/test/emqx_authn_SUITE.erl | 22 - .../test/emqx_authn_mongo_SUITE.erl | 409 ++++++++++++++++++ .../test/emqx_authn_mysql_SUITE.erl | 4 +- .../test/emqx_authn_pgsql_SUITE.erl | 4 +- .../test/emqx_authn_redis_SUITE.erl | 4 +- .../src/emqx_connector_mongo.erl | 3 +- 12 files changed, 429 insertions(+), 38 deletions(-) delete mode 100644 apps/emqx_authn/test/emqx_authn_SUITE.erl create mode 100644 apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env index 8c9d056cc..ae3d12c64 100644 --- a/.ci/docker-compose-file/.env +++ b/.ci/docker-compose-file/.env @@ -1,6 +1,6 @@ MYSQL_TAG=8 REDIS_TAG=6 -MONGO_TAG=4 +MONGO_TAG=5 PGSQL_TAG=13 LDAP_TAG=2.4.50 diff --git a/.ci/docker-compose-file/Makefile.local b/.ci/docker-compose-file/Makefile.local index d5ef99d66..096da64c5 100644 --- a/.ci/docker-compose-file/Makefile.local +++ b/.ci/docker-compose-file/Makefile.local @@ -14,7 +14,7 @@ up: env \ MYSQL_TAG=8 \ REDIS_TAG=6 \ - MONGO_TAG=4 \ + MONGO_TAG=5 \ PGSQL_TAG=13 \ LDAP_TAG=2.4.50 \ docker-compose \ diff --git a/.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml b/.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml index 494b42ce4..5bba6147c 100644 --- a/.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml +++ b/.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml @@ -2,11 +2,9 @@ version: '3.9' services: mongo_server: - container_name: mongo + container_name: mongo image: mongo:${MONGO_TAG} restart: always - environment: - MONGO_INITDB_DATABASE: mqtt networks: - emqx_bridge ports: diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index d1f8bf577..49e94c322 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -55,12 +55,14 @@ jobs: - uses: actions/checkout@v2 - name: docker compose up env: + MONGO_TAG: 5 MYSQL_TAG: 8 PGSQL_TAG: 13 REDIS_TAG: 6 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | docker-compose \ + -f .ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ diff --git a/apps/emqx_authn/src/emqx_authn_utils.erl b/apps/emqx_authn/src/emqx_authn_utils.erl index 56f485afc..2205d237d 100644 --- a/apps/emqx_authn/src/emqx_authn_utils.erl +++ b/apps/emqx_authn/src/emqx_authn_utils.erl @@ -93,6 +93,8 @@ is_superuser(#{<<"is_superuser">> := 0}) -> #{is_superuser => false}; is_superuser(#{<<"is_superuser">> := null}) -> #{is_superuser => false}; +is_superuser(#{<<"is_superuser">> := undefined}) -> + #{is_superuser => false}; is_superuser(#{<<"is_superuser">> := false}) -> #{is_superuser => false}; is_superuser(#{<<"is_superuser">> := _}) -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 40bd0c2c9..7e080dfee 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -115,6 +115,8 @@ create(#{selector := Selector} = Config) -> password_hash_algorithm, salt_position], Config), + #{password_hash_algorithm := Algorithm} = State, + ok = emqx_authn_utils:ensure_apps_started(Algorithm), ResourceId = emqx_authn_utils:make_resource_id(?MODULE), NState = State#{ selector => NSelector, @@ -155,7 +157,7 @@ authenticate(#{password := Password} = Credential, Doc -> case check_password(Password, Doc, State) of ok -> - {ok, #{is_superuser => is_superuser(Doc, State)}}; + {ok, is_superuser(Doc, State)}; {error, {cannot_find_password_hash_field, PasswordHashField}} -> ?SLOG(error, #{msg => "cannot_find_password_hash_field", resource => ResourceId, @@ -234,9 +236,8 @@ check_password(Password, end. is_superuser(Doc, #{is_superuser_field := IsSuperuserField}) -> - maps:get(IsSuperuserField, Doc, false); -is_superuser(_, _) -> - false. + IsSuperuser = maps:get(IsSuperuserField, Doc, false), + emqx_authn_utils:is_superuser(#{<<"is_superuser">> => IsSuperuser}). hash(Algorithm, Password, Salt, prefix) -> emqx_passwd:hash(Algorithm, <>); diff --git a/apps/emqx_authn/test/emqx_authn_SUITE.erl b/apps/emqx_authn/test/emqx_authn_SUITE.erl deleted file mode 100644 index d3704679f..000000000 --- a/apps/emqx_authn/test/emqx_authn_SUITE.erl +++ /dev/null @@ -1,22 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_authn_SUITE). - --compile(export_all). --compile(nowarn_export_all). - -all() -> emqx_common_test_helpers:all(?MODULE). diff --git a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl new file mode 100644 index 000000000..e6ae1706a --- /dev/null +++ b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl @@ -0,0 +1,409 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_mongo_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include("emqx_authn.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + + +-define(MONGO_HOST, "mongo"). +-define(MONGO_PORT, 27017). +-define(MONGO_CLIENT, 'emqx_authn_mongo_SUITE_client'). + +-define(PATH, [authentication]). + +all() -> + emqx_common_test_helpers:all(?MODULE). + + +init_per_testcase(_TestCase, Config) -> + emqx_authentication:initialize_authentication(?GLOBAL, []), + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL), + {ok, _} = mc_worker_api:connect(mongo_config()), + Config. + +end_per_testcase(_TestCase, _Config) -> + ok = mc_worker_api:disconnect(?MONGO_CLIENT). + + +init_per_suite(Config) -> + case emqx_authn_test_lib:is_tcp_server_available(?MONGO_HOST, ?MONGO_PORT) of + true -> + ok = emqx_common_test_helpers:start_apps([emqx_authn]), + ok = start_apps([emqx_resource, emqx_connector]), + Config; + false -> + {skip, no_mongo} + end. + +end_per_suite(_Config) -> + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL), + ok = stop_apps([emqx_resource, emqx_connector]), + ok = emqx_common_test_helpers:stop_apps([emqx_authn]). + +%%------------------------------------------------------------------------------ +%% Tests +%%------------------------------------------------------------------------------ + +t_create(_Config) -> + AuthConfig = raw_mongo_auth_config(), + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, AuthConfig}), + + {ok, [#{provider := emqx_authn_mongodb}]} = emqx_authentication:list_authenticators(?GLOBAL). + +t_create_invalid(_Config) -> + AuthConfig = raw_mongo_auth_config(), + + InvalidConfigs = + [ + AuthConfig#{mongo_type => <<"unknown">>}, + AuthConfig#{selector => <<"{ \"username\": \"${username}\" }">>} + ], + + lists:foreach( + fun(Config) -> + {error, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, Config}), + + {ok, []} = emqx_authentication:list_authenticators(?GLOBAL) + end, + InvalidConfigs). + +t_authenticate(_Config) -> + ok = init_seeds(), + ok = lists:foreach( + fun(Sample) -> + ct:pal("test_user_auth sample: ~p", [Sample]), + test_user_auth(Sample) + end, + user_seeds()), + ok = drop_seeds(). + +test_user_auth(#{credentials := Credentials0, + config_params := SpecificConfigParams, + result := Result}) -> + AuthConfig = maps:merge(raw_mongo_auth_config(), SpecificConfigParams), + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, AuthConfig}), + + Credentials = Credentials0#{ + listener => 'tcp:default', + protocol => mqtt + }, + ?assertEqual(Result, emqx_access_control:authenticate(Credentials)), + + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL). + +t_destroy(_Config) -> + ok = init_seeds(), + AuthConfig = raw_mongo_auth_config(), + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, AuthConfig}), + + {ok, [#{provider := emqx_authn_mongodb, state := State}]} + = emqx_authentication:list_authenticators(?GLOBAL), + + {ok, _} = emqx_authn_mongodb:authenticate( + #{username => <<"plain">>, + password => <<"plain">> + }, + State), + + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL), + + % Authenticator should not be usable anymore + ?assertException( + error, + _, + emqx_authn_mongodb:authenticate( + #{username => <<"plain">>, + password => <<"plain">> + }, + State)), + + ok = drop_seeds(). + +t_update(_Config) -> + ok = init_seeds(), + CorrectConfig = raw_mongo_auth_config(), + IncorrectConfig = + CorrectConfig#{selector => #{<<"wrongfield">> => <<"wrongvalue">>}}, + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, IncorrectConfig}), + + {error, not_authorized} = emqx_access_control:authenticate( + #{username => <<"plain">>, + password => <<"plain">>, + listener => 'tcp:default', + protocol => mqtt + }), + + % We update with config with correct selector, provider should update and work properly + {ok, _} = emqx:update_config( + ?PATH, + {update_authenticator, ?GLOBAL, <<"password-based:mongodb">>, CorrectConfig}), + + {ok,_} = emqx_access_control:authenticate( + #{username => <<"plain">>, + password => <<"plain">>, + listener => 'tcp:default', + protocol => mqtt + }), + ok = drop_seeds(). + +t_is_superuser(_Config) -> + Config = raw_mongo_auth_config(), + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, Config}), + + Checks = [ + {<<"0">>, false}, + {<<"">>, false}, + {null, false}, + {false, false}, + {0, false}, + + {<<"1">>, true}, + {<<"val">>, true}, + {1, true}, + {123, true}, + {true, true} + ], + + lists:foreach(fun test_is_superuser/1, Checks). + +test_is_superuser({Value, ExpectedValue}) -> + {true, _} = mc_worker_api:delete(?MONGO_CLIENT, <<"users">>, #{}), + + UserData = #{ + username => <<"user">>, + password_hash => <<"plainsalt">>, + salt => <<"salt">>, + is_superuser => Value + }, + + {{true, _}, _} = mc_worker_api:insert(?MONGO_CLIENT, <<"users">>, [UserData]), + + Credentials = #{ + listener => 'tcp:default', + protocol => mqtt, + username => <<"user">>, + password => <<"plain">> + }, + + ?assertEqual( + {ok, #{is_superuser => ExpectedValue}}, + emqx_access_control:authenticate(Credentials)). + +%%------------------------------------------------------------------------------ +%% Helpers +%%------------------------------------------------------------------------------ + +raw_mongo_auth_config() -> + #{ + mechanism => <<"password-based">>, + password_hash_algorithm => <<"plain">>, + salt_position => <<"suffix">>, + enable => <<"true">>, + + backend => <<"mongodb">>, + mongo_type => <<"single">>, + database => <<"mqtt">>, + collection => <<"users">>, + server => mongo_server(), + + selector => #{<<"username">> => <<"${username}">>}, + password_hash_field => <<"password_hash">>, + salt_field => <<"salt">>, + is_superuser_field => <<"is_superuser">> + }. + +user_seeds() -> + [#{data => #{ + username => <<"plain">>, + password_hash => <<"plainsalt">>, + salt => <<"salt">>, + is_superuser => <<"1">> + }, + credentials => #{ + username => <<"plain">>, + password => <<"plain">> + }, + config_params => #{ + }, + result => {ok,#{is_superuser => true}} + }, + + #{data => #{ + username => <<"md5">>, + password_hash => <<"9b4d0c43d206d48279e69b9ad7132e22">>, + salt => <<"salt">>, + is_superuser => <<"0">> + }, + credentials => #{ + username => <<"md5">>, + password => <<"md5">> + }, + config_params => #{ + password_hash_algorithm => <<"md5">>, + salt_position => <<"suffix">> + }, + result => {ok,#{is_superuser => false}} + }, + + #{data => #{ + username => <<"sha256">>, + password_hash => <<"ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf">>, + salt => <<"salt">>, + is_superuser => 1 + }, + credentials => #{ + clientid => <<"sha256">>, + password => <<"sha256">> + }, + config_params => #{ + selector => #{<<"username">> => <<"${clientid}">>}, + password_hash_algorithm => <<"sha256">>, + salt_position => <<"prefix">> + }, + result => {ok,#{is_superuser => true}} + }, + + #{data => #{ + username => <<"bcrypt">>, + password_hash => <<"$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u">>, + salt => <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>, + is_superuser => 0 + }, + credentials => #{ + username => <<"bcrypt">>, + password => <<"bcrypt">> + }, + config_params => #{ + password_hash_algorithm => <<"bcrypt">>, + salt_position => <<"suffix">> % should be ignored + }, + result => {ok,#{is_superuser => false}} + }, + + #{data => #{ + username => <<"bcrypt0">>, + password_hash => <<"$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u">>, + salt => <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>, + is_superuser => <<"0">> + }, + credentials => #{ + username => <<"bcrypt0">>, + password => <<"bcrypt">> + }, + config_params => #{ + % clientid variable & username credentials + selector => #{<<"username">> => <<"${clientid}">>}, + password_hash_algorithm => <<"bcrypt">>, + salt_position => <<"suffix">> + }, + result => {error,not_authorized} + }, + + #{data => #{ + username => <<"bcrypt1">>, + password_hash => <<"$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u">>, + salt => <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>, + is_superuser => <<"0">> + }, + credentials => #{ + username => <<"bcrypt1">>, + password => <<"bcrypt">> + }, + config_params => #{ + selector => #{<<"userid">> => <<"${clientid}">>}, + password_hash_algorithm => <<"bcrypt">>, + salt_position => <<"suffix">> + }, + result => {error,not_authorized} + }, + + #{data => #{ + username => <<"bcrypt2">>, + password_hash => <<"$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u">>, + salt => <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>, + is_superuser => <<"0">> + }, + credentials => #{ + username => <<"bcrypt2">>, + % Wrong password + password => <<"wrongpass">> + }, + config_params => #{ + password_hash_algorithm => <<"bcrypt">>, + salt_position => <<"suffix">> + }, + result => {error,bad_username_or_password} + } + ]. + +init_seeds() -> + Users = [Values || #{data := Values} <- user_seeds()], + {{true, _}, _} = mc_worker_api:insert(?MONGO_CLIENT, <<"users">>, Users), + ok. + +drop_seeds() -> + {true, _} = mc_worker_api:delete(?MONGO_CLIENT, <<"users">>, #{}), + ok. + +mongo_server() -> + iolist_to_binary( + io_lib:format( + "~s:~b", + [?MONGO_HOST, ?MONGO_PORT])). + +mongo_config() -> + [ + {database, <<"mqtt">>}, + {host, ?MONGO_HOST}, + {port, ?MONGO_PORT}, + {register, ?MONGO_CLIENT} + ]. + +start_apps(Apps) -> + lists:foreach(fun application:ensure_all_started/1, Apps). + +stop_apps(Apps) -> + lists:foreach(fun application:stop/1, Apps). diff --git a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl index 9073dd38a..48569ed36 100644 --- a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl @@ -117,9 +117,9 @@ t_authenticate(_Config) -> user_seeds()). test_user_auth(#{credentials := Credentials0, - config_params := SpecificConfgParams, + config_params := SpecificConfigParams, result := Result}) -> - AuthConfig = maps:merge(raw_mysql_auth_config(), SpecificConfgParams), + AuthConfig = maps:merge(raw_mysql_auth_config(), SpecificConfigParams), {ok, _} = emqx:update_config( ?PATH, diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl index 08bb2ee2e..e3848afbc 100644 --- a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl @@ -117,9 +117,9 @@ t_authenticate(_Config) -> user_seeds()). test_user_auth(#{credentials := Credentials0, - config_params := SpecificConfgParams, + config_params := SpecificConfigParams, result := Result}) -> - AuthConfig = maps:merge(raw_pgsql_auth_config(), SpecificConfgParams), + AuthConfig = maps:merge(raw_pgsql_auth_config(), SpecificConfigParams), {ok, _} = emqx:update_config( ?PATH, diff --git a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl index 8669080b0..be1ae6d13 100644 --- a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl @@ -124,9 +124,9 @@ t_authenticate(_Config) -> user_seeds()). test_user_auth(#{credentials := Credentials0, - config_params := SpecificConfgParams, + config_params := SpecificConfigParams, result := Result}) -> - AuthConfig = maps:merge(raw_redis_auth_config(), SpecificConfgParams), + AuthConfig = maps:merge(raw_redis_auth_config(), SpecificConfigParams), {ok, _} = emqx:update_config( ?PATH, diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index 11eac9c91..6cee46d75 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -181,12 +181,13 @@ health_check(PoolName) -> %% =================================================================== connect(Opts) -> - Type = proplists:get_value(mongo_type, Opts, single), + Type = proplists:get_value(type, Opts, single), Hosts = proplists:get_value(hosts, Opts, []), Options = proplists:get_value(options, Opts, []), WorkerOptions = proplists:get_value(worker_options, Opts, []), mongo_api:connect(Type, Hosts, Options, WorkerOptions). + mongo_query(Conn, find, Collection, Selector, Projector) -> mongo_api:find(Conn, Collection, Selector, Projector); From 0baec8e27d679f65cd46a777a82b0dd25730b05a Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 29 Nov 2021 21:39:28 +0300 Subject: [PATCH 07/72] chore(authn): unify Mongo type parameter naming --- apps/emqx_connector/src/emqx_connector_mongo.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index 6cee46d75..c121bb4c6 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -118,7 +118,7 @@ on_start(InstId, Config = #{mongo_type := Type, false -> [{ssl, false}] end, Topology = maps:get(topology, NConfig, #{}), - Opts = [{type, init_type(NConfig)}, + Opts = [{mongo_type, init_type(NConfig)}, {hosts, Hosts}, {pool_size, PoolSize}, {options, init_topology_options(maps:to_list(Topology), [])}, @@ -181,7 +181,7 @@ health_check(PoolName) -> %% =================================================================== connect(Opts) -> - Type = proplists:get_value(type, Opts, single), + Type = proplists:get_value(mongo_type, Opts, single), Hosts = proplists:get_value(hosts, Opts, []), Options = proplists:get_value(options, Opts, []), WorkerOptions = proplists:get_value(worker_options, Opts, []), From cfe2954a880c2e48dea79cef0f31bfc8d66ac96c Mon Sep 17 00:00:00 2001 From: lafirest Date: Tue, 30 Nov 2021 12:25:06 +0800 Subject: [PATCH 08/72] fix(emqx_gateway): fix the function_clause error when client disconnect after connection created --- apps/emqx_gateway/src/coap/emqx_coap_channel.erl | 12 +++++++----- apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl | 13 +++++++------ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl index 4c21e227b..d7b0a00c3 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl @@ -98,10 +98,10 @@ info(ctx, #channel{ctx = Ctx}) -> stats(_) -> []. -init(ConnInfo = #{peername := {PeerHost, _}, - sockname := {_, SockPort}}, +init(ConnInfoT = #{peername := {PeerHost, _}, + sockname := {_, SockPort}}, #{ctx := Ctx} = Config) -> - Peercert = maps:get(peercert, ConnInfo, undefined), + Peercert = maps:get(peercert, ConnInfoT, undefined), Mountpoint = maps:get(mountpoint, Config, <<>>), ListenerId = case maps:get(listener, Config, undefined) of undefined -> undefined; @@ -123,6 +123,10 @@ init(ConnInfo = #{peername := {PeerHost, _}, } ), + %% because it is possible to disconnect after init, and then trigger the $event.disconnected hook + %% and these two fields are required in the hook + ConnInfo = ConnInfoT#{proto_name => <<"CoAP">>, proto_ver => <<"1">>}, + Heartbeat = ?GET_IDLE_TIME(Config), #channel{ ctx = Ctx , conninfo = ConnInfo @@ -349,8 +353,6 @@ ensure_connected(Channel = #channel{ctx = Ctx, conninfo = ConnInfo, clientinfo = ClientInfo}) -> NConnInfo = ConnInfo#{ connected_at => erlang:system_time(millisecond) - , proto_name => <<"COAP">> - , proto_ver => <<"1">> }, ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]), _ = run_hooks(Ctx, 'client.connack', [NConnInfo, connection_accepted, []]), diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl index be328f32f..f3c64d678 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl @@ -93,10 +93,10 @@ info(ctx, #channel{ctx = Ctx}) -> stats(_) -> []. -init(ConnInfo = #{peername := {PeerHost, _}, - sockname := {_, SockPort}}, +init(ConnInfoT = #{peername := {PeerHost, _}, + sockname := {_, SockPort}}, #{ctx := Ctx} = Config) -> - Peercert = maps:get(peercert, ConnInfo, undefined), + Peercert = maps:get(peercert, ConnInfoT, undefined), Mountpoint = maps:get(mountpoint, Config, undefined), ListenerId = case maps:get(listener, Config, undefined) of undefined -> undefined; @@ -118,18 +118,20 @@ init(ConnInfo = #{peername := {PeerHost, _}, } ), + ConnInfo = ConnInfoT#{proto_name => <<"LwM2M">>, proto_ver => <<"0.0">>}, + #channel{ ctx = Ctx , conninfo = ConnInfo , clientinfo = ClientInfo , timers = #{} , session = emqx_lwm2m_session:new() - %% FIXME: don't store anonymouse func + %% FIXME: don't store anonymouse func , with_context = with_context(Ctx, ClientInfo) }. with_context(Ctx, ClientInfo) -> fun(Type, Topic) -> - with_context(Type, Topic, Ctx, ClientInfo) + with_context(Type, Topic, Ctx, ClientInfo) end. lookup_cmd(Channel, Path, Action) -> @@ -293,7 +295,6 @@ check_lwm2m_version(#coap_message{options = Opts}, end, if IsValid -> NConnInfo = ConnInfo#{ connected_at => erlang:system_time(millisecond) - , proto_name => <<"LwM2M">> , proto_ver => Ver }, {ok, Channel#channel{conninfo = NConnInfo}}; From 682b672b874db993e0416f161bd6a14d8fe22a4d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 30 Nov 2021 17:57:58 +0100 Subject: [PATCH 09/72] build: rebar3 no need to release before tar --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index 6ba943fab..c79dc711c 100755 --- a/build +++ b/build @@ -65,7 +65,7 @@ docgen() { make_rel() { # shellcheck disable=SC1010 - ./rebar3 as "$PROFILE" do release,tar + ./rebar3 as "$PROFILE" do tar if [ "$(find "_build/$PROFILE/rel/emqx/lib/" -maxdepth 1 -name 'gpb-*' -type d)" != "" ]; then echo "gpb should not be included in the release" exit 1 From a92970a9047c72a1181a8f3b7f2bac15344825f2 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Tue, 30 Nov 2021 11:44:15 +0100 Subject: [PATCH 10/72] feat(mria): Add RPC-related configuration to the schema --- apps/emqx_conf/src/emqx_conf_schema.erl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index d8bb2423b..bd897d730 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -347,6 +347,27 @@ to rlog. List of core nodes that the replicant will connect to.
Note: this parameter only takes effect when the backend is set to rlog and the role is set to replicant. +""" + })} + , {"rpc_module", + sc(hoconsc:enum([gen_rpc, rpc]), + #{ mapping => "mria.rlog_rpc_module" + , default => gen_rpc + , desc => """ +Protocol used for pushing transaction logs to the replicant nodes. +Important! This setting should be the same on all nodes in the cluster.
+Important! Changing this setting in the runtime is not allowed.
+""" + })} + , {"tlog_push_mode", + sc(hoconsc:enum([sync, async]), + #{ mapping => "mria.tlog_push_mode" + , default => sync + , desc => """ +In sync mode the core node waits for an ack from the replicant nodes before sending the next +transaction log entry. +Important! This setting should be the same on all nodes in the cluster.
+Important! Changing this setting in the runtime is not allowed.
""" })} ]; From d2fa0a71f471df86ad855b51fae11e1227a191e3 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 30 Nov 2021 17:43:12 +0100 Subject: [PATCH 11/72] refactor: give psk auth a better namespace --- .../src/emqx_dashboard_swagger.erl | 13 ++++++++++-- apps/emqx_psk/etc/emqx_psk.conf | 4 ++-- apps/emqx_psk/src/emqx_psk.erl | 8 +++---- apps/emqx_psk/src/emqx_psk_schema.erl | 21 ++++++++++++++++--- apps/emqx_psk/test/emqx_psk_SUITE.erl | 8 +++---- 5 files changed, 39 insertions(+), 15 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 59c0f560a..a699da8e6 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -337,19 +337,28 @@ components(Refs) -> components([], SpecAcc, []) -> SpecAcc; components([], SpecAcc, SubRefAcc) -> components(SubRefAcc, SpecAcc, []); components([{Module, Field} | Refs], SpecAcc, SubRefsAcc) -> - Props = apply(Module, fields, [Field]), + Props = hocon_schema_fields(Module, Field), Namespace = namespace(Module), {Object, SubRefs} = parse_object(Props, Module), NewSpecAcc = SpecAcc#{?TO_REF(Namespace, Field) => Object}, components(Refs, NewSpecAcc, SubRefs ++ SubRefsAcc); %% parameters in ref only have one value, not array components([{Module, Field, parameter} | Refs], SpecAcc, SubRefsAcc) -> - Props = apply(Module, fields, [Field]), + Props = hocon_schema_fields(Module, Field), {[Param], SubRefs} = parameters(Props, Module), Namespace = namespace(Module), NewSpecAcc = SpecAcc#{?TO_REF(Namespace, Field) => Param}, components(Refs, NewSpecAcc, SubRefs ++ SubRefsAcc). +hocon_schema_fields(Module, StructName) -> + case apply(Module, fields, [StructName]) of + #{fields := Fields, desc := _} -> + %% evil here, as it's match hocon_schema's internal representation + Fields; %% TODO: make use of desc ? + Other -> + Other + end. + %% Semantic error at components.schemas.xxx:xx:xx %% Component names can only contain the characters A-Z a-z 0-9 - . _ %% So replace ':' by '-'. diff --git a/apps/emqx_psk/etc/emqx_psk.conf b/apps/emqx_psk/etc/emqx_psk.conf index 80b29bfd4..ff9265fe1 100644 --- a/apps/emqx_psk/etc/emqx_psk.conf +++ b/apps/emqx_psk/etc/emqx_psk.conf @@ -2,11 +2,11 @@ ## EMQ X PSK ##-------------------------------------------------------------------- -psk { +psk_authentication { ## Whether to enable the PSK feature. enable = false - ## If init file is specified, emqx will import PSKs from the file + ## If init file is specified, emqx will import PSKs from the file ## into the built-in database at startup for use by the runtime. ## ## The file has to be structured line-by-line, each line must be in diff --git a/apps/emqx_psk/src/emqx_psk.erl b/apps/emqx_psk/src/emqx_psk.erl index ff89041ce..085a533d7 100644 --- a/apps/emqx_psk/src/emqx_psk.erl +++ b/apps/emqx_psk/src/emqx_psk.erl @@ -142,13 +142,13 @@ code_change(_OldVsn, State, _Extra) -> %%------------------------------------------------------------------------------ get_config(enable) -> - emqx_conf:get([psk, enable]); + emqx_conf:get([psk_authentication, enable]); get_config(init_file) -> - emqx_conf:get([psk, init_file], undefined); + emqx_conf:get([psk_authentication, init_file], undefined); get_config(separator) -> - emqx_conf:get([psk, separator], ?DEFAULT_DELIMITER); + emqx_conf:get([psk_authentication, separator], ?DEFAULT_DELIMITER); get_config(chunk_size) -> - emqx_conf:get([psk, chunk_size]). + emqx_conf:get([psk_authentication, chunk_size]). import_psks(SrcFile) -> case file:open(SrcFile, [read, raw, binary, read_ahead]) of diff --git a/apps/emqx_psk/src/emqx_psk_schema.erl b/apps/emqx_psk/src/emqx_psk_schema.erl index cce51d3fa..8097ade94 100644 --- a/apps/emqx_psk/src/emqx_psk_schema.erl +++ b/apps/emqx_psk/src/emqx_psk_schema.erl @@ -24,9 +24,24 @@ , fields/1 ]). -roots() -> ["psk"]. +roots() -> ["psk_authentication"]. -fields("psk") -> +fields("psk_authentication") -> + #{fields => fields(), + desc => """PSK stands for 'Pre-Shared Keys'. +This config to enable TLS-PSK authentication. + +Important! Make sure the SSL listener with +only tlsv1.2 enabled, and also PSK cipher suites +configured, such as RSA-PSK-AES256-GCM-SHA384. +See listener SSL options config for more details. + +The IDs and secrets can be provided from a file the path +to which is configurable by the init_file field. +""" + }. + +fields() -> [ {enable, fun enable/1} , {init_file, fun init_file/1} , {separator, fun separator/1} @@ -43,7 +58,7 @@ init_file(desc) -> <<"If init_file is specified, emqx will import PSKs from the file ", "into the built-in database at startup for use by the runtime. ", "The file has to be structured line-by-line, each line must be in ", - "the format: :">>; + "the format of 'PSKIdentity:SharedSecret' for example: mydevice1:c2VjcmV0">>; init_file(nullable) -> true; init_file(_) -> undefined. diff --git a/apps/emqx_psk/test/emqx_psk_SUITE.erl b/apps/emqx_psk/test/emqx_psk_SUITE.erl index 5794b8634..36d9521fe 100644 --- a/apps/emqx_psk/test/emqx_psk_SUITE.erl +++ b/apps/emqx_psk/test/emqx_psk_SUITE.erl @@ -26,13 +26,13 @@ all() -> init_per_suite(Config) -> meck:new(emqx_config, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_config, get, fun([psk, enable]) -> true; - ([psk, chunk_size]) -> 50; + meck:expect(emqx_config, get, fun([psk_authentication, enable]) -> true; + ([psk_authentication, chunk_size]) -> 50; (KeyPath) -> meck:passthrough([KeyPath]) end), - meck:expect(emqx_config, get, fun([psk, init_file], _) -> + meck:expect(emqx_config, get, fun([psk_authentication, init_file], _) -> filename:join([code:lib_dir(emqx_psk, test), "data/init.psk"]); - ([psk, separator], _) -> <<":">>; + ([psk_authentication, separator], _) -> <<":">>; (KeyPath, Default) -> meck:passthrough([KeyPath, Default]) end), emqx_common_test_helpers:start_apps([emqx_psk]), From 6b8997cbda109d6f11ec0d9706e85038fbe750cf Mon Sep 17 00:00:00 2001 From: zhouzb Date: Wed, 1 Dec 2021 09:23:36 +0800 Subject: [PATCH 12/72] fix(authn): fix superuser when missing is_superuser_field --- apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 7e080dfee..2deef8506 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -237,7 +237,9 @@ check_password(Password, is_superuser(Doc, #{is_superuser_field := IsSuperuserField}) -> IsSuperuser = maps:get(IsSuperuserField, Doc, false), - emqx_authn_utils:is_superuser(#{<<"is_superuser">> => IsSuperuser}). + emqx_authn_utils:is_superuser(#{<<"is_superuser">> => IsSuperuser}); +is_superuser(_, _) -> + emqx_authn_utils:is_superuser(#{<<"is_superuser">> => false}). hash(Algorithm, Password, Salt, prefix) -> emqx_passwd:hash(Algorithm, <>); From 1f6e2e73978e8f432b7f143f0e5c55bd92e92b77 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 1 Dec 2021 12:00:18 +0800 Subject: [PATCH 13/72] chore: add test for sub fields --- .../test/emqx_swagger_requestBody_SUITE.erl | 35 +++++++++++++++++-- .../test/emqx_swagger_response_SUITE.erl | 34 +++++++++++++++--- 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index ae74fc08e..84cac6229 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -7,7 +7,7 @@ -export([paths/0, api_spec/0, schema/1, fields/1]). -export([t_object/1, t_nest_object/1, t_api_spec/1, t_local_ref/1, t_remote_ref/1, t_bad_ref/1, t_none_ref/1, t_nest_ref/1, - t_ref_array_with_key/1, t_ref_array_without_key/1 + t_ref_array_with_key/1, t_ref_array_without_key/1, t_sub_fields/1 ]). -export([ t_object_trans/1, t_object_notrans/1, t_nest_object_trans/1, t_local_ref_trans/1, @@ -154,6 +154,17 @@ t_none_ref(_Config) -> emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path)), ok. +t_sub_fields(_Config) -> + Spec = #{ + post => #{parameters => [], + requestBody => #{<<"content">> => #{<<"application/json">> => + #{<<"schema">> => #{<<"$ref">> => + <<"#/components/schemas/emqx_swagger_requestBody_SUITE.sub_fields">>}}}}, + responses => #{<<"200">> => #{description => <<"ok">>}}}}, + Refs = [{?MODULE, sub_fields}], + validate("/fields/sub", Spec, Refs), + ok. + t_bad_ref(_Config) -> Path = "/ref/bad", Spec = #{ @@ -483,7 +494,7 @@ trans_requestBody(Path, Body, Filter) -> api_spec() -> emqx_dashboard_swagger:spec(?MODULE). paths() -> - ["/object", "/nest/object", "/ref/local", "/ref/nest/ref", + ["/object", "/nest/object", "/ref/local", "/ref/nest/ref", "/fields/sub", "/ref/array/with/key", "/ref/array/without/key"]. schema("/object") -> @@ -506,6 +517,8 @@ schema("/nest/object") -> ]); schema("/ref/local") -> to_schema(mk(hoconsc:ref(good_ref), #{})); +schema("/fields/sub") -> + to_schema(mk(hoconsc:ref(sub_fields), #{})); schema("/ref/remote") -> to_schema(mk(hoconsc:ref(emqx_swagger_remote_schema, "ref2"), #{})); schema("/ref/bad") -> @@ -544,4 +557,20 @@ fields(bad_ref) -> %% don't support maps #{ username => mk(string(), #{}), is_admin => mk(boolean(), #{}) - }. + }; +fields(sub_fields) -> + #{fields => [ + {enable, fun enable/1}, + {init_file, fun init_file/1} + ], + desc => <<"test sub fields">>}. + +enable(type) -> boolean(); +enable(desc) -> <<"Whether to enable tls psk support">>; +enable(default) -> false; +enable(_) -> undefined. + +init_file(type) -> binary(); +init_file(desc) -> <<"test test desc">>; +init_file(nullable) -> true; +init_file(_) -> undefined. diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index 4d4a9413e..b4785cf1a 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -14,7 +14,7 @@ -export([paths/0, api_spec/0, schema/1, fields/1]). -export([t_simple_binary/1, t_object/1, t_nest_object/1, t_empty/1, t_error/1, t_raw_local_ref/1, t_raw_remote_ref/1, t_hocon_schema_function/1, t_complicated_type/1, - t_local_ref/1, t_remote_ref/1, t_bad_ref/1, t_none_ref/1, t_nest_ref/1, + t_local_ref/1, t_remote_ref/1, t_bad_ref/1, t_none_ref/1, t_nest_ref/1, t_sub_fields/1, t_ref_array_with_key/1, t_ref_array_without_key/1, t_api_spec/1]). all() -> [{group, spec}]. @@ -23,7 +23,7 @@ groups() -> [ {spec, [parallel], [ t_api_spec, t_simple_binary, t_object, t_nest_object, t_error, t_complicated_type, t_raw_local_ref, t_raw_remote_ref, t_empty, t_hocon_schema_function, - t_local_ref, t_remote_ref, t_bad_ref, t_none_ref, + t_local_ref, t_remote_ref, t_bad_ref, t_none_ref, t_sub_fields, t_ref_array_with_key, t_ref_array_without_key, t_nest_ref]} ]. @@ -163,6 +163,14 @@ t_nest_ref(_Config) -> validate(Path, Object, ExpectRefs), ok. +t_sub_fields(_Config) -> + Path = "/fields/sub", + Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{ + <<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.sub_fields">>}}}}, + ExpectRefs = [{?MODULE, sub_fields}], + validate(Path, Object, ExpectRefs), + ok. + t_complicated_type(_Config) -> Path = "/ref/complicated_type", Object = #{<<"content">> => #{<<"application/json">> => @@ -366,7 +374,9 @@ schema("/ref/complicated_type") -> {fix_integer, hoconsc:mk(typerefl:integer(100), #{})} ] }} - }. + }; +schema("/fields/sub") -> + to_schema(hoconsc:ref(sub_fields)). validate(Path, ExpectObject, ExpectRefs) -> {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path), @@ -400,4 +410,20 @@ fields(bad_ref) -> %% don't support maps #{ username => mk(string(), #{}), is_admin => mk(boolean(), #{}) - }. + }; +fields(sub_fields) -> + #{fields => [ + {enable, fun enable/1}, + {init_file, fun init_file/1} + ], + desc => <<"test sub fields">>}. + +enable(type) -> boolean(); +enable(desc) -> <<"Whether to enable tls psk support">>; +enable(default) -> false; +enable(_) -> undefined. + +init_file(type) -> binary(); +init_file(desc) -> <<"test test desc">>; +init_file(nullable) -> true; +init_file(_) -> undefined. From c1a7d7bedead006c86832932a9d20a9c65214d75 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 1 Dec 2021 16:08:49 +0800 Subject: [PATCH 14/72] fix(conf): emqx_conf return nest error --- apps/emqx_conf/src/emqx_cluster_rpc.erl | 53 ++++++++++++++++--- apps/emqx_conf/src/emqx_conf.erl | 14 +++-- .../emqx_conf/test/emqx_cluster_rpc_SUITE.erl | 30 +++++++++-- apps/emqx_gateway/src/emqx_gateway_conf.erl | 2 +- 4 files changed, 83 insertions(+), 16 deletions(-) diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 153800414..17223234e 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -60,21 +60,28 @@ start_link() -> start_link(Node, Name, RetryMs) -> gen_server:start_link({local, Name}, ?MODULE, [Node, RetryMs], []). --spec multicall(Module, Function, Args) -> {ok, TnxId, term()} | {error, Reason} when +%% @doc return {ok, TnxId, MFARes} the first MFA result when all MFA run ok. +%% return {error, MFARes} when the first MFA result is no ok or {ok, term()}. +%% return {retry, TnxId, MFARes, Nodes} when some Nodes failed and some Node ok. +-spec multicall(Module, Function, Args) -> + {ok, TnxId, term()} | {error, Reason} | {retry, TnxId, MFARes, node()} when Module :: module(), Function :: atom(), Args :: [term()], + MFARes :: term(), TnxId :: pos_integer(), Reason :: string(). multicall(M, F, A) -> multicall(M, F, A, all, timer:minutes(2)). --spec multicall(Module, Function, Args, SucceedNum, Timeout) -> {ok, TnxId, term()} |{error, Reason} when +-spec multicall(Module, Function, Args, SucceedNum, Timeout) -> + {ok, TnxId, MFARes} | {error, Reason} | {retry, TnxId, MFARes, node()} when Module :: module(), Function :: atom(), Args :: [term()], SucceedNum :: pos_integer() | all, TnxId :: pos_integer(), + MFARes :: term(), Timeout :: timeout(), Reason :: string(). multicall(M, F, A, RequireNum, Timeout) when RequireNum =:= all orelse RequireNum >= 1 -> @@ -108,7 +115,10 @@ multicall(M, F, A, RequireNum, Timeout) when RequireNum =:= all orelse RequireNu end, case OkOrFailed of ok -> InitRes; - _ -> OkOrFailed + {error, Error0} -> {error, Error0}; + {retry, Node0} -> + {ok, TnxId0, MFARes} = InitRes, + {retry, TnxId0, MFARes, Node0} end. -spec query(pos_integer()) -> {'atomic', map()} | {'aborted', Reason :: term()}. @@ -136,6 +146,13 @@ get_node_tnx_id(Node) -> skip_failed_commit(Node) -> gen_server:call({?MODULE, Node}, skip_failed_commit). +%% Regardless of what MFA is returned, consider it a success), +%% then skip the specified TnxId. +%% If CurrTnxId >= TnxId, nothing happened. +%% If CurrTnxId < TnxId, the CurrTnxId will skip to TnxId. +-spec fast_forward_to_commit(node(), pos_integer()) -> pos_integer(). +fast_forward_to_commit(Node, ToTnxId) -> + gen_server:call({?MODULE, Node}, {fast_forward_to_commit, ToTnxId}). %%%=================================================================== %%% gen_server callbacks %%%=================================================================== @@ -165,8 +182,13 @@ handle_call({initiate, MFA}, _From, State = #{node := Node}) -> {aborted, Reason} -> {reply, {error, Reason}, State, {continue, ?CATCH_UP}} end; -handle_call(skip_failed_commit, _From, State) -> - {reply, ok, State, catch_up(State, true)}; +handle_call(skip_failed_commit, _From, State = #{node := Node}) -> + Timeout = catch_up(State, true), + {atomic, LatestId} = transaction(fun get_node_tnx_id/1, [Node]), + {reply, LatestId, State, Timeout}; +handle_call({fast_forward_to_commit, ToTnxId}, _From, State) -> + NodeId = do_fast_forward_to_commit(ToTnxId, State), + {reply, NodeId, State, catch_up(State)}; handle_call(_, _From, State) -> {reply, ok, State, catch_up(State)}. @@ -258,6 +280,20 @@ do_catch_up(ToTnxId, Node) -> commit(Node, TnxId) -> ok = mnesia:write(?CLUSTER_COMMIT, #cluster_rpc_commit{node = Node, tnx_id = TnxId}, write). +do_fast_forward_to_commit(ToTnxId, State = #{node := Node}) -> + {atomic, NodeId} = transaction(fun get_node_tnx_id/1, [Node]), + case NodeId >= ToTnxId of + true -> NodeId; + false -> + {atomic, LatestId} = transaction(fun get_latest_id/0, []), + case LatestId =< NodeId of + true -> NodeId; + false -> + catch_up(State, true), + do_fast_forward_to_commit(ToTnxId, State) + end + end. + get_latest_id() -> case mnesia:last(?CLUSTER_MFA) of '$end_of_table' -> 0; @@ -269,7 +305,8 @@ init_mfa(Node, MFA) -> LatestId = get_latest_id(), ok = do_catch_up_in_one_trans(LatestId, Node), TnxId = LatestId + 1, - MFARec = #cluster_rpc_mfa{tnx_id = TnxId, mfa = MFA, initiator = Node, created_at = erlang:localtime()}, + MFARec = #cluster_rpc_mfa{tnx_id = TnxId, mfa = MFA, + initiator = Node, created_at = erlang:localtime()}, ok = mnesia:write(?CLUSTER_MFA, MFARec, write), ok = commit(Node, TnxId), case apply_mfa(TnxId, MFA) of @@ -344,7 +381,7 @@ wait_for_all_nodes_commit(TnxId, Delay, Remain) -> ok = timer:sleep(Delay), wait_for_all_nodes_commit(TnxId, Delay, Remain - Delay); [] -> ok; - Nodes -> {error, Nodes} + Nodes -> {retry, Nodes} end. wait_for_nodes_commit(RequiredNum, TnxId, Delay, Remain) -> @@ -356,7 +393,7 @@ wait_for_nodes_commit(RequiredNum, TnxId, Delay, Remain) -> false -> case lagging_node(TnxId) of [] -> ok; %% All commit but The succeedNum > length(nodes()). - Nodes -> {error, Nodes} + Nodes -> {retry, Nodes} end end. diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index c3dfa8c49..c82623e72 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -16,6 +16,7 @@ -module(emqx_conf). -compile({no_auto_import, [get/1, get/2]}). +-include_lib("emqx/include/logger.hrl"). -export([add_handler/2, remove_handler/1]). -export([get/1, get/2, get_raw/2, get_all/1]). @@ -123,13 +124,18 @@ reset(Node, KeyPath, Opts) -> rpc:call(Node, ?MODULE, reset, [KeyPath, Opts]). %%-------------------------------------------------------------------- -%% Internal funcs +%% Internal functions %%-------------------------------------------------------------------- multicall(M, F, Args) -> case emqx_cluster_rpc:multicall(M, F, Args) of - {ok, _TnxId, Res} -> + {ok, _TnxId, Res} -> Res; + {retry, TnxId, Res, Nodes} -> + %% The init MFA return ok, but other nodes failed. + %% We return ok and alert an alarm. + ?SLOG(error, #{msg => "failed to update config in cluster", nodes => Nodes, + tnx_id => TnxId, mfa => {M, F, Args}}), Res; - {error, Reason} -> - {error, Reason} + {error, Error} -> %% all MFA return not ok or {ok, term()}. + Error end. diff --git a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl index 993ab3dc5..1710b90bd 100644 --- a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl +++ b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl @@ -33,7 +33,8 @@ all() -> [ t_commit_ok_but_apply_fail_on_other_node, t_commit_ok_apply_fail_on_other_node_then_recover, t_del_stale_mfa, - t_skip_failed_commit + t_skip_failed_commit, + t_fast_forward_commit ]. suite() -> [{timetrap, {minutes, 3}}]. groups() -> []. @@ -183,13 +184,36 @@ t_skip_failed_commit(_Config) -> ?assertEqual([{Node, 1}, {{Node, ?NODE2}, 1}, {{Node, ?NODE3}, 1}], tnx_ids(List1)), {M, F, A} = {?MODULE, failed_on_node, [erlang:whereis(?NODE1)]}, - {ok, _, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), - ok = gen_server:call(?NODE2, skip_failed_commit, 5000), + {ok, 2, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), + 2 = gen_server:call(?NODE2, skip_failed_commit, 5000), {atomic, List2} = emqx_cluster_rpc:status(), ?assertEqual([{Node, 2}, {{Node, ?NODE2}, 2}, {{Node, ?NODE3}, 1}], tnx_ids(List2)), ok. +t_fast_forward_commit(_Config) -> + emqx_cluster_rpc:reset(), + {atomic, []} = emqx_cluster_rpc:status(), + {ok, 1, ok} = emqx_cluster_rpc:multicall(io, format, ["test~n"], all, 1000), + ct:sleep(180), + {atomic, List1} = emqx_cluster_rpc:status(), + Node = node(), + ?assertEqual([{Node, 1}, {{Node, ?NODE2}, 1}, {{Node, ?NODE3}, 1}], + tnx_ids(List1)), + {M, F, A} = {?MODULE, failed_on_node, [erlang:whereis(?NODE1)]}, + {ok, 2, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), + {ok, 3, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), + {ok, 4, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), + {ok, 5, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), + 3 = gen_server:call(?NODE2, {fast_forward_to_commit, 3}, 5000), + 4 = gen_server:call(?NODE2, {fast_forward_to_commit, 4}, 5000), + 5 = gen_server:call(?NODE2, {fast_forward_to_commit, 6}, 5000), + 2 = gen_server:call(?NODE3, {fast_forward_to_commit, 2}, 5000), + {atomic, List2} = emqx_cluster_rpc:status(), + ?assertEqual([{Node, 5}, {{Node, ?NODE2}, 5}, {{Node, ?NODE3}, 2}], + tnx_ids(List2)), + ok. + tnx_ids(Status) -> lists:sort(lists:map(fun(#{tnx_id := TnxId, node := Node}) -> {Node, TnxId} end, Status)). diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl index ddbf99189..3612b428d 100644 --- a/apps/emqx_gateway/src/emqx_gateway_conf.erl +++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl @@ -232,7 +232,7 @@ update(Req) -> res(emqx_conf:update([gateway], Req, #{override_to => cluster})). res({ok, _Result}) -> ok; -res({error, {error, {pre_config_update,emqx_gateway_conf,Reason}}}) -> {error, Reason}; +res({error, {pre_config_update, emqx_gateway_conf, Reason}}) -> {error, Reason}; res({error, Reason}) -> {error, Reason}. bin({LType, LName}) -> From b983a18cdf9aa1a48d69cf342fddeb139377fc33 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Wed, 1 Dec 2021 20:24:43 +0800 Subject: [PATCH 15/72] fix(auth): replace query with cmd --- apps/emqx_authn/src/emqx_authn_api.erl | 2 +- .../src/simple_authn/emqx_authn_redis.erl | 24 +++++++++---------- .../test/emqx_authn_redis_SUITE.erl | 20 ++++++++-------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index b83d9d1af..83ca9e56d 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -1027,7 +1027,7 @@ authenticator_examples() -> backend => <<"redis">>, server => <<"127.0.0.1:6379">>, database => 0, - query => <<"HMGET ${username} password_hash salt">>, + cmd => <<"HMGET ${username} password_hash salt">>, password_hash_algorithm => <<"sha256">>, salt_position => <<"prefix">> } diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index 963536e0b..d238bc537 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -58,13 +58,13 @@ fields(sentinel) -> common_fields() -> [{mechanism, {enum, ['password-based']}}, {backend, {enum, [redis]}}, - {query, fun query/1}, + {cmd, fun cmd/1}, {password_hash_algorithm, fun password_hash_algorithm/1}, {salt_position, fun salt_position/1} ] ++ emqx_authn_schema:common_fields(). -query(type) -> string(); -query(_) -> undefined. +cmd(type) -> string(); +cmd(_) -> undefined. password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt]}; password_hash_algorithm(default) -> sha256; @@ -87,17 +87,17 @@ refs() -> create(_AuthenticatorID, Config) -> create(Config). -create(#{query := Query, +create(#{cmd := Cmd, password_hash_algorithm := Algorithm} = Config) -> try - NQuery = parse_query(Query), + NCmd = parse_cmd(Cmd), ok = emqx_authn_utils:ensure_apps_started(Algorithm), State = maps:with( [password_hash_algorithm, salt_position], Config), ResourceId = emqx_authn_utils:make_resource_id(?MODULE), NState = State#{ - query => NQuery, + cmd => NCmd, resource_id => ResourceId}, case emqx_resource:create_local(ResourceId, emqx_connector_redis, Config) of {ok, already_created} -> @@ -108,8 +108,8 @@ create(#{query := Query, {error, Reason} end catch - error:{unsupported_query, _Query} -> - {error, {unsupported_query, Query}}; + error:{unsupported_cmd, _Cmd} -> + {error, {unsupported_cmd, Cmd}}; error:missing_password_hash -> {error, missing_password_hash}; error:{unsupported_fields, Fields} -> @@ -128,7 +128,7 @@ update(Config, State) -> authenticate(#{auth_method := _}, _) -> ignore; authenticate(#{password := Password} = Credential, - #{query := {Command, Key, Fields}, + #{cmd := {Command, Key, Fields}, resource_id := ResourceId} = State) -> NKey = binary_to_list(iolist_to_binary(replace_placeholders(Key, Credential))), case emqx_resource:query(ResourceId, {cmd, [Command, NKey | Fields]}) of @@ -162,15 +162,15 @@ destroy(#{resource_id := ResourceId}) -> %%------------------------------------------------------------------------------ %% Only support HGET and HMGET -parse_query(Query) -> - case string:tokens(Query, " ") of +parse_cmd(Cmd) -> + case string:tokens(Cmd, " ") of [Command, Key, Field | Fields] when Command =:= "HGET" orelse Command =:= "HMGET" -> NFields = [Field | Fields], check_fields(NFields), NKey = parse_key(Key), {Command, NKey, NFields}; _ -> - error({unsupported_query, Query}) + error({unsupported_cmd, Cmd}) end. check_fields(Fields) -> diff --git a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl index be1ae6d13..02f3e0b7c 100644 --- a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl @@ -98,11 +98,11 @@ t_create_invalid(_Config) -> AuthConfig#{password => <<"wrongpass">>}, AuthConfig#{database => <<"5678">>}, AuthConfig#{ - query => <<"MGET password_hash:${username} salt:${username}">>}, + cmd => <<"MGET password_hash:${username} salt:${username}">>}, AuthConfig#{ - query => <<"HMGET mqtt_user:${username} password_hash invalid_field">>}, + cmd => <<"HMGET mqtt_user:${username} password_hash invalid_field">>}, AuthConfig#{ - query => <<"HMGET mqtt_user:${username} salt is_superuser">>} + cmd => <<"HMGET mqtt_user:${username} salt is_superuser">>} ], lists:foreach( @@ -177,7 +177,7 @@ t_update(_Config) -> CorrectConfig = raw_redis_auth_config(), IncorrectConfig = CorrectConfig#{ - query => <<"HMGET invalid_key:${username} password_hash salt is_superuser">>}, + cmd => <<"HMGET invalid_key:${username} password_hash salt is_superuser">>}, {ok, _} = emqx:update_config( ?PATH, @@ -214,7 +214,7 @@ raw_redis_auth_config() -> enable => <<"true">>, backend => <<"redis">>, - query => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>, + cmd => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>, database => <<"1">>, password => <<"public">>, server => redis_server() @@ -262,7 +262,7 @@ user_seeds() -> }, key => "mqtt_user:sha256", config_params => #{ - query => <<"HMGET mqtt_user:${clientid} password_hash salt is_superuser">>, + cmd => <<"HMGET mqtt_user:${clientid} password_hash salt is_superuser">>, password_hash_algorithm => <<"sha256">>, salt_position => <<"prefix">> }, @@ -298,7 +298,7 @@ user_seeds() -> key => "mqtt_user:bcrypt0", config_params => #{ % clientid variable & username credentials - query => <<"HMGET mqtt_client:${clientid} password_hash salt is_superuser">>, + cmd => <<"HMGET mqtt_client:${clientid} password_hash salt is_superuser">>, password_hash_algorithm => <<"bcrypt">>, salt_position => <<"suffix">> }, @@ -316,8 +316,8 @@ user_seeds() -> }, key => "mqtt_user:bcrypt1", config_params => #{ - % Bad key in query - query => <<"HMGET badkey:${username} password_hash salt is_superuser">>, + % Bad key in cmd + cmd => <<"HMGET badkey:${username} password_hash salt is_superuser">>, password_hash_algorithm => <<"bcrypt">>, salt_position => <<"suffix">> }, @@ -336,7 +336,7 @@ user_seeds() -> }, key => "mqtt_user:bcrypt2", config_params => #{ - query => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>, + cmd => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>, password_hash_algorithm => <<"bcrypt">>, salt_position => <<"suffix">> }, From e7a7d64004a5a64d975910cdd310f28c594e85d8 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 2 Dec 2021 09:30:27 +0800 Subject: [PATCH 16/72] feat(conf): skip/fast_forward tnx_id via cluster_call cli --- apps/emqx_conf/src/emqx_cluster_rpc.erl | 13 ++- apps/emqx_conf/src/emqx_conf_cli.erl | 92 +++++++++++++++++++ .../emqx_conf/test/emqx_cluster_rpc_SUITE.erl | 5 +- apps/emqx_modules/src/emqx_modules_app.erl | 2 + 4 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 apps/emqx_conf/src/emqx_conf_cli.erl diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 17223234e..7ebe7645b 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -18,8 +18,9 @@ %% API -export([start_link/0, mnesia/1]). --export([multicall/3, multicall/5, query/1, reset/0, status/0, skip_failed_commit/1]). --export([get_node_tnx_id/1]). +-export([multicall/3, multicall/5, query/1, reset/0, status/0, + skip_failed_commit/1, fast_forward_to_commit/2]). +-export([get_node_tnx_id/1, latest_tnx_id/0]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, handle_continue/2, code_change/3]). @@ -132,6 +133,11 @@ reset() -> gen_server:call(?MODULE, reset). status() -> transaction(fun trans_status/0, []). +-spec latest_tnx_id() -> pos_integer(). +latest_tnx_id() -> + {atomic, TnxId} = transaction(fun get_latest_id/0, []), + TnxId. + -spec get_node_tnx_id(node()) -> integer(). get_node_tnx_id(Node) -> case mnesia:wread({?CLUSTER_COMMIT, Node}) of @@ -267,7 +273,8 @@ do_catch_up(ToTnxId, Node) -> {false, Error} -> mnesia:abort(Error) end; [#cluster_rpc_commit{tnx_id = LastAppliedId}] -> - Reason = lists:flatten(io_lib:format("~p catch up failed by LastAppliedId(~p) > ToTnxId(~p)", + Reason = lists:flatten( + io_lib:format("~p catch up failed by LastAppliedId(~p) > ToTnxId(~p)", [Node, LastAppliedId, ToTnxId])), ?SLOG(error, #{ msg => "catch up failed!", diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl new file mode 100644 index 000000000..7fb421e75 --- /dev/null +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -0,0 +1,92 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_conf_cli). +-export([ load/0 + , admins/1 + , unload/0 + ]). + +-define(CMD, cluster_call). + +load() -> + emqx_ctl:register_command(?CMD, {?MODULE, admins}, []). + +unload() -> + emqx_ctl:unregister_command(?CMD). + +admins(["status"]) -> status(); + +admins(["skip"]) -> + status(), + Nodes = mria_mnesia:running_nodes(), + lists:foreach(fun emqx_cluster_rpc:skip_failed_commit/1, Nodes), + status(); + +admins(["skip", Node0]) -> + status(), + Node = list_to_existing_atom(Node0), + emqx_cluster_rpc:skip_failed_commit(Node), + status(); + +admins(["tnxid", TnxId0]) -> + TnxId = list_to_integer(TnxId0), + emqx_ctl:print("~p~n", [emqx_cluster_rpc:query(TnxId)]); + +admins(["fast_forward"]) -> + status(), + Nodes = mria_mnesia:running_nodes(), + TnxId = emqx_cluster_rpc:latest_tnx_id(), + lists:foreach(fun(N) -> emqx_cluster_rpc:fast_forward_to_commit(N, TnxId) end, Nodes), + status(); + +admins(["fast_forward", ToTnxId]) -> + status(), + Nodes = mria_mnesia:running_nodes(), + TnxId = list_to_integer(ToTnxId), + lists:foreach(fun(N) -> emqx_cluster_rpc:fast_forward_to_commit(N, TnxId) end, Nodes), + status(); + +admins(["fast_forward", Node0, ToTnxId]) -> + status(), + TnxId = list_to_integer(ToTnxId), + Node = list_to_existing_atom(Node0), + emqx_cluster_rpc:fast_forward_to_commit(Node, TnxId), + status(); + +admins(_) -> + emqx_ctl:usage( + [ + {"cluster_call status", "status"}, + {"cluster_call skip [node]", "increase one commit on specific node"}, + {"cluster_call tnxid ", "get detailed about TnxId"}, + {"cluster_call fast_forward [node] [tnx_id]", "fast forwards to tnx_id" } + ]). + +status() -> + emqx_ctl:print("-----------------------------------------------\n"), + {atomic, Status} = emqx_cluster_rpc:status(), + lists:foreach(fun(S) -> + #{ + node := Node, + tnx_id := TnxId, + mfa := {M, F, A}, + created_at := CreatedAt + } = S, + emqx_ctl:print("~p:[~w] CreatedAt:~p ~p:~p/~w\n", + [Node, TnxId, CreatedAt, M, F, length(A)]) + end, Status), + emqx_ctl:print("-----------------------------------------------\n"). diff --git a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl index 1710b90bd..ad74faf99 100644 --- a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl +++ b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl @@ -205,12 +205,13 @@ t_fast_forward_commit(_Config) -> {ok, 3, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), {ok, 4, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), {ok, 5, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), + {retry, 6, ok, _} = emqx_cluster_rpc:multicall(M, F, A, 2, 1000), 3 = gen_server:call(?NODE2, {fast_forward_to_commit, 3}, 5000), 4 = gen_server:call(?NODE2, {fast_forward_to_commit, 4}, 5000), - 5 = gen_server:call(?NODE2, {fast_forward_to_commit, 6}, 5000), + 6 = gen_server:call(?NODE2, {fast_forward_to_commit, 7}, 5000), 2 = gen_server:call(?NODE3, {fast_forward_to_commit, 2}, 5000), {atomic, List2} = emqx_cluster_rpc:status(), - ?assertEqual([{Node, 5}, {{Node, ?NODE2}, 5}, {{Node, ?NODE3}, 2}], + ?assertEqual([{Node, 6}, {{Node, ?NODE2}, 6}, {{Node, ?NODE3}, 2}], tnx_ids(List2)), ok. diff --git a/apps/emqx_modules/src/emqx_modules_app.erl b/apps/emqx_modules/src/emqx_modules_app.erl index 1605c3382..55c882f94 100644 --- a/apps/emqx_modules/src/emqx_modules_app.erl +++ b/apps/emqx_modules/src/emqx_modules_app.erl @@ -36,6 +36,7 @@ maybe_enable_modules() -> emqx_conf:get([telemetry, enable], true) andalso emqx_telemetry:enable(), emqx_conf:get([observer_cli, enable], true) andalso emqx_observer_cli:enable(), emqx_event_message:enable(), + emqx_conf_cli:load(), ok = emqx_rewrite:enable(), emqx_topic_metrics:enable(). @@ -45,4 +46,5 @@ maybe_disable_modules() -> emqx_conf:get([observer_cli, enable], true) andalso emqx_observer_cli:disable(), emqx_event_message:disable(), emqx_rewrite:disable(), + emqx_conf_cli:unload(), emqx_topic_metrics:disable(). From 68af28457045a37a1fb662a2eb31bb3c6d114c93 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 30 Nov 2021 15:57:36 +0800 Subject: [PATCH 17/72] fix(authz): http source create and update --- .../src/simple_authn/emqx_authn_http.erl | 2 +- apps/emqx_authz/src/emqx_authz.erl | 7 +- .../emqx_authz/src/emqx_authz_api_sources.erl | 2 +- apps/emqx_authz/src/emqx_authz_http.erl | 38 +++-- apps/emqx_authz/src/emqx_authz_schema.erl | 144 ++++++++++-------- 5 files changed, 120 insertions(+), 73 deletions(-) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index ec2da3237..829ce2282 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -284,7 +284,7 @@ replace_placeholders([{K, V0} | More], Credential, Acc) -> undefined -> error({cannot_get_variable, V0}); V -> - replace_placeholders(More, Credential, [{K, emqx_authn_utils:bin(V)} | Acc]) + replace_placeholders(More, Credential, [{K, to_bin(V)} | Acc]) end. append_query(Path, []) -> diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 2bcb7971d..0b5534608 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -207,7 +207,12 @@ check_dup_types([Source | Sources], Checked) -> create_dry_run(T, Source) -> case is_connector_source(T) of true -> - [NSource] = check_sources([Source]), + [CheckedSource] = check_sources([Source]), + case T of + http -> + URIMap = maps:get(url, CheckedSource), + NSource = maps:put(base_url, maps:remove(query, URIMap), CheckedSource) + end, emqx_resource:create_dry_run(connector_module(T), NSource); false -> ok diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 23c6077fa..4ba05037c 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -440,7 +440,7 @@ read_certs(#{<<"ssl">> := SSL} = Source) -> {error, Reason} -> ?SLOG(error, Reason#{msg => failed_to_readd_ssl_file}), throw(failed_to_readd_ssl_file); - NewSSL -> + {ok, NewSSL} -> Source#{<<"ssl">> => NewSSL} end; read_certs(Source) -> Source. diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index 4c17f90ec..af4e7f742 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -24,6 +24,7 @@ %% AuthZ Callbacks -export([ authorize/4 , description/0 + , parse_url/1 ]). -ifdef(TEST). @@ -36,7 +37,7 @@ description() -> authorize(Client, PubSub, Topic, #{type := http, - url := #{path := Path} = Url, + url := #{path := Path} = URL, headers := Headers, method := Method, request_timeout := RequestTimeout, @@ -44,7 +45,7 @@ authorize(Client, PubSub, Topic, } = Source) -> Request = case Method of get -> - Query = maps:get(query, Url, ""), + Query = maps:get(query, URL, ""), Path1 = replvar(Path ++ "?" ++ Query, PubSub, Topic, Client), {Path1, maps:to_list(Headers)}; _ -> @@ -56,10 +57,29 @@ authorize(Client, PubSub, Topic, Path1 = replvar(Path, PubSub, Topic, Client), {Path1, maps:to_list(Headers), Body1} end, - case emqx_resource:query(ResourceID, {Method, Request, RequestTimeout}) of - {ok, 204, _Headers} -> {matched, allow}; - {ok, 200, _Headers, _Body} -> {matched, allow}; - _ -> nomatch + case emqx_resource:query(ResourceID, {Method, Request, RequestTimeout}) of + {ok, 200, _Headers} -> + {matched, allow}; + {ok, 204, _Headers} -> + {matched, allow}; + {ok, 200, _Headers, _Body} -> + {matched, allow}; + {ok, _Status, _Headers, _Body} -> + nomatch; + {error, Reason} -> + ?SLOG(error, #{msg => "http_server_query_failed", + resource => ResourceID, + reason => Reason}), + ignore + end. + +parse_url(URL) -> + {ok, URIMap} = emqx_http_lib:uri_parse(URL), + case maps:get(query, URIMap, undefined) of + undefined -> + URIMap#{query => ""}; + _ -> + URIMap end. query_string(Body) -> @@ -88,7 +108,7 @@ replvar(Str0, PubSub, Topic, is_binary(Str0) -> NTopic = emqx_http_lib:uri_encode(Topic), Str1 = re:replace( Str0, emqx_authz:ph_to_re(?PH_S_CLIENTID) - , Clientid, [global, {return, binary}]), + , bin(Clientid), [global, {return, binary}]), Str2 = re:replace( Str1, emqx_authz:ph_to_re(?PH_S_USERNAME) , bin(Username), [global, {return, binary}]), Str3 = re:replace( Str2, emqx_authz:ph_to_re(?PH_S_HOST) @@ -96,9 +116,9 @@ replvar(Str0, PubSub, Topic, Str4 = re:replace( Str3, emqx_authz:ph_to_re(?PH_S_PROTONAME) , bin(Protocol), [global, {return, binary}]), Str5 = re:replace( Str4, emqx_authz:ph_to_re(?PH_S_MOUNTPOINT) - , Mountpoint, [global, {return, binary}]), + , bin(Mountpoint), [global, {return, binary}]), Str6 = re:replace( Str5, emqx_authz:ph_to_re(?PH_S_TOPIC) - , NTopic, [global, {return, binary}]), + , bin(NTopic), [global, {return, binary}]), Str7 = re:replace( Str6, emqx_authz:ph_to_re(?PH_S_ACTION) , bin(PubSub), [global, {return, binary}]), Str7. diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 884f6e82b..62d9bff17 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -32,10 +32,15 @@ -export([ namespace/0 , roots/0 , fields/1 + , validations/0 ]). -import(emqx_schema, [mk_duration/2]). +%%-------------------------------------------------------------------- +%% Hocon Schema +%%-------------------------------------------------------------------- + namespace() -> authz. %% @doc authorization schema is not exported @@ -98,66 +103,13 @@ and the new rules will override all rules from the old config file. }} ]; fields(http_get) -> - [ {type, #{type => http}} - , {enable, #{type => boolean(), - default => true}} - , {url, #{type => url()}} - , {method, #{type => get, default => get }} - , {headers, #{type => map(), - default => #{ <<"accept">> => <<"application/json">> - , <<"cache-control">> => <<"no-cache">> - , <<"connection">> => <<"keep-alive">> - , <<"keep-alive">> => <<"timeout=5">> - }, - converter => fun (Headers0) -> - Headers1 = maps:fold(fun(K0, V, AccIn) -> - K1 = iolist_to_binary(string:to_lower(to_list(K0))), - maps:put(K1, V, AccIn) - end, #{}, Headers0), - maps:merge(#{ <<"accept">> => <<"application/json">> - , <<"cache-control">> => <<"no-cache">> - , <<"connection">> => <<"keep-alive">> - , <<"keep-alive">> => <<"timeout=5">> - }, Headers1) - end - } - } - , {request_timeout, mk_duration("request timeout", #{default => "30s"})} - ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)); + [ {method, #{type => get, default => post}} + , {headers, fun headers_no_content_type/1} + ] ++ http_common_fields(); fields(http_post) -> - [ {type, #{type => http}} - , {enable, #{type => boolean(), - default => true}} - , {url, #{type => url()}} - , {method, #{type => post, - default => get}} - , {headers, #{type => map(), - default => #{ <<"accept">> => <<"application/json">> - , <<"cache-control">> => <<"no-cache">> - , <<"connection">> => <<"keep-alive">> - , <<"content-type">> => <<"application/json">> - , <<"keep-alive">> => <<"timeout=5">> - }, - converter => fun (Headers0) -> - Headers1 = maps:fold(fun(K0, V, AccIn) -> - K1 = iolist_to_binary(string:to_lower(binary_to_list(K0))), - maps:put(K1, V, AccIn) - end, #{}, Headers0), - maps:merge(#{ <<"accept">> => <<"application/json">> - , <<"cache-control">> => <<"no-cache">> - , <<"connection">> => <<"keep-alive">> - , <<"content-type">> => <<"application/json">> - , <<"keep-alive">> => <<"timeout=5">> - }, Headers1) - end - } - } - , {request_timeout, mk_duration("request timeout", #{default => "30s"})} - , {body, #{type => map(), - nullable => true - } - } - ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)); + [ {method, #{type => post, default => post}} + , {headers, fun headers/1} + ] ++ http_common_fields(); fields(mnesia) -> [ {type, #{type => 'built-in-database'}} , {enable, #{type => boolean(), @@ -203,10 +155,73 @@ fields(redis_cluster) -> connector_fields(redis, cluster) ++ [ {cmd, query()} ]. +http_common_fields() -> + [ {type, #{type => http}} + , {enable, #{type => boolean(), default => true}} + , {url, #{type => url()}} + , {request_timeout, mk_duration("request timeout", #{default => "30s"})} + , {body, #{type => map(), nullable => true}} + ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)). + +validations() -> + [ {check_ssl_opts, fun check_ssl_opts/1} + , {check_headers, fun check_headers/1} + ]. + +headers(type) -> map(); +headers(converter) -> + fun(Headers) -> + maps:merge(default_headers(), transform_header_name(Headers)) + end; +headers(default) -> default_headers(); +headers(_) -> undefined. + +headers_no_content_type(type) -> map(); +headers_no_content_type(converter) -> + fun(Headers) -> + maps:merge(default_headers_no_content_type(), transform_header_name(Headers)) + end; +headers_no_content_type(default) -> default_headers_no_content_type(); +headers_no_content_type(_) -> undefined. + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- +default_headers() -> + maps:put(<<"content-type">>, + <<"application/json">>, + default_headers_no_content_type()). + +default_headers_no_content_type() -> + #{ <<"accept">> => <<"application/json">> + , <<"cache-control">> => <<"no-cache">> + , <<"connection">> => <<"keep-alive">> + , <<"keep-alive">> => <<"timeout=5">> + }. + +transform_header_name(Headers) -> + maps:fold(fun(K0, V, Acc) -> + K = list_to_binary(string:to_lower(to_list(K0))), + maps:put(K, V, Acc) + end, #{}, Headers). + +check_ssl_opts(Conf) -> + case emqx_authz_http:parse_url(hocon_schema:get_value("config.url", Conf)) of + #{scheme := https} -> + case hocon_schema:get_value("config.ssl.enable", Conf) of + true -> ok; + false -> false + end; + #{scheme := http} -> + ok + end. + +check_headers(Conf) -> + Method = to_bin(hocon_schema:get_value("config.method", Conf)), + Headers = hocon_schema:get_value("config.headers", Conf), + Method =:= <<"post">> orelse (not maps:is_key(<<"content-type">>, Headers)). + union_array(Item) when is_list(Item) -> hoconsc:array(hoconsc:union(Item)). @@ -229,8 +244,8 @@ connector_fields(DB, Fields) -> catch error:badarg -> list_to_atom(Mod0); - Error -> - erlang:error(Error) + error:Reason -> + erlang:error(Reason) end, [ {type, #{type => DB}} , {enable, #{type => boolean(), @@ -241,3 +256,10 @@ to_list(A) when is_atom(A) -> atom_to_list(A); to_list(B) when is_binary(B) -> binary_to_list(B). + +to_bin(A) when is_atom(A) -> + atom_to_binary(A); +to_bin(B) when is_binary(B) -> + B; +to_bin(L) when is_list(L) -> + list_to_binary(L). From d7ec368884b9fd6fe8bef7a552a0d368805b2eba Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 1 Dec 2021 12:03:46 +0800 Subject: [PATCH 18/72] fix(authz): fix mongo resources create --- apps/emqx_authz/src/emqx_authz_api_sources.erl | 3 ++- apps/emqx_connector/src/emqx_connector_mongo.erl | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 4ba05037c..df5a6c819 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -447,7 +447,8 @@ read_certs(Source) -> Source. maybe_write_certs(#{<<"ssl">> := #{<<"enable">> := true} = SSL} = Source) -> Type = maps:get(<<"type">>, Source), - emqx_tls_lib:ensure_ssl_files(filename:join(["authz", Type]), SSL); + {ok, Return} = emqx_tls_lib:ensure_ssl_files(filename:join(["authz", Type]), SSL), + maps:put(<<"ssl">>, Return, Source); maybe_write_certs(Source) -> Source. write_file(Filename, Bytes0) -> diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index c121bb4c6..6a1b15e57 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -269,7 +269,7 @@ srv_record(_) -> undefined. parse_servers(Type, Servers) when is_binary(Servers) -> parse_servers(Type, binary_to_list(Servers)); parse_servers(Type, Servers) when is_list(Servers) -> - case string:split(Servers, ",", trailing) of + case string:split(Servers, ",", all) of [Host | _] when Type =:= single -> [Host]; Hosts -> From effa3b8b908c5a0144467f2f86068817ce89828b Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 1 Dec 2021 15:47:56 +0800 Subject: [PATCH 19/72] style: make elvis happy --- apps/emqx_authz/src/emqx_authz_schema.erl | 31 +++++++++-------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 62d9bff17..01f601857 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -116,26 +116,11 @@ fields(mnesia) -> default => true}} ]; fields(mongo_single) -> - [ {collection, #{type => atom()}} - , {selector, #{type => map()}} - , {type, #{type => mongodb}} - , {enable, #{type => boolean(), - default => true}} - ] ++ emqx_connector_mongo:fields(single); + mongo_common_fields() ++ emqx_connector_mongo:fields(single); fields(mongo_rs) -> - [ {collection, #{type => atom()}} - , {selector, #{type => map()}} - , {type, #{type => mongodb}} - , {enable, #{type => boolean(), - default => true}} - ] ++ emqx_connector_mongo:fields(rs); + mongo_common_fields() ++ emqx_connector_mongo:fields(rs); fields(mongo_sharded) -> - [ {collection, #{type => atom()}} - , {selector, #{type => map()}} - , {type, #{type => mongodb}} - , {enable, #{type => boolean(), - default => true}} - ] ++ emqx_connector_mongo:fields(sharded); + mongo_common_fields() ++ emqx_connector_mongo:fields(sharded); fields(mysql) -> connector_fields(mysql) ++ [ {query, query()} ]; @@ -163,6 +148,14 @@ http_common_fields() -> , {body, #{type => map(), nullable => true}} ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)). +mongo_common_fields() -> + [ {collection, #{type => atom()}} + , {selector, #{type => map()}} + , {type, #{type => mongodb}} + , {enable, #{type => boolean(), + default => true}} + ]. + validations() -> [ {check_ssl_opts, fun check_ssl_opts/1} , {check_headers, fun check_headers/1} @@ -250,7 +243,7 @@ connector_fields(DB, Fields) -> [ {type, #{type => DB}} , {enable, #{type => boolean(), default => true}} - ] ++ Mod:fields(Fields). + ] ++ erlang:apply(Mod, fields, [Fields]). to_list(A) when is_atom(A) -> atom_to_list(A); From a58493f7a41e17fad986d877c9cb816d4fccf5f8 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 2 Dec 2021 10:10:22 +0800 Subject: [PATCH 20/72] test(authz): fix test suite for schema check and app start --- apps/emqx_authz/src/emqx_authz_http.erl | 3 +++ apps/emqx_authz/src/emqx_authz_schema.erl | 6 ++++++ apps/emqx_authz/test/emqx_authz_SUITE.erl | 3 ++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index af4e7f742..6d1324c47 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -73,6 +73,9 @@ authorize(Client, PubSub, Topic, ignore end. +parse_url(URL) + when URL =:= undefined -> + #{}; parse_url(URL) -> {ok, URIMap} = emqx_http_lib:uri_parse(URL), case maps:get(query, URIMap, undefined) of diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 01f601857..4f7788849 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -199,6 +199,9 @@ transform_header_name(Headers) -> maps:put(K, V, Acc) end, #{}, Headers). +check_ssl_opts(Conf) + when Conf =:= #{} -> + true; check_ssl_opts(Conf) -> case emqx_authz_http:parse_url(hocon_schema:get_value("config.url", Conf)) of #{scheme := https} -> @@ -210,6 +213,9 @@ check_ssl_opts(Conf) -> ok end. +check_headers(Conf) + when Conf =:= #{} -> + true; check_headers(Conf) -> Method = to_bin(hocon_schema:get_value("config.method", Conf)), Headers = hocon_schema:get_value("config.headers", Conf), diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index 130e266fb..d965affee 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -36,7 +36,8 @@ init_per_suite(Config) -> meck:expect(emqx_resource, remove, fun(_) -> ok end ), ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_authz], fun set_special_configs/1), + [emqx_connector, emqx_conf, emqx_authz], + fun set_special_configs/1), Config. end_per_suite(_Config) -> From f6c61189c4787d7b1ed679ea76af0f731e0ea9ac Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 2 Dec 2021 10:30:50 +0800 Subject: [PATCH 21/72] chore(HTTP): prometheus or json format by accept header --- apps/emqx_prometheus/src/emqx_prometheus_api.erl | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index ff8cff158..9e8904a4e 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -52,7 +52,13 @@ prometheus_data_api() -> Metadata = #{ get => #{ description => <<"Get Prometheus Data">>, - responses => #{<<"200">> => schema(#{type => object})} + responses => #{<<"200">> => + #{content => + #{ + 'application/json' => #{schema => #{type => object}}, + 'text/plain' => #{schema => #{type => string}} + }} + } } }, {"/prometheus/stats", Metadata, stats}. @@ -72,8 +78,12 @@ prometheus(put, #{body := Body}) -> end, {200, emqx:get_raw_config([<<"prometheus">>], #{})}. -stats(get, #{query_string := Qs}) -> - Type = maps:get(<<"format_type">>, Qs, <<"json">>), +stats(get, #{headers := Headers}) -> + Type = + case maps:get(<<"accept">>, Headers, <<"text/plain">>) of + <<"application/json">> -> <<"json">>; + _ -> <<"prometheus">> + end, Data = emqx_prometheus:collect(Type), case Type of <<"json">> -> {200, Data}; From caf1784a90e7229d59ff23481fcc4a28dc9b56b7 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 2 Dec 2021 15:25:11 +0800 Subject: [PATCH 22/72] build: update otp version --- .ci/docker-compose-file/docker-compose.yaml | 4 ++-- .github/workflows/build_packages.yaml | 14 +++++++------- .github/workflows/build_slim_packages.yaml | 6 +++--- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/run_emqx_app_tests.yaml | 4 ++-- .github/workflows/run_fvt_tests.yaml | 6 +++--- .github/workflows/run_relup_tests.yaml | 4 ++-- .github/workflows/run_test_cases.yaml | 4 ++-- .tool-versions | 2 +- deploy/docker/Dockerfile | 2 +- 10 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 2f0137428..f1cce9364 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang23: container_name: erlang23 - image: ghcr.io/emqx/emqx-builder/5.0-2:23.3.4.9-3-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/5.0-3:23.3.4.9-3-ubuntu20.04 env_file: - conf.env environment: @@ -23,7 +23,7 @@ services: erlang24: container_name: erlang24 - image: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04 env_file: - conf.env environment: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index bf972f920..77376f02c 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -19,7 +19,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04" outputs: old_vsns: ${{ steps.find_old_versons.outputs.old_vsns }} @@ -129,7 +129,7 @@ jobs: - emqx - emqx-enterprise otp: - - 24.1.5-2 + - 24.1.5-3 macos: - macos-11 - macos-10.15 @@ -215,7 +215,7 @@ jobs: - emqx - emqx-enterprise otp: - - 24.1.5-2 # we test with OTP 23, but only build package on OTP 24 versions + - 24.1.5-3 # we test with OTP 23, but only build package on OTP 24 versions arch: - amd64 - arm64 @@ -301,7 +301,7 @@ jobs: -v $(pwd):/emqx \ --workdir /emqx \ --platform linux/$ARCH \ - ghcr.io/emqx/emqx-builder/5.0-2:$OTP-$SYSTEM \ + ghcr.io/emqx/emqx-builder/5.0-3:$OTP-$SYSTEM \ bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \ make $PROFILE-pkg || cat rebar3.crashdump; \ EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh" @@ -336,7 +336,7 @@ jobs: - emqx-enterprise # NOTE: for docker, only support latest otp version, not a matrix otp: - - 24.1.5-2 # update to latest + - 24.1.5-3 # update to latest steps: - uses: actions/download-artifact@v2 @@ -377,7 +377,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp }}-alpine3.14 + BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-alpine3.14 RUN_FROM=alpine:3.14 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile @@ -405,7 +405,7 @@ jobs: - emqx - emqx-enterprise otp: - - 24.1.5-2 + - 24.1.5-3 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index da62e296b..aaa56b30b 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -24,12 +24,12 @@ jobs: - emqx - emqx-enterprise otp: - - 24.1.5-2 + - 24.1.5-3 os: - ubuntu20.04 - centos7 - container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp }}-${{ matrix.os }}" + container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}" steps: - uses: actions/checkout@v1 @@ -55,7 +55,7 @@ jobs: - emqx - emqx-enterprise otp: - - 24.1.5-2 + - 24.1.5-3 macos: - macos-11 - macos-10.15 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index c1a457665..2318b9938 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04" steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index caa4e14ad..1aba6c594 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -13,10 +13,10 @@ jobs: matrix: otp: - 23.3.4.9-3 - - 24.1.5-2 + - 24.1.5-3 runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp }}-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-ubuntu20.04" steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 45f8d8965..2ca594de6 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -14,7 +14,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14 + container: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 steps: - uses: actions/checkout@v2 @@ -55,7 +55,7 @@ jobs: - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14 + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 run: | make ${{ matrix.profile }}-docker - name: run emqx @@ -100,7 +100,7 @@ jobs: - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14 + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 run: | make ${{ matrix.profile }}-docker echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 18d995d44..3b796d476 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -19,10 +19,10 @@ jobs: - emqx - emqx-enterprise otp_vsn: - - 24.1.5-2 + - 24.1.5-3 runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp_vsn }}-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp_vsn }}-ubuntu20.04" defaults: run: diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 49e94c322..bb0fb1c82 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -16,7 +16,7 @@ jobs: strategy: matrix: emqx_builder: - - 5.0-2:24.1.5-2 # run dialyzer on latest OTP + - 5.0-3:24.1.5-3 # run dialyzer on latest OTP runs-on: ubuntu-20.04 container: "ghcr.io/emqx/emqx-builder/${{ matrix.emqx_builder }}-ubuntu20.04" @@ -32,7 +32,7 @@ jobs: strategy: matrix: emqx_builder: - - 5.0-2:24.1.5-2 + - 5.0-3:24.1.5-3 runs-on: ubuntu-20.04 container: "ghcr.io/emqx/emqx-builder/${{ matrix.emqx_builder }}-ubuntu20.04" diff --git a/.tool-versions b/.tool-versions index 6d9b11e28..a6568713b 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -erlang 24.1.5-2 +erlang 24.1.5-3 diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 5538c5a0d..a075fcf5d 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 ARG RUN_FROM=alpine:3.14 FROM ${BUILD_FROM} AS builder From 5428278802ee531ad995252d7714fcc199a6d4a5 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 1 Dec 2021 12:03:53 +0100 Subject: [PATCH 23/72] fix(systemd): add restart on failure --- deploy/packages/rpm/emqx.service | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/packages/rpm/emqx.service b/deploy/packages/rpm/emqx.service index 2acbc765a..11248a55d 100644 --- a/deploy/packages/rpm/emqx.service +++ b/deploy/packages/rpm/emqx.service @@ -10,6 +10,8 @@ Environment=HOME=/var/lib/emqx ExecStart=/bin/sh /usr/bin/emqx start LimitNOFILE=1048576 ExecStop=/bin/sh /usr/bin/emqx stop +Restart=on-failure +RestartSec=5s [Install] WantedBy=multi-user.target From 99858adb9cff499e92a19e016f17f02b5e3ee353 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 2 Dec 2021 12:11:50 +0100 Subject: [PATCH 24/72] fix(systemd-start): /usr/bin/emqx is bash not sh --- deploy/packages/deb/debian/emqx.service | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 deploy/packages/deb/debian/emqx.service diff --git a/deploy/packages/deb/debian/emqx.service b/deploy/packages/deb/debian/emqx.service new file mode 100644 index 000000000..ef9abfb01 --- /dev/null +++ b/deploy/packages/deb/debian/emqx.service @@ -0,0 +1,17 @@ +[Unit] +Description=emqx daemon +After=network.target + +[Service] +User=emqx +Group=emqx +Type=forking +Environment=HOME=/var/lib/emqx +ExecStart=/usr/bin/emqx start +LimitNOFILE=1048576 +ExecStop=/usr/bin/emqx stop +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=multi-user.target From 0f796c886abfa41b0520f169c19cdb16bc0594c2 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 2 Dec 2021 14:33:20 +0100 Subject: [PATCH 25/72] build(deb): drop systemV init --- deploy/packages/deb/debian/rules | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deploy/packages/deb/debian/rules b/deploy/packages/deb/debian/rules index 1a9fe98e2..9ca6c8e17 100755 --- a/deploy/packages/deb/debian/rules +++ b/deploy/packages/deb/debian/rules @@ -46,8 +46,10 @@ install: build cp -R releases debian/emqx/usr/lib/emqx cp -R etc/* debian/emqx/etc/emqx cp -R data/* debian/emqx/var/lib/emqx - install -m755 debian/init.script debian/emqx/etc/init.d/emqx - + + install -d debian/emqx/lib/systemd/system + install -p -m0644 debian/emqx.service debian/emqx/lib/systemd/system/emqx.service + dh_shlibdeps # We have nothing to do by default. From bf72b9f297f45a217cbde4324e5a5fde8e089920 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 2 Dec 2021 14:43:24 +0100 Subject: [PATCH 26/72] build: rpm and deb use the same systemd service file --- deploy/packages/deb/Makefile | 1 + deploy/packages/deb/emqx.service | 1 + deploy/packages/{deb/debian => }/emqx.service | 0 deploy/packages/rpm/emqx.service | 4 ++-- 4 files changed, 4 insertions(+), 2 deletions(-) create mode 120000 deploy/packages/deb/emqx.service rename deploy/packages/{deb/debian => }/emqx.service (100%) diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 1f709d860..2cb3679ee 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -13,6 +13,7 @@ TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) .PHONY: all all: | $(BUILT) cp -r debian $(SRCDIR)/ + cp emqx.service $(SRCDIR)/debian/${EMQX_NAME}.service sed -i "s##$(shell date -u '+%a, %d %b %Y %T %z')#g" $(SRCDIR)/debian/changelog sed -i "s##$(PKG_VSN)#g" $(SRCDIR)/debian/changelog sed -i "s/emqx-pkg/$(EMQX_NAME)-pkg/g" $(SRCDIR)/debian/rules; \ diff --git a/deploy/packages/deb/emqx.service b/deploy/packages/deb/emqx.service new file mode 120000 index 000000000..2fc64d79d --- /dev/null +++ b/deploy/packages/deb/emqx.service @@ -0,0 +1 @@ +../emqx.service \ No newline at end of file diff --git a/deploy/packages/deb/debian/emqx.service b/deploy/packages/emqx.service similarity index 100% rename from deploy/packages/deb/debian/emqx.service rename to deploy/packages/emqx.service diff --git a/deploy/packages/rpm/emqx.service b/deploy/packages/rpm/emqx.service index 11248a55d..ef9abfb01 100644 --- a/deploy/packages/rpm/emqx.service +++ b/deploy/packages/rpm/emqx.service @@ -7,9 +7,9 @@ User=emqx Group=emqx Type=forking Environment=HOME=/var/lib/emqx -ExecStart=/bin/sh /usr/bin/emqx start +ExecStart=/usr/bin/emqx start LimitNOFILE=1048576 -ExecStop=/bin/sh /usr/bin/emqx stop +ExecStop=/usr/bin/emqx stop Restart=on-failure RestartSec=5s From 0c0d55115b4cf2759d6ea1974a76ef353e06f003 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 2 Dec 2021 20:33:38 +0100 Subject: [PATCH 27/72] build(deb): remove System V init --- deploy/packages/deb/debian/init.script | 150 ------------------------- deploy/packages/deb/debian/postinst | 3 - deploy/packages/deb/debian/postrm | 3 - 3 files changed, 156 deletions(-) delete mode 100755 deploy/packages/deb/debian/init.script diff --git a/deploy/packages/deb/debian/init.script b/deploy/packages/deb/debian/init.script deleted file mode 100755 index 0fcafd1d2..000000000 --- a/deploy/packages/deb/debian/init.script +++ /dev/null @@ -1,150 +0,0 @@ -#! /bin/bash -### BEGIN INIT INFO -# Provides: emqx -# Required-Start: $remote_fs $syslog -# Required-Stop: $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Erlang MQTT Broker -# Description: EMQX, a distributed, massively scalable, highly extensible MQTT message broker written in Erlang/OT -### END INIT INFO - -NAME=emqx -DAEMON=/usr/bin/$NAME -SCRIPTNAME=/etc/init.d/$NAME - -# Read configuration variable file if it is present -[ -r /etc/default/$NAME ] && . /etc/default/$NAME - -# Load the VERBOSE setting and other rcS variables -. /lib/init/vars.sh -. /lib/lsb/init-functions - -# `service` strips all environmental VARS so -# if no HOME was set in /etc/default/$NAME then set one here -# to the data directory for erlexec's sake -if [ -z "$HOME" ]; then - export HOME=/var/lib/emqx -fi - -# -# Function that starts the daemon/service -# -do_start() -{ - # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started - - # Startup with the appropriate user - start-stop-daemon --start \ - --name emqx \ - --user emqx \ - --exec $DAEMON -- start \ - || return 2 -} - -# -# Function that stops the daemon/service -# -do_stop() -{ - # Identify the erts directory - ERTS_PATH=`$DAEMON ertspath` - - # Attempt a clean shutdown. - $DAEMON stop - - # waiting stop done sleep 5 - sleep 5 - - # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - # Make sure it's down by using a more direct approach - start-stop-daemon --stop \ - --quiet \ - --retry=TERM/30/KILL/5 \ - --user emqx \ - --exec $ERTS_PATH/run_erl - return $? -} - -# -# Function that graceful reload the daemon/service -# -do_reload() { - # Restart the VM without exiting the process - $DAEMON restart && return $? || return 2 -} - -# Checks the status of a node -do_status() { - $DAEMON ping && echo $"$NAME is running" && return 0 - echo $"$NAME is stopped" && return 2 -} - -case "$1" in - start) - log_daemon_msg "Starting $NAME" - $DAEMON ping >/dev/null 2>&1 && echo $"$NAME is already running" && exit 0 - do_start - case "$?" in - 0|1) log_end_msg 0 ;; - 2) log_end_msg 1 - exit 1 - ;; - esac - ;; - stop) - log_daemon_msg "Stopping $NAME" - do_stop - case "$?" in - 0|1) log_end_msg 0 ;; - 2) log_end_msg 1 - exit 1 - ;; - esac - ;; - ping) - # See if the VM is alive - $DAEMON ping || exit $? - ;; - reload|force-reload) - log_daemon_msg "Reloading $NAME" - do_reload - ES=$? - log_end_msg $ES - exit $ES - ;; - restart) - log_daemon_msg "Restarting $NAME" - do_stop - case "$?" in - 0|1) - do_start - case "$?" in - 0) log_end_msg 0 ;; - 1) log_end_msg 1 && exit 1 ;; # Old process is still running - *) log_end_msg 1 && exit 1 ;; # Failed to start - esac - ;; - *) - # Failed to stop - log_end_msg 1 && exit 1 - ;; - esac - ;; - status) - do_status && exit 0 || exit $? - ;; - *) - echo "Usage: $SCRIPTNAME {start|stop|ping|restart|force-reload|status}" >&2 - exit 3 - ;; -esac - - diff --git a/deploy/packages/deb/debian/postinst b/deploy/packages/deb/debian/postinst index 3c114db64..1a8dd3be8 100755 --- a/deploy/packages/deb/debian/postinst +++ b/deploy/packages/deb/debian/postinst @@ -5,9 +5,6 @@ set -e -# install startup script -update-rc.d emqx defaults >/dev/null - # create group if ! getent group emqx >/dev/null; then addgroup --system emqx diff --git a/deploy/packages/deb/debian/postrm b/deploy/packages/deb/debian/postrm index 9d6072f58..caea4a520 100755 --- a/deploy/packages/deb/debian/postrm +++ b/deploy/packages/deb/debian/postrm @@ -23,9 +23,6 @@ case "$1" in purge) rm -f /etc/default/emqx - # ensure we remove the rc.d scripts installed by postinst - update-rc.d emqx remove >/dev/null - if [ -d /var/lib/emqx ]; then rm -r /var/lib/emqx fi From 4402327da0cb16b6a8a066467c24717cc47e351f Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 2 Dec 2021 22:46:22 +0100 Subject: [PATCH 28/72] ci(deb): remove system V test --- .ci/build_packages/tests.sh | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index a0f85fa22..c4b752da1 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -175,26 +175,6 @@ EOF cat /var/log/emqx/emqx.log.1 || true exit 1 fi - - if [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = ubuntu ] \ - || [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = debian ] ;then - if ! service emqx start; then - cat /var/log/emqx/erlang.log.1 || true - cat /var/log/emqx/emqx.log.1 || true - exit 1 - fi - IDLE_TIME=0 - while ! curl http://127.0.0.1:18083/api/v5/status >/dev/null 2>&1; do - if [ $IDLE_TIME -gt 10 ] - then - echo "emqx service error" - exit 1 - fi - sleep 10 - IDLE_TIME=$((IDLE_TIME+1)) - done - service emqx stop - fi } relup_test(){ From 858891dbf53b4f86f7059fae7cbd11119b0b1e4d Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 3 Dec 2021 09:54:44 +0800 Subject: [PATCH 29/72] fix: rewrite ct badmatch --- apps/emqx/test/emqx_trace_handler_SUITE.erl | 1 + apps/emqx_modules/test/emqx_rewrite_SUITE.erl | 11 +++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx/test/emqx_trace_handler_SUITE.erl b/apps/emqx/test/emqx_trace_handler_SUITE.erl index e010b6c5e..950f0faf6 100644 --- a/apps/emqx/test/emqx_trace_handler_SUITE.erl +++ b/apps/emqx/test/emqx_trace_handler_SUITE.erl @@ -199,6 +199,7 @@ t_trace_ip_address(_Config) -> ?assertEqual([], emqx_trace_handler:running()). filesync(Name, Type) -> + ct:sleep(50), filesync(Name, Type, 3). %% sometime the handler process is not started yet. diff --git a/apps/emqx_modules/test/emqx_rewrite_SUITE.erl b/apps/emqx_modules/test/emqx_rewrite_SUITE.erl index 4d51834c2..713468460 100644 --- a/apps/emqx_modules/test/emqx_rewrite_SUITE.erl +++ b/apps/emqx_modules/test/emqx_rewrite_SUITE.erl @@ -168,12 +168,11 @@ t_update_re_failed(_Config) -> }], Error = {badmatch, {error, - {error, - {emqx_modules_schema, - [{validation_error, - #{array_index => 1,path => "rewrite.re", - reason => {<<"*^test/*">>,{"nothing to repeat",0}}, - value => <<"*^test/*">>}}]}}}}, + {emqx_modules_schema, + [{validation_error, + #{array_index => 1,path => "rewrite.re", + reason => {<<"*^test/*">>,{"nothing to repeat",0}}, + value => <<"*^test/*">>}}]}}}, ?assertError(Error, emqx_rewrite:update(Rules)), ok. From e1bcbd012ca3aa3d7693ef1b53441a500741b61b Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 25 Nov 2021 11:11:38 +0800 Subject: [PATCH 30/72] feat(authn): support sync configuration in the cluster --- apps/emqx_authn/src/emqx_authn_api.erl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 83ca9e56d..827b4e09c 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -793,9 +793,10 @@ add_user(ChainName, AuthenticatorID, #{<<"user_id">> := UserID, <<"password">> := Password} = UserInfo) -> IsSuperuser = maps:get(<<"is_superuser">>, UserInfo, false), - case emqx_authentication:add_user(ChainName, AuthenticatorID, #{ user_id => UserID - , password => Password - , is_superuser => IsSuperuser}) of + case emqx_authentication:add_user(ChainName, AuthenticatorID, + #{ user_id => UserID + , password => Password + , is_superuser => IsSuperuser}) of {ok, User} -> {201, User}; {error, Reason} -> @@ -845,7 +846,8 @@ list_users(ChainName, AuthenticatorID, PageParams) -> end. update_config(Path, ConfigRequest) -> - emqx:update_config(Path, ConfigRequest, #{rawconf_with_defaults => true}). + emqx_conf:update(Path, ConfigRequest, #{rawconf_with_defaults => true, + override_to => cluster}). get_raw_config_with_defaults(ConfKeyPath) -> NConfKeyPath = [atom_to_binary(Key, utf8) || Key <- ConfKeyPath], From a7cd1ad30be7ad27b5f0e84392d141c34c148651 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 24 Nov 2021 18:49:13 +0800 Subject: [PATCH 31/72] fix(mgmt): node memory usage info --- apps/emqx_management/src/emqx_mgmt.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index f82b881a8..d45b5ad77 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -149,7 +149,7 @@ node_info(Node) when Node =:= node() -> Info#{node => node(), otp_release => iolist_to_binary(otp_rel()), memory_total => proplists:get_value(allocated, Memory), - memory_used => proplists:get_value(total, Memory), + memory_used => proplists:get_value(used, Memory), process_available => erlang:system_info(process_limit), process_used => erlang:system_info(process_count), @@ -650,4 +650,3 @@ max_row_limit() -> ?MAX_ROW_LIMIT. table_size(Tab) -> ets:info(Tab, size). - From d4246ad2f60870839f929ea85ef673ddc5a774bb Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 25 Nov 2021 14:41:40 +0800 Subject: [PATCH 32/72] fix(vm): add literal_alloc memory calculation --- apps/emqx/src/emqx_vm.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/src/emqx_vm.erl b/apps/emqx/src/emqx_vm.erl index 06e17513b..91b3e54f9 100644 --- a/apps/emqx/src/emqx_vm.erl +++ b/apps/emqx/src/emqx_vm.erl @@ -57,6 +57,7 @@ sl_alloc, ll_alloc, fix_alloc, + literal_alloc, std_alloc ]). From 4b3b29873a390849b62438a365619208052928c1 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 3 Dec 2021 14:41:42 +0800 Subject: [PATCH 33/72] feat(retainer): allow to stop publish clear msg --- apps/emqx_retainer/etc/emqx_retainer.conf | 7 +++++++ apps/emqx_retainer/src/emqx_retainer.erl | 10 +++++++++- apps/emqx_retainer/src/emqx_retainer_schema.erl | 1 + 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/apps/emqx_retainer/etc/emqx_retainer.conf b/apps/emqx_retainer/etc/emqx_retainer.conf index ba6bdfa6c..92dc62f24 100644 --- a/apps/emqx_retainer/etc/emqx_retainer.conf +++ b/apps/emqx_retainer/etc/emqx_retainer.conf @@ -29,6 +29,13 @@ emqx_retainer { ## Default: 0s msg_expiry_interval = 0s + ## When the retained flag of the PUBLISH message is set and Payload is empty, + ## whether to continue to publish the message. + ## see: http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html#_Toc398718038 + ## + ## Default: false + #stop_publish_clear_msg = false + ## The message read and deliver flow rate control ## When a client subscribe to a wildcard topic, may many retained messages will be loaded. ## If you don't want these data loaded to the memory all at once, you can use this to control. diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 5d248e638..9be449b60 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -88,7 +88,12 @@ on_message_publish(Msg = #message{flags = #{retain := true}, payload = <<>>}, Context) -> delete_message(Context, Topic), - {ok, Msg}; + case get_stop_publish_clear_msg() of + true -> + {ok, emqx_message:set_header(allow_publish, false, Msg)}; + _ -> + {ok, Msg} + end; on_message_publish(Msg = #message{flags = #{retain := true}}, Context) -> Msg1 = emqx_message:set_header(retained, true, Msg), @@ -157,6 +162,9 @@ get_expiry_time(#message{timestamp = Ts}) -> _ -> Ts + Interval end. +get_stop_publish_clear_msg() -> + emqx_conf:get([?APP, stop_publish_clear_msg], false). + -spec update_config(hocon:config()) -> ok. update_config(Conf) -> gen_server:call(?MODULE, {?FUNCTION_NAME, Conf}). diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 55cfa2fcc..e1fa8373a 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -14,6 +14,7 @@ fields("emqx_retainer") -> , {msg_clear_interval, sc(emqx_schema:duration_ms(), "0s")} , {flow_control, ?TYPE(hoconsc:ref(?MODULE, flow_control))} , {max_payload_size, sc(emqx_schema:bytesize(), "1MB")} + , {stop_publish_clear_msg, sc(boolean(), false)} , {config, config()} ]; From f645a4eada91292b1b6ecd0f7914c107a78ea863 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 3 Dec 2021 14:42:27 +0800 Subject: [PATCH 34/72] test(retainer): add test case for stopping publish clear msg --- apps/emqx_retainer/test/emqx_retainer_SUITE.erl | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl index ccc647ddc..5596e9539 100644 --- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl @@ -195,6 +195,21 @@ t_clean(_) -> ok = emqtt:disconnect(C1). +t_stop_publish_clear_msg(_) -> + emqx_retainer:update_config(#{<<"stop_publish_clear_msg">> => true}), + {ok, C1} = emqtt:start_link([{clean_start, true}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(C1), + emqtt:publish(C1, <<"retained/0">>, <<"this is a retained message 0">>, [{qos, 0}, {retain, true}]), + + {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained/#">>, [{qos, 0}, {rh, 0}]), + ?assertEqual(1, length(receive_messages(1))), + + emqtt:publish(C1, <<"retained/0">>, <<"">>, [{qos, 0}, {retain, true}]), + ?assertEqual(0, length(receive_messages(1))), + + emqx_retainer:update_config(#{<<"stop_publish_clear_msg">> => false}), + ok = emqtt:disconnect(C1). + t_flow_control(_) -> emqx_retainer:update_config(#{<<"flow_control">> => #{<<"max_read_number">> => 1, <<"msg_deliver_quota">> => 1, From 1dfc37cd182d74643151cc52972536623361408a Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 3 Dec 2021 15:55:34 +0800 Subject: [PATCH 35/72] test(authn): fix test cases --- apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 2 ++ apps/emqx_authn/test/emqx_authn_http_SUITE.erl | 2 ++ apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl | 5 +++++ apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl | 2 ++ apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl | 4 ++-- apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl | 2 ++ apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl | 2 ++ apps/emqx_authn/test/emqx_authn_redis_SUITE.erl | 4 ++-- .../test/emqx_enhanced_authn_scram_mnesia_SUITE.erl | 2 ++ 9 files changed, 21 insertions(+), 4 deletions(-) diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index c93bec582..ba9f4996a 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -43,6 +43,7 @@ groups() -> []. init_per_testcase(_, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL), @@ -55,6 +56,7 @@ init_per_testcase(_, Config) -> Config. init_per_suite(Config) -> + _ = application:load(emqx_conf), ok = emqx_common_test_helpers:start_apps( [emqx_authn, emqx_dashboard], fun set_special_configs/1), diff --git a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl index 4a966fe0e..67865d645 100644 --- a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl @@ -39,6 +39,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + _ = application:load(emqx_conf), emqx_common_test_helpers:start_apps([emqx_authn]), application:ensure_all_started(cowboy), Config. @@ -52,6 +53,7 @@ end_per_suite(_) -> ok. init_per_testcase(_Case, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL), diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl index fe730acb0..0a166616a 100644 --- a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -34,7 +34,12 @@ all() -> emqx_common_test_helpers:all(?MODULE). +init_per_testcase(_, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), + Config. + init_per_suite(Config) -> + _ = application:load(emqx_conf), emqx_common_test_helpers:start_apps([emqx_authn]), Config. diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index b9eadbef6..5c6eb694c 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -29,6 +29,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + _ = application:load(emqx_conf), emqx_common_test_helpers:start_apps([emqx_authn]), Config. @@ -37,6 +38,7 @@ end_per_suite(_) -> ok. init_per_testcase(_Case, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), mria:clear_table(emqx_authn_mnesia), Config. diff --git a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl index e6ae1706a..562c5aa1b 100644 --- a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl @@ -33,8 +33,8 @@ all() -> emqx_common_test_helpers:all(?MODULE). - init_per_testcase(_TestCase, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authentication:initialize_authentication(?GLOBAL, []), emqx_authn_test_lib:delete_authenticators( [authentication], @@ -45,8 +45,8 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> ok = mc_worker_api:disconnect(?MONGO_CLIENT). - init_per_suite(Config) -> + _ = application:load(emqx_conf), case emqx_authn_test_lib:is_tcp_server_available(?MONGO_HOST, ?MONGO_PORT) of true -> ok = emqx_common_test_helpers:start_apps([emqx_authn]), diff --git a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl index 48569ed36..bf66b034a 100644 --- a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl @@ -38,6 +38,7 @@ groups() -> [{require_seeds, [], [t_authenticate, t_update, t_destroy]}]. init_per_testcase(_, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authentication:initialize_authentication(?GLOBAL, []), emqx_authn_test_lib:delete_authenticators( [authentication], @@ -53,6 +54,7 @@ end_per_group(require_seeds, Config) -> Config. init_per_suite(Config) -> + _ = application:load(emqx_conf), case emqx_authn_test_lib:is_tcp_server_available(?MYSQL_HOST, ?MYSQL_PORT) of true -> ok = emqx_common_test_helpers:start_apps([emqx_authn]), diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl index e3848afbc..2a79179e1 100644 --- a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl @@ -38,6 +38,7 @@ groups() -> [{require_seeds, [], [t_authenticate, t_update, t_destroy, t_is_superuser]}]. init_per_testcase(_, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authentication:initialize_authentication(?GLOBAL, []), emqx_authn_test_lib:delete_authenticators( [authentication], @@ -53,6 +54,7 @@ end_per_group(require_seeds, Config) -> Config. init_per_suite(Config) -> + _ = application:load(emqx_conf), case emqx_authn_test_lib:is_tcp_server_available(?PGSQL_HOST, ?PGSQL_PORT) of true -> ok = emqx_common_test_helpers:start_apps([emqx_authn]), diff --git a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl index 02f3e0b7c..2e941e72f 100644 --- a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl @@ -23,12 +23,10 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). - -define(REDIS_HOST, "redis"). -define(REDIS_PORT, 6379). -define(REDIS_RESOURCE, <<"emqx_authn_redis_SUITE">>). - -define(PATH, [authentication]). all() -> @@ -38,6 +36,7 @@ groups() -> [{require_seeds, [], [t_authenticate, t_update, t_destroy]}]. init_per_testcase(_, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authentication:initialize_authentication(?GLOBAL, []), emqx_authn_test_lib:delete_authenticators( [authentication], @@ -53,6 +52,7 @@ end_per_group(require_seeds, Config) -> Config. init_per_suite(Config) -> + _ = application:load(emqx_conf), case emqx_authn_test_lib:is_tcp_server_available(?REDIS_HOST, ?REDIS_PORT) of true -> ok = emqx_common_test_helpers:start_apps([emqx_authn]), diff --git a/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl index 3504b88cc..b63e6638a 100644 --- a/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl @@ -33,6 +33,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + _ = application:load(emqx_conf), ok = emqx_common_test_helpers:start_apps([emqx_authn]), Config. @@ -40,6 +41,7 @@ end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_authn]). init_per_testcase(_Case, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), mria:clear_table(emqx_enhanced_authn_scram_mnesia), emqx_authn_test_lib:delete_authenticators( [authentication], From 2ceb660344215df48f20300918ae5cbbfbabb3f4 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Tue, 30 Nov 2021 22:06:33 +0100 Subject: [PATCH 36/72] chore(mria): Bump version to 0.1.4 --- apps/emqx_conf/src/emqx_conf_schema.erl | 6 +----- rebar.config | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index bd897d730..91ef5d812 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -355,19 +355,15 @@ to rlog and the role is set to replicant. , default => gen_rpc , desc => """ Protocol used for pushing transaction logs to the replicant nodes. -Important! This setting should be the same on all nodes in the cluster.
-Important! Changing this setting in the runtime is not allowed.
""" })} , {"tlog_push_mode", sc(hoconsc:enum([sync, async]), #{ mapping => "mria.tlog_push_mode" - , default => sync + , default => async , desc => """ In sync mode the core node waits for an ack from the replicant nodes before sending the next transaction log entry. -Important! This setting should be the same on all nodes in the cluster.
-Important! Changing this setting in the runtime is not allowed.
""" })} ]; diff --git a/rebar.config b/rebar.config index 9e7aba87c..4fbf1b136 100644 --- a/rebar.config +++ b/rebar.config @@ -52,7 +52,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.3"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}} - , {mria, {git, "https://github.com/emqx/mria", {tag, "0.1.3"}}} + , {mria, {git, "https://github.com/emqx/mria", {tag, "0.1.4"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.7"}}} From fd9f6bd14014e595f4b023706d2e66cf06fad1ea Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Fri, 3 Dec 2021 17:15:03 +0100 Subject: [PATCH 37/72] fix(crash_dump): Replace crash dump directory with a file --- apps/emqx_conf/etc/emqx_conf.conf | 6 +++--- apps/emqx_conf/src/emqx_conf_schema.erl | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_conf/etc/emqx_conf.conf b/apps/emqx_conf/etc/emqx_conf.conf index e57bc4869..bbfb91552 100644 --- a/apps/emqx_conf/etc/emqx_conf.conf +++ b/apps/emqx_conf/etc/emqx_conf.conf @@ -27,12 +27,12 @@ node { ## Default: "{{ platform_data_dir }}/" data_dir = "{{ platform_data_dir }}/" - ## Dir of crash dump file. + ## Location of crash dump file. ## - ## @doc node.crash_dump_dir + ## @doc node.crash_dump_file ## ValueType: Folder ## Default: "{{ platform_log_dir }}/" - crash_dump_dir = "{{ platform_log_dir }}/" + crash_dump_file = "{{ platform_log_dir }}/erl_crash.dump" ## Global GC Interval. ## diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 91ef5d812..caec61cc5 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -275,7 +275,7 @@ fields("node") -> #{ mapping => "emqx_machine.global_gc_interval" , default => "15m" })} - , {"crash_dump_dir", + , {"crash_dump_file", sc(file(), #{ mapping => "vm_args.-env ERL_CRASH_DUMP" })} From dfa45c909a7618f6779df23be1bdfb02ada96930 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Fri, 3 Dec 2021 18:17:45 +0100 Subject: [PATCH 38/72] fix(crash_dump): Fix hocon schema related to the erlang crash dump --- apps/emqx_conf/etc/emqx_conf.conf | 19 +++++++++++++++++-- apps/emqx_conf/src/emqx_conf_schema.erl | 16 ++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/apps/emqx_conf/etc/emqx_conf.conf b/apps/emqx_conf/etc/emqx_conf.conf index bbfb91552..ee869f3a3 100644 --- a/apps/emqx_conf/etc/emqx_conf.conf +++ b/apps/emqx_conf/etc/emqx_conf.conf @@ -30,10 +30,25 @@ node { ## Location of crash dump file. ## ## @doc node.crash_dump_file - ## ValueType: Folder - ## Default: "{{ platform_log_dir }}/" + ## ValueType: File + ## Default: "{{ platform_log_dir }}/erl_crash.dump" crash_dump_file = "{{ platform_log_dir }}/erl_crash.dump" + ## The number of seconds that the broker is allowed to spend writing + ## a crash dump + ## + ## @doc node.crash_dump_seconds + ## ValueType: seconds + ## Default: 30s + crash_dump_seconds = 30s + + ## The maximum size of a crash dump file in bytes. + ## + ## @doc node.crash_dump_bytes + ## ValueType: bytes + ## Default: 100MB + crash_dump_bytes = 100MB + ## Global GC Interval. ## ## @doc node.global_gc_interval diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index caec61cc5..617703cd2 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -278,6 +278,22 @@ fields("node") -> , {"crash_dump_file", sc(file(), #{ mapping => "vm_args.-env ERL_CRASH_DUMP" + , desc => "Location of the crash dump file" + })} + , {"crash_dump_seconds", + sc(emqx_schema:duration_s(), + #{ mapping => "vm_args.-env ERL_CRASH_DUMP_SECONDS" + , default => "30s" + , desc => """ +The number of seconds that the broker is allowed to spend writing +a crash dump +""" + })} + , {"crash_dump_bytes", + sc(emqx_schema:bytesize(), + #{ mapping => "vm_args.-env ERL_CRASH_DUMP_BYTES" + , default => "100MB" + , desc => "The maximum size of a crash dump file in bytes." })} , {"dist_net_ticktime", sc(emqx_schema:duration(), From bc8a0d7060f5af4c7b3d3dce9adf09e10630571b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 3 Dec 2021 15:25:51 -0300 Subject: [PATCH 39/72] fix(listeners): fix typo in listener type `emqx_listeners:{current,max}_conns` were matching on type `tcl`. However, this type doesn't exist (it's not defined in `?TYPES_STRING`). Therefore, this clause would never match. It seems that the intention was that it shouldbe `tcp`. --- apps/emqx/src/emqx_listeners.erl | 4 +-- apps/emqx/test/emqx_channel_SUITE.erl | 4 +-- apps/emqx/test/emqx_listeners_SUITE.erl | 33 ++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 2b9a76fe3..2869dfc75 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -111,7 +111,7 @@ current_conns(ID, ListenOn) -> {Type, Name} = parse_listener_id(ID), current_conns(Type, Name, ListenOn). -current_conns(Type, Name, ListenOn) when Type == tcl; Type == ssl -> +current_conns(Type, Name, ListenOn) when Type == tcp; Type == ssl -> esockd:get_current_connections({listener_id(Type, Name), ListenOn}); current_conns(Type, Name, _ListenOn) when Type =:= ws; Type =:= wss -> proplists:get_value(all_connections, ranch:info(listener_id(Type, Name))); @@ -122,7 +122,7 @@ max_conns(ID, ListenOn) -> {Type, Name} = parse_listener_id(ID), max_conns(Type, Name, ListenOn). -max_conns(Type, Name, ListenOn) when Type == tcl; Type == ssl -> +max_conns(Type, Name, ListenOn) when Type == tcp; Type == ssl -> esockd:get_max_connections({listener_id(Type, Name), ListenOn}); max_conns(Type, Name, _ListenOn) when Type =:= ws; Type =:= wss -> proplists:get_value(max_connections, ranch:info(listener_id(Type, Name))); diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 284cea784..5a626ed36 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -132,7 +132,7 @@ basic_conf() -> zones => zone_conf() }. -set_test_listenser_confs() -> +set_test_listener_confs() -> Conf = emqx_config:get([]), emqx_config:put(basic_conf()), Conf. @@ -179,7 +179,7 @@ end_per_suite(_Config) -> ]). init_per_testcase(_TestCase, Config) -> - NewConf = set_test_listenser_confs(), + NewConf = set_test_listener_confs(), [{config, NewConf}|Config]. end_per_testcase(_TestCase, Config) -> diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index 649fc0065..67e6154a3 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -22,6 +22,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). @@ -37,10 +38,33 @@ end_per_suite(_Config) -> application:stop(esockd), application:stop(cowboy). +init_per_testcase(Case, Config) + when Case =:= t_max_conns_tcp; Case =:= t_current_conns_tcp -> + {ok, _} = emqx_config_handler:start_link(), + PrevListeners = emqx_config:get([listeners, tcp], #{}), + emqx_config:put([listeners, tcp], #{ listener_test => + #{ bind => {"127.0.0.1", 9999} + , max_connections => 4321 + } + }), + emqx_config:put([rate_limit], #{max_conn_rate => 1000}), + ListenerConf = #{ bind => {"127.0.0.1", 9999} + }, + ok = emqx_listeners:start(), + [ {listener_conf, ListenerConf} + , {prev_listener_conf, PrevListeners} + | Config]; init_per_testcase(_, Config) -> {ok, _} = emqx_config_handler:start_link(), Config. +end_per_testcase(Case, Config) + when Case =:= t_max_conns_tcp; Case =:= t_current_conns_tcp -> + PrevListener = ?config(prev_listener_conf, Config), + emqx_config:put([listeners, tcp], PrevListener), + emqx_listeners:stop(), + _ = emqx_config_handler:stop(), + ok; end_per_testcase(_, _Config) -> _ = emqx_config_handler:stop(), ok. @@ -56,6 +80,14 @@ t_restart_listeners(_) -> ok = emqx_listeners:restart(), ok = emqx_listeners:stop(). +t_max_conns_tcp(_) -> + %% Note: Using a string representation for the bind address like + %% "127.0.0.1" does not work + ?assertEqual(4321, emqx_listeners:max_conns('tcp:listener_test', {{127,0,0,1}, 9999})). + +t_current_conns_tcp(_) -> + ?assertEqual(0, emqx_listeners:current_conns('tcp:listener_test', {{127,0,0,1}, 9999})). + render_config_file() -> Path = local_path(["etc", "emqx.conf"]), {ok, Temp} = file:read_file(Path), @@ -101,4 +133,3 @@ get_base_dir(Module) -> get_base_dir() -> get_base_dir(?MODULE). - From 5d3cb6ae1cc937808cbb61fd1cb9f485613d7447 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 29 Nov 2021 14:51:49 +0100 Subject: [PATCH 40/72] refactor(authn): align authn config root name authn configs are checked independently per-auth provider, this was to make authn providers more plugable. in order to make environment variable overrides work for authn, we need to have a unified view of the config layout, no matter from root level, or partially checking per-provider config independently, i.e. we try to use the same config envelop. --- apps/emqx/include/emqx_authentication.hrl | 31 +++++++++++++++++ apps/emqx/src/emqx_authentication.erl | 11 +++--- apps/emqx/src/emqx_authentication_config.erl | 20 ++++++----- apps/emqx/src/emqx_config.erl | 7 ++-- apps/emqx/src/emqx_schema.erl | 22 +++++++++--- apps/emqx/test/emqx_authentication_SUITE.erl | 27 ++++++--------- apps/emqx_authn/etc/emqx_authn.conf | 7 +--- apps/emqx_authn/include/emqx_authn.hrl | 7 ++++ apps/emqx_authn/src/emqx_authn.erl | 17 +++++++--- apps/emqx_authn/src/emqx_authn_api.erl | 7 ++-- apps/emqx_authn/src/emqx_authn_app.erl | 6 ++-- apps/emqx_authn/src/emqx_authn_schema.erl | 22 ++++++++++-- .../emqx_enhanced_authn_scram_mnesia.erl | 10 +++--- .../src/simple_authn/emqx_authn_http.erl | 20 ++++++----- .../src/simple_authn/emqx_authn_jwt.erl | 9 ++--- .../src/simple_authn/emqx_authn_mnesia.erl | 12 +++---- .../src/simple_authn/emqx_authn_mongodb.erl | 8 ++--- .../src/simple_authn/emqx_authn_mysql.erl | 10 +++--- .../src/simple_authn/emqx_authn_pgsql.erl | 10 +++--- .../src/simple_authn/emqx_authn_redis.erl | 14 ++++---- .../test/emqx_authn_mnesia_SUITE.erl | 6 ++-- apps/emqx_conf/src/emqx_conf_schema.erl | 6 ++++ .../src/coap/emqx_coap_channel.erl | 11 +++--- apps/emqx_gateway/src/emqx_gateway_api.erl | 5 +-- apps/emqx_gateway/src/emqx_gateway_conf.erl | 34 ++++++++++--------- apps/emqx_gateway/src/emqx_gateway_http.erl | 7 ++-- apps/emqx_gateway/src/emqx_gateway_schema.erl | 33 +++++------------- 27 files changed, 231 insertions(+), 148 deletions(-) create mode 100644 apps/emqx/include/emqx_authentication.hrl diff --git a/apps/emqx/include/emqx_authentication.hrl b/apps/emqx/include/emqx_authentication.hrl new file mode 100644 index 000000000..948842433 --- /dev/null +++ b/apps/emqx/include/emqx_authentication.hrl @@ -0,0 +1,31 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_AUTHENTICATION_HRL). +-define(EMQX_AUTHENTICATION_HRL, true). + +%% config root name all auth providers have to agree on. +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, "authentication"). +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM, authentication). +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, <<"authentication">>). + +%% key to a persistent term which stores a module name in order to inject +%% schema module at run-time to keep emqx app's compile time purity. +%% see emqx_schema.erl for more details +%% and emqx_conf_schema for an examples +-define(EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, emqx_authentication_schema_module). + +-endif. diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 77a5e2cee..1607497da 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -24,9 +24,12 @@ -include("emqx.hrl"). -include("logger.hrl"). +-include("emqx_authentication.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). +-define(CONF_ROOT, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). + %% The authentication entrypoint. -export([ authenticate/2 ]). @@ -383,8 +386,8 @@ list_users(ChainName, AuthenticatorID, Params) -> %%-------------------------------------------------------------------- init(_Opts) -> - ok = emqx_config_handler:add_handler([authentication], ?MODULE), - ok = emqx_config_handler:add_handler([listeners, '?', '?', authentication], ?MODULE), + ok = emqx_config_handler:add_handler([?CONF_ROOT], ?MODULE), + ok = emqx_config_handler:add_handler([listeners, '?', '?', ?CONF_ROOT], ?MODULE), {ok, #{hooked => false, providers => #{}}}. handle_call(get_providers, _From, #{providers := Providers} = State) -> @@ -496,8 +499,8 @@ terminate(Reason, _State) -> Other -> ?SLOG(error, #{msg => "emqx_authentication_terminating", reason => Other}) end, - emqx_config_handler:remove_handler([authentication]), - emqx_config_handler:remove_handler([listeners, '?', '?', authentication]), + emqx_config_handler:remove_handler([?CONF_ROOT]), + emqx_config_handler:remove_handler([listeners, '?', '?', ?CONF_ROOT]), ok. code_change(_OldVsn, State, _Extra) -> diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index a7fa5673a..795dd060e 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -34,6 +34,7 @@ -export_type([config/0]). -include("logger.hrl"). +-include("emqx_authentication.hrl"). -type parsed_config() :: #{mechanism := atom(), backend => atom(), @@ -132,9 +133,9 @@ do_post_config_update({move_authenticator, ChainName, AuthenticatorID, Position} check_configs(Configs) -> Providers = emqx_authentication:get_providers(), - lists:map(fun(C) -> do_check_conifg(C, Providers) end, Configs). + lists:map(fun(C) -> do_check_config(C, Providers) end, Configs). -do_check_conifg(Config, Providers) -> +do_check_config(Config, Providers) -> Type = authn_type(Config), case maps:get(Type, Providers, false) of false -> @@ -143,19 +144,20 @@ do_check_conifg(Config, Providers) -> providers => Providers}), throw({unknown_authn_type, Type}); Module -> - do_check_conifg(Type, Config, Module) + do_check_config(Type, Config, Module) end. -do_check_conifg(Type, Config, Module) -> +do_check_config(Type, Config, Module) -> F = case erlang:function_exported(Module, check_config, 1) of true -> fun Module:check_config/1; false -> fun(C) -> - #{config := R} = - hocon_schema:check_plain(Module, #{<<"config">> => C}, + Key = list_to_binary(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME), + AtomKey = list_to_atom(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME), + R = hocon_schema:check_plain(Module, #{Key => C}, #{atom_key => true}), - R + maps:get(AtomKey, R) end end, try @@ -261,8 +263,8 @@ authn_type(#{mechanism := M}) -> atom(M); authn_type(#{<<"mechanism">> := M, <<"backend">> := B}) -> {atom(M), atom(B)}; authn_type(#{<<"mechanism">> := M}) -> atom(M). -atom(Bin) -> - binary_to_existing_atom(Bin, utf8). +atom(A) when is_atom(A) -> A; +atom(Bin) -> binary_to_existing_atom(Bin, utf8). %% The relative dir for ssl files. certs_dir(ChainName, ConfigOrID) -> diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 8806d4bc3..2693d559e 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -268,12 +268,13 @@ init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) -> }), error(failed_to_load_hocon_conf) end; -init_load(SchemaMod, RawConf0) when is_map(RawConf0) -> +init_load(SchemaMod, RawConf) when is_map(RawConf) -> ok = save_schema_mod_and_names(SchemaMod), %% check and save configs - {_AppEnvs, CheckedConf} = check_config(SchemaMod, RawConf0), + {_AppEnvs, CheckedConf} = check_config(SchemaMod, RawConf), + RootNames = get_root_names(), ok = save_to_config_map(maps:with(get_atom_root_names(), CheckedConf), - maps:with(get_root_names(), RawConf0)). + maps:with(RootNames, RawConf)). include_dirs() -> [filename:join(emqx:data_dir(), "configs")]. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index e470ff478..c137f4a37 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -22,6 +22,7 @@ -dialyzer(no_unused). -dialyzer(no_fail_call). +-include("emqx_authentication.hrl"). -include_lib("typerefl/include/types.hrl"). -type duration() :: integer(). @@ -105,7 +106,7 @@ and can not be deleted.""" The configs here work as default values which can be overriden in zone configs""" })} - , {"authentication", + , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication( """Default authentication configs for all MQTT listeners.
For per-listener overrides see authentication @@ -972,7 +973,7 @@ mqtt_listener() -> sc(duration(), #{}) } - , {"authentication", + , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication("Per-listener authentication override") } ]. @@ -1436,8 +1437,21 @@ str(S) when is_list(S) -> S. authentication(Desc) -> - #{ type => hoconsc:lazy(hoconsc:union([typerefl:map(), hoconsc:array(typerefl:map())])) - , desc => iolist_to_binary([Desc, "
", """ + %% authentication schemais lazy to make it more 'plugable' + %% the type checks are done in emqx_auth application when it boots. + %% and in emqx_authentication_config module for rutime changes. + Default = hoconsc:lazy(hoconsc:union([typerefl:map(), hoconsc:array(typerefl:map())])), + %% as the type is lazy, the runtime module injection from EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY + %% is for now only affecting document generation. + %% maybe in the future, we can find a more straightforward way to support + %% * document generation (at compile time) + %% * type checks before boot (in bin/emqx config generation) + %% * type checks at runtime (when changing configs via management API) + #{ type => case persistent_term:get(?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, undefined) of + undefined -> Default; + Module -> hoconsc:lazy(Module:root_type()) + end + , desc => iolist_to_binary([Desc, """ Authentication can be one single authenticator instance or a chain of authenticators as an array. When authenticating a login (username, client ID, etc.) the authenticators are checked in the configured order.
diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index 10a4e4091..2c28a6076 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -25,18 +25,11 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("typerefl/include/types.hrl"). - --export([ roots/0, fields/1 ]). - --export([ create/2 - , update/2 - , authenticate/2 - , destroy/1 - , check_config/1 - ]). +-include("emqx_authentication.hrl"). -define(AUTHN, emqx_authentication). -define(config(KEY), (fun() -> {KEY, _V_} = lists:keyfind(KEY, 1, Config), _V_ end)()). +-define(CONF_ROOT, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). %%------------------------------------------------------------------------------ %% Hocon Schema @@ -250,7 +243,7 @@ t_update_config({init, Config}) -> {"auth2", AuthNType2} | Config]; t_update_config(Config) when is_list(Config) -> - emqx_config_handler:add_handler([authentication], emqx_authentication), + emqx_config_handler:add_handler([?CONF_ROOT], emqx_authentication), ok = register_provider(?config("auth1"), ?MODULE), ok = register_provider(?config("auth2"), ?MODULE), Global = ?config(global), @@ -267,7 +260,7 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch( {ok, _}, - update_config([authentication], {create_authenticator, Global, AuthenticatorConfig1})), + update_config([?CONF_ROOT], {create_authenticator, Global, AuthenticatorConfig1})), ?assertMatch( {ok, #{id := ID1, state := #{mark := 1}}}, @@ -275,7 +268,7 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch( {ok, _}, - update_config([authentication], {create_authenticator, Global, AuthenticatorConfig2})), + update_config([?CONF_ROOT], {create_authenticator, Global, AuthenticatorConfig2})), ?assertMatch( {ok, #{id := ID2, state := #{mark := 1}}}, @@ -283,7 +276,7 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch( {ok, _}, - update_config([authentication], + update_config([?CONF_ROOT], {update_authenticator, Global, ID1, @@ -296,25 +289,25 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch( {ok, _}, - update_config([authentication], {move_authenticator, Global, ID2, top})), + update_config([?CONF_ROOT], {move_authenticator, Global, ID2, top})), ?assertMatch({ok, [#{id := ID2}, #{id := ID1}]}, ?AUTHN:list_authenticators(Global)), - ?assertMatch({ok, _}, update_config([authentication], {delete_authenticator, Global, ID1})), + ?assertMatch({ok, _}, update_config([?CONF_ROOT], {delete_authenticator, Global, ID1})), ?assertEqual( {error, {not_found, {authenticator, ID1}}}, ?AUTHN:lookup_authenticator(Global, ID1)), ?assertMatch( {ok, _}, - update_config([authentication], {delete_authenticator, Global, ID2})), + update_config([?CONF_ROOT], {delete_authenticator, Global, ID2})), ?assertEqual( {error, {not_found, {authenticator, ID2}}}, ?AUTHN:lookup_authenticator(Global, ID2)), ListenerID = 'tcp:default', - ConfKeyPath = [listeners, tcp, default, authentication], + ConfKeyPath = [listeners, tcp, default, ?CONF_ROOT], ?assertMatch( {ok, _}, diff --git a/apps/emqx_authn/etc/emqx_authn.conf b/apps/emqx_authn/etc/emqx_authn.conf index d1d3d16f8..7503f3b89 100644 --- a/apps/emqx_authn/etc/emqx_authn.conf +++ b/apps/emqx_authn/etc/emqx_authn.conf @@ -1,6 +1 @@ -# authentication: { -# mechanism: password-based -# backend: built-in-database -# user_id_type: clientid -# } - +authentication: [] diff --git a/apps/emqx_authn/include/emqx_authn.hrl b/apps/emqx_authn/include/emqx_authn.hrl index 1c5c00e85..b5dfa8f01 100644 --- a/apps/emqx_authn/include/emqx_authn.hrl +++ b/apps/emqx_authn/include/emqx_authn.hrl @@ -17,6 +17,8 @@ -ifndef(EMQX_AUTHN_HRL). -define(EMQX_AUTHN_HRL, true). +-include_lib("emqx/include/emqx_authentication.hrl"). + -define(APP, emqx_authn). -define(AUTHN, emqx_authentication). @@ -27,4 +29,9 @@ -define(AUTH_SHARD, emqx_authn_shard). +%% has to be the same as the root field name defined in emqx_schema +-define(CONF_NS, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME). +-define(CONF_NS_ATOM, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). +-define(CONF_NS_BINARY, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY). + -endif. diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index fbd31c5d2..5b3d76822 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -22,6 +22,8 @@ , check_configs/1 ]). +-include("emqx_authn.hrl"). + providers() -> [ {{'password-based', 'built-in-database'}, emqx_authn_mnesia} , {{'password-based', mysql}, emqx_authn_mysql} @@ -44,8 +46,8 @@ check_config(Config) -> check_config(Config, Opts) -> case do_check_config(Config, Opts) of - #{config := Checked} -> Checked; - #{<<"config">> := WithDefaults} -> WithDefaults + #{?CONF_NS_ATOM := Checked} -> Checked; + #{?CONF_NS_BINARY := WithDefaults} -> WithDefaults end. do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) -> @@ -56,10 +58,15 @@ do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) -> case lists:keyfind(Key, 1, providers()) of false -> throw({unknown_handler, Key}); - {_, Provider} -> - hocon_schema:check_plain(Provider, #{<<"config">> => Config}, + {_, ProviderModule} -> + hocon_schema:check_plain(ProviderModule, #{?CONF_NS_BINARY => Config}, Opts#{atom_key => true}) end. atom(Bin) -> - binary_to_existing_atom(Bin, utf8). + try + binary_to_existing_atom(Bin, utf8) + catch + _ : _ -> + throw({unknown_auth_provider, Bin}) + end. diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 83ca9e56d..81964f3ea 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -22,6 +22,7 @@ -include("emqx_authn.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). -import(hoconsc, [mk/2, ref/1]). -import(emqx_dashboard_swagger, [error_codes/2]). @@ -32,8 +33,10 @@ % Swagger --define(API_TAGS_GLOBAL, [<<"authentication">>, <<"authentication config(global)">>]). --define(API_TAGS_SINGLE, [<<"authentication">>, <<"authentication config(single listener)">>]). +-define(API_TAGS_GLOBAL, [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, + <<"authentication config(global)">>]). +-define(API_TAGS_SINGLE, [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, + <<"authentication config(single listener)">>]). -export([ api_spec/0 , paths/0 diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index df7fbecd3..035e61910 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -25,6 +25,8 @@ , stop/1 ]). +-include_lib("emqx/include/emqx_authentication.hrl"). + -dialyzer({nowarn_function, [start/2]}). %%------------------------------------------------------------------------------ @@ -65,7 +67,7 @@ chain_configs() -> [global_chain_config() | listener_chain_configs()]. global_chain_config() -> - {?GLOBAL, emqx:get_raw_config([<<"authentication">>], [])}. + {?GLOBAL, emqx:get_raw_config([?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY], [])}. listener_chain_configs() -> lists:map( @@ -77,7 +79,7 @@ listener_chain_configs() -> auth_config_path(ListenerID) -> [<<"listeners">>] ++ binary:split(atom_to_binary(ListenerID), <<":">>) - ++ [<<"authentication">>]. + ++ [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY]. provider_types() -> lists:map(fun({Type, _Module}) -> Type end, emqx_authn:providers()). diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index 22f62f519..c2e963ec4 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -22,10 +22,12 @@ , roots/0 , fields/1 , authenticator_type/0 + , root_type/0 + , mechanism/1 + , backend/1 ]). -%% only for doc generation -roots() -> [{authenticator_config, hoconsc:mk(authenticator_type())}]. +roots() -> []. fields(_) -> []. @@ -35,6 +37,7 @@ common_fields() -> enable(type) -> boolean(); enable(default) -> true; +enable(desc) -> "Set to false to disable this auth provider"; enable(_) -> undefined. authenticator_type() -> @@ -42,3 +45,18 @@ authenticator_type() -> config_refs(Modules) -> lists:append([Module:refs() || Module <- Modules]). + +%% authn is a core functionality however implemented outside fo emqx app +%% in emqx_schema, 'authentication' is a map() type which is to allow +%% EMQ X more plugable. +root_type() -> + T = authenticator_type(), + hoconsc:union([T, hoconsc:array(T)]). + +mechanism(Name) -> + hoconsc:mk(hoconsc:enum([Name]), + #{nullable => false}). + +backend(Name) -> + hoconsc:mk(hoconsc:enum([Name]), + #{nullable => false}). diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl index f02655462..3604455dc 100644 --- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl +++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl @@ -83,11 +83,11 @@ mnesia(boot) -> namespace() -> "authn-scram-builtin_db". -roots() -> [config]. +roots() -> [?CONF_NS]. -fields(config) -> - [ {mechanism, {enum, [scram]}} - , {backend, {enum, ['built-in-database']}} +fields(?CONF_NS) -> + [ {mechanism, emqx_authn_schema:mechanism('scram')} + , {backend, emqx_authn_schema:backend('built-in-database')} , {algorithm, fun algorithm/1} , {iteration_count, fun iteration_count/1} ] ++ emqx_authn_schema:common_fields(). @@ -105,7 +105,7 @@ iteration_count(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, config)]. + [hoconsc:ref(?MODULE, ?CONF_NS)]. create(AuthenticatorID, #{algorithm := Algorithm, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 829ce2282..533301ea7 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -43,8 +43,9 @@ namespace() -> "authn-http". roots() -> - [ {config, hoconsc:mk(hoconsc:union(refs()), - #{})} + [ {?CONF_NS, + hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields(get) -> @@ -60,8 +61,8 @@ fields(post) -> ] ++ common_fields(). common_fields() -> - [ {mechanism, hoconsc:enum(['password-based'])} - , {backend, hoconsc:enum(['http'])} + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(http)} , {url, fun url/1} , {body, fun body/1} , {request_timeout, fun request_timeout/1} @@ -233,9 +234,9 @@ transform_header_name(Headers) -> end, #{}, Headers). check_ssl_opts(Conf) -> - case parse_url(hocon_schema:get_value("config.url", Conf)) of + case parse_url(get_conf_val("url", Conf)) of #{scheme := https} -> - case hocon_schema:get_value("config.ssl.enable", Conf) of + case get_conf_val("ssl.enable", Conf) of true -> ok; false -> false end; @@ -244,8 +245,8 @@ check_ssl_opts(Conf) -> end. check_headers(Conf) -> - Method = to_bin(hocon_schema:get_value("config.method", Conf)), - Headers = hocon_schema:get_value("config.headers", Conf), + Method = to_bin(get_conf_val("method", Conf)), + Headers = get_conf_val("headers", Conf), Method =:= <<"post">> orelse (not maps:is_key(<<"content-type">>, Headers)). parse_url(URL) -> @@ -340,3 +341,6 @@ to_bin(B) when is_binary(B) -> B; to_bin(L) when is_list(L) -> list_to_binary(L). + +get_conf_val(Name, Conf) -> + hocon_schema:get_value(?CONF_NS ++ "." ++ Name, Conf). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 916911ffa..9295e5c7e 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -16,6 +16,7 @@ -module(emqx_authn_jwt). +-include("emqx_authn.hrl"). -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). @@ -40,9 +41,9 @@ namespace() -> "authn-jwt". roots() -> - [ {config, hoconsc:mk(hoconsc:union(refs()), - #{} - )} + [ {?CONF_NS, + hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields('hmac-based') -> @@ -82,7 +83,7 @@ fields(ssl_disable) -> [ {enable, #{type => false}} ]. common_fields() -> - [ {mechanism, {enum, [jwt]}} + [ {mechanism, emqx_authn_schema:mechanism('jwt')} , {verify_claims, fun verify_claims/1} ] ++ emqx_authn_schema:common_fields(). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index fd02671fb..f609d8cac 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -85,11 +85,11 @@ mnesia(boot) -> namespace() -> "authn-builtin_db". -roots() -> [config]. +roots() -> [?CONF_NS]. -fields(config) -> - [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, ['built-in-database']}} +fields(?CONF_NS) -> + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend('built-in-database')} , {user_id_type, fun user_id_type/1} , {password_hash_algorithm, fun password_hash_algorithm/1} ] ++ emqx_authn_schema:common_fields(); @@ -104,7 +104,7 @@ fields(other_algorithms) -> ]. user_id_type(type) -> user_id_type(); -user_id_type(default) -> username; +user_id_type(default) -> <<"username">>; user_id_type(_) -> undefined. password_hash_algorithm(type) -> hoconsc:union([hoconsc:ref(?MODULE, bcrypt), @@ -121,7 +121,7 @@ salt_rounds(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, config)]. + [hoconsc:ref(?MODULE, ?CONF_NS)]. create(AuthenticatorID, #{user_id_type := Type, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 2deef8506..3b47bcd7b 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -42,8 +42,8 @@ namespace() -> "authn-mongodb". roots() -> - [ {config, hoconsc:mk(hoconsc:union(refs()), - #{})} + [ {?CONF_NS, hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields(standalone) -> @@ -56,8 +56,8 @@ fields('sharded-cluster') -> common_fields() ++ emqx_connector_mongo:fields(sharded). common_fields() -> - [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, [mongodb]}} + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(mongodb)} , {collection, fun collection/1} , {selector, fun selector/1} , {password_hash_field, fun password_hash_field/1} diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index 47ca0ae3c..fd0d09f57 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -41,11 +41,11 @@ namespace() -> "authn-mysql". -roots() -> [config]. +roots() -> [?CONF_NS]. -fields(config) -> - [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, [mysql]}} +fields(?CONF_NS) -> + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(mysql)} , {password_hash_algorithm, fun password_hash_algorithm/1} , {salt_position, fun salt_position/1} , {query, fun query/1} @@ -74,7 +74,7 @@ query_timeout(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, config)]. + [hoconsc:ref(?MODULE, ?CONF_NS)]. create(_AuthenticatorID, Config) -> create(Config). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 660acf566..fdd30b618 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -47,11 +47,11 @@ namespace() -> "authn-postgresql". -roots() -> [config]. +roots() -> [?CONF_NS]. -fields(config) -> - [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, [postgresql]}} +fields(?CONF_NS) -> + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(postgresql)} , {password_hash_algorithm, fun password_hash_algorithm/1} , {salt_position, fun salt_position/1} , {query, fun query/1} @@ -75,7 +75,7 @@ query(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, config)]. + [hoconsc:ref(?MODULE, ?CONF_NS)]. create(_AuthenticatorID, Config) -> create(Config). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index d238bc537..e17d0ad8f 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -42,8 +42,8 @@ namespace() -> "authn-redis". roots() -> - [ {config, hoconsc:mk(hoconsc:union(refs()), - #{})} + [ {?CONF_NS, hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields(standalone) -> @@ -56,11 +56,11 @@ fields(sentinel) -> common_fields() ++ emqx_connector_redis:fields(sentinel). common_fields() -> - [{mechanism, {enum, ['password-based']}}, - {backend, {enum, [redis]}}, - {cmd, fun cmd/1}, - {password_hash_algorithm, fun password_hash_algorithm/1}, - {salt_position, fun salt_position/1} + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(redis)} + , {cmd, fun cmd/1} + , {password_hash_algorithm, fun password_hash_algorithm/1} + , {salt_position, fun salt_position/1} ] ++ emqx_authn_schema:common_fields(). cmd(type) -> string(); diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index b9eadbef6..a9e474f11 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -47,6 +47,8 @@ end_per_testcase(_Case, Config) -> %% Tests %%------------------------------------------------------------------------------ +-define(CONF(Conf), #{?CONF_NS_BINARY => Conf}). + t_check_schema(_Config) -> ConfigOk = #{ <<"mechanism">> => <<"password-based">>, @@ -58,7 +60,7 @@ t_check_schema(_Config) -> } }, - hocon_schema:check_plain(emqx_authn_mnesia, #{<<"config">> => ConfigOk}), + hocon_schema:check_plain(emqx_authn_mnesia, ?CONF(ConfigOk)), ConfigNotOk = #{ <<"mechanism">> => <<"password-based">>, @@ -72,7 +74,7 @@ t_check_schema(_Config) -> ?assertException( throw, {emqx_authn_mnesia, _}, - hocon_schema:check_plain(emqx_authn_mnesia, #{<<"config">> => ConfigNotOk})). + hocon_schema:check_plain(emqx_authn_mnesia, ?CONF(ConfigNotOk))). t_create(_) -> Config0 = config(), diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 617703cd2..a84418916 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -24,6 +24,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). -type log_level() :: debug | info | notice | warning | error | critical | alert | emergency | all. -type file() :: string(). @@ -62,6 +63,11 @@ namespace() -> undefined. roots() -> + PtKey = ?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, + case persistent_term:get(PtKey, undefined) of + undefined -> persistent_term:put(PtKey, emqx_authn_schema); + _ -> ok + end, %% authorization configs are merged in THIS schema's "authorization" fields lists:keydelete("authorization", 1, emqx_schema:roots(high)) ++ [ {"node", diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl index d7b0a00c3..e1dad1dee 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl @@ -18,9 +18,6 @@ -behaviour(emqx_gateway_channel). --include_lib("emqx/include/logger.hrl"). --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). - %% API -export([ info/1 , info/2 @@ -44,6 +41,12 @@ -export_type([channel/0]). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). + +-define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). + -record(channel, { %% Context ctx :: emqx_gateway_ctx:context(), @@ -283,7 +286,7 @@ try_takeover(idle, DesireId, Msg, Channel) -> %% udp connection baseon the clientid call_session(handle_request, Msg, Channel); _ -> - case emqx_conf:get([gateway, coap, authentication], undefined) of + case emqx_conf:get([gateway, coap, ?AUTHN], undefined) of undefined -> call_session(handle_request, Msg, Channel); _ -> diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index d79a880a1..925b7cc0e 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -17,6 +17,7 @@ -module(emqx_gateway_api). -include_lib("emqx/include/emqx_placeholder.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). -behaviour(minirest_api). @@ -243,7 +244,7 @@ schema_gateway_overview_list() -> %% %% NOTE: It is a temporary measure to generate swagger-schema -define(COAP_GATEWAY_CONFS, -#{<<"authentication">> => +#{?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY => #{<<"mechanism">> => <<"password-based">>, <<"name">> => <<"authenticator1">>, <<"server_type">> => <<"built-in-database">>, @@ -331,7 +332,7 @@ schema_gateway_overview_list() -> ). -define(STOMP_GATEWAY_CONFS, -#{<<"authentication">> => +#{?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY => #{<<"mechanism">> => <<"password-based">>, <<"name">> => <<"authenticator1">>, <<"server_type">> => <<"built-in-database">>, diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl index 3612b428d..e97b0062d 100644 --- a/apps/emqx_gateway/src/emqx_gateway_conf.erl +++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl @@ -17,8 +17,6 @@ %% @doc The gateway configuration management module -module(emqx_gateway_conf). --include_lib("emqx/include/logger.hrl"). - %% Load/Unload -export([ load/0 , unload/0 @@ -56,6 +54,10 @@ , post_config_update/5 ]). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). +-define(AUTHN_BIN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY). + -type atom_or_bin() :: atom() | binary(). -type ok_or_err() :: ok_or_err(). -type listener_ref() :: {ListenerType :: atom_or_bin(), @@ -106,8 +108,9 @@ maps_key_take([K | Ks], M, Acc) -> -spec update_gateway(atom_or_bin(), map()) -> ok_or_err(). update_gateway(GwName, Conf0) -> - Conf = maps:without([listeners, authentication, - <<"listeners">>, <<"authentication">>], Conf0), + Exclude0 = [listeners, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM], + Exclude1 = [atom_to_binary(K, utf8) || K <- Exclude0], + Conf = maps:without(Exclude0 ++ Exclude1, Conf0), update({?FUNCTION_NAME, bin(GwName), Conf}). %% FIXME: delete cert files ?? @@ -263,8 +266,7 @@ pre_config_update(_, {update_gateway, GwName, Conf}, RawConf) -> undefined -> {error, not_found}; _ -> - NConf = maps:without([<<"listeners">>, - <<"authentication">>], Conf), + NConf = maps:without([<<"listeners">>, ?AUTHN_BIN], Conf), {ok, emqx_map_lib:deep_merge(RawConf, #{GwName => NConf})} end; pre_config_update(_, {unload_gateway, GwName}, RawConf) -> @@ -311,11 +313,11 @@ pre_config_update(_, {remove_listener, GwName, {LType, LName}}, RawConf) -> pre_config_update(_, {add_authn, GwName, Conf}, RawConf) -> case emqx_map_lib:deep_get( - [GwName, <<"authentication">>], RawConf, undefined) of + [GwName, ?AUTHN_BIN], RawConf, undefined) of undefined -> {ok, emqx_map_lib:deep_merge( RawConf, - #{GwName => #{<<"authentication">> => Conf}})}; + #{GwName => #{?AUTHN_BIN => Conf}})}; _ -> {error, already_exist} end; @@ -326,9 +328,9 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> undefined -> {error, not_found}; Listener -> - case maps:get(<<"authentication">>, Listener, undefined) of + case maps:get(?AUTHN_BIN, Listener, undefined) of undefined -> - NListener = maps:put(<<"authentication">>, Conf, Listener), + NListener = maps:put(?AUTHN_BIN, Conf, Listener), NGateway = #{GwName => #{<<"listeners">> => #{LType => #{LName => NListener}}}}, @@ -339,13 +341,13 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> end; pre_config_update(_, {update_authn, GwName, Conf}, RawConf) -> case emqx_map_lib:deep_get( - [GwName, <<"authentication">>], RawConf, undefined) of + [GwName, ?AUTHN_BIN], RawConf, undefined) of undefined -> {error, not_found}; _ -> {ok, emqx_map_lib:deep_merge( RawConf, - #{GwName => #{<<"authentication">> => Conf}})} + #{GwName => #{?AUTHN_BIN => Conf}})} end; pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> case emqx_map_lib:deep_get( @@ -354,12 +356,12 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> undefined -> {error, not_found}; Listener -> - case maps:get(<<"authentication">>, Listener, undefined) of + case maps:get(?AUTHN_BIN, Listener, undefined) of undefined -> {error, not_found}; Auth -> NListener = maps:put( - <<"authentication">>, + ?AUTHN_BIN, emqx_map_lib:deep_merge(Auth, Conf), Listener ), @@ -371,9 +373,9 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> end; pre_config_update(_, {remove_authn, GwName}, RawConf) -> {ok, emqx_map_lib:deep_remove( - [GwName, <<"authentication">>], RawConf)}; + [GwName, ?AUTHN_BIN], RawConf)}; pre_config_update(_, {remove_authn, GwName, {LType, LName}}, RawConf) -> - Path = [GwName, <<"listeners">>, LType, LName, <<"authentication">>], + Path = [GwName, <<"listeners">>, LType, LName, ?AUTHN_BIN], {ok, emqx_map_lib:deep_remove(Path, RawConf)}; pre_config_update(_, UnknownReq, _RawConf) -> diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index 0d2f765c5..98344f968 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -19,6 +19,9 @@ -include("include/emqx_gateway.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). + +-define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). %% Mgmt APIs - gateway -export([ gateways/1 @@ -166,7 +169,7 @@ remove_listener(ListenerId) -> -spec authn(gateway_name()) -> map(). authn(GwName) -> %% XXX: Need append chain-nanme, authenticator-id? - Path = [gateway, GwName, authentication], + Path = [gateway, GwName, ?AUTHN], ChainName = emqx_gateway_utils:global_chain(GwName), wrap_chain_name( ChainName, @@ -176,7 +179,7 @@ authn(GwName) -> -spec authn(gateway_name(), binary()) -> map(). authn(GwName, ListenerId) -> {_, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId), - Path = [gateway, GwName, listeners, Type, Name, authentication], + Path = [gateway, GwName, listeners, Type, Name, ?AUTHN], ChainName = emqx_gateway_utils:listener_chain(GwName, Type, Name), wrap_chain_name( ChainName, diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 9f35225b7..5f87131d7 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -24,6 +24,7 @@ -dialyzer(no_unused). -dialyzer(no_fail_call). +-include_lib("emqx/include/emqx_authentication.hrl"). -include_lib("typerefl/include/types.hrl"). -type ip_port() :: tuple(). @@ -144,7 +145,7 @@ The client just sends its PUBLISH messages to a GW" , desc => "The Pre-defined topic ids and topic names.
A 'pre-defined' topic id is a topic id whose mapping to a topic name -is known in advance by both the client’s application and the gateway" +is known in advance by both the client's application and the gateway" })} , {listeners, sc(ref(udp_listeners))} ] ++ gateway_common_options(); @@ -407,30 +408,14 @@ fields(dtls_opts) -> , ciphers => dtls_all_available }, false). -authentication() -> - sc(hoconsc:union( - [ hoconsc:ref(emqx_authn_mnesia, config) - , hoconsc:ref(emqx_authn_mysql, config) - , hoconsc:ref(emqx_authn_pgsql, config) - , hoconsc:ref(emqx_authn_mongodb, standalone) - , hoconsc:ref(emqx_authn_mongodb, 'replica-set') - , hoconsc:ref(emqx_authn_mongodb, 'sharded-cluster') - , hoconsc:ref(emqx_authn_redis, standalone) - , hoconsc:ref(emqx_authn_redis, cluster) - , hoconsc:ref(emqx_authn_redis, sentinel) - , hoconsc:ref(emqx_authn_http, get) - , hoconsc:ref(emqx_authn_http, post) - , hoconsc:ref(emqx_authn_jwt, 'hmac-based') - , hoconsc:ref(emqx_authn_jwt, 'public-key') - , hoconsc:ref(emqx_authn_jwt, 'jwks') - , hoconsc:ref(emqx_enhanced_authn_scram_mnesia, config) - ]), - #{ nullable => {true, recursively} - , desc => +authentication_schema() -> + sc(emqx_authn_schema:authenticator_type(), + #{ nullable => {true, recursively} + , desc => """Default authentication configs for all of the gateway listeners.
For per-listener overrides see authentication in listener configs""" - }). + }). gateway_common_options() -> [ {enable, @@ -464,7 +449,7 @@ it has two purposes: sc(ref(clientinfo_override), #{ desc => "" })} - , {authentication, authentication()} + , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication_schema()} ]. common_listener_opts() -> @@ -483,7 +468,7 @@ common_listener_opts() -> sc(integer(), #{ default => 1000 })} - , {authentication, authentication()} + , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication_schema()} , {mountpoint, sc(binary(), #{ default => undefined From 867cc3c4ade7c888a9a6ba6481f41a3b084145c3 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 2 Dec 2021 19:44:59 +0100 Subject: [PATCH 41/72] test: ensure emqx_connector is loaded --- apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl index 49056ed68..2bca1793d 100644 --- a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl @@ -31,7 +31,7 @@ groups() -> init_per_suite(Config) -> ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_authz], + [emqx_connector, emqx_conf, emqx_authz], fun set_special_configs/1 ), Config. From 1b0da3cc58a430d96b34d326604141e2e12b3a2f Mon Sep 17 00:00:00 2001 From: DDDHuang <904897578@qq.com> Date: Mon, 6 Dec 2021 14:36:51 +0800 Subject: [PATCH 42/72] fix(alarm): alarm message with .2f format --- apps/emqx/src/emqx_os_mon.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index e0cfac7af..263035db5 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -96,8 +96,8 @@ handle_info({timeout, _Timer, check}, State) -> _ = case emqx_vm:cpu_util() of %% TODO: should be improved? 0 -> ok; Busy when Busy >= CPUHighWatermark -> - Usage = io_lib:format("~p%", [Busy]), - Message = [Usage, " cpu usage"], + Usage = list_to_binary(io_lib:format("~.2f%", [Busy])), + Message = <>, emqx_alarm:activate(high_cpu_usage, #{ usage => Usage, @@ -107,8 +107,8 @@ handle_info({timeout, _Timer, check}, State) -> Message), start_check_timer(); Busy when Busy =< CPULowWatermark -> - Usage = io_lib:format("~p%", [Busy]), - Message = [Usage, " cpu usage"], + Usage = list_to_binary(io_lib:format("~.2f%", [Busy])), + Message = <>, emqx_alarm:deactivate(high_cpu_usage, #{ usage => Usage, From bfc36efa0eaf72709f2d63446ea91e4f36f3b05f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 2 Dec 2021 20:16:46 +0100 Subject: [PATCH 43/72] fix: pin hocon 0.21.1 --- apps/emqx/rebar.config | 2 +- apps/emqx/src/emqx_config.erl | 10 ++++++++-- build | 2 +- rebar.config | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 59c5cf045..5adcbd3bb 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -17,7 +17,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.20.6"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.15.0"}}} diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 2693d559e..ecb843ed5 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -272,9 +272,15 @@ init_load(SchemaMod, RawConf) when is_map(RawConf) -> ok = save_schema_mod_and_names(SchemaMod), %% check and save configs {_AppEnvs, CheckedConf} = check_config(SchemaMod, RawConf), + %% fill default values for raw config + Opts = #{only_fill_defaults => true, + logger => fun(_, _) -> ok end, %% everything should have been logged already + nullable => true %% TODO: evil, remove, nullable should be declared in schema + }, + RawConfWithDefaults = hocon_schema:check_plain(SchemaMod, RawConf, Opts), RootNames = get_root_names(), ok = save_to_config_map(maps:with(get_atom_root_names(), CheckedConf), - maps:with(RootNames, RawConf)). + maps:with(RootNames, RawConfWithDefaults)). include_dirs() -> [filename:join(emqx:data_dir(), "configs")]. @@ -283,7 +289,7 @@ include_dirs() -> when AppEnvs :: app_envs(), CheckedConf :: config(). check_config(SchemaMod, RawConf) -> Opts = #{return_plain => true, - nullable => true, + nullable => true, %% TODO: evil, remove, nullable should be declared in schema format => map }, {AppEnvs, CheckedConf} = diff --git a/build b/build index c79dc711c..9f9879df4 100755 --- a/build +++ b/build @@ -60,7 +60,7 @@ docgen() { conf_doc_markdown="$(pwd)/_build/${PROFILE}/rel/emqx/etc/emqx-config-doc.md" echo "===< Generating config document $conf_doc_markdown" # shellcheck disable=SC2086 - erl -noshell -pa $libs_dir1 $libs_dir2 -eval "file:write_file('$conf_doc_markdown', hocon_schema_doc:gen(emqx_conf_schema)), halt(0)." + erl -noshell -pa $libs_dir1 $libs_dir2 -eval "file:write_file('$conf_doc_markdown', hocon_schema_doc:gen(emqx_conf_schema, \"EMQ X ${PKG_VSN} Configuration\")), halt(0)." } make_rel() { diff --git a/rebar.config b/rebar.config index 4fbf1b136..33c1244d2 100644 --- a/rebar.config +++ b/rebar.config @@ -64,7 +64,7 @@ , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.15.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.20.6"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}} From e6e28f900250ae1d2b6bccd9dc67b76990bb4c3f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 29 Nov 2021 23:57:42 +0100 Subject: [PATCH 44/72] docs: fix emqx_schema doc style --- apps/emqx/src/emqx_schema.erl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index c137f4a37..2415f051c 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1232,16 +1232,18 @@ ciphers_schema(Default) -> false -> fun validate_ciphers/1 end , desc => -"""TLS cipher suite names separated by comma, or as an array of strings +"""This config holds TLS cipher suite names separated by comma, +or as an array of strings. e.g. \"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256\" or -[\"TLS_AES_256_GCM_SHA384\",\"TLS_AES_128_GCM_SHA256\"][\"TLS_AES_256_GCM_SHA384\",\"TLS_AES_128_GCM_SHA256\"].
Ciphers (and their ordering) define the way in which the -client and server encrypts information over the wire. +client and server encrypts information over the network connection. Selecting a good cipher suite is critical for the application's data security, confidentiality and performance. -The names should be in OpenSSL sting format (not RFC format). -Default values and examples proveded by EMQ X config + +The names should be in OpenSSL string format (not RFC format). +All default values and examples proveded by EMQ X config documentation are all in OpenSSL format.
NOTE: Certain cipher suites are only compatible with @@ -1455,7 +1457,5 @@ authentication(Desc) -> Authentication can be one single authenticator instance or a chain of authenticators as an array. When authenticating a login (username, client ID, etc.) the authenticators are checked in the configured order.
-EMQ X comes with a set of pre-built autenticators, for more details, see -autenticator_config """]) }. From d9a980c7e1945072ec7afac0ab08d4f7843e00cf Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 30 Nov 2021 17:09:13 +0100 Subject: [PATCH 45/72] docs: fix config doc, avoid using in doc body --- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- apps/emqx_bridge/src/emqx_bridge_schema.erl | 3 ++- apps/emqx_connector/src/emqx_connector_api.erl | 6 +++--- apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 417fa49f7..6e274903e 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -33,7 +33,7 @@ catch error:{invalid_bridge_id, Id0} -> {400, #{code => 'INVALID_ID', message => <<"invalid_bridge_id: ", Id0/binary, - ". Bridge Ids must be of format :">>}} + ". Bridge ID must be of format 'bridge_type:name'">>}} end). -define(METRICS(MATCH, SUCC, FAILED, RATE, RATE_5, RATE_MAX), diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl index 26a1d5bd1..3a3151ef0 100644 --- a/apps/emqx_bridge/src/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl @@ -108,7 +108,8 @@ connector_name() -> #{ nullable => false , desc =>""" The connector name to be used for this bridge. -Connectors are configured by 'connectors.. +Connectors are configured as 'connectors.type.name', +for example 'connectors.http.mybridge'. """ })}. diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index 6eb397519..7e934b997 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -38,7 +38,7 @@ catch error:{invalid_bridge_id, Id0} -> {400, #{code => 'INVALID_ID', message => <<"invalid_bridge_id: ", Id0/binary, - ". Bridge Ids must be of format :">>}} + ". Bridge ID must be of format 'bridge_type:name'">>}} end). namespace() -> "connector". @@ -74,7 +74,7 @@ schema("/connectors_test") -> post => #{ tags => [<<"connectors">>], description => <<"Test creating a new connector by given Id
" - "The Id must be of format :">>, + "The ID must be of format 'type:name'">>, summary => <<"Test creating connector">>, requestBody => connector_test_info(), responses => #{ @@ -98,7 +98,7 @@ schema("/connectors") -> post => #{ tags => [<<"connectors">>], description => <<"Create a new connector by given Id
" - "The Id must be of format :">>, + "The ID must be of format 'type:name'">>, summary => <<"Create connector">>, requestBody => connector_info(), responses => #{ diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 8daae99b5..995044fc7 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -56,7 +56,7 @@ A list of outputs of the rule.
An output can be a string that refers to the channel Id of a emqx bridge, or a object that refers to a function.
There a some built-in functions like \"republish\" and \"console\", and we also support user -provided functions like \":\".
+provided functions like \"ModuleName:FunctionName\".
The outputs in the list is executed one by one in order. This means that if one of the output is executing slowly, all of the outputs comes after it will not be executed until it returns.
From 2a3f58f8ed8b688a0b0693e97d7a00326404a43c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 3 Dec 2021 20:23:39 +0100 Subject: [PATCH 46/72] chore: add a comment in bin/emqx to warn the config overlay --- bin/emqx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/emqx b/bin/emqx index e1b72d1f9..920cd00f4 100755 --- a/bin/emqx +++ b/bin/emqx @@ -342,6 +342,9 @@ generate_config() { NOW_TIME="$(call_hocon now_time)" ## ths command populates two files: app.