yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #6311 from emqx/release-5.0-beta.2 

Sync bug fixes
This commit is contained in:
tigercl 2021-11-26 14:24:19 +08:00 committed by GitHub
parent b7928203b8 12d8ebcc15
commit 124ba7a071
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 152 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/emqx/etc/emqx.conf
View File

@ -577,8 +577,9 @@ mqtt {
## @doc mqtt.max_topic_levels ## @doc mqtt.max_topic_levels
## ValueType: Integer ## ValueType: Integer
## Range: [1, 65535] ## Range: [1, 65535]
## Default: 65535 ## Default: 128
max_topic_levels = 65535 ## Depth so big may lead to subscribing performance issues
max_topic_levels = 128
## Maximum QoS allowed. ## Maximum QoS allowed.
## ##

4
apps/emqx/test/emqx_channel_SUITE.erl
View File

@ -59,7 +59,7 @@ mqtt_conf() ->
max_inflight => 32,max_mqueue_len => 1000, max_inflight => 32,max_mqueue_len => 1000,
max_packet_size => 1048576,max_qos_allowed => 2, max_packet_size => 1048576,max_qos_allowed => 2,
max_subscriptions => infinity,max_topic_alias => 65535, max_subscriptions => infinity,max_topic_alias => 65535,
max_topic_levels => 65535,mqueue_default_priority => lowest, max_topic_levels => 128,mqueue_default_priority => lowest,
mqueue_priorities => disabled,mqueue_store_qos0 => true, mqueue_priorities => disabled,mqueue_store_qos0 => true,
peer_cert_as_clientid => disabled, peer_cert_as_clientid => disabled,
peer_cert_as_username => disabled, peer_cert_as_username => disabled,
@ -200,7 +200,7 @@ t_chan_caps(_) ->
#{max_clientid_len := 65535, #{max_clientid_len := 65535,
max_qos_allowed := 2, max_qos_allowed := 2,
max_topic_alias := 65535, max_topic_alias := 65535,
max_topic_levels := 65535, max_topic_levels := 128,
retain_available := true, retain_available := true,
shared_subscription := true, shared_subscription := true,
subscription_identifiers := true, subscription_identifiers := true,

28
apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl
View File

@ -125,7 +125,7 @@ verify_claims(default) -> #{};
verify_claims(validator) -> [fun do_check_verify_claims/1]; verify_claims(validator) -> [fun do_check_verify_claims/1];
verify_claims(converter) -> verify_claims(converter) ->
fun(VerifyClaims) -> fun(VerifyClaims) ->
maps:to_list(VerifyClaims) [{to_binary(K), V} || {K, V} <- maps:to_list(VerifyClaims)]
end; end;
verify_claims(_) -> undefined. verify_claims(_) -> undefined.
@ -201,15 +201,14 @@ create2(#{use_jwks := false,
secret := Secret0, secret := Secret0,
secret_base64_encoded := Base64Encoded, secret_base64_encoded := Base64Encoded,
verify_claims := VerifyClaims}) -> verify_claims := VerifyClaims}) ->
Secret = case Base64Encoded of case may_decode_secret(Base64Encoded, Secret0) of
true -> {error, Reason} ->
base64:decode(Secret0); {error, Reason};
false -> Secret ->
Secret0
end,
JWK = jose_jwk:from_oct(Secret), JWK = jose_jwk:from_oct(Secret),
{ok, #{jwk => JWK, {ok, #{jwk => JWK,
verify_claims => VerifyClaims}}; verify_claims => VerifyClaims}}
end;
create2(#{use_jwks := false, create2(#{use_jwks := false,
algorithm := 'public-key', algorithm := 'public-key',
@ -234,6 +233,14 @@ create2(#{use_jwks := true,
{error, Reason} {error, Reason}
end. end.
may_decode_secret(false, Secret) -> Secret;
may_decode_secret(true, Secret) ->
try base64:decode(Secret)
catch
error : _ ->
{error, {invalid_parameter, secret}}
end.
replace_placeholder(L, Variables) -> replace_placeholder(L, Variables) ->
replace_placeholder(L, Variables, []). replace_placeholder(L, Variables, []).
@ -349,3 +356,8 @@ validate_placeholder(<<"clientid">>) ->
clientid; clientid;
validate_placeholder(<<"username">>) -> validate_placeholder(<<"username">>) ->
username. username.
to_binary(A) when is_atom(A) ->
atom_to_binary(A);
to_binary(B) when is_binary(B) ->
B.

228
apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl
View File

@ -19,140 +19,142 @@
-compile(export_all). -compile(export_all).
-compile(nowarn_export_all). -compile(nowarn_export_all).
% -include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct.hrl").
% -include_lib("eunit/include/eunit.hrl"). -include_lib("eunit/include/eunit.hrl").
% -include("emqx_authn.hrl"). -include("emqx_authn.hrl").
% -define(AUTH, emqx_authn). -define(AUTHN_ID, <<"mechanism:jwt">>).
all() -> all() ->
emqx_common_test_helpers:all(?MODULE). emqx_common_test_helpers:all(?MODULE).
% init_per_suite(Config) -> init_per_suite(Config) ->
% emqx_common_test_helpers:start_apps([emqx_authn]), emqx_common_test_helpers:start_apps([emqx_authn]),
% Config. Config.
% end_per_suite(_) -> end_per_suite(_) ->
% emqx_common_test_helpers:stop_apps([emqx_authn]), emqx_common_test_helpers:stop_apps([emqx_authn]),
% ok. ok.
% t_jwt_authenticator(_) -> t_jwt_authenticator(_) ->
% AuthenticatorName = <<"myauthenticator">>, Secret = <<"abcdef">>,
% Config = #{name => AuthenticatorName, Config = #{mechanism => jwt,
% mechanism => jwt, use_jwks => false,
% use_jwks => false, algorithm => 'hmac-based',
% algorithm => 'hmac-based', secret => Secret,
% secret => <<"abcdef">>, secret_base64_encoded => false,
% secret_base64_encoded => false, verify_claims => []},
% verify_claims => []}, {ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config),
% {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, Config),
% Payload = #{<<"username">> => <<"myuser">>}, Payload = #{<<"username">> => <<"myuser">>},
% JWS = generate_jws('hmac-based', Payload, <<"abcdef">>), JWS = generate_jws('hmac-based', Payload, Secret),
% ClientInfo = #{username => <<"myuser">>, Credential = #{username => <<"myuser">>,
% password => JWS}, password => JWS},
% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential, State)),
% Payload1 = #{<<"username">> => <<"myuser">>, <<"is_superuser">> => true}, Payload1 = #{<<"username">> => <<"myuser">>, <<"is_superuser">> => true},
% JWS1 = generate_jws('hmac-based', Payload1, <<"abcdef">>), JWS1 = generate_jws('hmac-based', Payload1, Secret),
% ClientInfo1 = #{username => <<"myuser">>, Credential1 = #{username => <<"myuser">>,
% password => JWS1}, password => JWS1},
% ?assertEqual({stop, {ok, #{is_superuser => true}}}, ?AUTH:authenticate(ClientInfo1, ignored)), ?assertEqual({ok, #{is_superuser => true}}, emqx_authn_jwt:authenticate(Credential1, State)),
% BadJWS = generate_jws('hmac-based', Payload, <<"bad_secret">>), BadJWS = generate_jws('hmac-based', Payload, <<"bad_secret">>),
% ClientInfo2 = ClientInfo#{password => BadJWS}, Credential2 = Credential#{password => BadJWS},
% ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo2, ignored)), ?assertEqual(ignore, emqx_authn_jwt:authenticate(Credential2, State)),
% %% secret_base64_encoded %% secret_base64_encoded
% Config2 = Config#{secret => base64:encode(<<"abcdef">>), Config2 = Config#{secret => base64:encode(Secret),
% secret_base64_encoded => true}, secret_base64_encoded => true},
% ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config2)), {ok, State2} = emqx_authn_jwt:update(Config2, State),
% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential, State2)),
% Config3 = Config#{verify_claims => [{<<"username">>, <<"${mqtt-username}">>}]}, %% invalid secret
% ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config3)), BadConfig = Config#{secret => <<"emqxsecret">>,
% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), secret_base64_encoded => true},
% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo#{username => <<"otheruser">>}, ok)), {error, {invalid_parameter, secret}} = emqx_authn_jwt:create(?AUTHN_ID, BadConfig),
% %% Expiration Config3 = Config#{verify_claims => [{<<"username">>, <<"${username}">>}]},
% Payload3 = #{ <<"username">> => <<"myuser">> {ok, State3} = emqx_authn_jwt:update(Config3, State2),
% , <<"exp">> => erlang:system_time(second) - 60}, ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential, State3)),
% JWS3 = generate_jws('hmac-based', Payload3, <<"abcdef">>), ?assertEqual({error, bad_username_or_password}, emqx_authn_jwt:authenticate(Credential#{username => <<"otheruser">>}, State3)),
% ClientInfo3 = ClientInfo#{password => JWS3},
% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo3, ignored)),
% Payload4 = #{ <<"username">> => <<"myuser">> %% Expiration
% , <<"exp">> => erlang:system_time(second) + 60}, Payload3 = #{ <<"username">> => <<"myuser">>
% JWS4 = generate_jws('hmac-based', Payload4, <<"abcdef">>), , <<"exp">> => erlang:system_time(second) - 60},
% ClientInfo4 = ClientInfo#{password => JWS4}, JWS3 = generate_jws('hmac-based', Payload3, Secret),
% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo4, ignored)), Credential3 = Credential#{password => JWS3},
?assertEqual({error, bad_username_or_password}, emqx_authn_jwt:authenticate(Credential3, State3)),
% %% Issued At Payload4 = #{ <<"username">> => <<"myuser">>
% Payload5 = #{ <<"username">> => <<"myuser">> , <<"exp">> => erlang:system_time(second) + 60},
% , <<"iat">> => erlang:system_time(second) - 60}, JWS4 = generate_jws('hmac-based', Payload4, Secret),
% JWS5 = generate_jws('hmac-based', Payload5, <<"abcdef">>), Credential4 = Credential#{password => JWS4},
% ClientInfo5 = ClientInfo#{password => JWS5}, ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential4, State3)),
% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo5, ignored)),
% Payload6 = #{ <<"username">> => <<"myuser">> %% Issued At
% , <<"iat">> => erlang:system_time(second) + 60}, Payload5 = #{ <<"username">> => <<"myuser">>
% JWS6 = generate_jws('hmac-based', Payload6, <<"abcdef">>), , <<"iat">> => erlang:system_time(second) - 60},
% ClientInfo6 = ClientInfo#{password => JWS6}, JWS5 = generate_jws('hmac-based', Payload5, Secret),
% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo6, ignored)), Credential5 = Credential#{password => JWS5},
?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential5, State3)),
% %% Not Before Payload6 = #{ <<"username">> => <<"myuser">>
% Payload7 = #{ <<"username">> => <<"myuser">> , <<"iat">> => erlang:system_time(second) + 60},
% , <<"nbf">> => erlang:system_time(second) - 60}, JWS6 = generate_jws('hmac-based', Payload6, Secret),
% JWS7 = generate_jws('hmac-based', Payload7, <<"abcdef">>), Credential6 = Credential#{password => JWS6},
% ClientInfo7 = ClientInfo#{password => JWS7}, ?assertEqual({error, bad_username_or_password}, emqx_authn_jwt:authenticate(Credential6, State3)),
% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo7, ignored)),
% Payload8 = #{ <<"username">> => <<"myuser">> %% Not Before
% , <<"nbf">> => erlang:system_time(second) + 60}, Payload7 = #{ <<"username">> => <<"myuser">>
% JWS8 = generate_jws('hmac-based', Payload8, <<"abcdef">>), , <<"nbf">> => erlang:system_time(second) - 60},
% ClientInfo8 = ClientInfo#{password => JWS8}, JWS7 = generate_jws('hmac-based', Payload7, Secret),
% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo8, ignored)), Credential7 = Credential6#{password => JWS7},
?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential7, State3)),
% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), Payload8 = #{ <<"username">> => <<"myuser">>
% ok. , <<"nbf">> => erlang:system_time(second) + 60},
JWS8 = generate_jws('hmac-based', Payload8, Secret),
Credential8 = Credential#{password => JWS8},
?assertEqual({error, bad_username_or_password}, emqx_authn_jwt:authenticate(Credential8, State3)),
% t_jwt_authenticator2(_) -> ?assertEqual(ok, emqx_authn_jwt:destroy(State3)),
% Dir = code:lib_dir(emqx_authn, test), ok.
% PublicKey = list_to_binary(filename:join([Dir, "data/public_key.pem"])),
% PrivateKey = list_to_binary(filename:join([Dir, "data/private_key.pem"])),
% AuthenticatorName = <<"myauthenticator">>,
% Config = #{name => AuthenticatorName,
% mechanism => jwt,
% use_jwks => false,
% algorithm => 'public-key',
% certificate => PublicKey,
% verify_claims => []},
% {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, Config),
% Payload = #{<<"username">> => <<"myuser">>}, t_jwt_authenticator2(_) ->
% JWS = generate_jws('public-key', Payload, PrivateKey), Dir = code:lib_dir(emqx_authn, test),
% ClientInfo = #{username => <<"myuser">>, PublicKey = list_to_binary(filename:join([Dir, "data/public_key.pem"])),
% password => JWS}, PrivateKey = list_to_binary(filename:join([Dir, "data/private_key.pem"])),
% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), Config = #{mechanism => jwt,
% ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo#{password => <<"badpassword">>}, ignored)), use_jwks => false,
algorithm => 'public-key',
certificate => PublicKey,
verify_claims => []},
{ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config),
% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), Payload = #{<<"username">> => <<"myuser">>},
% ok. JWS = generate_jws('public-key', Payload, PrivateKey),
Credential = #{username => <<"myuser">>,
password => JWS},
?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential, State)),
?assertEqual(ignore, emqx_authn_jwt:authenticate(Credential#{password => <<"badpassword">>}, State)),
% generate_jws('hmac-based', Payload, Secret) -> ?assertEqual(ok, emqx_authn_jwt:destroy(State)),
% JWK = jose_jwk:from_oct(Secret), ok.
% Header = #{ <<"alg">> => <<"HS256">>
% , <<"typ">> => <<"JWT">> generate_jws('hmac-based', Payload, Secret) ->
% }, JWK = jose_jwk:from_oct(Secret),
% Signed = jose_jwt:sign(JWK, Header, Payload), Header = #{ <<"alg">> => <<"HS256">>
% {_, JWS} = jose_jws:compact(Signed), , <<"typ">> => <<"JWT">>
% JWS; },
% generate_jws('public-key', Payload, PrivateKey) -> Signed = jose_jwt:sign(JWK, Header, Payload),
% JWK = jose_jwk:from_pem_file(PrivateKey), {_, JWS} = jose_jws:compact(Signed),
% Header = #{ <<"alg">> => <<"RS256">> JWS;
% , <<"typ">> => <<"JWT">> generate_jws('public-key', Payload, PrivateKey) ->
% }, JWK = jose_jwk:from_pem_file(PrivateKey),
% Signed = jose_jwt:sign(JWK, Header, Payload), Header = #{ <<"alg">> => <<"RS256">>
% {_, JWS} = jose_jws:compact(Signed), , <<"typ">> => <<"JWT">>
% JWS. },
Signed = jose_jwt:sign(JWK, Header, Payload),
{_, JWS} = jose_jws:compact(Signed),
JWS.

2
apps/emqx_authz/include/emqx_authz.hrl
View File

@ -68,6 +68,8 @@
-define(CONF_KEY_PATH, [authorization, sources]). -define(CONF_KEY_PATH, [authorization, sources]).
-define(RE_PLACEHOLDER, "\\$\\{[a-z0-9\\-]+\\}").
-define(USERNAME_RULES_EXAMPLE, #{username => user1, -define(USERNAME_RULES_EXAMPLE, #{username => user1,
rules => [ #{topic => <<"test/toopic/1">>, rules => [ #{topic => <<"test/toopic/1">>,
permission => <<"allow">>, permission => <<"allow">>,

4
apps/emqx_authz/src/emqx_authz_mysql.erl
View File

@ -38,10 +38,10 @@ description() ->
parse_query(undefined) -> parse_query(undefined) ->
undefined; undefined;
parse_query(Sql) -> parse_query(Sql) ->
case re:run(Sql, "'%[ucCad]'", [global, {capture, all, list}]) of case re:run(Sql, ?RE_PLACEHOLDER, [global, {capture, all, list}]) of
{match, Variables} -> {match, Variables} ->
Params = [Var || [Var] <- Variables], Params = [Var || [Var] <- Variables],
{re:replace(Sql, "'%[ucCad]'", "?", [global, {return, list}]), Params}; {re:replace(Sql, ?RE_PLACEHOLDER, "?", [global, {return, list}]), Params};
nomatch -> nomatch ->
{Sql, []} {Sql, []}
end. end.

2
apps/emqx_authz/src/emqx_authz_postgresql.erl
View File

@ -38,7 +38,7 @@ description() ->
parse_query(undefined) -> parse_query(undefined) ->
undefined; undefined;
parse_query(Sql) -> parse_query(Sql) ->
case re:run(Sql, "'%[ucCad]'", [global, {capture, all, list}]) of case re:run(Sql, ?RE_PLACEHOLDER, [global, {capture, all, list}]) of
{match, Variables} -> {match, Variables} ->
Params = [Var || [Var] <- Variables], Params = [Var || [Var] <- Variables],
Vars = ["$" ++ integer_to_list(I) || I <- lists:seq(1, length(Params))], Vars = ["$" ++ integer_to_list(I) || I <- lists:seq(1, length(Params))],

7
apps/emqx_connector/src/emqx_connector_mongo.erl
View File

@ -169,8 +169,11 @@ health_check(PoolName) ->
case ecpool_worker:client(Worker) of case ecpool_worker:client(Worker) of
{ok, Conn} -> {ok, Conn} ->
%% we don't care if this returns something or not, we just to test the connection %% we don't care if this returns something or not, we just to test the connection
Res = mongo_api:find_one(Conn, <<"foo">>, {}, #{}), try mongo_api:find_one(Conn, <<"foo">>, {}, #{}) of
Res == undefined orelse is_map(Res); _ -> true
catch
_Class:_Error -> false
end;
_ -> false _ -> false
end end
end || {_WorkerName, Worker} <- ecpool:workers(PoolName)], end || {_WorkerName, Worker} <- ecpool:workers(PoolName)],