yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: use union member type selector for authz sources

This commit is contained in:
Zaiming (Stone) Shi 2022-12-31 19:52:38 +01:00
parent b80325f988
commit e52f9d5920
18 changed files with 229 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/emqx/rebar.config
View File

@ -29,7 +29,7 @@
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.34.0"}}},
{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
{recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},
{snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}

5
apps/emqx/src/emqx_config.erl
View File

@ -362,8 +362,8 @@ schema_default(Schema) ->
[];
?LAZY(?ARRAY(_)) ->
[];
?LAZY(?UNION(Unions)) ->
case [A || ?ARRAY(A) <- Unions] of
?LAZY(?UNION(Members)) ->
case [A || ?ARRAY(A) <- hoconsc:union_members(Members)] of
[_ | _] -> [];
_ -> #{}
end;
@ -402,7 +402,6 @@ merge_envs(SchemaMod, RawConf) ->
required => false,
format => map,
apply_override_envs => true,
remove_env_meta => true,
check_lazy => true
},
hocon_tconf:merge_env_overrides(SchemaMod, RawConf, all, Opts).

2
apps/emqx/test/emqx_schema_tests.erl
View File

@ -153,7 +153,7 @@ ssl_opts_gc_after_handshake_test_rancher_listener_test() ->
#{
kind := validation_error,
reason := unknown_fields,
unknown := <<"gc_after_handshake">>
unknown := "gc_after_handshake"
}
]},
validate(Sc, #{<<"gc_after_handshake">> => true})

2
apps/emqx_authz/src/emqx_authz.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_authz, [
{description, "An OTP application"},
{vsn, "0.1.10"},
{vsn, "0.1.11"},
{registered, []},
{mod, {emqx_authz_app, []}},
{applications, [

98
apps/emqx_authz/src/emqx_authz_schema.erl
View File

@ -47,14 +47,8 @@
%% Hocon Schema
%%--------------------------------------------------------------------
namespace() -> authz.
%% @doc authorization schema is not exported
%% but directly used by emqx_schema
roots() -> [].
fields("authorization") ->
Types = [
type_names() ->
[
file,
http_get,
http_post,
@ -67,12 +61,26 @@ fields("authorization") ->
redis_single,
redis_sentinel,
redis_cluster
],
Unions = [?R_REF(Type) || Type <- Types],
].
namespace() -> authz.
%% @doc authorization schema is not exported
%% but directly used by emqx_schema
roots() -> [].
fields("authorization") ->
Types = [?R_REF(Type) || Type <- type_names()],
UnionMemberSelector =
fun
(all_union_members) -> Types;
%% must return list
({value, Value}) -> [select_union_member(Value)]
end,
[
{sources,
?HOCON(
?ARRAY(?UNION(Unions)),
?ARRAY(?UNION(UnionMemberSelector)),
#{
default => [],
desc => ?DESC(sources)
@ -408,9 +416,75 @@ common_rate_field() ->
].
method(Method) ->
?HOCON(Method, #{default => Method, required => true, desc => ?DESC(method)}).
?HOCON(Method, #{required => true, desc => ?DESC(method)}).
array(Ref) -> array(Ref, Ref).
array(Ref, DescId) ->
?HOCON(?ARRAY(?R_REF(Ref)), #{desc => ?DESC(DescId)}).
select_union_member(#{<<"type">> := <<"mongodb">>} = Value) ->
MongoType = maps:get(<<"mongo_type">>, Value, undefined),
case MongoType of
<<"single">> ->
?R_REF(mongo_single);
<<"rs">> ->
?R_REF(mongo_rs);
<<"sharded">> ->
?R_REF(mongo_sharded);
Else ->
throw(#{
reason => "unknown_mongo_type",
expected => "single | rs | sharded",
got => Else
})
end;
select_union_member(#{<<"type">> := <<"redis">>} = Value) ->
RedisType = maps:get(<<"redis_type">>, Value, undefined),
case RedisType of
<<"single">> ->
?R_REF(redis_single);
<<"cluster">> ->
?R_REF(redis_cluster);
<<"sentinel">> ->
?R_REF(redis_sentinel);
Else ->
throw(#{
reason => "unknown_redis_type",
expected => "single | cluster | sentinel",
got => Else
})
end;
select_union_member(#{<<"type">> := <<"http">>} = Value) ->
RedisType = maps:get(<<"method">>, Value, undefined),
case RedisType of
<<"get">> ->
?R_REF(http_get);
<<"post">> ->
?R_REF(http_post);
Else ->
throw(#{
reason => "unknown_http_method",
expected => "get | post",
got => Else
})
end;
select_union_member(#{<<"type">> := <<"built_in_database">>}) ->
?R_REF(mnesia);
select_union_member(#{<<"type">> := Type}) ->
select_union_member_loop(Type, type_names());
select_union_member(_) ->
throw("missing_type_field").
select_union_member_loop(TypeValue, []) ->
throw(#{
reason => "unknown_authz_type",
got => TypeValue
});
select_union_member_loop(TypeValue, [Type | Types]) ->
case TypeValue =:= atom_to_binary(Type) of
true ->
?R_REF(Type);
false ->
select_union_member_loop(TypeValue, Types)
end.

116
apps/emqx_authz/test/emqx_authz_schema_tests.erl Normal file
View File

@ -0,0 +1,116 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_authz_schema_tests).
-include_lib("eunit/include/eunit.hrl").
bad_authz_type_test() ->
Txt = "[{type: foobar}]",
?assertThrow(
[
#{
reason := "unknown_authz_type",
got := <<"foobar">>
}
],
check(Txt)
).
bad_mongodb_type_test() ->
Txt = "[{type: mongodb, mongo_type: foobar}]",
?assertThrow(
[
#{
reason := "unknown_mongo_type",
got := <<"foobar">>
}
],
check(Txt)
).
missing_mongodb_type_test() ->
Txt = "[{type: mongodb}]",
?assertThrow(
[
#{
reason := "unknown_mongo_type",
got := undefined
}
],
check(Txt)
).
unknown_redis_type_test() ->
Txt = "[{type: redis, redis_type: foobar}]",
?assertThrow(
[
#{
reason := "unknown_redis_type",
got := <<"foobar">>
}
],
check(Txt)
).
missing_redis_type_test() ->
Txt = "[{type: redis}]",
?assertThrow(
[
#{
reason := "unknown_redis_type",
got := undefined
}
],
check(Txt)
).
unknown_http_method_test() ->
Txt = "[{type: http, method: getx}]",
?assertThrow(
[
#{
reason := "unknown_http_method",
got := <<"getx">>
}
],
check(Txt)
).
missing_http_method_test() ->
Txt = "[{type: http, methodx: get}]",
?assertThrow(
[
#{
reason := "unknown_http_method",
got := undefined
}
],
check(Txt)
).
check(Txt0) ->
Txt = ["sources: ", Txt0],
{ok, RawConf} = hocon:binary(Txt),
try
hocon_tconf:check_plain(schema(), RawConf, #{})
catch
throw:{_Schema, Errors} ->
throw(Errors)
end.
schema() ->
#{roots => emqx_authz_schema:fields("authorization")}.

2
apps/emqx_conf/src/emqx_conf.erl
View File

@ -316,7 +316,7 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) ->
{[Schema | Acc], SubRefs ++ RefsAcc}
end,
{[], []},
Types
hoconsc:union_members(Types)
),
{#{<<"oneOf">> => OneOf}, Refs};
hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->

3
apps/emqx_connector/src/emqx_connector_mongo.erl
View File

@ -67,7 +67,6 @@ fields(single) ->
[
{mongo_type, #{
type => single,
default => single,
required => true,
desc => ?DESC("single_mongo_type")
}},
@ -78,7 +77,6 @@ fields(rs) ->
[
{mongo_type, #{
type => rs,
default => rs,
required => true,
desc => ?DESC("rs_mongo_type")
}},
@ -91,7 +89,6 @@ fields(sharded) ->
[
{mongo_type, #{
type => sharded,
default => sharded,
required => true,
desc => ?DESC("sharded_mongo_type")
}},

3
apps/emqx_connector/src/emqx_connector_redis.erl
View File

@ -63,7 +63,6 @@ fields(single) ->
{server, server()},
{redis_type, #{
type => single,
default => single,
required => true,
desc => ?DESC("single")
}}
@ -75,7 +74,6 @@ fields(cluster) ->
{servers, servers()},
{redis_type, #{
type => cluster,
default => cluster,
required => true,
desc => ?DESC("cluster")
}}
@ -87,7 +85,6 @@ fields(sentinel) ->
{servers, servers()},
{redis_type, #{
type => sentinel,
default => sentinel,
required => true,
desc => ?DESC("sentinel")
}},

2
apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
View File

@ -623,7 +623,7 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) ->
{[Schema | Acc], SubRefs ++ RefsAcc}
end,
{[], []},
Types
hoconsc:union_members(Types)
),
{#{<<"oneOf">> => OneOf}, Refs};
hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->

1
changes/refactor-9653.en.md Normal file
View File

@ -0,0 +1 @@
Make authorization config validation error message more readable.

1
changes/refactor-9653.zh.md Normal file
View File

@ -0,0 +1 @@
改进授权配置检查错误日志的可读性。

3
lib-ee/emqx_ee_bridge/rebar.config
View File

@ -1,6 +1,5 @@
{erl_opts, [debug_info]}.
{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}
, {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.4"}}}
{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.4"}}}
, {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.2"}}}
, {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}}
, {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.7"}}}

5
lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl
View File

@ -50,19 +50,22 @@ values(Protocol, get) ->
values("single", post) ->
SpecificOpts = #{
server => <<"127.0.0.1:6379">>,
redis_type => single,
database => 1
},
values(common, "single", SpecificOpts);
values("sentinel", post) ->
SpecificOpts = #{
servers => [<<"127.0.0.1:26379">>],
redis_type => sentinel,
sentinel => <<"mymaster">>,
database => 1
},
values(common, "sentinel", SpecificOpts);
values("cluster", post) ->
SpecificOpts = #{
servers => [<<"127.0.0.1:6379">>]
servers => [<<"127.0.0.1:6379">>],
redis_type => cluster
},
values(common, "cluster", SpecificOpts);
values(Protocol, put) ->

3
lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl
View File

@ -151,6 +151,7 @@ mongo_config(MongoHost, MongoPort0, rs = Type) ->
" servers = [~p]\n"
" w_mode = safe\n"
" database = mqtt\n"
" mongo_type = rs\n"
"}",
[Name, Servers]
),
@ -167,6 +168,7 @@ mongo_config(MongoHost, MongoPort0, sharded = Type) ->
" servers = [~p]\n"
" w_mode = safe\n"
" database = mqtt\n"
" mongo_type = sharded\n"
"}",
[Name, Servers]
),
@ -183,6 +185,7 @@ mongo_config(MongoHost, MongoPort0, single = Type) ->
" server = ~p\n"
" w_mode = safe\n"
" database = mqtt\n"
" mongo_type = single\n"
"}",
[Name, Server]
),

9
lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl
View File

@ -17,7 +17,8 @@
%%------------------------------------------------------------------------------
-define(REDIS_TOXYPROXY_CONNECT_CONFIG, #{
<<"server">> => <<"toxiproxy:6379">>
<<"server">> => <<"toxiproxy:6379">>,
<<"redis_type">> => <<"single">>
}).
-define(COMMON_REDIS_OPTS, #{
@ -425,19 +426,23 @@ redis_connect_configs() ->
#{
redis_single => #{
tcp => #{
<<"redis_type">> => <<"single">>,
<<"server">> => <<"redis:6379">>
},
tls => #{
<<"redis_type">> => <<"single">>,
<<"server">> => <<"redis-tls:6380">>,
<<"ssl">> => redis_connect_ssl_opts(redis_single)
}
},
redis_sentinel => #{
tcp => #{
<<"redis_type">> => <<"sentinel">>,
<<"servers">> => <<"redis-sentinel:26379">>,
<<"sentinel">> => <<"mymaster">>
},
tls => #{
<<"redis_type">> => <<"sentinel">>,
<<"servers">> => <<"redis-sentinel-tls:26380">>,
<<"sentinel">> => <<"mymaster">>,
<<"ssl">> => redis_connect_ssl_opts(redis_sentinel)
@ -445,9 +450,11 @@ redis_connect_configs() ->
},
redis_cluster => #{
tcp => #{
<<"redis_type">> => <<"cluster">>,
<<"servers">> => <<"redis-cluster:7000,redis-cluster:7001,redis-cluster:7002">>
},
tls => #{
<<"redis_type">> => <<"cluster">>,
<<"servers">> =>
<<"redis-cluster-tls:8000,redis-cluster-tls:8001,redis-cluster-tls:8002">>,
<<"ssl">> => redis_connect_ssl_opts(redis_cluster)

2
mix.exs
View File

@ -68,7 +68,7 @@ defmodule EMQXUmbrella.MixProject do
# in conflict by emqtt and hocon
{:getopt, "1.0.2", override: true},
{:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true},
{:hocon, github: "emqx/hocon", tag: "0.33.0", override: true},
{:hocon, github: "emqx/hocon", tag: "0.34.0", override: true},
{:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true},
{:esasl, github: "emqx/esasl", tag: "0.2.0"},
{:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},

2
rebar.config
View File

@ -68,7 +68,7 @@
, {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}
, {getopt, "1.0.2"}
, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.34.0"}}}
, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}}
, {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
, {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}