Merge pull request #9653 from zmstone/0101-authz-schema-union-member-selection

0101 authz schema union member selection
This commit is contained in:
Zaiming (Stone) Shi 2023-01-09 22:17:51 +01:00 committed by GitHub
commit 67f2159a27
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 267 additions and 62 deletions

View File

@ -29,7 +29,7 @@
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.34.0"}}},
{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
{recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},
{snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}

View File

@ -362,8 +362,8 @@ schema_default(Schema) ->
[];
?LAZY(?ARRAY(_)) ->
[];
?LAZY(?UNION(Unions)) ->
case [A || ?ARRAY(A) <- Unions] of
?LAZY(?UNION(Members)) ->
case [A || ?ARRAY(A) <- hoconsc:union_members(Members)] of
[_ | _] -> [];
_ -> #{}
end;
@ -402,7 +402,6 @@ merge_envs(SchemaMod, RawConf) ->
required => false,
format => map,
apply_override_envs => true,
remove_env_meta => true,
check_lazy => true
},
hocon_tconf:merge_env_overrides(SchemaMod, RawConf, all, Opts).

View File

@ -153,7 +153,7 @@ ssl_opts_gc_after_handshake_test_rancher_listener_test() ->
#{
kind := validation_error,
reason := unknown_fields,
unknown := <<"gc_after_handshake">>
unknown := "gc_after_handshake"
}
]},
validate(Sc, #{<<"gc_after_handshake">> => true})

View File

@ -47,14 +47,8 @@
%% Hocon Schema
%%--------------------------------------------------------------------
namespace() -> authz.
%% @doc authorization schema is not exported
%% but directly used by emqx_schema
roots() -> [].
fields("authorization") ->
Types = [
type_names() ->
[
file,
http_get,
http_post,
@ -67,12 +61,26 @@ fields("authorization") ->
redis_single,
redis_sentinel,
redis_cluster
],
Unions = [?R_REF(Type) || Type <- Types],
].
namespace() -> authz.
%% @doc authorization schema is not exported
%% but directly used by emqx_schema
roots() -> [].
fields("authorization") ->
Types = [?R_REF(Type) || Type <- type_names()],
UnionMemberSelector =
fun
(all_union_members) -> Types;
%% must return list
({value, Value}) -> [select_union_member(Value)]
end,
[
{sources,
?HOCON(
?ARRAY(?UNION(Unions)),
?ARRAY(?UNION(UnionMemberSelector)),
#{
default => [],
desc => ?DESC(sources)
@ -408,9 +416,75 @@ common_rate_field() ->
].
method(Method) ->
?HOCON(Method, #{default => Method, required => true, desc => ?DESC(method)}).
?HOCON(Method, #{required => true, desc => ?DESC(method)}).
array(Ref) -> array(Ref, Ref).
array(Ref, DescId) ->
?HOCON(?ARRAY(?R_REF(Ref)), #{desc => ?DESC(DescId)}).
select_union_member(#{<<"type">> := <<"mongodb">>} = Value) ->
MongoType = maps:get(<<"mongo_type">>, Value, undefined),
case MongoType of
<<"single">> ->
?R_REF(mongo_single);
<<"rs">> ->
?R_REF(mongo_rs);
<<"sharded">> ->
?R_REF(mongo_sharded);
Else ->
throw(#{
reason => "unknown_mongo_type",
expected => "single | rs | sharded",
got => Else
})
end;
select_union_member(#{<<"type">> := <<"redis">>} = Value) ->
RedisType = maps:get(<<"redis_type">>, Value, undefined),
case RedisType of
<<"single">> ->
?R_REF(redis_single);
<<"cluster">> ->
?R_REF(redis_cluster);
<<"sentinel">> ->
?R_REF(redis_sentinel);
Else ->
throw(#{
reason => "unknown_redis_type",
expected => "single | cluster | sentinel",
got => Else
})
end;
select_union_member(#{<<"type">> := <<"http">>} = Value) ->
RedisType = maps:get(<<"method">>, Value, undefined),
case RedisType of
<<"get">> ->
?R_REF(http_get);
<<"post">> ->
?R_REF(http_post);
Else ->
throw(#{
reason => "unknown_http_method",
expected => "get | post",
got => Else
})
end;
select_union_member(#{<<"type">> := <<"built_in_database">>}) ->
?R_REF(mnesia);
select_union_member(#{<<"type">> := Type}) ->
select_union_member_loop(Type, type_names());
select_union_member(_) ->
throw("missing_type_field").
select_union_member_loop(TypeValue, []) ->
throw(#{
reason => "unknown_authz_type",
got => TypeValue
});
select_union_member_loop(TypeValue, [Type | Types]) ->
case TypeValue =:= atom_to_binary(Type) of
true ->
?R_REF(Type);
false ->
select_union_member_loop(TypeValue, Types)
end.

View File

@ -0,0 +1,116 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_authz_schema_tests).
-include_lib("eunit/include/eunit.hrl").
bad_authz_type_test() ->
Txt = "[{type: foobar}]",
?assertThrow(
[
#{
reason := "unknown_authz_type",
got := <<"foobar">>
}
],
check(Txt)
).
bad_mongodb_type_test() ->
Txt = "[{type: mongodb, mongo_type: foobar}]",
?assertThrow(
[
#{
reason := "unknown_mongo_type",
got := <<"foobar">>
}
],
check(Txt)
).
missing_mongodb_type_test() ->
Txt = "[{type: mongodb}]",
?assertThrow(
[
#{
reason := "unknown_mongo_type",
got := undefined
}
],
check(Txt)
).
unknown_redis_type_test() ->
Txt = "[{type: redis, redis_type: foobar}]",
?assertThrow(
[
#{
reason := "unknown_redis_type",
got := <<"foobar">>
}
],
check(Txt)
).
missing_redis_type_test() ->
Txt = "[{type: redis}]",
?assertThrow(
[
#{
reason := "unknown_redis_type",
got := undefined
}
],
check(Txt)
).
unknown_http_method_test() ->
Txt = "[{type: http, method: getx}]",
?assertThrow(
[
#{
reason := "unknown_http_method",
got := <<"getx">>
}
],
check(Txt)
).
missing_http_method_test() ->
Txt = "[{type: http, methodx: get}]",
?assertThrow(
[
#{
reason := "unknown_http_method",
got := undefined
}
],
check(Txt)
).
check(Txt0) ->
Txt = ["sources: ", Txt0],
{ok, RawConf} = hocon:binary(Txt),
try
hocon_tconf:check_plain(schema(), RawConf, #{})
catch
throw:{_Schema, Errors} ->
throw(Errors)
end.
schema() ->
#{roots => emqx_authz_schema:fields("authorization")}.

View File

@ -316,7 +316,7 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) ->
{[Schema | Acc], SubRefs ++ RefsAcc}
end,
{[], []},
Types
hoconsc:union_members(Types)
),
{#{<<"oneOf">> => OneOf}, Refs};
hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->

View File

@ -2,34 +2,34 @@ emqx_connector_mongo {
single_mongo_type {
desc {
en: "Standalone instance."
zh: "Standalone模式。"
en: "Standalone instance. Must be set to 'single' when MongoDB server is running in standalone mode."
zh: "Standalone 模式。当 MongoDB 服务运行在 standalone 模式下,该配置必须设置为 'single'。 "
}
label: {
en: "Standalone instance"
zh: "Standalone模式"
zh: "Standalone 模式"
}
}
rs_mongo_type {
desc {
en: "Replica set."
zh: "Replica set模式。"
en: "Replica set. Must be set to 'rs' when MongoDB server is running in 'replica set' mode."
zh: "Replica set模式。当 MongoDB 服务运行在 replica-set 模式下,该配置必须设置为 'rs'。"
}
label: {
en: "Replica set"
zh: "Replica set模式"
zh: "Replica set 模式"
}
}
sharded_mongo_type {
desc {
en: "Sharded cluster."
zh: "Sharded cluster模式。"
en: "Sharded cluster. Must be set to 'sharded' when MongoDB server is running in 'sharded' mode."
zh: "Sharded cluster模式。当 MongoDB 服务运行在 sharded 模式下,该配置必须设置为 'sharded'。"
}
label: {
en: "Sharded cluster"
zh: "Sharded cluster模式"
zh: "Sharded cluster 模式"
}
}

View File

@ -2,8 +2,8 @@ emqx_connector_redis {
single {
desc {
en: "Single mode"
zh: "单机模式。"
en: "Single mode. Must be set to 'single' when Redis server is running in single mode."
zh: "单机模式。当 Redis 服务运行在单机模式下,该配置必须设置为 'single'。"
}
label: {
en: "Single Mode"
@ -13,8 +13,8 @@ emqx_connector_redis {
cluster {
desc {
en: "Cluster mode"
zh: "集群模式。"
en: "Cluster mode. Must be set to 'cluster' when Redis server is running in clustered mode."
zh: "集群模式。当 Redis 服务运行在集群模式下,该配置必须设置为 'cluster'。"
}
label: {
en: "Cluster Mode"
@ -24,8 +24,8 @@ emqx_connector_redis {
sentinel {
desc {
en: "Sentinel mode"
zh: "哨兵模式。"
en: "Sentinel mode. Must be set to 'sentinel' when Redis server is running in sentinel mode."
zh: "哨兵模式。当 Redis 服务运行在哨兵模式下,该配置必须设置为 'sentinel'。"
}
label: {
en: "Sentinel Mode"

View File

@ -68,7 +68,6 @@ fields(single) ->
{mongo_type, #{
type => single,
default => single,
required => true,
desc => ?DESC("single_mongo_type")
}},
{server, server()},
@ -79,7 +78,6 @@ fields(rs) ->
{mongo_type, #{
type => rs,
default => rs,
required => true,
desc => ?DESC("rs_mongo_type")
}},
{servers, servers()},
@ -92,7 +90,6 @@ fields(sharded) ->
{mongo_type, #{
type => sharded,
default => sharded,
required => true,
desc => ?DESC("sharded_mongo_type")
}},
{servers, servers()},

View File

@ -64,7 +64,7 @@ fields(single) ->
{redis_type, #{
type => single,
default => single,
required => true,
required => false,
desc => ?DESC("single")
}}
] ++
@ -76,7 +76,7 @@ fields(cluster) ->
{redis_type, #{
type => cluster,
default => cluster,
required => true,
required => false,
desc => ?DESC("cluster")
}}
] ++
@ -88,7 +88,7 @@ fields(sentinel) ->
{redis_type, #{
type => sentinel,
default => sentinel,
required => true,
required => false,
desc => ?DESC("sentinel")
}},
{sentinel, #{

View File

@ -75,9 +75,9 @@ wait_for_redis(Checks) ->
wait_for_redis(Checks - 1)
end.
% %%------------------------------------------------------------------------------
% %% Testcases
% %%------------------------------------------------------------------------------
%%------------------------------------------------------------------------------
%% Testcases
%%------------------------------------------------------------------------------
t_single_lifecycle(_Config) ->
perform_lifecycle_check(

View File

@ -623,7 +623,7 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) ->
{[Schema | Acc], SubRefs ++ RefsAcc}
end,
{[], []},
Types
hoconsc:union_members(Types)
),
{#{<<"oneOf">> => OneOf}, Refs};
hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->

3
build
View File

@ -135,6 +135,9 @@ assert_no_compile_time_only_deps() {
make_rel() {
./scripts/pre-compile.sh "$PROFILE"
# make_elixir_rel always create rebar.lock
# delete it to make git clone + checkout work because we use shallow close for rebar deps
rm -f rebar.lock
# compile all beams
./rebar3 as "$PROFILE" compile
# generate docs (require beam compiled), generated to etc and priv dirs

View File

@ -0,0 +1 @@
Make authorization config validation error message more readable.

View File

@ -0,0 +1 @@
改进授权配置检查错误日志的可读性。

View File

@ -1,6 +1,5 @@
{erl_opts, [debug_info]}.
{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}
, {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.4"}}}
{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.4"}}}
, {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.2"}}}
, {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}}
, {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.7"}}}

View File

@ -50,19 +50,22 @@ values(Protocol, get) ->
values("single", post) ->
SpecificOpts = #{
server => <<"127.0.0.1:6379">>,
redis_type => single,
database => 1
},
values(common, "single", SpecificOpts);
values("sentinel", post) ->
SpecificOpts = #{
servers => [<<"127.0.0.1:26379">>],
redis_type => sentinel,
sentinel => <<"mymaster">>,
database => 1
},
values(common, "sentinel", SpecificOpts);
values("cluster", post) ->
SpecificOpts = #{
servers => [<<"127.0.0.1:6379">>]
servers => [<<"127.0.0.1:6379">>],
redis_type => cluster
},
values(common, "cluster", SpecificOpts);
values(Protocol, put) ->

View File

@ -17,7 +17,8 @@
%%------------------------------------------------------------------------------
-define(REDIS_TOXYPROXY_CONNECT_CONFIG, #{
<<"server">> => <<"toxiproxy:6379">>
<<"server">> => <<"toxiproxy:6379">>,
<<"redis_type">> => <<"single">>
}).
-define(COMMON_REDIS_OPTS, #{
@ -31,7 +32,7 @@
-define(PROXY_HOST, "toxiproxy").
-define(PROXY_PORT, "8474").
all() -> [{group, redis_types}, {group, rest}].
all() -> [{group, transport_types}, {group, rest}].
groups() ->
ResourceSpecificTCs = [t_create_delete_bridge],
@ -47,7 +48,7 @@ groups() ->
],
[
{rest, TCs},
{redis_types, [
{transport_types, [
{group, tcp},
{group, tls}
]},
@ -63,7 +64,7 @@ groups() ->
init_per_group(Group, Config) when
Group =:= redis_single; Group =:= redis_sentinel; Group =:= redis_cluster
->
[{redis_type, Group} | Config];
[{transport_type, Group} | Config];
init_per_group(Group, Config) when
Group =:= tcp; Group =:= tls
->
@ -79,6 +80,12 @@ end_per_group(_Group, _Config) ->
ok.
init_per_suite(Config) ->
wait_for_ci_redis(redis_checks(), Config).
wait_for_ci_redis(0, _Config) ->
throw(no_redis);
wait_for_ci_redis(Checks, Config) ->
timer:sleep(1000),
TestHosts = all_test_hosts(),
case emqx_common_test_helpers:is_all_tcp_servers_available(TestHosts) of
true ->
@ -96,15 +103,15 @@ init_per_suite(Config) ->
| Config
];
false ->
assert_ci()
wait_for_ci_redis(Checks - 1, Config)
end.
assert_ci() ->
redis_checks() ->
case os:getenv("IS_CI") of
"yes" ->
throw(no_redis);
10;
_ ->
{skip, no_redis}
1
end.
end_per_suite(_Config) ->
@ -116,7 +123,7 @@ end_per_suite(_Config) ->
init_per_testcase(_Testcase, Config) ->
ok = delete_all_bridges(),
case ?config(redis_type, Config) of
case ?config(transport_type, Config) of
undefined ->
Config;
RedisType ->
@ -139,7 +146,7 @@ end_per_testcase(_Testcase, Config) ->
t_create_delete_bridge(Config) ->
Name = <<"mybridge">>,
Type = ?config(redis_type, Config),
Type = ?config(transport_type, Config),
BridgeConfig = ?config(bridge_config, Config),
IsBatch = ?config(is_batch, Config),
?assertMatch(
@ -425,31 +432,37 @@ redis_connect_configs() ->
#{
redis_single => #{
tcp => #{
<<"server">> => <<"redis:6379">>
<<"server">> => <<"redis:6379">>,
<<"redis_type">> => <<"single">>
},
tls => #{
<<"server">> => <<"redis-tls:6380">>,
<<"ssl">> => redis_connect_ssl_opts(redis_single)
<<"ssl">> => redis_connect_ssl_opts(redis_single),
<<"redis_type">> => <<"single">>
}
},
redis_sentinel => #{
tcp => #{
<<"servers">> => <<"redis-sentinel:26379">>,
<<"redis_type">> => <<"sentinel">>,
<<"sentinel">> => <<"mymaster">>
},
tls => #{
<<"servers">> => <<"redis-sentinel-tls:26380">>,
<<"redis_type">> => <<"sentinel">>,
<<"sentinel">> => <<"mymaster">>,
<<"ssl">> => redis_connect_ssl_opts(redis_sentinel)
}
},
redis_cluster => #{
tcp => #{
<<"servers">> => <<"redis-cluster:7000,redis-cluster:7001,redis-cluster:7002">>
<<"servers">> => <<"redis-cluster:7000,redis-cluster:7001,redis-cluster:7002">>,
<<"redis_type">> => <<"cluster">>
},
tls => #{
<<"servers">> =>
<<"redis-cluster-tls:8000,redis-cluster-tls:8001,redis-cluster-tls:8002">>,
<<"redis_type">> => <<"cluster">>,
<<"ssl">> => redis_connect_ssl_opts(redis_cluster)
}
}

View File

@ -68,7 +68,7 @@ defmodule EMQXUmbrella.MixProject do
# in conflict by emqtt and hocon
{:getopt, "1.0.2", override: true},
{:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true},
{:hocon, github: "emqx/hocon", tag: "0.33.0", override: true},
{:hocon, github: "emqx/hocon", tag: "0.34.0", override: true},
{:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true},
{:esasl, github: "emqx/esasl", tag: "0.2.0"},
{:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},

View File

@ -68,7 +68,7 @@
, {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}
, {getopt, "1.0.2"}
, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.34.0"}}}
, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}}
, {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
, {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}

View File

@ -11,7 +11,6 @@ help() {
echo
echo "-h|--help: To display this usage info"
echo "--app lib_dir/app_name: For which app to run start docker-compose, and run common tests"
echo "--suites SUITE1,SUITE2: Comma separated SUITE names to run. e.g. apps/emqx/test/emqx_SUITE.erl"
echo "--console: Start EMQX in console mode but do not run test cases"
echo "--attach: Attach to the Erlang docker container without running any test case"
echo "--stop: Stop running containers for the given app"