diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 8ff4fea58..8d363aeed 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.34.0"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 49962e490..2fa39d094 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -362,8 +362,8 @@ schema_default(Schema) -> []; ?LAZY(?ARRAY(_)) -> []; - ?LAZY(?UNION(Unions)) -> - case [A || ?ARRAY(A) <- Unions] of + ?LAZY(?UNION(Members)) -> + case [A || ?ARRAY(A) <- hoconsc:union_members(Members)] of [_ | _] -> []; _ -> #{} end; @@ -402,7 +402,6 @@ merge_envs(SchemaMod, RawConf) -> required => false, format => map, apply_override_envs => true, - remove_env_meta => true, check_lazy => true }, hocon_tconf:merge_env_overrides(SchemaMod, RawConf, all, Opts). diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl index e4fadb192..b249dea92 100644 --- a/apps/emqx/test/emqx_schema_tests.erl +++ b/apps/emqx/test/emqx_schema_tests.erl @@ -153,7 +153,7 @@ ssl_opts_gc_after_handshake_test_rancher_listener_test() -> #{ kind := validation_error, reason := unknown_fields, - unknown := <<"gc_after_handshake">> + unknown := "gc_after_handshake" } ]}, validate(Sc, #{<<"gc_after_handshake">> => true}) diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index c876fbf16..f5b9f9da6 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.10"}, + {vsn, "0.1.11"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index d03747b84..a684ae6ba 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -47,14 +47,8 @@ %% Hocon Schema %%-------------------------------------------------------------------- -namespace() -> authz. - -%% @doc authorization schema is not exported -%% but directly used by emqx_schema -roots() -> []. - -fields("authorization") -> - Types = [ +type_names() -> + [ file, http_get, http_post, @@ -67,12 +61,26 @@ fields("authorization") -> redis_single, redis_sentinel, redis_cluster - ], - Unions = [?R_REF(Type) || Type <- Types], + ]. + +namespace() -> authz. + +%% @doc authorization schema is not exported +%% but directly used by emqx_schema +roots() -> []. + +fields("authorization") -> + Types = [?R_REF(Type) || Type <- type_names()], + UnionMemberSelector = + fun + (all_union_members) -> Types; + %% must return list + ({value, Value}) -> [select_union_member(Value)] + end, [ {sources, ?HOCON( - ?ARRAY(?UNION(Unions)), + ?ARRAY(?UNION(UnionMemberSelector)), #{ default => [], desc => ?DESC(sources) @@ -408,9 +416,75 @@ common_rate_field() -> ]. method(Method) -> - ?HOCON(Method, #{default => Method, required => true, desc => ?DESC(method)}). + ?HOCON(Method, #{required => true, desc => ?DESC(method)}). array(Ref) -> array(Ref, Ref). array(Ref, DescId) -> ?HOCON(?ARRAY(?R_REF(Ref)), #{desc => ?DESC(DescId)}). + +select_union_member(#{<<"type">> := <<"mongodb">>} = Value) -> + MongoType = maps:get(<<"mongo_type">>, Value, undefined), + case MongoType of + <<"single">> -> + ?R_REF(mongo_single); + <<"rs">> -> + ?R_REF(mongo_rs); + <<"sharded">> -> + ?R_REF(mongo_sharded); + Else -> + throw(#{ + reason => "unknown_mongo_type", + expected => "single | rs | sharded", + got => Else + }) + end; +select_union_member(#{<<"type">> := <<"redis">>} = Value) -> + RedisType = maps:get(<<"redis_type">>, Value, undefined), + case RedisType of + <<"single">> -> + ?R_REF(redis_single); + <<"cluster">> -> + ?R_REF(redis_cluster); + <<"sentinel">> -> + ?R_REF(redis_sentinel); + Else -> + throw(#{ + reason => "unknown_redis_type", + expected => "single | cluster | sentinel", + got => Else + }) + end; +select_union_member(#{<<"type">> := <<"http">>} = Value) -> + RedisType = maps:get(<<"method">>, Value, undefined), + case RedisType of + <<"get">> -> + ?R_REF(http_get); + <<"post">> -> + ?R_REF(http_post); + Else -> + throw(#{ + reason => "unknown_http_method", + expected => "get | post", + got => Else + }) + end; +select_union_member(#{<<"type">> := <<"built_in_database">>}) -> + ?R_REF(mnesia); +select_union_member(#{<<"type">> := Type}) -> + select_union_member_loop(Type, type_names()); +select_union_member(_) -> + throw("missing_type_field"). + +select_union_member_loop(TypeValue, []) -> + throw(#{ + reason => "unknown_authz_type", + got => TypeValue + }); +select_union_member_loop(TypeValue, [Type | Types]) -> + case TypeValue =:= atom_to_binary(Type) of + true -> + ?R_REF(Type); + false -> + select_union_member_loop(TypeValue, Types) + end. diff --git a/apps/emqx_authz/test/emqx_authz_schema_tests.erl b/apps/emqx_authz/test/emqx_authz_schema_tests.erl new file mode 100644 index 000000000..f7b2e3c10 --- /dev/null +++ b/apps/emqx_authz/test/emqx_authz_schema_tests.erl @@ -0,0 +1,116 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_schema_tests). + +-include_lib("eunit/include/eunit.hrl"). + +bad_authz_type_test() -> + Txt = "[{type: foobar}]", + ?assertThrow( + [ + #{ + reason := "unknown_authz_type", + got := <<"foobar">> + } + ], + check(Txt) + ). + +bad_mongodb_type_test() -> + Txt = "[{type: mongodb, mongo_type: foobar}]", + ?assertThrow( + [ + #{ + reason := "unknown_mongo_type", + got := <<"foobar">> + } + ], + check(Txt) + ). + +missing_mongodb_type_test() -> + Txt = "[{type: mongodb}]", + ?assertThrow( + [ + #{ + reason := "unknown_mongo_type", + got := undefined + } + ], + check(Txt) + ). + +unknown_redis_type_test() -> + Txt = "[{type: redis, redis_type: foobar}]", + ?assertThrow( + [ + #{ + reason := "unknown_redis_type", + got := <<"foobar">> + } + ], + check(Txt) + ). + +missing_redis_type_test() -> + Txt = "[{type: redis}]", + ?assertThrow( + [ + #{ + reason := "unknown_redis_type", + got := undefined + } + ], + check(Txt) + ). + +unknown_http_method_test() -> + Txt = "[{type: http, method: getx}]", + ?assertThrow( + [ + #{ + reason := "unknown_http_method", + got := <<"getx">> + } + ], + check(Txt) + ). + +missing_http_method_test() -> + Txt = "[{type: http, methodx: get}]", + ?assertThrow( + [ + #{ + reason := "unknown_http_method", + got := undefined + } + ], + check(Txt) + ). + +check(Txt0) -> + Txt = ["sources: ", Txt0], + {ok, RawConf} = hocon:binary(Txt), + try + hocon_tconf:check_plain(schema(), RawConf, #{}) + catch + throw:{_Schema, Errors} -> + throw(Errors) + end. + +schema() -> + #{roots => emqx_authz_schema:fields("authorization")}. diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 6fd9ac009..8b471a137 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -316,7 +316,7 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) -> {[Schema | Acc], SubRefs ++ RefsAcc} end, {[], []}, - Types + hoconsc:union_members(Types) ), {#{<<"oneOf">> => OneOf}, Refs}; hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) -> diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index f9e703d20..a735d8c31 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -67,7 +67,6 @@ fields(single) -> [ {mongo_type, #{ type => single, - default => single, required => true, desc => ?DESC("single_mongo_type") }}, @@ -78,7 +77,6 @@ fields(rs) -> [ {mongo_type, #{ type => rs, - default => rs, required => true, desc => ?DESC("rs_mongo_type") }}, @@ -91,7 +89,6 @@ fields(sharded) -> [ {mongo_type, #{ type => sharded, - default => sharded, required => true, desc => ?DESC("sharded_mongo_type") }}, diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 350d49f01..11d71f1df 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -63,7 +63,6 @@ fields(single) -> {server, server()}, {redis_type, #{ type => single, - default => single, required => true, desc => ?DESC("single") }} @@ -75,7 +74,6 @@ fields(cluster) -> {servers, servers()}, {redis_type, #{ type => cluster, - default => cluster, required => true, desc => ?DESC("cluster") }} @@ -87,7 +85,6 @@ fields(sentinel) -> {servers, servers()}, {redis_type, #{ type => sentinel, - default => sentinel, required => true, desc => ?DESC("sentinel") }}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 4b7a672bd..85b928dda 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -623,7 +623,7 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) -> {[Schema | Acc], SubRefs ++ RefsAcc} end, {[], []}, - Types + hoconsc:union_members(Types) ), {#{<<"oneOf">> => OneOf}, Refs}; hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) -> diff --git a/changes/refactor-9653.en.md b/changes/refactor-9653.en.md new file mode 100644 index 000000000..2807f81d5 --- /dev/null +++ b/changes/refactor-9653.en.md @@ -0,0 +1 @@ +Make authorization config validation error message more readable. diff --git a/changes/refactor-9653.zh.md b/changes/refactor-9653.zh.md new file mode 100644 index 000000000..755fd1683 --- /dev/null +++ b/changes/refactor-9653.zh.md @@ -0,0 +1 @@ +改进授权配置检查错误日志的可读性。 diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index 2bd4036e0..6ca554c72 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -1,6 +1,5 @@ {erl_opts, [debug_info]}. -{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}} - , {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.4"}}} +{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.4"}}} , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.2"}}} , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}} , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.7"}}} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl index 727a6df4b..a0c6ba834 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl @@ -50,19 +50,22 @@ values(Protocol, get) -> values("single", post) -> SpecificOpts = #{ server => <<"127.0.0.1:6379">>, + redis_type => single, database => 1 }, values(common, "single", SpecificOpts); values("sentinel", post) -> SpecificOpts = #{ servers => [<<"127.0.0.1:26379">>], + redis_type => sentinel, sentinel => <<"mymaster">>, database => 1 }, values(common, "sentinel", SpecificOpts); values("cluster", post) -> SpecificOpts = #{ - servers => [<<"127.0.0.1:6379">>] + servers => [<<"127.0.0.1:6379">>], + redis_type => cluster }, values(common, "cluster", SpecificOpts); values(Protocol, put) -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index fb8f1fcc3..05c513eb1 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -151,6 +151,7 @@ mongo_config(MongoHost, MongoPort0, rs = Type) -> " servers = [~p]\n" " w_mode = safe\n" " database = mqtt\n" + " mongo_type = rs\n" "}", [Name, Servers] ), @@ -167,6 +168,7 @@ mongo_config(MongoHost, MongoPort0, sharded = Type) -> " servers = [~p]\n" " w_mode = safe\n" " database = mqtt\n" + " mongo_type = sharded\n" "}", [Name, Servers] ), @@ -183,6 +185,7 @@ mongo_config(MongoHost, MongoPort0, single = Type) -> " server = ~p\n" " w_mode = safe\n" " database = mqtt\n" + " mongo_type = single\n" "}", [Name, Server] ), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl index 1f4b52ddc..fb5f688a6 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl @@ -17,7 +17,8 @@ %%------------------------------------------------------------------------------ -define(REDIS_TOXYPROXY_CONNECT_CONFIG, #{ - <<"server">> => <<"toxiproxy:6379">> + <<"server">> => <<"toxiproxy:6379">>, + <<"redis_type">> => <<"single">> }). -define(COMMON_REDIS_OPTS, #{ @@ -425,19 +426,23 @@ redis_connect_configs() -> #{ redis_single => #{ tcp => #{ + <<"redis_type">> => <<"single">>, <<"server">> => <<"redis:6379">> }, tls => #{ + <<"redis_type">> => <<"single">>, <<"server">> => <<"redis-tls:6380">>, <<"ssl">> => redis_connect_ssl_opts(redis_single) } }, redis_sentinel => #{ tcp => #{ + <<"redis_type">> => <<"sentinel">>, <<"servers">> => <<"redis-sentinel:26379">>, <<"sentinel">> => <<"mymaster">> }, tls => #{ + <<"redis_type">> => <<"sentinel">>, <<"servers">> => <<"redis-sentinel-tls:26380">>, <<"sentinel">> => <<"mymaster">>, <<"ssl">> => redis_connect_ssl_opts(redis_sentinel) @@ -445,9 +450,11 @@ redis_connect_configs() -> }, redis_cluster => #{ tcp => #{ + <<"redis_type">> => <<"cluster">>, <<"servers">> => <<"redis-cluster:7000,redis-cluster:7001,redis-cluster:7002">> }, tls => #{ + <<"redis_type">> => <<"cluster">>, <<"servers">> => <<"redis-cluster-tls:8000,redis-cluster-tls:8001,redis-cluster-tls:8002">>, <<"ssl">> => redis_connect_ssl_opts(redis_cluster) diff --git a/mix.exs b/mix.exs index 265448d72..b7fc7dac8 100644 --- a/mix.exs +++ b/mix.exs @@ -68,7 +68,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.33.0", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.34.0", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 4476a128c..878c911e5 100644 --- a/rebar.config +++ b/rebar.config @@ -68,7 +68,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.34.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}