refactor: use union member type selector for authz sources

apps/emqx/rebar.config

@@ -29,7 +29,7 @@
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}},
     {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
-    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}},
+    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.34.0"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
     {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},
     {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}

apps/emqx/src/emqx_config.erl

@@ -362,8 +362,8 @@ schema_default(Schema) ->
             [];
         ?LAZY(?ARRAY(_)) ->
             [];
-        ?LAZY(?UNION(Unions)) ->
+        ?LAZY(?UNION(Members)) ->
-            case [A || ?ARRAY(A) <- Unions] of
+            case [A || ?ARRAY(A) <- hoconsc:union_members(Members)] of
                 [_ | _] -> [];
                 _ -> #{}
             end;
@@ -402,7 +402,6 @@ merge_envs(SchemaMod, RawConf) ->
         required => false,
         format => map,
         apply_override_envs => true,
-        remove_env_meta => true,
         check_lazy => true
     },
     hocon_tconf:merge_env_overrides(SchemaMod, RawConf, all, Opts).

apps/emqx/test/emqx_schema_tests.erl

@@ -153,7 +153,7 @@ ssl_opts_gc_after_handshake_test_rancher_listener_test() ->
             #{
                 kind := validation_error,
                 reason := unknown_fields,
-                unknown := <<"gc_after_handshake">>
+                unknown := "gc_after_handshake"
             }
         ]},
         validate(Sc, #{<<"gc_after_handshake">> => true})

apps/emqx_authz/src/emqx_authz.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authz, [
     {description, "An OTP application"},
-    {vsn, "0.1.10"},
+    {vsn, "0.1.11"},
     {registered, []},
     {mod, {emqx_authz_app, []}},
     {applications, [

apps/emqx_authz/src/emqx_authz_schema.erl

@@ -47,14 +47,8 @@
 %% Hocon Schema
 %%--------------------------------------------------------------------
 
-namespace() -> authz.
+type_names() ->
-
+    [
-%% @doc authorization schema is not exported
-%% but directly used by emqx_schema
-roots() -> [].
-
-fields("authorization") ->
-    Types = [
         file,
         http_get,
         http_post,
@@ -67,12 +61,26 @@ fields("authorization") ->
         redis_single,
         redis_sentinel,
         redis_cluster
-    ],
+    ].
-    Unions = [?R_REF(Type) || Type <- Types],
+
+namespace() -> authz.
+
+%% @doc authorization schema is not exported
+%% but directly used by emqx_schema
+roots() -> [].
+
+fields("authorization") ->
+    Types = [?R_REF(Type) || Type <- type_names()],
+    UnionMemberSelector =
+        fun
+            (all_union_members) -> Types;
+            %% must return list
+            ({value, Value}) -> [select_union_member(Value)]
+        end,
     [
         {sources,
             ?HOCON(
-                ?ARRAY(?UNION(Unions)),
+                ?ARRAY(?UNION(UnionMemberSelector)),
                 #{
                     default => [],
                     desc => ?DESC(sources)
@@ -408,9 +416,75 @@ common_rate_field() ->
     ].
 
 method(Method) ->
-    ?HOCON(Method, #{default => Method, required => true, desc => ?DESC(method)}).
+    ?HOCON(Method, #{required => true, desc => ?DESC(method)}).
 
 array(Ref) -> array(Ref, Ref).
 
 array(Ref, DescId) ->
     ?HOCON(?ARRAY(?R_REF(Ref)), #{desc => ?DESC(DescId)}).
+
+select_union_member(#{<<"type">> := <<"mongodb">>} = Value) ->
+    MongoType = maps:get(<<"mongo_type">>, Value, undefined),
+    case MongoType of
+        <<"single">> ->
+            ?R_REF(mongo_single);
+        <<"rs">> ->
+            ?R_REF(mongo_rs);
+        <<"sharded">> ->
+            ?R_REF(mongo_sharded);
+        Else ->
+            throw(#{
+                reason => "unknown_mongo_type",
+                expected => "single | rs | sharded",
+                got => Else
+            })
+    end;
+select_union_member(#{<<"type">> := <<"redis">>} = Value) ->
+    RedisType = maps:get(<<"redis_type">>, Value, undefined),
+    case RedisType of
+        <<"single">> ->
+            ?R_REF(redis_single);
+        <<"cluster">> ->
+            ?R_REF(redis_cluster);
+        <<"sentinel">> ->
+            ?R_REF(redis_sentinel);
+        Else ->
+            throw(#{
+                reason => "unknown_redis_type",
+                expected => "single | cluster | sentinel",
+                got => Else
+            })
+    end;
+select_union_member(#{<<"type">> := <<"http">>} = Value) ->
+    RedisType = maps:get(<<"method">>, Value, undefined),
+    case RedisType of
+        <<"get">> ->
+            ?R_REF(http_get);
+        <<"post">> ->
+            ?R_REF(http_post);
+        Else ->
+            throw(#{
+                reason => "unknown_http_method",
+                expected => "get | post",
+                got => Else
+            })
+    end;
+select_union_member(#{<<"type">> := <<"built_in_database">>}) ->
+    ?R_REF(mnesia);
+select_union_member(#{<<"type">> := Type}) ->
+    select_union_member_loop(Type, type_names());
+select_union_member(_) ->
+    throw("missing_type_field").
+
+select_union_member_loop(TypeValue, []) ->
+    throw(#{
+        reason => "unknown_authz_type",
+        got => TypeValue
+    });
+select_union_member_loop(TypeValue, [Type | Types]) ->
+    case TypeValue =:= atom_to_binary(Type) of
+        true ->
+            ?R_REF(Type);
+        false ->
+            select_union_member_loop(TypeValue, Types)
+    end.

apps/emqx_authz/test/emqx_authz_schema_tests.erl

@@ -0,0 +1,116 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_authz_schema_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+bad_authz_type_test() ->
+    Txt = "[{type: foobar}]",
+    ?assertThrow(
+        [
+            #{
+                reason := "unknown_authz_type",
+                got := <<"foobar">>
+            }
+        ],
+        check(Txt)
+    ).
+
+bad_mongodb_type_test() ->
+    Txt = "[{type: mongodb, mongo_type: foobar}]",
+    ?assertThrow(
+        [
+            #{
+                reason := "unknown_mongo_type",
+                got := <<"foobar">>
+            }
+        ],
+        check(Txt)
+    ).
+
+missing_mongodb_type_test() ->
+    Txt = "[{type: mongodb}]",
+    ?assertThrow(
+        [
+            #{
+                reason := "unknown_mongo_type",
+                got := undefined
+            }
+        ],
+        check(Txt)
+    ).
+
+unknown_redis_type_test() ->
+    Txt = "[{type: redis, redis_type: foobar}]",
+    ?assertThrow(
+        [
+            #{
+                reason := "unknown_redis_type",
+                got := <<"foobar">>
+            }
+        ],
+        check(Txt)
+    ).
+
+missing_redis_type_test() ->
+    Txt = "[{type: redis}]",
+    ?assertThrow(
+        [
+            #{
+                reason := "unknown_redis_type",
+                got := undefined
+            }
+        ],
+        check(Txt)
+    ).
+
+unknown_http_method_test() ->
+    Txt = "[{type: http, method: getx}]",
+    ?assertThrow(
+        [
+            #{
+                reason := "unknown_http_method",
+                got := <<"getx">>
+            }
+        ],
+        check(Txt)
+    ).
+
+missing_http_method_test() ->
+    Txt = "[{type: http, methodx: get}]",
+    ?assertThrow(
+        [
+            #{
+                reason := "unknown_http_method",
+                got := undefined
+            }
+        ],
+        check(Txt)
+    ).
+
+check(Txt0) ->
+    Txt = ["sources: ", Txt0],
+    {ok, RawConf} = hocon:binary(Txt),
+    try
+        hocon_tconf:check_plain(schema(), RawConf, #{})
+    catch
+        throw:{_Schema, Errors} ->
+            throw(Errors)
+    end.
+
+schema() ->
+    #{roots => emqx_authz_schema:fields("authorization")}.

apps/emqx_conf/src/emqx_conf.erl

@@ -316,7 +316,7 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) ->
             {[Schema | Acc], SubRefs ++ RefsAcc}
         end,
         {[], []},
-        Types
+        hoconsc:union_members(Types)
     ),
     {#{<<"oneOf">> => OneOf}, Refs};
 hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->

apps/emqx_connector/src/emqx_connector_mongo.erl

@@ -67,7 +67,6 @@ fields(single) ->
     [
         {mongo_type, #{
             type => single,
-            default => single,
             required => true,
             desc => ?DESC("single_mongo_type")
         }},
@@ -78,7 +77,6 @@ fields(rs) ->
     [
         {mongo_type, #{
             type => rs,
-            default => rs,
             required => true,
             desc => ?DESC("rs_mongo_type")
         }},
@@ -91,7 +89,6 @@ fields(sharded) ->
     [
         {mongo_type, #{
             type => sharded,
-            default => sharded,
             required => true,
             desc => ?DESC("sharded_mongo_type")
         }},

apps/emqx_connector/src/emqx_connector_redis.erl

@@ -63,7 +63,6 @@ fields(single) ->
         {server, server()},
         {redis_type, #{
             type => single,
-            default => single,
             required => true,
             desc => ?DESC("single")
         }}
@@ -75,7 +74,6 @@ fields(cluster) ->
         {servers, servers()},
         {redis_type, #{
             type => cluster,
-            default => cluster,
             required => true,
             desc => ?DESC("cluster")
         }}
@@ -87,7 +85,6 @@ fields(sentinel) ->
         {servers, servers()},
         {redis_type, #{
             type => sentinel,
-            default => sentinel,
             required => true,
             desc => ?DESC("sentinel")
         }},

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -623,7 +623,7 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) ->
             {[Schema | Acc], SubRefs ++ RefsAcc}
         end,
         {[], []},
-        Types
+        hoconsc:union_members(Types)
     ),
     {#{<<"oneOf">> => OneOf}, Refs};
 hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->

changes/refactor-9653.en.md

@@ -0,0 +1 @@
+Make authorization config validation error message more readable.

changes/refactor-9653.zh.md

@@ -0,0 +1 @@
+改进授权配置检查错误日志的可读性。

lib-ee/emqx_ee_bridge/rebar.config

@@ -1,6 +1,5 @@
 {erl_opts, [debug_info]}.
-{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}
+{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.4"}}}
-       , {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.4"}}}
        , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.2"}}}
        , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}}
        , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.7"}}}

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl

@@ -50,19 +50,22 @@ values(Protocol, get) ->
 values("single", post) ->
     SpecificOpts = #{
         server => <<"127.0.0.1:6379">>,
+        redis_type => single,
         database => 1
     },
     values(common, "single", SpecificOpts);
 values("sentinel", post) ->
     SpecificOpts = #{
         servers => [<<"127.0.0.1:26379">>],
+        redis_type => sentinel,
         sentinel => <<"mymaster">>,
         database => 1
     },
     values(common, "sentinel", SpecificOpts);
 values("cluster", post) ->
     SpecificOpts = #{
-        servers => [<<"127.0.0.1:6379">>]
+        servers => [<<"127.0.0.1:6379">>],
+        redis_type => cluster
     },
     values(common, "cluster", SpecificOpts);
 values(Protocol, put) ->

lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl

@@ -151,6 +151,7 @@ mongo_config(MongoHost, MongoPort0, rs = Type) ->
             "  servers = [~p]\n"
             "  w_mode = safe\n"
             "  database = mqtt\n"
+            "  mongo_type = rs\n"
             "}",
             [Name, Servers]
         ),
@@ -167,6 +168,7 @@ mongo_config(MongoHost, MongoPort0, sharded = Type) ->
             "  servers = [~p]\n"
             "  w_mode = safe\n"
             "  database = mqtt\n"
+            "  mongo_type = sharded\n"
             "}",
             [Name, Servers]
         ),
@@ -183,6 +185,7 @@ mongo_config(MongoHost, MongoPort0, single = Type) ->
             "  server = ~p\n"
             "  w_mode = safe\n"
             "  database = mqtt\n"
+            "  mongo_type = single\n"
             "}",
             [Name, Server]
         ),

lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl

@@ -17,7 +17,8 @@
 %%------------------------------------------------------------------------------
 
 -define(REDIS_TOXYPROXY_CONNECT_CONFIG, #{
-    <<"server">> => <<"toxiproxy:6379">>
+    <<"server">> => <<"toxiproxy:6379">>,
+    <<"redis_type">> => <<"single">>
 }).
 
 -define(COMMON_REDIS_OPTS, #{
@@ -425,19 +426,23 @@ redis_connect_configs() ->
     #{
         redis_single => #{
             tcp => #{
+                <<"redis_type">> => <<"single">>,
                 <<"server">> => <<"redis:6379">>
             },
             tls => #{
+                <<"redis_type">> => <<"single">>,
                 <<"server">> => <<"redis-tls:6380">>,
                 <<"ssl">> => redis_connect_ssl_opts(redis_single)
             }
         },
         redis_sentinel => #{
             tcp => #{
+                <<"redis_type">> => <<"sentinel">>,
                 <<"servers">> => <<"redis-sentinel:26379">>,
                 <<"sentinel">> => <<"mymaster">>
             },
             tls => #{
+                <<"redis_type">> => <<"sentinel">>,
                 <<"servers">> => <<"redis-sentinel-tls:26380">>,
                 <<"sentinel">> => <<"mymaster">>,
                 <<"ssl">> => redis_connect_ssl_opts(redis_sentinel)
@@ -445,9 +450,11 @@ redis_connect_configs() ->
         },
         redis_cluster => #{
             tcp => #{
+                <<"redis_type">> => <<"cluster">>,
                 <<"servers">> => <<"redis-cluster:7000,redis-cluster:7001,redis-cluster:7002">>
             },
             tls => #{
+                <<"redis_type">> => <<"cluster">>,
                 <<"servers">> =>
                     <<"redis-cluster-tls:8000,redis-cluster-tls:8001,redis-cluster-tls:8002">>,
                 <<"ssl">> => redis_connect_ssl_opts(redis_cluster)

mix.exs

@@ -68,7 +68,7 @@ defmodule EMQXUmbrella.MixProject do
       # in conflict by emqtt and hocon
       {:getopt, "1.0.2", override: true},
       {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true},
-      {:hocon, github: "emqx/hocon", tag: "0.33.0", override: true},
+      {:hocon, github: "emqx/hocon", tag: "0.34.0", override: true},
       {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true},
       {:esasl, github: "emqx/esasl", tag: "0.2.0"},
       {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},

rebar.config

@@ -68,7 +68,7 @@
     , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}
     , {getopt, "1.0.2"}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.34.0"}}}
     , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}}
     , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
     , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}