Merge pull request #9653 from zmstone/0101-authz-schema-union-member-selection

0101 authz schema union member selection

apps/emqx/rebar.config

@@ -29,7 +29,7 @@
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}},
     {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
-    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}},
+    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.34.0"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
     {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},
     {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}

apps/emqx/src/emqx_config.erl

@@ -362,8 +362,8 @@ schema_default(Schema) ->
             [];
         ?LAZY(?ARRAY(_)) ->
             [];
-        ?LAZY(?UNION(Unions)) ->
+        ?LAZY(?UNION(Members)) ->
-            case [A || ?ARRAY(A) <- Unions] of
+            case [A || ?ARRAY(A) <- hoconsc:union_members(Members)] of
                 [_ | _] -> [];
                 _ -> #{}
             end;
@@ -402,7 +402,6 @@ merge_envs(SchemaMod, RawConf) ->
         required => false,
         format => map,
         apply_override_envs => true,
-        remove_env_meta => true,
         check_lazy => true
     },
     hocon_tconf:merge_env_overrides(SchemaMod, RawConf, all, Opts).

apps/emqx/test/emqx_schema_tests.erl

@@ -153,7 +153,7 @@ ssl_opts_gc_after_handshake_test_rancher_listener_test() ->
             #{
                 kind := validation_error,
                 reason := unknown_fields,
-                unknown := <<"gc_after_handshake">>
+                unknown := "gc_after_handshake"
             }
         ]},
         validate(Sc, #{<<"gc_after_handshake">> => true})

apps/emqx_authz/src/emqx_authz_schema.erl

@@ -47,14 +47,8 @@
 %% Hocon Schema
 %%--------------------------------------------------------------------
 
-namespace() -> authz.
+type_names() ->
-
+    [
-%% @doc authorization schema is not exported
-%% but directly used by emqx_schema
-roots() -> [].
-
-fields("authorization") ->
-    Types = [
         file,
         http_get,
         http_post,
@@ -67,12 +61,26 @@ fields("authorization") ->
         redis_single,
         redis_sentinel,
         redis_cluster
-    ],
+    ].
-    Unions = [?R_REF(Type) || Type <- Types],
+
+namespace() -> authz.
+
+%% @doc authorization schema is not exported
+%% but directly used by emqx_schema
+roots() -> [].
+
+fields("authorization") ->
+    Types = [?R_REF(Type) || Type <- type_names()],
+    UnionMemberSelector =
+        fun
+            (all_union_members) -> Types;
+            %% must return list
+            ({value, Value}) -> [select_union_member(Value)]
+        end,
     [
         {sources,
             ?HOCON(
-                ?ARRAY(?UNION(Unions)),
+                ?ARRAY(?UNION(UnionMemberSelector)),
                 #{
                     default => [],
                     desc => ?DESC(sources)
@@ -408,9 +416,75 @@ common_rate_field() ->
     ].
 
 method(Method) ->
-    ?HOCON(Method, #{default => Method, required => true, desc => ?DESC(method)}).
+    ?HOCON(Method, #{required => true, desc => ?DESC(method)}).
 
 array(Ref) -> array(Ref, Ref).
 
 array(Ref, DescId) ->
     ?HOCON(?ARRAY(?R_REF(Ref)), #{desc => ?DESC(DescId)}).
+
+select_union_member(#{<<"type">> := <<"mongodb">>} = Value) ->
+    MongoType = maps:get(<<"mongo_type">>, Value, undefined),
+    case MongoType of
+        <<"single">> ->
+            ?R_REF(mongo_single);
+        <<"rs">> ->
+            ?R_REF(mongo_rs);
+        <<"sharded">> ->
+            ?R_REF(mongo_sharded);
+        Else ->
+            throw(#{
+                reason => "unknown_mongo_type",
+                expected => "single | rs | sharded",
+                got => Else
+            })
+    end;
+select_union_member(#{<<"type">> := <<"redis">>} = Value) ->
+    RedisType = maps:get(<<"redis_type">>, Value, undefined),
+    case RedisType of
+        <<"single">> ->
+            ?R_REF(redis_single);
+        <<"cluster">> ->
+            ?R_REF(redis_cluster);
+        <<"sentinel">> ->
+            ?R_REF(redis_sentinel);
+        Else ->
+            throw(#{
+                reason => "unknown_redis_type",
+                expected => "single | cluster | sentinel",
+                got => Else
+            })
+    end;
+select_union_member(#{<<"type">> := <<"http">>} = Value) ->
+    RedisType = maps:get(<<"method">>, Value, undefined),
+    case RedisType of
+        <<"get">> ->
+            ?R_REF(http_get);
+        <<"post">> ->
+            ?R_REF(http_post);
+        Else ->
+            throw(#{
+                reason => "unknown_http_method",
+                expected => "get | post",
+                got => Else
+            })
+    end;
+select_union_member(#{<<"type">> := <<"built_in_database">>}) ->
+    ?R_REF(mnesia);
+select_union_member(#{<<"type">> := Type}) ->
+    select_union_member_loop(Type, type_names());
+select_union_member(_) ->
+    throw("missing_type_field").
+
+select_union_member_loop(TypeValue, []) ->
+    throw(#{
+        reason => "unknown_authz_type",
+        got => TypeValue
+    });
+select_union_member_loop(TypeValue, [Type | Types]) ->
+    case TypeValue =:= atom_to_binary(Type) of
+        true ->
+            ?R_REF(Type);
+        false ->
+            select_union_member_loop(TypeValue, Types)
+    end.

apps/emqx_authz/test/emqx_authz_schema_tests.erl

@@ -0,0 +1,116 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_authz_schema_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+bad_authz_type_test() ->
+    Txt = "[{type: foobar}]",
+    ?assertThrow(
+        [
+            #{
+                reason := "unknown_authz_type",
+                got := <<"foobar">>
+            }
+        ],
+        check(Txt)
+    ).
+
+bad_mongodb_type_test() ->
+    Txt = "[{type: mongodb, mongo_type: foobar}]",
+    ?assertThrow(
+        [
+            #{
+                reason := "unknown_mongo_type",
+                got := <<"foobar">>
+            }
+        ],
+        check(Txt)
+    ).
+
+missing_mongodb_type_test() ->
+    Txt = "[{type: mongodb}]",
+    ?assertThrow(
+        [
+            #{
+                reason := "unknown_mongo_type",
+                got := undefined
+            }
+        ],
+        check(Txt)
+    ).
+
+unknown_redis_type_test() ->
+    Txt = "[{type: redis, redis_type: foobar}]",
+    ?assertThrow(
+        [
+            #{
+                reason := "unknown_redis_type",
+                got := <<"foobar">>
+            }
+        ],
+        check(Txt)
+    ).
+
+missing_redis_type_test() ->
+    Txt = "[{type: redis}]",
+    ?assertThrow(
+        [
+            #{
+                reason := "unknown_redis_type",
+                got := undefined
+            }
+        ],
+        check(Txt)
+    ).
+
+unknown_http_method_test() ->
+    Txt = "[{type: http, method: getx}]",
+    ?assertThrow(
+        [
+            #{
+                reason := "unknown_http_method",
+                got := <<"getx">>
+            }
+        ],
+        check(Txt)
+    ).
+
+missing_http_method_test() ->
+    Txt = "[{type: http, methodx: get}]",
+    ?assertThrow(
+        [
+            #{
+                reason := "unknown_http_method",
+                got := undefined
+            }
+        ],
+        check(Txt)
+    ).
+
+check(Txt0) ->
+    Txt = ["sources: ", Txt0],
+    {ok, RawConf} = hocon:binary(Txt),
+    try
+        hocon_tconf:check_plain(schema(), RawConf, #{})
+    catch
+        throw:{_Schema, Errors} ->
+            throw(Errors)
+    end.
+
+schema() ->
+    #{roots => emqx_authz_schema:fields("authorization")}.

apps/emqx_conf/src/emqx_conf.erl

@@ -316,7 +316,7 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) ->
             {[Schema | Acc], SubRefs ++ RefsAcc}
         end,
         {[], []},
-        Types
+        hoconsc:union_members(Types)
     ),
     {#{<<"oneOf">> => OneOf}, Refs};
 hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->

apps/emqx_connector/i18n/emqx_connector_mongo.conf

@@ -2,8 +2,8 @@ emqx_connector_mongo {
 
     single_mongo_type {
         desc {
-          en: "Standalone instance."
+          en: "Standalone instance. Must be set to 'single' when MongoDB server is running in standalone mode."
-          zh: "Standalone模式。"
+          zh: "Standalone 模式。当 MongoDB 服务运行在 standalone 模式下，该配置必须设置为 'single'。 "
         }
         label: {
               en: "Standalone instance"
@@ -13,8 +13,8 @@ emqx_connector_mongo {
 
     rs_mongo_type {
         desc {
-          en: "Replica set."
+          en: "Replica set. Must be set to 'rs' when MongoDB server is running in 'replica set' mode."
-          zh: "Replica set模式。"
+          zh: "Replica set模式。当 MongoDB 服务运行在 replica-set 模式下，该配置必须设置为 'rs'。"
         }
         label: {
               en: "Replica set"
@@ -24,8 +24,8 @@ emqx_connector_mongo {
 
     sharded_mongo_type {
         desc {
-          en: "Sharded cluster."
+          en: "Sharded cluster. Must be set to 'sharded' when MongoDB server is running in 'sharded' mode."
-          zh: "Sharded cluster模式。"
+          zh: "Sharded cluster模式。当 MongoDB 服务运行在 sharded 模式下，该配置必须设置为 'sharded'。"
         }
         label: {
               en: "Sharded cluster"

apps/emqx_connector/i18n/emqx_connector_redis.conf

@@ -2,8 +2,8 @@ emqx_connector_redis {
 
     single {
         desc {
-          en: "Single mode"
+          en: "Single mode. Must be set to 'single' when Redis server is running in single mode."
-          zh: "单机模式。"
+          zh: "单机模式。当 Redis 服务运行在单机模式下，该配置必须设置为 'single'。"
         }
         label: {
               en: "Single Mode"
@@ -13,8 +13,8 @@ emqx_connector_redis {
 
     cluster {
         desc {
-          en: "Cluster mode"
+          en: "Cluster mode. Must be set to 'cluster' when Redis server is running in clustered mode."
-          zh: "集群模式。"
+          zh: "集群模式。当 Redis 服务运行在集群模式下，该配置必须设置为 'cluster'。"
         }
         label: {
               en: "Cluster Mode"
@@ -24,8 +24,8 @@ emqx_connector_redis {
 
     sentinel {
         desc {
-          en: "Sentinel mode"
+          en: "Sentinel mode. Must be set to 'sentinel' when Redis server is running in sentinel mode."
-          zh: "哨兵模式。"
+          zh: "哨兵模式。当 Redis 服务运行在哨兵模式下，该配置必须设置为 'sentinel'。"
         }
         label: {
               en: "Sentinel Mode"

apps/emqx_connector/src/emqx_connector_mongo.erl

@@ -68,7 +68,6 @@ fields(single) ->
         {mongo_type, #{
             type => single,
             default => single,
-            required => true,
             desc => ?DESC("single_mongo_type")
         }},
         {server, server()},
@@ -79,7 +78,6 @@ fields(rs) ->
         {mongo_type, #{
             type => rs,
             default => rs,
-            required => true,
             desc => ?DESC("rs_mongo_type")
         }},
         {servers, servers()},
@@ -92,7 +90,6 @@ fields(sharded) ->
         {mongo_type, #{
             type => sharded,
             default => sharded,
-            required => true,
             desc => ?DESC("sharded_mongo_type")
         }},
         {servers, servers()},

apps/emqx_connector/src/emqx_connector_redis.erl

@@ -64,7 +64,7 @@ fields(single) ->
         {redis_type, #{
             type => single,
             default => single,
-            required => true,
+            required => false,
             desc => ?DESC("single")
         }}
     ] ++
@@ -76,7 +76,7 @@ fields(cluster) ->
         {redis_type, #{
             type => cluster,
             default => cluster,
-            required => true,
+            required => false,
             desc => ?DESC("cluster")
         }}
     ] ++
@@ -88,7 +88,7 @@ fields(sentinel) ->
         {redis_type, #{
             type => sentinel,
             default => sentinel,
-            required => true,
+            required => false,
             desc => ?DESC("sentinel")
         }},
         {sentinel, #{

apps/emqx_connector/test/emqx_connector_redis_SUITE.erl

@@ -75,9 +75,9 @@ wait_for_redis(Checks) ->
             wait_for_redis(Checks - 1)
     end.
 
-% %%------------------------------------------------------------------------------
+%%------------------------------------------------------------------------------
-% %% Testcases
+%% Testcases
-% %%------------------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
 t_single_lifecycle(_Config) ->
     perform_lifecycle_check(

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -623,7 +623,7 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) ->
             {[Schema | Acc], SubRefs ++ RefsAcc}
         end,
         {[], []},
-        Types
+        hoconsc:union_members(Types)
     ),
     {#{<<"oneOf">> => OneOf}, Refs};
 hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->

build

@@ -135,6 +135,9 @@ assert_no_compile_time_only_deps() {
 
 make_rel() {
     ./scripts/pre-compile.sh "$PROFILE"
+    # make_elixir_rel always create rebar.lock
+    # delete it to make git clone + checkout work because we use shallow close for rebar deps
+    rm -f rebar.lock
     # compile all beams
     ./rebar3 as "$PROFILE" compile
     # generate docs (require beam compiled), generated to etc and priv dirs

changes/refactor-9653.en.md

@@ -0,0 +1 @@
+Make authorization config validation error message more readable.

changes/refactor-9653.zh.md

@@ -0,0 +1 @@
+改进授权配置检查错误日志的可读性。

lib-ee/emqx_ee_bridge/rebar.config

@@ -1,6 +1,5 @@
 {erl_opts, [debug_info]}.
-{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}
+{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.4"}}}
-       , {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.4"}}}
        , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.2"}}}
        , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}}
        , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.7"}}}

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl

@@ -50,19 +50,22 @@ values(Protocol, get) ->
 values("single", post) ->
     SpecificOpts = #{
         server => <<"127.0.0.1:6379">>,
+        redis_type => single,
         database => 1
     },
     values(common, "single", SpecificOpts);
 values("sentinel", post) ->
     SpecificOpts = #{
         servers => [<<"127.0.0.1:26379">>],
+        redis_type => sentinel,
         sentinel => <<"mymaster">>,
         database => 1
     },
     values(common, "sentinel", SpecificOpts);
 values("cluster", post) ->
     SpecificOpts = #{
-        servers => [<<"127.0.0.1:6379">>]
+        servers => [<<"127.0.0.1:6379">>],
+        redis_type => cluster
     },
     values(common, "cluster", SpecificOpts);
 values(Protocol, put) ->

lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl

@@ -17,7 +17,8 @@
 %%------------------------------------------------------------------------------
 
 -define(REDIS_TOXYPROXY_CONNECT_CONFIG, #{
-    <<"server">> => <<"toxiproxy:6379">>
+    <<"server">> => <<"toxiproxy:6379">>,
+    <<"redis_type">> => <<"single">>
 }).
 
 -define(COMMON_REDIS_OPTS, #{
@@ -31,7 +32,7 @@
 -define(PROXY_HOST, "toxiproxy").
 -define(PROXY_PORT, "8474").
 
-all() -> [{group, redis_types}, {group, rest}].
+all() -> [{group, transport_types}, {group, rest}].
 
 groups() ->
     ResourceSpecificTCs = [t_create_delete_bridge],
@@ -47,7 +48,7 @@ groups() ->
     ],
     [
         {rest, TCs},
-        {redis_types, [
+        {transport_types, [
             {group, tcp},
             {group, tls}
         ]},
@@ -63,7 +64,7 @@ groups() ->
 init_per_group(Group, Config) when
     Group =:= redis_single; Group =:= redis_sentinel; Group =:= redis_cluster
 ->
-    [{redis_type, Group} | Config];
+    [{transport_type, Group} | Config];
 init_per_group(Group, Config) when
     Group =:= tcp; Group =:= tls
 ->
@@ -79,6 +80,12 @@ end_per_group(_Group, _Config) ->
     ok.
 
 init_per_suite(Config) ->
+    wait_for_ci_redis(redis_checks(), Config).
+
+wait_for_ci_redis(0, _Config) ->
+    throw(no_redis);
+wait_for_ci_redis(Checks, Config) ->
+    timer:sleep(1000),
     TestHosts = all_test_hosts(),
     case emqx_common_test_helpers:is_all_tcp_servers_available(TestHosts) of
         true ->
@@ -96,15 +103,15 @@ init_per_suite(Config) ->
                 | Config
             ];
         false ->
-            assert_ci()
+            wait_for_ci_redis(Checks - 1, Config)
     end.
 
-assert_ci() ->
+redis_checks() ->
     case os:getenv("IS_CI") of
         "yes" ->
-            throw(no_redis);
+            10;
         _ ->
-            {skip, no_redis}
+            1
     end.
 
 end_per_suite(_Config) ->
@@ -116,7 +123,7 @@ end_per_suite(_Config) ->
 
 init_per_testcase(_Testcase, Config) ->
     ok = delete_all_bridges(),
-    case ?config(redis_type, Config) of
+    case ?config(transport_type, Config) of
         undefined ->
             Config;
         RedisType ->
@@ -139,7 +146,7 @@ end_per_testcase(_Testcase, Config) ->
 
 t_create_delete_bridge(Config) ->
     Name = <<"mybridge">>,
-    Type = ?config(redis_type, Config),
+    Type = ?config(transport_type, Config),
     BridgeConfig = ?config(bridge_config, Config),
     IsBatch = ?config(is_batch, Config),
     ?assertMatch(
@@ -425,31 +432,37 @@ redis_connect_configs() ->
     #{
         redis_single => #{
             tcp => #{
-                <<"server">> => <<"redis:6379">>
+                <<"server">> => <<"redis:6379">>,
+                <<"redis_type">> => <<"single">>
             },
             tls => #{
                 <<"server">> => <<"redis-tls:6380">>,
-                <<"ssl">> => redis_connect_ssl_opts(redis_single)
+                <<"ssl">> => redis_connect_ssl_opts(redis_single),
+                <<"redis_type">> => <<"single">>
             }
         },
         redis_sentinel => #{
             tcp => #{
                 <<"servers">> => <<"redis-sentinel:26379">>,
+                <<"redis_type">> => <<"sentinel">>,
                 <<"sentinel">> => <<"mymaster">>
             },
             tls => #{
                 <<"servers">> => <<"redis-sentinel-tls:26380">>,
+                <<"redis_type">> => <<"sentinel">>,
                 <<"sentinel">> => <<"mymaster">>,
                 <<"ssl">> => redis_connect_ssl_opts(redis_sentinel)
             }
         },
         redis_cluster => #{
             tcp => #{
-                <<"servers">> => <<"redis-cluster:7000,redis-cluster:7001,redis-cluster:7002">>
+                <<"servers">> => <<"redis-cluster:7000,redis-cluster:7001,redis-cluster:7002">>,
+                <<"redis_type">> => <<"cluster">>
             },
             tls => #{
                 <<"servers">> =>
                     <<"redis-cluster-tls:8000,redis-cluster-tls:8001,redis-cluster-tls:8002">>,
+                <<"redis_type">> => <<"cluster">>,
                 <<"ssl">> => redis_connect_ssl_opts(redis_cluster)
             }
         }

mix.exs

@@ -68,7 +68,7 @@ defmodule EMQXUmbrella.MixProject do
       # in conflict by emqtt and hocon
       {:getopt, "1.0.2", override: true},
       {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true},
-      {:hocon, github: "emqx/hocon", tag: "0.33.0", override: true},
+      {:hocon, github: "emqx/hocon", tag: "0.34.0", override: true},
       {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true},
       {:esasl, github: "emqx/esasl", tag: "0.2.0"},
       {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},

rebar.config

@@ -68,7 +68,7 @@
     , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}
     , {getopt, "1.0.2"}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.34.0"}}}
     , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}}
     , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
     , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}

scripts/ct/run.sh

@@ -11,7 +11,6 @@ help() {
     echo
     echo "-h|--help:              To display this usage info"
     echo "--app lib_dir/app_name: For which app to run start docker-compose, and run common tests"
-    echo "--suites SUITE1,SUITE2: Comma separated SUITE names to run. e.g. apps/emqx/test/emqx_SUITE.erl"
     echo "--console:              Start EMQX in console mode but do not run test cases"
     echo "--attach:               Attach to the Erlang docker container without running any test case"
     echo "--stop:                 Stop running containers for the given app"