From b80325f98808f47add468b15e1722d5893b7d507 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 2 Jan 2023 08:47:52 +0100 Subject: [PATCH 1/5] build: delete rebar.lock before making rebar release --- build | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build b/build index 05a3a33d3..87c719a53 100755 --- a/build +++ b/build @@ -135,6 +135,9 @@ assert_no_compile_time_only_deps() { make_rel() { ./scripts/pre-compile.sh "$PROFILE" + # make_elixir_rel always create rebar.lock + # delete it to make git clone + checkout work because we use shallow close for rebar deps + rm -f rebar.lock # compile all beams ./rebar3 as "$PROFILE" compile # generate docs (require beam compiled), generated to etc and priv dirs From e52f9d5920c1ff4f3ed6a85496081c12534ce5cf Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 31 Dec 2022 19:52:38 +0100 Subject: [PATCH 2/5] refactor: use union member type selector for authz sources --- apps/emqx/rebar.config | 2 +- apps/emqx/src/emqx_config.erl | 5 +- apps/emqx/test/emqx_schema_tests.erl | 2 +- apps/emqx_authz/src/emqx_authz.app.src | 2 +- apps/emqx_authz/src/emqx_authz_schema.erl | 98 +++++++++++++-- .../test/emqx_authz_schema_tests.erl | 116 ++++++++++++++++++ apps/emqx_conf/src/emqx_conf.erl | 2 +- .../src/emqx_connector_mongo.erl | 3 - .../src/emqx_connector_redis.erl | 3 - .../src/emqx_dashboard_swagger.erl | 2 +- changes/refactor-9653.en.md | 1 + changes/refactor-9653.zh.md | 1 + lib-ee/emqx_ee_bridge/rebar.config | 3 +- .../src/emqx_ee_bridge_redis.erl | 5 +- .../test/emqx_ee_bridge_mongodb_SUITE.erl | 3 + .../test/emqx_ee_bridge_redis_SUITE.erl | 9 +- mix.exs | 2 +- rebar.config | 2 +- 18 files changed, 229 insertions(+), 32 deletions(-) create mode 100644 apps/emqx_authz/test/emqx_authz_schema_tests.erl create mode 100644 changes/refactor-9653.en.md create mode 100644 changes/refactor-9653.zh.md diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 8ff4fea58..8d363aeed 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.34.0"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 49962e490..2fa39d094 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -362,8 +362,8 @@ schema_default(Schema) -> []; ?LAZY(?ARRAY(_)) -> []; - ?LAZY(?UNION(Unions)) -> - case [A || ?ARRAY(A) <- Unions] of + ?LAZY(?UNION(Members)) -> + case [A || ?ARRAY(A) <- hoconsc:union_members(Members)] of [_ | _] -> []; _ -> #{} end; @@ -402,7 +402,6 @@ merge_envs(SchemaMod, RawConf) -> required => false, format => map, apply_override_envs => true, - remove_env_meta => true, check_lazy => true }, hocon_tconf:merge_env_overrides(SchemaMod, RawConf, all, Opts). diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl index e4fadb192..b249dea92 100644 --- a/apps/emqx/test/emqx_schema_tests.erl +++ b/apps/emqx/test/emqx_schema_tests.erl @@ -153,7 +153,7 @@ ssl_opts_gc_after_handshake_test_rancher_listener_test() -> #{ kind := validation_error, reason := unknown_fields, - unknown := <<"gc_after_handshake">> + unknown := "gc_after_handshake" } ]}, validate(Sc, #{<<"gc_after_handshake">> => true}) diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index c876fbf16..f5b9f9da6 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.10"}, + {vsn, "0.1.11"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index d03747b84..a684ae6ba 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -47,14 +47,8 @@ %% Hocon Schema %%-------------------------------------------------------------------- -namespace() -> authz. - -%% @doc authorization schema is not exported -%% but directly used by emqx_schema -roots() -> []. - -fields("authorization") -> - Types = [ +type_names() -> + [ file, http_get, http_post, @@ -67,12 +61,26 @@ fields("authorization") -> redis_single, redis_sentinel, redis_cluster - ], - Unions = [?R_REF(Type) || Type <- Types], + ]. + +namespace() -> authz. + +%% @doc authorization schema is not exported +%% but directly used by emqx_schema +roots() -> []. + +fields("authorization") -> + Types = [?R_REF(Type) || Type <- type_names()], + UnionMemberSelector = + fun + (all_union_members) -> Types; + %% must return list + ({value, Value}) -> [select_union_member(Value)] + end, [ {sources, ?HOCON( - ?ARRAY(?UNION(Unions)), + ?ARRAY(?UNION(UnionMemberSelector)), #{ default => [], desc => ?DESC(sources) @@ -408,9 +416,75 @@ common_rate_field() -> ]. method(Method) -> - ?HOCON(Method, #{default => Method, required => true, desc => ?DESC(method)}). + ?HOCON(Method, #{required => true, desc => ?DESC(method)}). array(Ref) -> array(Ref, Ref). array(Ref, DescId) -> ?HOCON(?ARRAY(?R_REF(Ref)), #{desc => ?DESC(DescId)}). + +select_union_member(#{<<"type">> := <<"mongodb">>} = Value) -> + MongoType = maps:get(<<"mongo_type">>, Value, undefined), + case MongoType of + <<"single">> -> + ?R_REF(mongo_single); + <<"rs">> -> + ?R_REF(mongo_rs); + <<"sharded">> -> + ?R_REF(mongo_sharded); + Else -> + throw(#{ + reason => "unknown_mongo_type", + expected => "single | rs | sharded", + got => Else + }) + end; +select_union_member(#{<<"type">> := <<"redis">>} = Value) -> + RedisType = maps:get(<<"redis_type">>, Value, undefined), + case RedisType of + <<"single">> -> + ?R_REF(redis_single); + <<"cluster">> -> + ?R_REF(redis_cluster); + <<"sentinel">> -> + ?R_REF(redis_sentinel); + Else -> + throw(#{ + reason => "unknown_redis_type", + expected => "single | cluster | sentinel", + got => Else + }) + end; +select_union_member(#{<<"type">> := <<"http">>} = Value) -> + RedisType = maps:get(<<"method">>, Value, undefined), + case RedisType of + <<"get">> -> + ?R_REF(http_get); + <<"post">> -> + ?R_REF(http_post); + Else -> + throw(#{ + reason => "unknown_http_method", + expected => "get | post", + got => Else + }) + end; +select_union_member(#{<<"type">> := <<"built_in_database">>}) -> + ?R_REF(mnesia); +select_union_member(#{<<"type">> := Type}) -> + select_union_member_loop(Type, type_names()); +select_union_member(_) -> + throw("missing_type_field"). + +select_union_member_loop(TypeValue, []) -> + throw(#{ + reason => "unknown_authz_type", + got => TypeValue + }); +select_union_member_loop(TypeValue, [Type | Types]) -> + case TypeValue =:= atom_to_binary(Type) of + true -> + ?R_REF(Type); + false -> + select_union_member_loop(TypeValue, Types) + end. diff --git a/apps/emqx_authz/test/emqx_authz_schema_tests.erl b/apps/emqx_authz/test/emqx_authz_schema_tests.erl new file mode 100644 index 000000000..f7b2e3c10 --- /dev/null +++ b/apps/emqx_authz/test/emqx_authz_schema_tests.erl @@ -0,0 +1,116 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_schema_tests). + +-include_lib("eunit/include/eunit.hrl"). + +bad_authz_type_test() -> + Txt = "[{type: foobar}]", + ?assertThrow( + [ + #{ + reason := "unknown_authz_type", + got := <<"foobar">> + } + ], + check(Txt) + ). + +bad_mongodb_type_test() -> + Txt = "[{type: mongodb, mongo_type: foobar}]", + ?assertThrow( + [ + #{ + reason := "unknown_mongo_type", + got := <<"foobar">> + } + ], + check(Txt) + ). + +missing_mongodb_type_test() -> + Txt = "[{type: mongodb}]", + ?assertThrow( + [ + #{ + reason := "unknown_mongo_type", + got := undefined + } + ], + check(Txt) + ). + +unknown_redis_type_test() -> + Txt = "[{type: redis, redis_type: foobar}]", + ?assertThrow( + [ + #{ + reason := "unknown_redis_type", + got := <<"foobar">> + } + ], + check(Txt) + ). + +missing_redis_type_test() -> + Txt = "[{type: redis}]", + ?assertThrow( + [ + #{ + reason := "unknown_redis_type", + got := undefined + } + ], + check(Txt) + ). + +unknown_http_method_test() -> + Txt = "[{type: http, method: getx}]", + ?assertThrow( + [ + #{ + reason := "unknown_http_method", + got := <<"getx">> + } + ], + check(Txt) + ). + +missing_http_method_test() -> + Txt = "[{type: http, methodx: get}]", + ?assertThrow( + [ + #{ + reason := "unknown_http_method", + got := undefined + } + ], + check(Txt) + ). + +check(Txt0) -> + Txt = ["sources: ", Txt0], + {ok, RawConf} = hocon:binary(Txt), + try + hocon_tconf:check_plain(schema(), RawConf, #{}) + catch + throw:{_Schema, Errors} -> + throw(Errors) + end. + +schema() -> + #{roots => emqx_authz_schema:fields("authorization")}. diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 6fd9ac009..8b471a137 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -316,7 +316,7 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) -> {[Schema | Acc], SubRefs ++ RefsAcc} end, {[], []}, - Types + hoconsc:union_members(Types) ), {#{<<"oneOf">> => OneOf}, Refs}; hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) -> diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index f9e703d20..a735d8c31 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -67,7 +67,6 @@ fields(single) -> [ {mongo_type, #{ type => single, - default => single, required => true, desc => ?DESC("single_mongo_type") }}, @@ -78,7 +77,6 @@ fields(rs) -> [ {mongo_type, #{ type => rs, - default => rs, required => true, desc => ?DESC("rs_mongo_type") }}, @@ -91,7 +89,6 @@ fields(sharded) -> [ {mongo_type, #{ type => sharded, - default => sharded, required => true, desc => ?DESC("sharded_mongo_type") }}, diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 350d49f01..11d71f1df 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -63,7 +63,6 @@ fields(single) -> {server, server()}, {redis_type, #{ type => single, - default => single, required => true, desc => ?DESC("single") }} @@ -75,7 +74,6 @@ fields(cluster) -> {servers, servers()}, {redis_type, #{ type => cluster, - default => cluster, required => true, desc => ?DESC("cluster") }} @@ -87,7 +85,6 @@ fields(sentinel) -> {servers, servers()}, {redis_type, #{ type => sentinel, - default => sentinel, required => true, desc => ?DESC("sentinel") }}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 4b7a672bd..85b928dda 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -623,7 +623,7 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) -> {[Schema | Acc], SubRefs ++ RefsAcc} end, {[], []}, - Types + hoconsc:union_members(Types) ), {#{<<"oneOf">> => OneOf}, Refs}; hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) -> diff --git a/changes/refactor-9653.en.md b/changes/refactor-9653.en.md new file mode 100644 index 000000000..2807f81d5 --- /dev/null +++ b/changes/refactor-9653.en.md @@ -0,0 +1 @@ +Make authorization config validation error message more readable. diff --git a/changes/refactor-9653.zh.md b/changes/refactor-9653.zh.md new file mode 100644 index 000000000..755fd1683 --- /dev/null +++ b/changes/refactor-9653.zh.md @@ -0,0 +1 @@ +改进授权配置检查错误日志的可读性。 diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index 2bd4036e0..6ca554c72 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -1,6 +1,5 @@ {erl_opts, [debug_info]}. -{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}} - , {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.4"}}} +{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.4"}}} , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.2"}}} , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}} , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.7"}}} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl index 727a6df4b..a0c6ba834 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl @@ -50,19 +50,22 @@ values(Protocol, get) -> values("single", post) -> SpecificOpts = #{ server => <<"127.0.0.1:6379">>, + redis_type => single, database => 1 }, values(common, "single", SpecificOpts); values("sentinel", post) -> SpecificOpts = #{ servers => [<<"127.0.0.1:26379">>], + redis_type => sentinel, sentinel => <<"mymaster">>, database => 1 }, values(common, "sentinel", SpecificOpts); values("cluster", post) -> SpecificOpts = #{ - servers => [<<"127.0.0.1:6379">>] + servers => [<<"127.0.0.1:6379">>], + redis_type => cluster }, values(common, "cluster", SpecificOpts); values(Protocol, put) -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index fb8f1fcc3..05c513eb1 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -151,6 +151,7 @@ mongo_config(MongoHost, MongoPort0, rs = Type) -> " servers = [~p]\n" " w_mode = safe\n" " database = mqtt\n" + " mongo_type = rs\n" "}", [Name, Servers] ), @@ -167,6 +168,7 @@ mongo_config(MongoHost, MongoPort0, sharded = Type) -> " servers = [~p]\n" " w_mode = safe\n" " database = mqtt\n" + " mongo_type = sharded\n" "}", [Name, Servers] ), @@ -183,6 +185,7 @@ mongo_config(MongoHost, MongoPort0, single = Type) -> " server = ~p\n" " w_mode = safe\n" " database = mqtt\n" + " mongo_type = single\n" "}", [Name, Server] ), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl index 1f4b52ddc..fb5f688a6 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl @@ -17,7 +17,8 @@ %%------------------------------------------------------------------------------ -define(REDIS_TOXYPROXY_CONNECT_CONFIG, #{ - <<"server">> => <<"toxiproxy:6379">> + <<"server">> => <<"toxiproxy:6379">>, + <<"redis_type">> => <<"single">> }). -define(COMMON_REDIS_OPTS, #{ @@ -425,19 +426,23 @@ redis_connect_configs() -> #{ redis_single => #{ tcp => #{ + <<"redis_type">> => <<"single">>, <<"server">> => <<"redis:6379">> }, tls => #{ + <<"redis_type">> => <<"single">>, <<"server">> => <<"redis-tls:6380">>, <<"ssl">> => redis_connect_ssl_opts(redis_single) } }, redis_sentinel => #{ tcp => #{ + <<"redis_type">> => <<"sentinel">>, <<"servers">> => <<"redis-sentinel:26379">>, <<"sentinel">> => <<"mymaster">> }, tls => #{ + <<"redis_type">> => <<"sentinel">>, <<"servers">> => <<"redis-sentinel-tls:26380">>, <<"sentinel">> => <<"mymaster">>, <<"ssl">> => redis_connect_ssl_opts(redis_sentinel) @@ -445,9 +450,11 @@ redis_connect_configs() -> }, redis_cluster => #{ tcp => #{ + <<"redis_type">> => <<"cluster">>, <<"servers">> => <<"redis-cluster:7000,redis-cluster:7001,redis-cluster:7002">> }, tls => #{ + <<"redis_type">> => <<"cluster">>, <<"servers">> => <<"redis-cluster-tls:8000,redis-cluster-tls:8001,redis-cluster-tls:8002">>, <<"ssl">> => redis_connect_ssl_opts(redis_cluster) diff --git a/mix.exs b/mix.exs index 265448d72..b7fc7dac8 100644 --- a/mix.exs +++ b/mix.exs @@ -68,7 +68,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.33.0", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.34.0", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 4476a128c..878c911e5 100644 --- a/rebar.config +++ b/rebar.config @@ -68,7 +68,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.33.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.34.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From ffb09f0c4dca410e074e805ef6dfa905ced4dbd2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 3 Jan 2023 10:36:39 +0100 Subject: [PATCH 3/5] test: rename a test option name to avoid clashing with prod config name --- .../test/emqx_ee_bridge_redis_SUITE.erl | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl index fb5f688a6..05f066d1a 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl @@ -32,7 +32,7 @@ -define(PROXY_HOST, "toxiproxy"). -define(PROXY_PORT, "8474"). -all() -> [{group, redis_types}, {group, rest}]. +all() -> [{group, transport_types}, {group, rest}]. groups() -> ResourceSpecificTCs = [t_create_delete_bridge], @@ -48,7 +48,7 @@ groups() -> ], [ {rest, TCs}, - {redis_types, [ + {transport_types, [ {group, tcp}, {group, tls} ]}, @@ -64,7 +64,7 @@ groups() -> init_per_group(Group, Config) when Group =:= redis_single; Group =:= redis_sentinel; Group =:= redis_cluster -> - [{redis_type, Group} | Config]; + [{transport_type, Group} | Config]; init_per_group(Group, Config) when Group =:= tcp; Group =:= tls -> @@ -117,7 +117,7 @@ end_per_suite(_Config) -> init_per_testcase(_Testcase, Config) -> ok = delete_all_bridges(), - case ?config(redis_type, Config) of + case ?config(transport_type, Config) of undefined -> Config; RedisType -> @@ -140,7 +140,7 @@ end_per_testcase(_Testcase, Config) -> t_create_delete_bridge(Config) -> Name = <<"mybridge">>, - Type = ?config(redis_type, Config), + Type = ?config(transport_type, Config), BridgeConfig = ?config(bridge_config, Config), IsBatch = ?config(is_batch, Config), ?assertMatch( @@ -426,37 +426,37 @@ redis_connect_configs() -> #{ redis_single => #{ tcp => #{ - <<"redis_type">> => <<"single">>, - <<"server">> => <<"redis:6379">> + <<"server">> => <<"redis:6379">>, + <<"redis_type">> => <<"single">> }, tls => #{ - <<"redis_type">> => <<"single">>, <<"server">> => <<"redis-tls:6380">>, - <<"ssl">> => redis_connect_ssl_opts(redis_single) + <<"ssl">> => redis_connect_ssl_opts(redis_single), + <<"redis_type">> => <<"single">> } }, redis_sentinel => #{ tcp => #{ - <<"redis_type">> => <<"sentinel">>, <<"servers">> => <<"redis-sentinel:26379">>, + <<"redis_type">> => <<"sentinel">>, <<"sentinel">> => <<"mymaster">> }, tls => #{ - <<"redis_type">> => <<"sentinel">>, <<"servers">> => <<"redis-sentinel-tls:26380">>, + <<"redis_type">> => <<"sentinel">>, <<"sentinel">> => <<"mymaster">>, <<"ssl">> => redis_connect_ssl_opts(redis_sentinel) } }, redis_cluster => #{ tcp => #{ - <<"redis_type">> => <<"cluster">>, - <<"servers">> => <<"redis-cluster:7000,redis-cluster:7001,redis-cluster:7002">> + <<"servers">> => <<"redis-cluster:7000,redis-cluster:7001,redis-cluster:7002">>, + <<"redis_type">> => <<"cluster">> }, tls => #{ - <<"redis_type">> => <<"cluster">>, <<"servers">> => <<"redis-cluster-tls:8000,redis-cluster-tls:8001,redis-cluster-tls:8002">>, + <<"redis_type">> => <<"cluster">>, <<"ssl">> => redis_connect_ssl_opts(redis_cluster) } } From 0697c692ed2992920ed9b9988aa85d955af8defb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 8 Jan 2023 09:51:26 +0100 Subject: [PATCH 4/5] refactor: mongo_type and redis_type are not mandatory the connector schemas are shared between authn, auth and bridges, the difference is that authn and authz configs are unions like: [ {mongo_type = rs, ... } {another backend config} ] however the brdige types are maps like, for example: mongodb_rs.$name { mongo_type = rs ... } in which case, the mongo_type is not required. in order to keep the schema static as much as possible, this field is chanegd to 'required => false' with a default value. However, for authn and authz, the union selector will still raise exception if the there is no type provided. --- .../i18n/emqx_connector_mongo.conf | 18 +++++++++--------- .../i18n/emqx_connector_redis.conf | 12 ++++++------ .../src/emqx_connector_mongo.erl | 6 +++--- .../src/emqx_connector_redis.erl | 9 ++++++--- .../test/emqx_ee_bridge_mongodb_SUITE.erl | 3 --- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/apps/emqx_connector/i18n/emqx_connector_mongo.conf b/apps/emqx_connector/i18n/emqx_connector_mongo.conf index a598c084d..619a8e3b4 100644 --- a/apps/emqx_connector/i18n/emqx_connector_mongo.conf +++ b/apps/emqx_connector/i18n/emqx_connector_mongo.conf @@ -2,34 +2,34 @@ emqx_connector_mongo { single_mongo_type { desc { - en: "Standalone instance." - zh: "Standalone模式。" + en: "Standalone instance. Must be set to 'single' when MongoDB server is running in standalone mode." + zh: "Standalone 模式。当 MongoDB 服务运行在 standalone 模式下，该配置必须设置为 'single'。 " } label: { en: "Standalone instance" - zh: "Standalone模式" + zh: "Standalone 模式" } } rs_mongo_type { desc { - en: "Replica set." - zh: "Replica set模式。" + en: "Replica set. Must be set to 'rs' when MongoDB server is running in 'replica set' mode." + zh: "Replica set模式。当 MongoDB 服务运行在 replica-set 模式下，该配置必须设置为 'rs'。" } label: { en: "Replica set" - zh: "Replica set模式" + zh: "Replica set 模式" } } sharded_mongo_type { desc { - en: "Sharded cluster." - zh: "Sharded cluster模式。" + en: "Sharded cluster. Must be set to 'sharded' when MongoDB server is running in 'sharded' mode." + zh: "Sharded cluster模式。当 MongoDB 服务运行在 sharded 模式下，该配置必须设置为 'sharded'。" } label: { en: "Sharded cluster" - zh: "Sharded cluster模式" + zh: "Sharded cluster 模式" } } diff --git a/apps/emqx_connector/i18n/emqx_connector_redis.conf b/apps/emqx_connector/i18n/emqx_connector_redis.conf index 228d0805a..f42f38f30 100644 --- a/apps/emqx_connector/i18n/emqx_connector_redis.conf +++ b/apps/emqx_connector/i18n/emqx_connector_redis.conf @@ -2,8 +2,8 @@ emqx_connector_redis { single { desc { - en: "Single mode" - zh: "单机模式。" + en: "Single mode. Must be set to 'single' when Redis server is running in single mode." + zh: "单机模式。当 Redis 服务运行在单机模式下，该配置必须设置为 'single'。" } label: { en: "Single Mode" @@ -13,8 +13,8 @@ emqx_connector_redis { cluster { desc { - en: "Cluster mode" - zh: "集群模式。" + en: "Cluster mode. Must be set to 'cluster' when Redis server is running in clustered mode." + zh: "集群模式。当 Redis 服务运行在集群模式下，该配置必须设置为 'cluster'。" } label: { en: "Cluster Mode" @@ -24,8 +24,8 @@ emqx_connector_redis { sentinel { desc { - en: "Sentinel mode" - zh: "哨兵模式。" + en: "Sentinel mode. Must be set to 'sentinel' when Redis server is running in sentinel mode." + zh: "哨兵模式。当 Redis 服务运行在哨兵模式下，该配置必须设置为 'sentinel'。" } label: { en: "Sentinel Mode" diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index a735d8c31..0bcc39208 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -67,7 +67,7 @@ fields(single) -> [ {mongo_type, #{ type => single, - required => true, + default => single, desc => ?DESC("single_mongo_type") }}, {server, server()}, @@ -77,7 +77,7 @@ fields(rs) -> [ {mongo_type, #{ type => rs, - required => true, + default => rs, desc => ?DESC("rs_mongo_type") }}, {servers, servers()}, @@ -89,7 +89,7 @@ fields(sharded) -> [ {mongo_type, #{ type => sharded, - required => true, + default => sharded, desc => ?DESC("sharded_mongo_type") }}, {servers, servers()}, diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 11d71f1df..726af2d9b 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -63,7 +63,8 @@ fields(single) -> {server, server()}, {redis_type, #{ type => single, - required => true, + default => single, + required => false, desc => ?DESC("single") }} ] ++ @@ -74,7 +75,8 @@ fields(cluster) -> {servers, servers()}, {redis_type, #{ type => cluster, - required => true, + default => cluster, + required => false, desc => ?DESC("cluster") }} ] ++ @@ -85,7 +87,8 @@ fields(sentinel) -> {servers, servers()}, {redis_type, #{ type => sentinel, - required => true, + default => sentinel, + required => false, desc => ?DESC("sentinel") }}, {sentinel, #{ diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index 05c513eb1..fb8f1fcc3 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -151,7 +151,6 @@ mongo_config(MongoHost, MongoPort0, rs = Type) -> " servers = [~p]\n" " w_mode = safe\n" " database = mqtt\n" - " mongo_type = rs\n" "}", [Name, Servers] ), @@ -168,7 +167,6 @@ mongo_config(MongoHost, MongoPort0, sharded = Type) -> " servers = [~p]\n" " w_mode = safe\n" " database = mqtt\n" - " mongo_type = sharded\n" "}", [Name, Servers] ), @@ -185,7 +183,6 @@ mongo_config(MongoHost, MongoPort0, single = Type) -> " server = ~p\n" " w_mode = safe\n" " database = mqtt\n" - " mongo_type = single\n" "}", [Name, Server] ), From c3635f537a7dbc7da45248646caa45f7ca4206af Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 2 Jan 2023 20:37:38 +0100 Subject: [PATCH 5/5] ci: wait for redis in emqx_ee_bridge_redis_SUITE --- .../test/emqx_connector_redis_SUITE.erl | 6 +++--- .../test/emqx_ee_bridge_redis_SUITE.erl | 14 ++++++++++---- scripts/ct/run.sh | 1 - 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl index 060340fed..a1d8fe9d5 100644 --- a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl @@ -75,9 +75,9 @@ wait_for_redis(Checks) -> wait_for_redis(Checks - 1) end. -% %%------------------------------------------------------------------------------ -% %% Testcases -% %%------------------------------------------------------------------------------ +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ t_single_lifecycle(_Config) -> perform_lifecycle_check( diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl index 05f066d1a..c3529ddeb 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl @@ -80,6 +80,12 @@ end_per_group(_Group, _Config) -> ok. init_per_suite(Config) -> + wait_for_ci_redis(redis_checks(), Config). + +wait_for_ci_redis(0, _Config) -> + throw(no_redis); +wait_for_ci_redis(Checks, Config) -> + timer:sleep(1000), TestHosts = all_test_hosts(), case emqx_common_test_helpers:is_all_tcp_servers_available(TestHosts) of true -> @@ -97,15 +103,15 @@ init_per_suite(Config) -> | Config ]; false -> - assert_ci() + wait_for_ci_redis(Checks - 1, Config) end. -assert_ci() -> +redis_checks() -> case os:getenv("IS_CI") of "yes" -> - throw(no_redis); + 10; _ -> - {skip, no_redis} + 1 end. end_per_suite(_Config) -> diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 7fb9e00fd..3a7b40317 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -11,7 +11,6 @@ help() { echo echo "-h|--help: To display this usage info" echo "--app lib_dir/app_name: For which app to run start docker-compose, and run common tests" - echo "--suites SUITE1,SUITE2: Comma separated SUITE names to run. e.g. apps/emqx/test/emqx_SUITE.erl" echo "--console: Start EMQX in console mode but do not run test cases" echo "--attach: Attach to the Erlang docker container without running any test case" echo "--stop: Stop running containers for the given app"