From 32b23b9181c27f6046956e77562160c29a5f48b7 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 14 Jan 2021 16:12:19 +0800 Subject: [PATCH 01/24] Revert "feat(auth_http): support for getting websocket cookies" This reverts commit 5427057c2c0eb25fb826005e6cb1abe3fea5e20b. --- apps/emqx_auth_http/etc/emqx_auth_http.conf | 3 --- apps/emqx_auth_http/src/emqx_auth_http_cli.erl | 1 - apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl | 7 ------- src/emqx_types.erl | 3 +-- 4 files changed, 1 insertion(+), 13 deletions(-) diff --git a/apps/emqx_auth_http/etc/emqx_auth_http.conf b/apps/emqx_auth_http/etc/emqx_auth_http.conf index 56f76cf39..0df589169 100644 --- a/apps/emqx_auth_http/etc/emqx_auth_http.conf +++ b/apps/emqx_auth_http/etc/emqx_auth_http.conf @@ -28,7 +28,6 @@ auth.http.auth_req.content_type = x-www-form-urlencoded ## - %p: sockport of server accepted ## - %C: common name of client TLS cert ## - %d: subject of client TLS cert -## - %k: websocket cookie ## ## Value: Params auth.http.auth_req.params = clientid=%c,username=%u,password=%P @@ -59,7 +58,6 @@ auth.http.auth_req.params = clientid=%c,username=%u,password=%P ## - %p: sockport of server accepted ## - %C: common name of client TLS cert ## - %d: subject of client TLS cert -## - %k: websocket cookie ## ## Value: Params #auth.http.super_req.params = clientid=%c,username=%u @@ -89,7 +87,6 @@ auth.http.acl_req.content_type = x-www-form-urlencoded ## - %r: protocol ## - %m: mountpoint ## - %t: topic -## - %k: websocket cookie ## ## Value: Params auth.http.acl_req.params = access=%A,username=%u,clientid=%c,ipaddr=%a,topic=%t,mountpoint=%m diff --git a/apps/emqx_auth_http/src/emqx_auth_http_cli.erl b/apps/emqx_auth_http/src/emqx_auth_http_cli.erl index 32edd6190..c56e45d49 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_cli.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_cli.erl @@ -80,7 +80,6 @@ feedvar(Params, ClientInfo = #{clientid := ClientId, ({Param, "%A"}) -> {Param, maps:get(access, ClientInfo, null)}; ({Param, "%t"}) -> {Param, maps:get(topic, ClientInfo, null)}; ({Param, "%m"}) -> {Param, maps:get(mountpoint, ClientInfo, null)}; - ({Param, "%k"}) -> {Param, emqx_json:encode(maps:get(ws_cookie, ClientInfo, null))}; ({Param, Var}) -> {Param, Var} end, Params). diff --git a/apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl b/apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl index fc716f5ae..f9acdc638 100644 --- a/apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl +++ b/apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl @@ -171,10 +171,3 @@ t_comment_config(_) -> ?assertEqual(AuthCount - 1, length(emqx_hooks:lookup('client.authenticate'))), ?assertEqual(AclCount - 1, length(emqx_hooks:lookup('client.check_acl'))). -t_feedvar(_) -> - Params = [{"cookie", "%k"}], - User0 = ?USER(<<"client1">>, <<"testuser">>, mqtt, {127,0,0,1}, external), - ?assertEqual([{"cookie", <<"null">>}], emqx_auth_http_cli:feedvar(Params, User0)), - - User1 = User0#{ws_cookie => [{<<"k">>, <<"v">>}]}, - ?assertEqual([{"cookie", <<"{\"k\":\"v\"}">>}], emqx_auth_http_cli:feedvar(Params, User1)). diff --git a/src/emqx_types.erl b/src/emqx_types.erl index c54832b85..b92d457b8 100644 --- a/src/emqx_types.erl +++ b/src/emqx_types.erl @@ -136,7 +136,7 @@ is_bridge := boolean(), is_superuser := boolean(), mountpoint := maybe(binary()), - ws_cookie => wscookie(), + ws_cookie => maybe(list()), password => maybe(binary()), auth_result => auth_result(), anonymous => boolean(), @@ -150,7 +150,6 @@ -type(peerhost() :: inet:ip_address()). -type(peername() :: {inet:ip_address(), inet:port_number()} | inet:returned_non_ip_address()). --type(wscookie() :: [{binary(), binary()}]). -type(protocol() :: mqtt | 'mqtt-sn' | coap | lwm2m | stomp | none | atom()). -type(auth_result() :: success | client_identifier_not_valid From 67bfb4f10ab88a42be6b835516e718a890bc378e Mon Sep 17 00:00:00 2001 From: bignullnull <751957846@qq.com> Date: Fri, 15 Jan 2021 09:12:24 +0800 Subject: [PATCH 02/24] fix(rule_engine): resource update problem and add cli (#3991) --- .../emqx_rule_engine/src/emqx_rule_engine.erl | 72 ++++++++++++++----- .../src/emqx_rule_engine_api.erl | 54 ++++++++------ .../src/emqx_rule_engine_cli.erl | 35 ++++++++- .../test/emqx_rule_engine_SUITE.erl | 42 +++++++++-- 4 files changed, 156 insertions(+), 47 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index f453bb73c..26f6ef738 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -39,10 +39,11 @@ , get_resource_status/1 , get_resource_params/1 , delete_resource/1 - , update_resource/1 + , update_resource/2 ]). -export([ init_resource/4 + , init_resource/5 , init_action/4 , clear_resource/3 , clear_rule/1 @@ -244,27 +245,60 @@ create_resource(#{type := Type, config := Config0} = Params) -> {error, {resource_type_not_found, Type}} end. -update_resource(#{id := Id, type := Type, config := NewConfig, - description := Description} = NewResource) -> +-spec(update_resource(resource_id(), map()) -> ok | {error, Reason :: term()}). +update_resource(ResId, NewParams) -> + try + lists:foreach(fun(#rule{id = RuleId, enabled = Enabled, actions = Actions}) -> + lists:foreach( + fun (#action_instance{args = #{<<"$resource">> := ResId1}}) + when ResId =:= ResId1, Enabled == true -> + throw({dependency_exists, RuleId}); + (_) -> ok + end, Actions) + end, ets:tab2list(?RULE_TAB)), + do_update_resource_check(ResId, NewParams) + catch _ : Reason -> + {error, Reason} + end. + +do_update_resource_check(Id, NewParams) -> + case emqx_rule_registry:find_resource(Id) of + {ok, #resource{id = Id, + type = Type, + config = OldConfig, + description = OldDescription} = _OldResource} -> + try + do_update_resource(#{id => Id, + config => case maps:find(<<"config">>, NewParams) of + {ok, NewConfig} -> NewConfig; + error -> OldConfig + end, + type => Type, + description => case maps:find(<<"description">>, NewParams) of + {ok, NewDescription} -> NewDescription; + error -> OldDescription + end}), + ok + catch _ : Reason -> + {error, Reason} + end; + _Other -> + {error, not_found} + end. + +do_update_resource(#{id := Id, type := Type, description:= NewDescription, config:= NewConfig}) -> case emqx_rule_registry:find_resource_type(Type) of {ok, #resource_type{on_create = {Module, Create}, + on_destroy = {Module, Destroy}, params_spec = ParamSpec}} -> Config = emqx_rule_validator:validate_params(NewConfig, ParamSpec), - case delete_resource(Id) of - {error, not_found} -> {error, not_found}; - _ -> %% deletion might fail because of an associted rule. - emqx_rule_registry:add_resource( - #resource{ - id = Id, - config = Config, - type = Type, - description = Description, - created_at = erlang:system_time(millisecond)}), - catch cluster_call(init_resource, [Module, Create, Id, Config, true]), - {ok, NewResource} - end; - not_found -> - {error, {resource_type_not_found, Type}} + cluster_call(init_resource, [Module, Create, Id, Config]), + emqx_rule_registry:add_resource(#resource{id = Id, + type = Type, + config = Config, + description = NewDescription, + created_at = erlang:system_time(millisecond)}), + cluster_call(clear_resource, [Module, Destroy, Id]) end. -spec(start_resource(resource_id()) -> ok | {error, Reason :: term()}). @@ -499,7 +533,7 @@ init_resource(Module, OnCreate, ResId, Config, Restart) -> Params = ?RAISE( Module:OnCreate(ResId, Config), Restart andalso - timer:apply_after(timer:seconds(60), ?MODULE, do_init_resource, + timer:apply_after(timer:seconds(60), ?MODULE, init_resource, [Module, OnCreate, ResId, Config, Restart]), {{Module, OnCreate}, {_EXCLASS_, _EXCPTION_, _ST_}}), ResParams = #resource_params{id = ResId, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 03f34fa5b..987157d63 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -327,27 +327,34 @@ start_resource(#{id := Id}, _Params) -> return({error, 400, ?ERR_BADARGS(Reason)}) end. -update_resource(#{id := Id}, Params) -> - case parse_resource_params(Params) of - {ok, ParsedParams} -> - case emqx_rule_registry:find_resource(Id) of - {ok, #resource{id = Id, type = Type} = _OldResource} -> - Config = maps:get(config, ParsedParams), - Description = maps:get(description, ParsedParams), - _ = emqx_rule_engine:update_resource( - #{id => Id, - config => Config, - type => Type, - description => Description, - created_at => erlang:system_time(millisecond)}), - return(ok); - _Other -> - return({error, 400, ?ERR_NO_RESOURCE(Id)}) - end; +update_resource(#{id := Id}, NewParams) -> + P1 = case proplists:get_value(<<"description">>, NewParams) of + undefined -> #{}; + Value -> #{<<"description">> => Value} + end, + P2 = case proplists:get_value(<<"config">>, NewParams) of + undefined -> #{}; + <<"{}">> -> #{}; + Map -> #{<<"config">> => ?RAISE(maps:from_list(Map), {invalid_config, Map})} + end, + case emqx_rule_engine:update_resource(Id, maps:merge(P1, P2)) of + ok -> + return(ok); + {error, not_found} -> + ?LOG(error, "resource not found: ~0p", [Id]), + return({error, 400, list_to_binary("resource not found:" ++ binary_to_list(Id))}); + {error, {init_resource_failure, _}} -> + ?LOG(error, "init resource failure: ~0p", [Id]), + return({error, 500, list_to_binary("init resource failure:" ++ binary_to_list(Id))}); + {error, {dependency_exists, RuleId}} -> + ?LOG(error, "dependency exists: ~0p", [RuleId]), + return({error, 500, list_to_binary("resource dependency by rule:" ++ binary_to_list(RuleId))}); {error, Reason} -> - return({error, 400, ?ERR_BADARGS(Reason)}) + ?LOG(error, "update resource failed: ~0p", [Reason]), + return({error, 500, <<"update resource failed,error info have been written to logfile!">>}) end. + delete_resource(#{id := Id}, _Params) -> case emqx_rule_engine:delete_resource(Id) of ok -> return(ok); @@ -524,7 +531,14 @@ parse_resource_params([_ | Params], Res) -> parse_resource_params(Params, Res). json_term_to_map(List) -> - emqx_json:decode(emqx_json:encode(List), [return_maps]). + Data = lists:map(fun({K, V}) -> + case V of + {} ->{K, [{}]}; + _ -> {K, V} + end + end, + List), + emqx_json:decode(emqx_json:encode(Data), [return_maps]). sort_by_title(action, Actions) -> sort_by(#action.title, Actions); @@ -544,4 +558,4 @@ get_rule_metrics(Id) -> get_action_metrics(Id) -> [maps:put(node, Node, rpc:call(Node, emqx_rule_metrics, get_action_metrics, [Id])) - || Node <- ekka_mnesia:running_nodes()]. + || Node <- ekka_mnesia:running_nodes()]. \ No newline at end of file diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl index f165b95b1..8f950ab43 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl @@ -44,6 +44,12 @@ , {descr, $d, "descr", {binary, <<"">>}, "Description"} ]). +-define(OPTSPEC_RESOURCES_UPDATE, + [ {id, undefined, undefined, binary, "The resource id"} + , {config, $c, "config", {binary, undefined}, "Config"} + , {description, $d, "descr", {binary, undefined}, "Description"} + ]). + -define(OPTSPEC_RULES_CREATE, [ {sql, undefined, undefined, binary, "Filter Condition SQL"} , {actions, undefined, undefined, binary, "Action List in JSON format: [{\"name\": , \"params\": {: }}]"} @@ -61,7 +67,6 @@ , {on_action_failed, $g, "on_action_failed", {atom, undefined}, "'continue' or 'stop' when an action in the rule fails"} , {descr, $d, "descr", {binary, undefined}, "Description"} ]). - %%----------------------------------------------------------------------------- %% Load/Unload Commands %%----------------------------------------------------------------------------- @@ -148,6 +153,7 @@ actions(_Usage) -> %%------------------------------------------------------------------------------ %% 'resources' command %%------------------------------------------------------------------------------ + resources(["create" | Params]) -> with_opts(fun({Opts, _}) -> case emqx_rule_engine:create_resource(make_resource(Opts)) of @@ -158,6 +164,19 @@ resources(["create" | Params]) -> end end, Params, ?OPTSPEC_RESOURCES_CREATE, {?FUNCTION_NAME, create}); + +resources(["update" | Params]) -> + with_opts(fun({Opts, _}) -> + Id = maps:get(id, maps:from_list(Opts)), + Maps = make_updated_resource(Opts), + case emqx_rule_engine:update_resource(Id, Maps) of + ok -> + emqx_ctl:print("Resource update successfully~n"); + {error, Reason} -> + emqx_ctl:print("update resource failed, reason: ~p!~n", [Reason]) + end + end, Params, ?OPTSPEC_RESOURCES_UPDATE, {?FUNCTION_NAME, update}); + resources(["test" | Params]) -> with_opts(fun({Opts, _}) -> case emqx_rule_engine:test_resource(make_resource(Opts)) of @@ -192,7 +211,8 @@ resources(_Usage) -> emqx_ctl:usage([{"resources create", "Create a resource"}, {"resources list [-t ]", "List resources"}, {"resources show ", "Show a resource"}, - {"resources delete ", "Delete a resource"} + {"resources delete ", "Delete a resource"}, + {"resources update [-c ] [-d ]", "Update a resource"} ]). %%------------------------------------------------------------------------------ @@ -302,6 +322,17 @@ make_resource(Opts) -> config => ?RAISE(emqx_json:decode(Config, [return_maps]), {invalid_config, Config}), description => get_value(descr, Opts)}, id, <<"">>, Opts). +make_updated_resource(Opts) -> + P1 = case proplists:get_value(description, Opts) of + undefined -> #{}; + Value -> #{<<"description">> => Value} + end, + P2 = case proplists:get_value(config, Opts) of + undefined -> #{}; + Map -> #{<<"config">> => ?RAISE((emqx_json:decode(Map, [return_maps])), {invalid_config, Map})} + end, + maps:merge(P1, P2). + printable_actions(Actions) when is_list(Actions) -> emqx_json:encode([#{id => Id, name => Name, params => Args, metrics => get_action_metrics(Id), diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index a857ad60a..fe4c78fb1 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -260,6 +260,7 @@ init_per_testcase(_TestCase, Config) -> %ct:pal("============ ~p", [ets:tab2list(emqx_resource_type)]), Config. + end_per_testcase(t_events, Config) -> ets:delete(events_record_tab), ok = emqx_rule_registry:remove_rule(?config(hook_points_rules, Config)), @@ -443,24 +444,53 @@ t_crud_resources_api(_Config) -> ResId = maps:get(id, Resources1), {ok, #{code := 0, data := Resources}} = emqx_rule_engine_api:list_resources(#{},[]), ?assert(length(Resources) > 0), - {ok, #{code := 0, data := Resources2}} = emqx_rule_engine_api:show_resource(#{id => ResId},[]), ?assertEqual(ResId, maps:get(id, Resources2)), - + % {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId}, - [{<<"id">>, ResId}, - {<<"type">>, <<"built_in">>}, - {<<"config">>, [{<<"a">>, 2}]}, + [{<<"config">>, [{<<"a">>, 2}]}, {<<"description">>, <<"2">>}]), {ok, #{code := 0, data := Resources3}} = emqx_rule_engine_api:show_resource(#{id => ResId},[]), ?assertEqual(ResId, maps:get(id, Resources3)), ?assertEqual(#{<<"a">> => 2}, maps:get(config, Resources3)), ?assertEqual(<<"2">>, maps:get(description, Resources3)), - + {ok, #{code := 0, data := Resources3}} = emqx_rule_engine_api:show_resource(#{id => ResId},[]), + ?assertEqual(ResId, maps:get(id, Resources3)), + % + {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId}, + [{<<"config">>, [{<<"a">>, 3}]}]), + {ok, #{code := 0, data := Resources4}} = emqx_rule_engine_api:show_resource(#{id => ResId},[]), + ?assertEqual(ResId, maps:get(id, Resources4)), + ?assertEqual(#{<<"a">> => 3}, maps:get(config, Resources4)), + ?assertEqual(<<"2">>, maps:get(description, Resources4)), + % Only config + {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId}, + [{<<"config">>, [{<<"a">>, 1}, + {<<"b">>, 2}, + {<<"c">>, 3}]}]), + {ok, #{code := 0, data := Resources5}} = emqx_rule_engine_api:show_resource(#{id => ResId},[]), + ?assertEqual(ResId, maps:get(id, Resources5)), + ?assertEqual(#{<<"a">> => 1, <<"b">> => 2, <<"c">> => 3}, maps:get(config, Resources5)), + ?assertEqual(<<"2">>, maps:get(description, Resources5)), + % Only description + {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId}, + [{<<"description">>, <<"new5">>}]), + {ok, #{code := 0, data := Resources6}} = emqx_rule_engine_api:show_resource(#{id => ResId},[]), + ?assertEqual(ResId, maps:get(id, Resources6)), + ?assertEqual(#{<<"a">> => 1, <<"b">> => 2, <<"c">> => 3}, maps:get(config, Resources6)), + ?assertEqual(<<"new5">>, maps:get(description, Resources6)), + % None + {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId},[]), + {ok, #{code := 0, data := Resources7}} = emqx_rule_engine_api:show_resource(#{id => ResId},[]), + ?assertEqual(ResId, maps:get(id, Resources7)), + ?assertEqual(#{<<"a">> => 1, <<"b">> => 2, <<"c">> => 3}, maps:get(config, Resources7)), + ?assertEqual(<<"new5">>, maps:get(description, Resources7)), + % ?assertMatch({ok, #{code := 0}}, emqx_rule_engine_api:delete_resource(#{id => ResId},#{})), ?assertMatch({ok, #{code := 404}}, emqx_rule_engine_api:show_resource(#{id => ResId},[])), ok. + t_list_resource_types_api(_Config) -> {ok, #{code := 0, data := ResourceTypes}} = emqx_rule_engine_api:list_resource_types(#{},[]), ?assert(length(ResourceTypes) > 0), From 42f95bba0914f577a6fd0b79b74e5d27f62f7e94 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Wed, 13 Jan 2021 09:18:16 +0800 Subject: [PATCH 03/24] fix(alarms): add duration field for alarms --- apps/emqx_management/src/emqx_mgmt.erl | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index e2620bb9d..1458da73a 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -568,7 +568,7 @@ get_alarms(Type) -> [{Node, get_alarms(Node, Type)} || Node <- ekka_mnesia:running_nodes()]. get_alarms(Node, Type) when Node =:= node() -> - emqx_alarm:get_alarms(Type); + add_duration_field(emqx_alarm:get_alarms(Type)); get_alarms(Node, Type) -> rpc_call(Node, get_alarms, [Node, Type]). @@ -585,6 +585,17 @@ delete_all_deactivated_alarms(Node) when Node =:= node() -> delete_all_deactivated_alarms(Node) -> rpc_call(Node, delete_deactivated_alarms, [Node]). +add_duration_field(Alarms) -> + Now = erlang:system_time(microsecond), + add_duration_field(Alarms, Now, []). + +add_duration_field([], _Now, Acc) -> + Acc; +add_duration_field([Alarm = #{activated := true, activate_at := ActivateAt}| Rest], Now, Acc) -> + add_duration_field(Rest, Now, [Alarm#{duration => Now - ActivateAt} | Acc]); +add_duration_field([Alarm = #{activated := false, activate_at := ActivateAt, deactivate_at := DeactivateAt}| Rest], Now, Acc) -> + add_duration_field(Rest, Now, [Alarm#{duration => DeactivateAt - ActivateAt} | Acc]). + %%-------------------------------------------------------------------- %% Banned API %%-------------------------------------------------------------------- From d360e7ead1a45bc4e0a1eb2294a7173b781a04f6 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 11 Jan 2021 15:34:56 +0800 Subject: [PATCH 04/24] feat(exproto): use client streaming APIs for handler - Use the gRPC client streaming APIs to improve the ConnectionHandler server performance. - Change the 'conn' field type to term binary --- apps/emqx_exhook/rebar.config | 4 +- apps/emqx_exproto/priv/protos/exproto.proto | 10 +- apps/emqx_exproto/rebar.config | 4 +- .../emqx_exproto/src/emqx_exproto_channel.erl | 9 +- apps/emqx_exproto/src/emqx_exproto_gcli.erl | 62 +++++--- apps/emqx_exproto/src/emqx_exproto_gsvr.erl | 4 +- apps/emqx_exproto/test/emqx_exproto_SUITE.erl | 1 - .../test/emqx_exproto_echo_svr.erl | 133 +++++++++++------- 8 files changed, 137 insertions(+), 90 deletions(-) diff --git a/apps/emqx_exhook/rebar.config b/apps/emqx_exhook/rebar.config index ebeaddeab..d2e437b8b 100644 --- a/apps/emqx_exhook/rebar.config +++ b/apps/emqx_exhook/rebar.config @@ -1,11 +1,11 @@ %%-*- mode: erlang -*- {plugins, [rebar3_proper, - {grpc_plugin, {git, "https://github.com/HJianBo/grpcbox_plugin", {tag, "v0.9.1"}}} + {grpc_plugin, {git, "https://github.com/HJianBo/grpcbox_plugin", {tag, "v0.10.0"}}} ]}. {deps, - [{grpc, {git, "https://github.com/emqx/grpc", {tag, "0.5.0"}}} + [{grpc, {git, "https://github.com/emqx/grpc", {tag, "0.6.0"}}} ]}. {grpc, diff --git a/apps/emqx_exproto/priv/protos/exproto.proto b/apps/emqx_exproto/priv/protos/exproto.proto index 633cc1758..4b567693c 100644 --- a/apps/emqx_exproto/priv/protos/exproto.proto +++ b/apps/emqx_exproto/priv/protos/exproto.proto @@ -47,17 +47,17 @@ service ConnectionHandler { // -- socket layer - rpc OnSocketCreated(SocketCreatedRequest) returns (EmptySuccess) {}; + rpc OnSocketCreated(stream SocketCreatedRequest) returns (EmptySuccess) {}; - rpc OnSocketClosed(SocketClosedRequest) returns (EmptySuccess) {}; + rpc OnSocketClosed(stream SocketClosedRequest) returns (EmptySuccess) {}; - rpc OnReceivedBytes(ReceivedBytesRequest) returns (EmptySuccess) {}; + rpc OnReceivedBytes(stream ReceivedBytesRequest) returns (EmptySuccess) {}; // -- pub/sub layer - rpc OnTimerTimeout(TimerTimeoutRequest) returns (EmptySuccess) {}; + rpc OnTimerTimeout(stream TimerTimeoutRequest) returns (EmptySuccess) {}; - rpc OnReceivedMessages(ReceivedMessagesRequest) returns (EmptySuccess) {}; + rpc OnReceivedMessages(stream ReceivedMessagesRequest) returns (EmptySuccess) {}; } message EmptySuccess { } diff --git a/apps/emqx_exproto/rebar.config b/apps/emqx_exproto/rebar.config index 9dd5a2090..88831ce15 100644 --- a/apps/emqx_exproto/rebar.config +++ b/apps/emqx_exproto/rebar.config @@ -9,11 +9,11 @@ {parse_transform}]}. {plugins, [rebar3_proper, - {grpc_plugin, {git, "https://github.com/HJianBo/grpcbox_plugin", {tag, "v0.9.1"}}} + {grpc_plugin, {git, "https://github.com/HJianBo/grpcbox_plugin", {tag, "v0.10.0"}}} ]}. {deps, - [{grpc, {git, "https://github.com/emqx/grpc", {tag, "0.5.0"}}} + [{grpc, {git, "https://github.com/emqx/grpc", {tag, "0.6.0"}}} ]}. {grpc, diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl index 4865cca58..49ded91ff 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -364,7 +364,8 @@ handle_info({sock_closed, Reason}, case queue:len(Queue) =:= 0 andalso Inflight =:= undefined of true -> - {shutdown, {sock_closed, Reason}, Channel}; + Channel1 = ensure_disconnected({sock_closed, Reason}, Channel), + {shutdown, {sock_closed, Reason}, Channel1}; _ -> %% delayed close process for flushing all callback funcs to gRPC server Channel1 = Channel#channel{closed_reason = {sock_closed, Reason}}, @@ -372,9 +373,9 @@ handle_info({sock_closed, Reason}, {ok, ensure_disconnected({sock_closed, Reason}, Channel2)} end; -handle_info({hreply, on_socket_created, {ok, _}}, Channel) -> +handle_info({hreply, on_socket_created, ok}, Channel) -> dispatch_or_close_process(Channel#channel{inflight = undefined}); -handle_info({hreply, FunName, {ok, _}}, Channel) +handle_info({hreply, FunName, ok}, Channel) when FunName == on_socket_closed; FunName == on_received_bytes; FunName == on_received_messages; @@ -525,7 +526,7 @@ interval(alive_timer, #channel{keepalive = Keepalive}) -> %%-------------------------------------------------------------------- wrap(Req) -> - Req#{conn => pid_to_list(self())}. + Req#{conn => base64:encode(term_to_binary(self()))}. dispatch_or_close_process(Channel = #channel{ rqueue = Queue, diff --git a/apps/emqx_exproto/src/emqx_exproto_gcli.erl b/apps/emqx_exproto/src/emqx_exproto_gcli.erl index 69784b70c..90e5d75a3 100644 --- a/apps/emqx_exproto/src/emqx_exproto_gcli.erl +++ b/apps/emqx_exproto/src/emqx_exproto_gcli.erl @@ -37,6 +37,12 @@ , code_change/3 ]). +-record(state, { + pool, + id, + streams + }). + -define(CONN_ADAPTER_MOD, emqx_exproto_v_1_connection_handler_client). %%-------------------------------------------------------------------- @@ -68,32 +74,34 @@ pick(Conn) -> init([Pool, Id]) -> true = gproc_pool:connect_worker(Pool, {Pool, Id}), - {ok, #{pool => Pool, id => Id}}. + {ok, #state{pool = Pool, id = Id, streams = #{}}}. handle_call(_Request, _From, State) -> {reply, ok, State}. -handle_cast({rpc, Fun, Req, Options, From}, State) -> - try - case apply(?CONN_ADAPTER_MOD, Fun, [Req, Options]) of - {ok, Resp, _Metadata} -> - ?LOG(debug, "~p got {ok, ~0p, ~0p}", [Fun, Resp, _Metadata]), - reply(From, Fun, {ok, Resp}); - {error, {Code, Msg}, _Metadata} -> - ?LOG(error, "CALL ~0p:~0p(~0p, ~0p) response errcode: ~0p, errmsg: ~0p", - [?CONN_ADAPTER_MOD, Fun, Req, Options, Code, Msg]), - reply(From, Fun, {error, {Code, Msg}}); - {error, Reason} -> - ?LOG(error, "CALL ~0p:~0p(~0p, ~0p) error: ~0p", - [?CONN_ADAPTER_MOD, Fun, Req, Options, Reason]), - reply(From, Fun, {error, Reason}) - end - catch _ : Rsn : Stk -> - ?LOG(error, "CALL ~0p:~0p(~0p, ~0p) throw an exception: ~0p, stacktrace: ~0p", - [?CONN_ADAPTER_MOD, Fun, Req, Options, Rsn, Stk]), - reply(From, Fun, {error, Rsn}) - end, - {noreply, State}. +handle_cast({rpc, Fun, Req, Options, From}, State = #state{streams = Streams}) -> + case ensure_stream_opened(Fun, Options, Streams) of + {error, Reason} -> + ?LOG(error, "CALL ~0p:~0p(~0p) failed, reason: ~0p", + [?CONN_ADAPTER_MOD, Fun, Options, Reason]), + reply(From, Fun, {error, Reason}), + {noreply, State#state{streams = Streams#{Fun => undefined}}}; + {ok, Stream} -> + case catch grpc_client:send(Stream, Req) of + ok -> + ?LOG(debug, "Send to ~p method successfully, request: ~0p", [Fun, Req]), + reply(From, Fun, ok), + {noreply, State#state{streams = Streams#{Fun => Stream}}}; + {'EXIT', {timeout, _Stk}} -> + ?LOG(error, "Send to ~p method timeout, request: ~0p", [Fun, Req]), + reply(From, Fun, {error, timeout}), + {noreply, State#state{streams = Streams#{Fun => Stream}}}; + {'EXIT', {Reason1, _Stk}} -> + ?LOG(error, "Send to ~p method failure, request: ~0p, stacktrace: ~0p", [Fun, Req, _Stk]), + reply(From, Fun, {error, Reason1}), + {noreply, State#state{streams = Streams#{Fun => undefined}}} + end + end. handle_info(_Info, State) -> {noreply, State}. @@ -111,3 +119,13 @@ code_change(_OldVsn, State, _Extra) -> reply(Pid, Fun, Result) -> Pid ! {hreply, Fun, Result}, ok. + +ensure_stream_opened(Fun, Options, Streams) -> + case maps:get(Fun, Streams, undefined) of + undefined -> + case apply(?CONN_ADAPTER_MOD, Fun, [Options]) of + {ok, Stream} -> {ok, Stream}; + {error, Reason} -> {error, Reason} + end; + Stream -> {ok, Stream} + end. diff --git a/apps/emqx_exproto/src/emqx_exproto_gsvr.erl b/apps/emqx_exproto/src/emqx_exproto_gsvr.erl index 286784009..c1007ee1d 100644 --- a/apps/emqx_exproto/src/emqx_exproto_gsvr.erl +++ b/apps/emqx_exproto/src/emqx_exproto_gsvr.erl @@ -115,10 +115,10 @@ unsubscribe(Req = #{conn := Conn, topic := Topic}, Md) -> %%-------------------------------------------------------------------- to_pid(ConnStr) -> - list_to_pid(binary_to_list(ConnStr)). + binary_to_term(base64:decode(ConnStr)). call(ConnStr, Req) -> - case catch to_pid(ConnStr) of + case catch to_pid(ConnStr) of {'EXIT', {badarg, _}} -> {error, ?RESP_PARAMS_TYPE_ERROR, <<"The conn type error">>}; diff --git a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl index dc6a25c06..e3cab1792 100644 --- a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl @@ -239,7 +239,6 @@ t_hook_connected_disconnected(Cfg) -> emqx:hook('client.connected', HookFun1), emqx:hook('client.disconnected', HookFun2), - send(Sock, ConnBin), {ok, ConnAckBin} = recv(Sock, 5000), diff --git a/apps/emqx_exproto/test/emqx_exproto_echo_svr.erl b/apps/emqx_exproto/test/emqx_exproto_echo_svr.erl index 3742a29a8..031093b50 100644 --- a/apps/emqx_exproto/test/emqx_exproto_echo_svr.erl +++ b/apps/emqx_exproto/test/emqx_exproto_echo_svr.erl @@ -40,6 +40,8 @@ , on_received_messages/2 ]). +-define(LOG(Fmt, Args), io:format(standard_error, Fmt, Args)). + -define(HTTP, #{grpc_opts => #{service_protos => [emqx_exproto_pb], services => #{'emqx.exproto.v1.ConnectionHandler' => ?MODULE}}, listen_opts => #{port => 9001, @@ -48,23 +50,44 @@ transport_opts => #{ssl => false}}). -define(CLIENT, emqx_exproto_v_1_connection_adapter_client). --define(send(Req), ?CLIENT:send(Req, #{channel => ct_test_channel})). --define(close(Req), ?CLIENT:close(Req, #{channel => ct_test_channel})). --define(authenticate(Req), ?CLIENT:authenticate(Req, #{channel => ct_test_channel})). --define(start_timer(Req), ?CLIENT:start_timer(Req, #{channel => ct_test_channel})). --define(publish(Req), ?CLIENT:publish(Req, #{channel => ct_test_channel})). --define(subscribe(Req), ?CLIENT:subscribe(Req, #{channel => ct_test_channel})). --define(unsubscribe(Req), ?CLIENT:unsubscribe(Req, #{channel => ct_test_channel})). --define(TYPE_CONNECT, 1). --define(TYPE_CONNACK, 2). --define(TYPE_PUBLISH, 3). --define(TYPE_PUBACK, 4). --define(TYPE_SUBSCRIBE, 5). --define(TYPE_SUBACK, 6). +-define(send(Req), ?CLIENT:send(Req, #{channel => ct_test_channel})). +-define(close(Req), ?CLIENT:close(Req, #{channel => ct_test_channel})). +-define(authenticate(Req), ?CLIENT:authenticate(Req, #{channel => ct_test_channel})). +-define(start_timer(Req), ?CLIENT:start_timer(Req, #{channel => ct_test_channel})). +-define(publish(Req), ?CLIENT:publish(Req, #{channel => ct_test_channel})). +-define(subscribe(Req), ?CLIENT:subscribe(Req, #{channel => ct_test_channel})). +-define(unsubscribe(Req), ?CLIENT:unsubscribe(Req, #{channel => ct_test_channel})). + +-define(TYPE_CONNECT, 1). +-define(TYPE_CONNACK, 2). +-define(TYPE_PUBLISH, 3). +-define(TYPE_PUBACK, 4). +-define(TYPE_SUBSCRIBE, 5). +-define(TYPE_SUBACK, 6). -define(TYPE_UNSUBSCRIBE, 7). --define(TYPE_UNSUBACK, 8). --define(TYPE_DISCONNECT, 9). +-define(TYPE_UNSUBACK, 8). +-define(TYPE_DISCONNECT, 9). + +-define(loop_recv_and_reply_empty_success(Stream), + ?loop_recv_and_reply_empty_success(Stream, fun(_) -> ok end)). + +-define(loop_recv_and_reply_empty_success(Stream, Fun), + begin + LoopRecv = fun _Lp(_St) -> + case grpc_stream:recv(_St) of + {more, _Reqs, _NSt} -> + ?LOG("~p: ~p~n", [?FUNCTION_NAME, _Reqs]), + Fun(_Reqs), _Lp(_NSt); + {eos, _Reqs, _NSt} -> + ?LOG("~p: ~p~n", [?FUNCTION_NAME, _Reqs]), + Fun(_Reqs), _NSt + end + end, + NStream = LoopRecv(Stream), + grpc_stream:reply(NStream, #{}), + {ok, NStream} + end). %%-------------------------------------------------------------------- %% APIs @@ -92,47 +115,53 @@ stop([_ChannPid, _SvrPid]) -> %% Protocol Adapter callbacks %%-------------------------------------------------------------------- --spec on_socket_created(emqx_exproto_pb:socket_created_request(), grpc:metadata()) - -> {ok, emqx_exproto_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_socket_created(Req, Md) -> - io:format("~p: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. +-spec on_socket_created(grpc_stream:stream(), grpc:metadata()) + -> {ok, grpc_stream:stream()}. +on_socket_created(Stream, _Md) -> + ?loop_recv_and_reply_empty_success(Stream). --spec on_socket_closed(emqx_exproto_pb:socket_closed_request(), grpc:metadata()) - -> {ok, emqx_exproto_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_socket_closed(Req, Md) -> - io:format("~p: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. +-spec on_socket_closed(grpc_stream:stream(), grpc:metadata()) + -> {ok, grpc_stream:stream()}. +on_socket_closed(Stream, _Md) -> + ?loop_recv_and_reply_empty_success(Stream). --spec on_received_bytes(emqx_exproto_pb:received_bytes_request(), grpc:metadata()) - -> {ok, emqx_exproto_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_received_bytes(Req = #{conn := Conn, bytes := Bytes}, Md) -> - io:format("~p: ~0p~n", [?FUNCTION_NAME, Req]), - #{<<"type">> := Type} = Params = emqx_json:decode(Bytes, [return_maps]), - _ = handle_in(Conn, Type, Params), - {ok, #{}, Md}. +-spec on_received_bytes(grpc_stream:stream(), grpc:metadata()) + -> {ok, grpc_stream:stream()}. +on_received_bytes(Stream, _Md) -> + ?loop_recv_and_reply_empty_success(Stream, + fun(Reqs) -> + lists:foreach( + fun(#{conn := Conn, bytes := Bytes}) -> + #{<<"type">> := Type} = Params = emqx_json:decode(Bytes, [return_maps]), + _ = handle_in(Conn, Type, Params) + end, Reqs) + end). --spec on_timer_timeout(emqx_exproto_pb:timer_timeout_request(), grpc:metadata()) - -> {ok, emqx_exproto_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_timer_timeout(Req = #{conn := Conn, type := 'KEEPALIVE'}, Md) -> - io:format("~p: ~0p~n", [?FUNCTION_NAME, Req]), - handle_out(Conn, ?TYPE_DISCONNECT), - ?close(#{conn => Conn}), - {ok, #{}, Md}. +-spec on_timer_timeout(grpc_stream:stream(), grpc:metadata()) + -> {ok, grpc_stream:stream()}. +on_timer_timeout(Stream, _Md) -> + ?loop_recv_and_reply_empty_success(Stream, + fun(Reqs) -> + lists:foreach( + fun(#{conn := Conn, type := 'KEEPALIVE'}) -> + ?LOG("Close this connection ~p due to keepalive timeout", [Conn]), + handle_out(Conn, ?TYPE_DISCONNECT), + ?close(#{conn => Conn}) + end, Reqs) + end). --spec on_received_messages(emqx_exproto_pb:received_messages_request(), grpc:metadata()) - -> {ok, emqx_exproto_pb:empty_success(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. -on_received_messages(Req = #{conn := Conn, messages := Messages}, Md) -> - io:format("~p: ~0p~n", [?FUNCTION_NAME, Req]), - lists:foreach(fun(Message) -> - handle_out(Conn, ?TYPE_PUBLISH, Message) - end, Messages), - {ok, #{}, Md}. +-spec on_received_messages(grpc_stream:stream(), grpc:metadata()) + -> {ok, grpc_stream:stream()}. +on_received_messages(Stream, _Md) -> + ?loop_recv_and_reply_empty_success(Stream, + fun(Reqs) -> + lists:foreach( + fun(#{conn := Conn, messages := Messages}) -> + lists:foreach(fun(Message) -> + handle_out(Conn, ?TYPE_PUBLISH, Message) + end, Messages) + end, Reqs) + end). %%-------------------------------------------------------------------- %% The Protocol Example: From aed801d6fdb62bf7c2cb11b7338550051832219e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 11 Jan 2021 18:05:49 +0800 Subject: [PATCH 05/24] test(exproto): refine test cases --- apps/emqx_exproto/README.md | 4 --- .../docs/{design.md => design-cn.md} | 30 +++++++++---------- apps/emqx_exproto/test/emqx_exproto_SUITE.erl | 2 +- 3 files changed, 16 insertions(+), 20 deletions(-) rename apps/emqx_exproto/docs/{design.md => design-cn.md} (76%) diff --git a/apps/emqx_exproto/README.md b/apps/emqx_exproto/README.md index 4b59dcae3..a9375e5d3 100644 --- a/apps/emqx_exproto/README.md +++ b/apps/emqx_exproto/README.md @@ -22,7 +22,3 @@ See: `priv/protos/exproto.proto` ## Recommended gRPC Framework See: https://github.com/grpc-ecosystem/awesome-grpc - -## Thanks - -- [grpcbox](https://github.com/tsloughter/grpcbox) diff --git a/apps/emqx_exproto/docs/design.md b/apps/emqx_exproto/docs/design-cn.md similarity index 76% rename from apps/emqx_exproto/docs/design.md rename to apps/emqx_exproto/docs/design-cn.md index 0a6a082e2..7af7dbdb3 100644 --- a/apps/emqx_exproto/docs/design.md +++ b/apps/emqx_exproto/docs/design-cn.md @@ -8,7 +8,7 @@ - 极强的扩展能力。使用 gRPC 作为 RPC 通信框架,支持各个主流编程语言 - 高吞吐。连接层以完全的异步非阻塞式 I/O 的方式实现 -- 连接层透明。完全的支持 TCP\TLS UDP\DTLS 类型的连接管理,并对上层提供统一个 API +- 连接层透明。完全的支持 TCP\TLS UDP\DTLS 类型的连接管理,并对上层提供统一的 API 接口 - 连接层的管理能力。例如,最大连接数,连接和吞吐的速率限制,IP 黑名单 等 ## 架构 @@ -17,7 +17,7 @@ 该插件主要需要处理的内容包括: -1. **连接层:** 该部分主要**维持 Socket 的生命周期,和数据的收发**。它的功能要求包括: +1. **连接层:** 该部分主要 **维持 Socket 的生命周期,和数据的收发**。它的功能要求包括: - 监听某个端口。当有新的 TCP/UDP 连接到达后,启动一个连接进程,来维持连接的状态。 - 调用 `OnSocketCreated` 回调。用于通知外部模块**已新建立了一个连接**。 - 调用 `OnScoektClosed` 回调。用于通知外部模块连接**已关闭**。 @@ -37,7 +37,7 @@ ## 接口设计 -从 gRPC 上的逻辑来说,emqx-exproto 会作为客户端向用户的 `ProtocolHandler` 服务发送回调请求。同时,它也会作为服务端向用户提供 `ConnectionAdapter` 服务,以提供 emqx-exproto 各个接口的访问。如图: +从 gRPC 上的逻辑来说,emqx-exproto 会作为客户端向用户的 `ConnectionHandler` 服务发送回调请求。同时,它也会作为服务端向用户提供 `ConnectionAdapter` 服务,以提供 emqx-exproto 各个接口的访问。如图: ![Extension Protocol gRPC Arch](images/exproto-grpc-arch.jpg) @@ -78,25 +78,25 @@ service ConnectionHandler { // -- socket layer - rpc OnSocketCreated(SocketCreatedRequest) returns (EmptySuccess) {}; + rpc OnSocketCreated(stream SocketCreatedRequest) returns (EmptySuccess) {}; - rpc OnSocketClosed(SocketClosedRequest) returns (EmptySuccess) {}; + rpc OnSocketClosed(stream SocketClosedRequest) returns (EmptySuccess) {}; - rpc OnReceivedBytes(ReceivedBytesRequest) returns (EmptySuccess) {}; + rpc OnReceivedBytes(stream ReceivedBytesRequest) returns (EmptySuccess) {}; // -- pub/sub layer - rpc OnTimerTimeout(TimerTimeoutRequest) returns (EmptySuccess) {}; + rpc OnTimerTimeout(stream TimerTimeoutRequest) returns (EmptySuccess) {}; - rpc OnReceivedMessages(ReceivedMessagesRequest) returns (EmptySuccess) {}; + rpc OnReceivedMessages(stream ReceivedMessagesRequest) returns (EmptySuccess) {}; } ``` ## 配置项设计 -1. 以 **监听器( Listener)** 为基础,提供 TCP/UDP 的监听。 +1. 以 **监听器(Listener)** 为基础,提供 TCP/UDP 的监听。 - Listener 目前仅支持:TCP、TLS、UDP、DTLS。(ws、wss、quic 暂不支持) -2. 每个监听器,会指定一个 `ProtocolHandler` 的服务地址,用于调用外部模块的接口。 +2. 每个监听器,会指定一个 `ConnectionHandler` 的服务地址,用于调用外部模块的接口。 3. emqx-exproto 还会监听一个 gRPC 端口用于提供对 `ConnectionAdapter` 服务的访问。 例如: @@ -117,11 +117,11 @@ exproto.server.https.keyfile = key.pem ## 例如,名称为 protoname 协议的 TCP 监听器配置 exproto.listener.protoname = tcp://0.0.0.0:7993 -## ProtocolHandler 服务地址及 https 的证书配置 -exproto.listener.protoname.proto_handler_url = http://127.0.0.1:9001 -#exproto.listener.protoname.proto_handler_certfile = -#exproto.listener.protoname.proto_handler_cacertfile = -#exproto.listener.protoname.proto_handler_keyfile = +## ConnectionHandler 服务地址及 https 的证书配置 +exproto.listener.protoname.connection_handler_url = http://127.0.0.1:9001 +#exproto.listener.protoname.connection_handler_certfile = +#exproto.listener.protoname.connection_handler_cacertfile = +#exproto.listener.protoname.connection_handler_keyfile = # ... ``` diff --git a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl index e3cab1792..bf5c0943f 100644 --- a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl @@ -106,7 +106,7 @@ t_mountpoint_echo(Cfg) -> send(Sock, ConnBin), {ok, ConnAckBin} = recv(Sock, 5000), - SubBin = frame_subscribe(<<"t/#">>, 1), + SubBin = frame_subscribe(<<"t/dn">>, 1), SubAckBin = frame_suback(0), send(Sock, SubBin), From 981482d707b5fbc1a4bd74bd0003b950ee54b031 Mon Sep 17 00:00:00 2001 From: bignullnull <751957846@qq.com> Date: Fri, 15 Jan 2021 09:48:57 +0800 Subject: [PATCH 06/24] fix(test): fix test cases (#4012) * fix(test): fix test cases --- .ci/apps_tests/docker-compose.yaml | 3 +- .ci/apps_tests/emqx_ldap/certs/cacert.pem | 20 --- .ci/apps_tests/emqx_ldap/certs/cert.pem | 19 --- .../emqx_ldap/certs/client-cert.pem | 19 --- .ci/apps_tests/emqx_ldap/certs/client-key.pem | 27 ---- .ci/apps_tests/emqx_ldap/certs/key.pem | 27 ---- .ci/apps_tests/emqx_ldap/schema/emqx.io.ldif | 135 ------------------ .ci/apps_tests/emqx_ldap/schema/emqx.schema | 46 ------ .../{emqx_ldap => openldap}/Dockerfile | 8 +- .../{emqx_ldap => openldap}/slapd.conf | 0 .../docker-compose-ldap.yaml | 3 +- .../docker-compose-pgsql-tls.yaml | 30 ++-- .ci/compatibility_tests/openldap/Dockerfile | 8 +- .../openldap/certs/cacert.pem | 20 --- .../openldap/certs/cert.pem | 19 --- .../openldap/certs/client-cert.pem | 19 --- .../openldap/certs/client-key.pem | 27 ---- .../openldap/certs/key.pem | 27 ---- .../openldap/schema/emqx.io.ldif | 135 ------------------ .../openldap/schema/emqx.schema | 46 ------ .ci/compatibility_tests/pgsql/Dockerfile | 12 ++ .ci/compatibility_tests/pgsql/pg.conf | 21 --- .ci/compatibility_tests/pgsql/pg_hba.conf | 9 ++ .github/workflows/run_cts_tests.yaml | 25 ++-- .github/workflows/run_test_cases.yaml | 3 - .../test/emqx_auth_mysql_SUITE.erl | 30 ++-- apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf | 2 +- .../test/emqx_auth_pgsql_SUITE.erl | 118 ++++----------- .../test/emqx_auth_pgsql_SUITE_data/ca.pem | 19 --- .../client-cert.pem | 19 --- .../emqx_auth_pgsql_SUITE_data/client-key.pem | 27 ---- .../test/emqx_auth_pgsql_SUITE_data/pg.conf | 21 --- .../emqx_auth_pgsql_SUITE_data/postgresql.crt | 21 +++ .../emqx_auth_pgsql_SUITE_data/postgresql.csr | 17 +++ .../emqx_auth_pgsql_SUITE_data/postgresql.key | 27 ++++ .../test/emqx_auth_pgsql_SUITE_data/root.crt | 21 +++ .../test/emqx_auth_pgsql_SUITE_data/root.srl | 1 + .../server-cert.pem | 19 --- .../emqx_auth_pgsql_SUITE_data/server-key.pem | 27 ---- .../emqx_auth_pgsql_SUITE_data/server.crt | 21 +++ .../emqx_auth_pgsql_SUITE_data/server.key | 27 ++++ 41 files changed, 237 insertions(+), 888 deletions(-) delete mode 100644 .ci/apps_tests/emqx_ldap/certs/cacert.pem delete mode 100644 .ci/apps_tests/emqx_ldap/certs/cert.pem delete mode 100644 .ci/apps_tests/emqx_ldap/certs/client-cert.pem delete mode 100644 .ci/apps_tests/emqx_ldap/certs/client-key.pem delete mode 100644 .ci/apps_tests/emqx_ldap/certs/key.pem delete mode 100644 .ci/apps_tests/emqx_ldap/schema/emqx.io.ldif delete mode 100644 .ci/apps_tests/emqx_ldap/schema/emqx.schema rename .ci/apps_tests/{emqx_ldap => openldap}/Dockerfile (70%) rename .ci/apps_tests/{emqx_ldap => openldap}/slapd.conf (100%) delete mode 100644 .ci/compatibility_tests/openldap/certs/cacert.pem delete mode 100644 .ci/compatibility_tests/openldap/certs/cert.pem delete mode 100644 .ci/compatibility_tests/openldap/certs/client-cert.pem delete mode 100644 .ci/compatibility_tests/openldap/certs/client-key.pem delete mode 100644 .ci/compatibility_tests/openldap/certs/key.pem delete mode 100644 .ci/compatibility_tests/openldap/schema/emqx.io.ldif delete mode 100644 .ci/compatibility_tests/openldap/schema/emqx.schema create mode 100644 .ci/compatibility_tests/pgsql/Dockerfile delete mode 100644 .ci/compatibility_tests/pgsql/pg.conf create mode 100644 .ci/compatibility_tests/pgsql/pg_hba.conf delete mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem delete mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-cert.pem delete mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-key.pem delete mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/pg.conf create mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.crt create mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.csr create mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.key create mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt create mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.srl delete mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-cert.pem delete mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-key.pem create mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.crt create mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.key diff --git a/.ci/apps_tests/docker-compose.yaml b/.ci/apps_tests/docker-compose.yaml index d65b30a74..1c0d781e3 100644 --- a/.ci/apps_tests/docker-compose.yaml +++ b/.ci/apps_tests/docker-compose.yaml @@ -80,7 +80,8 @@ services: ldap_server: container_name: openldap build: - context: ./emqx_ldap + context: ../.. + dockerfile: .ci/apps_tests/openldap/Dockerfile args: LDAP_TAG: ${LDAP_TAG} image: emqx-ldap:1.0 diff --git a/.ci/apps_tests/emqx_ldap/certs/cacert.pem b/.ci/apps_tests/emqx_ldap/certs/cacert.pem deleted file mode 100644 index 604fd2362..000000000 --- a/.ci/apps_tests/emqx_ldap/certs/cacert.pem +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDUTCCAjmgAwIBAgIJAPPYCjTmxdt/MA0GCSqGSIb3DQEBCwUAMD8xCzAJBgNV -BAYTAkNOMREwDwYDVQQIDAhoYW5nemhvdTEMMAoGA1UECgwDRU1RMQ8wDQYDVQQD -DAZSb290Q0EwHhcNMjAwNTA4MDgwNjUyWhcNMzAwNTA2MDgwNjUyWjA/MQswCQYD -VQQGEwJDTjERMA8GA1UECAwIaGFuZ3pob3UxDDAKBgNVBAoMA0VNUTEPMA0GA1UE -AwwGUm9vdENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzcgVLex1 -EZ9ON64EX8v+wcSjzOZpiEOsAOuSXOEN3wb8FKUxCdsGrsJYB7a5VM/Jot25Mod2 -juS3OBMg6r85k2TWjdxUoUs+HiUB/pP/ARaaW6VntpAEokpij/przWMPgJnBF3Ur -MjtbLayH9hGmpQrI5c2vmHQ2reRZnSFbY+2b8SXZ+3lZZgz9+BaQYWdQWfaUWEHZ -uDaNiViVO0OT8DRjCuiDp3yYDj3iLWbTA/gDL6Tf5XuHuEwcOQUrd+h0hyIphO8D -tsrsHZ14j4AWYLk1CPA6pq1HIUvEl2rANx2lVUNv+nt64K/Mr3RnVQd9s8bK+TXQ -KGHd2Lv/PALYuwIDAQABo1AwTjAdBgNVHQ4EFgQUGBmW+iDzxctWAWxmhgdlE8Pj -EbQwHwYDVR0jBBgwFoAUGBmW+iDzxctWAWxmhgdlE8PjEbQwDAYDVR0TBAUwAwEB -/zANBgkqhkiG9w0BAQsFAAOCAQEAGbhRUjpIred4cFAFJ7bbYD9hKu/yzWPWkMRa -ErlCKHmuYsYk+5d16JQhJaFy6MGXfLgo3KV2itl0d+OWNH0U9ULXcglTxy6+njo5 -CFqdUBPwN1jxhzo9yteDMKF4+AHIxbvCAJa17qcwUKR5MKNvv09C6pvQDJLzid7y -E2dkgSuggik3oa0427KvctFf8uhOV94RvEDyqvT5+pgNYZ2Yfga9pD/jjpoHEUlo -88IGU8/wJCx3Ds2yc8+oBg/ynxG8f/HmCC1ET6EHHoe2jlo8FpU/SgGtghS1YL30 -IWxNsPrUP+XsZpBJy/mvOhE5QXo6Y35zDqqj8tI7AGmAWu22jg== ------END CERTIFICATE----- diff --git a/.ci/apps_tests/emqx_ldap/certs/cert.pem b/.ci/apps_tests/emqx_ldap/certs/cert.pem deleted file mode 100644 index 092390b1d..000000000 --- a/.ci/apps_tests/emqx_ldap/certs/cert.pem +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDEzCCAfugAwIBAgIBAjANBgkqhkiG9w0BAQsFADA/MQswCQYDVQQGEwJDTjER -MA8GA1UECAwIaGFuZ3pob3UxDDAKBgNVBAoMA0VNUTEPMA0GA1UEAwwGUm9vdENB -MB4XDTIwMDUwODA4MDcwNVoXDTMwMDUwNjA4MDcwNVowPzELMAkGA1UEBhMCQ04x -ETAPBgNVBAgMCGhhbmd6aG91MQwwCgYDVQQKDANFTVExDzANBgNVBAMMBlNlcnZl -cjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALNeWT3pE+QFfiRJzKmn -AMUrWo3K2j/Tm3+Xnl6WLz67/0rcYrJbbKvS3uyRP/stXyXEKw9CepyQ1ViBVFkW -Aoy8qQEOWFDsZc/5UzhXUnb6LXr3qTkFEjNmhj+7uzv/lbBxlUG1NlYzSeOB6/RT -8zH/lhOeKhLnWYPXdXKsa1FL6ij4X8DeDO1kY7fvAGmBn/THh1uTpDizM4YmeI+7 -4dmayA5xXvARte5h4Vu5SIze7iC057N+vymToMk2Jgk+ZZFpyXrnq+yo6RaD3ANc -lrc4FbeUQZ5a5s5Sxgs9a0Y3WMG+7c5VnVXcbjBRz/aq2NtOnQQjikKKQA8GF080 -BQkCAwEAAaMaMBgwCQYDVR0TBAIwADALBgNVHQ8EBAMCBeAwDQYJKoZIhvcNAQEL -BQADggEBAJefnMZpaRDHQSNUIEL3iwGXE9c6PmIsQVE2ustr+CakBp3TZ4l0enLt -iGMfEVFju69cO4oyokWv+hl5eCMkHBf14Kv51vj448jowYnF1zmzn7SEzm5Uzlsa -sqjtAprnLyof69WtLU1j5rYWBuFX86yOTwRAFNjm9fvhAcrEONBsQtqipBWkMROp -iUYMkRqbKcQMdwxov+lHBYKq9zbWRoqLROAn54SRqgQk6c15JdEfgOOjShbsOkIH -UhqcwRkQic7n1zwHVGVDgNIZVgmJ2IdIWBlPEC7oLrRrBD/X1iEEXtKab6p5o22n -KB5mN+iQaE+Oe2cpGKZJiJRdM+IqDDQ= ------END CERTIFICATE----- diff --git a/.ci/apps_tests/emqx_ldap/certs/client-cert.pem b/.ci/apps_tests/emqx_ldap/certs/client-cert.pem deleted file mode 100644 index 09d855221..000000000 --- a/.ci/apps_tests/emqx_ldap/certs/client-cert.pem +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDEzCCAfugAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MQswCQYDVQQGEwJDTjER -MA8GA1UECAwIaGFuZ3pob3UxDDAKBgNVBAoMA0VNUTEPMA0GA1UEAwwGUm9vdENB -MB4XDTIwMDUwODA4MDY1N1oXDTMwMDUwNjA4MDY1N1owPzELMAkGA1UEBhMCQ04x -ETAPBgNVBAgMCGhhbmd6aG91MQwwCgYDVQQKDANFTVExDzANBgNVBAMMBkNsaWVu -dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMy4hoksKcZBDbY680u6 -TS25U51nuB1FBcGMlF9B/t057wPOlxF/OcmbxY5MwepS41JDGPgulE1V7fpsXkiW -1LUimYV/tsqBfymIe0mlY7oORahKji7zKQ2UBIVFhdlvQxunlIDnw6F9popUgyHt -dMhtlgZK8oqRwHxO5dbfoukYd6J/r+etS5q26sgVkf3C6dt0Td7B25H9qW+f7oLV -PbcHYCa+i73u9670nrpXsC+Qc7Mygwa2Kq/jwU+ftyLQnOeW07DuzOwsziC/fQZa -nbxR+8U9FNftgRcC3uP/JMKYUqsiRAuaDokARZxVTV5hUElfpO6z6/NItSDvvh3i -eikCAwEAAaMaMBgwCQYDVR0TBAIwADALBgNVHQ8EBAMCBeAwDQYJKoZIhvcNAQEL -BQADggEBABchYxKo0YMma7g1qDswJXsR5s56Czx/I+B41YcpMBMTrRqpUC0nHtLk -M7/tZp592u/tT8gzEnQjZLKBAhFeZaR3aaKyknLqwiPqJIgg0pgsBGITrAK3Pv4z -5/YvAJJKgTe5UdeTz6U4lvNEux/4juZ4pmqH4qSFJTOzQS7LmgSmNIdd072rwXBd -UzcSHzsJgEMb88u/LDLjj1pQ7AtZ4Tta8JZTvcgBFmjB0QUi6fgkHY6oGat/W4kR -jSRUBlMUbM/drr2PVzRc2dwbFIl3X+ZE6n5Sl3ZwRAC/s92JU6CPMRW02muVu6xl -goraNgPISnrbpR6KjxLZkVembXzjNNc= ------END CERTIFICATE----- diff --git a/.ci/apps_tests/emqx_ldap/certs/client-key.pem b/.ci/apps_tests/emqx_ldap/certs/client-key.pem deleted file mode 100644 index 2b3f30cf6..000000000 --- a/.ci/apps_tests/emqx_ldap/certs/client-key.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAzLiGiSwpxkENtjrzS7pNLblTnWe4HUUFwYyUX0H+3TnvA86X -EX85yZvFjkzB6lLjUkMY+C6UTVXt+mxeSJbUtSKZhX+2yoF/KYh7SaVjug5FqEqO -LvMpDZQEhUWF2W9DG6eUgOfDoX2milSDIe10yG2WBkryipHAfE7l1t+i6Rh3on+v -561LmrbqyBWR/cLp23RN3sHbkf2pb5/ugtU9twdgJr6Lve73rvSeulewL5BzszKD -BrYqr+PBT5+3ItCc55bTsO7M7CzOIL99BlqdvFH7xT0U1+2BFwLe4/8kwphSqyJE -C5oOiQBFnFVNXmFQSV+k7rPr80i1IO++HeJ6KQIDAQABAoIBAGWgvPjfuaU3qizq -uti/FY07USz0zkuJdkANH6LiSjlchzDmn8wJ0pApCjuIE0PV/g9aS8z4opp5q/gD -UBLM/a8mC/xf2EhTXOMrY7i9p/I3H5FZ4ZehEqIw9sWKK9YzC6dw26HabB2BGOnW -5nozPSQ6cp2RGzJ7BIkxSZwPzPnVTgy3OAuPOiJytvK+hGLhsNaT+Y9bNDvplVT2 -ZwYTV8GlHZC+4b2wNROILm0O86v96O+Qd8nn3fXjGHbMsAnONBq10bZS16L4fvkH -5G+W/1PeSXmtZFppdRRDxIW+DWcXK0D48WRliuxcV4eOOxI+a9N2ZJZZiNLQZGwg -w3A8+mECgYEA8HuJFrlRvdoBe2U/EwUtG74dcyy30L4yEBnN5QscXmEEikhaQCfX -Wm6EieMcIB/5I5TQmSw0cmBMeZjSXYoFdoI16/X6yMMuATdxpvhOZGdUGXxhAH+x -xoTUavWZnEqW3fkUU71kT5E2f2i+0zoatFESXHeslJyz85aAYpP92H0CgYEA2e5A -Yozt5eaA1Gyhd8SeptkEU4xPirNUnVQHStpMWUb1kzTNXrPmNWccQ7JpfpG6DcYl -zUF6p6mlzY+zkMiyPQjwEJlhiHM2NlL1QS7td0R8ewgsFoyn8WsBI4RejWrEG9td -EDniuIw+pBFkcWthnTLHwECHdzgquToyTMjrBB0CgYEA28tdGbrZXhcyAZEhHAZA -Gzog+pKlkpEzeonLKIuGKzCrEKRecIK5jrqyQsCjhS0T7ZRnL4g6i0s+umiV5M5w -fcc292pEA1h45L3DD6OlKplSQVTv55/OYS4oY3YEJtf5mfm8vWi9lQeY8sxOlQpn -O+VZTdBHmTC8PGeTAgZXHZUCgYA6Tyv88lYowB7SN2qQgBQu8jvdGtqhcs/99GCr -H3N0I69LPsKAR0QeH8OJPXBKhDUywESXAaEOwS5yrLNP1tMRz5Vj65YUCzeDG3kx -gpvY4IMp7ArX0bSRvJ6mYSFnVxy3k174G3TVCfksrtagHioVBGQ7xUg5ltafjrms -n8l55QKBgQDVzU8tQvBVqY8/1lnw11Vj4fkE/drZHJ5UkdC1eenOfSWhlSLfUJ8j -ds7vEWpRPPoVuPZYeR1y78cyxKe1GBx6Wa2lF5c7xjmiu0xbRnrxYeLolce9/ntp -asClqpnHT8/VJYTD7Kqj0fouTTZf0zkig/y+2XERppd8k+pSKjUCPQ== ------END RSA PRIVATE KEY----- diff --git a/.ci/apps_tests/emqx_ldap/certs/key.pem b/.ci/apps_tests/emqx_ldap/certs/key.pem deleted file mode 100644 index 6c338216e..000000000 --- a/.ci/apps_tests/emqx_ldap/certs/key.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAs15ZPekT5AV+JEnMqacAxStajcraP9Obf5eeXpYvPrv/Stxi -sltsq9Le7JE/+y1fJcQrD0J6nJDVWIFUWRYCjLypAQ5YUOxlz/lTOFdSdvotevep -OQUSM2aGP7u7O/+VsHGVQbU2VjNJ44Hr9FPzMf+WE54qEudZg9d1cqxrUUvqKPhf -wN4M7WRjt+8AaYGf9MeHW5OkOLMzhiZ4j7vh2ZrIDnFe8BG17mHhW7lIjN7uILTn -s36/KZOgyTYmCT5lkWnJeuer7KjpFoPcA1yWtzgVt5RBnlrmzlLGCz1rRjdYwb7t -zlWdVdxuMFHP9qrY206dBCOKQopADwYXTzQFCQIDAQABAoIBAQCuvCbr7Pd3lvI/ -n7VFQG+7pHRe1VKwAxDkx2t8cYos7y/QWcm8Ptwqtw58HzPZGWYrgGMCRpzzkRSF -V9g3wP1S5Scu5C6dBu5YIGc157tqNGXB+SpdZddJQ4Nc6yGHXYERllT04ffBGc3N -WG/oYS/1cSteiSIrsDy/91FvGRCi7FPxH3wIgHssY/tw69s1Cfvaq5lr2NTFzxIG -xCvpJKEdSfVfS9I7LYiymVjst3IOR/w76/ZFY9cRa8ZtmQSWWsm0TUpRC1jdcbkm -ZoJptYWlP+gSwx/fpMYftrkJFGOJhHJHQhwxT5X/ajAISeqjjwkWSEJLwnHQd11C -Zy2+29lBAoGBANlEAIK4VxCqyPXNKfoOOi5dS64NfvyH4A1v2+KaHWc7lqaqPN49 -ezfN2n3X+KWx4cviDD914Yc2JQ1vVJjSaHci7yivocDo2OfZDmjBqzaMp/y+rX1R -/f3MmiTqMa468rjaxI9RRZu7vDgpTR+za1+OBCgMzjvAng8dJuN/5gjlAoGBANNY -uYPKtearBmkqdrSV7eTUe49Nhr0XotLaVBH37TCW0Xv9wjO2xmbm5Ga/DCtPIsBb -yPeYwX9FjoasuadUD7hRvbFu6dBa0HGLmkXRJZTcD7MEX2Lhu4BuC72yDLLFd0r+ -Ep9WP7F5iJyagYqIZtz+4uf7gBvUDdmvXz3sGr1VAoGAdXTD6eeKeiI6PlhKBztF -zOb3EQOO0SsLv3fnodu7ZaHbUgLaoTMPuB17r2jgrYM7FKQCBxTNdfGZmmfDjlLB -0xZ5wL8ibU30ZXL8zTlWPElST9sto4B+FYVVF/vcG9sWeUUb2ncPcJ/Po3UAktDG -jYQTTyuNGtSJHpad/YOZctkCgYBtWRaC7bq3of0rJGFOhdQT9SwItN/lrfj8hyHA -OjpqTV4NfPmhsAtu6j96OZaeQc+FHvgXwt06cE6Rt4RG4uNPRluTFgO7XYFDfitP -vCppnoIw6S5BBvHwPP+uIhUX2bsi/dm8vu8tb+gSvo4PkwtFhEr6I9HglBKmcmog -q6waEQKBgHyecFBeM6Ls11Cd64vborwJPAuxIW7HBAFj/BS99oeG4TjBx4Sz2dFd -rzUibJt4ndnHIvCN8JQkjNG14i9hJln+H3mRss8fbZ9vQdqG+2vOWADYSzzsNI55 -RFY7JjluKcVkp/zCDeUxTU3O6sS+v6/3VE11Cob6OYQx3lN5wrZ3 ------END RSA PRIVATE KEY----- diff --git a/.ci/apps_tests/emqx_ldap/schema/emqx.io.ldif b/.ci/apps_tests/emqx_ldap/schema/emqx.io.ldif deleted file mode 100644 index f9833cd88..000000000 --- a/.ci/apps_tests/emqx_ldap/schema/emqx.io.ldif +++ /dev/null @@ -1,135 +0,0 @@ -## create emqx.io - -dn:dc=emqx,dc=io -objectclass: top -objectclass: dcobject -objectclass: organization -dc:emqx -o:emqx,Inc. - -# create testdevice.emqx.io -dn:ou=testdevice,dc=emqx,dc=io -objectClass: top -objectclass:organizationalUnit -ou:testdevice - -# create user admin -dn:uid=admin,ou=testdevice,dc=emqx,dc=io -objectClass: top -objectClass: simpleSecurityObject -objectClass: account -userPassword:: e1NIQX1XNnBoNU1tNVB6OEdnaVVMYlBnekczN21qOWc9 -uid: admin - -## create user=mqttuser0001, -# password=mqttuser0001, -# passhash={SHA}mlb3fat40MKBTXUVZwCKmL73R/0= -# base64passhash=e1NIQX1tbGIzZmF0NDBNS0JUWFVWWndDS21MNzNSLzA9 -dn:uid=mqttuser0001,ou=testdevice,dc=emqx,dc=io -objectClass: top -objectClass: mqttUser -objectClass: mqttDevice -objectClass: mqttSecurity -uid: mqttuser0001 -isEnabled: TRUE -mqttAccountName: user1 -mqttPublishTopic: mqttuser0001/pub/1 -mqttPublishTopic: mqttuser0001/pub/+ -mqttPublishTopic: mqttuser0001/pub/# -mqttSubscriptionTopic: mqttuser0001/sub/1 -mqttSubscriptionTopic: mqttuser0001/sub/+ -mqttSubscriptionTopic: mqttuser0001/sub/# -mqttPubSubTopic: mqttuser0001/pubsub/1 -mqttPubSubTopic: mqttuser0001/pubsub/+ -mqttPubSubTopic: mqttuser0001/pubsub/# -userPassword:: e1NIQX1tbGIzZmF0NDBNS0JUWFVWWndDS21MNzNSLzA9 - -## create user=mqttuser0002 -# password=mqttuser0002, -# passhash={SSHA}n9XdtoG4Q/TQ3TQF4Y+khJbMBH4qXj4M -# base64passhash=e1NTSEF9bjlYZHRvRzRRL1RRM1RRRjRZK2toSmJNQkg0cVhqNE0= -dn:uid=mqttuser0002,ou=testdevice,dc=emqx,dc=io -objectClass: top -objectClass: mqttUser -objectClass: mqttDevice -objectClass: mqttSecurity -uid: mqttuser0002 -isEnabled: TRUE -mqttAccountName: user2 -mqttPublishTopic: mqttuser0002/pub/1 -mqttPublishTopic: mqttuser0002/pub/+ -mqttPublishTopic: mqttuser0002/pub/# -mqttSubscriptionTopic: mqttuser0002/sub/1 -mqttSubscriptionTopic: mqttuser0002/sub/+ -mqttSubscriptionTopic: mqttuser0002/sub/# -mqttPubSubTopic: mqttuser0002/pubsub/1 -mqttPubSubTopic: mqttuser0002/pubsub/+ -mqttPubSubTopic: mqttuser0002/pubsub/# -userPassword:: e1NTSEF9bjlYZHRvRzRRL1RRM1RRRjRZK2toSmJNQkg0cVhqNE0= - -## create user mqttuser0003 -# password=mqttuser0003, -# passhash={MD5}ybsPGoaK3nDyiQvveiCOIw== -# base64passhash=e01ENX15YnNQR29hSzNuRHlpUXZ2ZWlDT0l3PT0= -dn:uid=mqttuser0003,ou=testdevice,dc=emqx,dc=io -objectClass: top -objectClass: mqttUser -objectClass: mqttDevice -objectClass: mqttSecurity -uid: mqttuser0003 -isEnabled: TRUE -mqttPublishTopic: mqttuser0003/pub/1 -mqttPublishTopic: mqttuser0003/pub/+ -mqttPublishTopic: mqttuser0003/pub/# -mqttSubscriptionTopic: mqttuser0003/sub/1 -mqttSubscriptionTopic: mqttuser0003/sub/+ -mqttSubscriptionTopic: mqttuser0003/sub/# -mqttPubSubTopic: mqttuser0003/pubsub/1 -mqttPubSubTopic: mqttuser0003/pubsub/+ -mqttPubSubTopic: mqttuser0003/pubsub/# -userPassword:: e01ENX15YnNQR29hSzNuRHlpUXZ2ZWlDT0l3PT0= - -## create user mqttuser0004 -# password=mqttuser0004, -# passhash={MD5}2Br6pPDSEDIEvUlu9+s+MA== -# base64passhash=e01ENX0yQnI2cFBEU0VESUV2VWx1OStzK01BPT0= -dn:uid=mqttuser0004,ou=testdevice,dc=emqx,dc=io -objectClass: top -objectClass: mqttUser -objectClass: mqttDevice -objectClass: mqttSecurity -uid: mqttuser0004 -isEnabled: TRUE -mqttPublishTopic: mqttuser0004/pub/1 -mqttPublishTopic: mqttuser0004/pub/+ -mqttPublishTopic: mqttuser0004/pub/# -mqttSubscriptionTopic: mqttuser0004/sub/1 -mqttSubscriptionTopic: mqttuser0004/sub/+ -mqttSubscriptionTopic: mqttuser0004/sub/# -mqttPubSubTopic: mqttuser0004/pubsub/1 -mqttPubSubTopic: mqttuser0004/pubsub/+ -mqttPubSubTopic: mqttuser0004/pubsub/# -userPassword: {MD5}2Br6pPDSEDIEvUlu9+s+MA== - -## create user mqttuser0005 -# password=mqttuser0005, -# passhash={SHA}jKnxeEDGR14kE8AR7yuVFOelhz4= -# base64passhash=e1NIQX1qS254ZUVER1IxNGtFOEFSN3l1VkZPZWxoejQ9 -objectClass: top -dn:uid=mqttuser0005,ou=testdevice,dc=emqx,dc=io -objectClass: mqttUser -objectClass: mqttDevice -objectClass: mqttSecurity -uid: mqttuser0005 -isEnabled: TRUE -mqttPublishTopic: mqttuser0005/pub/1 -mqttPublishTopic: mqttuser0005/pub/+ -mqttPublishTopic: mqttuser0005/pub/# -mqttSubscriptionTopic: mqttuser0005/sub/1 -mqttSubscriptionTopic: mqttuser0005/sub/+ -mqttSubscriptionTopic: mqttuser0005/sub/# -mqttPubSubTopic: mqttuser0005/pubsub/1 -mqttPubSubTopic: mqttuser0005/pubsub/+ -mqttPubSubTopic: mqttuser0005/pubsub/# -userPassword: {SHA}jKnxeEDGR14kE8AR7yuVFOelhz4= - diff --git a/.ci/apps_tests/emqx_ldap/schema/emqx.schema b/.ci/apps_tests/emqx_ldap/schema/emqx.schema deleted file mode 100644 index 55f92269b..000000000 --- a/.ci/apps_tests/emqx_ldap/schema/emqx.schema +++ /dev/null @@ -1,46 +0,0 @@ -# -# Preliminary Apple OS X Native LDAP Schema -# This file is subject to change. -# -attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.1.3 NAME 'isEnabled' - EQUALITY booleanMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 - SINGLE-VALUE - USAGE userApplications ) - -attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4.1 NAME ( 'mqttPublishTopic' 'mpt' ) - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - USAGE userApplications ) -attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4.2 NAME ( 'mqttSubscriptionTopic' 'mst' ) - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - USAGE userApplications ) -attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4.3 NAME ( 'mqttPubSubTopic' 'mpst' ) - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - USAGE userApplications ) -attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4.4 NAME ( 'mqttAccountName' 'man' ) - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - USAGE userApplications ) - - -objectclass ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4 NAME 'mqttUser' - AUXILIARY - MAY ( mqttPublishTopic $ mqttSubscriptionTopic $ mqttPubSubTopic $ mqttAccountName) ) - -objectclass ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.2 NAME 'mqttDevice' - SUP top - STRUCTURAL - MUST ( uid ) - MAY ( isEnabled ) ) - -objectclass ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.3 NAME 'mqttSecurity' - SUP top - AUXILIARY - MAY ( userPassword $ userPKCS12 $ pwdAttribute $ pwdLockout ) ) diff --git a/.ci/apps_tests/emqx_ldap/Dockerfile b/.ci/apps_tests/openldap/Dockerfile similarity index 70% rename from .ci/apps_tests/emqx_ldap/Dockerfile rename to .ci/apps_tests/openldap/Dockerfile index a0cf7d25e..f15a48e69 100644 --- a/.ci/apps_tests/emqx_ldap/Dockerfile +++ b/.ci/apps_tests/openldap/Dockerfile @@ -9,10 +9,10 @@ RUN wget ftp://ftp.openldap.org/pub/OpenLDAP/openldap-release/openldap-${LDAP_TA && ./configure && make depend && make && make install \ && cd .. && rm -rf openldap-${LDAP_TAG} -COPY ./slapd.conf /usr/local/etc/openldap/slapd.conf -COPY ./schema/emqx.io.ldif /usr/local/etc/openldap/schema/emqx.io.ldif -COPY ./schema/emqx.schema /usr/local/etc/openldap/schema/emqx.schema -COPY ./certs/*.pem /usr/local/etc/openldap/ +COPY .ci/apps_tests/openldap/slapd.conf /usr/local/etc/openldap/slapd.conf +COPY apps/emqx_auth_ldap/emqx.io.ldif /usr/local/etc/openldap/schema/emqx.io.ldif +COPY apps/emqx_auth_ldap/emqx.schema /usr/local/etc/openldap/schema/emqx.schema +COPY apps/emqx_auth_ldap/test/certs/*.pem /usr/local/etc/openldap/ RUN mkdir -p /usr/local/etc/openldap/data \ && slapadd -l /usr/local/etc/openldap/schema/emqx.io.ldif -f /usr/local/etc/openldap/slapd.conf diff --git a/.ci/apps_tests/emqx_ldap/slapd.conf b/.ci/apps_tests/openldap/slapd.conf similarity index 100% rename from .ci/apps_tests/emqx_ldap/slapd.conf rename to .ci/apps_tests/openldap/slapd.conf diff --git a/.ci/compatibility_tests/docker-compose-ldap.yaml b/.ci/compatibility_tests/docker-compose-ldap.yaml index 69a403569..c41719b8a 100644 --- a/.ci/compatibility_tests/docker-compose-ldap.yaml +++ b/.ci/compatibility_tests/docker-compose-ldap.yaml @@ -16,7 +16,8 @@ services: ldap_server: container_name: ldap build: - context: ./openldap + context: ../.. + dockerfile: .ci/compatibility_tests/openldap/Dockerfile args: LDAP_TAG: ${LDAP_TAG} image: openldap diff --git a/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml b/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml index 35460e139..c7daf1e38 100644 --- a/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml +++ b/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml @@ -15,19 +15,31 @@ services: pgsql_server: container_name: pgsql - image: postgres:${PGSQL_TAG} + build: + context: ../.. + dockerfile: .ci/compatibility_tests/pgsql/Dockerfile + args: + POSTGRES_USER: postgres + BUILD_FROM: postgres:${PGSQL_TAG} + image: emqx_pgsql:${PGSQL_TAG} restart: always environment: - POSTGRES_PASSWORD: public - POSTGRES_USER: root - POSTGRES_DB: mqtt - volumes: - - ../../apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/pg.conf:/etc/postgresql/postgresql.conf - - ../../apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-cert.pem:/etc/postgresql/server-cert.pem - - ../../apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-key.pem:/etc/postgresql/server-key.pem + POSTGRES_DB: postgres + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - "5432:5432" command: - -c - - config_file=/etc/postgresql/postgresql.conf + - ssl=on + - -c + - ssl_cert_file=/var/lib/postgresql/server.crt + - -c + - ssl_key_file=/var/lib/postgresql/server.key + - -c + - ssl_ca_file=/var/lib/postgresql/root.crt + - -c + - hba_file=/var/lib/postgresql/pg_hba.conf networks: - emqx_bridge diff --git a/.ci/compatibility_tests/openldap/Dockerfile b/.ci/compatibility_tests/openldap/Dockerfile index a0cf7d25e..fa15ab5eb 100644 --- a/.ci/compatibility_tests/openldap/Dockerfile +++ b/.ci/compatibility_tests/openldap/Dockerfile @@ -9,10 +9,10 @@ RUN wget ftp://ftp.openldap.org/pub/OpenLDAP/openldap-release/openldap-${LDAP_TA && ./configure && make depend && make && make install \ && cd .. && rm -rf openldap-${LDAP_TAG} -COPY ./slapd.conf /usr/local/etc/openldap/slapd.conf -COPY ./schema/emqx.io.ldif /usr/local/etc/openldap/schema/emqx.io.ldif -COPY ./schema/emqx.schema /usr/local/etc/openldap/schema/emqx.schema -COPY ./certs/*.pem /usr/local/etc/openldap/ +COPY .ci/compatibility_tests/openldap/slapd.conf /usr/local/etc/openldap/slapd.conf +COPY apps/emqx_auth_ldap/emqx.io.ldif /usr/local/etc/openldap/schema/emqx.io.ldif +COPY apps/emqx_auth_ldap/emqx.schema /usr/local/etc/openldap/schema/emqx.schema +COPY apps/emqx_auth_ldap/test/certs/*.pem /usr/local/etc/openldap/ RUN mkdir -p /usr/local/etc/openldap/data \ && slapadd -l /usr/local/etc/openldap/schema/emqx.io.ldif -f /usr/local/etc/openldap/slapd.conf diff --git a/.ci/compatibility_tests/openldap/certs/cacert.pem b/.ci/compatibility_tests/openldap/certs/cacert.pem deleted file mode 100644 index 604fd2362..000000000 --- a/.ci/compatibility_tests/openldap/certs/cacert.pem +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDUTCCAjmgAwIBAgIJAPPYCjTmxdt/MA0GCSqGSIb3DQEBCwUAMD8xCzAJBgNV -BAYTAkNOMREwDwYDVQQIDAhoYW5nemhvdTEMMAoGA1UECgwDRU1RMQ8wDQYDVQQD -DAZSb290Q0EwHhcNMjAwNTA4MDgwNjUyWhcNMzAwNTA2MDgwNjUyWjA/MQswCQYD -VQQGEwJDTjERMA8GA1UECAwIaGFuZ3pob3UxDDAKBgNVBAoMA0VNUTEPMA0GA1UE -AwwGUm9vdENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzcgVLex1 -EZ9ON64EX8v+wcSjzOZpiEOsAOuSXOEN3wb8FKUxCdsGrsJYB7a5VM/Jot25Mod2 -juS3OBMg6r85k2TWjdxUoUs+HiUB/pP/ARaaW6VntpAEokpij/przWMPgJnBF3Ur -MjtbLayH9hGmpQrI5c2vmHQ2reRZnSFbY+2b8SXZ+3lZZgz9+BaQYWdQWfaUWEHZ -uDaNiViVO0OT8DRjCuiDp3yYDj3iLWbTA/gDL6Tf5XuHuEwcOQUrd+h0hyIphO8D -tsrsHZ14j4AWYLk1CPA6pq1HIUvEl2rANx2lVUNv+nt64K/Mr3RnVQd9s8bK+TXQ -KGHd2Lv/PALYuwIDAQABo1AwTjAdBgNVHQ4EFgQUGBmW+iDzxctWAWxmhgdlE8Pj -EbQwHwYDVR0jBBgwFoAUGBmW+iDzxctWAWxmhgdlE8PjEbQwDAYDVR0TBAUwAwEB -/zANBgkqhkiG9w0BAQsFAAOCAQEAGbhRUjpIred4cFAFJ7bbYD9hKu/yzWPWkMRa -ErlCKHmuYsYk+5d16JQhJaFy6MGXfLgo3KV2itl0d+OWNH0U9ULXcglTxy6+njo5 -CFqdUBPwN1jxhzo9yteDMKF4+AHIxbvCAJa17qcwUKR5MKNvv09C6pvQDJLzid7y -E2dkgSuggik3oa0427KvctFf8uhOV94RvEDyqvT5+pgNYZ2Yfga9pD/jjpoHEUlo -88IGU8/wJCx3Ds2yc8+oBg/ynxG8f/HmCC1ET6EHHoe2jlo8FpU/SgGtghS1YL30 -IWxNsPrUP+XsZpBJy/mvOhE5QXo6Y35zDqqj8tI7AGmAWu22jg== ------END CERTIFICATE----- diff --git a/.ci/compatibility_tests/openldap/certs/cert.pem b/.ci/compatibility_tests/openldap/certs/cert.pem deleted file mode 100644 index 092390b1d..000000000 --- a/.ci/compatibility_tests/openldap/certs/cert.pem +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDEzCCAfugAwIBAgIBAjANBgkqhkiG9w0BAQsFADA/MQswCQYDVQQGEwJDTjER -MA8GA1UECAwIaGFuZ3pob3UxDDAKBgNVBAoMA0VNUTEPMA0GA1UEAwwGUm9vdENB -MB4XDTIwMDUwODA4MDcwNVoXDTMwMDUwNjA4MDcwNVowPzELMAkGA1UEBhMCQ04x -ETAPBgNVBAgMCGhhbmd6aG91MQwwCgYDVQQKDANFTVExDzANBgNVBAMMBlNlcnZl -cjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALNeWT3pE+QFfiRJzKmn -AMUrWo3K2j/Tm3+Xnl6WLz67/0rcYrJbbKvS3uyRP/stXyXEKw9CepyQ1ViBVFkW -Aoy8qQEOWFDsZc/5UzhXUnb6LXr3qTkFEjNmhj+7uzv/lbBxlUG1NlYzSeOB6/RT -8zH/lhOeKhLnWYPXdXKsa1FL6ij4X8DeDO1kY7fvAGmBn/THh1uTpDizM4YmeI+7 -4dmayA5xXvARte5h4Vu5SIze7iC057N+vymToMk2Jgk+ZZFpyXrnq+yo6RaD3ANc -lrc4FbeUQZ5a5s5Sxgs9a0Y3WMG+7c5VnVXcbjBRz/aq2NtOnQQjikKKQA8GF080 -BQkCAwEAAaMaMBgwCQYDVR0TBAIwADALBgNVHQ8EBAMCBeAwDQYJKoZIhvcNAQEL -BQADggEBAJefnMZpaRDHQSNUIEL3iwGXE9c6PmIsQVE2ustr+CakBp3TZ4l0enLt -iGMfEVFju69cO4oyokWv+hl5eCMkHBf14Kv51vj448jowYnF1zmzn7SEzm5Uzlsa -sqjtAprnLyof69WtLU1j5rYWBuFX86yOTwRAFNjm9fvhAcrEONBsQtqipBWkMROp -iUYMkRqbKcQMdwxov+lHBYKq9zbWRoqLROAn54SRqgQk6c15JdEfgOOjShbsOkIH -UhqcwRkQic7n1zwHVGVDgNIZVgmJ2IdIWBlPEC7oLrRrBD/X1iEEXtKab6p5o22n -KB5mN+iQaE+Oe2cpGKZJiJRdM+IqDDQ= ------END CERTIFICATE----- diff --git a/.ci/compatibility_tests/openldap/certs/client-cert.pem b/.ci/compatibility_tests/openldap/certs/client-cert.pem deleted file mode 100644 index 09d855221..000000000 --- a/.ci/compatibility_tests/openldap/certs/client-cert.pem +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDEzCCAfugAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MQswCQYDVQQGEwJDTjER -MA8GA1UECAwIaGFuZ3pob3UxDDAKBgNVBAoMA0VNUTEPMA0GA1UEAwwGUm9vdENB -MB4XDTIwMDUwODA4MDY1N1oXDTMwMDUwNjA4MDY1N1owPzELMAkGA1UEBhMCQ04x -ETAPBgNVBAgMCGhhbmd6aG91MQwwCgYDVQQKDANFTVExDzANBgNVBAMMBkNsaWVu -dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMy4hoksKcZBDbY680u6 -TS25U51nuB1FBcGMlF9B/t057wPOlxF/OcmbxY5MwepS41JDGPgulE1V7fpsXkiW -1LUimYV/tsqBfymIe0mlY7oORahKji7zKQ2UBIVFhdlvQxunlIDnw6F9popUgyHt -dMhtlgZK8oqRwHxO5dbfoukYd6J/r+etS5q26sgVkf3C6dt0Td7B25H9qW+f7oLV -PbcHYCa+i73u9670nrpXsC+Qc7Mygwa2Kq/jwU+ftyLQnOeW07DuzOwsziC/fQZa -nbxR+8U9FNftgRcC3uP/JMKYUqsiRAuaDokARZxVTV5hUElfpO6z6/NItSDvvh3i -eikCAwEAAaMaMBgwCQYDVR0TBAIwADALBgNVHQ8EBAMCBeAwDQYJKoZIhvcNAQEL -BQADggEBABchYxKo0YMma7g1qDswJXsR5s56Czx/I+B41YcpMBMTrRqpUC0nHtLk -M7/tZp592u/tT8gzEnQjZLKBAhFeZaR3aaKyknLqwiPqJIgg0pgsBGITrAK3Pv4z -5/YvAJJKgTe5UdeTz6U4lvNEux/4juZ4pmqH4qSFJTOzQS7LmgSmNIdd072rwXBd -UzcSHzsJgEMb88u/LDLjj1pQ7AtZ4Tta8JZTvcgBFmjB0QUi6fgkHY6oGat/W4kR -jSRUBlMUbM/drr2PVzRc2dwbFIl3X+ZE6n5Sl3ZwRAC/s92JU6CPMRW02muVu6xl -goraNgPISnrbpR6KjxLZkVembXzjNNc= ------END CERTIFICATE----- diff --git a/.ci/compatibility_tests/openldap/certs/client-key.pem b/.ci/compatibility_tests/openldap/certs/client-key.pem deleted file mode 100644 index 2b3f30cf6..000000000 --- a/.ci/compatibility_tests/openldap/certs/client-key.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAzLiGiSwpxkENtjrzS7pNLblTnWe4HUUFwYyUX0H+3TnvA86X -EX85yZvFjkzB6lLjUkMY+C6UTVXt+mxeSJbUtSKZhX+2yoF/KYh7SaVjug5FqEqO -LvMpDZQEhUWF2W9DG6eUgOfDoX2milSDIe10yG2WBkryipHAfE7l1t+i6Rh3on+v -561LmrbqyBWR/cLp23RN3sHbkf2pb5/ugtU9twdgJr6Lve73rvSeulewL5BzszKD -BrYqr+PBT5+3ItCc55bTsO7M7CzOIL99BlqdvFH7xT0U1+2BFwLe4/8kwphSqyJE -C5oOiQBFnFVNXmFQSV+k7rPr80i1IO++HeJ6KQIDAQABAoIBAGWgvPjfuaU3qizq -uti/FY07USz0zkuJdkANH6LiSjlchzDmn8wJ0pApCjuIE0PV/g9aS8z4opp5q/gD -UBLM/a8mC/xf2EhTXOMrY7i9p/I3H5FZ4ZehEqIw9sWKK9YzC6dw26HabB2BGOnW -5nozPSQ6cp2RGzJ7BIkxSZwPzPnVTgy3OAuPOiJytvK+hGLhsNaT+Y9bNDvplVT2 -ZwYTV8GlHZC+4b2wNROILm0O86v96O+Qd8nn3fXjGHbMsAnONBq10bZS16L4fvkH -5G+W/1PeSXmtZFppdRRDxIW+DWcXK0D48WRliuxcV4eOOxI+a9N2ZJZZiNLQZGwg -w3A8+mECgYEA8HuJFrlRvdoBe2U/EwUtG74dcyy30L4yEBnN5QscXmEEikhaQCfX -Wm6EieMcIB/5I5TQmSw0cmBMeZjSXYoFdoI16/X6yMMuATdxpvhOZGdUGXxhAH+x -xoTUavWZnEqW3fkUU71kT5E2f2i+0zoatFESXHeslJyz85aAYpP92H0CgYEA2e5A -Yozt5eaA1Gyhd8SeptkEU4xPirNUnVQHStpMWUb1kzTNXrPmNWccQ7JpfpG6DcYl -zUF6p6mlzY+zkMiyPQjwEJlhiHM2NlL1QS7td0R8ewgsFoyn8WsBI4RejWrEG9td -EDniuIw+pBFkcWthnTLHwECHdzgquToyTMjrBB0CgYEA28tdGbrZXhcyAZEhHAZA -Gzog+pKlkpEzeonLKIuGKzCrEKRecIK5jrqyQsCjhS0T7ZRnL4g6i0s+umiV5M5w -fcc292pEA1h45L3DD6OlKplSQVTv55/OYS4oY3YEJtf5mfm8vWi9lQeY8sxOlQpn -O+VZTdBHmTC8PGeTAgZXHZUCgYA6Tyv88lYowB7SN2qQgBQu8jvdGtqhcs/99GCr -H3N0I69LPsKAR0QeH8OJPXBKhDUywESXAaEOwS5yrLNP1tMRz5Vj65YUCzeDG3kx -gpvY4IMp7ArX0bSRvJ6mYSFnVxy3k174G3TVCfksrtagHioVBGQ7xUg5ltafjrms -n8l55QKBgQDVzU8tQvBVqY8/1lnw11Vj4fkE/drZHJ5UkdC1eenOfSWhlSLfUJ8j -ds7vEWpRPPoVuPZYeR1y78cyxKe1GBx6Wa2lF5c7xjmiu0xbRnrxYeLolce9/ntp -asClqpnHT8/VJYTD7Kqj0fouTTZf0zkig/y+2XERppd8k+pSKjUCPQ== ------END RSA PRIVATE KEY----- diff --git a/.ci/compatibility_tests/openldap/certs/key.pem b/.ci/compatibility_tests/openldap/certs/key.pem deleted file mode 100644 index 6c338216e..000000000 --- a/.ci/compatibility_tests/openldap/certs/key.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAs15ZPekT5AV+JEnMqacAxStajcraP9Obf5eeXpYvPrv/Stxi -sltsq9Le7JE/+y1fJcQrD0J6nJDVWIFUWRYCjLypAQ5YUOxlz/lTOFdSdvotevep -OQUSM2aGP7u7O/+VsHGVQbU2VjNJ44Hr9FPzMf+WE54qEudZg9d1cqxrUUvqKPhf -wN4M7WRjt+8AaYGf9MeHW5OkOLMzhiZ4j7vh2ZrIDnFe8BG17mHhW7lIjN7uILTn -s36/KZOgyTYmCT5lkWnJeuer7KjpFoPcA1yWtzgVt5RBnlrmzlLGCz1rRjdYwb7t -zlWdVdxuMFHP9qrY206dBCOKQopADwYXTzQFCQIDAQABAoIBAQCuvCbr7Pd3lvI/ -n7VFQG+7pHRe1VKwAxDkx2t8cYos7y/QWcm8Ptwqtw58HzPZGWYrgGMCRpzzkRSF -V9g3wP1S5Scu5C6dBu5YIGc157tqNGXB+SpdZddJQ4Nc6yGHXYERllT04ffBGc3N -WG/oYS/1cSteiSIrsDy/91FvGRCi7FPxH3wIgHssY/tw69s1Cfvaq5lr2NTFzxIG -xCvpJKEdSfVfS9I7LYiymVjst3IOR/w76/ZFY9cRa8ZtmQSWWsm0TUpRC1jdcbkm -ZoJptYWlP+gSwx/fpMYftrkJFGOJhHJHQhwxT5X/ajAISeqjjwkWSEJLwnHQd11C -Zy2+29lBAoGBANlEAIK4VxCqyPXNKfoOOi5dS64NfvyH4A1v2+KaHWc7lqaqPN49 -ezfN2n3X+KWx4cviDD914Yc2JQ1vVJjSaHci7yivocDo2OfZDmjBqzaMp/y+rX1R -/f3MmiTqMa468rjaxI9RRZu7vDgpTR+za1+OBCgMzjvAng8dJuN/5gjlAoGBANNY -uYPKtearBmkqdrSV7eTUe49Nhr0XotLaVBH37TCW0Xv9wjO2xmbm5Ga/DCtPIsBb -yPeYwX9FjoasuadUD7hRvbFu6dBa0HGLmkXRJZTcD7MEX2Lhu4BuC72yDLLFd0r+ -Ep9WP7F5iJyagYqIZtz+4uf7gBvUDdmvXz3sGr1VAoGAdXTD6eeKeiI6PlhKBztF -zOb3EQOO0SsLv3fnodu7ZaHbUgLaoTMPuB17r2jgrYM7FKQCBxTNdfGZmmfDjlLB -0xZ5wL8ibU30ZXL8zTlWPElST9sto4B+FYVVF/vcG9sWeUUb2ncPcJ/Po3UAktDG -jYQTTyuNGtSJHpad/YOZctkCgYBtWRaC7bq3of0rJGFOhdQT9SwItN/lrfj8hyHA -OjpqTV4NfPmhsAtu6j96OZaeQc+FHvgXwt06cE6Rt4RG4uNPRluTFgO7XYFDfitP -vCppnoIw6S5BBvHwPP+uIhUX2bsi/dm8vu8tb+gSvo4PkwtFhEr6I9HglBKmcmog -q6waEQKBgHyecFBeM6Ls11Cd64vborwJPAuxIW7HBAFj/BS99oeG4TjBx4Sz2dFd -rzUibJt4ndnHIvCN8JQkjNG14i9hJln+H3mRss8fbZ9vQdqG+2vOWADYSzzsNI55 -RFY7JjluKcVkp/zCDeUxTU3O6sS+v6/3VE11Cob6OYQx3lN5wrZ3 ------END RSA PRIVATE KEY----- diff --git a/.ci/compatibility_tests/openldap/schema/emqx.io.ldif b/.ci/compatibility_tests/openldap/schema/emqx.io.ldif deleted file mode 100644 index f9833cd88..000000000 --- a/.ci/compatibility_tests/openldap/schema/emqx.io.ldif +++ /dev/null @@ -1,135 +0,0 @@ -## create emqx.io - -dn:dc=emqx,dc=io -objectclass: top -objectclass: dcobject -objectclass: organization -dc:emqx -o:emqx,Inc. - -# create testdevice.emqx.io -dn:ou=testdevice,dc=emqx,dc=io -objectClass: top -objectclass:organizationalUnit -ou:testdevice - -# create user admin -dn:uid=admin,ou=testdevice,dc=emqx,dc=io -objectClass: top -objectClass: simpleSecurityObject -objectClass: account -userPassword:: e1NIQX1XNnBoNU1tNVB6OEdnaVVMYlBnekczN21qOWc9 -uid: admin - -## create user=mqttuser0001, -# password=mqttuser0001, -# passhash={SHA}mlb3fat40MKBTXUVZwCKmL73R/0= -# base64passhash=e1NIQX1tbGIzZmF0NDBNS0JUWFVWWndDS21MNzNSLzA9 -dn:uid=mqttuser0001,ou=testdevice,dc=emqx,dc=io -objectClass: top -objectClass: mqttUser -objectClass: mqttDevice -objectClass: mqttSecurity -uid: mqttuser0001 -isEnabled: TRUE -mqttAccountName: user1 -mqttPublishTopic: mqttuser0001/pub/1 -mqttPublishTopic: mqttuser0001/pub/+ -mqttPublishTopic: mqttuser0001/pub/# -mqttSubscriptionTopic: mqttuser0001/sub/1 -mqttSubscriptionTopic: mqttuser0001/sub/+ -mqttSubscriptionTopic: mqttuser0001/sub/# -mqttPubSubTopic: mqttuser0001/pubsub/1 -mqttPubSubTopic: mqttuser0001/pubsub/+ -mqttPubSubTopic: mqttuser0001/pubsub/# -userPassword:: e1NIQX1tbGIzZmF0NDBNS0JUWFVWWndDS21MNzNSLzA9 - -## create user=mqttuser0002 -# password=mqttuser0002, -# passhash={SSHA}n9XdtoG4Q/TQ3TQF4Y+khJbMBH4qXj4M -# base64passhash=e1NTSEF9bjlYZHRvRzRRL1RRM1RRRjRZK2toSmJNQkg0cVhqNE0= -dn:uid=mqttuser0002,ou=testdevice,dc=emqx,dc=io -objectClass: top -objectClass: mqttUser -objectClass: mqttDevice -objectClass: mqttSecurity -uid: mqttuser0002 -isEnabled: TRUE -mqttAccountName: user2 -mqttPublishTopic: mqttuser0002/pub/1 -mqttPublishTopic: mqttuser0002/pub/+ -mqttPublishTopic: mqttuser0002/pub/# -mqttSubscriptionTopic: mqttuser0002/sub/1 -mqttSubscriptionTopic: mqttuser0002/sub/+ -mqttSubscriptionTopic: mqttuser0002/sub/# -mqttPubSubTopic: mqttuser0002/pubsub/1 -mqttPubSubTopic: mqttuser0002/pubsub/+ -mqttPubSubTopic: mqttuser0002/pubsub/# -userPassword:: e1NTSEF9bjlYZHRvRzRRL1RRM1RRRjRZK2toSmJNQkg0cVhqNE0= - -## create user mqttuser0003 -# password=mqttuser0003, -# passhash={MD5}ybsPGoaK3nDyiQvveiCOIw== -# base64passhash=e01ENX15YnNQR29hSzNuRHlpUXZ2ZWlDT0l3PT0= -dn:uid=mqttuser0003,ou=testdevice,dc=emqx,dc=io -objectClass: top -objectClass: mqttUser -objectClass: mqttDevice -objectClass: mqttSecurity -uid: mqttuser0003 -isEnabled: TRUE -mqttPublishTopic: mqttuser0003/pub/1 -mqttPublishTopic: mqttuser0003/pub/+ -mqttPublishTopic: mqttuser0003/pub/# -mqttSubscriptionTopic: mqttuser0003/sub/1 -mqttSubscriptionTopic: mqttuser0003/sub/+ -mqttSubscriptionTopic: mqttuser0003/sub/# -mqttPubSubTopic: mqttuser0003/pubsub/1 -mqttPubSubTopic: mqttuser0003/pubsub/+ -mqttPubSubTopic: mqttuser0003/pubsub/# -userPassword:: e01ENX15YnNQR29hSzNuRHlpUXZ2ZWlDT0l3PT0= - -## create user mqttuser0004 -# password=mqttuser0004, -# passhash={MD5}2Br6pPDSEDIEvUlu9+s+MA== -# base64passhash=e01ENX0yQnI2cFBEU0VESUV2VWx1OStzK01BPT0= -dn:uid=mqttuser0004,ou=testdevice,dc=emqx,dc=io -objectClass: top -objectClass: mqttUser -objectClass: mqttDevice -objectClass: mqttSecurity -uid: mqttuser0004 -isEnabled: TRUE -mqttPublishTopic: mqttuser0004/pub/1 -mqttPublishTopic: mqttuser0004/pub/+ -mqttPublishTopic: mqttuser0004/pub/# -mqttSubscriptionTopic: mqttuser0004/sub/1 -mqttSubscriptionTopic: mqttuser0004/sub/+ -mqttSubscriptionTopic: mqttuser0004/sub/# -mqttPubSubTopic: mqttuser0004/pubsub/1 -mqttPubSubTopic: mqttuser0004/pubsub/+ -mqttPubSubTopic: mqttuser0004/pubsub/# -userPassword: {MD5}2Br6pPDSEDIEvUlu9+s+MA== - -## create user mqttuser0005 -# password=mqttuser0005, -# passhash={SHA}jKnxeEDGR14kE8AR7yuVFOelhz4= -# base64passhash=e1NIQX1qS254ZUVER1IxNGtFOEFSN3l1VkZPZWxoejQ9 -objectClass: top -dn:uid=mqttuser0005,ou=testdevice,dc=emqx,dc=io -objectClass: mqttUser -objectClass: mqttDevice -objectClass: mqttSecurity -uid: mqttuser0005 -isEnabled: TRUE -mqttPublishTopic: mqttuser0005/pub/1 -mqttPublishTopic: mqttuser0005/pub/+ -mqttPublishTopic: mqttuser0005/pub/# -mqttSubscriptionTopic: mqttuser0005/sub/1 -mqttSubscriptionTopic: mqttuser0005/sub/+ -mqttSubscriptionTopic: mqttuser0005/sub/# -mqttPubSubTopic: mqttuser0005/pubsub/1 -mqttPubSubTopic: mqttuser0005/pubsub/+ -mqttPubSubTopic: mqttuser0005/pubsub/# -userPassword: {SHA}jKnxeEDGR14kE8AR7yuVFOelhz4= - diff --git a/.ci/compatibility_tests/openldap/schema/emqx.schema b/.ci/compatibility_tests/openldap/schema/emqx.schema deleted file mode 100644 index 55f92269b..000000000 --- a/.ci/compatibility_tests/openldap/schema/emqx.schema +++ /dev/null @@ -1,46 +0,0 @@ -# -# Preliminary Apple OS X Native LDAP Schema -# This file is subject to change. -# -attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.1.3 NAME 'isEnabled' - EQUALITY booleanMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 - SINGLE-VALUE - USAGE userApplications ) - -attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4.1 NAME ( 'mqttPublishTopic' 'mpt' ) - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - USAGE userApplications ) -attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4.2 NAME ( 'mqttSubscriptionTopic' 'mst' ) - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - USAGE userApplications ) -attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4.3 NAME ( 'mqttPubSubTopic' 'mpst' ) - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - USAGE userApplications ) -attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4.4 NAME ( 'mqttAccountName' 'man' ) - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - USAGE userApplications ) - - -objectclass ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4 NAME 'mqttUser' - AUXILIARY - MAY ( mqttPublishTopic $ mqttSubscriptionTopic $ mqttPubSubTopic $ mqttAccountName) ) - -objectclass ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.2 NAME 'mqttDevice' - SUP top - STRUCTURAL - MUST ( uid ) - MAY ( isEnabled ) ) - -objectclass ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.3 NAME 'mqttSecurity' - SUP top - AUXILIARY - MAY ( userPassword $ userPKCS12 $ pwdAttribute $ pwdLockout ) ) diff --git a/.ci/compatibility_tests/pgsql/Dockerfile b/.ci/compatibility_tests/pgsql/Dockerfile new file mode 100644 index 000000000..ca44acffa --- /dev/null +++ b/.ci/compatibility_tests/pgsql/Dockerfile @@ -0,0 +1,12 @@ +ARG BUILD_FROM=postgres:11 +FROM ${BUILD_FROM} +ARG POSTGRES_USER=postgres +COPY --chown=$POSTGRES_USER .ci/compatibility_tests/pgsql/pg_hba.conf /var/lib/postgresql/pg_hba.conf +COPY --chown=$POSTGRES_USER apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.key /var/lib/postgresql/server.key +COPY --chown=$POSTGRES_USER apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.crt /var/lib/postgresql/server.crt +COPY --chown=$POSTGRES_USER apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt /var/lib/postgresql/root.crt +RUN chmod 600 /var/lib/postgresql/pg_hba.conf +RUN chmod 600 /var/lib/postgresql/server.key +RUN chmod 600 /var/lib/postgresql/server.crt +RUN chmod 600 /var/lib/postgresql/root.crt +EXPOSE 5432 diff --git a/.ci/compatibility_tests/pgsql/pg.conf b/.ci/compatibility_tests/pgsql/pg.conf deleted file mode 100644 index 7b78cd1e3..000000000 --- a/.ci/compatibility_tests/pgsql/pg.conf +++ /dev/null @@ -1,21 +0,0 @@ -# - Connection Settings - - -listen_addresses = '*' -port = 5432 # (change requires restart) -max_connections = 100 # (change requires restart) -# - SSL - - -ssl = on -ssl_cert_file = '/etc/postgresql/server-cert.pem' -ssl_key_file = '/etc/postgresql/server-key.pem' -shared_buffers = 128MB # min 128kB -checkpoint_timeout = 5min # range 30s-1d -max_wal_size = 1GB -min_wal_size = 80MB -datestyle = 'iso, mdy' -timezone = 'Etc/UTC' -lc_messages = 'en_US.utf8' # locale for system error message -lc_monetary = 'en_US.utf8' # locale for monetary formatting -lc_numeric = 'en_US.utf8' # locale for number formatting -lc_time = 'en_US.utf8' # locale for time formatting -default_text_search_config = 'pg_catalog.english' diff --git a/.ci/compatibility_tests/pgsql/pg_hba.conf b/.ci/compatibility_tests/pgsql/pg_hba.conf new file mode 100644 index 000000000..8b4f9b5a6 --- /dev/null +++ b/.ci/compatibility_tests/pgsql/pg_hba.conf @@ -0,0 +1,9 @@ +# TYPE DATABASE USER CIDR-ADDRESS METHOD +local all all trust +host all all 0.0.0.0/0 trust +host all all ::/0 trust +hostssl all all 0.0.0.0/0 cert +hostssl all all ::/0 cert + +hostssl all www-data 0.0.0.0/0 cert clientcert=1 +hostssl all postgres 0.0.0.0/0 cert clientcert=1 diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index 60a841525..e125590e7 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -30,9 +30,6 @@ jobs: env: LDAP_TAG: ${{ matrix.ldap_tag }} run: | - cp -f apps/emqx_auth_ldap/emqx.io.ldif .ci/apps_tests/emqx_ldap/schema - cp -f apps/emqx_auth_ldap/emqx.schema .ci/apps_tests/emqx_ldap/schema - cp -f apps/emqx_auth_ldap/test/certs/* .ci/apps_tests/emqx_ldap/certs docker-compose -f .ci/apps_tests/docker-compose.yaml build --no-cache docker-compose -f .ci/compatibility_tests/docker-compose-ldap.yaml up -d - name: setup @@ -83,11 +80,6 @@ jobs: echo 'auth.mongo.ssl.cacertfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf echo 'auth.mongo.ssl.certfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf echo 'auth.mongo.ssl.keyfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - - # echo 'auth.mongo.ssl = true' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - # echo 'auth.mongo.ssl_opts.cacertfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - # echo 'auth.mongo.ssl_opts.certfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - # echo 'auth.mongo.ssl_opts.keyfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - name: setup env: MONGO_TAG: ${{ matrix.mongo_tag }} @@ -138,7 +130,7 @@ jobs: run: | docker-compose -f .ci/compatibility_tests/docker-compose-mysql-tls.yaml up -d echo '\n' >> apps/emqx_auth_mongo/etc/emqx_auth_mysql.conf - echo 'auth.mysql.ssl = on' >> apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + sed -i '/auth.mysql.ssl = off/c auth.mysql.ssl = on' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf echo "auth.mysql.ssl.cafile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem" >> apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf echo "auth.mysql.ssl.certfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem" >> apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf echo "auth.mysql.ssl.keyfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem" >> apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf @@ -170,7 +162,7 @@ jobs: pgsql: runs-on: ubuntu-20.04 - + strategy: matrix: pgsql_tag: @@ -183,9 +175,8 @@ jobs: - ipv4 - ipv6 connect_type: - # - tls + - tls - tcp - steps: - uses: actions/checkout@v1 - name: setup @@ -193,12 +184,14 @@ jobs: PGSQL_TAG: ${{ matrix.pgsql_tag }} if: matrix.connect_type == 'tls' run: | + docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml build --no-cache docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml up -d + sed -i 's|auth.pgsql.username[ \t]*=.*|auth.pgsql.username = postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i 's|auth.pgsql.password[ \t]*=.*|auth.pgsql.password = postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i 's|auth.pgsql.database[ \t]*=.*|auth.pgsql.database= postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i 's|auth.pgsql.ssl[ \t]*=.*|auth.pgsql.ssl = on|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf echo '\n' >> apps/emqx_auth_mongo/etc/emqx_auth_pgsql.conf - echo 'auth.pgsql.ssl = true' >> apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - echo "auth.pgsql.ssl_opts.cacertfile = /emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem" >> apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - echo "auth.pgsql.ssl_opts.certfile = /emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-cert.pem" >> apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - echo "auth.pgsql.ssl_opts.keyfile = /emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-key.pem" >> apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + echo 'auth.pgsql.ssl.cacertfile = /emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - name: setup env: PGSQL_TAG: ${{ matrix.pgsql_tag }} diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 41d224f48..13d911928 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -26,9 +26,6 @@ jobs: PGSQL_TAG: 13 LDAP_TAG: 2.4.50 run: | - cp -f apps/emqx_auth_ldap/emqx.io.ldif .ci/apps_tests/emqx_ldap/schema - cp -f apps/emqx_auth_ldap/emqx.schema .ci/apps_tests/emqx_ldap/schema - cp -f apps/emqx_auth_ldap/test/certs/* .ci/apps_tests/emqx_ldap/certs docker-compose -f .ci/apps_tests/docker-compose.yaml build --no-cache docker-compose -f .ci/apps_tests/docker-compose.yaml up -d - name: set config files diff --git a/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE.erl b/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE.erl index 044655ac1..62b8a821c 100644 --- a/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE.erl +++ b/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE.erl @@ -164,32 +164,22 @@ t_check_auth(_) -> BcryptFoo = #{clientid => <<"bcrypt_foo">>, username => <<"bcrypt_foo">>, zone => external}, User1 = #{clientid => <<"bcrypt_foo">>, username => <<"user">>, zone => external}, Bcrypt = #{clientid => <<"bcrypt">>, username => <<"bcrypt">>, zone => external}, - BcryptWrong = #{clientid => <<"bcrypt_wrong">>, username => <<"bcrypt_wrong">>, zone => external}, + % reload([{password_hash, plain}]), - {ok,#{is_superuser := true}} = - emqx_access_control:authenticate(Plain#{password => <<"plain">>}), + {ok, #{is_superuser := true}} = emqx_access_control:authenticate(Plain#{password => <<"plain">>}), reload([{password_hash, md5}]), - {ok,#{is_superuser := false}} = - emqx_access_control:authenticate(Md5#{password => <<"md5">>}), + {ok, #{is_superuser := false}} = emqx_access_control:authenticate(Md5#{password => <<"md5">>}), reload([{password_hash, sha}]), - {ok,#{is_superuser := false}} = - emqx_access_control:authenticate(Sha#{password => <<"sha">>}), + {ok, #{is_superuser := false}} = emqx_access_control:authenticate(Sha#{password => <<"sha">>}), reload([{password_hash, sha256}]), - {ok,#{is_superuser := false}} = - emqx_access_control:authenticate(Sha256#{password => <<"sha256">>}), + {ok, #{is_superuser := false}} = emqx_access_control:authenticate(Sha256#{password => <<"sha256">>}), reload([{password_hash, bcrypt}]), - {ok,#{is_superuser := false}} = - emqx_access_control:authenticate(Bcrypt#{password => <<"password">>}), - {error, not_authorized} = - emqx_access_control:authenticate(BcryptWrong#{password => <<"password">>}), - %%pbkdf2 sha - reload([{password_hash, {pbkdf2, sha, 1, 16}}, - {auth_query, "select password, salt from mqtt_user where username = '%u' limit 1"}]), - {ok,#{is_superuser := false}} = - emqx_access_control:authenticate(Pbkdf2#{password => <<"password">>}), + {ok, #{is_superuser := false}} = emqx_access_control:authenticate(Bcrypt#{password => <<"password">>}), + + reload([{password_hash, {pbkdf2, sha, 1, 16}}, {auth_query, "select password, salt from mqtt_user where username = '%u' limit 1"}]), + {ok, #{is_superuser := false}} = emqx_access_control:authenticate(Pbkdf2#{password => <<"password">>}), reload([{password_hash, {salt, bcrypt}}]), - {ok,#{is_superuser := false}} = - emqx_access_control:authenticate(BcryptFoo#{password => <<"foo">>}), + {ok, #{is_superuser := false}} = emqx_access_control:authenticate(BcryptFoo#{password => <<"foo">>}), {error, _} = emqx_access_control:authenticate(User1#{password => <<"foo">>}), {error, not_authorized} = emqx_access_control:authenticate(Bcrypt#{password => <<"password">>}). diff --git a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf index c3c6e2800..603a91908 100644 --- a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf +++ b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf @@ -22,7 +22,7 @@ auth.pgsql.username = root ## PostgreSQL password. ## ## Value: String -## auth.pgsql.password = +auth.pgsql.password = public ## PostgreSQL database. ## diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl index e3f849153..230166a62 100644 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl @@ -16,7 +16,6 @@ -module(emqx_auth_pgsql_SUITE). --compile(nowarn_export_all). -compile(export_all). -define(POOL, emqx_auth_pgsql). @@ -30,9 +29,9 @@ -include_lib("common_test/include/ct.hrl"). %%setp1 init table --define(DROP_ACL_TABLE, "DROP TABLE IF EXISTS mqtt_acl_test"). +-define(DROP_ACL_TABLE, "DROP TABLE IF EXISTS mqtt_acl"). --define(CREATE_ACL_TABLE, "CREATE TABLE mqtt_acl_test ( +-define(CREATE_ACL_TABLE, "CREATE TABLE mqtt_acl ( id SERIAL primary key, allow integer, ipaddr character varying(60), @@ -41,23 +40,23 @@ access integer, topic character varying(100))"). --define(INIT_ACL, "INSERT INTO mqtt_acl_test (id, allow, ipaddr, username, clientid, access, topic) +-define(INIT_ACL, "INSERT INTO mqtt_acl (id, allow, ipaddr, username, clientid, access, topic) VALUES (1,1,'127.0.0.1','u1','c1',1,'t1'), (2,0,'127.0.0.1','u2','c2',1,'t1'), (3,1,'10.10.0.110','u1','c1',1,'t1'), (4,1,'127.0.0.1','u3','c3',3,'t1')"). --define(DROP_AUTH_TABLE, "DROP TABLE IF EXISTS mqtt_user_test"). +-define(DROP_AUTH_TABLE, "DROP TABLE IF EXISTS mqtt_user"). --define(CREATE_AUTH_TABLE, "CREATE TABLE mqtt_user_test ( +-define(CREATE_AUTH_TABLE, "CREATE TABLE mqtt_user ( id SERIAL primary key, is_superuser boolean, username character varying(100), password character varying(100), salt character varying(40))"). --define(INIT_AUTH, "INSERT INTO mqtt_user_test (id, is_superuser, username, password, salt) +-define(INIT_AUTH, "INSERT INTO mqtt_user (id, is_superuser, username, password, salt) VALUES (1, true, 'plain', 'plain', 'salt'), (2, false, 'md5', '1bc29b36f623ba82aaf6724fd3b16718', 'salt'), @@ -68,61 +67,25 @@ (7, false, 'bcrypt', '$2y$16$rEVsDarhgHYB0TGnDFJzyu5f.T.Ha9iXMTk9J36NCMWWM7O16qyaK', 'salt')"). all() -> - [{group, ssl}, {group, nossl}]. + emqx_ct:all(?MODULE). -groups() -> - Cases = emqx_ct:all(?MODULE), - [{ssl, [sequence], Cases}, {nossl, [sequence], Cases}]. - -init_per_group(Name, Config) -> - case Name of - ssl -> - emqx_ct_helpers:start_apps([emqx_auth_pgsql], fun set_special_configs_ssl/1); - nossl -> - emqx_ct_helpers:start_apps([emqx_auth_pgsql], fun set_special_configs/1) - end, - init_auth_(), - init_acl_(), +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_auth_pgsql]), + drop_acl(), + init_auth(), + init_auth(), + init_acl(), + set_special_configs(), Config. -end_per_group(_, Config) -> - drop_auth_(), - drop_acl_(), +end_per_suite(Config) -> emqx_ct_helpers:stop_apps([emqx_auth_pgsql]), Config. -set_special_configs_ssl(Name) -> - Server = application:get_env(?APP, server, []), - Path = emqx_ct_helpers:deps_path(emqx_auth_pgsql, "test/emqx_auth_pgsql_SUITE_data/"), - Sslopts = [{keyfile, Path ++ "/client-key.pem"}, - {certfile, Path ++ "/client-cert.pem"}, - {cacertfile, Path ++ "/ca.pem"}], - Temp = lists:keyreplace(ssl, 1, Server, {ssl, true}), - application:set_env(?APP, server, Temp), - application:set_env(?APP, server, lists:keyreplace(ssl_opts, 1, Temp, {ssl_opts, Sslopts})), - set_special_configs(Name). - -set_special_configs(emqx) -> +set_special_configs() -> application:set_env(emqx, acl_nomatch, deny), - application:set_env(emqx, acl_file, - emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/acl.conf")), application:set_env(emqx, allow_anonymous, false), - application:set_env(emqx, enable_acl_cache, false), - application:set_env(emqx, plugins_loaded_file, - emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins")); - -set_special_configs(emqx_auth_pgsql) -> - Server = application:get_env(?APP, server, []), - application:set_env(?APP, server, - lists:keyreplace(password, - 1, - lists:keyreplace(pool_size, 1, Server, {pool_size, 1}), - {password, "public"})), - application:set_env(?APP, acl_query, "select allow, ipaddr, username, clientid, access, topic from mqtt_acl_test where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'"), - application:set_env(?APP, super_query, "select is_superuser from mqtt_user_test where username = '%u' limit 1"), - application:set_env(?APP, auth_query, "select password from mqtt_user_test where username = '%u' limit 1"); -set_special_configs(_App) -> - ok. + application:set_env(emqx, enable_acl_cache, false). t_comment_config(_) -> AuthCount = length(emqx_hooks:lookup('client.authenticate')), @@ -134,31 +97,6 @@ t_comment_config(_) -> ?assertEqual(AuthCount - 1, length(emqx_hooks:lookup('client.authenticate'))), ?assertEqual(AclCount - 1, length(emqx_hooks:lookup('client.check_acl'))). -t_placeholders(_) -> - ClientA = #{username => <<"plain">>, clientid => <<"plain">>, zone => external}, - reload([{password_hash, plain}, - {auth_query, "select password from mqtt_user_test where username = '%u' and 'a_cn_val' = '%C' limit 1"}]), - {error, not_authorized} = - emqx_access_control:authenticate(ClientA#{password => <<"plain">>}), - {error, not_authorized} = - emqx_access_control:authenticate(ClientA#{password => <<"plain">>, cn => undefined}), - {ok, _} = - emqx_access_control:authenticate(ClientA#{password => <<"plain">>, cn => <<"a_cn_val">>}), - - reload([{auth_query, "select password from mqtt_user_test where username = '%c' and 'a_dn_val' = '%d' limit 1"}]), - {error, not_authorized} = - emqx_access_control:authenticate(ClientA#{password => <<"plain">>}), - {error, not_authorized} = - emqx_access_control:authenticate(ClientA#{password => <<"plain">>, dn => undefined}), - {ok, _} = - emqx_access_control:authenticate(ClientA#{password => <<"plain">>, dn => <<"a_dn_val">>}), - - reload([{auth_query, "select password from mqtt_user_test where username = '%u' and '192.168.1.5' = '%a' limit 1"}]), - {error, not_authorized} = - emqx_access_control:authenticate(ClientA#{password => <<"plain">>}), - {ok, _} = - emqx_access_control:authenticate(ClientA#{password => <<"plain">>, peerhost => {192,168,1,5}}). - t_check_auth(_) -> Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external}, Md5 = #{clientid => <<"md5">>, username => <<"md5">>, zone => external}, @@ -168,8 +106,8 @@ t_check_auth(_) -> BcryptFoo = #{clientid => <<"bcrypt_foo">>, username => <<"bcrypt_foo">>, zone => external}, User1 = #{clientid => <<"bcrypt_foo">>, username => <<"user">>, zone => external}, Bcrypt = #{clientid => <<"bcrypt">>, username => <<"bcrypt">>, zone => external}, - reload([{password_hash, plain}, - {auth_query, "select password from mqtt_user_test where username = '%u' limit 1"}]), + % + reload([{password_hash, plain}]), {ok, #{is_superuser := true}} = emqx_access_control:authenticate(Plain#{password => <<"plain">>}), reload([{password_hash, md5}]), {ok, #{is_superuser := false}} = emqx_access_control:authenticate(Md5#{password => <<"md5">>}), @@ -179,18 +117,18 @@ t_check_auth(_) -> {ok, #{is_superuser := false}} = emqx_access_control:authenticate(Sha256#{password => <<"sha256">>}), reload([{password_hash, bcrypt}]), {ok, #{is_superuser := false}} = emqx_access_control:authenticate(Bcrypt#{password => <<"password">>}), - %%pbkdf2 sha - reload([{password_hash, {pbkdf2, sha, 1, 16}}, {auth_query, "select password, salt from mqtt_user_test where username = '%u' limit 1"}]), + + reload([{password_hash, {pbkdf2, sha, 1, 16}}, {auth_query, "select password, salt from mqtt_user where username = '%u' limit 1"}]), {ok, #{is_superuser := false}} = emqx_access_control:authenticate(Pbkdf2#{password => <<"password">>}), reload([{password_hash, {salt, bcrypt}}]), {ok, #{is_superuser := false}} = emqx_access_control:authenticate(BcryptFoo#{password => <<"foo">>}), {error, _} = emqx_access_control:authenticate(User1#{password => <<"foo">>}), {error, not_authorized} = emqx_access_control:authenticate(Bcrypt#{password => <<"password">>}). + t_check_acl(_) -> emqx_modules:load_module(emqx_mod_acl_internal, false), User1 = #{zone => external, peerhost => {127,0,0,1}, clientid => <<"c1">>, username => <<"u1">>}, User2 = #{zone => external, peerhost => {127,0,0,1}, clientid => <<"c2">>, username => <<"u2">>}, - reload([{acl_query, "select allow, ipaddr, username, clientid, access, topic from mqtt_acl_test where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'"}]), allow = emqx_access_control:check_acl(User1, subscribe, <<"t1">>), deny = emqx_access_control:check_acl(User2, subscribe, <<"t1">>), User3 = #{zone => external, peerhost => {10,10,0,110}, clientid => <<"c1">>, username => <<"u1">>}, @@ -204,7 +142,7 @@ t_check_acl(_) -> allow = emqx_access_control:check_acl(User5, publish, <<"t1">>). t_acl_super(_) -> - reload([{password_hash, plain}, {auth_query, "select password from mqtt_user_test where username = '%u' limit 1"}]), + reload([{password_hash, plain}, {auth_query, "select password from mqtt_user where username = '%u' limit 1"}]), {ok, C} = emqtt:start_link([{host, "localhost"}, {clientid, <<"simpleClient">>}, {username, <<"plain">>}, {password, <<"plain">>}]), {ok, _} = emqtt:connect(C), @@ -227,22 +165,22 @@ reload(Config) when is_list(Config) -> [application:set_env(?APP, K, V) || {K, V} <- Config], application:start(?APP). -init_acl_() -> +init_acl() -> {ok, Pid} = ecpool_worker:client(gproc_pool:pick_worker({ecpool, ?POOL})), {ok, [], []} = epgsql:squery(Pid, ?DROP_ACL_TABLE), {ok, [], []} = epgsql:squery(Pid, ?CREATE_ACL_TABLE), {ok, _} = epgsql:equery(Pid, ?INIT_ACL). -drop_acl_() -> +drop_acl() -> {ok, Pid} = ecpool_worker:client(gproc_pool:pick_worker({ecpool, ?POOL})), {ok, [], []}= epgsql:squery(Pid, ?DROP_ACL_TABLE). -init_auth_() -> +init_auth() -> {ok, Pid} = ecpool_worker:client(gproc_pool:pick_worker({ecpool, ?POOL})), {ok, [], []} = epgsql:squery(Pid, ?DROP_AUTH_TABLE), {ok, [], []} = epgsql:squery(Pid, ?CREATE_AUTH_TABLE), {ok, _} = epgsql:equery(Pid, ?INIT_AUTH). -drop_auth_() -> +drop_auth() -> {ok, Pid} = ecpool_worker:client(gproc_pool:pick_worker({ecpool, ?POOL})), - {ok, [], []} = epgsql:squery(Pid, ?DROP_AUTH_TABLE). + {ok, [], []} = epgsql:squery(Pid, ?DROP_AUTH_TABLE). \ No newline at end of file diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem deleted file mode 100644 index 00b31d8a4..000000000 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDAzCCAeugAwIBAgIBATANBgkqhkiG9w0BAQsFADA8MTowOAYDVQQDDDFNeVNR -TF9TZXJ2ZXJfOC4wLjE5X0F1dG9fR2VuZXJhdGVkX0NBX0NlcnRpZmljYXRlMB4X -DTIwMDYxMTAzMzg0NloXDTMwMDYwOTAzMzg0NlowPDE6MDgGA1UEAwwxTXlTUUxf -U2VydmVyXzguMC4xOV9BdXRvX0dlbmVyYXRlZF9DQV9DZXJ0aWZpY2F0ZTCCASIw -DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANJBlAYvTQ6euY4HcSn4syH7kq9s -KcG+OMjPUrj+KFEElCzgNuIhaS0f3ORQGB1PNcvVcfdXUI3WX332gWbr9s1b7Xl1 -JKJfDXs+26Cm6NhONTE3sPHnbTSmQEFb52hwAtjQmcY3IQs1AgxKFFHJfnCBEWfE -ePBQaiuYk1XDESMdWpMLrPnYQaj9MpAOUxjlmZCayzPWlF0j0IWvfsF5TqZL7tFK -9p5F/DzyZ4n1mqPVEoUmq5ZdSKj2TQkpWTMHBWHEDQQqXbyE1FGJR7zEUFeuG1KT -sVBg7iZEC93SygZTbgUZSQXIwQCsO6xZ8MB2XDJkPbWp/3Wc6c8I6P09F48CAwEA -AaMQMA4wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEADKz6bIpP5anp -GgLB0jkclRWuMlS4qqIt4itSsMXPJ/ezpHwECixmgW2TIQl6S1woRkUeMxhT2/Ay -Sn/7aKxuzRagyE5NEGOvrOuAP5RO2ZdNJ/X3/Rh533fK1sOTEEbSsWUvW6iSkZef -rsfZBVP32xBhRWkKRdLeLB4W99ADMa0IrTmZPCXHSSE2V4e1o6zWLXcOZeH1Qh8N -SkelBweR+8r1Fbvy1r3s7eH7DCbYoGEDVLQGOLvzHKBisQHmoDnnF5E9g1eeNRdg -o+vhOKfYCOzeNREJIqS42PHcGhdNRk90ycigPmfUJclz1mDHoMjKR2S5oosTpr65 -tNPx3CL7GA== ------END CERTIFICATE----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-cert.pem b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-cert.pem deleted file mode 100644 index aad1404ca..000000000 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-cert.pem +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDBDCCAeygAwIBAgIBAzANBgkqhkiG9w0BAQsFADA8MTowOAYDVQQDDDFNeVNR -TF9TZXJ2ZXJfOC4wLjE5X0F1dG9fR2VuZXJhdGVkX0NBX0NlcnRpZmljYXRlMB4X -DTIwMDYxMTAzMzg0N1oXDTMwMDYwOTAzMzg0N1owQDE+MDwGA1UEAww1TXlTUUxf -U2VydmVyXzguMC4xOV9BdXRvX0dlbmVyYXRlZF9DbGllbnRfQ2VydGlmaWNhdGUw -ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDVYSWpOvCTupz82fc85Opv -EQ7rkB8X2oOMyBCpkyHKBIr1ZQgRDWBp9UVOASq3GnSElm6+T3Kb1QbOffa8GIlw -sjAueKdq5L2eSkmPIEQ7eoO5kEW+4V866hE1LeL/PmHg2lGP0iqZiJYtElhHNQO8 -3y9I7cm3xWMAA3SSWikVtpJRn3qIp2QSrH+tK+/HHbE5QwtPxdir4ULSCSOaM5Yh -Wi5Oto88TZqe1v7SXC864JVvO4LuS7TuSreCdWZyPXTJFBFeCEWSAxonKZrqHbBe -CwKML6/0NuzjaQ51c2tzmVI6xpHj3nnu4cSRx6Jf9WBm+35vm0wk4pohX3ptdzeV -AgMBAAGjDTALMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEBAByQ5zSNeFUH -Aw7JlpZHtHaSEeiiyBHke20ziQ07BK1yi/ms2HAWwQkpZv149sjNuIRH8pkTmkZn -g8PDzSefjLbC9AsWpWV0XNV22T/cdobqLqMBDDZ2+5bsV+jTrOigWd9/AHVZ93PP -IJN8HJn6rtvo2l1bh/CdsX14uVSdofXnuWGabNTydqtMvmCerZsdf6qKqLL+PYwm -RDpgWiRUY7KPBSSlKm/9lJzA+bOe4dHeJzxWFVCJcbpoiTFs1je1V8kKQaHtuW39 -ifX6LTKUMlwEECCbDKM8Yq2tm8NjkjCcnFDtKg8zKGPUu+jrFMN5otiC3wnKcP7r -O9EkaPcgYH8= ------END CERTIFICATE----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-key.pem b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-key.pem deleted file mode 100644 index 6789d0291..000000000 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-key.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEA1WElqTrwk7qc/Nn3POTqbxEO65AfF9qDjMgQqZMhygSK9WUI -EQ1gafVFTgEqtxp0hJZuvk9ym9UGzn32vBiJcLIwLninauS9nkpJjyBEO3qDuZBF -vuFfOuoRNS3i/z5h4NpRj9IqmYiWLRJYRzUDvN8vSO3Jt8VjAAN0klopFbaSUZ96 -iKdkEqx/rSvvxx2xOUMLT8XYq+FC0gkjmjOWIVouTraPPE2antb+0lwvOuCVbzuC -7ku07kq3gnVmcj10yRQRXghFkgMaJyma6h2wXgsCjC+v9Dbs42kOdXNrc5lSOsaR -49557uHEkceiX/VgZvt+b5tMJOKaIV96bXc3lQIDAQABAoIBAF7yjXmSOn7h6P0y -WCuGiTLG2mbDiLJqj2LTm2Z5i+2Cu/qZ7E76Ls63TxF4v3MemH5vGfQhEhR5ZD/6 -GRJ1sKKvB3WGRqjwA9gtojHH39S/nWGy6vYW/vMOOH37XyjIr3EIdIaUtFQBTSHd -Kd71niYrAbVn6fyWHolhADwnVmTMOl5OOAhCdEF4GN3b5aIhIu8BJ7EUzTtHBJIj -CAEfjZFjDs1y1cIgGFJkuIQxMfCpq5recU2qwip7YO6fk//WEjOPu7kSf5IEswL8 -jg1dea9rGBV6KaD2xsgsC6Ll6Sb4BbsrHMfflG3K2Lk3RdVqqTFp1Fn1PTLQE/1S -S/SZPYECgYEA9qYcHKHd0+Q5Ty5wgpxKGa4UCWkpwvfvyv4bh8qlmxueB+l2AIdo -ZvkM8gTPagPQ3WypAyC2b9iQu70uOJo1NizTtKnpjDdN1YpDjISJuS/P0x73gZwy -gmoM5AzMtN4D6IbxXtXnPaYICvwLKU80ouEN5ZPM4/ODLUu6gsp0v2UCgYEA3Xgi -zMC4JF0vEKEaK0H6QstaoXUmw/lToZGH3TEojBIkb/2LrHUclygtONh9kJSFb89/ -jbmRRLAOrx3HZKCNGUmF4H9k5OQyAIv6OGBinvLGqcbqnyNlI+Le8zxySYwKMlEj -EMrBCLmSyi0CGFrbZ3mlj/oCET/ql9rNvcK+DHECgYAEx5dH3sMjtgp+RFId1dWB -xePRgt4yTwewkVgLO5wV82UOljGZNQaK6Eyd7AXw8f38LHzh+KJQbIvxd2sL4cEi -OaAoohpKg0/Y0YMZl//rPMf0OWdmdZZs/I0fZjgZUSwWN3c59T8z7KG/RL8an9RP -S7kvN7wCttdV61/D5RR6GQKBgDxCe/WKWpBKaovzydMLWLTj7/0Oi0W3iXHkzzr4 -LTgvl4qBSofaNbVLUUKuZTv5rXUG2IYPf99YqCYtzBstNDc1MiAriaBeFtzfOW4t -i6gEFtoLLbuvPc3N5Sv5vn8Ug5G9UfU3td5R4AbyyCcoUZqOFuZd+EIJSiOXfXOs -kVmBAoGBAIU9aPAqhU5LX902oq8KsrpdySONqv5mtoStvl3wo95WIqXNEsFY60wO -q02jKQmJJ2MqhkJm2EoF2Mq8+40EZ5sz8LdgeQ/M0yQ9lAhPi4rftwhpe55Ma9dk -SE9X1c/DMCBEaIjJqVXdy0/EeArwpb8sHkguVVAZUWxzD+phm1gs ------END RSA PRIVATE KEY----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/pg.conf b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/pg.conf deleted file mode 100644 index 7b78cd1e3..000000000 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/pg.conf +++ /dev/null @@ -1,21 +0,0 @@ -# - Connection Settings - - -listen_addresses = '*' -port = 5432 # (change requires restart) -max_connections = 100 # (change requires restart) -# - SSL - - -ssl = on -ssl_cert_file = '/etc/postgresql/server-cert.pem' -ssl_key_file = '/etc/postgresql/server-key.pem' -shared_buffers = 128MB # min 128kB -checkpoint_timeout = 5min # range 30s-1d -max_wal_size = 1GB -min_wal_size = 80MB -datestyle = 'iso, mdy' -timezone = 'Etc/UTC' -lc_messages = 'en_US.utf8' # locale for system error message -lc_monetary = 'en_US.utf8' # locale for monetary formatting -lc_numeric = 'en_US.utf8' # locale for number formatting -lc_time = 'en_US.utf8' # locale for time formatting -default_text_search_config = 'pg_catalog.english' diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.crt b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.crt new file mode 100644 index 000000000..9867681b9 --- /dev/null +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDYzCCAksCCQC7J1oPkDz7vTANBgkqhkiG9w0BAQUFADCBhTELMAkGA1UEBhMC +Q0ExGTAXBgNVBAgMEEJyaXRpc2ggQ29sdW1iaWExDjAMBgNVBAcMBUNvbW94MRQw +EgYDVQQKDAtUaGVCcmFpbi5jYTEUMBIGA1UEAwwLdGhlYnJhaW4uY2ExHzAdBgkq +hkiG9w0BCQEWEGluZm9AdGhlYnJhaW4uY2EwHhcNMjEwMTEzMDkwNzM2WhcNMjEw +MjEyMDkwNzM2WjBhMQswCQYDVQQGEwJDQTEZMBcGA1UECAwQQnJpdGlzaCBDb2x1 +bWJpYTEOMAwGA1UEBwwFQ29tb3gxFDASBgNVBAoMC1RoZUJyYWluLmNhMREwDwYD +VQQDDAh3d3ctZGF0YTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJv9 +yO5JGKBl+7w0HGkRDIPZ5Ku3lIAzB4ThszRHBqll7VjlTz+q16OQOONqeHBuxPjj +11WMXD2KnfYZW2ZWd0U8FKzuIGOCStGbSUi2hC0owp+KkJcDujfIafXQnAa0fUiS +FBB5iG98vm3QI4gv9135LgnO5oHopH6oZ/t0Id1LzFhp2sdhebdtczmImpo+nt7v +fduapptuIJ20ThdAvo3MlYoAhivsvJKntlWPAwPMQdyezww/q7T5Y8DCyJJTydr5 +PrMz9S/WQTkj/G0y4dZgQonG5r0d1Nf+rwkn78DdXGktVDMBBP41+VWnEDBCTlgS +FjQEY6Izaof8s8q8K2UCAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAdlAQkumOAKbQ +SW5gtkHgKyIQyfwk9maKqKccK04WlNk1t1jsvk7kaOEHr3t7YG28yKqicGHAcfFf +i/RU51v2GJVzWCbzkAAH/zNgDcYnYk6sn54YcuBzrPliVH1xxmZy/52+huTxy8Vd +3nmCjdYR/I764rd8gkRK+aHaUTLyitzX1kW90LtXonKY72CNZVXHEBom3XM/a6ff +ilybDloNVTfHstnfsnHHyNYn0SfapqXxPCO+FL9hQjlztUBZryRdS0nq66hB2GSB +CEst/vtNGo/2aa1Vw4bKl2oGepjKNzxp0ZTTVuIcwGzV6oKIsx1ZnWE3gQLEH/TX +dzMzesBayA== +-----END CERTIFICATE----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.csr b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.csr new file mode 100644 index 000000000..325fbe397 --- /dev/null +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.csr @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICpjCCAY4CAQAwYTELMAkGA1UEBhMCQ0ExGTAXBgNVBAgMEEJyaXRpc2ggQ29s +dW1iaWExDjAMBgNVBAcMBUNvbW94MRQwEgYDVQQKDAtUaGVCcmFpbi5jYTERMA8G +A1UEAwwId3d3LWRhdGEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCb +/cjuSRigZfu8NBxpEQyD2eSrt5SAMweE4bM0RwapZe1Y5U8/qtejkDjjanhwbsT4 +49dVjFw9ip32GVtmVndFPBSs7iBjgkrRm0lItoQtKMKfipCXA7o3yGn10JwGtH1I +khQQeYhvfL5t0COIL/dd+S4JzuaB6KR+qGf7dCHdS8xYadrHYXm3bXM5iJqaPp7e +733bmqabbiCdtE4XQL6NzJWKAIYr7LySp7ZVjwMDzEHcns8MP6u0+WPAwsiSU8na ++T6zM/Uv1kE5I/xtMuHWYEKJxua9HdTX/q8JJ+/A3VxpLVQzAQT+NflVpxAwQk5Y +EhY0BGOiM2qH/LPKvCtlAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEAN6Q8MEDx +g5xlpYB/fFmagpe15+G2QbqVf2mH1a4aBcBns4jMMqNidi4gyjGfzvNxX77R6KcI +AfcxENRVDYJbhAgEQ96jv4jv5pEMuyvQ8VLhn9AOXCaK/VHxbYlOiM7tfFtEDrrB +wTn8FvoEwjehfsSX2dWiwcUK4SPPeuklE/EGjRgoVCwg8EqWzf1fn+tzME8OpnRQ +I8coyALF6ANehvP7ADV3m5iOOaNhfnqmqGBEwjB3TTvE1gZ4UvAyl75bi+Zh3Osn +qemyxocp/ML4o6d/F+nKIZOe6309V2nyrY6RSd2fBCrhYj2rKTbrGTZrpKXeAhtI +jMivnjCK+WNHpQ== +-----END CERTIFICATE REQUEST----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.key b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.key new file mode 100644 index 000000000..787246f6f --- /dev/null +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAm/3I7kkYoGX7vDQcaREMg9nkq7eUgDMHhOGzNEcGqWXtWOVP +P6rXo5A442p4cG7E+OPXVYxcPYqd9hlbZlZ3RTwUrO4gY4JK0ZtJSLaELSjCn4qQ +lwO6N8hp9dCcBrR9SJIUEHmIb3y+bdAjiC/3XfkuCc7mgeikfqhn+3Qh3UvMWGna +x2F5t21zOYiamj6e3u9925qmm24gnbROF0C+jcyVigCGK+y8kqe2VY8DA8xB3J7P +DD+rtPljwMLIklPJ2vk+szP1L9ZBOSP8bTLh1mBCicbmvR3U1/6vCSfvwN1caS1U +MwEE/jX5VacQMEJOWBIWNARjojNqh/yzyrwrZQIDAQABAoIBAAOicycSLu+10Jq/ +ABZ2njsIPaq+mUgvaDJxa9KBASe7Rz92AFW0blfSSXELDwlXm2FNNbw5jACnFS0h +xB5rT1Yeo0CwP7Lx2zptCtUV45iFxZsgCGRsYs9f7RAcLzZ8yBqDxNHpcwNd/bXj +TqCitXnMD4WM+5P1TrfgxqN2Pj/Atg8w/4dP7KcFcTzcZzIz5rr3NTyjsrLdiFis +sR+7m7Qu4PyEfrDpR9Np111nQqVJ1bpt9qt/hv318FaBnpNY6MMBaSni99mvMXSd +SwHn3gnfHREWcNSLGA9gjEQmyIPHpV9T6SJ/zyr++6y8QCq4DiSP36A9zeA1XThP +YEIsWxUCgYEAyLppQerpOT2CnbTbKO/9rGwlbf8FT2GWFcPBtUm0lp21/C32BX+H +jNCmQsE1pZ6+sqv2mb1onr6Xl9cSEt6KsI1EJtFFR9Lnvqqu+JKo31U94z2yTqgv +sc+qMl7shy1kja8T5NaRc++UkCVzVNsnFB9torIaqQwY9IRdRwmYjisCgYEAxvHR +MwvWpOg25zz75OfupIOQhj9W6yphpY5/yoYBms/4OeabJhMrOV142s9souCHmuGU +EtzOQC5jbEc+3MUjx1ZlboHY7UuoEu87kykFEs9mnaD+T34PEAJcQjSzqzS5KMJE +Ro275xf+V/e3hS/Z3hQXmDQNQDNRYMcAZfTW9K8CgYBkHITOuYikYcc5PLBplHhi +fHWWjLBrTPJ73GxKLH6C+BmBsrKXP2mtk4q4lIBbH/dgSV/ugYciVVBqDHwZKSDm +uS4aZhk1nzyx3ZLyqsLK0ErTgTvi+wL+neH2yV0SdlNGTuGPKmzU89KWqfcBhWPS +J3KYyFd/pGb13OZgvap2jQKBgBXCXR84LEHdJCQmh2aB95gGy8fjJZ6TBBsXeuKr +xYEpPf0XO+DuN8wObSmBhmBKLorCIW/utqBOcpFlOXrsFP24dV+g1BkgLUHk6J8v +3V4xUQfsk+Qd5YfaujyDhyMyoQ3UMaOF3QdpmGgGsAvhL/MaP3pmNwzOkBgFrAV6 +wggBAoGBAMflqy2pfqGhaj9S6qZ3K95h7NdCUikdQzqmgbNtOHaZ2kHByyYtOPLB +1VnuDRQiacmum+fTZa6wNmvp2FWg+uxI/aspfF6SdPfGpyPrG5D+ITtqKF2xieK+ +XpzehKTrTuYQRAVhmWbhpuyahYnQyd/MrsCMGzUfAJtM7l5vKa2O +-----END RSA PRIVATE KEY----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt new file mode 100644 index 000000000..46b1e2a7a --- /dev/null +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDiDCCAnACCQCCsPcIlZO4TDANBgkqhkiG9w0BAQsFADCBhTELMAkGA1UEBhMC +Q0ExGTAXBgNVBAgMEEJyaXRpc2ggQ29sdW1iaWExDjAMBgNVBAcMBUNvbW94MRQw +EgYDVQQKDAtUaGVCcmFpbi5jYTEUMBIGA1UEAwwLdGhlYnJhaW4uY2ExHzAdBgkq +hkiG9w0BCQEWEGluZm9AdGhlYnJhaW4uY2EwHhcNMjEwMTEzMDkwNDIyWhcNMzEw +MTExMDkwNDIyWjCBhTELMAkGA1UEBhMCQ0ExGTAXBgNVBAgMEEJyaXRpc2ggQ29s +dW1iaWExDjAMBgNVBAcMBUNvbW94MRQwEgYDVQQKDAtUaGVCcmFpbi5jYTEUMBIG +A1UEAwwLdGhlYnJhaW4uY2ExHzAdBgkqhkiG9w0BCQEWEGluZm9AdGhlYnJhaW4u +Y2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC2YWuwplM2Hc5tzBMu +covW9nwZ8iNEFo5pbDc8710pmnkF+wsDztLy4afJe6OeVHyCgQxmE+rTZcoWbvoh +pxW3Zy/8es4My07RKHqI3NYadThUvDsmI10cF3tJbhOZaIrMaExLGookZYKwbNAy +7yJ1+MLyNCuFFsaOiNNxHOjH/InKSzEuGSLV68tdC7Pe+uanBcC7RKhOrjUC6Occ +naHPC+a/YMyRYx29T8CfkCBB7N6WanWylFN/1RBmAgq++kDflSaF9k+Zdl6I4jiF +mCPGS0k+AMre4PuAKOZOZOwhF0sWlXIxH6zPm9w0bSYdTLBupL846RTO72NtNP+X +KX5DAgMBAAEwDQYJKoZIhvcNAQELBQADggEBACXXFws+h+Zo9HsxW3BWpl2JU5u6 +KyfbLQt4kSN/gqltd4s84Q8c4z2jNdI0t8Oh5dXTjbLCpFjzuF2tdMtOWeYBCdsQ +4NJ69RrwkFdsSPxDPhSE0WGXPaOBaA92wJjTkVf+UYIek1ozeyWwFm1LPiZVei00 +mwDVgbAbIEb8cf6OqJrl2r5PMBCLWBwwg5aca3fe6TopJhyPA//DZDRPA5xzKb9e +PHUgF3apbcWxuxm8Mts4bAq8BcKoEvLHYWJ4fEWQvXPP7q1jYC3TkpSt5n3FQZTe +nLyQ+RNzsEHzmyOtTSa0Q+5KVluO1TE3ifpv8737pTLdY8t2waBamoboCu8= +-----END CERTIFICATE----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.srl b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.srl new file mode 100644 index 000000000..cf7e9e551 --- /dev/null +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.srl @@ -0,0 +1 @@ +BB275A0F903CFBBD diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-cert.pem b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-cert.pem deleted file mode 100644 index a2f9688df..000000000 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-cert.pem +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDBDCCAeygAwIBAgIBAjANBgkqhkiG9w0BAQsFADA8MTowOAYDVQQDDDFNeVNR -TF9TZXJ2ZXJfOC4wLjE5X0F1dG9fR2VuZXJhdGVkX0NBX0NlcnRpZmljYXRlMB4X -DTIwMDYxMTAzMzg0NloXDTMwMDYwOTAzMzg0NlowQDE+MDwGA1UEAww1TXlTUUxf -U2VydmVyXzguMC4xOV9BdXRvX0dlbmVyYXRlZF9TZXJ2ZXJfQ2VydGlmaWNhdGUw -ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCcEnEm5hqP1EbEJycOz8Ua -NWp29QdpFUzTWhkKGhVXk+0msmNTw4NBAFB42moY44OU8wvDideOlJNhPRWveD8z -G2lxzJA91p0UK4et8ia9MmeuCGhdC9jxJ8X69WNlUiPyy0hI/ZsqRq9Z0C2eW0iL -JPXsy4X8Xpw3SFwoXf5pR9RFY5Pb2tuyxqmSestu2VXT/NQjJg4CVDR3mFcHPXZB -4elRzH0WshExEGkgy0bg20MJeRc2Qdb5Xx+EakbmwroDWaCn3NSGqQ7jv6Vw0doy -TGvS6h6RHBxnyqRfRgKGlCoOMG9/5+rFJC00QpCUG2vHXHWGoWlMlJ3foN7rj5v9 -AgMBAAGjDTALMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEBAJ5zt2rj4Ag6 -zpN59AWC1Fur8g8l41ksHkSpKPp+PtyO/ngvbMqBpfmK1e7JCKZv/68QXfMyWWAI -hwalqZkXXWHKjuz3wE7dE25PXFXtGJtcZAaj10xt98fzdqt8lQSwh2kbfNwZIz1F -sgAStgE7+ZTcqTgvNB76Os1UK0to+/P0VBWktaVFdyub4Nc2SdPVnZNvrRBXBwOD -3V8ViwywDOFoE7DvCvwx/SVsvoC0Z4j3AMMovO6oHicP7uU83qsQgm1Qru3YeoLR -+DoVi7IPHbWvN7MqFYn3YjNlByO2geblY7MR0BlqbFlmFrqLsUfjsh2ys7/U/knC -dN/klu446fI= ------END CERTIFICATE----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-key.pem b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-key.pem deleted file mode 100644 index a1dfd5f78..000000000 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-key.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAnBJxJuYaj9RGxCcnDs/FGjVqdvUHaRVM01oZChoVV5PtJrJj -U8ODQQBQeNpqGOODlPMLw4nXjpSTYT0Vr3g/MxtpccyQPdadFCuHrfImvTJnrgho -XQvY8SfF+vVjZVIj8stISP2bKkavWdAtnltIiyT17MuF/F6cN0hcKF3+aUfURWOT -29rbssapknrLbtlV0/zUIyYOAlQ0d5hXBz12QeHpUcx9FrIRMRBpIMtG4NtDCXkX -NkHW+V8fhGpG5sK6A1mgp9zUhqkO47+lcNHaMkxr0uoekRwcZ8qkX0YChpQqDjBv -f+fqxSQtNEKQlBtrx1x1hqFpTJSd36De64+b/QIDAQABAoIBAFiah66Dt9SruLkn -WR8piUaFyLlcBib8Nq9OWSTJBhDAJERxxb4KIvvGB+l0ZgNXNp5bFPSfzsZdRwZP -PX5uj8Kd71Dxx3mz211WESMJdEC42u+MSmN4lGLkJ5t/sDwXU91E1vbJM0ve8THV -4/Ag9qA4DX2vVZOeyqT/6YHpSsPNZplqzrbAiwrfHwkctHfgqwOf3QLfhmVQgfCS -VwidBldEUv2whSIiIxh4Rv5St4kA68IBCbJxdpOpyuQBkk6CkxZ7VN9FqOuSd4Pk -Wm7iWyBMZsCmELZh5XAXld4BEt87C5R4CvbPBDZxAv3THk1DNNvpy3PFQfwARRFb -SAToYMECgYEAyL7U8yxpzHDYWd3oCx6vTi9p9N/z0FfAkWrRF6dm4UcSklNiT1Aq -EOnTA+SaW8tV3E64gCWcY23gNP8so/ZseWj6L+peHwtchaP9+KB7yGw2A+05+lOx -VetLTjAOmfpiUXFe5w1q4C1RGhLjZjjzW+GvwdAuchQgUEFaomrV+PUCgYEAxwfH -cmVGFbAktcjU4HSRjKSfawCrut+3YUOLybyku3Q/hP9amG8qkVTFe95CTLjLe2D0 -ccaTTpofFEJ32COeck0g0Ujn/qQ+KXRoauOYs4FB1DtqMpqB78wufWEUpDpbd9/h -J+gJdC/IADd4tJW9zA92g8IA7ZtFmqDtiSpQ0ekCgYAQGkaorvJZpN+l7cf0RGTZ -h7IfI2vCVZer0n6tQA9fmLzjoe6r4AlPzAHSOR8sp9XeUy43kUzHKQQoHCPvjw/K -eWJAP7OHF/k2+x2fOPhU7mEy1W+mJdp+wt4Kio5RSaVjVQ3AyPG+w8PSrJszEvRq -dWMMz+851WV2KpfjmWBKlQKBgQC++4j4DZQV5aMkSKV1CIZOBf3vaIJhXKEUFQPD -PmB4fBEjpwCg+zNGp6iktt65zi17o8qMjrb1mtCt2SY04eD932LZUHNFlwcLMmes -Ad+aiDLJ24WJL1f16eDGcOyktlblDZB5gZ/ovJzXEGOkLXglosTfo77OQculmDy2 -/UL2WQKBgGeKasmGNfiYAcWio+KXgFkHXWtAXB9B91B1OFnCa40wx+qnl71MIWQH -PQ/CZFNWOfGiNEJIZjrHsfNJoeXkhq48oKcT0AVCDYyLV0VxDO4ejT95mGW6njNd -JpvmhwwAjOvuWVr0tn4iXlSK8irjlJHmwcRjLTJq97vE9fsA2MjI ------END RSA PRIVATE KEY----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.crt b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.crt new file mode 100644 index 000000000..46b1e2a7a --- /dev/null +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDiDCCAnACCQCCsPcIlZO4TDANBgkqhkiG9w0BAQsFADCBhTELMAkGA1UEBhMC +Q0ExGTAXBgNVBAgMEEJyaXRpc2ggQ29sdW1iaWExDjAMBgNVBAcMBUNvbW94MRQw +EgYDVQQKDAtUaGVCcmFpbi5jYTEUMBIGA1UEAwwLdGhlYnJhaW4uY2ExHzAdBgkq +hkiG9w0BCQEWEGluZm9AdGhlYnJhaW4uY2EwHhcNMjEwMTEzMDkwNDIyWhcNMzEw +MTExMDkwNDIyWjCBhTELMAkGA1UEBhMCQ0ExGTAXBgNVBAgMEEJyaXRpc2ggQ29s +dW1iaWExDjAMBgNVBAcMBUNvbW94MRQwEgYDVQQKDAtUaGVCcmFpbi5jYTEUMBIG +A1UEAwwLdGhlYnJhaW4uY2ExHzAdBgkqhkiG9w0BCQEWEGluZm9AdGhlYnJhaW4u +Y2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC2YWuwplM2Hc5tzBMu +covW9nwZ8iNEFo5pbDc8710pmnkF+wsDztLy4afJe6OeVHyCgQxmE+rTZcoWbvoh +pxW3Zy/8es4My07RKHqI3NYadThUvDsmI10cF3tJbhOZaIrMaExLGookZYKwbNAy +7yJ1+MLyNCuFFsaOiNNxHOjH/InKSzEuGSLV68tdC7Pe+uanBcC7RKhOrjUC6Occ +naHPC+a/YMyRYx29T8CfkCBB7N6WanWylFN/1RBmAgq++kDflSaF9k+Zdl6I4jiF +mCPGS0k+AMre4PuAKOZOZOwhF0sWlXIxH6zPm9w0bSYdTLBupL846RTO72NtNP+X +KX5DAgMBAAEwDQYJKoZIhvcNAQELBQADggEBACXXFws+h+Zo9HsxW3BWpl2JU5u6 +KyfbLQt4kSN/gqltd4s84Q8c4z2jNdI0t8Oh5dXTjbLCpFjzuF2tdMtOWeYBCdsQ +4NJ69RrwkFdsSPxDPhSE0WGXPaOBaA92wJjTkVf+UYIek1ozeyWwFm1LPiZVei00 +mwDVgbAbIEb8cf6OqJrl2r5PMBCLWBwwg5aca3fe6TopJhyPA//DZDRPA5xzKb9e +PHUgF3apbcWxuxm8Mts4bAq8BcKoEvLHYWJ4fEWQvXPP7q1jYC3TkpSt5n3FQZTe +nLyQ+RNzsEHzmyOtTSa0Q+5KVluO1TE3ifpv8737pTLdY8t2waBamoboCu8= +-----END CERTIFICATE----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.key b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.key new file mode 100644 index 000000000..8bd131632 --- /dev/null +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAtmFrsKZTNh3ObcwTLnKL1vZ8GfIjRBaOaWw3PO9dKZp5BfsL +A87S8uGnyXujnlR8goEMZhPq02XKFm76IacVt2cv/HrODMtO0Sh6iNzWGnU4VLw7 +JiNdHBd7SW4TmWiKzGhMSxqKJGWCsGzQMu8idfjC8jQrhRbGjojTcRzox/yJyksx +Lhki1evLXQuz3vrmpwXAu0SoTq41AujnHJ2hzwvmv2DMkWMdvU/An5AgQezelmp1 +spRTf9UQZgIKvvpA35UmhfZPmXZeiOI4hZgjxktJPgDK3uD7gCjmTmTsIRdLFpVy +MR+sz5vcNG0mHUywbqS/OOkUzu9jbTT/lyl+QwIDAQABAoIBAA6UVR6G/UnrMhBW +6wWghItHov4T/Du6LeJBk1zcqa7kuV4ABo5kXzqpTVdu+dJzYIyyMkKKvw/tKC2I +65f7GmJR7mUZkBU3v3I68Si1tqvgyQMFFRlkZFIVknZ5RTnTQJ08jTTHx1lHgB4I +ZNBdi3ywySzBfOUjv/Wu/HAjZnxuEh2guBpRMZdwQwZLXr2koDa5inL3IwJrA4Ir +QzpZ0y6ql3A0tw7jAw36G1AKyyz74aFwJ0I8U8w+2Uk4iX5hcKGA8mFq4lyO4/3+ +7W2Z4V8cQzwMq2SMixI0Omxlc2BJUi9j17Ey//5dAXyPaG8QI1kzeL/3Gbs8YBMq +ekN8AZECgYEA5YxcFIVv3yO+ARNWUHovrsMuf9ElhyRuZd0I2+vjrq1b9zQsSy2d +PsyYWD17lO/GDmpTzZOdVsYtZHi+EiXmQnkzLJ4m2nlc7W4annWlbzlQMEn6vAji +l9bSHJXXiiIB7X/oHpDUdsnJp/uyAJppmnVLbSBboNCrG4Mf5cJqOnsCgYEAy2We +scp19h4UEKAU0Yh+5jh8W4VVtlISkH64vMgz/JZWXMPt1bM5C/5j+3UVUL5VmFqF +J1g0gXYkTGTL0+entb3SUiL42zrp3rZ3GgMU6V+aktq3dmri5bOifzihuLHLgjO5 +u/MJPBzvFxIiJxnNBybNLijIZfPm+9roUfpcBNkCgYBGE3Zc0WuYnEm5/FRCVzrN +SEqevJOPUSDeuf6lXLryLXxA2E2ZWcCCVmU/su1SR2yYI/+XZ7QFtJRQ8sdbtPQ5 +YNStj05fLeOfnBhGPbYWYVHInB0OYEwEfJFCJsBZLA6YmY6cHiyuYuXMAXuS0ZDh +lWNEWjd+vZUu3fXT52kUlwKBgDgq/eH3GRA4Si41JsqeOPz2iFD1xy+sBnhkpjtr +xf9wvLStXpZvAcfwHkgokxRTG2wRQ0gUMZu2tltqUmdYR5YGr3gDNFnGMSNRnB5Q +z4uK3TLEt3k6FyJ7stoTF4Xbg2mXQylF+jzheJ0UYt4NX/MjofGnTX/qFNVkJFfP +HW4xAoGBAMBb9cXTpzOMiMcSdQRlaLttV1p05pqxTgQNEQD8HB+lkx4AGnnHvtxW +XQJvPumtqdCEpfe4kaqLip8T+67sGfcDVQMogJc/tpvZ0AN4FuViFsf/YDuTPXEp +whMldPHtusbRP2fk/JFq4Ak0Xz2wAI1iMD3qfBeW6eJpvRllUo69 +-----END RSA PRIVATE KEY----- From 765e37ea26da19c17aa53122593f97af1ce963b8 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 14 Jan 2021 16:55:56 +0100 Subject: [PATCH 07/24] fix(ekka): Upgrade to 0.8.0 to allow remsh Before ekka 0.8.0, when epmd is not used, ekka_dist would calculate a port number from node name. This does not work for remote consoles because a remsh node name is like remsh81random0- i.e. the calculated port number is always the same as the target node. The fix in 0.8.0 allows remsh prefixed nodes to use a range of ports to listen. --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index c1db4712c..4baa9a252 100644 --- a/rebar.config +++ b/rebar.config @@ -45,7 +45,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.7.1"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.7.4"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.7.5"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.0"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.0"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.3"}}} From cbc07d993725c60e19281eb839708bbd207ee3dc Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 13 Jan 2021 22:38:15 +0100 Subject: [PATCH 08/24] fix(bin/emqx): Fix the boot file path for start_clean --- bin/emqx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/emqx b/bin/emqx index a684eb839..ead67c505 100755 --- a/bin/emqx +++ b/bin/emqx @@ -143,7 +143,7 @@ relx_get_pid() { relx_get_nodename() { id="longname$(relx_gen_id)-${NAME}" - "$BINDIR/erl" -boot start_clean -eval '[Host] = tl(string:tokens(atom_to_list(node()),"@")), io:format("~s~n", [Host]), halt()' -noshell ${NAME_TYPE} $id + "$BINDIR/erl" -boot "$REL_DIR/start_clean" -eval '[Host] = tl(string:tokens(atom_to_list(node()),"@")), io:format("~s~n", [Host]), halt()' -noshell ${NAME_TYPE} $id } # Connect to a remote node @@ -155,7 +155,7 @@ relx_rem_sh() { TICKTIME="$(relx_nodetool rpcterms net_kernel get_net_ticktime)" # Setup remote shell command to control node - exec "$BINDIR/erl" "$NAME_TYPE" "$id" -remsh "$NAME" -boot start_clean \ + exec "$BINDIR/erl" "$NAME_TYPE" "$id" -remsh "$NAME" -boot "$REL_DIR/start_clean" \ -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \ -setcookie "$COOKIE" -hidden -kernel net_ticktime $TICKTIME $EPMD_ARG } @@ -504,7 +504,7 @@ case "$1" in fi ;; console_clean) - BOOTFILE="$ROOTDIR/bin/start_clean" + BOOTFILE="$REL_DIR/start_clean" ;; console_boot) shift From 7df1dea4db4098079093ec9935bec5ff59b1f9f9 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 15 Jan 2021 18:54:43 +0800 Subject: [PATCH 09/24] perf(esockd): upgrade esockd to 5.8.0 (#4018) To better support hot upgrades, in esockd 5.8.0 we avoid saving anonymous functions in state. --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 4baa9a252..d97511c69 100644 --- a/rebar.config +++ b/rebar.config @@ -44,7 +44,7 @@ [ {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.7.1"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.7.4"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.0"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.0"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}} From 9e03d6fea1b96560a2472a9654ec60d5d9213351 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 15 Jan 2021 15:38:31 +0800 Subject: [PATCH 10/24] chore(auth): configuration format of unified authentication plug-in --- .github/workflows/run_cts_tests.yaml | 77 +++++++++++-------- .github/workflows/run_test_cases.yaml | 18 +++-- apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf | 2 +- .../priv/emqx_auth_mongo.schema | 60 +++++++++++---- apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf | 6 +- .../priv/emqx_auth_mysql.schema | 11 ++- apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf | 2 +- .../priv/emqx_auth_pgsql.schema | 36 +++++++-- apps/emqx_auth_redis/etc/emqx_auth_redis.conf | 2 +- .../priv/emqx_auth_redis.schema | 38 +++++++-- 10 files changed, 182 insertions(+), 70 deletions(-) diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index e125590e7..807af28e9 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -17,6 +17,7 @@ jobs: runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: ldap_tag: - 2.4.50 @@ -36,12 +37,12 @@ jobs: if: matrix.network_type == 'ipv4' run: | server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ldap) - sed -i "/auth.ldap.servers/c auth.ldap.servers = $server_address" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf + sed -i "s|^[#[:space:]]*auth.ldap.servers[[:space:]]*=.*|auth.ldap.servers = $server_address|g" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf - name: setup if: matrix.network_type == 'ipv6' run: | server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' ldap) - sed -i "/auth.ldap.servers/c auth.ldap.servers = $server_address" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf + sed -i "s|^[#[:space:]]*auth.ldap.servers[[:space:]]*=.*|auth.ldap.servers = $server_address|g" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf - name: run test cases run: | docker exec -i erlang sh -c "make ensure-rebar3" @@ -57,6 +58,7 @@ jobs: runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: mongo_tag: - 3 @@ -76,10 +78,10 @@ jobs: if: matrix.connect_type == 'tls' run: | docker-compose -f .ci/compatibility_tests/docker-compose-mongo-tls.yaml up -d - echo 'auth.mongo.ssl = on' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - echo 'auth.mongo.ssl.cacertfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - echo 'auth.mongo.ssl.certfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - echo 'auth.mongo.ssl.keyfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + sed -i 's|^[#[:space:]]*auth.mongo.ssl[[:space:]]*=.*|auth.mongo.ssl = on|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + sed -i 's|^[#[:space:]]*auth.mongo.cacertfile[[:space:]]*=.*|auth.mongo.cacertfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + sed -i 's|^[#[:space:]]*auth.mongo.certfile[[:space:]]*=.*|auth.mongo.certfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + sed -i 's|^[#[:space:]]*auth.mongo.keyfile[[:space:]]*=.*|auth.mongo.keyfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - name: setup env: MONGO_TAG: ${{ matrix.mongo_tag }} @@ -89,12 +91,12 @@ jobs: if: matrix.network_type == 'ipv4' run: | server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo) - sed -i "/auth.mongo.server/c auth.mongo.server = $server_address:27017" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + sed -i "s|^[#[:space:]]*auth.mongo.server[[:space:]]*=.*|auth.mongo.server = $server_address:27017|g" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - name: setup if: matrix.network_type == 'ipv6' run: | server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mongo) - sed -i "/auth.mongo.server/c auth.mongo.server = $server_address:27017" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + sed -i "s|^[#[:space:]]*auth.mongo.server[[:space:]]*=.*|auth.mongo.server = $server_address:27017|g" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - name: run test cases run: | docker exec -i erlang sh -c "make ensure-rebar3" @@ -110,6 +112,7 @@ jobs: runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: mysql_tag: - 5.7 @@ -129,11 +132,10 @@ jobs: if: matrix.connect_type == 'tls' run: | docker-compose -f .ci/compatibility_tests/docker-compose-mysql-tls.yaml up -d - echo '\n' >> apps/emqx_auth_mongo/etc/emqx_auth_mysql.conf - sed -i '/auth.mysql.ssl = off/c auth.mysql.ssl = on' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf - echo "auth.mysql.ssl.cafile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem" >> apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf - echo "auth.mysql.ssl.certfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem" >> apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf - echo "auth.mysql.ssl.keyfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem" >> apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + sed -i 's|^[#[:space:]]*auth.mysql.ssl[[:space:]]*=.*|auth.mysql.ssl = on|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + sed -i 's|^[#[:space:]]*auth.mysql.cacertfile[[:space:]]*=.*|auth.mysql.cacertfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + sed -i 's|^[#[:space:]]*auth.mysql.certfile[[:space:]]*=.*|auth.mysql.certfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + sed -i 's|^[#[:space:]]*auth.mysql.keyfile[[:space:]]*=.*|auth.mysql.keyfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf - name: setup env: MYSQL_TAG: ${{ matrix.mysql_tag }} @@ -149,6 +151,11 @@ jobs: run: | server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mysql) sed -i "/auth.mysql.server/c auth.mysql.server = $server_address:3306" apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + - name: setup + run: | + sed -i 's|^[#[:space:]]*auth.mysql.username[[:space:]]*=.*|auth.mysql.username = root|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + sed -i 's|^[#[:space:]]*auth.mysql.password[[:space:]]*=.*|auth.mysql.password = public|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + sed -i 's|^[#[:space:]]*auth.mysql.database[[:space:]]*=.*|auth.mysql.database = mqtt|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf - name: run test cases run: | docker exec -i erlang sh -c "make ensure-rebar3" @@ -164,6 +171,7 @@ jobs: runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: pgsql_tag: - 9 @@ -186,27 +194,30 @@ jobs: run: | docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml build --no-cache docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml up -d - sed -i 's|auth.pgsql.username[ \t]*=.*|auth.pgsql.username = postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - sed -i 's|auth.pgsql.password[ \t]*=.*|auth.pgsql.password = postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - sed -i 's|auth.pgsql.database[ \t]*=.*|auth.pgsql.database= postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - sed -i 's|auth.pgsql.ssl[ \t]*=.*|auth.pgsql.ssl = on|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - echo '\n' >> apps/emqx_auth_mongo/etc/emqx_auth_pgsql.conf - echo 'auth.pgsql.ssl.cacertfile = /emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i 's|^[#[:space:]]*auth.pgsql.username[ \t]*=.*|auth.pgsql.username = postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i 's|^[#[:space:]]*auth.pgsql.password[ \t]*=.*|auth.pgsql.password = postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i 's|^[#[:space:]]*auth.pgsql.database[ \t]*=.*|auth.pgsql.database = postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i 's|^[#[:space:]]*auth.pgsql.ssl[ \t]*=.*|auth.pgsql.ssl = on|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i 's|^[#[:space:]]*auth.pgsql.cacertfile[ \t]*=.*|auth.pgsql.cacertfile = /emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - name: setup env: PGSQL_TAG: ${{ matrix.pgsql_tag }} if: matrix.connect_type == 'tcp' - run: docker-compose -f .ci/compatibility_tests/docker-compose-pgsql.yaml up -d + run: | + docker-compose -f .ci/compatibility_tests/docker-compose-pgsql.yaml up -d + sed -i 's|^[#[:space:]]*auth.pgsql.username[ \t]*=.*|auth.pgsql.username = root|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i 's|^[#[:space:]]*auth.pgsql.password[ \t]*=.*|auth.pgsql.password = public|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i 's|^[#[:space:]]*auth.pgsql.database[ \t]*=.*|auth.pgsql.database = mqtt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - name: setup if: matrix.network_type == 'ipv4' run: | server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgsql) - sed -i "/auth.pgsql.server/c auth.pgsql.server = $server_address:5432" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i "s|^[#[:space:]]*auth.pgsql.server[[:space:]]*=.*|auth.pgsql.server = $server_address:5432|g" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - name: setup if: matrix.network_type == 'ipv6' run: | server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' pgsql) - sed -i "/auth.pgsql.server/c auth.pgsql.server = $server_address:5432" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i "s|^[#[:space:]]*auth.pgsql.server[[:space:]]*=.*|auth.pgsql.server = $server_address:5432|g" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - name: run test cases run: | docker exec -i erlang sh -c "make ensure-rebar3" @@ -222,6 +233,7 @@ jobs: runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: redis_tag: - 5 @@ -245,11 +257,10 @@ jobs: run: | set -exu docker-compose -f .ci/compatibility_tests/docker-compose-redis-${{ matrix.node_type }}-tls.yaml up -d - echo '\n' >> apps/emqx_auth_mongo/etc/emqx_auth_redis.conf - echo 'auth.redis.ssl = on' >> apps/emqx_auth_redis/etc/emqx_auth_redis.conf - echo 'auth.redis.ssl.cafile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/ca.crt' >> apps/emqx_auth_redis/etc/emqx_auth_redis.conf - echo 'auth.redis.ssl.certfile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.crt' >> apps/emqx_auth_redis/etc/emqx_auth_redis.conf - echo 'auth.redis.ssl.keyfile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.key' >> apps/emqx_auth_redis/etc/emqx_auth_redis.conf + sed -i 's|^[#[:space:]]*auth.redis.ssl[[:space:]]*=.*|auth.redis.ssl = on|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf + sed -i 's|^[#[:space:]]*auth.redis.ssl.cacertfile[[:space:]]*=.*|auth.redis.ssl.cacertfile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/ca.crt|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf + sed -i 's|^[#[:space:]]*auth.redis.ssl.certfile[[:space:]]*=.*|auth.redis.ssl.certfile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.crt|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf + sed -i 's|^[#[:space:]]*auth.redis.ssl.keyfile[[:space:]]*=.*|auth.redis.ssl.keyfile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.key|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf - name: setup env: REDIS_TAG: ${{ matrix.redis_tag }} @@ -267,24 +278,24 @@ jobs: if: matrix.node_type == 'singer' && matrix.connect_type == 'tcp' run: | set -exu - sed -i "/auth.redis.server/c auth.redis.server = ${redis_${{ matrix.network_type }}_address}:6379" apps/emqx_auth_redis/etc/emqx_auth_redis.conf + sed -i "s|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = ${redis_${{ matrix.network_type }}_address}:6379|g" apps/emqx_auth_redis/etc/emqx_auth_redis.conf - name: setup if: matrix.node_type == 'singer' && matrix.connect_type == 'tls' && matrix.redis_tag != '5' run: | set -exu - sed -i "/auth.redis.server/c auth.redis.server = ${redis_${{ matrix.network_type }}_address}:6380" apps/emqx_auth_redis/etc/emqx_auth_redis.conf + sed -i "s|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = ${redis_${{ matrix.network_type }}_address}:6380|g" apps/emqx_auth_redis/etc/emqx_auth_redis.conf - name: setup if: matrix.node_type == 'cluster' && matrix.connect_type == 'tcp' run: | set -exu - sed -i "/auth.redis.type/c auth.redis.type = cluster" apps/emqx_auth_redis/etc/emqx_auth_redis.conf - sed -i "/auth.redis.server/c auth.redis.server = ${redis_${{ matrix.network_type }}_address}:7000, ${redis_${{ matrix.network_type }}_address}:7001, ${redis_${{ matrix.network_type }}_address}:7002" apps/emqx_auth_redis/etc/emqx_auth_redis.conf + sed -i 's|^[#[:space:]]*auth.redis.type[[:space:]]*=.*|auth.redis.type = cluster|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf + sed -i "s|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = ${redis_${{ matrix.network_type }}_address}:7000, ${redis_${{ matrix.network_type }}_address}:7001, ${redis_${{ matrix.network_type }}_address}:7002|g" apps/emqx_auth_redis/etc/emqx_auth_redis.conf - name: setup if: matrix.node_type == 'cluster' && matrix.connect_type == 'tls' && matrix.redis_tag != '5' run: | set -exu - sed -i "/auth.redis.type/c auth.redis.type = cluster" apps/emqx_auth_redis/etc/emqx_auth_redis.conf - sed -i "/auth.redis.server/c auth.redis.server = ${redis_${{ matrix.network_type }}_address}:8000, ${redis_${{ matrix.network_type }}_address}:8001, ${redis_${{ matrix.network_type }}_address}:8002" apps/emqx_auth_redis/etc/emqx_auth_redis.conf + sed -i 's|^[#[:space:]]*auth.redis.type[[:space:]]*=.*|auth.redis.type = cluster|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf + sed -i "s|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = ${redis_${{ matrix.network_type }}_address}:8000, ${redis_${{ matrix.network_type }}_address}:8001, ${redis_${{ matrix.network_type }}_address}:8002|g" apps/emqx_auth_redis/etc/emqx_auth_redis.conf - name: run test cases if: matrix.connect_type == 'tcp' || (matrix.connect_type == 'tls' && matrix.redis_tag != '5') run: | diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 13d911928..fd0443d9a 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -30,11 +30,19 @@ jobs: docker-compose -f .ci/apps_tests/docker-compose.yaml up -d - name: set config files run: | - sed -i "/auth.mysql.server/c auth.mysql.server = mysql_server:3306" apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf - sed -i "/auth.redis.server/c auth.redis.server = redis_server:6379" apps/emqx_auth_redis/etc/emqx_auth_redis.conf - sed -i "/auth.mongo.server/c auth.mongo.server = mongo_server:27017" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - sed -i "/auth.pgsql.server/c auth.pgsql.server = pgsql_server:5432" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - sed -i "/auth.ldap.servers/c auth.ldap.servers = ldap_server" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf + sed -i 's|^[#[:space:]]*auth.ldap.servers[[:space:]]*=.*|auth.ldap.servers = ldap_server|g' apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf + sed -i 's|^[#[:space:]]*auth.mongo.server[[:space:]]*=.*|auth.mongo.server = mongo_server:27017|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + sed -i 's|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = redis_server:6379|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf + + sed -i 's|^[#[:space:]]*auth.mysql.server[[:space:]]*=.*|auth.mysql.server = mysql_server:3306|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + sed -i 's|^[#[:space:]]*auth.mysql.username[[:space:]]*=.*|auth.mysql.username = root|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + sed -i 's|^[#[:space:]]*auth.mysql.password[[:space:]]*=.*|auth.mysql.password = public|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + sed -i 's|^[#[:space:]]*auth.mysql.database[[:space:]]*=.*|auth.mysql.database = mqtt|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + + sed -i 's|^[#[:space:]]*auth.pgsql.server[[:space:]]*=.*|auth.pgsql.server = pgsql_server:5432|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i 's|^[#[:space:]]*auth.pgsql.username[[:space:]]*=.*|auth.pgsql.username = root|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i 's|^[#[:space:]]*auth.pgsql.password[[:space:]]*=.*|auth.pgsql.password = public|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i 's|^[#[:space:]]*auth.pgsql.database[[:space:]]*=.*|auth.pgsql.database = mqtt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - name: run tests run: | docker exec -i erlang bash -c "make xref" diff --git a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf index d2645c4e1..073feeb6d 100644 --- a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf +++ b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf @@ -27,7 +27,7 @@ auth.mongo.pool = 8 ## MongoDB login user. ## ## Value: String -## auth.mongo.login = +# auth.mongo.username = ## MongoDB password. ## diff --git a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema index ebc5480ae..15668ca5b 100644 --- a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema +++ b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema @@ -21,11 +21,17 @@ {datatype, integer} ]}. +%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 {mapping, "auth.mongo.login", "emqx_auth_mongo.server", [ {default, ""}, {datatype, string} ]}. +{mapping, "auth.mongo.username", "emqx_auth_mongo.server", [ + {default, ""}, + {datatype, string} +]}. + {mapping, "auth.mongo.password", "emqx_auth_mongo.server", [ {default, ""}, {datatype, string} @@ -43,7 +49,7 @@ {mapping, "auth.mongo.ssl", "emqx_auth_mongo.server", [ {default, off}, - {datatype, flag} + {datatype, {enum, [on, off, true, false]}} %% FIXME: ture/false is compatible with 4.0-4.2 version format, plan to delete in 5.0 ]}. {mapping, "auth.mongo.ssl.keyfile", "emqx_auth_mongo.server", [ @@ -58,6 +64,21 @@ {datatype, string} ]}. +%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 +{mapping, "auth.mongo.ssl_opts.keyfile", "emqx_auth_mongo.server", [ + {datatype, string} +]}. + +%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 +{mapping, "auth.mongo.ssl_opts.certfile", "emqx_auth_mongo.server", [ + {datatype, string} +]}. + +%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 +{mapping, "auth.mongo.ssl_opts.cacertfile", "emqx_auth_mongo.server", [ + {datatype, string} +]}. + {mapping, "auth.mongo.w_mode", "emqx_auth_mongo.server", [ {default, undef}, {datatype, {enum, [safe, unsafe, undef]}} @@ -77,7 +98,10 @@ Hosts = string:tokens(H, ","), Type0 = cuttlefish:conf_get("auth.mongo.type", Conf), Pool = cuttlefish:conf_get("auth.mongo.pool", Conf), - Login = cuttlefish:conf_get("auth.mongo.login", Conf), + %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 + Login = cuttlefish:conf_get("auth.mongo.username", Conf, + cuttlefish:conf_get("auth.mongo.login", Conf) + ), Passwd = cuttlefish:conf_get("auth.mongo.password", Conf), DB = cuttlefish:conf_get("auth.mongo.database", Conf), AuthSrc = cuttlefish:conf_get("auth.mongo.auth_source", Conf), @@ -99,18 +123,28 @@ true -> []; false -> [{r_mode, R}] end, + + Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, + SslOpts = fun(Prefix) -> + Filter([{keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, + {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, + {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}]) + end, + + %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 + GenSsl = case cuttlefish:conf_get("auth.mongo.ssl.cacertfile", Conf, undefined) of + undefined -> [{ssl, true}, {ssl_opts, SslOpts("auth.mongo.ssl_opts")}]; + _ -> [{ssl, true}, {ssl_opts, SslOpts("auth.mongo.ssl")}] + end, + + %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 Ssl = case cuttlefish:conf_get("auth.mongo.ssl", Conf) of - true -> - Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, - SslOpts = fun(Prefix) -> - Filter([{keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, - {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, - {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}]) - end, - [{ssl, true}, {ssl_opts, SslOpts("auth.mongo.ssl")}]; - false -> - [] - end, + on -> GenSsl; + off -> []; + true -> GenSsl; + false -> [] + end, + WorkerOptions = [{database, list_to_binary(DB)}, {auth_source, list_to_binary(AuthSrc)}] ++ Login0 ++ Passwd0 ++ W0 ++ R0 ++ Ssl, diff --git a/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf b/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf index c502e36b1..af68134cb 100644 --- a/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf +++ b/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf @@ -17,12 +17,12 @@ auth.mysql.pool = 8 ## MySQL username. ## ## Value: String -auth.mysql.username = root +# auth.mysql.username = ## MySQL password. ## ## Value: String -auth.mysql.password = public +# auth.mysql.password = ## MySQL database. ## @@ -103,7 +103,7 @@ auth.mysql.acl_query = select allow, ipaddr, username, clientid, access, topic f ## CA certificate. ## ## Value: File -## auth.mysql.ssl.cafile = path to your ca file +# auth.mysql.ssl.cacertfile = /path/to/ca.pem ## Client ssl certificate. ## diff --git a/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema b/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema index 37eed8a5f..8aacfd280 100644 --- a/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema +++ b/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema @@ -40,6 +40,12 @@ {datatype, string} ]}. +{mapping, "auth.mysql.ssl.cacertfile", "emqx_auth_mysql.server", [ + {default, ""}, + {datatype, string} +]}. + +%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 {mapping, "auth.mysql.ssl.certfile", "emqx_auth_mysql.server", [ {default, ""}, {datatype, string} @@ -84,7 +90,10 @@ Options1 = case cuttlefish:conf_get("auth.mysql.ssl", Conf) of true -> - CA = cuttlefish:conf_get("auth.mysql.ssl.cafile", Conf), + %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 + CA = cuttlefish:conf_get("auth.mysql.ssl.cacertfile", Conf, + cuttlefish:conf_get("auth.mysql.ssl.cafile", Conf) + ), Cert = cuttlefish:conf_get("auth.mysql.ssl.certfile", Conf), Key = cuttlefish:conf_get("auth.mysql.ssl.keyfile", Conf), Options ++ [{ssl, {server_name_indication, disable}, diff --git a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf index 603a91908..4bfcf9de6 100644 --- a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf +++ b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf @@ -22,7 +22,7 @@ auth.pgsql.username = root ## PostgreSQL password. ## ## Value: String -auth.pgsql.password = public +# auth.pgsql.password = ## PostgreSQL database. ## diff --git a/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema b/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema index 078158c0a..f57c15c08 100644 --- a/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema +++ b/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema @@ -32,7 +32,7 @@ {mapping, "auth.pgsql.ssl", "emqx_auth_pgsql.server", [ {default, off}, - {datatype, flag} + {datatype, {enum, [on, off, true, false]}} %% FIXME: true/fasle is compatible with 4.0-4.2 version format, plan to delete in 5.0 ]}. {mapping, "auth.pgsql.ssl.keyfile", "emqx_auth_pgsql.server", [ @@ -47,6 +47,21 @@ {datatype, string} ]}. +%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 +{mapping, "auth.pgsql.ssl_opts.keyfile", "emqx_auth_pgsql.server", [ + {datatype, string} +]}. + +%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 +{mapping, "auth.pgsql.ssl_opts.certfile", "emqx_auth_pgsql.server", [ + {datatype, string} +]}. + +%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 +{mapping, "auth.pgsql.ssl_opts.cacertfile", "emqx_auth_pgsql.server", [ + {datatype, string} +]}. + {translation, "emqx_auth_pgsql.server", fun(Conf) -> {PgHost, PgPort} = case cuttlefish:conf_get("auth.pgsql.server", Conf) of @@ -61,7 +76,6 @@ Passwd = cuttlefish:conf_get("auth.pgsql.password", Conf, ""), DB = cuttlefish:conf_get("auth.pgsql.database", Conf), Encoding = cuttlefish:conf_get("auth.pgsql.encoding", Conf), - Ssl = cuttlefish:conf_get("auth.pgsql.ssl", Conf), Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, SslOpts = fun(Prefix) -> @@ -69,6 +83,20 @@ {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}]) end, + + %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 + GenSsl = case cuttlefish:conf_get("auth.pgsql.ssl.cacertfile", Conf, undefined) of + undefined -> [{ssl, true}, {ssl_opts, SslOpts("auth.pgsql.ssl_opts")}]; + _ -> [{ssl, true}, {ssl_opts, SslOpts("auth.pgsql.ssl")}] + end, + + %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 + Ssl = case cuttlefish:conf_get("auth.pgsql.ssl", Conf) of + on -> GenSsl; + off -> []; + true -> GenSsl; + false -> [] + end, TempHost = case inet:parse_address(PgHost) of {ok, IpAddr} -> @@ -83,9 +111,7 @@ {username, Username}, {password, Passwd}, {database, DB}, - {encoding, Encoding}, - {ssl, Ssl}, - {ssl_opts, SslOpts("auth.pgsql.ssl")}] + {encoding, Encoding}] ++ Ssl end}. {mapping, "auth.pgsql.auth_query", "emqx_auth_pgsql.auth_query", [ diff --git a/apps/emqx_auth_redis/etc/emqx_auth_redis.conf b/apps/emqx_auth_redis/etc/emqx_auth_redis.conf index 644b90e4e..d56759c84 100644 --- a/apps/emqx_auth_redis/etc/emqx_auth_redis.conf +++ b/apps/emqx_auth_redis/etc/emqx_auth_redis.conf @@ -103,7 +103,7 @@ auth.redis.acl_cmd = HGETALL mqtt_acl:%u ## CA certificate. ## ## Value: File -#auth.redis.ssl.cafile = path/to/your/cafile +# auth.redis.ssl.cacertfile = path/to/your/cafile.pem ## Client ssl certificate. ## diff --git a/apps/emqx_auth_redis/priv/emqx_auth_redis.schema b/apps/emqx_auth_redis/priv/emqx_auth_redis.schema index a70be6a8d..93da345cc 100644 --- a/apps/emqx_auth_redis/priv/emqx_auth_redis.schema +++ b/apps/emqx_auth_redis/priv/emqx_auth_redis.schema @@ -38,11 +38,12 @@ {datatype, flag} ]}. -{mapping, "auth.redis.ssl.cafile", "emqx_auth_redis.options", [ +{mapping, "auth.redis.ssl.cacertfile", "emqx_auth_redis.options", [ {default, ""}, {datatype, string} ]}. +%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 {mapping, "auth.redis.ssl.certfile", "emqx_auth_redis.options", [ {default, ""}, {datatype, string} @@ -53,16 +54,39 @@ {datatype, string} ]}. +%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 +{mapping, "auth.redis.cafile", "emqx_auth_redis.options", [ + {default, ""}, + {datatype, string} +]}. + +%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 +{mapping, "auth.redis.certfile", "emqx_auth_redis.options", [ + {default, ""}, + {datatype, string} +]}. + +%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 +{mapping, "auth.redis.keyfile", "emqx_auth_redis.options", [ + {default, ""}, + {datatype, string} +]}. + {translation, "emqx_auth_redis.options", fun(Conf) -> Ssl = cuttlefish:conf_get("auth.redis.ssl", Conf, false), case Ssl of true -> - CA = cuttlefish:conf_get("auth.redis.ssl.cafile", Conf), - Cert = cuttlefish:conf_get("auth.redis.ssl.certfile", Conf), - Key = cuttlefish:conf_get("auth.redis.ssl.keyfile", Conf), - [{options, [{ssl_options, [{cacertfile, CA}, - {certfile, Cert}, - {keyfile, Key}]}]}]; + %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 + Prefix = case cuttlefish:conf_get("auth.redis.ssl.cacertfile", Conf, undefined) of + undefined -> "auth.redis"; + _ -> "auth.redis.ssl" + end, + CA = cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf), + Cert = cuttlefish:conf_get(Prefix ++ ".certfile", Conf), + Key = cuttlefish:conf_get(Prefix ++ ".keyfile", Conf), + [{options, [{ssl_options, [{cacertfile, CA}, + {certfile, Cert}, + {keyfile, Key}]}]}]; _ -> [{options, []}] end end}. From 5794a708ed28888bd27bc4b8f43a3071e67d3801 Mon Sep 17 00:00:00 2001 From: "ayodele.akingbule" Date: Mon, 11 Jan 2021 22:08:32 +0100 Subject: [PATCH 11/24] feat(emqx_ws_connection): Prevent EMQX from CSWSH Cross-Site Web-Socket Hijack --- etc/emqx.conf | 35 ++++++++++++++++++++++--- priv/emqx.schema | 53 +++++++++++++++++++++++++++++++++++++- src/emqx_ws_connection.erl | 32 ++++++++++++++++++++++- 3 files changed, 114 insertions(+), 6 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 0414d3c61..bf6e0470f 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -681,7 +681,7 @@ mqtt.ignore_loop_deliver = false mqtt.strict_mode = false ## Specify the response information returned to the client -## +## ## Value: String ## mqtt.response_information = example @@ -917,7 +917,7 @@ zone.external.ignore_loop_deliver = false zone.external.strict_mode = false ## Specify the response information returned to the client -## +## ## Value: String ## zone.external.response_information = example @@ -1012,7 +1012,7 @@ zone.internal.ignore_loop_deliver = false zone.internal.strict_mode = false ## Specify the response information returned to the client -## +## ## Value: String ## zone.internal.response_information = example @@ -1704,6 +1704,21 @@ listener.ws.external.nodelay = true ## Value: single | multiple listener.ws.external.mqtt_piggyback = multiple +## Enable origin check in header for websocket connection +## +## Value: true | false (default false) +listener.ws.external.check_origin_enable = false + +## Allow origin to be absent in header in websocket connection when check_origin_enable is true +## +## Value: true | false (default true) +listener.ws.external.allow_origin_absence = true + +## Comma separated list of allowed origin in header for websocket connection +## +## Value: http://url eg. local http dashboard url - http://localhost:18083, http://127.0.0.1:18083 +listener.ws.external.check_origins = http://localhost:18083, http://127.0.0.1:18083 + ##-------------------------------------------------------------------- ## External WebSocket/SSL listener for MQTT Protocol @@ -1984,6 +1999,18 @@ listener.wss.external.send_timeout_close = on ## ## Value: single | multiple listener.wss.external.mqtt_piggyback = multiple +## Enable origin check in header for secure websocket connection +## +## Value: true | false (default false) +listener.wss.external.check_origin_enable = false +## Allow origin to be absent in header in secure websocket connection when check_origin_enable is true +## +## Value: true | false (default true) +listener.wss.external.allow_origin_absence = true +## Comma separated list of allowed origin in header for secure websocket connection +## +## Value: http://url eg. https://localhost:8084, https://127.0.0.1:8084 +listener.wss.external.check_origins = https://localhost:8084, https://127.0.0.1:8084 ##-------------------------------------------------------------------- ## Modules @@ -2245,7 +2272,7 @@ alarm.actions = log,publish ## The maximum number of deactivated alarms ## -## Value: Integer +## Value: Integer ## ## Default: 1000 alarm.size_limit = 1000 diff --git a/priv/emqx.schema b/priv/emqx.schema index 8345f2f51..b6579f5fe 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1582,6 +1582,23 @@ end}. hidden ]}. +{mapping, "listener.ws.$name.check_origin_enable", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + {default, false}, + hidden +]}. + +{mapping, "listener.ws.$name.allow_origin_absence", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + {default, true}, + hidden +]}. + +{mapping, "listener.ws.$name.check_origins", "emqx.listeners", [ + {datatype, string}, + hidden +]}. + %%-------------------------------------------------------------------- %% MQTT/WebSocket/SSL Listeners @@ -1800,6 +1817,23 @@ end}. hidden ]}. +{mapping, "listener.wss.$name.check_origin_enable", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + {default, false}, + hidden +]}. + +{mapping, "listener.wss.$name.allow_origin_absence", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + {default, true}, + hidden +]}. + +{mapping, "listener.wss.$name.check_origins", "emqx.listeners", [ + {datatype, string}, + hidden +]}. + {translation, "emqx.listeners", fun(Conf) -> Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, @@ -1833,6 +1867,20 @@ end}. {Limit, Duration} end, + CheckOrigin = fun(S) -> + Origins = string:tokens(S, ","), + [ list_to_binary(string:trim(O)) || O <- Origins] + end, + + WsOpts = fun(Prefix) -> + case cuttlefish_variable:filter_by_prefix(Prefix ++ ".check_origins", Conf) of + [] -> undefined; + Rules -> + OriginList = [CheckOrigin(Rule) || {_, Rule} <- Rules], + lists:flatten(OriginList) + end + end, + LisOpts = fun(Prefix) -> Filter([{acceptors, cuttlefish:conf_get(Prefix ++ ".acceptors", Conf)}, {mqtt_path, cuttlefish:conf_get(Prefix ++ ".mqtt_path", Conf, undefined)}, @@ -1849,7 +1897,10 @@ end}. {compress, cuttlefish:conf_get(Prefix ++ ".compress", Conf, undefined)}, {idle_timeout, cuttlefish:conf_get(Prefix ++ ".idle_timeout", Conf, undefined)}, {max_frame_size, cuttlefish:conf_get(Prefix ++ ".max_frame_size", Conf, undefined)}, - {mqtt_piggyback, cuttlefish:conf_get(Prefix ++ ".mqtt_piggyback", Conf, undefined)} | AccOpts(Prefix)]) + {mqtt_piggyback, cuttlefish:conf_get(Prefix ++ ".mqtt_piggyback", Conf, undefined)}, + {check_origin_enable, cuttlefish:conf_get(Prefix ++ ".check_origin_enable", Conf, undefined)}, + {allow_origin_absence, cuttlefish:conf_get(Prefix ++ ".allow_origin_absence", Conf, undefined)}, + {check_origins, WsOpts(Prefix)} | AccOpts(Prefix)]) end, DeflateOpts = fun(Prefix) -> Filter([{level, cuttlefish:conf_get(Prefix ++ ".deflate_opts.level", Conf, undefined)}, diff --git a/src/emqx_ws_connection.erl b/src/emqx_ws_connection.erl index 1fe9a105f..f016c24a0 100644 --- a/src/emqx_ws_connection.erl +++ b/src/emqx_ws_connection.erl @@ -183,18 +183,48 @@ init(Req, Opts) -> max_frame_size => MaxFrameSize, idle_timeout => IdleTimeout }, + + case check_origin_header(Req, Opts) of + {error, Message} -> + ?LOG(error, "Invalid Origin Header ~p~n", [Message]), + {ok, cowboy_req:reply(403, Req), WsOpts}; + ok -> parse_sec_websocket_protocol(Req, Opts, WsOpts) + end. + +parse_sec_websocket_protocol(Req, Opts, WsOpts) -> case cowboy_req:parse_header(<<"sec-websocket-protocol">>, Req) of undefined -> %% TODO: why not reply 500??? {cowboy_websocket, Req, [Req, Opts], WsOpts}; [<<"mqtt", Vsn/binary>>] -> Resp = cowboy_req:set_resp_header( - <<"sec-websocket-protocol">>, <<"mqtt", Vsn/binary>>, Req), + <<"sec-websocket-protocol">>, <<"mqtt", Vsn/binary>>, Req), {cowboy_websocket, Resp, [Req, Opts], WsOpts}; _ -> {ok, cowboy_req:reply(400, Req), WsOpts} end. +parse_header_fun_origin(Req, Opts) -> + case cowboy_req:header(<<"origin">>, Req) of + undefined -> + case proplists:get_value(allow_origin_absence, Opts, true) of + true -> ok; + false -> {error, origin_header_cannot_be_absent} + end; + Value -> + Origins = proplists:get_value(check_origins, Opts, []), + case lists:member(Value, Origins) of + true -> ok; + false -> {origin_not_allowed, Value} + end + end. + +check_origin_header(Req, Opts) -> + case proplists:get_value(check_origin_enable, Opts) of + true -> parse_header_fun_origin(Req, Opts); + false -> ok + end. + websocket_init([Req, Opts]) -> Peername = case proplists:get_bool(proxy_protocol, Opts) andalso maps:get(proxy_header, Req) of From 598207e308ca193781589844495180190117f1fd Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 15 Jan 2021 09:35:44 +0100 Subject: [PATCH 12/24] fix(ws_connection): proplists:get_bool for boolean flags --- src/emqx_ws_connection.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/emqx_ws_connection.erl b/src/emqx_ws_connection.erl index f016c24a0..d8898f777 100644 --- a/src/emqx_ws_connection.erl +++ b/src/emqx_ws_connection.erl @@ -177,7 +177,7 @@ init(Req, Opts) -> 0 -> infinity; I -> I end, - Compress = proplists:get_value(compress, Opts, false), + Compress = proplists:get_bool(compress, Opts), WsOpts = #{compress => Compress, deflate_opts => DeflateOptions, max_frame_size => MaxFrameSize, @@ -207,7 +207,7 @@ parse_sec_websocket_protocol(Req, Opts, WsOpts) -> parse_header_fun_origin(Req, Opts) -> case cowboy_req:header(<<"origin">>, Req) of undefined -> - case proplists:get_value(allow_origin_absence, Opts, true) of + case proplists:get_bool(allow_origin_absence, Opts) of true -> ok; false -> {error, origin_header_cannot_be_absent} end; @@ -220,7 +220,7 @@ parse_header_fun_origin(Req, Opts) -> end. check_origin_header(Req, Opts) -> - case proplists:get_value(check_origin_enable, Opts) of + case proplists:get_bool(check_origin_enable, Opts) of true -> parse_header_fun_origin(Req, Opts); false -> ok end. From 114bf8e048c6d1a0f4514d69e7f9b803e66dbd48 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 14 Jan 2021 19:40:44 +0800 Subject: [PATCH 13/24] fix(webhook): only POST and PUT requests carry Content-Type header and using the original URL as host header --- apps/emqx_web_hook/etc/emqx_web_hook.conf | 2 +- .../src/emqx_web_hook_actions.erl | 52 +++++++++---------- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/apps/emqx_web_hook/etc/emqx_web_hook.conf b/apps/emqx_web_hook/etc/emqx_web_hook.conf index 41821fdc9..ae3eaa854 100644 --- a/apps/emqx_web_hook/etc/emqx_web_hook.conf +++ b/apps/emqx_web_hook/etc/emqx_web_hook.conf @@ -5,7 +5,7 @@ ## Webhook URL ## ## Value: String -web.hook.url = http://127.0.0.1:8080 +web.hook.url = http://127.0.0.1:80 ## HTTP Headers ## diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index 784b4c768..a02252acd 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -295,9 +295,12 @@ create_req(_, Path, Headers, Body) -> parse_action_params(Params = #{<<"url">> := URL}) -> try #{path := CommonPath} = uri_string:parse(URL), - #{method => method(maps:get(<<"method">>, Params, <<"POST">>)), + Method = method(maps:get(<<"method">>, Params, <<"POST">>)), + Headers = headers(maps:get(<<"headers">>, Params, undefined)), + NHeaders = ensure_content_type_header(Headers, Method), + #{method => Method, path => path(filename:join(CommonPath, maps:get(<<"path">>, Params, <<>>))), - headers => headers(maps:get(<<"headers">>, Params, undefined)), + headers => NHeaders, body => maps:get(<<"body">>, Params, <<>>), request_timeout => timer:seconds(maps:get(<<"request_timeout">>, Params, 5)), pool => maps:get(<<"pool">>, Params)} @@ -305,6 +308,11 @@ parse_action_params(Params = #{<<"url">> := URL}) -> throw({invalid_params, Params}) end. +ensure_content_type_header(Headers, Method) when Method =:= post orelse Method =:= put -> + Headers; +ensure_content_type_header(Headers, _Method) -> + lists:keydelete("content-type", 1, Headers). + path(<<>>) -> <<"/">>; path(Path) -> Path. @@ -314,30 +322,31 @@ method(PUT) when PUT == <<"PUT">>; PUT == <<"put">> -> put; method(DEL) when DEL == <<"DELETE">>; DEL == <<"delete">> -> delete. headers(undefined) -> []; -headers(Headers) when is_list(Headers) -> Headers; headers(Headers) when is_map(Headers) -> - maps:fold(fun(K, V, Acc) -> - [{str(K), str(V)} | Acc] - end, [], Headers). + headers(maps:to_list(Headers)); +headers(Headers) when is_list(Headers) -> + [{string:to_lower(str(K)), str(V)} || {K, V} <- Headers]. str(Str) when is_list(Str) -> Str; str(Atom) when is_atom(Atom) -> atom_to_list(Atom); str(Bin) when is_binary(Bin) -> binary_to_list(Bin). pool_opts(Params = #{<<"url">> := URL}) -> - #{host := Host0, - port := Port, - scheme := Scheme} = uri_string:parse(URL), - Host = get_addr(binary_to_list(Host0)), + #{host := Host, + scheme := Scheme} = URIMap = uri_string:parse(URL), + Port = maps:get(port, URIMap, case Scheme of + <<"https">> -> 443; + _ -> 80 + end), PoolSize = maps:get(<<"pool_size">>, Params, 32), ConnectTimeout = timer:seconds(maps:get(<<"connect_timeout">>, Params, 5)), - IPv6 = case tuple_size(Host) =:= 8 of - true -> [inet6]; - false -> [] + IPv6 = case inet:getaddr(binary_to_list(Host), inet6) of + {error, _} -> inet; + {ok, _} -> inet6 end, MoreOpts = case Scheme of <<"http">> -> - [{transport_opts, IPv6}]; + [{transport_opts, [IPv6]}]; <<"https">> -> KeyFile = maps:get(<<"keyfile">>, Params), CertFile = maps:get(<<"certfile">>, Params), @@ -357,7 +366,7 @@ pool_opts(Params = #{<<"url">> := URL}) -> {ciphers, lists:foldl(fun(TlsVer, Ciphers) -> Ciphers ++ ssl:cipher_suites(all, TlsVer) end, [], TlsVers)} | TLSOpts], - [{transport, ssl}, {transport_opts, NTLSOpts ++ IPv6}] + [{transport, ssl}, {transport_opts, [IPv6 | NTLSOpts]}] end, [{host, Host}, {port, Port}, @@ -367,18 +376,5 @@ pool_opts(Params = #{<<"url">> := URL}) -> {retry, 5}, {retry_timeout, 1000}] ++ MoreOpts. -get_addr(Hostname) -> - case inet:parse_address(Hostname) of - {ok, {_,_,_,_} = Addr} -> Addr; - {ok, {_,_,_,_,_,_,_,_} = Addr} -> Addr; - {error, einval} -> - case inet:getaddr(Hostname, inet) of - {error, _} -> - {ok, Addr} = inet:getaddr(Hostname, inet6), - Addr; - {ok, Addr} -> Addr - end - end. - pool_name(ResId) -> list_to_atom("webhook:" ++ str(ResId)). From 9f0dbeff29d8bb51e9e09c7d4db1b3ff2bcb9985 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 15 Jan 2021 18:35:24 +0800 Subject: [PATCH 14/24] fix(webhook): fix ipv6 in ip and hostname --- .../src/emqx_web_hook_actions.erl | 24 +++++++--- apps/emqx_web_hook/src/emqx_web_hook_app.erl | 47 +++++++++---------- 2 files changed, 40 insertions(+), 31 deletions(-) diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index a02252acd..4c34babcc 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -332,21 +332,31 @@ str(Atom) when is_atom(Atom) -> atom_to_list(Atom); str(Bin) when is_binary(Bin) -> binary_to_list(Bin). pool_opts(Params = #{<<"url">> := URL}) -> - #{host := Host, - scheme := Scheme} = URIMap = uri_string:parse(URL), + #{host := Host0, + scheme := Scheme} = URIMap = uri_string:parse(binary_to_list(URL)), Port = maps:get(port, URIMap, case Scheme of <<"https">> -> 443; _ -> 80 end), PoolSize = maps:get(<<"pool_size">>, Params, 32), ConnectTimeout = timer:seconds(maps:get(<<"connect_timeout">>, Params, 5)), - IPv6 = case inet:getaddr(binary_to_list(Host), inet6) of - {error, _} -> inet; - {ok, _} -> inet6 + Host = case inet:parse_address(Host0) of + {ok, {_,_,_,_} = Addr} -> Addr; + {ok, {_,_,_,_,_,_,_,_} = Addr} -> Addr; + {error, einval} -> Host0 + end, + Inet = case Host of + {_,_,_,_} -> inet; + {_,_,_,_,_,_,_,_} -> inet6; + _ -> + case inet:getaddr(Host, inet6) of + {error, _} -> inet; + {ok, _} -> inet6 + end end, MoreOpts = case Scheme of <<"http">> -> - [{transport_opts, [IPv6]}]; + [{transport_opts, [Inet]}]; <<"https">> -> KeyFile = maps:get(<<"keyfile">>, Params), CertFile = maps:get(<<"certfile">>, Params), @@ -366,7 +376,7 @@ pool_opts(Params = #{<<"url">> := URL}) -> {ciphers, lists:foldl(fun(TlsVer, Ciphers) -> Ciphers ++ ssl:cipher_suites(all, TlsVer) end, [], TlsVers)} | TLSOpts], - [{transport, ssl}, {transport_opts, [IPv6 | NTLSOpts]}] + [{transport, ssl}, {transport_opts, [Inet | NTLSOpts]}] end, [{host, Host}, {port, Port}, diff --git a/apps/emqx_web_hook/src/emqx_web_hook_app.erl b/apps/emqx_web_hook/src/emqx_web_hook_app.erl index 218002b2b..2c0697c81 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_app.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_app.erl @@ -40,7 +40,7 @@ stop(_State) -> ehttpc_sup:stop_pool(?APP). add_default_scheme(URL) when is_list(URL) -> - add_default_scheme(list_to_binary(URL)); + binary_to_list(add_default_scheme(list_to_binary(URL))); add_default_scheme(<<"http://", _/binary>> = URL) -> URL; add_default_scheme(<<"https://", _/binary>> = URL) -> @@ -51,19 +51,31 @@ add_default_scheme(URL) -> translate_env() -> {ok, URL} = application:get_env(?APP, url), #{host := Host0, - port := Port, path := Path0, - scheme := Scheme} = uri_string:parse(binary_to_list(add_default_scheme(URL))), - Host = get_addr(Host0), + scheme := Scheme} = URIMap = uri_string:parse(add_default_scheme(URL)), + Port = maps:get(port, URIMap, case Scheme of + "https" -> 443; + _ -> 80 + end), Path = path(Path0), - PoolSize = application:get_env(?APP, pool_size, 8), - IPv6 = case tuple_size(Host) =:= 8 of - true -> [inet6]; - false -> [] - end, + Host = case inet:parse_address(Host0) of + {ok, {_,_,_,_} = Addr} -> Addr; + {ok, {_,_,_,_,_,_,_,_} = Addr} -> Addr; + {error, einval} -> Host0 + end, + Inet = case Host of + {_,_,_,_} -> inet; + {_,_,_,_,_,_,_,_} -> inet6; + _ -> + case inet:getaddr(Host, inet6) of + {error, _} -> inet; + {ok, _} -> inet6 + end + end, + PoolSize = application:get_env(?APP, pool_size, 32), MoreOpts = case Scheme of "http" -> - [{transport_opts, IPv6}]; + [{transport_opts, [Inet]}]; "https" -> CACertFile = application:get_env(?APP, cacertfile, undefined), CertFile = application:get_env(?APP, certfile, undefined), @@ -84,7 +96,7 @@ translate_env() -> {ciphers, lists:foldl(fun(TlsVer, Ciphers) -> Ciphers ++ ssl:cipher_suites(all, TlsVer) end, [], TlsVers)} | TLSOpts], - [{transport, ssl}, {transport_opts, NTLSOpts ++ IPv6}] + [{transport, ssl}, {transport_opts, [Inet | NTLSOpts]}] end, PoolOpts = [{host, Host}, {port, Port}, @@ -99,19 +111,6 @@ translate_env() -> NHeaders = set_content_type(Headers), application:set_env(?APP, headers, NHeaders). -get_addr(Hostname) -> - case inet:parse_address(Hostname) of - {ok, {_,_,_,_} = Addr} -> Addr; - {ok, {_,_,_,_,_,_,_,_} = Addr} -> Addr; - {error, einval} -> - case inet:getaddr(Hostname, inet) of - {error, _} -> - {ok, Addr} = inet:getaddr(Hostname, inet6), - Addr; - {ok, Addr} -> Addr - end - end. - path("") -> "/"; path(Path) -> From bf84686b760798a70d10deac87d501c56d6543da Mon Sep 17 00:00:00 2001 From: zhouzb Date: Sat, 16 Jan 2021 09:54:41 +0800 Subject: [PATCH 15/24] chore(ehttpc): update tag of ehttpc --- apps/emqx_web_hook/rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_web_hook/rebar.config b/apps/emqx_web_hook/rebar.config index a923a9906..a1caccf57 100644 --- a/apps/emqx_web_hook/rebar.config +++ b/apps/emqx_web_hook/rebar.config @@ -1,7 +1,7 @@ {plugins, [rebar3_proper]}. {deps, - [{ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.0"}}}, + [{ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.1"}}}, {emqx_rule_engine, {git, "https://github.com/emqx/emqx-rule-engine"}} ]}. From 2751fa6a4126cc5169f8aaffa1322b7a061722d1 Mon Sep 17 00:00:00 2001 From: turtleDeng Date: Sat, 16 Jan 2021 18:25:48 +0800 Subject: [PATCH 16/24] fix(scheme): update emqx_web_hook_actions.erl --- apps/emqx_web_hook/src/emqx_web_hook_actions.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index 4c34babcc..0abd1553b 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -335,7 +335,7 @@ pool_opts(Params = #{<<"url">> := URL}) -> #{host := Host0, scheme := Scheme} = URIMap = uri_string:parse(binary_to_list(URL)), Port = maps:get(port, URIMap, case Scheme of - <<"https">> -> 443; + "https" -> 443; _ -> 80 end), PoolSize = maps:get(<<"pool_size">>, Params, 32), From a6de90c3f9642230250e1ad327918d57233aebae Mon Sep 17 00:00:00 2001 From: wwhai <751957846@qq.com> Date: Sat, 16 Jan 2021 22:28:15 +0800 Subject: [PATCH 17/24] fix(test): add two way ssl test (#4025) --- .github/workflows/run_cts_tests.yaml | 10 +++- apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf | 9 ++- .../priv/emqx_auth_pgsql.schema | 11 +++- .../test/emqx_auth_pgsql_SUITE.erl | 58 +++++++++++++++---- 4 files changed, 71 insertions(+), 17 deletions(-) diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index 807af28e9..123910e06 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -169,7 +169,7 @@ jobs: pgsql: runs-on: ubuntu-20.04 - + strategy: fail-fast: false matrix: @@ -194,6 +194,12 @@ jobs: run: | docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml build --no-cache docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml up -d + if [ "$PGSQL_TAG" = "12" ] || [ "$PGSQL_TAG" = "13" ]; then + sed -i 's|^[#[:space:]]*auth.pgsql.ssl.tls_versions[ \t]*=.*|auth.pgsql.ssl.tls_versions = tlsv1.3,tlsv1.2|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + else + sed -i 's|^[#[:space:]]*auth.pgsql.ssl.tls_versions[ \t]*=.*|auth.pgsql.ssl.tls_versions = tlsv1.2,tlsv1.1|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + fi + sed -i 's|^[#[:space:]]*auth.pgsql.username[ \t]*=.*|auth.pgsql.username = postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf sed -i 's|^[#[:space:]]*auth.pgsql.password[ \t]*=.*|auth.pgsql.password = postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf sed -i 's|^[#[:space:]]*auth.pgsql.database[ \t]*=.*|auth.pgsql.database = postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf @@ -291,7 +297,7 @@ jobs: sed -i 's|^[#[:space:]]*auth.redis.type[[:space:]]*=.*|auth.redis.type = cluster|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf sed -i "s|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = ${redis_${{ matrix.network_type }}_address}:7000, ${redis_${{ matrix.network_type }}_address}:7001, ${redis_${{ matrix.network_type }}_address}:7002|g" apps/emqx_auth_redis/etc/emqx_auth_redis.conf - name: setup - if: matrix.node_type == 'cluster' && matrix.connect_type == 'tls' && matrix.redis_tag != '5' + if: matrix.node_type == 'cluster' && matrix.connect_type == 'tls' && matrix.redis_tag != '5' run: | set -exu sed -i 's|^[#[:space:]]*auth.redis.type[[:space:]]*=.*|auth.redis.type = cluster|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf diff --git a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf index 4bfcf9de6..bf3d290a0 100644 --- a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf +++ b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf @@ -39,6 +39,14 @@ auth.pgsql.encoding = utf8 ## Value: on | off auth.pgsql.ssl = off +## TLS version +## You can configure multi-version use "," split, +## default value is :tlsv1.2 +## Example: +## tlsv1.1,tlsv1.2,tlsv1.3 +## +## auth.pgsql.ssl_opts.tls_versions = tlsv1.2 + ## SSL keyfile. ## ## Value: File @@ -107,4 +115,3 @@ auth.pgsql.super_query = select is_superuser from mqtt_user where username = '%u ## ## Note: You can add the 'ORDER BY' statement to control the rules match order auth.pgsql.acl_query = select allow, ipaddr, username, clientid, access, topic from mqtt_acl where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c' - diff --git a/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema b/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema index f57c15c08..29c186fe8 100644 --- a/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema +++ b/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema @@ -35,6 +35,11 @@ {datatype, {enum, [on, off, true, false]}} %% FIXME: true/fasle is compatible with 4.0-4.2 version format, plan to delete in 5.0 ]}. +{mapping, "auth.pgsql.ssl.tls_versions", "emqx_auth_pgsql.server", [ + {default, "tlsv1.2"}, + {datatype, string} +]}. + {mapping, "auth.pgsql.ssl.keyfile", "emqx_auth_pgsql.server", [ {datatype, string} ]}. @@ -81,7 +86,9 @@ SslOpts = fun(Prefix) -> Filter([{keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, - {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}]) + {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined), + {versions, [list_to_existing_atom(Value) + ||Value <- string:tokens(cuttlefish:conf_get("auth.pgsql.ssl.tls_versions", Conf), " ,")]}}]) end, %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 @@ -97,7 +104,7 @@ true -> GenSsl; false -> [] end, - + TempHost = case inet:parse_address(PgHost) of {ok, IpAddr} -> IpAddr; diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl index 230166a62..1306518cf 100644 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl @@ -72,7 +72,7 @@ all() -> init_per_suite(Config) -> emqx_ct_helpers:start_apps([emqx_auth_pgsql]), drop_acl(), - init_auth(), + drop_auth(), init_auth(), init_acl(), set_special_configs(), @@ -97,6 +97,30 @@ t_comment_config(_) -> ?assertEqual(AuthCount - 1, length(emqx_hooks:lookup('client.authenticate'))), ?assertEqual(AclCount - 1, length(emqx_hooks:lookup('client.check_acl'))). +t_placeholders(_) -> + ClientA = #{username => <<"plain">>, clientid => <<"plain">>, zone => external}, + reload([{password_hash, plain}, + {auth_query, "select password from mqtt_user where username = '%u' and 'a_cn_val' = '%C' limit 1"}]), + {error, not_authorized} = + emqx_access_control:authenticate(ClientA#{password => <<"plain">>}), + {error, not_authorized} = + emqx_access_control:authenticate(ClientA#{password => <<"plain">>, cn => undefined}), + {ok, _} = + emqx_access_control:authenticate(ClientA#{password => <<"plain">>, cn => <<"a_cn_val">>}), + + reload([{auth_query, "select password from mqtt_user where username = '%c' and 'a_dn_val' = '%d' limit 1"}]), + {error, not_authorized} = + emqx_access_control:authenticate(ClientA#{password => <<"plain">>}), + {error, not_authorized} = + emqx_access_control:authenticate(ClientA#{password => <<"plain">>, dn => undefined}), + {ok, _} = + emqx_access_control:authenticate(ClientA#{password => <<"plain">>, dn => <<"a_dn_val">>}), + + reload([{auth_query, "select password from mqtt_user where username = '%u' and '192.168.1.5' = '%a' limit 1"}]), + {error, not_authorized} = + emqx_access_control:authenticate(ClientA#{password => <<"plain">>}), + {ok, _} = + emqx_access_control:authenticate(ClientA#{password => <<"plain">>, peerhost => {192,168,1,5}}). t_check_auth(_) -> Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external}, Md5 = #{clientid => <<"md5">>, username => <<"md5">>, zone => external}, @@ -106,22 +130,32 @@ t_check_auth(_) -> BcryptFoo = #{clientid => <<"bcrypt_foo">>, username => <<"bcrypt_foo">>, zone => external}, User1 = #{clientid => <<"bcrypt_foo">>, username => <<"user">>, zone => external}, Bcrypt = #{clientid => <<"bcrypt">>, username => <<"bcrypt">>, zone => external}, - % + BcryptWrong = #{clientid => <<"bcrypt_wrong">>, username => <<"bcrypt_wrong">>, zone => external}, reload([{password_hash, plain}]), - {ok, #{is_superuser := true}} = emqx_access_control:authenticate(Plain#{password => <<"plain">>}), + {ok,#{is_superuser := true}} = + emqx_access_control:authenticate(Plain#{password => <<"plain">>}), reload([{password_hash, md5}]), - {ok, #{is_superuser := false}} = emqx_access_control:authenticate(Md5#{password => <<"md5">>}), + {ok,#{is_superuser := false}} = + emqx_access_control:authenticate(Md5#{password => <<"md5">>}), reload([{password_hash, sha}]), - {ok, #{is_superuser := false}} = emqx_access_control:authenticate(Sha#{password => <<"sha">>}), + {ok,#{is_superuser := false}} = + emqx_access_control:authenticate(Sha#{password => <<"sha">>}), reload([{password_hash, sha256}]), - {ok, #{is_superuser := false}} = emqx_access_control:authenticate(Sha256#{password => <<"sha256">>}), + {ok,#{is_superuser := false}} = + emqx_access_control:authenticate(Sha256#{password => <<"sha256">>}), reload([{password_hash, bcrypt}]), - {ok, #{is_superuser := false}} = emqx_access_control:authenticate(Bcrypt#{password => <<"password">>}), - - reload([{password_hash, {pbkdf2, sha, 1, 16}}, {auth_query, "select password, salt from mqtt_user where username = '%u' limit 1"}]), - {ok, #{is_superuser := false}} = emqx_access_control:authenticate(Pbkdf2#{password => <<"password">>}), + {ok,#{is_superuser := false}} = + emqx_access_control:authenticate(Bcrypt#{password => <<"password">>}), + {error, not_authorized} = + emqx_access_control:authenticate(BcryptWrong#{password => <<"password">>}), + %%pbkdf2 sha + reload([{password_hash, {pbkdf2, sha, 1, 16}}, + {auth_query, "select password, salt from mqtt_user where username = '%u' limit 1"}]), + {ok,#{is_superuser := false}} = + emqx_access_control:authenticate(Pbkdf2#{password => <<"password">>}), reload([{password_hash, {salt, bcrypt}}]), - {ok, #{is_superuser := false}} = emqx_access_control:authenticate(BcryptFoo#{password => <<"foo">>}), + {ok,#{is_superuser := false}} = + emqx_access_control:authenticate(BcryptFoo#{password => <<"foo">>}), {error, _} = emqx_access_control:authenticate(User1#{password => <<"foo">>}), {error, not_authorized} = emqx_access_control:authenticate(Bcrypt#{password => <<"password">>}). @@ -183,4 +217,4 @@ init_auth() -> drop_auth() -> {ok, Pid} = ecpool_worker:client(gproc_pool:pick_worker({ecpool, ?POOL})), - {ok, [], []} = epgsql:squery(Pid, ?DROP_AUTH_TABLE). \ No newline at end of file + {ok, [], []} = epgsql:squery(Pid, ?DROP_AUTH_TABLE). From fd2e9f147b3cef856cde1ef4aad4866d095fdaea Mon Sep 17 00:00:00 2001 From: tigercl Date: Sat, 16 Jan 2021 23:10:53 +0800 Subject: [PATCH 18/24] fix(auth http): using ehttpc (#4021) * fix(auth http): using ehttpc * chore(ehttpc): update tag of ehttpc * fix(config): update comment --- apps/emqx_auth_http/etc/emqx_auth_http.conf | 143 +++++----- .../emqx_auth_http/include/emqx_auth_http.hrl | 2 - .../emqx_auth_http/priv/emqx_auth_http.schema | 131 +++------ apps/emqx_auth_http/rebar.config | 3 +- apps/emqx_auth_http/src/emqx_acl_http.erl | 27 +- .../emqx_auth_http/src/emqx_auth_http.app.src | 2 +- apps/emqx_auth_http/src/emqx_auth_http.erl | 48 ++-- .../emqx_auth_http/src/emqx_auth_http_app.erl | 252 +++++++++-------- .../emqx_auth_http/src/emqx_auth_http_cli.erl | 11 +- .../emqx_auth_http/src/emqx_auth_http_sup.erl | 29 ++ apps/emqx_auth_http/src/emqx_http_client.erl | 256 ------------------ .../src/emqx_http_client_sup.erl | 48 ---- .../test/emqx_auth_http_SUITE.erl | 20 +- 13 files changed, 326 insertions(+), 646 deletions(-) create mode 100644 apps/emqx_auth_http/src/emqx_auth_http_sup.erl delete mode 100644 apps/emqx_auth_http/src/emqx_http_client.erl delete mode 100644 apps/emqx_auth_http/src/emqx_http_client_sup.erl diff --git a/apps/emqx_auth_http/etc/emqx_auth_http.conf b/apps/emqx_auth_http/etc/emqx_auth_http.conf index 0df589169..3d5c45ea7 100644 --- a/apps/emqx_auth_http/etc/emqx_auth_http.conf +++ b/apps/emqx_auth_http/etc/emqx_auth_http.conf @@ -2,24 +2,29 @@ ## HTTP Auth/ACL Plugin ##-------------------------------------------------------------------- -##-------------------------------------------------------------------- -## Authentication request. - -## HTTP URL API path for authentication request +## HTTP URL API path for Auth Request ## ## Value: URL ## -## Examples: http://127.0.0.1:8991/mqtt/auth, https://[::1]:8991/mqtt/auth -auth.http.auth_req = http://127.0.0.1:8991/mqtt/auth +## Examples: http://127.0.0.1:80/mqtt/auth, https://[::1]:80/mqtt/auth +auth.http.auth_req.url = http://127.0.0.1:80/mqtt/auth +## HTTP Request Method for Auth Request +## ## Value: post | get auth.http.auth_req.method = post -## It only works when method=post -## Value: json | x-www-form-urlencoded -auth.http.auth_req.content_type = x-www-form-urlencoded +## HTTP Request Headers for Auth Request, Content-Type header is configured by default. +## The possible values of the Content-Type header: application/x-www-form-urlencoded, application/json +## +## Examples: auth.http.auth_req.headers.accept = */* +auth.http.auth_req.headers.content-type = application/x-www-form-urlencoded -## Variables: +## Parameters used to construct the request body or query string parameters +## When the request method is GET, these parameters will be converted into query string parameters +## When the request method is POST, the final format is determined by content-type +## +## Available Variables: ## - %u: username ## - %c: clientid ## - %a: ipaddress @@ -29,27 +34,32 @@ auth.http.auth_req.content_type = x-www-form-urlencoded ## - %C: common name of client TLS cert ## - %d: subject of client TLS cert ## -## Value: Params +## Value: =,=,... auth.http.auth_req.params = clientid=%c,username=%u,password=%P -##-------------------------------------------------------------------- -## Superuser request. - -## HTTP URL API path for Superuser request +## HTTP URL API path for SuperUser Request ## ## Value: URL ## -## Examples: http://127.0.0.1:8991/mqtt/superuser, https://[::1]:8991/mqtt/superuser -#auth.http.super_req = http://127.0.0.1:8991/mqtt/superuser +## Examples: http://127.0.0.1:80/mqtt/superuser, https://[::1]:80/mqtt/superuser +auth.http.super_req.url = http://127.0.0.1:80/mqtt/superuser +## HTTP Request Method for SuperUser Request +## ## Value: post | get -#auth.http.super_req.method = post +auth.http.super_req.method = post -## It only works when method=pos -## Value: json | x-www-form-urlencoded -#auth.http.super_req.content_type = x-www-form-urlencoded +## HTTP Request Headers for SuperUser Request, Content-Type header is configured by default. +## The possible values of the Content-Type header: application/x-www-form-urlencoded, application/json +## +## Examples: auth.http.super_req.headers.accept = */* +auth.http.super_req.headers.content-type = application/x-www-form-urlencoded -## Variables: +## Parameters used to construct the request body or query string parameters +## When the request method is GET, these parameters will be converted into query string parameters +## When the request method is POST, the final format is determined by content-type +## +## Available Variables: ## - %u: username ## - %c: clientid ## - %a: ipaddress @@ -59,42 +69,45 @@ auth.http.auth_req.params = clientid=%c,username=%u,password=%P ## - %C: common name of client TLS cert ## - %d: subject of client TLS cert ## -## Value: Params -#auth.http.super_req.params = clientid=%c,username=%u +## Value: =,=,... +auth.http.super_req.params = clientid=%c,username=%u -##-------------------------------------------------------------------- -## ACL request. - -## HTTP URL API path for ACL request +## HTTP URL API path for ACL Request ## ## Value: URL ## -## Examples: http://127.0.0.1:8991/mqtt/acl, https://[::1]:8991/mqtt/acl -auth.http.acl_req = http://127.0.0.1:8991/mqtt/acl +## Examples: http://127.0.0.1:80/mqtt/acl, https://[::1]:80/mqtt/acl +auth.http.acl_req.url = http://127.0.0.1:80/mqtt/acl +## HTTP Request Method for ACL Request +## ## Value: post | get -auth.http.acl_req.method = get +auth.http.acl_req.method = post -## It only works when method=post -## Value: json | x-www-form-urlencoded -auth.http.acl_req.content_type = x-www-form-urlencoded +## HTTP Request Headers for ACL Request, Content-Type header is configured by default. +## The possible values of the Content-Type header: application/x-www-form-urlencoded, application/json +## +## Examples: auth.http.acl_req.headers.accept = */* +auth.http.acl_req.headers.content-type = application/x-www-form-urlencoded -## Variables: -## - %A: 1 | 2, 1 = sub, 2 = pub +## Parameters used to construct the request body or query string parameters +## When the request method is GET, these parameters will be converted into query string parameters +## When the request method is POST, the final format is determined by content-type +## +## Available Variables: ## - %u: username ## - %c: clientid ## - %a: ipaddress ## - %r: protocol -## - %m: mountpoint -## - %t: topic +## - %P: password +## - %p: sockport of server accepted +## - %C: common name of client TLS cert +## - %d: subject of client TLS cert ## -## Value: Params +## Value: =,=,... auth.http.acl_req.params = access=%A,username=%u,clientid=%c,ipaddr=%a,topic=%t,mountpoint=%m -##------------------------------------------------------------------------------ -## Http Reqeust options - -## Time-out time for the http request, 0 is never timeout. +## Time-out time for the request. ## ## Value: Duration ## -h: hour, e.g. '2h' for 2 hours @@ -102,37 +115,23 @@ auth.http.acl_req.params = access=%A,username=%u,clientid=%c,ipaddr=%a,topic=%t, ## -s: second, e.g. '30s' for 30 seconds ## ## Default: 5s -## auth.http.request.timeout = 5s +auth.http.timeout = 5s -## Connection time-out time, used during the initial request -## when the client is connecting to the server +## Connection time-out time, used during the initial request, +## when the client is connecting to the server. ## ## Value: Duration +## -h: hour, e.g. '2h' for 2 hours +## -m: minute, e.g. '5m' for 5 minutes +## -s: second, e.g. '30s' for 30 seconds ## -## Default is same with the timeout option -## auth.http.request.connect_timeout = 0 +## Default: 5s +auth.http.connect_timeout = 5s -## Re-send http reuqest times +## Connection process pool size ## -## Value: integer -## -## Default: 3 -auth.http.request.retry_times = 5 - -## The interval for re-sending the http request -## -## Value: Duration -## -## Default: 1s -auth.http.request.retry_interval = 1s - -## The 'Exponential Backoff' mechanism for re-sending request. The actually -## re-send time interval is `interval * backoff ^ times` -## -## Value: float -## -## Default: 2.0 -auth.http.request.retry_backoff = 2.0 +## Value: Number +auth.http.pool_size = 32 ##------------------------------------------------------------------------------ ## SSL options @@ -152,11 +151,3 @@ auth.http.request.retry_backoff = 2.0 ## ## Value: File ## auth.http.ssl.keyfile = {{ platform_etc_dir }}/certs/client-key.pem - -##-------------------------------------------------------------------- -## HTTP Request Headers -## -## Example: auth.http.header.Accept-Encoding = * -## -## Value: String -## auth.http.header.Accept = */* diff --git a/apps/emqx_auth_http/include/emqx_auth_http.hrl b/apps/emqx_auth_http/include/emqx_auth_http.hrl index 2bbe12827..9c1216357 100644 --- a/apps/emqx_auth_http/include/emqx_auth_http.hrl +++ b/apps/emqx_auth_http/include/emqx_auth_http.hrl @@ -1,8 +1,6 @@ -define(APP, emqx_auth_http). --record(http_request, {method = post, path, headers, params, request_timeout}). - -record(auth_metrics, { success = 'client.auth.success', failure = 'client.auth.failure', diff --git a/apps/emqx_auth_http/priv/emqx_auth_http.schema b/apps/emqx_auth_http/priv/emqx_auth_http.schema index 4f4289db0..afd71cfd9 100644 --- a/apps/emqx_auth_http/priv/emqx_auth_http.schema +++ b/apps/emqx_auth_http/priv/emqx_auth_http.schema @@ -1,6 +1,6 @@ %%-*- mode: erlang -*- %% emqx_auth_http config mapping -{mapping, "auth.http.auth_req", "emqx_auth_http.auth_req", [ +{mapping, "auth.http.auth_req.url", "emqx_auth_http.auth_req", [ {datatype, string} ]}. @@ -9,9 +9,8 @@ {datatype, {enum, [post, get]}} ]}. -{mapping, "auth.http.auth_req.content_type", "emqx_auth_http.auth_req", [ - {default, 'x-www-form-urlencoded'}, - {datatype, {enum, ['json', 'x-www-form-urlencoded']}} +{mapping, "auth.http.auth_req.headers.$field", "emqx_auth_http.auth_req", [ + {datatype, string} ]}. {mapping, "auth.http.auth_req.params", "emqx_auth_http.auth_req", [ @@ -19,18 +18,19 @@ ]}. {translation, "emqx_auth_http.auth_req", fun(Conf) -> - case cuttlefish:conf_get("auth.http.auth_req", Conf) of + case cuttlefish:conf_get("auth.http.auth_req.url", Conf, undefined) of undefined -> cuttlefish:unset(); Url -> + Headers = cuttlefish_variable:filter_by_prefix("auth.http.auth_req.headers", Conf), Params = cuttlefish:conf_get("auth.http.auth_req.params", Conf), [{url, Url}, - {method, cuttlefish:conf_get("auth.http.auth_req.method", Conf)}, - {content_type, list_to_binary("application/" ++ atom_to_list(cuttlefish:conf_get("auth.http.auth_req.content_type", Conf)))}, - {params, [list_to_tuple(string:tokens(S, "=")) || S <- string:tokens(Params, ",")]}] + {method, cuttlefish:conf_get("auth.http.auth_req.method", Conf)}, + {headers, [{K, V} || {[_, _, _, _, K], V} <- Headers]}, + {params, [list_to_tuple(string:tokens(S, "=")) || S <- string:tokens(Params, ",")]}] end end}. -{mapping, "auth.http.super_req", "emqx_auth_http.super_req", [ +{mapping, "auth.http.super_req.url", "emqx_auth_http.super_req", [ {datatype, string} ]}. @@ -39,9 +39,8 @@ end}. {datatype, {enum, [post, get]}} ]}. -{mapping, "auth.http.super_req.content_type", "emqx_auth_http.super_req", [ - {default, 'x-www-form-urlencoded'}, - {datatype, {enum, ['json', 'x-www-form-urlencoded']}} +{mapping, "auth.http.super_req.headers.$field", "emqx_auth_http.super_req", [ + {datatype, string} ]}. {mapping, "auth.http.super_req.params", "emqx_auth_http.super_req", [ @@ -49,17 +48,19 @@ end}. ]}. {translation, "emqx_auth_http.super_req", fun(Conf) -> - case cuttlefish:conf_get("auth.http.super_req", Conf, undefined) of + case cuttlefish:conf_get("auth.http.super_req.url", Conf, undefined) of undefined -> cuttlefish:unset(); - Url -> Params = cuttlefish:conf_get("auth.http.super_req.params", Conf), - [{url, Url}, {method, cuttlefish:conf_get("auth.http.super_req.method", Conf)}, - {content_type, list_to_binary("application/" ++ atom_to_list(cuttlefish:conf_get("auth.http.super_req.content_type", Conf)))}, - {params, [list_to_tuple(string:tokens(S, "=")) || S <- string:tokens(Params, ",")]}] + Url -> + Headers = cuttlefish_variable:filter_by_prefix("auth.http.super_req.headers", Conf), + Params = cuttlefish:conf_get("auth.http.super_req.params", Conf), + [{url, Url}, + {method, cuttlefish:conf_get("auth.http.super_req.method", Conf)}, + {headers, [{K, V} || {[_, _, _, _, K], V} <- Headers]}, + {params, [list_to_tuple(string:tokens(S, "=")) || S <- string:tokens(Params, ",")]}] end end}. -{mapping, "auth.http.acl_req", "emqx_auth_http.acl_req", [ - {default, undefined}, +{mapping, "auth.http.acl_req.url", "emqx_auth_http.acl_req", [ {datatype, string} ]}. @@ -68,9 +69,8 @@ end}. {datatype, {enum, [post, get]}} ]}. -{mapping, "auth.http.acl_req.content_type", "emqx_auth_http.acl_req", [ - {default, 'x-www-form-urlencoded'}, - {datatype, {enum, ['json', 'x-www-form-urlencoded']}} +{mapping, "auth.http.acl_req.headers.$field", "emqx_auth_http.acl_req", [ + {datatype, string} ]}. {mapping, "auth.http.acl_req.params", "emqx_auth_http.acl_req", [ @@ -78,92 +78,41 @@ end}. ]}. {translation, "emqx_auth_http.acl_req", fun(Conf) -> - case cuttlefish:conf_get("auth.http.acl_req", Conf, undefined) of + case cuttlefish:conf_get("auth.http.acl_req.url", Conf, undefined) of undefined -> cuttlefish:unset(); - Url -> Params = cuttlefish:conf_get("auth.http.acl_req.params", Conf), - [{url, Url}, - {method, cuttlefish:conf_get("auth.http.acl_req.method", Conf)}, - {content_type, list_to_binary("application/" ++ atom_to_list(cuttlefish:conf_get("auth.http.acl_req.content_type", Conf)))}, - {params, [list_to_tuple(string:tokens(S, "=")) || S <- string:tokens(Params, ",")]}] + Url -> + Headers = cuttlefish_variable:filter_by_prefix("auth.http.acl_req.headers", Conf), + Params = cuttlefish:conf_get("auth.http.acl_req.params", Conf), + [{url, Url}, + {method, cuttlefish:conf_get("auth.http.acl_req.method", Conf)}, + {headers, [{K, V} || {[_, _, _, _, K], V} <- Headers]}, + {params, [list_to_tuple(string:tokens(S, "=")) || S <- string:tokens(Params, ",")]}] end end}. -{mapping, "auth.http.request.timeout", "emqx_auth_http.request_timeout", [ +{mapping, "auth.http.timeout", "emqx_auth_http.timeout", [ {default, "5s"}, {datatype, [integer, {duration, ms}]} ]}. -{mapping, "auth.http.pool_size", "emqx_auth_http.pool_opts", [ +{mapping, "auth.http.connect_timeout", "emqx_auth_http.connect_timeout", [ + {default, "5s"}, + {datatype, [integer, {duration, ms}]} +]}. + +{mapping, "auth.http.pool_size", "emqx_auth_http.pool_size", [ {default, 8}, {datatype, integer} ]}. -{mapping, "auth.http.request.connect_timeout", "emqx_auth_http.pool_opts", [ - {default, "5s"}, - {datatype, [integer, {duration, ms}]} -]}. - -{mapping, "auth.http.ssl.cacertfile", "emqx_auth_http.pool_opts", [ +{mapping, "auth.http.ssl.cacertfile", "emqx_auth_http.cacertfile", [ {datatype, string} ]}. -{mapping, "auth.http.ssl.certfile", "emqx_auth_http.pool_opts", [ +{mapping, "auth.http.ssl.certfile", "emqx_auth_http.certfile", [ {datatype, string} ]}. -{mapping, "auth.http.ssl.keyfile", "emqx_auth_http.pool_opts", [ +{mapping, "auth.http.ssl.keyfile", "emqx_auth_http.keyfile", [ {datatype, string} ]}. - -{mapping, "auth.http.request.retry_times", "emqx_auth_http.pool_opts", [ - {default, 5}, - {datatype, integer} -]}. - -{mapping, "auth.http.request.retry_interval", "emqx_auth_http.pool_opts", [ - {default, "1s"}, - {datatype, {duration, ms}} -]}. - -{mapping, "auth.http.request.retry_backoff", "emqx_auth_http.pool_opts", [ - {default, 2.0}, - {datatype, float} -]}. - -{translation, "emqx_auth_http.pool_opts", fun(Conf) -> - Filter = fun(L) -> [{K, V} || {K, V} <- L, V =/= undefined] end, - InfinityFun = fun(0) -> infinity; - (Duration) -> Duration - end, - SslOpts = Filter([{cacertfile, cuttlefish:conf_get("auth.http.ssl.cacertfile", Conf, undefined)}, - {certfile, cuttlefish:conf_get("auth.http.ssl.certfile", Conf, undefined)}, - {keyfile, cuttlefish:conf_get("auth.http.ssl.keyfile", Conf, undefined)}]), - Opts = [{pool_size, cuttlefish:conf_get("auth.http.pool_size", Conf)}, - {connect_timeout, InfinityFun(cuttlefish:conf_get("auth.http.request.connect_timeout", Conf))}, - {retry, cuttlefish:conf_get("auth.http.request.retry_times", Conf)}, - {retry_timeout, cuttlefish:conf_get("auth.http.request.retry_interval", Conf)}], - case SslOpts of - [] -> Filter(Opts); - _ -> - TlsVers = ['tlsv1.2','tlsv1.1',tlsv1], - DefaultOpts = [{versions, TlsVers}, - {ciphers, lists:foldl( - fun(TlsVer, Ciphers) -> - Ciphers ++ ssl:cipher_suites(all, TlsVer) - end, [], TlsVers)}], - Filter([{ssl, DefaultOpts ++ SslOpts} | Opts]) - end -end}. - - -{mapping, "auth.http.header.$field", "emqx_auth_http.headers", [ - {datatype, string} -]}. - -{translation, "emqx_auth_http.headers", fun(Conf) -> - lists:map( - fun({["auth", "http", "header", Field], Value}) -> - {Field, Value} - end, - cuttlefish_variable:filter_by_prefix("auth.http.header", Conf)) -end}. \ No newline at end of file diff --git a/apps/emqx_auth_http/rebar.config b/apps/emqx_auth_http/rebar.config index 0f2e48298..1e9f04a08 100644 --- a/apps/emqx_auth_http/rebar.config +++ b/apps/emqx_auth_http/rebar.config @@ -1,6 +1,5 @@ {deps, - [{gun, {git, "https://github.com/emqx/gun", {tag, "1.3.4"}}}, - {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} + [{ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.1"}}} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/apps/emqx_auth_http/src/emqx_acl_http.erl b/apps/emqx_auth_http/src/emqx_acl_http.erl index a6f60b465..23f6ff62b 100644 --- a/apps/emqx_auth_http/src/emqx_acl_http.erl +++ b/apps/emqx_auth_http/src/emqx_acl_http.erl @@ -42,22 +42,20 @@ register_metrics() -> %% ACL callbacks %%-------------------------------------------------------------------- -check_acl(ClientInfo, PubSub, Topic, AclResult, State) -> +check_acl(ClientInfo, PubSub, Topic, AclResult, Params) -> return_with(fun inc_metrics/1, - do_check_acl(ClientInfo, PubSub, Topic, AclResult, State)). + do_check_acl(ClientInfo, PubSub, Topic, AclResult, Params)). -do_check_acl(#{username := <<$$, _/binary>>}, _PubSub, _Topic, _AclResult, _Config) -> +do_check_acl(#{username := <<$$, _/binary>>}, _PubSub, _Topic, _AclResult, _Params) -> ok; -do_check_acl(ClientInfo, PubSub, Topic, _AclResult, #{acl_req := AclReq, - pool_name := PoolName}) -> +do_check_acl(ClientInfo, PubSub, Topic, _AclResult, #{acl := ACLParams = #{path := Path}}) -> ClientInfo1 = ClientInfo#{access => access(PubSub), topic => Topic}, - case check_acl_request(PoolName, AclReq, ClientInfo1) of + case check_acl_request(ACLParams, ClientInfo1) of {ok, 200, <<"ignore">>} -> ok; {ok, 200, _Body} -> {stop, allow}; {ok, _Code, _Body} -> {stop, deny}; {error, Error} -> - ?LOG(error, "Request ACL path ~s, error: ~p", - [AclReq#http_request.path, Error]), + ?LOG(error, "Request ACL path ~s, error: ~p", [Path, Error]), ok end. @@ -77,12 +75,13 @@ inc_metrics({stop, deny}) -> return_with(Fun, Result) -> Fun(Result), Result. -check_acl_request(PoolName, #http_request{path = Path, - method = Method, - headers = Headers, - params = Params, - request_timeout = RequestTimeout}, ClientInfo) -> - request(PoolName, Method, Path, Headers, feedvar(Params, ClientInfo), RequestTimeout). +check_acl_request(#{pool_name := PoolName, + path := Path, + method := Method, + headers := Headers, + params := Params, + timeout := Timeout}, ClientInfo) -> + request(PoolName, Method, Path, Headers, feedvar(Params, ClientInfo), Timeout). access(subscribe) -> 1; access(publish) -> 2. diff --git a/apps/emqx_auth_http/src/emqx_auth_http.app.src b/apps/emqx_auth_http/src/emqx_auth_http.app.src index 207b24cf4..b2c3221e6 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.app.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.app.src @@ -3,7 +3,7 @@ {vsn, "4.3.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_http_sup]}, - {applications, [kernel,stdlib,gproc,gun]}, + {applications, [kernel,stdlib,ehttpc]}, {mod, {emqx_auth_http_app, []}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_auth_http/src/emqx_auth_http.erl b/apps/emqx_auth_http/src/emqx_auth_http.erl index 21fe2d325..e82a37625 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http.erl @@ -29,10 +29,6 @@ , feedvar/2 ]). --type http_request() :: #http_request{method::'get' | 'post',params::[any()]}. -%-type http_opts() :: #{clientid:=_, peerhost:=_, protocol:=_, _=>_}. -%-type retry_opts() :: #{backoff:=_, interval:=_, times:=_, _=>_}. - %% Callbacks -export([ register_metrics/0 , check/3 @@ -43,28 +39,26 @@ register_metrics() -> lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). -check(ClientInfo, AuthResult, #{auth_req := AuthReq, - super_req := SuperReq, - pool_name := PoolName}) -> - case authenticate(PoolName, AuthReq, ClientInfo) of +check(ClientInfo, AuthResult, #{auth := AuthParms = #{path := Path}, + super := SuperParams}) -> + case authenticate(AuthParms, ClientInfo) of {ok, 200, <<"ignore">>} -> emqx_metrics:inc(?AUTH_METRICS(ignore)), ok; {ok, 200, Body} -> emqx_metrics:inc(?AUTH_METRICS(success)), - IsSuperuser = is_superuser(PoolName, SuperReq, ClientInfo), + IsSuperuser = is_superuser(SuperParams, ClientInfo), {stop, AuthResult#{is_superuser => IsSuperuser, auth_result => success, anonymous => false, mountpoint => mountpoint(Body, ClientInfo)}}; {ok, Code, _Body} -> ?LOG(error, "Deny connection from path: ~s, response http code: ~p", - [AuthReq#http_request.path, Code]), + [Path, Code]), emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => http_to_connack_error(Code), anonymous => false}}; {error, Error} -> - ?LOG(error, "Request auth path: ~s, error: ~p", - [AuthReq#http_request.path, Error]), + ?LOG(error, "Request auth path: ~s, error: ~p", [Path, Error]), emqx_metrics:inc(?AUTH_METRICS(failure)), %%FIXME later: server_unavailable is not right. {stop, AuthResult#{auth_result => server_unavailable, @@ -77,22 +71,24 @@ description() -> "Authentication by HTTP API". %% Requests %%-------------------------------------------------------------------- -authenticate(PoolName, #http_request{path = Path, - method = Method, - headers = Headers, - params = Params, - request_timeout = RequestTimeout}, ClientInfo) -> - request(PoolName, Method, Path, Headers, feedvar(Params, ClientInfo), RequestTimeout). +authenticate(#{pool_name := PoolName, + path := Path, + method := Method, + headers := Headers, + params := Params, + timeout := Timeout}, ClientInfo) -> + request(PoolName, Method, Path, Headers, feedvar(Params, ClientInfo), Timeout). --spec(is_superuser(atom(), maybe(http_request()), emqx_types:client()) -> boolean()). -is_superuser(_PoolName, undefined, _ClientInfo) -> +-spec(is_superuser(maybe(map()), emqx_types:client()) -> boolean()). +is_superuser(undefined, _ClientInfo) -> false; -is_superuser(PoolName, #http_request{path = Path, - method = Method, - headers = Headers, - params = Params, - request_timeout = RequestTimeout}, ClientInfo) -> - case request(PoolName, Method, Path, Headers, feedvar(Params, ClientInfo), RequestTimeout) of +is_superuser(#{pool_name := PoolName, + path := Path, + method := Method, + headers := Headers, + params := Params, + timeout := Timeout}, ClientInfo) -> + case request(PoolName, Method, Path, Headers, feedvar(Params, ClientInfo), Timeout) of {ok, 200, _Body} -> true; {ok, _Code, _Body} -> false; {error, Error} -> ?LOG(error, "Request superuser path ~s, error: ~p", [Path, Error]), diff --git a/apps/emqx_auth_http/src/emqx_auth_http_app.erl b/apps/emqx_auth_http/src/emqx_auth_http_app.erl index c251962ce..487201e8e 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_app.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_app.erl @@ -25,139 +25,153 @@ -export([ start/2 , stop/1 ]). --export([init/1]). %%-------------------------------------------------------------------- %% Application Callbacks %%-------------------------------------------------------------------- start(_StartType, _StartArgs) -> - case translate_env() of - ok -> - {ok, PoolOpts} = application:get_env(?APP, pool_opts), - {ok, Sup} = emqx_http_client_sup:start_link(?APP, ssl(inet(PoolOpts))), - _ = with_env(auth_req, fun load_auth_hook/1), - _ = with_env(acl_req, fun load_acl_hook/1), - {ok, Sup}; - {error, Reason} -> - {error, Reason} - end. - -load_auth_hook(AuthReq) -> - ok = emqx_auth_http:register_metrics(), - SuperReq = r(application:get_env(?APP, super_req, undefined)), - Params = #{auth_req => AuthReq, - super_req => SuperReq, - pool_name => ?APP}, - emqx:hook('client.authenticate', {emqx_auth_http, check, [Params]}). - -load_acl_hook(AclReq) -> - ok = emqx_acl_http:register_metrics(), - Params = #{acl_req => AclReq, - pool_name => ?APP}, - emqx:hook('client.check_acl', {emqx_acl_http, check_acl, [Params]}). + {ok, Sup} = emqx_auth_http_sup:start_link(), + translate_env(), + load_hooks(), + {ok, Sup}. stop(_State) -> - emqx:unhook('client.authenticate', {emqx_auth_http, check}), - emqx:unhook('client.check_acl', {emqx_acl_http, check_acl}), - emqx_http_client_sup:stop_pool(?APP). - -%%-------------------------------------------------------------------- -%% Dummy supervisor -%%-------------------------------------------------------------------- - -init([]) -> - {ok, { {one_for_all, 10, 100}, []} }. + unload_hooks(). %%-------------------------------------------------------------------- %% Internel functions %%-------------------------------------------------------------------- -with_env(Par, Fun) -> - case application:get_env(?APP, Par) of - undefined -> ok; - {ok, Req} -> Fun(r(Req)) - end. - -r(undefined) -> - undefined; -r(Config) -> - Headers = application:get_env(?APP, headers, []), - Method = proplists:get_value(method, Config, post), - Path = proplists:get_value(path, Config), - NewHeaders = [{<<"content-type">>, proplists:get_value(content_type, Config, <<"application/x-www-form-urlencoded">>)} | Headers], - Params = proplists:get_value(params, Config), - {ok, RequestTimeout} = application:get_env(?APP, request_timeout), - #http_request{method = Method, path = Path, headers = NewHeaders, params = Params, request_timeout = RequestTimeout}. - -inet(PoolOpts) -> - case proplists:get_value(host, PoolOpts) of - Host when tuple_size(Host) =:= 8 -> - TransOpts = proplists:get_value(transport_opts, PoolOpts, []), - NewPoolOpts = proplists:delete(transport_opts, PoolOpts), - [{transport_opts, [inet6 | TransOpts]} | NewPoolOpts]; - _ -> - PoolOpts - end. - -ssl(PoolOpts) -> - case proplists:get_value(ssl, PoolOpts, []) of - [] -> - PoolOpts; - SSLOpts -> - TransOpts = proplists:get_value(transport_opts, PoolOpts, []), - NewPoolOpts = proplists:delete(transport_opts, PoolOpts), - [{transport_opts, SSLOpts ++ TransOpts}, {transport, ssl} | NewPoolOpts] - end. - translate_env() -> - URLs = lists:foldl(fun(Name, Acc) -> - case application:get_env(?APP, Name, []) of - [] -> Acc; - Env -> - URL = proplists:get_value(url, Env), - #{host := Host0, - port := Port, - path := Path} = uri_string:parse(URL), - Host = get_addr(Host0), - [{Name, {Host, Port, path(Path)}} | Acc] - end - end, [], [acl_req, auth_req, super_req]), - case same_host_and_port(URLs) of - true -> - [begin - {ok, Req} = application:get_env(?APP, Name), - application:set_env(?APP, Name, [{path, Path} | Req]) - end || {Name, {_, _, Path}} <- URLs], - {_, {Host, Port, _}} = lists:last(URLs), - PoolOpts = application:get_env(?APP, pool_opts, []), - application:set_env(?APP, pool_opts, [{host, Host}, {port, Port} | PoolOpts]), - ok; - false -> - {error, different_server} + lists:foreach(fun translate_env/1, [auth_req, super_req, acl_req]). + +translate_env(EnvName) -> + case application:get_env(?APP, EnvName) of + undefined -> ok; + {ok, Req} -> + {ok, PoolSize} = application:get_env(?APP, pool_size), + {ok, ConnectTimeout} = application:get_env(?APP, connect_timeout), + URL = proplists:get_value(url, Req), + #{host := Host0, + path := Path0, + scheme := Scheme} = URIMap = uri_string:parse(add_default_scheme(URL)), + Port = maps:get(port, URIMap, case Scheme of + "https" -> 443; + _ -> 80 + end), + Path = path(Path0), + Host = case inet:parse_address(Host0) of + {ok, {_,_,_,_} = Addr} -> Addr; + {ok, {_,_,_,_,_,_,_,_} = Addr} -> Addr; + {error, einval} -> Host0 + end, + Inet = case Host of + {_,_,_,_} -> inet; + {_,_,_,_,_,_,_,_} -> inet6; + _ -> + case inet:getaddr(Host, inet6) of + {error, _} -> inet; + {ok, _} -> inet6 + end + end, + MoreOpts = case Scheme of + "http" -> + [{transport_opts, [Inet]}]; + "https" -> + CACertFile = application:get_env(?APP, cafile, undefined), + CertFile = application:get_env(?APP, certfile, undefined), + KeyFile = application:get_env(?APP, keyfile, undefined), + TLSOpts = lists:filter(fun({_K, V}) when V =:= <<>> -> + false; + (_) -> + true + end, [{keyfile, KeyFile}, {certfile, CertFile}, {cacertfile, CACertFile}]), + TlsVers = ['tlsv1.2','tlsv1.1',tlsv1], + NTLSOpts = [{versions, TlsVers}, + {ciphers, lists:foldl(fun(TlsVer, Ciphers) -> + Ciphers ++ ssl:cipher_suites(all, TlsVer) + end, [], TlsVers)} | TLSOpts], + [{transport, ssl}, {transport_opts, [Inet | NTLSOpts]}] + end, + PoolOpts = [{host, Host}, + {port, Port}, + {pool_size, PoolSize}, + {pool_type, random}, + {connect_timeout, ConnectTimeout}, + {retry, 5}, + {retry_timeout, 1000}] ++ MoreOpts, + Method = proplists:get_value(method, Req), + Headers = proplists:get_value(headers, Req), + NHeaders = ensure_content_type_header(Method, to_lower(Headers)), + NReq = lists:keydelete(headers, 1, Req), + {ok, Timeout} = application:get_env(?APP, timeout), + application:set_env(?APP, EnvName, [{path, Path}, + {headers, NHeaders}, + {timeout, Timeout}, + {pool_name, list_to_atom("emqx_auth_http/" ++ atom_to_list(EnvName))}, + {pool_opts, PoolOpts} | NReq]) end. -path("") -> "/"; -path(Path) -> Path. - -same_host_and_port([_]) -> - true; -same_host_and_port([{_, {Host, Port, _}}, {_, {Host, Port, _}}]) -> - true; -same_host_and_port([{_, {Host, Port, _}}, URL = {_, {Host, Port, _}} | Rest]) -> - same_host_and_port([URL | Rest]); -same_host_and_port(_) -> - false. - -get_addr(Hostname) -> - case inet:parse_address(Hostname) of - {ok, {_,_,_,_} = Addr} -> Addr; - {ok, {_,_,_,_,_,_,_,_} = Addr} -> Addr; - {error, einval} -> - case inet:getaddr(Hostname, inet) of - {error, _} -> - {ok, Addr} = inet:getaddr(Hostname, inet6), - Addr; - {ok, Addr} -> Addr +load_hooks() -> + case application:get_env(?APP, auth_req) of + undefined -> ok; + {ok, AuthReq} -> + ok = emqx_auth_http:register_metrics(), + PoolOpts = proplists:get_value(pool_opts, AuthReq), + PoolName = proplists:get_value(pool_name, AuthReq), + ehttpc_sup:start_pool(PoolName, PoolOpts), + case application:get_env(?APP, super_req) of + undefined -> + emqx:hook('client.authenticate', {emqx_auth_http, check, [#{auth => maps:from_list(AuthReq), + super => undefined}]}); + {ok, SuperReq} -> + PoolOpts1 = proplists:get_value(pool_opts, SuperReq), + PoolName1 = proplists:get_value(pool_name, SuperReq), + ehttpc_sup:start_pool(PoolName1, PoolOpts1), + emqx:hook('client.authenticate', {emqx_auth_http, check, [#{auth => maps:from_list(AuthReq), + super => maps:from_list(SuperReq)}]}) end - end. + end, + case application:get_env(?APP, acl_req) of + undefined -> ok; + {ok, ACLReq} -> + ok = emqx_acl_http:register_metrics(), + PoolOpts2 = proplists:get_value(pool_opts, ACLReq), + PoolName2 = proplists:get_value(pool_name, ACLReq), + ehttpc_sup:start_pool(PoolName2, PoolOpts2), + emqx:hook('client.check_acl', {emqx_acl_http, check_acl, [#{acl => maps:from_list(ACLReq)}]}) + end, + ok. + +unload_hooks() -> + emqx:unhook('client.authenticate', {emqx_auth_http, check}), + emqx:unhook('client.check_acl', {emqx_acl_http, check_acl}), + ehttpc_sup:stop_pool('emqx_auth_http/auth_req'), + ehttpc_sup:stop_pool('emqx_auth_http/super_req'), + ehttpc_sup:stop_pool('emqx_auth_http/acl_req'), + ok. + +to_lower(Headers) -> + [{string:to_lower(K), V} || {K, V} <- Headers]. + +ensure_content_type_header(Method, Headers) + when Method =:= post orelse Method =:= put -> + Headers; +ensure_content_type_header(_Method, Headers) -> + lists:keydelete("content-type", 1, Headers). + +add_default_scheme(URL) when is_list(URL) -> + binary_to_list(add_default_scheme(list_to_binary(URL))); +add_default_scheme(<<"http://", _/binary>> = URL) -> + URL; +add_default_scheme(<<"https://", _/binary>> = URL) -> + URL; +add_default_scheme(URL) -> + <<"http://", URL/binary>>. + +path("") -> + "/"; +path(Path) -> + Path. + diff --git a/apps/emqx_auth_http/src/emqx_auth_http_cli.erl b/apps/emqx_auth_http/src/emqx_auth_http_cli.erl index c56e45d49..6ff8c445a 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_cli.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_cli.erl @@ -29,16 +29,16 @@ request(PoolName, get, Path, Headers, Params, Timeout) -> NewPath = Path ++ "?" ++ binary_to_list(cow_qs:qs(bin_kw(Params))), - reply(emqx_http_client:request(get, PoolName, {NewPath, Headers}, Timeout)); + reply(ehttpc:request(ehttpc_pool:pick_worker(PoolName), get, {NewPath, Headers}, Timeout)); request(PoolName, post, Path, Headers, Params, Timeout) -> - Body = case proplists:get_value(<<"content-type">>, Headers) of - <<"application/x-www-form-urlencoded">> -> + Body = case proplists:get_value("content-type", Headers) of + "application/x-www-form-urlencoded" -> cow_qs:qs(bin_kw(Params)); - <<"application/json">> -> + "application/json" -> emqx_json:encode(bin_kw(Params)) end, - reply(emqx_http_client:request(post, PoolName, {Path, Headers, Body}, Timeout)). + reply(ehttpc:request(ehttpc_pool:pick_worker(PoolName), post, {Path, Headers, Body}, Timeout)). reply({ok, StatusCode, _Headers}) -> {ok, StatusCode, <<>>}; @@ -80,6 +80,7 @@ feedvar(Params, ClientInfo = #{clientid := ClientId, ({Param, "%A"}) -> {Param, maps:get(access, ClientInfo, null)}; ({Param, "%t"}) -> {Param, maps:get(topic, ClientInfo, null)}; ({Param, "%m"}) -> {Param, maps:get(mountpoint, ClientInfo, null)}; + ({Param, "%k"}) -> {Param, emqx_json:encode(maps:get(ws_cookie, ClientInfo, null))}; ({Param, Var}) -> {Param, Var} end, Params). diff --git a/apps/emqx_auth_http/src/emqx_auth_http_sup.erl b/apps/emqx_auth_http/src/emqx_auth_http_sup.erl new file mode 100644 index 000000000..36b61a224 --- /dev/null +++ b/apps/emqx_auth_http/src/emqx_auth_http_sup.erl @@ -0,0 +1,29 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_auth_http_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + {ok, {{one_for_all, 0, 1}, []}}. diff --git a/apps/emqx_auth_http/src/emqx_http_client.erl b/apps/emqx_auth_http/src/emqx_http_client.erl deleted file mode 100644 index 217ee4b58..000000000 --- a/apps/emqx_auth_http/src/emqx_http_client.erl +++ /dev/null @@ -1,256 +0,0 @@ --module(emqx_http_client). - --behaviour(gen_server). - --include_lib("emqx/include/logger.hrl"). - -%% APIs --export([ start_link/3 - , request/3 - , request/4 - ]). - -%% gen_server callbacks --export([ init/1 - , handle_call/3 - , handle_cast/2 - , handle_info/2 - , terminate/2 - , code_change/3 - ]). - --record(state, { - pool :: ecpool:poo_name(), - id :: pos_integer(), - client :: pid() | undefined, - mref :: reference() | undefined, - host :: inet:hostname() | inet:ip_address(), - port :: inet:port_number(), - gun_opts :: proplists:proplist(), - gun_state :: down | up, - requests :: map() - }). - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -start_link(Pool, Id, Opts) -> - gen_server:start_link(?MODULE, [Pool, Id, Opts], []). - -request(Method, Pool, Req) -> - request(Method, Pool, Req, 5000). - -request(get, Pool, {Path, Headers}, Timeout) -> - call(pick(Pool), {get, {Path, Headers}, Timeout}, Timeout + 1000); -request(Method, Pool, {Path, Headers, Body}, Timeout) -> - call(pick(Pool), {Method, {Path, Headers, Body}, Timeout}, Timeout + 1000). - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([Pool, Id, Opts]) -> - State = #state{pool = Pool, - id = Id, - client = undefined, - mref = undefined, - host = proplists:get_value(host, Opts), - port = proplists:get_value(port, Opts), - gun_opts = gun_opts(Opts), - gun_state = down, - requests = #{}}, - true = gproc_pool:connect_worker(Pool, {Pool, Id}), - {ok, State}. - -handle_call(Req = {_, _, _}, From, State = #state{client = undefined, gun_state = down}) -> - case open(State) of - {ok, NewState} -> - handle_call(Req, From, NewState); - {error, Reason} -> - {reply, {error, Reason}, State} - end; - -handle_call(Req = {_, _, Timeout}, From, State = #state{client = Client, mref = MRef, gun_state = down}) when is_pid(Client) -> - case gun:await_up(Client, Timeout, MRef) of - {ok, _} -> - handle_call(Req, From, State#state{gun_state = up}); - {error, timeout} -> - {reply, {error, timeout}, State}; - {error, Reason} -> - true = erlang:demonitor(MRef, [flush]), - {reply, {error, Reason}, State#state{client = undefined, mref = undefined}} - end; - -handle_call({Method, Request, Timeout}, From, State = #state{client = Client, requests = Requests, gun_state = up}) when is_pid(Client) -> - StreamRef = do_request(Client, Method, Request), - ExpirationTime = erlang:system_time(millisecond) + Timeout, - {noreply, State#state{requests = maps:put(StreamRef, {From, ExpirationTime, undefined}, Requests)}}; - -handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), - {reply, ignored, State}. - -handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), - {noreply, State}. - -handle_info({gun_response, Client, StreamRef, IsFin, StatusCode, Headers}, State = #state{client = Client, requests = Requests}) -> - Now = erlang:system_time(millisecond), - case maps:take(StreamRef, Requests) of - error -> - ?LOG(error, "Received 'gun_response' message from unknown stream ref: ~p", [StreamRef]), - {noreply, State}; - {{_, ExpirationTime, _}, NRequests} when Now > ExpirationTime -> - gun:cancel(Client, StreamRef), - flush_stream(Client, StreamRef), - {noreply, State#state{requests = NRequests}}; - {{From, ExpirationTime, undefined}, NRequests} -> - case IsFin of - fin -> - gen_server:reply(From, {ok, StatusCode, Headers}), - {noreply, State#state{requests = NRequests}}; - nofin -> - {noreply, State#state{requests = NRequests#{StreamRef => {From, ExpirationTime, {StatusCode, Headers, <<>>}}}}} - end; - _ -> - ?LOG(error, "Received 'gun_response' message does not match the state"), - {noreply, State} - end; - -handle_info({gun_data, Client, StreamRef, IsFin, Data}, State = #state{client = Client, requests = Requests}) -> - Now = erlang:system_time(millisecond), - case maps:take(StreamRef, Requests) of - error -> - ?LOG(error, "Received 'gun_data' message from unknown stream ref: ~p", [StreamRef]), - {noreply, State}; - {{_, ExpirationTime, _}, NRequests} when Now > ExpirationTime -> - gun:cancel(Client, StreamRef), - flush_stream(Client, StreamRef), - {noreply, State#state{requests = NRequests}}; - {{From, ExpirationTime, {StatusCode, Headers, Acc}}, NRequests} -> - case IsFin of - fin -> - gen_server:reply(From, {ok, StatusCode, Headers, <>}), - {noreply, State#state{requests = NRequests}}; - nofin -> - {noreply, State#state{requests = NRequests#{StreamRef => {From, ExpirationTime, {StatusCode, Headers, <>}}}}} - end; - _ -> - ?LOG(error, "Received 'gun_data' message does not match the state"), - {noreply, State} - end; - -handle_info({gun_error, Client, StreamRef, Reason}, State = #state{client = Client, requests = Requests}) -> - Now = erlang:system_time(millisecond), - case maps:take(StreamRef, Requests) of - error -> - ?LOG(error, "Received 'gun_error' message from unknown stream ref: ~p~n", [StreamRef]), - {noreply, State}; - {{_, ExpirationTime, _}, NRequests} when Now > ExpirationTime -> - {noreply, State#state{requests = NRequests}}; - {{From, _, _}, NRequests} -> - gen_server:reply(From, {error, Reason}), - {noreply, State#state{requests = NRequests}} - end; - -handle_info({gun_up, Client, _}, State = #state{client = Client}) -> - {noreply, State#state{gun_state = up}}; - -handle_info({gun_down, Client, _, Reason, KilledStreams, _}, State = #state{client = Client, requests = Requests}) -> - Now = erlang:system_time(millisecond), - NRequests = lists:foldl(fun(StreamRef, Acc) -> - case maps:take(StreamRef, Acc) of - error -> Acc; - {{_, ExpirationTime, _}, NAcc} when Now > ExpirationTime -> - NAcc; - {{From, _, _}, NAcc} -> - gen_server:reply(From, {error, Reason}), - NAcc - end - end, Requests, KilledStreams), - {noreply, State#state{gun_state = down, requests = NRequests}}; - -handle_info({'DOWN', MRef, process, Client, Reason}, State = #state{mref = MRef, client = Client, requests = Requests}) -> - true = erlang:demonitor(MRef, [flush]), - Now = erlang:system_time(millisecond), - lists:foreach(fun({_, {_, ExpirationTime, _}}) when Now > ExpirationTime -> - ok; - ({_, {From, _, _}}) -> - gen_server:reply(From, {error, Reason}) - end, maps:to_list(Requests)), - case open(State#state{requests = #{}}) of - {ok, NewState} -> - {noreply, NewState}; - {error, Reason} -> - {noreply, State#state{mref = undefined, client = undefined}} - end; - -handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), - {noreply, State}. - -terminate(_Reason, #state{pool = Pool, id = Id}) -> - gproc_pool:disconnect_worker(Pool, {Pool, Id}), - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -open(State = #state{host = Host, port = Port, gun_opts = GunOpts}) -> - case gun:open(Host, Port, GunOpts) of - {ok, ConnPid} when is_pid(ConnPid) -> - MRef = monitor(process, ConnPid), - {ok, State#state{mref = MRef, client = ConnPid}}; - {error, Reason} -> - {error, Reason} - end. - -gun_opts(Opts) -> - gun_opts(Opts, #{retry => 5, - retry_timeout => 1000, - connect_timeout => 5000, - protocols => [http], - http_opts => #{keepalive => infinity}}). - -gun_opts([], Acc) -> - Acc; -gun_opts([{retry, Retry} | Opts], Acc) -> - gun_opts(Opts, Acc#{retry => Retry}); -gun_opts([{retry_timeout, RetryTimeout} | Opts], Acc) -> - gun_opts(Opts, Acc#{retry_timeout => RetryTimeout}); -gun_opts([{connect_timeout, ConnectTimeout} | Opts], Acc) -> - gun_opts(Opts, Acc#{connect_timeout => ConnectTimeout}); -gun_opts([{transport, Transport} | Opts], Acc) -> - gun_opts(Opts, Acc#{transport => Transport}); -gun_opts([{transport_opts, TransportOpts} | Opts], Acc) -> - gun_opts(Opts, Acc#{transport_opts => TransportOpts}); -gun_opts([_ | Opts], Acc) -> - gun_opts(Opts, Acc). - -call(ChannPid, Msg, Timeout) -> - gen_server:call(ChannPid, Msg, Timeout). - -pick(Pool) -> - gproc_pool:pick_worker(Pool). - -do_request(Client, get, {Path, Headers}) -> - gun:get(Client, Path, Headers); -do_request(Client, post, {Path, Headers, Body}) -> - gun:post(Client, Path, Headers, Body). - -flush_stream(Client, StreamRef) -> - receive - {gun_response, Client, StreamRef, _, _, _} -> - flush_stream(Client, StreamRef); - {gun_data, Client, StreamRef, _, _} -> - flush_stream(Client, StreamRef); - {gun_error, Client, StreamRef, _} -> - flush_stream(Client, StreamRef) - after 0 -> - ok - end. diff --git a/apps/emqx_auth_http/src/emqx_http_client_sup.erl b/apps/emqx_auth_http/src/emqx_http_client_sup.erl deleted file mode 100644 index 1beb1ba36..000000000 --- a/apps/emqx_auth_http/src/emqx_http_client_sup.erl +++ /dev/null @@ -1,48 +0,0 @@ --module(emqx_http_client_sup). - --behaviour(supervisor). - --export([ start_link/2 - , init/1 - , stop_pool/1 - ]). - -start_link(Pool, Opts) -> - supervisor:start_link(?MODULE, [Pool, Opts]). - -init([Pool, Opts]) -> - PoolSize = pool_size(Opts), - ok = ensure_pool(Pool, random, [{size, PoolSize}]), - {ok, {{one_for_one, 10, 100}, [ - begin - ensure_pool_worker(Pool, {Pool, I}, I), - #{id => {Pool, I}, - start => {emqx_http_client, start_link, [Pool, I, Opts]}, - restart => transient, - shutdown => 5000, - type => worker, - modules => [emqx_http_client]} - end || I <- lists:seq(1, PoolSize)]}}. - - -ensure_pool(Pool, Type, Opts) -> - try gproc_pool:new(Pool, Type, Opts) - catch - error:exists -> ok - end. - -ensure_pool_worker(Pool, Name, Slot) -> - try gproc_pool:add_worker(Pool, Name, Slot) - catch - error:exists -> ok - end. - -pool_size(Opts) -> - Schedulers = erlang:system_info(schedulers), - proplists:get_value(pool_size, Opts, Schedulers). - -stop_pool(Name) -> - Workers = gproc_pool:defined_workers(Name), - _ = [gproc_pool:remove_worker(Name, WokerName) || {WokerName, _, _} <- Workers], - gproc_pool:delete(Name), - ok. diff --git a/apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl b/apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl index f9acdc638..235f1b5d8 100644 --- a/apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl +++ b/apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl @@ -68,15 +68,15 @@ set_special_configs(emqx_auth_http, Schema, Inet) -> AuthReq = #{method => post, url => ServerAddr ++ "/mqtt/auth", - content_type => <<"application/json">>, + headers => [{"content-type", "application/json"}], params => [{"clientid", "%c"}, {"username", "%u"}, {"password", "%P"}]}, SuperReq = #{method => post, url => ServerAddr ++ "/mqtt/superuser", - content_type => <<"application/json">>, + headers => [{"content-type", "application/json"}], params => [{"clientid", "%c"}, {"username", "%u"}]}, AclReq = #{method => post, url => ServerAddr ++ "/mqtt/acl", - content_type => <<"application/json">>, + headers => [{"content-type", "application/json"}], params => [{"access", "%A"}, {"username", "%u"}, {"clientid", "%c"}, {"ipaddr", "%a"}, {"topic", "%t"}, {"mountpoint", "%m"}]}, Schema =:= https andalso set_https_client_opts(), @@ -87,9 +87,10 @@ set_special_configs(emqx_auth_http, Schema, Inet) -> %% @private set_https_client_opts() -> - TransportOpts = emqx_ct_helpers:client_ssl_twoway(), - {ok, PoolOpts} = application:get_env(emqx_auth_http, pool_opts), - application:set_env(emqx_auth_http, pool_opts, [{transport_opts, TransportOpts}, {transport, ssl} | PoolOpts]). + SSLOpt = emqx_ct_helpers:client_ssl_twoway(), + application:set_env(emqx_auth_http, cafile, proplists:get_value(cacertfile, SSLOpt, undefined)), + application:set_env(emqx_auth_http, certfile, proplists:get_value(certfile, SSLOpt, undefined)), + application:set_env(emqx_auth_http, keyfile, proplists:get_value(keyfile, SSLOpt, undefined)). %% @private http_server(http, inet) -> "http://127.0.0.1:8991"; @@ -171,3 +172,10 @@ t_comment_config(_) -> ?assertEqual(AuthCount - 1, length(emqx_hooks:lookup('client.authenticate'))), ?assertEqual(AclCount - 1, length(emqx_hooks:lookup('client.check_acl'))). +t_feedvar(_) -> + Params = [{"cookie", "%k"}], + User0 = ?USER(<<"client1">>, <<"testuser">>, mqtt, {127,0,0,1}, external), + ?assertEqual([{"cookie", <<"null">>}], emqx_auth_http_cli:feedvar(Params, User0)), + + User1 = User0#{ws_cookie => [{<<"k">>, <<"v">>}]}, + ?assertEqual([{"cookie", <<"{\"k\":\"v\"}">>}], emqx_auth_http_cli:feedvar(Params, User1)). From e2389ab8e70be0a515a5472410cf03beb1ae9349 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Mon, 18 Jan 2021 16:12:48 +0800 Subject: [PATCH 19/24] chore(CI): update erlang version --- .ci/apps_tests/docker-compose.yaml | 2 +- .ci/build_packages/Dockerfile | 2 +- .ci/compatibility_tests/docker-compose-ldap.yaml | 2 +- .ci/compatibility_tests/docker-compose-mongo-tls.yaml | 2 +- .ci/compatibility_tests/docker-compose-mongo.yaml | 2 +- .ci/compatibility_tests/docker-compose-mysql-tls.yaml | 2 +- .ci/compatibility_tests/docker-compose-mysql.yaml | 2 +- .ci/compatibility_tests/docker-compose-pgsql-tls.yaml | 2 +- .ci/compatibility_tests/docker-compose-pgsql.yaml | 2 +- .../docker-compose-redis-cluster-tls.yaml | 2 +- .../docker-compose-redis-cluster.yaml | 2 +- .../docker-compose-redis-sentinel.yaml | 2 +- .../docker-compose-redis-singer-tls.yaml | 2 +- .../docker-compose-redis-singer.yaml | 2 +- .github/workflows/build_packages.yaml | 10 +++++++--- .github/workflows/run_fvt_tests.yaml | 2 +- 16 files changed, 22 insertions(+), 18 deletions(-) diff --git a/.ci/apps_tests/docker-compose.yaml b/.ci/apps_tests/docker-compose.yaml index 1c0d781e3..d834da5b2 100644 --- a/.ci/apps_tests/docker-compose.yaml +++ b/.ci/apps_tests/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3 + image: erlang:22.3.4.13 depends_on: - mysql_server - redis_server diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile index a950d2a28..adfbb82a2 100644 --- a/.ci/build_packages/Dockerfile +++ b/.ci/build_packages/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=emqx/build-env:erl22.3-ubuntu20.04 +ARG BUILD_FROM=emqx/build-env:erl22.3.4.13-ubuntu20.04 FROM ${BUILD_FROM} ARG EMQX_NAME=emqx diff --git a/.ci/compatibility_tests/docker-compose-ldap.yaml b/.ci/compatibility_tests/docker-compose-ldap.yaml index c41719b8a..1bbdfe2aa 100644 --- a/.ci/compatibility_tests/docker-compose-ldap.yaml +++ b/.ci/compatibility_tests/docker-compose-ldap.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3 + image: erlang:22.3.4.13 depends_on: - ldap_server networks: diff --git a/.ci/compatibility_tests/docker-compose-mongo-tls.yaml b/.ci/compatibility_tests/docker-compose-mongo-tls.yaml index fed0d60d8..a22642a24 100644 --- a/.ci/compatibility_tests/docker-compose-mongo-tls.yaml +++ b/.ci/compatibility_tests/docker-compose-mongo-tls.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3 + image: erlang:22.3.4.13 volumes: - ../../:/emqx working_dir: /emqx diff --git a/.ci/compatibility_tests/docker-compose-mongo.yaml b/.ci/compatibility_tests/docker-compose-mongo.yaml index ec6310eac..8385f1fb0 100644 --- a/.ci/compatibility_tests/docker-compose-mongo.yaml +++ b/.ci/compatibility_tests/docker-compose-mongo.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3 + image: erlang:22.3.4.13 volumes: - ../..:/emqx working_dir: /emqx diff --git a/.ci/compatibility_tests/docker-compose-mysql-tls.yaml b/.ci/compatibility_tests/docker-compose-mysql-tls.yaml index 566240787..50de346c2 100644 --- a/.ci/compatibility_tests/docker-compose-mysql-tls.yaml +++ b/.ci/compatibility_tests/docker-compose-mysql-tls.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3 + image: erlang:22.3.4.13 volumes: - ../../:/emqx working_dir: /emqx diff --git a/.ci/compatibility_tests/docker-compose-mysql.yaml b/.ci/compatibility_tests/docker-compose-mysql.yaml index d49c51977..f7a1b5d39 100644 --- a/.ci/compatibility_tests/docker-compose-mysql.yaml +++ b/.ci/compatibility_tests/docker-compose-mysql.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3 + image: erlang:22.3.4.13 volumes: - ../../:/emqx working_dir: /emqx diff --git a/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml b/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml index c7daf1e38..ff26d88bd 100644 --- a/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml +++ b/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3 + image: erlang:22.3.4.13 volumes: - ../../:/emqx working_dir: /emqx diff --git a/.ci/compatibility_tests/docker-compose-pgsql.yaml b/.ci/compatibility_tests/docker-compose-pgsql.yaml index a1f578eaa..f76768e8e 100644 --- a/.ci/compatibility_tests/docker-compose-pgsql.yaml +++ b/.ci/compatibility_tests/docker-compose-pgsql.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3 + image: erlang:22.3.4.13 volumes: - ../../:/emqx working_dir: /emqx diff --git a/.ci/compatibility_tests/docker-compose-redis-cluster-tls.yaml b/.ci/compatibility_tests/docker-compose-redis-cluster-tls.yaml index 0048d94f5..592f857cc 100644 --- a/.ci/compatibility_tests/docker-compose-redis-cluster-tls.yaml +++ b/.ci/compatibility_tests/docker-compose-redis-cluster-tls.yaml @@ -5,7 +5,7 @@ version: '2.4' services: erlang: container_name: erlang - image: erlang:22.3 + image: erlang:22.3.4.13 volumes: - ../..:/emqx networks: diff --git a/.ci/compatibility_tests/docker-compose-redis-cluster.yaml b/.ci/compatibility_tests/docker-compose-redis-cluster.yaml index 01d4b7cb7..fa4a93d90 100644 --- a/.ci/compatibility_tests/docker-compose-redis-cluster.yaml +++ b/.ci/compatibility_tests/docker-compose-redis-cluster.yaml @@ -5,7 +5,7 @@ version: '2.4' services: erlang: container_name: erlang - image: erlang:22.3 + image: erlang:22.3.4.13 volumes: - ../..:/emqx networks: diff --git a/.ci/compatibility_tests/docker-compose-redis-sentinel.yaml b/.ci/compatibility_tests/docker-compose-redis-sentinel.yaml index 54c621130..2cc3743c0 100644 --- a/.ci/compatibility_tests/docker-compose-redis-sentinel.yaml +++ b/.ci/compatibility_tests/docker-compose-redis-sentinel.yaml @@ -5,7 +5,7 @@ version: '2.4' services: erlang: container_name: erlang - image: erlang:22.3 + image: erlang:22.3.4.13 volumes: - ../..:/emqx networks: diff --git a/.ci/compatibility_tests/docker-compose-redis-singer-tls.yaml b/.ci/compatibility_tests/docker-compose-redis-singer-tls.yaml index 643f18755..22d44a035 100644 --- a/.ci/compatibility_tests/docker-compose-redis-singer-tls.yaml +++ b/.ci/compatibility_tests/docker-compose-redis-singer-tls.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3 + image: erlang:22.3.4.13 volumes: - ../..:/emqx networks: diff --git a/.ci/compatibility_tests/docker-compose-redis-singer.yaml b/.ci/compatibility_tests/docker-compose-redis-singer.yaml index 2f7c73d53..bc9b01d12 100644 --- a/.ci/compatibility_tests/docker-compose-redis-singer.yaml +++ b/.ci/compatibility_tests/docker-compose-redis-singer.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3 + image: erlang:22.3.4.13 volumes: - ../..:/emqx networks: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 108ac29e0..8beaa9e57 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -153,13 +153,15 @@ jobs: docker run --rm --privileged tonistiigi/binfmt --install all - uses: actions/checkout@v1 - name: get deps + env: + ERL_OTP: erl22.3.4.13 run: | docker run -i --rm \ -e GITHUB_RUN_ID=$GITHUB_RUN_ID \ -e GITHUB_REF=$GITHUB_REF \ -v $(pwd):/emqx \ -w /emqx \ - emqx/build-env:erl22.3-debian10 \ + emqx/build-env:${ERL_OTP}-debian10 \ bash -c "make deps-all" - name: downloads emqx zip packages env: @@ -188,7 +190,7 @@ jobs: - name: build emqx packages if: (matrix.arch == 'amd64' && matrix.emqx == 'emqx') || startsWith(github.ref, 'refs/tags/') env: - ERL_OTP: erl22.3 + ERL_OTP: erl22.3.4.13 EMQX: ${{ matrix.emqx }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} @@ -241,13 +243,15 @@ jobs: steps: - uses: actions/checkout@v1 - name: get deps + env: + ERL_OTP: erl22.3.4.13 run: | docker run -i --rm \ -e GITHUB_RUN_ID=$GITHUB_RUN_ID \ -e GITHUB_REF=$GITHUB_REF \ -v $(pwd):/emqx \ -w /emqx \ - emqx/build-env:erl22.3-alpine-amd64 \ + emqx/build-env:${ERL_OTP}-alpine-amd64 \ sh -c "make deps-emqx" - name: build emqx docker image env: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index c76108633..e8ca663ea 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -111,7 +111,7 @@ jobs: relup_test: runs-on: ubuntu-20.04 - container: emqx/build-env:erl22.3-ubuntu20.04 + container: emqx/build-env:erl22.3.4.13-ubuntu20.04 defaults: run: shell: bash From c6785b1a34272eb129e832364fb2a97137e1363b Mon Sep 17 00:00:00 2001 From: Yudai Kiyofuji <57182816+z8674558@users.noreply.github.com> Date: Thu, 21 Jan 2021 11:03:06 +0900 Subject: [PATCH 20/24] test(coap): add test on acl and connection discarding (#4039) --- apps/emqx_coap/test/emqx_coap_SUITE.erl | 43 +++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/apps/emqx_coap/test/emqx_coap_SUITE.erl b/apps/emqx_coap/test/emqx_coap_SUITE.erl index b8d01cf4c..672113e57 100644 --- a/apps/emqx_coap/test/emqx_coap_SUITE.erl +++ b/apps/emqx_coap/test/emqx_coap_SUITE.erl @@ -218,6 +218,46 @@ t_invalid_topic(_Config) -> Reply4 = er_coap_client:request(put, URI4, #coap_content{format = <<"application/octet-stream">>, payload = Payload3}), ?assertMatch({error,bad_request}, Reply4). +% mqtt connection kicked by coap with same client id +t_kick_1(_Config) -> + URI = "coap://127.0.0.1/mqtt/abc?c=clientid&u=tom&p=secret", + % workaround: emqx:subscribe does not kick same client id. + spawn_monitor(fun() -> + {ok, C} = emqtt:start_link([{host, "localhost"}, + {clientid, <<"clientid">>}, + {username, <<"plain">>}, + {password, <<"plain">>}]), + {ok, _} = emqtt:connect(C) end), + er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, + payload = <<"123">>}), + receive + {'DOWN', _, _, _, _} -> ok + after 2000 -> + ?assert(false) + end. + +% mqtt connection kicked by coap with same client id +t_acl(Config) -> + %% Update acl file and reload mod_acl_internal + Path = filename:join([testdir(proplists:get_value(data_dir, Config)), "deny.conf"]), + ok = file:write_file(Path, <<"{deny, {user, \"coap\"}, publish, [\"abc\"]}.">>), + OldPath = emqx:get_env(acl_file), + emqx_mod_acl_internal:reload([{acl_file, Path}]), + + emqx:subscribe(<<"abc">>), + URI = "coap://127.0.0.1/mqtt/adbc?c=client1&u=coap&p=secret", + er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, + payload = <<"123">>}), + receive + _Something -> ?assert(false) + after 2000 -> + ok + end, + + application:set_env(emqx, acl_file, OldPath), + file:delete(Path), + emqx_mod_acl_internal:reload([{acl_file, OldPath}]). + t_stats(_) -> ok. @@ -238,3 +278,6 @@ receive_notification() -> receive_notification_timeout end. +testdir(DataPath) -> + Ls = filename:split(DataPath), + filename:join(lists:sublist(Ls, 1, length(Ls) - 1)). From 5aa63ed2032cce119f1ac3bbb7eb559acd84dd52 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 20 Jan 2021 08:35:50 +0100 Subject: [PATCH 21/24] fix(shared_sub): align default value and default config Shared subscription default dispatch strategy is 'random' in config but 'round_robin' in code. This commit is to make sure they are the same: both are 'random'. --- src/emqx_shared_sub.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqx_shared_sub.erl b/src/emqx_shared_sub.erl index 7ec03cd87..f4b9bf78c 100644 --- a/src/emqx_shared_sub.erl +++ b/src/emqx_shared_sub.erl @@ -126,7 +126,7 @@ dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) -> -spec(strategy() -> random | round_robin | sticky | hash). strategy() -> - emqx:get_env(shared_subscription_strategy, round_robin). + emqx:get_env(shared_subscription_strategy, random). -spec(ack_enabled() -> boolean()). ack_enabled() -> From 0087303b251baa549921599a23b66524daaaad8f Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 20 Jan 2021 08:42:31 +0100 Subject: [PATCH 22/24] feat(shared_sub): Support hashing from source topic. --- etc/emqx.conf | 4 ++- src/emqx_shared_sub.erl | 46 ++++++++++++++++++++++------------ test/emqx_shared_sub_SUITE.erl | 43 ++++++++++++++++++++++++++++++- 3 files changed, 75 insertions(+), 18 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index bf6e0470f..343622336 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -2122,7 +2122,9 @@ broker.session_locking_strategy = quorum ## - random ## - round_robin ## - sticky -## - hash +## - hash # same as hash_clientid +## - hash_clientid +## - hash_topic broker.shared_subscription_strategy = random ## Enable/disable shared dispatch acknowledgement for QoS1 and QoS2 messages diff --git a/src/emqx_shared_sub.erl b/src/emqx_shared_sub.erl index f4b9bf78c..edc276f2a 100644 --- a/src/emqx_shared_sub.erl +++ b/src/emqx_shared_sub.erl @@ -58,6 +58,15 @@ , code_change/3 ]). +-export_type([strategy/0]). + +-type strategy() :: random + | round_robin + | sticky + | hash %% same as hash_clientid, backward compatible + | hash_clientid + | hash_topic. + -define(SERVER, ?MODULE). -define(TAB, emqx_shared_subscription). -define(SHARED_SUBS, emqx_shared_subscriber). @@ -111,8 +120,8 @@ dispatch(Group, Topic, Delivery) -> dispatch(Group, Topic, Delivery, _FailedSubs = []). dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) -> - #message{from = ClientId} = Msg, - case pick(strategy(), ClientId, Group, Topic, FailedSubs) of + #message{from = ClientId, topic = SourceTopic} = Msg, + case pick(strategy(), ClientId, SourceTopic, Group, Topic, FailedSubs) of false -> {error, no_subscribers}; {Type, SubPid} -> @@ -124,7 +133,7 @@ dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) -> end end. --spec(strategy() -> random | round_robin | sticky | hash). +-spec(strategy() -> strategy()). strategy() -> emqx:get_env(shared_subscription_strategy, random). @@ -226,7 +235,7 @@ maybe_ack(Msg) -> without_ack_ref(Msg) end. -pick(sticky, ClientId, Group, Topic, FailedSubs) -> +pick(sticky, ClientId, SourceTopic, Group, Topic, FailedSubs) -> Sub0 = erlang:get({shared_sub_sticky, Group, Topic}), case is_active_sub(Sub0, FailedSubs) of true -> @@ -235,15 +244,15 @@ pick(sticky, ClientId, Group, Topic, FailedSubs) -> {fresh, Sub0}; false -> %% randomly pick one for the first message - {Type, Sub} = do_pick(random, ClientId, Group, Topic, [Sub0 | FailedSubs]), + {Type, Sub} = do_pick(random, ClientId, SourceTopic, Group, Topic, [Sub0 | FailedSubs]), %% stick to whatever pick result erlang:put({shared_sub_sticky, Group, Topic}, Sub), {Type, Sub} end; -pick(Strategy, ClientId, Group, Topic, FailedSubs) -> - do_pick(Strategy, ClientId, Group, Topic, FailedSubs). +pick(Strategy, ClientId, SourceTopic, Group, Topic, FailedSubs) -> + do_pick(Strategy, ClientId, SourceTopic, Group, Topic, FailedSubs). -do_pick(Strategy, ClientId, Group, Topic, FailedSubs) -> +do_pick(Strategy, ClientId, SourceTopic, Group, Topic, FailedSubs) -> All = subscribers(Group, Topic), case All -- FailedSubs of [] when All =:= [] -> @@ -251,22 +260,27 @@ do_pick(Strategy, ClientId, Group, Topic, FailedSubs) -> false; [] -> %% All offline? pick one anyway - {retry, pick_subscriber(Group, Topic, Strategy, ClientId, All)}; + {retry, pick_subscriber(Group, Topic, Strategy, ClientId, SourceTopic, All)}; Subs -> %% More than one available - {fresh, pick_subscriber(Group, Topic, Strategy, ClientId, Subs)} + {fresh, pick_subscriber(Group, Topic, Strategy, ClientId, SourceTopic, Subs)} end. -pick_subscriber(_Group, _Topic, _Strategy, _ClientId, [Sub]) -> Sub; -pick_subscriber(Group, Topic, Strategy, ClientId, Subs) -> - Nth = do_pick_subscriber(Group, Topic, Strategy, ClientId, length(Subs)), +pick_subscriber(_Group, _Topic, _Strategy, _ClientId, _SourceTopic, [Sub]) -> Sub; +pick_subscriber(Group, Topic, Strategy, ClientId, SourceTopic, Subs) -> + Nth = do_pick_subscriber(Group, Topic, Strategy, ClientId, SourceTopic, length(Subs)), lists:nth(Nth, Subs). -do_pick_subscriber(_Group, _Topic, random, _ClientId, Count) -> +do_pick_subscriber(_Group, _Topic, random, _ClientId, _SourceTopic, Count) -> rand:uniform(Count); -do_pick_subscriber(_Group, _Topic, hash, ClientId, Count) -> +do_pick_subscriber(Group, Topic, hash, ClientId, SourceTopic, Count) -> + %% backward compatible + do_pick_subscriber(Group, Topic, hash_clientid, ClientId, SourceTopic, Count); +do_pick_subscriber(_Group, _Topic, hash_clientid, ClientId, _SourceTopic, Count) -> 1 + erlang:phash2(ClientId) rem Count; -do_pick_subscriber(Group, Topic, round_robin, _ClientId, Count) -> +do_pick_subscriber(_Group, _Topic, hash_topic, _ClientId, SourceTopic, Count) -> + 1 + erlang:phash2(SourceTopic) rem Count; +do_pick_subscriber(Group, Topic, round_robin, _ClientId, _SourceTopic, Count) -> Rem = case erlang:get({shared_sub_round_robin, Group, Topic}) of undefined -> 0; N -> (N + 1) rem Count diff --git a/test/emqx_shared_sub_SUITE.erl b/test/emqx_shared_sub_SUITE.erl index 71f25e96e..c2d7468d9 100644 --- a/test/emqx_shared_sub_SUITE.erl +++ b/test/emqx_shared_sub_SUITE.erl @@ -169,6 +169,47 @@ t_sticky(_) -> t_hash(_) -> test_two_messages(hash, false). +t_hash_clinetid(_) -> + test_two_messages(hash_clientid, false). + +t_hash_topic(_) -> + ok = ensure_config(hash_topic, false), + ClientId1 = <<"ClientId1">>, + ClientId2 = <<"ClientId2">>, + {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}]), + {ok, _} = emqtt:connect(ConnPid1), + {ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2}]), + {ok, _} = emqtt:connect(ConnPid2), + + Topic1 = <<"foo/bar1">>, + Topic2 = <<"foo/bar2">>, + ?assert(erlang:phash2(Topic1) rem 2 =/= erlang:phash2(Topic2) rem 2), + Message1 = emqx_message:make(ClientId1, 0, Topic1, <<"hello1">>), + Message2 = emqx_message:make(ClientId1, 0, Topic2, <<"hello2">>), + emqtt:subscribe(ConnPid1, {<<"$share/group1/foo/#">>, 0}), + emqtt:subscribe(ConnPid2, {<<"$share/group1/foo/#">>, 0}), + ct:sleep(100), + emqx:publish(Message1), + Me = self(), + WaitF = fun(ExpectedPayload) -> + case last_message(ExpectedPayload, [ConnPid1, ConnPid2]) of + {true, Pid} -> + Me ! {subscriber, Pid}, + true; + Other -> + Other + end + end, + WaitF(<<"hello1">>), + UsedSubPid1 = receive {subscriber, P1} -> P1 end, + emqx_broker:publish(Message2), + WaitF(<<"hello2">>), + UsedSubPid2 = receive {subscriber, P2} -> P2 end, + ?assert(UsedSubPid1 =/= UsedSubPid2), + emqtt:stop(ConnPid1), + emqtt:stop(ConnPid2), + ok. + %% if the original subscriber dies, change to another one alive t_not_so_sticky(_) -> ok = ensure_config(sticky), @@ -246,7 +287,7 @@ last_message(ExpectedPayload, Pids) -> after 100 -> <<"not yet?">> end. - + t_dispatch(_) -> ok = ensure_config(random), Topic = <<"foo">>, From ca32ac310fa5f140e230f2eb5fb9db57382fd888 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 20 Jan 2021 12:31:51 +0100 Subject: [PATCH 23/24] chore(style): make elvis happy --- scripts/elvis-check.sh | 2 +- test/emqx_shared_sub_SUITE.erl | 22 +++++++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/scripts/elvis-check.sh b/scripts/elvis-check.sh index 9e6fb7320..7ba8b72eb 100755 --- a/scripts/elvis-check.sh +++ b/scripts/elvis-check.sh @@ -4,7 +4,7 @@ set -euo pipefail ELVIS_VERSION='1.0.0-emqx-1' -base=${GITHUB_BASE_REF:-$1} +base="${GITHUB_BASE_REF:-$1}" elvis_version="${2:-$ELVIS_VERSION}" echo "elvis -v: $elvis_version" diff --git a/test/emqx_shared_sub_SUITE.erl b/test/emqx_shared_sub_SUITE.erl index c2d7468d9..2e8e624c6 100644 --- a/test/emqx_shared_sub_SUITE.erl +++ b/test/emqx_shared_sub_SUITE.erl @@ -47,16 +47,21 @@ t_is_ack_required(_) -> t_maybe_nack_dropped(_) -> ?assertEqual(ok, emqx_shared_sub:maybe_nack_dropped(#message{headers = #{}})), - ?assertEqual(ok, emqx_shared_sub:maybe_nack_dropped(#message{headers = #{shared_dispatch_ack => {self(), for_test}}})), + Msg = #message{headers = #{shared_dispatch_ack => {self(), for_test}}}, + ?assertEqual(ok, emqx_shared_sub:maybe_nack_dropped(Msg)), ?assertEqual(ok,receive {for_test, {shared_sub_nack, dropped}} -> ok after 100 -> timeout end). t_nack_no_connection(_) -> - ?assertEqual(ok, emqx_shared_sub:nack_no_connection(#message{headers = #{shared_dispatch_ack => {self(), for_test}}})), - ?assertEqual(ok,receive {for_test, {shared_sub_nack, no_connection}} -> ok after 100 -> timeout end). + Msg = #message{headers = #{shared_dispatch_ack => {self(), for_test}}}, + ?assertEqual(ok, emqx_shared_sub:nack_no_connection(Msg)), + ?assertEqual(ok,receive {for_test, {shared_sub_nack, no_connection}} -> ok + after 100 -> timeout end). t_maybe_ack(_) -> ?assertEqual(#message{headers = #{}}, emqx_shared_sub:maybe_ack(#message{headers = #{}})), - ?assertEqual(#message{headers = #{shared_dispatch_ack => ?no_ack}}, emqx_shared_sub:maybe_ack(#message{headers = #{shared_dispatch_ack => {self(), for_test}}})), + Msg = #message{headers = #{shared_dispatch_ack => {self(), for_test}}}, + ?assertEqual(#message{headers = #{shared_dispatch_ack => ?no_ack}}, + emqx_shared_sub:maybe_ack(Msg)), ?assertEqual(ok,receive {for_test, ?ack} -> ok after 100 -> timeout end). % t_subscribers(_) -> @@ -124,7 +129,8 @@ t_no_connection_nack(_) -> %% This is the connection which was picked by broker to dispatch (sticky) for 1st message ?assertMatch([#{packet_id := 1}], recv_msgs(1)), - %% Now kill the connection, expect all following messages to be delivered to the other subscriber. + %% Now kill the connection, expect all following messages to be delivered to the other + %% subscriber. %emqx_mock_client:stop(ConnPid), %% sleep then make synced calls to session processes to ensure that %% the connection pid's 'EXIT' message is propagated to the session process @@ -291,9 +297,11 @@ last_message(ExpectedPayload, Pids) -> t_dispatch(_) -> ok = ensure_config(random), Topic = <<"foo">>, - ?assertEqual({error, no_subscribers}, emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}})), + ?assertEqual({error, no_subscribers}, + emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}})), emqx:subscribe(Topic, #{qos => 2, share => <<"group1">>}), - ?assertEqual({ok, 1}, emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}})). + ?assertEqual({ok, 1}, + emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}})). % t_unsubscribe(_) -> % error('TODO'). From ed25c62fed56b11aa7feb6dc1231312af6995cc6 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 20 Jan 2021 12:37:21 +0100 Subject: [PATCH 24/24] chore(elvis): Add usage print --- scripts/elvis-check.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/elvis-check.sh b/scripts/elvis-check.sh index 7ba8b72eb..0e105bf99 100755 --- a/scripts/elvis-check.sh +++ b/scripts/elvis-check.sh @@ -1,10 +1,18 @@ #!/bin/bash +## This script checks style of changed files. +## Expect argument 1 to be the git compare base. + set -euo pipefail ELVIS_VERSION='1.0.0-emqx-1' -base="${GITHUB_BASE_REF:-$1}" +base="${1:-}" +if [ "${base}" = "" ]; then + echo "Usage $0 " + exit 1 +fi + elvis_version="${2:-$ELVIS_VERSION}" echo "elvis -v: $elvis_version"