From 9087f0c1386b32852ea26f9682738c4196a5f6a2 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Thu, 11 Mar 2021 14:59:55 +0900 Subject: [PATCH 001/158] feat(emqx_message): add from_map --- src/emqx_message.erl | 37 ++++++++++++++++++++++++++++++++++++- test/emqx_message_SUITE.erl | 12 ++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/emqx_message.erl b/src/emqx_message.erl index decbb0c18..3bc7c2d3f 100644 --- a/src/emqx_message.erl +++ b/src/emqx_message.erl @@ -67,8 +67,21 @@ -export([ to_packet/2 , to_map/1 , to_list/1 + , from_map/1 ]). +-export_type([message_map/0]). + +-type(message_map() :: #{id := binary(), + qos := 0 | 1 | 2, + from := atom() | binary(), + flags := emqx_types:flags(), + headers := emqx_types:headers(), + topic := emqx_types:topic(), + payload := emqx_types:payload(), + timestamp := integer()} + ). + -export([format/1]). -spec(make(emqx_topic:topic(), emqx_types:payload()) -> emqx_types:message()). @@ -266,7 +279,7 @@ filter_pub_props(Props) -> ], Props). %% @doc Message to map --spec(to_map(emqx_types:message()) -> map()). +-spec(to_map(emqx_types:message()) -> message_map()). to_map(#message{ id = Id, qos = QoS, @@ -292,6 +305,28 @@ to_map(#message{ to_list(Msg) -> lists:zip(record_info(fields, message), tl(tuple_to_list(Msg))). +%% @doc Map to message +-spec(from_map(message_map()) -> emqx_types:message()). +from_map(#{id := Id, + qos := QoS, + from := From, + flags := Flags, + headers := Headers, + topic := Topic, + payload := Payload, + timestamp := Timestamp + }) -> + #message{ + id = Id, + qos = QoS, + from = From, + flags = Flags, + headers = Headers, + topic = Topic, + payload = Payload, + timestamp = Timestamp + }. + %% MilliSeconds elapsed(Since) -> max(0, erlang:system_time(millisecond) - Since). diff --git a/test/emqx_message_SUITE.erl b/test/emqx_message_SUITE.erl index fec7967c8..bbead8f7e 100644 --- a/test/emqx_message_SUITE.erl +++ b/test/emqx_message_SUITE.erl @@ -210,3 +210,15 @@ t_to_map(_) -> ?assertEqual(List, emqx_message:to_list(Msg)), ?assertEqual(maps:from_list(List), emqx_message:to_map(Msg)). +t_from_map(_) -> + Msg = emqx_message:make(<<"clientid">>, ?QOS_1, <<"topic">>, <<"payload">>), + Map = #{id => emqx_message:id(Msg), + qos => ?QOS_1, + from => <<"clientid">>, + flags => #{}, + headers => #{}, + topic => <<"topic">>, + payload => <<"payload">>, + timestamp => emqx_message:timestamp(Msg)}, + ?assertEqual(Map, emqx_message:to_map(Msg)), + ?assertEqual(Msg, emqx_message:from_map(emqx_message:to_map(Msg))). From e9180b9ce874d82ee14cc84076aeb82d591653fb Mon Sep 17 00:00:00 2001 From: z8674558 Date: Thu, 11 Mar 2021 15:59:35 +0900 Subject: [PATCH 002/158] chore(emqx_messages): fix elvis --- src/emqx_message.erl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/emqx_message.erl b/src/emqx_message.erl index 3bc7c2d3f..3050ae517 100644 --- a/src/emqx_message.erl +++ b/src/emqx_message.erl @@ -84,6 +84,8 @@ -export([format/1]). +-elvis([{elvis_style, god_modules, disable}]). + -spec(make(emqx_topic:topic(), emqx_types:payload()) -> emqx_types:message()). make(Topic, Payload) -> make(undefined, Topic, Payload). @@ -241,8 +243,9 @@ is_expired(#message{headers = #{properties := #{'Message-Expiry-Interval' := Int is_expired(_Msg) -> false. -spec(update_expiry(emqx_types:message()) -> emqx_types:message()). -update_expiry(Msg = #message{headers = #{properties := Props = #{'Message-Expiry-Interval' := Interval}}, +update_expiry(Msg = #message{headers = #{properties := #{'Message-Expiry-Interval' := Interval}}, timestamp = CreatedAt}) -> + Props = maps:get(properties, Msg#message.headers), case elapsed(CreatedAt) of Elapsed when Elapsed > 0 -> Interval1 = max(1, Interval - (Elapsed div 1000)), @@ -263,7 +266,8 @@ to_packet(PacketId, Msg = #message{qos = QoS, headers = Headers, }, variable = #mqtt_packet_publish{topic_name = Topic, packet_id = PacketId, - properties = filter_pub_props(maps:get(properties, Headers, #{})) + properties = filter_pub_props( + maps:get(properties, Headers, #{})) }, payload = Payload }. @@ -331,7 +335,12 @@ from_map(#{id := Id, elapsed(Since) -> max(0, erlang:system_time(millisecond) - Since). -format(#message{id = Id, qos = QoS, topic = Topic, from = From, flags = Flags, headers = Headers}) -> +format(#message{id = Id, + qos = QoS, + topic = Topic, + from = From, + flags = Flags, + headers = Headers}) -> io_lib:format("Message(Id=~s, QoS=~w, Topic=~s, From=~p, Flags=~s, Headers=~s)", [Id, QoS, Topic, From, format(flags, Flags), format(headers, Headers)]). From 969110dc537c9b798902330da2426d386ba3db5a Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 11 Mar 2021 11:58:07 +0100 Subject: [PATCH 003/158] chore(test): Add snabbkaffe --- rebar.config.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/rebar.config.erl b/rebar.config.erl index ddfaa7133..ceef906b7 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -80,6 +80,7 @@ test_plugins() -> test_deps() -> [ {bbmustache, "1.10.0"} , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.3.6"}}} + , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.8.2"}}} , meck ]. From 173a4d8feaca14e7fb8e55ed5aba73fd7926ad94 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 11 Mar 2021 15:11:39 +0100 Subject: [PATCH 004/158] chore(build): Do not use test profile dedicated plugins --- rebar.config.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rebar.config.erl b/rebar.config.erl index ceef906b7..7fb892124 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -70,7 +70,10 @@ project_app_dirs() -> plugins() -> [ {relup_helper,{git,"https://github.com/emqx/relup_helper", {branch,"master"}}}, {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}} - ]. + ] + %% test plugins are concatenated to default profile plugins + %% otherwise rebar3 test profile runs are super slow + ++ test_plugins(). test_plugins() -> [ rebar3_proper, @@ -119,7 +122,6 @@ profiles() -> , {check, [ {erl_opts, test_compile_opts()} ]} , {test, [ {deps, test_deps()} - , {plugins, test_plugins()} , {erl_opts, test_compile_opts() ++ erl_opts_i()} , {extra_src_dirs, [{"test", [{recursive,true}]}]} ]} From 5f064a8e42cbb23e5d1ec471aa468e33ab8e5846 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 11 Mar 2021 17:28:52 +0100 Subject: [PATCH 005/158] chore(test): enable cover only when necessary So that ad-hoc ./rebar3 eunit and ct can run faster --- Makefile | 10 +++++----- rebar.config | 1 - rebar.config.erl | 10 +++++++++- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 69682f29c..97072f81a 100644 --- a/Makefile +++ b/Makefile @@ -33,23 +33,23 @@ get-dashboard: .PHONY: eunit eunit: $(REBAR) - @$(REBAR) eunit -v -c + @ENABLE_COVER_COMPILE=1 $(REBAR) eunit -v -c .PHONY: proper proper: $(REBAR) - @$(REBAR) as test proper -d test/props -c + @ENABLE_COVER_COMPILE=1 $(REBAR) as test proper -d test/props -c .PHONY: ct ct: $(REBAR) - @$(REBAR) ct --name 'test@127.0.0.1' -c -v + @ENABLE_COVER_COMPILE=1 $(REBAR) ct --name 'test@127.0.0.1' -c -v .PHONY: cover cover: $(REBAR) - @$(REBAR) cover + @ENABLE_COVER_COMPILE=1 $(REBAR) cover .PHONY: coveralls coveralls: $(REBAR) - @$(REBAR) as test coveralls send + @ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send .PHONY: $(REL_PROFILES) $(REL_PROFILES:%=%): $(REBAR) get-dashboard diff --git a/rebar.config b/rebar.config index f1fe1d3b2..4b99dde2d 100644 --- a/rebar.config +++ b/rebar.config @@ -24,7 +24,6 @@ ] }. -{cover_enabled, true}. {cover_opts, [verbose]}. {cover_export_enabled, true}. {cover_excl_mods, [emqx_exproto_pb, emqx_exhook_pb]}. diff --git a/rebar.config.erl b/rebar.config.erl index 7fb892124..47ba3a3d9 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -50,11 +50,19 @@ community_plugin_overrides() -> [{add, App, [ {erl_opts, [{i, "include"}]}]} || App <- relx_plugin_apps_extra()]. config() -> - [ {plugins, plugins()} + [ {cover_enabled, is_cover_enabled()} + , {plugins, plugins()} , {profiles, profiles()} , {project_app_dirs, project_app_dirs()} ]. +is_cover_enabled() -> + case os:getenv("ENABLE_COVER_COMPILE") of + "1"-> true; + "true" -> true; + _ -> false + end. + is_enterprise() -> filelib:is_regular("EMQX_ENTERPRISE"). From 7138e3a3a9048e551b500d895777043556e71959 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 12 Mar 2021 08:18:51 +0100 Subject: [PATCH 006/158] chore(build): Pin a tag of relup_helper plugin Otherwise we are doomed when we need to re-build an old version in the future --- rebar.config.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.erl b/rebar.config.erl index 47ba3a3d9..f1623dfca 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -76,7 +76,7 @@ project_app_dirs() -> ["apps/*", alternative_lib_dir() ++ "/*", "."]. plugins() -> - [ {relup_helper,{git,"https://github.com/emqx/relup_helper", {branch,"master"}}}, + [ {relup_helper,{git,"https://github.com/emqx/relup_helper", {tag, "2.0.0"}}}, {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}} ] %% test plugins are concatenated to default profile plugins From f7b34cb09802abf65d1cb7e7a22893d5caa7595d Mon Sep 17 00:00:00 2001 From: wwhai Date: Sat, 6 Mar 2021 15:50:32 +0800 Subject: [PATCH 007/158] feat(rule-engine): list rule support order by timestrap --- apps/emqx_rule_engine/include/rule_engine.hrl | 1 + .../emqx_rule_engine/src/emqx_rule_engine.erl | 1 + .../src/emqx_rule_engine_api.erl | 6 +- .../src/emqx_rule_engine_cli.erl | 2 +- .../src/emqx_rule_registry.erl | 10 ++++ .../test/emqx_rule_engine_SUITE.erl | 58 +++++++++++++------ 6 files changed, 57 insertions(+), 21 deletions(-) diff --git a/apps/emqx_rule_engine/include/rule_engine.hrl b/apps/emqx_rule_engine/include/rule_engine.hrl index 97263099d..ccc2685e1 100644 --- a/apps/emqx_rule_engine/include/rule_engine.hrl +++ b/apps/emqx_rule_engine/include/rule_engine.hrl @@ -79,6 +79,7 @@ , on_action_failed :: continue | stop , actions :: list(#action_instance{}) , enabled :: boolean() + , created_at :: integer() %% epoch in millisecond precision , description :: binary() }). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 860b9e702..12f00c191 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -179,6 +179,7 @@ create_rule(Params = #{rawsql := Sql, actions := ActArgs}) -> on_action_failed = maps:get(on_action_failed, Params, continue), actions = Actions, enabled = Enabled, + created_at = erlang:system_time(millisecond), description = maps:get(description, Params, "") }, ok = emqx_rule_registry:add_rule(Rule), diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index beb1204df..e7287d98d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -230,7 +230,7 @@ update_rule(#{id := Id}, Params) -> end. list_rules(_Bindings, _Params) -> - return_all(emqx_rule_registry:get_rules()). + return_all(emqx_rule_registry:get_rules_ordered_by_ts()). show_rule(#{id := Id}, _Params) -> reply_with(fun emqx_rule_registry:get_rule/1, Id). @@ -495,8 +495,8 @@ parse_rule_params([{<<"actions">>, Actions} | Params], Rule) -> parse_rule_params(Params, Rule#{actions => parse_actions(Actions)}); parse_rule_params([{<<"description">>, Descr} | Params], Rule) -> parse_rule_params(Params, Rule#{description => Descr}); -parse_rule_params([_ | Params], Res) -> - parse_rule_params(Params, Res). +parse_rule_params([_ | Params], Rule) -> + parse_rule_params(Params, Rule). on_failed(<<"continue">>) -> continue; on_failed(<<"stop">>) -> stop; diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl index 9edc198f9..dbcf3e0e5 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl @@ -98,7 +98,7 @@ unload() -> %%----------------------------------------------------------------------------- -dialyzer([{nowarn_function, [rules/1]}]). rules(["list"]) -> - print_all(emqx_rule_registry:get_rules()); + print_all(emqx_rule_registry:get_rules_ordered_by_ts()); rules(["show", RuleId]) -> print_with(fun emqx_rule_registry:get_rule/1, list_to_binary(RuleId)); diff --git a/apps/emqx_rule_engine/src/emqx_rule_registry.erl b/apps/emqx_rule_engine/src/emqx_rule_registry.erl index dc7a33805..80667f995 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_registry.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_registry.erl @@ -20,6 +20,7 @@ -include("rule_engine.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("stdlib/include/qlc.hrl"). -export([start_link/0]). @@ -27,6 +28,7 @@ -export([ get_rules/0 , get_rules_for/1 , get_rules_with_same_event/1 + , get_rules_ordered_by_ts/0 , get_rule/1 , add_rule/1 , add_rules/1 @@ -168,6 +170,14 @@ start_link() -> get_rules() -> get_all_records(?RULE_TAB). +get_rules_ordered_by_ts() -> + F = fun() -> + Query = qlc:q([E || E <- mnesia:table(?RULE_TAB)]), + qlc:e(qlc:keysort(#rule.created_at, Query, [{order, ascending}])) + end, + {atomic, List} = mnesia:transaction(F), + List. + -spec(get_rules_for(Topic :: binary()) -> list(emqx_rule_engine:rule())). get_rules_for(Topic) -> [Rule || Rule = #rule{for = For} <- get_rules(), diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 456438b6d..29c1ee6b6 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -694,44 +694,44 @@ t_update_rule(_Config) -> ok. t_disable_rule(_Config) -> - ets:new(simpile_action_2, [named_table, set, public]), - ets:insert(simpile_action_2, {created, 0}), - ets:insert(simpile_action_2, {destroyed, 0}), + ets:new(simple_action_2, [named_table, set, public]), + ets:insert(simple_action_2, {created, 0}), + ets:insert(simple_action_2, {destroyed, 0}), Now = erlang:timestamp(), emqx_rule_registry:add_action( - #action{name = 'simpile_action_2', app = ?APP, + #action{name = 'simple_action_2', app = ?APP, module = ?MODULE, - on_create = simpile_action_2_create, - on_destroy = simpile_action_2_destroy, + on_create = simple_action_2_create, + on_destroy = simple_action_2_destroy, types=[], params_spec = #{}, title = #{en => <<"Simple Action">>}, description = #{en => <<"Simple Action">>}}), {ok, #rule{actions = [#action_instance{id = ActInsId0}]}} = emqx_rule_engine:create_rule( #{id => <<"simple_rule_2">>, rawsql => <<"select * from \"t/#\"">>, - actions => [#{name => 'simpile_action_2', args => #{}}] + actions => [#{name => 'simple_action_2', args => #{}}] }), - [{_, CAt}] = ets:lookup(simpile_action_2, created), + [{_, CAt}] = ets:lookup(simple_action_2, created), ?assert(CAt > Now), - [{_, DAt}] = ets:lookup(simpile_action_2, destroyed), + [{_, DAt}] = ets:lookup(simple_action_2, destroyed), ?assert(DAt < Now), %% disable the rule and verify the old action instances has been cleared Now2 = erlang:timestamp(), emqx_rule_engine:update_rule(#{ id => <<"simple_rule_2">>, enabled => false}), - [{_, CAt2}] = ets:lookup(simpile_action_2, created), + [{_, CAt2}] = ets:lookup(simple_action_2, created), ?assert(CAt2 < Now2), - [{_, DAt2}] = ets:lookup(simpile_action_2, destroyed), + [{_, DAt2}] = ets:lookup(simple_action_2, destroyed), ?assert(DAt2 > Now2), %% enable the rule again and verify the action instances has been created Now3 = erlang:timestamp(), emqx_rule_engine:update_rule(#{ id => <<"simple_rule_2">>, enabled => true}), - [{_, CAt3}] = ets:lookup(simpile_action_2, created), + [{_, CAt3}] = ets:lookup(simple_action_2, created), ?assert(CAt3 > Now3), - [{_, DAt3}] = ets:lookup(simpile_action_2, destroyed), + [{_, DAt3}] = ets:lookup(simple_action_2, destroyed), ?assert(DAt3 < Now3), ok = emqx_rule_engine:delete_rule(<<"simple_rule_2">>). @@ -744,6 +744,19 @@ t_get_rules_for(_Config) -> ok = emqx_rule_registry:remove_rules([<<"rule-debug-1">>, <<"rule-debug-2">>]), ok. +t_get_rules_ordered_by_ts(_Config) -> + Now = fun() -> erlang:system_time(nanosecond) end, + ok = emqx_rule_registry:add_rules( + [make_simple_rule_with_ts(<<"rule-debug-0">>, Now()), + make_simple_rule_with_ts(<<"rule-debug-1">>, Now()), + make_simple_rule_with_ts(<<"rule-debug-2">>, Now()) + ]), + ?assertMatch([ + #rule{id = <<"rule-debug-0">>}, + #rule{id = <<"rule-debug-1">>}, + #rule{id = <<"rule-debug-2">>} + ], emqx_rule_registry:get_rules_ordered_by_ts()). + t_get_rules_for_2(_Config) -> Len0 = length(emqx_rule_registry:get_rules_for(<<"simple/1">>)), ok = emqx_rule_registry:add_rules( @@ -2166,6 +2179,17 @@ make_simple_rule(RuleId) when is_binary(RuleId) -> actions = [{'inspect', #{}}], description = <<"simple rule">>}. +make_simple_rule_with_ts(RuleId, Ts) when is_binary(RuleId) -> + #rule{id = RuleId, + rawsql = <<"select * from \"simple/topic\"">>, + for = [<<"simple/topic">>], + fields = [<<"*">>], + is_foreach = false, + conditions = {}, + actions = [{'inspect', #{}}], + created_at = Ts, + description = <<"simple rule">>}. + make_simple_rule(RuleId, SQL, ForTopics) when is_binary(RuleId) -> #rule{id = RuleId, rawsql = SQL, @@ -2250,12 +2274,12 @@ crash_action(_Id, _Params) -> error(crash) end. -simpile_action_2_create(_Id, _Params) -> - ets:insert(simpile_action_2, {created, erlang:timestamp()}), +simple_action_2_create(_Id, _Params) -> + ets:insert(simple_action_2, {created, erlang:timestamp()}), fun(_Data, _Envs) -> ok end. -simpile_action_2_destroy(_Id, _Params) -> - ets:insert(simpile_action_2, {destroyed, erlang:timestamp()}), +simple_action_2_destroy(_Id, _Params) -> + ets:insert(simple_action_2, {destroyed, erlang:timestamp()}), fun(_Data, _Envs) -> ok end. init_plus_by_one_action() -> From d54410e56011990a9b8185c21d0e795a72cdf0c0 Mon Sep 17 00:00:00 2001 From: turtleDeng Date: Fri, 12 Mar 2021 21:21:20 +0800 Subject: [PATCH 008/158] chore(script): split-config optimize include position (#4322) --- scripts/split-config.escript | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/split-config.escript b/scripts/split-config.escript index 07caf6503..04f94269c 100755 --- a/scripts/split-config.escript +++ b/scripts/split-config.escript @@ -16,10 +16,13 @@ main(_) -> {ok, Bin} = file:read_file("etc/emqx.conf"), Lines = binary:split(Bin, <<"\n">>, [global]), Sections0 = parse_sections(Lines), - Sections = lists:filter(fun({<<"modules">>, _}) -> false; - (_) -> true - end, Sections0), - ok = dump_sections(Sections). + {value, _, Sections1} = lists:keytake(<<"modules">>, 1, Sections0), + {value, {N, Base}, Sections2} = lists:keytake(<<"emqx">>, 1, Sections1), + IncludeNames = proplists:get_keys(Sections2), + Includes = lists:map(fun(Name) -> + iolist_to_binary(["include {{ platform_etc_dir }}/", Name, ".conf"]) + end, IncludeNames), + ok = dump_sections([{N, Base ++ Includes}| Sections2]). parse_sections(Lines) -> {ok, P} = re:compile("#+\s*CONFIG_SECTION_(BGN|END)\s*=\s*([^\s-]+)\s*="), @@ -42,9 +45,7 @@ parse_sections([Line | Lines], Parse, Section, Sections) -> ?BASE = Section, %% assert true = (Name =/= ?BASE), %% assert false = maps:is_key(Name, Sections), %% assert - Include = iolist_to_binary(["include {{ platform_etc_dir }}/", Name, ".conf"]), - Base = maps:get(?BASE, Sections), - NewSections = Sections#{?BASE := [Include | Base], Name => []}, + NewSections = Sections#{?BASE := maps:get(?BASE, Sections), Name => []}, parse_sections(Lines, Parse, Name, NewSections); {section_end, Name} -> true = (Name =:= Section), %% assert From 8e45fa1c8b86c186483d91d652477bc8fa85a6dc Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sat, 13 Mar 2021 18:50:18 +0100 Subject: [PATCH 009/158] feat(observer_cli): Add observer_cli --- rebar.config | 1 + rebar.config.erl | 1 + 2 files changed, 2 insertions(+) diff --git a/rebar.config b/rebar.config index 4b99dde2d..ff64e9eec 100644 --- a/rebar.config +++ b/rebar.config @@ -50,6 +50,7 @@ , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.2"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} + , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.1"} ]}. diff --git a/rebar.config.erl b/rebar.config.erl index f1623dfca..0f35b97b6 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -226,6 +226,7 @@ relx_apps(ReleaseType) -> , {mnesia, load} , {ekka, load} , {emqx_plugin_libs, load} + , observer_cli ] ++ [emqx_modules || not is_enterprise()] ++ [emqx_license || is_enterprise()] From c5f0194af72b23c09c2a6e78c7eff04441d83563 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Mon, 15 Mar 2021 09:36:12 +0800 Subject: [PATCH 010/158] chore(CI): fix build error on windows --- Makefile | 2 +- scripts/git-hook-pre-push.sh | 2 +- scripts/git-hooks-init.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 97072f81a..2223f9702 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -$(shell scripts/git-hooks-init.sh) +$(shell $(CURDIR)/scripts/git-hooks-init.sh) REBAR_VERSION = 3.14.3-emqx-5 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build diff --git a/scripts/git-hook-pre-push.sh b/scripts/git-hook-pre-push.sh index 2f5a9abcd..5b3f2edf2 100755 --- a/scripts/git-hook-pre-push.sh +++ b/scripts/git-hook-pre-push.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail diff --git a/scripts/git-hooks-init.sh b/scripts/git-hooks-init.sh index a9f02ab3a..fd47dd46f 100755 --- a/scripts/git-hooks-init.sh +++ b/scripts/git-hooks-init.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail From 4ffb0e18481640424185498fbee835068b09f149 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sun, 14 Mar 2021 13:37:35 +0100 Subject: [PATCH 011/158] docs(README): Add windows --- README.md | 2 + Windows.md | 114 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 Windows.md diff --git a/README.md b/README.md index 14e0abde4..49f44d33f 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,8 @@ Starting from 3.0 release, *EMQ X* broker fully supports MQTT V5.0 protocol spec The *EMQ X* broker is cross-platform, which supports Linux, Unix, macOS and Windows. It means *EMQ X* can be deployed on x86_64 architecture servers and ARM devices like Raspberry Pi. +See more details for building and running *EMQ X* on Windows in [Windows.md](./Windows.md) + #### Installing via EMQ X Docker Image ``` diff --git a/Windows.md b/Windows.md new file mode 100644 index 000000000..2f3f08876 --- /dev/null +++ b/Windows.md @@ -0,0 +1,114 @@ +# Build and run EMQ X on Windows + +NOTE: The instructions and examples are based on Windows 10. + +## Build Environment + +### Visual studio for C/C++ compile and link + +EMQ X includes Erlang NIF (Native Implmented Function) components, implemented +in C/C++. To compile and link C/C++ libraries, the easiest way is perhaps to +install Visual Studio. + +Visual Studio 2019 is used in our tests. +If you (like @zmstone) do not know where to start, please follow this OTP guide: +https://github.com/erlang/otp/blob/master/HOWTO/INSTALL-WIN32.md + +NOTE: To avoid surprises, you may need to add below two paths to `Path` environment variable +and order them before other paths. + +``` +C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29910\bin\Hostx64\x64 +C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build +``` + +Depending on your visual studio version and OS, the paths may differ. +The first path is for rebar3 port compiler to find `cl.exe` and `link.exe` +The second path is for Powershell or CMD to setup environment variables. + +### Erlang/OTP + +Install Erlang/OTP 23.2 from https://www.erlang.org/downloads +You may need to edit the `Path` environment variable to allow running +Erlang commands such as `erl` from powershell. + +To validate Erlang installation in CMD or powershell: + +* Start (or restart) CMD or powershell + +* Execute `erl` command to enter Erlang shell + +* Evaluate Erlang expression `halt().` to exit Erlang shell. + +e.g. + +``` +Eshell V11.1.4 (abort with ^G) +1> halt(). +PS C:\Users\zmsto> erl +Eshell V11.1.4 (abort with ^G) +1> halt(). +``` + +### bash + +All EMQ X build/run scripts are either in `bash` or `escript`. +`escript` is installed as a part of Erlang. To install a `bash` +environment in Windows, there are quite a few options. + +Cygwin is what we tested with. + +* Add `cygwin\bin` dir to `Path` environment variable + To do so, search for Edit environment variable in control pannel and + add `C:\tools\cygwin\bin` (depending on the location where it was installed) + to `Path` list. + +* Validate installation. + Start (restart) CMD or powershell console and execute `which bash`, it should + print out `/usr/bin/bash` + +### Other tools + +Some of the unix world tools are required to build EMQ X. Including: + +* git +* curl +* make +* jq +* zip / unzip + +We recommend using [scoop](https://scoop.sh/), or [Chocolatey](https://chocolatey.org/install) to install the tools. + +When using scoop: + +``` +scoop install git curl make jq zip unzip +``` + +## Build EMQ X source code + +* Clone the repo: `git clone https://github.com/emqx/emqx.git` + +* Start CMD or Powershell + +* Execute `vcvarsall.bat x86_amd64` to load environment variables + +* Change to emqx directory and execute `make` + +### Possible errors + +* `'cl.exe' is not recognized as an internal or external command` + This error is likely due to Visual Studio executables are not set in `Path` environment variable. + To fix it, either add path like `C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29910\bin\Hostx64\x64` + to `Paht`. Or make sure `vcvarsall.bat x86_amd64` is executed prior to the `make` command + +* `fatal error C1083: Cannot open include file: 'assert.h': No such file or directory` + If Visual Studio is installed correctly, this is likely `LIB` and `LIB_PATH` environment + variables are not set. Make sure `vcvarsall.bat x86_amd64` is executed prior to the `make` command + +* `link: extra operand 'some.obj'` + This is likely due ot the usage of GNU `lnik.exe` but not the one from Visual Studio. + Exeucte `link.exe --version` to inspect which one is in use. The one installed from + Visual Studio should print out `Microsoft (R) Incremental Linker`. + To fix it, Visual Studio's bin paths should be ordered prior to Cygwin's (or similar installation's) + bin paths in `Path` environment variable. From 32a0392ec5dc1a3d51d5633055052bf184a0afe3 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sun, 14 Mar 2021 14:04:26 +0100 Subject: [PATCH 012/158] chore(git): Force eol=lf --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..fb9a97dba --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +* text=auto +*.* text eol=lf From 0fead391d2c8403e3b7b655939cea63808ff1025 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sun, 14 Mar 2021 16:10:49 +0100 Subject: [PATCH 013/158] chore(script): no -r option for rm command when delete a file --- scripts/get-dashboard.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index b53741048..e6960b286 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -55,4 +55,4 @@ get_assets unzip -q "$RELEASE_ASSET_FILE" -d "$DASHBOARD_PATH" rm -rf "$DASHBOARD_PATH/www" mv "$DASHBOARD_PATH/dist" "$DASHBOARD_PATH/www" -rm -rf "$RELEASE_ASSET_FILE" +rm -f "$RELEASE_ASSET_FILE" From 2c0c69142d0c021eb4c063cd3750e8073f461d07 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sun, 14 Mar 2021 16:23:19 +0100 Subject: [PATCH 014/158] chore(script): Delete CRLF from jq print-outs --- scripts/get-dashboard.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index e6960b286..e37badfb2 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -38,7 +38,8 @@ get_assets(){ --header "${AUTH}" \ --header "Accept: application/vnd.github.v3+json" \ "https://api.github.com/repos/emqx/${DASHBOARD_REPO}/releases/tags/${VERSION}" \ - | jq --raw-output ".assets[] | select(.name==\"${RELEASE_ASSET_FILE}\").url")" + | jq --raw-output ".assets[] | select(.name==\"${RELEASE_ASSET_FILE}\").url" \ + | tr -d '\n' | tr -d '\r')" # Get GitHub's S3 redirect URL redirect_url=$(curl --silent --show-error \ --header "${AUTH}" \ From d9a937cd34b86343585f0077edc5215c54a4f791 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sun, 14 Mar 2021 16:38:20 +0100 Subject: [PATCH 015/158] chore(build): No rebar3 color for windows --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 2223f9702..57af7b8db 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,9 @@ SCRIPTS = $(CURDIR)/scripts export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X export EMQX_CE_DASHBOARD_VERSION ?= v4.3.0-beta.1 +ifeq ($(OS),Windows_NT) + export REBAR_COLOR=none +endif PROFILE ?= emqx REL_PROFILES := emqx emqx-edge From 51dac159d373b131551fd4a7b2ef808cf5e9ac93 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 15 Mar 2021 00:33:21 +0100 Subject: [PATCH 016/158] chore(ci): No more make get-deps --- .github/workflows/build_packages.yaml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 2562a725f..f0c9aa2a3 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -23,15 +23,6 @@ jobs: otp-version: 23.2 - name: build run: | - # set-executionpolicy remotesigned -s cu - # iex (new-object net.webclient).downloadstring('https://get.scoop.sh') - # # $env:path + ";" + $env:USERPROFILE + "\scoop\shims" + ';C:\Program Files\erl10.4\bin' - # [environment]::SetEnvironmentvariable("Path", ";" + $env:USERPROFILE + "\scoop\shims") - # [environment]::SetEnvironmentvariable("Path", ';C:\Program Files\erl10.4\bin') - # scoop bucket add extras https://github.com/lukesampson/scoop-extras.git - # scoop update - # scoop install sudo curl vcredist2013 - $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH" $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" ) @@ -43,13 +34,6 @@ jobs: $pkg_name = "emqx-windows-$($version -replace '/').zip" } - make deps-emqx || cat rebar3.crashdump - $rebar3 = $env:USERPROFILE + "\rebar3" - (New-Object System.Net.WebClient).DownloadFile('https://s3.amazonaws.com/rebar3/rebar3', $rebar3) - cd _build/emqx/lib/jiffy/ - escript $rebar3 compile - cd ../../../../ - make emqx mkdir -p _packages/emqx Compress-Archive -Path _build/emqx/rel/emqx -DestinationPath _build/emqx/rel/$pkg_name From a691a424224b2e1e85e2ae3797c746865bded29e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 15 Mar 2021 08:10:21 +0100 Subject: [PATCH 017/158] chore(Windows.md): tidy up erl cmd test example --- Windows.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Windows.md b/Windows.md index 2f3f08876..4f24f4d3a 100644 --- a/Windows.md +++ b/Windows.md @@ -11,7 +11,8 @@ in C/C++. To compile and link C/C++ libraries, the easiest way is perhaps to install Visual Studio. Visual Studio 2019 is used in our tests. -If you (like @zmstone) do not know where to start, please follow this OTP guide: +If you are like me (@zmstone), do not know where to start, +please follow this OTP guide: https://github.com/erlang/otp/blob/master/HOWTO/INSTALL-WIN32.md NOTE: To avoid surprises, you may need to add below two paths to `Path` environment variable @@ -43,8 +44,6 @@ To validate Erlang installation in CMD or powershell: e.g. ``` -Eshell V11.1.4 (abort with ^G) -1> halt(). PS C:\Users\zmsto> erl Eshell V11.1.4 (abort with ^G) 1> halt(). From 3438a0ebca61468db9667160ac12771c8cdcdd4a Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 15 Mar 2021 08:17:01 +0100 Subject: [PATCH 018/158] fix(telemetry): get release version, not emqx app version --- apps/emqx_telemetry/src/emqx_telemetry.erl | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/apps/emqx_telemetry/src/emqx_telemetry.erl b/apps/emqx_telemetry/src/emqx_telemetry.erl index f5dc0de60..b436768a2 100644 --- a/apps/emqx_telemetry/src/emqx_telemetry.erl +++ b/apps/emqx_telemetry/src/emqx_telemetry.erl @@ -157,7 +157,7 @@ init([Opts]) -> [#telemetry{uuid = UUID, enabled = Enabled} | _] -> State#state{enabled = Enabled, uuid = UUID} end, - case official_version(emqx_version()) of + case official_version(emqx_app:get_release()) of true -> {ok, ensure_report_timer(NState), {continue, first_report}}; false -> @@ -233,10 +233,6 @@ official_version(Version) -> ensure_report_timer(State = #state{report_interval = ReportInterval}) -> State#state{timer = emqx_misc:start_timer(ReportInterval, time_to_report_telemetry_data)}. -emqx_version() -> - {ok, Version} = application:get_key(emqx, vsn), - Version. - license() -> case search_telemetry_license() of {error, not_found} -> @@ -347,7 +343,7 @@ generate_uuid() -> get_telemetry(#state{uuid = UUID}) -> OSInfo = os_info(), - [{emqx_version, bin(emqx_version())}, + [{emqx_version, bin(emqx_app:get_release())}, {license, license()}, {os_name, bin(get_value(os_name, OSInfo))}, {os_version, bin(get_value(os_version, OSInfo))}, From ef1be26bc8885663f3ec6d21a0ef37b946cc482a Mon Sep 17 00:00:00 2001 From: zhouzb Date: Mon, 15 Mar 2021 10:15:23 +0800 Subject: [PATCH 019/158] fix(subprotocol): fix bad configuration item name for subprotocol --- etc/emqx.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 684a1d0f8..55914fc11 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1580,7 +1580,7 @@ listener.ws.external.access.1 = allow all ## Supported subprotocols ## ## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 -## listener.ws.external.supported_protocols = mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 +## listener.ws.external.supported_subprotocols = mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 ## Enable the Proxy Protocol V1/2 if the EMQ cluster is deployed behind ## HAProxy or Nginx. @@ -1801,7 +1801,7 @@ listener.wss.external.access.1 = allow all ## Supported subprotocols ## ## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 -## listener.wss.external.supported_protocols = mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 +## listener.wss.external.supported_subprotocols = mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 ## Enable the Proxy Protocol V1/2 support. ## From d5886c0c66568ac045ed1e0bc9dc4aed00e013a5 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 16 Mar 2021 09:49:11 +0100 Subject: [PATCH 020/158] chore(build): Get rel vsn fallback (#4339) --- src/emqx_app.erl | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/emqx_app.erl b/src/emqx_app.erl index 4f8d47390..e62c85f25 100644 --- a/src/emqx_app.erl +++ b/src/emqx_app.erl @@ -70,18 +70,15 @@ get_description() -> Str -> string:strip(Str, both, $\n) end. --ifdef(TEST). -%% When testing, the 'cover' compiler stripps aways compile info -get_release() -> release_in_macro(). --else. -%% Otherwise print the build number, -%% which may have a git commit in its suffix. get_release() -> - {_, Vsn} = lists:keyfind(emqx_vsn, 1, ?MODULE:module_info(compile)), - VsnStr = release_in_macro(), - 1 = string:str(Vsn, VsnStr), %% assert - Vsn. --endif. + case lists:keyfind(emqx_vsn, 1, ?MODULE:module_info(compile)) of + false -> %% For TEST build or depedency build. + release_in_macro(); + {_, Vsn} -> %% For emqx release build + VsnStr = release_in_macro(), + 1 = string:str(Vsn, VsnStr), %% assert + Vsn + end. release_in_macro() -> element(2, ?EMQX_RELEASE). From 1ef51cae1d8b589caa4ad62fd5e7aa58bc1d878f Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 16 Mar 2021 10:34:52 +0800 Subject: [PATCH 021/158] chore(CI): update docker compose file for mysql --- .ci/apps_tests/docker-compose.yaml | 1 - .ci/compatibility_tests/docker-compose-mysql.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/.ci/apps_tests/docker-compose.yaml b/.ci/apps_tests/docker-compose.yaml index b8f84821d..42cfe82f6 100644 --- a/.ci/apps_tests/docker-compose.yaml +++ b/.ci/apps_tests/docker-compose.yaml @@ -38,7 +38,6 @@ services: MYSQL_DATABASE: mqtt command: --bind-address 0.0.0.0 - --default-authentication-plugin=mysql_native_password --character-set-server=utf8mb4 --collation-server=utf8mb4_general_ci --explicit_defaults_for_timestamp=true diff --git a/.ci/compatibility_tests/docker-compose-mysql.yaml b/.ci/compatibility_tests/docker-compose-mysql.yaml index 1f285cc5e..9fe8d1a33 100644 --- a/.ci/compatibility_tests/docker-compose-mysql.yaml +++ b/.ci/compatibility_tests/docker-compose-mysql.yaml @@ -24,7 +24,6 @@ services: - emqx_bridge command: --bind-address "::" - --default-authentication-plugin=mysql_native_password --character-set-server=utf8mb4 --collation-server=utf8mb4_general_ci --explicit_defaults_for_timestamp=true From 98c62df2e4bacaacd46ac1d2d4a2365bdc862310 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 16 Mar 2021 15:20:31 +0800 Subject: [PATCH 022/158] fix(log): fix web hook print error --- apps/emqx_web_hook/src/emqx_web_hook.erl | 2 +- apps/emqx_web_hook/test/http_server.erl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_web_hook/src/emqx_web_hook.erl b/apps/emqx_web_hook/src/emqx_web_hook.erl index ec525c759..6c63d6d49 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook.erl @@ -325,7 +325,7 @@ send_http_request(ClientID, Params) -> {ok, Path} = application:get_env(?APP, path), Headers = application:get_env(?APP, headers, []), Body = emqx_json:encode(Params), - ?LOG(debug, "Send to: ~0p, params: ~0s", [Path, Body]), + ?LOG(debug, "Send to: ~0p, params: ~s", [Path, Body]), case ehttpc:request(ehttpc_pool:pick_worker(?APP, ClientID), post, {Path, Headers, Body}) of {ok, StatusCode, _} when StatusCode >= 200 andalso StatusCode < 300 -> ok; diff --git a/apps/emqx_web_hook/test/http_server.erl b/apps/emqx_web_hook/test/http_server.erl index e0f367eba..6a23e1035 100644 --- a/apps/emqx_web_hook/test/http_server.erl +++ b/apps/emqx_web_hook/test/http_server.erl @@ -12,7 +12,7 @@ -export([start_link/0]). -export([get_received_data/0]). -export([stop/1]). --export([code_change/3, handle_call/3, handle_cast/2, handle_info/2, init/1, terminate/2]). +-export([code_change/3, handle_call/3, handle_cast/2, handle_info/2, init/1, init/2, terminate/2]). -define(HTTP_PORT, 9999). -define(HTTPS_PORT, 8888). -record(state, {}). @@ -102,4 +102,4 @@ init(Req, State) -> reply(Req, ok) -> cowboy_req:reply(200, #{<<"content-type">> => <<"text/plain">>}, <<"ok">>, Req); reply(Req, error) -> - cowboy_req:reply(404, #{<<"content-type">> => <<"text/plain">>}, <<"deny">>, Req). \ No newline at end of file + cowboy_req:reply(404, #{<<"content-type">> => <<"text/plain">>}, <<"deny">>, Req). From 96bd56179069b87459a27f67608f91423508ca56 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 15 Mar 2021 17:49:15 +0100 Subject: [PATCH 023/158] feat(windows): Support environment variable override for windows --- Windows.md | 14 ++++++++++++++ bin/emqx.cmd | 2 ++ 2 files changed, 16 insertions(+) diff --git a/Windows.md b/Windows.md index 4f24f4d3a..5e947a22e 100644 --- a/Windows.md +++ b/Windows.md @@ -111,3 +111,17 @@ scoop install git curl make jq zip unzip Visual Studio should print out `Microsoft (R) Incremental Linker`. To fix it, Visual Studio's bin paths should be ordered prior to Cygwin's (or similar installation's) bin paths in `Path` environment variable. + +## Run EMQ X + +To start EMQ X broker. + +Execute `_build\emqx\rel\emqx>.\bin\emqx console` or `_build\emqx\rel\emqx>.\bin\emqx start` to start EMQ X. + +Then execute `_build\emqx\rel\emqx>.\bin\emqx_ctl status` to check status. +If everything works fine, it should print out + +``` +Node 'emqx@127.0.0.1' 4.3-beta.1 is started +Application emqx 4.3.0 is running +``` diff --git a/bin/emqx.cmd b/bin/emqx.cmd index 9092e553c..bef7da355 100644 --- a/bin/emqx.cmd +++ b/bin/emqx.cmd @@ -19,6 +19,8 @@ @set erts_vsn={{ erts_vsn }} @set erl_opts={{ erl_opts }} +@set "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" + @set script=%~n0 :: Discover the release root directory from the directory From a1f78b5083be07e685b291b2ffd53529dadd310e Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 15 Mar 2021 18:51:10 +0100 Subject: [PATCH 024/158] refactor(cli): Print node name version and app name version So not to confuse application version with node release version --- apps/emqx_management/src/emqx_mgmt_cli.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 20b4f1b01..6cf69209d 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -115,12 +115,12 @@ mgmt(_) -> status([]) -> {InternalStatus, _ProvidedStatus} = init:get_status(), - emqx_ctl:print("Node ~p is ~p~n", [node(), InternalStatus]), + emqx_ctl:print("Node ~p ~s is ~p~n", [node(), emqx_app:get_release(), InternalStatus]), case lists:keysearch(?APP, 1, application:which_applications()) of false -> - emqx_ctl:print("~s is not running~n", [?APP]); + emqx_ctl:print("Application ~s is not running~n", [?APP]); {value, {?APP, _Desc, Vsn}} -> - emqx_ctl:print("~s ~s is running~n", [?APP, Vsn]) + emqx_ctl:print("Application ~s ~s is running~n", [?APP, Vsn]) end; status(_) -> emqx_ctl:usage("status", "Show broker status"). From 4bf0ad1bafd573bbb4d7590b8ea34e053b3467b7 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 16 Mar 2021 16:55:06 +0800 Subject: [PATCH 025/158] fix(config): delete peer_cert_as_username and peer_cert_as_clientid in tcp listener --- etc/emqx.conf | 14 -------------- priv/emqx.schema | 8 -------- 2 files changed, 22 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 55914fc11..990d82bfc 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1085,20 +1085,6 @@ listener.tcp.external.access.1 = allow all ## Value: Duration ## listener.tcp.external.proxy_protocol_timeout = 3s -## Enable the option for X.509 certificate based authentication. -## EMQX will use the common name of certificate as MQTT username. -## 'pem' encodes CRT in base64, and md5 is the md5 hash of CRT. -## -## Value: cn | dn | crt | pem | md5 -## listener.tcp.external.peer_cert_as_username = cn - -## Enable the option for X.509 certificate based authentication. -## EMQX will use the common name of certificate as MQTT clientid. -## 'pem' encodes CRT in base64, and md5 is the md5 hash of CRT. -## -## Value: cn | dn | crt | pem | md5 -## listener.tcp.external.peer_cert_as_clientid = cn - ## The TCP backlog defines the maximum length that the queue of pending ## connections can grow to. ## diff --git a/priv/emqx.schema b/priv/emqx.schema index 0e933c44f..26d907926 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1211,14 +1211,6 @@ end}. {datatype, {duration, ms}} ]}. -{mapping, "listener.tcp.$name.peer_cert_as_username", "emqx.listeners", [ - {datatype, {enum, [cn, dn, crt, pem, md5]}} -]}. - -{mapping, "listener.tcp.$name.peer_cert_as_clientid", "emqx.listeners", [ - {datatype, {enum, [cn, dn, crt, pem, md5]}} -]}. - {mapping, "listener.tcp.$name.backlog", "emqx.listeners", [ {datatype, integer}, {default, 1024} From 08366c2735a6e545e1c8a0e2160e87efaf65fdb5 Mon Sep 17 00:00:00 2001 From: Turtle Date: Wed, 17 Mar 2021 14:11:40 +0800 Subject: [PATCH 026/158] fix(conf): fix split modules conf error --- etc/emqx.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 990d82bfc..ac082f5bc 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -2092,6 +2092,8 @@ module.presence.qos = 1 ## module.rewrite.pub.rule.1 = x/# ^x/y/(.+)$ z/y/$1 ## module.rewrite.sub.rule.1 = y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2 +## CONFIG_SECTION_END=modules ================================================== + ##------------------------------------------------------------------- ## Plugins ##------------------------------------------------------------------- @@ -2163,8 +2165,6 @@ broker.shared_dispatch_ack_enabled = false ## Value: Flag broker.route_batch_clean = off -## CONFIG_SECTION_END=modules ================================================== - ## CONFIG_SECTION_BGN=sys_mon ================================================== ## Enable Long GC monitoring. Disable if the value is 0. From d6797760a1458dc3cb025c038af73df8beffac72 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 16 Mar 2021 16:51:10 +0800 Subject: [PATCH 027/158] fix(test): add testcases for ws subprotocols --- test/emqx_ws_connection_SUITE.erl | 93 ++++++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 7 deletions(-) diff --git a/test/emqx_ws_connection_SUITE.erl b/test/emqx_ws_connection_SUITE.erl index 203a05065..71c6e40e5 100644 --- a/test/emqx_ws_connection_SUITE.erl +++ b/test/emqx_ws_connection_SUITE.erl @@ -41,7 +41,9 @@ all() -> emqx_ct:all(?MODULE). %% CT callbacks %%-------------------------------------------------------------------- -init_per_suite(Config) -> +init_per_testcase(TestCase, Config) when + TestCase =/= t_ws_sub_protocols_mqtt_equivalents, + TestCase =/= t_ws_sub_protocols_mqtt -> %% Mock cowboy_req ok = meck:new(cowboy_req, [passthrough, no_history, no_link]), ok = meck:expect(cowboy_req, peer, fun(_) -> {{127,0,0,1}, 3456} end), @@ -75,9 +77,15 @@ init_per_suite(Config) -> ok = meck:expect(emqx_metrics, inc, fun(_, _) -> ok end), ok = meck:expect(emqx_metrics, inc_recv, fun(_) -> ok end), ok = meck:expect(emqx_metrics, inc_sent, fun(_) -> ok end), + Config; + +init_per_testcase(_, Config) -> Config. -end_per_suite(_Config) -> + +end_per_testcase(TestCase, Config) when + TestCase =/= t_ws_sub_protocols_mqtt_equivalents, + TestCase =/= t_ws_sub_protocols_mqtt -> lists:foreach(fun meck:unload/1, [cowboy_req, emqx_zone, @@ -85,12 +93,9 @@ end_per_suite(_Config) -> emqx_broker, emqx_hooks, emqx_metrics - ]). + ]); -init_per_testcase(_TestCase, Config) -> - Config. - -end_per_testcase(_TestCase, Config) -> +end_per_testcase(_, Config) -> Config. %%-------------------------------------------------------------------- @@ -145,6 +150,27 @@ t_call(_) -> end), ?assertEqual(Info, ?ws_conn:call(WsPid, info)). +t_ws_sub_protocols_mqtt(_) -> + ok = emqx_ct_helpers:start_apps([]), + {ok, _} = application:ensure_all_started(gun), + ?assertMatch({gun_upgrade, _}, + start_ws_client(#{protocols => [<<"mqtt">>]})), + emqx_ct_helpers:stop_apps([]). + +t_ws_sub_protocols_mqtt_equivalents(_) -> + ok = emqx_ct_helpers:start_apps([]), + {ok, _} = application:ensure_all_started(gun), + %% also support mqtt-v3, mqtt-v3.1.1, mqtt-v5 + ?assertMatch({gun_upgrade, _}, + start_ws_client(#{protocols => [<<"mqtt-v3">>]})), + ?assertMatch({gun_upgrade, _}, + start_ws_client(#{protocols => [<<"mqtt-v3.1.1">>]})), + ?assertMatch({gun_upgrade, _}, + start_ws_client(#{protocols => [<<"mqtt-v5">>]})), + ?assertMatch({gun_response, {_, 400, _}}, + start_ws_client(#{protocols => [<<"not-mqtt">>]})), + emqx_ct_helpers:stop_apps([]). + t_init(_) -> Opts = [{idle_timeout, 300000}, {fail_if_no_subprotocol, false}, @@ -395,3 +421,56 @@ channel(InitFields) -> conn_state => connected }, InitFields)). +start_ws_client(State) -> + Host = maps:get(host, State, "127.0.0.1"), + Port = maps:get(port, State, 8083), + {ok, WPID} = gun:open(Host, Port), + #{result := Result} = ws_client(State#{wpid => WPID}), + gun:close(WPID), + Result. + +ws_client(State) -> + receive + {gun_up, WPID, _Proto} -> + #{protocols := Protos} = State, + StreamRef = gun:ws_upgrade(WPID, "/mqtt", [], + #{protocols => [{P, gun_ws_h} || P <- Protos]}), + ws_client(State#{wref => StreamRef}); + {gun_down, _WPID, _, Reason, _, _} -> + State#{result => {gun_down, Reason}}; + {gun_upgrade, _WPID, _Ref, _Proto, Data} -> + ct:pal("-- gun_upgrade: ~p", [Data]), + State#{result => {gun_upgrade, Data}}; + {gun_response, _WPID, _Ref, _Code, _HttpStatusCode, _Headers} -> + Rsp = {_Code, _HttpStatusCode, _Headers}, + ct:pal("-- gun_response: ~p", [Rsp]), + State#{result => {gun_response, Rsp}}; + {gun_error, _WPID, _Ref, _Reason} -> + State#{result => {gun_error, _Reason}}; + {'DOWN',_PID,process,_WPID,_Reason} -> + State#{result => {down, _Reason}}; + {gun_ws, _WPID, Frame} -> + case Frame of + close -> + self() ! stop; + {close,_Code,_Message} -> + self() ! stop; + {text,TextData} -> + io:format("Received Text Frame: ~p~n",[TextData]); + {binary,BinData} -> + io:format("Received Binary Frame: ~p~n",[BinData]); + _ -> + io:format("Received Unhandled Frame: ~p~n",[Frame]) + end, + ws_client(State); + stop -> + #{wpid := WPID} = State, + gun:flush(WPID), + gun:shutdown(WPID), + State#{result => {stop, normal}}; + Message -> + ct:pal("Received Unknown Message on Gun: ~p~n",[Message]), + ws_client(State) + after 1000 -> + ct:fail(ws_timeout) + end. From 7a1993f13b659f766fa6dba2ecb74ea3f87d8d9b Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 17 Mar 2021 16:09:11 +0800 Subject: [PATCH 028/158] chore(auth-mnesia): update cli --- apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl | 15 ++++++++++++--- .../emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl | 10 ++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl index ae4fcee1f..ef852d04d 100644 --- a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl +++ b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl @@ -160,18 +160,27 @@ cli(["show", "username", Username]) -> [print_acl(Acl) || Acl <- lookup_acl({username, iolist_to_binary(Username)})]; cli(["del", "clientid", Clientid, Topic])-> + cli(["delete", "clientid", Clientid, Topic]); + +cli(["delete", "clientid", Clientid, Topic])-> case remove_acl({clientid, iolist_to_binary(Clientid)}, iolist_to_binary(Topic)) of ok -> emqx_ctl:print("ok~n"); {error, Reason} -> emqx_ctl:print("Error: ~p~n", [Reason]) end; cli(["del", "username", Username, Topic])-> + cli(["delete", "username", Username, Topic]); + +cli(["delete", "username", Username, Topic])-> case remove_acl({username, iolist_to_binary(Username)}, iolist_to_binary(Topic)) of ok -> emqx_ctl:print("ok~n"); {error, Reason} -> emqx_ctl:print("Error: ~p~n", [Reason]) end; cli(["del", "_all", Topic])-> + cli(["delete", "_all", Topic]); + +cli(["delete", "_all", Topic])-> case remove_acl(all, iolist_to_binary(Topic)) of ok -> emqx_ctl:print("ok~n"); {error, Reason} -> emqx_ctl:print("Error: ~p~n", [Reason]) @@ -186,9 +195,9 @@ cli(_) -> , {"acl aad clientid ", "Add clientid acl"} , {"acl add Username ", "Add username acl"} , {"acl add _all ", "Add $all acl"} - , {"acl del clientid ", "Delete clientid acl"} - , {"acl del username ", "Delete username acl"} - , {"acl del _all ", "Delete $all acl"} + , {"acl delete clientid ", "Delete clientid acl"} + , {"acl delete username ", "Delete username acl"} + , {"acl delete _all ", "Delete $all acl"} ]). %%-------------------------------------------------------------------- diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl index adefa704b..ef78b1c3c 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl @@ -144,6 +144,9 @@ auth_clientid_cli(["update", ClientId, NewPassword]) -> end; auth_clientid_cli(["del", ClientId]) -> + auth_clientid_cli(["delete", ClientId]); + +auth_clientid_cli(["delete", ClientId]) -> case remove_user({clientid, iolist_to_binary(ClientId)}) of ok -> emqx_ctl:print("ok~n"); {error, Reason} -> emqx_ctl:print("Error: ~p~n", [Reason]) @@ -153,7 +156,7 @@ auth_clientid_cli(_) -> emqx_ctl:usage([{"clientid list", "List clientid auth rules"}, {"clientid add ", "Add clientid auth rule"}, {"clientid update ", "Update clientid auth rule"}, - {"clientid del ", "Delete clientid auth rule"}]). + {"clientid delete ", "Delete clientid auth rule"}]). %%-------------------------------------------------------------------- %% Auth Username Cli @@ -176,6 +179,9 @@ auth_username_cli(["update", Username, NewPassword]) -> {error, Reason} -> emqx_ctl:print("Error: ~p~n", [Reason]) end; auth_username_cli(["del", Username]) -> + auth_username_cli(["delete", Username]); + +auth_username_cli(["delete", Username]) -> case remove_user({username, iolist_to_binary(Username)}) of ok -> emqx_ctl:print("ok~n"); {error, Reason} -> emqx_ctl:print("Error: ~p~n", [Reason]) @@ -185,4 +191,4 @@ auth_username_cli(_) -> emqx_ctl:usage([{"user list", "List username auth rules"}, {"user add ", "Add username auth rule"}, {"user update ", "Update username auth rule"}, - {"user del ", "Delete username auth rule"}]). + {"user delete ", "Delete username auth rule"}]). From 237603cee61a36908fa2aacf3b68e7d3e8f112cb Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 17 Mar 2021 14:07:17 +0800 Subject: [PATCH 029/158] revert: fix(config): delete peer_cert_as_username and peer_cert_as_clientid in tcp listener This reverts commit: 4bf0ad1bafd573bbb4d7590b8ea34e053b3467b7 --- etc/emqx.conf | 14 ++++++++++++++ priv/emqx.schema | 8 ++++++++ 2 files changed, 22 insertions(+) diff --git a/etc/emqx.conf b/etc/emqx.conf index ac082f5bc..16f1eaae3 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1085,6 +1085,20 @@ listener.tcp.external.access.1 = allow all ## Value: Duration ## listener.tcp.external.proxy_protocol_timeout = 3s +## Enable the option for X.509 certificate based authentication. +## EMQX will use the common name of certificate as MQTT username. +## 'pem' encodes CRT in base64, and md5 is the md5 hash of CRT. +## +## Value: cn | dn | crt | pem | md5 +## listener.tcp.external.peer_cert_as_username = cn + +## Enable the option for X.509 certificate based authentication. +## EMQX will use the common name of certificate as MQTT clientid. +## 'pem' encodes CRT in base64, and md5 is the md5 hash of CRT. +## +## Value: cn | dn | crt | pem | md5 +## listener.tcp.external.peer_cert_as_clientid = cn + ## The TCP backlog defines the maximum length that the queue of pending ## connections can grow to. ## diff --git a/priv/emqx.schema b/priv/emqx.schema index 26d907926..0e933c44f 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1211,6 +1211,14 @@ end}. {datatype, {duration, ms}} ]}. +{mapping, "listener.tcp.$name.peer_cert_as_username", "emqx.listeners", [ + {datatype, {enum, [cn, dn, crt, pem, md5]}} +]}. + +{mapping, "listener.tcp.$name.peer_cert_as_clientid", "emqx.listeners", [ + {datatype, {enum, [cn, dn, crt, pem, md5]}} +]}. + {mapping, "listener.tcp.$name.backlog", "emqx.listeners", [ {datatype, integer}, {default, 1024} From 2232bca1505d480748a0db5894399d1d9f3b27f7 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 17 Mar 2021 14:20:34 +0800 Subject: [PATCH 030/158] chore(config): update enum for config item The tcp listener's peer_cert_as_clientid and peer_cert_as_username can only be set to cn --- etc/emqx.conf | 8 ++++---- priv/emqx.schema | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 16f1eaae3..4a953ddb4 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1087,16 +1087,16 @@ listener.tcp.external.access.1 = allow all ## Enable the option for X.509 certificate based authentication. ## EMQX will use the common name of certificate as MQTT username. -## 'pem' encodes CRT in base64, and md5 is the md5 hash of CRT. +## The proxy-protocol protocol can get the certificate CN through tcp ## -## Value: cn | dn | crt | pem | md5 +## Value: cn ## listener.tcp.external.peer_cert_as_username = cn ## Enable the option for X.509 certificate based authentication. ## EMQX will use the common name of certificate as MQTT clientid. -## 'pem' encodes CRT in base64, and md5 is the md5 hash of CRT. +## The proxy-protocol protocol can get the certificate CN through tcp ## -## Value: cn | dn | crt | pem | md5 +## Value: cn ## listener.tcp.external.peer_cert_as_clientid = cn ## The TCP backlog defines the maximum length that the queue of pending diff --git a/priv/emqx.schema b/priv/emqx.schema index 0e933c44f..2e1248c50 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1211,12 +1211,14 @@ end}. {datatype, {duration, ms}} ]}. +%% The proxy-protocol protocol can get the certificate CN through tcp {mapping, "listener.tcp.$name.peer_cert_as_username", "emqx.listeners", [ - {datatype, {enum, [cn, dn, crt, pem, md5]}} + {datatype, {enum, [cn]}} ]}. +%% The proxy-protocol protocol can get the certificate CN through tcp {mapping, "listener.tcp.$name.peer_cert_as_clientid", "emqx.listeners", [ - {datatype, {enum, [cn, dn, crt, pem, md5]}} + {datatype, {enum, [cn]}} ]}. {mapping, "listener.tcp.$name.backlog", "emqx.listeners", [ From a5f0ed0356e2c34bd041d3959d74d0c640fe1bdb Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 17 Mar 2021 11:21:51 +0100 Subject: [PATCH 031/158] fix(test): Ensure emqx_modules loaded for emqx_management test --- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 6 +++--- apps/emqx_management/test/emqx_mgmt_api_SUITE.erl | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index a3e9a6fad..c9aea26d6 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -53,16 +53,16 @@ groups() -> ]}]. apps() -> - [emqx, emqx_management, emqx_auth_mnesia]. + [emqx_management, emqx_auth_mnesia, emqx_modules]. init_per_suite(Config) -> ekka_mnesia:start(), emqx_mgmt_auth:mnesia(boot), - emqx_ct_helpers:start_apps([emqx_management, emqx_auth_mnesia]), + emqx_ct_helpers:start_apps(apps()), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_management, emqx_auth_mnesia]). + emqx_ct_helpers:stop_apps(apps()). t_app(_Config) -> {ok, AppSecret} = emqx_mgmt_auth:add_app(<<"app_id">>, <<"app_name">>), diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index 8e98567c4..a11ab6f26 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -57,13 +57,13 @@ groups() -> }]. init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_management, emqx_auth_mnesia]), + emqx_ct_helpers:start_apps([emqx_management, emqx_auth_mnesia, emqx_modules]), ekka_mnesia:start(), emqx_mgmt_auth:mnesia(boot), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_auth_mnesia, emqx_management]), + emqx_ct_helpers:stop_apps([emqx_auth_mnesia, emqx_management, emqx_modules]), ekka_mnesia:ensure_stopped(). init_per_testcase(data, Config) -> From cac0ee7f846d5e6f5401a3c9442f606a951dd59b Mon Sep 17 00:00:00 2001 From: deen13 Date: Tue, 16 Mar 2021 16:58:19 +0100 Subject: [PATCH 032/158] fix(helm): Move pullSecrets up to pod level Previously the pullSecrets was defined on container level which is not supported by kubernetes. --- deploy/charts/emqx/templates/StatefulSet.yaml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index c364ec83d..db6c28815 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -88,17 +88,17 @@ spec: {{- if .Values.initContainers }} initContainers: {{ toYaml .Values.initContainers | indent 8 }} + {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} {{- end }} containers: - name: emqx image: "{{ .Values.image.repository }}:{{ .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- if .Values.image.pullSecrets }} - imagePullSecrets: - {{- range .Values.image.pullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} ports: - name: mqtt containerPort: {{ .Values.emqxConfig.EMQX_LISTENER__TCP__EXTERNAL | default 1883 }} @@ -120,14 +120,14 @@ spec: containerPort: 4370 envFrom: - configMapRef: - name: {{ include "emqx.fullname" . }}-env + name: {{ include "emqx.fullname" . }}-env env: - name: EMQX_NAME - value: {{ .Release.Name }} + value: {{ .Release.Name }} - name: EMQX_CLUSTER__K8S__APP_NAME - value: {{ .Release.Name }} + value: {{ .Release.Name }} - name: EMQX_CLUSTER__DISCOVERY - value: k8s + value: k8s - name: EMQX_CLUSTER__K8S__SERVICE_NAME value: {{ include "emqx.fullname" . }}-headless - name: EMQX_CLUSTER__K8S__NAMESPACE From c7b44caa1dc8eca211438b512caab42809950d40 Mon Sep 17 00:00:00 2001 From: Karol Kaczmarek Date: Wed, 17 Mar 2021 16:41:04 +0100 Subject: [PATCH 033/158] feat(acl): Add possibility to remove all acl cache --- src/emqx_acl_cache.erl | 30 ++++++++++++++++++++---------- test/emqx_acl_cache_SUITE.erl | 24 ++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/emqx_acl_cache.erl b/src/emqx_acl_cache.erl index 5e327e210..2ba2bf04d 100644 --- a/src/emqx_acl_cache.erl +++ b/src/emqx_acl_cache.erl @@ -27,6 +27,7 @@ , get_cache_max_size/0 , get_cache_ttl/0 , is_enabled/0 + , drain_cache/0 ]). %% export for test @@ -47,6 +48,7 @@ %% Wrappers for key and value cache_k(PubSub, Topic)-> {PubSub, Topic}. cache_v(AclResult)-> {AclResult, time_now()}. +drain_k() -> {?MODULE, drain_timestamp}. -spec(is_enabled() -> boolean()). is_enabled() -> @@ -86,10 +88,10 @@ get_acl_cache(PubSub, Topic) -> put_acl_cache(PubSub, Topic, AclResult) -> MaxSize = get_cache_max_size(), true = (MaxSize =/= 0), Size = get_cache_size(), - if - Size < MaxSize -> + case Size < MaxSize of + true -> add_acl(PubSub, Topic, AclResult); - Size =:= MaxSize -> + false -> NewestK = get_newest_key(), {_AclResult, CachedAt} = erlang:get(NewestK), if_expired(CachedAt, @@ -145,6 +147,11 @@ foreach_acl_cache(Fun) -> _ = map_acl_cache(Fun), ok. +%% All acl cache entries added before `drain_cache()` invocation will become expired +drain_cache() -> + _ = persistent_term:put(drain_k(), time_now()), + ok. + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- @@ -185,11 +192,14 @@ incr_cache_size() -> erlang:put(acl_cache_size, get_cache_size() + 1), ok. decr_cache_size() -> Size = get_cache_size(), - if Size > 1 -> + case Size > 1 of + true -> erlang:put(acl_cache_size, Size-1); - Size =< 1 -> + false -> erlang:put(acl_cache_size, 0) - end, ok. + end, + ok. + set_cache_size(N) -> erlang:put(acl_cache_size, N), ok. @@ -239,8 +249,8 @@ time_now() -> erlang:system_time(millisecond). if_expired(CachedAt, Fun) -> TTL = get_cache_ttl(), Now = time_now(), - if (CachedAt + TTL) =< Now -> - Fun(true); - true -> - Fun(false) + CurrentEvictTimestamp = persistent_term:get(drain_k(), 0), + case CachedAt =< CurrentEvictTimestamp orelse (CachedAt + TTL) =< Now of + true -> Fun(true); + false -> Fun(false) end. diff --git a/test/emqx_acl_cache_SUITE.erl b/test/emqx_acl_cache_SUITE.erl index 8c7685aa2..8d760c284 100644 --- a/test/emqx_acl_cache_SUITE.erl +++ b/test/emqx_acl_cache_SUITE.erl @@ -55,6 +55,30 @@ t_clean_acl_cache(_) -> ?assertEqual(0, length(gen_server:call(ClientPid, list_acl_cache))), emqtt:stop(Client). + +t_drain_acl_cache(_) -> + {ok, Client} = emqtt:start_link([{clientid, <<"emqx_c">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0), + ct:sleep(100), + ClientPid = case emqx_cm:lookup_channels(<<"emqx_c">>) of + [Pid] when is_pid(Pid) -> + Pid; + Pids when is_list(Pids) -> + lists:last(Pids); + _ -> {error, not_found} + end, + Caches = gen_server:call(ClientPid, list_acl_cache), + ct:log("acl caches: ~p", [Caches]), + ?assert(length(Caches) > 0), + emqx_acl_cache:drain_cache(), + ?assertEqual(0, length(gen_server:call(ClientPid, list_acl_cache))), + ct:sleep(100), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + ?assert(length(gen_server:call(ClientPid, list_acl_cache)) > 0), + emqtt:stop(Client). + % optimize?? t_reload_aclfile_and_cleanall(_Config) -> From 1ee0432301b445a73c6525c9f0cc2c8d26240c19 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 17 Mar 2021 12:59:33 +0100 Subject: [PATCH 034/158] chore(build): bump to OTP-23.2.7 --- .ci/apps_tests/docker-compose.yaml | 2 +- .ci/build_packages/Dockerfile | 2 +- .ci/compatibility_tests/docker-compose-ldap.yaml | 2 +- .ci/compatibility_tests/docker-compose-mongo-tls.yaml | 2 +- .ci/compatibility_tests/docker-compose-mongo.yaml | 2 +- .ci/compatibility_tests/docker-compose-mysql-tls.yaml | 2 +- .ci/compatibility_tests/docker-compose-mysql.yaml | 2 +- .ci/compatibility_tests/docker-compose-pgsql-tls.yaml | 2 +- .ci/compatibility_tests/docker-compose-pgsql.yaml | 2 +- .../docker-compose-redis-cluster-tls.yaml | 2 +- .../docker-compose-redis-cluster.yaml | 2 +- .../docker-compose-redis-sentinel.yaml | 2 +- .../docker-compose-redis-single-tls.yaml | 2 +- .../docker-compose-redis-single.yaml | 2 +- .github/workflows/build_packages.yaml | 10 +++++----- .github/workflows/build_slim_packages.yaml | 6 +++--- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 2 +- .github/workflows/run_test_cases.yaml | 2 +- deploy/docker/Dockerfile | 2 +- docker.mk | 2 +- 21 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.ci/apps_tests/docker-compose.yaml b/.ci/apps_tests/docker-compose.yaml index 42cfe82f6..2df39fe6f 100644 --- a/.ci/apps_tests/docker-compose.yaml +++ b/.ci/apps_tests/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.2-ubuntu20.04 + image: emqx/build-env:erl23.2.7-ubuntu20.04 depends_on: - mysql_server - redis_server diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile index 197b7e731..92450440d 100644 --- a/.ci/build_packages/Dockerfile +++ b/.ci/build_packages/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=emqx/build-env:erl23.2.2-ubuntu20.04 +ARG BUILD_FROM=emqx/build-env:erl23.2.7-ubuntu20.04 FROM ${BUILD_FROM} ARG EMQX_NAME=emqx diff --git a/.ci/compatibility_tests/docker-compose-ldap.yaml b/.ci/compatibility_tests/docker-compose-ldap.yaml index 33b37e00c..1c47756d5 100644 --- a/.ci/compatibility_tests/docker-compose-ldap.yaml +++ b/.ci/compatibility_tests/docker-compose-ldap.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.2-ubuntu20.04 + image: emqx/build-env:erl23.2.7-ubuntu20.04 depends_on: - ldap_server networks: diff --git a/.ci/compatibility_tests/docker-compose-mongo-tls.yaml b/.ci/compatibility_tests/docker-compose-mongo-tls.yaml index 1611534f6..610056121 100644 --- a/.ci/compatibility_tests/docker-compose-mongo-tls.yaml +++ b/.ci/compatibility_tests/docker-compose-mongo-tls.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.2-ubuntu20.04 + image: emqx/build-env:erl23.2.7-ubuntu20.04 volumes: - ../../:/emqx working_dir: /emqx diff --git a/.ci/compatibility_tests/docker-compose-mongo.yaml b/.ci/compatibility_tests/docker-compose-mongo.yaml index 2f769ac63..c3e2a1d66 100644 --- a/.ci/compatibility_tests/docker-compose-mongo.yaml +++ b/.ci/compatibility_tests/docker-compose-mongo.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.2-ubuntu20.04 + image: emqx/build-env:erl23.2.7-ubuntu20.04 volumes: - ../..:/emqx working_dir: /emqx diff --git a/.ci/compatibility_tests/docker-compose-mysql-tls.yaml b/.ci/compatibility_tests/docker-compose-mysql-tls.yaml index ab9cbeed3..f3c9142ed 100644 --- a/.ci/compatibility_tests/docker-compose-mysql-tls.yaml +++ b/.ci/compatibility_tests/docker-compose-mysql-tls.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.2-ubuntu20.04 + image: emqx/build-env:erl23.2.7-ubuntu20.04 volumes: - ../../:/emqx working_dir: /emqx diff --git a/.ci/compatibility_tests/docker-compose-mysql.yaml b/.ci/compatibility_tests/docker-compose-mysql.yaml index 9fe8d1a33..adc3aa97f 100644 --- a/.ci/compatibility_tests/docker-compose-mysql.yaml +++ b/.ci/compatibility_tests/docker-compose-mysql.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.2-ubuntu20.04 + image: emqx/build-env:erl23.2.7-ubuntu20.04 volumes: - ../../:/emqx working_dir: /emqx diff --git a/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml b/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml index 6bb3d321e..fd9855c01 100644 --- a/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml +++ b/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.2-ubuntu20.04 + image: emqx/build-env:erl23.2.7-ubuntu20.04 volumes: - ../../:/emqx working_dir: /emqx diff --git a/.ci/compatibility_tests/docker-compose-pgsql.yaml b/.ci/compatibility_tests/docker-compose-pgsql.yaml index c5492d971..5204d85c9 100644 --- a/.ci/compatibility_tests/docker-compose-pgsql.yaml +++ b/.ci/compatibility_tests/docker-compose-pgsql.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.2-ubuntu20.04 + image: emqx/build-env:erl23.2.7-ubuntu20.04 volumes: - ../../:/emqx working_dir: /emqx diff --git a/.ci/compatibility_tests/docker-compose-redis-cluster-tls.yaml b/.ci/compatibility_tests/docker-compose-redis-cluster-tls.yaml index 06518854f..bc9e24931 100644 --- a/.ci/compatibility_tests/docker-compose-redis-cluster-tls.yaml +++ b/.ci/compatibility_tests/docker-compose-redis-cluster-tls.yaml @@ -5,7 +5,7 @@ version: '2.4' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.2-ubuntu20.04 + image: emqx/build-env:erl23.2.7-ubuntu20.04 volumes: - ../..:/emqx networks: diff --git a/.ci/compatibility_tests/docker-compose-redis-cluster.yaml b/.ci/compatibility_tests/docker-compose-redis-cluster.yaml index 213a06866..bdbcf11af 100644 --- a/.ci/compatibility_tests/docker-compose-redis-cluster.yaml +++ b/.ci/compatibility_tests/docker-compose-redis-cluster.yaml @@ -5,7 +5,7 @@ version: '2.4' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.2-ubuntu20.04 + image: emqx/build-env:erl23.2.7-ubuntu20.04 volumes: - ../..:/emqx networks: diff --git a/.ci/compatibility_tests/docker-compose-redis-sentinel.yaml b/.ci/compatibility_tests/docker-compose-redis-sentinel.yaml index b2b58fefe..134724da7 100644 --- a/.ci/compatibility_tests/docker-compose-redis-sentinel.yaml +++ b/.ci/compatibility_tests/docker-compose-redis-sentinel.yaml @@ -5,7 +5,7 @@ version: '2.4' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.2-ubuntu20.04 + image: emqx/build-env:erl23.2.7-ubuntu20.04 volumes: - ../..:/emqx networks: diff --git a/.ci/compatibility_tests/docker-compose-redis-single-tls.yaml b/.ci/compatibility_tests/docker-compose-redis-single-tls.yaml index 03d643754..0e0a0dd25 100644 --- a/.ci/compatibility_tests/docker-compose-redis-single-tls.yaml +++ b/.ci/compatibility_tests/docker-compose-redis-single-tls.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.2-ubuntu20.04 + image: emqx/build-env:erl23.2.7-ubuntu20.04 volumes: - ../..:/emqx networks: diff --git a/.ci/compatibility_tests/docker-compose-redis-single.yaml b/.ci/compatibility_tests/docker-compose-redis-single.yaml index 5d7acb865..b603758ab 100644 --- a/.ci/compatibility_tests/docker-compose-redis-single.yaml +++ b/.ci/compatibility_tests/docker-compose-redis-single.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.2-ubuntu20.04 + image: emqx/build-env:erl23.2.7-ubuntu20.04 volumes: - ../..:/emqx networks: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index f0c9aa2a3..e22a97d86 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -63,11 +63,11 @@ jobs: - name: build erlang timeout-minutes: 60 run: | - kerl build 23.2.2 - kerl install 23.2.2 $HOME/.kerl/23.2.2 + kerl build 23.2.7 + kerl install 23.2.7 $HOME/.kerl/23.2.7 - name: build run: | - . $HOME/.kerl/23.2.2/activate + . $HOME/.kerl/23.2.7/activate make emqx-pkg - name: test run: | @@ -149,7 +149,7 @@ jobs: - uses: actions/checkout@v1 - name: get deps env: - ERL_OTP: erl23.2.2 + ERL_OTP: erl23.2.7 run: | docker run -i --rm \ -e GITHUB_RUN_ID=$GITHUB_RUN_ID \ @@ -184,7 +184,7 @@ jobs: cd - - name: build emqx packages env: - ERL_OTP: erl23.2.2 + ERL_OTP: erl23.2.7 EMQX: ${{ matrix.emqx }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 217e06917..658bae2e8 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -5,17 +5,17 @@ on: [pull_request] jobs: build: runs-on: ubuntu-20.04 - + strategy: matrix: erl_otp: - - erl23.2.2 + - erl23.2.7 os: - ubuntu20.04 - centos8 container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }} - + steps: - uses: actions/checkout@v1 - name: build packages diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 91e25e203..4eeefe46c 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.2-ubuntu20.04 + container: emqx/build-env:erl23.2.7-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 04638ca08..0d557b605 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -114,7 +114,7 @@ jobs: relup_test: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.2-ubuntu20.04 + container: emqx/build-env:erl23.2.7-ubuntu20.04 defaults: run: shell: bash diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 0a2f13eca..2eb7855d5 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -12,7 +12,7 @@ on: jobs: run_static_analysis: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.2-ubuntu20.04 + container: emqx/build-env:erl23.2.7-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index b3ca6f4c8..288c79cec 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=emqx/build-env:erl23.2.2-alpine-amd64 +ARG BUILD_FROM=emqx/build-env:erl23.2.7-alpine-amd64 ARG RUN_FROM=alpine:3.12 FROM ${BUILD_FROM} AS builder diff --git a/docker.mk b/docker.mk index 142fff92e..074d8b698 100644 --- a/docker.mk +++ b/docker.mk @@ -58,7 +58,7 @@ docker-build: @docker build --no-cache \ --build-arg PKG_VSN=$(PKG_VSN) \ - --build-arg BUILD_FROM=emqx/build-env:erl23.2.2-alpine-$(ARCH) \ + --build-arg BUILD_FROM=emqx/build-env:erl23.2.7-alpine-$(ARCH) \ --build-arg RUN_FROM=$(ARCH)/alpine:3.12 \ --build-arg EMQX_NAME=$(EMQX_NAME) \ --build-arg QEMU_ARCH=$(QEMU_ARCH) \ From 862e48494482664d3c56c829473d3a8d911a9b6a Mon Sep 17 00:00:00 2001 From: Rory Z Date: Thu, 18 Mar 2021 16:30:48 +0800 Subject: [PATCH 035/158] chore(CI): build workflow support enterprise repo (#4363) --- .github/workflows/build_packages.yaml | 370 +++++++++++++-------- .github/workflows/build_slim_packages.yaml | 20 +- Makefile | 2 +- deploy/packages/deb/Makefile | 8 +- docker.mk | 8 +- 5 files changed, 259 insertions(+), 149 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index e22a97d86..fd9c5a198 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -11,50 +11,120 @@ on: - published jobs: + prepare: + runs-on: ubuntu-20.04 + container: emqx/build-env:erl23.2.7-ubuntu20.04 + + outputs: + profiles: ${{ steps.set_profile.outputs.profiles}} + + steps: + - uses: actions/checkout@v2 + with: + path: source + - name: set profile + id: set_profile + shell: bash + run: | + if make -C source emqx-ee --dry-run > /dev/null 2>&1; then + echo "::set-output name=profiles::[\"emqx-ee\"]" + else + echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]" + fi + - name: get_all_deps + if: endsWith(github.repository, 'emqx') + run: | + make -C source deps-all + zip -ryq source.zip source + - name: get_all_deps + if: endsWith(github.repository, 'enterprise') + run: | + echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials + git config --global credential.helper store + echo "${{ secrets.CI_GIT_TOKEN }}" >> source/scripts/git-token + make -C source deps-all + zip -ryq source.zip source + - uses: actions/upload-artifact@v2 + with: + name: source + path: source.zip + windows: runs-on: windows-2019 + needs: prepare + + strategy: + matrix: + profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + exclude: + - profile: emqx-edge + steps: - - uses: actions/checkout@v1 + - uses: actions/download-artifact@v2 + with: + name: source + path: . + - name: unzip source code + run: Expand-Archive -Path source.zip -DestinationPath ./ - uses: ilammy/msvc-dev-cmd@v1 - uses: gleam-lang/setup-erlang@v1.1.0 id: install_erlang with: otp-version: 23.2 - name: build + env: + PYTHON: python + DIAGNOSTIC: 1 run: | $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH" $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" ) if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") { $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]" - $pkg_name = "emqx-windows-$([regex]::matches($version, $regex).value).zip" + $pkg_name = "${{ matrix.profile }}-windows-$([regex]::matches($version, $regex).value).zip" } else { - $pkg_name = "emqx-windows-$($version -replace '/').zip" + $pkg_name = "${{ matrix.profile }}-windows-$($version -replace '/').zip" } - make emqx - mkdir -p _packages/emqx - Compress-Archive -Path _build/emqx/rel/emqx -DestinationPath _build/emqx/rel/$pkg_name - mv _build/emqx/rel/$pkg_name _packages/emqx - Get-FileHash -Path "_packages/emqx/$pkg_name" | Format-List | grep 'Hash' | awk '{print $3}' > _packages/emqx/$pkg_name.sha256 + cd source + make ${{ matrix.profile }} + mkdir -p _packages/${{ matrix.profile }} + Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name + mv _build/${{ matrix.profile }}/rel/$pkg_name _packages/${{ matrix.profile }} + Get-FileHash -Path "_packages/${{ matrix.profile }}/$pkg_name" | Format-List | grep 'Hash' | awk '{print $3}' > _packages/${{ matrix.profile }}/$pkg_name.sha256 - name: run emqx run: | - ./_build/emqx/rel/emqx/bin/emqx start - ./_build/emqx/rel/emqx/bin/emqx stop - ./_build/emqx/rel/emqx/bin/emqx install - ./_build/emqx/rel/emqx/bin/emqx uninstall + cd source + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall - uses: actions/upload-artifact@v1 + if: startsWith(github.ref, 'refs/tags/') with: - name: emqx - path: ./_packages/emqx/. + name: ${{ matrix.profile }} + path: source/_packages/${{ matrix.profile }}/. mac: runs-on: macos-10.15 + needs: prepare + + strategy: + matrix: + profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + exclude: + - profile: emqx-edge + steps: - - uses: actions/checkout@v1 + - uses: actions/download-artifact@v2 + with: + name: source + path: . + - name: unzip source code + run: unzip -q source.zip - name: prepare run: | brew install curl zip unzip gnu-sed kerl unixodbc freetds @@ -68,11 +138,12 @@ jobs: - name: build run: | . $HOME/.kerl/23.2.7/activate - make emqx-pkg + make -C source ${{ matrix.profile }}-pkg - name: test run: | - pkg_name=$(basename _packages/emqx/emqx-macos-*.zip) - unzip _packages/emqx/$pkg_name + cd source + pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip) + unzip _packages/${{ matrix.profile }}/$pkg_name gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' @@ -91,46 +162,51 @@ jobs: ./emqx/bin/emqx_ctl status ./emqx/bin/emqx stop rm -rf emqx - openssl dgst -sha256 ./_packages/emqx/$pkg_name | awk '{print $2}' > ./_packages/emqx/$pkg_name.sha256 + openssl dgst -sha256 ./_packages/${{ matrix.profile }}/$pkg_name | awk '{print $2}' > ./_packages/${{ matrix.profile }}/$pkg_name.sha256 - uses: actions/upload-artifact@v1 + if: startsWith(github.ref, 'refs/tags/') with: - name: emqx - path: ./_packages/emqx/. + name: ${{ matrix.profile }} + path: source/_packages/${{ matrix.profile }}/. linux: runs-on: ubuntu-20.04 + needs: prepare + strategy: matrix: + profile: ${{fromJSON(needs.prepare.outputs.profiles)}} arch: - - amd64 - - arm64 - emqx: - - emqx - - emqx-edge + - amd64 + - arm64 os: - - ubuntu20.04 - - ubuntu18.04 - - ubuntu16.04 - - debian10 - - debian9 - - opensuse - - centos8 - - centos7 - - centos6 - - raspbian10 - - raspbian9 + - ubuntu20.04 + - ubuntu18.04 + - ubuntu16.04 + - debian10 + - debian9 + # - opensuse + - centos8 + - centos7 + - centos6 + - raspbian10 + - raspbian9 exclude: - - os: raspbian9 - arch: amd64 - - os: raspbian9 - emqx: emqx - - os: raspbian10 - arch: amd64 - - os: raspbian10 - emqx: emqx - os: centos6 arch: arm64 + - os: raspbian9 + arch: amd64 + - os: raspbian10 + arch: amd64 + - os: raspbian9 + profile: emqx + - os: raspbian10 + profile: emqx + - os: raspbian9 + profile: emqx-ee + - os: raspbian10 + profile: emqx-ee defaults: run: @@ -146,71 +222,72 @@ jobs: docker info docker buildx create --use --name mybuild docker run --rm --privileged tonistiigi/binfmt --install all - - uses: actions/checkout@v1 - - name: get deps - env: - ERL_OTP: erl23.2.7 - run: | - docker run -i --rm \ - -e GITHUB_RUN_ID=$GITHUB_RUN_ID \ - -e GITHUB_REF=$GITHUB_REF \ - -v $(pwd):/emqx \ - -w /emqx \ - emqx/build-env:${ERL_OTP}-debian10 \ - bash -c "make deps-all" + - uses: actions/download-artifact@v2 + with: + name: source + path: . + - name: unzip source code + run: unzip -q source.zip - name: downloads emqx zip packages env: - EMQX: ${{ matrix.emqx }} + PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} run: | set -e -u -x - if [ $EMQX = "emqx-edge" ];then broker="emqx-edge"; else broker="emqx-ce"; fi + cd source if [ $ARCH = "arm64" ];then arch="aarch64"; else arch="x86_64"; fi + if [ $PROFILE = "emqx" ];then broker="emqx-ce"; else broker="$PROFILE"; fi + if [ $PROFILE = "emqx-ee" ];then edition='enterprise'; else edition='opensource'; fi - vsn="$(grep -oE '\{vsn, (.*)\}' src/emqx.app.src | sed -r 's/\{vsn, (.*)\}/\1/g' | sed 's/\"//g')" + vsn="$(grep -E "define.+EMQX_RELEASE.+${edition}" include/emqx_release.hrl | cut -d '"' -f2)" pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" - old_vsns=($(git tag -l "$pre_vsn.[0-9]" | sed "s/$vsn//")) + if [ $PROFILE = "emqx-ee" ]; then + old_vsns=($(git tag -l "e$pre_vsn.[0-9]" | sed "s/e$vsn//")) + else + old_vsns=($(git tag -l "v$pre_vsn.[0-9]" | sed "s/v$vsn//")) + fi - mkdir -p tmp/relup_packages/$EMQX - cd tmp/relup_packages/$EMQX + mkdir -p tmp/relup_packages/$PROFILE + cd tmp/relup_packages/$PROFILE for tag in ${old_vsns[@]};do - if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/v${tag#[e|v]}/$EMQX-$SYSTEM-${tag#[e|v]}-$arch.zip) | grep -oE "^[23]+")" ];then - wget https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/v${tag#[e|v]}/$EMQX-$SYSTEM-${tag#[e|v]}-$arch.zip - wget https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/v${tag#[e|v]}/$EMQX-$SYSTEM-${tag#[e|v]}-$arch.zip.sha256 - echo "$(cat $EMQX-$SYSTEM-${tag#[e|v]}-$arch.zip.sha256) $EMQX-$SYSTEM-${tag#[e|v]}-$arch.zip" | sha256sum -c || exit 1 + if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$arch.zip) | grep -oE "^[23]+")" ];then + wget https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$arch.zip + wget https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$arch.zip.sha256 + echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$arch.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$arch.zip" | sha256sum -c || exit 1 fi done cd - - name: build emqx packages env: ERL_OTP: erl23.2.7 - EMQX: ${{ matrix.emqx }} + PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} run: | - set -e -u -x + set -e -u + cd source docker buildx build --no-cache \ --platform=linux/$ARCH \ -t cross_build_emqx_for_$SYSTEM \ -f .ci/build_packages/Dockerfile \ --build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-$SYSTEM \ - --build-arg EMQX_NAME=$EMQX \ - --output type=tar,dest=/tmp/cross-build-$EMQX-for-$SYSTEM.tar . + --build-arg EMQX_NAME=$PROFILE \ + --output type=tar,dest=/tmp/cross-build-$PROFILE-for-$SYSTEM.tar . - mkdir -p /tmp/packages/$EMQX - tar -xvf /tmp/cross-build-$EMQX-for-$SYSTEM.tar --wildcards emqx/_packages/$EMQX/* - mv emqx/_packages/$EMQX/* /tmp/packages/$EMQX/ - rm -rf /tmp/cross-build-$EMQX-for-$SYSTEM.tar + mkdir -p /tmp/packages/$PROFILE + tar -xvf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar --wildcards emqx/_packages/$PROFILE/* + mv emqx/_packages/$PROFILE/* /tmp/packages/$PROFILE/ + rm -rf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar docker rm -f $(docker ps -a -q) docker volume prune -f - name: create sha256 env: - EMQX: ${{ matrix.emqx }} + PROFILE: ${{ matrix.profile}} run: | - if [ -d /tmp/packages/$EMQX ]; then - cd /tmp/packages/$EMQX + if [ -d /tmp/packages/$PROFILE ]; then + cd /tmp/packages/$PROFILE for var in $(ls emqx-* ); do bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256" done @@ -219,52 +296,74 @@ jobs: - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: - name: ${{ matrix.emqx }} - path: /tmp/packages/${{ matrix.emqx }}/. + name: ${{ matrix.profile }} + path: /tmp/packages/${{ matrix.profile }}/. docker: runs-on: ubuntu-20.04 + needs: prepare + strategy: matrix: + profile: ${{fromJSON(needs.prepare.outputs.profiles)}} arch: - - [amd64, x86_64] - - [arm64v8, aarch64] - - [arm32v7, arm] - - [i386, i386] - - [s390x, s390x] + - [amd64, x86_64] + - [arm64v8, aarch64] + - [arm32v7, arm] + - [i386, i386] + - [s390x, s390x] + exclude: + - profile: emqx-ee + arch: [i386, i386] + - profile: emqx-ee + arch: [s390x, s390x] steps: - - uses: actions/checkout@v1 + - uses: actions/download-artifact@v2 + with: + name: source + path: . + - name: unzip source code + run: unzip -q source.zip - name: build emqx docker image env: + PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch[0] }} QEMU_ARCH: ${{ matrix.arch[1] }} run: | sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - sudo TARGET=emqx/emqx ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH make docker - cd _packages/emqx && for var in $(ls emqx-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd - + cd source + sudo TARGET=emqx/$PROFILE ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH make docker + cd _packages/$PROFILE && for var in $(ls ${PROFILE}-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd - + - uses: actions/upload-artifact@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + name: ${{ matrix.profile }} + path: source/_packages/${{ matrix.profile }}/. - sudo TARGET=emqx/emqx-edge ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH make docker - cd _packages/emqx-edge && for var in $(ls emqx-edge-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd - - - uses: actions/upload-artifact@v1 + delete-artifact: + runs-on: ubuntu-20.04 + needs: [prepare, windows, mac, linux, docker] + steps: + - uses: geekyeggo/delete-artifact@v1 with: - name: emqx - path: ./_packages/emqx/. - - uses: actions/upload-artifact@v1 - with: - name: emqx-edge - path: ./_packages/emqx-edge/. + name: source upload: runs-on: ubuntu-20.04 - needs: [windows, mac, linux, docker] - if: startsWith(github.ref, 'refs/tags/') + needs: [prepare, windows, mac, linux, docker] + + strategy: + matrix: + profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + steps: + - uses: actions/checkout@v2 - name: get_version run: | echo 'version<> $GITHUB_ENV @@ -272,47 +371,38 @@ jobs: echo 'EOF' >> $GITHUB_ENV - uses: actions/download-artifact@v2 with: - name: emqx - path: ./_packages/emqx - - uses: actions/download-artifact@v2 - with: - name: emqx-edge - path: ./_packages/emqx-edge + name: ${{ matrix.profile }} + path: ./_packages/${{ matrix.profile }} - name: install dos2unix run: sudo apt-get update && sudo apt install -y dos2unix - name: get packages run: | - set -e -x -u - for EMQX in emqx emqx-edge; do - cd _packages/$EMQX - for var in $( ls |grep emqx |grep -v sha256); do - dos2unix $var.sha256 - echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1 - done - cd - + set -e -u + cd _packages/${{ matrix.profile }} + for var in $( ls |grep emqx |grep -v sha256); do + dos2unix $var.sha256 + echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1 done + cd - - name: upload aws s3 run: | - set -e -x -u + set -e -u + if [ "${{ matrix.profile }}" == "emqx" ];then + broker="emqx-ce" + else + broker=${{ matrix.profile }} + fi aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws configure set default.region us-west-2 + aws configure set default.region ${{ secrets.AWS_DEFAULT_REGION }} - aws s3 cp --recursive _packages/emqx s3://packages.emqx/emqx-ce/${{ env.version }} - aws s3 cp --recursive _packages/emqx-edge s3://packages.emqx/emqx-edge/${{ env.version }} - aws cloudfront create-invalidation --distribution-id E170YEULGLT8XB --paths "/emqx-ce/${{ env.version }}/*,/emqx-edge/${{ env.version }}/*" - - mkdir packages - mv _packages/emqx/* packages - mv _packages/emqx-edge/* packages - - uses: actions/checkout@v2 - with: - path: emqx + aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$broker/${{ env.version }} + aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$broker/${{ env.version }}/*" - uses: Rory-Z/upload-release-asset@v1 if: github.event_name == 'release' with: repo: emqx - path: "packages/emqx-*" + path: "_packages/${{ matrix.profile }}/emqx-*" token: ${{ github.token }} - name: update to emqx.io if: github.event_name == 'release' @@ -329,15 +419,22 @@ jobs: if: github.event_name == 'release' run: | set -e -x -u - sudo make -C emqx docker-prepare - cd packages && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd - + sudo make docker-prepare + cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd - echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin - sudo TARGET=emqx/emqx make -C emqx docker-push - sudo TARGET=emqx/emqx make -C emqx docker-manifest-list - sudo TARGET=emqx/emqx-edge make -C emqx docker-push - sudo TARGET=emqx/emqx-edge make -C emqx docker-manifest-list + sudo TARGET=emqx/${{ matrix.profile }} make docker-push + sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list - name: update repo.emqx.io - if: github.event_name == 'release' + if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee' + run: | + curl --silent --show-error \ + -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + -X POST \ + -d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \ + "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" + - name: update repo.emqx.io + if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' run: | curl --silent --show-error \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ @@ -346,7 +443,7 @@ jobs: -d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \ "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" - name: update homebrew packages - if: github.event_name == 'release' + if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' run: | if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then curl --silent --show-error \ @@ -358,7 +455,4 @@ jobs: fi - uses: geekyeggo/delete-artifact@v1 with: - name: emqx - - uses: geekyeggo/delete-artifact@v1 - with: - name: emqx-edge + name: ${{ matrix.profile }} diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 658bae2e8..62613031f 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -1,6 +1,8 @@ name: Build slim packages -on: [pull_request] +on: + - pull_request + - workflow_dispatch jobs: build: @@ -12,15 +14,27 @@ jobs: - erl23.2.7 os: - ubuntu20.04 - - centos8 + - centos7 container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 - name: build packages - run: make emqx-pkg + run: | + if make -C source emqx-ee --dry-run > /dev/null 2>&1; then + echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials + git config --global credential.helper store + echo "${{ secrets.CI_GIT_TOKEN }}" >> source/scripts/git-token + make emqx-ee-pkg + else + make emqx-pkg + fi - name: pakcages test run: | export CODE_PATH=$GITHUB_WORKSPACE .ci/build_packages/tests.sh + - uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.os }} + path: _packages/**/*.zip diff --git a/Makefile b/Makefile index 57af7b8db..cb36fcaec 100644 --- a/Makefile +++ b/Makefile @@ -100,7 +100,7 @@ ifneq ($(OS),Windows_NT) endif .PHONY: $(REL_PROFILES:%=%-tar) $(PKG_PROFILES:%=%-tar) -$(REL_PROFILES:%=%-tar) $(PKG_PROFILES:%=%-tar): $(REBAR) get-dashboard +$(REL_PROFILES:%=%-tar) $(PKG_PROFILES:%=%-tar): $(REBAR) get-dashboard $(CONF_SEGS) @$(BUILD) $(subst -tar,,$(@)) tar ## zip targets depend on the corresponding relup and tar artifacts diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index c7a5ff2a0..28799a21d 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -14,11 +14,9 @@ all: | $(BUILT) cp -r debian $(SRCDIR)/ sed -i "s##$(shell date -u '+%a, %d %b %Y %T %z')#g" $(SRCDIR)/debian/changelog sed -i "s##$(PKG_VSN)#g" $(SRCDIR)/debian/changelog - if [ ! -z $(shell echo $(EMQX_NAME) |grep edge) ]; then \ - sed -i "s/emqx-pkg/emqx-edge-pkg/g" $(SRCDIR)/debian/rules; \ - sed -i "s debian/emqx debian/emqx-edge g" $(SRCDIR)/debian/rules; \ - sed -i "s/Package: emqx/Package: emqx-edge/1" $(SRCDIR)/debian/control; \ - fi + sed -i "s/emqx-pkg/$(EMQX_NAME)-pkg/g" $(SRCDIR)/debian/rules; \ + sed -i "s debian/emqx debian/$(EMQX_NAME) g" $(SRCDIR)/debian/rules; \ + sed -i "s/Package: emqx/Package: $(EMQX_NAME)/1" $(SRCDIR)/debian/control; \ cd $(SRCDIR) && dpkg-buildpackage -us -uc mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) cp $(SRCDIR)/../$(SOURCE_PKG).deb $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb diff --git a/docker.mk b/docker.mk index 074d8b698..d2333b1b0 100644 --- a/docker.mk +++ b/docker.mk @@ -9,12 +9,16 @@ QEMU_VERSION ?= v5.0.0-2 OS ?= alpine export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) -ifeq ($(findstring emqx-edge, $(TARGET)), emqx-edge) +ifeq ($(findstring emqx-ee, $(TARGET)), emqx-ee) + ARCH_LIST := amd64 arm64v8 arm32v7 + EMQX_NAME := emqx-ee +else ifeq ($(findstring emqx-edge, $(TARGET)), emqx-edge) + ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x EMQX_NAME := emqx-edge else + ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x EMQX_NAME := emqx endif -ARCH_LIST = amd64 arm64v8 arm32v7 i386 s390x .PHONY: docker docker: docker-build docker-tag docker-save From 2e8dbe9130b170732a5bfbe9453a9df3577d7541 Mon Sep 17 00:00:00 2001 From: Swilder-M Date: Thu, 18 Mar 2021 17:47:56 +0800 Subject: [PATCH 036/158] fix(README): update README files --- README-CN.md | 19 ++++++++++--------- README.md | 8 ++++---- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/README-CN.md b/README-CN.md index 045a089ee..0172e14bb 100644 --- a/README-CN.md +++ b/README-CN.md @@ -6,8 +6,9 @@ [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx)](https://hub.docker.com/r/emqx/emqx) [![Slack Invite]()](https://slack-invite.emqx.io) [![Twitter](https://img.shields.io/badge/Twitter-EMQ%20X-1DA1F2?logo=twitter)](https://twitter.com/emqtt) +[![Community](https://img.shields.io/badge/Community-EMQ%20X-yellow)](https://askemq.com) -[![最棒的物联网 MQTT 开源团队期待您的加入](https://www.emqx.io/static/img/github_readme_cn_bg.png)](https://www.emqx.io/cn/careers) +[![最棒的物联网 MQTT 开源团队期待您的加入](https://www.emqx.io/static/img/github_readme_cn_bg.png)](https://careers.emqx.cn/) [English](./README.md) | 简体中文 | [日本語](./README-JP.md) @@ -16,7 +17,7 @@ 从 3.0 版本开始，*EMQ X* 完整支持 MQTT V5.0 协议规范，向下兼容 MQTT V3.1 和 V3.1.1，并支持 MQTT-SN、CoAP、LwM2M、WebSocket 和 STOMP 等通信协议。EMQ X 3.0 单集群可支持千万级别的 MQTT 并发连接。 - 新功能的完整列表，请参阅 [EMQ X Release Notes](https://github.com/emqx/emqx/releases)。 -- 获取更多信息，请访问 [EMQ X 官网](https://www.emqx.io/cn/)。 +- 获取更多信息，请访问 [EMQ X 官网](https://www.emqx.cn/)。 ## 安装 @@ -25,15 +26,15 @@ #### EMQ X Docker 镜像安装 ``` -docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx +docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx ``` #### 二进制软件包安装 -需从 [EMQ X 下载](https://www.emqx.io/cn/downloads) 页面获取相应操作系统的二进制软件包。 +需从 [EMQ X 下载](https://www.emqx.cn/downloads) 页面获取相应操作系统的二进制软件包。 -- [单节点安装文档](https://docs.emqx.io/broker/latest/cn/getting-started/install.html) -- [集群配置文档](https://docs.emqx.io/broker/latest/cn/advanced/cluster.html) +- [单节点安装文档](https://docs.emqx.cn/broker/latest/getting-started/install.html) +- [集群配置文档](https://docs.emqx.cn/broker/latest/advanced/cluster.html) ## 从源码构建 @@ -75,7 +76,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer ## FAQ -访问 [EMQ X FAQ](https://docs.emqx.io/broker/latest/cn/faq/faq.html) 以获取常见问题的帮助。 +访问 [EMQ X FAQ](https://docs.emqx.cn/broker/latest/faq/faq.html) 以获取常见问题的帮助。 ## 产品路线 @@ -89,9 +90,9 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer - [Twitter](https://twitter.com/emqtt) - [Facebook](https://www.facebook.com/emqxmqtt) - [Reddit](https://www.reddit.com/r/emqx/) -- [Forum](https://groups.google.com/d/forum/emqtt) +- [Forum](https://askemq.com) - [Weibo](https://weibo.com/emqtt) -- [Blog](https://www.emqx.io/cn/blog) +- [Blog](https://www.emqx.cn/blog) 欢迎你将任何 bug、问题和功能请求提交到 [emqx/emqx](https://github.com/emqx/emqx/issues)。 diff --git a/README.md b/README.md index 49f44d33f..af683f881 100644 --- a/README.md +++ b/README.md @@ -28,15 +28,15 @@ See more details for building and running *EMQ X* on Windows in [Windows.md](./W #### Installing via EMQ X Docker Image ``` -docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx +docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx ``` #### Installing via Binary Package Get the binary package of the corresponding OS from [EMQ X Download](https://www.emqx.io/downloads) page. -- [Single Node Install](https://docs.emqx.io/broker/latest/en/getting-started/install.html) -- [Multi Node Install](https://docs.emqx.io/broker/latest/en/advanced/cluster.html) +- [Single Node Install](https://docs.emqx.io/en/broker/latest/getting-started/install.html) +- [Multi Node Install](https://docs.emqx.io/en/broker/latest/advanced/cluster.html) ## Build From Source @@ -113,7 +113,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer ## FAQ -Visiting [EMQ X FAQ](https://docs.emqx.io/broker/latest/en/faq/faq.html) to get help of common problems. +Visiting [EMQ X FAQ](https://docs.emqx.io/en/broker/latest/faq/faq.html) to get help of common problems. ## Roadmap From 01149bf6876f229bae1908c56005ec8b7a014a27 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 17 Mar 2021 13:30:30 +0100 Subject: [PATCH 037/158] feat(compile): support debug_info encryption if EMQX_COMPILE_SECRET_FILE is set, debug_info of beam files would be encrypted. EMQX_COMPILE_SECRET_FILE contains secret string. --- rebar.config.erl | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/rebar.config.erl b/rebar.config.erl index 0f35b97b6..1479a9358 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -101,9 +101,20 @@ common_compile_opts() -> | [{d, 'EMQX_ENTERPRISE'} || is_enterprise()] ]. +prod_compile_opts(false) -> + prod_compile_opts(); +prod_compile_opts(SecretFile) -> + SecretText = get_compile_secret(SecretFile), + beam_lib:crypto_key_fun( + fun(init) -> ok; + (clear) -> ok; + ({debug_info, _Mode, _Module, _Filename}) -> SecretText + end + ), + [{debug_info_key, SecretText} | prod_compile_opts()]. + prod_compile_opts() -> [ compressed - , no_debug_info , warnings_as_errors | common_compile_opts() ]. @@ -115,16 +126,18 @@ test_compile_opts() -> profiles() -> Vsn = get_vsn(), - [ {'emqx', [ {erl_opts, prod_compile_opts()} + SecretFile = os:getenv("EMQX_COMPILE_SECRET_FILE"), + SecretFile =/= false andalso io:format("debug_info encryption enabled !~n"), + [ {'emqx', [ {erl_opts, prod_compile_opts(SecretFile)} , {relx, relx(Vsn, cloud, bin)} ]} - , {'emqx-pkg', [ {erl_opts, prod_compile_opts()} + , {'emqx-pkg', [ {erl_opts, prod_compile_opts(SecretFile)} , {relx, relx(Vsn, cloud, pkg)} ]} - , {'emqx-edge', [ {erl_opts, prod_compile_opts()} + , {'emqx-edge', [ {erl_opts, prod_compile_opts(SecretFile)} , {relx, relx(Vsn, edge, bin)} ]} - , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()} + , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts(SecretFile)} , {relx, relx(Vsn, edge, pkg)} ]} , {check, [ {erl_opts, test_compile_opts()} @@ -457,6 +470,10 @@ list_dir(Dir) -> {ok, Names} = file:list_dir(Dir), [list_to_atom(Name) || Name <- Names, filelib:is_dir(filename:join([Dir, Name]))]. +get_compile_secret(SecretFile) -> + {ok, Secret} = file:read_file(SecretFile), + string:trim(binary_to_list(Secret)). + %% ==== Enterprise supports below ================================================================== ee_profiles(_Vsn) -> []. From 8828c483528af4602b1f629231dbe30e041644fe Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 18 Mar 2021 19:37:04 +0100 Subject: [PATCH 038/158] chore(build): avoid reading secret keys repeatedly --- rebar.config.erl | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/rebar.config.erl b/rebar.config.erl index 1479a9358..837b9a56b 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -101,17 +101,25 @@ common_compile_opts() -> | [{d, 'EMQX_ENTERPRISE'} || is_enterprise()] ]. +make_debug_info_key_fun() -> + case os:getenv("EMQX_COMPILE_SECRET_FILE") of + false -> false; + "" -> false; + Fn -> + io:format("===< using debug_info encryption key from file ~p!~n", [Fn]), + SecretText = get_compile_secret(Fn), + F = fun(init) -> ok; + (clear) -> ok; + ({debug_info, _Mode, _Module, _Filename}) -> SecretText + end, + beam_lib:crypto_key_fun(F), + F + end. + prod_compile_opts(false) -> prod_compile_opts(); -prod_compile_opts(SecretFile) -> - SecretText = get_compile_secret(SecretFile), - beam_lib:crypto_key_fun( - fun(init) -> ok; - (clear) -> ok; - ({debug_info, _Mode, _Module, _Filename}) -> SecretText - end - ), - [{debug_info_key, SecretText} | prod_compile_opts()]. +prod_compile_opts(KeyFun) -> + [{debug_info_key, KeyFun({debug_info, "", "", ""})} | prod_compile_opts()]. prod_compile_opts() -> [ compressed @@ -126,18 +134,17 @@ test_compile_opts() -> profiles() -> Vsn = get_vsn(), - SecretFile = os:getenv("EMQX_COMPILE_SECRET_FILE"), - SecretFile =/= false andalso io:format("debug_info encryption enabled !~n"), - [ {'emqx', [ {erl_opts, prod_compile_opts(SecretFile)} + KeyFun = make_debug_info_key_fun(), + [ {'emqx', [ {erl_opts, prod_compile_opts(KeyFun)} , {relx, relx(Vsn, cloud, bin)} ]} - , {'emqx-pkg', [ {erl_opts, prod_compile_opts(SecretFile)} + , {'emqx-pkg', [ {erl_opts, prod_compile_opts(KeyFun)} , {relx, relx(Vsn, cloud, pkg)} ]} - , {'emqx-edge', [ {erl_opts, prod_compile_opts(SecretFile)} + , {'emqx-edge', [ {erl_opts, prod_compile_opts(KeyFun)} , {relx, relx(Vsn, edge, bin)} ]} - , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts(SecretFile)} + , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts(KeyFun)} , {relx, relx(Vsn, edge, pkg)} ]} , {check, [ {erl_opts, test_compile_opts()} @@ -471,8 +478,13 @@ list_dir(Dir) -> [list_to_atom(Name) || Name <- Names, filelib:is_dir(filename:join([Dir, Name]))]. get_compile_secret(SecretFile) -> - {ok, Secret} = file:read_file(SecretFile), - string:trim(binary_to_list(Secret)). + case file:read_file(SecretFile) of + {ok, Secret} -> + string:trim(binary_to_list(Secret)); + {error, Reason} -> + io:format("===< Failed to read debug_info encryption key file ~s: ~p~n", [SecretFile, Reason]), + exit(Reason) + end. %% ==== Enterprise supports below ================================================================== From b397b56db74ca70e4bb225a47df1779050f1e585 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 18 Mar 2021 20:48:53 +0100 Subject: [PATCH 039/158] chore(gitattributes): fix file classification. --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitattributes b/.gitattributes index fb9a97dba..4ed73da9a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,5 @@ * text=auto *.* text eol=lf +*.jpg -text +*.png -text +*.pdf -text From 0d75ea874b8c360e881aca5bc5b83b84b79f95bd Mon Sep 17 00:00:00 2001 From: wwhai Date: Thu, 18 Mar 2021 11:55:11 +0800 Subject: [PATCH 040/158] fix(rule-engine): add 'undefined' type for create time --- apps/emqx_rule_engine/include/rule_engine.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/include/rule_engine.hrl b/apps/emqx_rule_engine/include/rule_engine.hrl index ccc2685e1..a7fe9c60a 100644 --- a/apps/emqx_rule_engine/include/rule_engine.hrl +++ b/apps/emqx_rule_engine/include/rule_engine.hrl @@ -87,7 +87,7 @@ { id :: resource_id() , type :: resource_type_name() , config :: #{} %% the configs got from API for initializing resource - , created_at :: integer() %% epoch in millisecond precision + , created_at :: integer() | undefined %% epoch in millisecond precision , description :: binary() }). From 20086bcff18a77b24b4ad923600cbd269de031e1 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 18 Mar 2021 14:27:05 +0800 Subject: [PATCH 041/158] chore(CI): fvt test support enterprise repo --- .ci/fvt_tests/.env | 1 + .ci/fvt_tests/docker-compose.yaml | 4 +- .github/workflows/run_fvt_tests.yaml | 74 +++++++++++++++++++++++----- 3 files changed, 66 insertions(+), 13 deletions(-) create mode 100644 .ci/fvt_tests/.env diff --git a/.ci/fvt_tests/.env b/.ci/fvt_tests/.env new file mode 100644 index 000000000..26b92be81 --- /dev/null +++ b/.ci/fvt_tests/.env @@ -0,0 +1 @@ +TARGET=emqx/emqx diff --git a/.ci/fvt_tests/docker-compose.yaml b/.ci/fvt_tests/docker-compose.yaml index 22d48bef7..a71b910eb 100644 --- a/.ci/fvt_tests/docker-compose.yaml +++ b/.ci/fvt_tests/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3' services: emqx1: container_name: node1.emqx.io - image: emqx/emqx:build-alpine-amd64 + image: ${TARGET}:build-alpine-amd64 environment: - "EMQX_NAME=emqx" - "EMQX_HOST=node1.emqx.io" @@ -30,7 +30,7 @@ services: emqx2: container_name: node2.emqx.io - image: emqx/emqx:build-alpine-amd64 + image: ${TARGET}:build-alpine-amd64 environment: - "EMQX_NAME=emqx" - "EMQX_HOST=node2.emqx.io" diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 0d557b605..a3568350a 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -15,8 +15,19 @@ jobs: steps: - uses: actions/checkout@v1 + - name: prepare + run: | + if make emqx-ee --dry-run > /dev/null 2>&1; then + echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials + git config --global credential.helper store + echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token + make deps-emqx-ee + echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV + else + echo "TARGET=emqx/emqx" >> $GITHUB_ENV + fi - name: make emqx image - run: TARGET=emqx/emqx make docker + run: make docker - name: run emqx timeout-minutes: 5 run: | @@ -45,8 +56,19 @@ jobs: steps: - uses: actions/checkout@v1 + - name: prepare + run: | + if make emqx-ee --dry-run > /dev/null 2>&1; then + echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials + git config --global credential.helper store + echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token + make deps-emqx-ee + echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV + else + echo "TARGET=emqx/emqx" >> $GITHUB_ENV + fi - name: make emqx image - run: TARGET=emqx/emqx make docker + run: make docker - name: install k3s env: KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" @@ -69,15 +91,21 @@ jobs: timeout-minutes: 5 run: | version=$(./pkg-vsn.sh) - sudo docker save emqx/emqx:$version -o emqx.tar.gz + sudo docker save ${TARGET}:$version -o emqx.tar.gz sudo k3s ctr image import emqx.tar.gz sed -i -r "s/^appVersion: .*$/appVersion: \"${version}\"/g" deploy/charts/emqx/Chart.yaml - sed -i -r 's/ pullPolicy: .*$/ pullPolicy: Never/g' deploy/charts/emqx/values.yaml sed -i '/emqx_telemetry/d' deploy/charts/emqx/values.yaml - helm install emqx --set emqxAclConfig="" --set emqxConfig.EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 deploy/charts/emqx --debug --dry-run - helm install emqx --set emqxAclConfig="" --set emqxConfig.EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 deploy/charts/emqx + helm install emqx \ + --set image.repository=${TARGET} \ + --set image.pullPolicy=Never \ + --set emqxAclConfig="" \ + --set image.pullPolicy=Never \ + --set emqxConfig.EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s \ + --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ + deploy/charts/emqx \ + --debug while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \ != "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do @@ -110,6 +138,7 @@ jobs: emqx2=$(kubectl get pods emqx-2 -o jsonpath='{.status.podIP}') pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host $emqx_svc + pytest -v paho.mqtt.testing/interoperability/test_client --host $emqx_svc pytest -v paho.mqtt.testing/interoperability/test_cluster --host1 $emqx1 --host2 $emqx2 relup_test: @@ -148,14 +177,37 @@ jobs: repository: ${{ github.repository }} path: emqx fetch-depth: 0 + - name: prepare + run: | + if make -C emqx emqx-ee --dry-run > /dev/null 2>&1; then + echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials + git config --global credential.helper store + echo "${{ secrets.CI_GIT_TOKEN }}" >> emqx/scripts/git-token + echo "PROFILE=emqx-ee" >> $GITHUB_ENV + else + echo "PROFILE=emqx" >> $GITHUB_ENV + fi - name: get version run: | set -e -x -u cd emqx - vsn="$(erl -eval '{ok, [{application,emqx, L} | _]} = file:consult("src/emqx.app.src"), {vsn, VSN} = lists:keyfind(vsn,1,L), io:fwrite(VSN), halt().' -noshell)" + if [ $PROFILE = "emqx" ];then + broker="emqx-ce" + edition='opensource' + else + broker="emqx-ee" + edition='enterprise' + fi + + vsn="$(grep -E "define.+EMQX_RELEASE.+${edition}" include/emqx_release.hrl | cut -d '"' -f2)" echo "VSN=$vsn" >> $GITHUB_ENV - pre_tag="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" - old_vsns="$(git tag -l "$pre_tag.[0-9]" | tr "\n" " " | sed "s/$vsn//")" + + pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" + if [ $PROFILE = "emqx" ]; then + old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | tr "\n" " " | sed "s/v$vsn//")" + else + old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | tr "\n" " " | sed "s/v$vsn//")" + fi echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV - name: download emqx run: | @@ -163,10 +215,10 @@ jobs: cd emqx old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for old_vsn in ${old_vsns[@]}; do - wget https://s3-us-west-2.amazonaws.com/packages.emqx/emqx-ce/v$old_vsn/emqx-ubuntu20.04-${old_vsn}-x86_64.zip + wget https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-x86_64.zip done - name: build emqx - run: make -C emqx emqx-zip + run: make -C emqx ${PROFILE}-zip - name: build emqtt-bench run: make -C emqtt-bench - name: build lux From ea384ec6b54738d544d9a86a1ac589a4a3479b2c Mon Sep 17 00:00:00 2001 From: Karol Kaczmarek Date: Thu, 18 Mar 2021 21:03:03 +0100 Subject: [PATCH 042/158] style(api): minirest imports removed --- .../src/emqx_mgmt_api_apps.erl | 22 +++---- .../src/emqx_mgmt_api_banned.erl | 14 ++-- .../src/emqx_mgmt_api_brokers.erl | 8 +-- .../src/emqx_mgmt_api_clients.erl | 64 +++++++++---------- .../src/emqx_mgmt_api_data.erl | 30 ++++----- .../src/emqx_mgmt_api_listeners.erl | 14 ++-- .../src/emqx_mgmt_api_metrics.erl | 8 +-- .../src/emqx_mgmt_api_nodes.erl | 6 +- .../src/emqx_mgmt_api_plugins.erl | 20 +++--- .../src/emqx_mgmt_api_pubsub.erl | 20 +++--- .../src/emqx_mgmt_api_routes.erl | 6 +- .../src/emqx_mgmt_api_stats.erl | 8 +-- .../src/emqx_mgmt_api_subscriptions.erl | 16 ++--- 13 files changed, 101 insertions(+), 135 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_apps.erl b/apps/emqx_management/src/emqx_mgmt_api_apps.erl index 55eaddbb1..7d5031698 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_apps.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_apps.erl @@ -20,10 +20,6 @@ -import(proplists, [get_value/2]). --import(minirest, [ return/0 - , return/1 - ]). - -rest_api(#{name => add_app, method => 'POST', path => "/apps/", @@ -69,30 +65,30 @@ add_app(_Bindings, Params) -> Status = get_value(<<"status">>, Params), Expired = get_value(<<"expired">>, Params), case emqx_mgmt_auth:add_app(AppId, Name, Secret, Desc, Status, Expired) of - {ok, AppSecret} -> return({ok, #{secret => AppSecret}}); - {error, Reason} -> return({error, Reason}) + {ok, AppSecret} -> minirest:return({ok, #{secret => AppSecret}}); + {error, Reason} -> minirest:return({error, Reason}) end. del_app(#{appid := AppId}, _Params) -> case emqx_mgmt_auth:del_app(AppId) of - ok -> return(); - {error, Reason} -> return({error, Reason}) + ok -> minirest:return(); + {error, Reason} -> minirest:return({error, Reason}) end. list_apps(_Bindings, _Params) -> - return({ok, [format(Apps)|| Apps <- emqx_mgmt_auth:list_apps()]}). + minirest:return({ok, [format(Apps)|| Apps <- emqx_mgmt_auth:list_apps()]}). lookup_app(#{appid := AppId}, _Params) -> case emqx_mgmt_auth:lookup_app(AppId) of {AppId, AppSecret, Name, Desc, Status, Expired} -> - return({ok, #{app_id => AppId, + minirest:return({ok, #{app_id => AppId, secret => AppSecret, name => Name, desc => Desc, status => Status, expired => Expired}}); undefined -> - return({ok, #{}}) + minirest:return({ok, #{}}) end. update_app(#{appid := AppId}, Params) -> @@ -101,8 +97,8 @@ update_app(#{appid := AppId}, Params) -> Status = get_value(<<"status">>, Params), Expired = get_value(<<"expired">>, Params), case emqx_mgmt_auth:update_app(AppId, Name, Desc, Status, Expired) of - ok -> return(); - {error, Reason} -> return({error, Reason}) + ok -> minirest:return(); + {error, Reason} -> minirest:return({error, Reason}) end. format({AppId, _AppSecret, Name, Desc, Status, Expired}) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index 483ab4be3..2d5c0da69 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -22,10 +22,6 @@ -import(proplists, [get_value/2]). --import(minirest, [ return/0 - , return/1 - ]). - -rest_api(#{name => list_banned, method => 'GET', path => "/banned/", @@ -50,7 +46,7 @@ ]). list(_Bindings, Params) -> - return({ok, emqx_mgmt_api:paginate(emqx_banned, Params, fun format/1)}). + minirest:return({ok, emqx_mgmt_api:paginate(emqx_banned, Params, fun format/1)}). create(_Bindings, Params) -> case pipeline([fun ensure_required/1, @@ -58,9 +54,9 @@ create(_Bindings, Params) -> {ok, NParams} -> {ok, Banned} = pack_banned(NParams), ok = emqx_mgmt:create_banned(Banned), - return({ok, maps:from_list(Params)}); + minirest:return({ok, maps:from_list(Params)}); {error, Code, Message} -> - return({error, Code, Message}) + minirest:return({error, Code, Message}) end. delete(#{as := As, who := Who}, _) -> @@ -70,9 +66,9 @@ delete(#{as := As, who := Who}, _) -> fun validate_params/1], Params) of {ok, NParams} -> do_delete(get_value(<<"as">>, NParams), get_value(<<"who">>, NParams)), - return(); + minirest:return(); {error, Code, Message} -> - return({error, Code, Message}) + minirest:return({error, Code, Message}) end. pipeline([], Params) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_brokers.erl b/apps/emqx_management/src/emqx_mgmt_api_brokers.erl index 2d6d82850..89707f49c 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_brokers.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_brokers.erl @@ -18,8 +18,6 @@ -include("emqx_mgmt.hrl"). --import(minirest, [return/1]). - -rest_api(#{name => list_brokers, method => 'GET', path => "/brokers/", @@ -37,13 +35,13 @@ ]). list(_Bindings, _Params) -> - return({ok, [Info || {_Node, Info} <- emqx_mgmt:list_brokers()]}). + minirest:return({ok, [Info || {_Node, Info} <- emqx_mgmt:list_brokers()]}). get(#{node := Node}, _Params) -> case emqx_mgmt:lookup_broker(Node) of {error, Reason} -> - return({error, ?ERROR2, Reason}); + minirest:return({error, ?ERROR2, Reason}); Info -> - return({ok, Info}) + minirest:return({ok, Info}) end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 7a4e4b0b7..8a2c55643 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -21,10 +21,6 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx.hrl"). --import(minirest, [ return/0 - , return/1 - ]). - -import(proplists, [get_value/2]). -define(CLIENT_QS_SCHEMA, {emqx_channel_info, @@ -146,87 +142,87 @@ -define(format_fun, {?MODULE, format_channel_info}). list(Bindings, Params) when map_size(Bindings) == 0 -> - return({ok, emqx_mgmt_api:cluster_query(Params, ?CLIENT_QS_SCHEMA, ?query_fun)}); + minirest:return({ok, emqx_mgmt_api:cluster_query(Params, ?CLIENT_QS_SCHEMA, ?query_fun)}); list(#{node := Node}, Params) when Node =:= node() -> - return({ok, emqx_mgmt_api:node_query(Node, Params, ?CLIENT_QS_SCHEMA, ?query_fun)}); + minirest:return({ok, emqx_mgmt_api:node_query(Node, Params, ?CLIENT_QS_SCHEMA, ?query_fun)}); list(Bindings = #{node := Node}, Params) -> case rpc:call(Node, ?MODULE, list, [Bindings, Params]) of - {badrpc, Reason} -> return({error, ?ERROR1, Reason}); + {badrpc, Reason} -> minirest:return({error, ?ERROR1, Reason}); Res -> Res end. lookup(#{node := Node, clientid := ClientId}, _Params) -> - return({ok, emqx_mgmt:lookup_client(Node, {clientid, emqx_mgmt_util:urldecode(ClientId)}, ?format_fun)}); + minirest:return({ok, emqx_mgmt:lookup_client(Node, {clientid, emqx_mgmt_util:urldecode(ClientId)}, ?format_fun)}); lookup(#{clientid := ClientId}, _Params) -> - return({ok, emqx_mgmt:lookup_client({clientid, emqx_mgmt_util:urldecode(ClientId)}, ?format_fun)}); + minirest:return({ok, emqx_mgmt:lookup_client({clientid, emqx_mgmt_util:urldecode(ClientId)}, ?format_fun)}); lookup(#{node := Node, username := Username}, _Params) -> - return({ok, emqx_mgmt:lookup_client(Node, {username, emqx_mgmt_util:urldecode(Username)}, ?format_fun)}); + minirest:return({ok, emqx_mgmt:lookup_client(Node, {username, emqx_mgmt_util:urldecode(Username)}, ?format_fun)}); lookup(#{username := Username}, _Params) -> - return({ok, emqx_mgmt:lookup_client({username, emqx_mgmt_util:urldecode(Username)}, ?format_fun)}). + minirest:return({ok, emqx_mgmt:lookup_client({username, emqx_mgmt_util:urldecode(Username)}, ?format_fun)}). kickout(#{clientid := ClientId}, _Params) -> case emqx_mgmt:kickout_client(emqx_mgmt_util:urldecode(ClientId)) of - ok -> return(); - {error, not_found} -> return({error, ?ERROR12, not_found}); - {error, Reason} -> return({error, ?ERROR1, Reason}) + ok -> minirest:return(); + {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); + {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) end. clean_acl_cache(#{clientid := ClientId}, _Params) -> case emqx_mgmt:clean_acl_cache(emqx_mgmt_util:urldecode(ClientId)) of - ok -> return(); - {error, not_found} -> return({error, ?ERROR12, not_found}); - {error, Reason} -> return({error, ?ERROR1, Reason}) + ok -> minirest:return(); + {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); + {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) end. list_acl_cache(#{clientid := ClientId}, _Params) -> case emqx_mgmt:list_acl_cache(emqx_mgmt_util:urldecode(ClientId)) of - {error, not_found} -> return({error, ?ERROR12, not_found}); - {error, Reason} -> return({error, ?ERROR1, Reason}); - Caches -> return({ok, [format_acl_cache(Cache) || Cache <- Caches]}) + {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); + {error, Reason} -> minirest:return({error, ?ERROR1, Reason}); + Caches -> minirest:return({ok, [format_acl_cache(Cache) || Cache <- Caches]}) end. set_ratelimit_policy(#{clientid := ClientId}, Params) -> P = [{conn_bytes_in, get_value(<<"conn_bytes_in">>, Params)}, {conn_messages_in, get_value(<<"conn_messages_in">>, Params)}], case [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined] of - [] -> return(); + [] -> minirest:return(); Policy -> case emqx_mgmt:set_ratelimit_policy(emqx_mgmt_util:urldecode(ClientId), Policy) of - ok -> return(); - {error, not_found} -> return({error, ?ERROR12, not_found}); - {error, Reason} -> return({error, ?ERROR1, Reason}) + ok -> minirest:return(); + {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); + {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) end end. clean_ratelimit(#{clientid := ClientId}, _Params) -> case emqx_mgmt:set_ratelimit_policy(emqx_mgmt_util:urldecode(ClientId), []) of - ok -> return(); - {error, not_found} -> return({error, ?ERROR12, not_found}); - {error, Reason} -> return({error, ?ERROR1, Reason}) + ok -> minirest:return(); + {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); + {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) end. set_quota_policy(#{clientid := ClientId}, Params) -> P = [{conn_messages_routing, get_value(<<"conn_messages_routing">>, Params)}], case [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined] of - [] -> return(); + [] -> minirest:return(); Policy -> case emqx_mgmt:set_quota_policy(emqx_mgmt_util:urldecode(ClientId), Policy) of - ok -> return(); - {error, not_found} -> return({error, ?ERROR12, not_found}); - {error, Reason} -> return({error, ?ERROR1, Reason}) + ok -> minirest:return(); + {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); + {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) end end. clean_quota(#{clientid := ClientId}, _Params) -> case emqx_mgmt:set_quota_policy(emqx_mgmt_util:urldecode(ClientId), []) of - ok -> return(); - {error, not_found} -> return({error, ?ERROR12, not_found}); - {error, Reason} -> return({error, ?ERROR1, Reason}) + ok -> minirest:return(); + {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); + {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) end. %% @private diff --git a/apps/emqx_management/src/emqx_mgmt_api_data.erl b/apps/emqx_management/src/emqx_mgmt_api_data.erl index 5c19e95af..da551782d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_data.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_data.erl @@ -22,10 +22,6 @@ -include("emqx_mgmt.hrl"). --import(minirest, [ return/0 - , return/1 - ]). - -rest_api(#{name => export, method => 'POST', path => "/data/export", @@ -77,14 +73,14 @@ export(_Bindings, _Params) -> case emqx_mgmt_data_backup:export() of {ok, File = #{filename := Filename}} -> - return({ok, File#{filename => filename:basename(Filename)}}); - Return -> return(Return) + minirest:return({ok, File#{filename => filename:basename(Filename)}}); + Return -> minirest:return(Return) end. list_exported(_Bindings, _Params) -> List = [ rpc:call(Node, ?MODULE, get_list_exported, []) || Node <- ekka_mnesia:running_nodes() ], NList = lists:map(fun({_, FileInfo}) -> FileInfo end, lists:keysort(1, lists:append(List))), - return({ok, NList}). + minirest:return({ok, NList}). get_list_exported() -> Dir = emqx:get_env(data_dir), @@ -114,7 +110,7 @@ get_list_exported() -> import(_Bindings, Params) -> case proplists:get_value(<<"filename">>, Params) of undefined -> - return({error, missing_required_params}); + minirest:return({error, missing_required_params}); Filename -> Result = case proplists:get_value(<<"node">>, Params) of undefined -> do_import(Filename); @@ -122,11 +118,11 @@ import(_Bindings, Params) -> case lists:member(Node, [ erlang:atom_to_binary(N, utf8) || N <- ekka_mnesia:running_nodes() ] ) of - true -> return(rpc:call(erlang:binary_to_atom(Node, utf8), ?MODULE, do_import, [Filename])); - false -> return({error, no_existent_node}) + true -> minirest:return(rpc:call(erlang:binary_to_atom(Node, utf8), ?MODULE, do_import, [Filename])); + false -> minirest:return({error, no_existent_node}) end end, - return(Result) + minirest:return(Result) end. do_import(Filename) -> @@ -140,7 +136,7 @@ download(#{filename := Filename}, _Params) -> {ok, #{filename => list_to_binary(Filename), file => Bin}}; {error, Reason} -> - return({error, Reason}) + minirest:return({error, Reason}) end. upload(Bindings, Params) -> @@ -151,9 +147,9 @@ do_upload(_Bindings, #{<<"filename">> := Filename, FullFilename = filename:join([emqx:get_env(data_dir), Filename]), case file:write_file(FullFilename, Bin) of ok -> - return({ok, [{node, node()}]}); + minirest:return({ok, [{node, node()}]}); {error, Reason} -> - return({error, Reason}) + minirest:return({error, Reason}) end; do_upload(Bindings, Params = #{<<"file">> := _}) -> Seconds = erlang:system_time(second), @@ -161,13 +157,13 @@ do_upload(Bindings, Params = #{<<"file">> := _}) -> Filename = io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]), do_upload(Bindings, Params#{<<"filename">> => Filename}); do_upload(_Bindings, _Params) -> - return({error, missing_required_params}). + minirest:return({error, missing_required_params}). delete(#{filename := Filename}, _Params) -> FullFilename = filename:join([emqx:get_env(data_dir), Filename]), case file:delete(FullFilename) of ok -> - return(); + minirest:return(); {error, Reason} -> - return({error, Reason}) + minirest:return({error, Reason}) end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index 382c1051b..5425d82b4 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -16,8 +16,6 @@ -module(emqx_mgmt_api_listeners). --import(minirest, [return/1]). - -rest_api(#{name => list_listeners, method => 'GET', path => "/listeners/", @@ -46,18 +44,18 @@ %% List listeners on a node. list(#{node := Node}, _Params) -> - return({ok, format(emqx_mgmt:list_listeners(Node))}); + minirest:return({ok, format(emqx_mgmt:list_listeners(Node))}); %% List listeners in the cluster. list(_Binding, _Params) -> - return({ok, [#{node => Node, listeners => format(Listeners)} + minirest:return({ok, [#{node => Node, listeners => format(Listeners)} || {Node, Listeners} <- emqx_mgmt:list_listeners()]}). %% Restart listeners on a node. restart(#{node := Node, identifier := Identifier}, _Params) -> case emqx_mgmt:restart_listener(Node, Identifier) of - ok -> return({ok, "Listener restarted."}); - {error, Error} -> return({error, Error}) + ok -> minirest:return({ok, "Listener restarted."}); + {error, Error} -> minirest:return({error, Error}) end; %% Restart listeners in the cluster. @@ -66,8 +64,8 @@ restart(#{identifier := <<"http", _/binary>>}, _Params) -> restart(#{identifier := Identifier}, _Params) -> Results = [{Node, emqx_mgmt:restart_listener(Node, Identifier)} || {Node, _Info} <- emqx_mgmt:list_nodes()], case lists:filter(fun({_, Result}) -> Result =/= ok end, Results) of - [] -> return(ok); - Errors -> return({error, {restart, Errors}}) + [] -> minirest:return(ok); + Errors -> minirest:return({error, {restart, Errors}}) end. format(Listeners) when is_list(Listeners) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl index 3e9c88db6..c265fd20e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl @@ -16,8 +16,6 @@ -module(emqx_mgmt_api_metrics). --import(minirest, [return/1]). - -rest_api(#{name => list_all_metrics, method => 'GET', path => "/metrics", @@ -33,12 +31,12 @@ -export([list/2]). list(Bindings, _Params) when map_size(Bindings) == 0 -> - return({ok, [#{node => Node, metrics => maps:from_list(Metrics)} + minirest:return({ok, [#{node => Node, metrics => maps:from_list(Metrics)} || {Node, Metrics} <- emqx_mgmt:get_metrics()]}); list(#{node := Node}, _Params) -> case emqx_mgmt:get_metrics(Node) of - {error, Reason} -> return({error, Reason}); - Metrics -> return({ok, maps:from_list(Metrics)}) + {error, Reason} -> minirest:return({error, Reason}); + Metrics -> minirest:return({ok, maps:from_list(Metrics)}) end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index c5791f3d5..89325193f 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -16,8 +16,6 @@ -module(emqx_mgmt_api_nodes). --import(minirest, [return/1]). - -rest_api(#{name => list_nodes, method => 'GET', path => "/nodes/", @@ -35,10 +33,10 @@ ]). list(_Bindings, _Params) -> - return({ok, [format(Node, Info) || {Node, Info} <- emqx_mgmt:list_nodes()]}). + minirest:return({ok, [format(Node, Info) || {Node, Info} <- emqx_mgmt:list_nodes()]}). get(#{node := Node}, _Params) -> - return({ok, emqx_mgmt:lookup_node(Node)}). + minirest:return({ok, emqx_mgmt:lookup_node(Node)}). format(Node, {error, Reason}) -> #{node => Node, error => Reason}; diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index 991dffad9..c50a806ea 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -20,8 +20,6 @@ -include_lib("emqx/include/emqx.hrl"). --import(minirest, [return/1]). - -rest_api(#{name => list_all_plugins, method => 'GET', path => "/plugins/", @@ -71,36 +69,36 @@ ]). list(#{node := Node}, _Params) -> - return({ok, [format(Plugin) || Plugin <- emqx_mgmt:list_plugins(Node)]}); + minirest:return({ok, [format(Plugin) || Plugin <- emqx_mgmt:list_plugins(Node)]}); list(_Bindings, _Params) -> - return({ok, [format({Node, Plugins}) || {Node, Plugins} <- emqx_mgmt:list_plugins()]}). + minirest:return({ok, [format({Node, Plugins}) || {Node, Plugins} <- emqx_mgmt:list_plugins()]}). load(#{node := Node, plugin := Plugin}, _Params) -> - return(emqx_mgmt:load_plugin(Node, Plugin)). + minirest:return(emqx_mgmt:load_plugin(Node, Plugin)). unload(#{node := Node, plugin := Plugin}, _Params) -> - return(emqx_mgmt:unload_plugin(Node, Plugin)); + minirest:return(emqx_mgmt:unload_plugin(Node, Plugin)); unload(#{plugin := Plugin}, _Params) -> Results = [emqx_mgmt:unload_plugin(Node, Plugin) || {Node, _Info} <- emqx_mgmt:list_nodes()], case lists:filter(fun(Item) -> Item =/= ok end, Results) of [] -> - return(ok); + minirest:return(ok); Errors -> - return(lists:last(Errors)) + minirest:return(lists:last(Errors)) end. reload(#{node := Node, plugin := Plugin}, _Params) -> - return(emqx_mgmt:reload_plugin(Node, Plugin)); + minirest:return(emqx_mgmt:reload_plugin(Node, Plugin)); reload(#{plugin := Plugin}, _Params) -> Results = [emqx_mgmt:reload_plugin(Node, Plugin) || {Node, _Info} <- emqx_mgmt:list_nodes()], case lists:filter(fun(Item) -> Item =/= ok end, Results) of [] -> - return(ok); + minirest:return(ok); Errors -> - return(lists:last(Errors)) + minirest:return(lists:last(Errors)) end. format({Node, Plugins}) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl index 3b7f7392f..e22bc78f0 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl @@ -24,8 +24,6 @@ , get_value/3 ]). --import(minirest, [return/1]). - -rest_api(#{name => mqtt_subscribe, method => 'POST', path => "/mqtt/subscribe", @@ -73,7 +71,7 @@ subscribe(_Bindings, Params) -> logger:debug("API subscribe Params:~p", [Params]), {ClientId, Topic, QoS} = parse_subscribe_params(Params), - return(do_subscribe(ClientId, Topic, QoS)). + minirest:return(do_subscribe(ClientId, Topic, QoS)). publish(_Bindings, Params) -> logger:debug("API publish Params:~p", [Params]), @@ -81,33 +79,33 @@ publish(_Bindings, Params) -> case do_publish(ClientId, Topic, Qos, Retain, Payload) of {ok, MsgIds} -> case get_value(<<"return">>, Params, undefined) of - undefined -> return(ok); + undefined -> minirest:return(ok); _Val -> case get_value(<<"topics">>, Params, undefined) of - undefined -> return({ok, #{msgid => lists:last(MsgIds)}}); - _ -> return({ok, #{msgids => MsgIds}}) + undefined -> minirest:return({ok, #{msgid => lists:last(MsgIds)}}); + _ -> minirest:return({ok, #{msgids => MsgIds}}) end end; Result -> - return(Result) + minirest:return(Result) end. unsubscribe(_Bindings, Params) -> logger:debug("API unsubscribe Params:~p", [Params]), {ClientId, Topic} = parse_unsubscribe_params(Params), - return(do_unsubscribe(ClientId, Topic)). + minirest:return(do_unsubscribe(ClientId, Topic)). subscribe_batch(_Bindings, Params) -> logger:debug("API subscribe batch Params:~p", [Params]), - return({ok, loop_subscribe(Params)}). + minirest:return({ok, loop_subscribe(Params)}). publish_batch(_Bindings, Params) -> logger:debug("API publish batch Params:~p", [Params]), - return({ok, loop_publish(Params)}). + minirest:return({ok, loop_publish(Params)}). unsubscribe_batch(_Bindings, Params) -> logger:debug("API unsubscribe batch Params:~p", [Params]), - return({ok, loop_unsubscribe(Params)}). + minirest:return({ok, loop_unsubscribe(Params)}). loop_subscribe(Params) -> loop_subscribe(Params, []). diff --git a/apps/emqx_management/src/emqx_mgmt_api_routes.erl b/apps/emqx_management/src/emqx_mgmt_api_routes.erl index 3a58a26a2..ed173436f 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_routes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_routes.erl @@ -18,8 +18,6 @@ -include_lib("emqx/include/emqx.hrl"). --import(minirest, [return/1]). - -rest_api(#{name => list_routes, method => 'GET', path => "/routes/", @@ -37,11 +35,11 @@ ]). list(Bindings, Params) when map_size(Bindings) == 0 -> - return({ok, emqx_mgmt_api:paginate(emqx_route, Params, fun format/1)}). + minirest:return({ok, emqx_mgmt_api:paginate(emqx_route, Params, fun format/1)}). lookup(#{topic := Topic}, _Params) -> Topic1 = emqx_mgmt_util:urldecode(Topic), - return({ok, [format(R) || R <- emqx_mgmt:lookup_routes(Topic1)]}). + minirest:return({ok, [format(R) || R <- emqx_mgmt:lookup_routes(Topic1)]}). format(#route{topic = Topic, dest = {_, Node}}) -> #{topic => Topic, node => Node}; format(#route{topic = Topic, dest = Node}) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_stats.erl b/apps/emqx_management/src/emqx_mgmt_api_stats.erl index 57e0b4fcf..97d80410b 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_stats.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_stats.erl @@ -16,8 +16,6 @@ -module(emqx_mgmt_api_stats). --import(minirest, [return/1]). - -rest_api(#{name => list_stats, method => 'GET', path => "/stats/", @@ -36,12 +34,12 @@ %% List stats of all nodes list(Bindings, _Params) when map_size(Bindings) == 0 -> - return({ok, [#{node => Node, stats => maps:from_list(Stats)} + minirest:return({ok, [#{node => Node, stats => maps:from_list(Stats)} || {Node, Stats} <- emqx_mgmt:get_stats()]}). %% List stats of a node lookup(#{node := Node}, _Params) -> case emqx_mgmt:get_stats(Node) of - {error, Reason} -> return({error, Reason}); - Stats -> return({ok, maps:from_list(Stats)}) + {error, Reason} -> minirest:return({error, Reason}); + Stats -> minirest:return({ok, maps:from_list(Stats)}) end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 51f7a6dec..f61ebfd97 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -18,8 +18,6 @@ -include_lib("emqx/include/emqx.hrl"). --import(minirest, [return/1]). - -define(SUBS_QS_SCHEMA, {emqx_suboption, [{<<"clientid">>, binary}, {<<"topic">>, binary}, @@ -65,9 +63,9 @@ list(Bindings, Params) when map_size(Bindings) == 0 -> case proplists:get_value(<<"topic">>, Params) of undefined -> - return({ok, emqx_mgmt_api:cluster_query(Params, ?SUBS_QS_SCHEMA, ?query_fun)}); + minirest:return({ok, emqx_mgmt_api:cluster_query(Params, ?SUBS_QS_SCHEMA, ?query_fun)}); Topic -> - return({ok, emqx_mgmt:list_subscriptions_via_topic(emqx_mgmt_util:urldecode(Topic), ?format_fun)}) + minirest:return({ok, emqx_mgmt:list_subscriptions_via_topic(emqx_mgmt_util:urldecode(Topic), ?format_fun)}) end; list(#{node := Node} = Bindings, Params) -> @@ -75,22 +73,22 @@ list(#{node := Node} = Bindings, Params) -> undefined -> case Node =:= node() of true -> - return({ok, emqx_mgmt_api:node_query(Node, Params, ?SUBS_QS_SCHEMA, ?query_fun)}); + minirest:return({ok, emqx_mgmt_api:node_query(Node, Params, ?SUBS_QS_SCHEMA, ?query_fun)}); false -> case rpc:call(Node, ?MODULE, list, [Bindings, Params]) of - {badrpc, Reason} -> return({error, Reason}); + {badrpc, Reason} -> minirest:return({error, Reason}); Res -> Res end end; Topic -> - return({ok, emqx_mgmt:list_subscriptions_via_topic(Node, emqx_mgmt_util:urldecode(Topic), ?format_fun)}) + minirest:return({ok, emqx_mgmt:list_subscriptions_via_topic(Node, emqx_mgmt_util:urldecode(Topic), ?format_fun)}) end. lookup(#{node := Node, clientid := ClientId}, _Params) -> - return({ok, format(emqx_mgmt:lookup_subscriptions(Node, emqx_mgmt_util:urldecode(ClientId)))}); + minirest:return({ok, format(emqx_mgmt:lookup_subscriptions(Node, emqx_mgmt_util:urldecode(ClientId)))}); lookup(#{clientid := ClientId}, _Params) -> - return({ok, format(emqx_mgmt:lookup_subscriptions(emqx_mgmt_util:urldecode(ClientId)))}). + minirest:return({ok, format(emqx_mgmt:lookup_subscriptions(emqx_mgmt_util:urldecode(ClientId)))}). format(Items) when is_list(Items) -> [format(Item) || Item <- Items]; From 796c071af5340b3f0a8dfd189fbc96bf8d0f34b0 Mon Sep 17 00:00:00 2001 From: Karol Kaczmarek Date: Thu, 18 Mar 2021 21:11:15 +0100 Subject: [PATCH 043/158] style(api): proplists imports removed --- apps/emqx_management/src/emqx_mgmt.erl | 8 ++--- .../src/emqx_mgmt_api_apps.erl | 22 ++++++------ .../src/emqx_mgmt_api_banned.erl | 6 ++-- .../src/emqx_mgmt_api_clients.erl | 8 ++--- .../src/emqx_mgmt_api_pubsub.erl | 36 +++++++++---------- apps/emqx_management/src/emqx_mgmt_http.erl | 6 ++-- 6 files changed, 36 insertions(+), 50 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 3ab1b80ee..fb4ca47ed 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -22,8 +22,6 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). --import(proplists, [get_value/2]). - %% Nodes and Brokers API -export([ list_nodes/0 , lookup_node/1 @@ -135,11 +133,11 @@ node_info(Node) when Node =:= node() -> BrokerInfo = emqx_sys:info(), Info#{node => node(), otp_release => iolist_to_binary(otp_rel()), - memory_total => get_value(allocated, Memory), - memory_used => get_value(used, Memory), + memory_total => proplists:get_value(allocated, Memory), + memory_used => proplists:get_value(used, Memory), process_available => erlang:system_info(process_limit), process_used => erlang:system_info(process_count), - max_fds => get_value(max_fds, lists:usort(lists:flatten(erlang:system_info(check_io)))), + max_fds => proplists:get_value(max_fds, lists:usort(lists:flatten(erlang:system_info(check_io)))), connections => ets:info(emqx_channel, size), node_status => 'Running', uptime => iolist_to_binary(proplists:get_value(uptime, BrokerInfo)), diff --git a/apps/emqx_management/src/emqx_mgmt_api_apps.erl b/apps/emqx_management/src/emqx_mgmt_api_apps.erl index 7d5031698..e9a4e0997 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_apps.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_apps.erl @@ -18,8 +18,6 @@ -include("emqx_mgmt.hrl"). --import(proplists, [get_value/2]). - -rest_api(#{name => add_app, method => 'POST', path => "/apps/", @@ -58,12 +56,12 @@ ]). add_app(_Bindings, Params) -> - AppId = get_value(<<"app_id">>, Params), - Name = get_value(<<"name">>, Params), - Secret = get_value(<<"secret">>, Params), - Desc = get_value(<<"desc">>, Params), - Status = get_value(<<"status">>, Params), - Expired = get_value(<<"expired">>, Params), + AppId = proplists:get_value(<<"app_id">>, Params), + Name = proplists:get_value(<<"name">>, Params), + Secret = proplists:get_value(<<"secret">>, Params), + Desc = proplists:get_value(<<"desc">>, Params), + Status = proplists:get_value(<<"status">>, Params), + Expired = proplists:get_value(<<"expired">>, Params), case emqx_mgmt_auth:add_app(AppId, Name, Secret, Desc, Status, Expired) of {ok, AppSecret} -> minirest:return({ok, #{secret => AppSecret}}); {error, Reason} -> minirest:return({error, Reason}) @@ -92,10 +90,10 @@ lookup_app(#{appid := AppId}, _Params) -> end. update_app(#{appid := AppId}, Params) -> - Name = get_value(<<"name">>, Params), - Desc = get_value(<<"desc">>, Params), - Status = get_value(<<"status">>, Params), - Expired = get_value(<<"expired">>, Params), + Name = proplists:get_value(<<"name">>, Params), + Desc = proplists:get_value(<<"desc">>, Params), + Status = proplists:get_value(<<"status">>, Params), + Expired = proplists:get_value(<<"expired">>, Params), case emqx_mgmt_auth:update_app(AppId, Name, Desc, Status, Expired) of ok -> minirest:return(); {error, Reason} -> minirest:return({error, Reason}) diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index 2d5c0da69..4d5856fd0 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -20,8 +20,6 @@ -include("emqx_mgmt.hrl"). --import(proplists, [get_value/2]). - -rest_api(#{name => list_banned, method => 'GET', path => "/banned/", @@ -65,7 +63,7 @@ delete(#{as := As, who := Who}, _) -> case pipeline([fun ensure_required/1, fun validate_params/1], Params) of {ok, NParams} -> - do_delete(get_value(<<"as">>, NParams), get_value(<<"who">>, NParams)), + do_delete(proplists:get_value(<<"as">>, NParams), proplists:get_value(<<"who">>, NParams)), minirest:return(); {error, Code, Message} -> minirest:return({error, Code, Message}) @@ -95,7 +93,7 @@ ensure_required(Params) when is_list(Params) -> validate_params(Params) -> #{enum_values := AsEnums, message := Msg} = enum_values(as), - case lists:member(get_value(<<"as">>, Params), AsEnums) of + case lists:member(proplists:get_value(<<"as">>, Params), AsEnums) of true -> {ok, Params}; false -> {error, ?ERROR8, Msg} diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 8a2c55643..23a70f6eb 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -21,8 +21,6 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx.hrl"). --import(proplists, [get_value/2]). - -define(CLIENT_QS_SCHEMA, {emqx_channel_info, [{<<"clientid">>, binary}, {<<"username">>, binary}, @@ -187,8 +185,8 @@ list_acl_cache(#{clientid := ClientId}, _Params) -> end. set_ratelimit_policy(#{clientid := ClientId}, Params) -> - P = [{conn_bytes_in, get_value(<<"conn_bytes_in">>, Params)}, - {conn_messages_in, get_value(<<"conn_messages_in">>, Params)}], + P = [{conn_bytes_in, proplists:get_value(<<"conn_bytes_in">>, Params)}, + {conn_messages_in, proplists:get_value(<<"conn_messages_in">>, Params)}], case [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined] of [] -> minirest:return(); Policy -> @@ -207,7 +205,7 @@ clean_ratelimit(#{clientid := ClientId}, _Params) -> end. set_quota_policy(#{clientid := ClientId}, Params) -> - P = [{conn_messages_routing, get_value(<<"conn_messages_routing">>, Params)}], + P = [{conn_messages_routing, proplists:get_value(<<"conn_messages_routing">>, Params)}], case [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined] of [] -> minirest:return(); Policy -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl index e22bc78f0..693f71a3e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl @@ -20,10 +20,6 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include("emqx_mgmt.hrl"). --import(proplists, [ get_value/2 - , get_value/3 - ]). - -rest_api(#{name => mqtt_subscribe, method => 'POST', path => "/mqtt/subscribe", @@ -78,10 +74,10 @@ publish(_Bindings, Params) -> {ClientId, Topic, Qos, Retain, Payload} = parse_publish_params(Params), case do_publish(ClientId, Topic, Qos, Retain, Payload) of {ok, MsgIds} -> - case get_value(<<"return">>, Params, undefined) of + case proplists:get_value(<<"return">>, Params, undefined) of undefined -> minirest:return(ok); _Val -> - case get_value(<<"topics">>, Params, undefined) of + case proplists:get_value(<<"topics">>, Params, undefined) of undefined -> minirest:return({ok, #{msgid => lists:last(MsgIds)}}); _ -> minirest:return({ok, #{msgids => MsgIds}}) end @@ -118,7 +114,7 @@ loop_subscribe([Params | ParamsN], Acc) -> {_, Code0, _Reason} -> Code0 end, Result = #{clientid => ClientId, - topic => resp_topic(get_value(<<"topic">>, Params), get_value(<<"topics">>, Params, <<"">>)), + topic => resp_topic(proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), code => Code}, loop_subscribe(ParamsN, [Result | Acc]). @@ -132,7 +128,7 @@ loop_publish([Params | ParamsN], Acc) -> {ok, _} -> 0; {_, Code0, _} -> Code0 end, - Result = #{topic => resp_topic(get_value(<<"topic">>, Params), get_value(<<"topics">>, Params, <<"">>)), + Result = #{topic => resp_topic(proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), code => Code}, loop_publish(ParamsN, [Result | Acc]). @@ -147,7 +143,7 @@ loop_unsubscribe([Params | ParamsN], Acc) -> {_, Code0, _} -> Code0 end, Result = #{clientid => ClientId, - topic => resp_topic(get_value(<<"topic">>, Params), get_value(<<"topics">>, Params, <<"">>)), + topic => resp_topic(proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), code => Code}, loop_unsubscribe(ParamsN, [Result | Acc]). @@ -182,24 +178,24 @@ do_unsubscribe(ClientId, Topic) -> end. parse_subscribe_params(Params) -> - ClientId = get_value(<<"clientid">>, Params), - Topics = topics(filter, get_value(<<"topic">>, Params), get_value(<<"topics">>, Params, <<"">>)), - QoS = get_value(<<"qos">>, Params, 0), + ClientId = proplists:get_value(<<"clientid">>, Params), + Topics = topics(filter, proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), + QoS = proplists:get_value(<<"qos">>, Params, 0), {ClientId, Topics, QoS}. parse_publish_params(Params) -> - Topics = topics(name, get_value(<<"topic">>, Params), get_value(<<"topics">>, Params, <<"">>)), - ClientId = get_value(<<"clientid">>, Params), - Payload = decode_payload(get_value(<<"payload">>, Params, <<>>), - get_value(<<"encoding">>, Params, <<"plain">>)), - Qos = get_value(<<"qos">>, Params, 0), - Retain = get_value(<<"retain">>, Params, false), + Topics = topics(name, proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), + ClientId = proplists:get_value(<<"clientid">>, Params), + Payload = decode_payload(proplists:get_value(<<"payload">>, Params, <<>>), + proplists:get_value(<<"encoding">>, Params, <<"plain">>)), + Qos = proplists:get_value(<<"qos">>, Params, 0), + Retain = proplists:get_value(<<"retain">>, Params, false), Payload1 = maybe_maps_to_binary(Payload), {ClientId, Topics, Qos, Retain, Payload1}. parse_unsubscribe_params(Params) -> - ClientId = get_value(<<"clientid">>, Params), - Topic = get_value(<<"topic">>, Params), + ClientId = proplists:get_value(<<"clientid">>, Params), + Topic = proplists:get_value(<<"topic">>, Params), {ClientId, Topic}. topics(Type, undefined, Topics0) -> diff --git a/apps/emqx_management/src/emqx_mgmt_http.erl b/apps/emqx_management/src/emqx_mgmt_http.erl index aee057d39..82b45b368 100644 --- a/apps/emqx_management/src/emqx_mgmt_http.erl +++ b/apps/emqx_management/src/emqx_mgmt_http.erl @@ -16,8 +16,6 @@ -module(emqx_mgmt_http). --import(proplists, [get_value/3]). - -export([ start_listeners/0 , handle_request/2 , stop_listeners/0 @@ -58,8 +56,8 @@ start_listener({Proto, Port, Options}) when Proto == https -> minirest:start_https(listener_name(Proto), ranch_opts(Port, Options), Dispatch). ranch_opts(Port, Options0) -> - NumAcceptors = get_value(num_acceptors, Options0, 4), - MaxConnections = get_value(max_connections, Options0, 512), + NumAcceptors = proplists:get_value(num_acceptors, Options0, 4), + MaxConnections = proplists:get_value(max_connections, Options0, 512), Options = lists:foldl(fun({K, _V}, Acc) when K =:= max_connections orelse K =:= num_acceptors -> Acc; ({inet6, true}, Acc) -> [inet6 | Acc]; From fa8b2a5ac5492a3110c81a3c2e5fff838d58cf41 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Wed, 17 Mar 2021 15:32:05 +0800 Subject: [PATCH 044/158] chore(test-cswsh): add test case for cswsh --- test/emqx_ws_connection_SUITE.erl | 45 ++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/test/emqx_ws_connection_SUITE.erl b/test/emqx_ws_connection_SUITE.erl index 71c6e40e5..d83d7d859 100644 --- a/test/emqx_ws_connection_SUITE.erl +++ b/test/emqx_ws_connection_SUITE.erl @@ -43,7 +43,9 @@ all() -> emqx_ct:all(?MODULE). init_per_testcase(TestCase, Config) when TestCase =/= t_ws_sub_protocols_mqtt_equivalents, - TestCase =/= t_ws_sub_protocols_mqtt -> + TestCase =/= t_ws_sub_protocols_mqtt, + TestCase =/= t_ws_check_origin, + TestCase =/= t_ws_non_check_origin -> %% Mock cowboy_req ok = meck:new(cowboy_req, [passthrough, no_history, no_link]), ok = meck:expect(cowboy_req, peer, fun(_) -> {{127,0,0,1}, 3456} end), @@ -85,7 +87,9 @@ init_per_testcase(_, Config) -> end_per_testcase(TestCase, Config) when TestCase =/= t_ws_sub_protocols_mqtt_equivalents, - TestCase =/= t_ws_sub_protocols_mqtt -> + TestCase =/= t_ws_sub_protocols_mqtt, + TestCase =/= t_ws_check_origin, + TestCase =/= t_ws_non_check_origin -> lists:foreach(fun meck:unload/1, [cowboy_req, emqx_zone, @@ -171,6 +175,41 @@ t_ws_sub_protocols_mqtt_equivalents(_) -> start_ws_client(#{protocols => [<<"not-mqtt">>]})), emqx_ct_helpers:stop_apps([]). +t_ws_check_origin(_) -> + emqx_ct_helpers:start_apps([], + fun(emqx) -> + {ok, Listeners} = application:get_env(emqx, listeners), + NListeners = lists:map(fun(#{listen_on := 8083, opts := Opts} = Listener) -> + NOpts = proplists:delete(check_origin_enable, Opts), + Listener#{opts => [{check_origin_enable, true} | NOpts]}; + (Listener) -> + Listener + end, Listeners), + application:set_env(emqx, listeners, NListeners), + ok; + (_) -> ok + end), + {ok, _} = application:ensure_all_started(gun), + ?assertMatch({gun_upgrade, _}, + start_ws_client(#{protocols => [<<"mqtt">>], + headers => [{<<"origin">>, <<"http://localhost:18083">>}]})), + ?assertMatch({gun_response, {_, 500, _}}, + start_ws_client(#{protocols => [<<"mqtt">>], + headers => [{<<"origin">>, <<"http://localhost:18080">>}]})), + emqx_ct_helpers:stop_apps([]). + +t_ws_non_check_origin(_) -> + emqx_ct_helpers:start_apps([]), + {ok, _} = application:ensure_all_started(gun), + ?assertMatch({gun_upgrade, _}, + start_ws_client(#{protocols => [<<"mqtt">>], + headers => [{<<"origin">>, <<"http://localhost:18083">>}]})), + ?assertMatch({gun_upgrade, _}, + start_ws_client(#{protocols => [<<"mqtt">>], + headers => [{<<"origin">>, <<"http://localhost:18080">>}]})), + emqx_ct_helpers:stop_apps([]). + + t_init(_) -> Opts = [{idle_timeout, 300000}, {fail_if_no_subprotocol, false}, @@ -433,7 +472,7 @@ ws_client(State) -> receive {gun_up, WPID, _Proto} -> #{protocols := Protos} = State, - StreamRef = gun:ws_upgrade(WPID, "/mqtt", [], + StreamRef = gun:ws_upgrade(WPID, "/mqtt", maps:get(headers, State, []), #{protocols => [{P, gun_ws_h} || P <- Protos]}), ws_client(State#{wref => StreamRef}); {gun_down, _WPID, _, Reason, _, _} -> From 81602c973c3dd9fd7d17d8353f83913ac3f79330 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 19 Mar 2021 13:17:47 +0800 Subject: [PATCH 045/158] fix(emqx): deny pingreq when mqtt not connected #4370 --- src/emqx_connection.erl | 3 --- src/emqx_ws_connection.erl | 3 --- test/emqx_connection_SUITE.erl | 21 ++++++++++------ test/emqx_ws_connection_SUITE.erl | 42 +++++++++++++++++++++++++++++-- 4 files changed, 54 insertions(+), 15 deletions(-) diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 11a3a9418..88a1ba4d7 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -359,9 +359,6 @@ handle_msg({incoming, Packet = ?CONNECT_PACKET(ConnPkt)}, }, handle_incoming(Packet, NState); -handle_msg({incoming, ?PACKET(?PINGREQ)}, State) -> - handle_outgoing(?PACKET(?PINGRESP), State); - handle_msg({incoming, Packet}, State) -> handle_incoming(Packet, State); diff --git a/src/emqx_ws_connection.erl b/src/emqx_ws_connection.erl index 98cc2b52f..01f7b5e2b 100644 --- a/src/emqx_ws_connection.erl +++ b/src/emqx_ws_connection.erl @@ -348,9 +348,6 @@ websocket_info({incoming, Packet = ?CONNECT_PACKET(ConnPkt)}, State) -> NState = State#state{serialize = Serialize}, handle_incoming(Packet, cancel_idle_timer(NState)); -websocket_info({incoming, ?PACKET(?PINGREQ)}, State) -> - return(enqueue(?PACKET(?PINGRESP), State)); - websocket_info({incoming, Packet}, State) -> handle_incoming(Packet, State); diff --git a/test/emqx_connection_SUITE.erl b/test/emqx_connection_SUITE.erl index b011c6978..22abd2ca8 100644 --- a/test/emqx_connection_SUITE.erl +++ b/test/emqx_connection_SUITE.erl @@ -60,7 +60,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> ok = meck:unload(emqx_transport), - ok = meck:unload(emqx_channel), + catch meck:unload(emqx_channel), ok = meck:unload(emqx_cm), ok = meck:unload(emqx_limiter), ok = meck:unload(emqx_pd), @@ -69,7 +69,8 @@ end_per_suite(_Config) -> ok = meck:unload(emqx_alarm), ok. -init_per_testcase(_TestCase, Config) -> +init_per_testcase(TestCase, Config) when + TestCase =/= t_ws_pingreq_before_connected -> ok = meck:expect(emqx_transport, wait, fun(Sock) -> {ok, Sock} end), ok = meck:expect(emqx_transport, type, fun(_Sock) -> tcp end), ok = meck:expect(emqx_transport, ensure_ok_or_exit, @@ -87,6 +88,8 @@ init_per_testcase(_TestCase, Config) -> ok = meck:expect(emqx_transport, async_send, fun(_Sock, _Data) -> ok end), ok = meck:expect(emqx_transport, async_send, fun(_Sock, _Data, _Opts) -> ok end), ok = meck:expect(emqx_transport, fast_close, fun(_Sock) -> ok end), + Config; +init_per_testcase(_, Config) -> Config. end_per_testcase(_TestCase, Config) -> @@ -95,6 +98,9 @@ end_per_testcase(_TestCase, Config) -> %%-------------------------------------------------------------------- %% Test cases %%-------------------------------------------------------------------- +t_ws_pingreq_before_connected(_) -> + ?assertMatch({ok, [_, {close,protocol_error}], _}, + handle_msg({incoming, ?PACKET(?PINGREQ)}, st(#{}, #{conn_state => disconnected}))). t_info(_) -> CPid = spawn(fun() -> @@ -175,7 +181,6 @@ t_handle_msg(_) -> t_handle_msg_incoming(_) -> ?assertMatch({ok, _Out, _St}, handle_msg({incoming, ?CONNECT_PACKET(#mqtt_packet_connect{})}, st())), - ?assertEqual(ok, handle_msg({incoming, ?PACKET(?PINGREQ)}, st())), ok = meck:expect(emqx_channel, handle_in, fun(_Packet, Channel) -> {ok, Channel} end), ?assertMatch({ok, _St}, handle_msg({incoming, ?PUBLISH_PACKET(?QOS_1, <<"t">>, 1, <<"payload">>)}, st())), @@ -277,7 +282,6 @@ t_handle_incoming(_) -> t_with_channel(_) -> State = st(), - ok = meck:expect(emqx_channel, handle_in, fun(_, _) -> ok end), ?assertEqual({ok, State}, emqx_connection:with_channel(handle_in, [for_testing], State)), @@ -300,7 +304,8 @@ t_with_channel(_) -> {shutdown, [for_testing], ?DISCONNECT_PACKET(), Channel} end), ?assertMatch({stop, {shutdown,[for_testing]}, _NState}, - emqx_connection:with_channel(handle_in, [for_testing], State)). + emqx_connection:with_channel(handle_in, [for_testing], State)), + meck:unload(emqx_channel). t_handle_outgoing(_) -> ?assertEqual(ok, emqx_connection:handle_outgoing(?PACKET(?PINGRESP), st())), @@ -432,11 +437,13 @@ make_frame(Packet) -> payload(Len) -> iolist_to_binary(lists:duplicate(Len, 1)). -st() -> st(#{}). +st() -> st(#{}, #{}). st(InitFields) when is_map(InitFields) -> + st(InitFields, #{}). +st(InitFields, ChannelFields) when is_map(InitFields) -> St = emqx_connection:init_state(emqx_transport, sock, [#{zone => external}]), maps:fold(fun(N, V, S) -> emqx_connection:set_field(N, V, S) end, - emqx_connection:set_field(channel, channel(), St), + emqx_connection:set_field(channel, channel(ChannelFields), St), InitFields ). diff --git a/test/emqx_ws_connection_SUITE.erl b/test/emqx_ws_connection_SUITE.erl index d83d7d859..4cf803024 100644 --- a/test/emqx_ws_connection_SUITE.erl +++ b/test/emqx_ws_connection_SUITE.erl @@ -45,7 +45,9 @@ init_per_testcase(TestCase, Config) when TestCase =/= t_ws_sub_protocols_mqtt_equivalents, TestCase =/= t_ws_sub_protocols_mqtt, TestCase =/= t_ws_check_origin, - TestCase =/= t_ws_non_check_origin -> + TestCase =/= t_ws_pingreq_before_connected, + TestCase =/= t_ws_non_check_origin + -> %% Mock cowboy_req ok = meck:new(cowboy_req, [passthrough, no_history, no_link]), ok = meck:expect(cowboy_req, peer, fun(_) -> {{127,0,0,1}, 3456} end), @@ -89,7 +91,9 @@ end_per_testcase(TestCase, Config) when TestCase =/= t_ws_sub_protocols_mqtt_equivalents, TestCase =/= t_ws_sub_protocols_mqtt, TestCase =/= t_ws_check_origin, - TestCase =/= t_ws_non_check_origin -> + TestCase =/= t_ws_non_check_origin, + TestCase =/= t_ws_pingreq_before_connected + -> lists:foreach(fun meck:unload/1, [cowboy_req, emqx_zone, @@ -154,6 +158,40 @@ t_call(_) -> end), ?assertEqual(Info, ?ws_conn:call(WsPid, info)). +t_ws_pingreq_before_connected(_) -> + ok = emqx_ct_helpers:start_apps([]), + {ok, _} = application:ensure_all_started(gun), + {ok, WPID} = gun:open("127.0.0.1", 8083), + ws_pingreq(#{}), + gun:close(WPID), + emqx_ct_helpers:stop_apps([]). + +ws_pingreq(State) -> + receive + {gun_up, WPID, _Proto} -> + StreamRef = gun:ws_upgrade(WPID, "/mqtt", [], #{ + protocols => [{<<"mqtt">>, gun_ws_h}]}), + ws_pingreq(State#{wref => StreamRef}); + {gun_down, _WPID, _, Reason, _, _} -> + State#{result => {gun_down, Reason}}; + {gun_upgrade, WPID, _Ref, _Proto, _Data} -> + ct:pal("-- gun_upgrade, send ping-req"), + PingReq = {binary, <<192,0>>}, + ok = gun:ws_send(WPID, PingReq), + gun:flush(WPID), + ws_pingreq(State); + {gun_ws, _WPID, _Ref, {binary, <<208,0>>}} -> + ct:fail(unexpected_pingresp); + {gun_ws, _WPID, _Ref, Frame} -> + ct:pal("gun received frame: ~p", [Frame]), + ws_pingreq(State); + Message -> + ct:pal("Received Unknown Message on Gun: ~p~n",[Message]), + ws_pingreq(State) + after 1000 -> + ct:fail(ws_timeout) + end. + t_ws_sub_protocols_mqtt(_) -> ok = emqx_ct_helpers:start_apps([]), {ok, _} = application:ensure_all_started(gun), From fc0dea75e441a9a7af7d75c3306e39bc60e6ee61 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 19 Mar 2021 14:06:36 +0800 Subject: [PATCH 046/158] feat(helm): allow custom securityContext --- deploy/charts/emqx/templates/StatefulSet.yaml | 8 ++++++-- deploy/charts/emqx/values.yaml | 11 +++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index db6c28815..3a385732a 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -83,8 +83,9 @@ spec: secretName: {{ .Values.emqxLicneseSecretName }} {{- end }} serviceAccountName: {{ include "emqx.fullname" . }} - securityContext: - fsGroup: 1000 + {{- if .Values.podSecurityContext.enabled }} + securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }} + {{- end }} {{- if .Values.initContainers }} initContainers: {{ toYaml .Values.initContainers | indent 8 }} @@ -99,6 +100,9 @@ spec: - name: emqx image: "{{ .Values.image.repository }}:{{ .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} ports: - name: mqtt containerPort: {{ .Values.emqxConfig.EMQX_LISTENER__TCP__EXTERNAL | default 1883 }} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 45b966c3b..34302e09a 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -176,4 +176,15 @@ ingress: - api.emqx.local tls: [] +podSecurityContext: + enabled: true + fsGroup: 1000 + fsGroupChangePolicy: Always + runAsUser: 1000 + supplementalGroups: + - 1000 +containerSecurityContext: + enabled: true + runAsNonRoot: true + runAsUser: 1000 From 568f4de6e567fbd406b3a330afa9926d2c1f274f Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Mon, 15 Mar 2021 19:49:59 +0800 Subject: [PATCH 047/158] fix(channel): fix the wrong acl check order when subscribe, check the acl rule before run 'client.subscribe' hooks --- src/emqx_channel.erl | 76 +++++++++++++++++++++---------------- test/emqx_channel_SUITE.erl | 9 +++-- 2 files changed, 49 insertions(+), 36 deletions(-) diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 4d66d5cdc..97a1fce23 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -391,22 +391,30 @@ handle_in(?PUBCOMP_PACKET(PacketId, _ReasonCode), Channel = #channel{session = S handle_in(Packet = ?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters), Channel = #channel{clientinfo = ClientInfo = #{zone := Zone}}) -> case emqx_packet:check(Packet) of - ok -> TopicFilters1 = parse_topic_filters(TopicFilters), - TopicFilters2 = put_subid_in_subopts(Properties, TopicFilters1), - TopicFilters3 = run_hooks('client.subscribe', - [ClientInfo, Properties], - TopicFilters2 - ), - {ReasonCodes, NChannel} = process_subscribe(TopicFilters3, Properties, Channel), - case emqx_zone:get_env(Zone, acl_deny_action, ignore) =:= disconnect andalso - lists:any(fun(ReasonCode) -> - ReasonCode =:= ?RC_NOT_AUTHORIZED - end, ReasonCodes) of - true -> - handle_out(disconnect, ?RC_NOT_AUTHORIZED, NChannel); - false -> - handle_out(suback, {PacketId, ReasonCodes}, NChannel) - end; + ok -> + TopicFilters0 = parse_topic_filters(TopicFilters), + TupleTopicFilters0 = check_sub_acls(TopicFilters0, Channel), + case emqx_zone:get_env(Zone, acl_deny_action, ignore) =:= disconnect andalso + lists:any(fun({_TopicFilter, ReasonCode}) -> + ReasonCode =:= ?RC_NOT_AUTHORIZED + end, TupleTopicFilters0) of + true -> handle_out(disconnect, ?RC_NOT_AUTHORIZED, Channel); + false -> + Replace = fun + _Fun(TupleList, [ Tuple = {Key, _Value} | More]) -> + _Fun(lists:keyreplace(Key, 1, TupleList, Tuple), More); + _Fun(TupleList, []) -> TupleList + end, + TopicFilters1 = [ TopicFilter || {TopicFilter, 0} <- TupleTopicFilters0], + TopicFilters2 = put_subid_in_subopts(Properties, TopicFilters1), + TopicFilters3 = run_hooks('client.subscribe', + [ClientInfo, Properties], + TopicFilters2), + {TupleTopicFilters1, NChannel} = process_subscribe(TopicFilters3, Properties, Channel), + TupleTopicFilters2 = Replace(TupleTopicFilters0, TupleTopicFilters1), + ReasonCodes2 = [ ReasonCode || {_TopicFilter, ReasonCode} <- TupleTopicFilters2], + handle_out(suback, {PacketId, ReasonCodes2}, NChannel) + end; {error, ReasonCode} -> handle_out(disconnect, ReasonCode, Channel) end; @@ -613,15 +621,15 @@ process_subscribe(TopicFilters, SubProps, Channel) -> process_subscribe([], _SubProps, Channel, Acc) -> {lists:reverse(Acc), Channel}; -process_subscribe([{TopicFilter, SubOpts}|More], SubProps, Channel, Acc) -> - case check_subscribe(TopicFilter, SubOpts, Channel) of +process_subscribe([Topic = {TopicFilter, SubOpts}|More], SubProps, Channel, Acc) -> + case check_sub_caps(TopicFilter, SubOpts, Channel) of ok -> - {RC, NChannel} = do_subscribe(TopicFilter, SubOpts#{sub_props => SubProps}, Channel), - process_subscribe(More, SubProps, NChannel, [RC|Acc]); - {error, RC} -> + {ReasonCode, NChannel} = do_subscribe(TopicFilter, SubOpts#{sub_props => SubProps}, Channel), + process_subscribe(More, SubProps, NChannel, [{Topic, ReasonCode} | Acc]); + {error, ReasonCode} -> ?LOG(warning, "Cannot subscribe ~s due to ~s.", - [TopicFilter, emqx_reason_codes:text(RC)]), - process_subscribe(More, SubProps, Channel, [RC|Acc]) + [TopicFilter, emqx_reason_codes:text(ReasonCode)]), + process_subscribe(More, SubProps, Channel, [{Topic, ReasonCode} | Acc]) end. do_subscribe(TopicFilter, SubOpts = #{qos := QoS}, Channel = @@ -1389,18 +1397,22 @@ check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, #channel{clientinfo = #{zone := Zone}}) -> emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic => Topic}). -%%-------------------------------------------------------------------- -%% Check Subscribe - -check_subscribe(TopicFilter, SubOpts, Channel) -> - case check_sub_acl(TopicFilter, Channel) of - allow -> check_sub_caps(TopicFilter, SubOpts, Channel); - deny -> {error, ?RC_NOT_AUTHORIZED} - end. - %%-------------------------------------------------------------------- %% Check Sub ACL +check_sub_acls(TopicFilters, Channel) -> + check_sub_acls(TopicFilters, Channel, []). + +check_sub_acls([ TopicFilter = {Topic, _} | More] , Channel, Acc) -> + case check_sub_acl(Topic, Channel) of + allow -> + check_sub_acls(More, Channel, [ {TopicFilter, 0} | Acc]); + deny -> + check_sub_acls(More, Channel, [ {TopicFilter, ?RC_NOT_AUTHORIZED} | Acc]) + end; +check_sub_acls([], _Channel, Acc) -> + lists:reverse(Acc). + check_sub_acl(TopicFilter, #channel{clientinfo = ClientInfo}) -> case is_acl_enabled(ClientInfo) andalso emqx_access_control:check_acl(ClientInfo, subscribe, TopicFilter) of diff --git a/test/emqx_channel_SUITE.erl b/test/emqx_channel_SUITE.erl index 15f3c6674..f3ee4aade 100644 --- a/test/emqx_channel_SUITE.erl +++ b/test/emqx_channel_SUITE.erl @@ -347,8 +347,8 @@ t_process_publish_qos1(_) -> t_process_subscribe(_) -> ok = meck:expect(emqx_session, subscribe, fun(_, _, _, Session) -> {ok, Session} end), - TopicFilters = [{<<"+">>, ?DEFAULT_SUBOPTS}], - {[?RC_SUCCESS], _Channel} = emqx_channel:process_subscribe(TopicFilters, #{}, channel()). + TopicFilters = [ TopicFilter = {<<"+">>, ?DEFAULT_SUBOPTS}], + {[{TopicFilter, ?RC_SUCCESS}], _Channel} = emqx_channel:process_subscribe(TopicFilters, #{}, channel()). t_process_unsubscribe(_) -> ok = meck:expect(emqx_session, unsubscribe, fun(_, _, _, Session) -> {ok, Session} end), @@ -650,10 +650,11 @@ t_check_pub_alias(_) -> Channel = emqx_channel:set_field(alias_maximum, #{inbound => 10}, channel()), ok = emqx_channel:check_pub_alias(#mqtt_packet{variable = Publish}, Channel). -t_check_subscribe(_) -> +t_check_sub_acls(_) -> ok = meck:new(emqx_zone, [passthrough, no_history]), ok = meck:expect(emqx_zone, enable_acl, fun(_) -> true end), - ok = emqx_channel:check_subscribe(<<"t">>, ?DEFAULT_SUBOPTS, channel()), + TopicFilter = {<<"t">>, ?DEFAULT_SUBOPTS}, + [{TopicFilter, 0}] = emqx_channel:check_sub_acls([TopicFilter], channel()), ok = meck:unload(emqx_zone). t_enrich_connack_caps(_) -> From 72c1ee264c1a2f6351c266eae5a0202e1ad1ee76 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 17 Mar 2021 15:49:01 +0800 Subject: [PATCH 048/158] chore(elvis): fix elvis error --- src/emqx_channel.erl | 11 ++++-- test/emqx_channel_SUITE.erl | 71 ++++++++++++++++++++++++++++--------- 2 files changed, 63 insertions(+), 19 deletions(-) diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 97a1fce23..202403399 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -410,9 +410,12 @@ handle_in(Packet = ?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters), TopicFilters3 = run_hooks('client.subscribe', [ClientInfo, Properties], TopicFilters2), - {TupleTopicFilters1, NChannel} = process_subscribe(TopicFilters3, Properties, Channel), + {TupleTopicFilters1, NChannel} = process_subscribe(TopicFilters3, + Properties, + Channel), TupleTopicFilters2 = Replace(TupleTopicFilters0, TupleTopicFilters1), - ReasonCodes2 = [ ReasonCode || {_TopicFilter, ReasonCode} <- TupleTopicFilters2], + ReasonCodes2 = [ ReasonCode + || {_TopicFilter, ReasonCode} <- TupleTopicFilters2], handle_out(suback, {PacketId, ReasonCodes2}, NChannel) end; {error, ReasonCode} -> @@ -624,7 +627,9 @@ process_subscribe([], _SubProps, Channel, Acc) -> process_subscribe([Topic = {TopicFilter, SubOpts}|More], SubProps, Channel, Acc) -> case check_sub_caps(TopicFilter, SubOpts, Channel) of ok -> - {ReasonCode, NChannel} = do_subscribe(TopicFilter, SubOpts#{sub_props => SubProps}, Channel), + {ReasonCode, NChannel} = do_subscribe(TopicFilter, + SubOpts#{sub_props => SubProps}, + Channel), process_subscribe(More, SubProps, NChannel, [{Topic, ReasonCode} | Acc]); {error, ReasonCode} -> ?LOG(warning, "Cannot subscribe ~s due to ~s.", diff --git a/test/emqx_channel_SUITE.erl b/test/emqx_channel_SUITE.erl index f3ee4aade..424d6ec48 100644 --- a/test/emqx_channel_SUITE.erl +++ b/test/emqx_channel_SUITE.erl @@ -133,7 +133,7 @@ t_handle_in_connect_auth_failed(_) -> clientid = <<"clientid">>, username = <<"username">> }, - {shutdown, not_authorized, ?CONNACK_PACKET(?RC_NOT_AUTHORIZED), _} = + {shutdown, not_authorized, ?CONNACK_PACKET(?RC_NOT_AUTHORIZED), _} = emqx_channel:handle_in(?CONNECT_PACKET(ConnPkt), channel(#{conn_state => idle})). t_handle_in_continue_auth(_) -> @@ -144,19 +144,33 @@ t_handle_in_continue_auth(_) -> {shutdown, bad_authentication_method, ?CONNACK_PACKET(?RC_BAD_AUTHENTICATION_METHOD), _} = emqx_channel:handle_in(?AUTH_PACKET(?RC_CONTINUE_AUTHENTICATION,Properties), channel()), {shutdown, not_authorized, ?CONNACK_PACKET(?RC_NOT_AUTHORIZED), _} = - emqx_channel:handle_in(?AUTH_PACKET(?RC_CONTINUE_AUTHENTICATION,Properties), channel(#{conninfo => #{proto_ver => ?MQTT_PROTO_V5, conn_props => Properties}})). + emqx_channel:handle_in( + ?AUTH_PACKET(?RC_CONTINUE_AUTHENTICATION,Properties), + channel(#{conninfo => #{proto_ver => ?MQTT_PROTO_V5, conn_props => Properties}}) + ). t_handle_in_re_auth(_) -> Properties = #{ 'Authentication-Method' => <<"failed_auth_method">>, 'Authentication-Data' => <<"failed_auth_data">> }, - {ok, [{outgoing, ?DISCONNECT_PACKET(?RC_BAD_AUTHENTICATION_METHOD)}, {close, bad_authentication_method}], _} = - emqx_channel:handle_in(?AUTH_PACKET(?RC_RE_AUTHENTICATE,Properties), channel()), - {ok, [{outgoing, ?DISCONNECT_PACKET(?RC_BAD_AUTHENTICATION_METHOD)}, {close, bad_authentication_method}], _} = - emqx_channel:handle_in(?AUTH_PACKET(?RC_RE_AUTHENTICATE,Properties), channel(#{conninfo => #{proto_ver => ?MQTT_PROTO_V5, conn_props => undefined}})), + {ok, [{outgoing, ?DISCONNECT_PACKET(?RC_BAD_AUTHENTICATION_METHOD)}, + {close, bad_authentication_method}], _} = + emqx_channel:handle_in( + ?AUTH_PACKET(?RC_RE_AUTHENTICATE,Properties), + channel() + ), + {ok, [{outgoing, ?DISCONNECT_PACKET(?RC_BAD_AUTHENTICATION_METHOD)}, + {close, bad_authentication_method}], _} = + emqx_channel:handle_in( + ?AUTH_PACKET(?RC_RE_AUTHENTICATE,Properties), + channel(#{conninfo => #{proto_ver => ?MQTT_PROTO_V5, conn_props => undefined}}) + ), {ok, [{outgoing, ?DISCONNECT_PACKET(?RC_NOT_AUTHORIZED)}, {close, not_authorized}], _} = - emqx_channel:handle_in(?AUTH_PACKET(?RC_RE_AUTHENTICATE,Properties), channel(#{conninfo => #{proto_ver => ?MQTT_PROTO_V5, conn_props => Properties}})). + emqx_channel:handle_in( + ?AUTH_PACKET(?RC_RE_AUTHENTICATE,Properties), + channel(#{conninfo => #{proto_ver => ?MQTT_PROTO_V5, conn_props => Properties}}) + ). t_handle_in_qos0_publish(_) -> ok = meck:expect(emqx_broker, publish, fun(_) -> [] end), @@ -348,7 +362,8 @@ t_process_publish_qos1(_) -> t_process_subscribe(_) -> ok = meck:expect(emqx_session, subscribe, fun(_, _, _, Session) -> {ok, Session} end), TopicFilters = [ TopicFilter = {<<"+">>, ?DEFAULT_SUBOPTS}], - {[{TopicFilter, ?RC_SUCCESS}], _Channel} = emqx_channel:process_subscribe(TopicFilters, #{}, channel()). + {[{TopicFilter, ?RC_SUCCESS}], _Channel} = + emqx_channel:process_subscribe(TopicFilters, #{}, channel()). t_process_unsubscribe(_) -> ok = meck:expect(emqx_session, unsubscribe, fun(_, _, _, Session) -> {ok, Session} end), @@ -358,8 +373,16 @@ t_process_unsubscribe(_) -> t_quota_qos0(_) -> esockd_limiter:start_link(), Cnter = counters:new(1, []), ok = meck:expect(emqx_broker, publish, fun(_) -> [{node(), <<"topic">>, {ok, 4}}] end), - ok = meck:expect(emqx_metrics, inc, fun('packets.publish.dropped') -> counters:add(Cnter, 1, 1) end), - ok = meck:expect(emqx_metrics, val, fun('packets.publish.dropped') -> counters:get(Cnter, 1) end), + ok = meck:expect( + emqx_metrics, + inc, + fun('packets.publish.dropped') -> counters:add(Cnter, 1, 1) end + ), + ok = meck:expect( + emqx_metrics, + val, + fun('packets.publish.dropped') -> counters:get(Cnter, 1) end + ), Chann = channel(#{conn_state => connected, quota => quota()}), Pub = ?PUBLISH_PACKET(?QOS_0, <<"topic">>, undefined, <<"payload">>), @@ -454,8 +477,12 @@ t_handle_out_connack_response_information(_) -> end), ok = meck:expect(emqx_zone, response_information, fun(_) -> test end), IdleChannel = channel(#{conn_state => idle}), - {ok, [{event, connected}, {connack, ?CONNACK_PACKET(?RC_SUCCESS, 0, #{'Response-Information' := test})}], _} = - emqx_channel:handle_in(?CONNECT_PACKET(connpkt(#{'Request-Response-Information' => 1})), IdleChannel). + {ok, [{event, connected}, + {connack, ?CONNACK_PACKET(?RC_SUCCESS, 0, #{'Response-Information' := test})}], + _} = emqx_channel:handle_in( + ?CONNECT_PACKET(connpkt(#{'Request-Response-Information' => 1})), + IdleChannel + ). t_handle_out_connack_not_response_information(_) -> ok = meck:expect(emqx_cm, open_session, @@ -465,7 +492,10 @@ t_handle_out_connack_not_response_information(_) -> ok = meck:expect(emqx_zone, response_information, fun(_) -> test end), IdleChannel = channel(#{conn_state => idle}), {ok, [{event, connected}, {connack, ?CONNACK_PACKET(?RC_SUCCESS, 0, AckProps)}], _} = - emqx_channel:handle_in(?CONNECT_PACKET(connpkt(#{'Request-Response-Information' => 0})), IdleChannel), + emqx_channel:handle_in( + ?CONNECT_PACKET(connpkt(#{'Request-Response-Information' => 0})), + IdleChannel + ), ?assertEqual(false, maps:is_key('Response-Information', AckProps)). t_handle_out_connack_failure(_) -> @@ -530,7 +560,10 @@ t_handle_call_takeover_end(_) -> emqx_channel:handle_call({takeover, 'end'}, channel()). t_handle_call_quota(_) -> - {reply, ok, _Chan} = emqx_channel:handle_call({quota, [{conn_messages_routing, {100,1}}]}, channel()). + {reply, ok, _Chan} = emqx_channel:handle_call( + {quota, [{conn_messages_routing, {100,1}}]}, + channel() + ). t_handle_call_unexpected(_) -> {reply, ignored, _Chan} = emqx_channel:handle_call(unexpected_req, channel()). @@ -634,9 +667,15 @@ t_packing_alias(_) -> }}}, RePacket2), {RePacket3, _} = emqx_channel:packing_alias(Packet2, NChannel2), - ?assertEqual(#mqtt_packet{variable = #mqtt_packet_publish{topic_name = <<"y">>, properties = #{}}}, RePacket3), + ?assertEqual( + #mqtt_packet{variable = #mqtt_packet_publish{topic_name = <<"y">>, properties = #{}}}, + RePacket3 + ), - ?assertMatch({#mqtt_packet{variable = #mqtt_packet_publish{topic_name = <<"z">>}}, _}, emqx_channel:packing_alias(#mqtt_packet{variable = #mqtt_packet_publish{topic_name = <<"z">>}}, channel())). + ?assertMatch({#mqtt_packet{variable = #mqtt_packet_publish{topic_name = <<"z">>}}, _}, + emqx_channel:packing_alias( + #mqtt_packet{variable = #mqtt_packet_publish{topic_name = <<"z">>}}, + channel())). t_check_pub_acl(_) -> ok = meck:new(emqx_zone, [passthrough, no_history]), From 109b2f4e9053a1a0de968bf1f0bc946b53d7870a Mon Sep 17 00:00:00 2001 From: wwhai Date: Mon, 15 Mar 2021 14:53:44 +0800 Subject: [PATCH 049/158] fix(backup): fix data import/export problem --- .../src/emqx_mgmt_data_backup.erl | 45 +++++++++++++++---- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index cd6ef6fb9..b9c3d9933 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -17,6 +17,7 @@ -module(emqx_mgmt_data_backup). -include("emqx_mgmt.hrl"). +-include_lib("emqx_rule_engine/include/rule_engine.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("kernel/include/file.hrl"). @@ -59,7 +60,11 @@ %%-------------------------------------------------------------------- export_rules() -> - lists:map(fun({_, RuleId, _, RawSQL, _, _, _, _, _, _, Actions, Enabled, Desc}) -> + lists:map(fun(#rule{id = RuleId, + rawsql = RawSQL, + actions = Actions, + enabled = Enabled, + description = Desc}) -> [{id, RuleId}, {rawsql, RawSQL}, {actions, actions_to_prop_list(Actions)}, @@ -68,7 +73,11 @@ export_rules() -> end, emqx_rule_registry:get_rules()). export_resources() -> - lists:map(fun({_, Id, Type, Config, CreatedAt, Desc}) -> + lists:map(fun(#resource{id = Id, + type = Type, + config = Config, + created_at = CreatedAt, + description = Desc}) -> NCreatedAt = case CreatedAt of undefined -> null; _ -> CreatedAt @@ -236,18 +245,25 @@ import_resources_and_rules(Resources, Rules, FromVersion) when FromVersion =:= "4.0" orelse FromVersion =:= "4.1" orelse FromVersion =:= "4.2" -> Configs = lists:foldl(fun(#{<<"id">> := ID, <<"type">> := <<"web_hook">>, - <<"config">> := #{<<"content_type">> := ContentType, + <<"config">> := #{<<"connect_timeout">> := ConnectTimeout, + <<"content_type">> := ContentType, <<"headers">> := Headers, <<"method">> := Method, + <<"pool_size">> := PoolSize, + <<"request_timeout">> := RequestTimeout, <<"url">> := URL}} = Resource, Acc) -> - NConfig = #{<<"connect_timeout">> => 5, - <<"request_timeout">> => 5, + CovertFun = fun(Int) -> + list_to_binary(integer_to_list(Int) ++ "s") + end, + NConfig = #{<<"connect_timeout">> => CovertFun(ConnectTimeout), + <<"method">> => Method, + <<"pool_size">> => PoolSize, + <<"request_timeout">> => CovertFun(RequestTimeout), <<"cacertfile">> => <<>>, <<"certfile">> => <<>>, <<"keyfile">> => <<>>, - <<"pool_size">> => 8, - <<"url">> => URL, - <<"verify">> => true}, + <<"verify">> => true, + <<"url">> => URL}, NResource = Resource#{<<"config">> := NConfig}, {ok, _Resource} = import_resource(NResource), NHeaders = maps:put(<<"content-type">>, ContentType, Headers), @@ -284,7 +300,18 @@ apply_new_config([Action = #{<<"name">> := <<"data_to_webserver">>, <<"method">> => Method, <<"path">> => Path}, apply_new_config(More, Configs, [Action#{<<"args">> := Args} | Acc]) - end. + end; + +apply_new_config([Action = #{<<"args">> := #{<<"$resource">> := ResourceId, + <<"forward_topic">> := ForwardTopic, + <<"payload_tmpl">> := PayloadTmpl}, + <<"fallbacks">> := _Fallbacks, + <<"id">> := _Id, + <<"name">> := <<"data_to_mqtt_broker">>} | More], Configs, Acc) -> + Args = #{<<"$resource">> => ResourceId, + <<"payload_tmpl">> => PayloadTmpl, + <<"forward_topic">> => ForwardTopic}, + apply_new_config(More, Configs, [Action#{<<"args">> := Args} | Acc]). -endif. From 3f2204fb5ccdf3a96cd5b5582a14a0d466b78d36 Mon Sep 17 00:00:00 2001 From: wwhai Date: Thu, 18 Mar 2021 11:16:50 +0800 Subject: [PATCH 050/158] fix(backup): add compatible for more lower version --- .../src/emqx_mgmt_data_backup.erl | 90 +++++++++++++------ 1 file changed, 61 insertions(+), 29 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index b9c3d9933..009b4c888 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -243,35 +243,7 @@ import_resources_and_rules(Resources, Rules, _FromVersion) -> -else. import_resources_and_rules(Resources, Rules, FromVersion) when FromVersion =:= "4.0" orelse FromVersion =:= "4.1" orelse FromVersion =:= "4.2" -> - Configs = lists:foldl(fun(#{<<"id">> := ID, - <<"type">> := <<"web_hook">>, - <<"config">> := #{<<"connect_timeout">> := ConnectTimeout, - <<"content_type">> := ContentType, - <<"headers">> := Headers, - <<"method">> := Method, - <<"pool_size">> := PoolSize, - <<"request_timeout">> := RequestTimeout, - <<"url">> := URL}} = Resource, Acc) -> - CovertFun = fun(Int) -> - list_to_binary(integer_to_list(Int) ++ "s") - end, - NConfig = #{<<"connect_timeout">> => CovertFun(ConnectTimeout), - <<"method">> => Method, - <<"pool_size">> => PoolSize, - <<"request_timeout">> => CovertFun(RequestTimeout), - <<"cacertfile">> => <<>>, - <<"certfile">> => <<>>, - <<"keyfile">> => <<>>, - <<"verify">> => true, - <<"url">> => URL}, - NResource = Resource#{<<"config">> := NConfig}, - {ok, _Resource} = import_resource(NResource), - NHeaders = maps:put(<<"content-type">>, ContentType, Headers), - [{ID, #{headers => NHeaders, method => Method}} | Acc]; - (Resource, Acc) -> - {ok, _Resource} = import_resource(Resource), - Acc - end, [], Resources), + Configs = lists:foldl(fun compatible_version/2 , [], Resources), lists:foreach(fun(#{<<"actions">> := Actions} = Rule) -> NActions = apply_new_config(Actions, Configs), import_rule(Rule#{<<"actions">> := NActions}) @@ -281,6 +253,66 @@ import_resources_and_rules(Resources, Rules, _FromVersion) -> import_resources(Resources), import_rules(Rules). + +%% 4.2.5 + +compatible_version(#{<<"id">> := ID, + <<"type">> := <<"web_hook">>, + <<"config">> := #{<<"connect_timeout">> := ConnectTimeout, + <<"content_type">> := ContentType, + <<"headers">> := Headers, + <<"method">> := Method, + <<"pool_size">> := PoolSize, + <<"request_timeout">> := RequestTimeout, + <<"url">> := URL}} = Resource, Acc) -> + CovertFun = fun(Int) -> + list_to_binary(integer_to_list(Int) ++ "s") + end, + Cfg = make_new_config(#{<<"method">> => Method, + <<"pool_size">> => PoolSize, + <<"connect_timeout">> => CovertFun(ConnectTimeout), + <<"request_timeout">> => CovertFun(RequestTimeout), + <<"url">> => URL}), + {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}), + NHeaders = maps:put(<<"content-type">>, ContentType, Headers), + [{ID, #{headers => NHeaders, method => Method}} | Acc]; +% 4.2.0 +compatible_version(#{<<"id">> := ID, + <<"type">> := <<"web_hook">>, + <<"config">> := #{<<"headers">> := Headers, + <<"method">> := Method,%% 4.2.0 Different here + <<"url">> := URL}} = Resource, Acc) -> + Cfg = make_new_config(#{<<"method">> => Method, + <<"url">> => URL}), + {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}), + NHeaders = maps:put(<<"content-type">>, <<"application/json">> , Headers), + [{ID, #{headers => NHeaders, method => Method}} | Acc]; +% 4.2.3 +compatible_version(#{<<"id">> := ID, + <<"type">> := <<"web_hook">>, + <<"config">> := #{<<"headers">> := Headers, + <<"content_type">> := ContentType,%% 4.2.3 Different here + <<"method">> := Method, + <<"url">> := URL}} = Resource, Acc) -> + Cfg = make_new_config(#{<<"method">> => Method, + <<"url">> => URL}), + {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}), + NHeaders = maps:put(<<"content-type">>, ContentType, Headers), + [{ID, #{headers => NHeaders, method => Method}} | Acc]; +% normal version +compatible_version(Resource, Acc) -> + {ok, _Resource} = import_resource(Resource), + Acc. + +make_new_config(Cfg) -> + Config = #{<<"pool_size">> => 8, + <<"connect_timeout">> => <<"3s">>, + <<"request_timeout">> => <<"3s">>, + <<"cacertfile">> => <<>>, + <<"certfile">> => <<>>, + <<"keyfile">> => <<>>, + <<"verify">> => true}, + maps:merge(Cfg, Config). + apply_new_config(Actions, Configs) -> apply_new_config(Actions, Configs, []). From 6d6fe23d4539dc702f6d654f6ad439abfe8e6270 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 19 Mar 2021 15:14:43 +0800 Subject: [PATCH 051/158] fix(CI): fix fvt test error --- .github/workflows/build_slim_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 62613031f..d1b00a2f1 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v1 - name: build packages run: | - if make -C source emqx-ee --dry-run > /dev/null 2>&1; then + if make emqx-ee --dry-run > /dev/null 2>&1; then echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials git config --global credential.helper store echo "${{ secrets.CI_GIT_TOKEN }}" >> source/scripts/git-token From 178404c8214692a357e397ec254724fe20149675 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Mar 2021 23:23:08 +0100 Subject: [PATCH 052/158] chore(build): add rebar_mix plugin --- bin/emqx | 1 + lib-extra/plugins | 5 +++++ rebar.config.erl | 32 ++++++++++++++++++++------------ 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/bin/emqx b/bin/emqx index 89a1d796a..4901c816a 100755 --- a/bin/emqx +++ b/bin/emqx @@ -221,6 +221,7 @@ generate_config() { TMP_ARG_FILE="$RUNNER_DATA_DIR/configs/vm.args.tmp" cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE" echo "" >> "$TMP_ARG_FILE" + echo "-pa ${REL_DIR}/consolidated" >> "$TMP_ARG_FILE" sed '/^#/d' "$CUTTLE_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}') ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}') diff --git a/lib-extra/plugins b/lib-extra/plugins index 0f3177b9e..f7af801c1 100644 --- a/lib-extra/plugins +++ b/lib-extra/plugins @@ -2,3 +2,8 @@ [ {emqx_plugin_template, {git, "https://github.com/emqx/emqx-plugin-template", {branch, "master"}}} ] }. + +{elixir_plugins, + [ {emqx_elixir_plugin, {git, "https://github.com/zmstone/emqx-elixir-plugin.git", {branch, "umbrella"}}} + ] +}. diff --git a/rebar.config.erl b/rebar.config.erl index 837b9a56b..7bcc030f7 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -3,9 +3,9 @@ -export([do/2]). do(_Dir, CONFIG) -> - C1 = deps(CONFIG), + {HasElixir, C1} = deps(CONFIG), Config = dialyzer(C1), - maybe_dump(Config ++ [{overrides, overrides()}] ++ coveralls() ++ config()). + maybe_dump(Config ++ [{overrides, overrides()}] ++ coveralls() ++ config(HasElixir)). bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {branch, "0.6.0"}}}. @@ -16,13 +16,17 @@ deps(Config) -> true -> [bcrypt()]; false -> [] end, - lists:keystore(deps, 1, Config, {deps, OldDeps ++ MoreDeps ++ extra_deps()}). + {HasElixir, ExtraDeps} = extra_deps(), + {HasElixir, lists:keystore(deps, 1, Config, {deps, OldDeps ++ MoreDeps ++ ExtraDeps})}. extra_deps() -> {ok, Proplist} = file:consult("lib-extra/plugins"), - AllPlugins = proplists:get_value(erlang_plugins, Proplist), + ErlPlugins0 = proplists:get_value(erlang_plugins, Proplist), + ExPlugins0 = proplists:get_value(elixir_plugins, Proplist), Filter = string:split(os:getenv("EMQX_EXTRA_PLUGINS", ""), ",", all), - filter_extra_deps(AllPlugins, Filter). + ErlPlugins = filter_extra_deps(ErlPlugins0, Filter), + ExPlugins = filter_extra_deps(ExPlugins0, Filter), + {ExPlugins=/= [], ErlPlugins ++ ExPlugins}. filter_extra_deps(AllPlugins, ["all"]) -> AllPlugins; @@ -49,11 +53,14 @@ overrides() -> community_plugin_overrides() -> [{add, App, [ {erl_opts, [{i, "include"}]}]} || App <- relx_plugin_apps_extra()]. -config() -> +config(HasElixir) -> [ {cover_enabled, is_cover_enabled()} - , {plugins, plugins()} , {profiles, profiles()} , {project_app_dirs, project_app_dirs()} + , {plugins, plugins(HasElixir)} + | [ {provider_hooks, [ {pre, [{compile, {mix, find_elixir_libs}}]} + , {post, [{compile, {mix, consolidate_protocols}}]} + ]} || HasElixir ] ]. is_cover_enabled() -> @@ -75,9 +82,10 @@ alternative_lib_dir() -> project_app_dirs() -> ["apps/*", alternative_lib_dir() ++ "/*", "."]. -plugins() -> - [ {relup_helper,{git,"https://github.com/emqx/relup_helper", {tag, "2.0.0"}}}, - {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}} +plugins(HasElixir) -> + [ {relup_helper,{git,"https://github.com/emqx/relup_helper", {tag, "2.0.0"}}} + , {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}} + | [ rebar_mix || HasElixir ] ] %% test plugins are concatenated to default profile plugins %% otherwise rebar3 test profile runs are super slow @@ -176,7 +184,6 @@ emqx_description(cloud, true) -> "EMQ X Enterprise"; emqx_description(cloud, false) -> "EMQ X Broker"; emqx_description(edge, _) -> "EMQ X Edge". - overlay_vars(_RelType, PkgType, true) -> ee_overlay_vars(PkgType); overlay_vars(RelType, PkgType, false) -> @@ -305,7 +312,8 @@ relx_plugin_apps_enterprise(true) -> relx_plugin_apps_enterprise(false) -> []. relx_plugin_apps_extra() -> - [Plugin || {Plugin, _} <- extra_deps()]. + {_HasElixir, ExtraDeps} = extra_deps(), + [Plugin || {Plugin, _} <- ExtraDeps]. relx_overlay(ReleaseType) -> [ {mkdir, "log/"} From 09ff109fda780602984d4ccc7a3d3462e3e88d1e Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Mar 2021 23:35:55 +0100 Subject: [PATCH 053/158] chore(build): Pin version tag for template plugin --- lib-extra/plugins | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-extra/plugins b/lib-extra/plugins index f7af801c1..add732e61 100644 --- a/lib-extra/plugins +++ b/lib-extra/plugins @@ -1,5 +1,5 @@ {erlang_plugins, - [ {emqx_plugin_template, {git, "https://github.com/emqx/emqx-plugin-template", {branch, "master"}}} + [ {emqx_plugin_template, {git, "https://github.com/emqx/emqx-plugin-template", {tag, "4.3.0"}}} ] }. From 73a8494760a6fb2dcff93e49c0e0086804235d86 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 17 Mar 2021 22:30:40 +0100 Subject: [PATCH 054/158] fix(build): rebar3 compile before release Doing release directly works fine for prue Erlang dependencies. However if we add a elixir dependency, the application info and release becomes problematic Elixir apps do not have .app or .app.src files so the app vsn can not be found during app discovery which leads to "No valid version ([]) of .app file found" error when creating the release. This is maybe fixed in https://github.com/erlang/rebar3/pull/2518 Another issue is "Undefined applications: [elixir,logger]" This seem to be caused by the rebar3 not being able to identify as a release dependency if an elixir app was not compiled before. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cb36fcaec..85fd7bf6d 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,7 @@ coveralls: $(REBAR) .PHONY: $(REL_PROFILES) $(REL_PROFILES:%=%): $(REBAR) get-dashboard - @$(REBAR) as $(@) release + @$(REBAR) as $(@) do compile,release ## Not calling rebar3 clean because ## 1. rebar3 clean relies on rebar3, meaning it reads config, fetches dependencies etc. From ddec66fe35c92e7becaa1b8c94e7d31d45a543cb Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 18 Mar 2021 00:30:25 +0100 Subject: [PATCH 055/158] chore(build): remove plugin regression for now We do not have Elixir in build environment --- .github/workflows/run_test_cases.yaml | 2 -- rebar.config.erl | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 2eb7855d5..35cb7c1a6 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -40,11 +40,9 @@ jobs: - name: run eunit run: | docker exec -i erlang bash -c "make eunit" - docker exec --env EMQX_EXTRA_PLUGINS=all -i erlang bash -c "./rebar3 eunit --dir $(find lib-extra/ -mindepth 1 -maxdepth 2 -type l | tr '\n' ',')" - name: run common test run: | docker exec -i erlang bash -c "make ct" - docker exec --env EMQX_EXTRA_PLUGINS=all -i erlang bash -c "./rebar3 ct --dir $(find lib-extra/ -mindepth 1 -maxdepth 2 -type l | tr '\n' ',')" - name: run cover run: | docker exec -i erlang bash -c "make cover" diff --git a/rebar.config.erl b/rebar.config.erl index 7bcc030f7..f1ed9fda8 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -26,7 +26,7 @@ extra_deps() -> Filter = string:split(os:getenv("EMQX_EXTRA_PLUGINS", ""), ",", all), ErlPlugins = filter_extra_deps(ErlPlugins0, Filter), ExPlugins = filter_extra_deps(ExPlugins0, Filter), - {ExPlugins=/= [], ErlPlugins ++ ExPlugins}. + {ExPlugins =/= [], ErlPlugins ++ ExPlugins}. filter_extra_deps(AllPlugins, ["all"]) -> AllPlugins; @@ -34,10 +34,10 @@ filter_extra_deps(AllPlugins, Filter) -> filter_extra_deps(AllPlugins, Filter, []). filter_extra_deps([], _, Acc) -> lists:reverse(Acc); -filter_extra_deps([{Plugin, _}=P|More], Filter, Acc) -> +filter_extra_deps([{Plugin, _} = P | More], Filter, Acc) -> case lists:member(atom_to_list(Plugin), Filter) of true -> - filter_extra_deps(More, Filter, [P|Acc]); + filter_extra_deps(More, Filter, [P | Acc]); false -> filter_extra_deps(More, Filter, Acc) end. From 8f6e02c3224ffc4118c57e2763fc4da92a6b3a60 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 19 Mar 2021 12:08:39 +0100 Subject: [PATCH 056/158] chore(lib-extra): Pin elixir plugin dependency with tag --- lib-extra/plugins | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-extra/plugins b/lib-extra/plugins index add732e61..6c0f1d9f0 100644 --- a/lib-extra/plugins +++ b/lib-extra/plugins @@ -4,6 +4,6 @@ }. {elixir_plugins, - [ {emqx_elixir_plugin, {git, "https://github.com/zmstone/emqx-elixir-plugin.git", {branch, "umbrella"}}} + [ {emqx_elixir_plugin, {git, "https://github.com/emqx/emqx-elixir-plugin.git", {branch, "3.0.0"}}} ] }. From 2d150127d40f194b917adf8e565ed8e73549ed62 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 19 Mar 2021 09:56:20 +0100 Subject: [PATCH 057/158] feat(listeners): Ensure no tlsv1.3 for otp 22 or earlier --- src/emqx_listeners.erl | 16 ++++++++++-- src/emqx_tls_lib.erl | 56 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/src/emqx_listeners.erl b/src/emqx_listeners.erl index ecbc54053..270a66085 100644 --- a/src/emqx_listeners.erl +++ b/src/emqx_listeners.erl @@ -134,7 +134,18 @@ start_listener(Proto, ListenOn, Options) when Proto == https; Proto == wss -> start_http_listener(fun cowboy:start_tls/3, 'mqtt:wss', ListenOn, ranch_opts(Options), ws_opts(Options)). -start_mqtt_listener(Name, ListenOn, Options) -> +replace(Opts, Key, Value) -> [{Key, Value} | proplists:delete(Key, Opts)]. + +drop_tls13_for_old_otp(Options) -> + case proplists:get_value(ssl_options, Options) of + undefined -> Options; + SslOpts -> + SslOpts1 = emqx_tls_lib:drop_tls13_for_old_otp(SslOpts), + replace(Options, ssl_options, SslOpts1) + end. + +start_mqtt_listener(Name, ListenOn, Options0) -> + Options = drop_tls13_for_old_otp(Options0), SockOpts = esockd:parse_opt(Options), esockd:open(Name, ListenOn, merge_default(SockOpts), {emqx_connection, start_link, [Options -- SockOpts]}). @@ -151,7 +162,8 @@ ws_opts(Options) -> ProxyProto = proplists:get_value(proxy_protocol, Options, false), #{env => #{dispatch => Dispatch}, proxy_header => ProxyProto}. -ranch_opts(Options) -> +ranch_opts(Options0) -> + Options = drop_tls13_for_old_otp(Options0), NumAcceptors = proplists:get_value(acceptors, Options, 4), MaxConnections = proplists:get_value(max_connections, Options, 1024), TcpOptions = proplists:get_value(tcp_options, Options, []), diff --git a/src/emqx_tls_lib.erl b/src/emqx_tls_lib.erl index 215f0b5ca..72a955962 100644 --- a/src/emqx_tls_lib.erl +++ b/src/emqx_tls_lib.erl @@ -21,6 +21,7 @@ , default_ciphers/0 , default_ciphers/1 , integral_ciphers/2 + , drop_tls13_for_old_otp/1 ]). %% non-empty string @@ -140,3 +141,58 @@ split_by_comma(Bin) -> %% trim spaces trim_space(Bin) -> hd([I || I <- binary:split(Bin, <<" ">>), I =/= <<>>]). + +%% @doc Drop tlsv1.3 version and ciphers from ssl options +%% if running on otp 22 or earlier. +drop_tls13_for_old_otp(SslOpts) -> + case list_to_integer(erlang:system_info(otp_release)) < 23 of + true -> drop_tls13(SslOpts); + false -> SslOpts + end. + +%% The ciphers that ssl:cipher_suites(exclusive, 'tlsv1.3', openssl) +%% should return when running on otp 23. +%% But we still have to hard-code them because tlsv1.3 on otp 22 is +%% not trustworthy. +-define(TLSV13_EXCLUSIVE_CIPHERS, [ "TLS_AES_256_GCM_SHA384" + , "TLS_AES_128_GCM_SHA256" + , "TLS_CHACHA20_POLY1305_SHA256" + , "TLS_AES_128_CCM_SHA256" + , "TLS_AES_128_CCM_8_SHA256" + ]). +drop_tls13(SslOpts0) -> + SslOpts1 = case proplists:get_value(versions, SslOpts0) of + undefined -> SslOpts0; + Vsns -> replace(SslOpts0, versions, Vsns -- ['tlsv1.3']) + end, + case proplists:get_value(ciphers, SslOpts1) of + undefined -> SslOpts1; + Ciphers -> replace(SslOpts1, ciphers, Ciphers -- ?TLSV13_EXCLUSIVE_CIPHERS) + end. + +replace(Opts, Key, Value) -> [{Key, Value} | proplists:delete(Key, Opts)]. + +-if(?OTP_RELEASE > 22). +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +drop_tls13_test() -> + Versions = default_versions(), + ?assert(lists:member('tlsv1.3', Versions)), + Ciphers = default_ciphers(), + ?assert(has_tlsv13_cipher(Ciphers)), + Opts0 = [{versions, Versions}, {ciphers, Ciphers}, other, {bool, true}], + Opts = drop_tls13(Opts0), + ?assertNot(lists:member('tlsv1.3', proplists:get_value(versions, Opts))), + ?assertNot(has_tlsv13_cipher(proplists:get_value(ciphers, Opts))). + +drop_tls13_no_versions_cipers_test() -> + Opts0 = [other, {bool, true}], + Opts = drop_tls13(Opts0), + ?_assertEqual(Opts0, Opts). + +has_tlsv13_cipher(Ciphers) -> + lists:any(fun(C) -> lists:member(C, Ciphers) end, ?TLSV13_EXCLUSIVE_CIPHERS). + +-endif. +-endif. From e06b54ff71c7022286feaf1c95a78a24d03d2b11 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 19 Mar 2021 10:35:28 +0100 Subject: [PATCH 058/158] chore(boot-log): print a warning at boot when running on old otp --- src/emqx_app.erl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/emqx_app.erl b/src/emqx_app.erl index e62c85f25..ca1a2ce09 100644 --- a/src/emqx_app.erl +++ b/src/emqx_app.erl @@ -33,6 +33,7 @@ %%-------------------------------------------------------------------- start(_Type, _Args) -> + print_otp_version_warning(), print_banner(), ekka:start(), {ok, Sup} = emqx_sup:start_link(), @@ -56,6 +57,15 @@ stop(_State) -> %% Print Banner %%-------------------------------------------------------------------- +-if(?OTP_RELEASE> 22). +print_otp_version_warning() -> ok. +-else. +print_otp_version_warning() -> + io:format("WARNING: Running on Erlang/OTP version ~p. Recommended: 23~n", + [?OTP_RELEASE]). +-endif. + + print_banner() -> io:format("Starting ~s on node ~s~n", [?APP, node()]). From beac1f5f59e9a6cc2bab8b30a62420d732b30916 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 19 Mar 2021 22:55:54 +0800 Subject: [PATCH 059/158] fix(test cases): fix test case error ensure emqx_modues loaded before emqx_management in test cases --- apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl | 4 ++-- apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl | 4 ++-- apps/emqx_management/test/emqx_mgmt_api_SUITE.erl | 3 --- lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl | 4 ++-- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl index 13b041491..f900fb91a 100644 --- a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl @@ -42,13 +42,13 @@ groups() -> []. init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_management, emqx_auth_mnesia], fun set_special_configs/1), + emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_auth_mnesia], fun set_special_configs/1), create_default_app(), Config. end_per_suite(_Config) -> delete_default_app(), - emqx_ct_helpers:stop_apps([emqx_management, emqx_auth_mnesia]). + emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_auth_mnesia]). init_per_testcase(t_check_acl_as_clientid, Config) -> emqx:hook('client.check_acl', fun emqx_acl_mnesia:check_acl/5, [#{key_as => clientid}]), diff --git a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl index 1c3dc50b4..8ac942257 100644 --- a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl @@ -47,13 +47,13 @@ groups() -> []. init_per_suite(Config) -> - ok = emqx_ct_helpers:start_apps([emqx_management, emqx_auth_mnesia], fun set_special_configs/1), + ok = emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_auth_mnesia], fun set_special_configs/1), create_default_app(), Config. end_per_suite(_Config) -> delete_default_app(), - emqx_ct_helpers:stop_apps([emqx_management, emqx_auth_mnesia]). + emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_auth_mnesia]). init_per_testcase(t_check_as_clientid, Config) -> Params = #{ diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index a11ab6f26..ea52690d7 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -87,9 +87,6 @@ init_per_testcase(_, Config) -> end_per_testcase(data, _Config) -> application:stop(emqx_dahboard), application:stop(emqx_rule_engine), - application:stop(emqx_modules), - application:stop(emqx_schema_registry), - application:stop(emqx_conf), meck:unload(emqx_sys), ok; diff --git a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 4290db22a..60714a639 100644 --- a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -54,11 +54,11 @@ groups() -> ]. init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx, emqx_management, emqx_dashboard]), + emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_dashboard]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_dashboard, emqx_management, emqx]), + emqx_ct_helpers:stop_apps([emqx_dashboard, emqx_management, emqx_modules]), ekka_mnesia:ensure_stopped(). t_overview(_) -> From 8201e4c820ba87918897f6e8766e5cb17dc02386 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 19 Mar 2021 20:51:05 +0100 Subject: [PATCH 060/158] feat(emqx.erl): Add a help function to load debug secret --- src/emqx.erl | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/emqx.erl b/src/emqx.erl index b8b8cffbc..ed4fd16e6 100644 --- a/src/emqx.erl +++ b/src/emqx.erl @@ -63,12 +63,39 @@ , reboot/0 ]). +%% Troubleshooting +-export([ set_debug_secret/1 + ]). + -define(APP, ?MODULE). -define(COPYRIGHT, "Copyright (c) 2020 EMQ Technologies Co., Ltd"). -define(LICENSE_MESSAGE, "Licensed under the Apache License, Version 2.0"). +%% @hidden Path to the file which has debug_info encryption secret in it. +%% Evaluate this function if there is a need to access encrypted debug_info. +%% NOTE: Do not change the API to accept the secret text because it may +%% get logged everywhere. +set_debug_secret(PathToSecretFile) -> + SecretText = + case file:read_file(PathToSecretFile) of + {ok, Secret} -> + try string:trim(binary_to_list(Secret)) + catch _ : _ -> error({badfile, PathToSecretFile}) + end; + {error, Reason} -> + io:format("Failed to read debug_info encryption key file ~s: ~p~n", + [PathToSecretFile, Reason]), + error(Reason) + end, + F = fun(init) -> ok; + (clear) -> ok; + ({debug_info, _Mode, _Module, _Filename}) -> SecretText + end, + _ = beam_lib:clear_crypto_key_fun(), + ok = beam_lib:crypto_key_fun(F). + %%-------------------------------------------------------------------- %% Bootstrap, is_running... %%-------------------------------------------------------------------- From be4faa6d7c4cde1bdcd36cab4ce4571c827f33c8 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 19 Mar 2021 16:48:08 +0800 Subject: [PATCH 061/158] chore(CI): app test workflows support enterprose repo --- .ci/apps_tests/{.env => .opensource.env} | 0 .ci/apps_tests/docker-compose.yaml | 7 ++++- .github/workflows/run_cts_tests.yaml | 11 +++---- .github/workflows/run_test_cases.yaml | 38 +++++++++++++++++++++++- 4 files changed, 49 insertions(+), 7 deletions(-) rename .ci/apps_tests/{.env => .opensource.env} (100%) diff --git a/.ci/apps_tests/.env b/.ci/apps_tests/.opensource.env similarity index 100% rename from .ci/apps_tests/.env rename to .ci/apps_tests/.opensource.env diff --git a/.ci/apps_tests/docker-compose.yaml b/.ci/apps_tests/docker-compose.yaml index 2df39fe6f..2fd49a893 100644 --- a/.ci/apps_tests/docker-compose.yaml +++ b/.ci/apps_tests/docker-compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: '3.9' services: erlang: @@ -26,6 +26,11 @@ services: - ../../.:/emqx working_dir: /emqx tty: true + deploy: + resources: + reservations: + cpus: '0.5' + memory: 500M mysql_server: container_name: mysql diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index c0eda97dd..9dafcedb3 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -250,13 +250,16 @@ jobs: node_type: - single - cluster + exclude: + - redis_tag: 5 + connect_type: tls steps: - uses: actions/checkout@v1 - name: setup env: REDIS_TAG: ${{ matrix.redis_tag }} - if: matrix.connect_type == 'tls' && matrix.redis_tag != '5' + if: matrix.connect_type == 'tls' run: | set -exu docker-compose -f .ci/compatibility_tests/docker-compose-redis-${{ matrix.node_type }}-tls.yaml up -d @@ -274,7 +277,6 @@ jobs: docker-compose -f .ci/compatibility_tests/docker-compose-redis-${{ matrix.node_type }}.yaml up -d echo EMQX_AUTH__REDIS__SSL=off >> "$GITHUB_ENV" - name: get server address - if: matrix.connect_type == 'tcp' || (matrix.connect_type == 'tls' && matrix.redis_tag != '5') run: | set -exu ipv4_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' redis) @@ -292,7 +294,7 @@ jobs: EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:6379 EOF - name: setup - if: matrix.node_type == 'single' && matrix.connect_type == 'tls' && matrix.redis_tag != '5' + if: matrix.node_type == 'single' && matrix.connect_type == 'tls' run: | set -exu cat <<-EOF >> "$GITHUB_ENV" @@ -308,7 +310,7 @@ jobs: EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:7000 EOF - name: setup - if: matrix.node_type == 'cluster' && matrix.connect_type == 'tls' && matrix.redis_tag != '5' + if: matrix.node_type == 'cluster' && matrix.connect_type == 'tls' run: | set -exu cat <<-EOF >> "$GITHUB_ENV" @@ -316,7 +318,6 @@ jobs: EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:8000 EOF - name: run test cases - if: matrix.connect_type == 'tcp' || (matrix.connect_type == 'tls' && matrix.redis_tag != '5') run: | export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ printenv > .env diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 35cb7c1a6..03f0119ff 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -16,6 +16,12 @@ jobs: steps: - uses: actions/checkout@v2 + - name: set git credentials + run: | + if make emqx-ee --dry-run > /dev/null 2>&1; then + echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials + git config --global credential.helper store + fi - name: xref run: make xref - name: dialyzer @@ -26,7 +32,16 @@ jobs: steps: - uses: actions/checkout@v2 - - name: set up + - name: set edition + id: set_edition + run: | + if make emqx-ee --dry-run > /dev/null 2>&1; then + echo "EDITION=enterprise" >> $GITHUB_ENV + else + echo "EDITION=opensource" >> $GITHUB_ENV + fi + - name: docker compose up + if: env.EDITION == 'opensource' env: MYSQL_TAG: 8 REDIS_TAG: 6 @@ -37,6 +52,27 @@ jobs: run: | docker-compose -f .ci/apps_tests/docker-compose.yaml build --no-cache docker-compose -f .ci/apps_tests/docker-compose.yaml up -d + - name: docker compose up + if: env.EDITION == 'enterprise' + env: + MYSQL_TAG: 8 + REDIS_TAG: 6 + MONGO_TAG: 4 + PGSQL_TAG: 13 + LDAP_TAG: 2.4.50 + OPENTSDB_TAG: latest + INFLUXDB_TAG: 1.7.6 + DYNAMODB_TAG: 1.11.477 + TIMESCALE_TAG: latest-pg11 + CASSANDRA_TAG: 3.11.6 + RABBITMQ_TAG: 3.7 + KAFKA_TAG: 2.5.0 + PULSAR_TAG: 2.3.2 + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + docker-compose -f .ci/apps_tests/docker-compose.yaml -f .ci/apps_tests/docker-compose.enterprise.yaml build --no-cache + docker-compose -f .ci/apps_tests/docker-compose.yaml -f .ci/apps_tests/docker-compose.enterprise.yaml up -d + docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" - name: run eunit run: | docker exec -i erlang bash -c "make eunit" From 280fedca0d306af43a3ac308b64ae6e763589c07 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 19 Mar 2021 10:37:25 +0000 Subject: [PATCH 062/158] chore(CI): cts support enterprise repos --- .ci/apps_tests/{.opensource.env => .env} | 0 .github/workflows/run_cts_tests.yaml | 25 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) rename .ci/apps_tests/{.opensource.env => .env} (100%) diff --git a/.ci/apps_tests/.opensource.env b/.ci/apps_tests/.env similarity index 100% rename from .ci/apps_tests/.opensource.env rename to .ci/apps_tests/.env diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index 9dafcedb3..6ce4719bb 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -38,6 +38,11 @@ jobs: if: matrix.network_type == 'ipv6' run: | echo EMQX_AUTH__LDAP__SERVERS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' ldap) >> "$GITHUB_ENV" + - name: set git token + run: | + if make emqx-ee --dry-run > /dev/null 2>&1; then + docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" + fi - name: run test cases run: | export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ @@ -96,6 +101,11 @@ jobs: if: matrix.network_type == 'ipv6' run: | echo "EMQX_AUTH__MONGO__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mongo):27017" >> "$GITHUB_ENV" + - name: set git token + run: | + if make emqx-ee --dry-run > /dev/null 2>&1; then + docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" + fi - name: run test cases run: | export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ @@ -154,6 +164,11 @@ jobs: if: matrix.network_type == 'ipv6' run: | echo "EMQX_AUTH__MYSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mysql):3306" >> "$GITHUB_ENV" + - name: set git token + run: | + if make emqx-ee --dry-run > /dev/null 2>&1; then + docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" + fi - name: run test cases run: | export EMQX_AUTH__MYSQL__USERNAME=root \ @@ -216,6 +231,11 @@ jobs: if: matrix.network_type == 'ipv6' run: | echo "EMQX_AUTH__PGSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' pgsql):5432" >> "$GITHUB_ENV" + - name: set git token + run: | + if make emqx-ee --dry-run > /dev/null 2>&1; then + docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" + fi - name: run test cases run: | export EMQX_AUTH__PGSQL__USERNAME=root \ @@ -317,6 +337,11 @@ jobs: EMQX_AUTH__REDIS__TYPE=cluster EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:8000 EOF + - name: set git token + run: | + if make emqx-ee --dry-run > /dev/null 2>&1; then + docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" + fi - name: run test cases run: | export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ From 99843e7b4be4937e26c406f748edaa1968a7836e Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 19 Mar 2021 07:27:37 +0000 Subject: [PATCH 063/158] chore(auth plugins): remove emqx_modules in test case --- apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl | 4 +--- .../test/emqx_auth_ldap_bind_as_user_SUITE.erl | 6 ++---- apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl | 13 ++++--------- apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl | 9 ++++----- apps/emqx_auth_redis/test/emqx_auth_redis_SUITE.erl | 13 ++++--------- 5 files changed, 15 insertions(+), 30 deletions(-) diff --git a/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl index e13099f00..c9c38f610 100644 --- a/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl @@ -44,13 +44,11 @@ groups() -> init_per_group(GrpName, Cfg) -> Fun = fun(App) -> set_special_configs(GrpName, App) end, - emqx_ct_helpers:start_apps([emqx_modules]), emqx_ct_helpers:start_apps([emqx_auth_ldap], Fun), - emqx_mod_acl_internal:unload([]), Cfg. end_per_group(_GrpName, _Cfg) -> - emqx_ct_helpers:stop_apps([emqx_auth_ldap, emqx_modules]). + emqx_ct_helpers:stop_apps([emqx_auth_ldap]). %%-------------------------------------------------------------------- %% Cases diff --git a/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl index 2960c4621..6a5e7bf47 100644 --- a/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl @@ -36,12 +36,11 @@ all() -> check_acl]. init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_modules, emqx_auth_ldap], fun set_special_configs/1), - emqx_mod_acl_internal:unload([]), + emqx_ct_helpers:start_apps([emqx_auth_ldap], fun set_special_configs/1), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_auth_ldap, emqx_modules]). + emqx_ct_helpers:stop_apps([emqx_auth_ldap]). check_auth(_) -> MqttUser1 = #{clientid => <<"mqttuser1">>, @@ -62,7 +61,6 @@ check_auth(_) -> ?assertEqual({error, not_authorized}, emqx_access_control:authenticate(NonExistUser1)). check_acl(_) -> - % emqx_modules:load_module(emqx_mod_acl_internal, false), MqttUser = #{clientid => <<"mqttuser1">>, username => <<"user1">>, zone => external}, NoMqttUser = #{clientid => <<"mqttuser2">>, username => <<"user7">>, zone => external}, allow = emqx_access_control:check_acl(MqttUser, publish, <<"mqttuser0001/pub/1">>), diff --git a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl index 53f22783a..03e1fa33e 100644 --- a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl +++ b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl @@ -50,23 +50,18 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Cfg) -> - emqx_ct_helpers:start_apps([emqx_modules, emqx_auth_mongo], fun set_special_confs/1), - emqx_modules:load_module(emqx_mod_acl_internal, false), + emqx_ct_helpers:start_apps([emqx_auth_mongo], fun set_special_confs/1), init_mongo_data(), Cfg. end_per_suite(_Cfg) -> deinit_mongo_data(), - emqx_ct_helpers:stop_apps([emqx_auth_mongo, emqx_modules]). + emqx_ct_helpers:stop_apps([emqx_auth_mongo]). set_special_confs(emqx) -> application:set_env(emqx, acl_nomatch, deny), - application:set_env(emqx, acl_file, - emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/acl.conf")), application:set_env(emqx, allow_anonymous, false), - application:set_env(emqx, enable_acl_cache, false), - application:set_env(emqx, plugins_loaded_file, - emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins")); + application:set_env(emqx, enable_acl_cache, false); set_special_confs(_App) -> ok. @@ -133,7 +128,7 @@ t_check_acl(_) -> allow = emqx_access_control:check_acl(User2, subscribe, <<"$SYS/testuser/1">>), allow = emqx_access_control:check_acl(User3, publish, <<"a/b/c">>), deny = emqx_access_control:check_acl(User3, publish, <<"c">>), - allow = emqx_access_control:check_acl(User4, publish, <<"a/b/c">>). + deny = emqx_access_control:check_acl(User4, publish, <<"a/b/c">>). t_acl_super(_) -> reload({auth_query, [{password_hash, plain}, {password_field, [<<"password">>]}]}), diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl index 7929af929..2d59171b3 100644 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl @@ -70,7 +70,7 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_modules, emqx_auth_pgsql]), + emqx_ct_helpers:start_apps([emqx_auth_pgsql]), drop_acl(), drop_auth(), init_auth(), @@ -79,7 +79,7 @@ init_per_suite(Config) -> Config. end_per_suite(Config) -> - emqx_ct_helpers:stop_apps([emqx_auth_pgsql, emqx_modules]), + emqx_ct_helpers:stop_apps([emqx_auth_pgsql]), Config. set_special_configs() -> @@ -161,7 +161,6 @@ t_check_auth(_) -> {error, not_authorized} = emqx_access_control:authenticate(Bcrypt#{password => <<"password">>}). t_check_acl(_) -> - emqx_modules:load_module(emqx_mod_acl_internal, false), User1 = #{zone => external, peerhost => {127,0,0,1}, clientid => <<"c1">>, username => <<"u1">>}, User2 = #{zone => external, peerhost => {127,0,0,1}, clientid => <<"c2">>, username => <<"u2">>}, allow = emqx_access_control:check_acl(User1, subscribe, <<"t1">>), @@ -170,8 +169,8 @@ t_check_acl(_) -> User4 = #{zone => external, peerhost => {10,10,10,110}, clientid => <<"c1">>, username => <<"u1">>}, allow = emqx_access_control:check_acl(User3, subscribe, <<"t1">>), allow = emqx_access_control:check_acl(User3, subscribe, <<"t1">>), - allow = emqx_access_control:check_acl(User3, subscribe, <<"t2">>),%% nomatch -> ignore -> emqttd acl - allow = emqx_access_control:check_acl(User4, subscribe, <<"t1">>),%% nomatch -> ignore -> emqttd acl + deny = emqx_access_control:check_acl(User3, subscribe, <<"t2">>),%% nomatch -> ignore -> emqx acl + deny = emqx_access_control:check_acl(User4, subscribe, <<"t1">>),%% nomatch -> ignore -> emqx acl User5 = #{zone => external, peerhost => {127,0,0,1}, clientid => <<"c3">>, username => <<"u3">>}, allow = emqx_access_control:check_acl(User5, subscribe, <<"t1">>), allow = emqx_access_control:check_acl(User5, publish, <<"t1">>). diff --git a/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE.erl b/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE.erl index a20f0a2e9..c8551c2ad 100644 --- a/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE.erl +++ b/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE.erl @@ -49,22 +49,18 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Cfg) -> - emqx_ct_helpers:start_apps([emqx_modules, emqx_auth_redis], fun set_special_configs/1), + emqx_ct_helpers:start_apps([emqx_auth_redis], fun set_special_configs/1), init_redis_rows(), Cfg. end_per_suite(_Cfg) -> deinit_redis_rows(), - emqx_ct_helpers:stop_apps([emqx_auth_redis, emqx_modules]). + emqx_ct_helpers:stop_apps([emqx_auth_redis]). set_special_configs(emqx) -> application:set_env(emqx, allow_anonymous, false), application:set_env(emqx, acl_nomatch, deny), - application:set_env(emqx, acl_file, - emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/acl.conf")), - application:set_env(emqx, enable_acl_cache, false), - application:set_env(emqx, plugins_loaded_file, - emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins")); + application:set_env(emqx, enable_acl_cache, false); set_special_configs(_App) -> ok. @@ -72,7 +68,6 @@ init_redis_rows() -> %% Users [q(["HMSET", Key|FiledValue]) || {Key, FiledValue} <- ?INIT_AUTH], %% ACLs - emqx_modules:load_module(emqx_mod_acl_internal, false), Result = [q(["HSET", Key, Filed, Value]) || {Key, Filed, Value} <- ?INIT_ACL], ct:pal("redis init result: ~p~n", [Result]). @@ -136,7 +131,7 @@ t_check_acl(_) -> allow = emqx_access_control:check_acl(User2, subscribe, <<"topic2">>), allow = emqx_access_control:check_acl(User3, publish, <<"topic3">>), allow = emqx_access_control:check_acl(User3, subscribe, <<"topic3">>), - allow = emqx_access_control:check_acl(User4, publish, <<"a/b/c">>). + deny = emqx_access_control:check_acl(User4, publish, <<"a/b/c">>). t_acl_super(_) -> reload([{password_hash, plain}]), From 3e36c0857640683b1ca6a8d0e520f6fd377bfa87 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Sat, 20 Mar 2021 11:06:39 +0800 Subject: [PATCH 064/158] chore(CI): update workflows --- .github/workflows/build_slim_packages.yaml | 2 +- .github/workflows/elvis_lint.yaml | 7 ++++++- .github/workflows/run_gitlint.yaml | 14 ++++++++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index d1b00a2f1..2e29292ec 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -25,7 +25,7 @@ jobs: if make emqx-ee --dry-run > /dev/null 2>&1; then echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials git config --global credential.helper store - echo "${{ secrets.CI_GIT_TOKEN }}" >> source/scripts/git-token + echo "${{ secrets.CI_GIT_TOKEN }}" >> ./scripts/git-token make emqx-ee-pkg else make emqx-pkg diff --git a/.github/workflows/elvis_lint.yaml b/.github/workflows/elvis_lint.yaml index af824f034..1fdbeba87 100644 --- a/.github/workflows/elvis_lint.yaml +++ b/.github/workflows/elvis_lint.yaml @@ -6,6 +6,11 @@ jobs: build: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 + - name: Set git token + if: endsWith(github.repository, 'enterprise') + run: | + echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials + git config --global credential.helper store - run: | ./scripts/elvis-check.sh $GITHUB_BASE_REF diff --git a/.github/workflows/run_gitlint.yaml b/.github/workflows/run_gitlint.yaml index 9d5d72ab6..01b35461f 100644 --- a/.github/workflows/run_gitlint.yaml +++ b/.github/workflows/run_gitlint.yaml @@ -12,15 +12,25 @@ jobs: run: | sudo apt-get update sudo apt install gitlint + - name: Set auth header + if: endsWith(github.repository, 'enterprise') + run: | + echo 'AUTH_HEADER<> $GITHUB_ENV + echo "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV - name: Run gitlint + shell: bash run: | pr_number=$(echo $GITHUB_REF | awk 'BEGIN { FS = "/" } ; { print $3 }') - messages=$(curl "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${pr_number}/commits") + messages="$(curl --silent --show-error \ + --header "${{ env.AUTH_HEADER }}" \ + --header "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${pr_number}/commits")" len=$(echo $messages | jq length) result=true for i in $( seq 0 $(($len - 1)) ); do message=$(echo $messages | jq -r .[$i].commit.message) - echo commit message: $message + echo "commit message: $message" status=0 echo $message | gitlint -C ./.github/workflows/.gitlint || status=$? if [ $status -ne 0 ]; then From 7ea69f56bed62f7618e0b62dd9a79bd1ae4ca85f Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Sat, 20 Mar 2021 13:55:24 +0800 Subject: [PATCH 065/158] chore(CI): fix cts error --- .github/workflows/run_cts_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index 6ce4719bb..f5d561f0e 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -28,7 +28,7 @@ jobs: env: LDAP_TAG: ${{ matrix.ldap_tag }} run: | - docker-compose -f .ci/apps_tests/docker-compose.yaml build --no-cache + docker-compose -f .ci/compatibility_tests/docker-compose-ldap.yaml build --no-cache docker-compose -f .ci/compatibility_tests/docker-compose-ldap.yaml up -d - name: setup if: matrix.network_type == 'ipv4' From 45dfc8a2fa5b1ee1035d03f3b40846bca92a4251 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sat, 20 Mar 2021 10:57:56 +0100 Subject: [PATCH 066/158] chore(test): make app-ct run easier --- Makefile | 10 ++++++++++ README.md | 9 ++------- scripts/find-apps.sh | 18 ++++++++++++++++++ scripts/find-suites.sh | 13 +++++++++++++ 4 files changed, 43 insertions(+), 7 deletions(-) create mode 100755 scripts/find-apps.sh create mode 100755 scripts/find-suites.sh diff --git a/Makefile b/Makefile index 85fd7bf6d..38251043c 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,16 @@ proper: $(REBAR) ct: $(REBAR) @ENABLE_COVER_COMPILE=1 $(REBAR) ct --name 'test@127.0.0.1' -c -v +APPS=$(shell $(CURDIR)/scripts/find-apps.sh) + +## app/name-ct targets are intended for local tests hence cover is not enabled +.PHONY: $(APPS:%=%-ct) +define gen-app-ct-target +$1-ct: + $(REBAR) ct --name 'test@127.0.0.1' -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) +endef +$(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) + .PHONY: cover cover: $(REBAR) @ENABLE_COVER_COMPILE=1 $(REBAR) cover diff --git a/README.md b/README.md index af683f881..cf672b8bb 100644 --- a/README.md +++ b/README.md @@ -89,17 +89,12 @@ make eunit ct ### To run subset of the common tests -examples +Examples ```bash -./rebar3 ct --name 'test@127.0.0.1' -c -v --dir test,apps/emqx_sn,apps/emqx_coap -./rebar3 ct --name 'test@127.0.0.1' -c -v --dir apps/emqx_auth_mnesi --suite emqx_acl_mnesia_SUITE -./rebar3 ct --name 'test@127.0.0.1' -c -v --dir apps/emqx_auth_mnesi --suite emqx_acl_mnesia_SUITE --case t_rest_api +make apps/emqx_bridge_mqtt-ct ``` -NOTE: Do *NOT* use full (relative) path to SUITE files like this `--suite apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl`, -because it will lead to a full copy of `apps` dir into `_buid/test/lib/emqx`. - ### Dialyzer ##### To Analyze all the apps ``` diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh new file mode 100755 index 000000000..ebb06f1ac --- /dev/null +++ b/scripts/find-apps.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# ensure dir +cd -P -- "$(dirname -- "$0")/.." + +find_app() { + local appdir="$1" + find "${appdir}/" -mindepth 1 -maxdepth 1 -type d +} + +find_app 'apps' +if [ -f 'EMQX_ENTERPRISE' ]; then + find_app 'lib-ee' +else + find_app 'lib-ce' +fi diff --git a/scripts/find-suites.sh b/scripts/find-suites.sh new file mode 100755 index 000000000..66296b758 --- /dev/null +++ b/scripts/find-suites.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +## this script prints out all test/*_SUITE.erl files of a given app, +## file names are separated by comma for rebar3 ct's `--suite` option + +set -euo pipefail + +# ensure dir +cd -P -- "$(dirname -- "$0")/.." + +APPDIR="$1" + +find "${APPDIR}/test" -name "*_SUITE.erl" | tr -d '\r' | tr '\n' ',' From 21416bf859ca147cba806b9292432de5011bb7ff Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sat, 20 Mar 2021 12:32:58 +0100 Subject: [PATCH 067/158] chore(build): always include debug_info for CE --- rebar.config.erl | 50 ++++++++---------------------------------------- 1 file changed, 8 insertions(+), 42 deletions(-) diff --git a/rebar.config.erl b/rebar.config.erl index f1ed9fda8..c95246b44 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -104,61 +104,36 @@ test_deps() -> ]. common_compile_opts() -> - [ deterministic + [ debug_info % alwyas include debug_info + , deterministic , {compile_info, [{emqx_vsn, get_vsn()}]} | [{d, 'EMQX_ENTERPRISE'} || is_enterprise()] ]. -make_debug_info_key_fun() -> - case os:getenv("EMQX_COMPILE_SECRET_FILE") of - false -> false; - "" -> false; - Fn -> - io:format("===< using debug_info encryption key from file ~p!~n", [Fn]), - SecretText = get_compile_secret(Fn), - F = fun(init) -> ok; - (clear) -> ok; - ({debug_info, _Mode, _Module, _Filename}) -> SecretText - end, - beam_lib:crypto_key_fun(F), - F - end. - -prod_compile_opts(false) -> - prod_compile_opts(); -prod_compile_opts(KeyFun) -> - [{debug_info_key, KeyFun({debug_info, "", "", ""})} | prod_compile_opts()]. - prod_compile_opts() -> [ compressed , warnings_as_errors | common_compile_opts() ]. -test_compile_opts() -> - [ debug_info - | common_compile_opts() - ]. - profiles() -> Vsn = get_vsn(), - KeyFun = make_debug_info_key_fun(), - [ {'emqx', [ {erl_opts, prod_compile_opts(KeyFun)} + [ {'emqx', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, bin)} ]} - , {'emqx-pkg', [ {erl_opts, prod_compile_opts(KeyFun)} + , {'emqx-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, pkg)} ]} - , {'emqx-edge', [ {erl_opts, prod_compile_opts(KeyFun)} + , {'emqx-edge', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, bin)} ]} - , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts(KeyFun)} + , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, pkg)} ]} - , {check, [ {erl_opts, test_compile_opts()} + , {check, [ {erl_opts, common_compile_opts()} ]} , {test, [ {deps, test_deps()} - , {erl_opts, test_compile_opts() ++ erl_opts_i()} + , {erl_opts, common_compile_opts() ++ erl_opts_i()} , {extra_src_dirs, [{"test", [{recursive,true}]}]} ]} ] ++ ee_profiles(Vsn). @@ -485,15 +460,6 @@ list_dir(Dir) -> {ok, Names} = file:list_dir(Dir), [list_to_atom(Name) || Name <- Names, filelib:is_dir(filename:join([Dir, Name]))]. -get_compile_secret(SecretFile) -> - case file:read_file(SecretFile) of - {ok, Secret} -> - string:trim(binary_to_list(Secret)); - {error, Reason} -> - io:format("===< Failed to read debug_info encryption key file ~s: ~p~n", [SecretFile, Reason]), - exit(Reason) - end. - %% ==== Enterprise supports below ================================================================== ee_profiles(_Vsn) -> []. From 71a0901c9213593adba508f6af31bdb836a7be78 Mon Sep 17 00:00:00 2001 From: Karol Kaczmarek Date: Wed, 17 Mar 2021 19:52:04 +0100 Subject: [PATCH 068/158] feat(acl): CLI and REST handlers for removing acl cache --- apps/emqx_management/src/emqx_mgmt.erl | 16 ++++++ .../emqx_management/src/emqx_mgmt_api_acl.erl | 50 +++++++++++++++++++ apps/emqx_management/src/emqx_mgmt_cli.erl | 18 +++++++ 3 files changed, 84 insertions(+) create mode 100644 apps/emqx_management/src/emqx_mgmt_api_acl.erl diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index fb4ca47ed..1f3c58f35 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -43,8 +43,10 @@ , lookup_client/3 , kickout_client/1 , list_acl_cache/1 + , clean_acl_cache/0 , clean_acl_cache/1 , clean_acl_cache/2 + , clean_acl_cache_all/1 , set_ratelimit_policy/2 , set_quota_policy/2 ]). @@ -231,6 +233,13 @@ kickout_client(Node, ClientId) -> list_acl_cache(ClientId) -> call_client(ClientId, list_acl_cache). +clean_acl_cache() -> + Results = [clean_acl_cache_all(Node) || Node <- ekka_mnesia:running_nodes()], + case lists:any(fun(Item) -> Item =:= ok end, Results) of + true -> ok; + false -> lists:last(Results) + end. + clean_acl_cache(ClientId) -> Results = [clean_acl_cache(Node, ClientId) || Node <- ekka_mnesia:running_nodes()], case lists:any(fun(Item) -> Item =:= ok end, Results) of @@ -249,6 +258,13 @@ clean_acl_cache(Node, ClientId) when Node =:= node() -> clean_acl_cache(Node, ClientId) -> rpc_call(Node, clean_acl_cache, [Node, ClientId]). +clean_acl_cache_all(Node) when Node =:= node() -> + _ = emqx_acl_cache:drain_cache(), + ok; + +clean_acl_cache_all(Node) -> + rpc_call(Node, clean_acl_cache, []). + set_ratelimit_policy(ClientId, Policy) -> call_client(ClientId, {ratelimit, Policy}). diff --git a/apps/emqx_management/src/emqx_mgmt_api_acl.erl b/apps/emqx_management/src/emqx_mgmt_api_acl.erl new file mode 100644 index 000000000..02b3a2910 --- /dev/null +++ b/apps/emqx_management/src/emqx_mgmt_api_acl.erl @@ -0,0 +1,50 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mgmt_api_acl). + +-include("emqx_mgmt.hrl"). + +-import(minirest, [ return/0 + , return/1 + ]). +-rest_api(#{name => clean_acl_cache_all, + method => 'DELETE', + path => "/acl-cache/", + func => clean_all, + descr => "Clean acl cache on all nodes"}). + +-rest_api(#{name => clean_acl_cache_node, + method => 'DELETE', + path => "/:atom:node/acl-cache", + func => clean_node, + descr => "Clean acl cache on specific node"}). + +-export([ clean_all/2 + , clean_node/2 + ]). + +clean_all(_Bindings, _Params) -> + case emqx_mgmt:clean_acl_cache() of + ok -> return(); + {error, Reason} -> return({error, ?ERROR1, Reason}) + end. + +clean_node(#{node := Node}, _Params) -> + case emqx_mgmt:clean_acl_cache(Node) of + ok -> return(); + {error, Reason} -> return({error, ?ERROR1, Reason}) + end. diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 6cf69209d..a4e3dae6d 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -39,6 +39,7 @@ , log/1 , mgmt/1 , data/1 + , acl/1 ]). -define(PROC_INFOKEYS, [status, @@ -576,6 +577,23 @@ data(_) -> emqx_ctl:usage([{"data import ", "Import data from the specified file"}, {"data export", "Export data"}]). +%%-------------------------------------------------------------------- +%% @doc acl Command + +acl(["cache-clean", "node", SNode]) -> + emqx_mgmt:clean_acl_cache_all(ekka_node:parse_name(SNode)); + +acl(["cache-clean", "all"]) -> + emqx_mgmt:clean_acl_cache(); + +acl(["cache-clean", ClientId]) -> + emqx_mgmt:clean_acl_cache(ClientId); + +acl(_) -> + emqx_ctl:usage([{"cache-clean all", "Clears acl cache on all nodes"}, + {"cache-clean node ", "Clears acl cache on given node"}, + {"cache-clean ", "Clears acl cache for given client"}]). + %%-------------------------------------------------------------------- %% Dump ETS %%-------------------------------------------------------------------- From 87ce9d666fb40fb4709b8b5e37388e8079d36754 Mon Sep 17 00:00:00 2001 From: Karol Kaczmarek Date: Wed, 17 Mar 2021 21:27:05 +0100 Subject: [PATCH 069/158] feat(acl): working cache drain cli/rest handlers --- apps/emqx_management/src/emqx_mgmt.erl | 21 ++++++++--------- .../emqx_management/src/emqx_mgmt_api_acl.erl | 6 ++--- apps/emqx_management/src/emqx_mgmt_cli.erl | 23 ++++++++++++++----- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 1f3c58f35..3c5b5d73e 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -43,9 +43,9 @@ , lookup_client/3 , kickout_client/1 , list_acl_cache/1 - , clean_acl_cache/0 , clean_acl_cache/1 , clean_acl_cache/2 + , clean_acl_cache_all/0 , clean_acl_cache_all/1 , set_ratelimit_policy/2 , set_quota_policy/2 @@ -233,13 +233,6 @@ kickout_client(Node, ClientId) -> list_acl_cache(ClientId) -> call_client(ClientId, list_acl_cache). -clean_acl_cache() -> - Results = [clean_acl_cache_all(Node) || Node <- ekka_mnesia:running_nodes()], - case lists:any(fun(Item) -> Item =:= ok end, Results) of - true -> ok; - false -> lists:last(Results) - end. - clean_acl_cache(ClientId) -> Results = [clean_acl_cache(Node, ClientId) || Node <- ekka_mnesia:running_nodes()], case lists:any(fun(Item) -> Item =:= ok end, Results) of @@ -258,12 +251,18 @@ clean_acl_cache(Node, ClientId) when Node =:= node() -> clean_acl_cache(Node, ClientId) -> rpc_call(Node, clean_acl_cache, [Node, ClientId]). +clean_acl_cache_all() -> + Results = [clean_acl_cache_all(Node) || Node <- ekka_mnesia:running_nodes()], + case lists:any(fun(Item) -> Item =:= ok end, Results) of + true -> ok; + false -> lists:last(Results) + end. + clean_acl_cache_all(Node) when Node =:= node() -> - _ = emqx_acl_cache:drain_cache(), - ok; + emqx_acl_cache:drain_cache(); clean_acl_cache_all(Node) -> - rpc_call(Node, clean_acl_cache, []). + rpc_call(Node, clean_acl_cache_all, [Node]). set_ratelimit_policy(ClientId, Policy) -> call_client(ClientId, {ratelimit, Policy}). diff --git a/apps/emqx_management/src/emqx_mgmt_api_acl.erl b/apps/emqx_management/src/emqx_mgmt_api_acl.erl index 02b3a2910..47a85e691 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_acl.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_acl.erl @@ -23,7 +23,7 @@ ]). -rest_api(#{name => clean_acl_cache_all, method => 'DELETE', - path => "/acl-cache/", + path => "/acl-cache", func => clean_all, descr => "Clean acl cache on all nodes"}). @@ -38,13 +38,13 @@ ]). clean_all(_Bindings, _Params) -> - case emqx_mgmt:clean_acl_cache() of + case emqx_mgmt:clean_acl_cache_all() of ok -> return(); {error, Reason} -> return({error, ?ERROR1, Reason}) end. clean_node(#{node := Node}, _Params) -> - case emqx_mgmt:clean_acl_cache(Node) of + case emqx_mgmt:clean_acl_cache_all(Node) of ok -> return(); {error, Reason} -> return({error, ?ERROR1, Reason}) end. diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index a4e3dae6d..ee37e811b 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -580,19 +580,30 @@ data(_) -> %%-------------------------------------------------------------------- %% @doc acl Command -acl(["cache-clean", "node", SNode]) -> - emqx_mgmt:clean_acl_cache_all(ekka_node:parse_name(SNode)); +acl(["cache-clean", "node", Node]) -> + case emqx_mgmt:clean_acl_cache_all(erlang:list_to_atom(Node)) of + ok -> + emqx_ctl:print("The emqx acl cache removed on node ~s.~n", [Node]); + {error, Reason} -> + emqx_ctl:print("The emqx acl cache-clean on node ~s failed: ~s.~n", [Node, Reason]) + end; acl(["cache-clean", "all"]) -> - emqx_mgmt:clean_acl_cache(); + case emqx_mgmt:clean_acl_cache_all() of + ok -> + emqx_ctl:print("The emqx acl cache removed on all nodes.~n"); + {error, Reason} -> + emqx_ctl:print("The emqx acl cache-clean failed: ~s.~n", [Reason]) + end; acl(["cache-clean", ClientId]) -> emqx_mgmt:clean_acl_cache(ClientId); acl(_) -> - emqx_ctl:usage([{"cache-clean all", "Clears acl cache on all nodes"}, - {"cache-clean node ", "Clears acl cache on given node"}, - {"cache-clean ", "Clears acl cache for given client"}]). + emqx_ctl:usage([{"cache-clean all", "Clears acl cache on all nodes"}, + {"cache-clean node ", "Clears acl cache on given node"}, + {"cache-clean ", "Clears acl cache for given client"} + ]). %%-------------------------------------------------------------------- %% Dump ETS From 844a1ba0af366e63de0711b252c746c04d542e0a Mon Sep 17 00:00:00 2001 From: Karol Kaczmarek Date: Thu, 18 Mar 2021 20:45:47 +0100 Subject: [PATCH 070/158] feat(acl): cache drain cr fixes --- apps/emqx_management/src/emqx_mgmt.erl | 8 ++++---- apps/emqx_management/src/emqx_mgmt_api_acl.erl | 11 ++++------- apps/emqx_management/src/emqx_mgmt_cli.erl | 16 ++++++++-------- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 3c5b5d73e..7747dca8c 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -252,10 +252,10 @@ clean_acl_cache(Node, ClientId) -> rpc_call(Node, clean_acl_cache, [Node, ClientId]). clean_acl_cache_all() -> - Results = [clean_acl_cache_all(Node) || Node <- ekka_mnesia:running_nodes()], - case lists:any(fun(Item) -> Item =:= ok end, Results) of - true -> ok; - false -> lists:last(Results) + Results = [{Node, clean_acl_cache_all(Node)} || Node <- ekka_mnesia:running_nodes()], + case lists:filter(fun({_Node, Item}) -> Item =/= ok end, Results) of + [] -> ok; + BadNodes -> {error, BadNodes} end. clean_acl_cache_all(Node) when Node =:= node() -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_acl.erl b/apps/emqx_management/src/emqx_mgmt_api_acl.erl index 47a85e691..d6af2df58 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_acl.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_acl.erl @@ -18,9 +18,6 @@ -include("emqx_mgmt.hrl"). --import(minirest, [ return/0 - , return/1 - ]). -rest_api(#{name => clean_acl_cache_all, method => 'DELETE', path => "/acl-cache", @@ -39,12 +36,12 @@ clean_all(_Bindings, _Params) -> case emqx_mgmt:clean_acl_cache_all() of - ok -> return(); - {error, Reason} -> return({error, ?ERROR1, Reason}) + ok -> minirest:return(); + {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) end. clean_node(#{node := Node}, _Params) -> case emqx_mgmt:clean_acl_cache_all(Node) of - ok -> return(); - {error, Reason} -> return({error, ?ERROR1, Reason}) + ok -> minirest:return(); + {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) end. diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index ee37e811b..e4f443cee 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -581,28 +581,28 @@ data(_) -> %% @doc acl Command acl(["cache-clean", "node", Node]) -> - case emqx_mgmt:clean_acl_cache_all(erlang:list_to_atom(Node)) of + case emqx_mgmt:clean_acl_cache_all(erlang:list_to_existing_atom(Node)) of ok -> - emqx_ctl:print("The emqx acl cache removed on node ~s.~n", [Node]); + emqx_ctl:print("ACL cache drain started on node ~s.~n", [Node]); {error, Reason} -> - emqx_ctl:print("The emqx acl cache-clean on node ~s failed: ~s.~n", [Node, Reason]) + emqx_ctl:print("ACL drain failed on node ~s: ~0p.~n", [Node, Reason]) end; acl(["cache-clean", "all"]) -> case emqx_mgmt:clean_acl_cache_all() of ok -> - emqx_ctl:print("The emqx acl cache removed on all nodes.~n"); + emqx_ctl:print("Started ACL cache drain in all nodes~n"); {error, Reason} -> - emqx_ctl:print("The emqx acl cache-clean failed: ~s.~n", [Reason]) + emqx_ctl:print("ACL cache-clean failed: ~p.~n", [Reason]) end; acl(["cache-clean", ClientId]) -> emqx_mgmt:clean_acl_cache(ClientId); acl(_) -> - emqx_ctl:usage([{"cache-clean all", "Clears acl cache on all nodes"}, - {"cache-clean node ", "Clears acl cache on given node"}, - {"cache-clean ", "Clears acl cache for given client"} + emqx_ctl:usage([{"acl cache-clean all", "Clears acl cache on all nodes"}, + {"acl cache-clean node ", "Clears acl cache on given node"}, + {"acl cache-clean ", "Clears acl cache for given client"} ]). %%-------------------------------------------------------------------- From ac7d097f16098ed0476ea2a62944e06b78112558 Mon Sep 17 00:00:00 2001 From: Karol Kaczmarek Date: Fri, 19 Mar 2021 22:05:39 +0100 Subject: [PATCH 071/158] chore(deps): minirest 0.3.5 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index ff64e9eec..454182185 100644 --- a/rebar.config +++ b/rebar.config @@ -43,7 +43,7 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.0"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.0"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.1.0"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.3"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.5"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.0"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} From 609a8566c7ffb12d8ea6b7381af7b28096904151 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Sat, 20 Mar 2021 23:20:36 +0800 Subject: [PATCH 072/158] chore(CI): update build workflws target event --- .github/workflows/build_packages.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index fd9c5a198..3d0afde1f 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -6,9 +6,11 @@ on: push: tags: - v* + - e* release: types: - published + workflow_dispatch: jobs: prepare: From eb246cf11bf51674d57ec1dc74ec8d3e8d7dc480 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sat, 20 Mar 2021 16:01:41 +0100 Subject: [PATCH 073/158] chore(build): pin port_compiler v1.11.1 Port compiler bug in v1.11.0: In windows builds, cl.exe produces .obj files link.exe looks for .o files to link. --- rebar.config.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rebar.config.erl b/rebar.config.erl index c95246b44..1fe655d21 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -85,6 +85,9 @@ project_app_dirs() -> plugins(HasElixir) -> [ {relup_helper,{git,"https://github.com/emqx/relup_helper", {tag, "2.0.0"}}} , {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}} + %% emqx main project does not require port-compiler + %% pin at root level for deterministic + , {pc, {git, "https://github.com/emqx/port_compiler.git", {tag, "v1.11.1"}}} | [ rebar_mix || HasElixir ] ] %% test plugins are concatenated to default profile plugins From aa802d129c44cc5e222229e063e504aec085fac5 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sat, 20 Mar 2021 20:47:48 +0100 Subject: [PATCH 074/158] chore(ci): Delete bcrypt in windows The dependency is downloaded in ubuntu, but copied to windows so we need a explicit deletion --- .github/workflows/build_packages.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 3d0afde1f..605a403d9 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -77,7 +77,7 @@ jobs: - name: build env: PYTHON: python - DIAGNOSTIC: 1 + DIAGNOSTIC: 1 run: | $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH" @@ -89,8 +89,10 @@ jobs: else { $pkg_name = "${{ matrix.profile }}-windows-$($version -replace '/').zip" } - cd source + ## We do not build/release bcrypt for windows package + Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ + Remove-Item -Force -Path rebar.lock make ${{ matrix.profile }} mkdir -p _packages/${{ matrix.profile }} Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name From b6894ae4c2f656587f7f7f53a52008abf75ba0bc Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sun, 21 Mar 2021 06:35:31 +0100 Subject: [PATCH 075/158] chore(build): remove debug flag --- .github/workflows/build_packages.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 605a403d9..8a7f77498 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -77,7 +77,6 @@ jobs: - name: build env: PYTHON: python - DIAGNOSTIC: 1 run: | $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH" From 817abd7e937d2f203c25b34888adcf3790c708a1 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 22 Mar 2021 21:01:41 +0800 Subject: [PATCH 076/158] fix(lwm2m): export functions for starting and stopping listeners --- apps/emqx_lwm2m/src/emqx_lwm2m_coap_server.erl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_coap_server.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_coap_server.erl index 31986da54..12755dcd4 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_coap_server.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_coap_server.erl @@ -22,6 +22,12 @@ , stop/1 ]). +-export([ start_listener/1 + , start_listener/3 + , stop_listener/1 + , stop_listener/2 + ]). + -define(LOG(Level, Format, Args), logger:Level("LwM2M: " ++ Format, Args)). From fd1b1936ad2d2fa01e08ebf98961c2647bb713a2 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 22 Mar 2021 14:09:38 +0100 Subject: [PATCH 077/158] docs(README): Update README to include plugin dev --- README.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index cf672b8bb..aba873046 100644 --- a/README.md +++ b/README.md @@ -106,19 +106,25 @@ make dialyzer DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer ``` -## FAQ +## Community + +### FAQ Visiting [EMQ X FAQ](https://docs.emqx.io/en/broker/latest/faq/faq.html) to get help of common problems. -## Roadmap -The [EMQ X Roadmap uses Github milestones](https://github.com/emqx/emqx/milestones) to track the progress of the project. +### Questions -## Community +[GitHub Discussions](https://github.com/emqx/emqx/discussions) is where you can ask questions, and share ideas. -The EMQ X community can be found on [GitHub Discussions](https://github.com/emqx/emqx/discussions), where you can ask questions, voice ideas, and share your projects. +### Proposals + +For more organised improvement proposals, you can send pull requests to [EIP](https://github.com/emqx/eip). + +### Plugin development + +To develop your own plugins, see [lib-extra/README.md](./lib-extra/README.md) -To chat with other community members you can join the [EMQ X Slack](https://slack-invite.emqx.io). ## MQTT Specifications From 507a6ee1c03f4eba33f9752ae9d209a1eb2eac9b Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 22 Mar 2021 14:34:29 +0100 Subject: [PATCH 078/158] docs(lib-extra/README): Add more details on how to run the code --- lib-extra/README.md | 62 +++++++++++++++++++++++++++++++------------- scripts/find-apps.sh | 5 ++++ 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/lib-extra/README.md b/lib-extra/README.md index 3196f4b8c..5247029ab 100644 --- a/lib-extra/README.md +++ b/lib-extra/README.md @@ -6,11 +6,11 @@ external plugins from open-source community. The (maybe broken) symlinks are keept to help testing plugins in this umbrella project. -## How to build `plugin_foo` +## Add a plugin to the project Add `plugin_foo` as a rebar3 dependency in `plugins` file. -e.g. +e.g. For an Erlang plugin named `plugin_foo`: ``` {erlang_plugins, @@ -19,32 +19,58 @@ e.g. }. ``` -Exeucte command +## Build a release ``` -export EMQX_EXTRA_PLUGINS='plugin_foo' -make +$ make ``` -The plugin source code should downloaded to `_build/default/lib/plugin_foo` +If all goes as expected, there should be two directories in the release: -NOTE: Shallow clone with depth=1 is used for git dependencies. +``` +_build/emqx/rel/emqx/lib/plugin_foo-<..vsn..>/ +``` -## How to test `plugin_foo` +## Run your code -If the source code in `_build` is already symlinked from `lib-extra/`, -you may directlly run tests with commands below. +Start the node (interactive mode) + +``` +./_build/emqx/rel/emqx/bin/emqx console +``` + +Load the plugin with command: + +``` +./_build/emqx/rel/emqx/bin/emqx_ctl plugins load plugin_foo +``` + +## Test the plugin + +If the plugin is added as a rebar dependency, it should be cloned +to `_build/default/lib/plugin_foo`. + +Before you can test it, you need to make sure there is a symlink +in `lib-extra` pointing to the clone. For instance, the `emqx_plugin_template` +plugin is linked like this + +`emqx_plugin_template -> ../_build/default/lib/emqx_plugin_template/` + +To run its teste cases: ```bash ./rebar3 eunit --dir lib-extra/plugin_foo -./rebar3 ct --dir lib-extra/plugin_foo +mkae lib-extra/plugin_foo-ct ``` -In case the plugin is being actively developed -it can be cloned to `lib-extra`, e.g. `lib-extra/plugin-bar-dev` -then it can be tested with commands below: +NOTE: we should `depth=1` shallow clone into `_build/` directory, +for plugins being actively developed, you can place the clone in `lib-extra/` -```bash -./rebar3 eunit --dir lib-extra/plugin-bar-dev -./rebar3 ct --dir lib-extra/plugin-bar-dev -``` +## Caveats + +* Elixir dependency in Erlang is not quite nicely supported as incremental builds, + meaning you will not be able to edit the code in this project and get recompiled + in the next `make` command. + +* To have the plugin enabled/loaded by default, you can include it in the template + `data/loaded_plugins.tmpl` diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index ebb06f1ac..7a7da4964 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -16,3 +16,8 @@ if [ -f 'EMQX_ENTERPRISE' ]; then else find_app 'lib-ce' fi + +## find directories in lib-extra +find_app 'lib-extra' +## find symlinks in lib-extra +find 'lib-extra/' -mindepth 1 -maxdepth 1 -type l From 8a41a4ee0097cd434bc117e0dfeb528d1af06aa0 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 22 Mar 2021 16:27:05 +0100 Subject: [PATCH 079/158] fix(ssl): Fix dir name for uploaded certificates The dir is "rules" not "rule" in the original code ref: https://github.com/emqx/emqx-rule-actions/blob/f1eba1c648af950f9cbb3765c28d4b9f16086857/src/emqx_rule_actions_utils.erl --- apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl | 2 +- apps/emqx_web_hook/src/emqx_web_hook_actions.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl index cbd7f28ed..1752de8e2 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl @@ -567,7 +567,7 @@ options(Options, PoolName, ResId) -> maybe_ssl(_Options, false, _ResId) -> []; maybe_ssl(Options, true, ResId) -> - Dir = filename:join([emqx:get_env(data_dir), "rule", ResId]), + Dir = filename:join([emqx:get_env(data_dir), "rules", ResId]), [{ssl, true}, {ssl_opts, emqx_plugin_libs_ssl:save_files_return_opts(Options, Dir)}]. mqtt_ver(ProtoVer) -> diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index 4e63b54df..2d7fe8801 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -346,7 +346,7 @@ pool_name(ResId) -> list_to_atom("webhook:" ++ str(ResId)). get_ssl_opts(Opts, ResId) -> - Dir = filename:join([emqx:get_env(data_dir), "rule", ResId]), + Dir = filename:join([emqx:get_env(data_dir), "rules", ResId]), [{ssl, true}, {ssl_opts, emqx_plugin_libs_ssl:save_files_return_opts(Opts, Dir)}]. parse_host(Host) -> From 71559dd001604923b1c6c18ba16a28b844162d59 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 22 Mar 2021 16:33:57 +0100 Subject: [PATCH 080/158] refactor(ssl): Move data dir for ssl certs upload down to lib module --- .../src/emqx_bridge_mqtt_actions.erl | 3 +-- apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl | 12 +++++++++++- apps/emqx_web_hook/src/emqx_web_hook_actions.erl | 3 +-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl index 1752de8e2..8f5ad16ef 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl @@ -567,8 +567,7 @@ options(Options, PoolName, ResId) -> maybe_ssl(_Options, false, _ResId) -> []; maybe_ssl(Options, true, ResId) -> - Dir = filename:join([emqx:get_env(data_dir), "rules", ResId]), - [{ssl, true}, {ssl_opts, emqx_plugin_libs_ssl:save_files_return_opts(Options, Dir)}]. + [{ssl, true}, {ssl_opts, emqx_plugin_libs_ssl:save_files_return_opts(Options, "rules", ResId)}]. mqtt_ver(ProtoVer) -> case ProtoVer of diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl index 4b0746335..d0b561430 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl @@ -16,7 +16,9 @@ -module(emqx_plugin_libs_ssl). --export([save_files_return_opts/2]). +-export([save_files_return_opts/2, + save_files_return_opts/3 + ]). -type file_input_key() :: binary(). %% <<"file">> | <<"filename">> -type file_input() :: #{file_input_key() => binary()}. @@ -36,6 +38,14 @@ -type opt_value() :: term(). -type opts() :: [{opt_key(), opt_value()}]. +%% @doc Parse ssl options input. +%% If the input contains file content, save the files in the given dir. +%% Returns ssl options for Erlang's ssl application. +-spec save_files_return_opts(opts_input(), string(), string() | binary()) -> opts(). +save_files_return_opts(Options, SubDir, ResId) -> + Dir = filename:join([emqx:get_env(data_dir), SubDir, ResId]), + save_files_return_opts(Options, Dir). + %% @doc Parse ssl options input. %% If the input contains file content, save the files in the given dir. %% Returns ssl options for Erlang's ssl application. diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index 2d7fe8801..b670b44b8 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -346,8 +346,7 @@ pool_name(ResId) -> list_to_atom("webhook:" ++ str(ResId)). get_ssl_opts(Opts, ResId) -> - Dir = filename:join([emqx:get_env(data_dir), "rules", ResId]), - [{ssl, true}, {ssl_opts, emqx_plugin_libs_ssl:save_files_return_opts(Opts, Dir)}]. + [{ssl, true}, {ssl_opts, emqx_plugin_libs_ssl:save_files_return_opts(Opts, "rules", ResId)}]. parse_host(Host) -> case inet:parse_address(Host) of From 8ecce74d3301c75afc611cf145b547def27d7c19 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 22 Mar 2021 19:09:39 +0100 Subject: [PATCH 081/158] docs(lib-extra/README): Add EMQX_EXTRA_PLUGINS --- lib-extra/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/lib-extra/README.md b/lib-extra/README.md index 5247029ab..a2fae3a17 100644 --- a/lib-extra/README.md +++ b/lib-extra/README.md @@ -22,6 +22,7 @@ e.g. For an Erlang plugin named `plugin_foo`: ## Build a release ``` +$ export EMQX_EXTRA_PLUGINS=plugin_foo,plugin_bar $ make ``` From dc510fb5724304af0ad543ab80461dc3cb0f31e7 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 22 Mar 2021 20:58:49 +0100 Subject: [PATCH 082/158] chore(build): Upgrade to elvis 1.0.0-emqx-2 This version ensures spaces around '|' and '||' --- scripts/elvis-check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/elvis-check.sh b/scripts/elvis-check.sh index e77e6a785..f15b59b1e 100755 --- a/scripts/elvis-check.sh +++ b/scripts/elvis-check.sh @@ -5,7 +5,7 @@ set -euo pipefail -ELVIS_VERSION='1.0.0-emqx-1' +ELVIS_VERSION='1.0.0-emqx-2' base="${1:-}" if [ "${base}" = "" ]; then From 5b126d60ad947f220917f1b826c8f08db7fef4df Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 22 Mar 2021 21:05:15 +0100 Subject: [PATCH 083/158] style(elvis): force spaces around | and || --- apps/emqx_exhook/src/emqx_exhook_cli.erl | 5 +++-- elvis.config | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook_cli.erl b/apps/emqx_exhook/src/emqx_exhook_cli.erl index 8bab9ced5..9fea1f50d 100644 --- a/apps/emqx_exhook/src/emqx_exhook_cli.erl +++ b/apps/emqx_exhook/src/emqx_exhook_cli.erl @@ -23,7 +23,8 @@ cli(["server", "list"]) -> if_enabled(fun() -> Services = emqx_exhook:list(), - [emqx_ctl:print("HookServer(~s)~n", [emqx_exhook_server:format(Service)]) || Service <- Services] + [emqx_ctl:print("HookServer(~s)~n", + [emqx_exhook_server:format(Service)]) || Service <- Services] end); cli(["server", "enable", Name0]) -> @@ -74,7 +75,7 @@ hint() -> stats() -> lists:usort(lists:foldr(fun({K, N}, Acc) -> case atom_to_list(K) of - "exhook." ++ Key -> [{Key, N}|Acc]; + "exhook." ++ Key -> [{Key, N} | Acc]; _ -> Acc end end, [], emqx_metrics:all())). diff --git a/elvis.config b/elvis.config index bed4ba2e8..ae47fc8f9 100644 --- a/elvis.config +++ b/elvis.config @@ -5,7 +5,7 @@ [ {config, [ - #{dirs => ["src", "apps/**/src", "lib-ce/**/src"], + #{dirs => ["src", "apps/**/src", "lib-ce/**/src", "lib-ee/**/src"], filter => "*.erl", ruleset => erl_files, rules => [ @@ -13,7 +13,11 @@ {elvis_style, no_common_caveats_call, #{}}, {elvis_style, no_debug_call, #{ debug_functions => [ {ct, pal} , {ct, print} - ]}} + ]}}, + {elvis_style, operator_spaces, #{rules => [{right, "|"}, + {left, "|"}, + {right, "||"}, + {left, "||"}]}} ] }, #{dirs => ["test", "apps/**/test", "lib-ce/**/src"], From dcfde3861fc37a9e197c3c778a6bbd2da5c9bc00 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 22 Mar 2021 20:30:42 +0100 Subject: [PATCH 084/158] feat(ssl): Export a lib function to save key/cert file --- .../src/emqx_plugin_libs_ssl.erl | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl index d0b561430..35ed79fd8 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl @@ -17,7 +17,8 @@ -module(emqx_plugin_libs_ssl). -export([save_files_return_opts/2, - save_files_return_opts/3 + save_files_return_opts/3, + save_file/2 ]). -type file_input_key() :: binary(). %% <<"file">> | <<"filename">> @@ -41,7 +42,8 @@ %% @doc Parse ssl options input. %% If the input contains file content, save the files in the given dir. %% Returns ssl options for Erlang's ssl application. --spec save_files_return_opts(opts_input(), string(), string() | binary()) -> opts(). +-spec save_files_return_opts(opts_input(), atom() | string() | binary(), + string() | binary()) -> opts(). save_files_return_opts(Options, SubDir, ResId) -> Dir = filename:join([emqx:get_env(data_dir), SubDir, ResId]), save_files_return_opts(Options, Dir). @@ -56,9 +58,9 @@ save_files_return_opts(Options, Dir) -> KeyFile = Get(<<"keyfile">>), CertFile = Get(<<"certfile">>), CAFile = GetD(<<"cacertfile">>, Get(<<"cafile">>)), - Key = save_file(KeyFile, Dir), - Cert = save_file(CertFile, Dir), - CA = save_file(CAFile, Dir), + Key = do_save_file(KeyFile, Dir), + Cert = do_save_file(CertFile, Dir), + CA = do_save_file(CAFile, Dir), Verify = case GetD(<<"verify">>, false) of false -> verify_none; _ -> verify_peer @@ -68,22 +70,30 @@ save_files_return_opts(Options, Dir) -> filter([{keyfile, Key}, {certfile, Cert}, {cacertfile, CA}, {verify, Verify}, {versions, Versions}, {ciphers, Ciphers}]). +%% @doc Save a key or certificate file in data dir, +%% and return path of the saved file. +%% empty string is returned if the input is empty. +-spec save_file(file_input(), atom() | string() | binary()) -> string(). +save_file(Param, SubDir) -> + Dir = filename:join([emqx:get_env(data_dir), SubDir]), + do_save_file( Param, Dir). + filter([]) -> []; filter([{_, ""} | T]) -> filter(T); filter([H | T]) -> [H | filter(T)]. -save_file(#{<<"filename">> := FileName, <<"file">> := Content}, Dir) +do_save_file(#{<<"filename">> := FileName, <<"file">> := Content}, Dir) when FileName =/= undefined andalso Content =/= undefined -> - save_file(ensure_str(FileName), iolist_to_binary(Content), Dir); -save_file(FilePath, _) when is_binary(FilePath) -> + do_save_file(ensure_str(FileName), iolist_to_binary(Content), Dir); +do_save_file(FilePath, _) when is_binary(FilePath) -> ensure_str(FilePath); -save_file(FilePath, _) when is_list(FilePath) -> +do_save_file(FilePath, _) when is_list(FilePath) -> FilePath; -save_file(_, _) -> "". +do_save_file(_, _) -> "". -save_file("", _, _Dir) -> ""; %% ignore -save_file(_, <<>>, _Dir) -> ""; %% ignore -save_file(FileName, Content, Dir) -> +do_save_file("", _, _Dir) -> ""; %% ignore +do_save_file(_, <<>>, _Dir) -> ""; %% ignore +do_save_file(FileName, Content, Dir) -> FullFilename = filename:join([Dir, FileName]), ok = filelib:ensure_dir(FullFilename), case file:write_file(FullFilename, Content) of From 749f8ff156638ba5f24b7ccaf229ff4b669aee04 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 22 Mar 2021 22:34:58 +0100 Subject: [PATCH 085/158] chore(build): exclude broken symlinks in find-apps.sh --- scripts/find-apps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index 7a7da4964..82743dd20 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -20,4 +20,4 @@ fi ## find directories in lib-extra find_app 'lib-extra' ## find symlinks in lib-extra -find 'lib-extra/' -mindepth 1 -maxdepth 1 -type l +find 'lib-extra/' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print From 5ea897f0f66a1baa0da2aa07ae39723b378667cf Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Sat, 20 Mar 2021 10:24:53 +0000 Subject: [PATCH 086/158] chore(CI): check client exited --- .github/workflows/run_test_cases.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 03f0119ff..601c885e3 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -73,6 +73,10 @@ jobs: docker-compose -f .ci/apps_tests/docker-compose.yaml -f .ci/apps_tests/docker-compose.enterprise.yaml build --no-cache docker-compose -f .ci/apps_tests/docker-compose.yaml -f .ci/apps_tests/docker-compose.enterprise.yaml up -d docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" + while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \ + != $(docker ps -a --filter name=client | wc -l) ]; do + sleep 5 + done - name: run eunit run: | docker exec -i erlang bash -c "make eunit" From 277cab86d3f0efbf66b000ecdb4fca8ad278f6b8 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 23 Mar 2021 17:13:01 +0800 Subject: [PATCH 087/158] refactor: improve the speed of hexstr and binary conversions --- apps/emqx_exhook/src/emqx_exhook_handler.erl | 8 +------- .../emqx_exproto/src/emqx_exproto_channel.erl | 5 +---- .../emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl | 13 +------------ apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 15 +++------------ src/emqx_guid.erl | 8 ++++---- src/emqx_misc.erl | 19 +++++++++++++++++++ src/emqx_tracer.erl | 7 +------ 7 files changed, 30 insertions(+), 45 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index e60eadaa7..e03e23813 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -273,7 +273,7 @@ clientinfo(ClientInfo = message(#message{id = Id, qos = Qos, from = From, topic = Topic, payload = Payload, timestamp = Ts}) -> #{node => stringfy(node()), - id => hexstr(Id), + id => emqx_guid:to_hexstr(Id), qos => Qos, from => stringfy(From), topic => Topic, @@ -304,12 +304,6 @@ stringfy(Term) when is_atom(Term) -> stringfy(Term) -> unicode:characters_to_binary((io_lib:format("~0p", [Term]))). -hexstr(B) -> - << <<(hexchar(H)), (hexchar(L))>> || <> <= B>>. - -hexchar(I) when I >= 0 andalso I < 10 -> I + $0; -hexchar(I) -> I - 10 + $A. - %%-------------------------------------------------------------------- %% Acc funcs diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl index 0eec36410..2966cbba9 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -205,7 +205,7 @@ handle_deliver(Delivers, Channel = #channel{clientinfo = ClientInfo}) -> [ClientInfo], Msg), NMsg = emqx_mountpoint:unmount(Mountpoint, Msg1), #{node => NodeStr, - id => hexstr(emqx_message:id(NMsg)), + id => emqx_guid:to_hexstr(emqx_message:id(NMsg)), qos => emqx_message:qos(NMsg), from => fmt_from(emqx_message:from(NMsg)), topic => emqx_message:topic(NMsg), @@ -591,9 +591,6 @@ default_clientinfo(#{peername := {PeerHost, _}, stringfy(Reason) -> unicode:characters_to_binary((io_lib:format("~0p", [Reason]))). -hexstr(Bin) -> - [io_lib:format("~2.16.0B",[X]) || <> <= Bin]. - fmt_from(undefined) -> <<>>; fmt_from(Bin) when is_binary(Bin) -> Bin; fmt_from(T) -> stringfy(T). diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl index d542acf91..cd22797fa 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl @@ -303,18 +303,7 @@ bin(Float) when is_float(Float) -> float_to_binary(Float). decoding(Datas, <<"hex">>) -> lists:map(fun(Data = #{<<"value">> := Value}) -> - Data#{<<"value">> => hexstr_to_bin(binary_to_list(Value))} + Data#{<<"value">> => emqx_misc:hexstr2bin(Value)} end, Datas); decoding(Datas, _) -> Datas. - -hexstr_to_bin(S) -> - hexstr_to_bin(S, []). -hexstr_to_bin([], Acc) -> - list_to_binary(lists:reverse(Acc)); -hexstr_to_bin([X,Y|T], Acc) -> - {ok, [V], []} = io_lib:fread("~16u", [X,Y]), - hexstr_to_bin(T, [V | Acc]); -hexstr_to_bin([X|T], Acc) -> - {ok, [V], []} = io_lib:fread("~16u", lists:flatten([X,"0"])), - hexstr_to_bin(T, [V | Acc]). \ No newline at end of file diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 55917f751..3e41b2520 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -512,12 +512,10 @@ map(Data) -> emqx_rule_utils:map(Data). bin2hexstr(Bin) when is_binary(Bin) -> - IntL = binary_to_list(Bin), - list_to_binary([io_lib:format("~2.16.0B", [Int]) || Int <- IntL]). + emqx_misc:bin2hexstr(Bin). hexstr2bin(Str) when is_binary(Str) -> - list_to_binary([binary_to_integer(W, 16) || <> <= Str]). - + emqx_misc:hexstr2bin(Str). %%------------------------------------------------------------------------------ %% NULL Funcs @@ -776,14 +774,7 @@ sha256(S) when is_binary(S) -> hash(sha256, S). hash(Type, Data) -> - hexstring(crypto:hash(Type, Data)). - -hexstring(<>) -> - iolist_to_binary(io_lib:format("~32.16.0b", [X])); -hexstring(<>) -> - iolist_to_binary(io_lib:format("~40.16.0b", [X])); -hexstring(<>) -> - iolist_to_binary(io_lib:format("~64.16.0b", [X])). + emqx_misc:bin2hexstr(crypto:hash(Type, Data)). %%------------------------------------------------------------------------------ %% Data encode and decode Funcs diff --git a/src/emqx_guid.erl b/src/emqx_guid.erl index 79e445d73..7a7657ad1 100644 --- a/src/emqx_guid.erl +++ b/src/emqx_guid.erl @@ -136,11 +136,11 @@ npid() -> PidByte3:8, PidByte4:8>>, NPid. -to_hexstr(<>) -> - list_to_binary(integer_to_list(I, 16)). +to_hexstr(I) when byte_size(I) =:= 16 -> + emqx_misc:bin2hexstr(I). -from_hexstr(S) -> - I = list_to_integer(binary_to_list(S), 16), <>. +from_hexstr(S) when byte_size(S) =:= 32 -> + emqx_misc:hexstr2bin(S). to_base62(<>) -> emqx_base62:encode(I). diff --git a/src/emqx_misc.erl b/src/emqx_misc.erl index 88e3c91a6..a96261f74 100644 --- a/src/emqx_misc.erl +++ b/src/emqx_misc.erl @@ -45,6 +45,10 @@ , index_of/2 ]). +-export([ bin2hexstr/1 + , hexstr2bin/1 + ]). + %% @doc Merge options -spec(merge_opts(Opts, Opts) -> Opts when Opts :: proplists:proplist()). merge_opts(Defaults, Options) -> @@ -233,3 +237,18 @@ index_of(E, I, [E|_]) -> index_of(E, I, [_|L]) -> index_of(E, I+1, L). +-spec(bin2hexstr(binary()) -> binary()). +bin2hexstr(B) when is_binary(B) -> + << <<(int2hexchar(H)), (int2hexchar(L))>> || <> <= B>>. + +int2hexchar(I) when I >= 0 andalso I < 10 -> I + $0; +int2hexchar(I) -> I - 10 + $A. + +-spec(hexstr2bin(binary()) -> binary()). +hexstr2bin(B) when is_binary(B) -> + << <<(hexchar2int(H)*16 + hexchar2int(L))>> || <> <= B>>. + +hexchar2int(I) when I >= $0 andalso I =< $9 -> I - $0; +hexchar2int(I) when I >= $A andalso I =< $F -> I - $A + 10; +hexchar2int(I) when I >= $a andalso I =< $f -> I - $a + 10. + diff --git a/src/emqx_tracer.erl b/src/emqx_tracer.erl index ea9247531..871c3dc81 100644 --- a/src/emqx_tracer.erl +++ b/src/emqx_tracer.erl @@ -161,9 +161,4 @@ handler_name(Bin) -> end. hashstr(Bin) -> - hexstr(crypto:hash(sha, Bin)). - -hexstr(Bin) -> - lists:flatten( - [io_lib:format("~2.16.0B", [Int]) - || Int <- binary_to_list(Bin)]). + binary_to_list(emqx_misc:bin2hexstr(Bin)). From f08ddec296bb2750dc3099041ac024d131f62c48 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 23 Mar 2021 19:48:04 +0800 Subject: [PATCH 088/158] Add server_name_indication and verify option (#4349) Add server_name_indication and verify option --- .ci/compatibility_tests/pgsql/Dockerfile | 6 ++--- .github/workflows/run_cts_tests.yaml | 18 ++++++++++--- apps/emqx_auth_http/etc/emqx_auth_http.conf | 25 +++++++++++++---- .../emqx_auth_http/priv/emqx_auth_http.schema | 9 +++++++ .../emqx_auth_http/src/emqx_auth_http_app.erl | 21 +++++++++++---- .../test/emqx_auth_http_SUITE.erl | 4 ++- apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf | 2 -- .../emqx_auth_ldap/priv/emqx_auth_ldap.schema | 10 +++---- apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf | 15 +++++++++++ .../priv/emqx_auth_mongo.schema | 23 ++++++++++++++-- apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf | 15 +++++++++++ .../priv/emqx_auth_mysql.schema | 23 ++++++++++++++-- apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf | 15 +++++++++++ .../priv/emqx_auth_pgsql.schema | 18 +++++++++++++ .../emqx_auth_pgsql_SUITE_data/ca-key.pem | 27 +++++++++++++++++++ .../test/emqx_auth_pgsql_SUITE_data/ca.pem | 19 +++++++++++++ .../client-cert.pem | 19 +++++++++++++ .../emqx_auth_pgsql_SUITE_data/client-key.pem | 27 +++++++++++++++++++ .../emqx_auth_pgsql_SUITE_data/postgresql.crt | 21 --------------- .../emqx_auth_pgsql_SUITE_data/postgresql.csr | 17 ------------ .../emqx_auth_pgsql_SUITE_data/postgresql.key | 27 ------------------- .../private_key.pem | 27 +++++++++++++++++++ .../emqx_auth_pgsql_SUITE_data/public_key.pem | 9 +++++++ .../test/emqx_auth_pgsql_SUITE_data/root.crt | 21 --------------- .../test/emqx_auth_pgsql_SUITE_data/root.srl | 1 - .../server-cert.pem | 19 +++++++++++++ .../emqx_auth_pgsql_SUITE_data/server-key.pem | 27 +++++++++++++++++++ .../emqx_auth_pgsql_SUITE_data/server.crt | 21 --------------- .../emqx_auth_pgsql_SUITE_data/server.key | 27 ------------------- apps/emqx_auth_redis/etc/emqx_auth_redis.conf | 14 ++++++++++ .../priv/emqx_auth_redis.schema | 27 +++++++++++++++---- .../src/emqx_plugin_libs_ssl.erl | 3 ++- apps/emqx_web_hook/etc/emqx_web_hook.conf | 9 +++++++ apps/emqx_web_hook/priv/emqx_web_hook.schema | 4 +++ .../src/emqx_web_hook_actions.erl | 16 +++++++---- apps/emqx_web_hook/src/emqx_web_hook_app.erl | 13 ++++++--- scripts/elvis-check.sh | 8 ++++-- 37 files changed, 426 insertions(+), 181 deletions(-) create mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca-key.pem create mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem create mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-cert.pem create mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-key.pem delete mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.crt delete mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.csr delete mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.key create mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/private_key.pem create mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/public_key.pem delete mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt delete mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.srl create mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-cert.pem create mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-key.pem delete mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.crt delete mode 100644 apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.key diff --git a/.ci/compatibility_tests/pgsql/Dockerfile b/.ci/compatibility_tests/pgsql/Dockerfile index ca44acffa..a3b6374fc 100644 --- a/.ci/compatibility_tests/pgsql/Dockerfile +++ b/.ci/compatibility_tests/pgsql/Dockerfile @@ -2,9 +2,9 @@ ARG BUILD_FROM=postgres:11 FROM ${BUILD_FROM} ARG POSTGRES_USER=postgres COPY --chown=$POSTGRES_USER .ci/compatibility_tests/pgsql/pg_hba.conf /var/lib/postgresql/pg_hba.conf -COPY --chown=$POSTGRES_USER apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.key /var/lib/postgresql/server.key -COPY --chown=$POSTGRES_USER apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.crt /var/lib/postgresql/server.crt -COPY --chown=$POSTGRES_USER apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt /var/lib/postgresql/root.crt +COPY --chown=$POSTGRES_USER apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-key.pem /var/lib/postgresql/server.key +COPY --chown=$POSTGRES_USER apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-cert.pem /var/lib/postgresql/server.crt +COPY --chown=$POSTGRES_USER apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem /var/lib/postgresql/root.crt RUN chmod 600 /var/lib/postgresql/pg_hba.conf RUN chmod 600 /var/lib/postgresql/server.key RUN chmod 600 /var/lib/postgresql/server.crt diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index f5d561f0e..c996dfb8c 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -82,9 +82,11 @@ jobs: docker-compose -f .ci/compatibility_tests/docker-compose-mongo-tls.yaml up -d cat <<-EOF >> "$GITHUB_ENV" EMQX_AUTH__MONGO__SSL=on - EMQX_AUTH__MONGO__CACERTFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem - EMQX_AUTH__MONGO__CERTFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem - EMQX_AUTH__MONGO__KEYFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem + EMQX_AUTH__MONGO__SSL__CACERTFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem + EMQX_AUTH__MONGO__SSL__CERTFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem + EMQX_AUTH__MONGO__SSL__KEYFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem + EMQX_AUTH__MONGO__SSL__VERIFY=true + EMQX_AUTH__MONGO__SSL__SERVER_NAME_INDICATION=disable EOF - name: setup env: @@ -148,6 +150,8 @@ jobs: EMQX_AUTH__MYSQL__SSL__CACERTFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem EMQX_AUTH__MYSQL__SSL__CERTFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem EMQX_AUTH__MYSQL__SSL__KEYFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem + EMQX_AUTH__MYSQL__SSL__VERIFY=true + EMQX_AUTH__MYSQL__SSL__SERVER_NAME_INDICATION=disable EOF - name: setup env: @@ -214,7 +218,11 @@ jobs: docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml up -d cat <<-EOF >> "$GITHUB_ENV" EMQX_AUTH__PGSQL__SSL=on - EMQX_AUTH__PGSQL__SSL__CACERTFILE=/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt + EMQX_AUTH__PGSQL__SSL__CACERTFILE=/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem + EMQX_AUTH__PGSQL__SSL__CERTFILE=/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-cert.pem + EMQX_AUTH__PGSQL__SSL__KEYFILE=/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-key.pem + EMQX_AUTH__PGSQL__SSL__VERIFY=true + EMQX_AUTH__PGSQL__SSL__SERVER_NAME_INDICATION=disable EOF - name: setup env: @@ -288,6 +296,8 @@ jobs: EMQX_AUTH__REDIS__SSL__CACERTFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/ca.crt EMQX_AUTH__REDIS__SSL__CERTFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.crt EMQX_AUTH__REDIS__SSL__KEYFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.key + EMQX_AUTH__REDIS__SSL__VERIFY=true + EMQX_AUTH__REDIS__SSL__SERVER_NAME_INDICATION=disable EOF - name: setup env: diff --git a/apps/emqx_auth_http/etc/emqx_auth_http.conf b/apps/emqx_auth_http/etc/emqx_auth_http.conf index 3d5c45ea7..3ebf3a45d 100644 --- a/apps/emqx_auth_http/etc/emqx_auth_http.conf +++ b/apps/emqx_auth_http/etc/emqx_auth_http.conf @@ -16,14 +16,14 @@ auth.http.auth_req.method = post ## HTTP Request Headers for Auth Request, Content-Type header is configured by default. ## The possible values of the Content-Type header: application/x-www-form-urlencoded, application/json -## +## ## Examples: auth.http.auth_req.headers.accept = */* auth.http.auth_req.headers.content-type = application/x-www-form-urlencoded ## Parameters used to construct the request body or query string parameters ## When the request method is GET, these parameters will be converted into query string parameters ## When the request method is POST, the final format is determined by content-type -## +## ## Available Variables: ## - %u: username ## - %c: clientid @@ -58,7 +58,7 @@ auth.http.super_req.headers.content-type = application/x-www-form-urlencoded ## Parameters used to construct the request body or query string parameters ## When the request method is GET, these parameters will be converted into query string parameters ## When the request method is POST, the final format is determined by content-type -## +## ## Available Variables: ## - %u: username ## - %c: clientid @@ -93,7 +93,7 @@ auth.http.acl_req.headers.content-type = application/x-www-form-urlencoded ## Parameters used to construct the request body or query string parameters ## When the request method is GET, these parameters will be converted into query string parameters ## When the request method is POST, the final format is determined by content-type -## +## ## Available Variables: ## - %u: username ## - %c: clientid @@ -117,7 +117,7 @@ auth.http.acl_req.params = access=%A,username=%u,clientid=%c,ipaddr=%a,topic=%t, ## Default: 5s auth.http.timeout = 5s -## Connection time-out time, used during the initial request, +## Connection time-out time, used during the initial request, ## when the client is connecting to the server. ## ## Value: Duration @@ -151,3 +151,18 @@ auth.http.pool_size = 32 ## ## Value: File ## auth.http.ssl.keyfile = {{ platform_etc_dir }}/certs/client-key.pem + +## In mode verify_none the default behavior is to allow all x509-path +## validation errors. +## +## Value: true | false +## auth.http.ssl.verify = false + +## If not specified, the server's names returned in server's certificate is validated against +## what's provided `auth.http.auth_req.url` config's host part. +## Setting to 'disable' will make EMQ X ignore unmatched server names. +## If set with a host name, the server's names returned in server's certificate is validated +## against this value. +## +## Value: String | disable +## auth.http.ssl.server_name_indication = disable \ No newline at end of file diff --git a/apps/emqx_auth_http/priv/emqx_auth_http.schema b/apps/emqx_auth_http/priv/emqx_auth_http.schema index afd71cfd9..e1f02ef49 100644 --- a/apps/emqx_auth_http/priv/emqx_auth_http.schema +++ b/apps/emqx_auth_http/priv/emqx_auth_http.schema @@ -116,3 +116,12 @@ end}. {mapping, "auth.http.ssl.keyfile", "emqx_auth_http.keyfile", [ {datatype, string} ]}. + +{mapping, "auth.http.ssl.verify", "emqx_auth_http.verify", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +{mapping, "auth.http.ssl.server_name_indication", "emqx_auth_http.server_name_indication", [ + {datatype, string} +]}. \ No newline at end of file diff --git a/apps/emqx_auth_http/src/emqx_auth_http_app.erl b/apps/emqx_auth_http/src/emqx_auth_http_app.erl index acbb67bf4..79b34effb 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_app.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_app.erl @@ -66,11 +66,22 @@ translate_env(EnvName) -> CACertFile = application:get_env(?APP, cacertfile, undefined), CertFile = application:get_env(?APP, certfile, undefined), KeyFile = application:get_env(?APP, keyfile, undefined), - TLSOpts = lists:filter(fun({_K, V}) when V =:= <<>> -> - false; - (_) -> - true - end, [{keyfile, KeyFile}, {certfile, CertFile}, {cacertfile, CACertFile}]), + Verify = case application:get_env(?APP, verify, fasle) of + true -> verify_peer; + false -> verify_none + end, + SNI = case application:get_env(?APP, server_name_indication, undefined) of + "disable" -> disable; + SNI0 -> SNI0 + end, + TLSOpts = lists:filter( + fun({_, V}) -> + V =/= <<>> andalso V =/= undefined + end, [{keyfile, KeyFile}, + {certfile, CertFile}, + {cacertfile, CACertFile}, + {verify, Verify}, + {server_name_indication, SNI}]), NTLSOpts = [ {versions, emqx_tls_lib:default_versions()} , {ciphers, emqx_tls_lib:default_ciphers()} | TLSOpts diff --git a/apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl b/apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl index c2ad0ac43..5ac5c18e8 100644 --- a/apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl +++ b/apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl @@ -90,7 +90,9 @@ set_https_client_opts() -> SSLOpt = emqx_ct_helpers:client_ssl_twoway(), application:set_env(emqx_auth_http, cacertfile, proplists:get_value(cacertfile, SSLOpt, undefined)), application:set_env(emqx_auth_http, certfile, proplists:get_value(certfile, SSLOpt, undefined)), - application:set_env(emqx_auth_http, keyfile, proplists:get_value(keyfile, SSLOpt, undefined)). + application:set_env(emqx_auth_http, keyfile, proplists:get_value(keyfile, SSLOpt, undefined)), + application:set_env(emqx_auth_http, verify, true), + application:set_env(emqx_auth_http, server_name_indication, "disable"). %% @private http_server(http, inet) -> "http://127.0.0.1:8991"; diff --git a/apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf b/apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf index 746510fb3..8eebefe97 100644 --- a/apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf +++ b/apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf @@ -73,6 +73,4 @@ auth.ldap.ssl = false #auth.ldap.ssl.verify = verify_peer -#auth.ldap.ssl.fail_if_no_peer_cert = true - #auth.ldap.ssl.server_name_indication = your_server_name diff --git a/apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema b/apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema index 554752a0b..b3d3de1a2 100644 --- a/apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema +++ b/apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema @@ -53,10 +53,6 @@ {datatype, {enum, [verify_none, verify_peer]}} ]}. -{mapping, "auth.ldap.ssl.fail_if_no_peer_cert", "emqx_auth_ldap.ldap", [ - {datatype, {enum, [true, false]}} -]}. - {mapping, "auth.ldap.ssl.server_name_indication", "emqx_auth_ldap.ldap", [ {datatype, string} ]}. @@ -75,8 +71,10 @@ {keyfile, cuttlefish:conf_get("auth.ldap.ssl.keyfile", Conf)}, {cacertfile, cuttlefish:conf_get("auth.ldap.ssl.cacertfile", Conf, undefined)}, {verify, cuttlefish:conf_get("auth.ldap.ssl.verify", Conf, undefined)}, - {server_name_indication, cuttlefish:conf_get("auth.ldap.ssl.server_name_indication", Conf, disable)}, - {fail_if_no_peer_cert, cuttlefish:conf_get("auth.ldap.ssl.fail_if_no_peer_cert", Conf, undefined)}] + {server_name_indication, case cuttlefish:conf_get("auth.ldap.ssl.server_name_indication", Conf, undefined) of + "disable" -> disable; + SNI -> SNI + end}] end, Opts = [{servers, Servers}, {port, Port}, diff --git a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf index 073feeb6d..2a3d038f0 100644 --- a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf +++ b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf @@ -70,6 +70,21 @@ auth.mongo.database = mqtt ## Value: File ## auth.mongo.ssl.cacertfile = +## In mode verify_none the default behavior is to allow all x509-path +## validation errors. +## +## Value: true | false +## auth.mongo.ssl.verify = false + +## If not specified, the server's names returned in server's certificate is validated against +## what's provided `auth.mongo.server` config's host part. +## Setting to 'disable' will make EMQ X ignore unmatched server names. +## If set with a host name, the server's names returned in server's certificate is validated +## against this value. +## +## Value: String | disable +## auth.mongo.ssl.server_name_indication = disable + ## MongoDB write mode. ## ## Value: unsafe | safe diff --git a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema index bef569306..15d7ebdef 100644 --- a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema +++ b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema @@ -62,6 +62,15 @@ {datatype, string} ]}. +{mapping, "auth.mongo.ssl.verify", "emqx_auth_mongo.server", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +{mapping, "auth.mongo.ssl.server_name_indication", "emqx_auth_mongo.server", [ + {datatype, string} +]}. + %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 {mapping, "auth.mongo.ssl_opts.keyfile", "emqx_auth_mongo.server", [ {datatype, string} @@ -122,10 +131,20 @@ false -> [{r_mode, R}] end, + Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, SslOpts = fun(Prefix) -> - Filter([{keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, - {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, + Verify = case cuttlefish:conf_get(Prefix ++ ".verify", Conf, false) of + true -> verify_peer; + flase -> verify_none + end, + Filter([{verify, Verify}, + {server_name_indication, case cuttlefish:conf_get(Prefix ++ ".server_name_indication", Conf, undefined) of + "disable" -> disable; + SNI -> SNI + end}, + {keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, + {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}]) end, diff --git a/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf b/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf index d367c2edc..6014329b3 100644 --- a/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf +++ b/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf @@ -114,3 +114,18 @@ auth.mysql.acl_query = select allow, ipaddr, username, clientid, access, topic f ## ## Value: File #auth.mysql.ssl.keyfile = /path/to/your/clientkey.pem + +## In mode verify_none the default behavior is to allow all x509-path +## validation errors. +## +## Value: true | false +#auth.mysql.ssl.verify = false + +## If not specified, the server's names returned in server's certificate is validated against +## what's provided `auth.mysql.server` config's host part. +## Setting to 'disable' will make EMQ X ignore unmatched server names. +## If set with a host name, the server's names returned in server's certificate is validated +## against this value. +## +## Value: String | disable +## auth.mysql.ssl.server_name_indication = disable diff --git a/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema b/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema index 8f9c069c4..f68b73436 100644 --- a/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema +++ b/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema @@ -52,6 +52,15 @@ {datatype, string} ]}. +{mapping, "auth.mysql.ssl.verify", "emqx_auth_mysql.server", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +{mapping, "auth.mysql.ssl.server_name_indication", "emqx_auth_mysql.server", [ + {datatype, string} +]}. + {translation, "emqx_auth_mysql.server", fun(Conf) -> {MyHost, MyPort} = case cuttlefish:conf_get("auth.mysql.server", Conf) of @@ -94,10 +103,20 @@ ), Cert = cuttlefish:conf_get("auth.mysql.ssl.certfile", Conf, undefined), Key = cuttlefish:conf_get("auth.mysql.ssl.keyfile", Conf, undefined), - Options ++ [{ssl, Filter([{server_name_indication, disable}, + Verify = case cuttlefish:conf_get("auth.mysql.ssl.verify", Conf, false) of + true -> verify_peer; + flase -> verify_none + end, + SNI = case cuttlefish:conf_get("auth.mysql.ssl.server_name_indication", Conf, undefined) of + "disable" -> disable; + SNI0 -> SNI0 + end, + Options ++ [{ssl, Filter([{server_name_indication, SNI}, {cacertfile, CA}, {certfile, Cert}, - {keyfile, Key}]) + {keyfile, Key}, + {verify, Verify} + ]) }]; _ -> Options diff --git a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf index d27956b16..e39d0c78a 100644 --- a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf +++ b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf @@ -62,6 +62,21 @@ auth.pgsql.ssl = off ## Value: File #auth.pgsql.ssl.cacertfile = +## In mode verify_none the default behavior is to allow all x509-path +## validation errors. +## +## Value: true | false +#auth.pgsql.ssl.verify = false + +## If not specified, the server's names returned in server's certificate is validated against +## what's provided `auth.pgsql.server` config's host part. +## Setting to 'disable' will make EMQ X ignore unmatched server names. +## If set with a host name, the server's names returned in server's certificate is validated +## against this value. +## +## Value: String | disable +## auth.pgsql.ssl.server_name_indication = disable + ## Authentication query. ## ## Value: SQL diff --git a/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema b/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema index 77a239ba9..6e5441413 100644 --- a/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema +++ b/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema @@ -52,6 +52,15 @@ {datatype, string} ]}. +{mapping, "auth.pgsql.ssl.verify", "emqx_auth_pgsql.server", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +{mapping, "auth.pgsql.ssl.server_name_indication", "emqx_auth_pgsql.server", [ + {datatype, string} +]}. + %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 {mapping, "auth.pgsql.ssl_opts.keyfile", "emqx_auth_pgsql.server", [ {datatype, string} @@ -90,9 +99,18 @@ Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, SslOpts = fun(Prefix) -> + Verify = case cuttlefish:conf_get(Prefix ++ ".verify", Conf, false) of + true -> verify_peer; + flase -> verify_none + end, Filter([{keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}, + {verify, Verify}, + {server_name_indication, case cuttlefish:conf_get(Prefix ++ ".server_name_indication", Conf, undefined) of + "disable" -> disable; + SNI -> SNI + end}, {versions, [list_to_existing_atom(Value) || Value <- string:tokens(cuttlefish:conf_get(Prefix ++ ".tls_versions", Conf), " ,")]}]) end, diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca-key.pem b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca-key.pem new file mode 100644 index 000000000..e9717011e --- /dev/null +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA0kGUBi9NDp65jgdxKfizIfuSr2wpwb44yM9SuP4oUQSULOA2 +4iFpLR/c5FAYHU81y9Vx91dQjdZfffaBZuv2zVvteXUkol8Nez7boKbo2E41MTew +8edtNKZAQVvnaHAC2NCZxjchCzUCDEoUUcl+cIERZ8R48FBqK5iTVcMRIx1akwus ++dhBqP0ykA5TGOWZkJrLM9aUXSPQha9+wXlOpkvu0Ur2nkX8PPJnifWao9UShSar +ll1IqPZNCSlZMwcFYcQNBCpdvITUUYlHvMRQV64bUpOxUGDuJkQL3dLKBlNuBRlJ +BcjBAKw7rFnwwHZcMmQ9tan/dZzpzwjo/T0XjwIDAQABAoIBAQCSHvUqnzDkWjcG +l/Fzg92qXlYBCCC0/ugj1sHcwvVt6Mq5rVE3MpUPwTcYjPlVVTlD4aEEjm/zQuq2 +ddxUlOS+r4aIhHrjRT/vSS4FpjnoKeIZxGR6maVxk6DQS3i1QjMYT1CvSpzyVvKH +a+xXMrtmoKxh+085ZAmFJtIuJhUA2yEa4zggCxWnvz8ecLClUPfVDPhdLBHc3KmL +CRpHEC6L/wanvDPRdkkzfKyaJuIJlTDaCg63AY5sDkTW2I57iI/nJ3haSeidfQKz +39EfbnM1A/YprIakafjAu3frBIsjBVcxwGihZmL/YriTHjOggJF841kT5zFkkv2L +/530Wk6xAoGBAOqZLZ4DIi/zLndEOz1mRbUfjc7GQUdYplBnBwJ22VdS0P4TOXnd +UbJth2MA92NM7ocTYVFl4TVIZY/Y+Prxk7KQdHWzR7JPpKfx9OEVgtSqV0vF9eGI +rKp79Y1T4Mvc3UcQCXX6TP7nHLihEzpS8odm2LW4txrOiLsn4Fq/IWrLAoGBAOVv +6U4tm3lImotUupKLZPKEBYwruo9qRysoug9FiorP4TjaBVOfltiiHbAQD6aGfVtN +SZpZZtrs17wL7Xl4db5asgMcZd+8Hkfo5siR7AuGW9FZloOjDcXb5wCh9EvjJ74J +Cjw7RqyVymq9t7IP6wnVwj5Ck48YhlOZCz/mzlnNAoGAWq7NYFgLvgc9feLFF23S +IjpJQZWHJEITP98jaYNxbfzYRm49+GphqxwFinKULjFNvq7yHlnIXSVYBOu1CqOZ +GRwXuGuNmlKI7lZr9xmukfAqgGLMMdr4C4qRF4lFyufcLRz42z7exmWlx4ST/yaT +E13hBRWayeTuG5JFei6Jh1MCgYEAqmX4LyC+JFBgvvQZcLboLRkSCa18bADxhENG +FAuAvmFvksqRRC71WETmqZj0Fqgxt7pp3KFjO1rFSprNLvbg85PmO1s+6fCLyLpX +lESTu2d5D71qhK93jigooxalGitFm+SY3mzjq0/AOpBWOn+J/w7rqVPGxXLgaHv0 +l+vx+00CgYBOvo9/ImjwYii2jFl+sHEoCzlvpITi2temRlT2j6ulSjCLJgjwEFw9 +8e+vvfQumQOsutakUVyURrkMGNDiNlIv8kv5YLCCkrwN22E6Ghyi69MJUvHQXkc/ +QZhjn/luyfpB5f/BeHFS2bkkxAXo+cfG45ApY3Qfz6/7o+H+vDa6/A== +-----END RSA PRIVATE KEY----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem new file mode 100644 index 000000000..00b31d8a4 --- /dev/null +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDAzCCAeugAwIBAgIBATANBgkqhkiG9w0BAQsFADA8MTowOAYDVQQDDDFNeVNR +TF9TZXJ2ZXJfOC4wLjE5X0F1dG9fR2VuZXJhdGVkX0NBX0NlcnRpZmljYXRlMB4X +DTIwMDYxMTAzMzg0NloXDTMwMDYwOTAzMzg0NlowPDE6MDgGA1UEAwwxTXlTUUxf +U2VydmVyXzguMC4xOV9BdXRvX0dlbmVyYXRlZF9DQV9DZXJ0aWZpY2F0ZTCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANJBlAYvTQ6euY4HcSn4syH7kq9s +KcG+OMjPUrj+KFEElCzgNuIhaS0f3ORQGB1PNcvVcfdXUI3WX332gWbr9s1b7Xl1 +JKJfDXs+26Cm6NhONTE3sPHnbTSmQEFb52hwAtjQmcY3IQs1AgxKFFHJfnCBEWfE +ePBQaiuYk1XDESMdWpMLrPnYQaj9MpAOUxjlmZCayzPWlF0j0IWvfsF5TqZL7tFK +9p5F/DzyZ4n1mqPVEoUmq5ZdSKj2TQkpWTMHBWHEDQQqXbyE1FGJR7zEUFeuG1KT +sVBg7iZEC93SygZTbgUZSQXIwQCsO6xZ8MB2XDJkPbWp/3Wc6c8I6P09F48CAwEA +AaMQMA4wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEADKz6bIpP5anp +GgLB0jkclRWuMlS4qqIt4itSsMXPJ/ezpHwECixmgW2TIQl6S1woRkUeMxhT2/Ay +Sn/7aKxuzRagyE5NEGOvrOuAP5RO2ZdNJ/X3/Rh533fK1sOTEEbSsWUvW6iSkZef +rsfZBVP32xBhRWkKRdLeLB4W99ADMa0IrTmZPCXHSSE2V4e1o6zWLXcOZeH1Qh8N +SkelBweR+8r1Fbvy1r3s7eH7DCbYoGEDVLQGOLvzHKBisQHmoDnnF5E9g1eeNRdg +o+vhOKfYCOzeNREJIqS42PHcGhdNRk90ycigPmfUJclz1mDHoMjKR2S5oosTpr65 +tNPx3CL7GA== +-----END CERTIFICATE----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-cert.pem b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-cert.pem new file mode 100644 index 000000000..aad1404ca --- /dev/null +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDBDCCAeygAwIBAgIBAzANBgkqhkiG9w0BAQsFADA8MTowOAYDVQQDDDFNeVNR +TF9TZXJ2ZXJfOC4wLjE5X0F1dG9fR2VuZXJhdGVkX0NBX0NlcnRpZmljYXRlMB4X +DTIwMDYxMTAzMzg0N1oXDTMwMDYwOTAzMzg0N1owQDE+MDwGA1UEAww1TXlTUUxf +U2VydmVyXzguMC4xOV9BdXRvX0dlbmVyYXRlZF9DbGllbnRfQ2VydGlmaWNhdGUw +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDVYSWpOvCTupz82fc85Opv +EQ7rkB8X2oOMyBCpkyHKBIr1ZQgRDWBp9UVOASq3GnSElm6+T3Kb1QbOffa8GIlw +sjAueKdq5L2eSkmPIEQ7eoO5kEW+4V866hE1LeL/PmHg2lGP0iqZiJYtElhHNQO8 +3y9I7cm3xWMAA3SSWikVtpJRn3qIp2QSrH+tK+/HHbE5QwtPxdir4ULSCSOaM5Yh +Wi5Oto88TZqe1v7SXC864JVvO4LuS7TuSreCdWZyPXTJFBFeCEWSAxonKZrqHbBe +CwKML6/0NuzjaQ51c2tzmVI6xpHj3nnu4cSRx6Jf9WBm+35vm0wk4pohX3ptdzeV +AgMBAAGjDTALMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEBAByQ5zSNeFUH +Aw7JlpZHtHaSEeiiyBHke20ziQ07BK1yi/ms2HAWwQkpZv149sjNuIRH8pkTmkZn +g8PDzSefjLbC9AsWpWV0XNV22T/cdobqLqMBDDZ2+5bsV+jTrOigWd9/AHVZ93PP +IJN8HJn6rtvo2l1bh/CdsX14uVSdofXnuWGabNTydqtMvmCerZsdf6qKqLL+PYwm +RDpgWiRUY7KPBSSlKm/9lJzA+bOe4dHeJzxWFVCJcbpoiTFs1je1V8kKQaHtuW39 +ifX6LTKUMlwEECCbDKM8Yq2tm8NjkjCcnFDtKg8zKGPUu+jrFMN5otiC3wnKcP7r +O9EkaPcgYH8= +-----END CERTIFICATE----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-key.pem b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-key.pem new file mode 100644 index 000000000..6789d0291 --- /dev/null +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA1WElqTrwk7qc/Nn3POTqbxEO65AfF9qDjMgQqZMhygSK9WUI +EQ1gafVFTgEqtxp0hJZuvk9ym9UGzn32vBiJcLIwLninauS9nkpJjyBEO3qDuZBF +vuFfOuoRNS3i/z5h4NpRj9IqmYiWLRJYRzUDvN8vSO3Jt8VjAAN0klopFbaSUZ96 +iKdkEqx/rSvvxx2xOUMLT8XYq+FC0gkjmjOWIVouTraPPE2antb+0lwvOuCVbzuC +7ku07kq3gnVmcj10yRQRXghFkgMaJyma6h2wXgsCjC+v9Dbs42kOdXNrc5lSOsaR +49557uHEkceiX/VgZvt+b5tMJOKaIV96bXc3lQIDAQABAoIBAF7yjXmSOn7h6P0y +WCuGiTLG2mbDiLJqj2LTm2Z5i+2Cu/qZ7E76Ls63TxF4v3MemH5vGfQhEhR5ZD/6 +GRJ1sKKvB3WGRqjwA9gtojHH39S/nWGy6vYW/vMOOH37XyjIr3EIdIaUtFQBTSHd +Kd71niYrAbVn6fyWHolhADwnVmTMOl5OOAhCdEF4GN3b5aIhIu8BJ7EUzTtHBJIj +CAEfjZFjDs1y1cIgGFJkuIQxMfCpq5recU2qwip7YO6fk//WEjOPu7kSf5IEswL8 +jg1dea9rGBV6KaD2xsgsC6Ll6Sb4BbsrHMfflG3K2Lk3RdVqqTFp1Fn1PTLQE/1S +S/SZPYECgYEA9qYcHKHd0+Q5Ty5wgpxKGa4UCWkpwvfvyv4bh8qlmxueB+l2AIdo +ZvkM8gTPagPQ3WypAyC2b9iQu70uOJo1NizTtKnpjDdN1YpDjISJuS/P0x73gZwy +gmoM5AzMtN4D6IbxXtXnPaYICvwLKU80ouEN5ZPM4/ODLUu6gsp0v2UCgYEA3Xgi +zMC4JF0vEKEaK0H6QstaoXUmw/lToZGH3TEojBIkb/2LrHUclygtONh9kJSFb89/ +jbmRRLAOrx3HZKCNGUmF4H9k5OQyAIv6OGBinvLGqcbqnyNlI+Le8zxySYwKMlEj +EMrBCLmSyi0CGFrbZ3mlj/oCET/ql9rNvcK+DHECgYAEx5dH3sMjtgp+RFId1dWB +xePRgt4yTwewkVgLO5wV82UOljGZNQaK6Eyd7AXw8f38LHzh+KJQbIvxd2sL4cEi +OaAoohpKg0/Y0YMZl//rPMf0OWdmdZZs/I0fZjgZUSwWN3c59T8z7KG/RL8an9RP +S7kvN7wCttdV61/D5RR6GQKBgDxCe/WKWpBKaovzydMLWLTj7/0Oi0W3iXHkzzr4 +LTgvl4qBSofaNbVLUUKuZTv5rXUG2IYPf99YqCYtzBstNDc1MiAriaBeFtzfOW4t +i6gEFtoLLbuvPc3N5Sv5vn8Ug5G9UfU3td5R4AbyyCcoUZqOFuZd+EIJSiOXfXOs +kVmBAoGBAIU9aPAqhU5LX902oq8KsrpdySONqv5mtoStvl3wo95WIqXNEsFY60wO +q02jKQmJJ2MqhkJm2EoF2Mq8+40EZ5sz8LdgeQ/M0yQ9lAhPi4rftwhpe55Ma9dk +SE9X1c/DMCBEaIjJqVXdy0/EeArwpb8sHkguVVAZUWxzD+phm1gs +-----END RSA PRIVATE KEY----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.crt b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.crt deleted file mode 100644 index 9867681b9..000000000 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDYzCCAksCCQC7J1oPkDz7vTANBgkqhkiG9w0BAQUFADCBhTELMAkGA1UEBhMC -Q0ExGTAXBgNVBAgMEEJyaXRpc2ggQ29sdW1iaWExDjAMBgNVBAcMBUNvbW94MRQw -EgYDVQQKDAtUaGVCcmFpbi5jYTEUMBIGA1UEAwwLdGhlYnJhaW4uY2ExHzAdBgkq -hkiG9w0BCQEWEGluZm9AdGhlYnJhaW4uY2EwHhcNMjEwMTEzMDkwNzM2WhcNMjEw -MjEyMDkwNzM2WjBhMQswCQYDVQQGEwJDQTEZMBcGA1UECAwQQnJpdGlzaCBDb2x1 -bWJpYTEOMAwGA1UEBwwFQ29tb3gxFDASBgNVBAoMC1RoZUJyYWluLmNhMREwDwYD -VQQDDAh3d3ctZGF0YTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJv9 -yO5JGKBl+7w0HGkRDIPZ5Ku3lIAzB4ThszRHBqll7VjlTz+q16OQOONqeHBuxPjj -11WMXD2KnfYZW2ZWd0U8FKzuIGOCStGbSUi2hC0owp+KkJcDujfIafXQnAa0fUiS -FBB5iG98vm3QI4gv9135LgnO5oHopH6oZ/t0Id1LzFhp2sdhebdtczmImpo+nt7v -fduapptuIJ20ThdAvo3MlYoAhivsvJKntlWPAwPMQdyezww/q7T5Y8DCyJJTydr5 -PrMz9S/WQTkj/G0y4dZgQonG5r0d1Nf+rwkn78DdXGktVDMBBP41+VWnEDBCTlgS -FjQEY6Izaof8s8q8K2UCAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAdlAQkumOAKbQ -SW5gtkHgKyIQyfwk9maKqKccK04WlNk1t1jsvk7kaOEHr3t7YG28yKqicGHAcfFf -i/RU51v2GJVzWCbzkAAH/zNgDcYnYk6sn54YcuBzrPliVH1xxmZy/52+huTxy8Vd -3nmCjdYR/I764rd8gkRK+aHaUTLyitzX1kW90LtXonKY72CNZVXHEBom3XM/a6ff -ilybDloNVTfHstnfsnHHyNYn0SfapqXxPCO+FL9hQjlztUBZryRdS0nq66hB2GSB -CEst/vtNGo/2aa1Vw4bKl2oGepjKNzxp0ZTTVuIcwGzV6oKIsx1ZnWE3gQLEH/TX -dzMzesBayA== ------END CERTIFICATE----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.csr b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.csr deleted file mode 100644 index 325fbe397..000000000 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.csr +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIICpjCCAY4CAQAwYTELMAkGA1UEBhMCQ0ExGTAXBgNVBAgMEEJyaXRpc2ggQ29s -dW1iaWExDjAMBgNVBAcMBUNvbW94MRQwEgYDVQQKDAtUaGVCcmFpbi5jYTERMA8G -A1UEAwwId3d3LWRhdGEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCb -/cjuSRigZfu8NBxpEQyD2eSrt5SAMweE4bM0RwapZe1Y5U8/qtejkDjjanhwbsT4 -49dVjFw9ip32GVtmVndFPBSs7iBjgkrRm0lItoQtKMKfipCXA7o3yGn10JwGtH1I -khQQeYhvfL5t0COIL/dd+S4JzuaB6KR+qGf7dCHdS8xYadrHYXm3bXM5iJqaPp7e -733bmqabbiCdtE4XQL6NzJWKAIYr7LySp7ZVjwMDzEHcns8MP6u0+WPAwsiSU8na -+T6zM/Uv1kE5I/xtMuHWYEKJxua9HdTX/q8JJ+/A3VxpLVQzAQT+NflVpxAwQk5Y -EhY0BGOiM2qH/LPKvCtlAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEAN6Q8MEDx -g5xlpYB/fFmagpe15+G2QbqVf2mH1a4aBcBns4jMMqNidi4gyjGfzvNxX77R6KcI -AfcxENRVDYJbhAgEQ96jv4jv5pEMuyvQ8VLhn9AOXCaK/VHxbYlOiM7tfFtEDrrB -wTn8FvoEwjehfsSX2dWiwcUK4SPPeuklE/EGjRgoVCwg8EqWzf1fn+tzME8OpnRQ -I8coyALF6ANehvP7ADV3m5iOOaNhfnqmqGBEwjB3TTvE1gZ4UvAyl75bi+Zh3Osn -qemyxocp/ML4o6d/F+nKIZOe6309V2nyrY6RSd2fBCrhYj2rKTbrGTZrpKXeAhtI -jMivnjCK+WNHpQ== ------END CERTIFICATE REQUEST----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.key b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.key deleted file mode 100644 index 787246f6f..000000000 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/postgresql.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAm/3I7kkYoGX7vDQcaREMg9nkq7eUgDMHhOGzNEcGqWXtWOVP -P6rXo5A442p4cG7E+OPXVYxcPYqd9hlbZlZ3RTwUrO4gY4JK0ZtJSLaELSjCn4qQ -lwO6N8hp9dCcBrR9SJIUEHmIb3y+bdAjiC/3XfkuCc7mgeikfqhn+3Qh3UvMWGna -x2F5t21zOYiamj6e3u9925qmm24gnbROF0C+jcyVigCGK+y8kqe2VY8DA8xB3J7P -DD+rtPljwMLIklPJ2vk+szP1L9ZBOSP8bTLh1mBCicbmvR3U1/6vCSfvwN1caS1U -MwEE/jX5VacQMEJOWBIWNARjojNqh/yzyrwrZQIDAQABAoIBAAOicycSLu+10Jq/ -ABZ2njsIPaq+mUgvaDJxa9KBASe7Rz92AFW0blfSSXELDwlXm2FNNbw5jACnFS0h -xB5rT1Yeo0CwP7Lx2zptCtUV45iFxZsgCGRsYs9f7RAcLzZ8yBqDxNHpcwNd/bXj -TqCitXnMD4WM+5P1TrfgxqN2Pj/Atg8w/4dP7KcFcTzcZzIz5rr3NTyjsrLdiFis -sR+7m7Qu4PyEfrDpR9Np111nQqVJ1bpt9qt/hv318FaBnpNY6MMBaSni99mvMXSd -SwHn3gnfHREWcNSLGA9gjEQmyIPHpV9T6SJ/zyr++6y8QCq4DiSP36A9zeA1XThP -YEIsWxUCgYEAyLppQerpOT2CnbTbKO/9rGwlbf8FT2GWFcPBtUm0lp21/C32BX+H -jNCmQsE1pZ6+sqv2mb1onr6Xl9cSEt6KsI1EJtFFR9Lnvqqu+JKo31U94z2yTqgv -sc+qMl7shy1kja8T5NaRc++UkCVzVNsnFB9torIaqQwY9IRdRwmYjisCgYEAxvHR -MwvWpOg25zz75OfupIOQhj9W6yphpY5/yoYBms/4OeabJhMrOV142s9souCHmuGU -EtzOQC5jbEc+3MUjx1ZlboHY7UuoEu87kykFEs9mnaD+T34PEAJcQjSzqzS5KMJE -Ro275xf+V/e3hS/Z3hQXmDQNQDNRYMcAZfTW9K8CgYBkHITOuYikYcc5PLBplHhi -fHWWjLBrTPJ73GxKLH6C+BmBsrKXP2mtk4q4lIBbH/dgSV/ugYciVVBqDHwZKSDm -uS4aZhk1nzyx3ZLyqsLK0ErTgTvi+wL+neH2yV0SdlNGTuGPKmzU89KWqfcBhWPS -J3KYyFd/pGb13OZgvap2jQKBgBXCXR84LEHdJCQmh2aB95gGy8fjJZ6TBBsXeuKr -xYEpPf0XO+DuN8wObSmBhmBKLorCIW/utqBOcpFlOXrsFP24dV+g1BkgLUHk6J8v -3V4xUQfsk+Qd5YfaujyDhyMyoQ3UMaOF3QdpmGgGsAvhL/MaP3pmNwzOkBgFrAV6 -wggBAoGBAMflqy2pfqGhaj9S6qZ3K95h7NdCUikdQzqmgbNtOHaZ2kHByyYtOPLB -1VnuDRQiacmum+fTZa6wNmvp2FWg+uxI/aspfF6SdPfGpyPrG5D+ITtqKF2xieK+ -XpzehKTrTuYQRAVhmWbhpuyahYnQyd/MrsCMGzUfAJtM7l5vKa2O ------END RSA PRIVATE KEY----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/private_key.pem b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/private_key.pem new file mode 100644 index 000000000..8fbf6bdec --- /dev/null +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/private_key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA1zVmMhPqpSPMmYkKh5wwlRD5XuS8YWJKEM6tjFx61VK8qxHE +YngkC2KnL5EuKAjQZIF3tJskwt0hAat047CCCZxrkNEpbVvSnvnk+A/8bg/Ww1n3 +qxzfifhsWfpUKlDnwrtH+ftt+5rZeEkf37XAPy7ZjzecAF9SDV6WSiPeAxUX2+hN +dId42Pf45woo4LFGUlQeagCFkD/R0dpNIMGwcnkKCUikiBqr2ijSIgvRtBfZ9fBG +jFGER2uE/Eay4AgcQsHue8skRwDCng8OnqtPnBtTytmqTy9V/BRgsVKUoksm6wsx +kUYwgHeaq7UCvlCm25SZ7yRyd4k8t0BKDf2h+wIDAQABAoIBAEQcrHmRACTADdNS +IjkFYALt2l8EOfMAbryfDSJtapr1kqz59JPNvmq0EIHnixo0n/APYdmReLML1ZR3 +tYkSpjVwgkLVUC1CcIjMQoGYXaZf8PLnGJHZk45RR8m6hsTV0mQ5bfBaeVa2jbma +OzJMjcnxg/3l9cPQZ2G/3AUfEPccMxOXp1KRz3mUQcGnKJGtDbN/kfmntcwYoxaE +Zg4RoeKAoMpK1SSHAiJKe7TnztINJ7uygR9XSzNd6auY8A3vomSIjpYO7XL+lh7L +izm4Ir3Gb/eCYBvWgQyQa2KCJgK/sQyEs3a09ngofSEUhQJQYhgZDwUj+fDDOGqj +hCZOA8ECgYEA+ZWuHdcUQ3ygYhLds2QcogUlIsx7C8n/Gk/FUrqqXJrTkuO0Eqqa +B47lCITvmn2zm0ODfSFIARgKEUEDLS/biZYv7SUTrFqBLcet+aGI7Dpv91CgB75R +tNzcIf8VxoiP0jPqdbh9mLbbxGi5Uc4p9TVXRljC4hkswaouebWee0sCgYEA3L2E +YB3kiHrhPI9LHS5Px9C1w+NOu5wP5snxrDGEgaFCvL6zgY6PflacppgnmTXl8D1x +im0IDKSw5dP3FFonSVXReq3CXDql7UnhfTCiLDahV7bLxTH42FofcBpDN3ERdOal +58RwQh6VrLkzQRVoObo+hbGlFiwwSAfQC509FhECgYBsRSBpVXo25IN2yBRg09cP ++gdoFyhxrsj5kw1YnB13WrrZh+oABv4WtUhp77E5ZbpaamlKCPwBbXpAjeFg4tfr +0bksuN7V79UGFQ9FsWuCfr8/nDwv38H2IbFlFhFONMOfPmJBey0Q6JJhm8R41mSh +OOiJXcv85UrjIH5U0hLUDQKBgQDVLOU5WcUJlPoOXSgiT0ZW5xWSzuOLRUUKEf6l +19BqzAzCcLy0orOrRAPW01xylt2v6/bJw1Ahva7k1ZZo/kOwjANYoZPxM+ZoSZBN +MXl8j2mzZuJVV1RFxItV3NcLJNPB/Lk+IbRz9kt/2f9InF7iWR3mSU/wIM6j0X+2 +p6yFsQKBgQCM/ldWb511lA+SNkqXB2P6WXAgAM/7+jwsNHX2ia2Ikufm4SUEKMSv +mti/nZkHDHsrHU4wb/2cOAywMELzv9EHzdcoenjBQP65OAc/1qWJs+LnBcCXfqKk +aHjEZW6+brkHdRGLLY3YAHlt/AUL+RsKPJfN72i/FSpmu+52G36eeQ== +-----END RSA PRIVATE KEY----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/public_key.pem b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/public_key.pem new file mode 100644 index 000000000..f9772b533 --- /dev/null +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/public_key.pem @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1zVmMhPqpSPMmYkKh5ww +lRD5XuS8YWJKEM6tjFx61VK8qxHEYngkC2KnL5EuKAjQZIF3tJskwt0hAat047CC +CZxrkNEpbVvSnvnk+A/8bg/Ww1n3qxzfifhsWfpUKlDnwrtH+ftt+5rZeEkf37XA +Py7ZjzecAF9SDV6WSiPeAxUX2+hNdId42Pf45woo4LFGUlQeagCFkD/R0dpNIMGw +cnkKCUikiBqr2ijSIgvRtBfZ9fBGjFGER2uE/Eay4AgcQsHue8skRwDCng8OnqtP +nBtTytmqTy9V/BRgsVKUoksm6wsxkUYwgHeaq7UCvlCm25SZ7yRyd4k8t0BKDf2h ++wIDAQAB +-----END PUBLIC KEY----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt deleted file mode 100644 index 46b1e2a7a..000000000 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDiDCCAnACCQCCsPcIlZO4TDANBgkqhkiG9w0BAQsFADCBhTELMAkGA1UEBhMC -Q0ExGTAXBgNVBAgMEEJyaXRpc2ggQ29sdW1iaWExDjAMBgNVBAcMBUNvbW94MRQw -EgYDVQQKDAtUaGVCcmFpbi5jYTEUMBIGA1UEAwwLdGhlYnJhaW4uY2ExHzAdBgkq -hkiG9w0BCQEWEGluZm9AdGhlYnJhaW4uY2EwHhcNMjEwMTEzMDkwNDIyWhcNMzEw -MTExMDkwNDIyWjCBhTELMAkGA1UEBhMCQ0ExGTAXBgNVBAgMEEJyaXRpc2ggQ29s -dW1iaWExDjAMBgNVBAcMBUNvbW94MRQwEgYDVQQKDAtUaGVCcmFpbi5jYTEUMBIG -A1UEAwwLdGhlYnJhaW4uY2ExHzAdBgkqhkiG9w0BCQEWEGluZm9AdGhlYnJhaW4u -Y2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC2YWuwplM2Hc5tzBMu -covW9nwZ8iNEFo5pbDc8710pmnkF+wsDztLy4afJe6OeVHyCgQxmE+rTZcoWbvoh -pxW3Zy/8es4My07RKHqI3NYadThUvDsmI10cF3tJbhOZaIrMaExLGookZYKwbNAy -7yJ1+MLyNCuFFsaOiNNxHOjH/InKSzEuGSLV68tdC7Pe+uanBcC7RKhOrjUC6Occ -naHPC+a/YMyRYx29T8CfkCBB7N6WanWylFN/1RBmAgq++kDflSaF9k+Zdl6I4jiF -mCPGS0k+AMre4PuAKOZOZOwhF0sWlXIxH6zPm9w0bSYdTLBupL846RTO72NtNP+X -KX5DAgMBAAEwDQYJKoZIhvcNAQELBQADggEBACXXFws+h+Zo9HsxW3BWpl2JU5u6 -KyfbLQt4kSN/gqltd4s84Q8c4z2jNdI0t8Oh5dXTjbLCpFjzuF2tdMtOWeYBCdsQ -4NJ69RrwkFdsSPxDPhSE0WGXPaOBaA92wJjTkVf+UYIek1ozeyWwFm1LPiZVei00 -mwDVgbAbIEb8cf6OqJrl2r5PMBCLWBwwg5aca3fe6TopJhyPA//DZDRPA5xzKb9e -PHUgF3apbcWxuxm8Mts4bAq8BcKoEvLHYWJ4fEWQvXPP7q1jYC3TkpSt5n3FQZTe -nLyQ+RNzsEHzmyOtTSa0Q+5KVluO1TE3ifpv8737pTLdY8t2waBamoboCu8= ------END CERTIFICATE----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.srl b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.srl deleted file mode 100644 index cf7e9e551..000000000 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.srl +++ /dev/null @@ -1 +0,0 @@ -BB275A0F903CFBBD diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-cert.pem b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-cert.pem new file mode 100644 index 000000000..a2f9688df --- /dev/null +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDBDCCAeygAwIBAgIBAjANBgkqhkiG9w0BAQsFADA8MTowOAYDVQQDDDFNeVNR +TF9TZXJ2ZXJfOC4wLjE5X0F1dG9fR2VuZXJhdGVkX0NBX0NlcnRpZmljYXRlMB4X +DTIwMDYxMTAzMzg0NloXDTMwMDYwOTAzMzg0NlowQDE+MDwGA1UEAww1TXlTUUxf +U2VydmVyXzguMC4xOV9BdXRvX0dlbmVyYXRlZF9TZXJ2ZXJfQ2VydGlmaWNhdGUw +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCcEnEm5hqP1EbEJycOz8Ua +NWp29QdpFUzTWhkKGhVXk+0msmNTw4NBAFB42moY44OU8wvDideOlJNhPRWveD8z +G2lxzJA91p0UK4et8ia9MmeuCGhdC9jxJ8X69WNlUiPyy0hI/ZsqRq9Z0C2eW0iL +JPXsy4X8Xpw3SFwoXf5pR9RFY5Pb2tuyxqmSestu2VXT/NQjJg4CVDR3mFcHPXZB +4elRzH0WshExEGkgy0bg20MJeRc2Qdb5Xx+EakbmwroDWaCn3NSGqQ7jv6Vw0doy +TGvS6h6RHBxnyqRfRgKGlCoOMG9/5+rFJC00QpCUG2vHXHWGoWlMlJ3foN7rj5v9 +AgMBAAGjDTALMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEBAJ5zt2rj4Ag6 +zpN59AWC1Fur8g8l41ksHkSpKPp+PtyO/ngvbMqBpfmK1e7JCKZv/68QXfMyWWAI +hwalqZkXXWHKjuz3wE7dE25PXFXtGJtcZAaj10xt98fzdqt8lQSwh2kbfNwZIz1F +sgAStgE7+ZTcqTgvNB76Os1UK0to+/P0VBWktaVFdyub4Nc2SdPVnZNvrRBXBwOD +3V8ViwywDOFoE7DvCvwx/SVsvoC0Z4j3AMMovO6oHicP7uU83qsQgm1Qru3YeoLR ++DoVi7IPHbWvN7MqFYn3YjNlByO2geblY7MR0BlqbFlmFrqLsUfjsh2ys7/U/knC +dN/klu446fI= +-----END CERTIFICATE----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-key.pem b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-key.pem new file mode 100644 index 000000000..a1dfd5f78 --- /dev/null +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAnBJxJuYaj9RGxCcnDs/FGjVqdvUHaRVM01oZChoVV5PtJrJj +U8ODQQBQeNpqGOODlPMLw4nXjpSTYT0Vr3g/MxtpccyQPdadFCuHrfImvTJnrgho +XQvY8SfF+vVjZVIj8stISP2bKkavWdAtnltIiyT17MuF/F6cN0hcKF3+aUfURWOT +29rbssapknrLbtlV0/zUIyYOAlQ0d5hXBz12QeHpUcx9FrIRMRBpIMtG4NtDCXkX +NkHW+V8fhGpG5sK6A1mgp9zUhqkO47+lcNHaMkxr0uoekRwcZ8qkX0YChpQqDjBv +f+fqxSQtNEKQlBtrx1x1hqFpTJSd36De64+b/QIDAQABAoIBAFiah66Dt9SruLkn +WR8piUaFyLlcBib8Nq9OWSTJBhDAJERxxb4KIvvGB+l0ZgNXNp5bFPSfzsZdRwZP +PX5uj8Kd71Dxx3mz211WESMJdEC42u+MSmN4lGLkJ5t/sDwXU91E1vbJM0ve8THV +4/Ag9qA4DX2vVZOeyqT/6YHpSsPNZplqzrbAiwrfHwkctHfgqwOf3QLfhmVQgfCS +VwidBldEUv2whSIiIxh4Rv5St4kA68IBCbJxdpOpyuQBkk6CkxZ7VN9FqOuSd4Pk +Wm7iWyBMZsCmELZh5XAXld4BEt87C5R4CvbPBDZxAv3THk1DNNvpy3PFQfwARRFb +SAToYMECgYEAyL7U8yxpzHDYWd3oCx6vTi9p9N/z0FfAkWrRF6dm4UcSklNiT1Aq +EOnTA+SaW8tV3E64gCWcY23gNP8so/ZseWj6L+peHwtchaP9+KB7yGw2A+05+lOx +VetLTjAOmfpiUXFe5w1q4C1RGhLjZjjzW+GvwdAuchQgUEFaomrV+PUCgYEAxwfH +cmVGFbAktcjU4HSRjKSfawCrut+3YUOLybyku3Q/hP9amG8qkVTFe95CTLjLe2D0 +ccaTTpofFEJ32COeck0g0Ujn/qQ+KXRoauOYs4FB1DtqMpqB78wufWEUpDpbd9/h +J+gJdC/IADd4tJW9zA92g8IA7ZtFmqDtiSpQ0ekCgYAQGkaorvJZpN+l7cf0RGTZ +h7IfI2vCVZer0n6tQA9fmLzjoe6r4AlPzAHSOR8sp9XeUy43kUzHKQQoHCPvjw/K +eWJAP7OHF/k2+x2fOPhU7mEy1W+mJdp+wt4Kio5RSaVjVQ3AyPG+w8PSrJszEvRq +dWMMz+851WV2KpfjmWBKlQKBgQC++4j4DZQV5aMkSKV1CIZOBf3vaIJhXKEUFQPD +PmB4fBEjpwCg+zNGp6iktt65zi17o8qMjrb1mtCt2SY04eD932LZUHNFlwcLMmes +Ad+aiDLJ24WJL1f16eDGcOyktlblDZB5gZ/ovJzXEGOkLXglosTfo77OQculmDy2 +/UL2WQKBgGeKasmGNfiYAcWio+KXgFkHXWtAXB9B91B1OFnCa40wx+qnl71MIWQH +PQ/CZFNWOfGiNEJIZjrHsfNJoeXkhq48oKcT0AVCDYyLV0VxDO4ejT95mGW6njNd +JpvmhwwAjOvuWVr0tn4iXlSK8irjlJHmwcRjLTJq97vE9fsA2MjI +-----END RSA PRIVATE KEY----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.crt b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.crt deleted file mode 100644 index 46b1e2a7a..000000000 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDiDCCAnACCQCCsPcIlZO4TDANBgkqhkiG9w0BAQsFADCBhTELMAkGA1UEBhMC -Q0ExGTAXBgNVBAgMEEJyaXRpc2ggQ29sdW1iaWExDjAMBgNVBAcMBUNvbW94MRQw -EgYDVQQKDAtUaGVCcmFpbi5jYTEUMBIGA1UEAwwLdGhlYnJhaW4uY2ExHzAdBgkq -hkiG9w0BCQEWEGluZm9AdGhlYnJhaW4uY2EwHhcNMjEwMTEzMDkwNDIyWhcNMzEw -MTExMDkwNDIyWjCBhTELMAkGA1UEBhMCQ0ExGTAXBgNVBAgMEEJyaXRpc2ggQ29s -dW1iaWExDjAMBgNVBAcMBUNvbW94MRQwEgYDVQQKDAtUaGVCcmFpbi5jYTEUMBIG -A1UEAwwLdGhlYnJhaW4uY2ExHzAdBgkqhkiG9w0BCQEWEGluZm9AdGhlYnJhaW4u -Y2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC2YWuwplM2Hc5tzBMu -covW9nwZ8iNEFo5pbDc8710pmnkF+wsDztLy4afJe6OeVHyCgQxmE+rTZcoWbvoh -pxW3Zy/8es4My07RKHqI3NYadThUvDsmI10cF3tJbhOZaIrMaExLGookZYKwbNAy -7yJ1+MLyNCuFFsaOiNNxHOjH/InKSzEuGSLV68tdC7Pe+uanBcC7RKhOrjUC6Occ -naHPC+a/YMyRYx29T8CfkCBB7N6WanWylFN/1RBmAgq++kDflSaF9k+Zdl6I4jiF -mCPGS0k+AMre4PuAKOZOZOwhF0sWlXIxH6zPm9w0bSYdTLBupL846RTO72NtNP+X -KX5DAgMBAAEwDQYJKoZIhvcNAQELBQADggEBACXXFws+h+Zo9HsxW3BWpl2JU5u6 -KyfbLQt4kSN/gqltd4s84Q8c4z2jNdI0t8Oh5dXTjbLCpFjzuF2tdMtOWeYBCdsQ -4NJ69RrwkFdsSPxDPhSE0WGXPaOBaA92wJjTkVf+UYIek1ozeyWwFm1LPiZVei00 -mwDVgbAbIEb8cf6OqJrl2r5PMBCLWBwwg5aca3fe6TopJhyPA//DZDRPA5xzKb9e -PHUgF3apbcWxuxm8Mts4bAq8BcKoEvLHYWJ4fEWQvXPP7q1jYC3TkpSt5n3FQZTe -nLyQ+RNzsEHzmyOtTSa0Q+5KVluO1TE3ifpv8737pTLdY8t2waBamoboCu8= ------END CERTIFICATE----- diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.key b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.key deleted file mode 100644 index 8bd131632..000000000 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/server.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAtmFrsKZTNh3ObcwTLnKL1vZ8GfIjRBaOaWw3PO9dKZp5BfsL -A87S8uGnyXujnlR8goEMZhPq02XKFm76IacVt2cv/HrODMtO0Sh6iNzWGnU4VLw7 -JiNdHBd7SW4TmWiKzGhMSxqKJGWCsGzQMu8idfjC8jQrhRbGjojTcRzox/yJyksx -Lhki1evLXQuz3vrmpwXAu0SoTq41AujnHJ2hzwvmv2DMkWMdvU/An5AgQezelmp1 -spRTf9UQZgIKvvpA35UmhfZPmXZeiOI4hZgjxktJPgDK3uD7gCjmTmTsIRdLFpVy -MR+sz5vcNG0mHUywbqS/OOkUzu9jbTT/lyl+QwIDAQABAoIBAA6UVR6G/UnrMhBW -6wWghItHov4T/Du6LeJBk1zcqa7kuV4ABo5kXzqpTVdu+dJzYIyyMkKKvw/tKC2I -65f7GmJR7mUZkBU3v3I68Si1tqvgyQMFFRlkZFIVknZ5RTnTQJ08jTTHx1lHgB4I -ZNBdi3ywySzBfOUjv/Wu/HAjZnxuEh2guBpRMZdwQwZLXr2koDa5inL3IwJrA4Ir -QzpZ0y6ql3A0tw7jAw36G1AKyyz74aFwJ0I8U8w+2Uk4iX5hcKGA8mFq4lyO4/3+ -7W2Z4V8cQzwMq2SMixI0Omxlc2BJUi9j17Ey//5dAXyPaG8QI1kzeL/3Gbs8YBMq -ekN8AZECgYEA5YxcFIVv3yO+ARNWUHovrsMuf9ElhyRuZd0I2+vjrq1b9zQsSy2d -PsyYWD17lO/GDmpTzZOdVsYtZHi+EiXmQnkzLJ4m2nlc7W4annWlbzlQMEn6vAji -l9bSHJXXiiIB7X/oHpDUdsnJp/uyAJppmnVLbSBboNCrG4Mf5cJqOnsCgYEAy2We -scp19h4UEKAU0Yh+5jh8W4VVtlISkH64vMgz/JZWXMPt1bM5C/5j+3UVUL5VmFqF -J1g0gXYkTGTL0+entb3SUiL42zrp3rZ3GgMU6V+aktq3dmri5bOifzihuLHLgjO5 -u/MJPBzvFxIiJxnNBybNLijIZfPm+9roUfpcBNkCgYBGE3Zc0WuYnEm5/FRCVzrN -SEqevJOPUSDeuf6lXLryLXxA2E2ZWcCCVmU/su1SR2yYI/+XZ7QFtJRQ8sdbtPQ5 -YNStj05fLeOfnBhGPbYWYVHInB0OYEwEfJFCJsBZLA6YmY6cHiyuYuXMAXuS0ZDh -lWNEWjd+vZUu3fXT52kUlwKBgDgq/eH3GRA4Si41JsqeOPz2iFD1xy+sBnhkpjtr -xf9wvLStXpZvAcfwHkgokxRTG2wRQ0gUMZu2tltqUmdYR5YGr3gDNFnGMSNRnB5Q -z4uK3TLEt3k6FyJ7stoTF4Xbg2mXQylF+jzheJ0UYt4NX/MjofGnTX/qFNVkJFfP -HW4xAoGBAMBb9cXTpzOMiMcSdQRlaLttV1p05pqxTgQNEQD8HB+lkx4AGnnHvtxW -XQJvPumtqdCEpfe4kaqLip8T+67sGfcDVQMogJc/tpvZ0AN4FuViFsf/YDuTPXEp -whMldPHtusbRP2fk/JFq4Ak0Xz2wAI1iMD3qfBeW6eJpvRllUo69 ------END RSA PRIVATE KEY----- diff --git a/apps/emqx_auth_redis/etc/emqx_auth_redis.conf b/apps/emqx_auth_redis/etc/emqx_auth_redis.conf index 77b247a06..5a56c5dce 100644 --- a/apps/emqx_auth_redis/etc/emqx_auth_redis.conf +++ b/apps/emqx_auth_redis/etc/emqx_auth_redis.conf @@ -115,3 +115,17 @@ auth.redis.acl_cmd = HGETALL mqtt_acl:%u ## Value: File #auth.redis.ssl.keyfile = path/to/your/keyfile +## In mode verify_none the default behavior is to allow all x509-path +## validation errors. +## +## Value: true | false +#auth.redis.ssl.verify = false + +## If not specified, the server's names returned in server's certificate is validated against +## what's provided `auth.redis.server` config's host part. +## Setting to 'disable' will make EMQ X ignore unmatched server names. +## If set with a host name, the server's names returned in server's certificate is validated +## against this value. +## +## Value: String | disable +## auth.redis.ssl.server_name_indication = disable \ No newline at end of file diff --git a/apps/emqx_auth_redis/priv/emqx_auth_redis.schema b/apps/emqx_auth_redis/priv/emqx_auth_redis.schema index 070f306af..ea5eb76bb 100644 --- a/apps/emqx_auth_redis/priv/emqx_auth_redis.schema +++ b/apps/emqx_auth_redis/priv/emqx_auth_redis.schema @@ -50,21 +50,27 @@ {datatype, string} ]}. +{mapping, "auth.redis.ssl.verify", "emqx_auth_redis.options", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +{mapping, "auth.redis.ssl.server_name_indication", "emqx_auth_redis.options", [ + {datatype, string} +]}. + %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 {mapping, "auth.redis.cafile", "emqx_auth_redis.options", [ - {default, ""}, {datatype, string} ]}. %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 {mapping, "auth.redis.certfile", "emqx_auth_redis.options", [ - {default, ""}, {datatype, string} ]}. %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 {mapping, "auth.redis.keyfile", "emqx_auth_redis.options", [ - {default, ""}, {datatype, string} ]}. @@ -76,7 +82,7 @@ %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 CA = cuttlefish:conf_get( "auth.redis.ssl.cacertfile", Conf, - cuttlefish:conf_get("auth.redis.cacertfile", Conf, undefined) + cuttlefish:conf_get("auth.redis.cafile", Conf, undefined) ), Cert = cuttlefish:conf_get( "auth.redis.ssl.certfile", Conf, @@ -86,10 +92,21 @@ "auth.redis.ssl.keyfile", Conf, cuttlefish:conf_get("auth.redis.keyfile", Conf, undefined) ), + Verify = case cuttlefish:conf_get("auth.redis.ssl.verify", Conf, false) of + true -> verify_peer; + flase -> verify_none + end, + SNI = case cuttlefish:conf_get("auth.redis.ssl.server_name_indication", Conf, undefined) of + "disable" -> disable; + SNI0 -> SNI0 + end, [{options, [{ssl_options, Filter([{cacertfile, CA}, {certfile, Cert}, - {keyfile, Key}]) + {keyfile, Key}, + {verify, Verify}, + {server_name_indication, SNI} + ]) }]}]; _ -> [{options, []}] end diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl index 35ed79fd8..9fc9e66ef 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl @@ -65,10 +65,11 @@ save_files_return_opts(Options, Dir) -> false -> verify_none; _ -> verify_peer end, + SNI = Get(<<"server_name_indication">>), Versions = emqx_tls_lib:integral_versions(Get(<<"tls_versions">>)), Ciphers = emqx_tls_lib:integral_ciphers(Versions, Get(<<"ciphers">>)), filter([{keyfile, Key}, {certfile, Cert}, {cacertfile, CA}, - {verify, Verify}, {versions, Versions}, {ciphers, Ciphers}]). + {verify, Verify}, {server_name_indication, SNI}, {versions, Versions}, {ciphers, Ciphers}]). %% @doc Save a key or certificate file in data dir, %% and return path of the saved file. diff --git a/apps/emqx_web_hook/etc/emqx_web_hook.conf b/apps/emqx_web_hook/etc/emqx_web_hook.conf index 6c50924ff..7b9d32dfb 100644 --- a/apps/emqx_web_hook/etc/emqx_web_hook.conf +++ b/apps/emqx_web_hook/etc/emqx_web_hook.conf @@ -43,6 +43,15 @@ web.hook.body.encoding_of_payload_field = plain ## Value: true | false ## web.hook.ssl.verify = false +## If not specified, the server's names returned in server's certificate is validated against +## what's provided `web.hook.url` config's host part. +## Setting to 'disable' will make EMQ X ignore unmatched server names. +## If set with a host name, the server's names returned in server's certificate is validated +## against this value. +## +## Value: String | disable +## web.hook.ssl.server_name_indication = disable + ## Connection process pool size ## ## Value: Number diff --git a/apps/emqx_web_hook/priv/emqx_web_hook.schema b/apps/emqx_web_hook/priv/emqx_web_hook.schema index 3a56b8b1d..8ba1cc0fd 100644 --- a/apps/emqx_web_hook/priv/emqx_web_hook.schema +++ b/apps/emqx_web_hook/priv/emqx_web_hook.schema @@ -34,6 +34,10 @@ {datatype, {enum, [true, false]}} ]}. +{mapping, "web.hook.ssl.server_name_indication", "emqx_web_hook.server_name_indication", [ + {datatype, string} +]}. + {mapping, "web.hook.pool_size", "emqx_web_hook.pool_size", [ {default, 32}, {datatype, integer} diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index b670b44b8..f026434c6 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -84,17 +84,23 @@ certfile => #{order => 7, type => file, default => <<"">>, - title =>#{en => <<"SSL Cert">>, - zh => <<"SSL Cert"/utf8>>}, + title => #{en => <<"SSL Cert">>, + zh => <<"SSL Cert"/utf8>>}, description => #{en => <<"Your ssl certfile">>, zh => <<"SSL 证书"/utf8>>}}, verify => #{order => 8, type => boolean, default => false, - title =>#{en => <<"Verify Server Certfile">>, - zh => <<"校验服务器证书"/utf8>>}, + title => #{en => <<"Verify Server Certfile">>, + zh => <<"校验服务器证书"/utf8>>}, description => #{en => <<"Whether to verify the server certificate. By default, the client will not verify the server's certificate. If verification is required, please set it to true.">>, - zh => <<"是否校验服务器证书。 默认客户端不会去校验服务器的证书，如果需要校验，请设置成true。"/utf8>>}} + zh => <<"是否校验服务器证书。 默认客户端不会去校验服务器的证书，如果需要校验，请设置成true。"/utf8>>}}, + server_name_indication => #{order => 9, + type => string, + title => #{en => <<"Server Name Indication">>, + zh => <<"服务器名称指示"/utf8>>}, + description => #{en => <<"Specify the hostname used for peer certificate verification, or set to disable to turn off this verification.">>, + zh => <<"指定用于对端证书验证时使用的主机名，或者设置为 disable 以关闭此项验证。"/utf8>>}} }). -define(ACTION_PARAM_RESOURCE, #{ diff --git a/apps/emqx_web_hook/src/emqx_web_hook_app.erl b/apps/emqx_web_hook/src/emqx_web_hook_app.erl index 67775e00f..c75c7cb01 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_app.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_app.erl @@ -60,11 +60,18 @@ translate_env() -> true -> verify_peer; false -> verify_none end, + SNI = case application:get_env(?APP, server_name_indication, undefined) of + "disable" -> disable; + SNI0 -> SNI0 + end, TLSOpts = lists:filter(fun({_K, V}) -> V /= <<>> andalso V /= undefined andalso V /= "" andalso true - end, [{keyfile, KeyFile}, {certfile, CertFile}, {cacertfile, CACertFile}]), - NTLSOpts = [ {verify, VerifyType} - , {versions, emqx_tls_lib:default_versions()} + end, [{keyfile, KeyFile}, + {certfile, CertFile}, + {cacertfile, CACertFile}, + {verify, VerifyType}, + {server_name_indication, SNI}]), + NTLSOpts = [ {versions, emqx_tls_lib:default_versions()} , {ciphers, emqx_tls_lib:default_ciphers()} | TLSOpts ], diff --git a/scripts/elvis-check.sh b/scripts/elvis-check.sh index f15b59b1e..5fe482865 100755 --- a/scripts/elvis-check.sh +++ b/scripts/elvis-check.sh @@ -19,7 +19,7 @@ echo "elvis -v: $elvis_version" echo "git diff base: $base" if [ ! -f ./elvis ] || [ "$(./elvis -v | grep -oE '[1-9]+\.[0-9]+\.[0-9]+\-emqx-[0-9]+')" != "$elvis_version" ]; then - curl -fLO "https://github.com/emqx/elvis/releases/download/$elvis_version/elvis" + curl --silent --show-error -fLO "https://github.com/emqx/elvis/releases/download/$elvis_version/elvis" chmod +x ./elvis fi @@ -27,7 +27,11 @@ if [[ "$base" =~ [0-9a-f]{8,40} ]]; then # base is a commit sha1 compare_base="$base" else - remote="$(git remote -v | grep -E 'github\.com(.|/)emqx' | grep fetch | awk '{print $1}')" + if [[ $CI == true ]];then + remote="$(git remote -v | grep -E "github\.com(.|/)$GITHUB_REPOSITORY" | grep fetch | awk '{print $1}')" + else + remote="$(git remote -v | grep -E 'github\.com(.|/)emqx' | grep fetch | awk '{print $1}')" + fi git fetch "$remote" "$base" compare_base="$remote/$base" fi From 1086c09242a805795359617b66ddaeb3b1aa71d2 Mon Sep 17 00:00:00 2001 From: "ayodele.akingbule" Date: Mon, 22 Mar 2021 22:44:42 +0100 Subject: [PATCH 089/158] docs(config): CSWSH(Cross-Site Web-Socket Hijack) Documentation --- etc/emqx.conf | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/etc/emqx.conf b/etc/emqx.conf index 4a953ddb4..812fe2159 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1727,6 +1727,28 @@ listener.ws.external.nodelay = true ## Value: single | multiple listener.ws.external.mqtt_piggyback = multiple +## By default, EMQX web socket connection does not restrict connections to specific origins. +## It also, by default, does not enforce the presence of origin in request headers for WebSocket connections. +## Because of this, a malicious user could potentially hijack an existing web-socket connection to EMQX. + +## To prevent this, users can set allowed origin headers in their ws connection to EMQX. +## WS configs are set in listener.ws.external.* +## WSS configs are set in listener.wss.external.* + +## Example for WS connection +## To enables origin check in header for websocket connnection, +## set `listener.ws.external.check_origin_enable = true`. By default it is false, +## When it is set to true and no origin is present in the header of a ws connection request, the request fails. + +## To allow origins to be absent in header in the websocket connection when check_origin_enable is true, +## set `listener.ws.external.allow_origin_absence = true` + +## Enabling origin check implies there are specific valid origins allowed for ws connection. +## To set the list of allowed origins in header for websocket connection +## listener.ws.external.check_origins = http://localhost:18083(localhost dashboard url), http://yourapp.com` +## check_origins config allows a comma separated list of origins so you can specify as many origins are you want. +## With these configs, you can allow only connections from only authorized origins to your broker + ## Enable origin check in header for websocket connection ## ## Value: true | false (default false) From 622edcd5366238ccdc4713cbed5b5ba4321f50e2 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Mon, 22 Mar 2021 19:34:23 +0800 Subject: [PATCH 090/158] chore(CI): rename arch package name --- build | 16 +++++++++++++++- deploy/packages/deb/Makefile | 5 +++-- deploy/packages/rpm/Makefile | 6 +++--- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/build b/build index d7cd01183..c4f57d4ea 100755 --- a/build +++ b/build @@ -28,6 +28,20 @@ elif [ "$(uname -s)" = 'Linux' ]; then SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')" fi +ARCH="$(uname -m)" +case "$ARCH" in + x86_64) + ARCH='amd64' + ;; + aarch64) + ARCH='arm64' + ;; + arm*) + ARCH=arm + ;; +esac +export ARCH + ## ## Support RPM and Debian based linux systems ## @@ -95,7 +109,7 @@ make_zip() { log "ERROR: $tarball is not found" fi local zipball - zipball="${pkgpath}/${PROFILE}-${SYSTEM}-${PKG_VSN}-$(uname -m).zip" + zipball="${pkgpath}/${PROFILE}-${SYSTEM}-${PKG_VSN}-${ARCH}.zip" tar zxf "${tarball}" -C "${tard}/emqx" (cd "${tard}" && zip -qr - emqx) > "${zipball}" } diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 28799a21d..48124d780 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -1,5 +1,6 @@ -# Keep this short to avoid bloating beam files with long file path info +ARCH ?= amd64 TOPDIR := /tmp/emqx +# Keep this short to avoid bloating beam files with long file path info SRCDIR := $(TOPDIR)/$(PKG_VSN) BUILT := $(SRCDIR)/BUILT @@ -7,7 +8,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) -TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(shell uname -m) +TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) .PHONY: all all: | $(BUILT) diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index 780a4bbd7..ae4bd37df 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -1,4 +1,5 @@ # Keep this short to avoid bloating beam files with long file path info +ARCH ?= amd64 TOPDIR := /tmp/emqx SRCDIR := $(TOPDIR)/$(PKG_VSN) BUILT := $(SRCDIR)/BUILT @@ -8,7 +9,6 @@ space := $(none) $(none) RPM_VSN ?= $(shell echo $(PKG_VSN) | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+)?") RPM_REL ?= $(shell echo $(PKG_VSN) | grep -oE "(alpha|beta|rc)\.[0-9]") -ARCH:=$(shell uname -m) ifeq ($(ARCH),mips64) ARCH:=mips64el endif @@ -21,7 +21,7 @@ ifeq ($(RPM_REL),) # no tail RPM_REL := 1 endif -SOURCE_PKG := emqx-$(SYSTEM)-$(RPM_VSN)-$(RPM_REL).$(ARCH) +SOURCE_PKG := emqx-$(SYSTEM)-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C @@ -51,7 +51,7 @@ all: | $(BUILT) --define "_sharedstatedir /var/lib" \ emqx.spec mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) - cp $(TOPDIR)/RPMS/$(ARCH)/$(SOURCE_PKG).rpm $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm + cp $(TOPDIR)/RPMS/$(shell uname -m)/$(SOURCE_PKG).rpm $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm $(BUILT): mkdir -p $(TOPDIR) $(SRCDIR) $(SRCDIR)/BUILT From 83b99c09a4b318e4a8257bdea322fff9385697ad Mon Sep 17 00:00:00 2001 From: wwhai Date: Tue, 23 Mar 2021 13:38:37 +0800 Subject: [PATCH 091/158] fix(import): add bridge_mqtt compatible <4.2.x --- apps/emqx_management/include/emqx_mgmt.hrl | 2 +- .../src/emqx_mgmt_data_backup.erl | 38 +++++++++++++++---- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/apps/emqx_management/include/emqx_mgmt.hrl b/apps/emqx_management/include/emqx_mgmt.hrl index 469820b66..e3e1f9fce 100644 --- a/apps/emqx_management/include/emqx_mgmt.hrl +++ b/apps/emqx_management/include/emqx_mgmt.hrl @@ -32,4 +32,4 @@ -define(ERROR14, 114). %% OldPassword error -define(ERROR15, 115). %% bad topic --define(VERSIONS, ["1", "3.2", "3.4", "4.0", "4.1", "4.2"]). \ No newline at end of file +-define(VERSIONS, ["1", "3.2", "3.4", "4.0", "4.1", "4.2", "4.3"]). \ No newline at end of file diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 009b4c888..58f4ab28b 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -212,8 +212,8 @@ map_to_action(Map = #{<<"id">> := ActionInstId, <<"name">> := Name, <<"args">> : -endif. import_rules(Rules) -> - lists:foreach(fun(Resource) -> - import_resource(Resource) + lists:foreach(fun(Rule) -> + import_rule(Rule) end, Rules). import_resources(Reources) -> @@ -242,7 +242,9 @@ import_resources_and_rules(Resources, Rules, _FromVersion) -> import_rules(Rules). -else. import_resources_and_rules(Resources, Rules, FromVersion) - when FromVersion =:= "4.0" orelse FromVersion =:= "4.1" orelse FromVersion =:= "4.2" -> + when FromVersion =:= "4.0" orelse + FromVersion =:= "4.1" orelse + FromVersion =:= "4.2" -> Configs = lists:foldl(fun compatible_version/2 , [], Resources), lists:foreach(fun(#{<<"actions">> := Actions} = Rule) -> NActions = apply_new_config(Actions, Configs), @@ -253,7 +255,6 @@ import_resources_and_rules(Resources, Rules, _FromVersion) -> import_resources(Resources), import_rules(Rules). - %% 4.2.5 + compatible_version(#{<<"id">> := ID, <<"type">> := <<"web_hook">>, @@ -284,9 +285,30 @@ compatible_version(#{<<"id">> := ID, Cfg = make_new_config(#{<<"method">> => Method, <<"url">> => URL}), {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}), - NHeaders = maps:put(<<"content-type">>, <<"application/json">> , Headers), + NHeaders = maps:put(<<"content-type">>, <<"application/json">> , + case Headers of + [] -> #{}; + Other -> Other + end), [{ID, #{headers => NHeaders, method => Method}} | Acc]; -% 4.2.3 + +%% bridge mqtt +%% 4.2.0 - 4.2.5 bridge_mqtt, ssl enabled from on/off to true/false +compatible_version(#{<<"type">> := <<"bridge_mqtt">>, + <<"id">> := ID, %% begin 4.2.0. + <<"config">> := #{<<"ssl">> := Ssl} = Config} = Resource, Acc) -> + F = fun(B) -> + case B of + <<"on">> -> true; + <<"off">> -> false; + Other -> Other + end + end, + NewConfig = Config#{<<"ssl">> := F(Ssl)}, + {ok, _Resource} = import_resource(Resource#{<<"config">> := NewConfig}), + [{ID, NewConfig} | Acc]; + +% 4.2.3, add :content_type compatible_version(#{<<"id">> := ID, <<"type">> := <<"web_hook">>, <<"config">> := #{<<"headers">> := Headers, @@ -310,7 +332,7 @@ make_new_config(Cfg) -> <<"cacertfile">> => <<>>, <<"certfile">> => <<>>, <<"keyfile">> => <<>>, - <<"verify">> => true}, + <<"verify">> => false}, maps:merge(Cfg, Config). apply_new_config(Actions, Configs) -> @@ -597,7 +619,7 @@ import(Filename) -> logger:debug("The emqx data has been imported successfully"), ok catch Class:Reason:Stack -> - logger:error("The emqx data import failed: ~0p", [{Class,Reason,Stack}]), + logger:error("The emqx data import failed: ~0p", [{Class, Reason, Stack}]), {error, import_failed} end; false -> From b9c913d18f4632ab4147588b51aa3d5d34019efd Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 23 Mar 2021 16:40:23 +0800 Subject: [PATCH 092/158] chore(CI): fix windows build error --- .github/workflows/build_packages.yaml | 11 ++++++++--- .github/workflows/run_fvt_tests.yaml | 1 - deploy/packages/rpm/Makefile | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 8a7f77498..ecd672a98 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -55,6 +55,7 @@ jobs: runs-on: windows-2019 needs: prepare + if: endsWith(github.repository, 'emqx') strategy: matrix: @@ -91,16 +92,20 @@ jobs: cd source ## We do not build/release bcrypt for windows package Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ - Remove-Item -Force -Path rebar.lock + if (Test-Path rebar.lock) { + Remove-Item -Force -Path rebar.lock + } make ${{ matrix.profile }} mkdir -p _packages/${{ matrix.profile }} Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name mv _build/${{ matrix.profile }}/rel/$pkg_name _packages/${{ matrix.profile }} Get-FileHash -Path "_packages/${{ matrix.profile }}/$pkg_name" | Format-List | grep 'Hash' | awk '{print $3}' > _packages/${{ matrix.profile }}/$pkg_name.sha256 - name: run emqx + timeout-minutes: 1 run: | cd source ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start + Start-Sleep -s 5 ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall @@ -348,7 +353,7 @@ jobs: delete-artifact: runs-on: ubuntu-20.04 - needs: [prepare, windows, mac, linux, docker] + needs: [prepare, mac, linux, docker] steps: - uses: geekyeggo/delete-artifact@v1 with: @@ -359,7 +364,7 @@ jobs: if: startsWith(github.ref, 'refs/tags/') - needs: [prepare, windows, mac, linux, docker] + needs: [prepare, mac, linux, docker] strategy: matrix: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index a3568350a..6aa1e6844 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -138,7 +138,6 @@ jobs: emqx2=$(kubectl get pods emqx-2 -o jsonpath='{.status.podIP}') pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host $emqx_svc - pytest -v paho.mqtt.testing/interoperability/test_client --host $emqx_svc pytest -v paho.mqtt.testing/interoperability/test_cluster --host1 $emqx1 --host2 $emqx2 relup_test: diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index ae4bd37df..5a6e6bee4 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -1,5 +1,4 @@ # Keep this short to avoid bloating beam files with long file path info -ARCH ?= amd64 TOPDIR := /tmp/emqx SRCDIR := $(TOPDIR)/$(PKG_VSN) BUILT := $(SRCDIR)/BUILT @@ -9,6 +8,7 @@ space := $(none) $(none) RPM_VSN ?= $(shell echo $(PKG_VSN) | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+)?") RPM_REL ?= $(shell echo $(PKG_VSN) | grep -oE "(alpha|beta|rc)\.[0-9]") +ARCH ?= amd64 ifeq ($(ARCH),mips64) ARCH:=mips64el endif From 09c4f6eef099284c980f5cd164a0b8ab7bce06b0 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 23 Mar 2021 14:26:56 +0100 Subject: [PATCH 093/158] chore(boot): Do not ps -ef inspect node name In case there are multiple nodes running the current code (before this commit) will fail If we choose to fix, it may stop the wrong node --- bin/emqx | 6 ------ bin/emqx_ctl | 8 +------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/bin/emqx b/bin/emqx index 4901c816a..a2c8a5be9 100755 --- a/bin/emqx +++ b/bin/emqx @@ -262,9 +262,6 @@ fi if [ -z "$NAME_ARG" ]; then NODENAME="${EMQX_NODE_NAME:-}" - # check if there is a node running, inspect its name - # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions - [ -z "$NODENAME" ] && NODENAME=$(ps -ef | grep -E '\-progname\s.*emqx\s' | grep -o -E '\-name (\S*)' | awk '{print $2}') [ -z "$NODENAME" ] && NODENAME=$(grep -E '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-) if [ -z "$NODENAME" ]; then echoerr "vm.args needs to have a -name parameter." @@ -287,9 +284,6 @@ PIPE_DIR="${PIPE_DIR:-/$RUNNER_DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}" # Extract the target cookie if [ -z "$COOKIE_ARG" ]; then COOKIE="${EMQX_NODE_COOKIE:-}" - # check if there is a node running, steal its cookie - # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions - [ -z "$COOKIE" ] && COOKIE=$(ps -ef | grep -E '\-progname\s.*emqx\s' | grep -o -E '\-setcookie (\S*)' | awk '{print $2}') [ -z "$COOKIE" ] && COOKIE=$(grep -E '^[ \t]*node.cookie[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-) if [ -z "$COOKIE" ]; then echoerr "vm.args needs to have a -setcookie parameter." diff --git a/bin/emqx_ctl b/bin/emqx_ctl index 5fa35c7a5..1c8496a7f 100755 --- a/bin/emqx_ctl +++ b/bin/emqx_ctl @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # -*- tab-width:4;indent-tabs-mode:nil -*- # ex: ts=4 sw=4 et @@ -34,9 +34,6 @@ relx_nodetool() { if [ -z "$NAME_ARG" ]; then NODENAME="${EMQX_NODE_NAME:-}" - # check if there is a node running, inspect its name - # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions - [ -z "$NODENAME" ] && NODENAME=$(ps -ef | grep -E '\progname\s.*emqx\s' | grep -o -E '\-name (\S*)' | awk '{print $2}') [ -z "$NODENAME" ] && NODENAME=$(grep -E '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-) if [ -z "$NODENAME" ]; then echoerr "vm.args needs to have a -name parameter." @@ -55,9 +52,6 @@ NAME="$(echo "$NAME_ARG" | awk '{print $2}')" # Extract the target cookie if [ -z "$COOKIE_ARG" ]; then COOKIE="${EMQX_NODE_COOKIE:-}" - # check if there is a node running, steal its cookie - # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions - [ -z "$COOKIE" ] && COOKIE=$(ps -ef | grep -E '\-progname\s.*emqx\s' | grep -o -E '\-setcookie (\S*)' | awk '{print $2}') [ -z "$COOKIE" ] && COOKIE=$(grep -E '^[ \t]*node.cookie[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-) if [ -z "$COOKIE" ]; then echoerr "vm.args needs to have a -setcookie parameter." From 3ea8039d90ee5eb31bbfb91764becafd828fd32a Mon Sep 17 00:00:00 2001 From: wwhai Date: Tue, 23 Mar 2021 21:30:27 +0800 Subject: [PATCH 094/158] fix(import): fix import error --- .../src/emqx_mgmt_data_backup.erl | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 58f4ab28b..51bd088bd 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -268,13 +268,12 @@ compatible_version(#{<<"id">> := ID, CovertFun = fun(Int) -> list_to_binary(integer_to_list(Int) ++ "s") end, - Cfg = make_new_config(#{<<"method">> => Method, - <<"pool_size">> => PoolSize, + Cfg = make_new_config(#{<<"pool_size">> => PoolSize, <<"connect_timeout">> => CovertFun(ConnectTimeout), <<"request_timeout">> => CovertFun(RequestTimeout), <<"url">> => URL}), {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}), - NHeaders = maps:put(<<"content-type">>, ContentType, Headers), + NHeaders = maps:put(<<"content-type">>, ContentType, covert_empty_headers(Headers)), [{ID, #{headers => NHeaders, method => Method}} | Acc]; % 4.2.0 compatible_version(#{<<"id">> := ID, @@ -282,14 +281,9 @@ compatible_version(#{<<"id">> := ID, <<"config">> := #{<<"headers">> := Headers, <<"method">> := Method,%% 4.2.0 Different here <<"url">> := URL}} = Resource, Acc) -> - Cfg = make_new_config(#{<<"method">> => Method, - <<"url">> => URL}), + Cfg = make_new_config(#{<<"url">> => URL}), {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}), - NHeaders = maps:put(<<"content-type">>, <<"application/json">> , - case Headers of - [] -> #{}; - Other -> Other - end), + NHeaders = maps:put(<<"content-type">>, <<"application/json">> , covert_empty_headers(Headers)), [{ID, #{headers => NHeaders, method => Method}} | Acc]; %% bridge mqtt @@ -315,10 +309,9 @@ compatible_version(#{<<"id">> := ID, <<"content_type">> := ContentType,%% 4.2.3 Different here <<"method">> := Method, <<"url">> := URL}} = Resource, Acc) -> - Cfg = make_new_config(#{<<"method">> => Method, - <<"url">> => URL}), + Cfg = make_new_config(#{<<"url">> => URL}), {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}), - NHeaders = maps:put(<<"content-type">>, ContentType, Headers), + NHeaders = maps:put(<<"content-type">>, ContentType, covert_empty_headers(Headers)), [{ID, #{headers => NHeaders, method => Method}} | Acc]; % normal version compatible_version(Resource, Acc) -> @@ -327,8 +320,8 @@ compatible_version(Resource, Acc) -> make_new_config(Cfg) -> Config = #{<<"pool_size">> => 8, - <<"connect_timeout">> => <<"3s">>, - <<"request_timeout">> => <<"3s">>, + <<"connect_timeout">> => <<"5s">>, + <<"request_timeout">> => <<"5s">>, <<"cacertfile">> => <<>>, <<"certfile">> => <<>>, <<"keyfile">> => <<>>, @@ -649,3 +642,9 @@ do_import_extra_data(Data, _Version) -> -else. do_import_extra_data(_Data, _Version) -> ok. -endif. + +covert_empty_headers(Headers) -> + case Headers of + [] -> #{}; + Other -> Other + end. From 78afdd8b576b0aa51d0473a53d7930383461a97f Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 23 Mar 2021 15:22:37 +0100 Subject: [PATCH 095/158] chore(script): make emqx_ctl work with docker entrypoint --- bin/emqx_ctl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/emqx_ctl b/bin/emqx_ctl index 1c8496a7f..7acb61782 100755 --- a/bin/emqx_ctl +++ b/bin/emqx_ctl @@ -34,6 +34,8 @@ relx_nodetool() { if [ -z "$NAME_ARG" ]; then NODENAME="${EMQX_NODE_NAME:-}" + # compatible with docker entrypoint + [ -z "$NODENAME" ] && [ -n "$EMQX_NAME" ] && [ -n "$EMQX_HOST" ] && NODENAME="${EMQX_NAME}@${EMQX_HOST}" [ -z "$NODENAME" ] && NODENAME=$(grep -E '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-) if [ -z "$NODENAME" ]; then echoerr "vm.args needs to have a -name parameter." From 106b738a89c81ac7637aeab54efa5806d1e4fea9 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 23 Mar 2021 22:46:41 +0800 Subject: [PATCH 096/158] chore(script): make emqx work with docker entrypoint --- .ci/fvt_tests/docker-compose.yaml | 2 +- bin/emqx | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.ci/fvt_tests/docker-compose.yaml b/.ci/fvt_tests/docker-compose.yaml index a71b910eb..00b5712bc 100644 --- a/.ci/fvt_tests/docker-compose.yaml +++ b/.ci/fvt_tests/docker-compose.yaml @@ -46,7 +46,7 @@ services: sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins /opt/emqx/bin/emqx foreground healthcheck: - test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"] + test: ["CMD", "/opt/emqx/bin/emqx", "ping"] interval: 5s timeout: 25s retries: 5 diff --git a/bin/emqx b/bin/emqx index a2c8a5be9..167cd5cf1 100755 --- a/bin/emqx +++ b/bin/emqx @@ -262,6 +262,8 @@ fi if [ -z "$NAME_ARG" ]; then NODENAME="${EMQX_NODE_NAME:-}" + # compatible with docker entrypoint + [ -z "$NODENAME" ] && [ -n "$EMQX_NAME" ] && [ -n "$EMQX_HOST" ] && NODENAME="${EMQX_NAME}@${EMQX_HOST}" [ -z "$NODENAME" ] && NODENAME=$(grep -E '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-) if [ -z "$NODENAME" ]; then echoerr "vm.args needs to have a -name parameter." From b732d87da996526a23f620c49071a598ac907489 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 23 Mar 2021 22:52:35 +0800 Subject: [PATCH 097/158] chore(script): delete emqx_env --- bin/emqx_env | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100755 bin/emqx_env diff --git a/bin/emqx_env b/bin/emqx_env deleted file mode 100755 index 6c89be240..000000000 --- a/bin/emqx_env +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -[ "x" = "x$EMQX_NODE_NAME" ] && EMQX_NODE_NAME=emqx@127.0.0.1 -[ "x" = "x$EMQX_NODE_COOKIE" ] && EMQX_NODE_COOKIE=emqxsecretcookie -[ "x" = "x$EMQX_MAX_PACKET_SIZE" ] && EMQX_MAX_PACKET_SIZE=64KB -[ "x" = "x$EMQX_MAX_PORTS" ] && EMQX_MAX_PORTS=65536 -[ "x" = "x$EMQX_TCP_PORT" ] && EMQX_TCP_PORT=1883 -[ "x" = "x$EMQX_SSL_PORT" ] && EMQX_SSL_PORT=8883 -[ "x" = "x$EMQX_WS_PORT" ] && EMQX_WS_PORT=8083 -[ "x" = "x$EMQX_WSS_PORT" ] && EMQX_WSS_PORT=8084 - From 4ce7229382b07ae67e03d7b715bb967923d6a7ad Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 23 Mar 2021 23:09:38 +0800 Subject: [PATCH 098/158] chore(release): update emqx release version --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index b2c93a2bb..1acc6a70e 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3-beta.1"}). +-define(EMQX_RELEASE, {opensource, "4.3-rc.1"}). -else. From 7e3b7bb0fefeb0935508c21c04b800810d463b8e Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 24 Mar 2021 00:18:40 +0800 Subject: [PATCH 099/158] chore(CI): fix pkg-vsn error --- pkg-vsn.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg-vsn.sh b/pkg-vsn.sh index 9e8c26320..9da5a1075 100755 --- a/pkg-vsn.sh +++ b/pkg-vsn.sh @@ -14,7 +14,7 @@ fi RELEASE="$(grep -E "define.+EMQX_RELEASE.+${EDITION}" include/emqx_release.hrl | cut -d '"' -f2)" -if [ -d .git ] && ! git describe --tags --match "${RELEASE}" --exact >/dev/null 2>&1; then +if [ -d .git ] && ! git describe --tags --match "[e|v]${RELEASE}" --exact >/dev/null 2>&1; then SUFFIX="-$(git rev-parse HEAD | cut -b1-8)" fi From 4c5273c1d16b3bf802bb74881b446b8da6aecd77 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 23 Mar 2021 19:43:51 +0100 Subject: [PATCH 100/158] style(emqx_inflight): elvis: User UPPER_CASE macro --- src/emqx_inflight.erl | 48 +++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/emqx_inflight.erl b/src/emqx_inflight.erl index 61b5c3e2e..c94a8cc48 100644 --- a/src/emqx_inflight.erl +++ b/src/emqx_inflight.erl @@ -45,83 +45,83 @@ -opaque(inflight() :: {inflight, max_size(), gb_trees:tree()}). --define(Inflight(Tree), {inflight, _MaxSize, Tree}). +-define(INFLIGHT(Tree), {inflight, _MaxSize, Tree}). --define(Inflight(MaxSize, Tree), {inflight, MaxSize, (Tree)}). +-define(INFLIGHT(MaxSize, Tree), {inflight, MaxSize, (Tree)}). -spec(new() -> inflight()). new() -> new(0). -spec(new(non_neg_integer()) -> inflight()). new(MaxSize) when MaxSize >= 0 -> - ?Inflight(MaxSize, gb_trees:empty()). + ?INFLIGHT(MaxSize, gb_trees:empty()). -spec(contain(key(), inflight()) -> boolean()). -contain(Key, ?Inflight(Tree)) -> +contain(Key, ?INFLIGHT(Tree)) -> gb_trees:is_defined(Key, Tree). -spec(lookup(key(), inflight()) -> {value, term()} | none). -lookup(Key, ?Inflight(Tree)) -> +lookup(Key, ?INFLIGHT(Tree)) -> gb_trees:lookup(Key, Tree). -spec(insert(key(), Val :: term(), inflight()) -> inflight()). -insert(Key, Val, ?Inflight(MaxSize, Tree)) -> - ?Inflight(MaxSize, gb_trees:insert(Key, Val, Tree)). +insert(Key, Val, ?INFLIGHT(MaxSize, Tree)) -> + ?INFLIGHT(MaxSize, gb_trees:insert(Key, Val, Tree)). -spec(delete(key(), inflight()) -> inflight()). -delete(Key, ?Inflight(MaxSize, Tree)) -> - ?Inflight(MaxSize, gb_trees:delete(Key, Tree)). +delete(Key, ?INFLIGHT(MaxSize, Tree)) -> + ?INFLIGHT(MaxSize, gb_trees:delete(Key, Tree)). -spec(update(key(), Val :: term(), inflight()) -> inflight()). -update(Key, Val, ?Inflight(MaxSize, Tree)) -> - ?Inflight(MaxSize, gb_trees:update(Key, Val, Tree)). +update(Key, Val, ?INFLIGHT(MaxSize, Tree)) -> + ?INFLIGHT(MaxSize, gb_trees:update(Key, Val, Tree)). -spec(resize(integer(), inflight()) -> inflight()). -resize(MaxSize, ?Inflight(Tree)) -> - ?Inflight(MaxSize, Tree). +resize(MaxSize, ?INFLIGHT(Tree)) -> + ?INFLIGHT(MaxSize, Tree). -spec(is_full(inflight()) -> boolean()). -is_full(?Inflight(0, _Tree)) -> +is_full(?INFLIGHT(0, _Tree)) -> false; -is_full(?Inflight(MaxSize, Tree)) -> +is_full(?INFLIGHT(MaxSize, Tree)) -> MaxSize =< gb_trees:size(Tree). -spec(is_empty(inflight()) -> boolean()). -is_empty(?Inflight(Tree)) -> +is_empty(?INFLIGHT(Tree)) -> gb_trees:is_empty(Tree). -spec(smallest(inflight()) -> {key(), term()}). -smallest(?Inflight(Tree)) -> +smallest(?INFLIGHT(Tree)) -> gb_trees:smallest(Tree). -spec(largest(inflight()) -> {key(), term()}). -largest(?Inflight(Tree)) -> +largest(?INFLIGHT(Tree)) -> gb_trees:largest(Tree). -spec(values(inflight()) -> list()). -values(?Inflight(Tree)) -> +values(?INFLIGHT(Tree)) -> gb_trees:values(Tree). -spec(to_list(inflight()) -> list({key(), term()})). -to_list(?Inflight(Tree)) -> +to_list(?INFLIGHT(Tree)) -> gb_trees:to_list(Tree). -spec(to_list(fun(), inflight()) -> list({key(), term()})). -to_list(SortFun, ?Inflight(Tree)) -> +to_list(SortFun, ?INFLIGHT(Tree)) -> lists:sort(SortFun, gb_trees:to_list(Tree)). -spec(window(inflight()) -> list()). -window(Inflight = ?Inflight(Tree)) -> +window(Inflight = ?INFLIGHT(Tree)) -> case gb_trees:is_empty(Tree) of true -> []; false -> [Key || {Key, _Val} <- [smallest(Inflight), largest(Inflight)]] end. -spec(size(inflight()) -> non_neg_integer()). -size(?Inflight(Tree)) -> +size(?INFLIGHT(Tree)) -> gb_trees:size(Tree). -spec(max_size(inflight()) -> non_neg_integer()). -max_size(?Inflight(MaxSize, _Tree)) -> +max_size(?INFLIGHT(MaxSize, _Tree)) -> MaxSize. From dee375d4db5a03bd6083080e1eb789321ad43e9a Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 23 Mar 2021 19:49:00 +0100 Subject: [PATCH 101/158] chore(config): delete unused template in emqx.conf the bbmustache template additional_configs is no longer used --- etc/emqx.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 812fe2159..70320e91b 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -2349,5 +2349,3 @@ alarm.size_limit = 1000 alarm.validity_period = 24h ## CONFIG_SECTION_END=sys_mon ================================================== - -{{ additional_configs }} From a4b30ea77cda182b2e158594af5c5a4f1236a15e Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 23 Mar 2021 19:58:22 +0100 Subject: [PATCH 102/158] feat(emqx): Add backtrace_depth configuration --- etc/emqx.conf | 2 ++ priv/emqx.schema | 5 +++++ src/emqx_app.erl | 9 +++++++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 70320e91b..1eebed652 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -301,6 +301,8 @@ node.crash_dump = {{ platform_log_dir }}/crash.dump node.dist_listen_min = 6369 node.dist_listen_max = 6369 +node.backtrace_depth = 16 + ## CONFIG_SECTION_BGN=rpc ====================================================== ## RPC Mode. diff --git a/priv/emqx.schema b/priv/emqx.schema index 2e1248c50..d254c24e6 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -334,6 +334,11 @@ end}. hidden ]}. +{mapping, "node.backtrace_depth", "emqx.backtrace_depth", [ + {default, 16}, + {datatype, integer} +]}. + %%-------------------------------------------------------------------- %% RPC %%-------------------------------------------------------------------- diff --git a/src/emqx_app.erl b/src/emqx_app.erl index ca1a2ce09..ca6ce1073 100644 --- a/src/emqx_app.erl +++ b/src/emqx_app.erl @@ -33,6 +33,7 @@ %%-------------------------------------------------------------------- start(_Type, _Args) -> + set_backtrace_depth(), print_otp_version_warning(), print_banner(), ekka:start(), @@ -40,8 +41,7 @@ start(_Type, _Args) -> ok = start_autocluster(), ok = emqx_plugins:init(), _ = emqx_plugins:load(), - emqx_boot:is_enabled(listeners) - andalso (ok = emqx_listeners:start()), + emqx_boot:is_enabled(listeners) andalso (ok = emqx_listeners:start()), register(emqx, self()), ok = emqx_alarm_handler:load(), print_vsn(), @@ -53,6 +53,11 @@ stop(_State) -> emqx_boot:is_enabled(listeners) andalso emqx_listeners:stop(). +set_backtrace_depth() -> + Depth = application:get_env(?APP, backtrace_depth, 16), + _ = erlang:system_flag(backtrace_depth, Depth), + ok. + %%-------------------------------------------------------------------- %% Print Banner %%-------------------------------------------------------------------- From 4ee0dbdea44e19b9a0c4e07264f74dda4750e2e2 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 24 Mar 2021 11:55:58 +0100 Subject: [PATCH 103/158] fix(scripts): Finds node name and cookie from vm.args For node name and cookie overriden from environment variable the only way to find it from another shell is to inspect the vm.args file. For node boot commands, the vm.args file may not have been created yet, so we need to inspect node name in emqx.conf --- bin/emqx | 61 ++++++++++++++++++++++++++++++++++++++-------------- bin/emqx_ctl | 28 +++++++++++------------- 2 files changed, 57 insertions(+), 32 deletions(-) diff --git a/bin/emqx b/bin/emqx index 167cd5cf1..69994389b 100755 --- a/bin/emqx +++ b/bin/emqx @@ -260,15 +260,43 @@ if [ -z "$RELX_CONFIG_PATH" ]; then fi fi +IS_BOOT_COMMAND='no' +case "$1" in + start|start_boot) + IS_BOOT_COMMAND='yes' + ;; + console|console_clean|console_boot) + IS_BOOT_COMMAND='yes' + ;; + foreground) + IS_BOOT_COMMAND='yes' + ;; +esac + + if [ -z "$NAME_ARG" ]; then NODENAME="${EMQX_NODE_NAME:-}" # compatible with docker entrypoint [ -z "$NODENAME" ] && [ -n "$EMQX_NAME" ] && [ -n "$EMQX_HOST" ] && NODENAME="${EMQX_NAME}@${EMQX_HOST}" - [ -z "$NODENAME" ] && NODENAME=$(grep -E '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-) + if [ -z "$NODENAME" ] && [ "$IS_BOOT_COMMAND" = 'no' ]; then + # for non-boot commands, inspect vm.