From 2121da3755b2d8669eff34fb07af7b87c2edd073 Mon Sep 17 00:00:00 2001 From: terry-xiaoyu <506895667@qq.com> Date: Sat, 8 Sep 2018 10:10:22 +0800 Subject: [PATCH 1/8] Ignore Will-Delay-Interval = 0 --- src/emqx_protocol.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/emqx_protocol.erl b/src/emqx_protocol.erl index 7e95a4d90..4602337eb 100644 --- a/src/emqx_protocol.erl +++ b/src/emqx_protocol.erl @@ -370,7 +370,7 @@ process_packet(?SUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters), end; true -> RawTopicFilters - end, + end, case check_subscribe( parse_topic_filters(?SUBSCRIBE, RawTopicFilters1), PState) of {ok, TopicFilters} -> @@ -732,7 +732,8 @@ shutdown(Reason, PState = #pstate{connected = true, send_willmsg(undefined) -> ignore; send_willmsg(WillMsg = #message{topic = Topic, - headers = #{'Will-Delay-Interval' := Interval}}) when is_integer(Interval) -> + headers = #{'Will-Delay-Interval' := Interval}}) + when is_integer(Interval), Interval > 0 -> SendAfter = integer_to_binary(Interval), emqx_broker:publish(WillMsg#message{topic = <<"$delayed/", SendAfter/binary, "/", Topic/binary>>}); send_willmsg(WillMsg) -> From 2d10d6971df821a5188d5604cf9fa47707bf5abe Mon Sep 17 00:00:00 2001 From: Gilbert Wong Date: Sat, 8 Sep 2018 12:01:22 +0800 Subject: [PATCH 2/8] add pattern match for topic alias Prior to this change, it assume that topic_alias exists defaultly which may cause the unexpected bug This change fix this bug above --- src/emqx_protocol.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/emqx_protocol.erl b/src/emqx_protocol.erl index 4602337eb..1dae228d7 100644 --- a/src/emqx_protocol.erl +++ b/src/emqx_protocol.erl @@ -650,9 +650,15 @@ check_publish(Packet, PState) -> check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain}, variable = #mqtt_packet_publish{ properties = Properties}}, #pstate{zone = Zone}) -> - #{'Topic-Alias' := TopicAlias} = Properties, + emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain}); +check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain}, + variable = #mqtt_packet_publish{ + properties = #{'Topic-Alias' := TopicAlias} + }}, + #pstate{zone = Zone}) -> emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic_alias => TopicAlias}). + check_pub_acl(_Packet, #pstate{is_super = IsSuper, enable_acl = EnableAcl}) when IsSuper orelse (not EnableAcl) -> ok; From 78020de302a7a9e5a43f8cc45043ee88c03a2c78 Mon Sep 17 00:00:00 2001 From: Gilbert Wong Date: Sat, 8 Sep 2018 12:50:14 +0800 Subject: [PATCH 3/8] fix pattern match bu Prior to this change, when packet have topic alias, the check_pub_caps function could not be matched correctly This change fix this bug. --- src/emqx_protocol.erl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/emqx_protocol.erl b/src/emqx_protocol.erl index 1dae228d7..49408d38e 100644 --- a/src/emqx_protocol.erl +++ b/src/emqx_protocol.erl @@ -647,16 +647,16 @@ check_publish(Packet, PState) -> run_check_steps([fun check_pub_caps/2, fun check_pub_acl/2], Packet, PState). -check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain}, - variable = #mqtt_packet_publish{ properties = Properties}}, - #pstate{zone = Zone}) -> - emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain}); check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain}, variable = #mqtt_packet_publish{ properties = #{'Topic-Alias' := TopicAlias} }}, #pstate{zone = Zone}) -> - emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic_alias => TopicAlias}). + emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic_alias => TopicAlias}); +check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain}, + variable = #mqtt_packet_publish{ properties = Properties}}, + #pstate{zone = Zone}) -> + emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain}). check_pub_acl(_Packet, #pstate{is_super = IsSuper, enable_acl = EnableAcl}) From caedcee2dec3b63009f78d8660750eb81ce8b9e3 Mon Sep 17 00:00:00 2001 From: Gilbert Wong Date: Sat, 8 Sep 2018 14:32:02 +0800 Subject: [PATCH 4/8] fix listerners and access suites Prior to this change, listeners test suites did not start cowboy which is responsible for websocket connections. and access suites start duplicated processes and these actions are wrong This change fix the problem states above. --- test/emqx_access_SUITE.erl | 4 +--- test/emqx_listeners_SUITE.erl | 8 +++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/emqx_access_SUITE.erl b/test/emqx_access_SUITE.erl index e08cec08e..b49c90d80 100644 --- a/test/emqx_access_SUITE.erl +++ b/test/emqx_access_SUITE.erl @@ -98,7 +98,7 @@ end_per_group(_Group, Config) -> Config. init_per_testcase(_TestCase, Config) -> - %% {ok, _Pid} = + %% {ok, _Pid} = ?AC:start_link(), Config. end_per_testcase(_TestCase, _Config) -> @@ -119,7 +119,6 @@ reload_acl(_) -> register_mod(_) -> ok = ?AC:register_mod(acl, emqx_acl_test_mod, []), - {error, already_existed} = ?AC:register_mod(acl, emqx_acl_test_mod, []), {emqx_acl_test_mod, _, 0} = hd(?AC:lookup_mods(acl)), ok = ?AC:register_mod(auth, emqx_auth_anonymous_test_mod,[]), ok = ?AC:register_mod(auth, emqx_auth_dashboard, [], 99), @@ -378,4 +377,3 @@ match_rule(_) -> {matched, allow} = match(User, <<"Topic">>, AndRule), OrRule = compile({allow, {'or', [{ipaddr, "127.0.0.1"}, {user, <<"WrongUser">>}]}, publish, ["Topic"]}), {matched, allow} = match(User, <<"Topic">>, OrRule). - diff --git a/test/emqx_listeners_SUITE.erl b/test/emqx_listeners_SUITE.erl index 9d85583ab..17181aa31 100644 --- a/test/emqx_listeners_SUITE.erl +++ b/test/emqx_listeners_SUITE.erl @@ -31,22 +31,24 @@ all() -> init_per_suite(Config) -> NewConfig = generate_config(), application:ensure_all_started(esockd), + application:ensure_all_started(cowboy), lists:foreach(fun set_app_env/1, NewConfig), Config. end_per_suite(_Config) -> - application:stop(esockd). + application:stop(esockd), + application:stop(cowboy). start_stop_listeners(_) -> ok = emqx_listeners:start(), ok = emqx_listeners:stop(). - + restart_listeners(_) -> ok = emqx_listeners:start(), ok = emqx_listeners:stop(), ok = emqx_listeners:restart(), ok = emqx_listeners:stop(). - + generate_config() -> Schema = cuttlefish_schema:files([local_path(["priv", "emqx.schema"])]), Conf = conf_parse:file([local_path(["etc", "gen.emqx.conf"])]), From 40977e529a9a8a29fdf4d260408ea0e861186d88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E5=AD=90=E5=8D=9A?= <349832309@qq.com> Date: Sat, 8 Sep 2018 13:35:23 +0800 Subject: [PATCH 5/8] Retain flag in retained message must set to 1 --- src/emqx_session.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 05142297e..8e6e150c5 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -726,9 +726,10 @@ run_dispatch_steps([{qos, SubQoS}|Steps], Msg = #message{qos = PubQoS}, State = run_dispatch_steps(Steps, Msg#message{qos = min(SubQoS, PubQoS)}, State); run_dispatch_steps([{qos, SubQoS}|Steps], Msg = #message{qos = PubQoS}, State = #state{upgrade_qos = true}) -> run_dispatch_steps(Steps, Msg#message{qos = max(SubQoS, PubQoS)}, State); +run_dispatch_steps([{rap, _Rap}|Steps], Msg = #message{flags = Flags, headers = #{retained := true}}, State = #state{}) -> + run_dispatch_steps(Steps, Msg#message{flags = maps:put(retain, true, Flags)}, State); run_dispatch_steps([{rap, 0}|Steps], Msg = #message{flags = Flags}, State = #state{}) -> - Flags1 = maps:put(retain, false, Flags), - run_dispatch_steps(Steps, Msg#message{flags = Flags1}, State); + run_dispatch_steps(Steps, Msg#message{flags = maps:put(retain, false, Flags)}, State); run_dispatch_steps([{rap, _}|Steps], Msg, State) -> run_dispatch_steps(Steps, Msg, State); run_dispatch_steps([{subid, SubId}|Steps], Msg, State) -> From 23c91c0a40430e6169761fd6f77d761bd32927c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E5=AD=90=E5=8D=9A?= <349832309@qq.com> Date: Sat, 8 Sep 2018 18:05:04 +0800 Subject: [PATCH 6/8] no message --- src/emqx_reason_codes.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqx_reason_codes.erl b/src/emqx_reason_codes.erl index fdc1377ec..75118b563 100644 --- a/src/emqx_reason_codes.erl +++ b/src/emqx_reason_codes.erl @@ -139,4 +139,4 @@ compat(connack, 16#9D) -> ?CONNACK_SERVER; compat(connack, 16#9F) -> ?CONNACK_SERVER; compat(suback, Code) when Code =< ?QOS2 -> Code; -compat(suback, Code) when Code > 16#80 -> 16#80. +compat(suback, Code) when Code >= 16#80 -> 16#80. From 3caa41f751d3a72e3b126939fa4d1b05b4f58fec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E5=AD=90=E5=8D=9A?= <349832309@qq.com> Date: Sat, 8 Sep 2018 18:59:57 +0800 Subject: [PATCH 7/8] Add run hook when duplicated subscription --- src/emqx_session.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 8e6e150c5..118ab6e21 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -444,6 +444,7 @@ handle_cast({subscribe, FromPid, {PacketId, _Properties, TopicFilters}}, lists:foldr(fun({Topic, SubOpts = #{qos := QoS}}, {RcAcc, SubMap}) -> {[QoS|RcAcc], case maps:find(Topic, SubMap) of {ok, SubOpts} -> + emqx_hooks:run('session.subscribed', [#{client_id => ClientId}, Topic, SubOpts#{first => false}]), SubMap; {ok, _SubOpts} -> emqx_broker:set_subopts(Topic, {self(), ClientId}, SubOpts), From 0c39a7620e2d19ac628880c12ea9b5e2ae943120 Mon Sep 17 00:00:00 2001 From: spring2maz <40776645+spring2maz@users.noreply.github.com> Date: Sun, 9 Sep 2018 15:51:42 +0200 Subject: [PATCH 8/8] Rebar3 and erlang.mk dual support. (#1806) * Rebar3 and erlang.mk dual support. There was only erlang.mk support prior to this change. The main reasons for this dual support are: * Cover report upload can only be done by rebar3 in travis.ci * We want to prepare for the future to build emqx releases using rebar3 * We do not want to stop supporting erlang.mk in one single step * Add depencency version consistency check between erlang.mk and rebar.config --- .gitignore | 3 +- .travis.yml | 9 ++++-- Makefile | 68 +++++++++++++++++++++++++++++++++++++++---- rebar.config | 38 ++++++++++++++++++++++++ rebar.config.script | 19 ++++++++++++ rebar.lock | 36 ----------------------- src/emqx_protocol.erl | 2 +- 7 files changed, 129 insertions(+), 46 deletions(-) create mode 100644 rebar.config create mode 100644 rebar.config.script delete mode 100644 rebar.lock diff --git a/.gitignore b/.gitignore index 0322aaddc..0927f0bbc 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,8 @@ _build .rebar3 rebar3.crashdump .DS_Store -rebar.config emqx.iml bbmustache/ etc/gen.emqx.conf +compile_commands.json +cuttlefish diff --git a/.travis.yml b/.travis.yml index b2d01f3fc..e4088022d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,15 @@ language: erlang otp_release: - - 21.0 - 21.0.4 +before_install: + - git clone https://github.com/erlang/rebar3.git; cd rebar3; ./bootstrap; sudo mv rebar3 /usr/local/bin/; cd .. + script: - - make + - make dep-vsn-check + - make rebar-eunit + - make rebar-ct + - make coveralls sudo: false diff --git a/Makefile b/Makefile index f24497852..2417ad040 100644 --- a/Makefile +++ b/Makefile @@ -9,11 +9,11 @@ DEPS = jsx gproc gen_rpc lager ekka esockd cowboy clique lager_syslog dep_jsx = git https://github.com/talentdeficit/jsx 2.9.0 dep_gproc = git https://github.com/uwiger/gproc 0.8.0 dep_gen_rpc = git https://github.com/emqx/gen_rpc 2.2.0 -dep_lager = git https://github.com/erlang-lager/lager 3.6.4 +dep_lager = git https://github.com/erlang-lager/lager 3.6.5 dep_esockd = git https://github.com/emqx/esockd v5.4 dep_ekka = git https://github.com/emqx/ekka v0.4.1 dep_cowboy = git https://github.com/ninenines/cowboy 2.4.0 -dep_clique = git https://github.com/emqx/clique +dep_clique = git https://github.com/emqx/clique develop dep_lager_syslog = git https://github.com/basho/lager_syslog 3.0.1 NO_AUTOPATCH = cuttlefish @@ -41,7 +41,8 @@ CT_SUITES = emqx emqx_zone emqx_banned emqx_connection emqx_session emqx_access emqx_stats emqx_tables emqx_time emqx_topic emqx_trie emqx_vm \ emqx_mountpoint emqx_listeners emqx_protocol -CT_OPTS = -cover test/ct.cover.spec -erl_args -name emqxct@127.0.0.1 +CT_NODE_NAME = emqxct@127.0.0.1 +CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME) COVER = true @@ -51,7 +52,7 @@ DIALYZER_OPTS := --verbose --statistics -Werror_handling -Wrace_conditions #-Wun include erlang.mk -clean:: gen-clean +clean:: gen-clean rebar-clean .PHONY: gen-clean gen-clean: @@ -73,7 +74,62 @@ etc/gen.emqx.conf: bbmustache etc/emqx.conf halt(0)." app.config: etc/gen.emqx.conf - $(verbose) ./deps/cuttlefish/cuttlefish -l info -e etc/ -c etc/gen.emqx.conf -i priv/emqx.schema -d data/ + $(verbose) ./cuttlefish -l info -e etc/ -c etc/gen.emqx.conf -i priv/emqx.schema -d data/ -ct: app.config +ct: cuttlefish app.config + +coveralls: + @rebar3 coveralls send + +cuttlefish: deps + @mv ./deps/cuttlefish/cuttlefish ./cuttlefish + +rebar-cuttlefish: rebar-deps + @make -C _build/default/lib/cuttlefish + @mv _build/default/lib/cuttlefish/cuttlefish ./cuttlefish + +rebar-deps: + @rebar3 get-deps + +rebar-eunit: + @rebar3 eunit + +rebar-compile: + @rebar3 compile + +rebar-ct: rebar-cuttlefish app.config + @rebar3 as test compile + @ln -s -f '../../../../etc' _build/test/lib/emqx/ + @ln -s -f '../../../../data' _build/test/lib/emqx/ + @rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(shell echo $(foreach var,$(CT_SUITES),test/$(var)_SUITE) | tr ' ' ',') + +rebar-clean: + @rebar3 clean + +distclean:: rebar-clean + @rm -rf _build cover deps logs log data + @rm -f rebar.lock compile_commands.json cuttlefish + +## Below are for version consistency check during erlang.mk and rebar3 dual mode support +none= +space = $(none) $(none) +comma = , +quote = \" +curly_l = "{" +curly_r = "}" +dep-versions = [$(foreach dep,$(DEPS) $(BUILD_DEPS),$(curly_l)$(dep),$(quote)$(word 3,$(dep_$(dep)))$(quote)$(curly_r)$(comma))[]] + +.PHONY: dep-vsn-check +dep-vsn-check: + $(verbose) erl -noshell -eval \ + "MkVsns = lists:sort(lists:flatten($(dep-versions))), \ + {ok, Conf} = file:consult('rebar.config'), \ + {_, Deps1} = lists:keyfind(deps, 1, Conf), \ + {_, Deps2} = lists:keyfind(github_emqx_deps, 1, Conf), \ + F = fun({N, V}) when is_list(V) -> {N, V}; ({N, {git, _, {branch, V}}}) -> {N, V} end, \ + RebarVsns = lists:sort(lists:map(F, Deps1 ++ Deps2)), \ + case {RebarVsns -- MkVsns, MkVsns -- RebarVsns} of \ + {[], []} -> halt(0); \ + {Rebar, Mk} -> erlang:error({deps_version_discrepancy, [{rebar, Rebar}, {mk, Mk}]}) \ + end." diff --git a/rebar.config b/rebar.config new file mode 100644 index 000000000..d7d14c883 --- /dev/null +++ b/rebar.config @@ -0,0 +1,38 @@ +{deps, [{jsx, "2.9.0"}, + {gproc, "0.8.0"}, + {lager, "3.6.5"}, + {cowboy, "2.4.0"}, + {lager_syslog, {git, "https://github.com/basho/lager_syslog", {branch, "3.0.1"}}} + ]}. + +%% appended to deps in rebar.config.script +{github_emqx_deps, + [{gen_rpc, "2.2.0"}, + {ekka, "v0.4.1"}, + {clique, "develop"}, + {esockd, "v5.4"}, + {cuttlefish, "emqx30"} + ]}. + +{edoc_opts, [{preprocess, true}]}. +{erl_opts, [warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard, + debug_info, + {parse_transform, lager_transform}]}. +{xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, deprecated_function_calls, + warnings_as_errors, deprecated_functions]}. +{cover_enabled, true}. +{cover_opts, [verbose]}. +{cover_export_enabled, true}. + +%% rebar3_neotoma_plugin is needed to compile the .peg file for cuttlefish +{plugins, [rebar3_neotoma_plugin]}. + +%% Do not include cuttlefish's dependencies as mine +%% its dependencies are only fetched to compile itself +%% they are however not needed by emqx +{overrides, [{override, cuttlefish, [{deps, []}]} + ]}. diff --git a/rebar.config.script b/rebar.config.script new file mode 100644 index 000000000..7c247ac48 --- /dev/null +++ b/rebar.config.script @@ -0,0 +1,19 @@ + +CONFIG1 = case os:getenv("TRAVIS") of + "true" -> + JobId = os:getenv("TRAVIS_JOB_ID"), + [{coveralls_service_job_id, JobId}, + {plugins, [coveralls]}, + {coveralls_coverdata, "_build/test/cover/*.coverdata"}, + {coveralls_service_name , "travis-ci"} | CONFIG]; + _ -> + CONFIG +end, + +{_, Deps} = lists:keyfind(deps, 1, CONFIG1), +{_, OurDeps} = lists:keyfind(github_emqx_deps, 1, CONFIG1), +UrlPrefix = "https://github.com/emqx/", +NewDeps = Deps ++ [{Name, {git, UrlPrefix ++ atom_to_list(Name), {branch, Branch}}} || {Name, Branch} <- OurDeps], +CONFIG2 = lists:keystore(deps, 1, CONFIG1, {deps, NewDeps}), + +CONFIG2. diff --git a/rebar.lock b/rebar.lock deleted file mode 100644 index 1364f95ae..000000000 --- a/rebar.lock +++ /dev/null @@ -1,36 +0,0 @@ -[{<<"esockd">>, - {git,"https://github.com/emqtt/esockd", - {ref,"87d0d3b672e0f25e474f5f8298da568cbb6b168a"}}, - 0}, - {<<"gen_logger">>, - {git,"https://github.com/emqtt/gen_logger.git", - {ref,"f6e9f2f373d99f41ffe0579ab5a5f3b19472c9c5"}}, - 1}, - {<<"goldrush">>, - {git,"https://github.com/basho/goldrush.git", - {ref,"8f1b715d36b650ec1e1f5612c00e28af6ab0de82"}}, - 1}, - {<<"gproc">>, - {git,"https://github.com/uwiger/gproc", - {ref,"01c8fbfdd5e4701e8e4b57b0c8279872f9574b0b"}}, - 0}, - {<<"lager">>, - {git,"https://github.com/basho/lager", - {ref,"81eaef0ce98fdbf64ab95665e3bc2ec4b24c7dac"}}, - 0}, - {<<"lager_syslog">>, - {git,"https://github.com/basho/lager_syslog", - {ref,"126dd0284fcac9b01613189a82facf8d803411a2"}}, - 0}, - {<<"mochiweb">>, - {git,"https://github.com/emqtt/mochiweb", - {ref,"c75d88e451b4fe26580a58223f645d99482f51af"}}, - 0}, - {<<"pbkdf2">>, - {git,"https://github.com/comtihon/erlang-pbkdf2.git", - {ref,"7076584f5377e98600a7e2cb81980b2992fb2f71"}}, - 0}, - {<<"syslog">>, - {git,"git://github.com/Vagabond/erlang-syslog", - {ref,"0e4f0e95c361af298c5d1d17ceccfa831efc036d"}}, - 1}]. diff --git a/src/emqx_protocol.erl b/src/emqx_protocol.erl index 49408d38e..8301cf014 100644 --- a/src/emqx_protocol.erl +++ b/src/emqx_protocol.erl @@ -654,7 +654,7 @@ check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Ret #pstate{zone = Zone}) -> emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic_alias => TopicAlias}); check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain}, - variable = #mqtt_packet_publish{ properties = Properties}}, + variable = #mqtt_packet_publish{ properties = _Properties}}, #pstate{zone = Zone}) -> emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain}).