diff --git a/.gitignore b/.gitignore index 0322aaddc..0927f0bbc 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,8 @@ _build .rebar3 rebar3.crashdump .DS_Store -rebar.config emqx.iml bbmustache/ etc/gen.emqx.conf +compile_commands.json +cuttlefish diff --git a/.travis.yml b/.travis.yml index b2d01f3fc..e4088022d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,15 @@ language: erlang otp_release: - - 21.0 - 21.0.4 +before_install: + - git clone https://github.com/erlang/rebar3.git; cd rebar3; ./bootstrap; sudo mv rebar3 /usr/local/bin/; cd .. + script: - - make + - make dep-vsn-check + - make rebar-eunit + - make rebar-ct + - make coveralls sudo: false diff --git a/Makefile b/Makefile index f24497852..2417ad040 100644 --- a/Makefile +++ b/Makefile @@ -9,11 +9,11 @@ DEPS = jsx gproc gen_rpc lager ekka esockd cowboy clique lager_syslog dep_jsx = git https://github.com/talentdeficit/jsx 2.9.0 dep_gproc = git https://github.com/uwiger/gproc 0.8.0 dep_gen_rpc = git https://github.com/emqx/gen_rpc 2.2.0 -dep_lager = git https://github.com/erlang-lager/lager 3.6.4 +dep_lager = git https://github.com/erlang-lager/lager 3.6.5 dep_esockd = git https://github.com/emqx/esockd v5.4 dep_ekka = git https://github.com/emqx/ekka v0.4.1 dep_cowboy = git https://github.com/ninenines/cowboy 2.4.0 -dep_clique = git https://github.com/emqx/clique +dep_clique = git https://github.com/emqx/clique develop dep_lager_syslog = git https://github.com/basho/lager_syslog 3.0.1 NO_AUTOPATCH = cuttlefish @@ -41,7 +41,8 @@ CT_SUITES = emqx emqx_zone emqx_banned emqx_connection emqx_session emqx_access emqx_stats emqx_tables emqx_time emqx_topic emqx_trie emqx_vm \ emqx_mountpoint emqx_listeners emqx_protocol -CT_OPTS = -cover test/ct.cover.spec -erl_args -name emqxct@127.0.0.1 +CT_NODE_NAME = emqxct@127.0.0.1 +CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME) COVER = true @@ -51,7 +52,7 @@ DIALYZER_OPTS := --verbose --statistics -Werror_handling -Wrace_conditions #-Wun include erlang.mk -clean:: gen-clean +clean:: gen-clean rebar-clean .PHONY: gen-clean gen-clean: @@ -73,7 +74,62 @@ etc/gen.emqx.conf: bbmustache etc/emqx.conf halt(0)." app.config: etc/gen.emqx.conf - $(verbose) ./deps/cuttlefish/cuttlefish -l info -e etc/ -c etc/gen.emqx.conf -i priv/emqx.schema -d data/ + $(verbose) ./cuttlefish -l info -e etc/ -c etc/gen.emqx.conf -i priv/emqx.schema -d data/ -ct: app.config +ct: cuttlefish app.config + +coveralls: + @rebar3 coveralls send + +cuttlefish: deps + @mv ./deps/cuttlefish/cuttlefish ./cuttlefish + +rebar-cuttlefish: rebar-deps + @make -C _build/default/lib/cuttlefish + @mv _build/default/lib/cuttlefish/cuttlefish ./cuttlefish + +rebar-deps: + @rebar3 get-deps + +rebar-eunit: + @rebar3 eunit + +rebar-compile: + @rebar3 compile + +rebar-ct: rebar-cuttlefish app.config + @rebar3 as test compile + @ln -s -f '../../../../etc' _build/test/lib/emqx/ + @ln -s -f '../../../../data' _build/test/lib/emqx/ + @rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(shell echo $(foreach var,$(CT_SUITES),test/$(var)_SUITE) | tr ' ' ',') + +rebar-clean: + @rebar3 clean + +distclean:: rebar-clean + @rm -rf _build cover deps logs log data + @rm -f rebar.lock compile_commands.json cuttlefish + +## Below are for version consistency check during erlang.mk and rebar3 dual mode support +none= +space = $(none) $(none) +comma = , +quote = \" +curly_l = "{" +curly_r = "}" +dep-versions = [$(foreach dep,$(DEPS) $(BUILD_DEPS),$(curly_l)$(dep),$(quote)$(word 3,$(dep_$(dep)))$(quote)$(curly_r)$(comma))[]] + +.PHONY: dep-vsn-check +dep-vsn-check: + $(verbose) erl -noshell -eval \ + "MkVsns = lists:sort(lists:flatten($(dep-versions))), \ + {ok, Conf} = file:consult('rebar.config'), \ + {_, Deps1} = lists:keyfind(deps, 1, Conf), \ + {_, Deps2} = lists:keyfind(github_emqx_deps, 1, Conf), \ + F = fun({N, V}) when is_list(V) -> {N, V}; ({N, {git, _, {branch, V}}}) -> {N, V} end, \ + RebarVsns = lists:sort(lists:map(F, Deps1 ++ Deps2)), \ + case {RebarVsns -- MkVsns, MkVsns -- RebarVsns} of \ + {[], []} -> halt(0); \ + {Rebar, Mk} -> erlang:error({deps_version_discrepancy, [{rebar, Rebar}, {mk, Mk}]}) \ + end." diff --git a/rebar.config b/rebar.config new file mode 100644 index 000000000..d7d14c883 --- /dev/null +++ b/rebar.config @@ -0,0 +1,38 @@ +{deps, [{jsx, "2.9.0"}, + {gproc, "0.8.0"}, + {lager, "3.6.5"}, + {cowboy, "2.4.0"}, + {lager_syslog, {git, "https://github.com/basho/lager_syslog", {branch, "3.0.1"}}} + ]}. + +%% appended to deps in rebar.config.script +{github_emqx_deps, + [{gen_rpc, "2.2.0"}, + {ekka, "v0.4.1"}, + {clique, "develop"}, + {esockd, "v5.4"}, + {cuttlefish, "emqx30"} + ]}. + +{edoc_opts, [{preprocess, true}]}. +{erl_opts, [warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard, + debug_info, + {parse_transform, lager_transform}]}. +{xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, deprecated_function_calls, + warnings_as_errors, deprecated_functions]}. +{cover_enabled, true}. +{cover_opts, [verbose]}. +{cover_export_enabled, true}. + +%% rebar3_neotoma_plugin is needed to compile the .peg file for cuttlefish +{plugins, [rebar3_neotoma_plugin]}. + +%% Do not include cuttlefish's dependencies as mine +%% its dependencies are only fetched to compile itself +%% they are however not needed by emqx +{overrides, [{override, cuttlefish, [{deps, []}]} + ]}. diff --git a/rebar.config.script b/rebar.config.script new file mode 100644 index 000000000..7c247ac48 --- /dev/null +++ b/rebar.config.script @@ -0,0 +1,19 @@ + +CONFIG1 = case os:getenv("TRAVIS") of + "true" -> + JobId = os:getenv("TRAVIS_JOB_ID"), + [{coveralls_service_job_id, JobId}, + {plugins, [coveralls]}, + {coveralls_coverdata, "_build/test/cover/*.coverdata"}, + {coveralls_service_name , "travis-ci"} | CONFIG]; + _ -> + CONFIG +end, + +{_, Deps} = lists:keyfind(deps, 1, CONFIG1), +{_, OurDeps} = lists:keyfind(github_emqx_deps, 1, CONFIG1), +UrlPrefix = "https://github.com/emqx/", +NewDeps = Deps ++ [{Name, {git, UrlPrefix ++ atom_to_list(Name), {branch, Branch}}} || {Name, Branch} <- OurDeps], +CONFIG2 = lists:keystore(deps, 1, CONFIG1, {deps, NewDeps}), + +CONFIG2. diff --git a/rebar.lock b/rebar.lock deleted file mode 100644 index 1364f95ae..000000000 --- a/rebar.lock +++ /dev/null @@ -1,36 +0,0 @@ -[{<<"esockd">>, - {git,"https://github.com/emqtt/esockd", - {ref,"87d0d3b672e0f25e474f5f8298da568cbb6b168a"}}, - 0}, - {<<"gen_logger">>, - {git,"https://github.com/emqtt/gen_logger.git", - {ref,"f6e9f2f373d99f41ffe0579ab5a5f3b19472c9c5"}}, - 1}, - {<<"goldrush">>, - {git,"https://github.com/basho/goldrush.git", - {ref,"8f1b715d36b650ec1e1f5612c00e28af6ab0de82"}}, - 1}, - {<<"gproc">>, - {git,"https://github.com/uwiger/gproc", - {ref,"01c8fbfdd5e4701e8e4b57b0c8279872f9574b0b"}}, - 0}, - {<<"lager">>, - {git,"https://github.com/basho/lager", - {ref,"81eaef0ce98fdbf64ab95665e3bc2ec4b24c7dac"}}, - 0}, - {<<"lager_syslog">>, - {git,"https://github.com/basho/lager_syslog", - {ref,"126dd0284fcac9b01613189a82facf8d803411a2"}}, - 0}, - {<<"mochiweb">>, - {git,"https://github.com/emqtt/mochiweb", - {ref,"c75d88e451b4fe26580a58223f645d99482f51af"}}, - 0}, - {<<"pbkdf2">>, - {git,"https://github.com/comtihon/erlang-pbkdf2.git", - {ref,"7076584f5377e98600a7e2cb81980b2992fb2f71"}}, - 0}, - {<<"syslog">>, - {git,"git://github.com/Vagabond/erlang-syslog", - {ref,"0e4f0e95c361af298c5d1d17ceccfa831efc036d"}}, - 1}]. diff --git a/src/emqx_protocol.erl b/src/emqx_protocol.erl index 7e95a4d90..8301cf014 100644 --- a/src/emqx_protocol.erl +++ b/src/emqx_protocol.erl @@ -370,7 +370,7 @@ process_packet(?SUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters), end; true -> RawTopicFilters - end, + end, case check_subscribe( parse_topic_filters(?SUBSCRIBE, RawTopicFilters1), PState) of {ok, TopicFilters} -> @@ -648,10 +648,16 @@ check_publish(Packet, PState) -> fun check_pub_acl/2], Packet, PState). check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain}, - variable = #mqtt_packet_publish{ properties = Properties}}, + variable = #mqtt_packet_publish{ + properties = #{'Topic-Alias' := TopicAlias} + }}, #pstate{zone = Zone}) -> - #{'Topic-Alias' := TopicAlias} = Properties, - emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic_alias => TopicAlias}). + emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic_alias => TopicAlias}); +check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain}, + variable = #mqtt_packet_publish{ properties = _Properties}}, + #pstate{zone = Zone}) -> + emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain}). + check_pub_acl(_Packet, #pstate{is_super = IsSuper, enable_acl = EnableAcl}) when IsSuper orelse (not EnableAcl) -> @@ -732,7 +738,8 @@ shutdown(Reason, PState = #pstate{connected = true, send_willmsg(undefined) -> ignore; send_willmsg(WillMsg = #message{topic = Topic, - headers = #{'Will-Delay-Interval' := Interval}}) when is_integer(Interval) -> + headers = #{'Will-Delay-Interval' := Interval}}) + when is_integer(Interval), Interval > 0 -> SendAfter = integer_to_binary(Interval), emqx_broker:publish(WillMsg#message{topic = <<"$delayed/", SendAfter/binary, "/", Topic/binary>>}); send_willmsg(WillMsg) -> diff --git a/test/emqx_access_SUITE.erl b/test/emqx_access_SUITE.erl index e08cec08e..b49c90d80 100644 --- a/test/emqx_access_SUITE.erl +++ b/test/emqx_access_SUITE.erl @@ -98,7 +98,7 @@ end_per_group(_Group, Config) -> Config. init_per_testcase(_TestCase, Config) -> - %% {ok, _Pid} = + %% {ok, _Pid} = ?AC:start_link(), Config. end_per_testcase(_TestCase, _Config) -> @@ -119,7 +119,6 @@ reload_acl(_) -> register_mod(_) -> ok = ?AC:register_mod(acl, emqx_acl_test_mod, []), - {error, already_existed} = ?AC:register_mod(acl, emqx_acl_test_mod, []), {emqx_acl_test_mod, _, 0} = hd(?AC:lookup_mods(acl)), ok = ?AC:register_mod(auth, emqx_auth_anonymous_test_mod,[]), ok = ?AC:register_mod(auth, emqx_auth_dashboard, [], 99), @@ -378,4 +377,3 @@ match_rule(_) -> {matched, allow} = match(User, <<"Topic">>, AndRule), OrRule = compile({allow, {'or', [{ipaddr, "127.0.0.1"}, {user, <<"WrongUser">>}]}, publish, ["Topic"]}), {matched, allow} = match(User, <<"Topic">>, OrRule). - diff --git a/test/emqx_listeners_SUITE.erl b/test/emqx_listeners_SUITE.erl index 9d85583ab..17181aa31 100644 --- a/test/emqx_listeners_SUITE.erl +++ b/test/emqx_listeners_SUITE.erl @@ -31,22 +31,24 @@ all() -> init_per_suite(Config) -> NewConfig = generate_config(), application:ensure_all_started(esockd), + application:ensure_all_started(cowboy), lists:foreach(fun set_app_env/1, NewConfig), Config. end_per_suite(_Config) -> - application:stop(esockd). + application:stop(esockd), + application:stop(cowboy). start_stop_listeners(_) -> ok = emqx_listeners:start(), ok = emqx_listeners:stop(). - + restart_listeners(_) -> ok = emqx_listeners:start(), ok = emqx_listeners:stop(), ok = emqx_listeners:restart(), ok = emqx_listeners:stop(). - + generate_config() -> Schema = cuttlefish_schema:files([local_path(["priv", "emqx.schema"])]), Conf = conf_parse:file([local_path(["etc", "gen.emqx.conf"])]),