Rebar3 and erlang.mk dual support. (#1806)

* Rebar3 and erlang.mk dual support.

There was only erlang.mk support prior to this change.
The main reasons for this dual support are:
* Cover report upload can only be done by rebar3 in travis.ci
* We want to prepare for the future to build emqx releases using rebar3
* We do not want to stop supporting erlang.mk in one single step

* Add depencency version consistency check between erlang.mk and rebar.config

.gitignore

@@ -30,7 +30,8 @@ _build
 .rebar3
 rebar3.crashdump
 .DS_Store
-rebar.config
 emqx.iml
 bbmustache/
 etc/gen.emqx.conf
+compile_commands.json
+cuttlefish

.travis.yml

@@ -1,10 +1,15 @@
 language: erlang
 
 otp_release:
-   - 21.0
    - 21.0.4
 
+before_install:
+    - git clone https://github.com/erlang/rebar3.git; cd rebar3; ./bootstrap; sudo mv rebar3 /usr/local/bin/; cd ..
+
 script:
-  - make
+  - make dep-vsn-check
+  - make rebar-eunit
+  - make rebar-ct
+  - make coveralls
 
 sudo: false

Makefile

@@ -9,11 +9,11 @@ DEPS = jsx gproc gen_rpc lager ekka esockd cowboy clique lager_syslog
 dep_jsx     = git https://github.com/talentdeficit/jsx 2.9.0
 dep_gproc   = git https://github.com/uwiger/gproc 0.8.0
 dep_gen_rpc = git https://github.com/emqx/gen_rpc 2.2.0
-dep_lager   = git https://github.com/erlang-lager/lager 3.6.4
+dep_lager   = git https://github.com/erlang-lager/lager 3.6.5
 dep_esockd  = git https://github.com/emqx/esockd v5.4
 dep_ekka    = git https://github.com/emqx/ekka v0.4.1
 dep_cowboy  = git https://github.com/ninenines/cowboy 2.4.0
-dep_clique  = git https://github.com/emqx/clique
+dep_clique  = git https://github.com/emqx/clique develop
 dep_lager_syslog = git https://github.com/basho/lager_syslog 3.0.1
 
 NO_AUTOPATCH = cuttlefish
@@ -41,7 +41,8 @@ CT_SUITES = emqx emqx_zone emqx_banned emqx_connection emqx_session emqx_access
 			emqx_stats emqx_tables emqx_time emqx_topic emqx_trie emqx_vm \
 		 	emqx_mountpoint emqx_listeners emqx_protocol
 
-CT_OPTS = -cover test/ct.cover.spec -erl_args -name emqxct@127.0.0.1
+CT_NODE_NAME = emqxct@127.0.0.1
+CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME)
 
 COVER = true
 
@@ -51,7 +52,7 @@ DIALYZER_OPTS := --verbose --statistics -Werror_handling -Wrace_conditions #-Wun
 
 include erlang.mk
 
-clean:: gen-clean
+clean:: gen-clean rebar-clean
 
 .PHONY: gen-clean
 gen-clean:
@@ -73,7 +74,62 @@ etc/gen.emqx.conf: bbmustache etc/emqx.conf
 		halt(0)."
 
 app.config: etc/gen.emqx.conf
-	$(verbose) ./deps/cuttlefish/cuttlefish -l info -e etc/ -c etc/gen.emqx.conf -i priv/emqx.schema -d data/
+	$(verbose) ./cuttlefish -l info -e etc/ -c etc/gen.emqx.conf -i priv/emqx.schema -d data/
 
-ct: app.config
+ct: cuttlefish app.config
+
+coveralls:
+	@rebar3 coveralls send
+
+cuttlefish: deps
+	@mv ./deps/cuttlefish/cuttlefish ./cuttlefish
+
+rebar-cuttlefish: rebar-deps
+	@make -C _build/default/lib/cuttlefish
+	@mv _build/default/lib/cuttlefish/cuttlefish ./cuttlefish
+
+rebar-deps:
+	@rebar3 get-deps
+
+rebar-eunit:
+	@rebar3 eunit
+
+rebar-compile:
+	@rebar3 compile
+
+rebar-ct: rebar-cuttlefish app.config
+	@rebar3 as test compile
+	@ln -s -f '../../../../etc' _build/test/lib/emqx/
+	@ln -s -f '../../../../data' _build/test/lib/emqx/
+	@rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(shell echo $(foreach var,$(CT_SUITES),test/$(var)_SUITE) | tr ' ' ',')
+
+rebar-clean:
+	@rebar3 clean
+
+distclean:: rebar-clean
+	@rm -rf _build cover deps logs log data
+	@rm -f rebar.lock compile_commands.json cuttlefish
+
+## Below are for version consistency check during erlang.mk and rebar3 dual mode support
+none=
+space = $(none) $(none)
+comma = ,
+quote = \"
+curly_l = "{"
+curly_r = "}"
+dep-versions = [$(foreach dep,$(DEPS) $(BUILD_DEPS),$(curly_l)$(dep),$(quote)$(word 3,$(dep_$(dep)))$(quote)$(curly_r)$(comma))[]]
+
+.PHONY: dep-vsn-check
+dep-vsn-check:
+	$(verbose) erl -noshell -eval \
+		"MkVsns = lists:sort(lists:flatten($(dep-versions))), \
+		{ok, Conf} = file:consult('rebar.config'), \
+		{_, Deps1} = lists:keyfind(deps, 1, Conf), \
+		{_, Deps2} = lists:keyfind(github_emqx_deps, 1, Conf), \
+		F = fun({N, V}) when is_list(V) -> {N, V}; ({N, {git, _, {branch, V}}}) -> {N, V} end, \
+		RebarVsns = lists:sort(lists:map(F, Deps1 ++ Deps2)), \
+		case {RebarVsns -- MkVsns, MkVsns -- RebarVsns} of \
+		  {[], []} -> halt(0); \
+		  {Rebar, Mk} -> erlang:error({deps_version_discrepancy, [{rebar, Rebar}, {mk, Mk}]}) \
+		end."
 

rebar.config

@@ -0,0 +1,38 @@
+{deps, [{jsx, "2.9.0"},
+        {gproc, "0.8.0"},
+        {lager, "3.6.5"},
+        {cowboy, "2.4.0"},
+        {lager_syslog, {git, "https://github.com/basho/lager_syslog", {branch, "3.0.1"}}}
+       ]}.
+
+%% appended to deps in rebar.config.script
+{github_emqx_deps,
+    [{gen_rpc, "2.2.0"},
+     {ekka, "v0.4.1"},
+     {clique, "develop"},
+     {esockd, "v5.4"},
+     {cuttlefish, "emqx30"}
+    ]}.
+
+{edoc_opts, [{preprocess, true}]}.
+{erl_opts, [warn_unused_vars,
+            warn_shadow_vars,
+            warn_unused_import,
+            warn_obsolete_guard,
+            debug_info,
+            {parse_transform, lager_transform}]}.
+{xref_checks, [undefined_function_calls, undefined_functions,
+               locals_not_used, deprecated_function_calls,
+               warnings_as_errors, deprecated_functions]}.
+{cover_enabled, true}.
+{cover_opts, [verbose]}.
+{cover_export_enabled, true}.
+
+%% rebar3_neotoma_plugin is needed to compile the .peg file for cuttlefish
+{plugins, [rebar3_neotoma_plugin]}.
+
+%% Do not include cuttlefish's dependencies as mine
+%% its dependencies are only fetched to compile itself
+%% they are however not needed by emqx
+{overrides, [{override, cuttlefish, [{deps, []}]}
+            ]}.

rebar.config.script

@@ -0,0 +1,19 @@
+
+CONFIG1 = case os:getenv("TRAVIS") of
+  "true" ->
+    JobId = os:getenv("TRAVIS_JOB_ID"),
+    [{coveralls_service_job_id, JobId},
+     {plugins, [coveralls]},
+     {coveralls_coverdata, "_build/test/cover/*.coverdata"},
+     {coveralls_service_name , "travis-ci"} | CONFIG];
+  _ ->
+    CONFIG
+end,
+
+{_, Deps} = lists:keyfind(deps, 1, CONFIG1),
+{_, OurDeps} = lists:keyfind(github_emqx_deps, 1, CONFIG1),
+UrlPrefix = "https://github.com/emqx/",
+NewDeps = Deps ++ [{Name, {git, UrlPrefix ++ atom_to_list(Name), {branch, Branch}}} || {Name, Branch} <- OurDeps],
+CONFIG2 = lists:keystore(deps, 1, CONFIG1, {deps, NewDeps}),
+
+CONFIG2.

rebar.lock

@@ -1,36 +0,0 @@
-[{<<"esockd">>,
-  {git,"https://github.com/emqtt/esockd",
-       {ref,"87d0d3b672e0f25e474f5f8298da568cbb6b168a"}},
-  0},
- {<<"gen_logger">>,
-  {git,"https://github.com/emqtt/gen_logger.git",
-       {ref,"f6e9f2f373d99f41ffe0579ab5a5f3b19472c9c5"}},
-  1},
- {<<"goldrush">>,
-  {git,"https://github.com/basho/goldrush.git",
-       {ref,"8f1b715d36b650ec1e1f5612c00e28af6ab0de82"}},
-  1},
- {<<"gproc">>,
-  {git,"https://github.com/uwiger/gproc",
-       {ref,"01c8fbfdd5e4701e8e4b57b0c8279872f9574b0b"}},
-  0},
- {<<"lager">>,
-  {git,"https://github.com/basho/lager",
-       {ref,"81eaef0ce98fdbf64ab95665e3bc2ec4b24c7dac"}},
-  0},
- {<<"lager_syslog">>,
-  {git,"https://github.com/basho/lager_syslog",
-       {ref,"126dd0284fcac9b01613189a82facf8d803411a2"}},
-  0},
- {<<"mochiweb">>,
-  {git,"https://github.com/emqtt/mochiweb",
-       {ref,"c75d88e451b4fe26580a58223f645d99482f51af"}},
-  0},
- {<<"pbkdf2">>,
-  {git,"https://github.com/comtihon/erlang-pbkdf2.git",
-       {ref,"7076584f5377e98600a7e2cb81980b2992fb2f71"}},
-  0},
- {<<"syslog">>,
-  {git,"git://github.com/Vagabond/erlang-syslog",
-       {ref,"0e4f0e95c361af298c5d1d17ceccfa831efc036d"}},
-  1}].

src/emqx_protocol.erl

@@ -654,7 +654,7 @@ check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Ret
                #pstate{zone = Zone}) ->
     emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic_alias => TopicAlias});
 check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain},
-                            variable = #mqtt_packet_publish{ properties = Properties}},
+                            variable = #mqtt_packet_publish{ properties = _Properties}},
                #pstate{zone = Zone}) ->
     emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain}).