Build with rebar3 (#2475)

* Pin cuttlefish rebar3 branch

* No shallow clone for rebar3

* Replace timer:sleep with receive after for deterministic

* Build with only rebar3

* bbmustache as a test dependency

* Change to plain deps

* Use shallow clone by default

* Add ct-helpers as test dep

.travis.yml

@@ -7,12 +7,11 @@ before_install:
     - git clone https://github.com/erlang/rebar3.git; cd rebar3; ./bootstrap; sudo mv rebar3 /usr/local/bin/; cd ..
 
 script:
-  - make dep-vsn-check
+  - make compile
-  - make rebar-compile
+  - make xref
-  - make rebar-xref
+  - make eunit
-  - make rebar-eunit
+  - make ct
-  - make rebar-ct
+  - make cover
-  - make rebar-cover
 
 after_success:
   - make coveralls

Makefile

@@ -1,37 +1,9 @@
-.PHONY: plugins tests
+## shallow clone for speed
 
-PROJECT = emqx
+REBAR_GIT_CLONE_OPTIONS += --depth 1
-PROJECT_DESCRIPTION = EMQ X Broker
+export REBAR_GIT_CLONE_OPTIONS
 
-DEPS = jsx gproc gen_rpc ekka esockd cowboy replayq
+# CT_SUITES = emqx_trie emqx_router emqx_frame emqx_mqtt_compat
-
-dep_jsx     = git-emqx https://github.com/talentdeficit/jsx 2.9.0
-dep_gproc   = git-emqx https://github.com/uwiger/gproc 0.8.0
-dep_gen_rpc = git-emqx https://github.com/emqx/gen_rpc 2.3.1
-dep_esockd  = git-emqx https://github.com/emqx/esockd v5.4.4
-dep_ekka    = git-emqx https://github.com/emqx/ekka v0.5.4
-dep_cowboy  = git-emqx https://github.com/ninenines/cowboy 2.6.1
-dep_replayq = git-emqx https://github.com/emqx/replayq v0.1.1
-
-NO_AUTOPATCH = cuttlefish
-
-ERLC_OPTS += +debug_info -DAPPLICATION=emqx
-
-BUILD_DEPS = cuttlefish
-dep_cuttlefish = git-emqx https://github.com/emqx/cuttlefish v2.2.1
-
-CUR_BRANCH := $(shell git branch | grep -e "^*" | cut -d' ' -f 2)
-BRANCH := $(if $(filter $(CUR_BRANCH), master develop), $(CUR_BRANCH), develop)
-
-TEST_DEPS = emqx_ct_helpers
-dep_emqx_ct_helpers = git-emqx https://github.com/emqx/emqx-ct-helpers.git v1.0
-
-TEST_ERLC_OPTS += +debug_info -DAPPLICATION=emqx
-
-EUNIT_OPTS = verbose
-
-# CT_SUITES = emqx_bridge
-## emqx_trie emqx_router emqx_frame emqx_mqtt_compat
 
 CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \
 			emqx_broker emqx_cm emqx_frame emqx_guid emqx_inflight emqx_json \
@@ -44,31 +16,32 @@ CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \
             emqx_vm_mon emqx_alarm_handler emqx_rpc emqx_flapping
 
 CT_NODE_NAME = emqxct@127.0.0.1
-CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME)
 
-COVER = true
+compile:
+	@rebar3 compile
 
-PLT_APPS = sasl asn1 ssl syntax_tools runtime_tools crypto xmerl os_mon inets public_key ssl compiler mnesia
+clean: gen-clean
-DIALYZER_DIRS := ebin/
-DIALYZER_OPTS := --verbose --statistics -Werror_handling -Wrace_conditions #-Wunmatched_returns
-
-$(shell [ -f erlang.mk ] || curl -s -o erlang.mk https://raw.githubusercontent.com/emqx/erlmk/master/erlang.mk)
-include erlang.mk
-
-clean:: gen-clean
 
 .PHONY: gen-clean
 gen-clean:
-	@rm -rf bbmustache
 	@rm -f etc/gen.emqx.conf
 
-bbmustache:
+## bbmustache is a mustache template library used to render templated config files
-	$(verbose) git clone https://github.com/soranoba/bbmustache.git && cd bbmustache && ./rebar3 compile && cd ..
+## for common tests.
+BBMUSTACHE := _build/test/lib/bbmustache
+$(BBMUSTACHE):
+	@rebar3 as test compile
 
-# This hack is to generate a conf file for testing
+## Cuttlefish escript is built by default when cuttlefish app (as dependency) was built
-# relx overlay is used for release
+CUTTLEFISH_SCRIPT := _build/default/lib/cuttlefish/cuttlefish
-etc/gen.emqx.conf: bbmustache etc/emqx.conf
+
-	$(verbose) erl -noshell -pa bbmustache/_build/default/lib/bbmustache/ebin -eval \
+app.config: etc/gen.emqx.conf
+	$(verbose) $(CUTTLEFISH_SCRIPT) -l info -e etc/ -c etc/gen.emqx.conf -i priv/emqx.schema -d data/
+
+## NOTE: Mustache templating was resolved by relx overlay when building a release.
+## This is only to generate a conf file for testing,
+etc/gen.emqx.conf: $(BBMUSTACHE) etc/emqx.conf
+	@$(verbose) erl -noshell -pa _build/test/lib/bbmustache/ebin -eval \
 		"{ok, Temp} = file:read_file('etc/emqx.conf'), \
 		{ok, Vars0} = file:consult('vars'), \
 		Vars = [{atom_to_list(N), list_to_binary(V)} || {N, V} <- Vars0], \
@@ -76,51 +49,46 @@ etc/gen.emqx.conf: bbmustache etc/emqx.conf
 		ok = file:write_file('etc/gen.emqx.conf', Targ), \
 		halt(0)."
 
-CUTTLEFISH_SCRIPT = _build/default/lib/cuttlefish/cuttlefish
+.PHONY: cover
-
+cover:
-app.config: $(CUTTLEFISH_SCRIPT) etc/gen.emqx.conf
-	$(verbose) $(CUTTLEFISH_SCRIPT) -l info -e etc/ -c etc/gen.emqx.conf -i priv/emqx.schema -d data/
-
-ct: app.config
-
-rebar-cover:
 	@rebar3 cover
 
+.PHONY: coveralls
 coveralls:
 	@rebar3 coveralls send
 
-
+.PHONY: xref
-$(CUTTLEFISH_SCRIPT): rebar-deps
+xref:
-	@if [ ! -f cuttlefish ]; then make -C _build/default/lib/cuttlefish; fi
-
-rebar-xref:
 	@rebar3 xref
 
-rebar-deps:
+.PHONY: deps
+deps:
 	@rebar3 get-deps
 
-rebar-eunit: $(CUTTLEFISH_SCRIPT)
+.PHONY: eunit
+eunit:
 	@rebar3 eunit -v
 
-rebar-compile:
+## 'ct-setup' is a pre hook for 'rebar3 ct',
-	@rebar3 compile
+## but not the makefile target ct's dependency
-
+## because 'ct-setup' requires test dependencies to be compiled first
-rebar-ct-setup: app.config
+.PHONY: ct-setup
+ct-setup:
 	@rebar3 as test compile
 	@ln -s -f '../../../../etc' _build/test/lib/emqx/
 	@ln -s -f '../../../../data' _build/test/lib/emqx/
 
-rebar-ct: rebar-ct-setup
+.PHONY: ct
+ct: app.config ct-setup
 	@rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(shell echo $(foreach var,$(CT_SUITES),test/$(var)_SUITE) | tr ' ' ',')
 
 ## Run one single CT with rebar3
 ## e.g. make ct-one-suite suite=emqx_bridge
-ct-one-suite: rebar-ct-setup
+.PHONY: ct-one-suite
+ct-one-suite: ct-setup
 	@rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(suite)_SUITE
 
-rebar-clean:
+.PHONY: clean
-	@rebar3 clean
+clean:
-
-distclean::
 	@rm -rf _build cover deps logs log data
 	@rm -f rebar.lock compile_commands.json cuttlefish

rebar.config

@@ -1,17 +1,13 @@
-{deps, [{jsx, "2.9.0"},
+{deps,
-        {gproc, "0.8.0"},
+    [ {jsx, "2.9.0"} % hex
-        {cowboy, "2.6.1"}]}.
+    , {cowboy, "2.6.1"} % hex
-
+    , {gproc, "0.8.0"} % hex
-%% appended to deps in rebar.config.script
+    , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.3.1"}}}
-{github_emqx_libs,
+    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "v0.5.4"}}}
- [{gen_rpc, "2.3.1"},
+    , {replayq, {git, "https://github.com/emqx/replayq", {tag, "v0.1.1"}}}
-  {ekka, "v0.5.4"},
+    , {esockd, {git, "https://github.com/emqx/esockd", {tag, "v5.4.4"}}}
-  {replayq, "v0.1.1"},
+    , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}}
-  {esockd, "v5.4.4"},
+    ]}.
-  {cuttlefish, "v2.2.1"}]}.
-
-{github_emqx_projects,
- [{emqx_ct_helpers, "v1.0"}]}.
 
 {edoc_opts, [{preprocess, true}]}.
 {erl_opts, [warn_unused_vars,
@@ -28,3 +24,13 @@
 {cover_export_enabled, true}.
 
 {plugins, [coveralls]}.
+
+{profiles,
+    [{test,
+        [{deps,
+            [ {meck, "0.8.13"} % hex
+            , {bbmustache, "1.7.0"} % hex
+            , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.0"}}}
+            ]}
+        ]}
+    ]}.

rebar.config.script

@@ -1,11 +1,3 @@
-CONFIG0 = case os:getenv("REBAR_GIT_CLONE_OPTIONS") of
-              "--depth 1" ->
-                  CONFIG;
-              _ ->
-                  os:putenv("REBAR_GIT_CLONE_OPTIONS", "--depth 1"),
-                  CONFIG
-          end,
-
 CONFIG1 = case os:getenv("TRAVIS") of
               "true" ->
                   JobId = os:getenv("TRAVIS_JOB_ID"),
@@ -16,28 +8,4 @@ CONFIG1 = case os:getenv("TRAVIS") of
                   CONFIG
           end,
 
-FindDeps = fun(DepsType, Config) ->
+CONFIG1.
-                   case lists:keyfind(DepsType, 1, Config) of
-                       {_, RawDeps} -> RawDeps;
-                       _ -> []
-                   end
-           end,
-Deps = FindDeps(deps, CONFIG1),
-LibDeps = FindDeps(github_emqx_libs, CONFIG1),
-ProjDeps = FindDeps(github_emqx_projects, CONFIG1),
-UrlPrefix = "https://github.com/emqx/",
-RealName = fun TransName([$_ | Tail], Result) ->
-                   TransName(Tail, [$- | Result]);
-               TransName([Head | Tail], Result) ->
-                   TransName(Tail, [Head | Result]);
-               TransName([], Result) ->
-                   lists:reverse(Result)
-           end,
-
-NewLibDeps = [{LibName, {git, UrlPrefix ++ atom_to_list(LibName), {branch, Branch}}}
-              || {LibName, Branch} <- LibDeps],
-NewProjDeps = [{ProjName, {git, UrlPrefix ++ RealName(atom_to_list(ProjName), []), {branch, Branch}}} || {ProjName, Branch} <- ProjDeps],
-
-NewDeps = Deps ++ NewLibDeps ++ NewProjDeps,
-CONFIG2 = lists:keystore(deps, 1, CONFIG1, {deps, NewDeps}),
-CONFIG2.

test/emqx_vm_mon_SUITE.erl

@@ -23,6 +23,15 @@
 
 -include_lib("common_test/include/ct.hrl").
 
+-define(WAIT(PATTERN, TIMEOUT),
+        receive
+            PATTERN ->
+                ok
+        after
+            TIMEOUT ->
+                error(timeout)
+        end).
+
 all() -> [t_api].
 
 init_per_suite(Config) ->
@@ -33,18 +42,36 @@ end_per_suite(_Config) ->
     application:stop(sasl).
 
 t_api(_) ->
-    gen_event:swap_handler(alarm_handler, {emqx_alarm_handler, swap}, {alarm_handler, []}),
+    meck:new(alarm_handler, [passthrough, no_history]),
-    {ok, _} = emqx_vm_mon:start_link([{check_interval, 1},
+    Tester = self(),
-                                      {process_high_watermark, 0},
+    Ref = make_ref(),
-                                      {process_low_watermark, 0.6}]),
+    try
-    timer:sleep(2000),
+        meck:expect(alarm_handler, set_alarm,
-    ?assertEqual(true, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())),
+                    fun(What) ->
-    emqx_vm_mon:set_process_high_watermark(0.8),
+                            Res = meck:passthrough([What]),
-    emqx_vm_mon:set_process_low_watermark(0.75),
+                            Tester ! {Ref, set_alarm, What},
-    ?assertEqual(0.8, emqx_vm_mon:get_process_high_watermark()),
+                            Res
-    ?assertEqual(0.75, emqx_vm_mon:get_process_low_watermark()),
+                    end),
-    timer:sleep(3000),
+        meck:expect(alarm_handler, clear_alarm,
-    ?assertEqual(false, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())),
+                    fun(What) ->
-    emqx_vm_mon:set_check_interval(20),
+                            Res = meck:passthrough([What]),
-    ?assertEqual(20, emqx_vm_mon:get_check_interval()),
+                            Tester ! {Ref, clear_alarm, What},
-    ok.
+                            Res
+                    end),
+        gen_event:swap_handler(alarm_handler, {emqx_alarm_handler, swap}, {alarm_handler, []}),
+        {ok, _} = emqx_vm_mon:start_link([{check_interval, 1},
+                                        {process_high_watermark, 0},
+                                        {process_low_watermark, 0.6}]),
+        ?WAIT({Ref, set_alarm, {too_many_processes, _Count}}, 2000),
+        ?assertEqual(true, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())),
+        emqx_vm_mon:set_process_high_watermark(0.8),
+        emqx_vm_mon:set_process_low_watermark(0.75),
+        ?assertEqual(0.8, emqx_vm_mon:get_process_high_watermark()),
+        ?assertEqual(0.75, emqx_vm_mon:get_process_low_watermark()),
+        ?WAIT({Ref, clear_alarm, too_many_processes}, 3000),
+        ?assertEqual(false, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())),
+        emqx_vm_mon:set_check_interval(20),
+        ?assertEqual(20, emqx_vm_mon:get_check_interval())
+    after
+        meck:unload(alarm_handler)
+    end.