From 97476b8bde73702c0331bc42a52e27e2ab70bc04 Mon Sep 17 00:00:00 2001 From: spring2maz <40776645+spring2maz@users.noreply.github.com> Date: Sun, 28 Apr 2019 03:40:39 +0200 Subject: [PATCH] Build with rebar3 (#2475) * Pin cuttlefish rebar3 branch * No shallow clone for rebar3 * Replace timer:sleep with receive after for deterministic * Build with only rebar3 * bbmustache as a test dependency * Change to plain deps * Use shallow clone by default * Add ct-helpers as test dep --- .travis.yml | 11 ++-- Makefile | 116 ++++++++++++++----------------------- rebar.config | 34 ++++++----- rebar.config.script | 34 +---------- test/emqx_vm_mon_SUITE.erl | 57 +++++++++++++----- 5 files changed, 110 insertions(+), 142 deletions(-) diff --git a/.travis.yml b/.travis.yml index f2c483cc6..bef59bf4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,12 +7,11 @@ before_install: - git clone https://github.com/erlang/rebar3.git; cd rebar3; ./bootstrap; sudo mv rebar3 /usr/local/bin/; cd .. script: - - make dep-vsn-check - - make rebar-compile - - make rebar-xref - - make rebar-eunit - - make rebar-ct - - make rebar-cover + - make compile + - make xref + - make eunit + - make ct + - make cover after_success: - make coveralls diff --git a/Makefile b/Makefile index acb854fa4..b515e7618 100644 --- a/Makefile +++ b/Makefile @@ -1,37 +1,9 @@ -.PHONY: plugins tests +## shallow clone for speed -PROJECT = emqx -PROJECT_DESCRIPTION = EMQ X Broker +REBAR_GIT_CLONE_OPTIONS += --depth 1 +export REBAR_GIT_CLONE_OPTIONS -DEPS = jsx gproc gen_rpc ekka esockd cowboy replayq - -dep_jsx = git-emqx https://github.com/talentdeficit/jsx 2.9.0 -dep_gproc = git-emqx https://github.com/uwiger/gproc 0.8.0 -dep_gen_rpc = git-emqx https://github.com/emqx/gen_rpc 2.3.1 -dep_esockd = git-emqx https://github.com/emqx/esockd v5.4.4 -dep_ekka = git-emqx https://github.com/emqx/ekka v0.5.4 -dep_cowboy = git-emqx https://github.com/ninenines/cowboy 2.6.1 -dep_replayq = git-emqx https://github.com/emqx/replayq v0.1.1 - -NO_AUTOPATCH = cuttlefish - -ERLC_OPTS += +debug_info -DAPPLICATION=emqx - -BUILD_DEPS = cuttlefish -dep_cuttlefish = git-emqx https://github.com/emqx/cuttlefish v2.2.1 - -CUR_BRANCH := $(shell git branch | grep -e "^*" | cut -d' ' -f 2) -BRANCH := $(if $(filter $(CUR_BRANCH), master develop), $(CUR_BRANCH), develop) - -TEST_DEPS = emqx_ct_helpers -dep_emqx_ct_helpers = git-emqx https://github.com/emqx/emqx-ct-helpers.git v1.0 - -TEST_ERLC_OPTS += +debug_info -DAPPLICATION=emqx - -EUNIT_OPTS = verbose - -# CT_SUITES = emqx_bridge -## emqx_trie emqx_router emqx_frame emqx_mqtt_compat +# CT_SUITES = emqx_trie emqx_router emqx_frame emqx_mqtt_compat CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ emqx_broker emqx_cm emqx_frame emqx_guid emqx_inflight emqx_json \ @@ -44,31 +16,32 @@ CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ emqx_vm_mon emqx_alarm_handler emqx_rpc emqx_flapping CT_NODE_NAME = emqxct@127.0.0.1 -CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME) -COVER = true +compile: + @rebar3 compile -PLT_APPS = sasl asn1 ssl syntax_tools runtime_tools crypto xmerl os_mon inets public_key ssl compiler mnesia -DIALYZER_DIRS := ebin/ -DIALYZER_OPTS := --verbose --statistics -Werror_handling -Wrace_conditions #-Wunmatched_returns - -$(shell [ -f erlang.mk ] || curl -s -o erlang.mk https://raw.githubusercontent.com/emqx/erlmk/master/erlang.mk) -include erlang.mk - -clean:: gen-clean +clean: gen-clean .PHONY: gen-clean gen-clean: - @rm -rf bbmustache @rm -f etc/gen.emqx.conf -bbmustache: - $(verbose) git clone https://github.com/soranoba/bbmustache.git && cd bbmustache && ./rebar3 compile && cd .. +## bbmustache is a mustache template library used to render templated config files +## for common tests. +BBMUSTACHE := _build/test/lib/bbmustache +$(BBMUSTACHE): + @rebar3 as test compile -# This hack is to generate a conf file for testing -# relx overlay is used for release -etc/gen.emqx.conf: bbmustache etc/emqx.conf - $(verbose) erl -noshell -pa bbmustache/_build/default/lib/bbmustache/ebin -eval \ +## Cuttlefish escript is built by default when cuttlefish app (as dependency) was built +CUTTLEFISH_SCRIPT := _build/default/lib/cuttlefish/cuttlefish + +app.config: etc/gen.emqx.conf + $(verbose) $(CUTTLEFISH_SCRIPT) -l info -e etc/ -c etc/gen.emqx.conf -i priv/emqx.schema -d data/ + +## NOTE: Mustache templating was resolved by relx overlay when building a release. +## This is only to generate a conf file for testing, +etc/gen.emqx.conf: $(BBMUSTACHE) etc/emqx.conf + @$(verbose) erl -noshell -pa _build/test/lib/bbmustache/ebin -eval \ "{ok, Temp} = file:read_file('etc/emqx.conf'), \ {ok, Vars0} = file:consult('vars'), \ Vars = [{atom_to_list(N), list_to_binary(V)} || {N, V} <- Vars0], \ @@ -76,51 +49,46 @@ etc/gen.emqx.conf: bbmustache etc/emqx.conf ok = file:write_file('etc/gen.emqx.conf', Targ), \ halt(0)." -CUTTLEFISH_SCRIPT = _build/default/lib/cuttlefish/cuttlefish - -app.config: $(CUTTLEFISH_SCRIPT) etc/gen.emqx.conf - $(verbose) $(CUTTLEFISH_SCRIPT) -l info -e etc/ -c etc/gen.emqx.conf -i priv/emqx.schema -d data/ - -ct: app.config - -rebar-cover: +.PHONY: cover +cover: @rebar3 cover +.PHONY: coveralls coveralls: @rebar3 coveralls send - -$(CUTTLEFISH_SCRIPT): rebar-deps - @if [ ! -f cuttlefish ]; then make -C _build/default/lib/cuttlefish; fi - -rebar-xref: +.PHONY: xref +xref: @rebar3 xref -rebar-deps: +.PHONY: deps +deps: @rebar3 get-deps -rebar-eunit: $(CUTTLEFISH_SCRIPT) +.PHONY: eunit +eunit: @rebar3 eunit -v -rebar-compile: - @rebar3 compile - -rebar-ct-setup: app.config +## 'ct-setup' is a pre hook for 'rebar3 ct', +## but not the makefile target ct's dependency +## because 'ct-setup' requires test dependencies to be compiled first +.PHONY: ct-setup +ct-setup: @rebar3 as test compile @ln -s -f '../../../../etc' _build/test/lib/emqx/ @ln -s -f '../../../../data' _build/test/lib/emqx/ -rebar-ct: rebar-ct-setup +.PHONY: ct +ct: app.config ct-setup @rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(shell echo $(foreach var,$(CT_SUITES),test/$(var)_SUITE) | tr ' ' ',') ## Run one single CT with rebar3 ## e.g. make ct-one-suite suite=emqx_bridge -ct-one-suite: rebar-ct-setup +.PHONY: ct-one-suite +ct-one-suite: ct-setup @rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(suite)_SUITE -rebar-clean: - @rebar3 clean - -distclean:: +.PHONY: clean +clean: @rm -rf _build cover deps logs log data @rm -f rebar.lock compile_commands.json cuttlefish diff --git a/rebar.config b/rebar.config index 8db9caa3e..b2dd5bcaa 100644 --- a/rebar.config +++ b/rebar.config @@ -1,17 +1,13 @@ -{deps, [{jsx, "2.9.0"}, - {gproc, "0.8.0"}, - {cowboy, "2.6.1"}]}. - -%% appended to deps in rebar.config.script -{github_emqx_libs, - [{gen_rpc, "2.3.1"}, - {ekka, "v0.5.4"}, - {replayq, "v0.1.1"}, - {esockd, "v5.4.4"}, - {cuttlefish, "v2.2.1"}]}. - -{github_emqx_projects, - [{emqx_ct_helpers, "v1.0"}]}. +{deps, + [ {jsx, "2.9.0"} % hex + , {cowboy, "2.6.1"} % hex + , {gproc, "0.8.0"} % hex + , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.3.1"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "v0.5.4"}}} + , {replayq, {git, "https://github.com/emqx/replayq", {tag, "v0.1.1"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "v5.4.4"}}} + , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}} + ]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, @@ -28,3 +24,13 @@ {cover_export_enabled, true}. {plugins, [coveralls]}. + +{profiles, + [{test, + [{deps, + [ {meck, "0.8.13"} % hex + , {bbmustache, "1.7.0"} % hex + , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.0"}}} + ]} + ]} + ]}. diff --git a/rebar.config.script b/rebar.config.script index 36028ee3d..558910385 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -1,11 +1,3 @@ -CONFIG0 = case os:getenv("REBAR_GIT_CLONE_OPTIONS") of - "--depth 1" -> - CONFIG; - _ -> - os:putenv("REBAR_GIT_CLONE_OPTIONS", "--depth 1"), - CONFIG - end, - CONFIG1 = case os:getenv("TRAVIS") of "true" -> JobId = os:getenv("TRAVIS_JOB_ID"), @@ -16,28 +8,4 @@ CONFIG1 = case os:getenv("TRAVIS") of CONFIG end, -FindDeps = fun(DepsType, Config) -> - case lists:keyfind(DepsType, 1, Config) of - {_, RawDeps} -> RawDeps; - _ -> [] - end - end, -Deps = FindDeps(deps, CONFIG1), -LibDeps = FindDeps(github_emqx_libs, CONFIG1), -ProjDeps = FindDeps(github_emqx_projects, CONFIG1), -UrlPrefix = "https://github.com/emqx/", -RealName = fun TransName([$_ | Tail], Result) -> - TransName(Tail, [$- | Result]); - TransName([Head | Tail], Result) -> - TransName(Tail, [Head | Result]); - TransName([], Result) -> - lists:reverse(Result) - end, - -NewLibDeps = [{LibName, {git, UrlPrefix ++ atom_to_list(LibName), {branch, Branch}}} - || {LibName, Branch} <- LibDeps], -NewProjDeps = [{ProjName, {git, UrlPrefix ++ RealName(atom_to_list(ProjName), []), {branch, Branch}}} || {ProjName, Branch} <- ProjDeps], - -NewDeps = Deps ++ NewLibDeps ++ NewProjDeps, -CONFIG2 = lists:keystore(deps, 1, CONFIG1, {deps, NewDeps}), -CONFIG2. +CONFIG1. diff --git a/test/emqx_vm_mon_SUITE.erl b/test/emqx_vm_mon_SUITE.erl index 41a717293..3718e3626 100644 --- a/test/emqx_vm_mon_SUITE.erl +++ b/test/emqx_vm_mon_SUITE.erl @@ -23,6 +23,15 @@ -include_lib("common_test/include/ct.hrl"). +-define(WAIT(PATTERN, TIMEOUT), + receive + PATTERN -> + ok + after + TIMEOUT -> + error(timeout) + end). + all() -> [t_api]. init_per_suite(Config) -> @@ -33,18 +42,36 @@ end_per_suite(_Config) -> application:stop(sasl). t_api(_) -> - gen_event:swap_handler(alarm_handler, {emqx_alarm_handler, swap}, {alarm_handler, []}), - {ok, _} = emqx_vm_mon:start_link([{check_interval, 1}, - {process_high_watermark, 0}, - {process_low_watermark, 0.6}]), - timer:sleep(2000), - ?assertEqual(true, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())), - emqx_vm_mon:set_process_high_watermark(0.8), - emqx_vm_mon:set_process_low_watermark(0.75), - ?assertEqual(0.8, emqx_vm_mon:get_process_high_watermark()), - ?assertEqual(0.75, emqx_vm_mon:get_process_low_watermark()), - timer:sleep(3000), - ?assertEqual(false, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())), - emqx_vm_mon:set_check_interval(20), - ?assertEqual(20, emqx_vm_mon:get_check_interval()), - ok. + meck:new(alarm_handler, [passthrough, no_history]), + Tester = self(), + Ref = make_ref(), + try + meck:expect(alarm_handler, set_alarm, + fun(What) -> + Res = meck:passthrough([What]), + Tester ! {Ref, set_alarm, What}, + Res + end), + meck:expect(alarm_handler, clear_alarm, + fun(What) -> + Res = meck:passthrough([What]), + Tester ! {Ref, clear_alarm, What}, + Res + end), + gen_event:swap_handler(alarm_handler, {emqx_alarm_handler, swap}, {alarm_handler, []}), + {ok, _} = emqx_vm_mon:start_link([{check_interval, 1}, + {process_high_watermark, 0}, + {process_low_watermark, 0.6}]), + ?WAIT({Ref, set_alarm, {too_many_processes, _Count}}, 2000), + ?assertEqual(true, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())), + emqx_vm_mon:set_process_high_watermark(0.8), + emqx_vm_mon:set_process_low_watermark(0.75), + ?assertEqual(0.8, emqx_vm_mon:get_process_high_watermark()), + ?assertEqual(0.75, emqx_vm_mon:get_process_low_watermark()), + ?WAIT({Ref, clear_alarm, too_many_processes}, 3000), + ?assertEqual(false, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())), + emqx_vm_mon:set_check_interval(20), + ?assertEqual(20, emqx_vm_mon:get_check_interval()) + after + meck:unload(alarm_handler) + end.