diff --git a/.gitignore b/.gitignore index ab0cbe156..16369a576 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ rebar.lock xrefr erlang.mk *.coverdata +etc/emqx.conf.rendered diff --git a/.travis.yml b/.travis.yml index f2c483cc6..bef59bf4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,12 +7,11 @@ before_install: - git clone https://github.com/erlang/rebar3.git; cd rebar3; ./bootstrap; sudo mv rebar3 /usr/local/bin/; cd .. script: - - make dep-vsn-check - - make rebar-compile - - make rebar-xref - - make rebar-eunit - - make rebar-ct - - make rebar-cover + - make compile + - make xref + - make eunit + - make ct + - make cover after_success: - make coveralls diff --git a/Makefile b/Makefile index acb854fa4..0296f7bb9 100644 --- a/Makefile +++ b/Makefile @@ -1,37 +1,9 @@ -.PHONY: plugins tests +## shallow clone for speed -PROJECT = emqx -PROJECT_DESCRIPTION = EMQ X Broker +REBAR_GIT_CLONE_OPTIONS += --depth 1 +export REBAR_GIT_CLONE_OPTIONS -DEPS = jsx gproc gen_rpc ekka esockd cowboy replayq - -dep_jsx = git-emqx https://github.com/talentdeficit/jsx 2.9.0 -dep_gproc = git-emqx https://github.com/uwiger/gproc 0.8.0 -dep_gen_rpc = git-emqx https://github.com/emqx/gen_rpc 2.3.1 -dep_esockd = git-emqx https://github.com/emqx/esockd v5.4.4 -dep_ekka = git-emqx https://github.com/emqx/ekka v0.5.4 -dep_cowboy = git-emqx https://github.com/ninenines/cowboy 2.6.1 -dep_replayq = git-emqx https://github.com/emqx/replayq v0.1.1 - -NO_AUTOPATCH = cuttlefish - -ERLC_OPTS += +debug_info -DAPPLICATION=emqx - -BUILD_DEPS = cuttlefish -dep_cuttlefish = git-emqx https://github.com/emqx/cuttlefish v2.2.1 - -CUR_BRANCH := $(shell git branch | grep -e "^*" | cut -d' ' -f 2) -BRANCH := $(if $(filter $(CUR_BRANCH), master develop), $(CUR_BRANCH), develop) - -TEST_DEPS = emqx_ct_helpers -dep_emqx_ct_helpers = git-emqx https://github.com/emqx/emqx-ct-helpers.git v1.0 - -TEST_ERLC_OPTS += +debug_info -DAPPLICATION=emqx - -EUNIT_OPTS = verbose - -# CT_SUITES = emqx_bridge -## emqx_trie emqx_router emqx_frame emqx_mqtt_compat +# CT_SUITES = emqx_trie emqx_router emqx_frame emqx_mqtt_compat CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ emqx_broker emqx_cm emqx_frame emqx_guid emqx_inflight emqx_json \ @@ -44,31 +16,67 @@ CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ emqx_vm_mon emqx_alarm_handler emqx_rpc emqx_flapping CT_NODE_NAME = emqxct@127.0.0.1 -CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME) -COVER = true +compile: + @rebar3 compile -PLT_APPS = sasl asn1 ssl syntax_tools runtime_tools crypto xmerl os_mon inets public_key ssl compiler mnesia -DIALYZER_DIRS := ebin/ -DIALYZER_OPTS := --verbose --statistics -Werror_handling -Wrace_conditions #-Wunmatched_returns +clean: distclean -$(shell [ -f erlang.mk ] || curl -s -o erlang.mk https://raw.githubusercontent.com/emqx/erlmk/master/erlang.mk) -include erlang.mk +## Cuttlefish escript is built by default when cuttlefish app (as dependency) was built +CUTTLEFISH_SCRIPT := _build/default/lib/cuttlefish/cuttlefish -clean:: gen-clean +.PHONY: cover +cover: + @rebar3 cover -.PHONY: gen-clean -gen-clean: - @rm -rf bbmustache - @rm -f etc/gen.emqx.conf +.PHONY: coveralls +coveralls: + @rebar3 coveralls send + +.PHONY: xref +xref: + @rebar3 xref + +.PHONY: deps +deps: + @rebar3 get-deps + +.PHONY: eunit +eunit: + @rebar3 eunit -v + +.PHONY: ct-setup +ct-setup: + @mkdir -p data + @if [ ! -f data/loaded_plugins ]; then touch data/loaded_plugins; fi + @ln -s -f '../../../../etc' _build/test/lib/emqx/ + @ln -s -f '../../../../data' _build/test/lib/emqx/ + +.PHONY: ct +ct: ct-setup + @rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(shell echo $(foreach var,$(CT_SUITES),test/$(var)_SUITE) | tr ' ' ',') + +## Run one single CT with rebar3 +## e.g. make ct-one-suite suite=emqx_bridge +.PHONY: ct-one-suite +ct-one-suite: ct-setup + @rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(suite)_SUITE + +.PHONY: app.config +app.config: $(CUTTLEFISH_SCRIPT) etc/gen.emqx.conf + $(CUTTLEFISH_SCRIPT) -l info -e etc/ -c etc/gen.emqx.conf -i priv/emqx.schema -d data/ + +$(CUTTLEFISH_SCRIPT): + @rebar3 get-deps + @if [ ! -f cuttlefish ]; then make -C _build/default/lib/cuttlefish; fi bbmustache: - $(verbose) git clone https://github.com/soranoba/bbmustache.git && cd bbmustache && ./rebar3 compile && cd .. + @git clone https://github.com/soranoba/bbmustache.git && cd bbmustache && ./rebar3 compile && cd .. # This hack is to generate a conf file for testing # relx overlay is used for release etc/gen.emqx.conf: bbmustache etc/emqx.conf - $(verbose) erl -noshell -pa bbmustache/_build/default/lib/bbmustache/ebin -eval \ + @erl -noshell -pa bbmustache/_build/default/lib/bbmustache/ebin -eval \ "{ok, Temp} = file:read_file('etc/emqx.conf'), \ {ok, Vars0} = file:consult('vars'), \ Vars = [{atom_to_list(N), list_to_binary(V)} || {N, V} <- Vars0], \ @@ -76,51 +84,12 @@ etc/gen.emqx.conf: bbmustache etc/emqx.conf ok = file:write_file('etc/gen.emqx.conf', Targ), \ halt(0)." -CUTTLEFISH_SCRIPT = _build/default/lib/cuttlefish/cuttlefish +.PHONY: gen-clean +gen-clean: + @rm -rf bbmustache + @rm -f etc/gen.emqx.conf etc/emqx.conf.rendered -app.config: $(CUTTLEFISH_SCRIPT) etc/gen.emqx.conf - $(verbose) $(CUTTLEFISH_SCRIPT) -l info -e etc/ -c etc/gen.emqx.conf -i priv/emqx.schema -d data/ - -ct: app.config - -rebar-cover: - @rebar3 cover - -coveralls: - @rebar3 coveralls send - - -$(CUTTLEFISH_SCRIPT): rebar-deps - @if [ ! -f cuttlefish ]; then make -C _build/default/lib/cuttlefish; fi - -rebar-xref: - @rebar3 xref - -rebar-deps: - @rebar3 get-deps - -rebar-eunit: $(CUTTLEFISH_SCRIPT) - @rebar3 eunit -v - -rebar-compile: - @rebar3 compile - -rebar-ct-setup: app.config - @rebar3 as test compile - @ln -s -f '../../../../etc' _build/test/lib/emqx/ - @ln -s -f '../../../../data' _build/test/lib/emqx/ - -rebar-ct: rebar-ct-setup - @rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(shell echo $(foreach var,$(CT_SUITES),test/$(var)_SUITE) | tr ' ' ',') - -## Run one single CT with rebar3 -## e.g. make ct-one-suite suite=emqx_bridge -ct-one-suite: rebar-ct-setup - @rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(suite)_SUITE - -rebar-clean: - @rebar3 clean - -distclean:: +.PHONY: distclean +distclean: gen-clean @rm -rf _build cover deps logs log data - @rm -f rebar.lock compile_commands.json cuttlefish + @rm -f rebar.lock compile_commands.json cuttlefish erl_crash.dump diff --git a/rebar.config b/rebar.config index 8db9caa3e..3bbe75002 100644 --- a/rebar.config +++ b/rebar.config @@ -1,17 +1,13 @@ -{deps, [{jsx, "2.9.0"}, - {gproc, "0.8.0"}, - {cowboy, "2.6.1"}]}. - -%% appended to deps in rebar.config.script -{github_emqx_libs, - [{gen_rpc, "2.3.1"}, - {ekka, "v0.5.4"}, - {replayq, "v0.1.1"}, - {esockd, "v5.4.4"}, - {cuttlefish, "v2.2.1"}]}. - -{github_emqx_projects, - [{emqx_ct_helpers, "v1.0"}]}. +{deps, + [ {jsx, "2.9.0"} % hex + , {cowboy, "2.6.1"} % hex + , {gproc, "0.8.0"} % hex + , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.3.1"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "v0.5.4"}}} + , {replayq, {git, "https://github.com/emqx/replayq", {tag, "v0.1.1"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "v5.4.4"}}} + , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}} + ]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, @@ -28,3 +24,13 @@ {cover_export_enabled, true}. {plugins, [coveralls]}. + +{profiles, + [{test, + [{deps, + [ {meck, "0.8.13"} % hex + , {bbmustache, "1.7.0"} % hex + , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.1.1"}}} + ]} + ]} + ]}. diff --git a/rebar.config.script b/rebar.config.script index 36028ee3d..558910385 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -1,11 +1,3 @@ -CONFIG0 = case os:getenv("REBAR_GIT_CLONE_OPTIONS") of - "--depth 1" -> - CONFIG; - _ -> - os:putenv("REBAR_GIT_CLONE_OPTIONS", "--depth 1"), - CONFIG - end, - CONFIG1 = case os:getenv("TRAVIS") of "true" -> JobId = os:getenv("TRAVIS_JOB_ID"), @@ -16,28 +8,4 @@ CONFIG1 = case os:getenv("TRAVIS") of CONFIG end, -FindDeps = fun(DepsType, Config) -> - case lists:keyfind(DepsType, 1, Config) of - {_, RawDeps} -> RawDeps; - _ -> [] - end - end, -Deps = FindDeps(deps, CONFIG1), -LibDeps = FindDeps(github_emqx_libs, CONFIG1), -ProjDeps = FindDeps(github_emqx_projects, CONFIG1), -UrlPrefix = "https://github.com/emqx/", -RealName = fun TransName([$_ | Tail], Result) -> - TransName(Tail, [$- | Result]); - TransName([Head | Tail], Result) -> - TransName(Tail, [Head | Result]); - TransName([], Result) -> - lists:reverse(Result) - end, - -NewLibDeps = [{LibName, {git, UrlPrefix ++ atom_to_list(LibName), {branch, Branch}}} - || {LibName, Branch} <- LibDeps], -NewProjDeps = [{ProjName, {git, UrlPrefix ++ RealName(atom_to_list(ProjName), []), {branch, Branch}}} || {ProjName, Branch} <- ProjDeps], - -NewDeps = Deps ++ NewLibDeps ++ NewProjDeps, -CONFIG2 = lists:keystore(deps, 1, CONFIG1, {deps, NewDeps}), -CONFIG2. +CONFIG1. diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index af7a32f9a..1c1185bf0 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -108,13 +108,20 @@ ensure_file(File) -> with_loaded_file(File, SuccFun) -> case read_loaded(File) of - {ok, Names} -> + {ok, Names0} -> + Names = filter_plugins(Names0), SuccFun(Names); {error, Error} -> ?LOG(alert, "[Plugins] Failed to read: ~p, error: ~p", [File, Error]), {error, Error} end. +filter_plugins(Names) -> + lists:filtermap(fun(Name1) when is_atom(Name1) -> {true, Name1}; + ({Name1, true}) -> {true, Name1}; + ({_Name1, false}) -> false + end, Names). + load_plugins(Names, Persistent) -> Plugins = list(), NotFound = Names -- names(Plugins), case NotFound of @@ -264,7 +271,7 @@ plugin_loaded(Name, true) -> case lists:member(Name, Names) of false -> %% write file if plugin is loaded - write_loaded(lists:append(Names, [Name])); + write_loaded(lists:append(Names, [{Name, true}])); true -> ignore end; @@ -277,10 +284,7 @@ plugin_unloaded(_Name, false) -> plugin_unloaded(Name, true) -> case read_loaded() of {ok, Names0} -> - Names = lists:filtermap(fun(Name1) when is_atom(Name1) -> {true, Name1}; - ({Name1, true}) -> {true, Name1}; - ({_Name1, false}) -> false - end, Names0), + Names = filter_plugins(Names0), case lists:member(Name, Names) of true -> write_loaded(lists:delete(Name, Names)); @@ -304,7 +308,7 @@ write_loaded(AppNames) -> case file:open(File, [binary, write]) of {ok, Fd} -> lists:foreach(fun(Name) -> - file:write(Fd, iolist_to_binary(io_lib:format("~s.~n", [Name]))) + file:write(Fd, iolist_to_binary(io_lib:format("~p.~n", [Name]))) end, AppNames); {error, Error} -> ?LOG(error, "[Plugins] Open File ~p Error: ~p", [File, Error]), diff --git a/test/emqx_listeners_SUITE.erl b/test/emqx_listeners_SUITE.erl index d969699bc..cab8707d5 100644 --- a/test/emqx_listeners_SUITE.erl +++ b/test/emqx_listeners_SUITE.erl @@ -49,9 +49,27 @@ restart_listeners(_) -> ok = emqx_listeners:restart(), ok = emqx_listeners:stop(). +render_config_file() -> + Path = local_path(["etc", "emqx.conf"]), + {ok, Temp} = file:read_file(Path), + Vars0 = mustache_vars(), + Vars = [{atom_to_list(N), iolist_to_binary(V)} || {N, V} <- Vars0], + Targ = bbmustache:render(Temp, Vars), + NewName = Path ++ ".rendered", + ok = file:write_file(NewName, Targ), + NewName. + +mustache_vars() -> + [{platform_data_dir, local_path(["data"])}, + {platform_etc_dir, local_path(["etc"])}, + {platform_log_dir, local_path(["log"])}, + {platform_plugins_dir, local_path(["plugins"])} + ]. + generate_config() -> Schema = cuttlefish_schema:files([local_path(["priv", "emqx.schema"])]), - Conf = conf_parse:file([local_path(["etc", "gen.emqx.conf"])]), + ConfFile = render_config_file(), + Conf = conf_parse:file(ConfFile), cuttlefish_generator:map(Schema, Conf). set_app_env({App, Lists}) -> diff --git a/test/emqx_vm_mon_SUITE.erl b/test/emqx_vm_mon_SUITE.erl index 41a717293..3718e3626 100644 --- a/test/emqx_vm_mon_SUITE.erl +++ b/test/emqx_vm_mon_SUITE.erl @@ -23,6 +23,15 @@ -include_lib("common_test/include/ct.hrl"). +-define(WAIT(PATTERN, TIMEOUT), + receive + PATTERN -> + ok + after + TIMEOUT -> + error(timeout) + end). + all() -> [t_api]. init_per_suite(Config) -> @@ -33,18 +42,36 @@ end_per_suite(_Config) -> application:stop(sasl). t_api(_) -> - gen_event:swap_handler(alarm_handler, {emqx_alarm_handler, swap}, {alarm_handler, []}), - {ok, _} = emqx_vm_mon:start_link([{check_interval, 1}, - {process_high_watermark, 0}, - {process_low_watermark, 0.6}]), - timer:sleep(2000), - ?assertEqual(true, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())), - emqx_vm_mon:set_process_high_watermark(0.8), - emqx_vm_mon:set_process_low_watermark(0.75), - ?assertEqual(0.8, emqx_vm_mon:get_process_high_watermark()), - ?assertEqual(0.75, emqx_vm_mon:get_process_low_watermark()), - timer:sleep(3000), - ?assertEqual(false, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())), - emqx_vm_mon:set_check_interval(20), - ?assertEqual(20, emqx_vm_mon:get_check_interval()), - ok. + meck:new(alarm_handler, [passthrough, no_history]), + Tester = self(), + Ref = make_ref(), + try + meck:expect(alarm_handler, set_alarm, + fun(What) -> + Res = meck:passthrough([What]), + Tester ! {Ref, set_alarm, What}, + Res + end), + meck:expect(alarm_handler, clear_alarm, + fun(What) -> + Res = meck:passthrough([What]), + Tester ! {Ref, clear_alarm, What}, + Res + end), + gen_event:swap_handler(alarm_handler, {emqx_alarm_handler, swap}, {alarm_handler, []}), + {ok, _} = emqx_vm_mon:start_link([{check_interval, 1}, + {process_high_watermark, 0}, + {process_low_watermark, 0.6}]), + ?WAIT({Ref, set_alarm, {too_many_processes, _Count}}, 2000), + ?assertEqual(true, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())), + emqx_vm_mon:set_process_high_watermark(0.8), + emqx_vm_mon:set_process_low_watermark(0.75), + ?assertEqual(0.8, emqx_vm_mon:get_process_high_watermark()), + ?assertEqual(0.75, emqx_vm_mon:get_process_low_watermark()), + ?WAIT({Ref, clear_alarm, too_many_processes}, 3000), + ?assertEqual(false, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())), + emqx_vm_mon:set_check_interval(20), + ?assertEqual(20, emqx_vm_mon:get_check_interval()) + after + meck:unload(alarm_handler) + end. diff --git a/vars b/vars index fedd69a45..359c27c22 100644 --- a/vars +++ b/vars @@ -5,5 +5,4 @@ {platform_etc_dir, "etc"}. {platform_lib_dir, "lib"}. {platform_log_dir, "log"}. -{platform_plugins_dir, "plugins"}. - +{platform_plugins_dir, "plugins"}.