Merge pull request #2521 from emqx/develop-3.2

.gitignore

@@ -39,3 +39,4 @@ rebar.lock
 xrefr
 erlang.mk
 *.coverdata
+etc/emqx.conf.rendered

.travis.yml

@@ -7,12 +7,11 @@ before_install:
     - git clone https://github.com/erlang/rebar3.git; cd rebar3; ./bootstrap; sudo mv rebar3 /usr/local/bin/; cd ..
 
 script:
-  - make dep-vsn-check
+  - make compile
-  - make rebar-compile
+  - make xref
-  - make rebar-xref
+  - make eunit
-  - make rebar-eunit
+  - make ct
-  - make rebar-ct
+  - make cover
-  - make rebar-cover
 
 after_success:
   - make coveralls

Makefile

@@ -1,37 +1,9 @@
-.PHONY: plugins tests
+## shallow clone for speed
 
-PROJECT = emqx
+REBAR_GIT_CLONE_OPTIONS += --depth 1
-PROJECT_DESCRIPTION = EMQ X Broker
+export REBAR_GIT_CLONE_OPTIONS
 
-DEPS = jsx gproc gen_rpc ekka esockd cowboy replayq
+# CT_SUITES = emqx_trie emqx_router emqx_frame emqx_mqtt_compat
-
-dep_jsx     = git-emqx https://github.com/talentdeficit/jsx 2.9.0
-dep_gproc   = git-emqx https://github.com/uwiger/gproc 0.8.0
-dep_gen_rpc = git-emqx https://github.com/emqx/gen_rpc 2.3.1
-dep_esockd  = git-emqx https://github.com/emqx/esockd v5.4.4
-dep_ekka    = git-emqx https://github.com/emqx/ekka v0.5.4
-dep_cowboy  = git-emqx https://github.com/ninenines/cowboy 2.6.1
-dep_replayq = git-emqx https://github.com/emqx/replayq v0.1.1
-
-NO_AUTOPATCH = cuttlefish
-
-ERLC_OPTS += +debug_info -DAPPLICATION=emqx
-
-BUILD_DEPS = cuttlefish
-dep_cuttlefish = git-emqx https://github.com/emqx/cuttlefish v2.2.1
-
-CUR_BRANCH := $(shell git branch | grep -e "^*" | cut -d' ' -f 2)
-BRANCH := $(if $(filter $(CUR_BRANCH), master develop), $(CUR_BRANCH), develop)
-
-TEST_DEPS = emqx_ct_helpers
-dep_emqx_ct_helpers = git-emqx https://github.com/emqx/emqx-ct-helpers.git v1.0
-
-TEST_ERLC_OPTS += +debug_info -DAPPLICATION=emqx
-
-EUNIT_OPTS = verbose
-
-# CT_SUITES = emqx_bridge
-## emqx_trie emqx_router emqx_frame emqx_mqtt_compat
 
 CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \
 			emqx_broker emqx_cm emqx_frame emqx_guid emqx_inflight emqx_json \
@@ -44,31 +16,67 @@ CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \
             emqx_vm_mon emqx_alarm_handler emqx_rpc emqx_flapping
 
 CT_NODE_NAME = emqxct@127.0.0.1
-CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME)
 
-COVER = true
+compile:
+	@rebar3 compile
 
-PLT_APPS = sasl asn1 ssl syntax_tools runtime_tools crypto xmerl os_mon inets public_key ssl compiler mnesia
+clean: distclean
-DIALYZER_DIRS := ebin/
-DIALYZER_OPTS := --verbose --statistics -Werror_handling -Wrace_conditions #-Wunmatched_returns
 
-$(shell [ -f erlang.mk ] || curl -s -o erlang.mk https://raw.githubusercontent.com/emqx/erlmk/master/erlang.mk)
+## Cuttlefish escript is built by default when cuttlefish app (as dependency) was built
-include erlang.mk
+CUTTLEFISH_SCRIPT := _build/default/lib/cuttlefish/cuttlefish
 
-clean:: gen-clean
+.PHONY: cover
+cover:
+	@rebar3 cover
 
-.PHONY: gen-clean
+.PHONY: coveralls
-gen-clean:
+coveralls:
-	@rm -rf bbmustache
+	@rebar3 coveralls send
-	@rm -f etc/gen.emqx.conf
+
+.PHONY: xref
+xref:
+	@rebar3 xref
+
+.PHONY: deps
+deps:
+	@rebar3 get-deps
+
+.PHONY: eunit
+eunit:
+	@rebar3 eunit -v
+
+.PHONY: ct-setup
+ct-setup:
+	@mkdir -p data
+	@if [ ! -f data/loaded_plugins ]; then touch data/loaded_plugins; fi
+	@ln -s -f '../../../../etc' _build/test/lib/emqx/
+	@ln -s -f '../../../../data' _build/test/lib/emqx/
+
+.PHONY: ct
+ct: ct-setup
+	@rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(shell echo $(foreach var,$(CT_SUITES),test/$(var)_SUITE) | tr ' ' ',')
+
+## Run one single CT with rebar3
+## e.g. make ct-one-suite suite=emqx_bridge
+.PHONY: ct-one-suite
+ct-one-suite: ct-setup
+	@rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(suite)_SUITE
+
+.PHONY: app.config
+app.config: $(CUTTLEFISH_SCRIPT) etc/gen.emqx.conf
+	$(CUTTLEFISH_SCRIPT) -l info -e etc/ -c etc/gen.emqx.conf -i priv/emqx.schema -d data/
+
+$(CUTTLEFISH_SCRIPT):
+	@rebar3 get-deps
+	@if [ ! -f cuttlefish ]; then make -C _build/default/lib/cuttlefish; fi
 
 bbmustache:
-	$(verbose) git clone https://github.com/soranoba/bbmustache.git && cd bbmustache && ./rebar3 compile && cd ..
+	@git clone https://github.com/soranoba/bbmustache.git && cd bbmustache && ./rebar3 compile && cd ..
 
 # This hack is to generate a conf file for testing
 # relx overlay is used for release
 etc/gen.emqx.conf: bbmustache etc/emqx.conf
-	$(verbose) erl -noshell -pa bbmustache/_build/default/lib/bbmustache/ebin -eval \
+	@erl -noshell -pa bbmustache/_build/default/lib/bbmustache/ebin -eval \
 		"{ok, Temp} = file:read_file('etc/emqx.conf'), \
 		{ok, Vars0} = file:consult('vars'), \
 		Vars = [{atom_to_list(N), list_to_binary(V)} || {N, V} <- Vars0], \
@@ -76,51 +84,12 @@ etc/gen.emqx.conf: bbmustache etc/emqx.conf
 		ok = file:write_file('etc/gen.emqx.conf', Targ), \
 		halt(0)."
 
-CUTTLEFISH_SCRIPT = _build/default/lib/cuttlefish/cuttlefish
+.PHONY: gen-clean
+gen-clean:
+	@rm -rf bbmustache
+	@rm -f etc/gen.emqx.conf etc/emqx.conf.rendered
 
-app.config: $(CUTTLEFISH_SCRIPT) etc/gen.emqx.conf
+.PHONY: distclean
-	$(verbose) $(CUTTLEFISH_SCRIPT) -l info -e etc/ -c etc/gen.emqx.conf -i priv/emqx.schema -d data/
+distclean: gen-clean
-
-ct: app.config
-
-rebar-cover:
-	@rebar3 cover
-
-coveralls:
-	@rebar3 coveralls send
-
-
-$(CUTTLEFISH_SCRIPT): rebar-deps
-	@if [ ! -f cuttlefish ]; then make -C _build/default/lib/cuttlefish; fi
-
-rebar-xref:
-	@rebar3 xref
-
-rebar-deps:
-	@rebar3 get-deps
-
-rebar-eunit: $(CUTTLEFISH_SCRIPT)
-	@rebar3 eunit -v
-
-rebar-compile:
-	@rebar3 compile
-
-rebar-ct-setup: app.config
-	@rebar3 as test compile
-	@ln -s -f '../../../../etc' _build/test/lib/emqx/
-	@ln -s -f '../../../../data' _build/test/lib/emqx/
-
-rebar-ct: rebar-ct-setup
-	@rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(shell echo $(foreach var,$(CT_SUITES),test/$(var)_SUITE) | tr ' ' ',')
-
-## Run one single CT with rebar3
-## e.g. make ct-one-suite suite=emqx_bridge
-ct-one-suite: rebar-ct-setup
-	@rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(suite)_SUITE
-
-rebar-clean:
-	@rebar3 clean
-
-distclean::
 	@rm -rf _build cover deps logs log data
-	@rm -f rebar.lock compile_commands.json cuttlefish
+	@rm -f rebar.lock compile_commands.json cuttlefish erl_crash.dump

rebar.config

@@ -1,17 +1,13 @@
-{deps, [{jsx, "2.9.0"},
+{deps,
-        {gproc, "0.8.0"},
+    [ {jsx, "2.9.0"} % hex
-        {cowboy, "2.6.1"}]}.
+    , {cowboy, "2.6.1"} % hex
-
+    , {gproc, "0.8.0"} % hex
-%% appended to deps in rebar.config.script
+    , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.3.1"}}}
-{github_emqx_libs,
+    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "v0.5.4"}}}
- [{gen_rpc, "2.3.1"},
+    , {replayq, {git, "https://github.com/emqx/replayq", {tag, "v0.1.1"}}}
-  {ekka, "v0.5.4"},
+    , {esockd, {git, "https://github.com/emqx/esockd", {tag, "v5.4.4"}}}
-  {replayq, "v0.1.1"},
+    , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}}
-  {esockd, "v5.4.4"},
+    ]}.
-  {cuttlefish, "v2.2.1"}]}.
-
-{github_emqx_projects,
- [{emqx_ct_helpers, "v1.0"}]}.
 
 {edoc_opts, [{preprocess, true}]}.
 {erl_opts, [warn_unused_vars,
@@ -28,3 +24,13 @@
 {cover_export_enabled, true}.
 
 {plugins, [coveralls]}.
+
+{profiles,
+    [{test,
+        [{deps,
+            [ {meck, "0.8.13"} % hex
+            , {bbmustache, "1.7.0"} % hex
+            , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.1.1"}}}
+            ]}
+        ]}
+    ]}.

rebar.config.script

@@ -1,11 +1,3 @@
-CONFIG0 = case os:getenv("REBAR_GIT_CLONE_OPTIONS") of
-              "--depth 1" ->
-                  CONFIG;
-              _ ->
-                  os:putenv("REBAR_GIT_CLONE_OPTIONS", "--depth 1"),
-                  CONFIG
-          end,
-
 CONFIG1 = case os:getenv("TRAVIS") of
               "true" ->
                   JobId = os:getenv("TRAVIS_JOB_ID"),
@@ -16,28 +8,4 @@ CONFIG1 = case os:getenv("TRAVIS") of
                   CONFIG
           end,
 
-FindDeps = fun(DepsType, Config) ->
+CONFIG1.
-                   case lists:keyfind(DepsType, 1, Config) of
-                       {_, RawDeps} -> RawDeps;
-                       _ -> []
-                   end
-           end,
-Deps = FindDeps(deps, CONFIG1),
-LibDeps = FindDeps(github_emqx_libs, CONFIG1),
-ProjDeps = FindDeps(github_emqx_projects, CONFIG1),
-UrlPrefix = "https://github.com/emqx/",
-RealName = fun TransName([$_ | Tail], Result) ->
-                   TransName(Tail, [$- | Result]);
-               TransName([Head | Tail], Result) ->
-                   TransName(Tail, [Head | Result]);
-               TransName([], Result) ->
-                   lists:reverse(Result)
-           end,
-
-NewLibDeps = [{LibName, {git, UrlPrefix ++ atom_to_list(LibName), {branch, Branch}}}
-              || {LibName, Branch} <- LibDeps],
-NewProjDeps = [{ProjName, {git, UrlPrefix ++ RealName(atom_to_list(ProjName), []), {branch, Branch}}} || {ProjName, Branch} <- ProjDeps],
-
-NewDeps = Deps ++ NewLibDeps ++ NewProjDeps,
-CONFIG2 = lists:keystore(deps, 1, CONFIG1, {deps, NewDeps}),
-CONFIG2.

src/emqx_plugins.erl

@@ -108,13 +108,20 @@ ensure_file(File) ->
 
 with_loaded_file(File, SuccFun) ->
     case read_loaded(File) of
-        {ok, Names} ->
+        {ok, Names0} ->
+            Names = filter_plugins(Names0),
             SuccFun(Names);
         {error, Error} ->
             ?LOG(alert, "[Plugins] Failed to read: ~p, error: ~p", [File, Error]),
             {error, Error}
     end.
 
+filter_plugins(Names) ->
+    lists:filtermap(fun(Name1) when is_atom(Name1) -> {true, Name1};
+                       ({Name1, true}) -> {true, Name1};
+                       ({_Name1, false}) -> false
+                    end, Names).
+
 load_plugins(Names, Persistent) ->
     Plugins = list(), NotFound = Names -- names(Plugins),
     case NotFound of
@@ -264,7 +271,7 @@ plugin_loaded(Name, true) ->
             case lists:member(Name, Names) of
                 false ->
                     %% write file if plugin is loaded
-                    write_loaded(lists:append(Names, [Name]));
+                    write_loaded(lists:append(Names, [{Name, true}]));
                 true ->
                     ignore
             end;
@@ -277,10 +284,7 @@ plugin_unloaded(_Name, false) ->
 plugin_unloaded(Name, true) ->
     case read_loaded() of
         {ok, Names0} ->
-            Names = lists:filtermap(fun(Name1) when is_atom(Name1) -> {true, Name1};
+            Names = filter_plugins(Names0),
-                                       ({Name1, true}) -> {true, Name1};
-                                       ({_Name1, false}) -> false
-                                    end, Names0),
             case lists:member(Name, Names) of
                 true ->
                     write_loaded(lists:delete(Name, Names));
@@ -304,7 +308,7 @@ write_loaded(AppNames) ->
     case file:open(File, [binary, write]) of
         {ok, Fd} ->
             lists:foreach(fun(Name) ->
-                file:write(Fd, iolist_to_binary(io_lib:format("~s.~n", [Name])))
+                file:write(Fd, iolist_to_binary(io_lib:format("~p.~n", [Name])))
             end, AppNames);
         {error, Error} ->
             ?LOG(error, "[Plugins] Open File ~p Error: ~p", [File, Error]),

test/emqx_listeners_SUITE.erl

@@ -49,9 +49,27 @@ restart_listeners(_) ->
     ok = emqx_listeners:restart(),
     ok = emqx_listeners:stop().
 
+render_config_file() ->
+    Path = local_path(["etc", "emqx.conf"]),
+    {ok, Temp} = file:read_file(Path),
+    Vars0 = mustache_vars(),
+    Vars = [{atom_to_list(N), iolist_to_binary(V)} || {N, V} <- Vars0],
+    Targ = bbmustache:render(Temp, Vars),
+    NewName = Path ++ ".rendered",
+    ok = file:write_file(NewName, Targ),
+    NewName.
+
+mustache_vars() ->
+    [{platform_data_dir, local_path(["data"])},
+     {platform_etc_dir,  local_path(["etc"])},
+     {platform_log_dir,  local_path(["log"])},
+     {platform_plugins_dir,  local_path(["plugins"])}
+    ].
+
 generate_config() ->
     Schema = cuttlefish_schema:files([local_path(["priv", "emqx.schema"])]),
-    Conf = conf_parse:file([local_path(["etc", "gen.emqx.conf"])]),
+    ConfFile = render_config_file(),
+    Conf = conf_parse:file(ConfFile),
     cuttlefish_generator:map(Schema, Conf).
 
 set_app_env({App, Lists}) ->

test/emqx_vm_mon_SUITE.erl

@@ -23,6 +23,15 @@
 
 -include_lib("common_test/include/ct.hrl").
 
+-define(WAIT(PATTERN, TIMEOUT),
+        receive
+            PATTERN ->
+                ok
+        after
+            TIMEOUT ->
+                error(timeout)
+        end).
+
 all() -> [t_api].
 
 init_per_suite(Config) ->
@@ -33,18 +42,36 @@ end_per_suite(_Config) ->
     application:stop(sasl).
 
 t_api(_) ->
-    gen_event:swap_handler(alarm_handler, {emqx_alarm_handler, swap}, {alarm_handler, []}),
+    meck:new(alarm_handler, [passthrough, no_history]),
-    {ok, _} = emqx_vm_mon:start_link([{check_interval, 1},
+    Tester = self(),
-                                      {process_high_watermark, 0},
+    Ref = make_ref(),
-                                      {process_low_watermark, 0.6}]),
+    try
-    timer:sleep(2000),
+        meck:expect(alarm_handler, set_alarm,
-    ?assertEqual(true, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())),
+                    fun(What) ->
-    emqx_vm_mon:set_process_high_watermark(0.8),
+                            Res = meck:passthrough([What]),
-    emqx_vm_mon:set_process_low_watermark(0.75),
+                            Tester ! {Ref, set_alarm, What},
-    ?assertEqual(0.8, emqx_vm_mon:get_process_high_watermark()),
+                            Res
-    ?assertEqual(0.75, emqx_vm_mon:get_process_low_watermark()),
+                    end),
-    timer:sleep(3000),
+        meck:expect(alarm_handler, clear_alarm,
-    ?assertEqual(false, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())),
+                    fun(What) ->
-    emqx_vm_mon:set_check_interval(20),
+                            Res = meck:passthrough([What]),
-    ?assertEqual(20, emqx_vm_mon:get_check_interval()),
+                            Tester ! {Ref, clear_alarm, What},
-    ok.
+                            Res
+                    end),
+        gen_event:swap_handler(alarm_handler, {emqx_alarm_handler, swap}, {alarm_handler, []}),
+        {ok, _} = emqx_vm_mon:start_link([{check_interval, 1},
+                                        {process_high_watermark, 0},
+                                        {process_low_watermark, 0.6}]),
+        ?WAIT({Ref, set_alarm, {too_many_processes, _Count}}, 2000),
+        ?assertEqual(true, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())),
+        emqx_vm_mon:set_process_high_watermark(0.8),
+        emqx_vm_mon:set_process_low_watermark(0.75),
+        ?assertEqual(0.8, emqx_vm_mon:get_process_high_watermark()),
+        ?assertEqual(0.75, emqx_vm_mon:get_process_low_watermark()),
+        ?WAIT({Ref, clear_alarm, too_many_processes}, 3000),
+        ?assertEqual(false, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())),
+        emqx_vm_mon:set_check_interval(20),
+        ?assertEqual(20, emqx_vm_mon:get_check_interval())
+    after
+        meck:unload(alarm_handler)
+    end.

vars

@@ -5,5 +5,4 @@
 {platform_etc_dir,  "etc"}.
 {platform_lib_dir,  "lib"}.
 {platform_log_dir,  "log"}.
-{platform_plugins_dir,  "plugins"}.
+{platform_plugins_dir, "plugins"}.
-