diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index d14974679..47994c9ad 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -39,7 +39,7 @@ emqx_test(){ unzip -q "${PACKAGE_PATH}/${packagename}" export EMQX_ZONE__EXTERNAL__SERVER_KEEPALIVE=60 \ EMQX_MQTT__MAX_TOPIC_ALIAS=10 - sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins + # sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins echo "running ${packagename} start" if ! "${PACKAGE_PATH}"/emqx/bin/emqx start; then @@ -115,7 +115,7 @@ emqx_test(){ running_test(){ export EMQX_ZONE__EXTERNAL__SERVER_KEEPALIVE=60 \ EMQX_MQTT__MAX_TOPIC_ALIAS=10 - sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins + # sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins if ! emqx start; then cat /var/log/emqx/erlang.log.1 || true diff --git a/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml index 6bc8e67e2..81d48aba7 100644 --- a/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml +++ b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml @@ -38,7 +38,7 @@ services: - -c - | sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf - sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins + # sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins /opt/emqx/bin/emqx foreground healthcheck: test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"] @@ -62,7 +62,7 @@ services: - -c - | sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf - sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins + # sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins /opt/emqx/bin/emqx foreground healthcheck: test: ["CMD", "/opt/emqx/bin/emqx", "ping"] diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index b983eaa67..6d1a757af 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -179,7 +179,7 @@ jobs: cd source pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip) unzip -q _packages/${{ matrix.profile }}/$pkg_name - gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins + # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' for i in {1..10}; do diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index bf85578c5..1f79b9ab2 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -108,7 +108,7 @@ jobs: run: | pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip) unzip -q _packages/${EMQX_NAME}/$pkg_name - gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins + # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' for i in {1..10}; do diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 035c0d0e3..e414537b9 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -48,13 +48,13 @@ jobs: echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx"; sleep 5; done - - name: verify EMQX_LOADED_PLUGINS override working - run: | - expected="{emqx_sn, true}." - output=$(docker exec -i node1.emqx.io bash -c "cat data/loaded_plugins" | tail -n1) - if [ "$expected" != "$output" ]; then - exit 1 - fi + # - name: verify EMQX_LOADED_PLUGINS override working + # run: | + # expected="{emqx_sn, true}." + # output=$(docker exec -i node1.emqx.io bash -c "cat data/loaded_plugins" | tail -n1) + # if [ "$expected" != "$output" ]; then + # exit 1 + # fi - name: make paho tests run: | if ! docker exec -i python /scripts/pytest.sh; then diff --git a/.gitignore b/.gitignore index dd8e9b82e..e7b31b394 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,6 @@ emqx_dialyzer_*_plt */emqx_dashboard/priv/www dist.zip scripts/git-token -etc/*.seg +apps/*/etc/*.all _upgrade_base/ TAGS diff --git a/Makefile b/Makefile index cc8cdb0db..c81b9d5a3 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,8 @@ coveralls: $(REBAR) @ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send .PHONY: $(REL_PROFILES) -$(REL_PROFILES:%=%): $(REBAR) get-dashboard + +$(REL_PROFILES:%=%): $(REBAR) get-dashboard conf-segs @$(REBAR) as $(@) do compile,release ## Not calling rebar3 clean because @@ -111,7 +112,7 @@ xref: $(REBAR) dialyzer: $(REBAR) @$(REBAR) as check dialyzer -COMMON_DEPS := $(REBAR) get-dashboard $(CONF_SEGS) +COMMON_DEPS := $(REBAR) get-dashboard conf-segs ## rel target is to create release package without relup .PHONY: $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel) @@ -152,3 +153,6 @@ quickrun: ./_build/$(PROFILE)/rel/emqx/bin/emqx console include docker.mk + +conf-segs: + @scripts/merge-config.escript diff --git a/apps/emqx/etc/acl.conf b/apps/emqx/etc/acl.conf deleted file mode 100644 index af2fb0dd1..000000000 --- a/apps/emqx/etc/acl.conf +++ /dev/null @@ -1,26 +0,0 @@ -%%-------------------------------------------------------------------- -%% [ACL](https://docs.emqx.io/broker/v3/en/config.html) -%% -%% -type(who() :: all | binary() | -%% {ipaddr, esockd_access:cidr()} | -%% {client, binary()} | -%% {user, binary()}). -%% -%% -type(access() :: subscribe | publish | pubsub). -%% -%% -type(topic() :: binary()). -%% -%% -type(rule() :: {allow, all} | -%% {allow, who(), access(), list(topic())} | -%% {deny, all} | -%% {deny, who(), access(), list(topic())}). -%%-------------------------------------------------------------------- - -{allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}. - -{allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}. - -{deny, all, subscribe, ["$SYS/#", {eq, "#"}]}. - -{allow, all}. - diff --git a/apps/emqx/etc/acl.conf.paho b/apps/emqx/etc/acl.conf.paho deleted file mode 100644 index 5beec4347..000000000 --- a/apps/emqx/etc/acl.conf.paho +++ /dev/null @@ -1,14 +0,0 @@ -%%-------------------------------------------------------------------- -%% For paho interoperability test cases -%%-------------------------------------------------------------------- - -{deny, {client, "myclientid"}, subscribe, ["test/nosubscribe"]}. - -{allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}. - -{allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}. - -{deny, all, subscribe, ["$SYS/#", {eq, "#"}]}. - -{allow, all}. - diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index d148e01a3..a11c30cb4 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -108,8 +108,7 @@ descr :: string(), vendor :: string() | undefined, active = false :: boolean(), - info = #{} :: map(), - type :: atom() + info = #{} :: map() }). %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx.appup.src b/apps/emqx/src/emqx.appup.src deleted file mode 100644 index 4f9f00673..000000000 --- a/apps/emqx/src/emqx.appup.src +++ /dev/null @@ -1,111 +0,0 @@ -%% -*- mode: erlang -*- -{VSN, - [ - {"4.3.4", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]} - ]}, - {"4.3.3", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]} - ]}, - {"4.3.2", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]} - ]}, - {"4.3.1", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, - {load_module,emqx_node_dump,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_trie,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_node_dump,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,upgrade_retained_delayed_counter_type,[]}}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}], - [ - {"4.3.4", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]} - ]}, - {"4.3.3", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]} - ]}, - {"4.3.2", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]} - ]}, - {"4.3.1", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, - {load_module,emqx_node_dump,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_trie,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_node_dump,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}]}. diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl index c700239ca..0a290ff4a 100644 --- a/apps/emqx/src/emqx.erl +++ b/apps/emqx/src/emqx.erl @@ -227,7 +227,6 @@ shutdown() -> shutdown(Reason) -> ?LOG(critical, "emqx shutdown for ~s", [Reason]), _ = emqx_alarm_handler:unload(), - _ = emqx_plugins:unload(), lists:foreach(fun application:stop/1 , lists:reverse(default_started_applications()) ). diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index 60f0fc40d..666a704f3 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -51,7 +51,7 @@ start(_Type, _Args) -> ok = ekka_rlog:wait_for_shards(?EMQX_SHARDS, infinity), {ok, Sup} = emqx_sup:start_link(), ok = start_autocluster(), - ok = emqx_plugins:init(), + % ok = emqx_plugins:init(), _ = emqx_plugins:load(), _ = start_ce_modules(), emqx_boot:is_enabled(listeners) andalso (ok = emqx_listeners:start()), diff --git a/apps/emqx/src/emqx_plugins.erl b/apps/emqx/src/emqx_plugins.erl index 8abc2b21f..ae324c71d 100644 --- a/apps/emqx/src/emqx_plugins.erl +++ b/apps/emqx/src/emqx_plugins.erl @@ -21,8 +21,6 @@ -logger_header("[Plugins]"). --export([init/0]). - -export([ load/0 , load/1 , unload/0 @@ -30,8 +28,6 @@ , reload/1 , list/0 , find_plugin/1 - , generate_configs/1 - , apply_configs/1 ]). -export([funlog/2]). @@ -41,35 +37,14 @@ -compile(nowarn_export_all). -endif. --dialyzer({no_match, [ plugin_loaded/2 - , plugin_unloaded/2 - ]}). - %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- -%% @doc Init plugins' config --spec(init() -> ok). -init() -> - case emqx:get_env(plugins_etc_dir) of - undefined -> ok; - PluginsEtc -> - CfgFiles = [filename:join(PluginsEtc, File) || - File <- filelib:wildcard("*.config", PluginsEtc)], - lists:foreach(fun init_config/1, CfgFiles) - end. - %% @doc Load all plugins when the broker started. -spec(load() -> ok | ignore | {error, term()}). load() -> - ok = load_ext_plugins(emqx:get_env(expand_plugins_dir)), - case emqx:get_env(plugins_loaded_file) of - undefined -> ignore; %% No plugins available - File -> - _ = ensure_file(File), - with_loaded_file(File, fun(Names) -> load_plugins(Names, false) end) - end. + ok = load_ext_plugins(emqx:get_env(expand_plugins_dir)). %% @doc Load a Plugin -spec(load(atom()) -> ok | {error, term()}). @@ -82,17 +57,13 @@ load(PluginName) when is_atom(PluginName) -> ?LOG(notice, "Plugin ~s is already started", [PluginName]), {error, already_started}; {_, false} -> - load_plugin(PluginName, true) + load_plugin(PluginName) end. %% @doc Unload all plugins before broker stopped. --spec(unload() -> list() | {error, term()}). +-spec(unload() -> ok). unload() -> - case emqx:get_env(plugins_loaded_file) of - undefined -> ignore; - File -> - with_loaded_file(File, fun stop_plugins/1) - end. + stop_plugins(list()). %% @doc UnLoad a Plugin -spec(unload(atom()) -> ok | {error, term()}). @@ -105,7 +76,7 @@ unload(PluginName) when is_atom(PluginName) -> ?LOG(error, "Plugin ~s is not started", [PluginName]), {error, not_started}; {_, _} -> - unload_plugin(PluginName, true) + unload_plugin(PluginName) end. reload(PluginName) when is_atom(PluginName)-> @@ -126,8 +97,8 @@ reload(PluginName) when is_atom(PluginName)-> -spec(list() -> [emqx_types:plugin()]). list() -> StartedApps = names(started_app), - lists:map(fun({Name, _, [Type| _]}) -> - Plugin = plugin(Name, Type), + lists:map(fun({Name, _, _}) -> + Plugin = plugin(Name), case lists:member(Name, StartedApps) of true -> Plugin#plugin{active = true}; false -> Plugin @@ -144,12 +115,6 @@ find_plugin(Name, Plugins) -> %% Internal functions %%-------------------------------------------------------------------- -init_config(CfgFile) -> - {ok, [AppsEnv]} = file:consult(CfgFile), - lists:foreach(fun({App, Envs}) -> - [application:set_env(App, Par, Val) || {Par, Val} <- Envs] - end, AppsEnv). - %% load external plugins which are placed in etc/plugins dir load_ext_plugins(undefined) -> ok; load_ext_plugins(Dir) -> @@ -173,15 +138,15 @@ load_ext_plugin(PluginDir) -> ?LOG(alert, "plugin_app_file_not_found: ~s", [AppFile]), error({plugin_app_file_not_found, AppFile}) end, - ok = load_plugin_app(AppName, Ebin), - try - ok = generate_configs(AppName, PluginDir) - catch - throw : {conf_file_not_found, ConfFile} -> - %% this is maybe a dependency of an external plugin - ?LOG(debug, "config_load_error_ignored for app=~p, path=~s", [AppName, ConfFile]), - ok - end. + ok = load_plugin_app(AppName, Ebin). + % try + % ok = generate_configs(AppName, PluginDir) + % catch + % throw : {conf_file_not_found, ConfFile} -> + % %% this is maybe a dependency of an external plugin + % ?LOG(debug, "config_load_error_ignored for app=~p, path=~s", [AppName, ConfFile]), + % ok + % end. load_plugin_app(AppName, Ebin) -> _ = code:add_patha(Ebin), @@ -199,57 +164,24 @@ load_plugin_app(AppName, Ebin) -> {error, {already_loaded, _}} -> ok end. -ensure_file(File) -> - case filelib:is_file(File) of false -> write_loaded([]); true -> ok end. - -with_loaded_file(File, SuccFun) -> - case read_loaded(File) of - {ok, Names0} -> - Names = filter_plugins(Names0), - SuccFun(Names); - {error, Error} -> - ?LOG(alert, "Failed to read: ~p, error: ~p", [File, Error]), - {error, Error} - end. - -filter_plugins(Names) -> - lists:filtermap(fun(Name1) when is_atom(Name1) -> {true, Name1}; - ({Name1, true}) -> {true, Name1}; - ({_Name1, false}) -> false - end, Names). - -load_plugins(Names, Persistent) -> - Plugins = list(), - NotFound = Names -- names(Plugins), - case NotFound of - [] -> ok; - NotFound -> ?LOG(alert, "cannot_find_plugins: ~p", [NotFound]) - end, - NeedToLoad = Names -- NotFound -- names(started_app), - lists:foreach(fun(Name) -> - Plugin = find_plugin(Name, Plugins), - load_plugin(Plugin#plugin.name, Persistent) - end, NeedToLoad). - %% Stop plugins -stop_plugins(Names) -> - _ = [stop_app(App) || App <- Names], +stop_plugins(Plugins) -> + _ = [stop_app(Plugin#plugin.name) || Plugin <- Plugins], ok. -plugin(AppName, Type) -> +plugin(AppName) -> case application:get_all_key(AppName) of {ok, Attrs} -> Descr = proplists:get_value(description, Attrs, ""), - #plugin{name = AppName, descr = Descr, type = plugin_type(Type)}; + #plugin{name = AppName, descr = Descr}; undefined -> error({plugin_not_found, AppName}) end. -load_plugin(Name, Persistent) -> +load_plugin(Name) -> try - ok = ?MODULE:generate_configs(Name), case load_app(Name) of ok -> - start_app(Name, fun(App) -> plugin_loaded(App, Persistent) end); + start_app(Name); {error, Error0} -> {error, Error0} end @@ -268,22 +200,21 @@ load_app(App) -> {error, Error} end. -start_app(App, SuccFun) -> +start_app(App) -> case application:ensure_all_started(App) of {ok, Started} -> ?LOG(info, "Started plugins: ~p", [Started]), ?LOG(info, "Load plugin ~s successfully", [App]), - _ = SuccFun(App), ok; {error, {ErrApp, Reason}} -> ?LOG(error, "Load plugin ~s failed, cannot start plugin ~s for ~0p", [App, ErrApp, Reason]), {error, {ErrApp, Reason}} end. -unload_plugin(App, Persistent) -> +unload_plugin(App) -> case stop_app(App) of ok -> - _ = plugin_unloaded(App, Persistent), ok; + ok; {error, Reason} -> {error, Reason} end. @@ -307,133 +238,5 @@ names(started_app) -> names(Plugins) -> [Name || #plugin{name = Name} <- Plugins]. -plugin_loaded(_Name, false) -> - ok; -plugin_loaded(Name, true) -> - case read_loaded() of - {ok, Names} -> - case lists:member(Name, Names) of - false -> - %% write file if plugin is loaded - write_loaded(lists:append(Names, [{Name, true}])); - true -> - ignore - end; - {error, Error} -> - ?LOG(error, "Cannot read loaded plugins: ~p", [Error]) - end. - -plugin_unloaded(_Name, false) -> - ok; -plugin_unloaded(Name, true) -> - case read_loaded() of - {ok, Names0} -> - Names = filter_plugins(Names0), - case lists:member(Name, Names) of - true -> - write_loaded(lists:delete(Name, Names)); - false -> - ?LOG(error, "Cannot find ~s in loaded_file", [Name]) - end; - {error, Error} -> - ?LOG(error, "Cannot read loaded_plugins: ~p", [Error]) - end. - -read_loaded() -> - case emqx:get_env(plugins_loaded_file) of - undefined -> {error, not_found}; - File -> read_loaded(File) - end. - -read_loaded(File) -> file:consult(File). - -write_loaded(AppNames) -> - FilePath = emqx:get_env(plugins_loaded_file), - case file:write_file(FilePath, [io_lib:format("~p.~n", [Name]) || Name <- AppNames]) of - ok -> ok; - {error, Error} -> - ?LOG(error, "Write File ~p Error: ~p", [FilePath, Error]), - {error, Error} - end. - -plugin_type(auth) -> auth; -plugin_type(protocol) -> protocol; -plugin_type(backend) -> backend; -plugin_type(bridge) -> bridge; -plugin_type(_) -> feature. - - funlog(Key, Value) -> ?LOG(info, "~s = ~p", [string:join(Key, "."), Value]). - -generate_configs(App) -> - PluginConfDir = emqx:get_env(plugins_etc_dir), - PluginSchemaDir = code:priv_dir(App), - generate_configs(App, PluginConfDir, PluginSchemaDir). - -generate_configs(App, PluginDir) -> - PluginConfDir = filename:join([PluginDir, "etc"]), - PluginSchemaDir = filename:join([PluginDir, "priv"]), - generate_configs(App, PluginConfDir, PluginSchemaDir). - -generate_configs(App, PluginConfDir, PluginSchemaDir) -> - ConfigFile = filename:join([PluginConfDir, App]) ++ ".config", - case filelib:is_file(ConfigFile) of - true -> - {ok, [Configs]} = file:consult(ConfigFile), - apply_configs(Configs); - false -> - SchemaFile = filename:join([PluginSchemaDir, App]) ++ ".schema", - case filelib:is_file(SchemaFile) of - true -> - AppsEnv = do_generate_configs(App), - apply_configs(AppsEnv); - false -> - SchemaMod = lists:concat([App, "_schema"]), - ConfName = filename:join([PluginConfDir, App]) ++ ".conf", - SchemaFile1 = filename:join([code:lib_dir(App), "ebin", SchemaMod]) ++ ".beam", - do_generate_hocon_configs(App, ConfName, SchemaFile1) - end - end. - -do_generate_configs(App) -> - Name1 = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".conf", - Name2 = filename:join([code:lib_dir(App), "etc", App]) ++ ".conf", - ConfFile = case {filelib:is_file(Name1), filelib:is_file(Name2)} of - {true, _} -> Name1; - {false, true} -> Name2; - {false, false} -> error({config_not_found, [Name1, Name2]}) - end, - SchemaFile = filename:join([code:priv_dir(App), App]) ++ ".schema", - case filelib:is_file(SchemaFile) of - true -> - Schema = cuttlefish_schema:files([SchemaFile]), - Conf = cuttlefish_conf:file(ConfFile), - cuttlefish_generator:map(Schema, Conf, undefined, fun ?MODULE:funlog/2); - false -> - error({schema_not_found, SchemaFile}) - end. - -do_generate_hocon_configs(App, ConfName, SchemaFile) -> - SchemaMod = lists:concat([App, "_schema"]), - case {filelib:is_file(ConfName), filelib:is_file(SchemaFile)} of - {true, true} -> - {ok, RawConfig} = hocon:load(ConfName, #{format => richmap}), - _ = hocon_schema:check(list_to_atom(SchemaMod), RawConfig, #{atom_key => true, - return_plain => true}), - ok; - % emqx_config:update_config([App], Config); - {true, false} -> - error({schema_not_found, [SchemaFile]}); - {false, true} -> - error({config_not_found, [ConfName]}); - {false, false} -> - error({conf_and_schema_not_found, [ConfName, SchemaFile]}) - end. - -apply_configs([]) -> - ok; -apply_configs([{App, Config} | More]) -> - lists:foreach(fun({Key, _}) -> application:unset_env(App, Key) end, application:get_all_env(App)), - lists:foreach(fun({Key, Val}) -> application:set_env(App, Key, Val) end, Config), - apply_configs(More). diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index bb8b41cf9..440dd8117 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -65,6 +65,12 @@ includes() -> [ "emqx_data_bridge" , "emqx_telemetry" , "emqx_retainer" + , "emqx_statsd" + , "emqx_authn" + , "emqx_authz" + , "emqx_bridge_mqtt" + , "emqx_modules" + , "emqx_management" ]. -endif. diff --git a/apps/emqx/test/emqx_plugins_SUITE.erl b/apps/emqx/test/emqx_plugins_SUITE.erl index 6a76cb9d2..a7dd91602 100644 --- a/apps/emqx/test/emqx_plugins_SUITE.erl +++ b/apps/emqx/test/emqx_plugins_SUITE.erl @@ -63,64 +63,28 @@ t_load(_) -> ?assertEqual({error, not_started}, emqx_plugins:unload(emqx_hocon_plugin)), application:set_env(emqx, expand_plugins_dir, undefined), - application:set_env(emqx, plugins_loaded_file, undefined), - ?assertEqual(ignore, emqx_plugins:load()), - ?assertEqual(ignore, emqx_plugins:unload()). - - -t_init_config(_) -> - ConfFile = "emqx_mini_plugin.config", - Data = "[{emqx_mini_plugin,[{mininame ,test}]}].", - file:write_file(ConfFile, list_to_binary(Data)), - ?assertEqual(ok, emqx_plugins:init_config(ConfFile)), - file:delete(ConfFile), - ?assertEqual({ok,test}, application:get_env(emqx_mini_plugin, mininame)). + application:set_env(emqx, plugins_loaded_file, undefined). t_load_ext_plugin(_) -> ?assertError({plugin_app_file_not_found, _}, emqx_plugins:load_ext_plugin("./not_existed_path/")). t_list(_) -> - ?assertMatch([{plugin, _, _, _, _, _, _, _} | _ ], emqx_plugins:list()). + ?assertMatch([{plugin, _, _, _, _, _, _} | _ ], emqx_plugins:list()). t_find_plugin(_) -> - ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_mini_plugin)), - ?assertMatch({plugin, emqx_hocon_plugin, _, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_hocon_plugin)). - -t_plugin_type(_) -> - ?assertEqual(auth, emqx_plugins:plugin_type(auth)), - ?assertEqual(protocol, emqx_plugins:plugin_type(protocol)), - ?assertEqual(backend, emqx_plugins:plugin_type(backend)), - ?assertEqual(bridge, emqx_plugins:plugin_type(bridge)), - ?assertEqual(feature, emqx_plugins:plugin_type(undefined)). - -t_with_loaded_file(_) -> - ?assertMatch({error, _}, emqx_plugins:with_loaded_file("./not_existed_path/", fun(_) -> ok end)). - -t_plugin_loaded(_) -> - ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_mini_plugin, false)), - ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_mini_plugin, true)), - ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_hocon_plugin, false)), - ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_hocon_plugin, true)). - -t_plugin_unloaded(_) -> - ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin, false)), - ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin, true)), - ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_hocon_plugin, false)), - ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_hocon_plugin, true)). + ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_mini_plugin)), + ?assertMatch({plugin, emqx_hocon_plugin, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_hocon_plugin)). t_plugin(_) -> try - emqx_plugins:plugin(not_existed_plugin, undefined) + emqx_plugins:plugin(not_existed_plugin) catch _Error:Reason:_Stacktrace -> ?assertEqual({plugin_not_found,not_existed_plugin}, Reason) end, - ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _, _}, emqx_plugins:plugin(emqx_mini_plugin, undefined)), - ?assertMatch({plugin, emqx_hocon_plugin, _, _, _, _, _, _}, emqx_plugins:plugin(emqx_hocon_plugin, undefined)). - -t_filter_plugins(_) -> - ?assertEqual([name1, name2], emqx_plugins:filter_plugins([name1, {name2,true}, {name3, false}])). + ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _}, emqx_plugins:plugin(emqx_mini_plugin)), + ?assertMatch({plugin, emqx_hocon_plugin, _, _, _, _, _}, emqx_plugins:plugin(emqx_hocon_plugin)). t_load_plugin(_) -> ok = meck:new(application, [unstick, non_strict, passthrough, no_history]), @@ -133,9 +97,9 @@ t_load_plugin(_) -> ok = meck:new(emqx_plugins, [unstick, non_strict, passthrough, no_history]), ok = meck:expect(emqx_plugins, generate_configs, fun(_) -> ok end), ok = meck:expect(emqx_plugins, apply_configs, fun(_) -> ok end), - ?assertMatch({error, _}, emqx_plugins:load_plugin(already_loaded_app, true)), - ?assertMatch(ok, emqx_plugins:load_plugin(normal, true)), - ?assertMatch({error,_}, emqx_plugins:load_plugin(error_app, true)), + ?assertMatch({error, _}, emqx_plugins:load_plugin(already_loaded_app)), + ?assertMatch(ok, emqx_plugins:load_plugin(normal)), + ?assertMatch({error,_}, emqx_plugins:load_plugin(error_app)), ok = meck:unload(emqx_plugins), ok = meck:unload(application). @@ -146,8 +110,8 @@ t_unload_plugin(_) -> (error_app) -> {error, error}; (_) -> ok end), - ?assertEqual(ok, emqx_plugins:unload_plugin(not_started_app, true)), - ?assertEqual(ok, emqx_plugins:unload_plugin(normal, true)), - ?assertEqual({error,error}, emqx_plugins:unload_plugin(error_app, true)), + ?assertEqual(ok, emqx_plugins:unload_plugin(not_started_app)), + ?assertEqual(ok, emqx_plugins:unload_plugin(normal)), + ?assertEqual({error,error}, emqx_plugins:unload_plugin(error_app)), ok = meck:unload(application). diff --git a/apps/emqx_authn/etc/emqx_authn.conf b/apps/emqx_authn/etc/emqx_authn.conf index 092bd5404..ecd49d5a5 100644 --- a/apps/emqx_authn/etc/emqx_authn.conf +++ b/apps/emqx_authn/etc/emqx_authn.conf @@ -1,4 +1,4 @@ -authn: { +emqx_authn: { chains: [ # { # id: "chain1" diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index 9e60e5dda..033c760af 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -38,11 +38,8 @@ stop(_State) -> ok. initialize() -> - ConfFile = filename:join([emqx:get_env(plugins_etc_dir), ?APP]) ++ ".conf", - {ok, RawConfig} = hocon:load(ConfFile), - #{authn := #{chains := Chains, - bindings := Bindings}} - = hocon_schema:check_plain(emqx_authn_schema, RawConfig, #{atom_key => true, nullable => true}), + #{chains := Chains, + bindings := Bindings} = emqx_config:get([authn], #{chains => [], bindings => []}), initialize_chains(Chains), initialize_bindings(Bindings). diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index 0464350dd..d9bf72910 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -27,9 +27,9 @@ , authenticator_name/0 ]). -structs() -> [authn]. +structs() -> ["emqx_authn"]. -fields(authn) -> +fields("emqx_authn") -> [ {chains, fun chains/1} , {bindings, fun bindings/1}]; @@ -80,7 +80,7 @@ fields(pgsql) -> , {config, hoconsc:t(hoconsc:ref(emqx_authn_pgsql, config))} ]. -chains(type) -> hoconsc:array({union, [hoconsc:ref('simple-chain')]}); +chains(type) -> hoconsc:array({union, [hoconsc:ref(?MODULE, 'simple-chain')]}); chains(default) -> []; chains(_) -> undefined. @@ -89,10 +89,10 @@ chain_id(nullable) -> false; chain_id(_) -> undefined. simple_authenticators(type) -> - hoconsc:array({union, [ hoconsc:ref('built-in-database') - , hoconsc:ref(jwt) - , hoconsc:ref(mysql) - , hoconsc:ref(pgsql)]}); + hoconsc:array({union, [ hoconsc:ref(?MODULE, 'built-in-database') + , hoconsc:ref(?MODULE, jwt) + , hoconsc:ref(?MODULE, mysql) + , hoconsc:ref(?MODULE, pgsql)]}); simple_authenticators(default) -> []; simple_authenticators(_) -> undefined. @@ -105,7 +105,7 @@ authenticator_name(type) -> authenticator_name(); authenticator_name(nullable) -> false; authenticator_name(_) -> undefined. -bindings(type) -> hoconsc:array(hoconsc:ref(binding)); +bindings(type) -> hoconsc:array(hoconsc:ref(?MODULE, binding)); bindings(default) -> []; bindings(_) -> undefined. diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 9be9fdce8..6710e4f69 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -36,11 +36,7 @@ register_metrics() -> init() -> ok = register_metrics(), - Conf = filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), - {ok, RawConf} = hocon:load(Conf), - #{emqx_authz := #{rules := Rules}} = hocon_schema:check_plain(emqx_authz_schema, RawConf, #{atom_key => true}), - emqx_config:put([emqx_authz], #{rules => Rules}), - % Rules = emqx_config:get([emqx_authz, rules], []), + Rules = emqx_config:get([emqx_authz, rules], []), NRules = [compile(Rule) || Rule <- Rules], ok = emqx_hooks:add('client.authorize', {?MODULE, authorize, [NRules]}, -1). diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index 944053f4e..66b2e62de 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -41,11 +41,7 @@ set_special_configs(emqx) -> application:set_env(emqx, enable_acl_cache, false), ok; set_special_configs(emqx_authz) -> - application:set_env(emqx, plugins_etc_dir, - emqx_ct_helpers:deps_path(emqx_authz, "test")), - Conf = #{<<"emqx_authz">> => #{<<"rules">> => []}}, - ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)), - % emqx_config:put([emqx_authz], #{rules => []}), + emqx_config:put([emqx_authz], #{rules => []}), ok; set_special_configs(_App) -> ok. diff --git a/apps/emqx_authz/test/emqx_authz_api_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_SUITE.erl index b813ad1f7..24683cd5b 100644 --- a/apps/emqx_authz/test/emqx_authz_api_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_SUITE.erl @@ -55,22 +55,13 @@ set_special_configs(emqx) -> application:set_env(emqx, enable_acl_cache, false), ok; set_special_configs(emqx_authz) -> - application:set_env(emqx, plugins_etc_dir, - emqx_ct_helpers:deps_path(emqx_authz, "test")), - Conf = #{<<"emqx_authz">> => #{<<"rules">> => []}}, - ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)), - % emqx_config:put([emqx_authz], #{rules => []}), + emqx_config:put([emqx_authz], #{rules => []}), ok; set_special_configs(emqx_management) -> - application:set_env(emqx, plugins_etc_dir, - emqx_ct_helpers:deps_path(emqx_management, "test")), - Conf = #{<<"emqx_management">> => #{ - <<"listeners">> => [#{ - <<"protocol">> => <<"http">> - }]} - }, - ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'), jsx:encode(Conf)), + emqx_config:put([emqx_management], #{listeners => [#{protocol => "http", port => 8081}], + default_application_id => <<"admin">>, + default_application_secret => <<"public">>}), ok; set_special_configs(_App) -> diff --git a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl index 039d50383..d84f0722a 100644 --- a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl @@ -47,22 +47,12 @@ set_special_configs(emqx) -> emqx_ct_helpers:deps_path(emqx, "test/loaded_plguins")), ok; set_special_configs(emqx_authz) -> - application:set_env(emqx, plugins_etc_dir, - emqx_ct_helpers:deps_path(emqx_authz, "test")), - Conf = #{<<"emqx_authz">> => - #{<<"rules">> => - [#{<<"config">> =>#{<<"meck">> => <<"fake">>}, - <<"principal">> => all, - <<"sql">> => <<"fake sql">>, - <<"type">> => mysql} - ]}}, - ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)), - % Rules = [#{config =>#{<<"meck">> => <<"fake">>}, - % principal => all, - % sql => <<"fake sql">>, - % type => mysql} - % ], - % emqx_config:put([emqx_authz], #{rules => Rules}), + Rules = [#{config =>#{<<"meck">> => <<"fake">>}, + principal => all, + sql => <<"fake sql">>, + type => mysql} + ], + emqx_config:put([emqx_authz], #{rules => Rules}), ok; set_special_configs(_App) -> ok. diff --git a/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl index 9dcdd7c18..78112b5bc 100644 --- a/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl @@ -47,22 +47,12 @@ set_special_configs(emqx) -> emqx_ct_helpers:deps_path(emqx, "test/loaded_plguins")), ok; set_special_configs(emqx_authz) -> - application:set_env(emqx, plugins_etc_dir, - emqx_ct_helpers:deps_path(emqx_authz, "test")), - Conf = #{<<"emqx_authz">> => - #{<<"rules">> => - [#{<<"config">> =>#{<<"meck">> => <<"fake">>}, - <<"principal">> => all, - <<"sql">> => <<"fake sql">>, - <<"type">> => pgsql} - ]}}, - ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)), - % Rules = [#{config =>#{<<"meck">> => <<"fake">>}, - % principal => all, - % sql => <<"fake sql">>, - % type => pgsql} - % ], - % emqx_config:put([emqx_authz], #{rules => Rules}), + Rules = [#{config =>#{<<"meck">> => <<"fake">>}, + principal => all, + sql => <<"fake sql">>, + type => pgsql} + ], + emqx_config:put([emqx_authz], #{rules => Rules}), ok; set_special_configs(_App) -> ok. diff --git a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl index 8cb7a51b9..edff8a2a9 100644 --- a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl @@ -47,28 +47,12 @@ set_special_configs(emqx) -> emqx_ct_helpers:deps_path(emqx, "test/loaded_plguins")), ok; set_special_configs(emqx_authz) -> - application:set_env(emqx, plugins_etc_dir, - emqx_ct_helpers:deps_path(emqx_authz, "test")), - Conf = #{<<"emqx_authz">> => - #{<<"rules">> => - [#{<<"config">> =>#{ - <<"server">> => <<"127.0.0.1:6379">>, - <<"password">> => <<"public">>, - <<"pool_size">> => 1, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false} - }, - <<"principal">> => all, - <<"cmd">> => <<"fake cmd">>, - <<"type">> => redis} - ]}}, - ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)), - % Rules = [#{config =>#{<<"meck">> => <<"fake">>}, - % principal => all, - % cmd => <<"fake cmd">>, - % type => redis} - % ], - % emqx_config:put([emqx_authz], #{rules => Rules}), + Rules = [#{config =>#{<<"meck">> => <<"fake">>}, + principal => all, + cmd => <<"fake cmd">>, + type => redis} + ], + emqx_config:put([emqx_authz], #{rules => Rules}), ok; set_special_configs(_App) -> ok. diff --git a/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf b/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf index 4593e04f0..c34567ee4 100644 --- a/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf +++ b/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf @@ -3,54 +3,56 @@ ##==================================================================== emqx_bridge_mqtt:{ - bridges:[{ - name: "mqtt1" - start_type: auto - forwards: ["test/#"], - forward_mountpoint: "" - reconnect_interval: "30s" - batch_size: 100 - queue:{ - replayq_dir: false - # replayq_seg_bytes: "100MB" - # replayq_offload_mode: false - # replayq_max_total_bytes: "1GB" - }, - config:{ - conn_type: mqtt - address: "127.0.0.1:1883" - proto_ver: v4 - bridge_mode: true - clientid: "client1" - clean_start: true - username: "username1" - password: "" - keepalive: 300 - subscriptions: [{ - topic: "t/#" - qos: 1 - }] - receive_mountpoint: "" - retry_interval: "30s" - max_inflight: 32 - } - }, - { - name: "rpc1" - start_type: auto - forwards: ["test/#"], - forward_mountpoint: "" - reconnect_interval: "30s" - batch_size: 100 - queue:{ - replayq_dir: "{{ platform_data_dir }}/replayq/bridge_mqtt/" - replayq_seg_bytes: "100MB" - replayq_offload_mode: false - replayq_max_total_bytes: "1GB" - }, - config:{ - conn_type: rpc - node: "emqx@127.0.0.1" - } - }] + bridges:[ + # { + # name: "mqtt1" + # start_type: auto + # forwards: ["test/#"], + # forward_mountpoint: "" + # reconnect_interval: "30s" + # batch_size: 100 + # queue:{ + # replayq_dir: "{{ platform_data_dir }}/replayq/bridge_mqtt/" + # replayq_seg_bytes: "100MB" + # replayq_offload_mode: false + # replayq_max_total_bytes: "1GB" + # }, + # config:{ + # conn_type: mqtt + # address: "127.0.0.1:1883" + # proto_ver: v4 + # bridge_mode: true + # clientid: "client1" + # clean_start: true + # username: "username1" + # password: "" + # keepalive: 300 + # subscriptions: [{ + # topic: "t/#" + # qos: 1 + # }] + # receive_mountpoint: "" + # retry_interval: "30s" + # max_inflight: 32 + # } + # }, + # { + # name: "rpc1" + # start_type: auto + # forwards: ["test/#"], + # forward_mountpoint: "" + # reconnect_interval: "30s" + # batch_size: 100 + # queue:{ + # replayq_dir: "{{ platform_data_dir }}/replayq/bridge_mqtt/" + # replayq_seg_bytes: "100MB" + # replayq_offload_mode: false + # replayq_max_total_bytes: "1GB" + # }, + # config:{ + # conn_type: rpc + # node: "emqx@127.0.0.1" + # } + # } + ] } diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl index fcd2f2c1d..8cc87ef64 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl @@ -26,7 +26,7 @@ structs() -> ["emqx_bridge_mqtt"]. fields("emqx_bridge_mqtt") -> - [ {bridges, hoconsc:array("bridges")} + [ {bridges, hoconsc:array(hoconsc:ref(?MODULE, "bridges"))} ]; fields("bridges") -> @@ -36,8 +36,8 @@ fields("bridges") -> , {forward_mountpoint, emqx_schema:t(string())} , {reconnect_interval, emqx_schema:t(emqx_schema:duration_ms(), undefined, "30s")} , {batch_size, emqx_schema:t(integer(), undefined, 100)} - , {queue, emqx_schema:t(hoconsc:ref("queue"))} - , {config, hoconsc:union([hoconsc:ref("mqtt"), hoconsc:ref("rpc")])} + , {queue, emqx_schema:t(hoconsc:ref(?MODULE, "queue"))} + , {config, hoconsc:union([hoconsc:ref(?MODULE, "mqtt"), hoconsc:ref(?MODULE, "rpc")])} ]; fields("mqtt") -> diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index afec49419..3ea8ab743 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -40,18 +40,7 @@ -define(OVERVIEWS, ['alarms/activated', 'alarms/deactivated', banned, brokers, stats, metrics, listeners, clients, subscriptions, routes, plugins]). all() -> - [{group, overview}, - {group, admins}, - {group, rest}, - {group, cli} - ]. - -groups() -> - [{overview, [sequence], [t_overview]}, - {admins, [sequence], [t_admins_add_delete]}, - {rest, [sequence], [t_rest_api]}, - {cli, [sequence], [t_cli]} - ]. + emqx_ct:all(?MODULE). init_per_suite(Config) -> emqx_ct_helpers:start_apps([emqx_management, emqx_dashboard],fun set_special_configs/1), @@ -62,29 +51,27 @@ end_per_suite(_Config) -> ekka_mnesia:ensure_stopped(). set_special_configs(emqx_management) -> - application:set_env(emqx, plugins_etc_dir, - emqx_ct_helpers:deps_path(emqx_management, "test")), - Conf = #{<<"emqx_management">> => #{ - <<"listeners">> => [#{ - <<"protocol">> => <<"http">> - }]} - }, - ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'), jsx:encode(Conf)), + emqx_config:put([emqx_management], #{listeners => [#{protocol => "http", port => 8081}], + default_application_id => <<"admin">>, + default_application_secret => <<"public">>}), ok; set_special_configs(_) -> ok. t_overview(_) -> + mnesia:clear_table(mqtt_admin), + emqx_dashboard_admin:add_user(<<"admin">>, <<"public">>, <<"tag">>), [?assert(request_dashboard(get, api_path(erlang:atom_to_list(Overview)), auth_header_()))|| Overview <- ?OVERVIEWS]. t_admins_add_delete(_) -> + mnesia:clear_table(mqtt_admin), ok = emqx_dashboard_admin:add_user(<<"username">>, <<"password">>, <<"tag">>), ok = emqx_dashboard_admin:add_user(<<"username1">>, <<"password1">>, <<"tag1">>), Admins = emqx_dashboard_admin:all_users(), - ?assertEqual(3, length(Admins)), + ?assertEqual(2, length(Admins)), ok = emqx_dashboard_admin:remove_user(<<"username1">>), Users = emqx_dashboard_admin:all_users(), - ?assertEqual(2, length(Users)), + ?assertEqual(1, length(Users)), ok = emqx_dashboard_admin:change_password(<<"username">>, <<"password">>, <<"pwd">>), timer:sleep(10), ?assert(request_dashboard(get, api_path("brokers"), auth_header_("username", "pwd"))), @@ -93,6 +80,8 @@ t_admins_add_delete(_) -> ?assertNotEqual(true, request_dashboard(get, api_path("brokers"), auth_header_("username", "pwd"))). t_rest_api(_Config) -> + mnesia:clear_table(mqtt_admin), + emqx_dashboard_admin:add_user(<<"admin">>, <<"public">>, <<"administrator">>), {ok, Res0} = http_get("users"), ?assertEqual([#{<<"username">> => <<"admin">>, diff --git a/apps/emqx_data_bridge/etc/emqx_data_bridge.conf b/apps/emqx_data_bridge/etc/emqx_data_bridge.conf index 696cadf05..c299b97a1 100644 --- a/apps/emqx_data_bridge/etc/emqx_data_bridge.conf +++ b/apps/emqx_data_bridge/etc/emqx_data_bridge.conf @@ -2,125 +2,128 @@ ## EMQ X Bridge Plugin ##-------------------------------------------------------------------- -emqx_data_bridge.bridges: [ -# {name: "mysql_bridge_1" -# type: mysql -# config: { -# server: "192.168.0.172:3306" -# database: mqtt -# pool_size: 1 -# username: root -# password: public -# auto_reconnect: true -# ssl: false -# } -# } -# , {name: "pgsql_bridge_1" -# type: pgsql -# config: { -# server: "192.168.0.172:5432" -# database: mqtt -# pool_size: 1 -# username: root -# password: public -# auto_reconnect: true -# ssl: false -# } -# } -# , {name: "mongodb_bridge_single" -# type: mongo -# config: { -# servers: "192.168.0.172:27017" -# mongo_type: single -# pool_size: 1 -# login: root -# password: public -# auth_source: mqtt -# database: mqtt -# ssl: false -# } -# } -# ,{name: "mongodb_bridge_rs" -# type: mongo -# config: { -# servers: "127.0.0.1:27017" -# mongo_type: rs -# rs_set_name: rs_name -# pool_size: 1 -# login: root -# password: public -# auth_source: mqtt -# database: mqtt -# ssl: false -# } -# } -# ,{name: "mongodb_bridge_shared" -# type: mongo -# config: { -# servers: "127.0.0.1:27017" -# mongo_type: shared -# pool_size: 1 -# login: root -# password: public -# auth_source: mqtt -# database: mqtt -# ssl: false -# max_overflow: 1 -# overflow_ttl: -# overflow_check_period: 10s -# local_threshold_ms: 10s -# connect_timeout_ms: 10s -# socket_timeout_ms: 10s -# server_selection_timeout_ms: 10s -# wait_queue_timeout_ms: 10s -# heartbeat_frequency_ms: 10s -# min_heartbeat_frequency_ms: 10s -# } -# } -# , {name: "redis_bridge_single" -# type: redis -# config: { -# servers: "192.168.0.172:6379" -# redis_type: single -# pool_size: 1 -# database: 0 -# password: public -# auto_reconnect: true -# ssl: false -# } -# } -# ,{name: "redis_bridge_sentinel" -# type: redis -# config: { -# servers: "127.0.0.1:6379, 127.0.0.2:6379, 127.0.0.3:6379" -# redis_type: sentinel -# sentinel_name: mymaster -# pool_size: 1 -# database: 0 -# ssl: false -# } -# } -# ,{name: "redis_bridge_cluster" -# type: redis -# config: { -# servers: "127.0.0.1:6379, 127.0.0.2:6379, 127.0.0.3:6379" -# redis_type: cluster -# pool_size: 1 -# database: 0 -# password: "public" -# ssl: false -# } -# } -# , {name: "ldap_bridge_1" -# type: ldap -# config: { -# servers: "192.168.0.172" -# port: 389 -# bind_dn: "cn=root,dc=emqx,dc=io" -# bind_password: "public" -# timeout: 30s -# pool_size: 1 -# ssl: false -# } -# } -] +emqx_data_bridge:{ + bridges:[ + # {name: "mysql_bridge_1" + # type: mysql + # config: { + # server: "192.168.0.172:3306" + # database: mqtt + # pool_size: 1 + # username: root + # password: public + # auto_reconnect: true + # ssl: false + # } + # } + # , {name: "pgsql_bridge_1" + # type: pgsql + # config: { + # server: "192.168.0.172:5432" + # database: mqtt + # pool_size: 1 + # username: root + # password: public + # auto_reconnect: true + # ssl: false + # } + # } + # , {name: "mongodb_bridge_single" + # type: mongo + # config: { + # servers: "192.168.0.172:27017" + # mongo_type: single + # pool_size: 1 + # login: root + # password: public + # auth_source: mqtt + # database: mqtt + # ssl: false + # } + # } + # ,{name: "mongodb_bridge_rs" + # type: mongo + # config: { + # servers: "127.0.0.1:27017" + # mongo_type: rs + # rs_set_name: rs_name + # pool_size: 1 + # login: root + # password: public + # auth_source: mqtt + # database: mqtt + # ssl: false + # } + # } + # ,{name: "mongodb_bridge_shared" + # type: mongo + # config: { + # servers: "127.0.0.1:27017" + # mongo_type: shared + # pool_size: 1 + # login: root + # password: public + # auth_source: mqtt + # database: mqtt + # ssl: false + # max_overflow: 1 + # overflow_ttl: + # overflow_check_period: 10s + # local_threshold_ms: 10s + # connect_timeout_ms: 10s + # socket_timeout_ms: 10s + # server_selection_timeout_ms: 10s + # wait_queue_timeout_ms: 10s + # heartbeat_frequency_ms: 10s + # min_heartbeat_frequency_ms: 10s + # } + # } + # , {name: "redis_bridge_single" + # type: redis + # config: { + # servers: "192.168.0.172:6379" + # redis_type: single + # pool_size: 1 + # database: 0 + # password: public + # auto_reconnect: true + # ssl: false + # } + # } + # ,{name: "redis_bridge_sentinel" + # type: redis + # config: { + # servers: "127.0.0.1:6379, 127.0.0.2:6379, 127.0.0.3:6379" + # redis_type: sentinel + # sentinel_name: mymaster + # pool_size: 1 + # database: 0 + # ssl: false + # } + # } + # ,{name: "redis_bridge_cluster" + # type: redis + # config: { + # servers: "127.0.0.1:6379, 127.0.0.2:6379, 127.0.0.3:6379" + # redis_type: cluster + # pool_size: 1 + # database: 0 + # password: "public" + # ssl: false + # } + # } + # , {name: "ldap_bridge_1" + # type: ldap + # config: { + # servers: "192.168.0.172" + # port: 389 + # bind_dn: "cn=root,dc=emqx,dc=io" + # bind_password: "public" + # timeout: 30s + # pool_size: 1 + # ssl: false + # } + # } + + ] +} diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl index 1ba7f2fc5..b4749af0e 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl @@ -23,4 +23,4 @@ fields(mysql) -> ?BRIDGE_FIELDS(mysql); fields(pgsql) -> ?BRIDGE_FIELDS(pgsql); fields(mongo) -> ?BRIDGE_FIELDS(mongo); fields(redis) -> ?BRIDGE_FIELDS(redis); -fields(ldap) -> ?BRIDGE_FIELDS(ldap). \ No newline at end of file +fields(ldap) -> ?BRIDGE_FIELDS(ldap). diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index fe68fef44..6cd38d928 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.4.0"}, % strict semver, bump manually! + {vsn, "5.0.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_management/src/emqx_management.appup.src b/apps/emqx_management/src/emqx_management.appup.src deleted file mode 100644 index 06945afad..000000000 --- a/apps/emqx_management/src/emqx_management.appup.src +++ /dev/null @@ -1,13 +0,0 @@ -%% -*- mode: erlang -*- -{VSN, - [ {<<"4.3.[0-2]">>, - [ {restart_application, emqx_management} - ]}, - {<<".*">>, []} - ], - [ {<<"4.3.[0-2]">>, - [ {restart_application, emqx_management} - ]}, - {<<".*">>, []} - ] -}. diff --git a/apps/emqx_management/src/emqx_management_schema.erl b/apps/emqx_management/src/emqx_management_schema.erl index 02f3e2f1a..0b99d1046 100644 --- a/apps/emqx_management/src/emqx_management_schema.erl +++ b/apps/emqx_management/src/emqx_management_schema.erl @@ -28,7 +28,7 @@ fields("emqx_management") -> [ {default_application_id, fun default_application_id/1} , {default_application_secret, fun default_application_secret/1} , {max_row_limit, fun max_row_limit/1} - , {listeners, hoconsc:array(hoconsc:union([hoconsc:ref("http"), hoconsc:ref("https")]))} + , {listeners, hoconsc:array(hoconsc:union([hoconsc:ref(?MODULE, "http"), hoconsc:ref(?MODULE, "https")]))} ]; fields("http") -> diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index ce83b9b71..1aea90459 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -525,7 +525,7 @@ check_row_limit([Tab|Tables], Limit) -> end. max_row_limit() -> - application:get_env(?APP, max_row_limit, ?MAX_ROW_LIMIT). + emqx_config:get([?APP, max_row_limit], ?MAX_ROW_LIMIT). table_size(Tab) -> ets:info(Tab, size). diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index e4908aeff..369ad6782 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -106,10 +106,8 @@ format({Node, Plugins}) -> format(#plugin{name = Name, descr = Descr, - active = Active, - type = Type}) -> + active = Active}) -> #{name => Name, description => iolist_to_binary(Descr), - active => Active, - type => Type}. + active => Active}. diff --git a/apps/emqx_management/src/emqx_mgmt_app.erl b/apps/emqx_management/src/emqx_mgmt_app.erl index 7161435f6..48d57f232 100644 --- a/apps/emqx_management/src/emqx_mgmt_app.erl +++ b/apps/emqx_management/src/emqx_mgmt_app.erl @@ -29,11 +29,6 @@ -include("emqx_mgmt.hrl"). start(_Type, _Args) -> - Conf = filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'), - {ok, RawConf} = hocon:load(Conf), - #{emqx_management := Config} = - hocon_schema:check_plain(emqx_management_schema, RawConf, #{atom_key => true}), - [application:set_env(?APP, Key, maps:get(Key, Config)) || Key <- maps:keys(Config)], {ok, Sup} = emqx_mgmt_sup:start_link(), ok = ekka_rlog:wait_for_shards([?MANAGEMENT_SHARD], infinity), _ = emqx_mgmt_auth:add_default_app(), diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 297de0196..fbba0b2a4 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -68,14 +68,14 @@ mnesia(copy) -> %%-------------------------------------------------------------------- -spec(add_default_app() -> ok | {ok, appsecret()} | {error, term()}). add_default_app() -> - AppId = application:get_env(?APP, default_application_id, undefined), - AppSecret = application:get_env(?APP, default_application_secret, undefined), + AppId = emqx_config:get([?APP, default_application_id], undefined), + AppSecret = emqx_config:get([?APP, default_application_secret], undefined), case {AppId, AppSecret} of {undefined, _} -> ok; {_, undefined} -> ok; {_, _} -> - AppId1 = erlang:list_to_binary(AppId), - AppSecret1 = erlang:list_to_binary(AppSecret), + AppId1 = to_binary(AppId), + AppSecret1 = to_binary(AppSecret), add_app(AppId1, <<"Default">>, AppSecret1, <<"Application user">>, true, undefined) end. @@ -210,3 +210,6 @@ is_authorized(AppId, AppSecret) -> is_expired(undefined) -> true; is_expired(Expired) -> Expired >= erlang:system_time(second). + +to_binary(L) when is_list(L) -> list_to_binary(L); +to_binary(B) when is_binary(B) -> B. diff --git a/apps/emqx_management/src/emqx_mgmt_http.erl b/apps/emqx_management/src/emqx_mgmt_http.erl index c23219e5d..ecf204128 100644 --- a/apps/emqx_management/src/emqx_mgmt_http.erl +++ b/apps/emqx_management/src/emqx_mgmt_http.erl @@ -80,7 +80,7 @@ stop_listener({Proto, Port, _}) -> listeners() -> [{list_to_atom(Protocol), Port, maps:to_list(maps:without([protocol, port], Map))} || Map = #{protocol := Protocol,port := Port} - <- application:get_env(?APP, listeners, [])]. + <- emqx_config:get([emqx_management, listeners], [])]. listener_name(Proto) -> list_to_atom(atom_to_list(Proto) ++ ":management"). diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index f14cc0fef..415bfca2e 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -29,50 +29,19 @@ -define(LOG_HANDLER_ID, [file, default]). all() -> - [{group, manage_apps}, - {group, check_cli}]. - -groups() -> - [{manage_apps, [sequence], - [t_app - ]}, - {check_cli, [sequence], - [t_cli, - t_log_cmd, - t_mgmt_cmd, - t_status_cmd, - t_clients_cmd, - t_vm_cmd, - t_plugins_cmd, - t_trace_cmd, - t_broker_cmd, - t_router_cmd, - t_subscriptions_cmd, - t_listeners_cmd_old, - t_listeners_cmd_new - ]}]. - -apps() -> - [emqx_management, emqx_retainer]. + emqx_ct:all(?MODULE). init_per_suite(Config) -> ekka_mnesia:start(), emqx_mgmt_auth:mnesia(boot), - emqx_ct_helpers:start_apps(apps(), fun set_special_configs/1), + emqx_ct_helpers:start_apps([emqx_retainer, emqx_management], fun set_special_configs/1), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps(apps()). + emqx_ct_helpers:stop_apps([emqx_management, emqx_retainer]). set_special_configs(emqx_management) -> - application:set_env(emqx, plugins_etc_dir, - emqx_ct_helpers:deps_path(emqx_management, "test")), - Conf = #{<<"emqx_management">> => #{ - <<"listeners">> => [#{ - <<"protocol">> => <<"http">> - }]} - }, - ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'), jsx:encode(Conf)), + emqx_config:put([emqx_management], #{listeners => [#{protocol => "http", port => 8081}]}), ok; set_special_configs(_App) -> ok. diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index 0785b9563..66d9328a6 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -51,14 +51,9 @@ end_per_testcase(_, Config) -> Config. set_special_configs(emqx_management) -> - application:set_env(emqx, plugins_etc_dir, - emqx_ct_helpers:deps_path(emqx_management, "test")), - Conf = #{<<"emqx_management">> => #{ - <<"listeners">> => [#{ - <<"protocol">> => <<"http">> - }]} - }, - ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'), jsx:encode(Conf)), + emqx_config:put([emqx_management], #{listeners => [#{protocol => "http", port => 8081}], + default_application_id => <<"admin">>, + default_application_secret => <<"public">>}), ok; set_special_configs(_App) -> ok. diff --git a/apps/emqx_modules/src/emqx_modules.erl b/apps/emqx_modules/src/emqx_modules.erl index 6610a4716..11b4cdcbe 100644 --- a/apps/emqx_modules/src/emqx_modules.erl +++ b/apps/emqx_modules/src/emqx_modules.erl @@ -143,6 +143,8 @@ cli(_) -> {"modules reload ", "Reload module"} ]). +name(Name) when is_binary(Name) -> + name(binary_to_atom(Name, utf8)); name(delayed) -> emqx_mod_delayed; name(presence) -> emqx_mod_presence; name(recon) -> emqx_mod_recon; diff --git a/apps/emqx_modules/test/emqx_modules_SUITE.erl b/apps/emqx_modules/test/emqx_modules_SUITE.erl index db85cbd0e..897c73d50 100644 --- a/apps/emqx_modules/test/emqx_modules_SUITE.erl +++ b/apps/emqx_modules/test/emqx_modules_SUITE.erl @@ -37,15 +37,9 @@ init_per_suite(Config) -> Config. set_special_configs(emqx_management) -> - application:set_env(emqx, modules_loaded_file, emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_modules")), - application:set_env(emqx, plugins_etc_dir, - emqx_ct_helpers:deps_path(emqx_management, "test")), - Conf = #{<<"emqx_management">> => #{ - <<"listeners">> => [#{ - <<"protocol">> => <<"http">> - }]} - }, - ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'), jsx:encode(Conf)), + emqx_config:put([emqx_management], #{listeners => [#{protocol => "http", port => 8081}], + default_application_id => <<"admin">>, + default_application_secret => <<"public">>}), ok; set_special_configs(_) -> ok. diff --git a/apps/emqx_sn/vars b/apps/emqx_sn/vars index a170916f3..e39aa2801 100644 --- a/apps/emqx_sn/vars +++ b/apps/emqx_sn/vars @@ -5,4 +5,4 @@ {platform_etc_dir, "etc"}. {platform_lib_dir, "lib"}. {platform_log_dir, "log"}. -{platform_plugins_dir, "plugins"}. +{platform_plugins_dir, "data/plugins"}. diff --git a/data/loaded_plugins.tmpl b/data/loaded_plugins.tmpl deleted file mode 100644 index dc23386dc..000000000 --- a/data/loaded_plugins.tmpl +++ /dev/null @@ -1,3 +0,0 @@ -{emqx_management, true}. -{emqx_dashboard, true}. -{emqx_retainer, {{enable_plugin_emqx_retainer}}}. diff --git a/rebar.config.erl b/rebar.config.erl index 13bbb1c15..01abf254f 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -127,8 +127,9 @@ prod_compile_opts() -> prod_overrides() -> [{add, [ {erl_opts, [deterministic]}]}]. -relup_deps(Profile) -> - {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "scripts/inject-deps.escript " ++ atom_to_list(Profile)}]}. +relup_deps(_Profile) -> + % {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "scripts/inject-deps.escript " ++ atom_to_list(Profile)}]}. + {post_hooks, []}. profiles() -> Vsn = get_vsn(), @@ -192,9 +193,8 @@ overlay_vars_rel(RelType) -> cloud -> "vm.args"; edge -> "vm.args.edge" end, - [ - {enable_plugin_emqx_retainer, true} - , {vm_args_file, VmArgs} + + [ {vm_args_file, VmArgs} ]. %% vars per packaging type, bin(zip/tar.gz/docker) or pkg(rpm/deb) @@ -204,7 +204,7 @@ overlay_vars_pkg(bin) -> , {platform_etc_dir, "etc"} , {platform_lib_dir, "lib"} , {platform_log_dir, "log"} - , {platform_plugins_dir, "etc/plugins"} + , {platform_plugins_dir, "plugins"} , {runner_root_dir, "$(cd $(dirname $(readlink $0 || echo $0))/..; pwd -P)"} , {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"} , {runner_etc_dir, "$RUNNER_ROOT_DIR/etc"} @@ -219,7 +219,7 @@ overlay_vars_pkg(pkg) -> , {platform_etc_dir, "/etc/emqx"} , {platform_lib_dir, ""} , {platform_log_dir, "/var/log/emqx"} - , {platform_plugins_dir, "/var/lib/emqx/plugins"} + , {platform_plugins_dir, "/var/lib/enqx/plugins"} , {runner_root_dir, "/usr/lib/emqx"} , {runner_bin_dir, "/usr/bin"} , {runner_etc_dir, "/etc/emqx"} @@ -255,8 +255,12 @@ relx_apps(ReleaseType) -> , emqx_connector , emqx_data_bridge , emqx_rule_engine + , emqx_rule_actions , emqx_bridge_mqtt , emqx_modules + , emqx_management + , emqx_retainer + , emqx_statsd ] ++ [emqx_telemetry || not is_enterprise()] ++ [emqx_license || is_enterprise()] @@ -279,25 +283,13 @@ is_app(Name) -> end. relx_plugin_apps(ReleaseType) -> - [ emqx_retainer - , emqx_management - , emqx_dashboard - , emqx_sn - , emqx_coap - , emqx_stomp - , emqx_statsd - , emqx_rule_actions - ] + [] ++ relx_plugin_apps_per_rel(ReleaseType) ++ relx_plugin_apps_enterprise(is_enterprise()) ++ relx_plugin_apps_extra(). relx_plugin_apps_per_rel(cloud) -> - [ emqx_lwm2m - , emqx_exhook - , emqx_exproto - , emqx_prometheus - ]; + []; relx_plugin_apps_per_rel(edge) -> []. @@ -313,11 +305,11 @@ relx_plugin_apps_extra() -> relx_overlay(ReleaseType) -> [ {mkdir, "log/"} , {mkdir, "data/"} + , {mkdir, "plugins"} , {mkdir, "data/mnesia"} , {mkdir, "data/configs"} , {mkdir, "data/patches"} , {mkdir, "data/scripts"} - , {template, "data/loaded_plugins.tmpl", "data/loaded_plugins"} , {template, "data/emqx_vars", "releases/emqx_vars"} , {template, "data/BUILT_ON", "releases/{{release_version}}/BUILT_ON"} , {copy, "bin/emqx", "bin/emqx"} @@ -337,12 +329,8 @@ relx_overlay(ReleaseType) -> end. etc_overlay(ReleaseType) -> - PluginApps = relx_plugin_apps(ReleaseType), - Templates = emqx_etc_overlay(ReleaseType) ++ - lists:append([plugin_etc_overlays(App) || App <- PluginApps]) ++ - [community_plugin_etc_overlays(App) || App <- relx_plugin_apps_extra()], + Templates = emqx_etc_overlay(ReleaseType), [ {mkdir, "etc/"} - , {mkdir, "etc/plugins"} , {copy, "{{base_dir}}/lib/emqx/etc/certs","etc/"} ] ++ lists:map( @@ -352,7 +340,7 @@ etc_overlay(ReleaseType) -> ++ extra_overlay(ReleaseType). extra_overlay(cloud) -> - [ {copy,"{{base_dir}}/lib/emqx_lwm2m/lwm2m_xml","etc/"} + [ ]; extra_overlay(edge) -> []. @@ -366,42 +354,9 @@ emqx_etc_overlay(edge) -> ]. emqx_etc_overlay_common() -> - [{"{{base_dir}}/lib/emqx/etc/acl.conf", "etc/acl.conf"}, - {"{{base_dir}}/lib/emqx/etc/emqx.conf", "etc/emqx.conf"}, - {"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"}, - {"{{base_dir}}/lib/emqx_data_bridge/etc/emqx_data_bridge.conf", "etc/plugins/emqx_data_bridge.conf"}, - {"{{base_dir}}/lib/emqx_telemetry/etc/emqx_telemetry.conf", "etc/plugins/emqx_telemetry.conf"}, - {"{{base_dir}}/lib/emqx_authn/etc/emqx_authn.conf", "etc/plugins/emqx_authn.conf"}, - {"{{base_dir}}/lib/emqx_authz/etc/emqx_authz.conf", "etc/plugins/authz.conf"}, - {"{{base_dir}}/lib/emqx_rule_engine/etc/emqx_rule_engine.conf", "etc/plugins/emqx_rule_engine.conf"}, - {"{{base_dir}}/lib/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf", "etc/plugins/emqx_bridge_mqtt.conf"}, - {"{{base_dir}}/lib/emqx_modules/etc/emqx_modules.conf", "etc/plugins/emqx_modules.conf"}, - %% TODO: check why it has to end with .paho - %% and why it is put to etc/plugins dir - {"{{base_dir}}/lib/emqx/etc/acl.conf.paho", "etc/plugins/acl.conf.paho"}]. - -plugin_etc_overlays(App0) -> - App = atom_to_list(App0), - ConfFiles = find_conf_files(App), - %% NOTE: not filename:join here since relx translates it for windows - [{"{{base_dir}}/lib/"++ App ++"/etc/" ++ F, "etc/plugins/" ++ F} - || F <- ConfFiles]. - -community_plugin_etc_overlays(App0) -> - App = atom_to_list(App0), - {"{{base_dir}}/lib/"++ App ++"/etc/" ++ App ++ ".conf", "etc/plugins/" ++ App ++ ".conf"}. - -%% NOTE: for apps fetched as rebar dependency (there is so far no such an app) -%% the overlay should be hand-coded but not to rely on build-time wildcards. -find_conf_files(App) -> - Dir1 = filename:join(["apps", App, "etc"]), - filelib:wildcard("*.conf", Dir1) ++ - case is_enterprise() of - true -> - Dir2 = filename:join(["lib-ee", App, "etc"]), - filelib:wildcard("*.conf", Dir2); - false -> [] - end. + [ {"{{base_dir}}/lib/emqx/etc/emqx.conf.all", "etc/emqx.conf"} + , {"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"} + ]. get_vsn() -> PkgVsn = os:cmd("./pkg-vsn.sh"), diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index 54467b62b..e437c49c5 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -7,7 +7,7 @@ cd -P -- "$(dirname -- "$0")/.." find_app() { local appdir="$1" - find "${appdir}" -mindepth 1 -maxdepth 1 -type d | grep -vE "emqx_exhook|emqx_exproto|emqx_lwm2m|emqx_sn|emqx_coap|emqx_stomp" + find "${appdir}" -mindepth 1 -maxdepth 1 -type d | grep -vE "emqx_exhook|emqx_exproto|emqx_lwm2m|emqx_sn|emqx_coap|emqx_stomp|emqx_dashboard" } find_app 'apps' diff --git a/scripts/merge-config.escript b/scripts/merge-config.escript new file mode 100755 index 000000000..a0fbdb4b3 --- /dev/null +++ b/scripts/merge-config.escript @@ -0,0 +1,36 @@ +#!/usr/bin/env escript + +%% This script reads up emqx.conf and split the sections +%% and dump sections to separate files. +%% Sections are grouped between CONFIG_SECTION_BGN and +%% CONFIG_SECTION_END pairs +%% +%% NOTE: this feature is so far not used in opensource +%% edition due to backward-compatibility reasons. + +-mode(compile). + +main(_) -> + BaseConf = "apps/emqx/etc/emqx.conf", + {ok, Bin} = file:read_file(BaseConf), + Apps = filelib:wildcard("emqx_*", "apps/"), + Conf = lists:foldl(fun(App, Acc) -> + case lists:member(App, ["emqx_exhook", + "emqx_exproto", + "emqx_lwm2m", + "emqx_sn", + "emqx_coap", + "emqx_stomp", + "emqx_dashboard"]) of + true -> Acc; + false -> + Filename = filename:join([apps, App, "etc", App]) ++ ".conf", + case filelib:is_regular(Filename) of + true -> + {ok, Bin1} = file:read_file(Filename), + <>; + false -> Acc + end + end + end, Bin, Apps), + ok = file:write_file("apps/emqx/etc/emqx.conf.all", Conf).