From dd06d70bce38b0531cf483dee638e63769ca5712 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 1 Mar 2021 22:03:49 +0100 Subject: [PATCH 01/52] refactor(build): Move parse_transform module to root app So we do not have to workaround the compile order issue --- rebar.config | 2 +- rebar.config.erl | 17 +---------------- .../src => src}/emqx_rule_actions_trans.erl | 0 3 files changed, 2 insertions(+), 17 deletions(-) rename {apps/emqx_rule_engine/src => src}/emqx_rule_actions_trans.erl (100%) diff --git a/rebar.config b/rebar.config index c692b185e..034c9225b 100644 --- a/rebar.config +++ b/rebar.config @@ -34,7 +34,7 @@ {post_hooks,[]}. -{erl_first_files, ["src/emqx_logger.erl"]}. +{erl_first_files, ["src/emqx_logger.erl", "src/emqx_rule_actions_trans.erl"]}. {deps, [ {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} diff --git a/rebar.config.erl b/rebar.config.erl index 95d79b188..6006719f1 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -2,8 +2,7 @@ -export([do/2]). -do(Dir, CONFIG) -> - ok = compile_and_load_pase_transforms(Dir), +do(_Dir, CONFIG) -> C1 = deps(CONFIG), Config = dialyzer(C1), dump(Config ++ [{overrides, overrides()}] ++ coveralls() ++ config()). @@ -299,20 +298,6 @@ provide_bcrypt_dep() -> provide_bcrypt_release(ReleaseType) -> provide_bcrypt_dep() andalso ReleaseType =:= cloud. -%% this is a silly but working patch. -%% rebar3 does not handle umberella project's cross-app parse_transform well -compile_and_load_pase_transforms(Dir) -> - PtFiles = - [ "apps/emqx_rule_engine/src/emqx_rule_actions_trans.erl" - ], - CompileOpts = [verbose,report_errors,report_warnings,return_errors,debug_info], - lists:foreach(fun(PtFile) -> {ok, _Mod} = compile:file(path(Dir, PtFile), CompileOpts) end, PtFiles). - -path(Dir, Path) -> str(filename:join([Dir, Path])). - -str(L) when is_list(L) -> L; -str(B) when is_binary(B) -> unicode:characters_to_list(B, utf8). - erl_opts_i() -> [{i, "apps"}] ++ [{i, Dir} || Dir <- filelib:wildcard(filename:join(["apps", "*", "include"]))] ++ diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions_trans.erl b/src/emqx_rule_actions_trans.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_actions_trans.erl rename to src/emqx_rule_actions_trans.erl From 4e9330a37e086c887d3059e5686bf18612b00411 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 25 Feb 2021 09:43:37 +0800 Subject: [PATCH 02/52] chore(emqx_management): merge enterprise to opensource --- .../src/emqx_mgmt_api_pubsub.erl | 23 +++-- lib-ce/emqx_management/src/emqx_mgmt_http.erl | 4 + .../src/emqx_mod_api_topic_metrics.erl} | 85 +++++++++++++++++-- lib-ce/emqx_modules/src/emqx_modules.erl | 40 +++++++++ .../src/emqx_modules_api.erl} | 52 +++++++++--- lib-ce/emqx_modules/src/emqx_modules_app.erl | 1 + 6 files changed, 183 insertions(+), 22 deletions(-) rename lib-ce/{emqx_management/src/emqx_mgmt_api_topic_metrics.erl => emqx_modules/src/emqx_mod_api_topic_metrics.erl} (56%) rename lib-ce/{emqx_management/src/emqx_mgmt_api_modules.erl => emqx_modules/src/emqx_modules_api.erl} (70%) diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_pubsub.erl b/lib-ce/emqx_management/src/emqx_mgmt_api_pubsub.erl index 956f2a45f..f49aecb0e 100644 --- a/lib-ce/emqx_management/src/emqx_mgmt_api_pubsub.erl +++ b/lib-ce/emqx_management/src/emqx_mgmt_api_pubsub.erl @@ -78,7 +78,19 @@ subscribe(_Bindings, Params) -> publish(_Bindings, Params) -> logger:debug("API publish Params:~p", [Params]), {ClientId, Topic, Qos, Retain, Payload} = parse_publish_params(Params), - return(do_publish(ClientId, Topic, Qos, Retain, Payload)). + case do_publish(ClientId, Topic, Qos, Retain, Payload) of + {ok, MsgIds} -> + case get_value(<<"return">>, Params, undefined) of + undefined -> return(ok); + _Val -> + case get_value(<<"topics">>, Params, undefined) of + undefined -> return({ok, #{msgid => lists:last(MsgIds)}}); + _ -> return({ok, #{msgids => MsgIds}}) + end + end; + Result -> + return(Result) + end. unsubscribe(_Bindings, Params) -> logger:debug("API unsubscribe Params:~p", [Params]), @@ -119,7 +131,7 @@ loop_publish([], Result) -> loop_publish([Params | ParamsN], Acc) -> {ClientId, Topic, Qos, Retain, Payload} = parse_publish_params(Params), Code = case do_publish(ClientId, Topic, Qos, Retain, Payload) of - ok -> 0; + {ok, _} -> 0; {_, Code0, _} -> Code0 end, Result = #{topic => resp_topic(get_value(<<"topic">>, Params), get_value(<<"topics">>, Params, <<"">>)), @@ -153,11 +165,12 @@ do_subscribe(ClientId, Topics, QoS) -> do_publish(_ClientId, [], _Qos, _Retain, _Payload) -> {ok, ?ERROR15, bad_topic}; do_publish(ClientId, Topics, Qos, Retain, Payload) -> - lists:foreach(fun(Topic) -> + MsgIds = lists:map(fun(Topic) -> Msg = emqx_message:make(ClientId, Qos, Topic, Payload), - emqx_mgmt:publish(Msg#message{flags = #{retain => Retain}}) + emqx_mgmt:publish(Msg#message{flags = #{retain => Retain}}), + emqx_guid:to_hexstr(Msg#message.id) end, Topics), - ok. + {ok, MsgIds}. do_unsubscribe(ClientId, Topic) -> case validate_by_filter(Topic) of diff --git a/lib-ce/emqx_management/src/emqx_mgmt_http.erl b/lib-ce/emqx_management/src/emqx_mgmt_http.erl index d6fcf9ab3..8fb4f74ad 100644 --- a/lib-ce/emqx_management/src/emqx_mgmt_http.erl +++ b/lib-ce/emqx_management/src/emqx_mgmt_http.erl @@ -119,11 +119,15 @@ authorize_appid(Req) -> _ -> false end. +-ifdef(EMQX_ENTERPRISE). +filter(_) -> true. +-else. filter(#{app := App}) -> case emqx_plugins:find_plugin(App) of false -> false; Plugin -> Plugin#plugin.active end. +-endif. format(Port) when is_integer(Port) -> io_lib:format("0.0.0.0:~w", [Port]); diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_topic_metrics.erl b/lib-ce/emqx_modules/src/emqx_mod_api_topic_metrics.erl similarity index 56% rename from lib-ce/emqx_management/src/emqx_mgmt_api_topic_metrics.erl rename to lib-ce/emqx_modules/src/emqx_mod_api_topic_metrics.erl index aac7845b7..20416da7f 100644 --- a/lib-ce/emqx_management/src/emqx_mgmt_api_topic_metrics.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_api_topic_metrics.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_mgmt_api_topic_metrics). +-module(emqx_mod_api_topic_metrics). -import(minirest, [return/1]). @@ -58,7 +58,7 @@ list(#{topic := Topic0}, _Params) -> Topic = emqx_mgmt_util:urldecode(Topic0), case safe_validate(Topic) of true -> - case emqx_mgmt:get_topic_metrics(Topic) of + case get_topic_metrics(Topic) of {error, Reason} -> return({error, Reason}); Metrics -> return({ok, maps:from_list(Metrics)}) end; @@ -69,7 +69,7 @@ list(#{topic := Topic0}, _Params) -> list(_Bindings, _Params) -> execute_when_enabled(fun() -> - case emqx_mgmt:get_all_topic_metrics() of + case get_all_topic_metrics() of {error, Reason} -> return({error, Reason}); Metrics -> return({ok, Metrics}) end @@ -83,7 +83,7 @@ register(_Bindings, Params) -> Topic -> case safe_validate(Topic) of true -> - emqx_mgmt:register_topic_metrics(Topic), + register_topic_metrics(Topic), return(ok); false -> return({error, invalid_topic_name}) @@ -93,7 +93,7 @@ register(_Bindings, Params) -> unregister(Bindings, _Params) when map_size(Bindings) =:= 0 -> execute_when_enabled(fun() -> - emqx_mgmt:unregister_all_topic_metrics(), + unregister_all_topic_metrics(), return(ok) end); @@ -102,7 +102,7 @@ unregister(#{topic := Topic0}, _Params) -> Topic = emqx_mgmt_util:urldecode(Topic0), case safe_validate(Topic) of true -> - emqx_mgmt:unregister_topic_metrics(Topic), + unregister_topic_metrics(Topic), return(ok); false -> return({error, invalid_topic_name}) @@ -128,3 +128,76 @@ safe_validate(Topic) -> error:_Error -> false end. + +get_all_topic_metrics() -> + lists:foldl(fun(Topic, Acc) -> + case get_topic_metrics(Topic) of + {error, _Reason} -> + Acc; + Metrics -> + [#{topic => Topic, metrics => Metrics} | Acc] + end + end, [], emqx_mod_topic_metrics:all_registered_topics()). + +get_topic_metrics(Topic) -> + lists:foldl(fun(Node, Acc) -> + case get_topic_metrics(Node, Topic) of + {error, _Reason} -> + Acc; + Metrics -> + case Acc of + [] -> Metrics; + _ -> + lists:foldl(fun({K, V}, Acc0) -> + [{K, V + proplists:get_value(K, Metrics, 0)} | Acc0] + end, [], Acc) + end + end + end, [], ekka_mnesia:running_nodes()). + +get_topic_metrics(Node, Topic) when Node =:= node() -> + emqx_mod_topic_metrics:metrics(Topic); +get_topic_metrics(Node, Topic) -> + rpc_call(Node, get_topic_metrics, [Node, Topic]). + +register_topic_metrics(Topic) -> + Results = [register_topic_metrics(Node, Topic) || Node <- ekka_mnesia:running_nodes()], + case lists:any(fun(Item) -> Item =:= ok end, Results) of + true -> ok; + false -> lists:last(Results) + end. + +register_topic_metrics(Node, Topic) when Node =:= node() -> + emqx_mod_topic_metrics:register(Topic); +register_topic_metrics(Node, Topic) -> + rpc_call(Node, register_topic_metrics, [Node, Topic]). + +unregister_topic_metrics(Topic) -> + Results = [unregister_topic_metrics(Node, Topic) || Node <- ekka_mnesia:running_nodes()], + case lists:any(fun(Item) -> Item =:= ok end, Results) of + true -> ok; + false -> lists:last(Results) + end. + +unregister_topic_metrics(Node, Topic) when Node =:= node() -> + emqx_mod_topic_metrics:unregister(Topic); +unregister_topic_metrics(Node, Topic) -> + rpc_call(Node, unregister_topic_metrics, [Node, Topic]). + +unregister_all_topic_metrics() -> + Results = [unregister_all_topic_metrics(Node) || Node <- ekka_mnesia:running_nodes()], + case lists:any(fun(Item) -> Item =:= ok end, Results) of + true -> ok; + false -> lists:last(Results) + end. + +unregister_all_topic_metrics(Node) when Node =:= node() -> + emqx_mod_topic_metrics:unregister_all(); +unregister_all_topic_metrics(Node) -> + rpc_call(Node, unregister_topic_metrics, [Node]). + +rpc_call(Node, Fun, Args) -> + case rpc:call(Node, ?MODULE, Fun, Args) of + {badrpc, Reason} -> {error, Reason}; + Res -> Res + end. diff --git a/lib-ce/emqx_modules/src/emqx_modules.erl b/lib-ce/emqx_modules/src/emqx_modules.erl index ffcc456f0..38cf2ac1b 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.erl +++ b/lib-ce/emqx_modules/src/emqx_modules.erl @@ -30,6 +30,8 @@ , load_module/2 ]). +-export([cli/1]). + %% @doc List all available plugins -spec(list() -> [{atom(), boolean()}]). list() -> @@ -170,3 +172,41 @@ write_loaded(true) -> ok end; write_loaded(false) -> ok. + +%%-------------------------------------------------------------------- +%% @doc Modules Command +cli(["list"]) -> + foreach(fun(Module) -> print({module, Module}) end, emqx_modules:list()); + +cli(["load", Name]) -> + case emqx_modules:load(list_to_atom(Name)) of + ok -> + emqx_ctl:print("Module ~s loaded successfully.~n", [Name]); + {error, Reason} -> + emqx_ctl:print("Load module ~s error: ~p.~n", [Name, Reason]) + end; + +cli(["unload", Name]) -> + case emqx_modules:unload(list_to_atom(Name)) of + ok -> + emqx_ctl:print("Module ~s unloaded successfully.~n", [Name]); + {error, Reason} -> + emqx_ctl:print("Unload module ~s error: ~p.~n", [Name, Reason]) + end; + +cli(["reload", "emqx_mod_acl_internal" = Name]) -> + case emqx_modules:reload(list_to_atom(Name)) of + ok -> + emqx_ctl:print("Module ~s reloaded successfully.~n", [Name]); + {error, Reason} -> + emqx_ctl:print("Reload module ~s error: ~p.~n", [Name, Reason]) + end; +cli(["reload", Name]) -> + emqx_ctl:print("Module: ~p does not need to be reloaded.~n", [Name]); + +cli(_) -> + emqx_ctl:usage([{"modules list", "Show loaded modules"}, + {"modules load ", "Load module"}, + {"modules unload ", "Unload module"}, + {"modules reload ", "Reload module"} + ]). diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_modules.erl b/lib-ce/emqx_modules/src/emqx_modules_api.erl similarity index 70% rename from lib-ce/emqx_management/src/emqx_mgmt_api_modules.erl rename to lib-ce/emqx_modules/src/emqx_modules_api.erl index 0e2821ec6..fae6d0e5e 100644 --- a/lib-ce/emqx_management/src/emqx_mgmt_api_modules.erl +++ b/lib-ce/emqx_modules/src/emqx_modules_api.erl @@ -14,9 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_mgmt_api_modules). - --include("emqx_mgmt.hrl"). +-module(emqx_modules_api). -import(minirest, [return/1]). @@ -75,16 +73,16 @@ ]). list(#{node := Node}, _Params) -> - return({ok, [format(Module) || Module <- emqx_mgmt:list_modules(Node)]}); + return({ok, [format(Module) || Module <- list_modules(Node)]}); list(_Bindings, _Params) -> - return({ok, [format(Node, Modules) || {Node, Modules} <- emqx_mgmt:list_modules()]}). + return({ok, [format(Node, Modules) || {Node, Modules} <- list_modules()]}). load(#{node := Node, module := Module}, _Params) -> - return(emqx_mgmt:load_module(Node, Module)); + return(load_module(Node, Module)); load(#{module := Module}, _Params) -> - Results = [emqx_mgmt:load_module(Node, Module) || {Node, _Info} <- emqx_mgmt:list_nodes()], + Results = [load_module(Node, Module) || {Node, _Info} <- list_nodes()], case lists:filter(fun(Item) -> Item =/= ok end, Results) of [] -> return(ok); @@ -93,10 +91,10 @@ load(#{module := Module}, _Params) -> end. unload(#{node := Node, module := Module}, _Params) -> - return(emqx_mgmt:unload_module(Node, Module)); + return(unload_module(Node, Module)); unload(#{module := Module}, _Params) -> - Results = [emqx_mgmt:unload_module(Node, Module) || {Node, _Info} <- emqx_mgmt:list_nodes()], + Results = [unload_module(Node, Module) || {Node, _Info} <- list_nodes()], case lists:filter(fun(Item) -> Item =/= ok end, Results) of [] -> return(ok); @@ -105,13 +103,13 @@ unload(#{module := Module}, _Params) -> end. reload(#{node := Node, module := Module}, _Params) -> - case emqx_mgmt:reload_module(Node, Module) of + case reload_module(Node, Module) of ignore -> return(ok); Result -> return(Result) end; reload(#{module := Module}, _Params) -> - Results = [emqx_mgmt:reload_module(Node, Module) || {Node, _Info} <- emqx_mgmt:list_nodes()], + Results = [reload_module(Node, Module) || {Node, _Info} <- list_nodes()], case lists:filter(fun(Item) -> Item =/= ok end, Results) of [] -> return(ok); @@ -119,6 +117,10 @@ reload(#{module := Module}, _Params) -> return(lists:last(Errors)) end. +%%------------------------------------------------------------------------------ +%% Internal Functions +%%------------------------------------------------------------------------------ + format(Node, Modules) -> #{node => Node, modules => [format(Module) || Module <- Modules]}. @@ -127,3 +129,31 @@ format({Name, Active}) -> description => iolist_to_binary(Name:description()), active => Active}. +list_modules() -> + [{Node, list_modules(Node)} || Node <- ekka_mnesia:running_nodes()]. + +list_modules(Node) when Node =:= node() -> + emqx_modules:list(); +list_modules(Node) -> + rpc_call(Node, list_modules, [Node]). + +load_module(Node, Module) when Node =:= node() -> + emqx_modules:load(Module); +load_module(Node, Module) -> + rpc_call(Node, load_module, [Node, Module]). + +unload_module(Node, Module) when Node =:= node() -> + emqx_modules:unload(Module); +unload_module(Node, Module) -> + rpc_call(Node, unload_module, [Node, Module]). + +reload_module(Node, Module) when Node =:= node() -> + emqx_modules:reload(Module); +reload_module(Node, Module) -> + rpc_call(Node, reload_module, [Node, Module]). + +rpc_call(Node, Fun, Args) -> + case rpc:call(Node, ?MODULE, Fun, Args) of + {badrpc, Reason} -> {error, Reason}; + Res -> Res + end. diff --git a/lib-ce/emqx_modules/src/emqx_modules_app.erl b/lib-ce/emqx_modules/src/emqx_modules_app.erl index 832a39c8e..34ff51267 100644 --- a/lib-ce/emqx_modules/src/emqx_modules_app.erl +++ b/lib-ce/emqx_modules/src/emqx_modules_app.erl @@ -30,6 +30,7 @@ start(_Type, _Args) -> application:load(emqx), {ok, Pid} = emqx_mod_sup:start_link(), ok = emqx_modules:load(), + emqx_ctl:register_command(modules, {emqx_modules, cli}, []), {ok, Pid}. stop(_State) -> From 033f8619ebd1573ec3ea5fa1f67692185abda5bd Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 25 Feb 2021 14:07:42 +0800 Subject: [PATCH 03/52] chore(emqx_management): abstract data backup --- lib-ce/emqx_management/src/emqx_mgmt.erl | 408 ------------- .../src/emqx_mgmt_api_data.erl | 75 +-- lib-ce/emqx_management/src/emqx_mgmt_cli.erl | 97 +-- .../src/emqx_mgmt_data_backup.erl | 557 ++++++++++++++++++ 4 files changed, 571 insertions(+), 566 deletions(-) create mode 100644 lib-ce/emqx_management/src/emqx_mgmt_data_backup.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt.erl b/lib-ce/emqx_management/src/emqx_mgmt.erl index 19474e5a1..e7d896c6a 100644 --- a/lib-ce/emqx_management/src/emqx_mgmt.erl +++ b/lib-ce/emqx_management/src/emqx_mgmt.erl @@ -36,15 +36,6 @@ %% Metrics and Stats -export([ get_metrics/0 , get_metrics/1 - , get_all_topic_metrics/0 - , get_topic_metrics/1 - , get_topic_metrics/2 - , register_topic_metrics/1 - , register_topic_metrics/2 - , unregister_topic_metrics/1 - , unregister_topic_metrics/2 - , unregister_all_topic_metrics/0 - , unregister_all_topic_metrics/1 , get_stats/0 , get_stats/1 ]). @@ -91,14 +82,6 @@ , reload_plugin/2 ]). -%% Modules --export([ list_modules/0 - , list_modules/1 - , load_module/2 - , unload_module/2 - , reload_module/2 - ]). - %% Listeners -export([ list_listeners/0 , list_listeners/1 @@ -118,26 +101,6 @@ , delete_banned/1 ]). -%% Export/Import --export([ export_rules/0 - , export_resources/0 - , export_blacklist/0 - , export_applications/0 - , export_users/0 - , export_auth_mnesia/0 - , export_acl_mnesia/0 - , import_rules/1 - , import_resources/1 - , import_resources_and_rules/3 - , import_blacklist/1 - , import_applications/1 - , import_users/1 - , import_auth_clientid/1 %% BACKW: 4.1.x - , import_auth_username/1 %% BACKW: 4.1.x - , import_auth_mnesia/2 - , import_acl_mnesia/2 - , to_version/1 - ]). -export([ enable_telemetry/0 , disable_telemetry/0 @@ -217,73 +180,6 @@ get_metrics(Node) when Node =:= node() -> get_metrics(Node) -> rpc_call(Node, get_metrics, [Node]). -get_all_topic_metrics() -> - lists:foldl(fun(Topic, Acc) -> - case get_topic_metrics(Topic) of - {error, _Reason} -> - Acc; - Metrics -> - [#{topic => Topic, metrics => Metrics} | Acc] - end - end, [], emqx_mod_topic_metrics:all_registered_topics()). - -get_topic_metrics(Topic) -> - lists:foldl(fun(Node, Acc) -> - case get_topic_metrics(Node, Topic) of - {error, _Reason} -> - Acc; - Metrics -> - case Acc of - [] -> Metrics; - _ -> - lists:foldl(fun({K, V}, Acc0) -> - [{K, V + proplists:get_value(K, Metrics, 0)} | Acc0] - end, [], Acc) - end - end - end, [], ekka_mnesia:running_nodes()). - -get_topic_metrics(Node, Topic) when Node =:= node() -> - emqx_mod_topic_metrics:metrics(Topic); -get_topic_metrics(Node, Topic) -> - rpc_call(Node, get_topic_metrics, [Node, Topic]). - -register_topic_metrics(Topic) -> - Results = [register_topic_metrics(Node, Topic) || Node <- ekka_mnesia:running_nodes()], - case lists:any(fun(Item) -> Item =:= ok end, Results) of - true -> ok; - false -> lists:last(Results) - end. - -register_topic_metrics(Node, Topic) when Node =:= node() -> - emqx_mod_topic_metrics:register(Topic); -register_topic_metrics(Node, Topic) -> - rpc_call(Node, register_topic_metrics, [Node, Topic]). - -unregister_topic_metrics(Topic) -> - Results = [unregister_topic_metrics(Node, Topic) || Node <- ekka_mnesia:running_nodes()], - case lists:any(fun(Item) -> Item =:= ok end, Results) of - true -> ok; - false -> lists:last(Results) - end. - -unregister_topic_metrics(Node, Topic) when Node =:= node() -> - emqx_mod_topic_metrics:unregister(Topic); -unregister_topic_metrics(Node, Topic) -> - rpc_call(Node, unregister_topic_metrics, [Node, Topic]). - -unregister_all_topic_metrics() -> - Results = [unregister_all_topic_metrics(Node) || Node <- ekka_mnesia:running_nodes()], - case lists:any(fun(Item) -> Item =:= ok end, Results) of - true -> ok; - false -> lists:last(Results) - end. - -unregister_all_topic_metrics(Node) when Node =:= node() -> - emqx_mod_topic_metrics:unregister_all(); -unregister_all_topic_metrics(Node) -> - rpc_call(Node, unregister_topic_metrics, [Node]). - get_stats() -> [{Node, get_stats(Node)} || Node <- ekka_mnesia:running_nodes()]. @@ -504,33 +400,6 @@ reload_plugin(Node, Plugin) when Node =:= node() -> reload_plugin(Node, Plugin) -> rpc_call(Node, reload_plugin, [Node, Plugin]). - -%%-------------------------------------------------------------------- -%% Modules -%%-------------------------------------------------------------------- - -list_modules() -> - [{Node, list_modules(Node)} || Node <- ekka_mnesia:running_nodes()]. - -list_modules(Node) when Node =:= node() -> - emqx_modules:list(); -list_modules(Node) -> - rpc_call(Node, list_modules, [Node]). - -load_module(Node, Module) when Node =:= node() -> - emqx_modules:load(Module); -load_module(Node, Module) -> - rpc_call(Node, load_module, [Node, Module]). - -unload_module(Node, Module) when Node =:= node() -> - emqx_modules:unload(Module); -unload_module(Node, Module) -> - rpc_call(Node, unload_module, [Node, Module]). - -reload_module(Node, Module) when Node =:= node() -> - emqx_modules:reload(Module); -reload_module(Node, Module) -> - rpc_call(Node, reload_module, [Node, Module]). %%-------------------------------------------------------------------- %% Listeners %%-------------------------------------------------------------------- @@ -613,282 +482,6 @@ create_banned(Banned) -> delete_banned(Who) -> emqx_banned:delete(Who). -%%-------------------------------------------------------------------- -%% Data Export and Import -%%-------------------------------------------------------------------- - -export_rules() -> - lists:map(fun({_, RuleId, _, RawSQL, _, _, _, _, _, _, Actions, Enabled, Desc}) -> - [{id, RuleId}, - {rawsql, RawSQL}, - {actions, actions_to_prop_list(Actions)}, - {enabled, Enabled}, - {description, Desc}] - end, emqx_rule_registry:get_rules()). - -export_resources() -> - lists:map(fun({_, Id, Type, Config, CreatedAt, Desc}) -> - NCreatedAt = case CreatedAt of - undefined -> null; - _ -> CreatedAt - end, - [{id, Id}, - {type, Type}, - {config, maps:to_list(Config)}, - {created_at, NCreatedAt}, - {description, Desc}] - end, emqx_rule_registry:get_resources()). - -export_blacklist() -> - lists:map(fun(#banned{who = Who, by = By, reason = Reason, at = At, until = Until}) -> - NWho = case Who of - {peerhost, Peerhost} -> {peerhost, inet:ntoa(Peerhost)}; - _ -> Who - end, - [{who, [NWho]}, {by, By}, {reason, Reason}, {at, At}, {until, Until}] - end, ets:tab2list(emqx_banned)). - -export_applications() -> - lists:map(fun({_, AppID, AppSecret, Name, Desc, Status, Expired}) -> - [{id, AppID}, {secret, AppSecret}, {name, Name}, {desc, Desc}, {status, Status}, {expired, Expired}] - end, ets:tab2list(mqtt_app)). - -export_users() -> - lists:map(fun({_, Username, Password, Tags}) -> - [{username, Username}, {password, base64:encode(Password)}, {tags, Tags}] - end, ets:tab2list(mqtt_admin)). - -export_auth_mnesia() -> - case ets:info(emqx_user) of - undefined -> []; - _ -> - lists:map(fun({_, {Type, Login}, Password, CreatedAt}) -> - [{login, Login}, {type, Type}, {password, base64:encode(Password)}, {created_at, CreatedAt}] - end, ets:tab2list(emqx_user)) - end. - -export_acl_mnesia() -> - case ets:info(emqx_acl) of - undefined -> []; - _ -> - lists:map(fun({_, Filter, Action, Access, CreatedAt}) -> - Filter1 = case Filter of - {{Type, TypeValue}, Topic} -> - [{type, Type}, {type_value, TypeValue}, {topic, Topic}]; - {Type, Topic} -> - [{type, Type}, {topic, Topic}] - end, - Filter1 ++ [{action, Action}, {access, Access}, {created_at, CreatedAt}] - end, ets:tab2list(emqx_acl)) - end. - -import_rules(Rules) -> - lists:foreach(fun(Rule) -> - import_rule(Rule) - end, Rules). - -import_resources(Reources) -> - lists:foreach(fun(Resource) -> - import_resource(Resource) - end, Reources). - -import_rule(#{<<"id">> := RuleId, - <<"rawsql">> := RawSQL, - <<"actions">> := Actions, - <<"enabled">> := Enabled, - <<"description">> := Desc}) -> - Rule = #{id => RuleId, - rawsql => RawSQL, - actions => map_to_actions(Actions), - enabled => Enabled, - description => Desc}, - try emqx_rule_engine:create_rule(Rule) - catch throw:{resource_not_initialized, _ResId} -> - emqx_rule_engine:create_rule(Rule#{enabled => false}) - end. - -import_resource(#{<<"id">> := Id, - <<"type">> := Type, - <<"config">> := Config, - <<"created_at">> := CreatedAt, - <<"description">> := Desc}) -> - NCreatedAt = case CreatedAt of - null -> undefined; - _ -> CreatedAt - end, - emqx_rule_engine:create_resource(#{id => Id, - type => any_to_atom(Type), - config => Config, - created_at => NCreatedAt, - description => Desc}). - -import_resources_and_rules(Resources, Rules, FromVersion) - when FromVersion =:= "4.0" orelse FromVersion =:= "4.1" orelse FromVersion =:= "4.2" -> - Configs = lists:foldl(fun(#{<<"id">> := ID, - <<"type">> := <<"web_hook">>, - <<"config">> := #{<<"content_type">> := ContentType, - <<"headers">> := Headers, - <<"method">> := Method, - <<"url">> := URL}} = Resource, Acc) -> - NConfig = #{<<"connect_timeout">> => 5, - <<"request_timeout">> => 5, - <<"cacertfile">> => <<>>, - <<"certfile">> => <<>>, - <<"keyfile">> => <<>>, - <<"pool_size">> => 8, - <<"url">> => URL, - <<"verify">> => true}, - NResource = Resource#{<<"config">> := NConfig}, - import_resource(NResource), - NHeaders = maps:put(<<"content-type">>, ContentType, Headers), - [{ID, #{headers => NHeaders, method => Method}} | Acc]; - (Resource, Acc) -> - import_resource(Resource), - Acc - end, [], Resources), - lists:foreach(fun(#{<<"actions">> := Actions} = Rule) -> - NActions = apply_new_config(Actions, Configs), - import_rule(Rule#{<<"actions">> := NActions}) - end, Rules); -import_resources_and_rules(Resources, Rules, _FromVersion) -> - import_resources(Resources), - import_rules(Rules). - -apply_new_config(Actions, Configs) -> - apply_new_config(Actions, Configs, []). - -apply_new_config([], _Configs, Acc) -> - Acc; -apply_new_config([Action = #{<<"name">> := <<"data_to_webserver">>, - <<"args">> := #{<<"$resource">> := ID, - <<"path">> := Path, - <<"payload_tmpl">> := PayloadTmpl}} | More], Configs, Acc) -> - case proplists:get_value(ID, Configs, undefined) of - undefined -> - apply_new_config(More, Configs, [Action | Acc]); - #{headers := Headers, method := Method} -> - Args = #{<<"$resource">> => ID, - <<"body">> => PayloadTmpl, - <<"headers">> => Headers, - <<"method">> => Method, - <<"path">> => Path}, - apply_new_config(More, Configs, [Action#{<<"args">> := Args} | Acc]) - end. - -import_blacklist(Blacklist) -> - lists:foreach(fun(#{<<"who">> := Who, - <<"by">> := By, - <<"reason">> := Reason, - <<"at">> := At, - <<"until">> := Until}) -> - NWho = case Who of - #{<<"peerhost">> := Peerhost} -> - {ok, NPeerhost} = inet:parse_address(Peerhost), - {peerhost, NPeerhost}; - #{<<"clientid">> := ClientId} -> {clientid, ClientId}; - #{<<"username">> := Username} -> {username, Username} - end, - emqx_banned:create(#banned{who = NWho, by = By, reason = Reason, at = At, until = Until}) - end, Blacklist). - -import_applications(Apps) -> - lists:foreach(fun(#{<<"id">> := AppID, - <<"secret">> := AppSecret, - <<"name">> := Name, - <<"desc">> := Desc, - <<"status">> := Status, - <<"expired">> := Expired}) -> - NExpired = case is_integer(Expired) of - true -> Expired; - false -> undefined - end, - emqx_mgmt_auth:force_add_app(AppID, Name, AppSecret, Desc, Status, NExpired) - end, Apps). - -import_users(Users) -> - lists:foreach(fun(#{<<"username">> := Username, - <<"password">> := Password, - <<"tags">> := Tags}) -> - NPassword = base64:decode(Password), - emqx_dashboard_admin:force_add_user(Username, NPassword, Tags) - end, Users). - -import_auth_clientid(Lists) -> - case ets:info(emqx_user) of - undefined -> ok; - _ -> - [ mnesia:dirty_write({emqx_user, {clientid, Clientid}, base64:decode(Password), erlang:system_time(millisecond)}) - || #{<<"clientid">> := Clientid, <<"password">> := Password} <- Lists ] - end. - -import_auth_username(Lists) -> - case ets:info(emqx_user) of - undefined -> ok; - _ -> - [ mnesia:dirty_write({emqx_user, {username, Username}, base64:decode(Password), erlang:system_time(millisecond)}) - || #{<<"username">> := Username, <<"password">> := Password} <- Lists ] - end. - -import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse - FromVersion =:= "4.1" -> - case ets:info(emqx_user) of - undefined -> ok; - _ -> - CreatedAt = erlang:system_time(millisecond), - [ begin - mnesia:dirty_write({emqx_user, {username, Login}, base64:decode(Password), CreatedAt}) - end - || #{<<"login">> := Login, - <<"password">> := Password} <- Auths ] - - end; - -import_auth_mnesia(Auths, _) -> - case ets:info(emqx_user) of - undefined -> ok; - _ -> - [ mnesia:dirty_write({emqx_user, {any_to_atom(Type), Login}, base64:decode(Password), CreatedAt}) - || #{<<"login">> := Login, - <<"type">> := Type, - <<"password">> := Password, - <<"created_at">> := CreatedAt } <- Auths ] - end. - -import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse - FromVersion =:= "4.1" -> - case ets:info(emqx_acl) of - undefined -> ok; - _ -> - CreatedAt = erlang:system_time(millisecond), - [begin - Allow1 = case any_to_atom(Allow) of - true -> allow; - false -> deny - end, - mnesia:dirty_write({emqx_acl, {{username, Login}, Topic}, any_to_atom(Action), Allow1, CreatedAt}) - end || #{<<"login">> := Login, - <<"topic">> := Topic, - <<"allow">> := Allow, - <<"action">> := Action} <- Acls] - end; - -import_acl_mnesia(Acls, _) -> - case ets:info(emqx_acl) of - undefined -> ok; - _ -> - [ begin - Filter = case maps:get(<<"type_value">>, Map, undefined) of - undefined -> - {any_to_atom(maps:get(<<"type">>, Map)), maps:get(<<"topic">>, Map)}; - Value -> - {{any_to_atom(maps:get(<<"type">>, Map)), Value}, maps:get(<<"topic">>, Map)} - end, - mnesia:dirty_write({emqx_acl ,Filter, any_to_atom(Action), any_to_atom(Access), CreatedAt}) - end - || Map = #{<<"action">> := Action, - <<"access">> := Access, - <<"created_at">> := CreatedAt} <- Acls ] - end. any_to_atom(L) when is_list(L) -> list_to_atom(L); any_to_atom(B) when is_binary(B) -> binary_to_atom(B, utf8); @@ -985,4 +578,3 @@ action_to_prop_list({action_instance, ActionInstId, Name, FallbackActions, Args} {name, Name}, {fallbacks, actions_to_prop_list(FallbackActions)}, {args, Args}]. - diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_data.erl b/lib-ce/emqx_management/src/emqx_mgmt_api_data.erl index 723824d99..d3d22e4d1 100644 --- a/lib-ce/emqx_management/src/emqx_mgmt_api_data.erl +++ b/lib-ce/emqx_management/src/emqx_mgmt_api_data.erl @@ -75,48 +75,8 @@ ]). export(_Bindings, _Params) -> - Rules = emqx_mgmt:export_rules(), - Resources = emqx_mgmt:export_resources(), - Blacklist = emqx_mgmt:export_blacklist(), - Apps = emqx_mgmt:export_applications(), - Users = emqx_mgmt:export_users(), - AuthMnesia = emqx_mgmt:export_auth_mnesia(), - AclMnesia = emqx_mgmt:export_acl_mnesia(), - Seconds = erlang:system_time(second), - {{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds), - Filename = io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]), - NFilename = filename:join([emqx:get_env(data_dir), Filename]), - Version = string:sub_string(emqx_sys:version(), 1, 3), - Data = [{version, erlang:list_to_binary(Version)}, - {date, erlang:list_to_binary(emqx_mgmt_util:strftime(Seconds))}, - {rules, Rules}, - {resources, Resources}, - {blacklist, Blacklist}, - {apps, Apps}, - {users, Users}, - {auth_mnesia, AuthMnesia}, - {acl_mnesia, AclMnesia} - ], - - Bin = emqx_json:encode(Data), - ok = filelib:ensure_dir(NFilename), - case file:write_file(NFilename, Bin) of - ok -> - case file:read_file_info(NFilename) of - {ok, #file_info{size = Size, ctime = {{Y0, M0, D0}, {H0, MM0, S0}}}} -> - CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y0, M0, D0, H0, MM0, S0]), - return({ok, [{filename, list_to_binary(Filename)}, - {size, Size}, - {created_at, list_to_binary(CreatedAt)}, - {node, node()} - ]}); - {error, Reason} -> - return({error, Reason}) - end; - {error, Reason} -> - return({error, Reason}) - end. - + return(emqx_mgmt_data_backup:export()). + list_exported(_Bindings, _Params) -> List = [ rpc:call(Node, ?MODULE, get_list_exported, []) || Node <- ekka_mnesia:running_nodes() ], NList = lists:map(fun({_, FileInfo}) -> FileInfo end, lists:keysort(1, lists:append(List))), @@ -158,7 +118,7 @@ import(_Bindings, Params) -> case lists:member(Node, [ erlang:atom_to_binary(N, utf8) || N <- ekka_mnesia:running_nodes() ] ) of - true -> rpc:call(erlang:binary_to_atom(Node, utf8), ?MODULE, do_import, [Filename]); + true -> return(rpc:call(erlang:binary_to_atom(Node, utf8), ?MODULE, do_import, [Filename])); false -> return({error, no_existent_node}) end end, @@ -167,34 +127,7 @@ import(_Bindings, Params) -> do_import(Filename) -> FullFilename = filename:join([emqx:get_env(data_dir), Filename]), - case file:read_file(FullFilename) of - {ok, Json} -> - Data = emqx_json:decode(Json, [return_maps]), - Version = emqx_mgmt:to_version(maps:get(<<"version">>, Data)), - case lists:member(Version, ?VERSIONS) of - true -> - try - emqx_mgmt:import_resources_and_rules(maps:get(<<"resources">>, Data, []), maps:get(<<"rules">>, Data, []), Version), - emqx_mgmt:import_blacklist(maps:get(<<"blacklist">>, Data, [])), - emqx_mgmt:import_applications(maps:get(<<"apps">>, Data, [])), - emqx_mgmt:import_users(maps:get(<<"users">>, Data, [])), - _ = emqx_mgmt:import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), - _ = emqx_mgmt:import_auth_username(maps:get(<<"auth_username">>, Data, [])), - _ = emqx_mgmt:import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version), - _ = emqx_mgmt:import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version), - logger:debug("The emqx data has been imported successfully"), - ok - catch Class:Reason:Stack -> - logger:error("The emqx data import failed: ~0p", [{Class,Reason,Stack}]), - {error, import_failed} - end; - false -> - logger:error("Unsupported version: ~p", [Version]), - {error, unsupported_version} - end; - {error, Reason} -> - {error, Reason} - end. + emqx_mgmt_data_backup:import(FullFilename). download(#{filename := Filename}, _Params) -> FullFilename = filename:join([emqx:get_env(data_dir), Filename]), diff --git a/lib-ce/emqx_management/src/emqx_mgmt_cli.erl b/lib-ce/emqx_management/src/emqx_mgmt_cli.erl index 8f3a09026..d2712b5e1 100644 --- a/lib-ce/emqx_management/src/emqx_mgmt_cli.erl +++ b/lib-ce/emqx_management/src/emqx_mgmt_cli.erl @@ -41,7 +41,6 @@ , log/1 , mgmt/1 , data/1 - , modules/1 ]). -define(PROC_INFOKEYS, [status, @@ -322,44 +321,6 @@ plugins(_) -> {"plugins reload ", "Reload plugin"} ]). -%%-------------------------------------------------------------------- -%% @doc Modules Command -modules(["list"]) -> - foreach(fun(Module) -> print({module, Module}) end, emqx_modules:list()); - -modules(["load", Name]) -> - case emqx_modules:load(list_to_atom(Name)) of - ok -> - emqx_ctl:print("Module ~s loaded successfully.~n", [Name]); - {error, Reason} -> - emqx_ctl:print("Load module ~s error: ~p.~n", [Name, Reason]) - end; - -modules(["unload", Name]) -> - case emqx_modules:unload(list_to_atom(Name)) of - ok -> - emqx_ctl:print("Module ~s unloaded successfully.~n", [Name]); - {error, Reason} -> - emqx_ctl:print("Unload module ~s error: ~p.~n", [Name, Reason]) - end; - -modules(["reload", "emqx_mod_acl_internal" = Name]) -> - case emqx_modules:reload(list_to_atom(Name)) of - ok -> - emqx_ctl:print("Module ~s reloaded successfully.~n", [Name]); - {error, Reason} -> - emqx_ctl:print("Reload module ~s error: ~p.~n", [Name, Reason]) - end; -modules(["reload", Name]) -> - emqx_ctl:print("Module: ~p does not need to be reloaded.~n", [Name]); - -modules(_) -> - emqx_ctl:usage([{"modules list", "Show loaded modules"}, - {"modules load ", "Load module"}, - {"modules unload ", "Unload module"}, - {"modules reload ", "Reload module"} - ]). - %%-------------------------------------------------------------------- %% @doc vm command @@ -582,59 +543,21 @@ stop_listener(#{listen_on := ListenOn} = Listener, _Input) -> %% @doc data Command data(["export"]) -> - Rules = emqx_mgmt:export_rules(), - Resources = emqx_mgmt:export_resources(), - Blacklist = emqx_mgmt:export_blacklist(), - Apps = emqx_mgmt:export_applications(), - Users = emqx_mgmt:export_users(), - AuthMnesia = emqx_mgmt:export_auth_mnesia(), - AclMnesia = emqx_mgmt:export_acl_mnesia(), - Seconds = erlang:system_time(second), - {{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds), - Filename = io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]), - NFilename = filename:join([emqx:get_env(data_dir), Filename]), - Version = string:sub_string(emqx_sys:version(), 1, 3), - Data = [{version, erlang:list_to_binary(Version)}, - {date, erlang:list_to_binary(emqx_mgmt_util:strftime(Seconds))}, - {rules, Rules}, - {resources, Resources}, - {blacklist, Blacklist}, - {apps, Apps}, - {users, Users}, - {auth_mnesia, AuthMnesia}, - {acl_mnesia, AclMnesia}], - ok = filelib:ensure_dir(NFilename), - case file:write_file(NFilename, emqx_json:encode(Data)) of - ok -> + case emqx_mgmt_data_backup:export() of + {ok, _} -> emqx_ctl:print("The emqx data has been successfully exported to ~s.~n", [NFilename]); {error, Reason} -> emqx_ctl:print("The emqx data export failed due to ~p.~n", [Reason]) - end; + end; data(["import", Filename]) -> - case file:read_file(Filename) of - {ok, Json} -> - Data = emqx_json:decode(Json, [return_maps]), - Version = emqx_mgmt:to_version(maps:get(<<"version">>, Data)), - case lists:member(Version, ?VERSIONS) of - true -> - try - emqx_mgmt:import_resources(maps:get(<<"resources">>, Data, [])), - emqx_mgmt:import_rules(maps:get(<<"rules">>, Data, [])), - emqx_mgmt:import_blacklist(maps:get(<<"blacklist">>, Data, [])), - emqx_mgmt:import_applications(maps:get(<<"apps">>, Data, [])), - emqx_mgmt:import_users(maps:get(<<"users">>, Data, [])), - _ = emqx_mgmt:import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), - _ = emqx_mgmt:import_auth_username(maps:get(<<"auth_username">>, Data, [])), - _ = emqx_mgmt:import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version), - _ = emqx_mgmt:import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version), - emqx_ctl:print("The emqx data has been imported successfully.~n") - catch Class:Reason:Stack -> - emqx_ctl:print("The emqx data import failed due: ~0p~n", [{Class,Reason,Stack}]) - end; - false -> - emqx_ctl:print("Unsupported version: ~p~n", [Version]) - end; + case emqx_mgmt_data_backup:import(Filename) of + ok -> + emqx_ctl:print("The emqx data has been imported successfully.~n"); + {error, import_failed} -> + emqx_ctl:print("The emqx data import failed due: ~0p~n", [{Class,Reason,Stack}]); + {error, unsupported_version} -> + emqx_ctl:print("Unsupported version: ~p~n", [Version]); {error, Reason} -> emqx_ctl:print("The emqx data import failed: ~0p while reading ~s.~n", [Reason, Filename]) end; diff --git a/lib-ce/emqx_management/src/emqx_mgmt_data_backup.erl b/lib-ce/emqx_management/src/emqx_mgmt_data_backup.erl new file mode 100644 index 000000000..c2e4be94e --- /dev/null +++ b/lib-ce/emqx_management/src/emqx_mgmt_data_backup.erl @@ -0,0 +1,557 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mgmt_data_backup). + +-ifdef(EMQX_ENTERPISE). +-export([ export_modules/0 + , export_schemas/0 + , export_confs/0 + , import_modules/1 + , import_schemas/1 + , import_confs/2 + ]). +-else. +-export([ import_resources_and_rules/3 ]). +-endif. + +-export([ export_rules/0 + , export_resources/0 + , export_blacklist/0 + , export_applications/0 + , export_users/0 + , export_auth_mnesia/0 + , export_acl_mnesia/0 + , import_rules/1 + , import_resources/1 + , import_blacklist/1 + , import_applications/1 + , import_users/1 + , import_auth_clientid/1 %% BACKW: 4.1.x + , import_auth_username/1 %% BACKW: 4.1.x + , import_auth_mnesia/2 + , import_acl_mnesia/2 + , to_version/1 + ]). + +-export([ export/0 + , import/1 + ]). + +%%-------------------------------------------------------------------- +%% Data Export and Import +%%-------------------------------------------------------------------- + +export_rules() -> + lists:map(fun({_, RuleId, _, RawSQL, _, _, _, _, _, _, Actions, Enabled, Desc}) -> + [{id, RuleId}, + {rawsql, RawSQL}, + {actions, actions_to_prop_list(Actions)}, + {enabled, Enabled}, + {description, Desc}] + end, emqx_rule_registry:get_rules()). + +export_resources() -> + lists:map(fun({_, Id, Type, Config, CreatedAt, Desc}) -> + NCreatedAt = case CreatedAt of + undefined -> null; + _ -> CreatedAt + end, + [{id, Id}, + {type, Type}, + {config, maps:to_list(Config)}, + {created_at, NCreatedAt}, + {description, Desc}] + end, emqx_rule_registry:get_resources()). + +export_blacklist() -> + lists:map(fun(#banned{who = Who, by = By, reason = Reason, at = At, until = Until}) -> + NWho = case Who of + {peerhost, Peerhost} -> {peerhost, inet:ntoa(Peerhost)}; + _ -> Who + end, + [{who, [NWho]}, {by, By}, {reason, Reason}, {at, At}, {until, Until}] + end, ets:tab2list(emqx_banned)). + +export_applications() -> + lists:map(fun({_, AppID, AppSecret, Name, Desc, Status, Expired}) -> + [{id, AppID}, {secret, AppSecret}, {name, Name}, {desc, Desc}, {status, Status}, {expired, Expired}] + end, ets:tab2list(mqtt_app)). + +export_users() -> + lists:map(fun({_, Username, Password, Tags}) -> + [{username, Username}, {password, base64:encode(Password)}, {tags, Tags}] + end, ets:tab2list(mqtt_admin)). + +export_auth_mnesia() -> + case ets:info(emqx_user) of + undefined -> []; + _ -> + lists:map(fun({_, {Type, Login}, Password, CreatedAt}) -> + [{login, Login}, {type, Type}, {password, base64:encode(Password)}, {created_at, CreatedAt}] + end, ets:tab2list(emqx_user)) + end. + +export_acl_mnesia() -> + case ets:info(emqx_acl) of + undefined -> []; + _ -> + lists:map(fun({_, Filter, Action, Access, CreatedAt}) -> + Filter1 = case Filter of + {{Type, TypeValue}, Topic} -> + [{type, Type}, {type_value, TypeValue}, {topic, Topic}]; + {Type, Topic} -> + [{type, Type}, {topic, Topic}] + end, + Filter1 ++ [{action, Action}, {access, Access}, {created_at, CreatedAt}] + end, ets:tab2list(emqx_acl)) + end. + +-ifdef(EMQX_ENTERPISE). +export_modules() -> + case ets:info(emqx_modules) of + undefined -> []; + _ -> + lists:map(fun({_, Id, Type, Config, Enabled, CreatedAt, Description}) -> + [{id, Id}, + {type, Type}, + {config, Config}, + {enabled, Enabled}, + {created_at, CreatedAt}, + {description, Description} + ] + end, ets:tab2list(emqx_modules)) + end. + +export_schemas() -> + case ets:info(emqx_schema) of + undefined -> []; + _ -> + [emqx_schema_api:format_schema(Schema) || Schema <- emqx_schema_registry:get_all_schemas()] + end. + +export_confs() -> + case ets:info(emqx_conf_info) of + undefined -> {[], []}; + _ -> + {lists:map(fun({_, Key, Confs}) -> + case Key of + {_Zone, Name} -> + [{zone, list_to_binary(Name)}, + {confs, confs_to_binary(Confs)}]; + {_Listener, Type, Name} -> + [{type, list_to_binary(Type)}, + {name, list_to_binary(Name)}, + {confs, confs_to_binary(Confs)}]; + Name -> + [{name, list_to_binary(Name)}, + {confs, confs_to_binary(Confs)}] + end + end, ets:tab2list(emqx_conf_b)), + lists:map(fun({_, {_Listener, Type, Name}, Status}) -> + [{type, list_to_binary(Type)}, + {name, list_to_binary(Name)}, + {status, Status}] + end, ets:tab2list(emqx_listeners_state))} + end. + +confs_to_binary(Confs) -> + [{list_to_binary(Key), list_to_binary(Val)} || {Key, Val} <-Confs]. + +-endif. + +import_rules(Rules) -> + lists:foreach(fun(Resource) -> + import_resource(Resource) + end, Rules). +import_rule(#{<<"id">> := RuleId, + <<"rawsql">> := RawSQL, + <<"actions">> := Actions, + <<"enabled">> := Enabled, + <<"description">> := Desc}) -> + Rule = #{id => RuleId, + rawsql => RawSQL, + actions => map_to_actions(Actions), + enabled => Enabled, + description => Desc}, + try emqx_rule_engine:create_rule(Rule) + catch throw:{resource_not_initialized, _ResId} -> + emqx_rule_engine:create_rule(Rule#{enabled => false}) + end. + +import_resources(Reources) -> + lists:foreach(fun(Resource) -> + import_resource(Resource) + end, Reources). + +import_resource(#{<<"id">> := Id, + <<"type">> := Type, + <<"config">> := Config, + <<"created_at">> := CreatedAt, + <<"description">> := Desc}) -> + NCreatedAt = case CreatedAt of + null -> undefined; + _ -> CreatedAt + end, + emqx_rule_engine:create_resource(#{id => Id, + type => any_to_atom(Type), + config => Config, + created_at => NCreatedAt, + description => Desc}). + +-ifndef(EMQX_ENTERPRISE). +import_resources_and_rules(Resources, Rules, FromVersion) + when FromVersion =:= "4.0" orelse FromVersion =:= "4.1" orelse FromVersion =:= "4.2" -> + Configs = lists:foldl(fun(#{<<"id">> := ID, + <<"type">> := <<"web_hook">>, + <<"config">> := #{<<"content_type">> := ContentType, + <<"headers">> := Headers, + <<"method">> := Method, + <<"url">> := URL}} = Resource, Acc) -> + NConfig = #{<<"connect_timeout">> => 5, + <<"request_timeout">> => 5, + <<"cacertfile">> => <<>>, + <<"certfile">> => <<>>, + <<"keyfile">> => <<>>, + <<"pool_size">> => 8, + <<"url">> => URL, + <<"verify">> => true}, + NResource = Resource#{<<"config">> := NConfig}, + import_resource(NResource), + NHeaders = maps:put(<<"content-type">>, ContentType, Headers), + [{ID, #{headers => NHeaders, method => Method}} | Acc]; + (Resource, Acc) -> + import_resource(Resource), + Acc + end, [], Resources), + lists:foreach(fun(#{<<"actions">> := Actions} = Rule) -> + NActions = apply_new_config(Actions, Configs), + import_rule(Rule#{<<"actions">> := NActions}) + end, Rules); +import_resources_and_rules(Resources, Rules, _FromVersion) -> + import_resources(Resources), + import_rules(Rules). + +apply_new_config(Actions, Configs) -> + apply_new_config(Actions, Configs, []). + +apply_new_config([], _Configs, Acc) -> + Acc; +apply_new_config([Action = #{<<"name">> := <<"data_to_webserver">>, + <<"args">> := #{<<"$resource">> := ID, + <<"path">> := Path, + <<"payload_tmpl">> := PayloadTmpl}} | More], Configs, Acc) -> + case proplists:get_value(ID, Configs, undefined) of + undefined -> + apply_new_config(More, Configs, [Action | Acc]); + #{headers := Headers, method := Method} -> + Args = #{<<"$resource">> => ID, + <<"body">> => PayloadTmpl, + <<"headers">> => Headers, + <<"method">> => Method, + <<"path">> => Path}, + apply_new_config(More, Configs, [Action#{<<"args">> := Args} | Acc]) + end. + + +-endif. + +import_blacklist(Blacklist) -> + lists:foreach(fun(#{<<"who">> := Who, + <<"by">> := By, + <<"reason">> := Reason, + <<"at">> := At, + <<"until">> := Until}) -> + NWho = case Who of + #{<<"peerhost">> := Peerhost} -> + {ok, NPeerhost} = inet:parse_address(Peerhost), + {peerhost, NPeerhost}; + #{<<"clientid">> := ClientId} -> {clientid, ClientId}; + #{<<"username">> := Username} -> {username, Username} + end, + emqx_banned:create(#banned{who = NWho, by = By, reason = Reason, at = At, until = Until}) + end, Blacklist). + +import_applications(Apps) -> + lists:foreach(fun(#{<<"id">> := AppID, + <<"secret">> := AppSecret, + <<"name">> := Name, + <<"desc">> := Desc, + <<"status">> := Status, + <<"expired">> := Expired}) -> + NExpired = case is_integer(Expired) of + true -> Expired; + false -> undefined + end, + emqx_mgmt_auth:force_add_app(AppID, Name, AppSecret, Desc, Status, NExpired) + end, Apps). + +import_users(Users) -> + lists:foreach(fun(#{<<"username">> := Username, + <<"password">> := Password, + <<"tags">> := Tags}) -> + NPassword = base64:decode(Password), + emqx_dashboard_admin:force_add_user(Username, NPassword, Tags) + end, Users). + +import_auth_clientid(Lists) -> + case ets:info(emqx_user) of + undefined -> ok; + _ -> + lists:foreach(fun(#{<<"clientid">> := Clientid, <<"password">> := Password}) -> + mnesia:dirty_write({emqx_user, {clientid, Clientid}, base64:decode(Password), erlang:system_time(millisecond)}) + end, Lists) + end. + +import_auth_username(Lists) -> + case ets:info(emqx_user) of + undefined -> ok; + _ -> + lists:foreach(fun(#{<<"username">> := Username, <<"password">> := Password}) -> + mnesia:dirty_write({emqx_user, {username, Username}, base64:decode(Password), erlang:system_time(millisecond)}) + end, Lists) + end. + +import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse + FromVersion =:= "4.1" -> + case ets:info(emqx_user) of + undefined -> ok; + _ -> + CreatedAt = erlang:system_time(millisecond), + lists:foreach(fun(#{<<"login">> := Login, + <<"password">> := Password}) -> + mnesia:dirty_write({emqx_user, {username, Login}, base64:decode(Password), CreatedAt}) + end, Auths) + end; + +import_auth_mnesia(Auths, _) -> + case ets:info(emqx_user) of + undefined -> ok; + _ -> + lists:foreach(fun(#{<<"login">> := Login, + <<"type">> := Type, + <<"password">> := Password, + <<"created_at">> := CreatedAt }) -> + mnesia:dirty_write({emqx_user, {any_to_atom(Type), Login}, base64:decode(Password), CreatedAt}) + end, Auths) + end. + +import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse + FromVersion =:= "4.1" -> + case ets:info(emqx_acl) of + undefined -> ok; + _ -> + CreatedAt = erlang:system_time(millisecond), + lists:foreach(fun(#{<<"login">> := Login, + <<"topic">> := Topic, + <<"allow">> := Allow, + <<"action">> := Action}) -> + Allow1 = case any_to_atom(Allow) of + true -> allow; + false -> deny + end, + mnesia:dirty_write({emqx_acl, {{username, Login}, Topic}, any_to_atom(Action), Allow1, CreatedAt}) + end, Acls) + end; + +import_acl_mnesia(Acls, _) -> + case ets:info(emqx_acl) of + undefined -> ok; + _ -> + lists:foreach(fun(Map = #{<<"action">> := Action, + <<"access">> := Access, + <<"created_at">> := CreatedAt}) -> + Filter = case maps:get(<<"type_value">>, Map, undefined) of + undefined -> + {any_to_atom(maps:get(<<"type">>, Map)), maps:get(<<"topic">>, Map)}; + Value -> + {{any_to_atom(maps:get(<<"type">>, Map)), Value}, maps:get(<<"topic">>, Map)} + end, + mnesia:dirty_write({emqx_acl ,Filter, any_to_atom(Action), any_to_atom(Access), CreatedAt}) + end, Acls) + end. + +-ifdef(EMQX_ENTERPRISE). +import_modules(Modules) -> + case ets:info(emqx_modules) of + undefined -> []; + _ -> + lists:foreach(fun(#{<<"id">> := Id, + <<"type">> := Type, + <<"config">> := Config, + <<"enabled">> := Enabled, + <<"created_at">> := CreatedAt, + <<"description">> := Description}) -> + emqx_modules:import_module({Id, any_to_atom(Type), Config, Enabled, CreatedAt, Description}) + end, Modules) + end. + + +import_schemas(Schemas) -> + case ets:info(emqx_schema) of + undefined -> ok; + _ -> [emqx_schema_registry:add_schema(emqx_schema_api:make_schema_params(Schema)) || Schema <- Schemas] + end. + +import_confs(Configs, ListenersState) -> + case ets:info(emqx_conf_info) of + undefined -> ok; + _ -> + emqx_conf:import_confs(Configs, ListenersState) + end. + +-endif. + +-ifdef(EMQX_ENTERPRISE). +export() -> + Modules = export_modules(), + Rules = export_rules(), + Resources = export_resources(), + Blacklist = export_blacklist(), + Apps = export_applications(), + Users = export_users(), + AuthMnesia = export_auth_mnesia(), + AclMnesia = export_acl_mnesia(), + Schemas = export_schemas(), + {Configs, State} = export_confs(), + Seconds = erlang:system_time(second), + {{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds), + Filename = io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]), + NFilename = filename:join([emqx:get_env(data_dir), Filename]), + Version = string:sub_string(emqx_sys:version(), 1, 3), + Data = [{version, erlang:list_to_binary(Version)}, + {date, erlang:list_to_binary(emqx_mgmt_util:strftime(Seconds))}, + {modules, Modules}, + {rules, Rules}, + {resources, Resources}, + {blacklist, Blacklist}, + {apps, Apps}, + {users, Users}, + {auth_mnesia, AuthMnesia}, + {acl_mnesia, AclMnesia}, + {schemas, Schemas}, + {configs, Configs}, + {listeners_state, State}], + write_file(NFilename, Data). + +import(Filename) -> + case file:read_file(FullFilename) of + {ok, Json} -> + Data = emqx_json:decode(Json, [return_maps]), + Version = to_version(maps:get(<<"version">>, Data)), + case lists:member(Version, ?VERSIONS) of + true -> + try + import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])), + import_resources(maps:get(<<"resources">>, Data, [])), + import_rules(maps:get(<<"rules">>, Data, [])), + import_blacklist(maps:get(<<"blacklist">>, Data, [])), + import_applications(maps:get(<<"apps">>, Data, [])), + import_users(maps:get(<<"users">>, Data, [])), + import_modules(maps:get(<<"modules">>, Data, [])), + import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), + import_auth_username(maps:get(<<"auth_username">>, Data, [])), + import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version), + import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version), + import_schemas(maps:get(<<"schemas">>, Data, [])), + logger:debug("The emqx data has been imported successfully"), + ok + catch Class:Reason:Stack -> + logger:error("The emqx data import failed: ~0p", [{Class,Reason,Stack}]), + {error, import_failed} + end; + false -> + logger:error("Unsupported version: ~p", [Version]), + {error, unsupported_version} + end; + {error, Reason} -> + {error, Reason} + end. + +-else. +export() -> + Rules = export_rules(), + Resources = export_resources(), + Blacklist = export_blacklist(), + Apps = export_applications(), + Users = export_users(), + AuthMnesia = export_auth_mnesia(), + AclMnesia = export_acl_mnesia(), + Seconds = erlang:system_time(second), + {{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds), + Filename = io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]), + NFilename = filename:join([emqx:get_env(data_dir), Filename]), + Version = string:sub_string(emqx_sys:version(), 1, 3), + Data = [{version, erlang:list_to_binary(Version)}, + {date, erlang:list_to_binary(emqx_mgmt_util:strftime(Seconds))}, + {rules, Rules}, + {resources, Resources}, + {blacklist, Blacklist}, + {apps, Apps}, + {users, Users}, + {auth_mnesia, AuthMnesia}, + {acl_mnesia, AclMnesia}], + write_file(NFilename, Data). + +import(Filename) -> + case file:read_file(FullFilename) of + {ok, Json} -> + Data = emqx_json:decode(Json, [return_maps]), + Version = to_version(maps:get(<<"version">>, Data)), + case lists:member(Version, ?VERSIONS) of + true -> + try + import_resources_and_rules(maps:get(<<"resources">>, Data, []), maps:get(<<"rules">>, Data, []), Version), + import_blacklist(maps:get(<<"blacklist">>, Data, [])), + import_applications(maps:get(<<"apps">>, Data, [])), + import_users(maps:get(<<"users">>, Data, [])), + import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), + import_auth_username(maps:get(<<"auth_username">>, Data, [])), + import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version), + import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version), + logger:debug("The emqx data has been imported successfully"), + ok + catch Class:Reason:Stack -> + logger:error("The emqx data import failed: ~0p", [{Class,Reason,Stack}]), + {error, import_failed} + end; + false -> + logger:error("Unsupported version: ~p", [Version]), + {error, unsupported_version} + end; + {error, Reason} -> + {error, Reason} + end. +-endif. + +write_file(Filename, Data) -> + ok = filelib:ensure_dir(Filename), + case file:write_file(Filename, emqx_json:encode(Data)) of + ok -> + case file:read_file_info(Filename) of + {ok, #file_info{size = Size, ctime = {{Y, M, D}, {H, MM, S}}}} -> + CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y, M, D, H, MM, S]), + {ok, [{filename, list_to_binary(Filename)}, + {size, Size}, + {created_at, list_to_binary(CreatedAt)}, + {node, node()} + ]}; + {error, Reason} -> + {error, Reason} + end; + {error, Reason} -> + {error, Reason} + end. From 24d954282d1d9b53e1301ebcadd9c4f9688b76f8 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 25 Feb 2021 14:10:30 +0800 Subject: [PATCH 04/52] chore(emqx_management): update test case --- .../src/emqx_management.app.src | 2 +- lib-ce/emqx_management/src/emqx_mgmt.erl | 26 ---- .../src/emqx_mgmt_api_data.erl | 6 +- lib-ce/emqx_management/src/emqx_mgmt_cli.erl | 18 +-- .../src/emqx_mgmt_data_backup.erl | 53 +++++-- .../emqx_management/test/emqx_mgmt_SUITE.erl | 18 +-- .../test/emqx_mgmt_api_SUITE.erl | 63 +------- lib-ce/emqx_modules/src/emqx_modules.erl | 5 +- lib-ce/emqx_modules/src/emqx_modules_api.erl | 26 ++-- .../emqx_modules/test/emqx_modules_SUITE.erl | 147 +++++++++++++++++- 10 files changed, 222 insertions(+), 142 deletions(-) diff --git a/lib-ce/emqx_management/src/emqx_management.app.src b/lib-ce/emqx_management/src/emqx_management.app.src index 6f29fb4ff..b1e04c439 100644 --- a/lib-ce/emqx_management/src/emqx_management.app.src +++ b/lib-ce/emqx_management/src/emqx_management.app.src @@ -3,7 +3,7 @@ {vsn, "4.3.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, - {applications, [kernel,stdlib,minirest,emqx_modules]}, + {applications, [kernel,stdlib,minirest]}, {mod, {emqx_mgmt_app,[]}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/lib-ce/emqx_management/src/emqx_mgmt.erl b/lib-ce/emqx_management/src/emqx_mgmt.erl index e7d896c6a..39bc2153a 100644 --- a/lib-ce/emqx_management/src/emqx_mgmt.erl +++ b/lib-ce/emqx_management/src/emqx_mgmt.erl @@ -483,16 +483,6 @@ delete_banned(Who) -> emqx_banned:delete(Who). -any_to_atom(L) when is_list(L) -> list_to_atom(L); -any_to_atom(B) when is_binary(B) -> binary_to_atom(B, utf8); -any_to_atom(A) when is_atom(A) -> A. - -to_version(Version) when is_integer(Version) -> - integer_to_list(Version); -to_version(Version) when is_binary(Version) -> - binary_to_list(Version); -to_version(Version) when is_list(Version) -> - Version. %%-------------------------------------------------------------------- %% Telemtry API @@ -561,20 +551,4 @@ max_row_limit() -> table_size(Tab) -> ets:info(Tab, size). -map_to_actions(Maps) -> - [map_to_action(M) || M <- Maps]. -map_to_action(Map = #{<<"id">> := ActionInstId, <<"name">> := Name, <<"args">> := Args}) -> - #{id => ActionInstId, - name => any_to_atom(Name), - args => Args, - fallbacks => map_to_actions(maps:get(<<"fallbacks">>, Map, []))}. - -actions_to_prop_list(Actions) -> - [action_to_prop_list(Act) || Act <- Actions]. - -action_to_prop_list({action_instance, ActionInstId, Name, FallbackActions, Args}) -> - [{id, ActionInstId}, - {name, Name}, - {fallbacks, actions_to_prop_list(FallbackActions)}, - {args, Args}]. diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_data.erl b/lib-ce/emqx_management/src/emqx_mgmt_api_data.erl index d3d22e4d1..18ba27265 100644 --- a/lib-ce/emqx_management/src/emqx_mgmt_api_data.erl +++ b/lib-ce/emqx_management/src/emqx_mgmt_api_data.erl @@ -75,7 +75,11 @@ ]). export(_Bindings, _Params) -> - return(emqx_mgmt_data_backup:export()). + case emqx_mgmt_data_backup:export() of + {ok, File = #{filename := Filename}} -> + return({ok, File#{filename => filename:basename(Filename)}}); + Return -> return(Return) + end. list_exported(_Bindings, _Params) -> List = [ rpc:call(Node, ?MODULE, get_list_exported, []) || Node <- ekka_mnesia:running_nodes() ], diff --git a/lib-ce/emqx_management/src/emqx_mgmt_cli.erl b/lib-ce/emqx_management/src/emqx_mgmt_cli.erl index d2712b5e1..19cf83f99 100644 --- a/lib-ce/emqx_management/src/emqx_mgmt_cli.erl +++ b/lib-ce/emqx_management/src/emqx_mgmt_cli.erl @@ -58,7 +58,7 @@ -spec(load() -> ok). load() -> Cmds = [Fun || {Fun, _} <- ?MODULE:module_info(exports), is_cmd(Fun)], - lists:foreach(fun(Cmd) -> emqx_ctl:register_command(Cmd, {?MODULE, Cmd}, []) end, Cmds). + foreach(fun(Cmd) -> emqx_ctl:register_command(Cmd, {?MODULE, Cmd}, []) end, Cmds). is_cmd(Fun) -> not lists:member(Fun, [init, load, module_info]). @@ -100,7 +100,7 @@ mgmt(["delete", AppId]) -> end; mgmt(["list"]) -> - lists:foreach(fun({AppId, AppSecret, Name, Desc, Status, Expired}) -> + foreach(fun({AppId, AppSecret, Name, Desc, Status, Expired}) -> emqx_ctl:print("app_id: ~s, secret: ~s, name: ~s, desc: ~s, status: ~s, expired: ~p~n", [AppId, AppSecret, Name, Desc, Status, Expired]) end, emqx_mgmt_auth:list_apps()); @@ -229,7 +229,7 @@ routes(_) -> {"routes show ", "Show a route"}]). subscriptions(["list"]) -> - lists:foreach(fun(Suboption) -> + foreach(fun(Suboption) -> print({emqx_suboption, Suboption}) end, ets:tab2list(emqx_suboption)); @@ -544,8 +544,8 @@ stop_listener(#{listen_on := ListenOn} = Listener, _Input) -> data(["export"]) -> case emqx_mgmt_data_backup:export() of - {ok, _} -> - emqx_ctl:print("The emqx data has been successfully exported to ~s.~n", [NFilename]); + {ok, #{filename := Filename}} -> + emqx_ctl:print("The emqx data has been successfully exported to ~s.~n", [Filename]); {error, Reason} -> emqx_ctl:print("The emqx data export failed due to ~p.~n", [Reason]) end; @@ -555,9 +555,9 @@ data(["import", Filename]) -> ok -> emqx_ctl:print("The emqx data has been imported successfully.~n"); {error, import_failed} -> - emqx_ctl:print("The emqx data import failed due: ~0p~n", [{Class,Reason,Stack}]); + emqx_ctl:print("The emqx data import failed. ~n"); {error, unsupported_version} -> - emqx_ctl:print("Unsupported version: ~p~n", [Version]); + emqx_ctl:print("The emqx data import failed: Unsupported version. ~n"); {error, Reason} -> emqx_ctl:print("The emqx data import failed: ~0p while reading ~s.~n", [Reason, Filename]) end; @@ -636,10 +636,6 @@ print(#plugin{name = Name, descr = Descr, active = Active}) -> emqx_ctl:print("Plugin(~s, description=~s, active=~s)~n", [Name, Descr, Active]); -print({module, {Name, Active}}) -> - emqx_ctl:print("Module(~s, description=~s, active=~s)~n", - [Name, Name:description(), Active]); - print({emqx_suboption, {{Pid, Topic}, Options}}) when is_pid(Pid) -> emqx_ctl:print("~s -> ~s~n", [maps:get(subid, Options), Topic]). diff --git a/lib-ce/emqx_management/src/emqx_mgmt_data_backup.erl b/lib-ce/emqx_management/src/emqx_mgmt_data_backup.erl index c2e4be94e..03703be91 100644 --- a/lib-ce/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/lib-ce/emqx_management/src/emqx_mgmt_data_backup.erl @@ -16,6 +16,10 @@ -module(emqx_mgmt_data_backup). +-include("emqx_mgmt.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("kernel/include/file.hrl"). + -ifdef(EMQX_ENTERPISE). -export([ export_modules/0 , export_schemas/0 @@ -266,6 +270,14 @@ apply_new_config([Action = #{<<"name">> := <<"data_to_webserver">>, apply_new_config(More, Configs, [Action#{<<"args">> := Args} | Acc]) end. +actions_to_prop_list(Actions) -> + [action_to_prop_list(Act) || Act <- Actions]. + +action_to_prop_list({action_instance, ActionInstId, Name, FallbackActions, Args}) -> + [{id, ActionInstId}, + {name, Name}, + {fallbacks, actions_to_prop_list(FallbackActions)}, + {args, Args}]. -endif. @@ -415,6 +427,26 @@ import_confs(Configs, ListenersState) -> -endif. +any_to_atom(L) when is_list(L) -> list_to_atom(L); +any_to_atom(B) when is_binary(B) -> binary_to_atom(B, utf8); +any_to_atom(A) when is_atom(A) -> A. + +map_to_actions(Maps) -> + [map_to_action(M) || M <- Maps]. + +map_to_action(Map = #{<<"id">> := ActionInstId, <<"name">> := Name, <<"args">> := Args}) -> + #{id => ActionInstId, + name => any_to_atom(Name), + args => Args, + fallbacks => map_to_actions(maps:get(<<"fallbacks">>, Map, []))}. + +to_version(Version) when is_integer(Version) -> + integer_to_list(Version); +to_version(Version) when is_binary(Version) -> + binary_to_list(Version); +to_version(Version) when is_list(Version) -> + Version. + -ifdef(EMQX_ENTERPRISE). export() -> Modules = export_modules(), @@ -507,7 +539,7 @@ export() -> write_file(NFilename, Data). import(Filename) -> - case file:read_file(FullFilename) of + case file:read_file(Filename) of {ok, Json} -> Data = emqx_json:decode(Json, [return_maps]), Version = to_version(maps:get(<<"version">>, Data)), @@ -532,8 +564,7 @@ import(Filename) -> logger:error("Unsupported version: ~p", [Version]), {error, unsupported_version} end; - {error, Reason} -> - {error, Reason} + Error -> Error end. -endif. @@ -544,14 +575,12 @@ write_file(Filename, Data) -> case file:read_file_info(Filename) of {ok, #file_info{size = Size, ctime = {{Y, M, D}, {H, MM, S}}}} -> CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y, M, D, H, MM, S]), - {ok, [{filename, list_to_binary(Filename)}, - {size, Size}, - {created_at, list_to_binary(CreatedAt)}, - {node, node()} - ]}; - {error, Reason} -> - {error, Reason} + {ok, #{filename => list_to_binary(Filename), + size => Size, + created_at => list_to_binary(CreatedAt), + node => node() + }}; + Error -> Error end; - {error, Reason} -> - {error, Reason} + Error -> Error end. diff --git a/lib-ce/emqx_management/test/emqx_mgmt_SUITE.erl b/lib-ce/emqx_management/test/emqx_mgmt_SUITE.erl index 56db2d118..a3e9a6fad 100644 --- a/lib-ce/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/lib-ce/emqx_management/test/emqx_mgmt_SUITE.erl @@ -44,7 +44,6 @@ groups() -> t_clients_cmd, t_vm_cmd, t_plugins_cmd, - t_modules_cmd, t_trace_cmd, t_broker_cmd, t_router_cmd, @@ -59,11 +58,11 @@ apps() -> init_per_suite(Config) -> ekka_mnesia:start(), emqx_mgmt_auth:mnesia(boot), - emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_auth_mnesia]), + emqx_ct_helpers:start_apps([emqx_management, emqx_auth_mnesia]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_management, emqx_auth_mnesia, emqx_modules]). + emqx_ct_helpers:stop_apps([emqx_management, emqx_auth_mnesia]). t_app(_Config) -> {ok, AppSecret} = emqx_mgmt_auth:add_app(<<"app_id">>, <<"app_name">>), @@ -329,19 +328,6 @@ t_plugins_cmd(_) -> ), unmock_print(). -t_modules_cmd(_) -> - mock_print(), - meck:new(emqx_modules, [non_strict, passthrough]), - meck:expect(emqx_modules, load, fun(_) -> ok end), - meck:expect(emqx_modules, unload, fun(_) -> ok end), - meck:expect(emqx_modules, reload, fun(_) -> ok end), - ?assertEqual(emqx_mgmt_cli:modules(["list"]), ok), - ?assertEqual(emqx_mgmt_cli:modules(["load", "emqx_mod_presence"]), - "Module emqx_mod_presence loaded successfully.\n"), - ?assertEqual(emqx_mgmt_cli:modules(["unload", "emqx_mod_presence"]), - "Module emqx_mod_presence unloaded successfully.\n"), - unmock_print(). - t_cli(_) -> mock_print(), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:status([""]), "status")), diff --git a/lib-ce/emqx_management/test/emqx_mgmt_api_SUITE.erl b/lib-ce/emqx_management/test/emqx_mgmt_api_SUITE.erl index fa063d73a..401da3009 100644 --- a/lib-ce/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/lib-ce/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -48,7 +48,6 @@ groups() -> , metrics , nodes , plugins - , modules , acl_cache , pubsub , routes_and_subscriptions @@ -58,13 +57,13 @@ groups() -> }]. init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_auth_mnesia]), + emqx_ct_helpers:start_apps([emqx_management, emqx_auth_mnesia]), ekka_mnesia:start(), emqx_mgmt_auth:mnesia(boot), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_auth_mnesia, emqx_management, emqx_modules]), + emqx_ct_helpers:stop_apps([emqx_auth_mnesia, emqx_management]), ekka_mnesia:ensure_stopped(). init_per_testcase(data, Config) -> @@ -382,64 +381,6 @@ plugins(_) -> auth_header_()), ?assertEqual(<<"not_started">>, get(<<"message">>, Error2)). -modules(_) -> - emqx_modules:load_module(emqx_mod_presence, false), - timer:sleep(50), - {ok, Modules1} = request_api(get, api_path(["modules"]), auth_header_()), - [Modules11] = filter(get(<<"data">>, Modules1), <<"node">>, atom_to_binary(node(), utf8)), - [Module1] = filter(maps:get(<<"modules">>, Modules11), <<"name">>, <<"emqx_mod_presence">>), - ?assertEqual(<<"emqx_mod_presence">>, maps:get(<<"name">>, Module1)), - ?assertEqual(true, maps:get(<<"active">>, Module1)), - - {ok, _} = request_api(put, - api_path(["modules", - atom_to_list(emqx_mod_presence), - "unload"]), - auth_header_()), - {ok, Error1} = request_api(put, - api_path(["modules", - atom_to_list(emqx_mod_presence), - "unload"]), - auth_header_()), - ?assertEqual(<<"not_started">>, get(<<"message">>, Error1)), - {ok, Modules2} = request_api(get, - api_path(["nodes", atom_to_list(node()), "modules"]), - auth_header_()), - [Module2] = filter(get(<<"data">>, Modules2), <<"name">>, <<"emqx_mod_presence">>), - ?assertEqual(<<"emqx_mod_presence">>, maps:get(<<"name">>, Module2)), - ?assertEqual(false, maps:get(<<"active">>, Module2)), - - {ok, _} = request_api(put, - api_path(["nodes", - atom_to_list(node()), - "modules", - atom_to_list(emqx_mod_presence), - "load"]), - auth_header_()), - {ok, Modules3} = request_api(get, - api_path(["nodes", atom_to_list(node()), "modules"]), - auth_header_()), - [Module3] = filter(get(<<"data">>, Modules3), <<"name">>, <<"emqx_mod_presence">>), - ?assertEqual(<<"emqx_mod_presence">>, maps:get(<<"name">>, Module3)), - ?assertEqual(true, maps:get(<<"active">>, Module3)), - - {ok, _} = request_api(put, - api_path(["nodes", - atom_to_list(node()), - "modules", - atom_to_list(emqx_mod_presence), - "unload"]), - auth_header_()), - {ok, Error2} = request_api(put, - api_path(["nodes", - atom_to_list(node()), - "modules", - atom_to_list(emqx_mod_presence), - "unload"]), - auth_header_()), - ?assertEqual(<<"not_started">>, get(<<"message">>, Error2)), - emqx_modules:unload(emqx_mod_presence). - acl_cache(_) -> ClientId = <<"client1">>, Topic = <<"mytopic">>, diff --git a/lib-ce/emqx_modules/src/emqx_modules.erl b/lib-ce/emqx_modules/src/emqx_modules.erl index 38cf2ac1b..74a6c1256 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.erl +++ b/lib-ce/emqx_modules/src/emqx_modules.erl @@ -176,7 +176,10 @@ write_loaded(false) -> ok. %%-------------------------------------------------------------------- %% @doc Modules Command cli(["list"]) -> - foreach(fun(Module) -> print({module, Module}) end, emqx_modules:list()); + lists:foreach(fun({Name, Active}) -> + emqx_ctl:print("Module(~s, description=~s, active=~s)~n", + [Name, Name:description(), Active]) + end, emqx_modules:list()); cli(["load", Name]) -> case emqx_modules:load(list_to_atom(Name)) of diff --git a/lib-ce/emqx_modules/src/emqx_modules_api.erl b/lib-ce/emqx_modules/src/emqx_modules_api.erl index fae6d0e5e..6b8a67406 100644 --- a/lib-ce/emqx_modules/src/emqx_modules_api.erl +++ b/lib-ce/emqx_modules/src/emqx_modules_api.erl @@ -72,6 +72,10 @@ , reload/2 ]). +-export([ do_load_module/2 + , do_unload_module/2 + ]). + list(#{node := Node}, _Params) -> return({ok, [format(Module) || Module <- list_modules(Node)]}); @@ -79,10 +83,10 @@ list(_Bindings, _Params) -> return({ok, [format(Node, Modules) || {Node, Modules} <- list_modules()]}). load(#{node := Node, module := Module}, _Params) -> - return(load_module(Node, Module)); + return(do_load_module(Node, Module)); load(#{module := Module}, _Params) -> - Results = [load_module(Node, Module) || {Node, _Info} <- list_nodes()], + Results = [do_load_module(Node, Module) || {Node, _Info} <- emqx_mgmt:list_nodes()], case lists:filter(fun(Item) -> Item =/= ok end, Results) of [] -> return(ok); @@ -91,10 +95,10 @@ load(#{module := Module}, _Params) -> end. unload(#{node := Node, module := Module}, _Params) -> - return(unload_module(Node, Module)); + return(do_unload_module(Node, Module)); unload(#{module := Module}, _Params) -> - Results = [unload_module(Node, Module) || {Node, _Info} <- list_nodes()], + Results = [do_unload_module(Node, Module) || {Node, _Info} <- emqx_mgmt:list_nodes()], case lists:filter(fun(Item) -> Item =/= ok end, Results) of [] -> return(ok); @@ -109,7 +113,7 @@ reload(#{node := Node, module := Module}, _Params) -> end; reload(#{module := Module}, _Params) -> - Results = [reload_module(Node, Module) || {Node, _Info} <- list_nodes()], + Results = [reload_module(Node, Module) || {Node, _Info} <- emqx_mgmt:list_nodes()], case lists:filter(fun(Item) -> Item =/= ok end, Results) of [] -> return(ok); @@ -137,15 +141,15 @@ list_modules(Node) when Node =:= node() -> list_modules(Node) -> rpc_call(Node, list_modules, [Node]). -load_module(Node, Module) when Node =:= node() -> +do_load_module(Node, Module) when Node =:= node() -> emqx_modules:load(Module); -load_module(Node, Module) -> - rpc_call(Node, load_module, [Node, Module]). +do_load_module(Node, Module) -> + rpc_call(Node, do_load_module, [Node, Module]). -unload_module(Node, Module) when Node =:= node() -> +do_unload_module(Node, Module) when Node =:= node() -> emqx_modules:unload(Module); -unload_module(Node, Module) -> - rpc_call(Node, unload_module, [Node, Module]). +do_unload_module(Node, Module) -> + rpc_call(Node, do_unload_module, [Node, Module]). reload_module(Node, Module) when Node =:= node() -> emqx_modules:reload(Module); diff --git a/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl b/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl index b0ff65758..cc4ea6662 100644 --- a/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl @@ -21,10 +21,19 @@ -include_lib("eunit/include/eunit.hrl"). +-define(CONTENT_TYPE, "application/x-www-form-urlencoded"). + +-define(HOST, "http://127.0.0.1:8081/"). + +-define(API_VERSION, "v4"). + +-define(BASE_PATH, "api"). + all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_modules], fun set_sepecial_cfg/1), + emqx_ct_helpers:start_apps([emqx_management, emqx_modules], fun set_sepecial_cfg/1), + emqx_ct_http:create_default_app(), Config. set_sepecial_cfg(_) -> @@ -32,7 +41,8 @@ set_sepecial_cfg(_) -> ok. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_modules]). + emqx_ct_http:delete_default_app(), + emqx_ct_helpers:stop_apps([emqx_modules, emqx_management]). t_load(_) -> ?assertEqual(ok, emqx_modules:unload()), @@ -45,3 +55,136 @@ t_load(_) -> t_list(_) -> ?assertMatch([{_, _} | _ ], emqx_modules:list()). +t_modules_api(_) -> + emqx_modules:load_module(emqx_mod_presence, false), + timer:sleep(50), + {ok, Modules1} = request_api(get, api_path(["modules"]), auth_header_()), + [Modules11] = filter(get(<<"data">>, Modules1), <<"node">>, atom_to_binary(node(), utf8)), + [Module1] = filter(maps:get(<<"modules">>, Modules11), <<"name">>, <<"emqx_mod_presence">>), + ?assertEqual(<<"emqx_mod_presence">>, maps:get(<<"name">>, Module1)), + ?assertEqual(true, maps:get(<<"active">>, Module1)), + + {ok, _} = request_api(put, + api_path(["modules", + atom_to_list(emqx_mod_presence), + "unload"]), + auth_header_()), + {ok, Error1} = request_api(put, + api_path(["modules", + atom_to_list(emqx_mod_presence), + "unload"]), + auth_header_()), + ?assertEqual(<<"not_started">>, get(<<"message">>, Error1)), + {ok, Modules2} = request_api(get, + api_path(["nodes", atom_to_list(node()), "modules"]), + auth_header_()), + [Module2] = filter(get(<<"data">>, Modules2), <<"name">>, <<"emqx_mod_presence">>), + ?assertEqual(<<"emqx_mod_presence">>, maps:get(<<"name">>, Module2)), + ?assertEqual(false, maps:get(<<"active">>, Module2)), + + {ok, _} = request_api(put, + api_path(["nodes", + atom_to_list(node()), + "modules", + atom_to_list(emqx_mod_presence), + "load"]), + auth_header_()), + {ok, Modules3} = request_api(get, + api_path(["nodes", atom_to_list(node()), "modules"]), + auth_header_()), + [Module3] = filter(get(<<"data">>, Modules3), <<"name">>, <<"emqx_mod_presence">>), + ?assertEqual(<<"emqx_mod_presence">>, maps:get(<<"name">>, Module3)), + ?assertEqual(true, maps:get(<<"active">>, Module3)), + + {ok, _} = request_api(put, + api_path(["nodes", + atom_to_list(node()), + "modules", + atom_to_list(emqx_mod_presence), + "unload"]), + auth_header_()), + {ok, Error2} = request_api(put, + api_path(["nodes", + atom_to_list(node()), + "modules", + atom_to_list(emqx_mod_presence), + "unload"]), + auth_header_()), + ?assertEqual(<<"not_started">>, get(<<"message">>, Error2)), + emqx_modules:unload(emqx_mod_presence). + + +t_modules_cmd(_) -> + mock_print(), + meck:new(emqx_modules, [non_strict, passthrough]), + meck:expect(emqx_modules, load, fun(_) -> ok end), + meck:expect(emqx_modules, unload, fun(_) -> ok end), + meck:expect(emqx_modules, reload, fun(_) -> ok end), + ?assertEqual(emqx_modules:cli(["list"]), ok), + ?assertEqual(emqx_modules:cli(["load", "emqx_mod_presence"]), + "Module emqx_mod_presence loaded successfully.\n"), + ?assertEqual(emqx_modules:cli(["unload", "emqx_mod_presence"]), + "Module emqx_mod_presence unloaded successfully.\n"), + unmock_print(). + +mock_print() -> + catch meck:unload(emqx_ctl), + meck:new(emqx_ctl, [non_strict, passthrough]), + meck:expect(emqx_ctl, print, fun(Arg) -> emqx_ctl:format(Arg) end), + meck:expect(emqx_ctl, print, fun(Msg, Arg) -> emqx_ctl:format(Msg, Arg) end), + meck:expect(emqx_ctl, usage, fun(Usages) -> emqx_ctl:format_usage(Usages) end), + meck:expect(emqx_ctl, usage, fun(Cmd, Descr) -> emqx_ctl:format_usage(Cmd, Descr) end). + +unmock_print() -> + meck:unload(emqx_ctl). + +get(Key, ResponseBody) -> + maps:get(Key, jiffy:decode(list_to_binary(ResponseBody), [return_maps])). + +request_api(Method, Url, Auth) -> + request_api(Method, Url, [], Auth, []). + +request_api(Method, Url, QueryParams, Auth) -> + request_api(Method, Url, QueryParams, Auth, []). + +request_api(Method, Url, QueryParams, Auth, []) -> + NewUrl = case QueryParams of + "" -> Url; + _ -> Url ++ "?" ++ QueryParams + end, + do_request_api(Method, {NewUrl, [Auth]}); +request_api(Method, Url, QueryParams, Auth, Body) -> + NewUrl = case QueryParams of + "" -> Url; + _ -> Url ++ "?" ++ QueryParams + end, + do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + +do_request_api(Method, Request)-> + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], []) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {ok, {{"HTTP/1.1", Code, _}, _, Return} } + when Code =:= 200 orelse Code =:= 201 -> + {ok, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +auth_header_() -> + AppId = <<"admin">>, + AppSecret = <<"public">>, + auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)). + +auth_header_(User, Pass) -> + Encoded = base64:encode_to_string(lists:append([User,":",Pass])), + {"Authorization","Basic " ++ Encoded}. + +api_path(Parts)-> + ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts). + +filter(List, Key, Value) -> + lists:filter(fun(Item) -> + maps:get(Key, Item) == Value + end, List). From baa9fd8255e4e2a70a3f76cd6afae43e4e11650c Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 25 Feb 2021 17:50:11 +0800 Subject: [PATCH 05/52] chore(emqx_management): move emqx management to apps --- {lib-ce => apps}/emqx_management/.gitignore | 0 {lib-ce => apps}/emqx_management/README.md | 0 {lib-ce => apps}/emqx_management/etc/emqx_management.conf | 0 {lib-ce => apps}/emqx_management/include/emqx_mgmt.hrl | 0 {lib-ce => apps}/emqx_management/priv/emqx_management.schema | 0 {lib-ce => apps}/emqx_management/rebar.config | 0 {lib-ce => apps}/emqx_management/src/emqx_management.app.src | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_api.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_alarms.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_apps.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_banned.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_brokers.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_clients.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_data.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_listeners.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_metrics.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_nodes.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_plugins.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_pubsub.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_routes.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_stats.erl | 0 .../emqx_management/src/emqx_mgmt_api_subscriptions.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_app.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_auth.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_cli.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_data_backup.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_http.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_sup.erl | 0 {lib-ce => apps}/emqx_management/src/emqx_mgmt_util.erl | 0 {lib-ce => apps}/emqx_management/test/emqx_mgmt_SUITE.erl | 0 {lib-ce => apps}/emqx_management/test/emqx_mgmt_api_SUITE.erl | 0 {lib-ce => apps}/emqx_management/test/etc/emqx_management.conf | 0 {lib-ce => apps}/emqx_management/test/etc/emqx_reloader.conf | 0 {lib-ce => apps}/emqx_management/test/rfc6455_client.erl | 0 35 files changed, 0 insertions(+), 0 deletions(-) rename {lib-ce => apps}/emqx_management/.gitignore (100%) rename {lib-ce => apps}/emqx_management/README.md (100%) rename {lib-ce => apps}/emqx_management/etc/emqx_management.conf (100%) rename {lib-ce => apps}/emqx_management/include/emqx_mgmt.hrl (100%) rename {lib-ce => apps}/emqx_management/priv/emqx_management.schema (100%) rename {lib-ce => apps}/emqx_management/rebar.config (100%) rename {lib-ce => apps}/emqx_management/src/emqx_management.app.src (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_api.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_alarms.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_apps.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_banned.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_brokers.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_clients.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_data.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_listeners.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_metrics.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_nodes.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_plugins.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_pubsub.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_routes.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_stats.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_api_subscriptions.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_app.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_auth.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_cli.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_data_backup.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_http.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_sup.erl (100%) rename {lib-ce => apps}/emqx_management/src/emqx_mgmt_util.erl (100%) rename {lib-ce => apps}/emqx_management/test/emqx_mgmt_SUITE.erl (100%) rename {lib-ce => apps}/emqx_management/test/emqx_mgmt_api_SUITE.erl (100%) rename {lib-ce => apps}/emqx_management/test/etc/emqx_management.conf (100%) rename {lib-ce => apps}/emqx_management/test/etc/emqx_reloader.conf (100%) rename {lib-ce => apps}/emqx_management/test/rfc6455_client.erl (100%) diff --git a/lib-ce/emqx_management/.gitignore b/apps/emqx_management/.gitignore similarity index 100% rename from lib-ce/emqx_management/.gitignore rename to apps/emqx_management/.gitignore diff --git a/lib-ce/emqx_management/README.md b/apps/emqx_management/README.md similarity index 100% rename from lib-ce/emqx_management/README.md rename to apps/emqx_management/README.md diff --git a/lib-ce/emqx_management/etc/emqx_management.conf b/apps/emqx_management/etc/emqx_management.conf similarity index 100% rename from lib-ce/emqx_management/etc/emqx_management.conf rename to apps/emqx_management/etc/emqx_management.conf diff --git a/lib-ce/emqx_management/include/emqx_mgmt.hrl b/apps/emqx_management/include/emqx_mgmt.hrl similarity index 100% rename from lib-ce/emqx_management/include/emqx_mgmt.hrl rename to apps/emqx_management/include/emqx_mgmt.hrl diff --git a/lib-ce/emqx_management/priv/emqx_management.schema b/apps/emqx_management/priv/emqx_management.schema similarity index 100% rename from lib-ce/emqx_management/priv/emqx_management.schema rename to apps/emqx_management/priv/emqx_management.schema diff --git a/lib-ce/emqx_management/rebar.config b/apps/emqx_management/rebar.config similarity index 100% rename from lib-ce/emqx_management/rebar.config rename to apps/emqx_management/rebar.config diff --git a/lib-ce/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src similarity index 100% rename from lib-ce/emqx_management/src/emqx_management.app.src rename to apps/emqx_management/src/emqx_management.app.src diff --git a/lib-ce/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt.erl rename to apps/emqx_management/src/emqx_mgmt.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_api.erl rename to apps/emqx_management/src/emqx_mgmt_api.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_alarms.erl b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_api_alarms.erl rename to apps/emqx_management/src/emqx_mgmt_api_alarms.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_apps.erl b/apps/emqx_management/src/emqx_mgmt_api_apps.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_api_apps.erl rename to apps/emqx_management/src/emqx_mgmt_api_apps.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_api_banned.erl rename to apps/emqx_management/src/emqx_mgmt_api_banned.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_brokers.erl b/apps/emqx_management/src/emqx_mgmt_api_brokers.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_api_brokers.erl rename to apps/emqx_management/src/emqx_mgmt_api_brokers.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_api_clients.erl rename to apps/emqx_management/src/emqx_mgmt_api_clients.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_data.erl b/apps/emqx_management/src/emqx_mgmt_api_data.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_api_data.erl rename to apps/emqx_management/src/emqx_mgmt_api_data.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_api_listeners.erl rename to apps/emqx_management/src/emqx_mgmt_api_listeners.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_metrics.erl b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_api_metrics.erl rename to apps/emqx_management/src/emqx_mgmt_api_metrics.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_api_nodes.erl rename to apps/emqx_management/src/emqx_mgmt_api_nodes.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_api_plugins.erl rename to apps/emqx_management/src/emqx_mgmt_api_plugins.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_pubsub.erl b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_api_pubsub.erl rename to apps/emqx_management/src/emqx_mgmt_api_pubsub.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_routes.erl b/apps/emqx_management/src/emqx_mgmt_api_routes.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_api_routes.erl rename to apps/emqx_management/src/emqx_mgmt_api_routes.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_stats.erl b/apps/emqx_management/src/emqx_mgmt_api_stats.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_api_stats.erl rename to apps/emqx_management/src/emqx_mgmt_api_stats.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_api_subscriptions.erl rename to apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_app.erl b/apps/emqx_management/src/emqx_mgmt_app.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_app.erl rename to apps/emqx_management/src/emqx_mgmt_app.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_auth.erl rename to apps/emqx_management/src/emqx_mgmt_auth.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_cli.erl rename to apps/emqx_management/src/emqx_mgmt_cli.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_data_backup.erl rename to apps/emqx_management/src/emqx_mgmt_data_backup.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_http.erl b/apps/emqx_management/src/emqx_mgmt_http.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_http.erl rename to apps/emqx_management/src/emqx_mgmt_http.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_sup.erl b/apps/emqx_management/src/emqx_mgmt_sup.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_sup.erl rename to apps/emqx_management/src/emqx_mgmt_sup.erl diff --git a/lib-ce/emqx_management/src/emqx_mgmt_util.erl b/apps/emqx_management/src/emqx_mgmt_util.erl similarity index 100% rename from lib-ce/emqx_management/src/emqx_mgmt_util.erl rename to apps/emqx_management/src/emqx_mgmt_util.erl diff --git a/lib-ce/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl similarity index 100% rename from lib-ce/emqx_management/test/emqx_mgmt_SUITE.erl rename to apps/emqx_management/test/emqx_mgmt_SUITE.erl diff --git a/lib-ce/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl similarity index 100% rename from lib-ce/emqx_management/test/emqx_mgmt_api_SUITE.erl rename to apps/emqx_management/test/emqx_mgmt_api_SUITE.erl diff --git a/lib-ce/emqx_management/test/etc/emqx_management.conf b/apps/emqx_management/test/etc/emqx_management.conf similarity index 100% rename from lib-ce/emqx_management/test/etc/emqx_management.conf rename to apps/emqx_management/test/etc/emqx_management.conf diff --git a/lib-ce/emqx_management/test/etc/emqx_reloader.conf b/apps/emqx_management/test/etc/emqx_reloader.conf similarity index 100% rename from lib-ce/emqx_management/test/etc/emqx_reloader.conf rename to apps/emqx_management/test/etc/emqx_reloader.conf diff --git a/lib-ce/emqx_management/test/rfc6455_client.erl b/apps/emqx_management/test/rfc6455_client.erl similarity index 100% rename from lib-ce/emqx_management/test/rfc6455_client.erl rename to apps/emqx_management/test/rfc6455_client.erl From 28653bb457c8fb9ce108d04d9660a80890caf5f5 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 25 Feb 2021 19:06:40 +0800 Subject: [PATCH 06/52] chore(emqx_management): format code --- .../src/emqx_mgmt_api_data.erl | 4 +- apps/emqx_management/src/emqx_mgmt_cli.erl | 28 ++- .../src/emqx_mgmt_data_backup.erl | 212 +++++++----------- .../test/emqx_mgmt_api_SUITE.erl | 12 +- .../src/emqx_mod_api_topic_metrics.erl | 8 +- 5 files changed, 107 insertions(+), 157 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_data.erl b/apps/emqx_management/src/emqx_mgmt_api_data.erl index 18ba27265..5c19e95af 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_data.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_data.erl @@ -80,7 +80,7 @@ export(_Bindings, _Params) -> return({ok, File#{filename => filename:basename(Filename)}}); Return -> return(Return) end. - + list_exported(_Bindings, _Params) -> List = [ rpc:call(Node, ?MODULE, get_list_exported, []) || Node <- ekka_mnesia:running_nodes() ], NList = lists:map(fun({_, FileInfo}) -> FileInfo end, lists:keysort(1, lists:append(List))), @@ -131,7 +131,7 @@ import(_Bindings, Params) -> do_import(Filename) -> FullFilename = filename:join([emqx:get_env(data_dir), Filename]), - emqx_mgmt_data_backup:import(FullFilename). + emqx_mgmt_data_backup:import(FullFilename). download(#{filename := Filename}, _Params) -> FullFilename = filename:join([emqx:get_env(data_dir), Filename]), diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 19cf83f99..fc18bc1bf 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -23,8 +23,6 @@ -define(PRINT_CMD(Cmd, Descr), io:format("~-48s# ~s~n", [Cmd, Descr])). --import(lists, [foreach/2]). - -export([load/0]). -export([ status/1 @@ -58,7 +56,7 @@ -spec(load() -> ok). load() -> Cmds = [Fun || {Fun, _} <- ?MODULE:module_info(exports), is_cmd(Fun)], - foreach(fun(Cmd) -> emqx_ctl:register_command(Cmd, {?MODULE, Cmd}, []) end, Cmds). + lists:foreach(fun(Cmd) -> emqx_ctl:register_command(Cmd, {?MODULE, Cmd}, []) end, Cmds). is_cmd(Fun) -> not lists:member(Fun, [init, load, module_info]). @@ -100,7 +98,7 @@ mgmt(["delete", AppId]) -> end; mgmt(["list"]) -> - foreach(fun({AppId, AppSecret, Name, Desc, Status, Expired}) -> + lists:foreach(fun({AppId, AppSecret, Name, Desc, Status, Expired}) -> emqx_ctl:print("app_id: ~s, secret: ~s, name: ~s, desc: ~s, status: ~s, expired: ~p~n", [AppId, AppSecret, Name, Desc, Status, Expired]) end, emqx_mgmt_auth:list_apps()); @@ -229,7 +227,7 @@ routes(_) -> {"routes show ", "Show a route"}]). subscriptions(["list"]) -> - foreach(fun(Suboption) -> + lists:foreach(fun(Suboption) -> print({emqx_suboption, Suboption}) end, ets:tab2list(emqx_suboption)); @@ -279,7 +277,7 @@ if_valid_qos(QoS, Fun) -> end. plugins(["list"]) -> - foreach(fun print/1, emqx_plugins:list()); + lists:foreach(fun print/1, emqx_plugins:list()); plugins(["load", Name]) -> case emqx_plugins:load(list_to_atom(Name)) of @@ -421,7 +419,7 @@ log(_) -> %% @doc Trace Command trace(["list"]) -> - foreach(fun({{Who, Name}, {Level, LogFile}}) -> + lists:foreach(fun({{Who, Name}, {Level, LogFile}}) -> emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Who, Name, Level, LogFile]) end, emqx_tracer:lookup_traces()); @@ -470,7 +468,7 @@ trace_off(Who, Name) -> %% @doc Listeners Command listeners([]) -> - foreach(fun({{Protocol, ListenOn}, _Pid}) -> + lists:foreach(fun({{Protocol, ListenOn}, _Pid}) -> Info = [{listen_on, {string, emqx_listeners:format_listen_on(ListenOn)}}, {acceptors, esockd:get_acceptors({Protocol, ListenOn})}, {max_conns, esockd:get_max_connections({Protocol, ListenOn})}, @@ -478,9 +476,9 @@ listeners([]) -> {shutdown_count, esockd:get_shutdown_count({Protocol, ListenOn})} ], emqx_ctl:print("~s~n", [listener_identifier(Protocol, ListenOn)]), - foreach(fun indent_print/1, Info) + lists:foreach(fun indent_print/1, Info) end, esockd:listeners()), - foreach(fun({Protocol, Opts}) -> + lists:foreach(fun({Protocol, Opts}) -> Port = proplists:get_value(port, Opts), Info = [{listen_on, {string, emqx_listeners:format_listen_on(Port)}}, {acceptors, maps:get(num_acceptors, proplists:get_value(transport_options, Opts, #{}), 0)}, @@ -488,7 +486,7 @@ listeners([]) -> {current_conn, proplists:get_value(all_connections, Opts)}, {shutdown_count, []}], emqx_ctl:print("~s~n", [listener_identifier(Protocol, Port)]), - foreach(fun indent_print/1, Info) + lists:foreach(fun indent_print/1, Info) end, ranch:info()); listeners(["stop", Name = "http" ++ _N | _MaybePort]) -> @@ -548,16 +546,16 @@ data(["export"]) -> emqx_ctl:print("The emqx data has been successfully exported to ~s.~n", [Filename]); {error, Reason} -> emqx_ctl:print("The emqx data export failed due to ~p.~n", [Reason]) - end; + end; data(["import", Filename]) -> case emqx_mgmt_data_backup:import(Filename) of ok -> emqx_ctl:print("The emqx data has been imported successfully.~n"); - {error, import_failed} -> - emqx_ctl:print("The emqx data import failed. ~n"); + {error, import_failed} -> + emqx_ctl:print("The emqx data import failed.~n"); {error, unsupported_version} -> - emqx_ctl:print("The emqx data import failed: Unsupported version. ~n"); + emqx_ctl:print("The emqx data import failed: Unsupported version.~n"); {error, Reason} -> emqx_ctl:print("The emqx data import failed: ~0p while reading ~s.~n", [Reason, Filename]) end; diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 03703be91..ef8987603 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -51,7 +51,7 @@ , to_version/1 ]). --export([ export/0 +-export([ export/0 , import/1 ]). @@ -323,7 +323,7 @@ import_auth_clientid(Lists) -> case ets:info(emqx_user) of undefined -> ok; _ -> - lists:foreach(fun(#{<<"clientid">> := Clientid, <<"password">> := Password}) -> + lists:foreach(fun(#{<<"clientid">> := Clientid, <<"password">> := Password}) -> mnesia:dirty_write({emqx_user, {clientid, Clientid}, base64:decode(Password), erlang:system_time(millisecond)}) end, Lists) end. @@ -333,7 +333,7 @@ import_auth_username(Lists) -> undefined -> ok; _ -> lists:foreach(fun(#{<<"username">> := Username, <<"password">> := Password}) -> - mnesia:dirty_write({emqx_user, {username, Username}, base64:decode(Password), erlang:system_time(millisecond)}) + mnesia:dirty_write({emqx_user, {username, Username}, base64:decode(Password), erlang:system_time(millisecond)}) end, Lists) end. @@ -344,8 +344,8 @@ import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse _ -> CreatedAt = erlang:system_time(millisecond), lists:foreach(fun(#{<<"login">> := Login, - <<"password">> := Password}) -> - mnesia:dirty_write({emqx_user, {username, Login}, base64:decode(Password), CreatedAt}) + <<"password">> := Password}) -> + mnesia:dirty_write({emqx_user, {username, Login}, base64:decode(Password), CreatedAt}) end, Auths) end; @@ -356,7 +356,7 @@ import_auth_mnesia(Auths, _) -> lists:foreach(fun(#{<<"login">> := Login, <<"type">> := Type, <<"password">> := Password, - <<"created_at">> := CreatedAt }) -> + <<"created_at">> := CreatedAt }) -> mnesia:dirty_write({emqx_user, {any_to_atom(Type), Login}, base64:decode(Password), CreatedAt}) end, Auths) end. @@ -375,7 +375,7 @@ import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse true -> allow; false -> deny end, - mnesia:dirty_write({emqx_acl, {{username, Login}, Topic}, any_to_atom(Action), Allow1, CreatedAt}) + mnesia:dirty_write({emqx_acl, {{username, Login}, Topic}, any_to_atom(Action), Allow1, CreatedAt}) end, Acls) end; @@ -385,14 +385,14 @@ import_acl_mnesia(Acls, _) -> _ -> lists:foreach(fun(Map = #{<<"action">> := Action, <<"access">> := Access, - <<"created_at">> := CreatedAt}) -> + <<"created_at">> := CreatedAt}) -> Filter = case maps:get(<<"type_value">>, Map, undefined) of undefined -> {any_to_atom(maps:get(<<"type">>, Map)), maps:get(<<"topic">>, Map)}; Value -> {{any_to_atom(maps:get(<<"type">>, Map)), Value}, maps:get(<<"topic">>, Map)} end, - mnesia:dirty_write({emqx_acl ,Filter, any_to_atom(Action), any_to_atom(Access), CreatedAt}) + mnesia:dirty_write({emqx_acl ,Filter, any_to_atom(Action), any_to_atom(Access), CreatedAt}) end, Acls) end. @@ -447,135 +447,19 @@ to_version(Version) when is_binary(Version) -> to_version(Version) when is_list(Version) -> Version. --ifdef(EMQX_ENTERPRISE). export() -> - Modules = export_modules(), - Rules = export_rules(), - Resources = export_resources(), - Blacklist = export_blacklist(), - Apps = export_applications(), - Users = export_users(), - AuthMnesia = export_auth_mnesia(), - AclMnesia = export_acl_mnesia(), - Schemas = export_schemas(), - {Configs, State} = export_confs(), Seconds = erlang:system_time(second), + Data = do_export_data() ++ [{date, erlang:list_to_binary(emqx_mgmt_util:strftime(Seconds))}], {{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds), Filename = io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]), NFilename = filename:join([emqx:get_env(data_dir), Filename]), - Version = string:sub_string(emqx_sys:version(), 1, 3), - Data = [{version, erlang:list_to_binary(Version)}, - {date, erlang:list_to_binary(emqx_mgmt_util:strftime(Seconds))}, - {modules, Modules}, - {rules, Rules}, - {resources, Resources}, - {blacklist, Blacklist}, - {apps, Apps}, - {users, Users}, - {auth_mnesia, AuthMnesia}, - {acl_mnesia, AclMnesia}, - {schemas, Schemas}, - {configs, Configs}, - {listeners_state, State}], - write_file(NFilename, Data). - -import(Filename) -> - case file:read_file(FullFilename) of - {ok, Json} -> - Data = emqx_json:decode(Json, [return_maps]), - Version = to_version(maps:get(<<"version">>, Data)), - case lists:member(Version, ?VERSIONS) of - true -> - try - import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])), - import_resources(maps:get(<<"resources">>, Data, [])), - import_rules(maps:get(<<"rules">>, Data, [])), - import_blacklist(maps:get(<<"blacklist">>, Data, [])), - import_applications(maps:get(<<"apps">>, Data, [])), - import_users(maps:get(<<"users">>, Data, [])), - import_modules(maps:get(<<"modules">>, Data, [])), - import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), - import_auth_username(maps:get(<<"auth_username">>, Data, [])), - import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version), - import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version), - import_schemas(maps:get(<<"schemas">>, Data, [])), - logger:debug("The emqx data has been imported successfully"), - ok - catch Class:Reason:Stack -> - logger:error("The emqx data import failed: ~0p", [{Class,Reason,Stack}]), - {error, import_failed} - end; - false -> - logger:error("Unsupported version: ~p", [Version]), - {error, unsupported_version} - end; - {error, Reason} -> - {error, Reason} - end. - --else. -export() -> - Rules = export_rules(), - Resources = export_resources(), - Blacklist = export_blacklist(), - Apps = export_applications(), - Users = export_users(), - AuthMnesia = export_auth_mnesia(), - AclMnesia = export_acl_mnesia(), - Seconds = erlang:system_time(second), - {{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds), - Filename = io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]), - NFilename = filename:join([emqx:get_env(data_dir), Filename]), - Version = string:sub_string(emqx_sys:version(), 1, 3), - Data = [{version, erlang:list_to_binary(Version)}, - {date, erlang:list_to_binary(emqx_mgmt_util:strftime(Seconds))}, - {rules, Rules}, - {resources, Resources}, - {blacklist, Blacklist}, - {apps, Apps}, - {users, Users}, - {auth_mnesia, AuthMnesia}, - {acl_mnesia, AclMnesia}], - write_file(NFilename, Data). - -import(Filename) -> - case file:read_file(Filename) of - {ok, Json} -> - Data = emqx_json:decode(Json, [return_maps]), - Version = to_version(maps:get(<<"version">>, Data)), - case lists:member(Version, ?VERSIONS) of - true -> - try - import_resources_and_rules(maps:get(<<"resources">>, Data, []), maps:get(<<"rules">>, Data, []), Version), - import_blacklist(maps:get(<<"blacklist">>, Data, [])), - import_applications(maps:get(<<"apps">>, Data, [])), - import_users(maps:get(<<"users">>, Data, [])), - import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), - import_auth_username(maps:get(<<"auth_username">>, Data, [])), - import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version), - import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version), - logger:debug("The emqx data has been imported successfully"), - ok - catch Class:Reason:Stack -> - logger:error("The emqx data import failed: ~0p", [{Class,Reason,Stack}]), - {error, import_failed} - end; - false -> - logger:error("Unsupported version: ~p", [Version]), - {error, unsupported_version} - end; - Error -> Error - end. --endif. - -write_file(Filename, Data) -> - ok = filelib:ensure_dir(Filename), - case file:write_file(Filename, emqx_json:encode(Data)) of + ok = filelib:ensure_dir(NFilename), + case file:write_file(NFilename, emqx_json:encode(Data)) of ok -> - case file:read_file_info(Filename) of + case file:read_file_info(NFilename) of {ok, #file_info{size = Size, ctime = {{Y, M, D}, {H, MM, S}}}} -> CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y, M, D, H, MM, S]), - {ok, #{filename => list_to_binary(Filename), + {ok, #{filename => list_to_binary(NFilename), size => Size, created_at => list_to_binary(CreatedAt), node => node() @@ -584,3 +468,71 @@ write_file(Filename, Data) -> end; Error -> Error end. + +do_export_data() -> + Version = string:sub_string(emqx_sys:version(), 1, 3), + [{version, erlang:list_to_binary(Version)}, + {rules, export_rules()}, + {resources, export_resources()}, + {blacklist, export_blacklist()}, + {apps, export_applications()}, + {users, export_users()}, + {auth_mnesia, export_auth_mnesia()}, + {acl_mnesia, export_acl_mnesia()} + ] ++ do_export_extra_data(). + +-ifdef(EMQX_ENTERPRISE). +do_export_extra_data() -> + {Configs, State} = export_confs(), + [{modules, export_modules()}, + {schemas, export_schemas()}, + {configs, Configs}, + {listeners_state, State} + ]. +-else. +do_export_extra_data() -> []. +-endif. + +import(Filename) -> + case file:read_file(Filename) of + {ok, Json} -> + Data = emqx_json:decode(Json, [return_maps]), + Version = to_version(maps:get(<<"version">>, Data)), + case lists:member(Version, ?VERSIONS) of + true -> + try + do_import_data(Data, Version), + logger:debug("The emqx data has been imported successfully"), + ok + catch Class:Reason:Stack -> + logger:error("The emqx data import failed: ~0p", [{Class,Reason,Stack}]), + {error, import_failed} + end; + false -> + logger:error("Unsupported version: ~p", [Version]), + {error, unsupported_version} + end; + Error -> Error + end. + +do_import_data(Data, Version) -> + import_blacklist(maps:get(<<"blacklist">>, Data, [])), + import_applications(maps:get(<<"apps">>, Data, [])), + import_users(maps:get(<<"users">>, Data, [])), + import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), + import_auth_username(maps:get(<<"auth_username">>, Data, [])), + import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version), + import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version), + do_import_extra_data(Data, Version). + +-ifdef(EMQX_ENTERPRISE). +do_import_extra_data(Data, Version) -> + import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])), + import_resources(maps:get(<<"resources">>, Data, [])), + import_rules(maps:get(<<"rules">>, Data, [])), + import_modules(maps:get(<<"modules">>, Data, [])), + import_schemas(maps:get(<<"schemas">>, Data, [])). +-else. +do_import_extra_data(Data, Version) -> + import_resources_and_rules(maps:get(<<"resources">>, Data, []), maps:get(<<"rules">>, Data, []), Version). +-endif. diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index 401da3009..8e98567c4 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -101,7 +101,7 @@ get(Key, ResponseBody) -> lookup_alarm(Name, [#{<<"name">> := Name} | _More]) -> true; -lookup_alarm(Name, [_Alarm | More]) -> +lookup_alarm(Name, [_Alarm | More]) -> lookup_alarm(Name, More); lookup_alarm(_Name, []) -> false. @@ -119,7 +119,7 @@ alarms(_) -> ?assert(is_existing(alarm1, emqx_alarm:get_alarms(activated))), ?assert(is_existing(alarm2, emqx_alarm:get_alarms(activated))), - + {ok, Return1} = request_api(get, api_path(["alarms/activated"]), auth_header_()), ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return1))))), ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return1))))), @@ -230,7 +230,7 @@ clients(_) -> {ok, Clients2} = request_api(get, api_path(["nodes", atom_to_list(node()), "clients", binary_to_list(ClientId2)]) , auth_header_()), - ?assertEqual(<<"client2">>, maps:get(<<"clientid">>, lists:nth(1, get(<<"data">>, Clients2)))), + ?assertEqual(<<"client2">>, maps:get(<<"clientid">>, lists:nth(1, get(<<"data">>, Clients2)))), {ok, Clients3} = request_api(get, api_path(["clients", "username", binary_to_list(Username1)]), @@ -245,7 +245,7 @@ clients(_) -> {ok, Clients5} = request_api(get, api_path(["clients"]), "_limit=100&_page=1", auth_header_()), ?assertEqual(2, maps:get(<<"count">>, get(<<"meta">>, Clients5))), - + meck:new(emqx_mgmt, [passthrough, no_history]), meck:expect(emqx_mgmt, kickout_client, 1, fun(_) -> {error, undefined} end), @@ -261,7 +261,7 @@ clients(_) -> ?assertEqual(?ERROR1, get(<<"code">>, MeckRet3)), meck:unload(emqx_mgmt), - + {ok, Ok} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), ?assertEqual(?SUCCESS, get(<<"code">>, Ok)), @@ -436,7 +436,7 @@ pubsub(_) -> <<"topics">> => <<"">>, <<"qos">> => 1, <<"payload">> => <<"hello">>}), - ?assertEqual(?ERROR15, get(<<"code">>, BadTopic2)), + ?assertEqual(?ERROR15, get(<<"code">>, BadTopic2)), {ok, BadTopic3} = request_api(post, api_path(["mqtt/unsubscribe"]), [], auth_header_(), #{<<"clientid">> => ClientId, diff --git a/lib-ce/emqx_modules/src/emqx_mod_api_topic_metrics.erl b/lib-ce/emqx_modules/src/emqx_mod_api_topic_metrics.erl index 20416da7f..1588cf0c8 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_api_topic_metrics.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_api_topic_metrics.erl @@ -57,7 +57,7 @@ list(#{topic := Topic0}, _Params) -> execute_when_enabled(fun() -> Topic = emqx_mgmt_util:urldecode(Topic0), case safe_validate(Topic) of - true -> + true -> case get_topic_metrics(Topic) of {error, Reason} -> return({error, Reason}); Metrics -> return({ok, maps:from_list(Metrics)}) @@ -74,7 +74,7 @@ list(_Bindings, _Params) -> Metrics -> return({ok, Metrics}) end end). - + register(_Bindings, Params) -> execute_when_enabled(fun() -> case proplists:get_value(<<"topic">>, Params) of @@ -82,7 +82,7 @@ register(_Bindings, Params) -> return({error, missing_required_params}); Topic -> case safe_validate(Topic) of - true -> + true -> register_topic_metrics(Topic), return(ok); false -> @@ -101,7 +101,7 @@ unregister(#{topic := Topic0}, _Params) -> execute_when_enabled(fun() -> Topic = emqx_mgmt_util:urldecode(Topic0), case safe_validate(Topic) of - true -> + true -> unregister_topic_metrics(Topic), return(ok); false -> From b53bdd1450093c2f9dc301a8ed945459f3d4e058 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 26 Feb 2021 09:34:57 +0800 Subject: [PATCH 07/52] chore(emqx_management): remove emqx_modules dependency on emqx_management --- .../src/emqx_mgmt_data_backup.erl | 22 ++++++++++--------- lib-ce/emqx_modules/src/emqx_modules_api.erl | 6 ++--- lib-ce/emqx_modules/src/emqx_modules_app.erl | 1 + 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index ef8987603..0feb7b7f3 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -28,8 +28,6 @@ , import_schemas/1 , import_confs/2 ]). --else. --export([ import_resources_and_rules/3 ]). -endif. -export([ export_rules/0 @@ -39,6 +37,7 @@ , export_users/0 , export_auth_mnesia/0 , export_acl_mnesia/0 + , import_resources_and_rules/3 , import_rules/1 , import_resources/1 , import_blacklist/1 @@ -245,6 +244,7 @@ import_resources_and_rules(Resources, Rules, FromVersion) NActions = apply_new_config(Actions, Configs), import_rule(Rule#{<<"actions">> := NActions}) end, Rules); + import_resources_and_rules(Resources, Rules, _FromVersion) -> import_resources(Resources), import_rules(Rules). @@ -278,7 +278,10 @@ action_to_prop_list({action_instance, ActionInstId, Name, FallbackActions, Args} {name, Name}, {fallbacks, actions_to_prop_list(FallbackActions)}, {args, Args}]. - +-else. +import_resources_and_rules(Resources, Rules, _FromVersion) -> + import_resources(Resources), + import_rules(Rules). -endif. import_blacklist(Blacklist) -> @@ -516,23 +519,22 @@ import(Filename) -> end. do_import_data(Data, Version) -> + do_import_extra_data(Data, Version), + import_resources_and_rules(maps:get(<<"resources">>, Data, []), maps:get(<<"rules">>, Data, []), Version), import_blacklist(maps:get(<<"blacklist">>, Data, [])), import_applications(maps:get(<<"apps">>, Data, [])), import_users(maps:get(<<"users">>, Data, [])), import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), import_auth_username(maps:get(<<"auth_username">>, Data, [])), import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version), - import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version), - do_import_extra_data(Data, Version). + import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version). -ifdef(EMQX_ENTERPRISE). do_import_extra_data(Data, Version) -> import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])), - import_resources(maps:get(<<"resources">>, Data, [])), - import_rules(maps:get(<<"rules">>, Data, [])), import_modules(maps:get(<<"modules">>, Data, [])), - import_schemas(maps:get(<<"schemas">>, Data, [])). + import_schemas(maps:get(<<"schemas">>, Data, [])), + ok. -else. -do_import_extra_data(Data, Version) -> - import_resources_and_rules(maps:get(<<"resources">>, Data, []), maps:get(<<"rules">>, Data, []), Version). +do_import_extra_data(_Data, _Version) -> ok. -endif. diff --git a/lib-ce/emqx_modules/src/emqx_modules_api.erl b/lib-ce/emqx_modules/src/emqx_modules_api.erl index 6b8a67406..5f4641ade 100644 --- a/lib-ce/emqx_modules/src/emqx_modules_api.erl +++ b/lib-ce/emqx_modules/src/emqx_modules_api.erl @@ -86,7 +86,7 @@ load(#{node := Node, module := Module}, _Params) -> return(do_load_module(Node, Module)); load(#{module := Module}, _Params) -> - Results = [do_load_module(Node, Module) || {Node, _Info} <- emqx_mgmt:list_nodes()], + Results = [do_load_module(Node, Module) || Node <- ekka_mnesia:running_nodes()], case lists:filter(fun(Item) -> Item =/= ok end, Results) of [] -> return(ok); @@ -98,7 +98,7 @@ unload(#{node := Node, module := Module}, _Params) -> return(do_unload_module(Node, Module)); unload(#{module := Module}, _Params) -> - Results = [do_unload_module(Node, Module) || {Node, _Info} <- emqx_mgmt:list_nodes()], + Results = [do_unload_module(Node, Module) || Node <- ekka_mnesia:running_nodes()], case lists:filter(fun(Item) -> Item =/= ok end, Results) of [] -> return(ok); @@ -113,7 +113,7 @@ reload(#{node := Node, module := Module}, _Params) -> end; reload(#{module := Module}, _Params) -> - Results = [reload_module(Node, Module) || {Node, _Info} <- emqx_mgmt:list_nodes()], + Results = [reload_module(Node, Module) || Node <- ekka_mnesia:running_nodes()], case lists:filter(fun(Item) -> Item =/= ok end, Results) of [] -> return(ok); diff --git a/lib-ce/emqx_modules/src/emqx_modules_app.erl b/lib-ce/emqx_modules/src/emqx_modules_app.erl index 34ff51267..6d82177d4 100644 --- a/lib-ce/emqx_modules/src/emqx_modules_app.erl +++ b/lib-ce/emqx_modules/src/emqx_modules_app.erl @@ -34,4 +34,5 @@ start(_Type, _Args) -> {ok, Pid}. stop(_State) -> + emqx_ctl:unregister_command(modules), emqx_modules:unload(). From f548888af9271c7a5389ca21a8c5c48ca04e237e Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Mon, 1 Mar 2021 17:26:52 +0800 Subject: [PATCH 08/52] chore(emqx_management): judge the version for import and export acl mnesia and auth mnesia --- .../src/emqx_mgmt_data_backup.erl | 52 ++++++++++++++----- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 0feb7b7f3..5534abacd 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -215,7 +215,11 @@ import_resource(#{<<"id">> := Id, created_at => NCreatedAt, description => Desc}). --ifndef(EMQX_ENTERPRISE). +-ifdef(EMQX_ENTERPRISE). +import_resources_and_rules(Resources, Rules, _FromVersion) -> + import_resources(Resources), + import_rules(Rules). +-else. import_resources_and_rules(Resources, Rules, FromVersion) when FromVersion =:= "4.0" orelse FromVersion =:= "4.1" orelse FromVersion =:= "4.2" -> Configs = lists:foldl(fun(#{<<"id">> := ID, @@ -278,10 +282,6 @@ action_to_prop_list({action_instance, ActionInstId, Name, FallbackActions, Args} {name, Name}, {fallbacks, actions_to_prop_list(FallbackActions)}, {args, Args}]. --else. -import_resources_and_rules(Resources, Rules, _FromVersion) -> - import_resources(Resources), - import_rules(Rules). -endif. import_blacklist(Blacklist) -> @@ -340,8 +340,37 @@ import_auth_username(Lists) -> end, Lists) end. +-ifdef(EMQX_ENTERPRISE). import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse FromVersion =:= "4.1" -> + do_import_auth_mnesia_by_old_data(Auths); +import_auth_mnesia(Auths, _) -> + do_import_auth_mnesia(Auths). + +import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse + FromVersion =:= "4.1" -> + do_import_acl_mnesia_by_old_data(Acls); + +import_acl_mnesia(Acls, _) -> + do_import_acl_mnesia(Acls). +-else. +import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse + FromVersion =:= "4.1" orelse + FromVersion =:= "4.2" -> + do_import_auth_mnesia_by_old_data(Auths); +import_auth_mnesia(Auths, _) -> + do_import_auth_mnesia(Auths). + +import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse + FromVersion =:= "4.1" orelse + FromVersion =:= "4.2" -> + do_import_acl_mnesia_by_old_data(Acls); + +import_acl_mnesia(Acls, _) -> + do_import_acl_mnesia(Acls). +-endif. + +do_import_auth_mnesia_by_old_data(Auths) -> case ets:info(emqx_user) of undefined -> ok; _ -> @@ -350,9 +379,8 @@ import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse <<"password">> := Password}) -> mnesia:dirty_write({emqx_user, {username, Login}, base64:decode(Password), CreatedAt}) end, Auths) - end; - -import_auth_mnesia(Auths, _) -> + end. +do_import_auth_mnesia(Auths) -> case ets:info(emqx_user) of undefined -> ok; _ -> @@ -364,8 +392,7 @@ import_auth_mnesia(Auths, _) -> end, Auths) end. -import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse - FromVersion =:= "4.1" -> +do_import_acl_mnesia_by_old_data(Acls) -> case ets:info(emqx_acl) of undefined -> ok; _ -> @@ -380,9 +407,8 @@ import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse end, mnesia:dirty_write({emqx_acl, {{username, Login}, Topic}, any_to_atom(Action), Allow1, CreatedAt}) end, Acls) - end; - -import_acl_mnesia(Acls, _) -> + end. +do_import_acl_mnesia(Acls) -> case ets:info(emqx_acl) of undefined -> ok; _ -> From 829a39eade00b94aa6963ca845799978ccaed6a7 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Tue, 2 Mar 2021 11:37:54 +0900 Subject: [PATCH 09/52] fix(emqx_bridge_mqtt_actions): string -> password --- apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl index 7a1c55798..7d1bd23e6 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl @@ -94,7 +94,7 @@ }, password => #{ order => 6, - type => string, + type => password, required => false, default => <<"">>, title => #{en => <<"Password">>, @@ -296,7 +296,7 @@ }, password => #{ order => 6, - type => string, + type => password, required => false, default => <<"">>, title => #{en => <<"Password">>, From 2e199126dc4d7fc086593f5e8135c6560c54f4dc Mon Sep 17 00:00:00 2001 From: z8674558 Date: Tue, 2 Mar 2021 12:43:37 +0900 Subject: [PATCH 10/52] fix(emqx_bridge_mqtt_actions): string -> file --- .../src/emqx_bridge_mqtt_actions.erl | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl index 7d1bd23e6..0dccbd353 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl @@ -370,10 +370,8 @@ }, ssl => #{ order => 11, - type => string, - required => false, - default => <<"off">>, - enum => [<<"on">>, <<"off">>], + type => boolean, + default => false, title => #{en => <<"Bridge SSL">>, zh => <<"Bridge SSL"/utf8>>}, description => #{en => <<"Switch which used to enable ssl connection of the bridge">>, @@ -381,7 +379,7 @@ }, cacertfile => #{ order => 12, - type => string, + type => file, required => false, default => <<"etc/certs/cacert.pem">>, title => #{en => <<"CA certificates">>, @@ -391,7 +389,7 @@ }, certfile => #{ order => 13, - type => string, + type => file, required => false, default => <<"etc/certs/client-cert.pem">>, title => #{en => <<"SSL Certfile">>, @@ -401,7 +399,7 @@ }, keyfile => #{ order => 14, - type => string, + type => file, required => false, default => <<"etc/certs/client-key.pem">>, title => #{en => <<"SSL Keyfile">>, @@ -750,7 +748,7 @@ options(Options, PoolName, ResId) -> {password, str(Get(<<"password">>))}, {proto_ver, mqtt_ver(Get(<<"proto_ver">>))}, {retry_interval, cuttlefish_duration:parse(str(GetD(<<"retry_interval">>, "30s")), s)} - | maybe_ssl(Options, cuttlefish_flag:parse(str(Get(<<"ssl">>))), ResId) + | maybe_ssl(Options, Get(<<"ssl">>), ResId) ] ++ Subscriptions1 end. From 9076af1c06f0b6d1681c5800bd55937090400fa5 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 2 Mar 2021 09:01:10 +0800 Subject: [PATCH 11/52] chore(coap): upgrade gen_coap to v0.3.2 The 0.3.2 fixes some redundant log printing in a DTLS connection --- apps/emqx_coap/rebar.config | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/apps/emqx_coap/rebar.config b/apps/emqx_coap/rebar.config index 0b85b4f18..0f8759b8a 100644 --- a/apps/emqx_coap/rebar.config +++ b/apps/emqx_coap/rebar.config @@ -1,28 +1,4 @@ {deps, [ - {gen_coap, {git, "https://github.com/emqx/gen_coap", {tag, "v0.3.1"}}} - ]}. - -{edoc_opts, [{preprocess, true}]}. -{erl_opts, [warn_unused_vars, - warn_shadow_vars, - warn_unused_import, - warn_obsolete_guard, - debug_info, - {parse_transform}]}. - -{xref_checks, [undefined_function_calls, undefined_functions, - locals_not_used, deprecated_function_calls, - warnings_as_errors, deprecated_functions]}. -{cover_enabled, true}. -{cover_opts, [verbose]}. -{cover_export_enabled, true}. - -{profiles, - [{test, - [{deps, - [{er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}}, - {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}} - ]} - ]} + {gen_coap, {git, "https://github.com/emqx/gen_coap", {tag, "v0.3.2"}}} ]}. From cc373df49a947e1c7f735a1e7767c395f7e2623a Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 2 Mar 2021 09:49:45 +0800 Subject: [PATCH 12/52] fix(etc): remove .d suffix from dir names --- etc/{emqx_cloud.d => emqx_cloud}/vm.args | 0 etc/{emqx_edge.d => emqx_edge}/vm.args | 0 rebar.config.erl | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) rename etc/{emqx_cloud.d => emqx_cloud}/vm.args (100%) rename etc/{emqx_edge.d => emqx_edge}/vm.args (100%) diff --git a/etc/emqx_cloud.d/vm.args b/etc/emqx_cloud/vm.args similarity index 100% rename from etc/emqx_cloud.d/vm.args rename to etc/emqx_cloud/vm.args diff --git a/etc/emqx_edge.d/vm.args b/etc/emqx_edge/vm.args similarity index 100% rename from etc/emqx_edge.d/vm.args rename to etc/emqx_edge/vm.args diff --git a/rebar.config.erl b/rebar.config.erl index 6006719f1..c88ff8c13 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -243,11 +243,11 @@ extra_overlay(edge) -> []. emqx_etc_overlay(cloud) -> emqx_etc_overlay_common() ++ - [ {"etc/emqx_cloud.d/vm.args","etc/vm.args"} + [ {"etc/emqx_cloud/vm.args","etc/vm.args"} ]; emqx_etc_overlay(edge) -> emqx_etc_overlay_common() ++ - [ {"etc/emqx_edge.d/vm.args","etc/vm.args"} + [ {"etc/emqx_edge/vm.args","etc/vm.args"} ]. emqx_etc_overlay_common() -> From 8477780e2b6e5c1a3e7cbc32ffd57258119f0c84 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 2 Mar 2021 14:37:35 +0800 Subject: [PATCH 13/52] fix(mgmt): remove the useless match Currently, the `{topic, Topic}` pattern is not used for management. Moreover it will casue a `function_caluse` while the previous caluse not matched --- apps/emqx_management/src/emqx_mgmt.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 39bc2153a..3ab1b80ee 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -303,8 +303,8 @@ list_subscriptions_via_topic(Node, Topic, {M,F}) when Node =:= node() -> MatchSpec = [{{{'_', '$1'}, '_'}, [{'=:=','$1', Topic}], ['$_']}], M:F(ets:select(emqx_suboption, MatchSpec)); -list_subscriptions_via_topic(Node, {topic, Topic}, FormatFun) -> - rpc_call(Node, list_subscriptions_via_topic, [Node, {topic, Topic}, FormatFun]). +list_subscriptions_via_topic(Node, Topic, FormatFun) -> + rpc_call(Node, list_subscriptions_via_topic, [Node, Topic, FormatFun]). lookup_subscriptions(ClientId) -> lists:append([lookup_subscriptions(Node, ClientId) || Node <- ekka_mnesia:running_nodes()]). From cae08491520642ecef707e2a2e21e66da5d06362 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 2 Mar 2021 16:57:14 +0800 Subject: [PATCH 14/52] fix(emqx_channel): fix bug when publish deny fix the bug of replying to puback when the publish message with Qos equal to 2 is deny --- src/emqx_channel.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 2b51b1d02..03556d193 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -501,8 +501,10 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), ignore -> case QoS of ?QOS_0 -> {ok, NChannel}; - _ -> - handle_out(puback, {PacketId, Rc}, NChannel) + ?QOS_1 -> + handle_out(puback, {PacketId, Rc}, NChannel); + ?QOS_2 -> + handle_out(pubrec, {PacketId, Rc}, NChannel) end; disconnect -> handle_out(disconnect, Rc, NChannel) From 1ea4a9eb7def53f13a67f1ef152ecdd7f3583ad2 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 2 Mar 2021 15:13:38 +0800 Subject: [PATCH 15/52] chore(emqx_modules): enable emqx modules by default --- rebar.config.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/rebar.config.erl b/rebar.config.erl index c88ff8c13..1492114a1 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -148,6 +148,7 @@ relx_apps(ReleaseType) -> , {mnesia, load} , {ekka, load} , {emqx_plugin_libs, load} + , emqx_modules ] ++ [bcrypt || provide_bcrypt_release(ReleaseType)] ++ relx_apps_per_rel(ReleaseType) From 8e5cfaf0cc13a3661c975b565a480779589dd75b Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 2 Mar 2021 16:20:52 +0800 Subject: [PATCH 16/52] chore(emqx_modules): delete plugin flag for emqx_modules --- lib-ce/emqx_modules/src/emqx_modules_app.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib-ce/emqx_modules/src/emqx_modules_app.erl b/lib-ce/emqx_modules/src/emqx_modules_app.erl index 6d82177d4..ca735826c 100644 --- a/lib-ce/emqx_modules/src/emqx_modules_app.erl +++ b/lib-ce/emqx_modules/src/emqx_modules_app.erl @@ -18,8 +18,6 @@ -behaviour(application). --emqx_plugin(?MODULE). - -export([start/2]). -export([stop/1]). From db92f79713bfcf1b0f4037746a978f3357acc4e9 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 2 Mar 2021 18:02:18 +0800 Subject: [PATCH 17/52] chore(emqx_modules): add emqx_modules api path --- apps/emqx_management/src/emqx_mgmt_http.erl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_http.erl b/apps/emqx_management/src/emqx_mgmt_http.erl index 8fb4f74ad..2bf6ef38a 100644 --- a/apps/emqx_management/src/emqx_mgmt_http.erl +++ b/apps/emqx_management/src/emqx_mgmt_http.erl @@ -85,7 +85,7 @@ listener_name(Proto) -> http_handlers() -> Plugins = lists:map(fun(Plugin) -> Plugin#plugin.name end, emqx_plugins:list()), - [{"/api/v4", minirest:handler(#{apps => Plugins -- ?EXCEPT_PLUGIN, + [{"/api/v4", minirest:handler(#{apps => Plugins ++ [emqx_modules] -- ?EXCEPT_PLUGIN, except => ?EXCEPT, filter => fun filter/1}), [{authorization, fun authorize_appid/1}]}]. @@ -119,15 +119,12 @@ authorize_appid(Req) -> _ -> false end. --ifdef(EMQX_ENTERPRISE). -filter(_) -> true. --else. +filter(#{app := emqx_modules}) -> true; filter(#{app := App}) -> case emqx_plugins:find_plugin(App) of false -> false; Plugin -> Plugin#plugin.active end. --endif. format(Port) when is_integer(Port) -> io_lib:format("0.0.0.0:~w", [Port]); From 7bf5bb26e7a4ec28b233c8d0e9e3a45d672f2f28 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 2 Mar 2021 11:08:13 +0100 Subject: [PATCH 18/52] fix(scripts): Ensure ekka in ERL_LIBS for remote nodes --- bin/emqx | 1 + bin/emqx_ctl | 2 +- data/emqx_vars | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/emqx b/bin/emqx index 2fe24af23..81b1a9594 100755 --- a/bin/emqx +++ b/bin/emqx @@ -176,6 +176,7 @@ relx_nodetool() { command="$1"; shift ERL_FLAGS="$ERL_FLAGS $EPMD_ARG" \ + ERL_LIBS="${LIB_EKKA_DIR}:${ERL_LIBS:-}" \ "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" "$NAME_TYPE" "$NAME" \ -setcookie "$COOKIE" "$command" "$@" } diff --git a/bin/emqx_ctl b/bin/emqx_ctl index 961832c86..5fa35c7a5 100755 --- a/bin/emqx_ctl +++ b/bin/emqx_ctl @@ -27,11 +27,11 @@ relx_nodetool() { command="$1"; shift ERL_FLAGS="$ERL_FLAGS $EPMD_ARG $PROTO_DIST_ARG" \ + ERL_LIBS="${LIB_EKKA_DIR}:${ERL_LIBS:-}" \ "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" "$NAME_TYPE" "$NAME" \ -setcookie "$COOKIE" "$command" "$@" } - if [ -z "$NAME_ARG" ]; then NODENAME="${EMQX_NODE_NAME:-}" # check if there is a node running, inspect its name diff --git a/data/emqx_vars b/data/emqx_vars index 2266f741b..5159fa246 100644 --- a/data/emqx_vars +++ b/data/emqx_vars @@ -14,6 +14,7 @@ RUNNER_ETC_DIR="{{ runner_etc_dir }}" RUNNER_DATA_DIR="{{ runner_data_dir }}" RUNNER_USER="{{ runner_user }}" EMQX_DISCR="{{ emqx_description }}" +LIB_EKKA_DIR="${RUNNER_LIB_DIR}/ekka-$(grep ekka "${RUNNER_ROOT_DIR}/releases/RELEASES" | awk -F '\"' '{print $2}')" ## computed vars REL_NAME="emqx" From 2fb5dbd54688e25561b3a25af9230f647a89f943 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 2 Mar 2021 17:35:13 +0800 Subject: [PATCH 19/52] revert(bridge-mqtt): remove the mqtt_sub resource We have several reasons to remove this feature: 1. The design does not make sense. A rule engine resource should not have an impact on the system's messages directly after it is created. This mqtt_sub actually conflicts with any design concept of the rules engine. 2. The implementation is incorrect. mqtt_sub uses a client pool to establish a subscription relationship to an MQTT Broker. This causes a message to be sent repeatedly to EMQ X. Unless a shared subscription is used, or a Pool Size of 1 is configured. 3. The emqx-bridge-mqtt supports all the features of mqtt_sub. This feature introduced by https://github.com/emqx/emqx-bridge-mqtt/pull/78. And it released to v4.2.0 (NOT WORK), v4.2.1-v4.2.7 (FIXED) --- .../src/emqx_bridge_mqtt_actions.erl | 221 ++---------------- 1 file changed, 14 insertions(+), 207 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl index 0dccbd353..cbd7f28ed 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl @@ -39,7 +39,6 @@ ]). -define(RESOURCE_TYPE_MQTT, 'bridge_mqtt'). --define(RESOURCE_TYPE_MQTT_SUB, 'bridge_mqtt_sub'). -define(RESOURCE_TYPE_RPC, 'bridge_rpc'). -define(RESOURCE_CONFIG_SPEC_MQTT, #{ @@ -111,7 +110,7 @@ zh => <<"桥接挂载点"/utf8>>}, description => #{ en => <<"MountPoint for bridge topic:
" - "Example: The topic of messages sent to `topic1` on local node" + "Example: The topic of messages sent to `topic1` on local node " "will be transformed to `bridge/aws/${node}/topic1`">>, zh => <<"桥接主题的挂载点:
" "示例: 本地节点向 `topic1` 发消息，远程桥接节点的主题" @@ -126,8 +125,8 @@ enum => [<<"on">>, <<"off">>], title => #{en => <<"Disk Cache">>, zh => <<"磁盘缓存"/utf8>>}, - description => #{en => <<"The flag which determines whether messages" - "can be cached on local disk when bridge is" + description => #{en => <<"The flag which determines whether messages " + "can be cached on local disk when bridge is " "disconnected">>, zh => <<"当桥接断开时用于控制是否将消息缓存到本地磁" "盘队列上"/utf8>>} @@ -244,181 +243,6 @@ } }). --define(RESOURCE_CONFIG_SPEC_MQTT_SUB, #{ - address => #{ - order => 1, - type => string, - required => true, - default => <<"127.0.0.1:1883">>, - title => #{en => <<" Broker Address">>, - zh => <<"远程 broker 地址"/utf8>>}, - description => #{en => <<"The MQTT Remote Address">>, - zh => <<"远程 MQTT Broker 的地址"/utf8>>} - }, - pool_size => #{ - order => 2, - type => number, - required => true, - default => 8, - title => #{en => <<"Pool Size">>, - zh => <<"连接池大小"/utf8>>}, - description => #{en => <<"MQTT Connection Pool Size">>, - zh => <<"连接池大小"/utf8>>} - }, - clientid => #{ - order => 3, - type => string, - required => true, - default => <<"client">>, - title => #{en => <<"ClientId">>, - zh => <<"客户端 Id"/utf8>>}, - description => #{en => <<"ClientId for connecting to remote MQTT broker">>, - zh => <<"连接远程 Broker 的 ClientId"/utf8>>} - }, - append => #{ - order => 4, - type => boolean, - required => true, - default => true, - title => #{en => <<"Append GUID">>, - zh => <<"附加 GUID"/utf8>>}, - description => #{en => <<"Append GUID to MQTT ClientId?">>, - zh => <<"是否将GUID附加到 MQTT ClientId 后"/utf8>>} - }, - username => #{ - order => 5, - type => string, - required => false, - default => <<"">>, - title => #{en => <<"Username">>, zh => <<"用户名"/utf8>>}, - description => #{en => <<"Username for connecting to remote MQTT Broker">>, - zh => <<"连接远程 Broker 的用户名"/utf8>>} - }, - password => #{ - order => 6, - type => password, - required => false, - default => <<"">>, - title => #{en => <<"Password">>, - zh => <<"密码"/utf8>>}, - description => #{en => <<"Password for connecting to remote MQTT Broker">>, - zh => <<"连接远程 Broker 的密码"/utf8>>} - }, - subscription_opts => #{ - order => 7, - type => array, - items => #{ - type => object, - schema => #{ - topic => #{ - order => 1, - type => string, - default => <<>>, - title => #{en => <<"MQTT Topic">>, - zh => <<"MQTT 主题"/utf8>>}, - description => #{en => <<"MQTT Topic">>, - zh => <<"MQTT 主题"/utf8>>} - }, - qos => #{ - order => 2, - type => number, - enum => [0, 1, 2], - default => 0, - title => #{en => <<"MQTT Topic QoS">>, - zh => <<"MQTT 服务质量"/utf8>>}, - description => #{en => <<"MQTT Topic QoS">>, - zh => <<"MQTT 服务质量"/utf8>>} - } - } - }, - default => [], - title => #{en => <<"Subscription Opts">>, - zh => <<"订阅选项"/utf8>>}, - description => #{en => <<"Subscription Opts">>, - zh => <<"订阅选项"/utf8>>} - }, - proto_ver => #{ - order => 8, - type => string, - required => false, - default => <<"mqttv4">>, - enum => [<<"mqttv3">>, <<"mqttv4">>, <<"mqttv5">>], - title => #{en => <<"Protocol Version">>, - zh => <<"协议版本"/utf8>>}, - description => #{en => <<"MQTTT Protocol version">>, - zh => <<"MQTT 协议版本"/utf8>>} - }, - keepalive => #{ - order => 9, - type => string, - required => false, - default => <<"60s">> , - title => #{en => <<"Keepalive">>, - zh => <<"心跳间隔"/utf8>>}, - description => #{en => <<"Keepalive">>, - zh => <<"心跳间隔"/utf8>>} - }, - reconnect_interval => #{ - order => 10, - type => string, - required => false, - default => <<"30s">>, - title => #{en => <<"Reconnect Interval">>, - zh => <<"重连间隔"/utf8>>}, - description => #{en => <<"Reconnect interval of bridge">>, - zh => <<"重连间隔"/utf8>>} - }, - ssl => #{ - order => 11, - type => boolean, - default => false, - title => #{en => <<"Bridge SSL">>, - zh => <<"Bridge SSL"/utf8>>}, - description => #{en => <<"Switch which used to enable ssl connection of the bridge">>, - zh => <<"是否启用 Bridge SSL 连接"/utf8>>} - }, - cacertfile => #{ - order => 12, - type => file, - required => false, - default => <<"etc/certs/cacert.pem">>, - title => #{en => <<"CA certificates">>, - zh => <<"CA 证书"/utf8>>}, - description => #{en => <<"The file path of the CA certificates">>, - zh => <<"CA 证书路径"/utf8>>} - }, - certfile => #{ - order => 13, - type => file, - required => false, - default => <<"etc/certs/client-cert.pem">>, - title => #{en => <<"SSL Certfile">>, - zh => <<"SSL 客户端证书"/utf8>>}, - description => #{en => <<"The file path of the client certfile">>, - zh => <<"客户端证书路径"/utf8>>} - }, - keyfile => #{ - order => 14, - type => file, - required => false, - default => <<"etc/certs/client-key.pem">>, - title => #{en => <<"SSL Keyfile">>, - zh => <<"SSL 密钥文件"/utf8>>}, - description => #{en => <<"The file path of the client keyfile">>, - zh => <<"客户端密钥路径"/utf8>>} - }, - ciphers => #{ - order => 15, - type => string, - required => false, - default => <<"ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384">>, - title => #{en => <<"SSL Ciphers">>, - zh => <<"SSL 加密算法"/utf8>>}, - description => #{en => <<"SSL Ciphers">>, - zh => <<"SSL 加密算法"/utf8>>} - } - }). - -define(RESOURCE_CONFIG_SPEC_RPC, #{ address => #{ order => 1, @@ -438,7 +262,7 @@ title => #{en => <<"Bridge MountPoint">>, zh => <<"桥接挂载点"/utf8>>}, description => #{en => <<"MountPoint for bridge topic
" - "Example: The topic of messages sent to `topic1` on local node" + "Example: The topic of messages sent to `topic1` on local node " "will be transformed to `bridge/aws/${node}/topic1`">>, zh => <<"桥接主题的挂载点
" "示例: 本地节点向 `topic1` 发消息，远程桥接节点的主题" @@ -482,8 +306,8 @@ enum => [<<"on">>, <<"off">>], title => #{en => <<"Disk Cache">>, zh => <<"磁盘缓存"/utf8>>}, - description => #{en => <<"The flag which determines whether messages" - "can be cached on local disk when bridge is" + description => #{en => <<"The flag which determines whether messages " + "can be cached on local disk when bridge is " "disconnected">>, zh => <<"当桥接断开时用于控制是否将消息缓存到本地磁" "盘队列上"/utf8>>} @@ -508,15 +332,6 @@ description => #{en => <<"MQTT Message Bridge">>, zh => <<"MQTT 消息桥接"/utf8>>} }). --resource_type(#{ - name => ?RESOURCE_TYPE_MQTT_SUB, - create => on_resource_create, - status => on_get_resource_status, - destroy => on_resource_destroy, - params => ?RESOURCE_CONFIG_SPEC_MQTT_SUB, - title => #{en => <<"MQTT Subscribe">>, zh => <<"MQTT Subscribe"/utf8>>}, - description => #{en => <<"MQTT Subscribe">>, zh => <<"MQTT 订阅消息"/utf8>>} - }). -resource_type(#{ name => ?RESOURCE_TYPE_RPC, @@ -542,7 +357,8 @@ default => <<"">>, title => #{en => <<"Forward Topic">>, zh => <<"转发消息主题"/utf8>>}, - description => #{en => <<"The topic used when forwarding the message. Defaults to the topic of the bridge message if not provided.">>, + description => #{en => <<"The topic used when forwarding the message. " + "Defaults to the topic of the bridge message if not provided.">>, zh => <<"转发消息时使用的主题。如果未提供，则默认为桥接消息的主题。"/utf8>>} }, payload_tmpl => #{ @@ -553,8 +369,11 @@ default => <<"">>, title => #{en => <<"Payload Template">>, zh => <<"消息内容模板"/utf8>>}, - description => #{en => <<"The payload template, variable interpolation is supported. If using empty template (default), then the payload will be all the available vars in JSON format">>, - zh => <<"消息内容模板，支持变量。若使用空模板（默认），消息内容为 JSON 格式的所有字段"/utf8>>} + description => #{en => <<"The payload template, variable interpolation is supported. " + "If using empty template (default), then the payload will be " + "all the available vars in JSON format">>, + zh => <<"消息内容模板，支持变量。" + "若使用空模板（默认），消息内容为 JSON 格式的所有字段"/utf8>>} } }, title => #{en => <<"Data bridge to MQTT Broker">>, @@ -731,12 +550,6 @@ options(Options, PoolName, ResId) -> {connect_module, emqx_bridge_rpc}, {batch_size, Get(<<"batch_size">>)}]; false -> - Subscriptions = format_subscriptions(GetD(<<"subscription_opts">>, [])), - Subscriptions1 = case Get(<<"topic">>) of - undefined -> Subscriptions; - Topic -> - [{subscriptions, [{Topic, Get(<<"qos">>)}]} | Subscriptions] - end, [{address, binary_to_list(Address)}, {bridge_mode, GetD(<<"bridge_mode">>, true)}, {clean_start, true}, @@ -748,8 +561,7 @@ options(Options, PoolName, ResId) -> {password, str(Get(<<"password">>))}, {proto_ver, mqtt_ver(Get(<<"proto_ver">>))}, {retry_interval, cuttlefish_duration:parse(str(GetD(<<"retry_interval">>, "30s")), s)} - | maybe_ssl(Options, Get(<<"ssl">>), ResId) - ] ++ Subscriptions1 + | maybe_ssl(Options, Get(<<"ssl">>), ResId)] end. maybe_ssl(_Options, false, _ResId) -> @@ -765,8 +577,3 @@ mqtt_ver(ProtoVer) -> <<"mqttv5">> -> v5; _ -> v4 end. - -format_subscriptions(SubOpts) -> - lists:map(fun(Sub) -> - {maps:get(<<"topic">>, Sub), maps:get(<<"qos">>, Sub)} - end, SubOpts). From 6b2949e2c1ea1d2e9472027bc78f20e8ae211aac Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 2 Mar 2021 12:37:49 +0100 Subject: [PATCH 20/52] chore(config): Change default log_to config By default log to file. When start from console or forgreground, only log to console. Only log to file for package builds. --- bin/emqx | 9 ++++++++- etc/emqx.conf | 2 +- priv/emqx.schema | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/bin/emqx b/bin/emqx index 81b1a9594..9aba13192 100755 --- a/bin/emqx +++ b/bin/emqx @@ -518,6 +518,9 @@ case "$1" in ;; esac + # set before generate_config + export EMQX_LOG__TO='console' + #generate app.config and vm.args generate_config @@ -534,7 +537,8 @@ case "$1" in # shellcheck disable=SC2086 # $RELX_CONFIG_PATH $CONFIG_ARGS $EPMD_ARG are supposed to be split by whitespace # Build an array of arguments to pass to exec later on # Build it here because this command will be used for logging. - set -- "$BINDIR/erlexec" -boot "$BOOTFILE" -mode "$CODE_LOADING_MODE" \ + set -- "$BINDIR/erlexec" \ + -boot "$BOOTFILE" -mode "$CODE_LOADING_MODE" \ -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \ -mnesia dir "\"${MNESIA_DATA_DIR}\"" \ $RELX_CONFIG_PATH $CONFIG_ARGS $EPMD_ARG @@ -557,6 +561,9 @@ case "$1" in # start up the release in the foreground for use by runit # or other supervision services + # set before generate_config + export EMQX_LOG__TO='console' + #generate app.config and vm.args generate_config diff --git a/etc/emqx.conf b/etc/emqx.conf index ddc17eef9..672ce8556 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -412,7 +412,7 @@ rpc.socket_buffer = 1MB ## - file: write logs only to file ## - console: write logs only to standard I/O ## - both: write logs both to file and standard I/O -log.to = both +log.to = file ## The log severity level. ## diff --git a/priv/emqx.schema b/priv/emqx.schema index fa9cf1efb..0e933c44f 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -449,7 +449,7 @@ end}. %%-------------------------------------------------------------------- {mapping, "log.to", "kernel.logger", [ - {default, console}, + {default, file}, {datatype, {enum, [off, file, console, both]}} ]}. From fc03d7aea9b8bdef613d99d0c0cbc042b9f5e298 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 2 Mar 2021 15:17:48 +0100 Subject: [PATCH 21/52] fix(config): use config value default log.to for daemon mode --- bin/emqx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bin/emqx b/bin/emqx index 9aba13192..c26c0f8fc 100755 --- a/bin/emqx +++ b/bin/emqx @@ -339,6 +339,10 @@ case "$1" in # Bootstrap daemon command (check perms & drop to $RUNNER_USER) bootstrapd + # this flag passes down to console mode + # so we know it's intended to be run in daemon mode + export _EMQX_START_MODE="$1" + # Save this for later. CMD=$1 case "$1" in @@ -519,7 +523,9 @@ case "$1" in esac # set before generate_config - export EMQX_LOG__TO='console' + if [ "${_EMQX_START_MODE:-}" = '' ]; then + export EMQX_LOG__TO="${EMQX_LOG__TO:-console}" + fi #generate app.config and vm.args generate_config @@ -562,7 +568,7 @@ case "$1" in # or other supervision services # set before generate_config - export EMQX_LOG__TO='console' + export EMQX_LOG__TO="${EMQX_LOG__TO:-console}" #generate app.config and vm.args generate_config From 5d71e51985347f5e84bbb8817b59e979bc06b673 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 2 Mar 2021 15:29:44 +0100 Subject: [PATCH 22/52] fix(config): Do not allow customizing cuttlefish env override prefix We have quite a few EMQX_ prefixed variables used in bash scripts having to meta-programming bash is not quite readable --- bin/emqx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/emqx b/bin/emqx index c26c0f8fc..3d5900ef7 100755 --- a/bin/emqx +++ b/bin/emqx @@ -20,8 +20,8 @@ mkdir -p "$RUNNER_LOG_DIR" # Make sure data directory exists mkdir -p "$RUNNER_DATA_DIR" -# cuttlefish try to read environment variables starting with "EMQX_", if not specified -export CUTTLEFISH_ENV_OVERRIDE_PREFIX="${CUTTLEFISH_ENV_OVERRIDE_PREFIX:-EMQX_}" +# cuttlefish try to read environment variables starting with "EMQX_" +export CUTTLEFISH_ENV_OVERRIDE_PREFIX='EMQX_' relx_usage() { command="$1" From 6fb5c9de424695b09a28d1020500f16c1355eee7 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 2 Mar 2021 18:10:26 +0100 Subject: [PATCH 23/52] chore(deps): upgrade replayq to version 0.3.2 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 034c9225b..f1fe1d3b2 100644 --- a/rebar.config +++ b/rebar.config @@ -46,7 +46,7 @@ , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.1.0"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.3"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.0"}}} - , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.1"}}} + , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.2"}}} From e3407b95565cd199c6f6d6bbe11cc6cdf8e6dd48 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 2 Mar 2021 20:14:06 +0100 Subject: [PATCH 24/52] chore(build): fix rebar dependency and add a script to ensure integrity --- apps/emqx_sasl/rebar.config | 2 +- apps/emqx_telemetry/rebar.config | 15 +------ scripts/check-deps-integrity.escript | 63 ++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 15 deletions(-) create mode 100755 scripts/check-deps-integrity.escript diff --git a/apps/emqx_sasl/rebar.config b/apps/emqx_sasl/rebar.config index 40adb664a..318f82a0b 100644 --- a/apps/emqx_sasl/rebar.config +++ b/apps/emqx_sasl/rebar.config @@ -1,5 +1,5 @@ {deps, - [{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.3"}}} + [{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/apps/emqx_telemetry/rebar.config b/apps/emqx_telemetry/rebar.config index 0e8d1ef9b..7b30a8fd8 100644 --- a/apps/emqx_telemetry/rebar.config +++ b/apps/emqx_telemetry/rebar.config @@ -1,14 +1 @@ -{edoc_opts, [{preprocess, true}]}. -{erl_opts, [warn_unused_vars, - warn_shadow_vars, - warn_unused_import, - warn_obsolete_guard, - debug_info, - {parse_transform}]}. - -{xref_checks, [undefined_function_calls, undefined_functions, - locals_not_used, deprecated_function_calls, - warnings_as_errors, deprecated_functions]}. -{cover_enabled, true}. -{cover_opts, [verbose]}. -{cover_export_enabled, true}. +{deps, []}. diff --git a/scripts/check-deps-integrity.escript b/scripts/check-deps-integrity.escript new file mode 100755 index 000000000..2dd16afb9 --- /dev/null +++ b/scripts/check-deps-integrity.escript @@ -0,0 +1,63 @@ +#!/usr/bin/env escript + +%% NOTE: this script should be executed at project root. + +-mode(compile). + +main([]) -> + AppsDir = case filelib:is_file("EMQX_ENTERPRISE") of + true -> "lib-ee"; + false -> "lib-ce" + end, + true = filelib:is_dir(AppsDir), + Files = ["rebar.config"] ++ + apps_rebar_config("apps") ++ + apps_rebar_config(AppsDir), + Deps = collect_deps(Files, #{}), + case count_bad_deps(Deps) of + 0 -> + io:format("OK~n"); + N -> + io:format(standard_error, "~p dependency discrepancies", [N]), + halt(1) + end. + +apps_rebar_config(Dir) -> + filelib:wildcard(filename:join([Dir, "*", "rebar.config"])). + +%% collect a kv-list of {DepName, [{DepReference, RebarConfigFile}]} +%% the value part should have unique DepReference +collect_deps([], Acc) -> maps:to_list(Acc); +collect_deps([File | Files], Acc) -> + Deps = + try + {ok, Config} = file:consult(File), + {deps, Deps0} = lists:keyfind(deps, 1, Config), + Deps0 + catch + C : E : St -> + erlang:raise(C, {E, {failed_to_find_deps_in_rebar_config, File}}, St) + end, + collect_deps(Files, do_collect_deps(Deps, File, Acc)). + +do_collect_deps([], _File, Acc) -> Acc; +do_collect_deps([{Name, Ref} | Deps], File, Acc) -> + Refs = maps:get(Name, Acc, []), + do_collect_deps(Deps, File, Acc#{Name => [{Ref, File} | Refs]}). + +count_bad_deps([]) -> 0; +count_bad_deps([{Name, Refs0} | Rest]) -> + Refs = lists:keysort(1, Refs0), + case is_unique_ref(Refs) of + true -> + count_bad_deps(Rest); + false -> + io:format(standard_error, "~p:~n~p~n", [Name, Refs]), + 1 + count_bad_deps(Rest) + end. + +is_unique_ref([_]) -> true; +is_unique_ref([{Ref, _File1}, {Ref, File2} | Rest]) -> + is_unique_ref([{Ref, File2} | Rest]); +is_unique_ref(_) -> + false. From c61080c8627e3b2fe3b7070af0a821b713817f2d Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 2 Mar 2021 20:37:45 +0100 Subject: [PATCH 25/52] chore(build): write rendered rebar.config only when debugging --- rebar.config.erl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/rebar.config.erl b/rebar.config.erl index 1492114a1..1753e97ff 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -5,7 +5,7 @@ do(_Dir, CONFIG) -> C1 = deps(CONFIG), Config = dialyzer(C1), - dump(Config ++ [{overrides, overrides()}] ++ coveralls() ++ config()). + maybe_dump(Config ++ [{overrides, overrides()}] ++ coveralls() ++ config()). bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {branch, "0.6.0"}}}. @@ -286,10 +286,19 @@ get_vsn() -> Vsn2 = re:replace(PkgVsn, "v", "", [{return ,list}]), re:replace(Vsn2, "\n", "", [{return ,list}]). -dump(Config) -> - file:write_file("rebar.config.rendered", [io_lib:format("~p.\n", [I]) || I <- Config]), +maybe_dump(Config) -> + is_debug() andalso file:write_file("rebar.config.rendered", [io_lib:format("~p.\n", [I]) || I <- Config]), Config. +is_debug() -> is_debug("DEBUG") orelse is_debug("DIAGNOSTIC"). + +is_debug(VarName) -> + case os:getenv(VarName) of + false -> false; + "" -> false; + _ -> true + end. + provide_bcrypt_dep() -> case os:type() of {win32, _} -> false; From b176300635d004d5ed382065c809a9d8abef934b Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 2 Mar 2021 20:38:44 +0100 Subject: [PATCH 26/52] chore(ci): check rebar dependency integrity in CI --- .github/workflows/check_deps_integrity.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/check_deps_integrity.yaml diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml new file mode 100644 index 000000000..eb0ff1fee --- /dev/null +++ b/.github/workflows/check_deps_integrity.yaml @@ -0,0 +1,13 @@ +name: Check Rebar Dependencies + +on: [pull_request] + +jobs: + check_deps_integrity: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Run check-deps-integrity.escript + run: | + docker run --rm -v "$(pwd):/emqx" emqx/build-env:erl23.2.2-ubuntu20.04 sh -c 'cd /emqx && ./scripts/check-deps-integrity.escript' + From cd0890796d58e57f44d04beacfa73b07f6751eb2 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 2 Mar 2021 23:32:25 +0100 Subject: [PATCH 27/52] chore(build): use only mark file to tell apart ce ee --- Makefile | 1 - rebar.config.erl | 17 ++++++++++++++--- scripts/get-dashboard.sh | 2 +- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 146841948..dadc9dd3b 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,6 @@ DASHBOARD_VERSION = v4.3.0-beta.1 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts -export EMQX_ENTERPRISE=false export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) PROFILE ?= emqx diff --git a/rebar.config.erl b/rebar.config.erl index 1492114a1..2ae7559bd 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -22,6 +22,7 @@ overrides() -> [ {add, [ {extra_src_dirs, [{"etc", [{recursive,true}]}]} , {erl_opts, [ deterministic , {compile_info, [{emqx_vsn, get_vsn()}]} + | [{d, 'EMQX_ENTERPRISE'} || is_enterprise()] ]} ]} ]. @@ -32,9 +33,11 @@ config() -> , {project_app_dirs, project_app_dirs()} ]. +is_enterprise() -> + filelib:is_regular("EMQX_ENTERPRISE"). + extra_lib_dir() -> - EnterpriseFlag = os:getenv("EMQX_ENTERPRISE"), - case EnterpriseFlag =:= "true" orelse EnterpriseFlag =:= "1" of + case is_enterprise() of true -> "lib-ee"; false -> "lib-ce" end. @@ -61,6 +64,7 @@ test_deps() -> common_compile_opts() -> [ deterministic , {compile_info, [{emqx_vsn, get_vsn()}]} + | [{d, 'EMQX_ENTERPRISE'} || is_enterprise()] ]. prod_compile_opts() -> @@ -180,7 +184,9 @@ relx_plugin_apps(ReleaseType) -> , emqx_sasl , emqx_telemetry , emqx_modules - ] ++ relx_plugin_apps_per_rel(ReleaseType). + ] + ++ relx_plugin_apps_per_rel(ReleaseType) + ++ relx_plugin_apps_enterprise(is_enterprise()). relx_plugin_apps_per_rel(cloud) -> [ emqx_lwm2m @@ -197,6 +203,11 @@ relx_plugin_apps_per_rel(cloud) -> relx_plugin_apps_per_rel(edge) -> []. +relx_plugin_apps_enterprise(true) -> + [list_to_atom(A) || A <- filelib:wildcard("*", "lib-ee"), + filelib:is_dir(filename:join(["lib-ee", A]))]; +relx_plugin_apps_enterprise(false) -> []. + relx_overlay(ReleaseType) -> [ {mkdir,"log/"} , {mkdir,"data/"} diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index bc28164a5..3e47490b0 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -13,7 +13,7 @@ else DOWNLOAD_URL="https://github.com/emqx/emqx-dashboard-frontend/releases/download/${VERSION}/emqx-dashboard.zip" fi -if [ "${EMQX_ENTERPRISE:-}" = 'true' ] || [ "${EMQX_ENTERPRISE:-}" == '1' ]; then +if [ -f 'EMQX_ENTERPRISE' ]; then DASHBOARD_PATH='lib-ee/emqx_dashboard/priv' else DASHBOARD_PATH='lib-ce/emqx_dashboard/priv' From 8316f8cc34e2c8c22ede1435414ab84935dccebd Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 2 Mar 2021 23:02:40 +0100 Subject: [PATCH 28/52] fix(rule-engin-api): transform input config for update requests --- apps/emqx_rule_engine/src/emqx_rule_engine_api.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index d07ad9b94..beb1204df 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -337,7 +337,7 @@ update_resource(#{id := Id}, NewParams) -> P2 = case proplists:get_value(<<"config">>, NewParams) of undefined -> #{}; [{}] -> #{}; - Map -> #{<<"config">> => ?RAISE(maps:from_list(Map), {invalid_config, Map})} + Config -> #{<<"config">> => ?RAISE(json_term_to_map(Config), {invalid_config, Config})} end, case emqx_rule_engine:update_resource(Id, maps:merge(P1, P2)) of ok -> @@ -552,4 +552,4 @@ get_rule_metrics(Id) -> get_action_metrics(Id) -> [maps:put(node, Node, rpc:call(Node, emqx_rule_metrics, get_action_metrics, [Id])) - || Node <- ekka_mnesia:running_nodes()]. \ No newline at end of file + || Node <- ekka_mnesia:running_nodes()]. From 15041649339658f5eb67e0d45664470645b975f7 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Tue, 2 Mar 2021 15:31:04 +0900 Subject: [PATCH 29/52] chore(src): fix dialyzer warnings (match values) --- apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl | 2 +- apps/emqx_management/src/emqx_mgmt_api_pubsub.erl | 2 +- apps/emqx_management/src/emqx_mgmt_data_backup.erl | 4 ++-- apps/emqx_rule_engine/src/emqx_rule_registry.erl | 4 ++-- apps/emqx_web_hook/src/emqx_web_hook_app.erl | 2 +- lib-ce/emqx_modules/src/emqx_modules_app.erl | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl index 45023783b..0a40d3765 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl @@ -121,7 +121,7 @@ update_reg_info(NewRegInfo, Lwm2mState = #lwm2m_state{ UpdatedRegInfo = maps:merge(RegInfo, NewRegInfo), - case proplists:get_value(update_msg_publish_condition, + _ = case proplists:get_value(update_msg_publish_condition, lwm2m_coap_responder:options(), contains_object_list) of always -> send_to_broker(<<"update">>, #{<<"data">> => UpdatedRegInfo}, Lwm2mState); diff --git a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl index f49aecb0e..3b7f7392f 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl @@ -167,7 +167,7 @@ do_publish(_ClientId, [], _Qos, _Retain, _Payload) -> do_publish(ClientId, Topics, Qos, Retain, Payload) -> MsgIds = lists:map(fun(Topic) -> Msg = emqx_message:make(ClientId, Qos, Topic, Payload), - emqx_mgmt:publish(Msg#message{flags = #{retain => Retain}}), + _ = emqx_mgmt:publish(Msg#message{flags = #{retain => Retain}}), emqx_guid:to_hexstr(Msg#message.id) end, Topics), {ok, MsgIds}. diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 5534abacd..09e270fbb 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -237,11 +237,11 @@ import_resources_and_rules(Resources, Rules, FromVersion) <<"url">> => URL, <<"verify">> => true}, NResource = Resource#{<<"config">> := NConfig}, - import_resource(NResource), + {ok, _Resource} = import_resource(NResource), NHeaders = maps:put(<<"content-type">>, ContentType, Headers), [{ID, #{headers => NHeaders, method => Method}} | Acc]; (Resource, Acc) -> - import_resource(Resource), + {ok, _Resource} = import_resource(Resource), Acc end, [], Resources), lists:foreach(fun(#{<<"actions">> := Actions} = Rule) -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_registry.erl b/apps/emqx_rule_engine/src/emqx_rule_registry.erl index 3e4e4b74c..dc7a33805 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_registry.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_registry.erl @@ -407,8 +407,8 @@ delete_resource_type(Type) -> init([]) -> %% Enable stats timer ok = emqx_stats:update_interval(rule_registery_stats, fun update_stats/0), - ets:new(?KV_TAB, [named_table, set, public, {write_concurrency, true}, - {read_concurrency, true}]), + _TableId = ets:new(?KV_TAB, [named_table, set, public, {write_concurrency, true}, + {read_concurrency, true}]), {ok, #{}}. handle_call({add_rules, Rules}, _From, State) -> diff --git a/apps/emqx_web_hook/src/emqx_web_hook_app.erl b/apps/emqx_web_hook/src/emqx_web_hook_app.erl index d31df9b93..54ac9c317 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_app.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_app.erl @@ -30,7 +30,7 @@ start(_StartType, _StartArgs) -> translate_env(), {ok, Sup} = emqx_web_hook_sup:start_link(), {ok, PoolOpts} = application:get_env(?APP, pool_opts), - ehttpc_sup:start_pool(?APP, PoolOpts), + {ok, _Pid} = ehttpc_sup:start_pool(?APP, PoolOpts), emqx_web_hook:register_metrics(), emqx_web_hook:load(), {ok, Sup}. diff --git a/lib-ce/emqx_modules/src/emqx_modules_app.erl b/lib-ce/emqx_modules/src/emqx_modules_app.erl index ca735826c..a10176829 100644 --- a/lib-ce/emqx_modules/src/emqx_modules_app.erl +++ b/lib-ce/emqx_modules/src/emqx_modules_app.erl @@ -25,7 +25,7 @@ start(_Type, _Args) -> % the configs for emqx_modules is so far still in emqx application % Ensure it's loaded - application:load(emqx), + _ = application:load(emqx), {ok, Pid} = emqx_mod_sup:start_link(), ok = emqx_modules:load(), emqx_ctl:register_command(modules, {emqx_modules, cli}, []), From 70625b194205e7ae19afb66c2741ef75d6035a6d Mon Sep 17 00:00:00 2001 From: z8674558 Date: Tue, 2 Mar 2021 16:01:04 +0900 Subject: [PATCH 30/52] chore(emqx_sn_gateway): rm unused clauses --- apps/emqx_sn/src/emqx_sn_gateway.erl | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/apps/emqx_sn/src/emqx_sn_gateway.erl b/apps/emqx_sn/src/emqx_sn_gateway.erl index f4448d65d..335bd5531 100644 --- a/apps/emqx_sn/src/emqx_sn_gateway.erl +++ b/apps/emqx_sn/src/emqx_sn_gateway.erl @@ -104,9 +104,6 @@ -define(NO_PEERCERT, undefined). -%% TODO: fix when https://github.com/emqx/emqx-sn/pull/170 is merged --dialyzer([{nowarn_function, [idle/3]}]). - %%-------------------------------------------------------------------- %% Exported APIs %%-------------------------------------------------------------------- @@ -201,7 +198,7 @@ idle(cast, {incoming, ?SN_PUBLISH_MSG(#mqtt_sn_flags{qos = ?QOS_NEG1, false -> emqx_sn_registry:lookup_topic(Registry, ClientId, TopicId); true -> <> end, - case TopicName =/= undefined of + _ = case TopicName =/= undefined of true -> Msg = emqx_message:make(?NEG_QOS_CLIENT_ID, ?QOS_0, TopicName, Data), emqx_broker:publish(Msg); @@ -590,11 +587,7 @@ handle_call(_From, Req, State = #state{channel = Channel}) -> {reply, Reply, NChannel} -> {reply, Reply, State#state{channel = NChannel}}; {shutdown, Reason, Reply, NChannel} -> - shutdown(Reason, Reply, State#state{channel = NChannel}); - {shutdown, Reason, Reply, OutPacket, NChannel} -> - NState = State#state{channel = NChannel}, - ok = handle_outgoing(OutPacket, NState), - shutdown(Reason, Reply, NState) + shutdown(Reason, Reply, State#state{channel = NChannel}) end. handle_info(Info, State = #state{channel = Channel}) -> @@ -609,8 +602,6 @@ handle_ping(_PingReq, State) -> handle_timeout(TRef, TMsg, State = #state{channel = Channel}) -> handle_return(emqx_channel:handle_timeout(TRef, TMsg, Channel), State). -handle_return(ok, State) -> - {keep_state, State}; handle_return({ok, NChannel}, State) -> {keep_state, State#state{channel = NChannel}}; handle_return({ok, Replies, NChannel}, State) -> @@ -621,13 +612,7 @@ handle_return({shutdown, Reason, NChannel}, State) -> handle_return({shutdown, Reason, OutPacket, NChannel}, State) -> NState = State#state{channel = NChannel}, ok = handle_outgoing(OutPacket, NState), - stop({shutdown, Reason}, NState); -handle_return({stop, Reason, NChannel}, State) -> - stop(Reason, State#state{channel = NChannel}); -handle_return({stop, Reason, OutPacket, NChannel}, State) -> - NState = State#state{channel = NChannel}, - ok = handle_outgoing(OutPacket, NState), - stop(Reason, NState). + stop({shutdown, Reason}, NState). next_events(Packet) when is_record(Packet, mqtt_packet) -> next_event({outgoing, Packet}); From 1f238f4c2641d5dbe9fce983e045f0e567d2cee0 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Tue, 2 Mar 2021 16:43:04 +0900 Subject: [PATCH 31/52] chore(emqx_retainer): avoid race condition --- apps/emqx_retainer/src/emqx_retainer.erl | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 70f49a16c..28812ebfa 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -205,14 +205,18 @@ store_retained(Msg = #message{topic = Topic, payload = Payload}, Env) -> msg = Msg, expiry_time = get_expiry_time(Msg, Env)}); {true, false} -> - case mnesia:dirty_read(?TAB, Topic) of - [_] -> - mnesia:dirty_write(?TAB, #retained{topic = topic2tokens(Topic), - msg = Msg, - expiry_time = get_expiry_time(Msg, Env)}); - [] -> - ?LOG(error, "Cannot retain message(topic=~s) for table is full!", [Topic]) - end; + {atomic, _} = mnesia:transaction( + fun() -> + case mnesia:read(?TAB, Topic) of + [_] -> + mnesia:write(?TAB, #retained{topic = topic2tokens(Topic), + msg = Msg, + expiry_time = get_expiry_time(Msg, Env)}, write); + [] -> + ?LOG(error, "Cannot retain message(topic=~s) for table is full!", [Topic]) + end + end), + ok; {true, _} -> ?LOG(error, "Cannot retain message(topic=~s) for table is full!", [Topic]); {_, true} -> From e1b915b91c9005b4732301e0c90e458b39d94be8 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Wed, 3 Mar 2021 00:31:08 +0900 Subject: [PATCH 32/52] chore(emqx_tracer): pass proper map --- src/emqx_tracer.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/emqx_tracer.erl b/src/emqx_tracer.erl index 365336f03..ef29732a0 100644 --- a/src/emqx_tracer.erl +++ b/src/emqx_tracer.erl @@ -103,7 +103,6 @@ install_trace_handler(Who, Level, LogFile) -> case logger:add_handler(handler_id(Who), logger_disk_log_h, #{level => Level, formatter => ?FORMAT, - filesync_repeat_interval => no_repeat, config => #{type => halt, file => LogFile}, filter_default => stop, filters => [{meta_key_filter, From 43fc842057f555ecb78adef6d323453d89907049 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Wed, 3 Mar 2021 00:40:10 +0900 Subject: [PATCH 33/52] chore(emqx_lwm2m_protocol): default username, password is null -> undefined --- apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl index 0a40d3765..39eef75ae 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl @@ -407,8 +407,8 @@ clientinfo(#lwm2m_state{peername = {PeerHost, _}, peerhost => PeerHost, sockport => 5683, %% FIXME: clientid => EndpointName, - username => null, - password => null, + username => undefined, + password => undefined, peercert => nossl, is_bridge => false, is_superuser => false, From 467569d5ac367ef9b361e32634dec678b4066e6c Mon Sep 17 00:00:00 2001 From: z8674558 Date: Wed, 3 Mar 2021 00:43:53 +0900 Subject: [PATCH 34/52] chore(emqx_lwm2m_protocol): rm unused clause --- apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl index 39eef75ae..d71f15e5e 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl @@ -202,7 +202,6 @@ terminate(Reason, Lwm2mState) -> ?LOG(error, "process terminated: ~p, lwm2m_state: ~p", [Reason, Lwm2mState]). clean_subscribe(_CoapPid, _Error, undefined, _Lwm2mState) -> ok; -clean_subscribe(_CoapPid, _Error, _SubTopic, undefined) -> ok; clean_subscribe(CoapPid, {shutdown, Error}, SubTopic, Lwm2mState) -> do_clean_subscribe(CoapPid, Error, SubTopic, Lwm2mState); clean_subscribe(CoapPid, Error, SubTopic, Lwm2mState) -> From 7edd66e9d61e09215c8d06d38d70d90e31fdb472 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Wed, 3 Mar 2021 00:59:50 +0900 Subject: [PATCH 35/52] chore(emqx_lwm2m_message): comment out unused clause --- apps/emqx_lwm2m/src/emqx_lwm2m_message.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl index d90459334..e36c04195 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl @@ -90,13 +90,13 @@ basename(OldBaseName, _ObjectId, ObjectInstanceId, _ResourceId, 2) -> [ObjId, ObjInsId, _ResId] -> <<$/, ObjId/binary, $/, ObjInsId/binary>>; [ObjId, ObjInsId] -> <<$/, ObjId/binary, $/, ObjInsId/binary>>; [ObjId] -> <<$/, ObjId/binary, $/, (integer_to_binary(ObjectInstanceId))/binary>> - end; -basename(OldBaseName, _ObjectId, _ObjectInstanceId, _ResourceId, 1) -> - case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of - [ObjId, _ObjInsId, _ResId] -> <<$/, ObjId/binary>>; - [ObjId, _ObjInsId] -> <<$/, ObjId/binary>>; - [ObjId] -> <<$/, ObjId/binary>> end. +% basename(OldBaseName, _ObjectId, _ObjectInstanceId, _ResourceId, 1) -> +% case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of +% [ObjId, _ObjInsId, _ResId] -> <<$/, ObjId/binary>>; +% [ObjId, _ObjInsId] -> <<$/, ObjId/binary>>; +% [ObjId] -> <<$/, ObjId/binary>> +% end. make_path(RelativePath, Id) -> <>. From 117c0cf252095f27a6bb49ced66121cb9f27e97d Mon Sep 17 00:00:00 2001 From: z8674558 Date: Wed, 3 Mar 2021 01:00:05 +0900 Subject: [PATCH 36/52] chore(emqx_lwm2m_message): pass integer from list --- apps/emqx_lwm2m/src/emqx_lwm2m_message.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl index e36c04195..a7c77cfa1 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl @@ -187,7 +187,7 @@ insert(Level, #{<<"path">> := EleName, <<"type">> := Type, <<"value">> := Value} case Level of object -> insert_resource_into_object(Path, BinaryValue, Acc); object_instance -> insert_resource_into_object_instance(Path, BinaryValue, Acc); - resource -> insert_resource_instance_into_resource(Path, BinaryValue, Acc) + resource -> insert_resource_instance_into_resource(hd(Path), BinaryValue, Acc) end. % json text to TLV binary From 71ca1e4d786b9b94ed9a5d9d315065ba18e361e3 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Wed, 3 Mar 2021 01:41:09 +0900 Subject: [PATCH 37/52] chore(emqx_lwm2m_coap_resource): ignore coap_discover/2 from dialyzer --- apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl index b1bb8e9eb..62c0e0904 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl @@ -47,6 +47,9 @@ -define(LOG(Level, Format, Args), logger:Level("LWM2M-RESOURCE: " ++ Format, Args)). +-dialyzer([{nowarn_function, [coap_discover/2]}]). +% we use {'absolute', string(), [{atom(), binary()}]} as coap_uri() +% https://github.com/emqx/lwm2m-coap/blob/258e9bd3762124395e83c1e68a1583b84718230f/src/lwm2m_coap_resource.erl#L61 % resource operations coap_discover(_Prefix, _Args) -> [{absolute, "mqtt", []}]. From 119bbc4881ac2585660bdec24f1d764691b49a4d Mon Sep 17 00:00:00 2001 From: z8674558 Date: Wed, 3 Mar 2021 01:41:41 +0900 Subject: [PATCH 38/52] chore(emqx_bridge_msg): add empty props --- apps/emqx_bridge_mqtt/src/emqx_bridge_msg.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_msg.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_msg.erl index 20600282e..19f5fa139 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_msg.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_msg.erl @@ -55,6 +55,7 @@ to_export(emqx_bridge_mqtt, Mountpoint, #mqtt_msg{qos = QoS, retain = Retain, topic = topic(Mountpoint, Topic), + props = #{}, payload = Payload}; to_export(_Module, Mountpoint, #message{topic = Topic} = Msg) -> From a8558bc7b5afe409eac1dccae81cee870a4866eb Mon Sep 17 00:00:00 2001 From: z8674558 Date: Wed, 3 Mar 2021 01:45:16 +0900 Subject: [PATCH 39/52] chore(emqx_channel): ignore peer_cert_as/5 from dialyzer --- src/emqx_channel.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 03556d193..9bfba11c9 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -228,6 +228,9 @@ setting_peercert_infos(Peercert, ClientInfo, Options) -> ClientId = peer_cert_as(peer_cert_as_clientid, Options, Peercert, DN, CN), ClientInfo#{username => Username, clientid => ClientId, dn => DN, cn => CN}. +-dialyzer([{nowarn_function, [peer_cert_as/5]}]). +% esockd_peercert:peercert is opaque +% https://github.com/emqx/esockd/blob/9b959fc11a1c398a589892f335235be6c5b4a454/src/esockd_peercert.erl#L23 peer_cert_as(Key, Options, Peercert, DN, CN) -> case proplists:get_value(Key, Options) of cn -> CN; From 3daefe954bb443b75e5637f93f53ad98934b05fd Mon Sep 17 00:00:00 2001 From: z8674558 Date: Wed, 3 Mar 2021 02:34:24 +0900 Subject: [PATCH 40/52] chore(src): fix elvis --- src/emqx_channel.erl | 2 +- src/emqx_tracer.erl | 29 +++++++++++++++++------------ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 9bfba11c9..4d66d5cdc 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -230,7 +230,7 @@ setting_peercert_infos(Peercert, ClientInfo, Options) -> -dialyzer([{nowarn_function, [peer_cert_as/5]}]). % esockd_peercert:peercert is opaque -% https://github.com/emqx/esockd/blob/9b959fc11a1c398a589892f335235be6c5b4a454/src/esockd_peercert.erl#L23 +% https://github.com/emqx/esockd/blob/master/src/esockd_peercert.erl peer_cert_as(Key, Options, Peercert, DN, CN) -> case proplists:get_value(Key, Options) of cn -> CN; diff --git a/src/emqx_tracer.erl b/src/emqx_tracer.erl index ef29732a0..ea9247531 100644 --- a/src/emqx_tracer.erl +++ b/src/emqx_tracer.erl @@ -33,21 +33,21 @@ -define(TRACER, ?MODULE). -define(FORMAT, {emqx_logger_formatter, #{template => - [time," [",level,"] ", + [time, " [", level, "] ", {clientid, [{peername, - [clientid,"@",peername," "], + [clientid, "@", peername, " "], [clientid, " "]}], [{peername, - [peername," "], + [peername, " "], []}]}, - msg,"\n"]}}). + msg, "\n"]}}). -define(TOPIC_TRACE_ID(T), "trace_topic_"++T). -define(CLIENT_TRACE_ID(C), "trace_clientid_"++C). --define(TOPIC_TRACE(T), {topic,T}). --define(CLIENT_TRACE(C), {clientid,C}). +-define(TOPIC_TRACE(T), {topic, T}). +-define(CLIENT_TRACE(C), {clientid, C}). --define(is_log_level(L), +-define(IS_LOG_LEVEL(L), L =:= emergency orelse L =:= alert orelse L =:= critical orelse @@ -67,19 +67,24 @@ trace(publish, #message{topic = <<"$SYS/", _/binary>>}) -> ignore; trace(publish, #message{from = From, topic = Topic, payload = Payload}) when is_binary(From); is_atom(From) -> - emqx_logger:info(#{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY} }, "PUBLISH to ~s: ~0p", [Topic, Payload]). + emqx_logger:info(#{topic => Topic, + mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY} }, + "PUBLISH to ~s: ~0p", [Topic, Payload]). %% @doc Start to trace clientid or topic. -spec(start_trace(trace_who(), logger:level() | all, string()) -> ok | {error, term()}). start_trace(Who, all, LogFile) -> start_trace(Who, debug, LogFile); start_trace(Who, Level, LogFile) -> - case ?is_log_level(Level) of + case ?IS_LOG_LEVEL(Level) of true -> #{level := PrimaryLevel} = logger:get_primary_config(), try logger:compare_levels(Level, PrimaryLevel) of lt -> - {error, io_lib:format("Cannot trace at a log level (~s) lower than the primary log level (~s)", [Level, PrimaryLevel])}; + {error, + io_lib:format("Cannot trace at a log level (~s) " + "lower than the primary log level (~s)", + [Level, PrimaryLevel])}; _GtOrEq -> install_trace_handler(Who, Level, LogFile) catch @@ -127,9 +132,9 @@ uninstall_trance_handler(Who) -> filter_traces(#{id := Id, level := Level, dst := Dst}, Acc) -> case atom_to_list(Id) of ?TOPIC_TRACE_ID(T)-> - [{?TOPIC_TRACE(T), {Level,Dst}} | Acc]; + [{?TOPIC_TRACE(T), {Level, Dst}} | Acc]; ?CLIENT_TRACE_ID(C) -> - [{?CLIENT_TRACE(C), {Level,Dst}} | Acc]; + [{?CLIENT_TRACE(C), {Level, Dst}} | Acc]; _ -> Acc end. From f0d42bc6f55969502982fe2cb64ddf905eb891cd Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 3 Mar 2021 18:27:07 +0800 Subject: [PATCH 41/52] build(CI): fix build error on windows in github actions --- .github/workflows/build_packages.yaml | 13 ------------- pkg-vsn.sh | 2 +- scripts/elvis-check.sh | 2 +- scripts/ensure-rebar3.sh | 2 +- scripts/get-dashboard.sh | 2 +- scripts/shellcheck.sh | 3 ++- scripts/start-two-nodes-in-docker.sh | 2 +- 7 files changed, 7 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 65bbe8542..e54cd5c92 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -16,8 +16,6 @@ jobs: windows: runs-on: windows-2019 - if: startsWith(github.ref, 'refs/tags/') - steps: - uses: actions/checkout@v1 - uses: ilammy/msvc-dev-cmd@v1 @@ -261,17 +259,6 @@ jobs: steps: - uses: actions/checkout@v1 - - name: get deps - env: - ERL_OTP: erl23.2.2 - run: | - docker run -i --rm \ - -e GITHUB_RUN_ID=$GITHUB_RUN_ID \ - -e GITHUB_REF=$GITHUB_REF \ - -v $(pwd):/emqx \ - -w /emqx \ - emqx/build-env:${ERL_OTP}-alpine-amd64 \ - sh -c "make deps-emqx" - name: build emqx docker image env: ARCH: ${{ matrix.arch[0] }} diff --git a/pkg-vsn.sh b/pkg-vsn.sh index 01b58e3a5..0c1c0cb33 100755 --- a/pkg-vsn.sh +++ b/pkg-vsn.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e -u # This script prints the release version for emqx diff --git a/scripts/elvis-check.sh b/scripts/elvis-check.sh index fe4fa2190..e77e6a785 100755 --- a/scripts/elvis-check.sh +++ b/scripts/elvis-check.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ## This script checks style of changed files. ## Expect argument 1 to be the git compare base. diff --git a/scripts/ensure-rebar3.sh b/scripts/ensure-rebar3.sh index 5612beab4..e19af1283 100755 --- a/scripts/ensure-rebar3.sh +++ b/scripts/ensure-rebar3.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index 3e47490b0..c6fc9d448 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail diff --git a/scripts/shellcheck.sh b/scripts/shellcheck.sh index eebf9049a..5f8cdfd51 100755 --- a/scripts/shellcheck.sh +++ b/scripts/shellcheck.sh @@ -1,4 +1,5 @@ -#!/bin/bash +#!/usr/bin/env bash + set -euo pipefail target_files=() diff --git a/scripts/start-two-nodes-in-docker.sh b/scripts/start-two-nodes-in-docker.sh index 45307ba7f..e0526c26f 100755 --- a/scripts/start-two-nodes-in-docker.sh +++ b/scripts/start-two-nodes-in-docker.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail From ee776c50998d20304de1669397584c7bf6c11d4e Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 3 Mar 2021 19:59:26 +0800 Subject: [PATCH 42/52] chore(CI): update events that trigger workflows --- .github/workflows/build_packages.yaml | 11 ++--------- .github/workflows/check_deps_integrity.yaml | 6 +++--- .github/workflows/run_cts_tests.yaml | 3 --- .github/workflows/run_fvt_tests.yaml | 3 --- .github/workflows/run_test_cases.yaml | 3 --- 5 files changed, 5 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index e54cd5c92..15bdaa3db 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -1,16 +1,14 @@ name: Cross build packages on: + schedule: + - cron: '0 */6 * * *' push: tags: - v* release: types: - published - pull_request: - workflow_dispatch: - repository_dispatch: - types: [run_actions] jobs: windows: @@ -71,8 +69,6 @@ jobs: mac: runs-on: macos-10.15 - if: startsWith(github.ref, 'refs/tags/') - steps: - uses: actions/checkout@v1 - name: prepare @@ -203,7 +199,6 @@ jobs: done cd - - name: build emqx packages - if: (matrix.arch == 'amd64' && matrix.emqx == 'emqx') || startsWith(github.ref, 'refs/tags/') env: ERL_OTP: erl23.2.2 EMQX: ${{ matrix.emqx }} @@ -246,8 +241,6 @@ jobs: docker: runs-on: ubuntu-20.04 - if: startsWith(github.ref, 'refs/tags/') - strategy: matrix: arch: diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index eb0ff1fee..91e25e203 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,9 +5,9 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 + container: emqx/build-env:erl23.2.2-ubuntu20.04 + steps: - uses: actions/checkout@v2 - name: Run check-deps-integrity.escript - run: | - docker run --rm -v "$(pwd):/emqx" emqx/build-env:erl23.2.2-ubuntu20.04 sh -c 'cd /emqx && ./scripts/check-deps-integrity.escript' - + run: ./scripts/check-deps-integrity.escript diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index 90fadc2fe..c0eda97dd 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -8,9 +8,6 @@ on: types: - published pull_request: - workflow_dispatch: - repository_dispatch: - types: [run_actions] jobs: ldap: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 761daa237..04638ca08 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -8,9 +8,6 @@ on: types: - published pull_request: - workflow_dispatch: - repository_dispatch: - types: [run_actions] jobs: docker_test: diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 745deb7f9..5095d2428 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -8,9 +8,6 @@ on: types: - published pull_request: - workflow_dispatch: - repository_dispatch: - types: [run_actions] jobs: run_test_case: From 7a915efaa2db2130224d1f40d396a868fd733fbf Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 3 Mar 2021 21:08:53 +0800 Subject: [PATCH 43/52] build(CI): update build packages workflows --- .../upload_github_release_asset.sh | 62 ------------------- .github/workflows/build_packages.yaml | 35 +++++------ 2 files changed, 16 insertions(+), 81 deletions(-) delete mode 100755 .ci/build_packages/upload_github_release_asset.sh diff --git a/.ci/build_packages/upload_github_release_asset.sh b/.ci/build_packages/upload_github_release_asset.sh deleted file mode 100755 index 42dc7e4ef..000000000 --- a/.ci/build_packages/upload_github_release_asset.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -# -# Author: Stefan Buck -# License: MIT -# https://gist.github.com/stefanbuck/ce788fee19ab6eb0b4447a85fc99f447 -# -# -# This script accepts the following parameters: -# -# * owner -# * repo -# * tag -# * filename -# * github_api_token -# -# Script to upload a release asset using the GitHub API v3. -# -# Example: -# -# upload-github-release-asset.sh github_api_token=TOKEN owner=stefanbuck repo=playground tag=v0.1.0 filename=./build.zip -# - -# Check dependencies. -set -e -xargs=$(which gxargs || which xargs) - -# Validate settings. -[ "$TRACE" ] && set -x - -CONFIG=$@ - -for line in $CONFIG; do - eval "$line" -done - -# Define variables. -GH_API="https://api.github.com" -GH_REPO="$GH_API/repos/$owner/$repo" -GH_TAGS="$GH_REPO/releases/tags/$tag" -AUTH="Authorization: token $github_api_token" -WGET_ARGS="--content-disposition --auth-no-challenge --no-cookie" -CURL_ARGS="-LJO#" - -if [[ "$tag" == 'LATEST' ]]; then - GH_TAGS="$GH_REPO/releases/latest" -fi - -# Validate token. -curl -o /dev/null -sH "$AUTH" $GH_REPO || { echo "Error: Invalid repo, token or network issue!"; exit 1; } - -# Read asset tags. -response=$(curl -sH "$AUTH" $GH_TAGS) - -# Get ID of the asset based on given filename. -eval $(echo "$response" | grep -m 1 "id.:" | grep -w id | tr : = | tr -cd '[[:alnum:]]=') -[ "$id" ] || { echo "Error: Failed to get release id for tag: $tag"; echo "$response" | awk 'length($0)<100' >&2; exit 1; } - -# Upload asset -# Construct url -GH_ASSET="https://uploads.github.com/repos/$owner/$repo/releases/$id/assets?name=$(basename $filename)" - -curl "$GITHUB_OAUTH_BASIC" --data-binary @"$filename" -H "Authorization: token $github_api_token" -H "Content-Type: application/octet-stream" $GH_ASSET diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 15bdaa3db..0038ed9e8 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -281,6 +281,10 @@ jobs: if: startsWith(github.ref, 'refs/tags/') steps: + - user: get_version + run: | + version=$(echo ${{ github.ref }} | sed -r "s ^refs/heads/|^refs/tags/(.*) \1 g") + echo "VERSION=$verison" >> $GITHUB_ENV - uses: actions/download-artifact@v2 with: name: emqx @@ -305,17 +309,13 @@ jobs: - name: upload aws s3 run: | set -e -x -u - version=$(echo ${{ github.ref }} | sed -r "s ^refs/heads/|^refs/tags/(.*) \1 g") - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip -q awscliv2.zip - sudo ./aws/install aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws configure set default.region us-west-2 - aws s3 cp --recursive _packages/emqx s3://packages.emqx/emqx-ce/$version - aws s3 cp --recursive _packages/emqx-edge s3://packages.emqx/emqx-edge/$version - aws cloudfront create-invalidation --distribution-id E170YEULGLT8XB --paths "/emqx-ce/$version/*,/emqx-edge/$version/*" + aws s3 cp --recursive _packages/emqx s3://packages.emqx/emqx-ce/$VERSION + aws s3 cp --recursive _packages/emqx-edge s3://packages.emqx/emqx-edge/$VERSION + aws cloudfront create-invalidation --distribution-id E170YEULGLT8XB --paths "/emqx-ce/$VERSION/*,/emqx-edge/$VERSION/*" mkdir packages mv _packages/emqx/* packages @@ -323,28 +323,27 @@ jobs: - uses: actions/checkout@v2 with: path: emqx - - name: update to github and emqx.io + - uses: Rory-Z/upload-release-asset@v1 + if: github.event_name == 'release' + with: + repo: emqx + path: "packages/emqx-*" + token: ${{ secrets.AccessToken }} + - name: update to emqx.io if: github.event_name == 'release' run: | set -e -x -u - version=$(echo ${{ github.ref }} | sed -r "s ^refs/heads/|^refs/tags/(.*) \1 g") - cd packages - for var in $(ls); do - ../emqx/.ci/build_packages/upload_github_release_asset.sh owner=emqx repo=emqx tag=$version filename=$var github_api_token=$(echo ${{ secrets.AccessToken }}) - sleep 1 - done curl -w %{http_code} \ --insecure \ -H "Content-Type: application/json" \ -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \ -X POST \ - -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${version}\" }" \ + -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${VERSION}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - name: push docker image to docker hub if: github.event_name == 'release' run: | set -e -x -u - version=$(echo ${{ github.ref }} | sed -r "s ^refs/heads/|^refs/tags/(.*) \1 g") sudo make -C emqx docker-prepare cd packages && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd - echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin @@ -356,12 +355,11 @@ jobs: if: github.event_name == 'release' run: | set -e -x -u - version=$(echo ${{ github.ref }} | sed -r "s ^refs/heads/|^refs/tags/(.*) \1 g") curl \ -H "Authorization: token ${{ secrets.AccessToken }}" \ -H "Accept: application/vnd.github.v3+json" \ -X POST \ - -d "{\"ref\":\"v1.0.0\",\"inputs\":{\"version\": \"${version}\", \"emqx_ce\": \"true\"}}" \ + -d "{\"ref\":\"v1.0.0\",\"inputs\":{\"version\": \"${VERSION}\", \"emqx_ce\": \"true\"}}" \ https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_repos.yaml/dispatches - uses: geekyeggo/delete-artifact@v1 with: @@ -371,7 +369,6 @@ jobs: name: emqx-edge # - name: update homebrew packages # run: | - # version=$(echo ${{ github.ref }} | sed -r "s .*/.*/(.*) \1 g") # if [ ! -z $(echo $version | grep -oE "v[0-9]+\.[0-9]+(\.[0-9]+)?") ] && [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then # curl -H "Authorization: token ${{ secrets.AccessToken }}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" -X POST -d "{\"event_type\":\"update_homebrew\",\"client_payload\":{\"version\": \"$version\"}}" https://api.github.com/repos/emqx/emqx-packages-docker/dispatches # fi From f391e039c99749e331afeb5fd6250031af565959 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 3 Mar 2021 21:41:09 +0800 Subject: [PATCH 44/52] build(CI): fix syntax error for actions --- .github/workflows/build_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 0038ed9e8..5485a9fcd 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -281,7 +281,7 @@ jobs: if: startsWith(github.ref, 'refs/tags/') steps: - - user: get_version + - name: get_version run: | version=$(echo ${{ github.ref }} | sed -r "s ^refs/heads/|^refs/tags/(.*) \1 g") echo "VERSION=$verison" >> $GITHUB_ENV From a368453768f0a3de0732eadcd0d311123022f3b2 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 3 Mar 2021 15:15:33 +0100 Subject: [PATCH 45/52] fix(mgmt): typo in ifdef macro --- apps/emqx_management/src/emqx_mgmt_data_backup.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 09e270fbb..92791e2b2 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -20,7 +20,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("kernel/include/file.hrl"). --ifdef(EMQX_ENTERPISE). +-ifdef(EMQX_ENTERPRISE). -export([ export_modules/0 , export_schemas/0 , export_confs/0 @@ -123,7 +123,7 @@ export_acl_mnesia() -> end, ets:tab2list(emqx_acl)) end. --ifdef(EMQX_ENTERPISE). +-ifdef(EMQX_ENTERPRISE). export_modules() -> case ets:info(emqx_modules) of undefined -> []; From 284fcab12d732b4e9576f23542370866f5c1d7de Mon Sep 17 00:00:00 2001 From: z8674558 Date: Wed, 3 Mar 2021 22:43:11 +0900 Subject: [PATCH 46/52] test(.github): run dialyzer on pull request --- .github/workflows/run_dialyzer.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/workflows/run_dialyzer.yaml diff --git a/.github/workflows/run_dialyzer.yaml b/.github/workflows/run_dialyzer.yaml new file mode 100644 index 000000000..575326d2d --- /dev/null +++ b/.github/workflows/run_dialyzer.yaml @@ -0,0 +1,12 @@ +name: Run dialyzer +on: pull_request +jobs: + run_dialyzer: + runs-on: ubuntu-latest + container: + image: erlang:23.2 + steps: + - uses: actions/checkout@v1 + - name: Code dialyzer + run: | + make dialyzer From eb03c343bc22bbba47e2a8fa2f56cc1229c67c4f Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 3 Mar 2021 18:40:52 +0100 Subject: [PATCH 47/52] fix(mgmt): Wrong ifdef EMQX_ENTERPRISE compile scope --- .../src/emqx_mgmt_data_backup.erl | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 92791e2b2..cd6ef6fb9 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -174,12 +174,8 @@ export_confs() -> confs_to_binary(Confs) -> [{list_to_binary(Key), list_to_binary(Val)} || {Key, Val} <-Confs]. --endif. +-else. -import_rules(Rules) -> - lists:foreach(fun(Resource) -> - import_resource(Resource) - end, Rules). import_rule(#{<<"id">> := RuleId, <<"rawsql">> := RawSQL, <<"actions">> := Actions, @@ -195,6 +191,22 @@ import_rule(#{<<"id">> := RuleId, emqx_rule_engine:create_rule(Rule#{enabled => false}) end. +map_to_actions(Maps) -> + [map_to_action(M) || M <- Maps]. + +map_to_action(Map = #{<<"id">> := ActionInstId, <<"name">> := Name, <<"args">> := Args}) -> + #{id => ActionInstId, + name => any_to_atom(Name), + args => Args, + fallbacks => map_to_actions(maps:get(<<"fallbacks">>, Map, []))}. + +-endif. + +import_rules(Rules) -> + lists:foreach(fun(Resource) -> + import_resource(Resource) + end, Rules). + import_resources(Reources) -> lists:foreach(fun(Resource) -> import_resource(Resource) @@ -274,6 +286,8 @@ apply_new_config([Action = #{<<"name">> := <<"data_to_webserver">>, apply_new_config(More, Configs, [Action#{<<"args">> := Args} | Acc]) end. +-endif. + actions_to_prop_list(Actions) -> [action_to_prop_list(Act) || Act <- Actions]. @@ -282,7 +296,6 @@ action_to_prop_list({action_instance, ActionInstId, Name, FallbackActions, Args} {name, Name}, {fallbacks, actions_to_prop_list(FallbackActions)}, {args, Args}]. --endif. import_blacklist(Blacklist) -> lists:foreach(fun(#{<<"who">> := Who, @@ -460,15 +473,6 @@ any_to_atom(L) when is_list(L) -> list_to_atom(L); any_to_atom(B) when is_binary(B) -> binary_to_atom(B, utf8); any_to_atom(A) when is_atom(A) -> A. -map_to_actions(Maps) -> - [map_to_action(M) || M <- Maps]. - -map_to_action(Map = #{<<"id">> := ActionInstId, <<"name">> := Name, <<"args">> := Args}) -> - #{id => ActionInstId, - name => any_to_atom(Name), - args => Args, - fallbacks => map_to_actions(maps:get(<<"fallbacks">>, Map, []))}. - to_version(Version) when is_integer(Version) -> integer_to_list(Version); to_version(Version) when is_binary(Version) -> @@ -556,7 +560,7 @@ do_import_data(Data, Version) -> import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version). -ifdef(EMQX_ENTERPRISE). -do_import_extra_data(Data, Version) -> +do_import_extra_data(Data, _Version) -> import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])), import_modules(maps:get(<<"modules">>, Data, [])), import_schemas(maps:get(<<"schemas">>, Data, [])), From 63c001a7aacea0cf8b95014e71921e61e97e2fee Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sun, 28 Feb 2021 21:54:41 +0100 Subject: [PATCH 48/52] refactor(http-lib): Add emqx_http_lib So far only uri_encode and uri_decode APIs --- .../test/emqx_acl_mnesia_SUITE.erl | 6 +- apps/emqx_coap/src/emqx_coap_ps_resource.erl | 10 +-- src/emqx_http_lib.erl | 69 +++++++++++++++++++ test/emqx_http_lib_tests.erl | 46 +++++++++++++ 4 files changed, 117 insertions(+), 14 deletions(-) create mode 100644 src/emqx_http_lib.erl create mode 100644 test/emqx_http_lib_tests.erl diff --git a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl index b2c0b8b5f..13b041491 100644 --- a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl @@ -250,8 +250,4 @@ uri(Parts) when is_list(Parts) -> NParts = [b2l(E) || E <- Parts], ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION, "acl"| NParts]). -%% @private -b2l(B) when is_binary(B) -> - http_uri:encode(binary_to_list(B)); -b2l(L) when is_list(L) -> - http_uri:encode(L). +b2l(B) -> binary_to_list(emqx_http_lib:uri_encode(iolist_to_binary(B))). diff --git a/apps/emqx_coap/src/emqx_coap_ps_resource.erl b/apps/emqx_coap/src/emqx_coap_ps_resource.erl index 144dba1bd..b2169521a 100644 --- a/apps/emqx_coap/src/emqx_coap_ps_resource.erl +++ b/apps/emqx_coap/src/emqx_coap_ps_resource.erl @@ -241,7 +241,7 @@ handle_received_publish(Topic, MaxAge, Format, Payload) -> handle_received_create(TopicPrefix, MaxAge, Payload) -> case core_link:decode(Payload) of [{rootless, [Topic], [{ct, CT}]}] when is_binary(Topic), Topic =/= <<>> -> - TrueTopic = percent_decode(Topic), + TrueTopic = emqx_http_lib:uri_decode(Topic), ?LOG(debug, "decoded link-format payload, the Topic=~p, CT=~p~n", [TrueTopic, CT]), LocPath = concatenate_location_path([<<"ps">>, TopicPrefix, TrueTopic]), FullTopic = binary:part(LocPath, 4, byte_size(LocPath)-4), @@ -259,14 +259,6 @@ handle_received_create(TopicPrefix, MaxAge, Payload) -> {error, bad_request} end. -%% @private Copy from http_uri.erl which has been deprecated since OTP-23 -percent_decode(<<$%, Hex:2/binary, Rest/bits>>) -> - <<(binary_to_integer(Hex, 16)), (percent_decode(Rest))/binary>>; -percent_decode(<>) -> - <>; -percent_decode(<<>>) -> - <<>>. - %% When topic is timeout, server should return nocontent here, %% but gen_coap only receive return value of #coap_content from coap_get, so temporarily we can't give the Code 2.07 {ok, nocontent} out.TBC!!! return_resource(Topic, Payload, MaxAge, TimeStamp, Content) -> diff --git a/src/emqx_http_lib.erl b/src/emqx_http_lib.erl new file mode 100644 index 000000000..695879236 --- /dev/null +++ b/src/emqx_http_lib.erl @@ -0,0 +1,69 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_http_lib). + +-export([uri_encode/1, uri_decode/1]). + +%% @doc Decode percent-encoded URI. +%% This is copied from http_uri.erl which has been deprecated since OTP-23 +%% The recommended replacement uri_string function is not quite equivalent +%% and not backward compatible. +-spec uri_decode(binary()) -> binary(). +uri_decode(<<$%, Hex:2/binary, Rest/bits>>) -> + <<(binary_to_integer(Hex, 16)), (uri_decode(Rest))/binary>>; +uri_decode(<>) -> + <>; +uri_decode(<<>>) -> + <<>>. + +%% @doc Encode URI. +-spec uri_encode(binary()) -> binary(). +uri_encode(URI) when is_binary(URI) -> + << <<(uri_encode_binary(Char))/binary>> || <> <= URI >>. + +uri_encode_binary(Char) -> + case reserved(Char) of + true -> + << $%, (integer_to_binary(Char, 16))/binary >>; + false -> + <> + end. + +reserved($;) -> true; +reserved($:) -> true; +reserved($@) -> true; +reserved($&) -> true; +reserved($=) -> true; +reserved($+) -> true; +reserved($,) -> true; +reserved($/) -> true; +reserved($?) -> true; +reserved($#) -> true; +reserved($[) -> true; +reserved($]) -> true; +reserved($<) -> true; +reserved($>) -> true; +reserved($\") -> true; +reserved(${) -> true; +reserved($}) -> true; +reserved($|) -> true; +reserved($\\) -> true; +reserved($') -> true; +reserved($^) -> true; +reserved($%) -> true; +reserved($\s) -> true; +reserved(_) -> false. diff --git a/test/emqx_http_lib_tests.erl b/test/emqx_http_lib_tests.erl new file mode 100644 index 000000000..78a647417 --- /dev/null +++ b/test/emqx_http_lib_tests.erl @@ -0,0 +1,46 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_http_lib_tests). + +-include_lib("proper/include/proper.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +uri_encode_decode_test_() -> + Opts = [{numtests, 1000}, {to_file, user}], + {timeout, 10, + fun() -> ?assert(proper:quickcheck(prop_run(), Opts)) end}. + +prop_run() -> + ?FORALL(Generated, prop_uri(), test_prop_uri(iolist_to_binary(Generated))). + +prop_uri() -> + proper_types:non_empty(proper_types:list(proper_types:union([prop_char(), prop_reserved()]))). + +prop_char() -> proper_types:integer(32, 126). + +prop_reserved() -> + proper_types:oneof([$;, $:, $@, $&, $=, $+, $,, $/, $?, + $#, $[, $], $<, $>, $\", ${, $}, $|, + $\\, $', $^, $%, $ ]). + +test_prop_uri(URI) -> + Encoded = emqx_http_lib:uri_encode(URI), + Decoded1 = emqx_http_lib:uri_decode(Encoded), + ?assertEqual(URI, Decoded1), + Decoded2 = uri_string:percent_decode(Encoded), + ?assertEqual(URI, Decoded2), + true. From 7d2afbe2afd7330a986785f50cedb10dc952191a Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 4 Mar 2021 08:07:53 +0800 Subject: [PATCH 49/52] build(CI): fix syntax error for actions --- .github/workflows/build_packages.yaml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 5485a9fcd..5c7ff8cfa 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -283,8 +283,9 @@ jobs: steps: - name: get_version run: | - version=$(echo ${{ github.ref }} | sed -r "s ^refs/heads/|^refs/tags/(.*) \1 g") - echo "VERSION=$verison" >> $GITHUB_ENV + echo 'version<> $GITHUB_ENV + echo ${{ github.ref }} | sed -r "s ^refs/heads/|^refs/tags/(.*) \1 g" >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV - uses: actions/download-artifact@v2 with: name: emqx @@ -313,9 +314,9 @@ jobs: aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws configure set default.region us-west-2 - aws s3 cp --recursive _packages/emqx s3://packages.emqx/emqx-ce/$VERSION - aws s3 cp --recursive _packages/emqx-edge s3://packages.emqx/emqx-edge/$VERSION - aws cloudfront create-invalidation --distribution-id E170YEULGLT8XB --paths "/emqx-ce/$VERSION/*,/emqx-edge/$VERSION/*" + aws s3 cp --recursive _packages/emqx s3://packages.emqx/emqx-ce/${{ env.version }} + aws s3 cp --recursive _packages/emqx-edge s3://packages.emqx/emqx-edge/${{ env.version }} + aws cloudfront create-invalidation --distribution-id E170YEULGLT8XB --paths "/emqx-ce/${{ env.version }}/*,/emqx-edge/${{ env.version }}/*" mkdir packages mv _packages/emqx/* packages @@ -338,7 +339,7 @@ jobs: -H "Content-Type: application/json" \ -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \ -X POST \ - -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${VERSION}\" }" \ + -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - name: push docker image to docker hub if: github.event_name == 'release' @@ -359,7 +360,7 @@ jobs: -H "Authorization: token ${{ secrets.AccessToken }}" \ -H "Accept: application/vnd.github.v3+json" \ -X POST \ - -d "{\"ref\":\"v1.0.0\",\"inputs\":{\"version\": \"${VERSION}\", \"emqx_ce\": \"true\"}}" \ + -d "{\"ref\":\"v1.0.0\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \ https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_repos.yaml/dispatches - uses: geekyeggo/delete-artifact@v1 with: From 67c8b2eed763216970e0c75f89b0f7c6dddb42c9 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 4 Mar 2021 08:10:48 +0800 Subject: [PATCH 50/52] chore: update emqx.app.src version --- src/emqx.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqx.app.src b/src/emqx.app.src index e7689a2b7..70d64fff8 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -1,7 +1,7 @@ {application, emqx, [{description, "EMQ X Broker"}, {id, "emqx"}, - {vsn, "4.3-alpha.1"}, % strict semver, bump manually! + {vsn, "4.3-beta.1"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon]}, From 9f0c88cda98cb390894e3aba9864c323004b4f76 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 4 Mar 2021 14:48:06 +0800 Subject: [PATCH 51/52] chore(CI): update workflows change target update repos event in build packages workflows merge dialyzer test to run test case workflows --- .ci/apps_tests/conf.env | 12 +++++++++ .ci/apps_tests/docker-compose.yaml | 2 ++ .github/workflows/build_packages.yaml | 25 +++++++++++-------- .github/workflows/run_dialyzer.yaml | 12 --------- .github/workflows/run_test_cases.yaml | 36 +++++++++++++-------------- 5 files changed, 46 insertions(+), 41 deletions(-) create mode 100644 .ci/apps_tests/conf.env delete mode 100644 .github/workflows/run_dialyzer.yaml diff --git a/.ci/apps_tests/conf.env b/.ci/apps_tests/conf.env new file mode 100644 index 000000000..eb5a98277 --- /dev/null +++ b/.ci/apps_tests/conf.env @@ -0,0 +1,12 @@ +EMQX_AUTH__LDAP__SERVERS=ldap_server +EMQX_AUTH__MONGO__SERVER=mongo_server:27017 +EMQX_AUTH__REDIS__SERVER=redis_server:6379 +EMQX_AUTH__MYSQL__SERVER=mysql_server:3306 +EMQX_AUTH__MYSQL__USERNAME=root +EMQX_AUTH__MYSQL__PASSWORD=public +EMQX_AUTH__MYSQL__DATABASE=mqtt +EMQX_AUTH__PGSQL__SERVER=pgsql_server:5432 +EMQX_AUTH__PGSQL__USERNAME=root +EMQX_AUTH__PGSQL__PASSWORD=public +EMQX_AUTH__PGSQL__DATABASE=mqtt +CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ diff --git a/.ci/apps_tests/docker-compose.yaml b/.ci/apps_tests/docker-compose.yaml index 0ef8d64c5..b8f84821d 100644 --- a/.ci/apps_tests/docker-compose.yaml +++ b/.ci/apps_tests/docker-compose.yaml @@ -12,6 +12,8 @@ services: - ldap_server networks: - emqx_bridge + env_file: + - conf.env environment: GITHUB_ACTIONS: ${GITHUB_ACTIONS} GITHUB_TOKEN: ${GITHUB_TOKEN} diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 5c7ff8cfa..057435e4e 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -329,7 +329,7 @@ jobs: with: repo: emqx path: "packages/emqx-*" - token: ${{ secrets.AccessToken }} + token: ${{ github.token }} - name: update to emqx.io if: github.event_name == 'release' run: | @@ -355,21 +355,26 @@ jobs: - name: update repo.emqx.io if: github.event_name == 'release' run: | - set -e -x -u - curl \ + curl --silent --show-error \ -H "Authorization: token ${{ secrets.AccessToken }}" \ -H "Accept: application/vnd.github.v3+json" \ -X POST \ - -d "{\"ref\":\"v1.0.0\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \ - https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_repos.yaml/dispatches + -d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \ + "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" + - name: update homebrew packages + if: github.event_name == 'release' + run: | + if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then + curl --silent --show-error \ + -H "Authorization: token ${{ secrets.AccessToken }}" \ + -H "Accept: application/vnd.github.v3+json" \ + -X POST \ + -d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \ + "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches" + fi - uses: geekyeggo/delete-artifact@v1 with: name: emqx - uses: geekyeggo/delete-artifact@v1 with: name: emqx-edge - # - name: update homebrew packages - # run: | - # if [ ! -z $(echo $version | grep -oE "v[0-9]+\.[0-9]+(\.[0-9]+)?") ] && [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then - # curl -H "Authorization: token ${{ secrets.AccessToken }}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" -X POST -d "{\"event_type\":\"update_homebrew\",\"client_payload\":{\"version\": \"$version\"}}" https://api.github.com/repos/emqx/emqx-packages-docker/dispatches - # fi diff --git a/.github/workflows/run_dialyzer.yaml b/.github/workflows/run_dialyzer.yaml deleted file mode 100644 index 575326d2d..000000000 --- a/.github/workflows/run_dialyzer.yaml +++ /dev/null @@ -1,12 +0,0 @@ -name: Run dialyzer -on: pull_request -jobs: - run_dialyzer: - runs-on: ubuntu-latest - container: - image: erlang:23.2 - steps: - - uses: actions/checkout@v1 - - name: Code dialyzer - run: | - make dialyzer diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 5095d2428..6adf099cb 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,7 +10,18 @@ on: pull_request: jobs: - run_test_case: + run_static_analysis: + runs-on: ubuntu-20.04 + container: emqx/build-env:erl23.2.2-ubuntu20.04 + + steps: + - uses: actions/checkout@v2 + - name: xref + run: make xref + - name: dialyzer + run: make dialyzer + + run_common_test: runs-on: ubuntu-20.04 steps: @@ -26,25 +37,12 @@ jobs: run: | docker-compose -f .ci/apps_tests/docker-compose.yaml build --no-cache docker-compose -f .ci/apps_tests/docker-compose.yaml up -d - - - name: run tests + - name: run eunit + run: docker exec -i erlang bash -c "make eunit" + - name: run common test + run: docker exec -i erlang bash -c "make ct" + - name: run cover run: | - export EMQX_AUTH__LDAP__SERVERS=ldap_server \ - EMQX_AUTH__MONGO__SERVER=mongo_server:27017 \ - EMQX_AUTH__REDIS__SERVER=redis_server:6379 \ - EMQX_AUTH__MYSQL__SERVER=mysql_server:3306 \ - EMQX_AUTH__MYSQL__USERNAME=root \ - EMQX_AUTH__MYSQL__PASSWORD=public \ - EMQX_AUTH__MYSQL__DATABASE=mqtt \ - EMQX_AUTH__PGSQL__SERVER=pgsql_server:5432 \ - EMQX_AUTH__PGSQL__USERNAME=root \ - EMQX_AUTH__PGSQL__PASSWORD=public \ - EMQX_AUTH__PGSQL__DATABASE=mqtt \ - CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ - printenv > .env - docker exec -i erlang bash -c "make xref" - docker exec --env-file .env -i erlang bash -c "make ct" - docker exec --env-file .env -i erlang bash -c "make eunit" docker exec -i erlang bash -c "make cover" docker exec -i erlang bash -c "make coveralls" - uses: actions/upload-artifact@v1 From 059d9fcaebb67c8759903940802c9f7d808abbe4 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 3 Mar 2021 14:58:42 +0800 Subject: [PATCH 52/52] build(dashboard): get dashboard script support enterprise --- .gitignore | 1 + deploy/docker/Dockerfile | 3 ++- scripts/get-dashboard.sh | 38 ++++++++++++++++++++++++++++---------- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 387a3ff90..59ebdd433 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,4 @@ elvis emqx_dialyzer_*_plt */emqx_dashboard/priv/www dist.zip +scripts/git-token diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 860446b8f..b3ca6f4c8 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -18,7 +18,8 @@ RUN apk add --no-cache \ bsd-compat-headers \ libc-dev \ libstdc++ \ - bash + bash \ + jq COPY . /emqx diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index c6fc9d448..a0f484d12 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -5,18 +5,17 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." -if [[ "$1" == https://* ]]; then - VERSION='*' # alwyas download - DOWNLOAD_URL="$1" -else - VERSION="$1" - DOWNLOAD_URL="https://github.com/emqx/emqx-dashboard-frontend/releases/download/${VERSION}/emqx-dashboard.zip" -fi +VERSION="$1" +RELEASE_ASSET_FILE="emqx-dashboard.zip" if [ -f 'EMQX_ENTERPRISE' ]; then DASHBOARD_PATH='lib-ee/emqx_dashboard/priv' + DASHBOARD_REPO='emqx-enterprise-dashboard-frontend-src' + AUTH="Authorization: token $(cat scripts/git-token)" else DASHBOARD_PATH='lib-ce/emqx_dashboard/priv' + DASHBOARD_REPO='emqx-dashboard-frontend' + AUTH="" fi case $(uname) in @@ -32,8 +31,27 @@ if [ -d "$DASHBOARD_PATH/www" ] && [ "$(version)" = "$VERSION" ]; then exit 0 fi -curl -f -L "${DOWNLOAD_URL}" -o ./emqx-dashboard.zip -unzip -q ./emqx-dashboard.zip -d "$DASHBOARD_PATH" +get_assets(){ + # Get the download URL of our desired asset + download_url="$(curl --silent --show-error \ + --header "${AUTH}" \ + --header "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/emqx/${DASHBOARD_REPO}/releases/tags/${VERSION}" \ + | jq --raw-output ".assets[] | select(.name==\"${RELEASE_ASSET_FILE}\").url")" + # Get GitHub's S3 redirect URL + redirect_url=$(curl --silent --show-error \ + --header "${AUTH}" \ + --header "Accept: application/octet-stream" \ + --write-out "%{redirect_url}" \ + "$download_url") + curl --silent --show-error \ + --header "Accept: application/octet-stream" \ + --output "${RELEASE_ASSET_FILE}" \ + "$redirect_url" +} + +get_assets +unzip -q "$RELEASE_ASSET_FILE" -d "$DASHBOARD_PATH" rm -rf "$DASHBOARD_PATH/www" mv "$DASHBOARD_PATH/dist" "$DASHBOARD_PATH/www" -rm -rf emqx-dashboard.zip +rm -rf "$RELEASE_ASSET_FILE"