From 740ff3587c934a9154f821cbc6ee0db18c7db05c Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 1 Jul 2022 11:44:29 +0800 Subject: [PATCH 1/7] fix: sync data's authz and certs --- apps/emqx_conf/src/emqx_conf_app.erl | 19 +++++++++++++++++++ .../src/proto/emqx_conf_proto_v1.erl | 7 ++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 6e62db325..08fdca43a 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -20,6 +20,7 @@ -export([start/2, stop/1]). -export([get_override_config_file/0]). +-export([sync_data_from_node/0]). -include_lib("emqx/include/logger.hrl"). -include("emqx_conf.hrl"). @@ -56,6 +57,12 @@ get_override_config_file() -> end end. +sync_data_from_node() -> + Dir = emqx:data_dir(), + {ok, Zip} = zip:zip(atom_to_list(node()) ++ "_data.zip", ["authz", "certs"], [{cwd, Dir}]), + Res = {ok, _Bin} = file:read_file(Zip), + Res. + %% ------------------------------------------------------------------------------ %% Internal functions %% ------------------------------------------------------------------------------ @@ -150,6 +157,7 @@ copy_override_conf_from_core_node() -> RawOverrideConf, #{override_to => cluster} ), + ok = sync_data_from_node(Node), {ok, TnxId} end end. @@ -173,3 +181,14 @@ conf_sort({ok, #{tnx_id := Id, wall_clock := W1}}, {ok, #{tnx_id := Id, wall_clo W1 > W2; conf_sort({ok, _}, {ok, _}) -> false. + +sync_data_from_node(Node) -> + case emqx_conf_proto_v1:sync_data_from_node(Node) of + {ok, DataBin} -> + {ok, Files} = zip:unzip(DataBin, [{cwd, emqx:data_dir()}]), + ?SLOG(debug, #{node => Node, msg => "sync_data_from_node_ok", files => Files}), + ok; + Error -> + ?SLOG(emergency, #{node => Node, msg => "sync_data_from_node_failed", reason => Error}), + error(Error) + end. diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl index 97e14b7c4..9b0b816f2 100644 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl @@ -33,7 +33,8 @@ reset/2, reset/3, - get_override_config_file/1 + get_override_config_file/1, + sync_data_from_node/1 ]). -include_lib("emqx/include/bpapi.hrl"). @@ -104,3 +105,7 @@ reset(Node, KeyPath, Opts) -> -spec get_override_config_file([node()]) -> emqx_rpc:multicall_result(). get_override_config_file(Nodes) -> rpc:multicall(Nodes, emqx_conf_app, get_override_config_file, [], 20000). + +-spec sync_data_from_node(node()) -> {ok, binary()} | emqx_rpc:badrpc(). +sync_data_from_node(Node) -> + rpc:call(Node, emqx_conf_app, sync_data_from_node, [], 20000). From 8ba2b6cc180eb48db52f1beb9deb17916a7e4609 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 1 Jul 2022 14:08:16 +0800 Subject: [PATCH 2/7] fix: bpapi chekc failed --- apps/emqx_conf/src/emqx_conf_app.erl | 2 +- .../src/proto/emqx_conf_proto_v1.erl | 7 +--- .../src/proto/emqx_conf_proto_v2.erl | 33 +++++++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 08fdca43a..67d5747a2 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -183,7 +183,7 @@ conf_sort({ok, _}, {ok, _}) -> false. sync_data_from_node(Node) -> - case emqx_conf_proto_v1:sync_data_from_node(Node) of + case emqx_conf_proto_v2:sync_data_from_node(Node) of {ok, DataBin} -> {ok, Files} = zip:unzip(DataBin, [{cwd, emqx:data_dir()}]), ?SLOG(debug, #{node => Node, msg => "sync_data_from_node_ok", files => Files}), diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl index 9b0b816f2..97e14b7c4 100644 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl @@ -33,8 +33,7 @@ reset/2, reset/3, - get_override_config_file/1, - sync_data_from_node/1 + get_override_config_file/1 ]). -include_lib("emqx/include/bpapi.hrl"). @@ -105,7 +104,3 @@ reset(Node, KeyPath, Opts) -> -spec get_override_config_file([node()]) -> emqx_rpc:multicall_result(). get_override_config_file(Nodes) -> rpc:multicall(Nodes, emqx_conf_app, get_override_config_file, [], 20000). - --spec sync_data_from_node(node()) -> {ok, binary()} | emqx_rpc:badrpc(). -sync_data_from_node(Node) -> - rpc:call(Node, emqx_conf_app, sync_data_from_node, [], 20000). diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl new file mode 100644 index 000000000..09d1599cd --- /dev/null +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl @@ -0,0 +1,33 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_conf_proto_v2). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + sync_data_from_node/1 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.1". + +-spec sync_data_from_node(node()) -> {ok, binary()} | emqx_rpc:badrpc(). +sync_data_from_node(Node) -> + rpc:call(Node, emqx_conf_app, sync_data_from_node, [], 20000). From aabdb074477fbdc58522a170e0fec9ca2107df38 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 1 Jul 2022 14:18:03 +0800 Subject: [PATCH 3/7] chore: always delete data zip file --- apps/emqx_conf/src/emqx_conf_app.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 67d5747a2..225446860 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -61,6 +61,7 @@ sync_data_from_node() -> Dir = emqx:data_dir(), {ok, Zip} = zip:zip(atom_to_list(node()) ++ "_data.zip", ["authz", "certs"], [{cwd, Dir}]), Res = {ok, _Bin} = file:read_file(Zip), + _ = file:delete(Zip), Res. %% ------------------------------------------------------------------------------ From 9af470dee9b0ceaf6917114c7c008a048d7d473b Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 1 Jul 2022 15:00:14 +0800 Subject: [PATCH 4/7] chore: upgrade emqx_conf bpapi to version 2 --- apps/emqx/priv/bpapi.versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 80782beca..430517fc2 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -4,7 +4,7 @@ {emqx_bridge,1}. {emqx_broker,1}. {emqx_cm,1}. -{emqx_conf,1}. +{emqx_conf,2}. {emqx_dashboard,1}. {emqx_delayed,1}. {emqx_exhook,1}. From 3d6f98e5f6dbd84d21f54c7d1b1c6c06601027d7 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 1 Jul 2022 15:32:12 +0800 Subject: [PATCH 5/7] fix: filter data's dir --- apps/emqx_conf/src/emqx_conf_app.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 225446860..80ffb4258 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -59,7 +59,10 @@ get_override_config_file() -> sync_data_from_node() -> Dir = emqx:data_dir(), - {ok, Zip} = zip:zip(atom_to_list(node()) ++ "_data.zip", ["authz", "certs"], [{cwd, Dir}]), + TargetDirs = lists:filter(fun(Type) -> filelib:is_dir(filename:join(Dir, Type)) end, [ + "authz", "certs" + ]), + {ok, Zip} = zip:zip(atom_to_list(node()) ++ "_data.zip", TargetDirs, [{cwd, Dir}]), Res = {ok, _Bin} = file:read_file(Zip), _ = file:delete(Zip), Res. From c23ccbd370ef49ef6466d534aab67cc70fd7245d Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 1 Jul 2022 15:46:36 +0800 Subject: [PATCH 6/7] chore: update bpapi.verions --- apps/emqx/priv/bpapi.versions | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 430517fc2..139514dfb 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -4,6 +4,7 @@ {emqx_bridge,1}. {emqx_broker,1}. {emqx_cm,1}. +{emqx_conf,1}. {emqx_conf,2}. {emqx_dashboard,1}. {emqx_delayed,1}. From cbf467ee3dde2e9fc1e8bdc0fccacf579e40b9ad Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 1 Jul 2022 16:39:51 +0800 Subject: [PATCH 7/7] chore: update emqx_conf.appup.src --- apps/emqx_conf/src/emqx_conf.appup.src | 29 ++++--- apps/emqx_conf/src/emqx_conf.erl | 18 ++--- apps/emqx_conf/src/emqx_conf_app.erl | 2 +- .../src/proto/emqx_conf_proto_v2.erl | 78 ++++++++++++++++++- 4 files changed, 107 insertions(+), 20 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf.appup.src b/apps/emqx_conf/src/emqx_conf.appup.src index 688456e1e..1937d1fd9 100644 --- a/apps/emqx_conf/src/emqx_conf.appup.src +++ b/apps/emqx_conf/src/emqx_conf.appup.src @@ -1,10 +1,21 @@ -%% Unless you know what you are doing, DO NOT edit manually!! +%% -*- mode: erlang -*- {VSN, - [{"0.1.0", - [{load_module,emqx_conf_schema,brutal_purge,soft_purge,[]}, - {load_module,emqx_conf,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}], - [{"0.1.0", - [{load_module,emqx_conf_schema,brutal_purge,soft_purge,[]}, - {load_module,emqx_conf,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}]}. + [ {"0.1.0", + [ + {add_module, emqx_conf_proto_v2}, + {load_module,emqx_conf_schema,brutal_purge,soft_purge,[]}, + {load_module,emqx_conf,brutal_purge,soft_purge,[]}, + {load_module,emqx_conf_app,brutal_purge,soft_purge,[]} + ]}, + {<<".*">>, []} + ], + [ {"0.1.0", + [ + {delete_module, emqx_conf_proto_v2}, + {load_module,emqx_conf_schema,brutal_purge,soft_purge,[]}, + {load_module,emqx_conf,brutal_purge,soft_purge,[]}, + {load_module,emqx_conf_app,brutal_purge,soft_purge,[]} + ]}, + {<<".*">>, []} + ] +}. diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index aaddf7c79..bc8b52702 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -62,7 +62,7 @@ get_raw(KeyPath) -> %% @doc Returns all values in the cluster. -spec get_all(emqx_map_lib:config_key_path()) -> #{node() => term()}. get_all(KeyPath) -> - {ResL, []} = emqx_conf_proto_v1:get_all(KeyPath), + {ResL, []} = emqx_conf_proto_v2:get_all(KeyPath), maps:from_list(ResL). %% @doc Returns the specified node's KeyPath, or exception if not found @@ -70,14 +70,14 @@ get_all(KeyPath) -> get_by_node(Node, KeyPath) when Node =:= node() -> emqx:get_config(KeyPath); get_by_node(Node, KeyPath) -> - emqx_conf_proto_v1:get_config(Node, KeyPath). + emqx_conf_proto_v2:get_config(Node, KeyPath). %% @doc Returns the specified node's KeyPath, or the default value if not found -spec get_by_node(node(), emqx_map_lib:config_key_path(), term()) -> term(). get_by_node(Node, KeyPath, Default) when Node =:= node() -> emqx:get_config(KeyPath, Default); get_by_node(Node, KeyPath, Default) -> - emqx_conf_proto_v1:get_config(Node, KeyPath, Default). + emqx_conf_proto_v2:get_config(Node, KeyPath, Default). %% @doc Returns the specified node's KeyPath, or config_not_found if key path not found -spec get_node_and_config(emqx_map_lib:config_key_path()) -> term(). @@ -92,7 +92,7 @@ get_node_and_config(KeyPath) -> ) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. update(KeyPath, UpdateReq, Opts) -> - emqx_conf_proto_v1:update(KeyPath, UpdateReq, Opts). + emqx_conf_proto_v2:update(KeyPath, UpdateReq, Opts). %% @doc Update the specified node's key path in local-override.conf. -spec update( @@ -105,13 +105,13 @@ update(KeyPath, UpdateReq, Opts) -> update(Node, KeyPath, UpdateReq, Opts0) when Node =:= node() -> emqx:update_config(KeyPath, UpdateReq, Opts0#{override_to => local}); update(Node, KeyPath, UpdateReq, Opts) -> - emqx_conf_proto_v1:update(Node, KeyPath, UpdateReq, Opts). + emqx_conf_proto_v2:update(Node, KeyPath, UpdateReq, Opts). %% @doc remove all value of key path in cluster-override.conf or local-override.conf. -spec remove(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove(KeyPath, Opts) -> - emqx_conf_proto_v1:remove_config(KeyPath, Opts). + emqx_conf_proto_v2:remove_config(KeyPath, Opts). %% @doc remove the specified node's key path in local-override.conf. -spec remove(node(), emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> @@ -119,13 +119,13 @@ remove(KeyPath, Opts) -> remove(Node, KeyPath, Opts) when Node =:= node() -> emqx:remove_config(KeyPath, Opts#{override_to => local}); remove(Node, KeyPath, Opts) -> - emqx_conf_proto_v1:remove_config(Node, KeyPath, Opts). + emqx_conf_proto_v2:remove_config(Node, KeyPath, Opts). %% @doc reset all value of key path in cluster-override.conf or local-override.conf. -spec reset(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. reset(KeyPath, Opts) -> - emqx_conf_proto_v1:reset(KeyPath, Opts). + emqx_conf_proto_v2:reset(KeyPath, Opts). %% @doc reset the specified node's key path in local-override.conf. -spec reset(node(), emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> @@ -133,7 +133,7 @@ reset(KeyPath, Opts) -> reset(Node, KeyPath, Opts) when Node =:= node() -> emqx:reset_config(KeyPath, Opts#{override_to => local}); reset(Node, KeyPath, Opts) -> - emqx_conf_proto_v1:reset(Node, KeyPath, Opts). + emqx_conf_proto_v2:reset(Node, KeyPath, Opts). %% @doc Called from build script. -spec dump_schema(file:name_all()) -> ok. diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 80ffb4258..3061223cf 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -97,7 +97,7 @@ copy_override_conf_from_core_node() -> ?SLOG(debug, #{msg => "skip_copy_overide_conf_from_core_node"}), {ok, ?DEFAULT_INIT_TXN_ID}; Nodes -> - {Results, Failed} = emqx_conf_proto_v1:get_override_config_file(Nodes), + {Results, Failed} = emqx_conf_proto_v2:get_override_config_file(Nodes), {Ready, NotReady0} = lists:partition(fun(Res) -> element(1, Res) =:= ok end, Results), NotReady = lists:filter(fun(Res) -> element(1, Res) =:= error end, NotReady0), case (Failed =/= [] orelse NotReady =/= []) andalso Ready =/= [] of diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl index 09d1599cd..acb62eb94 100644 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl @@ -20,7 +20,20 @@ -export([ introduced_in/0, - sync_data_from_node/1 + sync_data_from_node/1, + get_config/2, + get_config/3, + get_all/1, + + update/3, + update/4, + remove_config/2, + remove_config/3, + + reset/2, + reset/3, + + get_override_config_file/1 ]). -include_lib("emqx/include/bpapi.hrl"). @@ -31,3 +44,66 @@ introduced_in() -> -spec sync_data_from_node(node()) -> {ok, binary()} | emqx_rpc:badrpc(). sync_data_from_node(Node) -> rpc:call(Node, emqx_conf_app, sync_data_from_node, [], 20000). +-type update_config_key_path() :: [emqx_map_lib:config_key(), ...]. + +-spec get_config(node(), emqx_map_lib:config_key_path()) -> + term() | emqx_rpc:badrpc(). +get_config(Node, KeyPath) -> + rpc:call(Node, emqx, get_config, [KeyPath]). + +-spec get_config(node(), emqx_map_lib:config_key_path(), _Default) -> + term() | emqx_rpc:badrpc(). +get_config(Node, KeyPath, Default) -> + rpc:call(Node, emqx, get_config, [KeyPath, Default]). + +-spec get_all(emqx_map_lib:config_key_path()) -> emqx_rpc:multicall_result(). +get_all(KeyPath) -> + rpc:multicall(emqx_conf, get_node_and_config, [KeyPath], 5000). + +-spec update( + update_config_key_path(), + emqx_config:update_request(), + emqx_config:update_opts() +) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. +update(KeyPath, UpdateReq, Opts) -> + emqx_cluster_rpc:multicall(emqx, update_config, [KeyPath, UpdateReq, Opts]). + +-spec update( + node(), + update_config_key_path(), + emqx_config:update_request(), + emqx_config:update_opts() +) -> + {ok, emqx_config:update_result()} + | {error, emqx_config:update_error()} + | emqx_rpc:badrpc(). +update(Node, KeyPath, UpdateReq, Opts) -> + rpc:call(Node, emqx, update_config, [KeyPath, UpdateReq, Opts], 5000). + +-spec remove_config(update_config_key_path(), emqx_config:update_opts()) -> + {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. +remove_config(KeyPath, Opts) -> + emqx_cluster_rpc:multicall(emqx, remove_config, [KeyPath, Opts]). + +-spec remove_config(node(), update_config_key_path(), emqx_config:update_opts()) -> + {ok, emqx_config:update_result()} + | {error, emqx_config:update_error()} + | emqx_rpc:badrpc(). +remove_config(Node, KeyPath, Opts) -> + rpc:call(Node, emqx, remove_config, [KeyPath, Opts], 5000). + +-spec reset(update_config_key_path(), emqx_config:update_opts()) -> + {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. +reset(KeyPath, Opts) -> + emqx_cluster_rpc:multicall(emqx, reset_config, [KeyPath, Opts]). + +-spec reset(node(), update_config_key_path(), emqx_config:update_opts()) -> + {ok, emqx_config:update_result()} + | {error, emqx_config:update_error()} + | emqx_rpc:badrpc(). +reset(Node, KeyPath, Opts) -> + rpc:call(Node, emqx, reset_config, [KeyPath, Opts]). + +-spec get_override_config_file([node()]) -> emqx_rpc:multicall_result(). +get_override_config_file(Nodes) -> + rpc:multicall(Nodes, emqx_conf_app, get_override_config_file, [], 20000).