From d976943f991faa14c290b4c8052f40135cc6c7d0 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 6 Jan 2023 22:15:22 +0300 Subject: [PATCH 001/197] feat(ft): add basic hooks --- apps/emqx/src/emqx_cm.erl | 46 ++++++++----- apps/emqx/src/emqx_types.erl | 4 ++ apps/emqx/src/proto/emqx_cm_proto_v2.erl | 88 ++++++++++++++++++++++++ mix.exs | 1 + 4 files changed, 122 insertions(+), 17 deletions(-) create mode 100644 apps/emqx/src/proto/emqx_cm_proto_v2.erl diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index f8c510482..d2ac642ca 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -89,6 +89,7 @@ mark_channel_connected/1, mark_channel_disconnected/1, get_connected_client_count/0, + takeover_finish/2, do_kick_session/3, do_get_chan_stats/2, @@ -171,6 +172,7 @@ register_channel(ClientId, ChanPid, #{conn_mod := ConnMod}) when is_pid(ChanPid) true = ets:insert(?CHAN_CONN_TAB, {Chan, ConnMod}), ok = emqx_cm_registry:register_channel(Chan), mark_channel_connected(ChanPid), + ok = emqx_hooks:run('channel.registered', [ConnMod, ChanPid]), cast({registered, Chan}). %% @doc Unregister a channel. @@ -180,11 +182,13 @@ unregister_channel(ClientId) when is_binary(ClientId) -> ok. %% @private -do_unregister_channel(Chan) -> +do_unregister_channel({_ClientId, ChanPid} = Chan) -> ok = emqx_cm_registry:unregister_channel(Chan), true = ets:delete(?CHAN_CONN_TAB, Chan), true = ets:delete(?CHAN_INFO_TAB, Chan), - ets:delete_object(?CHAN_TAB, Chan). + ets:delete_object(?CHAN_TAB, Chan), + ok = emqx_hooks:run('channel.unregistered', [ChanPid]), + true. -spec connection_closed(emqx_types:clientid()) -> true. connection_closed(ClientId) -> @@ -212,7 +216,7 @@ do_get_chan_info(ClientId, ChanPid) -> -spec get_chan_info(emqx_types:clientid(), chan_pid()) -> maybe(emqx_types:infos()). get_chan_info(ClientId, ChanPid) -> - wrap_rpc(emqx_cm_proto_v1:get_chan_info(ClientId, ChanPid)). + wrap_rpc(emqx_cm_proto_v2:get_chan_info(ClientId, ChanPid)). %% @doc Update infos of the channel. -spec set_chan_info(emqx_types:clientid(), emqx_types:attrs()) -> boolean(). @@ -242,7 +246,7 @@ do_get_chan_stats(ClientId, ChanPid) -> -spec get_chan_stats(emqx_types:clientid(), chan_pid()) -> maybe(emqx_types:stats()). get_chan_stats(ClientId, ChanPid) -> - wrap_rpc(emqx_cm_proto_v1:get_chan_stats(ClientId, ChanPid)). + wrap_rpc(emqx_cm_proto_v2:get_chan_stats(ClientId, ChanPid)). %% @doc Set channel's stats. -spec set_chan_stats(emqx_types:clientid(), emqx_types:stats()) -> boolean(). @@ -278,7 +282,7 @@ open_session(true, ClientInfo = #{clientid := ClientId}, ConnInfo) -> {ok, #{session => Session1, present => false}} end, emqx_cm_locker:trans(ClientId, CleanStart); -open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> +open_session(false, ClientInfo = #{clientid := ClientId}, #{conn_mod := NewConnMod} = ConnInfo) -> Self = self(), ResumeStart = fun(_) -> CreateSess = @@ -304,18 +308,12 @@ open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> }}; {living, ConnMod, ChanPid, Session} -> ok = emqx_session:resume(ClientInfo, Session), - case - request_stepdown( - {takeover, 'end'}, - ConnMod, - ChanPid - ) - of - {ok, Pendings} -> + case wrap_rpc(emqx_cm_proto_v2:takeover_finish(ConnMod, ChanPid)) of + {ok, Pendings, TakoverData} -> Session1 = emqx_persistent_session:persist( ClientInfo, ConnInfo, Session ), - register_channel(ClientId, Self, ConnInfo), + ok = emqx_hooks:run('channel.takeovered', [NewConnMod, Self, TakoverData]), {ok, #{ session => clean_session(Session1), present => true, @@ -400,6 +398,20 @@ takeover_session(ClientId) -> takeover_session(ClientId, ChanPid) end. +takeover_finish(ConnMod, ChanPid) -> + TakoverData = emqx_hooks:run_fold('channel.takeover', [ConnMod, ChanPid], #{}), + case + %% node-local call + request_stepdown( + {takeover, 'end'}, + ConnMod, + ChanPid + ) + of + {ok, Pendings} -> {ok, Pendings, TakoverData}; + {error, _} = Error -> Error + end. + takeover_session(ClientId, Pid) -> try do_takeover_session(ClientId, Pid) @@ -429,7 +441,7 @@ do_takeover_session(ClientId, ChanPid) when node(ChanPid) == node() -> end end; do_takeover_session(ClientId, ChanPid) -> - wrap_rpc(emqx_cm_proto_v1:takeover_session(ClientId, ChanPid)). + wrap_rpc(emqx_cm_proto_v2:takeover_session(ClientId, ChanPid)). %% @doc Discard all the sessions identified by the ClientId. -spec discard_session(emqx_types:clientid()) -> ok. @@ -531,7 +543,7 @@ do_kick_session(Action, ClientId, ChanPid) -> %% @private This function is shared for session 'kick' and 'discard' (as the first arg Action). kick_session(Action, ClientId, ChanPid) -> try - wrap_rpc(emqx_cm_proto_v1:kick_session(Action, ClientId, ChanPid)) + wrap_rpc(emqx_cm_proto_v2:kick_session(Action, ClientId, ChanPid)) catch Error:Reason -> %% This should mostly be RPC failures. @@ -716,7 +728,7 @@ do_get_chann_conn_mod(ClientId, ChanPid) -> end. get_chann_conn_mod(ClientId, ChanPid) -> - wrap_rpc(emqx_cm_proto_v1:get_chann_conn_mod(ClientId, ChanPid)). + wrap_rpc(emqx_cm_proto_v2:get_chann_conn_mod(ClientId, ChanPid)). mark_channel_connected(ChanPid) -> ?tp(emqx_cm_connected_client_count_inc, #{}), diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl index 7223da245..8b7fdae9b 100644 --- a/apps/emqx/src/emqx_types.erl +++ b/apps/emqx/src/emqx_types.erl @@ -101,6 +101,8 @@ -export_type([oom_policy/0]). +-export_type([takeover_data/0]). + -type proto_ver() :: ?MQTT_PROTO_V3 | ?MQTT_PROTO_V4 @@ -242,3 +244,5 @@ max_heap_size => non_neg_integer(), enable => boolean() }. + +-type takeover_data() :: map(). diff --git a/apps/emqx/src/proto/emqx_cm_proto_v2.erl b/apps/emqx/src/proto/emqx_cm_proto_v2.erl new file mode 100644 index 000000000..2981dbd40 --- /dev/null +++ b/apps/emqx/src/proto/emqx_cm_proto_v2.erl @@ -0,0 +1,88 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_cm_proto_v2). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + lookup_client/2, + kickout_client/2, + + get_chan_stats/2, + get_chan_info/2, + get_chann_conn_mod/2, + + takeover_session/2, + takeover_finish/2, + kick_session/3 +]). + +-include("bpapi.hrl"). +-include("src/emqx_cm.hrl"). + +introduced_in() -> + "5.0.0". + +-spec kickout_client(node(), emqx_types:clientid()) -> ok | {badrpc, _}. +kickout_client(Node, ClientId) -> + rpc:call(Node, emqx_cm, kick_session, [ClientId]). + +-spec lookup_client(node(), {clientid, emqx_types:clientid()} | {username, emqx_types:username()}) -> + [emqx_cm:channel_info()] | {badrpc, _}. +lookup_client(Node, Key) -> + rpc:call(Node, emqx_cm, lookup_client, [Key]). + +-spec get_chan_stats(emqx_types:clientid(), emqx_cm:chan_pid()) -> emqx_types:stats() | {badrpc, _}. +get_chan_stats(ClientId, ChanPid) -> + rpc:call(node(ChanPid), emqx_cm, do_get_chan_stats, [ClientId, ChanPid], ?T_GET_INFO * 2). + +-spec get_chan_info(emqx_types:clientid(), emqx_cm:chan_pid()) -> emqx_types:infos() | {badrpc, _}. +get_chan_info(ClientId, ChanPid) -> + rpc:call(node(ChanPid), emqx_cm, do_get_chan_info, [ClientId, ChanPid], ?T_GET_INFO * 2). + +-spec get_chann_conn_mod(emqx_types:clientid(), emqx_cm:chan_pid()) -> + module() | undefined | {badrpc, _}. +get_chann_conn_mod(ClientId, ChanPid) -> + rpc:call(node(ChanPid), emqx_cm, do_get_chann_conn_mod, [ClientId, ChanPid], ?T_GET_INFO * 2). + +-spec takeover_session(emqx_types:clientid(), emqx_cm:chan_pid()) -> + none + | {expired | persistent, emqx_session:session()} + | {living, _ConnMod :: atom(), emqx_cm:chan_pid(), emqx_session:session()} + | {badrpc, _}. +takeover_session(ClientId, ChanPid) -> + rpc:call(node(ChanPid), emqx_cm, takeover_session, [ClientId, ChanPid], ?T_TAKEOVER * 2). + +-spec takeover_finish(module(), emqx_cm:chan_pid()) -> + {ok, emqx_type:takeover_data()} + | {ok, list(emqx_type:deliver()), emqx_type:takeover_data()} + | {error, term()} + | {badrpc, _}. +takeover_finish(ConnMod, ChanPid) -> + erpc:call( + node(ChanPid), + emqx_cm, + takeover_session_finish, + [ConnMod, ChanPid], + ?T_TAKEOVER * 2 + ). + +-spec kick_session(kick | discard, emqx_types:clientid(), emqx_cm:chan_pid()) -> ok | {badrpc, _}. +kick_session(Action, ClientId, ChanPid) -> + rpc:call(node(ChanPid), emqx_cm, do_kick_session, [Action, ClientId, ChanPid], ?T_KICK * 2). diff --git a/mix.exs b/mix.exs index 384b08100..92024f48d 100644 --- a/mix.exs +++ b/mix.exs @@ -293,6 +293,7 @@ defmodule EMQXUmbrella.MixProject do emqx_psk: :permanent, emqx_slow_subs: :permanent, emqx_plugins: :permanent, + emqx_ft: :permanent, emqx_mix: :none ] ++ if(enable_quicer?(), do: [quicer: :permanent], else: []) ++ From aaaef30be66ab7291732f3fb634afdd0210b983b Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 6 Jan 2023 23:53:25 +0300 Subject: [PATCH 002/197] feat(ft): add file transfer app and bootstrap replicated ft data structure --- apps/emqx/src/emqx_channel.erl | 7 +++ apps/emqx_ft/README.md | 9 +++ apps/emqx_ft/include/emqx_ft.hrl | 17 +++++ apps/emqx_ft/rebar.config | 11 ++++ apps/emqx_ft/src/emqx_ft.app.src | 12 ++++ apps/emqx_ft/src/emqx_ft.erl | 103 +++++++++++++++++++++++++++++++ apps/emqx_ft/src/emqx_ft_app.erl | 30 +++++++++ apps/emqx_ft/src/emqx_ft_sup.erl | 49 +++++++++++++++ 8 files changed, 238 insertions(+) create mode 100644 apps/emqx_ft/README.md create mode 100644 apps/emqx_ft/include/emqx_ft.hrl create mode 100644 apps/emqx_ft/rebar.config create mode 100644 apps/emqx_ft/src/emqx_ft.app.src create mode 100644 apps/emqx_ft/src/emqx_ft.erl create mode 100644 apps/emqx_ft/src/emqx_ft_app.erl create mode 100644 apps/emqx_ft/src/emqx_ft_sup.erl diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 29a59e482..009dc72ea 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -641,6 +641,13 @@ process_connect( %%-------------------------------------------------------------------- process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) -> + ?SLOG( + warning, + #{ + packet => Packet, + packet_id => PacketId + } + ), case pipeline( [ diff --git a/apps/emqx_ft/README.md b/apps/emqx_ft/README.md new file mode 100644 index 000000000..c483b3169 --- /dev/null +++ b/apps/emqx_ft/README.md @@ -0,0 +1,9 @@ +emqx_ft +===== + +EMQX file transfer over MQTT + +Build +----- + + $ rebar3 compile diff --git a/apps/emqx_ft/include/emqx_ft.hrl b/apps/emqx_ft/include/emqx_ft.hrl new file mode 100644 index 000000000..2cbd24fb4 --- /dev/null +++ b/apps/emqx_ft/include/emqx_ft.hrl @@ -0,0 +1,17 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-define(FT_TAB, emqx_ft). diff --git a/apps/emqx_ft/rebar.config b/apps/emqx_ft/rebar.config new file mode 100644 index 000000000..2c0962035 --- /dev/null +++ b/apps/emqx_ft/rebar.config @@ -0,0 +1,11 @@ +%% -*- mode: erlang -*- + +{erl_opts, [debug_info]}. +{deps, [{emqx, {path, "../emqx"}}]}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [emqx_ft]} +]}. + +{project_plugins, [erlfmt]}. diff --git a/apps/emqx_ft/src/emqx_ft.app.src b/apps/emqx_ft/src/emqx_ft.app.src new file mode 100644 index 000000000..855451bfb --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft.app.src @@ -0,0 +1,12 @@ +{application, emqx_ft, [ + {description, "EMQX file transfer over MQTT"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_ft_app, []}}, + {applications, [ + kernel, + stdlib + ]}, + {env, []}, + {modules, []} +]}. diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl new file mode 100644 index 000000000..0ba9c17a6 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -0,0 +1,103 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft). + +-include("emqx_ft.hrl"). + +-export([ + create_tab/0, + hook/0, + unhook/0 +]). + +-export([ + on_channel_unregistered/1, + on_channel_takeover/3, + on_channel_takeovered/3 +]). + +-type ft_data() :: #{ + nodes := list(node()) +}. + +-record(emqx_ft, { + chan_pid :: pid(), + ft_data :: ft_data() +}). + +%%-------------------------------------------------------------------- +%% API for app +%%-------------------------------------------------------------------- + +create_tab() -> + _Tab = ets:new(?FT_TAB, [ + set, + public, + named_table, + {keypos, #emqx_ft.chan_pid} + ]), + ok. + +hook() -> + % ok = emqx_hooks:put('channel.registered', {?MODULE, on_channel_registered, []}), + ok = emqx_hooks:put('channel.unregistered', {?MODULE, on_channel_unregistered, []}), + ok = emqx_hooks:put('channel.takeover', {?MODULE, on_channel_takeover, []}), + ok = emqx_hooks:put('channel.takeovered', {?MODULE, on_channel_takeovered, []}). + +unhook() -> + % ok = emqx_hooks:del('channel.registered', {?MODULE, on_channel_registered}), + ok = emqx_hooks:del('channel.unregistered', {?MODULE, on_channel_unregistered}), + ok = emqx_hooks:del('channel.takeover', {?MODULE, on_channel_takeover}), + ok = emqx_hooks:del('channel.takeovered', {?MODULE, on_channel_takeovered}). + +%%-------------------------------------------------------------------- +%% Hooks +%%-------------------------------------------------------------------- + +on_channel_unregistered(ChanPid) -> + ok = delete_ft_data(ChanPid). + +on_channel_takeover(_ConnMod, ChanPid, TakeoverData) -> + case get_ft_data(ChanPid) of + {ok, FTData} -> + {ok, TakeoverData#{ft_data => FTData}}; + none -> + ok + end. + +on_channel_takeovered(_ConnMod, ChanPid, #{ft_data := FTData}) -> + ok = put_ft_data(ChanPid, FTData); +on_channel_takeovered(_ConnMod, _ChanPid, _) -> + ok. + +%%-------------------------------------------------------------------- +%% Private funs +%%-------------------------------------------------------------------- + +get_ft_data(ChanPid) -> + case ets:lookup(?FT_TAB, ChanPid) of + [#emqx_ft{ft_data = FTData}] -> {ok, FTData}; + [] -> none + end. + +delete_ft_data(ChanPid) -> + true = ets:delete(?FT_TAB, ChanPid), + ok. + +put_ft_data(ChanPid, FTData) -> + true = ets:insert(?FT_TAB, #emqx_ft{chan_pid = ChanPid, ft_data = FTData}), + ok. diff --git a/apps/emqx_ft/src/emqx_ft_app.erl b/apps/emqx_ft/src/emqx_ft_app.erl new file mode 100644 index 000000000..4778da1a1 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_app.erl @@ -0,0 +1,30 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + {ok, Sup} = emqx_ft_sup:start_link(), + ok = emqx_ft:hook(), + {ok, Sup}. + +stop(_State) -> + ok = emqx_ft:unhook(), + ok. diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl new file mode 100644 index 000000000..4c976246b --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -0,0 +1,49 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%% sup_flags() = #{strategy => strategy(), % optional +%% intensity => non_neg_integer(), % optional +%% period => pos_integer()} % optional +%% child_spec() = #{id => child_id(), % mandatory +%% start => mfargs(), % mandatory +%% restart => restart(), % optional +%% shutdown => shutdown(), % optional +%% type => worker(), % optional +%% modules => modules()} % optional +init([]) -> + ok = emqx_ft:create_tab(), + SupFlags = #{ + strategy => one_for_all, + intensity => 100, + period => 10 + }, + ChildSpecs = [], + {ok, {SupFlags, ChildSpecs}}. + +%% internal functions From 81e04ce93aca0c0831f4934f390d5a58612ee25c Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 20 Jan 2023 18:55:45 +0300 Subject: [PATCH 003/197] feat(ft): introduce simple filesystem storage backend + assembler --- apps/emqx_ft/src/emqx_ft.app.src | 3 +- apps/emqx_ft/src/emqx_ft.erl | 14 + apps/emqx_ft/src/emqx_ft_assembler.erl | 160 ++++++++ apps/emqx_ft/src/emqx_ft_assembler_sup.erl | 44 +++ apps/emqx_ft/src/emqx_ft_assembly.erl | 364 ++++++++++++++++++ apps/emqx_ft/src/emqx_ft_storage_fs.erl | 425 +++++++++++++++++++++ 6 files changed, 1009 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_ft/src/emqx_ft_assembler.erl create mode 100644 apps/emqx_ft/src/emqx_ft_assembler_sup.erl create mode 100644 apps/emqx_ft/src/emqx_ft_assembly.erl create mode 100644 apps/emqx_ft/src/emqx_ft_storage_fs.erl diff --git a/apps/emqx_ft/src/emqx_ft.app.src b/apps/emqx_ft/src/emqx_ft.app.src index 855451bfb..80b4b47dd 100644 --- a/apps/emqx_ft/src/emqx_ft.app.src +++ b/apps/emqx_ft/src/emqx_ft.app.src @@ -5,7 +5,8 @@ {mod, {emqx_ft_app, []}}, {applications, [ kernel, - stdlib + stdlib, + gproc ]}, {env, []}, {modules, []} diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 0ba9c17a6..1f8a90a6f 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -30,6 +30,20 @@ on_channel_takeovered/3 ]). +-export_type([clientid/0]). +-export_type([transfer/0]). +-export_type([offset/0]). + +%% Number of bytes +-type bytes() :: non_neg_integer(). + +%% MQTT Client ID +-type clientid() :: emqx_types:clientid(). + +-type fileid() :: binary(). +-type transfer() :: {clientid(), fileid()}. +-type offset() :: bytes(). + -type ft_data() :: #{ nodes := list(node()) }. diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl new file mode 100644 index 000000000..2bd01c8b0 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -0,0 +1,160 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_assembler). + +-export([start_link/2]). + +-behaviour(gen_statem). +-export([callback_mode/0]). +-export([init/1]). +% -export([list_local_fragments/3]). +% -export([list_remote_fragments/3]). +% -export([start_assembling/3]). +-export([handle_event/4]). + +% -export([handle_continue/2]). +% -export([handle_call/3]). +% -export([handle_cast/2]). + +-record(st, { + storage :: _Storage, + transfer :: emqx_ft:transfer(), + assembly :: _TODO, + file :: io:device(), + hash +}). + +-define(RPC_LIST_TIMEOUT, 1000). +-define(RPC_READSEG_TIMEOUT, 5000). + +%% + +start_link(Storage, Transfer) -> + gen_server:start_link(?MODULE, {Storage, Transfer}, []). + +%% + +-define(internal(C), {next_event, internal, C}). + +callback_mode() -> + handle_event_function. + +init({Storage, Transfer}) -> + St = #st{ + storage = Storage, + transfer = Transfer, + assembly = emqx_ft_assembly:new(), + hash = crypto:hash_init(sha256) + }, + {ok, list_local_fragments, St, ?internal([])}. + +handle_event(list_local_fragments, internal, _, St = #st{assembly = Asm}) -> + % TODO: what we do with non-transients errors here (e.g. `eacces`)? + {ok, Fragments} = emqx_ft_storage_fs:list(St#st.storage, St#st.transfer), + NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(Asm, node(), Fragments)), + NSt = St#st{assembly = NAsm}, + case emqx_ft_assembly:status(NAsm) of + complete -> + {next_state, start_assembling, NSt, ?internal([])}; + {incomplete, _} -> + Nodes = ekka:nodelist() -- [node()], + {next_state, {list_remote_fragments, Nodes}, NSt, ?internal([])} + % TODO: recovery? + % {error, _} = Reason -> + % {stop, Reason} + end; +handle_event({list_remote_fragments, Nodes}, internal, _, St) -> + % TODO: portable "storage" ref + Args = [St#st.storage, St#st.transfer], + % TODO + % Async would better because we would not need to wait for some lagging nodes if + % the coverage is already complete. + % TODO: BP API? + Results = erpc:multicall(Nodes, emqx_ft_storage_fs, list, Args, ?RPC_LIST_TIMEOUT), + NodeResults = lists:zip(Nodes, Results), + NAsm = emqx_ft_assembly:update( + lists:foldl( + fun + ({Node, {ok, {ok, Fragments}}}, Asm) -> + emqx_ft_assembly:append(Asm, Node, Fragments); + ({Node, Result}, Asm) -> + % TODO: log? + Asm + end, + St#st.assembly, + NodeResults + ) + ), + NSt = St#st{assembly = NAsm}, + case emqx_ft_assembly:status(NAsm) of + complete -> + {next_state, start_assembling, NSt, ?internal([])}; + % TODO: retries / recovery? + {incomplete, _} = Status -> + {stop, {error, Status}} + end; +handle_event(start_assembling, internal, _, St = #st{assembly = Asm}) -> + Filemeta = emqx_ft_assembly:filemeta(Asm), + Coverage = emqx_ft_assembly:coverage(Asm), + % TODO: errors + {ok, Handle} = emqx_ft_storage_fs:open_file(St#st.storage, St#st.transfer, Filemeta), + {next_state, {assemble, Coverage}, St#st{file = Handle}, ?internal([])}; +handle_event({assemble, [{Node, Segment} | Rest]}, internal, _, St = #st{}) -> + % TODO + % Currently, race is possible between getting segment info from the remote node and + % this node garbage collecting the segment itself. + Args = [St#st.storage, St#st.transfer, Segment, 0, segsize(Segment)], + % TODO: pipelining + case erpc:call(Node, emqx_ft_storage_fs, read_segment, Args, ?RPC_READSEG_TIMEOUT) of + {ok, Content} -> + {ok, NHandle} = emqx_ft_storage_fs:write(St#st.file, Content), + {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])} + % {error, _} -> + % ... + end; +handle_event({assemble, []}, internal, _, St = #st{}) -> + {next_state, complete, St, ?internal([])}; +handle_event(complete, internal, _, St = #st{assembly = Asm, file = Handle}) -> + Filemeta = emqx_ft_assembly:filemeta(Asm), + ok = emqx_ft_storage_fs:complete(St#st.storage, St#st.transfer, Filemeta, Handle), + {stop, shutdown}. + +% handle_continue(list_local, St = #st{storage = Storage, transfer = Transfer, assembly = Asm}) -> +% % TODO: what we do with non-transients errors here (e.g. `eacces`)? +% {ok, Fragments} = emqx_ft_storage_fs:list(Storage, Transfer), +% NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(Asm, node(), Fragments)), +% NSt = St#st{assembly = NAsm}, +% case emqx_ft_assembly:status(NAsm) of +% complete -> +% {noreply, NSt, {continue}}; +% {more, _} -> +% error(noimpl); +% {error, _} -> +% error(noimpl) +% end, +% {noreply, St}. + +% handle_call(_Call, _From, St) -> +% {reply, {error, badcall}, St}. + +% handle_cast(_Cast, St) -> +% {noreply, St}. + +%% + +segsize(#{fragment := {segmentinfo, Info}}) -> + maps:get(size, Info). diff --git a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl new file mode 100644 index 000000000..fbf948fbb --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl @@ -0,0 +1,44 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_assembler_sup). + +-export([start_link/1]). +-export([start_child/3]). + +-behaviour(supervisor). +-export([init/1]). + +-define(REF(ID), {via, gproc, {n, l, {?MODULE, ID}}}). + +start_link(ID) -> + supervisor:start_link(?REF(ID), ?MODULE, []). + +start_child(ID, Storage, Transfer) -> + Childspec = #{ + id => {Storage, Transfer}, + start => {emqx_ft_assembler, start_link, [Storage, Transfer]}, + restart => transient + }, + supervisor:start_child(?REF(ID), Childspec). + +init(_) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 100, + period => 1000 + }, + {ok, SupFlags, []}. diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl new file mode 100644 index 000000000..854b420e0 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -0,0 +1,364 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_assembly). + +-export([new/0]). +-export([append/3]). +-export([update/1]). + +-export([status/1]). +-export([filemeta/1]). +-export([coverage/1]). +-export([properties/1]). + +-record(asm, { + status :: _TODO, + coverage :: _TODO, + properties :: _TODO, + meta :: _TODO, + % orddict:orddict(K, V) + segs :: _TODO, + size +}). + +new() -> + #asm{ + status = {incomplete, {missing, filemeta}}, + meta = orddict:new(), + segs = orddict:new(), + size = 0 + }. + +append(Asm, Node, Fragments) when is_list(Fragments) -> + lists:foldl(fun(F, AsmIn) -> append(AsmIn, Node, F) end, Asm, Fragments); +append(Asm, Node, Fragment = #{fragment := {filemeta, _}}) -> + append_filemeta(Asm, Node, Fragment); +append(Asm, Node, Segment = #{fragment := {segmentinfo, _}}) -> + append_segmentinfo(Asm, Node, Segment). + +update(Asm) -> + case status(meta, Asm) of + {complete, _Meta} -> + case status(coverage, Asm) of + {complete, Coverage, Props} -> + Asm#asm{ + status = complete, + coverage = Coverage, + properties = Props + }; + Status -> + Asm#asm{status = Status} + end; + Status -> + Asm#asm{status = Status} + end. + +status(#asm{status = Status}) -> + Status. + +filemeta(Asm) -> + case status(meta, Asm) of + {complete, Meta} -> Meta; + _Other -> undefined + end. + +coverage(#asm{coverage = Coverage}) -> + Coverage. + +properties(#asm{properties = Properties}) -> + Properties. + +status(meta, #asm{meta = Meta}) -> + status(meta, orddict:to_list(Meta)); +status(meta, [{Meta, {_Node, _Frag}}]) -> + {complete, Meta}; +status(meta, []) -> + {incomplete, {missing, filemeta}}; +status(meta, [_M1, _M2 | _] = Metas) -> + {error, {inconsistent, [Frag#{node => Node} || {_, {Node, Frag}} <- Metas]}}; +status(coverage, #asm{segs = Segments, size = Size}) -> + case coverage(orddict:to_list(Segments), 0, Size) of + Coverage when is_list(Coverage) -> + {complete, Coverage, #{ + dominant => dominant(Coverage) + }}; + Missing = {missing, _} -> + {incomplete, Missing} + end. + +append_filemeta(Asm, Node, Fragment = #{fragment := {filemeta, Meta}}) -> + Asm#asm{ + meta = orddict:store(Meta, {Node, Fragment}, Asm#asm.meta) + }. + +append_segmentinfo(Asm, Node, Fragment = #{fragment := {segmentinfo, Info}}) -> + Offset = maps:get(offset, Info), + Size = maps:get(size, Info), + End = Offset + Size, + Asm#asm{ + % TODO + % In theory it's possible to have two segments with same offset + size on + % different nodes but with differing content. We'd need a checksum to + % be able to disambiguate them though. + segs = orddict:store({Offset, locality(Node), -End, Node}, Fragment, Asm#asm.segs), + size = max(End, Asm#asm.size) + }. + +coverage([{{Offset, _, _, _}, _Segment} | Rest], Cursor, Sz) when Offset < Cursor -> + coverage(Rest, Cursor, Sz); +coverage([{{Cursor, _Locality, MEnd, Node}, Segment} | Rest], Cursor, Sz) -> + % NOTE + % We consider only whole fragments here, so for example from the point of view of + % this algo `[{Offset1 = 0, Size1 = 15}, {Offset2 = 10, Size2 = 10}]` has no + % coverage. + case coverage(Rest, -MEnd, Sz) of + Coverage when is_list(Coverage) -> + [{Node, Segment} | Coverage]; + Missing = {missing, _} -> + case coverage(Rest, Cursor, Sz) of + CoverageAlt when is_list(CoverageAlt) -> + CoverageAlt; + {missing, _} -> + Missing + end + end; +coverage([{{Offset, _MEnd, _, _}, _Segment} | _], Cursor, _Sz) when Offset > Cursor -> + {missing, {segment, Cursor, Offset}}; +coverage([], Cursor, Sz) when Cursor < Sz -> + {missing, {segment, Cursor, Sz}}; +coverage([], Cursor, Cursor) -> + []. + +dominant(Coverage) -> + % TODO: needs improvement, better defined _dominance_, maybe some score + Freqs = frequencies(fun({Node, Segment}) -> {Node, segsize(Segment)} end, Coverage), + maxfreq(Freqs, node()). + +frequencies(Fun, List) -> + lists:foldl( + fun(E, Acc) -> + {K, N} = Fun(E), + maps:update_with(K, fun(M) -> M + N end, N, Acc) + end, + #{}, + List + ). + +maxfreq(Freqs, Init) -> + {_, Max} = maps:fold( + fun + (F, N, {M, _MF}) when N > M -> {N, F}; + (_F, _N, {M, MF}) -> {M, MF} + end, + {0, Init}, + Freqs + ), + Max. + +locality(Node) when Node =:= node() -> + % NOTE + % This should prioritize locally available segments over those on remote nodes. + 0; +locality(_RemoteNode) -> + 1. + +segsize(#{fragment := {segmentinfo, Info}}) -> + maps:get(size, Info). + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +incomplete_new_test() -> + ?assertEqual( + {incomplete, {missing, filemeta}}, + status(update(new())) + ). + +incomplete_test() -> + ?assertEqual( + {incomplete, {missing, filemeta}}, + status( + update( + append(new(), node(), [ + segmentinfo(p1, 0, 42), + segmentinfo(p1, 42, 100) + ]) + ) + ) + ). + +consistent_test() -> + Asm1 = append(new(), n1, [filemeta(m1, "blarg")]), + Asm2 = append(Asm1, n2, [segmentinfo(s2, 0, 42)]), + Asm3 = append(Asm2, n3, [filemeta(m3, "blarg")]), + ?assertMatch({complete, _}, status(meta, Asm3)). + +inconsistent_test() -> + Asm1 = append(new(), node(), [segmentinfo(s1, 0, 42)]), + Asm2 = append(Asm1, n1, [filemeta(m1, "blarg")]), + Asm3 = append(Asm2, n2, [segmentinfo(s2, 0, 42), filemeta(m1, "blorg")]), + Asm4 = append(Asm3, n3, [filemeta(m3, "blarg")]), + ?assertMatch( + {error, + {inconsistent, [ + % blarg < blorg + #{node := n3, path := m3, fragment := {filemeta, #{name := "blarg"}}}, + #{node := n2, path := m1, fragment := {filemeta, #{name := "blorg"}}} + ]}}, + status(meta, Asm4) + ). + +simple_coverage_test() -> + Node = node(), + Segs = [ + {node42, segmentinfo(n1, 20, 30)}, + {Node, segmentinfo(n2, 0, 10)}, + {Node, segmentinfo(n3, 50, 50)}, + {Node, segmentinfo(n4, 10, 10)} + ], + Asm = append_many(new(), Segs), + ?assertMatch( + {complete, + [ + {Node, #{path := n2}}, + {Node, #{path := n4}}, + {node42, #{path := n1}}, + {Node, #{path := n3}} + ], + #{dominant := Node}}, + status(coverage, Asm) + ). + +redundant_coverage_test() -> + Node = node(), + Segs = [ + {Node, segmentinfo(n1, 0, 20)}, + {node1, segmentinfo(n2, 0, 10)}, + {Node, segmentinfo(n3, 20, 40)}, + {node2, segmentinfo(n4, 10, 10)}, + {node2, segmentinfo(n5, 50, 20)}, + {node3, segmentinfo(n6, 20, 20)}, + {Node, segmentinfo(n7, 50, 10)}, + {node1, segmentinfo(n8, 40, 10)} + ], + Asm = append_many(new(), Segs), + ?assertMatch( + {complete, + [ + {Node, #{path := n1}}, + {node3, #{path := n6}}, + {node1, #{path := n8}}, + {node2, #{path := n5}} + ], + #{dominant := _}}, + status(coverage, Asm) + ). + +redundant_coverage_prefer_local_test() -> + Node = node(), + Segs = [ + {node1, segmentinfo(n1, 0, 20)}, + {Node, segmentinfo(n2, 0, 10)}, + {Node, segmentinfo(n3, 10, 10)}, + {node2, segmentinfo(n4, 20, 20)}, + {Node, segmentinfo(n5, 30, 10)}, + {Node, segmentinfo(n6, 20, 10)} + ], + Asm = append_many(new(), Segs), + ?assertMatch( + {complete, + [ + {Node, #{path := n2}}, + {Node, #{path := n3}}, + {Node, #{path := n6}}, + {Node, #{path := n5}} + ], + #{dominant := Node}}, + status(coverage, Asm) + ). + +missing_coverage_test() -> + Node = node(), + Segs = [ + {Node, segmentinfo(n1, 0, 10)}, + {node1, segmentinfo(n3, 10, 20)}, + {Node, segmentinfo(n2, 0, 20)}, + {node2, segmentinfo(n4, 50, 50)}, + {Node, segmentinfo(n5, 40, 60)} + ], + Asm = append_many(new(), Segs), + ?assertEqual( + % {incomplete, {missing, {segment, 30, 40}}}, ??? + {incomplete, {missing, {segment, 20, 40}}}, + status(coverage, Asm) + ). + +missing_end_coverage_test() -> + Node = node(), + Segs = [ + {Node, segmentinfo(n1, 0, 15)}, + {node1, segmentinfo(n3, 10, 10)} + ], + Asm = append_many(new(), Segs), + ?assertEqual( + {incomplete, {missing, {segment, 15, 20}}}, + status(coverage, Asm) + ). + +missing_coverage_with_redudancy_test() -> + Segs = [ + {node(), segmentinfo(n1, 0, 10)}, + {node(), segmentinfo(n2, 0, 20)}, + {node42, segmentinfo(n3, 10, 20)}, + {node43, segmentinfo(n4, 10, 50)}, + {node(), segmentinfo(n5, 40, 60)} + ], + Asm = append_many(new(), Segs), + ?assertEqual( + % {incomplete, {missing, {segment, 50, 60}}}, ??? + {incomplete, {missing, {segment, 20, 40}}}, + status(coverage, Asm) + ). + +append_many(Asm, List) -> + lists:foldl( + fun({Node, Frag}, Acc) -> append(Acc, Node, Frag) end, + Asm, + List + ). + +filemeta(Path, Name) -> + #{ + path => Path, + fragment => + {filemeta, #{ + name => Name + }} + }. + +segmentinfo(Path, Offset, Size) -> + #{ + path => Path, + fragment => + {segmentinfo, #{ + offset => Offset, + size => Size + }} + }. + +-endif. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl new file mode 100644 index 000000000..2d50e1b18 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -0,0 +1,425 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +% -compile(export_all). + +-export([store_filemeta/3]). +-export([store_segment/3]). +-export([list/2]). +-export([assemble/2]). + +-export([open_file/3]). +-export([complete/4]). +-export([write/2]). +-export([discard/1]). + +% -behaviour(gen_server). +% -export([init/1]). +% -export([handle_call/3]). +% -export([handle_cast/2]). + +-type json_value() :: + null + | boolean() + | binary() + | number() + | [json_value()] + | #{binary() => json_value()}. + +-reflect_type([json_value/0]). + +-type transfer() :: emqx_ft:transfer(). +-type offset() :: emqx_ft:offset(). + +%% TODO: move to `emqx_ft` interface module +% -type sha256_hex() :: <<_:512>>. + +-type filemeta() :: #{ + %% Display name + name := string(), + %% Size in bytes, as advertised by the client. + %% Client is free to specify here whatever it wants, which means we can end + %% up with a file of different size after assembly. It's not clear from + %% specification what that means (e.g. what are clients' expectations), we + %% currently do not condider that an error (or, specifically, a signal that + %% the resulting file is corrupted during transmission). + size => _Bytes :: non_neg_integer(), + checksum => {sha256, <<_:256>>}, + expire_at := emqx_datetime:epoch_second(), + %% TTL of individual segments + %% Somewhat confusing that we won't know it on the nodes where the filemeta + %% is missing. + segments_ttl => _Seconds :: pos_integer(), + user_data => json_value() +}. + +-type segment() :: {offset(), _Content :: binary()}. + +-type segmentinfo() :: #{ + offset := offset(), + size := _Bytes :: non_neg_integer() +}. + +-type filefrag(T) :: #{ + path := file:name(), + timestamp := emqx_datetime:epoch_second(), + fragment := T +}. + +-type filefrag() :: filefrag({filemeta, filemeta()} | {segment, segmentinfo()}). + +-define(MANIFEST, "MANIFEST.json"). +-define(SEGMENT, "SEG"). +-define(TEMP, "TMP"). + +-type root() :: file:name(). + +% -record(st, { +% root :: file:name() +% }). + +%% TODO +-type storage() :: root(). + +%% + +-define(PROCREF(Root), {via, gproc, {n, l, {?MODULE, Root}}}). + +-spec start_link(root()) -> + {ok, pid()} | {error, already_started}. +start_link(Root) -> + gen_server:start_link(?PROCREF(Root), ?MODULE, [], []). + +%% Store manifest in the backing filesystem. +%% Atomic operation. +-spec store_filemeta(storage(), transfer(), filemeta()) -> + % Quota? Some lower level errors? + ok | {error, conflict} | {error, _TODO}. +store_filemeta(Storage, Transfer, Meta) -> + Filepath = mk_filepath(Storage, Transfer, ?MANIFEST), + case read_file(Filepath, fun decode_filemeta/1) of + {ok, Meta} -> + _ = touch_file(Filepath), + ok; + {ok, Conflict} -> + % TODO + % We won't see conflicts in case of concurrent `store_filemeta` + % requests. It's rather odd scenario so it's fine not to worry + % about it too much now. + {error, conflict}; + {error, Reason} when Reason =:= notfound; Reason =:= corrupted -> + write_file_atomic(Filepath, encode_filemeta(Meta)) + end. + +%% Store a segment in the backing filesystem. +%% Atomic operation. +-spec store_segment(storage(), transfer(), segment()) -> + % Where is the checksum gets verified? Upper level probably. + % Quota? Some lower level errors? + ok | {error, _TODO}. +store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> + Filepath = mk_filepath(Storage, Transfer, mk_segment_filename(Segment)), + write_file_atomic(Filepath, Content). + +-spec list(storage(), transfer()) -> + % Some lower level errors? {error, notfound}? + % Result will contain zero or only one filemeta. + {ok, list(filefrag())} | {error, _TODO}. +list(Storage, Transfer) -> + Dirname = mk_filedir(Storage, Transfer), + case file:list_dir(Dirname) of + {ok, Filenames} -> + {ok, filtermap_files(fun mk_filefrag/2, Dirname, Filenames)}; + {error, enoent} -> + {ok, []}; + {error, _} = Error -> + Error + end. + +-spec assemble(storage(), transfer()) -> + % {ok, _Assembler :: pid()} | {error, incomplete} | {error, badrpc} | {error, _TODO}. + {ok, _Assembler :: pid()} | {error, _TODO}. +assemble(Storage, Transfer) -> + emqx_ft_assembler_sup:start_child(Storage, Storage, Transfer). + +%% + +-opaque handle() :: {file:name(), io:device(), crypto:hash_state()}. + +-spec open_file(storage(), transfer(), filemeta()) -> + {ok, handle()} | {error, _TODO}. +open_file(Storage, Transfer, Filemeta) -> + Filename = maps:get(name, Filemeta), + Filepath = mk_filepath(Storage, Transfer, Filename), + TempFilepath = mk_temp_filepath(Filepath), + case file:open(TempFilepath, [write, raw]) of + {ok, Handle} -> + _ = file:truncate(Handle), + {ok, {TempFilepath, Handle, init_checksum(Filemeta)}}; + {error, _} = Error -> + Error + end. + +-spec write(handle(), iodata()) -> + ok | {error, _TODO}. +write({Filepath, IoDevice, Ctx}, IoData) -> + case file:write(IoDevice, IoData) of + ok -> + {ok, {Filepath, IoDevice, update_checksum(Ctx, IoData)}}; + {error, _} = Error -> + Error + end. + +-spec complete(storage(), transfer(), filemeta(), handle()) -> + ok | {error, {checksum, _Algo, _Computed}} | {error, _TODO}. +complete(Storage, Transfer, Filemeta, Handle = {Filepath, IoDevice, Ctx}) -> + TargetFilepath = mk_filepath(Storage, Transfer, maps:get(name, Filemeta)), + case verify_checksum(Ctx, Filemeta) of + ok -> + ok = file:close(IoDevice), + file:rename(Filepath, TargetFilepath); + {error, _} = Error -> + _ = discard(Handle), + Error + end. + +-spec discard(handle()) -> + ok. +discard({Filepath, IoDevice, _Ctx}) -> + ok = file:close(IoDevice), + file:delete(Filepath). + +init_checksum(#{checksum := {Algo, _}}) -> + crypto:hash_init(Algo); +init_checksum(#{}) -> + undefined. + +update_checksum(Ctx, IoData) when Ctx /= undefined -> + crypto:hash_update(Ctx, IoData); +update_checksum(undefined, _IoData) -> + undefined. + +verify_checksum(Ctx, #{checksum := {Algo, Digest}}) when Ctx /= undefined -> + case crypto:hash_final(Ctx) of + Digest -> + ok; + Mismatch -> + {error, {checksum, Algo, binary:encode_hex(Mismatch)}} + end; +verify_checksum(undefined, _) -> + ok. + +%% + +-spec init(root()) -> {ok, storage()}. +init(Root) -> + % TODO: garbage_collect(...) + {ok, Root}. + +%% + +-define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). + +schema() -> + #{ + roots => [ + {name, hoconsc:mk(string(), #{required => true})}, + {size, hoconsc:mk(non_neg_integer())}, + {expire_at, hoconsc:mk(non_neg_integer(), #{required => true})}, + {checksum, hoconsc:mk({atom(), binary()}, #{converter => converter(checksum)})}, + {segments_ttl, hoconsc:mk(pos_integer())}, + {user_data, hoconsc:mk(json_value())} + ] + }. + +% encode_filemeta(Meta) -> +% emqx_json:encode( +% ?PRELUDE( +% _Vsn = 1, +% maps:map( +% fun +% (name, Name) -> +% {<<"name">>, Name}; +% (size, Size) -> +% {<<"size">>, Size}; +% (checksum, {sha256, Hash}) -> +% {<<"checksum">>, <<"sha256:", (binary:encode_hex(Hash))/binary>>}; +% (expire_at, ExpiresAt) -> +% {<<"expire_at">>, ExpiresAt}; +% (segments_ttl, TTL) -> +% {<<"segments_ttl">>, TTL}; +% (user_data, UserData) -> +% {<<"user_data">>, UserData} +% end, +% Meta +% ) +% ) +% ). + +encode_filemeta(Meta) -> + % TODO: Looks like this should be hocon's responsibility. + Term = hocon_tconf:make_serializable(schema(), emqx_map_lib:binary_key_map(Meta), #{}), + emqx_json:encode(?PRELUDE(_Vsn = 1, Term)). + +decode_filemeta(Binary) -> + ?PRELUDE(_Vsn = 1, Term) = emqx_json:decode(Binary, [return_maps]), + hocon_tconf:check_plain(schema(), Term, #{atom_key => true, required => false}). + +converter(checksum) -> + fun + (undefined, #{}) -> + undefined; + ({sha256, Bin}, #{make_serializable := true}) -> + _ = is_binary(Bin) orelse throw({expected_type, string}), + _ = byte_size(Bin) =:= 32 orelse throw({expected_length, 32}), + binary:encode_hex(Bin); + (Hex, #{}) -> + _ = is_binary(Hex) orelse throw({expected_type, string}), + _ = byte_size(Hex) =:= 64 orelse throw({expected_length, 64}), + {sha256, binary:decode_hex(Hex)} + end. + +% map_into(Fun, Into, Ks, Map) -> +% map_foldr(map_into_fn(Fun, Into), Into, Ks, Map). + +% map_into_fn(Fun, L) when is_list(L) -> +% fun(K, V, Acc) -> [{K, Fun(K, V)} || Acc] end. + +% map_foldr(_Fun, Acc, [], _) -> +% Acc; +% map_foldr(Fun, Acc, [K | Ks], Map) when is_map_key(K, Map) -> +% Fun(K, maps:get(K, Map), map_foldr(Fun, Acc, Ks, Map)); +% map_foldr(Fun, Acc, [_ | Ks], Map) -> +% map_foldr(Fun, Acc, Ks, Map). + +%% + +mk_segment_filename({Offset, Content}) -> + lists:concat([?SEGMENT, ".", Offset, ".", byte_size(Content)]). + +break_segment_filename(Filename) -> + Regex = "^" ?SEGMENT "[.]([0-9]+)[.]([0-9]+)$", + Result = re:run(Filename, Regex, [{capture, all_but_first, list}]), + case Result of + {match, [Offset, Size]} -> + {ok, #{offset => list_to_integer(Offset), size => list_to_integer(Size)}}; + nomatch -> + {error, invalid} + end. + +mk_filedir(Storage, {ClientId, FileId}) -> + filename:join([get_storage_root(Storage), ClientId, FileId]). + +mk_filepath(Storage, Transfer, Filename) -> + filename:join(mk_filedir(Storage, Transfer), Filename). + +get_storage_root(Storage) -> + Storage. + +%% + +-include_lib("kernel/include/file.hrl"). + +read_file(Filepath) -> + file:read_file(Filepath). + +read_file(Filepath, DecodeFun) -> + case read_file(Filepath) of + {ok, Content} -> + safe_decode(Content, DecodeFun); + {error, _} = Error -> + Error + end. + +safe_decode(Content, DecodeFun) -> + try + {ok, DecodeFun(Content)} + catch + C:R:Stacktrace -> + % TODO: Log? + {error, corrupted} + end. + +write_file_atomic(Filepath, Content) when is_binary(Content) -> + Result = emqx_misc:pipeline( + [ + fun filelib:ensure_dir/1, + fun mk_temp_filepath/1, + fun write_contents/2, + fun mv_temp_file/1 + ], + Filepath, + Content + ), + case Result of + {ok, {Filepath, TempFilepath}, _} -> + _ = file:delete(TempFilepath), + ok; + {error, Reason, _} -> + {error, Reason} + end. + +mk_temp_filepath(Filepath) -> + Dirname = filename:dirname(Filepath), + Filename = filename:basename(Filepath), + Unique = erlang:unique_integer([positive]), + TempFilepath = filename:join(Dirname, ?TEMP ++ integer_to_list(Unique) ++ "." ++ Filename), + {Filepath, TempFilepath}. + +write_contents({_Filepath, TempFilepath}, Content) -> + file:write_file(TempFilepath, Content). + +mv_temp_file({Filepath, TempFilepath}) -> + file:rename(TempFilepath, Filepath). + +touch_file(Filepath) -> + Now = erlang:localtime(), + file:change_time(Filepath, _Mtime = Now, _Atime = Now). + +filtermap_files(Fun, Dirname, Filenames) -> + lists:filtermap(fun(Filename) -> Fun(Dirname, Filename) end, Filenames). + +mk_filefrag(Dirname, Filename = ?MANIFEST) -> + mk_filefrag(Dirname, Filename, fun read_filemeta/2); +mk_filefrag(Dirname, Filename = ?SEGMENT ++ _) -> + mk_filefrag(Dirname, Filename, fun read_segmentinfo/2); +mk_filefrag(_Dirname, _) -> + false. + +mk_filefrag(Dirname, Filename, Fun) -> + Filepath = filename:join(Dirname, Filename), + Fileinfo = file:read_file_info(Filepath), + case Fun(Filename, Filepath) of + {ok, Frag} -> + {true, #{ + path => Filepath, + timestamp => Fileinfo#file_info.mtime, + fragment => Frag + }}; + {error, Reason} -> + false + end. + +read_filemeta(_Filename, Filepath) -> + read_file(Filepath, fun decode_filemeta/1). + +read_segmentinfo(Filename, _Filepath) -> + break_segment_filename(Filename). From d36ca18bff8b71c1c3f8d44ae096da3fab6b8c7c Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 23 Jan 2023 00:30:26 +0200 Subject: [PATCH 004/197] feat(ft): tie file transfer frontend and backend together --- apps/emqx/src/emqx_channel.erl | 29 +++- apps/emqx/src/emqx_cm.erl | 2 +- apps/emqx/test/emqx_channel_SUITE.erl | 2 +- .../emqx_channel_delayed_puback_SUITE.erl | 70 +++++++++ apps/emqx_ft/src/emqx_ft.erl | 139 ++++++++++++++++-- apps/emqx_ft/src/emqx_ft_assembler.erl | 22 +-- apps/emqx_ft/src/emqx_ft_assembler_sup.erl | 16 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 40 ++--- apps/emqx_ft/src/emqx_ft_sup.erl | 12 +- rebar.config.erl | 3 +- 10 files changed, 277 insertions(+), 58 deletions(-) create mode 100644 apps/emqx/test/emqx_channel_delayed_puback_SUITE.erl diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 009dc72ea..c67c02d66 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -744,9 +744,13 @@ do_publish(_PacketId, Msg = #message{qos = ?QOS_0}, Channel) -> {ok, NChannel}; do_publish(PacketId, Msg = #message{qos = ?QOS_1}, Channel) -> PubRes = emqx_broker:publish(Msg), - RC = puback_reason_code(PubRes), - NChannel = ensure_quota(PubRes, Channel), - handle_out(puback, {PacketId, RC}, NChannel); + RC = puback_reason_code(PacketId, Msg, PubRes), + case RC of + undefined -> + {ok, Channel}; + _Value -> + do_finish_publish(PacketId, PubRes, RC, Channel) + end; do_publish( PacketId, Msg = #message{qos = ?QOS_2}, @@ -754,7 +758,7 @@ do_publish( ) -> case emqx_session:publish(ClientInfo, PacketId, Msg, Session) of {ok, PubRes, NSession} -> - RC = puback_reason_code(PubRes), + RC = pubrec_reason_code(PubRes), NChannel0 = set_session(NSession, Channel), NChannel1 = ensure_timer(await_timer, NChannel0), NChannel2 = ensure_quota(PubRes, NChannel1), @@ -767,6 +771,10 @@ do_publish( handle_out(disconnect, RC, Channel) end. +do_finish_publish(PacketId, PubRes, RC, Channel) -> + NChannel = ensure_quota(PubRes, Channel), + handle_out(puback, {PacketId, RC}, NChannel). + ensure_quota(_, Channel = #channel{quota = undefined}) -> Channel; ensure_quota(PubRes, Channel = #channel{quota = Limiter}) -> @@ -786,9 +794,14 @@ ensure_quota(PubRes, Channel = #channel{quota = Limiter}) -> ensure_timer(quota_timer, Intv, Channel#channel{quota = NLimiter}) end. --compile({inline, [puback_reason_code/1]}). -puback_reason_code([]) -> ?RC_NO_MATCHING_SUBSCRIBERS; -puback_reason_code([_ | _]) -> ?RC_SUCCESS. +-compile({inline, [pubrec_reason_code/1]}). +pubrec_reason_code([]) -> ?RC_NO_MATCHING_SUBSCRIBERS; +pubrec_reason_code([_ | _]) -> ?RC_SUCCESS. + +puback_reason_code(PacketId, Msg, [] = PubRes) -> + emqx_hooks:run_fold('message.puback', [PacketId, Msg, PubRes], ?RC_NO_MATCHING_SUBSCRIBERS); +puback_reason_code(PacketId, Msg, [_ | _] = PubRes) -> + emqx_hooks:run_fold('message.puback', [PacketId, Msg, PubRes], ?RC_SUCCESS). -compile({inline, [after_message_acked/3]}). after_message_acked(ClientInfo, Msg, PubAckProps) -> @@ -1283,6 +1296,8 @@ handle_info(die_if_test = Info, Channel) -> die_if_test_compiled(), ?SLOG(error, #{msg => "unexpected_info", info => Info}), {ok, Channel}; +handle_info({puback, PacketId, PubRes, RC}, Channel) -> + do_finish_publish(PacketId, PubRes, RC, Channel); handle_info(Info, Channel) -> ?SLOG(error, #{msg => "unexpected_info", info => Info}), {ok, Channel}. diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index d2ac642ca..724139142 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -313,7 +313,7 @@ open_session(false, ClientInfo = #{clientid := ClientId}, #{conn_mod := NewConnM Session1 = emqx_persistent_session:persist( ClientInfo, ConnInfo, Session ), - ok = emqx_hooks:run('channel.takeovered', [NewConnMod, Self, TakoverData]), + ok = emqx_hooks:run('channel.takenover', [NewConnMod, Self, TakoverData]), {ok, #{ session => clean_session(Session1), present => true, diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 6dd389350..633337f80 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -1133,7 +1133,7 @@ t_ws_cookie_init(_) -> ?assertMatch(#{ws_cookie := WsCookie}, emqx_channel:info(clientinfo, Channel)). %%-------------------------------------------------------------------- -%% Test cases for other mechnisms +%% Test cases for other mechanisms %%-------------------------------------------------------------------- t_flapping_detect(_) -> diff --git a/apps/emqx/test/emqx_channel_delayed_puback_SUITE.erl b/apps/emqx/test/emqx_channel_delayed_puback_SUITE.erl new file mode 100644 index 000000000..4f2938b24 --- /dev/null +++ b/apps/emqx/test/emqx_channel_delayed_puback_SUITE.erl @@ -0,0 +1,70 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_channel_delayed_puback_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_hooks.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([]). + +init_per_testcase(Case, Config) -> + ?MODULE:Case({init, Config}). + +end_per_testcase(Case, Config) -> + ?MODULE:Case({'end', Config}). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_delayed_puback({init, Config}) -> + emqx_hooks:put('message.puback', {?MODULE, on_message_puback, []}, ?HP_LOWEST), + Config; +t_delayed_puback({'end', _Config}) -> + emqx_hooks:del('message.puback', {?MODULE, on_message_puback}); +t_delayed_puback(_Config) -> + {ok, ConnPid} = emqtt:start_link([{clientid, <<"clientid">>}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(ConnPid), + {ok, #{reason_code := ?RC_UNSPECIFIED_ERROR}} = emqtt:publish( + ConnPid, <<"topic">>, <<"hello">>, 1 + ), + emqtt:disconnect(ConnPid). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +on_message_puback(PacketId, _Msg, PubRes, _RC) -> + erlang:send(self(), {puback, PacketId, PubRes, ?RC_UNSPECIFIED_ERROR}), + {stop, undefined}. diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 1f8a90a6f..c73d31559 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -17,6 +17,10 @@ -module(emqx_ft). -include("emqx_ft.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_hooks.hrl"). -export([ create_tab/0, @@ -27,9 +31,14 @@ -export([ on_channel_unregistered/1, on_channel_takeover/3, - on_channel_takeovered/3 + on_channel_takenover/3, + on_message_publish/1, + on_message_puback/4 ]). +%% For Debug +-export([transfer/2, storage/0]). + -export_type([clientid/0]). -export_type([transfer/0]). -export_type([offset/0]). @@ -67,16 +76,20 @@ create_tab() -> ok. hook() -> - % ok = emqx_hooks:put('channel.registered', {?MODULE, on_channel_registered, []}), - ok = emqx_hooks:put('channel.unregistered', {?MODULE, on_channel_unregistered, []}), - ok = emqx_hooks:put('channel.takeover', {?MODULE, on_channel_takeover, []}), - ok = emqx_hooks:put('channel.takeovered', {?MODULE, on_channel_takeovered, []}). + ok = emqx_hooks:put('channel.unregistered', {?MODULE, on_channel_unregistered, []}, ?HP_LOWEST), + ok = emqx_hooks:put('channel.takeover', {?MODULE, on_channel_takeover, []}, ?HP_LOWEST), + ok = emqx_hooks:put('channel.takenover', {?MODULE, on_channel_takenover, []}, ?HP_LOWEST), + + ok = emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_LOWEST), + ok = emqx_hooks:put('message.puback', {?MODULE, on_message_puback, []}, ?HP_LOWEST). unhook() -> - % ok = emqx_hooks:del('channel.registered', {?MODULE, on_channel_registered}), ok = emqx_hooks:del('channel.unregistered', {?MODULE, on_channel_unregistered}), ok = emqx_hooks:del('channel.takeover', {?MODULE, on_channel_takeover}), - ok = emqx_hooks:del('channel.takeovered', {?MODULE, on_channel_takeovered}). + ok = emqx_hooks:del('channel.takenover', {?MODULE, on_channel_takenover}), + + ok = emqx_hooks:del('message.publish', {?MODULE, on_message_publish}), + ok = emqx_hooks:del('message.puback', {?MODULE, on_message_puback}). %%-------------------------------------------------------------------- %% Hooks @@ -93,11 +106,30 @@ on_channel_takeover(_ConnMod, ChanPid, TakeoverData) -> ok end. -on_channel_takeovered(_ConnMod, ChanPid, #{ft_data := FTData}) -> +on_channel_takenover(_ConnMod, ChanPid, #{ft_data := FTData}) -> ok = put_ft_data(ChanPid, FTData); -on_channel_takeovered(_ConnMod, _ChanPid, _) -> +on_channel_takenover(_ConnMod, _ChanPid, _) -> ok. +on_message_publish( + Msg = #message{ + id = _Id, + topic = <<"$file/", _/binary>> + } +) -> + Headers = Msg#message.headers, + {stop, Msg#message{headers = Headers#{allow_publish => false}}}; +on_message_publish(Msg) -> + {ok, Msg}. + +on_message_puback(PacketId, #message{topic = Topic} = Msg, _PubRes, _RC) -> + case Topic of + <<"$file/", FileCommand/binary>> -> + {stop, on_file_command(PacketId, Msg, FileCommand)}; + _ -> + ignore + end. + %%-------------------------------------------------------------------- %% Private funs %%-------------------------------------------------------------------- @@ -115,3 +147,92 @@ delete_ft_data(ChanPid) -> put_ft_data(ChanPid, FTData) -> true = ets:insert(?FT_TAB, #emqx_ft{chan_pid = ChanPid, ft_data = FTData}), ok. + +on_file_command(PacketId, Msg, FileCommand) -> + case string:split(FileCommand, <<"/">>, all) of + [FileId, <<"init">>] -> + on_init(Msg, FileId); + [FileId, <<"fin">>] -> + on_fin(PacketId, Msg, FileId, undefined); + [FileId, <<"fin">>, Checksum] -> + on_fin(PacketId, Msg, FileId, Checksum); + [FileId, <<"abort">>] -> + on_abort(Msg, FileId); + [FileId, Offset] -> + on_segment(Msg, FileId, Offset, undefined); + [FileId, Offset, Checksum] -> + on_segment(Msg, FileId, Offset, Checksum); + _ -> + ?RC_UNSPECIFIED_ERROR + end. + +on_init(Msg, FileId) -> + ?SLOG(info, #{ + msg => "on_init", + mqtt_msg => Msg, + file_id => FileId + }), + % Payload = Msg#message.payload, + % %% Add validations here + % Meta = emqx_json:decode(Payload, [return_maps]), + % ok = emqx_ft_storage_fs:store_filemeta(storage(), transfer(Msg, FileId), Meta), + % ?RC_SUCCESS. + ?RC_UNSPECIFIED_ERROR. + +on_abort(_Msg, _FileId) -> + %% TODO + ?RC_SUCCESS. + +on_segment(Msg, FileId, Offset, Checksum) -> + ?SLOG(info, #{ + msg => "on_segment", + mqtt_msg => Msg, + file_id => FileId, + offset => Offset, + checksum => Checksum + }), + % %% TODO: handle checksum + % Payload = Msg#message.payload, + % %% Add offset/checksum validations + % ok = emqx_ft_storage_fs:store_segment( + % storage(), + % transfer(Msg, FileId), + % {binary_to_integer(Offset), Payload} + % ), + % ?RC_SUCCESS. + ?RC_UNSPECIFIED_ERROR. + +on_fin(PacketId, Msg, FileId, Checksum) -> + ?SLOG(info, #{ + msg => "on_fin", + mqtt_msg => Msg, + file_id => FileId, + checksum => Checksum, + packet_id => PacketId + }), + % %% TODO: handle checksum? Do we need it? + % {ok, _} = emqx_ft_storage_fs:assemble( + % storage(), + % transfer(Msg, FileId), + % callback(FileId, Msg) + % ), + Callback = callback(FileId, PacketId), + spawn(fun() -> Callback({error, not_implemented}) end), + undefined. + +callback(_FileId, PacketId) -> + ChanPid = self(), + fun + (ok) -> + erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS}); + ({error, _}) -> + erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}) + end. + +transfer(Msg, FileId) -> + ClientId = Msg#message.from, + {ClientId, FileId}. + +%% TODO: configure +storage() -> + filename:join(emqx:data_dir(), "file_transfer"). diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 2bd01c8b0..dfbb2bd3e 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -16,7 +16,7 @@ -module(emqx_ft_assembler). --export([start_link/2]). +-export([start_link/3]). -behaviour(gen_statem). -export([callback_mode/0]). @@ -35,7 +35,8 @@ transfer :: emqx_ft:transfer(), assembly :: _TODO, file :: io:device(), - hash + hash, + callback :: fun((ok | {error, term()}) -> any()) }). -define(RPC_LIST_TIMEOUT, 1000). @@ -43,8 +44,8 @@ %% -start_link(Storage, Transfer) -> - gen_server:start_link(?MODULE, {Storage, Transfer}, []). +start_link(Storage, Transfer, Callback) -> + gen_server:start_link(?MODULE, {Storage, Transfer, Callback}, []). %% @@ -53,12 +54,13 @@ start_link(Storage, Transfer) -> callback_mode() -> handle_event_function. -init({Storage, Transfer}) -> +init({Storage, Transfer, Callback}) -> St = #st{ storage = Storage, transfer = Transfer, assembly = emqx_ft_assembly:new(), - hash = crypto:hash_init(sha256) + hash = crypto:hash_init(sha256), + callback = Callback }, {ok, list_local_fragments, St, ?internal([])}. @@ -91,7 +93,7 @@ handle_event({list_remote_fragments, Nodes}, internal, _, St) -> fun ({Node, {ok, {ok, Fragments}}}, Asm) -> emqx_ft_assembly:append(Asm, Node, Fragments); - ({Node, Result}, Asm) -> + ({_Node, _Result}, Asm) -> % TODO: log? Asm end, @@ -128,9 +130,11 @@ handle_event({assemble, [{Node, Segment} | Rest]}, internal, _, St = #st{}) -> end; handle_event({assemble, []}, internal, _, St = #st{}) -> {next_state, complete, St, ?internal([])}; -handle_event(complete, internal, _, St = #st{assembly = Asm, file = Handle}) -> +handle_event(complete, internal, _, St = #st{assembly = Asm, file = Handle, callback = Callback}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), - ok = emqx_ft_storage_fs:complete(St#st.storage, St#st.transfer, Filemeta, Handle), + Result = emqx_ft_storage_fs:complete(St#st.storage, St#st.transfer, Filemeta, Handle), + %% TODO: safe apply + _ = Callback(Result), {stop, shutdown}. % handle_continue(list_local, St = #st{storage = Storage, transfer = Transfer, assembly = Asm}) -> diff --git a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl index fbf948fbb..e837d96f1 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl @@ -16,24 +16,22 @@ -module(emqx_ft_assembler_sup). --export([start_link/1]). +-export([start_link/0]). -export([start_child/3]). -behaviour(supervisor). -export([init/1]). --define(REF(ID), {via, gproc, {n, l, {?MODULE, ID}}}). +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). -start_link(ID) -> - supervisor:start_link(?REF(ID), ?MODULE, []). - -start_child(ID, Storage, Transfer) -> +start_child(Storage, Transfer, Callback) -> Childspec = #{ id => {Storage, Transfer}, - start => {emqx_ft_assembler, start_link, [Storage, Transfer]}, + start => {emqx_ft_assembler, start_link, [Storage, Transfer, Callback]}, restart => transient }, - supervisor:start_child(?REF(ID), Childspec). + supervisor:start_child(?MODULE, Childspec). init(_) -> SupFlags = #{ @@ -41,4 +39,4 @@ init(_) -> intensity => 100, period => 1000 }, - {ok, SupFlags, []}. + {ok, {SupFlags, []}}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 2d50e1b18..15efa142b 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -24,7 +24,7 @@ -export([store_filemeta/3]). -export([store_segment/3]). -export([list/2]). --export([assemble/2]). +-export([assemble/3]). -export([open_file/3]). -export([complete/4]). @@ -101,12 +101,12 @@ %% --define(PROCREF(Root), {via, gproc, {n, l, {?MODULE, Root}}}). +% -define(PROCREF(Root), {via, gproc, {n, l, {?MODULE, Root}}}). --spec start_link(root()) -> - {ok, pid()} | {error, already_started}. -start_link(Root) -> - gen_server:start_link(?PROCREF(Root), ?MODULE, [], []). +% -spec start_link(root()) -> +% {ok, pid()} | {error, already_started}. +% start_link(Root) -> +% gen_server:start_link(?PROCREF(Root), ?MODULE, [], []). %% Store manifest in the backing filesystem. %% Atomic operation. @@ -119,13 +119,13 @@ store_filemeta(Storage, Transfer, Meta) -> {ok, Meta} -> _ = touch_file(Filepath), ok; - {ok, Conflict} -> + {ok, _Conflict} -> % TODO % We won't see conflicts in case of concurrent `store_filemeta` % requests. It's rather odd scenario so it's fine not to worry % about it too much now. {error, conflict}; - {error, Reason} when Reason =:= notfound; Reason =:= corrupted -> + {error, Reason} when Reason =:= notfound; Reason =:= corrupted; Reason =:= enoent -> write_file_atomic(Filepath, encode_filemeta(Meta)) end. @@ -154,15 +154,15 @@ list(Storage, Transfer) -> Error end. --spec assemble(storage(), transfer()) -> +-spec assemble(storage(), transfer(), fun((ok | {error, term()}) -> any())) -> % {ok, _Assembler :: pid()} | {error, incomplete} | {error, badrpc} | {error, _TODO}. {ok, _Assembler :: pid()} | {error, _TODO}. -assemble(Storage, Transfer) -> - emqx_ft_assembler_sup:start_child(Storage, Storage, Transfer). +assemble(Storage, Transfer, Callback) -> + emqx_ft_assembler_sup:start_child(Storage, Transfer, Callback). %% --opaque handle() :: {file:name(), io:device(), crypto:hash_state()}. +-type handle() :: {file:name(), io:device(), crypto:hash_state()}. -spec open_file(storage(), transfer(), filemeta()) -> {ok, handle()} | {error, _TODO}. @@ -229,12 +229,12 @@ verify_checksum(undefined, _) -> %% --spec init(root()) -> {ok, storage()}. -init(Root) -> - % TODO: garbage_collect(...) - {ok, Root}. +% -spec init(root()) -> {ok, storage()}. +% init(Root) -> +% % TODO: garbage_collect(...) +% {ok, Root}. -%% +% %% -define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). @@ -243,7 +243,7 @@ schema() -> roots => [ {name, hoconsc:mk(string(), #{required => true})}, {size, hoconsc:mk(non_neg_integer())}, - {expire_at, hoconsc:mk(non_neg_integer(), #{required => true})}, + {expire_at, hoconsc:mk(non_neg_integer())}, {checksum, hoconsc:mk({atom(), binary()}, #{converter => converter(checksum)})}, {segments_ttl, hoconsc:mk(pos_integer())}, {user_data, hoconsc:mk(json_value())} @@ -353,7 +353,7 @@ safe_decode(Content, DecodeFun) -> try {ok, DecodeFun(Content)} catch - C:R:Stacktrace -> + _C:_R:_Stacktrace -> % TODO: Log? {error, corrupted} end. @@ -414,7 +414,7 @@ mk_filefrag(Dirname, Filename, Fun) -> timestamp => Fileinfo#file_info.mtime, fragment => Frag }}; - {error, Reason} -> + {error, _Reason} -> false end. diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index 4c976246b..fb7a6104f 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -43,7 +43,17 @@ init([]) -> intensity => 100, period => 10 }, - ChildSpecs = [], + + AssemblerSup = #{ + id => emqx_ft_assembler_sup, + start => {emqx_ft_assembler_sup, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [emqx_ft_assembler_sup] + }, + + ChildSpecs = [AssemblerSup], {ok, {SupFlags, ChildSpecs}}. %% internal functions diff --git a/rebar.config.erl b/rebar.config.erl index 98cd30570..ea0016ca9 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -400,7 +400,8 @@ relx_apps(ReleaseType, Edition) -> emqx_prometheus, emqx_psk, emqx_slow_subs, - emqx_plugins + emqx_plugins, + emqx_ft ] ++ [quicer || is_quicer_supported()] ++ [bcrypt || provide_bcrypt_release(ReleaseType)] ++ From cbff2e2309666679f2be8756e57f815a7604173b Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 1 Feb 2023 23:31:52 +0200 Subject: [PATCH 005/197] feat(ft): improve robustness of asynchronous acks * add auto ack after timeout * add fin file transfer packet registration to avoid duplication and multiple acks --- apps/emqx_ft/src/emqx_ft.erl | 83 ++++++++++++++---- apps/emqx_ft/src/emqx_ft_responder.erl | 114 +++++++++++++++++++++++++ apps/emqx_ft/src/emqx_ft_sup.erl | 11 ++- 3 files changed, 191 insertions(+), 17 deletions(-) create mode 100644 apps/emqx_ft/src/emqx_ft_responder.erl diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index c73d31559..014c74ac3 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -39,6 +39,8 @@ %% For Debug -export([transfer/2, storage/0]). +-export([on_assemble_timeout/1]). + -export_type([clientid/0]). -export_type([transfer/0]). -export_type([offset/0]). @@ -62,6 +64,8 @@ ft_data :: ft_data() }). +-define(ASSEMBLE_TIMEOUT, 5000). + %%-------------------------------------------------------------------- %% API for app %%-------------------------------------------------------------------- @@ -210,23 +214,66 @@ on_fin(PacketId, Msg, FileId, Checksum) -> checksum => Checksum, packet_id => PacketId }), - % %% TODO: handle checksum? Do we need it? - % {ok, _} = emqx_ft_storage_fs:assemble( - % storage(), - % transfer(Msg, FileId), - % callback(FileId, Msg) - % ), - Callback = callback(FileId, PacketId), - spawn(fun() -> Callback({error, not_implemented}) end), - undefined. + %% TODO: handle checksum? Do we need it? + FinPacketKey = {self(), PacketId}, + _ = + case + emqx_ft_responder:register( + FinPacketKey, fun ?MODULE:on_assemble_timeout/1, ?ASSEMBLE_TIMEOUT + ) + of + %% We have new fin packet + ok -> + Callback = callback(FinPacketKey, FileId), + case assemble(transfer(Msg, FileId), Callback) of + %% Assembling started, packet will be acked by the callback or the responder + ok -> + undefined; + %% Assembling failed, unregister the packet key + {error, _} -> + case emqx_ft_responder:unregister(FinPacketKey) of + %% We successfully unregistered the packet key, + %% so we can send the error code at once + ok -> + ?RC_UNSPECIFIED_ERROR; + %% Someone else already unregistered the key, + %% that is, either responder or someone else acked the packet, + %% we do not have to ack + {error, not_found} -> + undefined + end + end; + %% Fin packet already received. + %% Since we are still handling the previous one, + %% we probably have retransmit here + {error, already_registered} -> + undefined + end. -callback(_FileId, PacketId) -> - ChanPid = self(), - fun - (ok) -> - erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS}); - ({error, _}) -> - erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}) +assemble(_Transfer, _Callback) -> + % spawn(fun() -> Callback({error, not_implemented}) end), + ok. + +% assemble(Transfer, Callback) -> +% emqx_ft_storage_fs:assemble( +% storage(), +% Transfer, +% Callback +% ). + +callback({ChanPid, PacketId} = Key, _FileId) -> + fun(Result) -> + case emqx_ft_responder:unregister(Key) of + ok -> + case Result of + {ok, _} -> + erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS}); + {error, _} -> + erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}) + end; + {error, not_registered} -> + ok + end end. transfer(Msg, FileId) -> @@ -236,3 +283,7 @@ transfer(Msg, FileId) -> %% TODO: configure storage() -> filename:join(emqx:data_dir(), "file_transfer"). + +on_assemble_timeout({ChanPid, PacketId}) -> + ?SLOG(warning, #{msg => "on_assemble_timeout", packet_id => PacketId}), + erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}). diff --git a/apps/emqx_ft/src/emqx_ft_responder.erl b/apps/emqx_ft/src/emqx_ft_responder.erl new file mode 100644 index 000000000..dcb45d5d3 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_responder.erl @@ -0,0 +1,114 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_responder). + +-behaviour(gen_server). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). + +-export([start_link/0]). + +-export([ + register/3, + unregister/1 +]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). + +-define(SERVER, ?MODULE). +-define(TAB, ?MODULE). + +-type key() :: term(). + +%%-------------------------------------------------------------------- +%% API +%% ------------------------------------------------------------------- + +-spec start_link() -> startlink_ret(). +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + +-spec register(Key, DefaultAction, Timeout) -> ok | {error, already_registered} when + Key :: key(), + DefaultAction :: fun((Key) -> any()), + Timeout :: timeout(). +register(Key, DefaultAction, Timeout) -> + case ets:lookup(?TAB, Key) of + [] -> + gen_server:call(?SERVER, {register, Key, DefaultAction, Timeout}); + [{Key, _Action, _Ref}] -> + {error, already_registered} + end. + +-spec unregister(Key) -> ok | {error, not_found} when + Key :: key(). +unregister(Key) -> + gen_server:call(?SERVER, {unregister, Key}). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%% ------------------------------------------------------------------- + +init([]) -> + _ = ets:new(?TAB, [named_table, protected, set, {read_concurrency, true}]), + {ok, #{}}. + +handle_call({register, Key, DefaultAction, Timeout}, _From, State) -> + ?SLOG(warning, #{msg => "register", key => Key, timeout => Timeout}), + case ets:lookup(?TAB, Key) of + [] -> + TRef = erlang:start_timer(Timeout, self(), {timeout, Key}), + true = ets:insert(?TAB, {Key, DefaultAction, TRef}), + {reply, ok, State}; + [{_, _Action, _Ref}] -> + {reply, {error, already_registered}, State} + end; +handle_call({unregister, Key}, _From, State) -> + case ets:lookup(?TAB, Key) of + [] -> + {reply, {error, not_found}, State}; + [{_, _Action, TRef}] -> + _ = erlang:cancel_timer(TRef), + true = ets:delete(?TAB, Key), + {reply, ok, State} + end. + +handle_cast(Msg, State) -> + ?SLOG(warning, #{msg => "unknown cast", cast_msg => Msg}), + {noreply, State}. + +handle_info({timeout, TRef, {timeout, Key}}, State) -> + case ets:lookup(?TAB, Key) of + [] -> + {noreply, State}; + [{_, Action, TRef}] -> + _ = erlang:cancel_timer(TRef), + true = ets:delete(?TAB, Key), + %% TODO: safe apply + _ = Action(Key), + {noreply, State} + end; +handle_info(Msg, State) -> + ?SLOG(warning, #{msg => "unknown message", info_msg => Msg}), + {noreply, State}. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +terminate(_Reason, _State) -> + ok. diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index fb7a6104f..ef2d8033f 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -53,7 +53,16 @@ init([]) -> modules => [emqx_ft_assembler_sup] }, - ChildSpecs = [AssemblerSup], + Responder = #{ + id => emqx_ft_responder, + start => {emqx_ft_responder, start_link, []}, + restart => permanent, + shutdown => infinity, + type => worker, + modules => [emqx_ft_responder] + }, + + ChildSpecs = [Responder, AssemblerSup], {ok, {SupFlags, ChildSpecs}}. %% internal functions From 7b77e96ab9fe3020e93985fb9d570226db98d680 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 2 Feb 2023 21:25:31 +0300 Subject: [PATCH 006/197] test(ft): add some basic assembler tests --- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 163 +++++++++++++ apps/emqx_ft/test/emqx_ft_content_gen.erl | 229 ++++++++++++++++++ 2 files changed, 392 insertions(+) create mode 100644 apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl create mode 100644 apps/emqx_ft/test/emqx_ft_content_gen.erl diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl new file mode 100644 index 000000000..1112ccbb7 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -0,0 +1,163 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_assembler_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + % {ok, Apps} = application:ensure_all_started(emqx_ft), + % [{suite_apps, Apps} | Config]. + % ok = emqx_common_test_helpers:start_apps([emqx_ft]), + Config. + +end_per_suite(_Config) -> + % lists:foreach(fun application:stop/1, lists:reverse(?config(suite_apps, Config))). + % ok = emqx_common_test_helpers:stop_apps([emqx_ft]), + ok. + +init_per_testcase(TC, Config) -> + ok = snabbkaffe:start_trace(), + Root = filename:join(["roots", TC]), + {ok, Pid} = emqx_ft_assembler_sup:start_link(), + [{storage_root, Root}, {assembler_sup, Pid} | Config]. + +end_per_testcase(_TC, Config) -> + ok = inspect_storage_root(Config), + ok = gen:stop(?config(assembler_sup, Config)), + ok = snabbkaffe:stop(), + ok. + +%% + +-define(CLIENTID, <<"thatsme">>). + +t_assemble_empty_transfer(Config) -> + Storage = ?config(storage_root, Config), + Transfer = {?CLIENTID, mk_fileid()}, + Filename = "important.pdf", + Meta = #{ + name => Filename, + size => 0, + expire_at => 42 + }, + ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta), + ?assertMatch( + {ok, [ + #{ + path := _, + timestamp := {{_, _, _}, {_, _, _}}, + fragment := {filemeta, Meta} + } + ]}, + emqx_ft_storage_fs:list(Storage, Transfer) + ), + {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun on_assembly_finished/1), + {ok, Event} = ?block_until(#{?snk_kind := test_assembly_finished}), + ?assertMatch(#{result := ok}, Event), + ?assertEqual( + {ok, <<>>}, + % TODO + file:read_file(mk_assembly_filename(Config, Transfer, Filename)) + ), + ok. + +t_assemble_complete_local_transfer(Config) -> + Storage = ?config(storage_root, Config), + Transfer = {?CLIENTID, mk_fileid()}, + Filename = "topsecret.pdf", + TransferSize = 10000 + rand:uniform(50000), + SegmentSize = 4096, + Gen = emqx_ft_content_gen:new({Transfer, TransferSize}, SegmentSize), + Hash = emqx_ft_content_gen:hash(Gen, crypto:hash_init(sha256)), + Meta = #{ + name => Filename, + checksum => {sha256, Hash}, + expire_at => 42 + }, + + ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta), + _ = emqx_ft_content_gen:consume( + Gen, + fun({Content, SegmentNum, _SegmentCount}) -> + Offset = (SegmentNum - 1) * SegmentSize, + ?assertEqual( + ok, + emqx_ft_storage_fs:store_segment(Storage, Transfer, {Offset, Content}) + ) + end + ), + + {ok, Fragments} = emqx_ft_storage_fs:list(Storage, Transfer), + ?assertEqual((TransferSize div SegmentSize) + 1 + 1, length(Fragments)), + ?assertEqual( + [Meta], + [FM || #{fragment := {filemeta, FM}} <- Fragments], + Fragments + ), + + {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun on_assembly_finished/1), + {ok, Event} = ?block_until(#{?snk_kind := test_assembly_finished}), + ?assertMatch(#{result := ok}, Event), + + AssemblyFilename = mk_assembly_filename(Config, Transfer, Filename), + ?assertMatch( + {ok, #file_info{type = regular, size = TransferSize}}, + file:read_file_info(AssemblyFilename) + ), + ok = emqx_ft_content_gen:check_file_consistency( + {Transfer, TransferSize}, + 100, + AssemblyFilename + ). + +mk_assembly_filename(Config, {ClientID, FileID}, Filename) -> + filename:join([?config(storage_root, Config), ClientID, FileID, Filename]). + +on_assembly_finished(Result) -> + ?tp(test_assembly_finished, #{result => Result}). + +%% + +-include_lib("kernel/include/file.hrl"). + +inspect_storage_root(Config) -> + inspect_dir(?config(storage_root, Config)). + +inspect_dir(Dir) -> + FileInfos = filelib:fold_files( + Dir, + ".*", + true, + fun(Filename, Acc) -> orddict:store(Filename, inspect_file(Filename), Acc) end, + orddict:new() + ), + ct:pal("inspect '~s': ~p", [Dir, FileInfos]). + +inspect_file(Filename) -> + {ok, Info} = file:read_file_info(Filename), + {Info#file_info.type, Info#file_info.size, Info#file_info.mtime}. + +mk_fileid() -> + integer_to_binary(erlang:system_time(millisecond)). diff --git a/apps/emqx_ft/test/emqx_ft_content_gen.erl b/apps/emqx_ft/test/emqx_ft_content_gen.erl new file mode 100644 index 000000000..feca78949 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_content_gen.erl @@ -0,0 +1,229 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Inspired by +%% https://github.com/kafka4beam/kflow/blob/master/src/testbed/payload_gen.erl + +-module(emqx_ft_content_gen). + +-include_lib("eunit/include/eunit.hrl"). + +-dialyzer(no_improper_lists). + +-export([new/2]). +-export([generate/3]). +-export([next/1]). +-export([consume/1]). +-export([consume/2]). +-export([fold/3]). + +-export([hash/2]). +-export([check_file_consistency/3]). + +-export_type([cont/1]). +-export_type([stream/1]). +-export_type([binary_payload/0]). + +-define(hash_size, 16). + +-type payload() :: {Seed :: term(), Size :: integer()}. + +-type binary_payload() :: { + binary(), _ChunkNum :: non_neg_integer(), _ChunkCnt :: non_neg_integer() +}. + +-type cont(Data) :: + fun(() -> stream(Data)) + | stream(Data). + +-type stream(Data) :: + maybe_improper_list(Data, cont(Data)) + | eos. + +-record(chunk_state, { + seed :: term(), + payload_size :: non_neg_integer(), + offset :: non_neg_integer(), + chunk_size :: non_neg_integer() +}). + +-type chunk_state() :: #chunk_state{}. + +%% ----------------------------------------------------------------------------- +%% Generic streams +%% ----------------------------------------------------------------------------- + +%% @doc Consume one element from the stream. +-spec next(cont(A)) -> stream(A). +next(Fun) when is_function(Fun, 0) -> + Fun(); +next(L) -> + L. + +%% @doc Consume all elements of the stream and feed them into a +%% callback (e.g. brod:produce) +-spec consume(cont(A), fun((A) -> Ret)) -> [Ret]. +consume([Data | Cont], Callback) -> + [Callback(Data) | consume(next(Cont), Callback)]; +consume(Cont, Callback) when is_function(Cont, 0) -> + consume(next(Cont), Callback); +consume(eos, _Callback) -> + []. + +%% @equiv consume(Stream, fun(A) -> A end) +-spec consume(cont(A)) -> [A]. +consume(Stream) -> + consume(Stream, fun(A) -> A end). + +-spec fold(fun((A, Acc) -> Acc), Acc, cont(A)) -> Acc. +fold(Fun, Acc, [Data | Cont]) -> + fold(Fun, Fun(Data, Acc), next(Cont)); +fold(Fun, Acc, Cont) when is_function(Cont, 0) -> + fold(Fun, Acc, next(Cont)); +fold(_Fun, Acc, eos) -> + Acc. + +%% ----------------------------------------------------------------------------- +%% Binary streams +%% ----------------------------------------------------------------------------- + +%% @doc Stream of binary chunks. +%% Limitation: `ChunkSize' should be dividable by `?hash_size' +-spec new(payload(), integer()) -> cont(binary_payload()). +new({Seed, Size}, ChunkSize) when ChunkSize rem ?hash_size =:= 0 -> + fun() -> + generate_next_chunk(#chunk_state{ + seed = Seed, + payload_size = Size, + chunk_size = ChunkSize, + offset = 0 + }) + end. + +%% @doc Generate chunks of data and feed them into +%% `Callback' +-spec generate(payload(), integer(), fun((binary_payload()) -> A)) -> [A]. +generate(Payload, ChunkSize, Callback) -> + consume(new(Payload, ChunkSize), Callback). + +-spec hash(cont(binary_payload()), crypto:hash_state()) -> binary(). +hash(Stream, HashCtxIn) -> + crypto:hash_final( + fold( + fun({Chunk, _, _}, HashCtx) -> + crypto:hash_update(HashCtx, Chunk) + end, + HashCtxIn, + Stream + ) + ). + +-spec check_consistency( + payload(), + integer(), + fun((integer()) -> {ok, binary()} | undefined) +) -> ok. +check_consistency({Seed, Size}, SampleSize, Callback) -> + SeedHash = seed_hash(Seed), + Random = [rand:uniform(Size) - 1 || _ <- lists:seq(1, SampleSize)], + %% Always check first and last bytes, and one that should not exist: + Samples = [0, Size - 1, Size | Random], + lists:foreach( + fun + (N) when N < Size -> + Expected = do_get_byte(N, SeedHash), + ?assertEqual( + {N, {ok, Expected}}, + {N, Callback(N)} + ); + (N) -> + ?assertMatch(undefined, Callback(N)) + end, + Samples + ). + +-spec check_file_consistency( + payload(), + integer(), + file:filename() +) -> ok. +check_file_consistency(Payload, SampleSize, FileName) -> + {ok, FD} = file:open(FileName, [read, raw]), + try + Fun = fun(N) -> + case file:pread(FD, [{N, 1}]) of + {ok, [[X]]} -> {ok, X}; + {ok, [eof]} -> undefined + end + end, + check_consistency(Payload, SampleSize, Fun) + after + file:close(FD) + end. + +%% ============================================================================= +%% Internal functions +%% ============================================================================= + +%% @doc Continue generating chunks +-spec generate_next_chunk(chunk_state()) -> stream(binary()). +generate_next_chunk(#chunk_state{offset = Offset, payload_size = Size}) when Offset >= Size -> + eos; +generate_next_chunk(State0 = #chunk_state{offset = Offset, chunk_size = ChunkSize}) -> + State = State0#chunk_state{offset = Offset + ChunkSize}, + Payload = generate_chunk( + State#chunk_state.seed, + Offset, + ChunkSize, + State#chunk_state.payload_size + ), + [Payload | fun() -> generate_next_chunk(State) end]. + +generate_chunk(Seed, Offset, ChunkSize, Size) -> + SeedHash = seed_hash(Seed), + To = min(Offset + ChunkSize, Size) - 1, + Payload = iolist_to_binary([ + generator_fun(I, SeedHash) + || I <- lists:seq(Offset div 16, To div 16) + ]), + ChunkNum = Offset div ChunkSize + 1, + ChunkCnt = ceil(Size / ChunkSize), + Chunk = + case Offset + ChunkSize of + NextOffset when NextOffset > Size -> + binary:part(Payload, 0, Size rem ChunkSize); + _ -> + Payload + end, + {Chunk, ChunkNum, ChunkCnt}. + +%% @doc First argument is a chunk number, the second one is a seed. +%% This implementation is hardly efficient, but it was chosen for +%% clarity reasons +-spec generator_fun(integer(), binary()) -> binary(). +generator_fun(N, Seed) -> + crypto:hash(md5, <>). + +%% @doc Hash any term +-spec seed_hash(term()) -> binary(). +seed_hash(Seed) -> + crypto:hash(md5, term_to_binary(Seed)). + +%% @private Get byte at offset `N' +-spec do_get_byte(integer(), binary()) -> byte(). +do_get_byte(N, Seed) -> + Chunk = generator_fun(N div ?hash_size, Seed), + binary:at(Chunk, N rem ?hash_size). From 1fedae8a1633b54f30ca9fdc16a1a5732870e238 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 2 Feb 2023 21:29:35 +0300 Subject: [PATCH 007/197] fix(ft-asm): ensure module follows statem behaviour --- apps/emqx_ft/src/emqx_ft_assembler.erl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index dfbb2bd3e..4fd8d6e75 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -45,7 +45,7 @@ %% start_link(Storage, Transfer, Callback) -> - gen_server:start_link(?MODULE, {Storage, Transfer, Callback}, []). + gen_statem:start_link(?MODULE, {Storage, Transfer, Callback}, []). %% @@ -64,7 +64,7 @@ init({Storage, Transfer, Callback}) -> }, {ok, list_local_fragments, St, ?internal([])}. -handle_event(list_local_fragments, internal, _, St = #st{assembly = Asm}) -> +handle_event(internal, _, list_local_fragments, St = #st{assembly = Asm}) -> % TODO: what we do with non-transients errors here (e.g. `eacces`)? {ok, Fragments} = emqx_ft_storage_fs:list(St#st.storage, St#st.transfer), NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(Asm, node(), Fragments)), @@ -79,7 +79,7 @@ handle_event(list_local_fragments, internal, _, St = #st{assembly = Asm}) -> % {error, _} = Reason -> % {stop, Reason} end; -handle_event({list_remote_fragments, Nodes}, internal, _, St) -> +handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> % TODO: portable "storage" ref Args = [St#st.storage, St#st.transfer], % TODO @@ -109,13 +109,13 @@ handle_event({list_remote_fragments, Nodes}, internal, _, St) -> {incomplete, _} = Status -> {stop, {error, Status}} end; -handle_event(start_assembling, internal, _, St = #st{assembly = Asm}) -> +handle_event(internal, _, start_assembling, St = #st{assembly = Asm}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), Coverage = emqx_ft_assembly:coverage(Asm), % TODO: errors {ok, Handle} = emqx_ft_storage_fs:open_file(St#st.storage, St#st.transfer, Filemeta), {next_state, {assemble, Coverage}, St#st{file = Handle}, ?internal([])}; -handle_event({assemble, [{Node, Segment} | Rest]}, internal, _, St = #st{}) -> +handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % TODO % Currently, race is possible between getting segment info from the remote node and % this node garbage collecting the segment itself. @@ -128,9 +128,9 @@ handle_event({assemble, [{Node, Segment} | Rest]}, internal, _, St = #st{}) -> % {error, _} -> % ... end; -handle_event({assemble, []}, internal, _, St = #st{}) -> +handle_event(internal, _, {assemble, []}, St = #st{}) -> {next_state, complete, St, ?internal([])}; -handle_event(complete, internal, _, St = #st{assembly = Asm, file = Handle, callback = Callback}) -> +handle_event(internal, _, complete, St = #st{assembly = Asm, file = Handle, callback = Callback}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), Result = emqx_ft_storage_fs:complete(St#st.storage, St#st.transfer, Filemeta, Handle), %% TODO: safe apply @@ -160,5 +160,5 @@ handle_event(complete, internal, _, St = #st{assembly = Asm, file = Handle, call %% -segsize(#{fragment := {segmentinfo, Info}}) -> +segsize(#{fragment := {segment, Info}}) -> maps:get(size, Info). From 14b2a1013bedba30755b8f4c963556d2c19b6b24 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 2 Feb 2023 21:30:30 +0300 Subject: [PATCH 008/197] fix(ft-asm): follow proper `segment` fragment type --- apps/emqx_ft/src/emqx_ft_assembly.erl | 80 +++++++++++++-------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index 854b420e0..0f2729f42 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -47,7 +47,7 @@ append(Asm, Node, Fragments) when is_list(Fragments) -> lists:foldl(fun(F, AsmIn) -> append(AsmIn, Node, F) end, Asm, Fragments); append(Asm, Node, Fragment = #{fragment := {filemeta, _}}) -> append_filemeta(Asm, Node, Fragment); -append(Asm, Node, Segment = #{fragment := {segmentinfo, _}}) -> +append(Asm, Node, Segment = #{fragment := {segment, _}}) -> append_segmentinfo(Asm, Node, Segment). update(Asm) -> @@ -105,7 +105,7 @@ append_filemeta(Asm, Node, Fragment = #{fragment := {filemeta, Meta}}) -> meta = orddict:store(Meta, {Node, Fragment}, Asm#asm.meta) }. -append_segmentinfo(Asm, Node, Fragment = #{fragment := {segmentinfo, Info}}) -> +append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) -> Offset = maps:get(offset, Info), Size = maps:get(size, Info), End = Offset + Size, @@ -176,7 +176,7 @@ locality(Node) when Node =:= node() -> locality(_RemoteNode) -> 1. -segsize(#{fragment := {segmentinfo, Info}}) -> +segsize(#{fragment := {segment, Info}}) -> maps:get(size, Info). -ifdef(TEST). @@ -195,8 +195,8 @@ incomplete_test() -> status( update( append(new(), node(), [ - segmentinfo(p1, 0, 42), - segmentinfo(p1, 42, 100) + segment(p1, 0, 42), + segment(p1, 42, 100) ]) ) ) @@ -204,14 +204,14 @@ incomplete_test() -> consistent_test() -> Asm1 = append(new(), n1, [filemeta(m1, "blarg")]), - Asm2 = append(Asm1, n2, [segmentinfo(s2, 0, 42)]), + Asm2 = append(Asm1, n2, [segment(s2, 0, 42)]), Asm3 = append(Asm2, n3, [filemeta(m3, "blarg")]), ?assertMatch({complete, _}, status(meta, Asm3)). inconsistent_test() -> - Asm1 = append(new(), node(), [segmentinfo(s1, 0, 42)]), + Asm1 = append(new(), node(), [segment(s1, 0, 42)]), Asm2 = append(Asm1, n1, [filemeta(m1, "blarg")]), - Asm3 = append(Asm2, n2, [segmentinfo(s2, 0, 42), filemeta(m1, "blorg")]), + Asm3 = append(Asm2, n2, [segment(s2, 0, 42), filemeta(m1, "blorg")]), Asm4 = append(Asm3, n3, [filemeta(m3, "blarg")]), ?assertMatch( {error, @@ -226,10 +226,10 @@ inconsistent_test() -> simple_coverage_test() -> Node = node(), Segs = [ - {node42, segmentinfo(n1, 20, 30)}, - {Node, segmentinfo(n2, 0, 10)}, - {Node, segmentinfo(n3, 50, 50)}, - {Node, segmentinfo(n4, 10, 10)} + {node42, segment(n1, 20, 30)}, + {Node, segment(n2, 0, 10)}, + {Node, segment(n3, 50, 50)}, + {Node, segment(n4, 10, 10)} ], Asm = append_many(new(), Segs), ?assertMatch( @@ -247,14 +247,14 @@ simple_coverage_test() -> redundant_coverage_test() -> Node = node(), Segs = [ - {Node, segmentinfo(n1, 0, 20)}, - {node1, segmentinfo(n2, 0, 10)}, - {Node, segmentinfo(n3, 20, 40)}, - {node2, segmentinfo(n4, 10, 10)}, - {node2, segmentinfo(n5, 50, 20)}, - {node3, segmentinfo(n6, 20, 20)}, - {Node, segmentinfo(n7, 50, 10)}, - {node1, segmentinfo(n8, 40, 10)} + {Node, segment(n1, 0, 20)}, + {node1, segment(n2, 0, 10)}, + {Node, segment(n3, 20, 40)}, + {node2, segment(n4, 10, 10)}, + {node2, segment(n5, 50, 20)}, + {node3, segment(n6, 20, 20)}, + {Node, segment(n7, 50, 10)}, + {node1, segment(n8, 40, 10)} ], Asm = append_many(new(), Segs), ?assertMatch( @@ -272,12 +272,12 @@ redundant_coverage_test() -> redundant_coverage_prefer_local_test() -> Node = node(), Segs = [ - {node1, segmentinfo(n1, 0, 20)}, - {Node, segmentinfo(n2, 0, 10)}, - {Node, segmentinfo(n3, 10, 10)}, - {node2, segmentinfo(n4, 20, 20)}, - {Node, segmentinfo(n5, 30, 10)}, - {Node, segmentinfo(n6, 20, 10)} + {node1, segment(n1, 0, 20)}, + {Node, segment(n2, 0, 10)}, + {Node, segment(n3, 10, 10)}, + {node2, segment(n4, 20, 20)}, + {Node, segment(n5, 30, 10)}, + {Node, segment(n6, 20, 10)} ], Asm = append_many(new(), Segs), ?assertMatch( @@ -295,11 +295,11 @@ redundant_coverage_prefer_local_test() -> missing_coverage_test() -> Node = node(), Segs = [ - {Node, segmentinfo(n1, 0, 10)}, - {node1, segmentinfo(n3, 10, 20)}, - {Node, segmentinfo(n2, 0, 20)}, - {node2, segmentinfo(n4, 50, 50)}, - {Node, segmentinfo(n5, 40, 60)} + {Node, segment(n1, 0, 10)}, + {node1, segment(n3, 10, 20)}, + {Node, segment(n2, 0, 20)}, + {node2, segment(n4, 50, 50)}, + {Node, segment(n5, 40, 60)} ], Asm = append_many(new(), Segs), ?assertEqual( @@ -311,8 +311,8 @@ missing_coverage_test() -> missing_end_coverage_test() -> Node = node(), Segs = [ - {Node, segmentinfo(n1, 0, 15)}, - {node1, segmentinfo(n3, 10, 10)} + {Node, segment(n1, 0, 15)}, + {node1, segment(n3, 10, 10)} ], Asm = append_many(new(), Segs), ?assertEqual( @@ -322,11 +322,11 @@ missing_end_coverage_test() -> missing_coverage_with_redudancy_test() -> Segs = [ - {node(), segmentinfo(n1, 0, 10)}, - {node(), segmentinfo(n2, 0, 20)}, - {node42, segmentinfo(n3, 10, 20)}, - {node43, segmentinfo(n4, 10, 50)}, - {node(), segmentinfo(n5, 40, 60)} + {node(), segment(n1, 0, 10)}, + {node(), segment(n2, 0, 20)}, + {node42, segment(n3, 10, 20)}, + {node43, segment(n4, 10, 50)}, + {node(), segment(n5, 40, 60)} ], Asm = append_many(new(), Segs), ?assertEqual( @@ -351,11 +351,11 @@ filemeta(Path, Name) -> }} }. -segmentinfo(Path, Offset, Size) -> +segment(Path, Offset, Size) -> #{ path => Path, fragment => - {segmentinfo, #{ + {segment, #{ offset => Offset, size => Size }} From 97b831a160b6a622911ee7e199904c2a8bc8f741 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 2 Feb 2023 21:31:04 +0300 Subject: [PATCH 009/197] fix(ft-fs): add missing `read_segment/5` + fix atomic write --- apps/emqx_ft/src/emqx_ft_assembler_sup.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 59 +++++++++++++++++----- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl index e837d96f1..b60949a5e 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl @@ -36,7 +36,7 @@ start_child(Storage, Transfer, Callback) -> init(_) -> SupFlags = #{ strategy => one_for_one, - intensity => 100, + intensity => 10, period => 1000 }, {ok, {SupFlags, []}}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 15efa142b..cce7cc19e 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -24,6 +24,7 @@ -export([store_filemeta/3]). -export([store_segment/3]). -export([list/2]). +-export([read_segment/5]). -export([assemble/3]). -export([open_file/3]). @@ -154,6 +155,28 @@ list(Storage, Transfer) -> Error end. +-spec read_segment( + storage(), transfer(), filefrag(segmentinfo()), offset(), _Size :: non_neg_integer() +) -> + {ok, _Content :: iodata()} | {error, _TODO}. +read_segment(_Storage, _Transfer, Segment, Offset, Size) -> + Filepath = maps:get(path, Segment), + case file:open(Filepath, [raw, read]) of + {ok, IoDevice} -> + Read = file:pread(IoDevice, Offset, Size), + ok = file:close(IoDevice), + case Read of + {ok, Content} -> + {ok, Content}; + eof -> + {error, eof}; + {error, Reason} -> + {error, Reason} + end; + {error, Reason} -> + {error, Reason} + end. + -spec assemble(storage(), transfer(), fun((ok | {error, term()}) -> any())) -> % {ok, _Assembler :: pid()} | {error, incomplete} | {error, badrpc} | {error, _TODO}. {ok, _Assembler :: pid()} | {error, _TODO}. @@ -359,18 +382,18 @@ safe_decode(Content, DecodeFun) -> end. write_file_atomic(Filepath, Content) when is_binary(Content) -> + TempFilepath = mk_temp_filepath(Filepath), Result = emqx_misc:pipeline( [ fun filelib:ensure_dir/1, - fun mk_temp_filepath/1, fun write_contents/2, - fun mv_temp_file/1 + fun(FP) -> mv_temp_file(Filepath, FP) end ], - Filepath, + TempFilepath, Content ), case Result of - {ok, {Filepath, TempFilepath}, _} -> + {ok, _, _} -> _ = file:delete(TempFilepath), ok; {error, Reason, _} -> @@ -381,13 +404,20 @@ mk_temp_filepath(Filepath) -> Dirname = filename:dirname(Filepath), Filename = filename:basename(Filepath), Unique = erlang:unique_integer([positive]), - TempFilepath = filename:join(Dirname, ?TEMP ++ integer_to_list(Unique) ++ "." ++ Filename), - {Filepath, TempFilepath}. + filename:join(Dirname, mk_filename([?TEMP, Unique, ".", Filename])). -write_contents({_Filepath, TempFilepath}, Content) -> - file:write_file(TempFilepath, Content). +mk_filename(Comps) -> + lists:append(lists:map(fun mk_filename_component/1, Comps)). -mv_temp_file({Filepath, TempFilepath}) -> +mk_filename_component(I) when is_integer(I) -> integer_to_list(I); +mk_filename_component(A) when is_atom(A) -> atom_to_list(A); +mk_filename_component(B) when is_binary(B) -> unicode:characters_to_list(B); +mk_filename_component(S) when is_list(S) -> S. + +write_contents(Filepath, Content) -> + file:write_file(Filepath, Content). + +mv_temp_file(Filepath, TempFilepath) -> file:rename(TempFilepath, Filepath). touch_file(Filepath) -> @@ -398,21 +428,22 @@ filtermap_files(Fun, Dirname, Filenames) -> lists:filtermap(fun(Filename) -> Fun(Dirname, Filename) end, Filenames). mk_filefrag(Dirname, Filename = ?MANIFEST) -> - mk_filefrag(Dirname, Filename, fun read_filemeta/2); + mk_filefrag(Dirname, Filename, filemeta, fun read_filemeta/2); mk_filefrag(Dirname, Filename = ?SEGMENT ++ _) -> - mk_filefrag(Dirname, Filename, fun read_segmentinfo/2); + mk_filefrag(Dirname, Filename, segment, fun read_segmentinfo/2); mk_filefrag(_Dirname, _) -> false. -mk_filefrag(Dirname, Filename, Fun) -> +mk_filefrag(Dirname, Filename, Tag, Fun) -> Filepath = filename:join(Dirname, Filename), - Fileinfo = file:read_file_info(Filepath), + % TODO error handling? + {ok, Fileinfo} = file:read_file_info(Filepath), case Fun(Filename, Filepath) of {ok, Frag} -> {true, #{ path => Filepath, timestamp => Fileinfo#file_info.mtime, - fragment => Frag + fragment => {Tag, Frag} }}; {error, _Reason} -> false From 72e3eee6c93fa212eebe9ba62ab567b3dfd0421d Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 2 Feb 2023 20:23:12 +0200 Subject: [PATCH 010/197] feat(ft): add config & backend behaviour --- apps/emqx_conf/src/emqx_conf_schema.erl | 3 +- apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 14 +++ apps/emqx_ft/src/emqx_ft.erl | 126 +++++++++++++-------- apps/emqx_ft/src/emqx_ft_app.erl | 2 + apps/emqx_ft/src/emqx_ft_conf.erl | 65 +++++++++++ apps/emqx_ft/src/emqx_ft_responder.erl | 1 + apps/emqx_ft/src/emqx_ft_schema.erl | 49 ++++++++ apps/emqx_ft/src/emqx_ft_storage.erl | 75 ++++++++++++ apps/emqx_ft/src/emqx_ft_storage_dummy.erl | 35 ++++++ apps/emqx_ft/src/emqx_ft_storage_fs.erl | 71 +++--------- 10 files changed, 342 insertions(+), 99 deletions(-) create mode 100644 apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf create mode 100644 apps/emqx_ft/src/emqx_ft_conf.erl create mode 100644 apps/emqx_ft/src/emqx_ft_schema.erl create mode 100644 apps/emqx_ft/src/emqx_ft_storage.erl create mode 100644 apps/emqx_ft/src/emqx_ft_storage_dummy.erl diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 58bcf9700..1a0f8bbb1 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -63,7 +63,8 @@ emqx_psk_schema, emqx_limiter_schema, emqx_slow_subs_schema, - emqx_mgmt_api_key_schema + emqx_mgmt_api_key_schema, + emqx_ft_schema ]). %% root config should not have a namespace diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf new file mode 100644 index 000000000..85fb81cfe --- /dev/null +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -0,0 +1,14 @@ +emqx_ft_schema { + + local { + desc { + en: "Use local file system to store uploaded files and temporary data." + zh: "使用本地文件系统来存储上传的文件和临时数据。" + } + label: { + en: "Local Storage" + zh: "本地存储" + } + } + +} diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 014c74ac3..a286a6186 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -41,9 +41,13 @@ -export([on_assemble_timeout/1]). --export_type([clientid/0]). --export_type([transfer/0]). --export_type([offset/0]). +-export_type([ + clientid/0, + transfer/0, + offset/0, + filemeta/0, + segment/0 +]). %% Number of bytes -type bytes() :: non_neg_integer(). @@ -55,6 +59,26 @@ -type transfer() :: {clientid(), fileid()}. -type offset() :: bytes(). +-type filemeta() :: #{ + %% Display name + name := string(), + %% Size in bytes, as advertised by the client. + %% Client is free to specify here whatever it wants, which means we can end + %% up with a file of different size after assembly. It's not clear from + %% specification what that means (e.g. what are clients' expectations), we + %% currently do not condider that an error (or, specifically, a signal that + %% the resulting file is corrupted during transmission). + size => _Bytes :: non_neg_integer(), + checksum => {sha256, <<_:256>>}, + expire_at := emqx_datetime:epoch_second(), + %% TTL of individual segments + %% Somewhat confusing that we won't know it on the nodes where the filemeta + %% is missing. + segments_ttl => _Seconds :: pos_integer() +}. + +-type segment() :: {offset(), _Content :: binary()}. + -type ft_data() :: #{ nodes := list(node()) }. @@ -135,23 +159,9 @@ on_message_puback(PacketId, #message{topic = Topic} = Msg, _PubRes, _RC) -> end. %%-------------------------------------------------------------------- -%% Private funs +%% Handlers for transfer messages %%-------------------------------------------------------------------- -get_ft_data(ChanPid) -> - case ets:lookup(?FT_TAB, ChanPid) of - [#emqx_ft{ft_data = FTData}] -> {ok, FTData}; - [] -> none - end. - -delete_ft_data(ChanPid) -> - true = ets:delete(?FT_TAB, ChanPid), - ok. - -put_ft_data(ChanPid, FTData) -> - true = ets:insert(?FT_TAB, #emqx_ft{chan_pid = ChanPid, ft_data = FTData}), - ok. - on_file_command(PacketId, Msg, FileCommand) -> case string:split(FileCommand, <<"/">>, all) of [FileId, <<"init">>] -> @@ -176,12 +186,16 @@ on_init(Msg, FileId) -> mqtt_msg => Msg, file_id => FileId }), - % Payload = Msg#message.payload, + Payload = Msg#message.payload, % %% Add validations here - % Meta = emqx_json:decode(Payload, [return_maps]), - % ok = emqx_ft_storage_fs:store_filemeta(storage(), transfer(Msg, FileId), Meta), - % ?RC_SUCCESS. - ?RC_UNSPECIFIED_ERROR. + Meta = emqx_json:decode(Payload, [return_maps]), + case emqx_ft_storage:store_filemeta(transfer(Msg, FileId), Meta) of + {ok, Ctx} -> + ok = put_context(Ctx), + ?RC_SUCCESS; + {error, _Reason} -> + ?RC_UNSPECIFIED_ERROR + end. on_abort(_Msg, _FileId) -> %% TODO @@ -195,16 +209,17 @@ on_segment(Msg, FileId, Offset, Checksum) -> offset => Offset, checksum => Checksum }), - % %% TODO: handle checksum - % Payload = Msg#message.payload, - % %% Add offset/checksum validations - % ok = emqx_ft_storage_fs:store_segment( - % storage(), - % transfer(Msg, FileId), - % {binary_to_integer(Offset), Payload} - % ), - % ?RC_SUCCESS. - ?RC_UNSPECIFIED_ERROR. + %% TODO: handle checksum + Payload = Msg#message.payload, + Segment = {binary_to_integer(Offset), Payload}, + %% Add offset/checksum validations + case emqx_ft_storage:store_segment(get_context(), transfer(Msg, FileId), Segment) of + {ok, Ctx} -> + ok = put_context(Ctx), + ?RC_SUCCESS; + {error, _Reason} -> + ?RC_UNSPECIFIED_ERROR + end. on_fin(PacketId, Msg, FileId, Checksum) -> ?SLOG(info, #{ @@ -227,7 +242,7 @@ on_fin(PacketId, Msg, FileId, Checksum) -> Callback = callback(FinPacketKey, FileId), case assemble(transfer(Msg, FileId), Callback) of %% Assembling started, packet will be acked by the callback or the responder - ok -> + {ok, _} -> undefined; %% Assembling failed, unregister the packet key {error, _} -> @@ -250,16 +265,12 @@ on_fin(PacketId, Msg, FileId, Checksum) -> undefined end. -assemble(_Transfer, _Callback) -> - % spawn(fun() -> Callback({error, not_implemented}) end), - ok. - -% assemble(Transfer, Callback) -> -% emqx_ft_storage_fs:assemble( -% storage(), -% Transfer, -% Callback -% ). +assemble(Transfer, Callback) -> + emqx_ft_storage:assemble( + get_context(), + Transfer, + Callback + ). callback({ChanPid, PacketId} = Key, _FileId) -> fun(Result) -> @@ -281,9 +292,34 @@ transfer(Msg, FileId) -> {ClientId, FileId}. %% TODO: configure + storage() -> - filename:join(emqx:data_dir(), "file_transfer"). + emqx_config:get([file_transfer, storage]). on_assemble_timeout({ChanPid, PacketId}) -> ?SLOG(warning, #{msg => "on_assemble_timeout", packet_id => PacketId}), erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}). + +%%-------------------------------------------------------------------- +%% Context management +%%-------------------------------------------------------------------- + +get_context() -> + get_ft_data(self()). + +put_context(Context) -> + put_ft_data(self(), Context). + +get_ft_data(ChanPid) -> + case ets:lookup(?FT_TAB, ChanPid) of + [#emqx_ft{ft_data = FTData}] -> {ok, FTData}; + [] -> none + end. + +delete_ft_data(ChanPid) -> + true = ets:delete(?FT_TAB, ChanPid), + ok. + +put_ft_data(ChanPid, FTData) -> + true = ets:insert(?FT_TAB, #emqx_ft{chan_pid = ChanPid, ft_data = FTData}), + ok. diff --git a/apps/emqx_ft/src/emqx_ft_app.erl b/apps/emqx_ft/src/emqx_ft_app.erl index 4778da1a1..9b1513b46 100644 --- a/apps/emqx_ft/src/emqx_ft_app.erl +++ b/apps/emqx_ft/src/emqx_ft_app.erl @@ -23,8 +23,10 @@ start(_StartType, _StartArgs) -> {ok, Sup} = emqx_ft_sup:start_link(), ok = emqx_ft:hook(), + ok = emqx_ft_conf:load(), {ok, Sup}. stop(_State) -> + ok = emqx_ft_conf:unload(), ok = emqx_ft:unhook(), ok. diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl new file mode 100644 index 000000000..b88fd2532 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -0,0 +1,65 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc File Transfer configuration management module + +-module(emqx_ft_conf). + +-behaviour(emqx_config_handler). + +%% Load/Unload +-export([ + load/0, + unload/0 +]). + +%% callbacks for emqx_config_handler +-export([ + pre_config_update/3, + post_config_update/5 +]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec load() -> ok. +load() -> + emqx_conf:add_handler([file_transfer], ?MODULE). + +-spec unload() -> ok. +unload() -> + emqx_conf:remove_handler([file_transfer]). + +%%-------------------------------------------------------------------- +%% emqx_config_handler callbacks +%%-------------------------------------------------------------------- + +-spec pre_config_update(list(atom()), emqx_config:update_request(), emqx_config:raw_config()) -> + {ok, emqx_config:update_request()} | {error, term()}. +pre_config_update(_, _Req, Config) -> + {ok, Config}. + +-spec post_config_update( + list(atom()), + emqx_config:update_request(), + emqx_config:config(), + emqx_config:config(), + emqx_config:app_envs() +) -> + ok | {ok, Result :: any()} | {error, Reason :: term()}. +post_config_update(_, _Req, _NewConfig, _OldConfig, _AppEnvs) -> + ok. diff --git a/apps/emqx_ft/src/emqx_ft_responder.erl b/apps/emqx_ft/src/emqx_ft_responder.erl index dcb45d5d3..f58569a27 100644 --- a/apps/emqx_ft/src/emqx_ft_responder.erl +++ b/apps/emqx_ft/src/emqx_ft_responder.erl @@ -79,6 +79,7 @@ handle_call({register, Key, DefaultAction, Timeout}, _From, State) -> {reply, {error, already_registered}, State} end; handle_call({unregister, Key}, _From, State) -> + ?SLOG(warning, #{msg => "unregister", key => Key}), case ets:lookup(?TAB, Key) of [] -> {reply, {error, not_found}, State}; diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl new file mode 100644 index 000000000..f40d2f40e --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -0,0 +1,49 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_schema). + +-behaviour(hocon_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-export([namespace/0, roots/0, fields/1, tags/0]). + +namespace() -> file_transfer. + +tags() -> + [<<"File Transfer">>]. + +roots() -> [file_transfer]. + +fields(file_transfer) -> + [ + {storage, #{ + type => hoconsc:union([ + hoconsc:ref(?MODULE, local_storage) + ]) + }} + ]; +fields(local_storage) -> + [ + {type, #{ + type => local, + default => local, + required => false, + desc => ?DESC("local") + }} + ]. diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl new file mode 100644 index 000000000..5e945965d --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -0,0 +1,75 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage). + +-export( + [ + store_filemeta/2, + store_segment/3, + assemble/3 + ] +). + +-type ctx() :: term(). +-type storage() :: emqx_config:config(). + +-export_type([assemble_callback/0]). + +-type assemble_callback() :: fun((ok | {error, term()}) -> any()). + +%%-------------------------------------------------------------------- +%% behaviour +%%-------------------------------------------------------------------- + +-callback store_filemeta(storage(), emqx_ft:transfer(), emqx_ft:filemeta()) -> + {ok, ctx()} | {error, term()}. +-callback store_segment(storage(), ctx(), emqx_ft:transfer(), emqx_ft:segment()) -> + {ok, ctx()} | {error, term()}. +-callback assemble(storage(), ctx(), emqx_ft:transfer(), assemble_callback()) -> + {ok, pid()} | {error, term()}. + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec store_filemeta(emqx_ft:transfer(), emqx_ft:filemeta()) -> + {ok, ctx()} | {error, term()}. +store_filemeta(Transfer, FileMeta) -> + Mod = mod(), + Mod:store_filemeta(storage(), Transfer, FileMeta). + +-spec store_segment(ctx(), emqx_ft:transfer(), emqx_ft:segment()) -> + {ok, ctx()} | {error, term()}. +store_segment(Ctx, Transfer, Segment) -> + Mod = mod(), + Mod:store_segment(storage(), Ctx, Transfer, Segment). + +-spec assemble(ctx(), emqx_ft:transfer(), assemble_callback()) -> + {ok, pid()} | {error, term()}. +assemble(Ctx, Transfer, Callback) -> + Mod = mod(), + Mod:assemble(storage(), Ctx, Transfer, Callback). + +mod() -> + case storage() of + #{type := local} -> + % emqx_ft_storage_fs + emqx_ft_storage_dummy + end. + +storage() -> + emqx_config:get([file_transfer, storage]). diff --git a/apps/emqx_ft/src/emqx_ft_storage_dummy.erl b/apps/emqx_ft/src/emqx_ft_storage_dummy.erl new file mode 100644 index 000000000..1ab7f558e --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_dummy.erl @@ -0,0 +1,35 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_dummy). + +-behaviour(emqx_ft_storage). + +-export([ + store_filemeta/3, + store_segment/4, + assemble/4 +]). + +store_filemeta(_Storage, _Transfer, _Meta) -> + {ok, #{}}. + +store_segment(_Storage, Ctx, _Transfer, _Segment) -> + {ok, Ctx}. + +assemble(_Storage, _Ctx, _Transfer, Callback) -> + Pid = spawn(fun() -> Callback({error, not_implemented}) end), + {ok, Pid}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index cce7cc19e..bbd61eba9 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -19,10 +19,10 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -% -compile(export_all). +-behaviour(emqx_ft_storage). -export([store_filemeta/3]). --export([store_segment/3]). +-export([store_segment/4]). -export([list/2]). -export([read_segment/5]). -export([assemble/3]). @@ -32,45 +32,10 @@ -export([write/2]). -export([discard/1]). -% -behaviour(gen_server). -% -export([init/1]). -% -export([handle_call/3]). -% -export([handle_cast/2]). - --type json_value() :: - null - | boolean() - | binary() - | number() - | [json_value()] - | #{binary() => json_value()}. - --reflect_type([json_value/0]). - -type transfer() :: emqx_ft:transfer(). -type offset() :: emqx_ft:offset(). -%% TODO: move to `emqx_ft` interface module -% -type sha256_hex() :: <<_:512>>. - --type filemeta() :: #{ - %% Display name - name := string(), - %% Size in bytes, as advertised by the client. - %% Client is free to specify here whatever it wants, which means we can end - %% up with a file of different size after assembly. It's not clear from - %% specification what that means (e.g. what are clients' expectations), we - %% currently do not condider that an error (or, specifically, a signal that - %% the resulting file is corrupted during transmission). - size => _Bytes :: non_neg_integer(), - checksum => {sha256, <<_:256>>}, - expire_at := emqx_datetime:epoch_second(), - %% TTL of individual segments - %% Somewhat confusing that we won't know it on the nodes where the filemeta - %% is missing. - segments_ttl => _Seconds :: pos_integer(), - user_data => json_value() -}. +-type filemeta() :: emqx_ft:filemeta(). -type segment() :: {offset(), _Content :: binary()}. @@ -113,13 +78,14 @@ %% Atomic operation. -spec store_filemeta(storage(), transfer(), filemeta()) -> % Quota? Some lower level errors? - ok | {error, conflict} | {error, _TODO}. + {ok, emqx_ft_storage:ctx()} | {error, conflict} | {error, _TODO}. store_filemeta(Storage, Transfer, Meta) -> Filepath = mk_filepath(Storage, Transfer, ?MANIFEST), case read_file(Filepath, fun decode_filemeta/1) of {ok, Meta} -> _ = touch_file(Filepath), - ok; + %% No context is needed for this implementation. + {ok, #{}}; {ok, _Conflict} -> % TODO % We won't see conflicts in case of concurrent `store_filemeta` @@ -132,13 +98,18 @@ store_filemeta(Storage, Transfer, Meta) -> %% Store a segment in the backing filesystem. %% Atomic operation. --spec store_segment(storage(), transfer(), segment()) -> +-spec store_segment(storage(), emqx_ft_storage:ctx(), transfer(), segment()) -> % Where is the checksum gets verified? Upper level probably. % Quota? Some lower level errors? ok | {error, _TODO}. -store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> +store_segment(Storage, Ctx, Transfer, Segment = {_Offset, Content}) -> Filepath = mk_filepath(Storage, Transfer, mk_segment_filename(Segment)), - write_file_atomic(Filepath, Content). + case write_file_atomic(Filepath, Content) of + ok -> + {ok, Ctx}; + {error, _} = Error -> + Error + end. -spec list(storage(), transfer()) -> % Some lower level errors? {error, notfound}? @@ -178,13 +149,10 @@ read_segment(_Storage, _Transfer, Segment, Offset, Size) -> end. -spec assemble(storage(), transfer(), fun((ok | {error, term()}) -> any())) -> - % {ok, _Assembler :: pid()} | {error, incomplete} | {error, badrpc} | {error, _TODO}. {ok, _Assembler :: pid()} | {error, _TODO}. -assemble(Storage, Transfer, Callback) -> +assemble(Storage, _Ctx, Transfer, Callback) -> emqx_ft_assembler_sup:start_child(Storage, Transfer, Callback). -%% - -type handle() :: {file:name(), io:device(), crypto:hash_state()}. -spec open_file(storage(), transfer(), filemeta()) -> @@ -268,8 +236,7 @@ schema() -> {size, hoconsc:mk(non_neg_integer())}, {expire_at, hoconsc:mk(non_neg_integer())}, {checksum, hoconsc:mk({atom(), binary()}, #{converter => converter(checksum)})}, - {segments_ttl, hoconsc:mk(pos_integer())}, - {user_data, hoconsc:mk(json_value())} + {segments_ttl, hoconsc:mk(pos_integer())} ] }. @@ -354,10 +321,8 @@ mk_filedir(Storage, {ClientId, FileId}) -> mk_filepath(Storage, Transfer, Filename) -> filename:join(mk_filedir(Storage, Transfer), Filename). -get_storage_root(Storage) -> - Storage. - -%% +get_storage_root(_Storage) -> + filename:join(emqx:data_dir(), "file_transfer"). -include_lib("kernel/include/file.hrl"). From b4a42a447c603f5361c9969cd0aa44c98ff98e4c Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 2 Feb 2023 23:52:42 +0200 Subject: [PATCH 011/197] feat(ft): removed replicated data --- apps/emqx/src/emqx_cm.erl | 23 ++--- apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 10 +- apps/emqx_ft/include/emqx_ft.hrl | 2 - apps/emqx_ft/src/emqx_ft.erl | 92 +------------------ apps/emqx_ft/src/emqx_ft_storage.erl | 35 ++++--- apps/emqx_ft/src/emqx_ft_storage_dummy.erl | 12 +-- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 23 ++--- apps/emqx_ft/src/emqx_ft_sup.erl | 12 --- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 9 +- 9 files changed, 56 insertions(+), 162 deletions(-) diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index 724139142..766acd4b8 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -172,7 +172,6 @@ register_channel(ClientId, ChanPid, #{conn_mod := ConnMod}) when is_pid(ChanPid) true = ets:insert(?CHAN_CONN_TAB, {Chan, ConnMod}), ok = emqx_cm_registry:register_channel(Chan), mark_channel_connected(ChanPid), - ok = emqx_hooks:run('channel.registered', [ConnMod, ChanPid]), cast({registered, Chan}). %% @doc Unregister a channel. @@ -282,7 +281,7 @@ open_session(true, ClientInfo = #{clientid := ClientId}, ConnInfo) -> {ok, #{session => Session1, present => false}} end, emqx_cm_locker:trans(ClientId, CleanStart); -open_session(false, ClientInfo = #{clientid := ClientId}, #{conn_mod := NewConnMod} = ConnInfo) -> +open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> Self = self(), ResumeStart = fun(_) -> CreateSess = @@ -309,11 +308,10 @@ open_session(false, ClientInfo = #{clientid := ClientId}, #{conn_mod := NewConnM {living, ConnMod, ChanPid, Session} -> ok = emqx_session:resume(ClientInfo, Session), case wrap_rpc(emqx_cm_proto_v2:takeover_finish(ConnMod, ChanPid)) of - {ok, Pendings, TakoverData} -> + {ok, Pendings} -> Session1 = emqx_persistent_session:persist( ClientInfo, ConnInfo, Session ), - ok = emqx_hooks:run('channel.takenover', [NewConnMod, Self, TakoverData]), {ok, #{ session => clean_session(Session1), present => true, @@ -399,18 +397,11 @@ takeover_session(ClientId) -> end. takeover_finish(ConnMod, ChanPid) -> - TakoverData = emqx_hooks:run_fold('channel.takeover', [ConnMod, ChanPid], #{}), - case - %% node-local call - request_stepdown( - {takeover, 'end'}, - ConnMod, - ChanPid - ) - of - {ok, Pendings} -> {ok, Pendings, TakoverData}; - {error, _} = Error -> Error - end. + request_stepdown( + {takeover, 'end'}, + ConnMod, + ChanPid + ). takeover_session(ClientId, Pid) -> try diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf index 85fb81cfe..481ad8154 100644 --- a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -2,13 +2,13 @@ emqx_ft_schema { local { desc { - en: "Use local file system to store uploaded files and temporary data." - zh: "使用本地文件系统来存储上传的文件和临时数据。" + en: "Use local file system to store uploaded files and temporary data." + zh: "使用本地文件系统来存储上传的文件和临时数据。" } label: { - en: "Local Storage" - zh: "本地存储" - } + en: "Local Storage" + zh: "本地存储" + } } } diff --git a/apps/emqx_ft/include/emqx_ft.hrl b/apps/emqx_ft/include/emqx_ft.hrl index 2cbd24fb4..e46d79490 100644 --- a/apps/emqx_ft/include/emqx_ft.hrl +++ b/apps/emqx_ft/include/emqx_ft.hrl @@ -13,5 +13,3 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- - --define(FT_TAB, emqx_ft). diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index a286a6186..be2f80831 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -16,29 +16,21 @@ -module(emqx_ft). --include("emqx_ft.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). -export([ - create_tab/0, hook/0, unhook/0 ]). -export([ - on_channel_unregistered/1, - on_channel_takeover/3, - on_channel_takenover/3, on_message_publish/1, on_message_puback/4 ]). -%% For Debug --export([transfer/2, storage/0]). - -export([on_assemble_timeout/1]). -export_type([ @@ -79,43 +71,17 @@ -type segment() :: {offset(), _Content :: binary()}. --type ft_data() :: #{ - nodes := list(node()) -}. - --record(emqx_ft, { - chan_pid :: pid(), - ft_data :: ft_data() -}). - -define(ASSEMBLE_TIMEOUT, 5000). %%-------------------------------------------------------------------- %% API for app %%-------------------------------------------------------------------- -create_tab() -> - _Tab = ets:new(?FT_TAB, [ - set, - public, - named_table, - {keypos, #emqx_ft.chan_pid} - ]), - ok. - hook() -> - ok = emqx_hooks:put('channel.unregistered', {?MODULE, on_channel_unregistered, []}, ?HP_LOWEST), - ok = emqx_hooks:put('channel.takeover', {?MODULE, on_channel_takeover, []}, ?HP_LOWEST), - ok = emqx_hooks:put('channel.takenover', {?MODULE, on_channel_takenover, []}, ?HP_LOWEST), - ok = emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_LOWEST), ok = emqx_hooks:put('message.puback', {?MODULE, on_message_puback, []}, ?HP_LOWEST). unhook() -> - ok = emqx_hooks:del('channel.unregistered', {?MODULE, on_channel_unregistered}), - ok = emqx_hooks:del('channel.takeover', {?MODULE, on_channel_takeover}), - ok = emqx_hooks:del('channel.takenover', {?MODULE, on_channel_takenover}), - ok = emqx_hooks:del('message.publish', {?MODULE, on_message_publish}), ok = emqx_hooks:del('message.puback', {?MODULE, on_message_puback}). @@ -123,22 +89,6 @@ unhook() -> %% Hooks %%-------------------------------------------------------------------- -on_channel_unregistered(ChanPid) -> - ok = delete_ft_data(ChanPid). - -on_channel_takeover(_ConnMod, ChanPid, TakeoverData) -> - case get_ft_data(ChanPid) of - {ok, FTData} -> - {ok, TakeoverData#{ft_data => FTData}}; - none -> - ok - end. - -on_channel_takenover(_ConnMod, ChanPid, #{ft_data := FTData}) -> - ok = put_ft_data(ChanPid, FTData); -on_channel_takenover(_ConnMod, _ChanPid, _) -> - ok. - on_message_publish( Msg = #message{ id = _Id, @@ -190,8 +140,7 @@ on_init(Msg, FileId) -> % %% Add validations here Meta = emqx_json:decode(Payload, [return_maps]), case emqx_ft_storage:store_filemeta(transfer(Msg, FileId), Meta) of - {ok, Ctx} -> - ok = put_context(Ctx), + ok -> ?RC_SUCCESS; {error, _Reason} -> ?RC_UNSPECIFIED_ERROR @@ -213,9 +162,8 @@ on_segment(Msg, FileId, Offset, Checksum) -> Payload = Msg#message.payload, Segment = {binary_to_integer(Offset), Payload}, %% Add offset/checksum validations - case emqx_ft_storage:store_segment(get_context(), transfer(Msg, FileId), Segment) of - {ok, Ctx} -> - ok = put_context(Ctx), + case emqx_ft_storage:store_segment(transfer(Msg, FileId), Segment) of + ok -> ?RC_SUCCESS; {error, _Reason} -> ?RC_UNSPECIFIED_ERROR @@ -267,7 +215,6 @@ on_fin(PacketId, Msg, FileId, Checksum) -> assemble(Transfer, Callback) -> emqx_ft_storage:assemble( - get_context(), Transfer, Callback ). @@ -277,12 +224,12 @@ callback({ChanPid, PacketId} = Key, _FileId) -> case emqx_ft_responder:unregister(Key) of ok -> case Result of - {ok, _} -> + ok -> erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS}); {error, _} -> erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}) end; - {error, not_registered} -> + {error, not_found} -> ok end end. @@ -291,35 +238,6 @@ transfer(Msg, FileId) -> ClientId = Msg#message.from, {ClientId, FileId}. -%% TODO: configure - -storage() -> - emqx_config:get([file_transfer, storage]). - on_assemble_timeout({ChanPid, PacketId}) -> ?SLOG(warning, #{msg => "on_assemble_timeout", packet_id => PacketId}), erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}). - -%%-------------------------------------------------------------------- -%% Context management -%%-------------------------------------------------------------------- - -get_context() -> - get_ft_data(self()). - -put_context(Context) -> - put_ft_data(self(), Context). - -get_ft_data(ChanPid) -> - case ets:lookup(?FT_TAB, ChanPid) of - [#emqx_ft{ft_data = FTData}] -> {ok, FTData}; - [] -> none - end. - -delete_ft_data(ChanPid) -> - true = ets:delete(?FT_TAB, ChanPid), - ok. - -put_ft_data(ChanPid, FTData) -> - true = ets:insert(?FT_TAB, #emqx_ft{chan_pid = ChanPid, ft_data = FTData}), - ok. diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 5e945965d..8729a2ad4 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -19,12 +19,11 @@ -export( [ store_filemeta/2, - store_segment/3, - assemble/3 + store_segment/2, + assemble/2 ] ). --type ctx() :: term(). -type storage() :: emqx_config:config(). -export_type([assemble_callback/0]). @@ -32,14 +31,14 @@ -type assemble_callback() :: fun((ok | {error, term()}) -> any()). %%-------------------------------------------------------------------- -%% behaviour +%% Behaviour %%-------------------------------------------------------------------- -callback store_filemeta(storage(), emqx_ft:transfer(), emqx_ft:filemeta()) -> - {ok, ctx()} | {error, term()}. --callback store_segment(storage(), ctx(), emqx_ft:transfer(), emqx_ft:segment()) -> - {ok, ctx()} | {error, term()}. --callback assemble(storage(), ctx(), emqx_ft:transfer(), assemble_callback()) -> + ok | {error, term()}. +-callback store_segment(storage(), emqx_ft:transfer(), emqx_ft:segment()) -> + ok | {error, term()}. +-callback assemble(storage(), emqx_ft:transfer(), assemble_callback()) -> {ok, pid()} | {error, term()}. %%-------------------------------------------------------------------- @@ -47,28 +46,28 @@ %%-------------------------------------------------------------------- -spec store_filemeta(emqx_ft:transfer(), emqx_ft:filemeta()) -> - {ok, ctx()} | {error, term()}. + ok | {error, term()}. store_filemeta(Transfer, FileMeta) -> Mod = mod(), Mod:store_filemeta(storage(), Transfer, FileMeta). --spec store_segment(ctx(), emqx_ft:transfer(), emqx_ft:segment()) -> - {ok, ctx()} | {error, term()}. -store_segment(Ctx, Transfer, Segment) -> +-spec store_segment(emqx_ft:transfer(), emqx_ft:segment()) -> + ok | {error, term()}. +store_segment(Transfer, Segment) -> Mod = mod(), - Mod:store_segment(storage(), Ctx, Transfer, Segment). + Mod:store_segment(storage(), Transfer, Segment). --spec assemble(ctx(), emqx_ft:transfer(), assemble_callback()) -> +-spec assemble(emqx_ft:transfer(), assemble_callback()) -> {ok, pid()} | {error, term()}. -assemble(Ctx, Transfer, Callback) -> +assemble(Transfer, Callback) -> Mod = mod(), - Mod:assemble(storage(), Ctx, Transfer, Callback). + Mod:assemble(storage(), Transfer, Callback). mod() -> case storage() of #{type := local} -> - % emqx_ft_storage_fs - emqx_ft_storage_dummy + emqx_ft_storage_fs + % emqx_ft_storage_dummy end. storage() -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_dummy.erl b/apps/emqx_ft/src/emqx_ft_storage_dummy.erl index 1ab7f558e..d486c0c29 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_dummy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_dummy.erl @@ -20,16 +20,16 @@ -export([ store_filemeta/3, - store_segment/4, - assemble/4 + store_segment/3, + assemble/3 ]). store_filemeta(_Storage, _Transfer, _Meta) -> - {ok, #{}}. + ok. -store_segment(_Storage, Ctx, _Transfer, _Segment) -> - {ok, Ctx}. +store_segment(_Storage, _Transfer, _Segment) -> + ok. -assemble(_Storage, _Ctx, _Transfer, Callback) -> +assemble(_Storage, _Transfer, Callback) -> Pid = spawn(fun() -> Callback({error, not_implemented}) end), {ok, Pid}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index bbd61eba9..a530c9cf8 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -22,7 +22,7 @@ -behaviour(emqx_ft_storage). -export([store_filemeta/3]). --export([store_segment/4]). +-export([store_segment/3]). -export([list/2]). -export([read_segment/5]). -export([assemble/3]). @@ -84,8 +84,7 @@ store_filemeta(Storage, Transfer, Meta) -> case read_file(Filepath, fun decode_filemeta/1) of {ok, Meta} -> _ = touch_file(Filepath), - %% No context is needed for this implementation. - {ok, #{}}; + ok; {ok, _Conflict} -> % TODO % We won't see conflicts in case of concurrent `store_filemeta` @@ -98,18 +97,13 @@ store_filemeta(Storage, Transfer, Meta) -> %% Store a segment in the backing filesystem. %% Atomic operation. --spec store_segment(storage(), emqx_ft_storage:ctx(), transfer(), segment()) -> +-spec store_segment(storage(), transfer(), segment()) -> % Where is the checksum gets verified? Upper level probably. % Quota? Some lower level errors? ok | {error, _TODO}. -store_segment(Storage, Ctx, Transfer, Segment = {_Offset, Content}) -> +store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> Filepath = mk_filepath(Storage, Transfer, mk_segment_filename(Segment)), - case write_file_atomic(Filepath, Content) of - ok -> - {ok, Ctx}; - {error, _} = Error -> - Error - end. + write_file_atomic(Filepath, Content). -spec list(storage(), transfer()) -> % Some lower level errors? {error, notfound}? @@ -149,8 +143,9 @@ read_segment(_Storage, _Transfer, Segment, Offset, Size) -> end. -spec assemble(storage(), transfer(), fun((ok | {error, term()}) -> any())) -> + % {ok, _Assembler :: pid()} | {error, incomplete} | {error, badrpc} | {error, _TODO}. {ok, _Assembler :: pid()} | {error, _TODO}. -assemble(Storage, _Ctx, Transfer, Callback) -> +assemble(Storage, Transfer, Callback) -> emqx_ft_assembler_sup:start_child(Storage, Transfer, Callback). -type handle() :: {file:name(), io:device(), crypto:hash_state()}. @@ -321,8 +316,8 @@ mk_filedir(Storage, {ClientId, FileId}) -> mk_filepath(Storage, Transfer, Filename) -> filename:join(mk_filedir(Storage, Transfer), Filename). -get_storage_root(_Storage) -> - filename:join(emqx:data_dir(), "file_transfer"). +get_storage_root(Storage) -> + maps:get(root, Storage, filename:join(emqx:data_dir(), "file_transfer")). -include_lib("kernel/include/file.hrl"). diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index ef2d8033f..b4ce52edb 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -27,17 +27,7 @@ start_link() -> supervisor:start_link({local, ?SERVER}, ?MODULE, []). -%% sup_flags() = #{strategy => strategy(), % optional -%% intensity => non_neg_integer(), % optional -%% period => pos_integer()} % optional -%% child_spec() = #{id => child_id(), % mandatory -%% start => mfargs(), % mandatory -%% restart => restart(), % optional -%% shutdown => shutdown(), % optional -%% type => worker(), % optional -%% modules => modules()} % optional init([]) -> - ok = emqx_ft:create_tab(), SupFlags = #{ strategy => one_for_all, intensity => 100, @@ -64,5 +54,3 @@ init([]) -> ChildSpecs = [Responder, AssemblerSup], {ok, {SupFlags, ChildSpecs}}. - -%% internal functions diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 1112ccbb7..9ec2fce61 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -54,7 +54,7 @@ end_per_testcase(_TC, Config) -> -define(CLIENTID, <<"thatsme">>). t_assemble_empty_transfer(Config) -> - Storage = ?config(storage_root, Config), + Storage = storage(Config), Transfer = {?CLIENTID, mk_fileid()}, Filename = "important.pdf", Meta = #{ @@ -84,7 +84,7 @@ t_assemble_empty_transfer(Config) -> ok. t_assemble_complete_local_transfer(Config) -> - Storage = ?config(storage_root, Config), + Storage = storage(Config), Transfer = {?CLIENTID, mk_fileid()}, Filename = "topsecret.pdf", TransferSize = 10000 + rand:uniform(50000), @@ -161,3 +161,8 @@ inspect_file(Filename) -> mk_fileid() -> integer_to_binary(erlang:system_time(millisecond)). + +storage(Config) -> + #{ + root => ?config(storage_root, Config) + }. From 8298236908ef8ed7ae237acaf4ec6a6ec6681d61 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 3 Feb 2023 12:38:02 +0300 Subject: [PATCH 012/197] refactor(ft): bring back userdata to filemeta schema --- apps/emqx_ft/src/emqx_ft.erl | 3 +- apps/emqx_ft/src/emqx_ft_schema.erl | 40 +++++++++++++++++++++++++ apps/emqx_ft/src/emqx_ft_storage_fs.erl | 34 +++------------------ 3 files changed, 46 insertions(+), 31 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index be2f80831..ce3a7a97d 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -66,7 +66,8 @@ %% TTL of individual segments %% Somewhat confusing that we won't know it on the nodes where the filemeta %% is missing. - segments_ttl => _Seconds :: pos_integer() + segments_ttl => _Seconds :: pos_integer(), + user_data => emqx_ft_schema:json_value() }. -type segment() :: {offset(), _Content :: binary()}. diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index f40d2f40e..70acb8322 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -23,6 +23,20 @@ -export([namespace/0, roots/0, fields/1, tags/0]). +-export([schema/1]). + +-type json_value() :: + null + | boolean() + | binary() + | number() + | [json_value()] + | #{binary() => json_value()}. + +-reflect_type([json_value/0]). + +%% + namespace() -> file_transfer. tags() -> @@ -47,3 +61,29 @@ fields(local_storage) -> desc => ?DESC("local") }} ]. + +schema(filemeta) -> + #{ + roots => [ + {name, hoconsc:mk(string(), #{required => true})}, + {size, hoconsc:mk(non_neg_integer())}, + {expire_at, hoconsc:mk(non_neg_integer())}, + {checksum, hoconsc:mk({atom(), binary()}, #{converter => converter(checksum)})}, + {segments_ttl, hoconsc:mk(pos_integer())}, + {user_data, hoconsc:mk(json_value())} + ] + }. + +converter(checksum) -> + fun + (undefined, #{}) -> + undefined; + ({sha256, Bin}, #{make_serializable := true}) -> + _ = is_binary(Bin) orelse throw({expected_type, string}), + _ = byte_size(Bin) =:= 32 orelse throw({expected_length, 32}), + binary:encode_hex(Bin); + (Hex, #{}) -> + _ = is_binary(Hex) orelse throw({expected_type, string}), + _ = byte_size(Hex) =:= 64 orelse throw({expected_length, 64}), + {sha256, binary:decode_hex(Hex)} + end. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index a530c9cf8..e2bda8fcb 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -16,9 +16,6 @@ -module(emqx_ft_storage_fs). --include_lib("typerefl/include/types.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -behaviour(emqx_ft_storage). -export([store_filemeta/3]). @@ -224,17 +221,6 @@ verify_checksum(undefined, _) -> -define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). -schema() -> - #{ - roots => [ - {name, hoconsc:mk(string(), #{required => true})}, - {size, hoconsc:mk(non_neg_integer())}, - {expire_at, hoconsc:mk(non_neg_integer())}, - {checksum, hoconsc:mk({atom(), binary()}, #{converter => converter(checksum)})}, - {segments_ttl, hoconsc:mk(pos_integer())} - ] - }. - % encode_filemeta(Meta) -> % emqx_json:encode( % ?PRELUDE( @@ -261,26 +247,14 @@ schema() -> encode_filemeta(Meta) -> % TODO: Looks like this should be hocon's responsibility. - Term = hocon_tconf:make_serializable(schema(), emqx_map_lib:binary_key_map(Meta), #{}), + Schema = emqx_ft_schema:schema(filemeta), + Term = hocon_tconf:make_serializable(Schema, emqx_map_lib:binary_key_map(Meta), #{}), emqx_json:encode(?PRELUDE(_Vsn = 1, Term)). decode_filemeta(Binary) -> + Schema = emqx_ft_schema:schema(filemeta), ?PRELUDE(_Vsn = 1, Term) = emqx_json:decode(Binary, [return_maps]), - hocon_tconf:check_plain(schema(), Term, #{atom_key => true, required => false}). - -converter(checksum) -> - fun - (undefined, #{}) -> - undefined; - ({sha256, Bin}, #{make_serializable := true}) -> - _ = is_binary(Bin) orelse throw({expected_type, string}), - _ = byte_size(Bin) =:= 32 orelse throw({expected_length, 32}), - binary:encode_hex(Bin); - (Hex, #{}) -> - _ = is_binary(Hex) orelse throw({expected_type, string}), - _ = byte_size(Hex) =:= 64 orelse throw({expected_length, 64}), - {sha256, binary:decode_hex(Hex)} - end. + hocon_tconf:check_plain(Schema, Term, #{atom_key => true, required => false}). % map_into(Fun, Into, Ks, Map) -> % map_foldr(map_into_fn(Fun, Into), Into, Ks, Map). From 1308fa0e6b4b19d70fd07f4b6bbafdc4f57d0d1a Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 3 Feb 2023 12:40:22 +0300 Subject: [PATCH 013/197] fix(ft-fs): put fragments into separate directories In order to avoid potential filename collisions. --- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 49 ++++++++++--------- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 2 +- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index e2bda8fcb..cc9d94b35 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -49,9 +49,11 @@ -type filefrag() :: filefrag({filemeta, filemeta()} | {segment, segmentinfo()}). +-define(FRAGDIR, frags). +-define(TEMPDIR, tmp). +-define(RESULTDIR, result). -define(MANIFEST, "MANIFEST.json"). -define(SEGMENT, "SEG"). --define(TEMP, "TMP"). -type root() :: file:name(). @@ -77,7 +79,7 @@ % Quota? Some lower level errors? {ok, emqx_ft_storage:ctx()} | {error, conflict} | {error, _TODO}. store_filemeta(Storage, Transfer, Meta) -> - Filepath = mk_filepath(Storage, Transfer, ?MANIFEST), + Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], ?MANIFEST), case read_file(Filepath, fun decode_filemeta/1) of {ok, Meta} -> _ = touch_file(Filepath), @@ -89,7 +91,7 @@ store_filemeta(Storage, Transfer, Meta) -> % about it too much now. {error, conflict}; {error, Reason} when Reason =:= notfound; Reason =:= corrupted; Reason =:= enoent -> - write_file_atomic(Filepath, encode_filemeta(Meta)) + write_file_atomic(Storage, Transfer, Filepath, encode_filemeta(Meta)) end. %% Store a segment in the backing filesystem. @@ -99,15 +101,15 @@ store_filemeta(Storage, Transfer, Meta) -> % Quota? Some lower level errors? ok | {error, _TODO}. store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> - Filepath = mk_filepath(Storage, Transfer, mk_segment_filename(Segment)), - write_file_atomic(Filepath, Content). + Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], mk_segment_filename(Segment)), + write_file_atomic(Storage, Transfer, Filepath, Content). -spec list(storage(), transfer()) -> % Some lower level errors? {error, notfound}? % Result will contain zero or only one filemeta. {ok, list(filefrag())} | {error, _TODO}. list(Storage, Transfer) -> - Dirname = mk_filedir(Storage, Transfer), + Dirname = mk_filedir(Storage, Transfer, [?FRAGDIR]), case file:list_dir(Dirname) of {ok, Filenames} -> {ok, filtermap_files(fun mk_filefrag/2, Dirname, Filenames)}; @@ -151,8 +153,8 @@ assemble(Storage, Transfer, Callback) -> {ok, handle()} | {error, _TODO}. open_file(Storage, Transfer, Filemeta) -> Filename = maps:get(name, Filemeta), - Filepath = mk_filepath(Storage, Transfer, Filename), - TempFilepath = mk_temp_filepath(Filepath), + TempFilepath = mk_temp_filepath(Storage, Transfer, Filename), + _ = filelib:ensure_dir(TempFilepath), case file:open(TempFilepath, [write, raw]) of {ok, Handle} -> _ = file:truncate(Handle), @@ -174,11 +176,11 @@ write({Filepath, IoDevice, Ctx}, IoData) -> -spec complete(storage(), transfer(), filemeta(), handle()) -> ok | {error, {checksum, _Algo, _Computed}} | {error, _TODO}. complete(Storage, Transfer, Filemeta, Handle = {Filepath, IoDevice, Ctx}) -> - TargetFilepath = mk_filepath(Storage, Transfer, maps:get(name, Filemeta)), + TargetFilepath = mk_filepath(Storage, Transfer, [?RESULTDIR], maps:get(name, Filemeta)), case verify_checksum(Ctx, Filemeta) of ok -> ok = file:close(IoDevice), - file:rename(Filepath, TargetFilepath); + mv_temp_file(Filepath, TargetFilepath); {error, _} = Error -> _ = discard(Handle), Error @@ -284,11 +286,11 @@ break_segment_filename(Filename) -> {error, invalid} end. -mk_filedir(Storage, {ClientId, FileId}) -> - filename:join([get_storage_root(Storage), ClientId, FileId]). +mk_filedir(Storage, {ClientId, FileId}, SubDirs) -> + filename:join([get_storage_root(Storage), ClientId, FileId | SubDirs]). -mk_filepath(Storage, Transfer, Filename) -> - filename:join(mk_filedir(Storage, Transfer), Filename). +mk_filepath(Storage, Transfer, SubDirs, Filename) -> + filename:join(mk_filedir(Storage, Transfer, SubDirs), Filename). get_storage_root(Storage) -> maps:get(root, Storage, filename:join(emqx:data_dir(), "file_transfer")). @@ -315,13 +317,13 @@ safe_decode(Content, DecodeFun) -> {error, corrupted} end. -write_file_atomic(Filepath, Content) when is_binary(Content) -> - TempFilepath = mk_temp_filepath(Filepath), +write_file_atomic(Storage, Transfer, Filepath, Content) when is_binary(Content) -> + TempFilepath = mk_temp_filepath(Storage, Transfer, filename:basename(Filepath)), Result = emqx_misc:pipeline( [ fun filelib:ensure_dir/1, fun write_contents/2, - fun(FP) -> mv_temp_file(Filepath, FP) end + fun(_) -> mv_temp_file(TempFilepath, Filepath) end ], TempFilepath, Content @@ -334,11 +336,9 @@ write_file_atomic(Filepath, Content) when is_binary(Content) -> {error, Reason} end. -mk_temp_filepath(Filepath) -> - Dirname = filename:dirname(Filepath), - Filename = filename:basename(Filepath), +mk_temp_filepath(Storage, Transfer, Filename) -> Unique = erlang:unique_integer([positive]), - filename:join(Dirname, mk_filename([?TEMP, Unique, ".", Filename])). + filename:join(mk_filedir(Storage, Transfer, [?TEMPDIR]), mk_filename([Unique, ".", Filename])). mk_filename(Comps) -> lists:append(lists:map(fun mk_filename_component/1, Comps)). @@ -351,7 +351,8 @@ mk_filename_component(S) when is_list(S) -> S. write_contents(Filepath, Content) -> file:write_file(Filepath, Content). -mv_temp_file(Filepath, TempFilepath) -> +mv_temp_file(TempFilepath, Filepath) -> + _ = filelib:ensure_dir(Filepath), file:rename(TempFilepath, Filepath). touch_file(Filepath) -> @@ -365,7 +366,8 @@ mk_filefrag(Dirname, Filename = ?MANIFEST) -> mk_filefrag(Dirname, Filename, filemeta, fun read_filemeta/2); mk_filefrag(Dirname, Filename = ?SEGMENT ++ _) -> mk_filefrag(Dirname, Filename, segment, fun read_segmentinfo/2); -mk_filefrag(_Dirname, _) -> +mk_filefrag(_Dirname, _Filename) -> + % TODO this is unexpected, worth logging? false. mk_filefrag(Dirname, Filename, Tag, Fun) -> @@ -380,6 +382,7 @@ mk_filefrag(Dirname, Filename, Tag, Fun) -> fragment => {Tag, Frag} }}; {error, _Reason} -> + % TODO loss of information false end. diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 9ec2fce61..fdfdd432e 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -133,7 +133,7 @@ t_assemble_complete_local_transfer(Config) -> ). mk_assembly_filename(Config, {ClientID, FileID}, Filename) -> - filename:join([?config(storage_root, Config), ClientID, FileID, Filename]). + filename:join([?config(storage_root, Config), ClientID, FileID, result, Filename]). on_assembly_finished(Result) -> ?tp(test_assembly_finished, #{result => Result}). From 429eeaf029d69c896f46c8a6ee57bf90634b780b Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 3 Feb 2023 14:23:58 +0300 Subject: [PATCH 014/197] feat(ft-fs): make `list` / `read` more generic And usable in wider contexts as a consequence, for example querying and fetching resulting files from remote nodes. --- apps/emqx_ft/src/emqx_ft_assembler.erl | 6 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 60 ++++++++++++++----- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 19 +++++- 3 files changed, 65 insertions(+), 20 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 4fd8d6e75..af50dee3b 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -66,7 +66,7 @@ init({Storage, Transfer, Callback}) -> handle_event(internal, _, list_local_fragments, St = #st{assembly = Asm}) -> % TODO: what we do with non-transients errors here (e.g. `eacces`)? - {ok, Fragments} = emqx_ft_storage_fs:list(St#st.storage, St#st.transfer), + {ok, Fragments} = emqx_ft_storage_fs:list(St#st.storage, St#st.transfer, fragment), NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(Asm, node(), Fragments)), NSt = St#st{assembly = NAsm}, case emqx_ft_assembly:status(NAsm) of @@ -81,7 +81,7 @@ handle_event(internal, _, list_local_fragments, St = #st{assembly = Asm}) -> end; handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> % TODO: portable "storage" ref - Args = [St#st.storage, St#st.transfer], + Args = [St#st.storage, St#st.transfer, fragment], % TODO % Async would better because we would not need to wait for some lagging nodes if % the coverage is already complete. @@ -121,7 +121,7 @@ handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % this node garbage collecting the segment itself. Args = [St#st.storage, St#st.transfer, Segment, 0, segsize(Segment)], % TODO: pipelining - case erpc:call(Node, emqx_ft_storage_fs, read_segment, Args, ?RPC_READSEG_TIMEOUT) of + case erpc:call(Node, emqx_ft_storage_fs, pread, Args, ?RPC_READSEG_TIMEOUT) of {ok, Content} -> {ok, NHandle} = emqx_ft_storage_fs:write(St#st.file, Content), {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])} diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index cc9d94b35..fdf4558b8 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -20,8 +20,8 @@ -export([store_filemeta/3]). -export([store_segment/3]). --export([list/2]). --export([read_segment/5]). +-export([list/3]). +-export([pread/5]). -export([assemble/3]). -export([open_file/3]). @@ -41,13 +41,19 @@ size := _Bytes :: non_neg_integer() }. +% TODO naming -type filefrag(T) :: #{ path := file:name(), timestamp := emqx_datetime:epoch_second(), + size := _Bytes :: non_neg_integer(), fragment := T }. --type filefrag() :: filefrag({filemeta, filemeta()} | {segment, segmentinfo()}). +-type filefrag() :: filefrag( + {filemeta, filemeta()} + | {segment, segmentinfo()} + | {result, #{}} +). -define(FRAGDIR, frags). -define(TEMPDIR, tmp). @@ -104,29 +110,44 @@ store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], mk_segment_filename(Segment)), write_file_atomic(Storage, Transfer, Filepath, Content). --spec list(storage(), transfer()) -> +-spec list(storage(), transfer(), _What :: fragment | result) -> % Some lower level errors? {error, notfound}? % Result will contain zero or only one filemeta. - {ok, list(filefrag())} | {error, _TODO}. -list(Storage, Transfer) -> - Dirname = mk_filedir(Storage, Transfer, [?FRAGDIR]), + {ok, [filefrag({filemeta, filemeta()} | {segment, segmentinfo()})]} | {error, _TODO}. +list(Storage, Transfer, What) -> + Dirname = mk_filedir(Storage, Transfer, get_subdirs_for(What)), case file:list_dir(Dirname) of {ok, Filenames} -> - {ok, filtermap_files(fun mk_filefrag/2, Dirname, Filenames)}; + % TODO + % In case of `What = result` there might be more than one file (though + % extremely bad luck is needed for that, e.g. concurrent assemblers with + % different filemetas from different nodes). This might be unexpected for a + % client given the current protocol, yet might be helpful in the future. + {ok, filtermap_files(get_filefrag_fun_for(What), Dirname, Filenames)}; {error, enoent} -> {ok, []}; {error, _} = Error -> Error end. --spec read_segment( - storage(), transfer(), filefrag(segmentinfo()), offset(), _Size :: non_neg_integer() -) -> +get_subdirs_for(fragment) -> + [?FRAGDIR]; +get_subdirs_for(result) -> + [?RESULTDIR]. + +get_filefrag_fun_for(fragment) -> + fun mk_filefrag/2; +get_filefrag_fun_for(result) -> + fun mk_result_filefrag/2. + +-spec pread(storage(), transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> {ok, _Content :: iodata()} | {error, _TODO}. -read_segment(_Storage, _Transfer, Segment, Offset, Size) -> - Filepath = maps:get(path, Segment), - case file:open(Filepath, [raw, read]) of +pread(_Storage, _Transfer, Frag, Offset, Size) -> + Filepath = maps:get(path, Frag), + case file:open(Filepath, [read, raw, binary]) of {ok, IoDevice} -> + % NOTE + % Reading empty file is always `eof`. Read = file:pread(IoDevice, Offset, Size), ok = file:close(IoDevice), case Read of @@ -147,6 +168,8 @@ read_segment(_Storage, _Transfer, Segment, Offset, Size) -> assemble(Storage, Transfer, Callback) -> emqx_ft_assembler_sup:start_child(Storage, Transfer, Callback). +%% + -type handle() :: {file:name(), io:device(), crypto:hash_state()}. -spec open_file(storage(), transfer(), filemeta()) -> @@ -155,7 +178,7 @@ open_file(Storage, Transfer, Filemeta) -> Filename = maps:get(name, Filemeta), TempFilepath = mk_temp_filepath(Storage, Transfer, Filename), _ = filelib:ensure_dir(TempFilepath), - case file:open(TempFilepath, [write, raw]) of + case file:open(TempFilepath, [write, raw, binary]) of {ok, Handle} -> _ = file:truncate(Handle), {ok, {TempFilepath, Handle, init_checksum(Filemeta)}}; @@ -370,6 +393,12 @@ mk_filefrag(_Dirname, _Filename) -> % TODO this is unexpected, worth logging? false. +mk_result_filefrag(Dirname, Filename) -> + % NOTE + % Any file in the `?RESULTDIR` subdir is currently considered the result of + % the file transfer. + mk_filefrag(Dirname, Filename, result, fun(_, _) -> {ok, #{}} end). + mk_filefrag(Dirname, Filename, Tag, Fun) -> Filepath = filename:join(Dirname, Filename), % TODO error handling? @@ -379,6 +408,7 @@ mk_filefrag(Dirname, Filename, Tag, Fun) -> {true, #{ path => Filepath, timestamp => Fileinfo#file_info.mtime, + size => Fileinfo#file_info.size, fragment => {Tag, Frag} }}; {error, _Reason} -> diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index fdfdd432e..77a619fe3 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -71,7 +71,7 @@ t_assemble_empty_transfer(Config) -> fragment := {filemeta, Meta} } ]}, - emqx_ft_storage_fs:list(Storage, Transfer) + emqx_ft_storage_fs:list(Storage, Transfer, fragment) ), {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun on_assembly_finished/1), {ok, Event} = ?block_until(#{?snk_kind := test_assembly_finished}), @@ -81,6 +81,11 @@ t_assemble_empty_transfer(Config) -> % TODO file:read_file(mk_assembly_filename(Config, Transfer, Filename)) ), + {ok, [Result = #{size := Size = 0}]} = emqx_ft_storage_fs:list(Storage, Transfer, result), + ?assertEqual( + {error, eof}, + emqx_ft_storage_fs:pread(Storage, Transfer, Result, 0, Size) + ), ok. t_assemble_complete_local_transfer(Config) -> @@ -109,7 +114,7 @@ t_assemble_complete_local_transfer(Config) -> end ), - {ok, Fragments} = emqx_ft_storage_fs:list(Storage, Transfer), + {ok, Fragments} = emqx_ft_storage_fs:list(Storage, Transfer, fragment), ?assertEqual((TransferSize div SegmentSize) + 1 + 1, length(Fragments)), ?assertEqual( [Meta], @@ -122,6 +127,16 @@ t_assemble_complete_local_transfer(Config) -> ?assertMatch(#{result := ok}, Event), AssemblyFilename = mk_assembly_filename(Config, Transfer, Filename), + ?assertMatch( + {ok, [ + #{ + path := AssemblyFilename, + size := TransferSize, + fragment := {result, #{}} + } + ]}, + emqx_ft_storage_fs:list(Storage, Transfer, result) + ), ?assertMatch( {ok, #file_info{type = regular, size = TransferSize}}, file:read_file_info(AssemblyFilename) From 7ed06b0a2a3abbf76cf9dafd115ec203273833a8 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 3 Feb 2023 17:04:27 +0300 Subject: [PATCH 015/197] feat(ft-fs): allow to list all transfers in storage This is rather simplistic and thus, temporary solution. --- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 70 ++++++++++++++++++- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 48 ++++++++++--- 2 files changed, 104 insertions(+), 14 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index fdf4558b8..3a78559c1 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -24,6 +24,8 @@ -export([pread/5]). -export([assemble/3]). +-export([transfers/1]). + -export([open_file/3]). -export([complete/4]). -export([write/2]). @@ -31,16 +33,19 @@ -type transfer() :: emqx_ft:transfer(). -type offset() :: emqx_ft:offset(). - -type filemeta() :: emqx_ft:filemeta(). - --type segment() :: {offset(), _Content :: binary()}. +-type segment() :: emqx_ft:segment(). -type segmentinfo() :: #{ offset := offset(), size := _Bytes :: non_neg_integer() }. +-type transferinfo() :: #{ + status := complete | incomplete, + result => [filefrag({result, #{}})] +}. + % TODO naming -type filefrag(T) :: #{ path := file:name(), @@ -85,6 +90,7 @@ % Quota? Some lower level errors? {ok, emqx_ft_storage:ctx()} | {error, conflict} | {error, _TODO}. store_filemeta(Storage, Transfer, Meta) -> + % TODO safeguard against bad clientids / fileids. Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], ?MANIFEST), case read_file(Filepath, fun decode_filemeta/1) of {ok, Meta} -> @@ -170,6 +176,52 @@ assemble(Storage, Transfer, Callback) -> %% +-spec transfers(storage()) -> + {ok, #{transfer() => transferinfo()}}. +transfers(Storage) -> + % TODO `Continuation` + % There might be millions of transfers on the node, we need a protocol and + % storage schema to iterate through them effectively. + ClientIds = try_list_dir(get_storage_root(Storage)), + {ok, + lists:foldl( + fun(ClientId, Acc) -> transfers(Storage, ClientId, Acc) end, + #{}, + ClientIds + )}. + +transfers(Storage, ClientId, AccIn) -> + Dirname = mk_client_filedir(Storage, ClientId), + case file:list_dir(Dirname) of + {ok, FileIds} -> + lists:foldl( + fun(FileId, Acc) -> + Transfer = {filename_to_binary(ClientId), filename_to_binary(FileId)}, + read_transferinfo(Storage, Transfer, Acc) + end, + AccIn, + FileIds + ); + {error, _Reason} -> + % TODO worth logging + AccIn + end. + +read_transferinfo(Storage, Transfer, Acc) -> + case list(Storage, Transfer, result) of + {ok, Result = [_ | _]} -> + Info = #{status => complete, result => Result}, + Acc#{Transfer => Info}; + {ok, []} -> + Info = #{status => incomplete}, + Acc#{Transfer => Info}; + {error, _Reason} -> + % TODO worth logging + Acc + end. + +%% + -type handle() :: {file:name(), io:device(), crypto:hash_state()}. -spec open_file(storage(), transfer(), filemeta()) -> @@ -312,9 +364,18 @@ break_segment_filename(Filename) -> mk_filedir(Storage, {ClientId, FileId}, SubDirs) -> filename:join([get_storage_root(Storage), ClientId, FileId | SubDirs]). +mk_client_filedir(Storage, ClientId) -> + filename:join([get_storage_root(Storage), ClientId]). + mk_filepath(Storage, Transfer, SubDirs, Filename) -> filename:join(mk_filedir(Storage, Transfer, SubDirs), Filename). +try_list_dir(Dirname) -> + case file:list_dir(Dirname) of + {ok, List} -> List; + {error, _} -> [] + end. + get_storage_root(Storage) -> maps:get(root, Storage, filename:join(emqx:data_dir(), "file_transfer")). @@ -421,3 +482,6 @@ read_filemeta(_Filename, Filepath) -> read_segmentinfo(Filename, _Filepath) -> break_segment_filename(Filename). + +filename_to_binary(S) when is_list(S) -> unicode:characters_to_binary(S); +filename_to_binary(B) when is_binary(B) -> B. diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 77a619fe3..336580584 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -24,24 +24,31 @@ -include_lib("kernel/include/file.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -all() -> emqx_common_test_helpers:all(?MODULE). +all() -> + [ + t_assemble_empty_transfer, + t_assemble_complete_local_transfer, + + % NOTE + % It depends on the side effects of all previous testcases. + t_list_transfers + ]. init_per_suite(Config) -> - % {ok, Apps} = application:ensure_all_started(emqx_ft), - % [{suite_apps, Apps} | Config]. - % ok = emqx_common_test_helpers:start_apps([emqx_ft]), Config. end_per_suite(_Config) -> - % lists:foreach(fun application:stop/1, lists:reverse(?config(suite_apps, Config))). - % ok = emqx_common_test_helpers:stop_apps([emqx_ft]), ok. init_per_testcase(TC, Config) -> ok = snabbkaffe:start_trace(), - Root = filename:join(["roots", TC]), {ok, Pid} = emqx_ft_assembler_sup:start_link(), - [{storage_root, Root}, {assembler_sup, Pid} | Config]. + [ + {storage_root, "file_transfer_root"}, + {file_id, atom_to_binary(TC)}, + {assembler_sup, Pid} + | Config + ]. end_per_testcase(_TC, Config) -> ok = inspect_storage_root(Config), @@ -51,11 +58,12 @@ end_per_testcase(_TC, Config) -> %% --define(CLIENTID, <<"thatsme">>). +-define(CLIENTID1, <<"thatsme">>). +-define(CLIENTID2, <<"thatsnotme">>). t_assemble_empty_transfer(Config) -> Storage = storage(Config), - Transfer = {?CLIENTID, mk_fileid()}, + Transfer = {?CLIENTID1, ?config(file_id, Config)}, Filename = "important.pdf", Meta = #{ name => Filename, @@ -90,7 +98,7 @@ t_assemble_empty_transfer(Config) -> t_assemble_complete_local_transfer(Config) -> Storage = storage(Config), - Transfer = {?CLIENTID, mk_fileid()}, + Transfer = {?CLIENTID2, ?config(file_id, Config)}, Filename = "topsecret.pdf", TransferSize = 10000 + rand:uniform(50000), SegmentSize = 4096, @@ -155,6 +163,24 @@ on_assembly_finished(Result) -> %% +t_list_transfers(Config) -> + Storage = storage(Config), + ?assertMatch( + {ok, #{ + {?CLIENTID1, <<"t_assemble_empty_transfer">>} := #{ + status := complete, + result := [#{path := _, size := 0, fragment := {result, _}}] + }, + {?CLIENTID2, <<"t_assemble_complete_local_transfer">>} := #{ + status := complete, + result := [#{path := _, size := Size, fragment := {result, _}}] + } + }} when Size > 0, + emqx_ft_storage_fs:transfers(Storage) + ). + +%% + -include_lib("kernel/include/file.hrl"). inspect_storage_root(Config) -> From 92670bfe3df34ccd1fdd1e99b7bf0aa094bf4c3b Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 3 Feb 2023 18:48:50 +0300 Subject: [PATCH 016/197] feat(ft): add fs storage bpapi and use it in assembler --- apps/emqx_ft/src/emqx_ft_assembler.erl | 14 +++-- apps/emqx_ft/src/emqx_ft_storage.erl | 51 ++++++++++++++++- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 4 ++ .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 56 +++++++++++++++++++ 4 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index af50dee3b..ef4daf000 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -80,13 +80,11 @@ handle_event(internal, _, list_local_fragments, St = #st{assembly = Asm}) -> % {stop, Reason} end; handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> - % TODO: portable "storage" ref - Args = [St#st.storage, St#st.transfer, fragment], % TODO % Async would better because we would not need to wait for some lagging nodes if % the coverage is already complete. - % TODO: BP API? - Results = erpc:multicall(Nodes, emqx_ft_storage_fs, list, Args, ?RPC_LIST_TIMEOUT), + % TODO: portable "storage" ref + Results = emqx_ft_storage_fs_proto_v1:multilist(Nodes, St#st.transfer, fragment), NodeResults = lists:zip(Nodes, Results), NAsm = emqx_ft_assembly:update( lists:foldl( @@ -119,9 +117,8 @@ handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % TODO % Currently, race is possible between getting segment info from the remote node and % this node garbage collecting the segment itself. - Args = [St#st.storage, St#st.transfer, Segment, 0, segsize(Segment)], % TODO: pipelining - case erpc:call(Node, emqx_ft_storage_fs, pread, Args, ?RPC_READSEG_TIMEOUT) of + case pread(Node, Segment, St) of {ok, Content} -> {ok, NHandle} = emqx_ft_storage_fs:write(St#st.file, Content), {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])} @@ -158,6 +155,11 @@ handle_event(internal, _, complete, St = #st{assembly = Asm, file = Handle, call % handle_cast(_Cast, St) -> % {noreply, St}. +pread(Node, Segment, St) when Node =:= node() -> + emqx_ft_storage_fs:pread(St#st.storage, St#st.transfer, Segment, 0, segsize(Segment)); +pread(Node, Segment, St) -> + emqx_ft_storage_fs_proto_v1:pread(Node, St#st.transfer, Segment, 0, segsize(Segment)). + %% segsize(#{fragment := {segment, Info}}) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 8729a2ad4..819656551 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -24,6 +24,14 @@ ] ). +-export([list_local/2]). +-export([pread_local/4]). + +-export([local_transfers/0]). + +-type offset() :: emqx_ft:offset(). +-type transfer() :: emqx_ft:transfer(). + -type storage() :: emqx_config:config(). -export_type([assemble_callback/0]). @@ -63,8 +71,41 @@ assemble(Transfer, Callback) -> Mod = mod(), Mod:assemble(storage(), Transfer, Callback). +%%-------------------------------------------------------------------- +%% Local FS API +%%-------------------------------------------------------------------- + +-type filefrag() :: emqx_ft_storage_fs:filefrag(). +-type transferinfo() :: emqx_ft_storage_fs:transferinfo(). + +-spec list_local(transfer(), fragment | result) -> + {ok, [filefrag()]} | {error, term()}. +list_local(Transfer, What) -> + with_local_storage( + fun(Mod, Storage) -> Mod:list(Storage, Transfer, What) end + ). + +-spec pread_local(transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> + {ok, [filefrag()]} | {error, term()}. +pread_local(Transfer, Frag, Offset, Size) -> + with_local_storage( + fun(Mod, Storage) -> Mod:pread(Storage, Transfer, Frag, Offset, Size) end + ). + +-spec local_transfers() -> + {ok, node(), #{transfer() => transferinfo()}} | {error, term()}. +local_transfers() -> + with_local_storage( + fun(Mod, Storage) -> Mod:transfers(Storage) end + ). + +%% + mod() -> - case storage() of + mod(storage()). + +mod(Storage) -> + case Storage of #{type := local} -> emqx_ft_storage_fs % emqx_ft_storage_dummy @@ -72,3 +113,11 @@ mod() -> storage() -> emqx_config:get([file_transfer, storage]). + +with_local_storage(Fun) -> + case storage() of + #{type := local} = Storage -> + Fun(mod(Storage), Storage); + #{type := Type} -> + {error, {unsupported_storage_type, Type}} + end. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 3a78559c1..4afbc5276 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -31,6 +31,10 @@ -export([write/2]). -export([discard/1]). +-export_type([filefrag/1]). +-export_type([filefrag/0]). +-export_type([transferinfo/0]). + -type transfer() :: emqx_ft:transfer(). -type offset() :: emqx_ft:offset(). -type filemeta() :: emqx_ft:filemeta(). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl new file mode 100644 index 000000000..45dd93ab8 --- /dev/null +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -0,0 +1,56 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_proto_v1). + +-behaviour(emqx_bpapi). + +-export([introduced_in/0]). + +-export([list/3]). +-export([multilist/3]). +-export([pread/5]). +-export([transfers/1]). + +-type offset() :: emqx_ft:offset(). +-type transfer() :: emqx_ft:transfer(). +-type filefrag() :: emqx_ft_storage_fs:filefrag(). +-type transferinfo() :: emqx_ft_storage_fs:transferinfo(). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.17". + +-spec list(node(), transfer(), fragment | result) -> + {ok, [filefrag()]} | {error, term()}. +list(Node, Transfer, What) -> + erpc:call(Node, emqx_ft_storage, list_local, [Transfer, What]). + +-spec multilist([node()], transfer(), fragment | result) -> + emqx_rpc:erpc_multicall({ok, [filefrag()]} | {error, term()}). +multilist(Nodes, Transfer, What) -> + erpc:multicall(Nodes, emqx_ft_storage, list_local, [Transfer, What]). + +-spec pread(node(), transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> + {ok, [filefrag()]} | {error, term()}. +pread(Node, Transfer, Frag, Offset, Size) -> + erpc:call(Node, emqx_ft_storage, pread_local, [Transfer, Frag, Offset, Size]). + +-spec transfers([node()]) -> + emqx_rpc:erpc_multicall({ok, #{transfer() => transferinfo()}} | {error, term()}). +transfers(Nodes) -> + erpc:multicall(Nodes, emqx_ft_storage, local_transfers, []). From 04e5378bdaa320dec5d87489f81b16e2d1d316d3 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 6 Feb 2023 01:49:06 +0200 Subject: [PATCH 017/197] feat(ft): add API --- apps/emqx_ft/i18n/emqx_ft_api_i18n.conf | 25 +++ apps/emqx_ft/src/emqx_ft.erl | 8 +- apps/emqx_ft/src/emqx_ft_api.erl | 163 ++++++++++++++++++ apps/emqx_ft/src/emqx_ft_storage.erl | 97 ++++++----- apps/emqx_ft/src/emqx_ft_storage_dummy.erl | 10 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 159 ++++++++++------- .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 25 ++- 7 files changed, 360 insertions(+), 127 deletions(-) create mode 100644 apps/emqx_ft/i18n/emqx_ft_api_i18n.conf create mode 100644 apps/emqx_ft/src/emqx_ft_api.erl diff --git a/apps/emqx_ft/i18n/emqx_ft_api_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_api_i18n.conf new file mode 100644 index 000000000..0bda935f8 --- /dev/null +++ b/apps/emqx_ft/i18n/emqx_ft_api_i18n.conf @@ -0,0 +1,25 @@ +emqx_ft_api { + + file_list { + desc { + en: "List all uploaded files." + zh: "列出所有上传的文件。" + } + label: { + en: "List all uploaded files" + zh: "列出所有上传的文件" + } + } + + file_get { + desc { + en: "Get a file by its id." + zh: "根据文件 id 获取文件。" + } + label: { + en: "Get a file by its id" + zh: "根据文件 id 获取文件" + } + } + +} diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index ce3a7a97d..f54119440 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -189,7 +189,7 @@ on_fin(PacketId, Msg, FileId, Checksum) -> %% We have new fin packet ok -> Callback = callback(FinPacketKey, FileId), - case assemble(transfer(Msg, FileId), Callback) of + case emqx_ft_storage:assemble(transfer(Msg, FileId), Callback) of %% Assembling started, packet will be acked by the callback or the responder {ok, _} -> undefined; @@ -214,12 +214,6 @@ on_fin(PacketId, Msg, FileId, Checksum) -> undefined end. -assemble(Transfer, Callback) -> - emqx_ft_storage:assemble( - Transfer, - Callback - ). - callback({ChanPid, PacketId} = Key, _FileId) -> fun(Result) -> case emqx_ft_responder:unregister(Key) of diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl new file mode 100644 index 000000000..b2a822f36 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -0,0 +1,163 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_ft_api). + +-behaviour(minirest_api). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% Swagger specs from hocon schema +-export([ + api_spec/0, + paths/0, + schema/1, + namespace/0 +]). + +-export([ + fields/1, + roots/0 +]). + +%% API callbacks +-export([ + '/file_transfer/files'/2, + '/file_transfer/file'/2 +]). + +-import(hoconsc, [mk/2, ref/1, ref/2]). + +namespace() -> "file_transfer". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + [ + "/file_transfer/files", + "/file_transfer/file" + ]. + +schema("/file_transfer/files") -> + #{ + 'operationId' => '/file_transfer/files', + get => #{ + tags => [<<"file_transfer">>], + summary => <<"List all uploaded files">>, + description => ?DESC("file_list"), + responses => #{ + 200 => <<"Operation success">>, + 503 => emqx_dashboard_swagger:error_codes( + ['SERVICE_UNAVAILABLE'], <<"Service unavailable">> + ) + } + } + }; +schema("/file_transfer/file") -> + #{ + 'operationId' => '/file_transfer/file', + get => #{ + tags => [<<"file_transfer">>], + summary => <<"Download a particular file">>, + description => ?DESC("file_get"), + parameters => [ + ref(file_node), + ref(file_clientid), + ref(file_id) + ], + responses => #{ + 200 => <<"Operation success">>, + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Not found">>), + 503 => emqx_dashboard_swagger:error_codes( + ['SERVICE_UNAVAILABLE'], <<"Service unavailable">> + ) + } + } + }. + +'/file_transfer/files'(get, #{}) -> + case emqx_ft_storage:ready_transfers() of + {ok, Transfers} -> + FormattedTransfers = lists:map( + fun({Id, Info}) -> + #{id => Id, info => format_file_info(Info)} + end, + Transfers + ), + {200, #{<<"files">> => FormattedTransfers}}; + {error, _} -> + {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} + end. + +'/file_transfer/file'(get, #{query_string := Query}) -> + case emqx_ft_storage:get_ready_transfer(Query) of + {ok, FileData} -> + {200, #{<<"content-type">> => <<"application/data">>}, FileData}; + {error, enoent} -> + {404, error_msg('NOT_FOUND', <<"Not found">>)}; + {error, _} -> + {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} + end. + +error_msg(Code, Msg) -> + #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. + +-spec fields(hocon_schema:name()) -> hocon_schema:fields(). +fields(file_node) -> + Desc = <<"File Node">>, + Meta = #{ + in => query, desc => Desc, example => <<"emqx@127.0.0.1">>, required => false + }, + [{node, hoconsc:mk(binary(), Meta)}]; +fields(file_clientid) -> + Desc = <<"File ClientId">>, + Meta = #{ + in => query, desc => Desc, example => <<"client1">>, required => false + }, + [{clientid, hoconsc:mk(binary(), Meta)}]; +fields(file_id) -> + Desc = <<"File">>, + Meta = #{ + in => query, desc => Desc, example => <<"file1">>, required => false + }, + [{fileid, hoconsc:mk(binary(), Meta)}]. + +roots() -> + [ + file_node, + file_clientid, + file_id + ]. + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +format_file_info(#{path := Path, size := Size, timestamp := Timestamp}) -> + #{ + path => Path, + size => Size, + timestamp => format_datetime(Timestamp) + }. + +format_datetime({{Year, Month, Day}, {Hour, Minute, Second}}) -> + iolist_to_binary( + io_lib:format("~4..0w-~2..0w-~2..0wT~2..0w:~2..0w:~2..0w", [ + Year, Month, Day, Hour, Minute, Second + ]) + ). diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 819656551..e8f1d9c47 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -20,24 +20,27 @@ [ store_filemeta/2, store_segment/2, - assemble/2 + assemble/2, + + parse_id/1, + + ready_transfers/0, + get_ready_transfer/1, + + with_storage_type/3 ] ). --export([list_local/2]). --export([pread_local/4]). - --export([local_transfers/0]). - --type offset() :: emqx_ft:offset(). --type transfer() :: emqx_ft:transfer(). - -type storage() :: emqx_config:config(). -export_type([assemble_callback/0]). -type assemble_callback() :: fun((ok | {error, term()}) -> any()). +-type ready_transfer_id() :: term(). +-type ready_transfer_info() :: map(). +-type ready_transfer_data() :: binary(). + %%-------------------------------------------------------------------- %% Behaviour %%-------------------------------------------------------------------- @@ -48,6 +51,10 @@ ok | {error, term()}. -callback assemble(storage(), emqx_ft:transfer(), assemble_callback()) -> {ok, pid()} | {error, term()}. +-callback ready_transfers(storage()) -> + {ok, [{ready_transfer_id(), ready_transfer_info()}]} | {error, term()}. +-callback get_ready_transfer(storage(), ready_transfer_id()) -> + {ok, ready_transfer_data()} | {error, term()}. %%-------------------------------------------------------------------- %% API @@ -71,35 +78,46 @@ assemble(Transfer, Callback) -> Mod = mod(), Mod:assemble(storage(), Transfer, Callback). +-spec ready_transfers() -> {ok, [{ready_transfer_id(), ready_transfer_info()}]} | {error, term()}. +ready_transfers() -> + Mod = mod(), + Mod:ready_transfers(storage()). + +-spec get_ready_transfer(ready_transfer_id()) -> {ok, ready_transfer_data()} | {error, term()}. +get_ready_transfer(ReadyTransferId) -> + Mod = mod(), + Mod:get_ready_transfer(storage(), ReadyTransferId). + +-spec parse_id(map()) -> {ok, ready_transfer_id()} | {error, term()}. +parse_id(#{ + <<"type">> := local, <<"node">> := NodeBin, <<"clientid">> := ClientId, <<"id">> := Id +}) -> + case emqx_misc:safe_to_existing_atom(NodeBin) of + {ok, Node} -> + {ok, {local, Node, ClientId, Id}}; + {error, _} -> + {error, {invalid_node, NodeBin}} + end; +parse_id(#{}) -> + {error, invalid_file_id}. + +-spec with_storage_type(atom(), atom(), list(term())) -> any(). +with_storage_type(Type, Fun, Args) -> + Storage = storage(), + case Storage of + #{type := Type} -> + Mod = mod(Storage), + apply(Mod, Fun, [Storage | Args]); + _ -> + {error, {invalid_storage_type, Type}} + end. + %%-------------------------------------------------------------------- %% Local FS API %%-------------------------------------------------------------------- --type filefrag() :: emqx_ft_storage_fs:filefrag(). --type transferinfo() :: emqx_ft_storage_fs:transferinfo(). - --spec list_local(transfer(), fragment | result) -> - {ok, [filefrag()]} | {error, term()}. -list_local(Transfer, What) -> - with_local_storage( - fun(Mod, Storage) -> Mod:list(Storage, Transfer, What) end - ). - --spec pread_local(transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> - {ok, [filefrag()]} | {error, term()}. -pread_local(Transfer, Frag, Offset, Size) -> - with_local_storage( - fun(Mod, Storage) -> Mod:pread(Storage, Transfer, Frag, Offset, Size) end - ). - --spec local_transfers() -> - {ok, node(), #{transfer() => transferinfo()}} | {error, term()}. -local_transfers() -> - with_local_storage( - fun(Mod, Storage) -> Mod:transfers(Storage) end - ). - -%% +storage() -> + emqx_config:get([file_transfer, storage]). mod() -> mod(storage()). @@ -110,14 +128,3 @@ mod(Storage) -> emqx_ft_storage_fs % emqx_ft_storage_dummy end. - -storage() -> - emqx_config:get([file_transfer, storage]). - -with_local_storage(Fun) -> - case storage() of - #{type := local} = Storage -> - Fun(mod(Storage), Storage); - #{type := Type} -> - {error, {unsupported_storage_type, Type}} - end. diff --git a/apps/emqx_ft/src/emqx_ft_storage_dummy.erl b/apps/emqx_ft/src/emqx_ft_storage_dummy.erl index d486c0c29..4ed8ba487 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_dummy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_dummy.erl @@ -21,7 +21,9 @@ -export([ store_filemeta/3, store_segment/3, - assemble/3 + assemble/3, + ready_transfers/1, + get_ready_transfer/2 ]). store_filemeta(_Storage, _Transfer, _Meta) -> @@ -33,3 +35,9 @@ store_segment(_Storage, _Transfer, _Segment) -> assemble(_Storage, _Transfer, Callback) -> Pid = spawn(fun() -> Callback({error, not_implemented}) end), {ok, Pid}. + +ready_transfers(_Storage) -> + {ok, []}. + +get_ready_transfer(_Storage, _Id) -> + {error, not_implemented}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 4afbc5276..37a433433 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -18,6 +18,8 @@ -behaviour(emqx_ft_storage). +-include_lib("emqx/include/logger.hrl"). + -export([store_filemeta/3]). -export([store_segment/3]). -export([list/3]). @@ -26,6 +28,14 @@ -export([transfers/1]). +-export([pread_local/4]). +-export([list_local/2]). +-export([ready_transfers_local/0, ready_transfers_local/1]). +-export([get_ready_transfer_local/1, get_ready_transfer_local/2]). + +-export([ready_transfers/1]). +-export([get_ready_transfer/2]). + -export([open_file/3]). -export([complete/4]). -export([write/2]). @@ -70,23 +80,8 @@ -define(MANIFEST, "MANIFEST.json"). -define(SEGMENT, "SEG"). --type root() :: file:name(). - -% -record(st, { -% root :: file:name() -% }). - %% TODO --type storage() :: root(). - -%% - -% -define(PROCREF(Root), {via, gproc, {n, l, {?MODULE, Root}}}). - -% -spec start_link(root()) -> -% {ok, pid()} | {error, already_started}. -% start_link(Root) -> -% gen_server:start_link(?PROCREF(Root), ?MODULE, [], []). +-type storage() :: emqx_config:config(). %% Store manifest in the backing filesystem. %% Atomic operation. @@ -178,7 +173,89 @@ pread(_Storage, _Transfer, Frag, Offset, Size) -> assemble(Storage, Transfer, Callback) -> emqx_ft_assembler_sup:start_child(Storage, Transfer, Callback). -%% +-spec list_local(transfer(), fragment | result) -> + {ok, [filefrag()]} | {error, term()}. +list_local(Transfer, What) -> + emqx_ft_storage:with_storage_type(local, list, [Transfer, What]). + +-spec pread_local(transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> + {ok, [filefrag()]} | {error, term()}. +pread_local(Transfer, Frag, Offset, Size) -> + emqx_ft_storage:with_storage_type(local, pread, [Transfer, Frag, Offset, Size]). + +get_ready_transfer(_Storage, ReadyTransferId) -> + case parse_ready_transfer_id(ReadyTransferId) of + {ok, {Node, Transfer}} -> + try + emqx_ft_storage_fs_proto_v1:get_ready_transfer(Node, Transfer) + catch + error:Error -> + {error, Error}; + C:Error -> + {error, {C, Error}} + end; + {error, _} = Error -> + Error + end. + +get_ready_transfer_local(Transfer) -> + emqx_ft_storage:with_storage_type(local, get_ready_transfer_local, [Transfer]). + +get_ready_transfer_local(Storage, Transfer) -> + Dirname = mk_filedir(Storage, Transfer, get_subdirs_for(result)), + case file:list_dir(Dirname) of + {ok, [Filename | _]} -> + file:read_file(filename:join([Dirname, Filename])); + {error, _} = Error -> + Error + end. + +ready_transfers(_Storage) -> + Nodes = mria_mnesia:running_nodes(), + Results = emqx_ft_storage_fs_proto_v1:ready_transfers(Nodes), + {GoodResults, BadResults} = lists:partition( + fun + ({ok, _}) -> true; + (_) -> false + end, + Results + ), + ?SLOG(warning, #{msg => "ready_transfers", failures => BadResults}), + {ok, [File || {ok, Files} <- GoodResults, File <- Files]}. + +ready_transfers_local() -> + emqx_ft_storage:with_storage_type(local, ready_transfers_local, []). + +ready_transfers_local(Storage) -> + {ok, Transfers} = transfers(Storage), + lists:filtermap( + fun + ({Transfer, #{status := complete, result := [Result | _]}}) -> + {true, {ready_transfer_id(Transfer), maps:without([fragment], Result)}}; + (_) -> + false + end, + maps:to_list(Transfers) + ). + +ready_transfer_id({ClientId, FileId}) -> + #{ + <<"node">> => atom_to_binary(node()), + <<"clientid">> => ClientId, + <<"fileid">> => FileId + }. + +parse_ready_transfer_id(#{ + <<"node">> := NodeBin, <<"clientid">> := ClientId, <<"fileid">> := FileId +}) -> + case emqx_misc:safe_to_existing_atom(NodeBin) of + {ok, Node} -> + {ok, {Node, {ClientId, FileId}}}; + {error, _} -> + {error, {invalid_node, NodeBin}} + end; +parse_ready_transfer_id(#{}) -> + {error, invalid_file_id}. -spec transfers(storage()) -> {ok, #{transfer() => transferinfo()}}. @@ -291,41 +368,8 @@ verify_checksum(Ctx, #{checksum := {Algo, Digest}}) when Ctx /= undefined -> verify_checksum(undefined, _) -> ok. -%% - -% -spec init(root()) -> {ok, storage()}. -% init(Root) -> -% % TODO: garbage_collect(...) -% {ok, Root}. - -% %% - -define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). -% encode_filemeta(Meta) -> -% emqx_json:encode( -% ?PRELUDE( -% _Vsn = 1, -% maps:map( -% fun -% (name, Name) -> -% {<<"name">>, Name}; -% (size, Size) -> -% {<<"size">>, Size}; -% (checksum, {sha256, Hash}) -> -% {<<"checksum">>, <<"sha256:", (binary:encode_hex(Hash))/binary>>}; -% (expire_at, ExpiresAt) -> -% {<<"expire_at">>, ExpiresAt}; -% (segments_ttl, TTL) -> -% {<<"segments_ttl">>, TTL}; -% (user_data, UserData) -> -% {<<"user_data">>, UserData} -% end, -% Meta -% ) -% ) -% ). - encode_filemeta(Meta) -> % TODO: Looks like this should be hocon's responsibility. Schema = emqx_ft_schema:schema(filemeta), @@ -337,21 +381,6 @@ decode_filemeta(Binary) -> ?PRELUDE(_Vsn = 1, Term) = emqx_json:decode(Binary, [return_maps]), hocon_tconf:check_plain(Schema, Term, #{atom_key => true, required => false}). -% map_into(Fun, Into, Ks, Map) -> -% map_foldr(map_into_fn(Fun, Into), Into, Ks, Map). - -% map_into_fn(Fun, L) when is_list(L) -> -% fun(K, V, Acc) -> [{K, Fun(K, V)} || Acc] end. - -% map_foldr(_Fun, Acc, [], _) -> -% Acc; -% map_foldr(Fun, Acc, [K | Ks], Map) when is_map_key(K, Map) -> -% Fun(K, maps:get(K, Map), map_foldr(Fun, Acc, Ks, Map)); -% map_foldr(Fun, Acc, [_ | Ks], Map) -> -% map_foldr(Fun, Acc, Ks, Map). - -%% - mk_segment_filename({Offset, Content}) -> lists:concat([?SEGMENT, ".", Offset, ".", byte_size(Content)]). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index 45dd93ab8..4f354be63 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -23,12 +23,12 @@ -export([list/3]). -export([multilist/3]). -export([pread/5]). --export([transfers/1]). +-export([ready_transfers/1]). +-export([get_ready_transfer/2]). -type offset() :: emqx_ft:offset(). -type transfer() :: emqx_ft:transfer(). -type filefrag() :: emqx_ft_storage_fs:filefrag(). --type transferinfo() :: emqx_ft_storage_fs:transferinfo(). -include_lib("emqx/include/bpapi.hrl"). @@ -38,19 +38,26 @@ introduced_in() -> -spec list(node(), transfer(), fragment | result) -> {ok, [filefrag()]} | {error, term()}. list(Node, Transfer, What) -> - erpc:call(Node, emqx_ft_storage, list_local, [Transfer, What]). + erpc:call(Node, emqx_ft_storage_fs, list_local, [Transfer, What]). -spec multilist([node()], transfer(), fragment | result) -> emqx_rpc:erpc_multicall({ok, [filefrag()]} | {error, term()}). multilist(Nodes, Transfer, What) -> - erpc:multicall(Nodes, emqx_ft_storage, list_local, [Transfer, What]). + erpc:multicall(Nodes, emqx_ft_storage_fs, list_local, [Transfer, What]). -spec pread(node(), transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> {ok, [filefrag()]} | {error, term()}. pread(Node, Transfer, Frag, Offset, Size) -> - erpc:call(Node, emqx_ft_storage, pread_local, [Transfer, Frag, Offset, Size]). + erpc:call(Node, emqx_ft_storage_fs, pread_local, [Transfer, Frag, Offset, Size]). --spec transfers([node()]) -> - emqx_rpc:erpc_multicall({ok, #{transfer() => transferinfo()}} | {error, term()}). -transfers(Nodes) -> - erpc:multicall(Nodes, emqx_ft_storage, local_transfers, []). +-spec ready_transfers([node()]) -> + {ok, [{emqx_ft_storage:ready_transfer_id(), emqx_ft_storage:ready_transfer_info()}]} + | {error, term()}. +ready_transfers(Nodes) -> + erpc:multicall(Nodes, emqx_ft_storage_fs, ready_transfers_local, []). + +-spec get_ready_transfer(node(), emqx_ft_storage:ready_transfer_id()) -> + {ok, emqx_ft_storage:ready_transfer_data()} + | {error, term()}. +get_ready_transfer(Node, ReadyTransferId) -> + erpc:call(Node, emqx_ft_storage_fs, get_ready_transfer_local, [ReadyTransferId]). From 197ce32669a437dd7c26ecba71dc571f415dfcf8 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 6 Feb 2023 14:40:40 +0200 Subject: [PATCH 018/197] feat(ft): add proxy module for emqx_ft_storage_fs --- apps/emqx_ft/src/emqx_ft.erl | 13 +++++- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 22 +--------- apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl | 40 +++++++++++++++++++ .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 17 ++++---- 4 files changed, 63 insertions(+), 29 deletions(-) create mode 100644 apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index f54119440..c4f1caed5 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -189,7 +189,7 @@ on_fin(PacketId, Msg, FileId, Checksum) -> %% We have new fin packet ok -> Callback = callback(FinPacketKey, FileId), - case emqx_ft_storage:assemble(transfer(Msg, FileId), Callback) of + case assemble(transfer(Msg, FileId), Callback) of %% Assembling started, packet will be acked by the callback or the responder {ok, _} -> undefined; @@ -214,6 +214,17 @@ on_fin(PacketId, Msg, FileId, Checksum) -> undefined end. +assemble(Transfer, Callback) -> + try + emqx_ft_storage:assemble(Transfer, Callback) + catch + C:E:S -> + ?SLOG(warning, #{ + msg => "file_assemble_failed", class => C, reason => E, stacktrace => S + }), + {error, {internal_error, E}} + end. + callback({ChanPid, PacketId} = Key, _FileId) -> fun(Result) -> case emqx_ft_responder:unregister(Key) of diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 37a433433..088c316ae 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -28,10 +28,8 @@ -export([transfers/1]). --export([pread_local/4]). --export([list_local/2]). --export([ready_transfers_local/0, ready_transfers_local/1]). --export([get_ready_transfer_local/1, get_ready_transfer_local/2]). +-export([ready_transfers_local/1]). +-export([get_ready_transfer_local/2]). -export([ready_transfers/1]). -export([get_ready_transfer/2]). @@ -173,16 +171,6 @@ pread(_Storage, _Transfer, Frag, Offset, Size) -> assemble(Storage, Transfer, Callback) -> emqx_ft_assembler_sup:start_child(Storage, Transfer, Callback). --spec list_local(transfer(), fragment | result) -> - {ok, [filefrag()]} | {error, term()}. -list_local(Transfer, What) -> - emqx_ft_storage:with_storage_type(local, list, [Transfer, What]). - --spec pread_local(transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> - {ok, [filefrag()]} | {error, term()}. -pread_local(Transfer, Frag, Offset, Size) -> - emqx_ft_storage:with_storage_type(local, pread, [Transfer, Frag, Offset, Size]). - get_ready_transfer(_Storage, ReadyTransferId) -> case parse_ready_transfer_id(ReadyTransferId) of {ok, {Node, Transfer}} -> @@ -198,9 +186,6 @@ get_ready_transfer(_Storage, ReadyTransferId) -> Error end. -get_ready_transfer_local(Transfer) -> - emqx_ft_storage:with_storage_type(local, get_ready_transfer_local, [Transfer]). - get_ready_transfer_local(Storage, Transfer) -> Dirname = mk_filedir(Storage, Transfer, get_subdirs_for(result)), case file:list_dir(Dirname) of @@ -223,9 +208,6 @@ ready_transfers(_Storage) -> ?SLOG(warning, #{msg => "ready_transfers", failures => BadResults}), {ok, [File || {ok, Files} <- GoodResults, File <- Files]}. -ready_transfers_local() -> - emqx_ft_storage:with_storage_type(local, ready_transfers_local, []). - ready_transfers_local(Storage) -> {ok, Transfers} = transfers(Storage), lists:filtermap( diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl new file mode 100644 index 000000000..0c30f5567 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl @@ -0,0 +1,40 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% This methods are called via rpc by `emqx_ft_storage_fs` +%% They populate the call with actual storage which may be configured differently +%% on a concrete node. + +-module(emqx_ft_storage_fs_proxy). + +-export([ + list_local/2, + pread_local/4, + get_ready_transfer_local/1, + ready_transfers_local/0 +]). + +list_local(Transfer, What) -> + emqx_ft_storage:with_storage_type(local, list, [Transfer, What]). + +pread_local(Transfer, Frag, Offset, Size) -> + emqx_ft_storage:with_storage_type(local, pread, [Transfer, Frag, Offset, Size]). + +get_ready_transfer_local(Transfer) -> + emqx_ft_storage:with_storage_type(local, get_ready_transfer_local, [Transfer]). + +ready_transfers_local() -> + emqx_ft_storage:with_storage_type(local, ready_transfers_local, []). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index 4f354be63..2e3dc8632 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -36,28 +36,29 @@ introduced_in() -> "5.0.17". -spec list(node(), transfer(), fragment | result) -> - {ok, [filefrag()]} | {error, term()}. + {ok, [filefrag()]} | {error, term()} | no_return(). list(Node, Transfer, What) -> - erpc:call(Node, emqx_ft_storage_fs, list_local, [Transfer, What]). + erpc:call(Node, emqx_ft_storage_fs_proxy, list_local, [Transfer, What]). -spec multilist([node()], transfer(), fragment | result) -> emqx_rpc:erpc_multicall({ok, [filefrag()]} | {error, term()}). multilist(Nodes, Transfer, What) -> - erpc:multicall(Nodes, emqx_ft_storage_fs, list_local, [Transfer, What]). + erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, list_local, [Transfer, What]). -spec pread(node(), transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> - {ok, [filefrag()]} | {error, term()}. + {ok, [filefrag()]} | {error, term()} | no_return(). pread(Node, Transfer, Frag, Offset, Size) -> - erpc:call(Node, emqx_ft_storage_fs, pread_local, [Transfer, Frag, Offset, Size]). + erpc:call(Node, emqx_ft_storage_fs_proxy, pread_local, [Transfer, Frag, Offset, Size]). -spec ready_transfers([node()]) -> {ok, [{emqx_ft_storage:ready_transfer_id(), emqx_ft_storage:ready_transfer_info()}]} | {error, term()}. ready_transfers(Nodes) -> - erpc:multicall(Nodes, emqx_ft_storage_fs, ready_transfers_local, []). + erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, ready_transfers_local, []). -spec get_ready_transfer(node(), emqx_ft_storage:ready_transfer_id()) -> {ok, emqx_ft_storage:ready_transfer_data()} - | {error, term()}. + | {error, term()} + | no_return(). get_ready_transfer(Node, ReadyTransferId) -> - erpc:call(Node, emqx_ft_storage_fs, get_ready_transfer_local, [ReadyTransferId]). + erpc:call(Node, emqx_ft_storage_fs_proxy, get_ready_transfer_local, [ReadyTransferId]). From 0aefd4a8c7641635c852303fe923c249f134e699 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 6 Feb 2023 23:43:53 +0200 Subject: [PATCH 019/197] feat(ft): add streaming of file content when downloading --- apps/emqx_ft/src/emqx_ft_api.erl | 10 +- apps/emqx_ft/src/emqx_ft_storage.erl | 17 +-- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 37 ++++-- apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl | 6 +- .../emqx_ft/src/emqx_ft_storage_fs_reader.erl | 125 ++++++++++++++++++ .../src/emqx_ft_storage_fs_reader_sup.erl | 44 ++++++ apps/emqx_ft/src/emqx_ft_sup.erl | 11 +- .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 8 +- .../emqx_ft_storage_fs_reader_proto_v1.erl | 33 +++++ mix.exs | 2 +- rebar.config | 2 +- 11 files changed, 259 insertions(+), 36 deletions(-) create mode 100644 apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl create mode 100644 apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl create mode 100644 apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index b2a822f36..ddc6e761a 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -107,10 +107,16 @@ schema("/file_transfer/file") -> '/file_transfer/file'(get, #{query_string := Query}) -> case emqx_ft_storage:get_ready_transfer(Query) of {ok, FileData} -> - {200, #{<<"content-type">> => <<"application/data">>}, FileData}; + {200, + #{ + <<"content-type">> => <<"application/data">>, + <<"content-disposition">> => <<"attachment">> + }, + FileData}; {error, enoent} -> {404, error_msg('NOT_FOUND', <<"Not found">>)}; - {error, _} -> + {error, Error} -> + ?SLOG(warning, #{msg => "get_ready_transfer_fail", error => Error}), {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} end. diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index e8f1d9c47..0dd9d7989 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -22,8 +22,6 @@ store_segment/2, assemble/2, - parse_id/1, - ready_transfers/0, get_ready_transfer/1, @@ -39,7 +37,7 @@ -type ready_transfer_id() :: term(). -type ready_transfer_info() :: map(). --type ready_transfer_data() :: binary(). +-type ready_transfer_data() :: binary() | qlc:query_handle(). %%-------------------------------------------------------------------- %% Behaviour @@ -88,19 +86,6 @@ get_ready_transfer(ReadyTransferId) -> Mod = mod(), Mod:get_ready_transfer(storage(), ReadyTransferId). --spec parse_id(map()) -> {ok, ready_transfer_id()} | {error, term()}. -parse_id(#{ - <<"type">> := local, <<"node">> := NodeBin, <<"clientid">> := ClientId, <<"id">> := Id -}) -> - case emqx_misc:safe_to_existing_atom(NodeBin) of - {ok, Node} -> - {ok, {local, Node, ClientId, Id}}; - {error, _} -> - {error, {invalid_node, NodeBin}} - end; -parse_id(#{}) -> - {error, invalid_file_id}. - -spec with_storage_type(atom(), atom(), list(term())) -> any(). with_storage_type(Type, Fun, Args) -> Storage = storage(), diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 088c316ae..a120a4067 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -29,7 +29,7 @@ -export([transfers/1]). -export([ready_transfers_local/1]). --export([get_ready_transfer_local/2]). +-export([get_ready_transfer_local/3]). -export([ready_transfers/1]). -export([get_ready_transfer/2]). @@ -175,22 +175,43 @@ get_ready_transfer(_Storage, ReadyTransferId) -> case parse_ready_transfer_id(ReadyTransferId) of {ok, {Node, Transfer}} -> try - emqx_ft_storage_fs_proto_v1:get_ready_transfer(Node, Transfer) + case emqx_ft_storage_fs_proto_v1:get_ready_transfer(Node, self(), Transfer) of + {ok, ReaderPid} -> + {ok, emqx_ft_storage_fs_reader:table(ReaderPid)}; + {error, _} = Error -> + Error + end catch - error:Error -> - {error, Error}; - C:Error -> - {error, {C, Error}} + error:Exc:Stacktrace -> + ?SLOG(warning, #{ + msg => "get_ready_transfer_error", + node => Node, + transfer => Transfer, + exception => Exc, + stacktrace => Stacktrace + }), + {error, Exc}; + C:Exc:Stacktrace -> + ?SLOG(warning, #{ + msg => "get_ready_transfer_fail", + class => C, + node => Node, + transfer => Transfer, + exception => Exc, + stacktrace => Stacktrace + }), + {error, {C, Exc}} end; {error, _} = Error -> Error end. -get_ready_transfer_local(Storage, Transfer) -> +get_ready_transfer_local(Storage, CallerPid, Transfer) -> Dirname = mk_filedir(Storage, Transfer, get_subdirs_for(result)), case file:list_dir(Dirname) of {ok, [Filename | _]} -> - file:read_file(filename:join([Dirname, Filename])); + FullFilename = filename:join([Dirname, Filename]), + emqx_ft_storage_fs_reader:start_supervised(CallerPid, FullFilename); {error, _} = Error -> Error end. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl index 0c30f5567..7e19dd322 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl @@ -23,7 +23,7 @@ -export([ list_local/2, pread_local/4, - get_ready_transfer_local/1, + get_ready_transfer_local/2, ready_transfers_local/0 ]). @@ -33,8 +33,8 @@ list_local(Transfer, What) -> pread_local(Transfer, Frag, Offset, Size) -> emqx_ft_storage:with_storage_type(local, pread, [Transfer, Frag, Offset, Size]). -get_ready_transfer_local(Transfer) -> - emqx_ft_storage:with_storage_type(local, get_ready_transfer_local, [Transfer]). +get_ready_transfer_local(CallerPid, Transfer) -> + emqx_ft_storage:with_storage_type(local, get_ready_transfer_local, [CallerPid, Transfer]). ready_transfers_local() -> emqx_ft_storage:with_storage_type(local, ready_transfers_local, []). diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl new file mode 100644 index 000000000..a6307765e --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl @@ -0,0 +1,125 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_reader). + +-behaviour(gen_server). + +-include_lib("emqx/include/logger.hrl"). + +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). + +-export([ + start_link/2, + start_link/3, + start_supervised/2, + start_supervised/3, + read/1 +]). + +-export([ + table/1 +]). + +-define(DEFAULT_CHUNK_SIZE, 1024). + +table(ReaderPid) -> + NextFun = fun NextFun(Pid) -> + try + case emqx_ft_storage_fs_reader_proto_v1:read(node(Pid), Pid) of + eof -> + []; + {ok, Data} -> + [Data | fun() -> NextFun(Pid) end]; + {error, Reason} -> + ?SLOG(warning, #{msg => "file_read_error", reason => Reason}), + [] + end + catch + Class:Error:Stacktrace -> + ?SLOG(warning, #{ + msg => "file_read_error", + class => Class, + reason => Error, + stacktrace => Stacktrace + }), + [] + end + end, + qlc:table(fun() -> NextFun(ReaderPid) end, []). + +start_link(CallerPid, Filename) -> + start_link(CallerPid, Filename, ?DEFAULT_CHUNK_SIZE). + +start_link(CallerPid, Filename, ChunkSize) -> + gen_server:start_link(?MODULE, [CallerPid, Filename, ChunkSize], []). + +start_supervised(CallerPid, Filename) -> + start_supervised(CallerPid, Filename, ?DEFAULT_CHUNK_SIZE). + +start_supervised(CallerPid, Filename, ChunkSize) -> + emqx_ft_storage_fs_reader_sup:start_child(CallerPid, Filename, ChunkSize). + +read(Pid) -> + gen_server:call(Pid, read). + +init([CallerPid, Filename, ChunkSize]) -> + true = link(CallerPid), + case file:open(Filename, [read, raw, binary]) of + {ok, File} -> + {ok, #{ + filename => Filename, + file => File, + chunk_size => ChunkSize + }}; + {error, Reason} -> + {stop, Reason} + end. + +handle_call(read, _From, #{file := File, chunk_size := ChunkSize} = State) -> + case file:read(File, ChunkSize) of + {ok, Data} -> + ?SLOG(warning, #{msg => "read", bytes => byte_size(Data)}), + {reply, {ok, Data}, State}; + eof -> + ?SLOG(warning, #{msg => "read", eof => true}), + {stop, normal, eof, State}; + {error, Reason} = Error -> + {stop, Reason, Error, State} + end; +handle_call(Msg, _From, State) -> + {stop, {bad_call, Msg}, {bad_call, Msg}, State}. + +handle_info(Msg, State) -> + ?SLOG(warning, #{msg => "unexpected_message", info_msg => Msg}), + {noreply, State}. + +handle_cast(Msg, State) -> + ?SLOG(warning, #{msg => "unexpected_message", case_msg => Msg}), + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl new file mode 100644 index 000000000..7435d7e1c --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl @@ -0,0 +1,44 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_reader_sup). + +-behaviour(supervisor). + +-export([ + init/1, + start_link/0, + start_child/3 +]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +start_child(CallerPid, Filename, ChunkSize) -> + Childspec = #{ + id => {CallerPid, Filename}, + start => {emqx_ft_storage_fs_reader, start_link, [CallerPid, Filename, ChunkSize]}, + restart => temporary + }, + supervisor:start_child(?MODULE, Childspec). + +init(_) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 1000 + }, + {ok, {SupFlags, []}}. diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index b4ce52edb..5c2025860 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -43,6 +43,15 @@ init([]) -> modules => [emqx_ft_assembler_sup] }, + FileReaderSup = #{ + id => emqx_ft_storage_fs_reader_sup, + start => {emqx_ft_storage_fs_reader_sup, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [emqx_ft_storage_fs_reader_sup] + }, + Responder = #{ id => emqx_ft_responder, start => {emqx_ft_responder, start_link, []}, @@ -52,5 +61,5 @@ init([]) -> modules => [emqx_ft_responder] }, - ChildSpecs = [Responder, AssemblerSup], + ChildSpecs = [Responder, AssemblerSup, FileReaderSup], {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index 2e3dc8632..082df9ac0 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -24,7 +24,7 @@ -export([multilist/3]). -export([pread/5]). -export([ready_transfers/1]). --export([get_ready_transfer/2]). +-export([get_ready_transfer/3]). -type offset() :: emqx_ft:offset(). -type transfer() :: emqx_ft:transfer(). @@ -56,9 +56,9 @@ pread(Node, Transfer, Frag, Offset, Size) -> ready_transfers(Nodes) -> erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, ready_transfers_local, []). --spec get_ready_transfer(node(), emqx_ft_storage:ready_transfer_id()) -> +-spec get_ready_transfer(node(), pid(), emqx_ft_storage:ready_transfer_id()) -> {ok, emqx_ft_storage:ready_transfer_data()} | {error, term()} | no_return(). -get_ready_transfer(Node, ReadyTransferId) -> - erpc:call(Node, emqx_ft_storage_fs_proxy, get_ready_transfer_local, [ReadyTransferId]). +get_ready_transfer(Node, CallerPid, ReadyTransferId) -> + erpc:call(Node, emqx_ft_storage_fs_proxy, get_ready_transfer_local, [CallerPid, ReadyTransferId]). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl new file mode 100644 index 000000000..1bbb05471 --- /dev/null +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl @@ -0,0 +1,33 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_reader_proto_v1). + +-behaviour(emqx_bpapi). + +-export([introduced_in/0]). + +-export([read/2]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.17". + +-spec read(node(), pid()) -> + {ok, binary()} | eof | {error, term()} | no_return(). +read(Node, Pid) -> + erpc:call(Node, emqx_ft_storage_fs_reader, read, [Pid]). diff --git a/mix.exs b/mix.exs index 92024f48d..9a8c22e6b 100644 --- a/mix.exs +++ b/mix.exs @@ -58,7 +58,7 @@ defmodule EMQXUmbrella.MixProject do {:ekka, github: "emqx/ekka", tag: "0.14.6", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true}, - {:minirest, github: "emqx/minirest", tag: "1.3.8", override: true}, + {:minirest, github: "emqx/minirest", tag: "1.3.9", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, diff --git a/rebar.config b/rebar.config index f084f9827..0cc143e71 100644 --- a/rebar.config +++ b/rebar.config @@ -65,9 +65,9 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.8"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.9"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.5"}}} From b7d0bad97026381997e4bbf116c245674c6cfb64 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 7 Feb 2023 17:49:14 +0200 Subject: [PATCH 020/197] feat(ft): improve remote reader --- apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl | 17 +++++++++++++---- .../emqx_ft_storage_fs_reader_proto_v1.erl | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl index a6307765e..9c4aa5e0c 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl @@ -53,6 +53,9 @@ table(ReaderPid) -> [Data | fun() -> NextFun(Pid) end]; {error, Reason} -> ?SLOG(warning, #{msg => "file_read_error", reason => Reason}), + []; + {BadRPC, Reason} when BadRPC =:= badrpc orelse BadRPC =:= badtcp -> + ?SLOG(warning, #{msg => "file_read_rpc_error", kind => BadRPC, reason => Reason}), [] end catch @@ -84,13 +87,15 @@ read(Pid) -> gen_server:call(Pid, read). init([CallerPid, Filename, ChunkSize]) -> - true = link(CallerPid), + MRef = erlang:monitor(process, CallerPid), case file:open(Filename, [read, raw, binary]) of {ok, File} -> {ok, #{ filename => Filename, file => File, - chunk_size => ChunkSize + chunk_size => ChunkSize, + caller_pid => CallerPid, + mref => MRef }}; {error, Reason} -> {stop, Reason} @@ -99,10 +104,10 @@ init([CallerPid, Filename, ChunkSize]) -> handle_call(read, _From, #{file := File, chunk_size := ChunkSize} = State) -> case file:read(File, ChunkSize) of {ok, Data} -> - ?SLOG(warning, #{msg => "read", bytes => byte_size(Data)}), + ?SLOG(debug, #{msg => "read", bytes => byte_size(Data)}), {reply, {ok, Data}, State}; eof -> - ?SLOG(warning, #{msg => "read", eof => true}), + ?SLOG(debug, #{msg => "read", eof => true}), {stop, normal, eof, State}; {error, Reason} = Error -> {stop, Reason, Error, State} @@ -110,6 +115,10 @@ handle_call(read, _From, #{file := File, chunk_size := ChunkSize} = State) -> handle_call(Msg, _From, State) -> {stop, {bad_call, Msg}, {bad_call, Msg}, State}. +handle_info( + {'DOWN', MRef, process, CallerPid, _Reason}, #{mref := MRef, caller_pid := CallerPid} = State +) -> + {stop, {caller_down, CallerPid}, State}; handle_info(Msg, State) -> ?SLOG(warning, #{msg => "unexpected_message", info_msg => Msg}), {noreply, State}. diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl index 1bbb05471..982b9ca57 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl @@ -30,4 +30,4 @@ introduced_in() -> -spec read(node(), pid()) -> {ok, binary()} | eof | {error, term()} | no_return(). read(Node, Pid) -> - erpc:call(Node, emqx_ft_storage_fs_reader, read, [Pid]). + emqx_rpc:call(Node, emqx_ft_storage_fs_reader, read, [Pid]). From 8038a3fd4ac5b0f0b9726869f92d1698be6a703a Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 7 Feb 2023 23:29:22 +0200 Subject: [PATCH 021/197] feat(ft): add tests for remote reader --- apps/emqx_ft/etc/emqx_ft.conf | 5 + .../emqx_ft/src/emqx_ft_storage_fs_reader.erl | 108 +++++++------ .../src/emqx_ft_storage_fs_reader_sup.erl | 6 +- .../emqx_ft_storage_fs_reader_proto_v1.erl | 10 +- .../test/emqx_ft_storage_fs_reader_SUITE.erl | 153 ++++++++++++++++++ 5 files changed, 225 insertions(+), 57 deletions(-) create mode 100644 apps/emqx_ft/etc/emqx_ft.conf create mode 100644 apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl diff --git a/apps/emqx_ft/etc/emqx_ft.conf b/apps/emqx_ft/etc/emqx_ft.conf new file mode 100644 index 000000000..250dca6a9 --- /dev/null +++ b/apps/emqx_ft/etc/emqx_ft.conf @@ -0,0 +1,5 @@ +file_transfer { + storage { + type = local + } +} diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl index 9c4aa5e0c..782959e19 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl @@ -19,7 +19,18 @@ -behaviour(gen_server). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). +%% API +-export([ + start_link/2, + start_supervised/2, + table/1, + table/2, + read/2 +]). + +%% gen_server callbacks -export([ init/1, handle_call/3, @@ -29,71 +40,68 @@ code_change/3 ]). --export([ - start_link/2, - start_link/3, - start_supervised/2, - start_supervised/3, - read/1 -]). - --export([ - table/1 -]). - -define(DEFAULT_CHUNK_SIZE, 1024). +-define(IS_FILENAME(Filename), (is_list(Filename) or is_binary(Filename))). -table(ReaderPid) -> +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec table(pid()) -> qlc:query_handle(). +table(ReaderPid) when is_pid(ReaderPid) -> + table(ReaderPid, ?DEFAULT_CHUNK_SIZE). + +-spec table(pid(), pos_integer()) -> qlc:query_handle(). +table(ReaderPid, Bytes) when is_pid(ReaderPid) andalso is_integer(Bytes) andalso Bytes > 0 -> NextFun = fun NextFun(Pid) -> - try - case emqx_ft_storage_fs_reader_proto_v1:read(node(Pid), Pid) of - eof -> - []; - {ok, Data} -> - [Data | fun() -> NextFun(Pid) end]; - {error, Reason} -> - ?SLOG(warning, #{msg => "file_read_error", reason => Reason}), - []; - {BadRPC, Reason} when BadRPC =:= badrpc orelse BadRPC =:= badtcp -> - ?SLOG(warning, #{msg => "file_read_rpc_error", kind => BadRPC, reason => Reason}), - [] - end - catch - Class:Error:Stacktrace -> - ?SLOG(warning, #{ - msg => "file_read_error", - class => Class, - reason => Error, - stacktrace => Stacktrace - }), + case emqx_ft_storage_fs_reader_proto_v1:read(node(Pid), Pid, Bytes) of + eof -> + []; + {ok, Data} -> + [Data | fun() -> NextFun(Pid) end]; + {error, Reason} -> + ?SLOG(warning, #{msg => "file_read_error", reason => Reason}), + []; + {BadRPC, Reason} when BadRPC =:= badrpc orelse BadRPC =:= badtcp -> + ?SLOG(warning, #{msg => "file_read_rpc_error", kind => BadRPC, reason => Reason}), [] end end, qlc:table(fun() -> NextFun(ReaderPid) end, []). -start_link(CallerPid, Filename) -> - start_link(CallerPid, Filename, ?DEFAULT_CHUNK_SIZE). +-spec start_link(pid(), filename:filename()) -> startlink_ret(). +start_link(CallerPid, Filename) when + is_pid(CallerPid) andalso + ?IS_FILENAME(Filename) +-> + gen_server:start_link(?MODULE, [CallerPid, Filename], []). -start_link(CallerPid, Filename, ChunkSize) -> - gen_server:start_link(?MODULE, [CallerPid, Filename, ChunkSize], []). +-spec start_supervised(pid(), filename:filename()) -> startlink_ret(). +start_supervised(CallerPid, Filename) when + is_pid(CallerPid) andalso + ?IS_FILENAME(Filename) +-> + emqx_ft_storage_fs_reader_sup:start_child(CallerPid, Filename). -start_supervised(CallerPid, Filename) -> - start_supervised(CallerPid, Filename, ?DEFAULT_CHUNK_SIZE). +-spec read(pid(), pos_integer()) -> {ok, binary()} | eof | {error, term()}. +read(Pid, Bytes) when + is_pid(Pid) andalso + is_integer(Bytes) andalso + Bytes > 0 +-> + gen_server:call(Pid, {read, Bytes}). -start_supervised(CallerPid, Filename, ChunkSize) -> - emqx_ft_storage_fs_reader_sup:start_child(CallerPid, Filename, ChunkSize). +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- -read(Pid) -> - gen_server:call(Pid, read). - -init([CallerPid, Filename, ChunkSize]) -> +init([CallerPid, Filename]) -> MRef = erlang:monitor(process, CallerPid), case file:open(Filename, [read, raw, binary]) of {ok, File} -> {ok, #{ filename => Filename, file => File, - chunk_size => ChunkSize, caller_pid => CallerPid, mref => MRef }}; @@ -101,8 +109,8 @@ init([CallerPid, Filename, ChunkSize]) -> {stop, Reason} end. -handle_call(read, _From, #{file := File, chunk_size := ChunkSize} = State) -> - case file:read(File, ChunkSize) of +handle_call({read, Bytes}, _From, #{file := File} = State) -> + case file:read(File, Bytes) of {ok, Data} -> ?SLOG(debug, #{msg => "read", bytes => byte_size(Data)}), {reply, {ok, Data}, State}; @@ -113,7 +121,7 @@ handle_call(read, _From, #{file := File, chunk_size := ChunkSize} = State) -> {stop, Reason, Error, State} end; handle_call(Msg, _From, State) -> - {stop, {bad_call, Msg}, {bad_call, Msg}, State}. + {reply, {error, {bad_call, Msg}}, State}. handle_info( {'DOWN', MRef, process, CallerPid, _Reason}, #{mref := MRef, caller_pid := CallerPid} = State diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl index 7435d7e1c..934e2888c 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl @@ -21,16 +21,16 @@ -export([ init/1, start_link/0, - start_child/3 + start_child/2 ]). start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). -start_child(CallerPid, Filename, ChunkSize) -> +start_child(CallerPid, Filename) -> Childspec = #{ id => {CallerPid, Filename}, - start => {emqx_ft_storage_fs_reader, start_link, [CallerPid, Filename, ChunkSize]}, + start => {emqx_ft_storage_fs_reader, start_link, [CallerPid, Filename]}, restart => temporary }, supervisor:start_child(?MODULE, Childspec). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl index 982b9ca57..db5e35f94 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl @@ -20,14 +20,16 @@ -export([introduced_in/0]). --export([read/2]). +-export([read/3]). -include_lib("emqx/include/bpapi.hrl"). introduced_in() -> "5.0.17". --spec read(node(), pid()) -> +-spec read(node(), pid(), pos_integer()) -> {ok, binary()} | eof | {error, term()} | no_return(). -read(Node, Pid) -> - emqx_rpc:call(Node, emqx_ft_storage_fs_reader, read, [Pid]). +read(Node, Pid, Bytes) when + is_atom(Node) andalso is_pid(Pid) andalso is_integer(Bytes) andalso Bytes > 0 +-> + emqx_rpc:call(Node, emqx_ft_storage_fs_reader, read, [Pid, Bytes]). diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl new file mode 100644 index 000000000..e979d06fc --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl @@ -0,0 +1,153 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_reader_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_ft]), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft]), + ok. + +init_per_testcase(_Case, Config) -> + file:make_dir(?config(data_dir, Config)), + Data = <<"hello world">>, + Path = expand_path(Config, "test_file"), + ok = mk_test_file(Path, Data), + [{path, Path} | Config]. + +end_per_testcase(_Case, _Config) -> + ok. + +t_successful_read(Config) -> + Path = ?config(path, Config), + + {ok, ReaderPid} = emqx_ft_storage_fs_reader:start_link(self(), Path), + ?assertEqual( + {ok, <<"hello ">>}, + emqx_ft_storage_fs_reader:read(ReaderPid, 6) + ), + ?assertEqual( + {ok, <<"world">>}, + emqx_ft_storage_fs_reader:read(ReaderPid, 6) + ), + ?assertEqual( + eof, + emqx_ft_storage_fs_reader:read(ReaderPid, 6) + ), + ?assertNot(is_process_alive(ReaderPid)). + +t_caller_dead(Config) -> + erlang:process_flag(trap_exit, true), + + Path = ?config(path, Config), + + CallerPid = spawn_link( + fun() -> + receive + stop -> ok + end + end + ), + {ok, ReaderPid} = emqx_ft_storage_fs_reader:start_link(CallerPid, Path), + _ = erlang:monitor(process, ReaderPid), + ?assertEqual( + {ok, <<"hello ">>}, + emqx_ft_storage_fs_reader:read(ReaderPid, 6) + ), + CallerPid ! stop, + receive + {'DOWN', _, process, ReaderPid, _} -> ok + after 1000 -> + ct:fail("Reader process did not die") + end. + +t_tables(Config) -> + Path = ?config(path, Config), + + {ok, ReaderPid0} = emqx_ft_storage_fs_reader:start_link(self(), Path), + + ReaderQH0 = emqx_ft_storage_fs_reader:table(ReaderPid0, 6), + ?assertEqual( + [<<"hello ">>, <<"world">>], + qlc:eval(ReaderQH0) + ), + + {ok, ReaderPid1} = emqx_ft_storage_fs_reader:start_link(self(), Path), + + ReaderQH1 = emqx_ft_storage_fs_reader:table(ReaderPid1), + ?assertEqual( + [<<"hello world">>], + qlc:eval(ReaderQH1) + ). + +t_bad_messages(Config) -> + Path = ?config(path, Config), + + {ok, ReaderPid} = emqx_ft_storage_fs_reader:start_link(self(), Path), + + ReaderPid ! {bad, message}, + gen_server:cast(ReaderPid, {bad, message}), + + ?assertEqual( + {error, {bad_call, {bad, message}}}, + gen_server:call(ReaderPid, {bad, message}) + ). + +t_nonexistent_file(_Config) -> + ?assertEqual( + {error, enoent}, + emqx_ft_storage_fs_reader:start_link(self(), "/a/b/c/bar") + ). + +t_start_supervised(Config) -> + Path = ?config(path, Config), + + {ok, ReaderPid} = emqx_ft_storage_fs_reader:start_supervised(self(), Path), + ?assertEqual( + {ok, <<"hello ">>}, + emqx_ft_storage_fs_reader:read(ReaderPid, 6) + ). + +t_rpc_error(_Config) -> + ReaderQH = emqx_ft_storage_fs_reader:table(fake_remote_pid('dummy@127.0.0.1'), 6), + ?assertEqual( + [], + qlc:eval(ReaderQH) + ). + +mk_test_file(Path, Data) -> + ok = file:write_file(Path, Data). + +expand_path(Config, Filename) -> + filename:join([?config(data_dir, Config), Filename]). + +%% This is a hack to create a pid that is not registered on the local node. +%% https://www.erlang.org/doc/apps/erts/erl_ext_dist.html#new_pid_ext +fake_remote_pid(Node) -> + <<131, NodeAtom/binary>> = term_to_binary(Node), + PidBin = <<131, 88, NodeAtom/binary, 1:32/big, 1:32/big, 1:32/big>>, + binary_to_term(PidBin). From 836ec213c9c268032141075e8446e578326c78e0 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 8 Feb 2023 15:27:13 +0200 Subject: [PATCH 022/197] feat(ft): add responder tests --- apps/emqx/include/asserts.hrl | 38 +++++++ apps/emqx_ft/src/emqx_ft_responder.erl | 32 +++++- apps/emqx_ft/test/emqx_ft_responder_SUITE.erl | 99 +++++++++++++++++++ 3 files changed, 164 insertions(+), 5 deletions(-) create mode 100644 apps/emqx/include/asserts.hrl create mode 100644 apps/emqx_ft/test/emqx_ft_responder_SUITE.erl diff --git a/apps/emqx/include/asserts.hrl b/apps/emqx/include/asserts.hrl new file mode 100644 index 000000000..0e9f9477a --- /dev/null +++ b/apps/emqx/include/asserts.hrl @@ -0,0 +1,38 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-define(assertWaitEvent(Code, EventMatch, Timeout), + ?check_trace( + ?wait_async_action( + Code, + EventMatch, + Timeout + ), + fun(Trace) -> + ?assert( + lists:any( + fun + (EventMatch) -> true; + (_) -> false + end, + Trace + ) + ) + end + ) +). diff --git a/apps/emqx_ft/src/emqx_ft_responder.erl b/apps/emqx_ft/src/emqx_ft_responder.erl index f58569a27..8417053f6 100644 --- a/apps/emqx_ft/src/emqx_ft_responder.erl +++ b/apps/emqx_ft/src/emqx_ft_responder.erl @@ -21,6 +21,8 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/types.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + -export([start_link/0]). -export([ @@ -87,10 +89,13 @@ handle_call({unregister, Key}, _From, State) -> _ = erlang:cancel_timer(TRef), true = ets:delete(?TAB, Key), {reply, ok, State} - end. + end; +handle_call(Msg, _From, State) -> + ?SLOG(warning, #{msg => "unknown_call", call_msg => Msg}), + {reply, {error, unknown_call}, State}. handle_cast(Msg, State) -> - ?SLOG(warning, #{msg => "unknown cast", cast_msg => Msg}), + ?SLOG(warning, #{msg => "unknown_cast", cast_msg => Msg}), {noreply, State}. handle_info({timeout, TRef, {timeout, Key}}, State) -> @@ -100,12 +105,12 @@ handle_info({timeout, TRef, {timeout, Key}}, State) -> [{_, Action, TRef}] -> _ = erlang:cancel_timer(TRef), true = ets:delete(?TAB, Key), - %% TODO: safe apply - _ = Action(Key), + ok = safe_apply(Action, [Key]), + ?tp(ft_timeout_action_applied, #{key => Key}), {noreply, State} end; handle_info(Msg, State) -> - ?SLOG(warning, #{msg => "unknown message", info_msg => Msg}), + ?SLOG(warning, #{msg => "unknown_message", info_msg => Msg}), {noreply, State}. code_change(_OldVsn, State, _Extra) -> @@ -113,3 +118,20 @@ code_change(_OldVsn, State, _Extra) -> terminate(_Reason, _State) -> ok. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +safe_apply(Fun, Args) -> + try apply(Fun, Args) of + _ -> ok + catch + Class:Reason:Stacktrace -> + ?SLOG(error, #{ + msg => "safe_apply_failed", + class => Class, + reason => Reason, + stacktrace => Stacktrace + }) + end. diff --git a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl new file mode 100644 index 000000000..c08986120 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl @@ -0,0 +1,99 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_responder_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("emqx/include/asserts.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_ft]), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft]), + ok. + +init_per_testcase(_Case, Config) -> + Config. + +end_per_testcase(_Case, _Config) -> + ok. + +t_register_unregister(_Config) -> + Key = <<"test">>, + DefaultAction = fun(_) -> ok end, + ?assertEqual( + ok, + emqx_ft_responder:register(Key, DefaultAction, 1000) + ), + ?assertEqual( + {error, already_registered}, + emqx_ft_responder:register(Key, DefaultAction, 1000) + ), + ?assertEqual( + ok, + emqx_ft_responder:unregister(Key) + ), + ?assertEqual( + {error, not_found}, + emqx_ft_responder:unregister(Key) + ). + +t_timeout(_Config) -> + Key = <<"test">>, + Self = self(), + DefaultAction = fun(K) -> Self ! {timeout, K} end, + ok = emqx_ft_responder:register(Key, DefaultAction, 20), + receive + {timeout, Key} -> + ok + after 100 -> + ct:fail("emqx_ft_responder not called") + end, + ?assertEqual( + {error, not_found}, + emqx_ft_responder:unregister(Key) + ). + +t_action_exception(_Config) -> + Key = <<"test">>, + DefaultAction = fun(K) -> error({oops, K}) end, + + ?assertWaitEvent( + emqx_ft_responder:register(Key, DefaultAction, 10), + #{?snk_kind := ft_timeout_action_applied, key := <<"test">>}, + 1000 + ), + ?assertEqual( + {error, not_found}, + emqx_ft_responder:unregister(Key) + ). + +t_unknown_msgs(_Config) -> + Pid = whereis(emqx_ft_responder), + Pid ! {unknown_msg, <<"test">>}, + ok = gen_server:cast(Pid, {unknown_msg, <<"test">>}), + ?assertEqual( + {error, unknown_call}, + gen_server:call(Pid, {unknown_call, <<"test">>}) + ). From 9e4a37a39888bd71928436c1673d60a1e03599d9 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 8 Feb 2023 20:37:56 +0200 Subject: [PATCH 023/197] fix(ft): fix typespecs --- apps/emqx/src/proto/emqx_cm_proto_v2.erl | 2 +- apps/emqx_ft/src/emqx_ft_assembler.erl | 16 +++++++++++----- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 16 ++++++++++++---- apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl | 5 +---- .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 8 ++++++-- 5 files changed, 31 insertions(+), 16 deletions(-) diff --git a/apps/emqx/src/proto/emqx_cm_proto_v2.erl b/apps/emqx/src/proto/emqx_cm_proto_v2.erl index 2981dbd40..4208df97f 100644 --- a/apps/emqx/src/proto/emqx_cm_proto_v2.erl +++ b/apps/emqx/src/proto/emqx_cm_proto_v2.erl @@ -78,7 +78,7 @@ takeover_finish(ConnMod, ChanPid) -> erpc:call( node(ChanPid), emqx_cm, - takeover_session_finish, + takeover_finish, [ConnMod, ChanPid], ?T_TAKEOVER * 2 ). diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index ef4daf000..623e11714 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -34,7 +34,7 @@ storage :: _Storage, transfer :: emqx_ft:transfer(), assembly :: _TODO, - file :: io:device(), + file :: {file:filename(), io:device(), term()} | undefined, hash, callback :: fun((ok | {error, term()}) -> any()) }). @@ -120,10 +120,16 @@ handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % TODO: pipelining case pread(Node, Segment, St) of {ok, Content} -> - {ok, NHandle} = emqx_ft_storage_fs:write(St#st.file, Content), - {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])} - % {error, _} -> - % ... + case emqx_ft_storage_fs:write(St#st.file, Content) of + {ok, NHandle} -> + {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])}; + %% TODO: better error handling + {error, Error} -> + error(Error) + end; + {error, Error} -> + %% TODO: better error handling + error(Error) end; handle_event(internal, _, {assemble, []}, St = #st{}) -> {next_state, complete, St, ?internal([])}; diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index a120a4067..a6559d470 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -85,7 +85,7 @@ %% Atomic operation. -spec store_filemeta(storage(), transfer(), filemeta()) -> % Quota? Some lower level errors? - {ok, emqx_ft_storage:ctx()} | {error, conflict} | {error, _TODO}. + ok | {error, conflict} | {error, _TODO}. store_filemeta(Storage, Transfer, Meta) -> % TODO safeguard against bad clientids / fileids. Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], ?MANIFEST), @@ -226,8 +226,16 @@ ready_transfers(_Storage) -> end, Results ), - ?SLOG(warning, #{msg => "ready_transfers", failures => BadResults}), - {ok, [File || {ok, Files} <- GoodResults, File <- Files]}. + case {GoodResults, BadResults} of + {[], _} -> + ?SLOG(warning, #{msg => "ready_transfers", failures => BadResults}), + {error, no_nodes}; + {_, []} -> + {ok, [File || {ok, Files} <- GoodResults, File <- Files]}; + {_, _} -> + ?SLOG(warning, #{msg => "ready_transfers", failures => BadResults}), + {ok, [File || {ok, Files} <- GoodResults, File <- Files]} + end. ready_transfers_local(Storage) -> {ok, Transfers} = transfers(Storage), @@ -323,7 +331,7 @@ open_file(Storage, Transfer, Filemeta) -> end. -spec write(handle(), iodata()) -> - ok | {error, _TODO}. + {ok, handle()} | {error, _TODO}. write({Filepath, IoDevice, Ctx}, IoData) -> case file:write(IoDevice, IoData) of ok -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl index 782959e19..373b92753 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl @@ -58,12 +58,9 @@ table(ReaderPid, Bytes) when is_pid(ReaderPid) andalso is_integer(Bytes) andalso eof -> []; {ok, Data} -> - [Data | fun() -> NextFun(Pid) end]; + [Data] ++ fun() -> NextFun(Pid) end; {error, Reason} -> ?SLOG(warning, #{msg => "file_read_error", reason => Reason}), - []; - {BadRPC, Reason} when BadRPC =:= badrpc orelse BadRPC =:= badtcp -> - ?SLOG(warning, #{msg => "file_read_rpc_error", kind => BadRPC, reason => Reason}), [] end end, diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index 082df9ac0..992d62c48 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -51,8 +51,12 @@ pread(Node, Transfer, Frag, Offset, Size) -> erpc:call(Node, emqx_ft_storage_fs_proxy, pread_local, [Transfer, Frag, Offset, Size]). -spec ready_transfers([node()]) -> - {ok, [{emqx_ft_storage:ready_transfer_id(), emqx_ft_storage:ready_transfer_info()}]} - | {error, term()}. + [ + {ok, [{emqx_ft_storage:ready_transfer_id(), emqx_ft_storage:ready_transfer_info()}]} + | {error, term()} + | {exit, term()} + | {throw, term()} + ]. ready_transfers(Nodes) -> erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, ready_transfers_local, []). From bcfa22f3434d57e868060b9f126a1a11403d3f9a Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 8 Feb 2023 23:41:33 +0200 Subject: [PATCH 024/197] fix(ft): use correct supervison strategy for emqx_ft_sup --- apps/emqx_ft/src/emqx_ft_sup.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index 5c2025860..dfdde3a8a 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -29,7 +29,7 @@ start_link() -> init([]) -> SupFlags = #{ - strategy => one_for_all, + strategy => one_for_one, intensity => 100, period => 10 }, From eae3dc7b4b0c11fab106db0b69524542b311591d Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 10 Feb 2023 13:34:33 +0200 Subject: [PATCH 025/197] feat(ft): update BPAPI --- apps/emqx/priv/bpapi.versions | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 6190925d2..0e0957adf 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -7,11 +7,14 @@ {emqx_bridge,3}. {emqx_broker,1}. {emqx_cm,1}. +{emqx_cm,2}. {emqx_conf,1}. {emqx_conf,2}. {emqx_dashboard,1}. {emqx_delayed,1}. {emqx_exhook,1}. +{emqx_ft_storage_fs,1}. +{emqx_ft_storage_fs_reader,1}. {emqx_gateway_api_listeners,1}. {emqx_gateway_cm,1}. {emqx_gateway_http,1}. From c44fe92ef181aa81acc2193efed516552f5a2113 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 10 Feb 2023 14:57:10 +0200 Subject: [PATCH 026/197] feat(ft): add assembler tests --- apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 11 +++++ apps/emqx_ft/src/emqx_ft_assembler.erl | 35 +++++++++++---- apps/emqx_ft/src/emqx_ft_assembler_sup.erl | 2 +- apps/emqx_ft/src/emqx_ft_schema.erl | 5 +++ apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 43 +++++++++++++++++++ 5 files changed, 86 insertions(+), 10 deletions(-) diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf index 481ad8154..7e057fdf8 100644 --- a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -11,4 +11,15 @@ emqx_ft_schema { } } + local_storage_root { + desc { + en: "File system path to keep uploaded files and temporary data." + zh: "保存上传文件和临时数据的文件系统路径。" + } + label: { + en: "Local Storage Root" + zh: "本地存储根" + } + } + } diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 623e11714..38faebf03 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -16,6 +16,8 @@ -module(emqx_ft_assembler). +-include_lib("emqx/include/logger.hrl"). + -export([start_link/3]). -behaviour(gen_statem). @@ -73,7 +75,7 @@ handle_event(internal, _, list_local_fragments, St = #st{assembly = Asm}) -> complete -> {next_state, start_assembling, NSt, ?internal([])}; {incomplete, _} -> - Nodes = ekka:nodelist() -- [node()], + Nodes = mria_mnesia:running_nodes() -- [node()], {next_state, {list_remote_fragments, Nodes}, NSt, ?internal([])} % TODO: recovery? % {error, _} = Reason -> @@ -105,7 +107,7 @@ handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> {next_state, start_assembling, NSt, ?internal([])}; % TODO: retries / recovery? {incomplete, _} = Status -> - {stop, {error, Status}} + {next_state, {failure, {error, Status}}, NSt, ?internal([])} end; handle_event(internal, _, start_assembling, St = #st{assembly = Asm}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), @@ -124,21 +126,23 @@ handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> {ok, NHandle} -> {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])}; %% TODO: better error handling - {error, Error} -> - error(Error) + {error, _} = Error -> + {next_state, {failure, Error}, St, ?internal([])} end; - {error, Error} -> + {error, _} = Error -> %% TODO: better error handling - error(Error) + {next_state, {failure, Error}, St, ?internal([])} end; handle_event(internal, _, {assemble, []}, St = #st{}) -> {next_state, complete, St, ?internal([])}; handle_event(internal, _, complete, St = #st{assembly = Asm, file = Handle, callback = Callback}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), Result = emqx_ft_storage_fs:complete(St#st.storage, St#st.transfer, Filemeta, Handle), - %% TODO: safe apply - _ = Callback(Result), - {stop, shutdown}. + _ = safe_apply(Callback, Result), + {stop, shutdown}; +handle_event(internal, _, {failure, Error}, #st{callback = Callback}) -> + _ = safe_apply(Callback, Error), + {stop, Error}. % handle_continue(list_local, St = #st{storage = Storage, transfer = Transfer, assembly = Asm}) -> % % TODO: what we do with non-transients errors here (e.g. `eacces`)? @@ -170,3 +174,16 @@ pread(Node, Segment, St) -> segsize(#{fragment := {segment, Info}}) -> maps:get(size, Info). + +safe_apply(Callback, Result) -> + try apply(Callback, [Result]) of + _ -> ok + catch + Class:Reason:Stacktrace -> + ?SLOG(error, #{ + msg => "safe_apply_failed", + class => Class, + reason => Reason, + stacktrace => Stacktrace + }) + end. diff --git a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl index b60949a5e..17aa4d998 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl @@ -29,7 +29,7 @@ start_child(Storage, Transfer, Callback) -> Childspec = #{ id => {Storage, Transfer}, start => {emqx_ft_assembler, start_link, [Storage, Transfer, Callback]}, - restart => transient + restart => temporary }, supervisor:start_child(?MODULE, Childspec). diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 70acb8322..deb2cae6f 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -59,6 +59,11 @@ fields(local_storage) -> default => local, required => false, desc => ?DESC("local") + }}, + {root, #{ + type => binary(), + desc => ?DESC("local_storage_root"), + required => false }} ]. diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 336580584..dd3ffedad 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -28,6 +28,8 @@ all() -> [ t_assemble_empty_transfer, t_assemble_complete_local_transfer, + t_assemble_incomplete_transfer, + t_assemble_no_meta, % NOTE % It depends on the side effects of all previous testcases. @@ -155,6 +157,46 @@ t_assemble_complete_local_transfer(Config) -> AssemblyFilename ). +t_assemble_incomplete_transfer(Config) -> + Storage = storage(Config), + Transfer = {?CLIENTID2, ?config(file_id, Config)}, + Filename = "incomplete.pdf", + TransferSize = 10000 + rand:uniform(50000), + SegmentSize = 4096, + Gen = emqx_ft_content_gen:new({Transfer, TransferSize}, SegmentSize), + Hash = emqx_ft_content_gen:hash(Gen, crypto:hash_init(sha256)), + Meta = #{ + name => Filename, + checksum => {sha256, Hash}, + size => TransferSize, + expire_at => 42 + }, + ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta), + Self = self(), + {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun(Result) -> + Self ! {test_assembly_finished, Result} + end), + receive + {test_assembly_finished, Result} -> + ?assertMatch({error, _}, Result) + after 1000 -> + ct:fail("Assembler did not called callback") + end. + +t_assemble_no_meta(Config) -> + Storage = storage(Config), + Transfer = {?CLIENTID2, ?config(file_id, Config)}, + Self = self(), + {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun(Result) -> + Self ! {test_assembly_finished, Result} + end), + receive + {test_assembly_finished, Result} -> + ?assertMatch({error, _}, Result) + after 1000 -> + ct:fail("Assembler did not called callback") + end. + mk_assembly_filename(Config, {ClientID, FileID}, Filename) -> filename:join([?config(storage_root, Config), ClientID, FileID, result, Filename]). @@ -205,5 +247,6 @@ mk_fileid() -> storage(Config) -> #{ + type => local, root => ?config(storage_root, Config) }. From 1d48a97fd2b5ea32daf06745adfb06b21b1e73ff Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 13 Feb 2023 15:19:21 +0200 Subject: [PATCH 027/197] feat(ft): fix remote reader handling of gen_rpc errors --- apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl index 373b92753..4b1c4acb8 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl @@ -59,8 +59,8 @@ table(ReaderPid, Bytes) when is_pid(ReaderPid) andalso is_integer(Bytes) andalso []; {ok, Data} -> [Data] ++ fun() -> NextFun(Pid) end; - {error, Reason} -> - ?SLOG(warning, #{msg => "file_read_error", reason => Reason}), + {ErrorKind, Reason} when ErrorKind =:= badrpc; ErrorKind =:= error -> + ?SLOG(warning, #{msg => "file_read_error", kind => ErrorKind, reason => Reason}), [] end end, From 2e889f4ac76e93e26d7ebfc62a3060fa7c00995c Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 14 Feb 2023 01:18:15 +0200 Subject: [PATCH 028/197] feat(ft): add emqx_ft tests and fixes --- apps/emqx/test/emqx_common_test_helpers.erl | 10 + apps/emqx_ft/src/emqx_ft.erl | 100 +++++- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 12 +- .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 2 +- .../emqx_ft_storage_fs_reader_proto_v1.erl | 2 +- apps/emqx_ft/test/emqx_ft_SUITE.erl | 284 ++++++++++++++++++ apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_content_gen.erl | 2 +- apps/emqx_ft/test/emqx_ft_responder_SUITE.erl | 2 +- .../test/emqx_ft_storage_fs_reader_SUITE.erl | 2 +- 10 files changed, 398 insertions(+), 20 deletions(-) create mode 100644 apps/emqx_ft/test/emqx_ft_SUITE.erl diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 077ebe138..ecdf8f827 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -63,6 +63,7 @@ ]). -export([ + maybe_fix_gen_rpc/0, emqx_cluster/1, emqx_cluster/2, start_epmd/0, @@ -616,6 +617,15 @@ ensure_quic_listener(Name, UdpPort, ExtraSettings) -> listener_ports => [{Type :: tcp | ssl | ws | wss, inet:port_number()}] }. +-spec maybe_fix_gen_rpc() -> ok. +maybe_fix_gen_rpc() -> + %% When many tests run in an obscure order, it may occur that + %% `gen_rpc` started with its default settings before `emqx_conf`. + %% `gen_rpc` and `emqx_conf` have different default `port_discovery` modes, + %% so we reinitialize `gen_rpc` explicitly. + ok = application:stop(gen_rpc), + ok = application:start(gen_rpc). + -spec emqx_cluster(cluster_spec()) -> [{shortname(), node_opts()}]. emqx_cluster(Specs) -> emqx_cluster(Specs, #{}). diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index c4f1caed5..ed4edd33b 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -31,6 +31,10 @@ on_message_puback/4 ]). +-export([ + decode_filemeta/1 +]). + -export([on_assemble_timeout/1]). -export_type([ @@ -86,6 +90,27 @@ unhook() -> ok = emqx_hooks:del('message.publish', {?MODULE, on_message_publish}), ok = emqx_hooks:del('message.puback', {?MODULE, on_message_puback}). +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +decode_filemeta(Payload) when is_binary(Payload) -> + case emqx_json:safe_decode(Payload, [return_maps]) of + {ok, Map} -> + decode_filemeta(Map); + {error, Error} -> + {error, {invalid_filemeta_json, Error}} + end; +decode_filemeta(Map) when is_map(Map) -> + Schema = emqx_ft_schema:schema(filemeta), + try + Meta = hocon_tconf:check_plain(Schema, Map, #{atom_key => true, required => false}), + {ok, Meta} + catch + throw:Error -> + {error, {invalid_filemeta, Error}} + end. + %%-------------------------------------------------------------------- %% Hooks %%-------------------------------------------------------------------- @@ -113,6 +138,8 @@ on_message_puback(PacketId, #message{topic = Topic} = Msg, _PubRes, _RC) -> %% Handlers for transfer messages %%-------------------------------------------------------------------- +%% TODO Move to emqx_ft_mqtt? + on_file_command(PacketId, Msg, FileCommand) -> case string:split(FileCommand, <<"/">>, all) of [FileId, <<"init">>] -> @@ -123,10 +150,14 @@ on_file_command(PacketId, Msg, FileCommand) -> on_fin(PacketId, Msg, FileId, Checksum); [FileId, <<"abort">>] -> on_abort(Msg, FileId); - [FileId, Offset] -> - on_segment(Msg, FileId, Offset, undefined); - [FileId, Offset, Checksum] -> - on_segment(Msg, FileId, Offset, Checksum); + [FileId, OffsetBin] -> + validate([{offset, OffsetBin}], fun([Offset]) -> + on_segment(Msg, FileId, Offset, undefined) + end); + [FileId, OffsetBin, ChecksumBin] -> + validate([{offset, OffsetBin}, {checksum, ChecksumBin}], fun([Offset, Checksum]) -> + on_segment(Msg, FileId, Offset, Checksum) + end); _ -> ?RC_UNSPECIFIED_ERROR end. @@ -139,11 +170,21 @@ on_init(Msg, FileId) -> }), Payload = Msg#message.payload, % %% Add validations here - Meta = emqx_json:decode(Payload, [return_maps]), - case emqx_ft_storage:store_filemeta(transfer(Msg, FileId), Meta) of - ok -> - ?RC_SUCCESS; - {error, _Reason} -> + case decode_filemeta(Payload) of + {ok, Meta} -> + case emqx_ft_storage:store_filemeta(transfer(Msg, FileId), Meta) of + ok -> + ?RC_SUCCESS; + {error, _Reason} -> + ?RC_UNSPECIFIED_ERROR + end; + {error, Reason} -> + ?SLOG(error, #{ + msg => "on_init: invalid filemeta", + mqtt_msg => Msg, + file_id => FileId, + reason => Reason + }), ?RC_UNSPECIFIED_ERROR end. @@ -161,7 +202,7 @@ on_segment(Msg, FileId, Offset, Checksum) -> }), %% TODO: handle checksum Payload = Msg#message.payload, - Segment = {binary_to_integer(Offset), Payload}, + Segment = {Offset, Payload}, %% Add offset/checksum validations case emqx_ft_storage:store_segment(transfer(Msg, FileId), Segment) of ok -> @@ -247,3 +288,42 @@ transfer(Msg, FileId) -> on_assemble_timeout({ChanPid, PacketId}) -> ?SLOG(warning, #{msg => "on_assemble_timeout", packet_id => PacketId}), erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}). + +validate(Validations, Fun) -> + case do_validate(Validations, []) of + {ok, Parsed} -> + Fun(Parsed); + {error, Reason} -> + ?SLOG(error, #{ + msg => "validate: invalid $file command", + reason => Reason + }), + ?RC_UNSPECIFIED_ERROR + end. + +do_validate([], Parsed) -> + {ok, lists:reverse(Parsed)}; +do_validate([{offset, Offset} | Rest], Parsed) -> + case string:to_integer(Offset) of + {Int, <<>>} -> + do_validate(Rest, [Int | Parsed]); + _ -> + {error, {invalid_offset, Offset}} + end; +do_validate([{checksum, Checksum} | Rest], Parsed) -> + case parse_checksum(Checksum) of + {ok, Bin} -> + do_validate(Rest, [Bin | Parsed]); + {error, _Reason} -> + {error, {invalid_checksum, Checksum}} + end. + +parse_checksum(Checksum) when is_binary(Checksum) andalso byte_size(Checksum) =:= 64 -> + try + {ok, binary:decode_hex(Checksum)} + catch + error:badarg -> + {error, invalid_checksum} + end; +parse_checksum(_Checksum) -> + {error, invalid_checksum}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index a6559d470..01a3e19c2 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -387,10 +387,14 @@ encode_filemeta(Meta) -> Term = hocon_tconf:make_serializable(Schema, emqx_map_lib:binary_key_map(Meta), #{}), emqx_json:encode(?PRELUDE(_Vsn = 1, Term)). -decode_filemeta(Binary) -> - Schema = emqx_ft_schema:schema(filemeta), - ?PRELUDE(_Vsn = 1, Term) = emqx_json:decode(Binary, [return_maps]), - hocon_tconf:check_plain(Schema, Term, #{atom_key => true, required => false}). +decode_filemeta(Binary) when is_binary(Binary) -> + ?PRELUDE(_Vsn = 1, Map) = emqx_json:decode(Binary, [return_maps]), + case emqx_ft:decode_filemeta(Map) of + {ok, Meta} -> + Meta; + {error, Reason} -> + error(Reason) + end. mk_segment_filename({Offset, Content}) -> lists:concat([?SEGMENT, ".", Offset, ".", byte_size(Content)]). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index 992d62c48..f69033b4b 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl index db5e35f94..f8fe02d36 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl new file mode 100644 index 000000000..ff7cba403 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -0,0 +1,284 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-define(assertRCName(RCName, PublishRes), + ?assertMatch( + {ok, #{reason_code_name := RCName}}, + PublishRes + ) +). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], fun set_special_configs/1), + ok = emqx_common_test_helpers:maybe_fix_gen_rpc(), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok. + +set_special_configs(emqx_ft) -> + {ok, _} = emqx:update_config([file_transfer, storage], #{<<"type">> => <<"local">>}), + ok; +set_special_configs(_App) -> + ok. + +init_per_testcase(_Case, Config) -> + _ = file:del_dir_r(filename:join(emqx:data_dir(), "file_transfer")), + ClientId = <<"client">>, + {ok, C} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(C), + [{client, C}, {clientid, ClientId} | Config]. + +end_per_testcase(_Case, Config) -> + C = ?config(client, Config), + ok = emqtt:stop(C), + ok. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_invalid_topic_format(Config) -> + C = ?config(client, Config), + + %% TODO: more invalid topics + + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/XYZ">>, <<>>, 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/X/Y/Z">>, <<>>, 1) + ). + +t_simple_transfer(Config) -> + C = ?config(client, Config), + + Filename = <<"topsecret.pdf">>, + FileId = <<"f1">>, + + Data = [<<"first">>, <<"second">>, <<"third">>], + + Meta = meta(Filename, Data), + MetaPayload = emqx_json:encode(Meta), + + MetaTopic = <<"$file/", FileId/binary, "/init">>, + ?assertRCName( + success, + emqtt:publish(C, MetaTopic, MetaPayload, 1) + ), + + lists:foreach( + fun({Chunk, Offset}) -> + OffsetBin = integer_to_binary(Offset), + SegmentTopic = <<"$file/", FileId/binary, "/", OffsetBin/binary>>, + ?assertRCName( + success, + emqtt:publish(C, SegmentTopic, Chunk, 1) + ) + end, + with_offsets(Data) + ), + + FinTopic = <<"$file/", FileId/binary, "/fin">>, + ?assertRCName( + success, + emqtt:publish(C, FinTopic, <<>>, 1) + ), + + ReadyTransferId = #{ + <<"fileid">> => FileId, + <<"clientid">> => ?config(clientid, Config), + <<"node">> => atom_to_binary(node(), utf8) + }, + + {ok, TableQH} = emqx_ft_storage:get_ready_transfer(ReadyTransferId), + + ?assertEqual( + iolist_to_binary(Data), + iolist_to_binary(qlc:eval(TableQH)) + ). + +t_meta_conflict(Config) -> + C = ?config(client, Config), + + Filename = <<"topsecret.pdf">>, + FileId = <<"f1">>, + + Meta = meta(Filename, [<<"x">>]), + MetaPayload = emqx_json:encode(Meta), + + MetaTopic = <<"$file/", FileId/binary, "/init">>, + ?assertRCName( + success, + emqtt:publish(C, MetaTopic, MetaPayload, 1) + ), + + ConflictMeta = Meta#{name => <<"conflict.pdf">>}, + ConflictMetaPayload = emqx_json:encode(ConflictMeta), + + ?assertRCName( + unspecified_error, + emqtt:publish(C, MetaTopic, ConflictMetaPayload, 1) + ). + +t_no_meta(Config) -> + C = ?config(client, Config), + + FileId = <<"f1">>, + Data = <<"first">>, + + SegmentTopic = <<"$file/", FileId/binary, "/0">>, + ?assertRCName( + success, + emqtt:publish(C, SegmentTopic, Data, 1) + ), + + FinTopic = <<"$file/", FileId/binary, "/fin">>, + ?assertRCName( + unspecified_error, + emqtt:publish(C, FinTopic, <<>>, 1) + ). + +t_no_segment(Config) -> + C = ?config(client, Config), + + Filename = <<"topsecret.pdf">>, + FileId = <<"f1">>, + + Data = [<<"first">>, <<"second">>, <<"third">>], + + Meta = meta(Filename, Data), + MetaPayload = emqx_json:encode(Meta), + + MetaTopic = <<"$file/", FileId/binary, "/init">>, + ?assertRCName( + success, + emqtt:publish(C, MetaTopic, MetaPayload, 1) + ), + + lists:foreach( + fun({Chunk, Offset}) -> + OffsetBin = integer_to_binary(Offset), + SegmentTopic = <<"$file/", FileId/binary, "/", OffsetBin/binary>>, + ?assertRCName( + success, + emqtt:publish(C, SegmentTopic, Chunk, 1) + ) + end, + %% Skip the first segment + tl(with_offsets(Data)) + ), + + FinTopic = <<"$file/", FileId/binary, "/fin">>, + ?assertRCName( + unspecified_error, + emqtt:publish(C, FinTopic, <<>>, 1) + ). + +t_invalid_meta(Config) -> + C = ?config(client, Config), + + FileId = <<"f1">>, + + MetaTopic = <<"$file/", FileId/binary, "/init">>, + + %% Invalid schema + Meta = #{foo => <<"bar">>}, + MetaPayload = emqx_json:encode(Meta), + ?assertRCName( + unspecified_error, + emqtt:publish(C, MetaTopic, MetaPayload, 1) + ), + + %% Invalid JSON + ?assertRCName( + unspecified_error, + emqtt:publish(C, MetaTopic, <<"{oops;">>, 1) + ). + +t_invalid_checksum(Config) -> + C = ?config(client, Config), + + Filename = <<"topsecret.pdf">>, + FileId = <<"f1">>, + + Data = [<<"first">>, <<"second">>, <<"third">>], + + Meta = meta(Filename, Data), + MetaPayload = emqx_json:encode(Meta#{checksum => sha256hex(<<"invalid">>)}), + + MetaTopic = <<"$file/", FileId/binary, "/init">>, + ?assertRCName( + success, + emqtt:publish(C, MetaTopic, MetaPayload, 1) + ), + + lists:foreach( + fun({Chunk, Offset}) -> + OffsetBin = integer_to_binary(Offset), + SegmentTopic = <<"$file/", FileId/binary, "/", OffsetBin/binary>>, + ?assertRCName( + success, + emqtt:publish(C, SegmentTopic, Chunk, 1) + ) + end, + with_offsets(Data) + ), + + FinTopic = <<"$file/", FileId/binary, "/fin">>, + ?assertRCName( + unspecified_error, + emqtt:publish(C, FinTopic, <<>>, 1) + ). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +with_offsets(Items) -> + {List, _} = lists:mapfoldl( + fun(Item, Offset) -> + {{Item, Offset}, Offset + byte_size(Item)} + end, + 0, + Items + ), + List. + +sha256hex(Data) -> + binary:encode_hex(crypto:hash(sha256, Data)). + +meta(FileName, Data) -> + FullData = iolist_to_binary(Data), + #{ + name => FileName, + checksum => sha256hex(FullData), + expire_at => erlang:system_time(_Unit = second) + 3600, + size => byte_size(FullData) + }. diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index dd3ffedad..c5dbd418f 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/test/emqx_ft_content_gen.erl b/apps/emqx_ft/test/emqx_ft_content_gen.erl index feca78949..bd24d8c94 100644 --- a/apps/emqx_ft/test/emqx_ft_content_gen.erl +++ b/apps/emqx_ft/test/emqx_ft_content_gen.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl index c08986120..9098edcf6 100644 --- a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl index e979d06fc..0ac5d2844 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. From f6a0598f270e3dd907ff3f82efd6e2a9c839d536 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 15 Feb 2023 11:42:27 +0200 Subject: [PATCH 029/197] feat(ft): add file transfer tests --- apps/emqx/test/emqx_common_test_helpers.erl | 7 +- apps/emqx_ft/src/emqx_ft.erl | 16 +- apps/emqx_ft/src/emqx_ft_conf.erl | 4 +- apps/emqx_ft/src/emqx_ft_storage_dummy.erl | 43 ----- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 5 +- .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 6 - apps/emqx_ft/test/emqx_ft_SUITE.erl | 167 +++++++++++++++--- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 65 +++++++ .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 151 ++++++++++++++++ apps/emqx_ft/test/emqx_ft_test_helpers.erl | 77 ++++++++ 10 files changed, 456 insertions(+), 85 deletions(-) delete mode 100644 apps/emqx_ft/src/emqx_ft_storage_dummy.erl create mode 100644 apps/emqx_ft/test/emqx_ft_conf_SUITE.erl create mode 100644 apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl create mode 100644 apps/emqx_ft/test/emqx_ft_test_helpers.erl diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index ecdf8f827..18a3d9f3e 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -63,7 +63,7 @@ ]). -export([ - maybe_fix_gen_rpc/0, + set_gen_rpc_stateless/0, emqx_cluster/1, emqx_cluster/2, start_epmd/0, @@ -617,13 +617,14 @@ ensure_quic_listener(Name, UdpPort, ExtraSettings) -> listener_ports => [{Type :: tcp | ssl | ws | wss, inet:port_number()}] }. --spec maybe_fix_gen_rpc() -> ok. -maybe_fix_gen_rpc() -> +-spec set_gen_rpc_stateless() -> ok. +set_gen_rpc_stateless() -> %% When many tests run in an obscure order, it may occur that %% `gen_rpc` started with its default settings before `emqx_conf`. %% `gen_rpc` and `emqx_conf` have different default `port_discovery` modes, %% so we reinitialize `gen_rpc` explicitly. ok = application:stop(gen_rpc), + ok = application:set_env(gen_rpc, port_discovery, stateless), ok = application:start(gen_rpc). -spec emqx_cluster(cluster_spec()) -> [{shortname(), node_opts()}]. diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index ed4edd33b..2704e1e55 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -175,7 +175,13 @@ on_init(Msg, FileId) -> case emqx_ft_storage:store_filemeta(transfer(Msg, FileId), Meta) of ok -> ?RC_SUCCESS; - {error, _Reason} -> + {error, Reason} -> + ?SLOG(warning, #{ + msg => "store_filemeta_failed", + mqtt_msg => Msg, + file_id => FileId, + reason => Reason + }), ?RC_UNSPECIFIED_ERROR end; {error, Reason} -> @@ -235,7 +241,13 @@ on_fin(PacketId, Msg, FileId, Checksum) -> {ok, _} -> undefined; %% Assembling failed, unregister the packet key - {error, _} -> + {error, Reason} -> + ?SLOG(warning, #{ + msg => "assemble_not_started", + mqtt_msg => Msg, + file_id => FileId, + reason => Reason + }), case emqx_ft_responder:unregister(FinPacketKey) of %% We successfully unregistered the packet key, %% so we can send the error code at once diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index b88fd2532..d56dd8d32 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -50,8 +50,8 @@ unload() -> -spec pre_config_update(list(atom()), emqx_config:update_request(), emqx_config:raw_config()) -> {ok, emqx_config:update_request()} | {error, term()}. -pre_config_update(_, _Req, Config) -> - {ok, Config}. +pre_config_update(_, Req, _Config) -> + {ok, Req}. -spec post_config_update( list(atom()), diff --git a/apps/emqx_ft/src/emqx_ft_storage_dummy.erl b/apps/emqx_ft/src/emqx_ft_storage_dummy.erl deleted file mode 100644 index 4ed8ba487..000000000 --- a/apps/emqx_ft/src/emqx_ft_storage_dummy.erl +++ /dev/null @@ -1,43 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_ft_storage_dummy). - --behaviour(emqx_ft_storage). - --export([ - store_filemeta/3, - store_segment/3, - assemble/3, - ready_transfers/1, - get_ready_transfer/2 -]). - -store_filemeta(_Storage, _Transfer, _Meta) -> - ok. - -store_segment(_Storage, _Transfer, _Segment) -> - ok. - -assemble(_Storage, _Transfer, Callback) -> - Pid = spawn(fun() -> Callback({error, not_implemented}) end), - {ok, Pid}. - -ready_transfers(_Storage) -> - {ok, []}. - -get_ready_transfer(_Storage, _Id) -> - {error, not_implemented}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 01a3e19c2..c6cb09cf2 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -93,7 +93,10 @@ store_filemeta(Storage, Transfer, Meta) -> {ok, Meta} -> _ = touch_file(Filepath), ok; - {ok, _Conflict} -> + {ok, Conflict} -> + ?SLOG(warning, #{ + msg => "filemeta_conflict", transfer => Transfer, new => Meta, old => Conflict + }), % TODO % We won't see conflicts in case of concurrent `store_filemeta` % requests. It's rather odd scenario so it's fine not to worry diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index f69033b4b..e2c4c93d7 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -20,7 +20,6 @@ -export([introduced_in/0]). --export([list/3]). -export([multilist/3]). -export([pread/5]). -export([ready_transfers/1]). @@ -35,11 +34,6 @@ introduced_in() -> "5.0.17". --spec list(node(), transfer(), fragment | result) -> - {ok, [filefrag()]} | {error, term()} | no_return(). -list(Node, Transfer, What) -> - erpc:call(Node, emqx_ft_storage_fs_proxy, list_local, [Transfer, What]). - -spec multilist([node()], transfer(), fragment | result) -> emqx_rpc:erpc_multicall({ok, [filefrag()]} | {error, term()}). multilist(Nodes, Transfer, What) -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index ff7cba403..6a125449f 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -29,35 +29,58 @@ ) ). -all() -> emqx_common_test_helpers:all(?MODULE). +all() -> + [ + {group, single_node}, + {group, cluster} + ]. + +groups() -> + [ + {single_node, [sequence], emqx_common_test_helpers:all(?MODULE) -- [t_switch_node]}, + {cluster, [sequence], [t_switch_node]} + ]. init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], fun set_special_configs/1), - ok = emqx_common_test_helpers:maybe_fix_gen_rpc(), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], set_special_configs(Config)), + ok = emqx_common_test_helpers:set_gen_rpc_stateless(), Config. end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), ok. -set_special_configs(emqx_ft) -> - {ok, _} = emqx:update_config([file_transfer, storage], #{<<"type">> => <<"local">>}), - ok; -set_special_configs(_App) -> - ok. +set_special_configs(Config) -> + fun + (emqx_ft) -> + ok = emqx_config:put([file_transfer, storage], #{ + type => local, root => emqx_ft_test_helpers:ft_root(Config, node()) + }); + (_) -> + ok + end. -init_per_testcase(_Case, Config) -> - _ = file:del_dir_r(filename:join(emqx:data_dir(), "file_transfer")), - ClientId = <<"client">>, +init_per_testcase(Case, Config) -> + ClientId = atom_to_binary(Case), {ok, C} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}]), {ok, _} = emqtt:connect(C), [{client, C}, {clientid, ClientId} | Config]. - end_per_testcase(_Case, Config) -> C = ?config(client, Config), ok = emqtt:stop(C), ok. +init_per_group(cluster, Config) -> + Node = emqx_ft_test_helpers:start_additional_node(Config, test2), + [{additional_node, Node} | Config]; +init_per_group(_Group, Config) -> + Config. + +end_per_group(cluster, Config) -> + ok = emqx_ft_test_helpers:stop_additional_node(Config); +end_per_group(_Group, _Config) -> + ok. + %%-------------------------------------------------------------------- %% Tests %%-------------------------------------------------------------------- @@ -65,15 +88,34 @@ end_per_testcase(_Case, Config) -> t_invalid_topic_format(Config) -> C = ?config(client, Config), - %% TODO: more invalid topics - ?assertRCName( unspecified_error, - emqtt:publish(C, <<"$file/XYZ">>, <<>>, 1) + emqtt:publish(C, <<"$file/fileid">>, <<>>, 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/fileid/">>, <<>>, 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/fileid/offset">>, <<>>, 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/fileid/fin/offset">>, <<>>, 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/">>, <<>>, 1) ), ?assertRCName( unspecified_error, emqtt:publish(C, <<"$file/X/Y/Z">>, <<>>, 1) + ), + %% should not be handled by `emqx_ft` + ?assertRCName( + no_matching_subscribers, + emqtt:publish(C, <<"$file">>, <<>>, 1) ). t_simple_transfer(Config) -> @@ -95,8 +137,7 @@ t_simple_transfer(Config) -> lists:foreach( fun({Chunk, Offset}) -> - OffsetBin = integer_to_binary(Offset), - SegmentTopic = <<"$file/", FileId/binary, "/", OffsetBin/binary>>, + SegmentTopic = <<"$file/", FileId/binary, "/", Offset/binary>>, ?assertRCName( success, emqtt:publish(C, SegmentTopic, Chunk, 1) @@ -111,12 +152,7 @@ t_simple_transfer(Config) -> emqtt:publish(C, FinTopic, <<>>, 1) ), - ReadyTransferId = #{ - <<"fileid">> => FileId, - <<"clientid">> => ?config(clientid, Config), - <<"node">> => atom_to_binary(node(), utf8) - }, - + {ok, [{ReadyTransferId, _}]} = emqx_ft_storage:ready_transfers(), {ok, TableQH} = emqx_ft_storage:get_ready_transfer(ReadyTransferId), ?assertEqual( @@ -184,8 +220,7 @@ t_no_segment(Config) -> lists:foreach( fun({Chunk, Offset}) -> - OffsetBin = integer_to_binary(Offset), - SegmentTopic = <<"$file/", FileId/binary, "/", OffsetBin/binary>>, + SegmentTopic = <<"$file/", FileId/binary, "/", Offset/binary>>, ?assertRCName( success, emqtt:publish(C, SegmentTopic, Chunk, 1) @@ -241,8 +276,7 @@ t_invalid_checksum(Config) -> lists:foreach( fun({Chunk, Offset}) -> - OffsetBin = integer_to_binary(Offset), - SegmentTopic = <<"$file/", FileId/binary, "/", OffsetBin/binary>>, + SegmentTopic = <<"$file/", FileId/binary, "/", Offset/binary>>, ?assertRCName( success, emqtt:publish(C, SegmentTopic, Chunk, 1) @@ -257,6 +291,83 @@ t_invalid_checksum(Config) -> emqtt:publish(C, FinTopic, <<>>, 1) ). +t_switch_node(Config) -> + AdditionalNodePort = emqx_ft_test_helpers:tcp_port(?config(additional_node, Config)), + + ClientId = <<"t_switch_node-migrating_client">>, + + {ok, C1} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, AdditionalNodePort}]), + {ok, _} = emqtt:connect(C1), + + Filename = <<"multinode_upload.txt">>, + FileId = <<"f1">>, + + Data = [<<"first">>, <<"second">>, <<"third">>], + [{Data0, Offset0}, {Data1, Offset1}, {Data2, Offset2}] = with_offsets(Data), + + %% First, publist metadata and the first segment to the additional node + + Meta = meta(Filename, Data), + MetaPayload = emqx_json:encode(Meta), + + MetaTopic = <<"$file/", FileId/binary, "/init">>, + ?assertRCName( + success, + emqtt:publish(C1, MetaTopic, MetaPayload, 1) + ), + ?assertRCName( + success, + emqtt:publish(C1, <<"$file/", FileId/binary, "/", Offset0/binary>>, Data0, 1) + ), + + %% Then, switch the client to the main node + %% and publish the rest of the segments + + ok = emqtt:stop(C1), + {ok, C2} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(C2), + + ?assertRCName( + success, + emqtt:publish(C2, <<"$file/", FileId/binary, "/", Offset1/binary>>, Data1, 1) + ), + ?assertRCName( + success, + emqtt:publish(C2, <<"$file/", FileId/binary, "/", Offset2/binary>>, Data2, 1) + ), + + FinTopic = <<"$file/", FileId/binary, "/fin">>, + ?assertRCName( + success, + emqtt:publish(C2, FinTopic, <<>>, 1) + ), + + ok = emqtt:stop(C2), + + %% Now check consistency of the file + + {ok, ReadyTransfers} = emqx_ft_storage:ready_transfers(), + {ReadyTransferIds, _} = lists:unzip(ReadyTransfers), + [ReadyTransferId] = [Id || #{<<"clientid">> := CId} = Id <- ReadyTransferIds, CId == ClientId], + + {ok, TableQH} = emqx_ft_storage:get_ready_transfer(ReadyTransferId), + + ?assertEqual( + iolist_to_binary(Data), + iolist_to_binary(qlc:eval(TableQH)) + ). + +t_assemble_crash(Config) -> + C = ?config(client, Config), + + meck:new(emqx_ft_storage_fs), + meck:expect(emqx_ft_storage_fs, assemble, fun(_, _, _) -> meck:exception(error, oops) end), + + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/someid/fin">>, <<>>, 1) + ). + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- @@ -264,7 +375,7 @@ t_invalid_checksum(Config) -> with_offsets(Items) -> {List, _} = lists:mapfoldl( fun(Item, Offset) -> - {{Item, Offset}, Offset + byte_size(Item)} + {{Item, integer_to_binary(Offset)}, Offset + byte_size(Item)} end, 0, Items diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl new file mode 100644 index 000000000..cca69796d --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -0,0 +1,65 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_conf_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft]), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok. + +init_per_testcase(_Case, Config) -> + Config. + +end_per_testcase(_Case, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_update_config(_Config) -> + ?assertMatch( + {error, #{kind := validation_error}}, + emqx_conf:update( + [file_transfer], + #{<<"storage">> => #{<<"type">> => <<"unknown">>}}, + #{} + ) + ), + ?assertMatch( + {ok, _}, + emqx_conf:update( + [file_transfer], + #{<<"storage">> => #{<<"type">> => <<"local">>, <<"root">> => <<"/tmp/path">>}}, + #{} + ) + ), + ?assertEqual( + <<"/tmp/path">>, + emqx_config:get([file_transfer, storage, root]) + ). diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl new file mode 100644 index 000000000..20645bcd9 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -0,0 +1,151 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-define(assertInclude(Pattern, List), + ?assert( + lists:any( + fun + (Pattern) -> true; + (_) -> false + end, + List + ) + ) +). + +all() -> + [ + {group, single_node}, + {group, cluster} + ]. + +-define(CLUSTER_CASES, [t_multinode_ready_transfers]). + +groups() -> + [ + {single_node, [sequence], emqx_common_test_helpers:all(?MODULE) -- ?CLUSTER_CASES}, + {cluster, [sequence], ?CLUSTER_CASES} + ]. + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], set_special_configs(Config)), + ok = emqx_common_test_helpers:set_gen_rpc_stateless(), + Config. +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok. + +set_special_configs(Config) -> + fun + (emqx_ft) -> + ok = emqx_config:put([file_transfer, storage], #{ + type => local, root => emqx_ft_test_helpers:ft_root(Config, node()) + }); + (_) -> + ok + end. + +init_per_testcase(Case, Config) -> + [{tc, Case} | Config]. +end_per_testcase(_Case, _Config) -> + ok. + +init_per_group(cluster, Config) -> + Node = emqx_ft_test_helpers:start_additional_node(Config, test2), + [{additional_node, Node} | Config]; +init_per_group(_Group, Config) -> + Config. + +end_per_group(cluster, Config) -> + ok = emqx_ft_test_helpers:stop_additional_node(Config); +end_per_group(_Group, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_invalid_ready_transfer_id(Config) -> + ?assertMatch( + {error, _}, + emqx_ft_storage_fs:get_ready_transfer(storage(Config), #{ + <<"clientid">> => client_id(Config), + <<"fileid">> => <<"fileid">>, + <<"node">> => atom_to_binary('nonexistent@127.0.0.1') + }) + ), + ?assertMatch( + {error, _}, + emqx_ft_storage_fs:get_ready_transfer(storage(Config), #{ + <<"clientid">> => client_id(Config), + <<"fileid">> => <<"fileid">>, + <<"node">> => <<"nonexistent_as_atom@127.0.0.1">> + }) + ), + ?assertMatch( + {error, _}, + emqx_ft_storage_fs:get_ready_transfer(storage(Config), #{ + <<"clientid">> => client_id(Config), + <<"fileid">> => <<"nonexistent_file">>, + <<"node">> => node() + }) + ). + +t_multinode_ready_transfers(Config) -> + Node1 = ?config(additional_node, Config), + ok = emqx_ft_test_helpers:upload_file(<<"c1">>, <<"f1">>, <<"data">>, Node1), + + Node2 = node(), + ok = emqx_ft_test_helpers:upload_file(<<"c2">>, <<"f2">>, <<"data">>, Node2), + + ?assertInclude( + #{<<"clientid">> := <<"c1">>, <<"fileid">> := <<"f1">>}, + ready_transfer_ids(Config) + ), + + ?assertInclude( + #{<<"clientid">> := <<"c2">>, <<"fileid">> := <<"f2">>}, + ready_transfer_ids(Config) + ). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +client_id(Config) -> + atom_to_binary(?config(tc, Config), utf8). + +storage(Config) -> + #{ + type => local, + root => ft_root(Config) + }. + +ft_root(Config) -> + emqx_ft_test_helpers:ft_root(Config, node()). + +ready_transfer_ids(Config) -> + {ok, ReadyTransfers} = emqx_ft_storage_fs:ready_transfers(storage(Config)), + {ReadyTransferIds, _} = lists:unzip(ReadyTransfers), + ReadyTransferIds. diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl new file mode 100644 index 000000000..ca854bda0 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -0,0 +1,77 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_test_helpers). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). + +start_additional_node(Config, Node) -> + SelfNode = node(), + emqx_common_test_helpers:start_slave( + Node, + [ + {apps, [emqx_ft]}, + {join_to, SelfNode}, + {configure_gen_rpc, false}, + {env_handler, fun + (emqx_ft) -> + ok = emqx_config:put([file_transfer, storage], #{ + type => local, root => ft_root(Config, node()) + }); + (_) -> + ok + end} + ] + ). + +stop_additional_node(Config) -> + Node = ?config(additional_node, Config), + ok = rpc:call(Node, ekka, leave, []), + ok = rpc:call(Node, emqx_common_test_helpers, stop_apps, [[emqx_ft]]), + {ok, _} = emqx_common_test_helpers:stop_slave(Node), + ok. + +tcp_port(Node) -> + {_, Port} = rpc:call(Node, emqx_config, get, [[listeners, tcp, default, bind]]), + Port. + +ft_root(Config, Node) -> + filename:join([ + ?config(priv_dir, Config), <<"file_transfer">>, atom_to_binary(Node) + ]). + +upload_file(ClientId, FileId, Data, Node) -> + Port = tcp_port(Node), + + {ok, C1} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, Port}]), + {ok, _} = emqtt:connect(C1), + Meta = #{ + name => FileId, + expire_at => erlang:system_time(_Unit = second) + 3600, + size => byte_size(Data) + }, + MetaPayload = emqx_json:encode(Meta), + + MetaTopic = <<"$file/", FileId/binary, "/init">>, + {ok, _} = emqtt:publish(C1, MetaTopic, MetaPayload, 1), + {ok, _} = emqtt:publish(C1, <<"$file/", FileId/binary, "/0">>, Data, 1), + + FinTopic = <<"$file/", FileId/binary, "/fin">>, + {ok, _} = emqtt:publish(C1, FinTopic, <<>>, 1), + ok = emqtt:stop(C1). From 2cdf486bf499551f2657c0940ae41bb552876809 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 15 Feb 2023 16:02:18 +0300 Subject: [PATCH 030/197] feat(ft): simplify responder mechanism Make responder short-lived process responsible for a single task, and manage them with supervisor + gproc. --- apps/emqx_ft/src/emqx_ft.erl | 88 +++++-------- apps/emqx_ft/src/emqx_ft_responder.erl | 116 ++++++------------ apps/emqx_ft/src/emqx_ft_responder_sup.erl | 48 ++++++++ apps/emqx_ft/src/emqx_ft_sup.erl | 6 +- apps/emqx_ft/test/emqx_ft_responder_SUITE.erl | 65 +++++----- 5 files changed, 148 insertions(+), 175 deletions(-) create mode 100644 apps/emqx_ft/src/emqx_ft_responder_sup.erl diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 2704e1e55..ac419cb36 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -35,7 +35,7 @@ decode_filemeta/1 ]). --export([on_assemble_timeout/1]). +-export([on_assemble/2]). -export_type([ clientid/0, @@ -227,45 +227,25 @@ on_fin(PacketId, Msg, FileId, Checksum) -> }), %% TODO: handle checksum? Do we need it? FinPacketKey = {self(), PacketId}, - _ = - case - emqx_ft_responder:register( - FinPacketKey, fun ?MODULE:on_assemble_timeout/1, ?ASSEMBLE_TIMEOUT - ) - of - %% We have new fin packet - ok -> - Callback = callback(FinPacketKey, FileId), - case assemble(transfer(Msg, FileId), Callback) of - %% Assembling started, packet will be acked by the callback or the responder - {ok, _} -> - undefined; - %% Assembling failed, unregister the packet key - {error, Reason} -> - ?SLOG(warning, #{ - msg => "assemble_not_started", - mqtt_msg => Msg, - file_id => FileId, - reason => Reason - }), - case emqx_ft_responder:unregister(FinPacketKey) of - %% We successfully unregistered the packet key, - %% so we can send the error code at once - ok -> - ?RC_UNSPECIFIED_ERROR; - %% Someone else already unregistered the key, - %% that is, either responder or someone else acked the packet, - %% we do not have to ack - {error, not_found} -> - undefined - end - end; - %% Fin packet already received. - %% Since we are still handling the previous one, - %% we probably have retransmit here - {error, already_registered} -> - undefined - end. + case emqx_ft_responder:start(FinPacketKey, fun ?MODULE:on_assemble/2, ?ASSEMBLE_TIMEOUT) of + %% We have new fin packet + {ok, _} -> + Callback = fun(Result) -> emqx_ft_responder:ack(FinPacketKey, Result) end, + case assemble(transfer(Msg, FileId), Callback) of + %% Assembling started, packet will be acked by the callback or the responder + {ok, _} -> + ok; + %% Assembling failed, ack through the responder + {error, _} = Error -> + emqx_ft_responder:ack(FinPacketKey, Error) + end; + %% Fin packet already received. + %% Since we are still handling the previous one, + %% we probably have retransmit here + {error, {already_started, _}} -> + ok + end, + undefined. assemble(Transfer, Callback) -> try @@ -278,28 +258,20 @@ assemble(Transfer, Callback) -> {error, {internal_error, E}} end. -callback({ChanPid, PacketId} = Key, _FileId) -> - fun(Result) -> - case emqx_ft_responder:unregister(Key) of - ok -> - case Result of - ok -> - erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS}); - {error, _} -> - erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}) - end; - {error, not_found} -> - ok - end - end. - transfer(Msg, FileId) -> ClientId = Msg#message.from, {ClientId, FileId}. -on_assemble_timeout({ChanPid, PacketId}) -> - ?SLOG(warning, #{msg => "on_assemble_timeout", packet_id => PacketId}), - erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}). +on_assemble({ChanPid, PacketId}, Result) -> + ?SLOG(debug, #{msg => "on_assemble", packet_id => PacketId, result => Result}), + case Result of + {ack, ok} -> + erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS}); + {ack, {error, _}} -> + erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}); + timeout -> + erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}) + end. validate(Validations, Fun) -> case do_validate(Validations, []) of diff --git a/apps/emqx_ft/src/emqx_ft_responder.erl b/apps/emqx_ft/src/emqx_ft_responder.erl index 8417053f6..7b9220774 100644 --- a/apps/emqx_ft/src/emqx_ft_responder.erl +++ b/apps/emqx_ft/src/emqx_ft_responder.erl @@ -23,73 +23,50 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). --export([start_link/0]). +%% API +-export([start/3]). +-export([ack/2]). --export([ - register/3, - unregister/1 -]). +%% Supervisor API +-export([start_link/3]). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). --define(SERVER, ?MODULE). --define(TAB, ?MODULE). +-define(REF(Key), {via, gproc, {n, l, {?MODULE, Key}}}). -type key() :: term(). +-type respfun() :: fun(({ack, _Result} | timeout) -> _SideEffect). %%-------------------------------------------------------------------- %% API %% ------------------------------------------------------------------- --spec start_link() -> startlink_ret(). -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). +-spec start(key(), timeout(), respfun()) -> startlink_ret(). +start(Key, RespFun, Timeout) -> + emqx_ft_responder_sup:start_child(Key, RespFun, Timeout). --spec register(Key, DefaultAction, Timeout) -> ok | {error, already_registered} when - Key :: key(), - DefaultAction :: fun((Key) -> any()), - Timeout :: timeout(). -register(Key, DefaultAction, Timeout) -> - case ets:lookup(?TAB, Key) of - [] -> - gen_server:call(?SERVER, {register, Key, DefaultAction, Timeout}); - [{Key, _Action, _Ref}] -> - {error, already_registered} - end. +-spec ack(key(), _Result) -> _Return. +ack(Key, Result) -> + % TODO: it's possible to avoid term copy + gen_server:call(?REF(Key), {ack, Result}, infinity). --spec unregister(Key) -> ok | {error, not_found} when - Key :: key(). -unregister(Key) -> - gen_server:call(?SERVER, {unregister, Key}). +-spec start_link(key(), timeout(), respfun()) -> startlink_ret(). +start_link(Key, RespFun, Timeout) -> + gen_server:start_link(?REF(Key), ?MODULE, {Key, RespFun, Timeout}, []). %%-------------------------------------------------------------------- %% gen_server callbacks %% ------------------------------------------------------------------- -init([]) -> - _ = ets:new(?TAB, [named_table, protected, set, {read_concurrency, true}]), - {ok, #{}}. +init({Key, RespFun, Timeout}) -> + _ = erlang:process_flag(trap_exit, true), + _TRef = erlang:send_after(Timeout, self(), timeout), + {ok, {Key, RespFun}}. -handle_call({register, Key, DefaultAction, Timeout}, _From, State) -> - ?SLOG(warning, #{msg => "register", key => Key, timeout => Timeout}), - case ets:lookup(?TAB, Key) of - [] -> - TRef = erlang:start_timer(Timeout, self(), {timeout, Key}), - true = ets:insert(?TAB, {Key, DefaultAction, TRef}), - {reply, ok, State}; - [{_, _Action, _Ref}] -> - {reply, {error, already_registered}, State} - end; -handle_call({unregister, Key}, _From, State) -> - ?SLOG(warning, #{msg => "unregister", key => Key}), - case ets:lookup(?TAB, Key) of - [] -> - {reply, {error, not_found}, State}; - [{_, _Action, TRef}] -> - _ = erlang:cancel_timer(TRef), - true = ets:delete(?TAB, Key), - {reply, ok, State} - end; +handle_call({ack, Result}, _From, {Key, RespFun}) -> + Ret = apply(RespFun, [Key, {ack, Result}]), + ?tp(ft_responder_ack, #{key => Key, result => Result, return => Ret}), + {stop, {shutdown, Ret}, Ret, undefined}; handle_call(Msg, _From, State) -> ?SLOG(warning, #{msg => "unknown_call", call_msg => Msg}), {reply, {error, unknown_call}, State}. @@ -98,40 +75,17 @@ handle_cast(Msg, State) -> ?SLOG(warning, #{msg => "unknown_cast", cast_msg => Msg}), {noreply, State}. -handle_info({timeout, TRef, {timeout, Key}}, State) -> - case ets:lookup(?TAB, Key) of - [] -> - {noreply, State}; - [{_, Action, TRef}] -> - _ = erlang:cancel_timer(TRef), - true = ets:delete(?TAB, Key), - ok = safe_apply(Action, [Key]), - ?tp(ft_timeout_action_applied, #{key => Key}), - {noreply, State} - end; +handle_info(timeout, {Key, RespFun}) -> + Ret = apply(RespFun, [Key, timeout]), + ?tp(ft_responder_timeout, #{key => Key, return => Ret}), + {stop, {shutdown, Ret}, undefined}; handle_info(Msg, State) -> ?SLOG(warning, #{msg => "unknown_message", info_msg => Msg}), {noreply, State}. -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -terminate(_Reason, _State) -> +terminate(_Reason, undefined) -> + ok; +terminate(Reason, {Key, RespFun}) -> + Ret = apply(RespFun, [Key, timeout]), + ?tp(ft_responder_shutdown, #{key => Key, reason => Reason, return => Ret}), ok. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -safe_apply(Fun, Args) -> - try apply(Fun, Args) of - _ -> ok - catch - Class:Reason:Stacktrace -> - ?SLOG(error, #{ - msg => "safe_apply_failed", - class => Class, - reason => Reason, - stacktrace => Stacktrace - }) - end. diff --git a/apps/emqx_ft/src/emqx_ft_responder_sup.erl b/apps/emqx_ft/src/emqx_ft_responder_sup.erl new file mode 100644 index 000000000..23d4f55fa --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_responder_sup.erl @@ -0,0 +1,48 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_responder_sup). + +-export([start_link/0]). +-export([start_child/3]). + +-behaviour(supervisor). +-export([init/1]). + +-define(SUPERVISOR, ?MODULE). + +%% + +-spec start_link() -> {ok, pid()}. +start_link() -> + supervisor:start_link({local, ?SUPERVISOR}, ?MODULE, []). + +start_child(Key, RespFun, Timeout) -> + supervisor:start_child(?SUPERVISOR, [Key, RespFun, Timeout]). + +-spec init(_) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}. +init(_) -> + Flags = #{ + strategy => simple_one_for_one, + intensity => 100, + period => 100 + }, + ChildSpec = #{ + id => responder, + start => {emqx_ft_responder, start_link, []}, + restart => temporary + }, + {ok, {Flags, [ChildSpec]}}. diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index dfdde3a8a..8d388814c 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -53,12 +53,12 @@ init([]) -> }, Responder = #{ - id => emqx_ft_responder, - start => {emqx_ft_responder, start_link, []}, + id => emqx_ft_responder_sup, + start => {emqx_ft_responder_sup, start_link, []}, restart => permanent, shutdown => infinity, type => worker, - modules => [emqx_ft_responder] + modules => [emqx_ft_responder_sup] }, ChildSpecs = [Responder, AssemblerSup, FileReaderSup], diff --git a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl index 9098edcf6..447d41f11 100644 --- a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl @@ -19,7 +19,6 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). -include_lib("emqx/include/asserts.hrl"). @@ -39,58 +38,58 @@ init_per_testcase(_Case, Config) -> end_per_testcase(_Case, _Config) -> ok. -t_register_unregister(_Config) -> +t_start_ack(_Config) -> Key = <<"test">>, - DefaultAction = fun(_) -> ok end, - ?assertEqual( - ok, - emqx_ft_responder:register(Key, DefaultAction, 1000) + DefaultAction = fun(_Key, {ack, Ref}) -> Ref end, + ?assertMatch( + {ok, _Pid}, + emqx_ft_responder:start(Key, DefaultAction, 1000) ), - ?assertEqual( - {error, already_registered}, - emqx_ft_responder:register(Key, DefaultAction, 1000) + ?assertMatch( + {error, {already_started, _Pid}}, + emqx_ft_responder:start(Key, DefaultAction, 1000) ), + Ref = make_ref(), ?assertEqual( - ok, - emqx_ft_responder:unregister(Key) + Ref, + emqx_ft_responder:ack(Key, Ref) ), - ?assertEqual( - {error, not_found}, - emqx_ft_responder:unregister(Key) + ?assertExit( + {noproc, _}, + emqx_ft_responder:ack(Key, Ref) ). t_timeout(_Config) -> Key = <<"test">>, Self = self(), - DefaultAction = fun(K) -> Self ! {timeout, K} end, - ok = emqx_ft_responder:register(Key, DefaultAction, 20), + DefaultAction = fun(K, timeout) -> Self ! {timeout, K} end, + {ok, _Pid} = emqx_ft_responder:start(Key, DefaultAction, 20), receive {timeout, Key} -> ok after 100 -> ct:fail("emqx_ft_responder not called") end, - ?assertEqual( - {error, not_found}, - emqx_ft_responder:unregister(Key) + ?assertExit( + {noproc, _}, + emqx_ft_responder:ack(Key, oops) ). -t_action_exception(_Config) -> - Key = <<"test">>, - DefaultAction = fun(K) -> error({oops, K}) end, - - ?assertWaitEvent( - emqx_ft_responder:register(Key, DefaultAction, 10), - #{?snk_kind := ft_timeout_action_applied, key := <<"test">>}, - 1000 - ), - ?assertEqual( - {error, not_found}, - emqx_ft_responder:unregister(Key) - ). +% t_action_exception(_Config) -> +% Key = <<"test">>, +% DefaultAction = fun(K) -> error({oops, K}) end, +% ?assertWaitEvent( +% emqx_ft_responder:start(Key, DefaultAction, 10), +% #{?snk_kind := ft_timeout_action_applied, key := <<"test">>}, +% 1000 +% ), +% ?assertEqual( +% {error, not_found}, +% emqx_ft_responder:ack(Key, oops) +% ). t_unknown_msgs(_Config) -> - Pid = whereis(emqx_ft_responder), + {ok, Pid} = emqx_ft_responder:start(make_ref(), fun(_, _) -> ok end, 100), Pid ! {unknown_msg, <<"test">>}, ok = gen_server:cast(Pid, {unknown_msg, <<"test">>}), ?assertEqual( From f896fefa59f9eaeccf3f597c118e1594abc4ce66 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 16 Feb 2023 12:22:13 +0300 Subject: [PATCH 031/197] feat(ft): make storage backend fully async-aware Introduce an ad-hoc concept of tasks that need to be kicked off manually. Rework filesystem backend to accomodate for this change. Adapt responder logic for that "kickoff" protocol. --- apps/emqx_ft/src/emqx_ft.erl | 175 ++++++++++++------ apps/emqx_ft/src/emqx_ft_assembler.erl | 94 ++++------ apps/emqx_ft/src/emqx_ft_assembler_sup.erl | 15 +- apps/emqx_ft/src/emqx_ft_responder.erl | 33 +++- apps/emqx_ft/src/emqx_ft_storage.erl | 28 +-- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 12 +- apps/emqx_ft/test/emqx_ft_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 50 +++-- apps/emqx_ft/test/emqx_ft_responder_SUITE.erl | 6 +- 9 files changed, 240 insertions(+), 175 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index ac419cb36..348916920 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -35,7 +35,7 @@ decode_filemeta/1 ]). --export([on_assemble/2]). +-export([on_complete/4]). -export_type([ clientid/0, @@ -76,7 +76,8 @@ -type segment() :: {offset(), _Content :: binary()}. --define(ASSEMBLE_TIMEOUT, 5000). +-define(STORE_SEGMENT_TIMEOUT, 10000). +-define(ASSEMBLE_TIMEOUT, 60000). %%-------------------------------------------------------------------- %% API for app @@ -143,52 +144,59 @@ on_message_puback(PacketId, #message{topic = Topic} = Msg, _PubRes, _RC) -> on_file_command(PacketId, Msg, FileCommand) -> case string:split(FileCommand, <<"/">>, all) of [FileId, <<"init">>] -> - on_init(Msg, FileId); + on_init(PacketId, Msg, transfer(Msg, FileId)); [FileId, <<"fin">>] -> - on_fin(PacketId, Msg, FileId, undefined); + on_fin(PacketId, Msg, transfer(Msg, FileId), undefined); [FileId, <<"fin">>, Checksum] -> - on_fin(PacketId, Msg, FileId, Checksum); + on_fin(PacketId, Msg, transfer(Msg, FileId), Checksum); [FileId, <<"abort">>] -> - on_abort(Msg, FileId); + on_abort(Msg, transfer(Msg, FileId)); [FileId, OffsetBin] -> validate([{offset, OffsetBin}], fun([Offset]) -> - on_segment(Msg, FileId, Offset, undefined) + on_segment(PacketId, Msg, transfer(Msg, FileId), Offset, undefined) end); [FileId, OffsetBin, ChecksumBin] -> validate([{offset, OffsetBin}, {checksum, ChecksumBin}], fun([Offset, Checksum]) -> - on_segment(Msg, FileId, Offset, Checksum) + on_segment(PacketId, Msg, transfer(Msg, FileId), Offset, Checksum) end); _ -> ?RC_UNSPECIFIED_ERROR end. -on_init(Msg, FileId) -> +on_init(PacketId, Msg, Transfer) -> ?SLOG(info, #{ msg => "on_init", mqtt_msg => Msg, - file_id => FileId + packet_id => PacketId, + transfer => Transfer }), Payload = Msg#message.payload, + PacketKey = {self(), PacketId}, % %% Add validations here case decode_filemeta(Payload) of {ok, Meta} -> - case emqx_ft_storage:store_filemeta(transfer(Msg, FileId), Meta) of - ok -> - ?RC_SUCCESS; - {error, Reason} -> - ?SLOG(warning, #{ - msg => "store_filemeta_failed", - mqtt_msg => Msg, - file_id => FileId, - reason => Reason - }), - ?RC_UNSPECIFIED_ERROR - end; + Callback = fun(Result) -> + ?MODULE:on_complete("store_filemeta", PacketKey, Transfer, Result) + end, + with_responder(PacketKey, Callback, ?STORE_SEGMENT_TIMEOUT, fun() -> + case store_filemeta(Transfer, Meta) of + % Stored, ack through the responder right away + ok -> + emqx_ft_responder:ack(PacketKey, ok); + % Storage operation started, packet will be acked by the responder + {async, Pid} -> + ok = emqx_ft_responder:kickoff(PacketKey, Pid), + ok; + %% Storage operation failed, ack through the responder + {error, _} = Error -> + emqx_ft_responder:ack(PacketKey, Error) + end + end); {error, Reason} -> ?SLOG(error, #{ msg => "on_init: invalid filemeta", mqtt_msg => Msg, - file_id => FileId, + transfer => Transfer, reason => Reason }), ?RC_UNSPECIFIED_ERROR @@ -198,48 +206,69 @@ on_abort(_Msg, _FileId) -> %% TODO ?RC_SUCCESS. -on_segment(Msg, FileId, Offset, Checksum) -> +on_segment(PacketId, Msg, Transfer, Offset, Checksum) -> ?SLOG(info, #{ msg => "on_segment", mqtt_msg => Msg, - file_id => FileId, + packet_id => PacketId, + transfer => Transfer, offset => Offset, checksum => Checksum }), %% TODO: handle checksum Payload = Msg#message.payload, Segment = {Offset, Payload}, + PacketKey = {self(), PacketId}, + Callback = fun(Result) -> + ?MODULE:on_complete("store_segment", PacketKey, Transfer, Result) + end, %% Add offset/checksum validations - case emqx_ft_storage:store_segment(transfer(Msg, FileId), Segment) of - ok -> - ?RC_SUCCESS; - {error, _Reason} -> - ?RC_UNSPECIFIED_ERROR - end. + with_responder(PacketKey, Callback, ?STORE_SEGMENT_TIMEOUT, fun() -> + case store_segment(Transfer, Segment) of + ok -> + emqx_ft_responder:ack(PacketKey, ok); + {async, Pid} -> + ok = emqx_ft_responder:kickoff(PacketKey, Pid), + ok; + {error, _} = Error -> + emqx_ft_responder:ack(PacketKey, Error) + end + end). -on_fin(PacketId, Msg, FileId, Checksum) -> +on_fin(PacketId, Msg, Transfer, Checksum) -> ?SLOG(info, #{ msg => "on_fin", mqtt_msg => Msg, - file_id => FileId, - checksum => Checksum, - packet_id => PacketId + packet_id => PacketId, + transfer => Transfer, + checksum => Checksum }), %% TODO: handle checksum? Do we need it? FinPacketKey = {self(), PacketId}, - case emqx_ft_responder:start(FinPacketKey, fun ?MODULE:on_assemble/2, ?ASSEMBLE_TIMEOUT) of - %% We have new fin packet + Callback = fun(Result) -> + ?MODULE:on_complete("assemble", FinPacketKey, Transfer, Result) + end, + with_responder(FinPacketKey, Callback, ?ASSEMBLE_TIMEOUT, fun() -> + case assemble(Transfer) of + %% Assembling completed, ack through the responder right away + ok -> + emqx_ft_responder:ack(FinPacketKey, ok); + %% Assembling started, packet will be acked by the responder + {async, Pid} -> + ok = emqx_ft_responder:kickoff(FinPacketKey, Pid), + ok; + %% Assembling failed, ack through the responder + {error, _} = Error -> + emqx_ft_responder:ack(FinPacketKey, Error) + end + end). + +with_responder(Key, Callback, Timeout, CriticalSection) -> + case emqx_ft_responder:start(Key, Callback, Timeout) of + %% We have new packet {ok, _} -> - Callback = fun(Result) -> emqx_ft_responder:ack(FinPacketKey, Result) end, - case assemble(transfer(Msg, FileId), Callback) of - %% Assembling started, packet will be acked by the callback or the responder - {ok, _} -> - ok; - %% Assembling failed, ack through the responder - {error, _} = Error -> - emqx_ft_responder:ack(FinPacketKey, Error) - end; - %% Fin packet already received. + CriticalSection(); + %% Packet already received. %% Since we are still handling the previous one, %% we probably have retransmit here {error, {already_started, _}} -> @@ -247,13 +276,35 @@ on_fin(PacketId, Msg, FileId, Checksum) -> end, undefined. -assemble(Transfer, Callback) -> +store_filemeta(Transfer, Segment) -> try - emqx_ft_storage:assemble(Transfer, Callback) + emqx_ft_storage:store_filemeta(Transfer, Segment) catch C:E:S -> - ?SLOG(warning, #{ - msg => "file_assemble_failed", class => C, reason => E, stacktrace => S + ?SLOG(error, #{ + msg => "start_store_filemeta_failed", class => C, reason => E, stacktrace => S + }), + {error, {internal_error, E}} + end. + +store_segment(Transfer, Segment) -> + try + emqx_ft_storage:store_segment(Transfer, Segment) + catch + C:E:S -> + ?SLOG(error, #{ + msg => "start_store_segment_failed", class => C, reason => E, stacktrace => S + }), + {error, {internal_error, E}} + end. + +assemble(Transfer) -> + try + emqx_ft_storage:assemble(Transfer) + catch + C:E:S -> + ?SLOG(error, #{ + msg => "start_assemble_failed", class => C, reason => E, stacktrace => S }), {error, {internal_error, E}} end. @@ -262,14 +313,28 @@ transfer(Msg, FileId) -> ClientId = Msg#message.from, {ClientId, FileId}. -on_assemble({ChanPid, PacketId}, Result) -> - ?SLOG(debug, #{msg => "on_assemble", packet_id => PacketId, result => Result}), +on_complete(Op, {ChanPid, PacketId}, Transfer, Result) -> + ?SLOG(debug, #{ + msg => "on_complete", + operation => Op, + packet_id => PacketId, + transfer => Transfer + }), case Result of - {ack, ok} -> + {Mode, ok} when Mode == ack orelse Mode == down -> erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS}); - {ack, {error, _}} -> + {Mode, {error, _} = Reason} when Mode == ack orelse Mode == down -> + ?SLOG(error, #{ + msg => Op ++ "_failed", + transfer => Transfer, + reason => Reason + }), erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}); timeout -> + ?SLOG(error, #{ + msg => Op ++ "_timed_out", + transfer => Transfer + }), erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}) end. diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 38faebf03..083b4afcc 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -18,36 +18,31 @@ -include_lib("emqx/include/logger.hrl"). --export([start_link/3]). +-export([start_link/2]). -behaviour(gen_statem). -export([callback_mode/0]). -export([init/1]). -% -export([list_local_fragments/3]). -% -export([list_remote_fragments/3]). -% -export([start_assembling/3]). -export([handle_event/4]). -% -export([handle_continue/2]). -% -export([handle_call/3]). -% -export([handle_cast/2]). - -record(st, { storage :: _Storage, transfer :: emqx_ft:transfer(), assembly :: _TODO, file :: {file:filename(), io:device(), term()} | undefined, - hash, - callback :: fun((ok | {error, term()}) -> any()) + hash }). --define(RPC_LIST_TIMEOUT, 1000). --define(RPC_READSEG_TIMEOUT, 5000). +-define(NAME(Transfer), {n, l, {?MODULE, Transfer}}). +-define(REF(Transfer), {via, gproc, ?NAME(Transfer)}). %% -start_link(Storage, Transfer, Callback) -> - gen_statem:start_link(?MODULE, {Storage, Transfer, Callback}, []). +start_link(Storage, Transfer) -> + %% TODO + %% Additional callbacks? They won't survive restarts by the supervisor, which brings a + %% question if we even need to retry with the help of supervisor. + gen_statem:start_link(?REF(Transfer), ?MODULE, {Storage, Transfer}, []). %% @@ -56,16 +51,21 @@ start_link(Storage, Transfer, Callback) -> callback_mode() -> handle_event_function. -init({Storage, Transfer, Callback}) -> +init({Storage, Transfer}) -> St = #st{ storage = Storage, transfer = Transfer, assembly = emqx_ft_assembly:new(), - hash = crypto:hash_init(sha256), - callback = Callback + hash = crypto:hash_init(sha256) }, - {ok, list_local_fragments, St, ?internal([])}. + {ok, idle, St}. +handle_event(info, kickoff, idle, St) -> + % NOTE + % Someone's told us to start the work, which usually means that it has set up a monitor. + % We could wait for this message and handle it at the end of the assembling rather than at + % the beginning, however it would make error handling much more messier. + {next_state, list_local_fragments, St, ?internal([])}; handle_event(internal, _, list_local_fragments, St = #st{assembly = Asm}) -> % TODO: what we do with non-transients errors here (e.g. `eacces`)? {ok, Fragments} = emqx_ft_storage_fs:list(St#st.storage, St#st.transfer, fragment), @@ -76,10 +76,10 @@ handle_event(internal, _, list_local_fragments, St = #st{assembly = Asm}) -> {next_state, start_assembling, NSt, ?internal([])}; {incomplete, _} -> Nodes = mria_mnesia:running_nodes() -- [node()], - {next_state, {list_remote_fragments, Nodes}, NSt, ?internal([])} + {next_state, {list_remote_fragments, Nodes}, NSt, ?internal([])}; % TODO: recovery? - % {error, _} = Reason -> - % {stop, Reason} + {error, _} = Error -> + {stop, {shutdown, Error}} end; handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> % TODO @@ -107,12 +107,14 @@ handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> {next_state, start_assembling, NSt, ?internal([])}; % TODO: retries / recovery? {incomplete, _} = Status -> - {next_state, {failure, {error, Status}}, NSt, ?internal([])} + {stop, {shutdown, {error, Status}}}; + {error, _} = Error -> + {stop, {shutdown, Error}} end; handle_event(internal, _, start_assembling, St = #st{assembly = Asm}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), Coverage = emqx_ft_assembly:coverage(Asm), - % TODO: errors + % TODO: better error handling {ok, Handle} = emqx_ft_storage_fs:open_file(St#st.storage, St#st.transfer, Filemeta), {next_state, {assemble, Coverage}, St#st{file = Handle}, ?internal([])}; handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> @@ -120,50 +122,16 @@ handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % Currently, race is possible between getting segment info from the remote node and % this node garbage collecting the segment itself. % TODO: pipelining - case pread(Node, Segment, St) of - {ok, Content} -> - case emqx_ft_storage_fs:write(St#st.file, Content) of - {ok, NHandle} -> - {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])}; - %% TODO: better error handling - {error, _} = Error -> - {next_state, {failure, Error}, St, ?internal([])} - end; - {error, _} = Error -> - %% TODO: better error handling - {next_state, {failure, Error}, St, ?internal([])} - end; + % TODO: better error handling + {ok, Content} = pread(Node, Segment, St), + {ok, NHandle} = emqx_ft_storage_fs:write(St#st.file, Content), + {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])}; handle_event(internal, _, {assemble, []}, St = #st{}) -> {next_state, complete, St, ?internal([])}; -handle_event(internal, _, complete, St = #st{assembly = Asm, file = Handle, callback = Callback}) -> +handle_event(internal, _, complete, St = #st{assembly = Asm, file = Handle}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), Result = emqx_ft_storage_fs:complete(St#st.storage, St#st.transfer, Filemeta, Handle), - _ = safe_apply(Callback, Result), - {stop, shutdown}; -handle_event(internal, _, {failure, Error}, #st{callback = Callback}) -> - _ = safe_apply(Callback, Error), - {stop, Error}. - -% handle_continue(list_local, St = #st{storage = Storage, transfer = Transfer, assembly = Asm}) -> -% % TODO: what we do with non-transients errors here (e.g. `eacces`)? -% {ok, Fragments} = emqx_ft_storage_fs:list(Storage, Transfer), -% NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(Asm, node(), Fragments)), -% NSt = St#st{assembly = NAsm}, -% case emqx_ft_assembly:status(NAsm) of -% complete -> -% {noreply, NSt, {continue}}; -% {more, _} -> -% error(noimpl); -% {error, _} -> -% error(noimpl) -% end, -% {noreply, St}. - -% handle_call(_Call, _From, St) -> -% {reply, {error, badcall}, St}. - -% handle_cast(_Cast, St) -> -% {noreply, St}. + {stop, {shutdown, Result}}. pread(Node, Segment, St) when Node =:= node() -> emqx_ft_storage_fs:pread(St#st.storage, St#st.transfer, Segment, 0, segsize(Segment)); diff --git a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl index 17aa4d998..34783cbd3 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl @@ -17,7 +17,7 @@ -module(emqx_ft_assembler_sup). -export([start_link/0]). --export([start_child/3]). +-export([ensure_child/2]). -behaviour(supervisor). -export([init/1]). @@ -25,13 +25,18 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). -start_child(Storage, Transfer, Callback) -> +ensure_child(Storage, Transfer) -> Childspec = #{ - id => {Storage, Transfer}, - start => {emqx_ft_assembler, start_link, [Storage, Transfer, Callback]}, + id => Transfer, + start => {emqx_ft_assembler, start_link, [Storage, Transfer]}, restart => temporary }, - supervisor:start_child(?MODULE, Childspec). + case supervisor:start_child(?MODULE, Childspec) of + {ok, Pid} -> + {ok, Pid}; + {error, {already_started, Pid}} -> + {ok, Pid} + end. init(_) -> SupFlags = #{ diff --git a/apps/emqx_ft/src/emqx_ft_responder.erl b/apps/emqx_ft/src/emqx_ft_responder.erl index 7b9220774..fb823c433 100644 --- a/apps/emqx_ft/src/emqx_ft_responder.erl +++ b/apps/emqx_ft/src/emqx_ft_responder.erl @@ -25,6 +25,7 @@ %% API -export([start/3]). +-export([kickoff/2]). -export([ack/2]). %% Supervisor API @@ -35,7 +36,7 @@ -define(REF(Key), {via, gproc, {n, l, {?MODULE, Key}}}). -type key() :: term(). --type respfun() :: fun(({ack, _Result} | timeout) -> _SideEffect). +-type respfun() :: fun(({ack, _Result} | {down, _Result} | timeout) -> _SideEffect). %%-------------------------------------------------------------------- %% API @@ -45,6 +46,10 @@ start(Key, RespFun, Timeout) -> emqx_ft_responder_sup:start_child(Key, RespFun, Timeout). +-spec kickoff(key(), pid()) -> ok. +kickoff(Key, Pid) -> + gen_server:call(?REF(Key), {kickoff, Pid}). + -spec ack(key(), _Result) -> _Return. ack(Key, Result) -> % TODO: it's possible to avoid term copy @@ -63,8 +68,13 @@ init({Key, RespFun, Timeout}) -> _TRef = erlang:send_after(Timeout, self(), timeout), {ok, {Key, RespFun}}. +handle_call({kickoff, Pid}, _From, St) -> + % TODO: more state? + _MRef = erlang:monitor(process, Pid), + _ = Pid ! kickoff, + {reply, ok, St}; handle_call({ack, Result}, _From, {Key, RespFun}) -> - Ret = apply(RespFun, [Key, {ack, Result}]), + Ret = apply(RespFun, [{ack, Result}]), ?tp(ft_responder_ack, #{key => Key, result => Result, return => Ret}), {stop, {shutdown, Ret}, Ret, undefined}; handle_call(Msg, _From, State) -> @@ -76,9 +86,13 @@ handle_cast(Msg, State) -> {noreply, State}. handle_info(timeout, {Key, RespFun}) -> - Ret = apply(RespFun, [Key, timeout]), + Ret = apply(RespFun, [timeout]), ?tp(ft_responder_timeout, #{key => Key, return => Ret}), {stop, {shutdown, Ret}, undefined}; +handle_info({'DOWN', _MRef, process, _Pid, Reason}, {Key, RespFun}) -> + Ret = apply(RespFun, [{down, map_down_reason(Reason)}]), + ?tp(ft_responder_procdown, #{key => Key, reason => Reason, return => Ret}), + {stop, {shutdown, Ret}, undefined}; handle_info(Msg, State) -> ?SLOG(warning, #{msg => "unknown_message", info_msg => Msg}), {noreply, State}. @@ -86,6 +100,17 @@ handle_info(Msg, State) -> terminate(_Reason, undefined) -> ok; terminate(Reason, {Key, RespFun}) -> - Ret = apply(RespFun, [Key, timeout]), + Ret = apply(RespFun, [timeout]), ?tp(ft_responder_shutdown, #{key => Key, reason => Reason, return => Ret}), ok. + +map_down_reason(normal) -> + ok; +map_down_reason(shutdown) -> + ok; +map_down_reason({shutdown, Result}) -> + Result; +map_down_reason(noproc) -> + {error, noproc}; +map_down_reason(Error) -> + {error, {internal_error, Error}}. diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 0dd9d7989..6ca9e9ecb 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -20,7 +20,7 @@ [ store_filemeta/2, store_segment/2, - assemble/2, + assemble/1, ready_transfers/0, get_ready_transfer/1, @@ -43,12 +43,18 @@ %% Behaviour %%-------------------------------------------------------------------- +%% NOTE +%% An async task will wait for a `kickoff` message to start processing, to give some time +%% to set up monitors, etc. Async task will not explicitly report the processing result, +%% you are expected to receive and handle exit reason of the process, which is +%% -type result() :: `{shutdown, ok | {error, _}}`. + -callback store_filemeta(storage(), emqx_ft:transfer(), emqx_ft:filemeta()) -> - ok | {error, term()}. + ok | {async, pid()} | {error, term()}. -callback store_segment(storage(), emqx_ft:transfer(), emqx_ft:segment()) -> - ok | {error, term()}. --callback assemble(storage(), emqx_ft:transfer(), assemble_callback()) -> - {ok, pid()} | {error, term()}. + ok | {async, pid()} | {error, term()}. +-callback assemble(storage(), emqx_ft:transfer()) -> + ok | {async, pid()} | {error, term()}. -callback ready_transfers(storage()) -> {ok, [{ready_transfer_id(), ready_transfer_info()}]} | {error, term()}. -callback get_ready_transfer(storage(), ready_transfer_id()) -> @@ -59,22 +65,22 @@ %%-------------------------------------------------------------------- -spec store_filemeta(emqx_ft:transfer(), emqx_ft:filemeta()) -> - ok | {error, term()}. + ok | {async, pid()} | {error, term()}. store_filemeta(Transfer, FileMeta) -> Mod = mod(), Mod:store_filemeta(storage(), Transfer, FileMeta). -spec store_segment(emqx_ft:transfer(), emqx_ft:segment()) -> - ok | {error, term()}. + ok | {async, pid()} | {error, term()}. store_segment(Transfer, Segment) -> Mod = mod(), Mod:store_segment(storage(), Transfer, Segment). --spec assemble(emqx_ft:transfer(), assemble_callback()) -> - {ok, pid()} | {error, term()}. -assemble(Transfer, Callback) -> +-spec assemble(emqx_ft:transfer()) -> + ok | {async, pid()} | {error, term()}. +assemble(Transfer) -> Mod = mod(), - Mod:assemble(storage(), Transfer, Callback). + Mod:assemble(storage(), Transfer). -spec ready_transfers() -> {ok, [{ready_transfer_id(), ready_transfer_info()}]} | {error, term()}. ready_transfers() -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index c6cb09cf2..ef032a639 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -24,7 +24,7 @@ -export([store_segment/3]). -export([list/3]). -export([pread/5]). --export([assemble/3]). +-export([assemble/2]). -export([transfers/1]). @@ -168,11 +168,13 @@ pread(_Storage, _Transfer, Frag, Offset, Size) -> {error, Reason} end. --spec assemble(storage(), transfer(), fun((ok | {error, term()}) -> any())) -> +-spec assemble(storage(), transfer()) -> % {ok, _Assembler :: pid()} | {error, incomplete} | {error, badrpc} | {error, _TODO}. - {ok, _Assembler :: pid()} | {error, _TODO}. -assemble(Storage, Transfer, Callback) -> - emqx_ft_assembler_sup:start_child(Storage, Transfer, Callback). + {async, _Assembler :: pid()} | {error, _TODO}. +assemble(Storage, Transfer) -> + % TODO: ask cluster if the transfer is already assembled + {ok, Pid} = emqx_ft_assembler_sup:ensure_child(Storage, Transfer), + {async, Pid}. get_ready_transfer(_Storage, ReadyTransferId) -> case parse_ready_transfer_id(ReadyTransferId) of diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 6a125449f..0ca298265 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -361,7 +361,7 @@ t_assemble_crash(Config) -> C = ?config(client, Config), meck:new(emqx_ft_storage_fs), - meck:expect(emqx_ft_storage_fs, assemble, fun(_, _, _) -> meck:exception(error, oops) end), + meck:expect(emqx_ft_storage_fs, assemble, fun(_, _) -> meck:exception(error, oops) end), ?assertRCName( unspecified_error, diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index c5dbd418f..d4b619e43 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -37,7 +37,8 @@ all() -> ]. init_per_suite(Config) -> - Config. + Apps = application:ensure_all_started(gproc), + [{suite_apps, Apps} | Config]. end_per_suite(_Config) -> ok. @@ -83,9 +84,8 @@ t_assemble_empty_transfer(Config) -> ]}, emqx_ft_storage_fs:list(Storage, Transfer, fragment) ), - {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun on_assembly_finished/1), - {ok, Event} = ?block_until(#{?snk_kind := test_assembly_finished}), - ?assertMatch(#{result := ok}, Event), + Status = complete_assemble(Storage, Transfer), + ?assertEqual({shutdown, ok}, Status), ?assertEqual( {ok, <<>>}, % TODO @@ -132,9 +132,8 @@ t_assemble_complete_local_transfer(Config) -> Fragments ), - {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun on_assembly_finished/1), - {ok, Event} = ?block_until(#{?snk_kind := test_assembly_finished}), - ?assertMatch(#{result := ok}, Event), + Status = complete_assemble(Storage, Transfer), + ?assertEqual({shutdown, ok}, Status), AssemblyFilename = mk_assembly_filename(Config, Transfer, Filename), ?assertMatch( @@ -172,37 +171,32 @@ t_assemble_incomplete_transfer(Config) -> expire_at => 42 }, ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta), - Self = self(), - {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun(Result) -> - Self ! {test_assembly_finished, Result} - end), - receive - {test_assembly_finished, Result} -> - ?assertMatch({error, _}, Result) - after 1000 -> - ct:fail("Assembler did not called callback") - end. + Status = complete_assemble(Storage, Transfer), + ?assertMatch({shutdown, {error, _}}, Status). t_assemble_no_meta(Config) -> Storage = storage(Config), Transfer = {?CLIENTID2, ?config(file_id, Config)}, - Self = self(), - {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun(Result) -> - Self ! {test_assembly_finished, Result} - end), + Status = complete_assemble(Storage, Transfer), + ?assertMatch({shutdown, {error, {incomplete, _}}}, Status). + +complete_assemble(Storage, Transfer) -> + complete_assemble(Storage, Transfer, 1000). + +complete_assemble(Storage, Transfer, Timeout) -> + {async, Pid} = emqx_ft_storage_fs:assemble(Storage, Transfer), + MRef = erlang:monitor(process, Pid), + Pid ! kickoff, receive - {test_assembly_finished, Result} -> - ?assertMatch({error, _}, Result) - after 1000 -> - ct:fail("Assembler did not called callback") + {'DOWN', MRef, process, Pid, Result} -> + Result + after Timeout -> + ct:fail("Assembler did not finish in time") end. mk_assembly_filename(Config, {ClientID, FileID}, Filename) -> filename:join([?config(storage_root, Config), ClientID, FileID, result, Filename]). -on_assembly_finished(Result) -> - ?tp(test_assembly_finished, #{result => Result}). - %% t_list_transfers(Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl index 447d41f11..1674d05e4 100644 --- a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl @@ -40,7 +40,7 @@ end_per_testcase(_Case, _Config) -> t_start_ack(_Config) -> Key = <<"test">>, - DefaultAction = fun(_Key, {ack, Ref}) -> Ref end, + DefaultAction = fun({ack, Ref}) -> Ref end, ?assertMatch( {ok, _Pid}, emqx_ft_responder:start(Key, DefaultAction, 1000) @@ -62,7 +62,7 @@ t_start_ack(_Config) -> t_timeout(_Config) -> Key = <<"test">>, Self = self(), - DefaultAction = fun(K, timeout) -> Self ! {timeout, K} end, + DefaultAction = fun(timeout) -> Self ! {timeout, Key} end, {ok, _Pid} = emqx_ft_responder:start(Key, DefaultAction, 20), receive {timeout, Key} -> @@ -89,7 +89,7 @@ t_timeout(_Config) -> % ). t_unknown_msgs(_Config) -> - {ok, Pid} = emqx_ft_responder:start(make_ref(), fun(_, _) -> ok end, 100), + {ok, Pid} = emqx_ft_responder:start(make_ref(), fun(_) -> ok end, 100), Pid ! {unknown_msg, <<"test">>}, ok = gen_server:cast(Pid, {unknown_msg, <<"test">>}), ?assertEqual( From 6d9f780313dbf05ed6f3e6775ff69ec001cd20af Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 16 Feb 2023 18:35:10 +0300 Subject: [PATCH 032/197] chore: remove unused code --- apps/emqx_ft/src/emqx_ft_assembler.erl | 15 --------------- apps/emqx_ft/test/emqx_ft_responder_SUITE.erl | 14 -------------- 2 files changed, 29 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 083b4afcc..8ba7e42d4 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -16,8 +16,6 @@ -module(emqx_ft_assembler). --include_lib("emqx/include/logger.hrl"). - -export([start_link/2]). -behaviour(gen_statem). @@ -142,16 +140,3 @@ pread(Node, Segment, St) -> segsize(#{fragment := {segment, Info}}) -> maps:get(size, Info). - -safe_apply(Callback, Result) -> - try apply(Callback, [Result]) of - _ -> ok - catch - Class:Reason:Stacktrace -> - ?SLOG(error, #{ - msg => "safe_apply_failed", - class => Class, - reason => Reason, - stacktrace => Stacktrace - }) - end. diff --git a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl index 1674d05e4..e447ba03b 100644 --- a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl @@ -20,7 +20,6 @@ -compile(nowarn_export_all). -include_lib("stdlib/include/assert.hrl"). --include_lib("emqx/include/asserts.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). @@ -75,19 +74,6 @@ t_timeout(_Config) -> emqx_ft_responder:ack(Key, oops) ). -% t_action_exception(_Config) -> -% Key = <<"test">>, -% DefaultAction = fun(K) -> error({oops, K}) end, -% ?assertWaitEvent( -% emqx_ft_responder:start(Key, DefaultAction, 10), -% #{?snk_kind := ft_timeout_action_applied, key := <<"test">>}, -% 1000 -% ), -% ?assertEqual( -% {error, not_found}, -% emqx_ft_responder:ack(Key, oops) -% ). - t_unknown_msgs(_Config) -> {ok, Pid} = emqx_ft_responder:start(make_ref(), fun(_) -> ok end, 100), Pid ! {unknown_msg, <<"test">>}, From 16fa55e19aced78067859e9d5a9eabe318eab50b Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 15 Feb 2023 21:49:33 +0200 Subject: [PATCH 033/197] feat(ft): add API tests --- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 177 ++++++++++++++++++ .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 8 +- .../test/emqx_ft_storage_fs_reader_SUITE.erl | 5 +- 3 files changed, 185 insertions(+), 5 deletions(-) create mode 100644 apps/emqx_ft/test/emqx_ft_api_SUITE.erl diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl new file mode 100644 index 000000000..e420dac36 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -0,0 +1,177 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-define(assertInclude(Pattern, List), + ?assert( + lists:any( + fun(El) -> + case El of + Pattern -> true; + _ -> false + end + end, + List + ) + ) +). + +-import(emqx_mgmt_api_test_util, [request/3, uri/1]). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_mgmt_api_test_util:init_suite( + [emqx_conf, emqx_ft], set_special_configs(Config) + ), + ok = emqx_common_test_helpers:set_gen_rpc_stateless(), + Config. +end_per_suite(_Config) -> + ok = emqx_mgmt_api_test_util:end_suite([emqx_ft, emqx_conf]), + ok. + +set_special_configs(Config) -> + fun + (emqx_ft) -> + ok = emqx_config:put([file_transfer, storage], #{ + type => local, root => emqx_ft_test_helpers:ft_root(Config, node()) + }); + (_) -> + ok + end. + +init_per_testcase(Case, Config) -> + [{tc, Case} | Config]. +end_per_testcase(_Case, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_list_ready_transfers(Config) -> + ClientId = client_id(Config), + + ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, <<"data">>, node()), + + {ok, 200, Response} = request(get, uri(["file_transfer", "files"])), + + #{<<"files">> := Files} = emqx_json:decode(Response, [return_maps]), + + ?assertInclude( + #{<<"id">> := #{<<"clientid">> := ClientId, <<"fileid">> := <<"f1">>}}, + Files + ). + +%% This shouldn't happen in real life +%% but we need to test it anyway +t_list_ready_transfers_no_nodes(_Config) -> + _ = meck:new(mria_mnesia, [passthrough]), + _ = meck:expect(mria_mnesia, running_nodes, fun() -> [] end), + + ?assertMatch( + {ok, 503, _}, + request(get, uri(["file_transfer", "files"])) + ). + +t_download_transfer(Config) -> + ClientId = client_id(Config), + + ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, <<"data">>, node()), + + ?assertMatch( + {ok, 503, _}, + request( + get, + uri(["file_transfer", "file"]) ++ + query(#{ + clientid => ClientId, + fileid => <<"f1">> + }) + ) + ), + + ?assertMatch( + {ok, 503, _}, + request( + get, + uri(["file_transfer", "file"]) ++ + query(#{ + clientid => ClientId, + fileid => <<"f1">>, + node => <<"nonode@nohost">> + }) + ) + ), + + ?assertMatch( + {ok, 404, _}, + request( + get, + uri(["file_transfer", "file"]) ++ + query(#{ + clientid => ClientId, + fileid => <<"unknown_file">>, + node => node() + }) + ) + ), + + {ok, 200, Response} = request( + get, + uri(["file_transfer", "file"]) ++ + query(#{ + clientid => ClientId, + fileid => <<"f1">>, + node => node() + }) + ), + + ?assertEqual( + Response, + <<"data">> + ). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +client_id(Config) -> + atom_to_binary(?config(tc, Config), utf8). + +request(Method, Url) -> + request(Method, Url, []). + +query(Params) -> + KVs = lists:map(fun({K, V}) -> uri_encode(K) ++ "=" ++ uri_encode(V) end, maps:to_list(Params)), + "?" ++ string:join(KVs, "&"). + +uri_encode(T) -> + emqx_http_lib:uri_encode(to_list(T)). + +to_list(A) when is_atom(A) -> + atom_to_list(A); +to_list(B) when is_binary(B) -> + binary_to_list(B); +to_list(L) when is_list(L) -> + L. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index 20645bcd9..55f30300f 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -25,9 +25,11 @@ -define(assertInclude(Pattern, List), ?assert( lists:any( - fun - (Pattern) -> true; - (_) -> false + fun(El) -> + case El of + Pattern -> true; + _ -> false + end end, List ) diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl index 0ac5d2844..db15b8660 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl @@ -25,11 +25,12 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_ft]), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft]), + ok = emqx_common_test_helpers:set_gen_rpc_stateless(), Config. end_per_suite(_Config) -> - ok = emqx_common_test_helpers:stop_apps([emqx_ft]), + ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), ok. init_per_testcase(_Case, Config) -> From bc0a15afd7cb58931a7d46b73c032ab264660505 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 16 Feb 2023 00:28:32 +0200 Subject: [PATCH 034/197] feat(ft): fix test for CI --- apps/emqx/include/asserts.hrl | 14 ++++++++++++++ apps/emqx_ft/docker-ct | 0 apps/emqx_ft/test/emqx_ft_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 14 +------------- apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 16 ++-------------- 5 files changed, 18 insertions(+), 28 deletions(-) create mode 100644 apps/emqx_ft/docker-ct diff --git a/apps/emqx/include/asserts.hrl b/apps/emqx/include/asserts.hrl index 0e9f9477a..b3415fdf7 100644 --- a/apps/emqx/include/asserts.hrl +++ b/apps/emqx/include/asserts.hrl @@ -36,3 +36,17 @@ end ) ). + +-define(assertInclude(Pattern, List), + ?assert( + lists:any( + fun(El) -> + case El of + Pattern -> true; + _ -> false + end + end, + List + ) + ) +). diff --git a/apps/emqx_ft/docker-ct b/apps/emqx_ft/docker-ct new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 0ca298265..5b01c33cd 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -71,7 +71,7 @@ end_per_testcase(_Case, Config) -> ok. init_per_group(cluster, Config) -> - Node = emqx_ft_test_helpers:start_additional_node(Config, test2), + Node = emqx_ft_test_helpers:start_additional_node(Config, emqx_ft1), [{additional_node, Node} | Config]; init_per_group(_Group, Config) -> Config. diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index e420dac36..d38301a0f 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -22,19 +22,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). --define(assertInclude(Pattern, List), - ?assert( - lists:any( - fun(El) -> - case El of - Pattern -> true; - _ -> false - end - end, - List - ) - ) -). +-include_lib("emqx/include/asserts.hrl"). -import(emqx_mgmt_api_test_util, [request/3, uri/1]). diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index 55f30300f..d19d9c764 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -22,19 +22,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). --define(assertInclude(Pattern, List), - ?assert( - lists:any( - fun(El) -> - case El of - Pattern -> true; - _ -> false - end - end, - List - ) - ) -). +-include_lib("emqx/include/asserts.hrl"). all() -> [ @@ -74,7 +62,7 @@ end_per_testcase(_Case, _Config) -> ok. init_per_group(cluster, Config) -> - Node = emqx_ft_test_helpers:start_additional_node(Config, test2), + Node = emqx_ft_test_helpers:start_additional_node(Config, emqx_ft_storage_fs1), [{additional_node, Node} | Config]; init_per_group(_Group, Config) -> Config. From c6b3f2c2caaa93e7fb0a05f99adc231427c2729d Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 16 Feb 2023 12:28:40 +0200 Subject: [PATCH 035/197] feat(ft): add schema descriptions --- apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 13 ++++++++++++- apps/emqx_ft/src/emqx_ft_schema.erl | 12 +++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf index 7e057fdf8..576d7d8fe 100644 --- a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -1,6 +1,17 @@ emqx_ft_schema { - local { + storage { + desc { + en: "Storage settings for file transfer." + zh: "文件传输的存储设置。" + } + label: { + en: "Storage settings" + zh: "存储设置" + } + } + + local_type { desc { en: "Use local file system to store uploaded files and temporary data." zh: "使用本地文件系统来存储上传的文件和临时数据。" diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index deb2cae6f..f17c957a9 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -21,7 +21,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). --export([namespace/0, roots/0, fields/1, tags/0]). +-export([namespace/0, roots/0, fields/1, tags/0, desc/1]). -export([schema/1]). @@ -49,7 +49,8 @@ fields(file_transfer) -> {storage, #{ type => hoconsc:union([ hoconsc:ref(?MODULE, local_storage) - ]) + ]), + desc => ?DESC("storage") }} ]; fields(local_storage) -> @@ -58,7 +59,7 @@ fields(local_storage) -> type => local, default => local, required => false, - desc => ?DESC("local") + desc => ?DESC("local_type") }}, {root, #{ type => binary(), @@ -67,6 +68,11 @@ fields(local_storage) -> }} ]. +desc(file_transfer) -> + "File transfer settings"; +desc(local_storage) -> + "File transfer local storage settings". + schema(filemeta) -> #{ roots => [ From 1c61165a9185661127820e40c2bc3e8f31fc9843 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 16 Feb 2023 17:38:54 +0300 Subject: [PATCH 036/197] fix(ft): fix emqx_cm takover --- apps/emqx/src/emqx_cm.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index 766acd4b8..90b279fa4 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -312,6 +312,7 @@ open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> Session1 = emqx_persistent_session:persist( ClientInfo, ConnInfo, Session ), + register_channel(ClientId, Self, ConnInfo), {ok, #{ session => clean_session(Session1), present => true, From 2c9cd1397d5196ba078339faf4731ebdb74e0ea0 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 16 Feb 2023 18:13:14 +0200 Subject: [PATCH 037/197] chore(ft): fix `?tp` calls --- apps/emqx_ft/src/emqx_ft_responder.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_responder.erl b/apps/emqx_ft/src/emqx_ft_responder.erl index fb823c433..923e5ece0 100644 --- a/apps/emqx_ft/src/emqx_ft_responder.erl +++ b/apps/emqx_ft/src/emqx_ft_responder.erl @@ -75,7 +75,7 @@ handle_call({kickoff, Pid}, _From, St) -> {reply, ok, St}; handle_call({ack, Result}, _From, {Key, RespFun}) -> Ret = apply(RespFun, [{ack, Result}]), - ?tp(ft_responder_ack, #{key => Key, result => Result, return => Ret}), + ?tp(debug, ft_responder_ack, #{key => Key, result => Result, return => Ret}), {stop, {shutdown, Ret}, Ret, undefined}; handle_call(Msg, _From, State) -> ?SLOG(warning, #{msg => "unknown_call", call_msg => Msg}), @@ -87,11 +87,11 @@ handle_cast(Msg, State) -> handle_info(timeout, {Key, RespFun}) -> Ret = apply(RespFun, [timeout]), - ?tp(ft_responder_timeout, #{key => Key, return => Ret}), + ?tp(debug, ft_responder_timeout, #{key => Key, return => Ret}), {stop, {shutdown, Ret}, undefined}; handle_info({'DOWN', _MRef, process, _Pid, Reason}, {Key, RespFun}) -> Ret = apply(RespFun, [{down, map_down_reason(Reason)}]), - ?tp(ft_responder_procdown, #{key => Key, reason => Reason, return => Ret}), + ?tp(debug, ft_responder_procdown, #{key => Key, reason => Reason, return => Ret}), {stop, {shutdown, Ret}, undefined}; handle_info(Msg, State) -> ?SLOG(warning, #{msg => "unknown_message", info_msg => Msg}), @@ -101,7 +101,7 @@ terminate(_Reason, undefined) -> ok; terminate(Reason, {Key, RespFun}) -> Ret = apply(RespFun, [timeout]), - ?tp(ft_responder_shutdown, #{key => Key, reason => Reason, return => Ret}), + ?tp(debug, ft_responder_shutdown, #{key => Key, reason => Reason, return => Ret}), ok. map_down_reason(normal) -> From 228bf1a0cea5b11ecf5ffb81aabbe99c6ce3ef54 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 16 Feb 2023 20:32:12 +0200 Subject: [PATCH 038/197] chore(ft): fix typing issues --- apps/emqx_ft/src/emqx_ft.erl | 16 ++++++++-------- apps/emqx_ft/src/emqx_ft_responder.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 3 +-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 348916920..d55d5cad9 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -184,9 +184,9 @@ on_init(PacketId, Msg, Transfer) -> ok -> emqx_ft_responder:ack(PacketKey, ok); % Storage operation started, packet will be acked by the responder - {async, Pid} -> - ok = emqx_ft_responder:kickoff(PacketKey, Pid), - ok; + % {async, Pid} -> + % ok = emqx_ft_responder:kickoff(PacketKey, Pid), + % ok; %% Storage operation failed, ack through the responder {error, _} = Error -> emqx_ft_responder:ack(PacketKey, Error) @@ -227,9 +227,9 @@ on_segment(PacketId, Msg, Transfer, Offset, Checksum) -> case store_segment(Transfer, Segment) of ok -> emqx_ft_responder:ack(PacketKey, ok); - {async, Pid} -> - ok = emqx_ft_responder:kickoff(PacketKey, Pid), - ok; + % {async, Pid} -> + % ok = emqx_ft_responder:kickoff(PacketKey, Pid), + % ok; {error, _} = Error -> emqx_ft_responder:ack(PacketKey, Error) end @@ -251,8 +251,8 @@ on_fin(PacketId, Msg, Transfer, Checksum) -> with_responder(FinPacketKey, Callback, ?ASSEMBLE_TIMEOUT, fun() -> case assemble(Transfer) of %% Assembling completed, ack through the responder right away - ok -> - emqx_ft_responder:ack(FinPacketKey, ok); + % ok -> + % emqx_ft_responder:ack(FinPacketKey, ok); %% Assembling started, packet will be acked by the responder {async, Pid} -> ok = emqx_ft_responder:kickoff(FinPacketKey, Pid), diff --git a/apps/emqx_ft/src/emqx_ft_responder.erl b/apps/emqx_ft/src/emqx_ft_responder.erl index 923e5ece0..cbbfbe687 100644 --- a/apps/emqx_ft/src/emqx_ft_responder.erl +++ b/apps/emqx_ft/src/emqx_ft_responder.erl @@ -42,7 +42,7 @@ %% API %% ------------------------------------------------------------------- --spec start(key(), timeout(), respfun()) -> startlink_ret(). +-spec start(key(), respfun(), timeout()) -> startlink_ret(). start(Key, RespFun, Timeout) -> emqx_ft_responder_sup:start_child(Key, RespFun, Timeout). diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index ef032a639..81fed0f21 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -78,8 +78,7 @@ -define(MANIFEST, "MANIFEST.json"). -define(SEGMENT, "SEG"). -%% TODO --type storage() :: emqx_config:config(). +-type storage() :: emqx_ft_storage:storage(). %% Store manifest in the backing filesystem. %% Atomic operation. From c073914f7570982abcaea7e427058dc4003d431f Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 17 Feb 2023 13:56:22 +0300 Subject: [PATCH 039/197] chore(ft): fix gen_rpc flakyness --- apps/emqx/test/emqx_common_test_helpers.erl | 18 +----------------- apps/emqx_ft/docker-ct | 0 apps/emqx_ft/test/emqx_ft_SUITE.erl | 5 ++--- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 1 + apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 5 ++--- .../test/emqx_ft_storage_fs_reader_SUITE.erl | 5 ++--- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 2 +- 8 files changed, 10 insertions(+), 28 deletions(-) delete mode 100644 apps/emqx_ft/docker-ct diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 18a3d9f3e..5dd587a4e 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -63,7 +63,6 @@ ]). -export([ - set_gen_rpc_stateless/0, emqx_cluster/1, emqx_cluster/2, start_epmd/0, @@ -293,12 +292,7 @@ read_schema_configs(no_schema, _ConfigFile) -> ok; read_schema_configs(Schema, ConfigFile) -> NewConfig = generate_config(Schema, ConfigFile), - lists:foreach( - fun({App, Configs}) -> - [application:set_env(App, Par, Value) || {Par, Value} <- Configs] - end, - NewConfig - ). + application:set_env(NewConfig). generate_config(SchemaModule, ConfigFile) when is_atom(SchemaModule) -> {ok, Conf0} = hocon:load(ConfigFile, #{format => richmap}), @@ -617,16 +611,6 @@ ensure_quic_listener(Name, UdpPort, ExtraSettings) -> listener_ports => [{Type :: tcp | ssl | ws | wss, inet:port_number()}] }. --spec set_gen_rpc_stateless() -> ok. -set_gen_rpc_stateless() -> - %% When many tests run in an obscure order, it may occur that - %% `gen_rpc` started with its default settings before `emqx_conf`. - %% `gen_rpc` and `emqx_conf` have different default `port_discovery` modes, - %% so we reinitialize `gen_rpc` explicitly. - ok = application:stop(gen_rpc), - ok = application:set_env(gen_rpc, port_discovery, stateless), - ok = application:start(gen_rpc). - -spec emqx_cluster(cluster_spec()) -> [{shortname(), node_opts()}]. emqx_cluster(Specs) -> emqx_cluster(Specs, #{}). diff --git a/apps/emqx_ft/docker-ct b/apps/emqx_ft/docker-ct deleted file mode 100644 index e69de29bb..000000000 diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 5b01c33cd..8708f9808 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -42,12 +42,11 @@ groups() -> ]. init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], set_special_configs(Config)), - ok = emqx_common_test_helpers:set_gen_rpc_stateless(), + ok = emqx_common_test_helpers:start_apps([emqx_ft], set_special_configs(Config)), Config. end_per_suite(_Config) -> - ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok = emqx_common_test_helpers:stop_apps([emqx_ft]), ok. set_special_configs(Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index d38301a0f..40b5da58f 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -32,7 +32,7 @@ init_per_suite(Config) -> ok = emqx_mgmt_api_test_util:init_suite( [emqx_conf, emqx_ft], set_special_configs(Config) ), - ok = emqx_common_test_helpers:set_gen_rpc_stateless(), + {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. end_per_suite(_Config) -> ok = emqx_mgmt_api_test_util:end_suite([emqx_ft, emqx_conf]), diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index cca69796d..314b4a5f2 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -26,6 +26,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft]), + {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. end_per_suite(_Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index d19d9c764..cc3da8bc3 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -39,11 +39,10 @@ groups() -> ]. init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], set_special_configs(Config)), - ok = emqx_common_test_helpers:set_gen_rpc_stateless(), + ok = emqx_common_test_helpers:start_apps([emqx_ft], set_special_configs(Config)), Config. end_per_suite(_Config) -> - ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok = emqx_common_test_helpers:stop_apps([emqx_ft]), ok. set_special_configs(Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl index db15b8660..0ac5d2844 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl @@ -25,12 +25,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft]), - ok = emqx_common_test_helpers:set_gen_rpc_stateless(), + ok = emqx_common_test_helpers:start_apps([emqx_ft]), Config. end_per_suite(_Config) -> - ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok = emqx_common_test_helpers:stop_apps([emqx_ft]), ok. init_per_testcase(_Case, Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index ca854bda0..f7d8bc879 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -28,7 +28,7 @@ start_additional_node(Config, Node) -> [ {apps, [emqx_ft]}, {join_to, SelfNode}, - {configure_gen_rpc, false}, + {configure_gen_rpc, true}, {env_handler, fun (emqx_ft) -> ok = emqx_config:put([file_transfer, storage], #{ From 58115715ddbc11bf1a40d749d477424f8f967ed8 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 17 Feb 2023 17:27:38 +0300 Subject: [PATCH 040/197] fix(ft): require final size in `fin` packet Otherwise there are situations when it's not entirely clear if a transfer is really ready to be assembled. Since the `size` field in a filemeta is not required (and rightly so), we need client to tell us the final transfer size at the end of the process. Also synthesize a testcase to show why it's needed. Also worth noting that right now `fin` packets require final size, even if a client already told us the size through filemeta. The latter is regarded as serving informational purposes only (which means that, for example, it might differ from the final size, or some tranfer progress might show >100% somewhere because of that). --- apps/emqx/src/emqx_maybe.erl | 42 ++++ apps/emqx_ft/src/emqx_ft.erl | 26 ++- apps/emqx_ft/src/emqx_ft_assembler.erl | 10 +- apps/emqx_ft/src/emqx_ft_assembler_sup.erl | 6 +- apps/emqx_ft/src/emqx_ft_assembly.erl | 29 ++- apps/emqx_ft/src/emqx_ft_storage.erl | 10 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 8 +- apps/emqx_ft/test/emqx_ft_SUITE.erl | 212 ++++++++++++++++-- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 16 +- .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 15 +- 11 files changed, 295 insertions(+), 81 deletions(-) create mode 100644 apps/emqx/src/emqx_maybe.erl diff --git a/apps/emqx/src/emqx_maybe.erl b/apps/emqx/src/emqx_maybe.erl new file mode 100644 index 000000000..8c60d7ae7 --- /dev/null +++ b/apps/emqx/src/emqx_maybe.erl @@ -0,0 +1,42 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_maybe). + +-include_lib("emqx/include/types.hrl"). + +-export([to_list/1]). +-export([from_list/1]). +-export([apply/2]). + +-spec to_list(maybe(A)) -> [A]. +to_list(undefined) -> + []; +to_list(Term) -> + [Term]. + +-spec from_list([A]) -> maybe(A). +from_list([]) -> + undefined; +from_list([Term]) -> + Term. + +-spec apply(fun((maybe(A)) -> maybe(A)), maybe(A)) -> + maybe(A). +apply(_Fun, undefined) -> + undefined; +apply(Fun, Term) when is_function(Fun) -> + erlang:apply(Fun, [Term]). diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index d55d5cad9..266d38fe8 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -40,6 +40,7 @@ -export_type([ clientid/0, transfer/0, + bytes/0, offset/0, filemeta/0, segment/0 @@ -145,10 +146,11 @@ on_file_command(PacketId, Msg, FileCommand) -> case string:split(FileCommand, <<"/">>, all) of [FileId, <<"init">>] -> on_init(PacketId, Msg, transfer(Msg, FileId)); - [FileId, <<"fin">>] -> - on_fin(PacketId, Msg, transfer(Msg, FileId), undefined); - [FileId, <<"fin">>, Checksum] -> - on_fin(PacketId, Msg, transfer(Msg, FileId), Checksum); + [FileId, <<"fin">>, FinalSizeBin | ChecksumL] -> + validate([{size, FinalSizeBin}], fun([FinalSize]) -> + Checksum = emqx_maybe:from_list(ChecksumL), + on_fin(PacketId, Msg, transfer(Msg, FileId), FinalSize, Checksum) + end); [FileId, <<"abort">>] -> on_abort(Msg, transfer(Msg, FileId)); [FileId, OffsetBin] -> @@ -235,12 +237,13 @@ on_segment(PacketId, Msg, Transfer, Offset, Checksum) -> end end). -on_fin(PacketId, Msg, Transfer, Checksum) -> +on_fin(PacketId, Msg, Transfer, FinalSize, Checksum) -> ?SLOG(info, #{ msg => "on_fin", mqtt_msg => Msg, packet_id => PacketId, transfer => Transfer, + final_size => FinalSize, checksum => Checksum }), %% TODO: handle checksum? Do we need it? @@ -249,7 +252,7 @@ on_fin(PacketId, Msg, Transfer, Checksum) -> ?MODULE:on_complete("assemble", FinPacketKey, Transfer, Result) end, with_responder(FinPacketKey, Callback, ?ASSEMBLE_TIMEOUT, fun() -> - case assemble(Transfer) of + case assemble(Transfer, FinalSize) of %% Assembling completed, ack through the responder right away % ok -> % emqx_ft_responder:ack(FinPacketKey, ok); @@ -298,9 +301,9 @@ store_segment(Transfer, Segment) -> {error, {internal_error, E}} end. -assemble(Transfer) -> +assemble(Transfer, FinalSize) -> try - emqx_ft_storage:assemble(Transfer) + emqx_ft_storage:assemble(Transfer, FinalSize) catch C:E:S -> ?SLOG(error, #{ @@ -359,6 +362,13 @@ do_validate([{offset, Offset} | Rest], Parsed) -> _ -> {error, {invalid_offset, Offset}} end; +do_validate([{size, Size} | Rest], Parsed) -> + case string:to_integer(Size) of + {Int, <<>>} -> + do_validate(Rest, [Int | Parsed]); + _ -> + {error, {invalid_size, Size}} + end; do_validate([{checksum, Checksum} | Rest], Parsed) -> case parse_checksum(Checksum) of {ok, Bin} -> diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 8ba7e42d4..275d16499 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -16,7 +16,7 @@ -module(emqx_ft_assembler). --export([start_link/2]). +-export([start_link/3]). -behaviour(gen_statem). -export([callback_mode/0]). @@ -36,11 +36,11 @@ %% -start_link(Storage, Transfer) -> +start_link(Storage, Transfer, Size) -> %% TODO %% Additional callbacks? They won't survive restarts by the supervisor, which brings a %% question if we even need to retry with the help of supervisor. - gen_statem:start_link(?REF(Transfer), ?MODULE, {Storage, Transfer}, []). + gen_statem:start_link(?REF(Transfer), ?MODULE, {Storage, Transfer, Size}, []). %% @@ -49,11 +49,11 @@ start_link(Storage, Transfer) -> callback_mode() -> handle_event_function. -init({Storage, Transfer}) -> +init({Storage, Transfer, Size}) -> St = #st{ storage = Storage, transfer = Transfer, - assembly = emqx_ft_assembly:new(), + assembly = emqx_ft_assembly:new(Size), hash = crypto:hash_init(sha256) }, {ok, idle, St}. diff --git a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl index 34783cbd3..bdefdac47 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl @@ -17,7 +17,7 @@ -module(emqx_ft_assembler_sup). -export([start_link/0]). --export([ensure_child/2]). +-export([ensure_child/3]). -behaviour(supervisor). -export([init/1]). @@ -25,10 +25,10 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). -ensure_child(Storage, Transfer) -> +ensure_child(Storage, Transfer, Size) -> Childspec = #{ id => Transfer, - start => {emqx_ft_assembler, start_link, [Storage, Transfer]}, + start => {emqx_ft_assembler, start_link, [Storage, Transfer, Size]}, restart => temporary }, case supervisor:start_child(?MODULE, Childspec) of diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index 0f2729f42..2d78b540b 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -16,7 +16,7 @@ -module(emqx_ft_assembly). --export([new/0]). +-export([new/1]). -export([append/3]). -export([update/1]). @@ -35,12 +35,12 @@ size }). -new() -> +new(Size) -> #asm{ status = {incomplete, {missing, filemeta}}, meta = orddict:new(), segs = orddict:new(), - size = 0 + size = Size }. append(Asm, Node, Fragments) when is_list(Fragments) -> @@ -114,8 +114,7 @@ append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) -> % In theory it's possible to have two segments with same offset + size on % different nodes but with differing content. We'd need a checksum to % be able to disambiguate them though. - segs = orddict:store({Offset, locality(Node), -End, Node}, Fragment, Asm#asm.segs), - size = max(End, Asm#asm.size) + segs = orddict:store({Offset, locality(Node), -End, Node}, Fragment, Asm#asm.segs) }. coverage([{{Offset, _, _, _}, _Segment} | Rest], Cursor, Sz) when Offset < Cursor -> @@ -186,7 +185,7 @@ segsize(#{fragment := {segment, Info}}) -> incomplete_new_test() -> ?assertEqual( {incomplete, {missing, filemeta}}, - status(update(new())) + status(update(new(42))) ). incomplete_test() -> @@ -194,7 +193,7 @@ incomplete_test() -> {incomplete, {missing, filemeta}}, status( update( - append(new(), node(), [ + append(new(142), node(), [ segment(p1, 0, 42), segment(p1, 42, 100) ]) @@ -203,13 +202,13 @@ incomplete_test() -> ). consistent_test() -> - Asm1 = append(new(), n1, [filemeta(m1, "blarg")]), + Asm1 = append(new(42), n1, [filemeta(m1, "blarg")]), Asm2 = append(Asm1, n2, [segment(s2, 0, 42)]), Asm3 = append(Asm2, n3, [filemeta(m3, "blarg")]), ?assertMatch({complete, _}, status(meta, Asm3)). inconsistent_test() -> - Asm1 = append(new(), node(), [segment(s1, 0, 42)]), + Asm1 = append(new(42), node(), [segment(s1, 0, 42)]), Asm2 = append(Asm1, n1, [filemeta(m1, "blarg")]), Asm3 = append(Asm2, n2, [segment(s2, 0, 42), filemeta(m1, "blorg")]), Asm4 = append(Asm3, n3, [filemeta(m3, "blarg")]), @@ -231,7 +230,7 @@ simple_coverage_test() -> {Node, segment(n3, 50, 50)}, {Node, segment(n4, 10, 10)} ], - Asm = append_many(new(), Segs), + Asm = append_many(new(100), Segs), ?assertMatch( {complete, [ @@ -256,7 +255,7 @@ redundant_coverage_test() -> {Node, segment(n7, 50, 10)}, {node1, segment(n8, 40, 10)} ], - Asm = append_many(new(), Segs), + Asm = append_many(new(70), Segs), ?assertMatch( {complete, [ @@ -279,7 +278,7 @@ redundant_coverage_prefer_local_test() -> {Node, segment(n5, 30, 10)}, {Node, segment(n6, 20, 10)} ], - Asm = append_many(new(), Segs), + Asm = append_many(new(40), Segs), ?assertMatch( {complete, [ @@ -301,7 +300,7 @@ missing_coverage_test() -> {node2, segment(n4, 50, 50)}, {Node, segment(n5, 40, 60)} ], - Asm = append_many(new(), Segs), + Asm = append_many(new(100), Segs), ?assertEqual( % {incomplete, {missing, {segment, 30, 40}}}, ??? {incomplete, {missing, {segment, 20, 40}}}, @@ -314,7 +313,7 @@ missing_end_coverage_test() -> {Node, segment(n1, 0, 15)}, {node1, segment(n3, 10, 10)} ], - Asm = append_many(new(), Segs), + Asm = append_many(new(20), Segs), ?assertEqual( {incomplete, {missing, {segment, 15, 20}}}, status(coverage, Asm) @@ -328,7 +327,7 @@ missing_coverage_with_redudancy_test() -> {node43, segment(n4, 10, 50)}, {node(), segment(n5, 40, 60)} ], - Asm = append_many(new(), Segs), + Asm = append_many(new(100), Segs), ?assertEqual( % {incomplete, {missing, {segment, 50, 60}}}, ??? {incomplete, {missing, {segment, 20, 40}}}, diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 6ca9e9ecb..7a95a0454 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -20,7 +20,7 @@ [ store_filemeta/2, store_segment/2, - assemble/1, + assemble/2, ready_transfers/0, get_ready_transfer/1, @@ -53,7 +53,7 @@ ok | {async, pid()} | {error, term()}. -callback store_segment(storage(), emqx_ft:transfer(), emqx_ft:segment()) -> ok | {async, pid()} | {error, term()}. --callback assemble(storage(), emqx_ft:transfer()) -> +-callback assemble(storage(), emqx_ft:transfer(), _Size :: emqx_ft:bytes()) -> ok | {async, pid()} | {error, term()}. -callback ready_transfers(storage()) -> {ok, [{ready_transfer_id(), ready_transfer_info()}]} | {error, term()}. @@ -76,11 +76,11 @@ store_segment(Transfer, Segment) -> Mod = mod(), Mod:store_segment(storage(), Transfer, Segment). --spec assemble(emqx_ft:transfer()) -> +-spec assemble(emqx_ft:transfer(), emqx_ft:bytes()) -> ok | {async, pid()} | {error, term()}. -assemble(Transfer) -> +assemble(Transfer, Size) -> Mod = mod(), - Mod:assemble(storage(), Transfer). + Mod:assemble(storage(), Transfer, Size). -spec ready_transfers() -> {ok, [{ready_transfer_id(), ready_transfer_info()}]} | {error, term()}. ready_transfers() -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 81fed0f21..103f0e48d 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -24,7 +24,7 @@ -export([store_segment/3]). -export([list/3]). -export([pread/5]). --export([assemble/2]). +-export([assemble/3]). -export([transfers/1]). @@ -167,12 +167,12 @@ pread(_Storage, _Transfer, Frag, Offset, Size) -> {error, Reason} end. --spec assemble(storage(), transfer()) -> +-spec assemble(storage(), transfer(), emqx_ft:bytes()) -> % {ok, _Assembler :: pid()} | {error, incomplete} | {error, badrpc} | {error, _TODO}. {async, _Assembler :: pid()} | {error, _TODO}. -assemble(Storage, Transfer) -> +assemble(Storage, Transfer, Size) -> % TODO: ask cluster if the transfer is already assembled - {ok, Pid} = emqx_ft_assembler_sup:ensure_child(Storage, Transfer), + {ok, Pid} = emqx_ft_assembler_sup:ensure_child(Storage, Transfer, Size), {async, Pid}. get_ready_transfer(_Storage, ReadyTransferId) -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 8708f9808..8f00a1884 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -37,8 +37,14 @@ all() -> groups() -> [ - {single_node, [sequence], emqx_common_test_helpers:all(?MODULE) -- [t_switch_node]}, - {cluster, [sequence], [t_switch_node]} + {single_node, [sequence], emqx_common_test_helpers:all(?MODULE) -- group_cluster()}, + {cluster, [sequence], group_cluster()} + ]. + +group_cluster() -> + [ + t_switch_node, + t_unreliable_migrating_client ]. init_per_suite(Config) -> @@ -61,25 +67,61 @@ set_special_configs(Config) -> init_per_testcase(Case, Config) -> ClientId = atom_to_binary(Case), - {ok, C} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}]), - {ok, _} = emqtt:connect(C), - [{client, C}, {clientid, ClientId} | Config]. + case ?config(group, Config) of + cluster -> + [{clientid, ClientId} | Config]; + _ -> + {ok, C} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(C), + [{client, C}, {clientid, ClientId} | Config] + end. end_per_testcase(_Case, Config) -> - C = ?config(client, Config), - ok = emqtt:stop(C), + _ = [ok = emqtt:stop(C) || {client, C} <- Config], ok. -init_per_group(cluster, Config) -> - Node = emqx_ft_test_helpers:start_additional_node(Config, emqx_ft1), - [{additional_node, Node} | Config]; -init_per_group(_Group, Config) -> - Config. +init_per_group(Group = cluster, Config) -> + Cluster = mk_cluster_specs(Config), + ct:pal("Starting ~p", [Cluster]), + Nodes = [ + emqx_common_test_helpers:start_slave(Name, Opts#{join_to => node()}) + || {Name, Opts} <- Cluster + ], + [{group, Group}, {cluster_nodes, Nodes} | Config]; +init_per_group(Group, Config) -> + [{group, Group} | Config]. end_per_group(cluster, Config) -> - ok = emqx_ft_test_helpers:stop_additional_node(Config); + ok = lists:foreach( + fun emqx_ft_test_helpers:stop_additional_node/1, + ?config(cluster_nodes, Config) + ); end_per_group(_Group, _Config) -> ok. +mk_cluster_specs(Config) -> + Specs = [ + {core, emqx_ft_SUITE1, #{listener_ports => [{tcp, 2883}]}}, + {core, emqx_ft_SUITE2, #{listener_ports => [{tcp, 3883}]}} + ], + CommOpts = [ + {env, [{emqx, boot_modules, [broker, listeners]}]}, + {apps, [emqx_ft]}, + {conf, [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]]}, + {env_handler, fun + (emqx_ft) -> + ok = emqx_config:put([file_transfer, storage], #{ + type => local, + root => emqx_ft_test_helpers:ft_root(Config, node()) + }); + (_) -> + ok + end} + ], + emqx_common_test_helpers:emqx_cluster( + Specs, + CommOpts + ). + %%-------------------------------------------------------------------- %% Tests %%-------------------------------------------------------------------- @@ -125,7 +167,7 @@ t_simple_transfer(Config) -> Data = [<<"first">>, <<"second">>, <<"third">>], - Meta = meta(Filename, Data), + Meta = #{size := Filesize} = meta(Filename, Data), MetaPayload = emqx_json:encode(Meta), MetaTopic = <<"$file/", FileId/binary, "/init">>, @@ -145,7 +187,7 @@ t_simple_transfer(Config) -> with_offsets(Data) ), - FinTopic = <<"$file/", FileId/binary, "/fin">>, + FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( success, emqtt:publish(C, FinTopic, <<>>, 1) @@ -194,7 +236,7 @@ t_no_meta(Config) -> emqtt:publish(C, SegmentTopic, Data, 1) ), - FinTopic = <<"$file/", FileId/binary, "/fin">>, + FinTopic = <<"$file/", FileId/binary, "/fin/42">>, ?assertRCName( unspecified_error, emqtt:publish(C, FinTopic, <<>>, 1) @@ -208,7 +250,7 @@ t_no_segment(Config) -> Data = [<<"first">>, <<"second">>, <<"third">>], - Meta = meta(Filename, Data), + Meta = #{size := Filesize} = meta(Filename, Data), MetaPayload = emqx_json:encode(Meta), MetaTopic = <<"$file/", FileId/binary, "/init">>, @@ -229,7 +271,7 @@ t_no_segment(Config) -> tl(with_offsets(Data)) ), - FinTopic = <<"$file/", FileId/binary, "/fin">>, + FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( unspecified_error, emqtt:publish(C, FinTopic, <<>>, 1) @@ -264,7 +306,7 @@ t_invalid_checksum(Config) -> Data = [<<"first">>, <<"second">>, <<"third">>], - Meta = meta(Filename, Data), + Meta = #{size := Filesize} = meta(Filename, Data), MetaPayload = emqx_json:encode(Meta#{checksum => sha256hex(<<"invalid">>)}), MetaTopic = <<"$file/", FileId/binary, "/init">>, @@ -284,14 +326,15 @@ t_invalid_checksum(Config) -> with_offsets(Data) ), - FinTopic = <<"$file/", FileId/binary, "/fin">>, + FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( unspecified_error, emqtt:publish(C, FinTopic, <<>>, 1) ). t_switch_node(Config) -> - AdditionalNodePort = emqx_ft_test_helpers:tcp_port(?config(additional_node, Config)), + [Node | _] = ?config(cluster_nodes, Config), + AdditionalNodePort = emqx_ft_test_helpers:tcp_port(Node), ClientId = <<"t_switch_node-migrating_client">>, @@ -306,7 +349,7 @@ t_switch_node(Config) -> %% First, publist metadata and the first segment to the additional node - Meta = meta(Filename, Data), + Meta = #{size := Filesize} = meta(Filename, Data), MetaPayload = emqx_json:encode(Meta), MetaTopic = <<"$file/", FileId/binary, "/init">>, @@ -335,7 +378,7 @@ t_switch_node(Config) -> emqtt:publish(C2, <<"$file/", FileId/binary, "/", Offset2/binary>>, Data2, 1) ), - FinTopic = <<"$file/", FileId/binary, "/fin">>, + FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( success, emqtt:publish(C2, FinTopic, <<>>, 1) @@ -360,13 +403,134 @@ t_assemble_crash(Config) -> C = ?config(client, Config), meck:new(emqx_ft_storage_fs), - meck:expect(emqx_ft_storage_fs, assemble, fun(_, _) -> meck:exception(error, oops) end), + meck:expect(emqx_ft_storage_fs, assemble, fun(_, _, _) -> meck:exception(error, oops) end), ?assertRCName( unspecified_error, emqtt:publish(C, <<"$file/someid/fin">>, <<>>, 1) ). +t_unreliable_migrating_client(Config) -> + NodeSelf = node(), + [Node1, Node2] = ?config(cluster_nodes, Config), + + ClientId = ?config(clientid, Config), + FileId = emqx_guid:to_hexstr(emqx_guid:gen()), + Filename = "migratory-birds-in-southern-hemisphere-2013.pdf", + Filesize = 1000, + Gen = emqx_ft_content_gen:new({{ClientId, FileId}, Filesize}, 16), + Payload = iolist_to_binary(emqx_ft_content_gen:consume(Gen, fun({Chunk, _, _}) -> Chunk end)), + Meta = meta(Filename, Payload), + + Context = #{ + clientid => ClientId, + fileid => FileId, + filesize => Filesize, + payload => Payload + }, + Commands = [ + {fun connect_mqtt_client/2, [NodeSelf]}, + {fun send_filemeta/2, [Meta]}, + {fun send_segment/3, [0, 100]}, + {fun send_segment/3, [100, 100]}, + {fun send_segment/3, [200, 100]}, + {fun stop_mqtt_client/1, []}, + {fun connect_mqtt_client/2, [Node1]}, + {fun connect_mqtt_client/2, [Node2]}, + {fun send_filemeta/2, [Meta]}, + {fun send_segment/3, [0, 200]}, + {fun send_segment/3, [200, 200]}, + {fun send_segment/3, [400, 100]}, + {fun connect_mqtt_client/2, [Node2]}, + {fun send_segment/3, [200, 200]}, + {fun send_segment/3, [400, 200]}, + {fun connect_mqtt_client/2, [Node1]}, + {fun send_segment/3, [400, 200]}, + {fun send_segment/3, [600, eof]}, + {fun send_finish/1, []}, + {fun connect_mqtt_client/2, [NodeSelf]}, + {fun send_finish/1, []} + ], + _Context = run_commands(Commands, Context), + + {ok, ReadyTransfers} = emqx_ft_storage:ready_transfers(), + ReadyTransferIds = + [Id || {#{<<"clientid">> := CId} = Id, _Info} <- ReadyTransfers, CId == ClientId], + + Node1Bin = atom_to_binary(Node1), + NodeSelfBin = atom_to_binary(NodeSelf), + ?assertMatch( + [#{<<"node">> := Node1Bin}, #{<<"node">> := NodeSelfBin}], + lists:sort(ReadyTransferIds) + ), + + [ + begin + {ok, TableQH} = emqx_ft_storage:get_ready_transfer(Id), + ?assertEqual( + Payload, + iolist_to_binary(qlc:eval(TableQH)) + ) + end + || Id <- ReadyTransferIds + ]. + +run_commands(Commands, Context) -> + lists:foldl(fun run_command/2, Context, Commands). + +run_command({Command, Args}, Context) -> + ct:pal("COMMAND ~p ~p", [erlang:fun_info(Command, name), Args]), + erlang:apply(Command, Args ++ [Context]). + +connect_mqtt_client(Node, ContextIn) -> + Context = #{clientid := ClientId} = disown_mqtt_client(ContextIn), + NodePort = emqx_ft_test_helpers:tcp_port(Node), + {ok, Client} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, NodePort}]), + {ok, _} = emqtt:connect(Client), + Context#{client => Client}. + +stop_mqtt_client(Context = #{client := Client}) -> + _ = emqtt:stop(Client), + maps:remove(client, Context). + +disown_mqtt_client(Context = #{client := Client}) -> + _ = erlang:unlink(Client), + maps:remove(client, Context); +disown_mqtt_client(Context = #{}) -> + Context. + +send_filemeta(Meta, Context = #{client := Client, fileid := FileId}) -> + Topic = <<"$file/", FileId/binary, "/init">>, + MetaPayload = emqx_json:encode(Meta), + ?assertRCName( + success, + emqtt:publish(Client, Topic, MetaPayload, 1) + ), + Context. + +send_segment(Offset, Size, Context = #{client := Client, fileid := FileId, payload := Payload}) -> + Topic = <<"$file/", FileId/binary, "/", (integer_to_binary(Offset))/binary>>, + Data = + case Size of + eof -> + binary:part(Payload, Offset, byte_size(Payload) - Offset); + N -> + binary:part(Payload, Offset, N) + end, + ?assertRCName( + success, + emqtt:publish(Client, Topic, Data, 1) + ), + Context. + +send_finish(Context = #{client := Client, fileid := FileId, filesize := Filesize}) -> + Topic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, + ?assertRCName( + success, + emqtt:publish(Client, Topic, <<>>, 1) + ), + Context. + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index d4b619e43..bb8590cc7 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -84,7 +84,7 @@ t_assemble_empty_transfer(Config) -> ]}, emqx_ft_storage_fs:list(Storage, Transfer, fragment) ), - Status = complete_assemble(Storage, Transfer), + Status = complete_assemble(Storage, Transfer, 0), ?assertEqual({shutdown, ok}, Status), ?assertEqual( {ok, <<>>}, @@ -132,7 +132,7 @@ t_assemble_complete_local_transfer(Config) -> Fragments ), - Status = complete_assemble(Storage, Transfer), + Status = complete_assemble(Storage, Transfer, TransferSize), ?assertEqual({shutdown, ok}, Status), AssemblyFilename = mk_assembly_filename(Config, Transfer, Filename), @@ -171,20 +171,20 @@ t_assemble_incomplete_transfer(Config) -> expire_at => 42 }, ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta), - Status = complete_assemble(Storage, Transfer), + Status = complete_assemble(Storage, Transfer, TransferSize), ?assertMatch({shutdown, {error, _}}, Status). t_assemble_no_meta(Config) -> Storage = storage(Config), Transfer = {?CLIENTID2, ?config(file_id, Config)}, - Status = complete_assemble(Storage, Transfer), + Status = complete_assemble(Storage, Transfer, 42), ?assertMatch({shutdown, {error, {incomplete, _}}}, Status). -complete_assemble(Storage, Transfer) -> - complete_assemble(Storage, Transfer, 1000). +complete_assemble(Storage, Transfer, Size) -> + complete_assemble(Storage, Transfer, Size, 1000). -complete_assemble(Storage, Transfer, Timeout) -> - {async, Pid} = emqx_ft_storage_fs:assemble(Storage, Transfer), +complete_assemble(Storage, Transfer, Size, Timeout) -> + {async, Pid} = emqx_ft_storage_fs:assemble(Storage, Transfer, Size), MRef = erlang:monitor(process, Pid), Pid ! kickoff, receive diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index cc3da8bc3..3bda8042c 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -67,7 +67,7 @@ init_per_group(_Group, Config) -> Config. end_per_group(cluster, Config) -> - ok = emqx_ft_test_helpers:stop_additional_node(Config); + ok = emqx_ft_test_helpers:stop_additional_node(?config(additional_node, Config)); end_per_group(_Group, _Config) -> ok. diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index f7d8bc879..956e63553 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -21,13 +21,12 @@ -include_lib("common_test/include/ct.hrl"). -start_additional_node(Config, Node) -> - SelfNode = node(), +start_additional_node(Config, Name) -> emqx_common_test_helpers:start_slave( - Node, + Name, [ {apps, [emqx_ft]}, - {join_to, SelfNode}, + {join_to, node()}, {configure_gen_rpc, true}, {env_handler, fun (emqx_ft) -> @@ -40,8 +39,7 @@ start_additional_node(Config, Node) -> ] ). -stop_additional_node(Config) -> - Node = ?config(additional_node, Config), +stop_additional_node(Node) -> ok = rpc:call(Node, ekka, leave, []), ok = rpc:call(Node, emqx_common_test_helpers, stop_apps, [[emqx_ft]]), {ok, _} = emqx_common_test_helpers:stop_slave(Node), @@ -58,13 +56,14 @@ ft_root(Config, Node) -> upload_file(ClientId, FileId, Data, Node) -> Port = tcp_port(Node), + Size = byte_size(Data), {ok, C1} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, Port}]), {ok, _} = emqtt:connect(C1), Meta = #{ name => FileId, expire_at => erlang:system_time(_Unit = second) + 3600, - size => byte_size(Data) + size => Size }, MetaPayload = emqx_json:encode(Meta), @@ -72,6 +71,6 @@ upload_file(ClientId, FileId, Data, Node) -> {ok, _} = emqtt:publish(C1, MetaTopic, MetaPayload, 1), {ok, _} = emqtt:publish(C1, <<"$file/", FileId/binary, "/0">>, Data, 1), - FinTopic = <<"$file/", FileId/binary, "/fin">>, + FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Size))/binary>>, {ok, _} = emqtt:publish(C1, FinTopic, <<>>, 1), ok = emqtt:stop(C1). From 2cf2a2d95284b05551d48137f9ddd563196f0e6d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 17 Feb 2023 19:32:23 +0300 Subject: [PATCH 041/197] fix(ft): make `fin` packet parser stricter + safer --- apps/emqx_ft/src/emqx_ft.erl | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 266d38fe8..6a6599366 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -146,11 +146,14 @@ on_file_command(PacketId, Msg, FileCommand) -> case string:split(FileCommand, <<"/">>, all) of [FileId, <<"init">>] -> on_init(PacketId, Msg, transfer(Msg, FileId)); - [FileId, <<"fin">>, FinalSizeBin | ChecksumL] -> - validate([{size, FinalSizeBin}], fun([FinalSize]) -> - Checksum = emqx_maybe:from_list(ChecksumL), - on_fin(PacketId, Msg, transfer(Msg, FileId), FinalSize, Checksum) - end); + [FileId, <<"fin">>, FinalSizeBin | MaybeChecksum] when length(MaybeChecksum) =< 1 -> + ChecksumBin = emqx_maybe:from_list(MaybeChecksum), + validate( + [{size, FinalSizeBin}, {{maybe, checksum}, ChecksumBin}], + fun([FinalSize, Checksum]) -> + on_fin(PacketId, Msg, transfer(Msg, FileId), FinalSize, Checksum) + end + ); [FileId, <<"abort">>] -> on_abort(Msg, transfer(Msg, FileId)); [FileId, OffsetBin] -> @@ -375,7 +378,11 @@ do_validate([{checksum, Checksum} | Rest], Parsed) -> do_validate(Rest, [Bin | Parsed]); {error, _Reason} -> {error, {invalid_checksum, Checksum}} - end. + end; +do_validate([{{maybe, _}, undefined} | Rest], Parsed) -> + do_validate(Rest, [undefined | Parsed]); +do_validate([{{maybe, T}, Value} | Rest], Parsed) -> + do_validate([{T, Value} | Rest], Parsed). parse_checksum(Checksum) when is_binary(Checksum) andalso byte_size(Checksum) =:= 64 -> try From 10d4c4305a69bc6ac2c9976d1f1b3aacd55843c5 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 17 Feb 2023 19:45:10 +0300 Subject: [PATCH 042/197] feat(maybe): add basic tests for the new module --- apps/emqx/src/emqx_maybe.erl | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/apps/emqx/src/emqx_maybe.erl b/apps/emqx/src/emqx_maybe.erl index 8c60d7ae7..8da7c7c90 100644 --- a/apps/emqx/src/emqx_maybe.erl +++ b/apps/emqx/src/emqx_maybe.erl @@ -34,9 +34,40 @@ from_list([]) -> from_list([Term]) -> Term. +%% @doc Apply a function to a maybe argument. -spec apply(fun((maybe(A)) -> maybe(A)), maybe(A)) -> maybe(A). apply(_Fun, undefined) -> undefined; apply(Fun, Term) when is_function(Fun) -> erlang:apply(Fun, [Term]). + +%% + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +to_list_test_() -> + [ + ?_assertEqual([], to_list(undefined)), + ?_assertEqual([42], to_list(42)) + ]. + +from_list_test_() -> + [ + ?_assertEqual(undefined, from_list([])), + ?_assertEqual(3.1415, from_list([3.1415])), + ?_assertError(_, from_list([1, 2, 3])) + ]. + +apply_test_() -> + [ + ?_assertEqual(<<"42">>, ?MODULE:apply(fun erlang:integer_to_binary/1, 42)), + ?_assertEqual(undefined, ?MODULE:apply(fun erlang:integer_to_binary/1, undefined)), + ?_assertEqual(undefined, ?MODULE:apply(fun crash/1, undefined)) + ]. + +crash(_) -> + erlang:error(crashed). + +-endif. From a3f47641f3e3f0dfef2ff77899ebf00cb3807c72 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 17 Feb 2023 19:59:06 +0300 Subject: [PATCH 043/197] test(ft): describe the complex test case with comments In order to make the test client model and behavior clearer. --- apps/emqx_ft/test/emqx_ft_SUITE.erl | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 8f00a1884..be3b19a8c 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -429,25 +429,41 @@ t_unreliable_migrating_client(Config) -> payload => Payload }, Commands = [ + % Connect to the broker on the current node {fun connect_mqtt_client/2, [NodeSelf]}, + % Send filemeta and 3 initial segments + % (assuming client chose 100 bytes as a desired segment size) {fun send_filemeta/2, [Meta]}, {fun send_segment/3, [0, 100]}, {fun send_segment/3, [100, 100]}, {fun send_segment/3, [200, 100]}, + % Disconnect the client cleanly {fun stop_mqtt_client/1, []}, + % Connect to the broker on `Node1` {fun connect_mqtt_client/2, [Node1]}, + % Connect to the broker on `Node2` without first disconnecting from `Node1` + % Client forgot the state for some reason and started the transfer again. + % (assuming this is usual for a client on a device that was rebooted) {fun connect_mqtt_client/2, [Node2]}, {fun send_filemeta/2, [Meta]}, + % This time it chose 200 bytes as a segment size {fun send_segment/3, [0, 200]}, {fun send_segment/3, [200, 200]}, + % But now it downscaled back to 100 bytes segments {fun send_segment/3, [400, 100]}, + % Client lost connectivity and reconnected + % (also had last few segments unacked and decided to resend them) {fun connect_mqtt_client/2, [Node2]}, {fun send_segment/3, [200, 200]}, {fun send_segment/3, [400, 200]}, + % Client lost connectivity and reconnected, this time to another node + % (also had last segment unacked and decided to resend it) {fun connect_mqtt_client/2, [Node1]}, {fun send_segment/3, [400, 200]}, {fun send_segment/3, [600, eof]}, {fun send_finish/1, []}, + % Client lost connectivity and reconnected, this time to the current node + % (client had `fin` unacked and decided to resend it) {fun connect_mqtt_client/2, [NodeSelf]}, {fun send_finish/1, []} ], @@ -457,6 +473,9 @@ t_unreliable_migrating_client(Config) -> ReadyTransferIds = [Id || {#{<<"clientid">> := CId} = Id, _Info} <- ReadyTransfers, CId == ClientId], + % NOTE + % The cluster had 2 assemblers running on two different nodes, because client sent `fin` + % twice. This is currently expected, files must be identical anyway. Node1Bin = atom_to_binary(Node1), NodeSelfBin = atom_to_binary(NodeSelf), ?assertMatch( From 7ff253e80d3722bedb7e4dd73515cd48d736f582 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 17 Feb 2023 20:44:36 +0300 Subject: [PATCH 044/197] test(ft): attempt to avoid `gen_rpc` config reset hacks --- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index 40b5da58f..239edb267 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -136,8 +136,8 @@ t_download_transfer(Config) -> ), ?assertEqual( - Response, - <<"data">> + <<"data">>, + Response ). %%-------------------------------------------------------------------- From 2b925aa60bc11c2163c0d49fecce344863c634c4 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Feb 2023 18:03:26 +0300 Subject: [PATCH 045/197] fix(ft): drop unrelated TODO --- apps/emqx_ft/src/emqx_ft_assembler.erl | 3 --- 1 file changed, 3 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 275d16499..441303270 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -37,9 +37,6 @@ %% start_link(Storage, Transfer, Size) -> - %% TODO - %% Additional callbacks? They won't survive restarts by the supervisor, which brings a - %% question if we even need to retry with the help of supervisor. gen_statem:start_link(?REF(Transfer), ?MODULE, {Storage, Transfer, Size}, []). %% From 75070102ece6eb7513a947c6e6368a3b9b64486c Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Feb 2023 18:22:51 +0300 Subject: [PATCH 046/197] fix(ft): improve typespecs --- apps/emqx_ft/src/emqx_ft_assembler.erl | 2 +- apps/emqx_ft/src/emqx_ft_assembly.erl | 46 +++++++++++++++++++++---- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 23 ++++++++----- 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 441303270..9e3ddfcbd 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -26,7 +26,7 @@ -record(st, { storage :: _Storage, transfer :: emqx_ft:transfer(), - assembly :: _TODO, + assembly :: emqx_ft_assembly:t(), file :: {file:filename(), io:device(), term()} | undefined, hash }). diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index 2d78b540b..f0f0026a3 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -25,16 +25,43 @@ -export([coverage/1]). -export([properties/1]). +-export_type([t/0]). + +-type filemeta() :: emqx_ft:filemeta(). +-type filefrag() :: emqx_ft_storage_fs:filefrag(). +-type filefrag(T) :: emqx_ft_storage_fs:filefrag(T). +-type segmentinfo() :: emqx_ft_storage_fs:segmentinfo(). + -record(asm, { - status :: _TODO, - coverage :: _TODO, - properties :: _TODO, - meta :: _TODO, - % orddict:orddict(K, V) - segs :: _TODO, - size + status :: status(), + coverage :: coverage() | undefined, + properties :: properties() | undefined, + meta :: orddict:orddict( + filemeta(), + {node(), filefrag({filemeta, filemeta()})} + ), + segs :: orddict:orddict( + {emqx_ft:offset(), _Locality, _MEnd, node()}, + filefrag({segment, segmentinfo()}) + ), + size :: emqx_ft:bytes() }). +-type status() :: + {incomplete, {missing, _}} + | complete + | {error, {inconsistent, _}}. + +-type coverage() :: [{node(), filefrag({segment, segmentinfo()})}]. + +-type properties() :: #{ + %% Node where "most" of the segments are located. + dominant => node() +}. + +-opaque t() :: #asm{}. + +-spec new(emqx_ft:bytes()) -> t(). new(Size) -> #asm{ status = {incomplete, {missing, filemeta}}, @@ -43,6 +70,7 @@ new(Size) -> size = Size }. +-spec append(t(), node(), filefrag() | [filefrag()]) -> t(). append(Asm, Node, Fragments) when is_list(Fragments) -> lists:foldl(fun(F, AsmIn) -> append(AsmIn, Node, F) end, Asm, Fragments); append(Asm, Node, Fragment = #{fragment := {filemeta, _}}) -> @@ -50,6 +78,7 @@ append(Asm, Node, Fragment = #{fragment := {filemeta, _}}) -> append(Asm, Node, Segment = #{fragment := {segment, _}}) -> append_segmentinfo(Asm, Node, Segment). +-spec update(t()) -> t(). update(Asm) -> case status(meta, Asm) of {complete, _Meta} -> @@ -67,15 +96,18 @@ update(Asm) -> Asm#asm{status = Status} end. +-spec status(t()) -> status(). status(#asm{status = Status}) -> Status. +-spec filemeta(t()) -> filemeta(). filemeta(Asm) -> case status(meta, Asm) of {complete, Meta} -> Meta; _Other -> undefined end. +-spec coverage(t()) -> coverage() | undefined. coverage(#asm{coverage = Coverage}) -> Coverage. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 103f0e48d..d3331a091 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -80,11 +80,18 @@ -type storage() :: emqx_ft_storage:storage(). +-type file_error() :: + file:posix() + %% Filename is incompatible with the backing filesystem. + | badarg + %% System limit (e.g. number of ports) reached. + | system_limit. + %% Store manifest in the backing filesystem. %% Atomic operation. -spec store_filemeta(storage(), transfer(), filemeta()) -> % Quota? Some lower level errors? - ok | {error, conflict} | {error, _TODO}. + ok | {error, conflict} | {error, file_error()}. store_filemeta(Storage, Transfer, Meta) -> % TODO safeguard against bad clientids / fileids. Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], ?MANIFEST), @@ -110,7 +117,7 @@ store_filemeta(Storage, Transfer, Meta) -> -spec store_segment(storage(), transfer(), segment()) -> % Where is the checksum gets verified? Upper level probably. % Quota? Some lower level errors? - ok | {error, _TODO}. + ok | {error, file_error()}. store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], mk_segment_filename(Segment)), write_file_atomic(Storage, Transfer, Filepath, Content). @@ -118,7 +125,8 @@ store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> -spec list(storage(), transfer(), _What :: fragment | result) -> % Some lower level errors? {error, notfound}? % Result will contain zero or only one filemeta. - {ok, [filefrag({filemeta, filemeta()} | {segment, segmentinfo()})]} | {error, _TODO}. + {ok, [filefrag({filemeta, filemeta()} | {segment, segmentinfo()})]} + | {error, file_error()}. list(Storage, Transfer, What) -> Dirname = mk_filedir(Storage, Transfer, get_subdirs_for(What)), case file:list_dir(Dirname) of @@ -146,7 +154,7 @@ get_filefrag_fun_for(result) -> fun mk_result_filefrag/2. -spec pread(storage(), transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> - {ok, _Content :: iodata()} | {error, _TODO}. + {ok, _Content :: iodata()} | {error, eof} | {error, file_error()}. pread(_Storage, _Transfer, Frag, Offset, Size) -> Filepath = maps:get(path, Frag), case file:open(Filepath, [read, raw, binary]) of @@ -168,7 +176,6 @@ pread(_Storage, _Transfer, Frag, Offset, Size) -> end. -spec assemble(storage(), transfer(), emqx_ft:bytes()) -> - % {ok, _Assembler :: pid()} | {error, incomplete} | {error, badrpc} | {error, _TODO}. {async, _Assembler :: pid()} | {error, _TODO}. assemble(Storage, Transfer, Size) -> % TODO: ask cluster if the transfer is already assembled @@ -321,7 +328,7 @@ read_transferinfo(Storage, Transfer, Acc) -> -type handle() :: {file:name(), io:device(), crypto:hash_state()}. -spec open_file(storage(), transfer(), filemeta()) -> - {ok, handle()} | {error, _TODO}. + {ok, handle()} | {error, file_error()}. open_file(Storage, Transfer, Filemeta) -> Filename = maps:get(name, Filemeta), TempFilepath = mk_temp_filepath(Storage, Transfer, Filename), @@ -335,7 +342,7 @@ open_file(Storage, Transfer, Filemeta) -> end. -spec write(handle(), iodata()) -> - {ok, handle()} | {error, _TODO}. + {ok, handle()} | {error, file_error()}. write({Filepath, IoDevice, Ctx}, IoData) -> case file:write(IoDevice, IoData) of ok -> @@ -345,7 +352,7 @@ write({Filepath, IoDevice, Ctx}, IoData) -> end. -spec complete(storage(), transfer(), filemeta(), handle()) -> - ok | {error, {checksum, _Algo, _Computed}} | {error, _TODO}. + ok | {error, {checksum, _Algo, _Computed}} | {error, file_error()}. complete(Storage, Transfer, Filemeta, Handle = {Filepath, IoDevice, Ctx}) -> TargetFilepath = mk_filepath(Storage, Transfer, [?RESULTDIR], maps:get(name, Filemeta)), case verify_checksum(Ctx, Filemeta) of From 93865a79e9ae1f344036ef6aca79737cfb80021a Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Feb 2023 18:43:34 +0300 Subject: [PATCH 047/197] fix(ft): disallow empty fileids --- apps/emqx_ft/src/emqx_ft.erl | 40 ++++++++++++++++++------- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 1 - apps/emqx_ft/test/emqx_ft_SUITE.erl | 7 +++++ 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 6a6599366..baa0126d6 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -143,26 +143,37 @@ on_message_puback(PacketId, #message{topic = Topic} = Msg, _PubRes, _RC) -> %% TODO Move to emqx_ft_mqtt? on_file_command(PacketId, Msg, FileCommand) -> - case string:split(FileCommand, <<"/">>, all) of - [FileId, <<"init">>] -> - on_init(PacketId, Msg, transfer(Msg, FileId)); - [FileId, <<"fin">>, FinalSizeBin | MaybeChecksum] when length(MaybeChecksum) =< 1 -> + case emqx_topic:tokens(FileCommand) of + [FileIdIn | Rest] -> + validate([{fileid, FileIdIn}], fun([FileId]) -> + on_file_command(PacketId, FileId, Msg, Rest) + end); + [] -> + ?RC_UNSPECIFIED_ERROR + end. + +on_file_command(PacketId, FileId, Msg, FileCommand) -> + Transfer = transfer(Msg, FileId), + case FileCommand of + [<<"init">>] -> + on_init(PacketId, Msg, Transfer); + [<<"fin">>, FinalSizeBin | MaybeChecksum] when length(MaybeChecksum) =< 1 -> ChecksumBin = emqx_maybe:from_list(MaybeChecksum), validate( [{size, FinalSizeBin}, {{maybe, checksum}, ChecksumBin}], fun([FinalSize, Checksum]) -> - on_fin(PacketId, Msg, transfer(Msg, FileId), FinalSize, Checksum) + on_fin(PacketId, Msg, Transfer, FinalSize, Checksum) end ); - [FileId, <<"abort">>] -> - on_abort(Msg, transfer(Msg, FileId)); - [FileId, OffsetBin] -> + [<<"abort">>] -> + on_abort(Msg, Transfer); + [OffsetBin] -> validate([{offset, OffsetBin}], fun([Offset]) -> - on_segment(PacketId, Msg, transfer(Msg, FileId), Offset, undefined) + on_segment(PacketId, Msg, Transfer, Offset, undefined) end); - [FileId, OffsetBin, ChecksumBin] -> + [OffsetBin, ChecksumBin] -> validate([{offset, OffsetBin}, {checksum, ChecksumBin}], fun([Offset, Checksum]) -> - on_segment(PacketId, Msg, transfer(Msg, FileId), Offset, Checksum) + on_segment(PacketId, Msg, Transfer, Offset, Checksum) end); _ -> ?RC_UNSPECIFIED_ERROR @@ -358,6 +369,13 @@ validate(Validations, Fun) -> do_validate([], Parsed) -> {ok, lists:reverse(Parsed)}; +do_validate([{fileid, FileId} | Rest], Parsed) -> + case byte_size(FileId) of + S when S > 0 -> + do_validate(Rest, [FileId | Parsed]); + 0 -> + {error, {invalid_fileid, FileId}} + end; do_validate([{offset, Offset} | Rest], Parsed) -> case string:to_integer(Offset) of {Int, <<>>} -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index d3331a091..667ca091a 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -93,7 +93,6 @@ % Quota? Some lower level errors? ok | {error, conflict} | {error, file_error()}. store_filemeta(Storage, Transfer, Meta) -> - % TODO safeguard against bad clientids / fileids. Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], ?MANIFEST), case read_file(Filepath, fun decode_filemeta/1) of {ok, Meta} -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index be3b19a8c..37cc38d4f 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -159,6 +159,13 @@ t_invalid_topic_format(Config) -> emqtt:publish(C, <<"$file">>, <<>>, 1) ). +t_invalid_fileid(Config) -> + C = ?config(client, Config), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file//init">>, <<>>, 1) + ). + t_simple_transfer(Config) -> C = ?config(client, Config), From 5014cc15f4dcc62e80d3ece84c04df057b839ed7 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Feb 2023 18:44:13 +0300 Subject: [PATCH 048/197] test(ft): do not rely on fs backend implementation details --- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index bb8590cc7..a896922d1 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -22,7 +22,6 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). -include_lib("kernel/include/file.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). all() -> [ @@ -86,11 +85,6 @@ t_assemble_empty_transfer(Config) -> ), Status = complete_assemble(Storage, Transfer, 0), ?assertEqual({shutdown, ok}, Status), - ?assertEqual( - {ok, <<>>}, - % TODO - file:read_file(mk_assembly_filename(Config, Transfer, Filename)) - ), {ok, [Result = #{size := Size = 0}]} = emqx_ft_storage_fs:list(Storage, Transfer, result), ?assertEqual( {error, eof}, @@ -135,17 +129,16 @@ t_assemble_complete_local_transfer(Config) -> Status = complete_assemble(Storage, Transfer, TransferSize), ?assertEqual({shutdown, ok}, Status), - AssemblyFilename = mk_assembly_filename(Config, Transfer, Filename), ?assertMatch( {ok, [ #{ - path := AssemblyFilename, size := TransferSize, fragment := {result, #{}} } ]}, emqx_ft_storage_fs:list(Storage, Transfer, result) ), + {ok, [#{path := AssemblyFilename}]} = emqx_ft_storage_fs:list(Storage, Transfer, result), ?assertMatch( {ok, #file_info{type = regular, size = TransferSize}}, file:read_file_info(AssemblyFilename) @@ -194,9 +187,6 @@ complete_assemble(Storage, Transfer, Size, Timeout) -> ct:fail("Assembler did not finish in time") end. -mk_assembly_filename(Config, {ClientID, FileID}, Filename) -> - filename:join([?config(storage_root, Config), ClientID, FileID, result, Filename]). - %% t_list_transfers(Config) -> From 5998961f9f34f7f8d01475f38ba8128c3c7ddf33 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Feb 2023 18:45:07 +0300 Subject: [PATCH 049/197] fix(ft): log errors where they might get lost --- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 31 ++++++++++++++++++++----- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 667ca091a..2cc19d2a2 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -19,6 +19,7 @@ -behaviour(emqx_ft_storage). -include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). -export([store_filemeta/3]). -export([store_segment/3]). @@ -305,7 +306,10 @@ transfers(Storage, ClientId, AccIn) -> FileIds ); {error, _Reason} -> - % TODO worth logging + ?tp(warning, "list_dir_failed", #{ + storage => Storage, + directory => Dirname + }), AccIn end. @@ -318,7 +322,10 @@ read_transferinfo(Storage, Transfer, Acc) -> Info = #{status => incomplete}, Acc#{Transfer => Info}; {error, _Reason} -> - % TODO worth logging + ?tp(warning, "list_result_failed", #{ + storage => Storage, + transfer => Transfer + }), Acc end. @@ -454,8 +461,12 @@ safe_decode(Content, DecodeFun) -> try {ok, DecodeFun(Content)} catch - _C:_R:_Stacktrace -> - % TODO: Log? + C:E:Stacktrace -> + ?tp(warning, "safe_decode_failed", #{ + class => C, + exception => E, + stacktrace => Stacktrace + }), {error, corrupted} end. @@ -509,7 +520,10 @@ mk_filefrag(Dirname, Filename = ?MANIFEST) -> mk_filefrag(Dirname, Filename = ?SEGMENT ++ _) -> mk_filefrag(Dirname, Filename, segment, fun read_segmentinfo/2); mk_filefrag(_Dirname, _Filename) -> - % TODO this is unexpected, worth logging? + ?tp(warning, "rogue_file_found", #{ + directory => _Dirname, + filename => _Filename + }), false. mk_result_filefrag(Dirname, Filename) -> @@ -531,7 +545,12 @@ mk_filefrag(Dirname, Filename, Tag, Fun) -> fragment => {Tag, Frag} }}; {error, _Reason} -> - % TODO loss of information + ?tp(warning, "mk_filefrag_failed", #{ + directory => Dirname, + filename => Filename, + type => Tag, + reason => _Reason + }), false end. From 15d967459c7f3e8092465033afdae29c24ba6ab3 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Feb 2023 20:09:29 +0300 Subject: [PATCH 050/197] feat(ft): add segment checksum validation Also downgrade validation errors to mere info messages. --- apps/emqx_ft/src/emqx_ft.erl | 98 +++++++++-------- apps/emqx_ft/test/emqx_ft_SUITE.erl | 158 ++++++++++++++++++---------- 2 files changed, 157 insertions(+), 99 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index baa0126d6..2ccd5db15 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -20,6 +20,7 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). -export([ hook/0, @@ -156,7 +157,12 @@ on_file_command(PacketId, FileId, Msg, FileCommand) -> Transfer = transfer(Msg, FileId), case FileCommand of [<<"init">>] -> - on_init(PacketId, Msg, Transfer); + validate( + [{filemeta, Msg#message.payload}], + fun([Meta]) -> + on_init(PacketId, Msg, Transfer, Meta) + end + ); [<<"fin">>, FinalSizeBin | MaybeChecksum] when length(MaybeChecksum) =< 1 -> ChecksumBin = emqx_maybe:from_list(MaybeChecksum), validate( @@ -172,51 +178,47 @@ on_file_command(PacketId, FileId, Msg, FileCommand) -> on_segment(PacketId, Msg, Transfer, Offset, undefined) end); [OffsetBin, ChecksumBin] -> - validate([{offset, OffsetBin}, {checksum, ChecksumBin}], fun([Offset, Checksum]) -> - on_segment(PacketId, Msg, Transfer, Offset, Checksum) - end); + validate( + [{offset, OffsetBin}, {checksum, ChecksumBin}], + fun([Offset, Checksum]) -> + validate( + [{integrity, Msg#message.payload, Checksum}], + fun(_) -> + on_segment(PacketId, Msg, Transfer, Offset, Checksum) + end + ) + end + ); _ -> ?RC_UNSPECIFIED_ERROR end. -on_init(PacketId, Msg, Transfer) -> +on_init(PacketId, Msg, Transfer, Meta) -> ?SLOG(info, #{ msg => "on_init", mqtt_msg => Msg, packet_id => PacketId, - transfer => Transfer + transfer => Transfer, + filemeta => Meta }), - Payload = Msg#message.payload, PacketKey = {self(), PacketId}, - % %% Add validations here - case decode_filemeta(Payload) of - {ok, Meta} -> - Callback = fun(Result) -> - ?MODULE:on_complete("store_filemeta", PacketKey, Transfer, Result) - end, - with_responder(PacketKey, Callback, ?STORE_SEGMENT_TIMEOUT, fun() -> - case store_filemeta(Transfer, Meta) of - % Stored, ack through the responder right away - ok -> - emqx_ft_responder:ack(PacketKey, ok); - % Storage operation started, packet will be acked by the responder - % {async, Pid} -> - % ok = emqx_ft_responder:kickoff(PacketKey, Pid), - % ok; - %% Storage operation failed, ack through the responder - {error, _} = Error -> - emqx_ft_responder:ack(PacketKey, Error) - end - end); - {error, Reason} -> - ?SLOG(error, #{ - msg => "on_init: invalid filemeta", - mqtt_msg => Msg, - transfer => Transfer, - reason => Reason - }), - ?RC_UNSPECIFIED_ERROR - end. + Callback = fun(Result) -> + ?MODULE:on_complete("store_filemeta", PacketKey, Transfer, Result) + end, + with_responder(PacketKey, Callback, ?STORE_SEGMENT_TIMEOUT, fun() -> + case store_filemeta(Transfer, Meta) of + % Stored, ack through the responder right away + ok -> + emqx_ft_responder:ack(PacketKey, ok); + % Storage operation started, packet will be acked by the responder + % {async, Pid} -> + % ok = emqx_ft_responder:kickoff(PacketKey, Pid), + % ok; + %% Storage operation failed, ack through the responder + {error, _} = Error -> + emqx_ft_responder:ack(PacketKey, Error) + end + end). on_abort(_Msg, _FileId) -> %% TODO @@ -231,14 +233,11 @@ on_segment(PacketId, Msg, Transfer, Offset, Checksum) -> offset => Offset, checksum => Checksum }), - %% TODO: handle checksum - Payload = Msg#message.payload, - Segment = {Offset, Payload}, + Segment = {Offset, Msg#message.payload}, PacketKey = {self(), PacketId}, Callback = fun(Result) -> ?MODULE:on_complete("store_segment", PacketKey, Transfer, Result) end, - %% Add offset/checksum validations with_responder(PacketKey, Callback, ?STORE_SEGMENT_TIMEOUT, fun() -> case store_segment(Transfer, Segment) of ok -> @@ -360,10 +359,7 @@ validate(Validations, Fun) -> {ok, Parsed} -> Fun(Parsed); {error, Reason} -> - ?SLOG(error, #{ - msg => "validate: invalid $file command", - reason => Reason - }), + ?tp(info, "client_violated_protocol", #{reason => Reason}), ?RC_UNSPECIFIED_ERROR end. @@ -376,6 +372,13 @@ do_validate([{fileid, FileId} | Rest], Parsed) -> 0 -> {error, {invalid_fileid, FileId}} end; +do_validate([{filemeta, Payload} | Rest], Parsed) -> + case decode_filemeta(Payload) of + {ok, Meta} -> + do_validate(Rest, [Meta | Parsed]); + {error, Reason} -> + {error, {invalid_filemeta, Reason}} + end; do_validate([{offset, Offset} | Rest], Parsed) -> case string:to_integer(Offset) of {Int, <<>>} -> @@ -397,6 +400,13 @@ do_validate([{checksum, Checksum} | Rest], Parsed) -> {error, _Reason} -> {error, {invalid_checksum, Checksum}} end; +do_validate([{integrity, Payload, Checksum} | Rest], Parsed) -> + case crypto:hash(sha256, Payload) of + Checksum -> + do_validate(Rest, [Payload | Parsed]); + Mismatch -> + {error, {checksum_mismatch, binary:encode_hex(Mismatch)}} + end; do_validate([{{maybe, _}, undefined} | Rest], Parsed) -> do_validate(Rest, [undefined | Parsed]); do_validate([{{maybe, T}, Value} | Rest], Parsed) -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 37cc38d4f..bb67d3c56 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -37,8 +37,8 @@ all() -> groups() -> [ - {single_node, [sequence], emqx_common_test_helpers:all(?MODULE) -- group_cluster()}, - {cluster, [sequence], group_cluster()} + {single_node, [], emqx_common_test_helpers:all(?MODULE) -- group_cluster()}, + {cluster, [], group_cluster()} ]. group_cluster() -> @@ -177,32 +177,28 @@ t_simple_transfer(Config) -> Meta = #{size := Filesize} = meta(Filename, Data), MetaPayload = emqx_json:encode(Meta), - MetaTopic = <<"$file/", FileId/binary, "/init">>, ?assertRCName( success, - emqtt:publish(C, MetaTopic, MetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) ), lists:foreach( fun({Chunk, Offset}) -> - SegmentTopic = <<"$file/", FileId/binary, "/", Offset/binary>>, ?assertRCName( success, - emqtt:publish(C, SegmentTopic, Chunk, 1) + emqtt:publish(C, mk_segment_topic(FileId, Offset), Chunk, 1) ) end, with_offsets(Data) ), - FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( success, - emqtt:publish(C, FinTopic, <<>>, 1) + emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) ), - {ok, [{ReadyTransferId, _}]} = emqx_ft_storage:ready_transfers(), + [ReadyTransferId] = list_ready_transfers(?config(clientid, Config)), {ok, TableQH} = emqx_ft_storage:get_ready_transfer(ReadyTransferId), - ?assertEqual( iolist_to_binary(Data), iolist_to_binary(qlc:eval(TableQH)) @@ -217,10 +213,9 @@ t_meta_conflict(Config) -> Meta = meta(Filename, [<<"x">>]), MetaPayload = emqx_json:encode(Meta), - MetaTopic = <<"$file/", FileId/binary, "/init">>, ?assertRCName( success, - emqtt:publish(C, MetaTopic, MetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) ), ConflictMeta = Meta#{name => <<"conflict.pdf">>}, @@ -228,7 +223,7 @@ t_meta_conflict(Config) -> ?assertRCName( unspecified_error, - emqtt:publish(C, MetaTopic, ConflictMetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), ConflictMetaPayload, 1) ). t_no_meta(Config) -> @@ -237,16 +232,14 @@ t_no_meta(Config) -> FileId = <<"f1">>, Data = <<"first">>, - SegmentTopic = <<"$file/", FileId/binary, "/0">>, ?assertRCName( success, - emqtt:publish(C, SegmentTopic, Data, 1) + emqtt:publish(C, mk_segment_topic(FileId, 0), Data, 1) ), - FinTopic = <<"$file/", FileId/binary, "/fin/42">>, ?assertRCName( unspecified_error, - emqtt:publish(C, FinTopic, <<>>, 1) + emqtt:publish(C, mk_fin_topic(FileId, 42), <<>>, 1) ). t_no_segment(Config) -> @@ -260,28 +253,25 @@ t_no_segment(Config) -> Meta = #{size := Filesize} = meta(Filename, Data), MetaPayload = emqx_json:encode(Meta), - MetaTopic = <<"$file/", FileId/binary, "/init">>, ?assertRCName( success, - emqtt:publish(C, MetaTopic, MetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) ), lists:foreach( fun({Chunk, Offset}) -> - SegmentTopic = <<"$file/", FileId/binary, "/", Offset/binary>>, ?assertRCName( success, - emqtt:publish(C, SegmentTopic, Chunk, 1) + emqtt:publish(C, mk_segment_topic(FileId, Offset), Chunk, 1) ) end, %% Skip the first segment tl(with_offsets(Data)) ), - FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( unspecified_error, - emqtt:publish(C, FinTopic, <<>>, 1) + emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) ). t_invalid_meta(Config) -> @@ -289,20 +279,18 @@ t_invalid_meta(Config) -> FileId = <<"f1">>, - MetaTopic = <<"$file/", FileId/binary, "/init">>, - %% Invalid schema Meta = #{foo => <<"bar">>}, MetaPayload = emqx_json:encode(Meta), ?assertRCName( unspecified_error, - emqtt:publish(C, MetaTopic, MetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) ), %% Invalid JSON ?assertRCName( unspecified_error, - emqtt:publish(C, MetaTopic, <<"{oops;">>, 1) + emqtt:publish(C, mk_init_topic(FileId), <<"{oops;">>, 1) ). t_invalid_checksum(Config) -> @@ -316,27 +304,73 @@ t_invalid_checksum(Config) -> Meta = #{size := Filesize} = meta(Filename, Data), MetaPayload = emqx_json:encode(Meta#{checksum => sha256hex(<<"invalid">>)}), - MetaTopic = <<"$file/", FileId/binary, "/init">>, ?assertRCName( success, - emqtt:publish(C, MetaTopic, MetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) ), lists:foreach( fun({Chunk, Offset}) -> - SegmentTopic = <<"$file/", FileId/binary, "/", Offset/binary>>, ?assertRCName( success, - emqtt:publish(C, SegmentTopic, Chunk, 1) + emqtt:publish(C, mk_segment_topic(FileId, Offset), Chunk, 1) ) end, with_offsets(Data) ), - FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( unspecified_error, - emqtt:publish(C, FinTopic, <<>>, 1) + emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) + ). + +t_corrupted_segment_retry(Config) -> + C = ?config(client, Config), + + Filename = <<"corruption.pdf">>, + FileId = <<"4242-4242">>, + + Data = [<<"first">>, <<"second">>, <<"third">>], + [ + {Seg1, Offset1}, + {Seg2, Offset2}, + {Seg3, Offset3} + ] = with_offsets(Data), + [ + Checksum1, + Checksum2, + Checksum3 + ] = [sha256hex(S) || S <- Data], + + Meta = #{size := Filesize} = meta(Filename, Data), + + ?assertRCName(success, emqtt:publish(C, mk_init_topic(FileId), emqx_json:encode(Meta), 1)), + + ?assertRCName( + success, + emqtt:publish(C, mk_segment_topic(FileId, Offset1, Checksum1), Seg1, 1) + ), + + % segment is corrupted + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_segment_topic(FileId, Offset2, Checksum2), <>, 1) + ), + + % retry + ?assertRCName( + success, + emqtt:publish(C, mk_segment_topic(FileId, Offset2, Checksum2), Seg2, 1) + ), + + ?assertRCName( + success, + emqtt:publish(C, mk_segment_topic(FileId, Offset3, Checksum3), Seg3, 1) + ), + + ?assertRCName( + success, + emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) ). t_switch_node(Config) -> @@ -359,14 +393,13 @@ t_switch_node(Config) -> Meta = #{size := Filesize} = meta(Filename, Data), MetaPayload = emqx_json:encode(Meta), - MetaTopic = <<"$file/", FileId/binary, "/init">>, ?assertRCName( success, - emqtt:publish(C1, MetaTopic, MetaPayload, 1) + emqtt:publish(C1, mk_init_topic(FileId), MetaPayload, 1) ), ?assertRCName( success, - emqtt:publish(C1, <<"$file/", FileId/binary, "/", Offset0/binary>>, Data0, 1) + emqtt:publish(C1, mk_segment_topic(FileId, Offset0), Data0, 1) ), %% Then, switch the client to the main node @@ -378,29 +411,24 @@ t_switch_node(Config) -> ?assertRCName( success, - emqtt:publish(C2, <<"$file/", FileId/binary, "/", Offset1/binary>>, Data1, 1) + emqtt:publish(C2, mk_segment_topic(FileId, Offset1), Data1, 1) ), ?assertRCName( success, - emqtt:publish(C2, <<"$file/", FileId/binary, "/", Offset2/binary>>, Data2, 1) + emqtt:publish(C2, mk_segment_topic(FileId, Offset2), Data2, 1) ), - FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( success, - emqtt:publish(C2, FinTopic, <<>>, 1) + emqtt:publish(C2, mk_fin_topic(FileId, Filesize), <<>>, 1) ), ok = emqtt:stop(C2), %% Now check consistency of the file - {ok, ReadyTransfers} = emqx_ft_storage:ready_transfers(), - {ReadyTransferIds, _} = lists:unzip(ReadyTransfers), - [ReadyTransferId] = [Id || #{<<"clientid">> := CId} = Id <- ReadyTransferIds, CId == ClientId], - + [ReadyTransferId] = list_ready_transfers(ClientId), {ok, TableQH} = emqx_ft_storage:get_ready_transfer(ReadyTransferId), - ?assertEqual( iolist_to_binary(Data), iolist_to_binary(qlc:eval(TableQH)) @@ -476,9 +504,7 @@ t_unreliable_migrating_client(Config) -> ], _Context = run_commands(Commands, Context), - {ok, ReadyTransfers} = emqx_ft_storage:ready_transfers(), - ReadyTransferIds = - [Id || {#{<<"clientid">> := CId} = Id, _Info} <- ReadyTransfers, CId == ClientId], + ReadyTransferIds = list_ready_transfers(?config(clientid, Config)), % NOTE % The cluster had 2 assemblers running on two different nodes, because client sent `fin` @@ -526,16 +552,13 @@ disown_mqtt_client(Context = #{}) -> Context. send_filemeta(Meta, Context = #{client := Client, fileid := FileId}) -> - Topic = <<"$file/", FileId/binary, "/init">>, - MetaPayload = emqx_json:encode(Meta), ?assertRCName( success, - emqtt:publish(Client, Topic, MetaPayload, 1) + emqtt:publish(Client, mk_init_topic(FileId), emqx_json:encode(Meta), 1) ), Context. send_segment(Offset, Size, Context = #{client := Client, fileid := FileId, payload := Payload}) -> - Topic = <<"$file/", FileId/binary, "/", (integer_to_binary(Offset))/binary>>, Data = case Size of eof -> @@ -545,15 +568,14 @@ send_segment(Offset, Size, Context = #{client := Client, fileid := FileId, paylo end, ?assertRCName( success, - emqtt:publish(Client, Topic, Data, 1) + emqtt:publish(Client, mk_segment_topic(FileId, Offset), Data, 1) ), Context. send_finish(Context = #{client := Client, fileid := FileId, filesize := Filesize}) -> - Topic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( success, - emqtt:publish(Client, Topic, <<>>, 1) + emqtt:publish(Client, mk_fin_topic(FileId, Filesize), <<>>, 1) ), Context. @@ -561,6 +583,24 @@ send_finish(Context = #{client := Client, fileid := FileId, filesize := Filesize %% Helpers %%-------------------------------------------------------------------- +mk_init_topic(FileId) -> + <<"$file/", FileId/binary, "/init">>. + +mk_segment_topic(FileId, Offset) when is_integer(Offset) -> + mk_segment_topic(FileId, integer_to_binary(Offset)); +mk_segment_topic(FileId, Offset) when is_binary(Offset) -> + <<"$file/", FileId/binary, "/", Offset/binary>>. + +mk_segment_topic(FileId, Offset, Checksum) when is_integer(Offset) -> + mk_segment_topic(FileId, integer_to_binary(Offset), Checksum); +mk_segment_topic(FileId, Offset, Checksum) when is_binary(Offset) -> + <<"$file/", FileId/binary, "/", Offset/binary, "/", Checksum/binary>>. + +mk_fin_topic(FileId, Size) when is_integer(Size) -> + mk_fin_topic(FileId, integer_to_binary(Size)); +mk_fin_topic(FileId, Size) when is_binary(Size) -> + <<"$file/", FileId/binary, "/fin/", Size/binary>>. + with_offsets(Items) -> {List, _} = lists:mapfoldl( fun(Item, Offset) -> @@ -582,3 +622,11 @@ meta(FileName, Data) -> expire_at => erlang:system_time(_Unit = second) + 3600, size => byte_size(FullData) }. + +list_ready_transfers(ClientId) -> + {ok, ReadyTransfers} = emqx_ft_storage:ready_transfers(), + [ + Id + || {#{<<"clientid">> := CId} = Id, _Info} <- ReadyTransfers, + CId == ClientId + ]. From ccba65e1eacca6b8a2fee54f9c1f5139c71325f4 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 21 Feb 2023 11:24:30 +0300 Subject: [PATCH 051/197] chore(ft): add code owners for ft --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5db0f4465..a45d9af59 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,6 +8,7 @@ /apps/emqx_connector/ @emqx/emqx-review-board @JimMoen /apps/emqx_dashboard/ @emqx/emqx-review-board @JimMoen @lafirest /apps/emqx_exhook/ @emqx/emqx-review-board @JimMoen @lafirest +/apps/emqx_ft/ @emqx/emqx-review-board @savonarola @keynslug /apps/emqx_gateway/ @emqx/emqx-review-board @lafirest /apps/emqx_management/ @emqx/emqx-review-board @lafirest @sstrigler /apps/emqx_plugin_libs/ @emqx/emqx-review-board @lafirest From 0af7e2a00266b7ee121e69088c0960bd9103a324 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 27 Feb 2023 15:45:00 +0300 Subject: [PATCH 052/197] perf(ft-asm): optimize away remote segments identical to local --- apps/emqx_ft/src/emqx_ft_assembly.erl | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index f0f0026a3..2a8ccb485 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -137,6 +137,10 @@ append_filemeta(Asm, Node, Fragment = #{fragment := {filemeta, Meta}}) -> meta = orddict:store(Meta, {Node, Fragment}, Asm#asm.meta) }. +append_segmentinfo(Asm, _Node, #{fragment := {segment, #{size := 0}}}) -> + % NOTE + % Empty segments are valid but meaningless for coverage. + Asm; append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) -> Offset = maps:get(offset, Info), Size = maps:get(size, Info), @@ -151,16 +155,17 @@ append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) -> coverage([{{Offset, _, _, _}, _Segment} | Rest], Cursor, Sz) when Offset < Cursor -> coverage(Rest, Cursor, Sz); -coverage([{{Cursor, _Locality, MEnd, Node}, Segment} | Rest], Cursor, Sz) -> +coverage([{{Cursor, _Locality, MEnd, Node}, Segment} | _Rest] = Segments, Cursor, Sz) -> % NOTE % We consider only whole fragments here, so for example from the point of view of % this algo `[{Offset1 = 0, Size1 = 15}, {Offset2 = 10, Size2 = 10}]` has no % coverage. - case coverage(Rest, -MEnd, Sz) of + Tail = tail(Segments), + case coverage(Tail, -MEnd, Sz) of Coverage when is_list(Coverage) -> [{Node, Segment} | Coverage]; Missing = {missing, _} -> - case coverage(Rest, Cursor, Sz) of + case coverage(Tail, Cursor, Sz) of CoverageAlt when is_list(CoverageAlt) -> CoverageAlt; {missing, _} -> @@ -174,6 +179,20 @@ coverage([], Cursor, Sz) when Cursor < Sz -> coverage([], Cursor, Cursor) -> []. +tail([Segment | Rest]) -> + tail(Segment, Rest). + +tail({{Cursor, _, MEnd, _}, _} = Segment, [{{Cursor, _, MEnd, _}, _} | Rest]) -> + % NOTE + % Discarding segments with same offset / size, potentially located on other nodes. + % This is an optimization. They won't participate in coverage anyway given we're + % currently optimizing coverage towards locality. Yet if we instead decide to + % optimize for node dominance (e.g. compute such coverage that most of the data + % located on a single node) we'll need to account them again. + tail(Segment, Rest); +tail(_Segment, Tail) -> + Tail. + dominant(Coverage) -> % TODO: needs improvement, better defined _dominance_, maybe some score Freqs = frequencies(fun({Node, Segment}) -> {Node, segsize(Segment)} end, Coverage), From 97cfdf8eef5b44b7140eea5107b146efe96f6486 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 27 Feb 2023 15:46:47 +0300 Subject: [PATCH 053/197] test(ft-asm): add property tests for file assembly --- apps/emqx/test/emqx_proper_types.erl | 33 ++- .../test/props/prop_emqx_ft_assembly.erl | 214 ++++++++++++++++++ 2 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl diff --git a/apps/emqx/test/emqx_proper_types.erl b/apps/emqx/test/emqx_proper_types.erl index 2f0f9d494..e1d95227b 100644 --- a/apps/emqx/test/emqx_proper_types.erl +++ b/apps/emqx/test/emqx_proper_types.erl @@ -43,12 +43,21 @@ ip/0, port/0, limited_atom/0, - limited_latin_atom/0 + limited_latin_atom/0, + printable_utf8/0, + printable_codepoint/0 +]). + +%% Generic Types +-export([ + scaled/2 ]). %% Iterators -export([nof/1]). +-type proptype() :: proper_types:raw_type(). + %%-------------------------------------------------------------------- %% Types High level %%-------------------------------------------------------------------- @@ -606,6 +615,20 @@ limited_atom() -> limited_any_term() -> oneof([binary(), number(), string()]). +printable_utf8() -> + ?SUCHTHAT( + String, + ?LET(L, list(printable_codepoint()), unicode:characters_to_binary(L)), + is_binary(String) + ). + +printable_codepoint() -> + frequency([ + {7, range(16#20, 16#7E)}, + {2, range(16#00A0, 16#D7FF)}, + {1, range(16#E000, 16#FFFD)} + ]). + %%-------------------------------------------------------------------- %% Iterators %%-------------------------------------------------------------------- @@ -632,6 +655,14 @@ limited_list(N, T) -> end ). +%%-------------------------------------------------------------------- +%% Generic Types +%%-------------------------------------------------------------------- + +-spec scaled(number(), proptype()) -> proptype(). +scaled(F, T) when F > 0 -> + ?SIZED(S, resize(round(S * F), T)). + %%-------------------------------------------------------------------- %% Internal funcs %%-------------------------------------------------------------------- diff --git a/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl b/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl new file mode 100644 index 000000000..ebebf0e65 --- /dev/null +++ b/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl @@ -0,0 +1,214 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(prop_emqx_ft_assembly). + +-include_lib("proper/include/proper.hrl"). + +-import(emqx_proper_types, [scaled/2]). + +-define(COVERAGE_TIMEOUT, 5000). + +prop_coverage() -> + ?FORALL( + {Filesize, Segsizes}, + {filesize_t(), segsizes_t()}, + ?FORALL( + Fragments, + noshrink(fragments_t(Filesize, Segsizes)), + ?TIMEOUT( + ?COVERAGE_TIMEOUT, + begin + ASM1 = append_fragments(mk_assembly(Filesize), Fragments), + {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), + measure( + #{"Fragments" => length(Fragments), "Time" => Time}, + case emqx_ft_assembly:status(ASM2) of + complete -> + Coverage = emqx_ft_assembly:coverage(ASM2), + measure( + #{"CoverageLength" => length(Coverage)}, + is_coverage_complete(Coverage) + ); + {incomplete, {missing, {segment, _, _}}} -> + measure("CoverageLength", 0, true) + end + ) + end + ) + ) + ). + +prop_coverage_likely_incomplete() -> + ?FORALL( + {Filesize, Segsizes, Hole}, + {filesize_t(), segsizes_t(), filesize_t()}, + ?FORALL( + Fragments, + noshrink(fragments_t(Filesize, Segsizes, Hole)), + ?TIMEOUT( + ?COVERAGE_TIMEOUT, + begin + ASM1 = append_fragments(mk_assembly(Filesize), Fragments), + {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), + measure( + #{"Fragments" => length(Fragments), "Time" => Time}, + case emqx_ft_assembly:status(ASM2) of + complete -> + % NOTE: this is still possible due to the nature of `SUCHTHATMAYBE` + IsComplete = emqx_ft_assembly:coverage(ASM2), + collect(complete, is_coverage_complete(IsComplete)); + {incomplete, {missing, {segment, _, _}}} -> + collect(incomplete, true) + end + ) + end + ) + ) + ). + +prop_coverage_complete() -> + ?FORALL( + {Filesize, Segsizes}, + {filesize_t(), ?SUCHTHAT([BaseSegsize | _], segsizes_t(), BaseSegsize > 0)}, + ?FORALL( + {Fragments, MaxCoverage}, + noshrink({fragments_t(Filesize, Segsizes), coverage_t(Filesize, Segsizes)}), + begin + % Ensure that we have complete coverage + ASM1 = append_fragments(mk_assembly(Filesize), Fragments ++ MaxCoverage), + {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), + measure( + #{"CoverageMax" => length(MaxCoverage), "Time" => Time}, + case emqx_ft_assembly:status(ASM2) of + complete -> + Coverage = emqx_ft_assembly:coverage(ASM2), + measure( + #{"Coverage" => length(Coverage)}, + is_coverage_complete(Coverage) + ); + {incomplete, _} -> + false + end + ) + end + ) + ). + +measure(NamedSamples, Test) -> + maps:fold(fun(Name, Sample, Acc) -> measure(Name, Sample, Acc) end, Test, NamedSamples). + +is_coverage_complete([]) -> + true; +is_coverage_complete(Coverage = [_ | Tail]) -> + is_coverage_complete(Coverage, Tail). + +is_coverage_complete([_], []) -> + true; +is_coverage_complete( + [{_Node1, #{fragment := {segment, #{offset := O1, size := S1}}}} | Rest], + [{_Node2, #{fragment := {segment, #{offset := O2}}}} | Tail] +) -> + (O1 + S1 == O2) andalso is_coverage_complete(Rest, Tail). + +mk_assembly(Filesize) -> + emqx_ft_assembly:append(emqx_ft_assembly:new(Filesize), node(), mk_filemeta(Filesize)). + +append_fragments(ASMIn, Fragments) -> + lists:foldl( + fun({Node, Frag}, ASM) -> + emqx_ft_assembly:append(ASM, Node, Frag) + end, + ASMIn, + Fragments + ). + +mk_filemeta(Filesize) -> + #{ + path => "MANIFEST.json", + fragment => {filemeta, #{name => ?MODULE_STRING, size => Filesize}} + }. + +mk_segment(Offset, Size) -> + #{ + path => "SEG" ++ integer_to_list(Offset) ++ integer_to_list(Size), + fragment => {segment, #{offset => Offset, size => Size}} + }. + +fragments_t(Filesize, Segsizes = [BaseSegsize | _]) -> + NSegs = Filesize / max(1, BaseSegsize), + scaled(1 + NSegs, list({node_t(), fragment_t(Filesize, Segsizes)})). + +fragments_t(Filesize, Segsizes = [BaseSegsize | _], Hole) -> + NSegs = Filesize / max(1, BaseSegsize), + scaled(1 + NSegs, list({node_t(), fragment_t(Filesize, Segsizes, Hole)})). + +fragment_t(Filesize, Segsizes, Hole) -> + ?SUCHTHATMAYBE( + #{fragment := {segment, #{offset := Offset, size := Size}}}, + fragment_t(Filesize, Segsizes), + (Hole rem Filesize) =< Offset orelse (Hole rem Filesize) > (Offset + Size) + ). + +fragment_t(Filesize, Segsizes) -> + ?LET( + Segsize, + oneof(Segsizes), + ?LET( + Index, + range(0, Filesize div max(1, Segsize)), + mk_segment(Index * Segsize, min(Segsize, Filesize - (Index * Segsize))) + ) + ). + +coverage_t(Filesize, [Segsize | _]) -> + NSegs = Filesize div max(1, Segsize), + [ + {remote_node_t(), mk_segment(I * Segsize, min(Segsize, Filesize - (I * Segsize)))} + || I <- lists:seq(0, NSegs) + ]. + +filesize_t() -> + scaled(4000, non_neg_integer()). + +segsizes_t() -> + ?LET( + BaseSize, + segsize_t(), + oneof([ + [BaseSize, BaseSize * 2], + [BaseSize, BaseSize * 2, BaseSize * 3], + [BaseSize, BaseSize * 2, BaseSize * 5] + ]) + ). + +segsize_t() -> + scaled(50, non_neg_integer()). + +remote_node_t() -> + oneof([ + 'emqx42@emqx.local', + 'emqx43@emqx.local', + 'emqx44@emqx.local' + ]). + +node_t() -> + oneof([ + node(), + 'emqx42@emqx.local', + 'emqx43@emqx.local', + 'emqx44@emqx.local' + ]). From 0c84fc28b041d02558a48cd2a6cff66d1c1ecaf2 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 27 Feb 2023 15:59:52 +0300 Subject: [PATCH 054/197] perf(asm-ft): tradeoff optimality for computational complexity Through squashing segments table into consecutive "runs". --- apps/emqx_ft/src/emqx_ft_assembly.erl | 84 ++++++++++++++++++--------- 1 file changed, 55 insertions(+), 29 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index 2a8ccb485..4625e5ffc 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -40,9 +40,9 @@ filemeta(), {node(), filefrag({filemeta, filemeta()})} ), - segs :: orddict:orddict( + segs :: gb_trees:tree( {emqx_ft:offset(), _Locality, _MEnd, node()}, - filefrag({segment, segmentinfo()}) + [filefrag({segment, segmentinfo()})] ), size :: emqx_ft:bytes() }). @@ -66,7 +66,7 @@ new(Size) -> #asm{ status = {incomplete, {missing, filemeta}}, meta = orddict:new(), - segs = orddict:new(), + segs = gb_trees:empty(), size = Size }. @@ -123,7 +123,7 @@ status(meta, []) -> status(meta, [_M1, _M2 | _] = Metas) -> {error, {inconsistent, [Frag#{node => Node} || {_, {Node, Frag}} <- Metas]}}; status(coverage, #asm{segs = Segments, size = Size}) -> - case coverage(orddict:to_list(Segments), 0, Size) of + case coverage(squash(Segments), Size) of Coverage when is_list(Coverage) -> {complete, Coverage, #{ dominant => dominant(Coverage) @@ -145,54 +145,80 @@ append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) -> Offset = maps:get(offset, Info), Size = maps:get(size, Info), End = Offset + Size, + Index = {Offset, locality(Node), -End, Node}, + Segs = insert(Asm#asm.segs, Index, [Fragment]), Asm#asm{ % TODO % In theory it's possible to have two segments with same offset + size on % different nodes but with differing content. We'd need a checksum to % be able to disambiguate them though. - segs = orddict:store({Offset, locality(Node), -End, Node}, Fragment, Asm#asm.segs) + segs = Segs }. -coverage([{{Offset, _, _, _}, _Segment} | Rest], Cursor, Sz) when Offset < Cursor -> - coverage(Rest, Cursor, Sz); -coverage([{{Cursor, _Locality, MEnd, Node}, Segment} | _Rest] = Segments, Cursor, Sz) -> +squash(Segs) -> + % NOTE + % Here we're "compressing" information about every known segment by adjoining + % nearby segments on the same node into "runs". + squash(Segs, gb_trees:next(gb_trees:iterator(Segs))). + +squash(Segs, {Index = {Offset, Locality, MEnd, _}, Fragments, It}) -> + SegsSquashed = squash_run(gb_trees:delete(Index, Segs), Index, Fragments, It), + ItNext = gb_trees:iterator_from({Offset, Locality, MEnd + 1, 0}, SegsSquashed), + squash(SegsSquashed, gb_trees:next(ItNext)); +squash(Segs, none) -> + Segs. + +squash_run(Segs, {Offset, Locality, MEnd, Node} = Index, Fragments, It) -> + Next = gb_trees:next(It), + case Next of + {{OffsetNext, _, MEndNext, Node} = IndexNext, FragmentsNext, ItNext} when + OffsetNext == -MEnd + -> + SegsNext = gb_trees:delete(IndexNext, Segs), + IndexSquashed = {Offset, Locality, MEndNext, Node}, + squash_run(SegsNext, IndexSquashed, Fragments ++ FragmentsNext, ItNext); + {{OffsetNext, _, _, _}, _, ItNext} when OffsetNext =< -MEnd -> + squash_run(Segs, Index, Fragments, ItNext); + _ -> + insert(Segs, Index, Fragments) + end. + +insert(Segs, Index, Fragments) -> + try + gb_trees:insert(Index, Fragments, Segs) + catch + error:{key_exists, _} -> Segs + end. + +coverage(Segs, Size) -> + coverage(gb_trees:next(gb_trees:iterator(Segs)), 0, Size). + +coverage({{Offset, _, _, _}, _Fragments, It}, Cursor, Sz) when Offset < Cursor -> + coverage(gb_trees:next(It), Cursor, Sz); +coverage({{Cursor, _Locality, MEnd, Node}, Fragments, It}, Cursor, Sz) -> % NOTE % We consider only whole fragments here, so for example from the point of view of % this algo `[{Offset1 = 0, Size1 = 15}, {Offset2 = 10, Size2 = 10}]` has no % coverage. - Tail = tail(Segments), - case coverage(Tail, -MEnd, Sz) of + ItNext = gb_trees:next(It), + case coverage(ItNext, -MEnd, Sz) of Coverage when is_list(Coverage) -> - [{Node, Segment} | Coverage]; + [{Node, Frag} || Frag <- Fragments] ++ Coverage; Missing = {missing, _} -> - case coverage(Tail, Cursor, Sz) of + case coverage(ItNext, Cursor, Sz) of CoverageAlt when is_list(CoverageAlt) -> CoverageAlt; {missing, _} -> Missing end end; -coverage([{{Offset, _MEnd, _, _}, _Segment} | _], Cursor, _Sz) when Offset > Cursor -> +coverage({{Offset, _, _MEnd, _}, _Fragments, _It}, Cursor, _Sz) when Offset > Cursor -> {missing, {segment, Cursor, Offset}}; -coverage([], Cursor, Sz) when Cursor < Sz -> +coverage(none, Cursor, Sz) when Cursor < Sz -> {missing, {segment, Cursor, Sz}}; -coverage([], Cursor, Cursor) -> +coverage(none, Cursor, Cursor) -> []. -tail([Segment | Rest]) -> - tail(Segment, Rest). - -tail({{Cursor, _, MEnd, _}, _} = Segment, [{{Cursor, _, MEnd, _}, _} | Rest]) -> - % NOTE - % Discarding segments with same offset / size, potentially located on other nodes. - % This is an optimization. They won't participate in coverage anyway given we're - % currently optimizing coverage towards locality. Yet if we instead decide to - % optimize for node dominance (e.g. compute such coverage that most of the data - % located on a single node) we'll need to account them again. - tail(Segment, Rest); -tail(_Segment, Tail) -> - Tail. - dominant(Coverage) -> % TODO: needs improvement, better defined _dominance_, maybe some score Freqs = frequencies(fun({Node, Segment}) -> {Node, segsize(Segment)} end, Coverage), From 130601376aaa6018b016a8f77364181df8ea411f Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 27 Feb 2023 19:41:34 +0300 Subject: [PATCH 055/197] perf(ft-asm): express coverage through shortest path on graph Coverage becomes the shortest path problem on graph where nodes are offsets and edges are nodes' segments. Implement a simple Dijkstra algorithm to find one. --- apps/emqx/src/emqx_maybe.erl | 14 ++ apps/emqx_ft/src/emqx_ft_assembly.erl | 202 +++++++++++++++++--------- 2 files changed, 151 insertions(+), 65 deletions(-) diff --git a/apps/emqx/src/emqx_maybe.erl b/apps/emqx/src/emqx_maybe.erl index 8da7c7c90..0b919f7ab 100644 --- a/apps/emqx/src/emqx_maybe.erl +++ b/apps/emqx/src/emqx_maybe.erl @@ -20,6 +20,7 @@ -export([to_list/1]). -export([from_list/1]). +-export([define/2]). -export([apply/2]). -spec to_list(maybe(A)) -> [A]. @@ -34,6 +35,12 @@ from_list([]) -> from_list([Term]) -> Term. +-spec define(maybe(A), B) -> A | B. +define(undefined, Term) -> + Term; +define(Term, _) -> + Term. + %% @doc Apply a function to a maybe argument. -spec apply(fun((maybe(A)) -> maybe(A)), maybe(A)) -> maybe(A). @@ -60,6 +67,13 @@ from_list_test_() -> ?_assertError(_, from_list([1, 2, 3])) ]. +define_test_() -> + [ + ?_assertEqual(42, define(42, undefined)), + ?_assertEqual(<<"default">>, define(undefined, <<"default">>)), + ?_assertEqual(undefined, define(undefined, undefined)) + ]. + apply_test_() -> [ ?_assertEqual(<<"42">>, ?MODULE:apply(fun erlang:integer_to_binary/1, 42)), diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index 4625e5ffc..652636088 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -123,7 +123,7 @@ status(meta, []) -> status(meta, [_M1, _M2 | _] = Metas) -> {error, {inconsistent, [Frag#{node => Node} || {_, {Node, Frag}} <- Metas]}}; status(coverage, #asm{segs = Segments, size = Size}) -> - case coverage(squash(Segments), Size) of + case coverage(Segments, Size) of Coverage when is_list(Coverage) -> {complete, Coverage, #{ dominant => dominant(Coverage) @@ -145,8 +145,7 @@ append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) -> Offset = maps:get(offset, Info), Size = maps:get(size, Info), End = Offset + Size, - Index = {Offset, locality(Node), -End, Node}, - Segs = insert(Asm#asm.segs, Index, [Fragment]), + Segs = add_edge(Asm#asm.segs, Offset, End, locality(Node) * Size, {Node, Fragment}), Asm#asm{ % TODO % In theory it's possible to have two segments with same offset + size on @@ -155,69 +154,143 @@ append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) -> segs = Segs }. -squash(Segs) -> - % NOTE - % Here we're "compressing" information about every known segment by adjoining - % nearby segments on the same node into "runs". - squash(Segs, gb_trees:next(gb_trees:iterator(Segs))). - -squash(Segs, {Index = {Offset, Locality, MEnd, _}, Fragments, It}) -> - SegsSquashed = squash_run(gb_trees:delete(Index, Segs), Index, Fragments, It), - ItNext = gb_trees:iterator_from({Offset, Locality, MEnd + 1, 0}, SegsSquashed), - squash(SegsSquashed, gb_trees:next(ItNext)); -squash(Segs, none) -> - Segs. - -squash_run(Segs, {Offset, Locality, MEnd, Node} = Index, Fragments, It) -> - Next = gb_trees:next(It), - case Next of - {{OffsetNext, _, MEndNext, Node} = IndexNext, FragmentsNext, ItNext} when - OffsetNext == -MEnd - -> - SegsNext = gb_trees:delete(IndexNext, Segs), - IndexSquashed = {Offset, Locality, MEndNext, Node}, - squash_run(SegsNext, IndexSquashed, Fragments ++ FragmentsNext, ItNext); - {{OffsetNext, _, _, _}, _, ItNext} when OffsetNext =< -MEnd -> - squash_run(Segs, Index, Fragments, ItNext); - _ -> - insert(Segs, Index, Fragments) - end. - -insert(Segs, Index, Fragments) -> - try - gb_trees:insert(Index, Fragments, Segs) - catch - error:{key_exists, _} -> Segs - end. - coverage(Segs, Size) -> - coverage(gb_trees:next(gb_trees:iterator(Segs)), 0, Size). + find_shortest_path(Segs, 0, Size). -coverage({{Offset, _, _, _}, _Fragments, It}, Cursor, Sz) when Offset < Cursor -> - coverage(gb_trees:next(It), Cursor, Sz); -coverage({{Cursor, _Locality, MEnd, Node}, Fragments, It}, Cursor, Sz) -> +find_shortest_path(G1, From, To) -> % NOTE - % We consider only whole fragments here, so for example from the point of view of - % this algo `[{Offset1 = 0, Size1 = 15}, {Offset2 = 10, Size2 = 10}]` has no - % coverage. - ItNext = gb_trees:next(It), - case coverage(ItNext, -MEnd, Sz) of - Coverage when is_list(Coverage) -> - [{Node, Frag} || Frag <- Fragments] ++ Coverage; - Missing = {missing, _} -> - case coverage(ItNext, Cursor, Sz) of - CoverageAlt when is_list(CoverageAlt) -> - CoverageAlt; - {missing, _} -> - Missing + % This is a Dijkstra shortest path algorithm implemented on top of `gb_trees`. + % It is one-way right now, for simplicity sake. + G2 = set_cost(G1, From, 0, []), + case find_shortest_path(G2, From, 0, To) of + {found, G3} -> + construct_path(G3, From, To, []); + {error, Last} -> + % NOTE: this is actually just an estimation of what is missing. + {missing, {segment, Last, emqx_maybe:define(find_successor(G2, Last), To)}} + end. + +find_shortest_path(G1, Node, Cost, Target) -> + Edges = get_edges(G1, Node), + G2 = update_neighbours(G1, Node, Cost, Edges), + case take_queued(G2) of + {Target, _NextCost, G3} -> + {found, G3}; + {Next, NextCost, G3} -> + find_shortest_path(G3, Next, NextCost, Target); + none -> + {error, Node} + end. + +construct_path(_G, From, From, Acc) -> + Acc; +construct_path(G, From, To, Acc) -> + {Prev, Label} = get_label(G, To), + construct_path(G, From, Prev, [Label | Acc]). + +update_neighbours(G1, Node, NodeCost, Edges) -> + lists:foldl( + fun({Neighbour, Weight, Label}, GAcc) -> + case is_visited(GAcc, Neighbour) of + false -> + NeighCost = NodeCost + Weight, + CurrentCost = get_cost(GAcc, Neighbour), + case NeighCost < CurrentCost of + true -> + set_cost(GAcc, Neighbour, NeighCost, {Node, Label}); + false -> + GAcc + end; + true -> + GAcc end - end; -coverage({{Offset, _, _MEnd, _}, _Fragments, _It}, Cursor, _Sz) when Offset > Cursor -> - {missing, {segment, Cursor, Offset}}; -coverage(none, Cursor, Sz) when Cursor < Sz -> - {missing, {segment, Cursor, Sz}}; -coverage(none, Cursor, Cursor) -> - []. + end, + G1, + Edges + ). + +add_edge(G, Node, ToNode, WeightIn, EdgeLabel) -> + Edges = tree_lookup({Node}, G, []), + case lists:keyfind(ToNode, 1, Edges) of + {ToNode, Weight, _} when Weight =< WeightIn -> + % NOTE + % Discarding any edges with higher weight here. This is fine as long as we + % optimize for locality. + G; + _ -> + EdgesNext = lists:keystore(ToNode, 1, Edges, {ToNode, WeightIn, EdgeLabel}), + tree_update({Node}, EdgesNext, G) + end. + +get_edges(G, Node) -> + tree_lookup({Node}, G, []). + +get_cost(G, Node) -> + tree_lookup({Node, cost}, G, inf). + +get_label(G, Node) -> + gb_trees:get({Node, label}, G). + +set_cost(G1, Node, Cost, Label) -> + G3 = + case tree_lookup({Node, cost}, G1, inf) of + CostWas when CostWas /= inf -> + {true, G2} = gb_trees:take({queued, CostWas, Node}, G1), + tree_update({queued, Cost, Node}, true, G2); + inf -> + tree_update({queued, Cost, Node}, true, G1) + end, + G4 = tree_update({Node, cost}, Cost, G3), + G5 = tree_update({Node, label}, Label, G4), + G5. + +take_queued(G1) -> + It = gb_trees:iterator_from({queued, 0, 0}, G1), + case gb_trees:next(It) of + {{queued, Cost, Node} = Index, true, _It} -> + {Node, Cost, gb_trees:delete(Index, G1)}; + _ -> + none + end. + +is_visited(G, Node) -> + case tree_lookup({Node, cost}, G, inf) of + inf -> + false; + Cost -> + not tree_lookup({queued, Cost, Node}, G, false) + end. + +find_successor(G, Node) -> + case gb_trees:next(gb_trees:iterator_from({Node}, G)) of + {{Node}, _, It} -> + case gb_trees:next(It) of + {{Successor}, _, _} -> + Successor; + _ -> + undefined + end; + {{Successor}, _, _} -> + Successor; + _ -> + undefined + end. + +tree_lookup(Index, Tree, Default) -> + case gb_trees:lookup(Index, Tree) of + {value, V} -> + V; + none -> + Default + end. + +tree_update(Index, Value, Tree) -> + case gb_trees:take_any(Index, Tree) of + {_, TreeNext} -> + gb_trees:insert(Index, Value, TreeNext); + error -> + gb_trees:insert(Index, Value, Tree) + end. dominant(Coverage) -> % TODO: needs improvement, better defined _dominance_, maybe some score @@ -379,8 +452,7 @@ missing_coverage_test() -> ], Asm = append_many(new(100), Segs), ?assertEqual( - % {incomplete, {missing, {segment, 30, 40}}}, ??? - {incomplete, {missing, {segment, 20, 40}}}, + {incomplete, {missing, {segment, 30, 40}}}, status(coverage, Asm) ). @@ -407,7 +479,7 @@ missing_coverage_with_redudancy_test() -> Asm = append_many(new(100), Segs), ?assertEqual( % {incomplete, {missing, {segment, 50, 60}}}, ??? - {incomplete, {missing, {segment, 20, 40}}}, + {incomplete, {missing, {segment, 60, 100}}}, status(coverage, Asm) ). From b189ee463c72e421d79b023e4d143b867a065ae5 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 28 Feb 2023 14:01:01 +0300 Subject: [PATCH 056/197] feat: add weighted directional graph ADT with shortest path Basically, separate what abstraction was in `emqx_ft_assembly` into dedicated module with a compact interface and a basic testsuite. --- apps/emqx/src/emqx_wdgraph.erl | 186 ++++++++++++++++++++++++++ apps/emqx/test/emqx_wdgraph_tests.erl | 84 ++++++++++++ 2 files changed, 270 insertions(+) create mode 100644 apps/emqx/src/emqx_wdgraph.erl create mode 100644 apps/emqx/test/emqx_wdgraph_tests.erl diff --git a/apps/emqx/src/emqx_wdgraph.erl b/apps/emqx/src/emqx_wdgraph.erl new file mode 100644 index 000000000..3361c52d1 --- /dev/null +++ b/apps/emqx/src/emqx_wdgraph.erl @@ -0,0 +1,186 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Weighted directed graph. +%% +%% Purely functional, built on top of a single `gb_tree`. +%% Weights are currently assumed to be non-negative numbers, hovewer +%% presumably anything that is ≥ 0 should work (but won't typecheck 🥲). + +-module(emqx_wdgraph). + +-export([new/0]). +-export([insert_edge/5]). +-export([find_edge/3]). +-export([get_edges/2]). + +-export([find_shortest_path/3]). + +-export_type([t/0]). +-export_type([t/2]). +-export_type([weight/0]). + +-type gnode() :: term(). +-type weight() :: _NonNegative :: number(). +-type label() :: term(). + +-opaque t() :: t(gnode(), label()). +-opaque t(Node, Label) :: gb_trees:tree({Node}, {Node, weight(), Label}). + +%% + +-spec new() -> t(_, _). +new() -> + gb_trees:empty(). + +%% Add an edge. +%% Nodes are not expected to exist beforehand, and created lazily. +%% There could be only one edge between each pair of nodes, this function +%% replaces any existing edge in the graph. +-spec insert_edge(Node, Node, weight(), Label, t(Node, Label)) -> t(Node, Label). +insert_edge(From, To, Weight, EdgeLabel, G) -> + Edges = tree_lookup({From}, G, []), + EdgesNext = lists:keystore(To, 1, Edges, {To, Weight, EdgeLabel}), + tree_update({From}, EdgesNext, G). + +%% Find exising edge between two nodes, if any. +-spec find_edge(Node, Node, t(Node, Label)) -> {weight(), Label} | false. +find_edge(From, To, G) -> + Edges = tree_lookup({From}, G, []), + case lists:keyfind(To, 1, Edges) of + {To, Weight, Label} -> + {Weight, Label}; + false -> + false + end. + +%% Get all edges from the given node. +-spec get_edges(Node, t(Node, Label)) -> [{Node, weight(), Label}]. +get_edges(Node, G) -> + tree_lookup({Node}, G, []). + +% Find the shortest path between two nodes, if any. If the path exists, return list +% of edge labels along that path. +% This is a Dijkstra shortest path algorithm. It is one-way right now, for +% simplicity sake. +-spec find_shortest_path(Node, Node, t(Node, Label)) -> [Label] | {false, _StoppedAt :: Node}. +find_shortest_path(From, To, G1) -> + % NOTE + % If `From` and `To` are the same node, then path is `[]` even if this + % node does not exist in the graph. + G2 = set_cost(From, 0, [], G1), + case find_shortest_path(From, 0, To, G2) of + {true, G3} -> + construct_path(From, To, [], G3); + {false, Last} -> + {false, Last} + end. + +find_shortest_path(Node, Cost, Target, G1) -> + Edges = get_edges(Node, G1), + G2 = update_neighbours(Node, Cost, Edges, G1), + case take_queued(G2) of + {Target, _NextCost, G3} -> + {true, G3}; + {Next, NextCost, G3} -> + find_shortest_path(Next, NextCost, Target, G3); + none -> + {false, Node} + end. + +construct_path(From, From, Acc, _) -> + Acc; +construct_path(From, To, Acc, G) -> + {Prev, Label} = get_label(To, G), + construct_path(From, Prev, [Label | Acc], G). + +update_neighbours(Node, NodeCost, Edges, G1) -> + lists:foldl( + fun(Edge, GAcc) -> update_neighbour(Node, NodeCost, Edge, GAcc) end, + G1, + Edges + ). + +update_neighbour(Node, NodeCost, {Neighbour, Weight, Label}, G) -> + case is_visited(G, Neighbour) of + false -> + CurrentCost = get_cost(Neighbour, G), + case NodeCost + Weight of + NeighCost when NeighCost < CurrentCost -> + set_cost(Neighbour, NeighCost, {Node, Label}, G); + _ -> + G + end; + true -> + G + end. + +get_cost(Node, G) -> + case tree_lookup({Node, cost}, G, inf) of + {Cost, _Label} -> + Cost; + inf -> + inf + end. + +get_label(Node, G) -> + {_Cost, Label} = gb_trees:get({Node, cost}, G), + Label. + +set_cost(Node, Cost, Label, G1) -> + G3 = + case tree_lookup({Node, cost}, G1, inf) of + {CostWas, _Label} -> + {true, G2} = gb_trees:take({queued, CostWas, Node}, G1), + gb_trees:insert({queued, Cost, Node}, true, G2); + inf -> + gb_trees:insert({queued, Cost, Node}, true, G1) + end, + G4 = tree_update({Node, cost}, {Cost, Label}, G3), + G4. + +take_queued(G1) -> + It = gb_trees:iterator_from({queued, 0, 0}, G1), + case gb_trees:next(It) of + {{queued, Cost, Node} = Index, true, _It} -> + {Node, Cost, gb_trees:delete(Index, G1)}; + _ -> + none + end. + +is_visited(G, Node) -> + case tree_lookup({Node, cost}, G, inf) of + inf -> + false; + {Cost, _Label} -> + not tree_lookup({queued, Cost, Node}, G, false) + end. + +tree_lookup(Index, Tree, Default) -> + case gb_trees:lookup(Index, Tree) of + {value, V} -> + V; + none -> + Default + end. + +tree_update(Index, Value, Tree) -> + case gb_trees:is_defined(Index, Tree) of + true -> + gb_trees:update(Index, Value, Tree); + false -> + gb_trees:insert(Index, Value, Tree) + end. diff --git a/apps/emqx/test/emqx_wdgraph_tests.erl b/apps/emqx/test/emqx_wdgraph_tests.erl new file mode 100644 index 000000000..ece87b966 --- /dev/null +++ b/apps/emqx/test/emqx_wdgraph_tests.erl @@ -0,0 +1,84 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_wdgraph_tests). + +-include_lib("eunit/include/eunit.hrl"). + +empty_test_() -> + G = emqx_wdgraph:new(), + [ + ?_assertEqual([], emqx_wdgraph:get_edges(foo, G)), + ?_assertEqual(false, emqx_wdgraph:find_edge(foo, bar, G)) + ]. + +edges_nodes_test_() -> + G1 = emqx_wdgraph:new(), + G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1), + G3 = emqx_wdgraph:insert_edge(bar, baz, 1, "cheapest", G2), + G4 = emqx_wdgraph:insert_edge(bar, foo, 0, "free", G3), + G5 = emqx_wdgraph:insert_edge(foo, bar, 100, "luxury", G4), + [ + ?_assertEqual({42, "fancy"}, emqx_wdgraph:find_edge(foo, bar, G2)), + ?_assertEqual({100, "luxury"}, emqx_wdgraph:find_edge(foo, bar, G5)), + ?_assertEqual([{bar, 100, "luxury"}], emqx_wdgraph:get_edges(foo, G5)), + + ?_assertEqual({1, "cheapest"}, emqx_wdgraph:find_edge(bar, baz, G5)), + ?_assertEqual([{baz, 1, "cheapest"}, {foo, 0, "free"}], emqx_wdgraph:get_edges(bar, G5)) + ]. + +nonexistent_nodes_path_test_() -> + G1 = emqx_wdgraph:new(), + G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1), + G3 = emqx_wdgraph:insert_edge(bar, baz, 1, "cheapest", G2), + [ + ?_assertEqual( + {false, nosuchnode}, + emqx_wdgraph:find_shortest_path(nosuchnode, baz, G3) + ), + ?_assertEqual( + [], + emqx_wdgraph:find_shortest_path(nosuchnode, nosuchnode, G3) + ) + ]. + +nonexistent_path_test_() -> + G1 = emqx_wdgraph:new(), + G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1), + G3 = emqx_wdgraph:insert_edge(baz, boo, 1, "cheapest", G2), + G4 = emqx_wdgraph:insert_edge(boo, last, 3.5, "change", G3), + [ + ?_assertEqual( + {false, last}, + emqx_wdgraph:find_shortest_path(baz, foo, G4) + ), + ?_assertEqual( + {false, bar}, + emqx_wdgraph:find_shortest_path(foo, last, G4) + ) + ]. + +shortest_path_test() -> + G1 = emqx_wdgraph:new(), + G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1), + G3 = emqx_wdgraph:insert_edge(bar, baz, 1, "cheapest", G2), + G4 = emqx_wdgraph:insert_edge(baz, last, 0, "free", G3), + G5 = emqx_wdgraph:insert_edge(bar, last, 100, "luxury", G4), + G6 = emqx_wdgraph:insert_edge(bar, foo, 0, "comeback", G5), + ?assertEqual( + ["fancy", "cheapest", "free"], + emqx_wdgraph:find_shortest_path(foo, last, G6) + ). From 8e6f960c091c953160777af32a5ea8e56aacaa4f Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 28 Feb 2023 14:03:46 +0300 Subject: [PATCH 057/197] refactor(ft): employ `emqx_wdgraph` for coverage computation Also describe how coverage problem maps to shortest path problem. --- apps/emqx_ft/src/emqx_ft_assembly.erl | 158 ++++---------------------- 1 file changed, 23 insertions(+), 135 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index 652636088..bea320bbf 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -40,10 +40,7 @@ filemeta(), {node(), filefrag({filemeta, filemeta()})} ), - segs :: gb_trees:tree( - {emqx_ft:offset(), _Locality, _MEnd, node()}, - [filefrag({segment, segmentinfo()})] - ), + segs :: emqx_wdgraph:t(emqx_ft:offset(), {node(), filefrag({segment, segmentinfo()})}), size :: emqx_ft:bytes() }). @@ -66,7 +63,7 @@ new(Size) -> #asm{ status = {incomplete, {missing, filemeta}}, meta = orddict:new(), - segs = gb_trees:empty(), + segs = emqx_wdgraph:new(), size = Size }. @@ -154,142 +151,32 @@ append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) -> segs = Segs }. -coverage(Segs, Size) -> - find_shortest_path(Segs, 0, Size). - -find_shortest_path(G1, From, To) -> +add_edge(Segs, Offset, End, Weight, Label) -> % NOTE - % This is a Dijkstra shortest path algorithm implemented on top of `gb_trees`. - % It is one-way right now, for simplicity sake. - G2 = set_cost(G1, From, 0, []), - case find_shortest_path(G2, From, 0, To) of - {found, G3} -> - construct_path(G3, From, To, []); - {error, Last} -> - % NOTE: this is actually just an estimation of what is missing. - {missing, {segment, Last, emqx_maybe:define(find_successor(G2, Last), To)}} - end. - -find_shortest_path(G1, Node, Cost, Target) -> - Edges = get_edges(G1, Node), - G2 = update_neighbours(G1, Node, Cost, Edges), - case take_queued(G2) of - {Target, _NextCost, G3} -> - {found, G3}; - {Next, NextCost, G3} -> - find_shortest_path(G3, Next, NextCost, Target); - none -> - {error, Node} - end. - -construct_path(_G, From, From, Acc) -> - Acc; -construct_path(G, From, To, Acc) -> - {Prev, Label} = get_label(G, To), - construct_path(G, From, Prev, [Label | Acc]). - -update_neighbours(G1, Node, NodeCost, Edges) -> - lists:foldl( - fun({Neighbour, Weight, Label}, GAcc) -> - case is_visited(GAcc, Neighbour) of - false -> - NeighCost = NodeCost + Weight, - CurrentCost = get_cost(GAcc, Neighbour), - case NeighCost < CurrentCost of - true -> - set_cost(GAcc, Neighbour, NeighCost, {Node, Label}); - false -> - GAcc - end; - true -> - GAcc - end - end, - G1, - Edges - ). - -add_edge(G, Node, ToNode, WeightIn, EdgeLabel) -> - Edges = tree_lookup({Node}, G, []), - case lists:keyfind(ToNode, 1, Edges) of - {ToNode, Weight, _} when Weight =< WeightIn -> + % We are expressing coverage problem as a shortest path problem on weighted directed + % graph, where nodes are segments offsets, two nodes are connected with edge if + % there is a segment which "covers" these offsets (i.e. it starts at first node's + % offset and ends at second node's offst) and weights are segments sizes adjusted + % for locality (i.e. weight are always 0 for any local segment). + case emqx_wdgraph:find_edge(Offset, End, Segs) of + {WeightWas, _Label} when WeightWas =< Weight -> % NOTE % Discarding any edges with higher weight here. This is fine as long as we % optimize for locality. - G; + Segs; _ -> - EdgesNext = lists:keystore(ToNode, 1, Edges, {ToNode, WeightIn, EdgeLabel}), - tree_update({Node}, EdgesNext, G) + emqx_wdgraph:insert_edge(Offset, End, Weight, Label, Segs) end. -get_edges(G, Node) -> - tree_lookup({Node}, G, []). - -get_cost(G, Node) -> - tree_lookup({Node, cost}, G, inf). - -get_label(G, Node) -> - gb_trees:get({Node, label}, G). - -set_cost(G1, Node, Cost, Label) -> - G3 = - case tree_lookup({Node, cost}, G1, inf) of - CostWas when CostWas /= inf -> - {true, G2} = gb_trees:take({queued, CostWas, Node}, G1), - tree_update({queued, Cost, Node}, true, G2); - inf -> - tree_update({queued, Cost, Node}, true, G1) - end, - G4 = tree_update({Node, cost}, Cost, G3), - G5 = tree_update({Node, label}, Label, G4), - G5. - -take_queued(G1) -> - It = gb_trees:iterator_from({queued, 0, 0}, G1), - case gb_trees:next(It) of - {{queued, Cost, Node} = Index, true, _It} -> - {Node, Cost, gb_trees:delete(Index, G1)}; - _ -> - none - end. - -is_visited(G, Node) -> - case tree_lookup({Node, cost}, G, inf) of - inf -> - false; - Cost -> - not tree_lookup({queued, Cost, Node}, G, false) - end. - -find_successor(G, Node) -> - case gb_trees:next(gb_trees:iterator_from({Node}, G)) of - {{Node}, _, It} -> - case gb_trees:next(It) of - {{Successor}, _, _} -> - Successor; - _ -> - undefined - end; - {{Successor}, _, _} -> - Successor; - _ -> - undefined - end. - -tree_lookup(Index, Tree, Default) -> - case gb_trees:lookup(Index, Tree) of - {value, V} -> - V; - none -> - Default - end. - -tree_update(Index, Value, Tree) -> - case gb_trees:take_any(Index, Tree) of - {_, TreeNext} -> - gb_trees:insert(Index, Value, TreeNext); - error -> - gb_trees:insert(Index, Value, Tree) +coverage(Segs, Size) -> + case emqx_wdgraph:find_shortest_path(0, Size, Segs) of + Path when is_list(Path) -> + Path; + {false, LastOffset} -> + % NOTE + % This is far from being accurate, but needs no hairy specifics in the + % `emqx_wdgraph` interface. + {missing, {segment, LastOffset, Size}} end. dominant(Coverage) -> @@ -452,7 +339,8 @@ missing_coverage_test() -> ], Asm = append_many(new(100), Segs), ?assertEqual( - {incomplete, {missing, {segment, 30, 40}}}, + % {incomplete, {missing, {segment, 30, 40}}} would be more accurate + {incomplete, {missing, {segment, 30, 100}}}, status(coverage, Asm) ). From 788e76ed2d343aa2924f9707b2261948f57dbded Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 28 Feb 2023 16:55:23 +0300 Subject: [PATCH 058/197] test: make assembly proptests waste less memory --- .../test/props/prop_emqx_ft_assembly.erl | 71 ++++++++++--------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl b/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl index ebebf0e65..a437e8dfe 100644 --- a/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl +++ b/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl @@ -28,11 +28,11 @@ prop_coverage() -> {filesize_t(), segsizes_t()}, ?FORALL( Fragments, - noshrink(fragments_t(Filesize, Segsizes)), + noshrink(segments_t(Filesize, Segsizes)), ?TIMEOUT( ?COVERAGE_TIMEOUT, begin - ASM1 = append_fragments(mk_assembly(Filesize), Fragments), + ASM1 = append_segments(mk_assembly(Filesize), Fragments), {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), measure( #{"Fragments" => length(Fragments), "Time" => Time}, @@ -58,11 +58,11 @@ prop_coverage_likely_incomplete() -> {filesize_t(), segsizes_t(), filesize_t()}, ?FORALL( Fragments, - noshrink(fragments_t(Filesize, Segsizes, Hole)), + noshrink(segments_t(Filesize, Segsizes, Hole)), ?TIMEOUT( ?COVERAGE_TIMEOUT, begin - ASM1 = append_fragments(mk_assembly(Filesize), Fragments), + ASM1 = append_segments(mk_assembly(Filesize), Fragments), {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), measure( #{"Fragments" => length(Fragments), "Time" => Time}, @@ -85,17 +85,19 @@ prop_coverage_complete() -> {Filesize, Segsizes}, {filesize_t(), ?SUCHTHAT([BaseSegsize | _], segsizes_t(), BaseSegsize > 0)}, ?FORALL( - {Fragments, MaxCoverage}, - noshrink({fragments_t(Filesize, Segsizes), coverage_t(Filesize, Segsizes)}), + {Fragments, RemoteNode}, + noshrink({segments_t(Filesize, Segsizes), remote_node_t()}), begin % Ensure that we have complete coverage - ASM1 = append_fragments(mk_assembly(Filesize), Fragments ++ MaxCoverage), - {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), + ASM1 = mk_assembly(Filesize), + ASM2 = append_coverage(ASM1, RemoteNode, Filesize, Segsizes), + ASM3 = append_segments(ASM2, Fragments), + {Time, ASM4} = timer:tc(emqx_ft_assembly, update, [ASM3]), measure( - #{"CoverageMax" => length(MaxCoverage), "Time" => Time}, - case emqx_ft_assembly:status(ASM2) of + #{"CoverageMax" => nsegs(Filesize, Segsizes), "Time" => Time}, + case emqx_ft_assembly:status(ASM4) of complete -> - Coverage = emqx_ft_assembly:coverage(ASM2), + Coverage = emqx_ft_assembly:coverage(ASM4), measure( #{"Coverage" => length(Coverage)}, is_coverage_complete(Coverage) @@ -127,15 +129,26 @@ is_coverage_complete( mk_assembly(Filesize) -> emqx_ft_assembly:append(emqx_ft_assembly:new(Filesize), node(), mk_filemeta(Filesize)). -append_fragments(ASMIn, Fragments) -> +append_segments(ASMIn, Fragments) -> lists:foldl( - fun({Node, Frag}, ASM) -> - emqx_ft_assembly:append(ASM, Node, Frag) + fun({Node, {Offset, Size}}, ASM) -> + emqx_ft_assembly:append(ASM, Node, mk_segment(Offset, Size)) end, ASMIn, Fragments ). +append_coverage(ASM, Node, Filesize, Segsizes = [BaseSegsize | _]) -> + append_coverage(ASM, Node, Filesize, BaseSegsize, 0, nsegs(Filesize, Segsizes)). + +append_coverage(ASM, Node, Filesize, Segsize, I, NSegs) when I < NSegs -> + Offset = I * Segsize, + Size = min(Segsize, Filesize - Offset), + ASMNext = emqx_ft_assembly:append(ASM, Node, mk_segment(Offset, Size)), + append_coverage(ASMNext, Node, Filesize, Segsize, I + 1, NSegs); +append_coverage(ASM, _Node, _Filesize, _Segsize, _, _NSegs) -> + ASM. + mk_filemeta(Filesize) -> #{ path => "MANIFEST.json", @@ -148,39 +161,33 @@ mk_segment(Offset, Size) -> fragment => {segment, #{offset => Offset, size => Size}} }. -fragments_t(Filesize, Segsizes = [BaseSegsize | _]) -> - NSegs = Filesize / max(1, BaseSegsize), - scaled(1 + NSegs, list({node_t(), fragment_t(Filesize, Segsizes)})). +nsegs(Filesize, [BaseSegsize | _]) -> + Filesize div max(1, BaseSegsize) + 1. -fragments_t(Filesize, Segsizes = [BaseSegsize | _], Hole) -> - NSegs = Filesize / max(1, BaseSegsize), - scaled(1 + NSegs, list({node_t(), fragment_t(Filesize, Segsizes, Hole)})). +segments_t(Filesize, Segsizes) -> + scaled(nsegs(Filesize, Segsizes), list({node_t(), segment_t(Filesize, Segsizes)})). -fragment_t(Filesize, Segsizes, Hole) -> +segments_t(Filesize, Segsizes, Hole) -> + scaled(nsegs(Filesize, Segsizes), list({node_t(), segment_t(Filesize, Segsizes, Hole)})). + +segment_t(Filesize, Segsizes, Hole) -> ?SUCHTHATMAYBE( - #{fragment := {segment, #{offset := Offset, size := Size}}}, - fragment_t(Filesize, Segsizes), + {Offset, Size}, + segment_t(Filesize, Segsizes), (Hole rem Filesize) =< Offset orelse (Hole rem Filesize) > (Offset + Size) ). -fragment_t(Filesize, Segsizes) -> +segment_t(Filesize, Segsizes) -> ?LET( Segsize, oneof(Segsizes), ?LET( Index, range(0, Filesize div max(1, Segsize)), - mk_segment(Index * Segsize, min(Segsize, Filesize - (Index * Segsize))) + {Index * Segsize, min(Segsize, Filesize - (Index * Segsize))} ) ). -coverage_t(Filesize, [Segsize | _]) -> - NSegs = Filesize div max(1, Segsize), - [ - {remote_node_t(), mk_segment(I * Segsize, min(Segsize, Filesize - (I * Segsize)))} - || I <- lists:seq(0, NSegs) - ]. - filesize_t() -> scaled(4000, non_neg_integer()). From e9f98adca26b9cb13366bfadc6ddfde7337d8f1b Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 2 Mar 2023 18:53:19 +0300 Subject: [PATCH 059/197] test: make content_gen emit meta with each chunk Which will also contain total content size, to increase this tool's utility. --- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_content_gen.erl | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index a896922d1..8df471deb 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -109,7 +109,7 @@ t_assemble_complete_local_transfer(Config) -> ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta), _ = emqx_ft_content_gen:consume( Gen, - fun({Content, SegmentNum, _SegmentCount}) -> + fun({Content, SegmentNum, _Meta}) -> Offset = (SegmentNum - 1) * SegmentSize, ?assertEqual( ok, diff --git a/apps/emqx_ft/test/emqx_ft_content_gen.erl b/apps/emqx_ft/test/emqx_ft_content_gen.erl index bd24d8c94..286c1a588 100644 --- a/apps/emqx_ft/test/emqx_ft_content_gen.erl +++ b/apps/emqx_ft/test/emqx_ft_content_gen.erl @@ -42,7 +42,7 @@ -type payload() :: {Seed :: term(), Size :: integer()}. -type binary_payload() :: { - binary(), _ChunkNum :: non_neg_integer(), _ChunkCnt :: non_neg_integer() + binary(), _ChunkNum :: non_neg_integer(), _Meta :: #{} }. -type cont(Data) :: @@ -200,7 +200,10 @@ generate_chunk(Seed, Offset, ChunkSize, Size) -> || I <- lists:seq(Offset div 16, To div 16) ]), ChunkNum = Offset div ChunkSize + 1, - ChunkCnt = ceil(Size / ChunkSize), + Meta = #{ + chunk_size => ChunkSize, + chunk_count => ceil(Size / ChunkSize) + }, Chunk = case Offset + ChunkSize of NextOffset when NextOffset > Size -> @@ -208,7 +211,7 @@ generate_chunk(Seed, Offset, ChunkSize, Size) -> _ -> Payload end, - {Chunk, ChunkNum, ChunkCnt}. + {Chunk, ChunkNum, Meta}. %% @doc First argument is a chunk number, the second one is a seed. %% This implementation is hardly efficient, but it was chosen for From 715816e67bbf6ed206d02269c1c53c5d947aae35 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 2 Mar 2023 19:05:48 +0300 Subject: [PATCH 060/197] feat(ft): add GC logic and process for the FS storage backend --- apps/emqx/test/emqx_common_test_helpers.erl | 1 + apps/emqx_ft/include/emqx_ft_storage_fs.hrl | 29 ++ apps/emqx_ft/src/emqx_ft_assembler.erl | 17 + apps/emqx_ft/src/emqx_ft_conf.erl | 29 ++ apps/emqx_ft/src/emqx_ft_storage.erl | 21 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 80 ++++- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 337 ++++++++++++++++++ apps/emqx_ft/src/emqx_ft_sup.erl | 2 +- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 205 +++++++++++ 9 files changed, 702 insertions(+), 19 deletions(-) create mode 100644 apps/emqx_ft/include/emqx_ft_storage_fs.hrl create mode 100644 apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl create mode 100644 apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 5dd587a4e..16920d60b 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -30,6 +30,7 @@ start_apps/1, start_apps/2, start_apps/3, + start_app/2, stop_apps/1, reload/2, app_path/2, diff --git a/apps/emqx_ft/include/emqx_ft_storage_fs.hrl b/apps/emqx_ft/include/emqx_ft_storage_fs.hrl new file mode 100644 index 000000000..72ebe586a --- /dev/null +++ b/apps/emqx_ft/include/emqx_ft_storage_fs.hrl @@ -0,0 +1,29 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_FT_STORAGE_FS_HRL). +-define(EMQX_FT_STORAGE_FS_HRL, true). + +-record(gcstats, { + started_at :: integer(), + finished_at :: integer() | undefined, + files = 0 :: non_neg_integer(), + directories = 0 :: non_neg_integer(), + space = 0 :: non_neg_integer(), + errors = #{} :: #{_GCSubject => {error, _}} +}). + +-endif. diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 9e3ddfcbd..38ccf13ac 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -126,6 +126,7 @@ handle_event(internal, _, {assemble, []}, St = #st{}) -> handle_event(internal, _, complete, St = #st{assembly = Asm, file = Handle}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), Result = emqx_ft_storage_fs:complete(St#st.storage, St#st.transfer, Filemeta, Handle), + ok = maybe_garbage_collect(Result, St), {stop, {shutdown, Result}}. pread(Node, Segment, St) when Node =:= node() -> @@ -135,5 +136,21 @@ pread(Node, Segment, St) -> %% +maybe_garbage_collect(ok, St = #st{storage = Storage, transfer = Transfer}) -> + Nodes = get_coverage_nodes(St), + emqx_ft_storage_fs_gc:collect(Storage, Transfer, Nodes); +maybe_garbage_collect({error, _}, _St) -> + ok. + +get_coverage_nodes(St) -> + Coverage = emqx_ft_assembly:coverage(St#st.assembly), + ordsets:to_list( + lists:foldl( + fun({Node, _Segment}, Acc) -> ordsets:add_element(Node, Acc) end, + ordsets:new(), + Coverage + ) + ). + segsize(#{fragment := {segment, Info}}) -> maps:get(size, Info). diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index d56dd8d32..444462716 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -20,6 +20,11 @@ -behaviour(emqx_config_handler). +%% Accessors +-export([storage/0]). +-export([gc_interval/1]). +-export([segments_ttl/1]). + %% Load/Unload -export([ load/0, @@ -32,6 +37,30 @@ post_config_update/5 ]). +-type milliseconds() :: non_neg_integer(). +-type seconds() :: non_neg_integer(). + +%%-------------------------------------------------------------------- +%% Accessors +%%-------------------------------------------------------------------- + +-spec storage() -> _Storage | disabled. +storage() -> + emqx_config:get([file_transfer, storage], disabled). + +-spec gc_interval(_Storage) -> milliseconds(). +gc_interval(_Storage) -> + % TODO: config wiring + application:get_env(emqx_ft, gc_interval, timer:minutes(10)). + +-spec segments_ttl(_Storage) -> {_Min :: seconds(), _Max :: seconds()}. +segments_ttl(_Storage) -> + % TODO: config wiring + { + application:get_env(emqx_ft, min_segments_ttl, 60), + application:get_env(emqx_ft, max_segments_ttl, 72 * 3600) + }. + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 7a95a0454..0b8c38736 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -18,6 +18,8 @@ -export( [ + child_spec/0, + store_filemeta/2, store_segment/2, assemble/2, @@ -64,6 +66,17 @@ %% API %%-------------------------------------------------------------------- +-spec child_spec() -> + [supervisor:child_spec()]. +child_spec() -> + try + Mod = mod(), + Mod:child_spec(storage()) + catch + error:disabled -> []; + error:undef -> [] + end. + -spec store_filemeta(emqx_ft:transfer(), emqx_ft:filemeta()) -> ok | {async, pid()} | {error, term()}. store_filemeta(Transfer, FileMeta) -> @@ -99,6 +112,8 @@ with_storage_type(Type, Fun, Args) -> #{type := Type} -> Mod = mod(Storage), apply(Mod, Fun, [Storage | Args]); + disabled -> + {error, disabled}; _ -> {error, {invalid_storage_type, Type}} end. @@ -108,7 +123,7 @@ with_storage_type(Type, Fun, Args) -> %%-------------------------------------------------------------------- storage() -> - emqx_config:get([file_transfer, storage]). + emqx_ft_conf:storage(). mod() -> mod(storage()). @@ -116,6 +131,8 @@ mod() -> mod(Storage) -> case Storage of #{type := local} -> - emqx_ft_storage_fs + emqx_ft_storage_fs; + disabled -> + error(disabled) % emqx_ft_storage_dummy end. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 2cc19d2a2..b8aef5276 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -14,6 +14,12 @@ %% limitations under the License. %%-------------------------------------------------------------------- +%% Filesystem storage backend +%% +%% NOTE +%% If you plan to change storage layout please consult `emqx_ft_storage_fs_gc` +%% to see how much it would break or impair GC. + -module(emqx_ft_storage_fs). -behaviour(emqx_ft_storage). @@ -21,14 +27,22 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). +-export([child_spec/1]). + -export([store_filemeta/3]). -export([store_segment/3]). +-export([read_filemeta/2]). -export([list/3]). -export([pread/5]). -export([assemble/3]). -export([transfers/1]). +% GC API +% TODO: This is quickly becomes hairy. +-export([get_subdir/2]). +-export([get_subdir/3]). + -export([ready_transfers_local/1]). -export([get_ready_transfer_local/3]). @@ -40,6 +54,7 @@ -export([write/2]). -export([discard/1]). +-export_type([storage/0]). -export_type([filefrag/1]). -export_type([filefrag/0]). -export_type([transferinfo/0]). @@ -79,7 +94,9 @@ -define(MANIFEST, "MANIFEST.json"). -define(SEGMENT, "SEG"). --type storage() :: emqx_ft_storage:storage(). +-type storage() :: #{ + root => file:name() +}. -type file_error() :: file:posix() @@ -88,13 +105,25 @@ %% System limit (e.g. number of ports) reached. | system_limit. +%% Related resources childspecs +-spec child_spec(storage()) -> + [supervisor:child_spec()]. +child_spec(Storage) -> + [ + #{ + id => emqx_ft_storage_fs_gc, + start => {emqx_ft_storage_fs_gc, start_link, [Storage]}, + restart => permanent + } + ]. + %% Store manifest in the backing filesystem. %% Atomic operation. -spec store_filemeta(storage(), transfer(), filemeta()) -> % Quota? Some lower level errors? ok | {error, conflict} | {error, file_error()}. store_filemeta(Storage, Transfer, Meta) -> - Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], ?MANIFEST), + Filepath = mk_filepath(Storage, Transfer, get_subdirs_for(fragment), ?MANIFEST), case read_file(Filepath, fun decode_filemeta/1) of {ok, Meta} -> _ = touch_file(Filepath), @@ -119,9 +148,16 @@ store_filemeta(Storage, Transfer, Meta) -> % Quota? Some lower level errors? ok | {error, file_error()}. store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> - Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], mk_segment_filename(Segment)), + Filename = mk_segment_filename(Segment), + Filepath = mk_filepath(Storage, Transfer, get_subdirs_for(fragment), Filename), write_file_atomic(Storage, Transfer, Filepath, Content). +-spec read_filemeta(storage(), transfer()) -> + {ok, filefrag({filemeta, filemeta()})} | {error, corrupted} | {error, file_error()}. +read_filemeta(Storage, Transfer) -> + Filepath = mk_filepath(Storage, Transfer, get_subdirs_for(fragment), ?MANIFEST), + read_file(Filepath, fun decode_filemeta/1). + -spec list(storage(), transfer(), _What :: fragment | result) -> % Some lower level errors? {error, notfound}? % Result will contain zero or only one filemeta. @@ -143,11 +179,6 @@ list(Storage, Transfer, What) -> Error end. -get_subdirs_for(fragment) -> - [?FRAGDIR]; -get_subdirs_for(result) -> - [?RESULTDIR]. - get_filefrag_fun_for(fragment) -> fun mk_filefrag/2; get_filefrag_fun_for(result) -> @@ -329,6 +360,23 @@ read_transferinfo(Storage, Transfer, Acc) -> Acc end. +-spec get_subdir(storage(), transfer()) -> + file:name(). +get_subdir(Storage, Transfer) -> + mk_filedir(Storage, Transfer, []). + +-spec get_subdir(storage(), transfer(), fragment | temporary | result) -> + file:name(). +get_subdir(Storage, Transfer, What) -> + mk_filedir(Storage, Transfer, get_subdirs_for(What)). + +get_subdirs_for(fragment) -> + [?FRAGDIR]; +get_subdirs_for(temporary) -> + [?TEMPDIR]; +get_subdirs_for(result) -> + [?RESULTDIR]. + %% -type handle() :: {file:name(), io:device(), crypto:hash_state()}. @@ -341,7 +389,7 @@ open_file(Storage, Transfer, Filemeta) -> _ = filelib:ensure_dir(TempFilepath), case file:open(TempFilepath, [write, raw, binary]) of {ok, Handle} -> - _ = file:truncate(Handle), + % TODO: preserve filemeta {ok, {TempFilepath, Handle, init_checksum(Filemeta)}}; {error, _} = Error -> Error @@ -359,8 +407,8 @@ write({Filepath, IoDevice, Ctx}, IoData) -> -spec complete(storage(), transfer(), filemeta(), handle()) -> ok | {error, {checksum, _Algo, _Computed}} | {error, file_error()}. -complete(Storage, Transfer, Filemeta, Handle = {Filepath, IoDevice, Ctx}) -> - TargetFilepath = mk_filepath(Storage, Transfer, [?RESULTDIR], maps:get(name, Filemeta)), +complete(Storage, Transfer, Filemeta = #{name := Filename}, Handle = {Filepath, IoDevice, Ctx}) -> + TargetFilepath = mk_filepath(Storage, Transfer, get_subdirs_for(result), Filename), case verify_checksum(Ctx, Filemeta) of ok -> ok = file:close(IoDevice), @@ -491,7 +539,7 @@ write_file_atomic(Storage, Transfer, Filepath, Content) when is_binary(Content) mk_temp_filepath(Storage, Transfer, Filename) -> Unique = erlang:unique_integer([positive]), - filename:join(mk_filedir(Storage, Transfer, [?TEMPDIR]), mk_filename([Unique, ".", Filename])). + filename:join(get_subdir(Storage, Transfer, temporary), mk_filename([Unique, ".", Filename])). mk_filename(Comps) -> lists:append(lists:map(fun mk_filename_component/1, Comps)). @@ -516,9 +564,9 @@ filtermap_files(Fun, Dirname, Filenames) -> lists:filtermap(fun(Filename) -> Fun(Dirname, Filename) end, Filenames). mk_filefrag(Dirname, Filename = ?MANIFEST) -> - mk_filefrag(Dirname, Filename, filemeta, fun read_filemeta/2); + mk_filefrag(Dirname, Filename, filemeta, fun read_frag_filemeta/2); mk_filefrag(Dirname, Filename = ?SEGMENT ++ _) -> - mk_filefrag(Dirname, Filename, segment, fun read_segmentinfo/2); + mk_filefrag(Dirname, Filename, segment, fun read_frag_segmentinfo/2); mk_filefrag(_Dirname, _Filename) -> ?tp(warning, "rogue_file_found", #{ directory => _Dirname, @@ -554,10 +602,10 @@ mk_filefrag(Dirname, Filename, Tag, Fun) -> false end. -read_filemeta(_Filename, Filepath) -> +read_frag_filemeta(_Filename, Filepath) -> read_file(Filepath, fun decode_filemeta/1). -read_segmentinfo(Filename, _Filepath) -> +read_frag_segmentinfo(Filename, _Filepath) -> break_segment_filename(Filename). filename_to_binary(S) when is_list(S) -> unicode:characters_to_binary(S); diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl new file mode 100644 index 000000000..6d493337a --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -0,0 +1,337 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Filesystem storage GC +%% +%% This is conceptually a part of the Filesystem storage backend, even +%% though it's tied to the backend module with somewhat narrow interface. + +-module(emqx_ft_storage_fs_gc). + +-include_lib("emqx_ft/include/emqx_ft_storage_fs.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). + +-export([start_link/1]). + +-export([collect/1]). +-export([collect/3]). +-export([reset/1]). + +-behaviour(gen_server). +-export([init/1]). +-export([handle_call/3]). +-export([handle_cast/2]). +-export([handle_info/2]). + +-record(st, { + storage :: emqx_ft_storage_fs:storage(), + next_gc_timer :: maybe(reference()), + last_gc :: maybe(gcstats()) +}). + +-type gcstats() :: #gcstats{}. + +%% + +start_link(Storage) -> + gen_server:start_link(mk_server_ref(Storage), ?MODULE, Storage, []). + +-spec collect(emqx_ft_storage_fs:storage()) -> gcstats(). +collect(Storage) -> + gen_server:call(mk_server_ref(Storage), {collect, erlang:system_time()}, infinity). + +-spec reset(emqx_ft_storage_fs:storage()) -> ok. +reset(Storage) -> + gen_server:cast(mk_server_ref(Storage), reset). + +collect(Storage, Transfer, Nodes) -> + gen_server:cast(mk_server_ref(Storage), {collect, Transfer, Nodes}). + +mk_server_ref(Storage) -> + % TODO + {via, gproc, {n, l, {?MODULE, get_storage_root(Storage)}}}. + +%% + +init(Storage) -> + St = #st{storage = Storage}, + {ok, start_timer(St)}. + +handle_call({collect, CalledAt}, _From, St) -> + StNext = maybe_collect_garbage(CalledAt, St), + {reply, StNext#st.last_gc, StNext}; +handle_call(Call, From, St) -> + ?SLOG(error, #{msg => "unexpected_call", call => Call, from => From}), + {noreply, St}. + +% TODO +% handle_cast({collect, Transfer, [Node | Rest]}, St) -> +% ok = do_collect_transfer(Transfer, Node, St), +% ok = collect(self(), Transfer, Rest), +% {noreply, St}; +handle_cast(reset, St) -> + {noreply, reset_timer(St)}; +handle_cast(Cast, St) -> + ?SLOG(error, #{msg => "unexpected_cast", cast => Cast}), + {noreply, St}. + +handle_info({timeout, TRef, collect}, St = #st{next_gc_timer = TRef}) -> + StNext = do_collect_garbage(St), + {noreply, start_timer(StNext#st{next_gc_timer = undefined})}. + +% do_collect_transfer(Transfer, Node, St = #st{storage = Storage}) when Node == node() -> +% Stats = try_collect_transfer(Storage, Transfer, complete, init_gcstats()), +% ok = maybe_report(Stats, St), +% ok. + +maybe_collect_garbage(_CalledAt, St = #st{last_gc = undefined}) -> + do_collect_garbage(St); +maybe_collect_garbage(CalledAt, St = #st{last_gc = #gcstats{finished_at = FinishedAt}}) -> + case FinishedAt > CalledAt of + true -> + St; + false -> + reset_timer(do_collect_garbage(St)) + end. + +do_collect_garbage(St = #st{storage = Storage}) -> + Stats = collect_garbage(Storage), + ok = maybe_report(Stats, St), + St#st{last_gc = Stats}. + +maybe_report(#gcstats{errors = Errors}, #st{storage = Storage}) when map_size(Errors) > 0 -> + ?tp(warning, "garbage_collection_errors", #{errors => Errors, storage => Storage}); +maybe_report(#gcstats{} = _Stats, #st{storage = _Storage}) -> + ?tp(garbage_collection, #{stats => _Stats, storage => _Storage}). + +start_timer(St = #st{next_gc_timer = undefined}) -> + Delay = emqx_ft_conf:gc_interval(St#st.storage), + St#st{next_gc_timer = emqx_misc:start_timer(Delay, collect)}. + +reset_timer(St = #st{next_gc_timer = undefined}) -> + start_timer(St); +reset_timer(St = #st{next_gc_timer = TRef}) -> + ok = emqx_misc:cancel_timer(TRef), + start_timer(St#st{next_gc_timer = undefined}). + +%% + +collect_garbage(Storage) -> + Stats = init_gcstats(), + {ok, Transfers} = emqx_ft_storage_fs:transfers(Storage), + collect_garbage(Storage, Transfers, Stats). + +collect_garbage(Storage, Transfers, Stats) -> + finish_gcstats( + maps:fold( + fun(Transfer, TransferInfo, StatsAcc) -> + % TODO: throttling? + try_collect_transfer(Storage, Transfer, TransferInfo, StatsAcc) + end, + Stats, + Transfers + ) + ). + +try_collect_transfer(Storage, Transfer, #{status := complete}, Stats) -> + % File transfer is complete. + % We should be good to delete fragments and temporary files with their respective + % directories altogether. + % TODO: file expiration + {_, Stats1} = collect_fragments(Storage, Transfer, Stats), + {_, Stats2} = collect_tempfiles(Storage, Transfer, Stats1), + Stats2; +try_collect_transfer(Storage, Transfer, #{status := incomplete}, Stats) -> + % File transfer is still incomplete. + % Any outdated fragments and temporary files should be collectable. As a kind of + % heuristic we only delete transfer directory itself only if it is also outdated + % _and was empty at the start of GC_, as a precaution against races between + % writers and GCs. + TTL = get_segments_ttl(Storage, Transfer), + Cutoff = erlang:system_time(second) + TTL, + {FragCleaned, Stats1} = collect_outdated_fragments(Storage, Transfer, Cutoff, Stats), + {TempCleaned, Stats2} = collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats1), + % TODO: collect empty directories separately + case FragCleaned and TempCleaned of + true -> + collect_transfer_directory(Storage, Transfer, Stats2); + false -> + Stats2 + end. + +collect_fragments(Storage, Transfer, Stats) -> + Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, fragment), + collect_filepath(Dirname, true, Stats). + +collect_tempfiles(Storage, Transfer, Stats) -> + Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, temporary), + collect_filepath(Dirname, true, Stats). + +collect_outdated_fragments(Storage, Transfer, Cutoff, Stats) -> + Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, fragment), + Filter = fun(_Filepath, #file_info{mtime = ModifiedAt}) -> ModifiedAt < Cutoff end, + collect_filepath(Dirname, Filter, Stats). + +collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats) -> + Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, temporary), + Filter = fun(_Filepath, #file_info{mtime = ModifiedAt}) -> ModifiedAt < Cutoff end, + collect_filepath(Dirname, Filter, Stats). + +collect_transfer_directory(Storage, Transfer, Stats) -> + Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer), + StatsNext = collect_empty_directory(Dirname, Stats), + collect_parents(Dirname, StatsNext). + +collect_parents(Dirname, Stats) -> + Parent = filename:dirname(Dirname), + case file:del_dir(Parent) of + ok -> + collect_parents(Parent, account_gcstat_directory(Stats)); + {error, enoent} -> + collect_parents(Parent, Stats); + {error, eexist} -> + Stats; + {error, Reason} -> + register_gcstat_error({directory, Parent}, Reason, Stats) + end. + +% collect_outdated_fragment(#{path := Filepath, fileinfo := Fileinfo}, Cutoff, Stats) -> +% case Fileinfo#file_info.mtime of +% ModifiedAt when ModifiedAt < Cutoff -> +% collect_filepath(Filepath, Fileinfo, Stats); +% _ -> +% Stats +% end. + +-spec collect_filepath(file:name(), Filter, gcstats()) -> {boolean(), gcstats()} when + Filter :: boolean() | fun((file:name(), file:file_info()) -> boolean()). +collect_filepath(Filepath, Filter, Stats) -> + case file:read_file_info(Filepath) of + {ok, Fileinfo} -> + collect_filepath(Filepath, Fileinfo, Filter, Stats); + {error, enoent} -> + {true, Stats}; + {error, Reason} -> + {false, register_gcstat_error({path, Filepath}, Reason, Stats)} + end. + +collect_filepath(Filepath, #file_info{type = directory} = Fileinfo, Filter, Stats) -> + collect_directory(Filepath, Fileinfo, Filter, Stats); +collect_filepath(Filepath, #file_info{type = regular} = Fileinfo, Filter, Stats) -> + case filter_filepath(Filter, Filepath, Fileinfo) andalso file:delete(Filepath, [raw]) of + false -> + {false, Stats}; + ok -> + {true, account_gcstat(Fileinfo, Stats)}; + {error, enoent} -> + {true, Stats}; + {error, Reason} -> + {false, register_gcstat_error({file, Filepath}, Reason, Stats)} + end; +collect_filepath(Filepath, Fileinfo, _Filter, Stats) -> + {false, register_gcstat_error({file, Filepath}, {unexpected, Fileinfo}, Stats)}. + +collect_directory(Dirpath, Fileinfo, Filter, Stats) -> + case file:list_dir(Dirpath) of + {ok, Filenames} -> + {Clean, StatsNext} = collect_files(Dirpath, Filenames, Filter, Stats), + case Clean andalso filter_filepath(Filter, Dirpath, Fileinfo) of + true -> + {true, collect_empty_directory(Dirpath, StatsNext)}; + _ -> + {false, StatsNext} + end; + {error, Reason} -> + {false, register_gcstat_error({directory, Dirpath}, Reason, Stats)} + end. + +collect_files(Dirname, Filenames, Filter, Stats) -> + lists:foldl( + fun(Filename, {Complete, StatsAcc}) -> + Filepath = filename:join(Dirname, Filename), + {Collected, StatsNext} = collect_filepath(Filepath, Filter, StatsAcc), + {Collected andalso Complete, StatsNext} + end, + {true, Stats}, + Filenames + ). + +collect_empty_directory(Dirpath, Stats) -> + case file:del_dir(Dirpath) of + ok -> + account_gcstat_directory(Stats); + {error, enoent} -> + Stats; + {error, Reason} -> + register_gcstat_error({directory, Dirpath}, Reason, Stats) + end. + +filter_filepath(Filter, _, _) when is_boolean(Filter) -> + Filter; +filter_filepath(Filter, Filepath, Fileinfo) when is_function(Filter) -> + Filter(Filepath, Fileinfo). + +get_segments_ttl(Storage, Transfer) -> + {MinTTL, MaxTTL} = emqx_ft_conf:segments_ttl(Storage), + clamp(MinTTL, MaxTTL, try_get_filemeta_ttl(Storage, Transfer)). + +try_get_filemeta_ttl(Storage, Transfer) -> + case emqx_ft_storage_fs:read_filemeta(Storage, Transfer) of + {ok, Filemeta} -> + maps:get(segments_ttl, Filemeta, undefined); + {error, _} -> + undefined + end. + +clamp(Min, Max, V) -> + min(Max, max(Min, V)). + +% try_collect(_Subject, ok = Result, Then, _Stats) -> +% Then(Result); +% try_collect(_Subject, {ok, Result}, Then, _Stats) -> +% Then(Result); +% try_collect(Subject, {error, _} = Error, _Then, Stats) -> +% register_gcstat_error(Subject, Error, Stats). + +%% + +init_gcstats() -> + #gcstats{started_at = erlang:system_time()}. + +finish_gcstats(Stats) -> + Stats#gcstats{finished_at = erlang:system_time()}. + +account_gcstat(Fileinfo, Stats = #gcstats{files = Files, space = Space}) -> + Stats#gcstats{ + files = Files + 1, + space = Space + Fileinfo#file_info.size + }. + +account_gcstat_directory(Stats = #gcstats{directories = Directories}) -> + Stats#gcstats{ + directories = Directories + 1 + }. + +register_gcstat_error(Subject, Error, Stats = #gcstats{errors = Errors}) -> + Stats#gcstats{errors = Errors#{Subject => Error}}. + +%% + +get_storage_root(Storage) -> + maps:get(root, Storage, filename:join(emqx:data_dir(), "file_transfer")). diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index 8d388814c..3c28eae30 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -61,5 +61,5 @@ init([]) -> modules => [emqx_ft_responder_sup] }, - ChildSpecs = [Responder, AssemblerSup, FileReaderSup], + ChildSpecs = [Responder, AssemblerSup, FileReaderSup | emqx_ft_storage:child_spec()], {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl new file mode 100644 index 000000000..c7fffbacd --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -0,0 +1,205 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_gc_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx_ft/include/emqx_ft_storage_fs.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("snabbkaffe/include/test_macros.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + _ = application:load(emqx_ft), + ok = emqx_common_test_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([]), + ok. + +init_per_testcase(TC, Config) -> + _ = application:unset_env(emqx_ft, gc_interval), + _ = application:unset_env(emqx_ft, min_segments_ttl), + _ = application:unset_env(emqx_ft, max_segments_ttl), + ok = emqx_common_test_helpers:start_app( + emqx_ft, + fun(emqx_ft) -> + ok = emqx_config:put([file_transfer, storage], #{ + type => local, + root => mk_root(TC, Config) + }) + end + ), + Config. + +end_per_testcase(_TC, _Config) -> + ok = application:stop(emqx_ft), + ok. + +mk_root(TC, Config) -> + filename:join([?config(priv_dir, Config), <<"file_transfer">>, TC, atom_to_binary(node())]). + +%% + +t_gc_triggers_periodically(_Config) -> + Interval = 1000, + ok = application:set_env(emqx_ft, gc_interval, Interval), + ok = emqx_ft_storage_fs_gc:reset(emqx_ft_conf:storage()), + ?check_trace( + timer:sleep(Interval * 3), + fun(Trace) -> + [Event, _ | _] = ?of_kind(garbage_collection, Trace), + ?assertMatch( + #{ + stats := #gcstats{ + files = 0, + directories = 0, + space = 0, + errors = #{} = Errors + } + } when map_size(Errors) == 0, + Event + ) + end + ). + +t_gc_triggers_manually(_Config) -> + ?check_trace( + ?assertMatch( + #gcstats{files = 0, directories = 0, space = 0, errors = #{} = Errors} when + map_size(Errors) == 0, + emqx_ft_storage_fs_gc:collect(emqx_ft_conf:storage()) + ), + fun(Trace) -> + [Event] = ?of_kind(garbage_collection, Trace), + ?assertMatch( + #{stats := #gcstats{}}, + Event + ) + end + ). + +t_gc_complete_transfers(_Config) -> + Storage = emqx_ft_conf:storage(), + Transfers = [ + { + T1 = {<<"client1">>, mk_file_id()}, + "cat.cur", + emqx_ft_content_gen:new({?LINE, S1 = 42}, SS1 = 16) + }, + { + T2 = {<<"client2">>, mk_file_id()}, + "cat.ico", + emqx_ft_content_gen:new({?LINE, S2 = 420}, SS2 = 64) + }, + { + T3 = {<<"client42">>, mk_file_id()}, + "cat.jpg", + emqx_ft_content_gen:new({?LINE, S3 = 42000}, SS3 = 1024) + } + ], + % 1. Start all transfers + TransferSizes = emqx_misc:pmap( + fun(Transfer) -> start_transfer(Storage, Transfer) end, + Transfers + ), + ?assertEqual([S1, S2, S3], TransferSizes), + ?assertMatch( + #gcstats{files = 0, directories = 0, errors = #{} = Es} when map_size(Es) == 0, + emqx_ft_storage_fs_gc:collect(Storage) + ), + % 2. Complete just the first transfer + ?assertEqual( + ok, + complete_transfer(Storage, T1, S1) + ), + GCFiles1 = ceil(S1 / SS1) + 1, + ?assertMatch( + #gcstats{ + files = GCFiles1, + directories = 2, + space = Space, + errors = #{} = Es + } when Space > S1 andalso map_size(Es) == 0, + emqx_ft_storage_fs_gc:collect(Storage) + ), + % 3. Complete rest of transfers + ?assertEqual( + [ok, ok], + emqx_misc:pmap( + fun({Transfer, Size}) -> complete_transfer(Storage, Transfer, Size) end, + [{T2, S2}, {T3, S3}] + ) + ), + GCFiles2 = ceil(S2 / SS2) + 1, + GCFiles3 = ceil(S3 / SS3) + 1, + ?assertMatch( + #gcstats{ + files = Files, + directories = 4, + space = Space, + errors = #{} = Es + } when + Files == (GCFiles2 + GCFiles3) andalso + Space > (S2 + S3) andalso + map_size(Es) == 0, + emqx_ft_storage_fs_gc:collect(Storage) + ). + +start_transfer(Storage, {Transfer, Name, Gen}) -> + Meta = #{ + name => Name, + segments_ttl => 10 + }, + ?assertEqual( + ok, + emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta) + ), + emqx_ft_content_gen:fold( + fun({Content, SegmentNum, #{chunk_size := SegmentSize}}, _Transferred) -> + Offset = (SegmentNum - 1) * SegmentSize, + ?assertEqual( + ok, + emqx_ft_storage_fs:store_segment(Storage, Transfer, {Offset, Content}) + ), + Offset + byte_size(Content) + end, + 0, + Gen + ). + +complete_transfer(Storage, Transfer, Size) -> + complete_transfer(Storage, Transfer, Size, 100). + +complete_transfer(Storage, Transfer, Size, Timeout) -> + {async, Pid} = emqx_ft_storage_fs:assemble(Storage, Transfer, Size), + MRef = erlang:monitor(process, Pid), + Pid ! kickoff, + receive + {'DOWN', MRef, process, Pid, {shutdown, Result}} -> + Result + after Timeout -> + ct:fail("Assembler did not finish in time") + end. + +mk_file_id() -> + emqx_guid:to_hexstr(emqx_guid:gen()). From 15dc7c3e845febb9506d093c43dc1bb618a0e9d8 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 30 Mar 2023 14:05:35 +0300 Subject: [PATCH 061/197] fix(test): fix node helper usage --- apps/emqx/test/emqx_common_test_helpers.erl | 3 +++ apps/emqx_ft/test/emqx_ft_test_helpers.erl | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 16920d60b..e01f18a51 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -244,6 +244,9 @@ do_render_app_config(App, Schema, ConfigFile, Opts) -> copy_certs(App, RenderedConfigFile), ok. +start_app(App, SpecAppConfig) -> + start_app(App, SpecAppConfig, #{}). + start_app(App, SpecAppConfig, Opts) -> render_and_load_app_config(App, Opts), SpecAppConfig(App), diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index 956e63553..f054be762 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -42,7 +42,7 @@ start_additional_node(Config, Name) -> stop_additional_node(Node) -> ok = rpc:call(Node, ekka, leave, []), ok = rpc:call(Node, emqx_common_test_helpers, stop_apps, [[emqx_ft]]), - {ok, _} = emqx_common_test_helpers:stop_slave(Node), + ok = emqx_common_test_helpers:stop_slave(Node), ok. tcp_port(Node) -> From 344799f100e822440aace128417bcec02fea5843 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 3 Mar 2023 18:00:31 +0300 Subject: [PATCH 062/197] fix(fs-gc): add testcase on incomplete transfers GC And fix logic that made GC delete incomplete segments prematurely. --- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 6 +- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 104 +++++++++++++++--- 2 files changed, 93 insertions(+), 17 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 6d493337a..4b2d2bfd6 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -164,7 +164,7 @@ try_collect_transfer(Storage, Transfer, #{status := incomplete}, Stats) -> % _and was empty at the start of GC_, as a precaution against races between % writers and GCs. TTL = get_segments_ttl(Storage, Transfer), - Cutoff = erlang:system_time(second) + TTL, + Cutoff = erlang:system_time(second) - TTL, {FragCleaned, Stats1} = collect_outdated_fragments(Storage, Transfer, Cutoff, Stats), {TempCleaned, Stats2} = collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats1), % TODO: collect empty directories separately @@ -222,7 +222,7 @@ collect_parents(Dirname, Stats) -> -spec collect_filepath(file:name(), Filter, gcstats()) -> {boolean(), gcstats()} when Filter :: boolean() | fun((file:name(), file:file_info()) -> boolean()). collect_filepath(Filepath, Filter, Stats) -> - case file:read_file_info(Filepath) of + case file:read_file_info(Filepath, [{time, posix}]) of {ok, Fileinfo} -> collect_filepath(Filepath, Fileinfo, Filter, Stats); {error, enoent} -> @@ -238,6 +238,7 @@ collect_filepath(Filepath, #file_info{type = regular} = Fileinfo, Filter, Stats) false -> {false, Stats}; ok -> + ?tp(garbage_collected_file, #{path => Filepath}), {true, account_gcstat(Fileinfo, Stats)}; {error, enoent} -> {true, Stats}; @@ -275,6 +276,7 @@ collect_files(Dirname, Filenames, Filter, Stats) -> collect_empty_directory(Dirpath, Stats) -> case file:del_dir(Dirpath) of ok -> + ?tp(garbage_collected_directory, #{path => Dirpath}), account_gcstat_directory(Stats); {error, enoent} -> Stats; diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index c7fffbacd..858cdfd90 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -60,8 +60,10 @@ mk_root(TC, Config) -> %% +-define(NSEGS(Filesize, SegmentSize), (ceil(Filesize / SegmentSize) + 1)). + t_gc_triggers_periodically(_Config) -> - Interval = 1000, + Interval = 500, ok = application:set_env(emqx_ft, gc_interval, Interval), ok = emqx_ft_storage_fs_gc:reset(emqx_ft_conf:storage()), ?check_trace( @@ -103,17 +105,17 @@ t_gc_complete_transfers(_Config) -> Transfers = [ { T1 = {<<"client1">>, mk_file_id()}, - "cat.cur", + #{name => "cat.cur", segments_ttl => 10}, emqx_ft_content_gen:new({?LINE, S1 = 42}, SS1 = 16) }, { T2 = {<<"client2">>, mk_file_id()}, - "cat.ico", + #{name => "cat.ico", segments_ttl => 10}, emqx_ft_content_gen:new({?LINE, S2 = 420}, SS2 = 64) }, { T3 = {<<"client42">>, mk_file_id()}, - "cat.jpg", + #{name => "cat.jpg", segments_ttl => 10}, emqx_ft_content_gen:new({?LINE, S3 = 42000}, SS3 = 1024) } ], @@ -132,14 +134,13 @@ t_gc_complete_transfers(_Config) -> ok, complete_transfer(Storage, T1, S1) ), - GCFiles1 = ceil(S1 / SS1) + 1, ?assertMatch( #gcstats{ - files = GCFiles1, + files = Files, directories = 2, space = Space, errors = #{} = Es - } when Space > S1 andalso map_size(Es) == 0, + } when Files == ?NSEGS(S1, SS1) andalso Space > S1 andalso map_size(Es) == 0, emqx_ft_storage_fs_gc:collect(Storage) ), % 3. Complete rest of transfers @@ -150,8 +151,6 @@ t_gc_complete_transfers(_Config) -> [{T2, S2}, {T3, S3}] ) ), - GCFiles2 = ceil(S2 / SS2) + 1, - GCFiles3 = ceil(S3 / SS3) + 1, ?assertMatch( #gcstats{ files = Files, @@ -159,17 +158,92 @@ t_gc_complete_transfers(_Config) -> space = Space, errors = #{} = Es } when - Files == (GCFiles2 + GCFiles3) andalso + Files == (?NSEGS(S2, SS2) + ?NSEGS(S3, SS3)) andalso Space > (S2 + S3) andalso map_size(Es) == 0, emqx_ft_storage_fs_gc:collect(Storage) ). -start_transfer(Storage, {Transfer, Name, Gen}) -> - Meta = #{ - name => Name, - segments_ttl => 10 - }, +t_gc_incomplete_transfers(_Config) -> + _ = application:set_env(emqx_ft, min_segments_ttl, 0), + _ = application:set_env(emqx_ft, max_segments_ttl, 4), + ok = emqx_ft_storage_fs_gc:reset(emqx_ft_conf:storage()), + Storage = emqx_ft_conf:storage(), + Transfers = [ + { + {<<"client43"/utf8>>, <<"file-🦕"/utf8>>}, + #{name => "dog.cur", segments_ttl => 1}, + emqx_ft_content_gen:new({?LINE, S1 = 123}, SS1 = 32) + }, + { + {<<"client44">>, <<"file-🦖"/utf8>>}, + #{name => "dog.ico", segments_ttl => 2}, + emqx_ft_content_gen:new({?LINE, S2 = 456}, SS2 = 64) + }, + { + {<<"client1337">>, <<"file-🦀"/utf8>>}, + #{name => "dog.jpg", segments_ttl => 3000}, + emqx_ft_content_gen:new({?LINE, S3 = 7890}, SS3 = 128) + }, + { + {<<"client31337">>, <<"file-⏳"/utf8>>}, + #{name => "dog.jpg"}, + emqx_ft_content_gen:new({?LINE, S4 = 1230}, SS4 = 256) + } + ], + % 1. Start transfers, send all the segments but don't trigger completion. + _ = emqx_misc:pmap(fun(Transfer) -> start_transfer(Storage, Transfer) end, Transfers), + ?check_trace( + begin + % 2. Enable periodic GC every 0.5 seconds. + ok = application:set_env(emqx_ft, gc_interval, 500), + ok = emqx_ft_storage_fs_gc:reset(Storage), + % 3. First we need the first transfer to be collected. + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = Files, + directories = 4, + space = Space + } + } when Files == (?NSEGS(S1, SS1)) andalso Space > S1, + 5000, + 0 + ), + % 4. Then the second one. + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = Files, + directories = 4, + space = Space + } + } when Files == (?NSEGS(S2, SS2)) andalso Space > S2, + 5000, + 0 + ), + % 5. Then transfers 3 and 4 because 3rd has too big TTL and 4th has no specific TTL. + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = Files, + directories = 4 * 2, + space = Space + } + } when Files == (?NSEGS(S3, SS3) + ?NSEGS(S4, SS4)) andalso Space > S3 + S4, + 5000, + 0 + ) + end, + [] + ). + +%% + +start_transfer(Storage, {Transfer, Meta, Gen}) -> ?assertEqual( ok, emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta) From bcd2099ce1c194f2120c9bc28eb3787a13d07be7 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 3 Mar 2023 18:02:19 +0300 Subject: [PATCH 063/197] fix(fs-gc): make deletion empty transfer directories safer --- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 23 +++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 4b2d2bfd6..75a74681f 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -196,15 +196,18 @@ collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats) -> collect_transfer_directory(Storage, Transfer, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer), StatsNext = collect_empty_directory(Dirname, Stats), - collect_parents(Dirname, StatsNext). + collect_parents(Dirname, get_storage_root(Storage), StatsNext). -collect_parents(Dirname, Stats) -> +collect_parents(Dirname, Until, Stats) -> Parent = filename:dirname(Dirname), - case file:del_dir(Parent) of + case is_same_filepath(Parent, Until) orelse file:del_dir(Parent) of + true -> + Stats; ok -> - collect_parents(Parent, account_gcstat_directory(Stats)); + ?tp(garbage_collected_directory, #{path => Dirname}), + collect_parents(Parent, Until, account_gcstat_directory(Stats)); {error, enoent} -> - collect_parents(Parent, Stats); + collect_parents(Parent, Until, Stats); {error, eexist} -> Stats; {error, Reason} -> @@ -289,6 +292,16 @@ filter_filepath(Filter, _, _) when is_boolean(Filter) -> filter_filepath(Filter, Filepath, Fileinfo) when is_function(Filter) -> Filter(Filepath, Fileinfo). +is_same_filepath(P1, P2) when is_binary(P1) andalso is_binary(P2) -> + filename:absname(P1) == filename:absname(P2); +is_same_filepath(P1, P2) when is_list(P1) andalso is_list(P2) -> + filename:absname(P1) == filename:absname(P2); +is_same_filepath(P1, P2) when is_binary(P1) -> + is_same_filepath(P1, filepath_to_binary(P2)). + +filepath_to_binary(S) -> + unicode:characters_to_binary(S, unicode, file:native_name_encoding()). + get_segments_ttl(Storage, Transfer) -> {MinTTL, MaxTTL} = emqx_ft_conf:segments_ttl(Storage), clamp(MinTTL, MaxTTL, try_get_filemeta_ttl(Storage, Transfer)). From 50c6eef2bc6eb029ce0f7d13b4afea479246beac Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 3 Mar 2023 19:03:15 +0300 Subject: [PATCH 064/197] fix(fs-gc): do not hide `enoent`s Also use `file:read_link_info/2`, it actually fetches any file info while also not following symlinks automatically, which is better for GC usecases. --- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 30 +++++----- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 59 ++++++++++++++++++- 2 files changed, 72 insertions(+), 17 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 75a74681f..98b048108 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -177,21 +177,21 @@ try_collect_transfer(Storage, Transfer, #{status := incomplete}, Stats) -> collect_fragments(Storage, Transfer, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, fragment), - collect_filepath(Dirname, true, Stats). + maybe_collect_directory(Dirname, true, Stats). collect_tempfiles(Storage, Transfer, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, temporary), - collect_filepath(Dirname, true, Stats). + maybe_collect_directory(Dirname, true, Stats). collect_outdated_fragments(Storage, Transfer, Cutoff, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, fragment), Filter = fun(_Filepath, #file_info{mtime = ModifiedAt}) -> ModifiedAt < Cutoff end, - collect_filepath(Dirname, Filter, Stats). + maybe_collect_directory(Dirname, Filter, Stats). collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, temporary), Filter = fun(_Filepath, #file_info{mtime = ModifiedAt}) -> ModifiedAt < Cutoff end, - collect_filepath(Dirname, Filter, Stats). + maybe_collect_directory(Dirname, Filter, Stats). collect_transfer_directory(Storage, Transfer, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer), @@ -206,8 +206,6 @@ collect_parents(Dirname, Until, Stats) -> ok -> ?tp(garbage_collected_directory, #{path => Dirname}), collect_parents(Parent, Until, account_gcstat_directory(Stats)); - {error, enoent} -> - collect_parents(Parent, Until, Stats); {error, eexist} -> Stats; {error, Reason} -> @@ -222,16 +220,22 @@ collect_parents(Dirname, Until, Stats) -> % Stats % end. +maybe_collect_directory(Dirpath, Filter, Stats) -> + case filelib:is_dir(Dirpath) of + true -> + collect_filepath(Dirpath, Filter, Stats); + false -> + {true, Stats} + end. + -spec collect_filepath(file:name(), Filter, gcstats()) -> {boolean(), gcstats()} when Filter :: boolean() | fun((file:name(), file:file_info()) -> boolean()). collect_filepath(Filepath, Filter, Stats) -> - case file:read_file_info(Filepath, [{time, posix}]) of + case file:read_link_info(Filepath, [{time, posix}, raw]) of {ok, Fileinfo} -> collect_filepath(Filepath, Fileinfo, Filter, Stats); - {error, enoent} -> - {true, Stats}; {error, Reason} -> - {false, register_gcstat_error({path, Filepath}, Reason, Stats)} + {Reason == enoent, register_gcstat_error({path, Filepath}, Reason, Stats)} end. collect_filepath(Filepath, #file_info{type = directory} = Fileinfo, Filter, Stats) -> @@ -243,10 +247,8 @@ collect_filepath(Filepath, #file_info{type = regular} = Fileinfo, Filter, Stats) ok -> ?tp(garbage_collected_file, #{path => Filepath}), {true, account_gcstat(Fileinfo, Stats)}; - {error, enoent} -> - {true, Stats}; {error, Reason} -> - {false, register_gcstat_error({file, Filepath}, Reason, Stats)} + {Reason == enoent, register_gcstat_error({file, Filepath}, Reason, Stats)} end; collect_filepath(Filepath, Fileinfo, _Filter, Stats) -> {false, register_gcstat_error({file, Filepath}, {unexpected, Fileinfo}, Stats)}. @@ -281,8 +283,6 @@ collect_empty_directory(Dirpath, Stats) -> ok -> ?tp(garbage_collected_directory, #{path => Dirpath}), account_gcstat_directory(Stats); - {error, enoent} -> - Stats; {error, Reason} -> register_gcstat_error({directory, Dirpath}, Reason, Stats) end. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index 858cdfd90..a0b969edd 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -56,7 +56,7 @@ end_per_testcase(_TC, _Config) -> ok. mk_root(TC, Config) -> - filename:join([?config(priv_dir, Config), <<"file_transfer">>, TC, atom_to_binary(node())]). + filename:join([?config(priv_dir, Config), "file_transfer", TC, atom_to_list(node())]). %% @@ -167,7 +167,6 @@ t_gc_complete_transfers(_Config) -> t_gc_incomplete_transfers(_Config) -> _ = application:set_env(emqx_ft, min_segments_ttl, 0), _ = application:set_env(emqx_ft, max_segments_ttl, 4), - ok = emqx_ft_storage_fs_gc:reset(emqx_ft_conf:storage()), Storage = emqx_ft_conf:storage(), Transfers = [ { @@ -241,6 +240,62 @@ t_gc_incomplete_transfers(_Config) -> [] ). +t_gc_handling_errors(_Config) -> + _ = application:set_env(emqx_ft, min_segments_ttl, 0), + _ = application:set_env(emqx_ft, max_segments_ttl, 0), + Storage = emqx_ft_conf:storage(), + Transfer1 = {<<"client1">>, mk_file_id()}, + Transfer2 = {<<"client2">>, mk_file_id()}, + Filemeta = #{name => "oops.pdf"}, + Size = 420, + SegSize = 16, + _ = start_transfer( + Storage, + {Transfer1, Filemeta, emqx_ft_content_gen:new({?LINE, Size}, SegSize)} + ), + _ = start_transfer( + Storage, + {Transfer2, Filemeta, emqx_ft_content_gen:new({?LINE, Size}, SegSize)} + ), + % 1. Throw some chaos in the transfer directory. + DirFragment1 = emqx_ft_storage_fs:get_subdir(Storage, Transfer1, fragment), + DirTemporary1 = emqx_ft_storage_fs:get_subdir(Storage, Transfer1, temporary), + PathShadyLink = filename:join(DirTemporary1, "linked-here"), + ok = file:make_symlink(DirFragment1, PathShadyLink), + DirTransfer2 = emqx_ft_storage_fs:get_subdir(Storage, Transfer2), + PathTripUp = filename:join(DirTransfer2, "trip-up-here"), + ok = file:write_file(PathTripUp, <<"HAHA">>), + ok = timer:sleep(timer:seconds(1)), + % 2. Observe the errors are reported consistently. + ?check_trace( + ?assertMatch( + #gcstats{ + files = Files, + directories = 3, + space = Space, + errors = #{ + % NOTE: dangling symlink looks like `enoent` for some reason + {file, PathShadyLink} := {unexpected, _}, + {directory, DirTransfer2} := eexist + } + } when Files == ?NSEGS(Size, SegSize) * 2 andalso Space > Size * 2, + emqx_ft_storage_fs_gc:collect(Storage) + ), + fun(Trace) -> + ?assertMatch( + [ + #{ + errors := #{ + {file, PathShadyLink} := {unexpected, _}, + {directory, DirTransfer2} := eexist + } + } + ], + ?of_kind("garbage_collection_errors", Trace) + ) + end + ). + %% start_transfer(Storage, {Transfer, Meta, Gen}) -> From e1dc48fa2b4c5100f05f88b5ab9715d6ab8ff4d1 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 14 Mar 2023 19:10:47 +0300 Subject: [PATCH 065/197] feat(fs-gc): wire gc up with emqx config --- apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 35 +++++++++++++++++++ apps/emqx_ft/src/emqx_ft_conf.erl | 18 +++++++--- apps/emqx_ft/src/emqx_ft_schema.erl | 33 ++++++++++++++++- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 32 ++++++++++------- 4 files changed, 99 insertions(+), 19 deletions(-) diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf index 576d7d8fe..dd2d2a1dc 100644 --- a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -33,4 +33,39 @@ emqx_ft_schema { } } + local_storage_gc { + desc { + en: "Garbage collection settings for the intermediate and temporary files in the local file system." + zh: "" + } + label: { + en: "Local Storage GC" + zh: "" + } + } + + storage_gc_interval { + desc { + en: "Interval of periodic garbage collection." + zh: "" + } + label: { + en: "GC Interval" + zh: "" + } + } + + storage_gc_max_segments_ttl { + desc { + en: "Maximum TTL of a segment kept in the local file system.
" + "This is a hard limit: no segment will outlive this TTL, even if some file transfer specifies a " + "TTL more than that." + zh: "" + } + label: { + en: "GC Interval" + zh: "" + } + } + } diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 444462716..e925ee376 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -50,17 +50,25 @@ storage() -> -spec gc_interval(_Storage) -> milliseconds(). gc_interval(_Storage) -> - % TODO: config wiring - application:get_env(emqx_ft, gc_interval, timer:minutes(10)). + Conf = assert_storage(local), + emqx_map_lib:deep_get([gc, interval], Conf). -spec segments_ttl(_Storage) -> {_Min :: seconds(), _Max :: seconds()}. segments_ttl(_Storage) -> - % TODO: config wiring + Conf = assert_storage(local), { - application:get_env(emqx_ft, min_segments_ttl, 60), - application:get_env(emqx_ft, max_segments_ttl, 72 * 3600) + emqx_map_lib:deep_get([gc, minimum_segments_ttl], Conf), + emqx_map_lib:deep_get([gc, maximum_segments_ttl], Conf) }. +assert_storage(Type) -> + case storage() of + Conf = #{type := Type} -> + Conf; + Conf -> + error({inapplicable, Conf}) + end. + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index f17c957a9..d2b2b9299 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -65,13 +65,44 @@ fields(local_storage) -> type => binary(), desc => ?DESC("local_storage_root"), required => false + }}, + {gc, #{ + type => hoconsc:ref(?MODULE, local_storage_gc), + desc => ?DESC("local_storage_gc"), + required => false + }} + ]; +fields(local_storage_gc) -> + [ + {interval, #{ + type => emqx_schema:duration_ms(), + desc => ?DESC("storage_gc_interval"), + required => false, + default => "1h" + }}, + {maximum_segments_ttl, #{ + type => emqx_schema:duration_s(), + desc => ?DESC("storage_gc_max_segments_ttl"), + required => false, + default => "24h" + }}, + {minimum_segments_ttl, #{ + type => emqx_schema:duration_s(), + % desc => ?DESC("storage_gc_min_segments_ttl"), + required => false, + default => "5m", + % NOTE + % This setting does not seem to be useful to an end-user. + hidden => true }} ]. desc(file_transfer) -> "File transfer settings"; desc(local_storage) -> - "File transfer local storage settings". + "File transfer local storage settings"; +desc(local_storage_gc) -> + "Garbage collection settings for the File transfer local storage backend". schema(filemeta) -> #{ diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index a0b969edd..a83d915e6 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -37,16 +37,22 @@ end_per_suite(_Config) -> ok. init_per_testcase(TC, Config) -> - _ = application:unset_env(emqx_ft, gc_interval), - _ = application:unset_env(emqx_ft, min_segments_ttl), - _ = application:unset_env(emqx_ft, max_segments_ttl), ok = emqx_common_test_helpers:start_app( emqx_ft, fun(emqx_ft) -> - ok = emqx_config:put([file_transfer, storage], #{ - type => local, - root => mk_root(TC, Config) - }) + emqx_common_test_helpers:load_config( + emqx_ft_schema, + iolist_to_binary([ + "file_transfer {" + " storage = {" + " type = \"local\"," + " root = \"", + mk_root(TC, Config), + "\"" + " }" + "}" + ]) + ) end ), Config. @@ -64,7 +70,7 @@ mk_root(TC, Config) -> t_gc_triggers_periodically(_Config) -> Interval = 500, - ok = application:set_env(emqx_ft, gc_interval, Interval), + ok = emqx_config:put([file_transfer, storage, gc, interval], Interval), ok = emqx_ft_storage_fs_gc:reset(emqx_ft_conf:storage()), ?check_trace( timer:sleep(Interval * 3), @@ -165,8 +171,8 @@ t_gc_complete_transfers(_Config) -> ). t_gc_incomplete_transfers(_Config) -> - _ = application:set_env(emqx_ft, min_segments_ttl, 0), - _ = application:set_env(emqx_ft, max_segments_ttl, 4), + ok = emqx_config:put([file_transfer, storage, gc, minimum_segments_ttl], 0), + ok = emqx_config:put([file_transfer, storage, gc, maximum_segments_ttl], 4), Storage = emqx_ft_conf:storage(), Transfers = [ { @@ -195,7 +201,7 @@ t_gc_incomplete_transfers(_Config) -> ?check_trace( begin % 2. Enable periodic GC every 0.5 seconds. - ok = application:set_env(emqx_ft, gc_interval, 500), + ok = emqx_config:put([file_transfer, storage, gc, interval], 500), ok = emqx_ft_storage_fs_gc:reset(Storage), % 3. First we need the first transfer to be collected. {ok, _} = ?block_until( @@ -241,8 +247,8 @@ t_gc_incomplete_transfers(_Config) -> ). t_gc_handling_errors(_Config) -> - _ = application:set_env(emqx_ft, min_segments_ttl, 0), - _ = application:set_env(emqx_ft, max_segments_ttl, 0), + ok = emqx_config:put([file_transfer, storage, gc, minimum_segments_ttl], 0), + ok = emqx_config:put([file_transfer, storage, gc, maximum_segments_ttl], 0), Storage = emqx_ft_conf:storage(), Transfer1 = {<<"client1">>, mk_file_id()}, Transfer2 = {<<"client2">>, mk_file_id()}, From 0c821cd3bd323fa159b4c202bfca046e40607da5 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 15 Mar 2023 13:57:42 +0300 Subject: [PATCH 066/197] test(ft): inject configs through hocon subsystem So that relevant parts of config would be initialized with defaults. --- apps/emqx_ft/test/emqx_ft_SUITE.erl | 11 ++++------- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 5 ++--- apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 5 ++--- .../emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl | 16 +++------------- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 7 ++++--- 5 files changed, 15 insertions(+), 29 deletions(-) diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index bb67d3c56..9c9087732 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -58,9 +58,8 @@ end_per_suite(_Config) -> set_special_configs(Config) -> fun (emqx_ft) -> - ok = emqx_config:put([file_transfer, storage], #{ - type => local, root => emqx_ft_test_helpers:ft_root(Config, node()) - }); + Root = emqx_ft_test_helpers:ft_root(Config, node()), + emqx_ft_test_helpers:load_config(#{storage => #{type => local, root => Root}}); (_) -> ok end. @@ -109,10 +108,8 @@ mk_cluster_specs(Config) -> {conf, [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]]}, {env_handler, fun (emqx_ft) -> - ok = emqx_config:put([file_transfer, storage], #{ - type => local, - root => emqx_ft_test_helpers:ft_root(Config, node()) - }); + Root = emqx_ft_test_helpers:ft_root(Config, node()), + emqx_ft_test_helpers:load_config(#{storage => #{type => local, root => Root}}); (_) -> ok end} diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index 239edb267..7b191e229 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -41,9 +41,8 @@ end_per_suite(_Config) -> set_special_configs(Config) -> fun (emqx_ft) -> - ok = emqx_config:put([file_transfer, storage], #{ - type => local, root => emqx_ft_test_helpers:ft_root(Config, node()) - }); + Root = emqx_ft_test_helpers:ft_root(Config, node()), + emqx_ft_test_helpers:load_config(#{storage => #{type => local, root => Root}}); (_) -> ok end. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index 3bda8042c..5551cce27 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -48,9 +48,8 @@ end_per_suite(_Config) -> set_special_configs(Config) -> fun (emqx_ft) -> - ok = emqx_config:put([file_transfer, storage], #{ - type => local, root => emqx_ft_test_helpers:ft_root(Config, node()) - }); + Root = emqx_ft_test_helpers:ft_root(Config, node()), + emqx_ft_test_helpers:load_config(#{storage => #{type => local, root => Root}}); (_) -> ok end. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index a83d915e6..37ac7eedf 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -40,19 +40,9 @@ init_per_testcase(TC, Config) -> ok = emqx_common_test_helpers:start_app( emqx_ft, fun(emqx_ft) -> - emqx_common_test_helpers:load_config( - emqx_ft_schema, - iolist_to_binary([ - "file_transfer {" - " storage = {" - " type = \"local\"," - " root = \"", - mk_root(TC, Config), - "\"" - " }" - "}" - ]) - ) + emqx_ft_test_helpers:load_config(#{ + storage => #{type => local, root => mk_root(TC, Config)} + }) end ), Config. diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index f054be762..b756f8034 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -30,9 +30,7 @@ start_additional_node(Config, Name) -> {configure_gen_rpc, true}, {env_handler, fun (emqx_ft) -> - ok = emqx_config:put([file_transfer, storage], #{ - type => local, root => ft_root(Config, node()) - }); + load_config(#{storage => #{type => local, root => ft_root(Config, node())}}); (_) -> ok end} @@ -45,6 +43,9 @@ stop_additional_node(Node) -> ok = emqx_common_test_helpers:stop_slave(Node), ok. +load_config(Config) -> + emqx_common_test_helpers:load_config(emqx_ft_schema, #{file_transfer => Config}). + tcp_port(Node) -> {_, Port} = rpc:call(Node, emqx_config, get, [[listeners, tcp, default, bind]]), Port. From 0d395460809ce5c7cdc3b7a400124097201d0b5c Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Mar 2023 14:24:52 +0300 Subject: [PATCH 067/197] feat(wdgraph): add `fold/3` which folds over graph edges --- apps/emqx/src/emqx_wdgraph.erl | 24 +++++++++++++++++++++++- apps/emqx/test/emqx_wdgraph_tests.erl | 20 ++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_wdgraph.erl b/apps/emqx/src/emqx_wdgraph.erl index 3361c52d1..bd7f58e7c 100644 --- a/apps/emqx/src/emqx_wdgraph.erl +++ b/apps/emqx/src/emqx_wdgraph.erl @@ -27,6 +27,8 @@ -export([find_edge/3]). -export([get_edges/2]). +-export([fold/3]). + -export([find_shortest_path/3]). -export_type([t/0]). @@ -38,7 +40,7 @@ -type label() :: term(). -opaque t() :: t(gnode(), label()). --opaque t(Node, Label) :: gb_trees:tree({Node}, {Node, weight(), Label}). +-opaque t(Node, Label) :: gb_trees:tree({Node}, [{Node, weight(), Label}]). %% @@ -72,6 +74,26 @@ find_edge(From, To, G) -> get_edges(Node, G) -> tree_lookup({Node}, G, []). +-spec fold(FoldFun, Acc, t(Node, Label)) -> Acc when + FoldFun :: fun((Node, _Edge :: {Node, weight(), Label}, Acc) -> Acc). +fold(FoldFun, Acc, G) -> + fold_iterator(FoldFun, Acc, gb_trees:iterator(G)). + +fold_iterator(FoldFun, AccIn, It) -> + case gb_trees:next(It) of + {{Node}, Edges = [_ | _], ItNext} -> + AccNext = lists:foldl( + fun(Edge = {_To, _Weight, _Label}, Acc) -> + FoldFun(Node, Edge, Acc) + end, + AccIn, + Edges + ), + fold_iterator(FoldFun, AccNext, ItNext); + none -> + AccIn + end. + % Find the shortest path between two nodes, if any. If the path exists, return list % of edge labels along that path. % This is a Dijkstra shortest path algorithm. It is one-way right now, for diff --git a/apps/emqx/test/emqx_wdgraph_tests.erl b/apps/emqx/test/emqx_wdgraph_tests.erl index ece87b966..c159a0f49 100644 --- a/apps/emqx/test/emqx_wdgraph_tests.erl +++ b/apps/emqx/test/emqx_wdgraph_tests.erl @@ -40,6 +40,26 @@ edges_nodes_test_() -> ?_assertEqual([{baz, 1, "cheapest"}, {foo, 0, "free"}], emqx_wdgraph:get_edges(bar, G5)) ]. +fold_test_() -> + G1 = emqx_wdgraph:new(), + G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1), + G3 = emqx_wdgraph:insert_edge(bar, baz, 1, "cheapest", G2), + G4 = emqx_wdgraph:insert_edge(bar, foo, 0, "free", G3), + G5 = emqx_wdgraph:insert_edge(foo, bar, 100, "luxury", G4), + [ + ?_assertEqual( + % 100 + 0 + 1 + 101, + emqx_wdgraph:fold(fun(_From, {_, Weight, _}, Acc) -> Weight + Acc end, 0, G5) + ), + ?_assertEqual( + [bar, baz, foo], + lists:usort( + emqx_wdgraph:fold(fun(From, {To, _, _}, Acc) -> [From, To | Acc] end, [], G5) + ) + ) + ]. + nonexistent_nodes_path_test_() -> G1 = emqx_wdgraph:new(), G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1), From 4f2600b9f1e31c167442bbf0f07e8649742ec3a8 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Mar 2023 15:15:43 +0300 Subject: [PATCH 068/197] feat(ft-gc): treat all transfer as incomplete Since the concept of _complete transfers_ is being split out into the _export_ concept, we lose knowledge of completeness in the GC. Instead of asking exporters for transfer statuses we just treat all transfer as incomplete when GCing. --- apps/emqx_ft/src/emqx_ft_assembler.erl | 12 +- apps/emqx_ft/src/emqx_ft_assembly.erl | 19 +++ apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 48 +++--- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 141 +++++++++--------- 4 files changed, 121 insertions(+), 99 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 38ccf13ac..ff845fee9 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -137,20 +137,10 @@ pread(Node, Segment, St) -> %% maybe_garbage_collect(ok, St = #st{storage = Storage, transfer = Transfer}) -> - Nodes = get_coverage_nodes(St), + Nodes = emqx_ft_assembly:nodes(St#st.assembly), emqx_ft_storage_fs_gc:collect(Storage, Transfer, Nodes); maybe_garbage_collect({error, _}, _St) -> ok. -get_coverage_nodes(St) -> - Coverage = emqx_ft_assembly:coverage(St#st.assembly), - ordsets:to_list( - lists:foldl( - fun({Node, _Segment}, Acc) -> ordsets:add_element(Node, Acc) end, - ordsets:new(), - Coverage - ) - ). - segsize(#{fragment := {segment, Info}}) -> maps:get(size, Info). diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index bea320bbf..d0998d6ec 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -22,6 +22,7 @@ -export([status/1]). -export([filemeta/1]). +-export([nodes/1]). -export([coverage/1]). -export([properties/1]). @@ -108,6 +109,24 @@ filemeta(Asm) -> coverage(#asm{coverage = Coverage}) -> Coverage. +-spec nodes(t()) -> [node()]. +nodes(#asm{meta = Meta, segs = Segs}) -> + S1 = orddict:fold( + fun(_Meta, {Node, _Fragment}, Acc) -> + ordsets:add_element(Node, Acc) + end, + ordsets:new(), + Meta + ), + S2 = emqx_wdgraph:fold( + fun(_Offset, {_End, _, {Node, _Fragment}}, Acc) -> + ordsets:add_element(Node, Acc) + end, + ordsets:new(), + Segs + ), + ordsets:to_list(ordsets:union(S1, S2)). + properties(#asm{properties = Properties}) -> Properties. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 98b048108..58c5dbfdf 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -80,11 +80,15 @@ handle_call(Call, From, St) -> ?SLOG(error, #{msg => "unexpected_call", call => Call, from => From}), {noreply, St}. -% TODO -% handle_cast({collect, Transfer, [Node | Rest]}, St) -> -% ok = do_collect_transfer(Transfer, Node, St), -% ok = collect(self(), Transfer, Rest), -% {noreply, St}; +handle_cast({collect, Transfer, [Node | Rest]}, St) -> + ok = do_collect_transfer(Transfer, Node, St), + case Rest of + [_ | _] -> + gen_server:cast(self(), {collect, Transfer, Rest}); + [] -> + ok + end, + {noreply, St}; handle_cast(reset, St) -> {noreply, reset_timer(St)}; handle_cast(Cast, St) -> @@ -95,10 +99,13 @@ handle_info({timeout, TRef, collect}, St = #st{next_gc_timer = TRef}) -> StNext = do_collect_garbage(St), {noreply, start_timer(StNext#st{next_gc_timer = undefined})}. -% do_collect_transfer(Transfer, Node, St = #st{storage = Storage}) when Node == node() -> -% Stats = try_collect_transfer(Storage, Transfer, complete, init_gcstats()), -% ok = maybe_report(Stats, St), -% ok. +do_collect_transfer(Transfer, Node, St = #st{storage = Storage}) when Node == node() -> + Stats = try_collect_transfer(Storage, Transfer, complete, init_gcstats()), + ok = maybe_report(Stats, St), + ok; +do_collect_transfer(_Transfer, _Node, _St = #st{}) -> + % TODO + ok. maybe_collect_garbage(_CalledAt, St = #st{last_gc = undefined}) -> do_collect_garbage(St); @@ -149,21 +156,13 @@ collect_garbage(Storage, Transfers, Stats) -> ) ). -try_collect_transfer(Storage, Transfer, #{status := complete}, Stats) -> - % File transfer is complete. - % We should be good to delete fragments and temporary files with their respective - % directories altogether. - % TODO: file expiration - {_, Stats1} = collect_fragments(Storage, Transfer, Stats), - {_, Stats2} = collect_tempfiles(Storage, Transfer, Stats1), - Stats2; -try_collect_transfer(Storage, Transfer, #{status := incomplete}, Stats) -> - % File transfer is still incomplete. +try_collect_transfer(Storage, Transfer, TransferInfo = #{}, Stats) -> + % File transfer might still be incomplete. % Any outdated fragments and temporary files should be collectable. As a kind of % heuristic we only delete transfer directory itself only if it is also outdated % _and was empty at the start of GC_, as a precaution against races between % writers and GCs. - TTL = get_segments_ttl(Storage, Transfer), + TTL = get_segments_ttl(Storage, TransferInfo), Cutoff = erlang:system_time(second) - TTL, {FragCleaned, Stats1} = collect_outdated_fragments(Storage, Transfer, Cutoff, Stats), {TempCleaned, Stats2} = collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats1), @@ -173,7 +172,14 @@ try_collect_transfer(Storage, Transfer, #{status := incomplete}, Stats) -> collect_transfer_directory(Storage, Transfer, Stats2); false -> Stats2 - end. + end; +try_collect_transfer(Storage, Transfer, complete, Stats) -> + % File transfer is complete. + % We should be good to delete fragments and temporary files with their respective + % directories altogether. + {_, Stats1} = collect_fragments(Storage, Transfer, Stats), + {_, Stats2} = collect_tempfiles(Storage, Transfer, Stats1), + Stats2. collect_fragments(Storage, Transfer, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, fragment), diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index 37ac7eedf..89e9eb970 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -45,9 +45,11 @@ init_per_testcase(TC, Config) -> }) end ), + ok = snabbkaffe:start_trace(), Config. end_per_testcase(_TC, _Config) -> + ok = snabbkaffe:stop(), ok = application:stop(emqx_ft), ok. @@ -126,20 +128,29 @@ t_gc_complete_transfers(_Config) -> emqx_ft_storage_fs_gc:collect(Storage) ), % 2. Complete just the first transfer - ?assertEqual( - ok, - complete_transfer(Storage, T1, S1) + {ok, {ok, Event}} = ?wait_async_action( + ?assertEqual(ok, complete_transfer(Storage, T1, S1)), + #{?snk_kind := garbage_collection}, + 1000 ), ?assertMatch( - #gcstats{ - files = Files, - directories = 2, - space = Space, - errors = #{} = Es + #{ + stats := #gcstats{ + files = Files, + directories = 2, + space = Space, + errors = #{} = Es + } } when Files == ?NSEGS(S1, SS1) andalso Space > S1 andalso map_size(Es) == 0, - emqx_ft_storage_fs_gc:collect(Storage) + Event ), % 3. Complete rest of transfers + {ok, Sub} = snabbkaffe_collector:subscribe( + ?match_event(#{?snk_kind := garbage_collection}), + 2, + 1000, + 0 + ), ?assertEqual( [ok, ok], emqx_misc:pmap( @@ -147,18 +158,19 @@ t_gc_complete_transfers(_Config) -> [{T2, S2}, {T3, S3}] ) ), - ?assertMatch( - #gcstats{ - files = Files, - directories = 4, - space = Space, - errors = #{} = Es - } when - Files == (?NSEGS(S2, SS2) + ?NSEGS(S3, SS3)) andalso - Space > (S2 + S3) andalso - map_size(Es) == 0, - emqx_ft_storage_fs_gc:collect(Storage) - ). + {ok, Events} = snabbkaffe_collector:receive_events(Sub), + CFiles = lists:sum([Stats#gcstats.files || #{stats := Stats} <- Events]), + CDirectories = lists:sum([Stats#gcstats.directories || #{stats := Stats} <- Events]), + CSpace = lists:sum([Stats#gcstats.space || #{stats := Stats} <- Events]), + CErrors = lists:foldl( + fun maps:merge/2, + #{}, + [Stats#gcstats.errors || #{stats := Stats} <- Events] + ), + ?assertEqual(?NSEGS(S2, SS2) + ?NSEGS(S3, SS3), CFiles), + ?assertEqual(2 + 2, CDirectories), + ?assertMatch(Space when Space > S2 + S3, CSpace), + ?assertMatch(Errors when map_size(Errors) == 0, CErrors). t_gc_incomplete_transfers(_Config) -> ok = emqx_config:put([file_transfer, storage, gc, minimum_segments_ttl], 0), @@ -188,52 +200,47 @@ t_gc_incomplete_transfers(_Config) -> ], % 1. Start transfers, send all the segments but don't trigger completion. _ = emqx_misc:pmap(fun(Transfer) -> start_transfer(Storage, Transfer) end, Transfers), - ?check_trace( - begin - % 2. Enable periodic GC every 0.5 seconds. - ok = emqx_config:put([file_transfer, storage, gc, interval], 500), - ok = emqx_ft_storage_fs_gc:reset(Storage), - % 3. First we need the first transfer to be collected. - {ok, _} = ?block_until( - #{ - ?snk_kind := garbage_collection, - stats := #gcstats{ - files = Files, - directories = 4, - space = Space - } - } when Files == (?NSEGS(S1, SS1)) andalso Space > S1, - 5000, - 0 - ), - % 4. Then the second one. - {ok, _} = ?block_until( - #{ - ?snk_kind := garbage_collection, - stats := #gcstats{ - files = Files, - directories = 4, - space = Space - } - } when Files == (?NSEGS(S2, SS2)) andalso Space > S2, - 5000, - 0 - ), - % 5. Then transfers 3 and 4 because 3rd has too big TTL and 4th has no specific TTL. - {ok, _} = ?block_until( - #{ - ?snk_kind := garbage_collection, - stats := #gcstats{ - files = Files, - directories = 4 * 2, - space = Space - } - } when Files == (?NSEGS(S3, SS3) + ?NSEGS(S4, SS4)) andalso Space > S3 + S4, - 5000, - 0 - ) - end, - [] + % 2. Enable periodic GC every 0.5 seconds. + ok = emqx_config:put([file_transfer, storage, gc, interval], 500), + ok = emqx_ft_storage_fs_gc:reset(Storage), + % 3. First we need the first transfer to be collected. + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = Files, + directories = 4, + space = Space + } + } when Files == (?NSEGS(S1, SS1)) andalso Space > S1, + 5000, + 0 + ), + % 4. Then the second one. + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = Files, + directories = 4, + space = Space + } + } when Files == (?NSEGS(S2, SS2)) andalso Space > S2, + 5000, + 0 + ), + % 5. Then transfers 3 and 4 because 3rd has too big TTL and 4th has no specific TTL. + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = Files, + directories = 4 * 2, + space = Space + } + } when Files == (?NSEGS(S3, SS3) + ?NSEGS(S4, SS4)) andalso Space > S3 + S4, + 5000, + 0 ). t_gc_handling_errors(_Config) -> From 45f00e14a9df3a5ca889f2fc8da6fa601b0987fb Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Mar 2023 15:18:25 +0300 Subject: [PATCH 069/197] feat(ft): ensure that clientid is always binary For the sake of simplicity (e.g. transfer ids are now easier to compare). --- apps/emqx_ft/src/emqx_ft.erl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 2ccd5db15..518807d9f 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -51,7 +51,7 @@ -type bytes() :: non_neg_integer(). %% MQTT Client ID --type clientid() :: emqx_types:clientid(). +-type clientid() :: binary(). -type fileid() :: binary(). -type transfer() :: {clientid(), fileid()}. @@ -327,7 +327,7 @@ assemble(Transfer, FinalSize) -> transfer(Msg, FileId) -> ClientId = Msg#message.from, - {ClientId, FileId}. + {clientid_to_binary(ClientId), FileId}. on_complete(Op, {ChanPid, PacketId}, Transfer, Result) -> ?SLOG(debug, #{ @@ -421,3 +421,8 @@ parse_checksum(Checksum) when is_binary(Checksum) andalso byte_size(Checksum) =: end; parse_checksum(_Checksum) -> {error, invalid_checksum}. + +clientid_to_binary(A) when is_atom(A) -> + atom_to_binary(A); +clientid_to_binary(B) when is_binary(B) -> + B. From 2880c8f4eb6c69ffdcdeec3cccc481761448098d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 21 Mar 2023 00:11:30 +0300 Subject: [PATCH 070/197] fix(ft): unwrap error details when reader fails to start --- apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl index 934e2888c..fff6b7830 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl @@ -33,7 +33,12 @@ start_child(CallerPid, Filename) -> start => {emqx_ft_storage_fs_reader, start_link, [CallerPid, Filename]}, restart => temporary }, - supervisor:start_child(?MODULE, Childspec). + case supervisor:start_child(?MODULE, Childspec) of + {ok, Pid} -> + {ok, Pid}; + {error, {Reason, _Child}} -> + {error, Reason} + end. init(_) -> SupFlags = #{ From 4132f5a5fb3127878532d436613a20026e8da7b4 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 10 Mar 2023 15:27:50 +0300 Subject: [PATCH 071/197] feat(ft): introduce exporter concept in local storage backend The exporter is responsible for keeping fully transferred and successfully assembled files. This was on the local storage itself before. This abstraction is needed to give us an ability to support S3 destinations more easily, just by swapping the storage exporter. Also implement local filesystem exporter and reimplement parts of the `emqx_ft` API on top of it. --- apps/emqx_ft/etc/emqx_ft.conf | 3 + apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 38 +- apps/emqx_ft/src/emqx_ft.erl | 8 +- apps/emqx_ft/src/emqx_ft_api.erl | 174 ++++++--- apps/emqx_ft/src/emqx_ft_assembler.erl | 59 ++- apps/emqx_ft/src/emqx_ft_fs_util.erl | 101 +++++ apps/emqx_ft/src/emqx_ft_schema.erl | 26 +- apps/emqx_ft/src/emqx_ft_storage.erl | 35 +- .../src/emqx_ft_storage_exporter_fs.erl | 352 ++++++++++++++++++ apps/emqx_ft/src/emqx_ft_storage_fs.erl | 281 +++----------- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 22 +- apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl | 22 +- .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 29 +- apps/emqx_ft/test/emqx_ft_SUITE.erl | 67 ++-- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 66 ++-- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 48 ++- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 11 +- .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 69 +--- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 12 +- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 15 +- 20 files changed, 936 insertions(+), 502 deletions(-) create mode 100644 apps/emqx_ft/src/emqx_ft_fs_util.erl create mode 100644 apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl diff --git a/apps/emqx_ft/etc/emqx_ft.conf b/apps/emqx_ft/etc/emqx_ft.conf index 250dca6a9..8d921e79c 100644 --- a/apps/emqx_ft/etc/emqx_ft.conf +++ b/apps/emqx_ft/etc/emqx_ft.conf @@ -1,5 +1,8 @@ file_transfer { storage { type = local + exporter { + type = local + } } } diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf index dd2d2a1dc..15c42dcfa 100644 --- a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -13,7 +13,7 @@ emqx_ft_schema { local_type { desc { - en: "Use local file system to store uploaded files and temporary data." + en: "Use local file system to store uploaded fragments and temporary data." zh: "使用本地文件系统来存储上传的文件和临时数据。" } label: { @@ -24,7 +24,7 @@ emqx_ft_schema { local_storage_root { desc { - en: "File system path to keep uploaded files and temporary data." + en: "File system path to keep uploaded fragments and temporary data." zh: "保存上传文件和临时数据的文件系统路径。" } label: { @@ -33,6 +33,40 @@ emqx_ft_schema { } } + local_storage_exporter { + desc { + en: "Exporter for the local file system storage backend.
" + "Exporter defines where and how fully transferred and assembled files are stored." + zh: "" + } + label: { + en: "Local Storage Exporter" + zh: "" + } + } + + local_storage_exporter_type { + desc { + en: "Type of the Exporter to use." + zh: "" + } + label: { + en: "Local Storage Exporter Type" + zh: "" + } + } + + local_storage_exporter_root { + desc { + en: "File system path to keep uploaded files." + zh: "" + } + label: { + en: "Local Filesystem Exporter Root" + zh: "" + } + } + local_storage_gc { desc { en: "Garbage collection settings for the intermediate and temporary files in the local file system." diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 518807d9f..b7c8f0eac 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -33,7 +33,8 @@ ]). -export([ - decode_filemeta/1 + decode_filemeta/1, + encode_filemeta/1 ]). -export([on_complete/4]). @@ -114,6 +115,11 @@ decode_filemeta(Map) when is_map(Map) -> {error, {invalid_filemeta, Error}} end. +encode_filemeta(Meta = #{}) -> + % TODO: Looks like this should be hocon's responsibility. + Schema = emqx_ft_schema:schema(filemeta), + hocon_tconf:make_serializable(Schema, emqx_map_lib:binary_key_map(Meta), #{}). + %%-------------------------------------------------------------------- %% Hooks %%-------------------------------------------------------------------- diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index ddc6e761a..143d629de 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -40,6 +40,10 @@ '/file_transfer/file'/2 ]). +-export([ + mk_file_uri/3 +]). + -import(hoconsc, [mk/2, ref/1, ref/2]). namespace() -> "file_transfer". @@ -69,6 +73,11 @@ schema("/file_transfer/files") -> } }; schema("/file_transfer/file") -> + % TODO + % This is conceptually another API, because this logic is inherent only to the + % local filesystem exporter. Ideally, we won't even publish it if `emqx_ft` is + % configured with another exporter. Accordingly, things that look too specific + % for this module (i.e. `parse_filepath/1`) should go away in another API module. #{ 'operationId' => '/file_transfer/file', get => #{ @@ -77,8 +86,7 @@ schema("/file_transfer/file") -> description => ?DESC("file_get"), parameters => [ ref(file_node), - ref(file_clientid), - ref(file_id) + ref(file_ref) ], responses => #{ 200 => <<"Operation success">>, @@ -91,32 +99,40 @@ schema("/file_transfer/file") -> }. '/file_transfer/files'(get, #{}) -> - case emqx_ft_storage:ready_transfers() of + case emqx_ft_storage:exports() of {ok, Transfers} -> - FormattedTransfers = lists:map( - fun({Id, Info}) -> - #{id => Id, info => format_file_info(Info)} - end, - Transfers - ), - {200, #{<<"files">> => FormattedTransfers}}; + {200, #{<<"files">> => lists:map(fun format_export_info/1, Transfers)}}; {error, _} -> {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} end. '/file_transfer/file'(get, #{query_string := Query}) -> - case emqx_ft_storage:get_ready_transfer(Query) of - {ok, FileData} -> - {200, - #{ - <<"content-type">> => <<"application/data">>, - <<"content-disposition">> => <<"attachment">> - }, - FileData}; - {error, enoent} -> - {404, error_msg('NOT_FOUND', <<"Not found">>)}; - {error, Error} -> - ?SLOG(warning, #{msg => "get_ready_transfer_fail", error => Error}), + try + Node = parse_node(maps:get(<<"node">>, Query)), + Filepath = parse_filepath(maps:get(<<"fileref">>, Query)), + case emqx_ft_storage_fs_proto_v1:read_export_file(Node, Filepath, self()) of + {ok, ReaderPid} -> + FileData = emqx_ft_storage_fs_reader:table(ReaderPid), + {200, + #{ + <<"content-type">> => <<"application/data">>, + <<"content-disposition">> => <<"attachment">> + }, + FileData}; + {error, enoent} -> + {404, error_msg('NOT_FOUND', <<"Not found">>)}; + {error, Error} -> + ?SLOG(warning, #{msg => "get_ready_transfer_fail", error => Error}), + {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} + end + catch + throw:{invalid, Param} -> + {404, + error_msg( + 'NOT_FOUND', + iolist_to_binary(["Invalid query parameter: ", Param]) + )}; + error:{erpc, noconnection} -> {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} end. @@ -124,46 +140,100 @@ error_msg(Code, Msg) -> #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. -spec fields(hocon_schema:name()) -> hocon_schema:fields(). +fields(file_ref) -> + [ + {fileref, + hoconsc:mk(binary(), #{ + in => query, + desc => <<"File reference">>, + example => <<"file1">>, + required => true + })} + ]; fields(file_node) -> - Desc = <<"File Node">>, - Meta = #{ - in => query, desc => Desc, example => <<"emqx@127.0.0.1">>, required => false - }, - [{node, hoconsc:mk(binary(), Meta)}]; -fields(file_clientid) -> - Desc = <<"File ClientId">>, - Meta = #{ - in => query, desc => Desc, example => <<"client1">>, required => false - }, - [{clientid, hoconsc:mk(binary(), Meta)}]; -fields(file_id) -> - Desc = <<"File">>, - Meta = #{ - in => query, desc => Desc, example => <<"file1">>, required => false - }, - [{fileid, hoconsc:mk(binary(), Meta)}]. + [ + {node, + hoconsc:mk(binary(), #{ + in => query, + desc => <<"Node under which the file is located">>, + example => atom_to_list(node()), + required => true + })} + ]. roots() -> [ file_node, - file_clientid, - file_id + file_ref + ]. + +mk_file_uri(_Options, Node, Filepath) -> + % TODO: qualify with `?BASE_PATH` + [ + "/file_transfer/file?", + uri_string:compose_query([ + {"node", atom_to_list(Node)}, + {"fileref", Filepath} + ]) ]. %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- -format_file_info(#{path := Path, size := Size, timestamp := Timestamp}) -> - #{ - path => Path, +format_export_info( + Info = #{ + name := Name, + size := Size, + uri := URI, + timestamp := Timestamp, + transfer := {ClientId, FileId} + } +) -> + Res = #{ + name => iolist_to_binary(Name), size => Size, - timestamp => format_datetime(Timestamp) - }. + timestamp => format_timestamp(Timestamp), + clientid => ClientId, + fileid => FileId, + uri => iolist_to_binary(URI) + }, + case Info of + #{meta := Meta} -> + Res#{metadata => emqx_ft:encode_filemeta(Meta)}; + #{} -> + Res + end. -format_datetime({{Year, Month, Day}, {Hour, Minute, Second}}) -> - iolist_to_binary( - io_lib:format("~4..0w-~2..0w-~2..0wT~2..0w:~2..0w:~2..0w", [ - Year, Month, Day, Hour, Minute, Second - ]) - ). +format_timestamp(Timestamp) -> + iolist_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])). + +parse_node(NodeBin) -> + case emqx_misc:safe_to_existing_atom(NodeBin) of + {ok, Node} -> + Node; + {error, _} -> + throw({invalid, NodeBin}) + end. + +parse_filepath(PathBin) -> + case filename:pathtype(PathBin) of + relative -> + ok; + absolute -> + throw({invalid, PathBin}) + end, + PathComponents = filename:split(PathBin), + case lists:any(fun is_special_component/1, PathComponents) of + false -> + filename:join(PathComponents); + true -> + throw({invalid, PathBin}) + end. + +is_special_component(<<".", _/binary>>) -> + true; +is_special_component([$. | _]) -> + true; +is_special_component(_) -> + false. diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index ff845fee9..5489a232a 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -22,13 +22,13 @@ -export([callback_mode/0]). -export([init/1]). -export([handle_event/4]). +-export([terminate/3]). -record(st, { storage :: _Storage, transfer :: emqx_ft:transfer(), assembly :: emqx_ft_assembly:t(), - file :: {file:filename(), io:device(), term()} | undefined, - hash + export :: _Export | undefined }). -define(NAME(Transfer), {n, l, {?MODULE, Transfer}}). @@ -47,11 +47,11 @@ callback_mode() -> handle_event_function. init({Storage, Transfer, Size}) -> + _ = erlang:process_flag(trap_exit, true), St = #st{ storage = Storage, transfer = Transfer, - assembly = emqx_ft_assembly:new(Size), - hash = crypto:hash_init(sha256) + assembly = emqx_ft_assembly:new(Size) }, {ok, idle, St}. @@ -61,10 +61,10 @@ handle_event(info, kickoff, idle, St) -> % We could wait for this message and handle it at the end of the assembling rather than at % the beginning, however it would make error handling much more messier. {next_state, list_local_fragments, St, ?internal([])}; -handle_event(internal, _, list_local_fragments, St = #st{assembly = Asm}) -> +handle_event(internal, _, list_local_fragments, St = #st{}) -> % TODO: what we do with non-transients errors here (e.g. `eacces`)? {ok, Fragments} = emqx_ft_storage_fs:list(St#st.storage, St#st.transfer, fragment), - NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(Asm, node(), Fragments)), + NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(St#st.assembly, node(), Fragments)), NSt = St#st{assembly = NAsm}, case emqx_ft_assembly:status(NAsm) of complete -> @@ -110,8 +110,8 @@ handle_event(internal, _, start_assembling, St = #st{assembly = Asm}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), Coverage = emqx_ft_assembly:coverage(Asm), % TODO: better error handling - {ok, Handle} = emqx_ft_storage_fs:open_file(St#st.storage, St#st.transfer, Filemeta), - {next_state, {assemble, Coverage}, St#st{file = Handle}, ?internal([])}; + {ok, Export} = export_start(Filemeta, St), + {next_state, {assemble, Coverage}, St#st{export = Export}, ?internal([])}; handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % TODO % Currently, race is possible between getting segment info from the remote node and @@ -119,15 +119,17 @@ handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % TODO: pipelining % TODO: better error handling {ok, Content} = pread(Node, Segment, St), - {ok, NHandle} = emqx_ft_storage_fs:write(St#st.file, Content), - {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])}; + {ok, NExport} = export_write(St#st.export, Content), + {next_state, {assemble, Rest}, St#st{export = NExport}, ?internal([])}; handle_event(internal, _, {assemble, []}, St = #st{}) -> {next_state, complete, St, ?internal([])}; -handle_event(internal, _, complete, St = #st{assembly = Asm, file = Handle}) -> - Filemeta = emqx_ft_assembly:filemeta(Asm), - Result = emqx_ft_storage_fs:complete(St#st.storage, St#st.transfer, Filemeta, Handle), +handle_event(internal, _, complete, St = #st{}) -> + Result = export_complete(St#st.export), ok = maybe_garbage_collect(Result, St), - {stop, {shutdown, Result}}. + {stop, {shutdown, Result}, St#st{export = undefined}}. + +terminate(_Reason, _StateName, #st{export = Export}) -> + Export /= undefined andalso export_discard(Export). pread(Node, Segment, St) when Node =:= node() -> emqx_ft_storage_fs:pread(St#st.storage, St#st.transfer, Segment, 0, segsize(Segment)); @@ -136,8 +138,33 @@ pread(Node, Segment, St) -> %% -maybe_garbage_collect(ok, St = #st{storage = Storage, transfer = Transfer}) -> - Nodes = emqx_ft_assembly:nodes(St#st.assembly), +export_start(Filemeta, #st{storage = Storage, transfer = Transfer}) -> + {ExporterMod, Exporter} = emqx_ft_storage_fs:exporter(Storage), + case ExporterMod:start_export(Exporter, Transfer, Filemeta) of + {ok, Export} -> + {ok, {ExporterMod, Export}}; + {error, _} = Error -> + Error + end. + +export_write({ExporterMod, Export}, Content) -> + case ExporterMod:write(Export, Content) of + {ok, ExportNext} -> + {ok, {ExporterMod, ExportNext}}; + {error, _} = Error -> + Error + end. + +export_complete({ExporterMod, Export}) -> + ExporterMod:complete(Export). + +export_discard({ExporterMod, Export}) -> + ExporterMod:discard(Export). + +%% + +maybe_garbage_collect(ok, #st{storage = Storage, transfer = Transfer, assembly = Asm}) -> + Nodes = emqx_ft_assembly:nodes(Asm), emqx_ft_storage_fs_gc:collect(Storage, Transfer, Nodes); maybe_garbage_collect({error, _}, _St) -> ok. diff --git a/apps/emqx_ft/src/emqx_ft_fs_util.erl b/apps/emqx_ft/src/emqx_ft_fs_util.erl new file mode 100644 index 000000000..198f4ccc5 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_fs_util.erl @@ -0,0 +1,101 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_fs_util). + +-include_lib("snabbkaffe/include/trace.hrl"). + +-export([read_decode_file/2]). + +-export([fold/4]). + +-type glob() :: ['*' | globfun()]. +-type globfun() :: + fun((_Filename :: file:name()) -> boolean()). +-type foldfun(Acc) :: + fun( + ( + _Filepath :: file:name(), + _Info :: file:file_info() | {error, _IoError}, + _Stack :: [file:name()], + Acc + ) -> Acc + ). + +%% + +-spec read_decode_file(file:name(), fun((binary()) -> Value)) -> + {ok, Value} | {error, _IoError}. +read_decode_file(Filepath, DecodeFun) -> + case file:read_file(Filepath) of + {ok, Content} -> + safe_decode(Content, DecodeFun); + {error, _} = Error -> + Error + end. + +safe_decode(Content, DecodeFun) -> + try + {ok, DecodeFun(Content)} + catch + C:E:Stacktrace -> + ?tp(warning, "safe_decode_failed", #{ + class => C, + exception => E, + stacktrace => Stacktrace + }), + {error, corrupted} + end. + +-spec fold(foldfun(Acc), Acc, _Root :: file:name(), glob()) -> + Acc. +fold(Fun, Acc, Root, Glob) -> + fold(Fun, Acc, [], Root, Glob, []). + +fold(Fun, AccIn, Path, Root, [Glob | Rest], Stack) when Glob == '*' orelse is_function(Glob) -> + case file:list_dir(filename:join(Root, Path)) of + {ok, Filenames} -> + lists:foldl( + fun(FN, Acc) -> + case matches_glob(Glob, FN) of + true when Path == [] -> + fold(Fun, Acc, FN, Root, Rest, [FN | Stack]); + true -> + fold(Fun, Acc, filename:join(Path, FN), Root, Rest, [FN | Stack]); + false -> + Acc + end + end, + AccIn, + Filenames + ); + {error, enotdir} -> + AccIn; + {error, Reason} -> + Fun(Path, {error, Reason}, Stack, AccIn) + end; +fold(Fun, AccIn, Filepath, Root, [], Stack) -> + case file:read_link_info(filename:join(Root, Filepath), [{time, posix}, raw]) of + {ok, Info} -> + Fun(Filepath, Info, Stack, AccIn); + {error, Reason} -> + Fun(Filepath, {error, Reason}, Stack, AccIn) + end. + +matches_glob('*', _) -> + true; +matches_glob(FilterFun, Filename) when is_function(FilterFun) -> + FilterFun(Filename). diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index d2b2b9299..37e2adafc 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -66,12 +66,33 @@ fields(local_storage) -> desc => ?DESC("local_storage_root"), required => false }}, + {exporter, #{ + type => hoconsc:union([ + ?REF(local_storage_exporter) + ]), + desc => ?DESC("local_storage_exporter"), + required => true + }}, {gc, #{ - type => hoconsc:ref(?MODULE, local_storage_gc), + type => ?REF(local_storage_gc), desc => ?DESC("local_storage_gc"), required => false }} ]; +fields(local_storage_exporter) -> + [ + {type, #{ + type => local, + default => local, + required => false, + desc => ?DESC("local_storage_exporter_type") + }}, + {root, #{ + type => binary(), + desc => ?DESC("local_storage_exporter_root"), + required => false + }} + ]; fields(local_storage_gc) -> [ {interval, #{ @@ -101,12 +122,15 @@ desc(file_transfer) -> "File transfer settings"; desc(local_storage) -> "File transfer local storage settings"; +desc(local_storage_exporter) -> + "Exporter settings for the File transfer local storage backend"; desc(local_storage_gc) -> "Garbage collection settings for the File transfer local storage backend". schema(filemeta) -> #{ roots => [ + % TODO nonempty {name, hoconsc:mk(string(), #{required => true})}, {size, hoconsc:mk(non_neg_integer())}, {expire_at, hoconsc:mk(non_neg_integer())}, diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 0b8c38736..1d1c08ce9 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -24,8 +24,7 @@ store_segment/2, assemble/2, - ready_transfers/0, - get_ready_transfer/1, + exports/0, with_storage_type/3 ] @@ -34,12 +33,19 @@ -type storage() :: emqx_config:config(). -export_type([assemble_callback/0]). +-export_type([export_data/0]). -type assemble_callback() :: fun((ok | {error, term()}) -> any()). --type ready_transfer_id() :: term(). --type ready_transfer_info() :: map(). --type ready_transfer_data() :: binary() | qlc:query_handle(). +-type export_info() :: #{ + transfer := emqx_ft:transfer(), + name := file:name(), + size := _Bytes :: non_neg_integer(), + uri => uri_string:uri_string(), + meta => emqx_ft:filemeta() +}. + +-type export_data() :: binary() | qlc:query_handle(). %%-------------------------------------------------------------------- %% Behaviour @@ -57,10 +63,9 @@ ok | {async, pid()} | {error, term()}. -callback assemble(storage(), emqx_ft:transfer(), _Size :: emqx_ft:bytes()) -> ok | {async, pid()} | {error, term()}. --callback ready_transfers(storage()) -> - {ok, [{ready_transfer_id(), ready_transfer_info()}]} | {error, term()}. --callback get_ready_transfer(storage(), ready_transfer_id()) -> - {ok, ready_transfer_data()} | {error, term()}. + +-callback exports(storage()) -> + {ok, [export_info()]} | {error, term()}. %%-------------------------------------------------------------------- %% API @@ -95,15 +100,11 @@ assemble(Transfer, Size) -> Mod = mod(), Mod:assemble(storage(), Transfer, Size). --spec ready_transfers() -> {ok, [{ready_transfer_id(), ready_transfer_info()}]} | {error, term()}. -ready_transfers() -> +-spec exports() -> + {ok, [export_info()]} | {error, term()}. +exports() -> Mod = mod(), - Mod:ready_transfers(storage()). - --spec get_ready_transfer(ready_transfer_id()) -> {ok, ready_transfer_data()} | {error, term()}. -get_ready_transfer(ReadyTransferId) -> - Mod = mod(), - Mod:get_ready_transfer(storage(), ReadyTransferId). + Mod:exports(storage()). -spec with_storage_type(atom(), atom(), list(term())) -> any(). with_storage_type(Type, Fun, Args) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl new file mode 100644 index 000000000..e0bffd444 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -0,0 +1,352 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_exporter_fs). + +-include_lib("kernel/include/file.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% Exporter API +-export([start_export/3]). +-export([write/2]). +-export([complete/1]). +-export([discard/1]). + +-export([list_local/1]). +-export([list_local/2]). +-export([start_reader/3]). + +-export([list/1]). +% -export([list/2]). + +-export_type([export/0]). + +-type options() :: _TODO. +-type transfer() :: emqx_ft:transfer(). +-type filemeta() :: emqx_ft:filemeta(). +-type exportinfo() :: #{ + transfer := transfer(), + name := file:name(), + uri := uri_string:uri_string(), + timestamp := emqx_datetime:epoch_second(), + size := _Bytes :: non_neg_integer(), + meta => filemeta() +}. + +-type file_error() :: emqx_ft_storage_fs:file_error(). + +-opaque export() :: #{ + path := file:name(), + handle := io:device(), + result := file:name(), + meta := filemeta(), + hash := crypto:hash_state() +}. + +-type reader() :: pid(). + +-define(TEMPDIR, "tmp"). +-define(MANIFEST, ".MANIFEST.json"). + +%% NOTE +%% Bucketing of resulting files to accomodate the storage backend for considerably +%% large (e.g. > 10s of millions) amount of files. +-define(BUCKET_HASH, sha). + +%% 2 symbols = at most 256 directories on the upper level +-define(BUCKET1_LEN, 2). +%% 2 symbols = at most 256 directories on the second level +-define(BUCKET2_LEN, 2). + +-define(SLOG_UNEXPECTED(RelFilepath, Fileinfo, Options), + ?SLOG(notice, "filesystem_object_unexpected", #{ + relpath => RelFilepath, + fileinfo => Fileinfo, + options => Options + }) +). + +-define(SLOG_INACCESSIBLE(RelFilepath, Reason, Options), + ?SLOG(warning, "filesystem_object_inaccessible", #{ + relpath => RelFilepath, + reason => Reason, + options => Options + }) +). + +%% + +-spec start_export(options(), transfer(), filemeta()) -> + {ok, export()} | {error, file_error()}. +start_export(Options, Transfer, Filemeta = #{name := Filename}) -> + TempFilepath = mk_temp_absfilepath(Options, Transfer, Filename), + ResultFilepath = mk_absfilepath(Options, Transfer, result, Filename), + _ = filelib:ensure_dir(TempFilepath), + case file:open(TempFilepath, [write, raw, binary]) of + {ok, Handle} -> + {ok, #{ + path => TempFilepath, + handle => Handle, + result => ResultFilepath, + meta => Filemeta, + hash => init_checksum(Filemeta) + }}; + {error, _} = Error -> + Error + end. + +-spec write(export(), iodata()) -> + {ok, export()} | {error, file_error()}. +write(Export = #{handle := Handle, hash := Ctx}, IoData) -> + case file:write(Handle, IoData) of + ok -> + {ok, Export#{hash := update_checksum(Ctx, IoData)}}; + {error, _} = Error -> + Error + end. + +-spec complete(export()) -> + ok | {error, {checksum, _Algo, _Computed}} | {error, file_error()}. +complete( + Export = #{ + path := Filepath, + handle := Handle, + result := ResultFilepath, + meta := FilemetaIn, + hash := Ctx + } +) -> + case verify_checksum(Ctx, FilemetaIn) of + {ok, Filemeta} -> + ok = file:close(Handle), + _ = filelib:ensure_dir(ResultFilepath), + _ = file:write_file(mk_manifest_filename(ResultFilepath), encode_filemeta(Filemeta)), + file:rename(Filepath, ResultFilepath); + {error, _} = Error -> + _ = discard(Export), + Error + end. + +-spec discard(export()) -> + ok. +discard(#{path := Filepath, handle := Handle}) -> + ok = file:close(Handle), + file:delete(Filepath). + +%% + +-spec list_local(options(), transfer()) -> + {ok, [exportinfo(), ...]} | {error, file_error()}. +list_local(Options, Transfer) -> + TransferRoot = mk_absdir(Options, Transfer, result), + case + emqx_ft_fs_util:fold( + fun + (_Path, {error, Reason}, [], []) -> + {error, Reason}; + (_Path, Fileinfo = #file_info{type = regular}, [Filename | _], Acc) -> + RelFilepath = filename:join(mk_result_reldir(Transfer) ++ [Filename]), + Info = mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo), + [Info | Acc]; + (RelFilepath, Fileinfo = #file_info{}, _, Acc) -> + ?SLOG_UNEXPECTED(RelFilepath, Fileinfo, Options), + Acc; + (RelFilepath, {error, Reason}, _, Acc) -> + ?SLOG_INACCESSIBLE(RelFilepath, Reason, Options), + Acc + end, + [], + TransferRoot, + [fun filter_manifest/1] + ) + of + Infos = [_ | _] -> + {ok, Infos}; + [] -> + {error, enoent}; + {error, Reason} -> + {error, Reason} + end. + +-spec list_local(options()) -> + {ok, #{transfer() => [exportinfo(), ...]}}. +list_local(Options) -> + Pattern = [ + _Bucket1 = '*', + _Bucket2 = '*', + _Rest = '*', + _ClientId = '*', + _FileId = '*', + fun filter_manifest/1 + ], + Root = get_storage_root(Options), + {ok, + emqx_ft_fs_util:fold( + fun(RelFilepath, Info, Stack, Acc) -> + read_exportinfo(Options, RelFilepath, Info, Stack, Acc) + end, + [], + Root, + Pattern + )}. + +filter_manifest(?MANIFEST) -> + % Filename equals `?MANIFEST`, there should also be a manifest for it. + false; +filter_manifest(Filename) -> + ?MANIFEST =/= string:find(Filename, ?MANIFEST, trailing). + +read_exportinfo(Options, RelFilepath, Fileinfo = #file_info{type = regular}, Stack, Acc) -> + [Filename, FileId, ClientId | _] = Stack, + Transfer = dirnames_to_transfer(ClientId, FileId), + Info = mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo), + [Info | Acc]; +read_exportinfo(Options, RelFilepath, Fileinfo = #file_info{}, _Stack, Acc) -> + ?SLOG_UNEXPECTED(RelFilepath, Fileinfo, Options), + Acc; +read_exportinfo(Options, RelFilepath, {error, Reason}, _Stack, Acc) -> + ?SLOG_INACCESSIBLE(RelFilepath, Reason, Options), + Acc. + +mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo) -> + Root = get_storage_root(Options), + try_read_filemeta( + filename:join(Root, mk_manifest_filename(RelFilepath)), + #{ + transfer => Transfer, + name => Filename, + uri => mk_export_uri(Options, RelFilepath), + timestamp => Fileinfo#file_info.mtime, + size => Fileinfo#file_info.size, + path => filename:join(Root, RelFilepath) + } + ). + +try_read_filemeta(Filepath, Info) -> + case emqx_ft_fs_util:read_decode_file(Filepath, fun decode_filemeta/1) of + {ok, Filemeta} -> + Info#{meta => Filemeta}; + {error, Reason} -> + ?SLOG(warning, "filemeta_inaccessible", #{ + path => Filepath, + reason => Reason + }), + Info + end. + +mk_export_uri(Options, RelFilepath) -> + % emqx_ft_storage_exporter_fs_api:mk_export_uri(Options, RelFilepath). + emqx_ft_api:mk_file_uri(Options, node(), RelFilepath). + +-spec start_reader(options(), file:name(), _Caller :: pid()) -> + {ok, reader()} | {error, enoent}. +start_reader(Options, Filepath, CallerPid) -> + Root = get_storage_root(Options), + case filelib:safe_relative_path(Filepath, Root) of + SafeFilepath when SafeFilepath /= unsafe -> + AbsFilepath = filename:join(Root, SafeFilepath), + emqx_ft_storage_fs_reader:start_supervised(CallerPid, AbsFilepath); + unsafe -> + {error, enoent} + end. + +%% + +-spec list(options()) -> + {ok, [exportinfo(), ...]} | {error, file_error()}. +list(_Options) -> + Nodes = mria_mnesia:running_nodes(), + Results = emqx_ft_storage_fs_proto_v1:list_exports(Nodes), + {GoodResults, BadResults} = lists:partition( + fun + ({_Node, {ok, {ok, _}}}) -> true; + (_) -> false + end, + lists:zip(Nodes, Results) + ), + length(BadResults) > 0 andalso + ?SLOG(warning, #{msg => "list_remote_exports_failed", failures => BadResults}), + {ok, [File || {_Node, {ok, {ok, Files}}} <- GoodResults, File <- Files]}. + +%% + +init_checksum(#{checksum := {Algo, _}}) -> + crypto:hash_init(Algo); +init_checksum(#{}) -> + crypto:hash_init(sha256). + +update_checksum(Ctx, IoData) -> + crypto:hash_update(Ctx, IoData). + +verify_checksum(Ctx, Filemeta = #{checksum := {Algo, Digest}}) -> + case crypto:hash_final(Ctx) of + Digest -> + {ok, Filemeta}; + Mismatch -> + {error, {checksum, Algo, binary:encode_hex(Mismatch)}} + end; +verify_checksum(Ctx, Filemeta = #{}) -> + Digest = crypto:hash_final(Ctx), + {ok, Filemeta#{checksum => {sha256, Digest}}}. + +%% + +-define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). + +encode_filemeta(Meta) -> + emqx_json:encode(?PRELUDE(_Vsn = 1, emqx_ft:encode_filemeta(Meta))). + +decode_filemeta(Binary) when is_binary(Binary) -> + ?PRELUDE(_Vsn = 1, Map) = emqx_json:decode(Binary, [return_maps]), + case emqx_ft:decode_filemeta(Map) of + {ok, Meta} -> + Meta; + {error, Reason} -> + error(Reason) + end. + +mk_manifest_filename(Filename) when is_list(Filename) -> + Filename ++ ?MANIFEST; +mk_manifest_filename(Filename) when is_binary(Filename) -> + <>. + +mk_temp_absfilepath(Options, Transfer, Filename) -> + Unique = erlang:unique_integer([positive]), + TempFilename = integer_to_list(Unique) ++ "." ++ Filename, + filename:join(mk_absdir(Options, Transfer, temporary), TempFilename). + +mk_absdir(Options, _Transfer, temporary) -> + filename:join([get_storage_root(Options), ?TEMPDIR]); +mk_absdir(Options, Transfer, result) -> + filename:join([get_storage_root(Options) | mk_result_reldir(Transfer)]). + +mk_absfilepath(Options, Transfer, What, Filename) -> + filename:join(mk_absdir(Options, Transfer, What), Filename). + +mk_result_reldir(Transfer = {ClientId, FileId}) -> + Hash = mk_transfer_hash(Transfer), + << + Bucket1:?BUCKET1_LEN/binary, + Bucket2:?BUCKET2_LEN/binary, + BucketRest/binary + >> = binary:encode_hex(Hash), + [Bucket1, Bucket2, BucketRest, ClientId, FileId]. + +mk_transfer_hash(Transfer) -> + crypto:hash(?BUCKET_HASH, term_to_binary(Transfer)). + +get_storage_root(Options) -> + maps:get(root, Options, filename:join([emqx:data_dir(), "ft", "exports"])). diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index b8aef5276..4a613dbcf 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -29,6 +29,7 @@ -export([child_spec/1]). +% Segments-related API -export([store_filemeta/3]). -export([store_segment/3]). -export([read_filemeta/2]). @@ -43,22 +44,20 @@ -export([get_subdir/2]). -export([get_subdir/3]). --export([ready_transfers_local/1]). --export([get_ready_transfer_local/3]). +-export([exporter/1]). --export([ready_transfers/1]). --export([get_ready_transfer/2]). - --export([open_file/3]). --export([complete/4]). --export([write/2]). --export([discard/1]). +% Exporter-specific API +-export([exports/1]). +-export([exports_local/1]). +-export([exports_local/2]). -export_type([storage/0]). -export_type([filefrag/1]). -export_type([filefrag/0]). -export_type([transferinfo/0]). +-export_type([file_error/0]). + -type transfer() :: emqx_ft:transfer(). -type offset() :: emqx_ft:offset(). -type filemeta() :: emqx_ft:filemeta(). @@ -70,8 +69,7 @@ }. -type transferinfo() :: #{ - status := complete | incomplete, - result => [filefrag({result, #{}})] + filemeta => filemeta() }. % TODO naming @@ -85,16 +83,20 @@ -type filefrag() :: filefrag( {filemeta, filemeta()} | {segment, segmentinfo()} - | {result, #{}} ). -define(FRAGDIR, frags). -define(TEMPDIR, tmp). --define(RESULTDIR, result). -define(MANIFEST, "MANIFEST.json"). -define(SEGMENT, "SEG"). -type storage() :: #{ + root => file:name(), + exporter => exporter() +}. + +-type exporter() :: #{ + type := local, root => file:name() }. @@ -138,7 +140,9 @@ store_filemeta(Storage, Transfer, Meta) -> % about it too much now. {error, conflict}; {error, Reason} when Reason =:= notfound; Reason =:= corrupted; Reason =:= enoent -> - write_file_atomic(Storage, Transfer, Filepath, encode_filemeta(Meta)) + write_file_atomic(Storage, Transfer, Filepath, encode_filemeta(Meta)); + {error, _} = Error -> + Error end. %% Store a segment in the backing filesystem. @@ -153,17 +157,17 @@ store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> write_file_atomic(Storage, Transfer, Filepath, Content). -spec read_filemeta(storage(), transfer()) -> - {ok, filefrag({filemeta, filemeta()})} | {error, corrupted} | {error, file_error()}. + {ok, filemeta()} | {error, corrupted} | {error, file_error()}. read_filemeta(Storage, Transfer) -> Filepath = mk_filepath(Storage, Transfer, get_subdirs_for(fragment), ?MANIFEST), read_file(Filepath, fun decode_filemeta/1). --spec list(storage(), transfer(), _What :: fragment | result) -> +-spec list(storage(), transfer(), _What :: fragment) -> % Some lower level errors? {error, notfound}? % Result will contain zero or only one filemeta. {ok, [filefrag({filemeta, filemeta()} | {segment, segmentinfo()})]} | {error, file_error()}. -list(Storage, Transfer, What) -> +list(Storage, Transfer, What = fragment) -> Dirname = mk_filedir(Storage, Transfer, get_subdirs_for(What)), case file:list_dir(Dirname) of {ok, Filenames} -> @@ -172,18 +176,13 @@ list(Storage, Transfer, What) -> % extremely bad luck is needed for that, e.g. concurrent assemblers with % different filemetas from different nodes). This might be unexpected for a % client given the current protocol, yet might be helpful in the future. - {ok, filtermap_files(get_filefrag_fun_for(What), Dirname, Filenames)}; + {ok, filtermap_files(fun mk_filefrag/2, Dirname, Filenames)}; {error, enoent} -> {ok, []}; {error, _} = Error -> Error end. -get_filefrag_fun_for(fragment) -> - fun mk_filefrag/2; -get_filefrag_fun_for(result) -> - fun mk_result_filefrag/2. - -spec pread(storage(), transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> {ok, _Content :: iodata()} | {error, eof} | {error, file_error()}. pread(_Storage, _Transfer, Frag, Offset, Size) -> @@ -213,102 +212,28 @@ assemble(Storage, Transfer, Size) -> {ok, Pid} = emqx_ft_assembler_sup:ensure_child(Storage, Transfer, Size), {async, Pid}. -get_ready_transfer(_Storage, ReadyTransferId) -> - case parse_ready_transfer_id(ReadyTransferId) of - {ok, {Node, Transfer}} -> - try - case emqx_ft_storage_fs_proto_v1:get_ready_transfer(Node, self(), Transfer) of - {ok, ReaderPid} -> - {ok, emqx_ft_storage_fs_reader:table(ReaderPid)}; - {error, _} = Error -> - Error - end - catch - error:Exc:Stacktrace -> - ?SLOG(warning, #{ - msg => "get_ready_transfer_error", - node => Node, - transfer => Transfer, - exception => Exc, - stacktrace => Stacktrace - }), - {error, Exc}; - C:Exc:Stacktrace -> - ?SLOG(warning, #{ - msg => "get_ready_transfer_fail", - class => C, - node => Node, - transfer => Transfer, - exception => Exc, - stacktrace => Stacktrace - }), - {error, {C, Exc}} - end; - {error, _} = Error -> - Error +%% + +-spec exporter(storage()) -> {module(), _ExporterOptions}. +exporter(Storage) -> + case maps:get(exporter, Storage) of + #{type := local} = Options -> + {emqx_ft_storage_exporter_fs, Options} end. -get_ready_transfer_local(Storage, CallerPid, Transfer) -> - Dirname = mk_filedir(Storage, Transfer, get_subdirs_for(result)), - case file:list_dir(Dirname) of - {ok, [Filename | _]} -> - FullFilename = filename:join([Dirname, Filename]), - emqx_ft_storage_fs_reader:start_supervised(CallerPid, FullFilename); - {error, _} = Error -> - Error - end. +exports(Storage) -> + {ExporterMod, ExporterOpts} = exporter(Storage), + ExporterMod:list(ExporterOpts). -ready_transfers(_Storage) -> - Nodes = mria_mnesia:running_nodes(), - Results = emqx_ft_storage_fs_proto_v1:ready_transfers(Nodes), - {GoodResults, BadResults} = lists:partition( - fun - ({ok, _}) -> true; - (_) -> false - end, - Results - ), - case {GoodResults, BadResults} of - {[], _} -> - ?SLOG(warning, #{msg => "ready_transfers", failures => BadResults}), - {error, no_nodes}; - {_, []} -> - {ok, [File || {ok, Files} <- GoodResults, File <- Files]}; - {_, _} -> - ?SLOG(warning, #{msg => "ready_transfers", failures => BadResults}), - {ok, [File || {ok, Files} <- GoodResults, File <- Files]} - end. +exports_local(Storage) -> + {ExporterMod, ExporterOpts} = exporter(Storage), + ExporterMod:list_local(ExporterOpts). -ready_transfers_local(Storage) -> - {ok, Transfers} = transfers(Storage), - lists:filtermap( - fun - ({Transfer, #{status := complete, result := [Result | _]}}) -> - {true, {ready_transfer_id(Transfer), maps:without([fragment], Result)}}; - (_) -> - false - end, - maps:to_list(Transfers) - ). +exports_local(Storage, Transfer) -> + {ExporterMod, ExporterOpts} = exporter(Storage), + ExporterMod:list_local(ExporterOpts, Transfer). -ready_transfer_id({ClientId, FileId}) -> - #{ - <<"node">> => atom_to_binary(node()), - <<"clientid">> => ClientId, - <<"fileid">> => FileId - }. - -parse_ready_transfer_id(#{ - <<"node">> := NodeBin, <<"clientid">> := ClientId, <<"fileid">> := FileId -}) -> - case emqx_misc:safe_to_existing_atom(NodeBin) of - {ok, Node} -> - {ok, {Node, {ClientId, FileId}}}; - {error, _} -> - {error, {invalid_node, NodeBin}} - end; -parse_ready_transfer_id(#{}) -> - {error, invalid_file_id}. +%% -spec transfers(storage()) -> {ok, #{transfer() => transferinfo()}}. @@ -345,17 +270,16 @@ transfers(Storage, ClientId, AccIn) -> end. read_transferinfo(Storage, Transfer, Acc) -> - case list(Storage, Transfer, result) of - {ok, Result = [_ | _]} -> - Info = #{status => complete, result => Result}, - Acc#{Transfer => Info}; - {ok, []} -> - Info = #{status => incomplete}, - Acc#{Transfer => Info}; - {error, _Reason} -> - ?tp(warning, "list_result_failed", #{ + case read_filemeta(Storage, Transfer) of + {ok, Filemeta} -> + Acc#{Transfer => #{filemeta => Filemeta}}; + {error, enoent} -> + Acc#{Transfer => #{}}; + {error, Reason} -> + ?tp(warning, "read_transferinfo_failed", #{ storage => Storage, - transfer => Transfer + transfer => Transfer, + reason => Reason }), Acc end. @@ -365,7 +289,7 @@ read_transferinfo(Storage, Transfer, Acc) -> get_subdir(Storage, Transfer) -> mk_filedir(Storage, Transfer, []). --spec get_subdir(storage(), transfer(), fragment | temporary | result) -> +-spec get_subdir(storage(), transfer(), fragment | temporary) -> file:name(). get_subdir(Storage, Transfer, What) -> mk_filedir(Storage, Transfer, get_subdirs_for(What)). @@ -373,84 +297,12 @@ get_subdir(Storage, Transfer, What) -> get_subdirs_for(fragment) -> [?FRAGDIR]; get_subdirs_for(temporary) -> - [?TEMPDIR]; -get_subdirs_for(result) -> - [?RESULTDIR]. - -%% - --type handle() :: {file:name(), io:device(), crypto:hash_state()}. - --spec open_file(storage(), transfer(), filemeta()) -> - {ok, handle()} | {error, file_error()}. -open_file(Storage, Transfer, Filemeta) -> - Filename = maps:get(name, Filemeta), - TempFilepath = mk_temp_filepath(Storage, Transfer, Filename), - _ = filelib:ensure_dir(TempFilepath), - case file:open(TempFilepath, [write, raw, binary]) of - {ok, Handle} -> - % TODO: preserve filemeta - {ok, {TempFilepath, Handle, init_checksum(Filemeta)}}; - {error, _} = Error -> - Error - end. - --spec write(handle(), iodata()) -> - {ok, handle()} | {error, file_error()}. -write({Filepath, IoDevice, Ctx}, IoData) -> - case file:write(IoDevice, IoData) of - ok -> - {ok, {Filepath, IoDevice, update_checksum(Ctx, IoData)}}; - {error, _} = Error -> - Error - end. - --spec complete(storage(), transfer(), filemeta(), handle()) -> - ok | {error, {checksum, _Algo, _Computed}} | {error, file_error()}. -complete(Storage, Transfer, Filemeta = #{name := Filename}, Handle = {Filepath, IoDevice, Ctx}) -> - TargetFilepath = mk_filepath(Storage, Transfer, get_subdirs_for(result), Filename), - case verify_checksum(Ctx, Filemeta) of - ok -> - ok = file:close(IoDevice), - mv_temp_file(Filepath, TargetFilepath); - {error, _} = Error -> - _ = discard(Handle), - Error - end. - --spec discard(handle()) -> - ok. -discard({Filepath, IoDevice, _Ctx}) -> - ok = file:close(IoDevice), - file:delete(Filepath). - -init_checksum(#{checksum := {Algo, _}}) -> - crypto:hash_init(Algo); -init_checksum(#{}) -> - undefined. - -update_checksum(Ctx, IoData) when Ctx /= undefined -> - crypto:hash_update(Ctx, IoData); -update_checksum(undefined, _IoData) -> - undefined. - -verify_checksum(Ctx, #{checksum := {Algo, Digest}}) when Ctx /= undefined -> - case crypto:hash_final(Ctx) of - Digest -> - ok; - Mismatch -> - {error, {checksum, Algo, binary:encode_hex(Mismatch)}} - end; -verify_checksum(undefined, _) -> - ok. + [?TEMPDIR]. -define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). encode_filemeta(Meta) -> - % TODO: Looks like this should be hocon's responsibility. - Schema = emqx_ft_schema:schema(filemeta), - Term = hocon_tconf:make_serializable(Schema, emqx_map_lib:binary_key_map(Meta), #{}), - emqx_json:encode(?PRELUDE(_Vsn = 1, Term)). + emqx_json:encode(?PRELUDE(_Vsn = 1, emqx_ft:encode_filemeta(Meta))). decode_filemeta(Binary) when is_binary(Binary) -> ?PRELUDE(_Vsn = 1, Map) = emqx_json:decode(Binary, [return_maps]), @@ -490,33 +342,12 @@ try_list_dir(Dirname) -> end. get_storage_root(Storage) -> - maps:get(root, Storage, filename:join(emqx:data_dir(), "file_transfer")). + maps:get(root, Storage, filename:join([emqx:data_dir(), "ft", "transfers"])). -include_lib("kernel/include/file.hrl"). -read_file(Filepath) -> - file:read_file(Filepath). - read_file(Filepath, DecodeFun) -> - case read_file(Filepath) of - {ok, Content} -> - safe_decode(Content, DecodeFun); - {error, _} = Error -> - Error - end. - -safe_decode(Content, DecodeFun) -> - try - {ok, DecodeFun(Content)} - catch - C:E:Stacktrace -> - ?tp(warning, "safe_decode_failed", #{ - class => C, - exception => E, - stacktrace => Stacktrace - }), - {error, corrupted} - end. + emqx_ft_fs_util:read_decode_file(Filepath, DecodeFun). write_file_atomic(Storage, Transfer, Filepath, Content) when is_binary(Content) -> TempFilepath = mk_temp_filepath(Storage, Transfer, filename:basename(Filepath)), @@ -574,12 +405,6 @@ mk_filefrag(_Dirname, _Filename) -> }), false. -mk_result_filefrag(Dirname, Filename) -> - % NOTE - % Any file in the `?RESULTDIR` subdir is currently considered the result of - % the file transfer. - mk_filefrag(Dirname, Filename, result, fun(_, _) -> {ok, #{}} end). - mk_filefrag(Dirname, Filename, Tag, Fun) -> Filepath = filename:join(Dirname, Filename), % TODO error handling? diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 58c5dbfdf..20e4f468d 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -308,28 +308,18 @@ is_same_filepath(P1, P2) when is_binary(P1) -> filepath_to_binary(S) -> unicode:characters_to_binary(S, unicode, file:native_name_encoding()). -get_segments_ttl(Storage, Transfer) -> +get_segments_ttl(Storage, TransferInfo) -> {MinTTL, MaxTTL} = emqx_ft_conf:segments_ttl(Storage), - clamp(MinTTL, MaxTTL, try_get_filemeta_ttl(Storage, Transfer)). + clamp(MinTTL, MaxTTL, try_get_filemeta_ttl(TransferInfo)). -try_get_filemeta_ttl(Storage, Transfer) -> - case emqx_ft_storage_fs:read_filemeta(Storage, Transfer) of - {ok, Filemeta} -> - maps:get(segments_ttl, Filemeta, undefined); - {error, _} -> - undefined - end. +try_get_filemeta_ttl(#{filemeta := Filemeta}) -> + maps:get(segments_ttl, Filemeta, undefined); +try_get_filemeta_ttl(#{}) -> + undefined. clamp(Min, Max, V) -> min(Max, max(Min, V)). -% try_collect(_Subject, ok = Result, Then, _Stats) -> -% Then(Result); -% try_collect(_Subject, {ok, Result}, Then, _Stats) -> -% Then(Result); -% try_collect(Subject, {error, _} = Error, _Then, Stats) -> -% register_gcstat_error(Subject, Error, Stats). - %% init_gcstats() -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl index 7e19dd322..dbd0cd6dc 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl @@ -23,8 +23,8 @@ -export([ list_local/2, pread_local/4, - get_ready_transfer_local/2, - ready_transfers_local/0 + list_exports_local/0, + read_export_file_local/2 ]). list_local(Transfer, What) -> @@ -33,8 +33,18 @@ list_local(Transfer, What) -> pread_local(Transfer, Frag, Offset, Size) -> emqx_ft_storage:with_storage_type(local, pread, [Transfer, Frag, Offset, Size]). -get_ready_transfer_local(CallerPid, Transfer) -> - emqx_ft_storage:with_storage_type(local, get_ready_transfer_local, [CallerPid, Transfer]). +list_exports_local() -> + case emqx_ft_storage:with_storage_type(local, exporter, []) of + {emqx_ft_storage_exporter_fs, Options} -> + emqx_ft_storage_exporter_fs:list_local(Options); + InvalidExporter -> + {error, {invalid_exporter, InvalidExporter}} + end. -ready_transfers_local() -> - emqx_ft_storage:with_storage_type(local, ready_transfers_local, []). +read_export_file_local(Filepath, CallerPid) -> + case emqx_ft_storage:with_storage_type(local, exporter, []) of + {emqx_ft_storage_exporter_fs, Options} -> + emqx_ft_storage_exporter_fs:start_reader(Options, Filepath, CallerPid); + InvalidExporter -> + {error, {invalid_exporter, InvalidExporter}} + end. diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index e2c4c93d7..f152928fe 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -22,8 +22,10 @@ -export([multilist/3]). -export([pread/5]). --export([ready_transfers/1]). --export([get_ready_transfer/3]). + +%% TODO: These should be defined in a separate BPAPI +-export([list_exports/1]). +-export([read_export_file/3]). -type offset() :: emqx_ft:offset(). -type transfer() :: emqx_ft:transfer(). @@ -44,19 +46,16 @@ multilist(Nodes, Transfer, What) -> pread(Node, Transfer, Frag, Offset, Size) -> erpc:call(Node, emqx_ft_storage_fs_proxy, pread_local, [Transfer, Frag, Offset, Size]). --spec ready_transfers([node()]) -> - [ - {ok, [{emqx_ft_storage:ready_transfer_id(), emqx_ft_storage:ready_transfer_info()}]} - | {error, term()} - | {exit, term()} - | {throw, term()} - ]. -ready_transfers(Nodes) -> - erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, ready_transfers_local, []). +%% --spec get_ready_transfer(node(), pid(), emqx_ft_storage:ready_transfer_id()) -> - {ok, emqx_ft_storage:ready_transfer_data()} +-spec list_exports([node()]) -> + emqx_rpc:erpc_multicall([emqx_ft_storage:export_info()]). +list_exports(Nodes) -> + erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, list_exports_local, []). + +-spec read_export_file(node(), file:name(), pid()) -> + {ok, emqx_ft_storage:export_data()} | {error, term()} | no_return(). -get_ready_transfer(Node, CallerPid, ReadyTransferId) -> - erpc:call(Node, emqx_ft_storage_fs_proxy, get_ready_transfer_local, [CallerPid, ReadyTransferId]). +read_export_file(Node, Filepath, CallerPid) -> + erpc:call(Node, emqx_ft_storage_fs_proxy, read_export_file_local, [Filepath, CallerPid]). diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 9c9087732..5fb384c16 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -58,8 +58,9 @@ end_per_suite(_Config) -> set_special_configs(Config) -> fun (emqx_ft) -> - Root = emqx_ft_test_helpers:ft_root(Config, node()), - emqx_ft_test_helpers:load_config(#{storage => #{type => local, root => Root}}); + emqx_ft_test_helpers:load_config(#{ + storage => emqx_ft_test_helpers:local_storage(Config) + }); (_) -> ok end. @@ -108,8 +109,9 @@ mk_cluster_specs(Config) -> {conf, [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]]}, {env_handler, fun (emqx_ft) -> - Root = emqx_ft_test_helpers:ft_root(Config, node()), - emqx_ft_test_helpers:load_config(#{storage => #{type => local, root => Root}}); + emqx_ft_test_helpers:load_config(#{ + storage => emqx_ft_test_helpers:local_storage(Config) + }); (_) -> ok end} @@ -194,11 +196,10 @@ t_simple_transfer(Config) -> emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) ), - [ReadyTransferId] = list_ready_transfers(?config(clientid, Config)), - {ok, TableQH} = emqx_ft_storage:get_ready_transfer(ReadyTransferId), + [Export] = list_exports(?config(clientid, Config)), ?assertEqual( - iolist_to_binary(Data), - iolist_to_binary(qlc:eval(TableQH)) + {ok, iolist_to_binary(Data)}, + read_export(Export) ). t_meta_conflict(Config) -> @@ -424,11 +425,10 @@ t_switch_node(Config) -> %% Now check consistency of the file - [ReadyTransferId] = list_ready_transfers(ClientId), - {ok, TableQH} = emqx_ft_storage:get_ready_transfer(ReadyTransferId), + [Export] = list_exports(ClientId), ?assertEqual( - iolist_to_binary(Data), - iolist_to_binary(qlc:eval(TableQH)) + {ok, iolist_to_binary(Data)}, + read_export(Export) ). t_assemble_crash(Config) -> @@ -501,27 +501,30 @@ t_unreliable_migrating_client(Config) -> ], _Context = run_commands(Commands, Context), - ReadyTransferIds = list_ready_transfers(?config(clientid, Config)), + Exports = list_exports(?config(clientid, Config)), % NOTE % The cluster had 2 assemblers running on two different nodes, because client sent `fin` % twice. This is currently expected, files must be identical anyway. - Node1Bin = atom_to_binary(Node1), - NodeSelfBin = atom_to_binary(NodeSelf), + Node1Str = atom_to_list(Node1), + NodeSelfStr = atom_to_list(NodeSelf), ?assertMatch( - [#{<<"node">> := Node1Bin}, #{<<"node">> := NodeSelfBin}], - lists:sort(ReadyTransferIds) + [#{"node" := Node1Str}, #{"node" := NodeSelfStr}], + lists:map( + fun(#{uri := URIString}) -> + #{query := QS} = uri_string:parse(URIString), + uri_string:dissect_query(QS) + end, + lists:sort(Exports) + ) ), [ - begin - {ok, TableQH} = emqx_ft_storage:get_ready_transfer(Id), - ?assertEqual( - Payload, - iolist_to_binary(qlc:eval(TableQH)) - ) - end - || Id <- ReadyTransferIds + ?assertEqual( + {ok, Payload}, + read_export(Export) + ) + || Export <- Exports ]. run_commands(Commands, Context) -> @@ -620,10 +623,10 @@ meta(FileName, Data) -> size => byte_size(FullData) }. -list_ready_transfers(ClientId) -> - {ok, ReadyTransfers} = emqx_ft_storage:ready_transfers(), - [ - Id - || {#{<<"clientid">> := CId} = Id, _Info} <- ReadyTransfers, - CId == ClientId - ]. +list_exports(ClientId) -> + {ok, Exports} = emqx_ft_storage:exports(), + [Export || Export = #{transfer := {CId, _}} <- Exports, CId == ClientId]. + +read_export(#{path := AbsFilepath}) -> + % TODO: only works for the local filesystem exporter right now + file:read_file(AbsFilepath). diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index 7b191e229..aff4d864c 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -24,7 +24,7 @@ -include_lib("emqx/include/asserts.hrl"). --import(emqx_mgmt_api_test_util, [request/3, uri/1]). +-import(emqx_mgmt_api_test_util, [uri/1]). all() -> emqx_common_test_helpers:all(?MODULE). @@ -41,8 +41,9 @@ end_per_suite(_Config) -> set_special_configs(Config) -> fun (emqx_ft) -> - Root = emqx_ft_test_helpers:ft_root(Config, node()), - emqx_ft_test_helpers:load_config(#{storage => #{type => local, root => Root}}); + emqx_ft_test_helpers:load_config(#{ + storage => emqx_ft_test_helpers:local_storage(Config) + }); (_) -> ok end. @@ -61,40 +62,25 @@ t_list_ready_transfers(Config) -> ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, <<"data">>, node()), - {ok, 200, Response} = request(get, uri(["file_transfer", "files"])), - - #{<<"files">> := Files} = emqx_json:decode(Response, [return_maps]), + {ok, 200, #{<<"files">> := Files}} = + request(get, uri(["file_transfer", "files"]), fun json/1), ?assertInclude( - #{<<"id">> := #{<<"clientid">> := ClientId, <<"fileid">> := <<"f1">>}}, + #{<<"clientid">> := ClientId, <<"fileid">> := <<"f1">>}, Files ). -%% This shouldn't happen in real life -%% but we need to test it anyway -t_list_ready_transfers_no_nodes(_Config) -> - _ = meck:new(mria_mnesia, [passthrough]), - _ = meck:expect(mria_mnesia, running_nodes, fun() -> [] end), - - ?assertMatch( - {ok, 503, _}, - request(get, uri(["file_transfer", "files"])) - ). - t_download_transfer(Config) -> ClientId = client_id(Config), ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, <<"data">>, node()), ?assertMatch( - {ok, 503, _}, + {ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}}, request( get, - uri(["file_transfer", "file"]) ++ - query(#{ - clientid => ClientId, - fileid => <<"f1">> - }) + uri(["file_transfer", "file"]) ++ query(#{fileref => <<"f1">>}), + fun json/1 ) ), @@ -104,8 +90,7 @@ t_download_transfer(Config) -> get, uri(["file_transfer", "file"]) ++ query(#{ - clientid => ClientId, - fileid => <<"f1">>, + fileref => <<"f1">>, node => <<"nonode@nohost">> }) ) @@ -117,22 +102,16 @@ t_download_transfer(Config) -> get, uri(["file_transfer", "file"]) ++ query(#{ - clientid => ClientId, - fileid => <<"unknown_file">>, + fileref => <<"unknown_file">>, node => node() }) ) ), - {ok, 200, Response} = request( - get, - uri(["file_transfer", "file"]) ++ - query(#{ - clientid => ClientId, - fileid => <<"f1">>, - node => node() - }) - ), + {ok, 200, #{<<"files">> := [File]}} = + request(get, uri(["file_transfer", "files"]), fun json/1), + + {ok, 200, Response} = request(get, uri([]) ++ maps:get(<<"uri">>, File)), ?assertEqual( <<"data">>, @@ -147,7 +126,18 @@ client_id(Config) -> atom_to_binary(?config(tc, Config), utf8). request(Method, Url) -> - request(Method, Url, []). + emqx_mgmt_api_test_util:request(Method, Url, []). + +request(Method, Url, Decoder) when is_function(Decoder) -> + case emqx_mgmt_api_test_util:request(Method, Url, []) of + {ok, Code, Body} -> + {ok, Code, Decoder(Body)}; + Otherwise -> + Otherwise + end. + +json(Body) when is_binary(Body) -> + emqx_json:decode(Body, [return_maps]). query(Params) -> KVs = lists:map(fun({K, V}) -> uri_encode(K) ++ "=" ++ uri_encode(V) end, maps:to_list(Params)), diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 8df471deb..4c7c0f08c 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -47,6 +47,7 @@ init_per_testcase(TC, Config) -> {ok, Pid} = emqx_ft_assembler_sup:start_link(), [ {storage_root, "file_transfer_root"}, + {exports_root, "file_transfer_exports"}, {file_id, atom_to_binary(TC)}, {assembler_sup, Pid} | Config @@ -85,11 +86,12 @@ t_assemble_empty_transfer(Config) -> ), Status = complete_assemble(Storage, Transfer, 0), ?assertEqual({shutdown, ok}, Status), - {ok, [Result = #{size := Size = 0}]} = emqx_ft_storage_fs:list(Storage, Transfer, result), - ?assertEqual( - {error, eof}, - emqx_ft_storage_fs:pread(Storage, Transfer, Result, 0, Size) - ), + {ok, [_Result = #{size := _Size = 0}]} = + emqx_ft_storage_fs:exports_local(Storage, Transfer), + % ?assertEqual( + % {error, eof}, + % emqx_ft_storage_fs:pread(Storage, Transfer, Result, 0, Size) + % ), ok. t_assemble_complete_local_transfer(Config) -> @@ -133,12 +135,13 @@ t_assemble_complete_local_transfer(Config) -> {ok, [ #{ size := TransferSize, - fragment := {result, #{}} + meta := #{} } ]}, - emqx_ft_storage_fs:list(Storage, Transfer, result) + emqx_ft_storage_fs:exports_local(Storage, Transfer) ), - {ok, [#{path := AssemblyFilename}]} = emqx_ft_storage_fs:list(Storage, Transfer, result), + {ok, [#{path := AssemblyFilename}]} = + emqx_ft_storage_fs:exports_local(Storage, Transfer), ?assertMatch( {ok, #file_info{type = regular, size = TransferSize}}, file:read_file_info(AssemblyFilename) @@ -191,18 +194,23 @@ complete_assemble(Storage, Transfer, Size, Timeout) -> t_list_transfers(Config) -> Storage = storage(Config), + {ok, Exports} = emqx_ft_storage_fs:exports_local(Storage), ?assertMatch( - {ok, #{ - {?CLIENTID1, <<"t_assemble_empty_transfer">>} := #{ - status := complete, - result := [#{path := _, size := 0, fragment := {result, _}}] + [ + #{ + transfer := {?CLIENTID2, <<"t_assemble_complete_local_transfer">>}, + path := _, + size := Size, + meta := #{name := "topsecret.pdf"} }, - {?CLIENTID2, <<"t_assemble_complete_local_transfer">>} := #{ - status := complete, - result := [#{path := _, size := Size, fragment := {result, _}}] + #{ + transfer := {?CLIENTID1, <<"t_assemble_empty_transfer">>}, + path := _, + size := 0, + meta := #{name := "important.pdf"} } - }} when Size > 0, - emqx_ft_storage_fs:transfers(Storage) + ] when Size > 0, + lists:sort(Exports) ). %% @@ -232,5 +240,9 @@ mk_fileid() -> storage(Config) -> #{ type => local, - root => ?config(storage_root, Config) + root => ?config(storage_root, Config), + exporter => #{ + type => local, + root => ?config(exports_root, Config) + } }. diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index 314b4a5f2..e43abf095 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -56,7 +56,16 @@ t_update_config(_Config) -> {ok, _}, emqx_conf:update( [file_transfer], - #{<<"storage">> => #{<<"type">> => <<"local">>, <<"root">> => <<"/tmp/path">>}}, + #{ + <<"storage">> => #{ + <<"type">> => <<"local">>, + <<"root">> => <<"/tmp/path">>, + <<"exporter">> => #{ + <<"type">> => <<"local">>, + <<"root">> => <<"/tmp/exports">> + } + } + }, #{} ) ), diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index 5551cce27..5b3f56b7d 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -22,11 +22,8 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). --include_lib("emqx/include/asserts.hrl"). - all() -> [ - {group, single_node}, {group, cluster} ]. @@ -34,7 +31,6 @@ all() -> groups() -> [ - {single_node, [sequence], emqx_common_test_helpers:all(?MODULE) -- ?CLUSTER_CASES}, {cluster, [sequence], ?CLUSTER_CASES} ]. @@ -48,8 +44,9 @@ end_per_suite(_Config) -> set_special_configs(Config) -> fun (emqx_ft) -> - Root = emqx_ft_test_helpers:ft_root(Config, node()), - emqx_ft_test_helpers:load_config(#{storage => #{type => local, root => Root}}); + emqx_ft_test_helpers:load_config(#{ + storage => emqx_ft_test_helpers:local_storage(Config) + }); (_) -> ok end. @@ -74,47 +71,19 @@ end_per_group(_Group, _Config) -> %% Tests %%-------------------------------------------------------------------- -t_invalid_ready_transfer_id(Config) -> - ?assertMatch( - {error, _}, - emqx_ft_storage_fs:get_ready_transfer(storage(Config), #{ - <<"clientid">> => client_id(Config), - <<"fileid">> => <<"fileid">>, - <<"node">> => atom_to_binary('nonexistent@127.0.0.1') - }) - ), - ?assertMatch( - {error, _}, - emqx_ft_storage_fs:get_ready_transfer(storage(Config), #{ - <<"clientid">> => client_id(Config), - <<"fileid">> => <<"fileid">>, - <<"node">> => <<"nonexistent_as_atom@127.0.0.1">> - }) - ), - ?assertMatch( - {error, _}, - emqx_ft_storage_fs:get_ready_transfer(storage(Config), #{ - <<"clientid">> => client_id(Config), - <<"fileid">> => <<"nonexistent_file">>, - <<"node">> => node() - }) - ). - t_multinode_ready_transfers(Config) -> Node1 = ?config(additional_node, Config), - ok = emqx_ft_test_helpers:upload_file(<<"c1">>, <<"f1">>, <<"data">>, Node1), + ok = emqx_ft_test_helpers:upload_file(<<"c/1">>, <<"f:1">>, "fn1", <<"data">>, Node1), Node2 = node(), - ok = emqx_ft_test_helpers:upload_file(<<"c2">>, <<"f2">>, <<"data">>, Node2), + ok = emqx_ft_test_helpers:upload_file(<<"c/2">>, <<"f:2">>, "fn2", <<"data">>, Node2), - ?assertInclude( - #{<<"clientid">> := <<"c1">>, <<"fileid">> := <<"f1">>}, - ready_transfer_ids(Config) - ), - - ?assertInclude( - #{<<"clientid">> := <<"c2">>, <<"fileid">> := <<"f2">>}, - ready_transfer_ids(Config) + ?assertMatch( + [ + #{transfer := {<<"c/1">>, <<"f:1">>}, name := "fn1"}, + #{transfer := {<<"c/2">>, <<"f:2">>}, name := "fn2"} + ], + lists:sort(list_exports(Config)) ). %%-------------------------------------------------------------------- @@ -127,13 +96,13 @@ client_id(Config) -> storage(Config) -> #{ type => local, - root => ft_root(Config) + root => emqx_ft_test_helpers:root(Config, node(), ["transfers"]), + exporter => #{ + type => local, + root => emqx_ft_test_helpers:root(Config, node(), ["exports"]) + } }. -ft_root(Config) -> - emqx_ft_test_helpers:ft_root(Config, node()). - -ready_transfer_ids(Config) -> - {ok, ReadyTransfers} = emqx_ft_storage_fs:ready_transfers(storage(Config)), - {ReadyTransferIds, _} = lists:unzip(ReadyTransfers), - ReadyTransferIds. +list_exports(Config) -> + {ok, Exports} = emqx_ft_storage_fs:exports(storage(Config)), + Exports. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index 89e9eb970..25cd200f1 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -41,7 +41,14 @@ init_per_testcase(TC, Config) -> emqx_ft, fun(emqx_ft) -> emqx_ft_test_helpers:load_config(#{ - storage => #{type => local, root => mk_root(TC, Config)} + storage => #{ + type => local, + root => emqx_ft_test_helpers:root(Config, node(), [TC, transfers]), + exporter => #{ + type => local, + root => emqx_ft_test_helpers:root(Config, node(), [TC, exports]) + } + } }) end ), @@ -53,9 +60,6 @@ end_per_testcase(_TC, _Config) -> ok = application:stop(emqx_ft), ok. -mk_root(TC, Config) -> - filename:join([?config(priv_dir, Config), "file_transfer", TC, atom_to_list(node())]). - %% -define(NSEGS(Filesize, SegmentSize), (ceil(Filesize / SegmentSize) + 1)). diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index b756f8034..e62c74d81 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -30,7 +30,7 @@ start_additional_node(Config, Name) -> {configure_gen_rpc, true}, {env_handler, fun (emqx_ft) -> - load_config(#{storage => #{type => local, root => ft_root(Config, node())}}); + load_config(#{storage => local_storage(Config)}); (_) -> ok end} @@ -43,6 +43,13 @@ stop_additional_node(Node) -> ok = emqx_common_test_helpers:stop_slave(Node), ok. +local_storage(Config) -> + #{ + type => local, + root => root(Config, node(), [transfers]), + exporter => #{type => local, root => root(Config, node(), [exports])} + }. + load_config(Config) -> emqx_common_test_helpers:load_config(emqx_ft_schema, #{file_transfer => Config}). @@ -50,10 +57,8 @@ tcp_port(Node) -> {_, Port} = rpc:call(Node, emqx_config, get, [[listeners, tcp, default, bind]]), Port. -ft_root(Config, Node) -> - filename:join([ - ?config(priv_dir, Config), <<"file_transfer">>, atom_to_binary(Node) - ]). +root(Config, Node, Tail) -> + filename:join([?config(priv_dir, Config), "file_transfer", Node | Tail]). upload_file(ClientId, FileId, Data, Node) -> Port = tcp_port(Node), From 9aec01e7a3283b8b0021ce217e891e40d321809d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 23 Mar 2023 14:17:56 +0300 Subject: [PATCH 072/197] fix(ft-asm): use regular map for meta fragments There's actually no need to use an ordered data structure at all. --- apps/emqx_ft/src/emqx_ft_assembly.erl | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index d0998d6ec..dead1132e 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -37,10 +37,7 @@ status :: status(), coverage :: coverage() | undefined, properties :: properties() | undefined, - meta :: orddict:orddict( - filemeta(), - {node(), filefrag({filemeta, filemeta()})} - ), + meta :: #{filemeta() => {node(), filefrag({filemeta, filemeta()})}}, segs :: emqx_wdgraph:t(emqx_ft:offset(), {node(), filefrag({segment, segmentinfo()})}), size :: emqx_ft:bytes() }). @@ -63,7 +60,7 @@ new(Size) -> #asm{ status = {incomplete, {missing, filemeta}}, - meta = orddict:new(), + meta = #{}, segs = emqx_wdgraph:new(), size = Size }. @@ -111,7 +108,7 @@ coverage(#asm{coverage = Coverage}) -> -spec nodes(t()) -> [node()]. nodes(#asm{meta = Meta, segs = Segs}) -> - S1 = orddict:fold( + S1 = maps:fold( fun(_Meta, {Node, _Fragment}, Acc) -> ordsets:add_element(Node, Acc) end, @@ -131,7 +128,7 @@ properties(#asm{properties = Properties}) -> Properties. status(meta, #asm{meta = Meta}) -> - status(meta, orddict:to_list(Meta)); + status(meta, maps:to_list(Meta)); status(meta, [{Meta, {_Node, _Frag}}]) -> {complete, Meta}; status(meta, []) -> @@ -150,7 +147,7 @@ status(coverage, #asm{segs = Segments, size = Size}) -> append_filemeta(Asm, Node, Fragment = #{fragment := {filemeta, Meta}}) -> Asm#asm{ - meta = orddict:store(Meta, {Node, Fragment}, Asm#asm.meta) + meta = maps:put(Meta, {Node, Fragment}, Asm#asm.meta) }. append_segmentinfo(Asm, _Node, #{fragment := {segment, #{size := 0}}}) -> From 2707b4500fbe705ec6835699c8d54d4b3f9aceef Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 24 Mar 2023 00:16:17 +0300 Subject: [PATCH 073/197] feat(ft-fs): ensure filsystem-safe handling of client input Also restrict the filenames clients may specify in a filemeta to some safe subset, as a future proofing measure. --- apps/emqx_ft/src/emqx_ft_fs_util.erl | 84 +++++++++++++++++++ apps/emqx_ft/src/emqx_ft_schema.erl | 10 ++- .../src/emqx_ft_storage_exporter_fs.erl | 11 ++- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 18 ++-- apps/emqx_ft/test/emqx_ft_SUITE.erl | 41 +++++++++ apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 4 +- apps/emqx_ft/test/emqx_ft_fs_util_tests.erl | 65 ++++++++++++++ apps/emqx_ft/test/emqx_ft_test_helpers.erl | 7 +- 8 files changed, 225 insertions(+), 15 deletions(-) create mode 100644 apps/emqx_ft/test/emqx_ft_fs_util_tests.erl diff --git a/apps/emqx_ft/src/emqx_ft_fs_util.erl b/apps/emqx_ft/src/emqx_ft_fs_util.erl index 198f4ccc5..75fb47c27 100644 --- a/apps/emqx_ft/src/emqx_ft_fs_util.erl +++ b/apps/emqx_ft/src/emqx_ft_fs_util.erl @@ -18,6 +18,10 @@ -include_lib("snabbkaffe/include/trace.hrl"). +-export([is_filename_safe/1]). +-export([escape_filename/1]). +-export([unescape_filename/1]). + -export([read_decode_file/2]). -export([fold/4]). @@ -35,6 +39,86 @@ ) -> Acc ). +-define(IS_UNSAFE(C), + ((C) =:= $% orelse + (C) =:= $: orelse + (C) =:= $\\ orelse + (C) =:= $/) +). + +-define(IS_PRINTABLE(C), + % NOTE: See `io_lib:printable_unicode_list/1` + (((C) >= 32 andalso (C) =< 126) orelse + ((C) >= 16#A0 andalso (C) < 16#D800) orelse + ((C) > 16#DFFF andalso (C) < 16#FFFE) orelse + ((C) > 16#FFFF andalso (C) =< 16#10FFFF)) +). + +%% + +-spec is_filename_safe(file:filename_all()) -> ok | {error, atom()}. +is_filename_safe(FN) when is_binary(FN) -> + is_filename_safe(unicode:characters_to_list(FN)); +is_filename_safe("") -> + {error, empty}; +is_filename_safe(FN) when FN == "." orelse FN == ".." -> + {error, special}; +is_filename_safe(FN) -> + verify_filename_safe(FN). + +verify_filename_safe([$% | Rest]) -> + verify_filename_safe(Rest); +verify_filename_safe([C | _]) when ?IS_UNSAFE(C) -> + {error, unsafe}; +verify_filename_safe([C | _]) when not ?IS_PRINTABLE(C) -> + {error, nonprintable}; +verify_filename_safe([_ | Rest]) -> + verify_filename_safe(Rest); +verify_filename_safe([]) -> + ok. + +-spec escape_filename(binary()) -> file:name(). +escape_filename(Name) when Name == <<".">> orelse Name == <<"..">> -> + lists:reverse(percent_encode(Name, "")); +escape_filename(Name) -> + escape(Name, ""). + +escape(<>, Acc) when ?IS_UNSAFE(C) -> + escape(Rest, percent_encode(<>, Acc)); +escape(<>, Acc) when not ?IS_PRINTABLE(C) -> + escape(Rest, percent_encode(<>, Acc)); +escape(<>, Acc) -> + escape(Rest, [C | Acc]); +escape(<<>>, Acc) -> + lists:reverse(Acc). + +-spec unescape_filename(file:name()) -> binary(). +unescape_filename(Name) -> + unescape(Name, <<>>). + +unescape([$%, A, B | Rest], Acc) -> + unescape(Rest, percent_decode(A, B, Acc)); +unescape([C | Rest], Acc) -> + unescape(Rest, <>); +unescape([], Acc) -> + Acc. + +percent_encode(<>, Acc) -> + percent_encode(Rest, [dec2hex(B), dec2hex(A), $% | Acc]); +percent_encode(<<>>, Acc) -> + Acc. + +percent_decode(A, B, Acc) -> + <>. + +dec2hex(X) when (X >= 0) andalso (X =< 9) -> X + $0; +dec2hex(X) when (X >= 10) andalso (X =< 15) -> X + $A - 10. + +hex2dec(X) when (X >= $0) andalso (X =< $9) -> X - $0; +hex2dec(X) when (X >= $A) andalso (X =< $F) -> X - $A + 10; +hex2dec(X) when (X >= $a) andalso (X =< $f) -> X - $a + 10; +hex2dec(_) -> error(badarg). + %% -spec read_decode_file(file:name(), fun((binary()) -> Value)) -> diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 37e2adafc..5c0a764a2 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -130,8 +130,11 @@ desc(local_storage_gc) -> schema(filemeta) -> #{ roots => [ - % TODO nonempty - {name, hoconsc:mk(string(), #{required => true})}, + {name, + hoconsc:mk(string(), #{ + required => true, + validator => validator(filename) + })}, {size, hoconsc:mk(non_neg_integer())}, {expire_at, hoconsc:mk(non_neg_integer())}, {checksum, hoconsc:mk({atom(), binary()}, #{converter => converter(checksum)})}, @@ -140,6 +143,9 @@ schema(filemeta) -> ] }. +validator(filename) -> + fun emqx_ft_fs_util:is_filename_safe/1. + converter(checksum) -> fun (undefined, #{}) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index e0bffd444..6456e2ba5 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -343,7 +343,16 @@ mk_result_reldir(Transfer = {ClientId, FileId}) -> Bucket2:?BUCKET2_LEN/binary, BucketRest/binary >> = binary:encode_hex(Hash), - [Bucket1, Bucket2, BucketRest, ClientId, FileId]. + [ + Bucket1, + Bucket2, + BucketRest, + emqx_ft_fs_util:escape_filename(ClientId), + emqx_ft_fs_util:escape_filename(FileId) + ]. + +dirnames_to_transfer(ClientId, FileId) -> + {emqx_ft_fs_util:unescape_filename(ClientId), emqx_ft_fs_util:unescape_filename(FileId)}. mk_transfer_hash(Transfer) -> crypto:hash(?BUCKET_HASH, term_to_binary(Transfer)). diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 4a613dbcf..6c5b6962c 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -250,12 +250,12 @@ transfers(Storage) -> )}. transfers(Storage, ClientId, AccIn) -> - Dirname = mk_client_filedir(Storage, ClientId), + Dirname = filename:join(get_storage_root(Storage), ClientId), case file:list_dir(Dirname) of {ok, FileIds} -> lists:foldl( fun(FileId, Acc) -> - Transfer = {filename_to_binary(ClientId), filename_to_binary(FileId)}, + Transfer = dirnames_to_transfer(ClientId, FileId), read_transferinfo(Storage, Transfer, Acc) end, AccIn, @@ -327,10 +327,15 @@ break_segment_filename(Filename) -> end. mk_filedir(Storage, {ClientId, FileId}, SubDirs) -> - filename:join([get_storage_root(Storage), ClientId, FileId | SubDirs]). + filename:join([ + get_storage_root(Storage), + emqx_ft_fs_util:escape_filename(ClientId), + emqx_ft_fs_util:escape_filename(FileId) + | SubDirs + ]). -mk_client_filedir(Storage, ClientId) -> - filename:join([get_storage_root(Storage), ClientId]). +dirnames_to_transfer(ClientId, FileId) -> + {emqx_ft_fs_util:unescape_filename(ClientId), emqx_ft_fs_util:unescape_filename(FileId)}. mk_filepath(Storage, Transfer, SubDirs, Filename) -> filename:join(mk_filedir(Storage, Transfer, SubDirs), Filename). @@ -432,6 +437,3 @@ read_frag_filemeta(_Filename, Filepath) -> read_frag_segmentinfo(Filename, _Filepath) -> break_segment_filename(Filename). - -filename_to_binary(S) when is_list(S) -> unicode:characters_to_binary(S); -filename_to_binary(B) when is_binary(B) -> B. diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 5fb384c16..1a886c00d 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -165,6 +165,29 @@ t_invalid_fileid(Config) -> emqtt:publish(C, <<"$file//init">>, <<>>, 1) ). +t_invalid_filename(Config) -> + C = ?config(client, Config), + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_init_topic(<<"f1">>), emqx_json:encode(meta(".", <<>>)), 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_init_topic(<<"f2">>), emqx_json:encode(meta("..", <<>>)), 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_init_topic(<<"f2">>), emqx_json:encode(meta("../nice", <<>>)), 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_init_topic(<<"f3">>), emqx_json:encode(meta("/etc/passwd", <<>>)), 1) + ), + ?assertRCName( + success, + emqtt:publish(C, mk_init_topic(<<"f4">>), emqx_json:encode(meta("146%", <<>>)), 1) + ). + t_simple_transfer(Config) -> C = ?config(client, Config), @@ -202,6 +225,24 @@ t_simple_transfer(Config) -> read_export(Export) ). +t_nasty_clientids_fileids(_Config) -> + Transfers = [ + {<<".">>, <<".">>}, + {<<"🌚"/utf8>>, <<"🌝"/utf8>>}, + {<<"../..">>, <<"😤"/utf8>>}, + {<<"/etc/passwd">>, <<"whitehat">>}, + {<<"; rm -rf / ;">>, <<"whitehat">>} + ], + + ok = lists:foreach( + fun({ClientId, FileId}) -> + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "justfile", ClientId), + [Export] = list_exports(ClientId), + ?assertEqual({ok, ClientId}, read_export(Export)) + end, + Transfers + ). + t_meta_conflict(Config) -> C = ?config(client, Config), diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index aff4d864c..56622f3cd 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -60,7 +60,7 @@ end_per_testcase(_Case, _Config) -> t_list_ready_transfers(Config) -> ClientId = client_id(Config), - ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, <<"data">>, node()), + ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, "f1", <<"data">>), {ok, 200, #{<<"files">> := Files}} = request(get, uri(["file_transfer", "files"]), fun json/1), @@ -73,7 +73,7 @@ t_list_ready_transfers(Config) -> t_download_transfer(Config) -> ClientId = client_id(Config), - ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, <<"data">>, node()), + ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, "f1", <<"data">>), ?assertMatch( {ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}}, diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_tests.erl b/apps/emqx_ft/test/emqx_ft_fs_util_tests.erl new file mode 100644 index 000000000..1939e74c6 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_tests.erl @@ -0,0 +1,65 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_fs_util_tests). + +-include_lib("eunit/include/eunit.hrl"). + +filename_safe_test_() -> + [ + ?_assertEqual(ok, emqx_ft_fs_util:is_filename_safe("im.safe")), + ?_assertEqual(ok, emqx_ft_fs_util:is_filename_safe(<<"im.safe">>)), + ?_assertEqual(ok, emqx_ft_fs_util:is_filename_safe(<<".safe.100%">>)), + ?_assertEqual(ok, emqx_ft_fs_util:is_filename_safe(<<"safe.as.🦺"/utf8>>)) + ]. + +filename_unsafe_test_() -> + [ + ?_assertEqual({error, empty}, emqx_ft_fs_util:is_filename_safe("")), + ?_assertEqual({error, special}, emqx_ft_fs_util:is_filename_safe(".")), + ?_assertEqual({error, special}, emqx_ft_fs_util:is_filename_safe("..")), + ?_assertEqual({error, special}, emqx_ft_fs_util:is_filename_safe(<<"..">>)), + ?_assertEqual({error, unsafe}, emqx_ft_fs_util:is_filename_safe(<<".././..">>)), + ?_assertEqual({error, unsafe}, emqx_ft_fs_util:is_filename_safe("/etc/passwd")), + ?_assertEqual({error, unsafe}, emqx_ft_fs_util:is_filename_safe("../cookie")), + ?_assertEqual({error, unsafe}, emqx_ft_fs_util:is_filename_safe("C:$cookie")), + ?_assertEqual({error, nonprintable}, emqx_ft_fs_util:is_filename_safe([1, 2, 3])), + ?_assertEqual({error, nonprintable}, emqx_ft_fs_util:is_filename_safe(<<4, 5, 6>>)), + ?_assertEqual({error, nonprintable}, emqx_ft_fs_util:is_filename_safe([$a, 16#7F, $z])) + ]. + +-define(NAMES, [ + {"just.file", <<"just.file">>}, + {".hidden", <<".hidden">>}, + {".~what", <<".~what">>}, + {"100%25.file", <<"100%.file">>}, + {"%2E%2E", <<"..">>}, + {"...", <<"...">>}, + {"%2Fetc%2Fpasswd", <<"/etc/passwd">>}, + {"%01%02%0A ", <<1, 2, 10, 32>>} +]). + +escape_filename_test_() -> + [ + ?_assertEqual(Filename, emqx_ft_fs_util:escape_filename(Input)) + || {Filename, Input} <- ?NAMES + ]. + +unescape_filename_test_() -> + [ + ?_assertEqual(Input, emqx_ft_fs_util:unescape_filename(Filename)) + || {Filename, Input} <- ?NAMES + ]. diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index e62c74d81..a38242629 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -60,14 +60,17 @@ tcp_port(Node) -> root(Config, Node, Tail) -> filename:join([?config(priv_dir, Config), "file_transfer", Node | Tail]). -upload_file(ClientId, FileId, Data, Node) -> +upload_file(ClientId, FileId, Name, Data) -> + upload_file(ClientId, FileId, Name, Data, node()). + +upload_file(ClientId, FileId, Name, Data, Node) -> Port = tcp_port(Node), Size = byte_size(Data), {ok, C1} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, Port}]), {ok, _} = emqtt:connect(C1), Meta = #{ - name => FileId, + name => unicode:characters_to_binary(Name), expire_at => erlang:system_time(_Unit = second) + 3600, size => Size }, From 64f15f1fdbeef96573c3e6370d26e2c7f6a7fc17 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 27 Mar 2023 20:42:19 +0300 Subject: [PATCH 074/197] fix(ft-gc): ensure directories of complete transfers are GCed --- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 62 ++++++++++++------- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 19 +++++- 2 files changed, 57 insertions(+), 24 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 20e4f468d..eb274a4d5 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -169,7 +169,7 @@ try_collect_transfer(Storage, Transfer, TransferInfo = #{}, Stats) -> % TODO: collect empty directories separately case FragCleaned and TempCleaned of true -> - collect_transfer_directory(Storage, Transfer, Stats2); + collect_transfer_directory(Storage, Transfer, Cutoff, Stats2); false -> Stats2 end; @@ -191,18 +191,32 @@ collect_tempfiles(Storage, Transfer, Stats) -> collect_outdated_fragments(Storage, Transfer, Cutoff, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, fragment), - Filter = fun(_Filepath, #file_info{mtime = ModifiedAt}) -> ModifiedAt < Cutoff end, - maybe_collect_directory(Dirname, Filter, Stats). + maybe_collect_directory(Dirname, filter_older_than(Cutoff), Stats). collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, temporary), - Filter = fun(_Filepath, #file_info{mtime = ModifiedAt}) -> ModifiedAt < Cutoff end, - maybe_collect_directory(Dirname, Filter, Stats). + maybe_collect_directory(Dirname, filter_older_than(Cutoff), Stats). -collect_transfer_directory(Storage, Transfer, Stats) -> +collect_transfer_directory(Storage, Transfer, Cutoff, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer), - StatsNext = collect_empty_directory(Dirname, Stats), - collect_parents(Dirname, get_storage_root(Storage), StatsNext). + Filter = + case Stats of + #gcstats{directories = 0} -> + % Nothing were collected, this is a leftover from a past complete transfer GC. + filter_older_than(Cutoff); + #gcstats{} -> + % Usual incomplete transfer GC, collect directories unconditionally. + true + end, + case collect_empty_directory(Dirname, Filter, Stats) of + {true, StatsNext} -> + collect_parents(Dirname, get_storage_root(Storage), StatsNext); + {false, StatsNext} -> + StatsNext + end. + +filter_older_than(Cutoff) -> + fun(_Filepath, #file_info{mtime = ModifiedAt}) -> ModifiedAt =< Cutoff end. collect_parents(Dirname, Until, Stats) -> Parent = filename:dirname(Dirname), @@ -218,14 +232,6 @@ collect_parents(Dirname, Until, Stats) -> register_gcstat_error({directory, Parent}, Reason, Stats) end. -% collect_outdated_fragment(#{path := Filepath, fileinfo := Fileinfo}, Cutoff, Stats) -> -% case Fileinfo#file_info.mtime of -% ModifiedAt when ModifiedAt < Cutoff -> -% collect_filepath(Filepath, Fileinfo, Stats); -% _ -> -% Stats -% end. - maybe_collect_directory(Dirpath, Filter, Stats) -> case filelib:is_dir(Dirpath) of true -> @@ -263,10 +269,10 @@ collect_directory(Dirpath, Fileinfo, Filter, Stats) -> case file:list_dir(Dirpath) of {ok, Filenames} -> {Clean, StatsNext} = collect_files(Dirpath, Filenames, Filter, Stats), - case Clean andalso filter_filepath(Filter, Dirpath, Fileinfo) of + case Clean of true -> - {true, collect_empty_directory(Dirpath, StatsNext)}; - _ -> + collect_empty_directory(Dirpath, Fileinfo, Filter, StatsNext); + false -> {false, StatsNext} end; {error, Reason} -> @@ -284,13 +290,23 @@ collect_files(Dirname, Filenames, Filter, Stats) -> Filenames ). -collect_empty_directory(Dirpath, Stats) -> - case file:del_dir(Dirpath) of +collect_empty_directory(Dirpath, Filter, Stats) -> + case file:read_link_info(Dirpath, [{time, posix}, raw]) of + {ok, Dirinfo} -> + collect_empty_directory(Dirpath, Dirinfo, Filter, Stats); + {error, Reason} -> + {Reason == enoent, register_gcstat_error({directory, Dirpath}, Reason, Stats)} + end. + +collect_empty_directory(Dirpath, Dirinfo, Filter, Stats) -> + case filter_filepath(Filter, Dirpath, Dirinfo) andalso file:del_dir(Dirpath) of + false -> + {false, Stats}; ok -> ?tp(garbage_collected_directory, #{path => Dirpath}), - account_gcstat_directory(Stats); + {true, account_gcstat_directory(Stats)}; {error, Reason} -> - register_gcstat_error({directory, Dirpath}, Reason, Stats) + {false, register_gcstat_error({directory, Dirpath}, Reason, Stats)} end. filter_filepath(Filter, _, _) when is_boolean(Filter) -> diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index 25cd200f1..53a22b8b6 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -104,6 +104,10 @@ t_gc_triggers_manually(_Config) -> t_gc_complete_transfers(_Config) -> Storage = emqx_ft_conf:storage(), + ok = emqx_config:put([file_transfer, storage, gc, minimum_segments_ttl], 0), + ok = emqx_config:put([file_transfer, storage, gc, maximum_segments_ttl], 3), + ok = emqx_config:put([file_transfer, storage, gc, interval], 500), + ok = emqx_ft_storage_fs_gc:reset(Storage), Transfers = [ { T1 = {<<"client1">>, mk_file_id()}, @@ -174,7 +178,20 @@ t_gc_complete_transfers(_Config) -> ?assertEqual(?NSEGS(S2, SS2) + ?NSEGS(S3, SS3), CFiles), ?assertEqual(2 + 2, CDirectories), ?assertMatch(Space when Space > S2 + S3, CSpace), - ?assertMatch(Errors when map_size(Errors) == 0, CErrors). + ?assertMatch(Errors when map_size(Errors) == 0, CErrors), + % 4. Ensure that empty transfer directories will be eventually collected + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = 0, + directories = 6, + space = 0 + } + }, + 5000, + 0 + ). t_gc_incomplete_transfers(_Config) -> ok = emqx_config:put([file_transfer, storage, gc, minimum_segments_ttl], 0), From 54e54cc63dc2f614127e37aa6fd9e6059fd59f0a Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 27 Mar 2023 23:44:23 +0300 Subject: [PATCH 075/197] feat(api): provide utility to qualify API URIs --- apps/emqx_dashboard/src/emqx_dashboard.erl | 8 +++----- apps/emqx_dashboard/src/emqx_dashboard_swagger.erl | 12 ++++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 6f0c8334a..53b661ea8 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -41,8 +41,6 @@ -include_lib("emqx/include/http_api.hrl"). -include_lib("emqx/include/emqx_release.hrl"). --define(BASE_PATH, "/api/v5"). - -define(EMQX_MIDDLE, emqx_dashboard_middleware). %%-------------------------------------------------------------------- @@ -61,7 +59,7 @@ start_listeners(Listeners) -> GlobalSpec = #{ openapi => "3.0.0", info => #{title => "EMQX API", version => ?EMQX_API_VERSION}, - servers => [#{url => ?BASE_PATH}], + servers => [#{url => emqx_dashboard_swagger:base_path()}], components => #{ schemas => #{}, 'securitySchemes' => #{ @@ -78,11 +76,11 @@ start_listeners(Listeners) -> {"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, {emqx_mgmt_api_status:path(), emqx_mgmt_api_status, []}, - {?BASE_PATH ++ "/[...]", emqx_dashboard_bad_api, []}, + {emqx_dashboard_swagger:relative_uri("/[...]"), emqx_dashboard_bad_api, []}, {'_', cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}} ], BaseMinirest = #{ - base_path => ?BASE_PATH, + base_path => emqx_dashboard_swagger:base_path(), modules => minirest_api:find_api_modules(apps()), authorization => Authorization, security => [#{'basicAuth' => []}, #{'bearerAuth' => []}], diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index e2872c0d7..95f28e387 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -19,12 +19,16 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). +-define(BASE_PATH, "/api/v5"). + %% API -export([spec/1, spec/2]). -export([namespace/0, namespace/1, fields/1]). -export([schema_with_example/2, schema_with_examples/2]). -export([error_codes/1, error_codes/2]). -export([file_schema/1]). +-export([base_path/0]). +-export([relative_uri/1]). -export([filter_check_request/2, filter_check_request_and_translate_body/2]). @@ -177,6 +181,14 @@ error_codes(Codes = [_ | _], MsgDesc) -> })} ]. +-spec base_path() -> uri_string:uri_string(). +base_path() -> + ?BASE_PATH. + +-spec relative_uri(uri_string:uri_string()) -> uri_string:uri_string(). +relative_uri(Uri) -> + base_path() ++ Uri. + file_schema(FileName) -> #{ content => #{ From b6b044f42947fb0702006dce70feb4afbc7c1490 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 27 Mar 2023 22:58:25 +0300 Subject: [PATCH 076/197] feat(ft): move out exporter concept into dedicated modules --- apps/emqx/priv/bpapi.versions | 1 + .../test/emqx_dashboard_api_test_helpers.erl | 8 +- apps/emqx_ft/i18n/emqx_ft_api_i18n.conf | 4 + apps/emqx_ft/src/emqx_ft_api.erl | 133 +------------ apps/emqx_ft/src/emqx_ft_assembler.erl | 43 ++--- apps/emqx_ft/src/emqx_ft_storage.erl | 14 +- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 97 ++++++++++ .../src/emqx_ft_storage_exporter_fs.erl | 14 +- .../src/emqx_ft_storage_exporter_fs_api.erl | 180 ++++++++++++++++++ .../src/emqx_ft_storage_exporter_fs_proxy.erl | 46 +++++ apps/emqx_ft/src/emqx_ft_storage_fs.erl | 23 +-- apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl | 20 +- .../emqx_ft_storage_exporter_fs_proto_v1.erl | 51 +++++ .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 18 -- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 4 +- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 22 ++- 16 files changed, 437 insertions(+), 241 deletions(-) create mode 100644 apps/emqx_ft/src/emqx_ft_storage_exporter.erl create mode 100644 apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl create mode 100644 apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl create mode 100644 apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 0e0957adf..d597646d6 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -13,6 +13,7 @@ {emqx_dashboard,1}. {emqx_delayed,1}. {emqx_exhook,1}. +{emqx_ft_storage_exporter_fs,1}. {emqx_ft_storage_fs,1}. {emqx_ft_storage_fs_reader,1}. {emqx_gateway_api_listeners,1}. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl index 8df130897..f70f464b3 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl @@ -25,11 +25,12 @@ request/4, multipart_formdata_request/3, multipart_formdata_request/4, + host/0, uri/0, uri/1 ]). --define(HOST, "http://127.0.0.1:18083/"). +-define(HOST, "http://127.0.0.1:18083"). -define(API_VERSION, "v5"). -define(BASE_PATH, "api"). @@ -94,10 +95,13 @@ request(Username, Method, Url, Body) -> {error, Reason} end. +host() -> + ?HOST. + uri() -> uri([]). uri(Parts) when is_list(Parts) -> NParts = [E || E <- Parts], - ?HOST ++ to_list(filename:join([?BASE_PATH, ?API_VERSION | NParts])). + host() ++ "/" ++ to_list(filename:join([?BASE_PATH, ?API_VERSION | NParts])). auth_header(Username) -> Password = <<"public">>, diff --git a/apps/emqx_ft/i18n/emqx_ft_api_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_api_i18n.conf index 0bda935f8..6ac01ea23 100644 --- a/apps/emqx_ft/i18n/emqx_ft_api_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_api_i18n.conf @@ -11,6 +11,10 @@ emqx_ft_api { } } +} + +emqx_ft_storage_exporter_fs_api { + file_get { desc { en: "Get a file by its id." diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index 143d629de..eeb047b16 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -30,18 +30,12 @@ ]). -export([ - fields/1, roots/0 ]). %% API callbacks -export([ - '/file_transfer/files'/2, - '/file_transfer/file'/2 -]). - --export([ - mk_file_uri/3 + '/file_transfer/files'/2 ]). -import(hoconsc, [mk/2, ref/1, ref/2]). @@ -53,8 +47,7 @@ api_spec() -> paths() -> [ - "/file_transfer/files", - "/file_transfer/file" + "/file_transfer/files" ]. schema("/file_transfer/files") -> @@ -71,31 +64,6 @@ schema("/file_transfer/files") -> ) } } - }; -schema("/file_transfer/file") -> - % TODO - % This is conceptually another API, because this logic is inherent only to the - % local filesystem exporter. Ideally, we won't even publish it if `emqx_ft` is - % configured with another exporter. Accordingly, things that look too specific - % for this module (i.e. `parse_filepath/1`) should go away in another API module. - #{ - 'operationId' => '/file_transfer/file', - get => #{ - tags => [<<"file_transfer">>], - summary => <<"Download a particular file">>, - description => ?DESC("file_get"), - parameters => [ - ref(file_node), - ref(file_ref) - ], - responses => #{ - 200 => <<"Operation success">>, - 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Not found">>), - 503 => emqx_dashboard_swagger:error_codes( - ['SERVICE_UNAVAILABLE'], <<"Service unavailable">> - ) - } - } }. '/file_transfer/files'(get, #{}) -> @@ -106,76 +74,11 @@ schema("/file_transfer/file") -> {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} end. -'/file_transfer/file'(get, #{query_string := Query}) -> - try - Node = parse_node(maps:get(<<"node">>, Query)), - Filepath = parse_filepath(maps:get(<<"fileref">>, Query)), - case emqx_ft_storage_fs_proto_v1:read_export_file(Node, Filepath, self()) of - {ok, ReaderPid} -> - FileData = emqx_ft_storage_fs_reader:table(ReaderPid), - {200, - #{ - <<"content-type">> => <<"application/data">>, - <<"content-disposition">> => <<"attachment">> - }, - FileData}; - {error, enoent} -> - {404, error_msg('NOT_FOUND', <<"Not found">>)}; - {error, Error} -> - ?SLOG(warning, #{msg => "get_ready_transfer_fail", error => Error}), - {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} - end - catch - throw:{invalid, Param} -> - {404, - error_msg( - 'NOT_FOUND', - iolist_to_binary(["Invalid query parameter: ", Param]) - )}; - error:{erpc, noconnection} -> - {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} - end. - error_msg(Code, Msg) -> #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. --spec fields(hocon_schema:name()) -> hocon_schema:fields(). -fields(file_ref) -> - [ - {fileref, - hoconsc:mk(binary(), #{ - in => query, - desc => <<"File reference">>, - example => <<"file1">>, - required => true - })} - ]; -fields(file_node) -> - [ - {node, - hoconsc:mk(binary(), #{ - in => query, - desc => <<"Node under which the file is located">>, - example => atom_to_list(node()), - required => true - })} - ]. - roots() -> - [ - file_node, - file_ref - ]. - -mk_file_uri(_Options, Node, Filepath) -> - % TODO: qualify with `?BASE_PATH` - [ - "/file_transfer/file?", - uri_string:compose_query([ - {"node", atom_to_list(Node)}, - {"fileref", Filepath} - ]) - ]. + []. %%-------------------------------------------------------------------- %% Helpers @@ -207,33 +110,3 @@ format_export_info( format_timestamp(Timestamp) -> iolist_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])). - -parse_node(NodeBin) -> - case emqx_misc:safe_to_existing_atom(NodeBin) of - {ok, Node} -> - Node; - {error, _} -> - throw({invalid, NodeBin}) - end. - -parse_filepath(PathBin) -> - case filename:pathtype(PathBin) of - relative -> - ok; - absolute -> - throw({invalid, PathBin}) - end, - PathComponents = filename:split(PathBin), - case lists:any(fun is_special_component/1, PathComponents) of - false -> - filename:join(PathComponents); - true -> - throw({invalid, PathBin}) - end. - -is_special_component(<<".", _/binary>>) -> - true; -is_special_component([$. | _]) -> - true; -is_special_component(_) -> - false. diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 5489a232a..a86116b02 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -106,11 +106,15 @@ handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> {error, _} = Error -> {stop, {shutdown, Error}} end; -handle_event(internal, _, start_assembling, St = #st{assembly = Asm}) -> - Filemeta = emqx_ft_assembly:filemeta(Asm), - Coverage = emqx_ft_assembly:coverage(Asm), +handle_event(internal, _, start_assembling, St = #st{}) -> + Filemeta = emqx_ft_assembly:filemeta(St#st.assembly), + Coverage = emqx_ft_assembly:coverage(St#st.assembly), % TODO: better error handling - {ok, Export} = export_start(Filemeta, St), + {ok, Export} = emqx_ft_storage_exporter:start_export( + St#st.storage, + St#st.transfer, + Filemeta + ), {next_state, {assemble, Coverage}, St#st{export = Export}, ?internal([])}; handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % TODO @@ -119,17 +123,17 @@ handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % TODO: pipelining % TODO: better error handling {ok, Content} = pread(Node, Segment, St), - {ok, NExport} = export_write(St#st.export, Content), + {ok, NExport} = emqx_ft_storage_exporter:write(St#st.export, Content), {next_state, {assemble, Rest}, St#st{export = NExport}, ?internal([])}; handle_event(internal, _, {assemble, []}, St = #st{}) -> {next_state, complete, St, ?internal([])}; handle_event(internal, _, complete, St = #st{}) -> - Result = export_complete(St#st.export), + Result = emqx_ft_storage_exporter:complete(St#st.export), ok = maybe_garbage_collect(Result, St), {stop, {shutdown, Result}, St#st{export = undefined}}. terminate(_Reason, _StateName, #st{export = Export}) -> - Export /= undefined andalso export_discard(Export). + Export /= undefined andalso emqx_ft_storage_exporter:discard(Export). pread(Node, Segment, St) when Node =:= node() -> emqx_ft_storage_fs:pread(St#st.storage, St#st.transfer, Segment, 0, segsize(Segment)); @@ -138,31 +142,6 @@ pread(Node, Segment, St) -> %% -export_start(Filemeta, #st{storage = Storage, transfer = Transfer}) -> - {ExporterMod, Exporter} = emqx_ft_storage_fs:exporter(Storage), - case ExporterMod:start_export(Exporter, Transfer, Filemeta) of - {ok, Export} -> - {ok, {ExporterMod, Export}}; - {error, _} = Error -> - Error - end. - -export_write({ExporterMod, Export}, Content) -> - case ExporterMod:write(Export, Content) of - {ok, ExportNext} -> - {ok, {ExporterMod, ExportNext}}; - {error, _} = Error -> - Error - end. - -export_complete({ExporterMod, Export}) -> - ExporterMod:complete(Export). - -export_discard({ExporterMod, Export}) -> - ExporterMod:discard(Export). - -%% - maybe_garbage_collect(ok, #st{storage = Storage, transfer = Transfer, assembly = Asm}) -> Nodes = emqx_ft_assembly:nodes(Asm), emqx_ft_storage_fs_gc:collect(Storage, Transfer, Nodes); diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 1d1c08ce9..05a053fbf 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -26,6 +26,7 @@ exports/0, + with_storage_type/2, with_storage_type/3 ] ). @@ -33,7 +34,9 @@ -type storage() :: emqx_config:config(). -export_type([assemble_callback/0]). +-export_type([export_info/0]). -export_type([export_data/0]). +-export_type([reader/0]). -type assemble_callback() :: fun((ok | {error, term()}) -> any()). @@ -46,6 +49,7 @@ }. -type export_data() :: binary() | qlc:query_handle(). +-type reader() :: pid(). %%-------------------------------------------------------------------- %% Behaviour @@ -106,11 +110,17 @@ exports() -> Mod = mod(), Mod:exports(storage()). --spec with_storage_type(atom(), atom(), list(term())) -> any(). +-spec with_storage_type(atom(), atom() | function()) -> any(). +with_storage_type(Type, Fun) -> + with_storage_type(Type, Fun, []). + +-spec with_storage_type(atom(), atom() | function(), list(term())) -> any(). with_storage_type(Type, Fun, Args) -> Storage = storage(), case Storage of - #{type := Type} -> + #{type := Type} when is_function(Fun) -> + apply(Fun, [Storage | Args]); + #{type := Type} when is_atom(Fun) -> Mod = mod(Storage), apply(Mod, Fun, [Storage | Args]); disabled -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl new file mode 100644 index 000000000..97e260e3a --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -0,0 +1,97 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Filesystem storage exporter +%% +%% This is conceptually a part of the Filesystem storage backend that defines +%% how and where complete tranfers are assembled into files and stored. + +-module(emqx_ft_storage_exporter). + +%% Export API +-export([start_export/3]). +-export([write/2]). +-export([complete/1]). +-export([discard/1]). + +%% Listing API +-export([list/1]). +% TODO +% -export([list/2]). + +-export([exporter/1]). + +-type storage() :: emxt_ft_storage_fs:storage(). +-type transfer() :: emqx_ft:transfer(). +-type filemeta() :: emqx_ft:filemeta(). + +-type options() :: map(). +-type export() :: term(). + +-callback start_export(options(), transfer(), filemeta()) -> + {ok, export()} | {error, _Reason}. + +-callback write(ExportSt :: export(), iodata()) -> + {ok, ExportSt :: export()} | {error, _Reason}. + +-callback complete(ExportSt :: export()) -> + ok | {error, _Reason}. + +-callback discard(ExportSt :: export()) -> + ok | {error, _Reason}. + +-callback list(options()) -> + {ok, [emqx_ft_storage:export_info()]} | {error, _Reason}. + +%% + +start_export(Storage, Transfer, Filemeta) -> + {ExporterMod, Exporter} = exporter(Storage), + case ExporterMod:start_export(Exporter, Transfer, Filemeta) of + {ok, ExportSt} -> + {ok, {ExporterMod, ExportSt}}; + {error, _} = Error -> + Error + end. + +write({ExporterMod, ExportSt}, Content) -> + case ExporterMod:write(ExportSt, Content) of + {ok, ExportStNext} -> + {ok, {ExporterMod, ExportStNext}}; + {error, _} = Error -> + Error + end. + +complete({ExporterMod, ExportSt}) -> + ExporterMod:complete(ExportSt). + +discard({ExporterMod, ExportSt}) -> + ExporterMod:discard(ExportSt). + +%% + +list(Storage) -> + {ExporterMod, ExporterOpts} = exporter(Storage), + ExporterMod:list(ExporterOpts). + +%% + +-spec exporter(storage()) -> {module(), _ExporterOptions}. +exporter(Storage) -> + case maps:get(exporter, Storage) of + #{type := local} = Options -> + {emqx_ft_storage_exporter_fs, Options} + end. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index 6456e2ba5..7cf0ef6d5 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -30,6 +30,7 @@ -export([start_reader/3]). -export([list/1]). +% TODO % -export([list/2]). -export_type([export/0]). @@ -228,7 +229,7 @@ mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo) -> #{ transfer => Transfer, name => Filename, - uri => mk_export_uri(Options, RelFilepath), + uri => mk_export_uri(RelFilepath), timestamp => Fileinfo#file_info.mtime, size => Fileinfo#file_info.size, path => filename:join(Root, RelFilepath) @@ -247,15 +248,14 @@ try_read_filemeta(Filepath, Info) -> Info end. -mk_export_uri(Options, RelFilepath) -> - % emqx_ft_storage_exporter_fs_api:mk_export_uri(Options, RelFilepath). - emqx_ft_api:mk_file_uri(Options, node(), RelFilepath). +mk_export_uri(RelFilepath) -> + emqx_ft_storage_exporter_fs_api:mk_export_uri(node(), RelFilepath). -spec start_reader(options(), file:name(), _Caller :: pid()) -> {ok, reader()} | {error, enoent}. -start_reader(Options, Filepath, CallerPid) -> +start_reader(Options, RelFilepath, CallerPid) -> Root = get_storage_root(Options), - case filelib:safe_relative_path(Filepath, Root) of + case filelib:safe_relative_path(RelFilepath, Root) of SafeFilepath when SafeFilepath /= unsafe -> AbsFilepath = filename:join(Root, SafeFilepath), emqx_ft_storage_fs_reader:start_supervised(CallerPid, AbsFilepath); @@ -269,7 +269,7 @@ start_reader(Options, Filepath, CallerPid) -> {ok, [exportinfo(), ...]} | {error, file_error()}. list(_Options) -> Nodes = mria_mnesia:running_nodes(), - Results = emqx_ft_storage_fs_proto_v1:list_exports(Nodes), + Results = emqx_ft_storage_exporter_fs_proto_v1:list_exports(Nodes), {GoodResults, BadResults} = lists:partition( fun ({_Node, {ok, {ok, _}}}) -> true; diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl new file mode 100644 index 000000000..b7ad86436 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl @@ -0,0 +1,180 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_exporter_fs_api). + +-behaviour(minirest_api). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% Swagger specs from hocon schema +-export([ + api_spec/0, + paths/0, + schema/1, + namespace/0 +]). + +-export([ + fields/1, + roots/0 +]). + +%% API callbacks +-export([ + '/file_transfer/file'/2 +]). + +-export([mk_export_uri/2]). + +%% + +namespace() -> "file_transfer". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + [ + "/file_transfer/file" + ]. + +schema("/file_transfer/file") -> + #{ + 'operationId' => '/file_transfer/file', + get => #{ + tags => [<<"file_transfer">>], + summary => <<"Download a particular file">>, + description => ?DESC("file_get"), + parameters => [ + hoconsc:ref(file_node), + hoconsc:ref(file_ref) + ], + responses => #{ + 200 => <<"Operation success">>, + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Not found">>), + 503 => emqx_dashboard_swagger:error_codes( + ['SERVICE_UNAVAILABLE'], <<"Service unavailable">> + ) + } + } + }. + +roots() -> + [ + file_node, + file_ref + ]. + +-spec fields(hocon_schema:name()) -> hocon_schema:fields(). +fields(file_ref) -> + [ + {fileref, + hoconsc:mk(binary(), #{ + in => query, + desc => <<"File reference">>, + example => <<"file1">>, + required => true + })} + ]; +fields(file_node) -> + [ + {node, + hoconsc:mk(binary(), #{ + in => query, + desc => <<"Node under which the file is located">>, + example => atom_to_list(node()), + required => true + })} + ]. + +'/file_transfer/file'(get, #{query_string := Query}) -> + try + Node = parse_node(maps:get(<<"node">>, Query)), + Filepath = parse_filepath(maps:get(<<"fileref">>, Query)), + case emqx_ft_storage_exporter_fs_proto_v1:read_export_file(Node, Filepath, self()) of + {ok, ReaderPid} -> + FileData = emqx_ft_storage_fs_reader:table(ReaderPid), + {200, + #{ + <<"content-type">> => <<"application/data">>, + <<"content-disposition">> => <<"attachment">> + }, + FileData}; + {error, enoent} -> + {404, error_msg('NOT_FOUND', <<"Not found">>)}; + {error, Error} -> + ?SLOG(warning, #{msg => "get_ready_transfer_fail", error => Error}), + {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} + end + catch + throw:{invalid, Param} -> + {404, + error_msg( + 'NOT_FOUND', + iolist_to_binary(["Invalid query parameter: ", Param]) + )}; + error:{erpc, noconnection} -> + {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} + end. + +error_msg(Code, Msg) -> + #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. + +-spec mk_export_uri(node(), file:name()) -> + uri_string:uri_string(). +mk_export_uri(Node, Filepath) -> + emqx_dashboard_swagger:relative_uri([ + "/file_transfer/file?", + uri_string:compose_query([ + {"node", atom_to_list(Node)}, + {"fileref", Filepath} + ]) + ]). + +%% + +parse_node(NodeBin) -> + case emqx_misc:safe_to_existing_atom(NodeBin) of + {ok, Node} -> + Node; + {error, _} -> + throw({invalid, NodeBin}) + end. + +parse_filepath(PathBin) -> + case filename:pathtype(PathBin) of + relative -> + ok; + absolute -> + throw({invalid, PathBin}) + end, + PathComponents = filename:split(PathBin), + case lists:any(fun is_special_component/1, PathComponents) of + false -> + filename:join(PathComponents); + true -> + throw({invalid, PathBin}) + end. + +is_special_component(<<".", _/binary>>) -> + true; +is_special_component([$. | _]) -> + true; +is_special_component(_) -> + false. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl new file mode 100644 index 000000000..bdb64f543 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl @@ -0,0 +1,46 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% This methods are called via rpc by `emqx_ft_storage_exporter_fs` +%% They populate the call with actual storage which may be configured differently +%% on a concrete node. + +-module(emqx_ft_storage_exporter_fs_proxy). + +-export([ + list_exports_local/0, + read_export_file_local/2 +]). + +list_exports_local() -> + emqx_ft_storage:with_storage_type(local, fun(Storage) -> + case emqx_ft_storage_exporter:exporter(Storage) of + {emqx_ft_storage_exporter_fs, Options} -> + emqx_ft_storage_exporter_fs:list_local(Options); + InvalidExporter -> + {error, {invalid_exporter, InvalidExporter}} + end + end). + +read_export_file_local(Filepath, CallerPid) -> + emqx_ft_storage:with_storage_type(local, fun(Storage) -> + case emqx_ft_storage_exporter:exporter(Storage) of + {emqx_ft_storage_exporter_fs, Options} -> + emqx_ft_storage_exporter_fs:start_reader(Options, Filepath, CallerPid); + InvalidExporter -> + {error, {invalid_exporter, InvalidExporter}} + end + end). diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 6c5b6962c..b328cfd2b 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -44,12 +44,7 @@ -export([get_subdir/2]). -export([get_subdir/3]). --export([exporter/1]). - -% Exporter-specific API -export([exports/1]). --export([exports_local/1]). --export([exports_local/2]). -export_type([storage/0]). -export_type([filefrag/1]). @@ -214,24 +209,8 @@ assemble(Storage, Transfer, Size) -> %% --spec exporter(storage()) -> {module(), _ExporterOptions}. -exporter(Storage) -> - case maps:get(exporter, Storage) of - #{type := local} = Options -> - {emqx_ft_storage_exporter_fs, Options} - end. - exports(Storage) -> - {ExporterMod, ExporterOpts} = exporter(Storage), - ExporterMod:list(ExporterOpts). - -exports_local(Storage) -> - {ExporterMod, ExporterOpts} = exporter(Storage), - ExporterMod:list_local(ExporterOpts). - -exports_local(Storage, Transfer) -> - {ExporterMod, ExporterOpts} = exporter(Storage), - ExporterMod:list_local(ExporterOpts, Transfer). + emqx_ft_storage_exporter:list(Storage). %% diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl index dbd0cd6dc..597f84091 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl @@ -22,9 +22,7 @@ -export([ list_local/2, - pread_local/4, - list_exports_local/0, - read_export_file_local/2 + pread_local/4 ]). list_local(Transfer, What) -> @@ -32,19 +30,3 @@ list_local(Transfer, What) -> pread_local(Transfer, Frag, Offset, Size) -> emqx_ft_storage:with_storage_type(local, pread, [Transfer, Frag, Offset, Size]). - -list_exports_local() -> - case emqx_ft_storage:with_storage_type(local, exporter, []) of - {emqx_ft_storage_exporter_fs, Options} -> - emqx_ft_storage_exporter_fs:list_local(Options); - InvalidExporter -> - {error, {invalid_exporter, InvalidExporter}} - end. - -read_export_file_local(Filepath, CallerPid) -> - case emqx_ft_storage:with_storage_type(local, exporter, []) of - {emqx_ft_storage_exporter_fs, Options} -> - emqx_ft_storage_exporter_fs:start_reader(Options, Filepath, CallerPid); - InvalidExporter -> - {error, {invalid_exporter, InvalidExporter}} - end. diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl new file mode 100644 index 000000000..ee7a857be --- /dev/null +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl @@ -0,0 +1,51 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_exporter_fs_proto_v1). + +-behaviour(emqx_bpapi). + +-export([introduced_in/0]). + +-export([list_exports/1]). +-export([read_export_file/3]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.17". + +-spec list_exports([node()]) -> + emqx_rpc:erpc_multicall([emqx_ft_storage:export_info()]). +list_exports(Nodes) -> + erpc:multicall( + Nodes, + emqx_ft_storage_exporter_fs_proxy, + list_exports_local, + [] + ). + +-spec read_export_file(node(), file:name(), pid()) -> + {ok, emqx_ft_storage:reader()} + | {error, term()} + | no_return(). +read_export_file(Node, Filepath, CallerPid) -> + erpc:call( + Node, + emqx_ft_storage_exporter_fs_proxy, + read_export_file_local, + [Filepath, CallerPid] + ). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index f152928fe..3b91684e6 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -23,10 +23,6 @@ -export([multilist/3]). -export([pread/5]). -%% TODO: These should be defined in a separate BPAPI --export([list_exports/1]). --export([read_export_file/3]). - -type offset() :: emqx_ft:offset(). -type transfer() :: emqx_ft:transfer(). -type filefrag() :: emqx_ft_storage_fs:filefrag(). @@ -45,17 +41,3 @@ multilist(Nodes, Transfer, What) -> {ok, [filefrag()]} | {error, term()} | no_return(). pread(Node, Transfer, Frag, Offset, Size) -> erpc:call(Node, emqx_ft_storage_fs_proxy, pread_local, [Transfer, Frag, Offset, Size]). - -%% - --spec list_exports([node()]) -> - emqx_rpc:erpc_multicall([emqx_ft_storage:export_info()]). -list_exports(Nodes) -> - erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, list_exports_local, []). - --spec read_export_file(node(), file:name(), pid()) -> - {ok, emqx_ft_storage:export_data()} - | {error, term()} - | no_return(). -read_export_file(Node, Filepath, CallerPid) -> - erpc:call(Node, emqx_ft_storage_fs_proxy, read_export_file_local, [Filepath, CallerPid]). diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index 56622f3cd..a1bef2a2a 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -24,7 +24,7 @@ -include_lib("emqx/include/asserts.hrl"). --import(emqx_mgmt_api_test_util, [uri/1]). +-import(emqx_dashboard_api_test_helpers, [host/0, uri/1]). all() -> emqx_common_test_helpers:all(?MODULE). @@ -111,7 +111,7 @@ t_download_transfer(Config) -> {ok, 200, #{<<"files">> := [File]}} = request(get, uri(["file_transfer", "files"]), fun json/1), - {ok, 200, Response} = request(get, uri([]) ++ maps:get(<<"uri">>, File)), + {ok, 200, Response} = request(get, host() ++ maps:get(<<"uri">>, File)), ?assertEqual( <<"data">>, diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 4c7c0f08c..7c5d14b38 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -86,8 +86,7 @@ t_assemble_empty_transfer(Config) -> ), Status = complete_assemble(Storage, Transfer, 0), ?assertEqual({shutdown, ok}, Status), - {ok, [_Result = #{size := _Size = 0}]} = - emqx_ft_storage_fs:exports_local(Storage, Transfer), + {ok, [_Result = #{size := _Size = 0}]} = list_exports(Config, Transfer), % ?assertEqual( % {error, eof}, % emqx_ft_storage_fs:pread(Storage, Transfer, Result, 0, Size) @@ -138,10 +137,9 @@ t_assemble_complete_local_transfer(Config) -> meta := #{} } ]}, - emqx_ft_storage_fs:exports_local(Storage, Transfer) + list_exports(Config, Transfer) ), - {ok, [#{path := AssemblyFilename}]} = - emqx_ft_storage_fs:exports_local(Storage, Transfer), + {ok, [#{path := AssemblyFilename}]} = list_exports(Config, Transfer), ?assertMatch( {ok, #file_info{type = regular, size = TransferSize}}, file:read_file_info(AssemblyFilename) @@ -193,8 +191,7 @@ complete_assemble(Storage, Transfer, Size, Timeout) -> %% t_list_transfers(Config) -> - Storage = storage(Config), - {ok, Exports} = emqx_ft_storage_fs:exports_local(Storage), + {ok, Exports} = list_exports(Config), ?assertMatch( [ #{ @@ -237,6 +234,17 @@ inspect_file(Filename) -> mk_fileid() -> integer_to_binary(erlang:system_time(millisecond)). +list_exports(Config) -> + {emqx_ft_storage_exporter_fs, Options} = exporter(Config), + emqx_ft_storage_exporter_fs:list_local(Options). + +list_exports(Config, Transfer) -> + {emqx_ft_storage_exporter_fs, Options} = exporter(Config), + emqx_ft_storage_exporter_fs:list_local(Options, Transfer). + +exporter(Config) -> + emqx_ft_storage_exporter:exporter(storage(Config)). + storage(Config) -> #{ type => local, From 23cd78b8d63913efc1b6dc0e896d5d3d42015444 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 28 Mar 2023 13:26:49 +0300 Subject: [PATCH 077/197] refactor(ft-asm): turn state data record into a map Which should be more future-proof. --- apps/emqx_ft/src/emqx_ft_assembler.erl | 114 +++++++++++------- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 3 + 2 files changed, 75 insertions(+), 42 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index a86116b02..17fed012a 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -24,12 +24,12 @@ -export([handle_event/4]). -export([terminate/3]). --record(st, { - storage :: _Storage, - transfer :: emqx_ft:transfer(), - assembly :: emqx_ft_assembly:t(), - export :: _Export | undefined -}). +-type stdata() :: #{ + storage := emqx_ft_storage_fs:storage(), + transfer := emqx_ft:transfer(), + assembly := emqx_ft_assembly:t(), + export => emqx_ft_storage_exporter:export() +}. -define(NAME(Transfer), {n, l, {?MODULE, Transfer}}). -define(REF(Transfer), {via, gproc, ?NAME(Transfer)}). @@ -41,31 +41,48 @@ start_link(Storage, Transfer, Size) -> %% +-type state() :: + idle + | list_local_fragments + | {list_remote_fragments, [node()]} + | start_assembling + | {assemble, [{node(), emqx_ft_storage_fs:filefrag()}]} + | complete. + -define(internal(C), {next_event, internal, C}). callback_mode() -> handle_event_function. +-spec init(_Args) -> {ok, state(), stdata()}. init({Storage, Transfer, Size}) -> _ = erlang:process_flag(trap_exit, true), - St = #st{ - storage = Storage, - transfer = Transfer, - assembly = emqx_ft_assembly:new(Size) + St = #{ + storage => Storage, + transfer => Transfer, + assembly => emqx_ft_assembly:new(Size) }, {ok, idle, St}. +-spec handle_event(info | internal, _, state(), stdata()) -> + {next_state, state(), stdata(), {next_event, internal, _}} + | {stop, {shutdown, ok | {error, _}}, stdata()}. handle_event(info, kickoff, idle, St) -> % NOTE % Someone's told us to start the work, which usually means that it has set up a monitor. % We could wait for this message and handle it at the end of the assembling rather than at % the beginning, however it would make error handling much more messier. {next_state, list_local_fragments, St, ?internal([])}; -handle_event(internal, _, list_local_fragments, St = #st{}) -> +handle_event( + internal, + _, + list_local_fragments, + St = #{storage := Storage, transfer := Transfer, assembly := Asm} +) -> % TODO: what we do with non-transients errors here (e.g. `eacces`)? - {ok, Fragments} = emqx_ft_storage_fs:list(St#st.storage, St#st.transfer, fragment), - NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(St#st.assembly, node(), Fragments)), - NSt = St#st{assembly = NAsm}, + {ok, Fragments} = emqx_ft_storage_fs:list(Storage, Transfer, fragment), + NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(Asm, node(), Fragments)), + NSt = St#{assembly := NAsm}, case emqx_ft_assembly:status(NAsm) of complete -> {next_state, start_assembling, NSt, ?internal([])}; @@ -76,27 +93,32 @@ handle_event(internal, _, list_local_fragments, St = #st{}) -> {error, _} = Error -> {stop, {shutdown, Error}} end; -handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> +handle_event( + internal, + _, + {list_remote_fragments, Nodes}, + St = #{transfer := Transfer, assembly := Asm} +) -> % TODO % Async would better because we would not need to wait for some lagging nodes if % the coverage is already complete. % TODO: portable "storage" ref - Results = emqx_ft_storage_fs_proto_v1:multilist(Nodes, St#st.transfer, fragment), + Results = emqx_ft_storage_fs_proto_v1:multilist(Nodes, Transfer, fragment), NodeResults = lists:zip(Nodes, Results), NAsm = emqx_ft_assembly:update( lists:foldl( fun - ({Node, {ok, {ok, Fragments}}}, Asm) -> - emqx_ft_assembly:append(Asm, Node, Fragments); - ({_Node, _Result}, Asm) -> + ({Node, {ok, {ok, Fragments}}}, Acc) -> + emqx_ft_assembly:append(Acc, Node, Fragments); + ({_Node, _Result}, Acc) -> % TODO: log? - Asm + Acc end, - St#st.assembly, + Asm, NodeResults ) ), - NSt = St#st{assembly = NAsm}, + NSt = St#{assembly := NAsm}, case emqx_ft_assembly:status(NAsm) of complete -> {next_state, start_assembling, NSt, ?internal([])}; @@ -106,43 +128,51 @@ handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> {error, _} = Error -> {stop, {shutdown, Error}} end; -handle_event(internal, _, start_assembling, St = #st{}) -> - Filemeta = emqx_ft_assembly:filemeta(St#st.assembly), - Coverage = emqx_ft_assembly:coverage(St#st.assembly), +handle_event( + internal, + _, + start_assembling, + St = #{storage := Storage, transfer := Transfer, assembly := Asm} +) -> + Filemeta = emqx_ft_assembly:filemeta(Asm), + Coverage = emqx_ft_assembly:coverage(Asm), % TODO: better error handling {ok, Export} = emqx_ft_storage_exporter:start_export( - St#st.storage, - St#st.transfer, + Storage, + Transfer, Filemeta ), - {next_state, {assemble, Coverage}, St#st{export = Export}, ?internal([])}; -handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> + {next_state, {assemble, Coverage}, St#{export => Export}, ?internal([])}; +handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #{export := Export}) -> % TODO % Currently, race is possible between getting segment info from the remote node and % this node garbage collecting the segment itself. % TODO: pipelining % TODO: better error handling {ok, Content} = pread(Node, Segment, St), - {ok, NExport} = emqx_ft_storage_exporter:write(St#st.export, Content), - {next_state, {assemble, Rest}, St#st{export = NExport}, ?internal([])}; -handle_event(internal, _, {assemble, []}, St = #st{}) -> + {ok, NExport} = emqx_ft_storage_exporter:write(Export, Content), + {next_state, {assemble, Rest}, St#{export := NExport}, ?internal([])}; +handle_event(internal, _, {assemble, []}, St = #{}) -> {next_state, complete, St, ?internal([])}; -handle_event(internal, _, complete, St = #st{}) -> - Result = emqx_ft_storage_exporter:complete(St#st.export), +handle_event(internal, _, complete, St = #{export := Export}) -> + Result = emqx_ft_storage_exporter:complete(Export), ok = maybe_garbage_collect(Result, St), - {stop, {shutdown, Result}, St#st{export = undefined}}. + {stop, {shutdown, Result}, maps:remove(export, St)}. -terminate(_Reason, _StateName, #st{export = Export}) -> - Export /= undefined andalso emqx_ft_storage_exporter:discard(Export). +-spec terminate(_Reason, state(), stdata()) -> _. +terminate(_Reason, _StateName, #{export := Export}) -> + emqx_ft_storage_exporter:discard(Export); +terminate(_Reason, _StateName, #{}) -> + ok. -pread(Node, Segment, St) when Node =:= node() -> - emqx_ft_storage_fs:pread(St#st.storage, St#st.transfer, Segment, 0, segsize(Segment)); -pread(Node, Segment, St) -> - emqx_ft_storage_fs_proto_v1:pread(Node, St#st.transfer, Segment, 0, segsize(Segment)). +pread(Node, Segment, #{storage := Storage, transfer := Transfer}) when Node =:= node() -> + emqx_ft_storage_fs:pread(Storage, Transfer, Segment, 0, segsize(Segment)); +pread(Node, Segment, #{transfer := Transfer}) -> + emqx_ft_storage_fs_proto_v1:pread(Node, Transfer, Segment, 0, segsize(Segment)). %% -maybe_garbage_collect(ok, #st{storage = Storage, transfer = Transfer, assembly = Asm}) -> +maybe_garbage_collect(ok, #{storage := Storage, transfer := Transfer, assembly := Asm}) -> Nodes = emqx_ft_assembly:nodes(Asm), emqx_ft_storage_fs_gc:collect(Storage, Transfer, Nodes); maybe_garbage_collect({error, _}, _St) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index 97e260e3a..954423fba 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -34,6 +34,9 @@ -export([exporter/1]). +-export_type([options/0]). +-export_type([export/0]). + -type storage() :: emxt_ft_storage_fs:storage(). -type transfer() :: emqx_ft:transfer(). -type filemeta() :: emqx_ft:filemeta(). From c24c7eca34ca76e0dd0d727336e2c63bdfc5176b Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 28 Mar 2023 14:38:30 +0300 Subject: [PATCH 078/197] fix(ft-test): unbreak testcase by inhibiting local fs storage GC --- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 31 +++++++++++++++------- apps/emqx_ft/test/emqx_ft_SUITE.erl | 23 +++++++--------- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index eb274a4d5..258f3a7f3 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -81,12 +81,17 @@ handle_call(Call, From, St) -> {noreply, St}. handle_cast({collect, Transfer, [Node | Rest]}, St) -> - ok = do_collect_transfer(Transfer, Node, St), - case Rest of - [_ | _] -> - gen_server:cast(self(), {collect, Transfer, Rest}); - [] -> - ok + case gc_enabled(St) of + true -> + ok = do_collect_transfer(Transfer, Node, St), + case Rest of + [_ | _] -> + gen_server:cast(self(), {collect, Transfer, Rest}); + [] -> + ok + end; + false -> + skip end, {noreply, St}; handle_cast(reset, St) -> @@ -127,9 +132,14 @@ maybe_report(#gcstats{errors = Errors}, #st{storage = Storage}) when map_size(Er maybe_report(#gcstats{} = _Stats, #st{storage = _Storage}) -> ?tp(garbage_collection, #{stats => _Stats, storage => _Storage}). -start_timer(St = #st{next_gc_timer = undefined}) -> - Delay = emqx_ft_conf:gc_interval(St#st.storage), - St#st{next_gc_timer = emqx_misc:start_timer(Delay, collect)}. +start_timer(St = #st{storage = Storage, next_gc_timer = undefined}) -> + case emqx_ft_conf:gc_interval(Storage) of + Delay when Delay > 0 -> + St#st{next_gc_timer = emqx_misc:start_timer(Delay, collect)}; + 0 -> + ?SLOG(warning, #{msg => "periodic_gc_disabled"}), + St + end. reset_timer(St = #st{next_gc_timer = undefined}) -> start_timer(St); @@ -137,6 +147,9 @@ reset_timer(St = #st{next_gc_timer = TRef}) -> ok = emqx_misc:cancel_timer(TRef), start_timer(St#st{next_gc_timer = undefined}). +gc_enabled(St) -> + emqx_ft_conf:gc_interval(St#st.storage) > 0. + %% collect_garbage(Storage) -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 1a886c00d..c493a78c5 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -58,8 +58,12 @@ end_per_suite(_Config) -> set_special_configs(Config) -> fun (emqx_ft) -> + Storage = emqx_ft_test_helpers:local_storage(Config), emqx_ft_test_helpers:load_config(#{ - storage => emqx_ft_test_helpers:local_storage(Config) + % NOTE + % Inhibit local fs GC to simulate it isn't fast enough to collect + % complete transfers. + storage => Storage#{gc => #{interval => 0}} }); (_) -> ok @@ -107,14 +111,7 @@ mk_cluster_specs(Config) -> {env, [{emqx, boot_modules, [broker, listeners]}]}, {apps, [emqx_ft]}, {conf, [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]]}, - {env_handler, fun - (emqx_ft) -> - emqx_ft_test_helpers:load_config(#{ - storage => emqx_ft_test_helpers:local_storage(Config) - }); - (_) -> - ok - end} + {env_handler, set_special_configs(Config)} ], emqx_common_test_helpers:emqx_cluster( Specs, @@ -549,22 +546,20 @@ t_unreliable_migrating_client(Config) -> % twice. This is currently expected, files must be identical anyway. Node1Str = atom_to_list(Node1), NodeSelfStr = atom_to_list(NodeSelf), + % TODO: this testcase is specific to local fs storage backend ?assertMatch( [#{"node" := Node1Str}, #{"node" := NodeSelfStr}], lists:map( fun(#{uri := URIString}) -> #{query := QS} = uri_string:parse(URIString), - uri_string:dissect_query(QS) + maps:from_list(uri_string:dissect_query(QS)) end, lists:sort(Exports) ) ), [ - ?assertEqual( - {ok, Payload}, - read_export(Export) - ) + ?assertEqual({ok, Payload}, read_export(Export)) || Export <- Exports ]. From 45e3b62dc424e8b3e155863ab67c5846bbe53203 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 28 Mar 2023 17:48:10 +0300 Subject: [PATCH 079/197] refactor(ft): prefer plain `files` over `exports` in generic APIs So there's no more `exports` magically becomings `files` in the REST API which was slightly confusing. --- apps/emqx_ft/src/emqx_ft_api.erl | 8 ++++---- apps/emqx_ft/src/emqx_ft_storage.erl | 18 +++++++++--------- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 4 ++-- .../emqx_ft_storage_exporter_fs_proto_v1.erl | 2 +- apps/emqx_ft/test/emqx_ft_SUITE.erl | 14 +++++++------- apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 8 ++++---- 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index eeb047b16..a667454b6 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -67,9 +67,9 @@ schema("/file_transfer/files") -> }. '/file_transfer/files'(get, #{}) -> - case emqx_ft_storage:exports() of - {ok, Transfers} -> - {200, #{<<"files">> => lists:map(fun format_export_info/1, Transfers)}}; + case emqx_ft_storage:files() of + {ok, Files} -> + {200, #{<<"files">> => lists:map(fun format_file_info/1, Files)}}; {error, _} -> {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} end. @@ -84,7 +84,7 @@ roots() -> %% Helpers %%-------------------------------------------------------------------- -format_export_info( +format_file_info( Info = #{ name := Name, size := Size, diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 05a053fbf..bff149ce2 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -24,7 +24,7 @@ store_segment/2, assemble/2, - exports/0, + files/0, with_storage_type/2, with_storage_type/3 @@ -34,13 +34,13 @@ -type storage() :: emqx_config:config(). -export_type([assemble_callback/0]). --export_type([export_info/0]). +-export_type([file_info/0]). -export_type([export_data/0]). -export_type([reader/0]). -type assemble_callback() :: fun((ok | {error, term()}) -> any()). --type export_info() :: #{ +-type file_info() :: #{ transfer := emqx_ft:transfer(), name := file:name(), size := _Bytes :: non_neg_integer(), @@ -68,8 +68,8 @@ -callback assemble(storage(), emqx_ft:transfer(), _Size :: emqx_ft:bytes()) -> ok | {async, pid()} | {error, term()}. --callback exports(storage()) -> - {ok, [export_info()]} | {error, term()}. +-callback files(storage()) -> + {ok, [file_info()]} | {error, term()}. %%-------------------------------------------------------------------- %% API @@ -104,11 +104,11 @@ assemble(Transfer, Size) -> Mod = mod(), Mod:assemble(storage(), Transfer, Size). --spec exports() -> - {ok, [export_info()]} | {error, term()}. -exports() -> +-spec files() -> + {ok, [file_info()]} | {error, term()}. +files() -> Mod = mod(), - Mod:exports(storage()). + Mod:files(storage()). -spec with_storage_type(atom(), atom() | function()) -> any(). with_storage_type(Type, Fun) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index 954423fba..a7d1cd2e2 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -57,7 +57,7 @@ ok | {error, _Reason}. -callback list(options()) -> - {ok, [emqx_ft_storage:export_info()]} | {error, _Reason}. + {ok, [emqx_ft_storage:file_info()]} | {error, _Reason}. %% diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index b328cfd2b..10ca263da 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -44,7 +44,7 @@ -export([get_subdir/2]). -export([get_subdir/3]). --export([exports/1]). +-export([files/1]). -export_type([storage/0]). -export_type([filefrag/1]). @@ -209,7 +209,7 @@ assemble(Storage, Transfer, Size) -> %% -exports(Storage) -> +files(Storage) -> emqx_ft_storage_exporter:list(Storage). %% diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl index ee7a857be..4c64011de 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl @@ -29,7 +29,7 @@ introduced_in() -> "5.0.17". -spec list_exports([node()]) -> - emqx_rpc:erpc_multicall([emqx_ft_storage:export_info()]). + emqx_rpc:erpc_multicall([emqx_ft_storage:file_info()]). list_exports(Nodes) -> erpc:multicall( Nodes, diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index c493a78c5..3cfcebf93 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -216,7 +216,7 @@ t_simple_transfer(Config) -> emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) ), - [Export] = list_exports(?config(clientid, Config)), + [Export] = list_files(?config(clientid, Config)), ?assertEqual( {ok, iolist_to_binary(Data)}, read_export(Export) @@ -234,7 +234,7 @@ t_nasty_clientids_fileids(_Config) -> ok = lists:foreach( fun({ClientId, FileId}) -> ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "justfile", ClientId), - [Export] = list_exports(ClientId), + [Export] = list_files(ClientId), ?assertEqual({ok, ClientId}, read_export(Export)) end, Transfers @@ -463,7 +463,7 @@ t_switch_node(Config) -> %% Now check consistency of the file - [Export] = list_exports(ClientId), + [Export] = list_files(ClientId), ?assertEqual( {ok, iolist_to_binary(Data)}, read_export(Export) @@ -539,7 +539,7 @@ t_unreliable_migrating_client(Config) -> ], _Context = run_commands(Commands, Context), - Exports = list_exports(?config(clientid, Config)), + Exports = list_files(?config(clientid, Config)), % NOTE % The cluster had 2 assemblers running on two different nodes, because client sent `fin` @@ -659,9 +659,9 @@ meta(FileName, Data) -> size => byte_size(FullData) }. -list_exports(ClientId) -> - {ok, Exports} = emqx_ft_storage:exports(), - [Export || Export = #{transfer := {CId, _}} <- Exports, CId == ClientId]. +list_files(ClientId) -> + {ok, Files} = emqx_ft_storage:files(), + [File || File = #{transfer := {CId, _}} <- Files, CId == ClientId]. read_export(#{path := AbsFilepath}) -> % TODO: only works for the local filesystem exporter right now diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index 5b3f56b7d..e1574da0d 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -83,7 +83,7 @@ t_multinode_ready_transfers(Config) -> #{transfer := {<<"c/1">>, <<"f:1">>}, name := "fn1"}, #{transfer := {<<"c/2">>, <<"f:2">>}, name := "fn2"} ], - lists:sort(list_exports(Config)) + lists:sort(list_files(Config)) ). %%-------------------------------------------------------------------- @@ -103,6 +103,6 @@ storage(Config) -> } }. -list_exports(Config) -> - {ok, Exports} = emqx_ft_storage_fs:exports(storage(Config)), - Exports. +list_files(Config) -> + {ok, Files} = emqx_ft_storage_fs:files(storage(Config)), + Files. From 6ad7ce55d2490451eee5be6abe577d59a1d86a07 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 28 Mar 2023 18:49:02 +0300 Subject: [PATCH 080/197] fix(ft-conf): separate local segments storage settings To make things less ambiguous. --- apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 17 +++++++++--- apps/emqx_ft/src/emqx_ft_conf.erl | 17 +++++++++--- apps/emqx_ft/src/emqx_ft_schema.erl | 26 +++++++++++++------ apps/emqx_ft/src/emqx_ft_storage_fs.erl | 10 +++---- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 8 +++--- apps/emqx_ft/test/emqx_ft_SUITE.erl | 5 +++- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 4 ++- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 13 ++++++++-- .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 9 +------ .../test/emqx_ft_storage_fs_gc_SUITE.erl | 26 +++++++++++-------- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 9 +++++-- 11 files changed, 96 insertions(+), 48 deletions(-) diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf index 15c42dcfa..941611424 100644 --- a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -22,13 +22,24 @@ emqx_ft_schema { } } - local_storage_root { + local_storage_segments { + desc { + en: "Settings for local segments storage, which include uploaded transfer fragments and temporary data." + zh: "保存上传文件和临时数据的文件系统路径。" + } + label: { + en: "Local Segments Storage" + zh: "本地存储根" + } + } + + local_storage_segments_root { desc { en: "File system path to keep uploaded fragments and temporary data." zh: "保存上传文件和临时数据的文件系统路径。" } label: { - en: "Local Storage Root" + en: "Local Segments Storage Filesystem Root" zh: "本地存储根" } } @@ -67,7 +78,7 @@ emqx_ft_schema { } } - local_storage_gc { + local_storage_segments_gc { desc { en: "Garbage collection settings for the intermediate and temporary files in the local file system." zh: "" diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index e925ee376..b0e73cda4 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -22,6 +22,7 @@ %% Accessors -export([storage/0]). +-export([segments_root/1]). -export([gc_interval/1]). -export([segments_ttl/1]). @@ -48,17 +49,27 @@ storage() -> emqx_config:get([file_transfer, storage], disabled). +-spec segments_root(_Storage) -> file:name(). +segments_root(_Storage) -> + Conf = assert_storage(local), + case emqx_map_lib:deep_find([segments, root], Conf) of + {ok, Root} -> + Root; + {not_found, _, _} -> + filename:join([emqx:data_dir(), file_transfer, segments]) + end. + -spec gc_interval(_Storage) -> milliseconds(). gc_interval(_Storage) -> Conf = assert_storage(local), - emqx_map_lib:deep_get([gc, interval], Conf). + emqx_map_lib:deep_get([segments, gc, interval], Conf). -spec segments_ttl(_Storage) -> {_Min :: seconds(), _Max :: seconds()}. segments_ttl(_Storage) -> Conf = assert_storage(local), { - emqx_map_lib:deep_get([gc, minimum_segments_ttl], Conf), - emqx_map_lib:deep_get([gc, maximum_segments_ttl], Conf) + emqx_map_lib:deep_get([segments, gc, minimum_segments_ttl], Conf), + emqx_map_lib:deep_get([segments, gc, maximum_segments_ttl], Conf) }. assert_storage(Type) -> diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 5c0a764a2..2abbe4c45 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -61,9 +61,9 @@ fields(local_storage) -> required => false, desc => ?DESC("local_type") }}, - {root, #{ - type => binary(), - desc => ?DESC("local_storage_root"), + {segments, #{ + type => ?REF(local_storage_segments), + desc => ?DESC("local_storage_segments"), required => false }}, {exporter, #{ @@ -72,10 +72,18 @@ fields(local_storage) -> ]), desc => ?DESC("local_storage_exporter"), required => true + }} + ]; +fields(local_storage_segments) -> + [ + {root, #{ + type => binary(), + desc => ?DESC("local_storage_segments_root"), + required => false }}, {gc, #{ - type => ?REF(local_storage_gc), - desc => ?DESC("local_storage_gc"), + type => ?REF(local_storage_segments_gc), + desc => ?DESC("local_storage_segments_gc"), required => false }} ]; @@ -93,7 +101,7 @@ fields(local_storage_exporter) -> required => false }} ]; -fields(local_storage_gc) -> +fields(local_storage_segments_gc) -> [ {interval, #{ type => emqx_schema:duration_ms(), @@ -122,10 +130,12 @@ desc(file_transfer) -> "File transfer settings"; desc(local_storage) -> "File transfer local storage settings"; +desc(local_storage_segments) -> + "File transfer local segments storage settings"; desc(local_storage_exporter) -> "Exporter settings for the File transfer local storage backend"; -desc(local_storage_gc) -> - "Garbage collection settings for the File transfer local storage backend". +desc(local_storage_segments_gc) -> + "Garbage collection settings for the File transfer local segments storage". schema(filemeta) -> #{ diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 10ca263da..f0bc9ac67 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -220,7 +220,7 @@ transfers(Storage) -> % TODO `Continuation` % There might be millions of transfers on the node, we need a protocol and % storage schema to iterate through them effectively. - ClientIds = try_list_dir(get_storage_root(Storage)), + ClientIds = try_list_dir(get_segments_root(Storage)), {ok, lists:foldl( fun(ClientId, Acc) -> transfers(Storage, ClientId, Acc) end, @@ -229,7 +229,7 @@ transfers(Storage) -> )}. transfers(Storage, ClientId, AccIn) -> - Dirname = filename:join(get_storage_root(Storage), ClientId), + Dirname = filename:join(get_segments_root(Storage), ClientId), case file:list_dir(Dirname) of {ok, FileIds} -> lists:foldl( @@ -307,7 +307,7 @@ break_segment_filename(Filename) -> mk_filedir(Storage, {ClientId, FileId}, SubDirs) -> filename:join([ - get_storage_root(Storage), + get_segments_root(Storage), emqx_ft_fs_util:escape_filename(ClientId), emqx_ft_fs_util:escape_filename(FileId) | SubDirs @@ -325,8 +325,8 @@ try_list_dir(Dirname) -> {error, _} -> [] end. -get_storage_root(Storage) -> - maps:get(root, Storage, filename:join([emqx:data_dir(), "ft", "transfers"])). +get_segments_root(Storage) -> + emqx_ft_conf:segments_root(Storage). -include_lib("kernel/include/file.hrl"). diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 258f3a7f3..63b0ab500 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -65,7 +65,7 @@ collect(Storage, Transfer, Nodes) -> mk_server_ref(Storage) -> % TODO - {via, gproc, {n, l, {?MODULE, get_storage_root(Storage)}}}. + {via, gproc, {n, l, {?MODULE, get_segments_root(Storage)}}}. %% @@ -223,7 +223,7 @@ collect_transfer_directory(Storage, Transfer, Cutoff, Stats) -> end, case collect_empty_directory(Dirname, Filter, Stats) of {true, StatsNext} -> - collect_parents(Dirname, get_storage_root(Storage), StatsNext); + collect_parents(Dirname, get_segments_root(Storage), StatsNext); {false, StatsNext} -> StatsNext end. @@ -373,5 +373,5 @@ register_gcstat_error(Subject, Error, Stats = #gcstats{errors = Errors}) -> %% -get_storage_root(Storage) -> - maps:get(root, Storage, filename:join(emqx:data_dir(), "file_transfer")). +get_segments_root(Storage) -> + emqx_ft_conf:segments_root(Storage). diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 3cfcebf93..e365cba85 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -63,7 +63,10 @@ set_special_configs(Config) -> % NOTE % Inhibit local fs GC to simulate it isn't fast enough to collect % complete transfers. - storage => Storage#{gc => #{interval => 0}} + storage => emqx_map_lib:deep_merge( + Storage, + #{segments => #{gc => #{interval => 0}}} + ) }); (_) -> ok diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 7c5d14b38..4b5610f51 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -248,7 +248,9 @@ exporter(Config) -> storage(Config) -> #{ type => local, - root => ?config(storage_root, Config), + segments => #{ + root => ?config(storage_root, Config) + }, exporter => #{ type => local, root => ?config(exports_root, Config) diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index e43abf095..3811c1ef4 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -59,7 +59,12 @@ t_update_config(_Config) -> #{ <<"storage">> => #{ <<"type">> => <<"local">>, - <<"root">> => <<"/tmp/path">>, + <<"segments">> => #{ + <<"root">> => <<"/tmp/path">>, + <<"gc">> => #{ + <<"interval">> => <<"5m">> + } + }, <<"exporter">> => #{ <<"type">> => <<"local">>, <<"root">> => <<"/tmp/exports">> @@ -71,5 +76,9 @@ t_update_config(_Config) -> ), ?assertEqual( <<"/tmp/path">>, - emqx_config:get([file_transfer, storage, root]) + emqx_config:get([file_transfer, storage, segments, root]) + ), + ?assertEqual( + 5 * 60 * 1000, + emqx_ft_conf:gc_interval(emqx_ft_conf:storage()) ). diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index e1574da0d..e3decf0f5 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -94,14 +94,7 @@ client_id(Config) -> atom_to_binary(?config(tc, Config), utf8). storage(Config) -> - #{ - type => local, - root => emqx_ft_test_helpers:root(Config, node(), ["transfers"]), - exporter => #{ - type => local, - root => emqx_ft_test_helpers:root(Config, node(), ["exports"]) - } - }. + emqx_ft_test_helpers:local_storage(Config). list_files(Config) -> {ok, Files} = emqx_ft_storage_fs:files(storage(Config)), diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index 53a22b8b6..90039cd96 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -20,7 +20,6 @@ -compile(nowarn_export_all). -include_lib("emqx_ft/include/emqx_ft_storage_fs.hrl"). --include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). -include_lib("snabbkaffe/include/test_macros.hrl"). @@ -43,7 +42,9 @@ init_per_testcase(TC, Config) -> emqx_ft_test_helpers:load_config(#{ storage => #{ type => local, - root => emqx_ft_test_helpers:root(Config, node(), [TC, transfers]), + segments => #{ + root => emqx_ft_test_helpers:root(Config, node(), [TC, segments]) + }, exporter => #{ type => local, root => emqx_ft_test_helpers:root(Config, node(), [TC, exports]) @@ -66,7 +67,7 @@ end_per_testcase(_TC, _Config) -> t_gc_triggers_periodically(_Config) -> Interval = 500, - ok = emqx_config:put([file_transfer, storage, gc, interval], Interval), + ok = set_gc_config(interval, Interval), ok = emqx_ft_storage_fs_gc:reset(emqx_ft_conf:storage()), ?check_trace( timer:sleep(Interval * 3), @@ -104,9 +105,9 @@ t_gc_triggers_manually(_Config) -> t_gc_complete_transfers(_Config) -> Storage = emqx_ft_conf:storage(), - ok = emqx_config:put([file_transfer, storage, gc, minimum_segments_ttl], 0), - ok = emqx_config:put([file_transfer, storage, gc, maximum_segments_ttl], 3), - ok = emqx_config:put([file_transfer, storage, gc, interval], 500), + ok = set_gc_config(minimum_segments_ttl, 0), + ok = set_gc_config(maximum_segments_ttl, 3), + ok = set_gc_config(interval, 500), ok = emqx_ft_storage_fs_gc:reset(Storage), Transfers = [ { @@ -194,8 +195,8 @@ t_gc_complete_transfers(_Config) -> ). t_gc_incomplete_transfers(_Config) -> - ok = emqx_config:put([file_transfer, storage, gc, minimum_segments_ttl], 0), - ok = emqx_config:put([file_transfer, storage, gc, maximum_segments_ttl], 4), + ok = set_gc_config(minimum_segments_ttl, 0), + ok = set_gc_config(maximum_segments_ttl, 4), Storage = emqx_ft_conf:storage(), Transfers = [ { @@ -222,7 +223,7 @@ t_gc_incomplete_transfers(_Config) -> % 1. Start transfers, send all the segments but don't trigger completion. _ = emqx_misc:pmap(fun(Transfer) -> start_transfer(Storage, Transfer) end, Transfers), % 2. Enable periodic GC every 0.5 seconds. - ok = emqx_config:put([file_transfer, storage, gc, interval], 500), + ok = set_gc_config(interval, 500), ok = emqx_ft_storage_fs_gc:reset(Storage), % 3. First we need the first transfer to be collected. {ok, _} = ?block_until( @@ -265,8 +266,8 @@ t_gc_incomplete_transfers(_Config) -> ). t_gc_handling_errors(_Config) -> - ok = emqx_config:put([file_transfer, storage, gc, minimum_segments_ttl], 0), - ok = emqx_config:put([file_transfer, storage, gc, maximum_segments_ttl], 0), + ok = set_gc_config(minimum_segments_ttl, 0), + ok = set_gc_config(maximum_segments_ttl, 0), Storage = emqx_ft_conf:storage(), Transfer1 = {<<"client1">>, mk_file_id()}, Transfer2 = {<<"client2">>, mk_file_id()}, @@ -322,6 +323,9 @@ t_gc_handling_errors(_Config) -> %% +set_gc_config(Name, Value) -> + emqx_config:put([file_transfer, storage, segments, gc, Name], Value). + start_transfer(Storage, {Transfer, Meta, Gen}) -> ?assertEqual( ok, diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index a38242629..b8ee45b15 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -46,8 +46,13 @@ stop_additional_node(Node) -> local_storage(Config) -> #{ type => local, - root => root(Config, node(), [transfers]), - exporter => #{type => local, root => root(Config, node(), [exports])} + segments => #{ + root => root(Config, node(), [segments]) + }, + exporter => #{ + type => local, + root => root(Config, node(), [exports]) + } }. load_config(Config) -> From bef5cc9c0f3b10ac5d415383d06dff3b2d6b4366 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 29 Mar 2023 14:22:34 +0300 Subject: [PATCH 081/197] fix(fs-fold): avoid folding through symlinked directories Also a testsuite that verifies multilevel fold behaviour. --- apps/emqx_ft/src/emqx_ft_fs_util.erl | 23 ++- apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl | 159 ++++++++++++++++++ .../emqx_ft_fs_util_SUITE_data/a/b/foo/42 | 0 .../test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я | 1 + .../test/emqx_ft_fs_util_SUITE_data/a/link | 1 + .../emqx_ft_fs_util_SUITE_data/c/bar/中文 | 1 + .../test/emqx_ft_fs_util_SUITE_data/c/link | 1 + .../emqx_ft_fs_util_SUITE_data/d/e/baz/needle | 1 + .../emqx_ft_fs_util_SUITE_data/d/haystack | 1 + 9 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl create mode 100644 apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/42 create mode 100644 apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я create mode 120000 apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/link create mode 100644 apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/bar/中文 create mode 120000 apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/link create mode 100644 apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/e/baz/needle create mode 100644 apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/haystack diff --git a/apps/emqx_ft/src/emqx_ft_fs_util.erl b/apps/emqx_ft/src/emqx_ft_fs_util.erl index 75fb47c27..df9135816 100644 --- a/apps/emqx_ft/src/emqx_ft_fs_util.erl +++ b/apps/emqx_ft/src/emqx_ft_fs_util.erl @@ -17,12 +17,14 @@ -module(emqx_ft_fs_util). -include_lib("snabbkaffe/include/trace.hrl"). +-include_lib("kernel/include/file.hrl"). -export([is_filename_safe/1]). -export([escape_filename/1]). -export([unescape_filename/1]). -export([read_decode_file/2]). +-export([read_info/1]). -export([fold/4]). @@ -144,13 +146,20 @@ safe_decode(Content, DecodeFun) -> {error, corrupted} end. +-spec read_info(file:name_all()) -> + {ok, file:file_info()} | {error, file:posix() | badarg}. +read_info(AbsPath) -> + % NOTE + % Be aware that this function is occasionally mocked in `emqx_ft_fs_util_SUITE`. + file:read_link_info(AbsPath, [{time, posix}, raw]). + -spec fold(foldfun(Acc), Acc, _Root :: file:name(), glob()) -> Acc. fold(Fun, Acc, Root, Glob) -> fold(Fun, Acc, [], Root, Glob, []). fold(Fun, AccIn, Path, Root, [Glob | Rest], Stack) when Glob == '*' orelse is_function(Glob) -> - case file:list_dir(filename:join(Root, Path)) of + case list_dir(filename:join(Root, Path)) of {ok, Filenames} -> lists:foldl( fun(FN, Acc) -> @@ -172,7 +181,7 @@ fold(Fun, AccIn, Path, Root, [Glob | Rest], Stack) when Glob == '*' orelse is_fu Fun(Path, {error, Reason}, Stack, AccIn) end; fold(Fun, AccIn, Filepath, Root, [], Stack) -> - case file:read_link_info(filename:join(Root, Filepath), [{time, posix}, raw]) of + case ?MODULE:read_info(filename:join(Root, Filepath)) of {ok, Info} -> Fun(Filepath, Info, Stack, AccIn); {error, Reason} -> @@ -183,3 +192,13 @@ matches_glob('*', _) -> true; matches_glob(FilterFun, Filename) when is_function(FilterFun) -> FilterFun(Filename). + +list_dir(AbsPath) -> + case ?MODULE:read_info(AbsPath) of + {ok, #file_info{type = directory}} -> + file:list_dir(AbsPath); + {ok, #file_info{}} -> + {error, enotdir}; + {error, Reason} -> + {error, Reason} + end. diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl new file mode 100644 index 000000000..81a483651 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl @@ -0,0 +1,159 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_fs_util_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("kernel/include/file.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +t_fold_single_level(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {"a", #file_info{type = directory}, ["a"]}, + {"c", #file_info{type = directory}, ["c"]}, + {"d", #file_info{type = directory}, ["d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*'])) + ). + +t_fold_multi_level(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {"a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]}, + {"a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]}, + {"d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*'])) + ), + ?assertMatch( + [ + {"a/b/foo", #file_info{type = directory}, ["foo", "b", "a"]}, + {"c/bar/中文", #file_info{type = regular}, ["中文", "bar", "c"]}, + {"d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*'])) + ). + +t_fold_no_glob(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [{"", #file_info{type = directory}, []}], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, [])) + ). + +t_fold_glob_too_deep(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*', '*'])) + ). + +t_fold_invalid_root(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [], + sort(emqx_ft_fs_util:fold(fun cons/4, [], filename:join([Root, "a", "link"]), ['*'])) + ), + ?assertMatch( + [], + sort(emqx_ft_fs_util:fold(fun cons/4, [], filename:join([Root, "d", "haystack"]), ['*'])) + ). + +t_fold_filter_unicode(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {"a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]}, + {"d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', fun is_latin1/1])) + ), + ?assertMatch( + [ + {"a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', is_not(fun is_latin1/1)])) + ). + +t_fold_filter_levels(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {"a/b/foo", #file_info{type = directory}, ["foo", "b", "a"]}, + {"d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, [fun is_letter/1, fun is_letter/1, '*'])) + ). + +t_fold_errors(Config) -> + Root = ?config(data_dir, Config), + ok = meck:new(emqx_ft_fs_util, [passthrough]), + ok = meck:expect(emqx_ft_fs_util, read_info, fun(AbsFilepath) -> + ct:pal("read_info(~p)", [AbsFilepath]), + Filename = filename:basename(AbsFilepath), + case Filename of + "b" -> {error, eacces}; + "link" -> {error, enotsup}; + "bar" -> {error, enotdir}; + "needle" -> {error, ebusy}; + _ -> meck:passthrough([AbsFilepath]) + end + end), + ?assertMatch( + [ + {"a/b", {error, eacces}, ["b", "a"]}, + {"a/link", {error, enotsup}, ["link", "a"]}, + {"c/link", {error, enotsup}, ["link", "c"]}, + {"d/e/baz/needle", {error, ebusy}, ["needle", "baz", "e", "d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*'])) + ). + +%% + +is_not(F) -> + fun(X) -> not F(X) end. + +is_latin1(Filename) -> + case unicode:characters_to_binary(Filename, unicode, latin1) of + {error, _, _} -> + false; + _ -> + true + end. + +is_letter(Filename) -> + case Filename of + [_] -> + true; + _ -> + false + end. + +cons(Path, Info, Stack, Acc) -> + [{Path, Info, Stack} | Acc]. + +sort(L) when is_list(L) -> + lists:sort(L). diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/42 b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/42 new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я new file mode 100644 index 000000000..ac31ffd53 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я @@ -0,0 +1 @@ +Ты diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/link b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/link new file mode 120000 index 000000000..1b271d838 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/link @@ -0,0 +1 @@ +../c \ No newline at end of file diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/bar/中文 b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/bar/中文 new file mode 100644 index 000000000..2e11eb72f --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/bar/中文 @@ -0,0 +1 @@ +Zhōngwén diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/link b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/link new file mode 120000 index 000000000..82f488f26 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/link @@ -0,0 +1 @@ +../a \ No newline at end of file diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/e/baz/needle b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/e/baz/needle new file mode 100644 index 000000000..d755762d1 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/e/baz/needle @@ -0,0 +1 @@ +haystack diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/haystack b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/haystack new file mode 100644 index 000000000..a6b681bf4 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/haystack @@ -0,0 +1 @@ +needle From 11edfc1c6a9981176f95a9dfb2efca4beba3ba70 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 29 Mar 2023 14:37:29 +0300 Subject: [PATCH 082/197] fix(ft): comment out some clauses as "unreachable" --- .../src/emqx_ft_storage_exporter_fs_proxy.erl | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl index bdb64f543..943c053ff 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl @@ -29,9 +29,11 @@ list_exports_local() -> emqx_ft_storage:with_storage_type(local, fun(Storage) -> case emqx_ft_storage_exporter:exporter(Storage) of {emqx_ft_storage_exporter_fs, Options} -> - emqx_ft_storage_exporter_fs:list_local(Options); - InvalidExporter -> - {error, {invalid_exporter, InvalidExporter}} + emqx_ft_storage_exporter_fs:list_local(Options) + % NOTE + % This case clause is currently deemed unreachable by dialyzer. + % InvalidExporter -> + % {error, {invalid_exporter, InvalidExporter}} end end). @@ -39,8 +41,10 @@ read_export_file_local(Filepath, CallerPid) -> emqx_ft_storage:with_storage_type(local, fun(Storage) -> case emqx_ft_storage_exporter:exporter(Storage) of {emqx_ft_storage_exporter_fs, Options} -> - emqx_ft_storage_exporter_fs:start_reader(Options, Filepath, CallerPid); - InvalidExporter -> - {error, {invalid_exporter, InvalidExporter}} + emqx_ft_storage_exporter_fs:start_reader(Options, Filepath, CallerPid) + % NOTE + % This case clause is currently deemed unreachable by dialyzer. + % InvalidExporter -> + % {error, {invalid_exporter, InvalidExporter}} end end). From 258fabbf8b3b37fa2643af2ed978f6a0f37ab322 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 29 Mar 2023 14:38:41 +0300 Subject: [PATCH 083/197] fix(fs-exp): reply with error when listing failed everywhere --- .../src/emqx_ft_storage_exporter_fs.erl | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index 7cf0ef6d5..647d84124 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -266,20 +266,32 @@ start_reader(Options, RelFilepath, CallerPid) -> %% -spec list(options()) -> - {ok, [exportinfo(), ...]} | {error, file_error()}. + {ok, [exportinfo(), ...]} | {error, [{node(), _Reason}]}. list(_Options) -> Nodes = mria_mnesia:running_nodes(), - Results = emqx_ft_storage_exporter_fs_proto_v1:list_exports(Nodes), - {GoodResults, BadResults} = lists:partition( + Replies = emqx_ft_storage_exporter_fs_proto_v1:list_exports(Nodes), + {Results, Errors} = lists:foldl( fun - ({_Node, {ok, {ok, _}}}) -> true; - (_) -> false + ({_Node, {ok, {ok, Files}}}, {Acc, Errors}) -> + {Files ++ Acc, Errors}; + ({Node, {ok, {error, _} = Error}}, {Acc, Errors}) -> + {Acc, [{Node, Error} | Errors]}; + ({Node, Error}, {Acc, Errors}) -> + {Acc, [{Node, Error} | Errors]} end, - lists:zip(Nodes, Results) + {[], []}, + lists:zip(Nodes, Replies) ), - length(BadResults) > 0 andalso - ?SLOG(warning, #{msg => "list_remote_exports_failed", failures => BadResults}), - {ok, [File || {_Node, {ok, {ok, Files}}} <- GoodResults, File <- Files]}. + length(Errors) > 0 andalso + ?SLOG(warning, #{msg => "list_remote_exports_failed", errors => Errors}), + case Results of + [_ | _] -> + {ok, Results}; + [] when Errors =:= [] -> + {ok, Results}; + [] -> + {error, Errors} + end. %% From 28d87ca62dab2fdf482de96212c31d9436290b68 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 30 Mar 2023 11:47:02 +0300 Subject: [PATCH 084/197] fix(ft-fs): move default config to the backend impl --- apps/emqx_ft/src/emqx_ft_conf.erl | 11 ----------- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 20 ++++++++++++++------ apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 2 +- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index b0e73cda4..aafcd5ad3 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -22,7 +22,6 @@ %% Accessors -export([storage/0]). --export([segments_root/1]). -export([gc_interval/1]). -export([segments_ttl/1]). @@ -49,16 +48,6 @@ storage() -> emqx_config:get([file_transfer, storage], disabled). --spec segments_root(_Storage) -> file:name(). -segments_root(_Storage) -> - Conf = assert_storage(local), - case emqx_map_lib:deep_find([segments, root], Conf) of - {ok, Root} -> - Root; - {not_found, _, _} -> - filename:join([emqx:data_dir(), file_transfer, segments]) - end. - -spec gc_interval(_Storage) -> milliseconds(). gc_interval(_Storage) -> Conf = assert_storage(local), diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index f0bc9ac67..4e5cc9236 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -41,6 +41,7 @@ % GC API % TODO: This is quickly becomes hairy. +-export([get_root/1]). -export([get_subdir/2]). -export([get_subdir/3]). @@ -220,7 +221,7 @@ transfers(Storage) -> % TODO `Continuation` % There might be millions of transfers on the node, we need a protocol and % storage schema to iterate through them effectively. - ClientIds = try_list_dir(get_segments_root(Storage)), + ClientIds = try_list_dir(get_root(Storage)), {ok, lists:foldl( fun(ClientId, Acc) -> transfers(Storage, ClientId, Acc) end, @@ -229,7 +230,7 @@ transfers(Storage) -> )}. transfers(Storage, ClientId, AccIn) -> - Dirname = filename:join(get_segments_root(Storage), ClientId), + Dirname = filename:join(get_root(Storage), ClientId), case file:list_dir(Dirname) of {ok, FileIds} -> lists:foldl( @@ -263,6 +264,16 @@ read_transferinfo(Storage, Transfer, Acc) -> Acc end. +-spec get_root(storage()) -> + file:name(). +get_root(Storage) -> + case emqx_map_lib:deep_find([segments, root], Storage) of + {ok, Root} -> + Root; + {not_found, _, _} -> + filename:join([emqx:data_dir(), file_transfer, segments]) + end. + -spec get_subdir(storage(), transfer()) -> file:name(). get_subdir(Storage, Transfer) -> @@ -307,7 +318,7 @@ break_segment_filename(Filename) -> mk_filedir(Storage, {ClientId, FileId}, SubDirs) -> filename:join([ - get_segments_root(Storage), + get_root(Storage), emqx_ft_fs_util:escape_filename(ClientId), emqx_ft_fs_util:escape_filename(FileId) | SubDirs @@ -325,9 +336,6 @@ try_list_dir(Dirname) -> {error, _} -> [] end. -get_segments_root(Storage) -> - emqx_ft_conf:segments_root(Storage). - -include_lib("kernel/include/file.hrl"). read_file(Filepath, DecodeFun) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 63b0ab500..2ab30e88b 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -374,4 +374,4 @@ register_gcstat_error(Subject, Error, Stats = #gcstats{errors = Errors}) -> %% get_segments_root(Storage) -> - emqx_ft_conf:segments_root(Storage). + emqx_ft_storage_fs:get_root(Storage). From 31b441a46e1edc6340a200fc11d3e2091fc666b0 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 17 Mar 2023 01:30:31 +0200 Subject: [PATCH 085/197] feat(s3): add S3 client application --- .../docker-compose-minio-tcp.yaml | 21 + .../docker-compose-minio-tls.yaml | 23 + .../docker-compose-toxiproxy.yaml | 16 + .ci/docker-compose-file/toxiproxy.json | 12 + apps/emqx_machine/src/emqx_machine.app.src | 2 +- apps/emqx_machine/src/emqx_machine_boot.erl | 3 +- apps/emqx_s3/BSL.txt | 94 +++ apps/emqx_s3/README.md | 133 +++++ apps/emqx_s3/docker-ct | 2 + apps/emqx_s3/docs/s3_app.png | Bin 0 -> 202227 bytes apps/emqx_s3/rebar.config | 6 + apps/emqx_s3/src/emqx_s3.app.src | 14 + apps/emqx_s3/src/emqx_s3.erl | 65 +++ apps/emqx_s3/src/emqx_s3_app.erl | 16 + apps/emqx_s3/src/emqx_s3_client.erl | 293 ++++++++++ apps/emqx_s3/src/emqx_s3_profile_conf.erl | 390 +++++++++++++ .../src/emqx_s3_profile_http_pool_clients.erl | 35 ++ .../src/emqx_s3_profile_http_pools.erl | 123 ++++ apps/emqx_s3/src/emqx_s3_profile_sup.erl | 48 ++ .../src/emqx_s3_profile_uploader_sup.erl | 73 +++ apps/emqx_s3/src/emqx_s3_schema.erl | 143 +++++ apps/emqx_s3/src/emqx_s3_sup.erl | 47 ++ apps/emqx_s3/src/emqx_s3_uploader.erl | 318 +++++++++++ apps/emqx_s3/test/certs/ca.crt | 29 + apps/emqx_s3/test/emqx_s3_SUITE.erl | 66 +++ apps/emqx_s3/test/emqx_s3_client_SUITE.erl | 104 ++++ .../test/emqx_s3_profile_conf_SUITE.erl | 293 ++++++++++ apps/emqx_s3/test/emqx_s3_schema_SUITE.erl | 154 +++++ apps/emqx_s3/test/emqx_s3_test_helpers.erl | 135 +++++ apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl | 535 ++++++++++++++++++ rebar.config.erl | 3 +- scripts/ct/run.sh | 12 +- 32 files changed, 3204 insertions(+), 4 deletions(-) create mode 100644 .ci/docker-compose-file/docker-compose-minio-tcp.yaml create mode 100644 .ci/docker-compose-file/docker-compose-minio-tls.yaml create mode 100644 apps/emqx_s3/BSL.txt create mode 100644 apps/emqx_s3/README.md create mode 100644 apps/emqx_s3/docker-ct create mode 100644 apps/emqx_s3/docs/s3_app.png create mode 100644 apps/emqx_s3/rebar.config create mode 100644 apps/emqx_s3/src/emqx_s3.app.src create mode 100644 apps/emqx_s3/src/emqx_s3.erl create mode 100644 apps/emqx_s3/src/emqx_s3_app.erl create mode 100644 apps/emqx_s3/src/emqx_s3_client.erl create mode 100644 apps/emqx_s3/src/emqx_s3_profile_conf.erl create mode 100644 apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl create mode 100644 apps/emqx_s3/src/emqx_s3_profile_http_pools.erl create mode 100644 apps/emqx_s3/src/emqx_s3_profile_sup.erl create mode 100644 apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl create mode 100644 apps/emqx_s3/src/emqx_s3_schema.erl create mode 100644 apps/emqx_s3/src/emqx_s3_sup.erl create mode 100644 apps/emqx_s3/src/emqx_s3_uploader.erl create mode 100644 apps/emqx_s3/test/certs/ca.crt create mode 100644 apps/emqx_s3/test/emqx_s3_SUITE.erl create mode 100644 apps/emqx_s3/test/emqx_s3_client_SUITE.erl create mode 100644 apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl create mode 100644 apps/emqx_s3/test/emqx_s3_schema_SUITE.erl create mode 100644 apps/emqx_s3/test/emqx_s3_test_helpers.erl create mode 100644 apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl diff --git a/.ci/docker-compose-file/docker-compose-minio-tcp.yaml b/.ci/docker-compose-file/docker-compose-minio-tcp.yaml new file mode 100644 index 000000000..93e1c4ead --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-minio-tcp.yaml @@ -0,0 +1,21 @@ +version: '3.7' + +services: + minio: + hostname: minio + image: quay.io/minio/minio:RELEASE.2023-03-20T20-16-18Z + command: server --address ":9000" --console-address ":9001" /minio-data + expose: + - "9000" + - "9001" + ports: + - "9000:9000" + - "9001:9001" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 5s + retries: 3 + networks: + emqx_bridge: + diff --git a/.ci/docker-compose-file/docker-compose-minio-tls.yaml b/.ci/docker-compose-file/docker-compose-minio-tls.yaml new file mode 100644 index 000000000..2e7a6bea5 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-minio-tls.yaml @@ -0,0 +1,23 @@ +version: '3.7' + +services: + minio_tls: + hostname: minio-tls + image: quay.io/minio/minio:RELEASE.2023-03-20T20-16-18Z + command: server --certs-dir /etc/certs --address ":9100" --console-address ":9101" /minio-data + volumes: + - ./certs/server.crt:/etc/certs/public.crt + - ./certs/server.key:/etc/certs/private.key + expose: + - "9100" + - "9101" + ports: + - "9100:9100" + - "9101:9101" + healthcheck: + test: ["CMD", "curl", "-k", "-f", "https://localhost:9100/minio/health/live"] + interval: 30s + timeout: 5s + retries: 3 + networks: + emqx_bridge: diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index 9a1d08ba6..0cf689921 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -13,18 +13,34 @@ services: volumes: - "./toxiproxy.json:/config/toxiproxy.json" ports: + # Toxiproxy management API - 8474:8474 + # InfluxDB - 8086:8086 + # InfluxDB TLS - 8087:8087 + # MySQL - 13306:3306 + # MySQL TLS - 13307:3307 + # PostgreSQL - 15432:5432 + # PostgreSQL TLS - 15433:5433 + # TDEngine - 16041:6041 + # DynamoDB - 18000:8000 + # RocketMQ - 19876:9876 + # Cassandra - 19042:9042 + # Cassandra TLS - 19142:9142 + # S3 + - 19000:19000 + # S3 TLS + - 19100:19100 command: - "-host=0.0.0.0" - "-config=/config/toxiproxy.json" diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index 708cbf1ef..d8dd8a166 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -95,5 +95,17 @@ "listen": "0.0.0.0:9142", "upstream": "cassandra:9142", "enabled": true + }, + { + "name": "minio_tcp", + "listen": "0.0.0.0:19000", + "upstream": "minio:9000", + "enabled": true + }, + { + "name": "minio_tls", + "listen": "0.0.0.0:19100", + "upstream": "minio-tls:9100", + "enabled": true } ] diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 0bee30e35..6bd36aab5 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index 82b3d602f..feeb1ba75 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -146,7 +146,8 @@ basic_reboot_apps() -> emqx_authz, emqx_slow_subs, emqx_auto_subscribe, - emqx_plugins + emqx_plugins, + emqx_s3 ], case emqx_release:edition() of ce -> CE; diff --git a/apps/emqx_s3/BSL.txt b/apps/emqx_s3/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_s3/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_s3/README.md b/apps/emqx_s3/README.md new file mode 100644 index 000000000..3f049d1a8 --- /dev/null +++ b/apps/emqx_s3/README.md @@ -0,0 +1,133 @@ +# emqx_s3 + +EMQX S3 Application + +## Description + +This application provides functionality for uploading files to S3. + +## Usage + +The steps to integrate this application are: +* Integrate S3 configuration schema where needed. +* On _client_ application start: + * Call `emqx_s3:start_profile(ProfileName, ProfileConfig)` with configuration. + * Add `emqx_config_handler` hook to call `emqx_s3:start_profile(ProfileName, ProfileConfig)` when configuration is updated. +* On _client_ application stop, call `emqx_s3:stop_profile(ProfileName)`. + +`ProfileName` is a unique name used to distinguish different sets of S3 settings. Each profile has its own connection pool and configuration. + +To use S3 from a _client_ application: +* Create an uploader process with `{ok, Pid} = emqx_s3:start_uploader(ProfileName, #{key => MyKey})`. +* Write data with `emqx_s3_uploader:write(Pid, <<"data">>)`. +* Finish the uploader with `emqx_s3_uploader:complete(Pid)` or `emqx_s3_uploader:abort(Pid)`. + +### Configuration + +Example of integrating S3 configuration schema into a _client_ application `emqx_someapp`. + +```erlang +-module(emqx_someapp_schema). + +... + +roots() -> [someapp] +... + +fields(someapp) -> + [ + {other_setting, ...}, + {s3_settings, + mk( + hoconsc:ref(emqx_s3_schema, s3), + #{ + desc => ?DESC("s3_settings"), + required => true + } + )} + ]; +... + +``` + +### Application start and config hooks + +```erlang +-module(emqx_someapp_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +-export([ + pre_config_update/3, + post_config_update/5 +]). + +start(_StartType, _StartArgs) -> + ProfileConfig = emqx_config:get([someapp, s3_settings]), + ProfileName = someapp, + ok = emqx_s3:start_profile(ProfileName, ProfileConfig), + ok = emqx_config_handler:add_handler([someapp], ?MODULE). + +stop(_State) -> + ok = emqx_conf:remove_handler([someapp]), + ProfileName = someapp, + ok = emqx_s3:stop_profile(ProfileName). + +pre_config_update(_Path, NewConfig, _OldConfig) -> + {ok, NewConfig}. + +post_config_update(Path, _Req, NewConfig, _OldConfig, _AppEnvs) -> + NewProfileConfig = maps:get(s3_settings, NewConfig), + ProfileName = someapp, + %% more graceful error handling may be needed + ok = emqx_s3:update_profile(ProfileName, NewProfileConfig). + +``` + +### Uploader usage + +```erlang +-module(emqx_someapp_logic). +... + +-spec do_upload_data(Key :: string(), Data :: binary()) -> ok. +do_upload_data(Key, Data) -> + ProfileName = someapp, + {ok, Pid} = emqx_s3:start_uploader(ProfileName, #{key => Key}), + ok = emqx_s3_uploader:write(Pid, Data), + ok = emqx_s3_uploader:complete(Pid). + +``` + +## Design + +![Design](./docs/s3_app.png) + +* Each profile has its own supervisor `emqx_s3_profile_sup`. +* Under each profile supervisor, there is a + * `emqx_s3_profile_uploader_sup` supervisor for uploader processes. + * `emqx_s3_profile_conf` server for managing profile configuration. + +When an uploader process is started, it checkouts the actual S3 configuration for the profile from the `emqx_s3_profile_conf` server. It uses the obtained configuration and connection pool to upload data to S3 till the termination, even if the configuration is updated. + +`emqx_s3_profile_conf`: +* Keeps actual S3 configuration for the profile and creates a connection pool for the actual configuration. +* Creates a new connection pool when the configuration is updated. +* Keeps track of uploaders using connection pools. +* Drops connection pools when no uploaders are using it or after a timeout. + +The code is designed to allow a painless transition from `ehttpc` pool to any other HTTP pool/client. + +## Possible performance improvements + +One of the downsides of the current implementation is that there is a lot of message passing between the uploader client and the actual sockets. + +A possible improvement could be: +* Use a process-less HTTP client, like [Mint](https://github.com/elixir-mint/mint). +* Use a resource pool, like [NimblePool](https://github.com/dashbitco/nimble_pool) to manage the HTTP connections. It temporarily grants sockets to its clients. +* Do the buffering logic locally in the uploader client. +* Use `emqx_s3_client` directly from the uploader client. + +In this case, the data will be directly sent to the socket, without being sent to any intermediate processes. diff --git a/apps/emqx_s3/docker-ct b/apps/emqx_s3/docker-ct new file mode 100644 index 000000000..a5a001815 --- /dev/null +++ b/apps/emqx_s3/docker-ct @@ -0,0 +1,2 @@ +minio +toxiproxy diff --git a/apps/emqx_s3/docs/s3_app.png b/apps/emqx_s3/docs/s3_app.png new file mode 100644 index 0000000000000000000000000000000000000000..372f16d371d0b75b7d796736906eb1de004d1deb GIT binary patch literal 202227 zcmeFZXH-+&w>OFi(nOkopi-2k^xk_7q4zE|5K0Iw^sa&;#i*ck1px(VQluBfhJXg7 zN>xOp_YQXjlIQ=NbKX0~JMM@3=?sS<+1Y!oHGi|uHCOH!=xI`tGm{e#5Kuz2)QkuS zNUsnO5U-M+0asdMhN!@wQ+`I8Dg>q7muA7YM*iv+{$4>axTgyNr>N@5J5CWHH*Y_G zPEj>Z5fKL@QotGR;Ogt(!4a2VXh`Q#oEArVo2AyIx2NfTjdPElnc zA@CwBE+8!?WOZ`CgR2VyH=wwH5Ey_**TD&n@b~s};}lf^pCJf;7f_2eP*a7C?3;)*;SbALi9b8Xn(K9p%4${%o3h|J1zv1O8 z>SAhOazX>j#n%t+jlhwN9T3cqeb_%3>2h+_)!WM%&8{KDJ63JDulhFa`LKn{9fw#x&-bz0O4dBI1X(qKP@p)BTao1^B_%6RaHGv zEq}|CDJ9KN2A<-6J{kt%zJ3A0PRa(7P(MEr4@noOv{aBoh>O06k6W;trg^ZAxtOX6 z(gKXEqN}N?>#8dx;j8Yh8v+kOxGB2=MyO~3gkbJYe(Ks@KGLFy5U7(83S16$)l=3+ zC_6~1I{Ls2)!-pIFimi$u6~e?YoLf1(p=Ll$W#NV>}%l$fr)D%bu@fD{hU-pObi{I z{4BlQrS-jp%$>Cj5g{&q`s#j`(*9n4;%Y|XK45Gw7j;w9Krwv-A9G)6gFtaHGYxNP z2|Y)VkYFPhBWF{TvWT;}uuo8su$e1%DaMYf%FY2UUV18U*8oc^OUV!^e?3V(Rb!Dr zA5mo_xJ6V%%2dnG$z0viSk%Ws&A?IG00s$=3XqmU`6IPN)x}g`a4E2Sl$M%@2+Bvw z#Y@=)h=z)}xsjDg5F9FQ}{LcHCL{LQr`#l@8o9^yJ7Uiun-M&M2l zL+ofmaA7AEEmdI;LtPV;iH@qSsT9mBSYKHRr5)hzBLd*m@G#X1Hult!P_xh#LZKub z4Sc=4j4aep5F;VUARtUYEkdM4MAT4TqQPdGfQu^Pz81<3ZYT|eTCfNB2vJit^KcXO zkkSoAB24t5x_FcUmx9?K-m|f;_4u3CTK+13pR9w8V5_5 zn3<^uYnbT)-BCe$d1;uc2m2U02L||>IEE+-`DytC8k+?pG{s&0oz+4DlywYDE!3rT zJWf(p6^Mjb z1_2UGq$RcCT81zcLnA$54K*uaKUaT!M;(|p+}zw3p{nZb{ryD%(pGM+BF^3hfC1*tC_`rmOvS@fKR8%f z+eO_$OTpedqa`N{^nvq!Y)oQuzr}Xwh2VbU)cxgZ>Z_x z;%F>|3{rN}azW}_7@4cMqf|ZJ4TQzi1J$6KC^chKHDJtD5SCgFLTXl0zU~s9YUWU2 ztu>59)DXJC`tC@L5LY9Fr;nxu+)_GFTTMs{A`M4~>$+HJS*od_y!E^df|Yf2L_9+f znue|-mfjMou7S>uAzogt!VWI3l71l;8m5|hU_^6CEeCyNVLu5q4?{04LuaHJ(%-_< zM;sL*Ze#>E)fX{$u+l`Rn+1e8s<|1NhUmFyYeJ-?4Pg$ZzD6iVD-)QHyEaVQR2eB2 zV&vhV07y*O9~r{1h@tp>OmagBEG5`rqWo> zqO?W)AVQAjo@!QFMt;)f7Lt0RlB%9kJ`PZ6X{dyzi-(1S4^T%HlR)f(U0p=g9Ka`) z5N8R0QJ8M9G0IdA;pyq33A4aLqyzQPH}KX#X#;ef%*BM&v;rJ0#WZw~N=3}V$y3A4Kwr`{zzPNNbagjSg@XIke4M>uCMbjuGDOT=!pSfQ=IbfuVc;BW zgmQ2&PjWU>-e`kX!wtThKju(E7 zI+vQA7?MkNLIzt(MD!_DjY}hsz2&Z@bq198caN98O0|8Kmuit8E5qMsM^WQl9Q->Q zDy?AqT=7YjkT~H^Ir>wXm79%m*gcUHaqK)aXb z1g~74Q@gC{!WDFcru4Z-hNgOYw~vQcdO&cCw2=vqYBU4UxfbNM9p%msN~30n$#V-; zZRwpxQLPQc(L}_aO3OPHbQ=qOvu2532OoDXg${iSN9dK4*^r;`gos*82QbaC&xSM_ zM*Jso`8{s-dcyNpOa>mj#tBQ*RYJ{kOc6!-k(N@U^?U^ia{(?x#G1a)3q!Q!v&L~P z>)G#hx8>Zo9aKp+>*TBrzgppWXx|M$3yNb)IR^-u{doG-@eD}SC>0{fRjukH{LkMCKW`F&h>EO@qpNJd-5V?*|PvLlxYizyGm=t8U6DDY# z^wc}K^pj*1@0F<5OT@Pm&!xdXPHRxg*KeIc{-bP`47o@5r{}++Sc(2bZl}07s$tcl z=n>#%HF~-cKdkhzBxjf(vQh-P6*kq(mR`zvD3YeiJl`DkYA9swZP}lZ7rl>+7vDh7 zY_iNp^r^+;q)veR>k(48_}#B`peh3*+H%K3Ldb<0v-2eeR8EvIx{v%Uvs}$VGUa$! zdgru^XYHq#RuM1sUU>0Rs0u?FSM2eEXq|r==tb3fo=%u^&45*RD6N+y? z%;tSEeoWuiwN6Wa!oh9|ux5o0qCbzpnm^ELOC7KI4yi$W3aKcgGH%UP6qlr(x1}J( z!WUJBMb-DHA77&$eZ&FZ{~B#VwbFbdu>{ecKq)_(HM{X+<&Emtz_A(B1AL!n_z>YF zMvCj$;z>&UvdeCm~eFRfN@!X8C%U3?nx1Y1HZY23)Y=MR9?TJkxqrqwOC2F7tO0;BcP$0y11zEYr`fweX z%cq2L=hShASd!@3hUL~^$Yp*^MbkxP8e&2g(lBwCbVRuGJ$0FgKdlO*a->hFafYb% z6d3Cq$9?L?^I&k7VKQdyWVwWZ`0K4d4RC^iRGo;YxP2fA%&(Mlnw`6Bj}3xI{XA(9 zo6ygh8vz}*c@KML3hhF4#&7+98T%vq)rbeHl&3cbEucoSV9UFZ5vbi&9eQNZL(2xoK<58(RSd^(Y1jTypfjP8eMyoB8)I2~Q; zjFt*JZ_oDiLkXSe$*4nEtFf^EEu;k)=k2V>XPQ_Z^5{mX+{82vUc|E<`%qgxEn1TB zEK$eqN=bIucR?sLIUB|PWUqoS3}MIe_H$c3o?IQ8E- zz6+=m7DUvcThlloV82`O+p&~BlG4ItK8HnM8W z8xRmNlk?}8f7*0FjKu^uHE?1~0v_@gdU5ktBEW7eKV9iWV38V8hjF4WIH;+>C_#_? zRgUlHu?CP;&Pq|^AhV$XkUi3&jY|fQ?LH20JQ17j3TEa!+ru8KY(n={}SF% zmy!a=w7KrQ)dY}*e~}SB88yWV5IfY(`fAG>5HP;O9fD<0G=_*rji)Q239pUH;GxSp z*V%)w0_-GTaN%S$lLoL@=Q`OtRWJ$*S5^v687|`nzKcFQd8Op>tf1QW~+?Wth49KYV znxi*fMh(D2BoDN;#<1Awz9c(;!bVr%OQjxY6RsEl$l@T}&c~2d5(CIGl1}3xqX&@L ze&X=i#%gG?B{%G3)c;qt1|H*o)%t(c`hV5>|0lJ!dY%%oeJES)q1eLA%>2?{L(kX? zsBAI+=N(*VmD>*tsy1EoPJQS4_p}>DGiNeh4!jSC%06zQDXIj<>w2uO%&}G%0^LW- zG!ez&{8I-o7#8=?ihK8hRvqM#J+DIGeWJTztNCopF6I`7vWt~zn~RY*8RpAV6fZEv zKa6fY%W;PbXS7=Ru+xUBdcOX|oW8u6A+!3}*uSr$kRf0>Sy6AG$n`eyUg?Nmo~6QD z{lj^>CUJzPf#;9V*IsvJSY_F%L9jT{bnFhJlvaQ(J@4I-jkl#mz?LR))!Zv*sb4}} z4*z4v&r&xy1)Ytabn0_>daF}PHm*-4D!SFG$X5?%1dE;nLNWfG*VML`4+5*EZx<6Y z)~`*@RR5NdPZtOocr`FvrpbDK|8Q}pCAI&;=B6Dmid){N2#!J2#s+~ze5ikdK&sh!8iD{$WJe4Gj_N-r==+T!4LT7 z+8doyHovwKBtwTryc@L(FNInx567D;e5Hj7^j3{~w;U84?0G$9%!#y;;BEhC&TaN; zHZd`Acek@gd?{>!)^c+)qpYP*<+AJx-3$rSgs0iuj%?-GyEgbB>N5@U#gU=j{)*Gx z*%mSLc>C~&FN_P_CiN1;MsB)y#~b{eah(gaID+3-XXLHBVRn!EAkOijdo9&L^6-%F ziBH)A&KfCLsCE-4eVpXZ;cQ>~6)%SzK4F&LH*8w_eNlORZhcVc;?|hT0o6q*YoE3o z=f+f*^nvTlQ>}4uYM$@do@m; zIn!^pyVGOja$xU`Bt>}(eeve-gYS&HADBMOJ(*1IJo7iz)?a2J?olo+aU{BLjPn>ET2BNlYkobYO(AN7@;)(e3x*zTQX-;ep$=w_1jWT z*m`8c2;0OoV^*OoqM0kcr)G0aFn3X2q7UL~mpg4{@d7^c^UOT!m#}i59De_cHys(eH{RMy8;-og zc%|ffc=%}eTmWE5#l%`uc*I!I)^C~V`GoYlA<2JQ62@4LLV9QH<~~$;jN27v%bFDC z1m!m$t|sL6IF4u+m$ICv_;K)uf<8Q=raiv%FgR(kamOlLA=t?HqrBvxiDnKG_EoXi zKkvxkg?&I$;gR8G&F&biKupAe@HgVu6CPY(Vt?lioa#S)S5&3FS zK47S2>Dv;+PI&L!H>GUKdi7frw@6t1-&NHA#k6&0_ueJZlRMZi7*Ke;T(h}U`b1qaxr9$S?!}1>;`C?aZ$?(eZqa9o)aNrXzf+|8ucR}XDI0o zIekgW&qxiYiz&_ z|A700X6xC%)+}f3TBgt~b1|>=$s=kkFHN0(wnFzKsK>bS6s81mmIx;9SA|o>NaD&L zM@3>DkJfMeUVqZRD?c#4p=NONATFC%f}Z`|NN7jm(Kq(J`LCX&OA&`2LpeFG#vNwD zcRF0Hf7ShbI^*oc7YNWVwb@9{J2DJ6?mmvZCrSbOi@#Gp;(dx%bjdlQ;fI+AHA8Gz z0z^rUpy3kZX1xXBWe&;m7rv<3TTiA97_!GS!EOKD6nuS>x z+cs=6%fhOX6kkRPNx?fVP@N48NE+Cl+*?}DX+U6RxmoDv16!YJ&b{kR7sT+?RFEtk zXeI`&PPRt=5f6MhVm0a#XgKuc{0A0kzuWPzyIX7;77u17X0$12Vo~i@Oycq4x$NAn8I3=W-)@nS zmp7EWdC9`E6Ebvjo%M4;@ox0G;!C~HWXH;M36$sBCV$Z~<1)+h+$rCqTes78P>hYE zl&s^3`c0#(m4PFQ!}X7%-fp)_i-{CJP|z%fZ3n-q(@QRRTSCpw{PFelDNlHu+iTg? z1qz>$>Kv4VJ-Yql4IE7QCTqazJ_8Dq`d8 z%2=CohS#UJD+)7tGafxf^D~!NhrbGlePQ-<90Fm_NFh-y(RYD;{04sd$~ZNaJX_ z|40s)7fDa|yWs^jJ^eM8!+!ZN`>#s3^{_S{CDr^uR!_{dEA-gb+7-62gc~fUcip7+ zT2Z;_sHiwn;*?R+Vx}IL<(tfCxa74r+m4Orw$+2_e;ZWZ?w#@OQIY?VrN1z3Pxs8Nb|4n1s}?BD$EI zJe8XPp@@9zLs3VDB51dw>SnoD-gIBKDIASEC4>iM&nq7?QB+lIBj^X%sCcR16oRZG8~A1@xERsQiibdtW7 zs8dz#bZQH51rdkAw|yxo+xjYNOs(RVCzc1Qrml~|;IT|4jql{z#>vx0ig@1Kq=K;t zIUoh_!6-e2bM=uSeB#mO4J)exKC8P9Y4Yd3N;BHgi@nAi!>kyzZVyNUG1O1itk=K#S&qKl*7FQ_(7U80ec`HD+}{@TRR^1e8b#YZadTg*s`uv z?)U=LtAKMxgh zy_)c8cXh2NCWfjZwo^j3$InUeg_{{s1Mla>q`MNlTII4uo)eWIzOP0Ws>9%B&u@`B zDejlPC}NM8%Br1L{8ooohW$I~{Ob5bys({ivxPa>sDHdVK7nJAogS4Qh%8mS? zi(h3e#=2mFdmlKD&e^@JqJBEq)3d@vl^wp|#9u#1`$$AQRPG=yy*xF*XqVNc!iO4b zL!t^nz#fzy)Q+n*MIVtl`>-w7-iZ8u9ol8>&s;kWr?N#3CAJ+JvbRra=H$0rQuOoc zb4wtnneW^*=hk^LUHzM!#*^77Bgwz#fQmAe&kbSiQ}N_A2M8`xdSS9%b#dQ62C%Shd=0D_h-ms^(&8-*}R&+@R$3cRYVvVW`R> z($Y8Ea<2AMhc2K%aJ61 zvq#FP?mnW}(|?rSkz9CWpxgDiTm2|cL^0_Yn`La2-?FM(h12n`_mmh&2p1^!A2H;} z#>1YQnh!+BT^9ZDL*Vs1xs_{^U7G?x?EZ($XL(6kb$0NoM^^FFqu;B#Sodag%*SM` zFE(fF-gwo3S&C_qG>=Du5aMrsG)a%|j|2V0Z=Z`_8CGw-q#SPAd0))DF<)-KdE522 z;+Q|;qEJ}SuQ~VLuNrT2f_SzYx)#d~KIpx+JxaR~(eDoJvAOQBbg22qd&b7Z=c4PE zZAe+;cBY2BtWD3kNS&YKLDjRFW;{MUKl2`SPFwM-ZTJ43BmN)(@_^U;gCZBZyduZ^ zWD_rKx(PwwQ3lEz`p9iaojM)8VK9@9H*R;781`5nz4Q3c|MJLjrWD1laxEuEUwTt$ zG~(x%X}0&_a=L3@dTWecO0(!6JuPTd&&oX1G}M_t$rhAA{h!dA2$$?F+p!g&j{cU` zgZj|j>!fhDxm@7KSa@h-#KANpfAcS+zi*bm#6-C>Tp6q+uW4+rRe{a&iXEsvP(X(w3jSy#bTIL?mzQm9fh#t^bTM@D81LQZTi|gW4+ze}6}l7`Ut{%fI!Qmz<)&6Fn&@AOX4P zH9_u^TvP{0=|4M6VAHl9>xsO$PGuOj){w-t6wsT`NB^UzjTP9xxkO zxU_H6r9yPk|8<+>2utK4KYv5mfaNf+@f+*g*g#@%$)+%`>4$nUKgmsyV{({?5Bv%Dyl>OALHG< zEPE`^svG>zM(9+^KF0R~jBoXvj&Y$OuN4+d#4L_@%dKTH+B}U2y=-B&r;*J3qUoql zQfjke=DkN9y;=IS+{zp_F=?wLQx%jlW~9olyZ7GuL4;989ZVq5VdVAIY=fiG+H(U~ zP$471^Og~~r-`_wmOOq*8-CI^Tds} zNt}Xa%B%dzSw#KTHU1xPnjv3OeYvolGyA~<1vlS;F9Xl1L)8YqP=aVa7akn*dy(fl zXM?PFb@N!z!nFPDT@OUCR)Ro{TJL0k&DGr{+Wm-wtmIF$!0!I}ZPV8aGP zgTwL~h|~+_5Rp}UD!WuJ72oIC0DtnP-yQSk968_**v28l^M~%^s*?S_Uhu{3O69Vg zh)kBVA?hR5wersrL%heNr#U#g~6=|8GK1#%3*JPu=qMg~|`7T&l0hd^{uNmD?*} zo$8pi5%LKJzpydC%vD>Y^mel%Cc3Nt#*10;gby`E1i)NmlG8l%=xS+=QI6KGI@nA4 zP}Rm2S;Uha!tzDQUVfua0_OuCUQL-k)RbO%pZo%K)X_a^17e);2i0j_yO+Fka=e=Z z-W+e#jgk3=^5RESGoW?BWgPXJ#s#G)RcQe}jc*)_9Ga>1e4Ls$L46 z;>1=ahQj`@(w2?r*1e@>^+J}&cb+%2N6r#|$o4JW^=w|?qux|+p}b3ZVZC;OX8bUG zT3u$w`vXy|OPStdkxq{syL2ok%VScyX2(e}%fJ7YmGifb zbi0tFrdqK-%i$R?mv~mETQm~JDvd>XKTrqTKmBdjlY7%)3OsT>G^h3ZX+R)U_dB~` z_IcnqbK$|R7tMK2wG5`Cai2o?dws>vscq#?ggu`Sk35Su@d!M6o7BJKdJvGh6;JtQ znD+e0;v4Hahx{q2Gla(X&oD>ht7xRy0**>i*P>O|VS;UIYFOyXPGNN~VsExFp%h;K zJ8|IbcTlc)KD#B$5~Al3u*9^_l0(@Q5xKQ?gLik|^C12d$)W_btt=~WH2dOgnd>>j zTJioD@rj43>S3nxj}J9XrKOpU!@i#&?32p2Z&$_j9`qC2Tp+EAez7&i!g@9gTQX_$ zBpmIZFSQYE?D0{-74iJQNP-S&Cd$$!ps>4tbcy^d$2Xu zkLlzqG_8O2VrRIzOyfd;Jjvhj7KEbXQZJ0@=Sk29SpW6pdL9^#@^>?K@P=EjYsCdDr`fy+8=xNOk^zZz- zA{zZ$=9n4*&?qvY!f@vl&T3u9mZq;Xn^BeKQ@1T{5BXAizvQp2!?hnaagS)2tS!qUB&XaHeX@v{ECIpiUqlFlH1 zhOie+HjJ0&4p1cLF}pr=5Vo`b1O0Mog>er73RNZN_fCo|Y=-fz4Y*BiqDDqQQ;m0Y z5%nYx_HTL(3+5vUKAT+8#R8X0+FY}Xp34fB0Rb4WWI98brOO|EGR8y+Shs&ce_Bru z|Lh)1WW|#9E^CK&1?$N=&tF+)!S&6YB6>^{y+uIC#9+@R^!8KU?1`WRT~}g{DZIn& z$&KA(y{{2-e7lxb6cY{Yf2s>9c3`8XH{+t!Z6yytc@5XQIh>rONPvxKcqi$Ca&8cK z*I`}3{XeH(!tVPY@)LlTxC^9TkgIMoC--Dyy9UHoZZ6yeTgu!RE$ZVz$v_a1LM@ht z?+aQ7y$6rSetZ=nvZ4bZiL=`shk9Vsyb5f+XjyBxQ4w~4E(V^lyYeS;nM;kO6i1*S z*bT;RkBIQKD%szDpo#0eTQ&jPCjTeGRc_wQt^23-%p7datpC29%KkMSY~{QnTfr@z zG+!6&Xb+31gi`Z>4H(T8vw2){3flHaaB)>@D1fagl7b+`0lX2-luJ)KUKYqYLF0g~ zoP#WAK6`sdjP|&1Nl675B@Uu47kpIK1fVXEl&m@%-4A+AJayv-@M`A;p2wup499{E zr**}bH3c!DpfmKtNo11I~jo{tXt3kSxDamQ@tzc16TL89@ zBL)Jtx8My-Dp>6D$vyhmMiG$eqf`Lr>EIls*g>Ojg1sCq()7-gMzScS5FnyD-Uv4) z!dGNM)Pw2NWw>q-rD32U`TSR_NTR2}NdMX8Gl*{u{kIx_-K!-a_@8Qa^tSgMT@4-J zSEklg<>To%I6%vY%2%5+INUJ20O}Aw)lO1&R)uo8SsQc@7K^_>$%`zC_cXvBl8 z)LtW;(OR|y_NHfudy@~_YUjJ@b{s?~$rD|0FN>2Iz;`p6kX`a=D?j?}TX!yQ9s~87 zdBSVzl_yNf_O-J0UQXn9`oKgZg`Av_S>bTT_J8`iKgIw%$me`V;)+lMu&rbCC&u7^ z_V3|(YZoduWUTU?LP@T2a1i}CzwQ26Y;^bI1quY{vJJ1VW8yT#?Epo2083@l$+NP3 z1^B2m9lB$`u2c=oWl<`}4Bk0^A^&gftLZ~hZc>EJWQ_-tX(3^KvY#P=wj$;FisMt< zb~r#j7JxN-{!TOleRQ#SNBuv0?@r&oCnZs~uUH>osS_URA0VWnqSC4x?T|O#`-Uus zxA2D#wsU{5i{}m=4IAK^CapB?d|(8wt=TzU>eyetnnRij7ZEs{O%$!g1$cXY zY?6y?sQKICl>Ftr@>`l^-HG8njk&m4(Y~*bZ){B|hu>J?kC?)BQbc=#VC5rh%uO8N zf_FBcIrP7FkzJ>L)muNxRqVd=Tt=Fi(}8^=sg>kG^S9Wm<5f}bsV)x|X2{;BA;Ya? z8`~iq$48e3#bW)LtK!5fVApGevB4KnjQFW_lZS&>l7^#~#Ei3end73!_FcE7!o`Bg zJ4KdJ_{L}*>;j#K-Ksu-(d6_Lp8g-(5(UWqOJ}$cZw&MaCj%q#vxfVt;6TItg)mFB#1>^YsWp7Z^aV!Y=(L-ecx&9L}YFVw5o zR$Aej3!8`7H_7NQyZW6;^L21I;mhp_>(x7qC(iBzHr`A%X!kJ!V?Ka$9w!*U8(*-~ z|CcwGAWCXPzFYe|6B;mA`FMBpPkN$8uQrh!?FFzEk(jYu_pmdOImCa7QjR>!02dv# zDg*9g8yGWbfjF4RkR7{MHV3S~J2T^p^Oh-y=)7*${-%nj4pk4m|5R(Zz8dg);xGBQ zpMBzfEA*#Af6;&<`k{xXo9DdGN^Wym(0PTY_%6p-fbb>Iohgm$1SQ2XR4EIq8+3AM zc!^mjzZ5aK+{CASS4glt#hAtdz>w|?{V)~+U3z1SPM}+zQ4u>8hmYf}6b7(r%)gR~7qr0oEUR)JbTYeiimh9g75j$yYMUmx-D4%J;hI}cYoERV zyB#(Nv{_*5YREHu7xWDP_cj_nr30))a+PcR=CPVPL!u_v`CJle@R`^%LeV9i&!hI8 zVthZIW=UB1S@O4-6snE1c2F-{PQ*gS|KJXqwE3P*tlMw?a&ux$HfHbyc_g-@b!3OL zejOM(t;i%d_G4IT04ygB`zO$I@KKix4Hrz(POml1W3m{RioQ1p=qanT2T%A$kPQ+{ zTwK!a*L+a;<*Z!V#qZ13Kcf;zHo`ju$)4exn#Qrjr20(h#sXr*oTDXg^QxW(eiOC}%Oaml2QDewXv>=~$&SO`EwX zKrvWVsCcwm+?*kyI=&y6uCV9+k}E<&T~Q~8=4KXXYnS-N+}-SKztkomOMMZi!fXbh z-#d1Ti10f$7{)gtM73rDq9+dUsUYe3YeH_zgPWqmI?{4w`w?g}-c=!#Droy9en0cD zp?SgkvY_nIpm^b<2q{gRJ`oTOfg$f7T%uKm0eYReOvR4dVcDP{O>n+uT}6+J8jY^n z8yEckKOFAT(!B~NC49m7QT+E>;`oej9YlJT|C*%Cz@XGU&I?94Qq1^XWil+oE@XB} zUk3*1+dJCh%_1dj&}63tcXY#V9kCf)jZWl{zY34ZFBPxXNY@*&#Hc~WwRUaOnKi6& zSqnB2URG*Nt=TK%unzUGqnfIwR(wzuB$Y6h!SZhZSw0K8=Kgv-$qMZi9E9hu0eg4f zVE*Lb6UaZLu9O$%XS_Xi;tvvtyqM&7fsD z;{H7|jWZyvPZJYwWy_&HbNTu#DMS-e&!N8X>>5tv!2vFEn!o=#z(vZzhm6Vh!%Z|| zfzg8Gad=IHl)uC8qwLniWk9{Z$K^ z7^?6u=fCxyKao5PwiCR5O?3_rNwhnK4g0Kp@}k+09p(Vd^3Yikf8y#m-BR);EQW^I zVi?g3(8bDtCZoQYO2I#Z#U4;}-J7netrX4Ii=26%O;5%{DuJk<1g%Aiqbo{@j8I=| z_)Fe~>{hCOUVbE0CYOl&pANudTm6|hL_0(o!vTSsN7U zL%MDu%r29AAgAuAF{TT> z_^`@@ll$}6iZpE*8_dw8CpDA1j4s|1Taj&x5ZLl6b|>J{TT*Ap1P8?RR;*)LbCePAZOLDFD|>%Fi9_OFQ&DT_wHD0P!uQHj{M~%hUhO@~ zM)#RxayUlKxB5AHqj&}+F1}0sWP|?4tEXd)fFH{3KL$@ex|s7a&nzJ%_GRF3U(T#J z#6#Q40JHVNo1YTzZorhS{L#G}S=VQm(0WMTDrpVBd--dewgR(4`crmlm>|ezY1|O=dVk<4)-SoJK zfgn1cI4MQU1S15A)Y_d2XdLTCFFI0}Br#23{v!^}6mq<9RY-$&eXr;}s`e{HId|J+ zA(hXW`pGEtO9*aJ;+PH!EYxiv*T?TR1YIq^o6pV3&km1-4206{e5rO@KwPrE+n$_#lJlN0Y{cGV9fvlZ{r04xE9$yY#YTG-owgPVd*!{Nb;bn0L#L z@*GVM`)(QgUY<*^%IUjn_!|nza65w|j01Rcm6*YQonSSvheDEX59~eQ$Y5$K^wNh& zVi_HxK#V_dcg)>vHWFP2@fpd;4y4&lVoooCS%^M6aNq_P_M*|H z8-3B`CpxsO0i4o*j!3ahL@QMt{t2ASlAsEq_VZ(Bp+4t6w0hn;$X#jhUy{*hJwX9U zxD9>ZWVZCaZ)bM~B4dYqO?{T^lfeCYw+F>@hdI^^0t`q z_QTr5jgc(9LB~UQDPVZ+e83=h&_E9ZtE7sxT?p_NVOc zrzd?Wm!k0n?jfw!X9VYc$4TQvp;B@=frB+f@wQ^l5_gBEVJ)Srs)Zru6T167@xSaQ z(cde`%>0(P&JPZM{^MAky23oez2aOZ`>Z;#8H}c%i|K;>~Q(hV(k!Kio}A%f9Z-8 z&+_AZ{`Q8EYnce?wjX1rYCH%+FV=X<0pio*qE6311 zmn_ck%rH8U0QC$qhqUqzsP-seR$Pp#lzRzG``Pg4 z1w5bFn4(*5sgruO4^eqC5M@wCdznQmLhG;F#G#BKy{GV{2JG$8^`!IcvkZ_jh*g6C z|Hmf#N}n!93Bo8tZ|M~) zZGvfOhh{Qu9=V5h%x5=2OW4|AqvZ>nkYue86HL+kZH>+ol>hq?m=v$*c0!_t;-A_O zDwu3Zdd(p_WJIf81x-2juixbWx)M%QbGC&vn;mWl8UJ3$^rY$Ud$h*wSG9uQsUEBw z!t!2(zhy*MCFeJBY}~U&=xi&5@%^^k>qipfV0{dZ2osIre_bI;>42C=iARux8(P_L zi%WZ04Z}U1bvq9)Z&)eLB`18#v2ou&EC0|`Dkr(nLK&lr?oV3>#)ISFgJwLflbt!n z3|(>omN!@-p&HkPwpS<9e-xyL!tc6~gNU$*I^`|i#zm78fk|1t6Oclmy6t2A%+n&T zC;clBB#0&_o=>pPU>UCo;Apa%+zC?%8$-UY4OhL{%Xwgoi451c#=cqI9z(>d^XIbc z;;V8hCVH-iNNs!y6YUFTHsTTzx2iIjU_+}Q5tv#k^D$B}ixvGFp))j_nsX9q`i=g78b;tP?|CK^#-1ZqYj~R%j z9G?&VlV}#{v=;RLFa+_G**X1i&O0d4zCNe>bb+@GP4U{1)LI`vq!h) zo)4R3IEnQ7GUIl7NK1VRXEHrP77Dgw3I=T)g%sG1)iY3oqQ9 zZ!OT;(GBSCn&Y~pB238Hwz_OBp`^d0e`z^oKx{FZ2o?(ISt;X=D3 z5wF%Cp4r;+3vG9+xxdf9jL00^T?lXg@E%>#!ZgnO_8AfH&14qu9c9&rC&)e&1dG@~ zUBdVOfZS9G2i5sUH>s=-lVABp@4RKAdp32mIzxIo8Etomp35S-{o7pmbWg-c9r^R1rEgTFW6wCPh?Jm{)3JFBMy>rC&+BXcv=DuHLm+z`O$j15Zd4gzF zmWfjtXZ*2}nQ}Rro}-EDQ*(K>V4IhORE;xnO^SbReila;z1^-ZWb2l6#dG4pmsjnV z*2LSN3ws8i4INyPfU1Y>aq{WJGx9WAWpN=HPISK#8wG9pY9!z&&5a<0-o8bGGQUaB z=AR_q0h22~CB>@{R2!k|zxcU^t57!)p7Bc3W3IdMUrPcegJ{0feDB%!#V_z4`BmRflsg8hcH8YC3f)V zgY@cqbSIUA6zmU17^M)U;}eA!4CTtzZof~5W^;SKdKE3hw|ODwFHhybn~rwb;E>z8 ziiVirA&OWAT;Tx=)5OClde@QlBMH+cezJ0b8@`3tTO^<{T`Bg->g}mRw1j42M-Mr8 zb6L|~GBgZ4zU3&&W#Q2N7hPii)JgQ|@Y%a7?Kj(1Zp|*PP}5)~5&S;K`77HGy&Uo7 z07s(v#q6NTXv#}!BeE6kryCl6WjN}6RN$l-%rdRL8JmGx9z!DpcK9ygsW=wH@8<`_ zGEX5KAe+;h8k1RL-y6@&yiAIsXICR8Gs~nNFS+4COeUyZ8L=?(y^g~buHNeFNKD2n zu_kwe;)9+5kI7?>Pb@jNdyEZEOetFzSXaKZY!yCpL{qy7zR;Aq^<38p-bt{pfSROn zSc#_mDwm-aE7$UV5v`Y6Wo7ZlAvWua#Nd>}l_h;$Vl7%X@uOMjsf^(b zDTrCTeFlBh`ildXI>LXOfG^7Sry#Y^%J$PE^NYa^Z?mfxUhmXA;hlH&qUF^oZ|RVn zqqBBspF@w4<}7%7ZqWR>0w;z@q|!uMti&9!@V9+4nv0-Fpr2AFzj2dTU7|BjzB%#W z5E|b8>*XCLKf*)QCN$j~;Q#Vz`pQ%)nt0 zDScQjyE^ojS4h7^iT%2sTdDoi@`?QCW|8Iw`Cn}+RMu<%3?(_75P^VN0C^a#7o=ZsHrpc8kvO78%Mse4l0%?j7k= z+!DpF3@0W}hT$0YH`YDE%vZaFH5{Fp;25G1=i%-TQ}_0suJa;;Ml_q{15-Wx2bUJizckk*C(;b9 z(of4VhUzqHD`a9fc|Kx`iBF(RlqaV-T4hgNWaH(NvUtu!Z2fb&dhl(k^>(Z5D}NHt z)20pdd~fYz)6<~!lB2qX0sj($e7jci*MVr~@SvP2KmR}MePvXYTNkdBl*9%B6{Vz+kd*FDK^g(2Tcn#U zASK<6bcd95iy|%Eps?wX?z=YXiEn(j#{GML9LE`Z4Bqvwxn?}`nRCq-@&ci4Q}b2; z=RX{6I36^Ha`Tayr1Uz$1hsZ*Uz2`S{Wrnb{Zz`NmseQQ2Elx&_2fzg-ZSEQ8hQF` z;D*GFPQHz=TWuE^mqz(kbVgnbVgW(76B*Rv?_3gp@RH&I=;$X?b@(#~zgV6Cib_ZJHRz?I)9KyAwI9erDAbbDj>g)>bk!Gm{BD4#N) zaAKPXl!2V0o@!DT8xyx=_w$=S1y8ui<>h#LklRORHNXAJG+qmkt90qe!vkGRp%r@ktrCo7pW?sux>x)nlaeE zbKvw=n%Zi5SQ}q*-ykf5}z zAbIy*KHFc5v#7KPBzIwAY{1$mkj}IplaA8#zaO8+Z%M%`JL-K~?%(?c*UB+Yvf)@i zQ6&QZPiG+96Mw-NCt_B!FVl3MsG)j5x0-L5)H-sVVx_ca_>x3!y!d*DN%hriT6>%C zV-#6cwS}L_hzO-9lhsP;xV?_b8XsA56P`bm(hltm!q=jI7Jxxy?98aRqYH`%^A4xz z*F$nzgrR;^6R0LhcypVgeee!4Jzm*e z>Uw`15Rei5+Nek@>HPEh`5Q6{ZCStwNeX>7Zzl3Dsow>=u14Pn;7?*{9d18y9haj$ z+&dG$|CI^NPz&Tva=BrjuEqL9n62{EkQmX=gnxMt&xXI!1(B4zQvP=R(AAi8m1`d zPhW^MC_$6l&s|Bmi5dOS`f_4hu9v{WEt^2zXg98Z13R9hcWIOleC*C_CKG{k#$q9U z(>;$7G7&K&`@Cjow1JrF%S~>}fp!x72jyh9e&^uVB^>Iq#4MV2qrIXVA8|ZtDzgI) z$-?tmMk6yM8)z3MG@iB{i)nXs0^PWcKoM2;8Z_jubPEyn%0XqA+3$~1#}sf zGB$#R?g6`!5+yW60*5x=4+x zr!sccEV3xRh2*f)hRzQ~&3nhff@uCg4zq{KtZC;$XU*=#g75-+m>&zL_pS)=mMJOa z39*^G|A={plIiLn@3$jH>f)HtRxm%jW&-g|^J!aSZELlMR9S$`bC6@PgMqJqMtiH{ zlcye#|9Je&M>%m>gGMm2tF=BFA1F$eSWA=GqlVyF&l=_dnDxR7JrT8a4hFOUB9v(D zJ4((vvM}3&D=tnCwFXl%dWrx+NfeaMhSB)QBbwg4&=!$MOP-PA_8{HGu)K7#Q|*QA z!=n|O!#QtrOEyZ6>F)2j<~jj(zk$>>9uks9tlpSKjVwZI#qg=Q6+A48)c2zI3!N~hV z@}8zwRg&khwmAlu&A7_;Z%v8l!AEnP3# z7Qyv=VY^P0-p0O4*hn`nOgP;Y@vGDa5u~J_mIvc@3oxoPC+nDwT|}Q1VBQ+$nXa{< zO7ph{eSDEdOgO9^gMIDU>wAcV2=2gecTu6d-TEiJ<-Iy-5*ED+YmO zMN-P5IC^{gMEuZz@#K6wfSjRJ(XVhnhYNL;QP}KKVBE4CeWP5{h6wJNx8!iVBOYQADUW7{5eS)iA}q zxIClKD9W&o%u;OkznE2yfL?@!!57RE(^u$3O_O#N8T!6piOPB7^B*Ai>*y${^evtU z;$xp*Dd5R+Q&aUj;O;24U_byvTU|qG1qmHIr4a^q-jrzd#nE;%kk7C`TOYpbk7hyO zM3w>d+4s*LA4<01K#iyztQH>UURi6HaO>*h65P9e8RGEm!kH98g?MngY(mp9jIj|n z?W9G3U3)=z|F}FAYiaTy+Cw&9)e;kl9SNsXb!Bn+fbx)+~SL)#Xsv_+p~#MZ7yza z{8h`mej`eHeT&LDrZ3j#8ZycPgN1gFftozm7nHujhE(6Onctc51tWaZji3&2(pRP| z`~GYo0p<9UN5qH~;POXH5+Z^V@-hSMlK>1HLVC1aZRyf3V~zl#d+w{)s1bnBp+V$~ z3ML@cfV6G5$9MUH)Y+G11Ru5SU##;6Iv=f>wzP@NsrF%_BB04y5TG-ZGlZXBpggS} zW`ds#3k^d3$w&}n1B_TT2*gP}E<)8A}!K~SR0rDym zu`ub0TM24#j8lPD^dJeLHRo|vZ>&z)B2v=5cmK8(sfregNP)Ov+`hF>DZXZXgjS+0 zR?w*SH+ftbv9AYySe)kc(J)@V{$MBZRgp1{F^GwZ2+lB4u^ey*I|F5aWXf=!RMdx+v?FP zh!7Y-HXMg+9S+I}!BiJ%vQP*nBHFOdyi;UojJ=L?wi}hE@S9$(ugkT-W281Cpb5qx zV)}O8RC6AXRaTT%(ttWq7g~Agi^KRIIF$VC*9w8?6hHtPZbi&&gwpXThkKgxVH^rV><#* zQqwSJ;RpUXF62Lry1;vYS1>`|imLRUm4#+?QB8!9O-Z}ber+9a z1YRad=n53fVlD1audeLmOxAG(V2&vQn>d}YGx9>&n0r3AGf4A@FW^eMQp6{YvhfZf zNy8>HK?MBxq25?MDa2aiax7}T`VS2LlreuFKp~(o+A$)l!}rD1e9~ut#FASUBAH_B zE3z6>f z6BFwQhsc1&sOrCvcFdf98L0+3Yf5oDz;stt$Z>AgRTR7CknRE=iz@eHnlEpG8cck~ zFYd`{(TcppWe_TJsJ1+NHuSB$i+AEy*tPC+5R@62OLUgBDBt2(TK;wU`_IDZEv^e# zbfmIt$@31iw}G-Ky=y^q9Qh8-m{{Vw z7Pay=pRICP0Tm_p9a+}i1$fm`h#I9%%9Yu&R&{oY=jEf$@GsAXO#gwsaZM#!MirVQ zhCCuKec@9*!dant8f!kFu)j)YK?>^s&<*ta6sxbL4`;?nBDeZ<(JDXwRh~2kJ0U#v z!#N8#p6YnhhIBUY={nR5)Cr+Vr%9XBe7T%5^Lbrb<9WyJZ`1()zXZq=`Jw#COX}60 zp320x6mXtdjkd}sOVHm>MW7Sjm7+upMWTAnXqHTwgyp1V#ni6(4SwY(@q@7T2kJ8U zq8slUv$m;bh}9~l*x+X6E#%T=hxex!U=^`ir#C2)T6$7?NH#m|&BI?I_?L#=fIh># zg>K?Gr8tC@Z%r2hC_6nA$2j}KtXxW~Wd=l$-3pDMkVID&U$+n)RM@o*!!Op=MGvG$ zW3qFdK|SsSW8ezNyPTPkz=dWEeGz#obt*H|7ipw5RFPDCrhOJ&VmH3I1;OWcu&R0 zn5^66^iS_e9Z0qdQ>|@Wt!u(ItkPU?EVeI-ssz>#e%8S@Dn{8F(sJjXTa?bZ#%3i( zuE{RC*<8J2;1>q|c#}##X5Vh10PX9Ci92fw%{KzkR=IL+dZFFcyU>3s_p|lBmwkjT z{BSf0SQHUb7*@HYO{81}3ya^s{-?$Wk#R$xQzcyWSnz(G)O{q^B%m;aYP4ZU@^(V$ zSnoCC0=yeN-5naV7;`c^)etn!$Ie=uoDrbj!w$wWGb!o1*Xk(?N-ukr7I=pqFYeZg zQ@rUy4PVZE-jZyZAT-`sMs1hb(?A+*c~JRH1pyHaS*qnxyMs>_qn|XIaUj!wNVgDi z`E80z+jcc89v-zUt2CMwm4|Xs)H3E#Yves?Rz>_r z>|YNOY2o9M(W+aXJ}%W~VIXg~ZHevruUtd`H{cPkW!+ChI&}03s;Y?LjGbWT3cVkg zN0=CAdV~0%1@tp8(a7|nLN2@JQVVX=+Y$E9<+KCU32)t(7ivL5TjpedQRZRKl_Lw!Y6BMl<6`2 zSDo({oOr08ipWR3+%oQqyw}}t?DqwA`Ao9{Q3O2a{!j8ka!S!Ya8D*YOGtBfO#@z> zuibHA?X)>(&aTW5UYUkyliL^fv)APjCMA|>y*1v4|O4nD;T>e60|Lh&O6l2`{3F>^;?Z zKTo#>W#@rwsb=ZTD@%W;>}A%$dSP%{n052t29k+r#E+t(`_Q@WNv5cn5gbYjHN2z; zE~*YWfO$3YK&Y0I!dXAu^raT^Yswi$Gw&=?s^^re7nQ?e@2^8aCyFDz&QNJ>mMkSd z#&sd?I4hL=I3Vxl5ZE=#7dJz)W(Vohwm};tg?7^a2M| zluACXGBc|S^epV>j`0Y)!E0VPnA5bY*wR-~_C@|@!hgP{@vFfO8ib7Jf)PG~OF#5u z8)$_B!lEjV1069`9}`y(k>`?<0@;2ZX?hct>VSTu*@dyay>|+S;GX2EW9@N{QRA;$8GM{C#WsJ>Ub>y`4LeL1eK<$JHC6vakih(BON*-s=EW zr*xouRF#u;P%N0yuCl&+C-rKM&< zedoRPul(@_O1B$a42Vz2S6C4VF>y;Dk=##w3zh>SCT$T*Qlt>cc)77H4o#xOqO&{> zjGR4qNn;9cs+-4$_ET0x+81Q0dkoRk{Ttjrqg0xOOF}7v#7pu9R1O2Tlr>-eIU2Go zP`IL9LNJ?*c;|*=O;VN#=9x!!)>8!b^sivE-i(N6vRY(jQCTyTR@K=rDv|!B%AdW3 z%oGtm+!MJupARmm;t`VRQptTn0NGRSja@1QTEPtoiZ`SZ5dzUv0}?}1k>=8Mx}fj$ zx>jkZI&UuhkWxO{-ce0kWa*D&uVpN>j3RR_2UbDsVc=*8fFS5OUxNNIDFyH2$mArN zFbYU(RJ%mx_uJ0m%0$x39zSY6x!kJ1esY%L{6lb$V2#{NGO|GDPIiz-|JDdixJGc2v3GGmGl2^47iPsqdfFEoVqEuhNJ;49NLygw z6p@>WviiOHC#ej?Z~Obf-+iR_AUm};cDrFna!0tIGk_$5j}wX##eC(=Rn)0bNMh_G zvfL=2$9)OP$ArIF`ZK6PAPU1HVq&j5dOsZBj6V`;_~2p>+PY^6 z$(o$q&KJ%31JP_Bmq&FWAI9xxXEkH0iyn^}Z06R&&yEzOFJ?bFi-^kMzyNg>|1aeQ zj{Eism%B{iD+Ps;+t&E8mju}U>;59#ys`_)mes*^keyf|TEY8$SNyjzif@6-BJa81 zpP$oZ-u%&bpeDFs&A8jWwfo*d7eb~R!>QasRNA9AX;$Oyw(BQ`TrZh8N=5UkCd+~< zBx0DWmg33vMZXp!(10EizMq~o%M>SX`;XH`R@QGRa~Y|ArZvY~v~v=fh>8bQco=>c zibC%9Nqs`?J9GgFXEHv4{z4UVw_ESKv`rBqnoa{nlLkMjS#mQnUMMpq#jyqAaU)l~ z4C@smb8aTDj23;q!h$)PUi1f>_BN-eJVqe#eHnplJD!SJ2TA}Wj$5hXB?2y z2En)cEds#3wZ`M>Z6qaCem+CA{zSwi-XKW&W)x|!FWxB9n&Bb&f{^syk4GugdqHo< zd#C4Fxtj0oVmGDB6Q6?C?z^~ov;tvK_W4=LFrSJK<(s2W7$relNn;gwCUo&6 zxI6l|VNU%hIwhW-lvtVdv8mF1MC!o#`1p&L_;@8T_BC0&RQ$>x|AreK%gs@ zrB2vHJrB7rcDYoz$A@9J}3hPKynk zg0+DY7drX@r9o6itRi}+?tf8W|dw<~-q zCqI}<1*DuG$ytmP7A)?Gyw@}+vG(#MHgc`HwHTu=8S02C-tu)f=ol|rQ%10n9son2 zkcAIr>O+Jm6lXr%L$v$dNU+OoH#+=hz&G&82GFL5Ias$C+!cG*R2>v@)i8GDQ^O>a zN?cT18Xmf|P1#j3*+hw8Jm;9@i*|03U2X!u~%T#r+KDAd%Iw^cIMJtr&&#F z?LGJ$h}3ia)$S@+yHgRr_JxKAdlsAA?(@RLD$FCiRJf%LY-*La5lTB57BTGz5w_8y z34wC&^c@cl4FR~Z7CV9)&kxt(Wt4#a1g>0(UnmC-Zb{L_QZFr(YThcW4zx+~XELB2 ztISvB?^V4#I29@h5^DGfOU_ZhB-l{~T6*k6GQ{SEaXSp|IPH>sh6|lj>x?*j$$HUF zK{sMG8g81^-M>fT069uS1s>+~#SJHq;PP?1v-5Z(M&#b0V!ZQBP-8;0qwt1;bKQm+ z0yB8=;N0AMAvozmyT>A+5t0UqQ?E0tQd9JHz8xV~7Ch-L!eWKV+aQp5{{d6ulmj-$ z(P}5$@B1AKw8Z)AE6elEDnavhH^ljTg!|Os8{yIqf1C##WKPz&WdZ8KcZ^O~BoUZ9 zz+$8(Bpw)Vge>98hxTymN+9-Vn6Z%j*y|ja3EDV=kbAY0C725GN6WoC5hIKOwAu4B zr#a;rt?A?em!|=xvm>;)jO2j2@+0jAc_E=WtF&N(vtR736)%JYz-^g@^qfLz@_HX} zgW05DA4Xswagt%xzsk|@%@~S<$dMn_HgxyF6z4b35XXKQ;yiHq6OTxva>GPE^`KZ} zC^+-XX(iDaeU|r)^HSsE%ucjB*BimKBS5gh==em02WjAbTUnO{U=>`i+2B3wpPO#% za-SIpVgAh|@KNKskiM?L*Di1jF(o%;edE!cZ_T2?WKy`LZ&Q*0yEZovwlT&@ zN&}c@Q?x7ZL#5nzm%b=9FQ{nw*}Z<9{VJ(!zx34;9l_h(6BCA!Z2LeNh@eWIPm-_P zXm4xXjcke#u@gBtQIWvVImkx?;(=@@y2cf_g`^3i@t?uJ0IgUAL0al z*~O>SDb@~#8gV)tXb<}J>Iq4h!@k~Ah)uF$cUUZSs~BTOkFXiso3fUzeVzjH$5}#O zSnB<*YW0Y-{&9;Br)$oLfHm#zM3;~382rnD$I0=3eH2N^8|8SBzr{y!jGtNmEQG%X z8mMj^tW1KKfM3S13(;fcBFaNYs6la$2^etdsDz#@oF|FAS8_nAQW>E)*C>6vHd0ce zRj>AqX{5mPY?FRZfxpHCvP2#(j<6S+@IQe+mGu9FKaV)sc2x1xkOWa~prr$a_V!VT z``roI;(*ZP?35CX2>BfEy~%Yr{gzC(mQlt9%}}A`5QO?K(>m$Y$kT*7fUwbA`q{2iV^==n-HvBV?=3+iWrDF?rFfhOA_OSyxVQ{v|E(S4zaq6VNrH@ z8Igm0zSBw ze*JFqYuNaF(^UQJDaRWL?=#*5Cjridz>)q>xClev>8$U%`EHp4_u=)ypa%b<0>Zfw zrr`0_&JLOg2!fVxfoQ*>iOmO{gzkH2Ne)yDVPN6JTREdhV zzuW9N_+s~UOZx2-<5Dhlh%U0fM6C$(Yx?e7a@0rFotZ3 zc@y9p(96jqAiMuz_dcBJ^ox4 zt3f_qoMDvi7AD(+@5@fx_rD7>3NB2Ar2JPOi(MXwS|{aLCxNBVj&=0AliSAFeMKrh z>C9ZBr8#kB&O8(_iB9Uq-AMs@<0LBM|Jw#Poi$Q_iej?UU*ULR`X^3+F4-DeDZAX17>E7qd>OeMc0|eRJ6*v1wkc)6Z ze!MG5rXPI0rX$6FU-KD7+kb%|u~Q^)BSE+}lG)@YrG-~dY`V_1+^|WV2R?b*^E1v+ z3gr65h(OS?fPsd$E9Cw*lJr|(vnJ+b^S{oIQTD%|U%r^{=Wo?mYoh~n541j72?O_v z%K{zC=^F!yh9*I)pDm3*y|)wu{7gV5!1nizDn$6>ss9fi{}&3GPQ4lJFT{cZ;Gc{Z zOOD`#e0~q&q8+{KeUKe>0(jJe{s$bT3ju9UKu2>{Z#g7(v;K-#I!&0(hs;W&?*_1t zp*52dKB6_IGd1M>+zwswPdkyc&1Y3Xu8wBktC8T8oDd3$mqX~~c%?Y^$%3dP@an`$ zZ=>tv-4z1Ed_Y0;7Q!N%%3rhy83fAh?Hz|=*<22*X5_+{`R{9yf7oyergn%1;CBl9 z`cw$wSxw59nKj$j?<(K%TC~Hs{@SDw25wry{Xg&pFy}o8$PJIKJ%v^6(-k;r_@ACa zG!RXN#xI+Vr`js#;@;-gpIs0#o@qvzcoc^3FRolZsJj_Wo^5^fK-gUYJsiG`XJ8xn z|JU32dAVy%Q~hl7-pI6LIv+^x@>-bJxz=i(1h^XYSaQ7l2@#B>@~1jr(Z^5j07!=m z&i(J(;&_mFfVMQR&t>X$=D>e*x+Zv#l8;W^Z5BOLU)zdICr_J1h6G;Kv|Ik`sJ|xXo0sF^70KtUBc@~TRlx{|KA)?f|&0= zV2ILZtDD^bAf{vZwhyhq1R`&DlYVRV$WG_U8nXGLYewkcfQbLE52!t7@1l-$vl;=e zVJ#?@f`cDCoS#4{J$y|`5odvQxH)Z@a{tDyA0M_H#PWtlzX7czZyfnS2KMPurULRb z0qq%ags;QBVh*T7@z9a}b8I2HmP>|PI%pkxA}WBT2nQdx^0qK?gp1>dTC*{}U#jGpJzT;g4C z_nc#H1^g~0hBoT_O&gH9;^1+NQ>Y>M!c&r9LJsgzixo~?p?|tmLCs6YruMrO9h#vi zndul?N1zY{0gX{R(nPQuB8~Fn{VB9%}^8s^TgjaSU_LaU!`CfZq`sPEiGu)jL z;Q8wfB!Ax;6as%)QZsFv1^+YL%JsX8s+`7m9Vq5QuY=iQ-xC!KP#7VDkQ1-LOCoCM z0!~N{Ek^5(+0>4ogG`a1rM73ydMz03C`?D;AhJ)kjPWs zj1k2x69d{Sd-IJkh^2?fKv%j`<=i>!DDI@;y8-oC1Sx1o@005RfgrO#rvlp2{3&V{ zS@zd{^?Pkx*O`B9S8p!_{O&F0s~^JED1HN~41e4QkAeTMdcO*^K=pJRXSM1KBXqr1 zx&1`Q4^{=GV(8Tm$6AiFxNI7Eo))TK`0extV#@&m;|UF_V5c<`#Q(W zKCOb|=^@@rVdj+M$l1Fnb6vYD`%wCyUe#ae0uA4W?YVW`sHieRyhlcq_%pRuAhQD;`rE%C)OFH9>@w>RfhF20AY_nOO$H0WIb( zRmv42s0R&Sz|LlrTnnWe?MrVa4!bFUkG6+f!`;Ww%+<56Gy=KwpLaoy6bllU+CPbl zFwxH1f!c>aw6ViRp;84`iOtuF1DX9znuf%*0tJ12Tjx~D(x|sd!6+R z?~DlX;HEMnr~-9fx@=}#j5*6Gr<#OkSsiiqmzZQtSt@a2zWZy+U{tF!@EpLTA&ssS zru7W)do5~W`q+OFNZ^0~`A=*Y$2|LuS3&r!32^u!ZT6SJ6uQ|eW0%`{|NWU{rbKV8 zae~wpk$=UHS9;#Z8%=anznKW{8f44H*~td8%<8$OAbQmj9ePvEKr|PZ^^`I8cmvs~ zX)VZ$scNgqGS6c3&1yFtq$aC%wPM|a%mTG3X%35#(DisjC)U#947RpEy#R5?7GDWB zl|9Bj292K{&GoI*n-5RRk2p?pml(7ylzHk_bl)Z2OAP){ZSS$$?dFIrDKc zsaqk?HAh)JqwD6T>s#l8{#(+9;>duKr5_xR485=du7PL~6DGQJ_4TOj1h>>!N^(C3 zOsCoLg}dtNhm&y05d+CVg8&vHATCG!9ZHV1!6cdDG*6w9vn`UM2~w|>={sCpojAOU zcO1_9^LaLl_JVJTSyHrh=@k#FgoO@P;>-@!U^Y7*^3KnTI_wf4*)K|)JZVlk%ExJR z5Zd2ft}s3$P3N=<8{BeOTkBO%4ZG|e*7n+dTetGlVmR9Z2aEJ|=CX%?HciT?ynG_Z zSVu1D;Ex);!B2voPIwTxvzyrW3cv@JvO5ePZ{B5Hq3=ySd9iPka~{mJU5d$EPrpTc zmE9{6b!%X613!iMC(=!Amf(l{!+gHd73xYM`Qxz50MiAHTd_( zwwa_J!d*@R^4kr&86^!rHt^=*7O3mSHRMY}`e?Vt!lqW1=!hg8v!6$p)aw{(XK>X; zAS?TTeXjFb#a>^oK?$T(=B!4dLFUmVeyI!*8IUNTWCTfNw%+mhJROV` zifUz20pF@Ol)4se1zy@U*2WD0^ko9*bGRlLaqSU^OKLn%TtY(m%9_}2FTcH#Qj@GZ=BxxC!fWziXCgyYJ%xb+ej_L<% z92tRV5w)+^pBvcQ`-h%x*6=l+pF8tkjNgGNKCeF-e(;dQ(Np(qy`NP!H>r0zo%Hqg zlPZC8(~C>DREcSBDVv7#fyRCi5ka`FVm!%r?9*7LohBR>bKPd#@j`1zJeXrOJD z?VsL_(V5()6z@4T4eHyT96UMsY!EsUQWK{1-Q#fNr%`Z93{@(zVBg79%-C!J!zs>d zPtHQfk7~1f&rx0uXX2-*()=7P{f4^TSTa?zy}n_hR_45xjg#?(z?zDJ@35t@?(A?q zndfbohfLRfw~PX{;vAQ>{y~QeR~~clu+hiA&o4lo%H}r6 zw*vFv?rOL53|^0LS|rZ~R2(XZPcK#uaVK6;tT?6t8UuL)BnMNEu`B-o8PNUOelR+} zUh>M?Gs)<1E~z~Ia{=PX$w}SMp^AgqBxYW&eG~T+AGoikgtSh}dA(e$7sr*n)u@Rx zBRG7rQDNgX!ZhOK;Mr8YjpFJs#Ky`%&Ude>Sa33hN%nU`mZnCJkxR+{wlXmw@uuQLD_Z_}nw?P3^BI7Y)H@<^$mo z((>MmfoROlT-uD-<^+rk%r=b*=3W6&-Y+(cd-bTkyPWR&o3)PVJt?GZ;B%>V9DmWZ z@_2b$U2k+oQ@G7C?U6=tVwL$i5c|k?oX}SSXY)}uRZi_5L6t6T_&6)o#HZY8{O3LE z=3bUGFq0ZQ2-BwRhbWh=8`#ODo$NitUr9V2@ zK%FrebR3ar1TuFJ_4&&vitQ-F+42JoDG==Ep90Kzr;cBaGa0RJh6ZFi2)rO|a(stW z{0k4kyYLLpTp13H>of<`uduEdp}WmTvja=8v^Dq3^Yw2jBe!0vi`@}4`!Gq8qW$CJ z_!vbz5UdQQu`CtvsWLqm*2S-1zYZGaxGL;-(CXssGl?L;o_sJgAbo?yio=SjN#6Yw z(wnkZeviv+x#&lD521w<2r(;{lBvv3-$}n^k%ky2HNFbrw3*F1EXH{B_Li5F$>GP3 z9gm~tTsx&pSf+f{b9orJ#ZK*p%x3Jjkrd;eERX43DoWlj%iTLs9h=FSRiFEDzM%%l z!-M%4qSV*j4c}*Nc~S$8&FoC<*{RhA&53z$AMH4AmQwkE1}6OupZ>hPd=nyRu>g7Yfct!I z%lH+t5a}E^jyUaF$eV2go5DSZf$38h?*W5QCJD&OevJXV-Y4DxNMj`98j2H_+qAClB z3F)Njj~JhN?IB?g&ewlhnOF1Y_9~=>Ecls_jHDwpCVMn%tGuRLbmk8;RS(~YEM~k; zMZJu*!~@Yf7y_MD*{_8){6|>f$sI$^eau#Yp!#IE{w~@TaIWi!!u2fn#C%h54B-zj zL<hc8uLNMi#X9Vt95)3xAuDay#lP4byCvlNs~^1C^o+iaASfH@=u}~wnD2t9^>DkvmTf2_BhWTQHlm7JlwX!<2_y`47H^;@xe8QYlC%IZ zCj0o-f_vV4183@F)&Js12q|H*aZa55O5 zy-ZQ~OU74?BME`ir-cLON3qGL36H`_`b zl()$H9vc(*eLdBMox>ZgR$*LwFzzJ|^A9!=S@?Te@5&!;4Lmden%HczAGJ1AB?+vi zcCL7&ul^bHQ2#7Z@^TM(MNDh}ZmG%AX0@#O#xxJ@Z@pCbw_Y~%d(>(22 z_d&jqd@WUybjvglKS<&6!wtlb!*?i*fB20JwDDJsMyb_YrMFlVE4>akOADUmD?{Fa zxT@zg`#NNA;vKWs07xE;K)e{fYNeSo2TAR1Grf&(2c%wA%2iIK-y5vk8gg1J_q@jQ zHfxxrXtu|OJjM0QUx5jGHmxhjiE;=#1pGl39~3h0m}9E zY@_fCDf2VkS=TCSwK5D5P1zbf2Q#k5i&IJyK7rGhevXqaL^Lj~e0M*p`+Q2|+e^YT z*q=VOz2`ndpitZ2c=%rHfn(NQ1#uvn*JZ6HD(CbBU4O&9_DWvIv-MI&^Kz%wLhh2r zC8p=oZr%bX&WU}z7aFh%X`B9GVaBQ3yMmW_%ag}l+kGb4R3O4647b!eD_s)@S8}OA zc#xh{kUXebo;y9>r&NuG7ez$e=>*ETNh2E^1wbdE@NL{{YfCsaB>{Ln z}m0g<5;v}+-;XSfk<3ici1=f(?OyPANK`Knw9T3 zg0DQe6z>W?-hIBQ=7s}1s?klk<1qqt!JFfY&>Pq~vbR0DdLozJs=%2U|a1_S$K z?g>7+(j7ajXGx@d8A=B{45Z)r6aDkE-@%{3G(a5>1T_N9#gMDjAo1O33wYT8SMMeM zgFxolcfdV&;D=0rvG4A5*96ZBfKrpzfHwr~>IjQ=udVO@8Buu&DO+%55ERsdrP(GM z_k-E}9MXurIF|XU3|<3~+ZeiE^i%u1jxtXsGq+Ovw!Z)@(p1nAZP$GlW;!0ftB}BT z(EKba)q{`cA=W!YJswuU?r01PqoikwgVKk*aZ{AXOFn3BH zy*#bhtJhpg9~3xStG8L^Wl)Ll&9ju$WBoeM+ciAaFsUca`4 zEsOt!C1{t_^!z-{^q(oQ&D>^!PF=jwUGo?T5l?-S#(E-uz*8Drv912q%PG|{Y)0VX z$HSCw**<%NXBC9`!kN^4{p+Kb8HG&iC5py!4xTn~Zf{6D4&`AmkoG*i`syYYg-P-C zToMnWz`O1ai$N~`g0cL>?)z0JB+T>*37cu?t<_ZZDm22$bbn_`c7Jtk-DXU@v6Sv- z_ggU&)s2umI7PPIe5Msab0P1NqOet<{Wes&R|}=oAi#3@?KwQNVou7TcPq@!m_9lxGpF~EIpPkkb{x0oh@znBZU)Nf(v@1_(F!+fNGAo1x#$ zKD{quv}xz@IO}zX^@k*}2REr@9$)6We){HFbA%3uy+Lpo5!rugVR55aQbIL<2RumE6zt7ZE61(k^Bo$v= z=YR#yIS|k`9b|&DsLCf3xM=|QT+jGYJK};u7AJg;@xqi}o{7OhQebVl69(dPTkE+2 zQM_Iu$YV>BWwC)nNKms!hsmPdc?A`l8(tdQcR9x2VM@RB#+Wuin0#Hc z^VQLdGYLJ^r?wUtho3jcHE}|{I#mMEmh1Bkw5Xfv{9LkqT&hItfKMM{Om;o#C=9|1 zR3{+rtU%qRC&}(OEWbDd*+u7fY4`kb+Eo4k=3QEXweRhP}9Q+s1nCd6_@>DZzK&&v zt!i$O{$_!hlX_`|C)@i5DQPC27b*!~2PP&|wrjfK9JL=eyk^&JL%OR;Auz@3AJ~5w z3O<4L`%*&>+N(y@ayzik(vK+F+Y6ny!!`=)=a*>@+H$GPzZq9xq(MqWdo^m~EGZoAnXsIMcZwk76HEuhW)*x;VHd-ZjV z@z+sgX?w312l+9tjY_Y^p6149CGu7|cW!&l0=4*m~p-}N=`%Z%i!>s6S@ zjdY-gnE^hK+l!dfxIuyY`+Zrr2Hm{oM zjH?@WS_Os3Vv{uqq!Lq>L2D5|AB!@YR=hW_bHu0x&&&Q zC^n(r46c5aHZKz_;J-H{0^~gRi|UtYE5$j!sNvgekUdbgc9tpY*>7Imfej#c+n&8vRA^t zSMZk2O{a*wo%D6%0vs-SSxqP#pMiG!y=D1OC%_8IoV%>gd0F(X#S}cX*)h%r)z2Q+pvp30Vl6^ZKU1yS-SaVhLRzyP*HpKj}6sZ^56lWB93SQ)0q1W69BL zhKU&@gB`FT$>?J_cZGvFo?UKL;n3ehKS7r-(dWYhwKe{9WaKvtxRu*U0qU(O>-*DZ z*NsiVN3c8mXeFSlVCmh3q9D_lhZD}|MwpY58u}ZO@4D_!M(^XBiG_s5ktFFoEc z)xJzm_4rBNxOYm)F?A&bs&qpWxfx=}Dx(h#-ipJZ%Bo38w29-Y-6ZhRQ<8BwOaG(Wv=-^p7g$Axe?*q+_e6QZKpEq`xmz6ryz23 zxPK0Db?)KsGa)*x)8d-udE`3&I|s4#R6HO+lWo$@Q`p3Qw5JzlHYK~=H&nu5&h0t2 zFEBPyv)$5a?%6^v9?GV;LE@tCLG!e=A-1)Waf0!1hL$?8rLv-uvBtmz?k7zbTjJfW z3ZJ%H+ePxpxivpOuCmhr<*i01?4gfOonj`~qi*eVU~cmi@<&&dO6%6HUv{zo&OvgU zV0vj=(VDsxZAcHdU*Wy%p4_^#TUO_|o$x5hE?G0>J1B5SL>b61^iYmhnoaecd|PMc zFGNsJhMn_)0xp%>n)7BS`n~a7GEG=Lhjz-9yraAFnnp6}lxGHk3D<{e?Honqzl;)nf)OX*r zH5rUSk`rlb`5)AMcRZEh|2M}u;m97@A%%=6+2bU8?>$aZC^HHX=OiK-8QCL+%Ff=W zh>{s)bE1f{vPV4EeYC#6=llEb`RDn^>(x1(`?{{r_1WWnpF5ws**@XKdy72)HUpihO- z*FEx^#X)V_`(<*Q>`tcddzRY$2Qx+~TZ!vZLl&2CDG&5Z^=#aTO3lsy)?Z~`VeiS45hnk~vZrs5xN|;#c(?MyM#K?m4OXa@WfbUg zT9ucU+ptvQp;Y6F^Kjm5lecxANXl9-L;6I;JJ_+J06Xt;2lll(YOQUXF`VO-j>%3}7B zTf_EgJ-G?5o$I|HSX%W`3M$oRPPlFil5ET^j1+rC9aUHt;PCn$-gWgnsBfwbtbLp1 zZ4e*XCu$J;LgAQcOixS40$xOtfWv~I{diA8!S|H6^yenMw$5tqS5Tv;$eO`EYNT({~LH=#>y(>|p)9D|Gr({IO7H!dJ&re9d(;2T&d z%Kg)C_BM3T@0!`(tXvsJcpVFDO;mgeTpJZS+To)kZ~e|HIqWQ?Zq1E8!dFmCGsj7l ztcZL_pXj9Ag>ujqwD|rr-F;(^IYGx(@kM4?`>+)+-d--l@!E=}r|(J&(+5--_IE%N zf^p&$)pZ5;>h8vy3l+|~*KQ9D^#ANqvy&yKdFFCg{Mn}hb`Oh*Z&1FJLEuUc#`biv zjA(JH@7(JRt-|Riqvuz@b??utO20_{z}9h+>X{1<v-xD=pdkPg**F%uIU!Ud+?i z1zY)PO;DCVIr0;=9OEt55SGQhXvGe{*P#~NpT4{;t7xW@WlE+71r!^o0J<{7M`48)3>UUbnUy8YgKw?^Lez6NT! zbaA`S(>WTk;q{u_hUVXO3X31^FRV^Pve_(0>Ag?a#G`;KOZQdN^hW3BaIWQ=U2f>s z!n-ZVgN^@SHb2qfiW{(U{su=JqtVA^-I`FYM}Ae#8)>A0nw6}x9(n)n1+WVc>-?&W zLZ~bAODvU2migQG1#eF*P0$%l1uZr_de^P^VQW;V(EZ_Hv<>a2XNd|upVTU&IX6J3 z$8~gbI=HcWq?f-04+03hy><#cf%OwhP1GX;e#i!`0nS>G-#b1`PnDxyHSH{#7$i~tcDA-^n!(B&0bm>d!Dt|%5HH}C4YRJk$ugc z`s|;B%oqWU!Qe4r|d%tX|$!*@3 z$-H?3IAvfqBp<}D_t=lrY;o>vhrFTsm4&gMP9Q3--+$(HCTPC_nK8#M=N;cwQ{odY z5wP?tO*I2pyyGOrn67m0n{RAgb=Ad(Cw0_q-T=6<%c_{6!{e%EOq9Jt)VBz19kXU=auyi4r? z-;ZM&9Jxmyn9%XGbtIp)k=GUW{c_H8=8b~jL*ZUe*A1`0hKnuDbHfdJD_+mT$u1{^ zfXvW{z`J;h{2MRxW$depn|8&wYr~zr5}%J}AiN}dg+IP#ze;C5d|hhXVkp~YVR_iA zI8^1HHmLMGSk@>%_4d2_PPpmmTkW64l4WRl70l$Gx6EX#f)+G-TF`2;FMILNTOPRL zV9B-P?GBkwDX>{&@cEq+lP3C?C4gNzXq9WTP|Li{13IKiA4o{P?pCO=8J%*^;9R`f zr2@3qv~Xu%lu16Lz_Hjcj~#jHuisZhgnv|lYO7$7c`lgWNx!wgwoxFHe(iGjx{&2k zg6^J=*38O#x0t52Ic1x%Z)smWpN&2dv90k_8ZW3_T1vXm`+>=RLrl*o<;h*(Oc(a_ z@=~uh)!)_j?_8hF&NR6-kiy7mHcV9-PIgdH-|gFfv-hrX-?Z)M(CF6M+au#%v3vj% zVcLW#PSMu<$=5`_sDU*z+s^^(UpCmX1z{}<<$fzCIQpHA%p6;Jj$CO+mgS^`(bTgtIN-A#ELDKQW4s|@|oUWj~d^~D0RU?)qV1Rjsk@hmEWxTAL{0nN+gDk=&poSZp;VFli6>tTqlCH)xRCgI_9*xd^E~;JpG-I7!WmUQR ztfoxyV*!iN*^6`jrv*8bCA_A#zr6KI>t^am&7ZHFumu&Tz-Mc1O??y6Ps|#J- zpjs;jS}A^RQpGn$-YaCGfy~~s0KzfMUVS9U48J1Af z!0PXs67x#a{+?FE5k@)biE8IF^CituMZo5zzTEjRGA_`h)-CA$E`1}>shNIi!7Ia7 z(rQ`x&WL-xVYi&Z$E^`tN>(*`x?f2zQ%x`5siRF-U%yyPh>LstGv%oH6QMGfZH&T< ztYk^&dehLvZ1aA@u|CeSTaCWyH->A_UoeC7BO!L*fKg71R#n`-9km!>Y3O?YZ&gaw z_fJ(ysRdM}@_*bINdP5ffBJ8~Iw7>;r9Xv}dXG^4OzXb5p>dY>0_c&t?zz#WTeERL z(9ouKo!_9!Pv?!E;$9Z$zIryaN6I_D-nr}7RT`P{J)gs;fK}MBuT@d}MhnMHhwmjk z^I@GPv=9#|)F{ovl1YXsA76r+r3-Rv+{>YqD(KxGN5| zR$j^=+W$Ucm!4W*Iub`cQ7r3T)4JsZ+$g6FnKf%2P@+s>TbVx8;AeyTwx9ZqS5~>w z)v9VPt?$|;Q#w&A5KVNpW0QJF4dUZ%rw$6mUmkns|7xB zTDOs31L{92Te1w#7W=0xW%I>b8bcq|Gt&O}@iG5;*|09luZ|*--o1R0`P?eI%+H&F ztFP}ly-)$FzqI(PXmMRNaLKj$$G+8zq@3RBp+FeSN^r zs^5O`wppaL|2v!7>J6e>KbFPQeYQ)x9OK3_o&{T0hI87}8Ad5S_gk&t0QD^1(+c71 ze91pu6?-*v^^)aI#6c;E7itedO5%iq;_1{K$90<)xDX-Lw zZO{`A2ECRi(jKe5gj7^~X&~s!vVm&ycH7ooWh1nc1R7Ib#xKu3arAbbph)}C5&0=~v3^2BRa7!kjk->L`%dkaKhz{7uK4NA zv?b`c>eBICnZu4OQB%$&9H12q#b0+hupeK^D_K0+>WlJRxj^^ z1hzQn1H8HWlGcE9<-FUs6TK4hGM{uRWWKAv-Q=Kw7nI+Qssvv4 z*n88m=dUhyH=6KTeED7Ty7#N*l|KlNTyMJ(>z5OnocXupb&*i=0uq+17|wYSNLcpI zx2t=}a=@@B%GUzJ9&X&?7`~Y{>!g0`SN0Zz*?}e3k6&wAiw{GUAP9Emzu(05W#QN`G~E$xSn$vS%i5jVG(R`*fj34w)4}^x{`!vTRvs&g z$(;qZXj3s!tMuBn^Z~}CylKbIB-UHV@zEGb$>~m}I&JTkBG0(XmbITpzV1!0R4MFg z>ZHzRYR9EryvN6*j66+F1p>wJkC)u06{YqPoKCBP+BZnKJqP&*?^uD3%eFqfPMNpX z#w1tL6V7y8&R7ash@>`YxEblRGp4S*`DNzpoNr50wyWe?e|}RHQ^1;Ex%^$u8}S)| zvjEIaN;~+X9m$31^$Tk3ppt!Iy$zJm20{KSW$8z}lRW5ZDN=uh-SVLmyT{S;f{u2Q zBl8KLGK_RMFgpQd_Tt6W6Ht=i3$@sC@}|pWvSn$(M=_ee2D6B;Wa-0U zLI=(+amIJ=F)#oJI=v%i@|ZcuN}x2(;l`R$H&f2AQIF`c!W~3I|ITx^2)VOv58EW) zeNLSRO=L&>5^is~OvoVgwy0V5V!FN?EWvL%u!(0S+WJl#BrB7w$OONs1pZAV`^~bB zd@g&*6-%rjDb>dh4PBlJ65(WuLUydT5U(DzvFEpC2IakZPC9*Px%94N#CXr1?|Afc z(AxYYx6ygv=f)_j%~qB#x7q5^CxS(d8VOY+BHMY-7MpVR5;yo`eQCC)dO2jE_PXP? zD=|(}pu|&D_r2*PDHTZDDs*M(u~4a>Vv<`E8G9zrKCm`@ z+g|f_8$2?hoeYX+UL=Y)B6xXLIXogdf*#C+M!W0ZJ0gEVsht4!x+mKpd_3Zusi9i^ zY(FPu{pE;CyW|?w^gnhAnN;3WIkcp$bqvjqW_QV*9j9MF$@8SdGd%NFCmmF1hg2QW!Y7f!3wQXRj>e z0z3b7+h4N%YloQL=b$q|84|9;PeG#9d~JPn#DWEgmV}jq%ANZXn(yaK6z6spnVdhg zGwQx?&hS5L-1jr(NR0)@$cYupa7L{}r=j2XtwNfHr~;OswKpV!cgrcTE(3X1$JJ`kJ_JRj8_yB7MnpqT=%VODC@3^#skn z0_WoD*sxvs4(Ijj`uWZG3UD#%7MJFCcHI3^Hn!Mj7%4t^G|qi+&kBu9jZfu`1?{tq zKQA{LJ+YX5RZsP#TzG6MPIDi!CpqrPQW91N_9PM56N$3m&9rbbb&lW42z3(y-o2YJ zZfhU|dr3n!yrI*QkM6j7b5`$Q;3ab<$`)}R%+EweK>-l(1X*u&D%ZoMr4t-XgVi@K z+n!o(Tl6D0xfsjRQCE`sLD)QmJ2osvpPO@Hh}WlH&NJXugVWHqI^WCVV^+=px~}NV z_z$qPdsRn=hNJ0UX8!f5!-13Sf9?(#sxrLktJATCEo9sWf2+_BY&}eTagX4df_FJdG!v zi~zwrjX%}cxHR_8OLvHi4^A(af-s!2`O}pLX=F@afRRYEmDu0$Svwl6Mp5P$5Z29^ zZY<9BYs}1=9*mit)kG?BO&yeCE(P2(;D!|)%(oX&o<(rScmh}KwC;j#$V$_;b>4NC zDTejE0u_)r4wpsvSH};s32a3Ao2KhZ_cb>x+x59}9)${wnUr984(>^>w=7pznu135 zZ(Yj1B@s=%EKPp~Bl9~Rx(v?h!pSilirP9@fB6*bKW6*E+F$^Yi^dlkd2u<)r3-gG z$~}A+q*ueb$2nv{{`;Uge*wD2sAJGn+`)|23$b@k=>gm=Aye+tnruh|ppHSWnSP!w z=wQ@@##i!7G@TFg-L8xVFJ{4kUVq2MC=TJ@sdpcE_4u{62hh(@gf^g|q)YwbU{sBu zv}LEXp=mWRrRDO}2mpkdABQ?iOw4(C1{K&r+m9WXIlOf?%^LzefkMAn8hpeCZ;uo- zW=AOezHpfAJq$i349vwc8$-io;H*L{WGP^Z2Z9?=7W6;8{GlTp9iaOi^rwV_NQlX|}NG;J87ZlHhajxZ7vPSHX+;G-;uUx%G2XP|Y{{ z7MQaIb2Ve7kAEY2ftPR%SS?5yLQ`=ELcmsbUxwD>Kngq^(*zJ$7qXbJ7|QE&017yt zFp-ALk8kzhaoQp4Re;w67?1UZ!QS6skREE7Z^Az8gr5<*bP^&hRMW|-j}cO()n~d0 zgOcjXZ$f3R1AI>ts8z%w1YifuH=hIJdk8|6A7kdd7ZOLLjZLYNQt# z63dS{4|*E}VV_FGw`-z#Lw>Zay4wi|M!xaLX~{nvY{UT*A@}756LQw9F89uHHN#4m zUvF!%oX$8cC;OBNOGNJRSg_*diC@q}`juetZ0PUcS@q<&02}bm*WPtG<)0ieqmm*0 zsfIfPv)UVe?s=-qP>Q{a&0lAsD8tzoNqO8h2h#=+7YqnsF0rr4l|GO?;i6%KhH{8w z&x0nS5!mQgP1jxU@6PDQ;4WOi2z1%+5$M|@J&rB?0FKN@wlwF;z*8n~s_kfuBRtTe z$)LMorN(*jHUNPtU(#+ia^hb;iIAgG;ff$2tt1NR(mYq?LL5M?AY<4;ws@QguWEK> zN{vV9N+5eA5$qBWxa8kPjK&*+Dbhu1IeNQ{Xz!dImjBAWdX%`EOvrM*>_qprDfjH_ z-k-AGYyZ#IB0wS?%%3~=NPks?WzhZ_KYtvfG!V5g5t!u2nxS+)+)M!P$q%z&SW*J5 zBMzRo_0Pft$Ot$!hM3O)&uFQVNV3+DO_7e)aAnE~6sfE*n;qYONdh*DVwFuH$0Shq zYgzhpQ-f|1*V|p^fOm5rH&xu#do;C@pHDI1D+Dbi-?zlTs#&i|9DN=FR5${mk44bm zc}}Z-j!U8uCc*e(LqiAY$wSw#evLKO~X$UCWkODyJ?c%_bfr_V^Cb% z8*UmotcQ8Z(zDGflDz9yve(4I}0AKf_-bp+;0In@)3izaIwsb;J zUX{xTk|Ujz%{0QL06Pdk3Vv%>PJkSh*J&&@TB#Bz)9%_Qo^gHf4JHD~{=>VC(>6hipY+ZowD{$6`BU3E;|D=pgOT!^`u8 zfXMcrYL6_Q0qEE(r!=Tjnvw>@y`9F4xxY0HUF4-*%Eb7Q{^-}Euxr!rX?WMRy0{dx ze={W@I?U|th$k=14Y?@>RPAUQcYFxy2W+|Z$hT8wfcbWL@xUSEw}TGWNi*s7UsMtX zH48)N!9p`m7dt;a0J?d^I#7%NnH&do2*NBy+P&uE7uRb5!0jk9vrUr>sCOO3=jXV% zi51%fyYAAZAiv+oe1nCzfZMa4$wL8MagU%Y1pb%XYiIHpe_$=(SpMo$<>g{X!Z5kl zHPq4Xxnfj_S3-6MBFrEuLsZ;RD!W$sWrF|+#+ttt(Ay%|P0h`*ltln7d>tGYcf#M6 zD;ac)-k!Oe4(5hYFCYdrEy7FvFH4_f`YLNbjNA3t(|M%-w06ip@3~2Ti7n-ad>fmJ z8-=+x&rN#QM@**IC>f1jkiraz%Q}|qg4{p2vsF3GheleiX=OYtwk%eZaP?d#zuW95 z{U}XVb;DipaOEZ<%xu}Jln=HH5Lb~;Kz~{n^ZI$NKw?Qq-z?Xa0rzfG8@dQe{{Mx; zmdOt@OB4@3)codumr=paaAh+JxbFa7;XmP@PIw03sX@0>Ad)CX8jAx^Y+BPNj+{l< za(#mOX7MwJiOAq)1YQQ1q5nnqj53I7G2H#15LXt+nQ9B_OWb((|AukpiwLfJtXqF= z;7oeX&+^Y&d7JX7>}g$B_+B zY)1dfh0rY3;Ka+?o1Rc9Eb@(kcDh9X01F{!5)oSgB;3lV`w{a$P&}{7T~Bz`xmP{* z%S(?S(K8@}14x~z%l{yT^~lH024&K=%s7ln0e86_pq!ln9tu=)hyDbJ^w_@df2EKD zGH}P`ATW3IV*uRRYVb&hZ1X&3b28eqpybF$J|1h&#du1<8{3y0%S{sI1fP~`4CdjgIBNlM9wg;3!DTNLlrQd3>b!()z1Nw*}s>JfNM{_6bqRbQ!G1q zTP65Bgr^opw0w^ocsa9lU<_XC(U1eE3O)s-H)CIddv**$NShj z@IJ06z8G?w^paS6#zwpkP(FUjR<8a)c+j_!StdDib^6;k6A@CP1URoPkDh=t580=zE@kS2}F zr2X%kJAm)JKHH=oXU_OC_T?ksC+jN%tmQj@k##TBaH=aONr)CK$qT3S8vx8DCc5{$ zKq+M7;-w?b2EAnsVW_>~y}#aEKX})rs%DXF(yOu!vrW4ME@UYBv1aBU62hMtnw<~B zm7IhbfmcMh(DBbZUz$wNIi_qy!0LEicQMiU9YjJ1;&wN!#}R^X&9E8f{h&bw1?KlC zH^{H;Oez_Q<0a5b=eldct93#zLroqQS5~ zyYZ^W@K(f=3GR0vMsb6;?@elbyvtQaS)xGu(nERt{BN|!%OOv}YefS1O1!^VHb~!k zG&|r~8$b2C7vSLTY)H=xf)Msl=`$pj2n;Qgh`#au^BOg>T$CTUJY{o6?j)~Z+hgcd zz?YS@kL1ZV!GPRkX6gMpz6^EcFrodiEH}#ZSMtr(q=^8SE6e*g;F6zkFa`ac&F?TC z6~Kwcz%O(7&4hsL&j^pN?RZY6z!3n@yxSl&Z#L1C2;emUsA*QAs+75K66&>s0e9g>#`9!)bX zjvG1xv{)_>aR-YJul~W4CyNI#p!+Ila6S4)ce;4ck>Gk#9%XTpZ32Mm$vW=Hrt=w^ zzgh5Ke`I+hULMTy_53%N1bfJ{W~Ie3@tgn@3IUD~AN2maWCmeLZ?L5A${o&Q=wPs9 zs6iGbfcgYWc6&zVn}H=o<>%zD%}GmNkiQssf`z+C>@pu5c510xM??VNv{74+T~NXs zye9&HJ0vGnc<^3eg0P!3Y2g>d7!m9x6aTl_CdcE~6U$ZJuf%qp3JEu%y)EzaRUVcM z;Jq(drfmZVUw~;Xb;8vrV0O*^b>)1~@wKN3Ifb|9%|NS$qAu_lXSH|X#$4?rh zkN#LxUiRiuaxmYH0@_$HEeOJ>byFC5c7^e5f!X=BD`sYeHu=tpS4_%(-bnDf6&y4p zCQp%tvG}-C+*CDJ6=_v|qgl0Z`?%gav%-O+sU3n$IUZ7JiS+F?7a{Epdvj4IF}(ZZ zVFcwlu8UyC;l6><-`5CW(qYrrDW;ny$;lpkl!z;TD#~&p^I9#Mj#h$U&(TrO8Q?ki z+JLtqtkFLC3GiF>4FZPr-b=$J=yc*kDYvVf)>MaF=9 zjf~HFJvR2XYv1pac3snM+;_LuqJZv|jMJ?x6{>f)IA;zlc(_|Ng_7q67C)jHj?{$MsTQWLM1Q{z1q!2 zm#7s{Zv+E>BSx|Ko?_eOr>Hu3AbgZ^lvHSN{y3&zAFJ~2p3AW!=-8Y>ccs(PXh zi;VKlaw_c8?Z~<;Q%(2AZ(rMu7!N2&AR`wkEFb3cWMhBX2crY%`sO)!{tKBitvRSY z#5PGYMi#qiaZ&IoCWH);Zr*1I{m3Z-RcJVRJZJb-=DTz5u#bl%4i&w{zMBQ$iynL8AlnM&(N7Hg4^Y9ebV9RB#ruxJbv`CrK+#KlF zJxthPwx`u~mrzzb9zFHsY1A&%hhbfBt(IEz|KsUQ=0};4qhtjG-^%3AL3>IE`?hqP zS4YO{ph@tS_?u|-Q0ytAMkbzk=gLkK0?Q1+Qb`hXWJji`d{SX!T6|&8C{JZN`ovg~(@uOH@HzV-a7bn4E3%c%m3d96qGl_$U zx(@x9$2!95i2aH4DvrZQJ5zB=xF+lXHhG1qe);5q=n$?GRgAbns!KY#*!Eu@T^+dR zk2l62+0tt}+{%mQzFkata9UPAnw@~wJ-?AB@x z+!XR)UP|>|md_Xgr@X)zv% zIM`ie0L3ct33y|>JFDurJ=V9Oo>cpA2cpH45~9H$hT+dejc)mr2xc>SUF+1u-X>=! z*<}rrO^g;XcguZ@fz309B|Ng%3bo4>W*ugWd@QL0PLkU@TuNZbw{Rx7Dp5DlN`jD2 zU!fiTG};UAh#Ok-aispM0+9SM2G~Jr>2;O$SN^JeZ%P>S@l7y3*iKHS#R1jCHSs>U zQT-iq=4v{W3WF)>1uk48{}>e^cC`_4LY!5$=# zNDgc|X2nu0>teYYSYo|e_Bnx2%Wx6mN+O^j1#E9#?uTLhv0pG_*fUn58RvhQkOMjR zhPZNM+y}_fffB|ca5r#ND7h+#ZzAcc$p(>wtku;9>BD;Ka3SS4<7vzf2+vnQaKazL zrw?U7_Gj!ec5_N$uVkOPL~h9~b;{HC-y3(T$7S}ZY`FfBVh@3(zSRpCMlly5sb)Ry zp)#ByQ7IKw80Cs!ETMUnMSsh{OpCjYL!n8~PXC2v$%V-76YV8G{Mc=GNHvWSyVjSw z*xmP>Ec()M(W$>I3%EO!3~1#Z*PpER&V0aHSYy9o*LJf?YTm3Dp)?R~hdu&po5(HRkKQwGWUNj{oc3}n~A|50j z(o21@pRI?*g_Zf|`%|k4Vod_qg{>ndi`EL+dNOVJMP;xxn7i1w*gi~y1HlLTwF=f6 zsZCl>89=0+{_gN$OJYPjPloC#aCR^JtIxAaVWnRzT^iQO@P;#y6%~@;1O>Qa>_v<@rT2d8 z4Oe$0zy39>5vIZDff0l35E8l;EZ;ysf$S+KJveFJsn-afH%gulV8EWiWafRb&NBny z#oueWcD#hugUy4lwzWsxQNuc83XQ~+6wZjv$stFP1ypi}g_8f%gkxioE{ClV))Xx#{eQV071YQo zQjv};F|EMUtzoF-=hGY+JIHa5A4HT26p=7AqNJ$W3gbIF2A~fVzmE-0SCCNcCIC< zo>OC=Hv(DcM7*MaKkT>OX#-INZ4I9$*(28`-bG5eS<5W5BUrgSL_Lh>@_>#bzBL^3 zkFX%c_0UK04tQ~N@igVdZb)~H!J>&N@)DRYs;H)d|JlQj^})Qwkh6Vw^wR;N+V`%= z2L!jr!3e)Gd)vU+O@tyKqf<2UIWKI4$Wzj!+yRHCw|mxPyiMAax-y}~3Y%)@%7$vp^m4r@&=L_A8} z!j@};<*vCmj*d84LhEr|bHNv%sG`$f{}0{X)Zi8|c0@ZQd|%{_G@GHY_7uiOH|dG& zj>wvn8KJOD$d35Rq=qBrk3yzoj8g$OJJHuS6S(=aA>Nv6eYKv#{CFY@v$p}_BfN;T zRZoSywBy@V6`ha&vCK2v7H%Eah1$L43`x@qei}jUmV0DJiHsb@P!|nR*iobqRm+W2 zo<<+^uBC=xH_RdT#K_AhdUJ!#Eur3yyP(Y>59G zEimz%^4%s7_4(s)Ur-EO)~O=w2?9P|ozXZRl+eboV#w~@^8N2@TN*Fb{}EIE0$j)i8wRgk98_yo4czTsOcBEbvMq z5e>nr7NiIyR9OZvQs98g63?7{ZnX2Z2_?5?luH+PrCFG%}U~3V&h;_KFj(g0WjZgHAEYwxRBuuFw zO7!fK^5uG7&7l1q{2V*=legO9gN6Qk~3!G%GOjxP?H&f@3 zr)ocvZyRi=0;U*u@`HltEYuLZlC(=t!~L+ekp$6-@O_*TI8|AEFC)$XcQNE`kMh54 z-iWJ0WtN8wn+;J9kTL3KJ%P=@WI!3OmpiSCnRp!BG!H}S zU7zb1ECN>Xyn_Toe}*pc0>6GVP6UwcE{+t(;{1-b>grRq4_^Op#8+FiA)X$u#Z^gE z4qdbbBQvsqdKX?T^T!Q$yEm7JJmZy&Ik%w;9mvnGoNypNxGnKz;!vWxhkUMQgt4ga z8vf|ox(mDBVym~yx7(UDW10m12^o8+u*rQ7gWO?fDvuGhLHBf%e@PF`f>yz2aAr(h zfM}!#x4-(_{r&Zg4E9Aa=_4gH+%xn}n*{BuL%T^)A^>~~Hkx;i!wmc>D3^f}qm$}M za>soxSaM(R?TtNq^dH6;7NYUlm#31W?3kD~XNRmmb1E)k_U2$cuFMcRDJ5N4_Y+wx zDK>|#KX1&z8XWDuUd$%~x0hQ3o%vaO4u&%lj>Zu)Iu+uqpD}aaqs!GhZx@H8E4u2+ zAN7w5in@R>Rrg_ovhwUXR zAa;r8EX}kkKT^kVe6;XC#Q44m-V9fY3&#;#LKn`$ILI0{&oXu5PNsvnYBSL90Z3)B z=`P%RR6*kvK`c;E$CDh42=|Xf`a_ToQYGKzFsDs|Z4$!7N}zV% ztnV_X6NJnQzD+ViTt+fMRJt1A(v1c-E`vg7IEVkjC+}fL=aJCibxQfg&SU@bM~(;!`{cVs=EoR$l?`4M*mz9Zx7y%3agjKySO2W7 zkH&^$y0IsF4DfPL*z!YIlw|49l7^**_{0!%1F#7iTy9iyY@5ev2Buw)L#CRS-ZceM zZW@T}_oq$Ine%-Kvbp(~-sdvNdt_C>NH7)H57^ggEZdKaV`x;!H!38A)c$edvu0r{ zBzn2??=HJr0ln>n8N9Yo->k2;GGs*fv1}t1gjf4>- zLE12*T-cqbCHL38PokuXwaKQRqe2nWu#Z~N{Muh{nwgLKEfdk~7`6`h*2jOcKKxHG zhh2bwgu8j#eO^e~Z6w51eE4W60Xo7S(G0A5Dcs(qf3Usc=#p}&;hEG@C>i&Skc`W6 zf?4IdGBat0zc$q4BU)q9N@HJnMAgjad-zoEd1|2BMBIP<9h;f(ftI;IRb*cMw}#MZ&SDc^U~|j8$w)|^#!H~ zm(YzKjN(oi5iJj3W!DEe2W={$1Wv77fJcx!V^&*`q;Dnv>2ats8l)fysu%hrZi-jO z)8NhVSqD){7OELur28({opkrvg@$2}t%JV^=<>D)!S=ODqHLRlxrHR9cD5yxI^=rtdef@QozpdUI4&S`q^(K zOu!bn)L>I2g2aM(#hx?jnIg;7-(J9OYZc8n*;ATeCCEFZG(DAs&axmV;-ZgdBw~GA zt{^LBZ1#mkx(0t!0IdEs991wm@L)VB@O>P;oGP@RQrsM_5EOh~2y>2n0-Q7mGs|U+ zx@Z+@buaz-P$W>1w~NUk8ej{^gkq7@rX=8vWF)W5`i?4>!d|&rkPl0N-3`DL*_)Bb zxK*zFYRqT_3Xp=r^X1TAW2*0lwJ`L|_hCugxsF~(I@sW6fJ?Tu>iUASU>V$B5u8(R zU}73>`^Mz+A*(82a<0=4V+j{7MWzk!AQ2+(M^Xa~S0t?PIpTJ(KQ`!z+I9BU9}Hk` zS->59-CXckF5BtRy8D^clI8)U(pMw6o!RAa=El|y#rX?OoPYNca}>*n@yDoPU;m)5 zd;ntSw{W064pEUt@mnn9f`UyaV;I}@$=<4(=0SfZ$RJG?Fvo$t=pV*fmcATC$z?BP ztzVxOdm<&rc(g?&g;V7=p&+1d*qQO$Y>+QmC&|$XEdC~ zDrZ>p1d#f-Cq})2#lvzU?^7#tpPLt3X3l8as0%!$5caKTFpKF`9nK3k_G}ZsI7?E& z8TysrT;?KjHC`i2Z@%ic;apz4xV?Cje!CLbwgFhkxLmY-XqWLb&JS~s{l~{P+PrOd z>jX#oW#45KLv{*?tsnj7@uiXS&s&^+6yv_Y1s8{PKS>RHc!*O-=i`azfMiUAG@Dm% zy7Yg4`CZW;B(@yfb?-wyIkOp-AC^RtBrl(>j(#N)y|5=6RmgE>y!wz>yI*P0O$N66 z3(E$(+ACT=J|J0Z1zo-_?ATBo z&1Rm`!bhxDd|MfAikt_SFpcZ(y;Xk*`?V04jB`UV@Ul;g#K}V*uoD(EHZTPFsHD;@ zrPxsPa42D8$@-C}r{wkJsuHw;g^bmyo(1esgt+|O&`DoV>g#sJgQKWhaSAQr)K^xp>5~LjakxW_4f`P9eNv~>F&@8lgOYBtH|k_svTc_ z4pE(wp_*urYLG>$3j1@|L>K-efLoNW1*-cZ@5kX{aqYO1xLxOvVDj2ofpxN=NAIT6 z4Hl3bYq~w9KdXFhMfH{Y#X|Rol7ua9s0~bpT0VI7wJ~bme{9V_usv)(_{!@eHkoZr zYoD9ger_1L#PwFha0i+-QPoSN6fbUF`IWN6Dv|BV&WG2-JlwfYp9abRgdYrAD!u*k zxO*yET>;?U3Sk?>?{d3k?5|$7dJAX0b)nE90ForgrKwqc=X;z>M7QPi=;KOroq}^lRY{~h2F$Vtomen*NA-@&sn<5kgcdLJ> z#rwte@2XcEfIOOFZ(-uF)n0ygU+^SVfz?#9))n|o{R7Jcjtx5JLZZ96$_z{ugE?O9 z86Lg0dcFTrx5A-4a^(cHTy_sW2j-vj%QaRx5+;16)wG&sB9%N$v8yg{zUxcy>Z8_t z5I4~-vR&!E=x6fVh?T2xty@Me`8kZI)K53wI+@0;(1YWc`jZLT8M z7G%t_h1zJx1;Iy<7oFy9WQxL8lDm+(96qe>qvTD=t03x|rL}zSpkzXjwZ}{+?$0*0 z@iLB_yI+KQWLmJ)*h2UM*hz7$3RdlXC&Ej@l|dFNqt4NT#60ET;p*(hA~`F-L7T-; znJx1aHB9T9p)PPY+ocyTgA#Wky}1zMQDtJKCXwdWQxp6WVa26-@97IE-ahzcAi$;{ zGx3A;{7PFsB%d4hreda45VCUKjo0Vbpgt#9!SA zd&Z9z*w;qO#@*ek>VUh^9k4o*Jd#8rD{_;6cG1b0#5%~&$W<_M>8Tnv|9$hT;OQ=G zgCImOVB7Z@!(QFtSO036M$DI^4!~;hK*);+~L)NXfA!HnhhB`m zhzoxl%@@2qy_Y`RF^FKevgoK9xv>Sf&-=1~*u{e?2p0*gjf9j)jpVKa7w36MbX6J6 znSF)krxYUOAOeseGIR&nfWPR9Zn}|h43PbrxJ4+?i^+_Gbeo70?L@N0c|#1%yHqIV z0oMf7cRjgwvIuL->$ihbwd+athr?U$>22hF$-78CH~8b_fw@8@BSc6fNDmRu#Iw8_ zWz^>?gL&Gl*rTO-6w3@)p%j)5S_VJQ;O)Y=Wid`CI^GK0mL6s`e*+Zu8nS@sdtxP; z4C_YMK{ldA>?}{C4NH_qj9o==R9vEMRO0VV-_gK8oS1 zFl7PdC=Uos_+!ptCvw;dbcv`s&;g+!Yz5A#m*G1O+$x8~3m1l8J>t+LMj(Ye?6(hp(ME4T6_qh?T)Y<4dh#_L zwfp`)?2hz%XEP$;OIeBidYq>=yAhtvIR+_zm@;gs+;oMl>ly8&k5hVCLj+=UXY5U zndG@yzNx~BATtO95$Z1zGki_P-Nrq4-XHR{J9|B`kCW7mG&HApfqUqTQ8Yn}-sGZO zH75CMBeLZny%KX4`x<+{6^r`-;40@z_v+e4SX4GD0X7z1v;CsQ^J z^8b-EVX~`x@Pb5{h{<4nr7(N=Z;c2Za}i-bG{>)LE^sSyrsCuE9C($7Q$bzBLCC3w z2eo#zJAR{+<&`buywt+8^-sDr%CJji7iGvnJhGXIdkM0E3Ao%>@m9bZU+Zof63oIS zAuc0CG>^QRAgR>*@G|9ZG+M5T=H#kuyVsUt6I$T<$%E3v@v4&NFVlKcn5Latu zB=F`Kme8=G!{p0g4mT&Qu6cWL2*j%KYFwrk;iyzmIG=JJ7YNcLwH; zgi<>W+Gr(w1VdjrYBcN;#2KvIG>iVZqz_4~drN4ur5Bo&?y&1+JOa_A5&Yrok9W(q zcs?AP4$E8JzZs2I6|RNh=Y{b!BT~x#h)u1N>b_`enKkwUNd`fS=o0gih^P$+S_Hvo zAiKP*uq}d@MlYydzoH1(;U;#|U*6OzoDGr7-w;X!1SSGvQb2A?n}^&HUxE+agWSx6 z=q$z=dmJQjgfe+JQjm1nqMuT=)!aP@U#c)d$uMSxc1;D9&H+#`;0l865c9zjiI>m3 zgVYspR3-7+cnNf*s<+e8Ehdn_oLF&O1!hYMD~~lY@(F@fUbP-r|C{ydfnm*uuK1(X z(6`H9o4SsK(h4(Aa35&Mr-arHDU!kSOyF2t!7u|%i8Kv@oX=5I;XENnWgP=>Yni0@ zGX3jDHBd8!%zd*BJ6l37j8wRa-=L+-^7&Q@S8*V)~xko|10a>vf;Z(o5v8Fr|Sfw$vu0PryB#zmAvPXM(7gxrK!5&w?R z%Y(dwE62%(+biJ(Eg+#gMAiYTkKIl_@RA5k6T?wbrlLAY z*R&EQL8E;EqK9MBAvikNdinh8aPa6_2t9U(jM?!s4@0I6T>s;4VR&UA)8>rP=k2mdrF%b0JXME*)M=tAB=8x_{A|oPbZ&_W2 zKM8ghp~+$eH!p|1hBUzOgb8*-v=Unoxab7zgMip^(Vx-1y4m3fwco6^mj6xeY~P|< z^`l68)l{V=PWRI;F?RG{T&U-Y3qiRAsTcN|S?%0sLok$L-4GXMESKE4A;=~U08V!! z%2AV2-sB1n6)dyI7t{HZ4qVN6c7+>UjS)ReEm>=o{P}lRr6`Z;|CvR#4$GQp^sgcB zAW1e#_LG|EP07E*gcfWkECY=(ct+N`eiW!xfWb*`AT}-y)&(D?00>{LW`1ef8B>LM zzl#4sr-~aCfmAjf$-vuF5QbRvaMuScfl{LvcmDS;kbt>Wfl1hQ6@-M=qa6OGrECdA zMP>gEMg)WP4IJdk#AIWi6!*t@`Uw{g%$OEQU^F;p59 zK{{b?KI;HEa=IvvivRJ#$Td74HwS>G7hwpe|JxnS$9RDZcc2I~D}sN5KinBW*L&^v zAAq9%y6}#%`KW%>Mr5_sqsv#?3Gn|Z3>qmGNQkg9vHq;BnYTg(Yqf&8WH~`Or;8nZ zrx(x_cRr>UL-I|Xuj1EpP4qgj^^K?C1|O0cF?dUI(ho56X#Dzf(UHBlZDB2FK9?t2 z2D6|w`6s_wQVdf_Qq~g>-{$=rjIyvO;M=hCPye7HinjrKTg1W?8;{O(g@0K?2!a3Z zF+d2#`0Z{eSHrOm%#XW-{oennNQ9@H-o^TPp9Sd4zPRYirjSe0)5P!Hh*=GA1P_c# zL{cHnE&*!GtG~sen?O0CFcNTmgf3hb3V4$57(u;?gKe$B$8rn57;8NJ1I?wI|00mv zENOuDnS{j{+<`Ze&Yla!yGk}v4YIQ7m-Ba1hyh0lt%LML_G3H2e;8bJr{XYAF|`=y z;18FUzBv)F0XtK;@)fiw&5IG?d1fIM+Ozf-BZGfp>YxTX5anNg?P&*IbNsdo5KXoz zf*I=-JQ5L`z~@l~1i*lI0zaU2x$I&~{7Vcp)4E$wW*|9HnNv=eu7o>=mRtrVmM|Iv z(8Vfg1R*RI=8n+W^$U~^arwvTfy~h3fE;)qVO>kZy6}f9;EVP#@idX1*IWsBbB42tK~WUCz2;IE~L@@U;TM0C&lg#VEh4-lD)Vv*MA2 z!ss|bI&6g`J;T8C(<3&VNd^!DQfKYUE@D6stQQLx-et|j&;B2ms=ESXh~ii9ftM@& zWk!RC>%<=tZ10l$G$v7M;s+^-j*Zkrr2Pkx7ote~jhZy{>1kGTd_Yzd3i*GYqZ=U# zFUvzx-zs9eEM?~YAM~`1#+=-nG;zW*9ZG!wQFcHAXhYP{vyM}Vw7s&jC5Sg-skJa- zKA?0lea3hTuqpsG6@srDH%IA?CptYIX{$h<&n;@lbdmd_(M5pA!^!zKV}jcOQbZ#* z6}~mt3H8+%t?c6LJpzqPAfiL^WY~eoQyJyLy8grJ2G#|g9lpf!eHLloe3xdqkyETS4Evk_D}&_gTF82 zDlnWXqDLjWJ6FH97tsg0SxASpW^D;A1J+53lEtHi516ZeLo7=Y!U3y7;JXGU z9c3d}CA^Rb6~*M%xWHkl2yemtlRKl9-a8x1w6y()?BlhkmCfZrk~RLVF%<<>C|?P{ z=MTmyFAV-eR0U~B`$+Y;3PxpK?3*Aml-kn(k%0P7G< zh2PGLr?a~J(oz87_0Q@AWX)7?CQV{j!*0UV3z!zKIGs>^ZPVgjAW1x%>nq?JeS{z# zXC~f`O=!Bv0qjU7^$yK37`51IrDJZ0D$a4j6d(t+)$fRce1ZBO?W%+`uHjs(G>!6M=+TFhlqMx8EqMCh7}G|qrR%({$B{|E=H~PR}BlU z^z^CPL2WFkzCDmACoL!1KmiIU3G`SB2!I)3b#F*6(GQhO05g4tZ4J-FIznjV3DBwc zJy`mGAQDIqp3`kR-MWFD{|5oN;~t^cCSyB0CdEMSaw?jW zaw^u^FEd&TJkWxcIq|g5^ds}5HR8Qv$bllUop5p9q}2qD{ii%@071e5 z4ORW2uh6G%fa_ZaHO&)sqya7c+HDpBytNyr_k>eJLJ#gUR1v$lUoZ9i;B~w+b&9SJ zcKk0pLE{=rKY6+iTmVjoxuZ(NregODMrI+Ob0ln6IWRi44w`=B7;gU<ra#u%_+J zXTNZ)+vBPDt0Y4q{Q_GK8l$f^Ge<{?R*3xU#Rz9RC^!lzPQ})GmNG!CaNBCk&Z6Rp zA0`ST5zT}-gB$3)62X z#JFDtyIS2b7$pojrXSNVg5P_i63R3`Y`PBs2y{ePHWEH^ocG>ODdYJs8mQtQ1Xk<3 z+=IFZa~jj!)6+YWSIlzgfiy3hPP>>33tx_S%>!A~?J|_3l850EN9{xJ97uXHko0Uy zKkK7k6YTsEyk@Axy;OwxKQbc7qTs6o_F|wyv|0n=xa`z7|1&ferb9~yb0l z2y)lkv_WD3uCNfYf3F~{6Lz6z=&#HA;0ph4pQ*B!8 z4Dc@kBT4#Dp#md>;up{uSquRt5c5e^a#9I;%Nnv2M;sA@O}qq^NSL=!nUxA&e~lNl zdLi^4{$2?|)Hs=>#B}9+w0?~cWPO0KwvI7eEd4zX-6hV{ii>)mvdvM1Eud5MWOUZ{q zVw3_)sSpts6D&~&s#?0^Fq;@S#IX>)oRkB;A@Bl#fobRfv=s!+5k}vw4+9^yMSq!i z`o|tG+MrhcKSR6fb?xCym4JQURN2?goq8ybnRL1SPBT)sVlA z1V{u%h+#6xR=|{OO@~a;ee$z0FvX~~zozJz26l1&dM-HJ3rOXkp{DH{ka#M=gFZ{cson>yu7ka2I+8+v?KO|1%qc$6O#3tc5~+7VbE0bUhw3 z^Hv5HQP1BcQ0&Xef{%KI#mdQDyx{KFIDz&{FeJfT!FIuvXg_1=_qmieLBMDGoZkR+ zqsg+S7)eEqh89Bpb*gG(VDcqYUNVqB9wL5XEjOI|Zs#z91p{Bsa(CXi=ZH{l<#OnT zb_rGp(y@uK42tSR2ZW%3KX&qYvlc_GRZXrXOh4(}N5pq~k3{u6OOdypmsw=>F0jW! z4Jxn7c=hCs?z{KKXs#T$pu8&g;djR*-fw^2^9g~J1)2y?!iiJspY+Il?_YYKAlIi? zBH|31$XD_{zR^!vX`1)nuZe}w6=oli;GA>BvtdUmSK<#gm<#lIg%oy2mTqL0Jl4-# zd~>uijaHH~1UeGo6V@AjIm2hJHQn5&Y7fxDZE$+l!PlSXrE z1525+tJN*`%al0@?zf-+kf8qX`9Z-=uFT*Oo7Cqi`Dr0``RSpbxKe^!W@FVlMn2E1 z&_Pwq?N(F`fmzVD0CUhD<$AyYZCUF!nNZ6aJS&hXC`qx#`1Eb=SJxf8<5FM6)7+6K z&_@{1?9Rg;kFJl@t9wwIPnM?K;bpufJ`;cI`iJ;ibY|bSH;i_fSyIB^Xa}oOgXv>Z-UL5sRnN7>#7nEG zWL==c!H>t_Iw7d>?MuboydJ7Mg`z(#=voJmwUUELs&Z>cC%10Ap;=JHTdz*`Ux?72uyw^^#t%bz^?_OA;u~)LzCYi+h zN~8iEKoRk})A!9A^dYJcB&2L6bbZmUXFGv5m!@lduWy@Tx15{&NF+pOWw~`3Fn;gJ z${xMCdzqr@%>6Y~e{rlB(tm_9zCG(A>AYw{xj|c28{%4w*1Jx*(C?Zy8f6ePUc9D*WoRo!BJWv{j z2xhlfYO@;2BiITkA&@5y#h5D}#4E>SwOZH{zdKS<7Y=&funGT zjDx$GTUQOR-LANZl#{s7xLEaDNSK05Rtgxjflzm&>_Qj!(e*lydIjZ8mR1M0DP2@T zi(7G8%<$2XZ2iBfQ=;>zQ^l59>9e31g^EEkd7nI73s}ipXIm>Sui;~#v)v*BO>kd+ z;e*uGf|Z4D`!L>w=IrZs=`lO#8FYwhf93dEe)pHq&oe=d~oO@zb7y z@AkVxEflUNWj|=Pbf_$@;b0S7*?>+HV4r5j?*052C_M`i5L^)4fKCW*+D-}60o|4` z`G&Ezj+OU_Q1X{NcWD``2BEUuG~e?ss_-QL$dr8*PEGxf|h08BNaD zkuM_+eHRgToXr}vTUE#-8#6Qraxv)J2@0>A zRck8cxveiA7cViH`()jo1voE;?z(MmO`NDR6+8XBD-o4#W3oC0X@PFcH&GSmTtfyB}m-Z=#W!Ml%eO|rd)~%2} zsa~(5?tg|0wuKkbaMw##0YrM+*Zp?Za+|>_@(xSEo~6JquNh58(BlZ0ELz>zJq}23 zHGx0)kdjo8`}NE8ixsHG?aoKp)V$)l-?ZOEN&fl{Z-QFUl!2G*N|(}E%IAn^b@VMT zdtDwtI3wKQSvJ>=pMc`F5|0RBL7(4?&PwF8+Xh{de1i5KZ9h1u=p{Tjq=hm7c%V(+ z8nwr(`YcIFzvp$_K&15o%!9xnpR#^0rpsqBn#7bcr(D$0g5a{@bVSaRJ(yM`Q9-7a zFI83mOixSZo-2j8CznXZKfHfg)o1lB7Ty(Lo~w9D~@Fp;Q}KgIzBF6pK+OYF$=w_SRjHbh}+pj zi22jqK)KTszU$YEG%^~`gV%rV)<7bqy-ef38YAlQILeE&+&?Qu?t6y~{q2-Tqu3%e z?=FPCO5wdY!FBjkqpgr#TFro$^6aYHTwNORVd^D)3KLijBoS-V6k#+bQlxizqU_AWMrKg zrD7uqVqq}iAt{aLd6mprmc&G64$-SeF$8l)<{P`x0z=`OWu*qv^Unlm5;NTeKpI%{$p#qtX+ zwDABffMxp5JUa7S|E53P^KEan)sEIpTn#uw5{B&O`g`4{a)yRQ-|Y(0{HYSt9PS;( zNIYnH2|>Qw*&PV8pkvN`Sugf={}bK>_1_b`>=JU0r}Agb9SLD1j@xWY@jyg)Bd43@CA3ku2d1030SEyh+dZDBnLKW`Kr0=la~zf&X`38XR=e_ zO%bY$nF|mfuP~iKA~*XrjVVT7ExpD0d^l(j0=HJG)7{HnBe3hG#Rf&Q12W&o`6c!n zQ=WZBL4i{SZ+t(C-e>j6IV{1MFQGq+q47-W=WkXgOa1Jp2-kwmdzfC=XySwFrVAya!*sr|K?+R2l*&~**l+b_~ zYy}Nn=vX!k&6UhaU9_-{stB=F`7Uma#xGsQpF>}IZ`^{Dw_mH%y2}NUf*YQGk;`aa zg}QUz_D+D|s$NX-XD}nL6m)6#Xxb@y{Fo?qmF7=!DYJ%AXxuK3s2tFLJw{1rvOjW) zn*F`>OXp*f`{(0EIbZToBiOG={gA?Xjncwv~ETD!Hy49;{X9yt4$ASv`h&e0E~2eKSx$?`Yph3 zD&`wC()~jP+EX*%?dN5jx~4V$vgA;Y(k?vW1$*m#kBzYT=E?fW)tg{rdnJ91Ac}MF zXW!p5b5tmd$RZ-IIU^zw&eQ@5+n^dvBs3**#9zTWDgB)*ZT=2<;=*eA?GY4d+Al=( z5_HIKe|;wL2pgR7dT_GJwIX_NYOLsvL9)}b!J=p1!XgB15-H&YNqQh-Fp^1w{_8@PY)%612MRq9>R}+ zI%A;oVnDyNqB_dg^R*~Bcqtck*?0}XfMCb~o7Pfh7lEP$AcNsh8Ynj0)&;kwetk2) z1l5U*p}B5LGF>gl7*L2sHU3)UP+8B0U$G6c#OJIVo7C%~E6+5FKuGQMczuzVb}qoe zjyjlJw0f_THRbR9o0`~3i&$1jO!17khZ!vW?tncQLM#%j^}O$-uMfWFgJp3rcHwni zm#!nt*|*NTpr#CZgIl36|BSLtk|h*-u5?N4|M5e2ixGJvo3cwJWo*btjEnX@EJSfKp6iyz!g z1RE4-WtdMGIk0@DuzIQEOD{BG?zA2S%ZHi~7a@ofI>F-&h4UzZm9qK9^asX(MBmvN z`!J_7}AOba>S6)AhSM%Tt+Lli3@lX6AS8ur`e1_w2MT081jZSb{Jx zdGdgyC&EJ7o6n3tPE`B~?M(!l5nyl0#`sbWNjY-EvE|#IxzW~z=lE2<%Z)6OFfV{ zmC(Cra8^ohT)7meyz{ZIXcn>dU2?=WeYt3R31FZt7SgAhI@6DNHzH+_j!_+?P)Odk z{hw~a;xaU-z~}jKiaq}O(W4LX6xvDXx6l(@=8naLXSKGivJ0LH&JY_d#-Wwg1hx09 zjdjtSV6D>&9f%qG*DO(&5t)h4$$s@eyl6JYVs`ArLTSI*#+#1wk*=!|#XIrZ1mbAL_hPtuO)E35Bk}2j^wVLYNul zaqs|Zz%|9!Ob{@s16~GUgRZ2(j@4)F3-S!C{4zlB>i>{uDpBtTp6yG6tzH|c+m?Kp z5P(b51DIZ%^&-f}L+1~@)Ov5nH)xHXGL^9Xss6O;%9=E-gFq!CV?vC$+S~@DB`NId zMH%B;!%zLscDeMgAB^NIUy^D{C%3+bK(}D_3al+ z@kl)!qls3!2q^DQCiP@CPNv@8_Hmx8Gy5#-ol7XEuL5>h9Pv%l1$(+Wd?4uqHmQl; zgCS?MS)DXZ7Z@v!6RzN=6PV&SzBqQg#R4&4tQ>p(XM~`~j;VJV{Gu;u8^&WjueR+p z*m;BDg3_CP#brSB=jIP?55>}AdMmdX26se63A^XDA@o0hnHhOBSn1=e9lPM*$*t@$tUW^XMj2gqm51lOFM z++{ziA;A`5yExyDY*0wxN#=_|B9L&@d0ntUXB>=BZ(h0u`0Id(Ag|VsU5bk%4v`_U z=Yc$S#tS*sjQ^S70@Ggro8?F|HXuuNG*RcSVvcqnV1>7fd=bZf&gO!2v28@Py6e0Y0pq>y5|ks!_WtEl*mTUewZ$~B2+E7$ z4?R`|IFJqGGcaIt+o~Y>pUyeT;WD=?@0V-5nm(hp?T7&d?_@qV!^H_q=8qB1fd_2+ zGJcE+>RoB!r9nTUczJGRj^l;M;{NGSP&uP zwV@dDkjGDggX152J$vlsb@RBuL^00kO`dd~391K0nn$*Go-REl;a>%>s;Zg_oa*z- z?CN^>r&Frc&&Jbp&2QlFy3IA^DQeM9dhU#ryp+%8{ZcoAkvA}Uh@zm(ur6(8n1X_o zUs8rI=9}wqERr9UjTamZJ43R3OkJkhcqSMJ!#@y0w;PLxdcmr(g-3&Qgx$zl;6h}> z1k>)YXxOdougMHy&_TDXA!@^HWfV{GUUVh0zk=a|Pu6P(DJhjvi94G__DH9@Sq6uA6K+O(= z|MW{p;jfdh<1kl4_`p{X8#j8xTsmVE25rETXR&VZ1UJKeAr|1~%yzK&QPh!K_ys1IQd1qNLTG$! zuoB9h+EV*W)SVZ47Yke)-%;i&?zznS=wMs8*&eT$Pe#FPIWxX8EBvPRiCFM6SgKkM zb9yQgTsRbC1&fBKBje~XvKFl1(`Y1jY1tZv3K9tlp`$gu6CEJw5s+o2 zk~~_}XC)<>^CCC{??%sd8*J`)Fdarb{K@A?9l8V z?Qf9I2T<7j(f@lNndJ2W=U$FdHK@j{<$KoZblP1H)KhkhRU>Q9P{b`OQps zA++Foc*Qa@wAxv+s@1kvU+(gYojt#IlAjf*&640LdV8rn2yR|AMZpRKT6oKGn-;_N z8T}lVsFEj|4w>&Bvd3Yaz14n%)M;3b4QE>P!c=v2Bg`wXbEKiWoh|?`ZG3xbGt6O5 zjz-Ii7jB>UU{+#(6`(Tp^E6WvzCpT!;d?*9WNsI^&aRtrGzBzIKD;up*O|=l1bK^P z<5F-bEYU$CPBZzX-i4x*k)E(_T=T2JymQKsO366N8Bu|-Rk%O>hl=404Cp=H0K z4$LpJDW|ilU05J_ge0(Dw*EMF@ExI=+2@6wdhly~$sF>!0dY^?LT(j|bX1-`aaMC# zSt3SaV{5MKdD<({H)t82CZX5_=s)_wdf-Njt=u$HKr7gI*RX^Ywv~v9y^Jwjjo?wu z%@Det>OI=4-zr&=BNCT=*CH>(QTYE;9A`!JSt_Ku)LMp@@PEB`{#u+K2HnLC3_Wsf z{G2gOTOcl{e_v3PvqqwGj)sHu6VeyHKr+n*Q(^NFB-VzWU5K5O8N&q2f+gc+OSwY9 z2ZhA*&u%W@z`BivIl+tSDCHpK0rwGcxj2(Voi${BW%$+Ki}*pBVbX{JGMrY=UN7u) zR>oSxMJg+GtOz^{ zG)3SF@~xE8tk3h4wqgHR=mg$Bq2m$@i~G9BtJu-cK}XAm$m=i~xVe0=X&|N46HYjJ zIUPD4wukUG>HYe(%bxfvGw?(cvhJLRN)g1_q`@V%Tbcbwmo5U>5Og033&fD!9q@pc zP9D$DisL7vvZtPR-a)*6Fr&7q$D&>M^-5ni&tJoMuW3GT_`QB}UkJ}PrZAvK^X}a0 zrW6yt_cP2gNF&*Sly2H z?*{>g6GU+HiibJ>JE$W;_>x$QVIfjKRWPu+@p2%PRto#C!`9(5v^G-A2TBtmv(?tW z>f!kL`Nyas?z0W*7kvn%Ls7wEo39CmU&PG~@@_2ix*zYvf-7bigI8}t8bxKZ1#`mI zY#-k5e@TpQlmL5*M`c0_#dQ{)mHxJH64Kck8gBvBDDFKDw}POv6RAX@BPKPH^{&s$ zhMRX@)poyX6;2d7VXAXFjVT`4W^?F&-}9^ucW-(FeecG>UejBnlKXSlsXB7c>;2(Q zll9McXscMg8VUcS0oh>QcPoAla{=g?Q_$JjM|WRQ zs&09GGwGG`ovlKi`5iHz`Rx%^Saj@+&~OSLnl`(AQD>0$Cu+TSe#s+kNM-G$p~7ND z{lsHg$m&xj?OnH?y9^uK=q%R*3>k;NG`z>qQtHbz{p?xuak9JvTrgcKHfZfEvZd{K zdSZWSckz@Wb6W7!2N5phtV$|eq@BXE+x9K#(M^mpLN`X?E0q_H@OZnA6eXkpBvate$M>`l8y&_MVDzwjrW|8fDe$s_!M#VgCrx_m~lD%QIkTFny9 zYt0-l&bpcp5O)t)F#9LdV7G9iI|Za!?J7}Ok$yK?`wyIcaodO!iysy zy`#9S6;o+9t4fSXo;i)<7Zwv=aogCuQ*^hKg?Mf3S>q*>oOz710JZf%jAn&P|F!rs zUHZ1p{_6Cg$J`c>@6*WdU4fJ1r2!v{me4r;h_K`RHBm=SL5Z(ShYmV#2!Cgp|7+u{ zO|tSlB9&`K8vg~WyRZMml7m%mg22~5F@@;xeR^~Xg_*w~{Wafga@u%-+r-oZ_mr!M zKMz*w5&oWRe?f88>A;(V@KzYJS4j+Sl7O`H(~p#{YeP~6K@R)H3SZ}L3M`k6#GD=u zh&jqP>E};;ZR(5vXgx4+cms0@;Q{YMxKN@DV0GgQZC5Vx94)d3VU%;p6oj$v;T6r{Osw!xq~mLAd- ze^d5pcWT(|krX;;{Vei|SlX|JoZJU57rwRb)VbIYhKhoiMuo>QvZ0`mR%} zPW$H8tCxzQE~Q(amnC#KOmx0$gwj3aO;T75t0CS0TtCC`$t!KD;aplg%a{euv~bV* zx$`M{sj2sm^NUB=8x5WsgRWI*)eEemWKIzb>Ja2@L3`8bVjvYcMV1!^e^f4i)r8&J zd{IvmMnVDMk4O#HkPfPLWiO8CB^z3b*)o%$w|AR%n4UW}Y|X4o2R)c5qb7n#8wiW2kOAZm6yA=>_k+iqy`@Gq!>+xp#?6J? z#)aG#9nYT%(AQd=ffN3#k1bjr_nv+@#qB0rkR(}?sAT5-SvY6qH#>SsCh!D_Pu{6^ zf{YoM?291X`!Uw+AwhS?xtOkavVq`}-j&$+0Q-4`^8jypdaXVY-rB*p=n8B*&8|s2KXa#cZlM$!3iMi^RCw==KTfwj(}wGP zx|KELbUH852 zDU-TQGc$R5IZb^`bM9<&xM~r)q1Ji4zNq*9_$)kvO2GK-+2rv)vz6v=$xxc}35&}D zhg(P0YXoFW@_}#K=_;vq!=1bvL=21Rax_&7{7jTp)r#gIy}s`0_}}?=GK{2uSuo8WN_eytm*30S9lRt zg3|O)@jUO#9%&(dT!Bpf=8N=tt9+b@)9cRE;WfqsXMSM?+F5J{D;wvNFJB^oM6Qp4 zxcQ{+SM%>5SA4eS@LyvqgD3j9FFjuJfqn2s{rJ&rB5xI`sCD;%uY!U?cLHx6IsMQh zUFv&BH|;j4YlD6U`roQ^(UyF zp~X0^pa<7%e+agt52J2QSuB1W=}u)AqN`g89r@PzV6~@XNm$Ii)|A_z5V0g-pG&+N zzPB93Xd?cWAMVy|Tz=!`4&!yfP;Lcc%dQ}4Jo!?ih+gV}ceQzZ`f3e5H7*Ajr()yl zBr2`^u|G17%$M`UXr$`Hw?x&}5DT*2)xE}y=Z7r5UiM)$WvwykK5}YZnNMVmH5NI- zvk2#Jwbj4C%19xe+ayY<%GI22;l5oT>b}ykZJq7cq`EZ4RVIguqA4}o!5>?CNceWX z)$EAE?RNsvlgyFWUh21Egts!NG>yGJ^nJ>$u$xdfZQhEcyDhm-BTT@|rM;MrqA}!2 zp+1*ACSmUiiq-3i%{JTrQ5Y4fgJKDJ`2efY;@c>t;8kit1GqbxD$qfQn3WzmtI@c`1T|v;gczcia52y9H%OYZL}E;SF8T? zMV??s<|tI{G2vTOTO_w(LL z&KYialxxf-WttRVQsFDR+xYTS;50lsKqt}&ry+1xQPWN&r!4}TM@pTYg9Bjcr4 z4NY}2BdcQ)Wo>m+J0CyI#xxFEd_w5u+Lz$i4puiuqVnw~w&M61#FoWWqJ3YE^n9oL zDHxe1QTqD9cFa<Ma8JbiZW)!>tv1K7GuQt zp?hv%mG1%frkbGB=hSP2z0{?sM23`2kKVGYHlwD+W)1iA{tUzmP&|+44QY&CLbcQdbHV(+ zw5UgEwX;D_@~&~l$$EM2(2P%Xv{J}tp<1<0zt8yY$&7qyv1!74kkB@Bbt(5s?x+Uswr>Zlz_mA=-lXQ+m!|To03Z=T!+jd1`B4k9aB(6U}CuSID-^J}hS8>%^ z9~$$gH1FFIt0s&2h>MqvML8IjzMBbk{|UeM;#z}MzyE`P>w6s)ZC$rp?E*No zQ;1~8_C9vFQ{Q=q$3x4yw=&}1mnuW3@kj8(@5rQ#jBhqtv7WM{g#kHZ*=tW+kJ|0# zTW+4XTKFd~C|z@!!1E#e^R&1BSqVdrp*voX*%f6{9ZV=0MY(0KR=Tu@R?Wam zQNk_C6QbR;@B24cD5$1(i+)&st$UTReM5S))hGNxC<2WaF(2tD@40~&3&OrmR`kW81=g0kx)taQXuzEl0o5Au3GGu_nz*%bYD{l z*N2_kJ2#%S4NVqE;mHK-(eROICo-R}0l48;hl3Z3>|?#nh>`K3PpA`w#R%crLYT{3 z4t{uB0*O&7iSZj2YwO#O76q_tI5v1lrHD7^58~}3`TfO;){CA8pR)vO$oSU!P@$Z7 z>0Kt?n`Fp7<7zl<)trmmxn@>O^3}tjr8q6(F_mD_@j3JD!Ly0}C$6*GZV~d%zs2V@ zy`(lACf=r!^M_iC#NyHJwCR2ty`4Kge{eV`_ucN?kKV+j-4t7v=pYg0lDaqa6pwl!OLcbix8gmC=8CIh zN_pfB_lUplibEhg{QFD5tbhG5{Jy#}8&x9d%$Csnm-m63^P|}FTx?1BqDj9pyx0vQ zjRz1ci10(cx^SZcrd9f)n)tROJ9){kaKDcxu7ZZ7xcjTh6JaIkuW@|*=cY50WmL~% z@Fyzgd%|f6e>RMtOf`70)9EJ>vOXC5x_cZarzdWM^~vB%zU!{AnaS2s;!iUd)(||V zQL&P|Ov2BF-|a%>o^(l8+16Lp-Cy#i(H1njrq+zpCi1C;)z7dFarfoBpCPWTw8u*W z8YA&#X&N~E(?#~2q^?uV!f5KKX2V9ZPDf7b7(^})c8Y0cYmIOR*dew_D?;D$ik{?OO3eQxyRH#)|TrpMSsq=>IEk+7mqGI_h z0YmXy_ar+~cxL-orMr^^TYZg{S=lV`KELr-ThD2}LlvFQZ5t}AdlA{Qj#H0d&yz|cr0bk>^?H2;!+j_Vf2d5TmTWA1z`B(s zsXH-BZPNHa{;N%VDWyrgaZAZWMlgl=ic5tf!HdMC12hqtRsb5)T<~ z@tBA1p$$XPdYmgSllJPu%qj1a;N ze~8#DH)7U4cylui4pq>xQ)&51*51>vO4ZU6zX>cb6Ir82yUQ+wa%Z z$?zz#-KF}<`GUxsSnJ*~ky_8SMVI~cv1GXim-AzoOEXQHW+f4NQj>&iY|4W_} z6gxHsS4@mi^h(?X@|+b(7)lrLDkB+>E2g zisq^D^?-ifmoAL;t*dBf!>GoRQ}>NJ*&ms$6dk7qFT2U_DR&<94y0{P;$|l4i4c%k zS=?=^Oz3@EOwm-k_7UUsBwpUlGHAE|PJbmqBdLN{t;@5I-G_Iztol+F9#+xFTZA<3 z3hjAB;=KKm9d`Xd-u+ipk9v+x-yk+|>%@U(f*7enh*9f{(>MVy@%Q^@O+D0JPw_s! zRsEEzrK2=86R_d)Wx)0P49oFeD<%7p#!cO>v*aofM>twKve|Fh>005Wkr?F*)n&nw zIB_Lo(?)N6+0mWH);ER&{nMTYtj9DTG^#7j%fwu+#iPMx=0CadYnb6V=p&xv-Qaa2V~IizemVbW)9!f%BTpqz>+4C*){Q=* zZqVfmi8k>h*7-O+w3(&TV(Ynkf+2f#yQX$lgMD@;|0;X13|HOCvFD?Aqc?qmdE6O3 zVqI-`bp9}~>yq$k=lh|!frNh72qB4u->$hg|6r$R^B*L3r)Jsx}iK#ZfN zc{pOTN~cy=llo`WagI);*22&-{qFnm#(~rx9e0mEA9uS=rd0$?fa-ORP)^g4H>nVo zp9p;%`@LpaAF{MZ;SOY$N4_QfaBuRL3CQ98S54n}3*NCt^If4)%Gyw}tG`_uId=>S9?p%gKH$U6RJ zu5G4vJok;z+nQsNPKK)b5rYnc=eOq_x{i~QtyJs|Jt%EDlcW9ml@Dr`@d>q$!JLst zYL6XG?oA0gZHnYCubp+cOR0O(IyWo}Xfw-4_o)0vAP1e5&Gbv}-8}`3xW`N1bn^Os zK052eR+nM9BJqK%&y~SG^3?5&I{uRoCnp@J_r$bHUm!sfMf};&$!zBR9l?Q5vs>?z zbs|{K#rru&uC)eMe&^2Bg5EgdNb&2$)9a^x zBn`S!$d+$ZS{~0n-(wAlPhA)q+LXBE@?&e5Wa<>ggT=bXJJZ$sws4?Eg_o}aZS~S< z>zf6~jRUr(^AF9qI%V%In-0!-Q!Sax)-7|e0 z9qP>GLdMpeS~k{Xuxd8(W?b!CYRp%LExH#!pD#IkpA6C0Vu!2IG9^Pu)c9Pmh%H8q$f$(pLxC1iDSTMx*G;2X$(y#WxWbZ zlNlzr>)dp{YiaL0&vD`Ax}iqjco;rjkR$EHTDa7t&3^Z}zF`@ta@@LoH0q(V{os~0P50^Gf9M`r4x?f1Q_Jq2eEUxEkh=n%yiym~*2%7--qqSM*F1ArWuuNq zCYo=x9+gb0FeyQtaol@1%{Xhmy`t@{5r4r^K|BtR?nl=HbKG3g55z^v%r`Na5b$eo zaxWO@m}Vk=@uy|mj|_C0S?TU@o9Zx*s;dUSBV>5D7_Atpltj|N(6=_$Kyry!S}s!k z6GDnZ!uV_l+vWJjqif)#zDG@U=<&DbYz}1Kt)|a$vK2kL)*W&e!{{BDi*NxB}&7 z`YxMCsa4`cxP0WUC0Z})0qU%MNUK}maq6c$Y<0k$OJ`8YdE*^?x5^-u^{%-{@pudQ zm)urvy-Mp5y2ti^Y^i@dIG2!Z{g7@KL+&DPCi_}Mz*{MTg_D-UBrJZ_sN5+KkD-$Y ztJ2x(+}lE;Y)bisV@&ID)Ni&Z%bO!%t?MVzY@CrS_c|Z<@2OsmsW{DNmdN@1Gp1GB zMj-3iN<&o>9+|O+-P8P4h7`xaeH0JvMKgzxX8qTlVduFsjFiP|?!>G|o^xkVkBGF5b5Q*OF5IyQBL822W z5+zZBAQD6fB2lA82~mUS5?~?;H9}sQ|)>;C`xV=0^=51Gny3k7va8|3a>^O=#DCAYQf+7GPE1EUJeB(yJ+ zJ=@aSJn>BxshDs_Jqv@Nm<@4e4StL(WYG)$JhVc!R>X5~qbtOMcDVy*^xv1r971P7r3>cF6gJuw$ zm6Ug-d%Uwg7`yf5al^;vf|go`z8~EWuNd%tFFuoP^HpUiBxL{@xF36o_D6wt)X$1> zi@X;mgU0zYKhzE%PH-1%pG35q3{g;T5l-z=cg3Qvj#5zvI#4E51fx`vkwGih={QFm zOBZ=THUk`aVuo+q4u@w+eeqai8A}WeuylReP2vAqp?N=6t^Wx*bwct|F$>jH;ylCl z5AvJ)s--2v5~8e=evCdJ2Td-Dd~2e6@l}4$eSu*)u@-eeahmb50$*&MS?ikwkpRYqr9Dxx-^Y!U$7D_-1^{P&KmRX@*hV;|a0Zw=eUkRR#|90>g@dn^tm_i@xH_ z`umm-xv=`;=Vmeb?zAn=1P-exoX*(=?X0_b-eD& zSBN^9Fn>aDFwHYcT_PWzp6=e+CU9rkaBBLFH%8UzI_k}xA6I{4mUe#&5_DKB7sj36 zzJ1niBJ#dR=b7~w4=%R4xr;Vo<>D&pS7OmpTVHs5$DRB-OL|&Akme1)49P$hs%^=d z)_CU$P+_>?@8xHNj)8_Jm|X~M-jkY)rQNKZ1IOkm{QDNSK(_?qLY-&AKW|b~#MQU* zHIVjp^Qrf15E~<|f7-DxEXqjW!!v0xaI#Leuncy~4vaR~|8x2V0H}$Z5WoND^`R8; z&h2MA9!@+Fk9@5TpOqu8SI}@e=wvJZS{?}EYj&Mf`Tco{L}LTfmfWu&cITDT6zv_^ z#z$9uV=iB*Dg3|NIXccr%UR7qX1m>51y`YF2d0s*x0qL7qTt!pj%`#*vaX%WH|RLa zM`~I5zes;h_Jp>&}@{fy}l_Qnd+Jx7+pLyx-ex!b;&)@UnQx!Ta-R>KiJFZTAQmc4rf% zF;QQW*%ei5=aOg~8d~4;sb|TCk!{lVEQ?DeM=cZP+ z%TL-0gLmN&s844Q4me5OFHNdUkMZ~#R|-|t;BNk{y=mgnwACT*ldir-2Toow@eJs_ zZ2;|yOB>wLcJ?=$(@VzXg{{pAe~Ms>g4q963n0!J>3dhYNj0+B%Hn%WcA9mdOQY&% zR_TBv?Z>Kw_G|$PxRZB01%VSC))P#gTb8ax-C^G@RCvjp$vZkG^Kp9PeWI`Zb4YM> zE0QmxK(3f0kcV79k)qKsV5K-zRQP(V0^x$ zSBeLI>OUrv8a|28a>hQ!1m*Z$X#{KG^-WOC4XY2Rq=gdSIOJstv`~nN%9>y zcs8o?7k;SVWBEIkV64UUxG;3G{zrl=Ix*I&!JnRrL;igbM{um)WXP0E`uOxzrLW9f z$7|DhW}7DuzB(%=6rKo4jm+}|ISoCvGby)uvt7!B9{)9Jnlsayt%MQ@RlV=J88!aH zEbSS=H{wrY#V+}F74m{Z8pQkLXWwlHxTw}+TvCO8f76Y`I3L~=G%Xv00N#Ev=8s6X zlgC<*&e85RzoLFRBPoq6Wx9fxRjh3fdqJ{2okk>&dW!GbW@ar2dNotu9+}9gp1snO zD+WJqsJSY?0r)>fY$FO1w7t`kvJ6o2d6Xv`@<3@T-fme-D+rLl>VROy3qm?_=>)?PNM; z?P9*7#^Cos_qB8`wQTvY!_3*B;hLZ!w6mwy63@leY8d~v=TTMS=}^^w1@-NzsAM5^g8MZ_I^ z`dNILBW7gQQI)Xx!}LamS}c>q4W#!SmfCk6gDiuyoiEr-ve9d&yX^U^$wW%tA{GU# z!g-CcH`sCt9n0vSe)><wkx2GCq4)@uUO7F}-JeQL4)v-G->*_qtkSn~De4!Y=bTMnDB!>%4BJy#OzcLlt04lwVif_GQ#H<7hi6i?3HbywxoBg}y-Dc|WLCB5i=i_`2%K zc8!*Bki(CmL?@$(OL|g*n25$E#{{tVA0?0(kak^;G9fFtGs|rH(lhlVP4(=Dv+qN; zgceKT@tK0jzovwINsngk9zr?uv1^)`wu{Tx3e6NcpLc|F@f6;=GfZ;@lpktk;FG@A zb_{2RW{GTZ$Oq|VOo@*5@E42K*KxVD_vVMS12EZ!5?;DpE*93aoRT9xE6hb2_ZGwx zew}w;=%D@G)cdX8H?Zu|)OJLy#Cu?0A7%@|caqrZ9r^DnboJh(0vl@fWpxpq)}+%7 z^{tL-*qmUnN!I@Wutzh%dGCB)B zC-rqT#4#3lH;dPYt%o8pP|Pms{1rR+YJEddJ5@GH)`=nQHpR=#UhiiPs-}l#9hoxf zPUxcW>6YQd>$q8|B)guJs+0th>xCKFC=cbd9D^#CZeecMemyJy%|P*9h2Za^>E-Cs zE}M~BgKwXkYSYPxd3v&OefOk4n!eGGMUT>A??sHHc&HgY@^rp0RvM|AcX9J+jpz-t zjc8P%-{GX`=H1z#Ek}z?jF0&+o|HE^S9S1uk|2W}>DBQWzS0YxWgZT^9}WZ9^{u7_ zY$uvrC9s!cN9H(qF!r?xAJRI0jlMMg6u+pXE1uq99@D{A={g@+^XNhHQDnVahodH; zA3Yc14k#3Z$@~7MhwDguQKsdZ+Z`%kJuNA2(^0-s=jmnUI+IAyFYxeq^dVG5nLs#4 z75e%~pn3O;O->55Oe&7zv_4kjWg2Sc*Q^$w8Jg>b^}fbLfBw!p0W4Lzu_J}W+QVH@x9X7h$nj)9vWAB8r%uhBhCKk6|fe(P^~ zuvHlH*!-u&pm5NO;H;OoUmmD>+4lKnDcmU!Y_jiro5H-AH}>JJf=NSARlCbB+Md4i z&MmR7OyJC1=PjNky%`Mac(8(RezV^1@w2u*OLXmXA?eQAc+rVaOqb{5l_yM8c6Te1 z9bzBb)YQ|Ye6ZWxd3XQUg2gZE08_zH+BuZD{*bw@;bJy=4)(@3s;zg0mZbM2&i@x{ zU0D-G9hKLvba*9^w$?QJxi_1zX@L=2oC><^Z;LNoQ5Iko%mbRA_c4h>-#ohF`8nfs=Q8;f+L=GU<{ z#j3|CT}{^OLUl!>oA1RTZ+!CYVkgmFND_~FDf`;s&x{s#KXsZuuZ_v!zW9GL}M%0OY2`6-%St{9HDY!6b3pv-I6nezyX@5MH;Ud7&RswV~pkUT( zu;tIV+Eh}=SNOA$2%oJ%gD^2SaN5%ZFyC&e$??{0nZ>gzs@sd7(VS63zrRiQH>>@W zvDb_&j!7!I!QVwk3aVTwB<_DQYB_9K+NW61fAEZ*YPBQC-}g`NC%FuF}LLqDQQKiNS(^K===7sFOMx>#ox*oR+CM5pm23!gCT9lJ|@ZM;A(}lg7=Vk zc~mFwo9CBeQNbZtwfDm&ECTR2)}%k)d%R8iS1kvBJE}&4V!mR}&mRTeMQ%T6c48KH z=qfnSd4yABJCGiB?!d37Y)QmS53#65>M@8EZ{MVxxf{z9ZOFK=ZIizEim(OQk zRyYKw^iW;ziJktnSwEPnN?0dab!m>8kHEI{La@xczEmkLEl@NJqHNOGBID*1Ier@h zM7@U5>46XizK^h`47VmhhwaHyjyPd$?N6)wf30BEQQnPpe@ZBy`*c4PVz}NOs>XKv zCrS0EZ@-nl2`914T?eokxp3E-QDW3uUf)qhY?#)uap_Kbq zmObki<8*HAecxB~@%)fmfdXY93N=R&y}$1>ektvXU>SZP9g-%N7!=%>^y%@n zc@5@U_OIz@kL*aP6CSF6C~WEcAieCH>-|E@-#+f?4yZ>z8_=tVXa<+qBKwE^`*sw5 zZkW{RkBUEe2P*W?Qd8|YwJs4?t*k98H)~NUA31`v%SyZyWp1BH?{^BAw0s5!zv-BB zHjg7{jAEZp){LFX46mZ1OA{+}niebFak|_6HaNH12VF!W#>9K);8lv}SrT6k_atZi zH&bQyHiCIuoV4jk`N`O==$FBf-k`Wt?}AWLzg^&|;EG6ICsuHjblbn}Ml=DBwo0|8 z5F0qZEN4%vqHz@Y%YD0F?8ia+Q&X~cJksL$I4C1IyU5KlcXO10GdN&-BQ>o>UeV`- zZcbxy|2z{0;%Ij)K zGN^7X9S8Tyz6^R)aDaTK$Mt90TDNQX* zrgEqqCZ6|x_>q^5eCRf+x)UpLH@#*fy>9zUcTn(PiBQAupi4iE7gh%v;MjM{?&r*E z<3FYskYncpx<=6R*llBMAV)~9(qR)aXSiaHj#2QNzw&B`7`3N%9I<`*rhhWL7;SdR zVO^V;3SbRSTSV2nLpePk#*lOx;ZNUuuYG5k7@q*vRRbKc}j;wa& zb*>89uBW0Q-?dl9YDp&!C_Aqie0x&%o90}b@<$`V0~HaH#YG`jlEj>NZzl1W;bT6x zix|~(5jY#TR|&2Hs*dQLb6%H|T2AO@5NVXe6O0jmVkvMqL>P4zNdHw>uO?h@JPD$BPW z-}u#=fb3vd1;;ij#X};IbOSg4yg1%_isuSSV=WOGtM8u;5eogiKF(qM6J1Z5?@G{E z=n$Y4Cn&~m11dDm1j{Xk7517n+6lc{{Mr^^yZXc)R6)6Ej{@_X^$m8@{;^k%z ztX+;Xa0&Gcuzb&Ib>OG@OZ-KcTr5&$r!2h!>B{|){#v%Yx_sfciI#QFb>1X>|0$uo z7rJJQ28z68fBG@<_dT(ABK6z5LlaI{-c%i!^?jJmz=hmKkx>WVR#`o37zE?ea3fT< zp2Ig}ogRzcp!Fxgjlb97wYiUQva2O{m<%c#%8(yaGQJWkzQ}Oc>n_=bT6~(fXxyxtOwuh~^ zytV5NyT*U?J-kx4?sz$KD}s3`6rQ5i`Z>{V7ueD zjY)afNhodCcw+Qk*wnt>@3bBmDNXa;i;ABZuN&4eTtHBOgnVgmeZcXx_aPD>Q>61K z5^lHZ5ujk%?9D!0VSOoE)wo3e?(p^Z$U#tqhA!n$FGn+7Q$NCHpDVeN*hCel{5A<3 zyRltMR7}GufDGTprVpNA8$>@d<4aq6>q}vK)5W`~GFms9P8u9;NOpexk>DcbzB6a` zK9uW%;-~E=vX%Mgo#~^0v#lBDyk$+c?(v*iA@UIo+d41+`%VL_?k4rX4ee(~XFkmh z9hZ{dcmKUTaXs#!1&hD&@G3s1*VASlt^HIdVm+k@NQ!rYBp#mq{mfZ^hxUTVtz-l{}gH2)w6K zQ|9^3n{eQH6JCGPSs(M`&!W}5QHg|rl;RDwVg6r{Cg|6oX7})E7|rc`Cgie zOGZJrqsVnI5xWdnzyJ}?Qs1FKzv0He=bRg1*-LAC1BR}3VR?hUH5=uQx!YUD-FI(- zW22?r3A;M)W5xT%%xl}J7w9FK@yRM+^z{mq!N$La{x-odQEUtKk zjrt8QIBC;G7(a7+d<8wMD}F?q(d+I?vvC2$9Zd$35wl>+p@J*)P6mZXhwK{1vGR;k zeiR?RSx(;Q1;%w+>6?P9fY{S~j_DF6B7}?T+tzd`Q?Qt3hdROz+++z4;vz#H5f+Hz zWMDtl-yS{0Lp$}zRus$%_HdAr!i|O;${zb`qhok(2NEbV!Ja9w#!P6pR-dMi2SDY} zF_E)*o%{ti)5Q3}pLQP>mnQRDqqz*(z}FCycyE0hICDdUDd2&@c+-CKi%T?ewYQB* z(Fc@qj6|bS#aO;#-z0<4%g(`HLE^-Y@^WpNLV)u&-d3I?yQb3p}T)gb` zKKb9;83)5cTJ6FKdVNFz74Ae`?ex~3GCF6+I-R5S3M{d>mBbJ-Q8_|c1S6$wOQzL) z_eS)CVe#-R$^tvO{Z?lnrQo~Dz-@zu6-JD$v{T4?mr|4$ zPwDCXpVLzvb`kRglvjVCuqDE6o+RG=3~Wjr6W$ApM(k;v>vBI&7Bce{@OpHg_ePPxc`(@NsBR?ERq1ri4jWVnPMPUJMKU@u>7nBAKQ z+l6!S!q1`lsVHtD_CAtPekK9B@QtSb<+IrOCny~Ho2F`~4w@CP4oM;?kpp@X5Igps z4u;(j04mTFkIDz2vXKrnFAs081Z{FE>4D~z*RU(j8ORD5{E7j0Zwqp%+B=MZ@~vpt zqHt?4VW(^};cwj`fq)`g48r*OEhbzTEFD(asoMPcEKTecFtobBL4aDgfZu2%+gVD8R#e9kS-4@n>Nl5hJ9aZA&NpcB(KqfZbDscXbrq zw@cYAF=^%y=fVQ~!y=sFwoIJL%5=wh#um4CFs zzqs}=0s=C9G{6edlw-Ge>moD44Wq`8rtUu|(d%F#LCmP36Zpn4}gNzL$0gtQWXZG z*)#_{o@vc4C(G*`KVMs5Y5^-=Wh4v%8J0?g@VZyKMNQ_Kk%6_zv2e%?xWlVp=C{5- z5-A>_?lnMM(g54_$DAA;P5-WS0ho#w?L`K$QS>i(?Bw%L!YiqjZ@{Jq9xp2uN*IHG z-lI-QU!s7qe*Ibh$5HGL)B^#uX#fpJh+8Yh4!fKW>kTjJwEoi>q9Q>Z6LC${6Wof1 z@<2!^1s4Gu%>}@t56E=)I+djxq>EBRpE)n-58mSGX4uw%jaSCnc;5xGZ9#e?M9;Jm ztoFOdghll^0ef!JE|A8ZhZVsepRtW&=v*+u+MWZ0ncjZ=zc4Y5%aU+W9r7AhT*e9Y zg<_x#m5kxy(YyJNU_2H*N|5AjIS^g1Xhia1ixjrF5X-VM>~sj2$3NdWQnLgFlyM{* zQUmSQ(-s6Y+LYiOR0I&x_$lZD>ja$%5`7ZB^Hqbst?2L_?l-|^7eOH`8vc^BWPp-_ zphoiNogQD@7NnhY;gH5=Q58gA)abC)ZY6-Gf(eN?f&=PoZ<#;hEyMq+1-S6n1RBBk z?)T}wfwv>?gK1~DJ9bXd-T>l;m(}fs!9ZC1tFP^)w-@9lpc|)bh-dH*K`F>$U6}uZ zT%i~ZSV|hHb^rtamZxQqlHF_mGzQ!kGpM|Y-h%eDql5Qp&ix3o>ClFaGY|fy^?#d5 zWGSC&&EXE+U6?n?1__RHjA5slO{^GDS;-kr4U>hXyi=vde3-%f?R9|ZN3~ut=)tb@ za=I3!;-*dk%K{AY-}b)wZvd-VPJVq{X zSqDcO0$9Ba3GT_fq(oL9TzhrSAI1Z7vRjyYcn=b{B%(v@-3ja<fIgUA#j z&~V7$Z|uT*>cbpM|6N;-|KcjE*A0F>;Za2rR>DC9bTL>&Ula6GvGqW#--?MIeYGNJ zBDf|p0WtJK_&)3@e55eLKfv~m-x3>X7PPJO3TW}XKt^pH7fS(Ya4U@D*INL#pjM zWbGHYQmo}9_0QM};5sgB<*z|=#k5+C9qL%a@V}lU_*fS6Vr>;5!@Ou;AU`8CRA^?B)`F3L|Q*cqI z3~c|m&0+ql7QQKa?7;VSLjl*YLm4v(?HICJO?A}IOegs|)lYR)_4@!NdCYA~XO>A) z$$4vHW(8{t!J+Bx#Qz07Q~ZiV?y6pYcwwI8*{2DVODYBAILz348+aDeSwnY2pdtBa zFpIC0L&#>_G$SFYIut{2-YbupHIALeNc)~5D9wfd__QfL?Jm$&MgSD8tH60GBbuRL zV9K;pJC;)xrv$?ix-%;`XlLD30}@HW$q_OWWR;(1MHbwb%N;pwm86Q0QAu5Txe(=)oyJD zLsX88fiD&R?>b)nD{?=zlfQxXXz1k?M9MWUIR#wW4tY2`kkuY6ya;x|$wQl;n7>Q; zltCAfO%-t{PdJEy5IBpxGw&6#(-o&veXL`x`OSo)dv20=gq;3qdfVrpMn)vz1!&uq zCN>x%FWJS$$oSd7>8vi`tqo>f%F|9+mq0KS$=B}I-gOi9xI*SCO{@7iS)F|FU~rzY zFU$h|9$u;6_66{pJ+X$(ucwa(k3{!yPboiI2HmfRpzTQ8*PdLoBL zqpj0E2B5Zn4+}1^{q287vq6GL&OeIjy0?qw<6B(Yql}t8RO?BAg1vv`HXw*nc7Os3 z@ue}>70?CN$y$m)w9A*e-wx=I4I1tIr4FWznC4ww*lcM=G1^0jPlIUquqg%OmK0M> zmu}P1Y3)5_$LN}Oq9o}mjlDHcZOIXRI%mXWf(Gj>$xE0+j z07^Q^1A4D%-RVCvHzRiId;3efRRnT%RgSLbZ*>=;3p}gjoi}tgcmESZZVDfVQv~&L z!c66BLka~>P!jeaY7vE;^+MHxKIHjP*?^i@{f1hmZ>BNFB(R#>vodUuYBT$lpAs?xXq;Cq` z3(z|48Hgi13U6IF{^9Yx%?+un{G)1>JRb2k3;N%)L*M%9vj>9B+5+2cSKZ?kfxFz} z4sVAEIH&#~uIv-x7j*)HazpJgw@ThA0Pfr0-Y7{0aRRyGd(T6csUN&lNTzoeU1JgZ zTo7#dpO)d*y*oR3w=;G}p6@U#DT-?U|Dzq|p)0pw3oh0#U`CUh>|azM?cf$AQsxq> z{|>vbcdH-)ICk^^DPy<6d1BG?12^XgvC|}wUQBZWQ-SnCk3l z7Vs}cwE_uTp}8bp_gF4JE+HJulVyOWz)6giguu5YzQKF75Y3Pb$X~d2&S=fbC+rOW z#^G@OJ@K}0iHh`2M=vJazKsM8TKw#l!sI^38fLE@Iz2uE@sSL89#qtZ%xPhue=5*U z>qzJ1!Z9XSl8-Ys*MjN|M4&x+3a%=;(LtSY;jWlCRQ2rz3}xdSP8#-zN%u{bgxpOG zTnXA=X)u`e#ixi)vOG2sXF=2((_8U305(HFkLE*WH7=lBr$Sn9Bb%#A5+Vo%p@67c zn1T(0z;JgE@n9_LS0?M_a4O~~1q!8a3c?4V>7A_-;w(&XaGag0p!8U^pc4#7&*=_V zM&o>7&972yyrT$%qhZN)LQUyHEfWnaDk=d3Wk;|muSeCDW!`_Wp>>V$!n#s2Ekh^) z03oYU%BA;O7 zDN!YjvWy&w+7#eQ(Zg0@*e2Ve2aKtBoJAJM3fg*-i`@Rl7Vapa=!7nIB&TwP7?TI- z4MXq@_R+f2t#LM0KSs{N;C;mkVR>Hqooz8I%;d+=)=&{RuJRVm?G)(IaST|RT>9;) zg)o?UFnS2lO88CdJ?wN4Gy%>&XG;s_P~(C=?JP<|m|jIMRDLw6Cq$LZZCQU zsm%rrP|CfdofO$F;~1-QCY+8UhZq60_Yw$KhrMv_oj`27FkvgT0T0)r(T-L*E_ojh z8SWx-(xo*I7ip=cIa16?jCubsY)P@D`L#^Ng=V z7&>c2-it6CyYhJ2?6HlIxFu#gL|824m>3BTf`c0?=+nof}S#W>pHK zz2>UBFYy^oxDvOs67WonjhrK1w?$eC$SVxmCA17Z_zc3EJz;27u6NSFsv^uC0EBJ6 z=3_#Er>u}z0{5=z+lDZbmGAP$(4ttwJwcC6>kN|0Hc$2e49&(RAnOr#JB!4zv890i zEp|8@{_Fj5z1w^B>SUVfV!zp$)jf}ms}HmM6*^o5r)=NVly*bHnWg|uFlV~H{MqO` z?%Z2mKdk@t!^*PRgJhRo{^~mtBBF^zdeoi%ksYF#AbUmNFu+SQK(AH(ovMUR6D81Q zc*u%{N?z|-9IDzSWEqw<`&N48Gz~e4Mi<=;i4ON9Ijoafm}UeopQu62ea-8I;0e3= z{Y?C%>I#7FqEHI?&4SsDVh?t$mM`*1mEW-7pP$hyr%@1g+KteV7;SsC=O!sn!eaVT za|=31%617r!>_=b0$^oK{1eKBoryk%(f4Kfw8tG zKsu&Aj&1YTTwujD?3`CzVYYVlnzr`PXdCR|6|mqkid|M^W7zN;DS0gr1#d_o1}VOO zOoC}1m>r$DPbdhHswYU0TS*gF-f#Rk5h-a?pbUk+&}e}fG?0k zIafg$avGbSChru_k=Lf*JIASU2V=009FPa7)uM_j*$x=Yt0?)>;io<{%5}>EG(Z*kI$echtQ>At69@O{aA=n$EMZ zt`&CFF?2{`jNp95`YZr!u*&&qBz zs|2O)w?hZeuNI+p!TO~UC&{h z$=&dAbfG1MPRx{LVCM`T-oFevz0;qj(^MEIw2ISISmh_#j(%M+$uIKOkK$0m-*^zR zP>bYygmZ5mH_>MWCtn+2xa$DLv5%-3i3`#$u!!*HMCa0`{yz-VL~tPL3*)F2(aha6 zAXmnW8-p#F59!IC2Ib(@P;H=RN|xTy`e5iZ2o~F`AAJF7S4lS##G)g4-x3xWnh32a zRY4a0Gy*mlB4gSCUPdhub`Utd_=IO^ZPw0Dr3EDX-{dPFLI-w4HZ6;L5BfhKF2f`d zXemV=v;Xh3N7cH~t)F3v#O}1pB(N>P_48~%Wj7oV74VTW=ob2t;IQHM_X%#?28&KL zv?azJj0rNqS(CS!WZ=NR)I(8&u;p!igf`_g1MC_M%SF{q!&n&7HNe#B{{L0%OzWYf z=$5Fk0bMcHm#;d#lYmwalWJ#Ni1cRk(=+gU^OJiQe* zyBEYyuA(d|%BP_*FL0fghw<79y_NO)`!rzs-?H8nXAfuNB{1Y+RK zN&Zw`O^iYe)53}buW#`1*29~1aH`zQER{Z`9Vb=Oru_4#8zPMi@L_JP~sYH$~ zR=F_~t%NyE)&F@i>SZoTB%J2`q~sj870oB%fpdf?Wf+`rfL|Gf$=&LGOs$L#KoC-M z%9M3C3LtqKCMVWZ5AJb}TF6Gh5YE(#d`qM{wU!Cz56gQMUlwCh=9aOY@8p|m4ZVp(rx?N`kV_B! zG!tQnE?RUf>b18J2toH*qZ z!t$}p41V<{*iRp(3T0>z$T10*c7zeXoGcEY;}@PQrv ztln&Jh~V(|q$qXm@pN%CAxirRSgp3f5qzy}(j2eZSjsqg?+Vq@88GoBTPX%r(@zx5 zF<6Vf+-R`kIRR1Qfg#r)#CZCJK=h6pEc`nXrOuZmaZ@J=mqBx=j(`5#sn7W{o%DD{ zaFFIe;pY|3H>!ZaC}ngm;Ol_QdIB??5YG$PP2+W|H1<$icJTLt-8E9XKy08EFL>+| z;;;dw%B2&gBWpO&DtTAb7YMSZwG4kDLsDx%w?>hf1Q1XJO-m-htCOP4tHYslV2W2L z)41GvG7@lU;2FL_wxvYlJ$ExHS5*i9wav6w1_DAj|E}ZfvA(Auww&9&@ZF00Jx3bH{ zdQ{sr?qzCTO~95+q@~~z+3u=8gKA~pC1>!q@(TpLleyYJGJfdOZ}dtG3{gsrjx}QT z>t(hHzr+1Tq-%(?`o}9Lvhts6$n0~@?f+hzjPNqY$>bQxG|5W?8Ti{W#LPhDjBdft zrG=JRM7%xN%qjW89dh|+l1C^ZDz>IIAJ42;ZTlsiaU^cp%1e6>#o)S!VcI zZCfdwcie9)iCzc5mB#uuzltqxk4&D94uiDdFPfoa{TPc!7>ZX!wtAYep-iI9H2DC;FN=jkeu)R z))oVt33oEKi%Q<&cSH;rq+K!*b&cCK&kg)cG7l@60QVr_(wTxKhKI|@L)^py115@@ z?2JOydfse!dwfUk<@C)3LP(Ilh+4~@Pp+3O-c4j)_g8GR8ih3SvGFvYPIAMwyi{*K zLnIpI1_t<*rxa)t9DbL3^bd?9&D;Ce#XcrAd@D*k`XZ3O0qd&o#a zbE`vhzpLdYKCCAX^u_yG!rnfW7=SeQF>)NUBst8~Xg{ioyjm0W*L3@@8N1bOonPB6 zv`lUf1#?q_0jRT)4b&{*_E-hZC3c;%_m75`Ru+BnD^!IZ_zQ--@M1_!fnWoHV2h-9 zw~^ql4;o+=;;JDPLz)goCr=MvShf`#7J`l4?} z-ajbG4%WGp?V=eEP?8=Un=-sV(Mp6WXRWHi2uF$vRNS1gg&4j{=<9_cJe2AlbHoc* zOkhS?z*_sS;g7Z!U9gCNFG}yCQ0Ce>KM}~fAFGS3r`$-!O+0kWKrw4q z|0JmuyxWo#6x!Q*3GnO=f<@<`qND@=NN-GiApI3a)S3`44Cr2x=-Jy8a?dMS#^wqw zEd+BsoSnC_eO#iz&4yX(YU0Qx;Dj^{(BuSWE$kYB14?2`fR>wI)Vrv|t25TdwnPdr zqY-h68R>*8FO3fMzmD+=M{b0s%WB(z*?kc)M2Z!E^o=`55^R$sr)MJPa6 zK9^;pR!K#^_KtfKA*Ca$H&BBYP6&2eLjNV59|hPPp>rb^_$V^41euFlzt$w8o{v*^ z=vI>I9WJjGv(mzXDo4PcOXF>M_+mgG8R2c6`^DqZ>-QbHoBII= z-U5PYXi=^fkaZ=0nbSDZKLn|g$G&bR@^J8+YI%l%yJ9Q!0vOblnXeUH(zZ1bsBnrbv^`w zb+^6^f&iqt30Oy>gjYQ|dS$Nm=sZe!?@h;>QOE}Q5?kBxpf~UdAl&+#!8Pe~fJy2F zZ?G0)j$AtA+-5aPKWM}+L<8oXhqx#dp)4H^$0uceH5<^H@pW9LA#LJvCegV{{6$yk z^FMPJL;@b^#`Zm-2CyLy#Ppbc3&o&Z$--EEytIa`bz}_fouzZs4lATJ^M@M3?!&^4k$AIN^85#fN%nChMKsNWdQE|A@xl zR?srvCOY9-x}kGXr&`X~=`ZWiMyh~Y-0y#Ug%qA0FkdjrnWH>k&{=BtW0Nv$=9~P0 z*XaVL5NzBN!p>TL(~VZhx0#Lwf8(zan&H~7H8pPN5CZO^#CkAi);{c{E)t@bQS|#%H+7 zo3;WgoOue0BA>V2ftOK6gYOM8vf_S{{cXllN9snS#5ZXEL8RrmrQqGMG*=noO7Kt= zV5qV5(wWy+5$`4@D4uEOTzbY;I8j-qmpy#o^S6lQdRT;zipwgw4uf7G=B@n=z&P;T=UU_9UA`B{FPOE) z0et5_3oLdZ?&gRL`A7_@h4xe4xJ?Hc3Hp#u#kJ&T%UCO;?U!DvU?Pa-$H2T>A3&g@ z9j&Z;7rA5BYPV)LA9Z}g@4$?R@SL1J#EK$e_s541%X61cZYm7?s}|t#^C?$MF_6SWi4=jy>;MB>Y8*e__PhUX1<3)T858WFTQ9Bn{}WRsVe>{xTJQC z-s{WzBz*W^Tt^WAxr=3C8)AXtJ77|((W-~aFuS|3J~Yt4pK+29TYe%atY9w}3*w6$UK79Dy0u$gjurfX>x-fIm?+JhgMoC z-?9WivWVjyIYOVLe1QF`s?CQ!x;A-%K?oo$Q?H$tU1N`_xow^-eQfBGgv7I2##P$5#Qjm^|wz<|oy2F`! zmrkQ6G)e@p|5uJk;5{JqE4(Qw=!g(u->gHHu&GwlurrZV z%5X0lQig)sQ}ET(*^x^lLapSGLeH7%0uyp0BdlhSRvnndui?GTcN(i4BMI44zN!sW z;fd8B+jtZGW&Kyk{eQY5VFVPvUb*KkauqR1Ren%7wX$%xY{LFVTUp8sP%m+!F%Ut{ zxca*_qS}JGScQuLnVe)JiJ|mPvfpyOD#3|oA9jy^symk6^g(X!Rh_+50dP*bE(ec$s2DvY!Y}g7fb_N&P@cwyJU0pp+ z)scrRu`F<%XfS^>f9vOnv%Sw!Lt4&xw;a#Zjd#h;sJ9=}EQkn$1lKi>wa~$~3@Z?R zDq9k*2+u#m)_4-skA+0PmVQA{GzFBD32yPT9mIa;$Dm*+S$};I37pGSuvmw(=p+rJ z))TF776UEZ6ZCISK9}YY{RCU60UrELie%G*)d=F0U2wupC`gm~ME!1RxOi7lXAh(y zc2sS%5%jv!2PN-nzN~%2oK;0@szg$ZPlb`_jA2CIQxU);yD-tckD$eUCA{hZkF>ZJ%W7Fj5UanKoqa=Xd&+znQ+Dose>ow^j zNp=Mc6LM6Pl=T%opyG5UGp zNdq~0BscgJ8c7kg!7CcBXq!e@1kv+TCf*_P^~OX##~!aDkWycF3RVD&EdhW?X_mS9 z{xx|p=ab@+JJ^tvL%R_P1fn)HBja(9(ZC;#0wr$<6oA=7CW4e2?4x8XxkgN}C4kwM zZoFKdI|_E~)%5-wkHoAUBV8wU``>nY_XV~{x-o_W`+YP|LbbBo5x$=_8$m*^=&v?# z-#PDGM-mJXM>{fRqB1g=xMByq*MW=&m!mQOH!`?I&RFIX*Xc|d#lB-}llW$LLtyf+ zfpuhEL{Er3Z56>^z5lAY#4SGXhGC{#_|^YI*IP$L`GswxGSuK845%O=ozhAQ(m9}l zN;je)9nw9tN;ybKBOnb!ONSzglq21!NQZQD_Ji;5UEli7Ie)m8x@Ml)d*6NC*S()o z?w}Ier|gm?^5ByA#DFU9R ze8c*WbSEu`U7m@Myy1%I?++z+_b?qNIJABm#OG=@i|=!+^W&N)gp5RFwFUs;E1`Dv zb0?qukO}*h`3)G86kfxwIUa#w`q&tNQPh(eno4o3;M1?bazAumIf)DkjAyE-Da^5c zuz>W-k8kC}@J@Q=<@~U9V{`xKdoQ4ip{(-Woc+jF=8wjwG;7yB#4HV#U3Rdl0Yz{z zAr+rd@elIrKl%Fl&u$-GM(f1MQUt^KZgDdfQ!R|=(aZ*iitMBeL!LS5;>sU>xB>Fp zn5-`0Y*A8lxC7X56+u1FO%9ADKycPx>QFCU0^aEXF!<|MQPPR>nbkQj);=A3JbFp< z(Fk_q)#%goBQdKdZM~n-ph(b$8EM`dbQKygfR7~G>WuxhMXlO8Uw4m6Xbr|h zdjzzO8t6i54&^IcSa8x#tppV9UgaXNY5<5J?+f(F5g^VyT$r7YYyyIZ-J}kMvX|Ow zXT7_IAlG9@c;AhhVAzcn+YrYWvSDGTqSy(3RSj12DW0#xDRy~awUn~Gk;@I!3_jfm z0RTRy3x0nIyw&PWaY)n=)p}KGK`TsB)msI(zJ@QL9{=OtVr0R)p}|UKDHY-K9YgQ| zWpKvgO$G!Mc4=?z+1$to$VI!v8?@P~UJ|=21WM_ph)--|!7+X^%zOp1`o%EwGX{eT z=6a3cO6y=n@-SjhP#B`lp5ujtvku7AgS3PM>@YU=#gl7<)L5LN&btLmloj={U20|$ ztPfRT+Q^^is((|XyYAnDRI{PesZ38`YWMlrg8M|$;w5gl_90z#r0S`7euU%6;==mQ zV*dKzlfkgDJuDx~)Od13V%6V$R*neGtqI~e?OrA4`Xl%JTMF)d$P5_Q^!(Wz3 z63xj*&%gKtwP#t2Fziq?V?+tmHLtOM^*?y`FZTFM~FYyP-0&4Mq$$HAYB2^i6#8U1Wb77mBwQ8Jz(( z1yJ7Ix7jBcGxb5ehVf8jv&NJr4FwGTM%BzhmoO_u^2`mx&-K7moDZORia^3@9LeTG z1~d35LsSLsnd9K$v+GPYuwgX`Q{1&$s>R5Bx7gGFiSbv^RRk0qXTth^y$jhU^Sck{ z?Mn_l9r!NEg`73tmdj%tYoY94{w44z&XQ*?ETMdOhEsX@rBKFSe+kQ_OJ27vM}pZ_ z+x@^!`Z$4L*X4;fCLc~IZMZlL3j^_YLD{(!}kDXRf|;q7ei=X0HZOz7^8 z7gFjFvRjwU_fb%|oYfKlBf#0Jdz-RaUqVqiNoYJgU__ngQjii~jQEyyncd9ov7u@O z2BSk2mB5R4RNFs9U!D2Y7;|5JXyY32BG9(JQ>dyV+dc{0$&MkC5gI|TY)i}!q{2w7 z;Ved;KYw}K_jATiorGNR(Hyi$vORF6x#L7Te{AQrI=e zBy8)I#C*Z~AKNv%ffgpDDCTZwNZTtG{K6g^yUofF^cY z3A>saB2r1#qO+jWRB6NKRsx!pgE~6FL^}y%dHr%Bw*z{HQ%AQ3bMzGkOLX(Pr00LN zpZu)oINXZ4ZBuXK9)LMM*_%#2T)sP4q^oi-#j53QMATf2b>-mWozI2SnGS8A;|70I zd9iY{sJ3XF7{@ig{Y`j@!(Z~dR7Soj$M{aU&G>@sLTeDQ47JyiI6V*pQVe|=)V=&j zgUY97B^QGT>&|0-Gl4g7{9~B+YV{9s^am2+opQiuRzNN!igG;1xOM3Mp%G3Gm6Au# zI{T|uqrow5J2+=jFHd;3vS;J`T+~=%-!lc_9cfbpjhMqoP~O49U((C|mQS>qe9Rs7n!ohvMyoH7T@t)va6wn=2%DrW6$pn!Tse@>dp#dBrN zW4qmgaVdO#MW}1j5*_=4f2okq!>A1B)gGP4b~f_c2?sR9KWRUh!_O8S+@(!s*_-d* z5h7wxnkBki{a|^=py*zM=wZQ%MM>rCQhlROfV5%P?8#?-fcvPS>5VU#JBl4r@CRBU5r3n%ap)7)%)FqmIVzD6fN)9({Mk%o|HEGln-pM+qMK5;8J~=3u#|X?o?j)1%?hWzLZ$d+plEv%v?$J2Hho zJFwJi8dCfIQQk+k)kd{%InD%2y5dG`$w_Z(_vkum{XHBn^UbP^@yi+W*JqkVC;C{Y zi5BVDS(t}TNiht5spZgWn0B^WdG|wLEn&;^-9g!RT$7psGOKXz$}u&441x1baW84Q(7wr|Mb=Du-LJtTQhm26rZ3^=CP!WdaBkdqhHKTeIhDr5 zD~a%;K&R&g5}&+)e=mn|g5r&~R4pEK(ll0{(Od)G!X!|Jid9C*RS*WVQmfb*U3qlZ zP~$~{9olV@^3p2XsDxi}PvXjfrR%xRQ=(Ee1lQaZ6~071G1XyW{wf!;-h8MXzTz3D z;{4S~` z!%e2Q{QL)cY`L{ZEAKx2z5T}(#f}zJXT3hFxiJC*$+lvD)8}awH>NqMjRFE|7Au-9 zdy?`J)32B09k#4DRh_*yT>5E9zMF1H7~v;rZ1_MVYdu)S=jtWP_xQB>zQ%eF#&HHrhHBWCp|totsj);%e85 zP^8`_dq&rF1RjArD7fJFK9k+JvcS1`1*M|y%^Uhl<5%qJj|nTbp4U}HIpnyA)+CU! zKsjTGENw9S8xWpwDR*q_Z>AtVnT?HAM=t_LzvrQ}B^5z?$I(S5-#^yIpG_39=~WZ7 zjNT%hlrWj|YxtE;xA-2fQ~M(KcFTd=q~EIT)bhvA#_ufobcX%~>s8%`uJ}tXB@t%k z;aT{dHb;|U1NIu*`P&&^=NV5jf7MSdo~-QVoobA))}!|tMdd=Cmra_6?@W~byc2MC zQrj!#A2{WGfLON@@byfmF8(lwWA-l2(DqCqijg+8t+vrAaO}IYVY(yLawpR4q~=uP z#LZG^Fk(%`G53{&bUsd6=q$A;+r6Qus0-Wno^NNL?yT4PPUwDZ4ziP18)?3lS{ENo zh|WY)^TG+pPLs8pyH#uWp5G4L+>o;^8S_PHN7}{2*U`&`7|lU=?L!tU@KT_NJ1Rvm zk)pot({KRSr76aJV-b3hOy!dx8 zPsRri&z9=*NS!UQK&H9erJ{bP9jQ|ntooGxkGx*o8r`TB26uycr6ApE^?m02^Yx5n zuctM1{MU=~)@o&5=`FSCJ+=7orW{nG>V>h`J2qpgbKC zO*Qh16|#+=v^PUcL#NPps_{TyrWO(OiYmUq?mG(~_j6#ztG1)>*voY5jNgK{KC&5F zOdi#9ZB+CA@~NF{4@ESswhJTJx|463M$w84O$w}0TMmD5q-!(_=>I#erA3B~9;ZLE zqCdfk5j7ysKJR}UFuQ%EZFEuR&b7mX4x480Y}(G`^_TEd604r1A~Dlb`OBQgf3!*W zy>P<%s%?%3;GISy@S=;aNB?xEm(d#-|329-=GohU(T!I%wjQ10#>v0?hkKObp6*=| z*lVYqB%3nQ-IX*tar8F!Fx^W|E*;bS*ixZZs$QFzeyEz5c3QO;#Aja)@|PFnEWYA6 z2LnEnBHgld#ARL*q}J72&z17LU9q%^ZndQQ3`0PUA*j-i%YHm#kIuMb5^>s1(Qr6WL&m@cC+u z@I{_sQ}-HwK3Ehvs`HsVnHU$?0DqiF&qP`RJm_0(Uv19jNcMo`y@0 zaJQ%#9O~q}a|^y4ui5HfS1DX@aufA5gz`Q?-Z>-_5!uS^pU?EGh*~)RkH|RZde0h* ziNnEMW9Q~K0n?u6K268fb4~V%d(A^7x*9u6gM4QO!+tw0W!qabqVXBy2g}|^L)z7S z%R^fxEk7H1M4p(rtHTU0WDVCfJ-JGO`5^o;$a>u9UUErJ+W2S`>VSl0#b<=7ekEEVk#-7XBt=^`O-&i& z%Mc1sD;!*H=EVBJV)R|dm`l`FBcq#)H;=LFmR$Yr#C(&iQp#@8;^fbRei^>I zl%|Xf{|9399)I{RhLI|t`r3C;UTaJ9VU&?|Bt4io`X1i0tv%`O24@-AH+?Q43hKIO zFUvO|_stCqCFcz>Tdp*$yoI8Y^}Heg#efSiLqNKjoG353l5AX=WE}Y1Srz<0QyGX2 zn_XYqD7ipGwkXu*Rk1-=Y=Q}T)%cpZNIQJd@3No^^tmvYPF8D>E6CVp_G;cgNvi3O z_p8Fi{J`&9Jt|~46|KoA^<4bavbu~;E8@pze9!7Uy4x;v_-sQ+=Sk6$j`oGN>02-4 z2#UA_PbV@A3bW&_7ryfYEf=5MZg~w zGrbkPmtEV^B+nyek~PNEuCnfz7gs<1%l_y~|Bq_tJU4DKk)7eU5@spOJPG<6!El9i zhHP~WpRVy<7>w!)yv9tEg;9<`a(CwYylsh20*L`GXplB6szI^Gpt#s#_k;1T&!;&F z%c4IJ+1^F>C%fJ<7x|i9MHWj5{?TC3oyqd_!FE%l6)7 z*wg@kMw(Oa+F)~{<=u>I{#Ah;lUm7BMt|_q4ggO` zxc5mmBG!0Bjd~jdr_3v>o9v~}7KrJGckZtr=t!28`S+#INHHEB>+RfLBw0U*b39X7 z_Q0OG=DZE)9pqn*7s$q*tkEqU62}a=d>bC_o7VWj-joiN38vr0WFiFroUFV4Je`{* zEDGDm;Mv^?x`LqM#X(hm{=XaY*(5QByjaA|$tc)(AxG9j@x-4iovkLumL; z_%!b^r@3|*hkNx+`_CcL{8m_}woYOu9zu8%0MVEIWM;R%F4av>nq-<)mKYYDspomi zm>SjSskqi@iwdOAx5f>aZ@Fj~p3WAY-B~8QT|6nd@Wj5{Z#?z$BRrbFoB+?ghy0I( zQBEb^1(8m&v3-aVMU1wb-j(Io-@O3fUl@4eV-Y~Q%KWwcSoZ@`UjJWv=mNSBko-29 zL1K)aW9xa2`q(9+`#`-i@MP1>UpNA1`cOcIwWfb780vo6J9_rTtdzKoi4EWg-;4^N3;I}ZF7BjBS z3JcI?a9SweGmeK5?~ZLxpBWTt_#@4yCPF2igr^7}5c8Zq@aLubI6}_hHEkFlt}it2 zMSZ+lt#ImSC}8&>g(+y7rupF;-v77&bo1b{VxBagnJ-f|Rcc;SU*u`?7!PGTT&Vt- z;YBZ=rD%WUxHg47k!HC4fOExa_(9h=QA&;0ulGD&ZDe%e)lFm*cXg+qD|U#QcUE?z zqWfP#av=I&Ib0D}#PEHbDf4=ALrEqOyDdOkew_t0v(fO2BQD<)l1c>@-Kb@`1dfy; zy24|qFKBsL+2a+r+0&twUB)B)Y~2Ta-Cy7FV`~wtdU_q)tlL#7OYf$v4*{(bq;Dv^ zhM;HW!8=88`Vkjd?P~o&0@7Ow!~u%VpR6ykJke2rUzXjN=eL#%nG|cBXy|v;CM6;* z^74vK%%J(D&57Mz|F--6dz1TSy2jyQz|y2eAjkdu)a>$xlfGsf8T*m%n-c0{w>zcY z)ta2X`$_DNH5-boGr5Xad&h;Wo?ux@o;3iz>He^fs_0I)Id-KOU@#~MG%BCzY`hWQSY65g|a$$3^S z%+cx&QhKXm<=Q$4RiTJ3SB>f6ZBX>`xseqWJAM4vsYM_hL^tw~rpf1y0fh^^+5NC% z(sgA~a+&c=Z}YF*$ln3+nFu>@kQ0ia)`t2UOh_K|2>`V;hHVj>hj<~=79x}qpm*;~ zIr3dwZh^%b)Qd)Rd3gQ0Q1Af6SJ*lUK_T@AP2Aubd5jNQo=Ktgl&f0FeRcW)qIl$} zgR<~{+B&xgmk8yF-U5tGS3Pm%wjoXoQ!LtWHW*;y&*OgAFgXbBFA`z|gP(45@I}O~ z*RBqjS}iR*KN`rCld85GylY!(&+|-s^8JZp0)S`%#ww--5OwXbXMY%1<=TF#RUeGvNK_1t9aL=~ZM(0;~p+q>NqWHoG z`$l%EKfU^yeadHf2{J5Um1^GxC3r6M=SEd*142?0Dw-O+XJm zJFgcRcpuQtt`pLt4gp_x#HvTZ2%hCadpnvnyAR|xW_zDW!4Op z^+5SecCh=5c3J9so)bJFHhF`H*AH%a;*z3SY&Crmg6{5pZgYbmxXz`lJ{rG)@#!?Ml`YNtNCLT=7r@<=_`HIYEB;N1hGb5Qa#;46VMuEYCOYnk7}LQjidxoFj8ZD+GLVcGUc=j2Uce}w{{`q)r#WOv&QPNc zxK#8W9!wok%wheKO5iL_|J?*a+3^LXKWks!wm4$s1*+_@FG8cKUI!vxuz}XDH zIbx|=v02nM(?%Y;N~t0SEoh2Zid^CzK*8%RW~2*sTkiiyywIz1D!(t|6tOf^Qgqrk z`F6+OdMcJlKG`axoE2$}c7x%C?3U(HiPvnjSzx4Jx~?x?2Zgd+t8@VNLJ**8?wSHw zOOpg!kV09=HSk;JRU@L8V05a71IBO6^uvU^J!do6+_ zOMS@P|Cx}S*VXKdypNP&Bs^wa-P;gnxOTQv5cpz@36kyWSG|o7;{sCORhCZPUv_&& z^&hXHnMUlMnCV8dL22dt()!h(#e3>oVPFvC&QT;iTJ@qG1)NFPS?U;fUdf1s1KREd zwERV37j;2Bcw0IE%4!IL`8s5w0RO*~@kC~)_#A28Z>>H#7homqX-f<|^}hn>;cO$c zzyHa7g^>5ohqN5~LK22!{zZJ$`iB`j59z0lhW&hCwZHt_TbY_}Jxh{UA9yI$yLRQj z#WFNnM=Hl875|?E7gZ#4Pi5Z-MZ_CfT~tl?k~!K$e`BPbcvtUA!4!@LFv$u7wPnQB zamo&X9v_gfc6jI~6hz!oH-io(FJ40X=f zUSaq?uKf;2wU<0DK>QI}LX&`5$xFUYskOWc(+y>sm4EKpPJ~}#12`=LIFS%ojmhA5 z{=k0r{uoZ<{h2|^-Ka;vZ`Bj*x_%QJ(X|z*yU-rtdQ%YMxn@EZ)+mJ3W9e7N@DYva z;tpDwG|ljMXr7@%7|>fI6-@q2S*>~V8;~;b61cyq3iomCT_}$iufn3G?3$De-O*GK zxpS7&`qReVj+2ZbP7m;g=yz>$hnB@2G#&2J+yR?)p@8(7!|x<~ivF-km!xeBaI|vM z+-o08R@ni^_9Xk0D6lXq^dSB>0lS~k_VNA2o9L5@ZRhm9&RIxPmIziXg}^KeH?!W+ z1R;_EeLfsw_5v!WF2Q*zuHCSi<{ug_ON^G&}pm21rn04 z+=m_6oSE*956I@?%lO7}Bz}sFXOee#WNjV-N>E%;Bv3!}@kKrMkZcW!RHVGBP7s{A zZR9#0@Ellcz?@8$?5F1UHDe|~+<-_T=*RH?6u1wBg%q|E?MN~wGAKW~4}#@SVZ-g} zSQxP?E!Uzb>Wu&Q3vp3EBOW0XGc27or70U9`{U0wE4d9liQr-}GT++_FT`x}z>&R) z-}-~k2;+WL62w^cgW10g`#)%e%3RDIYP*W9hGFJA2}K1Kc~OJ8%lCOZV?X)?G0BrX zTLH`xb3%8wZvrE?NKfZS96wxyQ_1RG-wm{b;`o4kYawGQ``3i%s$DFCSx@lgdC=Aj zg$|Pn{su8(Jg>t&UC69Q!%hp!r5sUvsEhJJRm}SvvMq+na-m;2=O#1vg4FN;iS4L8 zVW;Nx#ahl}TEJ%Uk78dZ2m*y5Nzn`g0CWLFRvs&`nt@+Ym~jI9&OYViT|jwn)cNn6 zn9MQ^^#}=$Y*y{5Wv>NM0#6&*S^?2Z$mC?bHF1A2Yw@nPnBa=4nf01M@z=K%NItR* zm_p#F{^OlCwptB7)C^{sB$?00*uLBch6tKh@kXE*e_>V~ zWz2SxFp+huA_<{It%jBP#P>!}2iJ-5k3JAzlmndEVP3!<}cs(&tUTzxrO=XbYsC0%LkQvy(1{zu8jAV z)Wt=;prr{$#RY~^*1WFDW%F?e=q65?bhB$CIC|=pvIzIa)x2woM)Mb*XHwwWs-$@8<1AmHyLpb7j&JdV@2!W8VX~QEFZfor?8JMIB zlqDe$B_TjAHEdm$2Z^(@qk{cJIv-)4UGLH?CIz`JJVo7b!i$+J6JVpa5nK_mky_8_G-wlFl z)v=OE?!V6Sgwi)8m( z6GF;s#5d&L`6Ye8K56>(sT`u}r4+o&hRpXCWWGdT3X2q4T@p>c?*^sKx!?%Lz^U=n z`y*w!W<#8yNzG{vbZScoDUM_1OHE6DH}Hx4tvs?ZCjmLyZ{1e4d?p?98H>E$gN(&`1ww+Nkx8inIfkM|efg6!(HZWM(=- z=|WcE%ym$QAUB}hn5YJ|zQ*v!KCzV`9Op$@++r~0G;fW>LsMQ9PJA*OqRl=ku*O5k zVBWGfZ1Qd1X=Tk0!9nnmjW_P#8<6>Mb$rF!LdCzp#k*w z9B4xB@J}j5o=IkXE;Xs8CR3mw=gz|mz#Iz*d3oJ14Q=h*tp==tnoIh#$H2QlV{`?n zyNaY^U5YYHUlX2SpHkV?5rT{XiZ;bM_hj4nv16gyG9O~~UzmhV%NOJV}}QVSazw>^^+pVQT{C?FTp#hXP8;iMUc&klzPY zh0GbCp%?FWlRz5!-#oTlWlZXnG@DoTQPZpeP4St3^HPT!aSO)G}Y^G#S6`@?#@Re7>)PiO=(WgfSjSi_d>8 zlq8PW2QUM(o(O?$O#Map`X?{5Z7F}SoS$M=+t+|`=0igv>j;9!JlIzV#+4h^m5~p8 zxvi6Y^hVMlJyS$wM&(!-!Q0O4f-e99VFVxnUw}a)Sf1*ilx9@HynQm4H7jcC7tZv zjZFy9s#QVq6ry!kDEMxeY?JIpYl6uLlZ755K|Ap}r^S~@(B7xyz{Uf182tz4hzs)V z9B*fV(n5|B7+9P*_fhs2|5>ca(v<4Y3pAI(vBs5U5_nEG|$b z)pik=p=jx9+rFEE#YP2cvs7P%6E{$3xD8@T+OxHtvLJq0v7-U9qUxL&)1#t3^^ys8AgVw(5L(t3CMW^1*LylWch&4nBW!td>jw{2U}^e ze+wy_gfXF22J-~)Pmsqs{=;i8<-q-QZKA5-w%F@!x3LS?pa3Xu7JyPgNeBwkHZ1=p zp;p^xr^tSS%%#FUy$1S~Z%bL-Jz96x{B)L@!N@x|gxOm{^(}@OBo;JVU$`C%^m8Mn zu@~W&@L7}RxEb_-Hp zlFU+|I&+^g0L1sdFfxo}bV39~N!YgXFx=MbWOT9!Cc9?>o-=7UWV^2iC23}mA zK=cv;@>q?(uO8I+S98xL^+^vLPb8-RyK(I;T#EYGx7-hpZrOza6(*s^Ht6t1n#Js* z%cTVt`B9D>zQ?2j0Wirrs9VUK0r= z%rS}!aXuOb?n=*mH(`1R*Okv=1MIcn=zlY@(6x~pAEqZKb~ROtex0T z=hBNoCvEo?m~dc0EK*XgAbCfAF#tOp(z1(?<%dPFt#Vlt@74a)p&>r#0@;qI*&unB z%WaY)$eh`c;uC&ULxX8PkIfDUeeOdT0#Y>=SR}zcOIWzetDFFJ34L4>$#q5rD2@cd z&RnPyYTaax@8U!->0u-wQAuBGsz^U-{p$@TEi`hF5y9Zw;@y;f(D1Vj$mBFDDGZsA z56!+sZlGIIGXj!gjH@*f3?uc6P)^Lm4=W-_Fdb=k^VR}^k%u2Y3vd>}(q^Jz_aP*8Qoz_Nz5-`hVXF}NCIlzrdxKUxLes$-yaLFIzQ5gCe zh|3Xw8vA)5NfZOYVISLNJVf8;_r(_FOtSMO&VtKK8qgbfE%SNXQ9zMZBU(>Et2U4> zW>Rk}A4H?i>`x*|@xckmGKGNeBlYYIgz*gV*CkO2S4$Jx(>HBwMO zxhHmuVEA8|=q^4;Cp=vQ19JxkcV=*5bn;qfvscw zo@QD2K2_b<;>&|oS&2fFH|eqN?9 zd)g2)xg)9da5&!`;}3ZNB`WMIT$6F`HOZYx%zZu=Msef61Z2;dLk8KSoS8!?IwYm6 zL9u#z7jjU5ZyRo&CgKj&!p)Co=`3J z8-76T*%@Cxu8Pw=kh!A*nd@T#Crpmd1lP*P5_$_679um4&dCfFO6OAB*lmnM5D_{D zx=~~-@)iT|5=B7VRlUC4os+#MJ#zxJ{k#fo-1H;HtpHbeTA=l~A1w9>`VxTBWKr~! zvMq_*#W_}Nes6xa35f!AxqzMk;rF*gXL=}wo)QqzK{eYEVK}sUs8PagXudi@gbJjbXqX#56f9><)xE*Y1aF9-B>I=7>ba}&f+DZVU9xy;^a=YVjB8~B zJT5WV1qppG3zniv%WZuR74SjpP=kct78CAP0Xk6O=^71ZXZN59W5ar|ksO~>8Yyt_ zDnRXA{@j{l)fZs(G;5pbwv?MCXVWLtFmy3j>*~1%0Um~Cm0*b5PfV-j5TKXJoId*y z<^p~!11QL5!PoLlW;6DJGd^o-R04P2dC=lD%#6|M{Q#h<$lTFP0sV`5B6Zz1#Baax-u1}BM=9zz; z&JZ1j&ZXWZi40mIE>C+X0VNLqTN43PkmoCIo{#MwA<_B5Z1E6|tG{b%%G_(o_{Ly0 z(LdWT&Cj)wL@Esp+hpitAb>A1tSS(Guf_*TjRB$7gG%JH42nD0es{~tzO0}ud_(Rx zkZ^O9%*Mg)>BXLbGUe$m4JRaT0w7Bz@MaW!MDb6xSB4kbg5duQ!wKerk{DdANX|J6 z9ZxHH#WfR-xdb{OK=2*HP_dwj2I_d_u1jOsGmkAx9m!{yn<1t}`R;-; zCLeQZp%S`;Q0EeF6V`U-nU+kx2p{n|1sa;?YxQAKgZZe_{sfg)E)6s=0-_J$v0%Fq zO2_*Rl@y*#h~W27r1T3D>O~4zC9?-Pm zH+)Mx->G)1JWXOA$A4S^Vt3Y0a+qXE0TVtZwRys5J}{j-EO;w z$!5zgnAz?%I)4`3J#uq;LFjWThJhZ{ML0wT_HNt?qq0S#lOM9F2dqXEC|aN;XIO83K=)n{?XK-I`CcG^Y*83P@p_QfeU0N zd(_WUl`qDbO*wxZ_5#%F`Sujbtr!T7k?mT0a;*(s;vC`77&QTfOG?fE|8#a69idzi zG$D0QzR@JORYIG9jMrB)@8wA|MwkQSx%2!`Fq| z&R1dX>WL?9qw*Nz{C~B;!2KChjEPe?n6{6rmb~sW4{7bGlLuSl+h5Kjm(YqpZQ(Le zRPvr5GOm)xxsQ6OxGmQwiau*;UK%cAJT+TEgZsW)0e@J;zkS$yZZOMx+1{{6O=4{z z8_pWy834YHj$gZ%qIFsPi;+RwoA>G!DO%FsJb6{cn6sfPI}&cp=t^VDb|wV*st!%o`lK%Qu0qt~IXd7F zD7mFlMmx&7*{>G2w;2o~ef5{$gs|NSaK9nob8wKS6aDmg1 z3*_ti>vC1Rd$Pb*{nm)U!+$b3Cb{!O>;RR_W-UW~+%^6E_Cq(AKis?&!Q4m+a28e2 zp>IN@(+jGWk<$LrmKR^2sQkFv4cHm6)w0uc4x8MpuFK8gWYX__hK_Q*6hMKM?pGcE zgc;ulR1BAjC=QyExrtvramPzKf1zIr}V3tcM%u4mFl8A46bn-N!yTFIG5h>LX?oSoYGzL0=w5U8@# zWw-Bdb!|3d9=R$^+g5%3tfLqS{6#2o|M-= kv}&?Qa8ZP1>Y`sbAwk{og&pYB;} z<5#?le~ZRH@(;C9c8up2O4LchT)aU9suuVam=%3uRP;VgIml{6W=23ZS_#Zwhr$$=!V$~ju^1oNpA4Dz!*n`#cZ+eUDxwzL}U z@a^UuU96C^8Np+K8!=Dh?A%!pcGM3`H1WK@vph`x-}pF!nt!1Ai19ZxkXPb(L(auB>hTUJ=dQ8JnMu(n27@{DqgHAc(14S*;T2E@ET+lYJpp-wj=o zqc70aIzakg03;!=bJtFb0Q~}Epe`Yc%VbH7c?6xh^%c)g+6tbJaHZA+TU(_;y8ygtK!99zs20yB^>rDJ8tHwjr74L>>e$Pu|v{ zF%LFdFw3P?evA(8?r$qArBim-PAW^HAD;HrxR4`Y*93TZ6UH3*UciZB9R>LH!A# zXiknK|F5RLF1ofyLOowVof-V*wzO-A73gcvWwS zrj)P5w{-WzQ99Dj4!kOIc@+O&Q{pjzJ5qMX&ORPtHySd7Bk><;M>FtX+68c zL2mpTc^(q=;VU%Y5*7!uBM;bYut23a**{Xqf3^5KEmtQY6r*}Fd%**48UhJD3Xj-~#rVD}Mo;=zJPoD-P{>`wPvA-DN74{sIexi^mna!_%w z)Lg;v>tYV6MC1&`VVCHZB87Unx=}AZ2gqfPHM5nY9#&fRxPGEgx$?JLB;zHMe2;P) zA5>v>x!Cgb(38> zBbU*?5jpG;g0`P|OC~($3^bl;1;fGH1BU=3C5Pm4aqIn^62n>t5C<}7MY=Sm_AC3O zf4sTzXyBa^!oIn#MEs5M(PCc4K)HpA6la0g)*lgXE3YpY2ltbm;pO_}(yc$Oe^}H2Z|K7)$QFCmF@z+= zEWaxlL{2QM2NAat?{9o0(%Ato;Q|=}k{%sE)J|+jSpRg8>=;~ZCHF6KFvaQtPxvBj zOz+YPbYtJ|ypnLG^SI1aEQ_yuwXJmK4%gz@MW1>_#>p-&vbZYgs~nV4eWd&`Fl!Q) zyQkI_hpFFHntE4rd&k{-^#xzb5g_od`);%)OH>~d88*l^|F|z{{VPgrsL^ZdNmCep$1Y5${=nh6*6 z^@)$+@9zwnncIP}h0>22DUV+gDIRG_!hXKs#Deb z)1{6CN>4tTkKBh`=?Sh8IaZUy?>qu8vw-Pjz~Hho;s3u!#2+A7jt`g1#-3$IaZkB- zUVk20+I(nJQib6&Zq7LY!xl@Um3g(U!PO$eMvp8yqqR>CcWieO*dwqj7q3*e^Wl26 zKo;praX${jB>#$6o@lgm=ka3kSN%IL1$|NhgbJ1x1Jn8Y*`aic(SM^m!J(>)*ncOO z(CZFo_<0zRWugMM`1t0%7wf;>FuidDBNy^bpv91ee!qh`|G(2;IAc&Ed$CM}eWA>jD$%Q=n}5ks|~BNj2*F$_}N z{Dp>uaJnCR(}r%7n@>55X+QbxP3Bz^o521|T5sA*cW60r_uFVAo}#?^XE0Ztdh~SM zuJT^ISpM(iy!=sYqGeTr1@Nl;5ogJWh9xz0+T(qjmZ1= zYn5%Bx`6h!MTXz*_os=*I#0#h<*BGmJk6C_PU@s*0`VI&bg$-PV-;HbPlU}{FZk`H zX=4N&Q|xQr^4I;QbR3N~1GYT3-MJY~LBhf-P+(0KWA{UD3x*e*kw_uR^=gg{XMJwe zg0M5!S$uFm&_@kpeln;*zM-BJTmf+eK$3xi`p&;cqxwcnTyl$Q5`?*>C*S8m!d(Hx z(+VEiB6SH9+{bdp2hK!hjG}rae|sh|@-M@Tsukq=FY**?F6y~InBs3Xti14P zrY+d*`0s8@VV6CypE!G4zLA#h)fJXzLb77QV7QP~??5^bYB%B-sCWy5?}nV?bdT6{ zBTESKPAAsu0 z)KhIIzx#w?#6+lUP`B0;uWKBbZE<#tbp4qB_lpP+vr+5(r$wjfN>_ue%TumNK;`qX zpWv#RDq+&zoc=3;5@h3O_7&nhWl-FKX^s&#ZSAYf;$Gh_vMJt(Fw@3A++O_Ub^>f@ zdu^oey|K?~b^UA@UE@8K=3(L25fFKa$j3#WHa+S@ukIkNq6Pi%0^wro|~4XZ|0b3L>=&0}h`;lgyq)BlIG z?+&N>{r^9XgJU0?h(oqQ$jUfoh)`MCBO{bi_Bi%2;$%x^RI-W4mYt$x@0AJ}+4FZF zy+5D#`~AIsfBe4J@48Mdb>!Ty`+kncb3E?n^~Y@TQ@;20IE%>7iM`6bi4sOxiql(_ zWuv!eC*1pr;X0w@9DDOs7NL69sh6$8GyOKzOMODFkSZDSwvN4V7^o*94A>5jVD;%D zH9gwsWHZ}qKCt}wZ9+fIr*f zRX+*z+mTL)7WqEDm(OxvW(AC04L2ulF{@mJ->+L0e;`Uqxt9Yw3^+O3pCWh|IEuDF z%(l^JZzU@M8zyUWOeRh$G}0GxT)kB7*OF{}C-%-cIlmg|M?V*2&1+n8-X4WybiTe& zfAF*LH2_`h?59V+wM*IPUzIh75I~F01&6`pj<;?J`C0f=3VK?0R-nSFOoHLnPGOHf zKk$$#9sR~7z^r6RsQ0p_g(d!9c%+KzJ1%0e$e4CAy)5t593NAqYYU2% zK!N_cRrcEb{y_CRGrtlsCQIMJed)N{SL+n5K8TK3je6v_@^kjTW+*fEDdtcNR39DW zyheiu<0sC$jIn@R(NlqQ5a1K*l5p%XtQME*U3#fAX)IgQ+u9t2U-dz-Zb24fF-Wp< z|3ryW_O|{avXbCExR9+CIHUS}wjonb)6zMGGJ-fwz1$>Aw8YdWgYf%Tqls$<3|{DL)S#US_=D$@JoQh-l$gGK4qt%fQmZ)Gi_| zR?9nCZ@s-g`3BfG0}?eGkJjX>vU}uHzWc4Dk3LB)d0B6zEbYyNP`)_pf&#aUMy)QHtu)%QUZ{yX z@9sKyc>AFCyr}in<>@c8=*@MMUQQ4J>5BHR;Obhzk$ba4Gw)whe!ZR6Y?h_hKjrFz z+j6jTq$5Jk)hofW)x`BSEj1SFGc+@2ZvYVH33R4*yxcATR&+Ecqb{#TlKifx@3RyM z&Jv=;p&q33X(b|xyG|ZzKg84P?!oWIS0`^7X`E*ihgMNz0DSm`JlXHey0YKdeSrHj zUE1Vuazt=vM)q6XQkpq~m*>p5_=&-|2Oz6Im46Vxd-ZSnd4CE~#MoN|^Y~2yiD|z| zPkUplX2B`D0VfBD@%`qffkXjv0vx^1qy`GPEj}h2lwCa$O(&qhgiZ=)~y_UDG_?G#Vo;B`eLhvIzH%8lMDK2wQ>9(sPI7#i$M&E>;k7u{JJye{)UOqiDvdbOE0>{y zw|9Y$+emIIyDO1AC`77bh;1E+Bp1t;5Z{>(>%-&6H~mBpUnI>3*n@V^3>^t}feD zs(Zvw+ITG0EiKrH9`-aeNq7hOPfw6ysBh7*)#u{7v@7_zOx<(A&5${pyx|oE0fU7( zKDe54t+SvDA+b>2g?l{Jf958-2n6-So8GC=z{hqn_0>LlIL+ilh<*{TcMe54O&t9M zi#nnfPDjl4^}kkd*Nfcf$167X*T_h*u_{M~YNyfM3Q~+ZQ~1LlUsQgw zQB&u((W!>PIl{3c&(gU05kVm~Pp~(st#t3YVRv8a6~6FGE~1R4zogh1d;YTLFB4h; zeX`{tslMCw4g=g@3mHmWO{(rC@jJ|t=QHiBPEnjf0%3bZp@EcpYrg0v`6GNNhe<*a z4b>QI@BQ?&cRIvddwY0Hd85*wb^P!k%Q--Wnp~Nk1?$EZnZW6>KlQL`wpDg^*TAJ% z{m7|X&)(_U+1M3C7Hr`+xt14ldf+EFL;H_A=@5kzc6WKX^YbR0eRmPX*X8k?h;)YP?|%_Jo4?}?jeyV+_EVv}KG{FbMCm~`4S%RW;+2dW2f#END+MHb zC_Fy?=zGv}Nw*pJrwl1O1{Sqy{B9pa5|{c@m{C^Wy~Cc|MxGn-FmNywr}k{B#;M2ai=h_~RS?RKWCN+{9_)*XsuqOu3Sq?{AMrWl&gd4vCR%af36 zu3Iy@M*&-&FQ?>E_G{G?LyClKROq?nx`MZuI71vKR_FGkU)|iCWF-1Rig?puOU=>M z;=eK1?)u{k*J5U(VE@g7Yjn>r#LBmk+}cYYZXH%uFGjY{-_}MYDQFI{w?3uP+}=H%4WAj)99b z+L*YAbwk;W$lCHADs{(mt+s~B`5*9p4tf6Lx-q6pu|n^=3gMj>q@4lr-3lVo73}R3yn!)TPqJ@;TEP;ABx00mPkHBdM1yy1aXK=#J-0=PqsZ^kP0h_Dc@6QYTTXJJd6 zc;@Ug_t{&+PEDIXD|Ncet-G?wyRmdqW;GY5zvYU-!nMCOZ3ycmhJf?@NjumJE99<&7#ftE+^(1$&DHewV$!1fHH8 z^IM0evS5b+MsF~yw0%r1Y45SvLzmQQE)b0X|MehAz~Bb)g@rOR5mFc-p{mPE-RG*H zF*21~8R4n`j0Vw#oGs(^Tv*~Jud{l@D|lzdC<9WijKexGehFe`F<^RxGv zfZIajD>TgA6f{ond8nprqfRzFC)4rs*ZxwFnN@H!>pOH?0jG}mSna!ku>PLI_m(i+ z+cSf(yxXyfy-zwl416DP{&F|F(aSmg#%A4t+#>2Q{F@r5g%;G6j^o+!?~B#W6S40P zDd+cQ=doGa$(-uc3j&GS{&ak1j4B$m?boHuG=)b#-&MlgKxa!iiL(c@2!6foN*A^g1Q+q1L z_bF4hunn)y16=npNJ=GldgR`ldRggw51Wh^0toW0)#OXY;om-{7Y@5+3vm&NeSQz( zA*A+!;|pS{>id&XV^xby7j#Aujr-r2_YM!0MC5@DBr>h@sM<{fZZ9t}cqELJ;dZ^1 z+pDS9gMp`ypNNoh3?UNM^a?dZ&hrX#IQ6G~7Zp|`jOb3_DfCbZCsSA++FT;^+FQ@r zyxt>$X;;M{=b}`i=?faIHt+m45D{1VMp$khcuc4DR7|&&i@AM$_Wi0=EXXhxj;QD$ znK<%_=@SwkA@J0C68Vz)XR3x5`nZsbY@C(BF0=`Z3yq&s)r~0q3n;FBhXv?Xsn$q_ z=QEiY!%3)2=G*lD9H%Qc_Qq@b(Vl~Q)7v2$tEG?Ltq{3pV)6eV3GlTBHz51hY?rQ; zHp;E?6I7VGnW3C5To%kx1G+eYvY(2X0liKPwXx~f(6CnwC8A`_b{eg~enD7?1pb<+Ew_&N z-q%_wIZ^B0nfZ7Icrex7Uz3Br1f=wD;&|LInKzn5(&QbX+g{&w$2t*l^h-Ycj+GUc z>P-ppzH}jComT$MNPC>${$>FgwMc9X+jtE89IwQ~~77;%x`p z%d1D66dUTHgy(K;7z->ThpQh<-)FGz>QjlLRe1OCN15lE*;V{bP0C&jV&K8!-Fj{I z;N2oRSuUK*5xRgZDDL^$Mzn~>Io!T3gf5nzZBY=4j>AnE$DI@UE^f$XTE4h*c0B1e z5#$)}Xt%!hs_lPz0aD`~CiAV&^m71a!pn~LHgcsch1IHliP&DBtvS2VsIDp!P z2+tr44hAi@?K~b&+C!N+CSSg#n&@A6v8PdwP~|#p`)?uHHHx<~BH`BJunSPwM|M#X zsx}F!n&g*zL}ZK>@-7~sylE7wu`H2AMwQZwzK1)`+q8`~-2KJ?0)CyXW{HXZ5e9d& z7t-E(Fja5VU{LFJ7bB^lDZ;e%b5P)>I*bZmjT=(D70eRvILdhAnPd>q$}v$H`+(b} zX6JF06{~2WrlXtfyu-GAnZ-}lh{G~vea9xv8holuo!G5F&r{cxhRAGGC>e)+zlfCm z+I6WYa9Nfz!nVs7loJioSUKH8fG4|2s3;<1FX!kUX^65KsF1cFW|I$y0rFF@%c3=+ z&Q0<-{luTj2($LIOW`!rCKo{;5OC_hB@A++SK^AIxewO6ge)!+-c^NJ_XAJW5ncR8 zA#%Bpul2P2bQHvLDT)`_0>_=WxE2q!Ho@?>Klnjd9t`2@1+?H_IetD8aI}o8^u%z; zT|eCMShzvs$rUKZDL*JZ=H9&`H<2(gaR#YuQd%D2F`A@x&H9TR^k$~BB1Ksak8c3_&)l9{T<8q>$TxP51#AfD0ZcZ@<5?b zlRLTT5)bBp(|_%pDdW9z7nZwxF^yt@II#N;mO~{9t|esNSX4ob5mu<+zz7|LxOBQM zLd<$7yHZCh?F@*vd~(otc|i$3I`r4h)tRTOkJOl;ol*I z+LfzIg$%sZo(BQ^>kJ##{=Ow@&2%pX5haWyO5@^0Qy^PBf#MN1hLVFHbKf~Hqv+*2 zwubX+NafzviCQcVTW9QEjX{h z1sOfRw^Bt+sr(RWp3-$Tkrsy~nh$|)%=kNgb7YN%E40C#A)ySa`G1BKa?;{?Z=TjW zZ;H8f#OuVS+L-YLo1RB8KrU#`(M%%oN_l2@l_fp)B$GcHgBwNcPGub9gbXrG_?LsY zQCOb3hHg$~#kEfjpUaJ}C0@KGGF0Llf?ywNDSv(qu0~JLfM~gpu=hQimV?pQkc28` z!^#mS17(tU>ek#$l*$W2tszGgGB-vwIPMuyp1hG13I}6>&!Pi4+-%J(L)B6W#U2qg z?{G@U(aPYH{y+Yi9J-lxYy#n=v+|m9bbM+PiZxk}b7I3jE9WLu)q?)w$tI@SLU0-Lvwz{`#9C}Z0cU+a;%0{q%f?tvV=Dpn94Xm&HIiiJ8 zs2U6nM&dvY#CHl$C_P9KCS)v3Q*)hNafJc=#d_s$HEZr9Ryz>s z_R{&Byn$l9-#_l3zHSjx{I&Hy+fvIq?%2;>5FJeOI5LO$B7qKbvoS=R>y7BMth4tZ z4R~Y_VSJQ3>2ItPJf2QL8P1`a4Wyzi1LbNgCCJwu#;Kmbi>DYu$viW7bdw|bTMXEz zdZ8FhYKNi-!juPx1awGMo~x7apz)%_xa3Rbo1 z_Em_YKNe33j8gq$2>|Z}{7PBsr|4G}DxvEyw8+!@Ph0}dWJEfM9}%Ye8d2Sj^b?5U?mCM^33nhNsH%MQNv6sDl8@Uy{dtmK9AM11 z9;6C#(Q`Q4OeEGu5gP;N0KFBs(eKTSczm^7P}siU5M{FsgrUk3h&tg`p>|VCdy}B$ zcWe;v?J{)vC*01>hy-1@7MEijXXbU1@5Ctl(=p}HsFpnpu5aewaRa=M56e@1ilGpj z4e}FTwhzP`xi3&z7M3-6(EM5y_8-Z!18Z#)b-BS1iN;Wp;Pb*~8251p4MxESH=97# z@a>@%)qFBh@;m_qI+ObG3L2ug5!@y`*?qxf_^7hg$O8BUU{|Ge9HaujSrr-9pQD@R zk7J!Qywt%c3rxr`F&UFI3$yrr)omZW{xS#I;j4FQxWMpmGrw@$DxG0a7G)w1xvmvX zc(;C?7LT0dnsd+>^S>4d=E37eADb*&syv9r$NE@%<2`%B0qHms3~o+gR@7BU7NqRv z{W%T@1=I_Mlt+tln=rpqhT3w@q!L)Z(h6PgPpiaP-qz*Q2~^xp%WD~>W&KP2*=DBJ z_+?-?Jt_u9FK8Cred|R*1({OE?zQlxFP$S1EK2&jdMK$UQ?+XYy&L4#c|DL_EvwPN7mJbF)KMR7}g+|J|i#gwI@u#HV6c~G+syAsco<5Tsz>+l{c zh#AkZ#mItfrw!`w*EtH{e`D!_|66Xgj}f{@|& zUz(kal%V{E4~O=%HivqpC(%4Xds4gf2Cs)`g}PjL;R5ESM`C~Ps>BF8&Y2`J1p)b= zF@wev=ctcU@+t~rk`!Sf%_CrvGKdY7W~7Kd2f-24QcOOjC2hfQO2Kz_KCx$Wxc{g{ z32u_|BYmeIOwgN-nqR^i;4{H?FJ@l-d6$s*P1X4}ZM^<53-br}16uX@)#SRK9PHqf zdkMB>gg7Z9Mg(}!(TBlH;-e;R^oSLi*AhqUoNrHx4!o6!@J(D%hrT9LN?sCuuC@{U z>UJ^o!{xTQOyKzF`$xiXYmA}cEuvxVI@RMUEP6g!b@%E>$p;@K7t!C@AWm#H*fuqt zBRP5RJCMysf=|^-Azfrh0f_f%FQ;|urSX(LC2KTp#Ry86)mWB1WSs_006C{RA1Ach zH|nH~nIOVaK^(0+-aqc*tB-Gf3-KA1$+D$kXQWQJiJ3`SzVQ<0QIXUD7<%106Q6P7 zJP*YAL_%z9_r-Wi{R)<@6bvOr-6Rc#+J8(Vm<|psb}%#24P#I21Vf%m{$dD2&YnY) zK{$(3>xa@=T(~<|d!eB-erV8EbrPsO&p|NIW0$_!! zGXVEt7J|Ttr8>%fCkMJ(K&-}XNOkUe7}4nS^}Xs#AA$ffXpDF&E&T_N{8^nYA{!C( zR@D!Ipi&^1RQ6G5m4$TUORLKNOPS&4`48FG9Aqq;40p}MPm@f4c~w7j#n zjU)-e&owZ)N$Vrz4Rrmv${JC)7F3sN4-Z8&p1y{c%D`|NnA9-VLtIRT_QnNdnN`_5 z;B#MaAWs8_?4%GN&%`TJPhlaC#;;!6JfOU#qGX5LfYfAbaB1>UPhoK^>L70yX-f`giC^u2VT?EU#zLu z;XzW6bK|PXJPi}c9Lb8XxFQB3T-+Sw#RCvc{e`7aux%SkC^?}NHx7HquyX z($jYznJJY2)?hhRa3E~PItdZh3kAluO2YeAUu1!C%i^%uKK2$t>qG9~r_S9H_+X*d zEE!Y+2_Jjceal=shLXykl7wnPD{&qd$e>z*ShHK!xa)(5Hl*o-XJ?13BLiulhe>t2 zAs6Wst!1xX{VP>~6HWqfuof*U+A~Ql z78LRtviDK7)jC%P*rrn4r=cH!Piv7FdZLR3Kv#`48@+lxlxS?ZSGVn;Up*PW`ymdZ z{z16lE9L^QYIAVxBsq=CVu5j-|GM*tvB%ll%ZReglZ)DQn2l+bq1tIC2OH>S%gQ}S zm~*~{v=0gOpBmPgX$$~MMl{t+F@vY&gz`PZkSlww`-v5mKqY@6Ww2TQFxMb|k8Gx# zkcA{Eb*LfVy}X}B&aUd`bY{s|FIco6OW~s(Atm_O+ZkfiW&A;Dkf#tySoSL(b|CKR z?^x2$1UwGN>_+EP(I@D3V$RQZuV7aQ?<9w?e0Hg|pkaCDvL|@D_8hkx|9Q31;2x7$ z>}4dl69D&_K@z06cR5+`1b+2-L!QB0NWdKXh)mH0R50~-5{p;G--=}vcH(#%au6*E z52eI$Ki$TL`z>)->^ShE5$kced94DU+XBy_&786u8Xc)@$rAo!EkG7r3&_qbp*ZcU zODj0pS^vRh=K>_qOtJaip}PZQ9B&1p({X(Wb3Yz^mH0N^db%@;uT%I_M#xp>XhoPd z&9^=2{ z4Alx0jeT|)&5A`cAiB=0B!@tF8Y5J7Zz#ZsEX@!CrvJ^B06}p9ZVvh&GL$nr5ciqX zZd{TQxXn`Lw8%fMp8LX-irT!xcg68t{N}*TFVrncg8( zg{l&BLf1tgd^tMJ3SQW@Zi~~g3T8=L$-RN^ZBGA*ym5PCaUbVdeEt0(Dh>P^#K~{b;%%8h#67;5VzX;Ug#JMcpv6_H_nU;D;My< z*EZZ-s4)+qH~AHW1YP_xq)8$&p;%4Am(J7LKY9dABG-_TY<~+d7F`SXQpWUqI*f0g z;PshNlKvK0AyT_k#sIN_V9C(ezKJpD`zQf{xdy#?xQ2&bh6sk@1s?2&oe$Xg8t6GK zO(W6i>U7<6>BQ;BqTTupcxcRcl&nD9bNvVj-HJN_B2j6O-KA6}Z>8vOkQaC(MA^!I zS2aO!A8CHNL^xDB5vp_;JJ-|Rw6Rs-x1)^Dey8R5s{QKYc=TiPJ=@S^teV!L&-xE|j1 z7k=E=qVe&5^|_)M`b^(KdD9W22+kWMR90`t(ubVh7e%`B$zjgR?QnowFEG3~IIJ^! ze)OC}2{ka6dN)B~ctaH>w=_sf&hmjbFR~+H{4<7Ju)`Sk4D+{At2Mubx2fY!;e%7a zNl?ZSg8PbXzJ6+jShpyLQcsaux(v-tF0F_K^_%Rjxp`^}Sr}p}zn?pn!kjaFP)o1P z6=GI-Z_Hubueo3E(`z1`@|T?YGQC|hAJF!>L>7?nJ~J$K)!6PQm$vu(<9WBhoesU{ zeZ&gccl$d>sva@C5Zt56pSqZY5cC>-Hj&lf)bgzMkR3froJ6j`l8b2BGv(w6{V{Jd zijI1abZz^d-jm%y88{i4m>i=o&`4ff2)%c_z*l5gOTsK3&SUC0HG3h7ifn=nL}KXn z7o0P4&>s*<5;Cz3g-?tx_Ib{g2i0Hur@KJny`{z^A#&EG!phyH zuTPbutjPZmwh(!%c2J$-EbcxRbQrfb_vPb&WZ*pQ{Ub*q2}XLtrWVU#*6p=%s{c>#X-B3a(125RYFPc6;AqMV~hc5{ke4ZkxqbODu*XXm?8B^pso`N98SkCI=;^S z=l*3qq5bdlNX1?m439u?VK?GJP*`+|#b%-dwigbfMZ2_c#>g-UaI`8qPO8ujxQv~E z>tXKd;dKfl3S|r;LwyHY920CUMbPc@FMkMN$ps5T3?g&xsuLWtPjuN#?UcmrE`M-L zK}CD|z#m}}TPy1)8ve=DrA;G`P#v8!>Vd3}=2F()d~< zZ|qRTRCtw`w$BJk$tKg*Sn2yjEKMM5zt-Jjnl1XU^2d5 zFFg-B@fBg{HiaO~H`ycnQ$SPZHqqxlNqC>pgW9?UWFLK=$jd-XVB&^DdKO@!$BM?yhJt~+oWLtHMur}S2?zNH5yfCj!qML&eB z*;$ZnR0}<<6e{7jj?#wS#3gs~`cheqW5-l9H+}@?f2;U%;`Al#m=3ia`Y9ag1U!;Q z#G6;cW{(WFmv361AAJfOm2lAOD?eY!4baC)Obkbz2Z@Eb*M~N}4kleT*e~r}LRW1> zT)4%&zJZ4J|E9>!T$!0R@(z3OD%Ozz9b+``t!=sXHOCYAtk#VKJv#{J%sebphP3_H zK%vWia-vq-mC!J~-7A-&yLN0%kxXXw2%HZ8Y?_3Oy4!X}op)2c{#y)BV5Kk9mhrFoZ4iIcyp#Wuqi^pPE!9RJGKdPruqd=0Pd*7{Xj+NUe-eZuh zv!Y@8ew|yAggmoGl0N;Pzz)D-xH6w|S^nV^@)Q-wek{P7bX~3~?+gm>B1Rtha0yQA zDY3|{?ZW8%^EL5z?bd3EeiYOauZ+DTy%Zv+8pJ^$y!O_-ceNrAm3SO07HuV?(?4{ya#Deo;bML(qu-{2|0X|DRtvD=_0ivc}ypW#{Ui= z!1s_`T8=GBhsyIPhwf0FZ%KDT&jJxMb&Eqb_B*zr=Ub|{lQw#gEYH53tdJIVG4`Tn z-eRZMRh9*Lw{7_}P1m0@nN%cFHrB|ed*KBVt-oGfzO8r&QnlrD!j*_w&{R57ZcRkN zDuV$H2-ee${%TorvY@6Gr5Hw%g9?K38kDdc9Ung?SPdHT{58=6etNKLb^Pr$P}h_Z zj46lSsDfe8+~Qd+#mKBqDmjpY&`y zmU)TQe(Kq^LhXjXoLlgsX`ZeY>b6IgW+Z`2xm`@edHxY%;M6n-eT!Z3^sX~^E2Z-n z&OXfB%_}xigbb((Vqo&5h}F)mKj!&1zjN$}`pY6c%Y65ZZr5&D5+r|X6lYSHe)l(B z3%p%&BCe`+eo=#=+VujS`FPeOrejh`kFcy`N3VS{q|;UgtWH<75HUN~9*wI6fKvW* z@>hU1!YFZ{<>pI|E59FY`BAXR#ej|^_FFGBy;iG+O~SoTj=X94ZYozzSK3j6o~T7P z^UiQeR+;7(TIpXGzvN%PpmmN<=$BNwe4yM|wR7}RrmvvtROati{|kY;lb(z+US)Ui zjYvg7_M<~T1eo^vi$iy4?eDw@POV_D5$^gX4$-=eYz=;8_Q5xfEcWBK{~;syIS~L< zJTMxkJUVI(h?nAD3}pLp)5dN7AxqR*xRcfV(bu5-WUueuT$ylFhyOk8XUwN&ZT_kN z@7Q%wsiCwj=5`Z=eb(*2`j-kep1*>2p+=)iSNe$2pSlv2@E%tAlS_R0e9M_nSy*)_ zMiG>8#hgYEpu;(iM3tgT;PxwYFahb@?E47puEc|>7%+0npdstRHHu!)uu_OmL9AJA6ELV z5yh%P6=&+?rY$?#()L~rUfXj+X)CHDp;L|WJP7d%1DPQ2hEjlZ#ys4*Z43nF$rW5b zWs*QwmVA>e2Auia_=6@;`RC(+dF*fAeRFS6Bk4ZkL-HkN3KmIpGpf7!ye~+g9FWnL z^Dqa>rctq<82b8flTgWMja-<2`8ASQSqEwUv-ojCb73z%GZIx?Xc_J0<=55T zuq6HiPlA{NsqpT#B>F$S0L03B+R9DXYW!ow&fOPZg>okfx2eBgz5$Z*DCjTlvc&}} z;JH~X*O|Y*9BMQB#X6lZ?;5znX&URS2W^nv?fBw-{qwJAkG(u!o&w(7tp(-gVAaRxEaz2g$%8iL zJLemAr$h;xbE>G=TS1wa3W3O0b-8jO*v7TSbB$HRrV9<4$E3Cg?NvkTJVD{|^=-k< z0B-$&E-?A$zi>S3V19>h%d1!hUT9^13)~f!D!PzQ2gX1 zQV_CUh?Rgd-MT9p7V=~C zP!Zhc%N;&)KU^N5`#3J~RwN1*H)*`P1S4Xo#J?FeONuK*#?*Qry#Yv)(N2_@Y5@tl z^(OcsQqPjfh=6s86VC>1k-4DD3AM3Wy>R1&*H-zP3>76PP(BBy;9J|xV zVq0qbmI6=%S`Ezo!>l_`uCM53qwo?h8PkJ~&gxxo214ALspw0_)vvx#3rVoAsq;mGzRaAvACh;J9(6<5FK1l!yzwZdf+n8a z>Pwz_SBg-Ue=i^&SZ#g(JW}ial3$m-9yD9Qewe>|=P?`s=S_}gsdp!u8T%Dt~T zMx3CZ_hpBqx5&2`ciV!rtV!63dt`{5X79r9VP}cn+WEI1VlGtIw<~ zU*)ko4pk*asBf(wgN~v8B_33v2h)(=6uKFq!H8#=v>pN&z4;j z8WuFG$JlgpWEOVk)Uz~3M?$QAe95=}oU4+z&u>sd<{U)`AADC~;i#~m zs+SrnHD|lqq%D6;>wmCS-UM2bt~ZIBeeU7_=ABGU3co;pzQ@yGtjo}qibm#f7zXvh zX1&y4!sKE6zYS0`4;*asvC>%AOA)3t@)n+dKYXP*EE&mG2*EY!3_b)M;QRw{(GIcNYJeB^y#F2ZI1f`Q|^^*JfGydI93wKZ_jOsqvS8V}n`XayR` z+O7*V%GclZ=ual>`uyUNJYGj9MGWXffuIB9!y^fzre}-9v~)M!+KJKf@T|nBMMO$ za4gLk5|rFv30PPf4_$wM)8djt$hX;iRH4mHdW=wr@{4@*Yk~=|lW(Rmb2^3=4VIYJ zaZCz0%bm7b+t`G| z$~1$BM(?XZ75h_dpu<+!;g@n^F|S?qn^5ip@fEuXj02AXo&a^l(q%GLG5K~I$hApQLWDNvLB;>&@|wbKkMERSF2^a;P(h$(_l z<71V4_kZmzuDmN8e)f1-Qy94_V`&NDjBak=ZM?Q?cH95Wa^`x9^}nna&VV_(Oj z4*kY!%H|MCy7sGAMQfhr{8>#y8?pC$Ektr_*j7EX4p}p_5T2R(-qhMH(1SreRtfrq zn6~Stj_J{(akqu8q!5fIsyThfj%FsRJFE%M2CDNZnTh?y&c==$@a8({>X_{FW7lDy zdia#-(i(ePfeq6ww^DgaO2ug$B=hkU@NN|xOOuw3=fObY1Al&$OW%9il= zz+YWAor=@Ixa_T4CJ`uTMh|?Z?4}mJKOy~THv74!jo4NX5)xh9xP-4{i3g}zV^7U zOw`hNZz|tt+ytj&HP;ro<3+=BV}(vJwEK7G%;rn~+3`Ss=M-S?$oZ^`|SM)tS+b+~dbr>WNlr&63)U*N8JP$Bt);@@Br z)I`bg(h)h2kL)gbS8-gz%DD2Ex@N}W&I;uOp9u705TMjbDA^5h)QUfN_M+;9eIiS% zPV!-IAu-E@FM@682+nsazpX^x!{^b(#)W!`)jsZ}6A|=4HIIbWZk#Uwkdv~27NAxT z4A|GNAbD+QmZ>_co99qH5&jVZ!HAAHm80Gmw~i>i zcB4d{*>3o;8I3FY`KuXlPgrQn{{UiGG%@ksw&_@PeN*7b3tXDREvSS0;S<5LApDTH zNOp(j{&Z8gUEdKuz+k#xE~I_}?&7}scPu4Us+=WZ_|EyL_`4cb2OM0QxoM$jiIqNX z1-cC8NBJ;WwDZj`lTIffQ{bf1%cpMO|Iclr-+vHfLyNsgY6bRP z8MXl^(KT(#=Kz2s1xy#+QG&KSG9?fwb8J%*6YI{AJO{p2NzLh#CJh~?S)oErjgPqe zm55ZA@8D~_TXNTjB0zh14!D#d^{S^J4g@^{<5>4D>?`-#KPUEJm~8UE z<9AM1y*%xMk*o%^n=m>6!YS`hRJng0mxH(rs1%ta!xZ}}Cc<9pGmIg%z%iP6FVYPP z8kCr5z>xH}RR4-JbYDAbDux2e`UvHhmzrt8piS@E=5n0;Nm&Z2KNOC084*8uv@>t7 zs`ToB#$zr+gt@3~FXow)`|Dl6Jp)8TVnAfRy&4h{vT8(_8+^OcRtq$8Ub1cmm!CWV z$XV^9ipd7J5#<~`huHlIMsN_1#WE9_u_J8@{+6KQu`QR{O~!k(Eps-rzQ*OUlVv>;c>jDW+Q|Q&*1dZjYCz~+^}Z*RBD`MO z(3waF5>G!qrMU7_Z#SvTkAniYy^t*^gdFk)A7S=vQdt%3Z@VzqUy5H~qHQclSrj1i z9tI_Uk$?S4SreHHz3eS*p817gfRyS}#?=TM8=X;ctk|$z(KH$$uk97E)kR}F-#;t) zOO5>;EZhLkTlnEA!gA?T#w#<~qX+0jUaS+T4^5CFz_o*12BC1r;gYHpBS!gv`W~?v zdyoof+yA`;!ag*VfCRPn9%NN?a}>kW{6Qd4IKuQ1FkP;95L`>Rh=^}TX=+F$UNrh5 zsQG>1WgNAv~T$tnj3?0DYdY8}N zMgFJFZo7cjbi(djjn8Y!{s1BJBI;;MMdqNj0Jr6gPRL$@>xOfN+0M8v$ zHxq1OuVSE93JMC?A`+jvz|+Q_e&od%5-OW#;I@d-N@@G~ zpOsoTsLDUk0x}bi@rfvq;0A-JI5^i9cexrN>@pRm9x$MzUb@hw3tI*sR}E=b@YVxt z*5k=H&%P+rmP*AAy;FJu+H_0JwkzJ2;&ks;2h0uLGCk}AobHuz%QRQ)o21`f zS(GV^Af8=u?A~Q{BO)qvo@UxjVsbOz)hpYXiGkQ&l2%lthpXB1~JQuUU(Gs_8P}2M*FcL9mzZ>$E^E9wKK!4 ze#mXNEFXhV?q=#`t6zP#CUgf2q_FUM~t^L^w_)Zgyg2D;^Z?vCIitWlR08L_LnOgwLiTO2ISGY?; zf6<;z``evHzglV@wSBxuy$|1zqZG>7uHXCFeVyD=g@}INe1V$h3$QPbX4~Ifyv4+6 z!@?wA%|s=3!|l6}72LmwV@EAry!al+(K&G^3}Bm#@2j)SCBVtWUd+AKA5tiBo^G5J z?8jzB)A9uaLbK)CRD%a$Y#J+2pBK~T*1WS6VQWSLJ{tc@@B?>CtbhuI_d)iq@a{+Y zU5B$t^5E&pqFdZ1Pqucw`CLy}8fWx2gn+BV+|2x%`t&$I|jA-w7T*;Rxlk z^A{2{i9q5ts!C0OHV8@H53QcN7l%FBF)5t(NoUVW~7=y=2gVy&^ zpy;l0BICL73cz60a|zWT;;uW-_Yb!eS7VlK9MYAs|3YK!Q?N#&bPKXB5+4od^CGkT@y>T{@0O=OlRF z@4|z4a^zCVkZ9x9g`;O>+uLx%0QyVM9e3_K5x%OVMErP zjC7A|Mjg34n?KtB&7a;30ZKn$dvv&YoG;T`)4w!DdC;1mtT z;ODjB5-03N$TrIEJ!Ee1_aW7>ZXXiL$mAUB{>_850@QyTzpvhzQ zoH1NJUgp${`7i3hKTA~>S3oz6p>sPExTXM|fgY8YoL3-!pik{0ALL)RljD$KdvC`4 z?WJq6CQ=jf#Mfd(@haE)Tq*i1@tvSLtZ3Rs%8ixV<;)pi|6G3)0_cJn85@9cMhBZ= zW`6AZIrxD06f{Z0?Hoz9x1&a)%*{>&{`LyT;& zdx1H8b#!PO1N!@MzdsD_T1y1Vv8r)_rH}uj6aBXy48AEl2TF)V)pIc^phA12d4=uS zMs~P_0C7zU`fE{%UWvHi(Hc}$>zFmE(7Y0v&oSQ1b^@t0klqe5ZPhyW3R}34kz)nt1er2Ga*FHv%?%KU2_<0ia6VDK4 z!GxCn#Wp2-VRJ40ax89Z+lM$23_x-ipLhN*H~BgW$Lg^k%^`RP^K@OCVfk2m_PAFJ zcPWO*njbM1O)~Cw>bmQ_;m~K zBLGcL_->2g8Mouq%-DOI>FU2E3rJa;CTKNHWhy5C>J&#Ud`M&`^J?QWbDSj?JcxDW!;=*s&W))G2{ayr5Pwf(%FfeO@xl-IPumsyBa%df*V;KVO(M zAnlh$tw_PYvT^;9_W9=$tnW1tP_3V0?I05UT>L;n>;cc{vV$$R#m@0)+e$d(`+XXA z3}Z`TN7I9jG>jq-CB>a4Vt#JF&bA!zDe-*c zavdyVeNOb@zt#ZO;}QgxA&9b~0dGum{gZ|< z6FC*w7$@G;J!cs}D={`A29Pu z6_D zcfyu@XuEY68t;&f=rTKWm{(-|LXfMK7gkraA^}=S_L|27Q02r4FmZ5xK<_D={OHtR zCGyukz@<)U5jKG3}XXFUsMhFbTkJy(sMP>Vg~vzx478BBk=XsK1lq!5>xK>+>V6^|ICg}?Lex8A|HH(K0b23`qtfeHvSt% zHTE~5pR!YH3diAWC5>N2_*;wL17GwzDvea(&%r5`?3M?k*Cs0{?wfbLUDlI~d?d49 zW&Ey$>%{!zIUlg5nOD<7laG<^ETy<3?N1^!_?ruoJg_)}HME!C2H zH?!~G(n;)lOZNji^;tODLO_8yZeZ zrL1{2qChHQbNll0@|Q@xY#V^h^x98_v#(eD&ccprNVpAl*2qXm;2%mrUU`XBk8ylF z4g&I=VHqN>P!#CNPT_$=ss*H_au(q7o>a3tU7!BnBAT1EYV8>J`xExC%YfclMUG0( z0E5#ZI#dWjK01d^CPQ3>g@5-%iSfzkv^KXna&LlY)(piX7j8>5ccwcPp7banhY(P8 zd~bdo?@V_t+Ph~QAhfjUAkG`&VpnEMwXKX)jkJVr zKUqLu>%a}X68#h0-!1iWzK|j!e9^4yi>d##b}kxYciNVK3%p2h&fB_tBc0eOcbFxC z^IKi)Blr8UZC>YxWRlckj|X4KKXtC{7%CepG?W4-UXiR%S+g3Pt;TTJV)CV%Q_*-F z*KaY%M}42!X*-I~*3Hdj9l<|T#XrZ{v7JlT$~j1)EX$bwg~N>@Rkq6}@_IuB^M zx%)H5)0NH&C@Ld`Y{(g&o>S+277Fh}Muz8p`4CVDfx^Q-<$Rq@Q|&q1ZI%j6${9+J z_{ovvHVfji4lFtNqV&M%!A{>mZ0`WH&oub$kN2h!=>wOZ{sOy7<`a&czLlxLo(8e_ z_n)WJ@{kg5MBZ!?dwDdp0h#Tt;XDjSNb4;j5MA2$@x(txmoL*4B^emL%JsZEl4rc$ z1`pnFDex@)NlF`2y71q<0EugmtjA`cqr)~(*tjpEo^;%kh7gUR&eo5K_L)G4UNON7I@7Gws^v1q&QJKRqCOBP0aR!<0g4{C&xY%#G*21|9Pe z%SwmtxkLMQA6E;|o!ucP4?`;@F+TnzNO0b9c{Ed$RU-GI`xDQzMyU&2ljEg{@oMMT z=jI)Gxdl59)NjEgz_j_7+NF*b(evCI2$4KJ_ey2CK|ekG;BkLKZZ6ddGb>c0Hc z6M?t-%!*sXEFm%IInT0MTFS9OobHqL?Sf7G(klm{glNbUQ+zlg)!zGsrr1SDnL7$i zvd@_ALXbm>uOo5wO$F# zb}W0_u^M9ZHZ^5oMklx;>QfQZLNt(+OM>&CM74LY8Z&SQGS~x7iAnO*w`>^6EoL)B zgq$}?Ett1qh0O*7sZx$d4NQUk51iXPM+d)je3H3K`L>2P;QNL9CReddYXx&FK2LDy zeO_#A>{hKL?{|!i>~BhxK?brrROI-nL%MWr2akXlQw;`7Ig00gok}<^Z3dG)lnrP* z+O~U7Eqd291fDF7OE{YY+HyQW3;wc_vz<}iO@b;hS??=*$O^n0Hc@|(_dbGYApiqO z`t5F#ucd%gqYsvm|e%ppHA)2#^OHpme7zs z{1f;^ihn|7XOxb7cH{L!PNt9{BfN90v+&zZ%i*zmL+`~V@ZJhLZTm(T2xq%QgtP!Q znpAIWM)gylxopfYH^5O1#=vV8E=KIkw_j4*)&?3wJoXJ>c(4_YPdz7@9vmT^Hy-<> zS!yOd^1&jKRSJA`6H(y;{3kfG7N~-ugdYPo>3)P8x35sW8p$rNIkqiu<$1op`*3nX2qfhdJzX za7v=uEEHit3QBGO8Ws#iAHy&~v0hcXpYwH;gW)pK1oI&#i`=&34@XuO!7O=+F1+EP zszz!U)$9>f*o|7e?2}BcxJ*>G=k2)EvT3aI6BG>LQdzbMpXTNcsm;}6P;GqbeU~<( z^Rl(eqBiL1Vv5kcg8=E%+{UaH&qDkMB`BUntr?8C%<7ln5t~{Hujti= zZ%kkzQupmcVGki;B8~w}(*38QpQZ0})?IdXW$VMX9~Q(`I`Ll+RcaE_*+U}WmB6Ea zgSn558mz(AwQ^rXUg9jnw>WC|AwD3;@v{tcBNZZ@Qiv0OJkS2MK|Q!qcj;K?@gYVo zW?Y+$QF*&s1BHtTfR+=9(9#9Aj472o08m}BfS<+jT6^k`PTWDpcCa$6l)1bCb! z++5IQ^R#kAm$+3upBv4@ZsA1zpe-cUqLnPNYy2KU4SD4%RdUDcc+VU-;)I%t*X>x* z zooMnStGQYSso~Eav|!#}@LLMXtdhG928)4fcEKdg6P-AEdGe7>S7LElRDuY_uPqSIg?DGELk7iUg)3b=J)3Vg5tN}fF=?9B}cku za!*#l$b4X$&8f`|A-)S~gT#947zLf8KEQG}uU=<(?Ce|?~8!}ARgoYfs@~6>UPrc z7Nr@rRzgHLzJ&Jnqw?0f;rv;mrp=$%(^PXT8M7ns{>R#$$>ynEIySoLl0* zRKD3)Y`=Sn?3@f)9=vnue9Wbh^vEbBG^tDfbdydr8}7HjK}OtgdX7D!C2ak60}d)C zWFF2t8IzVLA%^LMBlR>o6fd8ACTJFe)mOlqgkJQr;yu&t>h|e3yF-ftZW}?MRbsUb zVpe3ggPbO<+PBiF_{k&PDhFj(R~3?FvL9w zuhUkqhm9o9oF@Ok_}~RxD3wqpCW-t*ECPlu$&W%$>C;!QvPVKrUpK|tR+F{eA`7@l zuklXSj7~0Fdzqt7wV_O5HAz(J(2V*$wP3`QfZgI?P?z@Q!E?`QlzEY}*q;vjJizS| zd}O_g6mD=Ct5xd?|3TqCqXIj}-3|zT+Va47ZF_i6EwY>F{9^I1 zR`ZRxPhTRV2qKs=$SUk6Q#gyJE71`wm3^W32Ko-Vyj*M6z})gMLEvn^>}r8cfIH14 zG;;i?Kf`u=I^fw?65bk2p+IyepC_(GN5K^@21U~5fNUUlSPki3Me=VVtFvEOdOwOi zM;khqZ~u*MQki!960pa;UVq>n$Je)d=H@Ly{(s^r(#a}mW!e{N= z#8AD*Y5eg`pkG7)NYx4C;iZ*Kg8C0+Tclw^1+fU0SWaDyAX?>Q{0^kKBRS+hsRgBT zu3T<7PpgI|7jo#lvE8njxUdN6Mi^5LJ)QY{hufn5y!d{u0^POl1sEntRL&ZnFq2LOAx3Y8sy za;W8+od39!@D+>Tt67CTrS;$8_|d%={9!TdxwtJapL~4-Lg^co!j^AEAa=!`BeKVt z!ebYPq>MwEpcXrsDzU>a_w|w4&E#LN^jeUB4wLtw0PC8O*H_MSArY$)hzX7p;3Et0 ztx5Q=Qd8wXcKwK`48`Z?3l>=Hz`CGZ4=NC~lgBTS{E@mbiu;cz(``{;l9NwPZp78c zq!GRrCDEa@)@@%Wz20-*gr*5d#Ph4_Bj2Ri!?`#jJ$!I?{~z_z-=7~6B3xclL&IYj zxm)Q%{J;zjpdM>h6f_E>ksfz-u}des@i-6@V$=|(@S}wq474PCdlDyeo3rESsg*!l zt1Oi|g)E>8v>q$aRu)k$(ixeAK0i&Lx2-Rw7V!v#)gGIUjZTkOm{2eb`Mo^DUT&`& zX`DO#K=jaOjTD6lBX!5qIia^mOrKKF3^jK+pU_Gyjfn17y*G(nov!Y??I)o$aBpEt z@$~9t<06%cmW`S^uI;a8(*2fWtDPUQyZE`B$pb!#MDAlew>mC-TtQNjVpr+&Sxm9_ z29n?wB&$uBS>>UaVFw)Lb$uN6Q%ixM7p$N?;aOXX?)pY9tpBEzYQ1f_oMQqYNwjRx zVQYcWN{0B=w|04X&0IWq(MjkXw%9o``*{EMPZsIOk8vVttq^Lfr~BbYJ?qU8ngjrh zxWJee;J=Y?pSp;PL|yU^y=kAd{w$ktjMh zT9Y~`W0Z|T+~W-~TPxj5>VhS?9$(qL9zvJ6wx^~A)QGRMVQN~(H+??*8`UC;zB5(M6^wl$Gr zEe7hBN?+g5dN-(z0e-L9WAf0XOnCat$5!j7k@zDi{i~CM^K|M zd6<~c@V&6$nG5Nu%^0hOrT8Vm)KR-q`jhDUgHGr7QD&9N-nd$}i7FN|NuO2?wVy8J z?FKOQNHY)YCfKWnJfM|KymT*it5+Z+Yd$1nO?L|TWN#1bg!=jR)vPF}1~3e2CzYR5 zTE!Hpj^P~Q5tcFA9;l8=J-CBCJ=EfUtXr=1aqrVMb#gMat=V#xdU@f0!U52z35z~%P!%s1`W zAi)a(*H6n~)hgd40kd!Dq@(OpDzlJInBZN=m+6Oh0Q-eMi*sD?8)6K=!bnb=D0HMZ znVFI#wqmAuabKbF*@MKF%e=p1(hfuM;``+)r_Ea}i7Bc{LLXc1oFbT0{`}K_jWC{% z;+CqVKk=iw=PLb>H0H!yDK3b%S?YEo4#pV0TFevMJ9HgYN30_ul>J@v&=zOx=y8eC zEak*h9oS0b6BIy7>n~bf_K0QH%9yL}!b8J3l4DP<#(PhoqQbo0khh4$aM1V+v`c(p zlMPJ4-4V>*unmWmY?%M5alp2rs8kbsf4=9bUZ^TKIIPz8q`AYc5s!LE6PKm}&Q~yW z>%wJQs#;;0dp>KAAefBr)orGCaruRUM06cT8p2u&KPb5#uL*$6UL#o28gvSDd}c{g z0}7T5*&T1rzH@u;hNDj9pnF$#`CnI_=zEid-{;x|n)Bi}zrj>KWC}5ZYF&sM3pqmE?=MDLc_DrL&X&SETrR6 zr^bCb8~c}BpmfD6Yu3Nq?Qi%R4b15h;kn1299~~Y`5n#v`bi^-sjU!NycElbNHoH% zE;zRbQWQO-k;R|jy4IGh=i+{x&D|&V`7EeuVANIxg>jfFq5 zs3Dn?oyf3L!Ns{%lG&<#@s8 z;SA?0l~$KT{fWZ89Q!jY{(0>SM$iV5X{d;f{F8xZg`M92JJF$1Bw-}x74@pJ-y?T6 zrY|_>PU`_OjwE?WSsY)(#=05`M@s<=K0Y-M>VCG#zJ0PY9rIm2{-qTNM23w)OmrMfOwawZHtkh zEPz#!j`9Igrm&H1&zLP=U?&&91aA@%6N3zM4g5?u9OSq*O20KsP=_HlG3ndAB{xtx zntqXfT> z+5fb^&Jg>^sqrORIPrH-qobcrn3O2RB#~~S;kQ2&AGT1RnyFBX=S0->jvQxGy!(sg zTeWAGqw2*=NJGtf^gi>SG8C7Ew; zFZX1`$IzVQRhw?dI{m6PyC-Tp-YFWKpMd#*l7p4je^lwdDxy2ia8$DGLkTLVhiX3k z3a-4I&m?#beMQ!B1`c zOylILU940QYPPPC8%BVmDj}o7ORS%-dPoxRE=Zb4kymr{PA$(jHg@9S%Yrw|Q85a%~vLw z#Si^4dFI8;&K}Fhw7K8UB+T7S`9QS)QgOqh8*ZNGFiotX{XpY-IPZ71VwVG~da(mE zmgI>uP$U$Y;+bi2zl#`?ay={M`Ui~->V4Kwz+;0S@}`5o{IvzFuICC!`XTm&~KKs zRJn+{#T?qvMV!f1eIB)Y@5m-)LUfr-2xWfP$6Ni>%jAmzo;A-*1QI5PEri=9Dz5w$ z@esD&8TuN4x#y>O^dpx{Znd zhpRfB3z%i&E|k7fJn6n^SRAhnC48#;s^`#!O}ox`&(@%7zm2U?LsB2+A^KKlmDi-B!o_xSi7L?TP0U zSsF}qyY)nyl=iCA>#5I4>-|2R&-3rK;HsULg!ef%+S z^Pn_sxPotxI(an?huK) zEV^h+lHSAHEdSC1pbDKt9aS`vm?qPc&T)LEF~l6&`E5_hb3EUBk5qWavwACz7V6|d z8)1femDWeZt$O7a%iho7TSVxd+o6}=JvI2P&jy#{#5@}+)dEqF%Pq!q4NkUmzpe-_ z^S4IOYCnYd`Fy=ftFWF)jVaLcSaqzlh;>sIhI%&RVVAuaAS{11@+_fdOO1(?V4_I{ ze}0SBoaL&}tM4rS$|tSdYSeu`)b_&l1O+R)I^Sn=bql)ks{NR1qr}eKd1ElA8|A8S zd7@sdJD~`ab=_ozcrAZ1dnpb?(AkaogV%_?d1oA);S>y4&R!}{RoILni+H+a4p7#c zEWT`)%h##b?9pAyHZMw?r=n&Z8hcVTb`(_j1e=aq3<_%$T3I!E`*>Pd7jNb$SwJAd zy`g^!E%?kG3XL?{vG?|I8Fq+b5RPBhmWK*Xw%M0HRw^z{^;a&p9hUd5N!;>qw4lBo zq!QRPELdc*UUSyji3Oi;^E2)AX+ilva!^3K&mTIsL4j|F-yETeN%erhhPt$*#H?0= z;i6=bk>BQ*c8(=`cR#esC)$I_>7Y;|@H5tYN*S@ecM6+Y0A^u>Q+J=4EK~HsfpAg% z;_Q_8) zPz&%8VV_xWyLJ`HSZHk!K3(CkN*1;}k9GYV2HR9-&1vVn+P9!l|O$m7#8RlN1+I11wQ z$F!}F0IP1N*El%%1zp|v1kdt#SG|-gwyvxN`&mE|G|C{|baW<2oAAV%wgM+QfvQ~o z8SFc8LfznFrH**UoGG4*q=GhQJ%%DWg~tfTCil5%vWM*u-c6G3`*Jl&e_X@_O${+* z2(i?^euz(kd71Xsa*GCQuk(djY;Yw4KkZ+nz8)i~!KAA?BIgmiXZz@RLQ;5mxbrYF z>_t1<1Se(%1&WpjR+XrjaJG+sx&Wg$YtH+CZMI7J$|t0nr0Vu-a2wnqp8~uC#Zn8z z6!FLblvxhglLxV0FVWvGuM*?-yt*ay{(l#N_*5(ls39WiX&z+u4eKzQ9ccY(PFb#iqA28f+O~blMDZ6yXWeY7=^uN^CO;?fA}K zWK5KS;Y#=&ul(N3!WAhh3RROC>=#1S9ydl0kUISlF^u1%pc1%A0LnR2G~l%8LL))h zfQn0YZpXiIGe=d~9r7&!-&z*6@bC`8I8@+d@)c2#jC4;bYonm2HD)cvKP1qm4rigR zmbyjJNpVVgh?%T!iL~o2zKEXgFrwYozE9`sH1+mIa_irc`fIzQ3Bi1E7Eas8NPbR) zzjr{W-sX_;uHPQlyD?`-!wlO3^!-f1M8Ah` zCi^yt0KKoc(9Nobei?k)gc<&G!n~$oysV9dQkT18UhM&uR{c=)AG){u&g6j7PDpeT z$|K9$FmN~*d*9)=SUOD!Va7iNp-@dv*DGLnGFNVQcc_9&Pd#;^73($i z)uaDMXLYf_jMpxc3^gzU^PTS|wgDfm`+154|0#oS{BJk%U^s zSvpkxq$Dbe{2L4~OIx&k~3q->&a5WNy#oxYh;dY30z|a^@65UzS5(! zy2|`Pi?iw=3=Ny@;oyHO>l{K^eJJ@R!7#m>1iz&q8T>`ki*Xri+atCM(V4AdpL3a z2vvB9ico`snX>K(v#P_)Uo9bv%we+NfBuPF?n{HI%)z&Kr=^knrhn^UlU{rdN}uRn zzHKesrRYN7mrp=%egCLV^?iAS7`}b0k{T?JdMtmQW@-~&BUo9-@1?`7J-8oQ7%*I- z+Mb=X`1m?jE@879our8-J}~qXAKfRo_uex4k~|0bpHBzMPlbv}m)^=eh=Gb93VuJr ziSgjD;hpIlci%8|n%zF}tgpH}@8Ub;`bq{ppnaEf26U0fnbjSyp2;FPU6JwZm?)S2 z-jdGeg$QCC;k6FiEC%K9sO#5c<=7qA(kg!rL-E|Hx%|~4%w~Xu_fOA-(ARV@#`!I! zi_kkH6sEazNfHUvnV}~{ZcB->hr1&0-KOu}-@-ee>Vb6eqMYlo&YfRw^^2Ryvqs&H zbSg{7qqNk^h}%D@?8~cl_DooQR&3bl-KU}idwTnuZ_>qqwA zy#UO`IhyazIf_K#i9J7a-aPw5E`v{jTx#Ai!z7UWvVY43|48MvDLJNeF#hYl?vVt% z3EKnxFTWR5HD1lwr=ZK2^F?YqEQf`@eRVq1x6p2)I4xq0pE>is8zy4cX}+OQ_uVa8 z-h_PeuMvT$%;UV_NErMliUq1Mr~@7>^q$jO3dv6d>)JN`NFJ)8(n(>;-t|s@EM_h{ z?Pykl@P|^S2Vo4t?a3;G+#!>9)YP!r5@wleb$hBM(Bkh7u2)*lI?aCWoTt6khVPi{ z`mp+j&sP!K!NoNm1;(pZ@7eT|K1Z>j{h<*=KqFqXFb~OsY2sSHHKMNS3yq4)y}*yo zt5TH0UvDlpREpM!ia`rt8iyykQPvXK!Rd>P^=dLlOyzNUhJ6&o>;u_Cd>fmmH$s@% zjI)&Of172wxO}m+oiap0ufk59J9}OLhn3P`#3otv-~I_1f{ib|dvZG90hChe_Yxey zT;p>|n)`vfV+2|oEsC6nGPM>j_R)Y>_{fE?fg{E6F1qZ=v(4(aZqFzn|@w3kGJY-02(!Bfph? z?|<{pvEavy;>NA#IOnScCmR{&hWiQ>M_bHCA}bxJpPmq`(;?I{^q`S3fp(ooUW&>> zO)}15Gq8?cF`Y$o-1qKkwrax5n=G-C*TPOE(ym;-S1+(A2wh>%fxfE_qLw*faJ}B5 ze^|$EI1;c0bP&(9k*`Dfi>ii(Oq(SP90UvB#wVpP1$J9A)19KY8QNnAvh5Q#wD zT)w-bd3Nr1SCXWt>7BeU3xT-7frf;n$Lh zHaFqME}Aa8pJ9DBd9$Y(eq5>}kW_kBQ*eP5zP8u~m##hjazjeUq?|OB3oDuDr0wRV~nr9#&*Tm`Wz5HP^Y0+RJxCH5_8{ zSfU9@!Y}1A4-lS@Ltk$gSFGy|~XT!*YBfBDt1&Pnw}Z+Y)HzE^o^ zHAy>4i?41Xz$ZEib+3nxn2mP6xOmd-t1@r#W;9cXQh@CM+8^qA8@LX+JtP$ou%Al5 zT}O_mK(?TZUu=@+1U7BH0U}A_9V+e$S^tW?#q7z6nhF-~6VY(*X5)i0iWW)v5INm1 zyX2G{#d9OrdgkV1r!=JO>yw!dBu6{1qtZAsbifArHlm6qTm;ADC*+CKSSdN|9mCPG z9(@}&5=k!q1ML+8q+{>=;uQ%P-fHlBMe-1|Y#Kil;zw?$i6KY7KK6NLB)%SHA$O5U z6xtf1^($9V&%{Kijy1~cpqZ~R-0X^sHyeI|xp27=8qhHLE!yj@SK zu4my#lH{XmO6{$H?W3KnWFYKbOY_yMtw|xa^#*1j0)DCSK1-R#Uy0rcF6Ninma{0Q zH>$0&6Q6*h1vGD5@NT9U5j~b$cT*8W!93NP>!O|o-8nitG%D%hdpu|N#Q1+S)~780 zu{)fI-EmNV1?*k@&2Jo-yTq3q1<+IC;kUL3@;qfGL=G`G}>N5H?H} z?DH{mas8UPw-6KuR%Wu-yGW|;B_vF{JNUnLbp44z5g1d+3lvn`1LEzd zF4nX2+rGx_-1&O7N;*0^hDavkcUsl!J)GNh6&)|xfye!dGSzR38LL<$hIS;Zv)r*K z<%wES(vB9uXh1PyktroaRfJwX@!`kM=2TT+O|SiM^v2f0&+M$GW0G!TzUW~YjTpjK z^0}P)B=8pfk42>y0Dx9Tr7HyRv*&fbMn!PU?|)g(iZ(-f65JNyq3|GcLZ@Z$gDmwt z8orPBT2WpJ*>E&1B5BchbPMcIrlOGgU6hdoKrrg%P5v6Q>zim$jBv%U8&R1tpKd4U*mn{s=G9OQBfjyE(OJn}VN@F8veilYb2N5Q2tL1MD9bd<-uC<66B-6HrRP{fajLf`VR`=aWoO$J_ zy9En1-l_)8AA#fP%n{Yax@;!@Rek0slr04@Doc2baCSe5PhU1;#8nQKSU(f@$gP?= zo=1~?4-=d?dM5g>)qwAUgc9Rl31ub`l~MWYXcF;a1APDHWO821Nv4+FRzTNbNEdMo#`oe51 z%k2eQyZ~WaCi~)dj3DAzd+_hIh+86XeOrPqfp`=0?^yM4H|m8kQCN8R(ZUdyGjuOVMI;57W)pm>Z)C@zTzRSi3PRC~fxAJcj;aAo_75a_y-NZvTELQ!$5L?^;fpo75~Jqx^yM@2Il5i;Ylv} zVXeL$Mqq%?;`XcFAn{Iw-~M>(qfrfZ3#)v)LR`$!`C!9oeETX+7Z$25HtYDX&{V1k zhO@8kB zk50Dhs<&Ji0;qbLqNV-ffhQr13|7EOk{wERAu6{S)BsATzF5N%r7bLj$OAk{v}@gi zVv4R*!^gS@00wQrL(Q-&NpF9=L@Zi&@~r9)2!Z4`48XYDJK=IVU*8+Qchr84anFHh z>l6)}=JI5StKQIQZMYemS{t*uVO6rp40&vIm0^6Eou#qtmtWM3oFvt49ls_JsPI%Z zM|Gmm*+{J^Y6?7Dp7Q0*J{FwOvhu%KCjmm06EgPQ{K+cyJbumrWs2BWvU?Tw3CL(M zs5d;67#X=%@W>Bj1}UV_{`4^4S-{OFCGoo;`dqbWGU^nLA2zZW0(0FP-^G<5i;1!D3-m{gRd$HRuR1GFX~$4NF=tIqV{i`l z;>GWH2Gf0~Ui6ZFq!gmqSQKMmFjA76r|W_f)KKYJjSwgiCc2`!dWn?GkK=NYOIW7O zX4(4cU=jFdZ`FCGzE;2V2G$eJ7{j|f_J?|*e}HP5xqHa7=mduD^LM;=dtZ28^i=9r zdu0IE4O;?bMmcQ1tu1Rly9s>^N+;j*tBNEgkqdRBqb^XrX+$UZoaB9!g#7NQ8uE(y z?5!;^1_PbRNeYJPo%k4YajEM9`QUMyf&z-Hdvl=igs~(#e!mPVezu37V6*XVt2oVO zYkt1vM?1>XUH@D=6uv(f7fFgdi$-AWx0zB)Jqr9GA-y;YkQBF{rj73UV57l8yf}(P zU?5Jt7XE?ba}gFCKWQv(v3t)Rn3##stoLPPz@84y5^3H?W0XlYmNZOYN2@_s^q)qV z*kDDgVI?gqwcS9=I7iM&^yWg_kZ-n>w?G9snlCllmKqTSqs=pMiaWzh7oyA@w@`V7 zweto2@|(5@(N=FS5!>#WVp#kU2$4{hZ=SM-+E3wBZcel(9i0zkDO2N+-Itc7_AUb> zh0`rZPOpyk!st~-P-J9e;=~{gOi$y|vl2ZN0FlB`oc)f>>PGNXVbR8ldX1<*W;9dY zF7nFicb%P_bTAEiEn3TPuVJgPzTbK4Tl>Z@$G)QmOK^Xj?_COU7TV{egu8dGelg3p zD(Jaar(Y!4@!&}R+vNXK4rHaBi>HxXq8rOrP2;uJUb}MDdo%Nz4FpRKW578`F6vdK z6++#DiAq3tAO!Za?OWUC^zMDN_t!eKIhox!&bC1s;YHYzrC-4*;I?x8~}iKI4Q zo;%3`>?|Px7{sEl029UnYexE8B4Zz=iAaG+b>{W0LE^7m>p7pwgTwx)`u**S{TQ$x zQcGicPga0=yJ5eggnn|~1{t~o7A2To94F+QJ5);tsu6aPUy>FNO0!mb;#Y(4B#i)% z$t@C(g*W$tUB$-Dle`y--V)+7-!b+n+}CQZxLKp|tY|Q|Wu;M&2-Y5g1K`}1bt`R~DLI=>k9NRU4UU^8S@ z>+nUs(Nat81f3D7z;eU#tH0P>OUKyJDABAuPAiDEcai zG?K5TpgN^5a{yf*3r(s5Kz1_5$6FI6LF%#Z&1pjMyE@?ew+`#qucB)HVw zUj<9M0{~wFP6(dQQJ#^E7PBiYehAWmBctzK_#qSpv!lqF!UPr{*&bF><-~6QDI@dq1Y#mB_}}pD2P{9!(hMy z)R?@A0RzK*0Wi!~FC~q{;|Obe^zN+NE-%Ue&^lSRI|trcF?WBc=nb9^BDG|HR$K%S z=O4QNzXsV9g3^mYfanb{#X%1cUCQ{0`JzIc27ednCHvK=Jf=V99uD?_qfR5R{6JTc z)7@6Qu1kj`?_*WKY-3hSJ+2kYX|T|`YJs|N+t??9sQ9w|gl!sa{_UVxqY=5}DIH6> zD)WEQ`nh6I?hC;P$)ufgMJBFUplAjS5p}4eWFeP~4gOJwPMH`4fJ!&BPxPYu;{0^+ zy_qa#5a-m*CGZh11kV}+l@nE;5NP@7D}8lx$yXY>`yv^%feV3S(tdwN$G>tYUp$Z` zoEk^IqC@eyW1Pa)@PR%QKgozLslvdW-G29MHOceJ=X^a2I7Oc?tR9DJDMU&SCBWtQ zAyrwb3~)Q1spRITAaP0H=ZM&f!2^xScoDy$=UubwP&vg>G}K2)gZok40_&OY+geBfSxn z?mma`6IL9Wuhgo8Rf7ijFWU`-(EgDX{;f{eFD;75qYseN&oHaln@-kq2i*{)YR#)hll-A^}BxFR1V(>qI3K`3BZ7XGr^VlHC=KrwEQLLZZ78Dc|+A3}x zu64D88eLcdGM9aFnV*>IDZpSdzQu#GJ9(@qOx^-%{z{{U_wN6zs6b^0^$}0QAKqNCv^NaX#*^Q|7M$HOaM={qJQ0?Q~CY0r}!!RK7hjbqy_zhkr?c7 z$QI3cYy4Zc5(cq&M<>>HiQ$=tI|_2+P0)Mb{9RXG$Y4-KUPKX^XaF|Ul+KfLkR#N1KDDD*V(H{yrH73q z56qxmM#t5yw56ivRMtY6od@U2>RQ3E(<-FFt?9zi$JT^~hcQ_3+m&fcgh6r9aYo}b zF@b0?zCWxx=s#ZNUuF*ul|w+;>fx>%uLBAo4<34RFEJZ1`EUUzr=-cN&iumJ-1eQwcN1L`7dig5?5+F)Ft%e{6ii48z%Y&(~1 zQ0uZsyYKW1_@QX}7e0kwwPalHafkM6Mg`$Tu)_rQ^JDr4u!%*B6Y#74%RVEc5;6dD zl8C%bl+@(Dnvey7Z5kPZ3KiZMnfcdwssL`27F3b_4*JO|hqMy`y?d+il(H<)YLqo%d zPQ~`~Em#~I!#Qbyb}a|YB$&}!y8z6}T2z3!G>kT=jSFb9pw!B0r@MxeE9F5cRW&{q z)nTmDcTQ)od+Sdl4CT5Xs$%^YrB?X>wi>mbqGGwqn{b?XlGSq}W(>@MXFvH?ZZ+1~ zEBSr}ko?D2YCr&)XXc^YYUB}b)6aMxe-$z9i*fzd zuffubWfx(J*o$#R77Z$C?OKllO<0*io+f<%NbUPc3b3ZihL)I0;5D9f(A?%PhUk@B zYB1W?YPMYJ^9Ub2I5>E_Je#@H*j2Bg`}6EfMcyT1(0340O4m8duudhfrqI~%OKR);LKO7m253q0t(n*x_wSlyH<$Vay9YgsGXk#P#Ez;yh_)*bDe~Q$XK^s3Iw zdXbic2=n0~)IJZr@|!KP5>}!|z+ow{zA*wv(q951Moz(Grf(x z_JS%YB;Jwp4Hp@SO#`iSSd#t!L+{|j~^eFnR5K-a9=T*~d+w!3G$2-I7VPPcIMp7G30>kSTlclU~vJ zV}dq54kGsAWC{0q6l}8FBqY%C=HpqLT3nlv3R7Y4M;5DffHA*ncL&|&yFWI`!9k*| z-`SV5iG=RHgTH$L+7C(-#$JoB$8LKWBL{la*6k7VyXg&vWXuO}aCR$}{n0b=AH3MX z0A$S*aSTlHFhuOKShbmdTVTJ zy|+nXkdw498&!|aPb_AljF4&PF%44ES5p-QS|KBq&wpS|`1=6H_*TClz-_M)fR3{t zXWV}4aAQOfFi;-)zyWpP2+$Hz=XCQ#_qKzawUXlRU0jV@urt6TJhELmyW@S;g3aY7 zdtch{gG5mKUC$>Z22QHo+a(x?Ad}$OD|jcyp_3)cZP7BE_YN8Xz;^RG20#sJE_P~L z&2+|FJ8`KPzgaO78@sKggtRncQ3R+di;UFvv@jiexsR3hiEc*MI;}q|2ehBRJgw(P z%DdXJ4U|-3NTkQBOwvY+47jGEX)eoM0#0BX8hKs8xSEB7D^KIob`-aT%10Qo-o&l& zY7Bj4vK+KtZ7i57-)q6x7AjMVKW?mpi0Dr0CNE9awqMq*(wh6sxDVqUI5yGSJIe-| z&(!L%k>rmMjVYfxO7Au?CG1y3>awjD3>N*2B9<1}bD;k(UdmVe86vuMkdq-5EB9Zm zD?|2)`vLWGE;d&kcO8B`VE^CcL{FiS%3lEK8OZ4J) zxZ|_-VUX8#Dd(__tPU-Ep(+(&MR?dY$o;lps&N zj#*&ytod2#>EEX~nq6*SZs!)i_(@o9J1w+QF~Nya;p~-t1^5{ETtbQ$d8{ij^>!;a zK`G(`8X+r@bg==Jq+Z_?|6zq<(7O>pU0uJ6m{L<;(o>V1+;ky&Kh#)EZ+xlYV&|X* z9ngB4JnS}E?ui#re)-(=l^+gUJ10des4zJL68V?0FGS?h8PK zNZZ}F?*cXUi6N;dWgC$|ohmo$J#pk+y{L}UJ1>G+*OqO~pZCUb{_Kq|MrtvS{z6g> z4#NKb5K40fklS`mau;8N+fHe|Q{6p>iNLWKLNdhCpb6#kuPVYR& z_d9d%lWM>0zNGY!iV_hk(kXwripwEZFuCmZxkR{f+?Ml>!Bahf;f;(LD75ez6|K z)**isWbA6K=YP2cBWb}DLW|!^e zle#Vow1q3|#wwC3EkNV9@YTChpFQP+a#PO_#C^Jg1^&enN7`_T@vaA(3=^?7j15`P zF+;DDnjHn^2RZFQ{M*j)NFyA{E~fvIt4 zJZD8oq3M8>_Rh+Cwo$nn@L|bYoxHsG^-BlqNSWPf)UM2T(f99ve3paqxYxCcy`E9% z>Zv)!&6kXUdk;3@smEXrOV7_NV`C7~c#viYKV28v*n8eNb_-`df#Rb1l?uzTYIt0# zLi7bVj5=Q+;lLi3W>Bd3qGbh(caUZ7*w@r;3wlopt%OlBal@~8X8~4OS&~>ADovTr zZ^r~jgM+=JR1kbFg5!O+?Jr?bXj(T?I5gRx^_|I=d;?+|=U@-L*-ug=m8zV-fsC1loU{y?vmZTJce@!HyO5SELL zJrW@qc0+BlMHpU@8ZUNz=s4(I$lRz)!xaH9rEG0B6coRE{N}m{m&mXtky4r^!ZPw+ ziH`ORcgG#Y0&*S;c<&)^YT3%TevR4UQQKj)lZt9iwaYBCKZV(Q(=)q2W-73#BpHw& zpIfUrTCwD@o~%!cEz~}9!+daj^fLCDEdYW6;F@*Q6}YYoV=i0scV-b4X9z*y>9J6` z*}l1&XK_p$d)y@KBM)iXEC314uA@HO*?`T#P8XA^b^ZpxP_UPvq5 z?$>M~zokr=hC$TYdSP;<=_Pv&l9TCB+cKX5103Z(c{F5idQsSQPp8H2hOb_u;De7e zLoN0WrF)YvThIw_SPUhV+gb3Q=w{mjUOpk*A!)*PJbAN!bFHOAUHudW&9M%g%W!o) zHqbR?d6UPi?;eN{s&e>~Px`Y}=#dqXEh2ZfHTt$b#UNq}U&PlIn>6xt?Gj3)RWfXD z!rK6k&0m>r+5|@NREFs6pY$t(Z>j~e?H((!WIP!W$7VGJac179F%tA(&+#TlYoAkp zf}d#4;{9Z40ee-zGX=DR=zNitdU$)QM+uRKemH%CO3g;kW7E8HU=cO+!eXyn^E`u3 zx%}G9*qzGiGG&q&kBGnf#x%I*E3Or3^!{J-m z;JxD_fJMRs_RCzEL0&zU@Cd#DN=90%Rc`#ma6|A=tLw$jooF>ekK^(Yz3Dwql35<) ziWWEZq}`!i0ns0P?EEDO-*ar0TitT}o2^_9W_7@US4tjh1CL0Hy*&R6ob~9^F*pX! ztY2^wF>ZgimVJt&(=qsgvL3jHDSwn&T!6s)OFk_ri@DzR^C81Vv4G3G?Pfi5aUMEP ziV{XCmV+;7fI~P9lcUwNw8-JG+8{s2rK}Qr<|(R^y%#C+k&_g598afZ*hQ^T2*fU{ zZnYMxM&Vj@2?aAafjV9Jtv*01;SYxGCZ3Znt~XMRWtrQ=fg`SWzh%C_Ib8EqUbrK_ zeEQpta2nE)O9b&X0OSTYw|u;VjHnsz?>qU>|2X~tDZmb!C^#)as$!JTiRBY*u;3(x zHG$LD3x_cR4m%fyS4_MCT-14JL*s>&sg`qJQ87qu734lS^l`5F@|Vr?nlDtK@)Avg zD1xaP>W+)kbIhN`-Zk-erUlyLcqUBkWrpLk@>4CDIfUxyIs+#t$y9@cAt-FLNqXf`!obGl|-(1$qIRdQN{o7awYI*Ed2M zt0R7&k1yNJ=)1!=Wl+xS1)RVWE4x*r@cV7yKch7nrEeyb$*MO@KRMSChgk7hYq)Hg z25!{Qk=<$DtE?2P0?q;6v2)hrFsVk{?;OnoCy=0of*|H>&N4_>T5;$GCrh3aD+;V| zBf^jDC0BjdKuY|jIHougEDbvIA3f0`wCy7)9mh{0cDpDwl?h{3xbx!EW$H4w{i0T!q>=`c*W;MgpV{2ok}m z+o&jj_ers@X+#kMK>IoO2p$4RvwYkWr*e+f*Sbf6_xY`hsGPZ7_|TmFVpw!^Cb-kh zW%o0ggyU-2TqP~_^1*DhWiL41`GIWM0XdHL`g%UC_{YH&K!J8Rk(#xiZGAlx z-2$ffzJ$qUt^!e=ITa+phO7WLQf30&CZG1adEQsH!z2%rxl@NXyTs51j^n}o0uu+z z;d?uqs7x^>D}nmcY+G~{EhdZT!vWNtJb223Y7Y4Yda#&TWwA$FGFP*hky)v<`NAH}*R|=(L7S!f z)X|p75Z|um3_+Y7mApV$jvlJkZj}z8L z!Ko!)65!NLi)ghc(+F`7E_|HSLXhaWo_L=<&u!94jhI8-VTU@YKZ>}_>od5$qZjNb ztc?LY(>CxjFB`HzZg}bi*=Ml5k;5zoZk7W&R&3yYY}jt({swGZ-Pk$l4BJaulAfc> zTOtO17C&Tg(sLAG`l9PXr*T_U^2rgxV<^wyg+tOaiv=o;rl5koG|CbTlf&Ms6@BHB zC)x;=y<)u+fY+&@z{?f@$Ed>2RyNy{P2xa z07j3XpqQP@@X(Bm%Y?rp``NawJm~2PX-K3%h^wJeQl%}m*)MHveIO%X-jqp36j^*8 zF0s^Im}{*IY8lVqM{PI!zjy^%PiGMm5!kX5HbLz%l_9w|3tAGxFIgD4w+C3KMEp>~ zpsYIs1msD#tFafWfVZ!t_zeZ>lQ7@ECl+`QCaR7i%7qq$>~nJ)J*^#V!HPzOf%`YP z8X|Rmwl9>9nl_Wuq9pa{Z(DgGU{`*wu?LGl&bh6V%evsYK)xHd%dp z?J3l2Q4fj|z#RS(-S^_8<^i&~JEfR~^~f8j>z+>}SfktpflmKkdij$F@6rOeooBVz zUsMX!MPT{%Pbz=C?Ej991Vbn>-&Bamut@tha^q!oXZ-;Ek%-UMPgIYcZfw{nx_Df%-Tng~)?wboHd7^yVN%f5TkK#BK!M*UR)daQPoQN(c$LR)mc`~@O)Opla$Xm|> z>0C?Krn@h~NKnatGR?7D$|npwY!J4E_(v+Rm3VxSUc>Ka#*ir*Ki>CH+RzY#p7%p| zqJhi1C0CJ^w>~dc-5<2PVX>*s;wFH}pEv1CdChh-e3gHDStSN>P{9@3cfgu??=I22 zn$?$id~NgVn*}cu#iI$V|3srxtcgi^VGlI2rZp;uCW zYMXiya~=yl&f;!Pau6KxVQcY9l{F~+_Ux$(FAj{}8hxozU|Qx``>~7{m+a_W3x%P( zy_2oUWl)r-@lgp?Z>#9^BmRNE18i{w&~f+{TY#@MLf>Nla&nOYT2$?&-S%7IP2>sG4@UQ!}+(q{mNg2*gD%fHas zq!wf=!_RCF>lMi$D3S1>yeVv1i1x+o0loCm?qN)Zp=u*8ED2S2jm`XA3XAylawq0k zRnoTReOWP*S$rn)Sz~JOj$ld)>EB>7Fx%+3OWHqF9iUDyU37p_4aNZzKa{u|cRz5a zYeL(G;}$g|h@-+j-A*7xi))RHH(YYh9g}bCU=@2!e`M(GO`_v}ksG<|o))UG6UtVcnj*IGLwxuUjE&2#_s zK)?Sn?SWpf7`n>`+q@-_VlMGI{+o+^>koja$yLyaP}gc)2_PoA0*)rS?Y2Kvp;2S_ z`p7=ivzDuy)tg|EZIdep6dmo!QilVP1maly%oxJ1P z7`W`LlR!$Z?hKz9s?(F{Nx6z&>nTj2Qa^( zuaWWCo0uecn;sqS5vMk7w(gIB^+}Z;6pGmO_x@~Or<6`Po!XNJIlErheiq%+;p-jT z9UGtmePvEcXlL01GlGZ232LRiX7p4nU}o?yrocfdrrEs&E|%w;i}h+#a1IA^QMY3j%2Apomv7OQh&$M$2}#o>&6Q4lL6+nwa?+x@5{$X-V%o5tsX$@ANol^` z^yTzDuzTFE7Wla69&4rsL&e5U-FOi%29vu~Ci)5c%Xzu_NEW_m&ndZ4^iD9|p^enh_GxTWy@xe|ZsG&qG~ z5}QV%zrsdtEvzT`qion>eb+V0Hfk7hL_DVibeS0{=PW#aho+e{ez{h!za`Qc{gxpL zL11n;0uo3B5(uyo{HX+@ML&6BgXktRJWDJAT2!Bz3{D)II0 z7pNr76lts6eP1^7Dl6V?o71rIt+2@t9XN`YjvkWj3M^FW& z{2dVy1OnNZyeg%SL^;KDxvxK2>HiL)L`gvNahkqW#xOWN0*xKxKVK0d1%~c6x$ZVZ z0xDuD_@aIW&isvty`l#>>n=?nc<`iy89)nV5^J|i)quop$4!0~Is83w zx7;P2c>3mix*G-h-PTfw2egXJb(c7rd#XIIU zE%;pH{{a=>4PlC^Ukp6cld65icf1Fd_f=>1Mc_ejlEn6ZKX8^9&?GAF?=K6SgS@%m&(`z5*;fi0>T27p zo0IMnN;D;PrR4nm_8)%>*#QFYcK3}zP>Z3@z3@Hr-<;|KhUW|S7YIkebvLtL{ikhU zC83T_x;>cA4H}BkazJqUH^D#r_RHb^a=3pR^N+v%a=5=7?vFL&Z}0NU;r?>Ce`~`J zzx|4Ee?_>^Zsb>l`~TAjcM%-Vj(BL_H&gX-tjIVl#0AY;>!WwHsL?c0nksRj|Eue_ z`tBOPr`a*8A~(%)%qZgBes@1p=&?(Ns;1`fr8&L(?f= z7<_J9tG@n;-u*#JBv89YpH@3Lz^}7ULsN}S2jm!Ucw?Ty6AQPT&*D5JyY&Q5iJ|Y( zm;vHH@Q%Ae^SW0V2;1lFZPlyb6(rFSP!>Z#9ijvVbzM!4EGLpMNwJ8AoFnWL4i*iu z`{yz$Wr8ca%-!Jl4JZXuu=kA zSzUp$PTR1z^}ePS=du|7069lu9Uz)D=Jum!o>*nNp=bWXkCXxbF>^omLooPs(^9<( zt*qAo-I2KhwgF&YWC`-8zroM|Lx@cxarynJ#l6@RhassZxTpjiLRFInkk1k>UmP^A zd5D3qucX{n1+TdGc^34lLy2-)1q>SJVw#eTP8el6Yy>&S_%$%a=epLbn8XrINB-P0 z(4zSQ94k}F#v?c2M;51mJRwQp1ZvmhJ(^$ueyxQG`BN-n86Zc|7W2+K=!9gp)QvzKpcEW_3q=Y{9;hAXLx=f`z|#-FpniG^4TA{Cv6J(%EwPyQ8SQt)kW^D&j|C3F zCA0_--+$Rk7@F6Q8Gx{F$tP!kSETv3$w4b?Qz{r#c(+?#`z#H2PjMUM97QL9Xs~N0 zNAMYZLk>wyL!c-58hGn_PiB6Fet+81uh8#%SNJRR`^%>odx-yaN5JoY`6~?kSKRR{ z4E)oV{LY$RVc@@11pQYS_?J)l{mZW~@VhwWR~Y!6h7kM;1HTKZe<6V1hibnN!0$Qo z3jzFBBL52k{N9iJLID2>K>R`gzta%WUkKnoHRRbZ1n~Rt>=y#~Jx6|R{(hI!{M!8e z-mg9r{k8e~9a{8j^Y=Tr2=;69_q(j-*XHl{d#b-Sf4}3%{|B4Dy*=o&5)%8n$HsZ- zbiP)u5`sI^>_*?w@qJ9tln3@QgCbNy`l~My?n>5U#$E|oGZ*uI`rCvXp|!XKQd~?h zKc0pp<$i7Q25A_b23wuQZ-2LXr^nSQs~f=xWD5mxJ%epq2(Z z?l(20YBilwlq4OC^W4b9YuTT#f9Es*`4|J(n-*WUkDBwtn^7TjB@>V3P3MK?kDwRW z&d%frMSHQ4jwOuykcM}P1^S6zRJDdw$d`L8%FU@XfCK(Fh5n;-#1#~;Oq)@)bt+tq z^B?aah?=C)%gvck){-eqv2|RISVCed- zz@vNUaXy{b&3YLADYbtnJ@}_sbx`DK^*K@w#qv!^9x(}eO|CyDD|~{7*>zu*4JV5qwV32JRjPAyXS2v z9S09h3LRmVsf3)951l?=;6;DmBU7@&Fgkl_I)QLHH*>l`>2O~QQ!-83ur}G2Ca-2I zb9tkV%9Fy)W3!moj9WYU)ori3x&_g6yZNBvnSm$Sql`?@)jiKdlYvdWzRfdXQ!-|? zza_hHY$He8j3X5mI+tqtRFTMf#F1Dwo#z@Ahrl(3NRliSBdMk7OI`e?&V}+(7g)2T z1CHNlq)Mc7Gu~#+3Z+yAp(4bc$5)%s|ruAO3VQ`Jxa${7C$zuX<<;!MTUuF;Wc$;x^ zLaWEMd*H}P@v=>~!H=Q?(}T+8)P7r}G?P~e*FVLrmL}WXmXCComyfjHnnz5hO9ve* zMsae5PvEVNkzno!@gG!pAZV44t4@4AFLr_lIvi*xHBITp0ek0tk#Z}@@PD!BdiOp4>XM$2}$azDYTgx4ZI#f30-m%;s9!XDlOCSj&Qf^Vn{+O z`DMG+r07~2DDU7=$VXT4rF6I}%E!2k%&RdHPoyGM@el0m*O$fijthe^qi*`aj$OSrQTxhD4ibK zK0g+79(uZ+r9e#sD=Pf?spfU9)fr{SLQ?XO>7*Lh6apiqV~Dtdou9P6bx%-V5Z7ApWpR8_$EwQ4=x&a zA(~mCE*+wRIo80)O~sdy(y{Lff;Riwv!xM%+QX~<6vO=z5(oEehG`~kWTTKr-R-X( z<}wXN|LMZNe=oHNm=!qX>7fH>X~rQ*vrOgHK*u(f&nfL%#09~o!CxB-CDRSRJdEp- z$B^>bZ47#WY0y}AI?r+-g(^5!LM!~uB$q{tOZAXEC3IQiGtsLkGiR7VMpoRna|ljq zCJ9ZM6X{!gx1ZX$4E`%rZ=9(-Y%sAzw1Zdl3ryu@IUGCG$v#zKlohca+K7#9cq<+O zy%40;0JnY=xMcrnJR`fsfDV2sl!ck9o8MHURlAgBmW7X5gI+0TxU5afasFehqj?Ly z{^4vNhpiPNoOof8;dA5gn_Zp0)*D9uQGW-Pf`n6uTKvOuUA5cYV=he3t~)J`@I9Tl z$>GrT6_-z8s<_gv$6|ecR&(lO^@JyqCr3U?rWk;8|| zSd#C>{6B#c_@^iZ6#X}-m`*ntCDU_KR9LXvjKCeR@u_A{IcfLD2-jDiCM5NeNsG*l zbm@xhlQ>SpG~XkmgU$){O-pxQU7#u{*`;i<9N?$@*LKPOE|H=ZfH9A!9&9q5K4NB* zqd8`w(Z0(j>@cBjy=9|Ao)w-tfMmfiW4~^zrzE)X(E$jTN>E+!Ylsx`p-bZz_TFP| z5@;h2LDexLm-P$71+pu)>SdGr@kO?I&h;3v2RX^L2b&fIwW+5LrAz1`+F$4o`$X#_ zf&vqRBWkiI#+y1_+EJe5btU2wG=sRKfR+Z`xG^|zw39@9La#98d$(npZ+0)DVdQzs z)82&p86<<%PIiK>Cm!`pR{O)3x{pGyAs#O+?>{Qqu9W_#Fuw~uVU?&_Z9Oh8w;Wxf zWG@dA(~XR$0tItLpE>A};)RN_pb(Lj2UL~sB}+Xk)qxp~FjpEHqayhzvP~sNr&i$R zPtW@iUF6TDFU~&ESazMxP7OlZYD#wuUt4sYgZ6dtI$-GSFt<15UYYmbQPeGrNfFJx zqSCaQWY%AOU%)kAAF7UHK4z;HJ%E>g=^!ABkpxGJ@mMU8YUX|GS?C0d3Ik~(m;1+kiUssS({JMG*~*w5!xQB zcFi^XGGnY`+eax6?Sf5+fd%nfF19b-XR%-mbiG2O-VZ`G7XD<~ln$YU_Ee^^CN$J4 z9KE_4oZvNs6f1XjOtSsD5b;5)r-^GJ1}a2`&7zttrm`^)Ds*Bcp4v&YM)TX;Vc{?0 z0?}bf5IvJNG3<1(Ww3(Tq5fF5!c1s!a%gpNRpRcFWMl_9B;mv;uccT)v5e3lRPrX! zpY2L*$V)xGODCJIn98Ur0pq6dd(nTax$OLCC44Fw+!j2qtzNI*2H80wY7HFLVr~np zn#58zY^!Ic+lG|wB?i6rgBL2V%rC%Lm!1TyM5_e{MW-}t4hyKcr>Gn8(nC7$K+t5Q zJQb6*uvE^e5ElHVg|cc-lEZS{iAPwh%!qifczt-isUccgTB&Bgw}p!2KIkd<$psFiEMIf%p`#(?{6C@I;xY~EpKLinC3z2 z!$Hm`AR6m--KLt+7R-kEoz?E^wnal~Fae({Fg>PozZ=I!(MF~;yZfD1O;;Tf)k^XB z8)(Bf+yyAVI&PZead+bk2{_m2M1RSE^gE*Coxo4vbuIsH&B-T(gqyQ$S~m_}pL550 z#F}e38hGb8!^d~jocR)2>+hIJgb$%wpe$@IICK%}FE|d--D}H>G=!&fyfA5@`M4^VZ}T9CNdf zKyLI#zSM8NloL$H_~*dDhH)KSId)ct?g`JgSWpN>^zTaaK`%&awt%bHmY?-Jc3;p= zj%J0YG=_2MQiTV<)trc?q`hv9hDkX_dHKnS{y7TrqwC(mg?3w&#*>N;|IyM%1NwGK zF;2T_*Sw)?p8FEa_5|U-V55{a&KgY6B&FA}bErQhikP1;jXHKlEfPjOekzcZnn(dc zNgwbC`Mj3|IVnA-ss77m^AMwndX2;);n%8MY-mV5>{O&O|QUs zZOCwi3?#PTUZf|vIfpege&Ak95!wsF4o+U#>cedex%;uI>x2CQG~J~|Sp{=*QC%Nf zA(d1fw3mD;c#FT|;K4)#cdifExBjGF!@oXkO#`bVE=6FT5Q9ZhBYR^3EMz4VMmKZ* zBUUyB;f~-qo;k@G#L(KoKG?X9v}kKs$@SW7Uv84Pd0GdKDZ z-M2SoR(*ZdaTHx8TeWTN^NOEDr4S8PFBA{vA7{=)I!FyU zOKHxL@^X_rdk86Q!64S|UvpXA;THqo}oWxr^VaJX$<{0s{aXe%qNR zS3vgpQfA7(xII?s22%OV|Ay25T4|{xfNdJ|KW(&&h#k~;d?`RC1+3d#MpQkqEy-4!e?J^;4KHUJQLpo%?#%@&3uc?_UxN@{NZ9?NP_8aPCPSR4n>lh27c) zi&#;wXJ|>kvAA|aIzg+TFfldL>NGf@3}Q4`D{x4$TD77l{w-lLBD#+w$?0P@Q%qt6 zX*Xv~o{46mCLUy0;(kNQ2Va7$`SOs!efNZZ<21;qS=0ugloItQg;ihMX3uc8 zfHIs2p53hLZV4F0mO2!g{i7ENn|*9|n;2I{t1z*Jawj0Q=$piSFD@gFtt2X4hL%6) zO>Ye-&ZCh_QRBdH9dsxF3c^Bl^etd})aAtmsXh!TqCJ>|{Qzf)z1B=Iv(0vMD*x?` zr9u5CL-)`du$D*nmCZ}1{45P+y0-$7W2zj`{-Yn86)c4Z6)ru#1FhPQ_gVNBNX;H* zoPqfe4Cb$n8GyV=g2=k9J192W$3!C>(xRtz;?upbPzjkL&5F|PEbEPqs;eN)?l}6y zEfaqq?H1-vebvc!q>Qu=O<+=DmBGw*QAQTD#}s-K)sxZ#43S>sB#Y;? zs;_9`*^$E|rN-JL0Ry_g0g;@7Ik~|88h1!pG)0K^oFP1OSn=eg%o|p|g->rQtkkQE z-(N;M&yef3xd=Ge;47n6Vz(751d|rhrcIM$W4F@HptFZrav>|A%fjH z|G;6%7~luew6_;7qmT(Z6HAvUsNqhbov?Pk6Z57Q!F@Q{BtFEy2==67=(bJvClBK;lz9X zB&n>31;(Za4c{C4Ka4`p6C;Yv_qyYezoleUY|VJRTK$NmvC{RDDXeKVy~C7Y{g#6q zDY|9_3(_1{S$BcHOAxky3eD+)Nd`5*c2?EuWkv=8tRuLKp*HQr?o1ov_fcr%G#erjf z0QyL#5Ypb}yQKiBWd5w3++yB5Zp@0{+GPu;n2w|U5^0dIeujToz`Az@1xbu-#>J7l zamj|OMA&CnR`_%3K&IL8ei%{Le6J6V9ZmQ;?xJ9_PjfmW^~}8dFk@88w~bnOTkzeI zvJ&TnD*wUuf2|x=3}|h{5{`T{v#d9#wmB^NtzYO*tD`Ag&2^ab%9{vce`5^q@9(ma zSHZP>069-Q&3H!7Z{{vXbA$vT*5n@7nAM;OHUUHV$DLf8Svy z3_03k`cBZ^$ur=f8<&#jS}RM6LO>p;rV_zCvCYRS`HMePaD>s_XRRu2bciIj?P!2< zYs&brPnd+OrK{cNHRZCFXu|lmN5=f_w!CuY%aDy%Mx(7eG+1Fh;ic)mNOp;TAwVu$ zTQ}_#?yY}u>wmxx(NT14Dv68pOYW9o^_a#TW-ldEb?)-YJU>gU@aN(s)3fKb(;kaY zVj;W`%G4`Tq&>ynD>`L!SVhh50EerUZjx#FhpK{I%+Im?wvb-D zS3G`P%zJg0UbfP*-IL1zc#pnMa=0iB*@s`A1pXtUCU)ZVN{kS~GGhW)4w7f8D9Qx4 zi_UeJdMKbUaaBn{j=9k2(xk9-8Lm_A&u$21W73u3s}7&IVJf%tYbNnDx{B0_ep5jaWDT&%LO_{>>i-9K_2OQ%jm zvE9}?O{AX!G~0^Mo`WUo59n%4`^?R4NkqV1)4N4C;IyLSvM{jUm$x$;9DF=n`-YLL zy>dQ7$O_s&UXK->@Wk@UEY~HZwVwLJtLZJFWA)1X#WaS`&rv%a8j1N|psoBm>=Opz z>Z0!Y3);}*h0vGoY_oBD8=hG$eCYY-2g{F_*5^}B{O{6v*lIL>5oyUjPY;>ngczg7 zJM%~8K;g2;{`rHG+@^UBFBX0t?ntZ&Km^bWr z6*J6t3asr498Qa&=g%xqqe+tT(hGg3YHEO1Mx&y|vvhD@s;tVec zk>C_0P1sYc1ac`~?xVxvb~tZeart9Nh7pU!D}UpDbeQkv1rbY0`M824tV52|Fs-x8 zPUg4FkrQ8JXsbuzT{iC7Q9b7%O}&hO;XXk^j*A?q3co@t!wT}vf-!*%;^Pbr(F%cVh-1DVN zv%!-5oW#lMTt+eWaFJzkNZj)`SRUH*vos$TmiUDf!UR&}1(*~C4|y#je}nO#{ZO-l zFgSrzNrt5uv#cyv6(}|`K(8lW*nqa{r5TxAQejI*twL8~d*FTXOPx1hNF2fIo!>Tg z@7!iL)r1d6oCMiHI**SM-v;X}nPim>b9V-7o&H2~3w?K|I(pP?z0Imju+}+9{0ZZw zkil~y<%yJsfw;JQERJryI+3)7VU|@GVT`v5J2-3Z^2i0*Hn!ccB2r0zrvJmsszb_P z6-6DQ)OVG9Y)jRA?EQ+3iI4fzAM?`Vxo^;Y=N{F%wL}cQH7|BamOM3 z?7dV-f78i;;`O=T5tZ`sBpid)Plkb$x~0N7<>ex9_nElH>`|9E>tfQHwbs1|b~o-I z;m7Sgu?rOw!|R^`ev=h#`4W@L*>MXo^IP3a&Po6?yqm(tsDtI5dLnM&9~sTl;kYMD z^G=$H+c~gW7gmRj5Nu;5j>MhtObi#hEv7>G7 zzz9}%xc^&WV_CfPmt);>Z%*!y*lsR@v+FfR+*|srKZQw(aBS@~+Bf7>&m65=4tSVt z)#x6T;Wzl}?tkv`&?(rdHgRzum=~eqLNx8oV|wroDGG3`?o9~YIezs#=cI90`eY~C z(pA-U=e&oOYb&`@>0BOe-ueq>TG`3VAXIR23exN&gS7QaDEquVSTkAdYy0)A@zG<{ z?$Vi8ZEn-mZifa#NYY>$_;8q0!|G6y4bq?Kcg5+fkXp$}#2`=bF#dO6Id_a4 zPPE(D3gC<09o$I0P}qL7f9DzaGO{+K6yRPYo-9(c>Ec1SYdJ!mBjhVu$C@5Qak~3sy3uPmb8+Webl2ZIE;t!Cs;9=0Nt1Kzhg`8=9XhY( zmd>8N7e{r;ZTAZ^+sk10wU`%LjW|W;FJ6LEnOW{=(+~UYx*&kA z1n449j}AFh*6~bq6&-uiV;S?Xq9=c~dlMcbFl}UE^5_xWYWd;uUWX4;6~NdoNncLgUC& z#|S^0>WIAMar8M>jj{C|2T%23mQs~op%U_FV5-hz^TauOf5t8F4u}1{jwSM^abR1| zY^u=OAl9ZjF_FtBpeAxp(qV7rkI*;DX4(y5^*%<-ud6_Qv}q^@Tn3$3w^xp2B)N58JJDEc}t9 zz*yamis`6WUCrYaD*fhUQlUBXs1%Obt+4>NL;MC@%fqU@ZsRA*1IKiSx-jlx3bWM} z^&2*i|#xTpVT;5 zlUX&zo(CeO9At>n%(=P4l2MAU=46Sf)#Y;?9?o^D>t)d=$eYH9DofSM#qLColl4t1 zza@{8z@w1_+eY;KQEB}ZC6S23DfpMghB@JCH0bIeNmk|SFR`-P?1sbWuEiW(M)-Gc z5K0R5!c30G6YBEjRn;2HrElyeofL>vmm6Pth%_~bW1P!tENML1&+KUAjefI>)E*^? zkg5~$$e-I))7m6mk2%@9(}A#25^y}Es%GSjl<$c1K!#&ZMlFdD1&ib*&UHi&y=mAo z4aa=1aFh0Z=9H))Qbad^J8(snC-BTfHYCP4uJ)&ZLw(;09nM$=ZzP-?29M2-dZ6Kd z&^kY-`*bCT?j7GltyVsRrytaJO*+g-g522nayXC8zN+c(vU?#>N&9$h-jVT_EW?X-4u_;B)B?Wspib$J>LTJRG%5wH6)PN<7&-KLvVAUUPSQy@gRgZ2@#l3bH^h4V^7 z>CQn-z=(KVfk;8SpiuSJ*UM85I-6fkEy@oWpuO0&3=YIUmXn}kX3D(WP78@&{RW%D z0P6p6JHMPN#T2=-717t|o5v;Z^bF%^&YcG}>+7+I*eVb}^!iyh6v|q!-n228@7&PU z)K*h-n0|AqU$~O7_{lIyIDO280%OIO1x7K1hK9+fM_zB;gq^P7{XDCCLn5p`c$VGN z6_*5Z3C1F4|E5{2b|v(LH4X=ry!G?zHVPmo5GER|FtZTMn{^45LTakoO}%z|B?S#b zo=Hdfl)|E3H%6K#Zv_v#)86KrgaqCCioT&6@3!PWSbUtVl~8i;K5sSG3G@9ckd}DF z_=n0<$%XA@uJ^F}pY35^Rk~FMZgTtB-nLtx-L@+g)u5}^$L;UrwVCjjlf;mNJ!e9ZX3=71 zMib3Kq0Yh8o|eJd>$LFl<1-w={vqPSVP2_6-7UqHU zK9F%D3*Xwohc~d*Inc3X_3BRRzNE#B3!qaVZOt)tP787F!GR<#l?_Fj*{ai0+A{pe zC0yRLwPM~am5mnd6j}#!?krqMI%>tgbv`t4P8+%H_7-Z1AfUFYLaj)k@Ebag zg%3FJONO_kBDqhYRk3tBJ<=82*kqTCYS_t$YG|$^*WrWIHEhxN+zW&ni>rz>Y_Z4r z2NyZ)rA`G% zS~TUsee#jb2agK!_6#F<;gM>e86iPZSS0~NnxuE}jGDZk`NEs9?kgN~uMy2SdXfYT zbTB)SLoHhe3cn^>g)A-C#(wcd1$Id4wt`U9P0MI#E>gRVA zZ+1dUBU__Zmb4I9MSTAvq%g7Njxh2SC|Y~`~1 zMJ0=U$wxH%O_hs2!T4HNIib_Y4m6As-_GL7RlCr@3ZK9zGGn&N8*wXrQhLf~1DLQDWPlS2i86?e51b`lr1_*kxDM+{sp7mAyZ^X{UUZxopxy^ljvMQ7 zGw(;V)JOMfRnE-;lpcaA3>M`O5Mi)x%Z15k{=u{2XN{>mXh*bb^IwK1JF8hkrS590 zj^=*w$o~SJKIjoMv`TvS4{fxjOzV-|;hZBu1cl{Kcv;_gd@Y0&Z8QecVAM_5m+rlQ z04;@%YF}?C>2pR;D2;~m74@R{_=oPw)t|s5oZ)YCsX$<&uoQA2M`CAfad!oiW%O8{CYBIuOYzD32n z*JNGwTfM)$<=@?i@Sf5~P0nzod$6E@byvd&UloK1`EOabAk)l`emKNPnok+YYOPFd zud;(>^jDvIkVS@Fq0gSWjV8$yvLZ}m=5c^umU=}(t=MeGvjs7zoJ0(L0`vG)Y%r*J zy8#IG3{A;i(sxvIGo`RN`$@3TE4!ktXsXp0Z#zgv)o)~64V3d&5^aPqQFN?#A4Knf z0IRrHH(bJbIQNdqRO|DB*blF5(EX*xyIz-kPH@wt37j6vSoEe0naP9)^y@rH| zrp|78Q-gN0VcpcSTBS>Ft4s{~fx*3^Rs+|dx^qW+v?UG zJl!7jr)a@Qes8rl^c{uHDTL$opoq#@?$bXB?cuQfSL-t^t}hJ%FZzHcR2kUrK{@;6JECe;n% z9wsrR>r?H+Z8_kK8-y4T9gFyd=pA;NX{pNc! zu>(Ci!jg`hCm)_euJXUZ-~_q8omwu6QFNJ4-*R2hI$tLp2fv)HPtarr^KyyvRmiOF zQZWci!I>T47J4;+!5jo>rDC+oTQ9N7WQ&S=M~MlkE(#ktxAM(pi2Xl>U3)y!`y220 z)rq5$Iz_q9+;f?0YAz=mRt+g3j94PKV(z!3G9he4BMK3@q_Bpe!&nE+<>VSVA@}97 z$hF^R)%o*$+v~O0{@CmF`EJkie%{aXyq{;^X&`UyJ8Bz+6$n4RhuxofJi_mi|L9J0 z?Yn-EUv+O#)phhuqMKK-vbnKq4?0id2TFVk_R8BtEte=c53betvQG4+ykWj6EV?o+ z-{-1j46y)l-LRhD1{Fl9t$e&ifp-n6%Zww|H)e7x!tnJ1Mkjc}l{?;+8vqUVEi)hx z>^W8fmza#O-v2}l!0mfRyec~OgPaii6aI+G%-QC?*6u~>ko{IZFMr){sO;8b#{XW%$DiCu@B zL4rQBJ|6Sp)fJ#VU{R!qGG9!$ z2!D;;5agIzEs5u#V3tXJ$?Ei>WJ1XuVkh}BT}3fJOD9Q#l+mETSK$*ZS>^R5LZx|F z+C-gSh(0x#m}O}g16kNf&hvm{BX*k&E)LAVGB!NeEgG!wQ@g};`x8)39ekq0^~&1% zeSRV)_%7->74K^Q4y{K{P!YZT*HnN>Az^E~Q$uO654Rsl%Q4#gA|ow!pDzc5BO|MdN;vdRpe~m4#fB@BV12lfsaFJ=!r>dVb47U zTj2H2^B2bH2W=!FPvfn<&i!mOy@7kZ*=hLhuNdi*vh5*W@fjg?B6kdj`H_|5TYYu^ z>`X=x3hs~TC<^HylUC45(2l+V9DS9?#nY>FGa8QBQV5*l6b%TUK`d=Iqup$1;1zWn z5zes72q6`?ufzf{5R2P)A=CQoqxLyt0U(#zCCexRH|Cd|oRrC)=>) zpXI)a#f;9k!IEj5kDz=JSauA!51r}u(&VV?nNJLXW5fg(@-OD*H^=X=HX-57I8SDH zIW$9xt4nv0Pcx{nd}(!OZHI!;*>b5+WC6I21tNQXpLc9(AWkQzczu2d6!d3CCz>mQ zeS-K398Ny;*&*(@MDL?|c(HHoQaX0(HtzZSbGuOSxSw&x*6-YtDl8Ri zVk|-}VhxSBxW6Lpd(e+++nL6584RIQWV~_j_=KIe47fmC21n!M$p zlG&@t*;k~-5Vw}?; zMyk4lthR02rHVoQaqd4k$Z!jQZ!-E&<-Z<%q?CVTI@l^&Ac}l|3uh*sB(;L+9=lyc zxviS*d1L}QqC0v4d9Rti{DC=@jVB9!qa0ej2!=G3>-p!w91zl@v6Wv1*8@tztLfiy zcVgDc9Ex&zf4>WCpr3ikqB-a)fdz3!adZTPtlI^5FU}*%hD_y%8 zt5+|fehSgIJ!`->(sN`!+T~JY!3#XopI@~`i<1JO!2d$28H#-MNBlhIy5B^@Q zGpsl^M=2S_T0bPq3UN4hTCYqr^SPJaRRN$KmDO-5RD5lN9m+Zngrlbl2Xh1$9fDoCr%7`uK0Ogv@CB zCf&5A45t{|-&gYbl0JpF*ertx<4gsgFZ4A-y?;L}=RA>LGx|QOT`x^ekQ(S(qyO^# z;&JUvW^gZbdT@FM(gU+Qw(^_w5{865NLDK6_D);z*1$wWWQmP!3qQeMl<1JS2_K;9 zf8jXFWpoh*fR-K>4LwcT{+lg;ytp~Fva$-!+1*}iY-^J~W_aHgj3 zh~}VwF*BokbIS4jTP^=FpSOyTOVQU2Kk))S6>HH?cwS9g&NL+@iibKF^(S>+8>p~G zF;dXs6zlmk*^WAYG;;MzcxN!)hxO{6hcFEL)ina!ZKSG9dJ2^d_DZS+?KlG4mD3Nb zqKYBCM09aT_Wk3H%?$Sf$Gt9cs-h@|-*JUC)RK6)>@UFt(U563Ka6i6rCIK2k;uIR zIO;LTW%}z4`~aK6Hv^ff88gF~n67%FX^42Ng?=%t8LMl+t?qei4%Ny~L2aKk5E4lc zy%Jd+OF(O5w?4IAIb5$5RAZ?N0e$e685ZzC;p*Vp ze{KkCUBty74m>Y@?tZ<@Z0K}yAJq>#+fO?N>vFlpjweQ*>>Be}@;KZx?(_L4@NH-S zWxlY@SAEv-fMp~WK2?c19G57gq#$xI*DPayuG#V9#^>9nmk$4GbYpW)#Ce1)&{8MY ziw>N4borzi$tDX+9xN&nsY3{_Sx7^B=vQtZWoZ5|d$H_72&0&kSD}yT8X)33%PY8Q zKAtfP8Ws(3wRkYZ*gV(SkUzyn3WF(cGj~{1A07_IaJr=r322HecF5hjk=s?PJxl!y zQg7ZC^$HOC#jEX4t&6<8O7(PL!7ik{GJ^;ODWR~wCxwWxHjXgjW4y4o>jk{}B|6FP zVKn~5gKKWA(Q8gDmupv9AHf`}^~K3)Tm3I9L{1H8o&@#~=!?(}bb7`$YcYA|aWyDQ zSs3yYGZN>y>1dZ`&e)d@8slc}hysEzmdBk8rm3%ZrQbC&XZ*7{2+Elfy;D0`IHW18 zS~|9-OOvL7$-NYTZLF>>KEV47^gfoQE9`It?}g99qyFOY}Dz{ zI%!P4=3g;)W1maq<`S4sNEtmR=0Y26h@D)4!w znV{KjObI|&)hPLvSuz8}V%epQCiM??=006AB>2xJmP)VZEHQ^CkJid{dG;;^BpD;K z;8UCTHJ6q=W$9%ahZehUr45BNyo6?1L(nCHm@Dylh4!(FV5E3jPu(Dq@)X-$mAc{g zMcgwVz!C0}6l`~&I0l)SH#Gn!yU(!A>@*%bJXI7LDfgG0PKK1nf8k-MfD)tc({rMmHtFb2Jm=U#BJk zIY5xsA|i_EKAHkAH-PnLz^B-rhs|XKkB8pd1M8=x;3dqfUHsHt|IS@f*;=c^UHJog gAk4dWa!1rx0avvWt=7jm<=emmgEm8zo^ZPHfA+x|3;+NC literal 0 HcmV?d00001 diff --git a/apps/emqx_s3/rebar.config b/apps/emqx_s3/rebar.config new file mode 100644 index 000000000..f8e4d4e42 --- /dev/null +++ b/apps/emqx_s3/rebar.config @@ -0,0 +1,6 @@ +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {erlcloud, {git, "https://github.com/savonarola/erlcloud", {tag, "3.6.7-emqx-1"}}} +]}. + +{project_plugins, [erlfmt]}. diff --git a/apps/emqx_s3/src/emqx_s3.app.src b/apps/emqx_s3/src/emqx_s3.app.src new file mode 100644 index 000000000..7864ffb29 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3.app.src @@ -0,0 +1,14 @@ +{application, emqx_s3, [ + {description, "EMQX S3"}, + {vsn, "5.0.6"}, + {modules, []}, + {registered, [emqx_s3_sup]}, + {applications, [ + kernel, + stdlib, + gproc, + erlcloud, + ehttpc + ]}, + {mod, {emqx_s3_app, []}} +]}. diff --git a/apps/emqx_s3/src/emqx_s3.erl b/apps/emqx_s3/src/emqx_s3.erl new file mode 100644 index 000000000..6d2577dca --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3.erl @@ -0,0 +1,65 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3). + +-include_lib("emqx/include/types.hrl"). + +-export([ + start_profile/2, + stop_profile/1, + update_profile/2, + start_uploader/2, + with_client/2 +]). + +-export_type([ + profile_id/0, + profile_config/0 +]). + +-type profile_id() :: term(). + +%% TODO: define fields +-type profile_config() :: map(). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec start_profile(profile_id(), profile_config()) -> ok_or_error(term()). +start_profile(ProfileId, ProfileConfig) -> + case emqx_s3_sup:start_profile(ProfileId, ProfileConfig) of + {ok, _} -> + ok; + {error, _} = Error -> + Error + end. + +-spec stop_profile(profile_id()) -> ok_or_error(term()). +stop_profile(ProfileId) -> + emqx_s3_sup:stop_profile(ProfileId). + +-spec update_profile(profile_id(), profile_config()) -> ok_or_error(term()). +update_profile(ProfileId, ProfileConfig) -> + emqx_s3_profile_conf:update_config(ProfileId, ProfileConfig). + +-spec start_uploader(profile_id(), emqx_s3_uploader:opts()) -> + supervisor:start_ret() | {error, profile_not_found}. +start_uploader(ProfileId, Opts) -> + emqx_s3_profile_uploader_sup:start_uploader(ProfileId, Opts). + +-spec with_client(profile_id(), fun((emqx_s3_client:client()) -> Result)) -> + {error, profile_not_found} | Result. +with_client(ProfileId, Fun) when is_function(Fun, 1) -> + case emqx_s3_profile_conf:checkout_config(ProfileId) of + {ok, ClientConfig, _UploadConfig} -> + try + Fun(emqx_s3_client:create(ClientConfig)) + after + emqx_s3_profile_conf:checkin_config(ProfileId) + end; + {error, _} = Error -> + Error + end. diff --git a/apps/emqx_s3/src/emqx_s3_app.erl b/apps/emqx_s3/src/emqx_s3_app.erl new file mode 100644 index 000000000..8d8b0f7b9 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_app.erl @@ -0,0 +1,16 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +start(_Type, _Args) -> + {ok, Sup} = emqx_s3_sup:start_link(), + {ok, Sup}. + +stop(_State) -> + ok. diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl new file mode 100644 index 000000000..01d677922 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -0,0 +1,293 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_client). + +-include_lib("emqx/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("erlcloud/include/erlcloud_aws.hrl"). + +-compile(nowarn_export_all). +-compile(export_all). + +-export([ + create/1, + + put_object/3, + + start_multipart/2, + upload_part/5, + complete_multipart/4, + abort_multipart/3, + list/2, + + format/1 +]). + +-export_type([client/0]). + +-type s3_bucket_acl() :: + private + | public_read + | public_read_write + | authenticated_read + | bucket_owner_read + | bucket_owner_full_control. + +-type headers() :: #{binary() => binary()}. + +-type key() :: string(). +-type part_number() :: non_neg_integer(). +-type upload_id() :: string(). +-type etag() :: string(). + +-type upload_options() :: list({acl, s3_bucket_acl()}). + +-opaque client() :: #{ + aws_config := aws_config(), + options := upload_options(), + bucket := string(), + headers := headers() +}. + +-type config() :: #{ + scheme := string(), + host := string(), + port := part_number(), + bucket := string(), + headers := headers(), + acl := s3_bucket_acl(), + access_key_id := string() | undefined, + secret_access_key := string() | undefined, + http_pool := ecpool:pool_name(), + request_timeout := timeout() +}. + +-type s3_options() :: list({string(), string()}). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec create(config()) -> client(). +create(Config) -> + #{ + aws_config => aws_config(Config), + upload_options => upload_options(Config), + bucket => maps:get(bucket, Config), + headers => headers(Config) + }. + +-spec put_object(client(), key(), iodata()) -> ok_or_error(term()). +put_object( + #{bucket := Bucket, upload_options := Options, headers := Headers, aws_config := AwsConfig}, + Key, + Value +) -> + try erlcloud_s3:put_object(Bucket, Key, Value, Options, Headers, AwsConfig) of + Props when is_list(Props) -> + ok + catch + error:{aws_error, Reason} -> + ?SLOG(debug, #{msg => "put_object_fail", key => Key, reason => Reason}), + {error, Reason} + end. + +-spec start_multipart(client(), key()) -> ok_or_error(upload_id(), term()). +start_multipart( + #{bucket := Bucket, upload_options := Options, headers := Headers, aws_config := AwsConfig}, + Key +) -> + case erlcloud_s3:start_multipart(Bucket, Key, Options, Headers, AwsConfig) of + {ok, Props} -> + {ok, proplists:get_value(uploadId, Props)}; + {error, Reason} -> + ?SLOG(debug, #{msg => "start_multipart_fail", key => Key, reason => Reason}), + {error, Reason} + end. + +-spec upload_part(client(), key(), upload_id(), part_number(), iodata()) -> + ok_or_error(etag(), term()). +upload_part( + #{bucket := Bucket, headers := Headers, aws_config := AwsConfig}, + Key, + UploadId, + PartNumber, + Value +) -> + case erlcloud_s3:upload_part(Bucket, Key, UploadId, PartNumber, Value, Headers, AwsConfig) of + {ok, Props} -> + {ok, proplists:get_value(etag, Props)}; + {error, Reason} -> + ?SLOG(debug, #{msg => "upload_part_fail", key => Key, reason => Reason}), + {error, Reason} + end. + +-spec complete_multipart(client(), key(), upload_id(), [etag()]) -> ok_or_error(term()). +complete_multipart( + #{bucket := Bucket, headers := Headers, aws_config := AwsConfig}, Key, UploadId, ETags +) -> + case erlcloud_s3:complete_multipart(Bucket, Key, UploadId, ETags, Headers, AwsConfig) of + ok -> + ok; + {error, Reason} -> + ?SLOG(debug, #{msg => "complete_multipart_fail", key => Key, reason => Reason}), + {error, Reason} + end. + +-spec abort_multipart(client(), key(), upload_id()) -> ok_or_error(term()). +abort_multipart(#{bucket := Bucket, headers := Headers, aws_config := AwsConfig}, Key, UploadId) -> + case erlcloud_s3:abort_multipart(Bucket, Key, UploadId, [], Headers, AwsConfig) of + ok -> + ok; + {error, Reason} -> + ?SLOG(debug, #{msg => "abort_multipart_fail", key => Key, reason => Reason}), + {error, Reason} + end. + +-spec list(client(), s3_options()) -> ok_or_error(term()). +list(#{bucket := Bucket, aws_config := AwsConfig}, Options) -> + try + {ok, erlcloud_s3:list_objects(Bucket, Options, AwsConfig)} + catch + error:{aws_error, Reason} -> + ?SLOG(debug, #{msg => "list_objects_fail", bucket => Bucket, reason => Reason}), + {error, Reason} + end. + +-spec format(client()) -> term(). +format(#{aws_config := AwsConfig} = Client) -> + Client#{aws_config => AwsConfig#aws_config{secret_access_key = "***"}}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +upload_options(Config) -> + [ + {acl, maps:get(acl, Config)} + ]. + +headers(#{headers := Headers}) -> + maps:to_list(Headers). + +aws_config(#{ + scheme := Scheme, + host := Host, + port := Port, + headers := Headers, + access_key_id := AccessKeyId, + secret_access_key := SecretAccessKey, + http_pool := HttpPool, + request_timeout := Timeout +}) -> + #aws_config{ + s3_scheme = Scheme, + s3_host = Host, + s3_port = Port, + s3_bucket_access_method = path, + + access_key_id = AccessKeyId, + secret_access_key = SecretAccessKey, + + http_client = request_fun(Headers, HttpPool), + timeout = Timeout + }. + +-type http_headers() :: [{binary(), binary()}]. +-type http_pool() :: term(). + +-spec request_fun(http_headers(), http_pool()) -> erlcloud_httpc:request_fun(). +request_fun(CustomHeaders, HttpPool) -> + fun(Url, Method, Headers, Body, Timeout, _Config) -> + with_path_and_query_only(Url, fun(PathQuery) -> + JoinedHeaders = join_headers(Headers, CustomHeaders), + Request = make_request(Method, PathQuery, JoinedHeaders, Body), + ehttpc_request(HttpPool, Method, Request, Timeout) + end) + end. + +ehttpc_request(HttpPool, Method, Request, Timeout) -> + try ehttpc:request(HttpPool, Method, Request, Timeout) of + {ok, StatusCode, RespHeaders} -> + {ok, {{StatusCode, undefined}, string_headers(RespHeaders), undefined}}; + {ok, StatusCode, RespHeaders, RespBody} -> + {ok, {{StatusCode, undefined}, string_headers(RespHeaders), RespBody}}; + {error, Reason} -> + ?SLOG(error, #{ + msg => "s3_ehttpc_request_fail", + reason => Reason, + timeout => Timeout, + pool => HttpPool, + method => Method + }), + {error, Reason} + catch + error:badarg -> + ?SLOG(error, #{ + msg => "s3_ehttpc_request_fail", + reason => badarg, + timeout => Timeout, + pool => HttpPool, + method => Method + }), + {error, no_ehttpc_pool}; + error:Reason -> + ?SLOG(error, #{ + msg => "s3_ehttpc_request_fail", + reason => Reason, + timeout => Timeout, + pool => HttpPool, + method => Method + }), + {error, Reason} + end. + +-define(IS_BODY_EMPTY(Body), (Body =:= undefined orelse Body =:= <<>>)). +-define(NEEDS_BODY(Method), (Method =:= get orelse Method =:= head orelse Method =:= delete)). + +make_request(Method, PathQuery, Headers, Body) when + ?IS_BODY_EMPTY(Body) andalso ?NEEDS_BODY(Method) +-> + {PathQuery, Headers}; +make_request(_Method, PathQuery, Headers, Body) when ?IS_BODY_EMPTY(Body) -> + {PathQuery, [{<<"content-length">>, <<"0">>} | Headers], <<>>}; +make_request(_Method, PathQuery, Headers, Body) -> + {PathQuery, Headers, Body}. + +format_request({PathQuery, Headers, _Body}) -> {PathQuery, Headers, <<"...">>}. + +join_headers(Headers, CustomHeaders) -> + MapHeaders = lists:foldl( + fun({K, V}, MHeaders) -> + maps:put(to_binary(K), V, MHeaders) + end, + #{}, + Headers ++ maps:to_list(CustomHeaders) + ), + maps:to_list(MapHeaders). + +with_path_and_query_only(Url, Fun) -> + case string:split(Url, "//", leading) of + [_Scheme, UrlRem] -> + case string:split(UrlRem, "/", leading) of + [_HostPort, PathQuery] -> + Fun([$/ | PathQuery]); + _ -> + {error, {invalid_url, Url}} + end; + _ -> + {error, {invalid_url, Url}} + end. + +to_binary(Val) when is_list(Val) -> list_to_binary(Val); +to_binary(Val) when is_binary(Val) -> Val. + +string_headers(Hdrs) -> + [{string:to_lower(to_list_string(K)), to_list_string(V)} || {K, V} <- Hdrs]. + +to_list_string(Val) when is_binary(Val) -> + binary_to_list(Val); +to_list_string(Val) when is_list(Val) -> + Val. diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl new file mode 100644 index 000000000..09e945edc --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -0,0 +1,390 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_conf). + +-behaviour(gen_server). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-export([ + start_link/2, + child_spec/2 +]). + +-export([ + checkout_config/1, + checkout_config/2, + checkin_config/1, + checkin_config/2, + + update_config/2, + update_config/3 +]). + +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). + +%% For test purposes +-export([ + client_config/2, + start_http_pool/2, + id/1 +]). + +-define(DEFAULT_CALL_TIMEOUT, 5000). + +-define(DEFAULT_HTTP_POOL_TIMEOUT, 60000). +-define(DEAFULT_HTTP_POOL_CLEANUP_INTERVAL, 60000). + +-spec child_spec(emqx_s3:profile_id(), emqx_s3:profile_config()) -> supervisor:child_spec(). +child_spec(ProfileId, ProfileConfig) -> + #{ + id => ProfileId, + start => {?MODULE, start_link, [ProfileId, ProfileConfig]}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [?MODULE] + }. + +-spec start_link(emqx_s3:profile_id(), emqx_s3:profile_config()) -> gen_server:start_ret(). +start_link(ProfileId, ProfileConfig) -> + gen_server:start_link(?MODULE, [ProfileId, ProfileConfig], []). + +-spec update_config(emqx_s3:profile_id(), emqx_s3:profile_config()) -> ok_or_error(term()). +update_config(ProfileId, ProfileConfig) -> + update_config(ProfileId, ProfileConfig, ?DEFAULT_CALL_TIMEOUT). + +-spec update_config(emqx_s3:profile_id(), emqx_s3:profile_config(), timeout()) -> + ok_or_error(term()). +update_config(ProfileId, ProfileConfig, Timeout) -> + case gproc:where({n, l, id(ProfileId)}) of + undefined -> + {error, profile_not_found}; + Pid -> + gen_server:call(Pid, {update_config, ProfileConfig}, Timeout) + end. + +-spec checkout_config(emqx_s3:profile_id()) -> + {ok, emqx_s3_client:config(), emqx_s3_uploader:config()} | {error, profile_not_found}. +checkout_config(ProfileId) -> + checkout_config(ProfileId, ?DEFAULT_CALL_TIMEOUT). + +-spec checkout_config(emqx_s3:profile_id(), timeout()) -> + {ok, emqx_s3_client:config(), emqx_s3_uploader:config()} | {error, profile_not_found}. +checkout_config(ProfileId, Timeout) -> + case gproc:where({n, l, id(ProfileId)}) of + undefined -> + {error, profile_not_found}; + Pid -> + gen_server:call(Pid, {checkout_config, self()}, Timeout) + end. + +-spec checkin_config(emqx_s3:profile_id()) -> ok | {error, profile_not_found}. +checkin_config(ProfileId) -> + checkin_config(ProfileId, ?DEFAULT_CALL_TIMEOUT). + +-spec checkin_config(emqx_s3:profile_id(), timeout()) -> ok | {error, profile_not_found}. +checkin_config(ProfileId, Timeout) -> + case gproc:where({n, l, id(ProfileId)}) of + undefined -> + {error, profile_not_found}; + Pid -> + gen_server:call(Pid, {checkin_config, self()}, Timeout) + end. + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([ProfileId, ProfileConfig]) -> + _ = process_flag(trap_exit, true), + ok = cleanup_orphaned_pools(ProfileId), + case start_http_pool(ProfileId, ProfileConfig) of + {ok, PoolName} -> + true = gproc:reg({n, l, id(ProfileId)}, ignored), + HttpPoolCleanupInterval = http_pool_cleanup_interval(ProfileConfig), + {ok, #{ + profile_id => ProfileId, + profile_config => ProfileConfig, + client_config => client_config(ProfileConfig, PoolName), + uploader_config => uploader_config(ProfileConfig), + pool_name => PoolName, + pool_clients => emqx_s3_profile_http_pool_clients:create_table(), + %% We don't expose these options to users currently, but use in tests + http_pool_timeout => http_pool_timeout(ProfileConfig), + http_pool_cleanup_interval => HttpPoolCleanupInterval, + + outdated_pool_cleanup_tref => erlang:send_after( + HttpPoolCleanupInterval, self(), cleanup_outdated + ) + }}; + {error, Reason} -> + {stop, Reason} + end. + +handle_call( + {checkout_config, Pid}, + _From, + #{ + client_config := ClientConfig, + uploader_config := UploaderConfig + } = State +) -> + ok = register_client(Pid, State), + {reply, {ok, ClientConfig, UploaderConfig}, State}; +handle_call({checkin_config, Pid}, _From, State) -> + ok = unregister_client(Pid, State), + {reply, ok, State}; +handle_call( + {update_config, NewProfileConfig}, + _From, + #{profile_id := ProfileId} = State +) -> + case update_http_pool(ProfileId, NewProfileConfig, State) of + {ok, PoolName} -> + NewState = State#{ + profile_config => NewProfileConfig, + client_config => client_config(NewProfileConfig, PoolName), + uploader_config => uploader_config(NewProfileConfig), + http_pool_timeout => http_pool_timeout(NewProfileConfig), + http_pool_cleanup_interval => http_pool_cleanup_interval(NewProfileConfig), + pool_name => PoolName + }, + {reply, ok, NewState}; + {error, Reason} -> + {reply, {error, Reason}, State} + end; +handle_call(_Request, _From, State) -> + {reply, {error, not_implemented}, State}. + +handle_cast(_Request, State) -> + {noreply, State}. + +handle_info({'DOWN', _Ref, process, Pid, _Reason}, State) -> + ok = unregister_client(Pid, State), + {noreply, State}; +handle_info(cleanup_outdated, #{http_pool_cleanup_interval := HttpPoolCleanupInterval} = State0) -> + %% Maybe cleanup asynchoronously + ok = cleanup_outdated_pools(State0), + State1 = State0#{ + outdated_pool_cleanup_tref => erlang:send_after( + HttpPoolCleanupInterval, self(), cleanup_outdated + ) + }, + {noreply, State1}; +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, #{profile_id := ProfileId}) -> + lists:foreach( + fun(PoolName) -> + ok = stop_http_pool(ProfileId, PoolName) + end, + emqx_s3_profile_http_pools:all(ProfileId) + ). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +id(ProfileId) -> + {?MODULE, ProfileId}. + +client_config(ProfileConfig, PoolName) -> + HTTPOpts = maps:get(transport_options, ProfileConfig, #{}), + #{ + scheme => scheme(HTTPOpts), + host => maps:get(host, ProfileConfig), + port => maps:get(port, ProfileConfig), + headers => maps:get(headers, HTTPOpts, #{}), + acl => maps:get(acl, ProfileConfig), + bucket => maps:get(bucket, ProfileConfig), + access_key_id => maps:get(access_key_id, ProfileConfig, undefined), + secret_access_key => maps:get(secret_access_key, ProfileConfig, undefined), + request_timeout => maps:get(request_timeout, HTTPOpts, undefined), + http_pool => PoolName + }. + +uploader_config(#{max_part_size := MaxPartSize, min_part_size := MinPartSize} = _ProfileConfig) -> + #{ + min_part_size => MinPartSize, + max_part_size => MaxPartSize + }. + +scheme(#{ssl := #{enable := true}}) -> "https://"; +scheme(_TransportOpts) -> "http://". + +start_http_pool(ProfileId, ProfileConfig) -> + HttpConfig = http_config(ProfileConfig), + PoolName = pool_name(ProfileId), + case do_start_http_pool(PoolName, HttpConfig) of + ok -> + ok = emqx_s3_profile_http_pools:register(ProfileId, PoolName), + ok = ?tp(debug, "s3_start_http_pool", #{pool_name => PoolName, profile_id => ProfileId}), + {ok, PoolName}; + {error, _} = Error -> + Error + end. + +update_http_pool(ProfileId, ProfileConfig, #{pool_name := OldPoolName} = State) -> + HttpConfig = http_config(ProfileConfig), + OldHttpConfig = old_http_config(State), + case OldHttpConfig =:= HttpConfig of + true -> + {ok, OldPoolName}; + false -> + PoolName = pool_name(ProfileId), + case do_start_http_pool(PoolName, HttpConfig) of + ok -> + ok = set_old_pool_outdated(State), + ok = emqx_s3_profile_http_pools:register(ProfileId, PoolName), + {ok, PoolName}; + {error, _} = Error -> + Error + end + end. + +pool_name(ProfileId) -> + iolist_to_binary([ + <<"s3-http-">>, + ProfileId, + <<"-">>, + integer_to_binary(erlang:system_time(millisecond)), + <<"-">>, + integer_to_binary(erlang:unique_integer([positive])) + ]). + +old_http_config(#{profile_config := ProfileConfig}) -> http_config(ProfileConfig). + +set_old_pool_outdated(#{ + profile_id := ProfileId, pool_name := PoolName, http_pool_timeout := HttpPoolTimeout +}) -> + _ = emqx_s3_profile_http_pools:set_outdated(ProfileId, PoolName, HttpPoolTimeout), + ok. + +cleanup_orphaned_pools(ProfileId) -> + lists:foreach( + fun(PoolName) -> + ok = stop_http_pool(ProfileId, PoolName) + end, + emqx_s3_profile_http_pools:all(ProfileId) + ). + +register_client(Pid, #{profile_id := ProfileId, pool_clients := PoolClients, pool_name := PoolName}) -> + MRef = monitor(process, Pid), + ok = emqx_s3_profile_http_pool_clients:register(PoolClients, Pid, MRef, PoolName), + _ = emqx_s3_profile_http_pools:register_client(ProfileId, PoolName), + ok. + +unregister_client( + Pid, + #{ + profile_id := ProfileId, pool_clients := PoolClients, pool_name := PoolName + } +) -> + case emqx_s3_profile_http_pool_clients:unregister(PoolClients, Pid) of + undefined -> + ok; + {MRef, PoolName} -> + true = erlang:demonitor(MRef, [flush]), + _ = emqx_s3_profile_http_pools:unregister_client(ProfileId, PoolName), + ok; + {MRef, OutdatedPoolName} -> + true = erlang:demonitor(MRef, [flush]), + ClientNum = emqx_s3_profile_http_pools:unregister_client(ProfileId, OutdatedPoolName), + maybe_stop_outdated_pool(ProfileId, OutdatedPoolName, ClientNum) + end. + +maybe_stop_outdated_pool(ProfileId, OutdatedPoolName, 0) -> + ok = stop_http_pool(ProfileId, OutdatedPoolName); +maybe_stop_outdated_pool(_ProfileId, _OutdatedPoolName, _ClientNum) -> + ok. + +cleanup_outdated_pools(#{profile_id := ProfileId}) -> + lists:foreach( + fun(PoolName) -> + ok = stop_http_pool(ProfileId, PoolName) + end, + emqx_s3_profile_http_pools:outdated(ProfileId) + ). + +%%-------------------------------------------------------------------- +%% HTTP Pool implementation dependent functions +%%-------------------------------------------------------------------- + +http_config( + #{ + host := Host, + port := Port, + transport_options := #{ + pool_type := PoolType, + pool_size := PoolSize, + enable_pipelining := EnablePipelining, + connect_timeout := ConnectTimeout + } = HTTPOpts + } +) -> + {Transport, TransportOpts} = + case scheme(HTTPOpts) of + "http://" -> + {tcp, []}; + "https://" -> + SSLOpts = emqx_tls_lib:to_client_opts(maps:get(ssl, HTTPOpts)), + {tls, SSLOpts} + end, + NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), + [ + {host, Host}, + {port, Port}, + {connect_timeout, ConnectTimeout}, + {keepalive, 30000}, + {pool_type, PoolType}, + {pool_size, PoolSize}, + {transport, Transport}, + {transport_opts, NTransportOpts}, + {enable_pipelining, EnablePipelining} + ]. + +http_pool_cleanup_interval(ProfileConfig) -> + maps:get( + http_pool_cleanup_interval, ProfileConfig, ?DEAFULT_HTTP_POOL_CLEANUP_INTERVAL + ). + +http_pool_timeout(ProfileConfig) -> + maps:get( + http_pool_timeout, ProfileConfig, ?DEFAULT_HTTP_POOL_TIMEOUT + ). + +stop_http_pool(ProfileId, PoolName) -> + case ehttpc_sup:stop_pool(PoolName) of + ok -> + ok; + {error, Reason} -> + ?SLOG(error, #{msg => "ehttpc_pool_stop_fail", pool_name => PoolName, reason => Reason}), + ok + end, + ok = emqx_s3_profile_http_pools:unregister(ProfileId, PoolName), + ok = ?tp(debug, "s3_stop_http_pool", #{pool_name => PoolName}). + +do_start_http_pool(PoolName, HttpConfig) -> + case ehttpc_sup:start_pool(PoolName, HttpConfig) of + {ok, _} -> + ok; + {error, _} = Error -> + Error + end. diff --git a/apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl b/apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl new file mode 100644 index 000000000..b4e640f7c --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl @@ -0,0 +1,35 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_http_pool_clients). + +-export([ + create_table/0, + + register/4, + unregister/2 +]). + +-define(TAB, ?MODULE). + +-spec create_table() -> ok. +create_table() -> + ets:new(?TAB, [ + private, + set + ]). + +-spec register(ets:tid(), pid(), reference(), emqx_s3_profile_http_pools:pool_name()) -> true. +register(Tab, Pid, MRef, PoolName) -> + true = ets:insert(Tab, {Pid, {MRef, PoolName}}), + ok. + +-spec unregister(ets:tid(), pid()) -> emqx_s3_profile_http_pools:pool_name() | undefined. +unregister(Tab, Pid) -> + case ets:take(Tab, Pid) of + [{Pid, {MRef, PoolName}}] -> + {MRef, PoolName}; + [] -> + undefined + end. diff --git a/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl b/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl new file mode 100644 index 000000000..e1b36c3be --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl @@ -0,0 +1,123 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_http_pools). + +-include_lib("stdlib/include/ms_transform.hrl"). + +-export([ + create_table/0, + + register/2, + unregister/2, + + register_client/2, + unregister_client/2, + + set_outdated/3, + + outdated/1, + all/1 +]). + +-export_type([pool_name/0]). + +-define(TAB, ?MODULE). + +-type pool_name() :: ecpool:pool_name(). + +-type pool_key() :: {emqx_s3:profile_id(), pool_name()}. + +-record(pool, { + key :: pool_key(), + client_count = 0 :: integer(), + deadline = undefined :: undefined | integer(), + extra = #{} :: map() +}). + +-spec create_table() -> ok. +create_table() -> + _ = ets:new(?TAB, [ + named_table, + public, + ordered_set, + {keypos, #pool.key}, + {read_concurrency, true}, + {write_concurrency, true} + ]), + ok. + +-spec register(emqx_s3:profile_id(), pool_name()) -> + ok. +register(ProfileId, PoolName) -> + Key = key(ProfileId, PoolName), + true = ets:insert(?TAB, #pool{ + key = Key, + client_count = 0, + deadline = undefined, + extra = #{} + }), + ok. + +-spec unregister(emqx_s3:profile_id(), pool_name()) -> + ok. +unregister(ProfileId, PoolName) -> + Key = key(ProfileId, PoolName), + true = ets:delete(?TAB, Key), + ok. + +-spec register_client(emqx_s3:profile_id(), pool_name()) -> + integer(). +register_client(ProfileId, PoolName) -> + Key = key(ProfileId, PoolName), + ets:update_counter(?TAB, Key, {#pool.client_count, 1}). + +-spec unregister_client(emqx_s3:profile_id(), pool_name()) -> + integer(). +unregister_client(ProfileId, PoolName) -> + Key = key(ProfileId, PoolName), + try + ets:update_counter(?TAB, Key, {#pool.client_count, -1}) + catch + error:badarg -> + undefined + end. + +-spec set_outdated(emqx_s3:profile_id(), pool_name(), integer()) -> + ok. +set_outdated(ProfileId, PoolName, Timeout) -> + Key = key(ProfileId, PoolName), + Now = erlang:monotonic_time(millisecond), + ets:update_element(?TAB, Key, {#pool.deadline, Now + Timeout}). + +-spec outdated(emqx_s3:profile_id()) -> + [pool_name()]. +outdated(ProfileId) -> + Now = erlang:monotonic_time(millisecond), + MS = ets:fun2ms( + fun(#pool{key = {ProfileId_, PoolName}, deadline = Deadline_}) when + ProfileId_ =:= ProfileId andalso + Deadline_ =/= undefined andalso Deadline_ < Now + -> + PoolName + end + ), + ets:select(?TAB, MS). + +-spec all(emqx_s3:profile_id()) -> + [pool_name()]. +all(ProfileId) -> + MS = ets:fun2ms( + fun(#pool{key = {ProfileId_, PoolName}}) when ProfileId_ =:= ProfileId -> + PoolName + end + ), + ets:select(?TAB, MS). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +key(ProfileId, PoolName) -> + {ProfileId, PoolName}. diff --git a/apps/emqx_s3/src/emqx_s3_profile_sup.erl b/apps/emqx_s3/src/emqx_s3_profile_sup.erl new file mode 100644 index 000000000..c39fc9f4b --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_profile_sup.erl @@ -0,0 +1,48 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_sup). + +-behaviour(supervisor). + +-include_lib("emqx/include/types.hrl"). + +-export([ + start_link/2, + child_spec/2 +]). + +-export([init/1]). + +-spec start_link(emqx_s3:profile_id(), emqx_s3:profile_config()) -> supervisor:start_ret(). +start_link(ProfileId, ProfileConfig) -> + supervisor:start_link(?MODULE, [ProfileId, ProfileConfig]). + +-spec child_spec(emqx_s3:profile_id(), emqx_s3:profile_config()) -> supervisor:child_spec(). +child_spec(ProfileId, ProfileConfig) -> + #{ + id => ProfileId, + start => {?MODULE, start_link, [ProfileId, ProfileConfig]}, + restart => permanent, + shutdown => 5000, + type => supervisor, + modules => [?MODULE] + }. + +%%-------------------------------------------------------------------- +%% supervisor callbacks +%%------------------------------------------------------------------- + +init([ProfileId, ProfileConfig]) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 5 + }, + ChildSpecs = [ + %% Order matters + emqx_s3_profile_conf:child_spec(ProfileId, ProfileConfig), + emqx_s3_profile_uploader_sup:child_spec(ProfileId) + ], + {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl b/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl new file mode 100644 index 000000000..1cd155a77 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl @@ -0,0 +1,73 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_uploader_sup). + +-behaviour(supervisor). + +-include_lib("emqx/include/types.hrl"). + +-export([ + start_link/1, + child_spec/1, + id/1, + start_uploader/2 +]). + +-export([init/1]). + +-export_type([id/0]). + +-type id() :: {?MODULE, emqx_s3:profile_id()}. + +-spec start_link(emqx_s3:profile_id()) -> supervisor:start_ret(). +start_link(ProfileId) -> + supervisor:start_link(?MODULE, [ProfileId]). + +-spec child_spec(emqx_s3:profile_id()) -> supervisor:child_spec(). +child_spec(ProfileId) -> + #{ + id => id(ProfileId), + start => {?MODULE, start_link, [ProfileId]}, + restart => permanent, + shutdown => 5000, + type => supervisor, + modules => [?MODULE] + }. + +-spec id(emqx_s3:profile_id()) -> id(). +id(ProfileId) -> + {?MODULE, ProfileId}. + +-spec start_uploader(emqx_s3:profile_id(), emqx_s3_uploader:opts()) -> + supervisor:start_ret() | {error, profile_not_found}. +start_uploader(ProfileId, Opts) -> + Id = id(ProfileId), + case gproc:where({n, l, Id}) of + undefined -> {error, profile_not_found}; + Pid -> supervisor:start_child(Pid, [Opts]) + end. + +%%-------------------------------------------------------------------- +%% supervisor callbacks +%%------------------------------------------------------------------- + +init([ProfileId]) -> + true = gproc:reg({n, l, id(ProfileId)}, ignored), + SupFlags = #{ + strategy => simple_one_for_one, + intensity => 10, + period => 5 + }, + ChildSpecs = [ + #{ + id => emqx_s3_uploader, + start => {emqx_s3_uploader, start_link, [ProfileId]}, + restart => temporary, + shutdown => 5000, + type => worker, + modules => [emqx_s3_uploader] + } + ], + {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl new file mode 100644 index 000000000..ceb0d1dd4 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -0,0 +1,143 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_schema). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-import(hoconsc, [mk/2, ref/1]). + +-export([roots/0, fields/1, namespace/0, tags/0]). + +-export([translate/1]). + +roots() -> + [s3]. + +namespace() -> "s3". + +tags() -> + [<<"S3">>]. + +fields(s3) -> + [ + {access_key_id, + mk( + string(), + #{ + desc => ?DESC("access_key_id"), + required => false + } + )}, + {secret_access_key, + mk( + string(), + #{ + desc => ?DESC("secret_access_key"), + required => false + } + )}, + {bucket, + mk( + string(), + #{ + desc => ?DESC("bucket"), + required => true + } + )}, + {host, + mk( + string(), + #{ + desc => ?DESC("host"), + required => true + } + )}, + {port, + mk( + pos_integer(), + #{ + desc => ?DESC("port"), + required => true + } + )}, + {min_part_size, + mk( + emqx_schema:bytesize(), + #{ + default => "5mb", + desc => ?DESC("min_part_size"), + required => true, + validator => fun part_size_validator/1 + } + )}, + {max_part_size, + mk( + emqx_schema:bytesize(), + #{ + default => "5gb", + desc => ?DESC("max_part_size"), + required => true, + validator => fun part_size_validator/1 + } + )}, + {acl, + mk( + hoconsc:enum([ + private, + public_read, + public_read_write, + authenticated_read, + bucket_owner_read, + bucket_owner_full_control + ]), + #{ + default => private, + desc => ?DESC("acl"), + required => true + } + )}, + {transport_options, + mk( + ref(transport_options), + #{ + desc => ?DESC("transport_options"), + required => false + } + )} + ]; +fields(transport_options) -> + props_without( + [base_url, max_retries, retry_interval, request], emqx_connector_http:fields(config) + ) ++ + props_with( + [headers, max_retries, request_timeout], emqx_connector_http:fields("request") + ). + +translate(Conf) -> + Options = #{atom_key => true}, + #{s3 := TranslatedConf} = hocon_tconf:check_plain( + emqx_s3_schema, #{<<"s3">> => Conf}, Options, [s3] + ), + TranslatedConf. + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +props_with(Keys, Proplist) -> + lists:filter(fun({K, _}) -> lists:member(K, Keys) end, Proplist). + +props_without(Keys, Proplist) -> + lists:filter(fun({K, _}) -> not lists:member(K, Keys) end, Proplist). + +part_size_validator(PartSizeLimit) -> + case + PartSizeLimit >= 5 * 1024 * 1024 andalso + PartSizeLimit =< 5 * 1024 * 1024 * 1024 + of + true -> ok; + false -> {error, "must be at least 5mb and less than 5gb"} + end. diff --git a/apps/emqx_s3/src/emqx_s3_sup.erl b/apps/emqx_s3/src/emqx_s3_sup.erl new file mode 100644 index 000000000..0f6b0160b --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_sup.erl @@ -0,0 +1,47 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_sup). + +-behaviour(supervisor). + +-include_lib("emqx/include/types.hrl"). + +-export([ + start_link/0, + start_profile/2, + stop_profile/1 +]). + +-export([init/1]). + +-spec start_link() -> supervisor:start_ret(). +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +-spec start_profile(emqx_s3:profile_id(), emqx_s3:profile_config()) -> supervisor:startchild_ret(). +start_profile(ProfileId, ProfileConfig) -> + supervisor:start_child(?MODULE, emqx_s3_profile_sup:child_spec(ProfileId, ProfileConfig)). + +-spec stop_profile(emqx_s3:profile_id()) -> ok_or_error(term()). +stop_profile(ProfileId) -> + case supervisor:terminate_child(?MODULE, ProfileId) of + ok -> + supervisor:delete_child(?MODULE, ProfileId); + {error, Reason} -> + {error, Reason} + end. + +%%-------------------------------------------------------------------- +%% supervisor callbacks +%%------------------------------------------------------------------- + +init([]) -> + ok = emqx_s3_profile_http_pools:create_table(), + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 5 + }, + {ok, {SupFlags, []}}. diff --git a/apps/emqx_s3/src/emqx_s3_uploader.erl b/apps/emqx_s3/src/emqx_s3_uploader.erl new file mode 100644 index 000000000..4e3fe15f2 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_uploader.erl @@ -0,0 +1,318 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_uploader). + +-include_lib("emqx/include/types.hrl"). + +-behaviour(gen_statem). + +-export([ + start_link/2, + + write/2, + complete/1, + abort/1 +]). + +-export([ + init/1, + callback_mode/0, + handle_event/4, + terminate/3, + code_change/4, + format_status/1, + format_status/2 +]). + +-export_type([opts/0, config/0]). + +-type opts() :: #{ + name := string() +}. + +-type config() :: #{ + min_part_size := pos_integer() +}. + +-type data() :: #{ + profile_id := emqx_s3:profile_id(), + client := emqx_s3_client:client(), + key := emqx_s3_client:key(), + buffer := iodata(), + buffer_size := non_neg_integer(), + min_part_size := pos_integer(), + max_part_size := pos_integer(), + upload_id := undefined | emqx_s3_client:upload_id(), + etags := [emqx_s3_client:etag()], + part_number := emqx_s3_client:part_number() +}. + +%% 5MB +-define(DEFAULT_MIN_PART_SIZE, 5242880). +%% 5GB +-define(DEFAULT_MAX_PART_SIZE, 5368709120). + +-spec start_link(emqx_s3:profile_id(), opts()) -> gen_statem:start_ret(). +start_link(ProfileId, #{key := Key} = Opts) when is_list(Key) -> + gen_statem:start_link(?MODULE, [ProfileId, Opts], []). + +-spec write(pid(), binary()) -> ok_or_error(term()). +write(Pid, WriteData) when is_binary(WriteData) -> + write(Pid, WriteData, infinity). + +-spec write(pid(), binary(), timeout()) -> ok_or_error(term()). +write(Pid, WriteData, Timeout) when is_binary(WriteData) -> + gen_statem:call(Pid, {write, wrap(WriteData)}, Timeout). + +-spec complete(pid()) -> ok_or_error(term()). +complete(Pid) -> + complete(Pid, infinity). + +-spec complete(pid(), timeout()) -> ok_or_error(term()). +complete(Pid, Timeout) -> + gen_statem:call(Pid, complete, Timeout). + +-spec abort(pid()) -> ok_or_error(term()). +abort(Pid) -> + abort(Pid, infinity). + +-spec abort(pid(), timeout()) -> ok_or_error(term()). +abort(Pid, Timeout) -> + gen_statem:call(Pid, abort, Timeout). + +%%-------------------------------------------------------------------- +%% gen_statem callbacks +%%-------------------------------------------------------------------- + +callback_mode() -> handle_event_function. + +init([ProfileId, #{key := Key}]) -> + process_flag(trap_exit, true), + {ok, ClientConfig, UploaderConfig} = emqx_s3_profile_conf:checkout_config(ProfileId), + Client = emqx_s3_client:create(ClientConfig), + {ok, upload_not_started, #{ + profile_id => ProfileId, + client => Client, + key => Key, + buffer => [], + buffer_size => 0, + min_part_size => maps:get(min_part_size, UploaderConfig, ?DEFAULT_MIN_PART_SIZE), + max_part_size => maps:get(max_part_size, UploaderConfig, ?DEFAULT_MAX_PART_SIZE), + upload_id => undefined, + etags => [], + part_number => 1 + }}. + +handle_event({call, From}, {write, WriteDataWrapped}, State, Data0) -> + WriteData = unwrap(WriteDataWrapped), + case is_valid_part(WriteData, Data0) of + true -> + handle_write(State, From, WriteData, Data0); + false -> + {keep_state_and_data, {reply, From, {error, {too_large, byte_size(WriteData)}}}} + end; +handle_event({call, From}, complete, upload_not_started, Data0) -> + case put_object(Data0) of + ok -> + {stop_and_reply, normal, {reply, From, ok}}; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data0} + end; +handle_event({call, From}, complete, upload_started, Data0) -> + case complete_upload(Data0) of + {ok, Data1} -> + {stop_and_reply, normal, {reply, From, ok}, Data1}; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data0} + end; +handle_event({call, From}, abort, upload_not_started, _Data) -> + {stop_and_reply, normal, {reply, From, ok}}; +handle_event({call, From}, abort, upload_started, Data0) -> + case abort_upload(Data0) of + ok -> + {stop_and_reply, normal, {reply, From, ok}}; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data0} + end. + +handle_write(upload_not_started, From, WriteData, Data0) -> + Data1 = append_buffer(Data0, WriteData), + case maybe_start_upload(Data1) of + not_started -> + {keep_state, Data1, {reply, From, ok}}; + {started, Data2} -> + case upload_part(Data2) of + {ok, Data3} -> + {next_state, upload_started, Data3, {reply, From, ok}}; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data2} + end; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data1} + end; +handle_write(upload_started, From, WriteData, Data0) -> + Data1 = append_buffer(Data0, WriteData), + case maybe_upload_part(Data1) of + {ok, Data2} -> + {keep_state, Data2, {reply, From, ok}}; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data1} + end. + +terminate(Reason, _State, #{client := Client, upload_id := UploadId, key := Key}) when + (UploadId =/= undefined) andalso (Reason =/= normal) +-> + emqx_s3_client:abort_multipart(Client, Key, UploadId); +terminate(_Reason, _State, _Data) -> + ok. + +code_change(_OldVsn, StateName, State, _Extra) -> + {ok, StateName, State}. + +format_status(#{data := #{client := Client} = Data} = Status) -> + Status#{ + data => Data#{ + client => emqx_s3_client:format(Client), + buffer => [<<"...">>] + } + }. + +format_status(_Opt, [PDict, State, #{client := Client} = Data]) -> + #{ + data => Data#{ + client => emqx_s3_client:format(Client), + buffer => [<<"...">>] + }, + state => State, + pdict => PDict + }. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +-spec maybe_start_upload(data()) -> not_started | {started, data()} | {error, term()}. +maybe_start_upload(#{buffer_size := BufferSize, min_part_size := MinPartSize} = Data) -> + case BufferSize >= MinPartSize of + true -> + start_upload(Data); + false -> + not_started + end. + +-spec start_upload(data()) -> {started, data()} | {error, term()}. +start_upload(#{client := Client, key := Key} = Data) -> + case emqx_s3_client:start_multipart(Client, Key) of + {ok, UploadId} -> + NewData = Data#{upload_id => UploadId}, + {started, NewData}; + {error, _} = Error -> + Error + end. + +-spec maybe_upload_part(data()) -> ok_or_error(data(), term()). +maybe_upload_part(#{buffer_size := BufferSize, min_part_size := MinPartSize} = Data) -> + case BufferSize >= MinPartSize of + true -> + upload_part(Data); + false -> + % ct:print("buffer size: ~p, max part size: ~p, no upload", [BufferSize, MinPartSize]), + {ok, Data} + end. + +-spec upload_part(data()) -> ok_or_error(data(), term()). +upload_part(#{buffer_size := 0} = Data) -> + {ok, Data}; +upload_part( + #{ + client := Client, + key := Key, + upload_id := UploadId, + buffer := Buffer, + part_number := PartNumber, + etags := ETags + } = Data +) -> + case emqx_s3_client:upload_part(Client, Key, UploadId, PartNumber, lists:reverse(Buffer)) of + {ok, ETag} -> + % ct:print("upload part ~p, etag: ~p", [PartNumber, ETag]), + NewData = Data#{ + buffer => [], + buffer_size => 0, + part_number => PartNumber + 1, + etags => [{PartNumber, ETag} | ETags] + }, + {ok, NewData}; + {error, _} = Error -> + % ct:print("upload part ~p failed: ~p", [PartNumber, Error]), + Error + end. + +-spec complete_upload(data()) -> ok_or_error(term()). +complete_upload( + #{ + client := Client, + key := Key, + upload_id := UploadId + } = Data0 +) -> + case upload_part(Data0) of + {ok, #{etags := ETags} = Data1} -> + case emqx_s3_client:complete_multipart(Client, Key, UploadId, lists:reverse(ETags)) of + ok -> + {ok, Data1}; + {error, _} = Error -> + Error + end; + {error, _} = Error -> + Error + end. + +-spec abort_upload(data()) -> ok_or_error(term()). +abort_upload( + #{ + client := Client, + key := Key, + upload_id := UploadId + } +) -> + case emqx_s3_client:abort_multipart(Client, Key, UploadId) of + ok -> + ok; + {error, _} = Error -> + Error + end. + +-spec put_object(data()) -> ok_or_error(term()). +put_object( + #{ + client := Client, + key := Key, + buffer := Buffer + } +) -> + case emqx_s3_client:put_object(Client, Key, lists:reverse(Buffer)) of + ok -> + ok; + {error, _} = Error -> + Error + end. + +-spec append_buffer(data(), binary()) -> data(). +append_buffer(#{buffer := Buffer, buffer_size := BufferSize} = Data, WriteData) -> + Data#{ + buffer => [WriteData | Buffer], + buffer_size => BufferSize + byte_size(WriteData) + }. + +-compile({inline, [wrap/1, unwrap/1]}). +wrap(Data) -> + fun() -> Data end. + +unwrap(WrappedData) -> + WrappedData(). + +is_valid_part(WriteData, #{max_part_size := MaxPartSize, buffer_size := BufferSize}) -> + BufferSize + byte_size(WriteData) =< MaxPartSize. diff --git a/apps/emqx_s3/test/certs/ca.crt b/apps/emqx_s3/test/certs/ca.crt new file mode 100644 index 000000000..8a9dafccd --- /dev/null +++ b/apps/emqx_s3/test/certs/ca.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE5DCCAswCCQCF3o0gIdaNDjANBgkqhkiG9w0BAQsFADA0MRIwEAYDVQQKDAlF +TVFYIFRlc3QxHjAcBgNVBAMMFUNlcnRpZmljYXRlIEF1dGhvcml0eTAeFw0yMTEy +MzAwODQxMTFaFw00OTA1MTcwODQxMTFaMDQxEjAQBgNVBAoMCUVNUVggVGVzdDEe +MBwGA1UEAwwVQ2VydGlmaWNhdGUgQXV0aG9yaXR5MIICIjANBgkqhkiG9w0BAQEF +AAOCAg8AMIICCgKCAgEAqmqSrxyH16j63QhqGLT1UO8I+m6BM3HfnJQM8laQdtJ0 +WgHqCh0/OphH3S7v4SfF4fNJDEJWMWuuzJzU9cTqHPLzhvo3+ZHcMIENgtY2p2Cf +7AQjEqFViEDyv2ZWNEe76BJeShntdY5NZr4gIPar99YGG/Ln8YekspleV+DU38rE +EX9WzhgBr02NN9z4NzIxeB+jdvPnxcXs3WpUxzfnUjOQf/T1tManvSdRbFmKMbxl +A8NLYK3oAYm8EbljWUINUNN6loqYhbigKv8bvo5S4xvRqmX86XB7sc0SApngtNcg +O0EKn8z/KVPDskE+8lMfGMiU2e2Tzw6Rph57mQPOPtIp5hPiKRik7ST9n0p6piXW +zRLplJEzSjf40I1u+VHmpXlWI/Fs8b1UkDSMiMVJf0LyWb4ziBSZOY2LtZzWHbWj +LbNgxQcwSS29tKgUwfEFmFcm+iOM59cPfkl2IgqVLh5h4zmKJJbfQKSaYb5fcKRf +50b1qsN40VbR3Pk/0lJ0/WqgF6kZCExmT1qzD5HJES/5grjjKA4zIxmHOVU86xOF +ouWvtilVR4PGkzmkFvwK5yRhBUoGH/A9BurhqOc0QCGay1kqHQFA6se4JJS+9KOS +x8Rn1Nm6Pi7sd6Le3cKmHTlyl5a/ofKqTCX2Qh+v/7y62V1V1wnoh3ipRjdPTnMC +AwEAATANBgkqhkiG9w0BAQsFAAOCAgEARCqaocvlMFUQjtFtepO2vyG1krn11xJ0 +e7md26i+g8SxCCYqQ9IqGmQBg0Im8fyNDKRN/LZoj5+A4U4XkG1yya91ZIrPpWyF +KUiRAItchNj3g1kHmI2ckl1N//6Kpx3DPaS7qXZaN3LTExf6Ph+StE1FnS0wVF+s +tsNIf6EaQ+ZewW3pjdlLeAws3jvWKUkROc408Ngvx74zbbKo/zAC4tz8oH9ZcpsT +WD8enVVEeUQKI6ItcpZ9HgTI9TFWgfZ1vYwvkoRwNIeabYI62JKmLEo2vGfGwWKr +c+GjnJ/tlVI2DpPljfWOnQ037/7yyJI/zo65+HPRmGRD6MuW/BdPDYOvOZUTcQKh +kANi5THSbJJgZcG3jb1NLebaUQ1H0zgVjn0g3KhUV+NJQYk8RQ7rHtB+MySqTKlM +kRkRjfTfR0Ykxpks7Mjvsb6NcZENf08ZFPd45+e/ptsxpiKu4e4W4bV7NZDvNKf9 +0/aD3oGYNMiP7s+KJ1lRSAjnBuG21Yk8FpzG+yr8wvJhV8aFgNQ5wIH86SuUTmN0 +5bVzFEIcUejIwvGoQEctNHBlOwHrb7zmB6OwyZeMapdXBQ+9UDhYg8ehDqdDOdfn +wsBcnjD2MwNhlE1hjL+tZWLNwSHiD6xx3LvNoXZu2HK8Cp3SOrkE69cFghYMIZZb +T+fp6tNL6LE= +-----END CERTIFICATE----- diff --git a/apps/emqx_s3/test/emqx_s3_SUITE.erl b/apps/emqx_s3/test/emqx_s3_SUITE.erl new file mode 100644 index 000000000..287dcb597 --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_SUITE.erl @@ -0,0 +1,66 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(emqx_s3), + Config. + +end_per_suite(_Config) -> + ok = application:stop(emqx_s3). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_start_stop_update(_Config) -> + ProfileId = <<"test">>, + ProfileConfig = profile_config(), + + ?assertMatch( + ok, + emqx_s3:start_profile(ProfileId, ProfileConfig) + ), + + ?assertMatch( + {error, _}, + emqx_s3:start_profile(ProfileId, ProfileConfig) + ), + + ?assertEqual( + ok, + emqx_s3:update_profile(ProfileId, ProfileConfig) + ), + + ?assertMatch( + {error, _}, + emqx_s3:update_profile(<<"unknown">>, ProfileConfig) + ), + + ?assertEqual( + ok, + emqx_s3:stop_profile(ProfileId) + ), + + ?assertMatch( + {error, _}, + emqx_s3:stop_profile(ProfileId) + ). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +profile_config() -> + emqx_s3_test_helpers:base_config(tcp). diff --git a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl new file mode 100644 index 000000000..3d0d7bb18 --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl @@ -0,0 +1,104 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_client_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(PROFILE_ID, atom_to_binary(?MODULE)). + +all() -> + [ + {group, tcp}, + {group, tls} + ]. + +groups() -> + AllCases = emqx_common_test_helpers:all(?MODULE), + [ + {tcp, [], AllCases}, + {tls, [], AllCases} + ]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(emqx_s3), + Config. + +end_per_suite(_Config) -> + ok = application:stop(emqx_s3). + +init_per_group(ConnType, Config) -> + [{conn_type, ConnType} | Config]. +end_per_group(_ConnType, _Config) -> + ok. + +init_per_testcase(_TestCase, Config0) -> + ConnType = ?config(conn_type, Config0), + + Bucket = emqx_s3_test_helpers:unique_bucket(), + TestAwsConfig = emqx_s3_test_helpers:aws_config(ConnType), + ok = erlcloud_s3:create_bucket(Bucket, TestAwsConfig), + Config1 = [ + {key, emqx_s3_test_helpers:unique_key()}, + {bucket, Bucket} + | Config0 + ], + {ok, PoolName} = emqx_s3_profile_conf:start_http_pool(?PROFILE_ID, profile_config(Config1)), + [{ehttpc_pool_name, PoolName} | Config1]. + +end_per_testcase(_TestCase, Config) -> + ok = ehttpc_sup:stop_pool(?config(ehttpc_pool_name, Config)). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_multipart_upload(Config) -> + Key = ?config(key, Config), + + Client = client(Config), + + {ok, UploadId} = emqx_s3_client:start_multipart(Client, Key), + + Data = data(6_000_000), + + {ok, Etag1} = emqx_s3_client:upload_part(Client, Key, UploadId, 1, Data), + {ok, Etag2} = emqx_s3_client:upload_part(Client, Key, UploadId, 2, Data), + + ok = emqx_s3_client:complete_multipart( + Client, Key, UploadId, [{1, Etag1}, {2, Etag2}] + ). + +t_simple_put(Config) -> + Key = ?config(key, Config), + + Client = client(Config), + + Data = data(6_000_000), + + ok = emqx_s3_client:put_object(Client, Key, Data). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +client(Config) -> + ClientConfig = emqx_s3_profile_conf:client_config( + profile_config(Config), ?config(ehttpc_pool_name, Config) + ), + emqx_s3_client:create(ClientConfig). + +profile_config(Config) -> + maps:put( + bucket, + ?config(bucket, Config), + emqx_s3_test_helpers:base_config(?config(conn_type, Config)) + ). + +data(Size) -> + iolist_to_binary([$a || _ <- lists:seq(1, Size)]). diff --git a/apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl b/apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl new file mode 100644 index 000000000..ce53525be --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl @@ -0,0 +1,293 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_conf_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-define(assertWaitEvent(Code, EventMatch, Timeout), + ?assertMatch( + {_, {ok, EventMatch}}, + ?wait_async_action( + Code, + EventMatch, + Timeout + ) + ) +). + +all() -> emqx_common_test_helpers:all(?MODULE). + +suite() -> [{timetrap, {minutes, 1}}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(emqx_s3), + Config. + +end_per_suite(_Config) -> + ok = application:stop(emqx_s3). + +init_per_testcase(_TestCase, Config) -> + ok = snabbkaffe:start_trace(), + TestAwsConfig = emqx_s3_test_helpers:aws_config(tcp), + + Bucket = emqx_s3_test_helpers:unique_bucket(), + ok = erlcloud_s3:create_bucket(Bucket, TestAwsConfig), + + ProfileBaseConfig = emqx_s3_test_helpers:base_config(tcp), + ProfileConfig = ProfileBaseConfig#{bucket => Bucket}, + ok = emqx_s3:start_profile(profile_id(), ProfileConfig), + + [{profile_config, ProfileConfig} | Config]. + +end_per_testcase(_TestCase, _Config) -> + ok = snabbkaffe:stop(), + _ = emqx_s3:stop_profile(profile_id()). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_regular_outdated_pool_cleanup(Config) -> + _ = process_flag(trap_exit, true), + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + [OldPool] = emqx_s3_profile_http_pools:all(profile_id()), + + ProfileBaseConfig = ?config(profile_config, Config), + ProfileConfig = emqx_map_lib:deep_put( + [transport_options, pool_size], ProfileBaseConfig, 16 + ), + ok = emqx_s3:update_profile(profile_id(), ProfileConfig), + + ?assertEqual( + 2, + length(emqx_s3_profile_http_pools:all(profile_id())) + ), + + ?assertWaitEvent( + ok = emqx_s3_uploader:abort(Pid), + #{?snk_kind := "s3_stop_http_pool", pool_name := OldPool}, + 1000 + ), + + [NewPool] = emqx_s3_profile_http_pools:all(profile_id()), + + ?assertWaitEvent( + ok = emqx_s3:stop_profile(profile_id()), + #{?snk_kind := "s3_stop_http_pool", pool_name := NewPool}, + 1000 + ), + + ?assertEqual( + 0, + length(emqx_s3_profile_http_pools:all(profile_id())) + ). + +t_timeout_pool_cleanup(Config) -> + _ = process_flag(trap_exit, true), + + %% We restart the profile to set `http_pool_timeout` value suitable for test + ok = emqx_s3:stop_profile(profile_id()), + ProfileBaseConfig = ?config(profile_config, Config), + ProfileConfig = ProfileBaseConfig#{ + http_pool_timeout => 500, + http_pool_cleanup_interval => 100 + }, + ok = emqx_s3:start_profile(profile_id(), ProfileConfig), + + %% Start uploader + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + ok = emqx_s3_uploader:write(Pid, <<"data">>), + + [OldPool] = emqx_s3_profile_http_pools:all(profile_id()), + + NewProfileConfig = emqx_map_lib:deep_put( + [transport_options, pool_size], ProfileConfig, 16 + ), + + %% We update profile to create new pool and wait for the old one to be stopped by timeout + ?assertWaitEvent( + ok = emqx_s3:update_profile(profile_id(), NewProfileConfig), + #{?snk_kind := "s3_stop_http_pool", pool_name := OldPool}, + 1000 + ), + + %% The uploader now has no valid pool and should fail + ?assertMatch( + {error, _}, + emqx_s3_uploader:complete(Pid) + ). + +t_checkout_no_profile(_Config) -> + ?assertEqual( + {error, profile_not_found}, + emqx_s3_profile_conf:checkout_config(<<"no_such_profile">>) + ). + +t_httpc_pool_start_error(Config) -> + %% `ehhtpc_pool`s are lazy so it is difficult to trigger an error + %% passing some bad connection options. + %% So we emulate some unknown crash with `meck`. + meck:new(ehttpc_pool, [passthrough]), + meck:expect(ehttpc_pool, init, fun(_) -> meck:raise(error, badarg) end), + + ?assertMatch( + {error, _}, + emqx_s3:start_profile(<<"profile">>, ?config(profile_config, Config)) + ). + +t_httpc_pool_update_error(Config) -> + %% `ehhtpc_pool`s are lazy so it is difficult to trigger an error + %% passing some bad connection options. + %% So we emulate some unknown crash with `meck`. + meck:new(ehttpc_pool, [passthrough]), + meck:expect(ehttpc_pool, init, fun(_) -> meck:raise(error, badarg) end), + + ProfileBaseConfig = ?config(profile_config, Config), + NewProfileConfig = emqx_map_lib:deep_put( + [transport_options, pool_size], ProfileBaseConfig, 16 + ), + + ?assertMatch( + {error, _}, + emqx_s3:start_profile(<<"profile">>, NewProfileConfig) + ). + +t_orphaned_pools_cleanup(_Config) -> + ProfileId = profile_id(), + Pid = gproc:where({n, l, emqx_s3_profile_conf:id(ProfileId)}), + + %% We kill conf and wait for it to restart + %% and create a new pool + ?assertWaitEvent( + exit(Pid, kill), + #{?snk_kind := "s3_start_http_pool", profile_id := ProfileId}, + 1000 + ), + + %% We should still have only one pool + ?assertEqual( + 1, + length(emqx_s3_profile_http_pools:all(ProfileId)) + ). + +t_orphaned_pools_cleanup_non_graceful(_Config) -> + ProfileId = profile_id(), + Pid = gproc:where({n, l, emqx_s3_profile_conf:id(ProfileId)}), + + %% We stop pool, conf server should not fail when attempting to stop it once more + [PoolName] = emqx_s3_profile_http_pools:all(ProfileId), + ok = ehttpc_pool:stop_pool(PoolName), + + %% We kill conf and wait for it to restart + %% and create a new pool + ?assertWaitEvent( + exit(Pid, kill), + #{?snk_kind := "s3_start_http_pool", profile_id := ProfileId}, + 1000 + ), + + %% We should still have only one pool + ?assertEqual( + 1, + length(emqx_s3_profile_http_pools:all(ProfileId)) + ). + +t_checkout_client(Config) -> + ProfileId = profile_id(), + Key = emqx_s3_test_helpers:unique_key(), + Caller = self(), + Pid = spawn_link(fun() -> + emqx_s3:with_client( + ProfileId, + fun(Client) -> + receive + put_object -> + Caller ! {put_object, emqx_s3_client:put_object(Client, Key, <<"data">>)} + end, + receive + list_objects -> + Caller ! {list_objects, emqx_s3_client:list(Client, [])} + end + end + ), + Caller ! client_released, + receive + stop -> ok + end + end), + + %% Ask spawned process to put object + Pid ! put_object, + receive + {put_object, ok} -> ok + after 1000 -> + ct:fail("put_object fail") + end, + + %% Now change config for the profile + ProfileBaseConfig = ?config(profile_config, Config), + NewProfileConfig0 = ProfileBaseConfig#{bucket => <<"new_bucket">>}, + NewProfileConfig1 = emqx_map_lib:deep_put( + [transport_options, pool_size], NewProfileConfig0, 16 + ), + ok = emqx_s3:update_profile(profile_id(), NewProfileConfig1), + + %% We should have two pools now, because the old one is still in use + %% by the spawned process + ?assertEqual( + 2, + length(emqx_s3_profile_http_pools:all(ProfileId)) + ), + + %% Ask spawned process to list objects + Pid ! list_objects, + receive + {list_objects, Result} -> + {ok, OkResult} = Result, + Contents = proplists:get_value(contents, OkResult), + ?assertEqual(1, length(Contents)), + ?assertEqual(Key, proplists:get_value(key, hd(Contents))) + after 1000 -> + ct:fail("list_objects fail") + end, + + %% Wait till spawned process releases client + receive + client_released -> ok + after 1000 -> + ct:fail("client not released") + end, + + %% We should have only one pool now, because the old one is released + ?assertEqual( + 1, + length(emqx_s3_profile_http_pools:all(ProfileId)) + ). + +t_unknown_messages(_Config) -> + Pid = gproc:where({n, l, emqx_s3_profile_conf:id(profile_id())}), + + Pid ! unknown, + ok = gen_server:cast(Pid, unknown), + + ?assertEqual( + {error, not_implemented}, + gen_server:call(Pid, unknown) + ). + +%%-------------------------------------------------------------------- +%% Test helpers +%%-------------------------------------------------------------------- + +profile_id() -> + <<"test">>. diff --git a/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl b/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl new file mode 100644 index 000000000..bba1a5ba8 --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl @@ -0,0 +1,154 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_schema_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_minimal_config(_Config) -> + ?assertMatch( + #{ + bucket := "bucket", + host := "s3.us-east-1.endpoint.com", + port := 443, + acl := private, + min_part_size := 5242880, + transport_options := + #{ + connect_timeout := 15000, + enable_pipelining := 100, + pool_size := 8, + pool_type := random, + ssl := #{enable := false} + } + }, + emqx_s3_schema:translate(#{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443 + }) + ). + +t_full_config(_Config) -> + ?assertMatch( + #{ + access_key_id := "access_key_id", + acl := public_read, + bucket := "bucket", + host := "s3.us-east-1.endpoint.com", + min_part_size := 10485760, + port := 443, + secret_access_key := "secret_access_key", + transport_options := + #{ + connect_timeout := 30000, + enable_pipelining := 200, + headers := #{<<"x-amz-acl">> := <<"public-read">>}, + max_retries := 3, + pool_size := 10, + pool_type := random, + request_timeout := 10000, + ssl := + #{ + cacertfile := <<"cacertfile.crt">>, + certfile := <<"server.crt">>, + ciphers := ["ECDHE-RSA-AES256-GCM-SHA384"], + depth := 10, + enable := true, + keyfile := <<"server.key">>, + reuse_sessions := true, + secure_renegotiate := true, + server_name_indication := "some-host", + verify := verify_peer, + versions := ['tlsv1.2'] + } + } + }, + emqx_s3_schema:translate(#{ + <<"access_key_id">> => <<"access_key_id">>, + <<"secret_access_key">> => <<"secret_access_key">>, + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"min_part_size">> => <<"10mb">>, + <<"acl">> => <<"public_read">>, + <<"transport_options">> => #{ + <<"connect_timeout">> => 30000, + <<"enable_pipelining">> => 200, + <<"pool_size">> => 10, + <<"pool_type">> => <<"random">>, + <<"ssl">> => #{ + <<"enable">> => true, + <<"keyfile">> => <<"server.key">>, + <<"certfile">> => <<"server.crt">>, + <<"cacertfile">> => <<"cacertfile.crt">>, + <<"server_name_indication">> => <<"some-host">>, + <<"verify">> => <<"verify_peer">>, + <<"versions">> => [<<"tlsv1.2">>], + <<"ciphers">> => [<<"ECDHE-RSA-AES256-GCM-SHA384">>] + }, + <<"request_timeout">> => <<"10s">>, + <<"max_retries">> => 3, + <<"headers">> => #{ + <<"x-amz-acl">> => <<"public-read">> + } + } + }) + ). + +t_invalid_limits(_Config) -> + ?assertException( + throw, + {emqx_s3_schema, [#{kind := validation_error, path := "s3.min_part_size"}]}, + emqx_s3_schema:translate(#{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"min_part_size">> => <<"1mb">> + }) + ), + + ?assertException( + throw, + {emqx_s3_schema, [#{kind := validation_error, path := "s3.min_part_size"}]}, + emqx_s3_schema:translate(#{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"min_part_size">> => <<"100000gb">> + }) + ), + + ?assertException( + throw, + {emqx_s3_schema, [#{kind := validation_error, path := "s3.max_part_size"}]}, + emqx_s3_schema:translate(#{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"max_part_size">> => <<"1mb">> + }) + ), + + ?assertException( + throw, + {emqx_s3_schema, [#{kind := validation_error, path := "s3.max_part_size"}]}, + emqx_s3_schema:translate(#{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"max_part_size">> => <<"100000gb">> + }) + ). diff --git a/apps/emqx_s3/test/emqx_s3_test_helpers.erl b/apps/emqx_s3/test/emqx_s3_test_helpers.erl new file mode 100644 index 000000000..c74e78a4d --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_test_helpers.erl @@ -0,0 +1,135 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_test_helpers). + +-compile(nowarn_export_all). +-compile(export_all). + +-define(ACCESS_KEY_ID, "minioadmin"). +-define(SECRET_ACCESS_KEY, "minioadmin"). + +-define(TOXIPROXY_HOST, "toxiproxy"). +-define(TOXIPROXY_PORT, 8474). + +-define(TCP_HOST, ?TOXIPROXY_HOST). +-define(TCP_PORT, 19000). +-define(TLS_HOST, ?TOXIPROXY_HOST). +-define(TLS_PORT, 19100). + +-include_lib("erlcloud/include/erlcloud_aws.hrl"). + +-export([ + aws_config/1, + base_raw_config/1, + base_config/1, + + unique_key/0, + unique_bucket/0, + + with_failure/3 +]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +aws_config(tcp) -> + erlcloud_s3_new( + ?ACCESS_KEY_ID, + ?SECRET_ACCESS_KEY, + ?TCP_HOST, + ?TCP_PORT, + "http://" + ); +aws_config(tls) -> + erlcloud_s3_new( + ?ACCESS_KEY_ID, + ?SECRET_ACCESS_KEY, + ?TLS_HOST, + ?TLS_PORT, + "https://" + ). + +base_raw_config(tcp) -> + #{ + <<"bucket">> => <<"bucket">>, + <<"access_key_id">> => bin(?ACCESS_KEY_ID), + <<"secret_access_key">> => bin(?SECRET_ACCESS_KEY), + <<"host">> => ?TCP_HOST, + <<"port">> => ?TCP_PORT, + <<"max_part_size">> => 10 * 1024 * 1024, + <<"transport_options">> => + #{ + <<"request_timeout">> => 2000 + } + }; +base_raw_config(tls) -> + #{ + <<"bucket">> => <<"bucket">>, + <<"access_key_id">> => bin(?ACCESS_KEY_ID), + <<"secret_access_key">> => bin(?SECRET_ACCESS_KEY), + <<"host">> => ?TLS_HOST, + <<"port">> => ?TLS_PORT, + <<"max_part_size">> => 10 * 1024 * 1024, + <<"transport_options">> => + #{ + <<"request_timeout">> => 2000, + <<"ssl">> => #{ + <<"enable">> => true, + <<"cacertfile">> => bin(cert_path("ca.crt")), + <<"server_name_indication">> => <<"authn-server">>, + <<"verify">> => <<"verify_peer">> + } + } + }. + +base_config(ConnType) -> + emqx_s3_schema:translate(base_raw_config(ConnType)). + +unique_key() -> + "key-" ++ integer_to_list(erlang:system_time(millisecond)) ++ "-" ++ + integer_to_list(erlang:unique_integer([positive])). + +unique_bucket() -> + "bucket-" ++ integer_to_list(erlang:system_time(millisecond)) ++ "-" ++ + integer_to_list(erlang:unique_integer([positive])). + +with_failure(_ConnType, ehttpc_500, Fun) -> + try + meck:new(ehttpc, [passthrough, no_history]), + meck:expect(ehttpc, request, fun(_, _, _, _) -> {ok, 500, []} end), + Fun() + after + meck:unload(ehttpc) + end; +with_failure(ConnType, FailureType, Fun) -> + emqx_common_test_helpers:with_failure( + FailureType, + toxproxy_name(ConnType), + ?TOXIPROXY_HOST, + ?TOXIPROXY_PORT, + Fun + ). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +toxproxy_name(tcp) -> "minio_tcp"; +toxproxy_name(tls) -> "minio_tls". + +cert_path(FileName) -> + Dir = code:lib_dir(emqx_s3, test), + filename:join([Dir, <<"certs">>, FileName]). + +bin(String) when is_list(String) -> list_to_binary(String); +bin(Binary) when is_binary(Binary) -> Binary. + +erlcloud_s3_new(AccessKeyId, SecretAccessKey, Host, Port, Scheme) -> + AwsConfig = erlcloud_s3:new(AccessKeyId, SecretAccessKey, Host, Port), + AwsConfig#aws_config{ + s3_scheme = Scheme, + s3_bucket_access_method = path + }. diff --git a/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl new file mode 100644 index 000000000..ef1d916c6 --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl @@ -0,0 +1,535 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_uploader_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(assertProcessExited(Reason, Pid), + receive + {'DOWN', _, _, Pid, Reason} -> + % ct:print("uploader process exited with reason: ~p", [R]), + ok + after 3000 -> + ct:fail("uploader process did not exit") + end +). + +-define(assertObjectEqual(Value, AwsConfig, Bucket, Key), + ?assertEqual( + Value, + proplists:get_value( + content, + erlcloud_s3:get_object( + Bucket, + Key, + AwsConfig + ) + ) + ) +). + +all() -> + [ + {group, tcp}, + {group, tls} + ]. + +groups() -> + [ + {tcp, [ + {group, common_cases}, + {group, tcp_cases} + ]}, + {tls, [ + {group, common_cases}, + {group, tls_cases} + ]}, + {common_cases, [], [ + t_happy_path_simple_put, + t_happy_path_multi, + t_abort_multi, + t_abort_simple_put, + + {group, noconn_errors}, + {group, timeout_errors}, + {group, http_errors} + ]}, + + {tcp_cases, [ + t_config_switch, + t_config_switch_http_settings, + t_too_large, + t_no_profile + ]}, + + {tls_cases, [ + t_tls_error + ]}, + + {noconn_errors, [{group, transport_errors}]}, + {timeout_errors, [{group, transport_errors}]}, + {http_errors, [{group, transport_errors}]}, + + {transport_errors, [ + t_start_multipart_error, + t_upload_part_error, + t_complete_multipart_error, + t_abort_multipart_error, + t_put_object_error + ]} + ]. + +suite() -> [{timetrap, {minutes, 1}}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(emqx_s3), + Config. + +end_per_suite(_Config) -> + ok = application:stop(emqx_s3). + +init_per_group(Group, Config) when Group =:= tcp orelse Group =:= tls -> + [{conn_type, Group} | Config]; +init_per_group(noconn_errors, Config) -> + [{failure, down} | Config]; +init_per_group(timeout_errors, Config) -> + [{failure, timeout} | Config]; +init_per_group(http_errors, Config) -> + [{failure, ehttpc_500} | Config]; +init_per_group(_ConnType, Config) -> + Config. + +end_per_group(_ConnType, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + ok = snabbkaffe:start_trace(), + ConnType = ?config(conn_type, Config), + TestAwsConfig = emqx_s3_test_helpers:aws_config(ConnType), + + Bucket = emqx_s3_test_helpers:unique_bucket(), + ok = erlcloud_s3:create_bucket(Bucket, TestAwsConfig), + + ProfileBaseConfig = emqx_s3_test_helpers:base_config(ConnType), + ProfileConfig = ProfileBaseConfig#{bucket => Bucket}, + ok = emqx_s3:start_profile(profile_id(), ProfileConfig), + + [{bucket, Bucket}, {test_aws_config, TestAwsConfig}, {profile_config, ProfileConfig} | Config]. + +end_per_testcase(_TestCase, _Config) -> + ok = snabbkaffe:stop(), + _ = emqx_s3:stop_profile(profile_id()). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_happy_path_simple_put(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + Data = data($a, 1024, 10), + + lists:foreach( + fun(Chunk) -> + ?assertEqual( + ok, + emqx_s3_uploader:write(Pid, Chunk) + ) + end, + Data + ), + + ok = emqx_s3_uploader:complete(Pid), + + ?assertProcessExited( + normal, + Pid + ), + + ?assertObjectEqual( + iolist_to_binary(Data), + ?config(test_aws_config, Config), + ?config(bucket, Config), + Key + ). + +t_happy_path_multi(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + Data = data($a, 1024 * 1024, 10), + + lists:foreach( + fun(Chunk) -> + ?assertEqual( + ok, + emqx_s3_uploader:write(Pid, Chunk) + ) + end, + Data + ), + + ok = emqx_s3_uploader:complete(Pid), + + ?assertProcessExited( + normal, + Pid + ), + + ?assertObjectEqual( + iolist_to_binary(Data), + ?config(test_aws_config, Config), + ?config(bucket, Config), + Key + ). + +t_abort_multi(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 6 * 1024 * 1024, 1), + + ok = emqx_s3_uploader:write(Pid, Data), + + ?assertMatch( + [], + list_objects(Config) + ), + + ok = emqx_s3_uploader:abort(Pid), + + ?assertMatch( + [], + list_objects(Config) + ), + + ?assertProcessExited( + normal, + Pid + ). + +t_abort_simple_put(_Config) -> + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 10 * 1024, 1), + + ok = emqx_s3_uploader:write(Pid, Data), + + ok = emqx_s3_uploader:abort(Pid), + + ?assertProcessExited( + normal, + Pid + ). + +t_config_switch(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + OldBucket = ?config(bucket, Config), + {ok, Pid0} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + [Data0, Data1] = data($a, 6 * 1024 * 1024, 2), + + ok = emqx_s3_uploader:write(Pid0, Data0), + + %% Switch to the new config, but without changing HTTP settings + ProfileConfig = ?config(profile_config, Config), + NewBucket = emqx_s3_test_helpers:unique_bucket(), + ok = erlcloud_s3:create_bucket(NewBucket, ?config(test_aws_config, Config)), + NewProfileConfig = ProfileConfig#{bucket => NewBucket}, + + ok = emqx_s3:update_profile(profile_id(), NewProfileConfig), + + %% Already started uploader should be OK and use previous config + ok = emqx_s3_uploader:write(Pid0, Data1), + ok = emqx_s3_uploader:complete(Pid0), + + ?assertObjectEqual( + iolist_to_binary([Data0, Data1]), + ?config(test_aws_config, Config), + OldBucket, + Key + ), + + %% Now check that new uploader uses new config + {ok, Pid1} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + ok = emqx_s3_uploader:write(Pid1, Data0), + ok = emqx_s3_uploader:complete(Pid1), + + ?assertObjectEqual( + iolist_to_binary(Data0), + ?config(test_aws_config, Config), + NewBucket, + Key + ). + +t_config_switch_http_settings(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + OldBucket = ?config(bucket, Config), + {ok, Pid0} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + [Data0, Data1] = data($a, 6 * 1024 * 1024, 2), + + ok = emqx_s3_uploader:write(Pid0, Data0), + + %% Switch to the new config, completely changing HTTP settings (tcp -> tls) + NewBucket = emqx_s3_test_helpers:unique_bucket(), + NewTestAwsConfig = emqx_s3_test_helpers:aws_config(tls), + ok = erlcloud_s3:create_bucket(NewBucket, NewTestAwsConfig), + NewProfileConfig0 = emqx_s3_test_helpers:base_config(tls), + NewProfileConfig1 = NewProfileConfig0#{bucket => NewBucket}, + + ok = emqx_s3:update_profile(profile_id(), NewProfileConfig1), + + %% Already started uploader should be OK and use previous config + ok = emqx_s3_uploader:write(Pid0, Data1), + ok = emqx_s3_uploader:complete(Pid0), + + ?assertObjectEqual( + iolist_to_binary([Data0, Data1]), + ?config(test_aws_config, Config), + OldBucket, + Key + ), + + %% Now check that new uploader uses new config + {ok, Pid1} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + ok = emqx_s3_uploader:write(Pid1, Data0), + ok = emqx_s3_uploader:complete(Pid1), + + ?assertObjectEqual( + iolist_to_binary(Data0), + NewTestAwsConfig, + NewBucket, + Key + ). + +t_start_multipart_error(Config) -> + _ = process_flag(trap_exit, true), + + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 6 * 1024 * 1024, 1), + + emqx_s3_test_helpers:with_failure( + ?config(conn_type, Config), + ?config(failure, Config), + fun() -> + ?assertMatch( + {error, _}, + emqx_s3_uploader:write(Pid, Data) + ) + end + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_upload_part_error(Config) -> + _ = process_flag(trap_exit, true), + + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data0, Data1] = data($a, 6 * 1024 * 1024, 2), + + ok = emqx_s3_uploader:write(Pid, Data0), + + emqx_s3_test_helpers:with_failure( + ?config(conn_type, Config), + ?config(failure, Config), + fun() -> + ?assertMatch( + {error, _}, + emqx_s3_uploader:write(Pid, Data1) + ) + end + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_abort_multipart_error(Config) -> + _ = process_flag(trap_exit, true), + + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 6 * 1024 * 1024, 1), + + ok = emqx_s3_uploader:write(Pid, Data), + + emqx_s3_test_helpers:with_failure( + ?config(conn_type, Config), + ?config(failure, Config), + fun() -> + ?assertMatch( + {error, _}, + emqx_s3_uploader:abort(Pid) + ) + end + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_complete_multipart_error(Config) -> + _ = process_flag(trap_exit, true), + + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 6 * 1024 * 1024, 1), + + ok = emqx_s3_uploader:write(Pid, Data), + + emqx_s3_test_helpers:with_failure( + ?config(conn_type, Config), + ?config(failure, Config), + fun() -> + ?assertMatch( + {error, _}, + emqx_s3_uploader:complete(Pid) + ) + end + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_put_object_error(Config) -> + _ = process_flag(trap_exit, true), + + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + %% Little data to avoid multipart upload + [Data] = data($a, 1024, 1), + + emqx_s3_test_helpers:with_failure( + ?config(conn_type, Config), + ?config(failure, Config), + fun() -> + ok = emqx_s3_uploader:write(Pid, Data), + ?assertMatch( + {error, _}, + emqx_s3_uploader:complete(Pid) + ) + end + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_too_large(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 1024, 1), + + [DataLarge] = data($a, 20 * 1024 * 1024, 1), + + ?assertMatch( + {error, {too_large, _}}, + emqx_s3_uploader:write(Pid, DataLarge) + ), + + ok = emqx_s3_uploader:write(Pid, Data), + ok = emqx_s3_uploader:complete(Pid), + + ?assertProcessExited( + normal, + Pid + ), + + ?assertObjectEqual( + iolist_to_binary(Data), + ?config(test_aws_config, Config), + ?config(bucket, Config), + Key + ). + +t_tls_error(Config) -> + _ = process_flag(trap_exit, true), + + ProfileBaseConfig = ?config(profile_config, Config), + ProfileConfig = emqx_map_lib:deep_put( + [transport_options, ssl, server_name_indication], ProfileBaseConfig, "invalid-hostname" + ), + ok = emqx_s3:update_profile(profile_id(), ProfileConfig), + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 6 * 1024 * 1024, 1), + + ?assertMatch( + {error, _}, + emqx_s3_uploader:write(Pid, Data) + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_no_profile(_Config) -> + Key = emqx_s3_test_helpers:unique_key(), + ?assertMatch( + {error, profile_not_found}, + emqx_s3:start_uploader(<<"no-profile">>, #{key => Key}) + ). + +%%-------------------------------------------------------------------- +%% Test helpers +%%-------------------------------------------------------------------- + +profile_id() -> + <<"test">>. + +data(Byte, ChunkSize, ChunkCount) -> + Chunk = iolist_to_binary([Byte || _ <- lists:seq(1, ChunkSize)]), + [Chunk || _ <- lists:seq(1, ChunkCount)]. + +list_objects(Config) -> + Props = erlcloud_s3:list_objects(?config(bucket, Config), [], ?config(test_aws_config, Config)), + proplists:get_value(contents, Props). diff --git a/rebar.config.erl b/rebar.config.erl index ea0016ca9..51a6946dc 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -401,7 +401,8 @@ relx_apps(ReleaseType, Edition) -> emqx_psk, emqx_slow_subs, emqx_plugins, - emqx_ft + emqx_ft, + emqx_s3 ] ++ [quicer || is_quicer_supported()] ++ [bcrypt || provide_bcrypt_release(ReleaseType)] ++ diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 82823720d..9644ec8b9 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -91,6 +91,12 @@ if [ "${WHICH_APP}" = 'novalue' ]; then exit 1 fi +if [ ! -d "${WHICH_APP}" ]; then + echo "must provide an existing path for --app arg" + help + exit 1 +fi + if [[ "${WHICH_APP}" == lib-ee* && (-z "${PROFILE+x}" || "${PROFILE}" != emqx-enterprise) ]]; then echo 'You are trying to run an enterprise test case without the emqx-enterprise profile.' echo 'This will most likely not work.' @@ -172,10 +178,14 @@ for dep in ${CT_DEPS}; do ;; rocketmq) FILES+=( '.ci/docker-compose-file/docker-compose-rocketmq.yaml' ) - ;; + ;; cassandra) FILES+=( '.ci/docker-compose-file/docker-compose-cassandra.yaml' ) ;; + minio) + FILES+=( '.ci/docker-compose-file/docker-compose-minio-tcp.yaml' + '.ci/docker-compose-file/docker-compose-minio-tls.yaml' ) + ;; *) echo "unknown_ct_dependency $dep" exit 1 From 818a5cacf298e1b4b981618effee40c298f762ac Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 30 Mar 2023 22:05:21 +0300 Subject: [PATCH 086/197] feat(ft-s3): add initial integration --- apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 13 +- apps/emqx_ft/src/emqx_ft.erl | 2 +- apps/emqx_ft/src/emqx_ft_assembler.erl | 21 +- apps/emqx_ft/src/emqx_ft_schema.erl | 186 +++++++++++------- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 47 +++-- .../src/emqx_ft_storage_exporter_fs.erl | 12 +- .../src/emqx_ft_storage_exporter_s3.erl | 69 +++++++ apps/emqx_ft/src/emqx_ft_storage_fs.erl | 16 +- apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf | 76 +++++++ apps/emqx_s3/src/emqx_s3.erl | 28 ++- apps/emqx_s3/src/emqx_s3_schema.erl | 4 +- 11 files changed, 362 insertions(+), 112 deletions(-) create mode 100644 apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl create mode 100644 apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf index 941611424..c33ea447e 100644 --- a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -58,7 +58,7 @@ emqx_ft_schema { local_storage_exporter_type { desc { - en: "Type of the Exporter to use." + en: "Exporter type for the exporter to the local file system" zh: "" } label: { @@ -67,6 +67,17 @@ emqx_ft_schema { } } + s3_exporter_type { + desc { + en: "Exporter type for the exporter to S3" + zh: "" + } + label: { + en: "S3 Exporter Type" + zh: "" + } + } + local_storage_exporter_root { desc { en: "File system path to keep uploaded files." diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index b7c8f0eac..e3f6cbe1c 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -336,7 +336,7 @@ transfer(Msg, FileId) -> {clientid_to_binary(ClientId), FileId}. on_complete(Op, {ChanPid, PacketId}, Transfer, Result) -> - ?SLOG(debug, #{ + ?SLOG(warning, #{ msg => "on_complete", operation => Op, packet_id => PacketId, diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 17fed012a..3a352cd10 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -136,13 +136,12 @@ handle_event( ) -> Filemeta = emqx_ft_assembly:filemeta(Asm), Coverage = emqx_ft_assembly:coverage(Asm), - % TODO: better error handling - {ok, Export} = emqx_ft_storage_exporter:start_export( - Storage, - Transfer, - Filemeta - ), - {next_state, {assemble, Coverage}, St#{export => Export}, ?internal([])}; + case emqx_ft_storage_exporter:start_export(Storage, Transfer, Filemeta) of + {ok, Export} -> + {next_state, {assemble, Coverage}, St#{export => Export}, ?internal([])}; + {error, _} = Error -> + {stop, {shutdown, Error}} + end; handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #{export := Export}) -> % TODO % Currently, race is possible between getting segment info from the remote node and @@ -150,8 +149,12 @@ handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #{export := % TODO: pipelining % TODO: better error handling {ok, Content} = pread(Node, Segment, St), - {ok, NExport} = emqx_ft_storage_exporter:write(Export, Content), - {next_state, {assemble, Rest}, St#{export := NExport}, ?internal([])}; + case emqx_ft_storage_exporter:write(Export, Content) of + {ok, NExport} -> + {next_state, {assemble, Rest}, St#{export := NExport}, ?internal([])}; + {error, _} = Error -> + {stop, {shutdown, Error}, maps:remove(export, St)} + end; handle_event(internal, _, {assemble, []}, St = #{}) -> {next_state, complete, St, ?internal([])}; handle_event(internal, _, complete, St = #{export := Export}) -> diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 2abbe4c45..df28de218 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -16,7 +16,7 @@ -module(emqx_ft_schema). --behaviour(hocon_schema). +% -behaviour(hocon_schema). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). @@ -35,7 +35,7 @@ -reflect_type([json_value/0]). -%% +-import(hoconsc, [ref/1, ref/2, mk/2]). namespace() -> file_transfer. @@ -46,84 +46,130 @@ roots() -> [file_transfer]. fields(file_transfer) -> [ - {storage, #{ - type => hoconsc:union([ - hoconsc:ref(?MODULE, local_storage) - ]), - desc => ?DESC("storage") - }} + {storage, + mk( + hoconsc:union([ + ref(local_storage) + ]), + #{ + required => true, + desc => ?DESC("storage") + } + )} ]; fields(local_storage) -> [ - {type, #{ - type => local, - default => local, - required => false, - desc => ?DESC("local_type") - }}, - {segments, #{ - type => ?REF(local_storage_segments), - desc => ?DESC("local_storage_segments"), - required => false - }}, - {exporter, #{ - type => hoconsc:union([ - ?REF(local_storage_exporter) - ]), - desc => ?DESC("local_storage_exporter"), - required => true - }} + {type, + mk( + local, + #{ + default => local, + required => false, + desc => ?DESC("local_type") + } + )}, + {segments, + mk( + ref(local_storage_segments), + #{ + desc => ?DESC("local_storage_segments"), + required => false + } + )}, + {exporter, + mk( + hoconsc:union([ + ref(local_storage_exporter), + ref(s3_exporter) + ]), + #{ + desc => ?DESC("local_storage_exporter"), + required => true + } + )} ]; fields(local_storage_segments) -> [ - {root, #{ - type => binary(), - desc => ?DESC("local_storage_segments_root"), - required => false - }}, - {gc, #{ - type => ?REF(local_storage_segments_gc), - desc => ?DESC("local_storage_segments_gc"), - required => false - }} + {root, + mk( + binary(), + #{ + desc => ?DESC("local_storage_segments_root"), + required => false + } + )}, + {gc, + mk( + ref(local_storage_segments_gc), #{ + desc => ?DESC("local_storage_segments_gc"), + required => false + } + )} ]; fields(local_storage_exporter) -> [ - {type, #{ - type => local, - default => local, - required => false, - desc => ?DESC("local_storage_exporter_type") - }}, - {root, #{ - type => binary(), - desc => ?DESC("local_storage_exporter_root"), - required => false - }} + {type, + mk( + local, + #{ + default => local, + required => false, + desc => ?DESC("local_storage_exporter_type") + } + )}, + {root, + mk( + binary(), + #{ + desc => ?DESC("local_storage_exporter_root"), + required => false + } + )} ]; +fields(s3_exporter) -> + [ + {type, + mk( + s3, + #{ + default => s3, + required => false, + desc => ?DESC("s3_exporter_type") + } + )} + ] ++ + emqx_s3_schema:fields(s3); fields(local_storage_segments_gc) -> [ - {interval, #{ - type => emqx_schema:duration_ms(), - desc => ?DESC("storage_gc_interval"), - required => false, - default => "1h" - }}, - {maximum_segments_ttl, #{ - type => emqx_schema:duration_s(), - desc => ?DESC("storage_gc_max_segments_ttl"), - required => false, - default => "24h" - }}, - {minimum_segments_ttl, #{ - type => emqx_schema:duration_s(), - % desc => ?DESC("storage_gc_min_segments_ttl"), - required => false, - default => "5m", - % NOTE - % This setting does not seem to be useful to an end-user. - hidden => true - }} + {interval, + mk( + emqx_schema:duration_ms(), + #{ + desc => ?DESC("storage_gc_interval"), + required => false, + default => "1h" + } + )}, + {maximum_segments_ttl, + mk( + emqx_schema:duration_s(), + #{ + desc => ?DESC("storage_gc_max_segments_ttl"), + required => false, + default => "24h" + } + )}, + {minimum_segments_ttl, + mk( + emqx_schema:duration_s(), + #{ + required => false, + default => "5m", + % NOTE + % This setting does not seem to be useful to an end-user. + hidden => true + } + )} ]. desc(file_transfer) -> @@ -133,7 +179,9 @@ desc(local_storage) -> desc(local_storage_segments) -> "File transfer local segments storage settings"; desc(local_storage_exporter) -> - "Exporter settings for the File transfer local storage backend"; + "Local Exporter settings for the File transfer local storage backend"; +desc(s3_exporter) -> + "S3 Exporter settings for the File transfer local storage backend"; desc(local_storage_segments_gc) -> "Garbage collection settings for the File transfer local segments storage". diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index a7d1cd2e2..297762d28 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -34,42 +34,47 @@ -export([exporter/1]). --export_type([options/0]). -export_type([export/0]). -type storage() :: emxt_ft_storage_fs:storage(). -type transfer() :: emqx_ft:transfer(). -type filemeta() :: emqx_ft:filemeta(). --type options() :: map(). --type export() :: term(). +-type exporter_conf() :: map(). +-type export_st() :: term(). +-opaque export() :: {module(), export_st()}. --callback start_export(options(), transfer(), filemeta()) -> - {ok, export()} | {error, _Reason}. +-callback start_export(exporter_conf(), transfer(), filemeta()) -> + {ok, export_st()} | {error, _Reason}. --callback write(ExportSt :: export(), iodata()) -> - {ok, ExportSt :: export()} | {error, _Reason}. +%% Exprter must discard the export itself in case of error +-callback write(ExportSt :: export_st(), iodata()) -> + {ok, ExportSt :: export_st()} | {error, _Reason}. --callback complete(ExportSt :: export()) -> +-callback complete(ExportSt :: export_st()) -> ok | {error, _Reason}. --callback discard(ExportSt :: export()) -> +-callback discard(ExportSt :: export_st()) -> ok | {error, _Reason}. --callback list(options()) -> +-callback list(storage()) -> {ok, [emqx_ft_storage:file_info()]} | {error, _Reason}. %% +-spec start_export(storage(), transfer(), filemeta()) -> + {ok, export()} | {error, _Reason}. start_export(Storage, Transfer, Filemeta) -> - {ExporterMod, Exporter} = exporter(Storage), - case ExporterMod:start_export(Exporter, Transfer, Filemeta) of + {ExporterMod, ExporterConf} = exporter(Storage), + case ExporterMod:start_export(ExporterConf, Transfer, Filemeta) of {ok, ExportSt} -> {ok, {ExporterMod, ExportSt}}; {error, _} = Error -> Error end. +-spec write(export(), iodata()) -> + {ok, export()} | {error, _Reason}. write({ExporterMod, ExportSt}, Content) -> case ExporterMod:write(ExportSt, Content) of {ok, ExportStNext} -> @@ -78,23 +83,31 @@ write({ExporterMod, ExportSt}, Content) -> Error end. +-spec complete(export()) -> + ok | {error, _Reason}. complete({ExporterMod, ExportSt}) -> ExporterMod:complete(ExportSt). +-spec discard(export()) -> + ok | {error, _Reason}. discard({ExporterMod, ExportSt}) -> ExporterMod:discard(ExportSt). -%% - +-spec list(storage()) -> + {ok, [emqx_ft_storage:file_info()]} | {error, _Reason}. list(Storage) -> {ExporterMod, ExporterOpts} = exporter(Storage), ExporterMod:list(ExporterOpts). -%% - -spec exporter(storage()) -> {module(), _ExporterOptions}. exporter(Storage) -> case maps:get(exporter, Storage) of #{type := local} = Options -> - {emqx_ft_storage_exporter_fs, Options} + {emqx_ft_storage_exporter_fs, without_type(Options)}; + #{type := s3} = Options -> + {emqx_ft_storage_exporter_s3, without_type(Options)} end. + +-spec without_type(exporter_conf()) -> exporter_conf(). +without_type(#{type := _} = Options) -> + maps:without([type], Options). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index 647d84124..64c0e325a 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -20,21 +20,22 @@ -include_lib("emqx/include/logger.hrl"). %% Exporter API +-behaviour(emqx_ft_storage_exporter). + -export([start_export/3]). -export([write/2]). -export([complete/1]). -export([discard/1]). +-export([list/1]). +%% Internal API for RPC -export([list_local/1]). -export([list_local/2]). -export([start_reader/3]). --export([list/1]). % TODO % -export([list/2]). --export_type([export/0]). - -type options() :: _TODO. -type transfer() :: emqx_ft:transfer(). -type filemeta() :: emqx_ft:filemeta(). @@ -49,7 +50,7 @@ -type file_error() :: emqx_ft_storage_fs:file_error(). --opaque export() :: #{ +-type export() :: #{ path := file:name(), handle := io:device(), result := file:name(), @@ -116,6 +117,7 @@ write(Export = #{handle := Handle, hash := Ctx}, IoData) -> ok -> {ok, Export#{hash := update_checksum(Ctx, IoData)}}; {error, _} = Error -> + _ = discard(Export), Error end. @@ -370,4 +372,4 @@ mk_transfer_hash(Transfer) -> crypto:hash(?BUCKET_HASH, term_to_binary(Transfer)). get_storage_root(Options) -> - maps:get(root, Options, filename:join([emqx:data_dir(), "ft", "exports"])). + maps:get(root, Options, filename:join([emqx:data_dir(), file_transfer, exports])). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl new file mode 100644 index 000000000..300f0dc85 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -0,0 +1,69 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_exporter_s3). + +-include_lib("emqx/include/logger.hrl"). + +%% Exporter API +-export([start_export/3]). +-export([write/2]). +-export([complete/1]). +-export([discard/1]). +-export([list/1]). + +-type options() :: emqx_s3:profile_config(). +-type transfer() :: emqx_ft:transfer(). +-type filemeta() :: emqx_ft:filemeta(). +-type exportinfo() :: #{ + transfer := transfer(), + name := file:name(), + uri := uri_string:uri_string(), + timestamp := emqx_datetime:epoch_second(), + size := _Bytes :: non_neg_integer(), + meta => filemeta() +}. + +-type export_st() :: #{ + pid := pid(), + meta := filemeta() +}. + +-spec start_export(options(), transfer(), filemeta()) -> + {ok, export_st()} | {error, term()}. +start_export(_Options, _Transfer, Filemeta = #{name := Filename}) -> + Pid = spawn(fun() -> Filename end), + #{meta => Filemeta, pid => Pid}. + +-spec write(export_st(), iodata()) -> + {ok, export_st()} | {error, term()}. +write(ExportSt, _IoData) -> + {ok, ExportSt}. + +-spec complete(export_st()) -> + ok | {error, term()}. +complete(_ExportSt) -> + ok. + +-spec discard(export_st()) -> + ok. +discard(_ExportSt) -> + ok. + +-spec list(options()) -> + {ok, [exportinfo()]} | {error, term()}. +list(_Options) -> + {ok, []}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 4e5cc9236..d871d1f32 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -86,14 +86,18 @@ -define(MANIFEST, "MANIFEST.json"). -define(SEGMENT, "SEG"). --type storage() :: #{ - root => file:name(), - exporter => exporter() +-type segments() :: #{ + root := file:name(), + gc := #{ + interval := non_neg_integer(), + maximum_segments_ttl := non_neg_integer(), + minimum_segments_ttl := non_neg_integer() + } }. --type exporter() :: #{ - type := local, - root => file:name() +-type storage() :: #{ + segments := segments(), + exporter := emqx_ft_storage_exporter:exporter() }. -type file_error() :: diff --git a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf new file mode 100644 index 000000000..4e0870bae --- /dev/null +++ b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf @@ -0,0 +1,76 @@ +emqx_s3_schema { + access_key_id { + desc { + en: "The access key id of the S3 bucket." + } + label { + en: "Access Key ID" + } + } + secret_access_key { + desc { + en: "The secret access key of the S3 bucket." + } + label { + en: "Secret Access Key" + } + } + bucket { + desc { + en: "The name of the S3 bucket." + } + label { + en: "Bucket" + } + } + host { + desc { + en: "The host of the S3 endpoint." + } + label { + en: "S3 endpoint Host" + } + } + port { + desc { + en: "The port of the S3 endpoint." + } + label { + en: "S3 endpoint port" + } + } + min_part_size { + desc { + en: """The minimum part size for multipart uploads. +Uploaded data will be accumulated in memory until this size is reached.""" + } + label { + en: "Minimum multipart upload part size" + } + } + max_part_size { + desc { + en: """The maximum part size for multipart uploads. +S3 uploader won't try to upload parts larger than this size.""" + } + label { + en: "Maximum multipart upload part size" + } + } + acl { + desc { + en: "The ACL to use for the uploaded objects." + } + label { + en: "ACL" + } + } + transport_options { + desc { + en: "Options for the HTTP transport layer used by the S3 client." + } + label { + en: "Transport Options" + } + } +} diff --git a/apps/emqx_s3/src/emqx_s3.erl b/apps/emqx_s3/src/emqx_s3.erl index 6d2577dca..7c78de979 100644 --- a/apps/emqx_s3/src/emqx_s3.erl +++ b/apps/emqx_s3/src/emqx_s3.erl @@ -21,8 +21,32 @@ -type profile_id() :: term(). -%% TODO: define fields --type profile_config() :: map(). +-type acl() :: + private + | public_read + | public_read_write + | authenticated_read + | bucket_owner_read + | bucket_owner_full_control. + +-type transport_options() :: #{ + connect_timeout => pos_integer(), + enable_pipelining => pos_integer(), + headers => map(), + max_retries => pos_integer(), + pool_size => pos_integer(), + pool_type => atom(), + ssl => map() +}. + +-type profile_config() :: #{ + bucket := string(), + host := string(), + port := pos_integer(), + acl => acl(), + min_part_size => pos_integer(), + transport_options => transport_options() +}. %%-------------------------------------------------------------------- %% API diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index ceb0d1dd4..5d76e7120 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -7,7 +7,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). --import(hoconsc, [mk/2, ref/1]). +-import(hoconsc, [mk/2, ref/2]). -export([roots/0, fields/1, namespace/0, tags/0]). @@ -101,7 +101,7 @@ fields(s3) -> )}, {transport_options, mk( - ref(transport_options), + ref(?MODULE, transport_options), #{ desc => ?DESC("transport_options"), required => false From 83612236487af0bd22269e22f5acf72767825fa7 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 30 Mar 2023 23:09:13 +0300 Subject: [PATCH 087/197] feat(ft-s3): extract checksum verification --- apps/emqx_ft/src/emqx_ft.erl | 6 +- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 69 +++++++++++++++--- .../src/emqx_ft_storage_exporter_fs.erl | 70 ++++++------------- 3 files changed, 84 insertions(+), 61 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index e3f6cbe1c..064b6e066 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -45,7 +45,8 @@ bytes/0, offset/0, filemeta/0, - segment/0 + segment/0, + checksum/0 ]). %% Number of bytes @@ -57,6 +58,7 @@ -type fileid() :: binary(). -type transfer() :: {clientid(), fileid()}. -type offset() :: bytes(). +-type checksum() :: {_Algo :: atom(), _Digest :: binary()}. -type filemeta() :: #{ %% Display name @@ -68,7 +70,7 @@ %% currently do not condider that an error (or, specifically, a signal that %% the resulting file is corrupted during transmission). size => _Bytes :: non_neg_integer(), - checksum => {sha256, <<_:256>>}, + checksum => checksum(), expire_at := emqx_datetime:epoch_second(), %% TTL of individual segments %% Somewhat confusing that we won't know it on the nodes where the filemeta diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index 297762d28..444d686a8 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -39,10 +39,21 @@ -type storage() :: emxt_ft_storage_fs:storage(). -type transfer() :: emqx_ft:transfer(). -type filemeta() :: emqx_ft:filemeta(). +-type checksum() :: emqx_ft:checksum(). -type exporter_conf() :: map(). -type export_st() :: term(). --opaque export() :: {module(), export_st()}. +-type hash_state() :: term(). +-opaque export() :: #{ + mod := module(), + st := export_st(), + hash := hash_state(), + filemeta := filemeta() +}. + +%%------------------------------------------------------------------------------ +%% Behaviour +%%------------------------------------------------------------------------------ -callback start_export(exporter_conf(), transfer(), filemeta()) -> {ok, export_st()} | {error, _Reason}. @@ -51,7 +62,7 @@ -callback write(ExportSt :: export_st(), iodata()) -> {ok, ExportSt :: export_st()} | {error, _Reason}. --callback complete(ExportSt :: export_st()) -> +-callback complete(_ExportSt :: export_st(), _Checksum :: checksum()) -> ok | {error, _Reason}. -callback discard(ExportSt :: export_st()) -> @@ -60,7 +71,9 @@ -callback list(storage()) -> {ok, [emqx_ft_storage:file_info()]} | {error, _Reason}. -%% +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ -spec start_export(storage(), transfer(), filemeta()) -> {ok, export()} | {error, _Reason}. @@ -68,29 +81,43 @@ start_export(Storage, Transfer, Filemeta) -> {ExporterMod, ExporterConf} = exporter(Storage), case ExporterMod:start_export(ExporterConf, Transfer, Filemeta) of {ok, ExportSt} -> - {ok, {ExporterMod, ExportSt}}; + {ok, #{ + mod => ExporterMod, + st => ExportSt, + hash => init_checksum(Filemeta), + filemeta => Filemeta + }}; {error, _} = Error -> Error end. -spec write(export(), iodata()) -> {ok, export()} | {error, _Reason}. -write({ExporterMod, ExportSt}, Content) -> +write(#{mod := ExporterMod, st := ExportSt, hash := Hash} = Export, Content) -> case ExporterMod:write(ExportSt, Content) of {ok, ExportStNext} -> - {ok, {ExporterMod, ExportStNext}}; + {ok, Export#{ + st := ExportStNext, + hash := update_checksum(Hash, Content) + }}; {error, _} = Error -> Error end. -spec complete(export()) -> ok | {error, _Reason}. -complete({ExporterMod, ExportSt}) -> - ExporterMod:complete(ExportSt). +complete(#{mod := ExporterMod, st := ExportSt, hash := Hash, filemeta := Filemeta}) -> + case verify_checksum(Hash, Filemeta) of + {ok, Checksum} -> + ExporterMod:complete(ExportSt, Checksum); + {error, _} = Error -> + _ = ExporterMod:discard(ExportSt), + Error + end. -spec discard(export()) -> ok | {error, _Reason}. -discard({ExporterMod, ExportSt}) -> +discard(#{mod := ExporterMod, st := ExportSt}) -> ExporterMod:discard(ExportSt). -spec list(storage()) -> @@ -99,7 +126,10 @@ list(Storage) -> {ExporterMod, ExporterOpts} = exporter(Storage), ExporterMod:list(ExporterOpts). --spec exporter(storage()) -> {module(), _ExporterOptions}. +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + exporter(Storage) -> case maps:get(exporter, Storage) of #{type := local} = Options -> @@ -111,3 +141,22 @@ exporter(Storage) -> -spec without_type(exporter_conf()) -> exporter_conf(). without_type(#{type := _} = Options) -> maps:without([type], Options). + +init_checksum(#{checksum := {Algo, _}}) -> + crypto:hash_init(Algo); +init_checksum(#{}) -> + crypto:hash_init(sha256). + +update_checksum(Ctx, IoData) -> + crypto:hash_update(Ctx, IoData). + +verify_checksum(Ctx, #{checksum := {Algo, Digest} = Checksum}) -> + case crypto:hash_final(Ctx) of + Digest -> + {ok, Checksum}; + Mismatch -> + {error, {checksum, Algo, binary:encode_hex(Mismatch)}} + end; +verify_checksum(Ctx, #{}) -> + Digest = crypto:hash_final(Ctx), + {ok, {sha256, Digest}}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index 64c0e325a..fd1956009 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -24,7 +24,7 @@ -export([start_export/3]). -export([write/2]). --export([complete/1]). +-export([complete/2]). -export([discard/1]). -export([list/1]). @@ -50,12 +50,11 @@ -type file_error() :: emqx_ft_storage_fs:file_error(). --type export() :: #{ +-type export_st() :: #{ path := file:name(), handle := io:device(), result := file:name(), - meta := filemeta(), - hash := crypto:hash_state() + meta := filemeta() }. -type reader() :: pid(). @@ -92,7 +91,7 @@ %% -spec start_export(options(), transfer(), filemeta()) -> - {ok, export()} | {error, file_error()}. + {ok, export_st()} | {error, file_error()}. start_export(Options, Transfer, Filemeta = #{name := Filename}) -> TempFilepath = mk_temp_absfilepath(Options, Transfer, Filename), ResultFilepath = mk_absfilepath(Options, Transfer, result, Filename), @@ -103,47 +102,41 @@ start_export(Options, Transfer, Filemeta = #{name := Filename}) -> path => TempFilepath, handle => Handle, result => ResultFilepath, - meta => Filemeta, - hash => init_checksum(Filemeta) + meta => Filemeta }}; {error, _} = Error -> Error end. --spec write(export(), iodata()) -> - {ok, export()} | {error, file_error()}. -write(Export = #{handle := Handle, hash := Ctx}, IoData) -> +-spec write(export_st(), iodata()) -> + {ok, export_st()} | {error, file_error()}. +write(ExportSt = #{handle := Handle}, IoData) -> case file:write(Handle, IoData) of ok -> - {ok, Export#{hash := update_checksum(Ctx, IoData)}}; + {ok, ExportSt}; {error, _} = Error -> - _ = discard(Export), + _ = discard(ExportSt), Error end. --spec complete(export()) -> +-spec complete(export_st(), emqx_ft:checksum()) -> ok | {error, {checksum, _Algo, _Computed}} | {error, file_error()}. complete( - Export = #{ + #{ path := Filepath, handle := Handle, result := ResultFilepath, - meta := FilemetaIn, - hash := Ctx - } + meta := FilemetaIn + }, + Checksum ) -> - case verify_checksum(Ctx, FilemetaIn) of - {ok, Filemeta} -> - ok = file:close(Handle), - _ = filelib:ensure_dir(ResultFilepath), - _ = file:write_file(mk_manifest_filename(ResultFilepath), encode_filemeta(Filemeta)), - file:rename(Filepath, ResultFilepath); - {error, _} = Error -> - _ = discard(Export), - Error - end. + Filemeta = FilemetaIn#{checksum => Checksum}, + ok = file:close(Handle), + _ = filelib:ensure_dir(ResultFilepath), + _ = file:write_file(mk_manifest_filename(ResultFilepath), encode_filemeta(Filemeta)), + file:rename(Filepath, ResultFilepath). --spec discard(export()) -> +-spec discard(export_st()) -> ok. discard(#{path := Filepath, handle := Handle}) -> ok = file:close(Handle), @@ -297,27 +290,6 @@ list(_Options) -> %% -init_checksum(#{checksum := {Algo, _}}) -> - crypto:hash_init(Algo); -init_checksum(#{}) -> - crypto:hash_init(sha256). - -update_checksum(Ctx, IoData) -> - crypto:hash_update(Ctx, IoData). - -verify_checksum(Ctx, Filemeta = #{checksum := {Algo, Digest}}) -> - case crypto:hash_final(Ctx) of - Digest -> - {ok, Filemeta}; - Mismatch -> - {error, {checksum, Algo, binary:encode_hex(Mismatch)}} - end; -verify_checksum(Ctx, Filemeta = #{}) -> - Digest = crypto:hash_final(Ctx), - {ok, Filemeta#{checksum => {sha256, Digest}}}. - -%% - -define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). encode_filemeta(Meta) -> From 5ac3543a768a75abb0904aadd7afe7fef3469d0e Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 31 Mar 2023 00:32:11 +0300 Subject: [PATCH 088/197] feat(ft-s3): integrate exporter configs --- apps/emqx_ft/src/emqx_ft_conf.erl | 10 ++++- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 44 +++++++++++++++++-- .../emqx_ft_storage_exporter_fs.erl | 34 +++++++++++++- .../emqx_ft_storage_exporter_fs_api.erl | 0 .../emqx_ft_storage_exporter_fs_proxy.erl | 0 .../emqx_ft_storage_exporter_s3.erl | 33 ++++++++++++++ 6 files changed, 114 insertions(+), 7 deletions(-) rename apps/emqx_ft/src/{ => emqx_ft_storage_exporter}/emqx_ft_storage_exporter_fs.erl (91%) rename apps/emqx_ft/src/{ => emqx_ft_storage_exporter}/emqx_ft_storage_exporter_fs_api.erl (100%) rename apps/emqx_ft/src/{ => emqx_ft_storage_exporter}/emqx_ft_storage_exporter_fs_proxy.erl (100%) rename apps/emqx_ft/src/{ => emqx_ft_storage_exporter}/emqx_ft_storage_exporter_s3.erl (68%) diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index aafcd5ad3..6ce10408e 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -75,6 +75,10 @@ assert_storage(Type) -> -spec load() -> ok. load() -> + ok = emqx_ft_storage_exporter:update_exporter( + undefined, + emqx_config:get([file_transfer, storage]) + ), emqx_conf:add_handler([file_transfer], ?MODULE). -spec unload() -> ok. @@ -98,5 +102,7 @@ pre_config_update(_, Req, _Config) -> emqx_config:app_envs() ) -> ok | {ok, Result :: any()} | {error, Reason :: term()}. -post_config_update(_, _Req, _NewConfig, _OldConfig, _AppEnvs) -> - ok. +post_config_update(_Path, _Req, NewConfig, OldConfig, _AppEnvs) -> + OldStorageConfig = maps:get(storage, OldConfig, undefined), + NewStorageConfig = maps:get(storage, NewConfig, undefined), + emqx_ft_storage_exporter:update_exporter(OldStorageConfig, NewStorageConfig). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index 444d686a8..61241bd6f 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -29,9 +29,11 @@ %% Listing API -export([list/1]). -% TODO -% -export([list/2]). +%% Lifecycle API +-export([update_exporter/2]). + +%% Internal API -export([exporter/1]). -export_type([export/0]). @@ -71,6 +73,17 @@ -callback list(storage()) -> {ok, [emqx_ft_storage:file_info()]} | {error, _Reason}. +%% Lifecycle callbacks + +-callback start(exporter_conf()) -> + ok | {error, _Reason}. + +-callback stop(exporter_conf()) -> + ok. + +-callback update(exporter_conf(), exporter_conf()) -> + ok | {error, _Reason}. + %%------------------------------------------------------------------------------ %% API %%------------------------------------------------------------------------------ @@ -126,6 +139,32 @@ list(Storage) -> {ExporterMod, ExporterOpts} = exporter(Storage), ExporterMod:list(ExporterOpts). +%% Lifecycle + +-spec update_exporter(emqx_config:config(), emqx_config:config()) -> ok | {error, term()}. +update_exporter( + #{exporter := #{type := OldType}} = OldConfig, + #{exporter := #{type := OldType}} = NewConfig +) -> + {ExporterMod, OldExporterOpts} = exporter(OldConfig), + {_NewExporterMod, NewExporterOpts} = exporter(NewConfig), + ExporterMod:update(OldExporterOpts, NewExporterOpts); +update_exporter( + #{exporter := _} = OldConfig, + #{exporter := _} = NewConfig +) -> + {OldExporterMod, OldExporterOpts} = exporter(OldConfig), + {NewExporterMod, NewExporterOpts} = exporter(NewConfig), + ok = OldExporterMod:stop(OldExporterOpts), + NewExporterMod:start(NewExporterOpts); +update_exporter(undefined, NewConfig) -> + {ExporterMod, ExporterOpts} = exporter(NewConfig), + ExporterMod:start(ExporterOpts); +update_exporter(OldConfig, undefined) -> + {ExporterMod, ExporterOpts} = exporter(OldConfig), + ExporterMod:stop(ExporterOpts); +update_exporter(_, _) -> + ok. %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ @@ -138,7 +177,6 @@ exporter(Storage) -> {emqx_ft_storage_exporter_s3, without_type(Options)} end. --spec without_type(exporter_conf()) -> exporter_conf(). without_type(#{type := _} = Options) -> maps:without([type], Options). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs.erl similarity index 91% rename from apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl rename to apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs.erl index fd1956009..f84189f7f 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs.erl @@ -28,6 +28,12 @@ -export([discard/1]). -export([list/1]). +-export([ + start/1, + stop/1, + update/2 +]). + %% Internal API for RPC -export([list_local/1]). -export([list_local/2]). @@ -88,7 +94,9 @@ }) ). -%% +%%-------------------------------------------------------------------- +%% Exporter behaviour +%%-------------------------------------------------------------------- -spec start_export(options(), transfer(), filemeta()) -> {ok, export_st()} | {error, file_error()}. @@ -142,7 +150,25 @@ discard(#{path := Filepath, handle := Handle}) -> ok = file:close(Handle), file:delete(Filepath). -%% +%%-------------------------------------------------------------------- +%% Exporter behaviour (lifecycle) +%%-------------------------------------------------------------------- + +%% FS Exporter does not have require any stateful entities, +%% so lifecycle callbacks are no-op. + +-spec start(options()) -> ok. +start(_Options) -> ok. + +-spec stop(options()) -> ok. +stop(_Options) -> ok. + +-spec update(options(), options()) -> ok. +update(_OldOptions, _NewOptions) -> ok. + +%%-------------------------------------------------------------------- +%% Internal API +%%-------------------------------------------------------------------- -spec list_local(options(), transfer()) -> {ok, [exportinfo(), ...]} | {error, file_error()}. @@ -199,6 +225,10 @@ list_local(Options) -> Pattern )}. +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + filter_manifest(?MANIFEST) -> % Filename equals `?MANIFEST`, there should also be a manifest for it. false; diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs_api.erl similarity index 100% rename from apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl rename to apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs_api.erl diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs_proxy.erl similarity index 100% rename from apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl rename to apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs_proxy.erl diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_s3.erl similarity index 68% rename from apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl rename to apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_s3.erl index 300f0dc85..977ff576a 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_s3.erl @@ -25,6 +25,12 @@ -export([discard/1]). -export([list/1]). +-export([ + start/1, + stop/1, + update/2 +]). + -type options() :: emqx_s3:profile_config(). -type transfer() :: emqx_ft:transfer(). -type filemeta() :: emqx_ft:filemeta(). @@ -42,6 +48,10 @@ meta := filemeta() }. +%%-------------------------------------------------------------------- +%% Exporter behaviour +%%-------------------------------------------------------------------- + -spec start_export(options(), transfer(), filemeta()) -> {ok, export_st()} | {error, term()}. start_export(_Options, _Transfer, Filemeta = #{name := Filename}) -> @@ -67,3 +77,26 @@ discard(_ExportSt) -> {ok, [exportinfo()]} | {error, term()}. list(_Options) -> {ok, []}. + +%%-------------------------------------------------------------------- +%% Exporter behaviour (lifecycle) +%%-------------------------------------------------------------------- + +-spec start(options()) -> ok | {error, term()}. +start(Options) -> + emqx_s3:start_profile(s3_profile_id(), Options). + +-spec stop(options()) -> ok. +stop(_Options) -> + ok = emqx_s3:stop_profile(s3_profile_id()). + +-spec update(options(), options()) -> ok. +update(_OldOptions, NewOptions) -> + emqx_s3:update_profile(s3_profile_id(), NewOptions). + +%%-------------------------------------------------------------------- +%% Internal functions +%% ------------------------------------------------------------------- + +s3_profile_id() -> + atom_to_binary(?MODULE). From 43f973742033fa6eb14e6bdf278facc27f13731c Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 3 Apr 2023 20:35:47 +0300 Subject: [PATCH 089/197] feat(ft-s3): integrate list API --- apps/emqx_ft/src/emqx_ft.erl | 1 - apps/emqx_ft/src/emqx_ft_api.erl | 8 +- .../emqx_ft_storage_exporter_s3.erl | 102 ---------- .../emqx_ft_storage_exporter_fs.erl | 0 .../emqx_ft_storage_exporter_fs_api.erl | 0 .../emqx_ft_storage_exporter_fs_proxy.erl | 0 .../src/emqx_ft_storage_exporter_s3.erl | 188 ++++++++++++++++++ apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf | 12 +- apps/emqx_s3/src/emqx_s3.erl | 3 + apps/emqx_s3/src/emqx_s3_client.erl | 89 +++++---- apps/emqx_s3/src/emqx_s3_profile_conf.erl | 1 + apps/emqx_s3/src/emqx_s3_schema.erl | 9 + apps/emqx_s3/src/emqx_s3_uploader.erl | 49 +++-- apps/emqx_s3/test/emqx_s3_client_SUITE.erl | 33 +++ 14 files changed, 333 insertions(+), 162 deletions(-) delete mode 100644 apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_s3.erl rename apps/emqx_ft/src/{emqx_ft_storage_exporter => }/emqx_ft_storage_exporter_fs.erl (100%) rename apps/emqx_ft/src/{emqx_ft_storage_exporter => }/emqx_ft_storage_exporter_fs_api.erl (100%) rename apps/emqx_ft/src/{emqx_ft_storage_exporter => }/emqx_ft_storage_exporter_fs_proxy.erl (100%) create mode 100644 apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 064b6e066..c5522cdbd 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -118,7 +118,6 @@ decode_filemeta(Map) when is_map(Map) -> end. encode_filemeta(Meta = #{}) -> - % TODO: Looks like this should be hocon's responsibility. Schema = emqx_ft_schema:schema(filemeta), hocon_tconf:make_serializable(Schema, emqx_map_lib:binary_key_map(Meta), #{}). diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index a667454b6..390c10557 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -69,6 +69,7 @@ schema("/file_transfer/files") -> '/file_transfer/files'(get, #{}) -> case emqx_ft_storage:files() of {ok, Files} -> + ?SLOG(warning, #{msg => "files", files => Files}), {200, #{<<"files">> => lists:map(fun format_file_info/1, Files)}}; {error, _} -> {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} @@ -94,7 +95,7 @@ format_file_info( } ) -> Res = #{ - name => iolist_to_binary(Name), + name => format_name(Name), size => Size, timestamp => format_timestamp(Timestamp), clientid => ClientId, @@ -110,3 +111,8 @@ format_file_info( format_timestamp(Timestamp) -> iolist_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])). + +format_name(NameBin) when is_binary(NameBin) -> + NameBin; +format_name(Name) when is_list(Name) -> + iolist_to_binary(Name). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_s3.erl deleted file mode 100644 index 977ff576a..000000000 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_s3.erl +++ /dev/null @@ -1,102 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_ft_storage_exporter_s3). - --include_lib("emqx/include/logger.hrl"). - -%% Exporter API --export([start_export/3]). --export([write/2]). --export([complete/1]). --export([discard/1]). --export([list/1]). - --export([ - start/1, - stop/1, - update/2 -]). - --type options() :: emqx_s3:profile_config(). --type transfer() :: emqx_ft:transfer(). --type filemeta() :: emqx_ft:filemeta(). --type exportinfo() :: #{ - transfer := transfer(), - name := file:name(), - uri := uri_string:uri_string(), - timestamp := emqx_datetime:epoch_second(), - size := _Bytes :: non_neg_integer(), - meta => filemeta() -}. - --type export_st() :: #{ - pid := pid(), - meta := filemeta() -}. - -%%-------------------------------------------------------------------- -%% Exporter behaviour -%%-------------------------------------------------------------------- - --spec start_export(options(), transfer(), filemeta()) -> - {ok, export_st()} | {error, term()}. -start_export(_Options, _Transfer, Filemeta = #{name := Filename}) -> - Pid = spawn(fun() -> Filename end), - #{meta => Filemeta, pid => Pid}. - --spec write(export_st(), iodata()) -> - {ok, export_st()} | {error, term()}. -write(ExportSt, _IoData) -> - {ok, ExportSt}. - --spec complete(export_st()) -> - ok | {error, term()}. -complete(_ExportSt) -> - ok. - --spec discard(export_st()) -> - ok. -discard(_ExportSt) -> - ok. - --spec list(options()) -> - {ok, [exportinfo()]} | {error, term()}. -list(_Options) -> - {ok, []}. - -%%-------------------------------------------------------------------- -%% Exporter behaviour (lifecycle) -%%-------------------------------------------------------------------- - --spec start(options()) -> ok | {error, term()}. -start(Options) -> - emqx_s3:start_profile(s3_profile_id(), Options). - --spec stop(options()) -> ok. -stop(_Options) -> - ok = emqx_s3:stop_profile(s3_profile_id()). - --spec update(options(), options()) -> ok. -update(_OldOptions, NewOptions) -> - emqx_s3:update_profile(s3_profile_id(), NewOptions). - -%%-------------------------------------------------------------------- -%% Internal functions -%% ------------------------------------------------------------------- - -s3_profile_id() -> - atom_to_binary(?MODULE). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl similarity index 100% rename from apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs.erl rename to apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs_api.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl similarity index 100% rename from apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs_api.erl rename to apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl similarity index 100% rename from apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs_proxy.erl rename to apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl new file mode 100644 index 000000000..1c97520e3 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -0,0 +1,188 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_exporter_s3). + +-include_lib("emqx/include/logger.hrl"). + +%% Exporter API +-export([start_export/3]). +-export([write/2]). +-export([complete/2]). +-export([discard/1]). +-export([list/1]). + +-export([ + start/1, + stop/1, + update/2 +]). + +-type options() :: emqx_s3:profile_config(). +-type transfer() :: emqx_ft:transfer(). +-type filemeta() :: emqx_ft:filemeta(). +-type exportinfo() :: #{ + transfer := transfer(), + name := file:name(), + uri := uri_string:uri_string(), + timestamp := emqx_datetime:epoch_second(), + size := _Bytes :: non_neg_integer(), + filemeta => filemeta() +}. + +-type export_st() :: #{ + pid := pid(), + filemeta := filemeta(), + transfer := transfer() +}. + +-define(S3_PROFILE_ID, <<"emqx_ft_storage_exporter_s3">>). +-define(FILEMETA_VSN, <<"1">>). +-define(S3_LIST_LIMIT, 500). + +%%-------------------------------------------------------------------- +%% Exporter behaviour +%%-------------------------------------------------------------------- + +-spec start_export(options(), transfer(), filemeta()) -> + {ok, export_st()} | {error, term()}. +start_export(_Options, Transfer, Filemeta) -> + Options = #{ + key => s3_key(Transfer, Filemeta), + headers => s3_headers(Transfer, Filemeta) + }, + case emqx_s3:start_uploader(?S3_PROFILE_ID, Options) of + {ok, Pid} -> + {ok, #{filemeta => Filemeta, pid => Pid}}; + {error, _Reason} = Error -> + Error + end. + +-spec write(export_st(), iodata()) -> + {ok, export_st()} | {error, term()}. +write(#{pid := Pid} = ExportSt, IoData) -> + case emqx_s3_uploader:write(Pid, IoData) of + ok -> + {ok, ExportSt}; + {error, _Reason} = Error -> + Error + end. + +-spec complete(export_st(), emqx_ft:checksum()) -> + ok | {error, term()}. +complete(#{pid := Pid} = _ExportSt, _Checksum) -> + emqx_s3_uploader:complete(Pid). + +-spec discard(export_st()) -> + ok. +discard(#{pid := Pid} = _ExportSt) -> + emqx_s3_uploader:abort(Pid). + +-spec list(options()) -> + {ok, [exportinfo()]} | {error, term()}. +list(Options) -> + emqx_s3:with_client(?S3_PROFILE_ID, fun(Client) -> list(Client, Options) end). + +%%-------------------------------------------------------------------- +%% Exporter behaviour (lifecycle) +%%-------------------------------------------------------------------- + +-spec start(options()) -> ok | {error, term()}. +start(Options) -> + emqx_s3:start_profile(?S3_PROFILE_ID, Options). + +-spec stop(options()) -> ok. +stop(_Options) -> + ok = emqx_s3:stop_profile(?S3_PROFILE_ID). + +-spec update(options(), options()) -> ok. +update(_OldOptions, NewOptions) -> + emqx_s3:update_profile(?S3_PROFILE_ID, NewOptions). + +%%-------------------------------------------------------------------- +%% Internal functions +%% ------------------------------------------------------------------- + +s3_key({ClientId, FileId} = _Transfer, #{name := Filename}) -> + filename:join([ + emqx_ft_fs_util:escape_filename(ClientId), + emqx_ft_fs_util:escape_filename(FileId), + Filename + ]). + +s3_headers({ClientId, FileId}, Filemeta) -> + #{ + %% The ClientID MUST be a UTF-8 Encoded String + <<"x-amz-meta-clientid">> => ClientId, + %% It [Topic Name] MUST be a UTF-8 Encoded String + <<"x-amz-meta-fileid">> => FileId, + <<"x-amz-meta-filemeta">> => emqx_json:encode(emqx_ft:encode_filemeta(Filemeta)), + <<"x-amz-meta-filemeta-vsn">> => ?FILEMETA_VSN + }. + +list(Client, Options) -> + case list_key_info(Client, Options) of + {ok, KeyInfos} -> + {ok, + lists:map( + fun(KeyInfo) -> key_info_to_exportinfo(Client, KeyInfo, Options) end, KeyInfos + )}; + {error, _Reason} = Error -> + Error + end. + +list_key_info(Client, Options) -> + list_key_info(Client, Options, _Marker = [], _Acc = []). + +list_key_info(Client, Options, Marker, Acc) -> + ListOptions = [{max_keys, ?S3_LIST_LIMIT}] ++ Marker, + case emqx_s3_client:list(Client, ListOptions) of + {ok, Result} -> + ?SLOG(warning, #{msg => "list_key_info", result => Result}), + KeyInfos = proplists:get_value(contents, Result, []), + case proplists:get_value(is_truncated, Result, false) of + true -> + NewMarker = [{marker, proplists:get_value(next_marker, Result)}], + list_key_info(Client, Options, NewMarker, [KeyInfos | Acc]); + false -> + {ok, lists:append(lists:reverse([KeyInfos | Acc]))} + end; + {error, _Reason} = Error -> + Error + end. + +key_info_to_exportinfo(Client, KeyInfo, _Options) -> + Key = proplists:get_value(key, KeyInfo), + {Transfer, Name} = parse_transfer_and_name(Key), + #{ + transfer => Transfer, + name => unicode:characters_to_binary(Name), + uri => emqx_s3_client:uri(Client, Key), + timestamp => datetime_to_epoch_second(proplists:get_value(last_modified, KeyInfo)), + size => proplists:get_value(size, KeyInfo) + }. + +-define(EPOCH_START, 62167219200). + +datetime_to_epoch_second(DateTime) -> + calendar:datetime_to_gregorian_seconds(DateTime) - ?EPOCH_START. + +parse_transfer_and_name(Key) -> + [ClientId, FileId, Name] = string:split(Key, "/", all), + Transfer = { + emqx_ft_fs_util:unescape_filename(ClientId), emqx_ft_fs_util:unescape_filename(FileId) + }, + {Transfer, Name}. diff --git a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf index 4e0870bae..99004d62a 100644 --- a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf +++ b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf @@ -39,9 +39,17 @@ emqx_s3_schema { en: "S3 endpoint port" } } + url_expire_time { + desc { + en: "The time in seconds for which the signed URLs to the S3 objects are valid." + } + label { + en: "Signed URL expiration time" + } + } min_part_size { desc { - en: """The minimum part size for multipart uploads. + en: """The minimum part size for multipart uploads.
Uploaded data will be accumulated in memory until this size is reached.""" } label { @@ -50,7 +58,7 @@ Uploaded data will be accumulated in memory until this size is reached.""" } max_part_size { desc { - en: """The maximum part size for multipart uploads. + en: """The maximum part size for multipart uploads.
S3 uploader won't try to upload parts larger than this size.""" } label { diff --git a/apps/emqx_s3/src/emqx_s3.erl b/apps/emqx_s3/src/emqx_s3.erl index 7c78de979..8798b608d 100644 --- a/apps/emqx_s3/src/emqx_s3.erl +++ b/apps/emqx_s3/src/emqx_s3.erl @@ -41,8 +41,11 @@ -type profile_config() :: #{ bucket := string(), + access_key_id => string(), + secret_access_key => string(), host := string(), port := pos_integer(), + url_expire_time := pos_integer(), acl => acl(), min_part_size => pos_integer(), transport_options => transport_options() diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index 01d677922..a4d470a97 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -8,9 +8,6 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("erlcloud/include/erlcloud_aws.hrl"). --compile(nowarn_export_all). --compile(export_all). - -export([ create/1, @@ -21,11 +18,16 @@ complete_multipart/4, abort_multipart/3, list/2, + uri/2, - format/1 + format/1, + format_request/1 ]). --export_type([client/0]). +-export_type([ + client/0, + headers/0 +]). -type s3_bucket_acl() :: private @@ -35,7 +37,7 @@ | bucket_owner_read | bucket_owner_full_control. --type headers() :: #{binary() => binary()}. +-type headers() :: #{binary() | string() => binary() | string()}. -type key() :: string(). -type part_number() :: non_neg_integer(). @@ -58,6 +60,7 @@ bucket := string(), headers := headers(), acl := s3_bucket_acl(), + url_expire_time := pos_integer(), access_key_id := string() | undefined, secret_access_key := string() | undefined, http_pool := ecpool:pool_name(), @@ -76,6 +79,7 @@ create(Config) -> aws_config => aws_config(Config), upload_options => upload_options(Config), bucket => maps:get(bucket, Config), + url_expire_time => maps:get(url_expire_time, Config), headers => headers(Config) }. @@ -85,7 +89,7 @@ put_object( Key, Value ) -> - try erlcloud_s3:put_object(Bucket, Key, Value, Options, Headers, AwsConfig) of + try erlcloud_s3:put_object(Bucket, key(Key), Value, Options, Headers, AwsConfig) of Props when is_list(Props) -> ok catch @@ -99,7 +103,7 @@ start_multipart( #{bucket := Bucket, upload_options := Options, headers := Headers, aws_config := AwsConfig}, Key ) -> - case erlcloud_s3:start_multipart(Bucket, Key, Options, Headers, AwsConfig) of + case erlcloud_s3:start_multipart(Bucket, key(Key), Options, Headers, AwsConfig) of {ok, Props} -> {ok, proplists:get_value(uploadId, Props)}; {error, Reason} -> @@ -116,7 +120,9 @@ upload_part( PartNumber, Value ) -> - case erlcloud_s3:upload_part(Bucket, Key, UploadId, PartNumber, Value, Headers, AwsConfig) of + case + erlcloud_s3:upload_part(Bucket, key(Key), UploadId, PartNumber, Value, Headers, AwsConfig) + of {ok, Props} -> {ok, proplists:get_value(etag, Props)}; {error, Reason} -> @@ -126,9 +132,12 @@ upload_part( -spec complete_multipart(client(), key(), upload_id(), [etag()]) -> ok_or_error(term()). complete_multipart( - #{bucket := Bucket, headers := Headers, aws_config := AwsConfig}, Key, UploadId, ETags + #{bucket := Bucket, headers := Headers, aws_config := AwsConfig}, + Key, + UploadId, + ETags ) -> - case erlcloud_s3:complete_multipart(Bucket, Key, UploadId, ETags, Headers, AwsConfig) of + case erlcloud_s3:complete_multipart(Bucket, key(Key), UploadId, ETags, Headers, AwsConfig) of ok -> ok; {error, Reason} -> @@ -138,7 +147,7 @@ complete_multipart( -spec abort_multipart(client(), key(), upload_id()) -> ok_or_error(term()). abort_multipart(#{bucket := Bucket, headers := Headers, aws_config := AwsConfig}, Key, UploadId) -> - case erlcloud_s3:abort_multipart(Bucket, Key, UploadId, [], Headers, AwsConfig) of + case erlcloud_s3:abort_multipart(Bucket, key(Key), UploadId, [], Headers, AwsConfig) of ok -> ok; {error, Reason} -> @@ -156,6 +165,10 @@ list(#{bucket := Bucket, aws_config := AwsConfig}, Options) -> {error, Reason} end. +-spec uri(client(), key()) -> iodata(). +uri(#{bucket := Bucket, aws_config := AwsConfig, url_expire_time := ExpireTime}, Key) -> + erlcloud_s3:make_get_url(ExpireTime, Bucket, key(Key), AwsConfig). + -spec format(client()) -> term(). format(#{aws_config := AwsConfig} = Client) -> Client#{aws_config => AwsConfig#aws_config{secret_access_key = "***"}}. @@ -170,13 +183,14 @@ upload_options(Config) -> ]. headers(#{headers := Headers}) -> - maps:to_list(Headers). + string_headers(maps:to_list(Headers)); +headers(#{}) -> + []. aws_config(#{ scheme := Scheme, host := Host, port := Port, - headers := Headers, access_key_id := AccessKeyId, secret_access_key := SecretAccessKey, http_pool := HttpPool, @@ -187,23 +201,29 @@ aws_config(#{ s3_host = Host, s3_port = Port, s3_bucket_access_method = path, + s3_bucket_after_host = true, access_key_id = AccessKeyId, secret_access_key = SecretAccessKey, - http_client = request_fun(Headers, HttpPool), + http_client = request_fun(HttpPool), timeout = Timeout }. --type http_headers() :: [{binary(), binary()}]. -type http_pool() :: term(). --spec request_fun(http_headers(), http_pool()) -> erlcloud_httpc:request_fun(). -request_fun(CustomHeaders, HttpPool) -> +-spec request_fun(http_pool()) -> erlcloud_httpc:request_fun(). +request_fun(HttpPool) -> fun(Url, Method, Headers, Body, Timeout, _Config) -> with_path_and_query_only(Url, fun(PathQuery) -> - JoinedHeaders = join_headers(Headers, CustomHeaders), - Request = make_request(Method, PathQuery, JoinedHeaders, Body), + Request = make_request(Method, PathQuery, binary_headers(Headers), Body), + ?SLOG(warning, #{ + msg => "s3_ehttpc_request", + timeout => Timeout, + pool => HttpPool, + method => Method, + request => Request + }), ehttpc_request(HttpPool, Method, Request, Timeout) end) end. @@ -211,9 +231,9 @@ request_fun(CustomHeaders, HttpPool) -> ehttpc_request(HttpPool, Method, Request, Timeout) -> try ehttpc:request(HttpPool, Method, Request, Timeout) of {ok, StatusCode, RespHeaders} -> - {ok, {{StatusCode, undefined}, string_headers(RespHeaders), undefined}}; + {ok, {{StatusCode, undefined}, erlcloud_string_headers(RespHeaders), undefined}}; {ok, StatusCode, RespHeaders, RespBody} -> - {ok, {{StatusCode, undefined}, string_headers(RespHeaders), RespBody}}; + {ok, {{StatusCode, undefined}, erlcloud_string_headers(RespHeaders), RespBody}}; {error, Reason} -> ?SLOG(error, #{ msg => "s3_ehttpc_request_fail", @@ -258,16 +278,6 @@ make_request(_Method, PathQuery, Headers, Body) -> format_request({PathQuery, Headers, _Body}) -> {PathQuery, Headers, <<"...">>}. -join_headers(Headers, CustomHeaders) -> - MapHeaders = lists:foldl( - fun({K, V}, MHeaders) -> - maps:put(to_binary(K), V, MHeaders) - end, - #{}, - Headers ++ maps:to_list(CustomHeaders) - ), - maps:to_list(MapHeaders). - with_path_and_query_only(Url, Fun) -> case string:split(Url, "//", leading) of [_Scheme, UrlRem] -> @@ -281,13 +291,22 @@ with_path_and_query_only(Url, Fun) -> {error, {invalid_url, Url}} end. +string_headers(Headers) -> + [{to_list_string(K), to_list_string(V)} || {K, V} <- Headers]. + +erlcloud_string_headers(Headers) -> + [{string:to_lower(K), V} || {K, V} <- string_headers(Headers)]. + +binary_headers(Headers) -> + [{to_binary(K), V} || {K, V} <- Headers]. + to_binary(Val) when is_list(Val) -> list_to_binary(Val); to_binary(Val) when is_binary(Val) -> Val. -string_headers(Hdrs) -> - [{string:to_lower(to_list_string(K)), to_list_string(V)} || {K, V} <- Hdrs]. - to_list_string(Val) when is_binary(Val) -> binary_to_list(Val); to_list_string(Val) when is_list(Val) -> Val. + +key(Characters) -> + binary_to_list(unicode:characters_to_binary(Characters)). diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index 09e945edc..3c5eb8f36 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -211,6 +211,7 @@ client_config(ProfileConfig, PoolName) -> scheme => scheme(HTTPOpts), host => maps:get(host, ProfileConfig), port => maps:get(port, ProfileConfig), + url_expire_time => maps:get(url_expire_time, ProfileConfig), headers => maps:get(headers, HTTPOpts, #{}), acl => maps:get(acl, ProfileConfig), bucket => maps:get(bucket, ProfileConfig), diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index 5d76e7120..6db9a5886 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -63,6 +63,15 @@ fields(s3) -> required => true } )}, + {url_expire_time, + mk( + emqx_schema:duration_s(), + #{ + default => "1h", + desc => ?DESC("url_expire_time"), + required => false + } + )}, {min_part_size, mk( emqx_schema:bytesize(), diff --git a/apps/emqx_s3/src/emqx_s3_uploader.erl b/apps/emqx_s3/src/emqx_s3_uploader.erl index 4e3fe15f2..07ae5bd19 100644 --- a/apps/emqx_s3/src/emqx_s3_uploader.erl +++ b/apps/emqx_s3/src/emqx_s3_uploader.erl @@ -12,8 +12,13 @@ start_link/2, write/2, + write/3, + complete/1, - abort/1 + complete/2, + + abort/1, + abort/2 ]). -export([ @@ -26,14 +31,11 @@ format_status/2 ]). --export_type([opts/0, config/0]). +-export_type([opts/0]). -type opts() :: #{ - name := string() -}. - --type config() :: #{ - min_part_size := pos_integer() + key := string(), + headers => emqx_s3_client:headers() }. -type data() :: #{ @@ -58,12 +60,12 @@ start_link(ProfileId, #{key := Key} = Opts) when is_list(Key) -> gen_statem:start_link(?MODULE, [ProfileId, Opts], []). --spec write(pid(), binary()) -> ok_or_error(term()). -write(Pid, WriteData) when is_binary(WriteData) -> +-spec write(pid(), iodata()) -> ok_or_error(term()). +write(Pid, WriteData) -> write(Pid, WriteData, infinity). --spec write(pid(), binary(), timeout()) -> ok_or_error(term()). -write(Pid, WriteData, Timeout) when is_binary(WriteData) -> +-spec write(pid(), iodata(), timeout()) -> ok_or_error(term()). +write(Pid, WriteData, Timeout) -> gen_statem:call(Pid, {write, wrap(WriteData)}, Timeout). -spec complete(pid()) -> ok_or_error(term()). @@ -88,10 +90,10 @@ abort(Pid, Timeout) -> callback_mode() -> handle_event_function. -init([ProfileId, #{key := Key}]) -> +init([ProfileId, #{key := Key} = Opts]) -> process_flag(trap_exit, true), {ok, ClientConfig, UploaderConfig} = emqx_s3_profile_conf:checkout_config(ProfileId), - Client = emqx_s3_client:create(ClientConfig), + Client = client(ClientConfig, Opts), {ok, upload_not_started, #{ profile_id => ProfileId, client => Client, @@ -111,7 +113,7 @@ handle_event({call, From}, {write, WriteDataWrapped}, State, Data0) -> true -> handle_write(State, From, WriteData, Data0); false -> - {keep_state_and_data, {reply, From, {error, {too_large, byte_size(WriteData)}}}} + {keep_state_and_data, {reply, From, {error, {too_large, iolist_size(WriteData)}}}} end; handle_event({call, From}, complete, upload_not_started, Data0) -> case put_object(Data0) of @@ -218,7 +220,6 @@ maybe_upload_part(#{buffer_size := BufferSize, min_part_size := MinPartSize} = D true -> upload_part(Data); false -> - % ct:print("buffer size: ~p, max part size: ~p, no upload", [BufferSize, MinPartSize]), {ok, Data} end. @@ -237,7 +238,6 @@ upload_part( ) -> case emqx_s3_client:upload_part(Client, Key, UploadId, PartNumber, lists:reverse(Buffer)) of {ok, ETag} -> - % ct:print("upload part ~p, etag: ~p", [PartNumber, ETag]), NewData = Data#{ buffer => [], buffer_size => 0, @@ -246,7 +246,6 @@ upload_part( }, {ok, NewData}; {error, _} = Error -> - % ct:print("upload part ~p failed: ~p", [PartNumber, Error]), Error end. @@ -260,7 +259,11 @@ complete_upload( ) -> case upload_part(Data0) of {ok, #{etags := ETags} = Data1} -> - case emqx_s3_client:complete_multipart(Client, Key, UploadId, lists:reverse(ETags)) of + case + emqx_s3_client:complete_multipart( + Client, Key, UploadId, lists:reverse(ETags) + ) + of ok -> {ok, Data1}; {error, _} = Error -> @@ -300,11 +303,11 @@ put_object( Error end. --spec append_buffer(data(), binary()) -> data(). +-spec append_buffer(data(), iodata()) -> data(). append_buffer(#{buffer := Buffer, buffer_size := BufferSize} = Data, WriteData) -> Data#{ buffer => [WriteData | Buffer], - buffer_size => BufferSize + byte_size(WriteData) + buffer_size => BufferSize + iolist_size(WriteData) }. -compile({inline, [wrap/1, unwrap/1]}). @@ -315,4 +318,8 @@ unwrap(WrappedData) -> WrappedData(). is_valid_part(WriteData, #{max_part_size := MaxPartSize, buffer_size := BufferSize}) -> - BufferSize + byte_size(WriteData) =< MaxPartSize. + BufferSize + iolist_size(WriteData) =< MaxPartSize. + +client(Config, Opts) -> + Headers = maps:get(headers, Opts, #{}), + emqx_s3_client:create(Config#{headers => Headers}). diff --git a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl index 3d0d7bb18..f5a507653 100644 --- a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl @@ -83,6 +83,39 @@ t_simple_put(Config) -> ok = emqx_s3_client:put_object(Client, Key, Data). +t_list(Config) -> + Key = ?config(key, Config), + + Client = client(Config), + + ok = emqx_s3_client:put_object(Client, Key, <<"data">>), + + {ok, List} = emqx_s3_client:list(Client, Key), + + [KeyInfo] = proplists:get_value(contents, List), + ?assertMatch( + #{ + key := Key, + size := 4, + etag := _, + last_modified := _ + }, + maps:from_list(KeyInfo) + ). + +t_url(Config) -> + Key = ?config(key, Config), + + Client = client(Config), + ok = emqx_s3_client:put_object(Client, Key, <<"data">>), + + Url = emqx_s3_client:url(Client, Key), + + ?assertMatch( + {ok, {{_StatusLine, 200, "OK"}, _Headers, "data"}}, + httpc:request(Url) + ). + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- From 9bb72ee020b5ed4350f1210aed259357b30d5360 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 3 Apr 2023 20:44:56 +0300 Subject: [PATCH 090/197] feat(ft-s3): fix behaviour specification --- apps/emqx_ft/src/emqx_ft_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index df28de218..697220e1e 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -16,7 +16,7 @@ -module(emqx_ft_schema). -% -behaviour(hocon_schema). +-behaviour(hocon_schema). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). From ca22f11161411bc5d92f2420bc5e2818626a282f Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 3 Apr 2023 20:46:59 +0300 Subject: [PATCH 091/197] feat(ft-s3): fix logging --- apps/emqx_ft/src/emqx_ft.erl | 2 +- apps/emqx_s3/src/emqx_s3_client.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index c5522cdbd..45ec2b933 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -337,7 +337,7 @@ transfer(Msg, FileId) -> {clientid_to_binary(ClientId), FileId}. on_complete(Op, {ChanPid, PacketId}, Transfer, Result) -> - ?SLOG(warning, #{ + ?SLOG(debug, #{ msg => "on_complete", operation => Op, packet_id => PacketId, diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index a4d470a97..b84022357 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -217,7 +217,7 @@ request_fun(HttpPool) -> fun(Url, Method, Headers, Body, Timeout, _Config) -> with_path_and_query_only(Url, fun(PathQuery) -> Request = make_request(Method, PathQuery, binary_headers(Headers), Body), - ?SLOG(warning, #{ + ?SLOG(debug, #{ msg => "s3_ehttpc_request", timeout => Timeout, pool => HttpPool, From 3ffa01e16085eed166faaeeb74710e1a4d6961c2 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 3 Apr 2023 20:47:41 +0300 Subject: [PATCH 092/197] feat(ft-s3): fix logging level --- apps/emqx_ft/src/emqx_ft_api.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index 390c10557..1ea710848 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -69,7 +69,6 @@ schema("/file_transfer/files") -> '/file_transfer/files'(get, #{}) -> case emqx_ft_storage:files() of {ok, Files} -> - ?SLOG(warning, #{msg => "files", files => Files}), {200, #{<<"files">> => lists:map(fun format_file_info/1, Files)}}; {error, _} -> {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} From 07ae50a54b2aed23385c119d6db74f97c2159d95 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 4 Apr 2023 12:53:58 +0300 Subject: [PATCH 093/197] fix(ft-s3): fix release settings --- apps/emqx_ft/src/emqx_ft.app.src | 3 ++- apps/emqx_machine/src/emqx_machine_boot.erl | 13 +++++++++---- mix.exs | 5 +++-- rebar.config.erl | 8 ++++---- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.app.src b/apps/emqx_ft/src/emqx_ft.app.src index 80b4b47dd..058fe984a 100644 --- a/apps/emqx_ft/src/emqx_ft.app.src +++ b/apps/emqx_ft/src/emqx_ft.app.src @@ -6,7 +6,8 @@ {applications, [ kernel, stdlib, - gproc + gproc, + emqx_s3 ]}, {env, []}, {modules, []} diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index feeb1ba75..83824db40 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -146,12 +146,17 @@ basic_reboot_apps() -> emqx_authz, emqx_slow_subs, emqx_auto_subscribe, - emqx_plugins, - emqx_s3 + emqx_plugins ], case emqx_release:edition() of - ce -> CE; - ee -> CE ++ [] + ce -> + CE; + ee -> + CE ++ + [ + emqx_s3, + emqx_ft + ] end. sorted_reboot_apps() -> diff --git a/mix.exs b/mix.exs index 9a8c22e6b..689253a76 100644 --- a/mix.exs +++ b/mix.exs @@ -293,7 +293,6 @@ defmodule EMQXUmbrella.MixProject do emqx_psk: :permanent, emqx_slow_subs: :permanent, emqx_plugins: :permanent, - emqx_ft: :permanent, emqx_mix: :none ] ++ if(enable_quicer?(), do: [quicer: :permanent], else: []) ++ @@ -308,7 +307,9 @@ defmodule EMQXUmbrella.MixProject do emqx_license: :permanent, emqx_ee_conf: :load, emqx_ee_connector: :permanent, - emqx_ee_bridge: :permanent + emqx_ee_bridge: :permanent, + emqx_s3: :permanent, + emqx_ft: :permanent ], else: [] ) diff --git a/rebar.config.erl b/rebar.config.erl index 51a6946dc..158f66cd6 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -400,9 +400,7 @@ relx_apps(ReleaseType, Edition) -> emqx_prometheus, emqx_psk, emqx_slow_subs, - emqx_plugins, - emqx_ft, - emqx_s3 + emqx_plugins ] ++ [quicer || is_quicer_supported()] ++ [bcrypt || provide_bcrypt_release(ReleaseType)] ++ @@ -424,7 +422,9 @@ relx_apps_per_edition(ee) -> emqx_license, {emqx_ee_conf, load}, emqx_ee_connector, - emqx_ee_bridge + emqx_ee_bridge, + emqx_s3, + emqx_ft ]; relx_apps_per_edition(ce) -> []. From 71965d90e41978e9aebdcbf9e32b4281fae1ce14 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 4 Apr 2023 12:55:26 +0300 Subject: [PATCH 094/197] fix(ft-s3): synchronize erlcloud version across apps --- lib-ee/emqx_ee_connector/rebar.config | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index e754bd573..9e4135fac 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -1,12 +1,12 @@ {erl_opts, [debug_info]}. {deps, [ - {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, - {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, - {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, - {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, - {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag,"3.5.16-emqx-1"}}}, - {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, - {emqx, {path, "../../apps/emqx"}} + {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, + {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, + {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, + {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, + {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag, "3.6.7-emqx-1"}}}, + {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, + {emqx, {path, "../../apps/emqx"}} ]}. {shell, [ From 4fbabe5a761461340678a6604ab7f6be35422444 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 4 Apr 2023 13:05:02 +0300 Subject: [PATCH 095/197] fix(ft-s3): fix atom and variable naming --- apps/emqx_s3/src/emqx_s3_client.erl | 2 +- apps/emqx_s3/src/emqx_s3_profile_http_pools.erl | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index b84022357..99e2de4da 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -105,7 +105,7 @@ start_multipart( ) -> case erlcloud_s3:start_multipart(Bucket, key(Key), Options, Headers, AwsConfig) of {ok, Props} -> - {ok, proplists:get_value(uploadId, Props)}; + {ok, proplists:get_value('uploadId', Props)}; {error, Reason} -> ?SLOG(debug, #{msg => "start_multipart_fail", key => Key, reason => Reason}), {error, Reason} diff --git a/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl b/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl index e1b36c3be..73774624e 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl @@ -96,11 +96,11 @@ set_outdated(ProfileId, PoolName, Timeout) -> outdated(ProfileId) -> Now = erlang:monotonic_time(millisecond), MS = ets:fun2ms( - fun(#pool{key = {ProfileId_, PoolName}, deadline = Deadline_}) when - ProfileId_ =:= ProfileId andalso - Deadline_ =/= undefined andalso Deadline_ < Now + fun(#pool{key = {CurProfileId, CurPoolName}, deadline = CurDeadline}) when + CurProfileId =:= ProfileId andalso + CurDeadline =/= undefined andalso CurDeadline < Now -> - PoolName + CurPoolName end ), ets:select(?TAB, MS). @@ -109,8 +109,8 @@ outdated(ProfileId) -> [pool_name()]. all(ProfileId) -> MS = ets:fun2ms( - fun(#pool{key = {ProfileId_, PoolName}}) when ProfileId_ =:= ProfileId -> - PoolName + fun(#pool{key = {CurProfileId, CurPoolName}}) when CurProfileId =:= ProfileId -> + CurPoolName end ), ets:select(?TAB, MS). From c7865e5eaea74a60f1f8a214040bb3e5d0ed8517 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 4 Apr 2023 14:19:58 +0300 Subject: [PATCH 096/197] fix(ft-s3): fix dependency dependencies --- apps/emqx_s3/rebar.config | 2 +- lib-ee/emqx_ee_connector/rebar.config | 2 +- mix.exs | 9 +- rebar.config | 157 ++++++++++++++------------ 4 files changed, 96 insertions(+), 74 deletions(-) diff --git a/apps/emqx_s3/rebar.config b/apps/emqx_s3/rebar.config index f8e4d4e42..92ab7895e 100644 --- a/apps/emqx_s3/rebar.config +++ b/apps/emqx_s3/rebar.config @@ -1,6 +1,6 @@ {deps, [ {emqx, {path, "../../apps/emqx"}}, - {erlcloud, {git, "https://github.com/savonarola/erlcloud", {tag, "3.6.7-emqx-1"}}} + {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.7-emqx-1"}}} ]}. {project_plugins, [erlfmt]}. diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index 9e4135fac..f36eb0ad6 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -4,7 +4,7 @@ {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, - {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag, "3.6.7-emqx-1"}}}, + {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.7-emqx-1"}}}, {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, {emqx, {path, "../../apps/emqx"}} ]}. diff --git a/mix.exs b/mix.exs index 689253a76..8cda3b9c0 100644 --- a/mix.exs +++ b/mix.exs @@ -93,7 +93,14 @@ defmodule EMQXUmbrella.MixProject do github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true}, # in conflict by grpc and eetcd {:gpb, "4.19.5", override: true, runtime: false}, - {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true} + {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true}, + {:erlcloud, github: "emqx/erlcloud", tag: "3.6.7-emqx-1", override: true}, + # erlcloud's rebar.config requires rebar3 and does not support Mix, + # so it tries to fetch deps from git. We need to override this. + {:lhttpc, "1.6.2", override: true}, + {:eini, "1.2.9", override: true}, + {:base16, "1.0.0", override: true} + # end of erlcloud's deps ] ++ emqx_apps(profile_info, version) ++ enterprise_deps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep() diff --git a/rebar.config b/rebar.config index 0cc143e71..5cedb47ff 100644 --- a/rebar.config +++ b/rebar.config @@ -7,24 +7,35 @@ %% with rebar.config.erl module. Final result is written to %% rebar.config.rendered if environment DEBUG is set. -{edoc_opts, [{preprocess,true}]}. -{erl_opts, [warn_unused_vars,warn_shadow_vars,warn_unused_import, - warn_obsolete_guard,compressed, nowarn_unused_import, - {d, snk_kind, msg} - ]}. +{edoc_opts, [{preprocess, true}]}. +{erl_opts, [ + warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard, + compressed, + nowarn_unused_import, + {d, snk_kind, msg} +]}. -{xref_checks,[undefined_function_calls,undefined_functions,locals_not_used, - deprecated_function_calls,warnings_as_errors,deprecated_functions]}. +{xref_checks, [ + undefined_function_calls, + undefined_functions, + locals_not_used, + deprecated_function_calls, + warnings_as_errors, + deprecated_functions +]}. %% Check the forbidden mnesia calls: -{xref_queries, - [ {"E || \"mnesia\":\"dirty_delete.*\"/\".*\" : Fun", []} - , {"E || \"mnesia\":\"transaction\"/\".*\" : Fun", []} - , {"E || \"mnesia\":\"async_dirty\"/\".*\" : Fun", []} - , {"E || \"mnesia\":\"clear_table\"/\".*\" : Fun", []} - , {"E || \"mnesia\":\"create_table\"/\".*\" : Fun", []} - , {"E || \"mnesia\":\"delete_table\"/\".*\" : Fun", []} - ]}. +{xref_queries, [ + {"E || \"mnesia\":\"dirty_delete.*\"/\".*\" : Fun", []}, + {"E || \"mnesia\":\"transaction\"/\".*\" : Fun", []}, + {"E || \"mnesia\":\"async_dirty\"/\".*\" : Fun", []}, + {"E || \"mnesia\":\"clear_table\"/\".*\" : Fun", []}, + {"E || \"mnesia\":\"create_table\"/\".*\" : Fun", []}, + {"E || \"mnesia\":\"delete_table\"/\".*\" : Fun", []} +]}. {dialyzer, [ {warnings, [unmatched_returns, error_handling]}, @@ -32,72 +43,76 @@ {plt_prefix, "emqx_dialyzer"}, {plt_apps, all_apps}, {statistics, true} - ] -}. +]}. {cover_opts, [verbose]}. {cover_export_enabled, true}. {cover_excl_mods, - [ %% generated protobuf modules - emqx_exproto_pb, - emqx_exhook_pb, - %% taken almost as-is from OTP - emqx_ssl_crl_cache - ]}. + %% generated protobuf modules + [ + emqx_exproto_pb, + emqx_exhook_pb, + %% taken almost as-is from OTP + emqx_ssl_crl_cache + ]}. {provider_hooks, [{pre, [{release, {relup_helper, gen_appups}}]}]}. -{post_hooks,[]}. +{post_hooks, []}. -{deps, - [ {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}} - , {redbug, "2.0.8"} - , {covertool, {git, "https://github.com/zmstone/covertool", {tag, "2.0.4.1"}}} - , {gpb, "4.19.5"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps - , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}} - , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}} - , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.7"}}} - , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} - , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} - , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}} - , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-9"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}} - , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} - , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} - , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}} - , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.9"}}} - , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} - , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}} - , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.5"}}} - , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x - , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} - , {getopt, "1.0.2"} - , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.0"}}} - , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} - , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} - , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} - , {telemetry, "1.1.0"} - , {hackney, {git, "https://github.com/emqx/hackney.git", {tag, "1.18.1-1"}}} - ]}. +{deps, [ + {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}}, + {redbug, "2.0.8"}, + {covertool, {git, "https://github.com/zmstone/covertool", {tag, "2.0.4.1"}}}, + %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps + {gpb, "4.19.5"}, + {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}}, + {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}}, + {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.7"}}}, + {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}, + {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, + {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, + {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, + {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-9"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}}, + {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, + {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}}, + {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}}, + {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}}, + {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.9"}}}, + {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}}, + {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}}, + % NOTE: depends on recon 2.5.x + {observer_cli, "1.7.1"}, + {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}, + {getopt, "1.0.2"}, + {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.0"}}}, + {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, + {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}, + {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}, + {telemetry, "1.1.0"}, + {hackney, {git, "https://github.com/emqx/hackney.git", {tag, "1.18.1-1"}}} +]}. {xref_ignores, - [ %% schema registry is for enterprise - {emqx_schema_registry,get_all_schemas,0}, - {emqx_schema_api,format_schema,1}, - {emqx_schema_api,make_schema_params,1}, - {emqx_schema_parser,decode,3}, - {emqx_schema_parser,encode,3}, - {emqx_schema_registry,add_schema,1}, - emqx_exhook_pb, % generated code for protobuf - emqx_exproto_pb % generated code for protobuf -]}. + %% schema registry is for enterprise + [ + {emqx_schema_registry, get_all_schemas, 0}, + {emqx_schema_api, format_schema, 1}, + {emqx_schema_api, make_schema_params, 1}, + {emqx_schema_parser, decode, 3}, + {emqx_schema_parser, encode, 3}, + {emqx_schema_registry, add_schema, 1}, + % generated code for protobuf + emqx_exhook_pb, + % generated code for protobuf + emqx_exproto_pb + ]}. -{project_plugins, - [ erlfmt, +{project_plugins, [ + erlfmt, {rebar3_hex, "7.0.2"}, - {rebar3_sbom, - {git, "https://github.com/emqx/rebar3_sbom.git", {tag, "v0.6.1-1"}}} + {rebar3_sbom, {git, "https://github.com/emqx/rebar3_sbom.git", {tag, "v0.6.1-1"}}} ]}. From 7d13862da54ad5e4931d31c8a5d783488cfead03 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 4 Apr 2023 15:25:53 +0300 Subject: [PATCH 097/197] fix(ft-s3): use uploader specific headers only for object creation --- apps/emqx_s3/src/emqx_s3_client.erl | 21 +++++++++++++++++++-- apps/emqx_s3/src/emqx_s3_uploader.erl | 17 +++++++++-------- apps/emqx_s3/test/emqx_s3_client_SUITE.erl | 2 +- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index 99e2de4da..8e54d90f9 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -12,8 +12,10 @@ create/1, put_object/3, + put_object/4, start_multipart/2, + start_multipart/3, upload_part/5, complete_multipart/4, abort_multipart/3, @@ -84,12 +86,18 @@ create(Config) -> }. -spec put_object(client(), key(), iodata()) -> ok_or_error(term()). +put_object(Client, Key, Value) -> + put_object(Client, #{}, Key, Value). + +-spec put_object(client(), headers(), key(), iodata()) -> ok_or_error(term()). put_object( #{bucket := Bucket, upload_options := Options, headers := Headers, aws_config := AwsConfig}, + SpecialHeaders, Key, Value ) -> - try erlcloud_s3:put_object(Bucket, key(Key), Value, Options, Headers, AwsConfig) of + AllHeaders = join_headers(Headers, SpecialHeaders), + try erlcloud_s3:put_object(Bucket, key(Key), Value, Options, AllHeaders, AwsConfig) of Props when is_list(Props) -> ok catch @@ -99,11 +107,17 @@ put_object( end. -spec start_multipart(client(), key()) -> ok_or_error(upload_id(), term()). +start_multipart(Client, Key) -> + start_multipart(Client, #{}, Key). + +-spec start_multipart(client(), headers(), key()) -> ok_or_error(upload_id(), term()). start_multipart( #{bucket := Bucket, upload_options := Options, headers := Headers, aws_config := AwsConfig}, + SpecialHeaders, Key ) -> - case erlcloud_s3:start_multipart(Bucket, key(Key), Options, Headers, AwsConfig) of + AllHeaders = join_headers(Headers, SpecialHeaders), + case erlcloud_s3:start_multipart(Bucket, key(Key), Options, AllHeaders, AwsConfig) of {ok, Props} -> {ok, proplists:get_value('uploadId', Props)}; {error, Reason} -> @@ -300,6 +314,9 @@ erlcloud_string_headers(Headers) -> binary_headers(Headers) -> [{to_binary(K), V} || {K, V} <- Headers]. +join_headers(Headers, SpecialHeaders) -> + Headers ++ string_headers(maps:to_list(SpecialHeaders)). + to_binary(Val) when is_list(Val) -> list_to_binary(Val); to_binary(Val) when is_binary(Val) -> Val. diff --git a/apps/emqx_s3/src/emqx_s3_uploader.erl b/apps/emqx_s3/src/emqx_s3_uploader.erl index 07ae5bd19..f6414669d 100644 --- a/apps/emqx_s3/src/emqx_s3_uploader.erl +++ b/apps/emqx_s3/src/emqx_s3_uploader.erl @@ -93,10 +93,11 @@ callback_mode() -> handle_event_function. init([ProfileId, #{key := Key} = Opts]) -> process_flag(trap_exit, true), {ok, ClientConfig, UploaderConfig} = emqx_s3_profile_conf:checkout_config(ProfileId), - Client = client(ClientConfig, Opts), + Client = client(ClientConfig), {ok, upload_not_started, #{ profile_id => ProfileId, client => Client, + headers => maps:get(headers, Opts, #{}), key => Key, buffer => [], buffer_size => 0, @@ -205,8 +206,8 @@ maybe_start_upload(#{buffer_size := BufferSize, min_part_size := MinPartSize} = end. -spec start_upload(data()) -> {started, data()} | {error, term()}. -start_upload(#{client := Client, key := Key} = Data) -> - case emqx_s3_client:start_multipart(Client, Key) of +start_upload(#{client := Client, key := Key, headers := Headers} = Data) -> + case emqx_s3_client:start_multipart(Client, Headers, Key) of {ok, UploadId} -> NewData = Data#{upload_id => UploadId}, {started, NewData}; @@ -293,10 +294,11 @@ put_object( #{ client := Client, key := Key, - buffer := Buffer + buffer := Buffer, + headers := Headers } ) -> - case emqx_s3_client:put_object(Client, Key, lists:reverse(Buffer)) of + case emqx_s3_client:put_object(Client, Headers, Key, lists:reverse(Buffer)) of ok -> ok; {error, _} = Error -> @@ -320,6 +322,5 @@ unwrap(WrappedData) -> is_valid_part(WriteData, #{max_part_size := MaxPartSize, buffer_size := BufferSize}) -> BufferSize + iolist_size(WriteData) =< MaxPartSize. -client(Config, Opts) -> - Headers = maps:get(headers, Opts, #{}), - emqx_s3_client:create(Config#{headers => Headers}). +client(Config) -> + emqx_s3_client:create(Config). diff --git a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl index f5a507653..ec7d5ebcf 100644 --- a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl @@ -109,7 +109,7 @@ t_url(Config) -> Client = client(Config), ok = emqx_s3_client:put_object(Client, Key, <<"data">>), - Url = emqx_s3_client:url(Client, Key), + Url = emqx_s3_client:uri(Client, Key), ?assertMatch( {ok, {{_StatusLine, 200, "OK"}, _Headers, "data"}}, From d7a85242de46fa47f4b0716ce8f644bb8c6672bc Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 4 Apr 2023 17:12:52 +0300 Subject: [PATCH 098/197] fix(ft-s3): fix s3 listing --- apps/emqx/src/emqx_channel.erl | 7 ------- apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl | 5 ++++- apps/emqx_s3/src/emqx_s3_client.erl | 11 +++++++++++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index c67c02d66..9d4b7eac2 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -641,13 +641,6 @@ process_connect( %%-------------------------------------------------------------------- process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) -> - ?SLOG( - warning, - #{ - packet => Packet, - packet_id => PacketId - } - ), case pipeline( [ diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index 1c97520e3..3258a5cd1 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -155,7 +155,7 @@ list_key_info(Client, Options, Marker, Acc) -> KeyInfos = proplists:get_value(contents, Result, []), case proplists:get_value(is_truncated, Result, false) of true -> - NewMarker = [{marker, proplists:get_value(next_marker, Result)}], + NewMarker = next_marker(KeyInfos), list_key_info(Client, Options, NewMarker, [KeyInfos | Acc]); false -> {ok, lists:append(lists:reverse([KeyInfos | Acc]))} @@ -164,6 +164,9 @@ list_key_info(Client, Options, Marker, Acc) -> Error end. +next_marker(KeyInfos) -> + [{marker, proplists:get_value(key, lists:last(KeyInfos))}]. + key_info_to_exportinfo(Client, KeyInfo, _Options) -> Key = proplists:get_value(key, KeyInfo), {Transfer, Name} = parse_transfer_and_name(Key), diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index 8e54d90f9..8639e0a25 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -245,8 +245,19 @@ request_fun(HttpPool) -> ehttpc_request(HttpPool, Method, Request, Timeout) -> try ehttpc:request(HttpPool, Method, Request, Timeout) of {ok, StatusCode, RespHeaders} -> + ?SLOG(debug, #{ + msg => "s3_ehttpc_request_ok", + status_code => StatusCode, + headers => RespHeaders + }), {ok, {{StatusCode, undefined}, erlcloud_string_headers(RespHeaders), undefined}}; {ok, StatusCode, RespHeaders, RespBody} -> + ?SLOG(debug, #{ + msg => "s3_ehttpc_request_ok", + status_code => StatusCode, + headers => RespHeaders, + body => RespBody + }), {ok, {{StatusCode, undefined}, erlcloud_string_headers(RespHeaders), RespBody}}; {error, Reason} -> ?SLOG(error, #{ From be99242e3279ed4a7dff72353002cda3ee0e83f1 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 4 Apr 2023 23:11:33 +0300 Subject: [PATCH 099/197] fix(ft-s3): fix review comments --- .ci/docker-compose-file/.env | 1 + .../docker-compose-minio-tcp.yaml | 2 +- .../docker-compose-minio-tls.yaml | 2 +- .github/workflows/run_test_cases.yaml | 1 + apps/emqx_ft/src/emqx_ft_conf.erl | 2 + .../src/emqx_ft_storage_exporter_s3.erl | 5 +- apps/emqx_s3/src/emqx_s3.erl | 19 +-- apps/emqx_s3/src/emqx_s3.hrl | 13 ++ apps/emqx_s3/src/emqx_s3_client.erl | 122 ++++++++++++------ apps/emqx_s3/src/emqx_s3_profile_conf.erl | 45 +++---- .../src/emqx_s3_profile_uploader_sup.erl | 14 +- apps/emqx_s3/src/emqx_s3_uploader.erl | 14 +- 12 files changed, 151 insertions(+), 89 deletions(-) create mode 100644 apps/emqx_s3/src/emqx_s3.hrl diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env index 956750e00..9675937d8 100644 --- a/.ci/docker-compose-file/.env +++ b/.ci/docker-compose-file/.env @@ -7,5 +7,6 @@ INFLUXDB_TAG=2.5.0 TDENGINE_TAG=3.0.2.4 DYNAMO_TAG=1.21.0 CASSANDRA_TAG=3.11.6 +MINIO_TAG=RELEASE.2023-03-20T20-16-18Z TARGET=emqx/emqx diff --git a/.ci/docker-compose-file/docker-compose-minio-tcp.yaml b/.ci/docker-compose-file/docker-compose-minio-tcp.yaml index 93e1c4ead..fa78e4426 100644 --- a/.ci/docker-compose-file/docker-compose-minio-tcp.yaml +++ b/.ci/docker-compose-file/docker-compose-minio-tcp.yaml @@ -3,7 +3,7 @@ version: '3.7' services: minio: hostname: minio - image: quay.io/minio/minio:RELEASE.2023-03-20T20-16-18Z + image: quay.io/minio/minio:${MINIO_TAG} command: server --address ":9000" --console-address ":9001" /minio-data expose: - "9000" diff --git a/.ci/docker-compose-file/docker-compose-minio-tls.yaml b/.ci/docker-compose-file/docker-compose-minio-tls.yaml index 2e7a6bea5..4999cccb5 100644 --- a/.ci/docker-compose-file/docker-compose-minio-tls.yaml +++ b/.ci/docker-compose-file/docker-compose-minio-tls.yaml @@ -3,7 +3,7 @@ version: '3.7' services: minio_tls: hostname: minio-tls - image: quay.io/minio/minio:RELEASE.2023-03-20T20-16-18Z + image: quay.io/minio/minio:${MINIO_TAG} command: server --certs-dir /etc/certs --address ":9100" --console-address ":9101" /minio-data volumes: - ./certs/server.crt:/etc/certs/public.crt diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 8702cd849..3360bec6c 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -168,6 +168,7 @@ jobs: REDIS_TAG: "7.0" INFLUXDB_TAG: "2.5.0" TDENGINE_TAG: "3.0.2.4" + MINIO_TAG: "RELEASE.2023-03-20T20-16-18Z" PROFILE: ${{ matrix.profile }} CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }} diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 6ce10408e..0e8bcc193 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -20,6 +20,8 @@ -behaviour(emqx_config_handler). +-include_lib("emqx/include/logger.hrl"). + %% Accessors -export([storage/0]). -export([gc_interval/1]). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index 3258a5cd1..24c3c52e5 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -49,7 +49,7 @@ transfer := transfer() }. --define(S3_PROFILE_ID, <<"emqx_ft_storage_exporter_s3">>). +-define(S3_PROFILE_ID, ?MODULE). -define(FILEMETA_VSN, <<"1">>). -define(S3_LIST_LIMIT, 500). @@ -66,6 +66,7 @@ start_export(_Options, Transfer, Filemeta) -> }, case emqx_s3:start_uploader(?S3_PROFILE_ID, Options) of {ok, Pid} -> + true = erlang:link(Pid), {ok, #{filemeta => Filemeta, pid => Pid}}; {error, _Reason} = Error -> Error @@ -151,7 +152,7 @@ list_key_info(Client, Options, Marker, Acc) -> ListOptions = [{max_keys, ?S3_LIST_LIMIT}] ++ Marker, case emqx_s3_client:list(Client, ListOptions) of {ok, Result} -> - ?SLOG(warning, #{msg => "list_key_info", result => Result}), + ?SLOG(debug, #{msg => "list_key_info", result => Result}), KeyInfos = proplists:get_value(contents, Result, []), case proplists:get_value(is_truncated, Result, false) of true -> diff --git a/apps/emqx_s3/src/emqx_s3.erl b/apps/emqx_s3/src/emqx_s3.erl index 8798b608d..15f63fbd5 100644 --- a/apps/emqx_s3/src/emqx_s3.erl +++ b/apps/emqx_s3/src/emqx_s3.erl @@ -16,10 +16,11 @@ -export_type([ profile_id/0, - profile_config/0 + profile_config/0, + acl/0 ]). --type profile_id() :: term(). +-type profile_id() :: atom() | binary(). -type acl() :: private @@ -30,9 +31,9 @@ | bucket_owner_full_control. -type transport_options() :: #{ + headers => map(), connect_timeout => pos_integer(), enable_pipelining => pos_integer(), - headers => map(), max_retries => pos_integer(), pool_size => pos_integer(), pool_type => atom(), @@ -51,12 +52,14 @@ transport_options => transport_options() }. +-define(IS_PROFILE_ID(ProfileId), (is_atom(ProfileId) orelse is_binary(ProfileId))). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- -spec start_profile(profile_id(), profile_config()) -> ok_or_error(term()). -start_profile(ProfileId, ProfileConfig) -> +start_profile(ProfileId, ProfileConfig) when ?IS_PROFILE_ID(ProfileId) -> case emqx_s3_sup:start_profile(ProfileId, ProfileConfig) of {ok, _} -> ok; @@ -65,21 +68,21 @@ start_profile(ProfileId, ProfileConfig) -> end. -spec stop_profile(profile_id()) -> ok_or_error(term()). -stop_profile(ProfileId) -> +stop_profile(ProfileId) when ?IS_PROFILE_ID(ProfileId) -> emqx_s3_sup:stop_profile(ProfileId). -spec update_profile(profile_id(), profile_config()) -> ok_or_error(term()). -update_profile(ProfileId, ProfileConfig) -> +update_profile(ProfileId, ProfileConfig) when ?IS_PROFILE_ID(ProfileId) -> emqx_s3_profile_conf:update_config(ProfileId, ProfileConfig). -spec start_uploader(profile_id(), emqx_s3_uploader:opts()) -> supervisor:start_ret() | {error, profile_not_found}. -start_uploader(ProfileId, Opts) -> +start_uploader(ProfileId, Opts) when ?IS_PROFILE_ID(ProfileId) -> emqx_s3_profile_uploader_sup:start_uploader(ProfileId, Opts). -spec with_client(profile_id(), fun((emqx_s3_client:client()) -> Result)) -> {error, profile_not_found} | Result. -with_client(ProfileId, Fun) when is_function(Fun, 1) -> +with_client(ProfileId, Fun) when is_function(Fun, 1) andalso ?IS_PROFILE_ID(ProfileId) -> case emqx_s3_profile_conf:checkout_config(ProfileId) of {ok, ClientConfig, _UploadConfig} -> try diff --git a/apps/emqx_s3/src/emqx_s3.hrl b/apps/emqx_s3/src/emqx_s3.hrl new file mode 100644 index 000000000..9a2cf8d2f --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3.hrl @@ -0,0 +1,13 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-define(VIA_GPROC(Id), {via, gproc, {n, l, Id}}). + +-define(SAFE_CALL_VIA_GPROC(Id, Message, Timeout, NoProcError), + try gen_server:call(?VIA_GPROC(Id), Message, Timeout) of + Result -> Result + catch + exit:{noproc, _} -> {error, NoProcError} + end +). diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index 8639e0a25..c84f32dc3 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -31,22 +31,14 @@ headers/0 ]). --type s3_bucket_acl() :: - private - | public_read - | public_read_write - | authenticated_read - | bucket_owner_read - | bucket_owner_full_control. - --type headers() :: #{binary() | string() => binary() | string()}. +-type headers() :: #{binary() | string() => iodata()}. -type key() :: string(). -type part_number() :: non_neg_integer(). -type upload_id() :: string(). -type etag() :: string(). --type upload_options() :: list({acl, s3_bucket_acl()}). +-type upload_options() :: list({acl, emqx_s3:acl()}). -opaque client() :: #{ aws_config := aws_config(), @@ -61,11 +53,11 @@ port := part_number(), bucket := string(), headers := headers(), - acl := s3_bucket_acl(), + acl := emqx_s3:acl(), url_expire_time := pos_integer(), access_key_id := string() | undefined, secret_access_key := string() | undefined, - http_pool := ecpool:pool_name(), + http_pool := ehttpc:pool_name(), request_timeout := timeout() }. @@ -97,7 +89,7 @@ put_object( Value ) -> AllHeaders = join_headers(Headers, SpecialHeaders), - try erlcloud_s3:put_object(Bucket, key(Key), Value, Options, AllHeaders, AwsConfig) of + try erlcloud_s3:put_object(Bucket, erlcloud_key(Key), Value, Options, AllHeaders, AwsConfig) of Props when is_list(Props) -> ok catch @@ -117,9 +109,9 @@ start_multipart( Key ) -> AllHeaders = join_headers(Headers, SpecialHeaders), - case erlcloud_s3:start_multipart(Bucket, key(Key), Options, AllHeaders, AwsConfig) of + case erlcloud_s3:start_multipart(Bucket, erlcloud_key(Key), Options, AllHeaders, AwsConfig) of {ok, Props} -> - {ok, proplists:get_value('uploadId', Props)}; + {ok, response_property('uploadId', Props)}; {error, Reason} -> ?SLOG(debug, #{msg => "start_multipart_fail", key => Key, reason => Reason}), {error, Reason} @@ -135,10 +127,12 @@ upload_part( Value ) -> case - erlcloud_s3:upload_part(Bucket, key(Key), UploadId, PartNumber, Value, Headers, AwsConfig) + erlcloud_s3:upload_part( + Bucket, erlcloud_key(Key), UploadId, PartNumber, Value, Headers, AwsConfig + ) of {ok, Props} -> - {ok, proplists:get_value(etag, Props)}; + {ok, response_property(etag, Props)}; {error, Reason} -> ?SLOG(debug, #{msg => "upload_part_fail", key => Key, reason => Reason}), {error, Reason} @@ -151,7 +145,11 @@ complete_multipart( UploadId, ETags ) -> - case erlcloud_s3:complete_multipart(Bucket, key(Key), UploadId, ETags, Headers, AwsConfig) of + case + erlcloud_s3:complete_multipart( + Bucket, erlcloud_key(Key), UploadId, ETags, Headers, AwsConfig + ) + of ok -> ok; {error, Reason} -> @@ -161,7 +159,7 @@ complete_multipart( -spec abort_multipart(client(), key(), upload_id()) -> ok_or_error(term()). abort_multipart(#{bucket := Bucket, headers := Headers, aws_config := AwsConfig}, Key, UploadId) -> - case erlcloud_s3:abort_multipart(Bucket, key(Key), UploadId, [], Headers, AwsConfig) of + case erlcloud_s3:abort_multipart(Bucket, erlcloud_key(Key), UploadId, [], Headers, AwsConfig) of ok -> ok; {error, Reason} -> @@ -181,7 +179,7 @@ list(#{bucket := Bucket, aws_config := AwsConfig}, Options) -> -spec uri(client(), key()) -> iodata(). uri(#{bucket := Bucket, aws_config := AwsConfig, url_expire_time := ExpireTime}, Key) -> - erlcloud_s3:make_get_url(ExpireTime, Bucket, key(Key), AwsConfig). + erlcloud_s3:make_get_url(ExpireTime, Bucket, erlcloud_key(Key), AwsConfig). -spec format(client()) -> term(). format(#{aws_config := AwsConfig} = Client) -> @@ -197,7 +195,7 @@ upload_options(Config) -> ]. headers(#{headers := Headers}) -> - string_headers(maps:to_list(Headers)); + headers_user_to_erlcloud_request(Headers); headers(#{}) -> []. @@ -230,7 +228,9 @@ aws_config(#{ request_fun(HttpPool) -> fun(Url, Method, Headers, Body, Timeout, _Config) -> with_path_and_query_only(Url, fun(PathQuery) -> - Request = make_request(Method, PathQuery, binary_headers(Headers), Body), + Request = make_request( + Method, PathQuery, headers_erlcloud_request_to_ehttpc(Headers), Body + ), ?SLOG(debug, #{ msg => "s3_ehttpc_request", timeout => Timeout, @@ -243,6 +243,13 @@ request_fun(HttpPool) -> end. ehttpc_request(HttpPool, Method, Request, Timeout) -> + ?SLOG(debug, #{ + msg => "s3_ehttpc_request", + timeout => Timeout, + pool => HttpPool, + method => Method, + request => format_request(Request) + }), try ehttpc:request(HttpPool, Method, Request, Timeout) of {ok, StatusCode, RespHeaders} -> ?SLOG(debug, #{ @@ -250,7 +257,9 @@ ehttpc_request(HttpPool, Method, Request, Timeout) -> status_code => StatusCode, headers => RespHeaders }), - {ok, {{StatusCode, undefined}, erlcloud_string_headers(RespHeaders), undefined}}; + {ok, { + {StatusCode, undefined}, headers_ehttpc_to_erlcloud_response(RespHeaders), undefined + }}; {ok, StatusCode, RespHeaders, RespBody} -> ?SLOG(debug, #{ msg => "s3_ehttpc_request_ok", @@ -258,7 +267,9 @@ ehttpc_request(HttpPool, Method, Request, Timeout) -> headers => RespHeaders, body => RespBody }), - {ok, {{StatusCode, undefined}, erlcloud_string_headers(RespHeaders), RespBody}}; + {ok, { + {StatusCode, undefined}, headers_ehttpc_to_erlcloud_response(RespHeaders), RespBody + }}; {error, Reason} -> ?SLOG(error, #{ msg => "s3_ehttpc_request_fail", @@ -290,10 +301,10 @@ ehttpc_request(HttpPool, Method, Request, Timeout) -> end. -define(IS_BODY_EMPTY(Body), (Body =:= undefined orelse Body =:= <<>>)). --define(NEEDS_BODY(Method), (Method =:= get orelse Method =:= head orelse Method =:= delete)). +-define(NEEDS_NO_BODY(Method), (Method =:= get orelse Method =:= head orelse Method =:= delete)). make_request(Method, PathQuery, Headers, Body) when - ?IS_BODY_EMPTY(Body) andalso ?NEEDS_BODY(Method) + ?IS_BODY_EMPTY(Body) andalso ?NEEDS_NO_BODY(Method) -> {PathQuery, Headers}; make_request(_Method, PathQuery, Headers, Body) when ?IS_BODY_EMPTY(Body) -> @@ -301,7 +312,8 @@ make_request(_Method, PathQuery, Headers, Body) when ?IS_BODY_EMPTY(Body) -> make_request(_Method, PathQuery, Headers, Body) -> {PathQuery, Headers, Body}. -format_request({PathQuery, Headers, _Body}) -> {PathQuery, Headers, <<"...">>}. +format_request({PathQuery, Headers, _Body}) -> {PathQuery, Headers, <<"...">>}; +format_request({PathQuery, Headers}) -> {PathQuery, Headers}. with_path_and_query_only(Url, Fun) -> case string:split(Url, "//", leading) of @@ -316,17 +328,41 @@ with_path_and_query_only(Url, Fun) -> {error, {invalid_url, Url}} end. -string_headers(Headers) -> - [{to_list_string(K), to_list_string(V)} || {K, V} <- Headers]. +%% We need some header conversions to tie the emqx_s3, erlcloud and ehttpc APIs together. -erlcloud_string_headers(Headers) -> - [{string:to_lower(K), V} || {K, V} <- string_headers(Headers)]. +%% The request header flow is: -binary_headers(Headers) -> - [{to_binary(K), V} || {K, V} <- Headers]. +%% UserHeaders -> [emqx_s3_client API] -> ErlcloudRequestHeaders0 -> +%% -> [erlcloud API] -> ErlcloudRequestHeaders1 -> [emqx_s3_client injected request_fun] -> +%% -> EhttpcRequestHeaders -> [ehttpc API] -join_headers(Headers, SpecialHeaders) -> - Headers ++ string_headers(maps:to_list(SpecialHeaders)). +%% The response header flow is: + +%% [ehttpc API] -> EhttpcResponseHeaders -> [emqx_s3_client injected request_fun] -> +%% -> ErlcloudResponseHeaders0 -> [erlcloud API] -> [emqx_s3_client API] + +%% UserHeders (emqx_s3 API headers) are maps with string/binary keys. +%% ErlcloudRequestHeaders are lists of tuples with string keys and iodata values +%% ErlcloudResponseHeders are lists of tuples with lower case string keys and iodata values. +%% EhttpcHeaders are lists of tuples with binary keys and iodata values. + +%% Users provide headers as a map, but erlcloud expects a list of tuples with string keys and values. +headers_user_to_erlcloud_request(UserHeaders) -> + [{to_list_string(K), V} || {K, V} <- maps:to_list(UserHeaders)]. + +%% Ehttpc returns operates on headers as a list of tuples with binary keys. +%% Erlcloud expects a list of tuples with string values and lowcase string keys +%% from the underlying http library. +headers_ehttpc_to_erlcloud_response(EhttpcHeaders) -> + [{string:to_lower(to_list_string(K)), to_list_string(V)} || {K, V} <- EhttpcHeaders]. + +%% Ehttpc expects a list of tuples with binary keys. +%% Erlcloud provides a list of tuples with string keys. +headers_erlcloud_request_to_ehttpc(ErlcloudHeaders) -> + [{to_binary(K), V} || {K, V} <- ErlcloudHeaders]. + +join_headers(ErlcloudHeaders, UserSpecialHeaders) -> + ErlcloudHeaders ++ headers_user_to_erlcloud_request(UserSpecialHeaders). to_binary(Val) when is_list(Val) -> list_to_binary(Val); to_binary(Val) when is_binary(Val) -> Val. @@ -336,5 +372,19 @@ to_list_string(Val) when is_binary(Val) -> to_list_string(Val) when is_list(Val) -> Val. -key(Characters) -> +erlcloud_key(Characters) -> binary_to_list(unicode:characters_to_binary(Characters)). + +response_property(Name, Props) -> + case proplists:get_value(Name, Props) of + undefined -> + %% This schould not happen for valid S3 implementations + ?SLOG(error, #{ + msg => "missing_s3_response_property", + name => Name, + props => Props + }), + error({missing_s3_response_property, Name}); + Value -> + Value + end. diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index 3c5eb8f36..3ab023d4d 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -11,6 +11,8 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include("src/emqx_s3.hrl"). + -export([ start_link/2, child_spec/2 @@ -47,6 +49,10 @@ -define(DEFAULT_HTTP_POOL_TIMEOUT, 60000). -define(DEAFULT_HTTP_POOL_CLEANUP_INTERVAL, 60000). +-define(SAFE_CALL_VIA_GPROC(ProfileId, Message, Timeout), + ?SAFE_CALL_VIA_GPROC(id(ProfileId), Message, Timeout, profile_not_found) +). + -spec child_spec(emqx_s3:profile_id(), emqx_s3:profile_config()) -> supervisor:child_spec(). child_spec(ProfileId, ProfileConfig) -> #{ @@ -60,7 +66,7 @@ child_spec(ProfileId, ProfileConfig) -> -spec start_link(emqx_s3:profile_id(), emqx_s3:profile_config()) -> gen_server:start_ret(). start_link(ProfileId, ProfileConfig) -> - gen_server:start_link(?MODULE, [ProfileId, ProfileConfig], []). + gen_server:start_link(?VIA_GPROC(id(ProfileId)), ?MODULE, [ProfileId, ProfileConfig], []). -spec update_config(emqx_s3:profile_id(), emqx_s3:profile_config()) -> ok_or_error(term()). update_config(ProfileId, ProfileConfig) -> @@ -69,12 +75,7 @@ update_config(ProfileId, ProfileConfig) -> -spec update_config(emqx_s3:profile_id(), emqx_s3:profile_config(), timeout()) -> ok_or_error(term()). update_config(ProfileId, ProfileConfig, Timeout) -> - case gproc:where({n, l, id(ProfileId)}) of - undefined -> - {error, profile_not_found}; - Pid -> - gen_server:call(Pid, {update_config, ProfileConfig}, Timeout) - end. + ?SAFE_CALL_VIA_GPROC(ProfileId, {update_config, ProfileConfig}, Timeout). -spec checkout_config(emqx_s3:profile_id()) -> {ok, emqx_s3_client:config(), emqx_s3_uploader:config()} | {error, profile_not_found}. @@ -84,12 +85,7 @@ checkout_config(ProfileId) -> -spec checkout_config(emqx_s3:profile_id(), timeout()) -> {ok, emqx_s3_client:config(), emqx_s3_uploader:config()} | {error, profile_not_found}. checkout_config(ProfileId, Timeout) -> - case gproc:where({n, l, id(ProfileId)}) of - undefined -> - {error, profile_not_found}; - Pid -> - gen_server:call(Pid, {checkout_config, self()}, Timeout) - end. + ?SAFE_CALL_VIA_GPROC(ProfileId, {checkout_config, self()}, Timeout). -spec checkin_config(emqx_s3:profile_id()) -> ok | {error, profile_not_found}. checkin_config(ProfileId) -> @@ -97,12 +93,7 @@ checkin_config(ProfileId) -> -spec checkin_config(emqx_s3:profile_id(), timeout()) -> ok | {error, profile_not_found}. checkin_config(ProfileId, Timeout) -> - case gproc:where({n, l, id(ProfileId)}) of - undefined -> - {error, profile_not_found}; - Pid -> - gen_server:call(Pid, {checkin_config, self()}, Timeout) - end. + ?SAFE_CALL_VIA_GPROC(ProfileId, {checkin_config, self()}, Timeout). %%-------------------------------------------------------------------- %% gen_server callbacks @@ -110,10 +101,9 @@ checkin_config(ProfileId, Timeout) -> init([ProfileId, ProfileConfig]) -> _ = process_flag(trap_exit, true), - ok = cleanup_orphaned_pools(ProfileId), + ok = cleanup_profile_pools(ProfileId), case start_http_pool(ProfileId, ProfileConfig) of {ok, PoolName} -> - true = gproc:reg({n, l, id(ProfileId)}, ignored), HttpPoolCleanupInterval = http_pool_cleanup_interval(ProfileConfig), {ok, #{ profile_id => ProfileId, @@ -188,12 +178,7 @@ handle_info(_Info, State) -> {noreply, State}. terminate(_Reason, #{profile_id := ProfileId}) -> - lists:foreach( - fun(PoolName) -> - ok = stop_http_pool(ProfileId, PoolName) - end, - emqx_s3_profile_http_pools:all(ProfileId) - ). + cleanup_profile_pools(ProfileId). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -263,12 +248,14 @@ update_http_pool(ProfileId, ProfileConfig, #{pool_name := OldPoolName} = State) pool_name(ProfileId) -> iolist_to_binary([ <<"s3-http-">>, - ProfileId, + profile_id_to_bin(ProfileId), <<"-">>, integer_to_binary(erlang:system_time(millisecond)), <<"-">>, integer_to_binary(erlang:unique_integer([positive])) ]). +profile_id_to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); +profile_id_to_bin(Bin) when is_binary(Bin) -> Bin. old_http_config(#{profile_config := ProfileConfig}) -> http_config(ProfileConfig). @@ -278,7 +265,7 @@ set_old_pool_outdated(#{ _ = emqx_s3_profile_http_pools:set_outdated(ProfileId, PoolName, HttpPoolTimeout), ok. -cleanup_orphaned_pools(ProfileId) -> +cleanup_profile_pools(ProfileId) -> lists:foreach( fun(PoolName) -> ok = stop_http_pool(ProfileId, PoolName) diff --git a/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl b/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl index 1cd155a77..fb7b93a15 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl @@ -7,6 +7,9 @@ -behaviour(supervisor). -include_lib("emqx/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-include("src/emqx_s3.hrl"). -export([ start_link/1, @@ -23,7 +26,7 @@ -spec start_link(emqx_s3:profile_id()) -> supervisor:start_ret(). start_link(ProfileId) -> - supervisor:start_link(?MODULE, [ProfileId]). + supervisor:start_link(?VIA_GPROC(id(ProfileId)), ?MODULE, [ProfileId]). -spec child_spec(emqx_s3:profile_id()) -> supervisor:child_spec(). child_spec(ProfileId) -> @@ -43,10 +46,10 @@ id(ProfileId) -> -spec start_uploader(emqx_s3:profile_id(), emqx_s3_uploader:opts()) -> supervisor:start_ret() | {error, profile_not_found}. start_uploader(ProfileId, Opts) -> - Id = id(ProfileId), - case gproc:where({n, l, Id}) of - undefined -> {error, profile_not_found}; - Pid -> supervisor:start_child(Pid, [Opts]) + try supervisor:start_child(?VIA_GPROC(id(ProfileId)), [Opts]) of + Result -> Result + catch + exit:{noproc, _} -> {error, profile_not_found} end. %%-------------------------------------------------------------------- @@ -54,7 +57,6 @@ start_uploader(ProfileId, Opts) -> %%------------------------------------------------------------------- init([ProfileId]) -> - true = gproc:reg({n, l, id(ProfileId)}, ignored), SupFlags = #{ strategy => simple_one_for_one, intensity => 10, diff --git a/apps/emqx_s3/src/emqx_s3_uploader.erl b/apps/emqx_s3/src/emqx_s3_uploader.erl index f6414669d..8327462c7 100644 --- a/apps/emqx_s3/src/emqx_s3_uploader.erl +++ b/apps/emqx_s3/src/emqx_s3_uploader.erl @@ -56,13 +56,15 @@ %% 5GB -define(DEFAULT_MAX_PART_SIZE, 5368709120). +-define(DEFAULT_TIMEOUT, 30000). + -spec start_link(emqx_s3:profile_id(), opts()) -> gen_statem:start_ret(). start_link(ProfileId, #{key := Key} = Opts) when is_list(Key) -> gen_statem:start_link(?MODULE, [ProfileId, Opts], []). -spec write(pid(), iodata()) -> ok_or_error(term()). write(Pid, WriteData) -> - write(Pid, WriteData, infinity). + write(Pid, WriteData, ?DEFAULT_TIMEOUT). -spec write(pid(), iodata(), timeout()) -> ok_or_error(term()). write(Pid, WriteData, Timeout) -> @@ -70,7 +72,7 @@ write(Pid, WriteData, Timeout) -> -spec complete(pid()) -> ok_or_error(term()). complete(Pid) -> - complete(Pid, infinity). + complete(Pid, ?DEFAULT_TIMEOUT). -spec complete(pid(), timeout()) -> ok_or_error(term()). complete(Pid, Timeout) -> @@ -78,7 +80,7 @@ complete(Pid, Timeout) -> -spec abort(pid()) -> ok_or_error(term()). abort(Pid) -> - abort(Pid, infinity). + abort(Pid, ?DEFAULT_TIMEOUT). -spec abort(pid(), timeout()) -> ok_or_error(term()). abort(Pid, Timeout) -> @@ -237,7 +239,7 @@ upload_part( etags := ETags } = Data ) -> - case emqx_s3_client:upload_part(Client, Key, UploadId, PartNumber, lists:reverse(Buffer)) of + case emqx_s3_client:upload_part(Client, Key, UploadId, PartNumber, Buffer) of {ok, ETag} -> NewData = Data#{ buffer => [], @@ -298,7 +300,7 @@ put_object( headers := Headers } ) -> - case emqx_s3_client:put_object(Client, Headers, Key, lists:reverse(Buffer)) of + case emqx_s3_client:put_object(Client, Headers, Key, Buffer) of ok -> ok; {error, _} = Error -> @@ -308,7 +310,7 @@ put_object( -spec append_buffer(data(), iodata()) -> data(). append_buffer(#{buffer := Buffer, buffer_size := BufferSize} = Data, WriteData) -> Data#{ - buffer => [WriteData | Buffer], + buffer => [Buffer, WriteData], buffer_size => BufferSize + iolist_size(WriteData) }. From 820e06d756bc81a2e2f9b8ddf7e65cc7d5af2817 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 00:02:26 +0300 Subject: [PATCH 100/197] fix(ft-s3): make controllable use of max_retries and request_timeout arguments --- apps/emqx_s3/src/emqx_s3_client.erl | 38 +++++++++++--------- apps/emqx_s3/src/emqx_s3_profile_conf.erl | 1 + apps/emqx_s3/test/emqx_s3_test_helpers.erl | 2 +- apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl | 1 - 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index c84f32dc3..2ba837e66 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -58,11 +58,15 @@ access_key_id := string() | undefined, secret_access_key := string() | undefined, http_pool := ehttpc:pool_name(), - request_timeout := timeout() + request_timeout := timeout() | undefined, + max_retries := non_neg_integer() | undefined }. -type s3_options() :: list({string(), string()}). +-define(DEFAULT_REQUEST_TIMEOUT, 30000). +-define(DEFAULT_MAX_RETRIES, 2). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -206,7 +210,8 @@ aws_config(#{ access_key_id := AccessKeyId, secret_access_key := SecretAccessKey, http_pool := HttpPool, - request_timeout := Timeout + request_timeout := Timeout, + max_retries := MaxRetries }) -> #aws_config{ s3_scheme = Scheme, @@ -218,39 +223,37 @@ aws_config(#{ access_key_id = AccessKeyId, secret_access_key = SecretAccessKey, - http_client = request_fun(HttpPool), - timeout = Timeout + http_client = request_fun(HttpPool, with_default(MaxRetries, ?DEFAULT_MAX_RETRIES)), + + %% This value will be transparently passed to ehttpc + timeout = with_default(Timeout, ?DEFAULT_REQUEST_TIMEOUT), + %% We rely on retry mechanism of ehttpc + retry_num = 1 }. -type http_pool() :: term(). --spec request_fun(http_pool()) -> erlcloud_httpc:request_fun(). -request_fun(HttpPool) -> +-spec request_fun(http_pool(), non_neg_integer()) -> erlcloud_httpc:request_fun(). +request_fun(HttpPool, MaxRetries) -> fun(Url, Method, Headers, Body, Timeout, _Config) -> with_path_and_query_only(Url, fun(PathQuery) -> Request = make_request( Method, PathQuery, headers_erlcloud_request_to_ehttpc(Headers), Body ), - ?SLOG(debug, #{ - msg => "s3_ehttpc_request", - timeout => Timeout, - pool => HttpPool, - method => Method, - request => Request - }), - ehttpc_request(HttpPool, Method, Request, Timeout) + ehttpc_request(HttpPool, Method, Request, Timeout, MaxRetries) end) end. -ehttpc_request(HttpPool, Method, Request, Timeout) -> +ehttpc_request(HttpPool, Method, Request, Timeout, MaxRetries) -> ?SLOG(debug, #{ msg => "s3_ehttpc_request", timeout => Timeout, pool => HttpPool, method => Method, + max_retries => MaxRetries, request => format_request(Request) }), - try ehttpc:request(HttpPool, Method, Request, Timeout) of + try ehttpc:request(HttpPool, Method, Request, Timeout, MaxRetries) of {ok, StatusCode, RespHeaders} -> ?SLOG(debug, #{ msg => "s3_ehttpc_request_ok", @@ -388,3 +391,6 @@ response_property(Name, Props) -> Value -> Value end. + +with_default(undefined, Default) -> Default; +with_default(Value, _Default) -> Value. diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index 3ab023d4d..13efb9d74 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -203,6 +203,7 @@ client_config(ProfileConfig, PoolName) -> access_key_id => maps:get(access_key_id, ProfileConfig, undefined), secret_access_key => maps:get(secret_access_key, ProfileConfig, undefined), request_timeout => maps:get(request_timeout, HTTPOpts, undefined), + max_retries => maps:get(max_retries, HTTPOpts, undefined), http_pool => PoolName }. diff --git a/apps/emqx_s3/test/emqx_s3_test_helpers.erl b/apps/emqx_s3/test/emqx_s3_test_helpers.erl index c74e78a4d..2edd52609 100644 --- a/apps/emqx_s3/test/emqx_s3_test_helpers.erl +++ b/apps/emqx_s3/test/emqx_s3_test_helpers.erl @@ -99,7 +99,7 @@ unique_bucket() -> with_failure(_ConnType, ehttpc_500, Fun) -> try meck:new(ehttpc, [passthrough, no_history]), - meck:expect(ehttpc, request, fun(_, _, _, _) -> {ok, 500, []} end), + meck:expect(ehttpc, request, fun(_, _, _, _, _) -> {ok, 500, []} end), Fun() after meck:unload(ehttpc) diff --git a/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl index ef1d916c6..5fda42fbe 100644 --- a/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl @@ -13,7 +13,6 @@ -define(assertProcessExited(Reason, Pid), receive {'DOWN', _, _, Pid, Reason} -> - % ct:print("uploader process exited with reason: ~p", [R]), ok after 3000 -> ct:fail("uploader process did not exit") From d333187c5a0ccbe563a10e24f729f040cad3ac70 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 00:25:31 +0300 Subject: [PATCH 101/197] docs(s3): update readme --- apps/emqx_s3/README.md | 2 ++ apps/emqx_s3/docs/s3_app.png | Bin 202227 -> 256486 bytes 2 files changed, 2 insertions(+) diff --git a/apps/emqx_s3/README.md b/apps/emqx_s3/README.md index 3f049d1a8..4ce1b0c0a 100644 --- a/apps/emqx_s3/README.md +++ b/apps/emqx_s3/README.md @@ -112,6 +112,8 @@ do_upload_data(Key, Data) -> When an uploader process is started, it checkouts the actual S3 configuration for the profile from the `emqx_s3_profile_conf` server. It uses the obtained configuration and connection pool to upload data to S3 till the termination, even if the configuration is updated. +Other processes (`emqx_XXX`) can also checkout the actual S3 configuration for the profile from the `emqx_s3_profile_conf` server. + `emqx_s3_profile_conf`: * Keeps actual S3 configuration for the profile and creates a connection pool for the actual configuration. * Creates a new connection pool when the configuration is updated. diff --git a/apps/emqx_s3/docs/s3_app.png b/apps/emqx_s3/docs/s3_app.png index 372f16d371d0b75b7d796736906eb1de004d1deb..cb7758844e0b12ffe5a46b34df2ddbec94287120 100644 GIT binary patch literal 256486 zcmeEv1wd5W_BS9Pph$@bf&vN%NOyyPf=EbrNDduCOIsL-q)19icZigUbfa8_lHd>kKX|xV z`MG(tcgGum%<2=X1$_{RITHeSQVhgo%vY_KS3%<+RLQSo~Z*Um= zR8#>!G{OHI>;@b>27HX*qlCS^wW)^bWjP3#%{d+}K2|PXaQL*GjH>)aI!;OO*&1SH z3Vxk8HMX*YUm|JYXlDbCoa5l&U}cB@3l6FnnHf1kP;LQVnJLuBe0Nps=ai(+^SFCJ zRW7M&I4c>b@_R_{uExdG(FtN_idEBJT!1k~N$boZ#aot?QgcpxVomoy!_1h_H$ zKQ2iGDZv+8BQT*I64Obs(@Aj9NpaBebME|PhaWk+`%`M?Gyl$UUc_-gjos%#5Dd_E zlHul3KCh^%;db6yQc{6S2CDV_vyn5faXas5WN#sFXRL^zXNZZZ)9$rMa@g5fLm~Ei4;tIq+L{_8zvs;JvT&D-NMbtL*@63^a2WM8a*&Oiq^UI=(nvZYo&%8g z?_c|MnT}s2>i_3L!D&C87-Eh@cvk&O9u=nbp~oYg4kHZ=jd1p#(DN2rCJ zxt*<%we+{6XTRYesP(ho#$2?s1K{65y`?D>>W)y9M$S;Xy)t3y27zjVk?gFTJm3uc zs}|xy_-83Ntl_ub`2wf$&KFyN`Q58_E`ooBU-k9ex69zCq>$GcJ3G3371n*8vVfPX8$2 z{N-?iu+@Kg+#p;n6#Z^$*Y`-fzD46=DYW@XN3tV|?Njqyh#~s1rGBGmc zH!}um#0lzXXJv|VhS!+i^s*T^v->=Iqu zUq|E-s_iGNvv;(E$5DS-7VhOBC(>;~W#Lcl{V&Ht4utcGB4You_dm7c2z~sU_8u7n z{`D0%f+Bwubol;tlm8!b`Tk`H0PPM}@PBy-APDmJGXR{(zzt!;|I8NP4;JG;PsE4R z?7wXS5I*f+UjC5@#Xrgf{Oc9~Bnbb=WWm1-9Uu*V57U99$Um_i&i2-JMkc0?f7K`g z8TcV=2p7_(phOXzNHYD}fglIM;Xz^ME{stW!~a+0&ST1N!u_+zouBhE7ccLBDRSpV za_FCMjDsJMK}F@*SNGu8iQPH4b~%K?GGyKb8N7q)l-Un#1SJZP?L;kT?9Haw+S!5( z&cX{G5Ky#|K%B6Fn3%vzDnG5De1i`>zq>oF-#1Ug z&4ciaklf}*=w=l89=>{xQk+17p7t`2Ubu`!(bfNq7XqxRDALm0`Sy z6bvfE1b)4=%s)=8A+6#M4E*~@CJ%x-NIoO^hsx)l8BQcVej&j`)ZJ0+MtLAaUdx269aB+4S_dA>;w`h$ojW5{C~s3B3lp;a?FF2PgFVnStSHH z?iZ5dAG;J(5Dpd!13>2s=m7dh9u7*6h>?^1Zd;HU#0_4m-s4WuadMfNnQRVHT^tYq`xyA>$ z%Z2QEU}r^Yzq4qw|PeviHf(ABs@sNdAYTz{J^7=j`w zn)nAS{ad;hHFyHjZfs(Dng23C$lgc`oZ;f-65xXChyRAQ=H^8xIi%Z#6yo2KcBBgb zg@hduwf!-|{!PHOfJh@n|KAh#uZrN?WxtECBM9}I!p@1Xc7J)di|Z$5{hLMjz3l`$ z>wmxM3D@>;%lO+E_`4{4E=2tKN2!lLPs!|G#?IfzSwKh=inCy90fpKd|7EKjNH+l) zB>W^sKzazMNjYSq?$_}ekT&i|*4cy29e-K=3GB_bar`W+`X^!Fk7;m(?ERG)$ia?m zMMOy8pP7R{=EeV`p_a+7QONjO_CV!*GqwN{h@ti$S7J1o+SH?1loV78x+Zc-;Rs^HnB0nwL@``{-fI2bmyO6 zFA;{UWBiuTBE6(vM>}&NwDTVm$^JV+`;Qv<{!3YJu75S_&H1-Ug8a@|?_X8ef98rI zLtIo>>=zXFzh(Htht$r0{^*Ab(Jg{vUnPv8&PMR}DZnGxAB^qarzMD+pBH$2J9%oP zD~%LN6sydK@Wg&?IYox+C~Q`AhME{bK`X%j7|qWCe@Va|P9r58h12|8NI3lBy#6sA z{6AhJUt_}k6JGzs{QhZpjezZ+$=QbYrN9fzC~@`g-(tdzP}9E|StCR6zkJmhNf8v2 zfXt%(S(^R6-!l0p?)KyU6ZVSzA%cwTQTFqc;D3;>yh$h-(B>>TLeMG`2FjQ{`s5v z;E5~HUI^aL_mdY3B7>o zo9^KD81b6B->O&r2UFkQ+1H<719ZTny!q~@l>1k=;&AW8w*LpP{W@>D`)BL_U48N2 zS&X?kk?9enApflJj7(vp3i2OnW`D|#A74WASJb=$NIwY4us^ZpMcB$emFC^;yGEu$ z{ts2h|A^{Elxu%8p4zPx{Pk4tKW--a&!s_rU)3uh@E20OKY8!RPBg2jsriEr{*?mQ z-J4{Yy3yCm<#;ThjahnW&Atm_WacY zR2#8#7Wr2hc26KKfWORj_x=2uabaiB(5TU5r6g3{^rr^*yQ!+J)qW-mc|fR?XSfe5 zNLh z8}Tbwg*>^UPN{H}UTImFI?Guavmihg`ecP~7DCO9Ilw^J+ig0jxQ^q|#U5k-w%l%H z@hCbbp*Y&tAJpU+*PJP#In&PTCC<~4cd8sO?w-F)+UlA7Xhl|XHqG$_kGkHnFzUd(5B1x@<@&4?^HM3~G-a6v3G8}4OG>mUQ9B`A<(puP< znVH|Y5ZY?LN~vq!If+Y{xBp>T=|eQ7Lw?`Ji*sUfa&jICi-?E_yg8&ui$+SrDUpWr zsQ1i!436r)Lzqllirw+caZ3}VRS7C(HYB37hi-nm9R6weg5UG!{bJ&Lt~1+sf_w3p^sJ6Q>L0RRppR}i@RhPV(^oD{uPF00zN}O` z-@*|4jox$VzRrrWvkYBFd@WlOo(lph1}HAimw)hKgTs{l-bd+^xeiDxP5M z0vKD5E4n)N&@dT5KnT5Zzs@L05@z%jSA2Fu6uLae>x8~L&CG0d$-&{Xl9(zs1MH<( zP&sC!$1H84Rpn&78pX5J@-J>wd-;jx_XTqTPt@kAACK$X_%lvqC-^+!I7%nR(W#W3 ztyV5(R8x8GjzUBG5^)x}0r&EH5(x%#gUIv=s>#Z<3vmmyd!z%0TIq&33txVgh=>>0 z?E{$LsdEp|x}t@7A06EPZ3=#>IGPP_`!WO4Qc{|j+1Pq4^W)YGDJ9ZWb&Z;Z&3o-t zs0j%-<9S5|Mj8W?G`?fY){#70lk~F>Qf6>OR@dhZ7XgKI{qoQmW}XSUK8KNiMLimZ zEg{Yd`jf`S#%pwRbT2B1-DqJW*O@uoy|VHt3*tify6OVP-h6j^oKunsL8jo|bo*kB z;md2oRmJ7kI>9}vZw!}$P}k|4V@uU{^+1I$~p)z zj4%|}(94)`PcsM+LLKL)UJjL7c135Y9=EEj4XTg({_bw_)J4!IHdoZIG2n~s3Q8gg z_3+_XG95Tfk;BFDFdZEo{ZIJ6RbTRaaE!^D1bX4wtB_%xmcQ{7FE% z@EHVka&QvQJrFOeQAs*?_lVtK`I17~5Ny8pFgqpS#69tF&tfbrEL8``mVq`E6sOhdN-qjrwCi=5g)fcPbrs5u5s6bP_ziDrXBV~X zzx1)w#1-`tnJ2!L8} z)vrA5HMQZMbOUub93O7#2@y3a;X;OSScZl`J-DK0@UL#-8E&8q{30d(Lc!yNg-!=S z`S|*yFkXKJ#?joRkt?)so;XRie5aiW#l^*5YiORox`jGS4nmTuy8wL^lUb-;Bm4Aq`Al;ZwTD0Ike$^OAGnGgS5R0Od6<-R@X@RIJh;UjI1?Nf zCz_j;m31SChK=psdWmh^#kC;Ri#c#%UMKYG6?8m2);0jB<2a~JH^<@#zQTRrJGk(W z*XH39(}sw**~XvG37}_CVxTT?XD!O~gcNS|Z{BDsX=@9{)Tq>_UL`r{D^7`_rKNS? z_U+qG$7%5J@J6%0jLsj*fZ>R=gkd2Uh|I?q5d4Fknjg#cgQuq%*Q&!R?^O0@-g zSlC16U|$u>0RiScgWw=Z7N-^ydv|8Z#0oP`nVlep7#%Z_n0n+eDcLbpCFA=J6G|?G zFJfc0bOi82wOWG$J#B62ll2eN9H#bT_hZPOQKds4O_*sX1;Uvt&OVY%(=;%kyM2M( z5mjQu6OQ-=dx|U(!|&-6X~I78GWz41U!(Q{b?Z6@4vDO z3j>|;^l6zdp9S__H!O{AMg9~(N&3~RSA(Gf=lm7wI`B=*%!1A^GIEf-N%INW-*o?b zy(C^!iP3$Rol3&Y?9=m*kPstUW!Ml=n-fq>nc@I)6##O_uf_}r1}q*!g_bg-;owJE zMa6gnpenhvyl7xylv)}5bhY70-}@TAzWcGBpD{o0%s|l0C3&{yJ;pQ7a_u_V7Yf{a zD2?U^!=&KGc^w?$Vs|4!1?r^S)9nY=b4j~jlP~#}ik@!89?96{z=k3m9s`PPCE1;; zN*O{9*QkBPMQ{imGxnP(p!oP`7(-;O1cd;dHY|)r4E_p%PlFzR2;-Lvx(W!sH+m2y zO(+jlWQa3587v57KD}_P5p~3{1p4~^KB-gh;kWs0z>OL6fo}u{bp)J;U>c<8>+<<` z?%csjNlAH;KJ!ucI^QjQa|Q!&8I3Nsnb%uejrJb-uc(sv^3EAEeFy#ggJxk@FG z24hICS9V^b*L-)d;|HAy*SHJ`U9pai86(k)2k=GU&Vio=mj8+yGZRzdU?hO_o;4*B zTC`FhmA@d<3k>+-FWMEdQjX4K-PZRG1!aFX6n-Zu9E)CANyWF?XKjP&5)CN#0ZVw2 zP`4n6AivzK8Fm_#p=FxX$;z$+Cv;E34zIofX26lTL~75&ccvdgPv{tMCr_n4S%4j5 zh~y2h?wY!~19u4uJnU@AiBRvzuTfJT85$m5{J5;F%u`r|4yIbFPc0J{IZ~2>ob{zOL-j*ViX^>(;FqL&m2jnNC+Qb=eF>8!2!wr7{J+k2NL~sV~X( z5~zi6UxCM}P*v}g2JPFu60iei#v@>khZWJ%&evdJL`6j%xNzY@xk~;! ztWaq(RHoDvV$iJEU#>s%swmW4x)lG6h=@paR@M~jW(Z%Ee!R`=8$X~Z7fcNEP>3kVBy(5r_f5jmENE`)kQ_tBX^kD2v|VB|i*#kc4L_}n<`>?Sed=t(Ui3eamd!wuhS29;)gf|HS zp3&^;m2cGa^})gsT6LWPkpmOblQ^%hA2;&plmB-v$S^hfWR*OKWoA|vTEuRg>*uN7 zANzRZ@Ob!hm+uw=zWN>jH-2K@4&ww|NqDGonNk364~T-f{WLQKR|Q~FZx|twwEOp} zpB*~JoD}b^6L6AHBz(-~2ZB2+5Zn!>e^NF05VJSpx~We+5+^T3o_dq_Ruyye&`A&| zOMyU{ecDt)!VeQ;?fvn+zQZmVd^ND}4jiCkV5mLF#=>Hw@9yr-mKzlAK;!HCz*52E zoyP2T`*#eDVJ5UuV|a%Z3(`HGA|oP>XRFA(se3}kDY8cq@8XY`1@8Trn4CGy!ZLjy z=43C7Nhnk4tJV~_UY#C)z+*2nI_coIHo%Gn2J^@fGBQ^csiBc>XU5Nhd+4@v(Xa?Q zzR);6_a@hFO2EN&_HGCf&9D`Eh5$&gkPD+;Qg%r^(|)p&f~bz{entQ5;mCwgcc)Ck zFlLvJbn@}2a=0rBelPJf`C5^lCJpT;TMB&O%CQ7TMDUhYR8%mWfI6=Tw|k-r*<7SJ zw_5B5A(VDxBzH2t?FSQ%hCzheE6Vct?HHlUt($l6651D_b;ZO83oU|sqY(MAbb>^H z;nno+3I~lF`X| z=~HZr2jlPr0hAkopolySR{6koD(JfeCLEYwq-*fLBL01(1D67(?wJJB&3(IEocv(a z>b&O(Xny4pxeFIt8G-!Sf1l1aN-05J+DH)CDo!z)}?0QHUeuz$iyTY8%(6i&zS*Jr= z4#oM=46~v~WIOd-Y17#=)dU5<{tQ8P8lbS!o8^4*Q8|mWA&I2aQ-`yUzC9UCj9zF> zb;4IZP=sM`NEAXlJeoj4la{Q@N%>t)XXh^B074E_)WdG@No*W$lh`u}Y7f7M5xGuI z(rJco7}?kq9i5u$oquM}(J_gc4A&1oIv!~{MV!&_jPRzMuj%B6560$u?pN~!qa;Bk z$oojsM7mA@FcUGJ*=<~VhU}U#CycAH!h9(Q-yFmEB#kflpLYF%!q5|Oa@;*Q-Bn&tfNYg;MH0cv*+l|$Hau>N)`Yyo#Kg$!G2By9-$}z< zkgxNJREYp=C*b(`+pP^8g>qWukT}jazp9h@2@gtfu3n8DaI0Bi7lfi7P{BR3z zVgFt&BVJ<+N4Ecpp{_I6>$R(gQf`X4m~w(_%2!E2e5#SgQ14|F5!svpYFU@eIW~uO zm=0ZKHQw`;{zK*0du^mDQo1(Hs=B4xtIHT+Cfzzc3JbVYtTsR18_iD@uUGWa(TcW+ z`Yn4~%qyprsAHSY0DYF2>jVsMdzfln2RcRl6{G-x*02b_fqc+}bj=lhUP7D)q-fvgABRX=Z}n^U)vX0bbubH#2FkG7 zX;yklaB}`g&6;V9l4G`KjK-GhlDBP%s9_@!QHk+;^+F6r$*ulRRkuR}6zE5iKbpoa z@temE{)Cd4A@m9HOr3chCFgn>A?=h${SthixHdAO=@RX^qwfmV05Oy^{!QCB^~lKP z?GU1$SUy{_yg_KkBa4Z;PSrFTUvXv3#UTe@&sy31(S*HsOTx5X(OuM+3sPYgRC`Pj z|Hh%SSkn9_jKQ=d!od|1SgJbabklbd3+au>x7IDG)a~V@pCHy*CJ3MX2Yj*xNDQzz z89#&xyAQCxEp8#;T)G(TO*_nqz97-}Gr9SQfxks&=cwCSmzCWdt$NSE?;yb*1HL0b zb9uhgpLD8@u&m4}kyBu#6XM*IK|c{Dk0mm;H*!)w85|mV5-=Dd*AvC}xS#pLp|nrW zhqLkzfLhuGEv+bS%g)ow0sL+)j4G*b?ul)kzH!34Y;k7r?V{>-toLHu`GWPCrwv$S zy2Cezap%Pd78Y2duYrpXAId0)J78wKd0-tI*+X<-@=w*U6SQ_nt@Kd{G=8bbS{tefx-P zrh4{BK?Cc;3EzNojSK4g<+1e~QhZrpVY3fXCbOKo7Ut(q@%0>bPNW&O@MF;_c>`-p zGJWQcRbc=AY)6lt>!4bY@VpE;r^(GlSlf%X&3ma%BA&}(s|#Zdzy~tf`10ATC)&1L z_VwLk4Es-9$-A=LXWmB1I%C5FEw=9Cg!JY;ih;JvKKod?`YF0S*FbDxDYtryNoc0b zcwp=U+8$&9v3p5tsPkOG)wNNiO!M_~Q5^n@HCcEyUM4r1@CMWA4X;HH>FJU?CIWnZ zz~Y8=mRe03&M|&AAHIoGW>Di1*Hh`~x^R)Jo!4{Oy06?hlhVEhy**Vvde~N4TAI_m znR206^nguw7Uxx&Oda2`Z0BPjf-T1Fohjs}hnWn;DBC_b(8Gce&1V-;<+WZPLMLZA zRdC(cw&L@-(u3*U-h$KnXH^tM=pyw!(#M^^J@aNUV}%`_aNz6_^%^dWWyNC!S7Xgn z!~K%=eG^nB#(a-=(k7s%wc4wU1$jn`XiurHvX_nK$a6@c`JKkc~HM;ad`(FZATY#8b5JaJ4ZvU%HGxo@z0!CuU5dm(=NL58g0Oyx?>l_Y(G?9_w~ z10PVH?#|{Kw0KulsrCGO+v+Qxfq6C}uYu_PbJeA53$(*De!!tVeE9H9Z~KVrzM+_B zB^4;_CkFDvL(63h2AU9%HyCUF)~}d9WgaE^;j`Wu7t;iVvB^KI+Y>r39=Vpb6MwQ!?p0Rum>r% zKd-ixo7(`t&{k)JXU6Gs?v-egTPU2 zVrC+Ldt*TnGF0&d)Y#8g__(`2*?70T89(mHEl<)XG+R-aoRV^FkY#{iNG2$^CyJG; zCs~$+E28}2vcY;{kEwjmgX2l52`YNErTQ8=_?E>MZD&i~+CG(W<$62LljGH475SZO z;w!EPl9Xl0+~#C|Tn~|Wf&#j~f-`Zxgx7&A-BPU~tqOJZ+_%;W@-^vI=DrK7dgA*O zJIs7+kM%Bhx@!^HIQh=A712-eFHqN}Ie zD6{dPeWsC7-dbX?X*%+*OI}m*KIwVSbv(Eha-PoIdTg+E;FTnSh{;*u2@9UsB<7s5 zvpH336HJd{fB@-5u<9|5#k$5(x()lygK5+iK{_s?$pt7_dc5;msXHVzMJU`I#;DTB zlefLK-T;Wd@!*o;m zmZI=ysSdkMzJ#7K{q24q@L`?G`^$TxCbnbW$OpaAHd?5@sh{QAx?9%Jos z&n)4&ntkp0#^;R_TXYOO#_8J3z4{lUdPUc#jN`&9x7H@-`^@99;+bav$5MNr9bl_4 zj8i-Av+-G3TiY@y-?EFziP8)Ev3zOea=%qm;i&z4CNFFe$B{R~?%7_BPEJh=&t)_m z-n%om=;*EAiw^2x91cq$k?ZafPz!u3&fL54@x(T}nD29*O&T7c_5~&$C;7AKh<)Ma zPBNmkl4W1tS__bVA*!zQdg~#!%%_%kyR)a&JM87<{gNYCbRIgvRu^vsV_)}Jgb^K- zf>4ijswnc@0Hq*a^w8X@wfZZYO?>a(08t_1>UC-nF~4GIN%6X!@IJ4e-Ey4g42O@; z>S$zByUns0!CA=TPwjGS*@Xt$baMMty4{{n+jnc3M=fYgmwgNh)haZLGC25rRi_#a zMmhClEY6ItxJRJX%>9OthQ;`-MfaU9_AYQ(=i>uKp=oX>D(&jv3V^pyWbyqEulEP> zf52HCPm8}unrbialiy*LY*6j}$y86h1+O3S#j3^^o&K?^N(Xi5W`9wR5B}5f;-k%) z+k$p1)UZtBVQzg8q2u#!!@2OaE^z0FCqX|wYDNy7fjF6-u zyy44gS_)ky_=FRH4GnQ&iGBsIpVeq#d={SV0_h?aFS8^4i|cD=6yl35kILTGx>~|v z{Q6DOJyD8f_b5^3$$=+M>@~^Kl$IW!-0KMt`5n3ZI32<-SDK#rxgs1` zV!e|UGq&jGF=N4t3QHAv^D%>Bt*_%taIu;Av4Tq z$-eXPrS@tcpF-}3*fCSj?g3~iVdGPr(pFGnUa%OJdVBta*WoO;Bnc*YEUU?(%BQmX zt@A>dGt>NxevEX82W%=pq?af?^H;~~Pzh@arlRdcv*Y8+P#&6A)c#B)TqRVbi zhI|6m$|yW(b<<=Xt8N)!Gn*b|C4@TbI=9xeerk#yjWUnJRj+AJ2*=JT>I+a#})tVO0Ib;S`+A5Y`AG5oO z9R~!AP34)~n0_wqbXs#Mt7{rf4$vk8)^>L#?k+uSt%=m%*T49%)!=#&o|?P_!{+Rr zA=Pe|w>G6w2JJ=h6PaD*S$O8lz8bKlTm@}uxt68H4#j9ot^T6^h-UKdB2qNJe$1>q z6+{&+sqHM9-vi*CzPeZ&6nMnVVuG_!mx?A$s=-kh^8-oVbA|FdgLloHZ_OG`-SrK? zB>r5Xp;6lQfuLOQu4rk;SaAQMR(8%C;Am{r3ZLxqKeD!=5Fa^S$nK@EzCK&s0G3I^ z5Pzkh?>sZ_c*dEvDdqU$erBHW!gF2`sn?D#MkQ_O#cx77Fb=N?t3_8*pQR=YCOUcU zf$;=a&uz9Ue$Dr?8lL%a65ZT%!gizXnE_i*+QU!b@YBFvhB(VU=zVx7-7lHfrvE{t zWM(&PMRapfMPMrLMdM`oMieP){(ceexgyXS3S&|^j*INr1D1R0i=WK#E8U!JzQaQvKh z4tq@7{Ah!8q*zC43dPa`^$tV%r!LbMAIk;kcPo1=e}rj5qxjbyEhs44@sw&Qoi0jo zc>%92+JOEni?jLDyOlo638AdaZktOz`&&}F?sA8jsm-<92V?t9NicYZGg;Ms{#Y9w za=#zicQ&(LEg2mAAPChG!xz$;oV7fy&|+G>3>_EPOsN}!_VMc}D^=&_>c~f~6f&8P zYi1u|g_-1Hr1X^Ne_T$LW3wzi9cx^fBf9m)c%fi(ji40sk-At1^r=_;P5S7&-^F`Z zm}`h#s$xB`<1~T861X&C!GWhK*PXAccgJ_?x_Ahde)PhVtaoM7Yc4h{3@+®}? zdo6xxe4cSm_j2O9qI?6Uk$V1P5Bcok7ALwIu59*OjXI)4o^;jPAv?~m}uFCCqtUrrg-lA332dINr z+AfVunU8%ec}`eqm1t@?-_^wDf^;P3VP^_LVT8g1-Pt-R6&1Se9UYlh3$ySS_ZNHB ziA}C+H_5!yL`GBp94qnG_yP_E{R9-zVIk zg?9m%Y53RYj)$aY#g76Kx;u2d<>b8B8pU9Au3pA}C; zQjD`PxiGU-r&s4$lnZfRJS^WT+cBa|Bc~&;_QbaAJ$75(gOZRC;!wd-gYEuE5YTCm z-@|S=4m=3UXBC>clBnIP@O;}`mG~QCm-I5WF)x#Iu>I%@>3r)q-fgg3yoQ}YXC6te zoOm~8*p^2EbVv*X`}xK8`S{GjJf_hx(xox)F zA_Mfn^$DS?FYjVsfJU9xp2wQ9O23{dlPvOp)AUxlMG`~{pAJ&!e|&jXtFrip=Tt+7 zR<4d8S*Cy3bsb>%jr6ys3S0b1!={o-o5|J!R7>0D?x+VA&U09m)RdyTuFb+ znVdiS4HSg)8taej#B#77K`%a(>UO-lyaYLa@3>8b(Cs3Kp=^LL#hbQO%}NjFyi-ZX zPwKX&rz+jQ?!=9m(xm;g`#2PDrQwdP6hCFrPwMfr#r!R{(TN}rcZfiJZV+OTuhFHwcb5#zl-Z6 z{bw5bgN}`V={vMH&jGLO<%QSzB31$+Jr9v#~pyESz-7B!LygB39Q9T2-XwpSkd&+I#ia<^4XMsG*&eSh-;45M7THqy$0c8Ah;7VCN_B-y zP!v|>h|LvEgy-NM%YhhLjl8*^nxUd6zfaLJRjyCEJge^^kCjQ|0}lFAq(DT}*Ea1X z7_uprUXmzYosI}EIWBso493Q!&d;Fh)+Ff5s*!eRdGrGg>3@(fafaRtT_&Epc}eLPgCf z`RxshpY=snc980MuZ=T2nR|Yq$)zEZEi&Mc;AzKTcAa^+pX8t?J~f5sr-~V?S->hC zc|GE@=Ry--Q(wH-a-X*6qfxx|m07!zeE;L`nXsjgK|O$TIvV5DoW8U_{JI|81$KTo8^5Mf}fsMHr@;oPPw5PM|-c`{HcUO6N!lUIW;Oc%V2DHbY7v5a# z=nAqkxUOK(Gfj&-1_Mf4SATY>^6s?~8Gi)|;lg^k33qNvb&Zb6wl%DC`)svclymvA z6O%voOiHkP#1+&+7s$q09*s1(J{_W7Q4Uf_O;*~MF4SInyOjG@`m$Ef7x(X)iEH99 zKz6mo-MkZ*gc=sce&NvJ+rIFWJuvzCuzUiHB{oAf=gX_*Yo%j%0!d|FFX&Gjar!HC zJJl)qrN~7-yiC-uo}6(NEkL0WqF<`?)B1v9`VEz?7sIB$%89x(X z4~Xbb!c2T#TWA#Jq9wQZhyizOeBeo8-zi-pES8A`=C?3Q!LfKB?`*lZpDUcJw1@)7 zvYXdjS_?`a@|-q+rCzdr_)zE|KT>fAw zOr#JwlUCV#X@PIk;ymgp*FpA!-7l@Eh`Wt2Z0m*w^QJ}N{@92!TC{pkU3lKjwk6%b z<$(~6Uq5CWj7viQVD^0ZMQ!;kZFiXf+5VjjjhcoY>42)_1&WZixuD{!@u|0Y+>+$C zJ5Llm4DL#OOK?I|xR;NoGeOx#z82o<%iCtDP+l37pPc;CP0y)`=h-07!DRLRDXY(x z{1<$_SlP=&t4kD|y-22OFJWJI*0YrnT2*{GVvG2ZA0NKO=9iE6+KV76P8Bi%3N8(D ztWREIb`%MQs4KTmUi883Fbj__3d(#}7;&eg+G>_Bh<4F0`fE-)p;WMV`bt968 z1YTO3Z(lpl95x=}L!r>&-u`*K%&aAj{caQ|SmKYd}H1XM&g-rW#3@ z`&5FOyv5UbfCnoUq-EF&F)nUzZH{(n7~m6c&b=T2Il$4D8o(aSoXTbG1y>eKBR2X+ zE!UQ`Y+YPREZY2NB(LAhd7e03#}I$mE9mhw$mtzC@}PCBwmhDfB_W}O#d-9YjW)H! zJv9}eT}Pc<;!4Zk8*q8uIiDaJpzzF6VeJIJwk0O_z$3APS*erp69oJ4LoDdPHR*3M zVv-FQ1YyYyFN&>tSlvYNV_`iH2a5TaVaZ~7+%-w)EH>}1EXcDbq^LbEDhS>3?K+Y} zteo6VakcG&6h%gR>X`6itE6<~LdS_)>d`z_5tL4i9G~Ryr-cE`pY?EycPGn+4$ph6 z`KR9JIjr`UKyNVT@nac5m#L$aX+Z`77V2|ylByIkg&BI|YzMXd>W@PU1QW+P4UU}Q z5{W2>44>Bl5kYlY?K`6zRQ}bcVJlO{UAZUm1;*4V=aC?7km>2jWn`AG)Z*lsJ13C;wZ_DTl*IqWjyg~~a;TcG#Um7VqD8E>g zp#rjea<>PW#7x}MrXSzzUn$x1Q7k#PYX#Ma)nTmqItHZ2$L;J@Ipyw$aq{7Xk z*9N*gY%^Q&XF*_HmW{k^s$5>rIy+X|CwKJWTx5?Ndnq!6b z2PhcV2Jy6TI$tC6?6{1pjg{fP@JX#ISFRUwjbYtMY*73*56nr{tBOqaqxJMijiyvtN<0C_Dhx84*EX}U4uQsRxZ*G zQ8%(2D9eX=WKrO>pHQW!k~OuLVkQ_Tux5GsxS^!jcCa7wT2cJ_hu0yUDv{b94iI0N zOx0^!zCkaSfSuy)%2vnGKN5 zNu>=_mn_)dQ$zqFmSwHn{d#NinWEzYJl5(RnkUHvypFLvN~ht^E>C&J=iHW|@%|xp zQ_+d?d)Tldw&m2GTU+PlBJ(;@=CVqRUSJn4KGNs9|6tOM2bR`T?J*e=L?sA)TsWa# zWNf~tNe#}bM52TY6>e?o+rEU*5418-HA^3vG|-Mh_3wo(`N6j}SYwdrXX$oBJ{WpI zWhH9{F_xApdIawsVQ(Fs0QD|^`*MoAIqiw0v5W47yjJ7opk7&bIJmb65{fG{zOodh z`?kc!>RL}tXwG6{i|~@xBTo|(~6Qphbc z)5kA}kwXT$9@ZG4vuHSiPh4kU0swkt*n~1UzDKU&;0w~N+zAvA1R>?eH<#com1{VH z6v{sGuF(+b}@EJfupcf5)y6b1`*ck4JHP``(v^Awq zlsn5-FB>0x6~40x#6H-&eQ3kj=ox}fj3Qu3lECc}FrnS(#rIe6 zj&XiHMG79RA0_+xy(;*i5KF#x&9@XYgbJGYFtju$(u2GSRIg42>?eqTm*jbE2Q=Bn zpbN4J{O>W{XDe4mCNgs)Kq*(6I}DzE52lec0fnpZu1t;dMFhLWpD!xn`xG9%PX*?l z_b7`?W|mBcV^8YVOQTMlmo#~{Dx=B`AgnmDxxRLHTgT&xO@mpxZLXQ)<;ZS_7(cz8 z?vsHth6jX3Z{Hj;NCE|TXE0G+Qbu@`#p$7NM|=SA@dNz`8crvZA%UQL6sL5v=(@GW z&CeBGq#aTWL^yb-gkVxU46MA!T6s+h#v>XAe17Md-ujfJM{jZX4L~(hF4E`3K@o^E z!*hk^cs>t!1K%l0-wcI1rO^k+4+C~QlXY$g2G}u8g)v}sf%LP3+HNo2u(zKHb}XJ0 z*yV%SSb*Ab=8A(_JQciHQ%ALMj;-=!f?pb>P>gS%Wk+IEOy}LvsS2*Aci<#Cb(v5Rv-ej>kKgCvE1m_`c{$uzY|Si}#4Vy_Spgz&}8l)rBec#ON(bt++iu|si` z<=4fj;9?<#XnLs@FYs4Tb#mM;eujfcyw;dwXxBlWcbb}YsyfW~jd1CL#wukOxyWt9 zH9iY*21rjql=1q7+MVIVY{30{8SN;g2ugkPozQYCcCg=$$6)aY`cpIshMfJoiy2@G zzLP-G6*J(Y$Bu#E(%hpONtMW`(Fq)ZRr`7Jf+Vc0yiitS*NR0$;2;%LH`apdbVK&0(X8OQ;H>};&Nw(|hh68X*%CHkq%vTZX@x(|+pn30o975?A?oFen69kVcHZpLAS5Rf|jmfVf0C}Ge!(! zP52@^o9P&pG&N5P2ncjDQk+`Qr96fn35e&|chx=pDQ0X1cl4cCc3^QEydUrE^t6Kw zvVn&Lqf93se;RoucZ~y4as~a{uNC#^GrMi*M@G7E&2;04@*?s&(B&x`Z(I=O%0IXR zH*o+rBI^9W6L7dWbW?Gf0USi(l=}gg0Jw=8lkKvDl0LXKVP}J-&0u*kb5{%{Zv*se zpZQ~nf0%3%0@s6=36y}uu|JR{|4(JMKypV)$QP7+a0T5hILU9#!b3sZdUAGnS0BJC zF*#iO>}Mmqj@&ymbfh&n^g75qHNRbGgjJl}$)pI*HZq=sZycp1&9nUwC{uNFM7sRM zTbf}bpdgON>826fO$hBUzqv)DR0zk~q1}}LtmP%BXlwTm%+Jrq!Lcp<4cjN+*#fF* zkS+LP+*d|BSFEoE&)AO`7QZwtGLoEMoESO{>0`;3?aHX9M9cXzGU!W#$MqWJlJgv zX7P1*d`%=qAkI!VV^_=-_f2+-UA__8=L0U1DIV8bDFz=56vYDIe&$s`Jxm--Fl`5p z%9E_SU8?0)M}1E+ZU9(()E`FqF%r)ruEs4PO2frbw~+$W)G&D*mNln|DVcdMC!fgS!v!P!ZbS0rRHfB89?5 zSSHXnJ45Bs&5`6}l9?os!qz! zK>uP~$hS_QVOn(C#&U}Suq~bjJZCgy$92!MoyfmJ)#HB;t{&XxRYag*&K}tOVi@@c z=DY=6gqC+Z7Hb$WK4XGJ3j;{z)T_1=AXp|&EeP&0|56G*;zFDaq$cWx*tE4Rm* zPL>7lq7%%}70=$dJ9gn*E|C&=kR#O&0egYFbo>M@&KD(M6bKjw86`h$7O9x|ds7ou z0!kB#gIexY?zZ3B`@`?I zKp$c{8N+^U5iZ6%{JAL)sG9#IpA18R5CD$E%uf_ff^@ReXU>GhP_le7p=jty@xB0) z*aZ2P3uXH3Akl%GZC_RToLcmf2Z{XCLrG?6i&V!LGx9? zx||M|kcNhaP_P9!E$FuQx2~X08bB3RBCr=4{9b9M5`oJ!F-y(7bKsFtH&&1o8cD31 z*Vb^~6*+HWFza};==_~VVmCGFhy;>`ZucciL)~5Bp&n)iKv=UZC(o*d^DZCPlpoGJ z(A%7$(57m0Zy{?cNW2~nxGQ)YZvoTR!Tzo%>sUahw**Wuckd!~40^dwFozroH3 z#lMl_JZJz1Tx3Uc0^o}u1r~VmLULZ7l7acEU=E2GoDv<@Le^P;b+%{q-8@1z%sbAF z4`fF|r`v?eWBVxlFyUhJEP-;jT;_Ke2Rw?ip_}yZVHxU#D3l$czB*W7`rT1U?{IF~ z8%JaUvVeDIumZy95{Sa1gpNSh=Xvg8RD)+?dKV0}ia#(vnfdgd6V^R&(`Bf^uwoxRmqRTdh8u z9VQw+C&xhc^&8@XHCv&Bf*>GwrPKKqIJr%5mAV+U3MhM?0dqkqETBJ20i-$F$X}t& zJ>S_8urOBFAZT}IlDHR+6bWEr`Pu`b=CAvdJ3|4QCvhKkPzPfIn9!q#OTvM~3Xcru zag*2c;r^ZUT`ywVn)UH*j)<_sw?!@bZJ#z$*u21st#7C|X^41dk`^0%W037WY3$Z) zqUy@?#hqq-t`eIEXnh;94IH-mtS#00}VRsn01mWsp^yX;p1e zYtZv~U`uc|ZBR5|@x-nub5#3SW^QcvA8Y9DqI)@s0JAT6Wf4F@VqIb;&Q%;h61kmz zZR!}bh8_pKFTmV|w$KmAzF1mcAJ5jgJ@S72{F@5@sQGD)hNezHe7&@63k#)-`!(!Gq4+U+d-2fk@Cl+UkLRt3ne7so~-0`3anUo-M~58SDP%qzByViAvqFN z_y2f%>#!)h=v`P5kWxTE6p#=B0Rd?Nfx!T!k&+f^knR={P!Lc_fgvPDx;qu=?(XjH zI{TRc&wGC7`o8P?>-*<@y=I2}?7iY%Yu#(_{Y>fYHZT01LJ#1R^A2#3RbA;S7@SG? zo%#}ggjQb5L7eBfjCJ2>&a7N_eTBApf6|n>@0biYK1vOW?10O$nrC)IpVY7Pkt2S-jkG`qf?qxdGU8ewhy54mt zqa5}R?lUr^15^KVU>!8KN`M9nx{rj208qBkM@>MQC&Zbao_?XN9cKooy*tash5i^a z>FS`R?YM)3lkLGZrABVjoiIh_3LF6`AiD=a(Ld-*xyD%bOGs=6T?0cEpFG>WGVgZg zWE+b8a%W*!=$3SjIp;w!?@S}EG_La~33;RL=HH`f54oo;OZo7nVk7HkS0t5=ruJB7 zi>O#MWTz7M8U=8lAKfyMXR`%6)54eXfA~qwFq&4I6^m;+o`UnN5ge|X`6J$=HeQmh z8;lDBN9c}|HH9rp*#W}6?8UaiAn@4E&Yz!oJ0tqUH6P4yx7Y&)!@;q^5cp5lIfQZS zqSL!#Ep|pYUi+XA3D2-cmqRnBO4dO%V~CL|3i4j++v_Ts9(+$!+OVld^<&`{`)*bf5fEKL3x>nAzHzN4gMVKo}2QZHLwNeGVB%ez5_(e~Tr|f?`+BK6eelfIPhj=k2 z-x|ELR1p?3=EA6q_qfN!eK?-2{_-7x21=IQ(3rk6>b0^BU&&jM=9qkCfT}IkDR@e-|Vah$ zv|6$yjhG53JCoxx8P}U#1C+|o+wRStRto{ZkQV-SoS#Q}hxQE<&NV`etUf@ghC<3q zXse{_OCVgeQP9F*9|1sOAitVGe+N4Mvk8izs&h~Qz8PlFH=(n&bY!GKX1cC87dg5`^aIA zWy-BR?$pYVsZuJ#Qz6^hyL{7G@gI)L4m-Q7=Ej>^`cxYljCW(ZvKe-R#G+I1WK~wQ z0%g9+Rhx5Y1^igg+ZI?tlh;m}i%W0WbE2}(NfzzN(yksCD)~74bZ;gilhmz3I%2KF zIYdYF&?`W$&S_@n>GW(|e9E%ES40T~oVKlkzp3n|c{^k0hS;Dxn`zyV5nBtr4l-R@ zPlbWaUZY%xJ2=|I=K>m6xm-CKzO($1)9WaM>{9m+@x@tTm$K_0C>RHK7H0?o|wv$Lq8j#SPm?kEjTT}mcqzNHPBTI z)y@q~5!f2`n~AmzjC2L8T)9S;1HY^OWn(N=lZU#xnNPV{GP?LeI@goMI0~ zIWx-`Mx1t&bK1^Ma$c@BRkpaS0krxsk#5K~n{jo8(dj|@JeS#~pOr#i^lzA=ic#|M ziT!6T!~XUMBM26djVDY6+EHlsNAGoMhCi=GngO_B1?(kI{gVt%4j#KDL)Td$Q?m7l z_gd!7PNtN%1BIaw4s8hlf`Q#b110^d*S|2}&-%6-{B)@bL;w$k4p-;5)SNHS8k14; zxTkTrlHg|G0~a^JF3M#q8h&7kXaU#Vc}niW+XPss0bWp_VB!U{6UDHz)sWj9zH3?g z-Tw7N{X}N|ZRlv%qEw*Jk#|2c56kOa&*hS%=$FOM@|l98n~cdEI)x0gQcMCzbGlkv zQlog0Ym#YvbH`0~R7Sw%!#pJ_^b%q{11Gkcdcjp(FU`#K9HsvQA4eF+PCC6OoOM_P zZx;A7kb%o10s|z)gf^2;Hc9}3WVU5_*3H%Jb)MyQ=@g4t9FkSlFHBG<~>VGK1d^|oD&9k z)!P{(HW+NPt_waAVepa08$neZ)Hojep3-;!kk{Ua4g5PwYvK2v%DlNm&kQh_w!k+F zSZR>mnq!tXBkM%pFi%6@Y*UhsSLD|*^fy#P(8a7CPvSZ2mQ#yv^<>D`ydpTK^VLB7 zbBO^(ir1sm{CsX~hzRcDkm|3N5*tCPSFIYtV4CfN1pLoTrufsi}S` zqj?LycW&Ap+o4hb=oJ($buWwx$1clhMX3H>IO0xC)UNjKoT&0lU9L2$ma%HGC2jnU zQBvevAC<`bEDSRbY}|2ka_y~5^{>W(d*Si}9sjyI<#7jO?jA37v5??XLHY|cW9g!B z^1cTt*?{9($QTFy;~-X*$*W(^>G|bv875ci3?=37Pf z0UWhQW_(R%6Vjt!S^1#X>2~IsA~{YP<9^qG!an$5^dp02D3SE4`DwIoY?tSmi>9>D zNfgrwbClrIj9U&V2ctS|GGSHh42?e>t*3wH2JnuRI?mrO-aMpH+`SEV_IoUNK(kdI zSaI^)&T#O-qTg2K6KtvxIP1(S*l#(Cm}{dmeRSM5M`bd~-a4bph`HDfbkgOVcA z9;nS7Eumk!z^7OwK3Oo`{*DjfQ}s)OdI+D00T%tNdMnQh2n4IG<>In6H&@~Ei+q0B z=*7Ip%eIDH0WNN>C6hkQ$1tVzeQ`400nM8%T5SGgdXar^!G^-T)2rAN6`2FV_S@)u zXOku$lv{=}WxhpbP#RW53(W<+2}x)mS_`@=dhME-ftU9-`IuY&FnR=(Nhi5)38|PI zW++CtE)OY&vK`gCXocl=#eOT$qhbrmm0mPE9WC!;-`sBE)G4uYAInh*=+De4=~*y1 zdAdp|@REZwx494C7Kh#Rour)Lg0|jcv*G2qw8+(4G|KA2f43En95)6SJ}G_fRFZFh zspY48%zmwYMqtBjaE9o`9;(o8-|QJ;^7CfVfnVI`0GLB?hZPdqVO7p(ZsqJtdhAf>dKoaaHs1Rf9Yjs5`th?4x|e@@ezWA7i-rWNXc@7R|E)gjbP zg4nNT3~xrZyYTr%3mMdC$H@Q}oAHBb&9-}O4S}fK{W*QTxIlG2$@J*Tbniqw-CTcA zisT_+>&{5(=l6~hJ`4VN{L}cl;y6M)!IITC!6rzbT!A@dH>&;T8;*BHvDC| z2ZvClqQW_fRZ;NvsmjCoJmpl^neVArn@(ug(mKViJ-bHECoR2bZK3KG}2JMXC zK=1zvJ`z=H0DQP3ztoyT;4}1vQxljJwG#jzztJy(>Yx|iU@JchzrN+B`pi>+x$>YB z-7_bL?6bGYb^ht~agMv6BUXgn#_p2yrty-%^Z0{U>Y2|5@7`*woQ|BOtJHzVI9lIQr|n9EM_yF!hmd=H_DDf_s=NzQy#qn%DlUGw+*#K}9TgblsoV2Ylrtr!c3IhnQ`Y@0}iuXW$sX`N-+(v3{1(`oh>cxm&IhT8MUJB zIe~40LFn2-Y5yId54Z`Q6^~#@gdc4Mj?tA`5`hE2+TusQ8%)4A4^S7yfn$zfTiy4d zp!i4<-%kg)*DeP_j7|T?5~D{T?Vo)|CosR&O3v0ap4uAdwB9wUW}KoyS4!u6+_38I zdM9SDYlp7I5w++S(J)hx_ui*(ga7YYFhaaxBce*~&V3LIMk98=A3JZj46JYQcNIbo zTFKcK<`JgOta>+QmICk({06M^&`km3x|fGGg|0;n%N%;czMu_P1`q~}OhPQ$v3J9x z4`ZG}ZdC3GeZ*@B+K{(ULj{mQQOvUP?TplSF+iPXv%vqKG;I%}49fZZZJ*h?GTH74 zsy3=&#JuRz7pQT<*_Frw3v^sjr zp9FL>m5v%g^o<&5xVPK5HWe5gy4UqhJo^p`7g`}2YPdMg02z9r81f1Plt=~5J_?te z#>_d6foN?cci`^~`P7cF*ZGd}zcM!h5uK zQ_yL#8?>|c-8sda^?n}P_U`Kab)j4h;g>i<{ zO>EOiPE~wvw3mi`z}n6Pl{Uke}=>vHhHSS20AXEpxPK9oV$4mdTSz;&u#Q9(cPp zZU-WbRmb}&-G!gUKNaCn9;>Z!w&s4uT={O0TE#zSNl&Hh>_{(&Ysk{zrG$h;|E*V! z)Mm{|uVSnrvt1okkum(CAJAs znf4%tF%JJiUBQ)-cW_oMvHyLrZaw<_ka@G&i0VE3G2th(B+bF*m8xd+^I7pFF9WMA ztr~iG6>-@{9BPVloO*aS1Di6!cuWQAgXbuNRMsjO-A@jDwuM%8`An-|3(k!{a$E+) z04PR7DJox5Xqj;G*nrj4R~N{mLU1R-&fW#H+v9&bz*zIcG*T@zVyla>@dnC#TfDYP zvspdZ1AbhZ*tUVFug3rKb#L^C{%+-dBl*miT(&f;G{!a|dNTgTcik^eEf;L9CXEbg zTsn-37CJWe8kyN18`UDxErV8O*5gZ>+6nRNg-myGDUS-4*f>HWou(-mWGcAg2l*Ie zij*mf48jji!-Q|X1e^f-o`k*n!HP16+4jR9g4Ry!k5pf#v|%F}SBiBKK+=L)RAKeuWpKT3}RJsm^Ek=`p8c8DC%E zn7e@{K85v9ygM7Qd4mX3^{E#-t;69`J9rjQ!B=$$>THJEHd=~hF3_P7LT#sT-)lrH z%SUWQo&%b_odZgKF)5;J-7fLNCZEWoJ~Ow!TB;O!=;0Z0&u(?>7qbtzlZusgF|ND5 zGV9?5Hc6yg-L{lsn)&TVerthag)h!3H1t*$Nw`=5>EN?xw)@}0A4UVxh%o7pGCQWxlLFb?VWs5TWZ13 zc}`6y=o~{WQ?_5==z776jpgGg79Di-MMv_8@!Kzv^f|mL=H%6vPi`xQqN+W(@%-#r zf=8*7v^n7~dC=m6KYRD}!DZgc80k>#y606a01WJ!Bs*kK$Y^}?SMpCH^Bcv+D5!>j z4!3BhHjUMhXZ){O(&qO9LY*^4zPjjp-~@t?%lhb0()aHL?G|51Y&!e6J|&R6zb>@> z;$U)1X4<={EtzA8cZ~0q6P0^Pa(zNw?vtllnQYjsu6C`-f-^z*q(4rq#AF_ZeBe1f z4od#XwKC(55oiFYF>=Fdp?~iPP&GA#2%tWjqq^uA){@7=7zW$l0z+9Jj=Agfidn(x zq}jDxWu%>`4K2=S+r%?<4#n;4bPkq-@b&6rB_#D|uQ`|R{6^$QPqhiVh4E^+YC@z~ zLFs&=^hB0 zm4~h~{#JuOt|VFp(dWdjpNu z;79sV>!F#sf(Y?Y`U^Nz3Bq=4Lo722`~Ti206RrZdPwHK+p2c_x2%@BTsRt@@8gx{ z8>qCET{sux^lY_a3v(;oevU1SXJwbyz3?jhqFG?U3o_jsMuRp7TpEY1#;S5JuJ8`( zvz*pGa4aw!r6`_Hn`+rxy>q4;qBqY~WOI^u<%2{nV6{uiS>t$h4i$3O`hCvm+)uOV zZdDZ9?i+TbyxQPmmh8^%BHSEorc=DmeUeW)6KZm%U9opCDEvpszp?a$KbJTrzdO=w z$h->HP`anxU^aX&I-?=Z)SinZJ)_L+)9_;(g%BN;xFlAwx%%A9R*p=-z3}h;b=$0$ z57{!#cNY{A5xtffHO5X9eO;}FDMO^U3t&V#gP}B(uYd*z1A`b2uI&tet_>WSc7Qg+ zX8p4g%UeL6E21WcUtm9>?uDUT0O~GB*a_}OT>hB_pH6K4o-9BRVlw3T$*=8nosAViYwhKyfNypL-znD*+*!!Kz3{A4Ragg|= z>1!#iekL9J7F`|~HcZpkC~+3?E>x6xe~;;9ksNovi3T zFWbo@0Nfl#WS15WE$ULsfK!tBh1 z(_^lviA!p;=xAsHCBJ-YKU15(n@y-5jWDM^2Fwk(k+Ys%V;7edpOn>wWP`f;-LK1u z9mRzTDsHWV%T)nB;;ZHO5{PO6F&kefU$7ss^B?HNaeAJo6YifE(K(I1*QkRZ$Su+D zRZxM5VyqSo2%0Tb^!~eu!$0EEg0b0~7uLM#Rq`dS;GAsW?kPYrjmj5HBexQUpDn0` zs6anZKGGT@idJv7>n+(Fl$bX4{-|sxSpNk0%s|Nma-WG>5W#Jd+Mw-#dtK0WKosb8 zXTpkNiOdO>?5{r}N2SDI56lfnQa$h6-JErAWu9E0zaOUQYMpN9Txdq?b7!FE9cY_m zlsXpSm2ej}x^-Z}(^6hBMDxQaot6`gUH4s^^!eUaYBcYGeq&O_elTA7f-lpNRLs_A zn@8*8A2o2vm=28(5IGLNuO=OKAAi{Mtm}pKeQ}JDP_5=-4BrVjs8B)}7fiMGv}&my z7E%-`&%Cm_!d9%bT4LZb>3Ozax}3Q(!r zI)PkA-nf9RyszQ?P`z}yEJ$zkDyLLPs(}`qWY6K1af9U~S8{=oIvz(d#3Hb^baBYD_=|u*YWOUdU=30~s9tu&8Dm ztTLELQq$+9@6SpwGP8sTs^$gXu}{IHpyL4NEm{{gt<#6{!uCkr6jzS5Tz0dgEM4?! z657F`^iqTfqbWaTI$mNrL@Mm4P>Jj7Pm`oRkx3KD%Bl;bq)M$bFg$emZ`!5 z?(a#vIp3+bX;!@Ygs8rg$6x^`mn#mW3+r8Hx%O0VLKSRB-?uT$NBrP^-+1ZicMV$t zb^_ubmx(@=;@tg2`1cd_ecv=gd{$q+_u5h>ub+RzppA{i`-G))?-n*k)%&-ve=lb| z8naGGpWTJ8R@goraZYJfYpZ@LWA9}>dbVIa*L%{E+!l3`uF|)HSKdNm<1664R9A^b z{-9)tziy@EWY+6Qgw4gsVWvA0bFnP}-%7`>lf=Rm4IPV10@j|aveP-(i1tA`BzgIf zu}4xx3}wFO7NOx!V9ru$5j;fHu=kfd2GB-C5PCEpLz+TaaG8~*-Rs}bg3rOlNLU1h z2$;Nib8X1iN#7TiN0W_y)vjj;mveEpPb4bIcioz+eb?aYzk{l$b_WsrPQfAwUWy~% z^_VqRHOtZ5ijDXzUB7gL{E75T;UtG=YT&waY3LRyhuYCnAM?yZr;S{;sIBs>&1E+4 z!aHY+lWlFn+rQRxl*`S=FYnC>pE15Goo21s$aEe$dg^jdb8fft>?mygz0{h5%MwYZ z<5P!4&KN4nydL43?ryWceQpN&FO60lrQ=qg%Jl0UMwz>6_3s(A-Tvh|R<>?3u^=;; z{%E&}^qBfx#SBkwYnLNjzS2f+<8`rp`Oa81wt2K$^LSwIF<)BY>Gss6sofpxw(D0{ zD#bQ-TtpCs<$F`wnG7q!OZ@DyH$9Cl$#>;ku84e@r{>icLto$m(yD((EmXoP^oR8$ z0hNZbi#BaL+GP>MP2%ii4a3Ewg(N2vUJp4O2Vp^RskysI5eE?x*hG9OO$EoLkP@e7 zZSN+}Ro2=GTJQH6t6W#x{UunfHnycT!D>GeyKj9wKy!7KJfrN`hejxRg*;3{I5(x^ zK4|Zoeqv^KJ?7N9W?y20) z50!Ez%&JkB;SSD;Y>A9hQ@O%c*-nYT)4lbwr-3+@bcp`SD|^oB*^LUrv$*^Yklf#5hzLarkGVwcWWD(Ld-RV(o{b=5} z7AAt&e^g!`Dwwrz=Q2G)+Ip}m+0KAz5uG*U=^IlcR!|c&vLS6xX}ImW62<1_iQk|4 zh-Yv)Yb)b17S@sv8I) zs2|FCB#nii949k)CC6V^Vc5Q*{MzXtH4wjv>SSM~g8c>YmFGT`2N9E%tt+%^?gkDg zCYnmGhx5Ci@hcf9oVT^DDJ$kGm&rA4hli>K zdCU5m+cHZg#U|=W1+DcJlUuLotTSb7ci$!>&(m-}1j-f|%X_OoleD@;vDvBfheoW7 zDb0OosP%Y$lgd^2vB_8+aU8A!A#^RC&vaJn~^u#6r5KzSrVR6U?HZ`+y1|GoU>gWHlRUWVn^!E>Dw`0$@iP1x2 zr`Mks&0NNJvABQsT_9kyhUf34o;I^0)_3JCCx2$u%!r_aToz-_>00U)TOGDjhBA)+ zV<)?_77g~5w=*5Q4I6Z8r{#e2)wpN z!lRM%y#HN;Qtk(_`1z-KC5xL>KIFaX^4-Q$QlTHy&lXN||0s|2F$NYdVfy_ERXdX> z6pu7n!sBy4ru>^JeKIU`S*P z)brg%-;FG|0b&|U@1}^vURy($ys;EV^Cwt$D6 zCty$QeH2!=kcN}kr~)RTa<{MY)RvT)pSv>4=^}8>(*a7DEpEb12wdZwp_#X|8@V^~ zK0eS@;?eqn&mJ_fsMCR&ZdkAhq&w;MfGu^oOogolk^oKh zFnYjg5D?E^f$iZLIItY)9Pa5$f zN8c}n8MGUzU}?{Qn*g4;*ueeI@dXw;f@aoSP+H;CXLgxae+ab*a9jmkRe`@jXh-vL z1_fN+xWLQG-b@}cD(_cS@AaOw>V-i^C*j?iCA$ai& zn9wtvxZ98od<72b>6@6Wh=C4I5g#z0MjMq0jOxb5{vrkZG+VBe06E!TQb0;Pstok! z>P}z*+8*5L;77y7E@C%vfgS>@L<4tMyn79T&g12Nf+rhZ{;+B=ESY8%J@44j0==@# zkToeNT=xiL&mbUe$Mk58l`s3vPGasLDJ7L3XMT8LP1ITMK{@Ar)ngW?h<#{y(1;IA zgPx9ho{Q13FI^5yMpPZ4`5I*1v>##C7$EwYt>i@^1sDw0U5kdCH8qSHlymx=EM;be zPr#y;g5?+>09OJomu*$r?|_aDuboU_!nei|F79wM<|_N@k?wfgnOSL}}<<2gZ607p;W^IB;=(mle`7WpQjU z;Y!{Fq3PXQwqgRPhPUi}gbY%hPq02ndtd&4U#17RPLKD2R>fn@t>OPq`T%m_QIMlTx)>1V z6|h!;OA$}8?%f5ezw6Zwj>=$hy#Ue|sp;%a0fJUcT;{a`ix!h!{D@vlKYp^^|4ZW& zQV$nifD|T}XXi}|9iEj#!#f^cz)!h_pohe2fhW4vERS6R|HmXwPM6wtD@hOO$)o@Y zO>11^;R|T3rlbTm#?D?lfyi-z{sJM}!2JKqHli#VYbJcTdA(fi#(3V9bzpu0e)Fkn zhj)}h`eZRtd#T3nkM@yS-&Z9Q=yFxauvw_*btNwxkAsB3rZI25WCRB+^8eWcvG{M_ zM2}zuQK7lSh#NW3Jb*k7`U`FR0QR7#>M&yF1$>sd@fQ(z)U8sOBkjYF>u0rYmZbL{ zI)Nv2MDpuA_uynzz0+D|VcW00kfiOC7^(QU(F`3+QuR%bUf)N>Fb(qEexsND)^&72 zW&um*D!Kw0Y8g8PL5DW93J5sqp`0_@cu?|w?ENFx(TM2%v8~X~81w`&D3!{esCs+l?!!WudIMv|Jh8HsuLuMA zxvtc~AI|nlHzD&8Fqfy;Sp3&&=H8$Y0lh`suk-`biOmMK?#}}EZi6(dE2$8R$6q5# zVOPaUWaLvVJ>zOo zowUTCXaUv*$1iuz^_&N0*qIU7O9*Kgj9^Rz4pI2`e#Y)*1+h#Z8y9lc5HMQ$1Cv%b zIJaDUh`m#eCtzAOfvFuMW*RBm#N`F9S(X*bk*;rWK#j5j^k6K*vhxus72pK&vdFka>No9j?faD*=!XQaZ3nou zf#rI$kh0TVwB&I;1tghJcO+mE=xn&c&@j z48($^$BTvy{DST^u@t1pE8s_DviGEx#lRD(SdMyGWkE%(d0(j3tS&Lq%b#~Jc6X@+ z$Y2R4I55>uc%z=57A2BA8Sodb+Ch61AlwkR%XclT=o5YTtMj{D$?du-tRughny zEMf5FY1y(Mrbkgu*8*tX2z85eGLKv^V9%}YHxV^`Qkd}jm($98L2#$%*%@Bmm{Dagra396+m!H)Zdv!QdFw*#h=53(!}bCGKcR|XI_PTl?=KzR5P zJn7|PsU4I|amVuOxuWT{GoGF`E+svc`6ILAC}8c3QK0#_{O+Jqq~w`aQF3{HFDG)A z4?*b@l#NmXNGuhAw7dQ+*)c<2FDD8IW?9t_WLGP_`e6qcQ_s@UGP*u`V_#stiJw+! zuX*u~{lVtU#ISYSwe!-*#MRaID{88S6=0EDCcmy;sKEmai}}`?z~dhH$M^f5qL9B@ zf=T-`|G;E>06^;_!L6Y8<89%w3^qT&HPAe1ud{#dn;RP&KS{SxLCF<5aV~;bRNB3F zZti*F;~(W&SwkDf4E9W`YspR9Qz{nlJa}-`-KSb6x-&J&Caq3qK zw7>*(qL`FJ>yZGhgUP2BT8A984yLf$T@WP}O?C0np5jj5vT0=54^ngYIZd=cLrwq9 zB={W>n+N8_VDz*Wwa-(le~@Nezd+Uou0M*{1oLYDR~U4qT0_W8Ho9Gd;P6EpSMC5BpZ6Op^@(4{0)GbcYr2gu!0R4}F2OdATmh)3P|Idk zfol*8i>kD*F27pyih^JA_=X*2(Rt8S3HBJ%a@yVnE1w3>4wgvFT|mUj^Cb3myu^Wo zlikMWT}pr8JTr(gktz!yoazI>Z$#`I3nU|k;{+$H^ zKWxZu1I8cl8vNbao)SLNi(o@Q(Sw@WCl^b2KrIXCBUU9_4h5_Pw()G+g}xWK08GE; z0E=zx=_t4iSt`sL%fk)B3hyM?H9(2a%dN4J0odbRPz3A4|oC5z>LabL#ni2muOoW z`$%ha%XCoD5IurT z)*H?Wk#Cx-RzR$ zQOlSpc!!e~kMdU(Kk$?F!o@^sENUVm|6n83)uY7wjfj6bJAWp=xEG*R#h1-Re63%;A2?qf3H+Sd+D$`kzPK5@0QO*eL%7 z0=n(7Noco0zQJxMz&i2(%Edo|Jn22`ps2*bC@?J&c3J&9P$CP6K*ri~TdUt=sGdgFAOV= zbDUTtZBq9&B;%@Iz`sOU7F~#dNkmv&R4eq z)KqQDZy}^hNyBa>tpEosksrClcrXPr4IzImC_et8mdyq7N)OCza#T#^SSImNQy-A% zPoC9!j9@^84JV-7oqIV@B8tKYm#7g4gE!82(9jnNKTRYRtrSmfLs1z_@J%;ZQ2b5X z3T$BfQbm_o$J{)uxMS3nIhmN%E8J$6+i%-74#b^Q?l>WfLG%b3An7msMs(-nQ)7E@ z%oCrv&)9awF-#!tiN_OOBkFhB*!DG4GvzpOpGfj1k+lYbZkGi@_r58eI|uxog9oeo zfvmThl~xS`&*k|lh`PxPP3c`C{ftdCv0cci{m*LSGB`;vT{P@=Kzb5o{->8aItsPy4FEW|S zXj9PNgyk(=xFyJjs0-a48RxmO<7WS$7GnffpU@O-p)V?K+87J^ZhsUF5u=!{#A zDN#JBJS;Pw92OdL9@FB150ND%SzlBHSoave4z;}CO$eg!CPR_5^q@9?>WV5u$VitK z-l6MbJG>hizVg$BMFX%v+27?>)#l=te`16Ui+wyr4(zc3sA2!Q0mTcrLZ`p~b?BO^ z2m<>$PZ+95ge-whokEKI88Br6?mB{B;3r4(z7OI!`-6^Xm-$;tuE>C-X!}uVif5zS zxs{Cp!MX!KbiPE}mtOpw0q`WdC z=)GT5aBDtW3iq!i(VE!bv$~e4;L<_TC1(Fgkh2mu$r+m0?{SgDaXe*Iv78lM8*Oz{zTy@+(zqD~2w z0jaVBld_|+ZJq6T;_#o^2lasP1lR%TXi=0)SzKuKY?94&#a=PwRN5;+SU%GacmYs@ zabeR?JqV+*&re6E_JC!q#b0HCXUW_0dHg5Cya`Z^(VG8t#6Q%qA8 z_+~sONV>_6U9+%_wL1>J`{2ae}W^hgW%N~XN;y3LVFn2Q*Ma)7S7KD=O;HAr{R{}F$jHC+8*lE3e{W1=r zjd2H+K@Eb+vHO&4iRJ<+nLSz~b2s$g;t9;;g`A=B)>X_Y7`UQb!xId6?dNRxN8rmh zSOb!{WvYy&g{gzzD2RAu8y2W%(2dF@ag~~3E zy=r@MOHWT>HqP{^W{KHytJpw|3HD=uQJ1)$=t&av7}2Ta2grF*KL-;cPW>r`nk5jj zE3VDxqh?!fS`kErk3S${AiXL99jSx}__X5i&&cYzrfF0;FARS6!wrs$Y7P1HmcYQ+ zxKXE1RN({DZ-hg#A!Xsb`FTYCKPf$EEFajM{{YBF^__E5j+Kk}$oFf`H9rdqaSnb} z&_C$T`LNi!kHJr+GUqz&aEuXMJ4KP%$-Gaz$5<^*>Bw7AQiLk$Zvi&2>pn3YMLmJJ zzIlrVr$G#dxwD;Ftzqu`^^V6yAum*ueP<1<-GR)a@&_$eGnS2al$AT1;Ad@`buPCZ zEADg$Dn>JSs)=nulE5*syZ#PhqpyhsY1vf}Ak|*|7r1mVK1P|b!0URs?QWSs4Xyv3 zuNyuFKb^9)3kk&uW=5;KBz9cKQP6<(fY`y8Qi%@yqwGJ)7?707(OV^k3m!>^`8@@u2J78P<}5IQ_S8WuEZE@@c3kzE5n64szAMH40~09dO?$Az^nlFqxV-O8NJp4*AUxAO9OBD3MbzJG6<>(jDDoz2zQ$pOr3 zA~iZa7DY~vkR{e8K(@xxQ>Tks#xNMR*CNELhiIDncEz8pmP`VPEc8ItTSvZ!yq;j1+o^(ya!n#nWwUqS9*E5)a6w)EeE35JvYj3r zNbWBHjl@_UVMutPoGJ>m5njC$+*-A7jg{TFZU;T*>GVy29vng!_9g9#ru00c%`VCy znr&O5KxE%2s^^zodaH%%!Tqw$dZttn+diJtV$Nd)%%3HdTIsgCs68D2C3_&{ef5Iv zF-b#k7;01~g&Q15xz1+gkA?F4ScvpvqY*X-bgz6hPEzp{$)`0EovgbRKCHZJh&nY- z{!FFbLcFGC(g;Gglw6#6zP(?6WZ~%xn%Kq_77CARQ{>{4<8vX6RXG3?u3cI&Mw$#H zKvyiA)6_19@W`B{$&kSG+z5H9D@_^5uR-fa1R;q#Hfn~;pI=zu#Pbp6LUqTJ2tI+m zmgsZIv^@@(VYY-`QR0!D5nR|rwoh%Wd-b8hz(niXUhb1_mp5Bp(Iu5*#ora?ia#c) z3KJvQmj-~7bn-aYE+|2Q`7$~A(J22hYk9lXY`KhCV)qhYNqqk-=^-Gmf`0$Aq!ip> z5AUM4@^eS?tFcdA*zR1x{&Tg*bg009kA6}xv}lLOdx3&i(UTSKRy#5+DsK-jo%3yV zj9*n!x_z~L_XpeS9`hk*4aC^iKnxTS=mFQu#pkSi!LJj9KdoYt?(unE91%qQ+LqH9 zxzolY3P{c>!;Sffb-ueXXHlV;Nrtgw&6i~+e(~|<$m<)ADeQp$(Hsf9<9DQ5 z+<--%U*!@g!JOH}+}*h}7jI_nSh3nJdTf1rn_&KzgzM=Rki&>b%4)76pIAt=|KelC zj2>^-n}@DHmJF;hQjS~mweQVe@!$dr?o9Jj`WI9Q!F(#a;ely>Gz`4g4tc?yQjmq# zgJvRQT}oj|8^~Z-&5s0>g~Zd;J(8C#UEKQ)C^NS_b=FP>g;}kZv@3rHaS(L5s#ne} z;Njwk%<{x+y6Q+{DD zfbXs{T7TuJJ2Gw;fXev!K)ySpexkIi(ViTlXX=h98N&sg)M53br3{}bqSx~sE{bS( z^s;`s9LLAj>>JOC8(}}#KH^aBH_kXoJ#JN8*-=l;uhA)0{=O-orr7O<9s!7??&U)x zLF5vVR3Mskbx4{uHv$woZFc9)nM)qx0I5U5m%Ax|ugrFfOFRvCJa+g@GYqQF@F>{r zici}9n@e73x7i=j>~j@66$k2);8a z_T?9*_(kze2HnAw)W{}%$7}Mgb-xgIkG=X5Z_-y=sE~pVWJCXc^}8+j!=B`&VOo;; zZ-pKeSNou-&pYoy~LeV?BssaeWg<&=LEf@ z+Ca+u|H-6tplz` zk>#IqnZE*{-PqoB=Ypks zXy;L^WbW&4@zlQPm5q08+ALR8VmvAx%z&PzRFQvj)5h4;Xs9E5RWa%#^`M+`IW+-B zBZ1C=EkgVtTp*4?#|y@Z?tSekwg`88tLlId(8q3%LLhR^z|D0?Kbd1qS^Z{j?k)~- zFZj@9K}Tps*4oyALc*#%fHB|BbE`b5kzG9JR!*5)E5dL6sfhsMT#;ddWq91JMv6($JTI~-X#lV(ScoXNW zhbBYB;_pZFODcAZpj(|8h7IdcqGT}322beX zIW2*@tpnvMJ|&G9lt5j9Kz9cCx1K?!^Jn#EIeOz)`QvsDA-iE6`b~gXbLD zmmUN8DjfTyS*jV5{AyJkpRD*J;}~$eJOk=zjWkAIJoN!aBk$Xhz(l`Wlv2uLU$ef; zyB4%cBBQev)Va;yOkXLCl{b+X8V+z=cCa=<%%Cq4&tDe|VUEM)=uTc8M(-q^H^BJS zil7f?>r6u^NulK3P-C0L7COL*Al#(7UVFo}8a(s$a*i zJnhfScL2-zVcTtt27KZ%8&amA)LISPl?Y|Q9uydpk+D$rHt%%%&7xh_@`|pQ=$wE0=D^bESLp)! z3~8ipz^xbYds* zTuoJORp3<}PUo~d$do716hBDmjW=wSF=%Ft%h?8HdfY@!LLHcwRk97w39(IMQn!(P zo;5c^YIcGKb2=?xvX7oZ#4_NwWxb={N;LJvO!hsIS?H6-y$YBmvgQ`K3){Mf&i_;``5c26`{d?teBu?DERc<5JS}s<(>Ug@u*^6xK_+Hc*${T#en*Y2tH4plV$ubX1 z5x~uYoxC^rt?V&KIygoxgU; z#Tbp$A9|v`5P)=q;Qra_ELCGq+)@Uk8OO*9}uchYj%MZ)MGXCHQ$fIE&CNA7h${H>cp2?j!o5x^$h^ykqv|0w&NJ`^U9kRClTEXlD{vJg*^BmSZt_yY*F z31={GHJF*o;jAyO(TL4MA1B-zxkscOJi~dEy?uqsoHzUXidpV|n$6UJy-*AVZ@DP@ zQDN^^xhO6<^V>STp4fesZCB-5x{3~CKzcEy%q?>>+~OHN)@@X$>tZiPt1mDTlDK81 zK-g{m^$9J(idOOWRqrkbW^vvHl`8@SNa+tFQ{DFIOyGx#!OQ2fmVEZ6V*J%R$;!a5 zk*QadbRu4;klS`R5G7NXTYF7Ej>wRk=>|7215tb}2P@ls8f$t9?7K@ftUi=)O$IHL zryYP3^B4qMFi_hl+UeSF=MdX*Jbnc-X7OaQK@^Ou(r3xaGTPaOhlI}lTvXRfK9aMV zIaPXVJU~sZZ9nPoKpBnsisT4lO?|=yG;KdB+k@s8%DthEzz%uopVv@v)VT5%j<#{4 zgGcdq?~~uByxnRaEw{i%KiFMuv*Q-NF2_pv}F(~`o zb)EoJmJ}D|EIklL&vXtI+9fQ>XRv;wyUVO4??H-kz9~WU{=<~4wkI_4#Sv;VCE%_Y zn&*}&ULV9a=JZ}Cmu0kf{$g&g059DNm^*|o>}_}}nHY~bbe!*!cvAodB0R6B@?MT) zr-%ooB&MSe(!Dt~QWo6th6SPX1CC^5D4{VwH~k;A_U66^lSjCl=c8;9#a6ZaiTKCL zJthT?mBnbHxLyGzSHwF+UnIBCCRA3m~+;oH<2b3oT?Jk^) zNo|lcD(c7sRL2gGqQB&k1nLIv@(+V!po^#b;$j1Yut*a#on8TL%sj~<0p|l$y%7vQ4xuX$TCAtX*p5ZDk7;!*|KjliA<5Eb4toy z6xl{0yOBvMYoW0-#4xsDF#F8!^`_2!-{0^3`P|p_yMDj_?yKt@UFUM%ulM`)dM=OW z^Z9r_U*O9}UzcrMi|9xf_@4@5+Kd0v$+As?j4m*1BZ-Ls0 zZT=nG^T+xG7dM^tfNf^9xk+RXoiNYo(#nnmtd?yZ%ysDCv#)9HT)}m$*|T`8d^|Zr z=alNv4qP2qwA3*(*nHXZaBm=VaFF0{ z0(G9-Pe69NtRQn&U=Hog?gg%2@W{2gOdGNvY**r$!)15OXo6rpc`>7Bm}H0R+B+ZS z)gPjD9D43UJC!Hr6uj%&Vv>A)$D_^8X1mziL~fbA6jU3b_-Hp57IexZ?nY14l1iRW(~e>g01S^XmHOVN*nDKH@1?fA>*f{XH#5d~*< z?fvuCYl-cm+T*50OX;AtFE3`$sI<-A(~~XI0c2v$THFAcI<*&9!0u~nd3X*^@h;+S zxYT5v9ZK3Ay` zAMYE6fhdP~R!UUwP-gg!SLPOf`U^e%($sLy>$S9nxti!E#WNS{d~HIr#4wTV$;^oC~z4P7JV znNxxyP!w#i#qZf!ErWzIi9Nr1aXdiT*+%V4_FvuLf6jC4g%}Ztfs*9TO&y>VebIRx z?csa#TlLvssYgY(rh|d5Zvl)?iS4>aGhaKM%dWX7u|G6*z>KpI;-F2U+)g9bJJtjK z(pn7xl*K_v{JH0+zfJZGW-%|eds@A=KV2=N=U8vDnjVNcAo3K`cd9XH8C+_0VYy35 zKuuxNpksTB-r{CMsBCamRK#%0rhm3Q8iFdCkm~G zu+gi*W2M+r%+11xWKTNv;4dx!2!M~5>^XlJWTzJnYkJqhjMY2z-fXVaT_^BNOvDi4 zwf>Vf_^&0OPuY@oh#rvH+Zk>U?j$Ma$5EmSSRmprhu1{anBEPgm&)Mny$8Ra;O4T9y+N_ty!V2^J(om9{QO(d=ztyRtT( zH&=BB+tt=>oil&iYO3qd36Ul3Q-}w9c=(8d)8u1c5>q40h71a!I*P7zlp(lvs4iln z4`yfZ*S-pkpIen#2CPY;wpWpg3>0w7%mvfMTJ}y}2JWWb5NNfdlws%Ukc5Oq=iHMCL0{sqlX5X=9U@n+6{A)Wz)6 z=gEK6>i>#5dIc6xUJ*Jz`s!-YzeODreTvz++F$cnU}Bq_>)+tR9(4tK35bh4#=g{w z{&T$O_Q_aD@ZEjtr+9Z-j$6&<7lID|HLd-H3HJ#^*>6Z|tMCXk_a6^5kmKVowXPB- zY!v{r;l(9#;em6qUvSNQ2jq#m@MO$lVQo6r3TQ^^%TNzPq@~PV0j~^ zdzOj4dT^Awy+$iI+WNbKfK=qn(dqzXC}vWo+P=W8KTZ2@QI$Zk@85qSTDo)67WFiE z;!fu_2tsBl+=Z{qzUln?nH2X3`p(siudDVWa=&1Ug*frEN!NY9g2YyNk4}^AAn*7l zcUfEd>HOD#y#u8nks}XBtx_&C0eP~*?$hhYvi~5w{=%WLn>->qcA+wfD&Bp7u`cxa zqtZ#-(UK5?WUT5LkDhs9_(N|8znCV9`)(u~CsVwaY zZB9_Q(=?wtJT?Qq^J84CNEfhLV((3CiIX>-1-b?MbfrX%?DETMMehFr63yH1TLI%2 z^-=u8LmLU>MmtW7lah;-$}egDPvq+aaihSZ^Ti8Clwts<_}zKy*~7Q$j|%mbCY$4R zC!T{Q4eLzUI$8%s;QKI!vc;wCcxIegCwZHBj;7r^sizx? zM?b@yzu=s9J@;1VC%14VqX7Nt!vZJMfLNc`6A9d58pUu8k~rANZ_+LrkT%E<&UVm( z(kk^8f*=FC=Nj=JGq%=7aegOQxYO9GvLeJO?V(Nck>_)I&Y2uv9mK~uYR(NJgSWd2 z7_$84e1A!KT`B#FkYuaO9Rep)p#}Fb0@O?)0@xF6NBL2-fYns@F(|vDYX!(kIgI3D zP^$NH?kihGNUI=G&|Ez_^NK)%s8J#|ZZYJ^lLWKDYYFwQU263p_N*&5?MV!f+;7RK zz@QH%t2LQ;AiLU{uZlK#b?!f$O;LbZatq(9uI|d{x0AT zjMRN`P&?A@$PPUU=wexo%Z@ubD&uTs&PLdR&B;GrupgK{#j}|a3B}2s^)VZNi7S9) z0P!C+!{qGjxC^$^y3)|8a$@JUoUS9-l}(eIHQ>FQ@sXp_v#h3T6Wa)BAgkSC(VF|n zezQl3*(2zq4q*R?RLZYk9B;O8YTe~6VCD8Q80wnSI|K*^2^x=%{Avw94;3*eixWMS zKccN13MzL&jW~^(R(|@BNxCMkcTFxj@9&A{d1qGC9>(4Z)krY%5CVm+*teg^oPJW>G=8$15$y}0;xctG5?9L0Uq zPevBcz?*-uljvJw7|~Ac|CnySSpzsCsIlykyvG+{=k%uj8f7KOPf8qQQmekc?1D(> zU7CRLTDE8eb*rt4f*Hgvh&vT@J#Fq{ZH+}Q>zpdO)Dhc6e1>@o zZ~JYl=0%vN2T3mlG=?NYN?hhqKlLR<$jHLV-ft_eHXj^5pfrt-opdO2+Hy(v`!65~ zWMVVqWKIRQZWY=p4aWRcZMD2*Ba>^Mxhr4McB-oByWQ!xQc+9bAXamNKX!J+fK%U9 ze)=_DO-2J*`tI+}Pe;?BuB}4ZNr{g|7aRvL<-c4KaDaCKX!@h)LMb9DJKD8*#I9Who=lJ|xnxJ~}zjRQM~r+@9#TksK|pBy1R*(AdVu&1|N0}vJOzMhJMypTIn~FuF1t~5g!54xgU?47ThfD(Ev{8@u!Pl?@x#@ z>Z=ZxeLDlz$W{eMv&+_8=^qt+0rCib|L{@3FLcC6t>hU2^O^V-9#smq>sq(Pb&($$ zYunZI*Xwo*QAT~320|D_3980Xi#m=l$D|8Wflvz*blwjkQM$R~YbHtX?~O((tQG(G zAr;UhL|Fa?3N`mHjKn-@(=|v(QE{y#0P#~ z4pxa`Yzr6&E9|(euO7Wh(KZswt3f@+nj~ON)4+6tHCiZitQTK^4UriSUib>4rG%lb zEux!5wq^6)J9mS7pI1-yKPkaO79=M9ce{oU3XKYXU06TO*tX;pmE){+^Kny?WgnFP zG*g8~Ipg59;O0Q&;ufv2<7=tRO~0D02qFM49l1MYD=0Fcxw`Y$cl?{3C(CLe?><1@ zH_w@??*tPz+Qu$@0=e2#3@Bv`dLcml_)AlI5dKIA*Mn%-q4evVZ9tYO`OL1_0#Ng? zm4+Jmmef~Q3s?JhtAT22$HBQSkQ8wK%MD5jT=oT8t!f>0#~E1_4SkZbTG%fR;!)7F zwkz?tN7G%f*pb!o9xd`mxzV+G;F`)j%_~TODd07}<=>N+{&Z4xdDVbOzsA$V0?-Vn zxwA#zWE}$IT$=d7KKB0miXmmd-6CQmuH(pSmJp3FNuf&-GBKJ0t*Jya+$^D^gE} zdM_Ryzvlka)T)_fBG(?Q^zR?63b%D^{hYa4iNV1?{?uWxf(*3Xr*?SFfERr&wYwT% zzYD6%z&5P>(eXO(F;^rTq)(3HcAo!3)gY=1f%q_D(!=q-PXFA+e`IRO3 zPtbL>Z}9Lf5D!B3q|Er=Sd;R%AO;7={(4`Q3Q)R-X+TD2ju(Ne&(=`tufY}Pp?>$| zN3|lGCT-S^$@1SF(+*uCrrX}165r7Vp7iNV{j}wiE5vKR^mh?*ZAnMhvw=;dv^Nh4 zirC6R8xS`Lnz%s=^?z^q%5|bD5xMV@ZCql4z*n0+0fPT_ceT|o!T%fT+N=9xYtPRi z8#~Q1U#yzxnOI<^HGmgB1md^JhY3|syA_2@C~hSQS^nZ*oBvtIOxmibwvwSns>@p7x&7a! z*O-C8>YM}5y?i3eH5z*E!~|fEhKa12W&yE(W16D`M={;tK3wm+zZ!OP!3$?W3bT4( zK+!&evYg(=D@4UC08>_5D6H1xK>lCs?ef9j+d*XQ_aTuuXv^Wp>38x0+UbBrf!0q= zqmu5O@rRg!O#aH!8S9VQBg<|BqeYe%IbicY`pm7i|A4bVgA?V3_8NzRuljK!I|?*z zuWE^%5Hv*rYZeCJ$dc!a!-6+ujz+?O0oDDzSt_T!#@FZvNNT7eqZp)Rn6dUVQN*}< zW>d6C2v_ls*1e#H0X}Xu4+jZkM*w*GD~D5Th(iQ*!Tb+ub1h1G3o>(Vqrq%(K{LJ91o4tOF0aGEp_vk^!WLx$USKiT z|41z;hv)WG9S2beq-x!V{)JM69=S}%bnP7}C46hWsf1^NRMfE0D2AB7r~2y7C0U2y z&7c4_v6;$pbpoHcYh;=5vt?NDKQbWT+oe^FK-=lG*s7c){|gu1rT;tbtiO#Rx1pPq zr*9yKl=l5H>tK%--MxDgvKTR|l_YQMRk`qj%1AS9YkT@k9;~_@K$?fQh*G}@r~fbf zdDGn)r#o(App%9@Q|IM9W``Etl&g9%@PXLTXS1WrY9+U)_e8A|szFJJkik{2khosF z45UUaE8vA5*v@ac(e|4>(pD8#Na4S?gH!Wr`-(pJ0W7S5ej8`wjeo^DaZxvlp-~Mw z_#KraezjH2{PDoE6j-t&N!DYcfBpf^5Ww}Hjz?r0FykBkX>l{tTsygA|KjBE*iTUV z^+JUa;1Ht@FDHS^Q>1Hz)$KzjSXQwgzckFm;&6;%T$Q?QDMAT&#X#8UFv7(P?FeWr5k z@2{y{=eBMG3b8s0=QCLL=sUfwYhllME#UO~zD@M)HP3O^)v$!VyEyXK(AldJJW?CJo3?Crp2z4Y`D2UZ0*LZ~%@eLeM@lu+6} zt%|wLib<=76$prq|2G^0OZ=x`cmTB@eA8W_FP57cJJUlxyp_%voH`>55Wyw8(tDsX zRD;sPT|K{sjgvupZr2aWHMTMk4}FRF3}@yTm1%?AG8(Zkngjz=jnLe!MWc@iVygpm zpovE&7yE@VD)2WEJ)H;ZPOLg8`GVKr-o#VgV%9-O+NoOJ#PhJo@U%gbt`iyw87UY_ zp#_C@>brT4*7pHfE5q^8OL93C$&VnqGeP8RqMKsvRh8-GEHFRsyufR5u+^X)e62CM zPYt|Y`gzJRU}tR!W{fXM2)OgoOl|yrmz&Ez1eH18rsEDw(iH_v2LHy%iY8M7v3|tl z^O6V5kHosIJPsLor*Mwl?%KJ-(cUN{O2UOt6b}dk}71BrU$G;&TV| zCojYD&XYdRxx?~^!zQ*kA6!WaCQS6@fhLT6b{E3F7=X1aw1GjZk9LQ zkxE}`Jg4111ukmB^SX5!PUAw|LfYb;?Je?Y#xJE-D21U=>@yqby}G}D?M>#kSY-L7 z&hP%Y#l!#g{SHjp7JZa%MuRxN%UBHR*}Vbq>{7|DKN3`yS*O*CY-B#21pwinKu-4v zFm-z`Rd*Qzv*3AZ<-N0Hu72t_;c)%Tpwze8^IwJ1ytVV{$0yP5U}1wcVq%}Ui^UH;8d{3y(k2(^Di9RvR5X-w7iH7YO% zGUoZA&EdNz^~|F9OHIg_{=qqTL)bJ9N&ZTUEDol%;&*}J0@$?cO5Mjbp#g>!QwI2r z!T6@w#IWFI7yedM5FwECAh^oEqJMDy+|*O^;5h6sdl=8zF%TGiIMHf>$YFID4eJq0;4`Ddz_!Nsg^OIJb>7V(~yC z+=TSGF$2F)hKn6@P(DbbF>1Z$%Jocc`Z-Phq!DpEnPTy~jL2`J(hdXw?n;&mdhR;PLRTp)3_N0$J zIA~J9`WU(&8Ba#HiMwpw7{MbYMj!T<{#g9goIQV95cR-hg5lfR?I6!Jq-(j$R{Ol| zl_yOJ*Xf!}hw6xySGIb6KX`ax{zG}q0jHpaa(9$G_TZ!t2E)x@a*Bt-rVIUBx*fRN zYv#Ij#D|#^LWlWd@{%zA2W|t&Qlj+@pY>plU`$PFD8s{B z4-DT-dA5Rr1_2ttg9FZ@YnKV1bI_4#BaTd(;$7qA6AE%olGEq>o?-(G)(vO&nl(%^UIIWkt2 zh9B$>uJlIsZ*mBH3#CV81pvPBH(bt7iUn{^3?!2WHLRwT?|~@iXn|_71<2?+BhQ~X zI=GeszXuWpJDgQpr=f=a+h;k&?7IpkUOVf*MSeoHhj>2?6e&G4VPlGUp1HAz>>OE{ta9T#~sJ@SgFn;or?+6Mj zu8>Mk{M|5MP#>9AVzA(z!5OG@C4>*UPmpuaD?eUX)4Z@@1NfL=G^49yFWK&4RbUe1 z{m=n=`brO_jxO%t%p)Ac4$tllmm^L|?5SN|Zi=C&QDZ}K^oJ@iX#{6V=MI{I>2IH* zCGJNmI0uX%XfT=e))^d5sOx8W$M%{1d6}K_sBr%tcWhb%C4Wx_odxH&93;6Ko4#c* z$=0U3ltVx5kH_g$vdSAZUDEo=JhlvJTqk{F`vo2iFVBa6EWtG-___m3b}9B$j9{w; z_t{1oP)q2X`Y40OiJ6Z=bNNOon5!vF_9?zw?wp!n8t7bvxwToVYORCYmg5Hi6$keQ zw)trcfGWpwtJj7leS(@-@X6&hO}DK~#W>9_921j>{>DbiDx?xWACUTW)!_Th0yPe{*%#gEn_!oB^{e8e`(~}!$Mkg?YFsc90GBJ|1hn=&VgfC z!@uC8DRAB|oK}t4%3J>2`8;2|9jU%LhtUNYLA1Ps4xjCBkarw1D5zt_UAj_WjarIq zYfx;nNOTncOuw%%jkTE#Tr%hWB7w`1H~LOtHNI7Lmn<)E(Y ztkDM|fEl8Fr+y^D?3~85=Xqy)mQMrYJhWg%`g9eI1VIy+uO@g?_WOhH)xWp^00y~rFRzW$ybl!bTFp?AeKkN^Qlns? z=B@_)g*QqrJeJhe!nmb!JXgEsM<^SkgI*~8U3Y0Y9w#|ni02h#+DJ@CFn;K?R(sd4P1)0X(>?{cK%&REVG#w4AvTi7ES@xig*{kW}F7dJ7lA6g*2?|37x!aph*upL#}5sWI=8HS2+- zj?t)HBQZl#`a!eVOEOZh-O+#kL+3F$Uy6L0Scr;lRYBgZTuXQ1FBB@>XW zQwm@o2XUi(d@1gop?RPV`@(@mUAmT#S1%6TrIoS0}@y5ZE~eY zE(~juuQ#)4$A*E>+HLXjpNCuxz_ITqKZ&z6;fpc0688I2_#m?r7<*em^hDp6hwa;ROm~`0?P?dn_W7k;Dt8-Z%u&1C zP4PGRtll@kwkz(ATbDIAOUXxVhOwz^g<;;O9K2J(zU41TCOHgRqr;$HiNG1MpI>*5 zus4c0>ZlXmJ>3B2h`aA(EM;UE?Kip9n?=5HwZN^wot!0T31*)w+|LhDK?Yu~m`lzm z^5?R3B2PUH18yluYUnxR94@mn2g!Ty+wzx=#!4-RjMnBM?~>cFsT(^_dj7L{l8D>F zOn{M3hNl3O92}^84OV4WM$C%fMrsJ6;mKV32{inCi2ddOkfj<6Y(6M@S-azjLBlx( zzo<5Amd*s_&@HoYu#u&phjK0WJ_C`CUT*H&`#T6+gjCTXy^W-MSk%!smJz%AfXl2_ zWkA%}J-o=j<%$B@c!pM?5@$ ztg4%1Mv=k4l~mWAQW7MT4S{N4_HK}D7^irwQ6&Bgsa0Ge78+Yj9bpgE_KbbDHr1`+ z&BD*R&x4qXFX_NxU>fP=Jz*)n?EYcb-_3KN_#%RoI$zjXr%!ZN2@3C7%344yGjVr> z5g{0AyAt1GfqdZ!a%aTkBliOT7IYomi`1WA2YXjJ{EoB(cXn?F?gUvlT)39{l9XuE z`97^})}PP4M_p8~7S2S43}!T7QNEx5ygOy%Vu-xK?@0T8a6$%-vNl|qB)nqN?$F}x z{{Es$jCfy|!J)pT#tg!2m1P{F!g$Xu<~`Jt*EQ8RX{liuq1(mr$KQ6Gyl(SM|OCRX5oiuHdHO?GG}lVMSm_^+wz3(<+c7 zdsK(L2DvR(nx0gv4zR2GVR_sE$iV8K0bB!!AR?F+#zjGnEK;m=TqiR!$T($CJu+HV z$wOtw!{@+{NyHa5ulcbS;K$rH1Xosp(nOd@Q}^xBtox&JCd0hXI1mvR8cg5C*6`Cv z%ec^U3$-|x2(KQhGyNGTpF8ul6u8@OR9$X9H#_a5GI{$1F(%wQ1UKpzvEX;skIgGX zi zty6BiJ8U?uggl>fB!e@_Zs{RVt~3UHm}R_HBVdN<4@RfwMmk<+u#OtCsL}BmDl3bxU&!B~Kg$r; zG4jnAoeoKZyvnJftFrKNTB*9Wj^`2-;90Q-D)+a-K@K3jK;ryizEoO3m~@axtUGtu&;?2~65G$?wW5 ziUgL+Si){>cN1>LzJ(oSc z9bIX|XFtq*l-ao*jdV%@6E(qUI^IKkw0cQZpzRGJz65NcEO4G6k0rZ`S$Op<4~uSH#=PGi)d!YHlvrgm#!36G|D9)n0IJbSu!}eXh$;6 z`6sR8;`(pEtnHJMT&dnq zJPka{?*Hsru7atrs=7GHvwV|Of7l>-n-n(U0j7BZ?D&`GmD08JXDKvU!nCaG(IN1& z!R#&;ZoJTnlbknRNSL09tS@od8iRWE0OHclnkviE+1% zFPWV~Y}5RZ%Z~1fnlOdv!{AmE;Rfy#Z-acv)k{&@`&|^Qz&#VhsZMEJgcp=_Juv-O zklfR&v)`EShoD5PA^Su|O4*sy1EKBcUW1wmhD^iY9D-X>bu&IHBWt5^?%BQh;qgr$ z64NhL6!-)8$R4J)85D`kqlqcs5AG}~T`H1F+c)MkN4;MgtRrwk7{RCdF1_)YDvvK& z;eWsk%RDe;6(4Mu?&su<4LO?}C>}tC(@v5euBSf}?MNIRp5M|gi5(idw_nQ}+(tto z`sY57^z(Hb%E-A1V15_>qMp>C=T+F-7dCvn`83QGMsfKH&g?jM9>i0iM%aGi0mw?3 zM|G*A#+9s86M_3*Zh+js25|q4c^)!rwFYQNMm6QN0Yp}fN-!x^M$b zYAl+QbqEEQKJYiy(P!fiIc>Q{(=Oiyb5~e@ispr8HfInLllU4OPOLrzbGGm9y_A2B zSkN_cFo?prI6uo@xJPBE&p8x)TY``>2j4%i6S`^PyC^I$eJ2b}SSH3oS2IwXmC*7B zhCNe8pGwLPdcEJ-h;reL!DFtY!VXsuBjqvBg$Z}ung&cglEPs0>fs?L!Ed8#li!QD zjuq{rtzzwGG0b-i&@J9=V9NyUy)t)IzJ3oy$(o4oZ6A@s9L5ZNcjUG zSx|Z@&3O{G))H{$XEUvS<}JCj4KXLii|0VHqkp;A1_1-!E`@D0`!Qv=Xfv$Ly=P*m zT>F7mv{hu<x>ix_u$gzL`R1O<@B$a9BnO>iThTV=Wc!VQv=_Ltd&z-w#Hvn2P)xoUG= zXht?P>YONL@pCLrt|L%kZ&xxLn!xw;y=7!&6|$#462opY*@N@!4~G0_f zHL0q2dWDN?lY@sUz#|>#fY4n|<&cBnCFQlDx#g!5O6Ywq0BnuyH5d@{TA`n z5@U$@H1DF%sq~#2{5Q-_8$P)H!u{t)#DFr6Qkkn1t>Z_Y2UyDo{L>W+ULp(k0`e=9 zmWlf}e{#yy_KDfAGgdQ5SMZ1U^i|-zx^_&1_EV_dzc#$56q+4E5{hiIp-TV@tffqy zQHTL|2srGko#LocROXF$%`kD+$6k4zym3^-#hVq=!jPj!9=f8GZ_=s z|IZ6e=4*gpT#t2n{J!8m1&?MWQcJU;51rdM@Cnee{^hqq6GK5c6{51RQdT)g8=b>J23c9 z0R;UhF?Un&rZkXwfBIhkEpJ%8M*t~KDL>CzFnPx+WeqUw3x+iccXa#)j0$hVrk#U- z7>Gb~ouobfsR6I$fE{wyTn}RwyT%T= zZMB7Wp0Y104D#*Mvytrfxeey*>jfBa@dBVhoLMYd;|TrN@#@n%qx{56$GMg4r(>{# zfGH6Gs1IMy83DrU5v+{}wgrqy}ru5=Xcsln)oyuI`d?R$%olEMd zyMdE}6@TV)BheZ;`4GynJSV>9M#MnIOKryUNzTpaOw%iL7HL_n?CT8pSDc5`8i<2< zcx=z)P^@6uKKO4{)|q>v-qj$1$1Rc`tr4Us=t_0_%!mn!?B4y$kbfE`F-V|x%1Nh| z`PV7e1%73cr11O;(gA&!^1T}NOCj`TmO;H0l;-*T-f9`$6L~D}dd$fq(YM84=!$8^ ze>L+J`s3L;C+Yjgbv^GNc~lW!ePqW<=17so9>06+&54=!l;$Kwe&4xO(7|_|AYJJ3 zWG1zhp6Z!bJJ{KLfdcNaBNviMTdr1lr$P5G=Jl7OBJ+~m9E&G_a93!pY0E1Ao@KSczR9($_9uvOoOFC^lRof&uRvpkjr6B+vVVq~ z)s-+qjVOD;RLptT?_gA}mPj^gHI+aaG@$a?XugheaElEgd}xJrg^i+z59@&Zm^vm( ziW+9n)~Lcua!0J@mR|D?4rL=|a4^}8Mn9YZRo?!glnStGh+#5eYkd^)|<(t$fPzh}BZ4aWW9`zVbVThjhi5*{+|NVqd>jA!*VyU^RH z4rp$9t$gk`nm-f7lfVRx3+#(yk9QxW<4dbgnw-8;c-v;Sc?DH@YY z>|w4;#rLvzD$E5>Jhye6R7LDZecq*j4@#Tn^qy*}VNvo4aGvj#ij^|lqb>tqC$BLD zB>R|LU**t>@$TO89K?zbp@~mh?>ZkjuQ)_^$%*wUzpx+gBEeOz8cy%dnt?49qa~VZ zM-vjTK4Tw;F-WJ-@wZ&Zy!XR=UT`9PrQZczqT{mMgpAGLF#h)CDh=d@afWU>ssymu zmdJx9bRnFRpHr4#S`lN`ry+<~>pW%^EzAM2x%8-E@W9k78!vTOL@G;f@HG&*li2do zu|V#+sOhF}``f@Kxqb8hYq}}>w)4+9j7#8_l-E*r-L;%}(c*g(@LC3ME z6?pW;GQQ&w`)l#xp5IMYNOUO&aHn05D@G~C08dsU^%0pR%rWl7VsjqOC3O5_o)iwv zM#p%)tNh+G?4uAtP9eo8z-18Z&)62Y8jRin6bD*aUeIqNBR-7)Dy${c`)xm!A-8BT%qErmIyV+BKEs2a|g)RZJz| zLXR;e9*$ds~MC6oZuOGU)b?VBi4~jiKs59bw*n#?NaWJqCaef5%Si2 zJhK`twqM6+FNm}cbSGUJAukXGv6mF{Hdad1?J4po9ZT>zbH_v8XRib;BZE0#Jgps+ z!T&b0S79FR^N>e)!mT02D&q6o52Z>bzqkMw;$5@K(bPxXo_5W%N(4Wh&{ju6ynDLz5bu?P z!)z^IEytnBY1iaZtjq(PKyb5H6Q0sa%Ecf&o!(Ib{Q}>p&HL9!FuY$!H~u!9CS{n* z`etc`atZk;19Rr`i70AHtNm6sjvwL^G9g25xJBmw$fLCTP4E9&9~nrakQ^_2oYbqhXahAZRG6d_P$N&)Ka&+zQ*pqU^J?*?fd77aq;7dmH?-u|rsN$!1j~ILaj%iQlgaOowJE zRXY{;k4WrqM;(%LzL&o{k^<@~O2e1QOknL*!xppPPNq-xY=~?#B##z%e|OXoF)GBf zUAwc!D6<@&NK<7Lb--;Dd3;$Sm6>PgMq^-4q97HN6e+9bW9h#y75g--L*Ws zCOf{1@H5I7my(2(qT|_Vc}-K=JeEPbvHAcX7F@uKEO(sU!sFtt3LC8bhI>|axftTa z&YpZb=uMw?(1M#z9BbcbjUHMM-x%1kAEq>#H8MKJ7+QvBoO}CBZDZvHK;T53PpBLk zrP%n%oC!%d5y>usKqc7qBlWGD!tuP(= zqjn`9X0GUPkJXXWzVZ5Zr||a?Arw$GvSLo6ZTTxNeK>{#BeBNzeDsf6I_~z-y&PGK zQ_BZt;;a^=BS0dtS#!7C`s;k&U!7!!=W$J2mD{DIB94ZPD-D?*a8W_r-B!ao?Z5nf zgU|7daFS2{f`MOFSP$#IX|A;MQ`S9n%=4Z=(WnDRRbTGeP_Zgh1^8ol@>f625omZ{ zYxB@&h!g1YyUO+s>nJD0uoun01!aF8Cz`D_V)EGAYw5n(t~KcQN;zPwM6=1t=XO!+ z@HTOS6!t%$nKC?<)ikR1X@f&R81Zw4BpY6V;sq)h@DMX4{ek)TCb^W+FDIiNo%WIm z@CXM*iD#>LNz;gqq_~o<;3Au@Mk-2>cW*q@Xj#zMl+&mkTSjS`ubt)Q#8emIn?4LT z3v(LV?2hi*kYg3n$&aVwgT^v_LLj7=io5r)0~(b z4o&hme0S{eimxtgkNMY1@gwx%Kd2V~JZw`0fp3Bb$MedHin`Pk(vF@?u_=T9fS|1KI zn6|FawdP53^uq81^ms~xv?U?{(IH20RzmZuJ?Q{m26bmV8p?&eL}Wdg~)s=+em4 z!*CdE?qO~I%m^ifDp4uNTd>7fn(sitHowjg1cYTI(b$Il)==V)x z8)4M@Hb@!R@(A0OZ;R8g#e5t!q}ZiXh!M6g{KAsesiZ|qx(b296D!=Va5GI2gRwD*1^-|=1Gh` z#fQd|BjI*h{we{VY8&<#^n1PErY>oea3kLNImPbU+vCP6%)0h!h1Q$`Xu=>$wHwCcz=sHrwy0#2$!y_sIVegxVHo-VlOC3hv zXlCaQ1-`2CO|ofTq%-#w{SJXgF;7`0_YWHy49s`*JMimlemkhJ-tq5E^x#vA73{XosrjYF0f-4f&$nJDF64*kxNm*v1rsH z(v~08M&Fa-^3}sP2a0q^IzE22l#J0V&Zru9^uV=_Yra>CT+I-nABhC*;0$r6gpx+P z&!cY3qsK4Bmn$};gnpjRvz%=7D$BHqh;WU#?>f-N|7_QtG4|a(_04@&G6;6E9yIa9 zh5A?!a#?;xX1(e7wCqO9d^Fue8MXwo=T>`_zkn%&n)yRQ_rJ$k3CZ?u5)Ih)u$f%# z75Sj|a6;)Z)nE|M4rtwo5d5ZalnY2{dgGa$BV!E@LPjha1~=9b<4RV7f(MW@Vex|s z8`4n4%!KOG0N!vHUXr}a`^!A(JxO&4lONL9-U6u{)QSx0t5mhi@_c->6`nuHzeA9! z8@yfr#g%TXS;K1O4^0M-BszFYxEy4+_12_|Ol3&q~Fvx8jsb#e~BVdt^W8wat>rx9`n8<-MZS17`1w^s6^vgFB;9EbKOR`1nAtbAv%28u z&33{Z{_>{lvQ3JI7!#p&@yH`n{0cS%7=oRg4b(fn3kw#(3GId;Fwi1mTZeTjQ4;*f%WGG4nPo}< zt=LC#>d+8M6VrV6e6z5V)Fu^kSGdvFM92^*P(xfk%B( zBg9*UG+!-Q#-ceE6(jxrTKs{V*xH6n>Qaf92<|K))tRHag22w z4$2T}io<$^eUyQp7P>sTQd!b2l@mGlcoSQu9Oq2UFCGje+atCPfJu$C25HPd5i5tt z>lY@}ZbXlqjZ~H}GNt%gi}OwPE0k^-;7uckl+u|WD(kxq&MM@~<>OHhWC37a(F2bJC|V)0Q<4Dp7`q|0jSj z4FQC603c`sfDjb9^uSE?ALkSIh?ChrE007O(lozKZ|wy=bGc)ObRI z&s+a2^PG?YMb5o-nGC}+Y{Z$HW0Jp7oq3u1bSS}d%5rj}#lj#3*KwGY=e>~i3<3#cP%b084> zoWVH*Q5&}K7M{}!dAzFfblJJkVc{F!mr|VB->6^4G90+a_QL)1{Rv8F_)70=Rd@3!W7JT%9LXVl zp(+?;gv*YVr1Sr#VzKBWjBn3!VlnVgI%U|3^+AE)GQ?}6>D-Gd4QXw1myF>;Lfp32?Ip_2&Lq%?Wn z8JBN)MwXerQ76!0p@vL?8ERoq_e#*eg5R`_OGXPNUzCNmXz>nP* zgDr}0VzfURl1X3A#^W9+M62@`Ul46wa}0Wd#F)DLLGJfrXCm;vLO@mjd#didjnrCEAPvyOQ1!1R&s>o9622BT2J zk)*l4$j8+gK^cyia_)5g1h^Dc`DV@_nZ4vQ&plf~KndHsVA*KK|HIjv$5S0{aii?E z+Jkz z_PMkl+9o=NJz}T7`PuygwQJA>b>9#7oYLCM#zUZe+sfJ#o{ZC&x|mMHZ6u%QyHrqfAw+BGq&wHE-|RIf`^VFY=7-l_8HYV=UOd$|lIbxg`ht)9#IK}! z_v*Sbcfo+g$RwKgq3){fZq!&{Y|E*OcAconWozW)#gyF<=OaVV>E%(WN^?XQN(A<-LnR~qB!nViS zf#s2?n+rGF8Mt$*ZxliC6g%rKJ@=cR@3`*!71TA-xnNp`W!WAJli2QGlns()6F+Dl zRV2}7m^AJkms(oFHMQS(>(uPs&%X`*FOLfD4jPwPPf8j5kyyXL+O++3;oSoF_SQsR zI*j(7m6`A?@ar5W=m!wk33ZEmM$%nH!^8frrgLZPyIHc|a{~cT5!6Sdn6+m8WQ*C5 z;|I6DHBA;Jo2)gpXS$n%R;tRk*7`FSf|!-^GPmah$1;3=FaK_8_x859xx5(d(Tp9* zl>6b+zxX3gQO+B8)ceNbF<*C=1)qmreG}8~6_%@hAA@`9L%p!oH8u4GaqdhRu11as zKJB(*d`OGkYnE9d_!1+*0xf2WBI>ITcwUE~>=qJ|y*x0!(cQ{iAii^F)1hU6mrB{# z4CV;H!Y?0+)8(akSr51_q(*G54Z|SI7`0&B1j;_sNB88PhN#v6`T?;+)!Q!7+|8Z4 zu=}1Cl!&1^sZZ{cwx3aj)VyF<1kwtJRw?gfz-2eV(lmoTe!KH}mAC&^j4f&V{eCM=d-!IkPPp)hc5KRr<~7P9cWh9N{_$=!8z z?a#=zc-rE45cQI-Z-|-y++bYhDdvogf##CGr*@}{H!c2Ij=e4C_VsJ)m*Jp}TZXBZ zDZc;svmni#bZXg2kxuT1XHP|w{^e$i<;uOwqOxKNubDHvHEz7n&t$6}hN>!w&cMlu zF^AZj1GlX7x7rYF)9szRpz|7ggRYz`yQkmUvgJXA#BJY0Tb=C8*HswA8rn2w-#h>y}UKizLv3GW%ELr`**ElU*UzC zf9AXKoAo$*rq2+ybvwWue`npj5(Q8HeEl0|wH3zxsJ=bmhoU0)Lz3;i&@TOMmMT45 zm$;yCZFl;k2ea)zJ=#`nvH*m;L>OAlZk{rhhvtq*6O~+;TO}1>BUS~?Z+k^*USZ;S z832kjK6X2{Br3LSc3EJ$cZxW3^9^`W{Kg{#(4mri($I9+Pt9U&)-oJa*3e>WR1$zhsnDHe4(;Vdalsc{=8ml`H!y4FK?0W4bJJB$ea?Ya`wRO}$?GGN;<1-~Pc8EmrfNzjAm1k&@XyRPU1!||dgV+>af34fhsBM5QOC{O>q4f-In z8<*qVk9c5z>J#=M;Adipmd*Q|f@bHBgr0}*lhT9eM#|H_UVmG1=2EBQpqc;XFl%$8 z|Klr{O&qtr?xO_=*@d+LJ1zB5$*;+SE;e9#*TNxRmYU`sykW4*;1@lmJcpW7b4ws2 z3B;*Dg~yjnWcy>oxJtRRXt?VG@A?FKY_WgnEFyn0ehiS>@R~|{3G{<~*o&GuY|E;G z)bD6tCZvti>~`nRgUQI&9$Xz+k*{t;i#hDYeuT9YqmQ}Yv|_%Vjp=Lr^#R==Lgg$mkNrs$h z9*f;*z0z2YzU;nA*PFChvEroJ_Q?|`zc1^LG>znTkHpVTY#fV@lq1)K@<>9XD_aFy zfvzY*T%W=5RIl*399)3D1pN<%J#+i%zXa73znVE6Z+L32fIbAU8njL(Xul&}HK5ny z2>P*2jh+cOMf6bqkMrrT=5q)=wbw1q2e&w)O1X=ue`Cq4HOT^C(LrV~GYKbgORKw> z{=nSziTT$1J|lw1RK5bqE%2;*3~ydt-FxN9Q!s-tE#hNo>@cH{62X;R+QHN$5q%$5 zgmN*c>GFii(lYx-U$8b zDG-v)#BMw~w=7NrrpvvyZ;1RwT4PNf@4B&(w0)12)H%Pl~ z8gS(c@Z#>n>bUL{`N+{?S4(ZN8U$H*Z=-NR47tXC7$|wG)t=aa_ z>`{0hBbWRU{gENxmDo0XD4@5=I`*mDgmctb??&lI>Grg== z+3RaL!H<8TMWtz~Ja1v;)^}^Yw(bvCx!IcNDKR4-E@B`TQ3(N`hW90GaayeiW4^hO zHdSrx<^^tyE&r&Bm(lkjXUSVyzWx2v7*FBRjqaSLYVuhgfU~sgeMZn-J3ejD-(Dy} z)oGxtdB$on++t4WLeWVvV~VM)Be}U+EJsF9SNT??N6>8Smh3%(zK#Xgks8atq?yBv zzRu*9jm>j(QP>Xv*{qW%x{d{$Qa#Hrq|5nVgy1H4{$7!?zr@$a?8fT*i+`|eiz$3` znb-Of@lw}-C%o|1r&S(XVgI;zXJjH;gg5|bq zW2Umkpd+po3w{VmI>N+df5m}^6nI`lj8Fyb^&l>_H|n%_+~HJT(muXP`Wl{HpvCAS zuVO+_jr*{DA$MZ$W4{;E7PTYI@seV2bh2&pkV?F8s-b)rcBzV1*ajnpue|r}2hW~YX_I0h-;{{H^T{XQ%sH1kzDmxXa<3zyTB(b+FsS3H7fWcMm^q(y|+5dA)=H~VPSuGCx7 z`ZV?P)%}QEd&uC%buh^EzVFssOnqYW%r9`;_u#YOx}eDBjvYV0MxWR7(}ur(ChQburNX zPw7gZox1Zs^o4hu7=+HZ_~xgXatPxM;Im=U%&9G;K1am9 z8vZE}1i^*6F8c~P9eb<)C?18rq(~!F$xBuzG4R<~@!-i+Wb~P7?~)NK`=v(0@WlZK zLm4^f;6wmh8j?g098VMYWSa_K-B!{^@6{HL;KRW;H%!3^53fpNQRXzYfu03r*T~G_ zaQNo~UgB5dZ)s7n$|Ypcms;rjf@a+b#OXcwaJg@ zzt42_0z5U5Am^ykRls+nlSv5IvYH)ynBy=>xlhAv-+~F$)g}ix&D?XGuIKPMMiTj! zJae``;d8^`b2&N&ftb%vImNro#|je-l=2KF9|bbJR)Oa_t3CUMgC;4ZW!;R^dhlTN zfNP{!1U?CR1LS)sWwUsSK_+XZk|?4X~mRp`DeAB&*4OggHct$sLargJ$Cuv|$q-T*krPK%vp40Dn6jSrK zMG~GY0fTMsfNFpZ?vgU#jHAHo!WpV`SGsbL$;(8p$@h2OeEeMx zf;sxclze*c67Nf<_-;MS84tL4IYnlv;)Be)H4keuU5*?3IM;s#bSKSl0v^bwTCE_T- za+5L-BvN`id=DoZZdnudR9cL9V;L1v#CDX`^dR6%5~hc4Kx4Qp5TLVQNSNfy_;Cbk z6=XO*3rG3UYGw-})zLwBI&1 z%~xEFrN9S5vVQf6i=QTsAiOKBFnPoTq4oVt?Nd2Ruz^>}iMPb3vU#Q)FK=ZQG68ik zI?h3MBClQmJ;Q&svkH=uy@^%c`!U7P($Xj9Pn+2845UBzRXsS73!3eRaaM=m;qJW+ zxfe*vMqnz=hLMNi403fHkOozVYyH0we=Udn*5Lv5f)5)|j%@+5@7vdN6|Ec zTl4{uec#Tj*}V6HLH4OxtRu}~Ov2$(WoDsE@WqSNR_i1aM4$qVxc{DXcP?Cl)WN|N zHaD9OwovBa1fN$b3O)KkI^W?-LE!^7mJtG4A{`$1Il=RJ@c)TM+}ju32)%i3s zh&=hBq6zJUr?TP;b+1?=kL*bMt)gia2{NM1D2rF7#q#T*&3Wz>zy-|>lRipz2kybQ zooVYz5CCnPqT_k@KGo0wNAf#b*iRE!u(b{t6)RhK?ENz;KDm(}plW^};=}$?jHdwjcLy57 z$^M-#NZ7fA`q1Yz*aV&L==SMX7ProI^P|E*+F6aiOW!nm;d(iM~=2737b(G6owChv?v^z(_ zy1UugP~+1C+H&37;`3cxg;Jij|I7=wj*D#J|Dv+gZcp%%BH0jV>jn6ChkkgpxQvx= z`G^klou}xJJyhhVfv#f%C$ed`9~mi8L%i{Pgx%}y z;E=_8qLCY2ZrvJAY)(#2_Qd82wt=aj#A1{Cr$Kehd*uRy zM?--u9G;Q^?) zDl3YtuJ?e%Di~bgJ`bxpNcpz3le)H|KP`t3iXSsz=We)IK<`%%cvI&^?8h6Zb9>T@ zJB|%MUV;H>iTk1ZOB?H{aWyTNCi2!8&Sqrr-~qficsF%A{p4s zz4#{ZZ&sdO-yp|D8!#uUw>f2I2^n-DpehK*@AaY-dYnG>7PwM!c|)B?6P#A5*+<4? zr_~s?=#r?}TrE01QY@fmmyb=ESx^Zx55ukHdl#W>svma8+!*(#4cVKqRA{Grfr^;g zyL9r`DXSW{fxq>deyU{)c({c7A|FSgnLwIU8*p{x27t$r#*XFrcKaX_oZIk`@snM{ zX7|J3o_-8aWr5G`3D`id7gd!djPTIHZ(LawmmO9(#DCEj#MA@ z_TGB^=~&xkbrXvC@^S(v7-m`uWex-t!qQ)%|8o`P6Kv%^UEbo_d)2Zaa*-nHz;C@ zIZ}_MvegCJ>Z|SImGpuwEns#cy$8c!0zBt8E3ctYPWZ{KHM^58bE`Re8y9#iL8Sei zoNUn>hvCc~s(t^Wt(14}X|$+^|0WXH4YbX_R{q84-Qn|>;{g6X)o$al+p$IU3zYR2 z-#FN|owkJgPjX`P!K!Osz|!wvDR%%%IpSxLN`T!24Vk6xGMAuZXLICUHK@$N)owWy zKL`t`)v9)(=_-wY;`jI0UiQhMuF=vB6JoBJwH3_MtmMh}$NueA)26QC1-%?zDYFCW zStFocLZziPm;*n({Gqy;R~9a;(p;!5Kp%;gohwPuhyK0tGsCK&2}V!TtSd`bR;d&l zj{5&f=}R9kUAph>vwC|7{#PqF{&d;4+$djKkP^gYCn@a! zq&Ku#+lmhojOagRhBsiX`(Z%RE?8PFbQCv-n8M4Pa-J3&ExCtC(BE9HS4^Xor&f%n zxN3}&X9VFJio8}o;`~E2Ov)Yv*d_ zURiX1m%V|y04Kx`Em3%Ms&HFRzUON8nO*0=e}c_773}9B9W+?C)WJ=C4|9z!VXBknC4*yMW~6H{!ac= zPfAxu^-Rj_Sj#R4&l6=kJRpn|U}n9ck8F-01JiVlsm(-wz^N5fxihI6LvW20<>2+u z{!Nx@IXzr?1bxz-MILysAz1e5n};r@j{~!vrI(_zmsNVvr=UYe+m3|N{$$u|I^^&J2D_hgT0@O_(xAL zFyko@Uze_~G8h?2DOdOoPrX%)~4oYFyfyg&nKyw5~|7v6{y=HKW z4l3i5BuiX+E`;Dxc2*JOeSJo|K8F28qiyfMbYR&1$IH%%*<72s9mJmk-v&>dJo~>8 zPOv;AH242SICYwnlhV`o&i0pOI-1dEJPdvoXDlBbPTFe*RVX&jlyBN>B~+1poHl z84j`=2cU!O(;gkMsU&d5T!Z5EJ@|I!57dE!LHuMn8;!)ATEyTY5V8`1Z?mwi`YrYq zJ-uGeD0Db{@j;b03fB-R$tvDU_6{INpq3kU&D})dhzp~cKIw)9<8ZrM{tuUprT&NCaIKbb6i%?+$-pJtA3a{pW^Z|@SfsgZd zbU2a#5Y(NnLa-H1m%6c~&Pbaz+!vIBvXj{*z>kk2b1npTdWCI}*}HxMqp}BX8kN7S z7nOQ=1Mj&rx%xHqCJ~N0e()tZGi3x9()9Z$c1n~w2E(4T|Gzrk5>7CgmV62THA3g6 z3g7PIC9RH-$rYOfa(vbdlZzQyffjEreN`tC{>c?+6T;&+~tLDgEaL^lh-{j3#RcjiWlvSl`+ZfjtDqr;;}}Uk zl1y4S2JcXFS{=aq8Ro6q@nU;IcO<8)7?$)TBj(dH_&LQdRJ04V+ph1+RqDqAqss%m zB99c&9pL_mluZi9nsf<&Cr*=_nW-AIxE>QQP12|{j7SGP&3(n*{=|U<>mBP1!7AKG z2v)lg*grdszB&bLt>c*{t>6|ZIQraz*#R)bd^$lc%@CkUo;Y#ho%)=zl_Mg?*ITksU) zFDmJC#RMzuJaS#5`Yx7(A37I;)hN~S1*jIbSA`rXz1qJxP**Y2;HH6pUfwe|=WNm( zxRrGN=W+5qhf7P!*u!zZqe=Pc_P+>Y^Mf*mstWu#T==>`v+3t;9wb|jEvEb+LX34B zJGO%HsY0ipXzj<0w0D#?Y;t~rsgy^ z?z#fzYys;1O~cNPhVNf|KHO#C?QCKtX)Al;gpFNZcmRMhB&icZM(f zx~fA(Ho@^?RtZ~`v#u-Xe|DIh3f>VNW$GJza)|8{(uqjF%*xSSo; zRXOw;T?M@zK9VjFAy?fE3{u|2^*?dk`)!c7rYR?vU`7{A=NE)IO%IU zQhh=qGF;UHU3E(RJYa^W!S`7jYH;*}clSK;vL%atG;S;W`NWG#VTXl1hv$0#Fdh7N zVWDPOf6sJE8i0E}QeK=lEn2coHAqWmkemjTK@its7&xz_J*!5*lowV2E$C&I5=^WCfY#O8(G+OCTr3JK;hu_7M1f zqS=<{4#x5aYNwvc=%Iwcqs=j44Pf#-4$2!5D7+n2@ucnbm2-r@qo>yTiCO^#DSR`T%S$h>x#i`1`GXl+rM2b3s0YWO70bf z`CmL;v>=HMyc3wd*-N4c+(w-l`JbO2g$fb@ znW~!;V0VZcjTSTZLdR8msvdLF>bpW=^~{km_v>++O5%ZKFBNAY(=xu_Sg~kZ(Pz)Z zj*20-C`we(Lu4GI50)WCq$~tJIWjmHFAN!!ew0zo94JIfNZ`ELBwH+a;VA^?Pd6dZ zW5rngm~Nyb^;42WgaW@GPQdi#Cj%spc}<9IC_KS{M!YGD`CrV2!oQ#jZKBPjpx)Qq zq!P}c_4S2o%aKdM=HxHhOWo_LF);>?P>E{Ky`t85zVcoXZTA(fYsT*nNh0$p;kw&{ z)G8Qjk33{Zigs<=a&~gjP{y@b9W0CiD<*_8(mQbOZ4(47U^^Y)A3q}#y4cjcOGJ|sf zk1Izv;^<9Yt9y1A_eSE%(8OQYjKt>*KYJ$PWZ-gqjz9hukHOkqGzuZj-9vexLMZdr zf9>Uv31zP?2XxaM|J}$hvgA{2ZP9j6oxHmz`QwBCM{WGk4IwokfwndnOT?K4qelF7 z=7(sG(SfZya7dGsBz^f>hsR=6TW-EeWPnB1_3VkGmAtDeguj7zgFcBW_NCVh>brL2 z1DXV4$-@Er;;iPtO!|T1fZwJ1mE5 zT)^Kq(w*S#Qlz8|`P-)vOZ`niXM0X)CEd(!nf*gcW0y@0hS#7u#j0{r_fj@V#aE)y zX&r?%-IKiKA04$XPjxIoRv}5f8RL7j3p-J^R6=c)!tx zhGZb>E1Z{?eVH8!{ddKreEs6eoU^}aRWuICvBJz#SCY)Pyhf>0rCsJZ!+zOTO@gQE z8E)%p{dxdrOWEZY(kDvH5kbkhLhXtPqd>J%e=OOKKLE!`t@H?1D!FBY;YBfrWE`*G>LiM* z@3Ilxshr?;dv6lmHl|VF zD?kmj=PtiUMDJGoQvOI^eL96ah&mh7TlCDN(Sb$!50j8OfnG7)d(dGyw%&|b;q$b^ zYM_Ar0u3%f+Wu}LV(2-ijQLa3GGClMUU6@)x7R~!rR*!*B%zbE!(#_nQZI@Ov21@OYU+#MYGrigv24CoDzK@#<)-jxyUYE$B;d#2#af$U(Ve})9?9W<@;6xLC^ z_;MnvzW7$PF;jks7H_EjMBIL=^mlBeS9aaU`A@Ldv^~$QlT#?4&zFSGbp)o*L$(`m_|+|=d<(m| z!`S{*e4JR^|I1^kLua*rkFxI42s(SPw0sQ_bOJJE@1s2~af0MBecjukCi3;OQ!)jf z0o<_tBsc8W86fiQlW`f`g9bvGm1wl zze1d;VdZu%f8&s`&y{$|-f_yK3fp}_mWN^@%REZwiZ>4;aSJ{JeAd;5;JW5`lJuT; zN;;&8F}aZQkHsNcpz&zxDJXQ2(Z()dtB!=jzU*X_@WV7meV!fd8nB#>-p|VQv1foAldPd z(zT{RrH&tyy;k_QoRQrO#LK3~mzNT?ml{5XS2Y6sh@+GXVXrPPKT4$9A5?06E-#Um za0)r^t6Ek|#KX|`|6}Kh!`G|a4ZpADF*Ogs2b^`q6Jm_C7cknGnGGx^aWO2KWdm3Nq>|H z=vazQw0f5MdIH12RI{@m$=(z6@w(i5WF71u)_2zrHPfEZ6#0R8AxW34awvNBM1N`? zOiEcP0j^YG($>wx<-DPi%-pn!Cr8=mYTi~ zXHpTJ#&CwyIkMUJ*~jXWuWhOLpO8_JD!y`qbq7fe>F9P^`rX-~j|K~#7-1Oxh2g+d zTJi$8JZJ!O_vq;4@M~KO!153d-`)`HAxab&R<^ak^ty5Se&wXypz~Yt;7s{d2PDf*AeTzM_++703FMZ7(6R z4BK5m=+6@=Z`*H{RtFl@i_7w%W7x+9V@^rQh>v9nTuiTZ3G`;KDyZitO{{GW-)jmk zx7&~wX^dEXdOb?4`sB5qT+LZ0;8=dN4@VqT06B}`^6$w&r{)?^M38|Nc@}m$d64}= zB>Z}Q{xi*)RX4rq<4EQN92P*%9B2$;%b4{pJUJOKosfDnApd~^(`#cm-W_Hc;qOJ= zfl3Kh(Xz~4u_bacp_1E4JP@r--eN@FS%_La{xFK8AVd=|D*bCC&q+iQ3J{h3dma&+ z_tj&2^^}Bo^EaoW8JUDG0_5=xIwh^Gb2L72k9T=*YdKp$*VWlrP>zK>pE3=`jGo#~ z7U+J5d7Qx$-E+;vif7u>4Kvoeu?Q!jEJxmNAb$xo>s}mpmLzR8Pgf(3^G0ORndqNM z4cIw@Jx{_i>&kMtKylp#U2!5N=0R~w?!XBczd9S|+&(mD*2qH|9iMWQ^m#Gn}YMQG`#Y0m2^VoLE7sROPc4rdH z*VIInE^Za%yto=pfiFP3|AhYM^=ymlbw8ZC-g05sS1O&cnlJ#WsKd@sD}=X84wQ(W zapgGnG?daI+>G=|S*5)3+z?7+LyrSZ(=}3-Vj_r3T{gMIn51DTQD13#m3~-rh1sv> zxj5wjvhuH0MGOs=gF5z2s`i%t=C;@Dr{9u7SqEoe>I$;tuHqKG*ym=BR6@`5u6*T$q3l=S z;-{>7Rvn=_-6Z~|^6?j8$`F{4YmZiA1jGO8!@w5GUD!}o3pIneArY@7wqommPvC+< zE4hOYy%yyJlr_kC2`=+`;qf*1VX=tc#>Q!x0{$ zmRe`mGelh8Qe}?+5zWf8NHor)UP015#@|1FUZN@tjXO^fz0agWbPsn3FT1f?UvwP! zn??!hMO=Q$1St>;wW1r_ugpGlJDJa#T5ckJ=SKDT;CwC3R}uJIpS3))$H12}jf4X* z2$}Hwy%#v>kY|{c*Y-S*-Mdiv`z>uMZ6;+wNq&%R;pbRqM--?P({oJou?YrDVs~8_ zg;c??QL%o}p*|us+6Y{(?i?!68bdzCHaMW+ls zPL+;xY`5z*2|qL#A>DmI7a-xW11B+D)o&5ezUJr@h2w&o>D${aq{38Q*rR&MZtkd+ zj!Js;16&8{MWV3r`4P-F2@t5i(m$3lv%jCsSAWUy{hnYv?De$bU#^h>aGuw|#yPmd znbD~VHq3|=Z+_0q4hac|t0Hofd^#mWmWT;9|7Bs~V?K8eS3RXNL7bHyenb8XLA#w)1ORamQ$6r$fzoge~p1OjnOp3o1NL||$1Wto14^Uw4_ zF3Z9avD0SW&%)rVAn-y6`oIh66NEg{YJsWPN?&IqP(z{TWhJaiU=PAMx#4V%AduHl ztA{yTOD9=k`JgZh(Xl-Eja{6)0N%UoB$;vUs9nvJhSBnX$GFmc0p|i?ZVU~wDO@kY zLQkIk`$jtDnMq(=s8~G-jC+wd2^+X8^WvdHXHI%0!7=Yeh%r!M!K0IB{dKy7kQDxJ z2`1;P3eFEw*Qd(-i;Z$({p;te<^D#Bmfk3JvYI);iuK-xLi2* zN<-9|I zUqXk9wRn>r>19@2~56H_rmiXW0;K}x(w5#7y4Gq0*XbC^h`P&{8G6cL9{IzpiW~*7 zdM^G!IejWVe#DghD;cwc#fVG75OQ|!l;mXUC2h$1Zb9%u^x0Zz za>w7WPK`d#>g_lrQ?RS<*+YXhC<#1x?j`DGyzMK_;y87T>AaaQw#by0anbr6}Na{^eNx_k$qZMA% zZ0GwiEQhUhL(kBBjkBir7k78wOpm)zFGh+o;41PQL%tuWJKNDFD+Yu>%b8=|@?r!R zW_A#$x=%PX7_#rva=M&8pF^I(6|B4Lnm z@XX2DJIAQ8(pl5&h&eyo!twAdO%}&Lv!sCzO+POD*qQ}05(b+r6y*&qx$FGad7HVn ze@T+4)GL}T5=VZKxV#duCk@2UQ{BDKN1R?UC~F;?lUfp(mO7TT3p^4YPDFZP zrUrAGzl+HQHKnIJAq4#Lf|rFH%lA;rvE?4=9D2oq^}Yt`+B7+-=;N%5v~LjQg;`Mr zv(3dSI#H@6`i~_K zr}TOv@Q$a@sB#+plJq~#^!k$3qHK96@q|S6m>r}};sGiyyog)`a!Nbk$$6iu|8hX0 z(7RpdHa@>u0i#0;ZX)ouCkZW8%32j-knTqE$;p^r+04zv00AoLe_k8zGHP{?-LR{G z5&KtONdU@wNciQE9KmPcOhNy@Sq%~p58d-gt_3K(_ z+9RlgqJPrH4g$$==3bY~-hYw_q;JV}rLaEnT;cfvbP7-z9EW=umsti}>gQvc$0J2E zIE+0b`>|%*=%Jz69dfEd-iMM$uapUvsP{@+g~lhJD66+ z0A&Q9NJZRNy7kHW?PAJ5>kZ|kHF31iF6s)>`VQ(2uXOzPPBVI^9AAm2Uzi+@x8h8s zz-NJKinv?}(--ua7N_5V7d+R6TD`gSpKDFO_2QT}(2I971*r37;m{*wv@HS&{n z5mS#agnuLHBr`L<)RdOo8@91d=_v~a^mCKL1}2>G6nIATX0a41(VUv$>Lr509_l$+ zt=V8@OmFic-_Fuw5mJ4cT@Hl}^uTt*d!sk61dss49cKh%y%<7UbSH%oy@7BNW3e-K zdZ<}tOy85)(DVm|ZzI#)J<1fHe$+=jqv>6EcA6;HPs726RA7gEEY*JBXvT6Kdot9G zSk*2%Ep^Sh9h4!C7M1wjh4Lcv&i$iAEkjPv-sNu!%YR>T9kpyN*vy)F>0094z0{jq z`EeFZuMJ!D*{+uqyE9Hl1bgTx660+oW^eZP-z`bFr|+!ZCEWhmRs)S;5!4B~Hz7-m z5^Y4{RMJAQjQK(O>RkoQfzeFt5ZgQ09+%+Jp*@O`>wNbvl-;i~M{sn~!cV%J50yi` zZujk@h)VA zQp~QMlZ8Fa1GN9|@mJ9t`<%_7oAovKdBl0Mrc^vDO-FS5&vq$OoAfyd3CJ<3B2=t9 z$EVMJpsEIssi&-tSXkd?jgiYS)}WkQaeQ0ut7{9WiJuLI!F;WADYb zgfqO*I24omd4{FW-e#uvj5a1-M5no*DOAjBH@hs}1u;{r5(TTz@9AU42G(tKjRCSH zs#(;-GOnt&?9C15;oPAng8OAlStNK@W0LAWIt(OQj^D^EwnV*{r%st5NooD|1&vEB zq#WT)ig^zB25X+X28UzSC=CVy9H6HfY~Oqq7Skq?UVC8 zt2$B5#|bFT{n|MVh)>X)8d6jS5l3)VS3#|Ve|8d&bL7oC$)c1;O+(CqPfe=EZI%nT zEGa+vE(6^yjGr(4a}K}L6t4yL-t4|U6@LGhI;eKb)sJ6>F)#NyU6~#ByQC}Y<>67r z;Zp*vSV10V2LXr$azJVyZrD{d3XQVt`8qQ}+}%f&-jNid0`@(YYW&=hri(znxOjZA zYH$yBb#`Kp3j!tVoI*BD@k*d)5+Uwac6Q{KOx!ZKiJhs~!$42FJm4VkH^y}HPxxMr z!Vj*_^PppYU3NlJVR^bGNT7^^P9Vd-N2*9_d5R$3`_|z^n_=4SRd|1LlnS)u(Wcrh zam3v&uMMB~{gzfp*|J%fyjYXVzYD~~U^25ct)?WnaGqHxL!6-&vH))kcqF~>RI!z2 zKOXkw4oAH>=;l#Yev3mKkESC2GEoT9E@U$`GzbCGqpZ0YM{PsEg_l&Aw zgg!POHUftKaaEoOdc_IKUjPl>UWoWRH+%JpS4_ zhmH?LH*jkrq))4};Da(gpVP9sj$i!(*2~@4!uLCVNk<_w`H<%##>m6*Q%^TEb}A}I1ViXNaJx~%n{!5z}b?Tz~_d>=3X@esq3_k0R8r5R*wNa zNva`19th7d!F_4D6GRA$=>!3(chg!Mykxax@Au_e&DW^X+VM!mS7G*lrunCP+Y4t- zP@>M!(s_8ianw4e%@{&XxC2CP*z%{l;3xJZz5on>W>}vR^o9EI zvqT$s;k|wOVVzJ|Vx-U$B_>s)7Duf&i2`q8#Fl@XFYMBit|sU|=+VkUu3BY_n(RaW z0h_gdN*|+P2Fr4wl~ec!s!E~u(iV?67L~*_!mNpXg|SWvc{yPvlf8@YJWhgOFu|z- zD^mb5Sw*L$=~8CK#;3fHW(iO7y6u?V3e^Rt<7bY@fS>tuzsbz22YqQjOB=iYd9!*9 zi-A&MBGbG6H3kyI(^GwlV7M~gGe=Z-*%D`%F)2k%y&3qtto1;l1GEo@O(fOIV)whz z6Kg5%$Tq(qYF138=43E&zt;ge_dyaLpFvgIsvs(H-_4Ijaa{l2cAXX8*57f^I0p3S zDWkrm<7_5DB>-oxLs!FnJ_YR;%YL$Kb8RiTs{|yWX1S-r@#F{Y5?U~O^B}^?7Qh_s_}J)>WEp>dun zcN+OdDi?-mfjpiI%_yGi-M-_DMsJ7%9@%89vrWBX)^aL;WYwwgmnb^*GPB2r;9(mJSzpB9z1RFi=72X%(bJQfNk_EJ4~CRbs-v5q^nv^Hrj~Eg{AJ^3uV7s!;;rg=;I9R4HdT1 zd2}?6n12c>gZhTz;FDW3E)}qAZHdDz%dEGv^43vOKmTz7R?VC%+BP&(WlGFWzw>L$ zTc4`)^053dI{_aMzi>q6I4S^P4w4xxEN=82PfA0w3wLR*HjA-24_=NE)A~~7ps#Vd zZ~J@flNT5$3PMmfo_is5tqQ)W@&~E$b(|a78nf%mWV>|yS00HE*1>|*5B9bAG=HYg`n$dH z;rO|`DR2&*n6$vZ=co30j!As5bnA-Rztxz$pEHaC1`l{Qj%Cn>SA3BU8d(j$X}I0A zK6~o!snugZ!$Ahxbk*WGy(5E4zg$VZRt+;5w{P^nGIPiM*8zYYKevmd`Ys5G%IxKZ zE}G~E=Fl9dRWmP5n(aE0N&%@zKws{ffwoIB-2RWwA(~y3wl^!6{VLPt<7OmhhEA@H zB7U)_DrYK2V!Ag5@`@`sD@h90`ufi{3#A8ID~ryoTHUQ9*E=Du2M-mV-Q?iV->Szw zL(Mc&nmYX88!a|eapKwCOb@vXLSsMU=!MFjAsQ|x*My4`O4dc(4P0(1C{2ro#KDQ)(4|Y z>%IRj$XWP2EUhaAIvEq&`fpbW^R8CQ&9w)_1;y=7e(R-vBIm4O#;1ATP}a;`9cbny zOu!z2s)GHut&2W8xlnMPo_}=+stBMo;iZSpX|^_wZYa-#Aw}0{d#4EN)0KO#$(G7P z0}$j0vR?eDIr)ucm)4*8MyD~6a8gNIE9PZO57co>ar;FU$x+Mg<+BIK^E%sYQQ56g zwh?N!ZQ?B|<>==*oMp0Xwj0aoY`fF0%N|oep$A#>TeXf+JT+LKnUo#iD;}8*G|W3a zS(-cu+02P)A%vD%K9ln4a;?>pYirFQV>Ba=PBB!DROHCIq0(u4@e2d@lcs=WRjvy5 zI9^q6^p?6fEw|$Ai^WED@uCgE(RkZrPo$Enk>NHTI{U8;6_(Wu+C{jH)Za)mm!Qdu zgE~hM9mPbKC8DN#TW+mdTwn6R@7|JNU>R1S_R;b1csZJQ_CM#6!2NH`>i|^}i;6AE@YWFHAo3{W{xB5uK&g;4xYo59H zAk+{~E7(!8iy={wl(>~w-m43o&=w~H^D?+cZ!C8pph#x*@$I?`1RVbR8SUA%)29zr zP5obFy#-X%(H1uf2vQ;-Ac`~s(n`wEA|f>)NVhaYDBT?@(j7xccXuix-7z#GF*HcZ zQ16WQ-fw;Hy|r{LUE=@5?!A9|A5xspisGgm5GA_}pkr!So;C#dPp0XZAVB9DK-R9C z%A+|17_g=ogx!BYOJ$cLk2D=1=m}Z=I$dlx9*+R>0eklT$H*!X&~;yI9tq~ZR&>wO z%z=b<*#q>3`wW;g1jPY9&hM42 zcO$)^qTpW#T!e|b2JnFgoC)(6D;sjUYKiGtxVc%n)grc(L6mn~qr(iC=y(Fj#-2)geV z*yB8O<{lKECgNLMJkbs4%`t0p%cy8)TFpfJ@bE8yzJ+*cmyM2{xe~3S^GLsi@5UGV z3G(&-`e956fB31*-=7(OPm4%WUdN~X zix!ZvUm6JBmk)MHuYr!kO4 zg97}SvA1NWyLehp3;$T)p*MqKeLs6nM+AnPRu-*jyx%gsI!?my@5O0z-ke!Zm4Gxu zRKO$Q%ebQ6hk@p9*kB-5nDuMyLoyQXp7J~}A;cbdtmwwwL@<^^!?CY8{S}`Xpex%g zk;^Tt|1)__J?*#kzbY=^o&7>W>C6_%CX;2yW|f=P;XNdP2t zIqCJH-hXfkmy`l%D>ih?8od;VU$Mi!Dp;X)nSREOr)K#&IWBmhem~*LjVKCm5%w5a zfu{4}-=<^hY*}nP8}5U$YJ-Q|sI~J+%Am9X5+JrV+!cS{xB$`oz6E8#28(f_7gJf9 z4lB<8q5k&Yj1)B5=zn!>B>jj>qyFu!I59oroFXF>1KLBb0#qyrq|Nl38L)N|UKb0FvCa|UOrx2j%)c~kOc-L(~ z|GQMZxWngCVNmMDJ|unM6{o*x%*p{-<0}sjk~9aC9?UdUIvGZ!{fQWYl1T#8*F3aD z^;x3)LBxJrhCk?k!-oa1(8EhU7DLdp>%a`^I^=)w<~o!O$N7n&u@bI!Xun{2Bm300 z)td-}CPvKu-=bn=CzCmV5A{Ib=-`bDb10at@+w0Fgisw5U|LElr1CZpP8p;3>cYjs z$=pB@asl52XrC%DgP;%0m3#zdBi&f%b%RgzD1n?${bwp-kmL?ZJ1oqN225J&R}m|V za2B~$`wvN$T6XTbjHL!tT^aV4`WldRb(|UnAnV9Xbi2;~T99M$MK^=~UOx-Urt_%p zy}LU3&&xBy@+Vt53*MS6_qM)dCO}Ia=mB#;_kaObHNeUZg_mr^A-n{supZ)R4YCC? zQegpP9ANo-`|*Q96_OEYfN7yL(qdZbm|)*Mv$3; zwYBlNz{^cA+Z_f4DU=$A7|E?|ZR&aI|DVh4#pnfwB-=P8XaL_=mR!5Ry;WqIqVF;yfUZ!30nk0ly6rP%{S< z;TMEc-uoL10QocjcSqKL`U3&rUzPOv)n#M|@MDsq)Yx%G!SfnbFBl}mos+JTwk zQl7NYYY!-iy)@O{A)>%>eZx zx^4XykY*zloOg*muO0gkz_?5or0RW)rvEg-j;IUXps&Tmzu~Kz-VB>vZV=!vNV-!B zt0X-^L!|{k7exj5sjvN;dGkcGgK4|3v^81?Jr(R}L1#F61VTz;EB>YL=^pmtPwViN zzvCr#V>zEb_mDI8sw98KjPHq(kE!HkX87^!?N52~_!m?p_!LCcj^ymUahB=0_3ll3 zNbR|1&okkRv}&X)s&p9Xv$5lV14TKhSKpx z@H>H=OE!VuBFWjLk{FoSzwb?>W+3i2N6zbM-<0xyIG0fpvuvz(6+N@S_G0b8^=#u{ zBYe@}h=t89rpe3l#jhNDXJ0C1)QH|s@*Wm8Jf>r1VigB;*V^|`kf1*pLkW`bEex8H z82+o+<|Q<&v!jKU%kNb4^}<%Uf6n01`(a=`!`Cv-hnS_v>u1;-%Z{++$p!?HNYZ;_ z*=Q$<>y~}Nz^Wu9Y5216nTb@A{simd)Uc}UqiO#2p&qHf5k83%fkvx-y-onMj02zS z1y~b{4X}NbnZ+nx!z-h%h~O%(ZbrV0`;(px<_3z`nAjwE0Uh%~tVw1lZ$gm(sFYiZ zHWOAEMP}eWmoxH7L*QMkRqFo_8)DOq>dbF!h0>#8;HKMLgUp2I0Yf4yGw^Erc`A1=;@!LM&$pP6BMJ?Zd_CJyV1 zAvuV`$g(G<2rKje)Q@$wUh7Z3=X7YMtxb-FNB>J0ESb*nknB|lCDy4O)J7W_7E^_X zO>$7-w^Uz|ogEuW2hEG{b47o6VfL9L_T^*n;f$hQ9g>4|zbmoVjt}o(le~@wu^PFQ zL-G%>Tjuqj+W*K}NC8N%qg)mO=!dUpD54>LZO5uR)%CLry>&?)oE_b1B0C6k>6ydt z-`Map6=XUdwn4`%3%kP0npA>YfKcyWfLnqVI4Wb4RA5gF##&(d^8T`9p!%(u=7PF(P6dSGfI787$4h3|H}$|+pWn# z@ZcyazrTC6l^(8$fuv;=UA;_vIZ2GWr9nTX8UM8A3|e{g=rP!R?VqVz#?DGj5P~ zgeEgH{rZCO4a;J_kpm0wC4ttQ%%CX{7BPapn?nqdqA!OZHjetBw zUl9EUkp6&c>2Io(IWZ#e#LjFb3cDP#Ed)g!b>O}TmpK( zcz>aOM*jQqeE}o~IT&*??o~F5RG5bN6}EZheZBAI*}>11#^d+2;6ZS5MmWjPJkba) zE0OZ!Cc`f#Od?^c=}+LM*x41N&^JZmbf_<wE^jJ@KLCaY z8gMUR^TS#Z-YU4LAJWSO;9Iiijl$q~ejPr5i9`v&LGdw*Q)3s4Wlm_58<@|~YK-Ky ziqT#w*+6n&Dpc-(U=I==0o^OYt(NKt#qugZc7CmTj4k>EtV7F5rS_^r77M%G`>kTa zZ}cbk%A?CIUXS!5*33SDW&AAABAxeEksL5Zg)JP-6hN;TVfk!ty&PdPB7WL0Ltl-B zO~|`FxBxD;2?O(1IuNq=svXy~am!Rs|b z+!;tj0fh+-2DP7I+1<-*&P^@)riaA<79ufo*=NLuG}a3a+yR$?7Xui8UOlyz5irZ# zl<>-sKCm9>Uc}w)*~u-Yd^$~*WCdX#ZYkDog$}^q*%)xbOD*-OAU`3T4Nl}>(=!1 zW&ajhCNuEVb`D!}ASjDDwH)Y6omyW+NQW!nTq4-*w)@TDm+gESVM@)-!%T z&L*eVd#;8L5jMPK?&mj*y^;RAI-rUgTa>fH{${@V<;$hi2WTR&0Ab{dot-DdOhu~? zTv@QPaNqFYKLdM8NaFPS4LGqAP&WDy=SD%W5>Q_-^f@-2*MSE6#fEut%6TI5cuuX_ zq6++-6%*2bUYJsXgSn4Sa(QYnm{b`F?&iE8$v}@lij_wrTo`R_uE~Jxx2IGwV$VMY zrS;8?We;8_2Rn&{)FCcDo^%b?DE z0*POP!#pk56)6BaN_G%H;68c}tu%`xEu=O!R%gbZeL!D=EvZaT@4rjHR`cW94$yYh z2=~)PbA$qOl58fLBbqLUc+$aqegUJ~;9wY7jM#8D0w8s8tWHYjXNe$P4mB$aM>|i- z{}ezMt`Iz?Ej8!74;wyqXJC!=pdEv5pScZ>_7o)IZvTg zeuFaj_J#HLFC+(HexPiw(~va5hWO6Y|BeRD51D8|{Vyx*_12D$_Qsl-jDa0>&^n9_ zvCT>agDvAdXYRP40_}A)#3oSzevsQ;lJ~&Qvr$D6~io?*I5@sIuCY6-LaU=dM zEZG6t?BIGDQJG4inQ9~K={Fv}7rUCmtG}edF93@fxVb))NR~j#xm&Dwod2#?Uj514 znllwlWO}{Bx;AIu9^*Jv%7skRoO@v7@Fd|nXOyc@8;7CKX>QH-7}VN8OU)puRkH2kRHV=YJksD{pl)(-nrh$w zQ%I<{vYUo!P)F#fR)I0>Vhw4yItXR&=y`@@GDL9%8msPHPQ zD;DrzeWZj2yL#ahA?J3vqRE1UI!_Vj+x{%w)SM?x_PQb^y=_gI{$g8~5u+1K#3KeFoi#Jnpvv56c1mPMbh3 z;;k}Cj(qCwd;@e|#G#1K5smIr>{O!RllIF?wRFeiqcJKFU>#9sz}Q(;@~!dD55tt# zBxLU=z~)?IIdlzJEfPw>2Qt;z>G;Yh^1;nK|Mr|36Z_qLU}*R#a9J1*#uqbk6z0kf zheBKrTjbk~uX6N}+p!H{7f@T36UOtcu<}(2*=yO^8ha+qnif6zL@uQfwjovTKQokL z6*^3g3Aev-x1$^y%T{U~OqZEfT4H>OZ`Sjb&ARH`UySs&3z(Y$-OJE1#VJYxCdr)tW`2>Xs#6XY@+m>wcI$!SHFd zvP6u;&0qZp;ZrwZRMN@-Vsm3P{oUATuUPHtZbjgiNXNr~QX~NOPq-&Y@<#Wf|2y)9 z{%4(~5qT;BtH;3v6ftJyrv269=@bhIilG@z$Cs%7a@E$i814kQaKQ%ylcnq=)k3JLYG$V{(4R|4#lxQve=2^gVigH}|_4$SU`BAyV z+CgXJ$7U~yE7x*z@uJrgN2hyNQlH6%S@K}lr*q{F8(E>fQn)na{DaJSvtoW;#xa#SPUm}R_uUA^ z+2BmRO4C2(yj8qV!G9-@rt;mxo6Dm}ubW+j>c#=n;bnT53ys+DwapJIC)6&&@WoOr zE**9M@%Qf4Gn+1F2Ga9waKO~^_PCV@38}jxB`PiX-Oeo#E6F$eYim*?z5SCWdw31N zDO0mKIY;jMu9ml7$GiybzS-wms46{pGb%EC>ALT`R=_CR#s5j-=2Gph=r2j(zfpg$ zX=um0VxPa(Z@WV2Rue54hpt^~+b^|wtF8>BunbI;DTr()I?Mi8m?}5m5vtoTY7?{` zYaOeR8Gmd}b~Ulzer52P+OdY(N#!`hKO3H_UvO9WII2JZK{9@}bVbja?iE0NZw+bJ zx;QLxWiwT};@?dd6%x+V=c-?M8-D$FCGtu_lP<-9^#aHDX4h2>VG<$o(y5iwD#9};6=DM`=n=p$s16V)c0yW+Gxskapp&e@@E21 z^z#BHnkxLAHl{>0>5-p$&w5*C_*T=lYIBcDz0H1_kgcuZ3@2B-inkV6{mr5&cL5bs zdSdx@il)k8h9j>(I^_A5fDNa#tkM95$5=&o&c&;~FrI+~v1XSflT?GrS7j|IoteMu zRNBw?A6hwS^KFj0el%@Ajrr2-nVD>HlH$Bt)=c-&%}%iWDq()IK9w4>mv$SgEvLoe(MMC7^w-ZCkujcubd*H1i$-aw+z2y!)4AFFt z3v5$4wvrGTX=hCn1^PnE?ybO}7HSvruH`C3`taKJ-8!88jdR#5OP2yGI_g+nSv!u& z?L1RiNeGAaFWp>nt+0r(?b|Ot)^n_)2y{)9n@ViT8}Xtk(0I~>%N;v z(SN`H^h_vqWmvepPETw9-R8({T@aR=a5P<$eNIcufpDSIX`{oLcjS7}x12vrrhmRR zs%E>0p-tP72jilLfsYiccLW`^h~lw(Z3xPFV^(3dX8WE%Ngh(s+cK?0w_5bw;-HQJ z>#oRI3_*O+s!pVMry9$b81;Tb0q3!U2=V&Cxp||u^5gTVBUy>(T)*ds#b=n#BB>a+ zHF${a6$*w+e0+6lYoH%bX5?4>4J(~0=Us=5W~$uP{vO}U?R#Ae6L}q4Rxg8n&{PLw znUyKoCDLC@dDOJp1Z+fE6p#4LOUR;Ev%CH1GnXIm!=azf4(>;=9H*IT)2Z0VJ`Nr+ z`!=45*I&Aa{)YvSN6fj7hyg<}Ev^DJ4c~+Sr>e=cksAx|jv={6r0ttl+y}zg$H=W2 zzV)8(jf7_7sFC(&`FJ*F0iF`J-R1@Qd~ylq-(NOk^ZPGncwSF#sjueg+Q%eEiZLtP z=X|D{2-n}Wv6{}ws@ZmGwQR6boO+;LYg229A=^pg{AeorTlQ^PA~7r09E)@}mX_Vr z+vZe)c8|L=-lu(`g}nNluJxAI-TX7BuWj(W@7a|(6gefTd3({@;>YEp=Y?4BmdHSm z>rkn4*}PzZtEvFvpf|{qHttNQ-Tvat1K;X{Euj@CQ}x0q}XOJ|5w~2 zZRloQTqqSJ%Ur4^kx*pbaH7DmoA+&f_z`aYO<`pl|r zGkm}O(@rk|uNC$fx6#d09+)p~V9iq8BXro}jz{wJ*C?=+`8Xw9+kYE-Hs#;@VuM#g zHhyF;7x*CpptHFJ_GhPuTM9?nO;*1wQE8ev4wGBUi2a8S&7#lNe8{jyo<@Qi{aqp` zn8?OL-BQ_J#*k%kpOe^iYgAGy5?ZQ6&ptbHVX4wbU}A|Vgq4+W*xPGa-1T)s9XX6L zC#JT;C^L(x;H2I$8;Y(1H%?c z{-7+wVc{LA0@L+e1pM&VlZQgQ3*B@!go@|t_Suz{Q>oLx zQaH`lIu<*%?e!HnSKqC3Ew=t(HTW?z!g-*TSW&?E!S4Iq+J@R;BQn+TXw4C;;ZTva z^;53zwf&p<3{9bVgRv`N=}9VLrLiU4uc$ITT)utQLr}TtTBs=y-4Z)>vTTlf*%E#f zf3jNYT%4ody|+9P=i@!U5NO&Ltd5V!9I`9zo6 zgJ-k0#TH0GcOU!BvwEMKwL1{2%pz9=Y!Ej3YI|FxH>#O6sM)XU(1uyH=G@UdW&c}C zhzs$!1%k@CDBCO?(0f2ZsIKj;29K{2zo|J6j^m=Tq33$Ajqq|# z4{<)3wi?SY!$zE(?Ow1X82Ag3DX(|gc{t+j*K#_x{f2nt{n&(Xq+D)M{P*%mDqbCP z#bo2ALCZusB8ydOD)N~IAGPOhCcgT&sFk^s7(!`L_8r%4(P#Z8#Nzjz?*c3SwJ~&2 zxq05wXamJD7kujtS-!-R6P{6XT$JMp-B{Hd4zk73O3M>QHtnL9--#~*gbBly4Wv7@ zX1iW@i#%G4mZinR8S56~g%VJF-a(lXzaV-fp{P@1n|{zTAwTG(B|ge6XuC+J+Irf4 zqcpMC_egR2)pjBA4JGo36vff%{z^)`yaKfdP8Az zfkGO)z@vej4@b-P742v~hDjE52HfVqU+yuCJ+@;^zU{H4R_DcB=s21SoPu#hzk#i- zTpFJ(!}%XJu2q|*wPV#s0_N9L5!y{Y`7_o^%`09PCz@aVS!tNMm7*PC21a#f<16oo zAXodoH+)kXuGz~bvZrWKm|9*i|c7Axi)HC)>Ip_ z*1F95kOP#)c9)0B z^+Yc(?=&K%3f)eH56>+!dIkz(P0>+U?Oy$o*>!$)rUgf|Emf_;+h8?fVv!R%-dFO8 zrSzV@mj-#_{06je{-3~|>Kjzu8W{1E0@+SNr6nQm65qvs)=xEaefdsDblaDvJkU2*9Ty;%`4P1f<-!y<9a(hWc z3q-*1gU(WkxeRaX0uHs^rrMnSs5aFi8>^X%6@%o{5i(xqv6c1OU=GTjhZgD4Nbh5d zSG!~5`|H=sWh(uP2JEb9aerN^Pcvf|wP3DYlZmNi+QrqH{U0%z2EA%lM!u>OcFqlH zOR>VLh`4R(k#OJix8nIRn?D7tcBJ!Q(GjV3oAd~$hM(ABb*=S-PLov-A8pJc9*LVV zn~OZaQ`#$4IXZ*Z4&Oh?4C(=K3@i4xxe^0;J%xG9@`K~gDAde8t*JSOHgIa&YGE!2+ zFrHIgRebg;dceyjZy-f{u4U>fLN|BYHtQy$8az*7;dIa%sAzN$E0*#4Mi)2eRzSI} zn>rM-_IT!KS+>e&(&OQjeNy93$MT`*7qZmc`EZ{iYNv|kI@!oBVR$#FrR8%YUSVM3 z;^2`#|2SLzF0U89=e$wU&0&SX?P=Rd(RAb?mIA-__Y-Fbo3-S4UG zn!89{N-z%}5-5*+Afm0KPl=Mq-Cf4_lx6Z(YQ`qo^(_{L)+>9Onm?J&k#5G(cQM`& znl6oyknp-GohTq7z_-GsziRTfL_TTFh)C17*f|^X>jqEuQKy%?thgT+1~=Bpea!#e zB(o$YeUYLj=zS1~zo#W#e^O*9R>PI%(lTay4Sb0p#q_6rHeD9sRTS*KKMEZg>wLp3 zpi`BnnQB92g67ajXjF8dG1GCex;z5nTDJ@hV<7jES(-%0ubeZL- z;5+M6WY##Bckcc#&w=e(fsHi2NCDprLC;SUWsYzDwrB-Hn*`EaNXD}C<92khwh-px z;&M)@qBN&Q3XCqjx8d(jrW%1o-o#br;x06=j%;mAT z#Eb7MXxM(*c$afDe)o;S{Xss=PIY@s;yPrUehR{`cL@ng}}ul&27B*h{pXFQ7&;_yJTC z7}ZUKO_M+`ZkacG`V9iSS9rkZ`1 zgVe+hO7lr)J*1)AyHw$BKOi`AEnbPdZncetDG5C$TI1CE^P+(xqAd@7&S?-Y51)Ek zx#qy^T3AGv2p{P^A(l8%QI*i#pKeksEAowRJX_7DyWg58aJ;mgIB9M3LZ9k*Yo3pM zqAHTO&X--i>;ruEt(ufhkK}g&6A*|4l`FIIe2UgnXy`d4r#bakSrI)3OlYC_Tg@o!xHqML%9<$aHIx+ITw{)Q6&nzo#9{DMuF2&$24*R8Sbg~iAWq_5hPNQrSHOz_dzSGGuy?Y3OBdJv4&~FJ2oT6T3|@oRDRiU2fVJ* zznH+g#_>u|*|raUZ}$635oe+vsz;=+Mu)Vuvsq-)_!{hml22B%Nih@BVq5-R60kI= zXS?WEbEQYT#fID2EzLV!RN*@BoYmp}K5iUEvW`aEjIe3g!wrWr$B`M{oF_%WP2|&^ zfv2@Chrf?l^s-k9&--&c1__l|^9T2Bp3*%{*DQwqzzuJ@%5fna-$m5uP9K%GSq;ID zCMMqJ=UeJBZ{3fgni?-%EZOJsPVZ0m`Y3GJDNV)LCv0F?rP8!Og345_zaGDJmSM%O7&11H>-_KB_1#?)}Y{?<4aJl1Ja4uEi6>-UAQI9aNsPm6vaeH zm%I(4B7UJPl3mN=jgpZm{nO0x5V1#$0d(6MrLlK4Lw)W~axL%&nb1*clrSy%UbH%G zjmmi}%&65cjmRMR`9r@n_%`TGvK;TjE-JnY3j_JAHbKCegX^`&B|o`jj!D#pz2T*a zJ-(*fzn%_CcCtifFl6t?fOKm}{^`ZO@6OE`N{V!ubKCE1(T8V#v!{)A*CXc}r)QGL z;y%0Q$mY4(utM@-^o`h8!FXt)4R`TEx)X=s(b<(QBL0c`i~A*7^td#~`@@^3>k42SA(5KukMkOJu?2BS;zADjg#(gt~mp+xB;d z&`OKzc**;8_jm^}X1nWixh!in-H`RRK##$zMQq1H;<}d7Fg9H%`1-;Lb!p{bH|WR& z2321;CziEJ!xc-7iQ^ZJeuehzq%fY+M=#fDR%L>SAC7)lk4}FJ?#>HMWv&cNs(98rfC8i-wHOnj? zNIx|aAxA0Hd-V<4qJD+yA6|7ut)b#O~%1H)4LRv#WQeSe~n$a*1% z5NdxtRjQrNp<*uaD$tM$qZ+|MA;r4OnKztlSBuEeKDK>xtUWr4LP5Xk&M`w`h#*~C zrAPe_opjn=g7sY0H|j64aW0OxPyHh8r1Ka0O8$)do_)6SKKmANwpOhy9us-CbC2k^ z)^NPoM$VkLIPV9iwk2ILk=kbq*=Si>=P7VVX@rM7;!rP457VxXsIEgix;ap^@=#C3 z$|!van=I-*KRgX_ULLPynhieTex~2a&zW@8sLTQYk#!%b9v7p_61YTntly7t1ZKu+ zvUiD+9;5^JWnSRXsBe%c9~U|Mr>W@I?PYM6n!rexcI_rM@(1++(9?2l&H7N!ow;%C za-jl`=$WeiE&AFI3$Gfl`&`!7kcmUFyc93sy1SiCK#|1KW+TS9-%Q>2L<#Kv7_X3f zA{xI_(WIn}7(Lfl+4mZ+-AJQCuFids4OE${w;|gNs&Smd<%OY24(fbbaZh6+7Np@8 zWDkLRVgUDb5)m$yZ4KM*&3Tw*Yg-Cl5ttfO)a1HI67{JPeeUtDY)JdmWPLcrzRq?0 zf3dJU$h2rHX7KH9O*YJ!y^*7Jrzfw0xqB;&mhRqrcp9O!LPx`zsOKZsmI%Tv-LdN; zmC*desg@`C86P~#R(Aaxj8{UFNm9j<1YG*G8h83sko+)9mG53ADxZFR#H=)Bn1ROz3tiz?hk?FU`D=f-sbBcy2F62t6cFtJrR)(KYSfr zJ^$|38?NodWbIjd-;RjmUbskZhdp`Isc@=dID|7B2-Wx~SKs4l)z$olt0p_u#icp` z3W@;Hf|@~f7az^OPz%ZE7yCIbnA`p+4U2q+b6ukA8NAvdl#=llgpsMn}Dcu zd4Kt{?TUEPxVbLAPQC4|a?6a0Gfc2#KU~-D(POdwkMkZZ_gV8JK*L*{=$TB%%|C0< zptcRIGC!Y9`dqFbGv!M{aH#VwySf<8rQH5tYtMuTDBs|APd@AKXk8Zd8Wj!XF7FoCQs4WD%E zx{~R!boM*(MW0`U)s#JOu|0mnl#HvEG;QuGGL50I*+6~!Jslp|OV%?U>&xr8ebrJ^ z%P@0}4s&9N7;1+}wb@|zqXL+%FHnNUv!ul zB=7b7GQz2OFv%q8(Ri&tsUKbu;IxoCRn-xWuSNR2*J&e|6-I*t2LJIJZk$^nRQh5G zohMAQM@Af)hNj$clb%q+NcR3A>LLsR3H3dr9Af-If z^8oy=0Z*5djj8+U!=>>s4q+ItM#0yYg}jG-TW4hH*%U3K6_c}P;*lQk2;0=Y&$(eT z%_@U#dy`n5Pc$@{Va3gsl)GLF6fsw`3Zyh`KnKs%D~v~o2{{wus2oz|$3J04zAcZh z(3g^8-F6SV@ASD_mohQXCu>ce5;@D+nS+q6A`pRDJIT$XDfF= zZAwW}r86i}k(x=3W&Tm6GknsUqn#v+VhdydPM@aV?&B`WxUpZ?6sylmGpKR`amV+r zS>LZN(K^)*g(bRhE|5pzh0m9a{#l<}i)h{xxJg16Y1EJjVvE!uos#3<u`<)D*LE;}!atGy&sZStt7ZvZD+ zY7bC=r@cbUZr`D~(azvo>>sAR*g~@WS7;ibLkGGU3ibLV*kqZyd^_psb%W-j)_F*Y zVXHv`SFa#Er(|?_{Za0)zW3vcc$@R}V*8xw!Y&6K-4YWz_(1`vMBG}Zt6tJ(kx?GF za#ZIe1NLFGNlzQ;YCAV*6RfckKy~=nF)M@ekPxEnU#V&)-y1-|^Ai5^xOd#_c;wl7 zt>%)443y1BeoTthb3<0dd|{gF_{hXyrRH9Kik~`TXPa0C#YEpeWOlT~9?`ZQ6|bv- zPL9NGzrWy<_-`HbV+1sE&}YI4qna2@zka*FQDifGH?B(ZN0_bR6TUU3puQoO2Hlsttu=v&M)9ZskYW19t z`K&BUtY>$KuILtvm(E{;+OxwC@*Rf2G+o5i2_j?cGKQ+^>S}PEn#j-VY!|Vf&QP|DX0EPh$@hwxm6C(IoB9z!$z$G< z%-Gh8Xz-lJ2D{FPTx8g!ZsZ_P<)KOG`VG9+M%#s~v@s=^Vgqp{CMDYi!vVRERyRx{ z!n8d0%MhPK3U**sb-@$2j<#sy`Cb<2u2_YaB2jx6FXmFFLw%NZ=kF72!ata610c`= z+9lb_ncMKb7(^UqFhvfWw6@sxH|LSX`n1n=S_q5$oY^lnmeVhfumm~U87{phHpE+r zE4O<7=R6Zx^fO$nWFKCyQwKnMH$H%p+$tJ&TpvL}Kc<;bqH9w})GsW4bL=>C(P_Y@ zxB7(5N9Wm*1A8F!G^)otGCVOX)gTRax4x!%GU%g2h{rW*o5#vo4mR`0r=YW_?fR;I zDYYF`R+)u?UWCoP>_!;+a4mTs!7YV~(-5<5X3b7E3%&EhDVoD~#nx&Q4TkHPVkY78 z$ePOq8{zAqA3owW_WJeabJlDQPSGFDwuVC~;o~FNRkN-z9`Deg@22NC@EwM{a5~$l z&buOgUYy&+~xyW~= zZ|02O?>7&)1S;u*;ZFh!SSqpx-=26Xm746(!0{kms>y$Wl_bxglx6bsUx#)}IsLI}4xUGV2;+9;>t^JMsW==B@0|K2T#GfOU3@ za_08M34OaqGi$q5=&sPItHZ21yPaxiT36D1xCJvC__5i@ZR~7Oh4qY!yOfQ}bhPw4 z@Rp!uw&G}{$(FiECw6emg3lnW6V-oQ;SK3qFiQk85%;E>={i;+qT++Uaoh(c+!U@^ zr&82hdQWl}L-`e~p#n3CgaVe*j3PE2>Kw*pL4i5@$1J22091GZP=#^odak_v^=@C+praWvq_xd7%c+@RBHsAN zbnn7y}L2%i7?v}*zszL<0^d%%+gOkL z!kJb^`8iwk+H;4y0OCE}$Oc?j(2vkOaB92q-}ExghmmrSqD`D=mvY%?J-*G7Nr9fspR|4_nkebEsv1`D4&{xs^wXnBZEsjxV;gHr~G%|2cD9zkY$)F(ch#@9eS)nX7o-!?jkYHsq%K zKP*7YMO#hN^NC!Ja!EbTh$af-!K$o1`P%KN7+DLM+pCG75I(!9*d3S8s<37qpQ|x5 zGMvseE0_r4gtaE6j3IB+KLN~nc?v?U+_Lw@VGTfKBr6E(D&{ZM04L*C2s6 zjZ<=#YeQN{OoRqE8)HKH%pRo8F+^DdAuGJU=J5gFjcwz&o33491rom&8!66D6GH3h zkrl_OK2@SA(tM|_wHRM-BdPf>QQ_M)y`!_Y0MP?Vv{pLpMLRPTsNFyA3R7}j)y2u} zz7;FpRx^Ju4+KA)KK(iOXiBb_1(~`sRQkbxz{{ubyusqG}9 z0SuLP1BKQB7b8fzr+ew!GODfH+0xBo0_{0@$+~BA`@A>RuD?fjOG6@3ESnZKR95`G zn|dA*Y>1wdWvR1* zcyVm}ZKty4BB=(?snoglXhxYA?N{*6nXw)Q4%gc29#nl)AOJBRN;u&Xs^cFHEVCF~ zAAVV{s^~N=ZA3M_HQ$=C5o$)II)0Ec^;XtEDqpMhqye32OF6m9_C-LAsNAZ4&RgX7 z`Wa+Nsz$a+uTF^v4lb2$Z!9IY)+&6akXZ22uYWsX2ZgxOP*tHrllA>s_npJr$e@Kh z@vE=+7+WJyTTuPwy!5uXSxgavHEq9#3N+^Gnqu70Hy1GC$>*OnvhNKa18l=QYpr#CGh9uac<@Lt(CnIm1|(GW2XN{!TMwiOo1 zFD5}=GNv+B5c(<vrfGL86n8oJj%mlp`j|;o-T@Q zjEo>^UgABgOn2QruZckT`drT-%~Z-qhTCQ<;veRX;AZv ziWS-ZUP6Lbg|V+7RVEIT?Qq9@0Ubj4J*1DS0WM_@ej|9K7Pyp~5JZXFLeRUCH@P}Q zY)*ekS)Lo?h}-Rnsd$_aI(Ljl_hqWp2y5wXSUv9%FUj?jqjLTyDywK zaVQ28Y#~k@>_j||d7ZXP6u6&r?5G?CpwLBK)Bg4F<&MbP_6a4k@0w;e5)72K)bT@Q zkJCNuuYUCtIBX(RNs~YbJGQ-5XFhu+drNAn;yW#BxH9MsZSr%Sn!s=Y{+Fli-Zy;y z_h6Q1B~u7jf*VJWHS~Dqna}JzSuqK~h3r(;)jwB;+c+sFU~#X7JCT+cB|srAH*K{# zX$?0!8~FfnYdJ}>l*$M7@+s^)rMCA^+C6Vpi%4z)@}(O>l>6R3i59A%W6^d!PzQ&njKR$3|jqW0l&H*5)xQ#K+O- zx&~!bqeHiWVCB9yG7-Gvj2m&5&hJDb|JPSoMnT=9=^=Nq>%(Z--vc*zYlcGXLl4wynhy zk<~w;-Lg)ql}iXgCGQfE+oOIOwTX&Ho1GGmE|IG*g$SvAvri zj-)7bHV!-}OcW}x#(J)S;PC!78#A!z-2*ZHHlJ_yx?R^HIO2G^EpR^?75-xG0gDL# z+Z_J~95y+K@spDllY+}X7xrI=lUx$UY;W$JOdXZ%65H>LDcm*!wns9P?2Ii8i+#L* z9sK#Kzi>P=de@XHqAKKeA}uu^XG?sRcSKXy+F~|XY4vjmyb^|&2D^!q+MCUf_Y)H4n3#H(~%*lPpN&6AJiFs+iM3X#>Aex{o_DQPImYwOfUxI0%Ulqv zr|&#RZGSN2T>Sd~A?&^5ss7*pap!c@q2UN=I|!MH%H~*Q9x{`ejAYM@a7bAd8QGMK zWEEwLin3=JktBO>*}v;~dc8jH_wV-od~e@By4~a)k8wS&eLb%GqhG7=TZ;F1YdwS7 zp>5MpyY82UO)fe0MyyHST`u_+iz=m?eUL22N^_(zM2+Ba2f&#<9{D<3X(iYSVejCa{5<(IR7_JHSCs^H3YQAK%u&n^DpKXk+hO*L-L5pJDe zEU9;F?bcR4tge}lt@2v5m^UPucR93u*vH{nAKTjLHP>HM>q679VXYuvEGRxm;q9e1 z(}r(`oX9ZT$5+l0Y?KcOoxY%Ht++?D{Lai%(9#W% z-r?m)a!Q31vHT&4<+g_mJzI$j6JrJ5jcy`aL7zr)3R?yIo*`G1$DiM1Davc?Nn8Cn zes6{;cCeK(#hdxc5k8(;qJ(^dE|>l2G22_+&wemXKUG`3%d)u|oapK4C^GY~bEN0D zOy5}!F(^T^O&v)feJkSa8~&73p8m(GHqD^c$)kn$sn^uX>fMqr;xxY7>oHM$x=Jl) zy)SDRzBDb(KmEcm(ao;UN>uZd&dcyOotDg}q;#zZtv0)6QVl)M~^W*>`#4$deM>)8MbG$-xrz{IyJ6+43DKKSd^ z)JNyv54*6HNTEN_<1we3$_j6%0Lv~~-KZ?!$={pQKb>IHX0?+Tx45L{_4~T%8%^c3 z94~bvp)R6A8Jk|k@5ITm#)fG&0lQ%U_UM4#y3ezQeX4LR`-{2m+Y3nzL#LoJvyjX* z{QKv%cRZ$IwbuZ8e;(i7ay9NKd=gh;nK3Kf#;MuSzYiMGU}n4K$b+K1v4!eBI}@2D z6d_61dCY&@TihuHcR|Y9$6%@BUbap?7hJhY$h-QzUGW}K`UITmlXs2aw}^JF7<<;H zv-5wMFM_5*`|hw_RvbSFD2iLSQNEfL5m1HavFvC`1`yEBj|Dk{h=AJ77`}0WAo84S zfpli*?WprVOoomfQ;pyD zBRPtVD`$G)**(kQEe5Iqv!cE3mR(|zcUb>k!lgj_&_cuK4<+^!E{n4bOUWKBxl>yu zm%9Q083vkHFTusFJ!luMdBT%e7Bq@}u#%G`Z&{BmfmK9?hJ4V%DWroluXWogw*E*$Qs!>+2kG09AnUrARcA z1W%T`dYjxqtpy$8tte>7RR-7VZ!ZHrTo1k(w=!x6t#2`;6;yJvma$yHZ_$K%@F#xD zP`y+N~~E3j2*& zErOsAwuS}LvPak+OGBIb{+?pfc8ACZiR30S=uj6Scc}ZB>~EV9=vlE`LBRpilUT|< zf?)uIQ=1vS@ilM?eIq6Bq9ztUS;LsQNq3Q2G=XC&-Ndx{;aRU%8I1~E3$aBJxsZ)-U zNK!D(dSPA;-23oHa`NsMEo(u5sy7CSnZQ_Yl~H7+97^#EU99ErfyWuZ{{8j6bWS5^ z3@E<$C!P}9>JMt093uCWgH%G3Dc%q{Z?;BPge9Ah`E!Ks$WowKP39s}e`WB<5XFxH z@dZ#HDfzguozi#`x`GA}3wVy9z@w~j9{k_2D)?Z=iHMOhnNZ&8a{>&CT$U!d9FN0hz|u^*lC)U?t^|oTz9v?Q@`To5K1{2-@$H=&kl|(p>ac5H6!xbY}-3ZIAFpywsN_rMa z6H<2=fxlov6s|O_9vaPuoMgvGeqKbc5MdFH{*J=F^wEFiZ|x``Ka1whT8U zu^2TF|G{sG46N!%u@D_B41fo~omq|V$U4>(#-q=s{{oCu0lyz&`0b7yIr5!;Jw}WB zq(I7^S1q4vZ%!iC%|V)M-5jzg=4APJ4|K|aKJX_Jb@7919OLx!`$!`QJOv+OwjyNs z4+1z5@JFD$xguz51^A{Pk67qs{3*0Z@vFwewA&P6h=wXrB@}=Ay2f+LCVQf2lAj6V z&rIMKKk8}`jXiS)*n#;HQ7IbMuBGU73f6|8k3cH=#g|gZIH!O@NTxwn3{as-TUW2> z_>n%LZM4T6P$d#T79Mz6%7W{V90h_VmB+RWPzUnAuz&kii3^-qxF)4scYdRXck7ywBcC6+8w9Gj}>vOfVrbgN=ew#R@1ph zxQ<*5yB{6;8Ki+mJ%LaUpNGUKf;DVes*bnGIM;?(Orz8<|PKKi~0JBxk zcHiVUA{G&_y9ms%3!($|xqiqI%f4J4ie3$;^HVve6E4C!NQ3fmTfA~gj}mPEkfqa3>&5wXi@L*Jo zI7lT9M#?NH5f=UitI*(WSV4xw97g8(>u&)Zb&~1Tp`*x&ZNtxG%S%$m0FV9M04@?t zVr;B17St1Rc+J%Yr%_TMWQ#YoCEl|86Hjv;(UgZl@AAN=ztoyrVTF;OfOuZWiOC}q zW9rRu^TiZ?DBtdwZ_I;2sDZ*QovU5<5ZfvOwzcUv>I>*E5hH&SfX~o;J=HpiFae-; zx*&KV`~C}H<-fEK+>Q71gS+wcD9lO5L5aqN!~Af*INE_w_X9BV4LDM&PDfLCA5t`3 z=7}}gcAepdGuM-DxkBMr0l#0cL4Lafp2KjMXaO~*17EG(&Gug&NnhijVamDPiE4S7AUNn~nb|f(Zoju;% zZaRj7$Fp4)FDJ#afvHdj7_3bH0QXaV)z}`X#7^Q_Jr_ADl^MOWXrm+0zrXQ5J?XFG zMTZVZB;$qs0dFu;ZU0oXIjzg0QhF>0;=kA-w5mAYtptQXHknw$h+?ktEO1uV{W(vS8#>-(CG?6t}ri?qsp>BK=AumyzM#CFrM!K}?3#jo2fw zARHp>WFQD%sy6Rx;}x@uSd2W7N}a1&l-8k&Lf^8i>T(kf)r#mFgZizDH(&#=IIwT10xkG061>_ z;Jws)U!?l_(8%*dX@$k>{SyziGS=IM5zqVVtQee0A#Yr#sa7T#+)g6@( zJvbwKM7v}}@c>{Rnh(x;kcOg>CZ6ObeT;==Bl9s2ZF?I9;s(w8!%?5p3Kn4*^hK35 z{Ds2#7xRcpbm0J*<%M~l^>tb4a{MSF7U$nYew789BuYs&4*_r!M%IbkTqiuB93P6h z;a~?Am=hwuC=DVXxX@J8pr&z;JK-4J;7H%-JDSB~%6`V!=Jmgrxo=grCk7FI_|)Ey zN-h7uwpS4N98}cauQFTm%e6mwTNIY#{d#|-LXul03*`+MwV+@m#T$l7;{*TmJC%W&(~grX3KFJl5I zMDPSS_AQiN1X_j50F^mwhBdGyvLu?{{a5!{fK3kGKQH;ToAgS;`&R&|9F7(td76?zhJs(@N&2I=GDk~q;WxJyQ|E?)9NL}}QW1f_M zyU`Tq@B$_g(z!2!aKHkbWysHmf;fj@k`G1Z%^+Jz7b! zE#sU}d02}VaNWOHY(YqHDSrKAD9IYrlhJZ9-Gl|sFAePF8I;H=;5>-dIMWji4xqlz z>FE4>7vO)aZ<==X_m4#T)V`i?IWA@NVs8|#&(+tGHN)096I?6}vH2Z=7@ zMRDKjgAun*81YJI-R!Q1lib`q9cNA#K1zCggi)J)S_^kPAMq2hatyEUICjiT#((9K z=HN&Mm#Z!o6zIdN&a8+G=7JvYXlSgZRfvWz^uO+(!#%*FRNX^)+^Itff+!M+4lVLC z5-TEdm2x(i0*x49@dJ5Bo|Q*tH+~vM*hY7M&mN-a)iUAIG6R)EsGo>00SIJZTrny@ z&}!-yr%HB0b-gviO~8l&%cTE};Y`gG>=LkoA_cOVD`6=Bu7i|M!z0nh-M+8ahb?~C znI(!c{^-d26AYP9sw)XuZ{8(EG7JqMYg$65c)#mb8S2cJO`_L9y=ptu!>6r4`M%Iz1P=vba2M_buw40XkIkFxMd)?%s~!RWk7$>P4*+`i3*iK%C8Ba?ILim{Z(Xb zy4!e@0P4%H0lC9bLXet z>&PszCCH1+-oObEVFz0x7Uh3bl)Dm%6q(sjYX-?6qF>^iW0`i6o!%Ej)Dm%RG*2G{ z1+Abop9YO@(vwd#7qm|I(+9!KgNJ|zXJ5+t6vC!4(5)xYE-Jx$LSa0#Onj$MWFk{}{k#* z{Y7(AM6aPyA2FPSVXz3{8mNI7jfVPKIHgYHlz3aVar2-kIMx_>xVe=d96EJ-N60K_ zd-sIPZhstkCXg=V?ANFZB_VtPaM#RdTJf$H3ix~y-LH%T45U6-&1VdFAm?w}K@$Ev z^;bIDZM@>{*XPX6K6^uH!#+WTB7y8cbH9Z@+$jW+DBz)uWch#4{v=)9x}OwNo`?>$ z9jL`)!1!b4b2N10DP-a1gcDa@Ns7m=A+|9Bg72st{21%tO+R?zvE8@1X<5S+H z1*OW$3;x7`Jf=c&&IY0xuy9B;14IN-pv>&bxd6|&Cw6>nkf$Y7r=wkz9m!58iqRyy zj3h=zl#Q#0JD5E)Ad!Jm|)x+V-Yd zn|RUZYe%eu1CRp>cI~Y`wPB*^x8K5`S7N~8{FNF9P@s`Fm-~;R)<7iTm|Y$~?g9}) zgnh`jd&mJO1!@;rim|HR)$f?Mk-^`ZYswV6cB*g=N+-k^!G~tV;?4k|KVbF(Tx;9R z^<_)JCC9^l&dGp^Hy;XHuhNQe#fKy5N-MZy!$u{Vj*3=hL2L*o}USC3lnz?E(eyHX^Suc z2m=7Z)sv0tKbVS#Z7+YVKjCWnluRtB#f;MhSx zC1At{oWuvs29b^XYnE}4Gni9F@rpNI#w)4sveAES6nG8(#P?@e zL&-_zTOGgSA7`I-AiYlto$f0>cPeHqp1gRV5|aIatKtXm{1B zAH|W$^$~y>tEo^jn7<&*XW>;Ei@!*e{~!$)9K~oL(d@hAiHhU@q%_EG*%V$Z`KCXN?FDGBse1&nJ?6Omyz} z*44+~Eehh{=pQb{tXiyka+UNR{u%V9fWu9cuyBL zHZ(*$mko^HJ$Ni)DTb%A@Q?)n2g1ChpCEGUxv4pjj`T;8J1?VY`pbIb`5f3RKY-6b zqF_d!rUk-g6Iv)Nyo-rZ9*C1Tfs}X-Y3Bu#0mMBr!$CLzXO%oAjt36z!z-a|V1Zr4 zC9${Fi=(YumwxZ_D0jkL7$`N4c5$TMpsU`Xb;u;yO+2rfv9q0dE>-v@`n8yKZ4P|n z5wKN9P-8mnd>NeRoL2aCk4&8=$WOp$*ASM|wnsp#caWUn&+mKP&0XxN9TxONLCUxx zq&SUWPQsGcIA(@wF2&pB9$A8P{aYj+;y%ZCrMeI8Lba%~eMp$_vqOG$rf)1aSWq9a zI{%%0!Q;-tPW-rvQy8wL1IuKmqQ?CpJ@1Zw7Tr=!D z!r^JXJ+nJv;~MdS%YhsAP)KO8opkqgaefzj0zkX*0mpJY3$H_zcNMJlL-msn2sqH+ z9@&!!PyP64*IQt&k)QldqOiED+J@`_6|f8Hh?RP`m~3>%N59jmF4&eR zZfL{~T7wgwRS0KM-~;PuMf)sN^ZK>YvtR}hcvq%Zb;ofEk-)qcK4eT3eC1YsZ2)M5 z$X?FCUjN>lC6PjreFFqJo0Q+yM^(?{710pjFQg19Ou0>EvJD7))!%RTY&>!+dgXkl zGWHabvjM$?a0+=g?&%uOJv7eQe+%s;t3Qk^Y>|Jmi3_9`=?~j7zDbkMpYK4Y z-|12EQU$_-4kRqdtuO$a8$n-J8lCYVal9t(3|A&xmET49IzJ-^>la1!n&EoGSwSae zh}*3*`no1oT@i}~^v5|MUcx`PxrBeyTWRVf2TkhprGT}KxeO?aCQ8jRwDUROJ$n?n zsK8_bNV*d#7ouno<*)u`Rc(c|XW7=!44?rA62t{T@52NDs}1nC*R-C3?({=+Y+XUQ zZTrRtT|?S8Mf`6@Gpk1O@7YfD>zp42vfF`(?9?Aox63+!L?_B5JbNn`jsGNG{P4(R zM(=Xt51ta$dT`*_P$eb*&A%JDi{!j4Da}RT`_~Yr<*kg|-%W_z33G^y)E)%aeb5#4 zagR|xe(4Gj)5USo_mLjB6aUFAkPEVvN*3nk!4DFFqw?LFj*>U=f0ob0-kNgv)6WRS zjQ0<#UoGJU7gYm1KaR%FX|&iV(oTPIQ;}&|Gv%u-9t5@WKb`<{DCk2_oe>lh4@ltLe2jt@VC z;^vpNeC9HqF&A%R{Lc#y2*3X?r3RYjhDm@3Ou!0J4m~k>0=984N@mYLSy(63K(+?4ff6A(EyR+P@G&Iy;8ll{Ut7xG z-5e#;X$Of~@;wWVgL9b&GC$ZQ`8ay4_*9b74tuOEm0?l?(S^Con4$Bw@qxar|L zavpMg#Z)~%^T!AOmN{_qP$TA0iwUb|4VOgxzbxab_iYSky+2P>2=sNQYsOgr4I^;n zFhx(U@L@z2p+L$Zr2FMfDXopu!{ifCIwDY}R?gja02w4OVV^!Cg+gL&cxZ_K;(tXQ z_;4p66emaXI3?V@j!H;C?9)G6JADWd!@u>DFK2Xb*j~}qm0O+@3H57+ zenTdta5u=8`4p3(!gA)6of}f8aZ;dXieY3KZ{$O}rq=Wn0&16_1H>{9+ zEdONmCzhjg@`#(OGtQn|{ot`9B&!CRgj5Z`*c%pYJ2xZ1oSD%b^=KEJ2+{32HIJu7 zdBvY5H)m*hmU^i35V*)Rz-g3!>QnkPSyda(A(f;Oc>Dry!iXQNSptbLq66o>o9pvk zu%_Jz&lwsW2_y``Plt=0`{G<&o%H2n|GpRy4yLnK*ETyrGfwG?^4ybmzsWD_;4L6+ zDW{tmTOaUDb6UqrQec309D6an6c2SQ@JKWX_s}BuvRveX>7IN7ANU2tZ~_g@&B$JQ zkReBFu3|An5+7lQOJrsFpZh8#(jD@2+&>D4pNN%Ll3U$e7up>!pdG2kFS)H9G~gZK zdiuoYlh+qlJ9Xq4ABB;d2QTXBx6$5qN>k}aJi;Kw>dE(+`jUG0NSZ1xC&AhtqV9o9 zmxUSc?I+02F(g@l|3nNZ?7~R;l-SQD?*xLIL-UBirH)8#1ISYku3?DU{O9blIJ(nD z4)S+7oFJ5};cc4tpvs3+KzMAq#7P??&KsVT#9QW?^w@iI)x(olWycux(OUchgZ{A#wH^fOsa@mL8GNrls*M1{=*;u2u$ z5Xzhs-P~Dcmt^Jh{4q#fRs*^FKt|tVqPixX>A5ml561BB+X9nw_w$G!@}#oKaks-| z(zDLpUNJ^t?y;=}U#-Od(Dp} z;P;>kgdOGZ6ew&!5JjqHkbkWPv-(+d5b-{X;UAB>NiFx$CM*j1s|9w__%QEI5OKi0 z{RFxcM}SfsO*<#sxy&{#<5H2aTCD28~u=kQoo2Y<5($4Vokr;fx+XXVCn(W(~RvCD{-MCFVc6< zJCvP%Q~b8CI9*5`HMec|jbm}PU(Kv| zpFOiF?dXbELyL!))A9pkSl~Y5jbQqz018Y56<*kpoxI~%WXJcubTEgXLrZurTCn_aBZA&y>g;xD@y<+}YVfN;sg3(2_CI`q$N)zk^|wW`TMxPCJ1ys^$SMOLlVX`>CGS_c=Zn0=oE}rBjGT$*+ataU(Aa zg|Kpg6ncrNbjogM6t*T2*4acu(HBVILLWRno{OSlZ^-=AU}NSQ;uO_;;xAeK2#Io; zA&v}AV?V6=b}dwFJ&^3UVMkO+e<$P~z14{<9L_zeXLB z^!?Mm*yC*|J;Jw41hmNHyK@ck`q{q9Z=s)!pAY_Q zENV_rc+q@3!=RAKv<>NSHuf&P8gJWO!@$_ARp$9kwAZHaK}U|frf&BwjvtLN)|cv5 z#q3xUgzTbvNv0 zYF2$-DM@JyM-q+&Po|vXebTqPk@F;yUHs0~Q;H4m?Kj033*1&bJ^svgn|=SJ9sFix zTcU3k@7*GLZR3!5+S4<`+Ad|@%Y};rTfU8snJ4jHMSap4Dv8|Z)O^*nUTL&X)EOn2 z)oDwZXJ=Hs`eeCYrPKIGml#<*=3}#A-D&+;(!-ataw|5f`=#7Un_+%KCrxwY2vZ*M zVb;f9dEL_-hG9MBUE)S-n$P-_TRQQg92~-!EVAo3F3oP15DeX+htWgNyxi_Vv+>Dgtx6 zO8~cKY+LG)`@U6j=jlVPTo2>TzKkWBNsqw0ZN{SeX-8 zi#G2l(00~)b62&&W4UrOBStCoR57t)x1n(VNI@Z!N;w}x@@#_R^XZ~7w_r&w*@Gvx zXyPQSt-ep~Jv7107&aNV;zNBWu-=wkj>CRyk}hfMNKAxo20Ai2cDYiEMKj|EUeuhW zW|K^{*PNP~svGUb$B!in9|vm1V$X!e7l{kTjen$<@1jOzl@bWZ7(R~n6Q(8H4c2^X zy{bf2{--walq|ybgkRX2t1e2>K)(vfUz`o8oq?gc^;TEbx&PFXn(k@yyuTE97| z1XjiKD2h#2JW}!9q_*mL?N$&imhWveTQ$PwY*^)dC|a{+BX|03U(w>|6jKAkn3QS3 zQ7en|`g|jHK>NO5F~2M(;4{5Hqsk)qcF#+{>9Q|wdZEOy{_Xhs*tD6D6KYRxs)-Pe zU$Qx?%$SEW?A!A7xi#Q5n){?+DX%X+zmOqmTBLMqL$YJ0?oz?d)ZB2KqW0o==E(MzuWN{%Px8?$cc5oqgR3_Trxt zoS(jCojos!wK+xBON%o2$ElJ2Dly;csYIgZ>CM{}8w({I^i{3K^_aG_i*}lyj!yOE zd+3}lc{l3XyqMOTY||;=DzR4n`L0*Z`OUgVbo;svKcB6fkT0Do+ZCLvdOgl%_1;@E zZbKn@YbD}KlXXjzO0R{BM%?5&$Iqvqe7cS7yv-bPQzF*}4EXkhm4B*m>wDoFF1u?i zw&co}+G(68rnr&fe9JU>a`N_i(4Sv-#eEcoN$~PEtptTmrwVwz5NZ_YrZ!6 zu4@$(eq9gk#e&E@{q8EyZ)I{_*^#Z{*YYZ|>~pQVyjOA)9lz$XXKR&v>o|@!UmrXHBQLVz&J{brZXDNA3-O{Sa6e^oE0DUR-L#wsffCO4AyH z+Sb)We7)seAHFWwaVFRqop7mtQg9_<#+xxo(tX!?-E-!0Z>e^n+-LUkAm0xz+|vKv z1$h5f?!j_Z>s@PemTgh}_K9xQqVNWqE8X?C-~IW~`jC{SYMwv)#g!i&8)6cFWAgjL zp2@B1Ji|&OpW;^I<|#%4pNEM{V=rpzmx6tFYUh>ggdQexh3#y+E-fZ9+%#MI{78K6 zOG-IBYU9W%njZO}Kaa0(YKfE0{CfF7WMb}A#@xjhkww`prz|A{XQcKri<*as&mn9X z#Nel;K|()!tC{l{m2jUh4sPCk$p9O&qF4o2xWMvDG%n~2YBMZlLn_m1frUU-?p~Yl zLx%i~8`xo47bT8EW{0UYg7@@l)LqcM^!t^;d~^3t<<8vVC*>I{Efzu5M!)I(Hx5lc zt-G7Hzf0=Q?t0U{rtjLz9#)E_7Y0lPmnsEsb}r35iZ~)>m*g1cW9d>rmj}=!9v+`% z)^APts(gOYDrv;ZCEYt3IG)y$B?E zTzgkb@3lSm-Pm@xAUX5&*%#bC=hg=Pr1oDN-A(ItQ#BhMDv07q@mN;nta2~#P*dK0 z_dQ#FSU%IE(74e(&;L8Ko>GVF_4Aw_%~=~6p1ePdQgX_gD=xn3{vlm^b*>DTBy;T2 zk%Ne)RYU8TI_)qkG!?YXz_6!dwv|K~^p`QVoA>FZwF}8Qau_ z7j$~wsW{a+uh$FR+>MQm={YOw z!TSTBHgrZ>Dl{6pg(i+)7)lH@8~u96@K@Eshr)EP*2fw7)IwYXFDX{Of4RsuE%44p zJ~MOYyl&q55}k#+Wqmort`Rq#y{~+-yNO#;maE~3s`Y`2*}!p^jtN8aue_NV{i)GU zTS}+TPJGKRO#JGaGjV6^=2(Nh*IPx8=E2`{QFq3?S@zwrXJrUW;WlSfdpZ&Jp7S4v z?##$VXKtzRj`wSzMtxUhdh;7+-yBJ*2qI*WG9n)fB>gr>PgD7Axg?pYdFgA(Ub2ou z6YzT2Hnyxmxz!8`OduPGZ!$KG?ow%r$43Qnz}1c?jnh98F4&L#BLm8l@!|8+m|o7W zziuv&4Pp2d^oCeRQA0z+bwQDz&OS-`BWZLr0#uZx)caeu&5m1x_ns%u%aHgs`VepQ z*{6A%h|}q7YrXSBTaaoyV_k|n z&B)xiUXTXivF`)!9L8Qq)`=G3#Y$p0^>C2vMrC{o#S>!qU%lfMj8?} z8eDzjOapQe=Pj&C@&eoEz*$diyZ4jFmIjuKFEa?qA+zxrf805(>0HKT4AXTu<&LfBsUQqCI#IIPhn%w=Qvb3kz9y@kD+L zKB9)!HTHg2Kc4oeiMSxXXZkqe7ks6+cFMm z`E>k%z+lBEYr_}luUUbQ1oPLKMQ18oiHo;JQl15)2?~#1)kf=b^`gvcg@v~xhv8*DybAomGfO?Y4cckEloF)J~0;;GwLm=bKJHI z*Us5VFx@|kzwd=pho))w>aQw~LF_!Z)r0`*U zv}>a7z=g?U9`0M&qv^UkE4o%?%}=f5Glh(I)my|oDke4uml&15w3M?wxL$cXX?eoU zJM};kLsEuX@;P%xiQ5}lq%@X@LJ@f$DLIZ)UjJI>QAdje&l~O99d|b~oVwbJ>r;!R zqMzo*H+DJ)Dt@|r5J(@FaN%BL5qPo@vwENol76f+SgTfxsc%c zbCc`S^TuCZ^?x`oN>%wX32A-~^OnB;&W>4g^sCv^5Sj94YZnR+3W%15vZqFh5tsdu zWPqPP1ih7tjM5%u{(=5f5R)?jt=F-U&ymnu?gI)t3TC&he+qjFo#)4B*5(-Z5^>>axSO!^!Z`7pLM`R*hIH;tWqeZvHJHczcEvPs$=b4fAZ5GoumJW97eB4 zXK^J9>&Cykb^HBpRp!8xC%$)k?s{)oC5`0YSrsy_TAY?Cot_489gd^{ZN94KVKzk` ztV`P+YdUozm2(0gJ5W_WWM{oIGQ@|FSb5Lm3i%6s55^+L6a)8IyWjSxu;%mb4>J8e z>oa**Wc+oP-VCIW28|(D+mHL3^CBNh1;YP|92w$<0}FyE3_hF#orU2j=oq28^a-(J zSV5FrCjXVNce2%>c<0pxMJM)1@fu#aM?&P`BoYdf2+UODDOYSnN^J`H(<5SSKBt>T zjXp{kkK#&*Jn^AEx$tE$v3T3{fw+75PC`yyRPU#>Z$`Y6+fT~Os<#(6nrbsz!J}-< z^RnUCKensiBDuO+;~KC15($9yt|U;4g;i{Ebm}G)HYND{v?*s7aCUq|xcKLfm^GE^ z#=!S;@3RWoxkbI#W@wzg{}O5l8L?Jjn#uQa_iFhv>ea0A&gaLS%V#^*h?}37FON3J zyA*`Bd&4tkt?x#CoL2dJH-z-#`MCTH`{&-vl}e2~^Y8h-o0nhvnKN+0)xLR0p=HKf z*R*wJ)t0mGLSIAB%`zI>FUd|M12G>$YFPI+{o?z$MgvI1jWdgS{Fb*TYv8*5r~;u8 zO743=Lliz-$)UrX*={dhm3kMD><$10gDzrQpJ1xvObKK!x&NF_dafpeNk*Ves zoQPQQDrOte(o%}PpLG-+so(E-I0=N>gEo<;73f)I?fZ^4_`?3q8b*PbbBPN*cOwlI z`%)Ub>f;!@qU5*SdwtD&27?#dJ9fW{ddBCD7g&2LZ*IF~t!%XU(KO^7+fSFT=V%2l zw#D-(#Ans_zEw%KxLH>6vvSnD&$w#GVS($IT#0wsJwf)VgW6KH2t8_!ncQ_=PXE7GxvX@$#yy$Hp&@cX?NjO}x@4Mi(Vr@ABLZ z+}%F&`^!#>xP$(W-Ivefp4+((<^)iTR;Vje9XhcU79clrcMjD0$aH9VXtP`PoW~Ef z&jes@;>bbTM}P3?WZ>{-{W9yFYqs^Q{ScrRuxWlH48M(6xnqyH5=7*e5&Y zXgObN&zh^Y%fS37KC#QO|J74u)Vf)(cl;yM$1mNEb<6**c0b40clo|e)zYN`-SSh- zmYF$GcNY*hqEX6#clUaQlFy%?VI5^Z9v5s5^mTua`MI$FyF0KV3uOErF_P1w1iU5M z%)a3m^B&kSiXFnH!NJ!TknP_<9IXdu-(1GhlStlJU_^=@<|G-W>(*`@39c_HD*CZ% zKb6fspZMz(Bt>mF%5O7NrlAy{$ZYeeWC|7W&pIuJaMkMiJ3;sYR)DRcC~ z(*sEOSO?bJ|KzEzy?$ug*XC1oZJPK&+E{iyx6~*@jDN!s#0=T649eejqE@|nRxl<^ z7RT6Lf6gt4==0=2w{Q@#Hkc9g<}BAN$0x0$vpBYS|3SLSMRA=#E09Xbu`)o!kct zOzAW_A?1C@di?%VhJxx6BDrD0 zh055pGbV*|>P3(DL_n2PZT(q8ESzv6)wMTS72+$t=%Ql@xCDP!CE%Od-xF6d8^6{I z4*&UC9Bou(KQ&{Da8Vvr|J~XX1=E%19uB7!(D5SnzUkpOlKsb9V8Rnjk2nId{`6Pz$fV~ubWdx}P;!jmcre^X;inEAasG~&)%Wxbv1x&9#R^yKK_jUa3)-512C zpun6=A8UxXb!(K}HY($HEKGd$jLDpBix<;~@Qb5Xt7J)&1L2aP=XpwvlA^k{9$-WC z(J;^1z~aRn$&C6{jleeC(4eHb(fM@O#mKLRzJ)Ged|1K`^7cl(8J%uUcXx}T-Yh+S z!i`rCBt6a6FPh)aDq_W&5Lqny9~CMWM0mAtt4v}z14|i~3oGHIN^39(9IUxPu5M)H#loIZ8I@uuT%LPUS|JVxCTcL#UF4CECo6+LzP| zJ{o*;c+utiu@Awre+`)*3-H5Bhu253^baU{6RAJN=}mHk^N3s7H+*-z4Pyx7Ugn=k zicWwFm)y)XekvE%g2tot4Xgt)kENzq;pSybw#)LicLQnj51H7 zO)9{AF(;c>*=OZDol!w&cFhlza*CffO9xi7!$eSW%=(rt4Bsa623TIXO!1HAx3Cy6 z(~17afP>c)cACnl=|+DSt!=MWUoClW z_~Y|*4bit3{_MB7{~TfT5dJp`$lyj)X^{yoC!38@Y1UWn?L`>$6-LAS^@|IZS}&U4 zBRWD3Da!`<*X!U19}vP$OFUFhrO|w6bLt>knuWAXpLXl@5%?=zw9b$Uo743;!ETWf z$v?0HeE$j0xok@|%rZaL;u{2+Mdd}x zrVm0s5{ufzSINmOTU%;oWoguy2|OoPB_XNyk;7lpAhlHzsp85ZhNosv(<*+DM>lgZ z(84)Y8qbAEWSezS%NpiLsBCIb`yTet;;IO0dJpv-kos;IT-?)rpv`HW7dAp9g zq3e4Sr+pBU+-s7<)XsU-(S!jZZ3Nb=;d|PTCkeur;0*e)>dAa|w3b!KN2Uru^#{_H zjSI|!QHgcK^eH7@yff_zB=s&6sg#U6dDOq-o)7ti^hjZD)3yD~95w#ZXuZ2BT%|9< z39u=mrO)MW;Ux|+fdb9hhrcD!sha(26q``uN6S%^a{l#T3D2M$M#*)O=WnKL zQ4yw1aXEGZP04OT88#gFMm&hu0Uyh!jlp}$CEjLDleT>O~yQB!oizTs-trAPb> zFJDxf;s@hMnn#NmBP;6#Kld1e1Z}cXkkGUWG0pYFuyL(t1UdKPa+%nDF}XnG8~A55 zb5q=knduYDe}&2pb5@P9IH=H=EYbYP$zgF=PE_XCG~-O}UiqxD@V&9-i~%G~{|>5| zH9Ps221nu){gqbUYrbl5@#eGseH~j92TuCX$?dJ_Q|+8m2FDoCM08w%NbmbPVq$^; z5M@Lr%o;gTclI_zI=fH>y%26|jaiL|7C|jOgwy5m(G;)Nt~9t_=4!I}OOOs)C`Q{5 z@Ma+NF|YO27d|x5aLM6i0(eSzgd8x5A0sUSl)g}VY3wbOn2)OXNKCX+j+0U3#Df{a7RHtktq3b{;wR?Ck^Y3Jjc>r&h?4GJgVw1nr^E!z|xBI+@{@;!numHU&FvmT5sM++y7L4F*Mhd}Z(>)LRf&>`4| z^Q4mk<5empDeKPsq=W68hcZ9yO0d8)m{GLM6n+Ya)#DVU1cFt~Y(vcc^wPt*!xbN) z5sENnGRg5iK=L6*XSb6qh;0ZW4b!JSd!vO7NF=bY*Xzx;_?%kgE!+yCBRGV(lVo2@ z$YtU%!E6sA2(N62Iwr4<>N4MXoQ>P8T>Une^pUPW^Xw<1ukx&-3=C3{;Cg)kJ7|-7 z($L}Nmj+<6Av(O5=EBYDzn4+)X)w3Uap9+@ywN&N8c$5Ra`f7N@{7Tx9JDfFgdtqZ*vfe>vtNEbTs?u$m#GZgGgL1A;Dq zRaXd%X}%v#TKbivM8sllJUTP=fUu0cHc`JdHNw3y7gKd1Epxn9?4cNv^Xv${(NAOj z|B?2VQB`(P+o)_&36(|>!A&VSpG^n)F-O?p34WgoStFR?R1Ox$5 zKv2qW-k{I(yzlwW7-x+0$HKkud(Ac1%x^T>X=?do+EC zl?OovJ0pB8;|ps$f*huSO#eNb(9W3$Gi~A!jonjiHJCfOz)_$T+TTE$I!&=yzDzmD z;Xrny84O)kw3WT;h(HL&e3Z%lI6ZvJ)T;+nUOih{e8-|O{9N669KpR3sS6D&4!03{J9B6k3Eeo=|<5fz`rZ7L>9Ht z8Kj4viN31t)Xiy#D}_8dRww5onjQA@9y#GVGB(5c4POkWu0lL)Q`+s)7MDt&-#uS;}=6=23RYMGpI8Ev1D$ zDRLLQ4)u^CXBB()Xiz73%~G$)}fe_WBSL zQ>Ob(it5DWeM&d{dG$Qg{#kEB0=S3w;Pyxe-Px!P>pE+b`R%^}C!A(?xi1e@9}rhX zSh$A^xCVUeMT`~^A;GAFM#DH(xS1?Pj$A#S5Mr5l z@JcA!Gt|g8LedO5A!UMcRevi5Iz?ydQX2{ceU2c9JUvp<`w;DY1`z6fv=TNf7(;e~ zdyMD;C`@Y*JC;};OJve<^*sKx99pM_bvDy7rdlhMmpA_&k`Ml3grnmXWof*BbpC@U zf**+FAN&9vj2|7qX4Dt%;CAt${_T+mQ@X}`J?WD0fN>fD6n-VVS?men(!vO6J$yb4 zQQ9HAO5r1D0;GNzOb0gV74MA0>3!S$LKfnL<674rMX_D05?@d(E-q$zW+sB*5L?M_ zP5xdNR*kwXl8T=i9h3ejI+|Ry(OF(j2|j~mxbd6}5kZvfQ5)J|3?9)@HGiZ$(D}}y zdm!owhxF%rX~=@uE@zBe>=6q7R=al4Ujl9U!sIvK_RCN95W(dd-a{9!5rV|E2*m=Z zD!qo$; zA?ytUlG0>dt9Ua{4kZlijqX0s&rN*oxT+Lk`}l3Vc^X1gr$H*T0U0a!M99ow_rE`q zlH}v{_xC=k@|IR_wKbM-Xf~4fmC5lcCFtPjFu&+vMUyh4EbNK2g{~r|)Na2a- z(YP%|vvg3?L1}_yzA;?)zl}yr3%hm=@5u_Q2w0sJ%3`gvhrLKDGBxWA?(PclAjjl) z>&;gn2;B=CT4^+c97R!Tr2b+7PT-P>$f;SuMqO8Jos5b3 zZB6H_qfUZ9c92Tk0;C<0_4rlS2Z%uBo?CDM6zwK{bd_XWLIQD;x7SPBI{&2=xyJLo zGr26sZApVspe*e2QBn_(-pT1~>Lp<&i6fHIB_~mMj8n16s`AVeW)4e-T}>otE}{YR=0;f_$u4i56rxW%YU&%mtwEppd{3N zSR5(YuGbN8Ni7#^V@p}vkxrxzp-XFP(n(YRq#DWEaUzw)Xf+-4>F3R9HZGKqTp7R01L31F3WF;IuM;d#eTy_x*F3Uw;_Au*#B4cQ<=7D$Zo%+(sIyYS8aBwF*-!Sj?2krh)v$z|Y=7 zzkmAxm;wB4uq#JIMM{qEF9}UbfV?NzreG%jE2i!W>(6g)ew|QI8yVc+)mBNo{G|s% zU((DNN@bln@h#^F0GXJNRtkEmkR}^MCz;H{Yxwm17b8pGBS`b+P=`96e*=&TVp@z~ ze2<}buhq{1X=RO6PDDdKGDKpWhUf}PROj-(wMEB)IYs2U3a~JamEa^GOTpp0Er2}v z=cU#zOmOzzw{;gvW;PjH0sX7+vjAb|5=o$=q-Bx5np(mek`Ls+QF`p%UNh8arul#^LWW}>vn$nyzr!w?zoToHbzs_lLe!yR zy(iAMGiGt@;MhRi5e>GC!B+4T{#pDpLIdYyq5#}hx3BSjB=`=5l51s57H+HlHg1>D z;+E}?inBmZJ3SqQJ%>lNL{o_SPY;-FOw`6KjTL5Z`qwdT;J*&1*g^ZrPP~3k=_utHVvzdQM-%{Blp)$O2YcqtQ$8p+p*@U zwx`6Vgbfv5<3r>!L~$AFlPVJ1p8g+>rP3el5R|^az|GLA9=+%137hcMU~nHHeFY3m zNQt}Yg5`&EogAKS#x#!b)5&T3Dsd=UFSZ-+mzL2Xt&3&Pc$+;pkKcnovL z=B%9%J-olAh~OahZgcmOB_kLO`WCCtiSo*Wr0Dv^R#nk)Yhnr~0b8m-!Kw!_2=}1Q zwo@W5P1iZgI8D@Ty%aW*W4CZ%)%B&KF$1E&AG7&pf7Zb{o^&f5w%?y!&vQ<{SJ(Bh#r2$&x{x>3I2Akx5 zN!(F7JlILc=0O!JTiKBVGfRr1oV4_fD=p_plZehpvm(zRJ%j4dcuYt|jA#(NQCkq` zve)%q%weSY`KOL#K^Q5;_xthF+^epvGZQ;&R)I)#u}UopuhKN1mINR1Pz=T-^+!}4 z+i!?$1RP=nq5uv4zHtIJin9)<`@49aTnk(3%k>BU3QzkD2liKcbQE+lK(_X}SP(lp zk^(Q~Z3wxWxVgF~zt`H!IIV1O^PNT)_8TvEtWcl;)I8>*j_-xUm&#uXn2~1FgRgTNrS497%DKeSU`ekWO{&}qb z3pibzXYH--9RPy%qmPdSk**!S@AT2bQ4n|UX84gYkTM{L;U0~}pzX`R69S$a4(^}R z?#w)?os9mq_BkwGVa)gUZmv!~X6W&yW8ypCUgny=44f263z}Sf_asL0Yu(F-UeAx5 z>ZloWN(nMp6`aXZ2($R|rf?`ng@Q)FLb)%7(g>R@dNaKzP4Wu-&U^Ui@eX$xA83cp zJOadxs43Y}8i~IOzn;WZ+;NQ!7kvIomi+_G&bXgnQzxZ_o<6(pW)YokrA36)G=IWB zm9S~Gi!00Ee1~6@pz2XD3+SoPL)zh7>ZOvn4>vsgt(=EGSR-vGEvIcXW*P-hD-# zR{CtugGSKmVTk{0M&%QrWkN{eNu+fXKaccCSnj@e^*|Hog22rN_~5yuRN={zd7e2oo_>4b2}7*dHHnnW!$sW}Ru7D$M33IK$>LD)`- ztc#Rsx*`%k-so8V`101ujB_eBHdeVkoNl&prG>)#_uDIGaW`9jy>5E8{k5?>Msl>4#tRoB!2S>A{HVg@37m3KRXqDI`fFD_0bLvnWosnF!`m)Tdx&8mpjLxZ%XkUS z%H9KQLGq%dww3d~cLh{05VWaa|5U8bXX(_u&>4`=*DKcVl-^#75i7AybB#ClTz*>l zR^RBYL~DlcUVW-dpLkw%T20!WNrOqBZAYBfTCa=5d@$yR^?)<+6}97bMvZPy#?-tP zq2oZ?l|}MkUN@GsE-D^tjtf?EyF{R_K*$qwuxFb1>`I?0lmq_|$7IpQa6Vs9@#)jz zp)9CEWJz&Qt0YxLjr+O~(`Gq2?bJ!M;7>D4*v3jOjq;N%0sAO31mm&N_~V^XZ+hSf zT2>~-hsvIbzB}KQ#FwjwACeX=mo|Ql_SxN<#r|v##>nih#|YWI>FD_^k{x69_KC=Jh>bsd^X#_|Z?X2@rZQ(1c6F=Cv1+4n><-aJs8}s45%l7rz;b zx9?UCK~-N?ya9FCT4Mif!fxF|g6+iH=6YnT*Q(5Wl7+i1+AFS6;`+!U*P+x1BVOd@pn&<=Ln#H{RRxRZ+#630)d4^W9zaAX$A8Ks+1F6n@^@ zu-1u3Pfu@d@sydD*7{~-s1RI#)Z|d{iu&p6ij`vSwIMY=6-&!pWB1v(N}Itf`DZ8T zBsv}k(R1E-{ZOT8V=m=p*U#^tW}l9j3g-SeDgE=>;Io@Hg*BH>inpx~3(p%m4Eu1J zOMxj1arW9ukAE+@y;NoSZR$Cpt&-r?g>>m(GZZfM@w3AY`kUh}5!E+8Nb~2#2Q@C& zj9K)hiswDZSe01qU{WYsdH2Lf)ygn*C4?>g{D{r)kTcv=*3@~=74{&7Kc2TYbx@_g zFGI#`Yh#&f`Tf(FW9Pgsr@PN3wMhV>lqKvGpU9fcf<#)oNL-y__}QeZwU1mgq`i9< z3m@>r6>xHLT1>ZvtimPjtgj`e{0`3)6_*NFoYA-~N)Ik`a0_rJ7PRf2BBGJElcoHRIgkKq6d&9gQa&|QXgQVK9 zg%kdX6u~L;IUn7uT%7|y0CS0XZ~Y@!Xiz1WnaWx3MWvknYU{IKy*Foy%UYuMU8bX;RZpZ~ z{{_M223H@;rPwyz=`&)tGMjc*yKT!FXQI{Yc1z{4&OdHYgksF8(j4N&cWZ++mcIzK z+8cS)$K6*G4$x7QGbDOdXdSh0oGvq?oS@vBynH3cQUDUy1Id!IBX!y73hTK<2wFmI zr*I;0@(EC}Woz_DRo}T=zosI+MIM|D=)&Td7!~h*_3!n-zM{I4n9F93Dq4v(yEEbH z{PKioB5##-zerMaQz(WONP=-(SCUIGoK&`XdEjtNl7(y;!P{nYR3So76kiukD^pA0 zopX)T79NfPiFVfKr`&A8SzNRmEq~m!IqB)#D?AcxWi_2l<}dJ> zT)ltICMY@$x!Mfxai$r3yb>U~J0N1wK`Q<^)kS@dqt_$1B3bf|Q`*|SdCIg8gaMRk zC8-t*w(+&PJoRR)o9CL>gN9V(MF z{bWUEYd%}s(9o{IZiA)w1<(;I*NksHKO*41Y9ScdJCn0g_`qZFNo}$dC!4{7%Cp~l z+bfSUZ*&)lE9A@UeUHd99v@UX=lE{Hfm3N=vCnmJ#BlNnM9XtOk_4?)Kli2ej>znM zyU!7kzh>i>|2f?gb4J)M_O!Mj`+|h-sP<#k0H)oEV)7yI&h|5`CXH2RGsAOV#Md~; zv+*qEdsb_F9(MK`&ezmz%A=`Ld~}B8&ZK+3yGHlqaz#7+)2jso!VaV7*OHyJc`^0s zYO51)3ZIq^?tYmRDt-gO3tO_3Hf>f+N=EXvjY5y?y!9N%GTx-?Ef=cBGI~E_SEJUF z)u*-cHh0n4E#kxkSrXs8vF2eE9pafD*CS@ zKu02o^L7-t2q-XIn^K2%x-2@xueU^rrlv>fW#!N@iw%FrxncyRBSzS!hwX}medyr0 za6|=LsU2A=tXE%;Wt%Vv{MreVVVV=RcqD9VSb7!Tl$ps4a%*(Y;*4ZVsc}Dk{D5VQwm}HzP3sdjqSqIL-LO+WXE9{VfKU_BBPG-2FS*2l1EAE_goKE7+ z(qKGas-Sgvwe4^oeE!*vb3H^$6x)vDb+?%n@-6V#&F<_Pe@;qZ=42@o9QnTX^d{ZN zo&!!i`dMvRLhPGTk%ZEWjoRPTkIm*>mLDQS97JtpRZ+3opRVoydEYz(u3TatS3VU8 zSqiP{IucVv%76iJX^(fnd=(ocMkNtE(85-d{9cSX^o4?;GVYk}-L1LB6{NP&u_l;j zC6}_f`80M+Lv+GoZjju8^H{yDp@c?vh3zo6is#q*{Iaa!2G>P(5oSk2AB-S1izrp{ zNtvBnYn3|K%H8Uc6f5x|au?2X-aFshvqnbe6tP3mPWO+V5J*V9{hFq7cVleCpxx+Z zb6f{w_Vw64YqbenrBCOn2{0ge*wm4S7|zSK`h_z!<}Z)x*!7FV#S1CEAL`WKN(XB5 zg$0lHh;ui)4%->L*m{?HFFw@p{akMf8^mT;*yRy2lZ#9BjS;04mA;KvTU2vn`n~qd zwOcuio+g#SvBw8{Py6^QVOPVLKm8Gz0C`7jh03uyk70Iwzb-L$y1#}P&b<$m{n~8( zWc@*=RHwy;k!VEuqMzsOxf>VfiuYk+qWJe%B>?qZQXTswb0+Y9iZl{7=d<-k#y2ie z16Yyc$%WnZVdpk7r8-<7M_S(O$7*lz=asLhjVvhBSTU*WzFWf?J;KSiJ8N`v7bksx zd+U&6#=s7chyzM{v1AdD^kvodBYjO@Aq=-`K*EL>4+w$!@lZLLMKy(XP=%#F;QZ_EownjIolu7 z2(o_qZdhgYXsLN~GS3W>tw$C)JUR;!yUkd{__id;=SN6yDW8;>M-PL0TZxnQ-g;v& z0zp+IJDIgY{>ro8lgg6d6NPv6;w8QmWIrU$wVwdTX93B~`IUOIG+pX5{MR2W_`cP< z(ut8=*5%7EZT1MU@~@Gs%{1=!5X7`T0Wmp^DU2W;`j3Q z^21p=$M5&W|0#7A5{`1KLnt92ay#@N<&}lORT8s!oR{R#bdas~A!gVA8YF*3GAAuu zh+d0ZQVpUn6AY>ZFmh2%)T`W5RpN_lvoKpro?xCh+d%!7_;geStIc2F^0}|lc~-UR z0?%^1%AA&gif_Bk@RD4zrE-c6c55a^(*h(~8lbc-1)=ZUQ}wK25}6&edlC;5Dc(fPFkzy3*9 z-Ws$gnE7yI_-qwIwxTS7ok#ZxUw|kgx~Sv|Ox!U43=GfkGMen@6sw9MZuLS#2s@*S zqEo8YGgOU}9lJUBaq1?WBf#Y9RnFC@vV-i}Dj37j8MLM#pMUH1BfUSq-wO>L;vsI0 zxRkkz4y$-#m;N!R{^}6x3Xn~E6Z`}qRHstn@qJvg5*SptdGv#xI6XISkSsagGX*w4 z2;q{4EH$)kJFP5+%66j@$MMBp(GySrv8_66TPV=W2?#c- zW=Pr~Z@RJ8OeDsZemCov$6Qa6VrHe|+akTZwj>_f1QocF7BR3)riI6IlA=UDc&!;e zM)Ep&1!hNmthRe$bv3JJe5ukav~}ZJpQPoR0+U?%gmC5h|{=@fI0_pSyoIqzwh3TygsL7o}f{@b2?O4$lmGdzxUppIB|}u zB2{BEizckqYN?)H-&4uiG%&Fw{KV>@*t~(SRV~-UaC-leS-@V&|Lvjn!vnUb!!4iWi zb&hj}C-CZSk)x;h{EZdZxJ2sXlB;)&_>H`Asb45kmy++ZN)$F_X-DyF=B@TZ&S~+% zdsEo5(`Rde&7j6139(*TNx=>&5K5kP>ERDqdA9Y1i|M(IBt&Mj{>SJe&335x}a31qMfVZ}oYW1J9rwXjLNQo;p$G|l;QcoY^02pl~hdc^V6J00&6p!974es8lOD80ohK{uAPo8gs6Qg z>%#(A&il5*{q6LgCBF>o3&bpw!~~UXj~0QIU&;G0T5)4`+@)XG%y)C*j+KewB*sW2 zE2hf`%rpOp-Ei5mhw!J^Pz{Ov`0e4%YFm*fv^Sgh8Nzt3mbWgM1hDkpf~nCD+tSzj zi8&L6ZHL-|)sNBN&2-o7{$y*#K3kx%nfdv(z;IbuCGFY%FoqPI?qa)$>ABTczZ^47C#h6Ugi7Y51?pU&rw^eUr1rvP-8y>WjIP?w(5BFEv{jKdO{cvg8&6C=JR*77 zn3HK?(m}rn!1hVC*b|Uurn&Rgb4d2)Sk{YwxDui^;>5!F*)v&1&0zFH{?_CP(eZ0@ zgLPL6OA=-B`|3Mk8WlN|rT0FcnHL)OnZ>nYyz`{!r~W`bnBfwMWo$|juB%KWnKh{_bWaaC zyCJR*`{j+w#^?sZ_cj@w0 zqvA8i>TYM-P;;3CS(SjZqM?l|&=nb8^$ntPOI*nRu1nv**UKAc=iB$gM&*)8<7+MV z8i+sJ1Ptb#8-uR@02+%T{l7FE=sn1{cBwA3vGxw=*4sJ;YbX$r5C3WPLev2`Y>8Rb z$VG-Jf5=FVUBx0^;PN5V2E!ApFTtLRgV{kG{_-Ng6~NzE5%t&2(s@-!q@owkd+K7x zimNAGzp9ak$-;nkSpwTu&_!&Qhl_FIDOB>2E)EIHWD6)EWu1E&L?U|bwiE1d7PK1w zQoo?ch@Q$jPIp}g5A~e#pHr1N0DDWMk^1YphF?%e#YveI1cgXMP)!n zPFb(=8Te~5!L>0W80vc$E!Cd0v>yI3^*S&P^yxdZY+iG6ozcmO#yg@kX~6mN^TpT2 zCJWoP^mthKTpuXt(Ad-3vSgOe_qzOOBhgfWEf>6_;9*Os{S5}jnBRf*zW5*O6({}1 zDZHQ?tRa5rv-UsQOez9paPzR!izlEVlz%xW;uSM})ek=LZMdxdmUcLaONF*<38JkY+Bsj`1wc{XJ z21JT{`oj|4bfQv;BjVWtVp+hqqFM5ONGGYke&gr2k8@0=NXKL5520yaxabpj)-`#R z9_%K~3sO0(;pG3gDL+{{So2vT0?J6}mEoBS0>cER&La(PnUf*Fu5N5%G@Mvk$rI}~ z9Smh}K=qc>StO?Q7j%Os4? zxuyNQu%u*8hAkr9C;bm5^JMKz5G`k2bo^-~4=D@(+0Zf368IRArWEYBKFmL=v zt>L0Mgsf+AmY@zzx*4>%8-S4~ z?@9k9Yx(p%t-P5ZzE9S=Kspbdm?pTPx`M~77FzlO4sBFJr8j74q_?z5W~Rr0uhCLr z{dxc4kjJ|4xo^Q?IIuy06cWi9Ny}t$^Sh$_G&%Q?-=Sf zpILr%Ocw^Bs>pW(UWh}GY5!~e00mwbm5f~z{?Vimnat=(IePjZZ6G)bdVj6WU$5=K zA(Ge|;)xohrY?ecqvtVdIf?6(lq~(Gh%TL7 zr1!WRePO*Ld#L4x5sk=}=49rpI#EDs-tT{hxbU_hzp+&pRs6!gn85=X__spKl*m~5 z>gB%}Gbkq#a3Q03VxJH%hFDI#VutHYau#pJ5S|1~Lz{y>TeEieh^-Y{*QopF4yq)= zwkKX`R<{hxr=<3ISvqrq(@XyU>L2{D){JA^BE^k95OzF zcM4zMYvZQD_XTI$dV4P0`fexQ(qK%&AEFjj+6k&_Y{8f#emw`)HIf8{BhrY?=vERs zjX)*-gpDvyHyG1$&4QGmj!8BIeJ6DuNue?$!3x^D+4Z>5jVX_v^kfpVqyZ)w_w(BaJRQntKfy6= zgTRn9gXG9$A{rLB9p<0???8I~eFQXzP{R1t_>nbR^wdlew+#_3rA(L2+jcQltNSil z_{}G@H8$F3*iI1KKpvPp=S>2!vW%aKW*MrO^iOZabn?f;=7qtz@k=Dn~& zJM%v4oF+A?EXqd((;zILf9Pl;%SCj^KHBRF^%Fw^T7n$--nNhX8|r_70uXcQc!C%H zk*}XD7OD*jZ1GFXX#7-?*H{(IM-Bt5>f3~4eRzb--G~QNLQm`(h`!HOB5`?EvpH4p zA;a|<=Us*o#jlA~4Ce?&Gf3#VA~MM2+0Na2K}y;oh}ShTOLI#;5Qdc~qjCG?i)o5m zL>djrmB-aQ6y!NYe+-r#@ZkR19RI5n&@yrTiSv>sV3s*s>oR{B>s?Ut1i*k^%%U>% zd9yb(A0l8(G!cHC%x`ii0kEgqY=GAKs9_ou!OhBt@-s~F!J~h53l?PqS3tOOi5M}W zlLXAf8Z2AB*B6scJm$Fh_}aC9ExmsM=tHQXJWdl$kmH$X`eB;=9A)r81pz#08XgoD z<)k+^pl@PA_dU2uilC0|8VlRA-|B($Gtx)bAIMe^=r(JyB)?Umom~v23Yh8-o2T44 zU9s;<6d0#+XHrcVwRNKIy{i7_KpjF2BFsRT5)|e~)-iWc0+))SmXAk>%F%~5KbrwM z)iPX}{1?mA-@!nq<$QEUc$({;9Jd z+ky`kL5`F;{xk+C#Z3QV8nYBm3@Kg#+~ZkpByc z;Kf~BD4lXmfQ(De=?f^4MuV1aF-D0ad>bVMs09L8P?gp>q;hu*`Twot-wd7ly42!Q z+#U?TJl@Llf1iLn?C-kL3Eym#a-4{r2eO=Jgv`N~IA z=bd7_FN;_6df&^) zX`r1ptsy8ocZGW8Fy6Lc=B3$2i!>A7tQZpi8w3#X`B9}|RVyLp@*>ES`$JqqlUkta zL)9BZtqa9>C*w1TS6d7Y#>xJDWaKgMQtp0Tk)+D3kzX!)FbEKk67IiHL5A%3S6JMR zd|MhqtCjaXNyH+~aZ?^r7Kx2>il`c|i9hb)bZu)g+!Qg$L2{V}ro;x5`#W@Z8yxK` zQQtaaIhTc%7)7@KpLKeHLe2A_&bcU`fSgX6|K+8=g+rAr^%vU<3}s7Drhk> zZv0TUVdqN-A>E(X_cxG$gbws?hGOnYg47MgEsNgDw%~3$vu&XhFF|Z%ECWUEDQ$gj zHzyTgYNdV4ofi7RC!f8I#xvI7-Q?u2z35n2{qD(S`RxNHazb*MGAr)X&BF1Fu=dHa zBu9~axrCCcMI~$(Pqe0tgh@Q_mTWxt&g_ao{iBYzq_;G$O)ib7e=eOkdB<1F&$DK6 zR1vdur)l0g`3@+~WYTRw@Ls-*s#jHSxN4->7oV8jWPI^bhs9sxv#)1=@aHT)bV6)@ zCFg3_J9xx1-9kbbh?3pxYifl63$FS;)o(qPCSl?EE)E71zd|6azUH6zmd(uNM8B{g^?#KT-lYkNaDw#(@8?9PY}mS&)7UoV zH8LwY3|8m14=$X9h&QhF%MUJcqu887L!u3Nikdw}5jBduIf_=Tw9!|A8P~*1z{;U$$-YoH?5j%<01(o$Gk>0OZ?cb)a%&scygNfe9 z=0Aw=Q&sO9x^1Y6Nf~o?c57c?X7xT7#o_h&9@XC-|H0XVi=wg4?iZrZWfr%+xxQ%f zaP+~z3_0Tj9L;|Gbv)0?pE(Bd4Ydk)E6>-I74!C;clNWP3rzI7$ODPb47fn z{A^JRg?ekvxy-L=((7bSeVO6cj3vO1RQF^9CC?~=;K>-PC z$9o4%^7Dl){t=JH#7WYQ!~O-9u9d+hOY-_`n?XkhJEgchCJge1gNLD4P!VgrZ5o>5;rqd%qsd^gfl8i*h{U29=Y@A`CuYJ7=DEI&|B`=a&ba z6ag+3+Y7tPn9s7P>!#fq;_Ny(Iwa-pN+>%^xvd%Xy*Isl98hr%02U#$pLMc7PuvSJz&ai;e%Ev;U^b{-Z^^5%%* zuPUQMXBR0)D%vw_3bb1KCO%%oI=6?Ik`kQu_v%n{3*+R{li0AIWns?qqeCH))_;T~ zj~Lg`L`A8RI=2}TNce^p5=EAHy0*rYOC!gSyhcno}T*J zA9aFnkQ=D3Rjf_fPGKMQziq z<*=(q|BjG}z%OZBX)i&+0W|aV9=Lj+``hOz=jsy<8<{hK@6%T{PXaE&{L1u zKV*a{3GenG&n}k7uDT)PUA|}CF(;JgDMs{t?f!m(d_xm$(kXT%W~E21L#;x_=g_}i z1UYM25cSyVJTNoac&N*p;v^u>Ag-`GS|tNnv!8dO=}=z6*}=I7$_}yHE(moF1x~%N z?qcxtZhf8naQ`Hb z!k6w%=guLZ6#D4DLWkI*17WKKj|R<$t_8b!vtLQruD=e0ZGiD6lpQ0jK0YZ~Pp6zZpPx&8fpkD*nOdJ3|`q#^filPDM*w`Xc{|UMyTyb&2 z{b`M^i>v@W)p&QKzWy6@##|GrpxT+olfH%tPFfdzKWIq3nn@+_#atGJqN+#S_<6Wc*Gj2jQJ?t zQv%6L0YmxSSH^H%5>)*XE!n>Ir`o{9rkU4`1IEQm<#^8;a^_xA+oek#G$sq~+Qq%K z3VXu=a-c+}vb`jgA5^i-*3p#G(=Ri16v;+mR8&H+OQTvX+g~LIH_g0%=>3A8OV2g{ zu^QErg;d_xI4(M_ad_DOf5t`M=ayLTX|i&a!*SJ zBvfdv4Iww4XQT_*b~&${XBxm`gbYkwn3NhJ;@n%J1O;mcaB;EzlW6Y^99u<5RiA_! z(z(ak6+f|aTw3dwNdPExdu+{c4QyLCfD3t`-BmAoNTr!9?Mr`V9L4xq-gnUf@5e2W zb>LxJc@?lkS7l$+r_RWv=4`!GRT+A>vvv4U%5gOb;TPIs{nW+(Yf+?JC>X8lP!mb5 zVbn#oj0w@3!f`_@%o_+-(=QX>g2t}*vq^Fz6y+C!nxZ!nSNfknzIun{BDYcgzLBs4 zn2w3}kGIYU^g{#sfa;15Pe-|3lG*#2o$B`9Hp^OT%;e4l3*esDj-m@|;ZhN3t3(4j zE{SX$FkRfb$ngfGQMXl!W4!YdQ$dL*X3tEMMI66)d_eFV1vxef3Niu`9K>;vLfi2v zkni(wKIJ$pIx<><@!ea?nC%hlk4|grQoZ=7oFT8v=QUWqQ9jS0g`$_^u^w|5%5{2wlBc`57Ckl45Xp(()7tKZ* z9|sVh>LOD*Yra+OG+|g&RMa!V*YY!~Kq!~77j~J?$e}{v$?c_W&Tgc?t_^&6AQ0wq z8&4U+A( z*Y1yzS)hf)h`Rtd)~9Q+gDHZtxy7i~i6hCDqXnQ7FO0L7%U_<1`4k~DY4Kd*_s&}H zX|)8azWE&E&;1#EaLrE~M3SBGKJ5L_|GZAu?C~Rdb#*m%HBQx-Nhw^tWbAc=W`^|K z58W5>s=8EERNDIb`u1iBAj;VFtzKTq<0`05Hy*>Uhw5)+wvM%9VI`VMz*Ae|Y9}t5 zQwOX5^-^H&WjZP91BmJ0pbl(s0GiZjFvNi&K+LoUaEp{FvZs-a6b+E z)Q(rayUMv+FrKY(q_|Kgzbo_N{VzRt-e@U>Y`mQ_Hgi^rd!TDGG)I{R_Lr^wtc!Xx zYEXqOjd}Bf@3Z=i*25&~%9uWJ1gRrFpLxGW?bGW=G0%3FZ`$Gt4q1xZvGu0s%>H(2 zSjdYV8GYJFm8PpY#BIoFZot6he$89ceYx&acu1{PPojAxf3Cg6Gt*E17m&2df{Lg?L5u`DB@RNC=+Z$pA zErAXg-hM&|*Zm{O32wY2{o2nU<-jfZ4I+elR9WXIGwv&tZ`j>4tifDCZljsO2Lw%t z(iP*|0k~Fxbf~{(5M6FqERCUcs(X%2lk%+Tj0KeLd+3S+UuVd|Rl87GPDVVlh4Oh!`I)&6y3yKXDkI+# zp6#$O*to;p5I%ZMLk`+zzPnGsp>$eg4CPhEsa|;x(@~g`{X4BPlFyspi)v`uSyf{a zU681@?a`B{V$9_+$X_`@Bf!j`ZdE$8qyuoSCoZO>Y2!O@9%O4CmuHiiWpXiMziy#p zd6Cgmo9yt5H~-PxKYx@VY}3Xv8U<%41y_LJGb!%6cE-T;X(I9BG0P#yyk+?kwp}t` zzMPgr3QBkJjt}pO-(9yWV=sWfAr1)~G$ZBn$(R)LKlMfb#R4!h({Sz8;d-+pq&ahQ zbMuO=%mBv)txn1QBd3hfF>IP5H%8SlK&!Av$SoS8aWN(9IUgXX+Eg6)_3b4|?>0bp zB7AqUIZoYKA5;n}9meT~4~?WPzAJm$WP}in7XJ{9Qp$XPemtf_ulad!3DYmNnF^3H zcK#Wd32yoBF(hYtoM&xC*y185y*Eujc<<9@54mzv)d1d5!qz}$BHK|SBmPs0w0DEM z%4Qv)uDc4fvS$DbFH~lWYn0j;&88-%Y5?wym8lM<>lH^r;V8D7r#4aDZ0qqdEKk65 zxI}wk>N_Jo~PO)fR_I*aGpUMh)Madlp^$xVQ4}N7}wzD0Lwb4twQH z_`f{?*plarq-uk<(@V4p=kjVl{JMtApMCy3l%N=a`c4Vrr3mNbXz+<=^%6=)d|kOU zp0PXIX#=c*`SKe|Ys*=B#CQ~>DDwuHWB3N|E%Bc^r13&VF}6S$J#UeVp14@#?6TlC z$9-uNaDAbncYOfj?5I*j2AA1t3aRNhyL9OT-C{6HAy3E@f&%r|tky18)|Ji?Eg2ZB zAyD2rr*nG03cAxJM-^pdy=W@Q8d#m})^Ou=On&@jeOR-sY9oxdL7CothO9p;#;QiU zQ&~_~JVn)x6K(4WE+hwn$svH~7^3u78LO8vtWt(VoM)5)R8@e><}nkq?mG(*>Ny$G zt2kTe3uifx0Bk31^d?xV8%z><&04d|K~fRTyH~rLD7SxI-zBR0GV}Wa`FVf#QwksBU|Wb3(Ro=EmWxDgFA@p%TO@B6Wn!)*sPiwLep@{ETI|FH1DX~ zP+P5Htt%O9hgV&aE793h-`efRyH(an%JS>1<2zZ?oBjo$J7$^wXJe{A?#@}SBQ&R)*AaS?~nYXFRD z1X2{jL#iCcNG7ce|6dT)Gp^!Q5)-%E>9VroYuwl7I!!&7sfY$`q0%*ecKLVzWLoQ( z?hoIyzZG0;F|)B+K(K6=EvGmbM=u{r%uRabJ+Gp z9#uJ0xMTLrHN*A&=Si2Z01EZ)GuTe;_h?Q(^%V!utPl1^n+spdK1?S@;ycKmkNdiF z)_&W5tj6(0sWA4}vpgAMXr3cChW?LVqNv#GGGivFRjf-~!X~5A#Eyo;|M^KqV4reC zW@O!yd$G!lKU@jz3KuyR8d+|4u$T1XM811U-NX328ke5T&5Y5fs(c)0o$jvqzS=g1 zJiV6u&4`&VPl{uWK`zxk7(2n(Ib^|%ix(Gfv)F)81b9AHBY;g{s|GoL+5$LDu1XB+ z+~QqQ;6>d(x-bA2Cui+40j}%b56Vzf?W9fx3YH^3mcR=E%t@kA=gqi$x@o;R7tPBVT=?9Jx!`Q@!o~<-RI2B#Dz6iF8U{5bhWjw zqmGvf-^1)#wSmeWy}$9n*VmxpdO$tl#QR<+_xPn!p^}`(ERKdI%EiZR*N&#O{G2An zL0&T5Fz_i4lV;bVZ!mx`IJJxQd0ZTR@9p;f?y5;J^-!s9h!mm4BHW{y#2exKX2U$= z)rPUPwug#b7vIgPf&Q8T335~f3z8I$i|(p3hE>EwdnkYtO;(aNcn2M>GKqg^by&Cu0y{joE^SU!IGeCt^| zE?4*dH7E0Yvymc01h{%J-@?~`@u%xO3|XpB;KTpNfp_P)eCuLsxIq7Efg#ou2XfI=UXzC4qfym3FbW9( zhoqs6IC{Uy5pKK^xHToama7YwMjwirG|F23_U4j_gKP0h%XXix{E*v_3R{SNz-kS| zb8mGMwTYiyWLpNd{A7q;Zso2=u_cdb)2ObU#QMOcSs*dQK!W=ArwwWM*t%);y%>=b znRIuj{9o&sJVi`+#fP!ACcVrrBV{+YMgXrS_Fnn7f2K}Oe#CR8|6xWEEu@EgY`d~M zXY`SaU@cHuccU-^A)inA_Ylm-yBZFv)RqiCS+sE?lsETwQ!ClAS<=4%`R^^~0$uE? zC{O8;G@+)f$Bc~=DS1ENbx`2!OOkvw9h^UQ7`e1U5dO^WD*M z8*>z0`^*$BCdaCth(<~TUO z4SzKs4aBO6&=vvJO?O-*Tx5QIYq6{w)@|K4XorDR_K9wxwnDY@TO_-KOz+Vnb6~sv zAI{zaDhjS^8CHfACP0qv{Vsj)6?h)fv|tL9AN z)S|7>5_H=g8brJ8nw`S#cJNbG)qJwbd0~>>JefFWN7YqUogIp0gwBZZZFjJ4fu`hk zx0JD26T`>${V5u=od|ALqm(g9mr|Bb7z+oSV)l;-rt;%Kc!}Fe4*B?7Bfi^=55@VEpalPEg0s#+jV)appVnM_A)#b!9aZU1a zL8M?b_R@m13fzadxg8)hZeRl9}u#m)5{ty#q4j%T|>%R;DIP z;a}}YA=Z~7BXjx5y4bO7X+nrE>eXkz?5h0?Jv5HGi#)*{-H;?5!@;DKLFiY6x7EWF2Abu$N_t|=# z+yR~i5i{x5vDd-|7&Be&xhfb5gD)(LQnO;JcSpnl|l zB8&Mb45=m}Xfh^FSk0--(Ts1LOjkBVcLkxl;c>0p0njhyL$I>)RtZR(wZDb2IVEE1cUmX{<>6#MDcn8HyW$ws3eC zQhHux&=d;xUNq__59!Ykv0mEaO3eJg* zb#x*dbc=P9r-p@84N2i{d(1r6y2|F3q|tKUKJ%i;lV$ zeebDrbD{?6%CJ43?&;la|})>;y0 z32<3Zm8@NDp`OZKTUU@Q_7sArrUOjblF6O(c|^hx!9x05yPHGU$dgV^u?t4Rvl0v< zA4ihC^aBd#E&cQn;Q}s?FVBRXpXbdyVF;KFUYe6wPB(pYw3ed2oVF&VF`?WsfW9+U zk)GB#60rzN$3^uXiN!``d(@O`=|ek{F0jv-03a)1X1G-iH|f833HCM)#9?k@{3#3A zKD8pokjK;RkJGPGW4ZuYiN0UJ=iY{<7E!pXWOog}oV8-z?(T>j2a$Qt2k0mcPdSg& zS+i(QYpvSXxEWKhu4pEYi9JLmljo-0)_y5uWhxgJEN>O~d)EBFj@S`QA}{CV{B#X` zAsnT;U8)WuJG?)~`^F*FG9@EK*%p-nW#dGj@PuNBYEQaWrGvb@{EwBVzMEEODSq)G zg3N(;xR;Yc3_xgs0!f{F9OjVCt1p>;_NCdb;`=p6?j8vM+1=kCCLnsT!R z)7SqUk>Qkqhuu0+u4jODD$@QuY=yKw85oXu;b><7=Gb8Q`O~r*fl7KV(C~AQ&nfRv z2WXPa27^_eIJB5R?)r{$SB>}3hQYcwJP2B<8}=mhMU!rSTStaTf%k4jB5*>I@A$5*2Qclld4BLt|kIE zU?-FbXQ$Ee3Qbfunf`W~E@7(O0?LZUn8^p6^PKu0F=^W+s^8s~5E21uDj+G4?heuRD_AU+j4psjVPQFGvPw)%}5Ig($DojE&n3y@a-8r{z+1c0| zc?Y#y>WB@~mC^8*L8j~PI1ut3lEw%pC_?URtNmcxgj0* zuEjpifrjqS}N@N>d%h)g*3=M$|plV9F6EX!b0-;8&V zCOM`4bL!x_Ekm8wtz;d^3FNBOgi9s%3d2N5kYKV8;*Dvuoh)<9_8(yp7AP6N@HCw1 zfXQP{1w$u7J!|+poK70Hj*5on*Gnfq>$smaVk1e}OsQcAS3h#w`d)ea*^HDq-ZOG7xQnGc2xb3d^HpK3|Nx4F}6HBPA~h)>FYTTz8?f(c%i zRO6%$*GlZ8au&WHkuO;@Iu@+1+Lk$zXkIA5Aq0H<$4j;ewWx7o)#E-=z(d8( zjlW*T5aMQFYgBC{&|x&3F0tKIAXYem%awds_u}-JT{x8u65IEeCBGq5hYnl2WN@); zl%|1`9R3x{GT46u{&@Mu7}bEbC7GZ+pP-Q4u0sbib;w-tW}RcO^6uSMK(_>bC;Jk~0+ z5&kyr7;44#Ta$|(F6*!Tt&gaufdTox?J4vDQ&wOn^jd{^Ft<8`p&kFZbh=V%Nry8$ zknh(t(N<%sPGq-b+36MTy(D9E3~*JAt=%CFn&;p6gj4yPdu-m2WpyVnl|+Y}3G^{b z1G{$|Us)sI?^-&u`?80QPYNJr-g)jHcLQT|3y>w6G(e+rzAx#EldF3P{np1neqAs# zUT!ym{$OT}`T;-LTeT`!PKi01)V5fc6_hHrD1kZ=pPGY&v3PoS?b(sWV6Fxv#wy?W z!^5sFJ#~V)W2@Q-#**RZ85SChGor`4OAv)PXYJ_CF-=+l*En;~*PK7=u3D@fXlfri z(zcPr3jK1w;R*~k9i}N-J(P7O=a7~!QvXl^`0YpSxe5EKW~mg0Au=2i%aiDG5EuxY zXkIu`QgFC%uBzlv5Lj+HK)>)*kQwEneOzd+#rTUgdMe6Mnet8~O6rp+&mB(Czz z=kPfpPrd-;%&h51^VeX)CH-BsXg46GB`2XeppuM4%&`ZLp(ycuGn_J24 zhW!DF23=&f^D()#pV5k;3Y)*GLC>b14)Q&~ZX>I7{wOmyA|TW92~jL1GvVZRa24OY zp&FaV=l)C`*}yfYw+QRWd-ByU?eDR=eX4r8+^9;EsC;fobNb;}vadQiyz=<+CUfwZ z6)EFpm-EnDuRXK!IC6)buQsaw4fBGq^Ja37W*`0Cu&|w#QK_?&RoM}TN(;j^EPaEI zjHT~(OC35vZ?75iAxPsUxdc!c%pH8DyJ~eyrWHa3fAG+MlD1Df^3JiUh;YSaVQ4tM?n+kWmPP@#IxD^N!#!d&-Nc1=?Ng0 z6u1FKHac&8gXr64VfDrjC9CK>xJqfGTW*c6X2&{B<2p}oT%GOeU5#^Ed7px(96(_f z%Dd^~C=h(LiJ#i4T43z%>bCWpF`v1gf&T5kETrOk)2##`yN;Pu=T$Y_DjVgelf$;j zLx+yN-)PNKwx=z4a;|Q|#jkVyyu_WmU(PbOWoB?B)t9U{>&IfUP$-D z9Y=_hT*B3D^5wzuN{nc34?o3RHGaP^b9l+sN24WyyCdt#qiE|261q8E*JkM-VOX(4 zT&J)dPWUk<8Q1c*VG-+ziN6haN!;BW{|oL=6xx8(sfisSf3->XO1>aOeGc@-I(ZlYiBpcL!l8o=w2%#^LV{pfJ8Fa zog{3v&3)4GmCo|uw)u8Bfx5E{k@M8~+KZ9=y}*S;0}z2uNp)9BPd;{jxNKoOEG42- zV#4l=5pgXPMhT}~=C0%zSE6*;5GHzEn4PdF;v;3c0vA|wcKlS0wvfAB1bN(R#AGi@8p^D_O ziNp;K_)cv~ZTfiPdh>u_+c%kW3-{cwVXH4S8C==ML7tE!wHAQYEBn4 z%I8OK;>Hp3?-iTmN(A*Wvn^8emlSLAOv|Th!61h_iLwU6>D7vtQRDlCwt`2u2whD;Qx zXn#6d4wpv;!LC`OS^Q#@ZTS&<=74Qji1qzx4Qe@d$QZ63WD2oRTf0-+P{gRIb~MGx z=7?RRQ7LW2 zrQx<2iepA*Ac>1pJXYGGIK^Gl{>$dn;+$QF%D!*irkT+(aw=lNWzAnHb3Um7$`bam z&HZfVgb@5imP2QVUP#?n{5-39r5IAn(TbhrT@)Um9@QqWc($lnNE}m2yiZ|X&s{+= zRV^E;96>(@HJ_4T%hF}IzPzXom&|;1*+SR{s&cw7d01{aoy96{2N3<{~=&}Cg|~Q zgii8=farWx^+07Tle0Q0=LEk)7hqtT`f9JHcy$6FcPhj!27lD(^(9oi1X~bv17Z%4PDk_S~(c-0}qibzpuU`p`S5W=3 zGaD_GUAhKE^B1+Aot`i0^Owgn)oCPQ@T_P*-XjxyMp;qVVN^ zumH3C=RdaRelRdIaqfK~kxCj;JAbC*p^G{-l47XajJ-7WaaU8{cc%EP}tW*zi=`iPB(cD9QKV8>y!_n z90c9N>-$aoHWB3g8}W6v*1+=8CMA7KpK-h%f~8JHt&|aZ#fVKhQJCy>dqKlI_NRKM zDJS#(ZVriE*vH%@~qkjx+NM zYy_bujfn*oOP*i1>L!wj6DD9Gx(vm&mZ2RH*coDTKp|HKDu8QHSGx_rCB`bqjDv)CAqv%X1JCDV09hJodl zm|0Da&)SdC3cBrFGG8FIMqq4(uwo)h+4?m+=j71l7tYWSjHDAJIJg)-r3YivXDk>{ z;H)O?zZgF6`p(Q>D~%NGp;IW>`FP<;{*SW01#`;WS{l?Q6`&s*JSl5ZbVE68568?! z5(T}-=c(pe6iqLcnkegABh;!M>l?wkqQxN|>$a4x8b(Eczh1aqvoMPxk{jMvfmF7< zmMd_cv9qB}T2Bx1p!9Mu$|-is;{w*MQ^E4B3I>rVA34szFT5|u@2yH<$%={p zaf~6?U%`*@atn>m9;g+%1Z1i174x08^yROQ_q&cK&SC~U6Yr70G9%Sw7%R$<|LB+e zt@uoYiG;wACSxgyxUA?_giBS4u2KE5(?Tz7Zsp_E$nKP?XQHHR>ex_+Q&E?%*htklhV%CNBgE*@ z8obrKG(-P?y`F@-cBbm7VQXroVIL8oNpT93Sjn~kd9(jlrz!NOyGQd={sw6TxTybb zRg=K8qT}aW>~;YL#iq#`mcrL{zrnvn;(UmkX=!HLQ zwu%SjBU|FO9eKR|t-Z}uN|t<7j3?qe6r&?<)xcP|tw${;>jb&yZ<`y_%#&qK3r*Bm zjaxXj1BZtw^UgfKW+`VSxV0ps?C3CUC_|B^`^>yi)##+=i61%&4~vah^KA2mmfVt` z5{4&hbQj8SE+TDWarcc67Na$CB!-K#zo2FdD7WI;U1_fI&}4Kdl@zM6!dlC$AG z-)vzOSbc=gm+E+g9B&{V15aLe+`-+jyI2^qsr;@qfQrta5^h}nY<1AnV&9Tv-_3~u zo{X6_#=&9SPkqM@m3<@5RbP@mbz$OJPOUg|XgtxX?Yu{^YMg>b+`5#JMyG_KS{#E% zjV50`a4ba&AOELtd&e8fBp~sLj>1bZgL}jscO~iJlh+I!DpbHV~EaD__q(X26|eRNMpGSbdFCjp7o?e@TQt2M*}%TS;aUPNn-RJ?TCtl}l5^ z*R7|p_C9NouH7QBb6MOjZMXQg09o4U9Ga3mJ{^r&yx&v20Wf@twQDN_3vjoz`-r!B` z{jG!3pqN~f$fcjx{iO=4_d|h4BhkQ5+7{JpPn=z3-~+AboaoICde%JU*4$~+fh!hK zIf(W-D9*KxL=2AIjDuH>GNnp|t9DkT44OxYt%Mzcv(gr>VIk2Y@!~%UuRr4D;Nr#= zvxSp;DK1DyIHgm=rQY}uj!pbmgNyxf%n^ZhAhp@r7U&xv^0BZ!5)X{kGldR%$15Dk z`f{!ny_xsk;r-F^3MANk>%}1#k_nzdk3k~rxt?7v(PoJ*63Q!=Lt$-EE}R2mB>}aQ z_lGV%6OJLBoKsKw<3l)AZC&>J=2up8={f!0J3>RrK3cKpFq7FGZ7L15i|h*+2=wKB zBH*LJZ#;j5lZXSl@fX2Wy*tV8*@8t9AS>k9=78fHS#zKH`lUQ zwTalaEOc0%c!-X=j_=8j=W{ICL33^A3NRv?;3>jcA%x5X0W^3#?30E0eEjR_HbZ;L z!>H_ayN{JGz*CXha;$X|L$f>Ez_n~Pk^$s3qH}1G@mT0=@@MH<<+l4mH|diK!*QH+ z^+Z>`{*_W-4d>49vWo@&DM4S&5RWQze zNM=X^5P<~B(WdKE45d?iqi^%lmUgfIa8Q5hTs5VO&5nZ zS&$LrrL!j^-K`M!Ug$3TfXr7GwwfR^7|3djej@&z+hS=&w_V02iqiQnLAo^b+b#Ye=quNxfpp z>F%PAjdE(CD8qd;+@eZj(>OUUGpANE(A+r`w{7Wk$-#hU(r#>A&8mF_5^j0OF&{vO zil51`)~XiVhacAJcR$r}-SHKhYo>A!#fNU{o$-MfV?OV?e^})Xh-AWVo>vzab)Q>A zgl`;FPZ+mo(09gE&+NF}(Y!Z)t1<(s6=&qe>%(Os{0Q zk3)abEXW@M@tEk)4&;m<22cCOzvn!<8_+RF`T zvz#U`TYshXhAP@g?r-YTcIm^RIc;d0mgQ}z<5D01`(b6~aP$a-c-)d8Fs*!!yL}WV z(e*CcGtLycH;Kb@ImXZ|Q2%LjJZx-p0K}J)HUbx_3e-0=J>wPLh`?O`W37K_4KF@l z9$9>@@J&3%PwWUWHS|_#l%f~Z<@!~}P^y~qZ*QYxlfzI zj(l9oX9FvMo0b1;q&YUsE8;qv$Kd(~rSq9ydWE%;j{X*N*b3~D4pSt_` zx}&r8mXwXsrU$(~4L*+vabhJE1#G-X2EO3DJd_xBSgkCCk`y1krDTUX=3f38A}u6y zy6JpkIbv2@-SJC3D!oiLpKteP-$M6nT$PGC>Fczb85TVO zSRgqC)q{7{P4${PN`O)vmoVhq2;n94%FAE?!RE>8GJLnM+V9P7SwTQB^yo<2k-X3G zn$3cvSww3`W3p1nRR_C59!vUtG0{}TP_8Q%;)euk;OF2KhA!M?H$a^x*}fJu9?anG z+_R)*2+t0djc_w{8K7>T8={rUMoMNr^rf&vJww1>b1EQPN*0beB0yH}v7-t{OTxFu z1mY%@XC-4p z-LJN(-*O8W)p}YfL17&YIU6qV7!9%O4w%12)K^q%4tDUjt&bB&+tYAMThV2b6NT;w zPFM$vj!GZ@{$!sy!vXLiBAt76ik*F+Qy$0F z3DjKDy2xiQX2=VGz1=zi%=MPYPc_ccPkLsnCltB`M48IwyvrD;?JAik_N2ymb*hL; z2N3SrYU>LNg&@t{E&)T_PE{Fq%;Wo2N0cB;JF(Hpk^LfREPGkmc{1f&#K$A8+nH=h z!gsRe^-ACIrdgPkuYRulk-=dyELJLBI1{x@6)N7T&fXU-N->lcV;Q7mNk1kL)b7&a z8?pCp3~JHT&YCpk8_|3@X3hC33B?l?ZAZg$i=QY`%_w^5k|4B$^DDr|;AHq0ExFy{ zw7U&R26i-~eci)R;A-iX(7A1WMR{18#`uNhlpDTdo5(a-Gn-^g(AGGJUIVesFzn+3 zGs{4DIJ`V-oKcfwT7KDGZojO&m;Q;0mRm7yyK{_kM1}jDhs^ZKkBx}$-ctA~;eHc4 z_p0@#3AUEtL5;}NnPlsrz>vM;Vosf*HtkBbVH$@|en@mQW>m$(y1_>vrR6l@YolJO zV7Sf55d?QYr$P<~-65SPkYE79eesJAoe&r;5ys_ z_~SXkpaWDIndC``iCY1KVscW_i_v0+?6oDEm~w*;9jYu*@geOY--qdMr)6aX1Ei}G zvS^umvW^G?sY@8ext8R`vPzz%(ghWrZkR7L?ENr;DzO$b={p!!gj#Ri^@c*B_&ygZ zl3wT{XtLLxMg@1x(?qwTpAXfJ=2=GE{Q()TkxbGglYOBJL#BtSMaC<1X@%CjP+!z+0GHxGy6JX#~-Jm}gZm?D5O2%jKje}}K_eAU)8WFTf-T99~4E|0r#P_YGXVQJC)7#v#sh$cR9BrSzHa5@;XFMN zT4o~Csl8*(7+PeKQL)R{Jpg~0Q>PA#SIB9OlpQM7Xd`lHJIQhfc-0QglhE|JB;nM@ zob)`R&U<%xFE~F=0E_aAZ%&bV>EjOK`egn`B0C%N=j7yptIQ}|p7PzEJOO`BD&axc z@ekEZ9b_k z2+?CYNU?i!rW;=t^rNN9yJ-Les!g~O1X?oHpR)gcDqisrq23BmTpuscj z&e;n|dJynK37ODO+eJ#NlXDcA0hNnpYl{8?;)2k~=YsF@6qb^#Db5NlCN@yT_ER0& zPw=aW?AkC{z+mIQ{1C4!9B?RjtR%wV6B}kA#&+J&a*vmHy~r>p*Nt(w%b!7_D1AW< zK4o*t>Rj$RPd*$uHL`G!hW>vO(l;I25Y-bUIUsIH&=?Y}zG?U2=o$-{w*A*idB7Sv z0b;I8UYIxEoQm=Yr(!zaDDT9(Ty-B(@|6zZ-w1YB9F@mx%4t44J=!BrN;tJC|N2M# zJ2bp^{ViDMlZSwXM(T5HB+E9UhvDxt#4*qTj`m(>`2+9?7-a^u|Mm|~NYG^xdvVX; z5Rs72^Iw5%!J-oD+n4`uU~s~4+RE)lHT{k-+~G~Vih&H}he?Jyo_Pvo3+Lt+LJCs`NER*t0(8&cc~xsuHtB1N%T#Se(AiT zFFZ{J&<2)#1e1A0V^ddRLfpnmx=t>=d3bs=-XwntTj7WKg;Nuv<^<x0aX@&jPf=2D#N`p&~?Z$^h%MY|BLq6uez2B4NU`tQrhjCAy z>MauGQ#62Z7}0a&UX8C=w*RrD2KI`)>*sr6uQgz=hqSYRzDtNY`k%daE`E?pg-pW_ z#arn38{Bx>fv6IjWzQuMBa6O-TtOyG0vX!9XK-S8nQbY_Vy5l#G}i+(*v>p))fOCa zD!}xMpRNSJW7qd)fHsMx0t|b93sX8!PLdNJ9-=Q={GW?XqGGp1&2bFUNr0T;8So5Pt~7Uj&U_y)QGKylpbZPI z;0%}d-h3w5y4dJ0*wqZ+F93Eqe#qM@{`aF7qK*VAN!J~LM)SRR`A3;P*na%1hwU;B zcQUTZrPOku?w?*hEU`ci_4^_J#|_8i)EA;=#5jfrGOG3cXg`p6i=?O3E^;ezD|=ls zk^}XD9uDL1%+F@o$o?%kw^P89^$5oefXKvq^=C=EpI@KEYtYUq3WXq>vMYeevzGcA z_a$g-Q0`-0 zG?R^`e|22nDVZlMOFbj`jq&x67q&<0W0IZFd})_zXYY zPpcc}T>Nmw1XRc;C<7K!F6pIiZNIK1EdBE^dOP3a>nVb4olp|~1(opTIe1G%eRb4m zUE1-Fpe)2;qs;w=mr-#{Odm;( z=8Yt>uFgELu7@X#$z2{4H~Df)vkD!hdgvNF@|P(8vJx2B1+>8TFzeUE0iU#F|B;*d zy~F9189XN)YWYSw%`vyNPN0H@77nW{wA3_?tAq0ta^iS$;&Q-+*v%JsP(QV4C`E66XfD>y1%Fk1LgsmJ^8@ z2R%2^#DP)d9x9C392<0oS8EIxo1i32l`|vE+;f6FnK8(A=8xl;OAr*AEqbY&gG?b& zALCf|K>79=F>(vosMIdt~E+ z4~&2Lce+sU7MmvX6c+DP+qQNW-~7p_TT3F}*~6*T1%2Fqe*U^N%WUjG)ugXXzw4yX zS^{l8 ztTB3(zB>DxKWTKaSbcAV|GG@%<%3VNR%Co=e)y)*ob{n^fguup0OlJ25RxkQG2-kFxtaKq+n~4~0eqS)%l<`k1{{$q z=*RQFI3nBA9PP^1s0dGxAMWtsoxNj21FQ-HKjL}rf8d{`%YBYRd~%6>nr*SOJf+Qx z0~31rT>R}7uo(Ghz|m4#PbRl>Iu7rl(;KdSTnw@=tr4D|h|;+4z7SB#c{HbZbP34p zH=RdJ7Zv+GG$8M$G7ma#0KFY(7Yni7jV6N(Al$dn*!<$jq-U_s~dv#!Aoth^}peL>T*~Vx^P=D^^u-c|H;iq_o6yIg4 zi>iSJNRS3f^D0_3xy5tL!1&@vOS1r9xrm&cg+p;Ve{ zX8j7kcYW2km*k!La4XO%%5PA?L;>+c-4}E{&Rh`CVkc2j8`xmkw14c1lc(0rpEVtQ z5k-uIruTvD*cm!~dPoS6^)&P}H~t3;5LYMxWl!xdf`Q3mn6jh8ZA|Q}fo(ZL09D?3 z8R+8R34Gh)Pq@REVzle#Z^UrU^_Tq0N2ORde~NmMtq)==i>ABG7Y^YO5kQ9)qNV%x zyMJ_4i_g+e3;D*@nby|rufVXvK6?D)3z`h&Ai~nim2#ZnPhWatPA5jsI~*0|G=ru}?R^0{;#r7}R;8`x~&PbZID@M4;+`@T!cfFQTLJtXgAaxl?X}4)CRa z;rza5{~1vddUc(E*TgVY(|lWsI5jl2om~8P)5rgJ8womC`U>SlDp9}Um`?IvSh-9= z+VXgn!o4*g?#^Z^#=S)qtd)X zu0u$Oaco-Zk|}P3eu%C|BfK+&~(4O zA3Rv>a&pxr3`i{A5c>F8=NBgma*~q1zeO6);6~Qh54Bv3mFPJ6zY%69aiv%B9AuVyMrDkYL}y()lS_VT%^0YN-E z>#7dA4q8Up%U^9^SIun2W0by_p{-`cgf4e83#aa^a}izG`#30l&keeP@i1WT`hUJe zXns0#S|BbLalhSSXD9G6VVD(ZY2rl*L>hyJ&pEB*)Mq5HWREgSZ@RQilHdLjHe8eQ z(`ceH4C#ktGtQ|^L7@?5HOD5-n)9^$vJEn$qih?IlAPorFFfOkf-auK+(+;P-J&Em z&~Gs9#UD@C>8%e==tL~^IKf-thFn)?iYJKNhwq5^UAvk6Bgq1cvDR~x^$5cmjz=rT zxpt-GvC)x~uc@R1D-xX+E78mYS;~a6*qFez#rsY&62iWgq{O~E^x;e}DtmON`U+x@ zDX4PlXuvT-E0gp7`0XDGK=;3puDF=xM4;a|ogj%AutxHj4LuzlaLbI#iSm?U?XV3Q_($YMeukWlT!}wB{6wv|VZ_X% zVsu=w)S(pla9~9mfa$}UH{1s8#^$|e=jXPASZmMUG_EB@pjKo+BQ?i)qA$8tPgBUR`SesUJB4fC#_p_LQ5# zD;QffPQ<%Imvb^WQXX_|3+vV?onM*|+UiFDrSO_Rbx2XKPQO7_G z1i=o7zM2m;Lb(NU#P{sC*G=}ny+_9zUg1|uLu27gw~uZ3SygazRe-OdW-6Q2J>!BR z!_>nAskQWbP@T7kdg~4c=Ed@D<~U17nu>}BXS~W-Ohxs?)5pS=i?QH~oR;%k{37UVC|~<4*R7ftFtean zCd=0V5k!?>TF~?1_wfo`3f)KKf%>A=!_;31a+dOsY!uJM%ILPw4&PiJggF|0ju_qO zxcpF{cUE_K zoYr$_c^uYDPfrF+2mWcQUu7&0%PIWbgZ7kL<`*ePq6ooTv0kA6TOO!`ozam3=N{bG z5?R@f;9Q7|s8-DR+1ZSXW1v-6Q+6m@tBp#O4a(9H9KgQ?@}TsG*61%$&98Xd5;c23 zc{j)L{$A4&qEZbq7s~6J^U3og<|MmU29&Amm1R?s2I(kX?8(W)ZCh}*+fAAn7gS#X zusd|TU35B)%J{vg^`l%;6Oc$Cz&IAsk3e@P;_W>FFJ`+C2EpgdXV>DY*^$@w+~uy6 zy{yDju-Z9rNms*MCc(JPBjc;5|K3caV|av|;RwT8|b z*EsvC%!yX$6V<=DAucE6!IgV1S=}hiXi%>(tH8%l=?}mfk({fxEAFkJ5J!0QLy~L^ zmAujV`5RChK8MLJ92x`3^o`ayw@W+;)PEAX;vI8~JM3c{u^wxJU!TQXeZkof+5=IN zVK+}Q13PFdAuk0Y8XCArjeX;dSl!vQ6UZ`cTx(;!PV4U_7=c-rX3+>za!9n&iD+|YMe zH{K|Fk}Q8qY1++RRgIfnj9^^$=HEmhwJ-)3JDe06=~#1iJ_yy|53@&kjE8KpQmmb& zZQSl)?i~S*TAayZ7vD_)csJu^kZO8n6h+JpKOav*4DF=jI>|SW(Mh5xfw3zwJ3J1@ z+?(tpUzqw>h0cZ1hqn|a7-35GTdVof3Mr*NkNLH;8L`Q^JVwJ460cAeyK^x$Q;zl# zd5_VZ34c zsNy8*RKe_+lnMoPb885AOqSw=7w!oR!xe^bF5Kt9^M(e>;4}!=`Cnp2!F6f40EYRg zU@gqdSU9GJ4Rs!7L26K-M_Yf*-JW}r9o*Uu?5(H5kplt*kFA%7YXVarf|L<&G6Up! z4;`Gh>q4B4Q3@omoSo`iwVf?YW3WnZp zl!h3Cu=BzQ;JsGgaNDtJy$7t6YF$pJxlS|D4;9Z9KwKml1Rhx|8SDZflBX+(0?A{D z{_BohOD4Rb=ZGxz5O*UV8FLYmK;G&y#=0ZK(Mc!)*GRFqr^1RKI@Otu`)UA9I3dX- zWDoI>y$U*twf~rPmx@oc-A;ZQ-i=Y8$73s)A*{Cj&6|7ClP?i_F9 z#c0OdyY{9}5y8*%7QYm3ivW%Ji*Gc~V1(T_;``yRaa$6c7s8&P%oCYx{OR)wG^`sS z9NGf7(Ek6w-SPy`uF{(C4BF=Zmiw@TgLITZX((ztt^fAg;q2PU9bI@}oQ}p}y>e}j z>_`Gr?fLaLjyPrX(*&un23iNyK6I6}DLbMzEl7`sbW9$Ayb0c4s{!{{2gnFFM+o}* zf}aq;09DV{Vw^|Y9S!r($qckL9e{dZ{<9e6GpfCn0S1m(FiNtGbIR=@qecmv9-jJZ zpaHJBvfwc;%@MGrBhE(1cphg;q|ps;hi!~xbuYJ-+f%L(v-2 zH2R`@-6y9mSay4^R7y}>9>cTNS-guU&xlrPU!cEuu#xg_+k6_(?1F$7A`Bp(v?iTq z32epCb8L%@d!HZeZ!-In+lUWt7NCIurB+rsm4GWJT`4pC5DrG&7yzC)6d1c*4mueM z1egePs{yl%JU~4z_k0x@%#B;Q?EE=4L-EJ#*0ZHK&W}YR(F+D`*5u)M&unfw? z6o+%;V;#*{yp#8!{;}F^HERPw=ELtJYQEa`-Nyu@JL1nh($msf5E4qMY?ruc(me%K zCOyRC&#N_B29Rk>-*xW%ecd-&NWcaIJ0A%{nrR=2G#moP;rXS+pl~q7kYSk^5X0UF zG++{bFN4c8RD7U-?I9P;?&R*OJpmQRHR}LSOJDqBh_(m=7GZ)57CSY3oKdA+#|N*+0o>eGjH4i4w4H|%~`{r3n3 z(rg6qkorVlzRn<=7oxTRHD~0c>(0`BB(EdK*ms#&KDDM#_a1_g@a%DD$}ALMBD@xm z;8qHHjda`(lTQHOi+r_pi~TdbDPBT{Z61nTD6A<;c7tytSk@$wh(7_60L|!Oj?D_0*wJ&a2O> zo+s3Wob6lP2MxTwY-8gWs)~n<=;w48Alb9rEqERZsHIxNH7u7kUX@~x0umsi*itZw zb_B3%Dy(dydA>BFEDBm_=_ul|efmX9;e8fM=n1zr5d-UYch1;5?3R-w+5o%hEyqus z=1F>K=f}0@_!dh%Ru!E+fD4b29k3M-Srx=9WMCbDA&Uluog5(noReGw3LVxiv9XDZ zw7@v;H%9GDQ!}F)?g=+m8}Kd}U`g+(@K$i2<8y?ax)k(QDo4HFxkOAf9sAjr~WQ(^2+(OZkFRZG5`F$RdK91uOB(iI?u>@j{ZN9dV+fSA_}u-wRJtFwc*G9E&U~ztNS3d6N2$`mf;AA$IEf) zWJK^ds*(lYQm`4ninm^F-CpPn`N&+uWy#BEuk1*mvAkIfFpWqsz%`|!AF$TZlW4vX z9(O`hmEGfd6ThsnsCj>*4R}ARoom>*Qo8+003p^GcEUnO6wxrv+V$ZIrby#R;nWxRX=^cPT{#)A zo9eln9TzcAnj5C-I#P52#YoIsAW$eE<_w3j4<~ux4~H`$*u%7eI(9_+dPw+vt@2l; zg%6h$#bvqo|87tdaz_^d&E2&?12ur}p^D8^oR5}Mi9VQ=&4XbuNoCpd zUen`rcVQn;<)w`jbd6OBrvbt}jV|s*L{W1xL}07sq0OjCD$V-r!E6xcc81`N^c_3X z!K|o!z!VlYZr3N1U&AJy0w2rXYtw##B@bAWT%VLB+rFvCg+w;6b)+mpObPjSf|nf| zKM>1JtJlX6k3r-JD`-zgzJ>KUUqQ4uT}8+$+ydRj6YxW z&>gUf{J;5dDmqE7STEHv(7f}s@vOhu%Bx+UENub8r0*F|0S#Mq`T5y?EvnoWrdDhk zW?a`wX34XfRVYmp2dLJR(3EAQ?|SzDfmMvq7Y+h>3ipGVg?%vmIqw>kz_@ws9-wWH z{xzHz01_3}pc^og-lQ+rEM6k>g-N52Sx$MAn-+tZmeY){i6HLkP%fu1lLe0eMeP-z z{^uT6KEkB8P0=JUt0$-XQz@e%%B%%SU&EPonC!tsSXXoP%jtj?G5?qf@PMaVCXZb; zEoLCNZQ{BR1$%)d>+TIzHgzpQ-vqQo#O%G?{X&u1@?ULLi%pfl*jc$zrYN4z@d}LvM-RSnWN#>y zOzh8TEl9Cv3A8U`NVVU-{G3yX?|^gjmV=e85FuHd>dW9Gz(%BfYs7sc4R8 z)IIV$UL|v&PhxCLc*c6j+xC2rmCExov z+)oaM_h8u?rGUg~2k@Tz;u~&vat0w2b%wgpu+}5kW^oY@N|Z$FHrXc7P!<_@&hFET z$L}KrJ!(dvGGl;PQuMe;#)k*ZLy2o|($j94yjcBi@}uQ;i9zd^;fUhr;mV#oYk3WB zLCvn-ekeBar0g{gXHthz!0*`sBV{4uwn@qb)P4;cWwX}{m5_cxRbEf3Hv#EiONgw1 z$$|)$24FMH$ISz??bpXjImd`shAb+7d|;SsBC}%i*zsG~09#izWQaI7bQA#n12c~{ z3&S3#CB?Z0$vRl_5EhqKd?`KpvKN+xslYRozC=Cs&jXB&~h?G>0J@&XXP&_i^*^%E&9r}f{)xUbTS+O zTzdR&k^E;%dlinQq^~jhf!s}RO^cSLOta5IAIFDu&HikK#6><0;W)jYb|o{K&E(eV z3Q#^Ffc&Bz+ofveiEG{X|Hao^hgF%i?ZdL&Ac6=Aill^ybPCcS(nv^2gOoJVT?&X& z(!Eh)3kWD(n^NiSMx-05%_hHfJMX;DJkI<3{uqbjn2~j_Yh4}ZRpm5Seo|sMvZ_Gd ziFdte9I#B6tFI2$8d$#0;pRoII8e0ae*aaRl@z1TtO{@|2N3B<+QMR6fZLrTAIpad zj%WlQo)OlaR3UP9ICqyO`!P%K^N(HKRZ~qp3kg)dFO}jner+EMYYxv0k|MbDhl>n! z7COs0FUs8ruUc0KEcQFI8gT1eu}yY<6O?d5Zo1<>y!1pLdZ48fFwv*Wv+tqos+c#M z{m42xn5V-o0W);Br1L%SSCiVz!s?*TlvZ32VsWd*xn1m9+E@oI7*liY1b>EeDp0c* zWr`=MVU{)aovG=^x*y&=I@*_^_V}jGmZw#eewo3t1dM5nuvYL-&P}Cv``TR;EDwyu zF}o@-Pqh0BC@-S?jQKQCFBF*O5Rv{Ws>9H3XOA=-u#(h~V+u3Re>_%K+B5*d3QK10 z)8m=*c;HiUubA=TW}{1iL#7R&n{%U0EA}Y>((?1TOy_YhVGKD_JV+3QDDZyP)K<8^ zT>9)gUddD{A3?>+O=q(dFiPUpm!mrmybQ}15{zlY6Com66|{rmv3vy%W*1FW^KU&# za*7pIKHx}<@zKwZ8eK#iiWv$mz?`g>Ze};ikC3l9t%Q9R^)sggAT}54KJ}b`QV%Wf z98^}z6+{EqUr`3&IO$?Qham3!lZIgK(I)nU5|zpkB@*}kbAt#+`el-ZvHu4U%>3x? zkDqV%AjnArU(7;mR;hfyve>CU@M*LYP62b!#7b0bc0F=##Qf%az5t)o~&R6;$$)uPw^!SU2hG1F|@q zlRtuF=NUA*$PcDSM?Xf5EU6FwFkQ1^p3QDYbo)hsQj5%6lWXD`DsUSOMwDSfe)Y|E zZ<19XPlV+C*N)pjb-x$=KFR;5>c&9h*7*IQXv?cstL9^WVK9xFVoq);3W(-iu8ds~L4U9}Knt zko6%OEPjt2Q#HBva^KUVw#WQargdjPjynL=n8}~W4wgQ7U67tV6a!d^aYk~GMhiH_ zT^{_C4lkdN<`u_V^{xOd%0tU)3l?)!Qv&qP8?FU6ig=wDrm8m+41JljR+tR~l|WtU zW0)W#z@`CRe=+ARf%^h3Ji|-hG(j>`OfpHh9~8J^&`LAFW{FDyzwal;K_bJ(g^@~) zsp-`A&4~zy%_QN1iD<49_d3Tv^;&pm1whVIE*pDv|R(Hxeb} z0YSMmq-z`jxN986R|ui>;UhB^!IhhK)n((g9=n|aM;M^!tbNY`_q60uXA6asKRN;) z$D@4hF+BNlj;Uy+uZo@`LhMuQTL5mC&m@`0LaC`^d{zDEeeegXk-`lxci{7BeF}`V z%S1QfbHuJe8?txCt&5U~M(>a4nZR@)JJeF3SM>yp<(c^w-Ke<9rgMkyQ<-@eInT&= zwc8dudeTX`s&2?PcVL~8>N4} z>X$Zq1w_*z{XcCbcr?RYs7}|FeR-riO?(g{o+EnQ!R%krHm(*Zi&UlucsYR>e~Qlz zB%96gAJ`ADwL-o@lW?`mB#2)-Ta0niAh>8CD4eJ1Rt2@s{_X_;yBEMVk;6)^Ull#Yk3pGV0_%4y`O zhps8)GwB0t<3($7QF2Zr|L5QD+J=_T4%m`Kh#o!n`1PIiv3t86c4GGh7cYS+bEOFY zZN(1Zf`6j8pRZd{h$)HK0`QK?0?S!TqZ`;RVbxOx0Ayh<*d6#`?Etx!+zkO{;w)}{ z6$$|vMFn6a>Wg*X7KSc0_DGUF4q?RO`Qt@E`0Mjy5vqhFmUKRNEDw+4S zGoU~R@Hoxu(pSsv+H#($osYEmEau8uVu*1XqLTo}b5_{MkUxoDf(h;*FgKf&NZayYMVo z_vVAXf}1;@AkEHQ}`(Y*UO*)&3de|Zlqbx2{`DTOjc#o)SG3ekPL?(e9> z65c4CodT1~XX0b-Q?87=R&u@{!~-SQ65yb1pwq8I>O~D_0wtM3CCu!iGI{gBIlF7w z&MgU~!xCcp-$^((U~nBR5ZmqSKpAHg)$Y8Z1Yy7zuG=8iE?$V&ePhRjG`?x8liRAP z)GrnSM#(m2T|t08x?dU{b%p_32f|BAC6GV^e79-K!D6!oi#{T-OH1+MG+lK-%#Km4 zNE|R3u++(xV{v-^Q9k9MWXAV6)NTnWc38RrG8ORqz7^6q^a4_Jc|w z!f3(w>hl2_Vfg15mXf9ipTyN+K%Cvpf%o#?$rGB zIee!ox{-D;kwEjOY~Y_wE7GQxIaLtrXb+4^eoGZ3C>O zxpq8)tiE9k5P7?pbR;YRT62HBpPw35M>$vm^>uIUH2G9fg!e=f0LPF10*goD=OTo< zPcR_q83$A&ot^A7o*o#l{{pQIIq(4g*E~?p5vSS}hfW2*=tM@tKrix4!_F z`PNEW0R(yf`P=wL9{m{|-j@|hBaqQ)hq>YliAbF5b`RwVFwGEMT1@j_3t$XX@<6w~ zko+T*!X<%bYg%%q7cyd=Sc)%!JtEPV-KCDB{k5)m>*|Fhf_BrFY8Tmc?#t$L+LjGz z0{cY2T!%@HMHXxj(1I|O+kw5c$`e|7KwJjU-6?v2L!`b} z2U-O+bBnPuIa`W!^(;Rs=0CNJQY^~U$ZC3C?UT_RGVK*keqWI@58&<^J5!Ccw8yb@ zX$Fx~cf3;V#R6cdcV&B-5^t|LefRwYinT|tKr2Lfw*w#WFQTlmC|v()X98XHKdgQL z9MiYT_$GhGA{PRjK~_9FGrrMDk5JHTd-$Ro74Nb$NP={Losr(*k}nmhgk1MrB4%E? zq@XSn-#rxRQt$7Ny1@q3K?1eGM=UPwfHcpqlCMXpr^vW{0npZY4e)PktiS%iSQEQi z(;h{%H5u zs;ci+(m$)^#IEowVrfCb)s(7n^luXAK@A|;zJb=n&hmZ_1M6g!&)ZW&#dATi>ZQnK za7|2=H{h&}617_5>yRPpZSHqQQ@^d1>IJ3C*!c`<0V%wqTXcV(6j5Z6piD99R& z8exS>5;>?tmD0PxQ2ni2`a-}wC^rjJLC@feH@ct3w!Y)!yL(z#Oyy3u9J{4Vg8sAp zUwjsH@;llTUjDxdi4OqwP&EE$(OqZgnK4Wmx&dIFS5gZ7dMv_857#ZOpaE2a9=a!2 z{;TGSCkfhuO?DN5BmA!dUJFQsMCtkOCn2E*qE<>hME_-Z8n8k6fxR=(;tCY;{i~)6HU76@KSg*5g1-r%$v+!25(EKi z&!f8>-8?ulBp4|?i}|fVUzBwkaz^`%sU|dR7LakkSbbCgenuDqQ6bs@YebE)5}*GmPtAzORfRb^b2b| z%9o&I#4VUB>NWuTB_SmxZGP;O;K4zp3FKaFT8w#;=FgY|8+1pseaZbltIl&^yhoP3 z;iz-qchhFDKhDFynwo>SmV%rUj=Xz7=YsG=IcBUMgfz4wZ2!+{1jE0Jd)T{w{Q3yc z+uxM5_ONc?kw!W*J{qL-=Na$1p{K3g_hw^qsQt zWXZ2=+1HnT+_UC|JF+&QcPW)GB*OeEU}4kAWs!zZfM^8}U z$^ZrO)K~t*zmF&QKaN)h6I`=h9<(zrn13_D$t%;P>BnUu;aQ_zWtv_a&(5A!47)Cl zcnaC2q%}I+fLz&Uu#P%?6Y1KMl-p8%^(QI)Vl6tBL9c1y>2!Qwu2Hhj^w2-Byh4Tr zPrdfz*uxbGuvc7BEU0nN0n`RMf#iSV2Fu)El5c!)0ZRtxF*1<3rv)SATdf~|hWvtR zf226BwF7Oxs;Hfx`MBAxYs1lf-4XqC% zN){g@{lGUlXjOFdUjJnTBufuTdK0osmi)+Wdydmc+HI++SF_zC2q3~f^rfb|s}H4Q zs3DSVz5tyN0u!a4)I2s}Vbpxhv11(U5yvtrmisMofUZm?Z$N!5yPZ9ky_7VI)AXkn0K4k z>r890Jw?4>XZWc8ye@iGNg(mHaP6I$o?VA0RSckQMbm$4X;$emSSSPL7TFC8b?87Q zDmr;+H7K9#q5XkWwQCt*2qI5t=%qE-|7`84Pg3DfYH41K^#*`OFW&t4YUX3R#co>)s#7@;%t{^tU6UkW0|x7l0vy^`X_V=# z=W5G413)v-`okAn1r=h6*sydLlx4d7tYxe>Z`nFvDfi~eK6)dSrJrPQSp?gn`f0S3dQ_kpj(@3Gmu;=93R~BH3Ulo1K8ej`cP> zg6*wlXMB0MkbFvWi=cqgumvMBNP?C~4zyUvDp9AlWbK7*s!LAB3 zNo9CCWvUoy@HYijQu6??1^WW*9CzC>7D-D5_#2toEb!M$9{Q%&u|fU1+`c*d>w{ni z($i5nKV``a0h74TgQCcI%o7V3Y@I~rv*uE(4&^X*8;&b~4L3aCcaPJ%&kP`52*6j} zrp3|t0p1Y%?1IBF^oWF5O-iDlAMo14qv2`mrwn+)D;BTPcNQ<>y3(VhV@qQQo_s*5 zv@sYtcLi35c2rf+|0XBR<^!zh8xy4;mi!zqB~!+lQ2|| zTG^^TX8LK5neEPaVnwQ>prM*wu49#JX8!2my`iiFP3zDPvFZ%O+KcB$=}AkANkVR$ zC1A~iM|#h$=XX}LN4Hb=ajI8j)sk2Jk_@BbJ2U-`Pu=tLWpm|7cCMLm%H?lhGnQ%# zAS{uzKWNUCPyLM&uiJn)jbv@dP~&@8`iN)5AOcvGJ;+F|jeD$rr;6k-sJ2;8>|1M3 z?4zgGYs&?sy8s)ynJ9w3pCp2IaY|Wfxqr;VuX#KiyOnrhx5|V{+65R|Em-GLUyaa6 zfD$oZ8f$(LFI}|`Rbj6`vZK3Za#Fi~3Fx{bTpJ-f-7&k#^2J3ydySD5VJE1w?^r-$$#E$j>2xrGFxqv}6mf%7}}C9aMYnD*h#i@FZjHCS4$xhIzL zQHfaD1MPG@YFw_>P1_ed-d)CbQ-~pB@9*yes}K>ESw!&|W+~_(Z_m_vv%rq?GF0ZZ+Qct(cSa2=jrA^`WL7j@e5wLr(2MwzmBYk8QLQ zI^X3()c_q_66o91*ecViFYQ*TO-Y%7CtU(hI(tc+{bE{d%LKJ$iA!_q;@6f}q;rb! z>$pVAu@@p0Ou*IvFv=!wfG}~nPfF|WZvtbl6BUwH1S|o@Ut!seNq}m||#EJ#b zs9UHaKxz6ww!IWxE)z2FlNEj)k5U*z%muo0A{$L2LWS8422|(BO5BHZ`S&1 z42sc~4CBfEJInA9bwQiP%HHmnDyheIi*^-y>b)Bvqsq_hNU@wHY)WvMU-kiu2o@}@ zza%Tm2#Z6vwHg6fnVkRrA?$7UkNvR^-5H?$H>%pdqo^m>M1=-$r_uWhjY>CN@igRF zRlbWe`b`~m>|`yKzL9k!p$#F1iB*3-?JF!S{6#*W)u=@ITmBC4-SIOC2ncxCManG$ zPw(dh58X5pEh^_+fK=5AJ|`lCr^_XSSX;7*;Ci>bvcEWTkCln(X~V)_)Zl=qQ3GSI z7p=)x>Vn?pIjo?dpufNW8GCyPra&nS`jYS)+GJkcm)RGVs!63jp_+q2?JMtFHY614JgM!n~tZ zgFpUag&Rvs%m}vuf@}fMK`MZ?R@CB);d5d@2o)uNBgST_c(@hi@;q|4Sa@Y?&ugn! zoUv;s>ym_WJON*M)t66_AFo1&P{M!1j?T#JM165L%#E`PNT4FP_9Zv_&5LO#+LC;_ z7GXA`-in{eEog|;!2J^)^lr*EOk9`a5Og@6Gn>!Lq|>+WXsZ>S=qDZoky8Kl*7h|g z9qL#}Csxt``Nc;)qWX$E$OYsV<)}NcGH1|BjHPTWSc0>pC{SM@MV`fJzE{m{zSl}V z`A!#7i?~?~4NcE=qBj%1dT;vP?uP+~r|812hr`U0Xs=x8$=j8`mEqF}9-`N@fI?Sz z*+cLAFnok~VC9y?X;mRRGU3Wy)Kk8XyARs2Dq-4QJ`rmTE|3m;3)|S(=%1Xl$(tS= zG%f!!P(+DadB;c&XzVcPd3s~EZowcf@r6{z%^1ekR|s_erypSq?U&D($@7Bd)F= zth5}QDBI-dD$X4d8;M%j(DZ8gV-TvdZ=jngUt1nDnPs_S=1`n)pT=m?;+t^fge zmhnRFPJoPI?c=l4KH;{9Q86P91_*irEG3|w6tIgAk8GIk4?*dLJi3fVrQ|8O`qe(b zf3WliL=I97P2!AVRM4B>xhc)IMwF*^BfzX9zRmb}lbJ_|ntS24_aT>KJK0hKYJ^8w zg#CD3bXvH4)ylf{A`c=#xv11pzsS&Ilv(4&dlfYH(_1d}gacMsyB<2xu`vu-i|4HS z=H(Mj-qe%za6wF1q1TD9$HuAFD z-`7g$sif|MbjBy~ctgaBBNSfP!;;6^@Ll^qDcRcEdeCQ}BJiG4FVAZ+EVl)6@ht?F zvlle?9lKm|b)bpT2=GJg2d@hnD_{=&ZYSNHcDJbUw(N{RCD%C2!Z6L=&6}fXD_^EZEXoX@YdHoe*rgidCHd-=6k<8f1Dhd?q*5-qh`lP}$B6PXx+p-) z1N3uTO|&@jI)Z0Qpy@>VWf~uaXt3=%_wExT8A*1rEyt?$+Kz<~waDnq9BMO}?2kda zuxhG){9W@DMI}$l%F|!mjzg;575SxDKq?CrEQw%Aq^%vG@mwex2MQb*z+L5d1Xmh0 zU9S*o!71^acVtngwD(<-{fb&QOReY%=zjWa$@4!@Q@$ykw!+Mu9N3UnaVD*l=orPc z;zZ9Mq92=1x9Z`Z4K&h_4D zqa))seV%o!wL5Q8S06d)hAvpVd1jPTFT9)OZ9Vd05xr*nKz`GES=nq&e`_t@QgX}= z?=$DBd40CN_2O=+)D(Lprr!R12)e6Zh0L8YLVDnmXord;eZ8rDFw`sV7`xqN^R=*d zT_KT|d8Qv+3U~v93v!9~lWcQ*P^K)aQM^^AoBUl@_hwPguq(k0pb4o1YU@OKExk7*aGS2F>ecRN_f(@%)H`cJ>3sub(iRfmrg%3`6++OA(wWme= z3u*`hx#qBxdxn@J18$7a)L2;UTwBZSIG*#ZTb7i)_xLA&z1BPLSlMY1spxQWI=A7b z_OY!%Eo5w+%WCS|5>)M-@GMN%*fH6y<>*$YsUIfUhCUe(S|k_z zET6VZ^)^coc}(T&ux;vercqy|JFMgNiqf}%+FbX6NOy{FnoTig*>k=5Hfc=Cb2uu<7!H$cisArE$kZlGejZjxfFyWShj+E58Nfi zc9)0+*x7gI@?sOyEJ#(8zlohS6xRwD#uo5E^{nyq0N|9*kVi`46(k4+=0l`3HER3e zRg&BRVc3EguE4k3&f!1~Q? zN%RbZ%s!mjl5=ofo63iMz#lO*@>GEuGbiFhJwTH;BX$@YrstRW)8vd>fur|FF28}- z(8*C`i9ysp-{@0npLt)*wC~jI<%wr|qzl8x)Ewq-tCu2D<|}Vg8c`n)`-ZeDvsZH8 z@NwzCz#>wrM9jEB$FJv4lqM>LI#S{CS5Z!=H<~(`bgWk6ndCipNJ9D zBOU%Ni4FzxtXkNKJuGO-03|H?8t42wuyj2irq7wvaKW)Y!ouMX&RF^r?^yqu84egnn_yPQ`C|KYw1ZV3PocD0(wMiB_hB{bNPFb3*MILaXxz z`F82JJiY0DTu8?%i_7_!)&TJ+5CBL#6gFVq-in-#^*eVdg|Rj zj~M0XH%GFI9yD|=%?DZ{la5wj3{3HuuGv;7YaILCJ{_?!UK*0YueWd38*`ZJdN>7` zG9>Q~$g$3x7w7s)FosKc~%bWUBqJl6}st# zvj*fsJ^ck{I{QUsMlkWFPgxUCZL~>6KAP2nZ_jrACZbzu|-Yd$cp3FQKv~Lw0loazVxVgBr-L-bJ zmvNitVzgkps`qyEW0oDy|cg{6&t39}`I&DXPACIOXyoDIL;XyS-UlnVrMVUcPG=1knA~$AiX47>jlW zu>s_>8S#%ui=J4YdagdW51m_;+i74-KUif0qeu18ZTHx*pA;Z8os-> zR(N}rZ@*4dok4(!6vi61_#x#YMVmLZY-}5G;(4Z8d;R(G#z4R8+(3Wr3}1ik7M}!i_W?{={AG%Cpeg=6Q$13cdPxu?q?mbs4_8Q<6O?8Z*P zDdF0U<-G*0rH-igh*gbX(q-r7s)P>FOVrbAU;4$7{D$Wt>!nl8i>Yr5D^*=jn=6h7 zm%cA8wo=EG^VM^ZF3%f&9&+;KI~~~xx8J;7t#7RC{pnOfz!{<6xsv?OD(%|Z&5Aam zs(oRB`Pa9#So45D+A=OJNCJWM-g8o6N$Pp|su=SC2&C_S1hVkAKync{TlEpDmzI{w zzR&QRZ~}KMgV*wtrX(k4foM~w;=LYV5fB0V?;mgB$tfvP=hTCe;2*T%9?KOAhI?KT zJe{lOHf`-S7?HXRAnOfxO;1Q=lGk*5suT}ImS2myJoj5R5IFG{>D=nT7EI$@psCs& zo*^kZo@^zUFAPEO>5gT)cpo+%7k3=??VNdSCDU#Btrv0I#PzgyoA5K)CAe*6I+f@?zWdJFLt9t)*sl+mj0MAT1>TOVgto8=Q~> zF;eO}AM<}N#JqF!Uf1TM2DN8T@Nk)aOh~J~blJNYNg0P-o53iOU(dG5owfq}QyXC^ z(W5QzG4sK^9Ily3G?4(XVMn|+zkVvugQaKu^)p6uk4?9K(B|R@f{83eK7FLu+p3FK zPv58d9()I)k&Z5foih|UgBLeDs*P@bZ;5?gp1R>DUN?$1~^#NsGryHM_)`pnnW z*x&z-shF=pch9s%MvnHJ7*urDEKFKx%`UV5cPiuWKg6wIbs5BJcgf1t zGP5&$f$Yfh@;1RmRtUIn9DZwh{(;M6=sR&;Wjf@cK0e44L+z!5mF*c?r-i7UX9CB2 zb6s21UI#APCIv1}2%P&J-DXza?)2|EnV|?|k`xum6&I)!5?J4zGH%_e?uwv36G!^h z?{Z+;ucoAg$y*nQG?{82&Ay+BO`;-9aw@9(MV<6{VC1@&b@{z6^6Zx3iSU@2$-d!^ zm5(nODTqaUdH0=nd;A8EkG@?X0YFbTrFlT_2*r1A>LgYkQqJ>BZf3vLDxn>%g{cIj zRynRO#|Eh?QKVjf{=65NCaa~Dpthe={P+EVv4&CzE4aX4r;?Gs45d#VAc-tI#?tUk zO+8naFS|R6E`zrx%4~e6W>=3HaqQ;`Ng7uYIOE;554xerDJFK*dg|CYdkmJ{q}3It zocyew0b9aUB5}~mzY<=7rW~^R`UM<}pP1liEV|(IYTQ6~-6&kh#g#o)!Ot~V&tRCp2%Rigk`aoY zeHg3j@vA*P(A^oLs<>xo>^5<#+`3#)T><}u-hVOCZFE2IkJCaqf`g4P#4>CW~3^w&#tHGIWI>_&v3!q)uY7(vjf)fuWMJYjz8-t zSSa8(OsGdqqRi6Fy{i414b8gBLQ)U|SDpI>DL;=W?I6b@Z{+f-Jl`zwTa{eG7Mwbo zFl=LL@tqMC#Ry$$Q|34``!$(TSo1{b+)pM6QD3*@Hs54;a#xd-Rli!jPRQfQ7X#p` zww{GJC+K<}l7`*q8A>BOmDALFp`u@O8w82~yiipX{`6LtjM$~JXpFGzy_!g#bM#C_ z=lK{FTHA&UP{Lpcc%CM;1%IV&iyz>CWq!+ zoF|9Iv3-Gz>uq3UEEf5~sT0GcR+0ims4)U7Zxnh~oO({Liav=r`o(14A-L7cDlbD_ ziNw6<786@78%jO18`KG3{8G|=9OQlMzqQZ2JD4lezGYWAZYtd0DKq^ABq*vJ#w|bS zE?@`=$ORW)1yXv=pDm7%FYm3yX~w_Eta(|;i<^I{?3ZAi?_KV$z4CJ-j(?KjvCtOz z*lx46A1xc>W3G{KGJaKGpS73YZM3q3EQ52mHqb6feFrkTy@ZiM8k&YC1=R+Ao#)n} zljiKLkr(hYF)M5GY-XZ8=m$n;95d&g=NMmu%3;sN-F6Y_j2LVB@UpL+713|F#h9iIX`Swfn z0%RB|icri5%(_+a%1zz&*DfPPmJ3jvxzl8LPKSTfn>8hJT7H8XU|9~9~T}HTN1P!6N72du^i)I9sbvnoKZp)oNUdQN+KV2)Y0>*D(uCKb8fx!8w zWyUZXXe%kZ5q2-{4x5FR0P&m9H#{qnwkO{8aONren08^;Z{t%{Obry>9!H2Xy0!4# zcT;beh|b8){_ZW90QHFc(*y@R!6I<)Cv11W{VrjBeO2YYKFN=X+lg$5^Cmu3_E|kpKsPcTOe)N?l1l2yZ&#STjGcH+40H5g zhe8}Try_C2^8WVkPJzx};-k?86qxDJU#h zwTn#yu{A#6p|C$qE=B~Y23xXWN3%N*7Z*KyS3}p9;b?N!vV1&8JNC#E? zG4;WtlkM8j>H0v0l)aV>;9c&HaAMwsqPMFqH#Poy{F3$hJ<{#L!NG6k%QyGm*=o@C zwv|(d>LIFaoLZWm&-?xaNo4T`W?*0nRwJr4f-$vrdy@du22>P*RpF)&P|Cf+gC{{# z>yN&hSw~i&J^0{11pVu$*K{i6+gT$!vt{=JF#;nM`^NcKV@^tiW7rDDApSRZJn6j$ zQq}K(v|Iz7^FjC_2E{wbB9^jPOzI2h*8qS4Z-c6H zTmY8pt>z^I^nqjP+()Ph@b5!FLFNmvHaFdr((j#KYtq91BiK1a?d#o`!!B==NUeUcj$MM|*pv3Tpq<_DQ ztb)Hkvh>arM~m0F#G{iftm+v~9U>W;d)smygGR_h>FHi;EKnJ&(#AQ)%`eK;4CmH7(AZALH1U~=$UWp&_fJbGshN<04)Fy_ld+usgp zdPBKXBpQzA^(7v)`{LPmR)!Yq_lTXQ|5gSK1mJ+FWu(;zNF`h0t3kU@RP-T!t^+<3 z-)Uxi!`s!Xvhp(GfOyY-^T|JmHz=P`?~eo%8PR5ynrSK_$It86jtNDyJv;d+L6m(D zN0eaZd47(EpI3QLE?kt$7h+Llv^+LN4ETNGF3^>Zk1nR^Jj&|r@23k2 z^o#+L)aT*n zr+HU-{7DYNEx2j}O0wB)nRz60?>p1ZsTFvJ97A+$1nh!Cu`qiDmbTN|Q3^UUi&-Il zal7o?zO#(99R~W))OJLL>%SBT%`OZsY*zR7@kNwPKV3gF-4T||Rrtfpdlj5{jY`HB z5qr{bv7n!ecfjAGaRJm-V{|DN|Bw4m0$r65$1fu$htY08d3)u+1(f+zpfX~fg+kju zs0)4LEcV`(8i9Ae-?T(La@&^=v4Zam(3#`iK6l0C*lFUv(!Vv-fMoPKi|aVd3jbyA zwsdMEMV|`?&lqJ44(FMyx|V#?RWwrN4iZX-<3dWSmb+Z6SFUHB`+SAl5Gargd6>bG zI}AS`uK4AT^WVW+cSu4zLSrb%-`F3=)txUrI?Xjbh5uv^^8~jo7PaNK`>-rlEIL$# zsN#X)nreYias9wHHvW5BPv+!Hr{6r82TF$qWAWn1)o>Ac=?p_Gpaqm^K1T%vLQD>Q z)?a@Zy%n$WXDp!5PPbWk@36l;?e_G=_Zt>IJ8g_YuWH_u5G~)7wKg8a_r{CaZ<};m zU0FPXi*a3tN}8tNG4Jknv=3Q!EouMCfciH5tsy(;^@+Dv4*kj6%F^=P!}mXQ4Al(b zsn@qfwLvLOZAu!{y8j_Gpv@#|p>3-0z^6gJ;?qFE8pwsPsdNSPacUz z6HW{r_yOq8D>yekE_z3BLf~)?D*&!asJQ7b4odRN6p=tBEXI8KtD5_d z`;Ol0HH>HUIe-51Tj0M1^eC+zY(>>tD%dEbl?bD_6Fe?IzS%-LT{?ete?`k82;#d&0-Oc zM>>n-Yb2v*^U`gUl7~Vs2K2)CSi`KgV=5NXG$-+cnQdR;2zfLr!$pHOpD8`M5D*5k zn#sSeUL6yrSlo@$Sm1?3B7hq@HblxYBPM;y>}bpTF!{W3W3H3tvrgdoBK zCovfMJsMEa`-)kekdyzTLB<5;G?j#jsxXambi1LAmC?G^picJFjr$7JZ2WibPOO)8m8 zp{lmfkFnElp-|0@kiwJ9i1bG{(~}L}`tvCZ@niAb`}tOm6_OX)j;Hrs#|`|pUTV;N z&>g-%VPLpwf$;CZ&oq9CP@k&bNJE$Spd`W}3N8b#mFDmGd4s)YNqe1e9NH@I2vpaL zST)&ubEQhcDoRU9W?sE{(+$)lkrHrd9F$W3z&rmDe?dZde~kl{29AjnvamJg1uuC( zKZYpgPPV)R!;k>ug*Y`BU;WgzvzouU+9bv|rs#couzwKJM0;rketv#hkGvo%lU?V7s0+81nK!Zke*K#3BO{c z0n2&%UkdjERz4ipGtbw}eJt~`b6%Z`=2p~FP~U?wM2@1!)D`SNMDbBrhCXg3V?)zN zFCWct@+A+^qF_cdhy%+YreQVU>J4S(DWfi&QW&Bf=jS#z{TDmkUl`ZzW90ol!7q^bk%11PJ-D_)>QW5~Og)$db;{Mc~yD_-iVn8>$>;skB`W9B@mU)tGlTJnfYJtfUZw|c@Mt++U|JMXKQL4 z-N4=#c^`sx)a@rW6~y_VWGnjiC?pVTYVyT*cE zCT(KCqShkhMIxO`za}g3fEV-muk7^RagL0M$y}yFJS$nzgykFYcF=Ug`}B(=XF&w+5<_3JdNN!}t04x}O%Cw!srtmGn2R8yT1K zonB)&j?torWmpaHPhw+@K#c>aO9Q!LyB>lOv5*=VQ~}}Dd7?e;g=SR=mK1-RufGhLcruK9aGPDb)xOSGv^7JEiWM}a~U>}0M5Lnu zh0Zk_xPT1Cd8gnzOtt(C{N)4OWjZD-Wz_M@AXTqs3lZxk?{05{o?k7fYp;O0yZu*vO z|Fc@gq~19w_`N-wBQiFQRm}h*w%N55rU&1-3%O`5Nx)r%qkg^5O-&ws!eYM)L&+9j~YsI3;)&rt9r4xjXU#9B$ zL<(CG#h#Z%(R>^iUaC~!#+n%}xZ4|Q#0X1^4*$5W*X5mFfDsm=kBO&IF#vo~gdd<3 zhhUQkP-3Hd;;gM2i+jLZS|21J&`W(-RuMj9Q%!swJ5_SLjNb1ww5vqwoi(ZrmY!u ze}4BRcQn3&wIyHw*K^1IF~=S`{iAax~1JKS1prciGhA!py~oW9KDyY z1y_dj7sd1!@j+*%x&*La%o$Ja{PS%f(Y*#g8B5@$?D6Mm8`Z`dI9L{#Y&$FF)+@uYWMdZ{6{Vq_=)VNKjC*SIuus zk(cNpkLZeOWCP=j7v2Wl2ePyWM6!1pnwEELl(Z*{+53LrR#xf}#Ts0{PZ3KB*C-kp zvLaV?b-}JBBq3hr{VJ|3Rw&R(ZsZ~q)V~J!6-ZS@V@voo*?T~$2xP(7MF7d7pCial z9h%NN!kcj2`*PueL47|d|68O$10)9#-x}1b^3aD3h8Qg`JeZ?db2BxL4eGNBd*Syv zhEzNc9eN#}z2sF?G>snm$YNH_@&o%k0_5Ea>rai{D55_v0>>EM*}BlV)VV}P*3L^G z0ud(-e3Zv_$YtmL>!Wq-AslqPRpl@nLoHqV%|DSC#h(Ox4p8<)Seg1fo2bqbY29A{ z410sDOf&QS?8VSBuZXO71oGBvgs*md3-wtlV+df=m);oqdW8}U7AQAjfoc&+S3Jdr=F_rx;?v?Et>+<#$ z&43`)o6@!v!)G-23R2Kci8d+*y0urlrnT(7YIRJfOS*X0_JP6(>iZP=>+^;QIFnFF z^8q0i1rQWaylOVf0_j$S?^Hr6(wri_H z`D1k)_?fI>)qG9I_^N!cwDv6*FE~=m3)0wo(!>n3z=T6T zXQ-0$^~!Y^A6L^nrk0_3?~ndafEDDOe2?~S!XKVF=nWOG+DJOmeT}xD>075iL*tE> zcQ=LPTROTy)6dP1&Wd&rr0=pizGL&wTG$3?`4!w4R zp*M76Es)oPGT;?>)&!!cj{gdDfQ5BXZ$Mz=Qi$#z=4MOH6vnt-vve&TQ=fPUTo3JA zWe!g@QSxUalMxXS21Sj=B@Znb6nG>(bm!wlHTQ8A4slq+iaC?#W@qwOAkGpdC34UP zz0DZm=O@x2#}2EhR&-H%H`Z2Aiht8#u-+fK*I!%{+xfkIN%)?rQ@^>DznYrZ2Jl`5J-c_idzc9~6e3A*kW^=v8o8W5ybU7~=$ zdjUF^Bus>@GKeQeutRqqz%FCglG?%Wh#xu%{HgKCZalZMJGj|-3R=29$DE~%4y>OA zB04>5B7_d~lf!j@p4ewG&j8!E=korS6Ge(;n4#t**ZYJifOMUN??U^h z3N*>X3+e_q>T*Kb5Vs*EXR6H;aD*-S>icJ|kqlK#KukDT&FGZI{0&lR$oDO_*Py&4 z^yV~D;N+do1D9m#8kKwSxSlx>uc$)Su=E=gVgC8}aH4Ivv#RdJG#mnQUMB#`DAXML z%ff4?zEO(_KX&>J&Bw$a?E1}3_wX~>!?jk&*;F=IRXqvqDMZePJ@^XbDlw1?`%B7|Q$g~%2>;2#3(za>&M|GOq=4JD@Tbx@Rm?!xuJzlPMYF2t z5OPaok5RbXdke9_8s~bYSvWi|j-frjj`WJgkrUg?Oz8Th(#ni$_FuvE?M`X313``p z^vH@5dIo+2ij@1YCSxFHpgWK`ci@Y}C->#aJ=#7^W&OIGQ>0fAm`n!XKD&yBz)e{?@E5ngJBIqpHi-0E}Q%id}WvpmBRF z)Au7iw<(vMZlSGKfMUp)-6(0PRjH>E>(OV2Ht4U}x@B=J?5BUa<#${kNkRD?qNSx6 zsUBM-c(KW$FDcTfq@DqlPxfRGLrc(e+GWISB8R{>1&LZ9O2Sz7$Aal5zfOL;k z0DK5jfR4AAdVlz-Wcie z&v-hG2&7lyRJYW;x8)0e6twK~Tg!1NayzxV)gS9u%?48_dPNBD$c_Q%jxN3zZZkd= zdH}g-2LPKTwHSCT+!zD%o>s$~AXGH)0;|~9*JCxb^ffefZOf8}T58_0;b+DLX4baw z%32XIb-nk?bha+nYAm3JJ=VY#s zPz6T2jI-Lb?K);YM%i?^_q}nt463`2%>cmeX&zJ)iXMuK?k4rTVutjPIZFw^4+$2m z*JR9nYj>P8(Mv!rT-js0e=%p=m%v;Qz#c9H;I?xD(g6P$Pf2Szu=VL<)Lls>!te5d{AZf55m!w2e}st*%xF`-=Tr;i9Xm#gdq{wETv9 zz;7_THH4lks{Dx;6Z#xTrTHhjI?N)sW&a%k02O35-}(-f?nd6QyJmQ<(;QC0$e8Z_ zLC>*`!Lfs_qBX_7j>)7YBCailF{NP4nag=~Fc-pvZ3WOSEGz}3(cY(PhWtt4?bsqK zryKx*?Ps~Y*G5?alXXGaF!%mb@bdydgiR1TnI&{=XF?_lK2GG6U`Ors8|-@l5GO^x z>>FBos&|!A?}?qRC=;XXCIRC?@J{VKJ;G|N763_b2eswXGpO*gc+VZcv6>D%=D$Ca zpLujhNFcovSh0s12LOa3s;Ir60#1vqTrHIJXJL<4KW&oRWY4Vi8D965yEi!ykzNb_><-XU#|^qiM$$#k|av zznERBKZG0K?|9rnlHK^EWU0ZrNl+NQkB=**l}iUKt7Df4;cN{QW)p*Ib4Rd=Uz z=YMt3Ver`r;qwWYI$%lub9V^!(t5n5a z)8Ei%>w@cqgy7a5=y@qZD+8*)=2xBRqk#*r#!b?ULuUpz9MXplAvU#VJl`ihFmMC{ z7rvV8Z@93n`S%AAmAr13H?YgBpS@@L(0z>o>j%wPGEdzn_cyO=I^hx}QnoG*8jdip z&wNh%3;Igg29u|CjsEb3wuof`lhwKx3J1Tv{SqZzx7#X>u#-5%? z(UR%wx`5?r+~C&|7rFjPQO!Zn9}hMOwA<1nwlz=QNVJY5`2J%4%7KH%yxf zS82$dp{P{!c#5^Z(vh)L_@3I`!dL$bC8oWF@f^LTP{Ogk{MhoB4|-Y&WbKt7$fzT5 z`@ARCdkd~F)lgtHCbdJ_?23()9=?s|pBXQ4zoq>U3Y0GprrJ1V?n2Ws-6r=Lx3YDd z?iVYdIj_-yao$rY7$AUTna5VC>5&+jAdlTrf_~kfPEV(fUj}94H(VmqY?UVrok=x! z?W_F&cG<8X^<9(W<%ivqNmPUP#H(|C+-4eb=5lgS#O1-OyWU;qhbFJJD4VXwqZJdD zhJHGoV+MwPRG127i--L#cVX8D={aJVZqvMz0;*6=Eg%6-1R1n6bch!W8UfzU3He%`#RX}oe* zXQWKe_c7?7X>=uMr`t%`*_4O($6y5gO1Vk(^>3cl$?N1> zq^_0Pdh%j2`EKhM-)=;oe@Uj_#dN`adN|w|I-m&OISqZmhQMxXb{(L#EBDyd_^Mdq zGW`0e1*n=t|xV{5n?P+?=45hCSE|@ z8|PG=`@Uk|k+-|1Q%*JOvh4`&s~mX&O;F`OjclT+taEJjQB{9H z?(C%7HGbiVb(wV&(xvcwve4oCCLZyo{uz#UH>WsbhPGpeD@Jf(ss;py3(74lt3 zqvc}+e{#>M@637ob*sz=@5-Q+8>ad$xa)#Q5w6(9l2TP$Tf4W^JSG%(02-1G=}iby(wTr2UPJ%JmH*yFXqOp59Qm(9u_TI^|BWHH!0G#UK`B!Vs4fZn~(o_j>KIx;VDn- zS6Uw`*l7a2qJ(6HoXre6L-au~v(1G9ndv{PP9uQJQSz}*x*w&9Z@bzjsD8cKoH;uai(H=l4xOeQ~by|^>d z^1$AwDOb4Fk}J?j<*m?y>45f=kx~~k)<4@q#UT)els70X2Bp1nw%A}}lxN!>0A z-^wuzri#&QJJ9y?IyJNJZJdy4haOrbB_KTZg@>eTMB?Xk_Uf%CTyygH?xmi$(nWik zV9NU%N4Or`J?icbdL!tmJo^#v{j!SAP@OT|?Bv!#+yC=|T8C6WS*PQ+NI`WIa6;O@ z<^B1w@BK!l>1QpC$siI+$_#fEZ~kF@AttZF>Ts|kx6j*8!ao9|wuQZ#s7em@RzVc! zMpp6X%?)PgfQ&9M7X0$SE#XkUmzmfM&Hv=mcDxsp9&cmVCTQlDWd{!16B0{Vg<4Du zwDtdFB@*LFj`S>z@-#6j&**%t+tvr>6Py}&u9kBk$I^cLAsVmp!w2=8NQ{i^sJg`@ zcAB1fZY6^vJG6%~mB-;NVX{u(l+7_n5zw~}pDZ>rl^)ncVJhv=`u+~opUOmS=`>0e zr|)e0;D1Hu<(?KYaq~p$$#8+p-En+N-O=-ue#M1}(?v?wtycC7j1;47PYJAexi=yH z??hvLe-vf(<-SOE{jqnbGa#IbvkY3wpdH^2Xc{?Sp9=*AS6wn{&)zn; z^MK#G>Ec_Op~ivBH6alVgz-$pVg4NGM6QBS%fdi_x4}X+ucieN{!$tlyWOwL{24s# zKiUoL4EOa!lIWU-+Mh_auAf+6<-sXbf!Wo#c@P?#Z8ARXe9uI#9{i2Z$wHqhp>OQA zQ`lMzXCt}V`WJ6@1{Z&KX`VDl#>61d4aKRs#h;{ysOpyVVM(xedms3v-`b9@>Uc@; z##V(n9L~tBF~8K_y?^YC^uYGccdk24+D4>3$0c8C-`o4K)Xe(Ehuw|A-Rb6lxG0qq zoKw3ls#1S~tVtK82+r__ou^;O@Tzq!6E8*xZq?w}8y&+!)tf=f4sWH9bNq4Zl zQ=6Q7Zj#>(ln@-g`anuKjWaR#SN+|dIJ_Rag0EoB6j%sbXfmk8ApkoC=7q5ADd%43 z06LVO?3Q@8EcmC=)ucbVHONg4ZU^+C(5Uh-P6n3JIHj3TKf90j?Xr%-e?sC0R`R+D zCtH657L%yZ2C|3wn|H0diVY$Dr}f$U3Ybq0uY^yzOM zCH1mHo1MVZ2-gxy9y&r3O}VSCC$x>XXhT!c;#PBV@$vN$NDH~~c9Fcgm`yId&pavF zctD8hzUZp9uxUHWbDK!rk|-6TL)rn-7n}SrRv#)V=j#%Ds5sD3 zrJVUG8`|ipgoc6jnwmM9s@Zjwg)zNa$5KA22~_#*ZIO1{bD@83p4v9(=JyJ@hzf-d zg=YU87)wNTb>vR{YDfib6+9r+S`B8^K7*9%KjXuyr+Y4SKK3ly=RW>P+Bjhn)UfI0 zJdo2xaBm}=F#dY`yguZob*GLef^&N?e6rw#Ie`HJw_F zj;)Ip3E`rIt{J~&-WLuhNW10hFAd@X1 z{BhdGD^_o5LH!T*9N_O+{@p%@0O(&!2@cJM@E_0|u?>v6r zL{l6*o#Fw(-<{9}aZ7P+q(YYU?-93JFgE0OI<}B7NdB-U+ZuD_Wp2p%gE0!GrZF2? z1Y=Km2h%`r`#(=dB_?~*4sQ_e5^DOMUagVYuzkNFCYi^;u1wwrVn)9#IJbpLue1CYqixx7j0icsRToW@T{9nC#E~r0>vZ{N?uc&XI=Z z>1wOHF$LfQ?>+td4fFSVT7BY304TA(#pbCY5`>uXl&=*k!u@V$3LCs_sw~B zuAm5R+>y^Dlo>Kf32NSbAg1V!l_*ur^lSh#^!8YV%Atv2@b)WuwtXWLM?{f{vM^0) zdFy-T)4b$n=1e@ry(7r22nBYx#iT^OJk|~tthhpcyzB7~*PpjfGd6xvJFT1Bia6{b z2kQRB${qo)YsxA13M|#z=f^r&>m6Q3h16V_=G8ls znN*PqmuPUn4I?}m8jYECCfh5{H@0JG`y!YP93MXv<2g^LkeU8@$3U0kK3snLjx*76 zHgcZ$tj$?$mPrNl_`yDvY)XR=d6;fx!d!G(MTj)?V>e^umsW~*jDrT1b+7n2(98JK zFl$|12=4%4ht{3z8rjT8>?x$DMk+WGzI@u3_tjF)qO_2bGsF-YoL)FBY~QWr@AL7K zQCr^8PZQ-2UKw4Ek`-XFYm>>jUgnP#k9$%4NO~H2|36g0-hp6V@~}`u1%&8~Q>mO} z_s4|TDE-uFykFcP{Uy6}S7k&{;B)LH+QbLvyT&|a{ybKr*9iyv{uC&u@bBlK5aaIf z7l~xG&>RM&Q*qhG$?);drr%G*Ifjy-4YjYq9S{DAf~L3AgC1pQj?$h@klwoi|}GqV$R z_Q9jM(-S=htZMgb!>uSPIL<`VtX8lR_lLPd@I}5MDkxM?Igr^fz+}pv`Vb9I1@8%# zqtRn!yvNLUK#S8~o*8b?KlLD9qbE75ePb)N&71rHiF?dSI`<~WA3{sik9nP!& z38F+hagtzL2X?#d9Cnh-OgV{U$~O1}TI8NNbRq`r1hL^8P>?QA`U1K6dXd$lfO4%1 z(HT8>%~$J@g$W`CMIghv6Q{U7G%0V|IY`F+awLZZx&qudrmo;v&K_Zx8Uq*W%1JfFVJ{8#gJaPTF&xFBN6_t7=XPEI~M4Ee_Foz}t9LL>WfNT@2< z(ox-EH%waxZbv_B_p)TSBuT2+d)wCd5hy!=G+0-x(#!q9Bhz}}7iP0E#bXBzoqB>G z<>W8T3@87nY}_`_o_gSP>(UZ`1}anEk{SrdJU;9x z{mk(7l|I(^DWLDo{&<9C|2FBM*(DB| zNUQOD99>9wrLRXsO6B@ld7LW#9@Dyc37t{ATLz77Etu}&loJ%wZJ-(qkS4aRTkv26 zm=2K2EqlzGrHzQKyBlbj(S~=|Az9_pi%&t;Z`>|yRS!mv+H4W+>t3S*FoY?|K6VGJ zkXKHT%b(ot-ja{M!)K7=n6szq35yuhYz}t^t$T17$s8K<_hX|dk=f!l?xvC@U?G(X zNvOlg-FPe;uq?r#4+w$OTB#vDl|m!0E4tP}6hZ%FC;85#T<|iI~9?Axo3R9zLY6 z^7`zv#cAxm&(-{;?$;TUJxVrPEed#}yrCA&k5n|2RC+`UWFeh$VEB=AD!X!LJ7TSr z-xc7oW0**arIhGKmo_!=7Z<;;^K*nU$ZnHueA5aghv)=N$GyugmCPrH zO;T4%UHMpM4)D7F33xRg%aP!J;uc@;m6taF^PGVB<+ZFH9(<5F&0s5!52t({WdN<3 z;N%4yqzt2^c)>f%vHW+M1$(}L#0zV8e8P*efD;}d{RG=BSUV#}&roL*48C1Lu|d-J z7ZhEWS5`9qw7@Q{DuJG7lqtsgiIG`qdb9BRNZ?*I_TBb)!tObA&hW%#4PcOw^+cq4XML&yq z{>1~N_n{=V=uNEg;nee0XMk&x8(LaFP(-5Vmf_vKrSlyVPFCtIGo(+0+Q=QFtGt8exo#-Z0$ z9ot5a!$n0!8N^A6sGzh%p1vLTJ|&1jEr%2TxU*nT=U(rbhNQR=hrvGHBDtcZt9aw!BUgmymGy7(rrwR zP}$qbpdDq0)Fc02wA2y8zUB2bH$fHT{EG0EVDnRfip0!aa3txc))0TUA)@HNMJ%wRcrH`<$=yuo|jsu=OQ?o~&_YG`- zvj)3J8v77g+7aa8p$xmh&E+=2D;fWlDq5Olq*SQAH%mb}FYabAGwZx;F-)#tU$qbF+d{3msgSJH_XesXd+(S?HEK&c@Ud%+B88OU@h^ zspd~MxDqvyYjwSXOX}YJKi*^rcioGIR)5@YYj)gkym7>=4o(Oin94aQmj|JK4}DUF z@QW|)?Cni$ztaVceR0j|`zFRlQ!YJK+is#V1U(6xG6#Z6h9_MKKYqcXxv_rs_Td%* zOHG1*VRz(l=uy<-+qw~T1(l=bvsrTM?)Oj&vpAv|xfjsi<$O%X8pLs5bAv9r>L8T3 z_q}L4Pw|@!X~xvx^T_P7yoct%v)6C9O17vWRW>1}>q#?^`I-R_uOn)rF`S9k0<|}J z1z(32Y@^WHj80%^U0P`a5d=rwqT5F;OD6S4<0u^D(Kqq~h8uJuWvR{`N#1+z$Q^GD z(g(BfRVo|)`dVWI|J&0BR(BPM>}zgFHrVq0l|AUU(UnK~ndgkNKg z9#i&9e!-xR=2PRok29{cRQ49*Q($z^pZWCA?i6>_4G;y@bdPiqcbuELGs8ZZ><~+& z)5Npli5a#XVjvbu%nK@@%l5}>Q1N_Gq<(8>WO{%33|$EQS3Go^^Ya+vJ{G1XdrZ=I zQIHs7`3FecamHnPj%-5h*z(Xu_K}nOug<9t67{6<$jRF_&6a!HP3=S==i|D`SS@Xh+eQXM*7)6n5!4O4`ntPWN`BeTpI#tPkp>qk9>fR=3VwbuQX5Tq`~{i3IFy3+pYqhu zhmC=GXJld|O}mpS0}kyJ7c=1|RCRS92@09_B0K3F{a|@JgXmU;JC^_MA!JL%@~Urc ztE?Y>#biYxx;Gyu@&(7b1zLOWpiU`P^_9pAkA{VDAa9L9G1S}9pkh3>o|FNaUxGn^ zRDJ(MfuUI91Lq|d?@$W%7PtJPauhaH>pzHLDN-7J7#ijFK+FmMWL?A8{acHm%f~@} zI0yXCUykmpxTF8-!TEp#q9Tt#-qiJ;)PXZEbs%JEo0F?Ih~M<; z*ETH|qbJb6Iw!^e#lAE&jcQI|%HFh#L=LR)9`)~oT(^&=c|GUc7r2=&MIrwa>9es& zKI7lPZ+qjm3eAGaMfKUm^i`*@NV}O|dsl(*@C>{69RrwnBVxJbUxs>SGq<)w1pkbW zR%c?GJDG5^Ju{*3!jL+<(?o9>lZfGe{lb;;^*Dnd+sYBdIGBprc>3JK-rV8)L00TO zNE<{w=2A4L{*FL?^U%w#X-P4c(4}zLAcO=Nt9BpK8TO7I$hi|+u7@{#EsZV7GIkCBoxQ8{X(KX@CQ{NTOQ{0vPTa zl$Ux7H8#jEDlK79^QQ&N&@Ufm=qrsn2GZn?LYEdnQ;k7YoRd`WyS?bi{|Ka3X9`4@@FWH*nr1Fo&|mtQ zo9;-{5wAy`A{yM&9|QHE_|A(5XX-aNsfz$o6xx>AlFg5ltKG zdZR0$-yI}RvH{|}7waPcuO+msUPY58UPANP56M`M!RyROs}J_T`Q1Ns^j-T-m*~%3fkqN1|@G%6m%^T7wbr~ z7`P~HEc%?>c^ozgiZ(-wxiW!>nm~aFh4$^&su8ll?;FO1(2@t=SuCAOlh?^&H*P}Rr~b{X;F&C- z^1*i}qT(m&&1HAKL=PcgEn)prZ9^s|7D! z-xAFS&7_}(so`SwTg>*Je#g)mI4XM^{gHd;?vta?D8W=6(L3qD4{9~_qZrOAbS-zI zfULO~9pz+%R=L&h!LnD0Js}ZHmL;!Yro4@APF_NZfj%%YpiHp=EP# zz)k)r`XyXLiL%QUpVD=jKZ<5|_R5nNjL_77gx62$s$fW2Rh5u5vrqJ0gsg!m78Vv0 zJCHU;6F-Y5WZgHRERZuY2QALSsWCAqP)#=AH06MfwB(z1h@SiT{k=r$6jX3;++(Vn z-|z;j8>d^_WxK3G7oS7vD2rXcVUsKL8G4VhYO`Kf+VqaJi7k%TpF>;r<$5yZ1JZk5 zQIa7f)TTG{!&L+rmp?vDalS~Y;AQiscH6N@KPG^W2K1|?S7>Nr&n>g78)(Do>gvRN z4P%0R+0Wnqk7Qr%%?|Y&;(^h#7RcX!t}zW>2_dK5DVdO#X6sqXY{Rx#5zlfyri$!9 zU|C^IRaA4@)QPGBT9J>j9%Iq`C6B#v`-_)cX2Z3&WlBmC6c#5FRLe-IEJ69~pEyb* zAt(lEu+X$173yWVSowiwR+Ni4IBm)*J6*YXu?#_HdALhB(3i>F%Fev|ZK>=62}751 z1f77#QmLf)3%&Ke1-A{kU(HNRRJGeHvFub}DzKd+`AFT-898-)Bd#1x zLsJdT0Dh1ZC2>9TyUM>382}I=qp-0_YmOCKhOIVDbPrep2`Gslws84I05IQgoV|UN z7SH>?SfBf(?oQH}p`UqSK&&#v-Xf(UD!16M8@wP1IdPx_^F3M zUxAV7Q$)8c8Cjgf-LE1azkZi)j^j_q9v;Q(D~&s`<)c2qUkT|*b*`&Ciebd%N~j|a zO{a=b1l!miT(U3VPl5@Y{2h%xzxnRjYyRXC z=B0?Ct;J-xUE&DE;$?f#kWS~RxpY_Xvh@j9`iL)?WA@M8P;?vTedh4t!!9RJ*2uYe z^}V=9dqC-*@FWKJE8Famk{4@$& zzNoCMIzc^z6j#QJmKpq~NM@SvOVxDl3bs1V%;Yz`6lox^H^QX}jCl+~wS{sP&?E6- zJ`HGqSYBv`KJnrCP88(wX!UbKvknB&I5MTX$#83Ijr;9H8pZv?Yt^#g-trHnK_MHrqvZ zF|IvEr}JkN^}-I;4P4NHgWCUK>>cFHzDLT_wJ~nEx6DgL77WM50}M>>HF8TSD;42A zW@vh%qdC%$+Xns_1>pmfr~vtr#n1jZ0X)pz_^eFOaj+%a4w_|EoGsMy=jV?yR>NfO zEKTgqIeZn0<(8T*=y1|$bPN)L>+bFSBSUXq5<{N2G#rI+HrE#9@btMWy;wK{Sh$1A{(ooVYc(eV;t8gA8Mg5iY4g8Td0(GHVw=4$e z7wY5pfL!S)`9)vP*=>BVSL35_W4X@84&|rm2DK)!Ex!HG46!iu)syH&FJRme{t#Y? zKgyA2p)~IujZ;GPB}I~o9s?9MwOxX6x!2#v5B!Jm(Byb|_s9yQtA;(PHIroam+KQC zV_=k#hFrFQF4aZQvp9K^PlGSdYld?pzbfF?%YEhVs?R_7&{-08oSmr_PPF<<<(3PA z#c2+??5zpM*;)$2d>?b*C<8wA%UvYsyUAJqg>%+-g<=#1W#wc?j;TE)*l|{-=R5vg z%G|?8Re>ccxit44ZT`L2F^YaLAb2mCu?k_N;`M}{xuxZ&Q+v&kJ2Qx46|TbaUDr^V zRdKR%>9H9Y{NiyB45y2WuLq~glM>!6)dtY-s{WDfqU#JlW_nDsR+p8%X0jjONxl-t$7VsE zT8Q(G{b4vz$S8!8eE+@mNe%_lUZt;B5aYLmDZvCcxln#Mt!rY_lyUOu-FXaq{#p57 z900FlI5Ur2KVs=9s}|jE)kxUSQzVRNX_zoVR1jHhe zGL)>*bQhftd}K-RXCBeYAt7Ns{w!KuflMjWpk@4< znZhyae}_EETdH=^|9*qgujqFTO+Vk|{KzC6IH*PGsH55-xK zqGEZ0q^NQ}2@wv@sD_B%CB$?_>)rj2#LfTo)v%F-nQ>W+;SKSiMZT!Vcy^4+MgJnd z+?&gy!3aKDt^xE|JI(mOUjHA8%44auj?FR73RH*l>G9K~VH6a0k?Z3){@srw^n=os z-)0OIN)%bXMuz`7HGPxz&obB%T0%txhVzp|Y*7_^XIxP}<3Q3n?Xa(X5*Gi2zA;T? zf-e;B9H@Krau;1!nyG$Mv6;-j)6Fxs$97<{tTMIqSmQjn2gx>LFkYVdBJFlE3dhuE zCu5Ntc@{|Wn=B`vV`?%g{IjaF_-oaT+n1wu9Z3gl0h?FU3FRKCFdk>)o7>ho@n^A* zH&M`yv|%_ki>EEbRbtK424eA-!!*$UIUX5T;(#Na9-taNRj&NHHWMS43j6;vu|3j6F z7&KQ(+_PtwKHS@*ytB)kO1E6BG)4=#&UY!>K}&SF66(@tTW*d)^L&q)lBUoN+V)`% zq?gqk)*teq(;?xze{DB`yYH9dP1dV?3QwZm-L%gCd0*u~kf?FV?$*j$|EMbuiw^wK zk}UMrzr^9M@|};|iW3`ogvK{mRWO z8T!hj#M|N8;TAsj=aT-^@R+>=TpcylRz{cZr_lEzySJH4CjQUMfXuOael=}RgXO1U9Poua@^{>JpbFmL+kKws42v^T)viN zLBQb}Z=2TTI<_^&hU?T8K6A(l4nM22Tb)|mSSE$e*@8aacy{2~ml^d4mmfr5x)yAq zP-d*(@GPz+?W8nu>V@GkJt$PDj;VQ68JWeInBKe#VdD1Pm4`hy<(}bvI>AavwQTy+ zE;94`n1rm;`&&Hf4SnX%OEs%CnSKvnX10IX&4KZlA8W8YDXDp)I`i^pjmc^Ek(o_f z-z&!!D$5anUw9B>7(#jl+j`AL`x9AeOU2#6ijPI-`lszi{Cn2_H=37BEAMk65v};i z)a*3}>I0ROv2vB~i{T_;k{+hN$YEgIy4O!@mqEPL$Bu|TVlBWEJe{W7@lkwq?^xZw zU4NgR!#4sb3mu)E#kx;Xtn{(XBs?xHchOpEYHDLGEiI44kKsKLvMk_c9sUDZ_f?g; z#&onlMzi&P|Dqq~P`N)It_)_5@FE?}mZ_SVjM`{+;Dy;Nn~Sk zut)UaInh@b_H1e60&7u&*5A=dE3E(D_wgZAaErFx{vtMv2Zn*Pl~THDpDwjDrK}LX z?;p~wIDh9;4}zv?u9Ocz9m*7YJv?>hl~dcd^t|l}OeC>bwZh@QoV}Px^GIlU0Zdh} z$A$#{R^nFQh11cmIm7;6Y;T4RS9fRggUza%D^C1(P5Yf-1b@?kLU`_D0_I>ghP6@I zZ5_{KX0KyTFZAZS@h4E8E zE%q^QTwhsFySJv;{{qJV=;5H(oo%0XT9dy7Auie$ z2e+)PiZ;}(|A2blVsZCofmgYYfft=4*L{mA48GQ}`R`U(1~nW2BWS)qy{?Y|c?b`3 znc;GL^Jt;3nkqUQEa%mUg?|9*%ZYssDEtYE9RnY;d3ys`?Kwv(sjMwmhn0dZmH?BI zc_tapk(mU~`&+R3l7n__moEEb&et)UHol!7cjfs5WP#8dHKDc*8Q^0kW=g{$y}5=5 z_fG$nX^i;lFq4pG(ZfbA=3+cFi)tSBi#d&9sqFo!cN)WZhLy|$)>v`=Pcl@f?}aw1 zx$y~Zqy-Iqh4c=6X_*IBhI1AN_(PPS<}LJY?U_Y>6mx9fFTr|xhdQhp&WUGgFF|%K zcq0%b&vPz5hl)b#hy8(W5W}m&xt|gdb{_qA=hy}FmNepeSZZTv?@;CbsO@X85Cx+q zVD+J`rqJwSV&gsZ!{WuA+-4bL<@IS9z#Bu!>mUFH1FN6HKf^gqoRBlhmNL$BU27^j zuA9ICisW%Lb8nu(^9}~MolubL8>3sogGiJiv64k63@H8x(sK@ z_WcIW+s@Ec5d*5L3W`8 zn*YI#YIECXWv)PXwy_eCyF(XEEouEUdpEU2<2uCZSosx1+I;~8+E+JJ%wY`woyzE2 zbQSxLhQ_!TwpwDVzlo6rMxqiIHJ)cAf42SgmvEmrqBhg61*N(GD2Q7$df9UjVU$tb z4h}_3pWk5X_|>FW1rveexgy|HHRb;N@Jjpk*Y{&BQDI6l4m4|sca?h{go@p-U3ll# zqRs;Q?D?~#MJs5S7T5I6iJ&DzOGmm67?1%mwD@162^~XdRw`&$y@Z4)_N5L zU*RNiaI-v6K)5&9zOi3x>{t9|;I_bvu{!h8+lE`+JwP9KO`plpCP1HGA&P5$BdQUrXP_>g%C9j2&F} zom`!W6m%4_ocj=)8yVNo26cXGT_d-PGr*gX?N7Vl~Fx}8TbZ5ZKwm82z?8V+d zNK*F&TNm!OF!D+twt_t{#6z5Ih#lP<5Uw42Z3P<4FBxqL`c-VZm(AR0_#$cwiR05- z))ARug>A7v7+lG*EAwOe$&j3h`BN+>zZ%8hIe%Vsr(ogwK}RCdwvsBdpKbk^mw3~Q z_s^Oe_g{wqV8TNWUe8cr`FE+Ll~3hXRAqrL zFs;A_K9919%F}NHemmRQ(WC!avawC3t+hY!<%k@>J!hG>4tUUrPs>FAh^I7-iHQ6a zVCCTfWS70Rd^86Gmk-9Rw$LHF-1|8D3QrTMra=As%tHqv@2R`uedwRr87ahAnR2eV z4WkeftMRDNe3VB^3(eZ)q6(%RTpLfM-U=VM0zrqzeTdsOM;*c&(mG!(vJDk#pP0v1Fv3{v5F+XAkKrrv?}vuns2!XPl&^<9t2iPSttRq31ZiJuNP z^kfC(6QN?%0b`-GRa0^v7>J-CIGOVO^n-Mb#pO`_K6BE5GlGBq8HgZSoZ-wCPybS> zv<8Q-ws`-zHD=o(W+ zz|L_JAR?oR{^B)#dy_=XRo8^+0@mc6&q#(;VaIr{uC(3my8n)qhhX#(zAW3<<)`!hDj=i&%NJS{>fR1?X@#OIMr zaL&obcwRqW!3!s2c=E`c=f7IvJw`d04Sn?($2{f;G^FYJp{Y@6@K0mtnyy~kF-$4< zgL4{;#8=Oerl#YSB0R&BGdyf>;GU!VqlBKzFir zL}u`%sI4lzy?6nO-u1Ht*!1ow5aU;PU#i-wxQMSikE|-!(4}4=*w}TFh#p0NZhjK> zM_6+I3X7Nr{HA6aN>enqg?N1Wr(TCP{A z25^I~jmuUdX;VmjllS&^GGdp5H~$TnC+r5X_E_q9GGaJU<##=QZ>poTE?J$2=p2JQ zb{%v2?=Osd4Noq_-WR@I^nvgL@WBUpb0TohNz)0M)r5h&QLJen7CpKM+HuCg_3^G> zh5pkGvh$a*x z{)J-M65H3^mTeSaleD@Jr!&*Zr;Q1Ywq+--ER}RS`&o0jr#HQkYDv) zmgcMO-F5hr2qyL}3>8ct3ry(Z7Grg!RBBb=Ac02t`?1zxJ^qD&ek?;Q>*>t@t+F#9 z;*joB{f0>8a%$06Nh9plM0Q!k6SEJ&rFvJe<3ZPzh(O9SBGhs*rjLn%idy2gOFm%m zs6Uu^%-dr;^VC9n&QXi^wwJ2dQ#HjU&;Kh7_JAwLQ;1F{il>LWfumG6=M=2Im*c@4 z(@MYD^I(3QQXRZbDR)CLcSBX7=f7hwnHHmqxMy!HG>H2$3ijoBx6G%NwL;3vaWVf5 zBn5XJ`5Sp7AvI>a=&HXhCZ2P3e0+h4Y=r545Lu9l3n0#a%pJ5Ubn1d2W}c|3tlu2>xRuE8N#jwHOH-5>Q|rD4zV>f%fpBf(byXRgi!la<<;4- z02~EzY(IW}TKqBUS>9ruNYd`su$;~9u0REl?%E_WuH8>FV~EB@JQ^(Uu56Pn(TNJH zPv?pHGZ=Tcbmszuewd61O{lX>^{Ul+{3?BoXl3aU*apJib%^e;x&{)eut&sx@7S7Q zXAUF`b>=sI>5Phvj zG(W3=9^Z|mef>^ExMA5VG z!`}}(poF7v`+_G2wy10ju3Kk5=(qxO>i@wLR5ZG9!{97V@QQS&Sm?p71X2I!T8GQ- z1uk0l0i%e7XAT^-CL+@YLYx5=uepd`;tHC1sY?G&8Ogq9vR^x=E)J=ST401lAN;nH z$XNZ&`cu$xB?2+b6%vTarsZ#9Isi6t(>|h^g^!0lf82g(Vs*9rTpbwd`osHwDSUi3 zI_L#&H-=gyke?wZj*7`9JMZOHH5+qk zo+RS)FIoS7V|EIlJaa%~;Tu7)qbku4KP_z71vLHoo7;+tgx_Oh=h62lA&KP-HR z^?)bHZakHrCrJM8S?x&wjGOJgstw=H)!#&`s{=n>*@Uel@)IfS@hxVjlvdZ?ZO0+R zl=j`R<7Dj*-suXvna;IFL@zyf*kP>*O>FR<0$5Bu+f z?05Jp`x%ft;5j*~TVBSYj)XPvMiGDnkMxEsY35grPKGr)g?g_Mi6BGV#*c&m#d#Lw zLTbZaZ-{kWm7w`Xh|Ts3B}uGZRi^>+SbWxg@Xxvz(-gnNfhHlf2fz$AtU3o6hD+@q zc2aFW?nx__!*ZpC39Ii0D9r_ED1T-xxQGO7%MC9_Te+v(_6!P!5j6XzjkVU0dlWms zm}v`LAct8XhDl3DD)kiD-DUU~yS8TE=}kKBrYzK*k@D=uSLGuNXr zq-adKb9Lvo^Z+2`z6P8{oaYQfIm4$Z70*0U(E*JiIfquI93=K(3I9Z$bofeDN(;|p z6T_~IuE6i}bl-zy66-MOM9<~pO$j^+AR9 zwrhPAO9&xCz91@zA4-u58($3IbR~%=PeEr7igf0dK6Y6LuCcf6;TdBP{ zJB>tQS|wf*jkUXBaZ!o@OsVNnY*x9JX*S6h{!ovRJ|C`rwO`gf+EIw)=nk>;J&dJ&>F2NT1 zVD}{xmP^hjH^)Sp+t|oCABz+?c<}X}g12)N|%1C{CTr=wSO>AO;*vmgufzF8U@3kieE<~gA z^NR-_ib&Slnmah8?q=QDuj+4f=H3VSyYGVh4?XzMYQA}FkNHc!u5OmCEJ`8{qZ^wV zUHVQsV3nkHuI0`z37*4Y-R$#DGv21dYyMvN*p646~4L8!f8vLI8wxpot+mwK#+3Aftzi#)Wm1g!dsAcFHY>9PLd>3JU zucvy4-V`EvL(U~X)lMwm}tfW+QsH25S=;fljr!}LCg9Er( zZC|hZ_oGKG0!qf%zK2&F6;}1>c;%$gd_!YjaKI7CHR^i_+`#tHwO@&wg)1yyfBOmE zkb5{jwR z7*kHtbvNY>u12lC%Sk~HZP}VSUvh%Cp#196>cYQOiTq@gV(Js=ua%7l`9j4^We=SD zLMCAQ#i>l{$2+C`jjWfYfAwAF`ZaJF{j0AjY!l7Sr<0pPzTS=On&^LS+V{T3{U=_@ z!)(67l(?jSY=C73egs*@uVm*7!I1;X6)!S=U69g@jyb57D;i|qEe{ium9m{}?%!~B zpknO(tC#`R4Ya#{{~Y_a`{pBm!SO#&UZtO2w;-vO2^cg~Tgm=gZH2!?{Wbl}`#0R? zL&}P~^8$`I*WVW55SSc!f2q9d_StrBW?z>?9)lMHL6fanoS7fOI5Xws#GgEn3_LP% z($jw=nP2zSW!)|MYgD9si~ymwGv*1$JmcqILXn?0aIzQQu}g)mNjuwa;_0}+r1u{9 zzp1l3>_Z9({@dfL&pK}6Ylq*pi>5df+!;_|e|699y{U7uzJulZPg!fREe+7o8k0`+ zvkCg7>!yrn~Yw3fyAo zn-$rnv@SGHqzAq!C8j`^D=xGG%p=Bf;}}tKouk}1l01Qt`*TFjSstfa62MR)Hk@=6 z8|LCPq7q7yL_>Pj^H)JycE5L{#K6%Fv@$Y7xnf!DY}1jM7y3*a29GK)L|*>kj(H01 zfbxdI9-`v-_$>cE&0cqDRsR`1YEf0GGuRND>f-{YO@U%bFI}Vhnw(rDhik1bbzW~| zFW%#@TXCY9dEe==%pW}`<_=>lUk>;pu3*jlu;q`R_t$y&Q?lHvhP*~HEfp$NLoX|Z zI9FSKZ_9dcT+hU&!5vql|8wM=jMw1}VPeOQ;v)6shCG!gs?R_CS|qu8(Yg9%V85Y~ zQ=W7B0+KrYb%S4&v&;LNmD!qt0;P=WpN>?foqq5+$f50>cz6g!ZE-!D_>zzRP5mRJ=EZyXML_m`MDymIcS^>)=yu|C~*Ff)a%r$YBq2G}9E9v=g07a^rgW zojtcEY0{2Tr-F1ZoZ^V>NEfRCrNsvGPdF5f!gf7CzIG`B=LSLBPRyE*I{ zZJ}Go#w0+&O8#t#t{ra6zwO4+RdcU%+UNbJyb<-d?KL}Aord6p2!eO%f<$F%U`TMU z+Q4>GO(FP8Z(KMRz@%kpH>h4r%-!t>_C7bwyqW8aGb`!nNYxF&a7$?Tqforv(xKHafrLk@|DR|DE3{~d&HINCeT z6E(>v6s~b-dRxgx)<1f3fAoIQni>pNiyBYg)$DPl$WdttV$FQtO6wZ&%fa12Y3%t} z31Wi@tP$?@<_E*0n7E$jM_b^gYrg(|rM#{555`o)E;&xWudy$FXnXngiA zaIQgrwMEb5-7;0~9V?1v$BQi8E5SLK5BE7FlV{O02IU3$hphkY-~KbmX?hAgI*sK6 zBANJ$Tm+RrsT<0PQ&|}UPA{N{GSf#k!3u8JZ5!AtZYymMi#o-6%f7cu+SdL2zZnMN zVu>WSW?@pshEYD*$Th zSIb)+s?3}1F~*!@jcpHJYNxH8<)1Xb?t6D%$8@J@AsSyhkPHvpQ_DsC?=b|%Cz#0Y)3^dZoU^!T5R`6fky3EpfF$Y9LnxoSy!V~Tw(giLP!!nQTn-UZ#a~R5 z`)0K;rWt!IM#iGSPS6Iw<7hC&!ajv z#Zld3sK;8Wusnk*pES3@wvX^4^pB51FS4=a!tTj!^epKu4rxb}ad%5{!{k%n=aN~8 zF-aF={q0O84Z|HqyR;6YP>IS+N47&ug?Do-KaH zDu`jcIc*%$OuimiX>WgRy^$>|&C*x0=_?zRjCbDhUJE?CejA-XKK$8_z(Bj@c+bad zvauHvT)JJ|MjC_Rmt}G>{<->~?!ozeWg(I-7mE??OGe0*l_0tSViCkn`GFq>t9IJe zNo*6ov)`sOd*QhigKn$@runtYz{e^c{cv*jJ7YLeLaY$P`J($cfbRErb}Av!CR_lx zJ=`wU9m2T`$LYt7*DR>#Khu~u1TG5TWxisNaKKDUxQ!IInKFS!XzHn z7$?g-45A|6Zf(b|BzOkG%7-nL6;&OV$CoGmiRwtlEArxFXBUnYdX z2dynjZOA<0=zW zR*GGZP!~UzhgN5Ah(q?@2aznT#{MuJ6L1Cio%WS13#6E&nDB~`E8wl3{LNv?OaK|X zCVhw41G-Fz{iRu`StadOisn<5`wl$^f8y=f7%>RKA`36?w}rO7HP=wtdrA4aKdnK~ z@lO_C#RX>6kh680nA#E2gZ02wuxTV!~tI?z)d|H0|Lg^$o>>I>h}S_dEk!$g@|OnjS1)Sn(EVz ztQICohW=3^e(qJp=%)7fP(>`V*u9w|YqK@UV)CDx#zCaUkU!uuuL*em$XeTUkT{1o(uZSVa~2Z= zPdq|7UtrD}m7#5wyWf*6J%(RDxO&<6bWJL;Rd*}!o1vbdb)YNZBSfx;;dKWIyux(X z=CZFM8U!95ckiAQA~MxP`2Tyw$3V}WP$+rM2p|+~R$oQD`Q(z;ozL3%2PK6? zK{8epKnY-jO>hf9+6yeLWZ-P97+Z-&-8;2Q3~1NCHN!HcGWSJO+~|yGK!o%USRy5e zi!wXh@O;irsT5vl$^* zJuQ?_#iDRxK=6C#et$Sd7*Vlb>pE0x-INWI0lZm31FP-?qRih2dmH7(aNJS0lLC$1 zOlki7wUPTLt?g8)b8p7oSzXw+0s1oz0R^V>6dO%htAf9v&fIf;K;*-X)!6&o){CU6 zQ)00+0bsh%wSEvOaK(A98s{3umgOed_9AzY2BZrh2n@p<_z=APeU>--GY|0|S5pej z=nSO8MN`Q){yK3eYhW+gT{>X-q|sPdyj1h(0H>iNf!eO?%iH|zM*jE!zXQk>S`Y73 z$nRdICiv=k-_`M23!|fp{Q3>PyFhW7b6?@fIIoSkX=CZ1fGIFd_~SAE$GQHI2ovJU zeg|s}wMt5fb(!!mU01RY{^%3E|HkG13Wwk-Sr^zzgkqe2PvOeanWc1+<&_}02wEUw zW9uOEtVvE_q-P%{#m7rod&jtRV(F>b9x}n7qt`u|I*=U2>~5$n;+dF^+bK}r(ttqy z+f0{m8qUY4^FXoXlQa5pVO5NquF%#xm53%^0a5wGKO)Pa%{1I6OP7Qn(KrV*=E@TIX#KCMT&bmDs(y6ixL3| z6)yLe(+7RJbwf9PK{*bM9 zUMJlot8_j4-eSho4AT9gY{5XP{%v@G`%rj$+Vmn^s_HM%33-+??!RGZ0umd8AwP^k zfo!WWIjQx5n`r=3?RxPVgiWitAGZ6@Ql=!;wxQ#!CaSWqPAJ>zn$a_4Sw}u$JYZ2E zDWh_jd*bc)S6L{Z-+BH&kB99g_ipL^^({ibpb`opE&}EiWpBoVd`REx^11cowugY$ zp%yRbrT2(q`sVDku3qY8jeknIIgCx6@X}E}`rC&HPbLRpchO1Jp{_MLV^+XE-oJN9 zl$!0xR=UG(HZX+KZgFsA>XjgAQ#uT0Uz}EN(V*Nm1HhDe;l8IZ2kA8`$hB8x;T}4w zU&5*^(x(5&4$%RHDp9$@ysc0&T4d2! zT|Ra88KT}t5yXCf3FlmsVSF%1@X0~oq2srq-|=CA9az%{rT0+Lj^#BX6_^r-E50~X z5^uA$19ON98Y-Lvf$GmKUzM{(6$j`1We?=R9tebL?1T0oj1XPUUag#JFwA8)ufJ6O zK^P=1E0oEwF4FW=wnp)|DhwfHjU$%->^OWGHUYv(MLrdSPRfUZ$Sw9n^nRK<#QGvL zGtQ^|AsDg*xL6jJfcyCMS`L^S;1wesvetUMuaa_7yWkQ4K8@Tycn`9RQAa^YInsKb z$@q_bz25%@DaE>P1?@ZTSGa71MC<%75AEnGpLFrzz~Qt;%f&#X%BPw!0FrJ-r{`Ap zpaLwMH_jEaul33Ufqg`}{F04|unn&tGyE3Vy&~zWSisE3f1FDhN@}4KC0ObsuJ0-w z)E~U4DZrV^A-pqO*4E?i5>v(hVg3ntf+aa&@3zS$n138Es}e_p(ZklP%3l)>kYHW! zZ-V#c;0nrpIR}4Mc}+Fk{{R2X+i-X^>b!uDhqL3divP1L?6S|S3plNs;~gud(t!h? zpHLeaatJ6*S@A#A{Rj*M_l${!L z%c6Noy<15Zk>}OmG{SBbhng)bk4LQyJsc^EBVFzN(suU~wKb{T%aF%4Iv`P;y&0o& z>Npjv!%ZgdX%CN&&%uO@l7b3Cmdg3QA#0~iy<1C>s!KK$_y32N7&fR#&?-}x@q%_; zmyB1RAycJgsVgYnPUn|}dEd?rn&QQQg8 zEQlsvc$@rUIjzzHeDbt~g7%MppZT*usH?s2L`X6!Gq$k*N&0^*5Hc2P$Ip$2&C+N< zRdeIv6$_*5$WY%F?(z>OuLek#=z81)+H6fZ-Lr5b-6Q{ev#*DFn{e!BD0bQ@Z)`g^ zG@n1!Hkk92n&iX+y^z_Btd(qlwc3w&l)HGOyMZz|A(6HQ>r`{nxpOdS`f`eA{3EF7 zrD8W_56W|3{AUi#8u1}uBE`t9Vu}ykWne+ZJ6p=%Y-3^Lf7w(m6X|Bqs8d9GrA^Xg zdt=vYS~DS=D}7L+*}l(?)~YJ^HD`OS{9*a7Sj?bWb-c zzwMz3k|CQXCB0)E=D>TXwwNhfW@>P4kv;k>lb(|qlZeCS?yN$P^^H4_Zd}IZ>?lDU zqSTlXn`ff=Kd$zy@vpf7P9<9=1Fu`ZRAO6#!);U(N+`P8&Ia+#P$@*6iue)JzPh~7 z_LA6uw8QW0+}X3nH(=U`%CTy7DdAHpQn*L$22Czx(IPID(lCe1XFVL9&Kl>4IM4La z)Tl^G1VxHn;!lrfOo_L@66|-syG-zE5dHB;m1Ko>q@@YDNCuKyBnv6pEF>lEVAP_$ zM@yTQB=oCD1N*kn0Ev{JTEfSoO&{M+>F#x%n%aVCF#Io4;@t|cn4;Yu{2SxxbUi;wRR^8WK667p zIx+FSRen~fU^_nWqmya`-TfnhB+(exNt31zoI#rVw-&8oMD{&;$`<67#m&~mO_x@! zMrg}qn)~n-9Cme)u=VSry0`RMTY@jW#g2ssOP)|gmGj2+E_q@aEn>uY(HjHH=gq$L zM`V^S7qHfNpzx9HDh}f(_toTj*aq5ZCsK0SxfSHxUrcp1AEDw1{N!bB*y2&Ab@?8t z9ZS$4|3{3rPH7x+@Z&iZey`T!ldMXd7NQyMRCavzC~0`+#XX00kUHjqLJD0CWMcV_ zMhXW>cs73*neE8s>%$DB4Jz%3 zQj%m4p6St%z8i2q;pu74JWE z_CIU`B!5e9fIdrA=F@Ith;!%2GJYIVW^dM0h?^^;7qcI%IHo-s^?y$IN>*{ z8u4zPlL9DAs#uziG;NEzL>ATs<}?ytNtapt7K^prn8FNxWi2hnVpH_#s)uFMjP+5i z;emO5c5WNf=zSb6x+%-NQMF^5CXGRX^OdrA7V6sOuHx!WHye4EPMSQgm&(&$-#aK{C$xZcl|a@)pNH?e89 z!|TQd2JJ#0EuUDDtTN>}0DKphUBHzsYYtyHYH4@_#iFDlv5t{C8vaBfOm~$`F&a5_ z{ch{5lM4bl)y#~->thnf+}GFr)i+EfiZc277BdBh5SRv!s$9B#Z|sAsx(f!LDWKd& z6cQXz4L5P9b_H;cptDg6zQ>(cdgeW83akhm>IK*y-)*z!rwR1e+gdIl&bdy3L+gAJ zJ;yPxMmpGdo}JbZUP1>=)p+szXpGanjn5N#Lr+aps+!(Bp}II!Fe5%Bjxs!^yOCvG z+-~Gn^~`Rj73Nlfy$Q@vXJ%>6WLeg*NRJNKo8(#N0@AtO{tAkJ#F#%t(n8ifjzT|r z3L{tfaw5fB0-+)whdkZ5Anq~~C$m#O)bey7j|4{ zxN(3ux>$e(wh$SSiCo<%ULAw2r_o=!}NiI;vcBko*! zj~8awxi+tETyo(;WPRG^X3rX{zr0J&VlFBc*H&0rn2GOx?fE|C-N=+Daw^#>eYHw@ zl*KS2BevZv^<38THzbo#{Q^(=_CVY7^FY2xdrDEVMa*XH+o{qUj1A^_ISV=5L*=Zu zSfr8I&$|Ced7)b(sOCq`TRMbYfY+U0NHxgz{15LJhQxiSAYi-M+%1|GqcRaHp+g)BR_@ES5V^{kw z-s?jrt`UFX46iXhS;iP2FVB-Am@ZUjeJg&Ek&qm3OnDx4596_ z7W>@HnS>tH^R~@)7fSmCNWa6PvOeWYm94{iig<0g(LbHs5_L%?fnDD{y39RG9N0Qe z{(oh$ehPK+{W#fi*RS!o=Zr%z*n~l^VW#ma|@Akyv1B)!;*Zn?PMOD9+-DE46?VyeD>M!{;5HDre&;G94C~C$p9OF!tHv@SI@^FSRfP}L68R9K;NG&YHOBeXMCDYn7|Eev34aGT>M1lItGTd zm~W4F#alZ!&2h&ECC~PD;;_s!E(#*i)#FO#DBPm)EoX1fktqDK-$%^m!obCuX}Xn- zQC<8gsns%8K&uBiPG zs${`=u)e7K%&~5+g;CEaM(>2WXDxOBlRu zbX}PG{N1`}fxyqQc#BV`@Uh|Q`;=d01kf~o8ovvtiaW=DSUO!yM{30kV>)K!87qVo z@orlPq?ek0#vPwwgyav3Tt-J^{4i`Ouv=8#qgG_>ja_f7#tL{VtR);bY4-9csO&QK z;%M*e0(<*`qT}r=(st=6(2c9l`L^df@X&(g`EXHBq;sgd8vnz5Dd)x=TEigj_WQo- z#>*eSN!FoTjS{aO4wf#T-dOz}d}}TjDLIm9m^1m})A$n@V-frV%A47#@}%9@2x(?p zSa+?w#r1oKW;=7v$GMK9IN0^{4cadve=XD!`!IF@g<}z5!v@6!dZ7$+(aFgF7uQ4% z>#<>87kyNh=*4|>>JGZ-*!q&8i*m&fj@%~b!{8vM>5?}RXOi) z1Bxxfxsi(u<^Rh<{|J`6p>UDzX1V9(>2X(f_Ru&3af{Dy+l>bxbGnmJdL)BCK<_vy zZ6FRodmQJS;&-RvaYljGm#Q`3v=>1&&B+ zxfRN*xpO^)Ej~+BsM*X=O1_XB?|%39Qr%{qLd?ZY4gO(pu zRLAyVgUcI%6a57%aE3NuT{FSy=Z393!ZQJi(YC|ogO2%pfh!+ft|m*pe6uwBTviT9N_ zl@{htI51{Sb)L~B;qAfF|5PUhxB^{Ud}wXu;DG62|F6Mbw$eJe!tjM-pdxs-Ix}O6 zBRcSstytki(EqltO~tFd!XX(wFF!CHZBg5_3~n!axfR+M<9S17dKg0{Vs40!dEA#Y z7$Mz$vO`+WX1^`+_F}Lb52KMj_T;(tvV(9p4G$~a3n_kQ<_Ri*b8zB>lj`$ub$ca} zEW{N+?cDg5k|?@uv-({Q<`>lCFDKQCQ$?;QoOl27k3RYSrI3mORo=l!ZD~KnX4bX_ z!?{eUt~L?q!2d%8L^(~GCJxQ(LBcw$WpdZytFZ+K4r>?M$3zqCZXji#87-r9z{Ycf zdw#>mSpF2EJkqFrm5Pg2jh+Mqlu57z_LX+#rxg_HSQ_4f>6i^Q+)g}IMO|Y|4(?@= zI_hfQshR_BHuF`9F%(ak_+J;AfD7_hkU)KOQOJC5m11-vQsu5xH({y!&GKS%Ci|v} z^8(W7FXXPr%ub(uVX_@l?c9OgWby()U&n(rpjq>MAb4HI$XC{eS~LVxYb(4G^;sfc zmoc%Zp~-o$_P9q4(+&u;=6wcTe=)1v1he||Cp1Arri_3O%a2jVON=*#4)t?*OB>&R zubf~;9_O_%Hs8~N8C$reWzfF#0de5e{{LDd;Q%$3z(f5u_fu^9*AYHo18zS!YPFcH z89L}YByh&|pNRuY@QTpy5Bz$&ML|NLx4yXgl~hObank26MGZ^s(7gd6zB0s#IbnB5U zA2>3u1q`8Aos}R2`&kK01FOFx-m3m-*K47f9?$%QQ*gwQj_iB4ZF+w(P}dp*$Ld|g zdF24H6Q6eqpv+Sd&+M4^?{G)t-6Z#XlNcF zdv`MXcI%gp;t#?IAt^yqqZYbr9O1`E3g`!bDb9SP3Exg8gymG2QVLv56>KRIVSizj zvLO%Rq_7;HadP)lFK4X55<3+(I8kM%9MHtH0UrlQYf16|$QW*`g!%yEw=5FIhA8B1?C}p&ydQ6W2)D=|>e6 zAUbEbY1_ITOu%;^x%smv`*fh=26c@g{nx9k7k1HxvuVVTAqypG)TKNfI74jWG}|w} zxu`Wx8m-U)B^|MEw%>yL@C!#R?`Pt#%X!*QSDGe{TB;A8Nlc_$-bK-+Q&{;bFnCzU zaLe}XxrMeTz0gO5g=##2x;WtQBRAVFz`7EI^di%omS(*;&+H4P_szX`gy#$~mP7R} zZgR$YZ}d>4`JH(L70B&0uJBv}mEf$ve{Sn{JXG+JAIpVY)MKizJ@o_PJ@wt5zCu*& z-fv@s@$bk-(}i5^k;==QkldI3>m4c4+66BG#;=YN`F8s+WMGu}{F&0cxpo#K2?G;9 zZHvBG#BJ^^g#A2_zB$s~+<-$OzEyU9bcjNqDgH~LSDjh~E3OXxjba^GLuzy0=wcUC+F^czxQ$V^?DwXRGo%95dLekDqK)L&07{whqsmF z#L~Fwidq9la0 zfMABPO4!^|xitTzwBz$pvUgnBE1zwv%6c2e_Qv+<8nmynz6)1lASF`?VFZO*n5cuS zjw5A>t8U76cy{&-d7Vzec4nak%sp6PW+TPA@jL+1UghOo3{@hzvt_@1w^?e%Q1}nS zg)_14%_CCyrHXp4cc-&Q;f&1y^>Zs7rK^A4z0}*Ra+)MVqLE6r*`%Aw%?r(*nY%!c zOBZhH6!nW60raqr(~kSYVk<_MF;&Sa=pdoNt&Alp$vv2EqR zP{`8&@|_JfwA_*njWPn{GxqaUKg=q4>9F+5#=^6gB{t-7?!#>zADs;OXN5s#DgI1! z|6@o?c5X2jA?Ojk4%rmijxS zmdA@6W zcGuq{qu?QW)!*l~%itI3F?RirKpKnrNQ6JcJzl4FwHcsLa55Lum80tj+Nb9DDCdf{Ij;qi1DY?`lytm)3`ww6(wB+VOr^&`T!9byR^5JP?;+LT zsAYJP{hY7S?A?>_E7apXg@P=`>ofV*_&ClBcJebZ$y1c?M7>DL1bLq?c)aLa3uq`Z zfZG{-i?AbB3k^SqQff}Npfs1<$l2!V>+k{hqjj%xPEc8o@w?KIo+2mnfnc`6L&J#q zyl?=47lRc!yf~X3&6dOR-lm#6XRYM;%ADK}LhKhH#Fpo*4dO(YTGW>=*IpS{rrIfJ z&`;oHM*wiA^IIXNv-E+D9YFXlp&U*INem$9#d3?^FItvKP^?T_26MBl2OB?E-?E8= zR3iu2w{POw0fj-3VNbB#XhjOhcDr*lb=`u=%H4X!XkT|o+>M$?1m5#f>p{n9_6O-1 z*2N_q>+J$NohtN`{$(HvrExD&5N&gpgJ2tVtr)n2GNkzLB{!Uo?sE)2N+1f)4O-%?`3Ht07ft0_L*6m! zTs^P(I-td9x6sx}>M_bYI!Y_9hI-;`ABP$4>kug#CjVq^8VA+0FZoAy@in zsJR0er1=iVZZ@#1B`#m5adjV|GR2N+7u}=^Ory7VU%rIu)Vp_LgtRJe6cExU2cW~K zu--3`n|=H0U3q(#D08aR)JW!5t%f`DY_ntl#`pyla1|i~B-=Z~f7ruj(qy&kC2^LhpRUQ>#f`$d9BWy(9?yrW2ObDj7w9`DCg2WNokG9(F@>IK z2zjwoKjB2_eSPT-`G4pHG0Uw1=t8PD#DR)oSu)m(8$Q0nZ7~&}70QGUiw!3&RK1k2 zC2N#il##$zs>2MWCCdvQTsG~C(j=KkVtcs{;0HGuU1+Waj}&~wpFt{yb0V;)AuT0_ zSz9>IPvB{gKM{`PA@M@0{Lz=P=BMNtuMYiw?>|4ljevEZAlpDw7j zSUc2OC&$s4rUeT~lfT$^i-^QrdO4M(q)ZNBSNENrvz5x^r>RA$WLa-K-_4szvdKm!fsj<9?SEtWVjJP`c*0R7 zc`d}#`C_a8T%KdDN06#APkkdFiK>i^@f+r zkpK2j7?E^$j!;Gn4*?p(NL1rvD{1jRmJ1$yMT)r;B0MPsvOP7wJ`d$AJ>PB#aQyo? zd!5gi^v_>tEsZ^Nfp3_2a85P!yLp54hWmhX8pMA52L=0QJa_aM1m~mtMec3nUo`7t zl}DIeJNH2m$MrI~)788+_zqCqa{iDwV}rtkobt&|;+*cCNjr@I;2Zqre-S5?9E$-^ zWX)DN)Gt}+2aAx+uOtEkLoeZI^LhEhn=dpSg%-*-H$R?pwne>UeCffjgq%!i=UY`8 za7yKXG=C?FEaMH%aHQSX36id-!|mc-y% zW-020uXKa3@ces0)1D+6%4F%lV?h;*j4T0x!T5InLte03RWJS18rK>czD#Ivlm3P; zge2U}u{YpqcY+5?kRw|C`l=J#^<7%*wUy(PfT-~|RmOf5c;KXZ->@ZR{f@B*X^e7B z57?m7@tHQ;Uo2xpbW)z;>=$GebQFI4DpWw18;$U0TPS$sXRO%`Z!%JM3aLt;+wxP! z)U~7t=G|nHbikQ7@ahsl?%|sWPIh0XIaSL;l7HehilHELPhZXBK{_bLoIH%+oHvQ$ zjA(1Hx^;b{mXoyo{DZ^caUf>In7)qBx0nuUT3j9Jr@otcon>AeGeg*KU7PNbg{q3J z+?Wx|#5Fl>78ywB!>-KdYa&npO zD$W#_1)0wTn-T8dOPE(R{vu)3q-BW$kHZ5an6X6RW|>g*jOV<= zvhb5TSPO84a;HHUT!>^w>j;kt@X!>V7xf_;rB1y+Dz+1=Z zAZdxdKrrQdpPyxiOt}~c<`)^86*FDuL!67uOu1PY!M}8X%e*djdGWOwpbGAMuYPX& zy;{L&k{#n)%-NjP+Qw&DZ)$ywsN8?V92J+zjd$&$;82pgd;@$-ojx$adaUJhkZk81 z+#>gWRNqyqbQ3VA2YhV6C0Fx+OP@q8(8~G=>l>{%fYKO3u-j%4QG=dgG@3ja_(`(1 zGLXRd)hV0$5l!wNUrczV5}--7(QwF9YE+Va*B@193_CTe&h7?dm{_HI%X!}XO8~v( zZFd5#?%T2qn-Y&Rrs)>R2*`-mk&|1uE;Vytm?)oCZ-!m+MnBfZp`INixu3ud5K=-k zTR8`uMz#OMni&6WgVNT*uEeOh7r^jzbgY*n8u(6`H znNyYSMpAfSO^nC)Aup|;{C|049HA^&`o_%f@w7((1|N+-5>mpegmBpyZPLmtC8RFr z&WD;-zW#cr_Z3SyKQLrP7G(N^#M?%jPi-@p1|rCg`&{ov z$>MI>nhs^y|Kr_{(DMN6vWF|<@V5NQur)#_Y(9EQ$z3g~x|@T2F`92=Z@kfJ>WeoV zX{K}8sS++dc?(!)m&!QinH&!f1B^xs@4jgEXwLZ|AysroyqQ5zJkmtF(b<_K!3cWu zP7)FzNi9uK|C5Z#BXS+lM&?|BtL1zzmZ&%eF&u-#DE?TG3zt=3Vv_Yy_azk4fBstS z1UtlV;ohTSPq#=y83UkPU0^f4%eSU`Kb~Hh_7usD^)Swe$8rpgS#e4;GIe{~I~smZ zGYX4f`q^2u*6AUHAMFBH9TKLu4eRP#cT zV?$;p18kgOnC$DiV{vq4E3Nne2j*y4@;7*%OC`(jM;V6Bq)TMJWsi(B&--YFP*HBa zD290bnl4vJBv~n4t=Ule}wer?XIe?x7him0pxB#NaQN#eI@yoiA*7+w}seJZRpg4KAjEs kqeP?dACkPJ(~sY?=lXcKbMEZIF7S_%yqa9*6_dyRA85Y)#sB~S literal 202227 zcmeFZXH-+&w>OFi(nOkopi-2k^xk_7q4zE|5K0Iw^sa&;#i*ck1px(VQluBfhJXg7 zN>xOp_YQXjlIQ=NbKX0~JMM@3=?sS<+1Y!oHGi|uHCOH!=xI`tGm{e#5Kuz2)QkuS zNUsnO5U-M+0asdMhN!@wQ+`I8Dg>q7muA7YM*iv+{$4>axTgyNr>N@5J5CWHH*Y_G zPEj>Z5fKL@QotGR;Ogt(!4a2VXh`Q#oEArVo2AyIx2NfTjdPElnc zA@CwBE+8!?WOZ`CgR2VyH=wwH5Ey_**TD&n@b~s};}lf^pCJf;7f_2eP*a7C?3;)*;SbALi9b8Xn(K9p%4${%o3h|J1zv1O8 z>SAhOazX>j#n%t+jlhwN9T3cqeb_%3>2h+_)!WM%&8{KDJ63JDulhFa`LKn{9fw#x&-bz0O4dBI1X(qKP@p)BTao1^B_%6RaHGv zEq}|CDJ9KN2A<-6J{kt%zJ3A0PRa(7P(MEr4@noOv{aBoh>O06k6W;trg^ZAxtOX6 z(gKXEqN}N?>#8dx;j8Yh8v+kOxGB2=MyO~3gkbJYe(Ks@KGLFy5U7(83S16$)l=3+ zC_6~1I{Ls2)!-pIFimi$u6~e?YoLf1(p=Ll$W#NV>}%l$fr)D%bu@fD{hU-pObi{I z{4BlQrS-jp%$>Cj5g{&q`s#j`(*9n4;%Y|XK45Gw7j;w9Krwv-A9G)6gFtaHGYxNP z2|Y)VkYFPhBWF{TvWT;}uuo8su$e1%DaMYf%FY2UUV18U*8oc^OUV!^e?3V(Rb!Dr zA5mo_xJ6V%%2dnG$z0viSk%Ws&A?IG00s$=3XqmU`6IPN)x}g`a4E2Sl$M%@2+Bvw z#Y@=)h=z)}xsjDg5F9FQ}{LcHCL{LQr`#l@8o9^yJ7Uiun-M&M2l zL+ofmaA7AEEmdI;LtPV;iH@qSsT9mBSYKHRr5)hzBLd*m@G#X1Hult!P_xh#LZKub z4Sc=4j4aep5F;VUARtUYEkdM4MAT4TqQPdGfQu^Pz81<3ZYT|eTCfNB2vJit^KcXO zkkSoAB24t5x_FcUmx9?K-m|f;_4u3CTK+13pR9w8V5_5 zn3<^uYnbT)-BCe$d1;uc2m2U02L||>IEE+-`DytC8k+?pG{s&0oz+4DlywYDE!3rT zJWf(p6^Mjb z1_2UGq$RcCT81zcLnA$54K*uaKUaT!M;(|p+}zw3p{nZb{ryD%(pGM+BF^3hfC1*tC_`rmOvS@fKR8%f z+eO_$OTpedqa`N{^nvq!Y)oQuzr}Xwh2VbU)cxgZ>Z_x z;%F>|3{rN}azW}_7@4cMqf|ZJ4TQzi1J$6KC^chKHDJtD5SCgFLTXl0zU~s9YUWU2 ztu>59)DXJC`tC@L5LY9Fr;nxu+)_GFTTMs{A`M4~>$+HJS*od_y!E^df|Yf2L_9+f znue|-mfjMou7S>uAzogt!VWI3l71l;8m5|hU_^6CEeCyNVLu5q4?{04LuaHJ(%-_< zM;sL*Ze#>E)fX{$u+l`Rn+1e8s<|1NhUmFyYeJ-?4Pg$ZzD6iVD-)QHyEaVQR2eB2 zV&vhV07y*O9~r{1h@tp>OmagBEG5`rqWo> zqO?W)AVQAjo@!QFMt;)f7Lt0RlB%9kJ`PZ6X{dyzi-(1S4^T%HlR)f(U0p=g9Ka`) z5N8R0QJ8M9G0IdA;pyq33A4aLqyzQPH}KX#X#;ef%*BM&v;rJ0#WZw~N=3}V$y3A4Kwr`{zzPNNbagjSg@XIke4M>uCMbjuGDOT=!pSfQ=IbfuVc;BW zgmQ2&PjWU>-e`kX!wtThKju(E7 zI+vQA7?MkNLIzt(MD!_DjY}hsz2&Z@bq198caN98O0|8Kmuit8E5qMsM^WQl9Q->Q zDy?AqT=7YjkT~H^Ir>wXm79%m*gcUHaqK)aXb z1g~74Q@gC{!WDFcru4Z-hNgOYw~vQcdO&cCw2=vqYBU4UxfbNM9p%msN~30n$#V-; zZRwpxQLPQc(L}_aO3OPHbQ=qOvu2532OoDXg${iSN9dK4*^r;`gos*82QbaC&xSM_ zM*Jso`8{s-dcyNpOa>mj#tBQ*RYJ{kOc6!-k(N@U^?U^ia{(?x#G1a)3q!Q!v&L~P z>)G#hx8>Zo9aKp+>*TBrzgppWXx|M$3yNb)IR^-u{doG-@eD}SC>0{fRjukH{LkMCKW`F&h>EO@qpNJd-5V?*|PvLlxYizyGm=t8U6DDY# z^wc}K^pj*1@0F<5OT@Pm&!xdXPHRxg*KeIc{-bP`47o@5r{}++Sc(2bZl}07s$tcl z=n>#%HF~-cKdkhzBxjf(vQh-P6*kq(mR`zvD3YeiJl`DkYA9swZP}lZ7rl>+7vDh7 zY_iNp^r^+;q)veR>k(48_}#B`peh3*+H%K3Ldb<0v-2eeR8EvIx{v%Uvs}$VGUa$! zdgru^XYHq#RuM1sUU>0Rs0u?FSM2eEXq|r==tb3fo=%u^&45*RD6N+y? z%;tSEeoWuiwN6Wa!oh9|ux5o0qCbzpnm^ELOC7KI4yi$W3aKcgGH%UP6qlr(x1}J( z!WUJBMb-DHA77&$eZ&FZ{~B#VwbFbdu>{ecKq)_(HM{X+<&Emtz_A(B1AL!n_z>YF zMvCj$;z>&UvdeCm~eFRfN@!X8C%U3?nx1Y1HZY23)Y=MR9?TJkxqrqwOC2F7tO0;BcP$0y11zEYr`fweX z%cq2L=hShASd!@3hUL~^$Yp*^MbkxP8e&2g(lBwCbVRuGJ$0FgKdlO*a->hFafYb% z6d3Cq$9?L?^I&k7VKQdyWVwWZ`0K4d4RC^iRGo;YxP2fA%&(Mlnw`6Bj}3xI{XA(9 zo6ygh8vz}*c@KML3hhF4#&7+98T%vq)rbeHl&3cbEucoSV9UFZ5vbi&9eQNZL(2xoK<58(RSd^(Y1jTypfjP8eMyoB8)I2~Q; zjFt*JZ_oDiLkXSe$*4nEtFf^EEu;k)=k2V>XPQ_Z^5{mX+{82vUc|E<`%qgxEn1TB zEK$eqN=bIucR?sLIUB|PWUqoS3}MIe_H$c3o?IQ8E- zz6+=m7DUvcThlloV82`O+p&~BlG4ItK8HnM8W z8xRmNlk?}8f7*0FjKu^uHE?1~0v_@gdU5ktBEW7eKV9iWV38V8hjF4WIH;+>C_#_? zRgUlHu?CP;&Pq|^AhV$XkUi3&jY|fQ?LH20JQ17j3TEa!+ru8KY(n={}SF% zmy!a=w7KrQ)dY}*e~}SB88yWV5IfY(`fAG>5HP;O9fD<0G=_*rji)Q239pUH;GxSp z*V%)w0_-GTaN%S$lLoL@=Q`OtRWJ$*S5^v687|`nzKcFQd8Op>tf1QW~+?Wth49KYV znxi*fMh(D2BoDN;#<1Awz9c(;!bVr%OQjxY6RsEl$l@T}&c~2d5(CIGl1}3xqX&@L ze&X=i#%gG?B{%G3)c;qt1|H*o)%t(c`hV5>|0lJ!dY%%oeJES)q1eLA%>2?{L(kX? zsBAI+=N(*VmD>*tsy1EoPJQS4_p}>DGiNeh4!jSC%06zQDXIj<>w2uO%&}G%0^LW- zG!ez&{8I-o7#8=?ihK8hRvqM#J+DIGeWJTztNCopF6I`7vWt~zn~RY*8RpAV6fZEv zKa6fY%W;PbXS7=Ru+xUBdcOX|oW8u6A+!3}*uSr$kRf0>Sy6AG$n`eyUg?Nmo~6QD z{lj^>CUJzPf#;9V*IsvJSY_F%L9jT{bnFhJlvaQ(J@4I-jkl#mz?LR))!Zv*sb4}} z4*z4v&r&xy1)Ytabn0_>daF}PHm*-4D!SFG$X5?%1dE;nLNWfG*VML`4+5*EZx<6Y z)~`*@RR5NdPZtOocr`FvrpbDK|8Q}pCAI&;=B6Dmid){N2#!J2#s+~ze5ikdK&sh!8iD{$WJe4Gj_N-r==+T!4LT7 z+8doyHovwKBtwTryc@L(FNInx567D;e5Hj7^j3{~w;U84?0G$9%!#y;;BEhC&TaN; zHZd`Acek@gd?{>!)^c+)qpYP*<+AJx-3$rSgs0iuj%?-GyEgbB>N5@U#gU=j{)*Gx z*%mSLc>C~&FN_P_CiN1;MsB)y#~b{eah(gaID+3-XXLHBVRn!EAkOijdo9&L^6-%F ziBH)A&KfCLsCE-4eVpXZ;cQ>~6)%SzK4F&LH*8w_eNlORZhcVc;?|hT0o6q*YoE3o z=f+f*^nvTlQ>}4uYM$@do@m; zIn!^pyVGOja$xU`Bt>}(eeve-gYS&HADBMOJ(*1IJo7iz)?a2J?olo+aU{BLjPn>ET2BNlYkobYO(AN7@;)(e3x*zTQX-;ep$=w_1jWT z*m`8c2;0OoV^*OoqM0kcr)G0aFn3X2q7UL~mpg4{@d7^c^UOT!m#}i59De_cHys(eH{RMy8;-og zc%|ffc=%}eTmWE5#l%`uc*I!I)^C~V`GoYlA<2JQ62@4LLV9QH<~~$;jN27v%bFDC z1m!m$t|sL6IF4u+m$ICv_;K)uf<8Q=raiv%FgR(kamOlLA=t?HqrBvxiDnKG_EoXi zKkvxkg?&I$;gR8G&F&biKupAe@HgVu6CPY(Vt?lioa#S)S5&3FS zK47S2>Dv;+PI&L!H>GUKdi7frw@6t1-&NHA#k6&0_ueJZlRMZi7*Ke;T(h}U`b1qaxr9$S?!}1>;`C?aZ$?(eZqa9o)aNrXzf+|8ucR}XDI0o zIekgW&qxiYiz&_ z|A700X6xC%)+}f3TBgt~b1|>=$s=kkFHN0(wnFzKsK>bS6s81mmIx;9SA|o>NaD&L zM@3>DkJfMeUVqZRD?c#4p=NONATFC%f}Z`|NN7jm(Kq(J`LCX&OA&`2LpeFG#vNwD zcRF0Hf7ShbI^*oc7YNWVwb@9{J2DJ6?mmvZCrSbOi@#Gp;(dx%bjdlQ;fI+AHA8Gz z0z^rUpy3kZX1xXBWe&;m7rv<3TTiA97_!GS!EOKD6nuS>x z+cs=6%fhOX6kkRPNx?fVP@N48NE+Cl+*?}DX+U6RxmoDv16!YJ&b{kR7sT+?RFEtk zXeI`&PPRt=5f6MhVm0a#XgKuc{0A0kzuWPzyIX7;77u17X0$12Vo~i@Oycq4x$NAn8I3=W-)@nS zmp7EWdC9`E6Ebvjo%M4;@ox0G;!C~HWXH;M36$sBCV$Z~<1)+h+$rCqTes78P>hYE zl&s^3`c0#(m4PFQ!}X7%-fp)_i-{CJP|z%fZ3n-q(@QRRTSCpw{PFelDNlHu+iTg? z1qz>$>Kv4VJ-Yql4IE7QCTqazJ_8Dq`d8 z%2=CohS#UJD+)7tGafxf^D~!NhrbGlePQ-<90Fm_NFh-y(RYD;{04sd$~ZNaJX_ z|40s)7fDa|yWs^jJ^eM8!+!ZN`>#s3^{_S{CDr^uR!_{dEA-gb+7-62gc~fUcip7+ zT2Z;_sHiwn;*?R+Vx}IL<(tfCxa74r+m4Orw$+2_e;ZWZ?w#@OQIY?VrN1z3Pxs8Nb|4n1s}?BD$EI zJe8XPp@@9zLs3VDB51dw>SnoD-gIBKDIASEC4>iM&nq7?QB+lIBj^X%sCcR16oRZG8~A1@xERsQiibdtW7 zs8dz#bZQH51rdkAw|yxo+xjYNOs(RVCzc1Qrml~|;IT|4jql{z#>vx0ig@1Kq=K;t zIUoh_!6-e2bM=uSeB#mO4J)exKC8P9Y4Yd3N;BHgi@nAi!>kyzZVyNUG1O1itk=K#S&qKl*7FQ_(7U80ec`HD+}{@TRR^1e8b#YZadTg*s`uv z?)U=LtAKMxgh zy_)c8cXh2NCWfjZwo^j3$InUeg_{{s1Mla>q`MNlTII4uo)eWIzOP0Ws>9%B&u@`B zDejlPC}NM8%Br1L{8ooohW$I~{Ob5bys({ivxPa>sDHdVK7nJAogS4Qh%8mS? zi(h3e#=2mFdmlKD&e^@JqJBEq)3d@vl^wp|#9u#1`$$AQRPG=yy*xF*XqVNc!iO4b zL!t^nz#fzy)Q+n*MIVtl`>-w7-iZ8u9ol8>&s;kWr?N#3CAJ+JvbRra=H$0rQuOoc zb4wtnneW^*=hk^LUHzM!#*^77Bgwz#fQmAe&kbSiQ}N_A2M8`xdSS9%b#dQ62C%Shd=0D_h-ms^(&8-*}R&+@R$3cRYVvVW`R> z($Y8Ea<2AMhc2K%aJ61 zvq#FP?mnW}(|?rSkz9CWpxgDiTm2|cL^0_Yn`La2-?FM(h12n`_mmh&2p1^!A2H;} z#>1YQnh!+BT^9ZDL*Vs1xs_{^U7G?x?EZ($XL(6kb$0NoM^^FFqu;B#Sodag%*SM` zFE(fF-gwo3S&C_qG>=Du5aMrsG)a%|j|2V0Z=Z`_8CGw-q#SPAd0))DF<)-KdE522 z;+Q|;qEJ}SuQ~VLuNrT2f_SzYx)#d~KIpx+JxaR~(eDoJvAOQBbg22qd&b7Z=c4PE zZAe+;cBY2BtWD3kNS&YKLDjRFW;{MUKl2`SPFwM-ZTJ43BmN)(@_^U;gCZBZyduZ^ zWD_rKx(PwwQ3lEz`p9iaojM)8VK9@9H*R;781`5nz4Q3c|MJLjrWD1laxEuEUwTt$ zG~(x%X}0&_a=L3@dTWecO0(!6JuPTd&&oX1G}M_t$rhAA{h!dA2$$?F+p!g&j{cU` zgZj|j>!fhDxm@7KSa@h-#KANpfAcS+zi*bm#6-C>Tp6q+uW4+rRe{a&iXEsvP(X(w3jSy#bTIL?mzQm9fh#t^bTM@D81LQZTi|gW4+ze}6}l7`Ut{%fI!Qmz<)&6Fn&@AOX4P zH9_u^TvP{0=|4M6VAHl9>xsO$PGuOj){w-t6wsT`NB^UzjTP9xxkO zxU_H6r9yPk|8<+>2utK4KYv5mfaNf+@f+*g*g#@%$)+%`>4$nUKgmsyV{({?5Bv%Dyl>OALHG< zEPE`^svG>zM(9+^KF0R~jBoXvj&Y$OuN4+d#4L_@%dKTH+B}U2y=-B&r;*J3qUoql zQfjke=DkN9y;=IS+{zp_F=?wLQx%jlW~9olyZ7GuL4;989ZVq5VdVAIY=fiG+H(U~ zP$471^Og~~r-`_wmOOq*8-CI^Tds} zNt}Xa%B%dzSw#KTHU1xPnjv3OeYvolGyA~<1vlS;F9Xl1L)8YqP=aVa7akn*dy(fl zXM?PFb@N!z!nFPDT@OUCR)Ro{TJL0k&DGr{+Wm-wtmIF$!0!I}ZPV8aGP zgTwL~h|~+_5Rp}UD!WuJ72oIC0DtnP-yQSk968_**v28l^M~%^s*?S_Uhu{3O69Vg zh)kBVA?hR5wersrL%heNr#U#g~6=|8GK1#%3*JPu=qMg~|`7T&l0hd^{uNmD?*} zo$8pi5%LKJzpydC%vD>Y^mel%Cc3Nt#*10;gby`E1i)NmlG8l%=xS+=QI6KGI@nA4 zP}Rm2S;Uha!tzDQUVfua0_OuCUQL-k)RbO%pZo%K)X_a^17e);2i0j_yO+Fka=e=Z z-W+e#jgk3=^5RESGoW?BWgPXJ#s#G)RcQe}jc*)_9Ga>1e4Ls$L46 z;>1=ahQj`@(w2?r*1e@>^+J}&cb+%2N6r#|$o4JW^=w|?qux|+p}b3ZVZC;OX8bUG zT3u$w`vXy|OPStdkxq{syL2ok%VScyX2(e}%fJ7YmGifb zbi0tFrdqK-%i$R?mv~mETQm~JDvd>XKTrqTKmBdjlY7%)3OsT>G^h3ZX+R)U_dB~` z_IcnqbK$|R7tMK2wG5`Cai2o?dws>vscq#?ggu`Sk35Su@d!M6o7BJKdJvGh6;JtQ znD+e0;v4Hahx{q2Gla(X&oD>ht7xRy0**>i*P>O|VS;UIYFOyXPGNN~VsExFp%h;K zJ8|IbcTlc)KD#B$5~Al3u*9^_l0(@Q5xKQ?gLik|^C12d$)W_btt=~WH2dOgnd>>j zTJioD@rj43>S3nxj}J9XrKOpU!@i#&?32p2Z&$_j9`qC2Tp+EAez7&i!g@9gTQX_$ zBpmIZFSQYE?D0{-74iJQNP-S&Cd$$!ps>4tbcy^d$2Xu zkLlzqG_8O2VrRIzOyfd;Jjvhj7KEbXQZJ0@=Sk29SpW6pdL9^#@^>?K@P=EjYsCdDr`fy+8=xNOk^zZz- zA{zZ$=9n4*&?qvY!f@vl&T3u9mZq;Xn^BeKQ@1T{5BXAizvQp2!?hnaagS)2tS!qUB&XaHeX@v{ECIpiUqlFlH1 zhOie+HjJ0&4p1cLF}pr=5Vo`b1O0Mog>er73RNZN_fCo|Y=-fz4Y*BiqDDqQQ;m0Y z5%nYx_HTL(3+5vUKAT+8#R8X0+FY}Xp34fB0Rb4WWI98brOO|EGR8y+Shs&ce_Bru z|Lh)1WW|#9E^CK&1?$N=&tF+)!S&6YB6>^{y+uIC#9+@R^!8KU?1`WRT~}g{DZIn& z$&KA(y{{2-e7lxb6cY{Yf2s>9c3`8XH{+t!Z6yytc@5XQIh>rONPvxKcqi$Ca&8cK z*I`}3{XeH(!tVPY@)LlTxC^9TkgIMoC--Dyy9UHoZZ6yeTgu!RE$ZVz$v_a1LM@ht z?+aQ7y$6rSetZ=nvZ4bZiL=`shk9Vsyb5f+XjyBxQ4w~4E(V^lyYeS;nM;kO6i1*S z*bT;RkBIQKD%szDpo#0eTQ&jPCjTeGRc_wQt^23-%p7datpC29%KkMSY~{QnTfr@z zG+!6&Xb+31gi`Z>4H(T8vw2){3flHaaB)>@D1fagl7b+`0lX2-luJ)KUKYqYLF0g~ zoP#WAK6`sdjP|&1Nl675B@Uu47kpIK1fVXEl&m@%-4A+AJayv-@M`A;p2wup499{E zr**}bH3c!DpfmKtNo11I~jo{tXt3kSxDamQ@tzc16TL89@ zBL)Jtx8My-Dp>6D$vyhmMiG$eqf`Lr>EIls*g>Ojg1sCq()7-gMzScS5FnyD-Uv4) z!dGNM)Pw2NWw>q-rD32U`TSR_NTR2}NdMX8Gl*{u{kIx_-K!-a_@8Qa^tSgMT@4-J zSEklg<>To%I6%vY%2%5+INUJ20O}Aw)lO1&R)uo8SsQc@7K^_>$%`zC_cXvBl8 z)LtW;(OR|y_NHfudy@~_YUjJ@b{s?~$rD|0FN>2Iz;`p6kX`a=D?j?}TX!yQ9s~87 zdBSVzl_yNf_O-J0UQXn9`oKgZg`Av_S>bTT_J8`iKgIw%$me`V;)+lMu&rbCC&u7^ z_V3|(YZoduWUTU?LP@T2a1i}CzwQ26Y;^bI1quY{vJJ1VW8yT#?Epo2083@l$+NP3 z1^B2m9lB$`u2c=oWl<`}4Bk0^A^&gftLZ~hZc>EJWQ_-tX(3^KvY#P=wj$;FisMt< zb~r#j7JxN-{!TOleRQ#SNBuv0?@r&oCnZs~uUH>osS_URA0VWnqSC4x?T|O#`-Uus zxA2D#wsU{5i{}m=4IAK^CapB?d|(8wt=TzU>eyetnnRij7ZEs{O%$!g1$cXY zY?6y?sQKICl>Ftr@>`l^-HG8njk&m4(Y~*bZ){B|hu>J?kC?)BQbc=#VC5rh%uO8N zf_FBcIrP7FkzJ>L)muNxRqVd=Tt=Fi(}8^=sg>kG^S9Wm<5f}bsV)x|X2{;BA;Ya? z8`~iq$48e3#bW)LtK!5fVApGevB4KnjQFW_lZS&>l7^#~#Ei3end73!_FcE7!o`Bg zJ4KdJ_{L}*>;j#K-Ksu-(d6_Lp8g-(5(UWqOJ}$cZw&MaCj%q#vxfVt;6TItg)mFB#1>^YsWp7Z^aV!Y=(L-ecx&9L}YFVw5o zR$Aej3!8`7H_7NQyZW6;^L21I;mhp_>(x7qC(iBzHr`A%X!kJ!V?Ka$9w!*U8(*-~ z|CcwGAWCXPzFYe|6B;mA`FMBpPkN$8uQrh!?FFzEk(jYu_pmdOImCa7QjR>!02dv# zDg*9g8yGWbfjF4RkR7{MHV3S~J2T^p^Oh-y=)7*${-%nj4pk4m|5R(Zz8dg);xGBQ zpMBzfEA*#Af6;&<`k{xXo9DdGN^Wym(0PTY_%6p-fbb>Iohgm$1SQ2XR4EIq8+3AM zc!^mjzZ5aK+{CASS4glt#hAtdz>w|?{V)~+U3z1SPM}+zQ4u>8hmYf}6b7(r%)gR~7qr0oEUR)JbTYeiimh9g75j$yYMUmx-D4%J;hI}cYoERV zyB#(Nv{_*5YREHu7xWDP_cj_nr30))a+PcR=CPVPL!u_v`CJle@R`^%LeV9i&!hI8 zVthZIW=UB1S@O4-6snE1c2F-{PQ*gS|KJXqwE3P*tlMw?a&ux$HfHbyc_g-@b!3OL zejOM(t;i%d_G4IT04ygB`zO$I@KKix4Hrz(POml1W3m{RioQ1p=qanT2T%A$kPQ+{ zTwK!a*L+a;<*Z!V#qZ13Kcf;zHo`ju$)4exn#Qrjr20(h#sXr*oTDXg^QxW(eiOC}%Oaml2QDewXv>=~$&SO`EwX zKrvWVsCcwm+?*kyI=&y6uCV9+k}E<&T~Q~8=4KXXYnS-N+}-SKztkomOMMZi!fXbh z-#d1Ti10f$7{)gtM73rDq9+dUsUYe3YeH_zgPWqmI?{4w`w?g}-c=!#Droy9en0cD zp?SgkvY_nIpm^b<2q{gRJ`oTOfg$f7T%uKm0eYReOvR4dVcDP{O>n+uT}6+J8jY^n z8yEckKOFAT(!B~NC49m7QT+E>;`oej9YlJT|C*%Cz@XGU&I?94Qq1^XWil+oE@XB} zUk3*1+dJCh%_1dj&}63tcXY#V9kCf)jZWl{zY34ZFBPxXNY@*&#Hc~WwRUaOnKi6& zSqnB2URG*Nt=TK%unzUGqnfIwR(wzuB$Y6h!SZhZSw0K8=Kgv-$qMZi9E9hu0eg4f zVE*Lb6UaZLu9O$%XS_Xi;tvvtyqM&7fsD z;{H7|jWZyvPZJYwWy_&HbNTu#DMS-e&!N8X>>5tv!2vFEn!o=#z(vZzhm6Vh!%Z|| zfzg8Gad=IHl)uC8qwLniWk9{Z$K^ z7^?6u=fCxyKao5PwiCR5O?3_rNwhnK4g0Kp@}k+09p(Vd^3Yikf8y#m-BR);EQW^I zVi?g3(8bDtCZoQYO2I#Z#U4;}-J7netrX4Ii=26%O;5%{DuJk<1g%Aiqbo{@j8I=| z_)Fe~>{hCOUVbE0CYOl&pANudTm6|hL_0(o!vTSsN7U zL%MDu%r29AAgAuAF{TT> z_^`@@ll$}6iZpE*8_dw8CpDA1j4s|1Taj&x5ZLl6b|>J{TT*Ap1P8?RR;*)LbCePAZOLDFD|>%Fi9_OFQ&DT_wHD0P!uQHj{M~%hUhO@~ zM)#RxayUlKxB5AHqj&}+F1}0sWP|?4tEXd)fFH{3KL$@ex|s7a&nzJ%_GRF3U(T#J z#6#Q40JHVNo1YTzZorhS{L#G}S=VQm(0WMTDrpVBd--dewgR(4`crmlm>|ezY1|O=dVk<4)-SoJK zfgn1cI4MQU1S15A)Y_d2XdLTCFFI0}Br#23{v!^}6mq<9RY-$&eXr;}s`e{HId|J+ zA(hXW`pGEtO9*aJ;+PH!EYxiv*T?TR1YIq^o6pV3&km1-4206{e5rO@KwPrE+n$_#lJlN0Y{cGV9fvlZ{r04xE9$yY#YTG-owgPVd*!{Nb;bn0L#L z@*GVM`)(QgUY<*^%IUjn_!|nza65w|j01Rcm6*YQonSSvheDEX59~eQ$Y5$K^wNh& zVi_HxK#V_dcg)>vHWFP2@fpd;4y4&lVoooCS%^M6aNq_P_M*|H z8-3B`CpxsO0i4o*j!3ahL@QMt{t2ASlAsEq_VZ(Bp+4t6w0hn;$X#jhUy{*hJwX9U zxD9>ZWVZCaZ)bM~B4dYqO?{T^lfeCYw+F>@hdI^^0t`q z_QTr5jgc(9LB~UQDPVZ+e83=h&_E9ZtE7sxT?p_NVOc zrzd?Wm!k0n?jfw!X9VYc$4TQvp;B@=frB+f@wQ^l5_gBEVJ)Srs)Zru6T167@xSaQ z(cde`%>0(P&JPZM{^MAky23oez2aOZ`>Z;#8H}c%i|K;>~Q(hV(k!Kio}A%f9Z-8 z&+_AZ{`Q8EYnce?wjX1rYCH%+FV=X<0pio*qE6311 zmn_ck%rH8U0QC$qhqUqzsP-seR$Pp#lzRzG``Pg4 z1w5bFn4(*5sgruO4^eqC5M@wCdznQmLhG;F#G#BKy{GV{2JG$8^`!IcvkZ_jh*g6C z|Hmf#N}n!93Bo8tZ|M~) zZGvfOhh{Qu9=V5h%x5=2OW4|AqvZ>nkYue86HL+kZH>+ol>hq?m=v$*c0!_t;-A_O zDwu3Zdd(p_WJIf81x-2juixbWx)M%QbGC&vn;mWl8UJ3$^rY$Ud$h*wSG9uQsUEBw z!t!2(zhy*MCFeJBY}~U&=xi&5@%^^k>qipfV0{dZ2osIre_bI;>42C=iARux8(P_L zi%WZ04Z}U1bvq9)Z&)eLB`18#v2ou&EC0|`Dkr(nLK&lr?oV3>#)ISFgJwLflbt!n z3|(>omN!@-p&HkPwpS<9e-xyL!tc6~gNU$*I^`|i#zm78fk|1t6Oclmy6t2A%+n&T zC;clBB#0&_o=>pPU>UCo;Apa%+zC?%8$-UY4OhL{%Xwgoi451c#=cqI9z(>d^XIbc z;;V8hCVH-iNNs!y6YUFTHsTTzx2iIjU_+}Q5tv#k^D$B}ixvGFp))j_nsX9q`i=g78b;tP?|CK^#-1ZqYj~R%j z9G?&VlV}#{v=;RLFa+_G**X1i&O0d4zCNe>bb+@GP4U{1)LI`vq!h) zo)4R3IEnQ7GUIl7NK1VRXEHrP77Dgw3I=T)g%sG1)iY3oqQ9 zZ!OT;(GBSCn&Y~pB238Hwz_OBp`^d0e`z^oKx{FZ2o?(ISt;X=D3 z5wF%Cp4r;+3vG9+xxdf9jL00^T?lXg@E%>#!ZgnO_8AfH&14qu9c9&rC&)e&1dG@~ zUBdVOfZS9G2i5sUH>s=-lVABp@4RKAdp32mIzxIo8Etomp35S-{o7pmbWg-c9r^R1rEgTFW6wCPh?Jm{)3JFBMy>rC&+BXcv=DuHLm+z`O$j15Zd4gzF zmWfjtXZ*2}nQ}Rro}-EDQ*(K>V4IhORE;xnO^SbReila;z1^-ZWb2l6#dG4pmsjnV z*2LSN3ws8i4INyPfU1Y>aq{WJGx9WAWpN=HPISK#8wG9pY9!z&&5a<0-o8bGGQUaB z=AR_q0h22~CB>@{R2!k|zxcU^t57!)p7Bc3W3IdMUrPcegJ{0feDB%!#V_z4`BmRflsg8hcH8YC3f)V zgY@cqbSIUA6zmU17^M)U;}eA!4CTtzZof~5W^;SKdKE3hw|ODwFHhybn~rwb;E>z8 ziiVirA&OWAT;Tx=)5OClde@QlBMH+cezJ0b8@`3tTO^<{T`Bg->g}mRw1j42M-Mr8 zb6L|~GBgZ4zU3&&W#Q2N7hPii)JgQ|@Y%a7?Kj(1Zp|*PP}5)~5&S;K`77HGy&Uo7 z07s(v#q6NTXv#}!BeE6kryCl6WjN}6RN$l-%rdRL8JmGx9z!DpcK9ygsW=wH@8<`_ zGEX5KAe+;h8k1RL-y6@&yiAIsXICR8Gs~nNFS+4COeUyZ8L=?(y^g~buHNeFNKD2n zu_kwe;)9+5kI7?>Pb@jNdyEZEOetFzSXaKZY!yCpL{qy7zR;Aq^<38p-bt{pfSROn zSc#_mDwm-aE7$UV5v`Y6Wo7ZlAvWua#Nd>}l_h;$Vl7%X@uOMjsf^(b zDTrCTeFlBh`ildXI>LXOfG^7Sry#Y^%J$PE^NYa^Z?mfxUhmXA;hlH&qUF^oZ|RVn zqqBBspF@w4<}7%7ZqWR>0w;z@q|!uMti&9!@V9+4nv0-Fpr2AFzj2dTU7|BjzB%#W z5E|b8>*XCLKf*)QCN$j~;Q#Vz`pQ%)nt0 zDScQjyE^ojS4h7^iT%2sTdDoi@`?QCW|8Iw`Cn}+RMu<%3?(_75P^VN0C^a#7o=ZsHrpc8kvO78%Mse4l0%?j7k= z+!DpF3@0W}hT$0YH`YDE%vZaFH5{Fp;25G1=i%-TQ}_0suJa;;Ml_q{15-Wx2bUJizckk*C(;b9 z(of4VhUzqHD`a9fc|Kx`iBF(RlqaV-T4hgNWaH(NvUtu!Z2fb&dhl(k^>(Z5D}NHt z)20pdd~fYz)6<~!lB2qX0sj($e7jci*MVr~@SvP2KmR}MePvXYTNkdBl*9%B6{Vz+kd*FDK^g(2Tcn#U zASK<6bcd95iy|%Eps?wX?z=YXiEn(j#{GML9LE`Z4Bqvwxn?}`nRCq-@&ci4Q}b2; z=RX{6I36^Ha`Tayr1Uz$1hsZ*Uz2`S{Wrnb{Zz`NmseQQ2Elx&_2fzg-ZSEQ8hQF` z;D*GFPQHz=TWuE^mqz(kbVgnbVgW(76B*Rv?_3gp@RH&I=;$X?b@(#~zgV6Cib_ZJHRz?I)9KyAwI9erDAbbDj>g)>bk!Gm{BD4#N) zaAKPXl!2V0o@!DT8xyx=_w$=S1y8ui<>h#LklRORHNXAJG+qmkt90qe!vkGRp%r@ktrCo7pW?sux>x)nlaeE zbKvw=n%Zi5SQ}q*-ykf5}z zAbIy*KHFc5v#7KPBzIwAY{1$mkj}IplaA8#zaO8+Z%M%`JL-K~?%(?c*UB+Yvf)@i zQ6&QZPiG+96Mw-NCt_B!FVl3MsG)j5x0-L5)H-sVVx_ca_>x3!y!d*DN%hriT6>%C zV-#6cwS}L_hzO-9lhsP;xV?_b8XsA56P`bm(hltm!q=jI7Jxxy?98aRqYH`%^A4xz z*F$nzgrR;^6R0LhcypVgeee!4Jzm*e z>Uw`15Rei5+Nek@>HPEh`5Q6{ZCStwNeX>7Zzl3Dsow>=u14Pn;7?*{9d18y9haj$ z+&dG$|CI^NPz&Tva=BrjuEqL9n62{EkQmX=gnxMt&xXI!1(B4zQvP=R(AAi8m1`d zPhW^MC_$6l&s|Bmi5dOS`f_4hu9v{WEt^2zXg98Z13R9hcWIOleC*C_CKG{k#$q9U z(>;$7G7&K&`@Cjow1JrF%S~>}fp!x72jyh9e&^uVB^>Iq#4MV2qrIXVA8|ZtDzgI) z$-?tmMk6yM8)z3MG@iB{i)nXs0^PWcKoM2;8Z_jubPEyn%0XqA+3$~1#}sf zGB$#R?g6`!5+yW60*5x=4+x zr!sccEV3xRh2*f)hRzQ~&3nhff@uCg4zq{KtZC;$XU*=#g75-+m>&zL_pS)=mMJOa z39*^G|A={plIiLn@3$jH>f)HtRxm%jW&-g|^J!aSZELlMR9S$`bC6@PgMqJqMtiH{ zlcye#|9Je&M>%m>gGMm2tF=BFA1F$eSWA=GqlVyF&l=_dnDxR7JrT8a4hFOUB9v(D zJ4((vvM}3&D=tnCwFXl%dWrx+NfeaMhSB)QBbwg4&=!$MOP-PA_8{HGu)K7#Q|*QA z!=n|O!#QtrOEyZ6>F)2j<~jj(zk$>>9uks9tlpSKjVwZI#qg=Q6+A48)c2zI3!N~hV z@}8zwRg&khwmAlu&A7_;Z%v8l!AEnP3# z7Qyv=VY^P0-p0O4*hn`nOgP;Y@vGDa5u~J_mIvc@3oxoPC+nDwT|}Q1VBQ+$nXa{< zO7ph{eSDEdOgO9^gMIDU>wAcV2=2gecTu6d-TEiJ<-Iy-5*ED+YmO zMN-P5IC^{gMEuZz@#K6wfSjRJ(XVhnhYNL;QP}KKVBE4CeWP5{h6wJNx8!iVBOYQADUW7{5eS)iA}q zxIClKD9W&o%u;OkznE2yfL?@!!57RE(^u$3O_O#N8T!6piOPB7^B*Ai>*y${^evtU z;$xp*Dd5R+Q&aUj;O;24U_byvTU|qG1qmHIr4a^q-jrzd#nE;%kk7C`TOYpbk7hyO zM3w>d+4s*LA4<01K#iyztQH>UURi6HaO>*h65P9e8RGEm!kH98g?MngY(mp9jIj|n z?W9G3U3)=z|F}FAYiaTy+Cw&9)e;kl9SNsXb!Bn+fbx)+~SL)#Xsv_+p~#MZ7yza z{8h`mej`eHeT&LDrZ3j#8ZycPgN1gFftozm7nHujhE(6Onctc51tWaZji3&2(pRP| z`~GYo0p<9UN5qH~;POXH5+Z^V@-hSMlK>1HLVC1aZRyf3V~zl#d+w{)s1bnBp+V$~ z3ML@cfV6G5$9MUH)Y+G11Ru5SU##;6Iv=f>wzP@NsrF%_BB04y5TG-ZGlZXBpggS} zW`ds#3k^d3$w&}n1B_TT2*gP}E<)8A}!K~SR0rDym zu`ub0TM24#j8lPD^dJeLHRo|vZ>&z)B2v=5cmK8(sfregNP)Ov+`hF>DZXZXgjS+0 zR?w*SH+ftbv9AYySe)kc(J)@V{$MBZRgp1{F^GwZ2+lB4u^ey*I|F5aWXf=!RMdx+v?FP zh!7Y-HXMg+9S+I}!BiJ%vQP*nBHFOdyi;UojJ=L?wi}hE@S9$(ugkT-W281Cpb5qx zV)}O8RC6AXRaTT%(ttWq7g~Agi^KRIIF$VC*9w8?6hHtPZbi&&gwpXThkKgxVH^rV><#* zQqwSJ;RpUXF62Lry1;vYS1>`|imLRUm4#+?QB8!9O-Z}ber+9a z1YRad=n53fVlD1audeLmOxAG(V2&vQn>d}YGx9>&n0r3AGf4A@FW^eMQp6{YvhfZf zNy8>HK?MBxq25?MDa2aiax7}T`VS2LlreuFKp~(o+A$)l!}rD1e9~ut#FASUBAH_B zE3z6>f z6BFwQhsc1&sOrCvcFdf98L0+3Yf5oDz;stt$Z>AgRTR7CknRE=iz@eHnlEpG8cck~ zFYd`{(TcppWe_TJsJ1+NHuSB$i+AEy*tPC+5R@62OLUgBDBt2(TK;wU`_IDZEv^e# zbfmIt$@31iw}G-Ky=y^q9Qh8-m{{Vw z7Pay=pRICP0Tm_p9a+}i1$fm`h#I9%%9Yu&R&{oY=jEf$@GsAXO#gwsaZM#!MirVQ zhCCuKec@9*!dant8f!kFu)j)YK?>^s&<*ta6sxbL4`;?nBDeZ<(JDXwRh~2kJ0U#v z!#N8#p6YnhhIBUY={nR5)Cr+Vr%9XBe7T%5^Lbrb<9WyJZ`1()zXZq=`Jw#COX}60 zp320x6mXtdjkd}sOVHm>MW7Sjm7+upMWTAnXqHTwgyp1V#ni6(4SwY(@q@7T2kJ8U zq8slUv$m;bh}9~l*x+X6E#%T=hxex!U=^`ir#C2)T6$7?NH#m|&BI?I_?L#=fIh># zg>K?Gr8tC@Z%r2hC_6nA$2j}KtXxW~Wd=l$-3pDMkVID&U$+n)RM@o*!!Op=MGvG$ zW3qFdK|SsSW8ezNyPTPkz=dWEeGz#obt*H|7ipw5RFPDCrhOJ&VmH3I1;OWcu&R0 zn5^66^iS_e9Z0qdQ>|@Wt!u(ItkPU?EVeI-ssz>#e%8S@Dn{8F(sJjXTa?bZ#%3i( zuE{RC*<8J2;1>q|c#}##X5Vh10PX9Ci92fw%{KzkR=IL+dZFFcyU>3s_p|lBmwkjT z{BSf0SQHUb7*@HYO{81}3ya^s{-?$Wk#R$xQzcyWSnz(G)O{q^B%m;aYP4ZU@^(V$ zSnoCC0=yeN-5naV7;`c^)etn!$Ie=uoDrbj!w$wWGb!o1*Xk(?N-ukr7I=pqFYeZg zQ@rUy4PVZE-jZyZAT-`sMs1hb(?A+*c~JRH1pyHaS*qnxyMs>_qn|XIaUj!wNVgDi z`E80z+jcc89v-zUt2CMwm4|Xs)H3E#Yves?Rz>_r z>|YNOY2o9M(W+aXJ}%W~VIXg~ZHevruUtd`H{cPkW!+ChI&}03s;Y?LjGbWT3cVkg zN0=CAdV~0%1@tp8(a7|nLN2@JQVVX=+Y$E9<+KCU32)t(7ivL5TjpedQRZRKl_Lw!Y6BMl<6`2 zSDo({oOr08ipWR3+%oQqyw}}t?DqwA`Ao9{Q3O2a{!j8ka!S!Ya8D*YOGtBfO#@z> zuibHA?X)>(&aTW5UYUkyliL^fv)APjCMA|>y*1v4|O4nD;T>e60|Lh&O6l2`{3F>^;?Z zKTo#>W#@rwsb=ZTD@%W;>}A%$dSP%{n052t29k+r#E+t(`_Q@WNv5cn5gbYjHN2z; zE~*YWfO$3YK&Y0I!dXAu^raT^Yswi$Gw&=?s^^re7nQ?e@2^8aCyFDz&QNJ>mMkSd z#&sd?I4hL=I3Vxl5ZE=#7dJz)W(Vohwm};tg?7^a2M| zluACXGBc|S^epV>j`0Y)!E0VPnA5bY*wR-~_C@|@!hgP{@vFfO8ib7Jf)PG~OF#5u z8)$_B!lEjV1069`9}`y(k>`?<0@;2ZX?hct>VSTu*@dyay>|+S;GX2EW9@N{QRA;$8GM{C#WsJ>Ub>y`4LeL1eK<$JHC6vakih(BON*-s=EW zr*xouRF#u;P%N0yuCl&+C-rKM&< zedoRPul(@_O1B$a42Vz2S6C4VF>y;Dk=##w3zh>SCT$T*Qlt>cc)77H4o#xOqO&{> zjGR4qNn;9cs+-4$_ET0x+81Q0dkoRk{Ttjrqg0xOOF}7v#7pu9R1O2Tlr>-eIU2Go zP`IL9LNJ?*c;|*=O;VN#=9x!!)>8!b^sivE-i(N6vRY(jQCTyTR@K=rDv|!B%AdW3 z%oGtm+!MJupARmm;t`VRQptTn0NGRSja@1QTEPtoiZ`SZ5dzUv0}?}1k>=8Mx}fj$ zx>jkZI&UuhkWxO{-ce0kWa*D&uVpN>j3RR_2UbDsVc=*8fFS5OUxNNIDFyH2$mArN zFbYU(RJ%mx_uJ0m%0$x39zSY6x!kJ1esY%L{6lb$V2#{NGO|GDPIiz-|JDdixJGc2v3GGmGl2^47iPsqdfFEoVqEuhNJ;49NLygw z6p@>WviiOHC#ej?Z~Obf-+iR_AUm};cDrFna!0tIGk_$5j}wX##eC(=Rn)0bNMh_G zvfL=2$9)OP$ArIF`ZK6PAPU1HVq&j5dOsZBj6V`;_~2p>+PY^6 z$(o$q&KJ%31JP_Bmq&FWAI9xxXEkH0iyn^}Z06R&&yEzOFJ?bFi-^kMzyNg>|1aeQ zj{Eism%B{iD+Ps;+t&E8mju}U>;59#ys`_)mes*^keyf|TEY8$SNyjzif@6-BJa81 zpP$oZ-u%&bpeDFs&A8jWwfo*d7eb~R!>QasRNA9AX;$Oyw(BQ`TrZh8N=5UkCd+~< zBx0DWmg33vMZXp!(10EizMq~o%M>SX`;XH`R@QGRa~Y|ArZvY~v~v=fh>8bQco=>c zibC%9Nqs`?J9GgFXEHv4{z4UVw_ESKv`rBqnoa{nlLkMjS#mQnUMMpq#jyqAaU)l~ z4C@smb8aTDj23;q!h$)PUi1f>_BN-eJVqe#eHnplJD!SJ2TA}Wj$5hXB?2y z2En)cEds#3wZ`M>Z6qaCem+CA{zSwi-XKW&W)x|!FWxB9n&Bb&f{^syk4GugdqHo< zd#C4Fxtj0oVmGDB6Q6?C?z^~ov;tvK_W4=LFrSJK<(s2W7$relNn;gwCUo&6 zxI6l|VNU%hIwhW-lvtVdv8mF1MC!o#`1p&L_;@8T_BC0&RQ$>x|AreK%gs@ zrB2vHJrB7rcDYoz$A@9J}3hPKynk zg0+DY7drX@r9o6itRi}+?tf8W|dw<~-q zCqI}<1*DuG$ytmP7A)?Gyw@}+vG(#MHgc`HwHTu=8S02C-tu)f=ol|rQ%10n9son2 zkcAIr>O+Jm6lXr%L$v$dNU+OoH#+=hz&G&82GFL5Ias$C+!cG*R2>v@)i8GDQ^O>a zN?cT18Xmf|P1#j3*+hw8Jm;9@i*|03U2X!u~%T#r+KDAd%Iw^cIMJtr&&#F z?LGJ$h}3ia)$S@+yHgRr_JxKAdlsAA?(@RLD$FCiRJf%LY-*La5lTB57BTGz5w_8y z34wC&^c@cl4FR~Z7CV9)&kxt(Wt4#a1g>0(UnmC-Zb{L_QZFr(YThcW4zx+~XELB2 ztISvB?^V4#I29@h5^DGfOU_ZhB-l{~T6*k6GQ{SEaXSp|IPH>sh6|lj>x?*j$$HUF zK{sMG8g81^-M>fT069uS1s>+~#SJHq;PP?1v-5Z(M&#b0V!ZQBP-8;0qwt1;bKQm+ z0yB8=;N0AMAvozmyT>A+5t0UqQ?E0tQd9JHz8xV~7Ch-L!eWKV+aQp5{{d6ulmj-$ z(P}5$@B1AKw8Z)AE6elEDnavhH^ljTg!|Os8{yIqf1C##WKPz&WdZ8KcZ^O~BoUZ9 zz+$8(Bpw)Vge>98hxTymN+9-Vn6Z%j*y|ja3EDV=kbAY0C725GN6WoC5hIKOwAu4B zr#a;rt?A?em!|=xvm>;)jO2j2@+0jAc_E=WtF&N(vtR736)%JYz-^g@^qfLz@_HX} zgW05DA4Xswagt%xzsk|@%@~S<$dMn_HgxyF6z4b35XXKQ;yiHq6OTxva>GPE^`KZ} zC^+-XX(iDaeU|r)^HSsE%ucjB*BimKBS5gh==em02WjAbTUnO{U=>`i+2B3wpPO#% za-SIpVgAh|@KNKskiM?L*Di1jF(o%;edE!cZ_T2?WKy`LZ&Q*0yEZovwlT&@ zN&}c@Q?x7ZL#5nzm%b=9FQ{nw*}Z<9{VJ(!zx34;9l_h(6BCA!Z2LeNh@eWIPm-_P zXm4xXjcke#u@gBtQIWvVImkx?;(=@@y2cf_g`^3i@t?uJ0IgUAL0al z*~O>SDb@~#8gV)tXb<}J>Iq4h!@k~Ah)uF$cUUZSs~BTOkFXiso3fUzeVzjH$5}#O zSnB<*YW0Y-{&9;Br)$oLfHm#zM3;~382rnD$I0=3eH2N^8|8SBzr{y!jGtNmEQG%X z8mMj^tW1KKfM3S13(;fcBFaNYs6la$2^etdsDz#@oF|FAS8_nAQW>E)*C>6vHd0ce zRj>AqX{5mPY?FRZfxpHCvP2#(j<6S+@IQe+mGu9FKaV)sc2x1xkOWa~prr$a_V!VT z``roI;(*ZP?35CX2>BfEy~%Yr{gzC(mQlt9%}}A`5QO?K(>m$Y$kT*7fUwbA`q{2iV^==n-HvBV?=3+iWrDF?rFfhOA_OSyxVQ{v|E(S4zaq6VNrH@ z8Igm0zSBw ze*JFqYuNaF(^UQJDaRWL?=#*5Cjridz>)q>xClev>8$U%`EHp4_u=)ypa%b<0>Zfw zrr`0_&JLOg2!fVxfoQ*>iOmO{gzkH2Ne)yDVPN6JTREdhV zzuW9N_+s~UOZx2-<5Dhlh%U0fM6C$(Yx?e7a@0rFotZ3 zc@y9p(96jqAiMuz_dcBJ^ox4 zt3f_qoMDvi7AD(+@5@fx_rD7>3NB2Ar2JPOi(MXwS|{aLCxNBVj&=0AliSAFeMKrh z>C9ZBr8#kB&O8(_iB9Uq-AMs@<0LBM|Jw#Poi$Q_iej?UU*ULR`X^3+F4-DeDZAX17>E7qd>OeMc0|eRJ6*v1wkc)6Z ze!MG5rXPI0rX$6FU-KD7+kb%|u~Q^)BSE+}lG)@YrG-~dY`V_1+^|WV2R?b*^E1v+ z3gr65h(OS?fPsd$E9Cw*lJr|(vnJ+b^S{oIQTD%|U%r^{=Wo?mYoh~n541j72?O_v z%K{zC=^F!yh9*I)pDm3*y|)wu{7gV5!1nizDn$6>ss9fi{}&3GPQ4lJFT{cZ;Gc{Z zOOD`#e0~q&q8+{KeUKe>0(jJe{s$bT3ju9UKu2>{Z#g7(v;K-#I!&0(hs;W&?*_1t zp*52dKB6_IGd1M>+zwswPdkyc&1Y3Xu8wBktC8T8oDd3$mqX~~c%?Y^$%3dP@an`$ zZ=>tv-4z1Ed_Y0;7Q!N%%3rhy83fAh?Hz|=*<22*X5_+{`R{9yf7oyergn%1;CBl9 z`cw$wSxw59nKj$j?<(K%TC~Hs{@SDw25wry{Xg&pFy}o8$PJIKJ%v^6(-k;r_@ACa zG!RXN#xI+Vr`js#;@;-gpIs0#o@qvzcoc^3FRolZsJj_Wo^5^fK-gUYJsiG`XJ8xn z|JU32dAVy%Q~hl7-pI6LIv+^x@>-bJxz=i(1h^XYSaQ7l2@#B>@~1jr(Z^5j07!=m z&i(J(;&_mFfVMQR&t>X$=D>e*x+Zv#l8;W^Z5BOLU)zdICr_J1h6G;Kv|Ik`sJ|xXo0sF^70KtUBc@~TRlx{|KA)?f|&0= zV2ILZtDD^bAf{vZwhyhq1R`&DlYVRV$WG_U8nXGLYewkcfQbLE52!t7@1l-$vl;=e zVJ#?@f`cDCoS#4{J$y|`5odvQxH)Z@a{tDyA0M_H#PWtlzX7czZyfnS2KMPurULRb z0qq%ags;QBVh*T7@z9a}b8I2HmP>|PI%pkxA}WBT2nQdx^0qK?gp1>dTC*{}U#jGpJzT;g4C z_nc#H1^g~0hBoT_O&gH9;^1+NQ>Y>M!c&r9LJsgzixo~?p?|tmLCs6YruMrO9h#vi zndul?N1zY{0gX{R(nPQuB8~Fn{VB9%}^8s^TgjaSU_LaU!`CfZq`sPEiGu)jL z;Q8wfB!Ax;6as%)QZsFv1^+YL%JsX8s+`7m9Vq5QuY=iQ-xC!KP#7VDkQ1-LOCoCM z0!~N{Ek^5(+0>4ogG`a1rM73ydMz03C`?D;AhJ)kjPWs zj1k2x69d{Sd-IJkh^2?fKv%j`<=i>!DDI@;y8-oC1Sx1o@005RfgrO#rvlp2{3&V{ zS@zd{^?Pkx*O`B9S8p!_{O&F0s~^JED1HN~41e4QkAeTMdcO*^K=pJRXSM1KBXqr1 zx&1`Q4^{=GV(8Tm$6AiFxNI7Eo))TK`0extV#@&m;|UF_V5c<`#Q(W zKCOb|=^@@rVdj+M$l1Fnb6vYD`%wCyUe#ae0uA4W?YVW`sHieRyhlcq_%pRuAhQD;`rE%C)OFH9>@w>RfhF20AY_nOO$H0WIb( zRmv42s0R&Sz|LlrTnnWe?MrVa4!bFUkG6+f!`;Ww%+<56Gy=KwpLaoy6bllU+CPbl zFwxH1f!c>aw6ViRp;84`iOtuF1DX9znuf%*0tJ12Tjx~D(x|sd!6+R z?~DlX;HEMnr~-9fx@=}#j5*6Gr<#OkSsiiqmzZQtSt@a2zWZy+U{tF!@EpLTA&ssS zru7W)do5~W`q+OFNZ^0~`A=*Y$2|LuS3&r!32^u!ZT6SJ6uQ|eW0%`{|NWU{rbKV8 zae~wpk$=UHS9;#Z8%=anznKW{8f44H*~td8%<8$OAbQmj9ePvEKr|PZ^^`I8cmvs~ zX)VZ$scNgqGS6c3&1yFtq$aC%wPM|a%mTG3X%35#(DisjC)U#947RpEy#R5?7GDWB zl|9Bj292K{&GoI*n-5RRk2p?pml(7ylzHk_bl)Z2OAP){ZSS$$?dFIrDKc zsaqk?HAh)JqwD6T>s#l8{#(+9;>duKr5_xR485=du7PL~6DGQJ_4TOj1h>>!N^(C3 zOsCoLg}dtNhm&y05d+CVg8&vHATCG!9ZHV1!6cdDG*6w9vn`UM2~w|>={sCpojAOU zcO1_9^LaLl_JVJTSyHrh=@k#FgoO@P;>-@!U^Y7*^3KnTI_wf4*)K|)JZVlk%ExJR z5Zd2ft}s3$P3N=<8{BeOTkBO%4ZG|e*7n+dTetGlVmR9Z2aEJ|=CX%?HciT?ynG_Z zSVu1D;Ex);!B2voPIwTxvzyrW3cv@JvO5ePZ{B5Hq3=ySd9iPka~{mJU5d$EPrpTc zmE9{6b!%X613!iMC(=!Amf(l{!+gHd73xYM`Qxz50MiAHTd_( zwwa_J!d*@R^4kr&86^!rHt^=*7O3mSHRMY}`e?Vt!lqW1=!hg8v!6$p)aw{(XK>X; zAS?TTeXjFb#a>^oK?$T(=B!4dLFUmVeyI!*8IUNTWCTfNw%+mhJROV` zifUz20pF@Ol)4se1zy@U*2WD0^ko9*bGRlLaqSU^OKLn%TtY(m%9_}2FTcH#Qj@GZ=BxxC!fWziXCgyYJ%xb+ej_L<% z92tRV5w)+^pBvcQ`-h%x*6=l+pF8tkjNgGNKCeF-e(;dQ(Np(qy`NP!H>r0zo%Hqg zlPZC8(~C>DREcSBDVv7#fyRCi5ka`FVm!%r?9*7LohBR>bKPd#@j`1zJeXrOJD z?VsL_(V5()6z@4T4eHyT96UMsY!EsUQWK{1-Q#fNr%`Z93{@(zVBg79%-C!J!zs>d zPtHQfk7~1f&rx0uXX2-*()=7P{f4^TSTa?zy}n_hR_45xjg#?(z?zDJ@35t@?(A?q zndfbohfLRfw~PX{;vAQ>{y~QeR~~clu+hiA&o4lo%H}r6 zw*vFv?rOL53|^0LS|rZ~R2(XZPcK#uaVK6;tT?6t8UuL)BnMNEu`B-o8PNUOelR+} zUh>M?Gs)<1E~z~Ia{=PX$w}SMp^AgqBxYW&eG~T+AGoikgtSh}dA(e$7sr*n)u@Rx zBRG7rQDNgX!ZhOK;Mr8YjpFJs#Ky`%&Ude>Sa33hN%nU`mZnCJkxR+{wlXmw@uuQLD_Z_}nw?P3^BI7Y)H@<^$mo z((>MmfoROlT-uD-<^+rk%r=b*=3W6&-Y+(cd-bTkyPWR&o3)PVJt?GZ;B%>V9DmWZ z@_2b$U2k+oQ@G7C?U6=tVwL$i5c|k?oX}SSXY)}uRZi_5L6t6T_&6)o#HZY8{O3LE z=3bUGFq0ZQ2-BwRhbWh=8`#ODo$NitUr9V2@ zK%FrebR3ar1TuFJ_4&&vitQ-F+42JoDG==Ep90Kzr;cBaGa0RJh6ZFi2)rO|a(stW z{0k4kyYLLpTp13H>of<`uduEdp}WmTvja=8v^Dq3^Yw2jBe!0vi`@}4`!Gq8qW$CJ z_!vbz5UdQQu`CtvsWLqm*2S-1zYZGaxGL;-(CXssGl?L;o_sJgAbo?yio=SjN#6Yw z(wnkZeviv+x#&lD521w<2r(;{lBvv3-$}n^k%ky2HNFbrw3*F1EXH{B_Li5F$>GP3 z9gm~tTsx&pSf+f{b9orJ#ZK*p%x3Jjkrd;eERX43DoWlj%iTLs9h=FSRiFEDzM%%l z!-M%4qSV*j4c}*Nc~S$8&FoC<*{RhA&53z$AMH4AmQwkE1}6OupZ>hPd=nyRu>g7Yfct!I z%lH+t5a}E^jyUaF$eV2go5DSZf$38h?*W5QCJD&OevJXV-Y4DxNMj`98j2H_+qAClB z3F)Njj~JhN?IB?g&ewlhnOF1Y_9~=>Ecls_jHDwpCVMn%tGuRLbmk8;RS(~YEM~k; zMZJu*!~@Yf7y_MD*{_8){6|>f$sI$^eau#Yp!#IE{w~@TaIWi!!u2fn#C%h54B-zj zL<hc8uLNMi#X9Vt95)3xAuDay#lP4byCvlNs~^1C^o+iaASfH@=u}~wnD2t9^>DkvmTf2_BhWTQHlm7JlwX!<2_y`47H^;@xe8QYlC%IZ zCj0o-f_vV4183@F)&Js12q|H*aZa55O5 zy-ZQ~OU74?BME`ir-cLON3qGL36H`_`b zl()$H9vc(*eLdBMox>ZgR$*LwFzzJ|^A9!=S@?Te@5&!;4Lmden%HczAGJ1AB?+vi zcCL7&ul^bHQ2#7Z@^TM(MNDh}ZmG%AX0@#O#xxJ@Z@pCbw_Y~%d(>(22 z_d&jqd@WUybjvglKS<&6!wtlb!*?i*fB20JwDDJsMyb_YrMFlVE4>akOADUmD?{Fa zxT@zg`#NNA;vKWs07xE;K)e{fYNeSo2TAR1Grf&(2c%wA%2iIK-y5vk8gg1J_q@jQ zHfxxrXtu|OJjM0QUx5jGHmxhjiE;=#1pGl39~3h0m}9E zY@_fCDf2VkS=TCSwK5D5P1zbf2Q#k5i&IJyK7rGhevXqaL^Lj~e0M*p`+Q2|+e^YT z*q=VOz2`ndpitZ2c=%rHfn(NQ1#uvn*JZ6HD(CbBU4O&9_DWvIv-MI&^Kz%wLhh2r zC8p=oZr%bX&WU}z7aFh%X`B9GVaBQ3yMmW_%ag}l+kGb4R3O4647b!eD_s)@S8}OA zc#xh{kUXebo;y9>r&NuG7ez$e=>*ETNh2E^1wbdE@NL{{YfCsaB>{Ln z}m0g<5;v}+-;XSfk<3ici1=f(?OyPANK`Knw9T3 zg0DQe6z>W?-hIBQ=7s}1s?klk<1qqt!JFfY&>Pq~vbR0DdLozJs=%2U|a1_S$K z?g>7+(j7ajXGx@d8A=B{45Z)r6aDkE-@%{3G(a5>1T_N9#gMDjAo1O33wYT8SMMeM zgFxolcfdV&;D=0rvG4A5*96ZBfKrpzfHwr~>IjQ=udVO@8Buu&DO+%55ERsdrP(GM z_k-E}9MXurIF|XU3|<3~+ZeiE^i%u1jxtXsGq+Ovw!Z)@(p1nAZP$GlW;!0ftB}BT z(EKba)q{`cA=W!YJswuU?r01PqoikwgVKk*aZ{AXOFn3BH zy*#bhtJhpg9~3xStG8L^Wl)Ll&9ju$WBoeM+ciAaFsUca`4 zEsOt!C1{t_^!z-{^q(oQ&D>^!PF=jwUGo?T5l?-S#(E-uz*8Drv912q%PG|{Y)0VX z$HSCw**<%NXBC9`!kN^4{p+Kb8HG&iC5py!4xTn~Zf{6D4&`AmkoG*i`syYYg-P-C zToMnWz`O1ai$N~`g0cL>?)z0JB+T>*37cu?t<_ZZDm22$bbn_`c7Jtk-DXU@v6Sv- z_ggU&)s2umI7PPIe5Msab0P1NqOet<{Wes&R|}=oAi#3@?KwQNVou7TcPq@!m_9lxGpF~EIpPkkb{x0oh@znBZU)Nf(v@1_(F!+fNGAo1x#$ zKD{quv}xz@IO}zX^@k*}2REr@9$)6We){HFbA%3uy+Lpo5!rugVR55aQbIL<2RumE6zt7ZE61(k^Bo$v= z=YR#yIS|k`9b|&DsLCf3xM=|QT+jGYJK};u7AJg;@xqi}o{7OhQebVl69(dPTkE+2 zQM_Iu$YV>BWwC)nNKms!hsmPdc?A`l8(tdQcR9x2VM@RB#+Wuin0#Hc z^VQLdGYLJ^r?wUtho3jcHE}|{I#mMEmh1Bkw5Xfv{9LkqT&hItfKMM{Om;o#C=9|1 zR3{+rtU%qRC&}(OEWbDd*+u7fY4`kb+Eo4k=3QEXweRhP}9Q+s1nCd6_@>DZzK&&v zt!i$O{$_!hlX_`|C)@i5DQPC27b*!~2PP&|wrjfK9JL=eyk^&JL%OR;Auz@3AJ~5w z3O<4L`%*&>+N(y@ayzik(vK+F+Y6ny!!`=)=a*>@+H$GPzZq9xq(MqWdo^m~EGZoAnXsIMcZwk76HEuhW)*x;VHd-ZjV z@z+sgX?w312l+9tjY_Y^p6149CGu7|cW!&l0=4*m~p-}N=`%Z%i!>s6S@ zjdY-gnE^hK+l!dfxIuyY`+Zrr2Hm{oM zjH?@WS_Os3Vv{uqq!Lq>L2D5|AB!@YR=hW_bHu0x&&&Q zC^n(r46c5aHZKz_;J-H{0^~gRi|UtYE5$j!sNvgekUdbgc9tpY*>7Imfej#c+n&8vRA^t zSMZk2O{a*wo%D6%0vs-SSxqP#pMiG!y=D1OC%_8IoV%>gd0F(X#S}cX*)h%r)z2Q+pvp30Vl6^ZKU1yS-SaVhLRzyP*HpKj}6sZ^56lWB93SQ)0q1W69BL zhKU&@gB`FT$>?J_cZGvFo?UKL;n3ehKS7r-(dWYhwKe{9WaKvtxRu*U0qU(O>-*DZ z*NsiVN3c8mXeFSlVCmh3q9D_lhZD}|MwpY58u}ZO@4D_!M(^XBiG_s5ktFFoEc z)xJzm_4rBNxOYm)F?A&bs&qpWxfx=}Dx(h#-ipJZ%Bo38w29-Y-6ZhRQ<8BwOaG(Wv=-^p7g$Axe?*q+_e6QZKpEq`xmz6ryz23 zxPK0Db?)KsGa)*x)8d-udE`3&I|s4#R6HO+lWo$@Q`p3Qw5JzlHYK~=H&nu5&h0t2 zFEBPyv)$5a?%6^v9?GV;LE@tCLG!e=A-1)Waf0!1hL$?8rLv-uvBtmz?k7zbTjJfW z3ZJ%H+ePxpxivpOuCmhr<*i01?4gfOonj`~qi*eVU~cmi@<&&dO6%6HUv{zo&OvgU zV0vj=(VDsxZAcHdU*Wy%p4_^#TUO_|o$x5hE?G0>J1B5SL>b61^iYmhnoaecd|PMc zFGNsJhMn_)0xp%>n)7BS`n~a7GEG=Lhjz-9yraAFnnp6}lxGHk3D<{e?Honqzl;)nf)OX*r zH5rUSk`rlb`5)AMcRZEh|2M}u;m97@A%%=6+2bU8?>$aZC^HHX=OiK-8QCL+%Ff=W zh>{s)bE1f{vPV4EeYC#6=llEb`RDn^>(x1(`?{{r_1WWnpF5ws**@XKdy72)HUpihO- z*FEx^#X)V_`(<*Q>`tcddzRY$2Qx+~TZ!vZLl&2CDG&5Z^=#aTO3lsy)?Z~`VeiS45hnk~vZrs5xN|;#c(?MyM#K?m4OXa@WfbUg zT9ucU+ptvQp;Y6F^Kjm5lecxANXl9-L;6I;JJ_+J06Xt;2lll(YOQUXF`VO-j>%3}7B zTf_EgJ-G?5o$I|HSX%W`3M$oRPPlFil5ET^j1+rC9aUHt;PCn$-gWgnsBfwbtbLp1 zZ4e*XCu$J;LgAQcOixS40$xOtfWv~I{diA8!S|H6^yenMw$5tqS5Tv;$eO`EYNT({~LH=#>y(>|p)9D|Gr({IO7H!dJ&re9d(;2T&d z%Kg)C_BM3T@0!`(tXvsJcpVFDO;mgeTpJZS+To)kZ~e|HIqWQ?Zq1E8!dFmCGsj7l ztcZL_pXj9Ag>ujqwD|rr-F;(^IYGx(@kM4?`>+)+-d--l@!E=}r|(J&(+5--_IE%N zf^p&$)pZ5;>h8vy3l+|~*KQ9D^#ANqvy&yKdFFCg{Mn}hb`Oh*Z&1FJLEuUc#`biv zjA(JH@7(JRt-|Riqvuz@b??utO20_{z}9h+>X{1<v-xD=pdkPg**F%uIU!Ud+?i z1zY)PO;DCVIr0;=9OEt55SGQhXvGe{*P#~NpT4{;t7xW@WlE+71r!^o0J<{7M`48)3>UUbnUy8YgKw?^Lez6NT! zbaA`S(>WTk;q{u_hUVXO3X31^FRV^Pve_(0>Ag?a#G`;KOZQdN^hW3BaIWQ=U2f>s z!n-ZVgN^@SHb2qfiW{(U{su=JqtVA^-I`FYM}Ae#8)>A0nw6}x9(n)n1+WVc>-?&W zLZ~bAODvU2migQG1#eF*P0$%l1uZr_de^P^VQW;V(EZ_Hv<>a2XNd|upVTU&IX6J3 z$8~gbI=HcWq?f-04+03hy><#cf%OwhP1GX;e#i!`0nS>G-#b1`PnDxyHSH{#7$i~tcDA-^n!(B&0bm>d!Dt|%5HH}C4YRJk$ugc z`s|;B%oqWU!Qe4r|d%tX|$!*@3 z$-H?3IAvfqBp<}D_t=lrY;o>vhrFTsm4&gMP9Q3--+$(HCTPC_nK8#M=N;cwQ{odY z5wP?tO*I2pyyGOrn67m0n{RAgb=Ad(Cw0_q-T=6<%c_{6!{e%EOq9Jt)VBz19kXU=auyi4r? z-;ZM&9Jxmyn9%XGbtIp)k=GUW{c_H8=8b~jL*ZUe*A1`0hKnuDbHfdJD_+mT$u1{^ zfXvW{z`J;h{2MRxW$depn|8&wYr~zr5}%J}AiN}dg+IP#ze;C5d|hhXVkp~YVR_iA zI8^1HHmLMGSk@>%_4d2_PPpmmTkW64l4WRl70l$Gx6EX#f)+G-TF`2;FMILNTOPRL zV9B-P?GBkwDX>{&@cEq+lP3C?C4gNzXq9WTP|Li{13IKiA4o{P?pCO=8J%*^;9R`f zr2@3qv~Xu%lu16Lz_Hjcj~#jHuisZhgnv|lYO7$7c`lgWNx!wgwoxFHe(iGjx{&2k zg6^J=*38O#x0t52Ic1x%Z)smWpN&2dv90k_8ZW3_T1vXm`+>=RLrl*o<;h*(Oc(a_ z@=~uh)!)_j?_8hF&NR6-kiy7mHcV9-PIgdH-|gFfv-hrX-?Z)M(CF6M+au#%v3vj% zVcLW#PSMu<$=5`_sDU*z+s^^(UpCmX1z{}<<$fzCIQpHA%p6;Jj$CO+mgS^`(bTgtIN-A#ELDKQW4s|@|oUWj~d^~D0RU?)qV1Rjsk@hmEWxTAL{0nN+gDk=&poSZp;VFli6>tTqlCH)xRCgI_9*xd^E~;JpG-I7!WmUQR ztfoxyV*!iN*^6`jrv*8bCA_A#zr6KI>t^am&7ZHFumu&Tz-Mc1O??y6Ps|#J- zpjs;jS}A^RQpGn$-YaCGfy~~s0KzfMUVS9U48J1Af z!0PXs67x#a{+?FE5k@)biE8IF^CituMZo5zzTEjRGA_`h)-CA$E`1}>shNIi!7Ia7 z(rQ`x&WL-xVYi&Z$E^`tN>(*`x?f2zQ%x`5siRF-U%yyPh>LstGv%oH6QMGfZH&T< ztYk^&dehLvZ1aA@u|CeSTaCWyH->A_UoeC7BO!L*fKg71R#n`-9km!>Y3O?YZ&gaw z_fJ(ysRdM}@_*bINdP5ffBJ8~Iw7>;r9Xv}dXG^4OzXb5p>dY>0_c&t?zz#WTeERL z(9ouKo!_9!Pv?!E;$9Z$zIryaN6I_D-nr}7RT`P{J)gs;fK}MBuT@d}MhnMHhwmjk z^I@GPv=9#|)F{ovl1YXsA76r+r3-Rv+{>YqD(KxGN5| zR$j^=+W$Ucm!4W*Iub`cQ7r3T)4JsZ+$g6FnKf%2P@+s>TbVx8;AeyTwx9ZqS5~>w z)v9VPt?$|;Q#w&A5KVNpW0QJF4dUZ%rw$6mUmkns|7xB zTDOs31L{92Te1w#7W=0xW%I>b8bcq|Gt&O}@iG5;*|09luZ|*--o1R0`P?eI%+H&F ztFP}ly-)$FzqI(PXmMRNaLKj$$G+8zq@3RBp+FeSN^r zs^5O`wppaL|2v!7>J6e>KbFPQeYQ)x9OK3_o&{T0hI87}8Ad5S_gk&t0QD^1(+c71 ze91pu6?-*v^^)aI#6c;E7itedO5%iq;_1{K$90<)xDX-Lw zZO{`A2ECRi(jKe5gj7^~X&~s!vVm&ycH7ooWh1nc1R7Ib#xKu3arAbbph)}C5&0=~v3^2BRa7!kjk->L`%dkaKhz{7uK4NA zv?b`c>eBICnZu4OQB%$&9H12q#b0+hupeK^D_K0+>WlJRxj^^ z1hzQn1H8HWlGcE9<-FUs6TK4hGM{uRWWKAv-Q=Kw7nI+Qssvv4 z*n88m=dUhyH=6KTeED7Ty7#N*l|KlNTyMJ(>z5OnocXupb&*i=0uq+17|wYSNLcpI zx2t=}a=@@B%GUzJ9&X&?7`~Y{>!g0`SN0Zz*?}e3k6&wAiw{GUAP9Emzu(05W#QN`G~E$xSn$vS%i5jVG(R`*fj34w)4}^x{`!vTRvs&g z$(;qZXj3s!tMuBn^Z~}CylKbIB-UHV@zEGb$>~m}I&JTkBG0(XmbITpzV1!0R4MFg z>ZHzRYR9EryvN6*j66+F1p>wJkC)u06{YqPoKCBP+BZnKJqP&*?^uD3%eFqfPMNpX z#w1tL6V7y8&R7ash@>`YxEblRGp4S*`DNzpoNr50wyWe?e|}RHQ^1;Ex%^$u8}S)| zvjEIaN;~+X9m$31^$Tk3ppt!Iy$zJm20{KSW$8z}lRW5ZDN=uh-SVLmyT{S;f{u2Q zBl8KLGK_RMFgpQd_Tt6W6Ht=i3$@sC@}|pWvSn$(M=_ee2D6B;Wa-0U zLI=(+amIJ=F)#oJI=v%i@|ZcuN}x2(;l`R$H&f2AQIF`c!W~3I|ITx^2)VOv58EW) zeNLSRO=L&>5^is~OvoVgwy0V5V!FN?EWvL%u!(0S+WJl#BrB7w$OONs1pZAV`^~bB zd@g&*6-%rjDb>dh4PBlJ65(WuLUydT5U(DzvFEpC2IakZPC9*Px%94N#CXr1?|Afc z(AxYYx6ygv=f)_j%~qB#x7q5^CxS(d8VOY+BHMY-7MpVR5;yo`eQCC)dO2jE_PXP? zD=|(}pu|&D_r2*PDHTZDDs*M(u~4a>Vv<`E8G9zrKCm`@ z+g|f_8$2?hoeYX+UL=Y)B6xXLIXogdf*#C+M!W0ZJ0gEVsht4!x+mKpd_3Zusi9i^ zY(FPu{pE;CyW|?w^gnhAnN;3WIkcp$bqvjqW_QV*9j9MF$@8SdGd%NFCmmF1hg2QW!Y7f!3wQXRj>e z0z3b7+h4N%YloQL=b$q|84|9;PeG#9d~JPn#DWEgmV}jq%ANZXn(yaK6z6spnVdhg zGwQx?&hS5L-1jr(NR0)@$cYupa7L{}r=j2XtwNfHr~;OswKpV!cgrcTE(3X1$JJ`kJ_JRj8_yB7MnpqT=%VODC@3^#skn z0_WoD*sxvs4(Ijj`uWZG3UD#%7MJFCcHI3^Hn!Mj7%4t^G|qi+&kBu9jZfu`1?{tq zKQA{LJ+YX5RZsP#TzG6MPIDi!CpqrPQW91N_9PM56N$3m&9rbbb&lW42z3(y-o2YJ zZfhU|dr3n!yrI*QkM6j7b5`$Q;3ab<$`)}R%+EweK>-l(1X*u&D%ZoMr4t-XgVi@K z+n!o(Tl6D0xfsjRQCE`sLD)QmJ2osvpPO@Hh}WlH&NJXugVWHqI^WCVV^+=px~}NV z_z$qPdsRn=hNJ0UX8!f5!-13Sf9?(#sxrLktJATCEo9sWf2+_BY&}eTagX4df_FJdG!v zi~zwrjX%}cxHR_8OLvHi4^A(af-s!2`O}pLX=F@afRRYEmDu0$Svwl6Mp5P$5Z29^ zZY<9BYs}1=9*mit)kG?BO&yeCE(P2(;D!|)%(oX&o<(rScmh}KwC;j#$V$_;b>4NC zDTejE0u_)r4wpsvSH};s32a3Ao2KhZ_cb>x+x59}9)${wnUr984(>^>w=7pznu135 zZ(Yj1B@s=%EKPp~Bl9~Rx(v?h!pSilirP9@fB6*bKW6*E+F$^Yi^dlkd2u<)r3-gG z$~}A+q*ueb$2nv{{`;Uge*wD2sAJGn+`)|23$b@k=>gm=Aye+tnruh|ppHSWnSP!w z=wQ@@##i!7G@TFg-L8xVFJ{4kUVq2MC=TJ@sdpcE_4u{62hh(@gf^g|q)YwbU{sBu zv}LEXp=mWRrRDO}2mpkdABQ?iOw4(C1{K&r+m9WXIlOf?%^LzefkMAn8hpeCZ;uo- zW=AOezHpfAJq$i349vwc8$-io;H*L{WGP^Z2Z9?=7W6;8{GlTp9iaOi^rwV_NQlX|}NG;J87ZlHhajxZ7vPSHX+;G-;uUx%G2XP|Y{{ z7MQaIb2Ve7kAEY2ftPR%SS?5yLQ`=ELcmsbUxwD>Kngq^(*zJ$7qXbJ7|QE&017yt zFp-ALk8kzhaoQp4Re;w67?1UZ!QS6skREE7Z^Az8gr5<*bP^&hRMW|-j}cO()n~d0 zgOcjXZ$f3R1AI>ts8z%w1YifuH=hIJdk8|6A7kdd7ZOLLjZLYNQt# z63dS{4|*E}VV_FGw`-z#Lw>Zay4wi|M!xaLX~{nvY{UT*A@}756LQw9F89uHHN#4m zUvF!%oX$8cC;OBNOGNJRSg_*diC@q}`juetZ0PUcS@q<&02}bm*WPtG<)0ieqmm*0 zsfIfPv)UVe?s=-qP>Q{a&0lAsD8tzoNqO8h2h#=+7YqnsF0rr4l|GO?;i6%KhH{8w z&x0nS5!mQgP1jxU@6PDQ;4WOi2z1%+5$M|@J&rB?0FKN@wlwF;z*8n~s_kfuBRtTe z$)LMorN(*jHUNPtU(#+ia^hb;iIAgG;ff$2tt1NR(mYq?LL5M?AY<4;ws@QguWEK> zN{vV9N+5eA5$qBWxa8kPjK&*+Dbhu1IeNQ{Xz!dImjBAWdX%`EOvrM*>_qprDfjH_ z-k-AGYyZ#IB0wS?%%3~=NPks?WzhZ_KYtvfG!V5g5t!u2nxS+)+)M!P$q%z&SW*J5 zBMzRo_0Pft$Ot$!hM3O)&uFQVNV3+DO_7e)aAnE~6sfE*n;qYONdh*DVwFuH$0Shq zYgzhpQ-f|1*V|p^fOm5rH&xu#do;C@pHDI1D+Dbi-?zlTs#&i|9DN=FR5${mk44bm zc}}Z-j!U8uCc*e(LqiAY$wSw#evLKO~X$UCWkODyJ?c%_bfr_V^Cb% z8*UmotcQ8Z(zDGflDz9yve(4I}0AKf_-bp+;0In@)3izaIwsb;J zUX{xTk|Ujz%{0QL06Pdk3Vv%>PJkSh*J&&@TB#Bz)9%_Qo^gHf4JHD~{=>VC(>6hipY+ZowD{$6`BU3E;|D=pgOT!^`u8 zfXMcrYL6_Q0qEE(r!=Tjnvw>@y`9F4xxY0HUF4-*%Eb7Q{^-}Euxr!rX?WMRy0{dx ze={W@I?U|th$k=14Y?@>RPAUQcYFxy2W+|Z$hT8wfcbWL@xUSEw}TGWNi*s7UsMtX zH48)N!9p`m7dt;a0J?d^I#7%NnH&do2*NBy+P&uE7uRb5!0jk9vrUr>sCOO3=jXV% zi51%fyYAAZAiv+oe1nCzfZMa4$wL8MagU%Y1pb%XYiIHpe_$=(SpMo$<>g{X!Z5kl zHPq4Xxnfj_S3-6MBFrEuLsZ;RD!W$sWrF|+#+ttt(Ay%|P0h`*ltln7d>tGYcf#M6 zD;ac)-k!Oe4(5hYFCYdrEy7FvFH4_f`YLNbjNA3t(|M%-w06ip@3~2Ti7n-ad>fmJ z8-=+x&rN#QM@**IC>f1jkiraz%Q}|qg4{p2vsF3GheleiX=OYtwk%eZaP?d#zuW95 z{U}XVb;DipaOEZ<%xu}Jln=HH5Lb~;Kz~{n^ZI$NKw?Qq-z?Xa0rzfG8@dQe{{Mx; zmdOt@OB4@3)codumr=paaAh+JxbFa7;XmP@PIw03sX@0>Ad)CX8jAx^Y+BPNj+{l< za(#mOX7MwJiOAq)1YQQ1q5nnqj53I7G2H#15LXt+nQ9B_OWb((|AukpiwLfJtXqF= z;7oeX&+^Y&d7JX7>}g$B_+B zY)1dfh0rY3;Ka+?o1Rc9Eb@(kcDh9X01F{!5)oSgB;3lV`w{a$P&}{7T~Bz`xmP{* z%S(?S(K8@}14x~z%l{yT^~lH024&K=%s7ln0e86_pq!ln9tu=)hyDbJ^w_@df2EKD zGH}P`ATW3IV*uRRYVb&hZ1X&3b28eqpybF$J|1h&#du1<8{3y0%S{sI1fP~`4CdjgIBNlM9wg;3!DTNLlrQd3>b!()z1Nw*}s>JfNM{_6bqRbQ!G1q zTP65Bgr^opw0w^ocsa9lU<_XC(U1eE3O)s-H)CIddv**$NShj z@IJ06z8G?w^paS6#zwpkP(FUjR<8a)c+j_!StdDib^6;k6A@CP1URoPkDh=t580=zE@kS2}F zr2X%kJAm)JKHH=oXU_OC_T?ksC+jN%tmQj@k##TBaH=aONr)CK$qT3S8vx8DCc5{$ zKq+M7;-w?b2EAnsVW_>~y}#aEKX})rs%DXF(yOu!vrW4ME@UYBv1aBU62hMtnw<~B zm7IhbfmcMh(DBbZUz$wNIi_qy!0LEicQMiU9YjJ1;&wN!#}R^X&9E8f{h&bw1?KlC zH^{H;Oez_Q<0a5b=eldct93#zLroqQS5~ zyYZ^W@K(f=3GR0vMsb6;?@elbyvtQaS)xGu(nERt{BN|!%OOv}YefS1O1!^VHb~!k zG&|r~8$b2C7vSLTY)H=xf)Msl=`$pj2n;Qgh`#au^BOg>T$CTUJY{o6?j)~Z+hgcd zz?YS@kL1ZV!GPRkX6gMpz6^EcFrodiEH}#ZSMtr(q=^8SE6e*g;F6zkFa`ac&F?TC z6~Kwcz%O(7&4hsL&j^pN?RZY6z!3n@yxSl&Z#L1C2;emUsA*QAs+75K66&>s0e9g>#`9!)bX zjvG1xv{)_>aR-YJul~W4CyNI#p!+Ila6S4)ce;4ck>Gk#9%XTpZ32Mm$vW=Hrt=w^ zzgh5Ke`I+hULMTy_53%N1bfJ{W~Ie3@tgn@3IUD~AN2maWCmeLZ?L5A${o&Q=wPs9 zs6iGbfcgYWc6&zVn}H=o<>%zD%}GmNkiQssf`z+C>@pu5c510xM??VNv{74+T~NXs zye9&HJ0vGnc<^3eg0P!3Y2g>d7!m9x6aTl_CdcE~6U$ZJuf%qp3JEu%y)EzaRUVcM z;Jq(drfmZVUw~;Xb;8vrV0O*^b>)1~@wKN3Ifb|9%|NS$qAu_lXSH|X#$4?rh zkN#LxUiRiuaxmYH0@_$HEeOJ>byFC5c7^e5f!X=BD`sYeHu=tpS4_%(-bnDf6&y4p zCQp%tvG}-C+*CDJ6=_v|qgl0Z`?%gav%-O+sU3n$IUZ7JiS+F?7a{Epdvj4IF}(ZZ zVFcwlu8UyC;l6><-`5CW(qYrrDW;ny$;lpkl!z;TD#~&p^I9#Mj#h$U&(TrO8Q?ki z+JLtqtkFLC3GiF>4FZPr-b=$J=yc*kDYvVf)>MaF=9 zjf~HFJvR2XYv1pac3snM+;_LuqJZv|jMJ?x6{>f)IA;zlc(_|Ng_7q67C)jHj?{$MsTQWLM1Q{z1q!2 zm#7s{Zv+E>BSx|Ko?_eOr>Hu3AbgZ^lvHSN{y3&zAFJ~2p3AW!=-8Y>ccs(PXh zi;VKlaw_c8?Z~<;Q%(2AZ(rMu7!N2&AR`wkEFb3cWMhBX2crY%`sO)!{tKBitvRSY z#5PGYMi#qiaZ&IoCWH);Zr*1I{m3Z-RcJVRJZJb-=DTz5u#bl%4i&w{zMBQ$iynL8AlnM&(N7Hg4^Y9ebV9RB#ruxJbv`CrK+#KlF zJxthPwx`u~mrzzb9zFHsY1A&%hhbfBt(IEz|KsUQ=0};4qhtjG-^%3AL3>IE`?hqP zS4YO{ph@tS_?u|-Q0ytAMkbzk=gLkK0?Q1+Qb`hXWJji`d{SX!T6|&8C{JZN`ovg~(@uOH@HzV-a7bn4E3%c%m3d96qGl_$U zx(@x9$2!95i2aH4DvrZQJ5zB=xF+lXHhG1qe);5q=n$?GRgAbns!KY#*!Eu@T^+dR zk2l62+0tt}+{%mQzFkata9UPAnw@~wJ-?AB@x z+!XR)UP|>|md_Xgr@X)zv% zIM`ie0L3ct33y|>JFDurJ=V9Oo>cpA2cpH45~9H$hT+dejc)mr2xc>SUF+1u-X>=! z*<}rrO^g;XcguZ@fz309B|Ng%3bo4>W*ugWd@QL0PLkU@TuNZbw{Rx7Dp5DlN`jD2 zU!fiTG};UAh#Ok-aispM0+9SM2G~Jr>2;O$SN^JeZ%P>S@l7y3*iKHS#R1jCHSs>U zQT-iq=4v{W3WF)>1uk48{}>e^cC`_4LY!5$=# zNDgc|X2nu0>teYYSYo|e_Bnx2%Wx6mN+O^j1#E9#?uTLhv0pG_*fUn58RvhQkOMjR zhPZNM+y}_fffB|ca5r#ND7h+#ZzAcc$p(>wtku;9>BD;Ka3SS4<7vzf2+vnQaKazL zrw?U7_Gj!ec5_N$uVkOPL~h9~b;{HC-y3(T$7S}ZY`FfBVh@3(zSRpCMlly5sb)Ry zp)#ByQ7IKw80Cs!ETMUnMSsh{OpCjYL!n8~PXC2v$%V-76YV8G{Mc=GNHvWSyVjSw z*xmP>Ec()M(W$>I3%EO!3~1#Z*PpER&V0aHSYy9o*LJf?YTm3Dp)?R~hdu&po5(HRkKQwGWUNj{oc3}n~A|50j z(o21@pRI?*g_Zf|`%|k4Vod_qg{>ndi`EL+dNOVJMP;xxn7i1w*gi~y1HlLTwF=f6 zsZCl>89=0+{_gN$OJYPjPloC#aCR^JtIxAaVWnRzT^iQO@P;#y6%~@;1O>Qa>_v<@rT2d8 z4Oe$0zy39>5vIZDff0l35E8l;EZ;ysf$S+KJveFJsn-afH%gulV8EWiWafRb&NBny z#oueWcD#hugUy4lwzWsxQNuc83XQ~+6wZjv$stFP1ypi}g_8f%gkxioE{ClV))Xx#{eQV071YQo zQjv};F|EMUtzoF-=hGY+JIHa5A4HT26p=7AqNJ$W3gbIF2A~fVzmE-0SCCNcCIC< zo>OC=Hv(DcM7*MaKkT>OX#-INZ4I9$*(28`-bG5eS<5W5BUrgSL_Lh>@_>#bzBL^3 zkFX%c_0UK04tQ~N@igVdZb)~H!J>&N@)DRYs;H)d|JlQj^})Qwkh6Vw^wR;N+V`%= z2L!jr!3e)Gd)vU+O@tyKqf<2UIWKI4$Wzj!+yRHCw|mxPyiMAax-y}~3Y%)@%7$vp^m4r@&=L_A8} z!j@};<*vCmj*d84LhEr|bHNv%sG`$f{}0{X)Zi8|c0@ZQd|%{_G@GHY_7uiOH|dG& zj>wvn8KJOD$d35Rq=qBrk3yzoj8g$OJJHuS6S(=aA>Nv6eYKv#{CFY@v$p}_BfN;T zRZoSywBy@V6`ha&vCK2v7H%Eah1$L43`x@qei}jUmV0DJiHsb@P!|nR*iobqRm+W2 zo<<+^uBC=xH_RdT#K_AhdUJ!#Eur3yyP(Y>59G zEimz%^4%s7_4(s)Ur-EO)~O=w2?9P|ozXZRl+eboV#w~@^8N2@TN*Fb{}EIE0$j)i8wRgk98_yo4czTsOcBEbvMq z5e>nr7NiIyR9OZvQs98g63?7{ZnX2Z2_?5?luH+PrCFG%}U~3V&h;_KFj(g0WjZgHAEYwxRBuuFw zO7!fK^5uG7&7l1q{2V*=legO9gN6Qk~3!G%GOjxP?H&f@3 zr)ocvZyRi=0;U*u@`HltEYuLZlC(=t!~L+ekp$6-@O_*TI8|AEFC)$XcQNE`kMh54 z-iWJ0WtN8wn+;J9kTL3KJ%P=@WI!3OmpiSCnRp!BG!H}S zU7zb1ECN>Xyn_Toe}*pc0>6GVP6UwcE{+t(;{1-b>grRq4_^Op#8+FiA)X$u#Z^gE z4qdbbBQvsqdKX?T^T!Q$yEm7JJmZy&Ik%w;9mvnGoNypNxGnKz;!vWxhkUMQgt4ga z8vf|ox(mDBVym~yx7(UDW10m12^o8+u*rQ7gWO?fDvuGhLHBf%e@PF`f>yz2aAr(h zfM}!#x4-(_{r&Zg4E9Aa=_4gH+%xn}n*{BuL%T^)A^>~~Hkx;i!wmc>D3^f}qm$}M za>soxSaM(R?TtNq^dH6;7NYUlm#31W?3kD~XNRmmb1E)k_U2$cuFMcRDJ5N4_Y+wx zDK>|#KX1&z8XWDuUd$%~x0hQ3o%vaO4u&%lj>Zu)Iu+uqpD}aaqs!GhZx@H8E4u2+ zAN7w5in@R>Rrg_ovhwUXR zAa;r8EX}kkKT^kVe6;XC#Q44m-V9fY3&#;#LKn`$ILI0{&oXu5PNsvnYBSL90Z3)B z=`P%RR6*kvK`c;E$CDh42=|Xf`a_ToQYGKzFsDs|Z4$!7N}zV% ztnV_X6NJnQzD+ViTt+fMRJt1A(v1c-E`vg7IEVkjC+}fL=aJCibxQfg&SU@bM~(;!`{cVs=EoR$l?`4M*mz9Zx7y%3agjKySO2W7 zkH&^$y0IsF4DfPL*z!YIlw|49l7^**_{0!%1F#7iTy9iyY@5ev2Buw)L#CRS-ZceM zZW@T}_oq$Ine%-Kvbp(~-sdvNdt_C>NH7)H57^ggEZdKaV`x;!H!38A)c$edvu0r{ zBzn2??=HJr0ln>n8N9Yo->k2;GGs*fv1}t1gjf4>- zLE12*T-cqbCHL38PokuXwaKQRqe2nWu#Z~N{Muh{nwgLKEfdk~7`6`h*2jOcKKxHG zhh2bwgu8j#eO^e~Z6w51eE4W60Xo7S(G0A5Dcs(qf3Usc=#p}&;hEG@C>i&Skc`W6 zf?4IdGBat0zc$q4BU)q9N@HJnMAgjad-zoEd1|2BMBIP<9h;f(ftI;IRb*cMw}#MZ&SDc^U~|j8$w)|^#!H~ zm(YzKjN(oi5iJj3W!DEe2W={$1Wv77fJcx!V^&*`q;Dnv>2ats8l)fysu%hrZi-jO z)8NhVSqD){7OELur28({opkrvg@$2}t%JV^=<>D)!S=ODqHLRlxrHR9cD5yxI^=rtdef@QozpdUI4&S`q^(K zOu!bn)L>I2g2aM(#hx?jnIg;7-(J9OYZc8n*;ATeCCEFZG(DAs&axmV;-ZgdBw~GA zt{^LBZ1#mkx(0t!0IdEs991wm@L)VB@O>P;oGP@RQrsM_5EOh~2y>2n0-Q7mGs|U+ zx@Z+@buaz-P$W>1w~NUk8ej{^gkq7@rX=8vWF)W5`i?4>!d|&rkPl0N-3`DL*_)Bb zxK*zFYRqT_3Xp=r^X1TAW2*0lwJ`L|_hCugxsF~(I@sW6fJ?Tu>iUASU>V$B5u8(R zU}73>`^Mz+A*(82a<0=4V+j{7MWzk!AQ2+(M^Xa~S0t?PIpTJ(KQ`!z+I9BU9}Hk` zS->59-CXckF5BtRy8D^clI8)U(pMw6o!RAa=El|y#rX?OoPYNca}>*n@yDoPU;m)5 zd;ntSw{W064pEUt@mnn9f`UyaV;I}@$=<4(=0SfZ$RJG?Fvo$t=pV*fmcATC$z?BP ztzVxOdm<&rc(g?&g;V7=p&+1d*qQO$Y>+QmC&|$XEdC~ zDrZ>p1d#f-Cq})2#lvzU?^7#tpPLt3X3l8as0%!$5caKTFpKF`9nK3k_G}ZsI7?E& z8TysrT;?KjHC`i2Z@%ic;apz4xV?Cje!CLbwgFhkxLmY-XqWLb&JS~s{l~{P+PrOd z>jX#oW#45KLv{*?tsnj7@uiXS&s&^+6yv_Y1s8{PKS>RHc!*O-=i`azfMiUAG@Dm% zy7Yg4`CZW;B(@yfb?-wyIkOp-AC^RtBrl(>j(#N)y|5=6RmgE>y!wz>yI*P0O$N66 z3(E$(+ACT=J|J0Z1zo-_?ATBo z&1Rm`!bhxDd|MfAikt_SFpcZ(y;Xk*`?V04jB`UV@Ul;g#K}V*uoD(EHZTPFsHD;@ zrPxsPa42D8$@-C}r{wkJsuHw;g^bmyo(1esgt+|O&`DoV>g#sJgQKWhaSAQr)K^xp>5~LjakxW_4f`P9eNv~>F&@8lgOYBtH|k_svTc_ z4pE(wp_*urYLG>$3j1@|L>K-efLoNW1*-cZ@5kX{aqYO1xLxOvVDj2ofpxN=NAIT6 z4Hl3bYq~w9KdXFhMfH{Y#X|Rol7ua9s0~bpT0VI7wJ~bme{9V_usv)(_{!@eHkoZr zYoD9ger_1L#PwFha0i+-QPoSN6fbUF`IWN6Dv|BV&WG2-JlwfYp9abRgdYrAD!u*k zxO*yET>;?U3Sk?>?{d3k?5|$7dJAX0b)nE90ForgrKwqc=X;z>M7QPi=;KOroq}^lRY{~h2F$Vtomen*NA-@&sn<5kgcdLJ> z#rwte@2XcEfIOOFZ(-uF)n0ygU+^SVfz?#9))n|o{R7Jcjtx5JLZZ96$_z{ugE?O9 z86Lg0dcFTrx5A-4a^(cHTy_sW2j-vj%QaRx5+;16)wG&sB9%N$v8yg{zUxcy>Z8_t z5I4~-vR&!E=x6fVh?T2xty@Me`8kZI)K53wI+@0;(1YWc`jZLT8M z7G%t_h1zJx1;Iy<7oFy9WQxL8lDm+(96qe>qvTD=t03x|rL}zSpkzXjwZ}{+?$0*0 z@iLB_yI+KQWLmJ)*h2UM*hz7$3RdlXC&Ej@l|dFNqt4NT#60ET;p*(hA~`F-L7T-; znJx1aHB9T9p)PPY+ocyTgA#Wky}1zMQDtJKCXwdWQxp6WVa26-@97IE-ahzcAi$;{ zGx3A;{7PFsB%d4hreda45VCUKjo0Vbpgt#9!SA zd&Z9z*w;qO#@*ek>VUh^9k4o*Jd#8rD{_;6cG1b0#5%~&$W<_M>8Tnv|9$hT;OQ=G zgCImOVB7Z@!(QFtSO036M$DI^4!~;hK*);+~L)NXfA!HnhhB`m zhzoxl%@@2qy_Y`RF^FKevgoK9xv>Sf&-=1~*u{e?2p0*gjf9j)jpVKa7w36MbX6J6 znSF)krxYUOAOeseGIR&nfWPR9Zn}|h43PbrxJ4+?i^+_Gbeo70?L@N0c|#1%yHqIV z0oMf7cRjgwvIuL->$ihbwd+athr?U$>22hF$-78CH~8b_fw@8@BSc6fNDmRu#Iw8_ zWz^>?gL&Gl*rTO-6w3@)p%j)5S_VJQ;O)Y=Wid`CI^GK0mL6s`e*+Zu8nS@sdtxP; z4C_YMK{ldA>?}{C4NH_qj9o==R9vEMRO0VV-_gK8oS1 zFl7PdC=Uos_+!ptCvw;dbcv`s&;g+!Yz5A#m*G1O+$x8~3m1l8J>t+LMj(Ye?6(hp(ME4T6_qh?T)Y<4dh#_L zwfp`)?2hz%XEP$;OIeBidYq>=yAhtvIR+_zm@;gs+;oMl>ly8&k5hVCLj+=UXY5U zndG@yzNx~BATtO95$Z1zGki_P-Nrq4-XHR{J9|B`kCW7mG&HApfqUqTQ8Yn}-sGZO zH75CMBeLZny%KX4`x<+{6^r`-;40@z_v+e4SX4GD0X7z1v;CsQ^J z^8b-EVX~`x@Pb5{h{<4nr7(N=Z;c2Za}i-bG{>)LE^sSyrsCuE9C($7Q$bzBLCC3w z2eo#zJAR{+<&`buywt+8^-sDr%CJji7iGvnJhGXIdkM0E3Ao%>@m9bZU+Zof63oIS zAuc0CG>^QRAgR>*@G|9ZG+M5T=H#kuyVsUt6I$T<$%E3v@v4&NFVlKcn5Latu zB=F`Kme8=G!{p0g4mT&Qu6cWL2*j%KYFwrk;iyzmIG=JJ7YNcLwH; zgi<>W+Gr(w1VdjrYBcN;#2KvIG>iVZqz_4~drN4ur5Bo&?y&1+JOa_A5&Yrok9W(q zcs?AP4$E8JzZs2I6|RNh=Y{b!BT~x#h)u1N>b_`enKkwUNd`fS=o0gih^P$+S_Hvo zAiKP*uq}d@MlYydzoH1(;U;#|U*6OzoDGr7-w;X!1SSGvQb2A?n}^&HUxE+agWSx6 z=q$z=dmJQjgfe+JQjm1nqMuT=)!aP@U#c)d$uMSxc1;D9&H+#`;0l865c9zjiI>m3 zgVYspR3-7+cnNf*s<+e8Ehdn_oLF&O1!hYMD~~lY@(F@fUbP-r|C{ydfnm*uuK1(X z(6`H9o4SsK(h4(Aa35&Mr-arHDU!kSOyF2t!7u|%i8Kv@oX=5I;XENnWgP=>Yni0@ zGX3jDHBd8!%zd*BJ6l37j8wRa-=L+-^7&Q@S8*V)~xko|10a>vf;Z(o5v8Fr|Sfw$vu0PryB#zmAvPXM(7gxrK!5&w?R z%Y(dwE62%(+biJ(Eg+#gMAiYTkKIl_@RA5k6T?wbrlLAY z*R&EQL8E;EqK9MBAvikNdinh8aPa6_2t9U(jM?!s4@0I6T>s;4VR&UA)8>rP=k2mdrF%b0JXME*)M=tAB=8x_{A|oPbZ&_W2 zKM8ghp~+$eH!p|1hBUzOgb8*-v=Unoxab7zgMip^(Vx-1y4m3fwco6^mj6xeY~P|< z^`l68)l{V=PWRI;F?RG{T&U-Y3qiRAsTcN|S?%0sLok$L-4GXMESKE4A;=~U08V!! z%2AV2-sB1n6)dyI7t{HZ4qVN6c7+>UjS)ReEm>=o{P}lRr6`Z;|CvR#4$GQp^sgcB zAW1e#_LG|EP07E*gcfWkECY=(ct+N`eiW!xfWb*`AT}-y)&(D?00>{LW`1ef8B>LM zzl#4sr-~aCfmAjf$-vuF5QbRvaMuScfl{LvcmDS;kbt>Wfl1hQ6@-M=qa6OGrECdA zMP>gEMg)WP4IJdk#AIWi6!*t@`Uw{g%$OEQU^F;p59 zK{{b?KI;HEa=IvvivRJ#$Td74HwS>G7hwpe|JxnS$9RDZcc2I~D}sN5KinBW*L&^v zAAq9%y6}#%`KW%>Mr5_sqsv#?3Gn|Z3>qmGNQkg9vHq;BnYTg(Yqf&8WH~`Or;8nZ zrx(x_cRr>UL-I|Xuj1EpP4qgj^^K?C1|O0cF?dUI(ho56X#Dzf(UHBlZDB2FK9?t2 z2D6|w`6s_wQVdf_Qq~g>-{$=rjIyvO;M=hCPye7HinjrKTg1W?8;{O(g@0K?2!a3Z zF+d2#`0Z{eSHrOm%#XW-{oennNQ9@H-o^TPp9Sd4zPRYirjSe0)5P!Hh*=GA1P_c# zL{cHnE&*!GtG~sen?O0CFcNTmgf3hb3V4$57(u;?gKe$B$8rn57;8NJ1I?wI|00mv zENOuDnS{j{+<`Ze&Yla!yGk}v4YIQ7m-Ba1hyh0lt%LML_G3H2e;8bJr{XYAF|`=y z;18FUzBv)F0XtK;@)fiw&5IG?d1fIM+Ozf-BZGfp>YxTX5anNg?P&*IbNsdo5KXoz zf*I=-JQ5L`z~@l~1i*lI0zaU2x$I&~{7Vcp)4E$wW*|9HnNv=eu7o>=mRtrVmM|Iv z(8Vfg1R*RI=8n+W^$U~^arwvTfy~h3fE;)qVO>kZy6}f9;EVP#@idX1*IWsBbB42tK~WUCz2;IE~L@@U;TM0C&lg#VEh4-lD)Vv*MA2 z!ss|bI&6g`J;T8C(<3&VNd^!DQfKYUE@D6stQQLx-et|j&;B2ms=ESXh~ii9ftM@& zWk!RC>%<=tZ10l$G$v7M;s+^-j*Zkrr2Pkx7ote~jhZy{>1kGTd_Yzd3i*GYqZ=U# zFUvzx-zs9eEM?~YAM~`1#+=-nG;zW*9ZG!wQFcHAXhYP{vyM}Vw7s&jC5Sg-skJa- zKA?0lea3hTuqpsG6@srDH%IA?CptYIX{$h<&n;@lbdmd_(M5pA!^!zKV}jcOQbZ#* z6}~mt3H8+%t?c6LJpzqPAfiL^WY~eoQyJyLy8grJ2G#|g9lpf!eHLloe3xdqkyETS4Evk_D}&_gTF82 zDlnWXqDLjWJ6FH97tsg0SxASpW^D;A1J+53lEtHi516ZeLo7=Y!U3y7;JXGU z9c3d}CA^Rb6~*M%xWHkl2yemtlRKl9-a8x1w6y()?BlhkmCfZrk~RLVF%<<>C|?P{ z=MTmyFAV-eR0U~B`$+Y;3PxpK?3*Aml-kn(k%0P7G< zh2PGLr?a~J(oz87_0Q@AWX)7?CQV{j!*0UV3z!zKIGs>^ZPVgjAW1x%>nq?JeS{z# zXC~f`O=!Bv0qjU7^$yK37`51IrDJZ0D$a4j6d(t+)$fRce1ZBO?W%+`uHjs(G>!6M=+TFhlqMx8EqMCh7}G|qrR%({$B{|E=H~PR}BlU z^z^CPL2WFkzCDmACoL!1KmiIU3G`SB2!I)3b#F*6(GQhO05g4tZ4J-FIznjV3DBwc zJy`mGAQDIqp3`kR-MWFD{|5oN;~t^cCSyB0CdEMSaw?jW zaw^u^FEd&TJkWxcIq|g5^ds}5HR8Qv$bllUop5p9q}2qD{ii%@071e5 z4ORW2uh6G%fa_ZaHO&)sqya7c+HDpBytNyr_k>eJLJ#gUR1v$lUoZ9i;B~w+b&9SJ zcKk0pLE{=rKY6+iTmVjoxuZ(NregODMrI+Ob0ln6IWRi44w`=B7;gU<ra#u%_+J zXTNZ)+vBPDt0Y4q{Q_GK8l$f^Ge<{?R*3xU#Rz9RC^!lzPQ})GmNG!CaNBCk&Z6Rp zA0`ST5zT}-gB$3)62X z#JFDtyIS2b7$pojrXSNVg5P_i63R3`Y`PBs2y{ePHWEH^ocG>ODdYJs8mQtQ1Xk<3 z+=IFZa~jj!)6+YWSIlzgfiy3hPP>>33tx_S%>!A~?J|_3l850EN9{xJ97uXHko0Uy zKkK7k6YTsEyk@Axy;OwxKQbc7qTs6o_F|wyv|0n=xa`z7|1&ferb9~yb0l z2y)lkv_WD3uCNfYf3F~{6Lz6z=&#HA;0ph4pQ*B!8 z4Dc@kBT4#Dp#md>;up{uSquRt5c5e^a#9I;%Nnv2M;sA@O}qq^NSL=!nUxA&e~lNl zdLi^4{$2?|)Hs=>#B}9+w0?~cWPO0KwvI7eEd4zX-6hV{ii>)mvdvM1Eud5MWOUZ{q zVw3_)sSpts6D&~&s#?0^Fq;@S#IX>)oRkB;A@Bl#fobRfv=s!+5k}vw4+9^yMSq!i z`o|tG+MrhcKSR6fb?xCym4JQURN2?goq8ybnRL1SPBT)sVlA z1V{u%h+#6xR=|{OO@~a;ee$z0FvX~~zozJz26l1&dM-HJ3rOXkp{DH{ka#M=gFZ{cson>yu7ka2I+8+v?KO|1%qc$6O#3tc5~+7VbE0bUhw3 z^Hv5HQP1BcQ0&Xef{%KI#mdQDyx{KFIDz&{FeJfT!FIuvXg_1=_qmieLBMDGoZkR+ zqsg+S7)eEqh89Bpb*gG(VDcqYUNVqB9wL5XEjOI|Zs#z91p{Bsa(CXi=ZH{l<#OnT zb_rGp(y@uK42tSR2ZW%3KX&qYvlc_GRZXrXOh4(}N5pq~k3{u6OOdypmsw=>F0jW! z4Jxn7c=hCs?z{KKXs#T$pu8&g;djR*-fw^2^9g~J1)2y?!iiJspY+Il?_YYKAlIi? zBH|31$XD_{zR^!vX`1)nuZe}w6=oli;GA>BvtdUmSK<#gm<#lIg%oy2mTqL0Jl4-# zd~>uijaHH~1UeGo6V@AjIm2hJHQn5&Y7fxDZE$+l!PlSXrE z1525+tJN*`%al0@?zf-+kf8qX`9Z-=uFT*Oo7Cqi`Dr0``RSpbxKe^!W@FVlMn2E1 z&_Pwq?N(F`fmzVD0CUhD<$AyYZCUF!nNZ6aJS&hXC`qx#`1Eb=SJxf8<5FM6)7+6K z&_@{1?9Rg;kFJl@t9wwIPnM?K;bpufJ`;cI`iJ;ibY|bSH;i_fSyIB^Xa}oOgXv>Z-UL5sRnN7>#7nEG zWL==c!H>t_Iw7d>?MuboydJ7Mg`z(#=voJmwUUELs&Z>cC%10Ap;=JHTdz*`Ux?72uyw^^#t%bz^?_OA;u~)LzCYi+h zN~8iEKoRk})A!9A^dYJcB&2L6bbZmUXFGv5m!@lduWy@Tx15{&NF+pOWw~`3Fn;gJ z${xMCdzqr@%>6Y~e{rlB(tm_9zCG(A>AYw{xj|c28{%4w*1Jx*(C?Zy8f6ePUc9D*WoRo!BJWv{j z2xhlfYO@;2BiITkA&@5y#h5D}#4E>SwOZH{zdKS<7Y=&funGT zjDx$GTUQOR-LANZl#{s7xLEaDNSK05Rtgxjflzm&>_Qj!(e*lydIjZ8mR1M0DP2@T zi(7G8%<$2XZ2iBfQ=;>zQ^l59>9e31g^EEkd7nI73s}ipXIm>Sui;~#v)v*BO>kd+ z;e*uGf|Z4D`!L>w=IrZs=`lO#8FYwhf93dEe)pHq&oe=d~oO@zb7y z@AkVxEflUNWj|=Pbf_$@;b0S7*?>+HV4r5j?*052C_M`i5L^)4fKCW*+D-}60o|4` z`G&Ezj+OU_Q1X{NcWD``2BEUuG~e?ss_-QL$dr8*PEGxf|h08BNaD zkuM_+eHRgToXr}vTUE#-8#6Qraxv)J2@0>A zRck8cxveiA7cViH`()jo1voE;?z(MmO`NDR6+8XBD-o4#W3oC0X@PFcH&GSmTtfyB}m-Z=#W!Ml%eO|rd)~%2} zsa~(5?tg|0wuKkbaMw##0YrM+*Zp?Za+|>_@(xSEo~6JquNh58(BlZ0ELz>zJq}23 zHGx0)kdjo8`}NE8ixsHG?aoKp)V$)l-?ZOEN&fl{Z-QFUl!2G*N|(}E%IAn^b@VMT zdtDwtI3wKQSvJ>=pMc`F5|0RBL7(4?&PwF8+Xh{de1i5KZ9h1u=p{Tjq=hm7c%V(+ z8nwr(`YcIFzvp$_K&15o%!9xnpR#^0rpsqBn#7bcr(D$0g5a{@bVSaRJ(yM`Q9-7a zFI83mOixSZo-2j8CznXZKfHfg)o1lB7Ty(Lo~w9D~@Fp;Q}KgIzBF6pK+OYF$=w_SRjHbh}+pj zi22jqK)KTszU$YEG%^~`gV%rV)<7bqy-ef38YAlQILeE&+&?Qu?t6y~{q2-Tqu3%e z?=FPCO5wdY!FBjkqpgr#TFro$^6aYHTwNORVd^D)3KLijBoS-V6k#+bQlxizqU_AWMrKg zrD7uqVqq}iAt{aLd6mprmc&G64$-SeF$8l)<{P`x0z=`OWu*qv^Unlm5;NTeKpI%{$p#qtX+ zwDABffMxp5JUa7S|E53P^KEan)sEIpTn#uw5{B&O`g`4{a)yRQ-|Y(0{HYSt9PS;( zNIYnH2|>Qw*&PV8pkvN`Sugf={}bK>_1_b`>=JU0r}Agb9SLD1j@xWY@jyg)Bd43@CA3ku2d1030SEyh+dZDBnLKW`Kr0=la~zf&X`38XR=e_ zO%bY$nF|mfuP~iKA~*XrjVVT7ExpD0d^l(j0=HJG)7{HnBe3hG#Rf&Q12W&o`6c!n zQ=WZBL4i{SZ+t(C-e>j6IV{1MFQGq+q47-W=WkXgOa1Jp2-kwmdzfC=XySwFrVAya!*sr|K?+R2l*&~**l+b_~ zYy}Nn=vX!k&6UhaU9_-{stB=F`7Uma#xGsQpF>}IZ`^{Dw_mH%y2}NUf*YQGk;`aa zg}QUz_D+D|s$NX-XD}nL6m)6#Xxb@y{Fo?qmF7=!DYJ%AXxuK3s2tFLJw{1rvOjW) zn*F`>OXp*f`{(0EIbZToBiOG={gA?Xjncwv~ETD!Hy49;{X9yt4$ASv`h&e0E~2eKSx$?`Yph3 zD&`wC()~jP+EX*%?dN5jx~4V$vgA;Y(k?vW1$*m#kBzYT=E?fW)tg{rdnJ91Ac}MF zXW!p5b5tmd$RZ-IIU^zw&eQ@5+n^dvBs3**#9zTWDgB)*ZT=2<;=*eA?GY4d+Al=( z5_HIKe|;wL2pgR7dT_GJwIX_NYOLsvL9)}b!J=p1!XgB15-H&YNqQh-Fp^1w{_8@PY)%612MRq9>R}+ zI%A;oVnDyNqB_dg^R*~Bcqtck*?0}XfMCb~o7Pfh7lEP$AcNsh8Ynj0)&;kwetk2) z1l5U*p}B5LGF>gl7*L2sHU3)UP+8B0U$G6c#OJIVo7C%~E6+5FKuGQMczuzVb}qoe zjyjlJw0f_THRbR9o0`~3i&$1jO!17khZ!vW?tncQLM#%j^}O$-uMfWFgJp3rcHwni zm#!nt*|*NTpr#CZgIl36|BSLtk|h*-u5?N4|M5e2ixGJvo3cwJWo*btjEnX@EJSfKp6iyz!g z1RE4-WtdMGIk0@DuzIQEOD{BG?zA2S%ZHi~7a@ofI>F-&h4UzZm9qK9^asX(MBmvN z`!J_7}AOba>S6)AhSM%Tt+Lli3@lX6AS8ur`e1_w2MT081jZSb{Jx zdGdgyC&EJ7o6n3tPE`B~?M(!l5nyl0#`sbWNjY-EvE|#IxzW~z=lE2<%Z)6OFfV{ zmC(Cra8^ohT)7meyz{ZIXcn>dU2?=WeYt3R31FZt7SgAhI@6DNHzH+_j!_+?P)Odk z{hw~a;xaU-z~}jKiaq}O(W4LX6xvDXx6l(@=8naLXSKGivJ0LH&JY_d#-Wwg1hx09 zjdjtSV6D>&9f%qG*DO(&5t)h4$$s@eyl6JYVs`ArLTSI*#+#1wk*=!|#XIrZ1mbAL_hPtuO)E35Bk}2j^wVLYNul zaqs|Zz%|9!Ob{@s16~GUgRZ2(j@4)F3-S!C{4zlB>i>{uDpBtTp6yG6tzH|c+m?Kp z5P(b51DIZ%^&-f}L+1~@)Ov5nH)xHXGL^9Xss6O;%9=E-gFq!CV?vC$+S~@DB`NId zMH%B;!%zLscDeMgAB^NIUy^D{C%3+bK(}D_3al+ z@kl)!qls3!2q^DQCiP@CPNv@8_Hmx8Gy5#-ol7XEuL5>h9Pv%l1$(+Wd?4uqHmQl; zgCS?MS)DXZ7Z@v!6RzN=6PV&SzBqQg#R4&4tQ>p(XM~`~j;VJV{Gu;u8^&WjueR+p z*m;BDg3_CP#brSB=jIP?55>}AdMmdX26se63A^XDA@o0hnHhOBSn1=e9lPM*$*t@$tUW^XMj2gqm51lOFM z++{ziA;A`5yExyDY*0wxN#=_|B9L&@d0ntUXB>=BZ(h0u`0Id(Ag|VsU5bk%4v`_U z=Yc$S#tS*sjQ^S70@Ggro8?F|HXuuNG*RcSVvcqnV1>7fd=bZf&gO!2v28@Py6e0Y0pq>y5|ks!_WtEl*mTUewZ$~B2+E7$ z4?R`|IFJqGGcaIt+o~Y>pUyeT;WD=?@0V-5nm(hp?T7&d?_@qV!^H_q=8qB1fd_2+ zGJcE+>RoB!r9nTUczJGRj^l;M;{NGSP&uP zwV@dDkjGDggX152J$vlsb@RBuL^00kO`dd~391K0nn$*Go-REl;a>%>s;Zg_oa*z- z?CN^>r&Frc&&Jbp&2QlFy3IA^DQeM9dhU#ryp+%8{ZcoAkvA}Uh@zm(ur6(8n1X_o zUs8rI=9}wqERr9UjTamZJ43R3OkJkhcqSMJ!#@y0w;PLxdcmr(g-3&Qgx$zl;6h}> z1k>)YXxOdougMHy&_TDXA!@^HWfV{GUUVh0zk=a|Pu6P(DJhjvi94G__DH9@Sq6uA6K+O(= z|MW{p;jfdh<1kl4_`p{X8#j8xTsmVE25rETXR&VZ1UJKeAr|1~%yzK&QPh!K_ys1IQd1qNLTG$! zuoB9h+EV*W)SVZ47Yke)-%;i&?zznS=wMs8*&eT$Pe#FPIWxX8EBvPRiCFM6SgKkM zb9yQgTsRbC1&fBKBje~XvKFl1(`Y1jY1tZv3K9tlp`$gu6CEJw5s+o2 zk~~_}XC)<>^CCC{??%sd8*J`)Fdarb{K@A?9l8V z?Qf9I2T<7j(f@lNndJ2W=U$FdHK@j{<$KoZblP1H)KhkhRU>Q9P{b`OQps zA++Foc*Qa@wAxv+s@1kvU+(gYojt#IlAjf*&640LdV8rn2yR|AMZpRKT6oKGn-;_N z8T}lVsFEj|4w>&Bvd3Yaz14n%)M;3b4QE>P!c=v2Bg`wXbEKiWoh|?`ZG3xbGt6O5 zjz-Ii7jB>UU{+#(6`(Tp^E6WvzCpT!;d?*9WNsI^&aRtrGzBzIKD;up*O|=l1bK^P z<5F-bEYU$CPBZzX-i4x*k)E(_T=T2JymQKsO366N8Bu|-Rk%O>hl=404Cp=H0K z4$LpJDW|ilU05J_ge0(Dw*EMF@ExI=+2@6wdhly~$sF>!0dY^?LT(j|bX1-`aaMC# zSt3SaV{5MKdD<({H)t82CZX5_=s)_wdf-Njt=u$HKr7gI*RX^Ywv~v9y^Jwjjo?wu z%@Det>OI=4-zr&=BNCT=*CH>(QTYE;9A`!JSt_Ku)LMp@@PEB`{#u+K2HnLC3_Wsf z{G2gOTOcl{e_v3PvqqwGj)sHu6VeyHKr+n*Q(^NFB-VzWU5K5O8N&q2f+gc+OSwY9 z2ZhA*&u%W@z`BivIl+tSDCHpK0rwGcxj2(Voi${BW%$+Ki}*pBVbX{JGMrY=UN7u) zR>oSxMJg+GtOz^{ zG)3SF@~xE8tk3h4wqgHR=mg$Bq2m$@i~G9BtJu-cK}XAm$m=i~xVe0=X&|N46HYjJ zIUPD4wukUG>HYe(%bxfvGw?(cvhJLRN)g1_q`@V%Tbcbwmo5U>5Og033&fD!9q@pc zP9D$DisL7vvZtPR-a)*6Fr&7q$D&>M^-5ni&tJoMuW3GT_`QB}UkJ}PrZAvK^X}a0 zrW6yt_cP2gNF&*Sly2H z?*{>g6GU+HiibJ>JE$W;_>x$QVIfjKRWPu+@p2%PRto#C!`9(5v^G-A2TBtmv(?tW z>f!kL`Nyas?z0W*7kvn%Ls7wEo39CmU&PG~@@_2ix*zYvf-7bigI8}t8bxKZ1#`mI zY#-k5e@TpQlmL5*M`c0_#dQ{)mHxJH64Kck8gBvBDDFKDw}POv6RAX@BPKPH^{&s$ zhMRX@)poyX6;2d7VXAXFjVT`4W^?F&-}9^ucW-(FeecG>UejBnlKXSlsXB7c>;2(Q zll9McXscMg8VUcS0oh>QcPoAla{=g?Q_$JjM|WRQ zs&09GGwGG`ovlKi`5iHz`Rx%^Saj@+&~OSLnl`(AQD>0$Cu+TSe#s+kNM-G$p~7ND z{lsHg$m&xj?OnH?y9^uK=q%R*3>k;NG`z>qQtHbz{p?xuak9JvTrgcKHfZfEvZd{K zdSZWSckz@Wb6W7!2N5phtV$|eq@BXE+x9K#(M^mpLN`X?E0q_H@OZnA6eXkpBvate$M>`l8y&_MVDzwjrW|8fDe$s_!M#VgCrx_m~lD%QIkTFny9 zYt0-l&bpcp5O)t)F#9LdV7G9iI|Za!?J7}Ok$yK?`wyIcaodO!iysy zy`#9S6;o+9t4fSXo;i)<7Zwv=aogCuQ*^hKg?Mf3S>q*>oOz710JZf%jAn&P|F!rs zUHZ1p{_6Cg$J`c>@6*WdU4fJ1r2!v{me4r;h_K`RHBm=SL5Z(ShYmV#2!Cgp|7+u{ zO|tSlB9&`K8vg~WyRZMml7m%mg22~5F@@;xeR^~Xg_*w~{Wafga@u%-+r-oZ_mr!M zKMz*w5&oWRe?f88>A;(V@KzYJS4j+Sl7O`H(~p#{YeP~6K@R)H3SZ}L3M`k6#GD=u zh&jqP>E};;ZR(5vXgx4+cms0@;Q{YMxKN@DV0GgQZC5Vx94)d3VU%;p6oj$v;T6r{Osw!xq~mLAd- ze^d5pcWT(|krX;;{Vei|SlX|JoZJU57rwRb)VbIYhKhoiMuo>QvZ0`mR%} zPW$H8tCxzQE~Q(amnC#KOmx0$gwj3aO;T75t0CS0TtCC`$t!KD;aplg%a{euv~bV* zx$`M{sj2sm^NUB=8x5WsgRWI*)eEemWKIzb>Ja2@L3`8bVjvYcMV1!^e^f4i)r8&J zd{IvmMnVDMk4O#HkPfPLWiO8CB^z3b*)o%$w|AR%n4UW}Y|X4o2R)c5qb7n#8wiW2kOAZm6yA=>_k+iqy`@Gq!>+xp#?6J? z#)aG#9nYT%(AQd=ffN3#k1bjr_nv+@#qB0rkR(}?sAT5-SvY6qH#>SsCh!D_Pu{6^ zf{YoM?291X`!Uw+AwhS?xtOkavVq`}-j&$+0Q-4`^8jypdaXVY-rB*p=n8B*&8|s2KXa#cZlM$!3iMi^RCw==KTfwj(}wGP zx|KELbUH852 zDU-TQGc$R5IZb^`bM9<&xM~r)q1Ji4zNq*9_$)kvO2GK-+2rv)vz6v=$xxc}35&}D zhg(P0YXoFW@_}#K=_;vq!=1bvL=21Rax_&7{7jTp)r#gIy}s`0_}}?=GK{2uSuo8WN_eytm*30S9lRt zg3|O)@jUO#9%&(dT!Bpf=8N=tt9+b@)9cRE;WfqsXMSM?+F5J{D;wvNFJB^oM6Qp4 zxcQ{+SM%>5SA4eS@LyvqgD3j9FFjuJfqn2s{rJ&rB5xI`sCD;%uY!U?cLHx6IsMQh zUFv&BH|;j4YlD6U`roQ^(UyF zp~X0^pa<7%e+agt52J2QSuB1W=}u)AqN`g89r@PzV6~@XNm$Ii)|A_z5V0g-pG&+N zzPB93Xd?cWAMVy|Tz=!`4&!yfP;Lcc%dQ}4Jo!?ih+gV}ceQzZ`f3e5H7*Ajr()yl zBr2`^u|G17%$M`UXr$`Hw?x&}5DT*2)xE}y=Z7r5UiM)$WvwykK5}YZnNMVmH5NI- zvk2#Jwbj4C%19xe+ayY<%GI22;l5oT>b}ykZJq7cq`EZ4RVIguqA4}o!5>?CNceWX z)$EAE?RNsvlgyFWUh21Egts!NG>yGJ^nJ>$u$xdfZQhEcyDhm-BTT@|rM;MrqA}!2 zp+1*ACSmUiiq-3i%{JTrQ5Y4fgJKDJ`2efY;@c>t;8kit1GqbxD$qfQn3WzmtI@c`1T|v;gczcia52y9H%OYZL}E;SF8T? zMV??s<|tI{G2vTOTO_w(LL z&KYialxxf-WttRVQsFDR+xYTS;50lsKqt}&ry+1xQPWN&r!4}TM@pTYg9Bjcr4 z4NY}2BdcQ)Wo>m+J0CyI#xxFEd_w5u+Lz$i4puiuqVnw~w&M61#FoWWqJ3YE^n9oL zDHxe1QTqD9cFa<Ma8JbiZW)!>tv1K7GuQt zp?hv%mG1%frkbGB=hSP2z0{?sM23`2kKVGYHlwD+W)1iA{tUzmP&|+44QY&CLbcQdbHV(+ zw5UgEwX;D_@~&~l$$EM2(2P%Xv{J}tp<1<0zt8yY$&7qyv1!74kkB@Bbt(5s?x+Uswr>Zlz_mA=-lXQ+m!|To03Z=T!+jd1`B4k9aB(6U}CuSID-^J}hS8>%^ z9~$$gH1FFIt0s&2h>MqvML8IjzMBbk{|UeM;#z}MzyE`P>w6s)ZC$rp?E*No zQ;1~8_C9vFQ{Q=q$3x4yw=&}1mnuW3@kj8(@5rQ#jBhqtv7WM{g#kHZ*=tW+kJ|0# zTW+4XTKFd~C|z@!!1E#e^R&1BSqVdrp*voX*%f6{9ZV=0MY(0KR=Tu@R?Wam zQNk_C6QbR;@B24cD5$1(i+)&st$UTReM5S))hGNxC<2WaF(2tD@40~&3&OrmR`kW81=g0kx)taQXuzEl0o5Au3GGu_nz*%bYD{l z*N2_kJ2#%S4NVqE;mHK-(eROICo-R}0l48;hl3Z3>|?#nh>`K3PpA`w#R%crLYT{3 z4t{uB0*O&7iSZj2YwO#O76q_tI5v1lrHD7^58~}3`TfO;){CA8pR)vO$oSU!P@$Z7 z>0Kt?n`Fp7<7zl<)trmmxn@>O^3}tjr8q6(F_mD_@j3JD!Ly0}C$6*GZV~d%zs2V@ zy`(lACf=r!^M_iC#NyHJwCR2ty`4Kge{eV`_ucN?kKV+j-4t7v=pYg0lDaqa6pwl!OLcbix8gmC=8CIh zN_pfB_lUplibEhg{QFD5tbhG5{Jy#}8&x9d%$Csnm-m63^P|}FTx?1BqDj9pyx0vQ zjRz1ci10(cx^SZcrd9f)n)tROJ9){kaKDcxu7ZZ7xcjTh6JaIkuW@|*=cY50WmL~% z@Fyzgd%|f6e>RMtOf`70)9EJ>vOXC5x_cZarzdWM^~vB%zU!{AnaS2s;!iUd)(||V zQL&P|Ov2BF-|a%>o^(l8+16Lp-Cy#i(H1njrq+zpCi1C;)z7dFarfoBpCPWTw8u*W z8YA&#X&N~E(?#~2q^?uV!f5KKX2V9ZPDf7b7(^})c8Y0cYmIOR*dew_D?;D$ik{?OO3eQxyRH#)|TrpMSsq=>IEk+7mqGI_h z0YmXy_ar+~cxL-orMr^^TYZg{S=lV`KELr-ThD2}LlvFQZ5t}AdlA{Qj#H0d&yz|cr0bk>^?H2;!+j_Vf2d5TmTWA1z`B(s zsXH-BZPNHa{;N%VDWyrgaZAZWMlgl=ic5tf!HdMC12hqtRsb5)T<~ z@tBA1p$$XPdYmgSllJPu%qj1a;N ze~8#DH)7U4cylui4pq>xQ)&51*51>vO4ZU6zX>cb6Ir82yUQ+wa%Z z$?zz#-KF}<`GUxsSnJ*~ky_8SMVI~cv1GXim-AzoOEXQHW+f4NQj>&iY|4W_} z6gxHsS4@mi^h(?X@|+b(7)lrLDkB+>E2g zisq^D^?-ifmoAL;t*dBf!>GoRQ}>NJ*&ms$6dk7qFT2U_DR&<94y0{P;$|l4i4c%k zS=?=^Oz3@EOwm-k_7UUsBwpUlGHAE|PJbmqBdLN{t;@5I-G_Iztol+F9#+xFTZA<3 z3hjAB;=KKm9d`Xd-u+ipk9v+x-yk+|>%@U(f*7enh*9f{(>MVy@%Q^@O+D0JPw_s! zRsEEzrK2=86R_d)Wx)0P49oFeD<%7p#!cO>v*aofM>twKve|Fh>005Wkr?F*)n&nw zIB_Lo(?)N6+0mWH);ER&{nMTYtj9DTG^#7j%fwu+#iPMx=0CadYnb6V=p&xv-Qaa2V~IizemVbW)9!f%BTpqz>+4C*){Q=* zZqVfmi8k>h*7-O+w3(&TV(Ynkf+2f#yQX$lgMD@;|0;X13|HOCvFD?Aqc?qmdE6O3 zVqI-`bp9}~>yq$k=lh|!frNh72qB4u->$hg|6r$R^B*L3r)Jsx}iK#ZfN zc{pOTN~cy=llo`WagI);*22&-{qFnm#(~rx9e0mEA9uS=rd0$?fa-ORP)^g4H>nVo zp9p;%`@LpaAF{MZ;SOY$N4_QfaBuRL3CQ98S54n}3*NCt^If4)%Gyw}tG`_uId=>S9?p%gKH$U6RJ zu5G4vJok;z+nQsNPKK)b5rYnc=eOq_x{i~QtyJs|Jt%EDlcW9ml@Dr`@d>q$!JLst zYL6XG?oA0gZHnYCubp+cOR0O(IyWo}Xfw-4_o)0vAP1e5&Gbv}-8}`3xW`N1bn^Os zK052eR+nM9BJqK%&y~SG^3?5&I{uRoCnp@J_r$bHUm!sfMf};&$!zBR9l?Q5vs>?z zbs|{K#rru&uC)eMe&^2Bg5EgdNb&2$)9a^x zBn`S!$d+$ZS{~0n-(wAlPhA)q+LXBE@?&e5Wa<>ggT=bXJJZ$sws4?Eg_o}aZS~S< z>zf6~jRUr(^AF9qI%V%In-0!-Q!Sax)-7|e0 z9qP>GLdMpeS~k{Xuxd8(W?b!CYRp%LExH#!pD#IkpA6C0Vu!2IG9^Pu)c9Pmh%H8q$f$(pLxC1iDSTMx*G;2X$(y#WxWbZ zlNlzr>)dp{YiaL0&vD`Ax}iqjco;rjkR$EHTDa7t&3^Z}zF`@ta@@LoH0q(V{os~0P50^Gf9M`r4x?f1Q_Jq2eEUxEkh=n%yiym~*2%7--qqSM*F1ArWuuNq zCYo=x9+gb0FeyQtaol@1%{Xhmy`t@{5r4r^K|BtR?nl=HbKG3g55z^v%r`Na5b$eo zaxWO@m}Vk=@uy|mj|_C0S?TU@o9Zx*s;dUSBV>5D7_Atpltj|N(6=_$Kyry!S}s!k z6GDnZ!uV_l+vWJjqif)#zDG@U=<&DbYz}1Kt)|a$vK2kL)*W&e!{{BDi*NxB}&7 z`YxMCsa4`cxP0WUC0Z})0qU%MNUK}maq6c$Y<0k$OJ`8YdE*^?x5^-u^{%-{@pudQ zm)urvy-Mp5y2ti^Y^i@dIG2!Z{g7@KL+&DPCi_}Mz*{MTg_D-UBrJZ_sN5+KkD-$Y ztJ2x(+}lE;Y)bisV@&ID)Ni&Z%bO!%t?MVzY@CrS_c|Z<@2OsmsW{DNmdN@1Gp1GB zMj-3iN<&o>9+|O+-P8P4h7`xaeH0JvMKgzxX8qTlVduFsjFiP|?!>G|o^xkVkBGF5b5Q*OF5IyQBL822W z5+zZBAQD6fB2lA82~mUS5?~?;H9}sQ|)>;C`xV=0^=51Gny3k7va8|3a>^O=#DCAYQf+7GPE1EUJeB(yJ+ zJ=@aSJn>BxshDs_Jqv@Nm<@4e4StL(WYG)$JhVc!R>X5~qbtOMcDVy*^xv1r971P7r3>cF6gJuw$ zm6Ug-d%Uwg7`yf5al^;vf|go`z8~EWuNd%tFFuoP^HpUiBxL{@xF36o_D6wt)X$1> zi@X;mgU0zYKhzE%PH-1%pG35q3{g;T5l-z=cg3Qvj#5zvI#4E51fx`vkwGih={QFm zOBZ=THUk`aVuo+q4u@w+eeqai8A}WeuylReP2vAqp?N=6t^Wx*bwct|F$>jH;ylCl z5AvJ)s--2v5~8e=evCdJ2Td-Dd~2e6@l}4$eSu*)u@-eeahmb50$*&MS?ikwkpRYqr9Dxx-^Y!U$7D_-1^{P&KmRX@*hV;|a0Zw=eUkRR#|90>g@dn^tm_i@xH_ z`umm-xv=`;=Vmeb?zAn=1P-exoX*(=?X0_b-eD& zSBN^9Fn>aDFwHYcT_PWzp6=e+CU9rkaBBLFH%8UzI_k}xA6I{4mUe#&5_DKB7sj36 zzJ1niBJ#dR=b7~w4=%R4xr;Vo<>D&pS7OmpTVHs5$DRB-OL|&Akme1)49P$hs%^=d z)_CU$P+_>?@8xHNj)8_Jm|X~M-jkY)rQNKZ1IOkm{QDNSK(_?qLY-&AKW|b~#MQU* zHIVjp^Qrf15E~<|f7-DxEXqjW!!v0xaI#Leuncy~4vaR~|8x2V0H}$Z5WoND^`R8; z&h2MA9!@+Fk9@5TpOqu8SI}@e=wvJZS{?}EYj&Mf`Tco{L}LTfmfWu&cITDT6zv_^ z#z$9uV=iB*Dg3|NIXccr%UR7qX1m>51y`YF2d0s*x0qL7qTt!pj%`#*vaX%WH|RLa zM`~I5zes;h_Jp>&}@{fy}l_Qnd+Jx7+pLyx-ex!b;&)@UnQx!Ta-R>KiJFZTAQmc4rf% zF;QQW*%ei5=aOg~8d~4;sb|TCk!{lVEQ?DeM=cZP+ z%TL-0gLmN&s844Q4me5OFHNdUkMZ~#R|-|t;BNk{y=mgnwACT*ldir-2Toow@eJs_ zZ2;|yOB>wLcJ?=$(@VzXg{{pAe~Ms>g4q963n0!J>3dhYNj0+B%Hn%WcA9mdOQY&% zR_TBv?Z>Kw_G|$PxRZB01%VSC))P#gTb8ax-C^G@RCvjp$vZkG^Kp9PeWI`Zb4YM> zE0QmxK(3f0kcV79k)qKsV5K-zRQP(V0^x$ zSBeLI>OUrv8a|28a>hQ!1m*Z$X#{KG^-WOC4XY2Rq=gdSIOJstv`~nN%9>y zcs8o?7k;SVWBEIkV64UUxG;3G{zrl=Ix*I&!JnRrL;igbM{um)WXP0E`uOxzrLW9f z$7|DhW}7DuzB(%=6rKo4jm+}|ISoCvGby)uvt7!B9{)9Jnlsayt%MQ@RlV=J88!aH zEbSS=H{wrY#V+}F74m{Z8pQkLXWwlHxTw}+TvCO8f76Y`I3L~=G%Xv00N#Ev=8s6X zlgC<*&e85RzoLFRBPoq6Wx9fxRjh3fdqJ{2okk>&dW!GbW@ar2dNotu9+}9gp1snO zD+WJqsJSY?0r)>fY$FO1w7t`kvJ6o2d6Xv`@<3@T-fme-D+rLl>VROy3qm?_=>)?PNM; z?P9*7#^Cos_qB8`wQTvY!_3*B;hLZ!w6mwy63@leY8d~v=TTMS=}^^w1@-NzsAM5^g8MZ_I^ z`dNILBW7gQQI)Xx!}LamS}c>q4W#!SmfCk6gDiuyoiEr-ve9d&yX^U^$wW%tA{GU# z!g-CcH`sCt9n0vSe)><wkx2GCq4)@uUO7F}-JeQL4)v-G->*_qtkSn~De4!Y=bTMnDB!>%4BJy#OzcLlt04lwVif_GQ#H<7hi6i?3HbywxoBg}y-Dc|WLCB5i=i_`2%K zc8!*Bki(CmL?@$(OL|g*n25$E#{{tVA0?0(kak^;G9fFtGs|rH(lhlVP4(=Dv+qN; zgceKT@tK0jzovwINsngk9zr?uv1^)`wu{Tx3e6NcpLc|F@f6;=GfZ;@lpktk;FG@A zb_{2RW{GTZ$Oq|VOo@*5@E42K*KxVD_vVMS12EZ!5?;DpE*93aoRT9xE6hb2_ZGwx zew}w;=%D@G)cdX8H?Zu|)OJLy#Cu?0A7%@|caqrZ9r^DnboJh(0vl@fWpxpq)}+%7 z^{tL-*qmUnN!I@Wutzh%dGCB)B zC-rqT#4#3lH;dPYt%o8pP|Pms{1rR+YJEddJ5@GH)`=nQHpR=#UhiiPs-}l#9hoxf zPUxcW>6YQd>$q8|B)guJs+0th>xCKFC=cbd9D^#CZeecMemyJy%|P*9h2Za^>E-Cs zE}M~BgKwXkYSYPxd3v&OefOk4n!eGGMUT>A??sHHc&HgY@^rp0RvM|AcX9J+jpz-t zjc8P%-{GX`=H1z#Ek}z?jF0&+o|HE^S9S1uk|2W}>DBQWzS0YxWgZT^9}WZ9^{u7_ zY$uvrC9s!cN9H(qF!r?xAJRI0jlMMg6u+pXE1uq99@D{A={g@+^XNhHQDnVahodH; zA3Yc14k#3Z$@~7MhwDguQKsdZ+Z`%kJuNA2(^0-s=jmnUI+IAyFYxeq^dVG5nLs#4 z75e%~pn3O;O->55Oe&7zv_4kjWg2Sc*Q^$w8Jg>b^}fbLfBw!p0W4Lzu_J}W+QVH@x9X7h$nj)9vWAB8r%uhBhCKk6|fe(P^~ zuvHlH*!-u&pm5NO;H;OoUmmD>+4lKnDcmU!Y_jiro5H-AH}>JJf=NSARlCbB+Md4i z&MmR7OyJC1=PjNky%`Mac(8(RezV^1@w2u*OLXmXA?eQAc+rVaOqb{5l_yM8c6Te1 z9bzBb)YQ|Ye6ZWxd3XQUg2gZE08_zH+BuZD{*bw@;bJy=4)(@3s;zg0mZbM2&i@x{ zU0D-G9hKLvba*9^w$?QJxi_1zX@L=2oC><^Z;LNoQ5Iko%mbRA_c4h>-#ohF`8nfs=Q8;f+L=GU<{ z#j3|CT}{^OLUl!>oA1RTZ+!CYVkgmFND_~FDf`;s&x{s#KXsZuuZ_v!zW9GL}M%0OY2`6-%St{9HDY!6b3pv-I6nezyX@5MH;Ud7&RswV~pkUT( zu;tIV+Eh}=SNOA$2%oJ%gD^2SaN5%ZFyC&e$??{0nZ>gzs@sd7(VS63zrRiQH>>@W zvDb_&j!7!I!QVwk3aVTwB<_DQYB_9K+NW61fAEZ*YPBQC-}g`NC%FuF}LLqDQQKiNS(^K===7sFOMx>#ox*oR+CM5pm23!gCT9lJ|@ZM;A(}lg7=Vk zc~mFwo9CBeQNbZtwfDm&ECTR2)}%k)d%R8iS1kvBJE}&4V!mR}&mRTeMQ%T6c48KH z=qfnSd4yABJCGiB?!d37Y)QmS53#65>M@8EZ{MVxxf{z9ZOFK=ZIizEim(OQk zRyYKw^iW;ziJktnSwEPnN?0dab!m>8kHEI{La@xczEmkLEl@NJqHNOGBID*1Ier@h zM7@U5>46XizK^h`47VmhhwaHyjyPd$?N6)wf30BEQQnPpe@ZBy`*c4PVz}NOs>XKv zCrS0EZ@-nl2`914T?eokxp3E-QDW3uUf)qhY?#)uap_Kbq zmObki<8*HAecxB~@%)fmfdXY93N=R&y}$1>ektvXU>SZP9g-%N7!=%>^y%@n zc@5@U_OIz@kL*aP6CSF6C~WEcAieCH>-|E@-#+f?4yZ>z8_=tVXa<+qBKwE^`*sw5 zZkW{RkBUEe2P*W?Qd8|YwJs4?t*k98H)~NUA31`v%SyZyWp1BH?{^BAw0s5!zv-BB zHjg7{jAEZp){LFX46mZ1OA{+}niebFak|_6HaNH12VF!W#>9K);8lv}SrT6k_atZi zH&bQyHiCIuoV4jk`N`O==$FBf-k`Wt?}AWLzg^&|;EG6ICsuHjblbn}Ml=DBwo0|8 z5F0qZEN4%vqHz@Y%YD0F?8ia+Q&X~cJksL$I4C1IyU5KlcXO10GdN&-BQ>o>UeV`- zZcbxy|2z{0;%Ij)K zGN^7X9S8Tyz6^R)aDaTK$Mt90TDNQX* zrgEqqCZ6|x_>q^5eCRf+x)UpLH@#*fy>9zUcTn(PiBQAupi4iE7gh%v;MjM{?&r*E z<3FYskYncpx<=6R*llBMAV)~9(qR)aXSiaHj#2QNzw&B`7`3N%9I<`*rhhWL7;SdR zVO^V;3SbRSTSV2nLpePk#*lOx;ZNUuuYG5k7@q*vRRbKc}j;wa& zb*>89uBW0Q-?dl9YDp&!C_Aqie0x&%o90}b@<$`V0~HaH#YG`jlEj>NZzl1W;bT6x zix|~(5jY#TR|&2Hs*dQLb6%H|T2AO@5NVXe6O0jmVkvMqL>P4zNdHw>uO?h@JPD$BPW z-}u#=fb3vd1;;ij#X};IbOSg4yg1%_isuSSV=WOGtM8u;5eogiKF(qM6J1Z5?@G{E z=n$Y4Cn&~m11dDm1j{Xk7517n+6lc{{Mr^^yZXc)R6)6Ej{@_X^$m8@{;^k%z ztX+;Xa0&Gcuzb&Ib>OG@OZ-KcTr5&$r!2h!>B{|){#v%Yx_sfciI#QFb>1X>|0$uo z7rJJQ28z68fBG@<_dT(ABK6z5LlaI{-c%i!^?jJmz=hmKkx>WVR#`o37zE?ea3fT< zp2Ig}ogRzcp!Fxgjlb97wYiUQva2O{m<%c#%8(yaGQJWkzQ}Oc>n_=bT6~(fXxyxtOwuh~^ zytV5NyT*U?J-kx4?sz$KD}s3`6rQ5i`Z>{V7ueD zjY)afNhodCcw+Qk*wnt>@3bBmDNXa;i;ABZuN&4eTtHBOgnVgmeZcXx_aPD>Q>61K z5^lHZ5ujk%?9D!0VSOoE)wo3e?(p^Z$U#tqhA!n$FGn+7Q$NCHpDVeN*hCel{5A<3 zyRltMR7}GufDGTprVpNA8$>@d<4aq6>q}vK)5W`~GFms9P8u9;NOpexk>DcbzB6a` zK9uW%;-~E=vX%Mgo#~^0v#lBDyk$+c?(v*iA@UIo+d41+`%VL_?k4rX4ee(~XFkmh z9hZ{dcmKUTaXs#!1&hD&@G3s1*VASlt^HIdVm+k@NQ!rYBp#mq{mfZ^hxUTVtz-l{}gH2)w6K zQ|9^3n{eQH6JCGPSs(M`&!W}5QHg|rl;RDwVg6r{Cg|6oX7})E7|rc`Cgie zOGZJrqsVnI5xWdnzyJ}?Qs1FKzv0He=bRg1*-LAC1BR}3VR?hUH5=uQx!YUD-FI(- zW22?r3A;M)W5xT%%xl}J7w9FK@yRM+^z{mq!N$La{x-odQEUtKk zjrt8QIBC;G7(a7+d<8wMD}F?q(d+I?vvC2$9Zd$35wl>+p@J*)P6mZXhwK{1vGR;k zeiR?RSx(;Q1;%w+>6?P9fY{S~j_DF6B7}?T+tzd`Q?Qt3hdROz+++z4;vz#H5f+Hz zWMDtl-yS{0Lp$}zRus$%_HdAr!i|O;${zb`qhok(2NEbV!Ja9w#!P6pR-dMi2SDY} zF_E)*o%{ti)5Q3}pLQP>mnQRDqqz*(z}FCycyE0hICDdUDd2&@c+-CKi%T?ewYQB* z(Fc@qj6|bS#aO;#-z0<4%g(`HLE^-Y@^WpNLV)u&-d3I?yQb3p}T)gb` zKKb9;83)5cTJ6FKdVNFz74Ae`?ex~3GCF6+I-R5S3M{d>mBbJ-Q8_|c1S6$wOQzL) z_eS)CVe#-R$^tvO{Z?lnrQo~Dz-@zu6-JD$v{T4?mr|4$ zPwDCXpVLzvb`kRglvjVCuqDE6o+RG=3~Wjr6W$ApM(k;v>vBI&7Bce{@OpHg_ePPxc`(@NsBR?ERq1ri4jWVnPMPUJMKU@u>7nBAKQ z+l6!S!q1`lsVHtD_CAtPekK9B@QtSb<+IrOCny~Ho2F`~4w@CP4oM;?kpp@X5Igps z4u;(j04mTFkIDz2vXKrnFAs081Z{FE>4D~z*RU(j8ORD5{E7j0Zwqp%+B=MZ@~vpt zqHt?4VW(^};cwj`fq)`g48r*OEhbzTEFD(asoMPcEKTecFtobBL4aDgfZu2%+gVD8R#e9kS-4@n>Nl5hJ9aZA&NpcB(KqfZbDscXbrq zw@cYAF=^%y=fVQ~!y=sFwoIJL%5=wh#um4CFs zzqs}=0s=C9G{6edlw-Ge>moD44Wq`8rtUu|(d%F#LCmP36Zpn4}gNzL$0gtQWXZG z*)#_{o@vc4C(G*`KVMs5Y5^-=Wh4v%8J0?g@VZyKMNQ_Kk%6_zv2e%?xWlVp=C{5- z5-A>_?lnMM(g54_$DAA;P5-WS0ho#w?L`K$QS>i(?Bw%L!YiqjZ@{Jq9xp2uN*IHG z-lI-QU!s7qe*Ibh$5HGL)B^#uX#fpJh+8Yh4!fKW>kTjJwEoi>q9Q>Z6LC${6Wof1 z@<2!^1s4Gu%>}@t56E=)I+djxq>EBRpE)n-58mSGX4uw%jaSCnc;5xGZ9#e?M9;Jm ztoFOdghll^0ef!JE|A8ZhZVsepRtW&=v*+u+MWZ0ncjZ=zc4Y5%aU+W9r7AhT*e9Y zg<_x#m5kxy(YyJNU_2H*N|5AjIS^g1Xhia1ixjrF5X-VM>~sj2$3NdWQnLgFlyM{* zQUmSQ(-s6Y+LYiOR0I&x_$lZD>ja$%5`7ZB^Hqbst?2L_?l-|^7eOH`8vc^BWPp-_ zphoiNogQD@7NnhY;gH5=Q58gA)abC)ZY6-Gf(eN?f&=PoZ<#;hEyMq+1-S6n1RBBk z?)T}wfwv>?gK1~DJ9bXd-T>l;m(}fs!9ZC1tFP^)w-@9lpc|)bh-dH*K`F>$U6}uZ zT%i~ZSV|hHb^rtamZxQqlHF_mGzQ!kGpM|Y-h%eDql5Qp&ix3o>ClFaGY|fy^?#d5 zWGSC&&EXE+U6?n?1__RHjA5slO{^GDS;-kr4U>hXyi=vde3-%f?R9|ZN3~ut=)tb@ za=I3!;-*dk%K{AY-}b)wZvd-VPJVq{X zSqDcO0$9Ba3GT_fq(oL9TzhrSAI1Z7vRjyYcn=b{B%(v@-3ja<fIgUA#j z&~V7$Z|uT*>cbpM|6N;-|KcjE*A0F>;Za2rR>DC9bTL>&Ula6GvGqW#--?MIeYGNJ zBDf|p0WtJK_&)3@e55eLKfv~m-x3>X7PPJO3TW}XKt^pH7fS(Ya4U@D*INL#pjM zWbGHYQmo}9_0QM};5sgB<*z|=#k5+C9qL%a@V}lU_*fS6Vr>;5!@Ou;AU`8CRA^?B)`F3L|Q*cqI z3~c|m&0+ql7QQKa?7;VSLjl*YLm4v(?HICJO?A}IOegs|)lYR)_4@!NdCYA~XO>A) z$$4vHW(8{t!J+Bx#Qz07Q~ZiV?y6pYcwwI8*{2DVODYBAILz348+aDeSwnY2pdtBa zFpIC0L&#>_G$SFYIut{2-YbupHIALeNc)~5D9wfd__QfL?Jm$&MgSD8tH60GBbuRL zV9K;pJC;)xrv$?ix-%;`XlLD30}@HW$q_OWWR;(1MHbwb%N;pwm86Q0QAu5Txe(=)oyJD zLsX88fiD&R?>b)nD{?=zlfQxXXz1k?M9MWUIR#wW4tY2`kkuY6ya;x|$wQl;n7>Q; zltCAfO%-t{PdJEy5IBpxGw&6#(-o&veXL`x`OSo)dv20=gq;3qdfVrpMn)vz1!&uq zCN>x%FWJS$$oSd7>8vi`tqo>f%F|9+mq0KS$=B}I-gOi9xI*SCO{@7iS)F|FU~rzY zFU$h|9$u;6_66{pJ+X$(ucwa(k3{!yPboiI2HmfRpzTQ8*PdLoBL zqpj0E2B5Zn4+}1^{q287vq6GL&OeIjy0?qw<6B(Yql}t8RO?BAg1vv`HXw*nc7Os3 z@ue}>70?CN$y$m)w9A*e-wx=I4I1tIr4FWznC4ww*lcM=G1^0jPlIUquqg%OmK0M> zmu}P1Y3)5_$LN}Oq9o}mjlDHcZOIXRI%mXWf(Gj>$xE0+j z07^Q^1A4D%-RVCvHzRiId;3efRRnT%RgSLbZ*>=;3p}gjoi}tgcmESZZVDfVQv~&L z!c66BLka~>P!jeaY7vE;^+MHxKIHjP*?^i@{f1hmZ>BNFB(R#>vodUuYBT$lpAs?xXq;Cq` z3(z|48Hgi13U6IF{^9Yx%?+un{G)1>JRb2k3;N%)L*M%9vj>9B+5+2cSKZ?kfxFz} z4sVAEIH&#~uIv-x7j*)HazpJgw@ThA0Pfr0-Y7{0aRRyGd(T6csUN&lNTzoeU1JgZ zTo7#dpO)d*y*oR3w=;G}p6@U#DT-?U|Dzq|p)0pw3oh0#U`CUh>|azM?cf$AQsxq> z{|>vbcdH-)ICk^^DPy<6d1BG?12^XgvC|}wUQBZWQ-SnCk3l z7Vs}cwE_uTp}8bp_gF4JE+HJulVyOWz)6giguu5YzQKF75Y3Pb$X~d2&S=fbC+rOW z#^G@OJ@K}0iHh`2M=vJazKsM8TKw#l!sI^38fLE@Iz2uE@sSL89#qtZ%xPhue=5*U z>qzJ1!Z9XSl8-Ys*MjN|M4&x+3a%=;(LtSY;jWlCRQ2rz3}xdSP8#-zN%u{bgxpOG zTnXA=X)u`e#ixi)vOG2sXF=2((_8U305(HFkLE*WH7=lBr$Sn9Bb%#A5+Vo%p@67c zn1T(0z;JgE@n9_LS0?M_a4O~~1q!8a3c?4V>7A_-;w(&XaGag0p!8U^pc4#7&*=_V zM&o>7&972yyrT$%qhZN)LQUyHEfWnaDk=d3Wk;|muSeCDW!`_Wp>>V$!n#s2Ekh^) z03oYU%BA;O7 zDN!YjvWy&w+7#eQ(Zg0@*e2Ve2aKtBoJAJM3fg*-i`@Rl7Vapa=!7nIB&TwP7?TI- z4MXq@_R+f2t#LM0KSs{N;C;mkVR>Hqooz8I%;d+=)=&{RuJRVm?G)(IaST|RT>9;) zg)o?UFnS2lO88CdJ?wN4Gy%>&XG;s_P~(C=?JP<|m|jIMRDLw6Cq$LZZCQU zsm%rrP|CfdofO$F;~1-QCY+8UhZq60_Yw$KhrMv_oj`27FkvgT0T0)r(T-L*E_ojh z8SWx-(xo*I7ip=cIa16?jCubsY)P@D`L#^Ng=V z7&>c2-it6CyYhJ2?6HlIxFu#gL|824m>3BTf`c0?=+nof}S#W>pHK zz2>UBFYy^oxDvOs67WonjhrK1w?$eC$SVxmCA17Z_zc3EJz;27u6NSFsv^uC0EBJ6 z=3_#Er>u}z0{5=z+lDZbmGAP$(4ttwJwcC6>kN|0Hc$2e49&(RAnOr#JB!4zv890i zEp|8@{_Fj5z1w^B>SUVfV!zp$)jf}ms}HmM6*^o5r)=NVly*bHnWg|uFlV~H{MqO` z?%Z2mKdk@t!^*PRgJhRo{^~mtBBF^zdeoi%ksYF#AbUmNFu+SQK(AH(ovMUR6D81Q zc*u%{N?z|-9IDzSWEqw<`&N48Gz~e4Mi<=;i4ON9Ijoafm}UeopQu62ea-8I;0e3= z{Y?C%>I#7FqEHI?&4SsDVh?t$mM`*1mEW-7pP$hyr%@1g+KteV7;SsC=O!sn!eaVT za|=31%617r!>_=b0$^oK{1eKBoryk%(f4Kfw8tG zKsu&Aj&1YTTwujD?3`CzVYYVlnzr`PXdCR|6|mqkid|M^W7zN;DS0gr1#d_o1}VOO zOoC}1m>r$DPbdhHswYU0TS*gF-f#Rk5h-a?pbUk+&}e}fG?0k zIafg$avGbSChru_k=Lf*JIASU2V=009FPa7)uM_j*$x=Yt0?)>;io<{%5}>EG(Z*kI$echtQ>At69@O{aA=n$EMZ zt`&CFF?2{`jNp95`YZr!u*&&qBz zs|2O)w?hZeuNI+p!TO~UC&{h z$=&dAbfG1MPRx{LVCM`T-oFevz0;qj(^MEIw2ISISmh_#j(%M+$uIKOkK$0m-*^zR zP>bYygmZ5mH_>MWCtn+2xa$DLv5%-3i3`#$u!!*HMCa0`{yz-VL~tPL3*)F2(aha6 zAXmnW8-p#F59!IC2Ib(@P;H=RN|xTy`e5iZ2o~F`AAJF7S4lS##G)g4-x3xWnh32a zRY4a0Gy*mlB4gSCUPdhub`Utd_=IO^ZPw0Dr3EDX-{dPFLI-w4HZ6;L5BfhKF2f`d zXemV=v;Xh3N7cH~t)F3v#O}1pB(N>P_48~%Wj7oV74VTW=ob2t;IQHM_X%#?28&KL zv?azJj0rNqS(CS!WZ=NR)I(8&u;p!igf`_g1MC_M%SF{q!&n&7HNe#B{{L0%OzWYf z=$5Fk0bMcHm#;d#lYmwalWJ#Ni1cRk(=+gU^OJiQe* zyBEYyuA(d|%BP_*FL0fghw<79y_NO)`!rzs-?H8nXAfuNB{1Y+RK zN&Zw`O^iYe)53}buW#`1*29~1aH`zQER{Z`9Vb=Oru_4#8zPMi@L_JP~sYH$~ zR=F_~t%NyE)&F@i>SZoTB%J2`q~sj870oB%fpdf?Wf+`rfL|Gf$=&LGOs$L#KoC-M z%9M3C3LtqKCMVWZ5AJb}TF6Gh5YE(#d`qM{wU!Cz56gQMUlwCh=9aOY@8p|m4ZVp(rx?N`kV_B! zG!tQnE?RUf>b18J2toH*qZ z!t$}p41V<{*iRp(3T0>z$T10*c7zeXoGcEY;}@PQrv ztln&Jh~V(|q$qXm@pN%CAxirRSgp3f5qzy}(j2eZSjsqg?+Vq@88GoBTPX%r(@zx5 zF<6Vf+-R`kIRR1Qfg#r)#CZCJK=h6pEc`nXrOuZmaZ@J=mqBx=j(`5#sn7W{o%DD{ zaFFIe;pY|3H>!ZaC}ngm;Ol_QdIB??5YG$PP2+W|H1<$icJTLt-8E9XKy08EFL>+| z;;;dw%B2&gBWpO&DtTAb7YMSZwG4kDLsDx%w?>hf1Q1XJO-m-htCOP4tHYslV2W2L z)41GvG7@lU;2FL_wxvYlJ$ExHS5*i9wav6w1_DAj|E}ZfvA(Auww&9&@ZF00Jx3bH{ zdQ{sr?qzCTO~95+q@~~z+3u=8gKA~pC1>!q@(TpLleyYJGJfdOZ}dtG3{gsrjx}QT z>t(hHzr+1Tq-%(?`o}9Lvhts6$n0~@?f+hzjPNqY$>bQxG|5W?8Ti{W#LPhDjBdft zrG=JRM7%xN%qjW89dh|+l1C^ZDz>IIAJ42;ZTlsiaU^cp%1e6>#o)S!VcI zZCfdwcie9)iCzc5mB#uuzltqxk4&D94uiDdFPfoa{TPc!7>ZX!wtAYep-iI9H2DC;FN=jkeu)R z))oVt33oEKi%Q<&cSH;rq+K!*b&cCK&kg)cG7l@60QVr_(wTxKhKI|@L)^py115@@ z?2JOydfse!dwfUk<@C)3LP(Ilh+4~@Pp+3O-c4j)_g8GR8ih3SvGFvYPIAMwyi{*K zLnIpI1_t<*rxa)t9DbL3^bd?9&D;Ce#XcrAd@D*k`XZ3O0qd&o#a zbE`vhzpLdYKCCAX^u_yG!rnfW7=SeQF>)NUBst8~Xg{ioyjm0W*L3@@8N1bOonPB6 zv`lUf1#?q_0jRT)4b&{*_E-hZC3c;%_m75`Ru+BnD^!IZ_zQ--@M1_!fnWoHV2h-9 zw~^ql4;o+=;;JDPLz)goCr=MvShf`#7J`l4?} z-ajbG4%WGp?V=eEP?8=Un=-sV(Mp6WXRWHi2uF$vRNS1gg&4j{=<9_cJe2AlbHoc* zOkhS?z*_sS;g7Z!U9gCNFG}yCQ0Ce>KM}~fAFGS3r`$-!O+0kWKrw4q z|0JmuyxWo#6x!Q*3GnO=f<@<`qND@=NN-GiApI3a)S3`44Cr2x=-Jy8a?dMS#^wqw zEd+BsoSnC_eO#iz&4yX(YU0Qx;Dj^{(BuSWE$kYB14?2`fR>wI)Vrv|t25TdwnPdr zqY-h68R>*8FO3fMzmD+=M{b0s%WB(z*?kc)M2Z!E^o=`55^R$sr)MJPa6 zK9^;pR!K#^_KtfKA*Ca$H&BBYP6&2eLjNV59|hPPp>rb^_$V^41euFlzt$w8o{v*^ z=vI>I9WJjGv(mzXDo4PcOXF>M_+mgG8R2c6`^DqZ>-QbHoBII= z-U5PYXi=^fkaZ=0nbSDZKLn|g$G&bR@^J8+YI%l%yJ9Q!0vOblnXeUH(zZ1bsBnrbv^`w zb+^6^f&iqt30Oy>gjYQ|dS$Nm=sZe!?@h;>QOE}Q5?kBxpf~UdAl&+#!8Pe~fJy2F zZ?G0)j$AtA+-5aPKWM}+L<8oXhqx#dp)4H^$0uceH5<^H@pW9LA#LJvCegV{{6$yk z^FMPJL;@b^#`Zm-2CyLy#Ppbc3&o&Z$--EEytIa`bz}_fouzZs4lATJ^M@M3?!&^4k$AIN^85#fN%nChMKsNWdQE|A@xl zR?srvCOY9-x}kGXr&`X~=`ZWiMyh~Y-0y#Ug%qA0FkdjrnWH>k&{=BtW0Nv$=9~P0 z*XaVL5NzBN!p>TL(~VZhx0#Lwf8(zan&H~7H8pPN5CZO^#CkAi);{c{E)t@bQS|#%H+7 zo3;WgoOue0BA>V2ftOK6gYOM8vf_S{{cXllN9snS#5ZXEL8RrmrQqGMG*=noO7Kt= zV5qV5(wWy+5$`4@D4uEOTzbY;I8j-qmpy#o^S6lQdRT;zipwgw4uf7G=B@n=z&P;T=UU_9UA`B{FPOE) z0et5_3oLdZ?&gRL`A7_@h4xe4xJ?Hc3Hp#u#kJ&T%UCO;?U!DvU?Pa-$H2T>A3&g@ z9j&Z;7rA5BYPV)LA9Z}g@4$?R@SL1J#EK$e_s541%X61cZYm7?s}|t#^C?$MF_6SWi4=jy>;MB>Y8*e__PhUX1<3)T858WFTQ9Bn{}WRsVe>{xTJQC z-s{WzBz*W^Tt^WAxr=3C8)AXtJ77|((W-~aFuS|3J~Yt4pK+29TYe%atY9w}3*w6$UK79Dy0u$gjurfX>x-fIm?+JhgMoC z-?9WivWVjyIYOVLe1QF`s?CQ!x;A-%K?oo$Q?H$tU1N`_xow^-eQfBGgv7I2##P$5#Qjm^|wz<|oy2F`! zmrkQ6G)e@p|5uJk;5{JqE4(Qw=!g(u->gHHu&GwlurrZV z%5X0lQig)sQ}ET(*^x^lLapSGLeH7%0uyp0BdlhSRvnndui?GTcN(i4BMI44zN!sW z;fd8B+jtZGW&Kyk{eQY5VFVPvUb*KkauqR1Ren%7wX$%xY{LFVTUp8sP%m+!F%Ut{ zxca*_qS}JGScQuLnVe)JiJ|mPvfpyOD#3|oA9jy^symk6^g(X!Rh_+50dP*bE(ec$s2DvY!Y}g7fb_N&P@cwyJU0pp+ z)scrRu`F<%XfS^>f9vOnv%Sw!Lt4&xw;a#Zjd#h;sJ9=}EQkn$1lKi>wa~$~3@Z?R zDq9k*2+u#m)_4-skA+0PmVQA{GzFBD32yPT9mIa;$Dm*+S$};I37pGSuvmw(=p+rJ z))TF776UEZ6ZCISK9}YY{RCU60UrELie%G*)d=F0U2wupC`gm~ME!1RxOi7lXAh(y zc2sS%5%jv!2PN-nzN~%2oK;0@szg$ZPlb`_jA2CIQxU);yD-tckD$eUCA{hZkF>ZJ%W7Fj5UanKoqa=Xd&+znQ+Dose>ow^j zNp=Mc6LM6Pl=T%opyG5UGp zNdq~0BscgJ8c7kg!7CcBXq!e@1kv+TCf*_P^~OX##~!aDkWycF3RVD&EdhW?X_mS9 z{xx|p=ab@+JJ^tvL%R_P1fn)HBja(9(ZC;#0wr$<6oA=7CW4e2?4x8XxkgN}C4kwM zZoFKdI|_E~)%5-wkHoAUBV8wU``>nY_XV~{x-o_W`+YP|LbbBo5x$=_8$m*^=&v?# z-#PDGM-mJXM>{fRqB1g=xMByq*MW=&m!mQOH!`?I&RFIX*Xc|d#lB-}llW$LLtyf+ zfpuhEL{Er3Z56>^z5lAY#4SGXhGC{#_|^YI*IP$L`GswxGSuK845%O=ozhAQ(m9}l zN;je)9nw9tN;ybKBOnb!ONSzglq21!NQZQD_Ji;5UEli7Ie)m8x@Ml)d*6NC*S()o z?w}Ier|gm?^5ByA#DFU9R ze8c*WbSEu`U7m@Myy1%I?++z+_b?qNIJABm#OG=@i|=!+^W&N)gp5RFwFUs;E1`Dv zb0?qukO}*h`3)G86kfxwIUa#w`q&tNQPh(eno4o3;M1?bazAumIf)DkjAyE-Da^5c zuz>W-k8kC}@J@Q=<@~U9V{`xKdoQ4ip{(-Woc+jF=8wjwG;7yB#4HV#U3Rdl0Yz{z zAr+rd@elIrKl%Fl&u$-GM(f1MQUt^KZgDdfQ!R|=(aZ*iitMBeL!LS5;>sU>xB>Fp zn5-`0Y*A8lxC7X56+u1FO%9ADKycPx>QFCU0^aEXF!<|MQPPR>nbkQj);=A3JbFp< z(Fk_q)#%goBQdKdZM~n-ph(b$8EM`dbQKygfR7~G>WuxhMXlO8Uw4m6Xbr|h zdjzzO8t6i54&^IcSa8x#tppV9UgaXNY5<5J?+f(F5g^VyT$r7YYyyIZ-J}kMvX|Ow zXT7_IAlG9@c;AhhVAzcn+YrYWvSDGTqSy(3RSj12DW0#xDRy~awUn~Gk;@I!3_jfm z0RTRy3x0nIyw&PWaY)n=)p}KGK`TsB)msI(zJ@QL9{=OtVr0R)p}|UKDHY-K9YgQ| zWpKvgO$G!Mc4=?z+1$to$VI!v8?@P~UJ|=21WM_ph)--|!7+X^%zOp1`o%EwGX{eT z=6a3cO6y=n@-SjhP#B`lp5ujtvku7AgS3PM>@YU=#gl7<)L5LN&btLmloj={U20|$ ztPfRT+Q^^is((|XyYAnDRI{PesZ38`YWMlrg8M|$;w5gl_90z#r0S`7euU%6;==mQ zV*dKzlfkgDJuDx~)Od13V%6V$R*neGtqI~e?OrA4`Xl%JTMF)d$P5_Q^!(Wz3 z63xj*&%gKtwP#t2Fziq?V?+tmHLtOM^*?y`FZTFM~FYyP-0&4Mq$$HAYB2^i6#8U1Wb77mBwQ8Jz(( z1yJ7Ix7jBcGxb5ehVf8jv&NJr4FwGTM%BzhmoO_u^2`mx&-K7moDZORia^3@9LeTG z1~d35LsSLsnd9K$v+GPYuwgX`Q{1&$s>R5Bx7gGFiSbv^RRk0qXTth^y$jhU^Sck{ z?Mn_l9r!NEg`73tmdj%tYoY94{w44z&XQ*?ETMdOhEsX@rBKFSe+kQ_OJ27vM}pZ_ z+x@^!`Z$4L*X4;fCLc~IZMZlL3j^_YLD{(!}kDXRf|;q7ei=X0HZOz7^8 z7gFjFvRjwU_fb%|oYfKlBf#0Jdz-RaUqVqiNoYJgU__ngQjii~jQEyyncd9ov7u@O z2BSk2mB5R4RNFs9U!D2Y7;|5JXyY32BG9(JQ>dyV+dc{0$&MkC5gI|TY)i}!q{2w7 z;Ved;KYw}K_jATiorGNR(Hyi$vORF6x#L7Te{AQrI=e zBy8)I#C*Z~AKNv%ffgpDDCTZwNZTtG{K6g^yUofF^cY z3A>saB2r1#qO+jWRB6NKRsx!pgE~6FL^}y%dHr%Bw*z{HQ%AQ3bMzGkOLX(Pr00LN zpZu)oINXZ4ZBuXK9)LMM*_%#2T)sP4q^oi-#j53QMATf2b>-mWozI2SnGS8A;|70I zd9iY{sJ3XF7{@ig{Y`j@!(Z~dR7Soj$M{aU&G>@sLTeDQ47JyiI6V*pQVe|=)V=&j zgUY97B^QGT>&|0-Gl4g7{9~B+YV{9s^am2+opQiuRzNN!igG;1xOM3Mp%G3Gm6Au# zI{T|uqrow5J2+=jFHd;3vS;J`T+~=%-!lc_9cfbpjhMqoP~O49U((C|mQS>qe9Rs7n!ohvMyoH7T@t)va6wn=2%DrW6$pn!Tse@>dp#dBrN zW4qmgaVdO#MW}1j5*_=4f2okq!>A1B)gGP4b~f_c2?sR9KWRUh!_O8S+@(!s*_-d* z5h7wxnkBki{a|^=py*zM=wZQ%MM>rCQhlROfV5%P?8#?-fcvPS>5VU#JBl4r@CRBU5r3n%ap)7)%)FqmIVzD6fN)9({Mk%o|HEGln-pM+qMK5;8J~=3u#|X?o?j)1%?hWzLZ$d+plEv%v?$J2Hho zJFwJi8dCfIQQk+k)kd{%InD%2y5dG`$w_Z(_vkum{XHBn^UbP^@yi+W*JqkVC;C{Y zi5BVDS(t}TNiht5spZgWn0B^WdG|wLEn&;^-9g!RT$7psGOKXz$}u&441x1baW84Q(7wr|Mb=Du-LJtTQhm26rZ3^=CP!WdaBkdqhHKTeIhDr5 zD~a%;K&R&g5}&+)e=mn|g5r&~R4pEK(ll0{(Od)G!X!|Jid9C*RS*WVQmfb*U3qlZ zP~$~{9olV@^3p2XsDxi}PvXjfrR%xRQ=(Ee1lQaZ6~071G1XyW{wf!;-h8MXzTz3D z;{4S~` z!%e2Q{QL)cY`L{ZEAKx2z5T}(#f}zJXT3hFxiJC*$+lvD)8}awH>NqMjRFE|7Au-9 zdy?`J)32B09k#4DRh_*yT>5E9zMF1H7~v;rZ1_MVYdu)S=jtWP_xQB>zQ%eF#&HHrhHBWCp|totsj);%e85 zP^8`_dq&rF1RjArD7fJFK9k+JvcS1`1*M|y%^Uhl<5%qJj|nTbp4U}HIpnyA)+CU! zKsjTGENw9S8xWpwDR*q_Z>AtVnT?HAM=t_LzvrQ}B^5z?$I(S5-#^yIpG_39=~WZ7 zjNT%hlrWj|YxtE;xA-2fQ~M(KcFTd=q~EIT)bhvA#_ufobcX%~>s8%`uJ}tXB@t%k z;aT{dHb;|U1NIu*`P&&^=NV5jf7MSdo~-QVoobA))}!|tMdd=Cmra_6?@W~byc2MC zQrj!#A2{WGfLON@@byfmF8(lwWA-l2(DqCqijg+8t+vrAaO}IYVY(yLawpR4q~=uP z#LZG^Fk(%`G53{&bUsd6=q$A;+r6Qus0-Wno^NNL?yT4PPUwDZ4ziP18)?3lS{ENo zh|WY)^TG+pPLs8pyH#uWp5G4L+>o;^8S_PHN7}{2*U`&`7|lU=?L!tU@KT_NJ1Rvm zk)pot({KRSr76aJV-b3hOy!dx8 zPsRri&z9=*NS!UQK&H9erJ{bP9jQ|ntooGxkGx*o8r`TB26uycr6ApE^?m02^Yx5n zuctM1{MU=~)@o&5=`FSCJ+=7orW{nG>V>h`J2qpgbKC zO*Qh16|#+=v^PUcL#NPps_{TyrWO(OiYmUq?mG(~_j6#ztG1)>*voY5jNgK{KC&5F zOdi#9ZB+CA@~NF{4@ESswhJTJx|463M$w84O$w}0TMmD5q-!(_=>I#erA3B~9;ZLE zqCdfk5j7ysKJR}UFuQ%EZFEuR&b7mX4x480Y}(G`^_TEd604r1A~Dlb`OBQgf3!*W zy>P<%s%?%3;GISy@S=;aNB?xEm(d#-|329-=GohU(T!I%wjQ10#>v0?hkKObp6*=| z*lVYqB%3nQ-IX*tar8F!Fx^W|E*;bS*ixZZs$QFzeyEz5c3QO;#Aja)@|PFnEWYA6 z2LnEnBHgld#ARL*q}J72&z17LU9q%^ZndQQ3`0PUA*j-i%YHm#kIuMb5^>s1(Qr6WL&m@cC+u z@I{_sQ}-HwK3Ehvs`HsVnHU$?0DqiF&qP`RJm_0(Uv19jNcMo`y@0 zaJQ%#9O~q}a|^y4ui5HfS1DX@aufA5gz`Q?-Z>-_5!uS^pU?EGh*~)RkH|RZde0h* ziNnEMW9Q~K0n?u6K268fb4~V%d(A^7x*9u6gM4QO!+tw0W!qabqVXBy2g}|^L)z7S z%R^fxEk7H1M4p(rtHTU0WDVCfJ-JGO`5^o;$a>u9UUErJ+W2S`>VSl0#b<=7ekEEVk#-7XBt=^`O-&i& z%Mc1sD;!*H=EVBJV)R|dm`l`FBcq#)H;=LFmR$Yr#C(&iQp#@8;^fbRei^>I zl%|Xf{|9399)I{RhLI|t`r3C;UTaJ9VU&?|Bt4io`X1i0tv%`O24@-AH+?Q43hKIO zFUvO|_stCqCFcz>Tdp*$yoI8Y^}Heg#efSiLqNKjoG353l5AX=WE}Y1Srz<0QyGX2 zn_XYqD7ipGwkXu*Rk1-=Y=Q}T)%cpZNIQJd@3No^^tmvYPF8D>E6CVp_G;cgNvi3O z_p8Fi{J`&9Jt|~46|KoA^<4bavbu~;E8@pze9!7Uy4x;v_-sQ+=Sk6$j`oGN>02-4 z2#UA_PbV@A3bW&_7ryfYEf=5MZg~w zGrbkPmtEV^B+nyek~PNEuCnfz7gs<1%l_y~|Bq_tJU4DKk)7eU5@spOJPG<6!El9i zhHP~WpRVy<7>w!)yv9tEg;9<`a(CwYylsh20*L`GXplB6szI^Gpt#s#_k;1T&!;&F z%c4IJ+1^F>C%fJ<7x|i9MHWj5{?TC3oyqd_!FE%l6)7 z*wg@kMw(Oa+F)~{<=u>I{#Ah;lUm7BMt|_q4ggO` zxc5mmBG!0Bjd~jdr_3v>o9v~}7KrJGckZtr=t!28`S+#INHHEB>+RfLBw0U*b39X7 z_Q0OG=DZE)9pqn*7s$q*tkEqU62}a=d>bC_o7VWj-joiN38vr0WFiFroUFV4Je`{* zEDGDm;Mv^?x`LqM#X(hm{=XaY*(5QByjaA|$tc)(AxG9j@x-4iovkLumL; z_%!b^r@3|*hkNx+`_CcL{8m_}woYOu9zu8%0MVEIWM;R%F4av>nq-<)mKYYDspomi zm>SjSskqi@iwdOAx5f>aZ@Fj~p3WAY-B~8QT|6nd@Wj5{Z#?z$BRrbFoB+?ghy0I( zQBEb^1(8m&v3-aVMU1wb-j(Io-@O3fUl@4eV-Y~Q%KWwcSoZ@`UjJWv=mNSBko-29 zL1K)aW9xa2`q(9+`#`-i@MP1>UpNA1`cOcIwWfb780vo6J9_rTtdzKoi4EWg-;4^N3;I}ZF7BjBS z3JcI?a9SweGmeK5?~ZLxpBWTt_#@4yCPF2igr^7}5c8Zq@aLubI6}_hHEkFlt}it2 zMSZ+lt#ImSC}8&>g(+y7rupF;-v77&bo1b{VxBagnJ-f|Rcc;SU*u`?7!PGTT&Vt- z;YBZ=rD%WUxHg47k!HC4fOExa_(9h=QA&;0ulGD&ZDe%e)lFm*cXg+qD|U#QcUE?z zqWfP#av=I&Ib0D}#PEHbDf4=ALrEqOyDdOkew_t0v(fO2BQD<)l1c>@-Kb@`1dfy; zy24|qFKBsL+2a+r+0&twUB)B)Y~2Ta-Cy7FV`~wtdU_q)tlL#7OYf$v4*{(bq;Dv^ zhM;HW!8=88`Vkjd?P~o&0@7Ow!~u%VpR6ykJke2rUzXjN=eL#%nG|cBXy|v;CM6;* z^74vK%%J(D&57Mz|F--6dz1TSy2jyQz|y2eAjkdu)a>$xlfGsf8T*m%n-c0{w>zcY z)ta2X`$_DNH5-boGr5Xad&h;Wo?ux@o;3iz>He^fs_0I)Id-KOU@#~MG%BCzY`hWQSY65g|a$$3^S z%+cx&QhKXm<=Q$4RiTJ3SB>f6ZBX>`xseqWJAM4vsYM_hL^tw~rpf1y0fh^^+5NC% z(sgA~a+&c=Z}YF*$ln3+nFu>@kQ0ia)`t2UOh_K|2>`V;hHVj>hj<~=79x}qpm*;~ zIr3dwZh^%b)Qd)Rd3gQ0Q1Af6SJ*lUK_T@AP2Aubd5jNQo=Ktgl&f0FeRcW)qIl$} zgR<~{+B&xgmk8yF-U5tGS3Pm%wjoXoQ!LtWHW*;y&*OgAFgXbBFA`z|gP(45@I}O~ z*RBqjS}iR*KN`rCld85GylY!(&+|-s^8JZp0)S`%#ww--5OwXbXMY%1<=TF#RUeGvNK_1t9aL=~ZM(0;~p+q>NqWHoG z`$l%EKfU^yeadHf2{J5Um1^GxC3r6M=SEd*142?0Dw-O+XJm zJFgcRcpuQtt`pLt4gp_x#HvTZ2%hCadpnvnyAR|xW_zDW!4Op z^+5SecCh=5c3J9so)bJFHhF`H*AH%a;*z3SY&Crmg6{5pZgYbmxXz`lJ{rG)@#!?Ml`YNtNCLT=7r@<=_`HIYEB;N1hGb5Qa#;46VMuEYCOYnk7}LQjidxoFj8ZD+GLVcGUc=j2Uce}w{{`q)r#WOv&QPNc zxK#8W9!wok%wheKO5iL_|J?*a+3^LXKWks!wm4$s1*+_@FG8cKUI!vxuz}XDH zIbx|=v02nM(?%Y;N~t0SEoh2Zid^CzK*8%RW~2*sTkiiyywIz1D!(t|6tOf^Qgqrk z`F6+OdMcJlKG`axoE2$}c7x%C?3U(HiPvnjSzx4Jx~?x?2Zgd+t8@VNLJ**8?wSHw zOOpg!kV09=HSk;JRU@L8V05a71IBO6^uvU^J!do6+_ zOMS@P|Cx}S*VXKdypNP&Bs^wa-P;gnxOTQv5cpz@36kyWSG|o7;{sCORhCZPUv_&& z^&hXHnMUlMnCV8dL22dt()!h(#e3>oVPFvC&QT;iTJ@qG1)NFPS?U;fUdf1s1KREd zwERV37j;2Bcw0IE%4!IL`8s5w0RO*~@kC~)_#A28Z>>H#7homqX-f<|^}hn>;cO$c zzyHa7g^>5ohqN5~LK22!{zZJ$`iB`j59z0lhW&hCwZHt_TbY_}Jxh{UA9yI$yLRQj z#WFNnM=Hl875|?E7gZ#4Pi5Z-MZ_CfT~tl?k~!K$e`BPbcvtUA!4!@LFv$u7wPnQB zamo&X9v_gfc6jI~6hz!oH-io(FJ40X=f zUSaq?uKf;2wU<0DK>QI}LX&`5$xFUYskOWc(+y>sm4EKpPJ~}#12`=LIFS%ojmhA5 z{=k0r{uoZ<{h2|^-Ka;vZ`Bj*x_%QJ(X|z*yU-rtdQ%YMxn@EZ)+mJ3W9e7N@DYva z;tpDwG|ljMXr7@%7|>fI6-@q2S*>~V8;~;b61cyq3iomCT_}$iufn3G?3$De-O*GK zxpS7&`qReVj+2ZbP7m;g=yz>$hnB@2G#&2J+yR?)p@8(7!|x<~ivF-km!xeBaI|vM z+-o08R@ni^_9Xk0D6lXq^dSB>0lS~k_VNA2o9L5@ZRhm9&RIxPmIziXg}^KeH?!W+ z1R;_EeLfsw_5v!WF2Q*zuHCSi<{ug_ON^G&}pm21rn04 z+=m_6oSE*956I@?%lO7}Bz}sFXOee#WNjV-N>E%;Bv3!}@kKrMkZcW!RHVGBP7s{A zZR9#0@Ellcz?@8$?5F1UHDe|~+<-_T=*RH?6u1wBg%q|E?MN~wGAKW~4}#@SVZ-g} zSQxP?E!Uzb>Wu&Q3vp3EBOW0XGc27or70U9`{U0wE4d9liQr-}GT++_FT`x}z>&R) z-}-~k2;+WL62w^cgW10g`#)%e%3RDIYP*W9hGFJA2}K1Kc~OJ8%lCOZV?X)?G0BrX zTLH`xb3%8wZvrE?NKfZS96wxyQ_1RG-wm{b;`o4kYawGQ``3i%s$DFCSx@lgdC=Aj zg$|Pn{su8(Jg>t&UC69Q!%hp!r5sUvsEhJJRm}SvvMq+na-m;2=O#1vg4FN;iS4L8 zVW;Nx#ahl}TEJ%Uk78dZ2m*y5Nzn`g0CWLFRvs&`nt@+Ym~jI9&OYViT|jwn)cNn6 zn9MQ^^#}=$Y*y{5Wv>NM0#6&*S^?2Z$mC?bHF1A2Yw@nPnBa=4nf01M@z=K%NItR* zm_p#F{^OlCwptB7)C^{sB$?00*uLBch6tKh@kXE*e_>V~ zWz2SxFp+huA_<{It%jBP#P>!}2iJ-5k3JAzlmndEVP3!<}cs(&tUTzxrO=XbYsC0%LkQvy(1{zu8jAV z)Wt=;prr{$#RY~^*1WFDW%F?e=q65?bhB$CIC|=pvIzIa)x2woM)Mb*XHwwWs-$@8<1AmHyLpb7j&JdV@2!W8VX~QEFZfor?8JMIB zlqDe$B_TjAHEdm$2Z^(@qk{cJIv-)4UGLH?CIz`JJVo7b!i$+J6JVpa5nK_mky_8_G-wlFl z)v=OE?!V6Sgwi)8m( z6GF;s#5d&L`6Ye8K56>(sT`u}r4+o&hRpXCWWGdT3X2q4T@p>c?*^sKx!?%Lz^U=n z`y*w!W<#8yNzG{vbZScoDUM_1OHE6DH}Hx4tvs?ZCjmLyZ{1e4d?p?98H>E$gN(&`1ww+Nkx8inIfkM|efg6!(HZWM(=- z=|WcE%ym$QAUB}hn5YJ|zQ*v!KCzV`9Op$@++r~0G;fW>LsMQ9PJA*OqRl=ku*O5k zVBWGfZ1Qd1X=Tk0!9nnmjW_P#8<6>Mb$rF!LdCzp#k*w z9B4xB@J}j5o=IkXE;Xs8CR3mw=gz|mz#Iz*d3oJ14Q=h*tp==tnoIh#$H2QlV{`?n zyNaY^U5YYHUlX2SpHkV?5rT{XiZ;bM_hj4nv16gyG9O~~UzmhV%NOJV}}QVSazw>^^+pVQT{C?FTp#hXP8;iMUc&klzPY zh0GbCp%?FWlRz5!-#oTlWlZXnG@DoTQPZpeP4St3^HPT!aSO)G}Y^G#S6`@?#@Re7>)PiO=(WgfSjSi_d>8 zlq8PW2QUM(o(O?$O#Map`X?{5Z7F}SoS$M=+t+|`=0igv>j;9!JlIzV#+4h^m5~p8 zxvi6Y^hVMlJyS$wM&(!-!Q0O4f-e99VFVxnUw}a)Sf1*ilx9@HynQm4H7jcC7tZv zjZFy9s#QVq6ry!kDEMxeY?JIpYl6uLlZ755K|Ap}r^S~@(B7xyz{Uf182tz4hzs)V z9B*fV(n5|B7+9P*_fhs2|5>ca(v<4Y3pAI(vBs5U5_nEG|$b z)pik=p=jx9+rFEE#YP2cvs7P%6E{$3xD8@T+OxHtvLJq0v7-U9qUxL&)1#t3^^ys8AgVw(5L(t3CMW^1*LylWch&4nBW!td>jw{2U}^e ze+wy_gfXF22J-~)Pmsqs{=;i8<-q-QZKA5-w%F@!x3LS?pa3Xu7JyPgNeBwkHZ1=p zp;p^xr^tSS%%#FUy$1S~Z%bL-Jz96x{B)L@!N@x|gxOm{^(}@OBo;JVU$`C%^m8Mn zu@~W&@L7}RxEb_-Hp zlFU+|I&+^g0L1sdFfxo}bV39~N!YgXFx=MbWOT9!Cc9?>o-=7UWV^2iC23}mA zK=cv;@>q?(uO8I+S98xL^+^vLPb8-RyK(I;T#EYGx7-hpZrOza6(*s^Ht6t1n#Js* z%cTVt`B9D>zQ?2j0Wirrs9VUK0r= z%rS}!aXuOb?n=*mH(`1R*Okv=1MIcn=zlY@(6x~pAEqZKb~ROtex0T z=hBNoCvEo?m~dc0EK*XgAbCfAF#tOp(z1(?<%dPFt#Vlt@74a)p&>r#0@;qI*&unB z%WaY)$eh`c;uC&ULxX8PkIfDUeeOdT0#Y>=SR}zcOIWzetDFFJ34L4>$#q5rD2@cd z&RnPyYTaax@8U!->0u-wQAuBGsz^U-{p$@TEi`hF5y9Zw;@y;f(D1Vj$mBFDDGZsA z56!+sZlGIIGXj!gjH@*f3?uc6P)^Lm4=W-_Fdb=k^VR}^k%u2Y3vd>}(q^Jz_aP*8Qoz_Nz5-`hVXF}NCIlzrdxKUxLes$-yaLFIzQ5gCe zh|3Xw8vA)5NfZOYVISLNJVf8;_r(_FOtSMO&VtKK8qgbfE%SNXQ9zMZBU(>Et2U4> zW>Rk}A4H?i>`x*|@xckmGKGNeBlYYIgz*gV*CkO2S4$Jx(>HBwMO zxhHmuVEA8|=q^4;Cp=vQ19JxkcV=*5bn;qfvscw zo@QD2K2_b<;>&|oS&2fFH|eqN?9 zd)g2)xg)9da5&!`;}3ZNB`WMIT$6F`HOZYx%zZu=Msef61Z2;dLk8KSoS8!?IwYm6 zL9u#z7jjU5ZyRo&CgKj&!p)Co=`3J z8-76T*%@Cxu8Pw=kh!A*nd@T#Crpmd1lP*P5_$_679um4&dCfFO6OAB*lmnM5D_{D zx=~~-@)iT|5=B7VRlUC4os+#MJ#zxJ{k#fo-1H;HtpHbeTA=l~A1w9>`VxTBWKr~! zvMq_*#W_}Nes6xa35f!AxqzMk;rF*gXL=}wo)QqzK{eYEVK}sUs8PagXudi@gbJjbXqX#56f9><)xE*Y1aF9-B>I=7>ba}&f+DZVU9xy;^a=YVjB8~B zJT5WV1qppG3zniv%WZuR74SjpP=kct78CAP0Xk6O=^71ZXZN59W5ar|ksO~>8Yyt_ zDnRXA{@j{l)fZs(G;5pbwv?MCXVWLtFmy3j>*~1%0Um~Cm0*b5PfV-j5TKXJoId*y z<^p~!11QL5!PoLlW;6DJGd^o-R04P2dC=lD%#6|M{Q#h<$lTFP0sV`5B6Zz1#Baax-u1}BM=9zz; z&JZ1j&ZXWZi40mIE>C+X0VNLqTN43PkmoCIo{#MwA<_B5Z1E6|tG{b%%G_(o_{Ly0 z(LdWT&Cj)wL@Esp+hpitAb>A1tSS(Guf_*TjRB$7gG%JH42nD0es{~tzO0}ud_(Rx zkZ^O9%*Mg)>BXLbGUe$m4JRaT0w7Bz@MaW!MDb6xSB4kbg5duQ!wKerk{DdANX|J6 z9ZxHH#WfR-xdb{OK=2*HP_dwj2I_d_u1jOsGmkAx9m!{yn<1t}`R;-; zCLeQZp%S`;Q0EeF6V`U-nU+kx2p{n|1sa;?YxQAKgZZe_{sfg)E)6s=0-_J$v0%Fq zO2_*Rl@y*#h~W27r1T3D>O~4zC9?-Pm zH+)Mx->G)1JWXOA$A4S^Vt3Y0a+qXE0TVtZwRys5J}{j-EO;w z$!5zgnAz?%I)4`3J#uq;LFjWThJhZ{ML0wT_HNt?qq0S#lOM9F2dqXEC|aN;XIO83K=)n{?XK-I`CcG^Y*83P@p_QfeU0N zd(_WUl`qDbO*wxZ_5#%F`Sujbtr!T7k?mT0a;*(s;vC`77&QTfOG?fE|8#a69idzi zG$D0QzR@JORYIG9jMrB)@8wA|MwkQSx%2!`Fq| z&R1dX>WL?9qw*Nz{C~B;!2KChjEPe?n6{6rmb~sW4{7bGlLuSl+h5Kjm(YqpZQ(Le zRPvr5GOm)xxsQ6OxGmQwiau*;UK%cAJT+TEgZsW)0e@J;zkS$yZZOMx+1{{6O=4{z z8_pWy834YHj$gZ%qIFsPi;+RwoA>G!DO%FsJb6{cn6sfPI}&cp=t^VDb|wV*st!%o`lK%Qu0qt~IXd7F zD7mFlMmx&7*{>G2w;2o~ef5{$gs|NSaK9nob8wKS6aDmg1 z3*_ti>vC1Rd$Pb*{nm)U!+$b3Cb{!O>;RR_W-UW~+%^6E_Cq(AKis?&!Q4m+a28e2 zp>IN@(+jGWk<$LrmKR^2sQkFv4cHm6)w0uc4x8MpuFK8gWYX__hK_Q*6hMKM?pGcE zgc;ulR1BAjC=QyExrtvramPzKf1zIr}V3tcM%u4mFl8A46bn-N!yTFIG5h>LX?oSoYGzL0=w5U8@# zWw-Bdb!|3d9=R$^+g5%3tfLqS{6#2o|M-= kv}&?Qa8ZP1>Y`sbAwk{og&pYB;} z<5#?le~ZRH@(;C9c8up2O4LchT)aU9suuVam=%3uRP;VgIml{6W=23ZS_#Zwhr$$=!V$~ju^1oNpA4Dz!*n`#cZ+eUDxwzL}U z@a^UuU96C^8Np+K8!=Dh?A%!pcGM3`H1WK@vph`x-}pF!nt!1Ai19ZxkXPb(L(auB>hTUJ=dQ8JnMu(n27@{DqgHAc(14S*;T2E@ET+lYJpp-wj=o zqc70aIzakg03;!=bJtFb0Q~}Epe`Yc%VbH7c?6xh^%c)g+6tbJaHZA+TU(_;y8ygtK!99zs20yB^>rDJ8tHwjr74L>>e$Pu|v{ zF%LFdFw3P?evA(8?r$qArBim-PAW^HAD;HrxR4`Y*93TZ6UH3*UciZB9R>LH!A# zXiknK|F5RLF1ofyLOowVof-V*wzO-A73gcvWwS zrj)P5w{-WzQ99Dj4!kOIc@+O&Q{pjzJ5qMX&ORPtHySd7Bk><;M>FtX+68c zL2mpTc^(q=;VU%Y5*7!uBM;bYut23a**{Xqf3^5KEmtQY6r*}Fd%**48UhJD3Xj-~#rVD}Mo;=zJPoD-P{>`wPvA-DN74{sIexi^mna!_%w z)Lg;v>tYV6MC1&`VVCHZB87Unx=}AZ2gqfPHM5nY9#&fRxPGEgx$?JLB;zHMe2;P) zA5>v>x!Cgb(38> zBbU*?5jpG;g0`P|OC~($3^bl;1;fGH1BU=3C5Pm4aqIn^62n>t5C<}7MY=Sm_AC3O zf4sTzXyBa^!oIn#MEs5M(PCc4K)HpA6la0g)*lgXE3YpY2ltbm;pO_}(yc$Oe^}H2Z|K7)$QFCmF@z+= zEWaxlL{2QM2NAat?{9o0(%Ato;Q|=}k{%sE)J|+jSpRg8>=;~ZCHF6KFvaQtPxvBj zOz+YPbYtJ|ypnLG^SI1aEQ_yuwXJmK4%gz@MW1>_#>p-&vbZYgs~nV4eWd&`Fl!Q) zyQkI_hpFFHntE4rd&k{-^#xzb5g_od`);%)OH>~d88*l^|F|z{{VPgrsL^ZdNmCep$1Y5${=nh6*6 z^@)$+@9zwnncIP}h0>22DUV+gDIRG_!hXKs#Deb z)1{6CN>4tTkKBh`=?Sh8IaZUy?>qu8vw-Pjz~Hho;s3u!#2+A7jt`g1#-3$IaZkB- zUVk20+I(nJQib6&Zq7LY!xl@Um3g(U!PO$eMvp8yqqR>CcWieO*dwqj7q3*e^Wl26 zKo;praX${jB>#$6o@lgm=ka3kSN%IL1$|NhgbJ1x1Jn8Y*`aic(SM^m!J(>)*ncOO z(CZFo_<0zRWugMM`1t0%7wf;>FuidDBNy^bpv91ee!qh`|G(2;IAc&Ed$CM}eWA>jD$%Q=n}5ks|~BNj2*F$_}N z{Dp>uaJnCR(}r%7n@>55X+QbxP3Bz^o521|T5sA*cW60r_uFVAo}#?^XE0Ztdh~SM zuJT^ISpM(iy!=sYqGeTr1@Nl;5ogJWh9xz0+T(qjmZ1= zYn5%Bx`6h!MTXz*_os=*I#0#h<*BGmJk6C_PU@s*0`VI&bg$-PV-;HbPlU}{FZk`H zX=4N&Q|xQr^4I;QbR3N~1GYT3-MJY~LBhf-P+(0KWA{UD3x*e*kw_uR^=gg{XMJwe zg0M5!S$uFm&_@kpeln;*zM-BJTmf+eK$3xi`p&;cqxwcnTyl$Q5`?*>C*S8m!d(Hx z(+VEiB6SH9+{bdp2hK!hjG}rae|sh|@-M@Tsukq=FY**?F6y~InBs3Xti14P zrY+d*`0s8@VV6CypE!G4zLA#h)fJXzLb77QV7QP~??5^bYB%B-sCWy5?}nV?bdT6{ zBTESKPAAsu0 z)KhIIzx#w?#6+lUP`B0;uWKBbZE<#tbp4qB_lpP+vr+5(r$wjfN>_ue%TumNK;`qX zpWv#RDq+&zoc=3;5@h3O_7&nhWl-FKX^s&#ZSAYf;$Gh_vMJt(Fw@3A++O_Ub^>f@ zdu^oey|K?~b^UA@UE@8K=3(L25fFKa$j3#WHa+S@ukIkNq6Pi%0^wro|~4XZ|0b3L>=&0}h`;lgyq)BlIG z?+&N>{r^9XgJU0?h(oqQ$jUfoh)`MCBO{bi_Bi%2;$%x^RI-W4mYt$x@0AJ}+4FZF zy+5D#`~AIsfBe4J@48Mdb>!Ty`+kncb3E?n^~Y@TQ@;20IE%>7iM`6bi4sOxiql(_ zWuv!eC*1pr;X0w@9DDOs7NL69sh6$8GyOKzOMODFkSZDSwvN4V7^o*94A>5jVD;%D zH9gwsWHZ}qKCt}wZ9+fIr*f zRX+*z+mTL)7WqEDm(OxvW(AC04L2ulF{@mJ->+L0e;`Uqxt9Yw3^+O3pCWh|IEuDF z%(l^JZzU@M8zyUWOeRh$G}0GxT)kB7*OF{}C-%-cIlmg|M?V*2&1+n8-X4WybiTe& zfAF*LH2_`h?59V+wM*IPUzIh75I~F01&6`pj<;?J`C0f=3VK?0R-nSFOoHLnPGOHf zKk$$#9sR~7z^r6RsQ0p_g(d!9c%+KzJ1%0e$e4CAy)5t593NAqYYU2% zK!N_cRrcEb{y_CRGrtlsCQIMJed)N{SL+n5K8TK3je6v_@^kjTW+*fEDdtcNR39DW zyheiu<0sC$jIn@R(NlqQ5a1K*l5p%XtQME*U3#fAX)IgQ+u9t2U-dz-Zb24fF-Wp< z|3ryW_O|{avXbCExR9+CIHUS}wjonb)6zMGGJ-fwz1$>Aw8YdWgYf%Tqls$<3|{DL)S#US_=D$@JoQh-l$gGK4qt%fQmZ)Gi_| zR?9nCZ@s-g`3BfG0}?eGkJjX>vU}uHzWc4Dk3LB)d0B6zEbYyNP`)_pf&#aUMy)QHtu)%QUZ{yX z@9sKyc>AFCyr}in<>@c8=*@MMUQQ4J>5BHR;Obhzk$ba4Gw)whe!ZR6Y?h_hKjrFz z+j6jTq$5Jk)hofW)x`BSEj1SFGc+@2ZvYVH33R4*yxcATR&+Ecqb{#TlKifx@3RyM z&Jv=;p&q33X(b|xyG|ZzKg84P?!oWIS0`^7X`E*ihgMNz0DSm`JlXHey0YKdeSrHj zUE1Vuazt=vM)q6XQkpq~m*>p5_=&-|2Oz6Im46Vxd-ZSnd4CE~#MoN|^Y~2yiD|z| zPkUplX2B`D0VfBD@%`qffkXjv0vx^1qy`GPEj}h2lwCa$O(&qhgiZ=)~y_UDG_?G#Vo;B`eLhvIzH%8lMDK2wQ>9(sPI7#i$M&E>;k7u{JJye{)UOqiDvdbOE0>{y zw|9Y$+emIIyDO1AC`77bh;1E+Bp1t;5Z{>(>%-&6H~mBpUnI>3*n@V^3>^t}feD zs(Zvw+ITG0EiKrH9`-aeNq7hOPfw6ysBh7*)#u{7v@7_zOx<(A&5${pyx|oE0fU7( zKDe54t+SvDA+b>2g?l{Jf958-2n6-So8GC=z{hqn_0>LlIL+ilh<*{TcMe54O&t9M zi#nnfPDjl4^}kkd*Nfcf$167X*T_h*u_{M~YNyfM3Q~+ZQ~1LlUsQgw zQB&u((W!>PIl{3c&(gU05kVm~Pp~(st#t3YVRv8a6~6FGE~1R4zogh1d;YTLFB4h; zeX`{tslMCw4g=g@3mHmWO{(rC@jJ|t=QHiBPEnjf0%3bZp@EcpYrg0v`6GNNhe<*a z4b>QI@BQ?&cRIvddwY0Hd85*wb^P!k%Q--Wnp~Nk1?$EZnZW6>KlQL`wpDg^*TAJ% z{m7|X&)(_U+1M3C7Hr`+xt14ldf+EFL;H_A=@5kzc6WKX^YbR0eRmPX*X8k?h;)YP?|%_Jo4?}?jeyV+_EVv}KG{FbMCm~`4S%RW;+2dW2f#END+MHb zC_Fy?=zGv}Nw*pJrwl1O1{Sqy{B9pa5|{c@m{C^Wy~Cc|MxGn-FmNywr}k{B#;M2ai=h_~RS?RKWCN+{9_)*XsuqOu3Sq?{AMrWl&gd4vCR%af36 zu3Iy@M*&-&FQ?>E_G{G?LyClKROq?nx`MZuI71vKR_FGkU)|iCWF-1Rig?puOU=>M z;=eK1?)u{k*J5U(VE@g7Yjn>r#LBmk+}cYYZXH%uFGjY{-_}MYDQFI{w?3uP+}=H%4WAj)99b z+L*YAbwk;W$lCHADs{(mt+s~B`5*9p4tf6Lx-q6pu|n^=3gMj>q@4lr-3lVo73}R3yn!)TPqJ@;TEP;ABx00mPkHBdM1yy1aXK=#J-0=PqsZ^kP0h_Dc@6QYTTXJJd6 zc;@Ug_t{&+PEDIXD|Ncet-G?wyRmdqW;GY5zvYU-!nMCOZ3ycmhJf?@NjumJE99<&7#ftE+^(1$&DHewV$!1fHH8 z^IM0evS5b+MsF~yw0%r1Y45SvLzmQQE)b0X|MehAz~Bb)g@rOR5mFc-p{mPE-RG*H zF*21~8R4n`j0Vw#oGs(^Tv*~Jud{l@D|lzdC<9WijKexGehFe`F<^RxGv zfZIajD>TgA6f{ond8nprqfRzFC)4rs*ZxwFnN@H!>pOH?0jG}mSna!ku>PLI_m(i+ z+cSf(yxXyfy-zwl416DP{&F|F(aSmg#%A4t+#>2Q{F@r5g%;G6j^o+!?~B#W6S40P zDd+cQ=doGa$(-uc3j&GS{&ak1j4B$m?boHuG=)b#-&MlgKxa!iiL(c@2!6foN*A^g1Q+q1L z_bF4hunn)y16=npNJ=GldgR`ldRggw51Wh^0toW0)#OXY;om-{7Y@5+3vm&NeSQz( zA*A+!;|pS{>id&XV^xby7j#Aujr-r2_YM!0MC5@DBr>h@sM<{fZZ9t}cqELJ;dZ^1 z+pDS9gMp`ypNNoh3?UNM^a?dZ&hrX#IQ6G~7Zp|`jOb3_DfCbZCsSA++FT;^+FQ@r zyxt>$X;;M{=b}`i=?faIHt+m45D{1VMp$khcuc4DR7|&&i@AM$_Wi0=EXXhxj;QD$ znK<%_=@SwkA@J0C68Vz)XR3x5`nZsbY@C(BF0=`Z3yq&s)r~0q3n;FBhXv?Xsn$q_ z=QEiY!%3)2=G*lD9H%Qc_Qq@b(Vl~Q)7v2$tEG?Ltq{3pV)6eV3GlTBHz51hY?rQ; zHp;E?6I7VGnW3C5To%kx1G+eYvY(2X0liKPwXx~f(6CnwC8A`_b{eg~enD7?1pb<+Ew_&N z-q%_wIZ^B0nfZ7Icrex7Uz3Br1f=wD;&|LInKzn5(&QbX+g{&w$2t*l^h-Ycj+GUc z>P-ppzH}jComT$MNPC>${$>FgwMc9X+jtE89IwQ~~77;%x`p z%d1D66dUTHgy(K;7z->ThpQh<-)FGz>QjlLRe1OCN15lE*;V{bP0C&jV&K8!-Fj{I z;N2oRSuUK*5xRgZDDL^$Mzn~>Io!T3gf5nzZBY=4j>AnE$DI@UE^f$XTE4h*c0B1e z5#$)}Xt%!hs_lPz0aD`~CiAV&^m71a!pn~LHgcsch1IHliP&DBtvS2VsIDp!P z2+tr44hAi@?K~b&+C!N+CSSg#n&@A6v8PdwP~|#p`)?uHHHx<~BH`BJunSPwM|M#X zsx}F!n&g*zL}ZK>@-7~sylE7wu`H2AMwQZwzK1)`+q8`~-2KJ?0)CyXW{HXZ5e9d& z7t-E(Fja5VU{LFJ7bB^lDZ;e%b5P)>I*bZmjT=(D70eRvILdhAnPd>q$}v$H`+(b} zX6JF06{~2WrlXtfyu-GAnZ-}lh{G~vea9xv8holuo!G5F&r{cxhRAGGC>e)+zlfCm z+I6WYa9Nfz!nVs7loJioSUKH8fG4|2s3;<1FX!kUX^65KsF1cFW|I$y0rFF@%c3=+ z&Q0<-{luTj2($LIOW`!rCKo{;5OC_hB@A++SK^AIxewO6ge)!+-c^NJ_XAJW5ncR8 zA#%Bpul2P2bQHvLDT)`_0>_=WxE2q!Ho@?>Klnjd9t`2@1+?H_IetD8aI}o8^u%z; zT|eCMShzvs$rUKZDL*JZ=H9&`H<2(gaR#YuQd%D2F`A@x&H9TR^k$~BB1Ksak8c3_&)l9{T<8q>$TxP51#AfD0ZcZ@<5?b zlRLTT5)bBp(|_%pDdW9z7nZwxF^yt@II#N;mO~{9t|esNSX4ob5mu<+zz7|LxOBQM zLd<$7yHZCh?F@*vd~(otc|i$3I`r4h)tRTOkJOl;ol*I z+LfzIg$%sZo(BQ^>kJ##{=Ow@&2%pX5haWyO5@^0Qy^PBf#MN1hLVFHbKf~Hqv+*2 zwubX+NafzviCQcVTW9QEjX{h z1sOfRw^Bt+sr(RWp3-$Tkrsy~nh$|)%=kNgb7YN%E40C#A)ySa`G1BKa?;{?Z=TjW zZ;H8f#OuVS+L-YLo1RB8KrU#`(M%%oN_l2@l_fp)B$GcHgBwNcPGub9gbXrG_?LsY zQCOb3hHg$~#kEfjpUaJ}C0@KGGF0Llf?ywNDSv(qu0~JLfM~gpu=hQimV?pQkc28` z!^#mS17(tU>ek#$l*$W2tszGgGB-vwIPMuyp1hG13I}6>&!Pi4+-%J(L)B6W#U2qg z?{G@U(aPYH{y+Yi9J-lxYy#n=v+|m9bbM+PiZxk}b7I3jE9WLu)q?)w$tI@SLU0-Lvwz{`#9C}Z0cU+a;%0{q%f?tvV=Dpn94Xm&HIiiJ8 zs2U6nM&dvY#CHl$C_P9KCS)v3Q*)hNafJc=#d_s$HEZr9Ryz>s z_R{&Byn$l9-#_l3zHSjx{I&Hy+fvIq?%2;>5FJeOI5LO$B7qKbvoS=R>y7BMth4tZ z4R~Y_VSJQ3>2ItPJf2QL8P1`a4Wyzi1LbNgCCJwu#;Kmbi>DYu$viW7bdw|bTMXEz zdZ8FhYKNi-!juPx1awGMo~x7apz)%_xa3Rbo1 z_Em_YKNe33j8gq$2>|Z}{7PBsr|4G}DxvEyw8+!@Ph0}dWJEfM9}%Ye8d2Sj^b?5U?mCM^33nhNsH%MQNv6sDl8@Uy{dtmK9AM11 z9;6C#(Q`Q4OeEGu5gP;N0KFBs(eKTSczm^7P}siU5M{FsgrUk3h&tg`p>|VCdy}B$ zcWe;v?J{)vC*01>hy-1@7MEijXXbU1@5Ctl(=p}HsFpnpu5aewaRa=M56e@1ilGpj z4e}FTwhzP`xi3&z7M3-6(EM5y_8-Z!18Z#)b-BS1iN;Wp;Pb*~8251p4MxESH=97# z@a>@%)qFBh@;m_qI+ObG3L2ug5!@y`*?qxf_^7hg$O8BUU{|Ge9HaujSrr-9pQD@R zk7J!Qywt%c3rxr`F&UFI3$yrr)omZW{xS#I;j4FQxWMpmGrw@$DxG0a7G)w1xvmvX zc(;C?7LT0dnsd+>^S>4d=E37eADb*&syv9r$NE@%<2`%B0qHms3~o+gR@7BU7NqRv z{W%T@1=I_Mlt+tln=rpqhT3w@q!L)Z(h6PgPpiaP-qz*Q2~^xp%WD~>W&KP2*=DBJ z_+?-?Jt_u9FK8Cred|R*1({OE?zQlxFP$S1EK2&jdMK$UQ?+XYy&L4#c|DL_EvwPN7mJbF)KMR7}g+|J|i#gwI@u#HV6c~G+syAsco<5Tsz>+l{c zh#AkZ#mItfrw!`w*EtH{e`D!_|66Xgj}f{@|& zUz(kal%V{E4~O=%HivqpC(%4Xds4gf2Cs)`g}PjL;R5ESM`C~Ps>BF8&Y2`J1p)b= zF@wev=ctcU@+t~rk`!Sf%_CrvGKdY7W~7Kd2f-24QcOOjC2hfQO2Kz_KCx$Wxc{g{ z32u_|BYmeIOwgN-nqR^i;4{H?FJ@l-d6$s*P1X4}ZM^<53-br}16uX@)#SRK9PHqf zdkMB>gg7Z9Mg(}!(TBlH;-e;R^oSLi*AhqUoNrHx4!o6!@J(D%hrT9LN?sCuuC@{U z>UJ^o!{xTQOyKzF`$xiXYmA}cEuvxVI@RMUEP6g!b@%E>$p;@K7t!C@AWm#H*fuqt zBRP5RJCMysf=|^-Azfrh0f_f%FQ;|urSX(LC2KTp#Ry86)mWB1WSs_006C{RA1Ach zH|nH~nIOVaK^(0+-aqc*tB-Gf3-KA1$+D$kXQWQJiJ3`SzVQ<0QIXUD7<%106Q6P7 zJP*YAL_%z9_r-Wi{R)<@6bvOr-6Rc#+J8(Vm<|psb}%#24P#I21Vf%m{$dD2&YnY) zK{$(3>xa@=T(~<|d!eB-erV8EbrPsO&p|NIW0$_!! zGXVEt7J|Ttr8>%fCkMJ(K&-}XNOkUe7}4nS^}Xs#AA$ffXpDF&E&T_N{8^nYA{!C( zR@D!Ipi&^1RQ6G5m4$TUORLKNOPS&4`48FG9Aqq;40p}MPm@f4c~w7j#n zjU)-e&owZ)N$Vrz4Rrmv${JC)7F3sN4-Z8&p1y{c%D`|NnA9-VLtIRT_QnNdnN`_5 z;B#MaAWs8_?4%GN&%`TJPhlaC#;;!6JfOU#qGX5LfYfAbaB1>UPhoK^>L70yX-f`giC^u2VT?EU#zLu z;XzW6bK|PXJPi}c9Lb8XxFQB3T-+Sw#RCvc{e`7aux%SkC^?}NHx7HquyX z($jYznJJY2)?hhRa3E~PItdZh3kAluO2YeAUu1!C%i^%uKK2$t>qG9~r_S9H_+X*d zEE!Y+2_Jjceal=shLXykl7wnPD{&qd$e>z*ShHK!xa)(5Hl*o-XJ?13BLiulhe>t2 zAs6Wst!1xX{VP>~6HWqfuof*U+A~Ql z78LRtviDK7)jC%P*rrn4r=cH!Piv7FdZLR3Kv#`48@+lxlxS?ZSGVn;Up*PW`ymdZ z{z16lE9L^QYIAVxBsq=CVu5j-|GM*tvB%ll%ZReglZ)DQn2l+bq1tIC2OH>S%gQ}S zm~*~{v=0gOpBmPgX$$~MMl{t+F@vY&gz`PZkSlww`-v5mKqY@6Ww2TQFxMb|k8Gx# zkcA{Eb*LfVy}X}B&aUd`bY{s|FIco6OW~s(Atm_O+ZkfiW&A;Dkf#tySoSL(b|CKR z?^x2$1UwGN>_+EP(I@D3V$RQZuV7aQ?<9w?e0Hg|pkaCDvL|@D_8hkx|9Q31;2x7$ z>}4dl69D&_K@z06cR5+`1b+2-L!QB0NWdKXh)mH0R50~-5{p;G--=}vcH(#%au6*E z52eI$Ki$TL`z>)->^ShE5$kced94DU+XBy_&786u8Xc)@$rAo!EkG7r3&_qbp*ZcU zODj0pS^vRh=K>_qOtJaip}PZQ9B&1p({X(Wb3Yz^mH0N^db%@;uT%I_M#xp>XhoPd z&9^=2{ z4Alx0jeT|)&5A`cAiB=0B!@tF8Y5J7Zz#ZsEX@!CrvJ^B06}p9ZVvh&GL$nr5ciqX zZd{TQxXn`Lw8%fMp8LX-irT!xcg68t{N}*TFVrncg8( zg{l&BLf1tgd^tMJ3SQW@Zi~~g3T8=L$-RN^ZBGA*ym5PCaUbVdeEt0(Dh>P^#K~{b;%%8h#67;5VzX;Ug#JMcpv6_H_nU;D;My< z*EZZ-s4)+qH~AHW1YP_xq)8$&p;%4Am(J7LKY9dABG-_TY<~+d7F`SXQpWUqI*f0g z;PshNlKvK0AyT_k#sIN_V9C(ezKJpD`zQf{xdy#?xQ2&bh6sk@1s?2&oe$Xg8t6GK zO(W6i>U7<6>BQ;BqTTupcxcRcl&nD9bNvVj-HJN_B2j6O-KA6}Z>8vOkQaC(MA^!I zS2aO!A8CHNL^xDB5vp_;JJ-|Rw6Rs-x1)^Dey8R5s{QKYc=TiPJ=@S^teV!L&-xE|j1 z7k=E=qVe&5^|_)M`b^(KdD9W22+kWMR90`t(ubVh7e%`B$zjgR?QnowFEG3~IIJ^! ze)OC}2{ka6dN)B~ctaH>w=_sf&hmjbFR~+H{4<7Ju)`Sk4D+{At2Mubx2fY!;e%7a zNl?ZSg8PbXzJ6+jShpyLQcsaux(v-tF0F_K^_%Rjxp`^}Sr}p}zn?pn!kjaFP)o1P z6=GI-Z_Hubueo3E(`z1`@|T?YGQC|hAJF!>L>7?nJ~J$K)!6PQm$vu(<9WBhoesU{ zeZ&gccl$d>sva@C5Zt56pSqZY5cC>-Hj&lf)bgzMkR3froJ6j`l8b2BGv(w6{V{Jd zijI1abZz^d-jm%y88{i4m>i=o&`4ff2)%c_z*l5gOTsK3&SUC0HG3h7ifn=nL}KXn z7o0P4&>s*<5;Cz3g-?tx_Ib{g2i0Hur@KJny`{z^A#&EG!phyH zuTPbutjPZmwh(!%c2J$-EbcxRbQrfb_vPb&WZ*pQ{Ub*q2}XLtrWVU#*6p=%s{c>#X-B3a(125RYFPc6;AqMV~hc5{ke4ZkxqbODu*XXm?8B^pso`N98SkCI=;^S z=l*3qq5bdlNX1?m439u?VK?GJP*`+|#b%-dwigbfMZ2_c#>g-UaI`8qPO8ujxQv~E z>tXKd;dKfl3S|r;LwyHY920CUMbPc@FMkMN$ps5T3?g&xsuLWtPjuN#?UcmrE`M-L zK}CD|z#m}}TPy1)8ve=DrA;G`P#v8!>Vd3}=2F()d~< zZ|qRTRCtw`w$BJk$tKg*Sn2yjEKMM5zt-Jjnl1XU^2d5 zFFg-B@fBg{HiaO~H`ycnQ$SPZHqqxlNqC>pgW9?UWFLK=$jd-XVB&^DdKO@!$BM?yhJt~+oWLtHMur}S2?zNH5yfCj!qML&eB z*;$ZnR0}<<6e{7jj?#wS#3gs~`cheqW5-l9H+}@?f2;U%;`Al#m=3ia`Y9ag1U!;Q z#G6;cW{(WFmv361AAJfOm2lAOD?eY!4baC)Obkbz2Z@Eb*M~N}4kleT*e~r}LRW1> zT)4%&zJZ4J|E9>!T$!0R@(z3OD%Ozz9b+``t!=sXHOCYAtk#VKJv#{J%sebphP3_H zK%vWia-vq-mC!J~-7A-&yLN0%kxXXw2%HZ8Y?_3Oy4!X}op)2c{#y)BV5Kk9mhrFoZ4iIcyp#Wuqi^pPE!9RJGKdPruqd=0Pd*7{Xj+NUe-eZuh zv!Y@8ew|yAggmoGl0N;Pzz)D-xH6w|S^nV^@)Q-wek{P7bX~3~?+gm>B1Rtha0yQA zDY3|{?ZW8%^EL5z?bd3EeiYOauZ+DTy%Zv+8pJ^$y!O_-ceNrAm3SO07HuV?(?4{ya#Deo;bML(qu-{2|0X|DRtvD=_0ivc}ypW#{Ui= z!1s_`T8=GBhsyIPhwf0FZ%KDT&jJxMb&Eqb_B*zr=Ub|{lQw#gEYH53tdJIVG4`Tn z-eRZMRh9*Lw{7_}P1m0@nN%cFHrB|ed*KBVt-oGfzO8r&QnlrD!j*_w&{R57ZcRkN zDuV$H2-ee${%TorvY@6Gr5Hw%g9?K38kDdc9Ung?SPdHT{58=6etNKLb^Pr$P}h_Z zj46lSsDfe8+~Qd+#mKBqDmjpY&`y zmU)TQe(Kq^LhXjXoLlgsX`ZeY>b6IgW+Z`2xm`@edHxY%;M6n-eT!Z3^sX~^E2Z-n z&OXfB%_}xigbb((Vqo&5h}F)mKj!&1zjN$}`pY6c%Y65ZZr5&D5+r|X6lYSHe)l(B z3%p%&BCe`+eo=#=+VujS`FPeOrejh`kFcy`N3VS{q|;UgtWH<75HUN~9*wI6fKvW* z@>hU1!YFZ{<>pI|E59FY`BAXR#ej|^_FFGBy;iG+O~SoTj=X94ZYozzSK3j6o~T7P z^UiQeR+;7(TIpXGzvN%PpmmN<=$BNwe4yM|wR7}RrmvvtROati{|kY;lb(z+US)Ui zjYvg7_M<~T1eo^vi$iy4?eDw@POV_D5$^gX4$-=eYz=;8_Q5xfEcWBK{~;syIS~L< zJTMxkJUVI(h?nAD3}pLp)5dN7AxqR*xRcfV(bu5-WUueuT$ylFhyOk8XUwN&ZT_kN z@7Q%wsiCwj=5`Z=eb(*2`j-kep1*>2p+=)iSNe$2pSlv2@E%tAlS_R0e9M_nSy*)_ zMiG>8#hgYEpu;(iM3tgT;PxwYFahb@?E47puEc|>7%+0npdstRHHu!)uu_OmL9AJA6ELV z5yh%P6=&+?rY$?#()L~rUfXj+X)CHDp;L|WJP7d%1DPQ2hEjlZ#ys4*Z43nF$rW5b zWs*QwmVA>e2Auia_=6@;`RC(+dF*fAeRFS6Bk4ZkL-HkN3KmIpGpf7!ye~+g9FWnL z^Dqa>rctq<82b8flTgWMja-<2`8ASQSqEwUv-ojCb73z%GZIx?Xc_J0<=55T zuq6HiPlA{NsqpT#B>F$S0L03B+R9DXYW!ow&fOPZg>okfx2eBgz5$Z*DCjTlvc&}} z;JH~X*O|Y*9BMQB#X6lZ?;5znX&URS2W^nv?fBw-{qwJAkG(u!o&w(7tp(-gVAaRxEaz2g$%8iL zJLemAr$h;xbE>G=TS1wa3W3O0b-8jO*v7TSbB$HRrV9<4$E3Cg?NvkTJVD{|^=-k< z0B-$&E-?A$zi>S3V19>h%d1!hUT9^13)~f!D!PzQ2gX1 zQV_CUh?Rgd-MT9p7V=~C zP!Zhc%N;&)KU^N5`#3J~RwN1*H)*`P1S4Xo#J?FeONuK*#?*Qry#Yv)(N2_@Y5@tl z^(OcsQqPjfh=6s86VC>1k-4DD3AM3Wy>R1&*H-zP3>76PP(BBy;9J|xV zVq0qbmI6=%S`Ezo!>l_`uCM53qwo?h8PkJ~&gxxo214ALspw0_)vvx#3rVoAsq;mGzRaAvACh;J9(6<5FK1l!yzwZdf+n8a z>Pwz_SBg-Ue=i^&SZ#g(JW}ial3$m-9yD9Qewe>|=P?`s=S_}gsdp!u8T%Dt~T zMx3CZ_hpBqx5&2`ciV!rtV!63dt`{5X79r9VP}cn+WEI1VlGtIw<~ zU*)ko4pk*asBf(wgN~v8B_33v2h)(=6uKFq!H8#=v>pN&z4;j z8WuFG$JlgpWEOVk)Uz~3M?$QAe95=}oU4+z&u>sd<{U)`AADC~;i#~m zs+SrnHD|lqq%D6;>wmCS-UM2bt~ZIBeeU7_=ABGU3co;pzQ@yGtjo}qibm#f7zXvh zX1&y4!sKE6zYS0`4;*asvC>%AOA)3t@)n+dKYXP*EE&mG2*EY!3_b)M;QRw{(GIcNYJeB^y#F2ZI1f`Q|^^*JfGydI93wKZ_jOsqvS8V}n`XayR` z+O7*V%GclZ=ual>`uyUNJYGj9MGWXffuIB9!y^fzre}-9v~)M!+KJKf@T|nBMMO$ za4gLk5|rFv30PPf4_$wM)8djt$hX;iRH4mHdW=wr@{4@*Yk~=|lW(Rmb2^3=4VIYJ zaZCz0%bm7b+t`G| z$~1$BM(?XZ75h_dpu<+!;g@n^F|S?qn^5ip@fEuXj02AXo&a^l(q%GLG5K~I$hApQLWDNvLB;>&@|wbKkMERSF2^a;P(h$(_l z<71V4_kZmzuDmN8e)f1-Qy94_V`&NDjBak=ZM?Q?cH95Wa^`x9^}nna&VV_(Oj z4*kY!%H|MCy7sGAMQfhr{8>#y8?pC$Ektr_*j7EX4p}p_5T2R(-qhMH(1SreRtfrq zn6~Stj_J{(akqu8q!5fIsyThfj%FsRJFE%M2CDNZnTh?y&c==$@a8({>X_{FW7lDy zdia#-(i(ePfeq6ww^DgaO2ug$B=hkU@NN|xOOuw3=fObY1Al&$OW%9il= zz+YWAor=@Ixa_T4CJ`uTMh|?Z?4}mJKOy~THv74!jo4NX5)xh9xP-4{i3g}zV^7U zOw`hNZz|tt+ytj&HP;ro<3+=BV}(vJwEK7G%;rn~+3`Ss=M-S?$oZ^`|SM)tS+b+~dbr>WNlr&63)U*N8JP$Bt);@@Br z)I`bg(h)h2kL)gbS8-gz%DD2Ex@N}W&I;uOp9u705TMjbDA^5h)QUfN_M+;9eIiS% zPV!-IAu-E@FM@682+nsazpX^x!{^b(#)W!`)jsZ}6A|=4HIIbWZk#Uwkdv~27NAxT z4A|GNAbD+QmZ>_co99qH5&jVZ!HAAHm80Gmw~i>i zcB4d{*>3o;8I3FY`KuXlPgrQn{{UiGG%@ksw&_@PeN*7b3tXDREvSS0;S<5LApDTH zNOp(j{&Z8gUEdKuz+k#xE~I_}?&7}scPu4Us+=WZ_|EyL_`4cb2OM0QxoM$jiIqNX z1-cC8NBJ;WwDZj`lTIffQ{bf1%cpMO|Iclr-+vHfLyNsgY6bRP z8MXl^(KT(#=Kz2s1xy#+QG&KSG9?fwb8J%*6YI{AJO{p2NzLh#CJh~?S)oErjgPqe zm55ZA@8D~_TXNTjB0zh14!D#d^{S^J4g@^{<5>4D>?`-#KPUEJm~8UE z<9AM1y*%xMk*o%^n=m>6!YS`hRJng0mxH(rs1%ta!xZ}}Cc<9pGmIg%z%iP6FVYPP z8kCr5z>xH}RR4-JbYDAbDux2e`UvHhmzrt8piS@E=5n0;Nm&Z2KNOC084*8uv@>t7 zs`ToB#$zr+gt@3~FXow)`|Dl6Jp)8TVnAfRy&4h{vT8(_8+^OcRtq$8Ub1cmm!CWV z$XV^9ipd7J5#<~`huHlIMsN_1#WE9_u_J8@{+6KQu`QR{O~!k(Eps-rzQ*OUlVv>;c>jDW+Q|Q&*1dZjYCz~+^}Z*RBD`MO z(3waF5>G!qrMU7_Z#SvTkAniYy^t*^gdFk)A7S=vQdt%3Z@VzqUy5H~qHQclSrj1i z9tI_Uk$?S4SreHHz3eS*p817gfRyS}#?=TM8=X;ctk|$z(KH$$uk97E)kR}F-#;t) zOO5>;EZhLkTlnEA!gA?T#w#<~qX+0jUaS+T4^5CFz_o*12BC1r;gYHpBS!gv`W~?v zdyoof+yA`;!ag*VfCRPn9%NN?a}>kW{6Qd4IKuQ1FkP;95L`>Rh=^}TX=+F$UNrh5 zsQG>1WgNAv~T$tnj3?0DYdY8}N zMgFJFZo7cjbi(djjn8Y!{s1BJBI;;MMdqNj0Jr6gPRL$@>xOfN+0M8v$ zHxq1OuVSE93JMC?A`+jvz|+Q_e&od%5-OW#;I@d-N@@G~ zpOsoTsLDUk0x}bi@rfvq;0A-JI5^i9cexrN>@pRm9x$MzUb@hw3tI*sR}E=b@YVxt z*5k=H&%P+rmP*AAy;FJu+H_0JwkzJ2;&ks;2h0uLGCk}AobHuz%QRQ)o21`f zS(GV^Af8=u?A~Q{BO)qvo@UxjVsbOz)hpYXiGkQ&l2%lthpXB1~JQuUU(Gs_8P}2M*FcL9mzZ>$E^E9wKK!4 ze#mXNEFXhV?q=#`t6zP#CUgf2q_FUM~t^L^w_)Zgyg2D;^Z?vCIitWlR08L_LnOgwLiTO2ISGY?; zf6<;z``evHzglV@wSBxuy$|1zqZG>7uHXCFeVyD=g@}INe1V$h3$QPbX4~Ifyv4+6 z!@?wA%|s=3!|l6}72LmwV@EAry!al+(K&G^3}Bm#@2j)SCBVtWUd+AKA5tiBo^G5J z?8jzB)A9uaLbK)CRD%a$Y#J+2pBK~T*1WS6VQWSLJ{tc@@B?>CtbhuI_d)iq@a{+Y zU5B$t^5E&pqFdZ1Pqucw`CLy}8fWx2gn+BV+|2x%`t&$I|jA-w7T*;Rxlk z^A{2{i9q5ts!C0OHV8@H53QcN7l%FBF)5t(NoUVW~7=y=2gVy&^ zpy;l0BICL73cz60a|zWT;;uW-_Yb!eS7VlK9MYAs|3YK!Q?N#&bPKXB5+4od^CGkT@y>T{@0O=OlRF z@4|z4a^zCVkZ9x9g`;O>+uLx%0QyVM9e3_K5x%OVMErP zjC7A|Mjg34n?KtB&7a;30ZKn$dvv&YoG;T`)4w!DdC;1mtT z;ODjB5-03N$TrIEJ!Ee1_aW7>ZXXiL$mAUB{>_850@QyTzpvhzQ zoH1NJUgp${`7i3hKTA~>S3oz6p>sPExTXM|fgY8YoL3-!pik{0ALL)RljD$KdvC`4 z?WJq6CQ=jf#Mfd(@haE)Tq*i1@tvSLtZ3Rs%8ixV<;)pi|6G3)0_cJn85@9cMhBZ= zW`6AZIrxD06f{Z0?Hoz9x1&a)%*{>&{`LyT;& zdx1H8b#!PO1N!@MzdsD_T1y1Vv8r)_rH}uj6aBXy48AEl2TF)V)pIc^phA12d4=uS zMs~P_0C7zU`fE{%UWvHi(Hc}$>zFmE(7Y0v&oSQ1b^@t0klqe5ZPhyW3R}34kz)nt1er2Ga*FHv%?%KU2_<0ia6VDK4 z!GxCn#Wp2-VRJ40ax89Z+lM$23_x-ipLhN*H~BgW$Lg^k%^`RP^K@OCVfk2m_PAFJ zcPWO*njbM1O)~Cw>bmQ_;m~K zBLGcL_->2g8Mouq%-DOI>FU2E3rJa;CTKNHWhy5C>J&#Ud`M&`^J?QWbDSj?JcxDW!;=*s&W))G2{ayr5Pwf(%FfeO@xl-IPumsyBa%df*V;KVO(M zAnlh$tw_PYvT^;9_W9=$tnW1tP_3V0?I05UT>L;n>;cc{vV$$R#m@0)+e$d(`+XXA z3}Z`TN7I9jG>jq-CB>a4Vt#JF&bA!zDe-*c zavdyVeNOb@zt#ZO;}QgxA&9b~0dGum{gZ|< z6FC*w7$@G;J!cs}D={`A29Pu z6_D zcfyu@XuEY68t;&f=rTKWm{(-|LXfMK7gkraA^}=S_L|27Q02r4FmZ5xK<_D={OHtR zCGyukz@<)U5jKG3}XXFUsMhFbTkJy(sMP>Vg~vzx478BBk=XsK1lq!5>xK>+>V6^|ICg}?Lex8A|HH(K0b23`qtfeHvSt% zHTE~5pR!YH3diAWC5>N2_*;wL17GwzDvea(&%r5`?3M?k*Cs0{?wfbLUDlI~d?d49 zW&Ey$>%{!zIUlg5nOD<7laG<^ETy<3?N1^!_?ruoJg_)}HME!C2H zH?!~G(n;)lOZNji^;tODLO_8yZeZ zrL1{2qChHQbNll0@|Q@xY#V^h^x98_v#(eD&ccprNVpAl*2qXm;2%mrUU`XBk8ylF z4g&I=VHqN>P!#CNPT_$=ss*H_au(q7o>a3tU7!BnBAT1EYV8>J`xExC%YfclMUG0( z0E5#ZI#dWjK01d^CPQ3>g@5-%iSfzkv^KXna&LlY)(piX7j8>5ccwcPp7banhY(P8 zd~bdo?@V_t+Ph~QAhfjUAkG`&VpnEMwXKX)jkJVr zKUqLu>%a}X68#h0-!1iWzK|j!e9^4yi>d##b}kxYciNVK3%p2h&fB_tBc0eOcbFxC z^IKi)Blr8UZC>YxWRlckj|X4KKXtC{7%CepG?W4-UXiR%S+g3Pt;TTJV)CV%Q_*-F z*KaY%M}42!X*-I~*3Hdj9l<|T#XrZ{v7JlT$~j1)EX$bwg~N>@Rkq6}@_IuB^M zx%)H5)0NH&C@Ld`Y{(g&o>S+277Fh}Muz8p`4CVDfx^Q-<$Rq@Q|&q1ZI%j6${9+J z_{ovvHVfji4lFtNqV&M%!A{>mZ0`WH&oub$kN2h!=>wOZ{sOy7<`a&czLlxLo(8e_ z_n)WJ@{kg5MBZ!?dwDdp0h#Tt;XDjSNb4;j5MA2$@x(txmoL*4B^emL%JsZEl4rc$ z1`pnFDex@)NlF`2y71q<0EugmtjA`cqr)~(*tjpEo^;%kh7gUR&eo5K_L)G4UNON7I@7Gws^v1q&QJKRqCOBP0aR!<0g4{C&xY%#G*21|9Pe z%SwmtxkLMQA6E;|o!ucP4?`;@F+TnzNO0b9c{Ed$RU-GI`xDQzMyU&2ljEg{@oMMT z=jI)Gxdl59)NjEgz_j_7+NF*b(evCI2$4KJ_ey2CK|ekG;BkLKZZ6ddGb>c0Hc z6M?t-%!*sXEFm%IInT0MTFS9OobHqL?Sf7G(klm{glNbUQ+zlg)!zGsrr1SDnL7$i zvd@_ALXbm>uOo5wO$F# zb}W0_u^M9ZHZ^5oMklx;>QfQZLNt(+OM>&CM74LY8Z&SQGS~x7iAnO*w`>^6EoL)B zgq$}?Ett1qh0O*7sZx$d4NQUk51iXPM+d)je3H3K`L>2P;QNL9CReddYXx&FK2LDy zeO_#A>{hKL?{|!i>~BhxK?brrROI-nL%MWr2akXlQw;`7Ig00gok}<^Z3dG)lnrP* z+O~U7Eqd291fDF7OE{YY+HyQW3;wc_vz<}iO@b;hS??=*$O^n0Hc@|(_dbGYApiqO z`t5F#ucd%gqYsvm|e%ppHA)2#^OHpme7zs z{1f;^ihn|7XOxb7cH{L!PNt9{BfN90v+&zZ%i*zmL+`~V@ZJhLZTm(T2xq%QgtP!Q znpAIWM)gylxopfYH^5O1#=vV8E=KIkw_j4*)&?3wJoXJ>c(4_YPdz7@9vmT^Hy-<> zS!yOd^1&jKRSJA`6H(y;{3kfG7N~-ugdYPo>3)P8x35sW8p$rNIkqiu<$1op`*3nX2qfhdJzX za7v=uEEHit3QBGO8Ws#iAHy&~v0hcXpYwH;gW)pK1oI&#i`=&34@XuO!7O=+F1+EP zszz!U)$9>f*o|7e?2}BcxJ*>G=k2)EvT3aI6BG>LQdzbMpXTNcsm;}6P;GqbeU~<( z^Rl(eqBiL1Vv5kcg8=E%+{UaH&qDkMB`BUntr?8C%<7ln5t~{Hujti= zZ%kkzQupmcVGki;B8~w}(*38QpQZ0})?IdXW$VMX9~Q(`I`Ll+RcaE_*+U}WmB6Ea zgSn558mz(AwQ^rXUg9jnw>WC|AwD3;@v{tcBNZZ@Qiv0OJkS2MK|Q!qcj;K?@gYVo zW?Y+$QF*&s1BHtTfR+=9(9#9Aj472o08m}BfS<+jT6^k`PTWDpcCa$6l)1bCb! z++5IQ^R#kAm$+3upBv4@ZsA1zpe-cUqLnPNYy2KU4SD4%RdUDcc+VU-;)I%t*X>x* z zooMnStGQYSso~Eav|!#}@LLMXtdhG928)4fcEKdg6P-AEdGe7>S7LElRDuY_uPqSIg?DGELk7iUg)3b=J)3Vg5tN}fF=?9B}cku za!*#l$b4X$&8f`|A-)S~gT#947zLf8KEQG}uU=<(?Ce|?~8!}ARgoYfs@~6>UPrc z7Nr@rRzgHLzJ&Jnqw?0f;rv;mrp=$%(^PXT8M7ns{>R#$$>ynEIySoLl0* zRKD3)Y`=Sn?3@f)9=vnue9Wbh^vEbBG^tDfbdydr8}7HjK}OtgdX7D!C2ak60}d)C zWFF2t8IzVLA%^LMBlR>o6fd8ACTJFe)mOlqgkJQr;yu&t>h|e3yF-ftZW}?MRbsUb zVpe3ggPbO<+PBiF_{k&PDhFj(R~3?FvL9w zuhUkqhm9o9oF@Ok_}~RxD3wqpCW-t*ECPlu$&W%$>C;!QvPVKrUpK|tR+F{eA`7@l zuklXSj7~0Fdzqt7wV_O5HAz(J(2V*$wP3`QfZgI?P?z@Q!E?`QlzEY}*q;vjJizS| zd}O_g6mD=Ct5xd?|3TqCqXIj}-3|zT+Va47ZF_i6EwY>F{9^I1 zR`ZRxPhTRV2qKs=$SUk6Q#gyJE71`wm3^W32Ko-Vyj*M6z})gMLEvn^>}r8cfIH14 zG;;i?Kf`u=I^fw?65bk2p+IyepC_(GN5K^@21U~5fNUUlSPki3Me=VVtFvEOdOwOi zM;khqZ~u*MQki!960pa;UVq>n$Je)d=H@Ly{(s^r(#a}mW!e{N= z#8AD*Y5eg`pkG7)NYx4C;iZ*Kg8C0+Tclw^1+fU0SWaDyAX?>Q{0^kKBRS+hsRgBT zu3T<7PpgI|7jo#lvE8njxUdN6Mi^5LJ)QY{hufn5y!d{u0^POl1sEntRL&ZnFq2LOAx3Y8sy za;W8+od39!@D+>Tt67CTrS;$8_|d%={9!TdxwtJapL~4-Lg^co!j^AEAa=!`BeKVt z!ebYPq>MwEpcXrsDzU>a_w|w4&E#LN^jeUB4wLtw0PC8O*H_MSArY$)hzX7p;3Et0 ztx5Q=Qd8wXcKwK`48`Z?3l>=Hz`CGZ4=NC~lgBTS{E@mbiu;cz(``{;l9NwPZp78c zq!GRrCDEa@)@@%Wz20-*gr*5d#Ph4_Bj2Ri!?`#jJ$!I?{~z_z-=7~6B3xclL&IYj zxm)Q%{J;zjpdM>h6f_E>ksfz-u}des@i-6@V$=|(@S}wq474PCdlDyeo3rESsg*!l zt1Oi|g)E>8v>q$aRu)k$(ixeAK0i&Lx2-Rw7V!v#)gGIUjZTkOm{2eb`Mo^DUT&`& zX`DO#K=jaOjTD6lBX!5qIia^mOrKKF3^jK+pU_Gyjfn17y*G(nov!Y??I)o$aBpEt z@$~9t<06%cmW`S^uI;a8(*2fWtDPUQyZE`B$pb!#MDAlew>mC-TtQNjVpr+&Sxm9_ z29n?wB&$uBS>>UaVFw)Lb$uN6Q%ixM7p$N?;aOXX?)pY9tpBEzYQ1f_oMQqYNwjRx zVQYcWN{0B=w|04X&0IWq(MjkXw%9o``*{EMPZsIOk8vVttq^Lfr~BbYJ?qU8ngjrh zxWJee;J=Y?pSp;PL|yU^y=kAd{w$ktjMh zT9Y~`W0Z|T+~W-~TPxj5>VhS?9$(qL9zvJ6wx^~A)QGRMVQN~(H+??*8`UC;zB5(M6^wl$Gr zEe7hBN?+g5dN-(z0e-L9WAf0XOnCat$5!j7k@zDi{i~CM^K|M zd6<~c@V&6$nG5Nu%^0hOrT8Vm)KR-q`jhDUgHGr7QD&9N-nd$}i7FN|NuO2?wVy8J z?FKOQNHY)YCfKWnJfM|KymT*it5+Z+Yd$1nO?L|TWN#1bg!=jR)vPF}1~3e2CzYR5 zTE!Hpj^P~Q5tcFA9;l8=J-CBCJ=EfUtXr=1aqrVMb#gMat=V#xdU@f0!U52z35z~%P!%s1`W zAi)a(*H6n~)hgd40kd!Dq@(OpDzlJInBZN=m+6Oh0Q-eMi*sD?8)6K=!bnb=D0HMZ znVFI#wqmAuabKbF*@MKF%e=p1(hfuM;``+)r_Ea}i7Bc{LLXc1oFbT0{`}K_jWC{% z;+CqVKk=iw=PLb>H0H!yDK3b%S?YEo4#pV0TFevMJ9HgYN30_ul>J@v&=zOx=y8eC zEak*h9oS0b6BIy7>n~bf_K0QH%9yL}!b8J3l4DP<#(PhoqQbo0khh4$aM1V+v`c(p zlMPJ4-4V>*unmWmY?%M5alp2rs8kbsf4=9bUZ^TKIIPz8q`AYc5s!LE6PKm}&Q~yW z>%wJQs#;;0dp>KAAefBr)orGCaruRUM06cT8p2u&KPb5#uL*$6UL#o28gvSDd}c{g z0}7T5*&T1rzH@u;hNDj9pnF$#`CnI_=zEid-{;x|n)Bi}zrj>KWC}5ZYF&sM3pqmE?=MDLc_DrL&X&SETrR6 zr^bCb8~c}BpmfD6Yu3Nq?Qi%R4b15h;kn1299~~Y`5n#v`bi^-sjU!NycElbNHoH% zE;zRbQWQO-k;R|jy4IGh=i+{x&D|&V`7EeuVANIxg>jfFq5 zs3Dn?oyf3L!Ns{%lG&<#@s8 z;SA?0l~$KT{fWZ89Q!jY{(0>SM$iV5X{d;f{F8xZg`M92JJF$1Bw-}x74@pJ-y?T6 zrY|_>PU`_OjwE?WSsY)(#=05`M@s<=K0Y-M>VCG#zJ0PY9rIm2{-qTNM23w)OmrMfOwawZHtkh zEPz#!j`9Igrm&H1&zLP=U?&&91aA@%6N3zM4g5?u9OSq*O20KsP=_HlG3ndAB{xtx zntqXfT> z+5fb^&Jg>^sqrORIPrH-qobcrn3O2RB#~~S;kQ2&AGT1RnyFBX=S0->jvQxGy!(sg zTeWAGqw2*=NJGtf^gi>SG8C7Ew; zFZX1`$IzVQRhw?dI{m6PyC-Tp-YFWKpMd#*l7p4je^lwdDxy2ia8$DGLkTLVhiX3k z3a-4I&m?#beMQ!B1`c zOylILU940QYPPPC8%BVmDj}o7ORS%-dPoxRE=Zb4kymr{PA$(jHg@9S%Yrw|Q85a%~vLw z#Si^4dFI8;&K}Fhw7K8UB+T7S`9QS)QgOqh8*ZNGFiotX{XpY-IPZ71VwVG~da(mE zmgI>uP$U$Y;+bi2zl#`?ay={M`Ui~->V4Kwz+;0S@}`5o{IvzFuICC!`XTm&~KKs zRJn+{#T?qvMV!f1eIB)Y@5m-)LUfr-2xWfP$6Ni>%jAmzo;A-*1QI5PEri=9Dz5w$ z@esD&8TuN4x#y>O^dpx{Znd zhpRfB3z%i&E|k7fJn6n^SRAhnC48#;s^`#!O}ox`&(@%7zm2U?LsB2+A^KKlmDi-B!o_xSi7L?TP0U zSsF}qyY)nyl=iCA>#5I4>-|2R&-3rK;HsULg!ef%+S z^Pn_sxPotxI(an?huK) zEV^h+lHSAHEdSC1pbDKt9aS`vm?qPc&T)LEF~l6&`E5_hb3EUBk5qWavwACz7V6|d z8)1femDWeZt$O7a%iho7TSVxd+o6}=JvI2P&jy#{#5@}+)dEqF%Pq!q4NkUmzpe-_ z^S4IOYCnYd`Fy=ftFWF)jVaLcSaqzlh;>sIhI%&RVVAuaAS{11@+_fdOO1(?V4_I{ ze}0SBoaL&}tM4rS$|tSdYSeu`)b_&l1O+R)I^Sn=bql)ks{NR1qr}eKd1ElA8|A8S zd7@sdJD~`ab=_ozcrAZ1dnpb?(AkaogV%_?d1oA);S>y4&R!}{RoILni+H+a4p7#c zEWT`)%h##b?9pAyHZMw?r=n&Z8hcVTb`(_j1e=aq3<_%$T3I!E`*>Pd7jNb$SwJAd zy`g^!E%?kG3XL?{vG?|I8Fq+b5RPBhmWK*Xw%M0HRw^z{^;a&p9hUd5N!;>qw4lBo zq!QRPELdc*UUSyji3Oi;^E2)AX+ilva!^3K&mTIsL4j|F-yETeN%erhhPt$*#H?0= z;i6=bk>BQ*c8(=`cR#esC)$I_>7Y;|@H5tYN*S@ecM6+Y0A^u>Q+J=4EK~HsfpAg% z;_Q_8) zPz&%8VV_xWyLJ`HSZHk!K3(CkN*1;}k9GYV2HR9-&1vVn+P9!l|O$m7#8RlN1+I11wQ z$F!}F0IP1N*El%%1zp|v1kdt#SG|-gwyvxN`&mE|G|C{|baW<2oAAV%wgM+QfvQ~o z8SFc8LfznFrH**UoGG4*q=GhQJ%%DWg~tfTCil5%vWM*u-c6G3`*Jl&e_X@_O${+* z2(i?^euz(kd71Xsa*GCQuk(djY;Yw4KkZ+nz8)i~!KAA?BIgmiXZz@RLQ;5mxbrYF z>_t1<1Se(%1&WpjR+XrjaJG+sx&Wg$YtH+CZMI7J$|t0nr0Vu-a2wnqp8~uC#Zn8z z6!FLblvxhglLxV0FVWvGuM*?-yt*ay{(l#N_*5(ls39WiX&z+u4eKzQ9ccY(PFb#iqA28f+O~blMDZ6yXWeY7=^uN^CO;?fA}K zWK5KS;Y#=&ul(N3!WAhh3RROC>=#1S9ydl0kUISlF^u1%pc1%A0LnR2G~l%8LL))h zfQn0YZpXiIGe=d~9r7&!-&z*6@bC`8I8@+d@)c2#jC4;bYonm2HD)cvKP1qm4rigR zmbyjJNpVVgh?%T!iL~o2zKEXgFrwYozE9`sH1+mIa_irc`fIzQ3Bi1E7Eas8NPbR) zzjr{W-sX_;uHPQlyD?`-!wlO3^!-f1M8Ah` zCi^yt0KKoc(9Nobei?k)gc<&G!n~$oysV9dQkT18UhM&uR{c=)AG){u&g6j7PDpeT z$|K9$FmN~*d*9)=SUOD!Va7iNp-@dv*DGLnGFNVQcc_9&Pd#;^73($i z)uaDMXLYf_jMpxc3^gzU^PTS|wgDfm`+154|0#oS{BJk%U^s zSvpkxq$Dbe{2L4~OIx&k~3q->&a5WNy#oxYh;dY30z|a^@65UzS5(! zy2|`Pi?iw=3=Ny@;oyHO>l{K^eJJ@R!7#m>1iz&q8T>`ki*Xri+atCM(V4AdpL3a z2vvB9ico`snX>K(v#P_)Uo9bv%we+NfBuPF?n{HI%)z&Kr=^knrhn^UlU{rdN}uRn zzHKesrRYN7mrp=%egCLV^?iAS7`}b0k{T?JdMtmQW@-~&BUo9-@1?`7J-8oQ7%*I- z+Mb=X`1m?jE@879our8-J}~qXAKfRo_uex4k~|0bpHBzMPlbv}m)^=eh=Gb93VuJr ziSgjD;hpIlci%8|n%zF}tgpH}@8Ub;`bq{ppnaEf26U0fnbjSyp2;FPU6JwZm?)S2 z-jdGeg$QCC;k6FiEC%K9sO#5c<=7qA(kg!rL-E|Hx%|~4%w~Xu_fOA-(ARV@#`!I! zi_kkH6sEazNfHUvnV}~{ZcB->hr1&0-KOu}-@-ee>Vb6eqMYlo&YfRw^^2Ryvqs&H zbSg{7qqNk^h}%D@?8~cl_DooQR&3bl-KU}idwTnuZ_>qqwA zy#UO`IhyazIf_K#i9J7a-aPw5E`v{jTx#Ai!z7UWvVY43|48MvDLJNeF#hYl?vVt% z3EKnxFTWR5HD1lwr=ZK2^F?YqEQf`@eRVq1x6p2)I4xq0pE>is8zy4cX}+OQ_uVa8 z-h_PeuMvT$%;UV_NErMliUq1Mr~@7>^q$jO3dv6d>)JN`NFJ)8(n(>;-t|s@EM_h{ z?Pykl@P|^S2Vo4t?a3;G+#!>9)YP!r5@wleb$hBM(Bkh7u2)*lI?aCWoTt6khVPi{ z`mp+j&sP!K!NoNm1;(pZ@7eT|K1Z>j{h<*=KqFqXFb~OsY2sSHHKMNS3yq4)y}*yo zt5TH0UvDlpREpM!ia`rt8iyykQPvXK!Rd>P^=dLlOyzNUhJ6&o>;u_Cd>fmmH$s@% zjI)&Of172wxO}m+oiap0ufk59J9}OLhn3P`#3otv-~I_1f{ib|dvZG90hChe_Yxey zT;p>|n)`vfV+2|oEsC6nGPM>j_R)Y>_{fE?fg{E6F1qZ=v(4(aZqFzn|@w3kGJY-02(!Bfph? z?|<{pvEavy;>NA#IOnScCmR{&hWiQ>M_bHCA}bxJpPmq`(;?I{^q`S3fp(ooUW&>> zO)}15Gq8?cF`Y$o-1qKkwrax5n=G-C*TPOE(ym;-S1+(A2wh>%fxfE_qLw*faJ}B5 ze^|$EI1;c0bP&(9k*`Dfi>ii(Oq(SP90UvB#wVpP1$J9A)19KY8QNnAvh5Q#wD zT)w-bd3Nr1SCXWt>7BeU3xT-7frf;n$Lh zHaFqME}Aa8pJ9DBd9$Y(eq5>}kW_kBQ*eP5zP8u~m##hjazjeUq?|OB3oDuDr0wRV~nr9#&*Tm`Wz5HP^Y0+RJxCH5_8{ zSfU9@!Y}1A4-lS@Ltk$gSFGy|~XT!*YBfBDt1&Pnw}Z+Y)HzE^o^ zHAy>4i?41Xz$ZEib+3nxn2mP6xOmd-t1@r#W;9cXQh@CM+8^qA8@LX+JtP$ou%Al5 zT}O_mK(?TZUu=@+1U7BH0U}A_9V+e$S^tW?#q7z6nhF-~6VY(*X5)i0iWW)v5INm1 zyX2G{#d9OrdgkV1r!=JO>yw!dBu6{1qtZAsbifArHlm6qTm;ADC*+CKSSdN|9mCPG z9(@}&5=k!q1ML+8q+{>=;uQ%P-fHlBMe-1|Y#Kil;zw?$i6KY7KK6NLB)%SHA$O5U z6xtf1^($9V&%{Kijy1~cpqZ~R-0X^sHyeI|xp27=8qhHLE!yj@SK zu4my#lH{XmO6{$H?W3KnWFYKbOY_yMtw|xa^#*1j0)DCSK1-R#Uy0rcF6Ninma{0Q zH>$0&6Q6*h1vGD5@NT9U5j~b$cT*8W!93NP>!O|o-8nitG%D%hdpu|N#Q1+S)~780 zu{)fI-EmNV1?*k@&2Jo-yTq3q1<+IC;kUL3@;qfGL=G`G}>N5H?H} z?DH{mas8UPw-6KuR%Wu-yGW|;B_vF{JNUnLbp44z5g1d+3lvn`1LEzd zF4nX2+rGx_-1&O7N;*0^hDavkcUsl!J)GNh6&)|xfye!dGSzR38LL<$hIS;Zv)r*K z<%wES(vB9uXh1PyktroaRfJwX@!`kM=2TT+O|SiM^v2f0&+M$GW0G!TzUW~YjTpjK z^0}P)B=8pfk42>y0Dx9Tr7HyRv*&fbMn!PU?|)g(iZ(-f65JNyq3|GcLZ@Z$gDmwt z8orPBT2WpJ*>E&1B5BchbPMcIrlOGgU6hdoKrrg%P5v6Q>zim$jBv%U8&R1tpKd4U*mn{s=G9OQBfjyE(OJn}VN@F8veilYb2N5Q2tL1MD9bd<-uC<66B-6HrRP{fajLf`VR`=aWoO$J_ zy9En1-l_)8AA#fP%n{Yax@;!@Rek0slr04@Doc2baCSe5PhU1;#8nQKSU(f@$gP?= zo=1~?4-=d?dM5g>)qwAUgc9Rl31ub`l~MWYXcF;a1APDHWO821Nv4+FRzTNbNEdMo#`oe51 z%k2eQyZ~WaCi~)dj3DAzd+_hIh+86XeOrPqfp`=0?^yM4H|m8kQCN8R(ZUdyGjuOVMI;57W)pm>Z)C@zTzRSi3PRC~fxAJcj;aAo_75a_y-NZvTELQ!$5L?^;fpo75~Jqx^yM@2Il5i;Ylv} zVXeL$Mqq%?;`XcFAn{Iw-~M>(qfrfZ3#)v)LR`$!`C!9oeETX+7Z$25HtYDX&{V1k zhO@8kB zk50Dhs<&Ji0;qbLqNV-ffhQr13|7EOk{wERAu6{S)BsATzF5N%r7bLj$OAk{v}@gi zVv4R*!^gS@00wQrL(Q-&NpF9=L@Zi&@~r9)2!Z4`48XYDJK=IVU*8+Qchr84anFHh z>l6)}=JI5StKQIQZMYemS{t*uVO6rp40&vIm0^6Eou#qtmtWM3oFvt49ls_JsPI%Z zM|Gmm*+{J^Y6?7Dp7Q0*J{FwOvhu%KCjmm06EgPQ{K+cyJbumrWs2BWvU?Tw3CL(M zs5d;67#X=%@W>Bj1}UV_{`4^4S-{OFCGoo;`dqbWGU^nLA2zZW0(0FP-^G<5i;1!D3-m{gRd$HRuR1GFX~$4NF=tIqV{i`l z;>GWH2Gf0~Ui6ZFq!gmqSQKMmFjA76r|W_f)KKYJjSwgiCc2`!dWn?GkK=NYOIW7O zX4(4cU=jFdZ`FCGzE;2V2G$eJ7{j|f_J?|*e}HP5xqHa7=mduD^LM;=dtZ28^i=9r zdu0IE4O;?bMmcQ1tu1Rly9s>^N+;j*tBNEgkqdRBqb^XrX+$UZoaB9!g#7NQ8uE(y z?5!;^1_PbRNeYJPo%k4YajEM9`QUMyf&z-Hdvl=igs~(#e!mPVezu37V6*XVt2oVO zYkt1vM?1>XUH@D=6uv(f7fFgdi$-AWx0zB)Jqr9GA-y;YkQBF{rj73UV57l8yf}(P zU?5Jt7XE?ba}gFCKWQv(v3t)Rn3##stoLPPz@84y5^3H?W0XlYmNZOYN2@_s^q)qV z*kDDgVI?gqwcS9=I7iM&^yWg_kZ-n>w?G9snlCllmKqTSqs=pMiaWzh7oyA@w@`V7 zweto2@|(5@(N=FS5!>#WVp#kU2$4{hZ=SM-+E3wBZcel(9i0zkDO2N+-Itc7_AUb> zh0`rZPOpyk!st~-P-J9e;=~{gOi$y|vl2ZN0FlB`oc)f>>PGNXVbR8ldX1<*W;9dY zF7nFicb%P_bTAEiEn3TPuVJgPzTbK4Tl>Z@$G)QmOK^Xj?_COU7TV{egu8dGelg3p zD(Jaar(Y!4@!&}R+vNXK4rHaBi>HxXq8rOrP2;uJUb}MDdo%Nz4FpRKW578`F6vdK z6++#DiAq3tAO!Za?OWUC^zMDN_t!eKIhox!&bC1s;YHYzrC-4*;I?x8~}iKI4Q zo;%3`>?|Px7{sEl029UnYexE8B4Zz=iAaG+b>{W0LE^7m>p7pwgTwx)`u**S{TQ$x zQcGicPga0=yJ5eggnn|~1{t~o7A2To94F+QJ5);tsu6aPUy>FNO0!mb;#Y(4B#i)% z$t@C(g*W$tUB$-Dle`y--V)+7-!b+n+}CQZxLKp|tY|Q|Wu;M&2-Y5g1K`}1bt`R~DLI=>k9NRU4UU^8S@ z>+nUs(Nat81f3D7z;eU#tH0P>OUKyJDABAuPAiDEcai zG?K5TpgN^5a{yf*3r(s5Kz1_5$6FI6LF%#Z&1pjMyE@?ew+`#qucB)HVw zUj<9M0{~wFP6(dQQJ#^E7PBiYehAWmBctzK_#qSpv!lqF!UPr{*&bF><-~6QDI@dq1Y#mB_}}pD2P{9!(hMy z)R?@A0RzK*0Wi!~FC~q{;|Obe^zN+NE-%Ue&^lSRI|trcF?WBc=nb9^BDG|HR$K%S z=O4QNzXsV9g3^mYfanb{#X%1cUCQ{0`JzIc27ednCHvK=Jf=V99uD?_qfR5R{6JTc z)7@6Qu1kj`?_*WKY-3hSJ+2kYX|T|`YJs|N+t??9sQ9w|gl!sa{_UVxqY=5}DIH6> zD)WEQ`nh6I?hC;P$)ufgMJBFUplAjS5p}4eWFeP~4gOJwPMH`4fJ!&BPxPYu;{0^+ zy_qa#5a-m*CGZh11kV}+l@nE;5NP@7D}8lx$yXY>`yv^%feV3S(tdwN$G>tYUp$Z` zoEk^IqC@eyW1Pa)@PR%QKgozLslvdW-G29MHOceJ=X^a2I7Oc?tR9DJDMU&SCBWtQ zAyrwb3~)Q1spRITAaP0H=ZM&f!2^xScoDy$=UubwP&vg>G}K2)gZok40_&OY+geBfSxn z?mma`6IL9Wuhgo8Rf7ijFWU`-(EgDX{;f{eFD;75qYseN&oHaln@-kq2i*{)YR#)hll-A^}BxFR1V(>qI3K`3BZ7XGr^VlHC=KrwEQLLZZ78Dc|+A3}x zu64D88eLcdGM9aFnV*>IDZpSdzQu#GJ9(@qOx^-%{z{{U_wN6zs6b^0^$}0QAKqNCv^NaX#*^Q|7M$HOaM={qJQ0?Q~CY0r}!!RK7hjbqy_zhkr?c7 z$QI3cYy4Zc5(cq&M<>>HiQ$=tI|_2+P0)Mb{9RXG$Y4-KUPKX^XaF|Ul+KfLkR#N1KDDD*V(H{yrH73q z56qxmM#t5yw56ivRMtY6od@U2>RQ3E(<-FFt?9zi$JT^~hcQ_3+m&fcgh6r9aYo}b zF@b0?zCWxx=s#ZNUuF*ul|w+;>fx>%uLBAo4<34RFEJZ1`EUUzr=-cN&iumJ-1eQwcN1L`7dig5?5+F)Ft%e{6ii48z%Y&(~1 zQ0uZsyYKW1_@QX}7e0kwwPalHafkM6Mg`$Tu)_rQ^JDr4u!%*B6Y#74%RVEc5;6dD zl8C%bl+@(Dnvey7Z5kPZ3KiZMnfcdwssL`27F3b_4*JO|hqMy`y?d+il(H<)YLqo%d zPQ~`~Em#~I!#Qbyb}a|YB$&}!y8z6}T2z3!G>kT=jSFb9pw!B0r@MxeE9F5cRW&{q z)nTmDcTQ)od+Sdl4CT5Xs$%^YrB?X>wi>mbqGGwqn{b?XlGSq}W(>@MXFvH?ZZ+1~ zEBSr}ko?D2YCr&)XXc^YYUB}b)6aMxe-$z9i*fzd zuffubWfx(J*o$#R77Z$C?OKllO<0*io+f<%NbUPc3b3ZihL)I0;5D9f(A?%PhUk@B zYB1W?YPMYJ^9Ub2I5>E_Je#@H*j2Bg`}6EfMcyT1(0340O4m8duudhfrqI~%OKR);LKO7m253q0t(n*x_wSlyH<$Vay9YgsGXk#P#Ez;yh_)*bDe~Q$XK^s3Iw zdXbic2=n0~)IJZr@|!KP5>}!|z+ow{zA*wv(q951Moz(Grf(x z_JS%YB;Jwp4Hp@SO#`iSSd#t!L+{|j~^eFnR5K-a9=T*~d+w!3G$2-I7VPPcIMp7G30>kSTlclU~vJ zV}dq54kGsAWC{0q6l}8FBqY%C=HpqLT3nlv3R7Y4M;5DffHA*ncL&|&yFWI`!9k*| z-`SV5iG=RHgTH$L+7C(-#$JoB$8LKWBL{la*6k7VyXg&vWXuO}aCR$}{n0b=AH3MX z0A$S*aSTlHFhuOKShbmdTVTJ zy|+nXkdw498&!|aPb_AljF4&PF%44ES5p-QS|KBq&wpS|`1=6H_*TClz-_M)fR3{t zXWV}4aAQOfFi;-)zyWpP2+$Hz=XCQ#_qKzawUXlRU0jV@urt6TJhELmyW@S;g3aY7 zdtch{gG5mKUC$>Z22QHo+a(x?Ad}$OD|jcyp_3)cZP7BE_YN8Xz;^RG20#sJE_P~L z&2+|FJ8`KPzgaO78@sKggtRncQ3R+di;UFvv@jiexsR3hiEc*MI;}q|2ehBRJgw(P z%DdXJ4U|-3NTkQBOwvY+47jGEX)eoM0#0BX8hKs8xSEB7D^KIob`-aT%10Qo-o&l& zY7Bj4vK+KtZ7i57-)q6x7AjMVKW?mpi0Dr0CNE9awqMq*(wh6sxDVqUI5yGSJIe-| z&(!L%k>rmMjVYfxO7Au?CG1y3>awjD3>N*2B9<1}bD;k(UdmVe86vuMkdq-5EB9Zm zD?|2)`vLWGE;d&kcO8B`VE^CcL{FiS%3lEK8OZ4J) zxZ|_-VUX8#Dd(__tPU-Ep(+(&MR?dY$o;lps&N zj#*&ytod2#>EEX~nq6*SZs!)i_(@o9J1w+QF~Nya;p~-t1^5{ETtbQ$d8{ij^>!;a zK`G(`8X+r@bg==Jq+Z_?|6zq<(7O>pU0uJ6m{L<;(o>V1+;ky&Kh#)EZ+xlYV&|X* z9ngB4JnS}E?ui#re)-(=l^+gUJ10des4zJL68V?0FGS?h8PK zNZZ}F?*cXUi6N;dWgC$|ohmo$J#pk+y{L}UJ1>G+*OqO~pZCUb{_Kq|MrtvS{z6g> z4#NKb5K40fklS`mau;8N+fHe|Q{6p>iNLWKLNdhCpb6#kuPVYR& z_d9d%lWM>0zNGY!iV_hk(kXwripwEZFuCmZxkR{f+?Ml>!Bahf;f;(LD75ez6|K z)**isWbA6K=YP2cBWb}DLW|!^e zle#Vow1q3|#wwC3EkNV9@YTChpFQP+a#PO_#C^Jg1^&enN7`_T@vaA(3=^?7j15`P zF+;DDnjHn^2RZFQ{M*j)NFyA{E~fvIt4 zJZD8oq3M8>_Rh+Cwo$nn@L|bYoxHsG^-BlqNSWPf)UM2T(f99ve3paqxYxCcy`E9% z>Zv)!&6kXUdk;3@smEXrOV7_NV`C7~c#viYKV28v*n8eNb_-`df#Rb1l?uzTYIt0# zLi7bVj5=Q+;lLi3W>Bd3qGbh(caUZ7*w@r;3wlopt%OlBal@~8X8~4OS&~>ADovTr zZ^r~jgM+=JR1kbFg5!O+?Jr?bXj(T?I5gRx^_|I=d;?+|=U@-L*-ug=m8zV-fsC1loU{y?vmZTJce@!HyO5SELL zJrW@qc0+BlMHpU@8ZUNz=s4(I$lRz)!xaH9rEG0B6coRE{N}m{m&mXtky4r^!ZPw+ ziH`ORcgG#Y0&*S;c<&)^YT3%TevR4UQQKj)lZt9iwaYBCKZV(Q(=)q2W-73#BpHw& zpIfUrTCwD@o~%!cEz~}9!+daj^fLCDEdYW6;F@*Q6}YYoV=i0scV-b4X9z*y>9J6` z*}l1&XK_p$d)y@KBM)iXEC314uA@HO*?`T#P8XA^b^ZpxP_UPvq5 z?$>M~zokr=hC$TYdSP;<=_Pv&l9TCB+cKX5103Z(c{F5idQsSQPp8H2hOb_u;De7e zLoN0WrF)YvThIw_SPUhV+gb3Q=w{mjUOpk*A!)*PJbAN!bFHOAUHudW&9M%g%W!o) zHqbR?d6UPi?;eN{s&e>~Px`Y}=#dqXEh2ZfHTt$b#UNq}U&PlIn>6xt?Gj3)RWfXD z!rK6k&0m>r+5|@NREFs6pY$t(Z>j~e?H((!WIP!W$7VGJac179F%tA(&+#TlYoAkp zf}d#4;{9Z40ee-zGX=DR=zNitdU$)QM+uRKemH%CO3g;kW7E8HU=cO+!eXyn^E`u3 zx%}G9*qzGiGG&q&kBGnf#x%I*E3Or3^!{J-m z;JxD_fJMRs_RCzEL0&zU@Cd#DN=90%Rc`#ma6|A=tLw$jooF>ekK^(Yz3Dwql35<) ziWWEZq}`!i0ns0P?EEDO-*ar0TitT}o2^_9W_7@US4tjh1CL0Hy*&R6ob~9^F*pX! ztY2^wF>ZgimVJt&(=qsgvL3jHDSwn&T!6s)OFk_ri@DzR^C81Vv4G3G?Pfi5aUMEP ziV{XCmV+;7fI~P9lcUwNw8-JG+8{s2rK}Qr<|(R^y%#C+k&_g598afZ*hQ^T2*fU{ zZnYMxM&Vj@2?aAafjV9Jtv*01;SYxGCZ3Znt~XMRWtrQ=fg`SWzh%C_Ib8EqUbrK_ zeEQpta2nE)O9b&X0OSTYw|u;VjHnsz?>qU>|2X~tDZmb!C^#)as$!JTiRBY*u;3(x zHG$LD3x_cR4m%fyS4_MCT-14JL*s>&sg`qJQ87qu734lS^l`5F@|Vr?nlDtK@)Avg zD1xaP>W+)kbIhN`-Zk-erUlyLcqUBkWrpLk@>4CDIfUxyIs+#t$y9@cAt-FLNqXf`!obGl|-(1$qIRdQN{o7awYI*Ed2M zt0R7&k1yNJ=)1!=Wl+xS1)RVWE4x*r@cV7yKch7nrEeyb$*MO@KRMSChgk7hYq)Hg z25!{Qk=<$DtE?2P0?q;6v2)hrFsVk{?;OnoCy=0of*|H>&N4_>T5;$GCrh3aD+;V| zBf^jDC0BjdKuY|jIHougEDbvIA3f0`wCy7)9mh{0cDpDwl?h{3xbx!EW$H4w{i0T!q>=`c*W;MgpV{2ok}m z+o&jj_ers@X+#kMK>IoO2p$4RvwYkWr*e+f*Sbf6_xY`hsGPZ7_|TmFVpw!^Cb-kh zW%o0ggyU-2TqP~_^1*DhWiL41`GIWM0XdHL`g%UC_{YH&K!J8Rk(#xiZGAlx z-2$ffzJ$qUt^!e=ITa+phO7WLQf30&CZG1adEQsH!z2%rxl@NXyTs51j^n}o0uu+z z;d?uqs7x^>D}nmcY+G~{EhdZT!vWNtJb223Y7Y4Yda#&TWwA$FGFP*hky)v<`NAH}*R|=(L7S!f z)X|p75Z|um3_+Y7mApV$jvlJkZj}z8L z!Ko!)65!NLi)ghc(+F`7E_|HSLXhaWo_L=<&u!94jhI8-VTU@YKZ>}_>od5$qZjNb ztc?LY(>CxjFB`HzZg}bi*=Ml5k;5zoZk7W&R&3yYY}jt({swGZ-Pk$l4BJaulAfc> zTOtO17C&Tg(sLAG`l9PXr*T_U^2rgxV<^wyg+tOaiv=o;rl5koG|CbTlf&Ms6@BHB zC)x;=y<)u+fY+&@z{?f@$Ed>2RyNy{P2xa z07j3XpqQP@@X(Bm%Y?rp``NawJm~2PX-K3%h^wJeQl%}m*)MHveIO%X-jqp36j^*8 zF0s^Im}{*IY8lVqM{PI!zjy^%PiGMm5!kX5HbLz%l_9w|3tAGxFIgD4w+C3KMEp>~ zpsYIs1msD#tFafWfVZ!t_zeZ>lQ7@ECl+`QCaR7i%7qq$>~nJ)J*^#V!HPzOf%`YP z8X|Rmwl9>9nl_Wuq9pa{Z(DgGU{`*wu?LGl&bh6V%evsYK)xHd%dp z?J3l2Q4fj|z#RS(-S^_8<^i&~JEfR~^~f8j>z+>}SfktpflmKkdij$F@6rOeooBVz zUsMX!MPT{%Pbz=C?Ej991Vbn>-&Bamut@tha^q!oXZ-;Ek%-UMPgIYcZfw{nx_Df%-Tng~)?wboHd7^yVN%f5TkK#BK!M*UR)daQPoQN(c$LR)mc`~@O)Opla$Xm|> z>0C?Krn@h~NKnatGR?7D$|npwY!J4E_(v+Rm3VxSUc>Ka#*ir*Ki>CH+RzY#p7%p| zqJhi1C0CJ^w>~dc-5<2PVX>*s;wFH}pEv1CdChh-e3gHDStSN>P{9@3cfgu??=I22 zn$?$id~NgVn*}cu#iI$V|3srxtcgi^VGlI2rZp;uCW zYMXiya~=yl&f;!Pau6KxVQcY9l{F~+_Ux$(FAj{}8hxozU|Qx``>~7{m+a_W3x%P( zy_2oUWl)r-@lgp?Z>#9^BmRNE18i{w&~f+{TY#@MLf>Nla&nOYT2$?&-S%7IP2>sG4@UQ!}+(q{mNg2*gD%fHas zq!wf=!_RCF>lMi$D3S1>yeVv1i1x+o0loCm?qN)Zp=u*8ED2S2jm`XA3XAylawq0k zRnoTReOWP*S$rn)Sz~JOj$ld)>EB>7Fx%+3OWHqF9iUDyU37p_4aNZzKa{u|cRz5a zYeL(G;}$g|h@-+j-A*7xi))RHH(YYh9g}bCU=@2!e`M(GO`_v}ksG<|o))UG6UtVcnj*IGLwxuUjE&2#_s zK)?Sn?SWpf7`n>`+q@-_VlMGI{+o+^>koja$yLyaP}gc)2_PoA0*)rS?Y2Kvp;2S_ z`p7=ivzDuy)tg|EZIdep6dmo!QilVP1maly%oxJ1P z7`W`LlR!$Z?hKz9s?(F{Nx6z&>nTj2Qa^( zuaWWCo0uecn;sqS5vMk7w(gIB^+}Z;6pGmO_x@~Or<6`Po!XNJIlErheiq%+;p-jT z9UGtmePvEcXlL01GlGZ232LRiX7p4nU}o?yrocfdrrEs&E|%w;i}h+#a1IA^QMY3j%2Apomv7OQh&$M$2}#o>&6Q4lL6+nwa?+x@5{$X-V%o5tsX$@ANol^` z^yTzDuzTFE7Wla69&4rsL&e5U-FOi%29vu~Ci)5c%Xzu_NEW_m&ndZ4^iD9|p^enh_GxTWy@xe|ZsG&qG~ z5}QV%zrsdtEvzT`qion>eb+V0Hfk7hL_DVibeS0{=PW#aho+e{ez{h!za`Qc{gxpL zL11n;0uo3B5(uyo{HX+@ML&6BgXktRJWDJAT2!Bz3{D)II0 z7pNr76lts6eP1^7Dl6V?o71rIt+2@t9XN`YjvkWj3M^FW& z{2dVy1OnNZyeg%SL^;KDxvxK2>HiL)L`gvNahkqW#xOWN0*xKxKVK0d1%~c6x$ZVZ z0xDuD_@aIW&isvty`l#>>n=?nc<`iy89)nV5^J|i)quop$4!0~Is83w zx7;P2c>3mix*G-h-PTfw2egXJb(c7rd#XIIU zE%;pH{{a=>4PlC^Ukp6cld65icf1Fd_f=>1Mc_ejlEn6ZKX8^9&?GAF?=K6SgS@%m&(`z5*;fi0>T27p zo0IMnN;D;PrR4nm_8)%>*#QFYcK3}zP>Z3@z3@Hr-<;|KhUW|S7YIkebvLtL{ikhU zC83T_x;>cA4H}BkazJqUH^D#r_RHb^a=3pR^N+v%a=5=7?vFL&Z}0NU;r?>Ce`~`J zzx|4Ee?_>^Zsb>l`~TAjcM%-Vj(BL_H&gX-tjIVl#0AY;>!WwHsL?c0nksRj|Eue_ z`tBOPr`a*8A~(%)%qZgBes@1p=&?(Ns;1`fr8&L(?f= z7<_J9tG@n;-u*#JBv89YpH@3Lz^}7ULsN}S2jm!Ucw?Ty6AQPT&*D5JyY&Q5iJ|Y( zm;vHH@Q%Ae^SW0V2;1lFZPlyb6(rFSP!>Z#9ijvVbzM!4EGLpMNwJ8AoFnWL4i*iu z`{yz$Wr8ca%-!Jl4JZXuu=kA zSzUp$PTR1z^}ePS=du|7069lu9Uz)D=Jum!o>*nNp=bWXkCXxbF>^omLooPs(^9<( zt*qAo-I2KhwgF&YWC`-8zroM|Lx@cxarynJ#l6@RhassZxTpjiLRFInkk1k>UmP^A zd5D3qucX{n1+TdGc^34lLy2-)1q>SJVw#eTP8el6Yy>&S_%$%a=epLbn8XrINB-P0 z(4zSQ94k}F#v?c2M;51mJRwQp1ZvmhJ(^$ueyxQG`BN-n86Zc|7W2+K=!9gp)QvzKpcEW_3q=Y{9;hAXLx=f`z|#-FpniG^4TA{Cv6J(%EwPyQ8SQt)kW^D&j|C3F zCA0_--+$Rk7@F6Q8Gx{F$tP!kSETv3$w4b?Qz{r#c(+?#`z#H2PjMUM97QL9Xs~N0 zNAMYZLk>wyL!c-58hGn_PiB6Fet+81uh8#%SNJRR`^%>odx-yaN5JoY`6~?kSKRR{ z4E)oV{LY$RVc@@11pQYS_?J)l{mZW~@VhwWR~Y!6h7kM;1HTKZe<6V1hibnN!0$Qo z3jzFBBL52k{N9iJLID2>K>R`gzta%WUkKnoHRRbZ1n~Rt>=y#~Jx6|R{(hI!{M!8e z-mg9r{k8e~9a{8j^Y=Tr2=;69_q(j-*XHl{d#b-Sf4}3%{|B4Dy*=o&5)%8n$HsZ- zbiP)u5`sI^>_*?w@qJ9tln3@QgCbNy`l~My?n>5U#$E|oGZ*uI`rCvXp|!XKQd~?h zKc0pp<$i7Q25A_b23wuQZ-2LXr^nSQs~f=xWD5mxJ%epq2(Z z?l(20YBilwlq4OC^W4b9YuTT#f9Es*`4|J(n-*WUkDBwtn^7TjB@>V3P3MK?kDwRW z&d%frMSHQ4jwOuykcM}P1^S6zRJDdw$d`L8%FU@XfCK(Fh5n;-#1#~;Oq)@)bt+tq z^B?aah?=C)%gvck){-eqv2|RISVCed- zz@vNUaXy{b&3YLADYbtnJ@}_sbx`DK^*K@w#qv!^9x(}eO|CyDD|~{7*>zu*4JV5qwV32JRjPAyXS2v z9S09h3LRmVsf3)951l?=;6;DmBU7@&Fgkl_I)QLHH*>l`>2O~QQ!-83ur}G2Ca-2I zb9tkV%9Fy)W3!moj9WYU)ori3x&_g6yZNBvnSm$Sql`?@)jiKdlYvdWzRfdXQ!-|? zza_hHY$He8j3X5mI+tqtRFTMf#F1Dwo#z@Ahrl(3NRliSBdMk7OI`e?&V}+(7g)2T z1CHNlq)Mc7Gu~#+3Z+yAp(4bc$5)%s|ruAO3VQ`Jxa${7C$zuX<<;!MTUuF;Wc$;x^ zLaWEMd*H}P@v=>~!H=Q?(}T+8)P7r}G?P~e*FVLrmL}WXmXCComyfjHnnz5hO9ve* zMsae5PvEVNkzno!@gG!pAZV44t4@4AFLr_lIvi*xHBITp0ek0tk#Z}@@PD!BdiOp4>XM$2}$azDYTgx4ZI#f30-m%;s9!XDlOCSj&Qf^Vn{+O z`DMG+r07~2DDU7=$VXT4rF6I}%E!2k%&RdHPoyGM@el0m*O$fijthe^qi*`aj$OSrQTxhD4ibK zK0g+79(uZ+r9e#sD=Pf?spfU9)fr{SLQ?XO>7*Lh6apiqV~Dtdou9P6bx%-V5Z7ApWpR8_$EwQ4=x&a zA(~mCE*+wRIo80)O~sdy(y{Lff;Riwv!xM%+QX~<6vO=z5(oEehG`~kWTTKr-R-X( z<}wXN|LMZNe=oHNm=!qX>7fH>X~rQ*vrOgHK*u(f&nfL%#09~o!CxB-CDRSRJdEp- z$B^>bZ47#WY0y}AI?r+-g(^5!LM!~uB$q{tOZAXEC3IQiGtsLkGiR7VMpoRna|ljq zCJ9ZM6X{!gx1ZX$4E`%rZ=9(-Y%sAzw1Zdl3ryu@IUGCG$v#zKlohca+K7#9cq<+O zy%40;0JnY=xMcrnJR`fsfDV2sl!ck9o8MHURlAgBmW7X5gI+0TxU5afasFehqj?Ly z{^4vNhpiPNoOof8;dA5gn_Zp0)*D9uQGW-Pf`n6uTKvOuUA5cYV=he3t~)J`@I9Tl z$>GrT6_-z8s<_gv$6|ecR&(lO^@JyqCr3U?rWk;8|| zSd#C>{6B#c_@^iZ6#X}-m`*ntCDU_KR9LXvjKCeR@u_A{IcfLD2-jDiCM5NeNsG*l zbm@xhlQ>SpG~XkmgU$){O-pxQU7#u{*`;i<9N?$@*LKPOE|H=ZfH9A!9&9q5K4NB* zqd8`w(Z0(j>@cBjy=9|Ao)w-tfMmfiW4~^zrzE)X(E$jTN>E+!Ylsx`p-bZz_TFP| z5@;h2LDexLm-P$71+pu)>SdGr@kO?I&h;3v2RX^L2b&fIwW+5LrAz1`+F$4o`$X#_ zf&vqRBWkiI#+y1_+EJe5btU2wG=sRKfR+Z`xG^|zw39@9La#98d$(npZ+0)DVdQzs z)82&p86<<%PIiK>Cm!`pR{O)3x{pGyAs#O+?>{Qqu9W_#Fuw~uVU?&_Z9Oh8w;Wxf zWG@dA(~XR$0tItLpE>A};)RN_pb(Lj2UL~sB}+Xk)qxp~FjpEHqayhzvP~sNr&i$R zPtW@iUF6TDFU~&ESazMxP7OlZYD#wuUt4sYgZ6dtI$-GSFt<15UYYmbQPeGrNfFJx zqSCaQWY%AOU%)kAAF7UHK4z;HJ%E>g=^!ABkpxGJ@mMU8YUX|GS?C0d3Ik~(m;1+kiUssS({JMG*~*w5!xQB zcFi^XGGnY`+eax6?Sf5+fd%nfF19b-XR%-mbiG2O-VZ`G7XD<~ln$YU_Ee^^CN$J4 z9KE_4oZvNs6f1XjOtSsD5b;5)r-^GJ1}a2`&7zttrm`^)Ds*Bcp4v&YM)TX;Vc{?0 z0?}bf5IvJNG3<1(Ww3(Tq5fF5!c1s!a%gpNRpRcFWMl_9B;mv;uccT)v5e3lRPrX! zpY2L*$V)xGODCJIn98Ur0pq6dd(nTax$OLCC44Fw+!j2qtzNI*2H80wY7HFLVr~np zn#58zY^!Ic+lG|wB?i6rgBL2V%rC%Lm!1TyM5_e{MW-}t4hyKcr>Gn8(nC7$K+t5Q zJQb6*uvE^e5ElHVg|cc-lEZS{iAPwh%!qifczt-isUccgTB&Bgw}p!2KIkd<$psFiEMIf%p`#(?{6C@I;xY~EpKLinC3z2 z!$Hm`AR6m--KLt+7R-kEoz?E^wnal~Fae({Fg>PozZ=I!(MF~;yZfD1O;;Tf)k^XB z8)(Bf+yyAVI&PZead+bk2{_m2M1RSE^gE*Coxo4vbuIsH&B-T(gqyQ$S~m_}pL550 z#F}e38hGb8!^d~jocR)2>+hIJgb$%wpe$@IICK%}FE|d--D}H>G=!&fyfA5@`M4^VZ}T9CNdf zKyLI#zSM8NloL$H_~*dDhH)KSId)ct?g`JgSWpN>^zTaaK`%&awt%bHmY?-Jc3;p= zj%J0YG=_2MQiTV<)trc?q`hv9hDkX_dHKnS{y7TrqwC(mg?3w&#*>N;|IyM%1NwGK zF;2T_*Sw)?p8FEa_5|U-V55{a&KgY6B&FA}bErQhikP1;jXHKlEfPjOekzcZnn(dc zNgwbC`Mj3|IVnA-ss77m^AMwndX2;);n%8MY-mV5>{O&O|QUs zZOCwi3?#PTUZf|vIfpege&Ak95!wsF4o+U#>cedex%;uI>x2CQG~J~|Sp{=*QC%Nf zA(d1fw3mD;c#FT|;K4)#cdifExBjGF!@oXkO#`bVE=6FT5Q9ZhBYR^3EMz4VMmKZ* zBUUyB;f~-qo;k@G#L(KoKG?X9v}kKs$@SW7Uv84Pd0GdKDZ z-M2SoR(*ZdaTHx8TeWTN^NOEDr4S8PFBA{vA7{=)I!FyU zOKHxL@^X_rdk86Q!64S|UvpXA;THqo}oWxr^VaJX$<{0s{aXe%qNR zS3vgpQfA7(xII?s22%OV|Ay25T4|{xfNdJ|KW(&&h#k~;d?`RC1+3d#MpQkqEy-4!e?J^;4KHUJQLpo%?#%@&3uc?_UxN@{NZ9?NP_8aPCPSR4n>lh27c) zi&#;wXJ|>kvAA|aIzg+TFfldL>NGf@3}Q4`D{x4$TD77l{w-lLBD#+w$?0P@Q%qt6 zX*Xv~o{46mCLUy0;(kNQ2Va7$`SOs!efNZZ<21;qS=0ugloItQg;ihMX3uc8 zfHIs2p53hLZV4F0mO2!g{i7ENn|*9|n;2I{t1z*Jawj0Q=$piSFD@gFtt2X4hL%6) zO>Ye-&ZCh_QRBdH9dsxF3c^Bl^etd})aAtmsXh!TqCJ>|{Qzf)z1B=Iv(0vMD*x?` zr9u5CL-)`du$D*nmCZ}1{45P+y0-$7W2zj`{-Yn86)c4Z6)ru#1FhPQ_gVNBNX;H* zoPqfe4Cb$n8GyV=g2=k9J192W$3!C>(xRtz;?upbPzjkL&5F|PEbEPqs;eN)?l}6y zEfaqq?H1-vebvc!q>Qu=O<+=DmBGw*QAQTD#}s-K)sxZ#43S>sB#Y;? zs;_9`*^$E|rN-JL0Ry_g0g;@7Ik~|88h1!pG)0K^oFP1OSn=eg%o|p|g->rQtkkQE z-(N;M&yef3xd=Ge;47n6Vz(751d|rhrcIM$W4F@HptFZrav>|A%fjH z|G;6%7~luew6_;7qmT(Z6HAvUsNqhbov?Pk6Z57Q!F@Q{BtFEy2==67=(bJvClBK;lz9X zB&n>31;(Za4c{C4Ka4`p6C;Yv_qyYezoleUY|VJRTK$NmvC{RDDXeKVy~C7Y{g#6q zDY|9_3(_1{S$BcHOAxky3eD+)Nd`5*c2?EuWkv=8tRuLKp*HQr?o1ov_fcr%G#erjf z0QyL#5Ypb}yQKiBWd5w3++yB5Zp@0{+GPu;n2w|U5^0dIeujToz`Az@1xbu-#>J7l zamj|OMA&CnR`_%3K&IL8ei%{Le6J6V9ZmQ;?xJ9_PjfmW^~}8dFk@88w~bnOTkzeI zvJ&TnD*wUuf2|x=3}|h{5{`T{v#d9#wmB^NtzYO*tD`Ag&2^ab%9{vce`5^q@9(ma zSHZP>069-Q&3H!7Z{{vXbA$vT*5n@7nAM;OHUUHV$DLf8Svy z3_03k`cBZ^$ur=f8<&#jS}RM6LO>p;rV_zCvCYRS`HMePaD>s_XRRu2bciIj?P!2< zYs&brPnd+OrK{cNHRZCFXu|lmN5=f_w!CuY%aDy%Mx(7eG+1Fh;ic)mNOp;TAwVu$ zTQ}_#?yY}u>wmxx(NT14Dv68pOYW9o^_a#TW-ldEb?)-YJU>gU@aN(s)3fKb(;kaY zVj;W`%G4`Tq&>ynD>`L!SVhh50EerUZjx#FhpK{I%+Im?wvb-D zS3G`P%zJg0UbfP*-IL1zc#pnMa=0iB*@s`A1pXtUCU)ZVN{kS~GGhW)4w7f8D9Qx4 zi_UeJdMKbUaaBn{j=9k2(xk9-8Lm_A&u$21W73u3s}7&IVJf%tYbNnDx{B0_ep5jaWDT&%LO_{>>i-9K_2OQ%jm zvE9}?O{AX!G~0^Mo`WUo59n%4`^?R4NkqV1)4N4C;IyLSvM{jUm$x$;9DF=n`-YLL zy>dQ7$O_s&UXK->@Wk@UEY~HZwVwLJtLZJFWA)1X#WaS`&rv%a8j1N|psoBm>=Opz z>Z0!Y3);}*h0vGoY_oBD8=hG$eCYY-2g{F_*5^}B{O{6v*lIL>5oyUjPY;>ngczg7 zJM%~8K;g2;{`rHG+@^UBFBX0t?ntZ&Km^bWr z6*J6t3asr498Qa&=g%xqqe+tT(hGg3YHEO1Mx&y|vvhD@s;tVec zk>C_0P1sYc1ac`~?xVxvb~tZeart9Nh7pU!D}UpDbeQkv1rbY0`M824tV52|Fs-x8 zPUg4FkrQ8JXsbuzT{iC7Q9b7%O}&hO;XXk^j*A?q3co@t!wT}vf-!*%;^Pbr(F%cVh-1DVN zv%!-5oW#lMTt+eWaFJzkNZj)`SRUH*vos$TmiUDf!UR&}1(*~C4|y#je}nO#{ZO-l zFgSrzNrt5uv#cyv6(}|`K(8lW*nqa{r5TxAQejI*twL8~d*FTXOPx1hNF2fIo!>Tg z@7!iL)r1d6oCMiHI**SM-v;X}nPim>b9V-7o&H2~3w?K|I(pP?z0Imju+}+9{0ZZw zkil~y<%yJsfw;JQERJryI+3)7VU|@GVT`v5J2-3Z^2i0*Hn!ccB2r0zrvJmsszb_P z6-6DQ)OVG9Y)jRA?EQ+3iI4fzAM?`Vxo^;Y=N{F%wL}cQH7|BamOM3 z?7dV-f78i;;`O=T5tZ`sBpid)Plkb$x~0N7<>ex9_nElH>`|9E>tfQHwbs1|b~o-I z;m7Sgu?rOw!|R^`ev=h#`4W@L*>MXo^IP3a&Po6?yqm(tsDtI5dLnM&9~sTl;kYMD z^G=$H+c~gW7gmRj5Nu;5j>MhtObi#hEv7>G7 zzz9}%xc^&WV_CfPmt);>Z%*!y*lsR@v+FfR+*|srKZQw(aBS@~+Bf7>&m65=4tSVt z)#x6T;Wzl}?tkv`&?(rdHgRzum=~eqLNx8oV|wroDGG3`?o9~YIezs#=cI90`eY~C z(pA-U=e&oOYb&`@>0BOe-ueq>TG`3VAXIR23exN&gS7QaDEquVSTkAdYy0)A@zG<{ z?$Vi8ZEn-mZifa#NYY>$_;8q0!|G6y4bq?Kcg5+fkXp$}#2`=bF#dO6Id_a4 zPPE(D3gC<09o$I0P}qL7f9DzaGO{+K6yRPYo-9(c>Ec1SYdJ!mBjhVu$C@5Qak~3sy3uPmb8+Webl2ZIE;t!Cs;9=0Nt1Kzhg`8=9XhY( zmd>8N7e{r;ZTAZ^+sk10wU`%LjW|W;FJ6LEnOW{=(+~UYx*&kA z1n449j}AFh*6~bq6&-uiV;S?Xq9=c~dlMcbFl}UE^5_xWYWd;uUWX4;6~NdoNncLgUC& z#|S^0>WIAMar8M>jj{C|2T%23mQs~op%U_FV5-hz^TauOf5t8F4u}1{jwSM^abR1| zY^u=OAl9ZjF_FtBpeAxp(qV7rkI*;DX4(y5^*%<-ud6_Qv}q^@Tn3$3w^xp2B)N58JJDEc}t9 zz*yamis`6WUCrYaD*fhUQlUBXs1%Obt+4>NL;MC@%fqU@ZsRA*1IKiSx-jlx3bWM} z^&2*i|#xTpVT;5 zlUX&zo(CeO9At>n%(=P4l2MAU=46Sf)#Y;?9?o^D>t)d=$eYH9DofSM#qLColl4t1 zza@{8z@w1_+eY;KQEB}ZC6S23DfpMghB@JCH0bIeNmk|SFR`-P?1sbWuEiW(M)-Gc z5K0R5!c30G6YBEjRn;2HrElyeofL>vmm6Pth%_~bW1P!tENML1&+KUAjefI>)E*^? zkg5~$$e-I))7m6mk2%@9(}A#25^y}Es%GSjl<$c1K!#&ZMlFdD1&ib*&UHi&y=mAo z4aa=1aFh0Z=9H))Qbad^J8(snC-BTfHYCP4uJ)&ZLw(;09nM$=ZzP-?29M2-dZ6Kd z&^kY-`*bCT?j7GltyVsRrytaJO*+g-g522nayXC8zN+c(vU?#>N&9$h-jVT_EW?X-4u_;B)B?Wspib$J>LTJRG%5wH6)PN<7&-KLvVAUUPSQy@gRgZ2@#l3bH^h4V^7 z>CQn-z=(KVfk;8SpiuSJ*UM85I-6fkEy@oWpuO0&3=YIUmXn}kX3D(WP78@&{RW%D z0P6p6JHMPN#T2=-717t|o5v;Z^bF%^&YcG}>+7+I*eVb}^!iyh6v|q!-n228@7&PU z)K*h-n0|AqU$~O7_{lIyIDO280%OIO1x7K1hK9+fM_zB;gq^P7{XDCCLn5p`c$VGN z6_*5Z3C1F4|E5{2b|v(LH4X=ry!G?zHVPmo5GER|FtZTMn{^45LTakoO}%z|B?S#b zo=Hdfl)|E3H%6K#Zv_v#)86KrgaqCCioT&6@3!PWSbUtVl~8i;K5sSG3G@9ckd}DF z_=n0<$%XA@uJ^F}pY35^Rk~FMZgTtB-nLtx-L@+g)u5}^$L;UrwVCjjlf;mNJ!e9ZX3=71 zMib3Kq0Yh8o|eJd>$LFl<1-w={vqPSVP2_6-7UqHU zK9F%D3*Xwohc~d*Inc3X_3BRRzNE#B3!qaVZOt)tP787F!GR<#l?_Fj*{ai0+A{pe zC0yRLwPM~am5mnd6j}#!?krqMI%>tgbv`t4P8+%H_7-Z1AfUFYLaj)k@Ebag zg%3FJONO_kBDqhYRk3tBJ<=82*kqTCYS_t$YG|$^*WrWIHEhxN+zW&ni>rz>Y_Z4r z2NyZ)rA`G% zS~TUsee#jb2agK!_6#F<;gM>e86iPZSS0~NnxuE}jGDZk`NEs9?kgN~uMy2SdXfYT zbTB)SLoHhe3cn^>g)A-C#(wcd1$Id4wt`U9P0MI#E>gRVA zZ+1dUBU__Zmb4I9MSTAvq%g7Njxh2SC|Y~`~1 zMJ0=U$wxH%O_hs2!T4HNIib_Y4m6As-_GL7RlCr@3ZK9zGGn&N8*wXrQhLf~1DLQDWPlS2i86?e51b`lr1_*kxDM+{sp7mAyZ^X{UUZxopxy^ljvMQ7 zGw(;V)JOMfRnE-;lpcaA3>M`O5Mi)x%Z15k{=u{2XN{>mXh*bb^IwK1JF8hkrS590 zj^=*w$o~SJKIjoMv`TvS4{fxjOzV-|;hZBu1cl{Kcv;_gd@Y0&Z8QecVAM_5m+rlQ z04;@%YF}?C>2pR;D2;~m74@R{_=oPw)t|s5oZ)YCsX$<&uoQA2M`CAfad!oiW%O8{CYBIuOYzD32n z*JNGwTfM)$<=@?i@Sf5~P0nzod$6E@byvd&UloK1`EOabAk)l`emKNPnok+YYOPFd zud;(>^jDvIkVS@Fq0gSWjV8$yvLZ}m=5c^umU=}(t=MeGvjs7zoJ0(L0`vG)Y%r*J zy8#IG3{A;i(sxvIGo`RN`$@3TE4!ktXsXp0Z#zgv)o)~64V3d&5^aPqQFN?#A4Knf z0IRrHH(bJbIQNdqRO|DB*blF5(EX*xyIz-kPH@wt37j6vSoEe0naP9)^y@rH| zrp|78Q-gN0VcpcSTBS>Ft4s{~fx*3^Rs+|dx^qW+v?UG zJl!7jr)a@Qes8rl^c{uHDTL$opoq#@?$bXB?cuQfSL-t^t}hJ%FZzHcR2kUrK{@;6JECe;n% z9wsrR>r?H+Z8_kK8-y4T9gFyd=pA;NX{pNc! zu>(Ci!jg`hCm)_euJXUZ-~_q8omwu6QFNJ4-*R2hI$tLp2fv)HPtarr^KyyvRmiOF zQZWci!I>T47J4;+!5jo>rDC+oTQ9N7WQ&S=M~MlkE(#ktxAM(pi2Xl>U3)y!`y220 z)rq5$Iz_q9+;f?0YAz=mRt+g3j94PKV(z!3G9he4BMK3@q_Bpe!&nE+<>VSVA@}97 z$hF^R)%o*$+v~O0{@CmF`EJkie%{aXyq{;^X&`UyJ8Bz+6$n4RhuxofJi_mi|L9J0 z?Yn-EUv+O#)phhuqMKK-vbnKq4?0id2TFVk_R8BtEte=c53betvQG4+ykWj6EV?o+ z-{-1j46y)l-LRhD1{Fl9t$e&ifp-n6%Zww|H)e7x!tnJ1Mkjc}l{?;+8vqUVEi)hx z>^W8fmza#O-v2}l!0mfRyec~OgPaii6aI+G%-QC?*6u~>ko{IZFMr){sO;8b#{XW%$DiCu@B zL4rQBJ|6Sp)fJ#VU{R!qGG9!$ z2!D;;5agIzEs5u#V3tXJ$?Ei>WJ1XuVkh}BT}3fJOD9Q#l+mETSK$*ZS>^R5LZx|F z+C-gSh(0x#m}O}g16kNf&hvm{BX*k&E)LAVGB!NeEgG!wQ@g};`x8)39ekq0^~&1% zeSRV)_%7->74K^Q4y{K{P!YZT*HnN>Az^E~Q$uO654Rsl%Q4#gA|ow!pDzc5BO|MdN;vdRpe~m4#fB@BV12lfsaFJ=!r>dVb47U zTj2H2^B2bH2W=!FPvfn<&i!mOy@7kZ*=hLhuNdi*vh5*W@fjg?B6kdj`H_|5TYYu^ z>`X=x3hs~TC<^HylUC45(2l+V9DS9?#nY>FGa8QBQV5*l6b%TUK`d=Iqup$1;1zWn z5zes72q6`?ufzf{5R2P)A=CQoqxLyt0U(#zCCexRH|Cd|oRrC)=>) zpXI)a#f;9k!IEj5kDz=JSauA!51r}u(&VV?nNJLXW5fg(@-OD*H^=X=HX-57I8SDH zIW$9xt4nv0Pcx{nd}(!OZHI!;*>b5+WC6I21tNQXpLc9(AWkQzczu2d6!d3CCz>mQ zeS-K398Ny;*&*(@MDL?|c(HHoQaX0(HtzZSbGuOSxSw&x*6-YtDl8Ri zVk|-}VhxSBxW6Lpd(e+++nL6584RIQWV~_j_=KIe47fmC21n!M$p zlG&@t*;k~-5Vw}?; zMyk4lthR02rHVoQaqd4k$Z!jQZ!-E&<-Z<%q?CVTI@l^&Ac}l|3uh*sB(;L+9=lyc zxviS*d1L}QqC0v4d9Rti{DC=@jVB9!qa0ej2!=G3>-p!w91zl@v6Wv1*8@tztLfiy zcVgDc9Ex&zf4>WCpr3ikqB-a)fdz3!adZTPtlI^5FU}*%hD_y%8 zt5+|fehSgIJ!`->(sN`!+T~JY!3#XopI@~`i<1JO!2d$28H#-MNBlhIy5B^@Q zGpsl^M=2S_T0bPq3UN4hTCYqr^SPJaRRN$KmDO-5RD5lN9m+Zngrlbl2Xh1$9fDoCr%7`uK0Ogv@CB zCf&5A45t{|-&gYbl0JpF*ertx<4gsgFZ4A-y?;L}=RA>LGx|QOT`x^ekQ(S(qyO^# z;&JUvW^gZbdT@FM(gU+Qw(^_w5{865NLDK6_D);z*1$wWWQmP!3qQeMl<1JS2_K;9 zf8jXFWpoh*fR-K>4LwcT{+lg;ytp~Fva$-!+1*}iY-^J~W_aHgj3 zh~}VwF*BokbIS4jTP^=FpSOyTOVQU2Kk))S6>HH?cwS9g&NL+@iibKF^(S>+8>p~G zF;dXs6zlmk*^WAYG;;MzcxN!)hxO{6hcFEL)ina!ZKSG9dJ2^d_DZS+?KlG4mD3Nb zqKYBCM09aT_Wk3H%?$Sf$Gt9cs-h@|-*JUC)RK6)>@UFt(U563Ka6i6rCIK2k;uIR zIO;LTW%}z4`~aK6Hv^ff88gF~n67%FX^42Ng?=%t8LMl+t?qei4%Ny~L2aKk5E4lc zy%Jd+OF(O5w?4IAIb5$5RAZ?N0e$e685ZzC;p*Vp ze{KkCUBty74m>Y@?tZ<@Z0K}yAJq>#+fO?N>vFlpjweQ*>>Be}@;KZx?(_L4@NH-S zWxlY@SAEv-fMp~WK2?c19G57gq#$xI*DPayuG#V9#^>9nmk$4GbYpW)#Ce1)&{8MY ziw>N4borzi$tDX+9xN&nsY3{_Sx7^B=vQtZWoZ5|d$H_72&0&kSD}yT8X)33%PY8Q zKAtfP8Ws(3wRkYZ*gV(SkUzyn3WF(cGj~{1A07_IaJr=r322HecF5hjk=s?PJxl!y zQg7ZC^$HOC#jEX4t&6<8O7(PL!7ik{GJ^;ODWR~wCxwWxHjXgjW4y4o>jk{}B|6FP zVKn~5gKKWA(Q8gDmupv9AHf`}^~K3)Tm3I9L{1H8o&@#~=!?(}bb7`$YcYA|aWyDQ zSs3yYGZN>y>1dZ`&e)d@8slc}hysEzmdBk8rm3%ZrQbC&XZ*7{2+Elfy;D0`IHW18 zS~|9-OOvL7$-NYTZLF>>KEV47^gfoQE9`It?}g99qyFOY}Dz{ zI%!P4=3g;)W1maq<`S4sNEtmR=0Y26h@D)4!w znV{KjObI|&)hPLvSuz8}V%epQCiM??=006AB>2xJmP)VZEHQ^CkJid{dG;;^BpD;K z;8UCTHJ6q=W$9%ahZehUr45BNyo6?1L(nCHm@Dylh4!(FV5E3jPu(Dq@)X-$mAc{g zMcgwVz!C0}6l`~&I0l)SH#Gn!yU(!A>@*%bJXI7LDfgG0PKK1nf8k-MfD)tc({rMmHtFb2Jm=U#BJk zIY5xsA|i_EKAHkAH-PnLz^B-rhs|XKkB8pd1M8=x;3dqfUHsHt|IS@f*;=c^UHJog gAk4dWa!1rx0avvWt=7jm<=emmgEm8zo^ZPHfA+x|3;+NC From 18aa39f892ba0fcdb7bedbb29bdcd46b43fe10dd Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 01:04:11 +0300 Subject: [PATCH 102/197] fix(ft): add emqx_ft_schema to ee schemas --- apps/emqx_conf/src/emqx_conf_schema.erl | 3 +-- lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src | 2 +- lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl | 5 ++++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 1a0f8bbb1..58bcf9700 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -63,8 +63,7 @@ emqx_psk_schema, emqx_limiter_schema, emqx_slow_subs_schema, - emqx_mgmt_api_key_schema, - emqx_ft_schema + emqx_mgmt_api_key_schema ]). %% root config should not have a namespace diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src index 324e7e308..771fdcb27 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_conf, [ {description, "EMQX Enterprise Edition configuration schema"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl index 5137574e3..10c6c0379 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl @@ -8,7 +8,10 @@ -export([namespace/0, roots/0, fields/1, translations/0, translation/1]). --define(EE_SCHEMA_MODULES, [emqx_license_schema]). +-define(EE_SCHEMA_MODULES, [ + emqx_license_schema, + emqx_ft_schema +]). namespace() -> emqx_conf_schema:namespace(). From 2f1970adbcb21387521cb8cbc092878202804676 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 13:36:34 +0300 Subject: [PATCH 103/197] fix(ft): make ee schema aware of ee fields --- apps/emqx_ft/src/emqx_ft_schema.erl | 5 ++++- lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 697220e1e..270543bc9 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -35,7 +35,7 @@ -reflect_type([json_value/0]). --import(hoconsc, [ref/1, ref/2, mk/2]). +-import(hoconsc, [ref/2, mk/2]). namespace() -> file_transfer. @@ -217,3 +217,6 @@ converter(checksum) -> _ = byte_size(Hex) =:= 64 orelse throw({expected_length, 64}), {sha256, binary:decode_hex(Hex)} end. + +ref(Ref) -> + ref(?MODULE, Ref). diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl index 10c6c0379..8e2800c68 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl @@ -26,10 +26,24 @@ roots() -> ). fields(Name) -> - emqx_conf_schema:fields(Name). + ee_fields(?EE_SCHEMA_MODULES, Name). translations() -> emqx_conf_schema:translations(). translation(Name) -> emqx_conf_schema:translation(Name). + +%%------------------------------------------------------------------------------ +%% helpers +%%------------------------------------------------------------------------------ + +ee_fields([EEMod | EEMods], Name) -> + case lists:member(Name, apply(EEMod, roots, [])) of + true -> + apply(EEMod, fields, [Name]); + false -> + ee_fields(EEMods, Name) + end; +ee_fields([], Name) -> + emqx_conf_schema:fields(Name). From f6461fe2872c66861fc086ed0230d0fe544e6d6d Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 15:46:28 +0300 Subject: [PATCH 104/197] fix(ft): fix typings --- apps/emqx_ft/src/emqx_ft_storage.erl | 1 + apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl | 10 +--------- apps/emqx_s3/src/emqx_s3_client.erl | 15 ++++++++------- .../src/emqx_s3_profile_http_pool_clients.erl | 5 +++-- apps/emqx_s3/src/emqx_s3_profile_http_pools.erl | 3 ++- apps/emqx_s3/src/emqx_s3_uploader.erl | 5 +++-- 6 files changed, 18 insertions(+), 21 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index bff149ce2..79538a9c7 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -44,6 +44,7 @@ transfer := emqx_ft:transfer(), name := file:name(), size := _Bytes :: non_neg_integer(), + timestamp := emqx_datetime:epoch_second(), uri => uri_string:uri_string(), meta => emqx_ft:filemeta() }. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index f84189f7f..ae8315510 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -45,15 +45,7 @@ -type options() :: _TODO. -type transfer() :: emqx_ft:transfer(). -type filemeta() :: emqx_ft:filemeta(). --type exportinfo() :: #{ - transfer := transfer(), - name := file:name(), - uri := uri_string:uri_string(), - timestamp := emqx_datetime:epoch_second(), - size := _Bytes :: non_neg_integer(), - meta => filemeta() -}. - +-type exportinfo() :: emqx_ft_storage:file_info(). -type file_error() :: emqx_ft_storage_fs:file_error(). -type export_st() :: #{ diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index 2ba837e66..b6578a7e1 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -32,19 +32,20 @@ ]). -type headers() :: #{binary() | string() => iodata()}. +-type erlcloud_headers() :: list({string(), iodata()}). -type key() :: string(). -type part_number() :: non_neg_integer(). -type upload_id() :: string(). -type etag() :: string(). - -type upload_options() :: list({acl, emqx_s3:acl()}). -opaque client() :: #{ aws_config := aws_config(), - options := upload_options(), + upload_options := upload_options(), bucket := string(), - headers := headers() + headers := erlcloud_headers(), + url_expire_time := non_neg_integer() }. -type config() :: #{ @@ -62,7 +63,7 @@ max_retries := non_neg_integer() | undefined }. --type s3_options() :: list({string(), string()}). +-type s3_options() :: proplists:proplist(). -define(DEFAULT_REQUEST_TIMEOUT, 30000). -define(DEFAULT_MAX_RETRIES, 2). @@ -171,10 +172,10 @@ abort_multipart(#{bucket := Bucket, headers := Headers, aws_config := AwsConfig} {error, Reason} end. --spec list(client(), s3_options()) -> ok_or_error(term()). +-spec list(client(), s3_options()) -> ok_or_error(proplists:proplist(), term()). list(#{bucket := Bucket, aws_config := AwsConfig}, Options) -> - try - {ok, erlcloud_s3:list_objects(Bucket, Options, AwsConfig)} + try erlcloud_s3:list_objects(Bucket, Options, AwsConfig) of + Result -> {ok, Result} catch error:{aws_error, Reason} -> ?SLOG(debug, #{msg => "list_objects_fail", bucket => Bucket, reason => Reason}), diff --git a/apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl b/apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl index b4e640f7c..19e4d2ddb 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl @@ -20,12 +20,13 @@ create_table() -> set ]). --spec register(ets:tid(), pid(), reference(), emqx_s3_profile_http_pools:pool_name()) -> true. +-spec register(ets:tid(), pid(), reference(), emqx_s3_profile_http_pools:pool_name()) -> ok. register(Tab, Pid, MRef, PoolName) -> true = ets:insert(Tab, {Pid, {MRef, PoolName}}), ok. --spec unregister(ets:tid(), pid()) -> emqx_s3_profile_http_pools:pool_name() | undefined. +-spec unregister(ets:tid(), pid()) -> + {reference(), emqx_s3_profile_http_pools:pool_name()} | undefined. unregister(Tab, Pid) -> case ets:take(Tab, Pid) of [{Pid, {MRef, PoolName}}] -> diff --git a/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl b/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl index 73774624e..944f2037d 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl @@ -89,7 +89,8 @@ unregister_client(ProfileId, PoolName) -> set_outdated(ProfileId, PoolName, Timeout) -> Key = key(ProfileId, PoolName), Now = erlang:monotonic_time(millisecond), - ets:update_element(?TAB, Key, {#pool.deadline, Now + Timeout}). + _ = ets:update_element(?TAB, Key, {#pool.deadline, Now + Timeout}), + ok. -spec outdated(emqx_s3:profile_id()) -> [pool_name()]. diff --git a/apps/emqx_s3/src/emqx_s3_uploader.erl b/apps/emqx_s3/src/emqx_s3_uploader.erl index 8327462c7..595612f62 100644 --- a/apps/emqx_s3/src/emqx_s3_uploader.erl +++ b/apps/emqx_s3/src/emqx_s3_uploader.erl @@ -48,7 +48,8 @@ max_part_size := pos_integer(), upload_id := undefined | emqx_s3_client:upload_id(), etags := [emqx_s3_client:etag()], - part_number := emqx_s3_client:part_number() + part_number := emqx_s3_client:part_number(), + headers := emqx_s3_client:headers() }. %% 5MB @@ -252,7 +253,7 @@ upload_part( Error end. --spec complete_upload(data()) -> ok_or_error(term()). +-spec complete_upload(data()) -> ok_or_error(data(), term()). complete_upload( #{ client := Client, From 6e50b168a07c1c7175009b335ed82ca9d269d15b Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 16:54:45 +0300 Subject: [PATCH 105/197] fix(ft): add empty strings for zh translations --- apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf index 99004d62a..c26fa477a 100644 --- a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf +++ b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf @@ -2,83 +2,103 @@ emqx_s3_schema { access_key_id { desc { en: "The access key id of the S3 bucket." + zh: "" } label { en: "Access Key ID" + zh: "" } } secret_access_key { desc { en: "The secret access key of the S3 bucket." + zh: "" } label { en: "Secret Access Key" + zh: "" } } bucket { desc { en: "The name of the S3 bucket." + zh: "" } label { en: "Bucket" + zh: "" } } host { desc { en: "The host of the S3 endpoint." + zh: "" } label { en: "S3 endpoint Host" + zh: "" } } port { desc { en: "The port of the S3 endpoint." + zh: "" } label { en: "S3 endpoint port" + zh: "" } } url_expire_time { desc { en: "The time in seconds for which the signed URLs to the S3 objects are valid." + zh: "" } label { en: "Signed URL expiration time" + zh: "" } } min_part_size { desc { en: """The minimum part size for multipart uploads.
Uploaded data will be accumulated in memory until this size is reached.""" + zh: "" } label { en: "Minimum multipart upload part size" + zh: "" } } max_part_size { desc { en: """The maximum part size for multipart uploads.
S3 uploader won't try to upload parts larger than this size.""" + zh: "" } label { en: "Maximum multipart upload part size" + zh: "" } } acl { desc { en: "The ACL to use for the uploaded objects." + zh: "" } label { en: "ACL" + zh: "" } } transport_options { desc { en: "Options for the HTTP transport layer used by the S3 client." + zh: "" } label { en: "Transport Options" + zh: "" } } } From 7bcb60a84a7fdb81048bb9086d2519de9817139a Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 18:18:03 +0300 Subject: [PATCH 106/197] fix(ft): fix config update tests --- apps/emqx_ft/etc/emqx_ft.conf | 7 +++++++ apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 23 ++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/apps/emqx_ft/etc/emqx_ft.conf b/apps/emqx_ft/etc/emqx_ft.conf index 8d921e79c..2058d24f5 100644 --- a/apps/emqx_ft/etc/emqx_ft.conf +++ b/apps/emqx_ft/etc/emqx_ft.conf @@ -1,7 +1,14 @@ file_transfer { storage { type = local + segments { + root = "{{ platform_data_dir }}/file_transfer/segments" + gc { + interval = 60000 + } + } exporter { + root = "{{ platform_data_dir }}/file_transfer/exports" type = local } } diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index 3811c1ef4..5788e4171 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -25,10 +25,31 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft]), + _ = emqx_config:save_schema_mod_and_names(emqx_ft_schema), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], fun set_special_config/1), {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. +set_special_config(emqx_ft) -> + emqx_config:put( + [file_transfer], + #{ + storage => #{ + type => local, + segments => #{ + gc => #{ + interval => 60000 + } + }, + exporter => #{ + type => local + } + } + } + ); +set_special_config(_) -> + ok. + end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), ok. From 8b13d249ebf66b5dd4691ff5a3c2a0b9f5167e1b Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 18:44:10 +0300 Subject: [PATCH 107/197] fix(ft): fix detection of ee apps in mix --- apps/emqx_ft/BSL.txt | 94 ++++++++++++++++++++++++++++++++++++++++++++ mix.exs | 33 ++++++++++++---- 2 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 apps/emqx_ft/BSL.txt diff --git a/apps/emqx_ft/BSL.txt b/apps/emqx_ft/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_ft/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/mix.exs b/mix.exs index 8cda3b9c0..e5e91b491 100644 --- a/mix.exs +++ b/mix.exs @@ -107,13 +107,12 @@ defmodule EMQXUmbrella.MixProject do end defp emqx_apps(profile_info, version) do - apps = umbrella_apps() ++ enterprise_apps(profile_info) + apps = community_apps() ++ enterprise_apps(profile_info) set_emqx_app_system_env(apps, profile_info, version) end - defp umbrella_apps() do - "apps/*" - |> Path.wildcard() + defp community_apps() do + community_app_paths() |> Enum.map(fn path -> app = path @@ -125,9 +124,7 @@ defmodule EMQXUmbrella.MixProject do end defp enterprise_apps(_profile_info = %{edition_type: :enterprise}) do - "lib-ee/*" - |> Path.wildcard() - |> Enum.filter(&File.dir?/1) + enterprise_app_paths() |> Enum.map(fn path -> app = path @@ -142,6 +139,28 @@ defmodule EMQXUmbrella.MixProject do [] end + defp community_app_paths() do + "apps/*" + |> Path.wildcard() + |> Enum.filter(&File.dir?/1) + |> Enum.reject(&File.exists?(Path.join(&1, "BSL.txt"))) + end + + defp enterprise_app_paths() do + lib_ee_paths = + "lib-ee/*" + |> Path.wildcard() + |> Enum.filter(&File.dir?/1) + + apps_paths = + "apps/*" + |> Path.wildcard() + |> Enum.filter(&File.dir?/1) + |> Enum.filter(&File.exists?(Path.join(&1, "BSL.txt"))) + + lib_ee_paths ++ apps_paths + end + defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do [ {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"}, From 918bdcae7d4cbf556a158c335552dbf0d7eb5845 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 21:10:07 +0300 Subject: [PATCH 108/197] fix(ft): fix schema descriptions --- apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 16 +++++++++++++++- apps/emqx_ft/src/emqx_ft_schema.erl | 1 + apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf index c33ea447e..15cf17a39 100644 --- a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -119,9 +119,23 @@ emqx_ft_schema { zh: "" } label: { - en: "GC Interval" + en: "Max segment TTL" zh: "" } } + storage_gc_min_segments_ttl { + desc { + en: "Minimum TTL of a segment kept in the local file system.
" + "This is a hard limit: no segment will be garbage collected before reaching this TTL, " + "even if some file transfer specifies a TTL less than that." + zh: "" + } + label: { + en: "Min segment TTL" + zh: "" + } + } + + } diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 270543bc9..28970f238 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -163,6 +163,7 @@ fields(local_storage_segments_gc) -> mk( emqx_schema:duration_s(), #{ + desc => ?DESC("storage_gc_min_segments_ttl"), required => false, default => "5m", % NOTE diff --git a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf index c26fa477a..9e9a52a41 100644 --- a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf +++ b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf @@ -1,7 +1,7 @@ emqx_s3_schema { access_key_id { desc { - en: "The access key id of the S3 bucket." + en: "The access key ID of the S3 bucket." zh: "" } label { From 4a144044b7c9a394a42d920bc2fc5d9a02aac4b5 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 21:10:32 +0300 Subject: [PATCH 109/197] fix(ft): set default ft config in tests --- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 24 +++---------------- apps/emqx_ft/test/emqx_ft_responder_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 15 +++++++----- 3 files changed, 13 insertions(+), 28 deletions(-) diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index 5788e4171..c616681dd 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -26,30 +26,12 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> _ = emqx_config:save_schema_mod_and_names(emqx_ft_schema), - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], fun set_special_config/1), + ok = emqx_common_test_helpers:start_apps( + [emqx_conf, emqx_ft], emqx_ft_test_helpers:env_handler(Config) + ), {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. -set_special_config(emqx_ft) -> - emqx_config:put( - [file_transfer], - #{ - storage => #{ - type => local, - segments => #{ - gc => #{ - interval => 60000 - } - }, - exporter => #{ - type => local - } - } - } - ); -set_special_config(_) -> - ok. - end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), ok. diff --git a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl index e447ba03b..751861206 100644 --- a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl @@ -24,7 +24,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_ft]), + ok = emqx_common_test_helpers:start_apps([emqx_ft], emqx_ft_test_helpers:env_handler(Config)), Config. end_per_suite(_Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index b8ee45b15..1e78f072e 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -28,12 +28,7 @@ start_additional_node(Config, Name) -> {apps, [emqx_ft]}, {join_to, node()}, {configure_gen_rpc, true}, - {env_handler, fun - (emqx_ft) -> - load_config(#{storage => local_storage(Config)}); - (_) -> - ok - end} + {env_handler, env_handler(Config)} ] ). @@ -43,6 +38,14 @@ stop_additional_node(Node) -> ok = emqx_common_test_helpers:stop_slave(Node), ok. +env_handler(Config) -> + fun + (emqx_ft) -> + load_config(#{storage => local_storage(Config)}); + (_) -> + ok + end. + local_storage(Config) -> #{ type => local, From bd7250cb1392619eb8d091f62991426d3c1e4b60 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 6 Apr 2023 00:04:10 +0300 Subject: [PATCH 110/197] fix(s3): fix hash pool type --- .../src/emqx_ft_storage_exporter_s3.erl | 44 ++++++---- apps/emqx_s3/src/emqx_s3_client.erl | 86 ++++++++++++------- apps/emqx_s3/src/emqx_s3_profile_conf.erl | 4 + apps/emqx_s3/test/emqx_s3_client_SUITE.erl | 36 ++++++-- 4 files changed, 117 insertions(+), 53 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index 24c3c52e5..63df71179 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -137,10 +137,11 @@ s3_headers({ClientId, FileId}, Filemeta) -> list(Client, Options) -> case list_key_info(Client, Options) of {ok, KeyInfos} -> - {ok, - lists:map( - fun(KeyInfo) -> key_info_to_exportinfo(Client, KeyInfo, Options) end, KeyInfos - )}; + MaybeExportInfos = lists:map( + fun(KeyInfo) -> key_info_to_exportinfo(Client, KeyInfo, Options) end, KeyInfos + ), + ExportInfos = [ExportInfo || {ok, ExportInfo} <- MaybeExportInfos], + {ok, ExportInfos}; {error, _Reason} = Error -> Error end. @@ -170,14 +171,18 @@ next_marker(KeyInfos) -> key_info_to_exportinfo(Client, KeyInfo, _Options) -> Key = proplists:get_value(key, KeyInfo), - {Transfer, Name} = parse_transfer_and_name(Key), - #{ - transfer => Transfer, - name => unicode:characters_to_binary(Name), - uri => emqx_s3_client:uri(Client, Key), - timestamp => datetime_to_epoch_second(proplists:get_value(last_modified, KeyInfo)), - size => proplists:get_value(size, KeyInfo) - }. + case parse_transfer_and_name(Key) of + {ok, {Transfer, Name}} -> + {ok, #{ + transfer => Transfer, + name => unicode:characters_to_binary(Name), + uri => emqx_s3_client:uri(Client, Key), + timestamp => datetime_to_epoch_second(proplists:get_value(last_modified, KeyInfo)), + size => proplists:get_value(size, KeyInfo) + }}; + {error, _Reason} = Error -> + Error + end. -define(EPOCH_START, 62167219200). @@ -185,8 +190,13 @@ datetime_to_epoch_second(DateTime) -> calendar:datetime_to_gregorian_seconds(DateTime) - ?EPOCH_START. parse_transfer_and_name(Key) -> - [ClientId, FileId, Name] = string:split(Key, "/", all), - Transfer = { - emqx_ft_fs_util:unescape_filename(ClientId), emqx_ft_fs_util:unescape_filename(FileId) - }, - {Transfer, Name}. + case string:split(Key, "/", all) of + [ClientId, FileId, Name] -> + Transfer = { + emqx_ft_fs_util:unescape_filename(ClientId), + emqx_ft_fs_util:unescape_filename(FileId) + }, + {ok, {Transfer, Name}}; + _ -> + {error, invalid_key} + end. diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index b6578a7e1..bfe87a602 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -38,6 +38,8 @@ -type part_number() :: non_neg_integer(). -type upload_id() :: string(). -type etag() :: string(). +-type http_pool() :: ehttpc:pool_name(). +-type pool_type() :: random | hash. -type upload_options() :: list({acl, emqx_s3:acl()}). -opaque client() :: #{ @@ -59,6 +61,7 @@ access_key_id := string() | undefined, secret_access_key := string() | undefined, http_pool := ehttpc:pool_name(), + pool_type := pool_type(), request_timeout := timeout() | undefined, max_retries := non_neg_integer() | undefined }. @@ -79,7 +82,8 @@ create(Config) -> upload_options => upload_options(Config), bucket => maps:get(bucket, Config), url_expire_time => maps:get(url_expire_time, Config), - headers => headers(Config) + headers => headers(Config), + pool_type => maps:get(pool_type, Config) }. -spec put_object(client(), key(), iodata()) -> ok_or_error(term()). @@ -211,6 +215,7 @@ aws_config(#{ access_key_id := AccessKeyId, secret_access_key := SecretAccessKey, http_pool := HttpPool, + pool_type := PoolType, request_timeout := Timeout, max_retries := MaxRetries }) -> @@ -224,7 +229,9 @@ aws_config(#{ access_key_id = AccessKeyId, secret_access_key = SecretAccessKey, - http_client = request_fun(HttpPool, with_default(MaxRetries, ?DEFAULT_MAX_RETRIES)), + http_client = request_fun( + HttpPool, PoolType, with_default(MaxRetries, ?DEFAULT_MAX_RETRIES) + ), %% This value will be transparently passed to ehttpc timeout = with_default(Timeout, ?DEFAULT_REQUEST_TIMEOUT), @@ -232,55 +239,63 @@ aws_config(#{ retry_num = 1 }. --type http_pool() :: term(). - --spec request_fun(http_pool(), non_neg_integer()) -> erlcloud_httpc:request_fun(). -request_fun(HttpPool, MaxRetries) -> +-spec request_fun(http_pool(), pool_type(), non_neg_integer()) -> erlcloud_httpc:request_fun(). +request_fun(HttpPool, PoolType, MaxRetries) -> fun(Url, Method, Headers, Body, Timeout, _Config) -> with_path_and_query_only(Url, fun(PathQuery) -> Request = make_request( Method, PathQuery, headers_erlcloud_request_to_ehttpc(Headers), Body ), - ehttpc_request(HttpPool, Method, Request, Timeout, MaxRetries) + case pick_worker_safe(HttpPool, PoolType) of + {ok, Worker} -> + ehttpc_request(Worker, Method, Request, Timeout, MaxRetries); + {error, Reason} -> + ?SLOG(error, #{ + msg => "s3_request_fun_fail", + reason => Reason, + http_pool => HttpPool, + pool_type => PoolType, + method => Method, + request => Request, + timeout => Timeout, + max_retries => MaxRetries + }), + {error, Reason} + end end) end. ehttpc_request(HttpPool, Method, Request, Timeout, MaxRetries) -> - ?SLOG(debug, #{ - msg => "s3_ehttpc_request", - timeout => Timeout, - pool => HttpPool, - method => Method, - max_retries => MaxRetries, - request => format_request(Request) - }), - try ehttpc:request(HttpPool, Method, Request, Timeout, MaxRetries) of - {ok, StatusCode, RespHeaders} -> - ?SLOG(debug, #{ - msg => "s3_ehttpc_request_ok", - status_code => StatusCode, - headers => RespHeaders - }), - {ok, { - {StatusCode, undefined}, headers_ehttpc_to_erlcloud_response(RespHeaders), undefined - }}; - {ok, StatusCode, RespHeaders, RespBody} -> + try timer:tc(fun() -> ehttpc:request(HttpPool, Method, Request, Timeout, MaxRetries) end) of + {Time, {ok, StatusCode, RespHeaders}} -> ?SLOG(debug, #{ msg => "s3_ehttpc_request_ok", status_code => StatusCode, headers => RespHeaders, - body => RespBody + time => Time + }), + {ok, { + {StatusCode, undefined}, headers_ehttpc_to_erlcloud_response(RespHeaders), undefined + }}; + {Time, {ok, StatusCode, RespHeaders, RespBody}} -> + ?SLOG(debug, #{ + msg => "s3_ehttpc_request_ok", + status_code => StatusCode, + headers => RespHeaders, + body => RespBody, + time => Time }), {ok, { {StatusCode, undefined}, headers_ehttpc_to_erlcloud_response(RespHeaders), RespBody }}; - {error, Reason} -> + {Time, {error, Reason}} -> ?SLOG(error, #{ msg => "s3_ehttpc_request_fail", reason => Reason, timeout => Timeout, pool => HttpPool, - method => Method + method => Method, + time => Time }), {error, Reason} catch @@ -304,6 +319,19 @@ ehttpc_request(HttpPool, Method, Request, Timeout, MaxRetries) -> {error, Reason} end. +pick_worker_safe(HttpPool, PoolType) -> + try + {ok, pick_worker(HttpPool, PoolType)} + catch + error:badarg -> + {error, no_ehttpc_pool} + end. + +pick_worker(HttpPool, random) -> + ehttpc_pool:pick_worker(HttpPool); +pick_worker(HttpPool, hash) -> + ehttpc_pool:pick_worker(HttpPool, self()). + -define(IS_BODY_EMPTY(Body), (Body =:= undefined orelse Body =:= <<>>)). -define(NEEDS_NO_BODY(Method), (Method =:= get orelse Method =:= head orelse Method =:= delete)). diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index 13efb9d74..e11dac530 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -204,6 +204,7 @@ client_config(ProfileConfig, PoolName) -> secret_access_key => maps:get(secret_access_key, ProfileConfig, undefined), request_timeout => maps:get(request_timeout, HTTPOpts, undefined), max_retries => maps:get(max_retries, HTTPOpts, undefined), + pool_type => maps:get(pool_type, HTTPOpts, random), http_pool => PoolName }. @@ -371,9 +372,12 @@ stop_http_pool(ProfileId, PoolName) -> ok = ?tp(debug, "s3_stop_http_pool", #{pool_name => PoolName}). do_start_http_pool(PoolName, HttpConfig) -> + ?SLOG(warning, #{msg => "s3_start_http_pool", pool_name => PoolName, config => HttpConfig}), case ehttpc_sup:start_pool(PoolName, HttpConfig) of {ok, _} -> + ?SLOG(warning, #{msg => "s3_start_http_pool_success", pool_name => PoolName}), ok; {error, _} = Error -> + ?SLOG(error, #{msg => "s3_start_http_pool_fail", pool_name => PoolName, error => Error}), Error end. diff --git a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl index ec7d5ebcf..4af803c67 100644 --- a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl @@ -20,9 +20,15 @@ all() -> groups() -> AllCases = emqx_common_test_helpers:all(?MODULE), + PoolGroups = [ + {group, pool_random}, + {group, pool_hash} + ], [ - {tcp, [], AllCases}, - {tls, [], AllCases} + {tcp, [], PoolGroups}, + {tls, [], PoolGroups}, + {pool_random, [], AllCases}, + {pool_hash, [], AllCases} ]. init_per_suite(Config) -> @@ -32,8 +38,17 @@ init_per_suite(Config) -> end_per_suite(_Config) -> ok = application:stop(emqx_s3). -init_per_group(ConnType, Config) -> - [{conn_type, ConnType} | Config]. +init_per_group(ConnTypeGroup, Config) when ConnTypeGroup =:= tcp; ConnTypeGroup =:= tls -> + [{conn_type, ConnTypeGroup} | Config]; +init_per_group(PoolTypeGroup, Config) when + PoolTypeGroup =:= pool_random; PoolTypeGroup =:= pool_hash +-> + PoolType = + case PoolTypeGroup of + pool_random -> random; + pool_hash -> hash + end, + [{pool_type, PoolType} | Config]. end_per_group(_ConnType, _Config) -> ok. @@ -127,11 +142,18 @@ client(Config) -> emqx_s3_client:create(ClientConfig). profile_config(Config) -> - maps:put( + ProfileConfig0 = emqx_s3_test_helpers:base_config(?config(conn_type, Config)), + ProfileConfig1 = maps:put( bucket, ?config(bucket, Config), - emqx_s3_test_helpers:base_config(?config(conn_type, Config)) - ). + ProfileConfig0 + ), + ProfileConfig2 = emqx_map_lib:deep_put( + [transport_options, pool_type], + ProfileConfig1, + ?config(pool_type, Config) + ), + ProfileConfig2. data(Size) -> iolist_to_binary([$a || _ <- lists:seq(1, Size)]). From eae2478678999ca4b569365fe04deb8aebc49972 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 6 Apr 2023 13:20:41 +0300 Subject: [PATCH 111/197] fix(ft): add descriptions for missing ft schema fields --- apps/emqx_s3/src/emqx_s3_schema.erl | 7 +++- .../emqx_ee_conf/src/emqx_ee_conf_schema.erl | 34 +++++++++++-------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index 6db9a5886..bdd88a9e8 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -9,7 +9,7 @@ -import(hoconsc, [mk/2, ref/2]). --export([roots/0, fields/1, namespace/0, tags/0]). +-export([roots/0, fields/1, namespace/0, tags/0, desc/1]). -export([translate/1]). @@ -125,6 +125,11 @@ fields(transport_options) -> [headers, max_retries, request_timeout], emqx_connector_http:fields("request") ). +desc(s3) -> + "S3 connection options"; +desc(transport_options) -> + "Options for the HTTP transport layer used by the S3 client". + translate(Conf) -> Options = #{atom_key => true}, #{s3 := TranslatedConf} = hocon_tconf:check_plain( diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl index 8e2800c68..907db70d9 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl @@ -6,10 +6,11 @@ -behaviour(hocon_schema). --export([namespace/0, roots/0, fields/1, translations/0, translation/1]). +-export([namespace/0, roots/0, fields/1, translations/0, translation/1, desc/1]). -define(EE_SCHEMA_MODULES, [ emqx_license_schema, + emqx_s3_schema, emqx_ft_schema ]). @@ -17,16 +18,10 @@ namespace() -> emqx_conf_schema:namespace(). roots() -> - lists:foldl( - fun(Module, Roots) -> - Roots ++ apply(Module, roots, []) - end, - emqx_conf_schema:roots(), - ?EE_SCHEMA_MODULES - ). + emqx_conf_schema:roots() ++ ee_roots(). fields(Name) -> - ee_fields(?EE_SCHEMA_MODULES, Name). + ee_delegate(fields, ?EE_SCHEMA_MODULES, Name). translations() -> emqx_conf_schema:translations(). @@ -34,16 +29,27 @@ translations() -> translation(Name) -> emqx_conf_schema:translation(Name). +desc(Name) -> + ee_delegate(desc, ?EE_SCHEMA_MODULES, Name). + %%------------------------------------------------------------------------------ %% helpers %%------------------------------------------------------------------------------ -ee_fields([EEMod | EEMods], Name) -> +ee_roots() -> + lists:flatmap( + fun(Module) -> + apply(Module, roots, []) + end, + ?EE_SCHEMA_MODULES + ). + +ee_delegate(Method, [EEMod | EEMods], Name) -> case lists:member(Name, apply(EEMod, roots, [])) of true -> - apply(EEMod, fields, [Name]); + EEMod:Method(Name); false -> - ee_fields(EEMods, Name) + ee_delegate(Method, EEMods, Name) end; -ee_fields([], Name) -> - emqx_conf_schema:fields(Name). +ee_delegate(Method, [], Name) -> + emqx_conf_schema:Method(Name). From 52f3189779070099ea8a6fa091660ff347ef28bb Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 6 Apr 2023 13:42:06 +0300 Subject: [PATCH 112/197] fix(ft): fix default ft config --- apps/emqx_ft/etc/emqx_ft.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_ft/etc/emqx_ft.conf b/apps/emqx_ft/etc/emqx_ft.conf index 2058d24f5..ec48d4e69 100644 --- a/apps/emqx_ft/etc/emqx_ft.conf +++ b/apps/emqx_ft/etc/emqx_ft.conf @@ -4,7 +4,7 @@ file_transfer { segments { root = "{{ platform_data_dir }}/file_transfer/segments" gc { - interval = 60000 + interval = "1h" } } exporter { From d21daae7aeb93fffc568b58b8ee7b7e88d3a6fb2 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 6 Apr 2023 13:43:17 +0300 Subject: [PATCH 113/197] fix(ft): fix dynamic function calls --- lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl index 907db70d9..08a6aa971 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl @@ -47,9 +47,9 @@ ee_roots() -> ee_delegate(Method, [EEMod | EEMods], Name) -> case lists:member(Name, apply(EEMod, roots, [])) of true -> - EEMod:Method(Name); + apply(EEMod, Method, [Name]); false -> ee_delegate(Method, EEMods, Name) end; ee_delegate(Method, [], Name) -> - emqx_conf_schema:Method(Name). + apply(emqx_conf_schema, Method, [Name]). From 6fbc7dc822eef92a69e8c36096fab701ea436e76 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 6 Apr 2023 14:11:02 +0300 Subject: [PATCH 114/197] fix(ft): remove s3 roots from ee schema --- lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl index 08a6aa971..df3382df1 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl @@ -10,7 +10,6 @@ -define(EE_SCHEMA_MODULES, [ emqx_license_schema, - emqx_s3_schema, emqx_ft_schema ]). From 7eeba326195b7183a73528edd5d438c78d3047e7 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 6 Apr 2023 21:53:12 +0300 Subject: [PATCH 115/197] fix(s3): fix typings --- apps/emqx_ft/src/emqx_ft.erl | 2 +- apps/emqx_s3/src/emqx_s3_client.erl | 9 +++++---- apps/emqx_s3/src/emqx_s3_profile_conf.erl | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 45ec2b933..c055680b0 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -82,7 +82,7 @@ -type segment() :: {offset(), _Content :: binary()}. -define(STORE_SEGMENT_TIMEOUT, 10000). --define(ASSEMBLE_TIMEOUT, 60000). +-define(ASSEMBLE_TIMEOUT, 300000). %%-------------------------------------------------------------------- %% API for app diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index bfe87a602..c01b1b240 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -47,7 +47,8 @@ upload_options := upload_options(), bucket := string(), headers := erlcloud_headers(), - url_expire_time := non_neg_integer() + url_expire_time := non_neg_integer(), + pool_type := pool_type() }. -type config() :: #{ @@ -60,7 +61,7 @@ url_expire_time := pos_integer(), access_key_id := string() | undefined, secret_access_key := string() | undefined, - http_pool := ehttpc:pool_name(), + http_pool := http_pool(), pool_type := pool_type(), request_timeout := timeout() | undefined, max_retries := non_neg_integer() | undefined @@ -268,7 +269,7 @@ request_fun(HttpPool, PoolType, MaxRetries) -> ehttpc_request(HttpPool, Method, Request, Timeout, MaxRetries) -> try timer:tc(fun() -> ehttpc:request(HttpPool, Method, Request, Timeout, MaxRetries) end) of {Time, {ok, StatusCode, RespHeaders}} -> - ?SLOG(debug, #{ + ?SLOG(info, #{ msg => "s3_ehttpc_request_ok", status_code => StatusCode, headers => RespHeaders, @@ -278,7 +279,7 @@ ehttpc_request(HttpPool, Method, Request, Timeout, MaxRetries) -> {StatusCode, undefined}, headers_ehttpc_to_erlcloud_response(RespHeaders), undefined }}; {Time, {ok, StatusCode, RespHeaders, RespBody}} -> - ?SLOG(debug, #{ + ?SLOG(info, #{ msg => "s3_ehttpc_request_ok", status_code => StatusCode, headers => RespHeaders, diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index e11dac530..006d06899 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -337,7 +337,7 @@ http_config( SSLOpts = emqx_tls_lib:to_client_opts(maps:get(ssl, HTTPOpts)), {tls, SSLOpts} end, - NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), + % NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), [ {host, Host}, {port, Port}, @@ -346,7 +346,7 @@ http_config( {pool_type, PoolType}, {pool_size, PoolSize}, {transport, Transport}, - {transport_opts, NTransportOpts}, + {transport_opts, TransportOpts}, {enable_pipelining, EnablePipelining} ]. From b2fd2dcbc114122336b196c6109c71955bc8ebd1 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 7 Apr 2023 01:58:25 +0300 Subject: [PATCH 116/197] fix(ft): make configs user friendlier --- apps/emqx_ft/src/emqx_ft_conf.erl | 27 +++++++++++++++++----- apps/emqx_ft/src/emqx_ft_schema.erl | 17 +++++++++++--- apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf | 10 ++++++++ apps/emqx_s3/src/emqx_s3.erl | 1 + apps/emqx_s3/src/emqx_s3_profile_conf.erl | 9 ++++++-- apps/emqx_s3/src/emqx_s3_schema.erl | 17 +++++++++++--- 6 files changed, 67 insertions(+), 14 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 0e8bcc193..5ca0b85ed 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -42,25 +42,40 @@ -type milliseconds() :: non_neg_integer(). -type seconds() :: non_neg_integer(). +%% 5 minutes (s) +-define(DEFAULT_MIN_SEGMENTS_TTL, 300). +%% 1 day (s) +-define(DEFAULT_MAX_SEGMENTS_TTL, 86400). +%% 1 minute (ms) +-define(DEFAULT_GC_INTERVAL, 60000). + %%-------------------------------------------------------------------- %% Accessors %%-------------------------------------------------------------------- --spec storage() -> _Storage | disabled. +-spec storage() -> _Storage. storage() -> - emqx_config:get([file_transfer, storage], disabled). + emqx_config:get([file_transfer, storage]). -spec gc_interval(_Storage) -> milliseconds(). gc_interval(_Storage) -> Conf = assert_storage(local), - emqx_map_lib:deep_get([segments, gc, interval], Conf). + emqx_map_lib:deep_get([segments, gc, interval], Conf, ?DEFAULT_GC_INTERVAL). -spec segments_ttl(_Storage) -> {_Min :: seconds(), _Max :: seconds()}. segments_ttl(_Storage) -> Conf = assert_storage(local), { - emqx_map_lib:deep_get([segments, gc, minimum_segments_ttl], Conf), - emqx_map_lib:deep_get([segments, gc, maximum_segments_ttl], Conf) + emqx_map_lib:deep_get( + [segments, gc, minimum_segments_ttl], + Conf, + ?DEFAULT_MIN_SEGMENTS_TTL + ), + emqx_map_lib:deep_get( + [segments, gc, maximum_segments_ttl], + Conf, + ?DEFAULT_MAX_SEGMENTS_TTL + ) }. assert_storage(Type) -> @@ -79,7 +94,7 @@ assert_storage(Type) -> load() -> ok = emqx_ft_storage_exporter:update_exporter( undefined, - emqx_config:get([file_transfer, storage]) + storage() ), emqx_conf:add_handler([file_transfer], ?MODULE). diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 28970f238..f22b93c6d 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -52,8 +52,9 @@ fields(file_transfer) -> ref(local_storage) ]), #{ - required => true, - desc => ?DESC("storage") + required => false, + desc => ?DESC("storage"), + default => default_storage() } )} ]; @@ -184,7 +185,9 @@ desc(local_storage_exporter) -> desc(s3_exporter) -> "S3 Exporter settings for the File transfer local storage backend"; desc(local_storage_segments_gc) -> - "Garbage collection settings for the File transfer local segments storage". + "Garbage collection settings for the File transfer local segments storage"; +desc(_) -> + undefined. schema(filemeta) -> #{ @@ -221,3 +224,11 @@ converter(checksum) -> ref(Ref) -> ref(?MODULE, Ref). + +default_storage() -> + #{ + <<"type">> => <<"local">>, + <<"exporter">> => #{ + <<"type">> => <<"local">> + } + }. diff --git a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf index 9e9a52a41..c391fa352 100644 --- a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf +++ b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf @@ -101,4 +101,14 @@ S3 uploader won't try to upload parts larger than this size.""" zh: "" } } + ipv6_probe { + desc { + en: "Whether to probe for IPv6 support." + zh: "" + } + label { + en: "IPv6 probe" + zh: "" + } + } } diff --git a/apps/emqx_s3/src/emqx_s3.erl b/apps/emqx_s3/src/emqx_s3.erl index 15f63fbd5..0c2592736 100644 --- a/apps/emqx_s3/src/emqx_s3.erl +++ b/apps/emqx_s3/src/emqx_s3.erl @@ -37,6 +37,7 @@ max_retries => pos_integer(), pool_size => pos_integer(), pool_type => atom(), + ipv6_probe => boolean(), ssl => map() }. diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index 006d06899..3d9d6ed3f 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -337,7 +337,7 @@ http_config( SSLOpts = emqx_tls_lib:to_client_opts(maps:get(ssl, HTTPOpts)), {tls, SSLOpts} end, - % NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), + NTransportOpts = maybe_ipv6_probe(TransportOpts, maps:get(ipv6_probe, HTTPOpts, true)), [ {host, Host}, {port, Port}, @@ -346,10 +346,15 @@ http_config( {pool_type, PoolType}, {pool_size, PoolSize}, {transport, Transport}, - {transport_opts, TransportOpts}, + {transport_opts, NTransportOpts}, {enable_pipelining, EnablePipelining} ]. +maybe_ipv6_probe(TransportOpts, true) -> + emqx_misc:ipv6_probe(TransportOpts); +maybe_ipv6_probe(TransportOpts, false) -> + TransportOpts. + http_pool_cleanup_interval(ProfileConfig) -> maps:get( http_pool_cleanup_interval, ProfileConfig, ?DEAFULT_HTTP_POOL_CLEANUP_INTERVAL diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index bdd88a9e8..4d017deed 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -118,9 +118,20 @@ fields(s3) -> )} ]; fields(transport_options) -> - props_without( - [base_url, max_retries, retry_interval, request], emqx_connector_http:fields(config) - ) ++ + [ + {ipv6_probe, + mk( + boolean(), + #{ + default => true, + desc => ?DESC("ipv6_probe"), + required => false + } + )} + ] ++ + props_without( + [base_url, max_retries, retry_interval, request], emqx_connector_http:fields(config) + ) ++ props_with( [headers, max_retries, request_timeout], emqx_connector_http:fields("request") ). From 0d86ef8d3a87a98efa6479d57b61aa853a5f41ef Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 7 Apr 2023 21:05:09 +0300 Subject: [PATCH 117/197] fix(ft-s3): use AWS4 signed urls for S3 export URIs This will ensure that the S3 export URIs are valid in all AWS regions. --- apps/emqx_s3/rebar.config | 2 +- apps/emqx_s3/src/emqx_s3_client.erl | 2 +- apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl | 57 ++++++++++++++++++++ lib-ee/emqx_ee_connector/rebar.config | 2 +- mix.exs | 4 +- 5 files changed, 62 insertions(+), 5 deletions(-) diff --git a/apps/emqx_s3/rebar.config b/apps/emqx_s3/rebar.config index 92ab7895e..b1483e028 100644 --- a/apps/emqx_s3/rebar.config +++ b/apps/emqx_s3/rebar.config @@ -1,6 +1,6 @@ {deps, [ {emqx, {path, "../../apps/emqx"}}, - {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.7-emqx-1"}}} + {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.8-emqx-1"}}} ]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index c01b1b240..6bb6c2e3b 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -189,7 +189,7 @@ list(#{bucket := Bucket, aws_config := AwsConfig}, Options) -> -spec uri(client(), key()) -> iodata(). uri(#{bucket := Bucket, aws_config := AwsConfig, url_expire_time := ExpireTime}, Key) -> - erlcloud_s3:make_get_url(ExpireTime, Bucket, erlcloud_key(Key), AwsConfig). + erlcloud_s3:make_presigned_v4_url(ExpireTime, Bucket, get, erlcloud_key(Key), [], AwsConfig). -spec format(client()) -> term(). format(#{aws_config := AwsConfig} = Client) -> diff --git a/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl index 5fda42fbe..e09ea2773 100644 --- a/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl @@ -54,6 +54,8 @@ groups() -> t_happy_path_multi, t_abort_multi, t_abort_simple_put, + t_signed_url_download, + t_signed_nonascii_url_download, {group, noconn_errors}, {group, timeout_errors}, @@ -193,6 +195,40 @@ t_happy_path_multi(Config) -> Key ). +t_signed_url_download(_Config) -> + Prefix = emqx_s3_test_helpers:unique_key(), + Key = Prefix ++ "/ascii.txt", + + {ok, Data} = upload(Key, 1024, 5), + + SignedUrl = emqx_s3:with_client(profile_id(), fun(Client) -> + emqx_s3_client:uri(Client, Key) + end), + + {ok, {_, _, Body}} = httpc:request(get, {SignedUrl, []}, [], []), + + ?assertEqual( + iolist_to_binary(Data), + iolist_to_binary(Body) + ). + +t_signed_nonascii_url_download(_Config) -> + Prefix = emqx_s3_test_helpers:unique_key(), + Key = Prefix ++ "/unicode-🫠.txt", + + {ok, Data} = upload(Key, 1024 * 1024, 8), + + SignedUrl = emqx_s3:with_client(profile_id(), fun(Client) -> + emqx_s3_client:uri(Client, Key) + end), + + {ok, {_, _, Body}} = httpc:request(get, {SignedUrl, []}, [], []), + + ?assertEqual( + iolist_to_binary(Data), + iolist_to_binary(Body) + ). + t_abort_multi(Config) -> Key = emqx_s3_test_helpers:unique_key(), {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), @@ -532,3 +568,24 @@ data(Byte, ChunkSize, ChunkCount) -> list_objects(Config) -> Props = erlcloud_s3:list_objects(?config(bucket, Config), [], ?config(test_aws_config, Config)), proplists:get_value(contents, Props). + +upload(Key, ChunkSize, ChunkCount) -> + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + Data = data($a, ChunkSize, ChunkCount), + + ok = lists:foreach( + fun(Chunk) -> ?assertEqual(ok, emqx_s3_uploader:write(Pid, Chunk)) end, + Data + ), + + ok = emqx_s3_uploader:complete(Pid), + + ok = ?assertProcessExited( + normal, + Pid + ), + + {ok, Data}. diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index f36eb0ad6..654b1bf54 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -4,7 +4,7 @@ {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, - {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.7-emqx-1"}}}, + {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.8-emqx-1"}}}, {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, {emqx, {path, "../../apps/emqx"}} ]}. diff --git a/mix.exs b/mix.exs index e5e91b491..4f3d5a217 100644 --- a/mix.exs +++ b/mix.exs @@ -94,10 +94,10 @@ defmodule EMQXUmbrella.MixProject do # in conflict by grpc and eetcd {:gpb, "4.19.5", override: true, runtime: false}, {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true}, - {:erlcloud, github: "emqx/erlcloud", tag: "3.6.7-emqx-1", override: true}, + {:erlcloud, github: "emqx/erlcloud", tag: "3.6.8-emqx-1", override: true}, # erlcloud's rebar.config requires rebar3 and does not support Mix, # so it tries to fetch deps from git. We need to override this. - {:lhttpc, "1.6.2", override: true}, + {:lhttpc, github: "https://github.com/erlcloud/lhttpc", tag: "1.6.2", override: true}, {:eini, "1.2.9", override: true}, {:base16, "1.0.0", override: true} # end of erlcloud's deps From 8daa38ef06a7091cde479f4ecec12e3d71482829 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 7 Apr 2023 21:58:27 +0300 Subject: [PATCH 118/197] feat(ft-s3): store metadata in ASCII-safe format Also ensure consistent encoding and decoding filenames throughout the `emqx_ft` application. --- apps/emqx_ft/src/emqx_ft_schema.erl | 14 ++++- .../src/emqx_ft_storage_exporter_s3.erl | 5 +- apps/emqx_ft/test/emqx_ft_SUITE.erl | 56 +++++++++---------- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 6 +- mix.exs | 2 +- 5 files changed, 49 insertions(+), 34 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index f22b93c6d..beb43adc3 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -195,7 +195,8 @@ schema(filemeta) -> {name, hoconsc:mk(string(), #{ required => true, - validator => validator(filename) + validator => validator(filename), + converter => converter(unicode_string) })}, {size, hoconsc:mk(non_neg_integer())}, {expire_at, hoconsc:mk(non_neg_integer())}, @@ -220,6 +221,17 @@ converter(checksum) -> _ = is_binary(Hex) orelse throw({expected_type, string}), _ = byte_size(Hex) =:= 64 orelse throw({expected_length, 64}), {sha256, binary:decode_hex(Hex)} + end; +converter(unicode_string) -> + fun + (undefined, #{}) -> + undefined; + (Str, #{make_serializable := true}) -> + _ = is_list(Str) orelse throw({expected_type, string}), + unicode:characters_to_binary(Str); + (Str, #{}) -> + _ = is_binary(Str) orelse throw({expected_type, string}), + unicode:characters_to_list(Str) end. ref(Ref) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index 63df71179..259cb4016 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -130,10 +130,13 @@ s3_headers({ClientId, FileId}, Filemeta) -> <<"x-amz-meta-clientid">> => ClientId, %% It [Topic Name] MUST be a UTF-8 Encoded String <<"x-amz-meta-fileid">> => FileId, - <<"x-amz-meta-filemeta">> => emqx_json:encode(emqx_ft:encode_filemeta(Filemeta)), + <<"x-amz-meta-filemeta">> => s3_header_filemeta(Filemeta), <<"x-amz-meta-filemeta-vsn">> => ?FILEMETA_VSN }. +s3_header_filemeta(Filemeta) -> + emqx_json:encode(emqx_ft:encode_filemeta(Filemeta), [force_utf8, uescape]). + list(Client, Options) -> case list_key_info(Client, Options) of {ok, KeyInfos} -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index e365cba85..c3ee7f271 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -169,39 +169,38 @@ t_invalid_filename(Config) -> C = ?config(client, Config), ?assertRCName( unspecified_error, - emqtt:publish(C, mk_init_topic(<<"f1">>), emqx_json:encode(meta(".", <<>>)), 1) + emqtt:publish(C, mk_init_topic(<<"f1">>), encode_meta(meta(".", <<>>)), 1) ), ?assertRCName( unspecified_error, - emqtt:publish(C, mk_init_topic(<<"f2">>), emqx_json:encode(meta("..", <<>>)), 1) + emqtt:publish(C, mk_init_topic(<<"f2">>), encode_meta(meta("..", <<>>)), 1) ), ?assertRCName( unspecified_error, - emqtt:publish(C, mk_init_topic(<<"f2">>), emqx_json:encode(meta("../nice", <<>>)), 1) + emqtt:publish(C, mk_init_topic(<<"f2">>), encode_meta(meta("../nice", <<>>)), 1) ), ?assertRCName( unspecified_error, - emqtt:publish(C, mk_init_topic(<<"f3">>), emqx_json:encode(meta("/etc/passwd", <<>>)), 1) + emqtt:publish(C, mk_init_topic(<<"f3">>), encode_meta(meta("/etc/passwd", <<>>)), 1) ), ?assertRCName( success, - emqtt:publish(C, mk_init_topic(<<"f4">>), emqx_json:encode(meta("146%", <<>>)), 1) + emqtt:publish(C, mk_init_topic(<<"f4">>), encode_meta(meta("146%", <<>>)), 1) ). t_simple_transfer(Config) -> C = ?config(client, Config), - Filename = <<"topsecret.pdf">>, + Filename = "topsecret.pdf", FileId = <<"f1">>, Data = [<<"first">>, <<"second">>, <<"third">>], Meta = #{size := Filesize} = meta(Filename, Data), - MetaPayload = emqx_json:encode(Meta), ?assertRCName( success, - emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1) ), lists:foreach( @@ -246,23 +245,21 @@ t_nasty_clientids_fileids(_Config) -> t_meta_conflict(Config) -> C = ?config(client, Config), - Filename = <<"topsecret.pdf">>, + Filename = "topsecret.pdf", FileId = <<"f1">>, Meta = meta(Filename, [<<"x">>]), - MetaPayload = emqx_json:encode(Meta), ?assertRCName( success, - emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1) ), - ConflictMeta = Meta#{name => <<"conflict.pdf">>}, - ConflictMetaPayload = emqx_json:encode(ConflictMeta), + ConflictMeta = Meta#{name => "conflict.pdf"}, ?assertRCName( unspecified_error, - emqtt:publish(C, mk_init_topic(FileId), ConflictMetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), encode_meta(ConflictMeta), 1) ). t_no_meta(Config) -> @@ -284,17 +281,16 @@ t_no_meta(Config) -> t_no_segment(Config) -> C = ?config(client, Config), - Filename = <<"topsecret.pdf">>, + Filename = "topsecret.pdf", FileId = <<"f1">>, Data = [<<"first">>, <<"second">>, <<"third">>], Meta = #{size := Filesize} = meta(Filename, Data), - MetaPayload = emqx_json:encode(Meta), ?assertRCName( success, - emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1) ), lists:foreach( @@ -335,13 +331,13 @@ t_invalid_meta(Config) -> t_invalid_checksum(Config) -> C = ?config(client, Config), - Filename = <<"topsecret.pdf">>, + Filename = "topsecret.pdf", FileId = <<"f1">>, Data = [<<"first">>, <<"second">>, <<"third">>], Meta = #{size := Filesize} = meta(Filename, Data), - MetaPayload = emqx_json:encode(Meta#{checksum => sha256hex(<<"invalid">>)}), + MetaPayload = encode_meta(Meta#{checksum => {sha256, sha256(<<"invalid">>)}}), ?assertRCName( success, @@ -366,7 +362,7 @@ t_invalid_checksum(Config) -> t_corrupted_segment_retry(Config) -> C = ?config(client, Config), - Filename = <<"corruption.pdf">>, + Filename = "corruption.pdf", FileId = <<"4242-4242">>, Data = [<<"first">>, <<"second">>, <<"third">>], @@ -379,11 +375,11 @@ t_corrupted_segment_retry(Config) -> Checksum1, Checksum2, Checksum3 - ] = [sha256hex(S) || S <- Data], + ] = [binary:encode_hex(sha256(S)) || S <- Data], Meta = #{size := Filesize} = meta(Filename, Data), - ?assertRCName(success, emqtt:publish(C, mk_init_topic(FileId), emqx_json:encode(Meta), 1)), + ?assertRCName(success, emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1)), ?assertRCName( success, @@ -421,7 +417,7 @@ t_switch_node(Config) -> {ok, C1} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, AdditionalNodePort}]), {ok, _} = emqtt:connect(C1), - Filename = <<"multinode_upload.txt">>, + Filename = "multinode_upload.txt", FileId = <<"f1">>, Data = [<<"first">>, <<"second">>, <<"third">>], @@ -430,11 +426,10 @@ t_switch_node(Config) -> %% First, publist metadata and the first segment to the additional node Meta = #{size := Filesize} = meta(Filename, Data), - MetaPayload = emqx_json:encode(Meta), ?assertRCName( success, - emqtt:publish(C1, mk_init_topic(FileId), MetaPayload, 1) + emqtt:publish(C1, mk_init_topic(FileId), encode_meta(Meta), 1) ), ?assertRCName( success, @@ -593,7 +588,7 @@ disown_mqtt_client(Context = #{}) -> send_filemeta(Meta, Context = #{client := Client, fileid := FileId}) -> ?assertRCName( success, - emqtt:publish(Client, mk_init_topic(FileId), emqx_json:encode(Meta), 1) + emqtt:publish(Client, mk_init_topic(FileId), encode_meta(Meta), 1) ), Context. @@ -650,18 +645,21 @@ with_offsets(Items) -> ), List. -sha256hex(Data) -> - binary:encode_hex(crypto:hash(sha256, Data)). +sha256(Data) -> + crypto:hash(sha256, Data). meta(FileName, Data) -> FullData = iolist_to_binary(Data), #{ name => FileName, - checksum => sha256hex(FullData), + checksum => {sha256, sha256(FullData)}, expire_at => erlang:system_time(_Unit = second) + 3600, size => byte_size(FullData) }. +encode_meta(Meta) -> + emqx_json:encode(emqx_ft:encode_filemeta(Meta)). + list_files(ClientId) -> {ok, Files} = emqx_ft_storage:files(), [File || File = #{transfer := {CId, _}} <- Files, CId == ClientId]. diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index 1e78f072e..b7004dcc6 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -78,11 +78,13 @@ upload_file(ClientId, FileId, Name, Data, Node) -> {ok, C1} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, Port}]), {ok, _} = emqtt:connect(C1), Meta = #{ - name => unicode:characters_to_binary(Name), + name => Name, expire_at => erlang:system_time(_Unit = second) + 3600, size => Size }, - MetaPayload = emqx_json:encode(Meta), + MetaPayload = emqx_json:encode(emqx_ft:encode_filemeta(Meta)), + + ct:pal("MetaPayload = ~ts", [MetaPayload]), MetaTopic = <<"$file/", FileId/binary, "/init">>, {ok, _} = emqtt:publish(C1, MetaTopic, MetaPayload, 1), diff --git a/mix.exs b/mix.exs index 4f3d5a217..cd6d08fd1 100644 --- a/mix.exs +++ b/mix.exs @@ -97,7 +97,7 @@ defmodule EMQXUmbrella.MixProject do {:erlcloud, github: "emqx/erlcloud", tag: "3.6.8-emqx-1", override: true}, # erlcloud's rebar.config requires rebar3 and does not support Mix, # so it tries to fetch deps from git. We need to override this. - {:lhttpc, github: "https://github.com/erlcloud/lhttpc", tag: "1.6.2", override: true}, + {:lhttpc, github: "erlcloud/lhttpc", tag: "1.6.2", override: true}, {:eini, "1.2.9", override: true}, {:base16, "1.0.0", override: true} # end of erlcloud's deps From 9224b339becf5b85f49e3d1547fb99553e80a1b1 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 7 Apr 2023 22:14:06 +0300 Subject: [PATCH 119/197] fix(dep): upgrade `rulesql` back to 0.1.5 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 5cedb47ff..908adefdb 100644 --- a/rebar.config +++ b/rebar.config @@ -82,7 +82,7 @@ {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.9"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}}, - {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}}, + {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.5"}}}, % NOTE: depends on recon 2.5.x {observer_cli, "1.7.1"}, {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}, From 37a520d79773ad66a4c7641fbcbdd6b453651e8c Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 10 Apr 2023 16:02:34 +0300 Subject: [PATCH 120/197] fix(i18n): relocate i18n files to rel/i18n --- .../i18n/emqx_ft_api_i18n.conf => rel/i18n/emqx_ft_api.hocon | 0 .../emqx_ft_schema_i18n.conf => rel/i18n/emqx_ft_schema.hocon | 0 .../emqx_s3_schema_i18n.conf => rel/i18n/emqx_s3_schema.hocon | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename apps/emqx_ft/i18n/emqx_ft_api_i18n.conf => rel/i18n/emqx_ft_api.hocon (100%) rename apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf => rel/i18n/emqx_ft_schema.hocon (100%) rename apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf => rel/i18n/emqx_s3_schema.hocon (100%) diff --git a/apps/emqx_ft/i18n/emqx_ft_api_i18n.conf b/rel/i18n/emqx_ft_api.hocon similarity index 100% rename from apps/emqx_ft/i18n/emqx_ft_api_i18n.conf rename to rel/i18n/emqx_ft_api.hocon diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/rel/i18n/emqx_ft_schema.hocon similarity index 100% rename from apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf rename to rel/i18n/emqx_ft_schema.hocon diff --git a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf b/rel/i18n/emqx_s3_schema.hocon similarity index 100% rename from apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf rename to rel/i18n/emqx_s3_schema.hocon From e22c1c01ec3dad342bc12651a4a7104b31ef5ee0 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 13 Apr 2023 19:47:12 +0300 Subject: [PATCH 121/197] feat(s3): make acl optional --- apps/emqx_s3/src/emqx_s3_client.erl | 8 +++++--- apps/emqx_s3/src/emqx_s3_profile_conf.erl | 2 +- apps/emqx_s3/src/emqx_s3_schema.erl | 2 +- apps/emqx_s3/test/emqx_s3_client_SUITE.erl | 13 ++++++++++++- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index 6bb6c2e3b..e3e1d84af 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -199,10 +199,12 @@ format(#{aws_config := AwsConfig} = Client) -> %% Internal functions %%-------------------------------------------------------------------- -upload_options(Config) -> +upload_options(#{acl := Acl}) when Acl =/= undefined -> [ - {acl, maps:get(acl, Config)} - ]. + {acl, Acl} + ]; +upload_options(#{}) -> + []. headers(#{headers := Headers}) -> headers_user_to_erlcloud_request(Headers); diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index 3d9d6ed3f..3d66823a7 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -198,7 +198,7 @@ client_config(ProfileConfig, PoolName) -> port => maps:get(port, ProfileConfig), url_expire_time => maps:get(url_expire_time, ProfileConfig), headers => maps:get(headers, HTTPOpts, #{}), - acl => maps:get(acl, ProfileConfig), + acl => maps:get(acl, ProfileConfig, undefined), bucket => maps:get(bucket, ProfileConfig), access_key_id => maps:get(access_key_id, ProfileConfig, undefined), secret_access_key => maps:get(secret_access_key, ProfileConfig, undefined), diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index 4d017deed..2d714bb7d 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -105,7 +105,7 @@ fields(s3) -> #{ default => private, desc => ?DESC("acl"), - required => true + required => false } )}, {transport_options, diff --git a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl index 4af803c67..cb55bc083 100644 --- a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl @@ -60,7 +60,8 @@ init_per_testcase(_TestCase, Config0) -> ok = erlcloud_s3:create_bucket(Bucket, TestAwsConfig), Config1 = [ {key, emqx_s3_test_helpers:unique_key()}, - {bucket, Bucket} + {bucket, Bucket}, + {aws_config, TestAwsConfig} | Config0 ], {ok, PoolName} = emqx_s3_profile_conf:start_http_pool(?PROFILE_ID, profile_config(Config1)), @@ -131,6 +132,16 @@ t_url(Config) -> httpc:request(Url) ). +t_no_acl(Config) -> + Key = ?config(key, Config), + + ClientConfig = emqx_s3_profile_conf:client_config( + profile_config(Config), ?config(ehttpc_pool_name, Config) + ), + Client = emqx_s3_client:create(maps:without([acl], ClientConfig)), + + ok = emqx_s3_client:put_object(Client, Key, <<"data">>). + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- From 92ff2b7d6ee8a11d11f9f1d67b564fcb5c1e9ad5 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 13 Apr 2023 19:47:32 +0300 Subject: [PATCH 122/197] feat(ft): make timeouts configurable --- apps/emqx_ft/src/emqx_ft.erl | 9 +++----- apps/emqx_ft/src/emqx_ft_conf.erl | 12 ++++++++++ apps/emqx_ft/src/emqx_ft_schema.erl | 27 ++++++++++++++++++++++ rel/i18n/emqx_ft_schema.hocon | 36 +++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 6 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index c055680b0..6ac6596ff 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -81,9 +81,6 @@ -type segment() :: {offset(), _Content :: binary()}. --define(STORE_SEGMENT_TIMEOUT, 10000). --define(ASSEMBLE_TIMEOUT, 300000). - %%-------------------------------------------------------------------- %% API for app %%-------------------------------------------------------------------- @@ -212,7 +209,7 @@ on_init(PacketId, Msg, Transfer, Meta) -> Callback = fun(Result) -> ?MODULE:on_complete("store_filemeta", PacketKey, Transfer, Result) end, - with_responder(PacketKey, Callback, ?STORE_SEGMENT_TIMEOUT, fun() -> + with_responder(PacketKey, Callback, emqx_ft_conf:init_timeout(), fun() -> case store_filemeta(Transfer, Meta) of % Stored, ack through the responder right away ok -> @@ -245,7 +242,7 @@ on_segment(PacketId, Msg, Transfer, Offset, Checksum) -> Callback = fun(Result) -> ?MODULE:on_complete("store_segment", PacketKey, Transfer, Result) end, - with_responder(PacketKey, Callback, ?STORE_SEGMENT_TIMEOUT, fun() -> + with_responder(PacketKey, Callback, emqx_ft_conf:store_segment_timeout(), fun() -> case store_segment(Transfer, Segment) of ok -> emqx_ft_responder:ack(PacketKey, ok); @@ -271,7 +268,7 @@ on_fin(PacketId, Msg, Transfer, FinalSize, Checksum) -> Callback = fun(Result) -> ?MODULE:on_complete("assemble", FinPacketKey, Transfer, Result) end, - with_responder(FinPacketKey, Callback, ?ASSEMBLE_TIMEOUT, fun() -> + with_responder(FinPacketKey, Callback, emqx_ft_conf:assemble_timeout(), fun() -> case assemble(Transfer, FinalSize) of %% Assembling completed, ack through the responder right away % ok -> diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 5ca0b85ed..14a79b94c 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -26,6 +26,9 @@ -export([storage/0]). -export([gc_interval/1]). -export([segments_ttl/1]). +-export([init_timeout/0]). +-export([store_segment_timeout/0]). +-export([assemble_timeout/0]). %% Load/Unload -export([ @@ -86,6 +89,15 @@ assert_storage(Type) -> error({inapplicable, Conf}) end. +init_timeout() -> + emqx_config:get([file_transfer, init_timeout]). + +assemble_timeout() -> + emqx_config:get([file_transfer, assemble_timeout]). + +store_segment_timeout() -> + emqx_config:get([file_transfer, store_segment_timeout]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index beb43adc3..bc780874a 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -46,6 +46,33 @@ roots() -> [file_transfer]. fields(file_transfer) -> [ + {init_timeout, + mk( + emqx_schema:duration_ms(), + #{ + desc => ?DESC("init_timeout"), + required => false, + default => "10s" + } + )}, + {store_segment_timeout, + mk( + emqx_schema:duration_ms(), + #{ + desc => ?DESC("store_segment_timeout"), + required => false, + default => "5m" + } + )}, + {assemble_timeout, + mk( + emqx_schema:duration_ms(), + #{ + desc => ?DESC("assemble_timeout"), + required => false, + default => "5m" + } + )}, {storage, mk( hoconsc:union([ diff --git a/rel/i18n/emqx_ft_schema.hocon b/rel/i18n/emqx_ft_schema.hocon index 15cf17a39..9df02e920 100644 --- a/rel/i18n/emqx_ft_schema.hocon +++ b/rel/i18n/emqx_ft_schema.hocon @@ -1,5 +1,41 @@ emqx_ft_schema { + init_timeout { + desc { + en: "Timeout for initializing the file transfer.
" + "After reaching the timeout, `init` message will be acked with an error" + zh: "" + } + label { + en: "File Transfer Init Timeout" + zh: "" + } + } + + assemble_timeout { + desc { + en: "Timeout for assembling and exporting file segments into a final file.
" + "After reaching the timeout, `fin` message will be acked with an error" + zh: "" + } + label { + en: "File Assemble Timeout" + zh: "" + } + } + + store_segment_timeout { + desc { + en: "Timeout for storing a file segment.
" + "After reaching the timeout, message with the segment will be acked with an error" + zh: "" + } + label { + en: "Store Segment Timeout" + zh: "" + } + } + storage { desc { en: "Storage settings for file transfer." From 04523b3f8153e8c35b80c30e43e385cce8c8c4a5 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 19 Apr 2023 18:48:40 +0300 Subject: [PATCH 123/197] fix(ft): restrict max filename length in transfers Reject transfers with too long filenames right away, during `init` handling, to avoid having to deal with filesystem errors later. --- apps/emqx_ft/src/emqx_ft.erl | 6 +++--- apps/emqx_ft/src/emqx_ft_schema.erl | 15 ++++++++++++++- apps/emqx_ft/test/emqx_ft_SUITE.erl | 11 ++++++++++- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 6ac6596ff..45207baa9 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -110,8 +110,8 @@ decode_filemeta(Map) when is_map(Map) -> Meta = hocon_tconf:check_plain(Schema, Map, #{atom_key => true, required => false}), {ok, Meta} catch - throw:Error -> - {error, {invalid_filemeta, Error}} + throw:{_Schema, Errors} -> + {error, {invalid_filemeta, Errors}} end. encode_filemeta(Meta = #{}) -> @@ -381,7 +381,7 @@ do_validate([{filemeta, Payload} | Rest], Parsed) -> {ok, Meta} -> do_validate(Rest, [Meta | Parsed]); {error, Reason} -> - {error, {invalid_filemeta, Reason}} + {error, Reason} end; do_validate([{offset, Offset} | Rest], Parsed) -> case string:to_integer(Offset) of diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index bc780874a..ce3710cc1 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -35,6 +35,13 @@ -reflect_type([json_value/0]). +%% NOTE +%% This is rather conservative limit, mostly dictated by the filename limitations +%% on most filesystems. Even though, say, S3 does not have such limitations, it's +%% still useful to have a limit on the filename length, to avoid having to deal with +%% limits in the storage backends. +-define(MAX_FILENAME_BYTELEN, 255). + -import(hoconsc, [ref/2, mk/2]). namespace() -> file_transfer. @@ -234,7 +241,13 @@ schema(filemeta) -> }. validator(filename) -> - fun emqx_ft_fs_util:is_filename_safe/1. + [ + fun(Value) -> + Bin = unicode:characters_to_binary(Value), + byte_size(Bin) =< ?MAX_FILENAME_BYTELEN orelse {error, max_length_exceeded} + end, + fun emqx_ft_fs_util:is_filename_safe/1 + ]. converter(checksum) -> fun diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index c3ee7f271..3649d02f8 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -183,9 +183,18 @@ t_invalid_filename(Config) -> unspecified_error, emqtt:publish(C, mk_init_topic(<<"f3">>), encode_meta(meta("/etc/passwd", <<>>)), 1) ), + ?assertRCName( + unspecified_error, + emqtt:publish( + C, + mk_init_topic(<<"f4">>), + encode_meta(meta(lists:duplicate(1000, $A), <<>>)), + 1 + ) + ), ?assertRCName( success, - emqtt:publish(C, mk_init_topic(<<"f4">>), encode_meta(meta("146%", <<>>)), 1) + emqtt:publish(C, mk_init_topic(<<"f5">>), encode_meta(meta("146%", <<>>)), 1) ). t_simple_transfer(Config) -> From f06300cbed896c290c37b370026c6631cfb85a88 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 19 Apr 2023 19:05:12 +0300 Subject: [PATCH 124/197] fix(s3): mark S3 secrets as `sensitive` in schema --- apps/emqx_s3/src/emqx_s3_schema.erl | 9 +++++++-- apps/emqx_s3/test/emqx_s3_schema_SUITE.erl | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index 2d714bb7d..23e69ec5d 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -12,6 +12,7 @@ -export([roots/0, fields/1, namespace/0, tags/0, desc/1]). -export([translate/1]). +-export([translate/2]). roots() -> [s3]. @@ -36,7 +37,8 @@ fields(s3) -> string(), #{ desc => ?DESC("secret_access_key"), - required => false + required => false, + sensitive => true } )}, {bucket, @@ -142,7 +144,10 @@ desc(transport_options) -> "Options for the HTTP transport layer used by the S3 client". translate(Conf) -> - Options = #{atom_key => true}, + translate(Conf, #{}). + +translate(Conf, OptionsIn) -> + Options = maps:merge(#{atom_key => true}, OptionsIn), #{s3 := TranslatedConf} = hocon_tconf:check_plain( emqx_s3_schema, #{<<"s3">> => Conf}, Options, [s3] ), diff --git a/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl b/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl index bba1a5ba8..89ec8a958 100644 --- a/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl @@ -108,6 +108,25 @@ t_full_config(_Config) -> }) ). +t_sensitive_config_hidden(_Config) -> + ?assertMatch( + #{ + access_key_id := "access_key_id", + secret_access_key := <<"******">> + }, + emqx_s3_schema:translate( + #{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"access_key_id">> => <<"access_key_id">>, + <<"secret_access_key">> => <<"secret_access_key">> + }, + % NOTE: this is what Config API handler is doing + #{obfuscate_sensitive_values => true} + ) + ). + t_invalid_limits(_Config) -> ?assertException( throw, From 12a8c070d4b2bec79e613ee386f6fa2746743135 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 21 Apr 2023 17:46:37 +0300 Subject: [PATCH 125/197] feat(ft): fix dependencies in mix.exs --- mix.exs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index f94755b09..d05a1b225 100644 --- a/mix.exs +++ b/mix.exs @@ -184,7 +184,14 @@ defmodule EMQXUmbrella.MixProject do {:brod, github: "kafka4beam/brod", tag: "3.16.8"}, {:snappyer, "1.2.8", override: true}, {:crc32cer, "0.1.8", override: true}, - {:supervisor3, "1.1.12", override: true} + {:supervisor3, "1.1.12", override: true}, + {:erlcloud, github: "emqx/erlcloud", tag: "3.6.8-emqx-1", override: true}, + # erlcloud's rebar.config requires rebar3 and does not support Mix, + # so it tries to fetch deps from git. We need to override this. + {:lhttpc, github: "erlcloud/lhttpc", tag: "1.6.2", override: true}, + {:eini, "1.2.9", override: true}, + {:base16, "1.0.0", override: true} + # end of erlcloud's deps ] end @@ -374,7 +381,8 @@ defmodule EMQXUmbrella.MixProject do emqx_bridge_rocketmq: :permanent, emqx_bridge_tdengine: :permanent, emqx_bridge_timescale: :permanent, - emqx_ee_schema_registry: :permanent + emqx_ee_schema_registry: :permanent, + emqx_ft: :permanent ], else: [] ) From ebb75b275e6509fb7bf864c0802d23f2c74b022e Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 21 Apr 2023 17:50:10 +0300 Subject: [PATCH 126/197] feat(ft): update app versions --- apps/emqx_machine/src/emqx_machine.app.src | 2 +- lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 6bd36aab5..a44d2b36e 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src index 771fdcb27..3df18ce7a 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_conf, [ {description, "EMQX Enterprise Edition configuration schema"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [ kernel, From b951de4c6ea4ef2dd5a61958739b83289169e191 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 21 Apr 2023 19:14:24 +0300 Subject: [PATCH 127/197] feat(ft): convert hocon files to the new format --- rel/i18n/emqx_ft_api.hocon | 24 +---- rel/i18n/emqx_ft_schema.hocon | 197 +++++++--------------------------- rel/i18n/emqx_s3_schema.hocon | 144 ++++++------------------- 3 files changed, 76 insertions(+), 289 deletions(-) diff --git a/rel/i18n/emqx_ft_api.hocon b/rel/i18n/emqx_ft_api.hocon index 6ac01ea23..0c67db554 100644 --- a/rel/i18n/emqx_ft_api.hocon +++ b/rel/i18n/emqx_ft_api.hocon @@ -1,29 +1,13 @@ emqx_ft_api { - file_list { - desc { - en: "List all uploaded files." - zh: "列出所有上传的文件。" - } - label: { - en: "List all uploaded files" - zh: "列出所有上传的文件" - } - } +file_list.desc: +"""List all uploaded files.""" } emqx_ft_storage_exporter_fs_api { - file_get { - desc { - en: "Get a file by its id." - zh: "根据文件 id 获取文件。" - } - label: { - en: "Get a file by its id" - zh: "根据文件 id 获取文件" - } - } +file_get.desc: +"""Get a file by its id.""" } diff --git a/rel/i18n/emqx_ft_schema.hocon b/rel/i18n/emqx_ft_schema.hocon index 9df02e920..28c93e1ef 100644 --- a/rel/i18n/emqx_ft_schema.hocon +++ b/rel/i18n/emqx_ft_schema.hocon @@ -1,177 +1,56 @@ emqx_ft_schema { - init_timeout { - desc { - en: "Timeout for initializing the file transfer.
" - "After reaching the timeout, `init` message will be acked with an error" - zh: "" - } - label { - en: "File Transfer Init Timeout" - zh: "" - } - } +init_timeout.desc: +"""Timeout for initializing the file transfer.
+After reaching the timeout, `init` message will be acked with an error""" - assemble_timeout { - desc { - en: "Timeout for assembling and exporting file segments into a final file.
" - "After reaching the timeout, `fin` message will be acked with an error" - zh: "" - } - label { - en: "File Assemble Timeout" - zh: "" - } - } +assemble_timeout.desc: +"""Timeout for assembling and exporting file segments into a final file.
+After reaching the timeout, `fin` message will be acked with an error""" - store_segment_timeout { - desc { - en: "Timeout for storing a file segment.
" - "After reaching the timeout, message with the segment will be acked with an error" - zh: "" - } - label { - en: "Store Segment Timeout" - zh: "" - } - } +store_segment_timeout.desc: +"""Timeout for storing a file segment.
+After reaching the timeout, message with the segment will be acked with an error""" - storage { - desc { - en: "Storage settings for file transfer." - zh: "文件传输的存储设置。" - } - label: { - en: "Storage settings" - zh: "存储设置" - } - } +storage.desc: +"""Storage settings for file transfer.""" - local_type { - desc { - en: "Use local file system to store uploaded fragments and temporary data." - zh: "使用本地文件系统来存储上传的文件和临时数据。" - } - label: { - en: "Local Storage" - zh: "本地存储" - } - } +local_type.desc: +"""Use local file system to store uploaded fragments and temporary data.""" - local_storage_segments { - desc { - en: "Settings for local segments storage, which include uploaded transfer fragments and temporary data." - zh: "保存上传文件和临时数据的文件系统路径。" - } - label: { - en: "Local Segments Storage" - zh: "本地存储根" - } - } +local_storage_segments.desc: +"""Settings for local segments storage, which include uploaded transfer fragments and temporary data.""" - local_storage_segments_root { - desc { - en: "File system path to keep uploaded fragments and temporary data." - zh: "保存上传文件和临时数据的文件系统路径。" - } - label: { - en: "Local Segments Storage Filesystem Root" - zh: "本地存储根" - } - } +local_storage_segments_root.desc: +"""File system path to keep uploaded fragments and temporary data.""" - local_storage_exporter { - desc { - en: "Exporter for the local file system storage backend.
" - "Exporter defines where and how fully transferred and assembled files are stored." - zh: "" - } - label: { - en: "Local Storage Exporter" - zh: "" - } - } +local_storage_exporter.desc: +"""Exporter for the local file system storage backend.
+Exporter defines where and how fully transferred and assembled files are stored.""" - local_storage_exporter_type { - desc { - en: "Exporter type for the exporter to the local file system" - zh: "" - } - label: { - en: "Local Storage Exporter Type" - zh: "" - } - } +local_storage_exporter_type.desc: +"""Exporter type for the exporter to the local file system""" - s3_exporter_type { - desc { - en: "Exporter type for the exporter to S3" - zh: "" - } - label: { - en: "S3 Exporter Type" - zh: "" - } - } +s3_exporter_type.desc: +"""Exporter type for the exporter to S3""" - local_storage_exporter_root { - desc { - en: "File system path to keep uploaded files." - zh: "" - } - label: { - en: "Local Filesystem Exporter Root" - zh: "" - } - } +local_storage_exporter_root.desc: +"""File system path to keep uploaded files.""" - local_storage_segments_gc { - desc { - en: "Garbage collection settings for the intermediate and temporary files in the local file system." - zh: "" - } - label: { - en: "Local Storage GC" - zh: "" - } - } +local_storage_segments_gc.desc: +"""Garbage collection settings for the intermediate and temporary files in the local file system.""" - storage_gc_interval { - desc { - en: "Interval of periodic garbage collection." - zh: "" - } - label: { - en: "GC Interval" - zh: "" - } - } +storage_gc_interval.desc: +"""Interval of periodic garbage collection.""" - storage_gc_max_segments_ttl { - desc { - en: "Maximum TTL of a segment kept in the local file system.
" - "This is a hard limit: no segment will outlive this TTL, even if some file transfer specifies a " - "TTL more than that." - zh: "" - } - label: { - en: "Max segment TTL" - zh: "" - } - } - - storage_gc_min_segments_ttl { - desc { - en: "Minimum TTL of a segment kept in the local file system.
" - "This is a hard limit: no segment will be garbage collected before reaching this TTL, " - "even if some file transfer specifies a TTL less than that." - zh: "" - } - label: { - en: "Min segment TTL" - zh: "" - } - } +storage_gc_max_segments_ttl.desc: +"""Maximum TTL of a segment kept in the local file system.
+This is a hard limit: no segment will outlive this TTL, even if some file transfer specifies a +TTL more than that.""" +storage_gc_min_segments_ttl.desc: +"""Minimum TTL of a segment kept in the local file system.
+This is a hard limit: no segment will be garbage collected before reaching this TTL, +even if some file transfer specifies a TTL less than that.""" } diff --git a/rel/i18n/emqx_s3_schema.hocon b/rel/i18n/emqx_s3_schema.hocon index c391fa352..df4b973fa 100644 --- a/rel/i18n/emqx_s3_schema.hocon +++ b/rel/i18n/emqx_s3_schema.hocon @@ -1,114 +1,38 @@ emqx_s3_schema { - access_key_id { - desc { - en: "The access key ID of the S3 bucket." - zh: "" - } - label { - en: "Access Key ID" - zh: "" - } - } - secret_access_key { - desc { - en: "The secret access key of the S3 bucket." - zh: "" - } - label { - en: "Secret Access Key" - zh: "" - } - } - bucket { - desc { - en: "The name of the S3 bucket." - zh: "" - } - label { - en: "Bucket" - zh: "" - } - } - host { - desc { - en: "The host of the S3 endpoint." - zh: "" - } - label { - en: "S3 endpoint Host" - zh: "" - } - } - port { - desc { - en: "The port of the S3 endpoint." - zh: "" - } - label { - en: "S3 endpoint port" - zh: "" - } - } - url_expire_time { - desc { - en: "The time in seconds for which the signed URLs to the S3 objects are valid." - zh: "" - } - label { - en: "Signed URL expiration time" - zh: "" - } - } - min_part_size { - desc { - en: """The minimum part size for multipart uploads.
+ +access_key_id.desc: +"""The access key ID of the S3 bucket.""" + +secret_access_key.desc: +"""The secret access key of the S3 bucket.""" + +bucket.desc: +"""The name of the S3 bucket.""" + +host.desc: +"""The host of the S3 endpoint.""" + +port.desc: +"""The port of the S3 endpoint.""" + +url_expire_time.desc: +"""The time in seconds for which the signed URLs to the S3 objects are valid.""" + +min_part_size.desc: +"""The minimum part size for multipart uploads.
Uploaded data will be accumulated in memory until this size is reached.""" - zh: "" - } - label { - en: "Minimum multipart upload part size" - zh: "" - } - } - max_part_size { - desc { - en: """The maximum part size for multipart uploads.
+ +max_part_size.desc: +"""The maximum part size for multipart uploads.
S3 uploader won't try to upload parts larger than this size.""" - zh: "" - } - label { - en: "Maximum multipart upload part size" - zh: "" - } - } - acl { - desc { - en: "The ACL to use for the uploaded objects." - zh: "" - } - label { - en: "ACL" - zh: "" - } - } - transport_options { - desc { - en: "Options for the HTTP transport layer used by the S3 client." - zh: "" - } - label { - en: "Transport Options" - zh: "" - } - } - ipv6_probe { - desc { - en: "Whether to probe for IPv6 support." - zh: "" - } - label { - en: "IPv6 probe" - zh: "" - } - } + +acl.desc: +"""The ACL to use for the uploaded objects.""" + +transport_options.desc: +"""Options for the HTTP transport layer used by the S3 client.""" + +ipv6_probe.desc: +"""Whether to probe for IPv6 support.""" + } From 69c4ba2a62a05f54d4e7dbd8985952d2785ba111 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Sat, 22 Apr 2023 22:50:19 +0300 Subject: [PATCH 128/197] feat(ft): use new utils application --- apps/emqx_ft/src/emqx_ft.erl | 4 ++-- apps/emqx_ft/src/emqx_ft_api.erl | 2 +- apps/emqx_ft/src/emqx_ft_conf.erl | 6 +++--- apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl | 4 ++-- apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl | 4 ++-- apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 8 ++++---- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 4 ++-- apps/emqx_ft/test/emqx_ft_SUITE.erl | 6 +++--- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl | 6 +++--- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 2 +- apps/emqx_s3/src/emqx_s3_profile_conf.erl | 2 +- apps/emqx_s3/test/emqx_s3_client_SUITE.erl | 2 +- apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl | 8 ++++---- apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl | 2 +- 16 files changed, 32 insertions(+), 32 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 45207baa9..95982b849 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -98,7 +98,7 @@ unhook() -> %%-------------------------------------------------------------------- decode_filemeta(Payload) when is_binary(Payload) -> - case emqx_json:safe_decode(Payload, [return_maps]) of + case emqx_utils_json:safe_decode(Payload, [return_maps]) of {ok, Map} -> decode_filemeta(Map); {error, Error} -> @@ -116,7 +116,7 @@ decode_filemeta(Map) when is_map(Map) -> encode_filemeta(Meta = #{}) -> Schema = emqx_ft_schema:schema(filemeta), - hocon_tconf:make_serializable(Schema, emqx_map_lib:binary_key_map(Meta), #{}). + hocon_tconf:make_serializable(Schema, emqx_utils_maps:binary_key_map(Meta), #{}). %%-------------------------------------------------------------------- %% Hooks diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index 1ea710848..85fd8d8cb 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -75,7 +75,7 @@ schema("/file_transfer/files") -> end. error_msg(Code, Msg) -> - #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. + #{code => Code, message => emqx_utils:readable_error_msg(Msg)}. roots() -> []. diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 14a79b94c..dd9806e95 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -63,18 +63,18 @@ storage() -> -spec gc_interval(_Storage) -> milliseconds(). gc_interval(_Storage) -> Conf = assert_storage(local), - emqx_map_lib:deep_get([segments, gc, interval], Conf, ?DEFAULT_GC_INTERVAL). + emqx_utils_maps:deep_get([segments, gc, interval], Conf, ?DEFAULT_GC_INTERVAL). -spec segments_ttl(_Storage) -> {_Min :: seconds(), _Max :: seconds()}. segments_ttl(_Storage) -> Conf = assert_storage(local), { - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [segments, gc, minimum_segments_ttl], Conf, ?DEFAULT_MIN_SEGMENTS_TTL ), - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [segments, gc, maximum_segments_ttl], Conf, ?DEFAULT_MAX_SEGMENTS_TTL diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index ae8315510..4b05c9a58 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -315,10 +315,10 @@ list(_Options) -> -define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). encode_filemeta(Meta) -> - emqx_json:encode(?PRELUDE(_Vsn = 1, emqx_ft:encode_filemeta(Meta))). + emqx_utils_json:encode(?PRELUDE(_Vsn = 1, emqx_ft:encode_filemeta(Meta))). decode_filemeta(Binary) when is_binary(Binary) -> - ?PRELUDE(_Vsn = 1, Map) = emqx_json:decode(Binary, [return_maps]), + ?PRELUDE(_Vsn = 1, Map) = emqx_utils_json:decode(Binary, [return_maps]), case emqx_ft:decode_filemeta(Map) of {ok, Meta} -> Meta; diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl index b7ad86436..f1a8c6dae 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl @@ -134,7 +134,7 @@ fields(file_node) -> end. error_msg(Code, Msg) -> - #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. + #{code => Code, message => emqx_utils:readable_error_msg(Msg)}. -spec mk_export_uri(node(), file:name()) -> uri_string:uri_string(). @@ -150,7 +150,7 @@ mk_export_uri(Node, Filepath) -> %% parse_node(NodeBin) -> - case emqx_misc:safe_to_existing_atom(NodeBin) of + case emqx_utils:safe_to_existing_atom(NodeBin) of {ok, Node} -> Node; {error, _} -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index 259cb4016..adf000346 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -135,7 +135,7 @@ s3_headers({ClientId, FileId}, Filemeta) -> }. s3_header_filemeta(Filemeta) -> - emqx_json:encode(emqx_ft:encode_filemeta(Filemeta), [force_utf8, uescape]). + emqx_utils_json:encode(emqx_ft:encode_filemeta(Filemeta), [force_utf8, uescape]). list(Client, Options) -> case list_key_info(Client, Options) of diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index d871d1f32..1b1d9ecf7 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -271,7 +271,7 @@ read_transferinfo(Storage, Transfer, Acc) -> -spec get_root(storage()) -> file:name(). get_root(Storage) -> - case emqx_map_lib:deep_find([segments, root], Storage) of + case emqx_utils_maps:deep_find([segments, root], Storage) of {ok, Root} -> Root; {not_found, _, _} -> @@ -296,10 +296,10 @@ get_subdirs_for(temporary) -> -define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). encode_filemeta(Meta) -> - emqx_json:encode(?PRELUDE(_Vsn = 1, emqx_ft:encode_filemeta(Meta))). + emqx_utils_json:encode(?PRELUDE(_Vsn = 1, emqx_ft:encode_filemeta(Meta))). decode_filemeta(Binary) when is_binary(Binary) -> - ?PRELUDE(_Vsn = 1, Map) = emqx_json:decode(Binary, [return_maps]), + ?PRELUDE(_Vsn = 1, Map) = emqx_utils_json:decode(Binary, [return_maps]), case emqx_ft:decode_filemeta(Map) of {ok, Meta} -> Meta; @@ -347,7 +347,7 @@ read_file(Filepath, DecodeFun) -> write_file_atomic(Storage, Transfer, Filepath, Content) when is_binary(Content) -> TempFilepath = mk_temp_filepath(Storage, Transfer, filename:basename(Filepath)), - Result = emqx_misc:pipeline( + Result = emqx_utils:pipeline( [ fun filelib:ensure_dir/1, fun write_contents/2, diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 2ab30e88b..0f61e65b7 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -135,7 +135,7 @@ maybe_report(#gcstats{} = _Stats, #st{storage = _Storage}) -> start_timer(St = #st{storage = Storage, next_gc_timer = undefined}) -> case emqx_ft_conf:gc_interval(Storage) of Delay when Delay > 0 -> - St#st{next_gc_timer = emqx_misc:start_timer(Delay, collect)}; + St#st{next_gc_timer = emqx_utils:start_timer(Delay, collect)}; 0 -> ?SLOG(warning, #{msg => "periodic_gc_disabled"}), St @@ -144,7 +144,7 @@ start_timer(St = #st{storage = Storage, next_gc_timer = undefined}) -> reset_timer(St = #st{next_gc_timer = undefined}) -> start_timer(St); reset_timer(St = #st{next_gc_timer = TRef}) -> - ok = emqx_misc:cancel_timer(TRef), + ok = emqx_utils:cancel_timer(TRef), start_timer(St#st{next_gc_timer = undefined}). gc_enabled(St) -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 3649d02f8..151b8e5fe 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -63,7 +63,7 @@ set_special_configs(Config) -> % NOTE % Inhibit local fs GC to simulate it isn't fast enough to collect % complete transfers. - storage => emqx_map_lib:deep_merge( + storage => emqx_utils_maps:deep_merge( Storage, #{segments => #{gc => #{interval => 0}}} ) @@ -325,7 +325,7 @@ t_invalid_meta(Config) -> %% Invalid schema Meta = #{foo => <<"bar">>}, - MetaPayload = emqx_json:encode(Meta), + MetaPayload = emqx_utils_json:encode(Meta), ?assertRCName( unspecified_error, emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) @@ -667,7 +667,7 @@ meta(FileName, Data) -> }. encode_meta(Meta) -> - emqx_json:encode(emqx_ft:encode_filemeta(Meta)). + emqx_utils_json:encode(emqx_ft:encode_filemeta(Meta)). list_files(ClientId) -> {ok, Files} = emqx_ft_storage:files(), diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index a1bef2a2a..5f3b213fb 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -137,7 +137,7 @@ request(Method, Url, Decoder) when is_function(Decoder) -> end. json(Body) when is_binary(Body) -> - emqx_json:decode(Body, [return_maps]). + emqx_utils_json:decode(Body, [return_maps]). query(Params) -> KVs = lists:map(fun({K, V}) -> uri_encode(K) ++ "=" ++ uri_encode(V) end, maps:to_list(Params)), diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index 90039cd96..abf2749ed 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -127,7 +127,7 @@ t_gc_complete_transfers(_Config) -> } ], % 1. Start all transfers - TransferSizes = emqx_misc:pmap( + TransferSizes = emqx_utils:pmap( fun(Transfer) -> start_transfer(Storage, Transfer) end, Transfers ), @@ -162,7 +162,7 @@ t_gc_complete_transfers(_Config) -> ), ?assertEqual( [ok, ok], - emqx_misc:pmap( + emqx_utils:pmap( fun({Transfer, Size}) -> complete_transfer(Storage, Transfer, Size) end, [{T2, S2}, {T3, S3}] ) @@ -221,7 +221,7 @@ t_gc_incomplete_transfers(_Config) -> } ], % 1. Start transfers, send all the segments but don't trigger completion. - _ = emqx_misc:pmap(fun(Transfer) -> start_transfer(Storage, Transfer) end, Transfers), + _ = emqx_utils:pmap(fun(Transfer) -> start_transfer(Storage, Transfer) end, Transfers), % 2. Enable periodic GC every 0.5 seconds. ok = set_gc_config(interval, 500), ok = emqx_ft_storage_fs_gc:reset(Storage), diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index b7004dcc6..704e55454 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -82,7 +82,7 @@ upload_file(ClientId, FileId, Name, Data, Node) -> expire_at => erlang:system_time(_Unit = second) + 3600, size => Size }, - MetaPayload = emqx_json:encode(emqx_ft:encode_filemeta(Meta)), + MetaPayload = emqx_utils_json:encode(emqx_ft:encode_filemeta(Meta)), ct:pal("MetaPayload = ~ts", [MetaPayload]), diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index 3d66823a7..87f006bcb 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -351,7 +351,7 @@ http_config( ]. maybe_ipv6_probe(TransportOpts, true) -> - emqx_misc:ipv6_probe(TransportOpts); + emqx_utils:ipv6_probe(TransportOpts); maybe_ipv6_probe(TransportOpts, false) -> TransportOpts. diff --git a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl index cb55bc083..434510867 100644 --- a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl @@ -159,7 +159,7 @@ profile_config(Config) -> ?config(bucket, Config), ProfileConfig0 ), - ProfileConfig2 = emqx_map_lib:deep_put( + ProfileConfig2 = emqx_utils_maps:deep_put( [transport_options, pool_type], ProfileConfig1, ?config(pool_type, Config) diff --git a/apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl b/apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl index ce53525be..4b70ba015 100644 --- a/apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl @@ -62,7 +62,7 @@ t_regular_outdated_pool_cleanup(Config) -> [OldPool] = emqx_s3_profile_http_pools:all(profile_id()), ProfileBaseConfig = ?config(profile_config, Config), - ProfileConfig = emqx_map_lib:deep_put( + ProfileConfig = emqx_utils_maps:deep_put( [transport_options, pool_size], ProfileBaseConfig, 16 ), ok = emqx_s3:update_profile(profile_id(), ProfileConfig), @@ -110,7 +110,7 @@ t_timeout_pool_cleanup(Config) -> [OldPool] = emqx_s3_profile_http_pools:all(profile_id()), - NewProfileConfig = emqx_map_lib:deep_put( + NewProfileConfig = emqx_utils_maps:deep_put( [transport_options, pool_size], ProfileConfig, 16 ), @@ -153,7 +153,7 @@ t_httpc_pool_update_error(Config) -> meck:expect(ehttpc_pool, init, fun(_) -> meck:raise(error, badarg) end), ProfileBaseConfig = ?config(profile_config, Config), - NewProfileConfig = emqx_map_lib:deep_put( + NewProfileConfig = emqx_utils_maps:deep_put( [transport_options, pool_size], ProfileBaseConfig, 16 ), @@ -237,7 +237,7 @@ t_checkout_client(Config) -> %% Now change config for the profile ProfileBaseConfig = ?config(profile_config, Config), NewProfileConfig0 = ProfileBaseConfig#{bucket => <<"new_bucket">>}, - NewProfileConfig1 = emqx_map_lib:deep_put( + NewProfileConfig1 = emqx_utils_maps:deep_put( [transport_options, pool_size], NewProfileConfig0, 16 ), ok = emqx_s3:update_profile(profile_id(), NewProfileConfig1), diff --git a/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl index e09ea2773..6ba0e3ed9 100644 --- a/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl @@ -526,7 +526,7 @@ t_tls_error(Config) -> _ = process_flag(trap_exit, true), ProfileBaseConfig = ?config(profile_config, Config), - ProfileConfig = emqx_map_lib:deep_put( + ProfileConfig = emqx_utils_maps:deep_put( [transport_options, ssl, server_name_indication], ProfileBaseConfig, "invalid-hostname" ), ok = emqx_s3:update_profile(profile_id(), ProfileConfig), From 4574597175649d8e11308698fcbda39784e221f4 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Sat, 22 Apr 2023 23:39:03 +0300 Subject: [PATCH 129/197] feat(ft): update minirest --- mix.exs | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index d05a1b225..2925c6580 100644 --- a/mix.exs +++ b/mix.exs @@ -58,7 +58,7 @@ defmodule EMQXUmbrella.MixProject do {:ekka, github: "emqx/ekka", tag: "0.14.6", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true}, - {:minirest, github: "emqx/minirest", tag: "1.3.8", override: true}, + {:minirest, github: "emqx/minirest", tag: "1.3.9", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, diff --git a/rebar.config b/rebar.config index de520f124..b1640991e 100644 --- a/rebar.config +++ b/rebar.config @@ -65,7 +65,7 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.8"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.9"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} From 5efd590ca47afddd12282ff064974ed6e4b0b7dc Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 24 Apr 2023 20:28:58 +0300 Subject: [PATCH 130/197] feat(ft): properly propagate config updates Ensure that: * Storage config might be removed. * Local FS GC process is set up when Local FS storage is configured. * Local FS GC process gets its timer reset on config updates. * Storage / exporter gets chosen based on `type` only. * Exporter config updates propagated as before. Also employ `emqx_ft_schema:translate/1` instead of duplicating defaults where applicable. --- apps/emqx/src/emqx_maybe.erl | 3 + apps/emqx_ft/src/emqx_ft.erl | 21 ++- apps/emqx_ft/src/emqx_ft_assembler.erl | 2 +- apps/emqx_ft/src/emqx_ft_conf.erl | 54 +++----- apps/emqx_ft/src/emqx_ft_schema.erl | 71 +++++++--- apps/emqx_ft/src/emqx_ft_storage.erl | 98 +++++++++++--- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 46 +++---- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 9 ++ apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 125 ++++++++++-------- apps/emqx_ft/src/emqx_ft_sup.erl | 2 +- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 28 ++-- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 84 ++++++++++++ .../test/emqx_ft_storage_fs_gc_SUITE.erl | 12 +- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 15 ++- 14 files changed, 376 insertions(+), 194 deletions(-) diff --git a/apps/emqx/src/emqx_maybe.erl b/apps/emqx/src/emqx_maybe.erl index 0b919f7ab..2629bc737 100644 --- a/apps/emqx/src/emqx_maybe.erl +++ b/apps/emqx/src/emqx_maybe.erl @@ -23,6 +23,9 @@ -export([define/2]). -export([apply/2]). +-type t(T) :: maybe(T). +-export_type([t/1]). + -spec to_list(maybe(A)) -> [A]. to_list(undefined) -> []; diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 95982b849..d500a6344 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -301,8 +301,8 @@ store_filemeta(Transfer, Segment) -> emqx_ft_storage:store_filemeta(Transfer, Segment) catch C:E:S -> - ?SLOG(error, #{ - msg => "start_store_filemeta_failed", class => C, reason => E, stacktrace => S + ?tp(error, "start_store_filemeta_failed", #{ + class => C, reason => E, stacktrace => S }), {error, {internal_error, E}} end. @@ -312,8 +312,8 @@ store_segment(Transfer, Segment) -> emqx_ft_storage:store_segment(Transfer, Segment) catch C:E:S -> - ?SLOG(error, #{ - msg => "start_store_segment_failed", class => C, reason => E, stacktrace => S + ?tp(error, "start_store_segment_failed", #{ + class => C, reason => E, stacktrace => S }), {error, {internal_error, E}} end. @@ -323,8 +323,8 @@ assemble(Transfer, FinalSize) -> emqx_ft_storage:assemble(Transfer, FinalSize) catch C:E:S -> - ?SLOG(error, #{ - msg => "start_assemble_failed", class => C, reason => E, stacktrace => S + ?tp(error, "start_assemble_failed", #{ + class => C, reason => E, stacktrace => S }), {error, {internal_error, E}} end. @@ -334,8 +334,7 @@ transfer(Msg, FileId) -> {clientid_to_binary(ClientId), FileId}. on_complete(Op, {ChanPid, PacketId}, Transfer, Result) -> - ?SLOG(debug, #{ - msg => "on_complete", + ?tp(debug, "on_complete", #{ operation => Op, packet_id => PacketId, transfer => Transfer @@ -344,15 +343,13 @@ on_complete(Op, {ChanPid, PacketId}, Transfer, Result) -> {Mode, ok} when Mode == ack orelse Mode == down -> erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS}); {Mode, {error, _} = Reason} when Mode == ack orelse Mode == down -> - ?SLOG(error, #{ - msg => Op ++ "_failed", + ?tp(error, Op ++ "_failed", #{ transfer => Transfer, reason => Reason }), erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}); timeout -> - ?SLOG(error, #{ - msg => Op ++ "_timed_out", + ?tp(error, Op ++ "_timed_out", #{ transfer => Transfer }), erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 3a352cd10..767930f98 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -159,7 +159,7 @@ handle_event(internal, _, {assemble, []}, St = #{}) -> {next_state, complete, St, ?internal([])}; handle_event(internal, _, complete, St = #{export := Export}) -> Result = emqx_ft_storage_exporter:complete(Export), - ok = maybe_garbage_collect(Result, St), + _ = maybe_garbage_collect(Result, St), {stop, {shutdown, Result}, maps:remove(export, St)}. -spec terminate(_Reason, state(), stdata()) -> _. diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index dd9806e95..1e531ecdb 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -45,49 +45,28 @@ -type milliseconds() :: non_neg_integer(). -type seconds() :: non_neg_integer(). -%% 5 minutes (s) --define(DEFAULT_MIN_SEGMENTS_TTL, 300). -%% 1 day (s) --define(DEFAULT_MAX_SEGMENTS_TTL, 86400). -%% 1 minute (ms) --define(DEFAULT_GC_INTERVAL, 60000). - %%-------------------------------------------------------------------- %% Accessors %%-------------------------------------------------------------------- -spec storage() -> _Storage. storage() -> - emqx_config:get([file_transfer, storage]). + emqx_config:get([file_transfer, storage], undefined). --spec gc_interval(_Storage) -> milliseconds(). -gc_interval(_Storage) -> - Conf = assert_storage(local), - emqx_utils_maps:deep_get([segments, gc, interval], Conf, ?DEFAULT_GC_INTERVAL). +-spec gc_interval(_Storage) -> emqx_maybe:t(milliseconds()). +gc_interval(Conf = #{type := local}) -> + emqx_utils_maps:deep_get([segments, gc, interval], Conf); +gc_interval(_) -> + undefined. --spec segments_ttl(_Storage) -> {_Min :: seconds(), _Max :: seconds()}. -segments_ttl(_Storage) -> - Conf = assert_storage(local), +-spec segments_ttl(_Storage) -> emqx_maybe:t({_Min :: seconds(), _Max :: seconds()}). +segments_ttl(Conf = #{type := local}) -> { - emqx_utils_maps:deep_get( - [segments, gc, minimum_segments_ttl], - Conf, - ?DEFAULT_MIN_SEGMENTS_TTL - ), - emqx_utils_maps:deep_get( - [segments, gc, maximum_segments_ttl], - Conf, - ?DEFAULT_MAX_SEGMENTS_TTL - ) - }. - -assert_storage(Type) -> - case storage() of - Conf = #{type := Type} -> - Conf; - Conf -> - error({inapplicable, Conf}) - end. + emqx_utils_maps:deep_get([segments, gc, minimum_segments_ttl], Conf), + emqx_utils_maps:deep_get([segments, gc, maximum_segments_ttl], Conf) + }; +segments_ttl(_) -> + undefined. init_timeout() -> emqx_config:get([file_transfer, init_timeout]). @@ -104,10 +83,7 @@ store_segment_timeout() -> -spec load() -> ok. load() -> - ok = emqx_ft_storage_exporter:update_exporter( - undefined, - storage() - ), + ok = emqx_ft_storage:on_config_update(undefined, storage()), emqx_conf:add_handler([file_transfer], ?MODULE). -spec unload() -> ok. @@ -134,4 +110,4 @@ pre_config_update(_, Req, _Config) -> post_config_update(_Path, _Req, NewConfig, OldConfig, _AppEnvs) -> OldStorageConfig = maps:get(storage, OldConfig, undefined), NewStorageConfig = maps:get(storage, NewConfig, undefined), - emqx_ft_storage_exporter:update_exporter(OldStorageConfig, NewStorageConfig). + emqx_ft_storage:on_config_update(OldStorageConfig, NewStorageConfig). diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index ce3710cc1..27c593b6c 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -25,6 +25,8 @@ -export([schema/1]). +-export([translate/1]). + -type json_value() :: null | boolean() @@ -82,13 +84,25 @@ fields(file_transfer) -> )}, {storage, mk( - hoconsc:union([ - ref(local_storage) - ]), + hoconsc:union( + fun + (all_union_members) -> + [ + % NOTE: by default storage is disabled + undefined, + ref(local_storage) + ]; + ({value, #{<<"type">> := <<"local">>}}) -> + [ref(local_storage)]; + ({value, #{<<"type">> := _}}) -> + throw(#{field_name => type, expected => "local"}); + (_) -> + [undefined] + end + ), #{ required => false, - desc => ?DESC("storage"), - default => default_storage() + desc => ?DESC("storage") } )} ]; @@ -108,18 +122,38 @@ fields(local_storage) -> ref(local_storage_segments), #{ desc => ?DESC("local_storage_segments"), - required => false + required => false, + default => #{ + <<"gc">> => #{} + } } )}, {exporter, mk( - hoconsc:union([ - ref(local_storage_exporter), - ref(s3_exporter) - ]), + hoconsc:union( + fun + (all_union_members) -> + [ + ref(local_storage_exporter), + ref(s3_exporter) + ]; + ({value, #{<<"type">> := <<"local">>}}) -> + [ref(local_storage_exporter)]; + ({value, #{<<"type">> := <<"s3">>}}) -> + [ref(s3_exporter)]; + ({value, #{<<"type">> := _}}) -> + throw(#{field_name => type, expected => "local | s3"}); + ({value, _}) -> + % NOTE: default + [ref(local_storage_exporter)] + end + ), #{ desc => ?DESC("local_storage_exporter"), - required => true + required => true, + default => #{ + <<"type">> => <<"local">> + } } )} ]; @@ -277,10 +311,11 @@ converter(unicode_string) -> ref(Ref) -> ref(?MODULE, Ref). -default_storage() -> - #{ - <<"type">> => <<"local">>, - <<"exporter">> => #{ - <<"type">> => <<"local">> - } - }. +translate(Conf) -> + [Root] = roots(), + maps:get( + Root, + hocon_tconf:check_plain( + ?MODULE, #{atom_to_binary(Root) => Conf}, #{atom_key => true}, [Root] + ) + ). diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 79538a9c7..fee16cd09 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -27,7 +27,9 @@ files/0, with_storage_type/2, - with_storage_type/3 + with_storage_type/3, + + on_config_update/2 ] ). @@ -90,26 +92,35 @@ child_spec() -> -spec store_filemeta(emqx_ft:transfer(), emqx_ft:filemeta()) -> ok | {async, pid()} | {error, term()}. store_filemeta(Transfer, FileMeta) -> - Mod = mod(), - Mod:store_filemeta(storage(), Transfer, FileMeta). + with_storage(store_filemeta, [Transfer, FileMeta]). -spec store_segment(emqx_ft:transfer(), emqx_ft:segment()) -> ok | {async, pid()} | {error, term()}. store_segment(Transfer, Segment) -> - Mod = mod(), - Mod:store_segment(storage(), Transfer, Segment). + with_storage(store_segment, [Transfer, Segment]). -spec assemble(emqx_ft:transfer(), emqx_ft:bytes()) -> ok | {async, pid()} | {error, term()}. assemble(Transfer, Size) -> - Mod = mod(), - Mod:assemble(storage(), Transfer, Size). + with_storage(assemble, [Transfer, Size]). -spec files() -> {ok, [file_info()]} | {error, term()}. files() -> - Mod = mod(), - Mod:files(storage()). + with_storage(files, []). + +-spec with_storage(atom() | function()) -> any(). +with_storage(Fun) -> + with_storage(Fun, []). + +-spec with_storage(atom() | function(), list(term())) -> any(). +with_storage(Fun, Args) -> + case storage() of + Storage = #{} -> + apply_storage(Storage, Fun, Args); + undefined -> + {error, disabled} + end. -spec with_storage_type(atom(), atom() | function()) -> any(). with_storage_type(Type, Fun) -> @@ -117,17 +128,61 @@ with_storage_type(Type, Fun) -> -spec with_storage_type(atom(), atom() | function(), list(term())) -> any(). with_storage_type(Type, Fun, Args) -> - Storage = storage(), - case Storage of - #{type := Type} when is_function(Fun) -> - apply(Fun, [Storage | Args]); - #{type := Type} when is_atom(Fun) -> - Mod = mod(Storage), - apply(Mod, Fun, [Storage | Args]); - disabled -> - {error, disabled}; - _ -> - {error, {invalid_storage_type, Type}} + with_storage(fun(Storage) -> + case Storage of + #{type := Type} -> + apply_storage(Storage, Fun, Args); + _ -> + {error, {invalid_storage_type, Storage}} + end + end). + +apply_storage(Storage, Fun, Args) when is_atom(Fun) -> + apply(mod(Storage), Fun, [Storage | Args]); +apply_storage(Storage, Fun, Args) when is_function(Fun) -> + apply(Fun, [Storage | Args]). + +%% + +-spec on_config_update(_Old :: emqx_maybe:t(storage()), _New :: emqx_maybe:t(storage())) -> + ok. +on_config_update(Storage, Storage) -> + ok; +on_config_update(#{type := Type} = StorageOld, #{type := Type} = StorageNew) -> + ok = (mod(StorageNew)):on_config_update(StorageOld, StorageNew); +on_config_update(StorageOld, StorageNew) -> + _ = emqx_maybe:apply(fun on_storage_stop/1, StorageOld), + _ = emqx_maybe:apply(fun on_storage_start/1, StorageNew), + _ = emqx_maybe:apply( + fun(Storage) -> (mod(Storage)):on_config_update(StorageOld, StorageNew) end, + StorageNew + ), + ok. + +on_storage_start(Storage = #{type := _}) -> + lists:foreach( + fun(ChildSpec) -> + {ok, _Child} = supervisor:start_child(emqx_ft_sup, ChildSpec) + end, + child_spec(Storage) + ). + +on_storage_stop(Storage = #{type := _}) -> + lists:foreach( + fun(#{id := ChildId}) -> + _ = supervisor:terminate_child(emqx_ft_sup, ChildId), + ok = supervisor:delete_child(emqx_ft_sup, ChildId) + end, + child_spec(Storage) + ). + +child_spec(Storage) -> + try + Mod = mod(Storage), + Mod:child_spec(Storage) + catch + error:disabled -> []; + error:undef -> [] end. %%-------------------------------------------------------------------- @@ -144,7 +199,6 @@ mod(Storage) -> case Storage of #{type := local} -> emqx_ft_storage_fs; - disabled -> + undefined -> error(disabled) - % emqx_ft_storage_dummy end. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index 61241bd6f..72128cb40 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -31,7 +31,7 @@ -export([list/1]). %% Lifecycle API --export([update_exporter/2]). +-export([on_config_update/2]). %% Internal API -export([exporter/1]). @@ -141,30 +141,28 @@ list(Storage) -> %% Lifecycle --spec update_exporter(emqx_config:config(), emqx_config:config()) -> ok | {error, term()}. -update_exporter( - #{exporter := #{type := OldType}} = OldConfig, - #{exporter := #{type := OldType}} = NewConfig -) -> - {ExporterMod, OldExporterOpts} = exporter(OldConfig), - {_NewExporterMod, NewExporterOpts} = exporter(NewConfig), - ExporterMod:update(OldExporterOpts, NewExporterOpts); -update_exporter( - #{exporter := _} = OldConfig, - #{exporter := _} = NewConfig -) -> - {OldExporterMod, OldExporterOpts} = exporter(OldConfig), - {NewExporterMod, NewExporterOpts} = exporter(NewConfig), - ok = OldExporterMod:stop(OldExporterOpts), - NewExporterMod:start(NewExporterOpts); -update_exporter(undefined, NewConfig) -> - {ExporterMod, ExporterOpts} = exporter(NewConfig), - ExporterMod:start(ExporterOpts); -update_exporter(OldConfig, undefined) -> - {ExporterMod, ExporterOpts} = exporter(OldConfig), - ExporterMod:stop(ExporterOpts); -update_exporter(_, _) -> +-spec on_config_update(storage(), storage()) -> ok | {error, term()}. +on_config_update(StorageOld, StorageNew) -> + on_exporter_update( + emqx_maybe:apply(fun exporter/1, StorageOld), + emqx_maybe:apply(fun exporter/1, StorageNew) + ). + +on_exporter_update(Config, Config) -> + ok; +on_exporter_update({ExporterMod, ConfigOld}, {ExporterMod, ConfigNew}) -> + ExporterMod:update(ConfigOld, ConfigNew); +on_exporter_update(ExporterOld, ExporterNew) -> + _ = emqx_maybe:apply(fun stop_exporter/1, ExporterOld), + _ = emqx_maybe:apply(fun start_exporter/1, ExporterNew), ok. + +start_exporter({ExporterMod, ExporterOpts}) -> + ok = ExporterMod:start(ExporterOpts). + +stop_exporter({ExporterMod, ExporterOpts}) -> + ok = ExporterMod:stop(ExporterOpts). + %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 1b1d9ecf7..5754d2cfc 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -47,6 +47,8 @@ -export([files/1]). +-export([on_config_update/2]). + -export_type([storage/0]). -export_type([filefrag/1]). -export_type([filefrag/0]). @@ -219,6 +221,13 @@ files(Storage) -> %% +on_config_update(StorageOld, StorageNew) -> + % NOTE: this will reset GC timer, frequent changes would postpone GC indefinitely + ok = emqx_ft_storage_fs_gc:reset(StorageNew), + emqx_ft_storage_exporter:on_config_update(StorageOld, StorageNew). + +%% + -spec transfers(storage()) -> {ok, #{transfer() => transferinfo()}}. transfers(Storage) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 0f61e65b7..692a270e3 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -29,8 +29,9 @@ -export([start_link/1]). --export([collect/1]). +-export([collect/0]). -export([collect/3]). +-export([reset/0]). -export([reset/1]). -behaviour(gen_server). @@ -40,38 +41,43 @@ -export([handle_info/2]). -record(st, { - storage :: emqx_ft_storage_fs:storage(), next_gc_timer :: maybe(reference()), last_gc :: maybe(gcstats()) }). -type gcstats() :: #gcstats{}. +-define(IS_ENABLED(INTERVAL), (is_integer(INTERVAL) andalso INTERVAL > 0)). + %% start_link(Storage) -> - gen_server:start_link(mk_server_ref(Storage), ?MODULE, Storage, []). + gen_server:start_link(mk_server_ref(global), ?MODULE, Storage, []). --spec collect(emqx_ft_storage_fs:storage()) -> gcstats(). -collect(Storage) -> - gen_server:call(mk_server_ref(Storage), {collect, erlang:system_time()}, infinity). +-spec collect() -> gcstats(). +collect() -> + gen_server:call(mk_server_ref(global), {collect, erlang:system_time()}, infinity). + +-spec reset() -> ok. +reset() -> + reset(emqx_ft_conf:storage()). -spec reset(emqx_ft_storage_fs:storage()) -> ok. reset(Storage) -> - gen_server:cast(mk_server_ref(Storage), reset). + gen_server:cast(mk_server_ref(global), {reset, gc_interval(Storage)}). collect(Storage, Transfer, Nodes) -> - gen_server:cast(mk_server_ref(Storage), {collect, Transfer, Nodes}). + gc_enabled(Storage) andalso cast_collect(mk_server_ref(global), Storage, Transfer, Nodes). -mk_server_ref(Storage) -> +mk_server_ref(Name) -> % TODO - {via, gproc, {n, l, {?MODULE, get_segments_root(Storage)}}}. + {via, gproc, {n, l, {?MODULE, Name}}}. %% init(Storage) -> - St = #st{storage = Storage}, - {ok, start_timer(St)}. + St = #st{}, + {ok, start_timer(gc_interval(Storage), St)}. handle_call({collect, CalledAt}, _From, St) -> StNext = maybe_collect_garbage(CalledAt, St), @@ -80,22 +86,17 @@ handle_call(Call, From, St) -> ?SLOG(error, #{msg => "unexpected_call", call => Call, from => From}), {noreply, St}. -handle_cast({collect, Transfer, [Node | Rest]}, St) -> - case gc_enabled(St) of - true -> - ok = do_collect_transfer(Transfer, Node, St), - case Rest of - [_ | _] -> - gen_server:cast(self(), {collect, Transfer, Rest}); - [] -> - ok - end; - false -> - skip +handle_cast({collect, Storage, Transfer, [Node | Rest]}, St) -> + ok = do_collect_transfer(Storage, Transfer, Node, St), + case Rest of + [_ | _] -> + cast_collect(self(), Storage, Transfer, Rest); + [] -> + ok end, {noreply, St}; -handle_cast(reset, St) -> - {noreply, reset_timer(St)}; +handle_cast({reset, Interval}, St) -> + {noreply, start_timer(Interval, cancel_timer(St))}; handle_cast(Cast, St) -> ?SLOG(error, #{msg => "unexpected_cast", cast => Cast}), {noreply, St}. @@ -104,14 +105,17 @@ handle_info({timeout, TRef, collect}, St = #st{next_gc_timer = TRef}) -> StNext = do_collect_garbage(St), {noreply, start_timer(StNext#st{next_gc_timer = undefined})}. -do_collect_transfer(Transfer, Node, St = #st{storage = Storage}) when Node == node() -> +do_collect_transfer(Storage, Transfer, Node, St = #st{}) when Node == node() -> Stats = try_collect_transfer(Storage, Transfer, complete, init_gcstats()), ok = maybe_report(Stats, St), ok; -do_collect_transfer(_Transfer, _Node, _St = #st{}) -> +do_collect_transfer(_Storage, _Transfer, _Node, _St = #st{}) -> % TODO ok. +cast_collect(Ref, Storage, Transfer, Nodes) -> + gen_server:cast(Ref, {collect, Storage, Transfer, Nodes}). + maybe_collect_garbage(_CalledAt, St = #st{last_gc = undefined}) -> do_collect_garbage(St); maybe_collect_garbage(CalledAt, St = #st{last_gc = #gcstats{finished_at = FinishedAt}}) -> @@ -119,36 +123,41 @@ maybe_collect_garbage(CalledAt, St = #st{last_gc = #gcstats{finished_at = Finish true -> St; false -> - reset_timer(do_collect_garbage(St)) + start_timer(do_collect_garbage(cancel_timer(St))) end. -do_collect_garbage(St = #st{storage = Storage}) -> - Stats = collect_garbage(Storage), - ok = maybe_report(Stats, St), - St#st{last_gc = Stats}. +do_collect_garbage(St = #st{}) -> + emqx_ft_storage:with_storage_type(local, fun(Storage) -> + Stats = collect_garbage(Storage), + ok = maybe_report(Stats, Storage), + St#st{last_gc = Stats} + end). -maybe_report(#gcstats{errors = Errors}, #st{storage = Storage}) when map_size(Errors) > 0 -> +maybe_report(#gcstats{errors = Errors}, Storage) when map_size(Errors) > 0 -> ?tp(warning, "garbage_collection_errors", #{errors => Errors, storage => Storage}); -maybe_report(#gcstats{} = _Stats, #st{storage = _Storage}) -> +maybe_report(#gcstats{} = _Stats, _Storage) -> ?tp(garbage_collection, #{stats => _Stats, storage => _Storage}). -start_timer(St = #st{storage = Storage, next_gc_timer = undefined}) -> - case emqx_ft_conf:gc_interval(Storage) of - Delay when Delay > 0 -> - St#st{next_gc_timer = emqx_utils:start_timer(Delay, collect)}; - 0 -> - ?SLOG(warning, #{msg => "periodic_gc_disabled"}), - St - end. +start_timer(St) -> + start_timer(gc_interval(emqx_ft_conf:storage()), St). -reset_timer(St = #st{next_gc_timer = undefined}) -> - start_timer(St); -reset_timer(St = #st{next_gc_timer = TRef}) -> +start_timer(Interval, St = #st{next_gc_timer = undefined}) when ?IS_ENABLED(Interval) -> + St#st{next_gc_timer = emqx_utils:start_timer(Interval, collect)}; +start_timer(Interval, St) -> + ?SLOG(warning, #{msg => "periodic_gc_disabled", interval => Interval}), + St. + +cancel_timer(St = #st{next_gc_timer = undefined}) -> + St; +cancel_timer(St = #st{next_gc_timer = TRef}) -> ok = emqx_utils:cancel_timer(TRef), - start_timer(St#st{next_gc_timer = undefined}). + St#st{next_gc_timer = undefined}. -gc_enabled(St) -> - emqx_ft_conf:gc_interval(St#st.storage) > 0. +gc_enabled(Storage) -> + ?IS_ENABLED(gc_interval(Storage)). + +gc_interval(Storage) -> + emqx_ft_conf:gc_interval(Storage). %% @@ -175,8 +184,13 @@ try_collect_transfer(Storage, Transfer, TransferInfo = #{}, Stats) -> % heuristic we only delete transfer directory itself only if it is also outdated % _and was empty at the start of GC_, as a precaution against races between % writers and GCs. - TTL = get_segments_ttl(Storage, TransferInfo), - Cutoff = erlang:system_time(second) - TTL, + Cutoff = + case get_segments_ttl(Storage, TransferInfo) of + TTL when is_integer(TTL) -> + erlang:system_time(second) - TTL; + undefined -> + 0 + end, {FragCleaned, Stats1} = collect_outdated_fragments(Storage, Transfer, Cutoff, Stats), {TempCleaned, Stats2} = collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats1), % TODO: collect empty directories separately @@ -338,16 +352,17 @@ filepath_to_binary(S) -> unicode:characters_to_binary(S, unicode, file:native_name_encoding()). get_segments_ttl(Storage, TransferInfo) -> - {MinTTL, MaxTTL} = emqx_ft_conf:segments_ttl(Storage), - clamp(MinTTL, MaxTTL, try_get_filemeta_ttl(TransferInfo)). + clamp(emqx_ft_conf:segments_ttl(Storage), try_get_filemeta_ttl(TransferInfo)). try_get_filemeta_ttl(#{filemeta := Filemeta}) -> maps:get(segments_ttl, Filemeta, undefined); try_get_filemeta_ttl(#{}) -> undefined. -clamp(Min, Max, V) -> - min(Max, max(Min, V)). +clamp({Min, Max}, V) -> + min(Max, max(Min, V)); +clamp(undefined, V) -> + V. %% diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index 3c28eae30..8d388814c 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -61,5 +61,5 @@ init([]) -> modules => [emqx_ft_responder_sup] }, - ChildSpecs = [Responder, AssemblerSup, FileReaderSup | emqx_ft_storage:child_spec()], + ChildSpecs = [Responder, AssemblerSup, FileReaderSup], {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 4b5610f51..a7323fc0e 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -46,8 +46,8 @@ init_per_testcase(TC, Config) -> ok = snabbkaffe:start_trace(), {ok, Pid} = emqx_ft_assembler_sup:start_link(), [ - {storage_root, "file_transfer_root"}, - {exports_root, "file_transfer_exports"}, + {storage_root, <<"file_transfer_root">>}, + {exports_root, <<"file_transfer_exports">>}, {file_id, atom_to_binary(TC)}, {assembler_sup, Pid} | Config @@ -246,13 +246,17 @@ exporter(Config) -> emqx_ft_storage_exporter:exporter(storage(Config)). storage(Config) -> - #{ - type => local, - segments => #{ - root => ?config(storage_root, Config) - }, - exporter => #{ - type => local, - root => ?config(exports_root, Config) - } - }. + maps:get( + storage, + emqx_ft_schema:translate(#{ + <<"storage">> => #{ + <<"type">> => <<"local">>, + <<"segments">> => #{ + <<"root">> => ?config(storage_root, Config) + }, + <<"exporter">> => #{ + <<"root">> => ?config(exports_root, Config) + } + } + }) + ). diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index c616681dd..89b0e895d 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -21,6 +21,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). +-include_lib("snabbkaffe/include/test_macros.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). @@ -85,3 +86,86 @@ t_update_config(_Config) -> 5 * 60 * 1000, emqx_ft_conf:gc_interval(emqx_ft_conf:storage()) ). + +t_remove_restore_config(Config) -> + ?assertMatch( + {ok, _}, + emqx_conf:update([file_transfer, storage], #{<<"type">> => <<"local">>}, #{}) + ), + ?assertEqual( + 60 * 60 * 1000, + emqx_ft_conf:gc_interval(emqx_ft_conf:storage()) + ), + % Verify that transfers work + ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <>), + ?assertMatch( + {ok, _}, + emqx_conf:remove([file_transfer, storage], #{}) + ), + ?assertEqual( + undefined, + emqx_ft_conf:storage() + ), + ?assertEqual( + undefined, + emqx_ft_conf:gc_interval(emqx_ft_conf:storage()) + ), + ClientId = gen_clientid(), + Client = emqx_ft_test_helpers:start_client(ClientId), + % Verify that transfers fail cleanly when storage is disabled + ?check_trace( + ?assertMatch( + {ok, #{reason_code_name := unspecified_error}}, + emqtt:publish( + Client, + <<"$file/f2/init">>, + emqx_utils_json:encode(emqx_ft:encode_filemeta(#{name => "f2", size => 42})), + 1 + ) + ), + fun(Trace) -> + ?assertMatch( + [#{transfer := {ClientId, <<"f2">>}, reason := {error, disabled}}], + ?of_kind("store_filemeta_failed", Trace) + ) + end + ), + % Restore local storage backend + Root = iolist_to_binary(emqx_ft_test_helpers:root(Config, node(), [segments])), + ?assertMatch( + {ok, _}, + emqx_conf:update( + [file_transfer, storage], + #{ + <<"type">> => <<"local">>, + <<"segments">> => #{ + <<"root">> => Root, + <<"gc">> => #{<<"interval">> => <<"1s">>} + } + }, + #{} + ) + ), + % Verify that GC is getting triggered eventually + ?check_trace( + ?block_until(#{?snk_kind := garbage_collection}, 5000, 0), + fun(Trace) -> + ?assertMatch( + [ + #{ + ?snk_kind := garbage_collection, + storage := #{ + type := local, + segments := #{root := Root} + } + } + ], + ?of_kind(garbage_collection, Trace) + ) + end + ), + % Verify that transfers work again + ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <>). + +gen_clientid() -> + emqx_base62:encode(emqx_guid:gen()). diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index abf2749ed..065e9ae0a 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -68,7 +68,7 @@ end_per_testcase(_TC, _Config) -> t_gc_triggers_periodically(_Config) -> Interval = 500, ok = set_gc_config(interval, Interval), - ok = emqx_ft_storage_fs_gc:reset(emqx_ft_conf:storage()), + ok = emqx_ft_storage_fs_gc:reset(), ?check_trace( timer:sleep(Interval * 3), fun(Trace) -> @@ -92,7 +92,7 @@ t_gc_triggers_manually(_Config) -> ?assertMatch( #gcstats{files = 0, directories = 0, space = 0, errors = #{} = Errors} when map_size(Errors) == 0, - emqx_ft_storage_fs_gc:collect(emqx_ft_conf:storage()) + emqx_ft_storage_fs_gc:collect() ), fun(Trace) -> [Event] = ?of_kind(garbage_collection, Trace), @@ -108,7 +108,7 @@ t_gc_complete_transfers(_Config) -> ok = set_gc_config(minimum_segments_ttl, 0), ok = set_gc_config(maximum_segments_ttl, 3), ok = set_gc_config(interval, 500), - ok = emqx_ft_storage_fs_gc:reset(Storage), + ok = emqx_ft_storage_fs_gc:reset(), Transfers = [ { T1 = {<<"client1">>, mk_file_id()}, @@ -134,7 +134,7 @@ t_gc_complete_transfers(_Config) -> ?assertEqual([S1, S2, S3], TransferSizes), ?assertMatch( #gcstats{files = 0, directories = 0, errors = #{} = Es} when map_size(Es) == 0, - emqx_ft_storage_fs_gc:collect(Storage) + emqx_ft_storage_fs_gc:collect() ), % 2. Complete just the first transfer {ok, {ok, Event}} = ?wait_async_action( @@ -224,7 +224,7 @@ t_gc_incomplete_transfers(_Config) -> _ = emqx_utils:pmap(fun(Transfer) -> start_transfer(Storage, Transfer) end, Transfers), % 2. Enable periodic GC every 0.5 seconds. ok = set_gc_config(interval, 500), - ok = emqx_ft_storage_fs_gc:reset(Storage), + ok = emqx_ft_storage_fs_gc:reset(), % 3. First we need the first transfer to be collected. {ok, _} = ?block_until( #{ @@ -304,7 +304,7 @@ t_gc_handling_errors(_Config) -> {directory, DirTransfer2} := eexist } } when Files == ?NSEGS(Size, SegSize) * 2 andalso Space > Size * 2, - emqx_ft_storage_fs_gc:collect(Storage) + emqx_ft_storage_fs_gc:collect() ), fun(Trace) -> ?assertMatch( diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index 704e55454..11ddf191b 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -68,15 +68,22 @@ tcp_port(Node) -> root(Config, Node, Tail) -> filename:join([?config(priv_dir, Config), "file_transfer", Node | Tail]). +start_client(ClientId) -> + start_client(ClientId, node()). + +start_client(ClientId, Node) -> + Port = tcp_port(Node), + {ok, Client} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, Port}]), + {ok, _} = emqtt:connect(Client), + Client. + upload_file(ClientId, FileId, Name, Data) -> upload_file(ClientId, FileId, Name, Data, node()). upload_file(ClientId, FileId, Name, Data, Node) -> - Port = tcp_port(Node), - Size = byte_size(Data), + C1 = start_client(ClientId, Node), - {ok, C1} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, Port}]), - {ok, _} = emqtt:connect(C1), + Size = byte_size(Data), Meta = #{ name => Name, expire_at => erlang:system_time(_Unit = second) + 3600, From a420c92d28562bb27be0084789eaaf2543398f9b Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 24 Apr 2023 20:37:50 +0300 Subject: [PATCH 131/197] fix(ft): fix typing issue --- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 5754d2cfc..823407307 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -98,6 +98,7 @@ }. -type storage() :: #{ + type := 'local', segments := segments(), exporter := emqx_ft_storage_exporter:exporter() }. From 811e449357dda9d2b874459c29c5d86e8c2997da Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 25 Apr 2023 19:16:14 +0300 Subject: [PATCH 132/197] feat(ft-conf): provide global killswitch --- apps/emqx_ft/src/emqx_ft.erl | 9 +- apps/emqx_ft/src/emqx_ft_app.erl | 2 - apps/emqx_ft/src/emqx_ft_conf.erl | 34 ++++- apps/emqx_ft/src/emqx_ft_schema.erl | 22 ++-- apps/emqx_ft/test/emqx_ft_SUITE.erl | 1 + apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 12 +- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 117 ++++++++++++++---- .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 12 +- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 2 +- rel/i18n/emqx_ft_schema.hocon | 6 + 10 files changed, 148 insertions(+), 69 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index d500a6344..42611e537 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -198,8 +198,7 @@ on_file_command(PacketId, FileId, Msg, FileCommand) -> end. on_init(PacketId, Msg, Transfer, Meta) -> - ?SLOG(info, #{ - msg => "on_init", + ?tp(info, "file_transfer_init", #{ mqtt_msg => Msg, packet_id => PacketId, transfer => Transfer, @@ -229,8 +228,7 @@ on_abort(_Msg, _FileId) -> ?RC_SUCCESS. on_segment(PacketId, Msg, Transfer, Offset, Checksum) -> - ?SLOG(info, #{ - msg => "on_segment", + ?tp(info, "file_transfer_segment", #{ mqtt_msg => Msg, packet_id => PacketId, transfer => Transfer, @@ -255,8 +253,7 @@ on_segment(PacketId, Msg, Transfer, Offset, Checksum) -> end). on_fin(PacketId, Msg, Transfer, FinalSize, Checksum) -> - ?SLOG(info, #{ - msg => "on_fin", + ?tp(info, "file_transfer_fin", #{ mqtt_msg => Msg, packet_id => PacketId, transfer => Transfer, diff --git a/apps/emqx_ft/src/emqx_ft_app.erl b/apps/emqx_ft/src/emqx_ft_app.erl index 9b1513b46..0bac6b592 100644 --- a/apps/emqx_ft/src/emqx_ft_app.erl +++ b/apps/emqx_ft/src/emqx_ft_app.erl @@ -22,11 +22,9 @@ start(_StartType, _StartArgs) -> {ok, Sup} = emqx_ft_sup:start_link(), - ok = emqx_ft:hook(), ok = emqx_ft_conf:load(), {ok, Sup}. stop(_State) -> ok = emqx_ft_conf:unload(), - ok = emqx_ft:unhook(), ok. diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 1e531ecdb..90b59c8d1 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -23,6 +23,7 @@ -include_lib("emqx/include/logger.hrl"). %% Accessors +-export([enabled/0]). -export([storage/0]). -export([gc_interval/1]). -export([segments_ttl/1]). @@ -49,6 +50,10 @@ %% Accessors %%-------------------------------------------------------------------- +-spec enabled() -> boolean(). +enabled() -> + emqx_config:get([file_transfer, enable], false). + -spec storage() -> _Storage. storage() -> emqx_config:get([file_transfer, storage], undefined). @@ -83,7 +88,7 @@ store_segment_timeout() -> -spec load() -> ok. load() -> - ok = emqx_ft_storage:on_config_update(undefined, storage()), + ok = on_config_update(#{}, emqx_config:get([file_transfer], #{})), emqx_conf:add_handler([file_transfer], ?MODULE). -spec unload() -> ok. @@ -107,7 +112,26 @@ pre_config_update(_, Req, _Config) -> emqx_config:app_envs() ) -> ok | {ok, Result :: any()} | {error, Reason :: term()}. -post_config_update(_Path, _Req, NewConfig, OldConfig, _AppEnvs) -> - OldStorageConfig = maps:get(storage, OldConfig, undefined), - NewStorageConfig = maps:get(storage, NewConfig, undefined), - emqx_ft_storage:on_config_update(OldStorageConfig, NewStorageConfig). +post_config_update([file_transfer | _], _Req, NewConfig, OldConfig, _AppEnvs) -> + on_config_update(OldConfig, NewConfig). + +on_config_update(OldConfig, NewConfig) -> + lists:foreach( + fun(ConfKey) -> + on_config_update( + ConfKey, + maps:get(ConfKey, OldConfig, undefined), + maps:get(ConfKey, NewConfig, undefined) + ) + end, + [storage, enable] + ). + +on_config_update(_, Config, Config) -> + ok; +on_config_update(storage, OldConfig, NewConfig) -> + ok = emqx_ft_storage:on_config_update(OldConfig, NewConfig); +on_config_update(enable, _, true) -> + ok = emqx_ft:hook(); +on_config_update(enable, _, false) -> + ok = emqx_ft:unhook(). diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 27c593b6c..e2eebbbb8 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -55,6 +55,15 @@ roots() -> [file_transfer]. fields(file_transfer) -> [ + {enable, + mk( + boolean(), + #{ + desc => ?DESC("enable"), + required => false, + default => false + } + )}, {init_timeout, mk( emqx_schema:duration_ms(), @@ -87,22 +96,19 @@ fields(file_transfer) -> hoconsc:union( fun (all_union_members) -> - [ - % NOTE: by default storage is disabled - undefined, - ref(local_storage) - ]; + [ref(local_storage)]; ({value, #{<<"type">> := <<"local">>}}) -> [ref(local_storage)]; ({value, #{<<"type">> := _}}) -> throw(#{field_name => type, expected => "local"}); - (_) -> - [undefined] + ({value, _}) -> + [ref(local_storage)] end ), #{ required => false, - desc => ?DESC("storage") + desc => ?DESC("storage"), + default => #{<<"type">> => <<"local">>} } )} ]; diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 151b8e5fe..d3a3aee21 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -63,6 +63,7 @@ set_special_configs(Config) -> % NOTE % Inhibit local fs GC to simulate it isn't fast enough to collect % complete transfers. + enable => true, storage => emqx_utils_maps:deep_merge( Storage, #{segments => #{gc => #{interval => 0}}} diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index 5f3b213fb..523026d5a 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -30,7 +30,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> ok = emqx_mgmt_api_test_util:init_suite( - [emqx_conf, emqx_ft], set_special_configs(Config) + [emqx_conf, emqx_ft], emqx_ft_test_helpers:env_handler(Config) ), {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. @@ -38,16 +38,6 @@ end_per_suite(_Config) -> ok = emqx_mgmt_api_test_util:end_suite([emqx_ft, emqx_conf]), ok. -set_special_configs(Config) -> - fun - (emqx_ft) -> - emqx_ft_test_helpers:load_config(#{ - storage => emqx_ft_test_helpers:local_storage(Config) - }); - (_) -> - ok - end. - init_per_testcase(Case, Config) -> [{tc, Case} | Config]. end_per_testcase(_Case, _Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index 89b0e895d..106c34702 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -26,22 +26,21 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - _ = emqx_config:save_schema_mod_and_names(emqx_ft_schema), - ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_ft], emqx_ft_test_helpers:env_handler(Config) - ), - {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. end_per_suite(_Config) -> - ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), ok. init_per_testcase(_Case, Config) -> + % NOTE: running each testcase with clean config + _ = emqx_config:save_schema_mod_and_names(emqx_ft_schema), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], fun(_) -> ok end), + {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. end_per_testcase(_Case, _Config) -> - ok. + ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok = emqx_config:erase(file_transfer). %%-------------------------------------------------------------------- %% Tests @@ -61,6 +60,7 @@ t_update_config(_Config) -> emqx_conf:update( [file_transfer], #{ + <<"enable">> => true, <<"storage">> => #{ <<"type">> => <<"local">>, <<"segments">> => #{ @@ -87,10 +87,14 @@ t_update_config(_Config) -> emqx_ft_conf:gc_interval(emqx_ft_conf:storage()) ). -t_remove_restore_config(Config) -> +t_disable_restore_config(Config) -> ?assertMatch( {ok, _}, - emqx_conf:update([file_transfer, storage], #{<<"type">> => <<"local">>}, #{}) + emqx_conf:update( + [file_transfer], + #{<<"enable">> => true, <<"storage">> => #{<<"type">> => <<"local">>}}, + #{} + ) ), ?assertEqual( 60 * 60 * 1000, @@ -98,24 +102,29 @@ t_remove_restore_config(Config) -> ), % Verify that transfers work ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <>), + % Verify that clearing storage settings reverts config to defaults ?assertMatch( {ok, _}, - emqx_conf:remove([file_transfer, storage], #{}) + emqx_conf:update( + [file_transfer], + #{<<"enable">> => false, <<"storage">> => undefined}, + #{} + ) ), ?assertEqual( - undefined, + false, + emqx_ft_conf:enabled() + ), + ?assertMatch( + #{type := local, exporter := #{type := local}}, emqx_ft_conf:storage() ), - ?assertEqual( - undefined, - emqx_ft_conf:gc_interval(emqx_ft_conf:storage()) - ), ClientId = gen_clientid(), Client = emqx_ft_test_helpers:start_client(ClientId), % Verify that transfers fail cleanly when storage is disabled ?check_trace( ?assertMatch( - {ok, #{reason_code_name := unspecified_error}}, + {ok, #{reason_code_name := no_matching_subscribers}}, emqtt:publish( Client, <<"$file/f2/init">>, @@ -124,23 +133,24 @@ t_remove_restore_config(Config) -> ) ), fun(Trace) -> - ?assertMatch( - [#{transfer := {ClientId, <<"f2">>}, reason := {error, disabled}}], - ?of_kind("store_filemeta_failed", Trace) - ) + ?assertMatch([], ?of_kind("file_transfer_init", Trace)) end ), + ok = emqtt:stop(Client), % Restore local storage backend Root = iolist_to_binary(emqx_ft_test_helpers:root(Config, node(), [segments])), ?assertMatch( {ok, _}, emqx_conf:update( - [file_transfer, storage], + [file_transfer], #{ - <<"type">> => <<"local">>, - <<"segments">> => #{ - <<"root">> => Root, - <<"gc">> => #{<<"interval">> => <<"1s">>} + <<"enable">> => true, + <<"storage">> => #{ + <<"type">> => <<"local">>, + <<"segments">> => #{ + <<"root">> => Root, + <<"gc">> => #{<<"interval">> => <<"1s">>} + } } }, #{} @@ -167,5 +177,62 @@ t_remove_restore_config(Config) -> % Verify that transfers work again ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <>). +t_switch_exporter(_Config) -> + ?assertMatch( + {ok, _}, + emqx_conf:update( + [file_transfer], + #{<<"enable">> => true}, + #{} + ) + ), + ?assertMatch( + #{type := local, exporter := #{type := local}}, + emqx_ft_conf:storage() + ), + % Verify that switching to a different exporter works + ?assertMatch( + {ok, _}, + emqx_conf:update( + [file_transfer, storage, exporter], + #{ + <<"type">> => <<"s3">>, + <<"bucket">> => <<"emqx">>, + <<"host">> => <<"https://localhost">>, + <<"port">> => 9000, + <<"transport_options">> => #{ + <<"ipv6_probe">> => false + } + }, + #{} + ) + ), + ?assertMatch( + #{type := local, exporter := #{type := s3}}, + emqx_ft_conf:storage() + ), + % Verify that switching back to local exporter works + ?assertMatch( + {ok, _}, + emqx_conf:remove( + [file_transfer, storage, exporter], + #{} + ) + ), + ?assertMatch( + {ok, _}, + emqx_conf:update( + [file_transfer, storage, exporter], + #{<<"type">> => <<"local">>}, + #{} + ) + ), + ?assertMatch( + #{type := local, exporter := #{type := local}}, + emqx_ft_conf:storage() + ), + % Verify that transfers work + ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <>). + gen_clientid() -> emqx_base62:encode(emqx_guid:gen()). diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index e3decf0f5..d4c13f7d1 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -35,22 +35,12 @@ groups() -> ]. init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_ft], set_special_configs(Config)), + ok = emqx_common_test_helpers:start_apps([emqx_ft], emqx_ft_test_helpers:env_handler(Config)), Config. end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_ft]), ok. -set_special_configs(Config) -> - fun - (emqx_ft) -> - emqx_ft_test_helpers:load_config(#{ - storage => emqx_ft_test_helpers:local_storage(Config) - }); - (_) -> - ok - end. - init_per_testcase(Case, Config) -> [{tc, Case} | Config]. end_per_testcase(_Case, _Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index 11ddf191b..89e349fae 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -41,7 +41,7 @@ stop_additional_node(Node) -> env_handler(Config) -> fun (emqx_ft) -> - load_config(#{storage => local_storage(Config)}); + load_config(#{enable => true, storage => local_storage(Config)}); (_) -> ok end. diff --git a/rel/i18n/emqx_ft_schema.hocon b/rel/i18n/emqx_ft_schema.hocon index 28c93e1ef..e7e551289 100644 --- a/rel/i18n/emqx_ft_schema.hocon +++ b/rel/i18n/emqx_ft_schema.hocon @@ -1,5 +1,11 @@ emqx_ft_schema { +enable.desc: +"""Enable the File Transfer feature.
+Enabling File Transfer implies reserving special MQTT topics in order to serve the protocol.
+This toggle does not have an effect neither on the availability of the File Transfer REST API, nor +on storage-dependent background activities (e.g. garbage collection).""" + init_timeout.desc: """Timeout for initializing the file transfer.
After reaching the timeout, `init` message will be acked with an error""" From a79cf681f7d405bc63fcb90ac045af1f18f5b799 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 27 Apr 2023 18:42:06 +0300 Subject: [PATCH 133/197] fix(s3): drop default from `acl` config field --- apps/emqx_s3/src/emqx_s3_client.erl | 2 +- apps/emqx_s3/src/emqx_s3_schema.erl | 1 - apps/emqx_s3/test/emqx_s3_schema_SUITE.erl | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index e3e1d84af..3bc5861c6 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -57,7 +57,7 @@ port := part_number(), bucket := string(), headers := headers(), - acl := emqx_s3:acl(), + acl := emqx_s3:acl() | undefined, url_expire_time := pos_integer(), access_key_id := string() | undefined, secret_access_key := string() | undefined, diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index 23e69ec5d..5866f8c2b 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -105,7 +105,6 @@ fields(s3) -> bucket_owner_full_control ]), #{ - default => private, desc => ?DESC("acl"), required => false } diff --git a/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl b/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl index 89ec8a958..63f659da0 100644 --- a/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl @@ -23,7 +23,6 @@ t_minimal_config(_Config) -> bucket := "bucket", host := "s3.us-east-1.endpoint.com", port := 443, - acl := private, min_part_size := 5242880, transport_options := #{ From 573bb22ada94110485dc3660a4033981d188cadb Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 30 Mar 2023 19:18:47 +0300 Subject: [PATCH 134/197] feat(ft-fs): introduce fs iterators concept + forward seeks In order to support paging over filesystem contents, to serve REST APIs effectively. --- apps/emqx_ft/src/emqx_ft_fs_iterator.erl | 219 ++++++++++++++++++++ apps/emqx_ft/src/emqx_ft_fs_util.erl | 66 ++---- apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl | 113 +++++++++- 3 files changed, 342 insertions(+), 56 deletions(-) create mode 100644 apps/emqx_ft/src/emqx_ft_fs_iterator.erl diff --git a/apps/emqx_ft/src/emqx_ft_fs_iterator.erl b/apps/emqx_ft/src/emqx_ft_fs_iterator.erl new file mode 100644 index 000000000..5c8857ab0 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_fs_iterator.erl @@ -0,0 +1,219 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_fs_iterator). + +-export([new/2]). +-export([next/1]). +-export([next_leaf/1]). + +-export([seek/3]). + +-export([fold/3]). + +-export_type([t/0]). +-export_type([glob/0]). +-export_type([pathstack/0]). + +-type root() :: file:name(). +-type glob() :: ['*' | globfun()]. +-type globfun() :: + fun((_Filename :: file:name()) -> boolean()) + | fun((_Filename :: file:name(), pathstack()) -> boolean()). + +% A path stack is a list of path components, in reverse order. +-type pathstack() :: [file:name(), ...]. + +-opaque t() :: #{ + root := root(), + queue := [_PathStack :: [file:name()]], + head := glob(), + stack := [{[pathstack()], glob()}] +}. + +-type entry() :: entry_leaf() | entry_node(). +-type entry_leaf() :: + {leaf, file:name(), file:file_info() | {error, file:posix()}, pathstack()}. +-type entry_node() :: + {node, file:name(), {error, file:posix()}, pathstack()}. + +-spec new(root(), glob()) -> + t(). +new(Root, Glob) -> + #{ + root => Root, + queue => [[]], + head => Glob, + stack => [] + }. + +-spec next(t()) -> + {entry(), t()} | none. +next(It = #{queue := [PathStack | Rest], head := []}) -> + {emit(PathStack, It), It#{queue => Rest}}; +next(It = #{queue := [PathStack | Rest], head := [Pat | _], root := Root}) -> + Filepath = mk_filepath(PathStack), + case emqx_ft_fs_util:list_dir(filename:join(Root, Filepath)) of + {ok, Filenames} -> + Sorted = lists:sort(Filenames), + Matches = [[Fn | PathStack] || Fn <- Sorted, matches_glob(Pat, Fn, [Fn | PathStack])], + ItNext = windup(It), + next(ItNext#{queue => Matches}); + {error, _} = Error -> + {{node, Filepath, Error, PathStack}, It#{queue => Rest}} + end; +next(It = #{queue := []}) -> + unwind(It). + +windup(It = #{queue := [_ | Rest], head := [Pat | Glob], stack := Stack}) -> + % NOTE + % Preserve unfinished paths and glob in the stack, so that we can resume traversal + % when the lower levels of the tree are exhausted. + It#{ + head => Glob, + stack => [{Rest, [Pat | Glob]} | Stack] + }. + +unwind(It = #{stack := [{Queue, Glob} | StackRest]}) -> + % NOTE + % Resume traversal of unfinished paths from the upper levels of the tree. + next(It#{ + queue => Queue, + head => Glob, + stack => StackRest + }); +unwind(#{stack := []}) -> + none. + +emit(PathStack, #{root := Root}) -> + Filepath = mk_filepath(PathStack), + case emqx_ft_fs_util:read_info(filename:join(Root, Filepath)) of + {ok, Fileinfo} -> + {leaf, Filepath, Fileinfo, PathStack}; + {error, _} = Error -> + {leaf, Filepath, Error, PathStack} + end. + +mk_filepath([]) -> + ""; +mk_filepath(PathStack) -> + filename:join(lists:reverse(PathStack)). + +matches_glob('*', _, _) -> + true; +matches_glob(FilterFun, Filename, _PathStack) when is_function(FilterFun, 1) -> + FilterFun(Filename); +matches_glob(FilterFun, Filename, PathStack) when is_function(FilterFun, 2) -> + FilterFun(Filename, PathStack). + +%% + +-spec next_leaf(t()) -> + {entry_leaf(), t()} | none. +next_leaf(It) -> + case next(It) of + {{leaf, _, _, _} = Leaf, ItNext} -> + {Leaf, ItNext}; + {{node, _Filename, _Error, _PathStack}, ItNext} -> + % NOTE + % Intentionally skipping intermediate traversal errors here, for simplicity. + next_leaf(ItNext); + none -> + none + end. + +%% + +-spec seek([file:name()], root(), glob()) -> + t(). +seek(PathSeek, Root, Glob) -> + SeekGlob = mk_seek_glob(PathSeek, Glob), + SeekStack = lists:reverse(PathSeek), + case next_leaf(new(Root, SeekGlob)) of + {{leaf, _Filepath, _Info, SeekStack}, It} -> + fixup_glob(Glob, It); + {{leaf, _Filepath, _Info, Successor}, It = #{queue := Queue}} -> + fixup_glob(Glob, It#{queue => [Successor | Queue]}); + none -> + none(Root) + end. + +mk_seek_glob(PathSeek, Glob) -> + % NOTE + % The seek glob is a glob that skips all the nodes / leaves that are lexicographically + % smaller than the seek path. For example, if the seek path is ["a", "b", "c"], and + % the glob is ['*', '*', '*', '*'], then the seek glob is: + % [ fun(Path) -> Path >= ["a"] end, + % fun(Path) -> Path >= ["a", "b"] end, + % fun(Path) -> Path >= ["a", "b", "c"] end, + % '*' + % ] + L = min(length(PathSeek), length(Glob)), + merge_glob([mk_seek_pat(lists:sublist(PathSeek, N)) || N <- lists:seq(1, L)], Glob). + +mk_seek_pat(PathSeek) -> + % NOTE + % The `PathStack` and `PathSeek` are of the same length here. + fun(_Filename, PathStack) -> lists:reverse(PathStack) >= PathSeek end. + +merge_glob([Pat | SeekRest], [PatOrig | Rest]) -> + [merge_pat(Pat, PatOrig) | merge_glob(SeekRest, Rest)]; +merge_glob([], [PatOrig | Rest]) -> + [PatOrig | merge_glob([], Rest)]; +merge_glob([], []) -> + []. + +merge_pat(Pat, PatOrig) -> + fun(Filename, PathStack) -> + Pat(Filename, PathStack) andalso matches_glob(PatOrig, Filename, PathStack) + end. + +fixup_glob(Glob, It = #{head := [], stack := Stack}) -> + % NOTE + % Restoring original glob through the stack. Strictly speaking, this is not usually + % necessary, it's a kind of optimization. + fixup_glob(Glob, lists:reverse(Stack), It#{stack => []}). + +fixup_glob(Glob = [_ | Rest], [{Queue, _} | StackRest], It = #{stack := Stack}) -> + fixup_glob(Rest, StackRest, It#{stack => [{Queue, Glob} | Stack]}); +fixup_glob(Rest, [], It) -> + It#{head => Rest}. + +%% + +-spec fold(fun((entry(), Acc) -> Acc), Acc, t()) -> + Acc. +fold(FoldFun, Acc, It) -> + case next(It) of + {Entry, ItNext} -> + fold(FoldFun, FoldFun(Entry, Acc), ItNext); + none -> + Acc + end. + +%% + +-spec none(root()) -> + t(). +none(Root) -> + % NOTE + % The _none_ iterator is a valid iterator, but it will never yield any entries. + #{ + root => Root, + queue => [], + head => [], + stack => [] + }. diff --git a/apps/emqx_ft/src/emqx_ft_fs_util.erl b/apps/emqx_ft/src/emqx_ft_fs_util.erl index df9135816..b731d3270 100644 --- a/apps/emqx_ft/src/emqx_ft_fs_util.erl +++ b/apps/emqx_ft/src/emqx_ft_fs_util.erl @@ -25,18 +25,16 @@ -export([read_decode_file/2]). -export([read_info/1]). +-export([list_dir/1]). -export([fold/4]). --type glob() :: ['*' | globfun()]. --type globfun() :: - fun((_Filename :: file:name()) -> boolean()). -type foldfun(Acc) :: fun( ( _Filepath :: file:name(), - _Info :: file:file_info() | {error, _IoError}, - _Stack :: [file:name()], + _Info :: file:file_info() | {error, file:posix()}, + _Stack :: emqx_ft_fs_iterator:pathstack(), Acc ) -> Acc ). @@ -153,46 +151,8 @@ read_info(AbsPath) -> % Be aware that this function is occasionally mocked in `emqx_ft_fs_util_SUITE`. file:read_link_info(AbsPath, [{time, posix}, raw]). --spec fold(foldfun(Acc), Acc, _Root :: file:name(), glob()) -> - Acc. -fold(Fun, Acc, Root, Glob) -> - fold(Fun, Acc, [], Root, Glob, []). - -fold(Fun, AccIn, Path, Root, [Glob | Rest], Stack) when Glob == '*' orelse is_function(Glob) -> - case list_dir(filename:join(Root, Path)) of - {ok, Filenames} -> - lists:foldl( - fun(FN, Acc) -> - case matches_glob(Glob, FN) of - true when Path == [] -> - fold(Fun, Acc, FN, Root, Rest, [FN | Stack]); - true -> - fold(Fun, Acc, filename:join(Path, FN), Root, Rest, [FN | Stack]); - false -> - Acc - end - end, - AccIn, - Filenames - ); - {error, enotdir} -> - AccIn; - {error, Reason} -> - Fun(Path, {error, Reason}, Stack, AccIn) - end; -fold(Fun, AccIn, Filepath, Root, [], Stack) -> - case ?MODULE:read_info(filename:join(Root, Filepath)) of - {ok, Info} -> - Fun(Filepath, Info, Stack, AccIn); - {error, Reason} -> - Fun(Filepath, {error, Reason}, Stack, AccIn) - end. - -matches_glob('*', _) -> - true; -matches_glob(FilterFun, Filename) when is_function(FilterFun) -> - FilterFun(Filename). - +-spec list_dir(file:name_all()) -> + {ok, [file:name()]} | {error, file:posix() | badarg}. list_dir(AbsPath) -> case ?MODULE:read_info(AbsPath) of {ok, #file_info{type = directory}} -> @@ -202,3 +162,19 @@ list_dir(AbsPath) -> {error, Reason} -> {error, Reason} end. + +-spec fold(foldfun(Acc), Acc, _Root :: file:name(), emqx_ft_fs_iterator:glob()) -> + Acc. +fold(FoldFun, Acc, Root, Glob) -> + fold(FoldFun, Acc, emqx_ft_fs_iterator:new(Root, Glob)). + +fold(FoldFun, Acc, It) -> + case emqx_ft_fs_iterator:next(It) of + {{node, _Path, {error, enotdir}, _PathStack}, ItNext} -> + fold(FoldFun, Acc, ItNext); + {{_Type, Path, Info, PathStack}, ItNext} -> + AccNext = FoldFun(Path, Info, PathStack, Acc), + fold(FoldFun, AccNext, ItNext); + none -> + Acc + end. diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl index 81a483651..e4aa70f81 100644 --- a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl @@ -34,7 +34,7 @@ t_fold_single_level(Config) -> {"c", #file_info{type = directory}, ["c"]}, {"d", #file_info{type = directory}, ["d"]} ], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*'])) + sort(fold(fun cons/4, [], Root, ['*'])) ). t_fold_multi_level(Config) -> @@ -45,7 +45,7 @@ t_fold_multi_level(Config) -> {"a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]}, {"d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} ], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*'])) + sort(fold(fun cons/4, [], Root, ['*', '*', '*', '*'])) ), ?assertMatch( [ @@ -53,32 +53,32 @@ t_fold_multi_level(Config) -> {"c/bar/中文", #file_info{type = regular}, ["中文", "bar", "c"]}, {"d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]} ], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*'])) + sort(fold(fun cons/4, [], Root, ['*', '*', '*'])) ). t_fold_no_glob(Config) -> Root = ?config(data_dir, Config), ?assertMatch( [{"", #file_info{type = directory}, []}], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, [])) + sort(fold(fun cons/4, [], Root, [])) ). t_fold_glob_too_deep(Config) -> Root = ?config(data_dir, Config), ?assertMatch( [], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*', '*'])) + sort(fold(fun cons/4, [], Root, ['*', '*', '*', '*', '*'])) ). t_fold_invalid_root(Config) -> Root = ?config(data_dir, Config), ?assertMatch( [], - sort(emqx_ft_fs_util:fold(fun cons/4, [], filename:join([Root, "a", "link"]), ['*'])) + sort(fold(fun cons/4, [], filename:join([Root, "a", "link"]), ['*'])) ), ?assertMatch( [], - sort(emqx_ft_fs_util:fold(fun cons/4, [], filename:join([Root, "d", "haystack"]), ['*'])) + sort(fold(fun cons/4, [], filename:join([Root, "d", "haystack"]), ['*'])) ). t_fold_filter_unicode(Config) -> @@ -88,13 +88,13 @@ t_fold_filter_unicode(Config) -> {"a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]}, {"d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} ], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', fun is_latin1/1])) + sort(fold(fun cons/4, [], Root, ['*', '*', '*', fun is_latin1/1])) ), ?assertMatch( [ {"a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]} ], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', is_not(fun is_latin1/1)])) + sort(fold(fun cons/4, [], Root, ['*', '*', '*', is_not(fun is_latin1/1)])) ). t_fold_filter_levels(Config) -> @@ -104,7 +104,7 @@ t_fold_filter_levels(Config) -> {"a/b/foo", #file_info{type = directory}, ["foo", "b", "a"]}, {"d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]} ], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, [fun is_letter/1, fun is_letter/1, '*'])) + sort(fold(fun cons/4, [], Root, [fun is_letter/1, fun is_letter/1, '*'])) ). t_fold_errors(Config) -> @@ -128,11 +128,99 @@ t_fold_errors(Config) -> {"c/link", {error, enotsup}, ["link", "c"]}, {"d/e/baz/needle", {error, ebusy}, ["needle", "baz", "e", "d"]} ], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*'])) + sort(fold(fun cons/4, [], Root, ['*', '*', '*', '*'])) + ). + +t_seek_fold(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {leaf, "a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]}, + {leaf, "a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]}, + {leaf, "d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} + | _Nodes + ], + sort( + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:seek(["a", "a"], Root, ['*', '*', '*', '*']) + ) + ) + ), + ?assertMatch( + [ + {leaf, "a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]}, + {leaf, "d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} + | _Nodes + ], + sort( + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:seek(["a", "b", "foo", "42"], Root, ['*', '*', '*', '*']) + ) + ) + ), + ?assertMatch( + [ + {leaf, "d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} + | _Nodes + ], + sort( + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:seek(["c", "d", "e", "f"], Root, ['*', '*', '*', '*']) + ) + ) + ). + +t_seek_empty(Config) -> + Root = ?config(data_dir, Config), + ?assertEqual( + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:new(Root, ['*', '*', '*', '*']) + ), + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:seek([], Root, ['*', '*', '*', '*']) + ) + ). + +t_seek_past_end(Config) -> + Root = ?config(data_dir, Config), + ?assertEqual( + none, + emqx_ft_fs_iterator:next( + emqx_ft_fs_iterator:seek(["g", "h"], Root, ['*', '*', '*', '*']) + ) + ). + +t_seek_with_filter(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {leaf, "d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]} + | _Nodes + ], + sort( + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:seek(["a", "link"], Root, ['*', fun is_letter/1, '*']) + ) + ) ). %% +fold(FoldFun, Acc, Root, Glob) -> + emqx_ft_fs_util:fold(FoldFun, Acc, Root, Glob). + is_not(F) -> fun(X) -> not F(X) end. @@ -155,5 +243,8 @@ is_letter(Filename) -> cons(Path, Info, Stack, Acc) -> [{Path, Info, Stack} | Acc]. +cons(Entry, Acc) -> + [Entry | Acc]. + sort(L) when is_list(L) -> lists:sort(L). From ed3756ea0928965c628a518015b5c5d0c79f01ef Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Sat, 1 Apr 2023 00:08:46 +0300 Subject: [PATCH 135/197] feat(ft-api): add paging support through cursors --- apps/emqx_ft/src/emqx_ft_api.erl | 131 +++++++- apps/emqx_ft/src/emqx_ft_fs_iterator.erl | 16 + apps/emqx_ft/src/emqx_ft_storage.erl | 29 +- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 16 +- .../src/emqx_ft_storage_exporter_fs.erl | 301 ++++++++++++------ .../src/emqx_ft_storage_exporter_fs_proxy.erl | 6 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 6 +- .../emqx_ft_storage_exporter_fs_proto_v1.erl | 13 +- apps/emqx_ft/test/emqx_ft_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 97 +++++- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 2 +- .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 2 +- rel/i18n/emqx_ft_api.hocon | 3 + 13 files changed, 482 insertions(+), 142 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index 85fd8d8cb..7e1ed97ad 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -19,7 +19,6 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). --include_lib("emqx/include/logger.hrl"). %% Swagger specs from hocon schema -export([ @@ -30,12 +29,14 @@ ]). -export([ - roots/0 + roots/0, + fields/1 ]). %% API callbacks -export([ - '/file_transfer/files'/2 + '/file_transfer/files'/2, + '/file_transfer/files/:clientid/:fileid'/2 ]). -import(hoconsc, [mk/2, ref/1, ref/2]). @@ -47,7 +48,8 @@ api_spec() -> paths() -> [ - "/file_transfer/files" + "/file_transfer/files", + "/file_transfer/files/:clientid/:fileid" ]. schema("/file_transfer/files") -> @@ -57,29 +59,133 @@ schema("/file_transfer/files") -> tags => [<<"file_transfer">>], summary => <<"List all uploaded files">>, description => ?DESC("file_list"), + parameters => [ + ref(following), + ref(emqx_dashboard_swagger, limit) + ], responses => #{ 200 => <<"Operation success">>, + 400 => emqx_dashboard_swagger:error_codes( + ['BAD_REQUEST'], <<"Invalid cursor">> + ), 503 => emqx_dashboard_swagger:error_codes( - ['SERVICE_UNAVAILABLE'], <<"Service unavailable">> + ['SERVICE_UNAVAILABLE'], error_desc('SERVICE_UNAVAILABLE') + ) + } + } + }; +schema("/file_transfer/files/:clientid/:fileid") -> + #{ + 'operationId' => '/file_transfer/files/:clientid/:fileid', + get => #{ + tags => [<<"file_transfer">>], + summary => <<"List files uploaded in a specific transfer">>, + description => ?DESC("file_list_transfer"), + parameters => [ + ref(client_id), + ref(file_id) + ], + responses => #{ + 200 => <<"Operation success">>, + 404 => emqx_dashboard_swagger:error_codes( + ['FILES_NOT_FOUND'], error_desc('FILES_NOT_FOUND') + ), + 503 => emqx_dashboard_swagger:error_codes( + ['SERVICE_UNAVAILABLE'], error_desc('SERVICE_UNAVAILABLE') ) } } }. -'/file_transfer/files'(get, #{}) -> - case emqx_ft_storage:files() of - {ok, Files} -> - {200, #{<<"files">> => lists:map(fun format_file_info/1, Files)}}; - {error, _} -> - {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} +'/file_transfer/files'(get, #{ + query_string := QueryString +}) -> + try + Limit = limit(QueryString), + Query = + case maps:get(<<"following">>, QueryString, undefined) of + undefined -> + #{limit => Limit}; + Cursor -> + #{limit => Limit, following => Cursor} + end, + case emqx_ft_storage:files(Query) of + {ok, Page} -> + {200, format_page(Page)}; + {error, _} -> + {503, error_msg('SERVICE_UNAVAILABLE')} + end + catch + error:{badarg, cursor} -> + {400, error_msg('BAD_REQUEST', <<"Invalid cursor">>)} end. +'/file_transfer/files/:clientid/:fileid'(get, #{ + bindings := #{clientid := ClientId, fileid := FileId} +}) -> + Transfer = {ClientId, FileId}, + case emqx_ft_storage:files(#{transfer => Transfer}) of + {ok, Page} -> + {200, format_page(Page)}; + {error, [{_Node, enoent} | _]} -> + {404, error_msg('FILES_NOT_FOUND')}; + {error, _} -> + {503, error_msg('SERVICE_UNAVAILABLE')} + end. + +format_page(#{items := Files, cursor := Cursor}) -> + #{ + <<"files">> => lists:map(fun format_file_info/1, Files), + <<"cursor">> => Cursor + }; +format_page(#{items := Files}) -> + #{ + <<"files">> => lists:map(fun format_file_info/1, Files) + }. + +error_msg(Code) -> + #{code => Code, message => error_desc(Code)}. + error_msg(Code, Msg) -> #{code => Code, message => emqx_utils:readable_error_msg(Msg)}. +error_desc('FILES_NOT_FOUND') -> + <<"Files requested for this transfer could not be found">>; +error_desc('SERVICE_UNAVAILABLE') -> + <<"Service unavailable">>. + roots() -> []. +-spec fields(hocon_schema:name()) -> [hoconsc:field()]. +fields(client_id) -> + [ + {clientid, + mk(binary(), #{ + in => path, + desc => <<"MQTT Client ID">>, + required => true + })} + ]; +fields(file_id) -> + [ + {fileid, + mk(binary(), #{ + in => path, + desc => <<"File ID">>, + required => true + })} + ]; +fields(following) -> + [ + {following, + mk(binary(), #{ + in => query, + desc => <<"Cursor to start listing files from">>, + required => false + })} + ]. + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- @@ -115,3 +221,6 @@ format_name(NameBin) when is_binary(NameBin) -> NameBin; format_name(Name) when is_list(Name) -> iolist_to_binary(Name). + +limit(QueryString) -> + maps:get(<<"limit">>, QueryString, emqx_mgmt:default_row_limit()). diff --git a/apps/emqx_ft/src/emqx_ft_fs_iterator.erl b/apps/emqx_ft/src/emqx_ft_fs_iterator.erl index 5c8857ab0..7fb8d8634 100644 --- a/apps/emqx_ft/src/emqx_ft_fs_iterator.erl +++ b/apps/emqx_ft/src/emqx_ft_fs_iterator.erl @@ -23,6 +23,7 @@ -export([seek/3]). -export([fold/3]). +-export([fold_n/4]). -export_type([t/0]). -export_type([glob/0]). @@ -204,6 +205,21 @@ fold(FoldFun, Acc, It) -> Acc end. +%% NOTE +%% Passing negative `N` is allowed, in which case the iterator will be exhausted +%% completely, like in `fold/3`. +-spec fold_n(fun((entry(), Acc) -> Acc), Acc, t(), _N :: integer()) -> + {Acc, {more, t()} | none}. +fold_n(_FoldFun, Acc, It, 0) -> + {Acc, {more, It}}; +fold_n(FoldFun, Acc, It, N) -> + case next(It) of + {Entry, ItNext} -> + fold_n(FoldFun, FoldFun(Entry, Acc), ItNext, N - 1); + none -> + {Acc, none} + end. + %% -spec none(root()) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index fee16cd09..5364211a4 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -25,6 +25,7 @@ assemble/2, files/0, + files/1, with_storage_type/2, with_storage_type/3, @@ -36,12 +37,27 @@ -type storage() :: emqx_config:config(). -export_type([assemble_callback/0]). + +-export_type([query/1]). +-export_type([page/2]). -export_type([file_info/0]). -export_type([export_data/0]). -export_type([reader/0]). -type assemble_callback() :: fun((ok | {error, term()}) -> any()). +-type query(Cursor) :: + #{transfer => emqx_ft:transfer()} + | #{ + limit => non_neg_integer(), + following => Cursor + }. + +-type page(Item, Cursor) :: #{ + items := [Item], + cursor => Cursor +}. + -type file_info() :: #{ transfer := emqx_ft:transfer(), name := file:name(), @@ -71,8 +87,8 @@ -callback assemble(storage(), emqx_ft:transfer(), _Size :: emqx_ft:bytes()) -> ok | {async, pid()} | {error, term()}. --callback files(storage()) -> - {ok, [file_info()]} | {error, term()}. +-callback files(storage(), query(Cursor)) -> + {ok, page(file_info(), Cursor)} | {error, term()}. %%-------------------------------------------------------------------- %% API @@ -105,9 +121,14 @@ assemble(Transfer, Size) -> with_storage(assemble, [Transfer, Size]). -spec files() -> - {ok, [file_info()]} | {error, term()}. + {ok, page(file_info(), _)} | {error, term()}. files() -> - with_storage(files, []). + files(#{}). + +-spec files(query(Cursor)) -> + {ok, page(file_info(), Cursor)} | {error, term()}. +files(Query) -> + with_storage(files, [Query]). -spec with_storage(atom() | function()) -> any(). with_storage(Fun) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index 72128cb40..fb44093c1 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -17,7 +17,7 @@ %% Filesystem storage exporter %% %% This is conceptually a part of the Filesystem storage backend that defines -%% how and where complete tranfers are assembled into files and stored. +%% how and where complete transfers are assembled into files and stored. -module(emqx_ft_storage_exporter). @@ -28,7 +28,7 @@ -export([discard/1]). %% Listing API --export([list/1]). +-export([list/2]). %% Lifecycle API -export([on_config_update/2]). @@ -70,8 +70,8 @@ -callback discard(ExportSt :: export_st()) -> ok | {error, _Reason}. --callback list(storage()) -> - {ok, [emqx_ft_storage:file_info()]} | {error, _Reason}. +-callback list(exporter_conf(), emqx_ft_storage:query(Cursor)) -> + {ok, emqx_ft_storage:page(emqx_ft_storage:file_info(), Cursor)} | {error, _Reason}. %% Lifecycle callbacks @@ -133,11 +133,11 @@ complete(#{mod := ExporterMod, st := ExportSt, hash := Hash, filemeta := Filemet discard(#{mod := ExporterMod, st := ExportSt}) -> ExporterMod:discard(ExportSt). --spec list(storage()) -> - {ok, [emqx_ft_storage:file_info()]} | {error, _Reason}. -list(Storage) -> +-spec list(storage(), emqx_ft_storage:query(Cursor)) -> + {ok, emqx_ft_storage:page(emqx_ft_storage:file_info(), Cursor)} | {error, _Reason}. +list(Storage, Query) -> {ExporterMod, ExporterOpts} = exporter(Storage), - ExporterMod:list(ExporterOpts). + ExporterMod:list(ExporterOpts, Query). %% Lifecycle diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index 4b05c9a58..9109dadbb 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -37,12 +37,23 @@ %% Internal API for RPC -export([list_local/1]). -export([list_local/2]). +-export([list_local_transfer/2]). -export([start_reader/3]). -% TODO -% -export([list/2]). +-export([list/2]). + +-export_type([export_st/0]). +-export_type([options/0]). + +-type options() :: #{ + root => file:name(), + _ => _ +}. + +-type query() :: emqx_ft_storage:query(cursor()). +-type page(T) :: emqx_ft_storage:page(T, cursor()). +-type cursor() :: iodata(). --type options() :: _TODO. -type transfer() :: emqx_ft:transfer(). -type filemeta() :: emqx_ft:filemeta(). -type exportinfo() :: emqx_ft_storage:file_info(). @@ -70,22 +81,6 @@ %% 2 symbols = at most 256 directories on the second level -define(BUCKET2_LEN, 2). --define(SLOG_UNEXPECTED(RelFilepath, Fileinfo, Options), - ?SLOG(notice, "filesystem_object_unexpected", #{ - relpath => RelFilepath, - fileinfo => Fileinfo, - options => Options - }) -). - --define(SLOG_INACCESSIBLE(RelFilepath, Reason, Options), - ?SLOG(warning, "filesystem_object_inaccessible", #{ - relpath => RelFilepath, - reason => Reason, - options => Options - }) -). - %%-------------------------------------------------------------------- %% Exporter behaviour %%-------------------------------------------------------------------- @@ -162,33 +157,33 @@ update(_OldOptions, _NewOptions) -> ok. %% Internal API %%-------------------------------------------------------------------- --spec list_local(options(), transfer()) -> - {ok, [exportinfo(), ...]} | {error, file_error()}. -list_local(Options, Transfer) -> - TransferRoot = mk_absdir(Options, Transfer, result), - case - emqx_ft_fs_util:fold( - fun - (_Path, {error, Reason}, [], []) -> - {error, Reason}; - (_Path, Fileinfo = #file_info{type = regular}, [Filename | _], Acc) -> - RelFilepath = filename:join(mk_result_reldir(Transfer) ++ [Filename]), - Info = mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo), - [Info | Acc]; - (RelFilepath, Fileinfo = #file_info{}, _, Acc) -> - ?SLOG_UNEXPECTED(RelFilepath, Fileinfo, Options), - Acc; - (RelFilepath, {error, Reason}, _, Acc) -> - ?SLOG_INACCESSIBLE(RelFilepath, Reason, Options), - Acc - end, - [], - TransferRoot, - [fun filter_manifest/1] - ) - of +-type local_query() :: emqx_ft_storage:query({transfer(), file:name()}). + +-spec list_local_transfer(options(), transfer()) -> + {ok, [exportinfo()]} | {error, file_error()}. +list_local_transfer(Options, Transfer) -> + It = emqx_ft_fs_iterator:new( + mk_absdir(Options, Transfer, result), + [fun filter_manifest/1] + ), + Result = emqx_ft_fs_iterator:fold( + fun + ({leaf, _Path, Fileinfo = #file_info{type = regular}, [Filename | _]}, Acc) -> + RelFilepath = filename:join(mk_result_reldir(Transfer) ++ [Filename]), + Info = mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo), + [Info | Acc]; + ({node, _Path, {error, Reason}, []}, []) -> + {error, Reason}; + (Entry, Acc) -> + ok = log_invalid_entry(Options, Entry), + Acc + end, + [], + It + ), + case Result of Infos = [_ | _] -> - {ok, Infos}; + {ok, lists:reverse(Infos)}; [] -> {error, enoent}; {error, Reason} -> @@ -196,9 +191,17 @@ list_local(Options, Transfer) -> end. -spec list_local(options()) -> - {ok, #{transfer() => [exportinfo(), ...]}}. + {ok, [exportinfo()]} | {error, file_error()}. list_local(Options) -> - Pattern = [ + list_local(Options, #{}). + +-spec list_local(options(), local_query()) -> + {ok, [exportinfo()]} | {error, file_error()}. +list_local(Options, #{transfer := Transfer}) -> + list_local_transfer(Options, Transfer); +list_local(Options, #{} = Query) -> + Root = get_storage_root(Options), + Glob = [ _Bucket1 = '*', _Bucket2 = '*', _Rest = '*', @@ -206,16 +209,30 @@ list_local(Options) -> _FileId = '*', fun filter_manifest/1 ], - Root = get_storage_root(Options), - {ok, - emqx_ft_fs_util:fold( - fun(RelFilepath, Info, Stack, Acc) -> - read_exportinfo(Options, RelFilepath, Info, Stack, Acc) - end, - [], - Root, - Pattern - )}. + It = + case Query of + #{following := Cursor} -> + emqx_ft_fs_iterator:seek(mk_path_seek(Cursor), Root, Glob); + #{} -> + emqx_ft_fs_iterator:new(Root, Glob) + end, + % NOTE + % In the rare case when some transfer contain more than one file, the paging mechanic + % here may skip over some files, when the cursor is transfer-only. + Limit = maps:get(limit, Query, -1), + {Exports, _} = emqx_ft_fs_iterator:fold_n( + fun(Entry, Acc) -> read_exportinfo(Options, Entry, Acc) end, + [], + It, + Limit + ), + {ok, Exports}. + +mk_path_seek(#{transfer := Transfer, name := Filename}) -> + mk_result_reldir(Transfer) ++ [Filename]; +mk_path_seek(#{transfer := Transfer}) -> + % NOTE: Any bitstring is greater than any list. + mk_result_reldir(Transfer) ++ [<<>>]. %%-------------------------------------------------------------------- %% Helpers @@ -227,16 +244,21 @@ filter_manifest(?MANIFEST) -> filter_manifest(Filename) -> ?MANIFEST =/= string:find(Filename, ?MANIFEST, trailing). -read_exportinfo(Options, RelFilepath, Fileinfo = #file_info{type = regular}, Stack, Acc) -> - [Filename, FileId, ClientId | _] = Stack, +read_exportinfo( + Options, + {leaf, RelFilepath, Fileinfo = #file_info{type = regular}, [Filename, FileId, ClientId | _]}, + Acc +) -> + % NOTE + % There might be more than one file for a single transfer (though + % extremely bad luck is needed for that, e.g. concurrent assemblers with + % different filemetas from different nodes). This might be unexpected for a + % client given the current protocol, yet might be helpful in the future. Transfer = dirnames_to_transfer(ClientId, FileId), Info = mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo), [Info | Acc]; -read_exportinfo(Options, RelFilepath, Fileinfo = #file_info{}, _Stack, Acc) -> - ?SLOG_UNEXPECTED(RelFilepath, Fileinfo, Options), - Acc; -read_exportinfo(Options, RelFilepath, {error, Reason}, _Stack, Acc) -> - ?SLOG_INACCESSIBLE(RelFilepath, Reason, Options), +read_exportinfo(Options, Entry, Acc) -> + ok = log_invalid_entry(Options, Entry), Acc. mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo) -> @@ -268,6 +290,19 @@ try_read_filemeta(Filepath, Info) -> mk_export_uri(RelFilepath) -> emqx_ft_storage_exporter_fs_api:mk_export_uri(node(), RelFilepath). +log_invalid_entry(Options, {_Type, RelFilepath, Fileinfo = #file_info{}, _Stack}) -> + ?SLOG(notice, "filesystem_object_unexpected", #{ + relpath => RelFilepath, + fileinfo => Fileinfo, + options => Options + }); +log_invalid_entry(Options, {_Type, RelFilepath, {error, Reason}, _Stack}) -> + ?SLOG(warning, "filesystem_object_inaccessible", #{ + relpath => RelFilepath, + reason => Reason, + options => Options + }). + -spec start_reader(options(), file:name(), _Caller :: pid()) -> {ok, reader()} | {error, enoent}. start_reader(Options, RelFilepath, CallerPid) -> @@ -282,32 +317,112 @@ start_reader(Options, RelFilepath, CallerPid) -> %% --spec list(options()) -> - {ok, [exportinfo(), ...]} | {error, [{node(), _Reason}]}. -list(_Options) -> - Nodes = mria_mnesia:running_nodes(), - Replies = emqx_ft_storage_exporter_fs_proto_v1:list_exports(Nodes), - {Results, Errors} = lists:foldl( - fun - ({_Node, {ok, {ok, Files}}}, {Acc, Errors}) -> - {Files ++ Acc, Errors}; - ({Node, {ok, {error, _} = Error}}, {Acc, Errors}) -> - {Acc, [{Node, Error} | Errors]}; - ({Node, Error}, {Acc, Errors}) -> - {Acc, [{Node, Error} | Errors]} - end, - {[], []}, - lists:zip(Nodes, Replies) - ), - length(Errors) > 0 andalso - ?SLOG(warning, #{msg => "list_remote_exports_failed", errors => Errors}), - case Results of - [_ | _] -> - {ok, Results}; - [] when Errors =:= [] -> - {ok, Results}; - [] -> - {error, Errors} +-spec list(options(), query()) -> + {ok, page(exportinfo())} | {error, [{node(), _Reason}]}. +list(_Options, Query = #{transfer := _Transfer}) -> + case list(Query) of + #{items := Exports = [_ | _]} -> + {ok, #{items => Exports}}; + #{items := [], errors := NodeErrors} -> + {error, NodeErrors} + end; +list(_Options, Query) -> + Result = list(Query), + case Result of + #{errors := NodeErrors} -> + ?SLOG(warning, "list_exports_errors", #{ + query => Query, + errors => NodeErrors + }); + #{} -> + ok + end, + case Result of + #{items := Exports, cursor := Cursor} -> + {ok, #{items => lists:reverse(Exports), cursor => encode_cursor(Cursor)}}; + #{items := Exports} -> + {ok, #{items => lists:reverse(Exports)}} + end. + +list(QueryIn) -> + {Nodes, NodeQuery} = decode_query(QueryIn, lists:sort(mria_mnesia:running_nodes())), + list_nodes(NodeQuery, Nodes, #{items => []}). + +list_nodes(Query, Nodes = [Node | Rest], Acc) -> + case emqx_ft_storage_exporter_fs_proto_v1:list_exports([Node], Query) of + [{ok, Result}] -> + list_accumulate(Result, Query, Nodes, Acc); + [Failure] -> + ?SLOG(warning, #{ + msg => "list_remote_exports_failed", + node => Node, + query => Query, + failure => Failure + }), + list_next(Query, Rest, Acc) + end; +list_nodes(_Query, [], Acc) -> + Acc. + +list_accumulate({ok, Exports}, Query, [Node | Rest], Acc = #{items := EAcc}) -> + NExports = length(Exports), + AccNext = Acc#{items := Exports ++ EAcc}, + case Query of + #{limit := Limit} when NExports < Limit -> + list_next(Query#{limit => Limit - NExports}, Rest, AccNext); + #{limit := _} -> + AccNext#{cursor => mk_cursor(Node, Exports)}; + #{} -> + list_next(Query, Rest, AccNext) + end; +list_accumulate({error, Reason}, Query, [Node | Rest], Acc) -> + EAcc = maps:get(errors, Acc, []), + list_next(Query, Rest, Acc#{errors => [{Node, Reason} | EAcc]}). + +list_next(Query, Nodes, Acc) -> + list_nodes(maps:remove(following, Query), Nodes, Acc). + +decode_query(Query = #{following := Cursor}, Nodes) -> + {Node, NodeCursor} = decode_cursor(Cursor), + {skip_query_nodes(Node, Nodes), Query#{following => NodeCursor}}; +decode_query(Query = #{}, Nodes) -> + {Nodes, Query}. + +skip_query_nodes(CNode, Nodes) -> + lists:dropwhile(fun(N) -> N < CNode end, Nodes). + +mk_cursor(Node, [_Last = #{transfer := Transfer, name := Name} | _]) -> + {Node, #{transfer => Transfer, name => Name}}. + +encode_cursor({Node, #{transfer := {ClientId, FileId}, name := Name}}) -> + emqx_utils_json:encode(#{ + <<"n">> => Node, + <<"cid">> => ClientId, + <<"fid">> => FileId, + <<"fn">> => unicode:characters_to_binary(Name) + }). + +decode_cursor(Cursor) -> + try + #{ + <<"n">> := NodeIn, + <<"cid">> := ClientId, + <<"fid">> := FileId, + <<"fn">> := NameIn + } = emqx_utils_json:decode(Cursor), + true = is_binary(ClientId), + true = is_binary(FileId), + Node = binary_to_existing_atom(NodeIn), + Name = unicode:characters_to_list(NameIn), + true = is_list(Name), + {Node, #{transfer => {ClientId, FileId}, name => Name}} + catch + error:{_, invalid_json} -> + error({badarg, cursor}); + error:{badmatch, _} -> + error({badarg, cursor}); + error:badarg -> + error({badarg, cursor}) end. %% @@ -352,9 +467,9 @@ mk_result_reldir(Transfer = {ClientId, FileId}) -> BucketRest/binary >> = binary:encode_hex(Hash), [ - Bucket1, - Bucket2, - BucketRest, + binary_to_list(Bucket1), + binary_to_list(Bucket2), + binary_to_list(BucketRest), emqx_ft_fs_util:escape_filename(ClientId), emqx_ft_fs_util:escape_filename(FileId) ]. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl index 943c053ff..13160bfc6 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl @@ -21,15 +21,15 @@ -module(emqx_ft_storage_exporter_fs_proxy). -export([ - list_exports_local/0, + list_exports_local/1, read_export_file_local/2 ]). -list_exports_local() -> +list_exports_local(Query) -> emqx_ft_storage:with_storage_type(local, fun(Storage) -> case emqx_ft_storage_exporter:exporter(Storage) of {emqx_ft_storage_exporter_fs, Options} -> - emqx_ft_storage_exporter_fs:list_local(Options) + emqx_ft_storage_exporter_fs:list_local(Options, Query) % NOTE % This case clause is currently deemed unreachable by dialyzer. % InvalidExporter -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 823407307..bd88727c0 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -45,7 +45,7 @@ -export([get_subdir/2]). -export([get_subdir/3]). --export([files/1]). +-export([files/2]). -export([on_config_update/2]). @@ -217,8 +217,8 @@ assemble(Storage, Transfer, Size) -> %% -files(Storage) -> - emqx_ft_storage_exporter:list(Storage). +files(Storage, Query) -> + emqx_ft_storage_exporter:list(Storage, Query). %% diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl index 4c64011de..9ca6db786 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl @@ -20,7 +20,7 @@ -export([introduced_in/0]). --export([list_exports/1]). +-export([list_exports/2]). -export([read_export_file/3]). -include_lib("emqx/include/bpapi.hrl"). @@ -28,14 +28,17 @@ introduced_in() -> "5.0.17". --spec list_exports([node()]) -> - emqx_rpc:erpc_multicall([emqx_ft_storage:file_info()]). -list_exports(Nodes) -> +-spec list_exports([node()], emqx_ft_storage:query(_LocalCursor)) -> + emqx_rpc:erpc_multicall( + {ok, [emqx_ft_storage:file_info()]} + | {error, file:posix() | disabled | {invalid_storage_type, _}} + ). +list_exports(Nodes, Query) -> erpc:multicall( Nodes, emqx_ft_storage_exporter_fs_proxy, list_exports_local, - [] + [Query] ). -spec read_export_file(node(), file:name(), pid()) -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index d3a3aee21..929665ca9 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -671,7 +671,7 @@ encode_meta(Meta) -> emqx_utils_json:encode(emqx_ft:encode_filemeta(Meta)). list_files(ClientId) -> - {ok, Files} = emqx_ft_storage:files(), + {ok, #{items := Files}} = emqx_ft_storage:files(), [File || File = #{transfer := {CId, _}} <- Files, CId == ClientId]. read_export(#{path := AbsFilepath}) -> diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index 523026d5a..d9abd5a22 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -47,17 +47,31 @@ end_per_testcase(_Case, _Config) -> %% Tests %%-------------------------------------------------------------------- -t_list_ready_transfers(Config) -> +t_list_files(Config) -> ClientId = client_id(Config), + FileId = <<"f1">>, - ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, "f1", <<"data">>), + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "f1", <<"data">>), {ok, 200, #{<<"files">> := Files}} = - request(get, uri(["file_transfer", "files"]), fun json/1), + request_json(get, uri(["file_transfer", "files"])), - ?assertInclude( - #{<<"clientid">> := ClientId, <<"fileid">> := <<"f1">>}, - Files + ?assertMatch( + [#{<<"clientid">> := ClientId, <<"fileid">> := <<"f1">>}], + [File || File = #{<<"clientid">> := CId} <- Files, CId == ClientId] + ), + + {ok, 200, #{<<"files">> := FilesTransfer}} = + request_json(get, uri(["file_transfer", "files", ClientId, FileId])), + + ?assertMatch( + [#{<<"clientid">> := ClientId, <<"fileid">> := <<"f1">>}], + FilesTransfer + ), + + ?assertMatch( + {ok, 404, #{<<"code">> := <<"FILES_NOT_FOUND">>}}, + request_json(get, uri(["file_transfer", "files", ClientId, <<"no-such-file">>])) ). t_download_transfer(Config) -> @@ -67,10 +81,9 @@ t_download_transfer(Config) -> ?assertMatch( {ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}}, - request( + request_json( get, - uri(["file_transfer", "file"]) ++ query(#{fileref => <<"f1">>}), - fun json/1 + uri(["file_transfer", "file"]) ++ query(#{fileref => <<"f1">>}) ) ), @@ -99,7 +112,7 @@ t_download_transfer(Config) -> ), {ok, 200, #{<<"files">> := [File]}} = - request(get, uri(["file_transfer", "files"]), fun json/1), + request_json(get, uri(["file_transfer", "files"])), {ok, 200, Response} = request(get, host() ++ maps:get(<<"uri">>, File)), @@ -108,6 +121,58 @@ t_download_transfer(Config) -> Response ). +t_list_files_paging(Config) -> + ClientId = client_id(Config), + NFiles = 20, + Uploads = [{mk_file_id("file:", N), mk_file_name(N)} || N <- lists:seq(1, NFiles)], + ok = lists:foreach( + fun({FileId, Name}) -> + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, Name, <<"data">>) + end, + Uploads + ), + + ?assertMatch( + {ok, 200, #{<<"files">> := [_, _, _], <<"cursor">> := _}}, + request_json(get, uri(["file_transfer", "files"]) ++ query(#{limit => 3})) + ), + + {ok, 200, #{<<"files">> := Files}} = + request_json(get, uri(["file_transfer", "files"]) ++ query(#{limit => 100})), + + ?assert(length(Files) >= NFiles), + + ?assertNotMatch( + {ok, 200, #{<<"cursor">> := _}}, + request_json(get, uri(["file_transfer", "files"]) ++ query(#{limit => 100})) + ), + + ?assertMatch( + {ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}}, + request_json(get, uri(["file_transfer", "files"]) ++ query(#{limit => 0})) + ), + + ?assertMatch( + {ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}}, + request_json( + get, + uri(["file_transfer", "files"]) ++ query(#{following => <<"whatsthat!?">>}) + ) + ), + + PageThrough = fun PageThrough(Query, Acc) -> + case request_json(get, uri(["file_transfer", "files"]) ++ query(Query)) of + {ok, 200, #{<<"files">> := FilesPage, <<"cursor">> := Cursor}} -> + PageThrough(Query#{following => Cursor}, Acc ++ FilesPage); + {ok, 200, #{<<"files">> := FilesPage}} -> + Acc ++ FilesPage + end + end, + + ?assertEqual(Files, PageThrough(#{limit => 1}, [])), + ?assertEqual(Files, PageThrough(#{limit => 8}, [])), + ?assertEqual(Files, PageThrough(#{limit => NFiles}, [])). + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- @@ -115,13 +180,19 @@ t_download_transfer(Config) -> client_id(Config) -> atom_to_binary(?config(tc, Config), utf8). +mk_file_id(Prefix, N) -> + iolist_to_binary([Prefix, integer_to_list(N)]). + +mk_file_name(N) -> + "file." ++ integer_to_list(N). + request(Method, Url) -> emqx_mgmt_api_test_util:request(Method, Url, []). -request(Method, Url, Decoder) when is_function(Decoder) -> +request_json(Method, Url) -> case emqx_mgmt_api_test_util:request(Method, Url, []) of {ok, Code, Body} -> - {ok, Code, Decoder(Body)}; + {ok, Code, json(Body)}; Otherwise -> Otherwise end. @@ -138,6 +209,8 @@ uri_encode(T) -> to_list(A) when is_atom(A) -> atom_to_list(A); +to_list(A) when is_integer(A) -> + integer_to_list(A); to_list(B) when is_binary(B) -> binary_to_list(B); to_list(L) when is_list(L) -> diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index a7323fc0e..24a4593c2 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -240,7 +240,7 @@ list_exports(Config) -> list_exports(Config, Transfer) -> {emqx_ft_storage_exporter_fs, Options} = exporter(Config), - emqx_ft_storage_exporter_fs:list_local(Options, Transfer). + emqx_ft_storage_exporter_fs:list_local_transfer(Options, Transfer). exporter(Config) -> emqx_ft_storage_exporter:exporter(storage(Config)). diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index d4c13f7d1..e0675d167 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -87,5 +87,5 @@ storage(Config) -> emqx_ft_test_helpers:local_storage(Config). list_files(Config) -> - {ok, Files} = emqx_ft_storage_fs:files(storage(Config)), + {ok, #{items := Files}} = emqx_ft_storage_fs:files(storage(Config), #{}), Files. diff --git a/rel/i18n/emqx_ft_api.hocon b/rel/i18n/emqx_ft_api.hocon index 0c67db554..bf6c22411 100644 --- a/rel/i18n/emqx_ft_api.hocon +++ b/rel/i18n/emqx_ft_api.hocon @@ -3,6 +3,9 @@ emqx_ft_api { file_list.desc: """List all uploaded files.""" +file_list_transfer.desc +"""List a file uploaded during specified transfer, identified by client id and file id.""" + } emqx_ft_storage_exporter_fs_api { From 75cceffa06d09d319c4288ed07b300751cfa97db Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Sat, 1 Apr 2023 00:17:55 +0300 Subject: [PATCH 136/197] fix(ft-test): rename testcases for consistency --- apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index e0675d167..5635d981b 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -27,7 +27,7 @@ all() -> {group, cluster} ]. --define(CLUSTER_CASES, [t_multinode_ready_transfers]). +-define(CLUSTER_CASES, [t_multinode_exports]). groups() -> [ @@ -61,7 +61,7 @@ end_per_group(_Group, _Config) -> %% Tests %%-------------------------------------------------------------------- -t_multinode_ready_transfers(Config) -> +t_multinode_exports(Config) -> Node1 = ?config(additional_node, Config), ok = emqx_ft_test_helpers:upload_file(<<"c/1">>, <<"f:1">>, "fn1", <<"data">>, Node1), From a9866fede4cb4feba41a78d79885843ade0ad729 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 25 Apr 2023 14:42:26 +0300 Subject: [PATCH 137/197] feat(ft-api): support paging in S3 storage exporter --- .../src/emqx_ft_storage_exporter_s3.erl | 118 ++++++++++++------ 1 file changed, 82 insertions(+), 36 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index adf000346..c7110c74a 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -23,7 +23,7 @@ -export([write/2]). -export([complete/2]). -export([discard/1]). --export([list/1]). +-export([list/2]). -export([ start/1, @@ -43,6 +43,10 @@ filemeta => filemeta() }. +-type query() :: emqx_ft_storage:query(cursor()). +-type page(T) :: emqx_ft_storage:page(T, cursor()). +-type cursor() :: iodata(). + -type export_st() :: #{ pid := pid(), filemeta := filemeta(), @@ -92,10 +96,10 @@ complete(#{pid := Pid} = _ExportSt, _Checksum) -> discard(#{pid := Pid} = _ExportSt) -> emqx_s3_uploader:abort(Pid). --spec list(options()) -> - {ok, [exportinfo()]} | {error, term()}. -list(Options) -> - emqx_s3:with_client(?S3_PROFILE_ID, fun(Client) -> list(Client, Options) end). +-spec list(options(), query()) -> + {ok, page(exportinfo())} | {error, term()}. +list(Options, Query) -> + emqx_s3:with_client(?S3_PROFILE_ID, fun(Client) -> list(Client, Options, Query) end). %%-------------------------------------------------------------------- %% Exporter behaviour (lifecycle) @@ -117,12 +121,11 @@ update(_OldOptions, NewOptions) -> %% Internal functions %% ------------------------------------------------------------------- -s3_key({ClientId, FileId} = _Transfer, #{name := Filename}) -> - filename:join([ - emqx_ft_fs_util:escape_filename(ClientId), - emqx_ft_fs_util:escape_filename(FileId), - Filename - ]). +s3_key(Transfer, #{name := Filename}) -> + s3_prefix(Transfer) ++ "/" ++ Filename. + +s3_prefix({ClientId, FileId} = _Transfer) -> + emqx_ft_fs_util:escape_filename(ClientId) ++ "/" ++ emqx_ft_fs_util:escape_filename(FileId). s3_headers({ClientId, FileId}, Filemeta) -> #{ @@ -137,54 +140,97 @@ s3_headers({ClientId, FileId}, Filemeta) -> s3_header_filemeta(Filemeta) -> emqx_utils_json:encode(emqx_ft:encode_filemeta(Filemeta), [force_utf8, uescape]). -list(Client, Options) -> - case list_key_info(Client, Options) of - {ok, KeyInfos} -> - MaybeExportInfos = lists:map( - fun(KeyInfo) -> key_info_to_exportinfo(Client, KeyInfo, Options) end, KeyInfos - ), - ExportInfos = [ExportInfo || {ok, ExportInfo} <- MaybeExportInfos], - {ok, ExportInfos}; +list(Client, _Options, #{transfer := Transfer}) -> + case list_key_info(Client, [{prefix, s3_prefix(Transfer)}, {max_keys, ?S3_LIST_LIMIT}]) of + {ok, {Exports, _Marker}} -> + {ok, #{items => Exports}}; + {error, _Reason} = Error -> + Error + end; +list(Client, _Options, Query) -> + Limit = maps:get(limit, Query, undefined), + Marker = emqx_maybe:apply(fun decode_cursor/1, maps:get(cursor, Query, undefined)), + case list_pages(Client, Marker, Limit, []) of + {ok, {Exports, undefined}} -> + {ok, #{items => Exports}}; + {ok, {Exports, NextMarker}} -> + {ok, #{items => Exports, cursor => encode_cursor(NextMarker)}}; {error, _Reason} = Error -> Error end. -list_key_info(Client, Options) -> - list_key_info(Client, Options, _Marker = [], _Acc = []). +list_pages(Client, Marker, Limit, Acc) -> + MaxKeys = min(?S3_LIST_LIMIT, Limit), + ListOptions = [{marker, Marker} || Marker =/= undefined], + case list_key_info(Client, [{max_keys, MaxKeys} | ListOptions]) of + {ok, {Exports, NextMarker}} -> + list_accumulate(Client, Limit, NextMarker, [Exports | Acc]); + {error, _Reason} = Error -> + Error + end. -list_key_info(Client, Options, Marker, Acc) -> - ListOptions = [{max_keys, ?S3_LIST_LIMIT}] ++ Marker, +list_accumulate(_Client, _Limit, undefined, Acc) -> + {ok, {flatten_pages(Acc), undefined}}; +list_accumulate(Client, undefined, Marker, Acc) -> + list_pages(Client, Marker, undefined, Acc); +list_accumulate(Client, Limit, Marker, Acc = [Exports | _]) -> + case Limit - length(Exports) of + 0 -> + {ok, {flatten_pages(Acc), Marker}}; + Left -> + list_pages(Client, Marker, Left, Acc) + end. + +flatten_pages(Pages) -> + lists:append(lists:reverse(Pages)). + +list_key_info(Client, ListOptions) -> case emqx_s3_client:list(Client, ListOptions) of {ok, Result} -> ?SLOG(debug, #{msg => "list_key_info", result => Result}), KeyInfos = proplists:get_value(contents, Result, []), - case proplists:get_value(is_truncated, Result, false) of - true -> - NewMarker = next_marker(KeyInfos), - list_key_info(Client, Options, NewMarker, [KeyInfos | Acc]); - false -> - {ok, lists:append(lists:reverse([KeyInfos | Acc]))} - end; + Exports = lists:filtermap( + fun(KeyInfo) -> key_info_to_exportinfo(Client, KeyInfo) end, KeyInfos + ), + Marker = + case proplists:get_value(is_truncated, Result, false) of + true -> + next_marker(KeyInfos); + false -> + undefined + end, + {ok, {Exports, Marker}}; {error, _Reason} = Error -> Error end. -next_marker(KeyInfos) -> - [{marker, proplists:get_value(key, lists:last(KeyInfos))}]. +encode_cursor(Key) -> + unicode:characters_to_binary(Key). -key_info_to_exportinfo(Client, KeyInfo, _Options) -> +decode_cursor(Cursor) -> + case unicode:characters_to_list(Cursor) of + Key when is_list(Key) -> + Key; + _ -> + error({badarg, cursor}) + end. + +next_marker(KeyInfos) -> + proplists:get_value(key, lists:last(KeyInfos)). + +key_info_to_exportinfo(Client, KeyInfo) -> Key = proplists:get_value(key, KeyInfo), case parse_transfer_and_name(Key) of {ok, {Transfer, Name}} -> - {ok, #{ + {true, #{ transfer => Transfer, name => unicode:characters_to_binary(Name), uri => emqx_s3_client:uri(Client, Key), timestamp => datetime_to_epoch_second(proplists:get_value(last_modified, KeyInfo)), size => proplists:get_value(size, KeyInfo) }}; - {error, _Reason} = Error -> - Error + {error, _Reason} -> + false end. -define(EPOCH_START, 62167219200). From bd9f129bb84953e3848b1db449470bb3dbb6aa73 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 25 Apr 2023 14:44:00 +0300 Subject: [PATCH 138/197] test(ft-api): also run testcases under cluster setup --- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 80 +++++++++++++++++++++---- 1 file changed, 68 insertions(+), 12 deletions(-) diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index d9abd5a22..4efa31205 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -22,11 +22,19 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). --include_lib("emqx/include/asserts.hrl"). - -import(emqx_dashboard_api_test_helpers, [host/0, uri/1]). -all() -> emqx_common_test_helpers:all(?MODULE). +all() -> + [ + {group, single}, + {group, cluster} + ]. + +groups() -> + [ + {single, [], emqx_common_test_helpers:all(?MODULE)}, + {cluster, [], emqx_common_test_helpers:all(?MODULE)} + ]. init_per_suite(Config) -> ok = emqx_mgmt_api_test_util:init_suite( @@ -38,6 +46,41 @@ end_per_suite(_Config) -> ok = emqx_mgmt_api_test_util:end_suite([emqx_ft, emqx_conf]), ok. +init_per_group(Group = cluster, Config) -> + Cluster = mk_cluster_specs(Config), + ct:pal("Starting ~p", [Cluster]), + Nodes = [ + emqx_common_test_helpers:start_slave(Name, Opts#{join_to => node()}) + || {Name, Opts} <- Cluster + ], + [{group, Group}, {cluster_nodes, Nodes} | Config]; +init_per_group(Group, Config) -> + [{group, Group} | Config]. + +end_per_group(cluster, Config) -> + ok = lists:foreach( + fun emqx_ft_test_helpers:stop_additional_node/1, + ?config(cluster_nodes, Config) + ); +end_per_group(_Group, _Config) -> + ok. + +mk_cluster_specs(Config) -> + Specs = [ + {core, emqx_ft_api_SUITE1, #{listener_ports => [{tcp, 2883}]}}, + {core, emqx_ft_api_SUITE2, #{listener_ports => [{tcp, 3883}]}} + ], + CommOpts = [ + {env, [{emqx, boot_modules, [broker, listeners]}]}, + {apps, [emqx_ft]}, + {conf, [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]]}, + {env_handler, emqx_ft_test_helpers:env_handler(Config)} + ], + emqx_common_test_helpers:emqx_cluster( + Specs, + CommOpts + ). + init_per_testcase(Case, Config) -> [{tc, Case} | Config]. end_per_testcase(_Case, _Config) -> @@ -51,7 +94,8 @@ t_list_files(Config) -> ClientId = client_id(Config), FileId = <<"f1">>, - ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "f1", <<"data">>), + Node = lists:last(cluster(Config)), + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "f1", <<"data">>, Node), {ok, 200, #{<<"files">> := Files}} = request_json(get, uri(["file_transfer", "files"])), @@ -76,14 +120,16 @@ t_list_files(Config) -> t_download_transfer(Config) -> ClientId = client_id(Config), + FileId = <<"f1">>, - ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, "f1", <<"data">>), + Node = lists:last(cluster(Config)), + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "f1", <<"data">>, Node), ?assertMatch( {ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}}, request_json( get, - uri(["file_transfer", "file"]) ++ query(#{fileref => <<"f1">>}) + uri(["file_transfer", "file"]) ++ query(#{fileref => FileId}) ) ), @@ -93,7 +139,7 @@ t_download_transfer(Config) -> get, uri(["file_transfer", "file"]) ++ query(#{ - fileref => <<"f1">>, + fileref => FileId, node => <<"nonode@nohost">> }) ) @@ -112,7 +158,7 @@ t_download_transfer(Config) -> ), {ok, 200, #{<<"files">> := [File]}} = - request_json(get, uri(["file_transfer", "files"])), + request_json(get, uri(["file_transfer", "files", ClientId, FileId])), {ok, 200, Response} = request(get, host() ++ maps:get(<<"uri">>, File)), @@ -124,10 +170,14 @@ t_download_transfer(Config) -> t_list_files_paging(Config) -> ClientId = client_id(Config), NFiles = 20, - Uploads = [{mk_file_id("file:", N), mk_file_name(N)} || N <- lists:seq(1, NFiles)], + Nodes = cluster(Config), + Uploads = [ + {mk_file_id("file:", N), mk_file_name(N), pick(N, Nodes)} + || N <- lists:seq(1, NFiles) + ], ok = lists:foreach( - fun({FileId, Name}) -> - ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, Name, <<"data">>) + fun({FileId, Name, Node}) -> + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, Name, <<"data">>, Node) end, Uploads ), @@ -177,8 +227,11 @@ t_list_files_paging(Config) -> %% Helpers %%-------------------------------------------------------------------- +cluster(Config) -> + [node() | proplists:get_value(cluster_nodes, Config, [])]. + client_id(Config) -> - atom_to_binary(?config(tc, Config), utf8). + iolist_to_binary(io_lib:format("~s.~s", [?config(group, Config), ?config(tc, Config)])). mk_file_id(Prefix, N) -> iolist_to_binary([Prefix, integer_to_list(N)]). @@ -215,3 +268,6 @@ to_list(B) when is_binary(B) -> binary_to_list(B); to_list(L) when is_list(L) -> L. + +pick(N, List) -> + lists:nth(1 + (N rem length(List)), List). From cff7788b2e6e3114449cd8cbc98bf43c6482f3c6 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 27 Apr 2023 23:23:52 +0300 Subject: [PATCH 139/197] chore(ft): add s3 exporter tests --- .../src/emqx_dashboard_swagger.erl | 66 ++++-- apps/emqx_ft/docker-ct | 1 + apps/emqx_ft/etc/emqx_ft.conf | 15 -- apps/emqx_ft/src/emqx_ft_api.erl | 15 +- apps/emqx_ft/src/emqx_ft_conf.erl | 44 ++-- apps/emqx_ft/src/emqx_ft_storage.erl | 51 ++--- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 8 +- .../src/emqx_ft_storage_exporter_fs_api.erl | 4 +- .../src/emqx_ft_storage_exporter_s3.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 23 ++ apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 31 +++ apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 5 +- .../emqx_ft_storage_exporter_s3_SUITE.erl | 199 ++++++++++++++++++ .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 4 +- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 15 +- .../test/emqx_ft_storage_fs_reader_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 54 +++-- apps/emqx_s3/test/emqx_s3_test_helpers.erl | 15 +- rel/i18n/emqx_ft_schema.hocon | 4 +- 19 files changed, 437 insertions(+), 121 deletions(-) create mode 100644 apps/emqx_ft/docker-ct create mode 100644 apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index e42d9c3ae..6422d627c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -29,6 +29,7 @@ -export([file_schema/1]). -export([base_path/0]). -export([relative_uri/1]). +-export([compose_filters/2]). -export([filter_check_request/2, filter_check_request_and_translate_body/2]). @@ -82,14 +83,30 @@ -type request() :: #{bindings => map(), query_string => map(), body => map()}. -type request_meta() :: #{module => module(), path => string(), method => atom()}. --type filter_result() :: {ok, request()} | {400, 'BAD_REQUEST', binary()}. --type filter() :: fun((request(), request_meta()) -> filter_result()). +%% More exact types are defined in minirest.hrl, but we don't want to include it +%% because it defines a lot of types and they may clash with the types declared locally. +-type status_code() :: pos_integer(). +-type error_code() :: atom() | binary(). +-type error_message() :: binary(). +-type response_body() :: term(). +-type headers() :: map(). + +-type response() :: + status_code() + | {status_code()} + | {status_code(), response_body()} + | {status_code(), headers(), response_body()} + | {status_code(), error_code(), error_message()}. + +-type filter_result() :: {ok, request()} | response(). +-type filter() :: emqx_maybe:t(fun((request(), request_meta()) -> filter_result())). -type spec_opts() :: #{ check_schema => boolean() | filter(), translate_body => boolean(), schema_converter => fun((hocon_schema:schema(), Module :: atom()) -> map()), - i18n_lang => atom() | string() | binary() + i18n_lang => atom() | string() | binary(), + filter => filter() }. -type route_path() :: string() | binary(). @@ -115,9 +132,9 @@ spec(Module, Options) -> lists:foldl( fun(Path, {AllAcc, AllRefsAcc}) -> {OperationId, Specs, Refs} = parse_spec_ref(Module, Path, Options), - CheckSchema = support_check_schema(Options), + Opts = #{filter => filter(Options)}, { - [{filename:join("/", Path), Specs, OperationId, CheckSchema} | AllAcc], + [{filename:join("/", Path), Specs, OperationId, Opts} | AllAcc], Refs ++ AllRefsAcc } end, @@ -204,6 +221,21 @@ file_schema(FileName) -> } }. +-spec compose_filters(filter(), filter()) -> filter(). +compose_filters(undefined, Filter2) -> + Filter2; +compose_filters(Filter1, undefined) -> + Filter1; +compose_filters(Filter1, Filter2) -> + fun(Request, RequestMeta) -> + case Filter1(Request, RequestMeta) of + {ok, Request1} -> + Filter2(Request1, RequestMeta); + Response -> + Response + end + end. + %%------------------------------------------------------------------------------ %% Private functions %%------------------------------------------------------------------------------ @@ -235,14 +267,22 @@ check_only(Schema, Map, Opts) -> _ = hocon_tconf:check_plain(Schema, Map, Opts), Map. -support_check_schema(#{check_schema := true, translate_body := true}) -> - #{filter => fun ?MODULE:filter_check_request_and_translate_body/2}; -support_check_schema(#{check_schema := true}) -> - #{filter => fun ?MODULE:filter_check_request/2}; -support_check_schema(#{check_schema := Filter}) when is_function(Filter, 2) -> - #{filter => Filter}; -support_check_schema(_) -> - #{filter => undefined}. +filter(Options) -> + CheckSchemaFilter = check_schema_filter(Options), + CustomFilter = custom_filter(Options), + compose_filters(CheckSchemaFilter, CustomFilter). + +custom_filter(Options) -> + maps:get(filter, Options, undefined). + +check_schema_filter(#{check_schema := true, translate_body := true}) -> + fun ?MODULE:filter_check_request_and_translate_body/2; +check_schema_filter(#{check_schema := true}) -> + fun ?MODULE:filter_check_request/2; +check_schema_filter(#{check_schema := Filter}) when is_function(Filter, 2) -> + Filter; +check_schema_filter(_) -> + undefined. parse_spec_ref(Module, Path, Options) -> Schema = diff --git a/apps/emqx_ft/docker-ct b/apps/emqx_ft/docker-ct new file mode 100644 index 000000000..36f9d86d3 --- /dev/null +++ b/apps/emqx_ft/docker-ct @@ -0,0 +1 @@ +minio diff --git a/apps/emqx_ft/etc/emqx_ft.conf b/apps/emqx_ft/etc/emqx_ft.conf index ec48d4e69..e69de29bb 100644 --- a/apps/emqx_ft/etc/emqx_ft.conf +++ b/apps/emqx_ft/etc/emqx_ft.conf @@ -1,15 +0,0 @@ -file_transfer { - storage { - type = local - segments { - root = "{{ platform_data_dir }}/file_transfer/segments" - gc { - interval = "1h" - } - } - exporter { - root = "{{ platform_data_dir }}/file_transfer/exports" - type = local - } - } -} diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index 7e1ed97ad..61a6a2c93 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -33,6 +33,9 @@ fields/1 ]). +%% Minirest filter for checking if file transfer is enabled +-export([check_ft_enabled/2]). + %% API callbacks -export([ '/file_transfer/files'/2, @@ -44,7 +47,9 @@ namespace() -> "file_transfer". api_spec() -> - emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + emqx_dashboard_swagger:spec(?MODULE, #{ + check_schema => true, filter => fun ?MODULE:check_ft_enabled/2 + }). paths() -> [ @@ -97,6 +102,14 @@ schema("/file_transfer/files/:clientid/:fileid") -> } }. +check_ft_enabled(Params, _Meta) -> + case emqx_ft_conf:enabled() of + true -> + {ok, Params}; + false -> + {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} + end. + '/file_transfer/files'(get, #{ query_string := QueryString }) -> diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 90b59c8d1..61f639271 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -56,7 +56,7 @@ enabled() -> -spec storage() -> _Storage. storage() -> - emqx_config:get([file_transfer, storage], undefined). + emqx_config:get([file_transfer, storage]). -spec gc_interval(_Storage) -> emqx_maybe:t(milliseconds()). gc_interval(Conf = #{type := local}) -> @@ -88,11 +88,12 @@ store_segment_timeout() -> -spec load() -> ok. load() -> - ok = on_config_update(#{}, emqx_config:get([file_transfer], #{})), + ok = maybe_start(), emqx_conf:add_handler([file_transfer], ?MODULE). -spec unload() -> ok. unload() -> + ok = stop(), emqx_conf:remove_handler([file_transfer]). %%-------------------------------------------------------------------- @@ -115,23 +116,26 @@ pre_config_update(_, Req, _Config) -> post_config_update([file_transfer | _], _Req, NewConfig, OldConfig, _AppEnvs) -> on_config_update(OldConfig, NewConfig). -on_config_update(OldConfig, NewConfig) -> - lists:foreach( - fun(ConfKey) -> - on_config_update( - ConfKey, - maps:get(ConfKey, OldConfig, undefined), - maps:get(ConfKey, NewConfig, undefined) - ) - end, - [storage, enable] - ). - -on_config_update(_, Config, Config) -> +on_config_update(#{enable := false}, #{enable := false}) -> ok; -on_config_update(storage, OldConfig, NewConfig) -> - ok = emqx_ft_storage:on_config_update(OldConfig, NewConfig); -on_config_update(enable, _, true) -> +on_config_update(#{enable := true, storage := OldStorage}, #{enable := false}) -> + ok = emqx_ft_storage:on_config_update(OldStorage, undefined), + ok = emqx_ft:unhook(); +on_config_update(#{enable := false}, #{enable := true, storage := NewStorage}) -> + ok = emqx_ft_storage:on_config_update(undefined, NewStorage), ok = emqx_ft:hook(); -on_config_update(enable, _, false) -> - ok = emqx_ft:unhook(). +on_config_update(#{enable := true, storage := OldStorage}, #{enable := true, storage := NewStorage}) -> + ok = emqx_ft_storage:on_config_update(OldStorage, NewStorage). + +maybe_start() -> + case emqx_config:get([file_transfer]) of + #{enable := true, storage := Storage} -> + ok = emqx_ft_storage:on_config_update(undefined, Storage), + ok = emqx_ft:hook(); + _ -> + ok + end. + +stop() -> + ok = emqx_ft:unhook(), + ok = emqx_ft_storage:on_config_update(storage(), undefined). diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 5364211a4..5ec342585 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -90,6 +90,12 @@ -callback files(storage(), query(Cursor)) -> {ok, page(file_info(), Cursor)} | {error, term()}. +-callback start(emqx_config:config()) -> any(). +-callback stop(emqx_config:config()) -> any(). + +-callback on_config_update(_OldConfig :: emqx_config:config(), _NewConfig :: emqx_config:config()) -> + any(). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -167,49 +173,28 @@ apply_storage(Storage, Fun, Args) when is_function(Fun) -> -spec on_config_update(_Old :: emqx_maybe:t(storage()), _New :: emqx_maybe:t(storage())) -> ok. -on_config_update(Storage, Storage) -> +on_config_update(#{type := _} = Storage, #{type := _} = Storage) -> ok; on_config_update(#{type := Type} = StorageOld, #{type := Type} = StorageNew) -> ok = (mod(StorageNew)):on_config_update(StorageOld, StorageNew); -on_config_update(StorageOld, StorageNew) -> +on_config_update(StorageOld, StorageNew) when + (StorageOld =:= undefined orelse is_map_key(type, StorageOld)) andalso + (StorageNew =:= undefined orelse is_map_key(type, StorageNew)) +-> _ = emqx_maybe:apply(fun on_storage_stop/1, StorageOld), _ = emqx_maybe:apply(fun on_storage_start/1, StorageNew), - _ = emqx_maybe:apply( - fun(Storage) -> (mod(Storage)):on_config_update(StorageOld, StorageNew) end, - StorageNew - ), ok. -on_storage_start(Storage = #{type := _}) -> - lists:foreach( - fun(ChildSpec) -> - {ok, _Child} = supervisor:start_child(emqx_ft_sup, ChildSpec) - end, - child_spec(Storage) - ). - -on_storage_stop(Storage = #{type := _}) -> - lists:foreach( - fun(#{id := ChildId}) -> - _ = supervisor:terminate_child(emqx_ft_sup, ChildId), - ok = supervisor:delete_child(emqx_ft_sup, ChildId) - end, - child_spec(Storage) - ). - -child_spec(Storage) -> - try - Mod = mod(Storage), - Mod:child_spec(Storage) - catch - error:disabled -> []; - error:undef -> [] - end. - %%-------------------------------------------------------------------- -%% Local FS API +%% Local API %%-------------------------------------------------------------------- +on_storage_start(Storage) -> + (mod(Storage)):start(Storage). + +on_storage_stop(Storage) -> + (mod(Storage)):stop(Storage). + storage() -> emqx_ft_conf:storage(). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index fb44093c1..e000fe5c6 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -153,14 +153,14 @@ on_exporter_update(Config, Config) -> on_exporter_update({ExporterMod, ConfigOld}, {ExporterMod, ConfigNew}) -> ExporterMod:update(ConfigOld, ConfigNew); on_exporter_update(ExporterOld, ExporterNew) -> - _ = emqx_maybe:apply(fun stop_exporter/1, ExporterOld), - _ = emqx_maybe:apply(fun start_exporter/1, ExporterNew), + _ = emqx_maybe:apply(fun stop/1, ExporterOld), + _ = emqx_maybe:apply(fun start/1, ExporterNew), ok. -start_exporter({ExporterMod, ExporterOpts}) -> +start({ExporterMod, ExporterOpts}) -> ok = ExporterMod:start(ExporterOpts). -stop_exporter({ExporterMod, ExporterOpts}) -> +stop({ExporterMod, ExporterOpts}) -> ok = ExporterMod:stop(ExporterOpts). %%------------------------------------------------------------------------------ diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl index f1a8c6dae..a278b01a5 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl @@ -47,7 +47,9 @@ namespace() -> "file_transfer". api_spec() -> - emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + emqx_dashboard_swagger:spec(?MODULE, #{ + check_schema => true, filter => fun emqx_ft_api:check_ft_enabled/2 + }). paths() -> [ diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index c7110c74a..5c5aade86 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -149,7 +149,7 @@ list(Client, _Options, #{transfer := Transfer}) -> end; list(Client, _Options, Query) -> Limit = maps:get(limit, Query, undefined), - Marker = emqx_maybe:apply(fun decode_cursor/1, maps:get(cursor, Query, undefined)), + Marker = emqx_maybe:apply(fun decode_cursor/1, maps:get(following, Query, undefined)), case list_pages(Client, Marker, Limit, []) of {ok, {Exports, undefined}} -> {ok, #{items => Exports}}; diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index bd88727c0..cdc86d218 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -48,6 +48,8 @@ -export([files/2]). -export([on_config_update/2]). +-export([start/1]). +-export([stop/1]). -export_type([storage/0]). -export_type([filefrag/1]). @@ -227,6 +229,27 @@ on_config_update(StorageOld, StorageNew) -> ok = emqx_ft_storage_fs_gc:reset(StorageNew), emqx_ft_storage_exporter:on_config_update(StorageOld, StorageNew). +start(Storage) -> + ok = lists:foreach( + fun(ChildSpec) -> + {ok, _Child} = supervisor:start_child(emqx_ft_sup, ChildSpec) + end, + child_spec(Storage) + ), + ok = emqx_ft_storage_exporter:on_config_update(undefined, Storage), + ok. + +stop(Storage) -> + ok = emqx_ft_storage_exporter:on_config_update(Storage, undefined), + ok = lists:foreach( + fun(#{id := ChildId}) -> + _ = supervisor:terminate_child(emqx_ft_sup, ChildId), + ok = supervisor:delete_child(emqx_ft_sup, ChildId) + end, + child_spec(Storage) + ), + ok. + %% -spec transfers(storage()) -> diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index 4efa31205..f69e13a6d 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -83,6 +83,8 @@ mk_cluster_specs(Config) -> init_per_testcase(Case, Config) -> [{tc, Case} | Config]. +end_per_testcase(t_ft_disabled, _Config) -> + emqx_config:put([file_transfer, enable], true); end_per_testcase(_Case, _Config) -> ok. @@ -223,6 +225,35 @@ t_list_files_paging(Config) -> ?assertEqual(Files, PageThrough(#{limit => 8}, [])), ?assertEqual(Files, PageThrough(#{limit => NFiles}, [])). +t_ft_disabled(_Config) -> + ?assertMatch( + {ok, 200, _}, + request_json(get, uri(["file_transfer", "files"])) + ), + + ?assertMatch( + {ok, 400, _}, + request_json( + get, + uri(["file_transfer", "file"]) ++ query(#{fileref => <<"f1">>}) + ) + ), + + ok = emqx_config:put([file_transfer, enable], false), + + ?assertMatch( + {ok, 503, _}, + request_json(get, uri(["file_transfer", "files"])) + ), + + ?assertMatch( + {ok, 503, _}, + request_json( + get, + uri(["file_transfer", "file"]) ++ query(#{fileref => <<"f1">>, node => node()}) + ) + ). + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index 106c34702..bc9eb5d98 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -32,9 +32,10 @@ end_per_suite(_Config) -> ok. init_per_testcase(_Case, Config) -> - % NOTE: running each testcase with clean config _ = emqx_config:save_schema_mod_and_names(emqx_ft_schema), - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], fun(_) -> ok end), + ok = emqx_common_test_helpers:start_apps( + [emqx_conf, emqx_ft], emqx_ft_test_helpers:env_handler(Config) + ), {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. diff --git a/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl new file mode 100644 index 000000000..86de74ee0 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl @@ -0,0 +1,199 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_exporter_s3_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-define(assertS3Data(Data, Url), + case httpc:request(Url) of + {ok, {{_StatusLine, 200, "OK"}, _Headers, Body}} -> + ?assertEqual(Data, list_to_binary(Body), "S3 data mismatch"); + OtherResponse -> + ct:fail("Unexpected response: ~p", [OtherResponse]) + end +). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Config. +end_per_suite(_Config) -> + ok. + +set_special_configs(Config) -> + fun + (emqx_ft) -> + Storage = emqx_ft_test_helpers:local_storage(Config, #{ + exporter => s3, bucket_name => ?config(bucket_name, Config) + }), + emqx_ft_test_helpers:load_config(#{<<"enable">> => true, <<"storage">> => Storage}); + (_) -> + ok + end. + +init_per_testcase(Case, Config0) -> + ClientId = atom_to_binary(Case), + BucketName = create_bucket(), + Config1 = [{bucket_name, BucketName}, {clientid, ClientId} | Config0], + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], set_special_configs(Config1)), + Config1. +end_per_testcase(_Case, _Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok. + +%%-------------------------------------------------------------------- +%% Test Cases +%%------------------------------------------------------------------- + +t_happy_path(Config) -> + ClientId = ?config(clientid, Config), + + FileId = <<"🌚"/utf8>>, + Name = "cool_name", + Data = <<"data"/utf8>>, + + ?assertEqual( + ok, + emqx_ft_test_helpers:upload_file(ClientId, FileId, Name, Data) + ), + + {ok, #{items := [#{uri := Uri}]}} = emqx_ft_storage:files(), + + ?assertS3Data(Data, Uri), + + Key = binary_to_list(ClientId) ++ "/" ++ binary_to_list(FileId) ++ "/" ++ Name, + Meta = erlcloud_s3:get_object_metadata( + ?config(bucket_name, Config), Key, emqx_ft_test_helpers:aws_config() + ), + + ?assertEqual( + ClientId, + metadata_field("clientid", Meta) + ), + + ?assertEqual( + FileId, + metadata_field("fileid", Meta) + ), + + NameBin = list_to_binary(Name), + ?assertMatch( + #{ + <<"name">> := NameBin, + <<"size">> := 4 + }, + emqx_utils_json:decode(metadata_field("filemeta", Meta), [return_maps]) + ). + +t_upload_error(Config) -> + ClientId = ?config(clientid, Config), + + FileId = <<"🌚"/utf8>>, + Name = "cool_name", + Data = <<"data"/utf8>>, + + {ok, _} = emqx_conf:update( + [file_transfer, storage, exporter, bucket], <<"invalid-bucket">>, #{} + ), + + ?assertEqual( + {error, unspecified_error}, + emqx_ft_test_helpers:upload_file(ClientId, FileId, Name, Data) + ). + +t_paging(Config) -> + ClientId = ?config(clientid, Config), + N = 1050, + + FileId = fun integer_to_binary/1, + Name = "cool_name", + Data = fun integer_to_binary/1, + + ok = lists:foreach( + fun(I) -> + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId(I), Name, Data(I)) + end, + lists:seq(1, N) + ), + + {ok, #{items := [#{uri := Uri}]}} = emqx_ft_storage:files(#{transfer => {ClientId, FileId(123)}}), + + ?assertS3Data(Data(123), Uri), + + lists:foreach( + fun(PageSize) -> + Pages = file_pages(#{limit => PageSize}), + ?assertEqual( + expected_page_count(PageSize, N), + length(Pages) + ), + FileIds = [ + FId + || #{transfer := {_, FId}} <- lists:concat(Pages) + ], + ?assertEqual( + lists:sort([FileId(I) || I <- lists:seq(1, N)]), + lists:sort(FileIds) + ) + end, + %% less than S3 limit, greater than S3 limit + [20, 550] + ). + +t_invalid_cursor(_Config) -> + InvalidUtf8 = <<16#80>>, + ?assertError( + {badarg, cursor}, + emqx_ft_storage:files(#{following => InvalidUtf8}) + ). + +%%-------------------------------------------------------------------- +%% Helper Functions +%%-------------------------------------------------------------------- + +expected_page_count(PageSize, Total) -> + case Total rem PageSize of + 0 -> Total div PageSize; + _ -> Total div PageSize + 1 + end. + +file_pages(Query) -> + case emqx_ft_storage:files(Query) of + {ok, #{items := Items, cursor := NewCursor}} -> + [Items] ++ file_pages(Query#{following => NewCursor}); + {ok, #{items := Items}} -> + [Items]; + {error, Error} -> + ct:fail("Failed to download files: ~p", [Error]) + end. + +metadata_field(Field, Meta) -> + Key = "x-amz-meta-" ++ Field, + case lists:keyfind(Key, 1, Meta) of + {Key, Value} -> list_to_binary(Value); + false -> false + end. + +create_bucket() -> + BucketName = emqx_s3_test_helpers:unique_bucket(), + _ = application:ensure_all_started(lhttpc), + ok = erlcloud_s3:create_bucket(BucketName, emqx_ft_test_helpers:aws_config()), + BucketName. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index 5635d981b..2acb57a8e 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -84,7 +84,9 @@ client_id(Config) -> atom_to_binary(?config(tc, Config), utf8). storage(Config) -> - emqx_ft_test_helpers:local_storage(Config). + RawConfig = #{<<"storage">> => emqx_ft_test_helpers:local_storage(Config)}, + #{storage := Storage} = emqx_ft_schema:translate(RawConfig), + Storage. list_files(Config) -> {ok, #{items := Files}} = emqx_ft_storage_fs:files(storage(Config), #{}), diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index 065e9ae0a..04aedf8f3 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -40,14 +40,15 @@ init_per_testcase(TC, Config) -> emqx_ft, fun(emqx_ft) -> emqx_ft_test_helpers:load_config(#{ - storage => #{ - type => local, - segments => #{ - root => emqx_ft_test_helpers:root(Config, node(), [TC, segments]) + <<"enable">> => true, + <<"storage">> => #{ + <<"type">> => <<"local">>, + <<"segments">> => #{ + <<"root">> => emqx_ft_test_helpers:root(Config, node(), [TC, segments]) }, - exporter => #{ - type => local, - root => emqx_ft_test_helpers:root(Config, node(), [TC, exports]) + <<"exporter">> => #{ + <<"type">> => <<"local">>, + <<"root">> => emqx_ft_test_helpers:root(Config, node(), [TC, exports]) } } }) diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl index 0ac5d2844..217205f6f 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl @@ -25,7 +25,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_ft]), + ok = emqx_common_test_helpers:start_apps([emqx_ft], emqx_ft_test_helpers:env_handler(Config)), Config. end_per_suite(_Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index 89e349fae..1482223d8 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -21,6 +21,9 @@ -include_lib("common_test/include/ct.hrl"). +-define(S3_HOST, <<"minio">>). +-define(S3_PORT, 9000). + start_additional_node(Config, Name) -> emqx_common_test_helpers:start_slave( Name, @@ -41,32 +44,41 @@ stop_additional_node(Node) -> env_handler(Config) -> fun (emqx_ft) -> - load_config(#{enable => true, storage => local_storage(Config)}); + load_config(#{<<"enable">> => true, <<"storage">> => local_storage(Config)}); (_) -> ok end. local_storage(Config) -> + local_storage(Config, #{exporter => local}). + +local_storage(Config, Opts) -> #{ - type => local, - segments => #{ - root => root(Config, node(), [segments]) - }, - exporter => #{ - type => local, - root => root(Config, node(), [exports]) - } + <<"type">> => <<"local">>, + <<"segments">> => #{<<"root">> => root(Config, node(), [segments])}, + <<"exporter">> => exporter(Config, Opts) + }. + +exporter(Config, #{exporter := local}) -> + #{<<"type">> => <<"local">>, <<"root">> => root(Config, node(), [exports])}; +exporter(_Config, #{exporter := s3, bucket_name := BucketName}) -> + BaseConfig = emqx_s3_test_helpers:base_raw_config(tcp), + BaseConfig#{ + <<"bucket">> => list_to_binary(BucketName), + <<"type">> => <<"s3">>, + <<"host">> => ?S3_HOST, + <<"port">> => ?S3_PORT }. load_config(Config) -> - emqx_common_test_helpers:load_config(emqx_ft_schema, #{file_transfer => Config}). + emqx_common_test_helpers:load_config(emqx_ft_schema, #{<<"file_transfer">> => Config}). tcp_port(Node) -> {_, Port} = rpc:call(Node, emqx_config, get, [[listeners, tcp, default, bind]]), Port. root(Config, Node, Tail) -> - filename:join([?config(priv_dir, Config), "file_transfer", Node | Tail]). + iolist_to_binary(filename:join([?config(priv_dir, Config), "file_transfer", Node | Tail])). start_client(ClientId) -> start_client(ClientId, node()). @@ -94,9 +106,21 @@ upload_file(ClientId, FileId, Name, Data, Node) -> ct:pal("MetaPayload = ~ts", [MetaPayload]), MetaTopic = <<"$file/", FileId/binary, "/init">>, - {ok, _} = emqtt:publish(C1, MetaTopic, MetaPayload, 1), - {ok, _} = emqtt:publish(C1, <<"$file/", FileId/binary, "/0">>, Data, 1), + {ok, #{reason_code_name := success}} = emqtt:publish(C1, MetaTopic, MetaPayload, 1), + {ok, #{reason_code_name := success}} = emqtt:publish( + C1, <<"$file/", FileId/binary, "/0">>, Data, 1 + ), FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Size))/binary>>, - {ok, _} = emqtt:publish(C1, FinTopic, <<>>, 1), - ok = emqtt:stop(C1). + FinResult = + case emqtt:publish(C1, FinTopic, <<>>, 1) of + {ok, #{reason_code_name := success}} -> + ok; + {ok, #{reason_code_name := Error}} -> + {error, Error} + end, + ok = emqtt:stop(C1), + FinResult. + +aws_config() -> + emqx_s3_test_helpers:aws_config(tcp, binary_to_list(?S3_HOST), ?S3_PORT). diff --git a/apps/emqx_s3/test/emqx_s3_test_helpers.erl b/apps/emqx_s3/test/emqx_s3_test_helpers.erl index 2edd52609..a73f618af 100644 --- a/apps/emqx_s3/test/emqx_s3_test_helpers.erl +++ b/apps/emqx_s3/test/emqx_s3_test_helpers.erl @@ -36,19 +36,24 @@ %%-------------------------------------------------------------------- aws_config(tcp) -> + aws_config(tcp, ?TCP_HOST, ?TCP_PORT); +aws_config(tls) -> + aws_config(tls, ?TLS_HOST, ?TLS_PORT). + +aws_config(tcp, Host, Port) -> erlcloud_s3_new( ?ACCESS_KEY_ID, ?SECRET_ACCESS_KEY, - ?TCP_HOST, - ?TCP_PORT, + Host, + Port, "http://" ); -aws_config(tls) -> +aws_config(tls, Host, Port) -> erlcloud_s3_new( ?ACCESS_KEY_ID, ?SECRET_ACCESS_KEY, - ?TLS_HOST, - ?TLS_PORT, + Host, + Port, "https://" ). diff --git a/rel/i18n/emqx_ft_schema.hocon b/rel/i18n/emqx_ft_schema.hocon index e7e551289..64b9bd67e 100644 --- a/rel/i18n/emqx_ft_schema.hocon +++ b/rel/i18n/emqx_ft_schema.hocon @@ -3,8 +3,8 @@ emqx_ft_schema { enable.desc: """Enable the File Transfer feature.
Enabling File Transfer implies reserving special MQTT topics in order to serve the protocol.
-This toggle does not have an effect neither on the availability of the File Transfer REST API, nor -on storage-dependent background activities (e.g. garbage collection).""" +This toggle also affects the availability of the File Transfer REST API and +storage-dependent background activities (e.g. garbage collection).""" init_timeout.desc: """Timeout for initializing the file transfer.
From 8ac881a140a68a0440e383ab14e904554746ce97 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 9 May 2023 14:17:50 +0500 Subject: [PATCH 140/197] chore(ft): handle multiple/concurrent fins more gracefully --- apps/emqx_ft/src/emqx_ft.erl | 4 +- apps/emqx_ft/src/emqx_ft_assembler.erl | 5 + .../src/emqx_ft_storage_exporter_fs.erl | 4 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 51 +++++++- apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl | 6 +- .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 6 + apps/emqx_ft/test/emqx_ft_SUITE.erl | 110 +++++++++++++++--- 7 files changed, 163 insertions(+), 23 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 42611e537..dba087484 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -268,8 +268,8 @@ on_fin(PacketId, Msg, Transfer, FinalSize, Checksum) -> with_responder(FinPacketKey, Callback, emqx_ft_conf:assemble_timeout(), fun() -> case assemble(Transfer, FinalSize) of %% Assembling completed, ack through the responder right away - % ok -> - % emqx_ft_responder:ack(FinPacketKey, ok); + ok -> + emqx_ft_responder:ack(FinPacketKey, ok); %% Assembling started, packet will be acked by the responder {async, Pid} -> ok = emqx_ft_responder:kickoff(FinPacketKey, Pid), diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 767930f98..19f6c48d1 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -24,6 +24,8 @@ -export([handle_event/4]). -export([terminate/3]). +-export([where/1]). + -type stdata() :: #{ storage := emqx_ft_storage_fs:storage(), transfer := emqx_ft:transfer(), @@ -39,6 +41,9 @@ start_link(Storage, Transfer, Size) -> gen_statem:start_link(?REF(Transfer), ?MODULE, {Storage, Transfer, Size}, []). +where(Transfer) -> + gproc:where(?NAME(Transfer)). + %% -type state() :: diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index 9109dadbb..208c6fda3 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -324,7 +324,9 @@ list(_Options, Query = #{transfer := _Transfer}) -> #{items := Exports = [_ | _]} -> {ok, #{items => Exports}}; #{items := [], errors := NodeErrors} -> - {error, NodeErrors} + {error, NodeErrors}; + #{items := []} -> + {ok, #{items => []}} end; list(_Options, Query) -> Result = list(Query), diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index cdc86d218..010d004a1 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -35,6 +35,7 @@ -export([read_filemeta/2]). -export([list/3]). -export([pread/5]). +-export([lookup_local_assembler/1]). -export([assemble/3]). -export([transfers/1]). @@ -211,11 +212,15 @@ pread(_Storage, _Transfer, Frag, Offset, Size) -> end. -spec assemble(storage(), transfer(), emqx_ft:bytes()) -> - {async, _Assembler :: pid()} | {error, _TODO}. + {async, _Assembler :: pid()} | ok | {error, _TODO}. assemble(Storage, Transfer, Size) -> - % TODO: ask cluster if the transfer is already assembled - {ok, Pid} = emqx_ft_assembler_sup:ensure_child(Storage, Transfer, Size), - {async, Pid}. + LookupSources = [ + fun() -> lookup_local_assembler(Transfer) end, + fun() -> lookup_remote_assembler(Transfer) end, + fun() -> check_if_already_exported(Storage, Transfer) end, + fun() -> ensure_local_assembler(Storage, Transfer, Size) end + ], + lookup_assembler(LookupSources). %% @@ -252,6 +257,44 @@ stop(Storage) -> %% +lookup_assembler([LastSource]) -> + LastSource(); +lookup_assembler([Source | Sources]) -> + case Source() of + {error, not_found} -> lookup_assembler(Sources); + Result -> Result + end. + +check_if_already_exported(Storage, Transfer) -> + case files(Storage, #{transfer => Transfer}) of + {ok, #{items := [_ | _]}} -> ok; + _ -> {error, not_found} + end. + +lookup_local_assembler(Transfer) -> + case emqx_ft_assembler:where(Transfer) of + Pid when is_pid(Pid) -> {async, Pid}; + _ -> {error, not_found} + end. + +lookup_remote_assembler(Transfer) -> + Nodes = emqx:running_nodes() -- [node()], + Assemblers = lists:flatmap( + fun + ({ok, {async, Pid}}) -> [Pid]; + (_) -> [] + end, + emqx_ft_storage_fs_proto_v1:list_assemblers(Nodes, Transfer) + ), + case Assemblers of + [Pid | _] -> {async, Pid}; + _ -> {error, not_found} + end. + +ensure_local_assembler(Storage, Transfer, Size) -> + {ok, Pid} = emqx_ft_assembler_sup:ensure_child(Storage, Transfer, Size), + {async, Pid}. + -spec transfers(storage()) -> {ok, #{transfer() => transferinfo()}}. transfers(Storage) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl index 597f84091..68cd4c2fd 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl @@ -22,7 +22,8 @@ -export([ list_local/2, - pread_local/4 + pread_local/4, + lookup_local_assembler/1 ]). list_local(Transfer, What) -> @@ -30,3 +31,6 @@ list_local(Transfer, What) -> pread_local(Transfer, Frag, Offset, Size) -> emqx_ft_storage:with_storage_type(local, pread, [Transfer, Frag, Offset, Size]). + +lookup_local_assembler(Transfer) -> + emqx_ft_storage:with_storage_type(local, lookup_local_assembler, [Transfer]). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index 3b91684e6..d09b9ec9a 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -22,6 +22,7 @@ -export([multilist/3]). -export([pread/5]). +-export([list_assemblers/2]). -type offset() :: emqx_ft:offset(). -type transfer() :: emqx_ft:transfer(). @@ -41,3 +42,8 @@ multilist(Nodes, Transfer, What) -> {ok, [filefrag()]} | {error, term()} | no_return(). pread(Node, Transfer, Frag, Offset, Size) -> erpc:call(Node, emqx_ft_storage_fs_proxy, pread_local, [Transfer, Frag, Offset, Size]). + +-spec list_assemblers([node()], transfer()) -> + emqx_rpc:erpc_multicall([pid()]). +list_assemblers(Nodes, Transfer) -> + erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, lookup_local_assembler, [Transfer]). diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 929665ca9..ae550eb12 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -44,7 +44,8 @@ groups() -> group_cluster() -> [ t_switch_node, - t_unreliable_migrating_client + t_unreliable_migrating_client, + t_concurrent_fins ]. init_per_suite(Config) -> @@ -549,21 +550,11 @@ t_unreliable_migrating_client(Config) -> Exports = list_files(?config(clientid, Config)), - % NOTE - % The cluster had 2 assemblers running on two different nodes, because client sent `fin` - % twice. This is currently expected, files must be identical anyway. Node1Str = atom_to_list(Node1), - NodeSelfStr = atom_to_list(NodeSelf), % TODO: this testcase is specific to local fs storage backend ?assertMatch( - [#{"node" := Node1Str}, #{"node" := NodeSelfStr}], - lists:map( - fun(#{uri := URIString}) -> - #{query := QS} = uri_string:parse(URIString), - maps:from_list(uri_string:dissect_query(QS)) - end, - lists:sort(Exports) - ) + [#{"node" := Node1Str}], + fs_exported_file_attributes(Exports) ), [ @@ -571,6 +562,84 @@ t_unreliable_migrating_client(Config) -> || Export <- Exports ]. +t_concurrent_fins(Config) -> + NodeSelf = node(), + [Node1, Node2] = ?config(cluster_nodes, Config), + + ClientId = ?config(clientid, Config), + FileId = emqx_guid:to_hexstr(emqx_guid:gen()), + Filename = "migratory-birds-in-southern-hemisphere-2013.pdf", + Filesize = 100, + Gen = emqx_ft_content_gen:new({{ClientId, FileId}, Filesize}, 16), + Payload = iolist_to_binary(emqx_ft_content_gen:consume(Gen, fun({Chunk, _, _}) -> Chunk end)), + Meta = meta(Filename, Payload), + + %% Send filemeta and segments to Node1 + Context0 = #{ + clientid => ClientId, + fileid => FileId, + filesize => Filesize, + payload => Payload + }, + + Context1 = run_commands( + [ + {fun connect_mqtt_client/2, [Node1]}, + {fun send_filemeta/2, [Meta]}, + {fun send_segment/3, [0, 100]}, + {fun stop_mqtt_client/1, []} + ], + Context0 + ), + + %% Now send fins concurrently to the 3 nodes + Self = self(), + Nodes = [Node1, Node2, NodeSelf], + FinSenders = lists:map( + fun(Node) -> + %% takeovers and disconnects will happen due to concurrency + _ = erlang:process_flag(trap_exit, true), + _Context = run_commands( + [ + {fun connect_mqtt_client/2, [Node]}, + {fun send_finish/1, []} + ], + Context1 + ), + Self ! {done, Node} + end, + Nodes + ), + ok = lists:foreach( + fun(F) -> + _Pid = spawn_link(F) + end, + FinSenders + ), + ok = lists:foreach( + fun(Node) -> + receive + {done, Node} -> ok + after 1000 -> + ct:fail("Node ~p did not send finish successfully", [Node]) + end + end, + Nodes + ), + + %% Only one node should have the file + Exports = list_files(?config(clientid, Config)), + ?assertMatch( + [#{"node" := _Node}], + fs_exported_file_attributes(Exports) + ). + +%%------------------------------------------------------------------------------ +%% Command helpers +%%------------------------------------------------------------------------------ + +%% Command runners + run_commands(Commands, Context) -> lists:foldl(fun run_command/2, Context, Commands). @@ -578,6 +647,8 @@ run_command({Command, Args}, Context) -> ct:pal("COMMAND ~p ~p", [erlang:fun_info(Command, name), Args]), erlang:apply(Command, Args ++ [Context]). +%% Commands + connect_mqtt_client(Node, ContextIn) -> Context = #{clientid := ClientId} = disown_mqtt_client(ContextIn), NodePort = emqx_ft_test_helpers:tcp_port(Node), @@ -623,9 +694,18 @@ send_finish(Context = #{client := Client, fileid := FileId, filesize := Filesize ), Context. -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ %% Helpers -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ + +fs_exported_file_attributes(FSExports) -> + lists:map( + fun(#{uri := URIString}) -> + #{query := QS} = uri_string:parse(URIString), + maps:from_list(uri_string:dissect_query(QS)) + end, + lists:sort(FSExports) + ). mk_init_topic(FileId) -> <<"$file/", FileId/binary, "/init">>. From 5e5f854ce1a157551c54b3d9cccccbdf6eb6f1fd Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 5 May 2023 23:34:45 +0300 Subject: [PATCH 141/197] feat(ft-conf): simplify schema of storage / exporter backends Assumption is this changes will make `emqx_ft` config schema user-friendlier and also more future-proof. --- apps/emqx_ft/src/emqx_ft_conf.erl | 30 +++-- apps/emqx_ft/src/emqx_ft_schema.erl | 112 +++++++--------- apps/emqx_ft/src/emqx_ft_storage.erl | 123 ++++++++---------- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 11 +- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 7 +- apps/emqx_ft/test/emqx_ft_SUITE.erl | 18 +-- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 19 +-- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 84 ++++++------ .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 2 +- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 22 ++-- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 20 +-- rel/i18n/emqx_ft_schema.hocon | 16 +-- 12 files changed, 228 insertions(+), 236 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 61f639271..74f2df27d 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -54,24 +54,26 @@ enabled() -> emqx_config:get([file_transfer, enable], false). --spec storage() -> _Storage. +-spec storage() -> emqx_config:config(). storage() -> emqx_config:get([file_transfer, storage]). --spec gc_interval(_Storage) -> emqx_maybe:t(milliseconds()). -gc_interval(Conf = #{type := local}) -> - emqx_utils_maps:deep_get([segments, gc, interval], Conf); -gc_interval(_) -> - undefined. +-spec gc_interval(emqx_ft_storage_fs:storage()) -> + emqx_maybe:t(milliseconds()). +gc_interval(Storage) -> + emqx_utils_maps:deep_get([segments, gc, interval], Storage, undefined). --spec segments_ttl(_Storage) -> emqx_maybe:t({_Min :: seconds(), _Max :: seconds()}). -segments_ttl(Conf = #{type := local}) -> - { - emqx_utils_maps:deep_get([segments, gc, minimum_segments_ttl], Conf), - emqx_utils_maps:deep_get([segments, gc, maximum_segments_ttl], Conf) - }; -segments_ttl(_) -> - undefined. +-spec segments_ttl(emqx_ft_storage_fs:storage()) -> + emqx_maybe:t({_Min :: seconds(), _Max :: seconds()}). +segments_ttl(Storage) -> + Min = emqx_utils_maps:deep_get([segments, gc, minimum_segments_ttl], Storage, undefined), + Max = emqx_utils_maps:deep_get([segments, gc, maximum_segments_ttl], Storage, undefined), + case is_integer(Min) andalso is_integer(Max) of + true -> + {Min, Max}; + false -> + undefined + end. init_timeout() -> emqx_config:get([file_transfer, init_timeout]). diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index e2eebbbb8..0440bdcb9 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -93,36 +93,30 @@ fields(file_transfer) -> )}, {storage, mk( - hoconsc:union( - fun - (all_union_members) -> - [ref(local_storage)]; - ({value, #{<<"type">> := <<"local">>}}) -> - [ref(local_storage)]; - ({value, #{<<"type">> := _}}) -> - throw(#{field_name => type, expected => "local"}); - ({value, _}) -> - [ref(local_storage)] - end - ), + ref(storage_backend), #{ + desc => ?DESC("storage_backend"), required => false, - desc => ?DESC("storage"), - default => #{<<"type">> => <<"local">>} + validator => validator(backend), + default => #{ + <<"local">> => #{} + } + } + )} + ]; +fields(storage_backend) -> + [ + {local, + mk( + ref(local_storage), + #{ + desc => ?DESC("local_storage"), + required => {false, recursively} } )} ]; fields(local_storage) -> [ - {type, - mk( - local, - #{ - default => local, - required => false, - desc => ?DESC("local_type") - } - )}, {segments, mk( ref(local_storage_segments), @@ -136,29 +130,13 @@ fields(local_storage) -> )}, {exporter, mk( - hoconsc:union( - fun - (all_union_members) -> - [ - ref(local_storage_exporter), - ref(s3_exporter) - ]; - ({value, #{<<"type">> := <<"local">>}}) -> - [ref(local_storage_exporter)]; - ({value, #{<<"type">> := <<"s3">>}}) -> - [ref(s3_exporter)]; - ({value, #{<<"type">> := _}}) -> - throw(#{field_name => type, expected => "local | s3"}); - ({value, _}) -> - % NOTE: default - [ref(local_storage_exporter)] - end - ), + ref(local_storage_exporter_backend), #{ - desc => ?DESC("local_storage_exporter"), - required => true, + desc => ?DESC("local_storage_exporter_backend"), + required => false, + validator => validator(backend), default => #{ - <<"type">> => <<"local">> + <<"local">> => #{} } } )} @@ -181,17 +159,27 @@ fields(local_storage_segments) -> } )} ]; -fields(local_storage_exporter) -> +fields(local_storage_exporter_backend) -> [ - {type, + {local, mk( - local, + ref(local_storage_exporter), #{ - default => local, - required => false, - desc => ?DESC("local_storage_exporter_type") + desc => ?DESC("local_storage_exporter"), + required => {false, recursively} } )}, + {s3, + mk( + ref(s3_exporter), + #{ + desc => ?DESC("s3_exporter"), + required => {false, recursively} + } + )} + ]; +fields(local_storage_exporter) -> + [ {root, mk( binary(), @@ -202,18 +190,7 @@ fields(local_storage_exporter) -> )} ]; fields(s3_exporter) -> - [ - {type, - mk( - s3, - #{ - default => s3, - required => false, - desc => ?DESC("s3_exporter_type") - } - )} - ] ++ - emqx_s3_schema:fields(s3); + emqx_s3_schema:fields(s3); fields(local_storage_segments_gc) -> [ {interval, @@ -287,7 +264,16 @@ validator(filename) -> byte_size(Bin) =< ?MAX_FILENAME_BYTELEN orelse {error, max_length_exceeded} end, fun emqx_ft_fs_util:is_filename_safe/1 - ]. + ]; +validator(backend) -> + fun(Config) -> + case maps:keys(Config) of + [_Type] -> + ok; + _Conflicts = [_ | _] -> + {error, multiple_conflicting_backends} + end + end. converter(checksum) -> fun diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 5ec342585..79fc2dcfc 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -18,8 +18,6 @@ -export( [ - child_spec/0, - store_filemeta/2, store_segment/2, assemble/2, @@ -30,11 +28,17 @@ with_storage_type/2, with_storage_type/3, + backend/0, on_config_update/2 ] ). --type storage() :: emqx_config:config(). +-type type() :: local. +-type backend() :: {type(), storage()}. +-type storage() :: config(). +-type config() :: emqx_config:config(). + +-export_type([backend/0]). -export_type([assemble_callback/0]). @@ -100,31 +104,20 @@ %% API %%-------------------------------------------------------------------- --spec child_spec() -> - [supervisor:child_spec()]. -child_spec() -> - try - Mod = mod(), - Mod:child_spec(storage()) - catch - error:disabled -> []; - error:undef -> [] - end. - -spec store_filemeta(emqx_ft:transfer(), emqx_ft:filemeta()) -> ok | {async, pid()} | {error, term()}. store_filemeta(Transfer, FileMeta) -> - with_storage(store_filemeta, [Transfer, FileMeta]). + dispatch(store_filemeta, [Transfer, FileMeta]). -spec store_segment(emqx_ft:transfer(), emqx_ft:segment()) -> ok | {async, pid()} | {error, term()}. store_segment(Transfer, Segment) -> - with_storage(store_segment, [Transfer, Segment]). + dispatch(store_segment, [Transfer, Segment]). -spec assemble(emqx_ft:transfer(), emqx_ft:bytes()) -> ok | {async, pid()} | {error, term()}. assemble(Transfer, Size) -> - with_storage(assemble, [Transfer, Size]). + dispatch(assemble, [Transfer, Size]). -spec files() -> {ok, page(file_info(), _)} | {error, term()}. @@ -134,77 +127,75 @@ files() -> -spec files(query(Cursor)) -> {ok, page(file_info(), Cursor)} | {error, term()}. files(Query) -> - with_storage(files, [Query]). + dispatch(files, [Query]). --spec with_storage(atom() | function()) -> any(). -with_storage(Fun) -> - with_storage(Fun, []). - --spec with_storage(atom() | function(), list(term())) -> any(). -with_storage(Fun, Args) -> - case storage() of - Storage = #{} -> - apply_storage(Storage, Fun, Args); - undefined -> +-spec dispatch(atom(), list(term())) -> any(). +dispatch(Fun, Args) when is_atom(Fun) -> + case backend() of + {Type, Storage} -> + apply(mod(Type), Fun, [Storage | Args]); + _ -> {error, disabled} end. +%% + -spec with_storage_type(atom(), atom() | function()) -> any(). with_storage_type(Type, Fun) -> with_storage_type(Type, Fun, []). -spec with_storage_type(atom(), atom() | function(), list(term())) -> any(). with_storage_type(Type, Fun, Args) -> - with_storage(fun(Storage) -> - case Storage of - #{type := Type} -> - apply_storage(Storage, Fun, Args); - _ -> - {error, {invalid_storage_type, Storage}} - end - end). - -apply_storage(Storage, Fun, Args) when is_atom(Fun) -> - apply(mod(Storage), Fun, [Storage | Args]); -apply_storage(Storage, Fun, Args) when is_function(Fun) -> - apply(Fun, [Storage | Args]). + case backend() of + {Type, Storage} when is_atom(Fun) -> + apply(mod(Type), Fun, [Storage | Args]); + {Type, Storage} when is_function(Fun) -> + apply(Fun, [Storage | Args]); + {_, _} = Backend -> + {error, {invalid_storage_backend, Backend}}; + _ -> + {error, disabled} + end. %% --spec on_config_update(_Old :: emqx_maybe:t(storage()), _New :: emqx_maybe:t(storage())) -> +-spec backend() -> backend(). +backend() -> + backend(emqx_ft_conf:storage()). + +-spec on_config_update(_Old :: emqx_maybe:t(config()), _New :: emqx_maybe:t(config())) -> ok. -on_config_update(#{type := _} = Storage, #{type := _} = Storage) -> +on_config_update(ConfigOld, ConfigNew) -> + on_backend_update( + emqx_maybe:apply(fun backend/1, ConfigOld), + emqx_maybe:apply(fun backend/1, ConfigNew) + ). + +on_backend_update({Type, _} = Backend, {Type, _} = Backend) -> ok; -on_config_update(#{type := Type} = StorageOld, #{type := Type} = StorageNew) -> - ok = (mod(StorageNew)):on_config_update(StorageOld, StorageNew); -on_config_update(StorageOld, StorageNew) when - (StorageOld =:= undefined orelse is_map_key(type, StorageOld)) andalso - (StorageNew =:= undefined orelse is_map_key(type, StorageNew)) +on_backend_update({Type, StorageOld}, {Type, StorageNew}) -> + ok = (mod(Type)):on_config_update(StorageOld, StorageNew); +on_backend_update(BackendOld, BackendNew) when + (BackendOld =:= undefined orelse is_tuple(BackendOld)) andalso + (BackendNew =:= undefined orelse is_tuple(BackendNew)) -> - _ = emqx_maybe:apply(fun on_storage_stop/1, StorageOld), - _ = emqx_maybe:apply(fun on_storage_start/1, StorageNew), + _ = emqx_maybe:apply(fun on_storage_stop/1, BackendOld), + _ = emqx_maybe:apply(fun on_storage_start/1, BackendNew), ok. %%-------------------------------------------------------------------- %% Local API %%-------------------------------------------------------------------- -on_storage_start(Storage) -> - (mod(Storage)):start(Storage). +-spec backend(config()) -> backend(). +backend(#{local := Storage}) -> + {local, Storage}. -on_storage_stop(Storage) -> - (mod(Storage)):stop(Storage). +on_storage_start({Type, Storage}) -> + (mod(Type)):start(Storage). -storage() -> - emqx_ft_conf:storage(). +on_storage_stop({Type, Storage}) -> + (mod(Type)):stop(Storage). -mod() -> - mod(storage()). - -mod(Storage) -> - case Storage of - #{type := local} -> - emqx_ft_storage_fs; - undefined -> - error(disabled) - end. +mod(local) -> + emqx_ft_storage_fs. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index e000fe5c6..e25ab158e 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -169,15 +169,12 @@ stop({ExporterMod, ExporterOpts}) -> exporter(Storage) -> case maps:get(exporter, Storage) of - #{type := local} = Options -> - {emqx_ft_storage_exporter_fs, without_type(Options)}; - #{type := s3} = Options -> - {emqx_ft_storage_exporter_s3, without_type(Options)} + #{local := Options} -> + {emqx_ft_storage_exporter_fs, Options}; + #{s3 := Options} -> + {emqx_ft_storage_exporter_s3, Options} end. -without_type(#{type := _} = Options) -> - maps:without([type], Options). - init_checksum(#{checksum := {Algo, _}}) -> crypto:hash_init(Algo); init_checksum(#{}) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 692a270e3..713649759 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -58,9 +58,9 @@ start_link(Storage) -> collect() -> gen_server:call(mk_server_ref(global), {collect, erlang:system_time()}, infinity). --spec reset() -> ok. +-spec reset() -> ok | {error, _}. reset() -> - reset(emqx_ft_conf:storage()). + emqx_ft_storage:with_storage_type(local, fun reset/1). -spec reset(emqx_ft_storage_fs:storage()) -> ok. reset(Storage) -> @@ -139,7 +139,8 @@ maybe_report(#gcstats{} = _Stats, _Storage) -> ?tp(garbage_collection, #{stats => _Stats, storage => _Storage}). start_timer(St) -> - start_timer(gc_interval(emqx_ft_conf:storage()), St). + Interval = emqx_ft_storage:with_storage_type(local, fun gc_interval/1), + start_timer(Interval, St). start_timer(Interval, St = #st{next_gc_timer = undefined}) when ?IS_ENABLED(Interval) -> St#st{next_gc_timer = emqx_utils:start_timer(Interval, collect)}; diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 929665ca9..6b675e0c0 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -58,16 +58,16 @@ end_per_suite(_Config) -> set_special_configs(Config) -> fun (emqx_ft) -> - Storage = emqx_ft_test_helpers:local_storage(Config), + % NOTE + % Inhibit local fs GC to simulate it isn't fast enough to collect + % complete transfers. + Storage = emqx_utils_maps:deep_merge( + emqx_ft_test_helpers:local_storage(Config), + #{<<"local">> => #{<<"segments">> => #{<<"gc">> => #{<<"interval">> => 0}}}} + ), emqx_ft_test_helpers:load_config(#{ - % NOTE - % Inhibit local fs GC to simulate it isn't fast enough to collect - % complete transfers. - enable => true, - storage => emqx_utils_maps:deep_merge( - Storage, - #{segments => #{gc => #{interval => 0}}} - ) + <<"enable">> => true, + <<"storage">> => Storage }); (_) -> ok diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 24a4593c2..c1deeb3bc 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -246,16 +246,19 @@ exporter(Config) -> emqx_ft_storage_exporter:exporter(storage(Config)). storage(Config) -> - maps:get( - storage, + emqx_utils_maps:deep_get( + [storage, local], emqx_ft_schema:translate(#{ <<"storage">> => #{ - <<"type">> => <<"local">>, - <<"segments">> => #{ - <<"root">> => ?config(storage_root, Config) - }, - <<"exporter">> => #{ - <<"root">> => ?config(exports_root, Config) + <<"local">> => #{ + <<"segments">> => #{ + <<"root">> => ?config(storage_root, Config) + }, + <<"exporter">> => #{ + <<"local">> => #{ + <<"root">> => ?config(exports_root, Config) + } + } } } }) diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index bc9eb5d98..1f53f88af 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -34,7 +34,12 @@ end_per_suite(_Config) -> init_per_testcase(_Case, Config) -> _ = emqx_config:save_schema_mod_and_names(emqx_ft_schema), ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_ft], emqx_ft_test_helpers:env_handler(Config) + [emqx_conf, emqx_ft], fun + (emqx_ft) -> + emqx_ft_test_helpers:load_config(#{}); + (_) -> + ok + end ), {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. @@ -52,7 +57,7 @@ t_update_config(_Config) -> {error, #{kind := validation_error}}, emqx_conf:update( [file_transfer], - #{<<"storage">> => #{<<"type">> => <<"unknown">>}}, + #{<<"storage">> => #{<<"unknown">> => #{<<"foo">> => 42}}}, #{} ) ), @@ -63,16 +68,18 @@ t_update_config(_Config) -> #{ <<"enable">> => true, <<"storage">> => #{ - <<"type">> => <<"local">>, - <<"segments">> => #{ - <<"root">> => <<"/tmp/path">>, - <<"gc">> => #{ - <<"interval">> => <<"5m">> + <<"local">> => #{ + <<"segments">> => #{ + <<"root">> => <<"/tmp/path">>, + <<"gc">> => #{ + <<"interval">> => <<"5m">> + } + }, + <<"exporter">> => #{ + <<"local">> => #{ + <<"root">> => <<"/tmp/exports">> + } } - }, - <<"exporter">> => #{ - <<"type">> => <<"local">>, - <<"root">> => <<"/tmp/exports">> } } }, @@ -81,11 +88,15 @@ t_update_config(_Config) -> ), ?assertEqual( <<"/tmp/path">>, - emqx_config:get([file_transfer, storage, segments, root]) + emqx_config:get([file_transfer, storage, local, segments, root]) ), ?assertEqual( 5 * 60 * 1000, - emqx_ft_conf:gc_interval(emqx_ft_conf:storage()) + emqx_ft_storage:with_storage_type(local, fun emqx_ft_conf:gc_interval/1) + ), + ?assertEqual( + {5 * 60, 24 * 60 * 60}, + emqx_ft_storage:with_storage_type(local, fun emqx_ft_conf:segments_ttl/1) ). t_disable_restore_config(Config) -> @@ -93,13 +104,13 @@ t_disable_restore_config(Config) -> {ok, _}, emqx_conf:update( [file_transfer], - #{<<"enable">> => true, <<"storage">> => #{<<"type">> => <<"local">>}}, + #{<<"enable">> => true, <<"storage">> => #{<<"local">> => #{}}}, #{} ) ), ?assertEqual( 60 * 60 * 1000, - emqx_ft_conf:gc_interval(emqx_ft_conf:storage()) + emqx_ft_storage:with_storage_type(local, fun emqx_ft_conf:gc_interval/1) ), % Verify that transfers work ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <>), @@ -117,7 +128,7 @@ t_disable_restore_config(Config) -> emqx_ft_conf:enabled() ), ?assertMatch( - #{type := local, exporter := #{type := local}}, + #{local := #{exporter := #{local := _}}}, emqx_ft_conf:storage() ), ClientId = gen_clientid(), @@ -147,10 +158,11 @@ t_disable_restore_config(Config) -> #{ <<"enable">> => true, <<"storage">> => #{ - <<"type">> => <<"local">>, - <<"segments">> => #{ - <<"root">> => Root, - <<"gc">> => #{<<"interval">> => <<"1s">>} + <<"local">> => #{ + <<"segments">> => #{ + <<"root">> => Root, + <<"gc">> => #{<<"interval">> => <<"1s">>} + } } } }, @@ -165,10 +177,7 @@ t_disable_restore_config(Config) -> [ #{ ?snk_kind := garbage_collection, - storage := #{ - type := local, - segments := #{root := Root} - } + storage := #{segments := #{root := Root}} } ], ?of_kind(garbage_collection, Trace) @@ -188,48 +197,49 @@ t_switch_exporter(_Config) -> ) ), ?assertMatch( - #{type := local, exporter := #{type := local}}, + #{local := #{exporter := #{local := _}}}, emqx_ft_conf:storage() ), % Verify that switching to a different exporter works ?assertMatch( {ok, _}, emqx_conf:update( - [file_transfer, storage, exporter], + [file_transfer, storage, local, exporter], #{ - <<"type">> => <<"s3">>, - <<"bucket">> => <<"emqx">>, - <<"host">> => <<"https://localhost">>, - <<"port">> => 9000, - <<"transport_options">> => #{ - <<"ipv6_probe">> => false + <<"s3">> => #{ + <<"bucket">> => <<"emqx">>, + <<"host">> => <<"https://localhost">>, + <<"port">> => 9000, + <<"transport_options">> => #{ + <<"ipv6_probe">> => false + } } }, #{} ) ), ?assertMatch( - #{type := local, exporter := #{type := s3}}, + #{local := #{exporter := #{s3 := _}}}, emqx_ft_conf:storage() ), % Verify that switching back to local exporter works ?assertMatch( {ok, _}, emqx_conf:remove( - [file_transfer, storage, exporter], + [file_transfer, storage, local, exporter], #{} ) ), ?assertMatch( {ok, _}, emqx_conf:update( - [file_transfer, storage, exporter], - #{<<"type">> => <<"local">>}, + [file_transfer, storage, local, exporter], + #{<<"local">> => #{}}, #{} ) ), ?assertMatch( - #{type := local, exporter := #{type := local}}, + #{local := #{exporter := #{local := #{}}}}, emqx_ft_conf:storage() ), % Verify that transfers work diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index 2acb57a8e..50925cfb9 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -85,7 +85,7 @@ client_id(Config) -> storage(Config) -> RawConfig = #{<<"storage">> => emqx_ft_test_helpers:local_storage(Config)}, - #{storage := Storage} = emqx_ft_schema:translate(RawConfig), + #{storage := #{local := Storage}} = emqx_ft_schema:translate(RawConfig), Storage. list_files(Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index 04aedf8f3..a7ffd5675 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -36,19 +36,19 @@ end_per_suite(_Config) -> ok. init_per_testcase(TC, Config) -> + SegmentsRoot = emqx_ft_test_helpers:root(Config, node(), [TC, segments]), + ExportsRoot = emqx_ft_test_helpers:root(Config, node(), [TC, exports]), ok = emqx_common_test_helpers:start_app( emqx_ft, fun(emqx_ft) -> emqx_ft_test_helpers:load_config(#{ <<"enable">> => true, <<"storage">> => #{ - <<"type">> => <<"local">>, - <<"segments">> => #{ - <<"root">> => emqx_ft_test_helpers:root(Config, node(), [TC, segments]) - }, - <<"exporter">> => #{ - <<"type">> => <<"local">>, - <<"root">> => emqx_ft_test_helpers:root(Config, node(), [TC, exports]) + <<"local">> => #{ + <<"segments">> => #{<<"root">> => SegmentsRoot}, + <<"exporter">> => #{ + <<"local">> => #{<<"root">> => ExportsRoot} + } } } }) @@ -105,7 +105,7 @@ t_gc_triggers_manually(_Config) -> ). t_gc_complete_transfers(_Config) -> - Storage = emqx_ft_conf:storage(), + {local, Storage} = emqx_ft_storage:backend(), ok = set_gc_config(minimum_segments_ttl, 0), ok = set_gc_config(maximum_segments_ttl, 3), ok = set_gc_config(interval, 500), @@ -198,7 +198,7 @@ t_gc_complete_transfers(_Config) -> t_gc_incomplete_transfers(_Config) -> ok = set_gc_config(minimum_segments_ttl, 0), ok = set_gc_config(maximum_segments_ttl, 4), - Storage = emqx_ft_conf:storage(), + {local, Storage} = emqx_ft_storage:backend(), Transfers = [ { {<<"client43"/utf8>>, <<"file-🦕"/utf8>>}, @@ -269,7 +269,7 @@ t_gc_incomplete_transfers(_Config) -> t_gc_handling_errors(_Config) -> ok = set_gc_config(minimum_segments_ttl, 0), ok = set_gc_config(maximum_segments_ttl, 0), - Storage = emqx_ft_conf:storage(), + {local, Storage} = emqx_ft_storage:backend(), Transfer1 = {<<"client1">>, mk_file_id()}, Transfer2 = {<<"client2">>, mk_file_id()}, Filemeta = #{name => "oops.pdf"}, @@ -325,7 +325,7 @@ t_gc_handling_errors(_Config) -> %% set_gc_config(Name, Value) -> - emqx_config:put([file_transfer, storage, segments, gc, Name], Value). + emqx_config:put([file_transfer, storage, local, segments, gc, Name], Value). start_transfer(Storage, {Transfer, Meta, Gen}) -> ?assertEqual( diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index 1482223d8..2eb6d84db 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -54,20 +54,22 @@ local_storage(Config) -> local_storage(Config, Opts) -> #{ - <<"type">> => <<"local">>, - <<"segments">> => #{<<"root">> => root(Config, node(), [segments])}, - <<"exporter">> => exporter(Config, Opts) + <<"local">> => #{ + <<"segments">> => #{<<"root">> => root(Config, node(), [segments])}, + <<"exporter">> => exporter(Config, Opts) + } }. exporter(Config, #{exporter := local}) -> - #{<<"type">> => <<"local">>, <<"root">> => root(Config, node(), [exports])}; + #{<<"local">> => #{<<"root">> => root(Config, node(), [exports])}}; exporter(_Config, #{exporter := s3, bucket_name := BucketName}) -> BaseConfig = emqx_s3_test_helpers:base_raw_config(tcp), - BaseConfig#{ - <<"bucket">> => list_to_binary(BucketName), - <<"type">> => <<"s3">>, - <<"host">> => ?S3_HOST, - <<"port">> => ?S3_PORT + #{ + <<"s3">> => BaseConfig#{ + <<"bucket">> => list_to_binary(BucketName), + <<"host">> => ?S3_HOST, + <<"port">> => ?S3_PORT + } }. load_config(Config) -> diff --git a/rel/i18n/emqx_ft_schema.hocon b/rel/i18n/emqx_ft_schema.hocon index 64b9bd67e..13bbc6970 100644 --- a/rel/i18n/emqx_ft_schema.hocon +++ b/rel/i18n/emqx_ft_schema.hocon @@ -18,11 +18,11 @@ store_segment_timeout.desc: """Timeout for storing a file segment.
After reaching the timeout, message with the segment will be acked with an error""" -storage.desc: +storage_backend.desc: """Storage settings for file transfer.""" -local_type.desc: -"""Use local file system to store uploaded fragments and temporary data.""" +local_storage.desc: +"""Local file system backend to store uploaded fragments and temporary data.""" local_storage_segments.desc: """Settings for local segments storage, which include uploaded transfer fragments and temporary data.""" @@ -30,15 +30,15 @@ local_storage_segments.desc: local_storage_segments_root.desc: """File system path to keep uploaded fragments and temporary data.""" -local_storage_exporter.desc: +local_storage_exporter_backend.desc: """Exporter for the local file system storage backend.
Exporter defines where and how fully transferred and assembled files are stored.""" -local_storage_exporter_type.desc: -"""Exporter type for the exporter to the local file system""" +local_storage_exporter.desc: +"""Exporter to the local file system.""" -s3_exporter_type.desc: -"""Exporter type for the exporter to S3""" +s3_exporter.desc: +"""Exporter to the S3 API compatible object storage.""" local_storage_exporter_root.desc: """File system path to keep uploaded files.""" From 079b8e94765027cd25345812a7b0105b67460f66 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 5 May 2023 23:37:28 +0300 Subject: [PATCH 142/197] fix(ft): silence warnings when some root is not yet `mkdir`ed --- apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index 9109dadbb..9f77c8afb 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -257,6 +257,9 @@ read_exportinfo( Transfer = dirnames_to_transfer(ClientId, FileId), Info = mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo), [Info | Acc]; +read_exportinfo(_Options, {node, _Root = "", {error, enoent}, []}, Acc) -> + % NOTE: Root directory does not exist, this is not an error. + Acc; read_exportinfo(Options, Entry, Acc) -> ok = log_invalid_entry(Options, Entry), Acc. From 9381c895bb86798d7fa19fc9e4d0cb6280b947ac Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 9 May 2023 12:19:55 +0500 Subject: [PATCH 143/197] fix(ft): fix dialyzer issues --- apps/emqx_ft/src/emqx_ft_storage.erl | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 79fc2dcfc..d35229bfc 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -131,12 +131,8 @@ files(Query) -> -spec dispatch(atom(), list(term())) -> any(). dispatch(Fun, Args) when is_atom(Fun) -> - case backend() of - {Type, Storage} -> - apply(mod(Type), Fun, [Storage | Args]); - _ -> - {error, disabled} - end. + {Type, Storage} = backend(), + apply(mod(Type), Fun, [Storage | Args]). %% @@ -152,9 +148,7 @@ with_storage_type(Type, Fun, Args) -> {Type, Storage} when is_function(Fun) -> apply(Fun, [Storage | Args]); {_, _} = Backend -> - {error, {invalid_storage_backend, Backend}}; - _ -> - {error, disabled} + {error, {invalid_storage_backend, Backend}} end. %% From 45875bfee56dfc68bd52d7db785c8c4255445bfd Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 9 May 2023 14:24:50 +0500 Subject: [PATCH 144/197] fix(ft): update s3 exporter tests to the new config --- apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl index 86de74ee0..e717fe262 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl @@ -111,7 +111,7 @@ t_upload_error(Config) -> Data = <<"data"/utf8>>, {ok, _} = emqx_conf:update( - [file_transfer, storage, exporter, bucket], <<"invalid-bucket">>, #{} + [file_transfer, storage, local, exporter, s3, bucket], <<"invalid-bucket">>, #{} ), ?assertEqual( From 244188982bde8a9a9c09e995a76e620b31e2d294 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 9 May 2023 17:08:58 +0500 Subject: [PATCH 145/197] fix(ft): add missing translations --- apps/emqx_ft/src/emqx_ft_schema.erl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 0440bdcb9..26eedb1d6 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -237,6 +237,10 @@ desc(s3_exporter) -> "S3 Exporter settings for the File transfer local storage backend"; desc(local_storage_segments_gc) -> "Garbage collection settings for the File transfer local segments storage"; +desc(local_storage_exporter_backend) -> + "Exporter for the local file system storage backend"; +desc(storage_backend) -> + "Storage backend settings for file transfer"; desc(_) -> undefined. From 04a5ab4498bbe7baa4f695382fdafc97a961fdbe Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 10 May 2023 11:40:05 +0300 Subject: [PATCH 146/197] chore: add changelog entry --- changes/ee/feat-9927.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ee/feat-9927.en.md diff --git a/changes/ee/feat-9927.en.md b/changes/ee/feat-9927.en.md new file mode 100644 index 000000000..c20a0c51e --- /dev/null +++ b/changes/ee/feat-9927.en.md @@ -0,0 +1 @@ +Introduce support for the File Transfer over MQTT feature as described in [EIP-0021](https://github.com/emqx/eip), with support to publish transferred files either to the node-local file system or to the S3 API compatible remote object storage. From 6b688d6646c5be7b871c7071e585516c358f7e04 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 10 May 2023 16:25:43 +0300 Subject: [PATCH 147/197] fix(ft): anticipate repeated `kickoff`s + fix testcase --- apps/emqx_ft/src/emqx_ft_assembler.erl | 2 + apps/emqx_ft/test/emqx_ft_SUITE.erl | 99 ++++++++++++++++---------- 2 files changed, 63 insertions(+), 38 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 19f6c48d1..7a14199ad 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -78,6 +78,8 @@ handle_event(info, kickoff, idle, St) -> % We could wait for this message and handle it at the end of the assembling rather than at % the beginning, however it would make error handling much more messier. {next_state, list_local_fragments, St, ?internal([])}; +handle_event(info, kickoff, _, _St) -> + keep_state_and_data; handle_event( internal, _, diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 97f0cbbcc..7d64f9716 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -37,15 +37,27 @@ all() -> groups() -> [ - {single_node, [], emqx_common_test_helpers:all(?MODULE) -- group_cluster()}, - {cluster, [], group_cluster()} - ]. - -group_cluster() -> - [ - t_switch_node, - t_unreliable_migrating_client, - t_concurrent_fins + {single_node, [parallel], [ + t_assemble_crash, + t_corrupted_segment_retry, + t_invalid_checksum, + t_invalid_fileid, + t_invalid_filename, + t_invalid_meta, + t_invalid_topic_format, + t_meta_conflict, + t_nasty_clientids_fileids, + t_no_meta, + t_no_segment, + t_simple_transfer + ]}, + {cluster, [], [ + t_switch_node, + t_unreliable_migrating_client, + {g_concurrent_fins, [{repeat_until_any_fail, 8}], [ + t_concurrent_fins + ]} + ]} ]. init_per_suite(Config) -> @@ -563,10 +575,15 @@ t_unreliable_migrating_client(Config) -> ]. t_concurrent_fins(Config) -> + ct:timetrap({seconds, 10}), + NodeSelf = node(), [Node1, Node2] = ?config(cluster_nodes, Config), - ClientId = ?config(clientid, Config), + ClientId = iolist_to_binary([ + ?config(clientid, Config), + integer_to_list(erlang:unique_integer()) + ]), FileId = emqx_guid:to_hexstr(emqx_guid:gen()), Filename = "migratory-birds-in-southern-hemisphere-2013.pdf", Filesize = 100, @@ -593,46 +610,52 @@ t_concurrent_fins(Config) -> ), %% Now send fins concurrently to the 3 nodes - Self = self(), Nodes = [Node1, Node2, NodeSelf], - FinSenders = lists:map( + SendFin = fun(Node) -> + run_commands( + [ + {fun connect_mqtt_client/2, [Node]}, + {fun send_finish/1, []} + ], + Context1 + ) + end, + + PidMons = lists:map( fun(Node) -> - %% takeovers and disconnects will happen due to concurrency - _ = erlang:process_flag(trap_exit, true), - _Context = run_commands( - [ - {fun connect_mqtt_client/2, [Node]}, - {fun send_finish/1, []} - ], - Context1 - ), - Self ! {done, Node} + erlang:spawn_monitor(fun F() -> + _ = erlang:process_flag(trap_exit, true), + try + SendFin(Node) + catch + C:E -> + % NOTE: random delay to avoid livelock conditions + ct:pal("Node ~p did not send finish successfully: ~p:~p", [Node, C, E]), + ok = timer:sleep(rand:uniform(10)), + F() + end + end) end, Nodes ), ok = lists:foreach( - fun(F) -> - _Pid = spawn_link(F) - end, - FinSenders - ), - ok = lists:foreach( - fun(Node) -> + fun({Pid, MRef}) -> receive - {done, Node} -> ok - after 1000 -> - ct:fail("Node ~p did not send finish successfully", [Node]) + {'DOWN', MRef, process, Pid, normal} -> ok end end, - Nodes + PidMons ), %% Only one node should have the file - Exports = list_files(?config(clientid, Config)), - ?assertMatch( - [#{"node" := _Node}], - fs_exported_file_attributes(Exports) - ). + Exports = list_files(ClientId), + case fs_exported_file_attributes(Exports) of + [#{"node" := _Node}] -> + ok; + [#{"node" := _Node} | _] = Files -> + % ...But we can't really guarantee that + ct:comment({multiple_files_on_different_nodes, Files}) + end. %%------------------------------------------------------------------------------ %% Command helpers From 5fdcfad60c305ae9a9b4c8562592e2c6489e0b7a Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 11 May 2023 16:18:45 +0500 Subject: [PATCH 148/197] fix(ft): synchronize erlcloud deps --- apps/emqx_bridge_dynamo/rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_dynamo/rebar.config b/apps/emqx_bridge_dynamo/rebar.config index fbccb5c9a..d3ba1093d 100644 --- a/apps/emqx_bridge_dynamo/rebar.config +++ b/apps/emqx_bridge_dynamo/rebar.config @@ -1,6 +1,6 @@ %% -*- mode: erlang; -*- {erl_opts, [debug_info]}. -{deps, [ {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag, "3.5.16-emqx-1"}}} +{deps, [ {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.8-emqx-1"}}} , {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} From 84db57f8dfc5fe0314789573ab819d49d288491d Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 11 May 2023 16:23:59 +0500 Subject: [PATCH 149/197] fix(ft): synchronize erlcloud mix deps --- mix.exs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/mix.exs b/mix.exs index 3336743bc..42f12cac8 100644 --- a/mix.exs +++ b/mix.exs @@ -192,13 +192,6 @@ defmodule EMQXUmbrella.MixProject do {:snappyer, "1.2.8", override: true}, {:crc32cer, "0.1.8", override: true}, {:supervisor3, "1.1.12", override: true}, - {:erlcloud, github: "emqx/erlcloud", tag: "3.5.16-emqx-1", override: true}, - # erlcloud's rebar.config requires rebar3 and does not support Mix, - # so it tries to fetch deps from git. We need to override this. - {:lhttpc, tag: "1.6.2", override: true}, - {:eini, "1.2.9", override: true}, - {:base16, "1.0.0", override: true}, - # end of erlcloud's deps {:opentsdb, github: "emqx/opentsdb-client-erl", tag: "v0.5.1", override: true}, # The following two are dependencies of rabbit_common. They are needed here to # make mix not complain about conflicting versions From be3a0ce974ab1976cc9640707b4880a5d55224af Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 16 May 2023 16:26:26 +0300 Subject: [PATCH 150/197] fix(maybe): correct `apply/2` typespec Co-authored-by: ieQu1 <99872536+ieQu1@users.noreply.github.com> --- apps/emqx/src/emqx_maybe.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_maybe.erl b/apps/emqx/src/emqx_maybe.erl index 2629bc737..5b5d5e94b 100644 --- a/apps/emqx/src/emqx_maybe.erl +++ b/apps/emqx/src/emqx_maybe.erl @@ -45,7 +45,7 @@ define(Term, _) -> Term. %% @doc Apply a function to a maybe argument. --spec apply(fun((maybe(A)) -> maybe(A)), maybe(A)) -> +-spec apply(fun((A) -> maybe(A)), maybe(A)) -> maybe(A). apply(_Fun, undefined) -> undefined; From 91ce1c2a9080760a0360489842535789089e09c9 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 16 May 2023 16:27:33 +0300 Subject: [PATCH 151/197] fix(ft-schema): make description more natural Co-authored-by: ieQu1 <99872536+ieQu1@users.noreply.github.com> --- rel/i18n/emqx_ft_schema.hocon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rel/i18n/emqx_ft_schema.hocon b/rel/i18n/emqx_ft_schema.hocon index 13bbc6970..bafda331a 100644 --- a/rel/i18n/emqx_ft_schema.hocon +++ b/rel/i18n/emqx_ft_schema.hocon @@ -41,7 +41,7 @@ s3_exporter.desc: """Exporter to the S3 API compatible object storage.""" local_storage_exporter_root.desc: -"""File system path to keep uploaded files.""" +"""Directory where the uploaded files are kept.""" local_storage_segments_gc.desc: """Garbage collection settings for the intermediate and temporary files in the local file system.""" From a7595ff46877c12ff4e31b9f6a7f2d8c4bc28250 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 16 May 2023 16:45:07 +0300 Subject: [PATCH 152/197] chore: fixup copyright years --- apps/emqx_ft/include/emqx_ft_storage_fs.hrl | 2 +- apps/emqx_ft/src/emqx_ft.erl | 3 +-- apps/emqx_ft/src/emqx_ft_api.erl | 2 +- apps/emqx_ft/src/emqx_ft_app.erl | 2 +- apps/emqx_ft/src/emqx_ft_assembler.erl | 2 +- apps/emqx_ft/src/emqx_ft_assembler_sup.erl | 2 +- apps/emqx_ft/src/emqx_ft_assembly.erl | 2 +- apps/emqx_ft/src/emqx_ft_conf.erl | 2 +- apps/emqx_ft/src/emqx_ft_fs_iterator.erl | 2 +- apps/emqx_ft/src/emqx_ft_fs_util.erl | 2 +- apps/emqx_ft/src/emqx_ft_responder.erl | 2 +- apps/emqx_ft/src/emqx_ft_responder_sup.erl | 2 +- apps/emqx_ft/src/emqx_ft_schema.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl | 2 +- apps/emqx_ft/src/emqx_ft_sup.erl | 2 +- .../emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl | 2 +- apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl | 2 +- apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl | 2 +- 27 files changed, 27 insertions(+), 28 deletions(-) diff --git a/apps/emqx_ft/include/emqx_ft_storage_fs.hrl b/apps/emqx_ft/include/emqx_ft_storage_fs.hrl index 72ebe586a..81ab9cfad 100644 --- a/apps/emqx_ft/include/emqx_ft_storage_fs.hrl +++ b/apps/emqx_ft/include/emqx_ft_storage_fs.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index dba087484..898203b51 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -17,7 +17,6 @@ -module(emqx_ft). -include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index 61a6a2c93..3fd279c76 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_app.erl b/apps/emqx_ft/src/emqx_ft_app.erl index 0bac6b592..299683e43 100644 --- a/apps/emqx_ft/src/emqx_ft_app.erl +++ b/apps/emqx_ft/src/emqx_ft_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 7a14199ad..873efc6ff 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl index bdefdac47..4ba65c290 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index dead1132e..d765a2bd2 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 74f2df27d..2e994925c 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_fs_iterator.erl b/apps/emqx_ft/src/emqx_ft_fs_iterator.erl index 7fb8d8634..7a58c5b38 100644 --- a/apps/emqx_ft/src/emqx_ft_fs_iterator.erl +++ b/apps/emqx_ft/src/emqx_ft_fs_iterator.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_fs_util.erl b/apps/emqx_ft/src/emqx_ft_fs_util.erl index b731d3270..9028722aa 100644 --- a/apps/emqx_ft/src/emqx_ft_fs_util.erl +++ b/apps/emqx_ft/src/emqx_ft_fs_util.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_responder.erl b/apps/emqx_ft/src/emqx_ft_responder.erl index cbbfbe687..c2c62e1c2 100644 --- a/apps/emqx_ft/src/emqx_ft_responder.erl +++ b/apps/emqx_ft/src/emqx_ft_responder.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_responder_sup.erl b/apps/emqx_ft/src/emqx_ft_responder_sup.erl index 23d4f55fa..fb3932425 100644 --- a/apps/emqx_ft/src/emqx_ft_responder_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_responder_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 26eedb1d6..09e9ab0a5 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index d35229bfc..4e1060d88 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index e25ab158e..591173615 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index 71f2f2748..6738d6fef 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl index a278b01a5..abb774f82 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl index 13160bfc6..50e02db6f 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index 5c5aade86..b9f07d5c0 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 713649759..4e9a6d56c 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl index 68cd4c2fd..e6358cb14 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl index 4b1c4acb8..513872edd 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl index fff6b7830..8c8aea6b3 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index 8d388814c..0308668ab 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl index 9ca6db786..222891f54 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index d09b9ec9a..989a48555 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl index f8fe02d36..ea089111d 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. From e3b822c1a046ee14ab7e6a5a9d3df9d132214f7d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 16 May 2023 16:45:41 +0300 Subject: [PATCH 153/197] chore: remove empty header Co-authored-by: ieQu1 <99872536+ieQu1@users.noreply.github.com> --- apps/emqx_ft/include/emqx_ft.hrl | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 apps/emqx_ft/include/emqx_ft.hrl diff --git a/apps/emqx_ft/include/emqx_ft.hrl b/apps/emqx_ft/include/emqx_ft.hrl deleted file mode 100644 index e46d79490..000000000 --- a/apps/emqx_ft/include/emqx_ft.hrl +++ /dev/null @@ -1,15 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- From 6f8f21106b9ef8dc45c2418c124ce45a438d891f Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 16 May 2023 16:46:05 +0300 Subject: [PATCH 154/197] fix(assert): use unpredictable binding names in macros Also translate macro vars to ALLCAPS for consistency. --- apps/emqx/include/asserts.hrl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx/include/asserts.hrl b/apps/emqx/include/asserts.hrl index 4b0afd67b..1be725d2d 100644 --- a/apps/emqx/include/asserts.hrl +++ b/apps/emqx/include/asserts.hrl @@ -30,16 +30,16 @@ ) ). --define(assertInclude(Pattern, List), +-define(assertInclude(PATTERN, LIST), ?assert( lists:any( - fun(El) -> - case El of - Pattern -> true; + fun(X__Elem_) -> + case X__Elem_ of + PATTERN -> true; _ -> false end end, - List + LIST ) ) ). From b71955e3685aebf3636e3b05e813fe28379c782b Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 17 May 2023 00:29:15 +0300 Subject: [PATCH 155/197] fix(ft): bump application versions --- apps/emqx/src/emqx.app.src | 2 +- apps/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- apps/emqx_machine/src/emqx_machine.app.src | 2 +- lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 5ca8fc797..be68b438f 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.25"}, + {vsn, "5.0.26"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index bd022f226..02fbdfb74 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.20"}, + {vsn, "5.0.21"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index a44d2b36e..7cf0e4b53 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.3"}, + {vsn, "0.2.4"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src index 3df18ce7a..599b0798c 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_conf, [ {description, "EMQX Enterprise Edition configuration schema"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [ kernel, From ab8b8ccaadcb92e7bc777f052e30f2936f97f481 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 16 May 2023 18:25:25 +0200 Subject: [PATCH 156/197] docs: make sure there is a summary for all API endpoints Many HTTP API endpoints did not have a summary when looking at the API docs at "http://emqx_host_name:18083/api-docs". This has been fixed by making sure there is a summary corresponding to all endpoints. Fixes: https://emqx.atlassian.net/browse/EMQX-9767 --- .../emqx_dashboard/src/emqx_dashboard.app.src | 2 +- .../src/emqx_dashboard_error_code_api.erl | 5 +- .../src/emqx_dashboard_monitor_api.erl | 9 +- .../src/emqx_management.app.src | 2 +- .../src/emqx_mgmt_api_api_keys.erl | 11 +- .../src/emqx_mgmt_api_clients.erl | 23 ++-- .../src/emqx_mgmt_api_cluster.erl | 6 +- .../src/emqx_mgmt_api_configs.erl | 45 ++++--- .../src/emqx_mgmt_api_listeners.erl | 20 +-- .../src/emqx_mgmt_api_metrics.erl | 3 +- .../src/emqx_mgmt_api_nodes.erl | 9 +- .../src/emqx_mgmt_api_stats.erl | 3 +- .../src/emqx_mgmt_api_subscriptions.erl | 3 +- .../src/emqx_mgmt_api_topics.erl | 5 +- .../src/emqx_mgmt_api_trace.erl | 46 +++---- .../src/emqx_prometheus.app.src | 2 +- .../src/emqx_prometheus_api.erl | 7 +- changes/ce/fix-10724.en.md | 1 + rel/i18n/emqx_authn_api.hocon | 58 +++++++++ rel/i18n/emqx_authn_user_import_api.hocon | 4 + rel/i18n/emqx_authz_api_cache.hocon | 2 + rel/i18n/emqx_authz_api_mnesia.hocon | 33 ++++- rel/i18n/emqx_authz_api_sources.hocon | 18 ++- rel/i18n/emqx_auto_subscribe_api.hocon | 6 + rel/i18n/emqx_dashboard_api.hocon | 16 +++ rel/i18n/emqx_dashboard_error_code_api.hocon | 13 ++ rel/i18n/emqx_dashboard_monitor_api.hocon | 23 ++++ rel/i18n/emqx_delayed_api.hocon | 46 +++++++ rel/i18n/emqx_exhook_api.hocon | 45 ++++++- rel/i18n/emqx_mgmt_api_alarms.hocon | 4 + rel/i18n/emqx_mgmt_api_api_keys.hocon | 33 +++++ rel/i18n/emqx_mgmt_api_banned.hocon | 6 + rel/i18n/emqx_mgmt_api_clients.hocon | 58 +++++++++ rel/i18n/emqx_mgmt_api_cluster.hocon | 18 +++ rel/i18n/emqx_mgmt_api_configs.hocon | 40 ++++++ rel/i18n/emqx_mgmt_api_listeners.hocon | 59 +++++++++ rel/i18n/emqx_mgmt_api_metrics.hocon | 8 ++ rel/i18n/emqx_mgmt_api_nodes.hocon | 23 ++++ rel/i18n/emqx_mgmt_api_stats.hocon | 8 ++ rel/i18n/emqx_mgmt_api_subscriptions.hocon | 8 ++ rel/i18n/emqx_mgmt_api_topics.hocon | 13 ++ rel/i18n/emqx_mgmt_api_trace.hocon | 118 ++++++++++++++++++ rel/i18n/emqx_prometheus_api.hocon | 18 +++ rel/i18n/emqx_retainer_api.hocon | 10 ++ rel/i18n/emqx_rewrite_api.hocon | 6 + rel/i18n/emqx_slow_subs_api.hocon | 8 ++ rel/i18n/emqx_telemetry_api.hocon | 6 + rel/i18n/emqx_topic_metrics_api.hocon | 68 ++++++++++ 48 files changed, 871 insertions(+), 107 deletions(-) create mode 100644 changes/ce/fix-10724.en.md create mode 100644 rel/i18n/emqx_dashboard_error_code_api.hocon create mode 100644 rel/i18n/emqx_dashboard_monitor_api.hocon create mode 100644 rel/i18n/emqx_mgmt_api_api_keys.hocon create mode 100644 rel/i18n/emqx_mgmt_api_clients.hocon create mode 100644 rel/i18n/emqx_mgmt_api_cluster.hocon create mode 100644 rel/i18n/emqx_mgmt_api_configs.hocon create mode 100644 rel/i18n/emqx_mgmt_api_listeners.hocon create mode 100644 rel/i18n/emqx_mgmt_api_metrics.hocon create mode 100644 rel/i18n/emqx_mgmt_api_nodes.hocon create mode 100644 rel/i18n/emqx_mgmt_api_stats.hocon create mode 100644 rel/i18n/emqx_mgmt_api_subscriptions.hocon create mode 100644 rel/i18n/emqx_mgmt_api_topics.hocon create mode 100644 rel/i18n/emqx_mgmt_api_trace.hocon create mode 100644 rel/i18n/emqx_prometheus_api.hocon diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index bd022f226..02fbdfb74 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.20"}, + {vsn, "5.0.21"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl index 131b08313..47ba02ebf 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl @@ -20,6 +20,7 @@ -include_lib("emqx/include/http_api.hrl"). -include("emqx_dashboard.hrl"). -include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -export([ api_spec/0, @@ -50,7 +51,7 @@ schema("/error_codes") -> 'operationId' => error_codes, get => #{ security => [], - description => <<"API Error Codes">>, + description => ?DESC(error_codes), tags => [<<"Error Codes">>], responses => #{ 200 => hoconsc:array(hoconsc:ref(?MODULE, error_code)) @@ -62,7 +63,7 @@ schema("/error_codes/:code") -> 'operationId' => error_code, get => #{ security => [], - description => <<"API Error Codes">>, + description => ?DESC(error_codes_u), tags => [<<"Error Codes">>], parameters => [ {code, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index c0e162b62..d86dffba0 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -6,6 +6,7 @@ -include("emqx_dashboard.hrl"). -include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hocon_types.hrl"). -behaviour(minirest_api). @@ -38,7 +39,7 @@ schema("/monitor") -> 'operationId' => monitor, get => #{ tags => [<<"Metrics">>], - desc => <<"List monitor data.">>, + description => ?DESC(list_monitor), parameters => [parameter_latest()], responses => #{ 200 => hoconsc:mk(hoconsc:array(hoconsc:ref(sampler)), #{}), @@ -51,7 +52,7 @@ schema("/monitor/nodes/:node") -> 'operationId' => monitor, get => #{ tags => [<<"Metrics">>], - desc => <<"List the monitor data on the node.">>, + description => ?DESC(list_monitor_node), parameters => [parameter_node(), parameter_latest()], responses => #{ 200 => hoconsc:mk(hoconsc:array(hoconsc:ref(sampler)), #{}), @@ -64,7 +65,7 @@ schema("/monitor_current") -> 'operationId' => monitor_current, get => #{ tags => [<<"Metrics">>], - desc => <<"Current status. Gauge and rate.">>, + description => ?DESC(current_status), responses => #{ 200 => hoconsc:mk(hoconsc:ref(sampler_current), #{}) } @@ -75,7 +76,7 @@ schema("/monitor_current/nodes/:node") -> 'operationId' => monitor_current, get => #{ tags => [<<"Metrics">>], - desc => <<"Node current status. Gauge and rate.">>, + description => ?DESC(current_status_node), parameters => [parameter_node()], responses => #{ 200 => hoconsc:mk(hoconsc:ref(sampler_current), #{}), diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 34f3dd1fe..f51a83923 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.21"}, + {vsn, "5.0.22"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl b/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl index c39b11273..ba21adaa5 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl @@ -18,6 +18,7 @@ -behaviour(minirest_api). -include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -export([api_spec/0, fields/1, paths/0, schema/1, namespace/0]). -export([api_key/2, api_key_by_name/2]). @@ -36,14 +37,14 @@ schema("/api_key") -> #{ 'operationId' => api_key, get => #{ - description => "Return api_key list", + description => ?DESC(api_key_list), tags => ?TAGS, responses => #{ 200 => delete([api_secret], fields(app)) } }, post => #{ - description => "Create new api_key", + description => ?DESC(create_new_api_key), tags => ?TAGS, 'requestBody' => delete([created_at, api_key, api_secret], fields(app)), responses => #{ @@ -56,7 +57,7 @@ schema("/api_key/:name") -> #{ 'operationId' => api_key_by_name, get => #{ - description => "Return the specific api_key", + description => ?DESC(get_api_key), tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ @@ -65,7 +66,7 @@ schema("/api_key/:name") -> } }, put => #{ - description => "Update the specific api_key", + description => ?DESC(update_api_key), tags => ?TAGS, parameters => [hoconsc:ref(name)], 'requestBody' => delete([created_at, api_key, api_secret, name], fields(app)), @@ -75,7 +76,7 @@ schema("/api_key/:name") -> } }, delete => #{ - description => "Delete the specific api_key", + description => ?DESC(delete_api_key), tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 681c851bf..27dc8c492 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -20,6 +20,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/emqx.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -101,7 +102,7 @@ schema("/clients") -> #{ 'operationId' => clients, get => #{ - description => <<"List clients">>, + description => ?DESC(list_clients), tags => ?TAGS, parameters => [ hoconsc:ref(emqx_dashboard_swagger, page), @@ -214,7 +215,7 @@ schema("/clients/:clientid") -> #{ 'operationId' => client, get => #{ - description => <<"Get clients info by client ID">>, + description => ?DESC(clients_info_from_id), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], responses => #{ @@ -225,7 +226,7 @@ schema("/clients/:clientid") -> } }, delete => #{ - description => <<"Kick out client by client ID">>, + description => ?DESC(kick_client_id), tags => ?TAGS, parameters => [ {clientid, hoconsc:mk(binary(), #{in => path})} @@ -242,7 +243,7 @@ schema("/clients/:clientid/authorization/cache") -> #{ 'operationId' => authz_cache, get => #{ - description => <<"Get client authz cache in the cluster.">>, + description => ?DESC(get_authz_cache), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], responses => #{ @@ -253,7 +254,7 @@ schema("/clients/:clientid/authorization/cache") -> } }, delete => #{ - description => <<"Clean client authz cache in the cluster.">>, + description => ?DESC(clean_authz_cache), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], responses => #{ @@ -268,7 +269,7 @@ schema("/clients/:clientid/subscriptions") -> #{ 'operationId' => subscriptions, get => #{ - description => <<"Get client subscriptions">>, + description => ?DESC(get_client_subs), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], responses => #{ @@ -285,7 +286,7 @@ schema("/clients/:clientid/subscribe") -> #{ 'operationId' => subscribe, post => #{ - description => <<"Subscribe">>, + description => ?DESC(subscribe), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], 'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, subscribe)), @@ -301,7 +302,7 @@ schema("/clients/:clientid/subscribe/bulk") -> #{ 'operationId' => subscribe_batch, post => #{ - description => <<"Subscribe">>, + description => ?DESC(subscribe_g), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], 'requestBody' => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, subscribe))), @@ -317,7 +318,7 @@ schema("/clients/:clientid/unsubscribe") -> #{ 'operationId' => unsubscribe, post => #{ - description => <<"Unsubscribe">>, + description => ?DESC(unsubscribe), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], 'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, unsubscribe)), @@ -333,7 +334,7 @@ schema("/clients/:clientid/unsubscribe/bulk") -> #{ 'operationId' => unsubscribe_batch, post => #{ - description => <<"Unsubscribe">>, + description => ?DESC(unsubscribe_g), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], 'requestBody' => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, unsubscribe))), @@ -349,7 +350,7 @@ schema("/clients/:clientid/keepalive") -> #{ 'operationId' => set_keepalive, put => #{ - description => <<"Set the online client keepalive by seconds">>, + description => ?DESC(set_keepalive_seconds), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], 'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, keepalive)), diff --git a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl index e74b6c362..ba185e369 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl @@ -39,7 +39,7 @@ schema("/cluster") -> #{ 'operationId' => cluster_info, get => #{ - description => "Get cluster info", + desc => ?DESC(get_cluster_info), tags => [<<"Cluster">>], responses => #{ 200 => [ @@ -54,7 +54,7 @@ schema("/cluster/:node/invite") -> #{ 'operationId' => invite_node, put => #{ - description => "Invite node to cluster", + desc => ?DESC(invite_node), tags => [<<"Cluster">>], parameters => [hoconsc:ref(node)], responses => #{ @@ -67,7 +67,7 @@ schema("/cluster/:node/force_leave") -> #{ 'operationId' => force_leave, delete => #{ - description => "Force leave node from cluster", + desc => ?DESC(force_remove_node), tags => [<<"Cluster">>], parameters => [hoconsc:ref(node)], responses => #{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 1d691c536..14fb07497 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -67,8 +67,7 @@ schema("/configs") -> 'operationId' => configs, get => #{ tags => ?TAGS, - description => - <<"Get all the configurations of the specified node, including hot and non-hot updatable items.">>, + description => ?DESC(get_conf_node), parameters => [ {node, hoconsc:mk( @@ -77,8 +76,7 @@ schema("/configs") -> in => query, required => false, example => <<"emqx@127.0.0.1">>, - desc => - <<"Node's name: If you do not fill in the fields, this node will be used by default.">> + description => ?DESC(node_name) } )} ], @@ -95,12 +93,7 @@ schema("/configs_reset/:rootname") -> 'operationId' => config_reset, post => #{ tags => ?TAGS, - description => - << - "Reset the config entry specified by the query string parameter `conf_path`.
" - "- For a config entry that has default value, this resets it to the default value;\n" - "- For a config entry that has no default value, an error 400 will be returned" - >>, + description => ?DESC(rest_conf_query), %% We only return "200" rather than the new configs that has been changed, as %% the schema of the changed configs is depends on the request parameter %% `conf_path`, it cannot be defined here. @@ -134,12 +127,12 @@ schema("/configs/global_zone") -> 'operationId' => global_zone_configs, get => #{ tags => ?TAGS, - description => <<"Get the global zone configs">>, + description => ?DESC(get_global_zone_configs), responses => #{200 => Schema} }, put => #{ tags => ?TAGS, - description => <<"Update globbal zone configs">>, + description => ?DESC(update_globar_zone_configs), 'requestBody' => Schema, responses => #{ 200 => Schema, @@ -153,7 +146,7 @@ schema("/configs/limiter") -> 'operationId' => limiter, get => #{ tags => ?TAGS, - description => <<"Get the node-level limiter configs">>, + description => ?DESC(get_node_level_limiter_congigs), responses => #{ 200 => hoconsc:mk(hoconsc:ref(emqx_limiter_schema, limiter)), 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"config not found">>) @@ -161,7 +154,7 @@ schema("/configs/limiter") -> }, put => #{ tags => ?TAGS, - description => <<"Update the node-level limiter configs">>, + description => ?DESC(update_node_level_limiter_congigs), 'requestBody' => hoconsc:mk(hoconsc:ref(emqx_limiter_schema, limiter)), responses => #{ 200 => hoconsc:mk(hoconsc:ref(emqx_limiter_schema, limiter)), @@ -172,15 +165,22 @@ schema("/configs/limiter") -> }; schema(Path) -> {RootKey, {_Root, Schema}} = find_schema(Path), + GetDesc = iolist_to_binary([ + <<"Get the sub-configurations under *">>, + RootKey, + <<"*">> + ]), + PutDesc = iolist_to_binary([ + <<"Update the sub-configurations under *">>, + RootKey, + <<"*">> + ]), #{ 'operationId' => config, get => #{ tags => ?TAGS, - description => iolist_to_binary([ - <<"Get the sub-configurations under *">>, - RootKey, - <<"*">> - ]), + desc => GetDesc, + summary => GetDesc, responses => #{ 200 => Schema, 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"config not found">>) @@ -188,11 +188,8 @@ schema(Path) -> }, put => #{ tags => ?TAGS, - description => iolist_to_binary([ - <<"Update the sub-configurations under *">>, - RootKey, - <<"*">> - ]), + desc => PutDesc, + summary => PutDesc, 'requestBody' => Schema, responses => #{ 200 => Schema, diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index 152ccc599..1f1dda5f2 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -66,7 +66,7 @@ schema("/listeners_status") -> 'operationId' => listener_type_status, get => #{ tags => [<<"listeners">>], - desc => <<"List all running node's listeners live status. group by listener type">>, + description => ?DESC(list_node_live_statuses), responses => #{ 200 => emqx_dashboard_swagger:schema_with_example( @@ -81,7 +81,7 @@ schema("/listeners") -> 'operationId' => list_listeners, get => #{ tags => [<<"listeners">>], - desc => <<"List all running node's listeners for the specified type.">>, + description => ?DESC(list_listeners), parameters => [ {type, ?HOCON( @@ -99,7 +99,7 @@ schema("/listeners") -> }, post => #{ tags => [<<"listeners">>], - desc => <<"Create the specified listener on all nodes.">>, + description => ?DESC(create_on_all_nodes), parameters => [], 'requestBody' => create_listener_schema(#{bind => true}), responses => #{ @@ -113,7 +113,7 @@ schema("/listeners/:id") -> 'operationId' => crud_listeners_by_id, get => #{ tags => [<<"listeners">>], - desc => <<"List all running node's listeners for the specified id.">>, + description => ?DESC(list_by_id), parameters => [?R_REF(listener_id)], responses => #{ 200 => listener_schema(#{bind => true}), @@ -122,7 +122,7 @@ schema("/listeners/:id") -> }, put => #{ tags => [<<"listeners">>], - desc => <<"Update the specified listener on all nodes.">>, + description => ?DESC(update_lisener), parameters => [?R_REF(listener_id)], 'requestBody' => listener_schema(#{bind => false}), responses => #{ @@ -133,7 +133,7 @@ schema("/listeners/:id") -> }, post => #{ tags => [<<"listeners">>], - desc => <<"Create the specified listener on all nodes.">>, + description => ?DESC(create_on_all_nodes), parameters => [?R_REF(listener_id)], 'requestBody' => listener_schema(#{bind => true}), responses => #{ @@ -144,7 +144,7 @@ schema("/listeners/:id") -> }, delete => #{ tags => [<<"listeners">>], - desc => <<"Delete the specified listener on all nodes.">>, + description => ?DESC(delete_on_all_nodes), parameters => [?R_REF(listener_id)], responses => #{ 204 => <<"Listener deleted">>, @@ -157,7 +157,7 @@ schema("/listeners/:id/start") -> 'operationId' => start_listeners_by_id, post => #{ tags => [<<"listeners">>], - desc => <<"Start the listener on all nodes.">>, + description => ?DESC(start_on_all_nodes), parameters => [ ?R_REF(listener_id) ], @@ -172,7 +172,7 @@ schema("/listeners/:id/stop") -> 'operationId' => stop_listeners_by_id, post => #{ tags => [<<"listeners">>], - desc => <<"Stop the listener on all nodes.">>, + description => ?DESC(stop_on_all_nodes), parameters => [ ?R_REF(listener_id) ], @@ -187,7 +187,7 @@ schema("/listeners/:id/restart") -> 'operationId' => restart_listeners_by_id, post => #{ tags => [<<"listeners">>], - desc => <<"Restart listeners on all nodes.">>, + description => ?DESC(restart_on_all_nodes), parameters => [ ?R_REF(listener_id) ], diff --git a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl index 0fcc45d8e..7ad0777c7 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl @@ -19,6 +19,7 @@ -behaviour(minirest_api). -include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hocon_types.hrl"). -import(hoconsc, [mk/2, ref/2]). @@ -73,7 +74,7 @@ schema("/metrics") -> 'operationId' => metrics, get => #{ - description => <<"EMQX metrics">>, + description => ?DESC(emqx_metrics), tags => [<<"Metrics">>], parameters => [ diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index ecf465f43..827fea44a 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -18,6 +18,7 @@ -behaviour(minirest_api). -include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -import(hoconsc, [mk/2, ref/1, ref/2, enum/1, array/1]). @@ -60,7 +61,7 @@ schema("/nodes") -> 'operationId' => nodes, get => #{ - description => <<"List EMQX nodes">>, + description => ?DESC(list_nodes), tags => [<<"Nodes">>], responses => #{ @@ -76,7 +77,7 @@ schema("/nodes/:node") -> 'operationId' => node, get => #{ - description => <<"Get node info">>, + description => ?DESC(get_node_info), tags => [<<"Nodes">>], parameters => [ref(node_name)], responses => @@ -94,7 +95,7 @@ schema("/nodes/:node/metrics") -> 'operationId' => node_metrics, get => #{ - description => <<"Get node metrics">>, + description => ?DESC(get_node_metrics), tags => [<<"Nodes">>], parameters => [ref(node_name)], responses => @@ -112,7 +113,7 @@ schema("/nodes/:node/stats") -> 'operationId' => node_stats, get => #{ - description => <<"Get node stats">>, + description => ?DESC(get_node_stats), tags => [<<"Nodes">>], parameters => [ref(node_name)], responses => diff --git a/apps/emqx_management/src/emqx_mgmt_api_stats.erl b/apps/emqx_management/src/emqx_mgmt_api_stats.erl index 5f4bbce65..b57565671 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_stats.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_stats.erl @@ -18,6 +18,7 @@ -behaviour(minirest_api). -include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -import( hoconsc, @@ -49,7 +50,7 @@ schema("/stats") -> 'operationId' => list, get => #{ - description => <<"EMQX stats">>, + description => ?DESC(emqx_stats), tags => [<<"Metrics">>], parameters => [ref(aggregate)], responses => diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 1b69835f9..08a90e623 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -21,6 +21,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -export([ api_spec/0, @@ -58,7 +59,7 @@ schema("/subscriptions") -> #{ 'operationId' => subscriptions, get => #{ - description => <<"List subscriptions">>, + description => ?DESC(list_subs), tags => [<<"Subscriptions">>], parameters => parameters(), responses => #{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_topics.erl b/apps/emqx_management/src/emqx_mgmt_api_topics.erl index 4100269e5..6b0e1f622 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_topics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_topics.erl @@ -18,6 +18,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). %% API -behaviour(minirest_api). @@ -51,7 +52,7 @@ schema("/topics") -> #{ 'operationId' => topics, get => #{ - description => <<"Topics list">>, + description => ?DESC(topic_list), tags => ?TAGS, parameters => [ topic_param(query), @@ -71,7 +72,7 @@ schema("/topics/:topic") -> #{ 'operationId' => topic, get => #{ - description => <<"Lookup topic info by name">>, + description => ?DESC(topic_info_by_name), tags => ?TAGS, parameters => [topic_param(path)], responses => #{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index 25cc2734f..331c62681 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -21,6 +21,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -export([ api_spec/0, @@ -73,14 +74,14 @@ schema("/trace") -> #{ 'operationId' => trace, get => #{ - description => "List all trace", + description => ?DESC(list_all), tags => ?TAGS, responses => #{ 200 => hoconsc:array(hoconsc:ref(trace)) } }, post => #{ - description => "Create new trace", + description => ?DESC(create_new), tags => ?TAGS, 'requestBody' => delete([status, log_size], fields(trace)), responses => #{ @@ -102,7 +103,7 @@ schema("/trace") -> } }, delete => #{ - description => "Clear all traces", + description => ?DESC(clear_all), tags => ?TAGS, responses => #{ 204 => <<"No Content">> @@ -113,7 +114,7 @@ schema("/trace/:name") -> #{ 'operationId' => delete_trace, delete => #{ - description => "Delete specified trace", + description => ?DESC(delete_trace), tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ @@ -126,7 +127,7 @@ schema("/trace/:name/stop") -> #{ 'operationId' => update_trace, put => #{ - description => "Stop trace by name", + description => ?DESC(stop_trace), tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ @@ -139,7 +140,7 @@ schema("/trace/:name/download") -> #{ 'operationId' => download_trace_log, get => #{ - description => "Download trace log by name", + description => ?DESC(download_log_by_name), tags => ?TAGS, parameters => [hoconsc:ref(name), hoconsc:ref(node)], responses => #{ @@ -161,7 +162,7 @@ schema("/trace/:name/log_detail") -> #{ 'operationId' => log_file_detail, get => #{ - description => "get trace log file's metadata, such as size, last update time", + description => ?DESC(get_trace_file_metadata), tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ @@ -174,7 +175,7 @@ schema("/trace/:name/log") -> #{ 'operationId' => stream_log_file, get => #{ - description => "view trace log", + description => ?DESC(view_trace_log), tags => ?TAGS, parameters => [ hoconsc:ref(name), @@ -204,9 +205,8 @@ schema("/trace/:name/log") -> fields(log_file_detail) -> fields(node) ++ [ - {size, hoconsc:mk(integer(), #{desc => "file size"})}, - {mtime, - hoconsc:mk(integer(), #{desc => "the modification and last access times of a file"})} + {size, hoconsc:mk(integer(), #{description => ?DESC(file_size)})}, + {mtime, hoconsc:mk(integer(), #{description => ?DESC(modification_date)})} ]; fields(trace) -> [ @@ -214,7 +214,7 @@ fields(trace) -> hoconsc:mk( binary(), #{ - desc => "Unique and format by [a-zA-Z0-9-_]", + description => ?DESC(format), validator => fun ?MODULE:validate_name/1, required => true, example => <<"EMQX-TRACE-1">> @@ -224,7 +224,7 @@ fields(trace) -> hoconsc:mk( hoconsc:enum([clientid, topic, ip_address]), #{ - desc => "" "Filter type" "", + description => ?DESC(filter_type), required => true, example => <<"clientid">> } @@ -233,7 +233,7 @@ fields(trace) -> hoconsc:mk( binary(), #{ - desc => "" "support mqtt wildcard topic." "", + description => ?DESC(support_wildcard), required => false, example => <<"/dev/#">> } @@ -242,7 +242,7 @@ fields(trace) -> hoconsc:mk( binary(), #{ - desc => "" "mqtt clientid." "", + description => ?DESC(mqtt_clientid), required => false, example => <<"dev-001">> } @@ -252,7 +252,7 @@ fields(trace) -> hoconsc:mk( binary(), #{ - desc => "client ip address", + description => ?DESC(client_ip_addess), required => false, example => <<"127.0.0.1">> } @@ -261,7 +261,7 @@ fields(trace) -> hoconsc:mk( hoconsc:enum([running, stopped, waiting]), #{ - desc => "trace status", + description => ?DESC(trace_status), required => false, example => running } @@ -283,7 +283,7 @@ fields(trace) -> hoconsc:mk( emqx_datetime:epoch_second(), #{ - desc => "rfc3339 timestamp or epoch second", + description => ?DESC(time_format), required => false, example => <<"2021-11-04T18:17:38+08:00">> } @@ -292,7 +292,7 @@ fields(trace) -> hoconsc:mk( emqx_datetime:epoch_second(), #{ - desc => "rfc3339 timestamp or epoch second", + description => ?DESC(time_format), required => false, example => <<"2021-11-05T18:17:38+08:00">> } @@ -301,7 +301,7 @@ fields(trace) -> hoconsc:mk( hoconsc:array(map()), #{ - desc => "trace log size", + description => ?DESC(trace_log_size), example => [#{<<"node">> => <<"emqx@127.0.0.1">>, <<"size">> => 1024}], required => false } @@ -326,7 +326,7 @@ fields(node) -> hoconsc:mk( binary(), #{ - desc => "Node name", + description => ?DESC(node_name), in => query, required => false, example => "emqx@127.0.0.1" @@ -341,7 +341,7 @@ fields(bytes) -> %% across different OS range(0, ?MAX_SINT32), #{ - desc => "Maximum number of bytes to send in response", + description => ?DESC(max_response_bytes), in => query, required => false, default => 1000, @@ -356,7 +356,7 @@ fields(position) -> hoconsc:mk( integer(), #{ - desc => "Offset from the current trace position.", + description => ?DESC(current_trace_offset), in => query, required => false, default => 0 diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index f94b22d81..a0d4dee04 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.10"}, + {vsn, "5.0.11"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index d3bfc0224..6fb57483d 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -19,6 +19,7 @@ -behaviour(minirest_api). -include("emqx_prometheus.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -import(hoconsc, [ref/2]). @@ -50,14 +51,14 @@ schema("/prometheus") -> 'operationId' => prometheus, get => #{ - description => <<"Get Prometheus config info">>, + description => ?DESC(get_prom_conf_info), tags => ?TAGS, responses => #{200 => prometheus_config_schema()} }, put => #{ - description => <<"Update Prometheus config">>, + description => ?DESC(update_prom_conf_info), tags => ?TAGS, 'requestBody' => prometheus_config_schema(), responses => @@ -69,7 +70,7 @@ schema("/prometheus/stats") -> 'operationId' => stats, get => #{ - description => <<"Get Prometheus Data">>, + description => ?DESC(get_prom_data), tags => ?TAGS, security => [], responses => diff --git a/changes/ce/fix-10724.en.md b/changes/ce/fix-10724.en.md new file mode 100644 index 000000000..a38a9e842 --- /dev/null +++ b/changes/ce/fix-10724.en.md @@ -0,0 +1 @@ +A summary has been added for all endpoints in the HTTP API documentation (accessible at "http://emqx_host_name:18083/api-docs"). diff --git a/rel/i18n/emqx_authn_api.hocon b/rel/i18n/emqx_authn_api.hocon index 07f9c6c3e..90240c4f4 100644 --- a/rel/i18n/emqx_authn_api.hocon +++ b/rel/i18n/emqx_authn_api.hocon @@ -2,42 +2,68 @@ emqx_authn_api { authentication_get.desc: """List authenticators for global authentication.""" +authentication_get.label: +"""List authenticators""" authentication_id_delete.desc: """Delete authenticator from global authentication chain.""" +authentication_id_delete.label: +"""Delete authenticator""" authentication_id_get.desc: """Get authenticator from global authentication chain.""" +authentication_id_get.label: +"""Get authenticator""" authentication_id_position_put.desc: """Move authenticator in global authentication chain.""" +authentication_id_position_put.label: +"""Move authenticator""" authentication_id_put.desc: """Update authenticator from global authentication chain.""" +authentication_id_put.label: +"""Update authenticator""" authentication_id_status_get.desc: """Get authenticator status from global authentication chain.""" +authentication_id_status_get.label: +"""Get authenticator status""" authentication_id_users_get.desc: """List users in authenticator in global authentication chain.""" +authentication_id_users_get.label: +"""List users in authenticator""" authentication_id_users_post.desc: """Create users for authenticator in global authentication chain.""" +authentication_id_users_post.label: +"""Create users for authenticator""" authentication_id_users_user_id_delete.desc: """Delete user in authenticator in global authentication chain.""" +authentication_id_users_user_id_delete.label: +"""Delete user in authenticator""" authentication_id_users_user_id_get.desc: """Get user from authenticator in global authentication chain.""" +authentication_id_users_user_id_get.label: +"""Get user from authenticator""" authentication_id_users_user_id_put.desc: """Update user in authenticator in global authentication chain.""" +authentication_id_users_user_id_put.label: +"""Update user in authenticator""" authentication_post.desc: """Create authenticator for global authentication.""" +authentication_post.label: +"""Create authenticator""" is_superuser.desc: """Is superuser""" +is_superuser.label: +"""Is superuser""" like_user_id.desc: """Fuzzy search user_id (username or clientid).""" @@ -47,50 +73,82 @@ like_user_id.label: listeners_listener_id_authentication_get.desc: """List authenticators for listener authentication.""" +listeners_listener_id_authentication_get.label: +"""List authenticators for listener""" listeners_listener_id_authentication_id_delete.desc: """Delete authenticator from listener authentication chain.""" +listeners_listener_id_authentication_id_delete.label: +"""Delete authenticator from listener""" listeners_listener_id_authentication_id_get.desc: """Get authenticator from listener authentication chain.""" +listeners_listener_id_authentication_id_get.label: +"""Get authenticator from listener""" listeners_listener_id_authentication_id_position_put.desc: """Move authenticator in listener authentication chain.""" +listeners_listener_id_authentication_id_position_put.label: +"""Move authenticator in listener""" listeners_listener_id_authentication_id_put.desc: """Update authenticator from listener authentication chain.""" +listeners_listener_id_authentication_id_put.label: +"""Update authenticator from listener""" listeners_listener_id_authentication_id_status_get.desc: """Get authenticator status from listener authentication chain.""" +listeners_listener_id_authentication_id_status_get.label: +"""Get authenticator status from listener""" listeners_listener_id_authentication_id_users_get.desc: """List users in authenticator in listener authentication chain.""" +listeners_listener_id_authentication_id_users_get.label: +"""List users in authenticator in listener""" listeners_listener_id_authentication_id_users_post.desc: """Create users for authenticator in listener authentication chain.""" +listeners_listener_id_authentication_id_users_post.label: +"""Create users for authenticator in listener""" listeners_listener_id_authentication_id_users_user_id_delete.desc: """Delete user in authenticator in listener authentication chain.""" +listeners_listener_id_authentication_id_users_user_id_delete.label: +"""Delete user in authenticator in listener""" listeners_listener_id_authentication_id_users_user_id_get.desc: """Get user from authenticator in listener authentication chain.""" +listeners_listener_id_authentication_id_users_user_id_get.label: +"""Get user from authenticator in listener""" listeners_listener_id_authentication_id_users_user_id_put.desc: """Update user in authenticator in listener authentication chain.""" +listeners_listener_id_authentication_id_users_user_id_put.label: +"""Update user in authenticator in listener""" listeners_listener_id_authentication_post.desc: """Create authenticator for listener authentication.""" +listeners_listener_id_authentication_post.label: +"""Create authenticator for listener""" param_auth_id.desc: """Authenticator ID.""" +param_auth_id.label: +"""Authenticator ID""" param_listener_id.desc: """Listener ID.""" +param_listener_id.label: +"""Listener ID""" param_position.desc: """Position of authenticator in chain. Possible values are 'front', 'rear', 'before:{other_authenticator}', 'after:{other_authenticator}'.""" +param_position.label: +"""Position of authenticator""" param_user_id.desc: """User ID.""" +param_user_id.label: +"""User ID""" } diff --git a/rel/i18n/emqx_authn_user_import_api.hocon b/rel/i18n/emqx_authn_user_import_api.hocon index f8fb1757c..3a97823df 100644 --- a/rel/i18n/emqx_authn_user_import_api.hocon +++ b/rel/i18n/emqx_authn_user_import_api.hocon @@ -2,8 +2,12 @@ emqx_authn_user_import_api { authentication_id_import_users_post.desc: """Import users into authenticator in global authentication chain.""" +authentication_id_import_users_post.label: +"""Global import users into authenticator""" listeners_listener_id_authentication_id_import_users_post.desc: """Import users into authenticator in listener authentication chain.""" +listeners_listener_id_authentication_id_import_users_post.label: +"""Import users into authenticator in listener""" } diff --git a/rel/i18n/emqx_authz_api_cache.hocon b/rel/i18n/emqx_authz_api_cache.hocon index 0789c1e71..f3e933d4b 100644 --- a/rel/i18n/emqx_authz_api_cache.hocon +++ b/rel/i18n/emqx_authz_api_cache.hocon @@ -2,5 +2,7 @@ emqx_authz_api_cache { authorization_cache_delete.desc: """Clean all authorization cache in the cluster.""" +authorization_cache_delete.label: +"""Clean authorization cache in cluster""" } diff --git a/rel/i18n/emqx_authz_api_mnesia.hocon b/rel/i18n/emqx_authz_api_mnesia.hocon index 4cfed2970..d0021c6a5 100644 --- a/rel/i18n/emqx_authz_api_mnesia.hocon +++ b/rel/i18n/emqx_authz_api_mnesia.hocon @@ -2,13 +2,11 @@ emqx_authz_api_mnesia { action.desc: """Authorized action (pub/sub/all)""" - action.label: """action""" clientid.desc: """ClientID""" - clientid.label: """clientid""" @@ -26,62 +24,87 @@ fuzzy_username.label: permission.desc: """Permission""" - permission.label: """permission""" rules_all_delete.desc: """Delete rules for 'all'""" +rules_all_delete.label: +"""Delete rules for 'all'""" rules_all_get.desc: """Show the list of rules for 'all'""" +rules_all_get.label: +"""Show rules for 'all'""" rules_all_post.desc: """Create/Update the list of rules for 'all'.""" +rules_all_post.label: +"""Update rules for 'all'""" rules_delete.desc: """Delete all rules for all 'users', 'clients' and 'all'""" +rules_delete.label: +"""Delete all rules""" topic.desc: """Rule on specific topic""" - topic.label: """topic""" user_clientid_delete.desc: """Delete rule for 'clientid'""" +user_clientid_delete.label: +"""Delete rule for 'clientid'""" user_clientid_get.desc: """Get rule for 'clientid'""" +user_clientid_get.label: +"""Get rule for 'clientid'""" user_clientid_put.desc: """Set rule for 'clientid'""" +user_clientid_put.label: +"""Set rule for 'clientid'""" user_username_delete.desc: """Delete rule for 'username'""" +user_username_delete.label: +"""Delete rule for 'username'""" user_username_get.desc: """Get rule for 'username'""" +user_username_get.label: +"""Get rule for 'username'""" user_username_put.desc: """Set rule for 'username'""" +user_username_put.label: +"""Set rule for 'username'""" username.desc: """Username""" - username.label: """username""" users_clientid_get.desc: """Show the list of rules for clients""" +users_clientid_get.label: +"""Show rules for clients""" users_clientid_post.desc: """Add new rule for 'clientid'""" +users_clientid_post.label: +"""Add rule for 'clientid'""" users_username_get.desc: """Show the list of rules for users""" +users_username_get.label: +"""Show rules for users""" users_username_post.desc: """Add new rule for 'username'""" +users_username_post.label: +"""Add rule for 'username'""" } diff --git a/rel/i18n/emqx_authz_api_sources.hocon b/rel/i18n/emqx_authz_api_sources.hocon index 5d2dda69e..1257d9eb8 100644 --- a/rel/i18n/emqx_authz_api_sources.hocon +++ b/rel/i18n/emqx_authz_api_sources.hocon @@ -2,46 +2,56 @@ emqx_authz_api_sources { authorization_sources_get.desc: """List all authorization sources""" +authorization_sources_get.label: +"""List all authorization sources""" authorization_sources_post.desc: """Add a new source""" +authorization_sources_post.label: +"""Add a new source""" authorization_sources_type_delete.desc: """Delete source""" +authorization_sources_type_delete.label: +"""Delete source""" authorization_sources_type_get.desc: """Get a authorization source""" +authorization_sources_type_get.label: +"""Get a authorization source""" authorization_sources_type_move_post.desc: """Change the exection order of sources""" +authorization_sources_type_move_post.label: +"""Change order of sources""" authorization_sources_type_put.desc: """Update source""" +authorization_sources_type_put.label: +"""Update source""" authorization_sources_type_status_get.desc: """Get a authorization source""" +authorization_sources_type_status_get.label: +"""Get a authorization source""" source.desc: """Authorization source""" - source.label: """source""" source_config.desc: """Source config""" - source_config.label: """source_config""" source_type.desc: """Authorization type""" - source_type.label: """source_type""" sources.desc: """Authorization sources""" - sources.label: """sources""" diff --git a/rel/i18n/emqx_auto_subscribe_api.hocon b/rel/i18n/emqx_auto_subscribe_api.hocon index df8e87e1a..518878a50 100644 --- a/rel/i18n/emqx_auto_subscribe_api.hocon +++ b/rel/i18n/emqx_auto_subscribe_api.hocon @@ -2,11 +2,17 @@ emqx_auto_subscribe_api { list_auto_subscribe_api.desc: """Get auto subscribe topic list""" +list_auto_subscribe_api.label: +"""Get auto subscribe topics""" update_auto_subscribe_api.desc: """Update auto subscribe topic list""" +update_auto_subscribe_api.label: +"""Update auto subscribe topics""" update_auto_subscribe_api_response409.desc: """Auto Subscribe topics max limit""" +update_auto_subscribe_api_response409.label: +"""Auto Subscribe topics max limit""" } diff --git a/rel/i18n/emqx_dashboard_api.hocon b/rel/i18n/emqx_dashboard_api.hocon index 3e5bb6239..01fc6eb16 100644 --- a/rel/i18n/emqx_dashboard_api.hocon +++ b/rel/i18n/emqx_dashboard_api.hocon @@ -2,24 +2,36 @@ emqx_dashboard_api { change_pwd_api.desc: """Change dashboard user password""" +change_pwd_api.label: +"""Change dashboard user password""" create_user_api.desc: """Create dashboard user""" +create_user_api.label: +"""Create dashboard user""" create_user_api_success.desc: """Create dashboard user success""" +create_user_api_success.label: +"""Create dashboard user success""" delete_user_api.desc: """Delete dashboard user""" +delete_user_api.label: +"""Delete dashboard user""" license.desc: """EMQX License. opensource or enterprise""" list_users_api.desc: """Dashboard list users""" +list_users_api.label: +"""Dashboard list users""" login_api.desc: """Get Dashboard Auth Token.""" +login_api.label: +"""Get Dashboard Auth Token.""" login_failed401.desc: """Login failed. Bad username or password""" @@ -32,6 +44,8 @@ login_success.desc: logout_api.desc: """Dashboard user logout""" +logout_api.label: +"""Dashboard user logout""" new_pwd.desc: """New password""" @@ -47,6 +61,8 @@ token.desc: update_user_api.desc: """Update dashboard user description""" +update_user_api.label: +"""Update dashboard user description""" update_user_api200.desc: """Update dashboard user success""" diff --git a/rel/i18n/emqx_dashboard_error_code_api.hocon b/rel/i18n/emqx_dashboard_error_code_api.hocon new file mode 100644 index 000000000..835f200ab --- /dev/null +++ b/rel/i18n/emqx_dashboard_error_code_api.hocon @@ -0,0 +1,13 @@ +emqx_dashboard_error_code_api { + +error_codes.desc: +"""API Error Codes""" +error_codes.label: +"""API Error Codes""" + +error_codes_u.desc: +"""API Error Codes""" +error_codes_u.label: +"""API Error Codes""" + +} diff --git a/rel/i18n/emqx_dashboard_monitor_api.hocon b/rel/i18n/emqx_dashboard_monitor_api.hocon new file mode 100644 index 000000000..1d45c45ae --- /dev/null +++ b/rel/i18n/emqx_dashboard_monitor_api.hocon @@ -0,0 +1,23 @@ +emqx_dashboard_monitor_api { + +list_monitor.desc: +"""List monitor data.""" +list_monitor.label: +"""List monitor data.""" + +list_monitor_node.desc: +"""List the monitor data on the node.""" +list_monitor_node.label: +"""List the monitor data on the node.""" + +current_status.desc: +"""Current status. Gauge and rate.""" +current_status.label: +"""Current status. Gauge and rate.""" + +current_status_node.desc: +"""Node current status. Gauge and rate.""" +current_status_node.label: +"""Node current status. Gauge and rate.""" + +} diff --git a/rel/i18n/emqx_delayed_api.hocon b/rel/i18n/emqx_delayed_api.hocon index 62e0fd775..b891c618c 100644 --- a/rel/i18n/emqx_delayed_api.hocon +++ b/rel/i18n/emqx_delayed_api.hocon @@ -2,71 +2,117 @@ emqx_delayed_api { bad_msgid_format.desc: """Bad Message ID format""" +bad_msgid_format.label: +"""Bad Message ID format""" count.desc: """Count of delayed messages""" +count.label: +"""Count of delayed messages""" delayed_interval.desc: """Delayed interval(second)""" +delayed_interval.label: +"""Delayed interval""" delayed_remaining.desc: """Delayed remaining(second)""" +delayed_remaining.label: +"""Delayed remaining""" delete_api.desc: """Delete delayed message""" +delete_api.label: +"""Delete delayed message""" expected_at.desc: """Expect publish time, in RFC 3339 format""" +expected_at.label: +"""Expect publish time""" from_clientid.desc: """From ClientID""" +from_clientid.label: +"""From ClientID""" from_username.desc: """From Username""" +from_username.label: +"""From Username""" get_message_api.desc: """View delayed message""" +get_message_api.label: +"""View delayed message""" illegality_limit.desc: """Max limit illegality""" +illegality_limit.label: +"""Max limit illegality""" list_api.desc: """List delayed messages""" +list_api.label: +"""List delayed messages""" msgid.desc: """Delayed Message ID""" +msgid.label: +"""Delayed Message ID""" msgid_not_found.desc: """Message ID not found""" +msgid_not_found.label: +"""Message ID not found""" node.desc: """The node where message from""" +node.label: +"""Node where message from""" payload.desc: """Payload, base64 encoded. Payload will be set to 'PAYLOAD_TO_LARGE' if its length is larger than 2048 bytes""" +payload.label: +"""Payload""" publish_at.desc: """Clinet publish message time, in RFC 3339 format""" +publish_at.label: +"""Client publish message time""" qos.desc: """QoS""" +qos.label: +"""QoS""" topic.desc: """Topic""" +topic.label: +"""Topic""" update_api.desc: """Enable or disable delayed, set max delayed messages""" +update_api.label: +"""Enable or disable delayed""" update_success.desc: """Enable or disable delayed successfully""" +update_success.label: +"""Enable or disable delayed successfully""" view_limit.desc: """Page limit""" +view_limit.label: +"""Page limit""" view_page.desc: """View page""" +view_page.label: +"""View page""" view_status_api.desc: """Get delayed status""" +view_status_api.label: +"""Get delayed status""" } diff --git a/rel/i18n/emqx_exhook_api.hocon b/rel/i18n/emqx_exhook_api.hocon index 9cb7177c1..dabf89364 100644 --- a/rel/i18n/emqx_exhook_api.hocon +++ b/rel/i18n/emqx_exhook_api.hocon @@ -2,70 +2,109 @@ emqx_exhook_api { add_server.desc: """Add a server""" +add_server.label: +"""Add a server""" delete_server.desc: """Delete the server""" +delete_server.label: +"""Delete the server""" get_detail.desc: """Get the detail information of Exhook server""" +get_detail.label: +"""Get server details""" get_hooks.desc: """Get the hooks information of server""" +get_hooks.label: +"""Get server hooks information""" hook_metrics.desc: """Metrics information of this hook in the current node""" +hook_metrics.label: +"""Hook metrics""" hook_name.desc: """The hook's name""" +hook_name.label: +"""Hook name""" hook_params.desc: """The parameters used when the hook is registered""" +hook_params.label: +"""Hook parameters""" list_all_servers.desc: """List all servers""" +list_all_servers.label: +"""List servers""" metric_failed.desc: """The number of times the hook execution failed""" +metric_failed.label: +"""Failed executions count""" metric_max_rate.desc: """Maximum call rate of hooks""" +metric_max_rate.label: +"""Max hook call rate""" metric_rate.desc: """The call rate of hooks""" +metric_rate.label: +"""Hook call rate""" metric_succeed.desc: """The number of times the hooks execution successful""" +metric_succeed.label: +"""Successful executions count""" metrics.desc: """Metrics information""" +metrics.label: +"""Metrics information""" move_api.desc: """Move the server. NOTE: The position should be "front | rear | before:{name} | after:{name}""" - move_api.label: """Change order of execution for registered Exhook server""" move_position.desc: """The target position to be moved""" +move_position.label: +"""Target position""" node.desc: """Node name""" +node.label: +"""Node name""" node_hook_metrics.desc: """Metrics information of this hook in all nodes""" +node_hook_metrics.label: +"""Node-wise hook metrics""" node_metrics.desc: """Metrics information of this server in all nodes""" +node_metrics.label: +"""Node-wise server metrics""" node_status.desc: """status of this server in all nodes""" +node_status.label: +"""Node-wise server status""" server_metrics.desc: """Metrics information of this server in the current node""" +server_metrics.label: +"""Server metrics""" server_name.desc: """The Exhook server name""" +server_name.label: +"""Server name""" status.desc: """The status of Exhook server.
@@ -74,8 +113,12 @@ connecting: connection failed, reconnecting
disconnected: failed to connect and didn't reconnect
disabled: this server is disabled
error: failed to view the status of this server""" +status.label: +"""Server status""" update_server.desc: """Update the server""" +update_server.label: +"""Update the server""" } diff --git a/rel/i18n/emqx_mgmt_api_alarms.hocon b/rel/i18n/emqx_mgmt_api_alarms.hocon index 0327fffcd..b9c289114 100644 --- a/rel/i18n/emqx_mgmt_api_alarms.hocon +++ b/rel/i18n/emqx_mgmt_api_alarms.hocon @@ -8,6 +8,8 @@ deactivate_at.desc: delete_alarms_api.desc: """Remove all historical alarms.""" +delete_alarms_api.label: +"""Remove all historical alarms.""" delete_alarms_api_response204.desc: """Historical alarms have been cleared successfully.""" @@ -26,6 +28,8 @@ The default is false.""" list_alarms_api.desc: """List currently activated alarms or historical alarms, determined by query parameters.""" +list_alarms_api.label: +"""List alarms""" message.desc: """Alarm message, which describes the alarm content in a human-readable format.""" diff --git a/rel/i18n/emqx_mgmt_api_api_keys.hocon b/rel/i18n/emqx_mgmt_api_api_keys.hocon new file mode 100644 index 000000000..85d5c4ec7 --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_api_keys.hocon @@ -0,0 +1,33 @@ +emqx_mgmt_api_api_keys { + +api_key_list.desc: +"""Return api_key list""" +api_key_list.label: +"""Return api_key list""" + +create_new_api_key.desc: +"""Create new api_key""" +create_new_api_key.label: +"""Create new api_key""" + +get_api_key.desc: +"""Return the specific api_key""" +get_api_key.label: +"""Return the specific api_key""" + +update_api_key.desc: +"""Update the specific api_key""" +update_api_key.label: +"""Update the specific api_key""" + +delete_api_key.desc: +"""Delete the specific api_key""" +delete_api_key.label: +"""Delete the specific api_key""" + +format.desc: +"""Unique and format by [a-zA-Z0-9-_]""" +format.label: +"""Unique and format by [a-zA-Z0-9-_]""" + +} diff --git a/rel/i18n/emqx_mgmt_api_banned.hocon b/rel/i18n/emqx_mgmt_api_banned.hocon index 1a9700641..0a5439402 100644 --- a/rel/i18n/emqx_mgmt_api_banned.hocon +++ b/rel/i18n/emqx_mgmt_api_banned.hocon @@ -20,18 +20,24 @@ by.label: create_banned_api.desc: """Add a client ID, username or IP address to the blacklist.""" +create_banned_api.label: +"""Ban client ID, username or IP address""" create_banned_api_response400.desc: """Bad request, possibly due to wrong parameters or the existence of a banned object.""" delete_banned_api.desc: """Remove a client ID, username or IP address from the blacklist.""" +delete_banned_api.label: +"""Unban a client ID, username or IP address""" delete_banned_api_response404.desc: """The banned object was not found in the blacklist.""" list_banned_api.desc: """List all currently banned client IDs, usernames and IP addresses.""" +list_banned_api.label: +"""List all banned client IDs""" reason.desc: """Ban reason, record the reason why the current object was banned.""" diff --git a/rel/i18n/emqx_mgmt_api_clients.hocon b/rel/i18n/emqx_mgmt_api_clients.hocon new file mode 100644 index 000000000..d82642b1e --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_clients.hocon @@ -0,0 +1,58 @@ +emqx_mgmt_api_clients { + +list_clients.desc: +"""List clients""" +list_clients.label: +"""List clients""" + +clients_info_from_id.desc: +"""Get clients info by client ID""" +clients_info_from_id.label: +"""Get clients info by client ID""" + +kick_client_id.desc: +"""Kick out client by client ID""" +kick_client_id.label: +"""Kick out client by client ID""" + +get_authz_cache.desc: +"""Get client authz cache in the cluster.""" +get_authz_cache.label: +"""Get client authz cache in the cluster.""" + +clean_authz_cache.desc: +"""Clean client authz cache in the cluster.""" +clean_authz_cache.label: +"""Clean client authz cache in the cluster.""" + +get_client_subs.desc: +"""Get client subscriptions""" +get_client_subs.label: +"""Get client subscriptions""" + +subscribe.desc: +"""Subscribe""" +subscribe.label: +"""Subscribe""" + +subscribe_g.desc: +"""Subscribe bulk""" +subscribe_g.label: +"""Subscribe bulk""" + +unsubscribe.desc: +"""Unsubscribe""" +unsubscribe.label: +"""Unsubscribe""" + +unsubscribe_g.desc: +"""Unsubscribe bulk""" +unsubscribe_g.label: +"""Unsubscribe bulk""" + +set_keepalive_seconds.desc: +"""Set the online client keepalive by seconds""" +set_keepalive_seconds.label: +"""Set the online client keepalive by seconds""" + +} diff --git a/rel/i18n/emqx_mgmt_api_cluster.hocon b/rel/i18n/emqx_mgmt_api_cluster.hocon new file mode 100644 index 000000000..f8b6de1a4 --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_cluster.hocon @@ -0,0 +1,18 @@ +emqx_mgmt_api_cluster { + +get_cluster_info.desc: +"""Get cluster info""" +get_cluster_info.label: +"""Get cluster info""" + +invite_node.desc: +"""Invite node to cluster""" +invite_node.label: +"""Invite node to cluster""" + +force_remove_node.desc: +"""Force leave node from cluster""" +force_remove_node.label: +"""Force leave node from cluster""" + +} diff --git a/rel/i18n/emqx_mgmt_api_configs.hocon b/rel/i18n/emqx_mgmt_api_configs.hocon new file mode 100644 index 000000000..42bda7899 --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_configs.hocon @@ -0,0 +1,40 @@ +emqx_mgmt_api_configs { + +get_conf_node.desc: +"""Get all the configurations of the specified node, including hot and non-hot updatable items.""" +get_conf_node.label: +"""Get all the configurations for node.""" + +node_name.desc: +"""Node's name: If you do not fill in the fields, this node will be used by default.""" +node_name.label: +"""Node's name.""" + +rest_conf_query.desc: +"""Reset the config entry specified by the query string parameter `conf_path`.
+- For a config entry that has default value, this resets it to the default value; +- For a config entry that has no default value, an error 400 will be returned""" +rest_conf_query.label: +"""Reset the config entry with query""" + +get_global_zone_configs.desc: +"""Get the global zone configs""" +get_global_zone_configs.label: +"""Get the global zone configs""" + +update_globar_zone_configs.desc: +"""Update globbal zone configs""" +update_globar_zone_configs.label: +"""Update globbal zone configs""" + +get_node_level_limiter_congigs.desc: +"""Get the node-level limiter configs""" +get_node_level_limiter_congigs.label: +"""Get the node-level limiter configs""" + +update_node_level_limiter_congigs.desc: +"""Update the node-level limiter configs""" +update_node_level_limiter_congigs.label: +"""Update the node-level limiter configs""" + +} diff --git a/rel/i18n/emqx_mgmt_api_listeners.hocon b/rel/i18n/emqx_mgmt_api_listeners.hocon new file mode 100644 index 000000000..b45f55977 --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_listeners.hocon @@ -0,0 +1,59 @@ +emqx_mgmt_api_listeners { + +list_node_live_statuses.desc: +"""List all running node's listeners live status. group by listener type""" +list_node_live_statuses.label: +"""List all running node's listeners live status. group by listener type""" + +list_listeners.desc: +"""List all running node's listeners for the specified type.""" +list_listeners.label: +"""List all running node's listeners for the specified type.""" + +listener_type.desc: +"""Listener type""" +listener_type.label: +"""Listener type""" + +create_on_all_nodes.desc: +"""Create the specified listener on all nodes.""" +create_on_all_nodes.label: +"""Create the specified listener on all nodes.""" + +list_by_id.desc: +"""List all running node's listeners for the specified id.""" +list_by_id.label: +"""List all running node's listeners for the specified id.""" + +update_lisener.desc: +"""Update the specified listener on all nodes.""" +update_lisener.label: +"""Update the specified listener on all nodes.""" + +create_on_all_nodes_2.desc: +"""Create the specified listener on all nodes.""" +create_on_all_nodes_2.label: +"""Create the specified listener on all nodes.""" + +delete_on_all_nodes.desc: +"""Delete the specified listener on all nodes.""" +delete_on_all_nodes.label: +"""Delete the specified listener on all nodes.""" + +start_on_all_nodes.desc: +"""Start the listener on all nodes.""" +start_on_all_nodes.label: +"""Start the listener on all nodes.""" + +stop_on_all_nodes.desc: +"""Stop the listener on all nodes.""" +stop_on_all_nodes.label: +"""Stop the listener on all nodes.""" + +restart_on_all_nodes.desc: +"""Restart listeners on all nodes.""" +restart_on_all_nodes.label: +"""Restart listeners on all nodes.""" + + +} diff --git a/rel/i18n/emqx_mgmt_api_metrics.hocon b/rel/i18n/emqx_mgmt_api_metrics.hocon new file mode 100644 index 000000000..de1e7f20c --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_metrics.hocon @@ -0,0 +1,8 @@ +emqx_mgmt_api_metrics { + +emqx_metrics.desc: +"""EMQX metrics""" +emqx_metrics.label: +"""EMQX metrics""" + +} diff --git a/rel/i18n/emqx_mgmt_api_nodes.hocon b/rel/i18n/emqx_mgmt_api_nodes.hocon new file mode 100644 index 000000000..e875b0dcc --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_nodes.hocon @@ -0,0 +1,23 @@ +emqx_mgmt_api_nodes { + +list_nodes.desc: +"""List EMQX nodes""" +list_nodes.label: +"""List EMQX nodes""" + +get_node_info.desc: +"""Get node info""" +get_node_info.label: +"""Get node info""" + +get_node_metrics.desc: +"""Get node metrics""" +get_node_metrics.label: +"""Get node metrics""" + +get_node_stats.desc: +"""Get node stats""" +get_node_stats.label: +"""Get node stats""" + +} diff --git a/rel/i18n/emqx_mgmt_api_stats.hocon b/rel/i18n/emqx_mgmt_api_stats.hocon new file mode 100644 index 000000000..d3c57d6e8 --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_stats.hocon @@ -0,0 +1,8 @@ +emqx_mgmt_api_stats { + +emqx_stats.desc: +"""EMQX stats""" +emqx_stats.label: +"""EMQX stats""" + +} diff --git a/rel/i18n/emqx_mgmt_api_subscriptions.hocon b/rel/i18n/emqx_mgmt_api_subscriptions.hocon new file mode 100644 index 000000000..a0c8a7a01 --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_subscriptions.hocon @@ -0,0 +1,8 @@ +emqx_mgmt_api_subscriptions { + +list_subs.desc: +"""List subscriptions""" +list_subs.label: +"""List subscriptions""" + +} diff --git a/rel/i18n/emqx_mgmt_api_topics.hocon b/rel/i18n/emqx_mgmt_api_topics.hocon new file mode 100644 index 000000000..5b65af95a --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_topics.hocon @@ -0,0 +1,13 @@ +emqx_mgmt_api_topics { + +topic_list.desc: +"""Topics list""" +topic_list.label: +"""Topics list""" + +topic_info_by_name.desc: +"""Lookup topic info by name""" +topic_info_by_name.label: +"""Lookup topic info by name""" + +} diff --git a/rel/i18n/emqx_mgmt_api_trace.hocon b/rel/i18n/emqx_mgmt_api_trace.hocon new file mode 100644 index 000000000..4af68a460 --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_trace.hocon @@ -0,0 +1,118 @@ +emqx_mgmt_api_trace { + +list_all.desc: +"""List all trace""" +list_all.label: +"""List all trace""" + +create_new.desc: +"""Create new trace""" +create_new.label: +"""Create new trace""" + +clear_all.desc: +"""Clear all traces""" +clear_all.label: +"""Clear all traces""" + +delete_trace.desc: +"""Delete specified trace""" +delete_trace.label: +"""Delete specified trace""" + +stop_trace.desc: +"""Stop trace by name""" +stop_trace.label: +"""Stop trace by name""" + +download_log_by_name.desc: +"""Download trace log by name""" +download_log_by_name.label: +"""Download trace log by name""" + +trace_zip_file.desc: +"""A trace zip file""" +trace_zip_file.label: +"""A trace zip file""" + +get_trace_file_metadata.desc: +"""get trace log file's metadata, such as size, last update time""" +get_trace_file_metadata.label: +"""get trace log file's metadata""" + +view_trace_log.desc: +"""view trace log""" +view_trace_log.label: +"""view trace log""" + +file_size.desc: +"""file size""" +file_size.label: +"""file size""" + +modification_date.desc: +"""the modification and last access times of a file""" +modification_date.label: +"""last access time""" + +format.desc: +"""Unique and format by [a-zA-Z0-9-_]""" +format.label: +"""Unique and format by [a-zA-Z0-9-_]""" + +filter_type.desc: +"""Filter type""" +filter_type.label: +"""Filter type""" + +support_wildcard.desc: +"""support mqtt wildcard topic.""" +support_wildcard.label: +"""support mqtt wildcard topic""" + +mqtt_clientid.desc: +"""mqtt clientid.""" +mqtt_clientid.label: +"""mqtt clientid""" + +client_ip_addess.desc: +"""client ip address""" +client_ip_addess.label: +"""client ip address""" + +trace_status.desc: +"""trace status""" +trace_status.label: +"""trace status""" + +time_format.desc: +"""rfc3339 timestamp or epoch second""" +time_format.label: +"""rfc3339 timestamp or epoch second""" + +time_format_g.desc: +"""rfc3339 timestamp or epoch second""" +time_format_g.label: +"""rfc3339 timestamp or epoch second""" + +trace_log_size.desc: +"""trace log size""" +trace_log_size.label: +"""trace log size""" + +node_name.desc: +"""Node name""" +node_name.label: +"""Node name""" + +max_response_bytes.desc: +"""Maximum number of bytes to send in response""" +max_response_bytes.label: +"""Maximum response bytes""" + +current_trace_offset.desc: +"""Offset from the current trace position.""" +current_trace_offset.label: +"""Offset from the current trace position.""" + +} diff --git a/rel/i18n/emqx_prometheus_api.hocon b/rel/i18n/emqx_prometheus_api.hocon new file mode 100644 index 000000000..c468ccc2f --- /dev/null +++ b/rel/i18n/emqx_prometheus_api.hocon @@ -0,0 +1,18 @@ +emqx_prometheus_api { + +get_prom_conf_info.desc: +"""Get Prometheus config info""" +get_prom_conf_info.label: +"""Get Prometheus config info""" + +update_prom_conf_info.desc: +"""Update Prometheus config""" +update_prom_conf_info.label: +"""Update Prometheus config""" + +get_prom_data.desc: +"""Get Prometheus Data""" +get_prom_data.label: +"""Get Prometheus Data""" + +} diff --git a/rel/i18n/emqx_retainer_api.hocon b/rel/i18n/emqx_retainer_api.hocon index 5c7084778..cdb49f8d0 100644 --- a/rel/i18n/emqx_retainer_api.hocon +++ b/rel/i18n/emqx_retainer_api.hocon @@ -8,6 +8,8 @@ config_not_found.desc: delete_matching_api.desc: """Delete matching messages.""" +delete_matching_api.label: +"""Delete matching messages""" from_clientid.desc: """The clientid of publisher.""" @@ -17,12 +19,18 @@ from_username.desc: get_config_api.desc: """View config""" +get_config_api.label: +"""View config""" list_retained_api.desc: """List retained messages.""" +list_retained_api.label: +"""List retained messages.""" lookup_api.desc: """Lookup a message by a topic without wildcards.""" +lookup_api.label: +"""Lookup a message""" message_detail.desc: """Details of the message.""" @@ -59,5 +67,7 @@ update_config_success.desc: update_retainer_api.desc: """Update retainer config.""" +update_retainer_api.label: +"""Update retainer config""" } diff --git a/rel/i18n/emqx_rewrite_api.hocon b/rel/i18n/emqx_rewrite_api.hocon index 7e090e590..1b56cf24d 100644 --- a/rel/i18n/emqx_rewrite_api.hocon +++ b/rel/i18n/emqx_rewrite_api.hocon @@ -2,11 +2,17 @@ emqx_rewrite_api { list_topic_rewrite_api.desc: """List all rewrite rules""" +list_topic_rewrite_api.label: +"""List all rewrite rules""" update_topic_rewrite_api.desc: """Update all rewrite rules""" +update_topic_rewrite_api.label: +"""Update all rewrite rules""" update_topic_rewrite_api_response413.desc: """Rules count exceed max limit""" +update_topic_rewrite_api_response413.label: +"""Rules count exceed limit""" } diff --git a/rel/i18n/emqx_slow_subs_api.hocon b/rel/i18n/emqx_slow_subs_api.hocon index edf473487..f1a31784b 100644 --- a/rel/i18n/emqx_slow_subs_api.hocon +++ b/rel/i18n/emqx_slow_subs_api.hocon @@ -2,15 +2,21 @@ emqx_slow_subs_api { clear_records_api.desc: """Clear current data and re count slow topic""" +clear_records_api.label: +"""Clear current data and re count slow topic""" clientid.desc: """Message clientid""" get_records_api.desc: """View slow topics statistics record data""" +get_records_api.label: +"""View slow topics statistics record data""" get_setting_api.desc: """View slow subs settings""" +get_setting_api.label: +"""View slow subs settings""" last_update_time.desc: """The timestamp of last update""" @@ -26,5 +32,7 @@ topic.desc: update_setting_api.desc: """Update slow subs settings""" +update_setting_api.label: +"""Update slow subs settings""" } diff --git a/rel/i18n/emqx_telemetry_api.hocon b/rel/i18n/emqx_telemetry_api.hocon index 5c61b8d3c..4651cc8a0 100644 --- a/rel/i18n/emqx_telemetry_api.hocon +++ b/rel/i18n/emqx_telemetry_api.hocon @@ -14,9 +14,13 @@ enable.desc: get_telemetry_data_api.desc: """Get telemetry data""" +get_telemetry_data_api.label: +"""Get telemetry data""" get_telemetry_status_api.desc: """Get telemetry status""" +get_telemetry_status_api.label: +"""Get telemetry status""" license.desc: """Get license information""" @@ -47,6 +51,8 @@ up_time.desc: update_telemetry_status_api.desc: """Enable or disable telemetry""" +update_telemetry_status_api.label: +"""Enable or disable telemetry""" uuid.desc: """Get UUID""" diff --git a/rel/i18n/emqx_topic_metrics_api.hocon b/rel/i18n/emqx_topic_metrics_api.hocon index 94c58f0cd..bffd831c3 100644 --- a/rel/i18n/emqx_topic_metrics_api.hocon +++ b/rel/i18n/emqx_topic_metrics_api.hocon @@ -2,104 +2,172 @@ emqx_topic_metrics_api { message_qos1_in_rate.desc: """QoS1 in messages rate""" +message_qos1_in_rate.label: +"""QoS1 in messages rate""" message_out_count.desc: """Out messages count""" +message_out_count.label: +"""Out messages count""" message_qos2_out_rate.desc: """QoS2 out messages rate""" +message_qos2_out_rate.label: +"""QoS2 out messages rate""" message_qos0_in_rate.desc: """QoS0 in messages rate""" +message_qos0_in_rate.label: +"""QoS0 in messages rate""" get_topic_metrics_api.desc: """List topic metrics""" +get_topic_metrics_api.label: +"""List topic metrics""" reset_time.desc: """Reset time. In rfc3339. Nullable if never reset""" +reset_time.label: +"""Reset time""" topic_metrics_api_response400.desc: """Bad request. Already exists or bad topic name""" +topic_metrics_api_response400.label: +"""Bad request""" reset_topic_desc.desc: """Topic Name. If this parameter is not present,all created topic metrics will be reset.""" +reset_topic_desc.label: +"""Topic Name""" topic_metrics_api_response409.desc: """Conflict. Topic metrics exceeded max limit 512""" +topic_metrics_api_response409.label: +"""Conflict""" post_topic_metrics_api.desc: """Create topic metrics""" +post_topic_metrics_api.label: +"""Create topic metrics""" message_dropped_rate.desc: """Dropped messages rate""" +message_dropped_rate.label: +"""Dropped messages rate""" message_qos2_in_rate.desc: """QoS2 in messages rate""" +message_qos2_in_rate.label: +"""QoS2 in messages rate""" message_in_rate.desc: """In messages rate""" +message_in_rate.label: +"""In messages rate""" message_qos0_out_rate.desc: """QoS0 out messages rate""" +message_qos0_out_rate.label: +"""QoS0 out messages rate""" message_qos2_in_count.desc: """QoS2 in messages count""" +message_qos2_in_count.label: +"""QoS2 in messages count""" message_dropped_count.desc: """Dropped messages count""" +message_dropped_count.label: +"""Dropped messages count""" topic_metrics_api_response404.desc: """Not Found. Topic metrics not found""" +topic_metrics_api_response404.label: +"""Not Found""" topic_in_path.desc: """Topic string. Notice: Topic string in url path must be encoded""" +topic_in_path.label: +"""Topic string""" action.desc: """Action. Only support reset""" +action.label: +"""Action""" message_qos0_in_count.desc: """QoS0 in messages count""" +message_qos0_in_count.label: +"""QoS0 in messages count""" message_qos1_out_rate.desc: """QoS1 out messages rate""" +message_qos1_out_rate.label: +"""QoS1 out messages rate""" topic.desc: """Topic""" +topic.label: +"""Topic""" reset_topic_metrics_api.desc: """Reset telemetry status""" +reset_topic_metrics_api.label: +"""Reset telemetry status""" create_time.desc: """Create time""" +create_time.label: +"""Create time""" metrics.desc: """Metrics""" +metrics.label: +"""Metrics""" message_qos1_out_count.desc: """QoS1 out messages count""" +message_qos1_out_count.label: +"""QoS1 out messages count""" gat_topic_metrics_data_api.desc: """Get topic metrics""" +gat_topic_metrics_data_api.label: +"""Get topic metrics""" message_qos1_in_count.desc: """QoS1 in messages count""" +message_qos1_in_count.label: +"""QoS1 in messages count""" delete_topic_metrics_data_api.desc: """Delete topic metrics""" +delete_topic_metrics_data_api.label: +"""Delete topic metrics""" message_qos0_out_count.desc: """QoS0 out messages count""" +message_qos0_out_count.label: +"""QoS0 out messages count""" topic_in_body.desc: """Raw topic string""" +topic_in_body.label: +"""Raw topic string""" message_in_count.desc: """In messages count""" +message_in_count.label: +"""In messages count""" message_qos2_out_count.desc: """QoS2 out messages count""" +message_qos2_out_count.label: +"""QoS2 out messages count""" message_out_rate.desc: """Out messages rate""" +message_out_rate.label: +"""Out messages rate""" } From c95ef71fb54493aaf49a766809fc3f816ffe3a8d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 17 May 2023 12:49:20 +0300 Subject: [PATCH 157/197] chore(ft): provide more details in README.md --- apps/emqx_ft/README.md | 89 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 6 deletions(-) diff --git a/apps/emqx_ft/README.md b/apps/emqx_ft/README.md index c483b3169..a479754d2 100644 --- a/apps/emqx_ft/README.md +++ b/apps/emqx_ft/README.md @@ -1,9 +1,86 @@ -emqx_ft -===== +# EMQX File Transfer -EMQX file transfer over MQTT +EMQX File Transfer application enables the _File Transfer over MQTT_ feature described in [EIP-0021](https://github.com/emqx/eip), and provides support to publish transferred files either to the node-local file system or to the S3 API compatible remote object storage. -Build ------ +## Usage - $ rebar3 compile +As almost any other EMQX application, `emqx_ft` is configured via the EMQX configuration system. The following snippet is the minimal configuration that will enable File Transfer over MQTT. + +``` +file_transfer { + enabled = true +} +``` + +The configuration above will make File Transfer available to all MQTT clients, and will use the default storage backend, which in turn uses node-local file system both for temporary storage and for the final destination of the transferred files. + +## Configuration + +Every configuration parameter is described in the `emqx_ft_schema` module. + +The most important configuration parameter is `storage`, which defines the storage backend to use. Currently, only `local` storage backend is available, which stores all the temporary data accumulating during file transfers in the node-local file system. Those go into `${EMQX_DATA_DIR}/file_transfer` directory by default, but can be configured via `local.storage.segments.root` parameter. The final destination of the transferred files on the other hand is defined by `local.storage.exporter` parameter, and currently can be either `local` or `s3`. + +### Local Exporter + +The `local` exporter is the default one, and it stores the transferred files in the node-local file system. The final destination directory is defined by `local.storage.exporter.local.root` parameter, and defaults to `${EMQX_DATA_DIR}/file_transfer/exports` directory. + +``` +file_transfer { + enabled = true + storage { + local { + exporter { + local { root = "/var/lib/emqx/transfers" } + } + } + } +} +``` + +Important to note that even though the transferred files go into the node-local file system, the File Transfer API provides a cluster-wide view of the transferred files, and any file can be downloaded from any node in the cluster. + +### S3 Exporter + +The `s3` exporter stores the transferred files in the S3 API compatible remote object storage. The destination bucket is defined by `local.storage.exporter.s3.bucket` parameter. + +This snippet configures File Transfer to store the transferred files in the `my-bucket` bucket in the `us-east-1` region of the AWS S3 service. + +``` +file_transfer { + enabled = true + storage { + local { + exporter { + s3 { + host = "s3.us-east-1.amazonaws.com" + port = "443" + access_key_id = "AKIA27EZDDM9XLINWXFE" + secret_access_key = "..." + bucket = "my-bucket" + } + } + } + } +} + +``` + +## API + +### MQTT + +When enabled, File Transfer application reserves MQTT topics starting with `$file/` prefix for the purpose of serving the File Transfer protocol, as described in [EIP-0021](https://github.com/emqx/eip). + +### REST + +Application publishes a basic set of APIs, to: +* List all the transferred files available for download. +* Configure the application, including the storage backend. +* (When using `local` storage exporter) Download the transferred files. + +Switching to the `s3` storage exporter is possible at any time, but the files transferred before the switch will not be +available for download anymore. Though, the files will still be available in the node-local file system. + +## Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). From 07e46592a8398f11480dd781340699c03b789451 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 11 May 2023 14:56:24 +0800 Subject: [PATCH 158/197] chore(ci): add conf tests --- .github/workflows/run_conf_tests.yaml | 44 ++ scripts/conf-test/old-confs/ce-v5.0.24.conf | 295 ++++++++ scripts/conf-test/old-confs/ee-v5.0.0.conf | 374 ++++++++++ scripts/conf-test/old-confs/ee-v5.0.1.conf | 375 ++++++++++ scripts/conf-test/old-confs/ee-v5.0.2.conf | 454 +++++++++++++ scripts/conf-test/old-confs/ee-v5.0.3.conf | 715 ++++++++++++++++++++ scripts/conf-test/run.sh | 33 + 7 files changed, 2290 insertions(+) create mode 100644 .github/workflows/run_conf_tests.yaml create mode 100644 scripts/conf-test/old-confs/ce-v5.0.24.conf create mode 100644 scripts/conf-test/old-confs/ee-v5.0.0.conf create mode 100644 scripts/conf-test/old-confs/ee-v5.0.1.conf create mode 100644 scripts/conf-test/old-confs/ee-v5.0.2.conf create mode 100644 scripts/conf-test/old-confs/ee-v5.0.3.conf create mode 100755 scripts/conf-test/run.sh diff --git a/.github/workflows/run_conf_tests.yaml b/.github/workflows/run_conf_tests.yaml new file mode 100644 index 000000000..9b3201772 --- /dev/null +++ b/.github/workflows/run_conf_tests.yaml @@ -0,0 +1,44 @@ +name: Run Configuration tests + +concurrency: + group: test-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - master + - 'ci/**' + tags: + - v* + - e* + pull_request: + +env: + IS_CI: "yes" + +jobs: + run_conf_tests: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + profile: + - emqx + - emqx-enterprise + container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04" + steps: + - uses: AutoModality/action-clean@v1 + - uses: actions/checkout@v3 + with: + path: source + - name: build_package + working-directory: source + run: | + make ${{ matrix.profile }} + - name: run_tests + working-directory: source + env: + PROFILE: ${{ matrix.profile }} + run: | + ./scripts/conf-test/run.sh diff --git a/scripts/conf-test/old-confs/ce-v5.0.24.conf b/scripts/conf-test/old-confs/ce-v5.0.24.conf new file mode 100644 index 000000000..80ce00245 --- /dev/null +++ b/scripts/conf-test/old-confs/ce-v5.0.24.conf @@ -0,0 +1,295 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +log { + file_handlers.default { + level = warning + file = "log/emqx.log" + } +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + } +] +authorization { + cache {enable = true} + deny_action = "ignore" + no_match = "allow" + sources = [ + { + body {username = "${username}"} + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + request_timeout = "30s" + ssl {enable = false, verify = "verify_peer"} + type = "http" + url = "http://127.0.0.1:8080" + }, + { + enable = true + path = "etc/acl.conf" + type = "file" + } + ] +} +bridges { + webhook { + test { + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + pool_type = "random" + request_timeout = "15s" + resource_opts { + async_inflight_window = 100 + auto_restart_interval = "60s" + health_check_interval = "15s" + max_queue_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080/webhook" + } + } +} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "test" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } + ] +} +gateway { + coap { + connection_required = false + enable_stats = true + listeners { + udp { + default { + bind = "5683" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + notify_type = "qos" + publish_qos = "coap" + subscribe_qos = "coap" + } +} +listeners { + ssl { + default { + bind = "0.0.0.0:8883" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + } + } + tcp { + default { + acceptors = 16 + bind = "0.0.0.0:1883" + enable_authn = true + enabled = true + limiter { + bytes {rate = "infinity"} + client { + bytes {rate = "infinity"} + messages {rate = "infinity"} + } + messages {rate = "infinity"} + } + max_connections = 5000 + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + ws { + default { + bind = "0.0.0.0:8083" + max_connections = 1024000 + websocket {mqtt_path = "/mqtt"} + } + } + wss { + default { + bind = "0.0.0.0:8084" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + websocket {mqtt_path = "/mqtt"} + } + } +} +log { + console_handler { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = true + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = 100 + overload_kill { + enable = true + mem_size = "30MB" + qlen = "10000" + restart_after = "5s" + } + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + file_handlers { + default { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = false + file = "log/emqx.log" + flush_qlen = 8000 + formatter = "text" + level = "info" + max_depth = 100 + max_size = "50MB" + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + rotation {count = 10, enable = true} + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = false + idle_timeout = "15s" + ignore_loop_deliver = false + keepalive_backoff = 0.75 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 32 + max_mqueue_len = 1000 + max_packet_size = "10MB" + max_qos_allowed = 2 + max_subscriptions = "infinity" + max_topic_alias = 65535 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "disabled" + peer_cert_as_username = "disabled" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = "disabled" + session_expiry_interval = "12h" + shared_subscription = true + strict_mode = false + upgrade_qos = false + use_username_as_clientid = false + wildcard_subscription = true +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 0 + storage_type = "ram" + type = "built_in_database" + } + enable = true + max_payload_size = "10MB" + msg_clear_interval = "0s" + msg_expiry_interval = "0s" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + rules { + "rule_s56b" { + actions = ["webhook:test"] + description = "" + metadata {created_at = 1683882654432} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = true + client_subscribed = true + client_unsubscribed = true + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} diff --git a/scripts/conf-test/old-confs/ee-v5.0.0.conf b/scripts/conf-test/old-confs/ee-v5.0.0.conf new file mode 100644 index 000000000..023c0a6ca --- /dev/null +++ b/scripts/conf-test/old-confs/ee-v5.0.0.conf @@ -0,0 +1,374 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +log { + file_handlers.default { + level = warning + file = "log/emqx.log" + } +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + + +listeners.tcp.default { + bind = "0.0.0.0:1883" + max_connections = 1024000 +} + +listeners.ssl.default { + bind = "0.0.0.0:8883" + max_connections = 512000 + ssl_options { + keyfile = "etc/certs/key.pem" + certfile = "etc/certs/cert.pem" + cacertfile = "etc/certs/cacert.pem" + } +} + +listeners.ws.default { + bind = "0.0.0.0:8083" + max_connections = 1024000 + websocket.mqtt_path = "/mqtt" +} + +listeners.wss.default { + bind = "0.0.0.0:8084" + max_connections = 512000 + websocket.mqtt_path = "/mqtt" + ssl_options { + keyfile = "etc/certs/key.pem" + certfile = "etc/certs/cert.pem" + cacertfile = "etc/certs/cacert.pem" + } +} + +dashboard { + listeners.http { + bind = 18083 + } + default_username = "admin" + default_password = "public" +} + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + } +] +authorization { + cache {enable = true} + deny_action = "ignore" + no_match = "allow" + sources = [ + { + body {username = "${username}"} + connect_timeout = "5s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + type = "http" + url = "http://127.0.0.1:8080" + }, + { + enable = true + path = "etc/acl.conf" + type = "file" + } + ] +} +bridges { + webhook { + default { + connect_timeout = "5s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 4 + pool_type = "random" + request_timeout = "5s" + resource_opts { + async_inflight_window = 100 + auto_restart_interval = "60s" + health_check_interval = "15s" + max_queue_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080/webhook" + } + } +} +conn_congestion {enable_alarm = true, min_alarm_sustain_duration = "1m"} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "exhook" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8000" + } + ] +} +flapping_detect { + ban_time = "5m" + enable = false + max_count = 15 + window_time = "1m" +} +force_gc { + bytes = "16MB" + count = 16000 + enable = true +} +force_shutdown { + enable = true + max_heap_size = "32MB" + max_message_queue_len = 1000 +} +gateway { + stomp { + enable_stats = true + frame { + max_body_length = 8192 + max_headers = 10 + max_headers_length = 1024 + } + idle_timeout = "30s" + listeners { + tcp { + default { + bind = "61613" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + } +} +limiter { + bytes_in {burst = "1000", rate = "infinity"} + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "1000KB/s" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "1000" + } + internal { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "1000" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection {burst = 0, rate = "infinity"} + internal {burst = 0, rate = "infinity"} + message_in {burst = 0, rate = "infinity"} + message_routing {burst = 0, rate = "1000"} +} +log { + console_handler { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = true + flush_qlen = 8000 + formatter = "text" + level = "info" + max_depth = 100 + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + file_handlers { + default { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = false + file = "log/emqx.log" + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = 100 + max_size = "50MB" + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + rotation {count = 10, enable = true} + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = false + idle_timeout = "15s" + ignore_loop_deliver = false + keepalive_backoff = 0.5 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 32 + max_mqueue_len = 1000 + max_packet_size = "1MB" + max_qos_allowed = 2 + max_subscriptions = "infinity" + max_topic_alias = 65535 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "disabled" + peer_cert_as_username = "disabled" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = "disabled" + session_expiry_interval = "2h" + shared_subscription = true + strict_mode = false + upgrade_qos = false + use_username_as_clientid = false + wildcard_subscription = true +} +overload_protection { + backoff_delay = 1 + backoff_gc = false + backoff_hibernation = true + backoff_new_conn = true + enable = false +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 0 + storage_type = "ram" + type = "built_in_database" + } + enable = true + flow_control { + batch_deliver_limiter { + capacity = "infinity" + client { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + initial = "0" + rate = "infinity" + } + batch_deliver_number = 0 + batch_read_number = 0 + } + max_payload_size = "10MB" + msg_clear_interval = "0s" + msg_expiry_interval = "0s" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + jq_implementation_module = "jq_nif" + rules { + "rule_k9y8" { + actions = ["webhook:default"] + description = "" + metadata {created_at = 1683875156258} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} +stats {enable = true} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = false + client_subscribed = false + client_unsubscribed = false + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} diff --git a/scripts/conf-test/old-confs/ee-v5.0.1.conf b/scripts/conf-test/old-confs/ee-v5.0.1.conf new file mode 100644 index 000000000..e60662b95 --- /dev/null +++ b/scripts/conf-test/old-confs/ee-v5.0.1.conf @@ -0,0 +1,375 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +log { + file_handlers.default { + level = warning + file = "log/emqx.log" + } +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + +dashboard { + listeners.http { + bind = 18083 + } + default_username = "admin" + default_password = "public" +} + + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + } +] +authorization { + cache {enable = true} + deny_action = "ignore" + no_match = "allow" + sources = [ + { + body {username = "${username}"} + connect_timeout = "5s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + type = "http" + url = "http://127.0.0.1:8080" + }, + { + enable = true + path = "etc/acl.conf" + type = "file" + } + ] +} +bridges { + webhook { + test { + connect_timeout = "5s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 4 + pool_type = "random" + request_timeout = "15s" + resource_opts { + async_inflight_window = 100 + auto_restart_interval = "60s" + health_check_interval = "15s" + max_queue_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } + } +} +gateway { + mqttsn { + broadcast = true + "enable_qos3" = true + enable_stats = true + gateway_id = 1 + idle_timeout = "30s" + listeners { + udp { + default { + bind = "1884" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + predefined = [] + } +} +limiter { + bytes_in {burst = "0", rate = "10MB"} + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "100KB" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + internal { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection {burst = "0", rate = "1000"} + internal {burst = "0", rate = "infinity"} + message_in {burst = "0", rate = "1000"} + message_routing {burst = "0", rate = "infinity"} +} +listeners { + ssl { + default { + bind = "0.0.0.0:8883" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + } + } + tcp { + default { + acceptors = 16 + bind = "0.0.0.0:1883" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = 1000 + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 2000 + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + ws { + default { + bind = "0.0.0.0:8083" + max_connections = 1024000 + websocket {mqtt_path = "/mqtt"} + } + } + wss { + default { + bind = "0.0.0.0:8084" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + websocket {mqtt_path = "/mqtt"} + } + } +} +log { + console_handler { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = true + flush_qlen = 8000 + formatter = "text" + level = "info" + max_depth = 100 + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + file_handlers { + default { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = false + file = "log/emqx.log" + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = 100 + max_size = "50MB" + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + rotation {count = 10, enable = true} + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + } +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 0 + storage_type = "ram" + type = "built_in_database" + } + enable = true + flow_control { + batch_deliver_limiter { + capacity = "infinity" + client { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + initial = "0" + rate = "infinity" + } + batch_deliver_number = 0 + batch_read_number = 0 + } + max_payload_size = "5MB" + msg_clear_interval = "0s" + msg_expiry_interval = "0s" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + jq_implementation_module = jq_nif + rules { + "rule_bw1q" { + actions = ["webhook:test"] + description = "" + metadata {created_at = 1683878250142} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} diff --git a/scripts/conf-test/old-confs/ee-v5.0.2.conf b/scripts/conf-test/old-confs/ee-v5.0.2.conf new file mode 100644 index 000000000..f5f3b1394 --- /dev/null +++ b/scripts/conf-test/old-confs/ee-v5.0.2.conf @@ -0,0 +1,454 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +log { + file_handlers.default { + level = warning + file = "log/emqx.log" + } +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + +dashboard { + listeners.http { + bind = 18083 + } + default_username = "admin" + default_password = "public" +} + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + } +] +authorization { + cache {enable = true} + deny_action = "ignore" + no_match = "allow" + sources = [ + { + body {username = "${username}"} + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + request_timeout = "30s" + ssl {enable = false, verify = "verify_peer"} + type = "http" + url = "http://127.0.0.1:8080" + }, + { + enable = true + path = "etc/acl.conf" + type = "file" + } + ] +} +bridges { + webhook { + tests { + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + pool_type = "random" + request_timeout = "15s" + resource_opts { + async_inflight_window = 100 + auto_restart_interval = "60s" + health_check_interval = "15s" + max_queue_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } + } +} +conn_congestion {enable_alarm = true, min_alarm_sustain_duration = "1m"} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "test" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } + ] +} +flapping_detect { + ban_time = "5m" + enable = false + max_count = 15 + window_time = "1m" +} +force_gc { + bytes = "16MB" + count = 16000 + enable = true +} +force_shutdown { + enable = true + max_heap_size = "32MB" + max_message_queue_len = 1000 +} +gateway { + coap { + connection_required = false + enable_stats = true + listeners { + udp { + default { + bind = "5683" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + notify_type = "qos" + publish_qos = "coap" + subscribe_qos = "coap" + } +} +limiter { + bytes_in {burst = "0", rate = "infinity"} + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + internal { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "500" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "200" + low_watermark = "60" + max_retry_time = "10s" + rate = "100" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection {burst = "0", rate = "infinity"} + internal {burst = "0", rate = "1000"} + message_in {burst = "0", rate = "10000"} + message_routing {burst = "0", rate = "infinity"} +} +listeners { + ssl { + default { + bind = "0.0.0.0:8883" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + } + } + tcp { + default { + acceptors = 16 + bind = "0.0.0.0:1883" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = "1000" + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 1024000 + mountpoint = "topic_prefix1/x" + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + ws { + default { + bind = "0.0.0.0:8083" + max_connections = 1024000 + websocket {mqtt_path = "/mqtt"} + } + } + wss { + default { + bind = "0.0.0.0:8084" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + websocket {mqtt_path = "/mqtt"} + } + } +} +log { + console_handler { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = true + flush_qlen = 8000 + formatter = "text" + level = "info" + max_depth = 100 + overload_kill { + enable = true + mem_size = "30MB" + qlen = "10000" + restart_after = "5s" + } + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + file_handlers { + default { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = true + file = "log/emqx.log" + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = 100 + max_size = "1000MB" + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + rotation {count = 10, enable = true} + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = false + idle_timeout = "15s" + ignore_loop_deliver = false + keepalive_backoff = 0.75 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 8 + max_mqueue_len = 1000 + max_packet_size = "1MB" + max_qos_allowed = 2 + max_subscriptions = "infinity" + max_topic_alias = 1024 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "disabled" + peer_cert_as_username = "disabled" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = "disabled" + session_expiry_interval = "2h" + shared_subscription = true + strict_mode = false + upgrade_qos = false + use_username_as_clientid = false + wildcard_subscription = true +} +overload_protection { + backoff_delay = 1 + backoff_gc = false + backoff_hibernation = true + backoff_new_conn = true + enable = false +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 0 + storage_type = "disc" + type = "built_in_database" + } + enable = true + flow_control { + batch_deliver_limiter { + capacity = "infinity" + client { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + initial = "0" + rate = "infinity" + } + batch_deliver_number = 0 + batch_read_number = 0 + } + max_payload_size = "1MB" + msg_clear_interval = "0s" + msg_expiry_interval = "120s" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + jq_implementation_module = "jq_nif" + rules { + "rule_bx87" { + actions = ["webhook:tests"] + description = "" + metadata {created_at = 1683880400358} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} +stats {enable = true} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = true + client_subscribed = true + client_unsubscribed = true + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} diff --git a/scripts/conf-test/old-confs/ee-v5.0.3.conf b/scripts/conf-test/old-confs/ee-v5.0.3.conf new file mode 100644 index 000000000..1981b8d09 --- /dev/null +++ b/scripts/conf-test/old-confs/ee-v5.0.3.conf @@ -0,0 +1,715 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + + +dashboard { + listeners.http { + bind = 18083 + } +} + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + } +] +authorization { + cache { + enable = true + max_size = 32 + ttl = "1m" + } + deny_action = "ignore" + no_match = "allow" + sources = [ + { + body {username = "${username}"} + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + request_timeout = "30s" + ssl {enable = false, verify = "verify_peer"} + type = "http" + url = "http://127.0.0.1:8080" + }, + { + enable = true + path = "${EMQX_ETC_DIR}/acl.conf" + type = "file" + } + ] +} +delayed {enable = true, max_delayed_messages = 200} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "test" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } + ] +} +gateway { + coap { + connection_required = false + enable_stats = true + listeners { + udp { + default { + bind = "5683" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + notify_type = "qos" + publish_qos = "coap" + subscribe_qos = "coap" + } +} +limiter { + bytes_in {burst = "10MB", rate = "10MB/s"} + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "10KB" + low_watermark = "0" + max_retry_time = "10s" + rate = "100KB/s" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "100" + low_watermark = "0" + max_retry_time = "10s" + rate = "100" + } + internal { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "100" + low_watermark = "0" + max_retry_time = "10s" + rate = "100" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "100" + low_watermark = "0" + max_retry_time = "10s" + rate = "100" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "100" + low_watermark = "0" + max_retry_time = "10s" + rate = "100" + } + } + connection {burst = "1000", rate = "1000"} + internal {burst = "1000", rate = "1000"} + message_in {burst = "1000", rate = "1000"} + message_routing {burst = "1000", rate = "1000"} +} +listeners { + ssl { + default { + acceptors = 16 + access_rules = ["allow all"] + bind = "0.0.0.0:8883" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = "1000" + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 5000000 + mountpoint = "" + proxy_protocol = false + proxy_protocol_timeout = "3s" + ssl_options { + cacertfile = "${EMQX_ETC_DIR}/certs/cacert.pem" + certfile = "${EMQX_ETC_DIR}/certs/cert.pem" + ciphers = [] + client_renegotiation = true + depth = 10 + enable_crl_check = false + fail_if_no_peer_cert = false + gc_after_handshake = false + handshake_timeout = "15s" + hibernate_after = "5s" + honor_cipher_order = true + keyfile = "${EMQX_ETC_DIR}/certs/key.pem" + ocsp { + enable_ocsp_stapling = false + refresh_http_timeout = "15s" + refresh_interval = "5m" + } + reuse_sessions = true + secure_renegotiate = true + verify = "verify_none" + versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] + } + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + tcp { + default { + acceptors = 16 + access_rules = ["allow all"] + bind = "0.0.0.0:1883" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = "1000" + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 12222 + mountpoint = "" + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + ws { + default { + acceptors = 16 + access_rules = ["allow all"] + bind = "0.0.0.0:8083" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = "1000" + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 5000000 + mountpoint = "" + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + websocket { + allow_origin_absence = true + check_origin_enable = false + check_origins = "http://localhost:18083, http://127.0.0.1:18083" + compress = false + deflate_opts { + client_context_takeover = "takeover" + client_max_window_bits = 15 + mem_level = 8 + server_context_takeover = "takeover" + server_max_window_bits = 15 + strategy = "default" + } + fail_if_no_subprotocol = true + idle_timeout = "7200s" + max_frame_size = "infinity" + mqtt_path = "/mqtt" + mqtt_piggyback = "multiple" + proxy_address_header = "x-forwarded-for" + proxy_port_header = "x-forwarded-port" + supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" + } + zone = "default" + } + } + wss { + default { + acceptors = 16 + access_rules = ["allow all"] + bind = "0.0.0.0:8084" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = "1000" + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 5000000 + mountpoint = "" + proxy_protocol = false + proxy_protocol_timeout = "3s" + ssl_options { + cacertfile = "${EMQX_ETC_DIR}/certs/cacert.pem" + certfile = "${EMQX_ETC_DIR}/certs/cert.pem" + ciphers = [] + client_renegotiation = true + depth = 10 + fail_if_no_peer_cert = false + handshake_timeout = "15s" + hibernate_after = "5s" + honor_cipher_order = true + keyfile = "${EMQX_ETC_DIR}/certs/key.pem" + reuse_sessions = true + secure_renegotiate = true + verify = "verify_none" + versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] + } + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + websocket { + allow_origin_absence = true + check_origin_enable = false + check_origins = "http://localhost:18083, http://127.0.0.1:18083" + compress = false + deflate_opts { + client_context_takeover = "takeover" + client_max_window_bits = 15 + mem_level = 8 + server_context_takeover = "takeover" + server_max_window_bits = 15 + strategy = "default" + } + fail_if_no_subprotocol = true + idle_timeout = "7200s" + max_frame_size = "infinity" + mqtt_path = "/mqtt" + mqtt_piggyback = "multiple" + proxy_address_header = "x-forwarded-for" + proxy_port_header = "x-forwarded-port" + supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" + } + zone = "default" + } + } +} +log { + console_handler { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = true + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = 100 + overload_kill { + enable = true + mem_size = "30MB" + qlen = "10000" + restart_after = "5s" + } + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + file_handlers { + default { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = false + file = "${EMQX_LOG_DIR}/emqx.log" + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = 100 + max_size = "500MB" + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + rotation {count = 10, enable = true} + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = false + idle_timeout = "15s" + ignore_loop_deliver = false + keepalive_backoff = 0.75 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 32 + max_mqueue_len = 1000 + max_packet_size = "20MB" + max_qos_allowed = 2 + max_subscriptions = "infinity" + max_topic_alias = 65535 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "disabled" + peer_cert_as_username = "disabled" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = "disabled" + session_expiry_interval = "12h" + shared_subscription = true + strict_mode = false + upgrade_qos = false + use_username_as_clientid = false + wildcard_subscription = true +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 0 + storage_type = "disc" + type = "built_in_database" + } + enable = true + flow_control { + batch_deliver_limiter { + capacity = "infinity" + client { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + initial = "0" + rate = "infinity" + } + batch_deliver_number = 0 + batch_read_number = 0 + } + max_payload_size = "20MB" + msg_clear_interval = "0s" + msg_expiry_interval = "0s" + stop_publish_clear_msg = false +} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = true + client_subscribed = true + client_unsubscribed = false + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} +sysmon { + os { + cpu_check_interval = "60s" + cpu_high_watermark = "80%" + cpu_low_watermark = "60%" + mem_check_interval = "60s" + procmem_high_watermark = "5%" + sysmem_high_watermark = "70%" + } + vm { + busy_dist_port = true + busy_port = true + large_heap = "disabled" + long_gc = "disabled" + long_schedule = "240ms" + process_check_interval = "30s" + process_high_watermark = "80%" + process_low_watermark = "60%" + } +} diff --git a/scripts/conf-test/run.sh b/scripts/conf-test/run.sh new file mode 100755 index 000000000..52bdd5df5 --- /dev/null +++ b/scripts/conf-test/run.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +PROFILE="${PROFILE:-emqx}" +EMQX_ROOT="${EMQX_ROOT:-_build/$PROFILE/rel/emqx}" +EMQX_WAIT_FOR_START="${EMQX_WAIT_FOR_START:-30}" +export EMQX_WAIT_FOR_START + +start_emqx_with_conf() { + echo "Starting $PROFILE with $1" + $EMQX_ROOT/bin/emqx start + $EMQX_ROOT/bin/emqx stop +} + +MINOR_VSN=$(./pkg-vsn.sh "$PROFILE" | cut -d. -f1,2) + +if [ "$PROFILE" = "emqx" ]; then + EDITION="ce" +else + EDITION="ee" +fi + +FILES=$(ls ./scripts/conf-test/old-confs/$EDITION-v"$MINOR_VSN"*) + +cp $EMQX_ROOT/etc/emqx.conf $EMQX_ROOT/etc/emqx.conf.bak +cleanup() { + cp $EMQX_ROOT/etc/emqx.conf.bak $EMQX_ROOT/etc/emqx.conf +} +trap cleanup EXIT + +for file in $FILES; do + cp $file $EMQX_ROOT/etc/emqx.conf + start_emqx_with_conf $file +done From 90d1a0096c80cf17d45910bfd5f5466c403ba305 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sun, 14 May 2023 13:03:25 +0800 Subject: [PATCH 159/197] chore: make spellcheck happy --- .github/workflows/run_conf_tests.yaml | 9 +++++++++ scripts/conf-test/run.sh | 14 ++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run_conf_tests.yaml b/.github/workflows/run_conf_tests.yaml index 9b3201772..93d9fee95 100644 --- a/.github/workflows/run_conf_tests.yaml +++ b/.github/workflows/run_conf_tests.yaml @@ -42,3 +42,12 @@ jobs: PROFILE: ${{ matrix.profile }} run: | ./scripts/conf-test/run.sh + - name: print_erlang_log + if: failure() + run: | + cat source/_build/${{ matrix.profile }}/rel/emqx/logs/erlang.log.* + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: logs-${{ matrix.profile }} + path: source/_build/${{ matrix.profile }}/rel/emqx/logs diff --git a/scripts/conf-test/run.sh b/scripts/conf-test/run.sh index 52bdd5df5..1fa3b3951 100755 --- a/scripts/conf-test/run.sh +++ b/scripts/conf-test/run.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -euo pipefail + PROFILE="${PROFILE:-emqx}" EMQX_ROOT="${EMQX_ROOT:-_build/$PROFILE/rel/emqx}" EMQX_WAIT_FOR_START="${EMQX_WAIT_FOR_START:-30}" @@ -7,8 +9,8 @@ export EMQX_WAIT_FOR_START start_emqx_with_conf() { echo "Starting $PROFILE with $1" - $EMQX_ROOT/bin/emqx start - $EMQX_ROOT/bin/emqx stop + "$EMQX_ROOT"/bin/emqx start + "$EMQX_ROOT"/bin/emqx stop } MINOR_VSN=$(./pkg-vsn.sh "$PROFILE" | cut -d. -f1,2) @@ -21,13 +23,13 @@ fi FILES=$(ls ./scripts/conf-test/old-confs/$EDITION-v"$MINOR_VSN"*) -cp $EMQX_ROOT/etc/emqx.conf $EMQX_ROOT/etc/emqx.conf.bak +cp "$EMQX_ROOT"/etc/emqx.conf "$EMQX_ROOT"/etc/emqx.conf.bak cleanup() { - cp $EMQX_ROOT/etc/emqx.conf.bak $EMQX_ROOT/etc/emqx.conf + cp "$EMQX_ROOT"/etc/emqx.conf.bak "$EMQX_ROOT"/etc/emqx.conf } trap cleanup EXIT for file in $FILES; do - cp $file $EMQX_ROOT/etc/emqx.conf - start_emqx_with_conf $file + cp "$file" "$EMQX_ROOT"/etc/emqx.conf + start_emqx_with_conf "$file" done From a05156df9ad210fe253a0cd6eaaabefb7dbe5b4b Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 16 May 2023 17:04:26 +0800 Subject: [PATCH 160/197] chore: tests v5.0.20~25 confs --- scripts/conf-test/old-confs/ce-v5.0.20.conf | 458 ++++++++++++++++++ scripts/conf-test/old-confs/ce-v5.0.21.conf | 498 +++++++++++++++++++ scripts/conf-test/old-confs/ce-v5.0.22.conf | 502 ++++++++++++++++++++ scripts/conf-test/old-confs/ce-v5.0.23.conf | 280 +++++++++++ scripts/conf-test/old-confs/ce-v5.0.25.conf | 437 +++++++++++++++++ 5 files changed, 2175 insertions(+) create mode 100644 scripts/conf-test/old-confs/ce-v5.0.20.conf create mode 100644 scripts/conf-test/old-confs/ce-v5.0.21.conf create mode 100644 scripts/conf-test/old-confs/ce-v5.0.22.conf create mode 100644 scripts/conf-test/old-confs/ce-v5.0.23.conf create mode 100644 scripts/conf-test/old-confs/ce-v5.0.25.conf diff --git a/scripts/conf-test/old-confs/ce-v5.0.20.conf b/scripts/conf-test/old-confs/ce-v5.0.20.conf new file mode 100644 index 000000000..b8361a7be --- /dev/null +++ b/scripts/conf-test/old-confs/ce-v5.0.20.conf @@ -0,0 +1,458 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +log { + file_handlers.default { + level = warning + file = "log/emqx.log" + } +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + +dashboard { + listeners.http { + bind = 18083 + } + default_username = "admin" + default_password = "public" +} + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + }, + { + backend = "mysql" + database = "mqtt_user" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + pool_size = 8 + query = "SELECT password_hash, salt FROM mqtt_user where username = ${username} LIMIT 1" + query_timeout = "5s" + server = "127.0.0.1:3306" + ssl {enable = false, verify = "verify_peer"} + username = "root" + }, + { + backend = "mongodb" + collection = "users" + database = "mqtt" + filter {username = "${username}"} + mechanism = "password_based" + mongo_type = "single" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + password_hash_field = "password_hash" + pool_size = 8 + salt_field = "salt" + server = "127.0.0.1:27017" + ssl {enable = false, verify = "verify_peer"} + topology {connect_timeout_ms = "20s"} + }, + { + backend = "postgresql" + database = "mqtt_user" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + pool_size = 8 + query = "SELECT password_hash, salt FROM mqtt_user where username = ${username} LIMIT 1" + server = "127.0.0.1:5432" + ssl {enable = false, verify = "verify_peer"} + username = "root" + }, + { + backend = "redis" + cmd = "HMGET mqtt_user:${username} password_hash salt" + database = 0 + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + pool_size = 8 + redis_type = "single" + server = "127.0.0.1:6379" + ssl {enable = false, verify = "verify_peer"} + }, + { + backend = "http" + body {password = "${password}", username = "${username}"} + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + mechanism = "password_based" + method = "post" + pool_size = 8 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } +] +authorization { + cache {enable = true} + deny_action = "ignore" + no_match = "allow" + sources = [ + {type = "built_in_database"}, + { + enable = true + path = "etc/acl.conf" + type = "file" + } + ] +} +bridges { + webhook { + test { + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + pool_type = "random" + request_timeout = "15s" + resource_opts { + async_inflight_window = 100 + auto_restart_interval = "60s" + health_check_interval = "15s" + max_queue_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080/webhook" + } + } +} +conn_congestion {enable_alarm = true, min_alarm_sustain_duration = "1m"} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "test" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } + ] +} +flapping_detect { + ban_time = "5m" + enable = false + max_count = 15 + window_time = "1m" +} +force_gc { + bytes = "16MB" + count = 16000 + enable = true +} +force_shutdown { + enable = true + max_heap_size = "32MB" + max_message_queue_len = 1000 +} +gateway { + stomp { + enable_stats = true + frame { + max_body_length = 8192 + max_headers = 10 + max_headers_length = 1024 + } + idle_timeout = "30s" + listeners { + tcp { + default { + bind = "61613" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + } +} +limiter { + bytes_in {burst = "0", rate = "infinity"} + client { + bytes_in { + capacity = "infinity" + divisible = true + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + internal { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "10" + low_watermark = "0" + max_retry_time = "10s" + rate = "100" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection {burst = "0", rate = "infinity"} + internal {burst = "0", rate = "infinity"} + message_in {burst = "100", rate = "1000/s"} + message_routing {burst = "0", rate = "infinity"} +} +listeners { + ssl { + default { + bind = "0.0.0.0:8883" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + } + } + tcp { + default { + acceptors = 32 + bind = "0.0.0.0:1883" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = 1000 + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 1024000 + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + ws { + default { + bind = "0.0.0.0:8083" + max_connections = 1024000 + websocket {mqtt_path = "/mqtt"} + } + } + wss { + default { + bind = "0.0.0.0:8084" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + websocket {mqtt_path = "/mqtt"} + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = false + idle_timeout = "15s" + ignore_loop_deliver = false + keepalive_backoff = 0.75 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 320 + max_mqueue_len = 1000 + max_packet_size = "10MB" + max_qos_allowed = 2 + max_subscriptions = "infinity" + max_topic_alias = 65535 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "disabled" + peer_cert_as_username = "disabled" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = "disabled" + session_expiry_interval = "2h" + shared_subscription = true + strict_mode = false + upgrade_qos = false + use_username_as_clientid = false + wildcard_subscription = true +} +overload_protection { + backoff_delay = 1 + backoff_gc = false + backoff_hibernation = true + backoff_new_conn = true + enable = false +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 0 + storage_type = "ram" + type = "built_in_database" + } + enable = true + flow_control { + batch_deliver_limiter { + capacity = "infinity" + client { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + initial = "0" + rate = "infinity" + } + batch_deliver_number = 0 + batch_read_number = 0 + } + max_payload_size = "10MB" + msg_clear_interval = "0s" + msg_expiry_interval = "0s" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + jq_implementation_module = "jq_nif" + rules { + "rule_kcz2" { + actions = ["webhook:test"] + description = "" + metadata {created_at = 1684227520811} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} +slow_subs { + enable = true + expire_interval = "300s" + stats_type = "whole" + threshold = "500ms" + top_k_num = 10 +} +stats {enable = true} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = true + client_subscribed = true + client_unsubscribed = true + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} diff --git a/scripts/conf-test/old-confs/ce-v5.0.21.conf b/scripts/conf-test/old-confs/ce-v5.0.21.conf new file mode 100644 index 000000000..c205a8ac2 --- /dev/null +++ b/scripts/conf-test/old-confs/ce-v5.0.21.conf @@ -0,0 +1,498 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +log { + file_handlers.default { + level = warning + file = "log/emqx.log" + } +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + +dashboard { + listeners.http { + bind = 18083 + } + default_username = "admin" + default_password = "public" +} + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + }, + { + algorithm = "hmac-based" + from = "password" + mechanism = "jwt" + secret = "emqxsecret" + "secret_base64_encoded" = false + use_jwks = false + }, + { + backend = "http" + body {password = "${password}", username = "${username}"} + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + mechanism = "password_based" + method = "post" + pool_size = 8 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } +] +authorization { + cache {enable = true} + deny_action = "ignore" + no_match = "allow" + sources = [ + { + cmd = "HGETALL mqtt_acl:${username}" + database = 0 + pool_size = 8 + redis_type = "single" + server = "127.0.0.1:6379" + ssl {enable = false, verify = "verify_peer"} + type = "redis" + }, + { + database = "mqtt_acl" + pool_size = 8 + query = "SELECT action, permission, topic FROM mqtt_acl where username = ${username}" + server = "127.0.0.1:5432" + ssl {enable = false, verify = "verify_peer"} + type = "postgresql" + username = "root" + }, + { + collection = "users" + database = "mqtt" + filter {username = "${username}"} + mongo_type = "single" + pool_size = 8 + server = "127.0.0.1:27017" + ssl {enable = false, verify = "verify_peer"} + topology {connect_timeout_ms = "20s"} + type = "mongodb" + }, + { + database = "mqtt_acl" + pool_size = 8 + query = "SELECT action, permission, topic FROM mqtt_acl where username = ${username}" + server = "127.0.0.1:3306" + ssl {enable = false, verify = "verify_peer"} + type = "mysql" + username = "root" + }, + { + body {username = "${username}"} + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + request_timeout = "30s" + ssl {enable = false, verify = "verify_peer"} + type = "http" + url = "http://127.0.0.1:8080" + }, + { + enable = true + path = "etc/acl.conf" + type = "file" + } + ] +} +bridges { + webhook { + test { + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + pool_type = "random" + request_timeout = "15s" + resource_opts { + async_inflight_window = 100 + auto_restart_interval = "60s" + health_check_interval = "15s" + max_queue_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080/test" + } + } +} +conn_congestion {enable_alarm = true, min_alarm_sustain_duration = "1m"} +delayed {enable = true, max_delayed_messages = 10} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "testx" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080/exhook" + } + ] +} +flapping_detect { + ban_time = "5m" + enable = false + max_count = 15 + window_time = "1m" +} +force_gc { + bytes = "16MB" + count = 16000 + enable = true +} +force_shutdown { + enable = true + max_heap_size = "32MB" + max_message_queue_len = 1000 +} +gateway { + coap { + connection_required = false + enable_stats = true + listeners { + udp { + default { + bind = "5683" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + notify_type = "qos" + publish_qos = "coap" + subscribe_qos = "coap" + } + exproto { + enable_stats = true + handler { + address = "http://127.0.0.1:9001" + ssl_options {enable = false} + } + idle_timeout = "30s" + listeners { + tcp { + default { + acceptors = 16 + bind = "7993" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + server { + bind = "127.0.0.1:9100" + ssl_options {verify = "verify_none"} + } + } + "lwm2m" { + auto_observe = true + enable_stats = true + idle_timeout = "30s" + lifetime_max = "86400s" + lifetime_min = "1s" + listeners { + udp { + default { + bind = "5783" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + qmode_time_window = "22s" + translators { + command {qos = 0, topic = "dn/#"} + notify {qos = 0, topic = "up/notify"} + register {qos = 0, topic = "up/resp"} + response {qos = 0, topic = "up/resp"} + update {qos = 0, topic = "up/update"} + } + update_msg_publish_condition = "contains_object_list" + xml_dir = "etc/lwm2m_xml/" + } + mqttsn { + broadcast = true + "enable_qos3" = true + enable_stats = true + gateway_id = 1 + idle_timeout = "30s" + listeners { + udp { + default { + bind = "1884" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + predefined = [] + } + stomp { + enable_stats = true + frame { + max_body_length = 8192 + max_headers = 10 + max_headers_length = 1024 + } + idle_timeout = "30s" + listeners { + tcp { + default { + bind = "61613" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + } +} +listeners { + ssl { + default { + bind = "0.0.0.0:8883" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + } + } + tcp { + default { + acceptors = 32 + bind = "0.0.0.0:1883" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = "1000" + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 1024000 + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + ws { + default { + bind = "0.0.0.0:8083" + max_connections = 1024000 + websocket {mqtt_path = "/mqtt"} + } + } + wss { + default { + bind = "0.0.0.0:8084" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + websocket {mqtt_path = "/mqtt"} + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = false + idle_timeout = "15s" + ignore_loop_deliver = false + keepalive_backoff = 0.75 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 320 + max_mqueue_len = 1000 + max_packet_size = "10MB" + max_qos_allowed = 2 + max_subscriptions = "infinity" + max_topic_alias = 65535 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "disabled" + peer_cert_as_username = "disabled" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = "disabled" + session_expiry_interval = "2h" + shared_subscription = true + strict_mode = false + upgrade_qos = false + use_username_as_clientid = false + wildcard_subscription = true +} +overload_protection { + backoff_delay = 1 + backoff_gc = false + backoff_hibernation = true + backoff_new_conn = true + enable = false +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 0 + storage_type = "ram" + type = "built_in_database" + } + enable = true + flow_control { + batch_deliver_limiter { + capacity = "infinity" + client { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + initial = "0" + rate = "infinity" + } + batch_deliver_number = 0 + batch_read_number = 0 + } + max_payload_size = "10MB" + msg_clear_interval = "0s" + msg_expiry_interval = "0s" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + jq_implementation_module = "jq_nif" + rules { + rule_plna { + actions = ["webhook:test"] + description = "" + metadata {created_at = 1684226199152} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} +slow_subs { + enable = true + expire_interval = "300s" + stats_type = "whole" + threshold = "500ms" + top_k_num = 10 +} +stats {enable = true} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = true + client_subscribed = true + client_unsubscribed = true + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} diff --git a/scripts/conf-test/old-confs/ce-v5.0.22.conf b/scripts/conf-test/old-confs/ce-v5.0.22.conf new file mode 100644 index 000000000..c5cd6e1c9 --- /dev/null +++ b/scripts/conf-test/old-confs/ce-v5.0.22.conf @@ -0,0 +1,502 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +log { + file_handlers.default { + level = warning + file = "log/emqx.log" + } +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + +dashboard { + listeners.http { + bind = 18083 + } + default_username = "admin" + default_password = "public" +} + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + } +] +authorization { + cache {enable = true} + deny_action = "ignore" + no_match = "allow" + sources = [ + { + body {username = "${username}"} + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + request_timeout = "30s" + ssl {enable = false, verify = "verify_peer"} + type = "http" + url = "http://127.0.0.1:8080" + }, + { + enable = true + path = "etc/acl.conf" + type = "file" + } + ] +} +bridges { + webhook { + test { + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + pool_type = "random" + request_timeout = "15s" + resource_opts { + async_inflight_window = 100 + auto_restart_interval = "60s" + health_check_interval = "15s" + max_queue_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080/webhook" + } + } +} +conn_congestion {enable_alarm = true, min_alarm_sustain_duration = "1m"} +delayed {enable = true, max_delayed_messages = 1000} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "test" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } + ] +} +flapping_detect { + ban_time = "5m" + enable = false + max_count = 15 + window_time = "1m" +} +force_gc { + bytes = "16MB" + count = 16000 + enable = true +} +force_shutdown { + enable = true + max_heap_size = "32MB" + max_message_queue_len = 1000 +} +gateway { + mqttsn { + broadcast = true + "enable_qos3" = true + enable_stats = true + gateway_id = 1 + idle_timeout = "30s" + listeners { + udp { + default { + bind = "1884" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + predefined = [] + } +} +limiter { + bytes_in {burst = "0", rate = "10MB/s"} + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "100KB" + low_watermark = "0" + max_retry_time = "10s" + rate = "2000" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + internal { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection {burst = "0", rate = "infinity"} + internal {burst = "1000", rate = "1000/s"} + message_in {burst = "0", rate = "infinity"} + message_routing {burst = "0", rate = "2000/m"} +} +listeners { + ssl { + default { + bind = "0.0.0.0:8883" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + } + } + tcp { + default {bind = "0.0.0.0:1883", max_connections = 1024000} + } + ws { + default { + acceptors = 32 + bind = "0.0.0.0:8083" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = "1000" + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 1024000 + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + websocket { + allow_origin_absence = true + check_origin_enable = false + check_origins = "http://localhost:18083, http://127.0.0.1:18083" + compress = false + deflate_opts { + client_context_takeover = "takeover" + client_max_window_bits = 15 + mem_level = 8 + server_context_takeover = "takeover" + server_max_window_bits = 15 + strategy = "default" + } + fail_if_no_subprotocol = true + idle_timeout = "7200s" + max_frame_size = "infinity" + mqtt_path = "/mqtt" + mqtt_piggyback = "multiple" + proxy_address_header = "x-forwarded-for" + proxy_port_header = "x-forwarded-port" + supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" + } + zone = "default" + } + } + wss { + default { + bind = "0.0.0.0:8084" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + websocket {mqtt_path = "/mqtt"} + } + } +} +log { + console_handler { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = 1000 + drop_mode_qlen = 3000 + enable = true + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = "unlimited" + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + file_handlers { + default { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = false + file = "log/emqx.log" + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = 100 + max_size = "50MB" + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + rotation {count = 10, enable = true} + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = true + idle_timeout = "15s" + ignore_loop_deliver = true + keepalive_backoff = 1 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 32 + max_mqueue_len = 1000 + max_packet_size = "10MB" + max_qos_allowed = 2 + max_subscriptions = 2000 + max_topic_alias = 65535 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "disabled" + peer_cert_as_username = "disabled" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = 20 + session_expiry_interval = "2h" + shared_subscription = true + strict_mode = false + upgrade_qos = true + use_username_as_clientid = true + wildcard_subscription = true +} +overload_protection { + backoff_delay = 1 + backoff_gc = false + backoff_hibernation = true + backoff_new_conn = true + enable = false +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 2000 + storage_type = "ram" + type = "built_in_database" + } + enable = true + flow_control { + batch_deliver_limiter { + capacity = "infinity" + client { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + initial = "0" + rate = "infinity" + } + batch_deliver_number = 0 + batch_read_number = 0 + } + max_payload_size = "1MB" + msg_clear_interval = "2000ms" + msg_expiry_interval = "1000ms" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + rules { + "rule_fhd9" { + actions = ["webhook:test"] + description = "" + metadata {created_at = 1684225481677} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} +stats {enable = true} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = true + client_subscribed = true + client_unsubscribed = true + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} +sysmon { + os { + cpu_check_interval = "60s" + cpu_high_watermark = "80%" + cpu_low_watermark = "60%" + mem_check_interval = "60s" + procmem_high_watermark = "5%" + sysmem_high_watermark = "70%" + } + top { + db_hostname = "" + db_name = "postgres" + db_password = "******" + db_port = 5432 + db_username = "system_monitor" + max_procs = 1000000 + num_items = 10 + sample_interval = "2s" + } + vm { + busy_dist_port = false + busy_port = false + large_heap = "disabled" + long_gc = "disabled" + long_schedule = "disabled" + process_check_interval = "30s" + process_high_watermark = "80%" + process_low_watermark = "60%" + } +} diff --git a/scripts/conf-test/old-confs/ce-v5.0.23.conf b/scripts/conf-test/old-confs/ce-v5.0.23.conf new file mode 100644 index 000000000..2bf5056d2 --- /dev/null +++ b/scripts/conf-test/old-confs/ce-v5.0.23.conf @@ -0,0 +1,280 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +log { + file_handlers.default { + level = warning + file = "log/emqx.log" + } +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + +dashboard { + listeners.http { + bind = 18083 + } + default_username = "admin" + default_password = "public" +} + +authentication = [ + { + algorithm = "hmac-based" + from = "password" + mechanism = "jwt" + secret = "emqxsecret" + "secret_base64_encoded" = false + use_jwks = false + verify_claims {} + } +] +authorization { + cache {enable = true} + deny_action = "ignore" + no_match = "allow" + sources = [ + {type = "built_in_database"}, + { + enable = true + path = "etc/acl.conf" + type = "file" + } + ] +} +bridges { + mqtt { + "to-public-broker" { + bridge_mode = false + clean_start = true + egress { + local {topic = "t/#"} + remote { + payload = "${payload}" + qos = 1 + retain = false + topic = "b/t" + } + } + enable = true + keepalive = "300s" + mode = "cluster_shareload" + proto_ver = "v4" + resource_opts { + auto_restart_interval = "60s" + health_check_interval = "15s" + inflight_window = 100 + max_buffer_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + retry_interval = "15s" + server = "broker.emqx.io:1883" + ssl {enable = false, verify = "verify_peer"} + } + } +} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "test" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080/exhook" + } + ] +} +gateway { + mqttsn { + broadcast = true + "enable_qos3" = true + enable_stats = true + gateway_id = 1 + idle_timeout = "30s" + listeners { + udp { + default { + bind = "1884" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + predefined = [] + } +} +listeners { + ssl { + default { + bind = "0.0.0.0:8883" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + } + } + tcp { + default { + acceptors = 32 + bind = "0.0.0.0:1883" + enable_authn = true + enabled = true + limiter { + bytes {rate = "infinity"} + client { + bytes {rate = "infinity"} + messages {rate = "infinity"} + } + messages {rate = "infinity"} + } + max_connections = 1024000 + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + ws { + default { + bind = "0.0.0.0:8083" + max_connections = 1024000 + websocket {mqtt_path = "/mqtt"} + } + } + wss { + default { + bind = "0.0.0.0:8084" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + websocket {mqtt_path = "/mqtt"} + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = true + idle_timeout = "15s" + ignore_loop_deliver = true + keepalive_backoff = 1 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 32 + max_mqueue_len = 1000 + max_packet_size = "1MB" + max_qos_allowed = 2 + max_subscriptions = 100 + max_topic_alias = 65535 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "cn" + peer_cert_as_username = "cn" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = 200 + session_expiry_interval = "2h" + shared_subscription = true + strict_mode = true + upgrade_qos = true + use_username_as_clientid = true + wildcard_subscription = true +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 1000 + storage_type = "ram" + type = "built_in_database" + } + enable = true + max_payload_size = "1MB" + msg_clear_interval = "20000ms" + msg_expiry_interval = "10000ms" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + rules { + rule_tcbl { + actions = ["mqtt:to-public-broker"] + description = "" + metadata {created_at = 1684223521242} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = true + client_subscribed = true + client_unsubscribed = true + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} +sysmon { + os { + cpu_check_interval = "60s" + cpu_high_watermark = "80%" + cpu_low_watermark = "60%" + mem_check_interval = "60s" + procmem_high_watermark = "5%" + sysmem_high_watermark = "70%" + } + top { + db_hostname = "" + db_name = "postgres" + db_password = "******" + db_port = 5432 + db_username = "system_monitor" + max_procs = 1000000 + num_items = 10 + sample_interval = "2s" + } + vm { + busy_dist_port = true + busy_port = true + large_heap = "32MB" + long_gc = "100ms" + long_schedule = "240ms" + process_check_interval = "30s" + process_high_watermark = "80%" + process_low_watermark = "60%" + } +} diff --git a/scripts/conf-test/old-confs/ce-v5.0.25.conf b/scripts/conf-test/old-confs/ce-v5.0.25.conf new file mode 100644 index 000000000..11c6e4dc0 --- /dev/null +++ b/scripts/conf-test/old-confs/ce-v5.0.25.conf @@ -0,0 +1,437 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + + +dashboard { + listeners.http { + bind = 18083 + } +} + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + } +] +authorization { + cache { + enable = true + max_size = 32 + ttl = "1m" + } + deny_action = "ignore" + no_match = "allow" + sources = [ + { + body {username = "${username}"} + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + request_timeout = "30s" + ssl {enable = false, verify = "verify_peer"} + type = "http" + url = "http://127.0.0.1:8080" + }, + { + enable = true + path = "${EMQX_ETC_DIR}/acl.conf" + type = "file" + } + ] +} +bridges { + webhook { + test { + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + pool_type = "random" + request_timeout = "15s" + resource_opts { + auto_restart_interval = "60s" + health_check_interval = "15s" + inflight_window = 100 + max_buffer_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080/webhook" + } + } +} +delayed {enable = false, max_delayed_messages = 2000} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "test" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } + ] +} +gateway { + "lwm2m" { + auto_observe = true + enable_stats = true + idle_timeout = "30s" + lifetime_max = "86400s" + lifetime_min = "1s" + listeners { + udp { + default { + bind = "5783" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + qmode_time_window = "22s" + translators { + command {qos = 0, topic = "dn/#"} + notify {qos = 0, topic = "up/notify"} + register {qos = 0, topic = "up/resp"} + response {qos = 0, topic = "up/resp"} + update {qos = 0, topic = "up/update"} + } + update_msg_publish_condition = "contains_object_list" + xml_dir = "etc/lwm2m_xml/" + } +} +limiter {max_conn_rate = "2000/s"} +listeners { + ssl { + default { + acceptors = 16 + access_rules = ["allow all"] + bind = "0.0.0.0:8883" + enable_authn = true + enabled = true + max_connections = 5000000 + mountpoint = "" + proxy_protocol = false + proxy_protocol_timeout = "3s" + ssl_options { + cacertfile = "${EMQX_ETC_DIR}/certs/cacert.pem" + certfile = "${EMQX_ETC_DIR}/certs/cert.pem" + ciphers = [] + client_renegotiation = true + depth = 10 + enable_crl_check = false + fail_if_no_peer_cert = false + gc_after_handshake = false + handshake_timeout = "15s" + hibernate_after = "5s" + honor_cipher_order = true + keyfile = "${EMQX_ETC_DIR}/certs/key.pem" + ocsp { + enable_ocsp_stapling = false + refresh_http_timeout = "15s" + refresh_interval = "5m" + } + reuse_sessions = true + secure_renegotiate = true + verify = "verify_none" + versions = ["tlsv1.3", "tlsv1.2"] + } + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + tcp { + default { + acceptors = 16 + access_rules = ["allow all"] + bind = "0.0.0.0:1883" + enable_authn = true + enabled = true + max_connections = 5000000 + mountpoint = "" + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + ws { + default { + acceptors = 32 + access_rules = ["allow all"] + bind = "0.0.0.0:8083" + enable_authn = true + enabled = true + max_connections = 5000000 + mountpoint = "" + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "40KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + websocket { + allow_origin_absence = true + check_origin_enable = false + check_origins = "http://localhost:18083, http://127.0.0.1:18083" + compress = false + deflate_opts { + client_context_takeover = "takeover" + client_max_window_bits = 15 + mem_level = 8 + server_context_takeover = "takeover" + server_max_window_bits = 15 + strategy = "default" + } + fail_if_no_subprotocol = true + idle_timeout = "7200s" + max_frame_size = "infinity" + mqtt_path = "/mqtt" + mqtt_piggyback = "multiple" + proxy_address_header = "x-forwarded-for" + proxy_port_header = "x-forwarded-port" + supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" + } + zone = "default" + } + } + wss { + default { + acceptors = 16 + access_rules = ["allow all"] + bind = "0.0.0.0:8084" + enable_authn = true + enabled = true + max_connections = 5000000 + mountpoint = "" + proxy_protocol = false + proxy_protocol_timeout = "3s" + ssl_options { + cacertfile = "${EMQX_ETC_DIR}/certs/cacert.pem" + certfile = "${EMQX_ETC_DIR}/certs/cert.pem" + ciphers = [] + client_renegotiation = true + depth = 10 + fail_if_no_peer_cert = false + handshake_timeout = "15s" + hibernate_after = "5s" + honor_cipher_order = true + keyfile = "${EMQX_ETC_DIR}/certs/key.pem" + reuse_sessions = true + secure_renegotiate = true + verify = "verify_none" + versions = ["tlsv1.3", "tlsv1.2"] + } + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + websocket { + allow_origin_absence = true + check_origin_enable = false + check_origins = "http://localhost:18083, http://127.0.0.1:18083" + compress = false + deflate_opts { + client_context_takeover = "takeover" + client_max_window_bits = 15 + mem_level = 8 + server_context_takeover = "takeover" + server_max_window_bits = 15 + strategy = "default" + } + fail_if_no_subprotocol = true + idle_timeout = "7200s" + max_frame_size = "infinity" + mqtt_path = "/mqtt" + mqtt_piggyback = "multiple" + proxy_address_header = "x-forwarded-for" + proxy_port_header = "x-forwarded-port" + supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" + } + zone = "default" + } + } +} +log { + console_handler { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = 10000 + drop_mode_qlen = 3000 + enable = true + flush_qlen = 8000 + formatter = "text" + level = "info" + max_depth = 100 + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + file_handlers { + default { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = false + file = "${EMQX_LOG_DIR}/emqx.log" + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = 100 + max_size = "50MB" + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + rotation {count = 10, enable = true} + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = true + idle_timeout = "15s" + ignore_loop_deliver = true + keepalive_backoff = 1 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 32 + max_mqueue_len = 1000 + max_packet_size = "10MB" + max_qos_allowed = 2 + max_subscriptions = 15 + max_topic_alias = 65535 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "cn" + peer_cert_as_username = "cn" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = 15 + session_expiry_interval = "2h" + shared_subscription = true + strict_mode = true + upgrade_qos = true + use_username_as_clientid = true + wildcard_subscription = true +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 120 + storage_type = "ram" + type = "built_in_database" + } + enable = true + max_payload_size = "1MB" + msg_clear_interval = "2000ms" + msg_expiry_interval = "10000ms" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + rules { + "rule_t7cl" { + actions = ["webhook:test"] + description = "" + metadata {created_at = 1684222459298} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} +slow_subs { + enable = true + expire_interval = "3000s" + stats_type = "internal" + threshold = "5000ms" + top_k_num = 1000 +} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = true + client_subscribed = true + client_unsubscribed = true + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} From 73f1421d1f96be78f987832906b0b8fd071a2e94 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 17 May 2023 17:51:22 +0800 Subject: [PATCH 161/197] ci: prefixed config files wit v/e --- scripts/conf-test/old-confs/{ee-v5.0.0.conf => e5.0.0.conf} | 0 scripts/conf-test/old-confs/{ee-v5.0.1.conf => e5.0.1.conf} | 0 scripts/conf-test/old-confs/{ee-v5.0.2.conf => e5.0.2.conf} | 0 scripts/conf-test/old-confs/{ee-v5.0.3.conf => e5.0.3.conf} | 0 .../conf-test/old-confs/{ce-v5.0.20.conf => v5.0.20.conf} | 0 .../conf-test/old-confs/{ce-v5.0.21.conf => v5.0.21.conf} | 0 .../conf-test/old-confs/{ce-v5.0.22.conf => v5.0.22.conf} | 0 .../conf-test/old-confs/{ce-v5.0.23.conf => v5.0.23.conf} | 0 .../conf-test/old-confs/{ce-v5.0.24.conf => v5.0.24.conf} | 0 .../conf-test/old-confs/{ce-v5.0.25.conf => v5.0.25.conf} | 0 scripts/conf-test/run.sh | 6 +++--- 11 files changed, 3 insertions(+), 3 deletions(-) rename scripts/conf-test/old-confs/{ee-v5.0.0.conf => e5.0.0.conf} (100%) rename scripts/conf-test/old-confs/{ee-v5.0.1.conf => e5.0.1.conf} (100%) rename scripts/conf-test/old-confs/{ee-v5.0.2.conf => e5.0.2.conf} (100%) rename scripts/conf-test/old-confs/{ee-v5.0.3.conf => e5.0.3.conf} (100%) rename scripts/conf-test/old-confs/{ce-v5.0.20.conf => v5.0.20.conf} (100%) rename scripts/conf-test/old-confs/{ce-v5.0.21.conf => v5.0.21.conf} (100%) rename scripts/conf-test/old-confs/{ce-v5.0.22.conf => v5.0.22.conf} (100%) rename scripts/conf-test/old-confs/{ce-v5.0.23.conf => v5.0.23.conf} (100%) rename scripts/conf-test/old-confs/{ce-v5.0.24.conf => v5.0.24.conf} (100%) rename scripts/conf-test/old-confs/{ce-v5.0.25.conf => v5.0.25.conf} (100%) diff --git a/scripts/conf-test/old-confs/ee-v5.0.0.conf b/scripts/conf-test/old-confs/e5.0.0.conf similarity index 100% rename from scripts/conf-test/old-confs/ee-v5.0.0.conf rename to scripts/conf-test/old-confs/e5.0.0.conf diff --git a/scripts/conf-test/old-confs/ee-v5.0.1.conf b/scripts/conf-test/old-confs/e5.0.1.conf similarity index 100% rename from scripts/conf-test/old-confs/ee-v5.0.1.conf rename to scripts/conf-test/old-confs/e5.0.1.conf diff --git a/scripts/conf-test/old-confs/ee-v5.0.2.conf b/scripts/conf-test/old-confs/e5.0.2.conf similarity index 100% rename from scripts/conf-test/old-confs/ee-v5.0.2.conf rename to scripts/conf-test/old-confs/e5.0.2.conf diff --git a/scripts/conf-test/old-confs/ee-v5.0.3.conf b/scripts/conf-test/old-confs/e5.0.3.conf similarity index 100% rename from scripts/conf-test/old-confs/ee-v5.0.3.conf rename to scripts/conf-test/old-confs/e5.0.3.conf diff --git a/scripts/conf-test/old-confs/ce-v5.0.20.conf b/scripts/conf-test/old-confs/v5.0.20.conf similarity index 100% rename from scripts/conf-test/old-confs/ce-v5.0.20.conf rename to scripts/conf-test/old-confs/v5.0.20.conf diff --git a/scripts/conf-test/old-confs/ce-v5.0.21.conf b/scripts/conf-test/old-confs/v5.0.21.conf similarity index 100% rename from scripts/conf-test/old-confs/ce-v5.0.21.conf rename to scripts/conf-test/old-confs/v5.0.21.conf diff --git a/scripts/conf-test/old-confs/ce-v5.0.22.conf b/scripts/conf-test/old-confs/v5.0.22.conf similarity index 100% rename from scripts/conf-test/old-confs/ce-v5.0.22.conf rename to scripts/conf-test/old-confs/v5.0.22.conf diff --git a/scripts/conf-test/old-confs/ce-v5.0.23.conf b/scripts/conf-test/old-confs/v5.0.23.conf similarity index 100% rename from scripts/conf-test/old-confs/ce-v5.0.23.conf rename to scripts/conf-test/old-confs/v5.0.23.conf diff --git a/scripts/conf-test/old-confs/ce-v5.0.24.conf b/scripts/conf-test/old-confs/v5.0.24.conf similarity index 100% rename from scripts/conf-test/old-confs/ce-v5.0.24.conf rename to scripts/conf-test/old-confs/v5.0.24.conf diff --git a/scripts/conf-test/old-confs/ce-v5.0.25.conf b/scripts/conf-test/old-confs/v5.0.25.conf similarity index 100% rename from scripts/conf-test/old-confs/ce-v5.0.25.conf rename to scripts/conf-test/old-confs/v5.0.25.conf diff --git a/scripts/conf-test/run.sh b/scripts/conf-test/run.sh index 1fa3b3951..4c2f18615 100755 --- a/scripts/conf-test/run.sh +++ b/scripts/conf-test/run.sh @@ -16,12 +16,12 @@ start_emqx_with_conf() { MINOR_VSN=$(./pkg-vsn.sh "$PROFILE" | cut -d. -f1,2) if [ "$PROFILE" = "emqx" ]; then - EDITION="ce" + PREFIX="v" else - EDITION="ee" + PREFIX="e" fi -FILES=$(ls ./scripts/conf-test/old-confs/$EDITION-v"$MINOR_VSN"*) +FILES=$(ls ./scripts/conf-test/old-confs/$PREFIX"$MINOR_VSN"*) cp "$EMQX_ROOT"/etc/emqx.conf "$EMQX_ROOT"/etc/emqx.conf.bak cleanup() { From 060efd696445aa87db42179a80c9bf65fae33386 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 16 May 2023 17:47:34 -0300 Subject: [PATCH 162/197] chore: bump gproc -> 0.9.0.1 (r5.0) Includes this fix: https://github.com/uwiger/gproc/pull/193 --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 18119607e..425c49fb3 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -24,7 +24,7 @@ {deps, [ {emqx_utils, {path, "../emqx_utils"}}, {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}}, - {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}, + {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.1"}}}, diff --git a/mix.exs b/mix.exs index 0284ed54a..3e6098258 100644 --- a/mix.exs +++ b/mix.exs @@ -50,7 +50,7 @@ defmodule EMQXUmbrella.MixProject do {:covertool, github: "zmstone/covertool", tag: "2.0.4.1", override: true}, {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true}, {:ehttpc, github: "emqx/ehttpc", tag: "0.4.8", override: true}, - {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, + {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true}, {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.6", override: true}, diff --git a/rebar.config b/rebar.config index 55a4583de..b6f5a479f 100644 --- a/rebar.config +++ b/rebar.config @@ -57,7 +57,7 @@ , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}} , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}} , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.8"}}} - , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} + , {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}} From 38ef99caf59f3584e63c4feac242e81da15622c2 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 17 May 2023 11:49:59 -0300 Subject: [PATCH 163/197] chore(dashboard): bump dashboard -> e1.0.7-beta.3 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1ad4421aa..785458601 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ endif # Dashbord version # from https://github.com/emqx/emqx-dashboard5 export EMQX_DASHBOARD_VERSION ?= v1.2.4-1 -export EMQX_EE_DASHBOARD_VERSION ?= e1.0.6 +export EMQX_EE_DASHBOARD_VERSION ?= e1.0.7-beta.3 # `:=` should be used here, otherwise the `$(shell ...)` will be executed every time when the variable is used # In make 4.4+, for backward-compatibility the value from the original environment is used. From dcccc0910aad87d289dff5bd2a2aa6c7625a2b36 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 17 May 2023 11:44:19 -0300 Subject: [PATCH 164/197] fix(pulsar): mark whole auth struct as sensitive (r5.0) Fixes https://emqx.atlassian.net/browse/EMQX-9900 I tried to patch hocon itself to filter the sensitive data, but the way it's currently structured doesn't seem to keep that field metadata. So, for now, we can just mark the whole auth union as sensitive. --- apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl index 5a87d8a0c..56c20130c 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl @@ -46,7 +46,14 @@ fields(config) -> )}, {authentication, mk(hoconsc:union([none, ref(auth_basic), ref(auth_token)]), #{ - default => none, desc => ?DESC("authentication") + default => none, + %% must mark this whole union as sensitive because + %% hocon ignores the `sensitive' metadata in struct + %% fields... Also, when trying to type check a struct + %% that doesn't match the intended type, it won't have + %% sensitivity information from sibling types. + sensitive => true, + desc => ?DESC("authentication") })} ] ++ emqx_connector_schema_lib:ssl_fields(); fields(producer_opts) -> From 659cf64ad7f40e9f436b7dd234a5c55b124ea098 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 17 May 2023 17:56:53 -0300 Subject: [PATCH 165/197] feat(pulsar): use an union member selector for better error messages --- .../src/emqx_bridge_pulsar.erl | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl index 56c20130c..7d1b20d24 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl @@ -45,16 +45,19 @@ fields(config) -> } )}, {authentication, - mk(hoconsc:union([none, ref(auth_basic), ref(auth_token)]), #{ - default => none, - %% must mark this whole union as sensitive because - %% hocon ignores the `sensitive' metadata in struct - %% fields... Also, when trying to type check a struct - %% that doesn't match the intended type, it won't have - %% sensitivity information from sibling types. - sensitive => true, - desc => ?DESC("authentication") - })} + mk( + hoconsc:union(fun auth_union_member_selector/1), + #{ + default => none, + %% must mark this whole union as sensitive because + %% hocon ignores the `sensitive' metadata in struct + %% fields... Also, when trying to type check a struct + %% that doesn't match the intended type, it won't have + %% sensitivity information from sibling types. + sensitive => true, + desc => ?DESC("authentication") + } + )} ] ++ emqx_connector_schema_lib:ssl_fields(); fields(producer_opts) -> [ @@ -233,3 +236,21 @@ override_default(OriginalFn, NewDefault) -> (default) -> NewDefault; (Field) -> OriginalFn(Field) end. + +auth_union_member_selector(all_union_members) -> + [none, ref(auth_basic), ref(auth_token)]; +auth_union_member_selector({value, V}) -> + case V of + #{<<"password">> := _} -> + [ref(auth_basic)]; + #{<<"jwt">> := _} -> + [ref(auth_token)]; + <<"none">> -> + [none]; + _ -> + Expected = "none | basic | token", + throw(#{ + field_name => authentication, + expected => Expected + }) + end. From 5d5c16a56d7e62a356b10cf855d09a4ab26a2822 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 17 May 2023 17:57:29 -0300 Subject: [PATCH 166/197] feat(bridges): use union member selector function for better error messages --- .../src/schema/emqx_bridge_schema.erl | 34 ++++++++++- .../emqx_ee_bridge/src/emqx_ee_bridge.app.src | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 61 +++++++++++-------- 3 files changed, 66 insertions(+), 31 deletions(-) diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index b590f0cd4..90e62764a 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -51,11 +51,39 @@ post_request() -> api_schema(Method) -> Broker = [ - ref(Mod, Method) - || Mod <- [emqx_bridge_webhook_schema, emqx_bridge_mqtt_schema] + {Type, ref(Mod, Method)} + || {Type, Mod} <- [ + {<<"webhook">>, emqx_bridge_webhook_schema}, + {<<"mqtt">>, emqx_bridge_mqtt_schema} + ] ], EE = ee_api_schemas(Method), - hoconsc:union(Broker ++ EE). + hoconsc:union(bridge_api_union(Broker ++ EE)). + +bridge_api_union(Refs) -> + Index = maps:from_list(Refs), + fun + (all_union_members) -> + maps:values(Index); + ({value, V}) -> + case V of + #{<<"type">> := T} -> + case maps:get(T, Index, undefined) of + undefined -> + throw(#{ + field_name => type, + reason => <<"unknown bridge type">> + }); + Ref -> + [Ref] + end; + _ -> + throw(#{ + field_name => type, + reason => <<"unknown bridge type">> + }) + end + end. -if(?EMQX_RELEASE_EDITION == ee). ee_api_schemas(Method) -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index 6e2dbcbce..bff42dd98 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.12"}, + {vsn, "0.1.13"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 17ffe9b9b..804e7d814 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -14,33 +14,37 @@ api_schemas(Method) -> [ - ref(emqx_bridge_gcp_pubsub, Method), - ref(emqx_bridge_kafka, Method ++ "_consumer"), - ref(emqx_bridge_kafka, Method ++ "_producer"), - ref(emqx_bridge_cassandra, Method), - ref(emqx_ee_bridge_mysql, Method), - ref(emqx_bridge_pgsql, Method), - ref(emqx_ee_bridge_mongodb, Method ++ "_rs"), - ref(emqx_ee_bridge_mongodb, Method ++ "_sharded"), - ref(emqx_ee_bridge_mongodb, Method ++ "_single"), - ref(emqx_ee_bridge_hstreamdb, Method), - ref(emqx_bridge_influxdb, Method ++ "_api_v1"), - ref(emqx_bridge_influxdb, Method ++ "_api_v2"), - ref(emqx_ee_bridge_redis, Method ++ "_single"), - ref(emqx_ee_bridge_redis, Method ++ "_sentinel"), - ref(emqx_ee_bridge_redis, Method ++ "_cluster"), - ref(emqx_bridge_timescale, Method), - ref(emqx_bridge_matrix, Method), - ref(emqx_bridge_tdengine, Method), - ref(emqx_ee_bridge_clickhouse, Method), - ref(emqx_bridge_dynamo, Method), - ref(emqx_bridge_rocketmq, Method), - ref(emqx_bridge_sqlserver, Method), - ref(emqx_bridge_opents, Method), - ref(emqx_bridge_pulsar, Method ++ "_producer"), - ref(emqx_bridge_oracle, Method), - ref(emqx_bridge_iotdb, Method), - ref(emqx_bridge_rabbitmq, Method) + %% We need to map the `type' field of a request (binary) to a + %% bridge schema module. + api_ref(emqx_bridge_gcp_pubsub, <<"gcp_pubsub">>, Method), + api_ref(emqx_bridge_kafka, <<"kafka_consumer">>, Method ++ "_consumer"), + %% TODO: rename this to `kafka_producer' after alias support is added + %% to hocon; keeping this as just `kafka' for backwards compatibility. + api_ref(emqx_bridge_kafka, <<"kafka">>, Method ++ "_producer"), + api_ref(emqx_bridge_cassandra, <<"cassandra">>, Method), + api_ref(emqx_ee_bridge_mysql, <<"mysql">>, Method), + api_ref(emqx_bridge_pgsql, <<"pgsql">>, Method), + api_ref(emqx_ee_bridge_mongodb, <<"mongodb_rs">>, Method ++ "_rs"), + api_ref(emqx_ee_bridge_mongodb, <<"mongodb_sharded">>, Method ++ "_sharded"), + api_ref(emqx_ee_bridge_mongodb, <<"mongodb_single">>, Method ++ "_single"), + api_ref(emqx_ee_bridge_hstreamdb, <<"hstreamdb">>, Method), + api_ref(emqx_bridge_influxdb, <<"influxdb_api_v1">>, Method ++ "_api_v1"), + api_ref(emqx_bridge_influxdb, <<"influxdb_api_v2">>, Method ++ "_api_v2"), + api_ref(emqx_ee_bridge_redis, <<"redis_single">>, Method ++ "_single"), + api_ref(emqx_ee_bridge_redis, <<"redis_sentinel">>, Method ++ "_sentinel"), + api_ref(emqx_ee_bridge_redis, <<"redis_cluster">>, Method ++ "_cluster"), + api_ref(emqx_bridge_timescale, <<"timescale">>, Method), + api_ref(emqx_bridge_matrix, <<"matrix">>, Method), + api_ref(emqx_bridge_tdengine, <<"tdengine">>, Method), + api_ref(emqx_ee_bridge_clickhouse, <<"clickhouse">>, Method), + api_ref(emqx_bridge_dynamo, <<"dynamo">>, Method), + api_ref(emqx_bridge_rocketmq, <<"rocketmq">>, Method), + api_ref(emqx_bridge_sqlserver, <<"sqlserver">>, Method), + api_ref(emqx_bridge_opents, <<"opents">>, Method), + api_ref(emqx_bridge_pulsar, <<"pulsar_producer">>, Method ++ "_producer"), + api_ref(emqx_bridge_oracle, <<"oracle">>, Method), + api_ref(emqx_bridge_iotdb, <<"iotdb">>, Method), + api_ref(emqx_bridge_rabbitmq, <<"rabbitmq">>, Method) ]. schema_modules() -> @@ -338,3 +342,6 @@ rabbitmq_structs() -> } )} ]. + +api_ref(Module, Type, Method) -> + {Type, ref(Module, Method)}. From 6ff77b221b5c5abdb5ea6c67511ccd1faf160eb9 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 18 May 2023 14:30:51 +0800 Subject: [PATCH 167/197] fix(tdengine): add supports for the `automatically create` feature in the SQL template --- .../src/emqx_bridge_tdengine_connector.erl | 138 ++++++++++++------ .../src/emqx_plugin_libs_rule.erl | 12 +- 2 files changed, 103 insertions(+), 47 deletions(-) diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl index 46a70e8b6..876743be5 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl @@ -25,7 +25,7 @@ on_get_status/2 ]). --export([connect/1, do_get_status/1, execute/3]). +-export([connect/1, do_get_status/1, execute/3, do_batch_insert/4]). -import(hoconsc, [mk/2, enum/1, ref/2]). @@ -124,32 +124,36 @@ on_stop(InstanceId, #{pool_name := PoolName}) -> on_query(InstanceId, {query, SQL}, State) -> do_query(InstanceId, SQL, State); -on_query(InstanceId, Request, State) -> - %% because the `emqx-tdengine` client only supports a single SQL cmd - %% so the `on_query` and `on_batch_query` have the same process, that is: - %% we need to collect all data into one SQL cmd and then call the insert API - on_batch_query(InstanceId, [Request], State). - -on_batch_query( - InstanceId, - BatchReq, - #{batch_inserts := Inserts, batch_params_tokens := ParamsTokens} = State -) -> - case hd(BatchReq) of - {Key, _} -> - case maps:get(Key, Inserts, undefined) of - undefined -> - {error, {unrecoverable_error, batch_prepare_not_implemented}}; - InsertSQL -> - Tokens = maps:get(Key, ParamsTokens), - do_batch_insert(InstanceId, BatchReq, InsertSQL, Tokens, State) - end; - Request -> - LogMeta = #{connector => InstanceId, first_request => Request, state => State}, - ?SLOG(error, LogMeta#{msg => "invalid request"}), +on_query(InstanceId, {Key, Data}, #{insert_tokens := InsertTksMap} = State) -> + case maps:find(Key, InsertTksMap) of + {ok, Tokens} -> + SQL = emqx_plugin_libs_rule:proc_sql_param_str(Tokens, Data), + do_query(InstanceId, SQL, State); + _ -> {error, {unrecoverable_error, invalid_request}} end. +%% aggregate the batch queries to one SQL is a heavy job, we should put it in the worker process +on_batch_query( + InstanceId, + [{Key, _} | _] = BatchReq, + #{batch_tokens := BatchTksMap, query_opts := Opts} = State +) -> + case maps:find(Key, BatchTksMap) of + {ok, Tokens} -> + do_query_job( + InstanceId, + {?MODULE, do_batch_insert, [Tokens, BatchReq, Opts]}, + State + ); + _ -> + {error, {unrecoverable_error, batch_prepare_not_implemented}} + end; +on_batch_query(InstanceId, BatchReq, State) -> + LogMeta = #{connector => InstanceId, request => BatchReq, state => State}, + ?SLOG(error, LogMeta#{msg => "invalid request"}), + {error, {unrecoverable_error, invalid_request}}. + on_get_status(_InstanceId, #{pool_name := PoolName}) -> Health = emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1), status_result(Health). @@ -167,17 +171,16 @@ status_result(_Status = false) -> connecting. %% Helper fns %%======================================================================================== -do_batch_insert(InstanceId, BatchReqs, InsertPart, Tokens, State) -> - SQL = emqx_plugin_libs_rule:proc_batch_sql(BatchReqs, InsertPart, Tokens), - do_query(InstanceId, SQL, State). +do_query(InstanceId, Query, #{query_opts := Opts} = State) -> + do_query_job(InstanceId, {?MODULE, execute, [Query, Opts]}, State). -do_query(InstanceId, Query, #{pool_name := PoolName, query_opts := Opts} = State) -> +do_query_job(InstanceId, Job, #{pool_name := PoolName} = State) -> ?TRACE( "QUERY", "tdengine_connector_received", - #{connector => InstanceId, query => Query, state => State} + #{connector => InstanceId, job => Job, state => State} ), - Result = ecpool:pick_and_do(PoolName, {?MODULE, execute, [Query, Opts]}, no_handover), + Result = ecpool:pick_and_do(PoolName, Job, no_handover), case Result of {error, Reason} -> @@ -188,7 +191,7 @@ do_query(InstanceId, Query, #{pool_name := PoolName, query_opts := Opts} = State ?SLOG(error, #{ msg => "tdengine_connector_do_query_failed", connector => InstanceId, - query => Query, + job => Job, reason => Reason }), Result; @@ -203,6 +206,37 @@ do_query(InstanceId, Query, #{pool_name := PoolName, query_opts := Opts} = State execute(Conn, Query, Opts) -> tdengine:insert(Conn, Query, Opts). +do_batch_insert(Conn, Tokens, BatchReqs, Opts) -> + Queries = aggregate_query(Tokens, BatchReqs), + SQL = lists:foldl( + fun({InsertPart, Values}, Acc) -> + lists:foldl( + fun(ValuePart, IAcc) -> + <> + end, + <>, + Values + ) + end, + <<"INSERT INTO">>, + Queries + ), + execute(Conn, SQL, Opts). + +aggregate_query({InsertPartTks, ParamsPartTks}, BatchReqs) -> + maps:to_list( + lists:foldl( + fun({_, Data}, Acc) -> + InsertPart = emqx_plugin_libs_rule:proc_sql_param_str(InsertPartTks, Data), + ParamsPart = emqx_plugin_libs_rule:proc_sql_param_str(ParamsPartTks, Data), + Values = maps:get(InsertPart, Acc, []), + maps:put(InsertPart, [ParamsPart | Values], Acc) + end, + #{}, + BatchReqs + ) + ). + connect(Opts) -> tdengine:start_link(Opts). @@ -218,32 +252,46 @@ parse_prepare_sql(Config) -> parse_batch_prepare_sql(maps:to_list(SQL), #{}, #{}). -parse_batch_prepare_sql([{Key, H} | T], BatchInserts, BatchTks) -> +parse_batch_prepare_sql([{Key, H} | T], InsertTksMap, BatchTksMap) -> case emqx_plugin_libs_rule:detect_sql_type(H) of {ok, select} -> - parse_batch_prepare_sql(T, BatchInserts, BatchTks); + parse_batch_prepare_sql(T, InsertTksMap, BatchTksMap); {ok, insert} -> - case emqx_plugin_libs_rule:split_insert_sql(H) of - {ok, {InsertSQL, Params}} -> - ParamsTks = emqx_plugin_libs_rule:preproc_tmpl(Params), + InsertTks = emqx_plugin_libs_rule:preproc_tmpl(H), + H1 = string:trim(H, trailing, ";"), + case split_insert_sql(H1) of + [_InsertStr, InsertPart, _ValuesStr, ParamsPart] -> + InsertPartTks = emqx_plugin_libs_rule:preproc_tmpl(InsertPart), + ParamsPartTks = emqx_plugin_libs_rule:preproc_tmpl(ParamsPart), parse_batch_prepare_sql( T, - BatchInserts#{Key => InsertSQL}, - BatchTks#{Key => ParamsTks} + InsertTksMap#{Key => InsertTks}, + BatchTksMap#{Key => {InsertPartTks, ParamsPartTks}} ); - {error, Reason} -> - ?SLOG(error, #{msg => "split sql failed", sql => H, reason => Reason}), - parse_batch_prepare_sql(T, BatchInserts, BatchTks) + _ -> + ?SLOG(error, #{msg => "split sql failed", sql => H}), + parse_batch_prepare_sql(T, InsertTksMap, BatchTksMap) end; {error, Reason} -> ?SLOG(error, #{msg => "detect sql type failed", sql => H, reason => Reason}), - parse_batch_prepare_sql(T, BatchInserts, BatchTks) + parse_batch_prepare_sql(T, InsertTksMap, BatchTksMap) end; -parse_batch_prepare_sql([], BatchInserts, BatchTks) -> +parse_batch_prepare_sql([], InsertTksMap, BatchTksMap) -> #{ - batch_inserts => BatchInserts, - batch_params_tokens => BatchTks + insert_tokens => InsertTksMap, + batch_tokens => BatchTksMap }. to_bin(List) when is_list(List) -> unicode:characters_to_binary(List, utf8). + +split_insert_sql(SQL0) -> + SQL = emqx_plugin_libs_rule:formalize_sql(SQL0), + lists:foldr( + fun + (<<>>, Acc) -> Acc; + (E, Acc) -> [string:trim(E) | Acc] + end, + [], + re:split(SQL, "(?i)(insert into)|(?i)(values)") + ). diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl index 9a4c01a2b..3bfac1ec4 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl @@ -32,7 +32,8 @@ proc_cql_param_str/2, split_insert_sql/1, detect_sql_type/1, - proc_batch_sql/3 + proc_batch_sql/3, + formalize_sql/1 ]). %% type converting @@ -126,7 +127,8 @@ proc_cql_param_str(Tokens, Data) -> -spec split_insert_sql(binary()) -> {ok, {InsertSQL, Params}} | {error, atom()} when InsertSQL :: binary(), Params :: binary(). -split_insert_sql(SQL) -> +split_insert_sql(SQL0) -> + SQL = formalize_sql(SQL0), case re:split(SQL, "((?i)values)", [{return, binary}]) of [Part1, _, Part3] -> case string:trim(Part1, leading) of @@ -173,6 +175,12 @@ proc_batch_sql(BatchReqs, InsertPart, Tokens) -> ), <>. +formalize_sql(Input) -> + %% 1. replace all whitespaces like '\r' '\n' or spaces to a single space char. + SQL = re:replace(Input, "\\s+", " ", [global, {return, binary}]), + %% 2. trims the result + string:trim(SQL). + unsafe_atom_key(Key) when is_atom(Key) -> Key; unsafe_atom_key(Key) when is_binary(Key) -> From 142125b9e43b6c8ebaecae201bcf1e773936a76c Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 18 May 2023 17:01:27 +0800 Subject: [PATCH 168/197] test(tdengine): add test cases to cover the super table feature --- .../test/emqx_bridge_tdengine_SUITE.erl | 204 ++++++++++++++++-- 1 file changed, 187 insertions(+), 17 deletions(-) diff --git a/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl b/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl index 1b8db1aaa..00a887852 100644 --- a/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl +++ b/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl @@ -24,9 +24,21 @@ ");" ). -define(SQL_DROP_TABLE, "DROP TABLE t_mqtt_msg"). --define(SQL_DELETE, "DELETE from t_mqtt_msg"). +-define(SQL_DROP_STABLE, "DROP STABLE s_tab"). +-define(SQL_DELETE, "DELETE FROM t_mqtt_msg"). -define(SQL_SELECT, "SELECT payload FROM t_mqtt_msg"). +-define(AUTO_CREATE_BRIDGE, + "insert into ${clientid} USING s_tab TAGS (${clientid}) values (${timestamp}, ${payload})" +). + +-define(SQL_CREATE_STABLE, + "CREATE STABLE s_tab (\n" + " ts timestamp,\n" + " payload BINARY(1024)\n" + ") TAGS (clientid BINARY(128));" +). + % DB defaults -define(TD_DATABASE, "mqtt"). -define(TD_USERNAME, "root"). @@ -53,12 +65,13 @@ all() -> groups() -> TCs = emqx_common_test_helpers:all(?MODULE), NonBatchCases = [t_write_timeout], + MustBatchCases = [t_batch_insert, t_auto_create_batch_insert], BatchingGroups = [{group, with_batch}, {group, without_batch}], [ {async, BatchingGroups}, {sync, BatchingGroups}, {with_batch, TCs -- NonBatchCases}, - {without_batch, TCs} + {without_batch, TCs -- MustBatchCases} ]. init_per_group(async, Config) -> @@ -117,7 +130,8 @@ common_init(ConfigT) -> Config0 = [ {td_host, Host}, {td_port, Port}, - {proxy_name, "tdengine_restful"} + {proxy_name, "tdengine_restful"}, + {template, ?SQL_BRIDGE} | ConfigT ], @@ -165,6 +179,7 @@ tdengine_config(BridgeType, Config) -> false -> 1 end, QueryMode = ?config(query_mode, Config), + Template = ?config(template, Config), ConfigString = io_lib:format( "bridges.~s.~s {\n" @@ -187,7 +202,7 @@ tdengine_config(BridgeType, Config) -> ?TD_DATABASE, ?TD_USERNAME, ?TD_PASSWORD, - ?SQL_BRIDGE, + Template, BatchSize, QueryMode ] @@ -272,11 +287,15 @@ connect_direct_tdengine(Config) -> connect_and_create_table(Config) -> ?WITH_CON(begin {ok, _} = directly_query(Con, ?SQL_CREATE_DATABASE, []), - {ok, _} = directly_query(Con, ?SQL_CREATE_TABLE) + {ok, _} = directly_query(Con, ?SQL_CREATE_TABLE), + {ok, _} = directly_query(Con, ?SQL_CREATE_STABLE) end). connect_and_drop_table(Config) -> - ?WITH_CON({ok, _} = directly_query(Con, ?SQL_DROP_TABLE)). + ?WITH_CON(begin + {ok, _} = directly_query(Con, ?SQL_DROP_TABLE), + {ok, _} = directly_query(Con, ?SQL_DROP_STABLE) + end). connect_and_clear_table(Config) -> ?WITH_CON({ok, _} = directly_query(Con, ?SQL_DELETE)). @@ -287,6 +306,15 @@ connect_and_get_payload(Config) -> ), Result. +connect_and_exec(Config, SQL) -> + ?WITH_CON({ok, _} = directly_query(Con, SQL)). + +connect_and_query(Config, SQL) -> + ?WITH_CON( + {ok, #{<<"code">> := 0, <<"data">> := Data}} = directly_query(Con, SQL) + ), + Data. + directly_query(Con, Query) -> directly_query(Con, Query, [{db_name, ?TD_DATABASE}]). @@ -407,7 +435,7 @@ t_write_failure(Config) -> #{?snk_kind := buffer_worker_flush_ack}, 2_000 ), - ?assertMatch({error, econnrefused}, Result), + ?assertMatch({error, Reason} when Reason =:= econnrefused; Reason =:= closed, Result), ok end), ok. @@ -490,26 +518,19 @@ t_missing_data(Config) -> ok. t_bad_sql_parameter(Config) -> - EnableBatch = ?config(enable_batch, Config), ?assertMatch( {ok, _}, create_bridge(Config) ), - Request = {sql, <<"">>, [bad_parameter]}, + Request = {send_message, <<"">>}, {_, {ok, #{result := Result}}} = ?wait_async_action( query_resource(Config, Request), #{?snk_kind := buffer_worker_flush_ack}, 2_000 ), - case EnableBatch of - true -> - ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result); - false -> - ?assertMatch( - {error, {unrecoverable_error, _}}, Result - ) - end, + + ?assertMatch({error, #{<<"code">> := _}}, Result), ok. t_nasty_sql_string(Config) -> @@ -544,7 +565,156 @@ t_nasty_sql_string(Config) -> connect_and_get_payload(Config) ). +t_simple_insert(Config) -> + connect_and_clear_table(Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + + SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000}, + Request = {send_message, SentData}, + {_, {ok, #{result := _Result}}} = + ?wait_async_action( + query_resource(Config, Request), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + ?assertMatch( + ?PAYLOAD, + connect_and_get_payload(Config) + ). + +t_batch_insert(Config) -> + connect_and_clear_table(Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + + Size = 5, + Ts = erlang:system_time(millisecond), + {_, {ok, #{result := Result}}} = + ?wait_async_action( + lists:foreach( + fun(Idx) -> + SentData = #{payload => ?PAYLOAD, timestamp => Ts + Idx}, + Request = {send_message, SentData}, + query_resource(Config, Request) + end, + lists:seq(1, Size) + ), + + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + + timer:sleep(200), + + ?assertMatch( + [[Size]], + connect_and_query(Config, "SELECT COUNT(1) FROM t_mqtt_msg") + ). + +t_auto_create_simple_insert(Config0) -> + ClientId = to_str(?FUNCTION_NAME), + Config = get_auto_create_config(Config0), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + + SentData = #{ + payload => ?PAYLOAD, + timestamp => 1668602148000, + clientid => ClientId + }, + Request = {send_message, SentData}, + {_, {ok, #{result := _Result}}} = + ?wait_async_action( + query_resource(Config, Request), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + ?assertMatch( + [[?PAYLOAD]], + connect_and_query(Config, "SELECT payload FROM " ++ ClientId) + ), + + ?assertMatch( + [[0]], + connect_and_query(Config, "DROP TABLE " ++ ClientId) + ). + +t_auto_create_batch_insert(Config0) -> + ClientId1 = "client1", + ClientId2 = "client2", + Config = get_auto_create_config(Config0), + + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + + Size1 = 2, + Size2 = 3, + + Ts = erlang:system_time(millisecond), + {_, {ok, #{result := Result}}} = + ?wait_async_action( + lists:foreach( + fun({Offset, ClientId, Size}) -> + lists:foreach( + fun(Idx) -> + SentData = #{ + payload => ?PAYLOAD, + timestamp => Ts + Idx + Offset, + clientid => ClientId + }, + Request = {send_message, SentData}, + query_resource(Config, Request) + end, + lists:seq(1, Size) + ) + end, + [{0, ClientId1, Size1}, {100, ClientId2, Size2}] + ), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + + timer:sleep(200), + + ?assertMatch( + [[Size1]], + connect_and_query(Config, "SELECT COUNT(1) FROM " ++ ClientId1) + ), + + ?assertMatch( + [[Size2]], + connect_and_query(Config, "SELECT COUNT(1) FROM " ++ ClientId2) + ), + + ?assertMatch( + [[0]], + connect_and_query(Config, "DROP TABLE " ++ ClientId1) + ), + + ?assertMatch( + [[0]], + connect_and_query(Config, "DROP TABLE " ++ ClientId2) + ). + to_bin(List) when is_list(List) -> unicode:characters_to_binary(List, utf8); to_bin(Bin) when is_binary(Bin) -> Bin. + +to_str(Atom) when is_atom(Atom) -> + erlang:atom_to_list(Atom). + +get_auto_create_config(Config0) -> + Config = lists:keyreplace(template, 1, Config0, {template, ?AUTO_CREATE_BRIDGE}), + BridgeType = proplists:get_value(bridge_type, Config, <<"tdengine">>), + {_Name, TDConf} = tdengine_config(BridgeType, Config), + lists:keyreplace(tdengine_config, 1, Config, {tdengine_config, TDConf}). From f1a3e5965e0862b4b10f003fdcd98a697c70325c Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 18 May 2023 17:34:28 +0800 Subject: [PATCH 169/197] chore: update apps version && changes --- apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src | 2 +- apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src | 2 +- changes/ee/fix-10738.en.md | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 changes/ee/fix-10738.en.md diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src index 141973e1e..321a2b724 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_tdengine, [ {description, "EMQX Enterprise TDEngine Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [kernel, stdlib, tdengine]}, {env, []}, diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index bfd7e68fa..82a95c377 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugin_libs, [ {description, "EMQX Plugin utility libs"}, - {vsn, "4.3.10"}, + {vsn, "4.3.11"}, {modules, []}, {applications, [kernel, stdlib]}, {env, []} diff --git a/changes/ee/fix-10738.en.md b/changes/ee/fix-10738.en.md new file mode 100644 index 000000000..e2fa14bfc --- /dev/null +++ b/changes/ee/fix-10738.en.md @@ -0,0 +1,3 @@ +Add supports for the `Supertable` and `Create Tables Automatically` features of TDEngine to its data bridge. +Before this fix, an insert with a supertable in the template will fail, like this: + `insert into ${clientid} using msg TAGS (${clientid}) values (${ts},${msg})`. From 5d289ade56c2331879a47925e910f1d8bb1c3ddf Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Tue, 16 May 2023 18:15:46 -0300 Subject: [PATCH 170/197] fix: validate range for some bridge options Fixes https://emqx.atlassian.net/browse/EMQX-9864 Setting a very large interval can cause `erlang:start_timer` to crash. Also, setting auto_restart_interval or health_check_interval to "0s" causes the state machine to be in loop as time 0 is handled separately: | state_timeout() = timeout() | integer() | (...) | If Time is relative and 0 no timer is actually started, instead the the | time-out event is enqueued to ensure that it gets processed before any | not yet received external event. from "https://www.erlang.org/doc/man/gen_statem.html#type-state_timeout" Therefore, both fields are now validated against the range [1ms, 1h], which doesn't cause above issues. --- .../src/schema/emqx_resource_schema.erl | 34 +++++++++++++++++++ changes/ce/fix-10726.en.md | 1 + 2 files changed, 35 insertions(+) create mode 100644 changes/ce/fix-10726.en.md diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 3b4fb66e5..9b1ae64fa 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -23,6 +23,12 @@ -export([namespace/0, roots/0, fields/1, desc/1]). +%% range interval in ms +-define(HEALTH_CHECK_INTERVAL_RANGE_MIN, 1). +-define(HEALTH_CHECK_INTERVAL_RANGE_MAX, 3_600_000). +-define(AUTO_RESTART_INTERVAL_RANGE_MIN, 1). +-define(AUTO_RESTART_INTERVAL_RANGE_MAX, 3_600_000). + %% ------------------------------------------------------------------------------------------------- %% Hocon Schema Definitions @@ -81,8 +87,22 @@ health_check_interval(type) -> emqx_schema:duration_ms(); health_check_interval(desc) -> ?DESC("health_check_interval"); health_check_interval(default) -> ?HEALTHCHECK_INTERVAL_RAW; health_check_interval(required) -> false; +health_check_interval(validator) -> fun health_check_interval_range/1; health_check_interval(_) -> undefined. +health_check_interval_range(HealthCheckInterval) when + is_integer(HealthCheckInterval) andalso + HealthCheckInterval >= ?HEALTH_CHECK_INTERVAL_RANGE_MIN andalso + HealthCheckInterval =< ?HEALTH_CHECK_INTERVAL_RANGE_MAX +-> + ok; +health_check_interval_range(_HealthCheckInterval) -> + {error, #{ + msg => <<"Health Check Interval out of range">>, + min => ?HEALTH_CHECK_INTERVAL_RANGE_MIN, + max => ?HEALTH_CHECK_INTERVAL_RANGE_MAX + }}. + start_after_created(type) -> boolean(); start_after_created(desc) -> ?DESC("start_after_created"); start_after_created(default) -> ?START_AFTER_CREATED_RAW; @@ -99,8 +119,22 @@ auto_restart_interval(type) -> hoconsc:union([infinity, emqx_schema:duration_ms( auto_restart_interval(desc) -> ?DESC("auto_restart_interval"); auto_restart_interval(default) -> ?AUTO_RESTART_INTERVAL_RAW; auto_restart_interval(required) -> false; +auto_restart_interval(validator) -> fun auto_restart_interval_range/1; auto_restart_interval(_) -> undefined. +auto_restart_interval_range(AutoRestartInterval) when + is_integer(AutoRestartInterval) andalso + AutoRestartInterval >= ?AUTO_RESTART_INTERVAL_RANGE_MIN andalso + AutoRestartInterval =< ?AUTO_RESTART_INTERVAL_RANGE_MAX +-> + ok; +auto_restart_interval_range(_AutoRestartInterval) -> + {error, #{ + msg => <<"Auto Restart Interval out of range">>, + min => ?AUTO_RESTART_INTERVAL_RANGE_MIN, + max => ?AUTO_RESTART_INTERVAL_RANGE_MAX + }}. + query_mode(type) -> enum([sync, async]); query_mode(desc) -> ?DESC("query_mode"); query_mode(default) -> async; diff --git a/changes/ce/fix-10726.en.md b/changes/ce/fix-10726.en.md new file mode 100644 index 000000000..d7a1f5bab --- /dev/null +++ b/changes/ce/fix-10726.en.md @@ -0,0 +1 @@ +Validate Health Check Interval and Auto Restart Interval against the range from 1ms to 1 hour. From 09ea2e2224f093cfcb77639742066531b25039a4 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 18 May 2023 11:37:45 -0300 Subject: [PATCH 171/197] fix(bridge_api): don't crash when formatting empty/unknown bridge metrics Fixes https://emqx.atlassian.net/browse/EMQX-9872 --- apps/emqx/test/emqx_common_test_helpers.erl | 8 ++- apps/emqx_bridge/src/emqx_bridge_api.erl | 23 +++++++ .../test/emqx_bridge_api_SUITE.erl | 66 ++++++++++++++++++- changes/ce/fix-10743.en.md | 1 + 4 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 changes/ce/fix-10743.en.md diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 61373f638..3654b8fae 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -653,11 +653,17 @@ emqx_cluster(Specs0, CommonOpts) -> ]), %% Set the default node of the cluster: CoreNodes = [node_name(Name) || {{core, Name, _}, _} <- Specs], - JoinTo = + JoinTo0 = case CoreNodes of [First | _] -> First; _ -> undefined end, + JoinTo = + case maps:find(join_to, CommonOpts) of + {ok, true} -> JoinTo0; + {ok, JT} -> JT; + error -> JoinTo0 + end, [ {Name, merge_opts(Opts, #{ diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 9802d5fe8..847270664 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -882,6 +882,29 @@ format_metrics(#{ Rate5m, RateMax, Rcvd + ); +format_metrics(_Metrics) -> + %% Empty metrics: can happen when a node joins another and a + %% bridge is not yet replicated to it, so the counters map is + %% empty. + ?METRICS( + _Dropped = 0, + _DroppedOther = 0, + _DroppedExpired = 0, + _DroppedQueueFull = 0, + _DroppedResourceNotFound = 0, + _DroppedResourceStopped = 0, + _Matched = 0, + _Queued = 0, + _Retried = 0, + _LateReply = 0, + _SentFailed = 0, + _SentInflight = 0, + _SentSucc = 0, + _Rate = 0, + _Rate5m = 0, + _RateMax = 0, + _Rcvd = 0 ). fill_defaults(Type, RawConf) -> diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 288b1da29..27c1c779a 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -70,18 +70,22 @@ all() -> [ {group, single}, + {group, cluster_later_join}, {group, cluster} ]. groups() -> + AllTCs = emqx_common_test_helpers:all(?MODULE), SingleOnlyTests = [ t_broken_bpapi_vsn, t_old_bpapi_vsn, t_bridges_probe ], + ClusterLaterJoinOnlyTCs = [t_cluster_later_join_metrics], [ - {single, [], emqx_common_test_helpers:all(?MODULE)}, - {cluster, [], emqx_common_test_helpers:all(?MODULE) -- SingleOnlyTests} + {single, [], AllTCs -- ClusterLaterJoinOnlyTCs}, + {cluster_later_join, [], ClusterLaterJoinOnlyTCs}, + {cluster, [], (AllTCs -- SingleOnlyTests) -- ClusterLaterJoinOnlyTCs} ]. suite() -> @@ -104,6 +108,17 @@ init_per_group(cluster, Config) -> ok = erpc:call(NodePrimary, fun() -> init_node(primary) end), _ = [ok = erpc:call(Node, fun() -> init_node(regular) end) || Node <- NodesRest], [{group, cluster}, {cluster_nodes, Nodes}, {api_node, NodePrimary} | Config]; +init_per_group(cluster_later_join, Config) -> + Cluster = mk_cluster_specs(Config, #{join_to => undefined}), + ct:pal("Starting ~p", [Cluster]), + Nodes = [ + emqx_common_test_helpers:start_slave(Name, Opts) + || {Name, Opts} <- Cluster + ], + [NodePrimary | NodesRest] = Nodes, + ok = erpc:call(NodePrimary, fun() -> init_node(primary) end), + _ = [ok = erpc:call(Node, fun() -> init_node(regular) end) || Node <- NodesRest], + [{group, cluster_later_join}, {cluster_nodes, Nodes}, {api_node, NodePrimary} | Config]; init_per_group(_, Config) -> ok = emqx_mgmt_api_test_util:init_suite(?SUITE_APPS), ok = load_suite_config(emqx_rule_engine), @@ -111,6 +126,9 @@ init_per_group(_, Config) -> [{group, single}, {api_node, node()} | Config]. mk_cluster_specs(Config) -> + mk_cluster_specs(Config, #{}). + +mk_cluster_specs(Config, Opts) -> Specs = [ {core, emqx_bridge_api_SUITE1, #{}}, {core, emqx_bridge_api_SUITE2, #{}} @@ -132,6 +150,7 @@ mk_cluster_specs(Config) -> load_apps => ?SUITE_APPS ++ [emqx_dashboard], env_handler => fun load_suite_config/1, load_schema => false, + join_to => maps:get(join_to, Opts, true), priv_data_dir => ?config(priv_dir, Config) }, emqx_common_test_helpers:emqx_cluster(Specs, CommonOpts). @@ -164,7 +183,10 @@ load_suite_config(emqx_bridge) -> load_suite_config(_) -> ok. -end_per_group(cluster, Config) -> +end_per_group(Group, Config) when + Group =:= cluster; + Group =:= cluster_later_join +-> ok = lists:foreach( fun(Node) -> _ = erpc:call(Node, emqx_common_test_helpers, stop_apps, [?SUITE_APPS]), @@ -1298,6 +1320,44 @@ t_inconsistent_webhook_request_timeouts(Config) -> validate_resource_request_timeout(proplists:get_value(group, Config), 1000, Name), ok. +t_cluster_later_join_metrics(Config) -> + Port = ?config(port, Config), + APINode = ?config(api_node, Config), + ClusterNodes = ?config(cluster_nodes, Config), + [OtherNode | _] = ClusterNodes -- [APINode], + URL1 = ?URL(Port, "path1"), + Name = ?BRIDGE_NAME, + BridgeParams = ?HTTP_BRIDGE(URL1, Name), + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), + ?check_trace( + begin + %% Create a bridge on only one of the nodes. + ?assertMatch({ok, 201, _}, request_json(post, uri(["bridges"]), BridgeParams, Config)), + %% Pre-condition. + ?assertMatch( + {ok, 200, #{ + <<"metrics">> := #{<<"success">> := _}, + <<"node_metrics">> := [_ | _] + }}, + request_json(get, uri(["bridges", BridgeID, "metrics"]), Config) + ), + %% Now join the other node join with the api node. + ok = erpc:call(OtherNode, ekka, join, [APINode]), + %% Check metrics; shouldn't crash even if the bridge is not + %% ready on the node that just joined the cluster. + ?assertMatch( + {ok, 200, #{ + <<"metrics">> := #{<<"success">> := _}, + <<"node_metrics">> := [#{<<"metrics">> := #{}}, #{<<"metrics">> := #{}} | _] + }}, + request_json(get, uri(["bridges", BridgeID, "metrics"]), Config) + ), + ok + end, + [] + ), + ok. + validate_resource_request_timeout(single, Timeout, Name) -> SentData = #{payload => <<"Hello EMQX">>, timestamp => 1668602148000}, BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), diff --git a/changes/ce/fix-10743.en.md b/changes/ce/fix-10743.en.md new file mode 100644 index 000000000..95e6b3652 --- /dev/null +++ b/changes/ce/fix-10743.en.md @@ -0,0 +1 @@ +Fixes an issue where trying to get a bridge info or metrics could result in a crash when a node is joining a cluster. From 5a08a7b9de1393a115ed4bb89d4703af6e5433fd Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 19 May 2023 11:08:58 +0800 Subject: [PATCH 172/197] fix(tdengine): minor improvement of code and changes --- .../src/emqx_bridge_tdengine_connector.erl | 41 ++++++++++--------- .../test/emqx_bridge_tdengine_SUITE.erl | 37 ++++++++++------- changes/ee/fix-10738.en.md | 2 +- 3 files changed, 45 insertions(+), 35 deletions(-) diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl index 876743be5..5644d0446 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl @@ -208,8 +208,8 @@ execute(Conn, Query, Opts) -> do_batch_insert(Conn, Tokens, BatchReqs, Opts) -> Queries = aggregate_query(Tokens, BatchReqs), - SQL = lists:foldl( - fun({InsertPart, Values}, Acc) -> + SQL = maps:fold( + fun(InsertPart, Values, Acc) -> lists:foldl( fun(ValuePart, IAcc) -> <> @@ -224,17 +224,15 @@ do_batch_insert(Conn, Tokens, BatchReqs, Opts) -> execute(Conn, SQL, Opts). aggregate_query({InsertPartTks, ParamsPartTks}, BatchReqs) -> - maps:to_list( - lists:foldl( - fun({_, Data}, Acc) -> - InsertPart = emqx_plugin_libs_rule:proc_sql_param_str(InsertPartTks, Data), - ParamsPart = emqx_plugin_libs_rule:proc_sql_param_str(ParamsPartTks, Data), - Values = maps:get(InsertPart, Acc, []), - maps:put(InsertPart, [ParamsPart | Values], Acc) - end, - #{}, - BatchReqs - ) + lists:foldl( + fun({_, Data}, Acc) -> + InsertPart = emqx_plugin_libs_rule:proc_sql_param_str(InsertPartTks, Data), + ParamsPart = emqx_plugin_libs_rule:proc_sql_param_str(ParamsPartTks, Data), + Values = maps:get(InsertPart, Acc, []), + maps:put(InsertPart, [ParamsPart | Values], Acc) + end, + #{}, + BatchReqs ). connect(Opts) -> @@ -268,8 +266,8 @@ parse_batch_prepare_sql([{Key, H} | T], InsertTksMap, BatchTksMap) -> InsertTksMap#{Key => InsertTks}, BatchTksMap#{Key => {InsertPartTks, ParamsPartTks}} ); - _ -> - ?SLOG(error, #{msg => "split sql failed", sql => H}), + Result -> + ?SLOG(error, #{msg => "split sql failed", sql => H, result => Result}), parse_batch_prepare_sql(T, InsertTksMap, BatchTksMap) end; {error, Reason} -> @@ -287,11 +285,14 @@ to_bin(List) when is_list(List) -> split_insert_sql(SQL0) -> SQL = emqx_plugin_libs_rule:formalize_sql(SQL0), - lists:foldr( - fun - (<<>>, Acc) -> Acc; - (E, Acc) -> [string:trim(E) | Acc] + lists:filtermap( + fun(E) -> + case string:trim(E) of + <<>> -> + false; + E1 -> + {true, E1} + end end, - [], re:split(SQL, "(?i)(insert into)|(?i)(values)") ). diff --git a/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl b/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl index 00a887852..3d06aee52 100644 --- a/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl +++ b/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl @@ -594,7 +594,7 @@ t_batch_insert(Config) -> Size = 5, Ts = erlang:system_time(millisecond), - {_, {ok, #{result := Result}}} = + {_, {ok, #{result := _Result}}} = ?wait_async_action( lists:foreach( fun(Idx) -> @@ -609,11 +609,13 @@ t_batch_insert(Config) -> 2_000 ), - timer:sleep(200), - - ?assertMatch( - [[Size]], - connect_and_query(Config, "SELECT COUNT(1) FROM t_mqtt_msg") + ?retry( + _Sleep = 50, + _Attempts = 30, + ?assertMatch( + [[Size]], + connect_and_query(Config, "SELECT COUNT(1) FROM t_mqtt_msg") + ) ). t_auto_create_simple_insert(Config0) -> @@ -660,7 +662,7 @@ t_auto_create_batch_insert(Config0) -> Size2 = 3, Ts = erlang:system_time(millisecond), - {_, {ok, #{result := Result}}} = + {_, {ok, #{result := _Result}}} = ?wait_async_action( lists:foreach( fun({Offset, ClientId, Size}) -> @@ -683,16 +685,23 @@ t_auto_create_batch_insert(Config0) -> 2_000 ), - timer:sleep(200), + ?retry( + _Sleep = 50, + _Attempts = 30, - ?assertMatch( - [[Size1]], - connect_and_query(Config, "SELECT COUNT(1) FROM " ++ ClientId1) + ?assertMatch( + [[Size1]], + connect_and_query(Config, "SELECT COUNT(1) FROM " ++ ClientId1) + ) ), - ?assertMatch( - [[Size2]], - connect_and_query(Config, "SELECT COUNT(1) FROM " ++ ClientId2) + ?retry( + 50, + 30, + ?assertMatch( + [[Size2]], + connect_and_query(Config, "SELECT COUNT(1) FROM " ++ ClientId2) + ) ), ?assertMatch( diff --git a/changes/ee/fix-10738.en.md b/changes/ee/fix-10738.en.md index e2fa14bfc..203fb5823 100644 --- a/changes/ee/fix-10738.en.md +++ b/changes/ee/fix-10738.en.md @@ -1,3 +1,3 @@ -Add supports for the `Supertable` and `Create Tables Automatically` features of TDEngine to its data bridge. +Add support for the `Supertable` and `Create Tables Automatically` features of TDEngine to its data bridge. Before this fix, an insert with a supertable in the template will fail, like this: `insert into ${clientid} using msg TAGS (${clientid}) values (${ts},${msg})`. From fa799e95a4cb1ee24c136d323f6f153bb1a24c48 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 19 May 2023 14:20:26 +0800 Subject: [PATCH 173/197] fix: supports test the `$events/delivery_dropped` event by API --- apps/emqx_rule_engine/src/emqx_rule_api_schema.erl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index c9926f56f..0424bfb60 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -112,7 +112,8 @@ fields("rule_test") -> ref("ctx_disconnected"), ref("ctx_connack"), ref("ctx_check_authz_complete"), - ref("ctx_bridge_mqtt") + ref("ctx_bridge_mqtt"), + ref("ctx_delivery_dropped") ]), #{ desc => ?DESC("test_context"), @@ -276,6 +277,15 @@ fields("ctx_bridge_mqtt") -> {"retain", sc(binary(), #{desc => ?DESC("event_retain")})}, {"message_received_at", publish_received_at_sc()}, qos() + ]; +fields("ctx_delivery_dropped") -> + [ + {"event_type", event_type_sc(delivery_dropped)}, + {"id", sc(binary(), #{desc => ?DESC("event_id")})}, + {"reason", sc(binary(), #{desc => ?DESC("event_ctx_dropped")})}, + {"from_clientid", sc(binary(), #{desc => ?DESC("event_from_clientid")})}, + {"from_username", sc(binary(), #{desc => ?DESC("event_from_username")})} + | msg_event_common_fields() ]. qos() -> From 0eea8438bf9c1fa9b6397901b17eb605fd24c134 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 25 Apr 2023 16:56:20 +0800 Subject: [PATCH 174/197] fix(resource): make some logging of the resource manager more secure --- apps/emqx_resource/src/emqx_resource_manager.erl | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index f42d3c1b5..c9417583e 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -387,7 +387,7 @@ handle_event(EventType, EventData, State, Data) -> event_type => EventType, event_data => EventData, state => State, - data => Data + data => redact_data(Data) } ), keep_state_and_data. @@ -397,15 +397,15 @@ log_state_consistency(State, #data{status = State} = Data) -> log_state_consistency(State, Data) -> ?tp(warning, "inconsistent_state", #{ state => State, - data => Data + data => redact_data(Data) }). log_cache_consistency(Data, Data) -> ok; log_cache_consistency(DataCached, Data) -> ?tp(warning, "inconsistent_cache", #{ - cache => DataCached, - data => Data + cache => redact_data(DataCached), + data => redact_data(Data) }). %%------------------------------------------------------------------------------ @@ -661,3 +661,9 @@ safe_call(ResId, Message, Timeout) -> exit:{timeout, _} -> {error, timeout} end. + +%% the config and state of a bridge often contains some sensitive data +%% we shouldn't expose them to logs +redact_data(Data) -> + Msg = <<"this data is redacted due to security reasons">>, + Data#data{config = Msg, state = Msg}. From baeb96a6e48c24a2275a80b158f43e8d5a3d2cdb Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 25 Apr 2023 17:23:54 +0800 Subject: [PATCH 175/197] chore: update changes --- apps/emqx_resource/src/emqx_resource_manager.erl | 14 ++++---------- changes/ce/perf-10511.en.md | 1 + 2 files changed, 5 insertions(+), 10 deletions(-) create mode 100644 changes/ce/perf-10511.en.md diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index c9417583e..cd858bda3 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -387,7 +387,7 @@ handle_event(EventType, EventData, State, Data) -> event_type => EventType, event_data => EventData, state => State, - data => redact_data(Data) + data => emqx_utils:redact(Data) } ), keep_state_and_data. @@ -397,15 +397,15 @@ log_state_consistency(State, #data{status = State} = Data) -> log_state_consistency(State, Data) -> ?tp(warning, "inconsistent_state", #{ state => State, - data => redact_data(Data) + data => emqx_utils:redact(Data) }). log_cache_consistency(Data, Data) -> ok; log_cache_consistency(DataCached, Data) -> ?tp(warning, "inconsistent_cache", #{ - cache => redact_data(DataCached), - data => redact_data(Data) + cache => emqx_utils:redact(DataCached), + data => emqx_utils:redact(Data) }). %%------------------------------------------------------------------------------ @@ -661,9 +661,3 @@ safe_call(ResId, Message, Timeout) -> exit:{timeout, _} -> {error, timeout} end. - -%% the config and state of a bridge often contains some sensitive data -%% we shouldn't expose them to logs -redact_data(Data) -> - Msg = <<"this data is redacted due to security reasons">>, - Data#data{config = Msg, state = Msg}. diff --git a/changes/ce/perf-10511.en.md b/changes/ce/perf-10511.en.md new file mode 100644 index 000000000..953ecf693 --- /dev/null +++ b/changes/ce/perf-10511.en.md @@ -0,0 +1 @@ +Improve the security and privacy of some resource logs by masking sensitive information in the data. From 4327e4074087842cccdf3fee1d51a2b96c6e14cc Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 19 May 2023 10:07:30 -0300 Subject: [PATCH 176/197] fix(pulsar): redact error reason Fixes https://emqx.atlassian.net/browse/EMQX-9940 Error log after fix: ``` 2023-05-19T13:09:26.304769+00:00 [error] msg: failed_to_start_pulsar_client, mfa: emqx_bridge_pulsar_impl_producer:on_start/2, line: 104, instance_id: <<"bridge:pulsar_producer:pprodu">>, pulsar_hosts: pulsar://pulsar:6652, reason: {#{{"pulsar",6652} => #{error => 'AuthenticationError',message => "Unable to authenticate",request_id => 18446744073709551615}},{child,undefined,'pulsar_producer:pprodu:emqx@127.0.0.1',{pulsar_client,start_link,['pulsar_producer:pprodu:emqx@127.0.0.1',["pulsar://pulsar:6652"],#{conn_opts => #{auth_data => <<"******">>,auth_method_name => <<"basic">>},ssl_opts => []}]},transient,false,5000,worker,[pulsar_client]}} ``` --- .../src/emqx_bridge_pulsar_impl_producer.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl index 300fe9b2d..59956e1b6 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl @@ -100,7 +100,7 @@ on_start(InstanceId, Config) -> msg => "failed_to_start_pulsar_client", instance_id => InstanceId, pulsar_hosts => Servers, - reason => Reason + reason => emqx_utils:redact(Reason, fun is_sensitive_key/1) }), throw(failed_to_start_pulsar_client) end, @@ -332,7 +332,7 @@ start_producer(Config, InstanceId, ClientId, ClientOpts) -> #{ instance_id => InstanceId, kind => Kind, - reason => Error, + reason => emqx_utils:redact(Error, fun is_sensitive_key/1), stacktrace => Stacktrace } ), @@ -419,3 +419,6 @@ get_producer_status(Producers) -> partition_strategy(key_dispatch) -> first_key_dispatch; partition_strategy(Strategy) -> Strategy. + +is_sensitive_key(auth_data) -> true; +is_sensitive_key(_) -> false. From 7eef86363af3e104b304c82bdaf6411406e8cb4c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 19 May 2023 17:46:36 +0200 Subject: [PATCH 177/197] test: make erlfmt happy --- .../test/emqx_bridge_tdengine_SUITE.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl b/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl index 3d06aee52..0b8d20f15 100644 --- a/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl +++ b/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl @@ -435,7 +435,12 @@ t_write_failure(Config) -> #{?snk_kind := buffer_worker_flush_ack}, 2_000 ), - ?assertMatch({error, Reason} when Reason =:= econnrefused; Reason =:= closed, Result), + case Result of + {error, Reason} when Reason =:= econnrefused; Reason =:= closed -> + ok; + _ -> + throw({unexpected, Result}) + end, ok end), ok. From 5bbcf4b712439c9f29069bd6371bc964bbb5a299 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 17 May 2023 21:53:35 +0200 Subject: [PATCH 178/197] fix(mqtt-connector): faster shutdown --- apps/emqx_connector/src/emqx_connector.app.src | 2 +- apps/emqx_connector/src/emqx_connector_mqtt.erl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index db55c7032..76c3e8bb9 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "EMQX Data Integration Connectors"}, - {vsn, "0.1.22"}, + {vsn, "0.1.23"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 5cafd2d50..bb8cc00d1 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -112,7 +112,7 @@ bridge_spec(Config) -> id => Name, start => {emqx_connector_mqtt_worker, start_link, [Name, NConfig]}, restart => temporary, - shutdown => 5000 + shutdown => 1000 }. -spec bridges() -> [{_Name, _Status}]. @@ -181,7 +181,7 @@ on_stop(_InstId, #{name := InstanceId}) -> ok; {error, Reason} -> ?SLOG(error, #{ - msg => "stop_mqtt_connector", + msg => "stop_mqtt_connector_error", connector => InstanceId, reason => Reason }) From be90c63c786ba4445196a33f4849dd2c4f7d9651 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 18 May 2023 11:54:14 +0200 Subject: [PATCH 179/197] chore(mqtt-connector): refine logging level connect failure should be at warning level but not error, the connecting state is visiable from dashbaord also the resource manager logs connection failures in general at warning level --- apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index 880a99313..e49603e51 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -202,13 +202,13 @@ connect(Name) -> Error end; {error, Reason} = Error -> - ?SLOG(error, #{ + ?SLOG(warning, #{ msg => "client_connect_failed", - reason => Reason + reason => Reason, + name => Name }), Error end. - subscribe_remote_topics(Ref, #{remote := #{topic := FromTopic, qos := QoS}}) -> emqtt:subscribe(ref(Ref), FromTopic, QoS); subscribe_remote_topics(_Ref, undefined) -> From 21de0f8274851bb11ba0456e308c106c923c3e8d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 17 May 2023 22:42:15 +0200 Subject: [PATCH 180/197] fix(buffer-worker-sup): fast stop the timeout shutdown in child spec may significantly slow down the deletion of a resource this commit chagnes the shutdown to brutal kill also, the pool worker removal code has been delete because it's not necessary since the entier pool is going to be force-delete later anyway --- .../src/emqx_resource_buffer_worker_sup.erl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker_sup.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker_sup.erl index 104ad7ade..e4fa04d36 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker_sup.erl @@ -115,7 +115,8 @@ ensure_worker_started(ResId, Idx, Opts) -> id => ?CHILD_ID(Mod, ResId, Idx), start => {Mod, start_link, [ResId, Idx, Opts]}, restart => transient, - shutdown => 5000, + %% if we delay shutdown, when the pool is big, it will take a long time + shutdown => brutal_kill, type => worker, modules => [Mod] }, @@ -130,13 +131,12 @@ ensure_worker_removed(ResId, Idx) -> ChildId = ?CHILD_ID(emqx_resource_buffer_worker, ResId, Idx), case supervisor:terminate_child(?SERVER, ChildId) of ok -> - Res = supervisor:delete_child(?SERVER, ChildId), - _ = gproc_pool:remove_worker(ResId, {ResId, Idx}), - Res; - {error, not_found} -> + _ = supervisor:delete_child(?SERVER, ChildId), + %% no need to remove worker from the pool, + %% because the entire pool will be forece deleted later ok; - {error, Reason} -> - {error, Reason} + {error, not_found} -> + ok end. ensure_disk_queue_dir_absent(ResourceId, Index) -> From f5e5c59763b1805326c8e70dc5d271c4626708d2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 19 May 2023 18:02:38 +0200 Subject: [PATCH 181/197] refactor(resource-manager-sup): do not force kill resource manager the shutdown timeout is now set to infinity so it will never force kill a resource manager, otherwise there will be resource leaks --- .../src/emqx_resource_buffer_worker_sup.erl | 2 +- apps/emqx_resource/src/emqx_resource_manager_sup.erl | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker_sup.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker_sup.erl index e4fa04d36..4b85fe92f 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker_sup.erl @@ -133,7 +133,7 @@ ensure_worker_removed(ResId, Idx) -> ok -> _ = supervisor:delete_child(?SERVER, ChildId), %% no need to remove worker from the pool, - %% because the entire pool will be forece deleted later + %% because the entire pool will be force deleted later ok; {error, not_found} -> ok diff --git a/apps/emqx_resource/src/emqx_resource_manager_sup.erl b/apps/emqx_resource/src/emqx_resource_manager_sup.erl index 2f442cd56..73f1988c6 100644 --- a/apps/emqx_resource/src/emqx_resource_manager_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_manager_sup.erl @@ -17,7 +17,7 @@ -behaviour(supervisor). --export([ensure_child/5]). +-export([ensure_child/5, delete_child/1]). -export([start_link/0]). @@ -27,6 +27,11 @@ ensure_child(ResId, Group, ResourceType, Config, Opts) -> _ = supervisor:start_child(?MODULE, [ResId, Group, ResourceType, Config, Opts]), ok. +delete_child(Pid) -> + _ = supervisor:terminate_child(?MODULE, Pid), + _ = supervisor:delete_child(?MODULE, Pid), + ok. + start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). @@ -36,7 +41,10 @@ init([]) -> id => emqx_resource_manager, start => {emqx_resource_manager, start_link, []}, restart => transient, - shutdown => brutal_kill, + %% never force kill a resource manager. + %% becasue otherwise it may lead to release leak, + %% resource_manager's terminate callback calls resource on_stop + shutdown => infinity, type => worker, modules => [emqx_resource_manager] } From 0d8ffc0d59b8abca7f0b5330ecd0eacf2da5ac5b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 19 May 2023 18:28:22 +0200 Subject: [PATCH 182/197] fix(resource-manager): ensure no false creation Update is implemented as remove + create. If a dleete call is made while the create is in progress the remove call is likely to timeout too. This causes the follwing creation to falsely succeed, because there is alreay a running child under the supervisor. As a result, the resource is permanently removed after resource_manager eventually handles the remove call. --- .../src/emqx_resource_manager.erl | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index cd858bda3..9eb08e80d 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -53,7 +53,18 @@ % State record -record(data, { - id, group, mod, callback_mode, query_mode, config, opts, status, state, error, pid + id, + group, + mod, + callback_mode, + query_mode, + config, + opts, + status, + state, + error, + pid, + extra }). -type data() :: #data{}. @@ -181,7 +192,15 @@ remove(ResId) when is_binary(ResId) -> %% @doc Stops a running resource_manager and optionally clears the metrics for the resource -spec remove(resource_id(), boolean()) -> ok | {error, Reason :: term()}. remove(ResId, ClearMetrics) when is_binary(ResId) -> - safe_call(ResId, {remove, ClearMetrics}, ?T_OPERATION). + ResourceManagerPid = gproc:whereis_name(?NAME(ResId)), + try + safe_call(ResId, {remove, ClearMetrics}, ?T_OPERATION) + after + %% Ensure the supervisor has it removed, otherwise the immediate re-add will see a stale process + %% If the 'remove' call babove had succeeded, this is mostly a no-op but still needed to avoid race condition. + %% Otherwise this is a 'infinity' shutdown, so it may take arbitrary long. + emqx_resource_manager_sup:delete_child(ResourceManagerPid) + end. %% @doc Stops and then starts an instance that was already running -spec restart(resource_id(), creation_opts()) -> ok | {error, Reason :: term()}. @@ -439,8 +458,10 @@ health_check_actions(Data) -> [{state_timeout, health_check_interval(Data#data.opts), health_check}]. handle_remove_event(From, ClearMetrics, Data) -> - _ = stop_resource(Data), + %% stop the buffer workers first, brutal_kill, so it should be fast ok = emqx_resource_buffer_worker_sup:stop_workers(Data#data.id, Data#data.opts), + %% no stop the resource, this can be slow + _ = stop_resource(Data), case ClearMetrics of true -> ok = emqx_metrics_worker:clear_metrics(?RES_METRICS, Data#data.id); false -> ok From cb76e5a2418120d8ebfc8ac704a206dcb9aebde9 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 19 May 2023 18:11:46 +0200 Subject: [PATCH 183/197] docs: add changelog for 10755 --- apps/emqx_resource/src/emqx_resource_manager.erl | 2 +- changes/ce/fix-10755.en.md | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 changes/ce/fix-10755.en.md diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 9eb08e80d..97ac355f4 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -460,7 +460,7 @@ health_check_actions(Data) -> handle_remove_event(From, ClearMetrics, Data) -> %% stop the buffer workers first, brutal_kill, so it should be fast ok = emqx_resource_buffer_worker_sup:stop_workers(Data#data.id, Data#data.opts), - %% no stop the resource, this can be slow + %% now stop the resource, this can be slow _ = stop_resource(Data), case ClearMetrics of true -> ok = emqx_metrics_worker:clear_metrics(?RES_METRICS, Data#data.id); diff --git a/changes/ce/fix-10755.en.md b/changes/ce/fix-10755.en.md new file mode 100644 index 000000000..6c887166b --- /dev/null +++ b/changes/ce/fix-10755.en.md @@ -0,0 +1,10 @@ +Fixed data bridge resource update race condition. + +In the 'delete + create' process for EMQX resource updates, +long bridge creation times could cause dashboard request timeouts. +If a bridge resource update was initiated before completion of its creation, +it led to an erroneous deletion from the runtime, despite being present in the config file. + +This fix addresses the race condition in bridge resource updates, +ensuring the accurate identification and addition of new resources, +maintaining consistency between runtime and configuration file statuses. From 7c2aac64bca7147dd45cb641558a9d098982153d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Sat, 20 May 2023 20:29:03 +0800 Subject: [PATCH 184/197] fix: bad cert file path in dashboard https listener --- apps/emqx/src/emqx_schema.erl | 6 + apps/emqx/src/emqx_tls_lib.erl | 6 +- apps/emqx_dashboard/src/emqx_dashboard.erl | 38 ++-- .../test/emqx_dashboard_SUITE.erl | 2 - .../test/emqx_dashboard_https_SUITE.erl | 214 ++++++++++++++++++ 5 files changed, 246 insertions(+), 20 deletions(-) create mode 100644 apps/emqx_dashboard/test/emqx_dashboard_https_SUITE.erl diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 1779457e1..dfeae6d64 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -28,6 +28,7 @@ -include("emqx_access_control.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). +-include_lib("logger.hrl"). -type duration() :: integer(). -type duration_s() :: integer(). @@ -3290,6 +3291,11 @@ naive_env_interpolation("$" ++ Maybe = Original) -> {ok, Path} -> filename:join([Path, Tail]); error -> + ?SLOG(warning, #{ + msg => "failed_to_resolve_env_variable", + env => Env, + original => Original + }), Original end; naive_env_interpolation(Other) -> diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 2683d2a9d..db0996e56 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -620,9 +620,11 @@ ensure_bin(A) when is_atom(A) -> atom_to_binary(A, utf8). ensure_ssl_file_key(_SSL, []) -> ok; ensure_ssl_file_key(SSL, RequiredKeyPaths) -> - NotFoundRef = make_ref(), Filter = fun(KeyPath) -> - NotFoundRef =:= emqx_utils_maps:deep_get(KeyPath, SSL, NotFoundRef) + case emqx_utils_maps:deep_find(KeyPath, SSL) of + {not_found, _, _} -> true; + _ -> false + end end, case lists:filter(Filter, RequiredKeyPaths) of [] -> ok; diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 13fd18267..aec811e5d 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -95,7 +95,7 @@ start_listeners(Listeners) -> end end, {[], []}, - listeners(Listeners) + listeners(ensure_ssl_cert(Listeners)) ), case ErrListeners of [] -> @@ -140,18 +140,18 @@ apps() -> listeners(Listeners) -> lists:filtermap( - fun({Protocol, Conf}) -> - maps:get(enable, Conf) andalso - begin - {Conf1, Bind} = ip_port(Conf), - {true, { - listener_name(Protocol), - Protocol, - Bind, - ranch_opts(Conf1), - proto_opts(Conf1) - }} - end + fun + ({Protocol, Conf = #{enable := true}}) -> + {Conf1, Bind} = ip_port(Conf), + {true, { + listener_name(Protocol), + Protocol, + Bind, + ranch_opts(Conf1), + proto_opts(Conf1) + }}; + ({_Protocol, #{enable := false}}) -> + false end, maps:to_list(Listeners) ). @@ -191,8 +191,8 @@ ranch_opts(Options) -> end, RanchOpts#{socket_opts => InetOpts ++ SocketOpts}. -proto_opts(Options) -> - maps:with([proxy_header], Options). +proto_opts(#{proxy_header := ProxyHeader}) -> + #{proxy_header => ProxyHeader}. filter_false(_K, false, S) -> S; filter_false(K, V, S) -> [{K, V} | S]. @@ -224,7 +224,7 @@ return_unauthorized(Code, Message) -> {401, #{ <<"WWW-Authenticate">> => - <<"Basic Realm=\"minirest-server\"">> + <<"Basic Realm=\"emqx-dashboard\"">> }, #{code => Code, message => Message}}. @@ -247,3 +247,9 @@ api_key_authorize(Req, Key, Secret) -> <<"Check api_key/api_secret">> ) end. + +ensure_ssl_cert(Listeners = #{https := Https0}) -> + Https1 = emqx_tls_lib:to_server_opts(tls, Https0), + Listeners#{https => maps:from_list(Https1)}; +ensure_ssl_cert(Listeners) -> + Listeners. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 1f14b02c0..783c00dad 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -36,8 +36,6 @@ -define(HOST, "http://127.0.0.1:18083"). -%% -define(API_VERSION, "v5"). - -define(BASE_PATH, "/api/v5"). -define(APP_DASHBOARD, emqx_dashboard). diff --git a/apps/emqx_dashboard/test/emqx_dashboard_https_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_https_SUITE.erl new file mode 100644 index 000000000..fefaeb7f1 --- /dev/null +++ b/apps/emqx_dashboard/test/emqx_dashboard_https_SUITE.erl @@ -0,0 +1,214 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_dashboard_https_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include("emqx_dashboard.hrl"). + +-define(NAME, 'https:dashboard'). +-define(HOST, "https://127.0.0.1:18084"). +-define(BASE_PATH, "/api/v5"). +-define(OVERVIEWS, [ + "alarms", + "banned", + "stats", + "metrics", + "listeners", + "clients", + "subscriptions" +]). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> Config. +end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite([emqx_management]). + +init_per_testcase(_TestCase, Config) -> Config. +end_per_testcase(_TestCase, _Config) -> emqx_mgmt_api_test_util:end_suite([emqx_management]). + +t_default_ssl_cert(_Config) -> + Conf = #{dashboard => #{listeners => #{https => #{bind => 18084, enable => true}}}}, + validate_https(Conf, 512, default_ssl_cert(), verify_none), + ok. + +t_normal_ssl_cert(_Config) -> + MaxConnection = 1000, + Conf = #{ + dashboard => #{ + listeners => #{ + https => #{ + bind => 18084, + enable => true, + cacertfile => naive_env_interpolation(<<"${EMQX_ETC_DIR}/certs/cacert.pem">>), + certfile => naive_env_interpolation(<<"${EMQX_ETC_DIR}/certs/cert.pem">>), + keyfile => naive_env_interpolation(<<"${EMQX_ETC_DIR}/certs/key.pem">>), + max_connections => MaxConnection + } + } + } + }, + validate_https(Conf, MaxConnection, default_ssl_cert(), verify_none), + ok. + +t_verify_cacertfile(_Config) -> + MaxConnection = 1024, + DefaultSSLCert = default_ssl_cert(), + SSLCert = DefaultSSLCert#{cacertfile => <<"">>}, + %% default #{verify => verify_none} + Conf = #{ + dashboard => #{ + listeners => #{ + https => #{ + bind => 18084, + enable => true, + cacertfile => <<"">>, + max_connections => MaxConnection + } + } + } + }, + validate_https(Conf, MaxConnection, SSLCert, verify_none), + %% verify_peer but cacertfile is empty + VerifyPeerConf1 = emqx_utils_maps:deep_put( + [dashboard, listeners, https, verify], + Conf, + verify_peer + ), + emqx_common_test_helpers:load_config(emqx_dashboard_schema, VerifyPeerConf1), + ?assertMatch({error, [?NAME]}, emqx_dashboard:start_listeners()), + %% verify_peer and cacertfile is ok. + VerifyPeerConf2 = emqx_utils_maps:deep_put( + [dashboard, listeners, https, cacertfile], + VerifyPeerConf1, + naive_env_interpolation(<<"${EMQX_ETC_DIR}/certs/cacert.pem">>) + ), + validate_https(VerifyPeerConf2, MaxConnection, DefaultSSLCert, verify_peer), + ok. + +t_bad_certfile(_Config) -> + Conf = #{ + dashboard => #{ + listeners => #{ + https => #{ + bind => 18084, + enable => true, + certfile => <<"${EMQX_ETC_DIR}/certs/not_found_cert.pem">> + } + } + } + }, + emqx_common_test_helpers:load_config(emqx_dashboard_schema, Conf), + ?assertMatch({error, [?NAME]}, emqx_dashboard:start_listeners()), + ok. + +validate_https(Conf, MaxConnection, SSLCert, Verify) -> + emqx_common_test_helpers:load_config(emqx_dashboard_schema, Conf), + emqx_mgmt_api_test_util:init_suite([emqx_management], fun(X) -> X end), + assert_ranch_options(MaxConnection, SSLCert, Verify), + assert_https_request(), + emqx_mgmt_api_test_util:end_suite([emqx_management]). + +assert_ranch_options(MaxConnections0, SSLCert, Verify) -> + Middlewares = [emqx_dashboard_middleware, cowboy_router, cowboy_handler], + [ + ?NAME, + ranch_ssl, + #{ + max_connections := MaxConnections, + num_acceptors := _, + socket_opts := SocketOpts + }, + cowboy_tls, + #{ + env := #{ + dispatch := {persistent_term, ?NAME}, + options := #{ + name := ?NAME, + protocol := https, + protocol_options := #{proxy_header := false}, + security := [#{basicAuth := []}, #{bearerAuth := []}], + swagger_global_spec := _ + } + }, + middlewares := Middlewares, + proxy_header := false + } + ] = ranch_server:get_listener_start_args(?NAME), + ?assertEqual(MaxConnections0, MaxConnections), + ?assert(lists:member(inet, SocketOpts), SocketOpts), + #{ + backlog := 1024, + ciphers := Ciphers, + port := 18084, + send_timeout := 10000, + verify := Verify, + versions := Versions + } = SocketMaps = maps:from_list(SocketOpts -- [inet]), + %% without tlsv1.1 tlsv1 + ?assertMatch(['tlsv1.3', 'tlsv1.2'], Versions), + ?assert(Ciphers =/= []), + maps:foreach( + fun(K, ConfVal) -> + case maps:find(K, SocketMaps) of + {ok, File} -> ?assertEqual(naive_env_interpolation(ConfVal), File); + error -> ?assertEqual(<<"">>, ConfVal) + end + end, + SSLCert + ), + ?assertMatch( + #{ + env := #{dispatch := {persistent_term, ?NAME}}, + middlewares := Middlewares, + proxy_header := false + }, + ranch:get_protocol_options(?NAME) + ), + ok. + +assert_https_request() -> + Headers = emqx_dashboard_SUITE:auth_header_(), + lists:foreach( + fun(Path) -> + ApiPath = api_path([Path]), + ?assertMatch( + {ok, _}, + emqx_dashboard_SUITE:request_dashboard(get, ApiPath, Headers) + ) + end, + ?OVERVIEWS + ). + +api_path(Parts) -> + ?HOST ++ filename:join([?BASE_PATH | Parts]). + +naive_env_interpolation(Str0) -> + Str1 = emqx_schema:naive_env_interpolation(Str0), + %% assert all envs are replaced + ?assertNot(lists:member($$, Str1)), + Str1. + +default_ssl_cert() -> + #{ + cacertfile => <<"${EMQX_ETC_DIR}/certs/cacert.pem">>, + certfile => <<"${EMQX_ETC_DIR}/certs/cert.pem">>, + keyfile => <<"${EMQX_ETC_DIR}/certs/key.pem">> + }. From cd753622e38ae376963dff44b87cd7e1faf0aafb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Sun, 21 May 2023 07:42:59 +0800 Subject: [PATCH 185/197] chore: add change for fix bad default SSL certificate --- changes/ce/fix-10761.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-10761.en.md diff --git a/changes/ce/fix-10761.en.md b/changes/ce/fix-10761.en.md new file mode 100644 index 000000000..3df84bb86 --- /dev/null +++ b/changes/ce/fix-10761.en.md @@ -0,0 +1 @@ +Fixing the issue where the default value of SSL certificate for Dashboard Listener was not correctly interpolated, which caused HTTPS to be inaccessible when verify_peer and cacertfile were using the default configuration. From 56a6b699ac8bdbd5b677b63e2fe45098fe2d7721 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 19 May 2023 18:07:06 +0800 Subject: [PATCH 186/197] fix: port the `emqx_calendar` from v4.4 --- apps/emqx/src/emqx_calendar.erl | 456 ++++++++++++++++++ apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 37 +- .../test/emqx_rule_funcs_SUITE.erl | 42 +- 3 files changed, 489 insertions(+), 46 deletions(-) create mode 100644 apps/emqx/src/emqx_calendar.erl diff --git a/apps/emqx/src/emqx_calendar.erl b/apps/emqx/src/emqx_calendar.erl new file mode 100644 index 000000000..17218c6a5 --- /dev/null +++ b/apps/emqx/src/emqx_calendar.erl @@ -0,0 +1,456 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_calendar). + +-define(SECONDS_PER_MINUTE, 60). +-define(SECONDS_PER_HOUR, 3600). +-define(SECONDS_PER_DAY, 86400). +-define(DAYS_PER_YEAR, 365). +-define(DAYS_PER_LEAP_YEAR, 366). +-define(DAYS_FROM_0_TO_1970, 719528). +-define(SECONDS_FROM_0_TO_1970, ?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY). + +-export([ + formatter/1, + format/3, + format/4, + parse/3, + offset_second/1 +]). + +-define(DATE_PART, [ + year, + month, + day, + hour, + minute, + second, + nanosecond, + millisecond, + microsecond +]). + +-define(DATE_ZONE_NAME, [ + timezone, + timezone1, + timezone2 +]). + +formatter(FormatterStr) when is_list(FormatterStr) -> + formatter(list_to_binary(FormatterStr)); +formatter(FormatterBin) when is_binary(FormatterBin) -> + do_formatter(FormatterBin, []). + +offset_second(Offset) -> + offset_second_(Offset). + +format(Time, Unit, Formatter) -> + format(Time, Unit, undefined, Formatter). + +format(Time, Unit, Offset, FormatterBin) when is_binary(FormatterBin) -> + format(Time, Unit, Offset, formatter(FormatterBin)); +format(Time, Unit, Offset, Formatter) -> + do_format(Time, time_unit(Unit), offset_second(Offset), Formatter). + +parse(DateStr, Unit, FormatterBin) when is_binary(FormatterBin) -> + parse(DateStr, Unit, formatter(FormatterBin)); +parse(DateStr, Unit, Formatter) -> + do_parse(DateStr, Unit, Formatter). +%% ------------------------------------------------------------------------------------------------- +%% internal + +time_unit(second) -> second; +time_unit(millisecond) -> millisecond; +time_unit(microsecond) -> microsecond; +time_unit(nanosecond) -> nanosecond; +time_unit("second") -> second; +time_unit("millisecond") -> millisecond; +time_unit("microsecond") -> microsecond; +time_unit("nanosecond") -> nanosecond; +time_unit(<<"second">>) -> second; +time_unit(<<"millisecond">>) -> millisecond; +time_unit(<<"microsecond">>) -> microsecond; +time_unit(<<"nanosecond">>) -> nanosecond. + +%% ------------------------------------------------------------------------------------------------- +%% internal: format part + +do_formatter(<<>>, Formatter) -> + lists:reverse(Formatter); +do_formatter(<<"%Y", Tail/binary>>, Formatter) -> + do_formatter(Tail, [year | Formatter]); +do_formatter(<<"%m", Tail/binary>>, Formatter) -> + do_formatter(Tail, [month | Formatter]); +do_formatter(<<"%d", Tail/binary>>, Formatter) -> + do_formatter(Tail, [day | Formatter]); +do_formatter(<<"%H", Tail/binary>>, Formatter) -> + do_formatter(Tail, [hour | Formatter]); +do_formatter(<<"%M", Tail/binary>>, Formatter) -> + do_formatter(Tail, [minute | Formatter]); +do_formatter(<<"%S", Tail/binary>>, Formatter) -> + do_formatter(Tail, [second | Formatter]); +do_formatter(<<"%N", Tail/binary>>, Formatter) -> + do_formatter(Tail, [nanosecond | Formatter]); +do_formatter(<<"%3N", Tail/binary>>, Formatter) -> + do_formatter(Tail, [millisecond | Formatter]); +do_formatter(<<"%6N", Tail/binary>>, Formatter) -> + do_formatter(Tail, [microsecond | Formatter]); +do_formatter(<<"%z", Tail/binary>>, Formatter) -> + do_formatter(Tail, [timezone | Formatter]); +do_formatter(<<"%:z", Tail/binary>>, Formatter) -> + do_formatter(Tail, [timezone1 | Formatter]); +do_formatter(<<"%::z", Tail/binary>>, Formatter) -> + do_formatter(Tail, [timezone2 | Formatter]); +do_formatter(<>, [Str | Formatter]) when is_list(Str) -> + do_formatter(Tail, [lists:append(Str, [Char]) | Formatter]); +do_formatter(<>, Formatter) -> + do_formatter(Tail, [[Char] | Formatter]). + +offset_second_(OffsetSecond) when is_integer(OffsetSecond) -> OffsetSecond; +offset_second_(undefined) -> + 0; +offset_second_("local") -> + offset_second_(local); +offset_second_(<<"local">>) -> + offset_second_(local); +offset_second_(local) -> + UniversalTime = calendar:system_time_to_universal_time(erlang:system_time(second), second), + LocalTime = erlang:universaltime_to_localtime(UniversalTime), + LocalSecs = calendar:datetime_to_gregorian_seconds(LocalTime), + UniversalSecs = calendar:datetime_to_gregorian_seconds(UniversalTime), + LocalSecs - UniversalSecs; +offset_second_(Offset) when is_binary(Offset) -> + offset_second_(erlang:binary_to_list(Offset)); +offset_second_("Z") -> + 0; +offset_second_("z") -> + 0; +offset_second_(Offset) when is_list(Offset) -> + Sign = hd(Offset), + ((Sign == $+) orelse (Sign == $-)) orelse + error({bad_time_offset, Offset}), + Signs = #{$+ => 1, $- => -1}, + PosNeg = maps:get(Sign, Signs), + [Sign | HM] = Offset, + {HourStr, MinuteStr, SecondStr} = + case string:tokens(HM, ":") of + [H, M] -> + {H, M, "0"}; + [H, M, S] -> + {H, M, S}; + [HHMM] when erlang:length(HHMM) == 4 -> + {string:sub_string(HHMM, 1, 2), string:sub_string(HHMM, 3, 4), "0"}; + _ -> + error({bad_time_offset, Offset}) + end, + Hour = erlang:list_to_integer(HourStr), + Minute = erlang:list_to_integer(MinuteStr), + Second = erlang:list_to_integer(SecondStr), + (Hour =< 23) orelse error({bad_time_offset_hour, Hour}), + (Minute =< 59) orelse error({bad_time_offset_minute, Minute}), + (Second =< 59) orelse error({bad_time_offset_second, Second}), + PosNeg * (Hour * 3600 + Minute * 60 + Second). + +do_format(Time, Unit, Offset, Formatter) -> + Adjustment = erlang:convert_time_unit(Offset, second, Unit), + AdjustedTime = Time + Adjustment, + Factor = factor(Unit), + Secs = AdjustedTime div Factor, + DateTime = system_time_to_datetime(Secs), + {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime, + Date = #{ + year => padding(Year, 4), + month => padding(Month, 2), + day => padding(Day, 2), + hour => padding(Hour, 2), + minute => padding(Min, 2), + second => padding(Sec, 2), + millisecond => trans_x_second(Unit, millisecond, Time), + microsecond => trans_x_second(Unit, microsecond, Time), + nanosecond => trans_x_second(Unit, nanosecond, Time) + }, + Timezones = formatter_timezones(Offset, Formatter, #{}), + DateWithZone = maps:merge(Date, Timezones), + [maps:get(Key, DateWithZone, Key) || Key <- Formatter]. + +formatter_timezones(_Offset, [], Zones) -> + Zones; +formatter_timezones(Offset, [Timezone | Formatter], Zones) -> + case lists:member(Timezone, [timezone, timezone1, timezone2]) of + true -> + NZones = Zones#{Timezone => offset_to_timezone(Offset, Timezone)}, + formatter_timezones(Offset, Formatter, NZones); + false -> + formatter_timezones(Offset, Formatter, Zones) + end. + +offset_to_timezone(Offset, Timezone) -> + Sign = + case Offset >= 0 of + true -> + $+; + false -> + $- + end, + {H, M, S} = seconds_to_time(abs(Offset)), + %% TODO: Support zone define %:::z + %% Numeric time zone with ":" to necessary precision (e.g., -04, +05:30). + case Timezone of + timezone -> + %% +0800 + io_lib:format("~c~2.10.0B~2.10.0B", [Sign, H, M]); + timezone1 -> + %% +08:00 + io_lib:format("~c~2.10.0B:~2.10.0B", [Sign, H, M]); + timezone2 -> + %% +08:00:00 + io_lib:format("~c~2.10.0B:~2.10.0B:~2.10.0B", [Sign, H, M, S]) + end. + +factor(second) -> 1; +factor(millisecond) -> 1000; +factor(microsecond) -> 1000000; +factor(nanosecond) -> 1000000000. + +system_time_to_datetime(Seconds) -> + gregorian_seconds_to_datetime(Seconds + ?SECONDS_FROM_0_TO_1970). + +gregorian_seconds_to_datetime(Secs) when Secs >= 0 -> + Days = Secs div ?SECONDS_PER_DAY, + Rest = Secs rem ?SECONDS_PER_DAY, + {gregorian_days_to_date(Days), seconds_to_time(Rest)}. + +seconds_to_time(Secs) when Secs >= 0, Secs < ?SECONDS_PER_DAY -> + Secs0 = Secs rem ?SECONDS_PER_DAY, + Hour = Secs0 div ?SECONDS_PER_HOUR, + Secs1 = Secs0 rem ?SECONDS_PER_HOUR, + Minute = Secs1 div ?SECONDS_PER_MINUTE, + Second = Secs1 rem ?SECONDS_PER_MINUTE, + {Hour, Minute, Second}. + +gregorian_days_to_date(Days) -> + {Year, DayOfYear} = day_to_year(Days), + {Month, DayOfMonth} = year_day_to_date(Year, DayOfYear), + {Year, Month, DayOfMonth}. + +day_to_year(DayOfEpoch) when DayOfEpoch >= 0 -> + YMax = DayOfEpoch div ?DAYS_PER_YEAR, + YMin = DayOfEpoch div ?DAYS_PER_LEAP_YEAR, + {Y1, D1} = dty(YMin, YMax, DayOfEpoch, dy(YMin), dy(YMax)), + {Y1, DayOfEpoch - D1}. + +year_day_to_date(Year, DayOfYear) -> + ExtraDay = + case is_leap_year(Year) of + true -> + 1; + false -> + 0 + end, + {Month, Day} = year_day_to_date2(ExtraDay, DayOfYear), + {Month, Day + 1}. + +dty(Min, Max, _D1, DMin, _DMax) when Min == Max -> + {Min, DMin}; +dty(Min, Max, D1, DMin, DMax) -> + Diff = Max - Min, + Mid = Min + Diff * (D1 - DMin) div (DMax - DMin), + MidLength = + case is_leap_year(Mid) of + true -> + ?DAYS_PER_LEAP_YEAR; + false -> + ?DAYS_PER_YEAR + end, + case dy(Mid) of + D2 when D1 < D2 -> + NewMax = Mid - 1, + dty(Min, NewMax, D1, DMin, dy(NewMax)); + D2 when D1 - D2 >= MidLength -> + NewMin = Mid + 1, + dty(NewMin, Max, D1, dy(NewMin), DMax); + D2 -> + {Mid, D2} + end. + +dy(Y) when Y =< 0 -> + 0; +dy(Y) -> + X = Y - 1, + X div 4 - X div 100 + X div 400 + X * ?DAYS_PER_YEAR + ?DAYS_PER_LEAP_YEAR. + +is_leap_year(Y) when is_integer(Y), Y >= 0 -> + is_leap_year1(Y). + +is_leap_year1(Year) when Year rem 4 =:= 0, Year rem 100 > 0 -> + true; +is_leap_year1(Year) when Year rem 400 =:= 0 -> + true; +is_leap_year1(_) -> + false. + +year_day_to_date2(_, Day) when Day < 31 -> + {1, Day}; +year_day_to_date2(E, Day) when 31 =< Day, Day < 59 + E -> + {2, Day - 31}; +year_day_to_date2(E, Day) when 59 + E =< Day, Day < 90 + E -> + {3, Day - (59 + E)}; +year_day_to_date2(E, Day) when 90 + E =< Day, Day < 120 + E -> + {4, Day - (90 + E)}; +year_day_to_date2(E, Day) when 120 + E =< Day, Day < 151 + E -> + {5, Day - (120 + E)}; +year_day_to_date2(E, Day) when 151 + E =< Day, Day < 181 + E -> + {6, Day - (151 + E)}; +year_day_to_date2(E, Day) when 181 + E =< Day, Day < 212 + E -> + {7, Day - (181 + E)}; +year_day_to_date2(E, Day) when 212 + E =< Day, Day < 243 + E -> + {8, Day - (212 + E)}; +year_day_to_date2(E, Day) when 243 + E =< Day, Day < 273 + E -> + {9, Day - (243 + E)}; +year_day_to_date2(E, Day) when 273 + E =< Day, Day < 304 + E -> + {10, Day - (273 + E)}; +year_day_to_date2(E, Day) when 304 + E =< Day, Day < 334 + E -> + {11, Day - (304 + E)}; +year_day_to_date2(E, Day) when 334 + E =< Day -> + {12, Day - (334 + E)}. + +trans_x_second(FromUnit, ToUnit, Time) -> + XSecond = do_trans_x_second(FromUnit, ToUnit, Time), + Len = + case ToUnit of + millisecond -> 3; + microsecond -> 6; + nanosecond -> 9 + end, + padding(XSecond, Len). + +do_trans_x_second(second, _, _Time) -> 0; +do_trans_x_second(millisecond, millisecond, Time) -> Time rem 1000; +do_trans_x_second(millisecond, microsecond, Time) -> (Time rem 1000) * 1000; +do_trans_x_second(millisecond, nanosecond, Time) -> (Time rem 1000) * 1000_000; +do_trans_x_second(microsecond, millisecond, Time) -> Time div 1000 rem 1000; +do_trans_x_second(microsecond, microsecond, Time) -> Time rem 1000000; +do_trans_x_second(microsecond, nanosecond, Time) -> (Time rem 1000000) * 1000; +do_trans_x_second(nanosecond, millisecond, Time) -> Time div 1000000 rem 1000; +do_trans_x_second(nanosecond, microsecond, Time) -> Time div 1000 rem 1000000; +do_trans_x_second(nanosecond, nanosecond, Time) -> Time rem 1000000000. + +padding(Data, Len) when is_integer(Data) -> + padding(integer_to_list(Data), Len); +padding(Data, Len) when Len > 0 andalso erlang:length(Data) < Len -> + [$0 | padding(Data, Len - 1)]; +padding(Data, _Len) -> + Data. + +%% ------------------------------------------------------------------------------------------------- +%% internal +%% parse part + +do_parse(DateStr, Unit, Formatter) -> + DateInfo = do_parse_date_str(DateStr, Formatter, #{}), + {Precise, PrecisionUnit} = precision(DateInfo), + Counter = + fun + (year, V, Res) -> + Res + dy(V) * ?SECONDS_PER_DAY * Precise - (?SECONDS_FROM_0_TO_1970 * Precise); + (month, V, Res) -> + Res + dm(V) * ?SECONDS_PER_DAY * Precise; + (day, V, Res) -> + Res + (V * ?SECONDS_PER_DAY * Precise); + (hour, V, Res) -> + Res + (V * ?SECONDS_PER_HOUR * Precise); + (minute, V, Res) -> + Res + (V * ?SECONDS_PER_MINUTE * Precise); + (second, V, Res) -> + Res + V * Precise; + (millisecond, V, Res) -> + case PrecisionUnit of + millisecond -> + Res + V; + microsecond -> + Res + (V * 1000); + nanosecond -> + Res + (V * 1000000) + end; + (microsecond, V, Res) -> + case PrecisionUnit of + microsecond -> + Res + V; + nanosecond -> + Res + (V * 1000) + end; + (nanosecond, V, Res) -> + Res + V; + (parsed_offset, V, Res) -> + Res - V + end, + Count = maps:fold(Counter, 0, DateInfo) - (?SECONDS_PER_DAY * Precise), + erlang:convert_time_unit(Count, PrecisionUnit, Unit). + +precision(#{nanosecond := _}) -> {1000_000_000, nanosecond}; +precision(#{microsecond := _}) -> {1000_000, microsecond}; +precision(#{millisecond := _}) -> {1000, millisecond}; +precision(#{second := _}) -> {1, second}; +precision(_) -> {1, second}. + +do_parse_date_str(<<>>, _, Result) -> + Result; +do_parse_date_str(_, [], Result) -> + Result; +do_parse_date_str(Date, [Key | Formatter], Result) -> + Size = date_size(Key), + <> = Date, + case lists:member(Key, ?DATE_PART) of + true -> + do_parse_date_str(Tail, Formatter, Result#{Key => erlang:binary_to_integer(DatePart)}); + false -> + case lists:member(Key, ?DATE_ZONE_NAME) of + true -> + do_parse_date_str(Tail, Formatter, Result#{ + parsed_offset => offset_second(DatePart) + }); + false -> + do_parse_date_str(Tail, Formatter, Result) + end + end. + +date_size(Str) when is_list(Str) -> erlang:length(Str); +date_size(year) -> 4; +date_size(month) -> 2; +date_size(day) -> 2; +date_size(hour) -> 2; +date_size(minute) -> 2; +date_size(second) -> 2; +date_size(millisecond) -> 3; +date_size(microsecond) -> 6; +date_size(nanosecond) -> 9; +date_size(timezone) -> 5; +date_size(timezone1) -> 6; +date_size(timezone2) -> 9. + +dm(1) -> 0; +dm(2) -> 31; +dm(3) -> 59; +dm(4) -> 90; +dm(5) -> 120; +dm(6) -> 151; +dm(7) -> 181; +dm(8) -> 212; +dm(9) -> 243; +dm(10) -> 273; +dm(11) -> 304; +dm(12) -> 334. diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 02163f95b..47017f718 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -1075,37 +1075,34 @@ now_timestamp(Unit) -> time_unit(<<"second">>) -> second; time_unit(<<"millisecond">>) -> millisecond; time_unit(<<"microsecond">>) -> microsecond; -time_unit(<<"nanosecond">>) -> nanosecond. +time_unit(<<"nanosecond">>) -> nanosecond; +time_unit(second) -> second; +time_unit(millisecond) -> millisecond; +time_unit(microsecond) -> microsecond; +time_unit(nanosecond) -> nanosecond. format_date(TimeUnit, Offset, FormatString) -> - emqx_plugin_libs_rule:bin( - emqx_rule_date:date( - time_unit(TimeUnit), - emqx_plugin_libs_rule:str(Offset), - emqx_plugin_libs_rule:str(FormatString) - ) - ). + Unit = time_unit(TimeUnit), + TimeEpoch = erlang:system_time(Unit), + format_date(Unit, Offset, FormatString, TimeEpoch). format_date(TimeUnit, Offset, FormatString, TimeEpoch) -> + Unit = time_unit(TimeUnit), emqx_plugin_libs_rule:bin( - emqx_rule_date:date( - time_unit(TimeUnit), - emqx_plugin_libs_rule:str(Offset), - emqx_plugin_libs_rule:str(FormatString), - TimeEpoch + lists:concat( + emqx_calendar:format(TimeEpoch, Unit, Offset, FormatString) ) ). date_to_unix_ts(TimeUnit, FormatString, InputString) -> - date_to_unix_ts(TimeUnit, "Z", FormatString, InputString). + Unit = time_unit(TimeUnit), + emqx_calendar:parse(InputString, Unit, FormatString). date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) -> - emqx_rule_date:parse_date( - time_unit(TimeUnit), - emqx_plugin_libs_rule:str(Offset), - emqx_plugin_libs_rule:str(FormatString), - emqx_plugin_libs_rule:str(InputString) - ). + Unit = time_unit(TimeUnit), + OffsetSecond = emqx_calendar:offset_second(Offset), + OffsetDelta = erlang:convert_time_unit(OffsetSecond, second, Unit), + date_to_unix_ts(Unit, FormatString, InputString) - OffsetDelta. %% @doc This is for sql funcs that should be handled in the specific modules. %% Here the emqx_rule_funcs module acts as a proxy, forwarding diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index d88637312..9bb6e2e6f 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -955,10 +955,10 @@ t_format_date_funcs(_) -> ?PROPTEST(prop_format_date_fun). prop_format_date_fun() -> - Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%y---%H:%M:%S%Z">>], + Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%Y---%H:%M:%S%z">>], ?FORALL( S, - range(0, 4000000000), + erlang:system_time(second), S == apply_func( date_to_unix_ts, @@ -971,14 +971,15 @@ prop_format_date_fun() -> ] ) ), - Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%y---%H:%M:%S%Z">>], + Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S:%3N%z">>], + Args2DTUS = [<<"millisecond">>, <<"--%m--%d--%Y---%H:%M:%S:%3N%z">>], ?FORALL( S, - range(0, 4000000000), + erlang:system_time(millisecond), S == apply_func( date_to_unix_ts, - Args2 ++ + Args2DTUS ++ [ apply_func( format_date, @@ -987,14 +988,15 @@ prop_format_date_fun() -> ] ) ), - Args = [<<"second">>, <<"+08:00">>, <<"%y-%m-%d-%H:%M:%S%Z">>], + Args = [<<"second">>, <<"+08:00">>, <<"%Y-%m-%d-%H:%M:%S%z">>], + ArgsDTUS = [<<"second">>, <<"%Y-%m-%d-%H:%M:%S%z">>], ?FORALL( S, - range(0, 4000000000), + erlang:system_time(second), S == apply_func( date_to_unix_ts, - Args ++ + ArgsDTUS ++ [ apply_func( format_date, @@ -1003,24 +1005,12 @@ prop_format_date_fun() -> ] ) ), - %% When no offset is specified, the offset should be taken from the formatted time string - ArgsNoOffset = [<<"second">>, <<"%y-%m-%d-%H:%M:%S%Z">>], - ArgsOffset = [<<"second">>, <<"+08:00">>, <<"%y-%m-%d-%H:%M:%S%Z">>], - ?FORALL( - S, - range(0, 4000000000), - S == - apply_func( - date_to_unix_ts, - ArgsNoOffset ++ - [ - apply_func( - format_date, - ArgsOffset ++ [S] - ) - ] - ) - ). + % no offset in format string. force add offset + Second = erlang:system_time(second), + Args3 = [<<"second">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Second], + Formatters3 = apply_func(format_date, Args3), + Args3DTUS = [<<"second">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Formatters3], + Second == apply_func(date_to_unix_ts, Args3DTUS). %%------------------------------------------------------------------------------ %% Utility functions From d3e38bd7f97030cc4ec34d6c80520cc98570c316 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 22 May 2023 10:06:29 +0800 Subject: [PATCH 187/197] chore: update changes --- changes/ce/fix-10747.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-10747.en.md diff --git a/changes/ce/fix-10747.en.md b/changes/ce/fix-10747.en.md new file mode 100644 index 000000000..a07c24074 --- /dev/null +++ b/changes/ce/fix-10747.en.md @@ -0,0 +1 @@ +Refactor date and time functions, `format_date` and `date_to_unix_ts`, in the rule engine to fix the implementation problem. From 27fca0ef3c03a406e638c049f307395e034a71ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Thu, 18 May 2023 21:36:45 +0800 Subject: [PATCH 188/197] fix: check authz's file rule before save to file --- apps/emqx_authz/src/emqx_authz.erl | 34 ++++++++---- apps/emqx_authz/src/emqx_authz_api_schema.erl | 6 +- apps/emqx_authz/src/emqx_authz_file.erl | 2 +- apps/emqx_authz/src/emqx_authz_rule.erl | 8 ++- apps/emqx_authz/test/emqx_authz_SUITE.erl | 55 +++++++++++++++++-- .../emqx_authz/test/emqx_authz_file_SUITE.erl | 4 +- changes/ce/fix-10742.en.md | 2 + 7 files changed, 90 insertions(+), 21 deletions(-) create mode 100644 changes/ce/fix-10742.en.md diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 682ad7f2e..5fffb61d2 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -140,7 +140,12 @@ update(Cmd, Sources) -> emqx_authz_utils:update_config(?CONF_KEY_PATH, {Cmd, Sources}). pre_config_update(_, Cmd, Sources) -> - {ok, do_pre_config_update(Cmd, Sources)}. + try do_pre_config_update(Cmd, Sources) of + {error, Reason} -> {error, Reason}; + NSources -> {ok, NSources} + catch + _:Reason -> {error, Reason} + end. do_pre_config_update({?CMD_MOVE, _, _} = Cmd, Sources) -> do_move(Cmd, Sources); @@ -475,11 +480,12 @@ maybe_write_files(#{<<"type">> := <<"file">>} = Source) -> maybe_write_files(NewSource) -> maybe_write_certs(NewSource). -write_acl_file(#{<<"rules">> := Rules} = Source) -> - NRules = check_acl_file_rules(Rules), - Path = ?MODULE:acl_conf_file(), - {ok, _Filename} = write_file(Path, NRules), - maps:without([<<"rules">>], Source#{<<"path">> => Path}). +write_acl_file(#{<<"rules">> := Rules} = Source0) -> + AclPath = ?MODULE:acl_conf_file(), + ok = check_acl_file_rules(AclPath, Rules), + ok = write_file(AclPath, Rules), + Source1 = maps:remove(<<"rules">>, Source0), + maps:put(<<"path">>, AclPath, Source1). %% @doc where the acl.conf file is stored. acl_conf_file() -> @@ -506,7 +512,7 @@ write_file(Filename, Bytes) -> ok = filelib:ensure_dir(Filename), case file:write_file(Filename, Bytes) of ok -> - {ok, iolist_to_binary(Filename)}; + ok; {error, Reason} -> ?SLOG(error, #{filename => Filename, msg => "write_file_error", reason => Reason}), throw(Reason) @@ -528,6 +534,14 @@ get_source_by_type(Type, Sources) -> update_authz_chain(Actions) -> emqx_hooks:put('client.authorize', {?MODULE, authorize, [Actions]}, ?HP_AUTHZ). -check_acl_file_rules(RawRules) -> - %% TODO: make sure the bin rules checked - RawRules. +check_acl_file_rules(Path, Rules) -> + TmpPath = Path ++ ".tmp", + try + ok = write_file(Path, Rules), + #{annotations := #{rules := _}} = emqx_authz_file:create(#{path => Path}), + ok + catch + throw:Reason -> throw(Reason) + after + _ = file:delete(TmpPath) + end. diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index 4adada182..d6e5a3eb2 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -39,8 +39,10 @@ fields(file) -> type => binary(), required => true, example => - <<"{allow,{username,\"^dashboard?\"},", "subscribe,[\"$SYS/#\"]}.\n", - "{allow,{ipaddr,\"127.0.0.1\"},all,[\"$SYS/#\",\"#\"]}.">>, + << + "{allow,{username,{re,\"^dashboard$\"}},subscribe,[\"$SYS/#\"]}.\n", + "{allow,{ipaddr,\"127.0.0.1\"},all,[\"$SYS/#\",\"#\"]}." + >>, desc => ?DESC(rules) }} ]; diff --git a/apps/emqx_authz/src/emqx_authz_file.erl b/apps/emqx_authz/src/emqx_authz_file.erl index 54f1775c6..dfd28f0c0 100644 --- a/apps/emqx_authz/src/emqx_authz_file.erl +++ b/apps/emqx_authz/src/emqx_authz_file.erl @@ -54,7 +54,7 @@ create(#{path := Path0} = Source) -> throw(failed_to_read_acl_file); {error, Reason} -> ?SLOG(alert, #{msg => bad_acl_file_content, path => Path, reason => Reason}), - throw(bad_acl_file_content) + throw({bad_acl_file_content, Reason}) end, Source#{annotations => #{rules => Rules}}. diff --git a/apps/emqx_authz/src/emqx_authz_rule.erl b/apps/emqx_authz/src/emqx_authz_rule.erl index bdd0904f7..ec1a8c5de 100644 --- a/apps/emqx_authz/src/emqx_authz_rule.erl +++ b/apps/emqx_authz/src/emqx_authz_rule.erl @@ -68,7 +68,13 @@ compile({Permission, Who, Action, TopicFilters}) when {atom(Permission), compile_who(Who), atom(Action), [ compile_topic(Topic) || Topic <- TopicFilters - ]}. + ]}; +compile({Permission, _Who, _Action, _TopicFilter}) when not ?ALLOW_DENY(Permission) -> + throw({invalid_authorization_permission, Permission}); +compile({_Permission, _Who, Action, _TopicFilter}) when not ?PUBSUB(Action) -> + throw({invalid_authorization_action, Action}); +compile(BadRule) -> + throw({invalid_authorization_rule, BadRule}). compile_who(all) -> all; diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index 84b1d903e..2c57d807a 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -155,22 +155,36 @@ set_special_configs(_App) -> <<"ssl">> => #{<<"enable">> => false}, <<"cmd">> => <<"HGETALL mqtt_authz:", ?PH_USERNAME/binary>> }). --define(SOURCE6, #{ + +-define(FILE_SOURCE(Rules), #{ <<"type">> => <<"file">>, <<"enable">> => true, - <<"rules">> => + <<"rules">> => Rules +}). + +-define(SOURCE6, + ?FILE_SOURCE( << "{allow,{username,\"^dashboard?\"},subscribe,[\"$SYS/#\"]}." "\n{allow,{ipaddr,\"127.0.0.1\"},all,[\"$SYS/#\",\"#\"]}." >> -}). --define(SOURCE7, #{ + ) +). +-define(SOURCE7, + ?FILE_SOURCE( + << + "{allow,{username,\"some_client\"},publish,[\"some_client/lwt\"]}.\n" + "{deny, all}." + >> + ) +). + +-define(BAD_FILE_SOURCE2, #{ <<"type">> => <<"file">>, <<"enable">> => true, <<"rules">> => << - "{allow,{username,\"some_client\"},publish,[\"some_client/lwt\"]}.\n" - "{deny, all}." + "{not_allow,{username,\"some_client\"},publish,[\"some_client/lwt\"]}." >> }). @@ -178,6 +192,35 @@ set_special_configs(_App) -> %% Testcases %%------------------------------------------------------------------------------ +-define(UPDATE_ERROR(Err), {error, {pre_config_update, emqx_authz, Err}}). + +t_bad_file_source(_) -> + BadContent = ?FILE_SOURCE(<<"{allow,{username,\"bar\"}, publish, [\"test\"]}">>), + BadContentErr = {bad_acl_file_content, {1, erl_parse, ["syntax error before: ", []]}}, + BadRule = ?FILE_SOURCE(<<"{allow,{username,\"bar\"},publish}.">>), + BadRuleErr = {invalid_authorization_rule, {allow, {username, "bar"}, publish}}, + BadPermission = ?FILE_SOURCE(<<"{not_allow,{username,\"bar\"},publish,[\"test\"]}.">>), + BadPermissionErr = {invalid_authorization_permission, not_allow}, + BadAction = ?FILE_SOURCE(<<"{allow,{username,\"bar\"},pubsub,[\"test\"]}.">>), + BadActionErr = {invalid_authorization_action, pubsub}, + lists:foreach( + fun({Source, Error}) -> + ?assertEqual(?UPDATE_ERROR(Error), emqx_authz:update(?CMD_REPLACE, [Source])), + ?assertEqual(?UPDATE_ERROR(Error), emqx_authz:update(?CMD_PREPEND, Source)), + ?assertEqual(?UPDATE_ERROR(Error), emqx_authz:update(?CMD_APPEND, Source)) + end, + [ + {BadContent, BadContentErr}, + {BadRule, BadRuleErr}, + {BadPermission, BadPermissionErr}, + {BadAction, BadActionErr} + ] + ), + ?assertMatch( + [], + emqx_conf:get([authorization, sources], []) + ). + t_update_source(_) -> %% replace all {ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE3]), diff --git a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl index 124fe904f..be8907feb 100644 --- a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl @@ -120,7 +120,9 @@ t_superuser(_Config) -> t_invalid_file(_Config) -> ?assertMatch( - {error, bad_acl_file_content}, + {error, + {pre_config_update, emqx_authz, + {bad_acl_file_content, {1, erl_parse, ["syntax error before: ", "term"]}}}}, emqx_authz:update(?CMD_REPLACE, [?RAW_SOURCE#{<<"rules">> => <<"{{invalid term">>}]) ). diff --git a/changes/ce/fix-10742.en.md b/changes/ce/fix-10742.en.md new file mode 100644 index 000000000..cc8232a04 --- /dev/null +++ b/changes/ce/fix-10742.en.md @@ -0,0 +1,2 @@ +Check the correctness of the rules before saving the authorization file source. +Previously, Saving wrong rules could lead to restart failure. From 082214d0397adeaefe90281bcfa4eef0795b4d56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Fri, 19 May 2023 08:55:04 +0800 Subject: [PATCH 189/197] feat: add authz file rule validator --- apps/emqx/test/emqx_common_test_helpers.erl | 12 + apps/emqx_authz/src/emqx_authz.app.src | 2 +- apps/emqx_authz/src/emqx_authz.erl | 5 +- apps/emqx_authz/src/emqx_authz_schema.erl | 24 +- .../emqx_conf/test/emqx_conf_schema_tests.erl | 206 ++++++++++++++++++ 5 files changed, 243 insertions(+), 6 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 40e9ca5fc..15869fb36 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -238,6 +238,8 @@ render_and_load_app_config(App, Opts) -> end. do_render_app_config(App, Schema, ConfigFile, Opts) -> + %% copy acl_conf must run before read_schema_configs + copy_acl_conf(), Vars = mustache_vars(App, Opts), RenderedConfigFile = render_config_file(ConfigFile, Vars), read_schema_configs(Schema, RenderedConfigFile), @@ -497,6 +499,16 @@ copy_certs(emqx_conf, Dest0) -> copy_certs(_, _) -> ok. +copy_acl_conf() -> + Dest = filename:join([code:lib_dir(emqx), "etc/acl.conf"]), + case code:lib_dir(emqx_authz) of + {error, bad_name} -> + (not filelib:is_regular(Dest)) andalso file:write_file(Dest, <<"">>); + _ -> + {ok, _} = file:copy(deps_path(emqx_authz, "etc/acl.conf"), Dest) + end, + ok. + load_config(SchemaModule, Config) -> ConfigBin = case is_map(Config) of diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index dd0325694..4856008e6 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.19"}, + {vsn, "0.1.20"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 5fffb61d2..e7f59cb71 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -482,6 +482,8 @@ maybe_write_files(NewSource) -> write_acl_file(#{<<"rules">> := Rules} = Source0) -> AclPath = ?MODULE:acl_conf_file(), + %% Always check if the rules are valid before writing to the file + %% If the rules are invalid, the old file will be kept ok = check_acl_file_rules(AclPath, Rules), ok = write_file(AclPath, Rules), Source1 = maps:remove(<<"rules">>, Source0), @@ -538,8 +540,7 @@ check_acl_file_rules(Path, Rules) -> TmpPath = Path ++ ".tmp", try ok = write_file(Path, Rules), - #{annotations := #{rules := _}} = emqx_authz_file:create(#{path => Path}), - ok + emqx_authz_schema:validate_file_rules(Path) catch throw:Reason -> throw(Reason) after diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index a2a7c6b52..af4a04d96 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -42,7 +42,8 @@ -export([ headers_no_content_type/1, - headers/1 + headers/1, + validate_file_rules/1 ]). %%-------------------------------------------------------------------- @@ -78,7 +79,17 @@ fields("authorization") -> authz_fields(); fields(file) -> authz_common_fields(file) ++ - [{path, ?HOCON(string(), #{required => true, desc => ?DESC(path)})}]; + [ + {path, + ?HOCON( + string(), + #{ + required => true, + validator => fun ?MODULE:validate_file_rules/1, + desc => ?DESC(path) + } + )} + ]; fields(http_get) -> authz_common_fields(http) ++ http_common_fields() ++ @@ -496,7 +507,7 @@ authz_fields() -> %% doc_lift is force a root level reference instead of nesting sub-structs extra => #{doc_lift => true}, %% it is recommended to configure authz sources from dashboard - %% hance the importance level for config is low + %% hence the importance level for config is low importance => ?IMPORTANCE_LOW } )} @@ -508,3 +519,10 @@ default_authz() -> <<"enable">> => true, <<"path">> => <<"${EMQX_ETC_DIR}/acl.conf">> }. + +validate_file_rules(Path) -> + %% Don't need assert the create result here, all error is thrown + %% some test mock the create function + %% #{annotations := #{rules := _}} + _ = emqx_authz_file:create(#{path => Path}), + ok. diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 79fe30293..d94975863 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -23,6 +23,7 @@ """). array_nodes_test() -> + ensure_acl_conf(), ExpectNodes = ['emqx1@127.0.0.1', 'emqx2@127.0.0.1'], lists:foreach( fun(Nodes) -> @@ -47,6 +48,200 @@ array_nodes_test() -> ), ok. +%% erlfmt-ignore +-define(OUTDATED_LOG_CONF, + """ +log.console_handler { + burst_limit { + enable = true + max_count = 10000 + window_time = 1000 + } + chars_limit = unlimited + drop_mode_qlen = 3000 + enable = true + flush_qlen = 8000 + formatter = text + level = warning + max_depth = 100 + overload_kill { + enable = true + mem_size = 31457280 + qlen = 20000 + restart_after = 5000 + } + single_line = true + supervisor_reports = error + sync_mode_qlen = 100 + time_offset = \"+02:00\" +} +log.file_handlers { + default { + burst_limit { + enable = true + max_count = 10000 + window_time = 1000 + } + chars_limit = unlimited + drop_mode_qlen = 3000 + enable = true + file = \"log/my-emqx.log\" + flush_qlen = 8000 + formatter = text + level = debug + max_depth = 100 + max_size = \"1024MB\" + overload_kill { + enable = true + mem_size = 31457280 + qlen = 20000 + restart_after = 5000 + } + rotation {count = 20, enable = true} + single_line = true + supervisor_reports = error + sync_mode_qlen = 100 + time_offset = \"+01:00\" + } +} + """ +). +-define(FORMATTER(TimeOffset), + {emqx_logger_textfmt, #{ + chars_limit => unlimited, + depth => 100, + single_line => true, + template => [time, " [", level, "] ", msg, "\n"], + time_offset => TimeOffset + }} +). + +-define(FILTERS, [{drop_progress_reports, {fun logger_filters:progress/2, stop}}]). +-define(LOG_CONFIG, #{ + burst_limit_enable => true, + burst_limit_max_count => 10000, + burst_limit_window_time => 1000, + drop_mode_qlen => 3000, + flush_qlen => 8000, + overload_kill_enable => true, + overload_kill_mem_size => 31457280, + overload_kill_qlen => 20000, + overload_kill_restart_after => 5000, + sync_mode_qlen => 100 +}). + +outdated_log_test() -> + validate_log(?OUTDATED_LOG_CONF). + +validate_log(Conf) -> + ensure_acl_conf(), + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + Conf0 = <>, + {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), + ConfList = hocon_tconf:generate(emqx_conf_schema, ConfMap0), + Kernel = proplists:get_value(kernel, ConfList), + + ?assertEqual(silent, proplists:get_value(error_logger, Kernel)), + ?assertEqual(debug, proplists:get_value(logger_level, Kernel)), + Loggers = proplists:get_value(logger, Kernel), + FileHandler = lists:keyfind(logger_disk_log_h, 3, Loggers), + ?assertEqual( + {handler, default, logger_disk_log_h, #{ + config => ?LOG_CONFIG#{ + type => wrap, + file => "log/my-emqx.log", + max_no_bytes => 1073741824, + max_no_files => 20 + }, + filesync_repeat_interval => no_repeat, + filters => ?FILTERS, + formatter => ?FORMATTER("+01:00"), + level => debug + }}, + FileHandler + ), + ConsoleHandler = lists:keyfind(logger_std_h, 3, Loggers), + ?assertEqual( + {handler, console, logger_std_h, #{ + config => ?LOG_CONFIG#{type => standard_io}, + filters => ?FILTERS, + formatter => ?FORMATTER("+02:00"), + level => warning + }}, + ConsoleHandler + ). + +%% erlfmt-ignore +-define(KERNEL_LOG_CONF, + """ + log.console { + enable = true + formatter = text + level = warning + time_offset = \"+02:00\" + } + log.file { + enable = false + file = \"log/xx-emqx.log\" + formatter = text + level = debug + rotation_count = 20 + rotation_size = \"1024MB\" + time_offset = \"+01:00\" + } + log.file_handlers.default { + enable = true + file = \"log/my-emqx.log\" + } + """ +). + +log_test() -> + validate_log(?KERNEL_LOG_CONF). + +%% erlfmt-ignore +log_rotation_count_limit_test() -> + ensure_acl_conf(), + Format = + """ + log.file { + enable = true + to = \"log/emqx.log\" + formatter = text + level = debug + rotation = {count = ~w} + rotation_size = \"1024MB\" + } + """, + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + lists:foreach(fun({Conf, Count}) -> + Conf0 = <>, + {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), + ConfList = hocon_tconf:generate(emqx_conf_schema, ConfMap0), + Kernel = proplists:get_value(kernel, ConfList), + Loggers = proplists:get_value(logger, Kernel), + ?assertMatch( + {handler, default, logger_disk_log_h, #{ + config := #{max_no_files := Count} + }}, + lists:keyfind(logger_disk_log_h, 3, Loggers) + ) + end, + [{to_bin(Format, [1]), 1}, {to_bin(Format, [128]), 128}]), + lists:foreach(fun({Conf, Count}) -> + Conf0 = <>, + {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), + ?assertThrow({emqx_conf_schema, + [#{kind := validation_error, + mismatches := #{"handler_name" := + #{kind := validation_error, + path := "log.file.default.rotation_count", + reason := #{expected_type := "1..128"}, + value := Count} + }}]}, + hocon_tconf:generate(emqx_conf_schema, ConfMap0)) + end, [{to_bin(Format, [0]), 0}, {to_bin(Format, [129]), 129}]). + %% erlfmt-ignore -define(BASE_AUTHN_ARRAY, """ @@ -79,6 +274,7 @@ array_nodes_test() -> ). authn_validations_test() -> + ensure_acl_conf(), BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), OKHttps = to_bin(?BASE_AUTHN_ARRAY, [post, true, <<"https://127.0.0.1:8080">>]), @@ -128,6 +324,7 @@ authn_validations_test() -> ). listeners_test() -> + ensure_acl_conf(), BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), Conf = <>, @@ -198,6 +395,7 @@ listeners_test() -> ok. doc_gen_test() -> + ensure_acl_conf(), %% the json file too large to encode. { timeout, @@ -220,3 +418,11 @@ doc_gen_test() -> to_bin(Format, Args) -> iolist_to_binary(io_lib:format(Format, Args)). + +ensure_acl_conf() -> + File = emqx_schema:naive_env_interpolation(<<"${EMQX_ETC_DIR}/acl.conf">>), + ok = filelib:ensure_dir(filename:dirname(File)), + case filelib:is_regular(File) of + true -> ok; + false -> file:write_file(File, <<"">>) + end. From 218fc4a83907fb92ca66d42208314ffa99add4b0 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 22 May 2023 11:03:23 +0800 Subject: [PATCH 190/197] refactor: add emqx_authz_file validate function --- apps/emqx_authz/src/emqx_authz.erl | 5 +- apps/emqx_authz/src/emqx_authz_file.erl | 7 +- apps/emqx_authz/src/emqx_authz_schema.erl | 12 +- apps/emqx_authz/test/emqx_authz_SUITE.erl | 7 +- .../emqx_conf/test/emqx_conf_schema_tests.erl | 194 ------------------ 5 files changed, 17 insertions(+), 208 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index e7f59cb71..c7db65992 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -539,8 +539,9 @@ update_authz_chain(Actions) -> check_acl_file_rules(Path, Rules) -> TmpPath = Path ++ ".tmp", try - ok = write_file(Path, Rules), - emqx_authz_schema:validate_file_rules(Path) + ok = write_file(TmpPath, Rules), + {ok, _} = emqx_authz_file:validate(TmpPath), + ok catch throw:Reason -> throw(Reason) after diff --git a/apps/emqx_authz/src/emqx_authz_file.erl b/apps/emqx_authz/src/emqx_authz_file.erl index dfd28f0c0..317395a45 100644 --- a/apps/emqx_authz/src/emqx_authz_file.erl +++ b/apps/emqx_authz/src/emqx_authz_file.erl @@ -33,13 +33,14 @@ update/1, destroy/1, authorize/4, + validate/1, read_file/1 ]). description() -> "AuthZ with static rules". -create(#{path := Path0} = Source) -> +validate(Path0) -> Path = filename(Path0), Rules = case file:consult(Path) of @@ -56,6 +57,10 @@ create(#{path := Path0} = Source) -> ?SLOG(alert, #{msg => bad_acl_file_content, path => Path, reason => Reason}), throw({bad_acl_file_content, Reason}) end, + {ok, Rules}. + +create(#{path := Path} = Source) -> + {ok, Rules} = validate(Path), Source#{annotations => #{rules => Rules}}. update(#{path := _Path} = Source) -> diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index af4a04d96..3b318c39f 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -42,8 +42,7 @@ -export([ headers_no_content_type/1, - headers/1, - validate_file_rules/1 + headers/1 ]). %%-------------------------------------------------------------------- @@ -85,7 +84,7 @@ fields(file) -> string(), #{ required => true, - validator => fun ?MODULE:validate_file_rules/1, + validator => fun(Path) -> element(1, emqx_authz_file:validate(Path)) end, desc => ?DESC(path) } )} @@ -519,10 +518,3 @@ default_authz() -> <<"enable">> => true, <<"path">> => <<"${EMQX_ETC_DIR}/acl.conf">> }. - -validate_file_rules(Path) -> - %% Don't need assert the create result here, all error is thrown - %% some test mock the create function - %% #{annotations := #{rules := _}} - _ = emqx_authz_file:create(#{path => Path}), - ok. diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index 2c57d807a..9c1b7fd51 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -205,9 +205,14 @@ t_bad_file_source(_) -> BadActionErr = {invalid_authorization_action, pubsub}, lists:foreach( fun({Source, Error}) -> + File = emqx_authz:acl_conf_file(), + {ok, Bin1} = file:read_file(File), ?assertEqual(?UPDATE_ERROR(Error), emqx_authz:update(?CMD_REPLACE, [Source])), ?assertEqual(?UPDATE_ERROR(Error), emqx_authz:update(?CMD_PREPEND, Source)), - ?assertEqual(?UPDATE_ERROR(Error), emqx_authz:update(?CMD_APPEND, Source)) + ?assertEqual(?UPDATE_ERROR(Error), emqx_authz:update(?CMD_APPEND, Source)), + %% Check file content not changed if update failed + {ok, Bin2} = file:read_file(File), + ?assertEqual(Bin1, Bin2) end, [ {BadContent, BadContentErr}, diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index d94975863..06784e32d 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -48,200 +48,6 @@ array_nodes_test() -> ), ok. -%% erlfmt-ignore --define(OUTDATED_LOG_CONF, - """ -log.console_handler { - burst_limit { - enable = true - max_count = 10000 - window_time = 1000 - } - chars_limit = unlimited - drop_mode_qlen = 3000 - enable = true - flush_qlen = 8000 - formatter = text - level = warning - max_depth = 100 - overload_kill { - enable = true - mem_size = 31457280 - qlen = 20000 - restart_after = 5000 - } - single_line = true - supervisor_reports = error - sync_mode_qlen = 100 - time_offset = \"+02:00\" -} -log.file_handlers { - default { - burst_limit { - enable = true - max_count = 10000 - window_time = 1000 - } - chars_limit = unlimited - drop_mode_qlen = 3000 - enable = true - file = \"log/my-emqx.log\" - flush_qlen = 8000 - formatter = text - level = debug - max_depth = 100 - max_size = \"1024MB\" - overload_kill { - enable = true - mem_size = 31457280 - qlen = 20000 - restart_after = 5000 - } - rotation {count = 20, enable = true} - single_line = true - supervisor_reports = error - sync_mode_qlen = 100 - time_offset = \"+01:00\" - } -} - """ -). --define(FORMATTER(TimeOffset), - {emqx_logger_textfmt, #{ - chars_limit => unlimited, - depth => 100, - single_line => true, - template => [time, " [", level, "] ", msg, "\n"], - time_offset => TimeOffset - }} -). - --define(FILTERS, [{drop_progress_reports, {fun logger_filters:progress/2, stop}}]). --define(LOG_CONFIG, #{ - burst_limit_enable => true, - burst_limit_max_count => 10000, - burst_limit_window_time => 1000, - drop_mode_qlen => 3000, - flush_qlen => 8000, - overload_kill_enable => true, - overload_kill_mem_size => 31457280, - overload_kill_qlen => 20000, - overload_kill_restart_after => 5000, - sync_mode_qlen => 100 -}). - -outdated_log_test() -> - validate_log(?OUTDATED_LOG_CONF). - -validate_log(Conf) -> - ensure_acl_conf(), - BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), - Conf0 = <>, - {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), - ConfList = hocon_tconf:generate(emqx_conf_schema, ConfMap0), - Kernel = proplists:get_value(kernel, ConfList), - - ?assertEqual(silent, proplists:get_value(error_logger, Kernel)), - ?assertEqual(debug, proplists:get_value(logger_level, Kernel)), - Loggers = proplists:get_value(logger, Kernel), - FileHandler = lists:keyfind(logger_disk_log_h, 3, Loggers), - ?assertEqual( - {handler, default, logger_disk_log_h, #{ - config => ?LOG_CONFIG#{ - type => wrap, - file => "log/my-emqx.log", - max_no_bytes => 1073741824, - max_no_files => 20 - }, - filesync_repeat_interval => no_repeat, - filters => ?FILTERS, - formatter => ?FORMATTER("+01:00"), - level => debug - }}, - FileHandler - ), - ConsoleHandler = lists:keyfind(logger_std_h, 3, Loggers), - ?assertEqual( - {handler, console, logger_std_h, #{ - config => ?LOG_CONFIG#{type => standard_io}, - filters => ?FILTERS, - formatter => ?FORMATTER("+02:00"), - level => warning - }}, - ConsoleHandler - ). - -%% erlfmt-ignore --define(KERNEL_LOG_CONF, - """ - log.console { - enable = true - formatter = text - level = warning - time_offset = \"+02:00\" - } - log.file { - enable = false - file = \"log/xx-emqx.log\" - formatter = text - level = debug - rotation_count = 20 - rotation_size = \"1024MB\" - time_offset = \"+01:00\" - } - log.file_handlers.default { - enable = true - file = \"log/my-emqx.log\" - } - """ -). - -log_test() -> - validate_log(?KERNEL_LOG_CONF). - -%% erlfmt-ignore -log_rotation_count_limit_test() -> - ensure_acl_conf(), - Format = - """ - log.file { - enable = true - to = \"log/emqx.log\" - formatter = text - level = debug - rotation = {count = ~w} - rotation_size = \"1024MB\" - } - """, - BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), - lists:foreach(fun({Conf, Count}) -> - Conf0 = <>, - {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), - ConfList = hocon_tconf:generate(emqx_conf_schema, ConfMap0), - Kernel = proplists:get_value(kernel, ConfList), - Loggers = proplists:get_value(logger, Kernel), - ?assertMatch( - {handler, default, logger_disk_log_h, #{ - config := #{max_no_files := Count} - }}, - lists:keyfind(logger_disk_log_h, 3, Loggers) - ) - end, - [{to_bin(Format, [1]), 1}, {to_bin(Format, [128]), 128}]), - lists:foreach(fun({Conf, Count}) -> - Conf0 = <>, - {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), - ?assertThrow({emqx_conf_schema, - [#{kind := validation_error, - mismatches := #{"handler_name" := - #{kind := validation_error, - path := "log.file.default.rotation_count", - reason := #{expected_type := "1..128"}, - value := Count} - }}]}, - hocon_tconf:generate(emqx_conf_schema, ConfMap0)) - end, [{to_bin(Format, [0]), 0}, {to_bin(Format, [129]), 129}]). - %% erlfmt-ignore -define(BASE_AUTHN_ARRAY, """ From e797f93a53d7bf46012a6b8dbf3d928969a99540 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 22 May 2023 15:17:50 +0800 Subject: [PATCH 191/197] fix: can't get file_list_transfer desc cause 500 error on /swagger.json api --- rel/i18n/emqx_ft_api.hocon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rel/i18n/emqx_ft_api.hocon b/rel/i18n/emqx_ft_api.hocon index bf6c22411..9d88fcddd 100644 --- a/rel/i18n/emqx_ft_api.hocon +++ b/rel/i18n/emqx_ft_api.hocon @@ -3,7 +3,7 @@ emqx_ft_api { file_list.desc: """List all uploaded files.""" -file_list_transfer.desc +file_list_transfer.desc: """List a file uploaded during specified transfer, identified by client id and file id.""" } From da7d351cc6ccb7d2fc43f8e129ac0a6689fbf3e6 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 19 May 2023 14:29:18 +0800 Subject: [PATCH 192/197] chore: update changes --- changes/ce/fix-10746.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-10746.en.md diff --git a/changes/ce/fix-10746.en.md b/changes/ce/fix-10746.en.md new file mode 100644 index 000000000..c8cf2d084 --- /dev/null +++ b/changes/ce/fix-10746.en.md @@ -0,0 +1 @@ +Add missing support of the event `$events/delivery_dropped` into the rule engine test API `rule_test`. From 087dc591151bf87c0b4749526c843a7a6961beef Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 22 May 2023 14:46:14 +0800 Subject: [PATCH 193/197] test: add test cases for the `rule_test` API --- .../emqx_rule_engine_api_rule_test_SUITE.erl | 277 ++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 apps/emqx_rule_engine/test/emqx_rule_engine_api_rule_test_SUITE.erl diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_rule_test_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_rule_test_SUITE.erl new file mode 100644 index 000000000..575d35238 --- /dev/null +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_rule_test_SUITE.erl @@ -0,0 +1,277 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_rule_engine_api_rule_test_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(CONF_DEFAULT, <<"rule_engine {rules {}}">>). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + application:load(emqx_conf), + ok = emqx_common_test_helpers:load_config(emqx_rule_engine_schema, ?CONF_DEFAULT), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_rule_engine]), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([emqx_conf, emqx_rule_engine]), + ok. + +t_ctx_pub(_) -> + SQL = <<"SELECT payload.msg as msg, clientid, username, payload, topic, qos FROM \"t/#\"">>, + Context = #{ + clientid => <<"c_emqx">>, + event_type => message_publish, + payload => <<"{\"msg\": \"hello\"}">>, + qos => 1, + topic => <<"t/a">>, + username => <<"u_emqx">> + }, + Expected = Context#{msg => <<"hello">>}, + do_test(SQL, Context, Expected). + +t_ctx_sub(_) -> + SQL = <<"SELECT clientid, username, topic, qos FROM \"$events/session_subscribed\"">>, + Context = #{ + clientid => <<"c_emqx">>, + event_type => session_subscribed, + qos => 1, + topic => <<"t/a">>, + username => <<"u_emqx">> + }, + + do_test(SQL, Context, Context). + +t_ctx_unsub(_) -> + SQL = <<"SELECT clientid, username, topic, qos FROM \"$events/session_unsubscribed\"">>, + Context = #{ + clientid => <<"c_emqx">>, + event_type => session_unsubscribed, + qos => 1, + topic => <<"t/a">>, + username => <<"u_emqx">> + }, + do_test(SQL, Context, Context). + +t_ctx_delivered(_) -> + SQL = + <<"SELECT from_clientid, from_username, topic, qos, node, timestamp FROM \"$events/message_delivered\"">>, + Context = #{ + clientid => <<"c_emqx_2">>, + event_type => message_delivered, + from_clientid => <<"c_emqx_1">>, + from_username => <<"u_emqx_1">>, + payload => <<"{\"msg\": \"hello\"}">>, + qos => 1, + topic => <<"t/a">>, + username => <<"u_emqx_2">> + }, + Expected = check_result([from_clientid, from_username, topic, qos], [node, timestamp], Context), + do_test(SQL, Context, Expected). + +t_ctx_acked(_) -> + SQL = + <<"SELECT from_clientid, from_username, topic, qos, node, timestamp FROM \"$events/message_acked\"">>, + + Context = #{ + clientid => <<"c_emqx_2">>, + event_type => message_acked, + from_clientid => <<"c_emqx_1">>, + from_username => <<"u_emqx_1">>, + payload => <<"{\"msg\": \"hello\"}">>, + qos => 1, + topic => <<"t/a">>, + username => <<"u_emqx_2">> + }, + + Expected = with_node_timestampe([from_clientid, from_username, topic, qos], Context), + + do_test(SQL, Context, Expected). + +t_ctx_droped(_) -> + SQL = <<"SELECT reason, topic, qos, node, timestamp FROM \"$events/message_dropped\"">>, + Topic = <<"t/a">>, + QoS = 1, + Reason = <<"no_subscribers">>, + Context = #{ + clientid => <<"c_emqx">>, + event_type => message_dropped, + payload => <<"{\"msg\": \"hello\"}">>, + qos => QoS, + reason => Reason, + topic => Topic, + username => <<"u_emqx">> + }, + + Expected = with_node_timestampe([reason, topic, qos], Context), + do_test(SQL, Context, Expected). + +t_ctx_connected(_) -> + SQL = + <<"SELECT clientid, username, keepalive, is_bridge FROM \"$events/client_connected\"">>, + + Context = + #{ + clean_start => true, + clientid => <<"c_emqx">>, + event_type => client_connected, + is_bridge => false, + peername => <<"127.0.0.1:52918">>, + username => <<"u_emqx">> + }, + Expected = check_result([clientid, username, keepalive, is_bridge], [], Context), + do_test(SQL, Context, Expected). + +t_ctx_disconnected(_) -> + SQL = + <<"SELECT clientid, username, reason, disconnected_at, node FROM \"$events/client_disconnected\"">>, + + Context = + #{ + clientid => <<"c_emqx">>, + event_type => client_disconnected, + reason => <<"normal">>, + username => <<"u_emqx">> + }, + Expected = check_result([clientid, username, reason], [disconnected_at, node], Context), + do_test(SQL, Context, Expected). + +t_ctx_connack(_) -> + SQL = + <<"SELECT clientid, username, reason_code, node FROM \"$events/client_connack\"">>, + + Context = + #{ + clean_start => true, + clientid => <<"c_emqx">>, + event_type => client_connack, + reason_code => <<"sucess">>, + username => <<"u_emqx">> + }, + Expected = check_result([clientid, username, reason_code], [node], Context), + do_test(SQL, Context, Expected). + +t_ctx_check_authz_complete(_) -> + SQL = + << + "SELECT clientid, username, topic, action, result,\n" + "authz_source, node FROM \"$events/client_check_authz_complete\"" + >>, + + Context = + #{ + action => <<"publish">>, + clientid => <<"c_emqx">>, + event_type => client_check_authz_complete, + result => <<"allow">>, + topic => <<"t/1">>, + username => <<"u_emqx">> + }, + Expected = check_result( + [clientid, username, topic, action], + [authz_source, node, result], + Context + ), + + do_test(SQL, Context, Expected). + +t_ctx_delivery_dropped(_) -> + SQL = + <<"SELECT from_clientid, from_username, reason, topic, qos FROM \"$events/delivery_dropped\"">>, + + Context = + #{ + clientid => <<"c_emqx_2">>, + event_type => delivery_dropped, + from_clientid => <<"c_emqx_1">>, + from_username => <<"u_emqx_1">>, + payload => <<"{\"msg\": \"hello\"}">>, + qos => 1, + reason => <<"queue_full">>, + topic => <<"t/a">>, + username => <<"u_emqx_2">> + }, + Expected = check_result([from_clientid, from_username, reason, qos, topic], [], Context), + do_test(SQL, Context, Expected). + +do_test(SQL, Context, Expected0) -> + Res = emqx_rule_engine_api:'/rule_test'( + post, + test_rule_params(SQL, Context) + ), + ?assertMatch({200, _}, Res), + {200, Result0} = Res, + Result = emqx_utils_maps:unsafe_atom_key_map(Result0), + case is_function(Expected0) of + false -> + Expected = maps:without([event_type], Expected0), + ?assertMatch(Expected, Result, Expected); + _ -> + Expected0(Result) + end, + ok. + +test_rule_params(Sql, Context) -> + #{ + body => #{ + <<"context">> => Context, + <<"sql">> => Sql + } + }. + +with_node_timestampe(Keys, Context) -> + check_result(Keys, [node, timestamp], Context). + +check_result(Keys, Exists, Context) -> + Log = fun(Format, Args) -> + lists:flatten(io_lib:format(Format, Args)) + end, + + Base = maps:with(Keys, Context), + + fun(Result) -> + maps:foreach( + fun(Key, Value) -> + ?assertEqual( + Value, + maps:get(Key, Result, undefined), + Log("Key:~p value error~nResult:~p~n", [Key, Result]) + ) + end, + Base + ), + + NotExists = fun(Key) -> Log("Key:~p not exists in result:~p~n", [Key, Result]) end, + lists:foreach( + fun(Key) -> + Find = maps:find(Key, Result), + Formatter = NotExists(Key), + ?assertMatch({ok, _}, Find, Formatter), + ?assertNotMatch({ok, undefined}, Find, Formatter), + ?assertNotMatch({ok, <<"undefined">>}, Find, Formatter) + end, + Exists + ), + + ?assertEqual(erlang:length(Keys) + erlang:length(Exists), maps:size(Result), Result) + end. From 966f7485513ac1fe5dddb85000081cc9f61c2a7d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 22 May 2023 10:45:27 +0300 Subject: [PATCH 194/197] test(ft): make proptest less aggressive So that the chance of getting huge coverage and as a result, high chance of running out of memory, is reduced. --- apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl b/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl index a437e8dfe..9a2b7df0b 100644 --- a/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl +++ b/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl @@ -189,7 +189,7 @@ segment_t(Filesize, Segsizes) -> ). filesize_t() -> - scaled(4000, non_neg_integer()). + scaled(2500, non_neg_integer()). segsizes_t() -> ?LET( From cb5a596c5715719e4bfa4052a9bde88fb3a75d3f Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 22 May 2023 16:58:13 +0800 Subject: [PATCH 195/197] fix: make sure swagger.json is fully generated --- apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl | 4 ++++ apps/emqx_machine/src/emqx_machine_boot.erl | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 783c00dad..0d446a7f2 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -55,6 +55,10 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + %% Load all applications to ensure swagger.json is fully generated. + Apps = emqx_machine_boot:reboot_apps(), + ct:pal("load apps:~p~n", [Apps]), + lists:foreach(fun(App) -> application:load(App) end, Apps), emqx_mgmt_api_test_util:init_suite([emqx_management]), Config. diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index f74db45ec..266855877 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -26,7 +26,7 @@ -dialyzer({no_match, [basic_reboot_apps/0]}). -ifdef(TEST). --export([sorted_reboot_apps/1]). +-export([sorted_reboot_apps/1, reboot_apps/0]). -endif. %% these apps are always (re)started by emqx_machine @@ -120,7 +120,7 @@ restart_type(App) -> %% the list of (re)started apps depends on release type/edition reboot_apps() -> - {ok, ConfigApps0} = application:get_env(emqx_machine, applications), + ConfigApps0 = application:get_env(emqx_machine, applications, []), BaseRebootApps = basic_reboot_apps(), ConfigApps = lists:filter(fun(App) -> not lists:member(App, BaseRebootApps) end, ConfigApps0), BaseRebootApps ++ ConfigApps. From d7bd2227dbd1a3dbf34ad2ba28695fe4e6ecda30 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 22 May 2023 17:34:33 +0800 Subject: [PATCH 196/197] chore: bad change log file --- changes/ce/{fix-10340-en.md => fix-10340.en.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename changes/ce/{fix-10340-en.md => fix-10340.en.md} (97%) diff --git a/changes/ce/fix-10340-en.md b/changes/ce/fix-10340.en.md similarity index 97% rename from changes/ce/fix-10340-en.md rename to changes/ce/fix-10340.en.md index c9ae7b81b..0f0436cec 100644 --- a/changes/ce/fix-10340-en.md +++ b/changes/ce/fix-10340.en.md @@ -1,4 +1,4 @@ -Fixed the issue that could lead to crash logs being printed when stopping EMQ X via systemd. +Fixed the issue that could lead to crash logs being printed when stopping EMQX via systemd. ``` 2023-03-29T16:43:25.915761+08:00 [error] Generic server memsup terminating. Reason: {port_died,normal}. Last message: {'EXIT',<0.2117.0>,{port_died,normal}}. State: [{data,[{"Timeout",60000}]},{items,{"Memory Usage",[{"Allocated",929959936},{"Total",3832242176}]}},{items,{"Worst Memory User",[{"Pid",<0.2031.0>},{"Memory",4720472}]}}]. 2023-03-29T16:43:25.924764+08:00 [error] crasher: initial call: memsup:init/1, pid: <0.2116.0>, registered_name: memsup, exit: {{port_died,normal},[{gen_server,handle_common_reply,8,[{file,"gen_server.erl"},{line,811}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,226}]}]}, ancestors: [os_mon_sup,<0.2114.0>], message_queue_len: 0, messages: [], links: [<0.2115.0>], dictionary: [], trap_exit: true, status: running, heap_size: 4185, stack_size: 29, reductions: 187637; neighbours: From d22541e8b3377fb7f39b5492666b5329e501f4c1 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 22 May 2023 15:32:08 +0300 Subject: [PATCH 197/197] fix(ft): correct mistyped option in README --- apps/emqx_ft/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_ft/README.md b/apps/emqx_ft/README.md index a479754d2..019c63dea 100644 --- a/apps/emqx_ft/README.md +++ b/apps/emqx_ft/README.md @@ -8,7 +8,7 @@ As almost any other EMQX application, `emqx_ft` is configured via the EMQX confi ``` file_transfer { - enabled = true + enable = true } ``` @@ -26,7 +26,7 @@ The `local` exporter is the default one, and it stores the transferred files in ``` file_transfer { - enabled = true + enable = true storage { local { exporter { @@ -47,7 +47,7 @@ This snippet configures File Transfer to store the transferred files in the `my- ``` file_transfer { - enabled = true + enable = true storage { local { exporter {