From d976943f991faa14c290b4c8052f40135cc6c7d0 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 6 Jan 2023 22:15:22 +0300 Subject: [PATCH 001/156] feat(ft): add basic hooks --- apps/emqx/src/emqx_cm.erl | 46 ++++++++----- apps/emqx/src/emqx_types.erl | 4 ++ apps/emqx/src/proto/emqx_cm_proto_v2.erl | 88 ++++++++++++++++++++++++ mix.exs | 1 + 4 files changed, 122 insertions(+), 17 deletions(-) create mode 100644 apps/emqx/src/proto/emqx_cm_proto_v2.erl diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index f8c510482..d2ac642ca 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -89,6 +89,7 @@ mark_channel_connected/1, mark_channel_disconnected/1, get_connected_client_count/0, + takeover_finish/2, do_kick_session/3, do_get_chan_stats/2, @@ -171,6 +172,7 @@ register_channel(ClientId, ChanPid, #{conn_mod := ConnMod}) when is_pid(ChanPid) true = ets:insert(?CHAN_CONN_TAB, {Chan, ConnMod}), ok = emqx_cm_registry:register_channel(Chan), mark_channel_connected(ChanPid), + ok = emqx_hooks:run('channel.registered', [ConnMod, ChanPid]), cast({registered, Chan}). %% @doc Unregister a channel. @@ -180,11 +182,13 @@ unregister_channel(ClientId) when is_binary(ClientId) -> ok. %% @private -do_unregister_channel(Chan) -> +do_unregister_channel({_ClientId, ChanPid} = Chan) -> ok = emqx_cm_registry:unregister_channel(Chan), true = ets:delete(?CHAN_CONN_TAB, Chan), true = ets:delete(?CHAN_INFO_TAB, Chan), - ets:delete_object(?CHAN_TAB, Chan). + ets:delete_object(?CHAN_TAB, Chan), + ok = emqx_hooks:run('channel.unregistered', [ChanPid]), + true. -spec connection_closed(emqx_types:clientid()) -> true. connection_closed(ClientId) -> @@ -212,7 +216,7 @@ do_get_chan_info(ClientId, ChanPid) -> -spec get_chan_info(emqx_types:clientid(), chan_pid()) -> maybe(emqx_types:infos()). get_chan_info(ClientId, ChanPid) -> - wrap_rpc(emqx_cm_proto_v1:get_chan_info(ClientId, ChanPid)). + wrap_rpc(emqx_cm_proto_v2:get_chan_info(ClientId, ChanPid)). %% @doc Update infos of the channel. -spec set_chan_info(emqx_types:clientid(), emqx_types:attrs()) -> boolean(). @@ -242,7 +246,7 @@ do_get_chan_stats(ClientId, ChanPid) -> -spec get_chan_stats(emqx_types:clientid(), chan_pid()) -> maybe(emqx_types:stats()). get_chan_stats(ClientId, ChanPid) -> - wrap_rpc(emqx_cm_proto_v1:get_chan_stats(ClientId, ChanPid)). + wrap_rpc(emqx_cm_proto_v2:get_chan_stats(ClientId, ChanPid)). %% @doc Set channel's stats. -spec set_chan_stats(emqx_types:clientid(), emqx_types:stats()) -> boolean(). @@ -278,7 +282,7 @@ open_session(true, ClientInfo = #{clientid := ClientId}, ConnInfo) -> {ok, #{session => Session1, present => false}} end, emqx_cm_locker:trans(ClientId, CleanStart); -open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> +open_session(false, ClientInfo = #{clientid := ClientId}, #{conn_mod := NewConnMod} = ConnInfo) -> Self = self(), ResumeStart = fun(_) -> CreateSess = @@ -304,18 +308,12 @@ open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> }}; {living, ConnMod, ChanPid, Session} -> ok = emqx_session:resume(ClientInfo, Session), - case - request_stepdown( - {takeover, 'end'}, - ConnMod, - ChanPid - ) - of - {ok, Pendings} -> + case wrap_rpc(emqx_cm_proto_v2:takeover_finish(ConnMod, ChanPid)) of + {ok, Pendings, TakoverData} -> Session1 = emqx_persistent_session:persist( ClientInfo, ConnInfo, Session ), - register_channel(ClientId, Self, ConnInfo), + ok = emqx_hooks:run('channel.takeovered', [NewConnMod, Self, TakoverData]), {ok, #{ session => clean_session(Session1), present => true, @@ -400,6 +398,20 @@ takeover_session(ClientId) -> takeover_session(ClientId, ChanPid) end. +takeover_finish(ConnMod, ChanPid) -> + TakoverData = emqx_hooks:run_fold('channel.takeover', [ConnMod, ChanPid], #{}), + case + %% node-local call + request_stepdown( + {takeover, 'end'}, + ConnMod, + ChanPid + ) + of + {ok, Pendings} -> {ok, Pendings, TakoverData}; + {error, _} = Error -> Error + end. + takeover_session(ClientId, Pid) -> try do_takeover_session(ClientId, Pid) @@ -429,7 +441,7 @@ do_takeover_session(ClientId, ChanPid) when node(ChanPid) == node() -> end end; do_takeover_session(ClientId, ChanPid) -> - wrap_rpc(emqx_cm_proto_v1:takeover_session(ClientId, ChanPid)). + wrap_rpc(emqx_cm_proto_v2:takeover_session(ClientId, ChanPid)). %% @doc Discard all the sessions identified by the ClientId. -spec discard_session(emqx_types:clientid()) -> ok. @@ -531,7 +543,7 @@ do_kick_session(Action, ClientId, ChanPid) -> %% @private This function is shared for session 'kick' and 'discard' (as the first arg Action). kick_session(Action, ClientId, ChanPid) -> try - wrap_rpc(emqx_cm_proto_v1:kick_session(Action, ClientId, ChanPid)) + wrap_rpc(emqx_cm_proto_v2:kick_session(Action, ClientId, ChanPid)) catch Error:Reason -> %% This should mostly be RPC failures. @@ -716,7 +728,7 @@ do_get_chann_conn_mod(ClientId, ChanPid) -> end. get_chann_conn_mod(ClientId, ChanPid) -> - wrap_rpc(emqx_cm_proto_v1:get_chann_conn_mod(ClientId, ChanPid)). + wrap_rpc(emqx_cm_proto_v2:get_chann_conn_mod(ClientId, ChanPid)). mark_channel_connected(ChanPid) -> ?tp(emqx_cm_connected_client_count_inc, #{}), diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl index 7223da245..8b7fdae9b 100644 --- a/apps/emqx/src/emqx_types.erl +++ b/apps/emqx/src/emqx_types.erl @@ -101,6 +101,8 @@ -export_type([oom_policy/0]). +-export_type([takeover_data/0]). + -type proto_ver() :: ?MQTT_PROTO_V3 | ?MQTT_PROTO_V4 @@ -242,3 +244,5 @@ max_heap_size => non_neg_integer(), enable => boolean() }. + +-type takeover_data() :: map(). diff --git a/apps/emqx/src/proto/emqx_cm_proto_v2.erl b/apps/emqx/src/proto/emqx_cm_proto_v2.erl new file mode 100644 index 000000000..2981dbd40 --- /dev/null +++ b/apps/emqx/src/proto/emqx_cm_proto_v2.erl @@ -0,0 +1,88 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_cm_proto_v2). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + lookup_client/2, + kickout_client/2, + + get_chan_stats/2, + get_chan_info/2, + get_chann_conn_mod/2, + + takeover_session/2, + takeover_finish/2, + kick_session/3 +]). + +-include("bpapi.hrl"). +-include("src/emqx_cm.hrl"). + +introduced_in() -> + "5.0.0". + +-spec kickout_client(node(), emqx_types:clientid()) -> ok | {badrpc, _}. +kickout_client(Node, ClientId) -> + rpc:call(Node, emqx_cm, kick_session, [ClientId]). + +-spec lookup_client(node(), {clientid, emqx_types:clientid()} | {username, emqx_types:username()}) -> + [emqx_cm:channel_info()] | {badrpc, _}. +lookup_client(Node, Key) -> + rpc:call(Node, emqx_cm, lookup_client, [Key]). + +-spec get_chan_stats(emqx_types:clientid(), emqx_cm:chan_pid()) -> emqx_types:stats() | {badrpc, _}. +get_chan_stats(ClientId, ChanPid) -> + rpc:call(node(ChanPid), emqx_cm, do_get_chan_stats, [ClientId, ChanPid], ?T_GET_INFO * 2). + +-spec get_chan_info(emqx_types:clientid(), emqx_cm:chan_pid()) -> emqx_types:infos() | {badrpc, _}. +get_chan_info(ClientId, ChanPid) -> + rpc:call(node(ChanPid), emqx_cm, do_get_chan_info, [ClientId, ChanPid], ?T_GET_INFO * 2). + +-spec get_chann_conn_mod(emqx_types:clientid(), emqx_cm:chan_pid()) -> + module() | undefined | {badrpc, _}. +get_chann_conn_mod(ClientId, ChanPid) -> + rpc:call(node(ChanPid), emqx_cm, do_get_chann_conn_mod, [ClientId, ChanPid], ?T_GET_INFO * 2). + +-spec takeover_session(emqx_types:clientid(), emqx_cm:chan_pid()) -> + none + | {expired | persistent, emqx_session:session()} + | {living, _ConnMod :: atom(), emqx_cm:chan_pid(), emqx_session:session()} + | {badrpc, _}. +takeover_session(ClientId, ChanPid) -> + rpc:call(node(ChanPid), emqx_cm, takeover_session, [ClientId, ChanPid], ?T_TAKEOVER * 2). + +-spec takeover_finish(module(), emqx_cm:chan_pid()) -> + {ok, emqx_type:takeover_data()} + | {ok, list(emqx_type:deliver()), emqx_type:takeover_data()} + | {error, term()} + | {badrpc, _}. +takeover_finish(ConnMod, ChanPid) -> + erpc:call( + node(ChanPid), + emqx_cm, + takeover_session_finish, + [ConnMod, ChanPid], + ?T_TAKEOVER * 2 + ). + +-spec kick_session(kick | discard, emqx_types:clientid(), emqx_cm:chan_pid()) -> ok | {badrpc, _}. +kick_session(Action, ClientId, ChanPid) -> + rpc:call(node(ChanPid), emqx_cm, do_kick_session, [Action, ClientId, ChanPid], ?T_KICK * 2). diff --git a/mix.exs b/mix.exs index 384b08100..92024f48d 100644 --- a/mix.exs +++ b/mix.exs @@ -293,6 +293,7 @@ defmodule EMQXUmbrella.MixProject do emqx_psk: :permanent, emqx_slow_subs: :permanent, emqx_plugins: :permanent, + emqx_ft: :permanent, emqx_mix: :none ] ++ if(enable_quicer?(), do: [quicer: :permanent], else: []) ++ From aaaef30be66ab7291732f3fb634afdd0210b983b Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 6 Jan 2023 23:53:25 +0300 Subject: [PATCH 002/156] feat(ft): add file transfer app and bootstrap replicated ft data structure --- apps/emqx/src/emqx_channel.erl | 7 +++ apps/emqx_ft/README.md | 9 +++ apps/emqx_ft/include/emqx_ft.hrl | 17 +++++ apps/emqx_ft/rebar.config | 11 ++++ apps/emqx_ft/src/emqx_ft.app.src | 12 ++++ apps/emqx_ft/src/emqx_ft.erl | 103 +++++++++++++++++++++++++++++++ apps/emqx_ft/src/emqx_ft_app.erl | 30 +++++++++ apps/emqx_ft/src/emqx_ft_sup.erl | 49 +++++++++++++++ 8 files changed, 238 insertions(+) create mode 100644 apps/emqx_ft/README.md create mode 100644 apps/emqx_ft/include/emqx_ft.hrl create mode 100644 apps/emqx_ft/rebar.config create mode 100644 apps/emqx_ft/src/emqx_ft.app.src create mode 100644 apps/emqx_ft/src/emqx_ft.erl create mode 100644 apps/emqx_ft/src/emqx_ft_app.erl create mode 100644 apps/emqx_ft/src/emqx_ft_sup.erl diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 29a59e482..009dc72ea 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -641,6 +641,13 @@ process_connect( %%-------------------------------------------------------------------- process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) -> + ?SLOG( + warning, + #{ + packet => Packet, + packet_id => PacketId + } + ), case pipeline( [ diff --git a/apps/emqx_ft/README.md b/apps/emqx_ft/README.md new file mode 100644 index 000000000..c483b3169 --- /dev/null +++ b/apps/emqx_ft/README.md @@ -0,0 +1,9 @@ +emqx_ft +===== + +EMQX file transfer over MQTT + +Build +----- + + $ rebar3 compile diff --git a/apps/emqx_ft/include/emqx_ft.hrl b/apps/emqx_ft/include/emqx_ft.hrl new file mode 100644 index 000000000..2cbd24fb4 --- /dev/null +++ b/apps/emqx_ft/include/emqx_ft.hrl @@ -0,0 +1,17 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-define(FT_TAB, emqx_ft). diff --git a/apps/emqx_ft/rebar.config b/apps/emqx_ft/rebar.config new file mode 100644 index 000000000..2c0962035 --- /dev/null +++ b/apps/emqx_ft/rebar.config @@ -0,0 +1,11 @@ +%% -*- mode: erlang -*- + +{erl_opts, [debug_info]}. +{deps, [{emqx, {path, "../emqx"}}]}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [emqx_ft]} +]}. + +{project_plugins, [erlfmt]}. diff --git a/apps/emqx_ft/src/emqx_ft.app.src b/apps/emqx_ft/src/emqx_ft.app.src new file mode 100644 index 000000000..855451bfb --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft.app.src @@ -0,0 +1,12 @@ +{application, emqx_ft, [ + {description, "EMQX file transfer over MQTT"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_ft_app, []}}, + {applications, [ + kernel, + stdlib + ]}, + {env, []}, + {modules, []} +]}. diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl new file mode 100644 index 000000000..0ba9c17a6 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -0,0 +1,103 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft). + +-include("emqx_ft.hrl"). + +-export([ + create_tab/0, + hook/0, + unhook/0 +]). + +-export([ + on_channel_unregistered/1, + on_channel_takeover/3, + on_channel_takeovered/3 +]). + +-type ft_data() :: #{ + nodes := list(node()) +}. + +-record(emqx_ft, { + chan_pid :: pid(), + ft_data :: ft_data() +}). + +%%-------------------------------------------------------------------- +%% API for app +%%-------------------------------------------------------------------- + +create_tab() -> + _Tab = ets:new(?FT_TAB, [ + set, + public, + named_table, + {keypos, #emqx_ft.chan_pid} + ]), + ok. + +hook() -> + % ok = emqx_hooks:put('channel.registered', {?MODULE, on_channel_registered, []}), + ok = emqx_hooks:put('channel.unregistered', {?MODULE, on_channel_unregistered, []}), + ok = emqx_hooks:put('channel.takeover', {?MODULE, on_channel_takeover, []}), + ok = emqx_hooks:put('channel.takeovered', {?MODULE, on_channel_takeovered, []}). + +unhook() -> + % ok = emqx_hooks:del('channel.registered', {?MODULE, on_channel_registered}), + ok = emqx_hooks:del('channel.unregistered', {?MODULE, on_channel_unregistered}), + ok = emqx_hooks:del('channel.takeover', {?MODULE, on_channel_takeover}), + ok = emqx_hooks:del('channel.takeovered', {?MODULE, on_channel_takeovered}). + +%%-------------------------------------------------------------------- +%% Hooks +%%-------------------------------------------------------------------- + +on_channel_unregistered(ChanPid) -> + ok = delete_ft_data(ChanPid). + +on_channel_takeover(_ConnMod, ChanPid, TakeoverData) -> + case get_ft_data(ChanPid) of + {ok, FTData} -> + {ok, TakeoverData#{ft_data => FTData}}; + none -> + ok + end. + +on_channel_takeovered(_ConnMod, ChanPid, #{ft_data := FTData}) -> + ok = put_ft_data(ChanPid, FTData); +on_channel_takeovered(_ConnMod, _ChanPid, _) -> + ok. + +%%-------------------------------------------------------------------- +%% Private funs +%%-------------------------------------------------------------------- + +get_ft_data(ChanPid) -> + case ets:lookup(?FT_TAB, ChanPid) of + [#emqx_ft{ft_data = FTData}] -> {ok, FTData}; + [] -> none + end. + +delete_ft_data(ChanPid) -> + true = ets:delete(?FT_TAB, ChanPid), + ok. + +put_ft_data(ChanPid, FTData) -> + true = ets:insert(?FT_TAB, #emqx_ft{chan_pid = ChanPid, ft_data = FTData}), + ok. diff --git a/apps/emqx_ft/src/emqx_ft_app.erl b/apps/emqx_ft/src/emqx_ft_app.erl new file mode 100644 index 000000000..4778da1a1 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_app.erl @@ -0,0 +1,30 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + {ok, Sup} = emqx_ft_sup:start_link(), + ok = emqx_ft:hook(), + {ok, Sup}. + +stop(_State) -> + ok = emqx_ft:unhook(), + ok. diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl new file mode 100644 index 000000000..4c976246b --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -0,0 +1,49 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%% sup_flags() = #{strategy => strategy(), % optional +%% intensity => non_neg_integer(), % optional +%% period => pos_integer()} % optional +%% child_spec() = #{id => child_id(), % mandatory +%% start => mfargs(), % mandatory +%% restart => restart(), % optional +%% shutdown => shutdown(), % optional +%% type => worker(), % optional +%% modules => modules()} % optional +init([]) -> + ok = emqx_ft:create_tab(), + SupFlags = #{ + strategy => one_for_all, + intensity => 100, + period => 10 + }, + ChildSpecs = [], + {ok, {SupFlags, ChildSpecs}}. + +%% internal functions From 81e04ce93aca0c0831f4934f390d5a58612ee25c Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 20 Jan 2023 18:55:45 +0300 Subject: [PATCH 003/156] feat(ft): introduce simple filesystem storage backend + assembler --- apps/emqx_ft/src/emqx_ft.app.src | 3 +- apps/emqx_ft/src/emqx_ft.erl | 14 + apps/emqx_ft/src/emqx_ft_assembler.erl | 160 ++++++++ apps/emqx_ft/src/emqx_ft_assembler_sup.erl | 44 +++ apps/emqx_ft/src/emqx_ft_assembly.erl | 364 ++++++++++++++++++ apps/emqx_ft/src/emqx_ft_storage_fs.erl | 425 +++++++++++++++++++++ 6 files changed, 1009 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_ft/src/emqx_ft_assembler.erl create mode 100644 apps/emqx_ft/src/emqx_ft_assembler_sup.erl create mode 100644 apps/emqx_ft/src/emqx_ft_assembly.erl create mode 100644 apps/emqx_ft/src/emqx_ft_storage_fs.erl diff --git a/apps/emqx_ft/src/emqx_ft.app.src b/apps/emqx_ft/src/emqx_ft.app.src index 855451bfb..80b4b47dd 100644 --- a/apps/emqx_ft/src/emqx_ft.app.src +++ b/apps/emqx_ft/src/emqx_ft.app.src @@ -5,7 +5,8 @@ {mod, {emqx_ft_app, []}}, {applications, [ kernel, - stdlib + stdlib, + gproc ]}, {env, []}, {modules, []} diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 0ba9c17a6..1f8a90a6f 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -30,6 +30,20 @@ on_channel_takeovered/3 ]). +-export_type([clientid/0]). +-export_type([transfer/0]). +-export_type([offset/0]). + +%% Number of bytes +-type bytes() :: non_neg_integer(). + +%% MQTT Client ID +-type clientid() :: emqx_types:clientid(). + +-type fileid() :: binary(). +-type transfer() :: {clientid(), fileid()}. +-type offset() :: bytes(). + -type ft_data() :: #{ nodes := list(node()) }. diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl new file mode 100644 index 000000000..2bd01c8b0 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -0,0 +1,160 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_assembler). + +-export([start_link/2]). + +-behaviour(gen_statem). +-export([callback_mode/0]). +-export([init/1]). +% -export([list_local_fragments/3]). +% -export([list_remote_fragments/3]). +% -export([start_assembling/3]). +-export([handle_event/4]). + +% -export([handle_continue/2]). +% -export([handle_call/3]). +% -export([handle_cast/2]). + +-record(st, { + storage :: _Storage, + transfer :: emqx_ft:transfer(), + assembly :: _TODO, + file :: io:device(), + hash +}). + +-define(RPC_LIST_TIMEOUT, 1000). +-define(RPC_READSEG_TIMEOUT, 5000). + +%% + +start_link(Storage, Transfer) -> + gen_server:start_link(?MODULE, {Storage, Transfer}, []). + +%% + +-define(internal(C), {next_event, internal, C}). + +callback_mode() -> + handle_event_function. + +init({Storage, Transfer}) -> + St = #st{ + storage = Storage, + transfer = Transfer, + assembly = emqx_ft_assembly:new(), + hash = crypto:hash_init(sha256) + }, + {ok, list_local_fragments, St, ?internal([])}. + +handle_event(list_local_fragments, internal, _, St = #st{assembly = Asm}) -> + % TODO: what we do with non-transients errors here (e.g. `eacces`)? + {ok, Fragments} = emqx_ft_storage_fs:list(St#st.storage, St#st.transfer), + NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(Asm, node(), Fragments)), + NSt = St#st{assembly = NAsm}, + case emqx_ft_assembly:status(NAsm) of + complete -> + {next_state, start_assembling, NSt, ?internal([])}; + {incomplete, _} -> + Nodes = ekka:nodelist() -- [node()], + {next_state, {list_remote_fragments, Nodes}, NSt, ?internal([])} + % TODO: recovery? + % {error, _} = Reason -> + % {stop, Reason} + end; +handle_event({list_remote_fragments, Nodes}, internal, _, St) -> + % TODO: portable "storage" ref + Args = [St#st.storage, St#st.transfer], + % TODO + % Async would better because we would not need to wait for some lagging nodes if + % the coverage is already complete. + % TODO: BP API? + Results = erpc:multicall(Nodes, emqx_ft_storage_fs, list, Args, ?RPC_LIST_TIMEOUT), + NodeResults = lists:zip(Nodes, Results), + NAsm = emqx_ft_assembly:update( + lists:foldl( + fun + ({Node, {ok, {ok, Fragments}}}, Asm) -> + emqx_ft_assembly:append(Asm, Node, Fragments); + ({Node, Result}, Asm) -> + % TODO: log? + Asm + end, + St#st.assembly, + NodeResults + ) + ), + NSt = St#st{assembly = NAsm}, + case emqx_ft_assembly:status(NAsm) of + complete -> + {next_state, start_assembling, NSt, ?internal([])}; + % TODO: retries / recovery? + {incomplete, _} = Status -> + {stop, {error, Status}} + end; +handle_event(start_assembling, internal, _, St = #st{assembly = Asm}) -> + Filemeta = emqx_ft_assembly:filemeta(Asm), + Coverage = emqx_ft_assembly:coverage(Asm), + % TODO: errors + {ok, Handle} = emqx_ft_storage_fs:open_file(St#st.storage, St#st.transfer, Filemeta), + {next_state, {assemble, Coverage}, St#st{file = Handle}, ?internal([])}; +handle_event({assemble, [{Node, Segment} | Rest]}, internal, _, St = #st{}) -> + % TODO + % Currently, race is possible between getting segment info from the remote node and + % this node garbage collecting the segment itself. + Args = [St#st.storage, St#st.transfer, Segment, 0, segsize(Segment)], + % TODO: pipelining + case erpc:call(Node, emqx_ft_storage_fs, read_segment, Args, ?RPC_READSEG_TIMEOUT) of + {ok, Content} -> + {ok, NHandle} = emqx_ft_storage_fs:write(St#st.file, Content), + {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])} + % {error, _} -> + % ... + end; +handle_event({assemble, []}, internal, _, St = #st{}) -> + {next_state, complete, St, ?internal([])}; +handle_event(complete, internal, _, St = #st{assembly = Asm, file = Handle}) -> + Filemeta = emqx_ft_assembly:filemeta(Asm), + ok = emqx_ft_storage_fs:complete(St#st.storage, St#st.transfer, Filemeta, Handle), + {stop, shutdown}. + +% handle_continue(list_local, St = #st{storage = Storage, transfer = Transfer, assembly = Asm}) -> +% % TODO: what we do with non-transients errors here (e.g. `eacces`)? +% {ok, Fragments} = emqx_ft_storage_fs:list(Storage, Transfer), +% NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(Asm, node(), Fragments)), +% NSt = St#st{assembly = NAsm}, +% case emqx_ft_assembly:status(NAsm) of +% complete -> +% {noreply, NSt, {continue}}; +% {more, _} -> +% error(noimpl); +% {error, _} -> +% error(noimpl) +% end, +% {noreply, St}. + +% handle_call(_Call, _From, St) -> +% {reply, {error, badcall}, St}. + +% handle_cast(_Cast, St) -> +% {noreply, St}. + +%% + +segsize(#{fragment := {segmentinfo, Info}}) -> + maps:get(size, Info). diff --git a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl new file mode 100644 index 000000000..fbf948fbb --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl @@ -0,0 +1,44 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_assembler_sup). + +-export([start_link/1]). +-export([start_child/3]). + +-behaviour(supervisor). +-export([init/1]). + +-define(REF(ID), {via, gproc, {n, l, {?MODULE, ID}}}). + +start_link(ID) -> + supervisor:start_link(?REF(ID), ?MODULE, []). + +start_child(ID, Storage, Transfer) -> + Childspec = #{ + id => {Storage, Transfer}, + start => {emqx_ft_assembler, start_link, [Storage, Transfer]}, + restart => transient + }, + supervisor:start_child(?REF(ID), Childspec). + +init(_) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 100, + period => 1000 + }, + {ok, SupFlags, []}. diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl new file mode 100644 index 000000000..854b420e0 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -0,0 +1,364 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_assembly). + +-export([new/0]). +-export([append/3]). +-export([update/1]). + +-export([status/1]). +-export([filemeta/1]). +-export([coverage/1]). +-export([properties/1]). + +-record(asm, { + status :: _TODO, + coverage :: _TODO, + properties :: _TODO, + meta :: _TODO, + % orddict:orddict(K, V) + segs :: _TODO, + size +}). + +new() -> + #asm{ + status = {incomplete, {missing, filemeta}}, + meta = orddict:new(), + segs = orddict:new(), + size = 0 + }. + +append(Asm, Node, Fragments) when is_list(Fragments) -> + lists:foldl(fun(F, AsmIn) -> append(AsmIn, Node, F) end, Asm, Fragments); +append(Asm, Node, Fragment = #{fragment := {filemeta, _}}) -> + append_filemeta(Asm, Node, Fragment); +append(Asm, Node, Segment = #{fragment := {segmentinfo, _}}) -> + append_segmentinfo(Asm, Node, Segment). + +update(Asm) -> + case status(meta, Asm) of + {complete, _Meta} -> + case status(coverage, Asm) of + {complete, Coverage, Props} -> + Asm#asm{ + status = complete, + coverage = Coverage, + properties = Props + }; + Status -> + Asm#asm{status = Status} + end; + Status -> + Asm#asm{status = Status} + end. + +status(#asm{status = Status}) -> + Status. + +filemeta(Asm) -> + case status(meta, Asm) of + {complete, Meta} -> Meta; + _Other -> undefined + end. + +coverage(#asm{coverage = Coverage}) -> + Coverage. + +properties(#asm{properties = Properties}) -> + Properties. + +status(meta, #asm{meta = Meta}) -> + status(meta, orddict:to_list(Meta)); +status(meta, [{Meta, {_Node, _Frag}}]) -> + {complete, Meta}; +status(meta, []) -> + {incomplete, {missing, filemeta}}; +status(meta, [_M1, _M2 | _] = Metas) -> + {error, {inconsistent, [Frag#{node => Node} || {_, {Node, Frag}} <- Metas]}}; +status(coverage, #asm{segs = Segments, size = Size}) -> + case coverage(orddict:to_list(Segments), 0, Size) of + Coverage when is_list(Coverage) -> + {complete, Coverage, #{ + dominant => dominant(Coverage) + }}; + Missing = {missing, _} -> + {incomplete, Missing} + end. + +append_filemeta(Asm, Node, Fragment = #{fragment := {filemeta, Meta}}) -> + Asm#asm{ + meta = orddict:store(Meta, {Node, Fragment}, Asm#asm.meta) + }. + +append_segmentinfo(Asm, Node, Fragment = #{fragment := {segmentinfo, Info}}) -> + Offset = maps:get(offset, Info), + Size = maps:get(size, Info), + End = Offset + Size, + Asm#asm{ + % TODO + % In theory it's possible to have two segments with same offset + size on + % different nodes but with differing content. We'd need a checksum to + % be able to disambiguate them though. + segs = orddict:store({Offset, locality(Node), -End, Node}, Fragment, Asm#asm.segs), + size = max(End, Asm#asm.size) + }. + +coverage([{{Offset, _, _, _}, _Segment} | Rest], Cursor, Sz) when Offset < Cursor -> + coverage(Rest, Cursor, Sz); +coverage([{{Cursor, _Locality, MEnd, Node}, Segment} | Rest], Cursor, Sz) -> + % NOTE + % We consider only whole fragments here, so for example from the point of view of + % this algo `[{Offset1 = 0, Size1 = 15}, {Offset2 = 10, Size2 = 10}]` has no + % coverage. + case coverage(Rest, -MEnd, Sz) of + Coverage when is_list(Coverage) -> + [{Node, Segment} | Coverage]; + Missing = {missing, _} -> + case coverage(Rest, Cursor, Sz) of + CoverageAlt when is_list(CoverageAlt) -> + CoverageAlt; + {missing, _} -> + Missing + end + end; +coverage([{{Offset, _MEnd, _, _}, _Segment} | _], Cursor, _Sz) when Offset > Cursor -> + {missing, {segment, Cursor, Offset}}; +coverage([], Cursor, Sz) when Cursor < Sz -> + {missing, {segment, Cursor, Sz}}; +coverage([], Cursor, Cursor) -> + []. + +dominant(Coverage) -> + % TODO: needs improvement, better defined _dominance_, maybe some score + Freqs = frequencies(fun({Node, Segment}) -> {Node, segsize(Segment)} end, Coverage), + maxfreq(Freqs, node()). + +frequencies(Fun, List) -> + lists:foldl( + fun(E, Acc) -> + {K, N} = Fun(E), + maps:update_with(K, fun(M) -> M + N end, N, Acc) + end, + #{}, + List + ). + +maxfreq(Freqs, Init) -> + {_, Max} = maps:fold( + fun + (F, N, {M, _MF}) when N > M -> {N, F}; + (_F, _N, {M, MF}) -> {M, MF} + end, + {0, Init}, + Freqs + ), + Max. + +locality(Node) when Node =:= node() -> + % NOTE + % This should prioritize locally available segments over those on remote nodes. + 0; +locality(_RemoteNode) -> + 1. + +segsize(#{fragment := {segmentinfo, Info}}) -> + maps:get(size, Info). + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +incomplete_new_test() -> + ?assertEqual( + {incomplete, {missing, filemeta}}, + status(update(new())) + ). + +incomplete_test() -> + ?assertEqual( + {incomplete, {missing, filemeta}}, + status( + update( + append(new(), node(), [ + segmentinfo(p1, 0, 42), + segmentinfo(p1, 42, 100) + ]) + ) + ) + ). + +consistent_test() -> + Asm1 = append(new(), n1, [filemeta(m1, "blarg")]), + Asm2 = append(Asm1, n2, [segmentinfo(s2, 0, 42)]), + Asm3 = append(Asm2, n3, [filemeta(m3, "blarg")]), + ?assertMatch({complete, _}, status(meta, Asm3)). + +inconsistent_test() -> + Asm1 = append(new(), node(), [segmentinfo(s1, 0, 42)]), + Asm2 = append(Asm1, n1, [filemeta(m1, "blarg")]), + Asm3 = append(Asm2, n2, [segmentinfo(s2, 0, 42), filemeta(m1, "blorg")]), + Asm4 = append(Asm3, n3, [filemeta(m3, "blarg")]), + ?assertMatch( + {error, + {inconsistent, [ + % blarg < blorg + #{node := n3, path := m3, fragment := {filemeta, #{name := "blarg"}}}, + #{node := n2, path := m1, fragment := {filemeta, #{name := "blorg"}}} + ]}}, + status(meta, Asm4) + ). + +simple_coverage_test() -> + Node = node(), + Segs = [ + {node42, segmentinfo(n1, 20, 30)}, + {Node, segmentinfo(n2, 0, 10)}, + {Node, segmentinfo(n3, 50, 50)}, + {Node, segmentinfo(n4, 10, 10)} + ], + Asm = append_many(new(), Segs), + ?assertMatch( + {complete, + [ + {Node, #{path := n2}}, + {Node, #{path := n4}}, + {node42, #{path := n1}}, + {Node, #{path := n3}} + ], + #{dominant := Node}}, + status(coverage, Asm) + ). + +redundant_coverage_test() -> + Node = node(), + Segs = [ + {Node, segmentinfo(n1, 0, 20)}, + {node1, segmentinfo(n2, 0, 10)}, + {Node, segmentinfo(n3, 20, 40)}, + {node2, segmentinfo(n4, 10, 10)}, + {node2, segmentinfo(n5, 50, 20)}, + {node3, segmentinfo(n6, 20, 20)}, + {Node, segmentinfo(n7, 50, 10)}, + {node1, segmentinfo(n8, 40, 10)} + ], + Asm = append_many(new(), Segs), + ?assertMatch( + {complete, + [ + {Node, #{path := n1}}, + {node3, #{path := n6}}, + {node1, #{path := n8}}, + {node2, #{path := n5}} + ], + #{dominant := _}}, + status(coverage, Asm) + ). + +redundant_coverage_prefer_local_test() -> + Node = node(), + Segs = [ + {node1, segmentinfo(n1, 0, 20)}, + {Node, segmentinfo(n2, 0, 10)}, + {Node, segmentinfo(n3, 10, 10)}, + {node2, segmentinfo(n4, 20, 20)}, + {Node, segmentinfo(n5, 30, 10)}, + {Node, segmentinfo(n6, 20, 10)} + ], + Asm = append_many(new(), Segs), + ?assertMatch( + {complete, + [ + {Node, #{path := n2}}, + {Node, #{path := n3}}, + {Node, #{path := n6}}, + {Node, #{path := n5}} + ], + #{dominant := Node}}, + status(coverage, Asm) + ). + +missing_coverage_test() -> + Node = node(), + Segs = [ + {Node, segmentinfo(n1, 0, 10)}, + {node1, segmentinfo(n3, 10, 20)}, + {Node, segmentinfo(n2, 0, 20)}, + {node2, segmentinfo(n4, 50, 50)}, + {Node, segmentinfo(n5, 40, 60)} + ], + Asm = append_many(new(), Segs), + ?assertEqual( + % {incomplete, {missing, {segment, 30, 40}}}, ??? + {incomplete, {missing, {segment, 20, 40}}}, + status(coverage, Asm) + ). + +missing_end_coverage_test() -> + Node = node(), + Segs = [ + {Node, segmentinfo(n1, 0, 15)}, + {node1, segmentinfo(n3, 10, 10)} + ], + Asm = append_many(new(), Segs), + ?assertEqual( + {incomplete, {missing, {segment, 15, 20}}}, + status(coverage, Asm) + ). + +missing_coverage_with_redudancy_test() -> + Segs = [ + {node(), segmentinfo(n1, 0, 10)}, + {node(), segmentinfo(n2, 0, 20)}, + {node42, segmentinfo(n3, 10, 20)}, + {node43, segmentinfo(n4, 10, 50)}, + {node(), segmentinfo(n5, 40, 60)} + ], + Asm = append_many(new(), Segs), + ?assertEqual( + % {incomplete, {missing, {segment, 50, 60}}}, ??? + {incomplete, {missing, {segment, 20, 40}}}, + status(coverage, Asm) + ). + +append_many(Asm, List) -> + lists:foldl( + fun({Node, Frag}, Acc) -> append(Acc, Node, Frag) end, + Asm, + List + ). + +filemeta(Path, Name) -> + #{ + path => Path, + fragment => + {filemeta, #{ + name => Name + }} + }. + +segmentinfo(Path, Offset, Size) -> + #{ + path => Path, + fragment => + {segmentinfo, #{ + offset => Offset, + size => Size + }} + }. + +-endif. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl new file mode 100644 index 000000000..2d50e1b18 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -0,0 +1,425 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +% -compile(export_all). + +-export([store_filemeta/3]). +-export([store_segment/3]). +-export([list/2]). +-export([assemble/2]). + +-export([open_file/3]). +-export([complete/4]). +-export([write/2]). +-export([discard/1]). + +% -behaviour(gen_server). +% -export([init/1]). +% -export([handle_call/3]). +% -export([handle_cast/2]). + +-type json_value() :: + null + | boolean() + | binary() + | number() + | [json_value()] + | #{binary() => json_value()}. + +-reflect_type([json_value/0]). + +-type transfer() :: emqx_ft:transfer(). +-type offset() :: emqx_ft:offset(). + +%% TODO: move to `emqx_ft` interface module +% -type sha256_hex() :: <<_:512>>. + +-type filemeta() :: #{ + %% Display name + name := string(), + %% Size in bytes, as advertised by the client. + %% Client is free to specify here whatever it wants, which means we can end + %% up with a file of different size after assembly. It's not clear from + %% specification what that means (e.g. what are clients' expectations), we + %% currently do not condider that an error (or, specifically, a signal that + %% the resulting file is corrupted during transmission). + size => _Bytes :: non_neg_integer(), + checksum => {sha256, <<_:256>>}, + expire_at := emqx_datetime:epoch_second(), + %% TTL of individual segments + %% Somewhat confusing that we won't know it on the nodes where the filemeta + %% is missing. + segments_ttl => _Seconds :: pos_integer(), + user_data => json_value() +}. + +-type segment() :: {offset(), _Content :: binary()}. + +-type segmentinfo() :: #{ + offset := offset(), + size := _Bytes :: non_neg_integer() +}. + +-type filefrag(T) :: #{ + path := file:name(), + timestamp := emqx_datetime:epoch_second(), + fragment := T +}. + +-type filefrag() :: filefrag({filemeta, filemeta()} | {segment, segmentinfo()}). + +-define(MANIFEST, "MANIFEST.json"). +-define(SEGMENT, "SEG"). +-define(TEMP, "TMP"). + +-type root() :: file:name(). + +% -record(st, { +% root :: file:name() +% }). + +%% TODO +-type storage() :: root(). + +%% + +-define(PROCREF(Root), {via, gproc, {n, l, {?MODULE, Root}}}). + +-spec start_link(root()) -> + {ok, pid()} | {error, already_started}. +start_link(Root) -> + gen_server:start_link(?PROCREF(Root), ?MODULE, [], []). + +%% Store manifest in the backing filesystem. +%% Atomic operation. +-spec store_filemeta(storage(), transfer(), filemeta()) -> + % Quota? Some lower level errors? + ok | {error, conflict} | {error, _TODO}. +store_filemeta(Storage, Transfer, Meta) -> + Filepath = mk_filepath(Storage, Transfer, ?MANIFEST), + case read_file(Filepath, fun decode_filemeta/1) of + {ok, Meta} -> + _ = touch_file(Filepath), + ok; + {ok, Conflict} -> + % TODO + % We won't see conflicts in case of concurrent `store_filemeta` + % requests. It's rather odd scenario so it's fine not to worry + % about it too much now. + {error, conflict}; + {error, Reason} when Reason =:= notfound; Reason =:= corrupted -> + write_file_atomic(Filepath, encode_filemeta(Meta)) + end. + +%% Store a segment in the backing filesystem. +%% Atomic operation. +-spec store_segment(storage(), transfer(), segment()) -> + % Where is the checksum gets verified? Upper level probably. + % Quota? Some lower level errors? + ok | {error, _TODO}. +store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> + Filepath = mk_filepath(Storage, Transfer, mk_segment_filename(Segment)), + write_file_atomic(Filepath, Content). + +-spec list(storage(), transfer()) -> + % Some lower level errors? {error, notfound}? + % Result will contain zero or only one filemeta. + {ok, list(filefrag())} | {error, _TODO}. +list(Storage, Transfer) -> + Dirname = mk_filedir(Storage, Transfer), + case file:list_dir(Dirname) of + {ok, Filenames} -> + {ok, filtermap_files(fun mk_filefrag/2, Dirname, Filenames)}; + {error, enoent} -> + {ok, []}; + {error, _} = Error -> + Error + end. + +-spec assemble(storage(), transfer()) -> + % {ok, _Assembler :: pid()} | {error, incomplete} | {error, badrpc} | {error, _TODO}. + {ok, _Assembler :: pid()} | {error, _TODO}. +assemble(Storage, Transfer) -> + emqx_ft_assembler_sup:start_child(Storage, Storage, Transfer). + +%% + +-opaque handle() :: {file:name(), io:device(), crypto:hash_state()}. + +-spec open_file(storage(), transfer(), filemeta()) -> + {ok, handle()} | {error, _TODO}. +open_file(Storage, Transfer, Filemeta) -> + Filename = maps:get(name, Filemeta), + Filepath = mk_filepath(Storage, Transfer, Filename), + TempFilepath = mk_temp_filepath(Filepath), + case file:open(TempFilepath, [write, raw]) of + {ok, Handle} -> + _ = file:truncate(Handle), + {ok, {TempFilepath, Handle, init_checksum(Filemeta)}}; + {error, _} = Error -> + Error + end. + +-spec write(handle(), iodata()) -> + ok | {error, _TODO}. +write({Filepath, IoDevice, Ctx}, IoData) -> + case file:write(IoDevice, IoData) of + ok -> + {ok, {Filepath, IoDevice, update_checksum(Ctx, IoData)}}; + {error, _} = Error -> + Error + end. + +-spec complete(storage(), transfer(), filemeta(), handle()) -> + ok | {error, {checksum, _Algo, _Computed}} | {error, _TODO}. +complete(Storage, Transfer, Filemeta, Handle = {Filepath, IoDevice, Ctx}) -> + TargetFilepath = mk_filepath(Storage, Transfer, maps:get(name, Filemeta)), + case verify_checksum(Ctx, Filemeta) of + ok -> + ok = file:close(IoDevice), + file:rename(Filepath, TargetFilepath); + {error, _} = Error -> + _ = discard(Handle), + Error + end. + +-spec discard(handle()) -> + ok. +discard({Filepath, IoDevice, _Ctx}) -> + ok = file:close(IoDevice), + file:delete(Filepath). + +init_checksum(#{checksum := {Algo, _}}) -> + crypto:hash_init(Algo); +init_checksum(#{}) -> + undefined. + +update_checksum(Ctx, IoData) when Ctx /= undefined -> + crypto:hash_update(Ctx, IoData); +update_checksum(undefined, _IoData) -> + undefined. + +verify_checksum(Ctx, #{checksum := {Algo, Digest}}) when Ctx /= undefined -> + case crypto:hash_final(Ctx) of + Digest -> + ok; + Mismatch -> + {error, {checksum, Algo, binary:encode_hex(Mismatch)}} + end; +verify_checksum(undefined, _) -> + ok. + +%% + +-spec init(root()) -> {ok, storage()}. +init(Root) -> + % TODO: garbage_collect(...) + {ok, Root}. + +%% + +-define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). + +schema() -> + #{ + roots => [ + {name, hoconsc:mk(string(), #{required => true})}, + {size, hoconsc:mk(non_neg_integer())}, + {expire_at, hoconsc:mk(non_neg_integer(), #{required => true})}, + {checksum, hoconsc:mk({atom(), binary()}, #{converter => converter(checksum)})}, + {segments_ttl, hoconsc:mk(pos_integer())}, + {user_data, hoconsc:mk(json_value())} + ] + }. + +% encode_filemeta(Meta) -> +% emqx_json:encode( +% ?PRELUDE( +% _Vsn = 1, +% maps:map( +% fun +% (name, Name) -> +% {<<"name">>, Name}; +% (size, Size) -> +% {<<"size">>, Size}; +% (checksum, {sha256, Hash}) -> +% {<<"checksum">>, <<"sha256:", (binary:encode_hex(Hash))/binary>>}; +% (expire_at, ExpiresAt) -> +% {<<"expire_at">>, ExpiresAt}; +% (segments_ttl, TTL) -> +% {<<"segments_ttl">>, TTL}; +% (user_data, UserData) -> +% {<<"user_data">>, UserData} +% end, +% Meta +% ) +% ) +% ). + +encode_filemeta(Meta) -> + % TODO: Looks like this should be hocon's responsibility. + Term = hocon_tconf:make_serializable(schema(), emqx_map_lib:binary_key_map(Meta), #{}), + emqx_json:encode(?PRELUDE(_Vsn = 1, Term)). + +decode_filemeta(Binary) -> + ?PRELUDE(_Vsn = 1, Term) = emqx_json:decode(Binary, [return_maps]), + hocon_tconf:check_plain(schema(), Term, #{atom_key => true, required => false}). + +converter(checksum) -> + fun + (undefined, #{}) -> + undefined; + ({sha256, Bin}, #{make_serializable := true}) -> + _ = is_binary(Bin) orelse throw({expected_type, string}), + _ = byte_size(Bin) =:= 32 orelse throw({expected_length, 32}), + binary:encode_hex(Bin); + (Hex, #{}) -> + _ = is_binary(Hex) orelse throw({expected_type, string}), + _ = byte_size(Hex) =:= 64 orelse throw({expected_length, 64}), + {sha256, binary:decode_hex(Hex)} + end. + +% map_into(Fun, Into, Ks, Map) -> +% map_foldr(map_into_fn(Fun, Into), Into, Ks, Map). + +% map_into_fn(Fun, L) when is_list(L) -> +% fun(K, V, Acc) -> [{K, Fun(K, V)} || Acc] end. + +% map_foldr(_Fun, Acc, [], _) -> +% Acc; +% map_foldr(Fun, Acc, [K | Ks], Map) when is_map_key(K, Map) -> +% Fun(K, maps:get(K, Map), map_foldr(Fun, Acc, Ks, Map)); +% map_foldr(Fun, Acc, [_ | Ks], Map) -> +% map_foldr(Fun, Acc, Ks, Map). + +%% + +mk_segment_filename({Offset, Content}) -> + lists:concat([?SEGMENT, ".", Offset, ".", byte_size(Content)]). + +break_segment_filename(Filename) -> + Regex = "^" ?SEGMENT "[.]([0-9]+)[.]([0-9]+)$", + Result = re:run(Filename, Regex, [{capture, all_but_first, list}]), + case Result of + {match, [Offset, Size]} -> + {ok, #{offset => list_to_integer(Offset), size => list_to_integer(Size)}}; + nomatch -> + {error, invalid} + end. + +mk_filedir(Storage, {ClientId, FileId}) -> + filename:join([get_storage_root(Storage), ClientId, FileId]). + +mk_filepath(Storage, Transfer, Filename) -> + filename:join(mk_filedir(Storage, Transfer), Filename). + +get_storage_root(Storage) -> + Storage. + +%% + +-include_lib("kernel/include/file.hrl"). + +read_file(Filepath) -> + file:read_file(Filepath). + +read_file(Filepath, DecodeFun) -> + case read_file(Filepath) of + {ok, Content} -> + safe_decode(Content, DecodeFun); + {error, _} = Error -> + Error + end. + +safe_decode(Content, DecodeFun) -> + try + {ok, DecodeFun(Content)} + catch + C:R:Stacktrace -> + % TODO: Log? + {error, corrupted} + end. + +write_file_atomic(Filepath, Content) when is_binary(Content) -> + Result = emqx_misc:pipeline( + [ + fun filelib:ensure_dir/1, + fun mk_temp_filepath/1, + fun write_contents/2, + fun mv_temp_file/1 + ], + Filepath, + Content + ), + case Result of + {ok, {Filepath, TempFilepath}, _} -> + _ = file:delete(TempFilepath), + ok; + {error, Reason, _} -> + {error, Reason} + end. + +mk_temp_filepath(Filepath) -> + Dirname = filename:dirname(Filepath), + Filename = filename:basename(Filepath), + Unique = erlang:unique_integer([positive]), + TempFilepath = filename:join(Dirname, ?TEMP ++ integer_to_list(Unique) ++ "." ++ Filename), + {Filepath, TempFilepath}. + +write_contents({_Filepath, TempFilepath}, Content) -> + file:write_file(TempFilepath, Content). + +mv_temp_file({Filepath, TempFilepath}) -> + file:rename(TempFilepath, Filepath). + +touch_file(Filepath) -> + Now = erlang:localtime(), + file:change_time(Filepath, _Mtime = Now, _Atime = Now). + +filtermap_files(Fun, Dirname, Filenames) -> + lists:filtermap(fun(Filename) -> Fun(Dirname, Filename) end, Filenames). + +mk_filefrag(Dirname, Filename = ?MANIFEST) -> + mk_filefrag(Dirname, Filename, fun read_filemeta/2); +mk_filefrag(Dirname, Filename = ?SEGMENT ++ _) -> + mk_filefrag(Dirname, Filename, fun read_segmentinfo/2); +mk_filefrag(_Dirname, _) -> + false. + +mk_filefrag(Dirname, Filename, Fun) -> + Filepath = filename:join(Dirname, Filename), + Fileinfo = file:read_file_info(Filepath), + case Fun(Filename, Filepath) of + {ok, Frag} -> + {true, #{ + path => Filepath, + timestamp => Fileinfo#file_info.mtime, + fragment => Frag + }}; + {error, Reason} -> + false + end. + +read_filemeta(_Filename, Filepath) -> + read_file(Filepath, fun decode_filemeta/1). + +read_segmentinfo(Filename, _Filepath) -> + break_segment_filename(Filename). From d36ca18bff8b71c1c3f8d44ae096da3fab6b8c7c Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 23 Jan 2023 00:30:26 +0200 Subject: [PATCH 004/156] feat(ft): tie file transfer frontend and backend together --- apps/emqx/src/emqx_channel.erl | 29 +++- apps/emqx/src/emqx_cm.erl | 2 +- apps/emqx/test/emqx_channel_SUITE.erl | 2 +- .../emqx_channel_delayed_puback_SUITE.erl | 70 +++++++++ apps/emqx_ft/src/emqx_ft.erl | 139 ++++++++++++++++-- apps/emqx_ft/src/emqx_ft_assembler.erl | 22 +-- apps/emqx_ft/src/emqx_ft_assembler_sup.erl | 16 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 40 ++--- apps/emqx_ft/src/emqx_ft_sup.erl | 12 +- rebar.config.erl | 3 +- 10 files changed, 277 insertions(+), 58 deletions(-) create mode 100644 apps/emqx/test/emqx_channel_delayed_puback_SUITE.erl diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 009dc72ea..c67c02d66 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -744,9 +744,13 @@ do_publish(_PacketId, Msg = #message{qos = ?QOS_0}, Channel) -> {ok, NChannel}; do_publish(PacketId, Msg = #message{qos = ?QOS_1}, Channel) -> PubRes = emqx_broker:publish(Msg), - RC = puback_reason_code(PubRes), - NChannel = ensure_quota(PubRes, Channel), - handle_out(puback, {PacketId, RC}, NChannel); + RC = puback_reason_code(PacketId, Msg, PubRes), + case RC of + undefined -> + {ok, Channel}; + _Value -> + do_finish_publish(PacketId, PubRes, RC, Channel) + end; do_publish( PacketId, Msg = #message{qos = ?QOS_2}, @@ -754,7 +758,7 @@ do_publish( ) -> case emqx_session:publish(ClientInfo, PacketId, Msg, Session) of {ok, PubRes, NSession} -> - RC = puback_reason_code(PubRes), + RC = pubrec_reason_code(PubRes), NChannel0 = set_session(NSession, Channel), NChannel1 = ensure_timer(await_timer, NChannel0), NChannel2 = ensure_quota(PubRes, NChannel1), @@ -767,6 +771,10 @@ do_publish( handle_out(disconnect, RC, Channel) end. +do_finish_publish(PacketId, PubRes, RC, Channel) -> + NChannel = ensure_quota(PubRes, Channel), + handle_out(puback, {PacketId, RC}, NChannel). + ensure_quota(_, Channel = #channel{quota = undefined}) -> Channel; ensure_quota(PubRes, Channel = #channel{quota = Limiter}) -> @@ -786,9 +794,14 @@ ensure_quota(PubRes, Channel = #channel{quota = Limiter}) -> ensure_timer(quota_timer, Intv, Channel#channel{quota = NLimiter}) end. --compile({inline, [puback_reason_code/1]}). -puback_reason_code([]) -> ?RC_NO_MATCHING_SUBSCRIBERS; -puback_reason_code([_ | _]) -> ?RC_SUCCESS. +-compile({inline, [pubrec_reason_code/1]}). +pubrec_reason_code([]) -> ?RC_NO_MATCHING_SUBSCRIBERS; +pubrec_reason_code([_ | _]) -> ?RC_SUCCESS. + +puback_reason_code(PacketId, Msg, [] = PubRes) -> + emqx_hooks:run_fold('message.puback', [PacketId, Msg, PubRes], ?RC_NO_MATCHING_SUBSCRIBERS); +puback_reason_code(PacketId, Msg, [_ | _] = PubRes) -> + emqx_hooks:run_fold('message.puback', [PacketId, Msg, PubRes], ?RC_SUCCESS). -compile({inline, [after_message_acked/3]}). after_message_acked(ClientInfo, Msg, PubAckProps) -> @@ -1283,6 +1296,8 @@ handle_info(die_if_test = Info, Channel) -> die_if_test_compiled(), ?SLOG(error, #{msg => "unexpected_info", info => Info}), {ok, Channel}; +handle_info({puback, PacketId, PubRes, RC}, Channel) -> + do_finish_publish(PacketId, PubRes, RC, Channel); handle_info(Info, Channel) -> ?SLOG(error, #{msg => "unexpected_info", info => Info}), {ok, Channel}. diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index d2ac642ca..724139142 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -313,7 +313,7 @@ open_session(false, ClientInfo = #{clientid := ClientId}, #{conn_mod := NewConnM Session1 = emqx_persistent_session:persist( ClientInfo, ConnInfo, Session ), - ok = emqx_hooks:run('channel.takeovered', [NewConnMod, Self, TakoverData]), + ok = emqx_hooks:run('channel.takenover', [NewConnMod, Self, TakoverData]), {ok, #{ session => clean_session(Session1), present => true, diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 6dd389350..633337f80 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -1133,7 +1133,7 @@ t_ws_cookie_init(_) -> ?assertMatch(#{ws_cookie := WsCookie}, emqx_channel:info(clientinfo, Channel)). %%-------------------------------------------------------------------- -%% Test cases for other mechnisms +%% Test cases for other mechanisms %%-------------------------------------------------------------------- t_flapping_detect(_) -> diff --git a/apps/emqx/test/emqx_channel_delayed_puback_SUITE.erl b/apps/emqx/test/emqx_channel_delayed_puback_SUITE.erl new file mode 100644 index 000000000..4f2938b24 --- /dev/null +++ b/apps/emqx/test/emqx_channel_delayed_puback_SUITE.erl @@ -0,0 +1,70 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_channel_delayed_puback_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_hooks.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([]). + +init_per_testcase(Case, Config) -> + ?MODULE:Case({init, Config}). + +end_per_testcase(Case, Config) -> + ?MODULE:Case({'end', Config}). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_delayed_puback({init, Config}) -> + emqx_hooks:put('message.puback', {?MODULE, on_message_puback, []}, ?HP_LOWEST), + Config; +t_delayed_puback({'end', _Config}) -> + emqx_hooks:del('message.puback', {?MODULE, on_message_puback}); +t_delayed_puback(_Config) -> + {ok, ConnPid} = emqtt:start_link([{clientid, <<"clientid">>}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(ConnPid), + {ok, #{reason_code := ?RC_UNSPECIFIED_ERROR}} = emqtt:publish( + ConnPid, <<"topic">>, <<"hello">>, 1 + ), + emqtt:disconnect(ConnPid). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +on_message_puback(PacketId, _Msg, PubRes, _RC) -> + erlang:send(self(), {puback, PacketId, PubRes, ?RC_UNSPECIFIED_ERROR}), + {stop, undefined}. diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 1f8a90a6f..c73d31559 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -17,6 +17,10 @@ -module(emqx_ft). -include("emqx_ft.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_hooks.hrl"). -export([ create_tab/0, @@ -27,9 +31,14 @@ -export([ on_channel_unregistered/1, on_channel_takeover/3, - on_channel_takeovered/3 + on_channel_takenover/3, + on_message_publish/1, + on_message_puback/4 ]). +%% For Debug +-export([transfer/2, storage/0]). + -export_type([clientid/0]). -export_type([transfer/0]). -export_type([offset/0]). @@ -67,16 +76,20 @@ create_tab() -> ok. hook() -> - % ok = emqx_hooks:put('channel.registered', {?MODULE, on_channel_registered, []}), - ok = emqx_hooks:put('channel.unregistered', {?MODULE, on_channel_unregistered, []}), - ok = emqx_hooks:put('channel.takeover', {?MODULE, on_channel_takeover, []}), - ok = emqx_hooks:put('channel.takeovered', {?MODULE, on_channel_takeovered, []}). + ok = emqx_hooks:put('channel.unregistered', {?MODULE, on_channel_unregistered, []}, ?HP_LOWEST), + ok = emqx_hooks:put('channel.takeover', {?MODULE, on_channel_takeover, []}, ?HP_LOWEST), + ok = emqx_hooks:put('channel.takenover', {?MODULE, on_channel_takenover, []}, ?HP_LOWEST), + + ok = emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_LOWEST), + ok = emqx_hooks:put('message.puback', {?MODULE, on_message_puback, []}, ?HP_LOWEST). unhook() -> - % ok = emqx_hooks:del('channel.registered', {?MODULE, on_channel_registered}), ok = emqx_hooks:del('channel.unregistered', {?MODULE, on_channel_unregistered}), ok = emqx_hooks:del('channel.takeover', {?MODULE, on_channel_takeover}), - ok = emqx_hooks:del('channel.takeovered', {?MODULE, on_channel_takeovered}). + ok = emqx_hooks:del('channel.takenover', {?MODULE, on_channel_takenover}), + + ok = emqx_hooks:del('message.publish', {?MODULE, on_message_publish}), + ok = emqx_hooks:del('message.puback', {?MODULE, on_message_puback}). %%-------------------------------------------------------------------- %% Hooks @@ -93,11 +106,30 @@ on_channel_takeover(_ConnMod, ChanPid, TakeoverData) -> ok end. -on_channel_takeovered(_ConnMod, ChanPid, #{ft_data := FTData}) -> +on_channel_takenover(_ConnMod, ChanPid, #{ft_data := FTData}) -> ok = put_ft_data(ChanPid, FTData); -on_channel_takeovered(_ConnMod, _ChanPid, _) -> +on_channel_takenover(_ConnMod, _ChanPid, _) -> ok. +on_message_publish( + Msg = #message{ + id = _Id, + topic = <<"$file/", _/binary>> + } +) -> + Headers = Msg#message.headers, + {stop, Msg#message{headers = Headers#{allow_publish => false}}}; +on_message_publish(Msg) -> + {ok, Msg}. + +on_message_puback(PacketId, #message{topic = Topic} = Msg, _PubRes, _RC) -> + case Topic of + <<"$file/", FileCommand/binary>> -> + {stop, on_file_command(PacketId, Msg, FileCommand)}; + _ -> + ignore + end. + %%-------------------------------------------------------------------- %% Private funs %%-------------------------------------------------------------------- @@ -115,3 +147,92 @@ delete_ft_data(ChanPid) -> put_ft_data(ChanPid, FTData) -> true = ets:insert(?FT_TAB, #emqx_ft{chan_pid = ChanPid, ft_data = FTData}), ok. + +on_file_command(PacketId, Msg, FileCommand) -> + case string:split(FileCommand, <<"/">>, all) of + [FileId, <<"init">>] -> + on_init(Msg, FileId); + [FileId, <<"fin">>] -> + on_fin(PacketId, Msg, FileId, undefined); + [FileId, <<"fin">>, Checksum] -> + on_fin(PacketId, Msg, FileId, Checksum); + [FileId, <<"abort">>] -> + on_abort(Msg, FileId); + [FileId, Offset] -> + on_segment(Msg, FileId, Offset, undefined); + [FileId, Offset, Checksum] -> + on_segment(Msg, FileId, Offset, Checksum); + _ -> + ?RC_UNSPECIFIED_ERROR + end. + +on_init(Msg, FileId) -> + ?SLOG(info, #{ + msg => "on_init", + mqtt_msg => Msg, + file_id => FileId + }), + % Payload = Msg#message.payload, + % %% Add validations here + % Meta = emqx_json:decode(Payload, [return_maps]), + % ok = emqx_ft_storage_fs:store_filemeta(storage(), transfer(Msg, FileId), Meta), + % ?RC_SUCCESS. + ?RC_UNSPECIFIED_ERROR. + +on_abort(_Msg, _FileId) -> + %% TODO + ?RC_SUCCESS. + +on_segment(Msg, FileId, Offset, Checksum) -> + ?SLOG(info, #{ + msg => "on_segment", + mqtt_msg => Msg, + file_id => FileId, + offset => Offset, + checksum => Checksum + }), + % %% TODO: handle checksum + % Payload = Msg#message.payload, + % %% Add offset/checksum validations + % ok = emqx_ft_storage_fs:store_segment( + % storage(), + % transfer(Msg, FileId), + % {binary_to_integer(Offset), Payload} + % ), + % ?RC_SUCCESS. + ?RC_UNSPECIFIED_ERROR. + +on_fin(PacketId, Msg, FileId, Checksum) -> + ?SLOG(info, #{ + msg => "on_fin", + mqtt_msg => Msg, + file_id => FileId, + checksum => Checksum, + packet_id => PacketId + }), + % %% TODO: handle checksum? Do we need it? + % {ok, _} = emqx_ft_storage_fs:assemble( + % storage(), + % transfer(Msg, FileId), + % callback(FileId, Msg) + % ), + Callback = callback(FileId, PacketId), + spawn(fun() -> Callback({error, not_implemented}) end), + undefined. + +callback(_FileId, PacketId) -> + ChanPid = self(), + fun + (ok) -> + erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS}); + ({error, _}) -> + erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}) + end. + +transfer(Msg, FileId) -> + ClientId = Msg#message.from, + {ClientId, FileId}. + +%% TODO: configure +storage() -> + filename:join(emqx:data_dir(), "file_transfer"). diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 2bd01c8b0..dfbb2bd3e 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -16,7 +16,7 @@ -module(emqx_ft_assembler). --export([start_link/2]). +-export([start_link/3]). -behaviour(gen_statem). -export([callback_mode/0]). @@ -35,7 +35,8 @@ transfer :: emqx_ft:transfer(), assembly :: _TODO, file :: io:device(), - hash + hash, + callback :: fun((ok | {error, term()}) -> any()) }). -define(RPC_LIST_TIMEOUT, 1000). @@ -43,8 +44,8 @@ %% -start_link(Storage, Transfer) -> - gen_server:start_link(?MODULE, {Storage, Transfer}, []). +start_link(Storage, Transfer, Callback) -> + gen_server:start_link(?MODULE, {Storage, Transfer, Callback}, []). %% @@ -53,12 +54,13 @@ start_link(Storage, Transfer) -> callback_mode() -> handle_event_function. -init({Storage, Transfer}) -> +init({Storage, Transfer, Callback}) -> St = #st{ storage = Storage, transfer = Transfer, assembly = emqx_ft_assembly:new(), - hash = crypto:hash_init(sha256) + hash = crypto:hash_init(sha256), + callback = Callback }, {ok, list_local_fragments, St, ?internal([])}. @@ -91,7 +93,7 @@ handle_event({list_remote_fragments, Nodes}, internal, _, St) -> fun ({Node, {ok, {ok, Fragments}}}, Asm) -> emqx_ft_assembly:append(Asm, Node, Fragments); - ({Node, Result}, Asm) -> + ({_Node, _Result}, Asm) -> % TODO: log? Asm end, @@ -128,9 +130,11 @@ handle_event({assemble, [{Node, Segment} | Rest]}, internal, _, St = #st{}) -> end; handle_event({assemble, []}, internal, _, St = #st{}) -> {next_state, complete, St, ?internal([])}; -handle_event(complete, internal, _, St = #st{assembly = Asm, file = Handle}) -> +handle_event(complete, internal, _, St = #st{assembly = Asm, file = Handle, callback = Callback}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), - ok = emqx_ft_storage_fs:complete(St#st.storage, St#st.transfer, Filemeta, Handle), + Result = emqx_ft_storage_fs:complete(St#st.storage, St#st.transfer, Filemeta, Handle), + %% TODO: safe apply + _ = Callback(Result), {stop, shutdown}. % handle_continue(list_local, St = #st{storage = Storage, transfer = Transfer, assembly = Asm}) -> diff --git a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl index fbf948fbb..e837d96f1 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl @@ -16,24 +16,22 @@ -module(emqx_ft_assembler_sup). --export([start_link/1]). +-export([start_link/0]). -export([start_child/3]). -behaviour(supervisor). -export([init/1]). --define(REF(ID), {via, gproc, {n, l, {?MODULE, ID}}}). +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). -start_link(ID) -> - supervisor:start_link(?REF(ID), ?MODULE, []). - -start_child(ID, Storage, Transfer) -> +start_child(Storage, Transfer, Callback) -> Childspec = #{ id => {Storage, Transfer}, - start => {emqx_ft_assembler, start_link, [Storage, Transfer]}, + start => {emqx_ft_assembler, start_link, [Storage, Transfer, Callback]}, restart => transient }, - supervisor:start_child(?REF(ID), Childspec). + supervisor:start_child(?MODULE, Childspec). init(_) -> SupFlags = #{ @@ -41,4 +39,4 @@ init(_) -> intensity => 100, period => 1000 }, - {ok, SupFlags, []}. + {ok, {SupFlags, []}}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 2d50e1b18..15efa142b 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -24,7 +24,7 @@ -export([store_filemeta/3]). -export([store_segment/3]). -export([list/2]). --export([assemble/2]). +-export([assemble/3]). -export([open_file/3]). -export([complete/4]). @@ -101,12 +101,12 @@ %% --define(PROCREF(Root), {via, gproc, {n, l, {?MODULE, Root}}}). +% -define(PROCREF(Root), {via, gproc, {n, l, {?MODULE, Root}}}). --spec start_link(root()) -> - {ok, pid()} | {error, already_started}. -start_link(Root) -> - gen_server:start_link(?PROCREF(Root), ?MODULE, [], []). +% -spec start_link(root()) -> +% {ok, pid()} | {error, already_started}. +% start_link(Root) -> +% gen_server:start_link(?PROCREF(Root), ?MODULE, [], []). %% Store manifest in the backing filesystem. %% Atomic operation. @@ -119,13 +119,13 @@ store_filemeta(Storage, Transfer, Meta) -> {ok, Meta} -> _ = touch_file(Filepath), ok; - {ok, Conflict} -> + {ok, _Conflict} -> % TODO % We won't see conflicts in case of concurrent `store_filemeta` % requests. It's rather odd scenario so it's fine not to worry % about it too much now. {error, conflict}; - {error, Reason} when Reason =:= notfound; Reason =:= corrupted -> + {error, Reason} when Reason =:= notfound; Reason =:= corrupted; Reason =:= enoent -> write_file_atomic(Filepath, encode_filemeta(Meta)) end. @@ -154,15 +154,15 @@ list(Storage, Transfer) -> Error end. --spec assemble(storage(), transfer()) -> +-spec assemble(storage(), transfer(), fun((ok | {error, term()}) -> any())) -> % {ok, _Assembler :: pid()} | {error, incomplete} | {error, badrpc} | {error, _TODO}. {ok, _Assembler :: pid()} | {error, _TODO}. -assemble(Storage, Transfer) -> - emqx_ft_assembler_sup:start_child(Storage, Storage, Transfer). +assemble(Storage, Transfer, Callback) -> + emqx_ft_assembler_sup:start_child(Storage, Transfer, Callback). %% --opaque handle() :: {file:name(), io:device(), crypto:hash_state()}. +-type handle() :: {file:name(), io:device(), crypto:hash_state()}. -spec open_file(storage(), transfer(), filemeta()) -> {ok, handle()} | {error, _TODO}. @@ -229,12 +229,12 @@ verify_checksum(undefined, _) -> %% --spec init(root()) -> {ok, storage()}. -init(Root) -> - % TODO: garbage_collect(...) - {ok, Root}. +% -spec init(root()) -> {ok, storage()}. +% init(Root) -> +% % TODO: garbage_collect(...) +% {ok, Root}. -%% +% %% -define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). @@ -243,7 +243,7 @@ schema() -> roots => [ {name, hoconsc:mk(string(), #{required => true})}, {size, hoconsc:mk(non_neg_integer())}, - {expire_at, hoconsc:mk(non_neg_integer(), #{required => true})}, + {expire_at, hoconsc:mk(non_neg_integer())}, {checksum, hoconsc:mk({atom(), binary()}, #{converter => converter(checksum)})}, {segments_ttl, hoconsc:mk(pos_integer())}, {user_data, hoconsc:mk(json_value())} @@ -353,7 +353,7 @@ safe_decode(Content, DecodeFun) -> try {ok, DecodeFun(Content)} catch - C:R:Stacktrace -> + _C:_R:_Stacktrace -> % TODO: Log? {error, corrupted} end. @@ -414,7 +414,7 @@ mk_filefrag(Dirname, Filename, Fun) -> timestamp => Fileinfo#file_info.mtime, fragment => Frag }}; - {error, Reason} -> + {error, _Reason} -> false end. diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index 4c976246b..fb7a6104f 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -43,7 +43,17 @@ init([]) -> intensity => 100, period => 10 }, - ChildSpecs = [], + + AssemblerSup = #{ + id => emqx_ft_assembler_sup, + start => {emqx_ft_assembler_sup, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [emqx_ft_assembler_sup] + }, + + ChildSpecs = [AssemblerSup], {ok, {SupFlags, ChildSpecs}}. %% internal functions diff --git a/rebar.config.erl b/rebar.config.erl index 98cd30570..ea0016ca9 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -400,7 +400,8 @@ relx_apps(ReleaseType, Edition) -> emqx_prometheus, emqx_psk, emqx_slow_subs, - emqx_plugins + emqx_plugins, + emqx_ft ] ++ [quicer || is_quicer_supported()] ++ [bcrypt || provide_bcrypt_release(ReleaseType)] ++ From cbff2e2309666679f2be8756e57f815a7604173b Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 1 Feb 2023 23:31:52 +0200 Subject: [PATCH 005/156] feat(ft): improve robustness of asynchronous acks * add auto ack after timeout * add fin file transfer packet registration to avoid duplication and multiple acks --- apps/emqx_ft/src/emqx_ft.erl | 83 ++++++++++++++---- apps/emqx_ft/src/emqx_ft_responder.erl | 114 +++++++++++++++++++++++++ apps/emqx_ft/src/emqx_ft_sup.erl | 11 ++- 3 files changed, 191 insertions(+), 17 deletions(-) create mode 100644 apps/emqx_ft/src/emqx_ft_responder.erl diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index c73d31559..014c74ac3 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -39,6 +39,8 @@ %% For Debug -export([transfer/2, storage/0]). +-export([on_assemble_timeout/1]). + -export_type([clientid/0]). -export_type([transfer/0]). -export_type([offset/0]). @@ -62,6 +64,8 @@ ft_data :: ft_data() }). +-define(ASSEMBLE_TIMEOUT, 5000). + %%-------------------------------------------------------------------- %% API for app %%-------------------------------------------------------------------- @@ -210,23 +214,66 @@ on_fin(PacketId, Msg, FileId, Checksum) -> checksum => Checksum, packet_id => PacketId }), - % %% TODO: handle checksum? Do we need it? - % {ok, _} = emqx_ft_storage_fs:assemble( - % storage(), - % transfer(Msg, FileId), - % callback(FileId, Msg) - % ), - Callback = callback(FileId, PacketId), - spawn(fun() -> Callback({error, not_implemented}) end), - undefined. + %% TODO: handle checksum? Do we need it? + FinPacketKey = {self(), PacketId}, + _ = + case + emqx_ft_responder:register( + FinPacketKey, fun ?MODULE:on_assemble_timeout/1, ?ASSEMBLE_TIMEOUT + ) + of + %% We have new fin packet + ok -> + Callback = callback(FinPacketKey, FileId), + case assemble(transfer(Msg, FileId), Callback) of + %% Assembling started, packet will be acked by the callback or the responder + ok -> + undefined; + %% Assembling failed, unregister the packet key + {error, _} -> + case emqx_ft_responder:unregister(FinPacketKey) of + %% We successfully unregistered the packet key, + %% so we can send the error code at once + ok -> + ?RC_UNSPECIFIED_ERROR; + %% Someone else already unregistered the key, + %% that is, either responder or someone else acked the packet, + %% we do not have to ack + {error, not_found} -> + undefined + end + end; + %% Fin packet already received. + %% Since we are still handling the previous one, + %% we probably have retransmit here + {error, already_registered} -> + undefined + end. -callback(_FileId, PacketId) -> - ChanPid = self(), - fun - (ok) -> - erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS}); - ({error, _}) -> - erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}) +assemble(_Transfer, _Callback) -> + % spawn(fun() -> Callback({error, not_implemented}) end), + ok. + +% assemble(Transfer, Callback) -> +% emqx_ft_storage_fs:assemble( +% storage(), +% Transfer, +% Callback +% ). + +callback({ChanPid, PacketId} = Key, _FileId) -> + fun(Result) -> + case emqx_ft_responder:unregister(Key) of + ok -> + case Result of + {ok, _} -> + erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS}); + {error, _} -> + erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}) + end; + {error, not_registered} -> + ok + end end. transfer(Msg, FileId) -> @@ -236,3 +283,7 @@ transfer(Msg, FileId) -> %% TODO: configure storage() -> filename:join(emqx:data_dir(), "file_transfer"). + +on_assemble_timeout({ChanPid, PacketId}) -> + ?SLOG(warning, #{msg => "on_assemble_timeout", packet_id => PacketId}), + erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}). diff --git a/apps/emqx_ft/src/emqx_ft_responder.erl b/apps/emqx_ft/src/emqx_ft_responder.erl new file mode 100644 index 000000000..dcb45d5d3 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_responder.erl @@ -0,0 +1,114 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_responder). + +-behaviour(gen_server). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). + +-export([start_link/0]). + +-export([ + register/3, + unregister/1 +]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). + +-define(SERVER, ?MODULE). +-define(TAB, ?MODULE). + +-type key() :: term(). + +%%-------------------------------------------------------------------- +%% API +%% ------------------------------------------------------------------- + +-spec start_link() -> startlink_ret(). +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + +-spec register(Key, DefaultAction, Timeout) -> ok | {error, already_registered} when + Key :: key(), + DefaultAction :: fun((Key) -> any()), + Timeout :: timeout(). +register(Key, DefaultAction, Timeout) -> + case ets:lookup(?TAB, Key) of + [] -> + gen_server:call(?SERVER, {register, Key, DefaultAction, Timeout}); + [{Key, _Action, _Ref}] -> + {error, already_registered} + end. + +-spec unregister(Key) -> ok | {error, not_found} when + Key :: key(). +unregister(Key) -> + gen_server:call(?SERVER, {unregister, Key}). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%% ------------------------------------------------------------------- + +init([]) -> + _ = ets:new(?TAB, [named_table, protected, set, {read_concurrency, true}]), + {ok, #{}}. + +handle_call({register, Key, DefaultAction, Timeout}, _From, State) -> + ?SLOG(warning, #{msg => "register", key => Key, timeout => Timeout}), + case ets:lookup(?TAB, Key) of + [] -> + TRef = erlang:start_timer(Timeout, self(), {timeout, Key}), + true = ets:insert(?TAB, {Key, DefaultAction, TRef}), + {reply, ok, State}; + [{_, _Action, _Ref}] -> + {reply, {error, already_registered}, State} + end; +handle_call({unregister, Key}, _From, State) -> + case ets:lookup(?TAB, Key) of + [] -> + {reply, {error, not_found}, State}; + [{_, _Action, TRef}] -> + _ = erlang:cancel_timer(TRef), + true = ets:delete(?TAB, Key), + {reply, ok, State} + end. + +handle_cast(Msg, State) -> + ?SLOG(warning, #{msg => "unknown cast", cast_msg => Msg}), + {noreply, State}. + +handle_info({timeout, TRef, {timeout, Key}}, State) -> + case ets:lookup(?TAB, Key) of + [] -> + {noreply, State}; + [{_, Action, TRef}] -> + _ = erlang:cancel_timer(TRef), + true = ets:delete(?TAB, Key), + %% TODO: safe apply + _ = Action(Key), + {noreply, State} + end; +handle_info(Msg, State) -> + ?SLOG(warning, #{msg => "unknown message", info_msg => Msg}), + {noreply, State}. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +terminate(_Reason, _State) -> + ok. diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index fb7a6104f..ef2d8033f 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -53,7 +53,16 @@ init([]) -> modules => [emqx_ft_assembler_sup] }, - ChildSpecs = [AssemblerSup], + Responder = #{ + id => emqx_ft_responder, + start => {emqx_ft_responder, start_link, []}, + restart => permanent, + shutdown => infinity, + type => worker, + modules => [emqx_ft_responder] + }, + + ChildSpecs = [Responder, AssemblerSup], {ok, {SupFlags, ChildSpecs}}. %% internal functions From 7b77e96ab9fe3020e93985fb9d570226db98d680 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 2 Feb 2023 21:25:31 +0300 Subject: [PATCH 006/156] test(ft): add some basic assembler tests --- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 163 +++++++++++++ apps/emqx_ft/test/emqx_ft_content_gen.erl | 229 ++++++++++++++++++ 2 files changed, 392 insertions(+) create mode 100644 apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl create mode 100644 apps/emqx_ft/test/emqx_ft_content_gen.erl diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl new file mode 100644 index 000000000..1112ccbb7 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -0,0 +1,163 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_assembler_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + % {ok, Apps} = application:ensure_all_started(emqx_ft), + % [{suite_apps, Apps} | Config]. + % ok = emqx_common_test_helpers:start_apps([emqx_ft]), + Config. + +end_per_suite(_Config) -> + % lists:foreach(fun application:stop/1, lists:reverse(?config(suite_apps, Config))). + % ok = emqx_common_test_helpers:stop_apps([emqx_ft]), + ok. + +init_per_testcase(TC, Config) -> + ok = snabbkaffe:start_trace(), + Root = filename:join(["roots", TC]), + {ok, Pid} = emqx_ft_assembler_sup:start_link(), + [{storage_root, Root}, {assembler_sup, Pid} | Config]. + +end_per_testcase(_TC, Config) -> + ok = inspect_storage_root(Config), + ok = gen:stop(?config(assembler_sup, Config)), + ok = snabbkaffe:stop(), + ok. + +%% + +-define(CLIENTID, <<"thatsme">>). + +t_assemble_empty_transfer(Config) -> + Storage = ?config(storage_root, Config), + Transfer = {?CLIENTID, mk_fileid()}, + Filename = "important.pdf", + Meta = #{ + name => Filename, + size => 0, + expire_at => 42 + }, + ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta), + ?assertMatch( + {ok, [ + #{ + path := _, + timestamp := {{_, _, _}, {_, _, _}}, + fragment := {filemeta, Meta} + } + ]}, + emqx_ft_storage_fs:list(Storage, Transfer) + ), + {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun on_assembly_finished/1), + {ok, Event} = ?block_until(#{?snk_kind := test_assembly_finished}), + ?assertMatch(#{result := ok}, Event), + ?assertEqual( + {ok, <<>>}, + % TODO + file:read_file(mk_assembly_filename(Config, Transfer, Filename)) + ), + ok. + +t_assemble_complete_local_transfer(Config) -> + Storage = ?config(storage_root, Config), + Transfer = {?CLIENTID, mk_fileid()}, + Filename = "topsecret.pdf", + TransferSize = 10000 + rand:uniform(50000), + SegmentSize = 4096, + Gen = emqx_ft_content_gen:new({Transfer, TransferSize}, SegmentSize), + Hash = emqx_ft_content_gen:hash(Gen, crypto:hash_init(sha256)), + Meta = #{ + name => Filename, + checksum => {sha256, Hash}, + expire_at => 42 + }, + + ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta), + _ = emqx_ft_content_gen:consume( + Gen, + fun({Content, SegmentNum, _SegmentCount}) -> + Offset = (SegmentNum - 1) * SegmentSize, + ?assertEqual( + ok, + emqx_ft_storage_fs:store_segment(Storage, Transfer, {Offset, Content}) + ) + end + ), + + {ok, Fragments} = emqx_ft_storage_fs:list(Storage, Transfer), + ?assertEqual((TransferSize div SegmentSize) + 1 + 1, length(Fragments)), + ?assertEqual( + [Meta], + [FM || #{fragment := {filemeta, FM}} <- Fragments], + Fragments + ), + + {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun on_assembly_finished/1), + {ok, Event} = ?block_until(#{?snk_kind := test_assembly_finished}), + ?assertMatch(#{result := ok}, Event), + + AssemblyFilename = mk_assembly_filename(Config, Transfer, Filename), + ?assertMatch( + {ok, #file_info{type = regular, size = TransferSize}}, + file:read_file_info(AssemblyFilename) + ), + ok = emqx_ft_content_gen:check_file_consistency( + {Transfer, TransferSize}, + 100, + AssemblyFilename + ). + +mk_assembly_filename(Config, {ClientID, FileID}, Filename) -> + filename:join([?config(storage_root, Config), ClientID, FileID, Filename]). + +on_assembly_finished(Result) -> + ?tp(test_assembly_finished, #{result => Result}). + +%% + +-include_lib("kernel/include/file.hrl"). + +inspect_storage_root(Config) -> + inspect_dir(?config(storage_root, Config)). + +inspect_dir(Dir) -> + FileInfos = filelib:fold_files( + Dir, + ".*", + true, + fun(Filename, Acc) -> orddict:store(Filename, inspect_file(Filename), Acc) end, + orddict:new() + ), + ct:pal("inspect '~s': ~p", [Dir, FileInfos]). + +inspect_file(Filename) -> + {ok, Info} = file:read_file_info(Filename), + {Info#file_info.type, Info#file_info.size, Info#file_info.mtime}. + +mk_fileid() -> + integer_to_binary(erlang:system_time(millisecond)). diff --git a/apps/emqx_ft/test/emqx_ft_content_gen.erl b/apps/emqx_ft/test/emqx_ft_content_gen.erl new file mode 100644 index 000000000..feca78949 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_content_gen.erl @@ -0,0 +1,229 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Inspired by +%% https://github.com/kafka4beam/kflow/blob/master/src/testbed/payload_gen.erl + +-module(emqx_ft_content_gen). + +-include_lib("eunit/include/eunit.hrl"). + +-dialyzer(no_improper_lists). + +-export([new/2]). +-export([generate/3]). +-export([next/1]). +-export([consume/1]). +-export([consume/2]). +-export([fold/3]). + +-export([hash/2]). +-export([check_file_consistency/3]). + +-export_type([cont/1]). +-export_type([stream/1]). +-export_type([binary_payload/0]). + +-define(hash_size, 16). + +-type payload() :: {Seed :: term(), Size :: integer()}. + +-type binary_payload() :: { + binary(), _ChunkNum :: non_neg_integer(), _ChunkCnt :: non_neg_integer() +}. + +-type cont(Data) :: + fun(() -> stream(Data)) + | stream(Data). + +-type stream(Data) :: + maybe_improper_list(Data, cont(Data)) + | eos. + +-record(chunk_state, { + seed :: term(), + payload_size :: non_neg_integer(), + offset :: non_neg_integer(), + chunk_size :: non_neg_integer() +}). + +-type chunk_state() :: #chunk_state{}. + +%% ----------------------------------------------------------------------------- +%% Generic streams +%% ----------------------------------------------------------------------------- + +%% @doc Consume one element from the stream. +-spec next(cont(A)) -> stream(A). +next(Fun) when is_function(Fun, 0) -> + Fun(); +next(L) -> + L. + +%% @doc Consume all elements of the stream and feed them into a +%% callback (e.g. brod:produce) +-spec consume(cont(A), fun((A) -> Ret)) -> [Ret]. +consume([Data | Cont], Callback) -> + [Callback(Data) | consume(next(Cont), Callback)]; +consume(Cont, Callback) when is_function(Cont, 0) -> + consume(next(Cont), Callback); +consume(eos, _Callback) -> + []. + +%% @equiv consume(Stream, fun(A) -> A end) +-spec consume(cont(A)) -> [A]. +consume(Stream) -> + consume(Stream, fun(A) -> A end). + +-spec fold(fun((A, Acc) -> Acc), Acc, cont(A)) -> Acc. +fold(Fun, Acc, [Data | Cont]) -> + fold(Fun, Fun(Data, Acc), next(Cont)); +fold(Fun, Acc, Cont) when is_function(Cont, 0) -> + fold(Fun, Acc, next(Cont)); +fold(_Fun, Acc, eos) -> + Acc. + +%% ----------------------------------------------------------------------------- +%% Binary streams +%% ----------------------------------------------------------------------------- + +%% @doc Stream of binary chunks. +%% Limitation: `ChunkSize' should be dividable by `?hash_size' +-spec new(payload(), integer()) -> cont(binary_payload()). +new({Seed, Size}, ChunkSize) when ChunkSize rem ?hash_size =:= 0 -> + fun() -> + generate_next_chunk(#chunk_state{ + seed = Seed, + payload_size = Size, + chunk_size = ChunkSize, + offset = 0 + }) + end. + +%% @doc Generate chunks of data and feed them into +%% `Callback' +-spec generate(payload(), integer(), fun((binary_payload()) -> A)) -> [A]. +generate(Payload, ChunkSize, Callback) -> + consume(new(Payload, ChunkSize), Callback). + +-spec hash(cont(binary_payload()), crypto:hash_state()) -> binary(). +hash(Stream, HashCtxIn) -> + crypto:hash_final( + fold( + fun({Chunk, _, _}, HashCtx) -> + crypto:hash_update(HashCtx, Chunk) + end, + HashCtxIn, + Stream + ) + ). + +-spec check_consistency( + payload(), + integer(), + fun((integer()) -> {ok, binary()} | undefined) +) -> ok. +check_consistency({Seed, Size}, SampleSize, Callback) -> + SeedHash = seed_hash(Seed), + Random = [rand:uniform(Size) - 1 || _ <- lists:seq(1, SampleSize)], + %% Always check first and last bytes, and one that should not exist: + Samples = [0, Size - 1, Size | Random], + lists:foreach( + fun + (N) when N < Size -> + Expected = do_get_byte(N, SeedHash), + ?assertEqual( + {N, {ok, Expected}}, + {N, Callback(N)} + ); + (N) -> + ?assertMatch(undefined, Callback(N)) + end, + Samples + ). + +-spec check_file_consistency( + payload(), + integer(), + file:filename() +) -> ok. +check_file_consistency(Payload, SampleSize, FileName) -> + {ok, FD} = file:open(FileName, [read, raw]), + try + Fun = fun(N) -> + case file:pread(FD, [{N, 1}]) of + {ok, [[X]]} -> {ok, X}; + {ok, [eof]} -> undefined + end + end, + check_consistency(Payload, SampleSize, Fun) + after + file:close(FD) + end. + +%% ============================================================================= +%% Internal functions +%% ============================================================================= + +%% @doc Continue generating chunks +-spec generate_next_chunk(chunk_state()) -> stream(binary()). +generate_next_chunk(#chunk_state{offset = Offset, payload_size = Size}) when Offset >= Size -> + eos; +generate_next_chunk(State0 = #chunk_state{offset = Offset, chunk_size = ChunkSize}) -> + State = State0#chunk_state{offset = Offset + ChunkSize}, + Payload = generate_chunk( + State#chunk_state.seed, + Offset, + ChunkSize, + State#chunk_state.payload_size + ), + [Payload | fun() -> generate_next_chunk(State) end]. + +generate_chunk(Seed, Offset, ChunkSize, Size) -> + SeedHash = seed_hash(Seed), + To = min(Offset + ChunkSize, Size) - 1, + Payload = iolist_to_binary([ + generator_fun(I, SeedHash) + || I <- lists:seq(Offset div 16, To div 16) + ]), + ChunkNum = Offset div ChunkSize + 1, + ChunkCnt = ceil(Size / ChunkSize), + Chunk = + case Offset + ChunkSize of + NextOffset when NextOffset > Size -> + binary:part(Payload, 0, Size rem ChunkSize); + _ -> + Payload + end, + {Chunk, ChunkNum, ChunkCnt}. + +%% @doc First argument is a chunk number, the second one is a seed. +%% This implementation is hardly efficient, but it was chosen for +%% clarity reasons +-spec generator_fun(integer(), binary()) -> binary(). +generator_fun(N, Seed) -> + crypto:hash(md5, <>). + +%% @doc Hash any term +-spec seed_hash(term()) -> binary(). +seed_hash(Seed) -> + crypto:hash(md5, term_to_binary(Seed)). + +%% @private Get byte at offset `N' +-spec do_get_byte(integer(), binary()) -> byte(). +do_get_byte(N, Seed) -> + Chunk = generator_fun(N div ?hash_size, Seed), + binary:at(Chunk, N rem ?hash_size). From 1fedae8a1633b54f30ca9fdc16a1a5732870e238 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 2 Feb 2023 21:29:35 +0300 Subject: [PATCH 007/156] fix(ft-asm): ensure module follows statem behaviour --- apps/emqx_ft/src/emqx_ft_assembler.erl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index dfbb2bd3e..4fd8d6e75 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -45,7 +45,7 @@ %% start_link(Storage, Transfer, Callback) -> - gen_server:start_link(?MODULE, {Storage, Transfer, Callback}, []). + gen_statem:start_link(?MODULE, {Storage, Transfer, Callback}, []). %% @@ -64,7 +64,7 @@ init({Storage, Transfer, Callback}) -> }, {ok, list_local_fragments, St, ?internal([])}. -handle_event(list_local_fragments, internal, _, St = #st{assembly = Asm}) -> +handle_event(internal, _, list_local_fragments, St = #st{assembly = Asm}) -> % TODO: what we do with non-transients errors here (e.g. `eacces`)? {ok, Fragments} = emqx_ft_storage_fs:list(St#st.storage, St#st.transfer), NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(Asm, node(), Fragments)), @@ -79,7 +79,7 @@ handle_event(list_local_fragments, internal, _, St = #st{assembly = Asm}) -> % {error, _} = Reason -> % {stop, Reason} end; -handle_event({list_remote_fragments, Nodes}, internal, _, St) -> +handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> % TODO: portable "storage" ref Args = [St#st.storage, St#st.transfer], % TODO @@ -109,13 +109,13 @@ handle_event({list_remote_fragments, Nodes}, internal, _, St) -> {incomplete, _} = Status -> {stop, {error, Status}} end; -handle_event(start_assembling, internal, _, St = #st{assembly = Asm}) -> +handle_event(internal, _, start_assembling, St = #st{assembly = Asm}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), Coverage = emqx_ft_assembly:coverage(Asm), % TODO: errors {ok, Handle} = emqx_ft_storage_fs:open_file(St#st.storage, St#st.transfer, Filemeta), {next_state, {assemble, Coverage}, St#st{file = Handle}, ?internal([])}; -handle_event({assemble, [{Node, Segment} | Rest]}, internal, _, St = #st{}) -> +handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % TODO % Currently, race is possible between getting segment info from the remote node and % this node garbage collecting the segment itself. @@ -128,9 +128,9 @@ handle_event({assemble, [{Node, Segment} | Rest]}, internal, _, St = #st{}) -> % {error, _} -> % ... end; -handle_event({assemble, []}, internal, _, St = #st{}) -> +handle_event(internal, _, {assemble, []}, St = #st{}) -> {next_state, complete, St, ?internal([])}; -handle_event(complete, internal, _, St = #st{assembly = Asm, file = Handle, callback = Callback}) -> +handle_event(internal, _, complete, St = #st{assembly = Asm, file = Handle, callback = Callback}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), Result = emqx_ft_storage_fs:complete(St#st.storage, St#st.transfer, Filemeta, Handle), %% TODO: safe apply @@ -160,5 +160,5 @@ handle_event(complete, internal, _, St = #st{assembly = Asm, file = Handle, call %% -segsize(#{fragment := {segmentinfo, Info}}) -> +segsize(#{fragment := {segment, Info}}) -> maps:get(size, Info). From 14b2a1013bedba30755b8f4c963556d2c19b6b24 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 2 Feb 2023 21:30:30 +0300 Subject: [PATCH 008/156] fix(ft-asm): follow proper `segment` fragment type --- apps/emqx_ft/src/emqx_ft_assembly.erl | 80 +++++++++++++-------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index 854b420e0..0f2729f42 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -47,7 +47,7 @@ append(Asm, Node, Fragments) when is_list(Fragments) -> lists:foldl(fun(F, AsmIn) -> append(AsmIn, Node, F) end, Asm, Fragments); append(Asm, Node, Fragment = #{fragment := {filemeta, _}}) -> append_filemeta(Asm, Node, Fragment); -append(Asm, Node, Segment = #{fragment := {segmentinfo, _}}) -> +append(Asm, Node, Segment = #{fragment := {segment, _}}) -> append_segmentinfo(Asm, Node, Segment). update(Asm) -> @@ -105,7 +105,7 @@ append_filemeta(Asm, Node, Fragment = #{fragment := {filemeta, Meta}}) -> meta = orddict:store(Meta, {Node, Fragment}, Asm#asm.meta) }. -append_segmentinfo(Asm, Node, Fragment = #{fragment := {segmentinfo, Info}}) -> +append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) -> Offset = maps:get(offset, Info), Size = maps:get(size, Info), End = Offset + Size, @@ -176,7 +176,7 @@ locality(Node) when Node =:= node() -> locality(_RemoteNode) -> 1. -segsize(#{fragment := {segmentinfo, Info}}) -> +segsize(#{fragment := {segment, Info}}) -> maps:get(size, Info). -ifdef(TEST). @@ -195,8 +195,8 @@ incomplete_test() -> status( update( append(new(), node(), [ - segmentinfo(p1, 0, 42), - segmentinfo(p1, 42, 100) + segment(p1, 0, 42), + segment(p1, 42, 100) ]) ) ) @@ -204,14 +204,14 @@ incomplete_test() -> consistent_test() -> Asm1 = append(new(), n1, [filemeta(m1, "blarg")]), - Asm2 = append(Asm1, n2, [segmentinfo(s2, 0, 42)]), + Asm2 = append(Asm1, n2, [segment(s2, 0, 42)]), Asm3 = append(Asm2, n3, [filemeta(m3, "blarg")]), ?assertMatch({complete, _}, status(meta, Asm3)). inconsistent_test() -> - Asm1 = append(new(), node(), [segmentinfo(s1, 0, 42)]), + Asm1 = append(new(), node(), [segment(s1, 0, 42)]), Asm2 = append(Asm1, n1, [filemeta(m1, "blarg")]), - Asm3 = append(Asm2, n2, [segmentinfo(s2, 0, 42), filemeta(m1, "blorg")]), + Asm3 = append(Asm2, n2, [segment(s2, 0, 42), filemeta(m1, "blorg")]), Asm4 = append(Asm3, n3, [filemeta(m3, "blarg")]), ?assertMatch( {error, @@ -226,10 +226,10 @@ inconsistent_test() -> simple_coverage_test() -> Node = node(), Segs = [ - {node42, segmentinfo(n1, 20, 30)}, - {Node, segmentinfo(n2, 0, 10)}, - {Node, segmentinfo(n3, 50, 50)}, - {Node, segmentinfo(n4, 10, 10)} + {node42, segment(n1, 20, 30)}, + {Node, segment(n2, 0, 10)}, + {Node, segment(n3, 50, 50)}, + {Node, segment(n4, 10, 10)} ], Asm = append_many(new(), Segs), ?assertMatch( @@ -247,14 +247,14 @@ simple_coverage_test() -> redundant_coverage_test() -> Node = node(), Segs = [ - {Node, segmentinfo(n1, 0, 20)}, - {node1, segmentinfo(n2, 0, 10)}, - {Node, segmentinfo(n3, 20, 40)}, - {node2, segmentinfo(n4, 10, 10)}, - {node2, segmentinfo(n5, 50, 20)}, - {node3, segmentinfo(n6, 20, 20)}, - {Node, segmentinfo(n7, 50, 10)}, - {node1, segmentinfo(n8, 40, 10)} + {Node, segment(n1, 0, 20)}, + {node1, segment(n2, 0, 10)}, + {Node, segment(n3, 20, 40)}, + {node2, segment(n4, 10, 10)}, + {node2, segment(n5, 50, 20)}, + {node3, segment(n6, 20, 20)}, + {Node, segment(n7, 50, 10)}, + {node1, segment(n8, 40, 10)} ], Asm = append_many(new(), Segs), ?assertMatch( @@ -272,12 +272,12 @@ redundant_coverage_test() -> redundant_coverage_prefer_local_test() -> Node = node(), Segs = [ - {node1, segmentinfo(n1, 0, 20)}, - {Node, segmentinfo(n2, 0, 10)}, - {Node, segmentinfo(n3, 10, 10)}, - {node2, segmentinfo(n4, 20, 20)}, - {Node, segmentinfo(n5, 30, 10)}, - {Node, segmentinfo(n6, 20, 10)} + {node1, segment(n1, 0, 20)}, + {Node, segment(n2, 0, 10)}, + {Node, segment(n3, 10, 10)}, + {node2, segment(n4, 20, 20)}, + {Node, segment(n5, 30, 10)}, + {Node, segment(n6, 20, 10)} ], Asm = append_many(new(), Segs), ?assertMatch( @@ -295,11 +295,11 @@ redundant_coverage_prefer_local_test() -> missing_coverage_test() -> Node = node(), Segs = [ - {Node, segmentinfo(n1, 0, 10)}, - {node1, segmentinfo(n3, 10, 20)}, - {Node, segmentinfo(n2, 0, 20)}, - {node2, segmentinfo(n4, 50, 50)}, - {Node, segmentinfo(n5, 40, 60)} + {Node, segment(n1, 0, 10)}, + {node1, segment(n3, 10, 20)}, + {Node, segment(n2, 0, 20)}, + {node2, segment(n4, 50, 50)}, + {Node, segment(n5, 40, 60)} ], Asm = append_many(new(), Segs), ?assertEqual( @@ -311,8 +311,8 @@ missing_coverage_test() -> missing_end_coverage_test() -> Node = node(), Segs = [ - {Node, segmentinfo(n1, 0, 15)}, - {node1, segmentinfo(n3, 10, 10)} + {Node, segment(n1, 0, 15)}, + {node1, segment(n3, 10, 10)} ], Asm = append_many(new(), Segs), ?assertEqual( @@ -322,11 +322,11 @@ missing_end_coverage_test() -> missing_coverage_with_redudancy_test() -> Segs = [ - {node(), segmentinfo(n1, 0, 10)}, - {node(), segmentinfo(n2, 0, 20)}, - {node42, segmentinfo(n3, 10, 20)}, - {node43, segmentinfo(n4, 10, 50)}, - {node(), segmentinfo(n5, 40, 60)} + {node(), segment(n1, 0, 10)}, + {node(), segment(n2, 0, 20)}, + {node42, segment(n3, 10, 20)}, + {node43, segment(n4, 10, 50)}, + {node(), segment(n5, 40, 60)} ], Asm = append_many(new(), Segs), ?assertEqual( @@ -351,11 +351,11 @@ filemeta(Path, Name) -> }} }. -segmentinfo(Path, Offset, Size) -> +segment(Path, Offset, Size) -> #{ path => Path, fragment => - {segmentinfo, #{ + {segment, #{ offset => Offset, size => Size }} From 97b831a160b6a622911ee7e199904c2a8bc8f741 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 2 Feb 2023 21:31:04 +0300 Subject: [PATCH 009/156] fix(ft-fs): add missing `read_segment/5` + fix atomic write --- apps/emqx_ft/src/emqx_ft_assembler_sup.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 59 +++++++++++++++++----- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl index e837d96f1..b60949a5e 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl @@ -36,7 +36,7 @@ start_child(Storage, Transfer, Callback) -> init(_) -> SupFlags = #{ strategy => one_for_one, - intensity => 100, + intensity => 10, period => 1000 }, {ok, {SupFlags, []}}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 15efa142b..cce7cc19e 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -24,6 +24,7 @@ -export([store_filemeta/3]). -export([store_segment/3]). -export([list/2]). +-export([read_segment/5]). -export([assemble/3]). -export([open_file/3]). @@ -154,6 +155,28 @@ list(Storage, Transfer) -> Error end. +-spec read_segment( + storage(), transfer(), filefrag(segmentinfo()), offset(), _Size :: non_neg_integer() +) -> + {ok, _Content :: iodata()} | {error, _TODO}. +read_segment(_Storage, _Transfer, Segment, Offset, Size) -> + Filepath = maps:get(path, Segment), + case file:open(Filepath, [raw, read]) of + {ok, IoDevice} -> + Read = file:pread(IoDevice, Offset, Size), + ok = file:close(IoDevice), + case Read of + {ok, Content} -> + {ok, Content}; + eof -> + {error, eof}; + {error, Reason} -> + {error, Reason} + end; + {error, Reason} -> + {error, Reason} + end. + -spec assemble(storage(), transfer(), fun((ok | {error, term()}) -> any())) -> % {ok, _Assembler :: pid()} | {error, incomplete} | {error, badrpc} | {error, _TODO}. {ok, _Assembler :: pid()} | {error, _TODO}. @@ -359,18 +382,18 @@ safe_decode(Content, DecodeFun) -> end. write_file_atomic(Filepath, Content) when is_binary(Content) -> + TempFilepath = mk_temp_filepath(Filepath), Result = emqx_misc:pipeline( [ fun filelib:ensure_dir/1, - fun mk_temp_filepath/1, fun write_contents/2, - fun mv_temp_file/1 + fun(FP) -> mv_temp_file(Filepath, FP) end ], - Filepath, + TempFilepath, Content ), case Result of - {ok, {Filepath, TempFilepath}, _} -> + {ok, _, _} -> _ = file:delete(TempFilepath), ok; {error, Reason, _} -> @@ -381,13 +404,20 @@ mk_temp_filepath(Filepath) -> Dirname = filename:dirname(Filepath), Filename = filename:basename(Filepath), Unique = erlang:unique_integer([positive]), - TempFilepath = filename:join(Dirname, ?TEMP ++ integer_to_list(Unique) ++ "." ++ Filename), - {Filepath, TempFilepath}. + filename:join(Dirname, mk_filename([?TEMP, Unique, ".", Filename])). -write_contents({_Filepath, TempFilepath}, Content) -> - file:write_file(TempFilepath, Content). +mk_filename(Comps) -> + lists:append(lists:map(fun mk_filename_component/1, Comps)). -mv_temp_file({Filepath, TempFilepath}) -> +mk_filename_component(I) when is_integer(I) -> integer_to_list(I); +mk_filename_component(A) when is_atom(A) -> atom_to_list(A); +mk_filename_component(B) when is_binary(B) -> unicode:characters_to_list(B); +mk_filename_component(S) when is_list(S) -> S. + +write_contents(Filepath, Content) -> + file:write_file(Filepath, Content). + +mv_temp_file(Filepath, TempFilepath) -> file:rename(TempFilepath, Filepath). touch_file(Filepath) -> @@ -398,21 +428,22 @@ filtermap_files(Fun, Dirname, Filenames) -> lists:filtermap(fun(Filename) -> Fun(Dirname, Filename) end, Filenames). mk_filefrag(Dirname, Filename = ?MANIFEST) -> - mk_filefrag(Dirname, Filename, fun read_filemeta/2); + mk_filefrag(Dirname, Filename, filemeta, fun read_filemeta/2); mk_filefrag(Dirname, Filename = ?SEGMENT ++ _) -> - mk_filefrag(Dirname, Filename, fun read_segmentinfo/2); + mk_filefrag(Dirname, Filename, segment, fun read_segmentinfo/2); mk_filefrag(_Dirname, _) -> false. -mk_filefrag(Dirname, Filename, Fun) -> +mk_filefrag(Dirname, Filename, Tag, Fun) -> Filepath = filename:join(Dirname, Filename), - Fileinfo = file:read_file_info(Filepath), + % TODO error handling? + {ok, Fileinfo} = file:read_file_info(Filepath), case Fun(Filename, Filepath) of {ok, Frag} -> {true, #{ path => Filepath, timestamp => Fileinfo#file_info.mtime, - fragment => Frag + fragment => {Tag, Frag} }}; {error, _Reason} -> false From 72e3eee6c93fa212eebe9ba62ab567b3dfd0421d Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 2 Feb 2023 20:23:12 +0200 Subject: [PATCH 010/156] feat(ft): add config & backend behaviour --- apps/emqx_conf/src/emqx_conf_schema.erl | 3 +- apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 14 +++ apps/emqx_ft/src/emqx_ft.erl | 126 +++++++++++++-------- apps/emqx_ft/src/emqx_ft_app.erl | 2 + apps/emqx_ft/src/emqx_ft_conf.erl | 65 +++++++++++ apps/emqx_ft/src/emqx_ft_responder.erl | 1 + apps/emqx_ft/src/emqx_ft_schema.erl | 49 ++++++++ apps/emqx_ft/src/emqx_ft_storage.erl | 75 ++++++++++++ apps/emqx_ft/src/emqx_ft_storage_dummy.erl | 35 ++++++ apps/emqx_ft/src/emqx_ft_storage_fs.erl | 71 +++--------- 10 files changed, 342 insertions(+), 99 deletions(-) create mode 100644 apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf create mode 100644 apps/emqx_ft/src/emqx_ft_conf.erl create mode 100644 apps/emqx_ft/src/emqx_ft_schema.erl create mode 100644 apps/emqx_ft/src/emqx_ft_storage.erl create mode 100644 apps/emqx_ft/src/emqx_ft_storage_dummy.erl diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 58bcf9700..1a0f8bbb1 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -63,7 +63,8 @@ emqx_psk_schema, emqx_limiter_schema, emqx_slow_subs_schema, - emqx_mgmt_api_key_schema + emqx_mgmt_api_key_schema, + emqx_ft_schema ]). %% root config should not have a namespace diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf new file mode 100644 index 000000000..85fb81cfe --- /dev/null +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -0,0 +1,14 @@ +emqx_ft_schema { + + local { + desc { + en: "Use local file system to store uploaded files and temporary data." + zh: "使用本地文件系统来存储上传的文件和临时数据。" + } + label: { + en: "Local Storage" + zh: "本地存储" + } + } + +} diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 014c74ac3..a286a6186 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -41,9 +41,13 @@ -export([on_assemble_timeout/1]). --export_type([clientid/0]). --export_type([transfer/0]). --export_type([offset/0]). +-export_type([ + clientid/0, + transfer/0, + offset/0, + filemeta/0, + segment/0 +]). %% Number of bytes -type bytes() :: non_neg_integer(). @@ -55,6 +59,26 @@ -type transfer() :: {clientid(), fileid()}. -type offset() :: bytes(). +-type filemeta() :: #{ + %% Display name + name := string(), + %% Size in bytes, as advertised by the client. + %% Client is free to specify here whatever it wants, which means we can end + %% up with a file of different size after assembly. It's not clear from + %% specification what that means (e.g. what are clients' expectations), we + %% currently do not condider that an error (or, specifically, a signal that + %% the resulting file is corrupted during transmission). + size => _Bytes :: non_neg_integer(), + checksum => {sha256, <<_:256>>}, + expire_at := emqx_datetime:epoch_second(), + %% TTL of individual segments + %% Somewhat confusing that we won't know it on the nodes where the filemeta + %% is missing. + segments_ttl => _Seconds :: pos_integer() +}. + +-type segment() :: {offset(), _Content :: binary()}. + -type ft_data() :: #{ nodes := list(node()) }. @@ -135,23 +159,9 @@ on_message_puback(PacketId, #message{topic = Topic} = Msg, _PubRes, _RC) -> end. %%-------------------------------------------------------------------- -%% Private funs +%% Handlers for transfer messages %%-------------------------------------------------------------------- -get_ft_data(ChanPid) -> - case ets:lookup(?FT_TAB, ChanPid) of - [#emqx_ft{ft_data = FTData}] -> {ok, FTData}; - [] -> none - end. - -delete_ft_data(ChanPid) -> - true = ets:delete(?FT_TAB, ChanPid), - ok. - -put_ft_data(ChanPid, FTData) -> - true = ets:insert(?FT_TAB, #emqx_ft{chan_pid = ChanPid, ft_data = FTData}), - ok. - on_file_command(PacketId, Msg, FileCommand) -> case string:split(FileCommand, <<"/">>, all) of [FileId, <<"init">>] -> @@ -176,12 +186,16 @@ on_init(Msg, FileId) -> mqtt_msg => Msg, file_id => FileId }), - % Payload = Msg#message.payload, + Payload = Msg#message.payload, % %% Add validations here - % Meta = emqx_json:decode(Payload, [return_maps]), - % ok = emqx_ft_storage_fs:store_filemeta(storage(), transfer(Msg, FileId), Meta), - % ?RC_SUCCESS. - ?RC_UNSPECIFIED_ERROR. + Meta = emqx_json:decode(Payload, [return_maps]), + case emqx_ft_storage:store_filemeta(transfer(Msg, FileId), Meta) of + {ok, Ctx} -> + ok = put_context(Ctx), + ?RC_SUCCESS; + {error, _Reason} -> + ?RC_UNSPECIFIED_ERROR + end. on_abort(_Msg, _FileId) -> %% TODO @@ -195,16 +209,17 @@ on_segment(Msg, FileId, Offset, Checksum) -> offset => Offset, checksum => Checksum }), - % %% TODO: handle checksum - % Payload = Msg#message.payload, - % %% Add offset/checksum validations - % ok = emqx_ft_storage_fs:store_segment( - % storage(), - % transfer(Msg, FileId), - % {binary_to_integer(Offset), Payload} - % ), - % ?RC_SUCCESS. - ?RC_UNSPECIFIED_ERROR. + %% TODO: handle checksum + Payload = Msg#message.payload, + Segment = {binary_to_integer(Offset), Payload}, + %% Add offset/checksum validations + case emqx_ft_storage:store_segment(get_context(), transfer(Msg, FileId), Segment) of + {ok, Ctx} -> + ok = put_context(Ctx), + ?RC_SUCCESS; + {error, _Reason} -> + ?RC_UNSPECIFIED_ERROR + end. on_fin(PacketId, Msg, FileId, Checksum) -> ?SLOG(info, #{ @@ -227,7 +242,7 @@ on_fin(PacketId, Msg, FileId, Checksum) -> Callback = callback(FinPacketKey, FileId), case assemble(transfer(Msg, FileId), Callback) of %% Assembling started, packet will be acked by the callback or the responder - ok -> + {ok, _} -> undefined; %% Assembling failed, unregister the packet key {error, _} -> @@ -250,16 +265,12 @@ on_fin(PacketId, Msg, FileId, Checksum) -> undefined end. -assemble(_Transfer, _Callback) -> - % spawn(fun() -> Callback({error, not_implemented}) end), - ok. - -% assemble(Transfer, Callback) -> -% emqx_ft_storage_fs:assemble( -% storage(), -% Transfer, -% Callback -% ). +assemble(Transfer, Callback) -> + emqx_ft_storage:assemble( + get_context(), + Transfer, + Callback + ). callback({ChanPid, PacketId} = Key, _FileId) -> fun(Result) -> @@ -281,9 +292,34 @@ transfer(Msg, FileId) -> {ClientId, FileId}. %% TODO: configure + storage() -> - filename:join(emqx:data_dir(), "file_transfer"). + emqx_config:get([file_transfer, storage]). on_assemble_timeout({ChanPid, PacketId}) -> ?SLOG(warning, #{msg => "on_assemble_timeout", packet_id => PacketId}), erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}). + +%%-------------------------------------------------------------------- +%% Context management +%%-------------------------------------------------------------------- + +get_context() -> + get_ft_data(self()). + +put_context(Context) -> + put_ft_data(self(), Context). + +get_ft_data(ChanPid) -> + case ets:lookup(?FT_TAB, ChanPid) of + [#emqx_ft{ft_data = FTData}] -> {ok, FTData}; + [] -> none + end. + +delete_ft_data(ChanPid) -> + true = ets:delete(?FT_TAB, ChanPid), + ok. + +put_ft_data(ChanPid, FTData) -> + true = ets:insert(?FT_TAB, #emqx_ft{chan_pid = ChanPid, ft_data = FTData}), + ok. diff --git a/apps/emqx_ft/src/emqx_ft_app.erl b/apps/emqx_ft/src/emqx_ft_app.erl index 4778da1a1..9b1513b46 100644 --- a/apps/emqx_ft/src/emqx_ft_app.erl +++ b/apps/emqx_ft/src/emqx_ft_app.erl @@ -23,8 +23,10 @@ start(_StartType, _StartArgs) -> {ok, Sup} = emqx_ft_sup:start_link(), ok = emqx_ft:hook(), + ok = emqx_ft_conf:load(), {ok, Sup}. stop(_State) -> + ok = emqx_ft_conf:unload(), ok = emqx_ft:unhook(), ok. diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl new file mode 100644 index 000000000..b88fd2532 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -0,0 +1,65 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc File Transfer configuration management module + +-module(emqx_ft_conf). + +-behaviour(emqx_config_handler). + +%% Load/Unload +-export([ + load/0, + unload/0 +]). + +%% callbacks for emqx_config_handler +-export([ + pre_config_update/3, + post_config_update/5 +]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec load() -> ok. +load() -> + emqx_conf:add_handler([file_transfer], ?MODULE). + +-spec unload() -> ok. +unload() -> + emqx_conf:remove_handler([file_transfer]). + +%%-------------------------------------------------------------------- +%% emqx_config_handler callbacks +%%-------------------------------------------------------------------- + +-spec pre_config_update(list(atom()), emqx_config:update_request(), emqx_config:raw_config()) -> + {ok, emqx_config:update_request()} | {error, term()}. +pre_config_update(_, _Req, Config) -> + {ok, Config}. + +-spec post_config_update( + list(atom()), + emqx_config:update_request(), + emqx_config:config(), + emqx_config:config(), + emqx_config:app_envs() +) -> + ok | {ok, Result :: any()} | {error, Reason :: term()}. +post_config_update(_, _Req, _NewConfig, _OldConfig, _AppEnvs) -> + ok. diff --git a/apps/emqx_ft/src/emqx_ft_responder.erl b/apps/emqx_ft/src/emqx_ft_responder.erl index dcb45d5d3..f58569a27 100644 --- a/apps/emqx_ft/src/emqx_ft_responder.erl +++ b/apps/emqx_ft/src/emqx_ft_responder.erl @@ -79,6 +79,7 @@ handle_call({register, Key, DefaultAction, Timeout}, _From, State) -> {reply, {error, already_registered}, State} end; handle_call({unregister, Key}, _From, State) -> + ?SLOG(warning, #{msg => "unregister", key => Key}), case ets:lookup(?TAB, Key) of [] -> {reply, {error, not_found}, State}; diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl new file mode 100644 index 000000000..f40d2f40e --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -0,0 +1,49 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_schema). + +-behaviour(hocon_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-export([namespace/0, roots/0, fields/1, tags/0]). + +namespace() -> file_transfer. + +tags() -> + [<<"File Transfer">>]. + +roots() -> [file_transfer]. + +fields(file_transfer) -> + [ + {storage, #{ + type => hoconsc:union([ + hoconsc:ref(?MODULE, local_storage) + ]) + }} + ]; +fields(local_storage) -> + [ + {type, #{ + type => local, + default => local, + required => false, + desc => ?DESC("local") + }} + ]. diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl new file mode 100644 index 000000000..5e945965d --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -0,0 +1,75 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage). + +-export( + [ + store_filemeta/2, + store_segment/3, + assemble/3 + ] +). + +-type ctx() :: term(). +-type storage() :: emqx_config:config(). + +-export_type([assemble_callback/0]). + +-type assemble_callback() :: fun((ok | {error, term()}) -> any()). + +%%-------------------------------------------------------------------- +%% behaviour +%%-------------------------------------------------------------------- + +-callback store_filemeta(storage(), emqx_ft:transfer(), emqx_ft:filemeta()) -> + {ok, ctx()} | {error, term()}. +-callback store_segment(storage(), ctx(), emqx_ft:transfer(), emqx_ft:segment()) -> + {ok, ctx()} | {error, term()}. +-callback assemble(storage(), ctx(), emqx_ft:transfer(), assemble_callback()) -> + {ok, pid()} | {error, term()}. + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec store_filemeta(emqx_ft:transfer(), emqx_ft:filemeta()) -> + {ok, ctx()} | {error, term()}. +store_filemeta(Transfer, FileMeta) -> + Mod = mod(), + Mod:store_filemeta(storage(), Transfer, FileMeta). + +-spec store_segment(ctx(), emqx_ft:transfer(), emqx_ft:segment()) -> + {ok, ctx()} | {error, term()}. +store_segment(Ctx, Transfer, Segment) -> + Mod = mod(), + Mod:store_segment(storage(), Ctx, Transfer, Segment). + +-spec assemble(ctx(), emqx_ft:transfer(), assemble_callback()) -> + {ok, pid()} | {error, term()}. +assemble(Ctx, Transfer, Callback) -> + Mod = mod(), + Mod:assemble(storage(), Ctx, Transfer, Callback). + +mod() -> + case storage() of + #{type := local} -> + % emqx_ft_storage_fs + emqx_ft_storage_dummy + end. + +storage() -> + emqx_config:get([file_transfer, storage]). diff --git a/apps/emqx_ft/src/emqx_ft_storage_dummy.erl b/apps/emqx_ft/src/emqx_ft_storage_dummy.erl new file mode 100644 index 000000000..1ab7f558e --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_dummy.erl @@ -0,0 +1,35 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_dummy). + +-behaviour(emqx_ft_storage). + +-export([ + store_filemeta/3, + store_segment/4, + assemble/4 +]). + +store_filemeta(_Storage, _Transfer, _Meta) -> + {ok, #{}}. + +store_segment(_Storage, Ctx, _Transfer, _Segment) -> + {ok, Ctx}. + +assemble(_Storage, _Ctx, _Transfer, Callback) -> + Pid = spawn(fun() -> Callback({error, not_implemented}) end), + {ok, Pid}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index cce7cc19e..bbd61eba9 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -19,10 +19,10 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -% -compile(export_all). +-behaviour(emqx_ft_storage). -export([store_filemeta/3]). --export([store_segment/3]). +-export([store_segment/4]). -export([list/2]). -export([read_segment/5]). -export([assemble/3]). @@ -32,45 +32,10 @@ -export([write/2]). -export([discard/1]). -% -behaviour(gen_server). -% -export([init/1]). -% -export([handle_call/3]). -% -export([handle_cast/2]). - --type json_value() :: - null - | boolean() - | binary() - | number() - | [json_value()] - | #{binary() => json_value()}. - --reflect_type([json_value/0]). - -type transfer() :: emqx_ft:transfer(). -type offset() :: emqx_ft:offset(). -%% TODO: move to `emqx_ft` interface module -% -type sha256_hex() :: <<_:512>>. - --type filemeta() :: #{ - %% Display name - name := string(), - %% Size in bytes, as advertised by the client. - %% Client is free to specify here whatever it wants, which means we can end - %% up with a file of different size after assembly. It's not clear from - %% specification what that means (e.g. what are clients' expectations), we - %% currently do not condider that an error (or, specifically, a signal that - %% the resulting file is corrupted during transmission). - size => _Bytes :: non_neg_integer(), - checksum => {sha256, <<_:256>>}, - expire_at := emqx_datetime:epoch_second(), - %% TTL of individual segments - %% Somewhat confusing that we won't know it on the nodes where the filemeta - %% is missing. - segments_ttl => _Seconds :: pos_integer(), - user_data => json_value() -}. +-type filemeta() :: emqx_ft:filemeta(). -type segment() :: {offset(), _Content :: binary()}. @@ -113,13 +78,14 @@ %% Atomic operation. -spec store_filemeta(storage(), transfer(), filemeta()) -> % Quota? Some lower level errors? - ok | {error, conflict} | {error, _TODO}. + {ok, emqx_ft_storage:ctx()} | {error, conflict} | {error, _TODO}. store_filemeta(Storage, Transfer, Meta) -> Filepath = mk_filepath(Storage, Transfer, ?MANIFEST), case read_file(Filepath, fun decode_filemeta/1) of {ok, Meta} -> _ = touch_file(Filepath), - ok; + %% No context is needed for this implementation. + {ok, #{}}; {ok, _Conflict} -> % TODO % We won't see conflicts in case of concurrent `store_filemeta` @@ -132,13 +98,18 @@ store_filemeta(Storage, Transfer, Meta) -> %% Store a segment in the backing filesystem. %% Atomic operation. --spec store_segment(storage(), transfer(), segment()) -> +-spec store_segment(storage(), emqx_ft_storage:ctx(), transfer(), segment()) -> % Where is the checksum gets verified? Upper level probably. % Quota? Some lower level errors? ok | {error, _TODO}. -store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> +store_segment(Storage, Ctx, Transfer, Segment = {_Offset, Content}) -> Filepath = mk_filepath(Storage, Transfer, mk_segment_filename(Segment)), - write_file_atomic(Filepath, Content). + case write_file_atomic(Filepath, Content) of + ok -> + {ok, Ctx}; + {error, _} = Error -> + Error + end. -spec list(storage(), transfer()) -> % Some lower level errors? {error, notfound}? @@ -178,13 +149,10 @@ read_segment(_Storage, _Transfer, Segment, Offset, Size) -> end. -spec assemble(storage(), transfer(), fun((ok | {error, term()}) -> any())) -> - % {ok, _Assembler :: pid()} | {error, incomplete} | {error, badrpc} | {error, _TODO}. {ok, _Assembler :: pid()} | {error, _TODO}. -assemble(Storage, Transfer, Callback) -> +assemble(Storage, _Ctx, Transfer, Callback) -> emqx_ft_assembler_sup:start_child(Storage, Transfer, Callback). -%% - -type handle() :: {file:name(), io:device(), crypto:hash_state()}. -spec open_file(storage(), transfer(), filemeta()) -> @@ -268,8 +236,7 @@ schema() -> {size, hoconsc:mk(non_neg_integer())}, {expire_at, hoconsc:mk(non_neg_integer())}, {checksum, hoconsc:mk({atom(), binary()}, #{converter => converter(checksum)})}, - {segments_ttl, hoconsc:mk(pos_integer())}, - {user_data, hoconsc:mk(json_value())} + {segments_ttl, hoconsc:mk(pos_integer())} ] }. @@ -354,10 +321,8 @@ mk_filedir(Storage, {ClientId, FileId}) -> mk_filepath(Storage, Transfer, Filename) -> filename:join(mk_filedir(Storage, Transfer), Filename). -get_storage_root(Storage) -> - Storage. - -%% +get_storage_root(_Storage) -> + filename:join(emqx:data_dir(), "file_transfer"). -include_lib("kernel/include/file.hrl"). From b4a42a447c603f5361c9969cd0aa44c98ff98e4c Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 2 Feb 2023 23:52:42 +0200 Subject: [PATCH 011/156] feat(ft): removed replicated data --- apps/emqx/src/emqx_cm.erl | 23 ++--- apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 10 +- apps/emqx_ft/include/emqx_ft.hrl | 2 - apps/emqx_ft/src/emqx_ft.erl | 92 +------------------ apps/emqx_ft/src/emqx_ft_storage.erl | 35 ++++--- apps/emqx_ft/src/emqx_ft_storage_dummy.erl | 12 +-- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 23 ++--- apps/emqx_ft/src/emqx_ft_sup.erl | 12 --- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 9 +- 9 files changed, 56 insertions(+), 162 deletions(-) diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index 724139142..766acd4b8 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -172,7 +172,6 @@ register_channel(ClientId, ChanPid, #{conn_mod := ConnMod}) when is_pid(ChanPid) true = ets:insert(?CHAN_CONN_TAB, {Chan, ConnMod}), ok = emqx_cm_registry:register_channel(Chan), mark_channel_connected(ChanPid), - ok = emqx_hooks:run('channel.registered', [ConnMod, ChanPid]), cast({registered, Chan}). %% @doc Unregister a channel. @@ -282,7 +281,7 @@ open_session(true, ClientInfo = #{clientid := ClientId}, ConnInfo) -> {ok, #{session => Session1, present => false}} end, emqx_cm_locker:trans(ClientId, CleanStart); -open_session(false, ClientInfo = #{clientid := ClientId}, #{conn_mod := NewConnMod} = ConnInfo) -> +open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> Self = self(), ResumeStart = fun(_) -> CreateSess = @@ -309,11 +308,10 @@ open_session(false, ClientInfo = #{clientid := ClientId}, #{conn_mod := NewConnM {living, ConnMod, ChanPid, Session} -> ok = emqx_session:resume(ClientInfo, Session), case wrap_rpc(emqx_cm_proto_v2:takeover_finish(ConnMod, ChanPid)) of - {ok, Pendings, TakoverData} -> + {ok, Pendings} -> Session1 = emqx_persistent_session:persist( ClientInfo, ConnInfo, Session ), - ok = emqx_hooks:run('channel.takenover', [NewConnMod, Self, TakoverData]), {ok, #{ session => clean_session(Session1), present => true, @@ -399,18 +397,11 @@ takeover_session(ClientId) -> end. takeover_finish(ConnMod, ChanPid) -> - TakoverData = emqx_hooks:run_fold('channel.takeover', [ConnMod, ChanPid], #{}), - case - %% node-local call - request_stepdown( - {takeover, 'end'}, - ConnMod, - ChanPid - ) - of - {ok, Pendings} -> {ok, Pendings, TakoverData}; - {error, _} = Error -> Error - end. + request_stepdown( + {takeover, 'end'}, + ConnMod, + ChanPid + ). takeover_session(ClientId, Pid) -> try diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf index 85fb81cfe..481ad8154 100644 --- a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -2,13 +2,13 @@ emqx_ft_schema { local { desc { - en: "Use local file system to store uploaded files and temporary data." - zh: "使用本地文件系统来存储上传的文件和临时数据。" + en: "Use local file system to store uploaded files and temporary data." + zh: "使用本地文件系统来存储上传的文件和临时数据。" } label: { - en: "Local Storage" - zh: "本地存储" - } + en: "Local Storage" + zh: "本地存储" + } } } diff --git a/apps/emqx_ft/include/emqx_ft.hrl b/apps/emqx_ft/include/emqx_ft.hrl index 2cbd24fb4..e46d79490 100644 --- a/apps/emqx_ft/include/emqx_ft.hrl +++ b/apps/emqx_ft/include/emqx_ft.hrl @@ -13,5 +13,3 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- - --define(FT_TAB, emqx_ft). diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index a286a6186..be2f80831 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -16,29 +16,21 @@ -module(emqx_ft). --include("emqx_ft.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). -export([ - create_tab/0, hook/0, unhook/0 ]). -export([ - on_channel_unregistered/1, - on_channel_takeover/3, - on_channel_takenover/3, on_message_publish/1, on_message_puback/4 ]). -%% For Debug --export([transfer/2, storage/0]). - -export([on_assemble_timeout/1]). -export_type([ @@ -79,43 +71,17 @@ -type segment() :: {offset(), _Content :: binary()}. --type ft_data() :: #{ - nodes := list(node()) -}. - --record(emqx_ft, { - chan_pid :: pid(), - ft_data :: ft_data() -}). - -define(ASSEMBLE_TIMEOUT, 5000). %%-------------------------------------------------------------------- %% API for app %%-------------------------------------------------------------------- -create_tab() -> - _Tab = ets:new(?FT_TAB, [ - set, - public, - named_table, - {keypos, #emqx_ft.chan_pid} - ]), - ok. - hook() -> - ok = emqx_hooks:put('channel.unregistered', {?MODULE, on_channel_unregistered, []}, ?HP_LOWEST), - ok = emqx_hooks:put('channel.takeover', {?MODULE, on_channel_takeover, []}, ?HP_LOWEST), - ok = emqx_hooks:put('channel.takenover', {?MODULE, on_channel_takenover, []}, ?HP_LOWEST), - ok = emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_LOWEST), ok = emqx_hooks:put('message.puback', {?MODULE, on_message_puback, []}, ?HP_LOWEST). unhook() -> - ok = emqx_hooks:del('channel.unregistered', {?MODULE, on_channel_unregistered}), - ok = emqx_hooks:del('channel.takeover', {?MODULE, on_channel_takeover}), - ok = emqx_hooks:del('channel.takenover', {?MODULE, on_channel_takenover}), - ok = emqx_hooks:del('message.publish', {?MODULE, on_message_publish}), ok = emqx_hooks:del('message.puback', {?MODULE, on_message_puback}). @@ -123,22 +89,6 @@ unhook() -> %% Hooks %%-------------------------------------------------------------------- -on_channel_unregistered(ChanPid) -> - ok = delete_ft_data(ChanPid). - -on_channel_takeover(_ConnMod, ChanPid, TakeoverData) -> - case get_ft_data(ChanPid) of - {ok, FTData} -> - {ok, TakeoverData#{ft_data => FTData}}; - none -> - ok - end. - -on_channel_takenover(_ConnMod, ChanPid, #{ft_data := FTData}) -> - ok = put_ft_data(ChanPid, FTData); -on_channel_takenover(_ConnMod, _ChanPid, _) -> - ok. - on_message_publish( Msg = #message{ id = _Id, @@ -190,8 +140,7 @@ on_init(Msg, FileId) -> % %% Add validations here Meta = emqx_json:decode(Payload, [return_maps]), case emqx_ft_storage:store_filemeta(transfer(Msg, FileId), Meta) of - {ok, Ctx} -> - ok = put_context(Ctx), + ok -> ?RC_SUCCESS; {error, _Reason} -> ?RC_UNSPECIFIED_ERROR @@ -213,9 +162,8 @@ on_segment(Msg, FileId, Offset, Checksum) -> Payload = Msg#message.payload, Segment = {binary_to_integer(Offset), Payload}, %% Add offset/checksum validations - case emqx_ft_storage:store_segment(get_context(), transfer(Msg, FileId), Segment) of - {ok, Ctx} -> - ok = put_context(Ctx), + case emqx_ft_storage:store_segment(transfer(Msg, FileId), Segment) of + ok -> ?RC_SUCCESS; {error, _Reason} -> ?RC_UNSPECIFIED_ERROR @@ -267,7 +215,6 @@ on_fin(PacketId, Msg, FileId, Checksum) -> assemble(Transfer, Callback) -> emqx_ft_storage:assemble( - get_context(), Transfer, Callback ). @@ -277,12 +224,12 @@ callback({ChanPid, PacketId} = Key, _FileId) -> case emqx_ft_responder:unregister(Key) of ok -> case Result of - {ok, _} -> + ok -> erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS}); {error, _} -> erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}) end; - {error, not_registered} -> + {error, not_found} -> ok end end. @@ -291,35 +238,6 @@ transfer(Msg, FileId) -> ClientId = Msg#message.from, {ClientId, FileId}. -%% TODO: configure - -storage() -> - emqx_config:get([file_transfer, storage]). - on_assemble_timeout({ChanPid, PacketId}) -> ?SLOG(warning, #{msg => "on_assemble_timeout", packet_id => PacketId}), erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}). - -%%-------------------------------------------------------------------- -%% Context management -%%-------------------------------------------------------------------- - -get_context() -> - get_ft_data(self()). - -put_context(Context) -> - put_ft_data(self(), Context). - -get_ft_data(ChanPid) -> - case ets:lookup(?FT_TAB, ChanPid) of - [#emqx_ft{ft_data = FTData}] -> {ok, FTData}; - [] -> none - end. - -delete_ft_data(ChanPid) -> - true = ets:delete(?FT_TAB, ChanPid), - ok. - -put_ft_data(ChanPid, FTData) -> - true = ets:insert(?FT_TAB, #emqx_ft{chan_pid = ChanPid, ft_data = FTData}), - ok. diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 5e945965d..8729a2ad4 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -19,12 +19,11 @@ -export( [ store_filemeta/2, - store_segment/3, - assemble/3 + store_segment/2, + assemble/2 ] ). --type ctx() :: term(). -type storage() :: emqx_config:config(). -export_type([assemble_callback/0]). @@ -32,14 +31,14 @@ -type assemble_callback() :: fun((ok | {error, term()}) -> any()). %%-------------------------------------------------------------------- -%% behaviour +%% Behaviour %%-------------------------------------------------------------------- -callback store_filemeta(storage(), emqx_ft:transfer(), emqx_ft:filemeta()) -> - {ok, ctx()} | {error, term()}. --callback store_segment(storage(), ctx(), emqx_ft:transfer(), emqx_ft:segment()) -> - {ok, ctx()} | {error, term()}. --callback assemble(storage(), ctx(), emqx_ft:transfer(), assemble_callback()) -> + ok | {error, term()}. +-callback store_segment(storage(), emqx_ft:transfer(), emqx_ft:segment()) -> + ok | {error, term()}. +-callback assemble(storage(), emqx_ft:transfer(), assemble_callback()) -> {ok, pid()} | {error, term()}. %%-------------------------------------------------------------------- @@ -47,28 +46,28 @@ %%-------------------------------------------------------------------- -spec store_filemeta(emqx_ft:transfer(), emqx_ft:filemeta()) -> - {ok, ctx()} | {error, term()}. + ok | {error, term()}. store_filemeta(Transfer, FileMeta) -> Mod = mod(), Mod:store_filemeta(storage(), Transfer, FileMeta). --spec store_segment(ctx(), emqx_ft:transfer(), emqx_ft:segment()) -> - {ok, ctx()} | {error, term()}. -store_segment(Ctx, Transfer, Segment) -> +-spec store_segment(emqx_ft:transfer(), emqx_ft:segment()) -> + ok | {error, term()}. +store_segment(Transfer, Segment) -> Mod = mod(), - Mod:store_segment(storage(), Ctx, Transfer, Segment). + Mod:store_segment(storage(), Transfer, Segment). --spec assemble(ctx(), emqx_ft:transfer(), assemble_callback()) -> +-spec assemble(emqx_ft:transfer(), assemble_callback()) -> {ok, pid()} | {error, term()}. -assemble(Ctx, Transfer, Callback) -> +assemble(Transfer, Callback) -> Mod = mod(), - Mod:assemble(storage(), Ctx, Transfer, Callback). + Mod:assemble(storage(), Transfer, Callback). mod() -> case storage() of #{type := local} -> - % emqx_ft_storage_fs - emqx_ft_storage_dummy + emqx_ft_storage_fs + % emqx_ft_storage_dummy end. storage() -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_dummy.erl b/apps/emqx_ft/src/emqx_ft_storage_dummy.erl index 1ab7f558e..d486c0c29 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_dummy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_dummy.erl @@ -20,16 +20,16 @@ -export([ store_filemeta/3, - store_segment/4, - assemble/4 + store_segment/3, + assemble/3 ]). store_filemeta(_Storage, _Transfer, _Meta) -> - {ok, #{}}. + ok. -store_segment(_Storage, Ctx, _Transfer, _Segment) -> - {ok, Ctx}. +store_segment(_Storage, _Transfer, _Segment) -> + ok. -assemble(_Storage, _Ctx, _Transfer, Callback) -> +assemble(_Storage, _Transfer, Callback) -> Pid = spawn(fun() -> Callback({error, not_implemented}) end), {ok, Pid}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index bbd61eba9..a530c9cf8 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -22,7 +22,7 @@ -behaviour(emqx_ft_storage). -export([store_filemeta/3]). --export([store_segment/4]). +-export([store_segment/3]). -export([list/2]). -export([read_segment/5]). -export([assemble/3]). @@ -84,8 +84,7 @@ store_filemeta(Storage, Transfer, Meta) -> case read_file(Filepath, fun decode_filemeta/1) of {ok, Meta} -> _ = touch_file(Filepath), - %% No context is needed for this implementation. - {ok, #{}}; + ok; {ok, _Conflict} -> % TODO % We won't see conflicts in case of concurrent `store_filemeta` @@ -98,18 +97,13 @@ store_filemeta(Storage, Transfer, Meta) -> %% Store a segment in the backing filesystem. %% Atomic operation. --spec store_segment(storage(), emqx_ft_storage:ctx(), transfer(), segment()) -> +-spec store_segment(storage(), transfer(), segment()) -> % Where is the checksum gets verified? Upper level probably. % Quota? Some lower level errors? ok | {error, _TODO}. -store_segment(Storage, Ctx, Transfer, Segment = {_Offset, Content}) -> +store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> Filepath = mk_filepath(Storage, Transfer, mk_segment_filename(Segment)), - case write_file_atomic(Filepath, Content) of - ok -> - {ok, Ctx}; - {error, _} = Error -> - Error - end. + write_file_atomic(Filepath, Content). -spec list(storage(), transfer()) -> % Some lower level errors? {error, notfound}? @@ -149,8 +143,9 @@ read_segment(_Storage, _Transfer, Segment, Offset, Size) -> end. -spec assemble(storage(), transfer(), fun((ok | {error, term()}) -> any())) -> + % {ok, _Assembler :: pid()} | {error, incomplete} | {error, badrpc} | {error, _TODO}. {ok, _Assembler :: pid()} | {error, _TODO}. -assemble(Storage, _Ctx, Transfer, Callback) -> +assemble(Storage, Transfer, Callback) -> emqx_ft_assembler_sup:start_child(Storage, Transfer, Callback). -type handle() :: {file:name(), io:device(), crypto:hash_state()}. @@ -321,8 +316,8 @@ mk_filedir(Storage, {ClientId, FileId}) -> mk_filepath(Storage, Transfer, Filename) -> filename:join(mk_filedir(Storage, Transfer), Filename). -get_storage_root(_Storage) -> - filename:join(emqx:data_dir(), "file_transfer"). +get_storage_root(Storage) -> + maps:get(root, Storage, filename:join(emqx:data_dir(), "file_transfer")). -include_lib("kernel/include/file.hrl"). diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index ef2d8033f..b4ce52edb 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -27,17 +27,7 @@ start_link() -> supervisor:start_link({local, ?SERVER}, ?MODULE, []). -%% sup_flags() = #{strategy => strategy(), % optional -%% intensity => non_neg_integer(), % optional -%% period => pos_integer()} % optional -%% child_spec() = #{id => child_id(), % mandatory -%% start => mfargs(), % mandatory -%% restart => restart(), % optional -%% shutdown => shutdown(), % optional -%% type => worker(), % optional -%% modules => modules()} % optional init([]) -> - ok = emqx_ft:create_tab(), SupFlags = #{ strategy => one_for_all, intensity => 100, @@ -64,5 +54,3 @@ init([]) -> ChildSpecs = [Responder, AssemblerSup], {ok, {SupFlags, ChildSpecs}}. - -%% internal functions diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 1112ccbb7..9ec2fce61 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -54,7 +54,7 @@ end_per_testcase(_TC, Config) -> -define(CLIENTID, <<"thatsme">>). t_assemble_empty_transfer(Config) -> - Storage = ?config(storage_root, Config), + Storage = storage(Config), Transfer = {?CLIENTID, mk_fileid()}, Filename = "important.pdf", Meta = #{ @@ -84,7 +84,7 @@ t_assemble_empty_transfer(Config) -> ok. t_assemble_complete_local_transfer(Config) -> - Storage = ?config(storage_root, Config), + Storage = storage(Config), Transfer = {?CLIENTID, mk_fileid()}, Filename = "topsecret.pdf", TransferSize = 10000 + rand:uniform(50000), @@ -161,3 +161,8 @@ inspect_file(Filename) -> mk_fileid() -> integer_to_binary(erlang:system_time(millisecond)). + +storage(Config) -> + #{ + root => ?config(storage_root, Config) + }. From 8298236908ef8ed7ae237acaf4ec6a6ec6681d61 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 3 Feb 2023 12:38:02 +0300 Subject: [PATCH 012/156] refactor(ft): bring back userdata to filemeta schema --- apps/emqx_ft/src/emqx_ft.erl | 3 +- apps/emqx_ft/src/emqx_ft_schema.erl | 40 +++++++++++++++++++++++++ apps/emqx_ft/src/emqx_ft_storage_fs.erl | 34 +++------------------ 3 files changed, 46 insertions(+), 31 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index be2f80831..ce3a7a97d 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -66,7 +66,8 @@ %% TTL of individual segments %% Somewhat confusing that we won't know it on the nodes where the filemeta %% is missing. - segments_ttl => _Seconds :: pos_integer() + segments_ttl => _Seconds :: pos_integer(), + user_data => emqx_ft_schema:json_value() }. -type segment() :: {offset(), _Content :: binary()}. diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index f40d2f40e..70acb8322 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -23,6 +23,20 @@ -export([namespace/0, roots/0, fields/1, tags/0]). +-export([schema/1]). + +-type json_value() :: + null + | boolean() + | binary() + | number() + | [json_value()] + | #{binary() => json_value()}. + +-reflect_type([json_value/0]). + +%% + namespace() -> file_transfer. tags() -> @@ -47,3 +61,29 @@ fields(local_storage) -> desc => ?DESC("local") }} ]. + +schema(filemeta) -> + #{ + roots => [ + {name, hoconsc:mk(string(), #{required => true})}, + {size, hoconsc:mk(non_neg_integer())}, + {expire_at, hoconsc:mk(non_neg_integer())}, + {checksum, hoconsc:mk({atom(), binary()}, #{converter => converter(checksum)})}, + {segments_ttl, hoconsc:mk(pos_integer())}, + {user_data, hoconsc:mk(json_value())} + ] + }. + +converter(checksum) -> + fun + (undefined, #{}) -> + undefined; + ({sha256, Bin}, #{make_serializable := true}) -> + _ = is_binary(Bin) orelse throw({expected_type, string}), + _ = byte_size(Bin) =:= 32 orelse throw({expected_length, 32}), + binary:encode_hex(Bin); + (Hex, #{}) -> + _ = is_binary(Hex) orelse throw({expected_type, string}), + _ = byte_size(Hex) =:= 64 orelse throw({expected_length, 64}), + {sha256, binary:decode_hex(Hex)} + end. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index a530c9cf8..e2bda8fcb 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -16,9 +16,6 @@ -module(emqx_ft_storage_fs). --include_lib("typerefl/include/types.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -behaviour(emqx_ft_storage). -export([store_filemeta/3]). @@ -224,17 +221,6 @@ verify_checksum(undefined, _) -> -define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). -schema() -> - #{ - roots => [ - {name, hoconsc:mk(string(), #{required => true})}, - {size, hoconsc:mk(non_neg_integer())}, - {expire_at, hoconsc:mk(non_neg_integer())}, - {checksum, hoconsc:mk({atom(), binary()}, #{converter => converter(checksum)})}, - {segments_ttl, hoconsc:mk(pos_integer())} - ] - }. - % encode_filemeta(Meta) -> % emqx_json:encode( % ?PRELUDE( @@ -261,26 +247,14 @@ schema() -> encode_filemeta(Meta) -> % TODO: Looks like this should be hocon's responsibility. - Term = hocon_tconf:make_serializable(schema(), emqx_map_lib:binary_key_map(Meta), #{}), + Schema = emqx_ft_schema:schema(filemeta), + Term = hocon_tconf:make_serializable(Schema, emqx_map_lib:binary_key_map(Meta), #{}), emqx_json:encode(?PRELUDE(_Vsn = 1, Term)). decode_filemeta(Binary) -> + Schema = emqx_ft_schema:schema(filemeta), ?PRELUDE(_Vsn = 1, Term) = emqx_json:decode(Binary, [return_maps]), - hocon_tconf:check_plain(schema(), Term, #{atom_key => true, required => false}). - -converter(checksum) -> - fun - (undefined, #{}) -> - undefined; - ({sha256, Bin}, #{make_serializable := true}) -> - _ = is_binary(Bin) orelse throw({expected_type, string}), - _ = byte_size(Bin) =:= 32 orelse throw({expected_length, 32}), - binary:encode_hex(Bin); - (Hex, #{}) -> - _ = is_binary(Hex) orelse throw({expected_type, string}), - _ = byte_size(Hex) =:= 64 orelse throw({expected_length, 64}), - {sha256, binary:decode_hex(Hex)} - end. + hocon_tconf:check_plain(Schema, Term, #{atom_key => true, required => false}). % map_into(Fun, Into, Ks, Map) -> % map_foldr(map_into_fn(Fun, Into), Into, Ks, Map). From 1308fa0e6b4b19d70fd07f4b6bbafdc4f57d0d1a Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 3 Feb 2023 12:40:22 +0300 Subject: [PATCH 013/156] fix(ft-fs): put fragments into separate directories In order to avoid potential filename collisions. --- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 49 ++++++++++--------- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 2 +- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index e2bda8fcb..cc9d94b35 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -49,9 +49,11 @@ -type filefrag() :: filefrag({filemeta, filemeta()} | {segment, segmentinfo()}). +-define(FRAGDIR, frags). +-define(TEMPDIR, tmp). +-define(RESULTDIR, result). -define(MANIFEST, "MANIFEST.json"). -define(SEGMENT, "SEG"). --define(TEMP, "TMP"). -type root() :: file:name(). @@ -77,7 +79,7 @@ % Quota? Some lower level errors? {ok, emqx_ft_storage:ctx()} | {error, conflict} | {error, _TODO}. store_filemeta(Storage, Transfer, Meta) -> - Filepath = mk_filepath(Storage, Transfer, ?MANIFEST), + Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], ?MANIFEST), case read_file(Filepath, fun decode_filemeta/1) of {ok, Meta} -> _ = touch_file(Filepath), @@ -89,7 +91,7 @@ store_filemeta(Storage, Transfer, Meta) -> % about it too much now. {error, conflict}; {error, Reason} when Reason =:= notfound; Reason =:= corrupted; Reason =:= enoent -> - write_file_atomic(Filepath, encode_filemeta(Meta)) + write_file_atomic(Storage, Transfer, Filepath, encode_filemeta(Meta)) end. %% Store a segment in the backing filesystem. @@ -99,15 +101,15 @@ store_filemeta(Storage, Transfer, Meta) -> % Quota? Some lower level errors? ok | {error, _TODO}. store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> - Filepath = mk_filepath(Storage, Transfer, mk_segment_filename(Segment)), - write_file_atomic(Filepath, Content). + Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], mk_segment_filename(Segment)), + write_file_atomic(Storage, Transfer, Filepath, Content). -spec list(storage(), transfer()) -> % Some lower level errors? {error, notfound}? % Result will contain zero or only one filemeta. {ok, list(filefrag())} | {error, _TODO}. list(Storage, Transfer) -> - Dirname = mk_filedir(Storage, Transfer), + Dirname = mk_filedir(Storage, Transfer, [?FRAGDIR]), case file:list_dir(Dirname) of {ok, Filenames} -> {ok, filtermap_files(fun mk_filefrag/2, Dirname, Filenames)}; @@ -151,8 +153,8 @@ assemble(Storage, Transfer, Callback) -> {ok, handle()} | {error, _TODO}. open_file(Storage, Transfer, Filemeta) -> Filename = maps:get(name, Filemeta), - Filepath = mk_filepath(Storage, Transfer, Filename), - TempFilepath = mk_temp_filepath(Filepath), + TempFilepath = mk_temp_filepath(Storage, Transfer, Filename), + _ = filelib:ensure_dir(TempFilepath), case file:open(TempFilepath, [write, raw]) of {ok, Handle} -> _ = file:truncate(Handle), @@ -174,11 +176,11 @@ write({Filepath, IoDevice, Ctx}, IoData) -> -spec complete(storage(), transfer(), filemeta(), handle()) -> ok | {error, {checksum, _Algo, _Computed}} | {error, _TODO}. complete(Storage, Transfer, Filemeta, Handle = {Filepath, IoDevice, Ctx}) -> - TargetFilepath = mk_filepath(Storage, Transfer, maps:get(name, Filemeta)), + TargetFilepath = mk_filepath(Storage, Transfer, [?RESULTDIR], maps:get(name, Filemeta)), case verify_checksum(Ctx, Filemeta) of ok -> ok = file:close(IoDevice), - file:rename(Filepath, TargetFilepath); + mv_temp_file(Filepath, TargetFilepath); {error, _} = Error -> _ = discard(Handle), Error @@ -284,11 +286,11 @@ break_segment_filename(Filename) -> {error, invalid} end. -mk_filedir(Storage, {ClientId, FileId}) -> - filename:join([get_storage_root(Storage), ClientId, FileId]). +mk_filedir(Storage, {ClientId, FileId}, SubDirs) -> + filename:join([get_storage_root(Storage), ClientId, FileId | SubDirs]). -mk_filepath(Storage, Transfer, Filename) -> - filename:join(mk_filedir(Storage, Transfer), Filename). +mk_filepath(Storage, Transfer, SubDirs, Filename) -> + filename:join(mk_filedir(Storage, Transfer, SubDirs), Filename). get_storage_root(Storage) -> maps:get(root, Storage, filename:join(emqx:data_dir(), "file_transfer")). @@ -315,13 +317,13 @@ safe_decode(Content, DecodeFun) -> {error, corrupted} end. -write_file_atomic(Filepath, Content) when is_binary(Content) -> - TempFilepath = mk_temp_filepath(Filepath), +write_file_atomic(Storage, Transfer, Filepath, Content) when is_binary(Content) -> + TempFilepath = mk_temp_filepath(Storage, Transfer, filename:basename(Filepath)), Result = emqx_misc:pipeline( [ fun filelib:ensure_dir/1, fun write_contents/2, - fun(FP) -> mv_temp_file(Filepath, FP) end + fun(_) -> mv_temp_file(TempFilepath, Filepath) end ], TempFilepath, Content @@ -334,11 +336,9 @@ write_file_atomic(Filepath, Content) when is_binary(Content) -> {error, Reason} end. -mk_temp_filepath(Filepath) -> - Dirname = filename:dirname(Filepath), - Filename = filename:basename(Filepath), +mk_temp_filepath(Storage, Transfer, Filename) -> Unique = erlang:unique_integer([positive]), - filename:join(Dirname, mk_filename([?TEMP, Unique, ".", Filename])). + filename:join(mk_filedir(Storage, Transfer, [?TEMPDIR]), mk_filename([Unique, ".", Filename])). mk_filename(Comps) -> lists:append(lists:map(fun mk_filename_component/1, Comps)). @@ -351,7 +351,8 @@ mk_filename_component(S) when is_list(S) -> S. write_contents(Filepath, Content) -> file:write_file(Filepath, Content). -mv_temp_file(Filepath, TempFilepath) -> +mv_temp_file(TempFilepath, Filepath) -> + _ = filelib:ensure_dir(Filepath), file:rename(TempFilepath, Filepath). touch_file(Filepath) -> @@ -365,7 +366,8 @@ mk_filefrag(Dirname, Filename = ?MANIFEST) -> mk_filefrag(Dirname, Filename, filemeta, fun read_filemeta/2); mk_filefrag(Dirname, Filename = ?SEGMENT ++ _) -> mk_filefrag(Dirname, Filename, segment, fun read_segmentinfo/2); -mk_filefrag(_Dirname, _) -> +mk_filefrag(_Dirname, _Filename) -> + % TODO this is unexpected, worth logging? false. mk_filefrag(Dirname, Filename, Tag, Fun) -> @@ -380,6 +382,7 @@ mk_filefrag(Dirname, Filename, Tag, Fun) -> fragment => {Tag, Frag} }}; {error, _Reason} -> + % TODO loss of information false end. diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 9ec2fce61..fdfdd432e 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -133,7 +133,7 @@ t_assemble_complete_local_transfer(Config) -> ). mk_assembly_filename(Config, {ClientID, FileID}, Filename) -> - filename:join([?config(storage_root, Config), ClientID, FileID, Filename]). + filename:join([?config(storage_root, Config), ClientID, FileID, result, Filename]). on_assembly_finished(Result) -> ?tp(test_assembly_finished, #{result => Result}). From 429eeaf029d69c896f46c8a6ee57bf90634b780b Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 3 Feb 2023 14:23:58 +0300 Subject: [PATCH 014/156] feat(ft-fs): make `list` / `read` more generic And usable in wider contexts as a consequence, for example querying and fetching resulting files from remote nodes. --- apps/emqx_ft/src/emqx_ft_assembler.erl | 6 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 60 ++++++++++++++----- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 19 +++++- 3 files changed, 65 insertions(+), 20 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 4fd8d6e75..af50dee3b 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -66,7 +66,7 @@ init({Storage, Transfer, Callback}) -> handle_event(internal, _, list_local_fragments, St = #st{assembly = Asm}) -> % TODO: what we do with non-transients errors here (e.g. `eacces`)? - {ok, Fragments} = emqx_ft_storage_fs:list(St#st.storage, St#st.transfer), + {ok, Fragments} = emqx_ft_storage_fs:list(St#st.storage, St#st.transfer, fragment), NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(Asm, node(), Fragments)), NSt = St#st{assembly = NAsm}, case emqx_ft_assembly:status(NAsm) of @@ -81,7 +81,7 @@ handle_event(internal, _, list_local_fragments, St = #st{assembly = Asm}) -> end; handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> % TODO: portable "storage" ref - Args = [St#st.storage, St#st.transfer], + Args = [St#st.storage, St#st.transfer, fragment], % TODO % Async would better because we would not need to wait for some lagging nodes if % the coverage is already complete. @@ -121,7 +121,7 @@ handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % this node garbage collecting the segment itself. Args = [St#st.storage, St#st.transfer, Segment, 0, segsize(Segment)], % TODO: pipelining - case erpc:call(Node, emqx_ft_storage_fs, read_segment, Args, ?RPC_READSEG_TIMEOUT) of + case erpc:call(Node, emqx_ft_storage_fs, pread, Args, ?RPC_READSEG_TIMEOUT) of {ok, Content} -> {ok, NHandle} = emqx_ft_storage_fs:write(St#st.file, Content), {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])} diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index cc9d94b35..fdf4558b8 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -20,8 +20,8 @@ -export([store_filemeta/3]). -export([store_segment/3]). --export([list/2]). --export([read_segment/5]). +-export([list/3]). +-export([pread/5]). -export([assemble/3]). -export([open_file/3]). @@ -41,13 +41,19 @@ size := _Bytes :: non_neg_integer() }. +% TODO naming -type filefrag(T) :: #{ path := file:name(), timestamp := emqx_datetime:epoch_second(), + size := _Bytes :: non_neg_integer(), fragment := T }. --type filefrag() :: filefrag({filemeta, filemeta()} | {segment, segmentinfo()}). +-type filefrag() :: filefrag( + {filemeta, filemeta()} + | {segment, segmentinfo()} + | {result, #{}} +). -define(FRAGDIR, frags). -define(TEMPDIR, tmp). @@ -104,29 +110,44 @@ store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], mk_segment_filename(Segment)), write_file_atomic(Storage, Transfer, Filepath, Content). --spec list(storage(), transfer()) -> +-spec list(storage(), transfer(), _What :: fragment | result) -> % Some lower level errors? {error, notfound}? % Result will contain zero or only one filemeta. - {ok, list(filefrag())} | {error, _TODO}. -list(Storage, Transfer) -> - Dirname = mk_filedir(Storage, Transfer, [?FRAGDIR]), + {ok, [filefrag({filemeta, filemeta()} | {segment, segmentinfo()})]} | {error, _TODO}. +list(Storage, Transfer, What) -> + Dirname = mk_filedir(Storage, Transfer, get_subdirs_for(What)), case file:list_dir(Dirname) of {ok, Filenames} -> - {ok, filtermap_files(fun mk_filefrag/2, Dirname, Filenames)}; + % TODO + % In case of `What = result` there might be more than one file (though + % extremely bad luck is needed for that, e.g. concurrent assemblers with + % different filemetas from different nodes). This might be unexpected for a + % client given the current protocol, yet might be helpful in the future. + {ok, filtermap_files(get_filefrag_fun_for(What), Dirname, Filenames)}; {error, enoent} -> {ok, []}; {error, _} = Error -> Error end. --spec read_segment( - storage(), transfer(), filefrag(segmentinfo()), offset(), _Size :: non_neg_integer() -) -> +get_subdirs_for(fragment) -> + [?FRAGDIR]; +get_subdirs_for(result) -> + [?RESULTDIR]. + +get_filefrag_fun_for(fragment) -> + fun mk_filefrag/2; +get_filefrag_fun_for(result) -> + fun mk_result_filefrag/2. + +-spec pread(storage(), transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> {ok, _Content :: iodata()} | {error, _TODO}. -read_segment(_Storage, _Transfer, Segment, Offset, Size) -> - Filepath = maps:get(path, Segment), - case file:open(Filepath, [raw, read]) of +pread(_Storage, _Transfer, Frag, Offset, Size) -> + Filepath = maps:get(path, Frag), + case file:open(Filepath, [read, raw, binary]) of {ok, IoDevice} -> + % NOTE + % Reading empty file is always `eof`. Read = file:pread(IoDevice, Offset, Size), ok = file:close(IoDevice), case Read of @@ -147,6 +168,8 @@ read_segment(_Storage, _Transfer, Segment, Offset, Size) -> assemble(Storage, Transfer, Callback) -> emqx_ft_assembler_sup:start_child(Storage, Transfer, Callback). +%% + -type handle() :: {file:name(), io:device(), crypto:hash_state()}. -spec open_file(storage(), transfer(), filemeta()) -> @@ -155,7 +178,7 @@ open_file(Storage, Transfer, Filemeta) -> Filename = maps:get(name, Filemeta), TempFilepath = mk_temp_filepath(Storage, Transfer, Filename), _ = filelib:ensure_dir(TempFilepath), - case file:open(TempFilepath, [write, raw]) of + case file:open(TempFilepath, [write, raw, binary]) of {ok, Handle} -> _ = file:truncate(Handle), {ok, {TempFilepath, Handle, init_checksum(Filemeta)}}; @@ -370,6 +393,12 @@ mk_filefrag(_Dirname, _Filename) -> % TODO this is unexpected, worth logging? false. +mk_result_filefrag(Dirname, Filename) -> + % NOTE + % Any file in the `?RESULTDIR` subdir is currently considered the result of + % the file transfer. + mk_filefrag(Dirname, Filename, result, fun(_, _) -> {ok, #{}} end). + mk_filefrag(Dirname, Filename, Tag, Fun) -> Filepath = filename:join(Dirname, Filename), % TODO error handling? @@ -379,6 +408,7 @@ mk_filefrag(Dirname, Filename, Tag, Fun) -> {true, #{ path => Filepath, timestamp => Fileinfo#file_info.mtime, + size => Fileinfo#file_info.size, fragment => {Tag, Frag} }}; {error, _Reason} -> diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index fdfdd432e..77a619fe3 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -71,7 +71,7 @@ t_assemble_empty_transfer(Config) -> fragment := {filemeta, Meta} } ]}, - emqx_ft_storage_fs:list(Storage, Transfer) + emqx_ft_storage_fs:list(Storage, Transfer, fragment) ), {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun on_assembly_finished/1), {ok, Event} = ?block_until(#{?snk_kind := test_assembly_finished}), @@ -81,6 +81,11 @@ t_assemble_empty_transfer(Config) -> % TODO file:read_file(mk_assembly_filename(Config, Transfer, Filename)) ), + {ok, [Result = #{size := Size = 0}]} = emqx_ft_storage_fs:list(Storage, Transfer, result), + ?assertEqual( + {error, eof}, + emqx_ft_storage_fs:pread(Storage, Transfer, Result, 0, Size) + ), ok. t_assemble_complete_local_transfer(Config) -> @@ -109,7 +114,7 @@ t_assemble_complete_local_transfer(Config) -> end ), - {ok, Fragments} = emqx_ft_storage_fs:list(Storage, Transfer), + {ok, Fragments} = emqx_ft_storage_fs:list(Storage, Transfer, fragment), ?assertEqual((TransferSize div SegmentSize) + 1 + 1, length(Fragments)), ?assertEqual( [Meta], @@ -122,6 +127,16 @@ t_assemble_complete_local_transfer(Config) -> ?assertMatch(#{result := ok}, Event), AssemblyFilename = mk_assembly_filename(Config, Transfer, Filename), + ?assertMatch( + {ok, [ + #{ + path := AssemblyFilename, + size := TransferSize, + fragment := {result, #{}} + } + ]}, + emqx_ft_storage_fs:list(Storage, Transfer, result) + ), ?assertMatch( {ok, #file_info{type = regular, size = TransferSize}}, file:read_file_info(AssemblyFilename) From 7ed06b0a2a3abbf76cf9dafd115ec203273833a8 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 3 Feb 2023 17:04:27 +0300 Subject: [PATCH 015/156] feat(ft-fs): allow to list all transfers in storage This is rather simplistic and thus, temporary solution. --- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 70 ++++++++++++++++++- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 48 ++++++++++--- 2 files changed, 104 insertions(+), 14 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index fdf4558b8..3a78559c1 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -24,6 +24,8 @@ -export([pread/5]). -export([assemble/3]). +-export([transfers/1]). + -export([open_file/3]). -export([complete/4]). -export([write/2]). @@ -31,16 +33,19 @@ -type transfer() :: emqx_ft:transfer(). -type offset() :: emqx_ft:offset(). - -type filemeta() :: emqx_ft:filemeta(). - --type segment() :: {offset(), _Content :: binary()}. +-type segment() :: emqx_ft:segment(). -type segmentinfo() :: #{ offset := offset(), size := _Bytes :: non_neg_integer() }. +-type transferinfo() :: #{ + status := complete | incomplete, + result => [filefrag({result, #{}})] +}. + % TODO naming -type filefrag(T) :: #{ path := file:name(), @@ -85,6 +90,7 @@ % Quota? Some lower level errors? {ok, emqx_ft_storage:ctx()} | {error, conflict} | {error, _TODO}. store_filemeta(Storage, Transfer, Meta) -> + % TODO safeguard against bad clientids / fileids. Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], ?MANIFEST), case read_file(Filepath, fun decode_filemeta/1) of {ok, Meta} -> @@ -170,6 +176,52 @@ assemble(Storage, Transfer, Callback) -> %% +-spec transfers(storage()) -> + {ok, #{transfer() => transferinfo()}}. +transfers(Storage) -> + % TODO `Continuation` + % There might be millions of transfers on the node, we need a protocol and + % storage schema to iterate through them effectively. + ClientIds = try_list_dir(get_storage_root(Storage)), + {ok, + lists:foldl( + fun(ClientId, Acc) -> transfers(Storage, ClientId, Acc) end, + #{}, + ClientIds + )}. + +transfers(Storage, ClientId, AccIn) -> + Dirname = mk_client_filedir(Storage, ClientId), + case file:list_dir(Dirname) of + {ok, FileIds} -> + lists:foldl( + fun(FileId, Acc) -> + Transfer = {filename_to_binary(ClientId), filename_to_binary(FileId)}, + read_transferinfo(Storage, Transfer, Acc) + end, + AccIn, + FileIds + ); + {error, _Reason} -> + % TODO worth logging + AccIn + end. + +read_transferinfo(Storage, Transfer, Acc) -> + case list(Storage, Transfer, result) of + {ok, Result = [_ | _]} -> + Info = #{status => complete, result => Result}, + Acc#{Transfer => Info}; + {ok, []} -> + Info = #{status => incomplete}, + Acc#{Transfer => Info}; + {error, _Reason} -> + % TODO worth logging + Acc + end. + +%% + -type handle() :: {file:name(), io:device(), crypto:hash_state()}. -spec open_file(storage(), transfer(), filemeta()) -> @@ -312,9 +364,18 @@ break_segment_filename(Filename) -> mk_filedir(Storage, {ClientId, FileId}, SubDirs) -> filename:join([get_storage_root(Storage), ClientId, FileId | SubDirs]). +mk_client_filedir(Storage, ClientId) -> + filename:join([get_storage_root(Storage), ClientId]). + mk_filepath(Storage, Transfer, SubDirs, Filename) -> filename:join(mk_filedir(Storage, Transfer, SubDirs), Filename). +try_list_dir(Dirname) -> + case file:list_dir(Dirname) of + {ok, List} -> List; + {error, _} -> [] + end. + get_storage_root(Storage) -> maps:get(root, Storage, filename:join(emqx:data_dir(), "file_transfer")). @@ -421,3 +482,6 @@ read_filemeta(_Filename, Filepath) -> read_segmentinfo(Filename, _Filepath) -> break_segment_filename(Filename). + +filename_to_binary(S) when is_list(S) -> unicode:characters_to_binary(S); +filename_to_binary(B) when is_binary(B) -> B. diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 77a619fe3..336580584 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -24,24 +24,31 @@ -include_lib("kernel/include/file.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -all() -> emqx_common_test_helpers:all(?MODULE). +all() -> + [ + t_assemble_empty_transfer, + t_assemble_complete_local_transfer, + + % NOTE + % It depends on the side effects of all previous testcases. + t_list_transfers + ]. init_per_suite(Config) -> - % {ok, Apps} = application:ensure_all_started(emqx_ft), - % [{suite_apps, Apps} | Config]. - % ok = emqx_common_test_helpers:start_apps([emqx_ft]), Config. end_per_suite(_Config) -> - % lists:foreach(fun application:stop/1, lists:reverse(?config(suite_apps, Config))). - % ok = emqx_common_test_helpers:stop_apps([emqx_ft]), ok. init_per_testcase(TC, Config) -> ok = snabbkaffe:start_trace(), - Root = filename:join(["roots", TC]), {ok, Pid} = emqx_ft_assembler_sup:start_link(), - [{storage_root, Root}, {assembler_sup, Pid} | Config]. + [ + {storage_root, "file_transfer_root"}, + {file_id, atom_to_binary(TC)}, + {assembler_sup, Pid} + | Config + ]. end_per_testcase(_TC, Config) -> ok = inspect_storage_root(Config), @@ -51,11 +58,12 @@ end_per_testcase(_TC, Config) -> %% --define(CLIENTID, <<"thatsme">>). +-define(CLIENTID1, <<"thatsme">>). +-define(CLIENTID2, <<"thatsnotme">>). t_assemble_empty_transfer(Config) -> Storage = storage(Config), - Transfer = {?CLIENTID, mk_fileid()}, + Transfer = {?CLIENTID1, ?config(file_id, Config)}, Filename = "important.pdf", Meta = #{ name => Filename, @@ -90,7 +98,7 @@ t_assemble_empty_transfer(Config) -> t_assemble_complete_local_transfer(Config) -> Storage = storage(Config), - Transfer = {?CLIENTID, mk_fileid()}, + Transfer = {?CLIENTID2, ?config(file_id, Config)}, Filename = "topsecret.pdf", TransferSize = 10000 + rand:uniform(50000), SegmentSize = 4096, @@ -155,6 +163,24 @@ on_assembly_finished(Result) -> %% +t_list_transfers(Config) -> + Storage = storage(Config), + ?assertMatch( + {ok, #{ + {?CLIENTID1, <<"t_assemble_empty_transfer">>} := #{ + status := complete, + result := [#{path := _, size := 0, fragment := {result, _}}] + }, + {?CLIENTID2, <<"t_assemble_complete_local_transfer">>} := #{ + status := complete, + result := [#{path := _, size := Size, fragment := {result, _}}] + } + }} when Size > 0, + emqx_ft_storage_fs:transfers(Storage) + ). + +%% + -include_lib("kernel/include/file.hrl"). inspect_storage_root(Config) -> From 92670bfe3df34ccd1fdd1e99b7bf0aa094bf4c3b Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 3 Feb 2023 18:48:50 +0300 Subject: [PATCH 016/156] feat(ft): add fs storage bpapi and use it in assembler --- apps/emqx_ft/src/emqx_ft_assembler.erl | 14 +++-- apps/emqx_ft/src/emqx_ft_storage.erl | 51 ++++++++++++++++- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 4 ++ .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 56 +++++++++++++++++++ 4 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index af50dee3b..ef4daf000 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -80,13 +80,11 @@ handle_event(internal, _, list_local_fragments, St = #st{assembly = Asm}) -> % {stop, Reason} end; handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> - % TODO: portable "storage" ref - Args = [St#st.storage, St#st.transfer, fragment], % TODO % Async would better because we would not need to wait for some lagging nodes if % the coverage is already complete. - % TODO: BP API? - Results = erpc:multicall(Nodes, emqx_ft_storage_fs, list, Args, ?RPC_LIST_TIMEOUT), + % TODO: portable "storage" ref + Results = emqx_ft_storage_fs_proto_v1:multilist(Nodes, St#st.transfer, fragment), NodeResults = lists:zip(Nodes, Results), NAsm = emqx_ft_assembly:update( lists:foldl( @@ -119,9 +117,8 @@ handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % TODO % Currently, race is possible between getting segment info from the remote node and % this node garbage collecting the segment itself. - Args = [St#st.storage, St#st.transfer, Segment, 0, segsize(Segment)], % TODO: pipelining - case erpc:call(Node, emqx_ft_storage_fs, pread, Args, ?RPC_READSEG_TIMEOUT) of + case pread(Node, Segment, St) of {ok, Content} -> {ok, NHandle} = emqx_ft_storage_fs:write(St#st.file, Content), {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])} @@ -158,6 +155,11 @@ handle_event(internal, _, complete, St = #st{assembly = Asm, file = Handle, call % handle_cast(_Cast, St) -> % {noreply, St}. +pread(Node, Segment, St) when Node =:= node() -> + emqx_ft_storage_fs:pread(St#st.storage, St#st.transfer, Segment, 0, segsize(Segment)); +pread(Node, Segment, St) -> + emqx_ft_storage_fs_proto_v1:pread(Node, St#st.transfer, Segment, 0, segsize(Segment)). + %% segsize(#{fragment := {segment, Info}}) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 8729a2ad4..819656551 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -24,6 +24,14 @@ ] ). +-export([list_local/2]). +-export([pread_local/4]). + +-export([local_transfers/0]). + +-type offset() :: emqx_ft:offset(). +-type transfer() :: emqx_ft:transfer(). + -type storage() :: emqx_config:config(). -export_type([assemble_callback/0]). @@ -63,8 +71,41 @@ assemble(Transfer, Callback) -> Mod = mod(), Mod:assemble(storage(), Transfer, Callback). +%%-------------------------------------------------------------------- +%% Local FS API +%%-------------------------------------------------------------------- + +-type filefrag() :: emqx_ft_storage_fs:filefrag(). +-type transferinfo() :: emqx_ft_storage_fs:transferinfo(). + +-spec list_local(transfer(), fragment | result) -> + {ok, [filefrag()]} | {error, term()}. +list_local(Transfer, What) -> + with_local_storage( + fun(Mod, Storage) -> Mod:list(Storage, Transfer, What) end + ). + +-spec pread_local(transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> + {ok, [filefrag()]} | {error, term()}. +pread_local(Transfer, Frag, Offset, Size) -> + with_local_storage( + fun(Mod, Storage) -> Mod:pread(Storage, Transfer, Frag, Offset, Size) end + ). + +-spec local_transfers() -> + {ok, node(), #{transfer() => transferinfo()}} | {error, term()}. +local_transfers() -> + with_local_storage( + fun(Mod, Storage) -> Mod:transfers(Storage) end + ). + +%% + mod() -> - case storage() of + mod(storage()). + +mod(Storage) -> + case Storage of #{type := local} -> emqx_ft_storage_fs % emqx_ft_storage_dummy @@ -72,3 +113,11 @@ mod() -> storage() -> emqx_config:get([file_transfer, storage]). + +with_local_storage(Fun) -> + case storage() of + #{type := local} = Storage -> + Fun(mod(Storage), Storage); + #{type := Type} -> + {error, {unsupported_storage_type, Type}} + end. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 3a78559c1..4afbc5276 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -31,6 +31,10 @@ -export([write/2]). -export([discard/1]). +-export_type([filefrag/1]). +-export_type([filefrag/0]). +-export_type([transferinfo/0]). + -type transfer() :: emqx_ft:transfer(). -type offset() :: emqx_ft:offset(). -type filemeta() :: emqx_ft:filemeta(). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl new file mode 100644 index 000000000..45dd93ab8 --- /dev/null +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -0,0 +1,56 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_proto_v1). + +-behaviour(emqx_bpapi). + +-export([introduced_in/0]). + +-export([list/3]). +-export([multilist/3]). +-export([pread/5]). +-export([transfers/1]). + +-type offset() :: emqx_ft:offset(). +-type transfer() :: emqx_ft:transfer(). +-type filefrag() :: emqx_ft_storage_fs:filefrag(). +-type transferinfo() :: emqx_ft_storage_fs:transferinfo(). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.17". + +-spec list(node(), transfer(), fragment | result) -> + {ok, [filefrag()]} | {error, term()}. +list(Node, Transfer, What) -> + erpc:call(Node, emqx_ft_storage, list_local, [Transfer, What]). + +-spec multilist([node()], transfer(), fragment | result) -> + emqx_rpc:erpc_multicall({ok, [filefrag()]} | {error, term()}). +multilist(Nodes, Transfer, What) -> + erpc:multicall(Nodes, emqx_ft_storage, list_local, [Transfer, What]). + +-spec pread(node(), transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> + {ok, [filefrag()]} | {error, term()}. +pread(Node, Transfer, Frag, Offset, Size) -> + erpc:call(Node, emqx_ft_storage, pread_local, [Transfer, Frag, Offset, Size]). + +-spec transfers([node()]) -> + emqx_rpc:erpc_multicall({ok, #{transfer() => transferinfo()}} | {error, term()}). +transfers(Nodes) -> + erpc:multicall(Nodes, emqx_ft_storage, local_transfers, []). From 04e5378bdaa320dec5d87489f81b16e2d1d316d3 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 6 Feb 2023 01:49:06 +0200 Subject: [PATCH 017/156] feat(ft): add API --- apps/emqx_ft/i18n/emqx_ft_api_i18n.conf | 25 +++ apps/emqx_ft/src/emqx_ft.erl | 8 +- apps/emqx_ft/src/emqx_ft_api.erl | 163 ++++++++++++++++++ apps/emqx_ft/src/emqx_ft_storage.erl | 97 ++++++----- apps/emqx_ft/src/emqx_ft_storage_dummy.erl | 10 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 159 ++++++++++------- .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 25 ++- 7 files changed, 360 insertions(+), 127 deletions(-) create mode 100644 apps/emqx_ft/i18n/emqx_ft_api_i18n.conf create mode 100644 apps/emqx_ft/src/emqx_ft_api.erl diff --git a/apps/emqx_ft/i18n/emqx_ft_api_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_api_i18n.conf new file mode 100644 index 000000000..0bda935f8 --- /dev/null +++ b/apps/emqx_ft/i18n/emqx_ft_api_i18n.conf @@ -0,0 +1,25 @@ +emqx_ft_api { + + file_list { + desc { + en: "List all uploaded files." + zh: "列出所有上传的文件。" + } + label: { + en: "List all uploaded files" + zh: "列出所有上传的文件" + } + } + + file_get { + desc { + en: "Get a file by its id." + zh: "根据文件 id 获取文件。" + } + label: { + en: "Get a file by its id" + zh: "根据文件 id 获取文件" + } + } + +} diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index ce3a7a97d..f54119440 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -189,7 +189,7 @@ on_fin(PacketId, Msg, FileId, Checksum) -> %% We have new fin packet ok -> Callback = callback(FinPacketKey, FileId), - case assemble(transfer(Msg, FileId), Callback) of + case emqx_ft_storage:assemble(transfer(Msg, FileId), Callback) of %% Assembling started, packet will be acked by the callback or the responder {ok, _} -> undefined; @@ -214,12 +214,6 @@ on_fin(PacketId, Msg, FileId, Checksum) -> undefined end. -assemble(Transfer, Callback) -> - emqx_ft_storage:assemble( - Transfer, - Callback - ). - callback({ChanPid, PacketId} = Key, _FileId) -> fun(Result) -> case emqx_ft_responder:unregister(Key) of diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl new file mode 100644 index 000000000..b2a822f36 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -0,0 +1,163 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_ft_api). + +-behaviour(minirest_api). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% Swagger specs from hocon schema +-export([ + api_spec/0, + paths/0, + schema/1, + namespace/0 +]). + +-export([ + fields/1, + roots/0 +]). + +%% API callbacks +-export([ + '/file_transfer/files'/2, + '/file_transfer/file'/2 +]). + +-import(hoconsc, [mk/2, ref/1, ref/2]). + +namespace() -> "file_transfer". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + [ + "/file_transfer/files", + "/file_transfer/file" + ]. + +schema("/file_transfer/files") -> + #{ + 'operationId' => '/file_transfer/files', + get => #{ + tags => [<<"file_transfer">>], + summary => <<"List all uploaded files">>, + description => ?DESC("file_list"), + responses => #{ + 200 => <<"Operation success">>, + 503 => emqx_dashboard_swagger:error_codes( + ['SERVICE_UNAVAILABLE'], <<"Service unavailable">> + ) + } + } + }; +schema("/file_transfer/file") -> + #{ + 'operationId' => '/file_transfer/file', + get => #{ + tags => [<<"file_transfer">>], + summary => <<"Download a particular file">>, + description => ?DESC("file_get"), + parameters => [ + ref(file_node), + ref(file_clientid), + ref(file_id) + ], + responses => #{ + 200 => <<"Operation success">>, + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Not found">>), + 503 => emqx_dashboard_swagger:error_codes( + ['SERVICE_UNAVAILABLE'], <<"Service unavailable">> + ) + } + } + }. + +'/file_transfer/files'(get, #{}) -> + case emqx_ft_storage:ready_transfers() of + {ok, Transfers} -> + FormattedTransfers = lists:map( + fun({Id, Info}) -> + #{id => Id, info => format_file_info(Info)} + end, + Transfers + ), + {200, #{<<"files">> => FormattedTransfers}}; + {error, _} -> + {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} + end. + +'/file_transfer/file'(get, #{query_string := Query}) -> + case emqx_ft_storage:get_ready_transfer(Query) of + {ok, FileData} -> + {200, #{<<"content-type">> => <<"application/data">>}, FileData}; + {error, enoent} -> + {404, error_msg('NOT_FOUND', <<"Not found">>)}; + {error, _} -> + {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} + end. + +error_msg(Code, Msg) -> + #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. + +-spec fields(hocon_schema:name()) -> hocon_schema:fields(). +fields(file_node) -> + Desc = <<"File Node">>, + Meta = #{ + in => query, desc => Desc, example => <<"emqx@127.0.0.1">>, required => false + }, + [{node, hoconsc:mk(binary(), Meta)}]; +fields(file_clientid) -> + Desc = <<"File ClientId">>, + Meta = #{ + in => query, desc => Desc, example => <<"client1">>, required => false + }, + [{clientid, hoconsc:mk(binary(), Meta)}]; +fields(file_id) -> + Desc = <<"File">>, + Meta = #{ + in => query, desc => Desc, example => <<"file1">>, required => false + }, + [{fileid, hoconsc:mk(binary(), Meta)}]. + +roots() -> + [ + file_node, + file_clientid, + file_id + ]. + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +format_file_info(#{path := Path, size := Size, timestamp := Timestamp}) -> + #{ + path => Path, + size => Size, + timestamp => format_datetime(Timestamp) + }. + +format_datetime({{Year, Month, Day}, {Hour, Minute, Second}}) -> + iolist_to_binary( + io_lib:format("~4..0w-~2..0w-~2..0wT~2..0w:~2..0w:~2..0w", [ + Year, Month, Day, Hour, Minute, Second + ]) + ). diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 819656551..e8f1d9c47 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -20,24 +20,27 @@ [ store_filemeta/2, store_segment/2, - assemble/2 + assemble/2, + + parse_id/1, + + ready_transfers/0, + get_ready_transfer/1, + + with_storage_type/3 ] ). --export([list_local/2]). --export([pread_local/4]). - --export([local_transfers/0]). - --type offset() :: emqx_ft:offset(). --type transfer() :: emqx_ft:transfer(). - -type storage() :: emqx_config:config(). -export_type([assemble_callback/0]). -type assemble_callback() :: fun((ok | {error, term()}) -> any()). +-type ready_transfer_id() :: term(). +-type ready_transfer_info() :: map(). +-type ready_transfer_data() :: binary(). + %%-------------------------------------------------------------------- %% Behaviour %%-------------------------------------------------------------------- @@ -48,6 +51,10 @@ ok | {error, term()}. -callback assemble(storage(), emqx_ft:transfer(), assemble_callback()) -> {ok, pid()} | {error, term()}. +-callback ready_transfers(storage()) -> + {ok, [{ready_transfer_id(), ready_transfer_info()}]} | {error, term()}. +-callback get_ready_transfer(storage(), ready_transfer_id()) -> + {ok, ready_transfer_data()} | {error, term()}. %%-------------------------------------------------------------------- %% API @@ -71,35 +78,46 @@ assemble(Transfer, Callback) -> Mod = mod(), Mod:assemble(storage(), Transfer, Callback). +-spec ready_transfers() -> {ok, [{ready_transfer_id(), ready_transfer_info()}]} | {error, term()}. +ready_transfers() -> + Mod = mod(), + Mod:ready_transfers(storage()). + +-spec get_ready_transfer(ready_transfer_id()) -> {ok, ready_transfer_data()} | {error, term()}. +get_ready_transfer(ReadyTransferId) -> + Mod = mod(), + Mod:get_ready_transfer(storage(), ReadyTransferId). + +-spec parse_id(map()) -> {ok, ready_transfer_id()} | {error, term()}. +parse_id(#{ + <<"type">> := local, <<"node">> := NodeBin, <<"clientid">> := ClientId, <<"id">> := Id +}) -> + case emqx_misc:safe_to_existing_atom(NodeBin) of + {ok, Node} -> + {ok, {local, Node, ClientId, Id}}; + {error, _} -> + {error, {invalid_node, NodeBin}} + end; +parse_id(#{}) -> + {error, invalid_file_id}. + +-spec with_storage_type(atom(), atom(), list(term())) -> any(). +with_storage_type(Type, Fun, Args) -> + Storage = storage(), + case Storage of + #{type := Type} -> + Mod = mod(Storage), + apply(Mod, Fun, [Storage | Args]); + _ -> + {error, {invalid_storage_type, Type}} + end. + %%-------------------------------------------------------------------- %% Local FS API %%-------------------------------------------------------------------- --type filefrag() :: emqx_ft_storage_fs:filefrag(). --type transferinfo() :: emqx_ft_storage_fs:transferinfo(). - --spec list_local(transfer(), fragment | result) -> - {ok, [filefrag()]} | {error, term()}. -list_local(Transfer, What) -> - with_local_storage( - fun(Mod, Storage) -> Mod:list(Storage, Transfer, What) end - ). - --spec pread_local(transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> - {ok, [filefrag()]} | {error, term()}. -pread_local(Transfer, Frag, Offset, Size) -> - with_local_storage( - fun(Mod, Storage) -> Mod:pread(Storage, Transfer, Frag, Offset, Size) end - ). - --spec local_transfers() -> - {ok, node(), #{transfer() => transferinfo()}} | {error, term()}. -local_transfers() -> - with_local_storage( - fun(Mod, Storage) -> Mod:transfers(Storage) end - ). - -%% +storage() -> + emqx_config:get([file_transfer, storage]). mod() -> mod(storage()). @@ -110,14 +128,3 @@ mod(Storage) -> emqx_ft_storage_fs % emqx_ft_storage_dummy end. - -storage() -> - emqx_config:get([file_transfer, storage]). - -with_local_storage(Fun) -> - case storage() of - #{type := local} = Storage -> - Fun(mod(Storage), Storage); - #{type := Type} -> - {error, {unsupported_storage_type, Type}} - end. diff --git a/apps/emqx_ft/src/emqx_ft_storage_dummy.erl b/apps/emqx_ft/src/emqx_ft_storage_dummy.erl index d486c0c29..4ed8ba487 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_dummy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_dummy.erl @@ -21,7 +21,9 @@ -export([ store_filemeta/3, store_segment/3, - assemble/3 + assemble/3, + ready_transfers/1, + get_ready_transfer/2 ]). store_filemeta(_Storage, _Transfer, _Meta) -> @@ -33,3 +35,9 @@ store_segment(_Storage, _Transfer, _Segment) -> assemble(_Storage, _Transfer, Callback) -> Pid = spawn(fun() -> Callback({error, not_implemented}) end), {ok, Pid}. + +ready_transfers(_Storage) -> + {ok, []}. + +get_ready_transfer(_Storage, _Id) -> + {error, not_implemented}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 4afbc5276..37a433433 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -18,6 +18,8 @@ -behaviour(emqx_ft_storage). +-include_lib("emqx/include/logger.hrl"). + -export([store_filemeta/3]). -export([store_segment/3]). -export([list/3]). @@ -26,6 +28,14 @@ -export([transfers/1]). +-export([pread_local/4]). +-export([list_local/2]). +-export([ready_transfers_local/0, ready_transfers_local/1]). +-export([get_ready_transfer_local/1, get_ready_transfer_local/2]). + +-export([ready_transfers/1]). +-export([get_ready_transfer/2]). + -export([open_file/3]). -export([complete/4]). -export([write/2]). @@ -70,23 +80,8 @@ -define(MANIFEST, "MANIFEST.json"). -define(SEGMENT, "SEG"). --type root() :: file:name(). - -% -record(st, { -% root :: file:name() -% }). - %% TODO --type storage() :: root(). - -%% - -% -define(PROCREF(Root), {via, gproc, {n, l, {?MODULE, Root}}}). - -% -spec start_link(root()) -> -% {ok, pid()} | {error, already_started}. -% start_link(Root) -> -% gen_server:start_link(?PROCREF(Root), ?MODULE, [], []). +-type storage() :: emqx_config:config(). %% Store manifest in the backing filesystem. %% Atomic operation. @@ -178,7 +173,89 @@ pread(_Storage, _Transfer, Frag, Offset, Size) -> assemble(Storage, Transfer, Callback) -> emqx_ft_assembler_sup:start_child(Storage, Transfer, Callback). -%% +-spec list_local(transfer(), fragment | result) -> + {ok, [filefrag()]} | {error, term()}. +list_local(Transfer, What) -> + emqx_ft_storage:with_storage_type(local, list, [Transfer, What]). + +-spec pread_local(transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> + {ok, [filefrag()]} | {error, term()}. +pread_local(Transfer, Frag, Offset, Size) -> + emqx_ft_storage:with_storage_type(local, pread, [Transfer, Frag, Offset, Size]). + +get_ready_transfer(_Storage, ReadyTransferId) -> + case parse_ready_transfer_id(ReadyTransferId) of + {ok, {Node, Transfer}} -> + try + emqx_ft_storage_fs_proto_v1:get_ready_transfer(Node, Transfer) + catch + error:Error -> + {error, Error}; + C:Error -> + {error, {C, Error}} + end; + {error, _} = Error -> + Error + end. + +get_ready_transfer_local(Transfer) -> + emqx_ft_storage:with_storage_type(local, get_ready_transfer_local, [Transfer]). + +get_ready_transfer_local(Storage, Transfer) -> + Dirname = mk_filedir(Storage, Transfer, get_subdirs_for(result)), + case file:list_dir(Dirname) of + {ok, [Filename | _]} -> + file:read_file(filename:join([Dirname, Filename])); + {error, _} = Error -> + Error + end. + +ready_transfers(_Storage) -> + Nodes = mria_mnesia:running_nodes(), + Results = emqx_ft_storage_fs_proto_v1:ready_transfers(Nodes), + {GoodResults, BadResults} = lists:partition( + fun + ({ok, _}) -> true; + (_) -> false + end, + Results + ), + ?SLOG(warning, #{msg => "ready_transfers", failures => BadResults}), + {ok, [File || {ok, Files} <- GoodResults, File <- Files]}. + +ready_transfers_local() -> + emqx_ft_storage:with_storage_type(local, ready_transfers_local, []). + +ready_transfers_local(Storage) -> + {ok, Transfers} = transfers(Storage), + lists:filtermap( + fun + ({Transfer, #{status := complete, result := [Result | _]}}) -> + {true, {ready_transfer_id(Transfer), maps:without([fragment], Result)}}; + (_) -> + false + end, + maps:to_list(Transfers) + ). + +ready_transfer_id({ClientId, FileId}) -> + #{ + <<"node">> => atom_to_binary(node()), + <<"clientid">> => ClientId, + <<"fileid">> => FileId + }. + +parse_ready_transfer_id(#{ + <<"node">> := NodeBin, <<"clientid">> := ClientId, <<"fileid">> := FileId +}) -> + case emqx_misc:safe_to_existing_atom(NodeBin) of + {ok, Node} -> + {ok, {Node, {ClientId, FileId}}}; + {error, _} -> + {error, {invalid_node, NodeBin}} + end; +parse_ready_transfer_id(#{}) -> + {error, invalid_file_id}. -spec transfers(storage()) -> {ok, #{transfer() => transferinfo()}}. @@ -291,41 +368,8 @@ verify_checksum(Ctx, #{checksum := {Algo, Digest}}) when Ctx /= undefined -> verify_checksum(undefined, _) -> ok. -%% - -% -spec init(root()) -> {ok, storage()}. -% init(Root) -> -% % TODO: garbage_collect(...) -% {ok, Root}. - -% %% - -define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). -% encode_filemeta(Meta) -> -% emqx_json:encode( -% ?PRELUDE( -% _Vsn = 1, -% maps:map( -% fun -% (name, Name) -> -% {<<"name">>, Name}; -% (size, Size) -> -% {<<"size">>, Size}; -% (checksum, {sha256, Hash}) -> -% {<<"checksum">>, <<"sha256:", (binary:encode_hex(Hash))/binary>>}; -% (expire_at, ExpiresAt) -> -% {<<"expire_at">>, ExpiresAt}; -% (segments_ttl, TTL) -> -% {<<"segments_ttl">>, TTL}; -% (user_data, UserData) -> -% {<<"user_data">>, UserData} -% end, -% Meta -% ) -% ) -% ). - encode_filemeta(Meta) -> % TODO: Looks like this should be hocon's responsibility. Schema = emqx_ft_schema:schema(filemeta), @@ -337,21 +381,6 @@ decode_filemeta(Binary) -> ?PRELUDE(_Vsn = 1, Term) = emqx_json:decode(Binary, [return_maps]), hocon_tconf:check_plain(Schema, Term, #{atom_key => true, required => false}). -% map_into(Fun, Into, Ks, Map) -> -% map_foldr(map_into_fn(Fun, Into), Into, Ks, Map). - -% map_into_fn(Fun, L) when is_list(L) -> -% fun(K, V, Acc) -> [{K, Fun(K, V)} || Acc] end. - -% map_foldr(_Fun, Acc, [], _) -> -% Acc; -% map_foldr(Fun, Acc, [K | Ks], Map) when is_map_key(K, Map) -> -% Fun(K, maps:get(K, Map), map_foldr(Fun, Acc, Ks, Map)); -% map_foldr(Fun, Acc, [_ | Ks], Map) -> -% map_foldr(Fun, Acc, Ks, Map). - -%% - mk_segment_filename({Offset, Content}) -> lists:concat([?SEGMENT, ".", Offset, ".", byte_size(Content)]). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index 45dd93ab8..4f354be63 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -23,12 +23,12 @@ -export([list/3]). -export([multilist/3]). -export([pread/5]). --export([transfers/1]). +-export([ready_transfers/1]). +-export([get_ready_transfer/2]). -type offset() :: emqx_ft:offset(). -type transfer() :: emqx_ft:transfer(). -type filefrag() :: emqx_ft_storage_fs:filefrag(). --type transferinfo() :: emqx_ft_storage_fs:transferinfo(). -include_lib("emqx/include/bpapi.hrl"). @@ -38,19 +38,26 @@ introduced_in() -> -spec list(node(), transfer(), fragment | result) -> {ok, [filefrag()]} | {error, term()}. list(Node, Transfer, What) -> - erpc:call(Node, emqx_ft_storage, list_local, [Transfer, What]). + erpc:call(Node, emqx_ft_storage_fs, list_local, [Transfer, What]). -spec multilist([node()], transfer(), fragment | result) -> emqx_rpc:erpc_multicall({ok, [filefrag()]} | {error, term()}). multilist(Nodes, Transfer, What) -> - erpc:multicall(Nodes, emqx_ft_storage, list_local, [Transfer, What]). + erpc:multicall(Nodes, emqx_ft_storage_fs, list_local, [Transfer, What]). -spec pread(node(), transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> {ok, [filefrag()]} | {error, term()}. pread(Node, Transfer, Frag, Offset, Size) -> - erpc:call(Node, emqx_ft_storage, pread_local, [Transfer, Frag, Offset, Size]). + erpc:call(Node, emqx_ft_storage_fs, pread_local, [Transfer, Frag, Offset, Size]). --spec transfers([node()]) -> - emqx_rpc:erpc_multicall({ok, #{transfer() => transferinfo()}} | {error, term()}). -transfers(Nodes) -> - erpc:multicall(Nodes, emqx_ft_storage, local_transfers, []). +-spec ready_transfers([node()]) -> + {ok, [{emqx_ft_storage:ready_transfer_id(), emqx_ft_storage:ready_transfer_info()}]} + | {error, term()}. +ready_transfers(Nodes) -> + erpc:multicall(Nodes, emqx_ft_storage_fs, ready_transfers_local, []). + +-spec get_ready_transfer(node(), emqx_ft_storage:ready_transfer_id()) -> + {ok, emqx_ft_storage:ready_transfer_data()} + | {error, term()}. +get_ready_transfer(Node, ReadyTransferId) -> + erpc:call(Node, emqx_ft_storage_fs, get_ready_transfer_local, [ReadyTransferId]). From 197ce32669a437dd7c26ecba71dc571f415dfcf8 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 6 Feb 2023 14:40:40 +0200 Subject: [PATCH 018/156] feat(ft): add proxy module for emqx_ft_storage_fs --- apps/emqx_ft/src/emqx_ft.erl | 13 +++++- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 22 +--------- apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl | 40 +++++++++++++++++++ .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 17 ++++---- 4 files changed, 63 insertions(+), 29 deletions(-) create mode 100644 apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index f54119440..c4f1caed5 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -189,7 +189,7 @@ on_fin(PacketId, Msg, FileId, Checksum) -> %% We have new fin packet ok -> Callback = callback(FinPacketKey, FileId), - case emqx_ft_storage:assemble(transfer(Msg, FileId), Callback) of + case assemble(transfer(Msg, FileId), Callback) of %% Assembling started, packet will be acked by the callback or the responder {ok, _} -> undefined; @@ -214,6 +214,17 @@ on_fin(PacketId, Msg, FileId, Checksum) -> undefined end. +assemble(Transfer, Callback) -> + try + emqx_ft_storage:assemble(Transfer, Callback) + catch + C:E:S -> + ?SLOG(warning, #{ + msg => "file_assemble_failed", class => C, reason => E, stacktrace => S + }), + {error, {internal_error, E}} + end. + callback({ChanPid, PacketId} = Key, _FileId) -> fun(Result) -> case emqx_ft_responder:unregister(Key) of diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 37a433433..088c316ae 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -28,10 +28,8 @@ -export([transfers/1]). --export([pread_local/4]). --export([list_local/2]). --export([ready_transfers_local/0, ready_transfers_local/1]). --export([get_ready_transfer_local/1, get_ready_transfer_local/2]). +-export([ready_transfers_local/1]). +-export([get_ready_transfer_local/2]). -export([ready_transfers/1]). -export([get_ready_transfer/2]). @@ -173,16 +171,6 @@ pread(_Storage, _Transfer, Frag, Offset, Size) -> assemble(Storage, Transfer, Callback) -> emqx_ft_assembler_sup:start_child(Storage, Transfer, Callback). --spec list_local(transfer(), fragment | result) -> - {ok, [filefrag()]} | {error, term()}. -list_local(Transfer, What) -> - emqx_ft_storage:with_storage_type(local, list, [Transfer, What]). - --spec pread_local(transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> - {ok, [filefrag()]} | {error, term()}. -pread_local(Transfer, Frag, Offset, Size) -> - emqx_ft_storage:with_storage_type(local, pread, [Transfer, Frag, Offset, Size]). - get_ready_transfer(_Storage, ReadyTransferId) -> case parse_ready_transfer_id(ReadyTransferId) of {ok, {Node, Transfer}} -> @@ -198,9 +186,6 @@ get_ready_transfer(_Storage, ReadyTransferId) -> Error end. -get_ready_transfer_local(Transfer) -> - emqx_ft_storage:with_storage_type(local, get_ready_transfer_local, [Transfer]). - get_ready_transfer_local(Storage, Transfer) -> Dirname = mk_filedir(Storage, Transfer, get_subdirs_for(result)), case file:list_dir(Dirname) of @@ -223,9 +208,6 @@ ready_transfers(_Storage) -> ?SLOG(warning, #{msg => "ready_transfers", failures => BadResults}), {ok, [File || {ok, Files} <- GoodResults, File <- Files]}. -ready_transfers_local() -> - emqx_ft_storage:with_storage_type(local, ready_transfers_local, []). - ready_transfers_local(Storage) -> {ok, Transfers} = transfers(Storage), lists:filtermap( diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl new file mode 100644 index 000000000..0c30f5567 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl @@ -0,0 +1,40 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% This methods are called via rpc by `emqx_ft_storage_fs` +%% They populate the call with actual storage which may be configured differently +%% on a concrete node. + +-module(emqx_ft_storage_fs_proxy). + +-export([ + list_local/2, + pread_local/4, + get_ready_transfer_local/1, + ready_transfers_local/0 +]). + +list_local(Transfer, What) -> + emqx_ft_storage:with_storage_type(local, list, [Transfer, What]). + +pread_local(Transfer, Frag, Offset, Size) -> + emqx_ft_storage:with_storage_type(local, pread, [Transfer, Frag, Offset, Size]). + +get_ready_transfer_local(Transfer) -> + emqx_ft_storage:with_storage_type(local, get_ready_transfer_local, [Transfer]). + +ready_transfers_local() -> + emqx_ft_storage:with_storage_type(local, ready_transfers_local, []). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index 4f354be63..2e3dc8632 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -36,28 +36,29 @@ introduced_in() -> "5.0.17". -spec list(node(), transfer(), fragment | result) -> - {ok, [filefrag()]} | {error, term()}. + {ok, [filefrag()]} | {error, term()} | no_return(). list(Node, Transfer, What) -> - erpc:call(Node, emqx_ft_storage_fs, list_local, [Transfer, What]). + erpc:call(Node, emqx_ft_storage_fs_proxy, list_local, [Transfer, What]). -spec multilist([node()], transfer(), fragment | result) -> emqx_rpc:erpc_multicall({ok, [filefrag()]} | {error, term()}). multilist(Nodes, Transfer, What) -> - erpc:multicall(Nodes, emqx_ft_storage_fs, list_local, [Transfer, What]). + erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, list_local, [Transfer, What]). -spec pread(node(), transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> - {ok, [filefrag()]} | {error, term()}. + {ok, [filefrag()]} | {error, term()} | no_return(). pread(Node, Transfer, Frag, Offset, Size) -> - erpc:call(Node, emqx_ft_storage_fs, pread_local, [Transfer, Frag, Offset, Size]). + erpc:call(Node, emqx_ft_storage_fs_proxy, pread_local, [Transfer, Frag, Offset, Size]). -spec ready_transfers([node()]) -> {ok, [{emqx_ft_storage:ready_transfer_id(), emqx_ft_storage:ready_transfer_info()}]} | {error, term()}. ready_transfers(Nodes) -> - erpc:multicall(Nodes, emqx_ft_storage_fs, ready_transfers_local, []). + erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, ready_transfers_local, []). -spec get_ready_transfer(node(), emqx_ft_storage:ready_transfer_id()) -> {ok, emqx_ft_storage:ready_transfer_data()} - | {error, term()}. + | {error, term()} + | no_return(). get_ready_transfer(Node, ReadyTransferId) -> - erpc:call(Node, emqx_ft_storage_fs, get_ready_transfer_local, [ReadyTransferId]). + erpc:call(Node, emqx_ft_storage_fs_proxy, get_ready_transfer_local, [ReadyTransferId]). From 0aefd4a8c7641635c852303fe923c249f134e699 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 6 Feb 2023 23:43:53 +0200 Subject: [PATCH 019/156] feat(ft): add streaming of file content when downloading --- apps/emqx_ft/src/emqx_ft_api.erl | 10 +- apps/emqx_ft/src/emqx_ft_storage.erl | 17 +-- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 37 ++++-- apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl | 6 +- .../emqx_ft/src/emqx_ft_storage_fs_reader.erl | 125 ++++++++++++++++++ .../src/emqx_ft_storage_fs_reader_sup.erl | 44 ++++++ apps/emqx_ft/src/emqx_ft_sup.erl | 11 +- .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 8 +- .../emqx_ft_storage_fs_reader_proto_v1.erl | 33 +++++ mix.exs | 2 +- rebar.config | 2 +- 11 files changed, 259 insertions(+), 36 deletions(-) create mode 100644 apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl create mode 100644 apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl create mode 100644 apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index b2a822f36..ddc6e761a 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -107,10 +107,16 @@ schema("/file_transfer/file") -> '/file_transfer/file'(get, #{query_string := Query}) -> case emqx_ft_storage:get_ready_transfer(Query) of {ok, FileData} -> - {200, #{<<"content-type">> => <<"application/data">>}, FileData}; + {200, + #{ + <<"content-type">> => <<"application/data">>, + <<"content-disposition">> => <<"attachment">> + }, + FileData}; {error, enoent} -> {404, error_msg('NOT_FOUND', <<"Not found">>)}; - {error, _} -> + {error, Error} -> + ?SLOG(warning, #{msg => "get_ready_transfer_fail", error => Error}), {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} end. diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index e8f1d9c47..0dd9d7989 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -22,8 +22,6 @@ store_segment/2, assemble/2, - parse_id/1, - ready_transfers/0, get_ready_transfer/1, @@ -39,7 +37,7 @@ -type ready_transfer_id() :: term(). -type ready_transfer_info() :: map(). --type ready_transfer_data() :: binary(). +-type ready_transfer_data() :: binary() | qlc:query_handle(). %%-------------------------------------------------------------------- %% Behaviour @@ -88,19 +86,6 @@ get_ready_transfer(ReadyTransferId) -> Mod = mod(), Mod:get_ready_transfer(storage(), ReadyTransferId). --spec parse_id(map()) -> {ok, ready_transfer_id()} | {error, term()}. -parse_id(#{ - <<"type">> := local, <<"node">> := NodeBin, <<"clientid">> := ClientId, <<"id">> := Id -}) -> - case emqx_misc:safe_to_existing_atom(NodeBin) of - {ok, Node} -> - {ok, {local, Node, ClientId, Id}}; - {error, _} -> - {error, {invalid_node, NodeBin}} - end; -parse_id(#{}) -> - {error, invalid_file_id}. - -spec with_storage_type(atom(), atom(), list(term())) -> any(). with_storage_type(Type, Fun, Args) -> Storage = storage(), diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 088c316ae..a120a4067 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -29,7 +29,7 @@ -export([transfers/1]). -export([ready_transfers_local/1]). --export([get_ready_transfer_local/2]). +-export([get_ready_transfer_local/3]). -export([ready_transfers/1]). -export([get_ready_transfer/2]). @@ -175,22 +175,43 @@ get_ready_transfer(_Storage, ReadyTransferId) -> case parse_ready_transfer_id(ReadyTransferId) of {ok, {Node, Transfer}} -> try - emqx_ft_storage_fs_proto_v1:get_ready_transfer(Node, Transfer) + case emqx_ft_storage_fs_proto_v1:get_ready_transfer(Node, self(), Transfer) of + {ok, ReaderPid} -> + {ok, emqx_ft_storage_fs_reader:table(ReaderPid)}; + {error, _} = Error -> + Error + end catch - error:Error -> - {error, Error}; - C:Error -> - {error, {C, Error}} + error:Exc:Stacktrace -> + ?SLOG(warning, #{ + msg => "get_ready_transfer_error", + node => Node, + transfer => Transfer, + exception => Exc, + stacktrace => Stacktrace + }), + {error, Exc}; + C:Exc:Stacktrace -> + ?SLOG(warning, #{ + msg => "get_ready_transfer_fail", + class => C, + node => Node, + transfer => Transfer, + exception => Exc, + stacktrace => Stacktrace + }), + {error, {C, Exc}} end; {error, _} = Error -> Error end. -get_ready_transfer_local(Storage, Transfer) -> +get_ready_transfer_local(Storage, CallerPid, Transfer) -> Dirname = mk_filedir(Storage, Transfer, get_subdirs_for(result)), case file:list_dir(Dirname) of {ok, [Filename | _]} -> - file:read_file(filename:join([Dirname, Filename])); + FullFilename = filename:join([Dirname, Filename]), + emqx_ft_storage_fs_reader:start_supervised(CallerPid, FullFilename); {error, _} = Error -> Error end. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl index 0c30f5567..7e19dd322 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl @@ -23,7 +23,7 @@ -export([ list_local/2, pread_local/4, - get_ready_transfer_local/1, + get_ready_transfer_local/2, ready_transfers_local/0 ]). @@ -33,8 +33,8 @@ list_local(Transfer, What) -> pread_local(Transfer, Frag, Offset, Size) -> emqx_ft_storage:with_storage_type(local, pread, [Transfer, Frag, Offset, Size]). -get_ready_transfer_local(Transfer) -> - emqx_ft_storage:with_storage_type(local, get_ready_transfer_local, [Transfer]). +get_ready_transfer_local(CallerPid, Transfer) -> + emqx_ft_storage:with_storage_type(local, get_ready_transfer_local, [CallerPid, Transfer]). ready_transfers_local() -> emqx_ft_storage:with_storage_type(local, ready_transfers_local, []). diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl new file mode 100644 index 000000000..a6307765e --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl @@ -0,0 +1,125 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_reader). + +-behaviour(gen_server). + +-include_lib("emqx/include/logger.hrl"). + +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). + +-export([ + start_link/2, + start_link/3, + start_supervised/2, + start_supervised/3, + read/1 +]). + +-export([ + table/1 +]). + +-define(DEFAULT_CHUNK_SIZE, 1024). + +table(ReaderPid) -> + NextFun = fun NextFun(Pid) -> + try + case emqx_ft_storage_fs_reader_proto_v1:read(node(Pid), Pid) of + eof -> + []; + {ok, Data} -> + [Data | fun() -> NextFun(Pid) end]; + {error, Reason} -> + ?SLOG(warning, #{msg => "file_read_error", reason => Reason}), + [] + end + catch + Class:Error:Stacktrace -> + ?SLOG(warning, #{ + msg => "file_read_error", + class => Class, + reason => Error, + stacktrace => Stacktrace + }), + [] + end + end, + qlc:table(fun() -> NextFun(ReaderPid) end, []). + +start_link(CallerPid, Filename) -> + start_link(CallerPid, Filename, ?DEFAULT_CHUNK_SIZE). + +start_link(CallerPid, Filename, ChunkSize) -> + gen_server:start_link(?MODULE, [CallerPid, Filename, ChunkSize], []). + +start_supervised(CallerPid, Filename) -> + start_supervised(CallerPid, Filename, ?DEFAULT_CHUNK_SIZE). + +start_supervised(CallerPid, Filename, ChunkSize) -> + emqx_ft_storage_fs_reader_sup:start_child(CallerPid, Filename, ChunkSize). + +read(Pid) -> + gen_server:call(Pid, read). + +init([CallerPid, Filename, ChunkSize]) -> + true = link(CallerPid), + case file:open(Filename, [read, raw, binary]) of + {ok, File} -> + {ok, #{ + filename => Filename, + file => File, + chunk_size => ChunkSize + }}; + {error, Reason} -> + {stop, Reason} + end. + +handle_call(read, _From, #{file := File, chunk_size := ChunkSize} = State) -> + case file:read(File, ChunkSize) of + {ok, Data} -> + ?SLOG(warning, #{msg => "read", bytes => byte_size(Data)}), + {reply, {ok, Data}, State}; + eof -> + ?SLOG(warning, #{msg => "read", eof => true}), + {stop, normal, eof, State}; + {error, Reason} = Error -> + {stop, Reason, Error, State} + end; +handle_call(Msg, _From, State) -> + {stop, {bad_call, Msg}, {bad_call, Msg}, State}. + +handle_info(Msg, State) -> + ?SLOG(warning, #{msg => "unexpected_message", info_msg => Msg}), + {noreply, State}. + +handle_cast(Msg, State) -> + ?SLOG(warning, #{msg => "unexpected_message", case_msg => Msg}), + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl new file mode 100644 index 000000000..7435d7e1c --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl @@ -0,0 +1,44 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_reader_sup). + +-behaviour(supervisor). + +-export([ + init/1, + start_link/0, + start_child/3 +]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +start_child(CallerPid, Filename, ChunkSize) -> + Childspec = #{ + id => {CallerPid, Filename}, + start => {emqx_ft_storage_fs_reader, start_link, [CallerPid, Filename, ChunkSize]}, + restart => temporary + }, + supervisor:start_child(?MODULE, Childspec). + +init(_) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 1000 + }, + {ok, {SupFlags, []}}. diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index b4ce52edb..5c2025860 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -43,6 +43,15 @@ init([]) -> modules => [emqx_ft_assembler_sup] }, + FileReaderSup = #{ + id => emqx_ft_storage_fs_reader_sup, + start => {emqx_ft_storage_fs_reader_sup, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [emqx_ft_storage_fs_reader_sup] + }, + Responder = #{ id => emqx_ft_responder, start => {emqx_ft_responder, start_link, []}, @@ -52,5 +61,5 @@ init([]) -> modules => [emqx_ft_responder] }, - ChildSpecs = [Responder, AssemblerSup], + ChildSpecs = [Responder, AssemblerSup, FileReaderSup], {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index 2e3dc8632..082df9ac0 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -24,7 +24,7 @@ -export([multilist/3]). -export([pread/5]). -export([ready_transfers/1]). --export([get_ready_transfer/2]). +-export([get_ready_transfer/3]). -type offset() :: emqx_ft:offset(). -type transfer() :: emqx_ft:transfer(). @@ -56,9 +56,9 @@ pread(Node, Transfer, Frag, Offset, Size) -> ready_transfers(Nodes) -> erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, ready_transfers_local, []). --spec get_ready_transfer(node(), emqx_ft_storage:ready_transfer_id()) -> +-spec get_ready_transfer(node(), pid(), emqx_ft_storage:ready_transfer_id()) -> {ok, emqx_ft_storage:ready_transfer_data()} | {error, term()} | no_return(). -get_ready_transfer(Node, ReadyTransferId) -> - erpc:call(Node, emqx_ft_storage_fs_proxy, get_ready_transfer_local, [ReadyTransferId]). +get_ready_transfer(Node, CallerPid, ReadyTransferId) -> + erpc:call(Node, emqx_ft_storage_fs_proxy, get_ready_transfer_local, [CallerPid, ReadyTransferId]). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl new file mode 100644 index 000000000..1bbb05471 --- /dev/null +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl @@ -0,0 +1,33 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_reader_proto_v1). + +-behaviour(emqx_bpapi). + +-export([introduced_in/0]). + +-export([read/2]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.17". + +-spec read(node(), pid()) -> + {ok, binary()} | eof | {error, term()} | no_return(). +read(Node, Pid) -> + erpc:call(Node, emqx_ft_storage_fs_reader, read, [Pid]). diff --git a/mix.exs b/mix.exs index 92024f48d..9a8c22e6b 100644 --- a/mix.exs +++ b/mix.exs @@ -58,7 +58,7 @@ defmodule EMQXUmbrella.MixProject do {:ekka, github: "emqx/ekka", tag: "0.14.6", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true}, - {:minirest, github: "emqx/minirest", tag: "1.3.8", override: true}, + {:minirest, github: "emqx/minirest", tag: "1.3.9", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, diff --git a/rebar.config b/rebar.config index f084f9827..0cc143e71 100644 --- a/rebar.config +++ b/rebar.config @@ -65,9 +65,9 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.8"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.9"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.5"}}} From b7d0bad97026381997e4bbf116c245674c6cfb64 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 7 Feb 2023 17:49:14 +0200 Subject: [PATCH 020/156] feat(ft): improve remote reader --- apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl | 17 +++++++++++++---- .../emqx_ft_storage_fs_reader_proto_v1.erl | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl index a6307765e..9c4aa5e0c 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl @@ -53,6 +53,9 @@ table(ReaderPid) -> [Data | fun() -> NextFun(Pid) end]; {error, Reason} -> ?SLOG(warning, #{msg => "file_read_error", reason => Reason}), + []; + {BadRPC, Reason} when BadRPC =:= badrpc orelse BadRPC =:= badtcp -> + ?SLOG(warning, #{msg => "file_read_rpc_error", kind => BadRPC, reason => Reason}), [] end catch @@ -84,13 +87,15 @@ read(Pid) -> gen_server:call(Pid, read). init([CallerPid, Filename, ChunkSize]) -> - true = link(CallerPid), + MRef = erlang:monitor(process, CallerPid), case file:open(Filename, [read, raw, binary]) of {ok, File} -> {ok, #{ filename => Filename, file => File, - chunk_size => ChunkSize + chunk_size => ChunkSize, + caller_pid => CallerPid, + mref => MRef }}; {error, Reason} -> {stop, Reason} @@ -99,10 +104,10 @@ init([CallerPid, Filename, ChunkSize]) -> handle_call(read, _From, #{file := File, chunk_size := ChunkSize} = State) -> case file:read(File, ChunkSize) of {ok, Data} -> - ?SLOG(warning, #{msg => "read", bytes => byte_size(Data)}), + ?SLOG(debug, #{msg => "read", bytes => byte_size(Data)}), {reply, {ok, Data}, State}; eof -> - ?SLOG(warning, #{msg => "read", eof => true}), + ?SLOG(debug, #{msg => "read", eof => true}), {stop, normal, eof, State}; {error, Reason} = Error -> {stop, Reason, Error, State} @@ -110,6 +115,10 @@ handle_call(read, _From, #{file := File, chunk_size := ChunkSize} = State) -> handle_call(Msg, _From, State) -> {stop, {bad_call, Msg}, {bad_call, Msg}, State}. +handle_info( + {'DOWN', MRef, process, CallerPid, _Reason}, #{mref := MRef, caller_pid := CallerPid} = State +) -> + {stop, {caller_down, CallerPid}, State}; handle_info(Msg, State) -> ?SLOG(warning, #{msg => "unexpected_message", info_msg => Msg}), {noreply, State}. diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl index 1bbb05471..982b9ca57 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl @@ -30,4 +30,4 @@ introduced_in() -> -spec read(node(), pid()) -> {ok, binary()} | eof | {error, term()} | no_return(). read(Node, Pid) -> - erpc:call(Node, emqx_ft_storage_fs_reader, read, [Pid]). + emqx_rpc:call(Node, emqx_ft_storage_fs_reader, read, [Pid]). From 8038a3fd4ac5b0f0b9726869f92d1698be6a703a Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 7 Feb 2023 23:29:22 +0200 Subject: [PATCH 021/156] feat(ft): add tests for remote reader --- apps/emqx_ft/etc/emqx_ft.conf | 5 + .../emqx_ft/src/emqx_ft_storage_fs_reader.erl | 108 +++++++------ .../src/emqx_ft_storage_fs_reader_sup.erl | 6 +- .../emqx_ft_storage_fs_reader_proto_v1.erl | 10 +- .../test/emqx_ft_storage_fs_reader_SUITE.erl | 153 ++++++++++++++++++ 5 files changed, 225 insertions(+), 57 deletions(-) create mode 100644 apps/emqx_ft/etc/emqx_ft.conf create mode 100644 apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl diff --git a/apps/emqx_ft/etc/emqx_ft.conf b/apps/emqx_ft/etc/emqx_ft.conf new file mode 100644 index 000000000..250dca6a9 --- /dev/null +++ b/apps/emqx_ft/etc/emqx_ft.conf @@ -0,0 +1,5 @@ +file_transfer { + storage { + type = local + } +} diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl index 9c4aa5e0c..782959e19 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl @@ -19,7 +19,18 @@ -behaviour(gen_server). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). +%% API +-export([ + start_link/2, + start_supervised/2, + table/1, + table/2, + read/2 +]). + +%% gen_server callbacks -export([ init/1, handle_call/3, @@ -29,71 +40,68 @@ code_change/3 ]). --export([ - start_link/2, - start_link/3, - start_supervised/2, - start_supervised/3, - read/1 -]). - --export([ - table/1 -]). - -define(DEFAULT_CHUNK_SIZE, 1024). +-define(IS_FILENAME(Filename), (is_list(Filename) or is_binary(Filename))). -table(ReaderPid) -> +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec table(pid()) -> qlc:query_handle(). +table(ReaderPid) when is_pid(ReaderPid) -> + table(ReaderPid, ?DEFAULT_CHUNK_SIZE). + +-spec table(pid(), pos_integer()) -> qlc:query_handle(). +table(ReaderPid, Bytes) when is_pid(ReaderPid) andalso is_integer(Bytes) andalso Bytes > 0 -> NextFun = fun NextFun(Pid) -> - try - case emqx_ft_storage_fs_reader_proto_v1:read(node(Pid), Pid) of - eof -> - []; - {ok, Data} -> - [Data | fun() -> NextFun(Pid) end]; - {error, Reason} -> - ?SLOG(warning, #{msg => "file_read_error", reason => Reason}), - []; - {BadRPC, Reason} when BadRPC =:= badrpc orelse BadRPC =:= badtcp -> - ?SLOG(warning, #{msg => "file_read_rpc_error", kind => BadRPC, reason => Reason}), - [] - end - catch - Class:Error:Stacktrace -> - ?SLOG(warning, #{ - msg => "file_read_error", - class => Class, - reason => Error, - stacktrace => Stacktrace - }), + case emqx_ft_storage_fs_reader_proto_v1:read(node(Pid), Pid, Bytes) of + eof -> + []; + {ok, Data} -> + [Data | fun() -> NextFun(Pid) end]; + {error, Reason} -> + ?SLOG(warning, #{msg => "file_read_error", reason => Reason}), + []; + {BadRPC, Reason} when BadRPC =:= badrpc orelse BadRPC =:= badtcp -> + ?SLOG(warning, #{msg => "file_read_rpc_error", kind => BadRPC, reason => Reason}), [] end end, qlc:table(fun() -> NextFun(ReaderPid) end, []). -start_link(CallerPid, Filename) -> - start_link(CallerPid, Filename, ?DEFAULT_CHUNK_SIZE). +-spec start_link(pid(), filename:filename()) -> startlink_ret(). +start_link(CallerPid, Filename) when + is_pid(CallerPid) andalso + ?IS_FILENAME(Filename) +-> + gen_server:start_link(?MODULE, [CallerPid, Filename], []). -start_link(CallerPid, Filename, ChunkSize) -> - gen_server:start_link(?MODULE, [CallerPid, Filename, ChunkSize], []). +-spec start_supervised(pid(), filename:filename()) -> startlink_ret(). +start_supervised(CallerPid, Filename) when + is_pid(CallerPid) andalso + ?IS_FILENAME(Filename) +-> + emqx_ft_storage_fs_reader_sup:start_child(CallerPid, Filename). -start_supervised(CallerPid, Filename) -> - start_supervised(CallerPid, Filename, ?DEFAULT_CHUNK_SIZE). +-spec read(pid(), pos_integer()) -> {ok, binary()} | eof | {error, term()}. +read(Pid, Bytes) when + is_pid(Pid) andalso + is_integer(Bytes) andalso + Bytes > 0 +-> + gen_server:call(Pid, {read, Bytes}). -start_supervised(CallerPid, Filename, ChunkSize) -> - emqx_ft_storage_fs_reader_sup:start_child(CallerPid, Filename, ChunkSize). +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- -read(Pid) -> - gen_server:call(Pid, read). - -init([CallerPid, Filename, ChunkSize]) -> +init([CallerPid, Filename]) -> MRef = erlang:monitor(process, CallerPid), case file:open(Filename, [read, raw, binary]) of {ok, File} -> {ok, #{ filename => Filename, file => File, - chunk_size => ChunkSize, caller_pid => CallerPid, mref => MRef }}; @@ -101,8 +109,8 @@ init([CallerPid, Filename, ChunkSize]) -> {stop, Reason} end. -handle_call(read, _From, #{file := File, chunk_size := ChunkSize} = State) -> - case file:read(File, ChunkSize) of +handle_call({read, Bytes}, _From, #{file := File} = State) -> + case file:read(File, Bytes) of {ok, Data} -> ?SLOG(debug, #{msg => "read", bytes => byte_size(Data)}), {reply, {ok, Data}, State}; @@ -113,7 +121,7 @@ handle_call(read, _From, #{file := File, chunk_size := ChunkSize} = State) -> {stop, Reason, Error, State} end; handle_call(Msg, _From, State) -> - {stop, {bad_call, Msg}, {bad_call, Msg}, State}. + {reply, {error, {bad_call, Msg}}, State}. handle_info( {'DOWN', MRef, process, CallerPid, _Reason}, #{mref := MRef, caller_pid := CallerPid} = State diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl index 7435d7e1c..934e2888c 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl @@ -21,16 +21,16 @@ -export([ init/1, start_link/0, - start_child/3 + start_child/2 ]). start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). -start_child(CallerPid, Filename, ChunkSize) -> +start_child(CallerPid, Filename) -> Childspec = #{ id => {CallerPid, Filename}, - start => {emqx_ft_storage_fs_reader, start_link, [CallerPid, Filename, ChunkSize]}, + start => {emqx_ft_storage_fs_reader, start_link, [CallerPid, Filename]}, restart => temporary }, supervisor:start_child(?MODULE, Childspec). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl index 982b9ca57..db5e35f94 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl @@ -20,14 +20,16 @@ -export([introduced_in/0]). --export([read/2]). +-export([read/3]). -include_lib("emqx/include/bpapi.hrl"). introduced_in() -> "5.0.17". --spec read(node(), pid()) -> +-spec read(node(), pid(), pos_integer()) -> {ok, binary()} | eof | {error, term()} | no_return(). -read(Node, Pid) -> - emqx_rpc:call(Node, emqx_ft_storage_fs_reader, read, [Pid]). +read(Node, Pid, Bytes) when + is_atom(Node) andalso is_pid(Pid) andalso is_integer(Bytes) andalso Bytes > 0 +-> + emqx_rpc:call(Node, emqx_ft_storage_fs_reader, read, [Pid, Bytes]). diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl new file mode 100644 index 000000000..e979d06fc --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl @@ -0,0 +1,153 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_reader_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_ft]), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft]), + ok. + +init_per_testcase(_Case, Config) -> + file:make_dir(?config(data_dir, Config)), + Data = <<"hello world">>, + Path = expand_path(Config, "test_file"), + ok = mk_test_file(Path, Data), + [{path, Path} | Config]. + +end_per_testcase(_Case, _Config) -> + ok. + +t_successful_read(Config) -> + Path = ?config(path, Config), + + {ok, ReaderPid} = emqx_ft_storage_fs_reader:start_link(self(), Path), + ?assertEqual( + {ok, <<"hello ">>}, + emqx_ft_storage_fs_reader:read(ReaderPid, 6) + ), + ?assertEqual( + {ok, <<"world">>}, + emqx_ft_storage_fs_reader:read(ReaderPid, 6) + ), + ?assertEqual( + eof, + emqx_ft_storage_fs_reader:read(ReaderPid, 6) + ), + ?assertNot(is_process_alive(ReaderPid)). + +t_caller_dead(Config) -> + erlang:process_flag(trap_exit, true), + + Path = ?config(path, Config), + + CallerPid = spawn_link( + fun() -> + receive + stop -> ok + end + end + ), + {ok, ReaderPid} = emqx_ft_storage_fs_reader:start_link(CallerPid, Path), + _ = erlang:monitor(process, ReaderPid), + ?assertEqual( + {ok, <<"hello ">>}, + emqx_ft_storage_fs_reader:read(ReaderPid, 6) + ), + CallerPid ! stop, + receive + {'DOWN', _, process, ReaderPid, _} -> ok + after 1000 -> + ct:fail("Reader process did not die") + end. + +t_tables(Config) -> + Path = ?config(path, Config), + + {ok, ReaderPid0} = emqx_ft_storage_fs_reader:start_link(self(), Path), + + ReaderQH0 = emqx_ft_storage_fs_reader:table(ReaderPid0, 6), + ?assertEqual( + [<<"hello ">>, <<"world">>], + qlc:eval(ReaderQH0) + ), + + {ok, ReaderPid1} = emqx_ft_storage_fs_reader:start_link(self(), Path), + + ReaderQH1 = emqx_ft_storage_fs_reader:table(ReaderPid1), + ?assertEqual( + [<<"hello world">>], + qlc:eval(ReaderQH1) + ). + +t_bad_messages(Config) -> + Path = ?config(path, Config), + + {ok, ReaderPid} = emqx_ft_storage_fs_reader:start_link(self(), Path), + + ReaderPid ! {bad, message}, + gen_server:cast(ReaderPid, {bad, message}), + + ?assertEqual( + {error, {bad_call, {bad, message}}}, + gen_server:call(ReaderPid, {bad, message}) + ). + +t_nonexistent_file(_Config) -> + ?assertEqual( + {error, enoent}, + emqx_ft_storage_fs_reader:start_link(self(), "/a/b/c/bar") + ). + +t_start_supervised(Config) -> + Path = ?config(path, Config), + + {ok, ReaderPid} = emqx_ft_storage_fs_reader:start_supervised(self(), Path), + ?assertEqual( + {ok, <<"hello ">>}, + emqx_ft_storage_fs_reader:read(ReaderPid, 6) + ). + +t_rpc_error(_Config) -> + ReaderQH = emqx_ft_storage_fs_reader:table(fake_remote_pid('dummy@127.0.0.1'), 6), + ?assertEqual( + [], + qlc:eval(ReaderQH) + ). + +mk_test_file(Path, Data) -> + ok = file:write_file(Path, Data). + +expand_path(Config, Filename) -> + filename:join([?config(data_dir, Config), Filename]). + +%% This is a hack to create a pid that is not registered on the local node. +%% https://www.erlang.org/doc/apps/erts/erl_ext_dist.html#new_pid_ext +fake_remote_pid(Node) -> + <<131, NodeAtom/binary>> = term_to_binary(Node), + PidBin = <<131, 88, NodeAtom/binary, 1:32/big, 1:32/big, 1:32/big>>, + binary_to_term(PidBin). From 836ec213c9c268032141075e8446e578326c78e0 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 8 Feb 2023 15:27:13 +0200 Subject: [PATCH 022/156] feat(ft): add responder tests --- apps/emqx/include/asserts.hrl | 38 +++++++ apps/emqx_ft/src/emqx_ft_responder.erl | 32 +++++- apps/emqx_ft/test/emqx_ft_responder_SUITE.erl | 99 +++++++++++++++++++ 3 files changed, 164 insertions(+), 5 deletions(-) create mode 100644 apps/emqx/include/asserts.hrl create mode 100644 apps/emqx_ft/test/emqx_ft_responder_SUITE.erl diff --git a/apps/emqx/include/asserts.hrl b/apps/emqx/include/asserts.hrl new file mode 100644 index 000000000..0e9f9477a --- /dev/null +++ b/apps/emqx/include/asserts.hrl @@ -0,0 +1,38 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-define(assertWaitEvent(Code, EventMatch, Timeout), + ?check_trace( + ?wait_async_action( + Code, + EventMatch, + Timeout + ), + fun(Trace) -> + ?assert( + lists:any( + fun + (EventMatch) -> true; + (_) -> false + end, + Trace + ) + ) + end + ) +). diff --git a/apps/emqx_ft/src/emqx_ft_responder.erl b/apps/emqx_ft/src/emqx_ft_responder.erl index f58569a27..8417053f6 100644 --- a/apps/emqx_ft/src/emqx_ft_responder.erl +++ b/apps/emqx_ft/src/emqx_ft_responder.erl @@ -21,6 +21,8 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/types.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + -export([start_link/0]). -export([ @@ -87,10 +89,13 @@ handle_call({unregister, Key}, _From, State) -> _ = erlang:cancel_timer(TRef), true = ets:delete(?TAB, Key), {reply, ok, State} - end. + end; +handle_call(Msg, _From, State) -> + ?SLOG(warning, #{msg => "unknown_call", call_msg => Msg}), + {reply, {error, unknown_call}, State}. handle_cast(Msg, State) -> - ?SLOG(warning, #{msg => "unknown cast", cast_msg => Msg}), + ?SLOG(warning, #{msg => "unknown_cast", cast_msg => Msg}), {noreply, State}. handle_info({timeout, TRef, {timeout, Key}}, State) -> @@ -100,12 +105,12 @@ handle_info({timeout, TRef, {timeout, Key}}, State) -> [{_, Action, TRef}] -> _ = erlang:cancel_timer(TRef), true = ets:delete(?TAB, Key), - %% TODO: safe apply - _ = Action(Key), + ok = safe_apply(Action, [Key]), + ?tp(ft_timeout_action_applied, #{key => Key}), {noreply, State} end; handle_info(Msg, State) -> - ?SLOG(warning, #{msg => "unknown message", info_msg => Msg}), + ?SLOG(warning, #{msg => "unknown_message", info_msg => Msg}), {noreply, State}. code_change(_OldVsn, State, _Extra) -> @@ -113,3 +118,20 @@ code_change(_OldVsn, State, _Extra) -> terminate(_Reason, _State) -> ok. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +safe_apply(Fun, Args) -> + try apply(Fun, Args) of + _ -> ok + catch + Class:Reason:Stacktrace -> + ?SLOG(error, #{ + msg => "safe_apply_failed", + class => Class, + reason => Reason, + stacktrace => Stacktrace + }) + end. diff --git a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl new file mode 100644 index 000000000..c08986120 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl @@ -0,0 +1,99 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_responder_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("emqx/include/asserts.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_ft]), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft]), + ok. + +init_per_testcase(_Case, Config) -> + Config. + +end_per_testcase(_Case, _Config) -> + ok. + +t_register_unregister(_Config) -> + Key = <<"test">>, + DefaultAction = fun(_) -> ok end, + ?assertEqual( + ok, + emqx_ft_responder:register(Key, DefaultAction, 1000) + ), + ?assertEqual( + {error, already_registered}, + emqx_ft_responder:register(Key, DefaultAction, 1000) + ), + ?assertEqual( + ok, + emqx_ft_responder:unregister(Key) + ), + ?assertEqual( + {error, not_found}, + emqx_ft_responder:unregister(Key) + ). + +t_timeout(_Config) -> + Key = <<"test">>, + Self = self(), + DefaultAction = fun(K) -> Self ! {timeout, K} end, + ok = emqx_ft_responder:register(Key, DefaultAction, 20), + receive + {timeout, Key} -> + ok + after 100 -> + ct:fail("emqx_ft_responder not called") + end, + ?assertEqual( + {error, not_found}, + emqx_ft_responder:unregister(Key) + ). + +t_action_exception(_Config) -> + Key = <<"test">>, + DefaultAction = fun(K) -> error({oops, K}) end, + + ?assertWaitEvent( + emqx_ft_responder:register(Key, DefaultAction, 10), + #{?snk_kind := ft_timeout_action_applied, key := <<"test">>}, + 1000 + ), + ?assertEqual( + {error, not_found}, + emqx_ft_responder:unregister(Key) + ). + +t_unknown_msgs(_Config) -> + Pid = whereis(emqx_ft_responder), + Pid ! {unknown_msg, <<"test">>}, + ok = gen_server:cast(Pid, {unknown_msg, <<"test">>}), + ?assertEqual( + {error, unknown_call}, + gen_server:call(Pid, {unknown_call, <<"test">>}) + ). From 9e4a37a39888bd71928436c1673d60a1e03599d9 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 8 Feb 2023 20:37:56 +0200 Subject: [PATCH 023/156] fix(ft): fix typespecs --- apps/emqx/src/proto/emqx_cm_proto_v2.erl | 2 +- apps/emqx_ft/src/emqx_ft_assembler.erl | 16 +++++++++++----- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 16 ++++++++++++---- apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl | 5 +---- .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 8 ++++++-- 5 files changed, 31 insertions(+), 16 deletions(-) diff --git a/apps/emqx/src/proto/emqx_cm_proto_v2.erl b/apps/emqx/src/proto/emqx_cm_proto_v2.erl index 2981dbd40..4208df97f 100644 --- a/apps/emqx/src/proto/emqx_cm_proto_v2.erl +++ b/apps/emqx/src/proto/emqx_cm_proto_v2.erl @@ -78,7 +78,7 @@ takeover_finish(ConnMod, ChanPid) -> erpc:call( node(ChanPid), emqx_cm, - takeover_session_finish, + takeover_finish, [ConnMod, ChanPid], ?T_TAKEOVER * 2 ). diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index ef4daf000..623e11714 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -34,7 +34,7 @@ storage :: _Storage, transfer :: emqx_ft:transfer(), assembly :: _TODO, - file :: io:device(), + file :: {file:filename(), io:device(), term()} | undefined, hash, callback :: fun((ok | {error, term()}) -> any()) }). @@ -120,10 +120,16 @@ handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % TODO: pipelining case pread(Node, Segment, St) of {ok, Content} -> - {ok, NHandle} = emqx_ft_storage_fs:write(St#st.file, Content), - {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])} - % {error, _} -> - % ... + case emqx_ft_storage_fs:write(St#st.file, Content) of + {ok, NHandle} -> + {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])}; + %% TODO: better error handling + {error, Error} -> + error(Error) + end; + {error, Error} -> + %% TODO: better error handling + error(Error) end; handle_event(internal, _, {assemble, []}, St = #st{}) -> {next_state, complete, St, ?internal([])}; diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index a120a4067..a6559d470 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -85,7 +85,7 @@ %% Atomic operation. -spec store_filemeta(storage(), transfer(), filemeta()) -> % Quota? Some lower level errors? - {ok, emqx_ft_storage:ctx()} | {error, conflict} | {error, _TODO}. + ok | {error, conflict} | {error, _TODO}. store_filemeta(Storage, Transfer, Meta) -> % TODO safeguard against bad clientids / fileids. Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], ?MANIFEST), @@ -226,8 +226,16 @@ ready_transfers(_Storage) -> end, Results ), - ?SLOG(warning, #{msg => "ready_transfers", failures => BadResults}), - {ok, [File || {ok, Files} <- GoodResults, File <- Files]}. + case {GoodResults, BadResults} of + {[], _} -> + ?SLOG(warning, #{msg => "ready_transfers", failures => BadResults}), + {error, no_nodes}; + {_, []} -> + {ok, [File || {ok, Files} <- GoodResults, File <- Files]}; + {_, _} -> + ?SLOG(warning, #{msg => "ready_transfers", failures => BadResults}), + {ok, [File || {ok, Files} <- GoodResults, File <- Files]} + end. ready_transfers_local(Storage) -> {ok, Transfers} = transfers(Storage), @@ -323,7 +331,7 @@ open_file(Storage, Transfer, Filemeta) -> end. -spec write(handle(), iodata()) -> - ok | {error, _TODO}. + {ok, handle()} | {error, _TODO}. write({Filepath, IoDevice, Ctx}, IoData) -> case file:write(IoDevice, IoData) of ok -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl index 782959e19..373b92753 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl @@ -58,12 +58,9 @@ table(ReaderPid, Bytes) when is_pid(ReaderPid) andalso is_integer(Bytes) andalso eof -> []; {ok, Data} -> - [Data | fun() -> NextFun(Pid) end]; + [Data] ++ fun() -> NextFun(Pid) end; {error, Reason} -> ?SLOG(warning, #{msg => "file_read_error", reason => Reason}), - []; - {BadRPC, Reason} when BadRPC =:= badrpc orelse BadRPC =:= badtcp -> - ?SLOG(warning, #{msg => "file_read_rpc_error", kind => BadRPC, reason => Reason}), [] end end, diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index 082df9ac0..992d62c48 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -51,8 +51,12 @@ pread(Node, Transfer, Frag, Offset, Size) -> erpc:call(Node, emqx_ft_storage_fs_proxy, pread_local, [Transfer, Frag, Offset, Size]). -spec ready_transfers([node()]) -> - {ok, [{emqx_ft_storage:ready_transfer_id(), emqx_ft_storage:ready_transfer_info()}]} - | {error, term()}. + [ + {ok, [{emqx_ft_storage:ready_transfer_id(), emqx_ft_storage:ready_transfer_info()}]} + | {error, term()} + | {exit, term()} + | {throw, term()} + ]. ready_transfers(Nodes) -> erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, ready_transfers_local, []). From bcfa22f3434d57e868060b9f126a1a11403d3f9a Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 8 Feb 2023 23:41:33 +0200 Subject: [PATCH 024/156] fix(ft): use correct supervison strategy for emqx_ft_sup --- apps/emqx_ft/src/emqx_ft_sup.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index 5c2025860..dfdde3a8a 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -29,7 +29,7 @@ start_link() -> init([]) -> SupFlags = #{ - strategy => one_for_all, + strategy => one_for_one, intensity => 100, period => 10 }, From eae3dc7b4b0c11fab106db0b69524542b311591d Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 10 Feb 2023 13:34:33 +0200 Subject: [PATCH 025/156] feat(ft): update BPAPI --- apps/emqx/priv/bpapi.versions | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 6190925d2..0e0957adf 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -7,11 +7,14 @@ {emqx_bridge,3}. {emqx_broker,1}. {emqx_cm,1}. +{emqx_cm,2}. {emqx_conf,1}. {emqx_conf,2}. {emqx_dashboard,1}. {emqx_delayed,1}. {emqx_exhook,1}. +{emqx_ft_storage_fs,1}. +{emqx_ft_storage_fs_reader,1}. {emqx_gateway_api_listeners,1}. {emqx_gateway_cm,1}. {emqx_gateway_http,1}. From c44fe92ef181aa81acc2193efed516552f5a2113 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 10 Feb 2023 14:57:10 +0200 Subject: [PATCH 026/156] feat(ft): add assembler tests --- apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 11 +++++ apps/emqx_ft/src/emqx_ft_assembler.erl | 35 +++++++++++---- apps/emqx_ft/src/emqx_ft_assembler_sup.erl | 2 +- apps/emqx_ft/src/emqx_ft_schema.erl | 5 +++ apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 43 +++++++++++++++++++ 5 files changed, 86 insertions(+), 10 deletions(-) diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf index 481ad8154..7e057fdf8 100644 --- a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -11,4 +11,15 @@ emqx_ft_schema { } } + local_storage_root { + desc { + en: "File system path to keep uploaded files and temporary data." + zh: "保存上传文件和临时数据的文件系统路径。" + } + label: { + en: "Local Storage Root" + zh: "本地存储根" + } + } + } diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 623e11714..38faebf03 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -16,6 +16,8 @@ -module(emqx_ft_assembler). +-include_lib("emqx/include/logger.hrl"). + -export([start_link/3]). -behaviour(gen_statem). @@ -73,7 +75,7 @@ handle_event(internal, _, list_local_fragments, St = #st{assembly = Asm}) -> complete -> {next_state, start_assembling, NSt, ?internal([])}; {incomplete, _} -> - Nodes = ekka:nodelist() -- [node()], + Nodes = mria_mnesia:running_nodes() -- [node()], {next_state, {list_remote_fragments, Nodes}, NSt, ?internal([])} % TODO: recovery? % {error, _} = Reason -> @@ -105,7 +107,7 @@ handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> {next_state, start_assembling, NSt, ?internal([])}; % TODO: retries / recovery? {incomplete, _} = Status -> - {stop, {error, Status}} + {next_state, {failure, {error, Status}}, NSt, ?internal([])} end; handle_event(internal, _, start_assembling, St = #st{assembly = Asm}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), @@ -124,21 +126,23 @@ handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> {ok, NHandle} -> {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])}; %% TODO: better error handling - {error, Error} -> - error(Error) + {error, _} = Error -> + {next_state, {failure, Error}, St, ?internal([])} end; - {error, Error} -> + {error, _} = Error -> %% TODO: better error handling - error(Error) + {next_state, {failure, Error}, St, ?internal([])} end; handle_event(internal, _, {assemble, []}, St = #st{}) -> {next_state, complete, St, ?internal([])}; handle_event(internal, _, complete, St = #st{assembly = Asm, file = Handle, callback = Callback}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), Result = emqx_ft_storage_fs:complete(St#st.storage, St#st.transfer, Filemeta, Handle), - %% TODO: safe apply - _ = Callback(Result), - {stop, shutdown}. + _ = safe_apply(Callback, Result), + {stop, shutdown}; +handle_event(internal, _, {failure, Error}, #st{callback = Callback}) -> + _ = safe_apply(Callback, Error), + {stop, Error}. % handle_continue(list_local, St = #st{storage = Storage, transfer = Transfer, assembly = Asm}) -> % % TODO: what we do with non-transients errors here (e.g. `eacces`)? @@ -170,3 +174,16 @@ pread(Node, Segment, St) -> segsize(#{fragment := {segment, Info}}) -> maps:get(size, Info). + +safe_apply(Callback, Result) -> + try apply(Callback, [Result]) of + _ -> ok + catch + Class:Reason:Stacktrace -> + ?SLOG(error, #{ + msg => "safe_apply_failed", + class => Class, + reason => Reason, + stacktrace => Stacktrace + }) + end. diff --git a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl index b60949a5e..17aa4d998 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl @@ -29,7 +29,7 @@ start_child(Storage, Transfer, Callback) -> Childspec = #{ id => {Storage, Transfer}, start => {emqx_ft_assembler, start_link, [Storage, Transfer, Callback]}, - restart => transient + restart => temporary }, supervisor:start_child(?MODULE, Childspec). diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 70acb8322..deb2cae6f 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -59,6 +59,11 @@ fields(local_storage) -> default => local, required => false, desc => ?DESC("local") + }}, + {root, #{ + type => binary(), + desc => ?DESC("local_storage_root"), + required => false }} ]. diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 336580584..dd3ffedad 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -28,6 +28,8 @@ all() -> [ t_assemble_empty_transfer, t_assemble_complete_local_transfer, + t_assemble_incomplete_transfer, + t_assemble_no_meta, % NOTE % It depends on the side effects of all previous testcases. @@ -155,6 +157,46 @@ t_assemble_complete_local_transfer(Config) -> AssemblyFilename ). +t_assemble_incomplete_transfer(Config) -> + Storage = storage(Config), + Transfer = {?CLIENTID2, ?config(file_id, Config)}, + Filename = "incomplete.pdf", + TransferSize = 10000 + rand:uniform(50000), + SegmentSize = 4096, + Gen = emqx_ft_content_gen:new({Transfer, TransferSize}, SegmentSize), + Hash = emqx_ft_content_gen:hash(Gen, crypto:hash_init(sha256)), + Meta = #{ + name => Filename, + checksum => {sha256, Hash}, + size => TransferSize, + expire_at => 42 + }, + ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta), + Self = self(), + {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun(Result) -> + Self ! {test_assembly_finished, Result} + end), + receive + {test_assembly_finished, Result} -> + ?assertMatch({error, _}, Result) + after 1000 -> + ct:fail("Assembler did not called callback") + end. + +t_assemble_no_meta(Config) -> + Storage = storage(Config), + Transfer = {?CLIENTID2, ?config(file_id, Config)}, + Self = self(), + {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun(Result) -> + Self ! {test_assembly_finished, Result} + end), + receive + {test_assembly_finished, Result} -> + ?assertMatch({error, _}, Result) + after 1000 -> + ct:fail("Assembler did not called callback") + end. + mk_assembly_filename(Config, {ClientID, FileID}, Filename) -> filename:join([?config(storage_root, Config), ClientID, FileID, result, Filename]). @@ -205,5 +247,6 @@ mk_fileid() -> storage(Config) -> #{ + type => local, root => ?config(storage_root, Config) }. From 1d48a97fd2b5ea32daf06745adfb06b21b1e73ff Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 13 Feb 2023 15:19:21 +0200 Subject: [PATCH 027/156] feat(ft): fix remote reader handling of gen_rpc errors --- apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl index 373b92753..4b1c4acb8 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl @@ -59,8 +59,8 @@ table(ReaderPid, Bytes) when is_pid(ReaderPid) andalso is_integer(Bytes) andalso []; {ok, Data} -> [Data] ++ fun() -> NextFun(Pid) end; - {error, Reason} -> - ?SLOG(warning, #{msg => "file_read_error", reason => Reason}), + {ErrorKind, Reason} when ErrorKind =:= badrpc; ErrorKind =:= error -> + ?SLOG(warning, #{msg => "file_read_error", kind => ErrorKind, reason => Reason}), [] end end, From 2e889f4ac76e93e26d7ebfc62a3060fa7c00995c Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 14 Feb 2023 01:18:15 +0200 Subject: [PATCH 028/156] feat(ft): add emqx_ft tests and fixes --- apps/emqx/test/emqx_common_test_helpers.erl | 10 + apps/emqx_ft/src/emqx_ft.erl | 100 +++++- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 12 +- .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 2 +- .../emqx_ft_storage_fs_reader_proto_v1.erl | 2 +- apps/emqx_ft/test/emqx_ft_SUITE.erl | 284 ++++++++++++++++++ apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_content_gen.erl | 2 +- apps/emqx_ft/test/emqx_ft_responder_SUITE.erl | 2 +- .../test/emqx_ft_storage_fs_reader_SUITE.erl | 2 +- 10 files changed, 398 insertions(+), 20 deletions(-) create mode 100644 apps/emqx_ft/test/emqx_ft_SUITE.erl diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 077ebe138..ecdf8f827 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -63,6 +63,7 @@ ]). -export([ + maybe_fix_gen_rpc/0, emqx_cluster/1, emqx_cluster/2, start_epmd/0, @@ -616,6 +617,15 @@ ensure_quic_listener(Name, UdpPort, ExtraSettings) -> listener_ports => [{Type :: tcp | ssl | ws | wss, inet:port_number()}] }. +-spec maybe_fix_gen_rpc() -> ok. +maybe_fix_gen_rpc() -> + %% When many tests run in an obscure order, it may occur that + %% `gen_rpc` started with its default settings before `emqx_conf`. + %% `gen_rpc` and `emqx_conf` have different default `port_discovery` modes, + %% so we reinitialize `gen_rpc` explicitly. + ok = application:stop(gen_rpc), + ok = application:start(gen_rpc). + -spec emqx_cluster(cluster_spec()) -> [{shortname(), node_opts()}]. emqx_cluster(Specs) -> emqx_cluster(Specs, #{}). diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index c4f1caed5..ed4edd33b 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -31,6 +31,10 @@ on_message_puback/4 ]). +-export([ + decode_filemeta/1 +]). + -export([on_assemble_timeout/1]). -export_type([ @@ -86,6 +90,27 @@ unhook() -> ok = emqx_hooks:del('message.publish', {?MODULE, on_message_publish}), ok = emqx_hooks:del('message.puback', {?MODULE, on_message_puback}). +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +decode_filemeta(Payload) when is_binary(Payload) -> + case emqx_json:safe_decode(Payload, [return_maps]) of + {ok, Map} -> + decode_filemeta(Map); + {error, Error} -> + {error, {invalid_filemeta_json, Error}} + end; +decode_filemeta(Map) when is_map(Map) -> + Schema = emqx_ft_schema:schema(filemeta), + try + Meta = hocon_tconf:check_plain(Schema, Map, #{atom_key => true, required => false}), + {ok, Meta} + catch + throw:Error -> + {error, {invalid_filemeta, Error}} + end. + %%-------------------------------------------------------------------- %% Hooks %%-------------------------------------------------------------------- @@ -113,6 +138,8 @@ on_message_puback(PacketId, #message{topic = Topic} = Msg, _PubRes, _RC) -> %% Handlers for transfer messages %%-------------------------------------------------------------------- +%% TODO Move to emqx_ft_mqtt? + on_file_command(PacketId, Msg, FileCommand) -> case string:split(FileCommand, <<"/">>, all) of [FileId, <<"init">>] -> @@ -123,10 +150,14 @@ on_file_command(PacketId, Msg, FileCommand) -> on_fin(PacketId, Msg, FileId, Checksum); [FileId, <<"abort">>] -> on_abort(Msg, FileId); - [FileId, Offset] -> - on_segment(Msg, FileId, Offset, undefined); - [FileId, Offset, Checksum] -> - on_segment(Msg, FileId, Offset, Checksum); + [FileId, OffsetBin] -> + validate([{offset, OffsetBin}], fun([Offset]) -> + on_segment(Msg, FileId, Offset, undefined) + end); + [FileId, OffsetBin, ChecksumBin] -> + validate([{offset, OffsetBin}, {checksum, ChecksumBin}], fun([Offset, Checksum]) -> + on_segment(Msg, FileId, Offset, Checksum) + end); _ -> ?RC_UNSPECIFIED_ERROR end. @@ -139,11 +170,21 @@ on_init(Msg, FileId) -> }), Payload = Msg#message.payload, % %% Add validations here - Meta = emqx_json:decode(Payload, [return_maps]), - case emqx_ft_storage:store_filemeta(transfer(Msg, FileId), Meta) of - ok -> - ?RC_SUCCESS; - {error, _Reason} -> + case decode_filemeta(Payload) of + {ok, Meta} -> + case emqx_ft_storage:store_filemeta(transfer(Msg, FileId), Meta) of + ok -> + ?RC_SUCCESS; + {error, _Reason} -> + ?RC_UNSPECIFIED_ERROR + end; + {error, Reason} -> + ?SLOG(error, #{ + msg => "on_init: invalid filemeta", + mqtt_msg => Msg, + file_id => FileId, + reason => Reason + }), ?RC_UNSPECIFIED_ERROR end. @@ -161,7 +202,7 @@ on_segment(Msg, FileId, Offset, Checksum) -> }), %% TODO: handle checksum Payload = Msg#message.payload, - Segment = {binary_to_integer(Offset), Payload}, + Segment = {Offset, Payload}, %% Add offset/checksum validations case emqx_ft_storage:store_segment(transfer(Msg, FileId), Segment) of ok -> @@ -247,3 +288,42 @@ transfer(Msg, FileId) -> on_assemble_timeout({ChanPid, PacketId}) -> ?SLOG(warning, #{msg => "on_assemble_timeout", packet_id => PacketId}), erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}). + +validate(Validations, Fun) -> + case do_validate(Validations, []) of + {ok, Parsed} -> + Fun(Parsed); + {error, Reason} -> + ?SLOG(error, #{ + msg => "validate: invalid $file command", + reason => Reason + }), + ?RC_UNSPECIFIED_ERROR + end. + +do_validate([], Parsed) -> + {ok, lists:reverse(Parsed)}; +do_validate([{offset, Offset} | Rest], Parsed) -> + case string:to_integer(Offset) of + {Int, <<>>} -> + do_validate(Rest, [Int | Parsed]); + _ -> + {error, {invalid_offset, Offset}} + end; +do_validate([{checksum, Checksum} | Rest], Parsed) -> + case parse_checksum(Checksum) of + {ok, Bin} -> + do_validate(Rest, [Bin | Parsed]); + {error, _Reason} -> + {error, {invalid_checksum, Checksum}} + end. + +parse_checksum(Checksum) when is_binary(Checksum) andalso byte_size(Checksum) =:= 64 -> + try + {ok, binary:decode_hex(Checksum)} + catch + error:badarg -> + {error, invalid_checksum} + end; +parse_checksum(_Checksum) -> + {error, invalid_checksum}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index a6559d470..01a3e19c2 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -387,10 +387,14 @@ encode_filemeta(Meta) -> Term = hocon_tconf:make_serializable(Schema, emqx_map_lib:binary_key_map(Meta), #{}), emqx_json:encode(?PRELUDE(_Vsn = 1, Term)). -decode_filemeta(Binary) -> - Schema = emqx_ft_schema:schema(filemeta), - ?PRELUDE(_Vsn = 1, Term) = emqx_json:decode(Binary, [return_maps]), - hocon_tconf:check_plain(Schema, Term, #{atom_key => true, required => false}). +decode_filemeta(Binary) when is_binary(Binary) -> + ?PRELUDE(_Vsn = 1, Map) = emqx_json:decode(Binary, [return_maps]), + case emqx_ft:decode_filemeta(Map) of + {ok, Meta} -> + Meta; + {error, Reason} -> + error(Reason) + end. mk_segment_filename({Offset, Content}) -> lists:concat([?SEGMENT, ".", Offset, ".", byte_size(Content)]). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index 992d62c48..f69033b4b 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl index db5e35f94..f8fe02d36 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl new file mode 100644 index 000000000..ff7cba403 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -0,0 +1,284 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-define(assertRCName(RCName, PublishRes), + ?assertMatch( + {ok, #{reason_code_name := RCName}}, + PublishRes + ) +). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], fun set_special_configs/1), + ok = emqx_common_test_helpers:maybe_fix_gen_rpc(), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok. + +set_special_configs(emqx_ft) -> + {ok, _} = emqx:update_config([file_transfer, storage], #{<<"type">> => <<"local">>}), + ok; +set_special_configs(_App) -> + ok. + +init_per_testcase(_Case, Config) -> + _ = file:del_dir_r(filename:join(emqx:data_dir(), "file_transfer")), + ClientId = <<"client">>, + {ok, C} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(C), + [{client, C}, {clientid, ClientId} | Config]. + +end_per_testcase(_Case, Config) -> + C = ?config(client, Config), + ok = emqtt:stop(C), + ok. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_invalid_topic_format(Config) -> + C = ?config(client, Config), + + %% TODO: more invalid topics + + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/XYZ">>, <<>>, 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/X/Y/Z">>, <<>>, 1) + ). + +t_simple_transfer(Config) -> + C = ?config(client, Config), + + Filename = <<"topsecret.pdf">>, + FileId = <<"f1">>, + + Data = [<<"first">>, <<"second">>, <<"third">>], + + Meta = meta(Filename, Data), + MetaPayload = emqx_json:encode(Meta), + + MetaTopic = <<"$file/", FileId/binary, "/init">>, + ?assertRCName( + success, + emqtt:publish(C, MetaTopic, MetaPayload, 1) + ), + + lists:foreach( + fun({Chunk, Offset}) -> + OffsetBin = integer_to_binary(Offset), + SegmentTopic = <<"$file/", FileId/binary, "/", OffsetBin/binary>>, + ?assertRCName( + success, + emqtt:publish(C, SegmentTopic, Chunk, 1) + ) + end, + with_offsets(Data) + ), + + FinTopic = <<"$file/", FileId/binary, "/fin">>, + ?assertRCName( + success, + emqtt:publish(C, FinTopic, <<>>, 1) + ), + + ReadyTransferId = #{ + <<"fileid">> => FileId, + <<"clientid">> => ?config(clientid, Config), + <<"node">> => atom_to_binary(node(), utf8) + }, + + {ok, TableQH} = emqx_ft_storage:get_ready_transfer(ReadyTransferId), + + ?assertEqual( + iolist_to_binary(Data), + iolist_to_binary(qlc:eval(TableQH)) + ). + +t_meta_conflict(Config) -> + C = ?config(client, Config), + + Filename = <<"topsecret.pdf">>, + FileId = <<"f1">>, + + Meta = meta(Filename, [<<"x">>]), + MetaPayload = emqx_json:encode(Meta), + + MetaTopic = <<"$file/", FileId/binary, "/init">>, + ?assertRCName( + success, + emqtt:publish(C, MetaTopic, MetaPayload, 1) + ), + + ConflictMeta = Meta#{name => <<"conflict.pdf">>}, + ConflictMetaPayload = emqx_json:encode(ConflictMeta), + + ?assertRCName( + unspecified_error, + emqtt:publish(C, MetaTopic, ConflictMetaPayload, 1) + ). + +t_no_meta(Config) -> + C = ?config(client, Config), + + FileId = <<"f1">>, + Data = <<"first">>, + + SegmentTopic = <<"$file/", FileId/binary, "/0">>, + ?assertRCName( + success, + emqtt:publish(C, SegmentTopic, Data, 1) + ), + + FinTopic = <<"$file/", FileId/binary, "/fin">>, + ?assertRCName( + unspecified_error, + emqtt:publish(C, FinTopic, <<>>, 1) + ). + +t_no_segment(Config) -> + C = ?config(client, Config), + + Filename = <<"topsecret.pdf">>, + FileId = <<"f1">>, + + Data = [<<"first">>, <<"second">>, <<"third">>], + + Meta = meta(Filename, Data), + MetaPayload = emqx_json:encode(Meta), + + MetaTopic = <<"$file/", FileId/binary, "/init">>, + ?assertRCName( + success, + emqtt:publish(C, MetaTopic, MetaPayload, 1) + ), + + lists:foreach( + fun({Chunk, Offset}) -> + OffsetBin = integer_to_binary(Offset), + SegmentTopic = <<"$file/", FileId/binary, "/", OffsetBin/binary>>, + ?assertRCName( + success, + emqtt:publish(C, SegmentTopic, Chunk, 1) + ) + end, + %% Skip the first segment + tl(with_offsets(Data)) + ), + + FinTopic = <<"$file/", FileId/binary, "/fin">>, + ?assertRCName( + unspecified_error, + emqtt:publish(C, FinTopic, <<>>, 1) + ). + +t_invalid_meta(Config) -> + C = ?config(client, Config), + + FileId = <<"f1">>, + + MetaTopic = <<"$file/", FileId/binary, "/init">>, + + %% Invalid schema + Meta = #{foo => <<"bar">>}, + MetaPayload = emqx_json:encode(Meta), + ?assertRCName( + unspecified_error, + emqtt:publish(C, MetaTopic, MetaPayload, 1) + ), + + %% Invalid JSON + ?assertRCName( + unspecified_error, + emqtt:publish(C, MetaTopic, <<"{oops;">>, 1) + ). + +t_invalid_checksum(Config) -> + C = ?config(client, Config), + + Filename = <<"topsecret.pdf">>, + FileId = <<"f1">>, + + Data = [<<"first">>, <<"second">>, <<"third">>], + + Meta = meta(Filename, Data), + MetaPayload = emqx_json:encode(Meta#{checksum => sha256hex(<<"invalid">>)}), + + MetaTopic = <<"$file/", FileId/binary, "/init">>, + ?assertRCName( + success, + emqtt:publish(C, MetaTopic, MetaPayload, 1) + ), + + lists:foreach( + fun({Chunk, Offset}) -> + OffsetBin = integer_to_binary(Offset), + SegmentTopic = <<"$file/", FileId/binary, "/", OffsetBin/binary>>, + ?assertRCName( + success, + emqtt:publish(C, SegmentTopic, Chunk, 1) + ) + end, + with_offsets(Data) + ), + + FinTopic = <<"$file/", FileId/binary, "/fin">>, + ?assertRCName( + unspecified_error, + emqtt:publish(C, FinTopic, <<>>, 1) + ). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +with_offsets(Items) -> + {List, _} = lists:mapfoldl( + fun(Item, Offset) -> + {{Item, Offset}, Offset + byte_size(Item)} + end, + 0, + Items + ), + List. + +sha256hex(Data) -> + binary:encode_hex(crypto:hash(sha256, Data)). + +meta(FileName, Data) -> + FullData = iolist_to_binary(Data), + #{ + name => FileName, + checksum => sha256hex(FullData), + expire_at => erlang:system_time(_Unit = second) + 3600, + size => byte_size(FullData) + }. diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index dd3ffedad..c5dbd418f 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/test/emqx_ft_content_gen.erl b/apps/emqx_ft/test/emqx_ft_content_gen.erl index feca78949..bd24d8c94 100644 --- a/apps/emqx_ft/test/emqx_ft_content_gen.erl +++ b/apps/emqx_ft/test/emqx_ft_content_gen.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl index c08986120..9098edcf6 100644 --- a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl index e979d06fc..0ac5d2844 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. From f6a0598f270e3dd907ff3f82efd6e2a9c839d536 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 15 Feb 2023 11:42:27 +0200 Subject: [PATCH 029/156] feat(ft): add file transfer tests --- apps/emqx/test/emqx_common_test_helpers.erl | 7 +- apps/emqx_ft/src/emqx_ft.erl | 16 +- apps/emqx_ft/src/emqx_ft_conf.erl | 4 +- apps/emqx_ft/src/emqx_ft_storage_dummy.erl | 43 ----- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 5 +- .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 6 - apps/emqx_ft/test/emqx_ft_SUITE.erl | 167 +++++++++++++++--- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 65 +++++++ .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 151 ++++++++++++++++ apps/emqx_ft/test/emqx_ft_test_helpers.erl | 77 ++++++++ 10 files changed, 456 insertions(+), 85 deletions(-) delete mode 100644 apps/emqx_ft/src/emqx_ft_storage_dummy.erl create mode 100644 apps/emqx_ft/test/emqx_ft_conf_SUITE.erl create mode 100644 apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl create mode 100644 apps/emqx_ft/test/emqx_ft_test_helpers.erl diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index ecdf8f827..18a3d9f3e 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -63,7 +63,7 @@ ]). -export([ - maybe_fix_gen_rpc/0, + set_gen_rpc_stateless/0, emqx_cluster/1, emqx_cluster/2, start_epmd/0, @@ -617,13 +617,14 @@ ensure_quic_listener(Name, UdpPort, ExtraSettings) -> listener_ports => [{Type :: tcp | ssl | ws | wss, inet:port_number()}] }. --spec maybe_fix_gen_rpc() -> ok. -maybe_fix_gen_rpc() -> +-spec set_gen_rpc_stateless() -> ok. +set_gen_rpc_stateless() -> %% When many tests run in an obscure order, it may occur that %% `gen_rpc` started with its default settings before `emqx_conf`. %% `gen_rpc` and `emqx_conf` have different default `port_discovery` modes, %% so we reinitialize `gen_rpc` explicitly. ok = application:stop(gen_rpc), + ok = application:set_env(gen_rpc, port_discovery, stateless), ok = application:start(gen_rpc). -spec emqx_cluster(cluster_spec()) -> [{shortname(), node_opts()}]. diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index ed4edd33b..2704e1e55 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -175,7 +175,13 @@ on_init(Msg, FileId) -> case emqx_ft_storage:store_filemeta(transfer(Msg, FileId), Meta) of ok -> ?RC_SUCCESS; - {error, _Reason} -> + {error, Reason} -> + ?SLOG(warning, #{ + msg => "store_filemeta_failed", + mqtt_msg => Msg, + file_id => FileId, + reason => Reason + }), ?RC_UNSPECIFIED_ERROR end; {error, Reason} -> @@ -235,7 +241,13 @@ on_fin(PacketId, Msg, FileId, Checksum) -> {ok, _} -> undefined; %% Assembling failed, unregister the packet key - {error, _} -> + {error, Reason} -> + ?SLOG(warning, #{ + msg => "assemble_not_started", + mqtt_msg => Msg, + file_id => FileId, + reason => Reason + }), case emqx_ft_responder:unregister(FinPacketKey) of %% We successfully unregistered the packet key, %% so we can send the error code at once diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index b88fd2532..d56dd8d32 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -50,8 +50,8 @@ unload() -> -spec pre_config_update(list(atom()), emqx_config:update_request(), emqx_config:raw_config()) -> {ok, emqx_config:update_request()} | {error, term()}. -pre_config_update(_, _Req, Config) -> - {ok, Config}. +pre_config_update(_, Req, _Config) -> + {ok, Req}. -spec post_config_update( list(atom()), diff --git a/apps/emqx_ft/src/emqx_ft_storage_dummy.erl b/apps/emqx_ft/src/emqx_ft_storage_dummy.erl deleted file mode 100644 index 4ed8ba487..000000000 --- a/apps/emqx_ft/src/emqx_ft_storage_dummy.erl +++ /dev/null @@ -1,43 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_ft_storage_dummy). - --behaviour(emqx_ft_storage). - --export([ - store_filemeta/3, - store_segment/3, - assemble/3, - ready_transfers/1, - get_ready_transfer/2 -]). - -store_filemeta(_Storage, _Transfer, _Meta) -> - ok. - -store_segment(_Storage, _Transfer, _Segment) -> - ok. - -assemble(_Storage, _Transfer, Callback) -> - Pid = spawn(fun() -> Callback({error, not_implemented}) end), - {ok, Pid}. - -ready_transfers(_Storage) -> - {ok, []}. - -get_ready_transfer(_Storage, _Id) -> - {error, not_implemented}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 01a3e19c2..c6cb09cf2 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -93,7 +93,10 @@ store_filemeta(Storage, Transfer, Meta) -> {ok, Meta} -> _ = touch_file(Filepath), ok; - {ok, _Conflict} -> + {ok, Conflict} -> + ?SLOG(warning, #{ + msg => "filemeta_conflict", transfer => Transfer, new => Meta, old => Conflict + }), % TODO % We won't see conflicts in case of concurrent `store_filemeta` % requests. It's rather odd scenario so it's fine not to worry diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index f69033b4b..e2c4c93d7 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -20,7 +20,6 @@ -export([introduced_in/0]). --export([list/3]). -export([multilist/3]). -export([pread/5]). -export([ready_transfers/1]). @@ -35,11 +34,6 @@ introduced_in() -> "5.0.17". --spec list(node(), transfer(), fragment | result) -> - {ok, [filefrag()]} | {error, term()} | no_return(). -list(Node, Transfer, What) -> - erpc:call(Node, emqx_ft_storage_fs_proxy, list_local, [Transfer, What]). - -spec multilist([node()], transfer(), fragment | result) -> emqx_rpc:erpc_multicall({ok, [filefrag()]} | {error, term()}). multilist(Nodes, Transfer, What) -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index ff7cba403..6a125449f 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -29,35 +29,58 @@ ) ). -all() -> emqx_common_test_helpers:all(?MODULE). +all() -> + [ + {group, single_node}, + {group, cluster} + ]. + +groups() -> + [ + {single_node, [sequence], emqx_common_test_helpers:all(?MODULE) -- [t_switch_node]}, + {cluster, [sequence], [t_switch_node]} + ]. init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], fun set_special_configs/1), - ok = emqx_common_test_helpers:maybe_fix_gen_rpc(), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], set_special_configs(Config)), + ok = emqx_common_test_helpers:set_gen_rpc_stateless(), Config. end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), ok. -set_special_configs(emqx_ft) -> - {ok, _} = emqx:update_config([file_transfer, storage], #{<<"type">> => <<"local">>}), - ok; -set_special_configs(_App) -> - ok. +set_special_configs(Config) -> + fun + (emqx_ft) -> + ok = emqx_config:put([file_transfer, storage], #{ + type => local, root => emqx_ft_test_helpers:ft_root(Config, node()) + }); + (_) -> + ok + end. -init_per_testcase(_Case, Config) -> - _ = file:del_dir_r(filename:join(emqx:data_dir(), "file_transfer")), - ClientId = <<"client">>, +init_per_testcase(Case, Config) -> + ClientId = atom_to_binary(Case), {ok, C} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}]), {ok, _} = emqtt:connect(C), [{client, C}, {clientid, ClientId} | Config]. - end_per_testcase(_Case, Config) -> C = ?config(client, Config), ok = emqtt:stop(C), ok. +init_per_group(cluster, Config) -> + Node = emqx_ft_test_helpers:start_additional_node(Config, test2), + [{additional_node, Node} | Config]; +init_per_group(_Group, Config) -> + Config. + +end_per_group(cluster, Config) -> + ok = emqx_ft_test_helpers:stop_additional_node(Config); +end_per_group(_Group, _Config) -> + ok. + %%-------------------------------------------------------------------- %% Tests %%-------------------------------------------------------------------- @@ -65,15 +88,34 @@ end_per_testcase(_Case, Config) -> t_invalid_topic_format(Config) -> C = ?config(client, Config), - %% TODO: more invalid topics - ?assertRCName( unspecified_error, - emqtt:publish(C, <<"$file/XYZ">>, <<>>, 1) + emqtt:publish(C, <<"$file/fileid">>, <<>>, 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/fileid/">>, <<>>, 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/fileid/offset">>, <<>>, 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/fileid/fin/offset">>, <<>>, 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/">>, <<>>, 1) ), ?assertRCName( unspecified_error, emqtt:publish(C, <<"$file/X/Y/Z">>, <<>>, 1) + ), + %% should not be handled by `emqx_ft` + ?assertRCName( + no_matching_subscribers, + emqtt:publish(C, <<"$file">>, <<>>, 1) ). t_simple_transfer(Config) -> @@ -95,8 +137,7 @@ t_simple_transfer(Config) -> lists:foreach( fun({Chunk, Offset}) -> - OffsetBin = integer_to_binary(Offset), - SegmentTopic = <<"$file/", FileId/binary, "/", OffsetBin/binary>>, + SegmentTopic = <<"$file/", FileId/binary, "/", Offset/binary>>, ?assertRCName( success, emqtt:publish(C, SegmentTopic, Chunk, 1) @@ -111,12 +152,7 @@ t_simple_transfer(Config) -> emqtt:publish(C, FinTopic, <<>>, 1) ), - ReadyTransferId = #{ - <<"fileid">> => FileId, - <<"clientid">> => ?config(clientid, Config), - <<"node">> => atom_to_binary(node(), utf8) - }, - + {ok, [{ReadyTransferId, _}]} = emqx_ft_storage:ready_transfers(), {ok, TableQH} = emqx_ft_storage:get_ready_transfer(ReadyTransferId), ?assertEqual( @@ -184,8 +220,7 @@ t_no_segment(Config) -> lists:foreach( fun({Chunk, Offset}) -> - OffsetBin = integer_to_binary(Offset), - SegmentTopic = <<"$file/", FileId/binary, "/", OffsetBin/binary>>, + SegmentTopic = <<"$file/", FileId/binary, "/", Offset/binary>>, ?assertRCName( success, emqtt:publish(C, SegmentTopic, Chunk, 1) @@ -241,8 +276,7 @@ t_invalid_checksum(Config) -> lists:foreach( fun({Chunk, Offset}) -> - OffsetBin = integer_to_binary(Offset), - SegmentTopic = <<"$file/", FileId/binary, "/", OffsetBin/binary>>, + SegmentTopic = <<"$file/", FileId/binary, "/", Offset/binary>>, ?assertRCName( success, emqtt:publish(C, SegmentTopic, Chunk, 1) @@ -257,6 +291,83 @@ t_invalid_checksum(Config) -> emqtt:publish(C, FinTopic, <<>>, 1) ). +t_switch_node(Config) -> + AdditionalNodePort = emqx_ft_test_helpers:tcp_port(?config(additional_node, Config)), + + ClientId = <<"t_switch_node-migrating_client">>, + + {ok, C1} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, AdditionalNodePort}]), + {ok, _} = emqtt:connect(C1), + + Filename = <<"multinode_upload.txt">>, + FileId = <<"f1">>, + + Data = [<<"first">>, <<"second">>, <<"third">>], + [{Data0, Offset0}, {Data1, Offset1}, {Data2, Offset2}] = with_offsets(Data), + + %% First, publist metadata and the first segment to the additional node + + Meta = meta(Filename, Data), + MetaPayload = emqx_json:encode(Meta), + + MetaTopic = <<"$file/", FileId/binary, "/init">>, + ?assertRCName( + success, + emqtt:publish(C1, MetaTopic, MetaPayload, 1) + ), + ?assertRCName( + success, + emqtt:publish(C1, <<"$file/", FileId/binary, "/", Offset0/binary>>, Data0, 1) + ), + + %% Then, switch the client to the main node + %% and publish the rest of the segments + + ok = emqtt:stop(C1), + {ok, C2} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(C2), + + ?assertRCName( + success, + emqtt:publish(C2, <<"$file/", FileId/binary, "/", Offset1/binary>>, Data1, 1) + ), + ?assertRCName( + success, + emqtt:publish(C2, <<"$file/", FileId/binary, "/", Offset2/binary>>, Data2, 1) + ), + + FinTopic = <<"$file/", FileId/binary, "/fin">>, + ?assertRCName( + success, + emqtt:publish(C2, FinTopic, <<>>, 1) + ), + + ok = emqtt:stop(C2), + + %% Now check consistency of the file + + {ok, ReadyTransfers} = emqx_ft_storage:ready_transfers(), + {ReadyTransferIds, _} = lists:unzip(ReadyTransfers), + [ReadyTransferId] = [Id || #{<<"clientid">> := CId} = Id <- ReadyTransferIds, CId == ClientId], + + {ok, TableQH} = emqx_ft_storage:get_ready_transfer(ReadyTransferId), + + ?assertEqual( + iolist_to_binary(Data), + iolist_to_binary(qlc:eval(TableQH)) + ). + +t_assemble_crash(Config) -> + C = ?config(client, Config), + + meck:new(emqx_ft_storage_fs), + meck:expect(emqx_ft_storage_fs, assemble, fun(_, _, _) -> meck:exception(error, oops) end), + + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/someid/fin">>, <<>>, 1) + ). + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- @@ -264,7 +375,7 @@ t_invalid_checksum(Config) -> with_offsets(Items) -> {List, _} = lists:mapfoldl( fun(Item, Offset) -> - {{Item, Offset}, Offset + byte_size(Item)} + {{Item, integer_to_binary(Offset)}, Offset + byte_size(Item)} end, 0, Items diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl new file mode 100644 index 000000000..cca69796d --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -0,0 +1,65 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_conf_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft]), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok. + +init_per_testcase(_Case, Config) -> + Config. + +end_per_testcase(_Case, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_update_config(_Config) -> + ?assertMatch( + {error, #{kind := validation_error}}, + emqx_conf:update( + [file_transfer], + #{<<"storage">> => #{<<"type">> => <<"unknown">>}}, + #{} + ) + ), + ?assertMatch( + {ok, _}, + emqx_conf:update( + [file_transfer], + #{<<"storage">> => #{<<"type">> => <<"local">>, <<"root">> => <<"/tmp/path">>}}, + #{} + ) + ), + ?assertEqual( + <<"/tmp/path">>, + emqx_config:get([file_transfer, storage, root]) + ). diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl new file mode 100644 index 000000000..20645bcd9 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -0,0 +1,151 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-define(assertInclude(Pattern, List), + ?assert( + lists:any( + fun + (Pattern) -> true; + (_) -> false + end, + List + ) + ) +). + +all() -> + [ + {group, single_node}, + {group, cluster} + ]. + +-define(CLUSTER_CASES, [t_multinode_ready_transfers]). + +groups() -> + [ + {single_node, [sequence], emqx_common_test_helpers:all(?MODULE) -- ?CLUSTER_CASES}, + {cluster, [sequence], ?CLUSTER_CASES} + ]. + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], set_special_configs(Config)), + ok = emqx_common_test_helpers:set_gen_rpc_stateless(), + Config. +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok. + +set_special_configs(Config) -> + fun + (emqx_ft) -> + ok = emqx_config:put([file_transfer, storage], #{ + type => local, root => emqx_ft_test_helpers:ft_root(Config, node()) + }); + (_) -> + ok + end. + +init_per_testcase(Case, Config) -> + [{tc, Case} | Config]. +end_per_testcase(_Case, _Config) -> + ok. + +init_per_group(cluster, Config) -> + Node = emqx_ft_test_helpers:start_additional_node(Config, test2), + [{additional_node, Node} | Config]; +init_per_group(_Group, Config) -> + Config. + +end_per_group(cluster, Config) -> + ok = emqx_ft_test_helpers:stop_additional_node(Config); +end_per_group(_Group, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_invalid_ready_transfer_id(Config) -> + ?assertMatch( + {error, _}, + emqx_ft_storage_fs:get_ready_transfer(storage(Config), #{ + <<"clientid">> => client_id(Config), + <<"fileid">> => <<"fileid">>, + <<"node">> => atom_to_binary('nonexistent@127.0.0.1') + }) + ), + ?assertMatch( + {error, _}, + emqx_ft_storage_fs:get_ready_transfer(storage(Config), #{ + <<"clientid">> => client_id(Config), + <<"fileid">> => <<"fileid">>, + <<"node">> => <<"nonexistent_as_atom@127.0.0.1">> + }) + ), + ?assertMatch( + {error, _}, + emqx_ft_storage_fs:get_ready_transfer(storage(Config), #{ + <<"clientid">> => client_id(Config), + <<"fileid">> => <<"nonexistent_file">>, + <<"node">> => node() + }) + ). + +t_multinode_ready_transfers(Config) -> + Node1 = ?config(additional_node, Config), + ok = emqx_ft_test_helpers:upload_file(<<"c1">>, <<"f1">>, <<"data">>, Node1), + + Node2 = node(), + ok = emqx_ft_test_helpers:upload_file(<<"c2">>, <<"f2">>, <<"data">>, Node2), + + ?assertInclude( + #{<<"clientid">> := <<"c1">>, <<"fileid">> := <<"f1">>}, + ready_transfer_ids(Config) + ), + + ?assertInclude( + #{<<"clientid">> := <<"c2">>, <<"fileid">> := <<"f2">>}, + ready_transfer_ids(Config) + ). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +client_id(Config) -> + atom_to_binary(?config(tc, Config), utf8). + +storage(Config) -> + #{ + type => local, + root => ft_root(Config) + }. + +ft_root(Config) -> + emqx_ft_test_helpers:ft_root(Config, node()). + +ready_transfer_ids(Config) -> + {ok, ReadyTransfers} = emqx_ft_storage_fs:ready_transfers(storage(Config)), + {ReadyTransferIds, _} = lists:unzip(ReadyTransfers), + ReadyTransferIds. diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl new file mode 100644 index 000000000..ca854bda0 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -0,0 +1,77 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_test_helpers). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). + +start_additional_node(Config, Node) -> + SelfNode = node(), + emqx_common_test_helpers:start_slave( + Node, + [ + {apps, [emqx_ft]}, + {join_to, SelfNode}, + {configure_gen_rpc, false}, + {env_handler, fun + (emqx_ft) -> + ok = emqx_config:put([file_transfer, storage], #{ + type => local, root => ft_root(Config, node()) + }); + (_) -> + ok + end} + ] + ). + +stop_additional_node(Config) -> + Node = ?config(additional_node, Config), + ok = rpc:call(Node, ekka, leave, []), + ok = rpc:call(Node, emqx_common_test_helpers, stop_apps, [[emqx_ft]]), + {ok, _} = emqx_common_test_helpers:stop_slave(Node), + ok. + +tcp_port(Node) -> + {_, Port} = rpc:call(Node, emqx_config, get, [[listeners, tcp, default, bind]]), + Port. + +ft_root(Config, Node) -> + filename:join([ + ?config(priv_dir, Config), <<"file_transfer">>, atom_to_binary(Node) + ]). + +upload_file(ClientId, FileId, Data, Node) -> + Port = tcp_port(Node), + + {ok, C1} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, Port}]), + {ok, _} = emqtt:connect(C1), + Meta = #{ + name => FileId, + expire_at => erlang:system_time(_Unit = second) + 3600, + size => byte_size(Data) + }, + MetaPayload = emqx_json:encode(Meta), + + MetaTopic = <<"$file/", FileId/binary, "/init">>, + {ok, _} = emqtt:publish(C1, MetaTopic, MetaPayload, 1), + {ok, _} = emqtt:publish(C1, <<"$file/", FileId/binary, "/0">>, Data, 1), + + FinTopic = <<"$file/", FileId/binary, "/fin">>, + {ok, _} = emqtt:publish(C1, FinTopic, <<>>, 1), + ok = emqtt:stop(C1). From 2cdf486bf499551f2657c0940ae41bb552876809 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 15 Feb 2023 16:02:18 +0300 Subject: [PATCH 030/156] feat(ft): simplify responder mechanism Make responder short-lived process responsible for a single task, and manage them with supervisor + gproc. --- apps/emqx_ft/src/emqx_ft.erl | 88 +++++-------- apps/emqx_ft/src/emqx_ft_responder.erl | 116 ++++++------------ apps/emqx_ft/src/emqx_ft_responder_sup.erl | 48 ++++++++ apps/emqx_ft/src/emqx_ft_sup.erl | 6 +- apps/emqx_ft/test/emqx_ft_responder_SUITE.erl | 65 +++++----- 5 files changed, 148 insertions(+), 175 deletions(-) create mode 100644 apps/emqx_ft/src/emqx_ft_responder_sup.erl diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 2704e1e55..ac419cb36 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -35,7 +35,7 @@ decode_filemeta/1 ]). --export([on_assemble_timeout/1]). +-export([on_assemble/2]). -export_type([ clientid/0, @@ -227,45 +227,25 @@ on_fin(PacketId, Msg, FileId, Checksum) -> }), %% TODO: handle checksum? Do we need it? FinPacketKey = {self(), PacketId}, - _ = - case - emqx_ft_responder:register( - FinPacketKey, fun ?MODULE:on_assemble_timeout/1, ?ASSEMBLE_TIMEOUT - ) - of - %% We have new fin packet - ok -> - Callback = callback(FinPacketKey, FileId), - case assemble(transfer(Msg, FileId), Callback) of - %% Assembling started, packet will be acked by the callback or the responder - {ok, _} -> - undefined; - %% Assembling failed, unregister the packet key - {error, Reason} -> - ?SLOG(warning, #{ - msg => "assemble_not_started", - mqtt_msg => Msg, - file_id => FileId, - reason => Reason - }), - case emqx_ft_responder:unregister(FinPacketKey) of - %% We successfully unregistered the packet key, - %% so we can send the error code at once - ok -> - ?RC_UNSPECIFIED_ERROR; - %% Someone else already unregistered the key, - %% that is, either responder or someone else acked the packet, - %% we do not have to ack - {error, not_found} -> - undefined - end - end; - %% Fin packet already received. - %% Since we are still handling the previous one, - %% we probably have retransmit here - {error, already_registered} -> - undefined - end. + case emqx_ft_responder:start(FinPacketKey, fun ?MODULE:on_assemble/2, ?ASSEMBLE_TIMEOUT) of + %% We have new fin packet + {ok, _} -> + Callback = fun(Result) -> emqx_ft_responder:ack(FinPacketKey, Result) end, + case assemble(transfer(Msg, FileId), Callback) of + %% Assembling started, packet will be acked by the callback or the responder + {ok, _} -> + ok; + %% Assembling failed, ack through the responder + {error, _} = Error -> + emqx_ft_responder:ack(FinPacketKey, Error) + end; + %% Fin packet already received. + %% Since we are still handling the previous one, + %% we probably have retransmit here + {error, {already_started, _}} -> + ok + end, + undefined. assemble(Transfer, Callback) -> try @@ -278,28 +258,20 @@ assemble(Transfer, Callback) -> {error, {internal_error, E}} end. -callback({ChanPid, PacketId} = Key, _FileId) -> - fun(Result) -> - case emqx_ft_responder:unregister(Key) of - ok -> - case Result of - ok -> - erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS}); - {error, _} -> - erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}) - end; - {error, not_found} -> - ok - end - end. - transfer(Msg, FileId) -> ClientId = Msg#message.from, {ClientId, FileId}. -on_assemble_timeout({ChanPid, PacketId}) -> - ?SLOG(warning, #{msg => "on_assemble_timeout", packet_id => PacketId}), - erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}). +on_assemble({ChanPid, PacketId}, Result) -> + ?SLOG(debug, #{msg => "on_assemble", packet_id => PacketId, result => Result}), + case Result of + {ack, ok} -> + erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS}); + {ack, {error, _}} -> + erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}); + timeout -> + erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}) + end. validate(Validations, Fun) -> case do_validate(Validations, []) of diff --git a/apps/emqx_ft/src/emqx_ft_responder.erl b/apps/emqx_ft/src/emqx_ft_responder.erl index 8417053f6..7b9220774 100644 --- a/apps/emqx_ft/src/emqx_ft_responder.erl +++ b/apps/emqx_ft/src/emqx_ft_responder.erl @@ -23,73 +23,50 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). --export([start_link/0]). +%% API +-export([start/3]). +-export([ack/2]). --export([ - register/3, - unregister/1 -]). +%% Supervisor API +-export([start_link/3]). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). --define(SERVER, ?MODULE). --define(TAB, ?MODULE). +-define(REF(Key), {via, gproc, {n, l, {?MODULE, Key}}}). -type key() :: term(). +-type respfun() :: fun(({ack, _Result} | timeout) -> _SideEffect). %%-------------------------------------------------------------------- %% API %% ------------------------------------------------------------------- --spec start_link() -> startlink_ret(). -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). +-spec start(key(), timeout(), respfun()) -> startlink_ret(). +start(Key, RespFun, Timeout) -> + emqx_ft_responder_sup:start_child(Key, RespFun, Timeout). --spec register(Key, DefaultAction, Timeout) -> ok | {error, already_registered} when - Key :: key(), - DefaultAction :: fun((Key) -> any()), - Timeout :: timeout(). -register(Key, DefaultAction, Timeout) -> - case ets:lookup(?TAB, Key) of - [] -> - gen_server:call(?SERVER, {register, Key, DefaultAction, Timeout}); - [{Key, _Action, _Ref}] -> - {error, already_registered} - end. +-spec ack(key(), _Result) -> _Return. +ack(Key, Result) -> + % TODO: it's possible to avoid term copy + gen_server:call(?REF(Key), {ack, Result}, infinity). --spec unregister(Key) -> ok | {error, not_found} when - Key :: key(). -unregister(Key) -> - gen_server:call(?SERVER, {unregister, Key}). +-spec start_link(key(), timeout(), respfun()) -> startlink_ret(). +start_link(Key, RespFun, Timeout) -> + gen_server:start_link(?REF(Key), ?MODULE, {Key, RespFun, Timeout}, []). %%-------------------------------------------------------------------- %% gen_server callbacks %% ------------------------------------------------------------------- -init([]) -> - _ = ets:new(?TAB, [named_table, protected, set, {read_concurrency, true}]), - {ok, #{}}. +init({Key, RespFun, Timeout}) -> + _ = erlang:process_flag(trap_exit, true), + _TRef = erlang:send_after(Timeout, self(), timeout), + {ok, {Key, RespFun}}. -handle_call({register, Key, DefaultAction, Timeout}, _From, State) -> - ?SLOG(warning, #{msg => "register", key => Key, timeout => Timeout}), - case ets:lookup(?TAB, Key) of - [] -> - TRef = erlang:start_timer(Timeout, self(), {timeout, Key}), - true = ets:insert(?TAB, {Key, DefaultAction, TRef}), - {reply, ok, State}; - [{_, _Action, _Ref}] -> - {reply, {error, already_registered}, State} - end; -handle_call({unregister, Key}, _From, State) -> - ?SLOG(warning, #{msg => "unregister", key => Key}), - case ets:lookup(?TAB, Key) of - [] -> - {reply, {error, not_found}, State}; - [{_, _Action, TRef}] -> - _ = erlang:cancel_timer(TRef), - true = ets:delete(?TAB, Key), - {reply, ok, State} - end; +handle_call({ack, Result}, _From, {Key, RespFun}) -> + Ret = apply(RespFun, [Key, {ack, Result}]), + ?tp(ft_responder_ack, #{key => Key, result => Result, return => Ret}), + {stop, {shutdown, Ret}, Ret, undefined}; handle_call(Msg, _From, State) -> ?SLOG(warning, #{msg => "unknown_call", call_msg => Msg}), {reply, {error, unknown_call}, State}. @@ -98,40 +75,17 @@ handle_cast(Msg, State) -> ?SLOG(warning, #{msg => "unknown_cast", cast_msg => Msg}), {noreply, State}. -handle_info({timeout, TRef, {timeout, Key}}, State) -> - case ets:lookup(?TAB, Key) of - [] -> - {noreply, State}; - [{_, Action, TRef}] -> - _ = erlang:cancel_timer(TRef), - true = ets:delete(?TAB, Key), - ok = safe_apply(Action, [Key]), - ?tp(ft_timeout_action_applied, #{key => Key}), - {noreply, State} - end; +handle_info(timeout, {Key, RespFun}) -> + Ret = apply(RespFun, [Key, timeout]), + ?tp(ft_responder_timeout, #{key => Key, return => Ret}), + {stop, {shutdown, Ret}, undefined}; handle_info(Msg, State) -> ?SLOG(warning, #{msg => "unknown_message", info_msg => Msg}), {noreply, State}. -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -terminate(_Reason, _State) -> +terminate(_Reason, undefined) -> + ok; +terminate(Reason, {Key, RespFun}) -> + Ret = apply(RespFun, [Key, timeout]), + ?tp(ft_responder_shutdown, #{key => Key, reason => Reason, return => Ret}), ok. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -safe_apply(Fun, Args) -> - try apply(Fun, Args) of - _ -> ok - catch - Class:Reason:Stacktrace -> - ?SLOG(error, #{ - msg => "safe_apply_failed", - class => Class, - reason => Reason, - stacktrace => Stacktrace - }) - end. diff --git a/apps/emqx_ft/src/emqx_ft_responder_sup.erl b/apps/emqx_ft/src/emqx_ft_responder_sup.erl new file mode 100644 index 000000000..23d4f55fa --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_responder_sup.erl @@ -0,0 +1,48 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_responder_sup). + +-export([start_link/0]). +-export([start_child/3]). + +-behaviour(supervisor). +-export([init/1]). + +-define(SUPERVISOR, ?MODULE). + +%% + +-spec start_link() -> {ok, pid()}. +start_link() -> + supervisor:start_link({local, ?SUPERVISOR}, ?MODULE, []). + +start_child(Key, RespFun, Timeout) -> + supervisor:start_child(?SUPERVISOR, [Key, RespFun, Timeout]). + +-spec init(_) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}. +init(_) -> + Flags = #{ + strategy => simple_one_for_one, + intensity => 100, + period => 100 + }, + ChildSpec = #{ + id => responder, + start => {emqx_ft_responder, start_link, []}, + restart => temporary + }, + {ok, {Flags, [ChildSpec]}}. diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index dfdde3a8a..8d388814c 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -53,12 +53,12 @@ init([]) -> }, Responder = #{ - id => emqx_ft_responder, - start => {emqx_ft_responder, start_link, []}, + id => emqx_ft_responder_sup, + start => {emqx_ft_responder_sup, start_link, []}, restart => permanent, shutdown => infinity, type => worker, - modules => [emqx_ft_responder] + modules => [emqx_ft_responder_sup] }, ChildSpecs = [Responder, AssemblerSup, FileReaderSup], diff --git a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl index 9098edcf6..447d41f11 100644 --- a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl @@ -19,7 +19,6 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). -include_lib("emqx/include/asserts.hrl"). @@ -39,58 +38,58 @@ init_per_testcase(_Case, Config) -> end_per_testcase(_Case, _Config) -> ok. -t_register_unregister(_Config) -> +t_start_ack(_Config) -> Key = <<"test">>, - DefaultAction = fun(_) -> ok end, - ?assertEqual( - ok, - emqx_ft_responder:register(Key, DefaultAction, 1000) + DefaultAction = fun(_Key, {ack, Ref}) -> Ref end, + ?assertMatch( + {ok, _Pid}, + emqx_ft_responder:start(Key, DefaultAction, 1000) ), - ?assertEqual( - {error, already_registered}, - emqx_ft_responder:register(Key, DefaultAction, 1000) + ?assertMatch( + {error, {already_started, _Pid}}, + emqx_ft_responder:start(Key, DefaultAction, 1000) ), + Ref = make_ref(), ?assertEqual( - ok, - emqx_ft_responder:unregister(Key) + Ref, + emqx_ft_responder:ack(Key, Ref) ), - ?assertEqual( - {error, not_found}, - emqx_ft_responder:unregister(Key) + ?assertExit( + {noproc, _}, + emqx_ft_responder:ack(Key, Ref) ). t_timeout(_Config) -> Key = <<"test">>, Self = self(), - DefaultAction = fun(K) -> Self ! {timeout, K} end, - ok = emqx_ft_responder:register(Key, DefaultAction, 20), + DefaultAction = fun(K, timeout) -> Self ! {timeout, K} end, + {ok, _Pid} = emqx_ft_responder:start(Key, DefaultAction, 20), receive {timeout, Key} -> ok after 100 -> ct:fail("emqx_ft_responder not called") end, - ?assertEqual( - {error, not_found}, - emqx_ft_responder:unregister(Key) + ?assertExit( + {noproc, _}, + emqx_ft_responder:ack(Key, oops) ). -t_action_exception(_Config) -> - Key = <<"test">>, - DefaultAction = fun(K) -> error({oops, K}) end, - - ?assertWaitEvent( - emqx_ft_responder:register(Key, DefaultAction, 10), - #{?snk_kind := ft_timeout_action_applied, key := <<"test">>}, - 1000 - ), - ?assertEqual( - {error, not_found}, - emqx_ft_responder:unregister(Key) - ). +% t_action_exception(_Config) -> +% Key = <<"test">>, +% DefaultAction = fun(K) -> error({oops, K}) end, +% ?assertWaitEvent( +% emqx_ft_responder:start(Key, DefaultAction, 10), +% #{?snk_kind := ft_timeout_action_applied, key := <<"test">>}, +% 1000 +% ), +% ?assertEqual( +% {error, not_found}, +% emqx_ft_responder:ack(Key, oops) +% ). t_unknown_msgs(_Config) -> - Pid = whereis(emqx_ft_responder), + {ok, Pid} = emqx_ft_responder:start(make_ref(), fun(_, _) -> ok end, 100), Pid ! {unknown_msg, <<"test">>}, ok = gen_server:cast(Pid, {unknown_msg, <<"test">>}), ?assertEqual( From f896fefa59f9eaeccf3f597c118e1594abc4ce66 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 16 Feb 2023 12:22:13 +0300 Subject: [PATCH 031/156] feat(ft): make storage backend fully async-aware Introduce an ad-hoc concept of tasks that need to be kicked off manually. Rework filesystem backend to accomodate for this change. Adapt responder logic for that "kickoff" protocol. --- apps/emqx_ft/src/emqx_ft.erl | 175 ++++++++++++------ apps/emqx_ft/src/emqx_ft_assembler.erl | 94 ++++------ apps/emqx_ft/src/emqx_ft_assembler_sup.erl | 15 +- apps/emqx_ft/src/emqx_ft_responder.erl | 33 +++- apps/emqx_ft/src/emqx_ft_storage.erl | 28 +-- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 12 +- apps/emqx_ft/test/emqx_ft_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 50 +++-- apps/emqx_ft/test/emqx_ft_responder_SUITE.erl | 6 +- 9 files changed, 240 insertions(+), 175 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index ac419cb36..348916920 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -35,7 +35,7 @@ decode_filemeta/1 ]). --export([on_assemble/2]). +-export([on_complete/4]). -export_type([ clientid/0, @@ -76,7 +76,8 @@ -type segment() :: {offset(), _Content :: binary()}. --define(ASSEMBLE_TIMEOUT, 5000). +-define(STORE_SEGMENT_TIMEOUT, 10000). +-define(ASSEMBLE_TIMEOUT, 60000). %%-------------------------------------------------------------------- %% API for app @@ -143,52 +144,59 @@ on_message_puback(PacketId, #message{topic = Topic} = Msg, _PubRes, _RC) -> on_file_command(PacketId, Msg, FileCommand) -> case string:split(FileCommand, <<"/">>, all) of [FileId, <<"init">>] -> - on_init(Msg, FileId); + on_init(PacketId, Msg, transfer(Msg, FileId)); [FileId, <<"fin">>] -> - on_fin(PacketId, Msg, FileId, undefined); + on_fin(PacketId, Msg, transfer(Msg, FileId), undefined); [FileId, <<"fin">>, Checksum] -> - on_fin(PacketId, Msg, FileId, Checksum); + on_fin(PacketId, Msg, transfer(Msg, FileId), Checksum); [FileId, <<"abort">>] -> - on_abort(Msg, FileId); + on_abort(Msg, transfer(Msg, FileId)); [FileId, OffsetBin] -> validate([{offset, OffsetBin}], fun([Offset]) -> - on_segment(Msg, FileId, Offset, undefined) + on_segment(PacketId, Msg, transfer(Msg, FileId), Offset, undefined) end); [FileId, OffsetBin, ChecksumBin] -> validate([{offset, OffsetBin}, {checksum, ChecksumBin}], fun([Offset, Checksum]) -> - on_segment(Msg, FileId, Offset, Checksum) + on_segment(PacketId, Msg, transfer(Msg, FileId), Offset, Checksum) end); _ -> ?RC_UNSPECIFIED_ERROR end. -on_init(Msg, FileId) -> +on_init(PacketId, Msg, Transfer) -> ?SLOG(info, #{ msg => "on_init", mqtt_msg => Msg, - file_id => FileId + packet_id => PacketId, + transfer => Transfer }), Payload = Msg#message.payload, + PacketKey = {self(), PacketId}, % %% Add validations here case decode_filemeta(Payload) of {ok, Meta} -> - case emqx_ft_storage:store_filemeta(transfer(Msg, FileId), Meta) of - ok -> - ?RC_SUCCESS; - {error, Reason} -> - ?SLOG(warning, #{ - msg => "store_filemeta_failed", - mqtt_msg => Msg, - file_id => FileId, - reason => Reason - }), - ?RC_UNSPECIFIED_ERROR - end; + Callback = fun(Result) -> + ?MODULE:on_complete("store_filemeta", PacketKey, Transfer, Result) + end, + with_responder(PacketKey, Callback, ?STORE_SEGMENT_TIMEOUT, fun() -> + case store_filemeta(Transfer, Meta) of + % Stored, ack through the responder right away + ok -> + emqx_ft_responder:ack(PacketKey, ok); + % Storage operation started, packet will be acked by the responder + {async, Pid} -> + ok = emqx_ft_responder:kickoff(PacketKey, Pid), + ok; + %% Storage operation failed, ack through the responder + {error, _} = Error -> + emqx_ft_responder:ack(PacketKey, Error) + end + end); {error, Reason} -> ?SLOG(error, #{ msg => "on_init: invalid filemeta", mqtt_msg => Msg, - file_id => FileId, + transfer => Transfer, reason => Reason }), ?RC_UNSPECIFIED_ERROR @@ -198,48 +206,69 @@ on_abort(_Msg, _FileId) -> %% TODO ?RC_SUCCESS. -on_segment(Msg, FileId, Offset, Checksum) -> +on_segment(PacketId, Msg, Transfer, Offset, Checksum) -> ?SLOG(info, #{ msg => "on_segment", mqtt_msg => Msg, - file_id => FileId, + packet_id => PacketId, + transfer => Transfer, offset => Offset, checksum => Checksum }), %% TODO: handle checksum Payload = Msg#message.payload, Segment = {Offset, Payload}, + PacketKey = {self(), PacketId}, + Callback = fun(Result) -> + ?MODULE:on_complete("store_segment", PacketKey, Transfer, Result) + end, %% Add offset/checksum validations - case emqx_ft_storage:store_segment(transfer(Msg, FileId), Segment) of - ok -> - ?RC_SUCCESS; - {error, _Reason} -> - ?RC_UNSPECIFIED_ERROR - end. + with_responder(PacketKey, Callback, ?STORE_SEGMENT_TIMEOUT, fun() -> + case store_segment(Transfer, Segment) of + ok -> + emqx_ft_responder:ack(PacketKey, ok); + {async, Pid} -> + ok = emqx_ft_responder:kickoff(PacketKey, Pid), + ok; + {error, _} = Error -> + emqx_ft_responder:ack(PacketKey, Error) + end + end). -on_fin(PacketId, Msg, FileId, Checksum) -> +on_fin(PacketId, Msg, Transfer, Checksum) -> ?SLOG(info, #{ msg => "on_fin", mqtt_msg => Msg, - file_id => FileId, - checksum => Checksum, - packet_id => PacketId + packet_id => PacketId, + transfer => Transfer, + checksum => Checksum }), %% TODO: handle checksum? Do we need it? FinPacketKey = {self(), PacketId}, - case emqx_ft_responder:start(FinPacketKey, fun ?MODULE:on_assemble/2, ?ASSEMBLE_TIMEOUT) of - %% We have new fin packet + Callback = fun(Result) -> + ?MODULE:on_complete("assemble", FinPacketKey, Transfer, Result) + end, + with_responder(FinPacketKey, Callback, ?ASSEMBLE_TIMEOUT, fun() -> + case assemble(Transfer) of + %% Assembling completed, ack through the responder right away + ok -> + emqx_ft_responder:ack(FinPacketKey, ok); + %% Assembling started, packet will be acked by the responder + {async, Pid} -> + ok = emqx_ft_responder:kickoff(FinPacketKey, Pid), + ok; + %% Assembling failed, ack through the responder + {error, _} = Error -> + emqx_ft_responder:ack(FinPacketKey, Error) + end + end). + +with_responder(Key, Callback, Timeout, CriticalSection) -> + case emqx_ft_responder:start(Key, Callback, Timeout) of + %% We have new packet {ok, _} -> - Callback = fun(Result) -> emqx_ft_responder:ack(FinPacketKey, Result) end, - case assemble(transfer(Msg, FileId), Callback) of - %% Assembling started, packet will be acked by the callback or the responder - {ok, _} -> - ok; - %% Assembling failed, ack through the responder - {error, _} = Error -> - emqx_ft_responder:ack(FinPacketKey, Error) - end; - %% Fin packet already received. + CriticalSection(); + %% Packet already received. %% Since we are still handling the previous one, %% we probably have retransmit here {error, {already_started, _}} -> @@ -247,13 +276,35 @@ on_fin(PacketId, Msg, FileId, Checksum) -> end, undefined. -assemble(Transfer, Callback) -> +store_filemeta(Transfer, Segment) -> try - emqx_ft_storage:assemble(Transfer, Callback) + emqx_ft_storage:store_filemeta(Transfer, Segment) catch C:E:S -> - ?SLOG(warning, #{ - msg => "file_assemble_failed", class => C, reason => E, stacktrace => S + ?SLOG(error, #{ + msg => "start_store_filemeta_failed", class => C, reason => E, stacktrace => S + }), + {error, {internal_error, E}} + end. + +store_segment(Transfer, Segment) -> + try + emqx_ft_storage:store_segment(Transfer, Segment) + catch + C:E:S -> + ?SLOG(error, #{ + msg => "start_store_segment_failed", class => C, reason => E, stacktrace => S + }), + {error, {internal_error, E}} + end. + +assemble(Transfer) -> + try + emqx_ft_storage:assemble(Transfer) + catch + C:E:S -> + ?SLOG(error, #{ + msg => "start_assemble_failed", class => C, reason => E, stacktrace => S }), {error, {internal_error, E}} end. @@ -262,14 +313,28 @@ transfer(Msg, FileId) -> ClientId = Msg#message.from, {ClientId, FileId}. -on_assemble({ChanPid, PacketId}, Result) -> - ?SLOG(debug, #{msg => "on_assemble", packet_id => PacketId, result => Result}), +on_complete(Op, {ChanPid, PacketId}, Transfer, Result) -> + ?SLOG(debug, #{ + msg => "on_complete", + operation => Op, + packet_id => PacketId, + transfer => Transfer + }), case Result of - {ack, ok} -> + {Mode, ok} when Mode == ack orelse Mode == down -> erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS}); - {ack, {error, _}} -> + {Mode, {error, _} = Reason} when Mode == ack orelse Mode == down -> + ?SLOG(error, #{ + msg => Op ++ "_failed", + transfer => Transfer, + reason => Reason + }), erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}); timeout -> + ?SLOG(error, #{ + msg => Op ++ "_timed_out", + transfer => Transfer + }), erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}) end. diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 38faebf03..083b4afcc 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -18,36 +18,31 @@ -include_lib("emqx/include/logger.hrl"). --export([start_link/3]). +-export([start_link/2]). -behaviour(gen_statem). -export([callback_mode/0]). -export([init/1]). -% -export([list_local_fragments/3]). -% -export([list_remote_fragments/3]). -% -export([start_assembling/3]). -export([handle_event/4]). -% -export([handle_continue/2]). -% -export([handle_call/3]). -% -export([handle_cast/2]). - -record(st, { storage :: _Storage, transfer :: emqx_ft:transfer(), assembly :: _TODO, file :: {file:filename(), io:device(), term()} | undefined, - hash, - callback :: fun((ok | {error, term()}) -> any()) + hash }). --define(RPC_LIST_TIMEOUT, 1000). --define(RPC_READSEG_TIMEOUT, 5000). +-define(NAME(Transfer), {n, l, {?MODULE, Transfer}}). +-define(REF(Transfer), {via, gproc, ?NAME(Transfer)}). %% -start_link(Storage, Transfer, Callback) -> - gen_statem:start_link(?MODULE, {Storage, Transfer, Callback}, []). +start_link(Storage, Transfer) -> + %% TODO + %% Additional callbacks? They won't survive restarts by the supervisor, which brings a + %% question if we even need to retry with the help of supervisor. + gen_statem:start_link(?REF(Transfer), ?MODULE, {Storage, Transfer}, []). %% @@ -56,16 +51,21 @@ start_link(Storage, Transfer, Callback) -> callback_mode() -> handle_event_function. -init({Storage, Transfer, Callback}) -> +init({Storage, Transfer}) -> St = #st{ storage = Storage, transfer = Transfer, assembly = emqx_ft_assembly:new(), - hash = crypto:hash_init(sha256), - callback = Callback + hash = crypto:hash_init(sha256) }, - {ok, list_local_fragments, St, ?internal([])}. + {ok, idle, St}. +handle_event(info, kickoff, idle, St) -> + % NOTE + % Someone's told us to start the work, which usually means that it has set up a monitor. + % We could wait for this message and handle it at the end of the assembling rather than at + % the beginning, however it would make error handling much more messier. + {next_state, list_local_fragments, St, ?internal([])}; handle_event(internal, _, list_local_fragments, St = #st{assembly = Asm}) -> % TODO: what we do with non-transients errors here (e.g. `eacces`)? {ok, Fragments} = emqx_ft_storage_fs:list(St#st.storage, St#st.transfer, fragment), @@ -76,10 +76,10 @@ handle_event(internal, _, list_local_fragments, St = #st{assembly = Asm}) -> {next_state, start_assembling, NSt, ?internal([])}; {incomplete, _} -> Nodes = mria_mnesia:running_nodes() -- [node()], - {next_state, {list_remote_fragments, Nodes}, NSt, ?internal([])} + {next_state, {list_remote_fragments, Nodes}, NSt, ?internal([])}; % TODO: recovery? - % {error, _} = Reason -> - % {stop, Reason} + {error, _} = Error -> + {stop, {shutdown, Error}} end; handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> % TODO @@ -107,12 +107,14 @@ handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> {next_state, start_assembling, NSt, ?internal([])}; % TODO: retries / recovery? {incomplete, _} = Status -> - {next_state, {failure, {error, Status}}, NSt, ?internal([])} + {stop, {shutdown, {error, Status}}}; + {error, _} = Error -> + {stop, {shutdown, Error}} end; handle_event(internal, _, start_assembling, St = #st{assembly = Asm}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), Coverage = emqx_ft_assembly:coverage(Asm), - % TODO: errors + % TODO: better error handling {ok, Handle} = emqx_ft_storage_fs:open_file(St#st.storage, St#st.transfer, Filemeta), {next_state, {assemble, Coverage}, St#st{file = Handle}, ?internal([])}; handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> @@ -120,50 +122,16 @@ handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % Currently, race is possible between getting segment info from the remote node and % this node garbage collecting the segment itself. % TODO: pipelining - case pread(Node, Segment, St) of - {ok, Content} -> - case emqx_ft_storage_fs:write(St#st.file, Content) of - {ok, NHandle} -> - {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])}; - %% TODO: better error handling - {error, _} = Error -> - {next_state, {failure, Error}, St, ?internal([])} - end; - {error, _} = Error -> - %% TODO: better error handling - {next_state, {failure, Error}, St, ?internal([])} - end; + % TODO: better error handling + {ok, Content} = pread(Node, Segment, St), + {ok, NHandle} = emqx_ft_storage_fs:write(St#st.file, Content), + {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])}; handle_event(internal, _, {assemble, []}, St = #st{}) -> {next_state, complete, St, ?internal([])}; -handle_event(internal, _, complete, St = #st{assembly = Asm, file = Handle, callback = Callback}) -> +handle_event(internal, _, complete, St = #st{assembly = Asm, file = Handle}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), Result = emqx_ft_storage_fs:complete(St#st.storage, St#st.transfer, Filemeta, Handle), - _ = safe_apply(Callback, Result), - {stop, shutdown}; -handle_event(internal, _, {failure, Error}, #st{callback = Callback}) -> - _ = safe_apply(Callback, Error), - {stop, Error}. - -% handle_continue(list_local, St = #st{storage = Storage, transfer = Transfer, assembly = Asm}) -> -% % TODO: what we do with non-transients errors here (e.g. `eacces`)? -% {ok, Fragments} = emqx_ft_storage_fs:list(Storage, Transfer), -% NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(Asm, node(), Fragments)), -% NSt = St#st{assembly = NAsm}, -% case emqx_ft_assembly:status(NAsm) of -% complete -> -% {noreply, NSt, {continue}}; -% {more, _} -> -% error(noimpl); -% {error, _} -> -% error(noimpl) -% end, -% {noreply, St}. - -% handle_call(_Call, _From, St) -> -% {reply, {error, badcall}, St}. - -% handle_cast(_Cast, St) -> -% {noreply, St}. + {stop, {shutdown, Result}}. pread(Node, Segment, St) when Node =:= node() -> emqx_ft_storage_fs:pread(St#st.storage, St#st.transfer, Segment, 0, segsize(Segment)); diff --git a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl index 17aa4d998..34783cbd3 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl @@ -17,7 +17,7 @@ -module(emqx_ft_assembler_sup). -export([start_link/0]). --export([start_child/3]). +-export([ensure_child/2]). -behaviour(supervisor). -export([init/1]). @@ -25,13 +25,18 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). -start_child(Storage, Transfer, Callback) -> +ensure_child(Storage, Transfer) -> Childspec = #{ - id => {Storage, Transfer}, - start => {emqx_ft_assembler, start_link, [Storage, Transfer, Callback]}, + id => Transfer, + start => {emqx_ft_assembler, start_link, [Storage, Transfer]}, restart => temporary }, - supervisor:start_child(?MODULE, Childspec). + case supervisor:start_child(?MODULE, Childspec) of + {ok, Pid} -> + {ok, Pid}; + {error, {already_started, Pid}} -> + {ok, Pid} + end. init(_) -> SupFlags = #{ diff --git a/apps/emqx_ft/src/emqx_ft_responder.erl b/apps/emqx_ft/src/emqx_ft_responder.erl index 7b9220774..fb823c433 100644 --- a/apps/emqx_ft/src/emqx_ft_responder.erl +++ b/apps/emqx_ft/src/emqx_ft_responder.erl @@ -25,6 +25,7 @@ %% API -export([start/3]). +-export([kickoff/2]). -export([ack/2]). %% Supervisor API @@ -35,7 +36,7 @@ -define(REF(Key), {via, gproc, {n, l, {?MODULE, Key}}}). -type key() :: term(). --type respfun() :: fun(({ack, _Result} | timeout) -> _SideEffect). +-type respfun() :: fun(({ack, _Result} | {down, _Result} | timeout) -> _SideEffect). %%-------------------------------------------------------------------- %% API @@ -45,6 +46,10 @@ start(Key, RespFun, Timeout) -> emqx_ft_responder_sup:start_child(Key, RespFun, Timeout). +-spec kickoff(key(), pid()) -> ok. +kickoff(Key, Pid) -> + gen_server:call(?REF(Key), {kickoff, Pid}). + -spec ack(key(), _Result) -> _Return. ack(Key, Result) -> % TODO: it's possible to avoid term copy @@ -63,8 +68,13 @@ init({Key, RespFun, Timeout}) -> _TRef = erlang:send_after(Timeout, self(), timeout), {ok, {Key, RespFun}}. +handle_call({kickoff, Pid}, _From, St) -> + % TODO: more state? + _MRef = erlang:monitor(process, Pid), + _ = Pid ! kickoff, + {reply, ok, St}; handle_call({ack, Result}, _From, {Key, RespFun}) -> - Ret = apply(RespFun, [Key, {ack, Result}]), + Ret = apply(RespFun, [{ack, Result}]), ?tp(ft_responder_ack, #{key => Key, result => Result, return => Ret}), {stop, {shutdown, Ret}, Ret, undefined}; handle_call(Msg, _From, State) -> @@ -76,9 +86,13 @@ handle_cast(Msg, State) -> {noreply, State}. handle_info(timeout, {Key, RespFun}) -> - Ret = apply(RespFun, [Key, timeout]), + Ret = apply(RespFun, [timeout]), ?tp(ft_responder_timeout, #{key => Key, return => Ret}), {stop, {shutdown, Ret}, undefined}; +handle_info({'DOWN', _MRef, process, _Pid, Reason}, {Key, RespFun}) -> + Ret = apply(RespFun, [{down, map_down_reason(Reason)}]), + ?tp(ft_responder_procdown, #{key => Key, reason => Reason, return => Ret}), + {stop, {shutdown, Ret}, undefined}; handle_info(Msg, State) -> ?SLOG(warning, #{msg => "unknown_message", info_msg => Msg}), {noreply, State}. @@ -86,6 +100,17 @@ handle_info(Msg, State) -> terminate(_Reason, undefined) -> ok; terminate(Reason, {Key, RespFun}) -> - Ret = apply(RespFun, [Key, timeout]), + Ret = apply(RespFun, [timeout]), ?tp(ft_responder_shutdown, #{key => Key, reason => Reason, return => Ret}), ok. + +map_down_reason(normal) -> + ok; +map_down_reason(shutdown) -> + ok; +map_down_reason({shutdown, Result}) -> + Result; +map_down_reason(noproc) -> + {error, noproc}; +map_down_reason(Error) -> + {error, {internal_error, Error}}. diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 0dd9d7989..6ca9e9ecb 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -20,7 +20,7 @@ [ store_filemeta/2, store_segment/2, - assemble/2, + assemble/1, ready_transfers/0, get_ready_transfer/1, @@ -43,12 +43,18 @@ %% Behaviour %%-------------------------------------------------------------------- +%% NOTE +%% An async task will wait for a `kickoff` message to start processing, to give some time +%% to set up monitors, etc. Async task will not explicitly report the processing result, +%% you are expected to receive and handle exit reason of the process, which is +%% -type result() :: `{shutdown, ok | {error, _}}`. + -callback store_filemeta(storage(), emqx_ft:transfer(), emqx_ft:filemeta()) -> - ok | {error, term()}. + ok | {async, pid()} | {error, term()}. -callback store_segment(storage(), emqx_ft:transfer(), emqx_ft:segment()) -> - ok | {error, term()}. --callback assemble(storage(), emqx_ft:transfer(), assemble_callback()) -> - {ok, pid()} | {error, term()}. + ok | {async, pid()} | {error, term()}. +-callback assemble(storage(), emqx_ft:transfer()) -> + ok | {async, pid()} | {error, term()}. -callback ready_transfers(storage()) -> {ok, [{ready_transfer_id(), ready_transfer_info()}]} | {error, term()}. -callback get_ready_transfer(storage(), ready_transfer_id()) -> @@ -59,22 +65,22 @@ %%-------------------------------------------------------------------- -spec store_filemeta(emqx_ft:transfer(), emqx_ft:filemeta()) -> - ok | {error, term()}. + ok | {async, pid()} | {error, term()}. store_filemeta(Transfer, FileMeta) -> Mod = mod(), Mod:store_filemeta(storage(), Transfer, FileMeta). -spec store_segment(emqx_ft:transfer(), emqx_ft:segment()) -> - ok | {error, term()}. + ok | {async, pid()} | {error, term()}. store_segment(Transfer, Segment) -> Mod = mod(), Mod:store_segment(storage(), Transfer, Segment). --spec assemble(emqx_ft:transfer(), assemble_callback()) -> - {ok, pid()} | {error, term()}. -assemble(Transfer, Callback) -> +-spec assemble(emqx_ft:transfer()) -> + ok | {async, pid()} | {error, term()}. +assemble(Transfer) -> Mod = mod(), - Mod:assemble(storage(), Transfer, Callback). + Mod:assemble(storage(), Transfer). -spec ready_transfers() -> {ok, [{ready_transfer_id(), ready_transfer_info()}]} | {error, term()}. ready_transfers() -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index c6cb09cf2..ef032a639 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -24,7 +24,7 @@ -export([store_segment/3]). -export([list/3]). -export([pread/5]). --export([assemble/3]). +-export([assemble/2]). -export([transfers/1]). @@ -168,11 +168,13 @@ pread(_Storage, _Transfer, Frag, Offset, Size) -> {error, Reason} end. --spec assemble(storage(), transfer(), fun((ok | {error, term()}) -> any())) -> +-spec assemble(storage(), transfer()) -> % {ok, _Assembler :: pid()} | {error, incomplete} | {error, badrpc} | {error, _TODO}. - {ok, _Assembler :: pid()} | {error, _TODO}. -assemble(Storage, Transfer, Callback) -> - emqx_ft_assembler_sup:start_child(Storage, Transfer, Callback). + {async, _Assembler :: pid()} | {error, _TODO}. +assemble(Storage, Transfer) -> + % TODO: ask cluster if the transfer is already assembled + {ok, Pid} = emqx_ft_assembler_sup:ensure_child(Storage, Transfer), + {async, Pid}. get_ready_transfer(_Storage, ReadyTransferId) -> case parse_ready_transfer_id(ReadyTransferId) of diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 6a125449f..0ca298265 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -361,7 +361,7 @@ t_assemble_crash(Config) -> C = ?config(client, Config), meck:new(emqx_ft_storage_fs), - meck:expect(emqx_ft_storage_fs, assemble, fun(_, _, _) -> meck:exception(error, oops) end), + meck:expect(emqx_ft_storage_fs, assemble, fun(_, _) -> meck:exception(error, oops) end), ?assertRCName( unspecified_error, diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index c5dbd418f..d4b619e43 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -37,7 +37,8 @@ all() -> ]. init_per_suite(Config) -> - Config. + Apps = application:ensure_all_started(gproc), + [{suite_apps, Apps} | Config]. end_per_suite(_Config) -> ok. @@ -83,9 +84,8 @@ t_assemble_empty_transfer(Config) -> ]}, emqx_ft_storage_fs:list(Storage, Transfer, fragment) ), - {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun on_assembly_finished/1), - {ok, Event} = ?block_until(#{?snk_kind := test_assembly_finished}), - ?assertMatch(#{result := ok}, Event), + Status = complete_assemble(Storage, Transfer), + ?assertEqual({shutdown, ok}, Status), ?assertEqual( {ok, <<>>}, % TODO @@ -132,9 +132,8 @@ t_assemble_complete_local_transfer(Config) -> Fragments ), - {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun on_assembly_finished/1), - {ok, Event} = ?block_until(#{?snk_kind := test_assembly_finished}), - ?assertMatch(#{result := ok}, Event), + Status = complete_assemble(Storage, Transfer), + ?assertEqual({shutdown, ok}, Status), AssemblyFilename = mk_assembly_filename(Config, Transfer, Filename), ?assertMatch( @@ -172,37 +171,32 @@ t_assemble_incomplete_transfer(Config) -> expire_at => 42 }, ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta), - Self = self(), - {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun(Result) -> - Self ! {test_assembly_finished, Result} - end), - receive - {test_assembly_finished, Result} -> - ?assertMatch({error, _}, Result) - after 1000 -> - ct:fail("Assembler did not called callback") - end. + Status = complete_assemble(Storage, Transfer), + ?assertMatch({shutdown, {error, _}}, Status). t_assemble_no_meta(Config) -> Storage = storage(Config), Transfer = {?CLIENTID2, ?config(file_id, Config)}, - Self = self(), - {ok, _AsmPid} = emqx_ft_storage_fs:assemble(Storage, Transfer, fun(Result) -> - Self ! {test_assembly_finished, Result} - end), + Status = complete_assemble(Storage, Transfer), + ?assertMatch({shutdown, {error, {incomplete, _}}}, Status). + +complete_assemble(Storage, Transfer) -> + complete_assemble(Storage, Transfer, 1000). + +complete_assemble(Storage, Transfer, Timeout) -> + {async, Pid} = emqx_ft_storage_fs:assemble(Storage, Transfer), + MRef = erlang:monitor(process, Pid), + Pid ! kickoff, receive - {test_assembly_finished, Result} -> - ?assertMatch({error, _}, Result) - after 1000 -> - ct:fail("Assembler did not called callback") + {'DOWN', MRef, process, Pid, Result} -> + Result + after Timeout -> + ct:fail("Assembler did not finish in time") end. mk_assembly_filename(Config, {ClientID, FileID}, Filename) -> filename:join([?config(storage_root, Config), ClientID, FileID, result, Filename]). -on_assembly_finished(Result) -> - ?tp(test_assembly_finished, #{result => Result}). - %% t_list_transfers(Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl index 447d41f11..1674d05e4 100644 --- a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl @@ -40,7 +40,7 @@ end_per_testcase(_Case, _Config) -> t_start_ack(_Config) -> Key = <<"test">>, - DefaultAction = fun(_Key, {ack, Ref}) -> Ref end, + DefaultAction = fun({ack, Ref}) -> Ref end, ?assertMatch( {ok, _Pid}, emqx_ft_responder:start(Key, DefaultAction, 1000) @@ -62,7 +62,7 @@ t_start_ack(_Config) -> t_timeout(_Config) -> Key = <<"test">>, Self = self(), - DefaultAction = fun(K, timeout) -> Self ! {timeout, K} end, + DefaultAction = fun(timeout) -> Self ! {timeout, Key} end, {ok, _Pid} = emqx_ft_responder:start(Key, DefaultAction, 20), receive {timeout, Key} -> @@ -89,7 +89,7 @@ t_timeout(_Config) -> % ). t_unknown_msgs(_Config) -> - {ok, Pid} = emqx_ft_responder:start(make_ref(), fun(_, _) -> ok end, 100), + {ok, Pid} = emqx_ft_responder:start(make_ref(), fun(_) -> ok end, 100), Pid ! {unknown_msg, <<"test">>}, ok = gen_server:cast(Pid, {unknown_msg, <<"test">>}), ?assertEqual( From 6d9f780313dbf05ed6f3e6775ff69ec001cd20af Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 16 Feb 2023 18:35:10 +0300 Subject: [PATCH 032/156] chore: remove unused code --- apps/emqx_ft/src/emqx_ft_assembler.erl | 15 --------------- apps/emqx_ft/test/emqx_ft_responder_SUITE.erl | 14 -------------- 2 files changed, 29 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 083b4afcc..8ba7e42d4 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -16,8 +16,6 @@ -module(emqx_ft_assembler). --include_lib("emqx/include/logger.hrl"). - -export([start_link/2]). -behaviour(gen_statem). @@ -142,16 +140,3 @@ pread(Node, Segment, St) -> segsize(#{fragment := {segment, Info}}) -> maps:get(size, Info). - -safe_apply(Callback, Result) -> - try apply(Callback, [Result]) of - _ -> ok - catch - Class:Reason:Stacktrace -> - ?SLOG(error, #{ - msg => "safe_apply_failed", - class => Class, - reason => Reason, - stacktrace => Stacktrace - }) - end. diff --git a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl index 1674d05e4..e447ba03b 100644 --- a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl @@ -20,7 +20,6 @@ -compile(nowarn_export_all). -include_lib("stdlib/include/assert.hrl"). --include_lib("emqx/include/asserts.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). @@ -75,19 +74,6 @@ t_timeout(_Config) -> emqx_ft_responder:ack(Key, oops) ). -% t_action_exception(_Config) -> -% Key = <<"test">>, -% DefaultAction = fun(K) -> error({oops, K}) end, -% ?assertWaitEvent( -% emqx_ft_responder:start(Key, DefaultAction, 10), -% #{?snk_kind := ft_timeout_action_applied, key := <<"test">>}, -% 1000 -% ), -% ?assertEqual( -% {error, not_found}, -% emqx_ft_responder:ack(Key, oops) -% ). - t_unknown_msgs(_Config) -> {ok, Pid} = emqx_ft_responder:start(make_ref(), fun(_) -> ok end, 100), Pid ! {unknown_msg, <<"test">>}, From 16fa55e19aced78067859e9d5a9eabe318eab50b Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 15 Feb 2023 21:49:33 +0200 Subject: [PATCH 033/156] feat(ft): add API tests --- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 177 ++++++++++++++++++ .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 8 +- .../test/emqx_ft_storage_fs_reader_SUITE.erl | 5 +- 3 files changed, 185 insertions(+), 5 deletions(-) create mode 100644 apps/emqx_ft/test/emqx_ft_api_SUITE.erl diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl new file mode 100644 index 000000000..e420dac36 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -0,0 +1,177 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-define(assertInclude(Pattern, List), + ?assert( + lists:any( + fun(El) -> + case El of + Pattern -> true; + _ -> false + end + end, + List + ) + ) +). + +-import(emqx_mgmt_api_test_util, [request/3, uri/1]). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_mgmt_api_test_util:init_suite( + [emqx_conf, emqx_ft], set_special_configs(Config) + ), + ok = emqx_common_test_helpers:set_gen_rpc_stateless(), + Config. +end_per_suite(_Config) -> + ok = emqx_mgmt_api_test_util:end_suite([emqx_ft, emqx_conf]), + ok. + +set_special_configs(Config) -> + fun + (emqx_ft) -> + ok = emqx_config:put([file_transfer, storage], #{ + type => local, root => emqx_ft_test_helpers:ft_root(Config, node()) + }); + (_) -> + ok + end. + +init_per_testcase(Case, Config) -> + [{tc, Case} | Config]. +end_per_testcase(_Case, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_list_ready_transfers(Config) -> + ClientId = client_id(Config), + + ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, <<"data">>, node()), + + {ok, 200, Response} = request(get, uri(["file_transfer", "files"])), + + #{<<"files">> := Files} = emqx_json:decode(Response, [return_maps]), + + ?assertInclude( + #{<<"id">> := #{<<"clientid">> := ClientId, <<"fileid">> := <<"f1">>}}, + Files + ). + +%% This shouldn't happen in real life +%% but we need to test it anyway +t_list_ready_transfers_no_nodes(_Config) -> + _ = meck:new(mria_mnesia, [passthrough]), + _ = meck:expect(mria_mnesia, running_nodes, fun() -> [] end), + + ?assertMatch( + {ok, 503, _}, + request(get, uri(["file_transfer", "files"])) + ). + +t_download_transfer(Config) -> + ClientId = client_id(Config), + + ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, <<"data">>, node()), + + ?assertMatch( + {ok, 503, _}, + request( + get, + uri(["file_transfer", "file"]) ++ + query(#{ + clientid => ClientId, + fileid => <<"f1">> + }) + ) + ), + + ?assertMatch( + {ok, 503, _}, + request( + get, + uri(["file_transfer", "file"]) ++ + query(#{ + clientid => ClientId, + fileid => <<"f1">>, + node => <<"nonode@nohost">> + }) + ) + ), + + ?assertMatch( + {ok, 404, _}, + request( + get, + uri(["file_transfer", "file"]) ++ + query(#{ + clientid => ClientId, + fileid => <<"unknown_file">>, + node => node() + }) + ) + ), + + {ok, 200, Response} = request( + get, + uri(["file_transfer", "file"]) ++ + query(#{ + clientid => ClientId, + fileid => <<"f1">>, + node => node() + }) + ), + + ?assertEqual( + Response, + <<"data">> + ). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +client_id(Config) -> + atom_to_binary(?config(tc, Config), utf8). + +request(Method, Url) -> + request(Method, Url, []). + +query(Params) -> + KVs = lists:map(fun({K, V}) -> uri_encode(K) ++ "=" ++ uri_encode(V) end, maps:to_list(Params)), + "?" ++ string:join(KVs, "&"). + +uri_encode(T) -> + emqx_http_lib:uri_encode(to_list(T)). + +to_list(A) when is_atom(A) -> + atom_to_list(A); +to_list(B) when is_binary(B) -> + binary_to_list(B); +to_list(L) when is_list(L) -> + L. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index 20645bcd9..55f30300f 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -25,9 +25,11 @@ -define(assertInclude(Pattern, List), ?assert( lists:any( - fun - (Pattern) -> true; - (_) -> false + fun(El) -> + case El of + Pattern -> true; + _ -> false + end end, List ) diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl index 0ac5d2844..db15b8660 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl @@ -25,11 +25,12 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_ft]), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft]), + ok = emqx_common_test_helpers:set_gen_rpc_stateless(), Config. end_per_suite(_Config) -> - ok = emqx_common_test_helpers:stop_apps([emqx_ft]), + ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), ok. init_per_testcase(_Case, Config) -> From bc0a15afd7cb58931a7d46b73c032ab264660505 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 16 Feb 2023 00:28:32 +0200 Subject: [PATCH 034/156] feat(ft): fix test for CI --- apps/emqx/include/asserts.hrl | 14 ++++++++++++++ apps/emqx_ft/docker-ct | 0 apps/emqx_ft/test/emqx_ft_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 14 +------------- apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 16 ++-------------- 5 files changed, 18 insertions(+), 28 deletions(-) create mode 100644 apps/emqx_ft/docker-ct diff --git a/apps/emqx/include/asserts.hrl b/apps/emqx/include/asserts.hrl index 0e9f9477a..b3415fdf7 100644 --- a/apps/emqx/include/asserts.hrl +++ b/apps/emqx/include/asserts.hrl @@ -36,3 +36,17 @@ end ) ). + +-define(assertInclude(Pattern, List), + ?assert( + lists:any( + fun(El) -> + case El of + Pattern -> true; + _ -> false + end + end, + List + ) + ) +). diff --git a/apps/emqx_ft/docker-ct b/apps/emqx_ft/docker-ct new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 0ca298265..5b01c33cd 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -71,7 +71,7 @@ end_per_testcase(_Case, Config) -> ok. init_per_group(cluster, Config) -> - Node = emqx_ft_test_helpers:start_additional_node(Config, test2), + Node = emqx_ft_test_helpers:start_additional_node(Config, emqx_ft1), [{additional_node, Node} | Config]; init_per_group(_Group, Config) -> Config. diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index e420dac36..d38301a0f 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -22,19 +22,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). --define(assertInclude(Pattern, List), - ?assert( - lists:any( - fun(El) -> - case El of - Pattern -> true; - _ -> false - end - end, - List - ) - ) -). +-include_lib("emqx/include/asserts.hrl"). -import(emqx_mgmt_api_test_util, [request/3, uri/1]). diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index 55f30300f..d19d9c764 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -22,19 +22,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). --define(assertInclude(Pattern, List), - ?assert( - lists:any( - fun(El) -> - case El of - Pattern -> true; - _ -> false - end - end, - List - ) - ) -). +-include_lib("emqx/include/asserts.hrl"). all() -> [ @@ -74,7 +62,7 @@ end_per_testcase(_Case, _Config) -> ok. init_per_group(cluster, Config) -> - Node = emqx_ft_test_helpers:start_additional_node(Config, test2), + Node = emqx_ft_test_helpers:start_additional_node(Config, emqx_ft_storage_fs1), [{additional_node, Node} | Config]; init_per_group(_Group, Config) -> Config. From c6b3f2c2caaa93e7fb0a05f99adc231427c2729d Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 16 Feb 2023 12:28:40 +0200 Subject: [PATCH 035/156] feat(ft): add schema descriptions --- apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 13 ++++++++++++- apps/emqx_ft/src/emqx_ft_schema.erl | 12 +++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf index 7e057fdf8..576d7d8fe 100644 --- a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -1,6 +1,17 @@ emqx_ft_schema { - local { + storage { + desc { + en: "Storage settings for file transfer." + zh: "文件传输的存储设置。" + } + label: { + en: "Storage settings" + zh: "存储设置" + } + } + + local_type { desc { en: "Use local file system to store uploaded files and temporary data." zh: "使用本地文件系统来存储上传的文件和临时数据。" diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index deb2cae6f..f17c957a9 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -21,7 +21,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). --export([namespace/0, roots/0, fields/1, tags/0]). +-export([namespace/0, roots/0, fields/1, tags/0, desc/1]). -export([schema/1]). @@ -49,7 +49,8 @@ fields(file_transfer) -> {storage, #{ type => hoconsc:union([ hoconsc:ref(?MODULE, local_storage) - ]) + ]), + desc => ?DESC("storage") }} ]; fields(local_storage) -> @@ -58,7 +59,7 @@ fields(local_storage) -> type => local, default => local, required => false, - desc => ?DESC("local") + desc => ?DESC("local_type") }}, {root, #{ type => binary(), @@ -67,6 +68,11 @@ fields(local_storage) -> }} ]. +desc(file_transfer) -> + "File transfer settings"; +desc(local_storage) -> + "File transfer local storage settings". + schema(filemeta) -> #{ roots => [ From 1c61165a9185661127820e40c2bc3e8f31fc9843 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 16 Feb 2023 17:38:54 +0300 Subject: [PATCH 036/156] fix(ft): fix emqx_cm takover --- apps/emqx/src/emqx_cm.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index 766acd4b8..90b279fa4 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -312,6 +312,7 @@ open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> Session1 = emqx_persistent_session:persist( ClientInfo, ConnInfo, Session ), + register_channel(ClientId, Self, ConnInfo), {ok, #{ session => clean_session(Session1), present => true, From 2c9cd1397d5196ba078339faf4731ebdb74e0ea0 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 16 Feb 2023 18:13:14 +0200 Subject: [PATCH 037/156] chore(ft): fix `?tp` calls --- apps/emqx_ft/src/emqx_ft_responder.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_responder.erl b/apps/emqx_ft/src/emqx_ft_responder.erl index fb823c433..923e5ece0 100644 --- a/apps/emqx_ft/src/emqx_ft_responder.erl +++ b/apps/emqx_ft/src/emqx_ft_responder.erl @@ -75,7 +75,7 @@ handle_call({kickoff, Pid}, _From, St) -> {reply, ok, St}; handle_call({ack, Result}, _From, {Key, RespFun}) -> Ret = apply(RespFun, [{ack, Result}]), - ?tp(ft_responder_ack, #{key => Key, result => Result, return => Ret}), + ?tp(debug, ft_responder_ack, #{key => Key, result => Result, return => Ret}), {stop, {shutdown, Ret}, Ret, undefined}; handle_call(Msg, _From, State) -> ?SLOG(warning, #{msg => "unknown_call", call_msg => Msg}), @@ -87,11 +87,11 @@ handle_cast(Msg, State) -> handle_info(timeout, {Key, RespFun}) -> Ret = apply(RespFun, [timeout]), - ?tp(ft_responder_timeout, #{key => Key, return => Ret}), + ?tp(debug, ft_responder_timeout, #{key => Key, return => Ret}), {stop, {shutdown, Ret}, undefined}; handle_info({'DOWN', _MRef, process, _Pid, Reason}, {Key, RespFun}) -> Ret = apply(RespFun, [{down, map_down_reason(Reason)}]), - ?tp(ft_responder_procdown, #{key => Key, reason => Reason, return => Ret}), + ?tp(debug, ft_responder_procdown, #{key => Key, reason => Reason, return => Ret}), {stop, {shutdown, Ret}, undefined}; handle_info(Msg, State) -> ?SLOG(warning, #{msg => "unknown_message", info_msg => Msg}), @@ -101,7 +101,7 @@ terminate(_Reason, undefined) -> ok; terminate(Reason, {Key, RespFun}) -> Ret = apply(RespFun, [timeout]), - ?tp(ft_responder_shutdown, #{key => Key, reason => Reason, return => Ret}), + ?tp(debug, ft_responder_shutdown, #{key => Key, reason => Reason, return => Ret}), ok. map_down_reason(normal) -> From 228bf1a0cea5b11ecf5ffb81aabbe99c6ce3ef54 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 16 Feb 2023 20:32:12 +0200 Subject: [PATCH 038/156] chore(ft): fix typing issues --- apps/emqx_ft/src/emqx_ft.erl | 16 ++++++++-------- apps/emqx_ft/src/emqx_ft_responder.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 3 +-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 348916920..d55d5cad9 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -184,9 +184,9 @@ on_init(PacketId, Msg, Transfer) -> ok -> emqx_ft_responder:ack(PacketKey, ok); % Storage operation started, packet will be acked by the responder - {async, Pid} -> - ok = emqx_ft_responder:kickoff(PacketKey, Pid), - ok; + % {async, Pid} -> + % ok = emqx_ft_responder:kickoff(PacketKey, Pid), + % ok; %% Storage operation failed, ack through the responder {error, _} = Error -> emqx_ft_responder:ack(PacketKey, Error) @@ -227,9 +227,9 @@ on_segment(PacketId, Msg, Transfer, Offset, Checksum) -> case store_segment(Transfer, Segment) of ok -> emqx_ft_responder:ack(PacketKey, ok); - {async, Pid} -> - ok = emqx_ft_responder:kickoff(PacketKey, Pid), - ok; + % {async, Pid} -> + % ok = emqx_ft_responder:kickoff(PacketKey, Pid), + % ok; {error, _} = Error -> emqx_ft_responder:ack(PacketKey, Error) end @@ -251,8 +251,8 @@ on_fin(PacketId, Msg, Transfer, Checksum) -> with_responder(FinPacketKey, Callback, ?ASSEMBLE_TIMEOUT, fun() -> case assemble(Transfer) of %% Assembling completed, ack through the responder right away - ok -> - emqx_ft_responder:ack(FinPacketKey, ok); + % ok -> + % emqx_ft_responder:ack(FinPacketKey, ok); %% Assembling started, packet will be acked by the responder {async, Pid} -> ok = emqx_ft_responder:kickoff(FinPacketKey, Pid), diff --git a/apps/emqx_ft/src/emqx_ft_responder.erl b/apps/emqx_ft/src/emqx_ft_responder.erl index 923e5ece0..cbbfbe687 100644 --- a/apps/emqx_ft/src/emqx_ft_responder.erl +++ b/apps/emqx_ft/src/emqx_ft_responder.erl @@ -42,7 +42,7 @@ %% API %% ------------------------------------------------------------------- --spec start(key(), timeout(), respfun()) -> startlink_ret(). +-spec start(key(), respfun(), timeout()) -> startlink_ret(). start(Key, RespFun, Timeout) -> emqx_ft_responder_sup:start_child(Key, RespFun, Timeout). diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index ef032a639..81fed0f21 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -78,8 +78,7 @@ -define(MANIFEST, "MANIFEST.json"). -define(SEGMENT, "SEG"). -%% TODO --type storage() :: emqx_config:config(). +-type storage() :: emqx_ft_storage:storage(). %% Store manifest in the backing filesystem. %% Atomic operation. From c073914f7570982abcaea7e427058dc4003d431f Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 17 Feb 2023 13:56:22 +0300 Subject: [PATCH 039/156] chore(ft): fix gen_rpc flakyness --- apps/emqx/test/emqx_common_test_helpers.erl | 18 +----------------- apps/emqx_ft/docker-ct | 0 apps/emqx_ft/test/emqx_ft_SUITE.erl | 5 ++--- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 1 + apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 5 ++--- .../test/emqx_ft_storage_fs_reader_SUITE.erl | 5 ++--- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 2 +- 8 files changed, 10 insertions(+), 28 deletions(-) delete mode 100644 apps/emqx_ft/docker-ct diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 18a3d9f3e..5dd587a4e 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -63,7 +63,6 @@ ]). -export([ - set_gen_rpc_stateless/0, emqx_cluster/1, emqx_cluster/2, start_epmd/0, @@ -293,12 +292,7 @@ read_schema_configs(no_schema, _ConfigFile) -> ok; read_schema_configs(Schema, ConfigFile) -> NewConfig = generate_config(Schema, ConfigFile), - lists:foreach( - fun({App, Configs}) -> - [application:set_env(App, Par, Value) || {Par, Value} <- Configs] - end, - NewConfig - ). + application:set_env(NewConfig). generate_config(SchemaModule, ConfigFile) when is_atom(SchemaModule) -> {ok, Conf0} = hocon:load(ConfigFile, #{format => richmap}), @@ -617,16 +611,6 @@ ensure_quic_listener(Name, UdpPort, ExtraSettings) -> listener_ports => [{Type :: tcp | ssl | ws | wss, inet:port_number()}] }. --spec set_gen_rpc_stateless() -> ok. -set_gen_rpc_stateless() -> - %% When many tests run in an obscure order, it may occur that - %% `gen_rpc` started with its default settings before `emqx_conf`. - %% `gen_rpc` and `emqx_conf` have different default `port_discovery` modes, - %% so we reinitialize `gen_rpc` explicitly. - ok = application:stop(gen_rpc), - ok = application:set_env(gen_rpc, port_discovery, stateless), - ok = application:start(gen_rpc). - -spec emqx_cluster(cluster_spec()) -> [{shortname(), node_opts()}]. emqx_cluster(Specs) -> emqx_cluster(Specs, #{}). diff --git a/apps/emqx_ft/docker-ct b/apps/emqx_ft/docker-ct deleted file mode 100644 index e69de29bb..000000000 diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 5b01c33cd..8708f9808 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -42,12 +42,11 @@ groups() -> ]. init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], set_special_configs(Config)), - ok = emqx_common_test_helpers:set_gen_rpc_stateless(), + ok = emqx_common_test_helpers:start_apps([emqx_ft], set_special_configs(Config)), Config. end_per_suite(_Config) -> - ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok = emqx_common_test_helpers:stop_apps([emqx_ft]), ok. set_special_configs(Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index d38301a0f..40b5da58f 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -32,7 +32,7 @@ init_per_suite(Config) -> ok = emqx_mgmt_api_test_util:init_suite( [emqx_conf, emqx_ft], set_special_configs(Config) ), - ok = emqx_common_test_helpers:set_gen_rpc_stateless(), + {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. end_per_suite(_Config) -> ok = emqx_mgmt_api_test_util:end_suite([emqx_ft, emqx_conf]), diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index cca69796d..314b4a5f2 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -26,6 +26,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft]), + {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. end_per_suite(_Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index d19d9c764..cc3da8bc3 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -39,11 +39,10 @@ groups() -> ]. init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], set_special_configs(Config)), - ok = emqx_common_test_helpers:set_gen_rpc_stateless(), + ok = emqx_common_test_helpers:start_apps([emqx_ft], set_special_configs(Config)), Config. end_per_suite(_Config) -> - ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok = emqx_common_test_helpers:stop_apps([emqx_ft]), ok. set_special_configs(Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl index db15b8660..0ac5d2844 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl @@ -25,12 +25,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft]), - ok = emqx_common_test_helpers:set_gen_rpc_stateless(), + ok = emqx_common_test_helpers:start_apps([emqx_ft]), Config. end_per_suite(_Config) -> - ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok = emqx_common_test_helpers:stop_apps([emqx_ft]), ok. init_per_testcase(_Case, Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index ca854bda0..f7d8bc879 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -28,7 +28,7 @@ start_additional_node(Config, Node) -> [ {apps, [emqx_ft]}, {join_to, SelfNode}, - {configure_gen_rpc, false}, + {configure_gen_rpc, true}, {env_handler, fun (emqx_ft) -> ok = emqx_config:put([file_transfer, storage], #{ From 58115715ddbc11bf1a40d749d477424f8f967ed8 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 17 Feb 2023 17:27:38 +0300 Subject: [PATCH 040/156] fix(ft): require final size in `fin` packet Otherwise there are situations when it's not entirely clear if a transfer is really ready to be assembled. Since the `size` field in a filemeta is not required (and rightly so), we need client to tell us the final transfer size at the end of the process. Also synthesize a testcase to show why it's needed. Also worth noting that right now `fin` packets require final size, even if a client already told us the size through filemeta. The latter is regarded as serving informational purposes only (which means that, for example, it might differ from the final size, or some tranfer progress might show >100% somewhere because of that). --- apps/emqx/src/emqx_maybe.erl | 42 ++++ apps/emqx_ft/src/emqx_ft.erl | 26 ++- apps/emqx_ft/src/emqx_ft_assembler.erl | 10 +- apps/emqx_ft/src/emqx_ft_assembler_sup.erl | 6 +- apps/emqx_ft/src/emqx_ft_assembly.erl | 29 ++- apps/emqx_ft/src/emqx_ft_storage.erl | 10 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 8 +- apps/emqx_ft/test/emqx_ft_SUITE.erl | 212 ++++++++++++++++-- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 16 +- .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 15 +- 11 files changed, 295 insertions(+), 81 deletions(-) create mode 100644 apps/emqx/src/emqx_maybe.erl diff --git a/apps/emqx/src/emqx_maybe.erl b/apps/emqx/src/emqx_maybe.erl new file mode 100644 index 000000000..8c60d7ae7 --- /dev/null +++ b/apps/emqx/src/emqx_maybe.erl @@ -0,0 +1,42 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_maybe). + +-include_lib("emqx/include/types.hrl"). + +-export([to_list/1]). +-export([from_list/1]). +-export([apply/2]). + +-spec to_list(maybe(A)) -> [A]. +to_list(undefined) -> + []; +to_list(Term) -> + [Term]. + +-spec from_list([A]) -> maybe(A). +from_list([]) -> + undefined; +from_list([Term]) -> + Term. + +-spec apply(fun((maybe(A)) -> maybe(A)), maybe(A)) -> + maybe(A). +apply(_Fun, undefined) -> + undefined; +apply(Fun, Term) when is_function(Fun) -> + erlang:apply(Fun, [Term]). diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index d55d5cad9..266d38fe8 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -40,6 +40,7 @@ -export_type([ clientid/0, transfer/0, + bytes/0, offset/0, filemeta/0, segment/0 @@ -145,10 +146,11 @@ on_file_command(PacketId, Msg, FileCommand) -> case string:split(FileCommand, <<"/">>, all) of [FileId, <<"init">>] -> on_init(PacketId, Msg, transfer(Msg, FileId)); - [FileId, <<"fin">>] -> - on_fin(PacketId, Msg, transfer(Msg, FileId), undefined); - [FileId, <<"fin">>, Checksum] -> - on_fin(PacketId, Msg, transfer(Msg, FileId), Checksum); + [FileId, <<"fin">>, FinalSizeBin | ChecksumL] -> + validate([{size, FinalSizeBin}], fun([FinalSize]) -> + Checksum = emqx_maybe:from_list(ChecksumL), + on_fin(PacketId, Msg, transfer(Msg, FileId), FinalSize, Checksum) + end); [FileId, <<"abort">>] -> on_abort(Msg, transfer(Msg, FileId)); [FileId, OffsetBin] -> @@ -235,12 +237,13 @@ on_segment(PacketId, Msg, Transfer, Offset, Checksum) -> end end). -on_fin(PacketId, Msg, Transfer, Checksum) -> +on_fin(PacketId, Msg, Transfer, FinalSize, Checksum) -> ?SLOG(info, #{ msg => "on_fin", mqtt_msg => Msg, packet_id => PacketId, transfer => Transfer, + final_size => FinalSize, checksum => Checksum }), %% TODO: handle checksum? Do we need it? @@ -249,7 +252,7 @@ on_fin(PacketId, Msg, Transfer, Checksum) -> ?MODULE:on_complete("assemble", FinPacketKey, Transfer, Result) end, with_responder(FinPacketKey, Callback, ?ASSEMBLE_TIMEOUT, fun() -> - case assemble(Transfer) of + case assemble(Transfer, FinalSize) of %% Assembling completed, ack through the responder right away % ok -> % emqx_ft_responder:ack(FinPacketKey, ok); @@ -298,9 +301,9 @@ store_segment(Transfer, Segment) -> {error, {internal_error, E}} end. -assemble(Transfer) -> +assemble(Transfer, FinalSize) -> try - emqx_ft_storage:assemble(Transfer) + emqx_ft_storage:assemble(Transfer, FinalSize) catch C:E:S -> ?SLOG(error, #{ @@ -359,6 +362,13 @@ do_validate([{offset, Offset} | Rest], Parsed) -> _ -> {error, {invalid_offset, Offset}} end; +do_validate([{size, Size} | Rest], Parsed) -> + case string:to_integer(Size) of + {Int, <<>>} -> + do_validate(Rest, [Int | Parsed]); + _ -> + {error, {invalid_size, Size}} + end; do_validate([{checksum, Checksum} | Rest], Parsed) -> case parse_checksum(Checksum) of {ok, Bin} -> diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 8ba7e42d4..275d16499 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -16,7 +16,7 @@ -module(emqx_ft_assembler). --export([start_link/2]). +-export([start_link/3]). -behaviour(gen_statem). -export([callback_mode/0]). @@ -36,11 +36,11 @@ %% -start_link(Storage, Transfer) -> +start_link(Storage, Transfer, Size) -> %% TODO %% Additional callbacks? They won't survive restarts by the supervisor, which brings a %% question if we even need to retry with the help of supervisor. - gen_statem:start_link(?REF(Transfer), ?MODULE, {Storage, Transfer}, []). + gen_statem:start_link(?REF(Transfer), ?MODULE, {Storage, Transfer, Size}, []). %% @@ -49,11 +49,11 @@ start_link(Storage, Transfer) -> callback_mode() -> handle_event_function. -init({Storage, Transfer}) -> +init({Storage, Transfer, Size}) -> St = #st{ storage = Storage, transfer = Transfer, - assembly = emqx_ft_assembly:new(), + assembly = emqx_ft_assembly:new(Size), hash = crypto:hash_init(sha256) }, {ok, idle, St}. diff --git a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl index 34783cbd3..bdefdac47 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl @@ -17,7 +17,7 @@ -module(emqx_ft_assembler_sup). -export([start_link/0]). --export([ensure_child/2]). +-export([ensure_child/3]). -behaviour(supervisor). -export([init/1]). @@ -25,10 +25,10 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). -ensure_child(Storage, Transfer) -> +ensure_child(Storage, Transfer, Size) -> Childspec = #{ id => Transfer, - start => {emqx_ft_assembler, start_link, [Storage, Transfer]}, + start => {emqx_ft_assembler, start_link, [Storage, Transfer, Size]}, restart => temporary }, case supervisor:start_child(?MODULE, Childspec) of diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index 0f2729f42..2d78b540b 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -16,7 +16,7 @@ -module(emqx_ft_assembly). --export([new/0]). +-export([new/1]). -export([append/3]). -export([update/1]). @@ -35,12 +35,12 @@ size }). -new() -> +new(Size) -> #asm{ status = {incomplete, {missing, filemeta}}, meta = orddict:new(), segs = orddict:new(), - size = 0 + size = Size }. append(Asm, Node, Fragments) when is_list(Fragments) -> @@ -114,8 +114,7 @@ append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) -> % In theory it's possible to have two segments with same offset + size on % different nodes but with differing content. We'd need a checksum to % be able to disambiguate them though. - segs = orddict:store({Offset, locality(Node), -End, Node}, Fragment, Asm#asm.segs), - size = max(End, Asm#asm.size) + segs = orddict:store({Offset, locality(Node), -End, Node}, Fragment, Asm#asm.segs) }. coverage([{{Offset, _, _, _}, _Segment} | Rest], Cursor, Sz) when Offset < Cursor -> @@ -186,7 +185,7 @@ segsize(#{fragment := {segment, Info}}) -> incomplete_new_test() -> ?assertEqual( {incomplete, {missing, filemeta}}, - status(update(new())) + status(update(new(42))) ). incomplete_test() -> @@ -194,7 +193,7 @@ incomplete_test() -> {incomplete, {missing, filemeta}}, status( update( - append(new(), node(), [ + append(new(142), node(), [ segment(p1, 0, 42), segment(p1, 42, 100) ]) @@ -203,13 +202,13 @@ incomplete_test() -> ). consistent_test() -> - Asm1 = append(new(), n1, [filemeta(m1, "blarg")]), + Asm1 = append(new(42), n1, [filemeta(m1, "blarg")]), Asm2 = append(Asm1, n2, [segment(s2, 0, 42)]), Asm3 = append(Asm2, n3, [filemeta(m3, "blarg")]), ?assertMatch({complete, _}, status(meta, Asm3)). inconsistent_test() -> - Asm1 = append(new(), node(), [segment(s1, 0, 42)]), + Asm1 = append(new(42), node(), [segment(s1, 0, 42)]), Asm2 = append(Asm1, n1, [filemeta(m1, "blarg")]), Asm3 = append(Asm2, n2, [segment(s2, 0, 42), filemeta(m1, "blorg")]), Asm4 = append(Asm3, n3, [filemeta(m3, "blarg")]), @@ -231,7 +230,7 @@ simple_coverage_test() -> {Node, segment(n3, 50, 50)}, {Node, segment(n4, 10, 10)} ], - Asm = append_many(new(), Segs), + Asm = append_many(new(100), Segs), ?assertMatch( {complete, [ @@ -256,7 +255,7 @@ redundant_coverage_test() -> {Node, segment(n7, 50, 10)}, {node1, segment(n8, 40, 10)} ], - Asm = append_many(new(), Segs), + Asm = append_many(new(70), Segs), ?assertMatch( {complete, [ @@ -279,7 +278,7 @@ redundant_coverage_prefer_local_test() -> {Node, segment(n5, 30, 10)}, {Node, segment(n6, 20, 10)} ], - Asm = append_many(new(), Segs), + Asm = append_many(new(40), Segs), ?assertMatch( {complete, [ @@ -301,7 +300,7 @@ missing_coverage_test() -> {node2, segment(n4, 50, 50)}, {Node, segment(n5, 40, 60)} ], - Asm = append_many(new(), Segs), + Asm = append_many(new(100), Segs), ?assertEqual( % {incomplete, {missing, {segment, 30, 40}}}, ??? {incomplete, {missing, {segment, 20, 40}}}, @@ -314,7 +313,7 @@ missing_end_coverage_test() -> {Node, segment(n1, 0, 15)}, {node1, segment(n3, 10, 10)} ], - Asm = append_many(new(), Segs), + Asm = append_many(new(20), Segs), ?assertEqual( {incomplete, {missing, {segment, 15, 20}}}, status(coverage, Asm) @@ -328,7 +327,7 @@ missing_coverage_with_redudancy_test() -> {node43, segment(n4, 10, 50)}, {node(), segment(n5, 40, 60)} ], - Asm = append_many(new(), Segs), + Asm = append_many(new(100), Segs), ?assertEqual( % {incomplete, {missing, {segment, 50, 60}}}, ??? {incomplete, {missing, {segment, 20, 40}}}, diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 6ca9e9ecb..7a95a0454 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -20,7 +20,7 @@ [ store_filemeta/2, store_segment/2, - assemble/1, + assemble/2, ready_transfers/0, get_ready_transfer/1, @@ -53,7 +53,7 @@ ok | {async, pid()} | {error, term()}. -callback store_segment(storage(), emqx_ft:transfer(), emqx_ft:segment()) -> ok | {async, pid()} | {error, term()}. --callback assemble(storage(), emqx_ft:transfer()) -> +-callback assemble(storage(), emqx_ft:transfer(), _Size :: emqx_ft:bytes()) -> ok | {async, pid()} | {error, term()}. -callback ready_transfers(storage()) -> {ok, [{ready_transfer_id(), ready_transfer_info()}]} | {error, term()}. @@ -76,11 +76,11 @@ store_segment(Transfer, Segment) -> Mod = mod(), Mod:store_segment(storage(), Transfer, Segment). --spec assemble(emqx_ft:transfer()) -> +-spec assemble(emqx_ft:transfer(), emqx_ft:bytes()) -> ok | {async, pid()} | {error, term()}. -assemble(Transfer) -> +assemble(Transfer, Size) -> Mod = mod(), - Mod:assemble(storage(), Transfer). + Mod:assemble(storage(), Transfer, Size). -spec ready_transfers() -> {ok, [{ready_transfer_id(), ready_transfer_info()}]} | {error, term()}. ready_transfers() -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 81fed0f21..103f0e48d 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -24,7 +24,7 @@ -export([store_segment/3]). -export([list/3]). -export([pread/5]). --export([assemble/2]). +-export([assemble/3]). -export([transfers/1]). @@ -167,12 +167,12 @@ pread(_Storage, _Transfer, Frag, Offset, Size) -> {error, Reason} end. --spec assemble(storage(), transfer()) -> +-spec assemble(storage(), transfer(), emqx_ft:bytes()) -> % {ok, _Assembler :: pid()} | {error, incomplete} | {error, badrpc} | {error, _TODO}. {async, _Assembler :: pid()} | {error, _TODO}. -assemble(Storage, Transfer) -> +assemble(Storage, Transfer, Size) -> % TODO: ask cluster if the transfer is already assembled - {ok, Pid} = emqx_ft_assembler_sup:ensure_child(Storage, Transfer), + {ok, Pid} = emqx_ft_assembler_sup:ensure_child(Storage, Transfer, Size), {async, Pid}. get_ready_transfer(_Storage, ReadyTransferId) -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 8708f9808..8f00a1884 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -37,8 +37,14 @@ all() -> groups() -> [ - {single_node, [sequence], emqx_common_test_helpers:all(?MODULE) -- [t_switch_node]}, - {cluster, [sequence], [t_switch_node]} + {single_node, [sequence], emqx_common_test_helpers:all(?MODULE) -- group_cluster()}, + {cluster, [sequence], group_cluster()} + ]. + +group_cluster() -> + [ + t_switch_node, + t_unreliable_migrating_client ]. init_per_suite(Config) -> @@ -61,25 +67,61 @@ set_special_configs(Config) -> init_per_testcase(Case, Config) -> ClientId = atom_to_binary(Case), - {ok, C} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}]), - {ok, _} = emqtt:connect(C), - [{client, C}, {clientid, ClientId} | Config]. + case ?config(group, Config) of + cluster -> + [{clientid, ClientId} | Config]; + _ -> + {ok, C} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(C), + [{client, C}, {clientid, ClientId} | Config] + end. end_per_testcase(_Case, Config) -> - C = ?config(client, Config), - ok = emqtt:stop(C), + _ = [ok = emqtt:stop(C) || {client, C} <- Config], ok. -init_per_group(cluster, Config) -> - Node = emqx_ft_test_helpers:start_additional_node(Config, emqx_ft1), - [{additional_node, Node} | Config]; -init_per_group(_Group, Config) -> - Config. +init_per_group(Group = cluster, Config) -> + Cluster = mk_cluster_specs(Config), + ct:pal("Starting ~p", [Cluster]), + Nodes = [ + emqx_common_test_helpers:start_slave(Name, Opts#{join_to => node()}) + || {Name, Opts} <- Cluster + ], + [{group, Group}, {cluster_nodes, Nodes} | Config]; +init_per_group(Group, Config) -> + [{group, Group} | Config]. end_per_group(cluster, Config) -> - ok = emqx_ft_test_helpers:stop_additional_node(Config); + ok = lists:foreach( + fun emqx_ft_test_helpers:stop_additional_node/1, + ?config(cluster_nodes, Config) + ); end_per_group(_Group, _Config) -> ok. +mk_cluster_specs(Config) -> + Specs = [ + {core, emqx_ft_SUITE1, #{listener_ports => [{tcp, 2883}]}}, + {core, emqx_ft_SUITE2, #{listener_ports => [{tcp, 3883}]}} + ], + CommOpts = [ + {env, [{emqx, boot_modules, [broker, listeners]}]}, + {apps, [emqx_ft]}, + {conf, [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]]}, + {env_handler, fun + (emqx_ft) -> + ok = emqx_config:put([file_transfer, storage], #{ + type => local, + root => emqx_ft_test_helpers:ft_root(Config, node()) + }); + (_) -> + ok + end} + ], + emqx_common_test_helpers:emqx_cluster( + Specs, + CommOpts + ). + %%-------------------------------------------------------------------- %% Tests %%-------------------------------------------------------------------- @@ -125,7 +167,7 @@ t_simple_transfer(Config) -> Data = [<<"first">>, <<"second">>, <<"third">>], - Meta = meta(Filename, Data), + Meta = #{size := Filesize} = meta(Filename, Data), MetaPayload = emqx_json:encode(Meta), MetaTopic = <<"$file/", FileId/binary, "/init">>, @@ -145,7 +187,7 @@ t_simple_transfer(Config) -> with_offsets(Data) ), - FinTopic = <<"$file/", FileId/binary, "/fin">>, + FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( success, emqtt:publish(C, FinTopic, <<>>, 1) @@ -194,7 +236,7 @@ t_no_meta(Config) -> emqtt:publish(C, SegmentTopic, Data, 1) ), - FinTopic = <<"$file/", FileId/binary, "/fin">>, + FinTopic = <<"$file/", FileId/binary, "/fin/42">>, ?assertRCName( unspecified_error, emqtt:publish(C, FinTopic, <<>>, 1) @@ -208,7 +250,7 @@ t_no_segment(Config) -> Data = [<<"first">>, <<"second">>, <<"third">>], - Meta = meta(Filename, Data), + Meta = #{size := Filesize} = meta(Filename, Data), MetaPayload = emqx_json:encode(Meta), MetaTopic = <<"$file/", FileId/binary, "/init">>, @@ -229,7 +271,7 @@ t_no_segment(Config) -> tl(with_offsets(Data)) ), - FinTopic = <<"$file/", FileId/binary, "/fin">>, + FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( unspecified_error, emqtt:publish(C, FinTopic, <<>>, 1) @@ -264,7 +306,7 @@ t_invalid_checksum(Config) -> Data = [<<"first">>, <<"second">>, <<"third">>], - Meta = meta(Filename, Data), + Meta = #{size := Filesize} = meta(Filename, Data), MetaPayload = emqx_json:encode(Meta#{checksum => sha256hex(<<"invalid">>)}), MetaTopic = <<"$file/", FileId/binary, "/init">>, @@ -284,14 +326,15 @@ t_invalid_checksum(Config) -> with_offsets(Data) ), - FinTopic = <<"$file/", FileId/binary, "/fin">>, + FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( unspecified_error, emqtt:publish(C, FinTopic, <<>>, 1) ). t_switch_node(Config) -> - AdditionalNodePort = emqx_ft_test_helpers:tcp_port(?config(additional_node, Config)), + [Node | _] = ?config(cluster_nodes, Config), + AdditionalNodePort = emqx_ft_test_helpers:tcp_port(Node), ClientId = <<"t_switch_node-migrating_client">>, @@ -306,7 +349,7 @@ t_switch_node(Config) -> %% First, publist metadata and the first segment to the additional node - Meta = meta(Filename, Data), + Meta = #{size := Filesize} = meta(Filename, Data), MetaPayload = emqx_json:encode(Meta), MetaTopic = <<"$file/", FileId/binary, "/init">>, @@ -335,7 +378,7 @@ t_switch_node(Config) -> emqtt:publish(C2, <<"$file/", FileId/binary, "/", Offset2/binary>>, Data2, 1) ), - FinTopic = <<"$file/", FileId/binary, "/fin">>, + FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( success, emqtt:publish(C2, FinTopic, <<>>, 1) @@ -360,13 +403,134 @@ t_assemble_crash(Config) -> C = ?config(client, Config), meck:new(emqx_ft_storage_fs), - meck:expect(emqx_ft_storage_fs, assemble, fun(_, _) -> meck:exception(error, oops) end), + meck:expect(emqx_ft_storage_fs, assemble, fun(_, _, _) -> meck:exception(error, oops) end), ?assertRCName( unspecified_error, emqtt:publish(C, <<"$file/someid/fin">>, <<>>, 1) ). +t_unreliable_migrating_client(Config) -> + NodeSelf = node(), + [Node1, Node2] = ?config(cluster_nodes, Config), + + ClientId = ?config(clientid, Config), + FileId = emqx_guid:to_hexstr(emqx_guid:gen()), + Filename = "migratory-birds-in-southern-hemisphere-2013.pdf", + Filesize = 1000, + Gen = emqx_ft_content_gen:new({{ClientId, FileId}, Filesize}, 16), + Payload = iolist_to_binary(emqx_ft_content_gen:consume(Gen, fun({Chunk, _, _}) -> Chunk end)), + Meta = meta(Filename, Payload), + + Context = #{ + clientid => ClientId, + fileid => FileId, + filesize => Filesize, + payload => Payload + }, + Commands = [ + {fun connect_mqtt_client/2, [NodeSelf]}, + {fun send_filemeta/2, [Meta]}, + {fun send_segment/3, [0, 100]}, + {fun send_segment/3, [100, 100]}, + {fun send_segment/3, [200, 100]}, + {fun stop_mqtt_client/1, []}, + {fun connect_mqtt_client/2, [Node1]}, + {fun connect_mqtt_client/2, [Node2]}, + {fun send_filemeta/2, [Meta]}, + {fun send_segment/3, [0, 200]}, + {fun send_segment/3, [200, 200]}, + {fun send_segment/3, [400, 100]}, + {fun connect_mqtt_client/2, [Node2]}, + {fun send_segment/3, [200, 200]}, + {fun send_segment/3, [400, 200]}, + {fun connect_mqtt_client/2, [Node1]}, + {fun send_segment/3, [400, 200]}, + {fun send_segment/3, [600, eof]}, + {fun send_finish/1, []}, + {fun connect_mqtt_client/2, [NodeSelf]}, + {fun send_finish/1, []} + ], + _Context = run_commands(Commands, Context), + + {ok, ReadyTransfers} = emqx_ft_storage:ready_transfers(), + ReadyTransferIds = + [Id || {#{<<"clientid">> := CId} = Id, _Info} <- ReadyTransfers, CId == ClientId], + + Node1Bin = atom_to_binary(Node1), + NodeSelfBin = atom_to_binary(NodeSelf), + ?assertMatch( + [#{<<"node">> := Node1Bin}, #{<<"node">> := NodeSelfBin}], + lists:sort(ReadyTransferIds) + ), + + [ + begin + {ok, TableQH} = emqx_ft_storage:get_ready_transfer(Id), + ?assertEqual( + Payload, + iolist_to_binary(qlc:eval(TableQH)) + ) + end + || Id <- ReadyTransferIds + ]. + +run_commands(Commands, Context) -> + lists:foldl(fun run_command/2, Context, Commands). + +run_command({Command, Args}, Context) -> + ct:pal("COMMAND ~p ~p", [erlang:fun_info(Command, name), Args]), + erlang:apply(Command, Args ++ [Context]). + +connect_mqtt_client(Node, ContextIn) -> + Context = #{clientid := ClientId} = disown_mqtt_client(ContextIn), + NodePort = emqx_ft_test_helpers:tcp_port(Node), + {ok, Client} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, NodePort}]), + {ok, _} = emqtt:connect(Client), + Context#{client => Client}. + +stop_mqtt_client(Context = #{client := Client}) -> + _ = emqtt:stop(Client), + maps:remove(client, Context). + +disown_mqtt_client(Context = #{client := Client}) -> + _ = erlang:unlink(Client), + maps:remove(client, Context); +disown_mqtt_client(Context = #{}) -> + Context. + +send_filemeta(Meta, Context = #{client := Client, fileid := FileId}) -> + Topic = <<"$file/", FileId/binary, "/init">>, + MetaPayload = emqx_json:encode(Meta), + ?assertRCName( + success, + emqtt:publish(Client, Topic, MetaPayload, 1) + ), + Context. + +send_segment(Offset, Size, Context = #{client := Client, fileid := FileId, payload := Payload}) -> + Topic = <<"$file/", FileId/binary, "/", (integer_to_binary(Offset))/binary>>, + Data = + case Size of + eof -> + binary:part(Payload, Offset, byte_size(Payload) - Offset); + N -> + binary:part(Payload, Offset, N) + end, + ?assertRCName( + success, + emqtt:publish(Client, Topic, Data, 1) + ), + Context. + +send_finish(Context = #{client := Client, fileid := FileId, filesize := Filesize}) -> + Topic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, + ?assertRCName( + success, + emqtt:publish(Client, Topic, <<>>, 1) + ), + Context. + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index d4b619e43..bb8590cc7 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -84,7 +84,7 @@ t_assemble_empty_transfer(Config) -> ]}, emqx_ft_storage_fs:list(Storage, Transfer, fragment) ), - Status = complete_assemble(Storage, Transfer), + Status = complete_assemble(Storage, Transfer, 0), ?assertEqual({shutdown, ok}, Status), ?assertEqual( {ok, <<>>}, @@ -132,7 +132,7 @@ t_assemble_complete_local_transfer(Config) -> Fragments ), - Status = complete_assemble(Storage, Transfer), + Status = complete_assemble(Storage, Transfer, TransferSize), ?assertEqual({shutdown, ok}, Status), AssemblyFilename = mk_assembly_filename(Config, Transfer, Filename), @@ -171,20 +171,20 @@ t_assemble_incomplete_transfer(Config) -> expire_at => 42 }, ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta), - Status = complete_assemble(Storage, Transfer), + Status = complete_assemble(Storage, Transfer, TransferSize), ?assertMatch({shutdown, {error, _}}, Status). t_assemble_no_meta(Config) -> Storage = storage(Config), Transfer = {?CLIENTID2, ?config(file_id, Config)}, - Status = complete_assemble(Storage, Transfer), + Status = complete_assemble(Storage, Transfer, 42), ?assertMatch({shutdown, {error, {incomplete, _}}}, Status). -complete_assemble(Storage, Transfer) -> - complete_assemble(Storage, Transfer, 1000). +complete_assemble(Storage, Transfer, Size) -> + complete_assemble(Storage, Transfer, Size, 1000). -complete_assemble(Storage, Transfer, Timeout) -> - {async, Pid} = emqx_ft_storage_fs:assemble(Storage, Transfer), +complete_assemble(Storage, Transfer, Size, Timeout) -> + {async, Pid} = emqx_ft_storage_fs:assemble(Storage, Transfer, Size), MRef = erlang:monitor(process, Pid), Pid ! kickoff, receive diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index cc3da8bc3..3bda8042c 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -67,7 +67,7 @@ init_per_group(_Group, Config) -> Config. end_per_group(cluster, Config) -> - ok = emqx_ft_test_helpers:stop_additional_node(Config); + ok = emqx_ft_test_helpers:stop_additional_node(?config(additional_node, Config)); end_per_group(_Group, _Config) -> ok. diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index f7d8bc879..956e63553 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -21,13 +21,12 @@ -include_lib("common_test/include/ct.hrl"). -start_additional_node(Config, Node) -> - SelfNode = node(), +start_additional_node(Config, Name) -> emqx_common_test_helpers:start_slave( - Node, + Name, [ {apps, [emqx_ft]}, - {join_to, SelfNode}, + {join_to, node()}, {configure_gen_rpc, true}, {env_handler, fun (emqx_ft) -> @@ -40,8 +39,7 @@ start_additional_node(Config, Node) -> ] ). -stop_additional_node(Config) -> - Node = ?config(additional_node, Config), +stop_additional_node(Node) -> ok = rpc:call(Node, ekka, leave, []), ok = rpc:call(Node, emqx_common_test_helpers, stop_apps, [[emqx_ft]]), {ok, _} = emqx_common_test_helpers:stop_slave(Node), @@ -58,13 +56,14 @@ ft_root(Config, Node) -> upload_file(ClientId, FileId, Data, Node) -> Port = tcp_port(Node), + Size = byte_size(Data), {ok, C1} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, Port}]), {ok, _} = emqtt:connect(C1), Meta = #{ name => FileId, expire_at => erlang:system_time(_Unit = second) + 3600, - size => byte_size(Data) + size => Size }, MetaPayload = emqx_json:encode(Meta), @@ -72,6 +71,6 @@ upload_file(ClientId, FileId, Data, Node) -> {ok, _} = emqtt:publish(C1, MetaTopic, MetaPayload, 1), {ok, _} = emqtt:publish(C1, <<"$file/", FileId/binary, "/0">>, Data, 1), - FinTopic = <<"$file/", FileId/binary, "/fin">>, + FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Size))/binary>>, {ok, _} = emqtt:publish(C1, FinTopic, <<>>, 1), ok = emqtt:stop(C1). From 2cf2a2d95284b05551d48137f9ddd563196f0e6d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 17 Feb 2023 19:32:23 +0300 Subject: [PATCH 041/156] fix(ft): make `fin` packet parser stricter + safer --- apps/emqx_ft/src/emqx_ft.erl | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 266d38fe8..6a6599366 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -146,11 +146,14 @@ on_file_command(PacketId, Msg, FileCommand) -> case string:split(FileCommand, <<"/">>, all) of [FileId, <<"init">>] -> on_init(PacketId, Msg, transfer(Msg, FileId)); - [FileId, <<"fin">>, FinalSizeBin | ChecksumL] -> - validate([{size, FinalSizeBin}], fun([FinalSize]) -> - Checksum = emqx_maybe:from_list(ChecksumL), - on_fin(PacketId, Msg, transfer(Msg, FileId), FinalSize, Checksum) - end); + [FileId, <<"fin">>, FinalSizeBin | MaybeChecksum] when length(MaybeChecksum) =< 1 -> + ChecksumBin = emqx_maybe:from_list(MaybeChecksum), + validate( + [{size, FinalSizeBin}, {{maybe, checksum}, ChecksumBin}], + fun([FinalSize, Checksum]) -> + on_fin(PacketId, Msg, transfer(Msg, FileId), FinalSize, Checksum) + end + ); [FileId, <<"abort">>] -> on_abort(Msg, transfer(Msg, FileId)); [FileId, OffsetBin] -> @@ -375,7 +378,11 @@ do_validate([{checksum, Checksum} | Rest], Parsed) -> do_validate(Rest, [Bin | Parsed]); {error, _Reason} -> {error, {invalid_checksum, Checksum}} - end. + end; +do_validate([{{maybe, _}, undefined} | Rest], Parsed) -> + do_validate(Rest, [undefined | Parsed]); +do_validate([{{maybe, T}, Value} | Rest], Parsed) -> + do_validate([{T, Value} | Rest], Parsed). parse_checksum(Checksum) when is_binary(Checksum) andalso byte_size(Checksum) =:= 64 -> try From 10d4c4305a69bc6ac2c9976d1f1b3aacd55843c5 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 17 Feb 2023 19:45:10 +0300 Subject: [PATCH 042/156] feat(maybe): add basic tests for the new module --- apps/emqx/src/emqx_maybe.erl | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/apps/emqx/src/emqx_maybe.erl b/apps/emqx/src/emqx_maybe.erl index 8c60d7ae7..8da7c7c90 100644 --- a/apps/emqx/src/emqx_maybe.erl +++ b/apps/emqx/src/emqx_maybe.erl @@ -34,9 +34,40 @@ from_list([]) -> from_list([Term]) -> Term. +%% @doc Apply a function to a maybe argument. -spec apply(fun((maybe(A)) -> maybe(A)), maybe(A)) -> maybe(A). apply(_Fun, undefined) -> undefined; apply(Fun, Term) when is_function(Fun) -> erlang:apply(Fun, [Term]). + +%% + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +to_list_test_() -> + [ + ?_assertEqual([], to_list(undefined)), + ?_assertEqual([42], to_list(42)) + ]. + +from_list_test_() -> + [ + ?_assertEqual(undefined, from_list([])), + ?_assertEqual(3.1415, from_list([3.1415])), + ?_assertError(_, from_list([1, 2, 3])) + ]. + +apply_test_() -> + [ + ?_assertEqual(<<"42">>, ?MODULE:apply(fun erlang:integer_to_binary/1, 42)), + ?_assertEqual(undefined, ?MODULE:apply(fun erlang:integer_to_binary/1, undefined)), + ?_assertEqual(undefined, ?MODULE:apply(fun crash/1, undefined)) + ]. + +crash(_) -> + erlang:error(crashed). + +-endif. From a3f47641f3e3f0dfef2ff77899ebf00cb3807c72 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 17 Feb 2023 19:59:06 +0300 Subject: [PATCH 043/156] test(ft): describe the complex test case with comments In order to make the test client model and behavior clearer. --- apps/emqx_ft/test/emqx_ft_SUITE.erl | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 8f00a1884..be3b19a8c 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -429,25 +429,41 @@ t_unreliable_migrating_client(Config) -> payload => Payload }, Commands = [ + % Connect to the broker on the current node {fun connect_mqtt_client/2, [NodeSelf]}, + % Send filemeta and 3 initial segments + % (assuming client chose 100 bytes as a desired segment size) {fun send_filemeta/2, [Meta]}, {fun send_segment/3, [0, 100]}, {fun send_segment/3, [100, 100]}, {fun send_segment/3, [200, 100]}, + % Disconnect the client cleanly {fun stop_mqtt_client/1, []}, + % Connect to the broker on `Node1` {fun connect_mqtt_client/2, [Node1]}, + % Connect to the broker on `Node2` without first disconnecting from `Node1` + % Client forgot the state for some reason and started the transfer again. + % (assuming this is usual for a client on a device that was rebooted) {fun connect_mqtt_client/2, [Node2]}, {fun send_filemeta/2, [Meta]}, + % This time it chose 200 bytes as a segment size {fun send_segment/3, [0, 200]}, {fun send_segment/3, [200, 200]}, + % But now it downscaled back to 100 bytes segments {fun send_segment/3, [400, 100]}, + % Client lost connectivity and reconnected + % (also had last few segments unacked and decided to resend them) {fun connect_mqtt_client/2, [Node2]}, {fun send_segment/3, [200, 200]}, {fun send_segment/3, [400, 200]}, + % Client lost connectivity and reconnected, this time to another node + % (also had last segment unacked and decided to resend it) {fun connect_mqtt_client/2, [Node1]}, {fun send_segment/3, [400, 200]}, {fun send_segment/3, [600, eof]}, {fun send_finish/1, []}, + % Client lost connectivity and reconnected, this time to the current node + % (client had `fin` unacked and decided to resend it) {fun connect_mqtt_client/2, [NodeSelf]}, {fun send_finish/1, []} ], @@ -457,6 +473,9 @@ t_unreliable_migrating_client(Config) -> ReadyTransferIds = [Id || {#{<<"clientid">> := CId} = Id, _Info} <- ReadyTransfers, CId == ClientId], + % NOTE + % The cluster had 2 assemblers running on two different nodes, because client sent `fin` + % twice. This is currently expected, files must be identical anyway. Node1Bin = atom_to_binary(Node1), NodeSelfBin = atom_to_binary(NodeSelf), ?assertMatch( From 7ff253e80d3722bedb7e4dd73515cd48d736f582 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 17 Feb 2023 20:44:36 +0300 Subject: [PATCH 044/156] test(ft): attempt to avoid `gen_rpc` config reset hacks --- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index 40b5da58f..239edb267 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -136,8 +136,8 @@ t_download_transfer(Config) -> ), ?assertEqual( - Response, - <<"data">> + <<"data">>, + Response ). %%-------------------------------------------------------------------- From 2b925aa60bc11c2163c0d49fecce344863c634c4 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Feb 2023 18:03:26 +0300 Subject: [PATCH 045/156] fix(ft): drop unrelated TODO --- apps/emqx_ft/src/emqx_ft_assembler.erl | 3 --- 1 file changed, 3 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 275d16499..441303270 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -37,9 +37,6 @@ %% start_link(Storage, Transfer, Size) -> - %% TODO - %% Additional callbacks? They won't survive restarts by the supervisor, which brings a - %% question if we even need to retry with the help of supervisor. gen_statem:start_link(?REF(Transfer), ?MODULE, {Storage, Transfer, Size}, []). %% From 75070102ece6eb7513a947c6e6368a3b9b64486c Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Feb 2023 18:22:51 +0300 Subject: [PATCH 046/156] fix(ft): improve typespecs --- apps/emqx_ft/src/emqx_ft_assembler.erl | 2 +- apps/emqx_ft/src/emqx_ft_assembly.erl | 46 +++++++++++++++++++++---- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 23 ++++++++----- 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 441303270..9e3ddfcbd 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -26,7 +26,7 @@ -record(st, { storage :: _Storage, transfer :: emqx_ft:transfer(), - assembly :: _TODO, + assembly :: emqx_ft_assembly:t(), file :: {file:filename(), io:device(), term()} | undefined, hash }). diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index 2d78b540b..f0f0026a3 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -25,16 +25,43 @@ -export([coverage/1]). -export([properties/1]). +-export_type([t/0]). + +-type filemeta() :: emqx_ft:filemeta(). +-type filefrag() :: emqx_ft_storage_fs:filefrag(). +-type filefrag(T) :: emqx_ft_storage_fs:filefrag(T). +-type segmentinfo() :: emqx_ft_storage_fs:segmentinfo(). + -record(asm, { - status :: _TODO, - coverage :: _TODO, - properties :: _TODO, - meta :: _TODO, - % orddict:orddict(K, V) - segs :: _TODO, - size + status :: status(), + coverage :: coverage() | undefined, + properties :: properties() | undefined, + meta :: orddict:orddict( + filemeta(), + {node(), filefrag({filemeta, filemeta()})} + ), + segs :: orddict:orddict( + {emqx_ft:offset(), _Locality, _MEnd, node()}, + filefrag({segment, segmentinfo()}) + ), + size :: emqx_ft:bytes() }). +-type status() :: + {incomplete, {missing, _}} + | complete + | {error, {inconsistent, _}}. + +-type coverage() :: [{node(), filefrag({segment, segmentinfo()})}]. + +-type properties() :: #{ + %% Node where "most" of the segments are located. + dominant => node() +}. + +-opaque t() :: #asm{}. + +-spec new(emqx_ft:bytes()) -> t(). new(Size) -> #asm{ status = {incomplete, {missing, filemeta}}, @@ -43,6 +70,7 @@ new(Size) -> size = Size }. +-spec append(t(), node(), filefrag() | [filefrag()]) -> t(). append(Asm, Node, Fragments) when is_list(Fragments) -> lists:foldl(fun(F, AsmIn) -> append(AsmIn, Node, F) end, Asm, Fragments); append(Asm, Node, Fragment = #{fragment := {filemeta, _}}) -> @@ -50,6 +78,7 @@ append(Asm, Node, Fragment = #{fragment := {filemeta, _}}) -> append(Asm, Node, Segment = #{fragment := {segment, _}}) -> append_segmentinfo(Asm, Node, Segment). +-spec update(t()) -> t(). update(Asm) -> case status(meta, Asm) of {complete, _Meta} -> @@ -67,15 +96,18 @@ update(Asm) -> Asm#asm{status = Status} end. +-spec status(t()) -> status(). status(#asm{status = Status}) -> Status. +-spec filemeta(t()) -> filemeta(). filemeta(Asm) -> case status(meta, Asm) of {complete, Meta} -> Meta; _Other -> undefined end. +-spec coverage(t()) -> coverage() | undefined. coverage(#asm{coverage = Coverage}) -> Coverage. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 103f0e48d..d3331a091 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -80,11 +80,18 @@ -type storage() :: emqx_ft_storage:storage(). +-type file_error() :: + file:posix() + %% Filename is incompatible with the backing filesystem. + | badarg + %% System limit (e.g. number of ports) reached. + | system_limit. + %% Store manifest in the backing filesystem. %% Atomic operation. -spec store_filemeta(storage(), transfer(), filemeta()) -> % Quota? Some lower level errors? - ok | {error, conflict} | {error, _TODO}. + ok | {error, conflict} | {error, file_error()}. store_filemeta(Storage, Transfer, Meta) -> % TODO safeguard against bad clientids / fileids. Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], ?MANIFEST), @@ -110,7 +117,7 @@ store_filemeta(Storage, Transfer, Meta) -> -spec store_segment(storage(), transfer(), segment()) -> % Where is the checksum gets verified? Upper level probably. % Quota? Some lower level errors? - ok | {error, _TODO}. + ok | {error, file_error()}. store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], mk_segment_filename(Segment)), write_file_atomic(Storage, Transfer, Filepath, Content). @@ -118,7 +125,8 @@ store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> -spec list(storage(), transfer(), _What :: fragment | result) -> % Some lower level errors? {error, notfound}? % Result will contain zero or only one filemeta. - {ok, [filefrag({filemeta, filemeta()} | {segment, segmentinfo()})]} | {error, _TODO}. + {ok, [filefrag({filemeta, filemeta()} | {segment, segmentinfo()})]} + | {error, file_error()}. list(Storage, Transfer, What) -> Dirname = mk_filedir(Storage, Transfer, get_subdirs_for(What)), case file:list_dir(Dirname) of @@ -146,7 +154,7 @@ get_filefrag_fun_for(result) -> fun mk_result_filefrag/2. -spec pread(storage(), transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> - {ok, _Content :: iodata()} | {error, _TODO}. + {ok, _Content :: iodata()} | {error, eof} | {error, file_error()}. pread(_Storage, _Transfer, Frag, Offset, Size) -> Filepath = maps:get(path, Frag), case file:open(Filepath, [read, raw, binary]) of @@ -168,7 +176,6 @@ pread(_Storage, _Transfer, Frag, Offset, Size) -> end. -spec assemble(storage(), transfer(), emqx_ft:bytes()) -> - % {ok, _Assembler :: pid()} | {error, incomplete} | {error, badrpc} | {error, _TODO}. {async, _Assembler :: pid()} | {error, _TODO}. assemble(Storage, Transfer, Size) -> % TODO: ask cluster if the transfer is already assembled @@ -321,7 +328,7 @@ read_transferinfo(Storage, Transfer, Acc) -> -type handle() :: {file:name(), io:device(), crypto:hash_state()}. -spec open_file(storage(), transfer(), filemeta()) -> - {ok, handle()} | {error, _TODO}. + {ok, handle()} | {error, file_error()}. open_file(Storage, Transfer, Filemeta) -> Filename = maps:get(name, Filemeta), TempFilepath = mk_temp_filepath(Storage, Transfer, Filename), @@ -335,7 +342,7 @@ open_file(Storage, Transfer, Filemeta) -> end. -spec write(handle(), iodata()) -> - {ok, handle()} | {error, _TODO}. + {ok, handle()} | {error, file_error()}. write({Filepath, IoDevice, Ctx}, IoData) -> case file:write(IoDevice, IoData) of ok -> @@ -345,7 +352,7 @@ write({Filepath, IoDevice, Ctx}, IoData) -> end. -spec complete(storage(), transfer(), filemeta(), handle()) -> - ok | {error, {checksum, _Algo, _Computed}} | {error, _TODO}. + ok | {error, {checksum, _Algo, _Computed}} | {error, file_error()}. complete(Storage, Transfer, Filemeta, Handle = {Filepath, IoDevice, Ctx}) -> TargetFilepath = mk_filepath(Storage, Transfer, [?RESULTDIR], maps:get(name, Filemeta)), case verify_checksum(Ctx, Filemeta) of From 93865a79e9ae1f344036ef6aca79737cfb80021a Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Feb 2023 18:43:34 +0300 Subject: [PATCH 047/156] fix(ft): disallow empty fileids --- apps/emqx_ft/src/emqx_ft.erl | 40 ++++++++++++++++++------- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 1 - apps/emqx_ft/test/emqx_ft_SUITE.erl | 7 +++++ 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 6a6599366..baa0126d6 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -143,26 +143,37 @@ on_message_puback(PacketId, #message{topic = Topic} = Msg, _PubRes, _RC) -> %% TODO Move to emqx_ft_mqtt? on_file_command(PacketId, Msg, FileCommand) -> - case string:split(FileCommand, <<"/">>, all) of - [FileId, <<"init">>] -> - on_init(PacketId, Msg, transfer(Msg, FileId)); - [FileId, <<"fin">>, FinalSizeBin | MaybeChecksum] when length(MaybeChecksum) =< 1 -> + case emqx_topic:tokens(FileCommand) of + [FileIdIn | Rest] -> + validate([{fileid, FileIdIn}], fun([FileId]) -> + on_file_command(PacketId, FileId, Msg, Rest) + end); + [] -> + ?RC_UNSPECIFIED_ERROR + end. + +on_file_command(PacketId, FileId, Msg, FileCommand) -> + Transfer = transfer(Msg, FileId), + case FileCommand of + [<<"init">>] -> + on_init(PacketId, Msg, Transfer); + [<<"fin">>, FinalSizeBin | MaybeChecksum] when length(MaybeChecksum) =< 1 -> ChecksumBin = emqx_maybe:from_list(MaybeChecksum), validate( [{size, FinalSizeBin}, {{maybe, checksum}, ChecksumBin}], fun([FinalSize, Checksum]) -> - on_fin(PacketId, Msg, transfer(Msg, FileId), FinalSize, Checksum) + on_fin(PacketId, Msg, Transfer, FinalSize, Checksum) end ); - [FileId, <<"abort">>] -> - on_abort(Msg, transfer(Msg, FileId)); - [FileId, OffsetBin] -> + [<<"abort">>] -> + on_abort(Msg, Transfer); + [OffsetBin] -> validate([{offset, OffsetBin}], fun([Offset]) -> - on_segment(PacketId, Msg, transfer(Msg, FileId), Offset, undefined) + on_segment(PacketId, Msg, Transfer, Offset, undefined) end); - [FileId, OffsetBin, ChecksumBin] -> + [OffsetBin, ChecksumBin] -> validate([{offset, OffsetBin}, {checksum, ChecksumBin}], fun([Offset, Checksum]) -> - on_segment(PacketId, Msg, transfer(Msg, FileId), Offset, Checksum) + on_segment(PacketId, Msg, Transfer, Offset, Checksum) end); _ -> ?RC_UNSPECIFIED_ERROR @@ -358,6 +369,13 @@ validate(Validations, Fun) -> do_validate([], Parsed) -> {ok, lists:reverse(Parsed)}; +do_validate([{fileid, FileId} | Rest], Parsed) -> + case byte_size(FileId) of + S when S > 0 -> + do_validate(Rest, [FileId | Parsed]); + 0 -> + {error, {invalid_fileid, FileId}} + end; do_validate([{offset, Offset} | Rest], Parsed) -> case string:to_integer(Offset) of {Int, <<>>} -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index d3331a091..667ca091a 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -93,7 +93,6 @@ % Quota? Some lower level errors? ok | {error, conflict} | {error, file_error()}. store_filemeta(Storage, Transfer, Meta) -> - % TODO safeguard against bad clientids / fileids. Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], ?MANIFEST), case read_file(Filepath, fun decode_filemeta/1) of {ok, Meta} -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index be3b19a8c..37cc38d4f 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -159,6 +159,13 @@ t_invalid_topic_format(Config) -> emqtt:publish(C, <<"$file">>, <<>>, 1) ). +t_invalid_fileid(Config) -> + C = ?config(client, Config), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file//init">>, <<>>, 1) + ). + t_simple_transfer(Config) -> C = ?config(client, Config), From 5014cc15f4dcc62e80d3ece84c04df057b839ed7 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Feb 2023 18:44:13 +0300 Subject: [PATCH 048/156] test(ft): do not rely on fs backend implementation details --- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index bb8590cc7..a896922d1 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -22,7 +22,6 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). -include_lib("kernel/include/file.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). all() -> [ @@ -86,11 +85,6 @@ t_assemble_empty_transfer(Config) -> ), Status = complete_assemble(Storage, Transfer, 0), ?assertEqual({shutdown, ok}, Status), - ?assertEqual( - {ok, <<>>}, - % TODO - file:read_file(mk_assembly_filename(Config, Transfer, Filename)) - ), {ok, [Result = #{size := Size = 0}]} = emqx_ft_storage_fs:list(Storage, Transfer, result), ?assertEqual( {error, eof}, @@ -135,17 +129,16 @@ t_assemble_complete_local_transfer(Config) -> Status = complete_assemble(Storage, Transfer, TransferSize), ?assertEqual({shutdown, ok}, Status), - AssemblyFilename = mk_assembly_filename(Config, Transfer, Filename), ?assertMatch( {ok, [ #{ - path := AssemblyFilename, size := TransferSize, fragment := {result, #{}} } ]}, emqx_ft_storage_fs:list(Storage, Transfer, result) ), + {ok, [#{path := AssemblyFilename}]} = emqx_ft_storage_fs:list(Storage, Transfer, result), ?assertMatch( {ok, #file_info{type = regular, size = TransferSize}}, file:read_file_info(AssemblyFilename) @@ -194,9 +187,6 @@ complete_assemble(Storage, Transfer, Size, Timeout) -> ct:fail("Assembler did not finish in time") end. -mk_assembly_filename(Config, {ClientID, FileID}, Filename) -> - filename:join([?config(storage_root, Config), ClientID, FileID, result, Filename]). - %% t_list_transfers(Config) -> From 5998961f9f34f7f8d01475f38ba8128c3c7ddf33 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Feb 2023 18:45:07 +0300 Subject: [PATCH 049/156] fix(ft): log errors where they might get lost --- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 31 ++++++++++++++++++++----- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 667ca091a..2cc19d2a2 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -19,6 +19,7 @@ -behaviour(emqx_ft_storage). -include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). -export([store_filemeta/3]). -export([store_segment/3]). @@ -305,7 +306,10 @@ transfers(Storage, ClientId, AccIn) -> FileIds ); {error, _Reason} -> - % TODO worth logging + ?tp(warning, "list_dir_failed", #{ + storage => Storage, + directory => Dirname + }), AccIn end. @@ -318,7 +322,10 @@ read_transferinfo(Storage, Transfer, Acc) -> Info = #{status => incomplete}, Acc#{Transfer => Info}; {error, _Reason} -> - % TODO worth logging + ?tp(warning, "list_result_failed", #{ + storage => Storage, + transfer => Transfer + }), Acc end. @@ -454,8 +461,12 @@ safe_decode(Content, DecodeFun) -> try {ok, DecodeFun(Content)} catch - _C:_R:_Stacktrace -> - % TODO: Log? + C:E:Stacktrace -> + ?tp(warning, "safe_decode_failed", #{ + class => C, + exception => E, + stacktrace => Stacktrace + }), {error, corrupted} end. @@ -509,7 +520,10 @@ mk_filefrag(Dirname, Filename = ?MANIFEST) -> mk_filefrag(Dirname, Filename = ?SEGMENT ++ _) -> mk_filefrag(Dirname, Filename, segment, fun read_segmentinfo/2); mk_filefrag(_Dirname, _Filename) -> - % TODO this is unexpected, worth logging? + ?tp(warning, "rogue_file_found", #{ + directory => _Dirname, + filename => _Filename + }), false. mk_result_filefrag(Dirname, Filename) -> @@ -531,7 +545,12 @@ mk_filefrag(Dirname, Filename, Tag, Fun) -> fragment => {Tag, Frag} }}; {error, _Reason} -> - % TODO loss of information + ?tp(warning, "mk_filefrag_failed", #{ + directory => Dirname, + filename => Filename, + type => Tag, + reason => _Reason + }), false end. From 15d967459c7f3e8092465033afdae29c24ba6ab3 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Feb 2023 20:09:29 +0300 Subject: [PATCH 050/156] feat(ft): add segment checksum validation Also downgrade validation errors to mere info messages. --- apps/emqx_ft/src/emqx_ft.erl | 98 +++++++++-------- apps/emqx_ft/test/emqx_ft_SUITE.erl | 158 ++++++++++++++++++---------- 2 files changed, 157 insertions(+), 99 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index baa0126d6..2ccd5db15 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -20,6 +20,7 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). -export([ hook/0, @@ -156,7 +157,12 @@ on_file_command(PacketId, FileId, Msg, FileCommand) -> Transfer = transfer(Msg, FileId), case FileCommand of [<<"init">>] -> - on_init(PacketId, Msg, Transfer); + validate( + [{filemeta, Msg#message.payload}], + fun([Meta]) -> + on_init(PacketId, Msg, Transfer, Meta) + end + ); [<<"fin">>, FinalSizeBin | MaybeChecksum] when length(MaybeChecksum) =< 1 -> ChecksumBin = emqx_maybe:from_list(MaybeChecksum), validate( @@ -172,51 +178,47 @@ on_file_command(PacketId, FileId, Msg, FileCommand) -> on_segment(PacketId, Msg, Transfer, Offset, undefined) end); [OffsetBin, ChecksumBin] -> - validate([{offset, OffsetBin}, {checksum, ChecksumBin}], fun([Offset, Checksum]) -> - on_segment(PacketId, Msg, Transfer, Offset, Checksum) - end); + validate( + [{offset, OffsetBin}, {checksum, ChecksumBin}], + fun([Offset, Checksum]) -> + validate( + [{integrity, Msg#message.payload, Checksum}], + fun(_) -> + on_segment(PacketId, Msg, Transfer, Offset, Checksum) + end + ) + end + ); _ -> ?RC_UNSPECIFIED_ERROR end. -on_init(PacketId, Msg, Transfer) -> +on_init(PacketId, Msg, Transfer, Meta) -> ?SLOG(info, #{ msg => "on_init", mqtt_msg => Msg, packet_id => PacketId, - transfer => Transfer + transfer => Transfer, + filemeta => Meta }), - Payload = Msg#message.payload, PacketKey = {self(), PacketId}, - % %% Add validations here - case decode_filemeta(Payload) of - {ok, Meta} -> - Callback = fun(Result) -> - ?MODULE:on_complete("store_filemeta", PacketKey, Transfer, Result) - end, - with_responder(PacketKey, Callback, ?STORE_SEGMENT_TIMEOUT, fun() -> - case store_filemeta(Transfer, Meta) of - % Stored, ack through the responder right away - ok -> - emqx_ft_responder:ack(PacketKey, ok); - % Storage operation started, packet will be acked by the responder - % {async, Pid} -> - % ok = emqx_ft_responder:kickoff(PacketKey, Pid), - % ok; - %% Storage operation failed, ack through the responder - {error, _} = Error -> - emqx_ft_responder:ack(PacketKey, Error) - end - end); - {error, Reason} -> - ?SLOG(error, #{ - msg => "on_init: invalid filemeta", - mqtt_msg => Msg, - transfer => Transfer, - reason => Reason - }), - ?RC_UNSPECIFIED_ERROR - end. + Callback = fun(Result) -> + ?MODULE:on_complete("store_filemeta", PacketKey, Transfer, Result) + end, + with_responder(PacketKey, Callback, ?STORE_SEGMENT_TIMEOUT, fun() -> + case store_filemeta(Transfer, Meta) of + % Stored, ack through the responder right away + ok -> + emqx_ft_responder:ack(PacketKey, ok); + % Storage operation started, packet will be acked by the responder + % {async, Pid} -> + % ok = emqx_ft_responder:kickoff(PacketKey, Pid), + % ok; + %% Storage operation failed, ack through the responder + {error, _} = Error -> + emqx_ft_responder:ack(PacketKey, Error) + end + end). on_abort(_Msg, _FileId) -> %% TODO @@ -231,14 +233,11 @@ on_segment(PacketId, Msg, Transfer, Offset, Checksum) -> offset => Offset, checksum => Checksum }), - %% TODO: handle checksum - Payload = Msg#message.payload, - Segment = {Offset, Payload}, + Segment = {Offset, Msg#message.payload}, PacketKey = {self(), PacketId}, Callback = fun(Result) -> ?MODULE:on_complete("store_segment", PacketKey, Transfer, Result) end, - %% Add offset/checksum validations with_responder(PacketKey, Callback, ?STORE_SEGMENT_TIMEOUT, fun() -> case store_segment(Transfer, Segment) of ok -> @@ -360,10 +359,7 @@ validate(Validations, Fun) -> {ok, Parsed} -> Fun(Parsed); {error, Reason} -> - ?SLOG(error, #{ - msg => "validate: invalid $file command", - reason => Reason - }), + ?tp(info, "client_violated_protocol", #{reason => Reason}), ?RC_UNSPECIFIED_ERROR end. @@ -376,6 +372,13 @@ do_validate([{fileid, FileId} | Rest], Parsed) -> 0 -> {error, {invalid_fileid, FileId}} end; +do_validate([{filemeta, Payload} | Rest], Parsed) -> + case decode_filemeta(Payload) of + {ok, Meta} -> + do_validate(Rest, [Meta | Parsed]); + {error, Reason} -> + {error, {invalid_filemeta, Reason}} + end; do_validate([{offset, Offset} | Rest], Parsed) -> case string:to_integer(Offset) of {Int, <<>>} -> @@ -397,6 +400,13 @@ do_validate([{checksum, Checksum} | Rest], Parsed) -> {error, _Reason} -> {error, {invalid_checksum, Checksum}} end; +do_validate([{integrity, Payload, Checksum} | Rest], Parsed) -> + case crypto:hash(sha256, Payload) of + Checksum -> + do_validate(Rest, [Payload | Parsed]); + Mismatch -> + {error, {checksum_mismatch, binary:encode_hex(Mismatch)}} + end; do_validate([{{maybe, _}, undefined} | Rest], Parsed) -> do_validate(Rest, [undefined | Parsed]); do_validate([{{maybe, T}, Value} | Rest], Parsed) -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 37cc38d4f..bb67d3c56 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -37,8 +37,8 @@ all() -> groups() -> [ - {single_node, [sequence], emqx_common_test_helpers:all(?MODULE) -- group_cluster()}, - {cluster, [sequence], group_cluster()} + {single_node, [], emqx_common_test_helpers:all(?MODULE) -- group_cluster()}, + {cluster, [], group_cluster()} ]. group_cluster() -> @@ -177,32 +177,28 @@ t_simple_transfer(Config) -> Meta = #{size := Filesize} = meta(Filename, Data), MetaPayload = emqx_json:encode(Meta), - MetaTopic = <<"$file/", FileId/binary, "/init">>, ?assertRCName( success, - emqtt:publish(C, MetaTopic, MetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) ), lists:foreach( fun({Chunk, Offset}) -> - SegmentTopic = <<"$file/", FileId/binary, "/", Offset/binary>>, ?assertRCName( success, - emqtt:publish(C, SegmentTopic, Chunk, 1) + emqtt:publish(C, mk_segment_topic(FileId, Offset), Chunk, 1) ) end, with_offsets(Data) ), - FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( success, - emqtt:publish(C, FinTopic, <<>>, 1) + emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) ), - {ok, [{ReadyTransferId, _}]} = emqx_ft_storage:ready_transfers(), + [ReadyTransferId] = list_ready_transfers(?config(clientid, Config)), {ok, TableQH} = emqx_ft_storage:get_ready_transfer(ReadyTransferId), - ?assertEqual( iolist_to_binary(Data), iolist_to_binary(qlc:eval(TableQH)) @@ -217,10 +213,9 @@ t_meta_conflict(Config) -> Meta = meta(Filename, [<<"x">>]), MetaPayload = emqx_json:encode(Meta), - MetaTopic = <<"$file/", FileId/binary, "/init">>, ?assertRCName( success, - emqtt:publish(C, MetaTopic, MetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) ), ConflictMeta = Meta#{name => <<"conflict.pdf">>}, @@ -228,7 +223,7 @@ t_meta_conflict(Config) -> ?assertRCName( unspecified_error, - emqtt:publish(C, MetaTopic, ConflictMetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), ConflictMetaPayload, 1) ). t_no_meta(Config) -> @@ -237,16 +232,14 @@ t_no_meta(Config) -> FileId = <<"f1">>, Data = <<"first">>, - SegmentTopic = <<"$file/", FileId/binary, "/0">>, ?assertRCName( success, - emqtt:publish(C, SegmentTopic, Data, 1) + emqtt:publish(C, mk_segment_topic(FileId, 0), Data, 1) ), - FinTopic = <<"$file/", FileId/binary, "/fin/42">>, ?assertRCName( unspecified_error, - emqtt:publish(C, FinTopic, <<>>, 1) + emqtt:publish(C, mk_fin_topic(FileId, 42), <<>>, 1) ). t_no_segment(Config) -> @@ -260,28 +253,25 @@ t_no_segment(Config) -> Meta = #{size := Filesize} = meta(Filename, Data), MetaPayload = emqx_json:encode(Meta), - MetaTopic = <<"$file/", FileId/binary, "/init">>, ?assertRCName( success, - emqtt:publish(C, MetaTopic, MetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) ), lists:foreach( fun({Chunk, Offset}) -> - SegmentTopic = <<"$file/", FileId/binary, "/", Offset/binary>>, ?assertRCName( success, - emqtt:publish(C, SegmentTopic, Chunk, 1) + emqtt:publish(C, mk_segment_topic(FileId, Offset), Chunk, 1) ) end, %% Skip the first segment tl(with_offsets(Data)) ), - FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( unspecified_error, - emqtt:publish(C, FinTopic, <<>>, 1) + emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) ). t_invalid_meta(Config) -> @@ -289,20 +279,18 @@ t_invalid_meta(Config) -> FileId = <<"f1">>, - MetaTopic = <<"$file/", FileId/binary, "/init">>, - %% Invalid schema Meta = #{foo => <<"bar">>}, MetaPayload = emqx_json:encode(Meta), ?assertRCName( unspecified_error, - emqtt:publish(C, MetaTopic, MetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) ), %% Invalid JSON ?assertRCName( unspecified_error, - emqtt:publish(C, MetaTopic, <<"{oops;">>, 1) + emqtt:publish(C, mk_init_topic(FileId), <<"{oops;">>, 1) ). t_invalid_checksum(Config) -> @@ -316,27 +304,73 @@ t_invalid_checksum(Config) -> Meta = #{size := Filesize} = meta(Filename, Data), MetaPayload = emqx_json:encode(Meta#{checksum => sha256hex(<<"invalid">>)}), - MetaTopic = <<"$file/", FileId/binary, "/init">>, ?assertRCName( success, - emqtt:publish(C, MetaTopic, MetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) ), lists:foreach( fun({Chunk, Offset}) -> - SegmentTopic = <<"$file/", FileId/binary, "/", Offset/binary>>, ?assertRCName( success, - emqtt:publish(C, SegmentTopic, Chunk, 1) + emqtt:publish(C, mk_segment_topic(FileId, Offset), Chunk, 1) ) end, with_offsets(Data) ), - FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( unspecified_error, - emqtt:publish(C, FinTopic, <<>>, 1) + emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) + ). + +t_corrupted_segment_retry(Config) -> + C = ?config(client, Config), + + Filename = <<"corruption.pdf">>, + FileId = <<"4242-4242">>, + + Data = [<<"first">>, <<"second">>, <<"third">>], + [ + {Seg1, Offset1}, + {Seg2, Offset2}, + {Seg3, Offset3} + ] = with_offsets(Data), + [ + Checksum1, + Checksum2, + Checksum3 + ] = [sha256hex(S) || S <- Data], + + Meta = #{size := Filesize} = meta(Filename, Data), + + ?assertRCName(success, emqtt:publish(C, mk_init_topic(FileId), emqx_json:encode(Meta), 1)), + + ?assertRCName( + success, + emqtt:publish(C, mk_segment_topic(FileId, Offset1, Checksum1), Seg1, 1) + ), + + % segment is corrupted + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_segment_topic(FileId, Offset2, Checksum2), <>, 1) + ), + + % retry + ?assertRCName( + success, + emqtt:publish(C, mk_segment_topic(FileId, Offset2, Checksum2), Seg2, 1) + ), + + ?assertRCName( + success, + emqtt:publish(C, mk_segment_topic(FileId, Offset3, Checksum3), Seg3, 1) + ), + + ?assertRCName( + success, + emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) ). t_switch_node(Config) -> @@ -359,14 +393,13 @@ t_switch_node(Config) -> Meta = #{size := Filesize} = meta(Filename, Data), MetaPayload = emqx_json:encode(Meta), - MetaTopic = <<"$file/", FileId/binary, "/init">>, ?assertRCName( success, - emqtt:publish(C1, MetaTopic, MetaPayload, 1) + emqtt:publish(C1, mk_init_topic(FileId), MetaPayload, 1) ), ?assertRCName( success, - emqtt:publish(C1, <<"$file/", FileId/binary, "/", Offset0/binary>>, Data0, 1) + emqtt:publish(C1, mk_segment_topic(FileId, Offset0), Data0, 1) ), %% Then, switch the client to the main node @@ -378,29 +411,24 @@ t_switch_node(Config) -> ?assertRCName( success, - emqtt:publish(C2, <<"$file/", FileId/binary, "/", Offset1/binary>>, Data1, 1) + emqtt:publish(C2, mk_segment_topic(FileId, Offset1), Data1, 1) ), ?assertRCName( success, - emqtt:publish(C2, <<"$file/", FileId/binary, "/", Offset2/binary>>, Data2, 1) + emqtt:publish(C2, mk_segment_topic(FileId, Offset2), Data2, 1) ), - FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( success, - emqtt:publish(C2, FinTopic, <<>>, 1) + emqtt:publish(C2, mk_fin_topic(FileId, Filesize), <<>>, 1) ), ok = emqtt:stop(C2), %% Now check consistency of the file - {ok, ReadyTransfers} = emqx_ft_storage:ready_transfers(), - {ReadyTransferIds, _} = lists:unzip(ReadyTransfers), - [ReadyTransferId] = [Id || #{<<"clientid">> := CId} = Id <- ReadyTransferIds, CId == ClientId], - + [ReadyTransferId] = list_ready_transfers(ClientId), {ok, TableQH} = emqx_ft_storage:get_ready_transfer(ReadyTransferId), - ?assertEqual( iolist_to_binary(Data), iolist_to_binary(qlc:eval(TableQH)) @@ -476,9 +504,7 @@ t_unreliable_migrating_client(Config) -> ], _Context = run_commands(Commands, Context), - {ok, ReadyTransfers} = emqx_ft_storage:ready_transfers(), - ReadyTransferIds = - [Id || {#{<<"clientid">> := CId} = Id, _Info} <- ReadyTransfers, CId == ClientId], + ReadyTransferIds = list_ready_transfers(?config(clientid, Config)), % NOTE % The cluster had 2 assemblers running on two different nodes, because client sent `fin` @@ -526,16 +552,13 @@ disown_mqtt_client(Context = #{}) -> Context. send_filemeta(Meta, Context = #{client := Client, fileid := FileId}) -> - Topic = <<"$file/", FileId/binary, "/init">>, - MetaPayload = emqx_json:encode(Meta), ?assertRCName( success, - emqtt:publish(Client, Topic, MetaPayload, 1) + emqtt:publish(Client, mk_init_topic(FileId), emqx_json:encode(Meta), 1) ), Context. send_segment(Offset, Size, Context = #{client := Client, fileid := FileId, payload := Payload}) -> - Topic = <<"$file/", FileId/binary, "/", (integer_to_binary(Offset))/binary>>, Data = case Size of eof -> @@ -545,15 +568,14 @@ send_segment(Offset, Size, Context = #{client := Client, fileid := FileId, paylo end, ?assertRCName( success, - emqtt:publish(Client, Topic, Data, 1) + emqtt:publish(Client, mk_segment_topic(FileId, Offset), Data, 1) ), Context. send_finish(Context = #{client := Client, fileid := FileId, filesize := Filesize}) -> - Topic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Filesize))/binary>>, ?assertRCName( success, - emqtt:publish(Client, Topic, <<>>, 1) + emqtt:publish(Client, mk_fin_topic(FileId, Filesize), <<>>, 1) ), Context. @@ -561,6 +583,24 @@ send_finish(Context = #{client := Client, fileid := FileId, filesize := Filesize %% Helpers %%-------------------------------------------------------------------- +mk_init_topic(FileId) -> + <<"$file/", FileId/binary, "/init">>. + +mk_segment_topic(FileId, Offset) when is_integer(Offset) -> + mk_segment_topic(FileId, integer_to_binary(Offset)); +mk_segment_topic(FileId, Offset) when is_binary(Offset) -> + <<"$file/", FileId/binary, "/", Offset/binary>>. + +mk_segment_topic(FileId, Offset, Checksum) when is_integer(Offset) -> + mk_segment_topic(FileId, integer_to_binary(Offset), Checksum); +mk_segment_topic(FileId, Offset, Checksum) when is_binary(Offset) -> + <<"$file/", FileId/binary, "/", Offset/binary, "/", Checksum/binary>>. + +mk_fin_topic(FileId, Size) when is_integer(Size) -> + mk_fin_topic(FileId, integer_to_binary(Size)); +mk_fin_topic(FileId, Size) when is_binary(Size) -> + <<"$file/", FileId/binary, "/fin/", Size/binary>>. + with_offsets(Items) -> {List, _} = lists:mapfoldl( fun(Item, Offset) -> @@ -582,3 +622,11 @@ meta(FileName, Data) -> expire_at => erlang:system_time(_Unit = second) + 3600, size => byte_size(FullData) }. + +list_ready_transfers(ClientId) -> + {ok, ReadyTransfers} = emqx_ft_storage:ready_transfers(), + [ + Id + || {#{<<"clientid">> := CId} = Id, _Info} <- ReadyTransfers, + CId == ClientId + ]. From ccba65e1eacca6b8a2fee54f9c1f5139c71325f4 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 21 Feb 2023 11:24:30 +0300 Subject: [PATCH 051/156] chore(ft): add code owners for ft --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5db0f4465..a45d9af59 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,6 +8,7 @@ /apps/emqx_connector/ @emqx/emqx-review-board @JimMoen /apps/emqx_dashboard/ @emqx/emqx-review-board @JimMoen @lafirest /apps/emqx_exhook/ @emqx/emqx-review-board @JimMoen @lafirest +/apps/emqx_ft/ @emqx/emqx-review-board @savonarola @keynslug /apps/emqx_gateway/ @emqx/emqx-review-board @lafirest /apps/emqx_management/ @emqx/emqx-review-board @lafirest @sstrigler /apps/emqx_plugin_libs/ @emqx/emqx-review-board @lafirest From 0af7e2a00266b7ee121e69088c0960bd9103a324 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 27 Feb 2023 15:45:00 +0300 Subject: [PATCH 052/156] perf(ft-asm): optimize away remote segments identical to local --- apps/emqx_ft/src/emqx_ft_assembly.erl | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index f0f0026a3..2a8ccb485 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -137,6 +137,10 @@ append_filemeta(Asm, Node, Fragment = #{fragment := {filemeta, Meta}}) -> meta = orddict:store(Meta, {Node, Fragment}, Asm#asm.meta) }. +append_segmentinfo(Asm, _Node, #{fragment := {segment, #{size := 0}}}) -> + % NOTE + % Empty segments are valid but meaningless for coverage. + Asm; append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) -> Offset = maps:get(offset, Info), Size = maps:get(size, Info), @@ -151,16 +155,17 @@ append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) -> coverage([{{Offset, _, _, _}, _Segment} | Rest], Cursor, Sz) when Offset < Cursor -> coverage(Rest, Cursor, Sz); -coverage([{{Cursor, _Locality, MEnd, Node}, Segment} | Rest], Cursor, Sz) -> +coverage([{{Cursor, _Locality, MEnd, Node}, Segment} | _Rest] = Segments, Cursor, Sz) -> % NOTE % We consider only whole fragments here, so for example from the point of view of % this algo `[{Offset1 = 0, Size1 = 15}, {Offset2 = 10, Size2 = 10}]` has no % coverage. - case coverage(Rest, -MEnd, Sz) of + Tail = tail(Segments), + case coverage(Tail, -MEnd, Sz) of Coverage when is_list(Coverage) -> [{Node, Segment} | Coverage]; Missing = {missing, _} -> - case coverage(Rest, Cursor, Sz) of + case coverage(Tail, Cursor, Sz) of CoverageAlt when is_list(CoverageAlt) -> CoverageAlt; {missing, _} -> @@ -174,6 +179,20 @@ coverage([], Cursor, Sz) when Cursor < Sz -> coverage([], Cursor, Cursor) -> []. +tail([Segment | Rest]) -> + tail(Segment, Rest). + +tail({{Cursor, _, MEnd, _}, _} = Segment, [{{Cursor, _, MEnd, _}, _} | Rest]) -> + % NOTE + % Discarding segments with same offset / size, potentially located on other nodes. + % This is an optimization. They won't participate in coverage anyway given we're + % currently optimizing coverage towards locality. Yet if we instead decide to + % optimize for node dominance (e.g. compute such coverage that most of the data + % located on a single node) we'll need to account them again. + tail(Segment, Rest); +tail(_Segment, Tail) -> + Tail. + dominant(Coverage) -> % TODO: needs improvement, better defined _dominance_, maybe some score Freqs = frequencies(fun({Node, Segment}) -> {Node, segsize(Segment)} end, Coverage), From 97cfdf8eef5b44b7140eea5107b146efe96f6486 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 27 Feb 2023 15:46:47 +0300 Subject: [PATCH 053/156] test(ft-asm): add property tests for file assembly --- apps/emqx/test/emqx_proper_types.erl | 33 ++- .../test/props/prop_emqx_ft_assembly.erl | 214 ++++++++++++++++++ 2 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl diff --git a/apps/emqx/test/emqx_proper_types.erl b/apps/emqx/test/emqx_proper_types.erl index 2f0f9d494..e1d95227b 100644 --- a/apps/emqx/test/emqx_proper_types.erl +++ b/apps/emqx/test/emqx_proper_types.erl @@ -43,12 +43,21 @@ ip/0, port/0, limited_atom/0, - limited_latin_atom/0 + limited_latin_atom/0, + printable_utf8/0, + printable_codepoint/0 +]). + +%% Generic Types +-export([ + scaled/2 ]). %% Iterators -export([nof/1]). +-type proptype() :: proper_types:raw_type(). + %%-------------------------------------------------------------------- %% Types High level %%-------------------------------------------------------------------- @@ -606,6 +615,20 @@ limited_atom() -> limited_any_term() -> oneof([binary(), number(), string()]). +printable_utf8() -> + ?SUCHTHAT( + String, + ?LET(L, list(printable_codepoint()), unicode:characters_to_binary(L)), + is_binary(String) + ). + +printable_codepoint() -> + frequency([ + {7, range(16#20, 16#7E)}, + {2, range(16#00A0, 16#D7FF)}, + {1, range(16#E000, 16#FFFD)} + ]). + %%-------------------------------------------------------------------- %% Iterators %%-------------------------------------------------------------------- @@ -632,6 +655,14 @@ limited_list(N, T) -> end ). +%%-------------------------------------------------------------------- +%% Generic Types +%%-------------------------------------------------------------------- + +-spec scaled(number(), proptype()) -> proptype(). +scaled(F, T) when F > 0 -> + ?SIZED(S, resize(round(S * F), T)). + %%-------------------------------------------------------------------- %% Internal funcs %%-------------------------------------------------------------------- diff --git a/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl b/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl new file mode 100644 index 000000000..ebebf0e65 --- /dev/null +++ b/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl @@ -0,0 +1,214 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(prop_emqx_ft_assembly). + +-include_lib("proper/include/proper.hrl"). + +-import(emqx_proper_types, [scaled/2]). + +-define(COVERAGE_TIMEOUT, 5000). + +prop_coverage() -> + ?FORALL( + {Filesize, Segsizes}, + {filesize_t(), segsizes_t()}, + ?FORALL( + Fragments, + noshrink(fragments_t(Filesize, Segsizes)), + ?TIMEOUT( + ?COVERAGE_TIMEOUT, + begin + ASM1 = append_fragments(mk_assembly(Filesize), Fragments), + {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), + measure( + #{"Fragments" => length(Fragments), "Time" => Time}, + case emqx_ft_assembly:status(ASM2) of + complete -> + Coverage = emqx_ft_assembly:coverage(ASM2), + measure( + #{"CoverageLength" => length(Coverage)}, + is_coverage_complete(Coverage) + ); + {incomplete, {missing, {segment, _, _}}} -> + measure("CoverageLength", 0, true) + end + ) + end + ) + ) + ). + +prop_coverage_likely_incomplete() -> + ?FORALL( + {Filesize, Segsizes, Hole}, + {filesize_t(), segsizes_t(), filesize_t()}, + ?FORALL( + Fragments, + noshrink(fragments_t(Filesize, Segsizes, Hole)), + ?TIMEOUT( + ?COVERAGE_TIMEOUT, + begin + ASM1 = append_fragments(mk_assembly(Filesize), Fragments), + {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), + measure( + #{"Fragments" => length(Fragments), "Time" => Time}, + case emqx_ft_assembly:status(ASM2) of + complete -> + % NOTE: this is still possible due to the nature of `SUCHTHATMAYBE` + IsComplete = emqx_ft_assembly:coverage(ASM2), + collect(complete, is_coverage_complete(IsComplete)); + {incomplete, {missing, {segment, _, _}}} -> + collect(incomplete, true) + end + ) + end + ) + ) + ). + +prop_coverage_complete() -> + ?FORALL( + {Filesize, Segsizes}, + {filesize_t(), ?SUCHTHAT([BaseSegsize | _], segsizes_t(), BaseSegsize > 0)}, + ?FORALL( + {Fragments, MaxCoverage}, + noshrink({fragments_t(Filesize, Segsizes), coverage_t(Filesize, Segsizes)}), + begin + % Ensure that we have complete coverage + ASM1 = append_fragments(mk_assembly(Filesize), Fragments ++ MaxCoverage), + {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), + measure( + #{"CoverageMax" => length(MaxCoverage), "Time" => Time}, + case emqx_ft_assembly:status(ASM2) of + complete -> + Coverage = emqx_ft_assembly:coverage(ASM2), + measure( + #{"Coverage" => length(Coverage)}, + is_coverage_complete(Coverage) + ); + {incomplete, _} -> + false + end + ) + end + ) + ). + +measure(NamedSamples, Test) -> + maps:fold(fun(Name, Sample, Acc) -> measure(Name, Sample, Acc) end, Test, NamedSamples). + +is_coverage_complete([]) -> + true; +is_coverage_complete(Coverage = [_ | Tail]) -> + is_coverage_complete(Coverage, Tail). + +is_coverage_complete([_], []) -> + true; +is_coverage_complete( + [{_Node1, #{fragment := {segment, #{offset := O1, size := S1}}}} | Rest], + [{_Node2, #{fragment := {segment, #{offset := O2}}}} | Tail] +) -> + (O1 + S1 == O2) andalso is_coverage_complete(Rest, Tail). + +mk_assembly(Filesize) -> + emqx_ft_assembly:append(emqx_ft_assembly:new(Filesize), node(), mk_filemeta(Filesize)). + +append_fragments(ASMIn, Fragments) -> + lists:foldl( + fun({Node, Frag}, ASM) -> + emqx_ft_assembly:append(ASM, Node, Frag) + end, + ASMIn, + Fragments + ). + +mk_filemeta(Filesize) -> + #{ + path => "MANIFEST.json", + fragment => {filemeta, #{name => ?MODULE_STRING, size => Filesize}} + }. + +mk_segment(Offset, Size) -> + #{ + path => "SEG" ++ integer_to_list(Offset) ++ integer_to_list(Size), + fragment => {segment, #{offset => Offset, size => Size}} + }. + +fragments_t(Filesize, Segsizes = [BaseSegsize | _]) -> + NSegs = Filesize / max(1, BaseSegsize), + scaled(1 + NSegs, list({node_t(), fragment_t(Filesize, Segsizes)})). + +fragments_t(Filesize, Segsizes = [BaseSegsize | _], Hole) -> + NSegs = Filesize / max(1, BaseSegsize), + scaled(1 + NSegs, list({node_t(), fragment_t(Filesize, Segsizes, Hole)})). + +fragment_t(Filesize, Segsizes, Hole) -> + ?SUCHTHATMAYBE( + #{fragment := {segment, #{offset := Offset, size := Size}}}, + fragment_t(Filesize, Segsizes), + (Hole rem Filesize) =< Offset orelse (Hole rem Filesize) > (Offset + Size) + ). + +fragment_t(Filesize, Segsizes) -> + ?LET( + Segsize, + oneof(Segsizes), + ?LET( + Index, + range(0, Filesize div max(1, Segsize)), + mk_segment(Index * Segsize, min(Segsize, Filesize - (Index * Segsize))) + ) + ). + +coverage_t(Filesize, [Segsize | _]) -> + NSegs = Filesize div max(1, Segsize), + [ + {remote_node_t(), mk_segment(I * Segsize, min(Segsize, Filesize - (I * Segsize)))} + || I <- lists:seq(0, NSegs) + ]. + +filesize_t() -> + scaled(4000, non_neg_integer()). + +segsizes_t() -> + ?LET( + BaseSize, + segsize_t(), + oneof([ + [BaseSize, BaseSize * 2], + [BaseSize, BaseSize * 2, BaseSize * 3], + [BaseSize, BaseSize * 2, BaseSize * 5] + ]) + ). + +segsize_t() -> + scaled(50, non_neg_integer()). + +remote_node_t() -> + oneof([ + 'emqx42@emqx.local', + 'emqx43@emqx.local', + 'emqx44@emqx.local' + ]). + +node_t() -> + oneof([ + node(), + 'emqx42@emqx.local', + 'emqx43@emqx.local', + 'emqx44@emqx.local' + ]). From 0c84fc28b041d02558a48cd2a6cff66d1c1ecaf2 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 27 Feb 2023 15:59:52 +0300 Subject: [PATCH 054/156] perf(asm-ft): tradeoff optimality for computational complexity Through squashing segments table into consecutive "runs". --- apps/emqx_ft/src/emqx_ft_assembly.erl | 84 ++++++++++++++++++--------- 1 file changed, 55 insertions(+), 29 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index 2a8ccb485..4625e5ffc 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -40,9 +40,9 @@ filemeta(), {node(), filefrag({filemeta, filemeta()})} ), - segs :: orddict:orddict( + segs :: gb_trees:tree( {emqx_ft:offset(), _Locality, _MEnd, node()}, - filefrag({segment, segmentinfo()}) + [filefrag({segment, segmentinfo()})] ), size :: emqx_ft:bytes() }). @@ -66,7 +66,7 @@ new(Size) -> #asm{ status = {incomplete, {missing, filemeta}}, meta = orddict:new(), - segs = orddict:new(), + segs = gb_trees:empty(), size = Size }. @@ -123,7 +123,7 @@ status(meta, []) -> status(meta, [_M1, _M2 | _] = Metas) -> {error, {inconsistent, [Frag#{node => Node} || {_, {Node, Frag}} <- Metas]}}; status(coverage, #asm{segs = Segments, size = Size}) -> - case coverage(orddict:to_list(Segments), 0, Size) of + case coverage(squash(Segments), Size) of Coverage when is_list(Coverage) -> {complete, Coverage, #{ dominant => dominant(Coverage) @@ -145,54 +145,80 @@ append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) -> Offset = maps:get(offset, Info), Size = maps:get(size, Info), End = Offset + Size, + Index = {Offset, locality(Node), -End, Node}, + Segs = insert(Asm#asm.segs, Index, [Fragment]), Asm#asm{ % TODO % In theory it's possible to have two segments with same offset + size on % different nodes but with differing content. We'd need a checksum to % be able to disambiguate them though. - segs = orddict:store({Offset, locality(Node), -End, Node}, Fragment, Asm#asm.segs) + segs = Segs }. -coverage([{{Offset, _, _, _}, _Segment} | Rest], Cursor, Sz) when Offset < Cursor -> - coverage(Rest, Cursor, Sz); -coverage([{{Cursor, _Locality, MEnd, Node}, Segment} | _Rest] = Segments, Cursor, Sz) -> +squash(Segs) -> + % NOTE + % Here we're "compressing" information about every known segment by adjoining + % nearby segments on the same node into "runs". + squash(Segs, gb_trees:next(gb_trees:iterator(Segs))). + +squash(Segs, {Index = {Offset, Locality, MEnd, _}, Fragments, It}) -> + SegsSquashed = squash_run(gb_trees:delete(Index, Segs), Index, Fragments, It), + ItNext = gb_trees:iterator_from({Offset, Locality, MEnd + 1, 0}, SegsSquashed), + squash(SegsSquashed, gb_trees:next(ItNext)); +squash(Segs, none) -> + Segs. + +squash_run(Segs, {Offset, Locality, MEnd, Node} = Index, Fragments, It) -> + Next = gb_trees:next(It), + case Next of + {{OffsetNext, _, MEndNext, Node} = IndexNext, FragmentsNext, ItNext} when + OffsetNext == -MEnd + -> + SegsNext = gb_trees:delete(IndexNext, Segs), + IndexSquashed = {Offset, Locality, MEndNext, Node}, + squash_run(SegsNext, IndexSquashed, Fragments ++ FragmentsNext, ItNext); + {{OffsetNext, _, _, _}, _, ItNext} when OffsetNext =< -MEnd -> + squash_run(Segs, Index, Fragments, ItNext); + _ -> + insert(Segs, Index, Fragments) + end. + +insert(Segs, Index, Fragments) -> + try + gb_trees:insert(Index, Fragments, Segs) + catch + error:{key_exists, _} -> Segs + end. + +coverage(Segs, Size) -> + coverage(gb_trees:next(gb_trees:iterator(Segs)), 0, Size). + +coverage({{Offset, _, _, _}, _Fragments, It}, Cursor, Sz) when Offset < Cursor -> + coverage(gb_trees:next(It), Cursor, Sz); +coverage({{Cursor, _Locality, MEnd, Node}, Fragments, It}, Cursor, Sz) -> % NOTE % We consider only whole fragments here, so for example from the point of view of % this algo `[{Offset1 = 0, Size1 = 15}, {Offset2 = 10, Size2 = 10}]` has no % coverage. - Tail = tail(Segments), - case coverage(Tail, -MEnd, Sz) of + ItNext = gb_trees:next(It), + case coverage(ItNext, -MEnd, Sz) of Coverage when is_list(Coverage) -> - [{Node, Segment} | Coverage]; + [{Node, Frag} || Frag <- Fragments] ++ Coverage; Missing = {missing, _} -> - case coverage(Tail, Cursor, Sz) of + case coverage(ItNext, Cursor, Sz) of CoverageAlt when is_list(CoverageAlt) -> CoverageAlt; {missing, _} -> Missing end end; -coverage([{{Offset, _MEnd, _, _}, _Segment} | _], Cursor, _Sz) when Offset > Cursor -> +coverage({{Offset, _, _MEnd, _}, _Fragments, _It}, Cursor, _Sz) when Offset > Cursor -> {missing, {segment, Cursor, Offset}}; -coverage([], Cursor, Sz) when Cursor < Sz -> +coverage(none, Cursor, Sz) when Cursor < Sz -> {missing, {segment, Cursor, Sz}}; -coverage([], Cursor, Cursor) -> +coverage(none, Cursor, Cursor) -> []. -tail([Segment | Rest]) -> - tail(Segment, Rest). - -tail({{Cursor, _, MEnd, _}, _} = Segment, [{{Cursor, _, MEnd, _}, _} | Rest]) -> - % NOTE - % Discarding segments with same offset / size, potentially located on other nodes. - % This is an optimization. They won't participate in coverage anyway given we're - % currently optimizing coverage towards locality. Yet if we instead decide to - % optimize for node dominance (e.g. compute such coverage that most of the data - % located on a single node) we'll need to account them again. - tail(Segment, Rest); -tail(_Segment, Tail) -> - Tail. - dominant(Coverage) -> % TODO: needs improvement, better defined _dominance_, maybe some score Freqs = frequencies(fun({Node, Segment}) -> {Node, segsize(Segment)} end, Coverage), From 130601376aaa6018b016a8f77364181df8ea411f Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 27 Feb 2023 19:41:34 +0300 Subject: [PATCH 055/156] perf(ft-asm): express coverage through shortest path on graph Coverage becomes the shortest path problem on graph where nodes are offsets and edges are nodes' segments. Implement a simple Dijkstra algorithm to find one. --- apps/emqx/src/emqx_maybe.erl | 14 ++ apps/emqx_ft/src/emqx_ft_assembly.erl | 202 +++++++++++++++++--------- 2 files changed, 151 insertions(+), 65 deletions(-) diff --git a/apps/emqx/src/emqx_maybe.erl b/apps/emqx/src/emqx_maybe.erl index 8da7c7c90..0b919f7ab 100644 --- a/apps/emqx/src/emqx_maybe.erl +++ b/apps/emqx/src/emqx_maybe.erl @@ -20,6 +20,7 @@ -export([to_list/1]). -export([from_list/1]). +-export([define/2]). -export([apply/2]). -spec to_list(maybe(A)) -> [A]. @@ -34,6 +35,12 @@ from_list([]) -> from_list([Term]) -> Term. +-spec define(maybe(A), B) -> A | B. +define(undefined, Term) -> + Term; +define(Term, _) -> + Term. + %% @doc Apply a function to a maybe argument. -spec apply(fun((maybe(A)) -> maybe(A)), maybe(A)) -> maybe(A). @@ -60,6 +67,13 @@ from_list_test_() -> ?_assertError(_, from_list([1, 2, 3])) ]. +define_test_() -> + [ + ?_assertEqual(42, define(42, undefined)), + ?_assertEqual(<<"default">>, define(undefined, <<"default">>)), + ?_assertEqual(undefined, define(undefined, undefined)) + ]. + apply_test_() -> [ ?_assertEqual(<<"42">>, ?MODULE:apply(fun erlang:integer_to_binary/1, 42)), diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index 4625e5ffc..652636088 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -123,7 +123,7 @@ status(meta, []) -> status(meta, [_M1, _M2 | _] = Metas) -> {error, {inconsistent, [Frag#{node => Node} || {_, {Node, Frag}} <- Metas]}}; status(coverage, #asm{segs = Segments, size = Size}) -> - case coverage(squash(Segments), Size) of + case coverage(Segments, Size) of Coverage when is_list(Coverage) -> {complete, Coverage, #{ dominant => dominant(Coverage) @@ -145,8 +145,7 @@ append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) -> Offset = maps:get(offset, Info), Size = maps:get(size, Info), End = Offset + Size, - Index = {Offset, locality(Node), -End, Node}, - Segs = insert(Asm#asm.segs, Index, [Fragment]), + Segs = add_edge(Asm#asm.segs, Offset, End, locality(Node) * Size, {Node, Fragment}), Asm#asm{ % TODO % In theory it's possible to have two segments with same offset + size on @@ -155,69 +154,143 @@ append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) -> segs = Segs }. -squash(Segs) -> - % NOTE - % Here we're "compressing" information about every known segment by adjoining - % nearby segments on the same node into "runs". - squash(Segs, gb_trees:next(gb_trees:iterator(Segs))). - -squash(Segs, {Index = {Offset, Locality, MEnd, _}, Fragments, It}) -> - SegsSquashed = squash_run(gb_trees:delete(Index, Segs), Index, Fragments, It), - ItNext = gb_trees:iterator_from({Offset, Locality, MEnd + 1, 0}, SegsSquashed), - squash(SegsSquashed, gb_trees:next(ItNext)); -squash(Segs, none) -> - Segs. - -squash_run(Segs, {Offset, Locality, MEnd, Node} = Index, Fragments, It) -> - Next = gb_trees:next(It), - case Next of - {{OffsetNext, _, MEndNext, Node} = IndexNext, FragmentsNext, ItNext} when - OffsetNext == -MEnd - -> - SegsNext = gb_trees:delete(IndexNext, Segs), - IndexSquashed = {Offset, Locality, MEndNext, Node}, - squash_run(SegsNext, IndexSquashed, Fragments ++ FragmentsNext, ItNext); - {{OffsetNext, _, _, _}, _, ItNext} when OffsetNext =< -MEnd -> - squash_run(Segs, Index, Fragments, ItNext); - _ -> - insert(Segs, Index, Fragments) - end. - -insert(Segs, Index, Fragments) -> - try - gb_trees:insert(Index, Fragments, Segs) - catch - error:{key_exists, _} -> Segs - end. - coverage(Segs, Size) -> - coverage(gb_trees:next(gb_trees:iterator(Segs)), 0, Size). + find_shortest_path(Segs, 0, Size). -coverage({{Offset, _, _, _}, _Fragments, It}, Cursor, Sz) when Offset < Cursor -> - coverage(gb_trees:next(It), Cursor, Sz); -coverage({{Cursor, _Locality, MEnd, Node}, Fragments, It}, Cursor, Sz) -> +find_shortest_path(G1, From, To) -> % NOTE - % We consider only whole fragments here, so for example from the point of view of - % this algo `[{Offset1 = 0, Size1 = 15}, {Offset2 = 10, Size2 = 10}]` has no - % coverage. - ItNext = gb_trees:next(It), - case coverage(ItNext, -MEnd, Sz) of - Coverage when is_list(Coverage) -> - [{Node, Frag} || Frag <- Fragments] ++ Coverage; - Missing = {missing, _} -> - case coverage(ItNext, Cursor, Sz) of - CoverageAlt when is_list(CoverageAlt) -> - CoverageAlt; - {missing, _} -> - Missing + % This is a Dijkstra shortest path algorithm implemented on top of `gb_trees`. + % It is one-way right now, for simplicity sake. + G2 = set_cost(G1, From, 0, []), + case find_shortest_path(G2, From, 0, To) of + {found, G3} -> + construct_path(G3, From, To, []); + {error, Last} -> + % NOTE: this is actually just an estimation of what is missing. + {missing, {segment, Last, emqx_maybe:define(find_successor(G2, Last), To)}} + end. + +find_shortest_path(G1, Node, Cost, Target) -> + Edges = get_edges(G1, Node), + G2 = update_neighbours(G1, Node, Cost, Edges), + case take_queued(G2) of + {Target, _NextCost, G3} -> + {found, G3}; + {Next, NextCost, G3} -> + find_shortest_path(G3, Next, NextCost, Target); + none -> + {error, Node} + end. + +construct_path(_G, From, From, Acc) -> + Acc; +construct_path(G, From, To, Acc) -> + {Prev, Label} = get_label(G, To), + construct_path(G, From, Prev, [Label | Acc]). + +update_neighbours(G1, Node, NodeCost, Edges) -> + lists:foldl( + fun({Neighbour, Weight, Label}, GAcc) -> + case is_visited(GAcc, Neighbour) of + false -> + NeighCost = NodeCost + Weight, + CurrentCost = get_cost(GAcc, Neighbour), + case NeighCost < CurrentCost of + true -> + set_cost(GAcc, Neighbour, NeighCost, {Node, Label}); + false -> + GAcc + end; + true -> + GAcc end - end; -coverage({{Offset, _, _MEnd, _}, _Fragments, _It}, Cursor, _Sz) when Offset > Cursor -> - {missing, {segment, Cursor, Offset}}; -coverage(none, Cursor, Sz) when Cursor < Sz -> - {missing, {segment, Cursor, Sz}}; -coverage(none, Cursor, Cursor) -> - []. + end, + G1, + Edges + ). + +add_edge(G, Node, ToNode, WeightIn, EdgeLabel) -> + Edges = tree_lookup({Node}, G, []), + case lists:keyfind(ToNode, 1, Edges) of + {ToNode, Weight, _} when Weight =< WeightIn -> + % NOTE + % Discarding any edges with higher weight here. This is fine as long as we + % optimize for locality. + G; + _ -> + EdgesNext = lists:keystore(ToNode, 1, Edges, {ToNode, WeightIn, EdgeLabel}), + tree_update({Node}, EdgesNext, G) + end. + +get_edges(G, Node) -> + tree_lookup({Node}, G, []). + +get_cost(G, Node) -> + tree_lookup({Node, cost}, G, inf). + +get_label(G, Node) -> + gb_trees:get({Node, label}, G). + +set_cost(G1, Node, Cost, Label) -> + G3 = + case tree_lookup({Node, cost}, G1, inf) of + CostWas when CostWas /= inf -> + {true, G2} = gb_trees:take({queued, CostWas, Node}, G1), + tree_update({queued, Cost, Node}, true, G2); + inf -> + tree_update({queued, Cost, Node}, true, G1) + end, + G4 = tree_update({Node, cost}, Cost, G3), + G5 = tree_update({Node, label}, Label, G4), + G5. + +take_queued(G1) -> + It = gb_trees:iterator_from({queued, 0, 0}, G1), + case gb_trees:next(It) of + {{queued, Cost, Node} = Index, true, _It} -> + {Node, Cost, gb_trees:delete(Index, G1)}; + _ -> + none + end. + +is_visited(G, Node) -> + case tree_lookup({Node, cost}, G, inf) of + inf -> + false; + Cost -> + not tree_lookup({queued, Cost, Node}, G, false) + end. + +find_successor(G, Node) -> + case gb_trees:next(gb_trees:iterator_from({Node}, G)) of + {{Node}, _, It} -> + case gb_trees:next(It) of + {{Successor}, _, _} -> + Successor; + _ -> + undefined + end; + {{Successor}, _, _} -> + Successor; + _ -> + undefined + end. + +tree_lookup(Index, Tree, Default) -> + case gb_trees:lookup(Index, Tree) of + {value, V} -> + V; + none -> + Default + end. + +tree_update(Index, Value, Tree) -> + case gb_trees:take_any(Index, Tree) of + {_, TreeNext} -> + gb_trees:insert(Index, Value, TreeNext); + error -> + gb_trees:insert(Index, Value, Tree) + end. dominant(Coverage) -> % TODO: needs improvement, better defined _dominance_, maybe some score @@ -379,8 +452,7 @@ missing_coverage_test() -> ], Asm = append_many(new(100), Segs), ?assertEqual( - % {incomplete, {missing, {segment, 30, 40}}}, ??? - {incomplete, {missing, {segment, 20, 40}}}, + {incomplete, {missing, {segment, 30, 40}}}, status(coverage, Asm) ). @@ -407,7 +479,7 @@ missing_coverage_with_redudancy_test() -> Asm = append_many(new(100), Segs), ?assertEqual( % {incomplete, {missing, {segment, 50, 60}}}, ??? - {incomplete, {missing, {segment, 20, 40}}}, + {incomplete, {missing, {segment, 60, 100}}}, status(coverage, Asm) ). From b189ee463c72e421d79b023e4d143b867a065ae5 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 28 Feb 2023 14:01:01 +0300 Subject: [PATCH 056/156] feat: add weighted directional graph ADT with shortest path Basically, separate what abstraction was in `emqx_ft_assembly` into dedicated module with a compact interface and a basic testsuite. --- apps/emqx/src/emqx_wdgraph.erl | 186 ++++++++++++++++++++++++++ apps/emqx/test/emqx_wdgraph_tests.erl | 84 ++++++++++++ 2 files changed, 270 insertions(+) create mode 100644 apps/emqx/src/emqx_wdgraph.erl create mode 100644 apps/emqx/test/emqx_wdgraph_tests.erl diff --git a/apps/emqx/src/emqx_wdgraph.erl b/apps/emqx/src/emqx_wdgraph.erl new file mode 100644 index 000000000..3361c52d1 --- /dev/null +++ b/apps/emqx/src/emqx_wdgraph.erl @@ -0,0 +1,186 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Weighted directed graph. +%% +%% Purely functional, built on top of a single `gb_tree`. +%% Weights are currently assumed to be non-negative numbers, hovewer +%% presumably anything that is ≥ 0 should work (but won't typecheck 🥲). + +-module(emqx_wdgraph). + +-export([new/0]). +-export([insert_edge/5]). +-export([find_edge/3]). +-export([get_edges/2]). + +-export([find_shortest_path/3]). + +-export_type([t/0]). +-export_type([t/2]). +-export_type([weight/0]). + +-type gnode() :: term(). +-type weight() :: _NonNegative :: number(). +-type label() :: term(). + +-opaque t() :: t(gnode(), label()). +-opaque t(Node, Label) :: gb_trees:tree({Node}, {Node, weight(), Label}). + +%% + +-spec new() -> t(_, _). +new() -> + gb_trees:empty(). + +%% Add an edge. +%% Nodes are not expected to exist beforehand, and created lazily. +%% There could be only one edge between each pair of nodes, this function +%% replaces any existing edge in the graph. +-spec insert_edge(Node, Node, weight(), Label, t(Node, Label)) -> t(Node, Label). +insert_edge(From, To, Weight, EdgeLabel, G) -> + Edges = tree_lookup({From}, G, []), + EdgesNext = lists:keystore(To, 1, Edges, {To, Weight, EdgeLabel}), + tree_update({From}, EdgesNext, G). + +%% Find exising edge between two nodes, if any. +-spec find_edge(Node, Node, t(Node, Label)) -> {weight(), Label} | false. +find_edge(From, To, G) -> + Edges = tree_lookup({From}, G, []), + case lists:keyfind(To, 1, Edges) of + {To, Weight, Label} -> + {Weight, Label}; + false -> + false + end. + +%% Get all edges from the given node. +-spec get_edges(Node, t(Node, Label)) -> [{Node, weight(), Label}]. +get_edges(Node, G) -> + tree_lookup({Node}, G, []). + +% Find the shortest path between two nodes, if any. If the path exists, return list +% of edge labels along that path. +% This is a Dijkstra shortest path algorithm. It is one-way right now, for +% simplicity sake. +-spec find_shortest_path(Node, Node, t(Node, Label)) -> [Label] | {false, _StoppedAt :: Node}. +find_shortest_path(From, To, G1) -> + % NOTE + % If `From` and `To` are the same node, then path is `[]` even if this + % node does not exist in the graph. + G2 = set_cost(From, 0, [], G1), + case find_shortest_path(From, 0, To, G2) of + {true, G3} -> + construct_path(From, To, [], G3); + {false, Last} -> + {false, Last} + end. + +find_shortest_path(Node, Cost, Target, G1) -> + Edges = get_edges(Node, G1), + G2 = update_neighbours(Node, Cost, Edges, G1), + case take_queued(G2) of + {Target, _NextCost, G3} -> + {true, G3}; + {Next, NextCost, G3} -> + find_shortest_path(Next, NextCost, Target, G3); + none -> + {false, Node} + end. + +construct_path(From, From, Acc, _) -> + Acc; +construct_path(From, To, Acc, G) -> + {Prev, Label} = get_label(To, G), + construct_path(From, Prev, [Label | Acc], G). + +update_neighbours(Node, NodeCost, Edges, G1) -> + lists:foldl( + fun(Edge, GAcc) -> update_neighbour(Node, NodeCost, Edge, GAcc) end, + G1, + Edges + ). + +update_neighbour(Node, NodeCost, {Neighbour, Weight, Label}, G) -> + case is_visited(G, Neighbour) of + false -> + CurrentCost = get_cost(Neighbour, G), + case NodeCost + Weight of + NeighCost when NeighCost < CurrentCost -> + set_cost(Neighbour, NeighCost, {Node, Label}, G); + _ -> + G + end; + true -> + G + end. + +get_cost(Node, G) -> + case tree_lookup({Node, cost}, G, inf) of + {Cost, _Label} -> + Cost; + inf -> + inf + end. + +get_label(Node, G) -> + {_Cost, Label} = gb_trees:get({Node, cost}, G), + Label. + +set_cost(Node, Cost, Label, G1) -> + G3 = + case tree_lookup({Node, cost}, G1, inf) of + {CostWas, _Label} -> + {true, G2} = gb_trees:take({queued, CostWas, Node}, G1), + gb_trees:insert({queued, Cost, Node}, true, G2); + inf -> + gb_trees:insert({queued, Cost, Node}, true, G1) + end, + G4 = tree_update({Node, cost}, {Cost, Label}, G3), + G4. + +take_queued(G1) -> + It = gb_trees:iterator_from({queued, 0, 0}, G1), + case gb_trees:next(It) of + {{queued, Cost, Node} = Index, true, _It} -> + {Node, Cost, gb_trees:delete(Index, G1)}; + _ -> + none + end. + +is_visited(G, Node) -> + case tree_lookup({Node, cost}, G, inf) of + inf -> + false; + {Cost, _Label} -> + not tree_lookup({queued, Cost, Node}, G, false) + end. + +tree_lookup(Index, Tree, Default) -> + case gb_trees:lookup(Index, Tree) of + {value, V} -> + V; + none -> + Default + end. + +tree_update(Index, Value, Tree) -> + case gb_trees:is_defined(Index, Tree) of + true -> + gb_trees:update(Index, Value, Tree); + false -> + gb_trees:insert(Index, Value, Tree) + end. diff --git a/apps/emqx/test/emqx_wdgraph_tests.erl b/apps/emqx/test/emqx_wdgraph_tests.erl new file mode 100644 index 000000000..ece87b966 --- /dev/null +++ b/apps/emqx/test/emqx_wdgraph_tests.erl @@ -0,0 +1,84 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_wdgraph_tests). + +-include_lib("eunit/include/eunit.hrl"). + +empty_test_() -> + G = emqx_wdgraph:new(), + [ + ?_assertEqual([], emqx_wdgraph:get_edges(foo, G)), + ?_assertEqual(false, emqx_wdgraph:find_edge(foo, bar, G)) + ]. + +edges_nodes_test_() -> + G1 = emqx_wdgraph:new(), + G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1), + G3 = emqx_wdgraph:insert_edge(bar, baz, 1, "cheapest", G2), + G4 = emqx_wdgraph:insert_edge(bar, foo, 0, "free", G3), + G5 = emqx_wdgraph:insert_edge(foo, bar, 100, "luxury", G4), + [ + ?_assertEqual({42, "fancy"}, emqx_wdgraph:find_edge(foo, bar, G2)), + ?_assertEqual({100, "luxury"}, emqx_wdgraph:find_edge(foo, bar, G5)), + ?_assertEqual([{bar, 100, "luxury"}], emqx_wdgraph:get_edges(foo, G5)), + + ?_assertEqual({1, "cheapest"}, emqx_wdgraph:find_edge(bar, baz, G5)), + ?_assertEqual([{baz, 1, "cheapest"}, {foo, 0, "free"}], emqx_wdgraph:get_edges(bar, G5)) + ]. + +nonexistent_nodes_path_test_() -> + G1 = emqx_wdgraph:new(), + G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1), + G3 = emqx_wdgraph:insert_edge(bar, baz, 1, "cheapest", G2), + [ + ?_assertEqual( + {false, nosuchnode}, + emqx_wdgraph:find_shortest_path(nosuchnode, baz, G3) + ), + ?_assertEqual( + [], + emqx_wdgraph:find_shortest_path(nosuchnode, nosuchnode, G3) + ) + ]. + +nonexistent_path_test_() -> + G1 = emqx_wdgraph:new(), + G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1), + G3 = emqx_wdgraph:insert_edge(baz, boo, 1, "cheapest", G2), + G4 = emqx_wdgraph:insert_edge(boo, last, 3.5, "change", G3), + [ + ?_assertEqual( + {false, last}, + emqx_wdgraph:find_shortest_path(baz, foo, G4) + ), + ?_assertEqual( + {false, bar}, + emqx_wdgraph:find_shortest_path(foo, last, G4) + ) + ]. + +shortest_path_test() -> + G1 = emqx_wdgraph:new(), + G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1), + G3 = emqx_wdgraph:insert_edge(bar, baz, 1, "cheapest", G2), + G4 = emqx_wdgraph:insert_edge(baz, last, 0, "free", G3), + G5 = emqx_wdgraph:insert_edge(bar, last, 100, "luxury", G4), + G6 = emqx_wdgraph:insert_edge(bar, foo, 0, "comeback", G5), + ?assertEqual( + ["fancy", "cheapest", "free"], + emqx_wdgraph:find_shortest_path(foo, last, G6) + ). From 8e6f960c091c953160777af32a5ea8e56aacaa4f Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 28 Feb 2023 14:03:46 +0300 Subject: [PATCH 057/156] refactor(ft): employ `emqx_wdgraph` for coverage computation Also describe how coverage problem maps to shortest path problem. --- apps/emqx_ft/src/emqx_ft_assembly.erl | 158 ++++---------------------- 1 file changed, 23 insertions(+), 135 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index 652636088..bea320bbf 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -40,10 +40,7 @@ filemeta(), {node(), filefrag({filemeta, filemeta()})} ), - segs :: gb_trees:tree( - {emqx_ft:offset(), _Locality, _MEnd, node()}, - [filefrag({segment, segmentinfo()})] - ), + segs :: emqx_wdgraph:t(emqx_ft:offset(), {node(), filefrag({segment, segmentinfo()})}), size :: emqx_ft:bytes() }). @@ -66,7 +63,7 @@ new(Size) -> #asm{ status = {incomplete, {missing, filemeta}}, meta = orddict:new(), - segs = gb_trees:empty(), + segs = emqx_wdgraph:new(), size = Size }. @@ -154,142 +151,32 @@ append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) -> segs = Segs }. -coverage(Segs, Size) -> - find_shortest_path(Segs, 0, Size). - -find_shortest_path(G1, From, To) -> +add_edge(Segs, Offset, End, Weight, Label) -> % NOTE - % This is a Dijkstra shortest path algorithm implemented on top of `gb_trees`. - % It is one-way right now, for simplicity sake. - G2 = set_cost(G1, From, 0, []), - case find_shortest_path(G2, From, 0, To) of - {found, G3} -> - construct_path(G3, From, To, []); - {error, Last} -> - % NOTE: this is actually just an estimation of what is missing. - {missing, {segment, Last, emqx_maybe:define(find_successor(G2, Last), To)}} - end. - -find_shortest_path(G1, Node, Cost, Target) -> - Edges = get_edges(G1, Node), - G2 = update_neighbours(G1, Node, Cost, Edges), - case take_queued(G2) of - {Target, _NextCost, G3} -> - {found, G3}; - {Next, NextCost, G3} -> - find_shortest_path(G3, Next, NextCost, Target); - none -> - {error, Node} - end. - -construct_path(_G, From, From, Acc) -> - Acc; -construct_path(G, From, To, Acc) -> - {Prev, Label} = get_label(G, To), - construct_path(G, From, Prev, [Label | Acc]). - -update_neighbours(G1, Node, NodeCost, Edges) -> - lists:foldl( - fun({Neighbour, Weight, Label}, GAcc) -> - case is_visited(GAcc, Neighbour) of - false -> - NeighCost = NodeCost + Weight, - CurrentCost = get_cost(GAcc, Neighbour), - case NeighCost < CurrentCost of - true -> - set_cost(GAcc, Neighbour, NeighCost, {Node, Label}); - false -> - GAcc - end; - true -> - GAcc - end - end, - G1, - Edges - ). - -add_edge(G, Node, ToNode, WeightIn, EdgeLabel) -> - Edges = tree_lookup({Node}, G, []), - case lists:keyfind(ToNode, 1, Edges) of - {ToNode, Weight, _} when Weight =< WeightIn -> + % We are expressing coverage problem as a shortest path problem on weighted directed + % graph, where nodes are segments offsets, two nodes are connected with edge if + % there is a segment which "covers" these offsets (i.e. it starts at first node's + % offset and ends at second node's offst) and weights are segments sizes adjusted + % for locality (i.e. weight are always 0 for any local segment). + case emqx_wdgraph:find_edge(Offset, End, Segs) of + {WeightWas, _Label} when WeightWas =< Weight -> % NOTE % Discarding any edges with higher weight here. This is fine as long as we % optimize for locality. - G; + Segs; _ -> - EdgesNext = lists:keystore(ToNode, 1, Edges, {ToNode, WeightIn, EdgeLabel}), - tree_update({Node}, EdgesNext, G) + emqx_wdgraph:insert_edge(Offset, End, Weight, Label, Segs) end. -get_edges(G, Node) -> - tree_lookup({Node}, G, []). - -get_cost(G, Node) -> - tree_lookup({Node, cost}, G, inf). - -get_label(G, Node) -> - gb_trees:get({Node, label}, G). - -set_cost(G1, Node, Cost, Label) -> - G3 = - case tree_lookup({Node, cost}, G1, inf) of - CostWas when CostWas /= inf -> - {true, G2} = gb_trees:take({queued, CostWas, Node}, G1), - tree_update({queued, Cost, Node}, true, G2); - inf -> - tree_update({queued, Cost, Node}, true, G1) - end, - G4 = tree_update({Node, cost}, Cost, G3), - G5 = tree_update({Node, label}, Label, G4), - G5. - -take_queued(G1) -> - It = gb_trees:iterator_from({queued, 0, 0}, G1), - case gb_trees:next(It) of - {{queued, Cost, Node} = Index, true, _It} -> - {Node, Cost, gb_trees:delete(Index, G1)}; - _ -> - none - end. - -is_visited(G, Node) -> - case tree_lookup({Node, cost}, G, inf) of - inf -> - false; - Cost -> - not tree_lookup({queued, Cost, Node}, G, false) - end. - -find_successor(G, Node) -> - case gb_trees:next(gb_trees:iterator_from({Node}, G)) of - {{Node}, _, It} -> - case gb_trees:next(It) of - {{Successor}, _, _} -> - Successor; - _ -> - undefined - end; - {{Successor}, _, _} -> - Successor; - _ -> - undefined - end. - -tree_lookup(Index, Tree, Default) -> - case gb_trees:lookup(Index, Tree) of - {value, V} -> - V; - none -> - Default - end. - -tree_update(Index, Value, Tree) -> - case gb_trees:take_any(Index, Tree) of - {_, TreeNext} -> - gb_trees:insert(Index, Value, TreeNext); - error -> - gb_trees:insert(Index, Value, Tree) +coverage(Segs, Size) -> + case emqx_wdgraph:find_shortest_path(0, Size, Segs) of + Path when is_list(Path) -> + Path; + {false, LastOffset} -> + % NOTE + % This is far from being accurate, but needs no hairy specifics in the + % `emqx_wdgraph` interface. + {missing, {segment, LastOffset, Size}} end. dominant(Coverage) -> @@ -452,7 +339,8 @@ missing_coverage_test() -> ], Asm = append_many(new(100), Segs), ?assertEqual( - {incomplete, {missing, {segment, 30, 40}}}, + % {incomplete, {missing, {segment, 30, 40}}} would be more accurate + {incomplete, {missing, {segment, 30, 100}}}, status(coverage, Asm) ). From 788e76ed2d343aa2924f9707b2261948f57dbded Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 28 Feb 2023 16:55:23 +0300 Subject: [PATCH 058/156] test: make assembly proptests waste less memory --- .../test/props/prop_emqx_ft_assembly.erl | 71 ++++++++++--------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl b/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl index ebebf0e65..a437e8dfe 100644 --- a/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl +++ b/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl @@ -28,11 +28,11 @@ prop_coverage() -> {filesize_t(), segsizes_t()}, ?FORALL( Fragments, - noshrink(fragments_t(Filesize, Segsizes)), + noshrink(segments_t(Filesize, Segsizes)), ?TIMEOUT( ?COVERAGE_TIMEOUT, begin - ASM1 = append_fragments(mk_assembly(Filesize), Fragments), + ASM1 = append_segments(mk_assembly(Filesize), Fragments), {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), measure( #{"Fragments" => length(Fragments), "Time" => Time}, @@ -58,11 +58,11 @@ prop_coverage_likely_incomplete() -> {filesize_t(), segsizes_t(), filesize_t()}, ?FORALL( Fragments, - noshrink(fragments_t(Filesize, Segsizes, Hole)), + noshrink(segments_t(Filesize, Segsizes, Hole)), ?TIMEOUT( ?COVERAGE_TIMEOUT, begin - ASM1 = append_fragments(mk_assembly(Filesize), Fragments), + ASM1 = append_segments(mk_assembly(Filesize), Fragments), {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), measure( #{"Fragments" => length(Fragments), "Time" => Time}, @@ -85,17 +85,19 @@ prop_coverage_complete() -> {Filesize, Segsizes}, {filesize_t(), ?SUCHTHAT([BaseSegsize | _], segsizes_t(), BaseSegsize > 0)}, ?FORALL( - {Fragments, MaxCoverage}, - noshrink({fragments_t(Filesize, Segsizes), coverage_t(Filesize, Segsizes)}), + {Fragments, RemoteNode}, + noshrink({segments_t(Filesize, Segsizes), remote_node_t()}), begin % Ensure that we have complete coverage - ASM1 = append_fragments(mk_assembly(Filesize), Fragments ++ MaxCoverage), - {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), + ASM1 = mk_assembly(Filesize), + ASM2 = append_coverage(ASM1, RemoteNode, Filesize, Segsizes), + ASM3 = append_segments(ASM2, Fragments), + {Time, ASM4} = timer:tc(emqx_ft_assembly, update, [ASM3]), measure( - #{"CoverageMax" => length(MaxCoverage), "Time" => Time}, - case emqx_ft_assembly:status(ASM2) of + #{"CoverageMax" => nsegs(Filesize, Segsizes), "Time" => Time}, + case emqx_ft_assembly:status(ASM4) of complete -> - Coverage = emqx_ft_assembly:coverage(ASM2), + Coverage = emqx_ft_assembly:coverage(ASM4), measure( #{"Coverage" => length(Coverage)}, is_coverage_complete(Coverage) @@ -127,15 +129,26 @@ is_coverage_complete( mk_assembly(Filesize) -> emqx_ft_assembly:append(emqx_ft_assembly:new(Filesize), node(), mk_filemeta(Filesize)). -append_fragments(ASMIn, Fragments) -> +append_segments(ASMIn, Fragments) -> lists:foldl( - fun({Node, Frag}, ASM) -> - emqx_ft_assembly:append(ASM, Node, Frag) + fun({Node, {Offset, Size}}, ASM) -> + emqx_ft_assembly:append(ASM, Node, mk_segment(Offset, Size)) end, ASMIn, Fragments ). +append_coverage(ASM, Node, Filesize, Segsizes = [BaseSegsize | _]) -> + append_coverage(ASM, Node, Filesize, BaseSegsize, 0, nsegs(Filesize, Segsizes)). + +append_coverage(ASM, Node, Filesize, Segsize, I, NSegs) when I < NSegs -> + Offset = I * Segsize, + Size = min(Segsize, Filesize - Offset), + ASMNext = emqx_ft_assembly:append(ASM, Node, mk_segment(Offset, Size)), + append_coverage(ASMNext, Node, Filesize, Segsize, I + 1, NSegs); +append_coverage(ASM, _Node, _Filesize, _Segsize, _, _NSegs) -> + ASM. + mk_filemeta(Filesize) -> #{ path => "MANIFEST.json", @@ -148,39 +161,33 @@ mk_segment(Offset, Size) -> fragment => {segment, #{offset => Offset, size => Size}} }. -fragments_t(Filesize, Segsizes = [BaseSegsize | _]) -> - NSegs = Filesize / max(1, BaseSegsize), - scaled(1 + NSegs, list({node_t(), fragment_t(Filesize, Segsizes)})). +nsegs(Filesize, [BaseSegsize | _]) -> + Filesize div max(1, BaseSegsize) + 1. -fragments_t(Filesize, Segsizes = [BaseSegsize | _], Hole) -> - NSegs = Filesize / max(1, BaseSegsize), - scaled(1 + NSegs, list({node_t(), fragment_t(Filesize, Segsizes, Hole)})). +segments_t(Filesize, Segsizes) -> + scaled(nsegs(Filesize, Segsizes), list({node_t(), segment_t(Filesize, Segsizes)})). -fragment_t(Filesize, Segsizes, Hole) -> +segments_t(Filesize, Segsizes, Hole) -> + scaled(nsegs(Filesize, Segsizes), list({node_t(), segment_t(Filesize, Segsizes, Hole)})). + +segment_t(Filesize, Segsizes, Hole) -> ?SUCHTHATMAYBE( - #{fragment := {segment, #{offset := Offset, size := Size}}}, - fragment_t(Filesize, Segsizes), + {Offset, Size}, + segment_t(Filesize, Segsizes), (Hole rem Filesize) =< Offset orelse (Hole rem Filesize) > (Offset + Size) ). -fragment_t(Filesize, Segsizes) -> +segment_t(Filesize, Segsizes) -> ?LET( Segsize, oneof(Segsizes), ?LET( Index, range(0, Filesize div max(1, Segsize)), - mk_segment(Index * Segsize, min(Segsize, Filesize - (Index * Segsize))) + {Index * Segsize, min(Segsize, Filesize - (Index * Segsize))} ) ). -coverage_t(Filesize, [Segsize | _]) -> - NSegs = Filesize div max(1, Segsize), - [ - {remote_node_t(), mk_segment(I * Segsize, min(Segsize, Filesize - (I * Segsize)))} - || I <- lists:seq(0, NSegs) - ]. - filesize_t() -> scaled(4000, non_neg_integer()). From e9f98adca26b9cb13366bfadc6ddfde7337d8f1b Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 2 Mar 2023 18:53:19 +0300 Subject: [PATCH 059/156] test: make content_gen emit meta with each chunk Which will also contain total content size, to increase this tool's utility. --- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_content_gen.erl | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index a896922d1..8df471deb 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -109,7 +109,7 @@ t_assemble_complete_local_transfer(Config) -> ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta), _ = emqx_ft_content_gen:consume( Gen, - fun({Content, SegmentNum, _SegmentCount}) -> + fun({Content, SegmentNum, _Meta}) -> Offset = (SegmentNum - 1) * SegmentSize, ?assertEqual( ok, diff --git a/apps/emqx_ft/test/emqx_ft_content_gen.erl b/apps/emqx_ft/test/emqx_ft_content_gen.erl index bd24d8c94..286c1a588 100644 --- a/apps/emqx_ft/test/emqx_ft_content_gen.erl +++ b/apps/emqx_ft/test/emqx_ft_content_gen.erl @@ -42,7 +42,7 @@ -type payload() :: {Seed :: term(), Size :: integer()}. -type binary_payload() :: { - binary(), _ChunkNum :: non_neg_integer(), _ChunkCnt :: non_neg_integer() + binary(), _ChunkNum :: non_neg_integer(), _Meta :: #{} }. -type cont(Data) :: @@ -200,7 +200,10 @@ generate_chunk(Seed, Offset, ChunkSize, Size) -> || I <- lists:seq(Offset div 16, To div 16) ]), ChunkNum = Offset div ChunkSize + 1, - ChunkCnt = ceil(Size / ChunkSize), + Meta = #{ + chunk_size => ChunkSize, + chunk_count => ceil(Size / ChunkSize) + }, Chunk = case Offset + ChunkSize of NextOffset when NextOffset > Size -> @@ -208,7 +211,7 @@ generate_chunk(Seed, Offset, ChunkSize, Size) -> _ -> Payload end, - {Chunk, ChunkNum, ChunkCnt}. + {Chunk, ChunkNum, Meta}. %% @doc First argument is a chunk number, the second one is a seed. %% This implementation is hardly efficient, but it was chosen for From 715816e67bbf6ed206d02269c1c53c5d947aae35 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 2 Mar 2023 19:05:48 +0300 Subject: [PATCH 060/156] feat(ft): add GC logic and process for the FS storage backend --- apps/emqx/test/emqx_common_test_helpers.erl | 1 + apps/emqx_ft/include/emqx_ft_storage_fs.hrl | 29 ++ apps/emqx_ft/src/emqx_ft_assembler.erl | 17 + apps/emqx_ft/src/emqx_ft_conf.erl | 29 ++ apps/emqx_ft/src/emqx_ft_storage.erl | 21 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 80 ++++- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 337 ++++++++++++++++++ apps/emqx_ft/src/emqx_ft_sup.erl | 2 +- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 205 +++++++++++ 9 files changed, 702 insertions(+), 19 deletions(-) create mode 100644 apps/emqx_ft/include/emqx_ft_storage_fs.hrl create mode 100644 apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl create mode 100644 apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 5dd587a4e..16920d60b 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -30,6 +30,7 @@ start_apps/1, start_apps/2, start_apps/3, + start_app/2, stop_apps/1, reload/2, app_path/2, diff --git a/apps/emqx_ft/include/emqx_ft_storage_fs.hrl b/apps/emqx_ft/include/emqx_ft_storage_fs.hrl new file mode 100644 index 000000000..72ebe586a --- /dev/null +++ b/apps/emqx_ft/include/emqx_ft_storage_fs.hrl @@ -0,0 +1,29 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_FT_STORAGE_FS_HRL). +-define(EMQX_FT_STORAGE_FS_HRL, true). + +-record(gcstats, { + started_at :: integer(), + finished_at :: integer() | undefined, + files = 0 :: non_neg_integer(), + directories = 0 :: non_neg_integer(), + space = 0 :: non_neg_integer(), + errors = #{} :: #{_GCSubject => {error, _}} +}). + +-endif. diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 9e3ddfcbd..38ccf13ac 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -126,6 +126,7 @@ handle_event(internal, _, {assemble, []}, St = #st{}) -> handle_event(internal, _, complete, St = #st{assembly = Asm, file = Handle}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), Result = emqx_ft_storage_fs:complete(St#st.storage, St#st.transfer, Filemeta, Handle), + ok = maybe_garbage_collect(Result, St), {stop, {shutdown, Result}}. pread(Node, Segment, St) when Node =:= node() -> @@ -135,5 +136,21 @@ pread(Node, Segment, St) -> %% +maybe_garbage_collect(ok, St = #st{storage = Storage, transfer = Transfer}) -> + Nodes = get_coverage_nodes(St), + emqx_ft_storage_fs_gc:collect(Storage, Transfer, Nodes); +maybe_garbage_collect({error, _}, _St) -> + ok. + +get_coverage_nodes(St) -> + Coverage = emqx_ft_assembly:coverage(St#st.assembly), + ordsets:to_list( + lists:foldl( + fun({Node, _Segment}, Acc) -> ordsets:add_element(Node, Acc) end, + ordsets:new(), + Coverage + ) + ). + segsize(#{fragment := {segment, Info}}) -> maps:get(size, Info). diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index d56dd8d32..444462716 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -20,6 +20,11 @@ -behaviour(emqx_config_handler). +%% Accessors +-export([storage/0]). +-export([gc_interval/1]). +-export([segments_ttl/1]). + %% Load/Unload -export([ load/0, @@ -32,6 +37,30 @@ post_config_update/5 ]). +-type milliseconds() :: non_neg_integer(). +-type seconds() :: non_neg_integer(). + +%%-------------------------------------------------------------------- +%% Accessors +%%-------------------------------------------------------------------- + +-spec storage() -> _Storage | disabled. +storage() -> + emqx_config:get([file_transfer, storage], disabled). + +-spec gc_interval(_Storage) -> milliseconds(). +gc_interval(_Storage) -> + % TODO: config wiring + application:get_env(emqx_ft, gc_interval, timer:minutes(10)). + +-spec segments_ttl(_Storage) -> {_Min :: seconds(), _Max :: seconds()}. +segments_ttl(_Storage) -> + % TODO: config wiring + { + application:get_env(emqx_ft, min_segments_ttl, 60), + application:get_env(emqx_ft, max_segments_ttl, 72 * 3600) + }. + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 7a95a0454..0b8c38736 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -18,6 +18,8 @@ -export( [ + child_spec/0, + store_filemeta/2, store_segment/2, assemble/2, @@ -64,6 +66,17 @@ %% API %%-------------------------------------------------------------------- +-spec child_spec() -> + [supervisor:child_spec()]. +child_spec() -> + try + Mod = mod(), + Mod:child_spec(storage()) + catch + error:disabled -> []; + error:undef -> [] + end. + -spec store_filemeta(emqx_ft:transfer(), emqx_ft:filemeta()) -> ok | {async, pid()} | {error, term()}. store_filemeta(Transfer, FileMeta) -> @@ -99,6 +112,8 @@ with_storage_type(Type, Fun, Args) -> #{type := Type} -> Mod = mod(Storage), apply(Mod, Fun, [Storage | Args]); + disabled -> + {error, disabled}; _ -> {error, {invalid_storage_type, Type}} end. @@ -108,7 +123,7 @@ with_storage_type(Type, Fun, Args) -> %%-------------------------------------------------------------------- storage() -> - emqx_config:get([file_transfer, storage]). + emqx_ft_conf:storage(). mod() -> mod(storage()). @@ -116,6 +131,8 @@ mod() -> mod(Storage) -> case Storage of #{type := local} -> - emqx_ft_storage_fs + emqx_ft_storage_fs; + disabled -> + error(disabled) % emqx_ft_storage_dummy end. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 2cc19d2a2..b8aef5276 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -14,6 +14,12 @@ %% limitations under the License. %%-------------------------------------------------------------------- +%% Filesystem storage backend +%% +%% NOTE +%% If you plan to change storage layout please consult `emqx_ft_storage_fs_gc` +%% to see how much it would break or impair GC. + -module(emqx_ft_storage_fs). -behaviour(emqx_ft_storage). @@ -21,14 +27,22 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). +-export([child_spec/1]). + -export([store_filemeta/3]). -export([store_segment/3]). +-export([read_filemeta/2]). -export([list/3]). -export([pread/5]). -export([assemble/3]). -export([transfers/1]). +% GC API +% TODO: This is quickly becomes hairy. +-export([get_subdir/2]). +-export([get_subdir/3]). + -export([ready_transfers_local/1]). -export([get_ready_transfer_local/3]). @@ -40,6 +54,7 @@ -export([write/2]). -export([discard/1]). +-export_type([storage/0]). -export_type([filefrag/1]). -export_type([filefrag/0]). -export_type([transferinfo/0]). @@ -79,7 +94,9 @@ -define(MANIFEST, "MANIFEST.json"). -define(SEGMENT, "SEG"). --type storage() :: emqx_ft_storage:storage(). +-type storage() :: #{ + root => file:name() +}. -type file_error() :: file:posix() @@ -88,13 +105,25 @@ %% System limit (e.g. number of ports) reached. | system_limit. +%% Related resources childspecs +-spec child_spec(storage()) -> + [supervisor:child_spec()]. +child_spec(Storage) -> + [ + #{ + id => emqx_ft_storage_fs_gc, + start => {emqx_ft_storage_fs_gc, start_link, [Storage]}, + restart => permanent + } + ]. + %% Store manifest in the backing filesystem. %% Atomic operation. -spec store_filemeta(storage(), transfer(), filemeta()) -> % Quota? Some lower level errors? ok | {error, conflict} | {error, file_error()}. store_filemeta(Storage, Transfer, Meta) -> - Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], ?MANIFEST), + Filepath = mk_filepath(Storage, Transfer, get_subdirs_for(fragment), ?MANIFEST), case read_file(Filepath, fun decode_filemeta/1) of {ok, Meta} -> _ = touch_file(Filepath), @@ -119,9 +148,16 @@ store_filemeta(Storage, Transfer, Meta) -> % Quota? Some lower level errors? ok | {error, file_error()}. store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> - Filepath = mk_filepath(Storage, Transfer, [?FRAGDIR], mk_segment_filename(Segment)), + Filename = mk_segment_filename(Segment), + Filepath = mk_filepath(Storage, Transfer, get_subdirs_for(fragment), Filename), write_file_atomic(Storage, Transfer, Filepath, Content). +-spec read_filemeta(storage(), transfer()) -> + {ok, filefrag({filemeta, filemeta()})} | {error, corrupted} | {error, file_error()}. +read_filemeta(Storage, Transfer) -> + Filepath = mk_filepath(Storage, Transfer, get_subdirs_for(fragment), ?MANIFEST), + read_file(Filepath, fun decode_filemeta/1). + -spec list(storage(), transfer(), _What :: fragment | result) -> % Some lower level errors? {error, notfound}? % Result will contain zero or only one filemeta. @@ -143,11 +179,6 @@ list(Storage, Transfer, What) -> Error end. -get_subdirs_for(fragment) -> - [?FRAGDIR]; -get_subdirs_for(result) -> - [?RESULTDIR]. - get_filefrag_fun_for(fragment) -> fun mk_filefrag/2; get_filefrag_fun_for(result) -> @@ -329,6 +360,23 @@ read_transferinfo(Storage, Transfer, Acc) -> Acc end. +-spec get_subdir(storage(), transfer()) -> + file:name(). +get_subdir(Storage, Transfer) -> + mk_filedir(Storage, Transfer, []). + +-spec get_subdir(storage(), transfer(), fragment | temporary | result) -> + file:name(). +get_subdir(Storage, Transfer, What) -> + mk_filedir(Storage, Transfer, get_subdirs_for(What)). + +get_subdirs_for(fragment) -> + [?FRAGDIR]; +get_subdirs_for(temporary) -> + [?TEMPDIR]; +get_subdirs_for(result) -> + [?RESULTDIR]. + %% -type handle() :: {file:name(), io:device(), crypto:hash_state()}. @@ -341,7 +389,7 @@ open_file(Storage, Transfer, Filemeta) -> _ = filelib:ensure_dir(TempFilepath), case file:open(TempFilepath, [write, raw, binary]) of {ok, Handle} -> - _ = file:truncate(Handle), + % TODO: preserve filemeta {ok, {TempFilepath, Handle, init_checksum(Filemeta)}}; {error, _} = Error -> Error @@ -359,8 +407,8 @@ write({Filepath, IoDevice, Ctx}, IoData) -> -spec complete(storage(), transfer(), filemeta(), handle()) -> ok | {error, {checksum, _Algo, _Computed}} | {error, file_error()}. -complete(Storage, Transfer, Filemeta, Handle = {Filepath, IoDevice, Ctx}) -> - TargetFilepath = mk_filepath(Storage, Transfer, [?RESULTDIR], maps:get(name, Filemeta)), +complete(Storage, Transfer, Filemeta = #{name := Filename}, Handle = {Filepath, IoDevice, Ctx}) -> + TargetFilepath = mk_filepath(Storage, Transfer, get_subdirs_for(result), Filename), case verify_checksum(Ctx, Filemeta) of ok -> ok = file:close(IoDevice), @@ -491,7 +539,7 @@ write_file_atomic(Storage, Transfer, Filepath, Content) when is_binary(Content) mk_temp_filepath(Storage, Transfer, Filename) -> Unique = erlang:unique_integer([positive]), - filename:join(mk_filedir(Storage, Transfer, [?TEMPDIR]), mk_filename([Unique, ".", Filename])). + filename:join(get_subdir(Storage, Transfer, temporary), mk_filename([Unique, ".", Filename])). mk_filename(Comps) -> lists:append(lists:map(fun mk_filename_component/1, Comps)). @@ -516,9 +564,9 @@ filtermap_files(Fun, Dirname, Filenames) -> lists:filtermap(fun(Filename) -> Fun(Dirname, Filename) end, Filenames). mk_filefrag(Dirname, Filename = ?MANIFEST) -> - mk_filefrag(Dirname, Filename, filemeta, fun read_filemeta/2); + mk_filefrag(Dirname, Filename, filemeta, fun read_frag_filemeta/2); mk_filefrag(Dirname, Filename = ?SEGMENT ++ _) -> - mk_filefrag(Dirname, Filename, segment, fun read_segmentinfo/2); + mk_filefrag(Dirname, Filename, segment, fun read_frag_segmentinfo/2); mk_filefrag(_Dirname, _Filename) -> ?tp(warning, "rogue_file_found", #{ directory => _Dirname, @@ -554,10 +602,10 @@ mk_filefrag(Dirname, Filename, Tag, Fun) -> false end. -read_filemeta(_Filename, Filepath) -> +read_frag_filemeta(_Filename, Filepath) -> read_file(Filepath, fun decode_filemeta/1). -read_segmentinfo(Filename, _Filepath) -> +read_frag_segmentinfo(Filename, _Filepath) -> break_segment_filename(Filename). filename_to_binary(S) when is_list(S) -> unicode:characters_to_binary(S); diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl new file mode 100644 index 000000000..6d493337a --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -0,0 +1,337 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Filesystem storage GC +%% +%% This is conceptually a part of the Filesystem storage backend, even +%% though it's tied to the backend module with somewhat narrow interface. + +-module(emqx_ft_storage_fs_gc). + +-include_lib("emqx_ft/include/emqx_ft_storage_fs.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). + +-export([start_link/1]). + +-export([collect/1]). +-export([collect/3]). +-export([reset/1]). + +-behaviour(gen_server). +-export([init/1]). +-export([handle_call/3]). +-export([handle_cast/2]). +-export([handle_info/2]). + +-record(st, { + storage :: emqx_ft_storage_fs:storage(), + next_gc_timer :: maybe(reference()), + last_gc :: maybe(gcstats()) +}). + +-type gcstats() :: #gcstats{}. + +%% + +start_link(Storage) -> + gen_server:start_link(mk_server_ref(Storage), ?MODULE, Storage, []). + +-spec collect(emqx_ft_storage_fs:storage()) -> gcstats(). +collect(Storage) -> + gen_server:call(mk_server_ref(Storage), {collect, erlang:system_time()}, infinity). + +-spec reset(emqx_ft_storage_fs:storage()) -> ok. +reset(Storage) -> + gen_server:cast(mk_server_ref(Storage), reset). + +collect(Storage, Transfer, Nodes) -> + gen_server:cast(mk_server_ref(Storage), {collect, Transfer, Nodes}). + +mk_server_ref(Storage) -> + % TODO + {via, gproc, {n, l, {?MODULE, get_storage_root(Storage)}}}. + +%% + +init(Storage) -> + St = #st{storage = Storage}, + {ok, start_timer(St)}. + +handle_call({collect, CalledAt}, _From, St) -> + StNext = maybe_collect_garbage(CalledAt, St), + {reply, StNext#st.last_gc, StNext}; +handle_call(Call, From, St) -> + ?SLOG(error, #{msg => "unexpected_call", call => Call, from => From}), + {noreply, St}. + +% TODO +% handle_cast({collect, Transfer, [Node | Rest]}, St) -> +% ok = do_collect_transfer(Transfer, Node, St), +% ok = collect(self(), Transfer, Rest), +% {noreply, St}; +handle_cast(reset, St) -> + {noreply, reset_timer(St)}; +handle_cast(Cast, St) -> + ?SLOG(error, #{msg => "unexpected_cast", cast => Cast}), + {noreply, St}. + +handle_info({timeout, TRef, collect}, St = #st{next_gc_timer = TRef}) -> + StNext = do_collect_garbage(St), + {noreply, start_timer(StNext#st{next_gc_timer = undefined})}. + +% do_collect_transfer(Transfer, Node, St = #st{storage = Storage}) when Node == node() -> +% Stats = try_collect_transfer(Storage, Transfer, complete, init_gcstats()), +% ok = maybe_report(Stats, St), +% ok. + +maybe_collect_garbage(_CalledAt, St = #st{last_gc = undefined}) -> + do_collect_garbage(St); +maybe_collect_garbage(CalledAt, St = #st{last_gc = #gcstats{finished_at = FinishedAt}}) -> + case FinishedAt > CalledAt of + true -> + St; + false -> + reset_timer(do_collect_garbage(St)) + end. + +do_collect_garbage(St = #st{storage = Storage}) -> + Stats = collect_garbage(Storage), + ok = maybe_report(Stats, St), + St#st{last_gc = Stats}. + +maybe_report(#gcstats{errors = Errors}, #st{storage = Storage}) when map_size(Errors) > 0 -> + ?tp(warning, "garbage_collection_errors", #{errors => Errors, storage => Storage}); +maybe_report(#gcstats{} = _Stats, #st{storage = _Storage}) -> + ?tp(garbage_collection, #{stats => _Stats, storage => _Storage}). + +start_timer(St = #st{next_gc_timer = undefined}) -> + Delay = emqx_ft_conf:gc_interval(St#st.storage), + St#st{next_gc_timer = emqx_misc:start_timer(Delay, collect)}. + +reset_timer(St = #st{next_gc_timer = undefined}) -> + start_timer(St); +reset_timer(St = #st{next_gc_timer = TRef}) -> + ok = emqx_misc:cancel_timer(TRef), + start_timer(St#st{next_gc_timer = undefined}). + +%% + +collect_garbage(Storage) -> + Stats = init_gcstats(), + {ok, Transfers} = emqx_ft_storage_fs:transfers(Storage), + collect_garbage(Storage, Transfers, Stats). + +collect_garbage(Storage, Transfers, Stats) -> + finish_gcstats( + maps:fold( + fun(Transfer, TransferInfo, StatsAcc) -> + % TODO: throttling? + try_collect_transfer(Storage, Transfer, TransferInfo, StatsAcc) + end, + Stats, + Transfers + ) + ). + +try_collect_transfer(Storage, Transfer, #{status := complete}, Stats) -> + % File transfer is complete. + % We should be good to delete fragments and temporary files with their respective + % directories altogether. + % TODO: file expiration + {_, Stats1} = collect_fragments(Storage, Transfer, Stats), + {_, Stats2} = collect_tempfiles(Storage, Transfer, Stats1), + Stats2; +try_collect_transfer(Storage, Transfer, #{status := incomplete}, Stats) -> + % File transfer is still incomplete. + % Any outdated fragments and temporary files should be collectable. As a kind of + % heuristic we only delete transfer directory itself only if it is also outdated + % _and was empty at the start of GC_, as a precaution against races between + % writers and GCs. + TTL = get_segments_ttl(Storage, Transfer), + Cutoff = erlang:system_time(second) + TTL, + {FragCleaned, Stats1} = collect_outdated_fragments(Storage, Transfer, Cutoff, Stats), + {TempCleaned, Stats2} = collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats1), + % TODO: collect empty directories separately + case FragCleaned and TempCleaned of + true -> + collect_transfer_directory(Storage, Transfer, Stats2); + false -> + Stats2 + end. + +collect_fragments(Storage, Transfer, Stats) -> + Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, fragment), + collect_filepath(Dirname, true, Stats). + +collect_tempfiles(Storage, Transfer, Stats) -> + Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, temporary), + collect_filepath(Dirname, true, Stats). + +collect_outdated_fragments(Storage, Transfer, Cutoff, Stats) -> + Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, fragment), + Filter = fun(_Filepath, #file_info{mtime = ModifiedAt}) -> ModifiedAt < Cutoff end, + collect_filepath(Dirname, Filter, Stats). + +collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats) -> + Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, temporary), + Filter = fun(_Filepath, #file_info{mtime = ModifiedAt}) -> ModifiedAt < Cutoff end, + collect_filepath(Dirname, Filter, Stats). + +collect_transfer_directory(Storage, Transfer, Stats) -> + Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer), + StatsNext = collect_empty_directory(Dirname, Stats), + collect_parents(Dirname, StatsNext). + +collect_parents(Dirname, Stats) -> + Parent = filename:dirname(Dirname), + case file:del_dir(Parent) of + ok -> + collect_parents(Parent, account_gcstat_directory(Stats)); + {error, enoent} -> + collect_parents(Parent, Stats); + {error, eexist} -> + Stats; + {error, Reason} -> + register_gcstat_error({directory, Parent}, Reason, Stats) + end. + +% collect_outdated_fragment(#{path := Filepath, fileinfo := Fileinfo}, Cutoff, Stats) -> +% case Fileinfo#file_info.mtime of +% ModifiedAt when ModifiedAt < Cutoff -> +% collect_filepath(Filepath, Fileinfo, Stats); +% _ -> +% Stats +% end. + +-spec collect_filepath(file:name(), Filter, gcstats()) -> {boolean(), gcstats()} when + Filter :: boolean() | fun((file:name(), file:file_info()) -> boolean()). +collect_filepath(Filepath, Filter, Stats) -> + case file:read_file_info(Filepath) of + {ok, Fileinfo} -> + collect_filepath(Filepath, Fileinfo, Filter, Stats); + {error, enoent} -> + {true, Stats}; + {error, Reason} -> + {false, register_gcstat_error({path, Filepath}, Reason, Stats)} + end. + +collect_filepath(Filepath, #file_info{type = directory} = Fileinfo, Filter, Stats) -> + collect_directory(Filepath, Fileinfo, Filter, Stats); +collect_filepath(Filepath, #file_info{type = regular} = Fileinfo, Filter, Stats) -> + case filter_filepath(Filter, Filepath, Fileinfo) andalso file:delete(Filepath, [raw]) of + false -> + {false, Stats}; + ok -> + {true, account_gcstat(Fileinfo, Stats)}; + {error, enoent} -> + {true, Stats}; + {error, Reason} -> + {false, register_gcstat_error({file, Filepath}, Reason, Stats)} + end; +collect_filepath(Filepath, Fileinfo, _Filter, Stats) -> + {false, register_gcstat_error({file, Filepath}, {unexpected, Fileinfo}, Stats)}. + +collect_directory(Dirpath, Fileinfo, Filter, Stats) -> + case file:list_dir(Dirpath) of + {ok, Filenames} -> + {Clean, StatsNext} = collect_files(Dirpath, Filenames, Filter, Stats), + case Clean andalso filter_filepath(Filter, Dirpath, Fileinfo) of + true -> + {true, collect_empty_directory(Dirpath, StatsNext)}; + _ -> + {false, StatsNext} + end; + {error, Reason} -> + {false, register_gcstat_error({directory, Dirpath}, Reason, Stats)} + end. + +collect_files(Dirname, Filenames, Filter, Stats) -> + lists:foldl( + fun(Filename, {Complete, StatsAcc}) -> + Filepath = filename:join(Dirname, Filename), + {Collected, StatsNext} = collect_filepath(Filepath, Filter, StatsAcc), + {Collected andalso Complete, StatsNext} + end, + {true, Stats}, + Filenames + ). + +collect_empty_directory(Dirpath, Stats) -> + case file:del_dir(Dirpath) of + ok -> + account_gcstat_directory(Stats); + {error, enoent} -> + Stats; + {error, Reason} -> + register_gcstat_error({directory, Dirpath}, Reason, Stats) + end. + +filter_filepath(Filter, _, _) when is_boolean(Filter) -> + Filter; +filter_filepath(Filter, Filepath, Fileinfo) when is_function(Filter) -> + Filter(Filepath, Fileinfo). + +get_segments_ttl(Storage, Transfer) -> + {MinTTL, MaxTTL} = emqx_ft_conf:segments_ttl(Storage), + clamp(MinTTL, MaxTTL, try_get_filemeta_ttl(Storage, Transfer)). + +try_get_filemeta_ttl(Storage, Transfer) -> + case emqx_ft_storage_fs:read_filemeta(Storage, Transfer) of + {ok, Filemeta} -> + maps:get(segments_ttl, Filemeta, undefined); + {error, _} -> + undefined + end. + +clamp(Min, Max, V) -> + min(Max, max(Min, V)). + +% try_collect(_Subject, ok = Result, Then, _Stats) -> +% Then(Result); +% try_collect(_Subject, {ok, Result}, Then, _Stats) -> +% Then(Result); +% try_collect(Subject, {error, _} = Error, _Then, Stats) -> +% register_gcstat_error(Subject, Error, Stats). + +%% + +init_gcstats() -> + #gcstats{started_at = erlang:system_time()}. + +finish_gcstats(Stats) -> + Stats#gcstats{finished_at = erlang:system_time()}. + +account_gcstat(Fileinfo, Stats = #gcstats{files = Files, space = Space}) -> + Stats#gcstats{ + files = Files + 1, + space = Space + Fileinfo#file_info.size + }. + +account_gcstat_directory(Stats = #gcstats{directories = Directories}) -> + Stats#gcstats{ + directories = Directories + 1 + }. + +register_gcstat_error(Subject, Error, Stats = #gcstats{errors = Errors}) -> + Stats#gcstats{errors = Errors#{Subject => Error}}. + +%% + +get_storage_root(Storage) -> + maps:get(root, Storage, filename:join(emqx:data_dir(), "file_transfer")). diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index 8d388814c..3c28eae30 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -61,5 +61,5 @@ init([]) -> modules => [emqx_ft_responder_sup] }, - ChildSpecs = [Responder, AssemblerSup, FileReaderSup], + ChildSpecs = [Responder, AssemblerSup, FileReaderSup | emqx_ft_storage:child_spec()], {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl new file mode 100644 index 000000000..c7fffbacd --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -0,0 +1,205 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_gc_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx_ft/include/emqx_ft_storage_fs.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("snabbkaffe/include/test_macros.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + _ = application:load(emqx_ft), + ok = emqx_common_test_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([]), + ok. + +init_per_testcase(TC, Config) -> + _ = application:unset_env(emqx_ft, gc_interval), + _ = application:unset_env(emqx_ft, min_segments_ttl), + _ = application:unset_env(emqx_ft, max_segments_ttl), + ok = emqx_common_test_helpers:start_app( + emqx_ft, + fun(emqx_ft) -> + ok = emqx_config:put([file_transfer, storage], #{ + type => local, + root => mk_root(TC, Config) + }) + end + ), + Config. + +end_per_testcase(_TC, _Config) -> + ok = application:stop(emqx_ft), + ok. + +mk_root(TC, Config) -> + filename:join([?config(priv_dir, Config), <<"file_transfer">>, TC, atom_to_binary(node())]). + +%% + +t_gc_triggers_periodically(_Config) -> + Interval = 1000, + ok = application:set_env(emqx_ft, gc_interval, Interval), + ok = emqx_ft_storage_fs_gc:reset(emqx_ft_conf:storage()), + ?check_trace( + timer:sleep(Interval * 3), + fun(Trace) -> + [Event, _ | _] = ?of_kind(garbage_collection, Trace), + ?assertMatch( + #{ + stats := #gcstats{ + files = 0, + directories = 0, + space = 0, + errors = #{} = Errors + } + } when map_size(Errors) == 0, + Event + ) + end + ). + +t_gc_triggers_manually(_Config) -> + ?check_trace( + ?assertMatch( + #gcstats{files = 0, directories = 0, space = 0, errors = #{} = Errors} when + map_size(Errors) == 0, + emqx_ft_storage_fs_gc:collect(emqx_ft_conf:storage()) + ), + fun(Trace) -> + [Event] = ?of_kind(garbage_collection, Trace), + ?assertMatch( + #{stats := #gcstats{}}, + Event + ) + end + ). + +t_gc_complete_transfers(_Config) -> + Storage = emqx_ft_conf:storage(), + Transfers = [ + { + T1 = {<<"client1">>, mk_file_id()}, + "cat.cur", + emqx_ft_content_gen:new({?LINE, S1 = 42}, SS1 = 16) + }, + { + T2 = {<<"client2">>, mk_file_id()}, + "cat.ico", + emqx_ft_content_gen:new({?LINE, S2 = 420}, SS2 = 64) + }, + { + T3 = {<<"client42">>, mk_file_id()}, + "cat.jpg", + emqx_ft_content_gen:new({?LINE, S3 = 42000}, SS3 = 1024) + } + ], + % 1. Start all transfers + TransferSizes = emqx_misc:pmap( + fun(Transfer) -> start_transfer(Storage, Transfer) end, + Transfers + ), + ?assertEqual([S1, S2, S3], TransferSizes), + ?assertMatch( + #gcstats{files = 0, directories = 0, errors = #{} = Es} when map_size(Es) == 0, + emqx_ft_storage_fs_gc:collect(Storage) + ), + % 2. Complete just the first transfer + ?assertEqual( + ok, + complete_transfer(Storage, T1, S1) + ), + GCFiles1 = ceil(S1 / SS1) + 1, + ?assertMatch( + #gcstats{ + files = GCFiles1, + directories = 2, + space = Space, + errors = #{} = Es + } when Space > S1 andalso map_size(Es) == 0, + emqx_ft_storage_fs_gc:collect(Storage) + ), + % 3. Complete rest of transfers + ?assertEqual( + [ok, ok], + emqx_misc:pmap( + fun({Transfer, Size}) -> complete_transfer(Storage, Transfer, Size) end, + [{T2, S2}, {T3, S3}] + ) + ), + GCFiles2 = ceil(S2 / SS2) + 1, + GCFiles3 = ceil(S3 / SS3) + 1, + ?assertMatch( + #gcstats{ + files = Files, + directories = 4, + space = Space, + errors = #{} = Es + } when + Files == (GCFiles2 + GCFiles3) andalso + Space > (S2 + S3) andalso + map_size(Es) == 0, + emqx_ft_storage_fs_gc:collect(Storage) + ). + +start_transfer(Storage, {Transfer, Name, Gen}) -> + Meta = #{ + name => Name, + segments_ttl => 10 + }, + ?assertEqual( + ok, + emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta) + ), + emqx_ft_content_gen:fold( + fun({Content, SegmentNum, #{chunk_size := SegmentSize}}, _Transferred) -> + Offset = (SegmentNum - 1) * SegmentSize, + ?assertEqual( + ok, + emqx_ft_storage_fs:store_segment(Storage, Transfer, {Offset, Content}) + ), + Offset + byte_size(Content) + end, + 0, + Gen + ). + +complete_transfer(Storage, Transfer, Size) -> + complete_transfer(Storage, Transfer, Size, 100). + +complete_transfer(Storage, Transfer, Size, Timeout) -> + {async, Pid} = emqx_ft_storage_fs:assemble(Storage, Transfer, Size), + MRef = erlang:monitor(process, Pid), + Pid ! kickoff, + receive + {'DOWN', MRef, process, Pid, {shutdown, Result}} -> + Result + after Timeout -> + ct:fail("Assembler did not finish in time") + end. + +mk_file_id() -> + emqx_guid:to_hexstr(emqx_guid:gen()). From 15dc7c3e845febb9506d093c43dc1bb618a0e9d8 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 30 Mar 2023 14:05:35 +0300 Subject: [PATCH 061/156] fix(test): fix node helper usage --- apps/emqx/test/emqx_common_test_helpers.erl | 3 +++ apps/emqx_ft/test/emqx_ft_test_helpers.erl | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 16920d60b..e01f18a51 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -244,6 +244,9 @@ do_render_app_config(App, Schema, ConfigFile, Opts) -> copy_certs(App, RenderedConfigFile), ok. +start_app(App, SpecAppConfig) -> + start_app(App, SpecAppConfig, #{}). + start_app(App, SpecAppConfig, Opts) -> render_and_load_app_config(App, Opts), SpecAppConfig(App), diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index 956e63553..f054be762 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -42,7 +42,7 @@ start_additional_node(Config, Name) -> stop_additional_node(Node) -> ok = rpc:call(Node, ekka, leave, []), ok = rpc:call(Node, emqx_common_test_helpers, stop_apps, [[emqx_ft]]), - {ok, _} = emqx_common_test_helpers:stop_slave(Node), + ok = emqx_common_test_helpers:stop_slave(Node), ok. tcp_port(Node) -> From 344799f100e822440aace128417bcec02fea5843 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 3 Mar 2023 18:00:31 +0300 Subject: [PATCH 062/156] fix(fs-gc): add testcase on incomplete transfers GC And fix logic that made GC delete incomplete segments prematurely. --- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 6 +- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 104 +++++++++++++++--- 2 files changed, 93 insertions(+), 17 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 6d493337a..4b2d2bfd6 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -164,7 +164,7 @@ try_collect_transfer(Storage, Transfer, #{status := incomplete}, Stats) -> % _and was empty at the start of GC_, as a precaution against races between % writers and GCs. TTL = get_segments_ttl(Storage, Transfer), - Cutoff = erlang:system_time(second) + TTL, + Cutoff = erlang:system_time(second) - TTL, {FragCleaned, Stats1} = collect_outdated_fragments(Storage, Transfer, Cutoff, Stats), {TempCleaned, Stats2} = collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats1), % TODO: collect empty directories separately @@ -222,7 +222,7 @@ collect_parents(Dirname, Stats) -> -spec collect_filepath(file:name(), Filter, gcstats()) -> {boolean(), gcstats()} when Filter :: boolean() | fun((file:name(), file:file_info()) -> boolean()). collect_filepath(Filepath, Filter, Stats) -> - case file:read_file_info(Filepath) of + case file:read_file_info(Filepath, [{time, posix}]) of {ok, Fileinfo} -> collect_filepath(Filepath, Fileinfo, Filter, Stats); {error, enoent} -> @@ -238,6 +238,7 @@ collect_filepath(Filepath, #file_info{type = regular} = Fileinfo, Filter, Stats) false -> {false, Stats}; ok -> + ?tp(garbage_collected_file, #{path => Filepath}), {true, account_gcstat(Fileinfo, Stats)}; {error, enoent} -> {true, Stats}; @@ -275,6 +276,7 @@ collect_files(Dirname, Filenames, Filter, Stats) -> collect_empty_directory(Dirpath, Stats) -> case file:del_dir(Dirpath) of ok -> + ?tp(garbage_collected_directory, #{path => Dirpath}), account_gcstat_directory(Stats); {error, enoent} -> Stats; diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index c7fffbacd..858cdfd90 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -60,8 +60,10 @@ mk_root(TC, Config) -> %% +-define(NSEGS(Filesize, SegmentSize), (ceil(Filesize / SegmentSize) + 1)). + t_gc_triggers_periodically(_Config) -> - Interval = 1000, + Interval = 500, ok = application:set_env(emqx_ft, gc_interval, Interval), ok = emqx_ft_storage_fs_gc:reset(emqx_ft_conf:storage()), ?check_trace( @@ -103,17 +105,17 @@ t_gc_complete_transfers(_Config) -> Transfers = [ { T1 = {<<"client1">>, mk_file_id()}, - "cat.cur", + #{name => "cat.cur", segments_ttl => 10}, emqx_ft_content_gen:new({?LINE, S1 = 42}, SS1 = 16) }, { T2 = {<<"client2">>, mk_file_id()}, - "cat.ico", + #{name => "cat.ico", segments_ttl => 10}, emqx_ft_content_gen:new({?LINE, S2 = 420}, SS2 = 64) }, { T3 = {<<"client42">>, mk_file_id()}, - "cat.jpg", + #{name => "cat.jpg", segments_ttl => 10}, emqx_ft_content_gen:new({?LINE, S3 = 42000}, SS3 = 1024) } ], @@ -132,14 +134,13 @@ t_gc_complete_transfers(_Config) -> ok, complete_transfer(Storage, T1, S1) ), - GCFiles1 = ceil(S1 / SS1) + 1, ?assertMatch( #gcstats{ - files = GCFiles1, + files = Files, directories = 2, space = Space, errors = #{} = Es - } when Space > S1 andalso map_size(Es) == 0, + } when Files == ?NSEGS(S1, SS1) andalso Space > S1 andalso map_size(Es) == 0, emqx_ft_storage_fs_gc:collect(Storage) ), % 3. Complete rest of transfers @@ -150,8 +151,6 @@ t_gc_complete_transfers(_Config) -> [{T2, S2}, {T3, S3}] ) ), - GCFiles2 = ceil(S2 / SS2) + 1, - GCFiles3 = ceil(S3 / SS3) + 1, ?assertMatch( #gcstats{ files = Files, @@ -159,17 +158,92 @@ t_gc_complete_transfers(_Config) -> space = Space, errors = #{} = Es } when - Files == (GCFiles2 + GCFiles3) andalso + Files == (?NSEGS(S2, SS2) + ?NSEGS(S3, SS3)) andalso Space > (S2 + S3) andalso map_size(Es) == 0, emqx_ft_storage_fs_gc:collect(Storage) ). -start_transfer(Storage, {Transfer, Name, Gen}) -> - Meta = #{ - name => Name, - segments_ttl => 10 - }, +t_gc_incomplete_transfers(_Config) -> + _ = application:set_env(emqx_ft, min_segments_ttl, 0), + _ = application:set_env(emqx_ft, max_segments_ttl, 4), + ok = emqx_ft_storage_fs_gc:reset(emqx_ft_conf:storage()), + Storage = emqx_ft_conf:storage(), + Transfers = [ + { + {<<"client43"/utf8>>, <<"file-🦕"/utf8>>}, + #{name => "dog.cur", segments_ttl => 1}, + emqx_ft_content_gen:new({?LINE, S1 = 123}, SS1 = 32) + }, + { + {<<"client44">>, <<"file-🦖"/utf8>>}, + #{name => "dog.ico", segments_ttl => 2}, + emqx_ft_content_gen:new({?LINE, S2 = 456}, SS2 = 64) + }, + { + {<<"client1337">>, <<"file-🦀"/utf8>>}, + #{name => "dog.jpg", segments_ttl => 3000}, + emqx_ft_content_gen:new({?LINE, S3 = 7890}, SS3 = 128) + }, + { + {<<"client31337">>, <<"file-⏳"/utf8>>}, + #{name => "dog.jpg"}, + emqx_ft_content_gen:new({?LINE, S4 = 1230}, SS4 = 256) + } + ], + % 1. Start transfers, send all the segments but don't trigger completion. + _ = emqx_misc:pmap(fun(Transfer) -> start_transfer(Storage, Transfer) end, Transfers), + ?check_trace( + begin + % 2. Enable periodic GC every 0.5 seconds. + ok = application:set_env(emqx_ft, gc_interval, 500), + ok = emqx_ft_storage_fs_gc:reset(Storage), + % 3. First we need the first transfer to be collected. + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = Files, + directories = 4, + space = Space + } + } when Files == (?NSEGS(S1, SS1)) andalso Space > S1, + 5000, + 0 + ), + % 4. Then the second one. + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = Files, + directories = 4, + space = Space + } + } when Files == (?NSEGS(S2, SS2)) andalso Space > S2, + 5000, + 0 + ), + % 5. Then transfers 3 and 4 because 3rd has too big TTL and 4th has no specific TTL. + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = Files, + directories = 4 * 2, + space = Space + } + } when Files == (?NSEGS(S3, SS3) + ?NSEGS(S4, SS4)) andalso Space > S3 + S4, + 5000, + 0 + ) + end, + [] + ). + +%% + +start_transfer(Storage, {Transfer, Meta, Gen}) -> ?assertEqual( ok, emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta) From bcd2099ce1c194f2120c9bc28eb3787a13d07be7 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 3 Mar 2023 18:02:19 +0300 Subject: [PATCH 063/156] fix(fs-gc): make deletion empty transfer directories safer --- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 23 +++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 4b2d2bfd6..75a74681f 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -196,15 +196,18 @@ collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats) -> collect_transfer_directory(Storage, Transfer, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer), StatsNext = collect_empty_directory(Dirname, Stats), - collect_parents(Dirname, StatsNext). + collect_parents(Dirname, get_storage_root(Storage), StatsNext). -collect_parents(Dirname, Stats) -> +collect_parents(Dirname, Until, Stats) -> Parent = filename:dirname(Dirname), - case file:del_dir(Parent) of + case is_same_filepath(Parent, Until) orelse file:del_dir(Parent) of + true -> + Stats; ok -> - collect_parents(Parent, account_gcstat_directory(Stats)); + ?tp(garbage_collected_directory, #{path => Dirname}), + collect_parents(Parent, Until, account_gcstat_directory(Stats)); {error, enoent} -> - collect_parents(Parent, Stats); + collect_parents(Parent, Until, Stats); {error, eexist} -> Stats; {error, Reason} -> @@ -289,6 +292,16 @@ filter_filepath(Filter, _, _) when is_boolean(Filter) -> filter_filepath(Filter, Filepath, Fileinfo) when is_function(Filter) -> Filter(Filepath, Fileinfo). +is_same_filepath(P1, P2) when is_binary(P1) andalso is_binary(P2) -> + filename:absname(P1) == filename:absname(P2); +is_same_filepath(P1, P2) when is_list(P1) andalso is_list(P2) -> + filename:absname(P1) == filename:absname(P2); +is_same_filepath(P1, P2) when is_binary(P1) -> + is_same_filepath(P1, filepath_to_binary(P2)). + +filepath_to_binary(S) -> + unicode:characters_to_binary(S, unicode, file:native_name_encoding()). + get_segments_ttl(Storage, Transfer) -> {MinTTL, MaxTTL} = emqx_ft_conf:segments_ttl(Storage), clamp(MinTTL, MaxTTL, try_get_filemeta_ttl(Storage, Transfer)). From 50c6eef2bc6eb029ce0f7d13b4afea479246beac Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 3 Mar 2023 19:03:15 +0300 Subject: [PATCH 064/156] fix(fs-gc): do not hide `enoent`s Also use `file:read_link_info/2`, it actually fetches any file info while also not following symlinks automatically, which is better for GC usecases. --- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 30 +++++----- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 59 ++++++++++++++++++- 2 files changed, 72 insertions(+), 17 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 75a74681f..98b048108 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -177,21 +177,21 @@ try_collect_transfer(Storage, Transfer, #{status := incomplete}, Stats) -> collect_fragments(Storage, Transfer, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, fragment), - collect_filepath(Dirname, true, Stats). + maybe_collect_directory(Dirname, true, Stats). collect_tempfiles(Storage, Transfer, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, temporary), - collect_filepath(Dirname, true, Stats). + maybe_collect_directory(Dirname, true, Stats). collect_outdated_fragments(Storage, Transfer, Cutoff, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, fragment), Filter = fun(_Filepath, #file_info{mtime = ModifiedAt}) -> ModifiedAt < Cutoff end, - collect_filepath(Dirname, Filter, Stats). + maybe_collect_directory(Dirname, Filter, Stats). collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, temporary), Filter = fun(_Filepath, #file_info{mtime = ModifiedAt}) -> ModifiedAt < Cutoff end, - collect_filepath(Dirname, Filter, Stats). + maybe_collect_directory(Dirname, Filter, Stats). collect_transfer_directory(Storage, Transfer, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer), @@ -206,8 +206,6 @@ collect_parents(Dirname, Until, Stats) -> ok -> ?tp(garbage_collected_directory, #{path => Dirname}), collect_parents(Parent, Until, account_gcstat_directory(Stats)); - {error, enoent} -> - collect_parents(Parent, Until, Stats); {error, eexist} -> Stats; {error, Reason} -> @@ -222,16 +220,22 @@ collect_parents(Dirname, Until, Stats) -> % Stats % end. +maybe_collect_directory(Dirpath, Filter, Stats) -> + case filelib:is_dir(Dirpath) of + true -> + collect_filepath(Dirpath, Filter, Stats); + false -> + {true, Stats} + end. + -spec collect_filepath(file:name(), Filter, gcstats()) -> {boolean(), gcstats()} when Filter :: boolean() | fun((file:name(), file:file_info()) -> boolean()). collect_filepath(Filepath, Filter, Stats) -> - case file:read_file_info(Filepath, [{time, posix}]) of + case file:read_link_info(Filepath, [{time, posix}, raw]) of {ok, Fileinfo} -> collect_filepath(Filepath, Fileinfo, Filter, Stats); - {error, enoent} -> - {true, Stats}; {error, Reason} -> - {false, register_gcstat_error({path, Filepath}, Reason, Stats)} + {Reason == enoent, register_gcstat_error({path, Filepath}, Reason, Stats)} end. collect_filepath(Filepath, #file_info{type = directory} = Fileinfo, Filter, Stats) -> @@ -243,10 +247,8 @@ collect_filepath(Filepath, #file_info{type = regular} = Fileinfo, Filter, Stats) ok -> ?tp(garbage_collected_file, #{path => Filepath}), {true, account_gcstat(Fileinfo, Stats)}; - {error, enoent} -> - {true, Stats}; {error, Reason} -> - {false, register_gcstat_error({file, Filepath}, Reason, Stats)} + {Reason == enoent, register_gcstat_error({file, Filepath}, Reason, Stats)} end; collect_filepath(Filepath, Fileinfo, _Filter, Stats) -> {false, register_gcstat_error({file, Filepath}, {unexpected, Fileinfo}, Stats)}. @@ -281,8 +283,6 @@ collect_empty_directory(Dirpath, Stats) -> ok -> ?tp(garbage_collected_directory, #{path => Dirpath}), account_gcstat_directory(Stats); - {error, enoent} -> - Stats; {error, Reason} -> register_gcstat_error({directory, Dirpath}, Reason, Stats) end. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index 858cdfd90..a0b969edd 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -56,7 +56,7 @@ end_per_testcase(_TC, _Config) -> ok. mk_root(TC, Config) -> - filename:join([?config(priv_dir, Config), <<"file_transfer">>, TC, atom_to_binary(node())]). + filename:join([?config(priv_dir, Config), "file_transfer", TC, atom_to_list(node())]). %% @@ -167,7 +167,6 @@ t_gc_complete_transfers(_Config) -> t_gc_incomplete_transfers(_Config) -> _ = application:set_env(emqx_ft, min_segments_ttl, 0), _ = application:set_env(emqx_ft, max_segments_ttl, 4), - ok = emqx_ft_storage_fs_gc:reset(emqx_ft_conf:storage()), Storage = emqx_ft_conf:storage(), Transfers = [ { @@ -241,6 +240,62 @@ t_gc_incomplete_transfers(_Config) -> [] ). +t_gc_handling_errors(_Config) -> + _ = application:set_env(emqx_ft, min_segments_ttl, 0), + _ = application:set_env(emqx_ft, max_segments_ttl, 0), + Storage = emqx_ft_conf:storage(), + Transfer1 = {<<"client1">>, mk_file_id()}, + Transfer2 = {<<"client2">>, mk_file_id()}, + Filemeta = #{name => "oops.pdf"}, + Size = 420, + SegSize = 16, + _ = start_transfer( + Storage, + {Transfer1, Filemeta, emqx_ft_content_gen:new({?LINE, Size}, SegSize)} + ), + _ = start_transfer( + Storage, + {Transfer2, Filemeta, emqx_ft_content_gen:new({?LINE, Size}, SegSize)} + ), + % 1. Throw some chaos in the transfer directory. + DirFragment1 = emqx_ft_storage_fs:get_subdir(Storage, Transfer1, fragment), + DirTemporary1 = emqx_ft_storage_fs:get_subdir(Storage, Transfer1, temporary), + PathShadyLink = filename:join(DirTemporary1, "linked-here"), + ok = file:make_symlink(DirFragment1, PathShadyLink), + DirTransfer2 = emqx_ft_storage_fs:get_subdir(Storage, Transfer2), + PathTripUp = filename:join(DirTransfer2, "trip-up-here"), + ok = file:write_file(PathTripUp, <<"HAHA">>), + ok = timer:sleep(timer:seconds(1)), + % 2. Observe the errors are reported consistently. + ?check_trace( + ?assertMatch( + #gcstats{ + files = Files, + directories = 3, + space = Space, + errors = #{ + % NOTE: dangling symlink looks like `enoent` for some reason + {file, PathShadyLink} := {unexpected, _}, + {directory, DirTransfer2} := eexist + } + } when Files == ?NSEGS(Size, SegSize) * 2 andalso Space > Size * 2, + emqx_ft_storage_fs_gc:collect(Storage) + ), + fun(Trace) -> + ?assertMatch( + [ + #{ + errors := #{ + {file, PathShadyLink} := {unexpected, _}, + {directory, DirTransfer2} := eexist + } + } + ], + ?of_kind("garbage_collection_errors", Trace) + ) + end + ). + %% start_transfer(Storage, {Transfer, Meta, Gen}) -> From e1dc48fa2b4c5100f05f88b5ab9715d6ab8ff4d1 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 14 Mar 2023 19:10:47 +0300 Subject: [PATCH 065/156] feat(fs-gc): wire gc up with emqx config --- apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 35 +++++++++++++++++++ apps/emqx_ft/src/emqx_ft_conf.erl | 18 +++++++--- apps/emqx_ft/src/emqx_ft_schema.erl | 33 ++++++++++++++++- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 32 ++++++++++------- 4 files changed, 99 insertions(+), 19 deletions(-) diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf index 576d7d8fe..dd2d2a1dc 100644 --- a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -33,4 +33,39 @@ emqx_ft_schema { } } + local_storage_gc { + desc { + en: "Garbage collection settings for the intermediate and temporary files in the local file system." + zh: "" + } + label: { + en: "Local Storage GC" + zh: "" + } + } + + storage_gc_interval { + desc { + en: "Interval of periodic garbage collection." + zh: "" + } + label: { + en: "GC Interval" + zh: "" + } + } + + storage_gc_max_segments_ttl { + desc { + en: "Maximum TTL of a segment kept in the local file system.
" + "This is a hard limit: no segment will outlive this TTL, even if some file transfer specifies a " + "TTL more than that." + zh: "" + } + label: { + en: "GC Interval" + zh: "" + } + } + } diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 444462716..e925ee376 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -50,17 +50,25 @@ storage() -> -spec gc_interval(_Storage) -> milliseconds(). gc_interval(_Storage) -> - % TODO: config wiring - application:get_env(emqx_ft, gc_interval, timer:minutes(10)). + Conf = assert_storage(local), + emqx_map_lib:deep_get([gc, interval], Conf). -spec segments_ttl(_Storage) -> {_Min :: seconds(), _Max :: seconds()}. segments_ttl(_Storage) -> - % TODO: config wiring + Conf = assert_storage(local), { - application:get_env(emqx_ft, min_segments_ttl, 60), - application:get_env(emqx_ft, max_segments_ttl, 72 * 3600) + emqx_map_lib:deep_get([gc, minimum_segments_ttl], Conf), + emqx_map_lib:deep_get([gc, maximum_segments_ttl], Conf) }. +assert_storage(Type) -> + case storage() of + Conf = #{type := Type} -> + Conf; + Conf -> + error({inapplicable, Conf}) + end. + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index f17c957a9..d2b2b9299 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -65,13 +65,44 @@ fields(local_storage) -> type => binary(), desc => ?DESC("local_storage_root"), required => false + }}, + {gc, #{ + type => hoconsc:ref(?MODULE, local_storage_gc), + desc => ?DESC("local_storage_gc"), + required => false + }} + ]; +fields(local_storage_gc) -> + [ + {interval, #{ + type => emqx_schema:duration_ms(), + desc => ?DESC("storage_gc_interval"), + required => false, + default => "1h" + }}, + {maximum_segments_ttl, #{ + type => emqx_schema:duration_s(), + desc => ?DESC("storage_gc_max_segments_ttl"), + required => false, + default => "24h" + }}, + {minimum_segments_ttl, #{ + type => emqx_schema:duration_s(), + % desc => ?DESC("storage_gc_min_segments_ttl"), + required => false, + default => "5m", + % NOTE + % This setting does not seem to be useful to an end-user. + hidden => true }} ]. desc(file_transfer) -> "File transfer settings"; desc(local_storage) -> - "File transfer local storage settings". + "File transfer local storage settings"; +desc(local_storage_gc) -> + "Garbage collection settings for the File transfer local storage backend". schema(filemeta) -> #{ diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index a0b969edd..a83d915e6 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -37,16 +37,22 @@ end_per_suite(_Config) -> ok. init_per_testcase(TC, Config) -> - _ = application:unset_env(emqx_ft, gc_interval), - _ = application:unset_env(emqx_ft, min_segments_ttl), - _ = application:unset_env(emqx_ft, max_segments_ttl), ok = emqx_common_test_helpers:start_app( emqx_ft, fun(emqx_ft) -> - ok = emqx_config:put([file_transfer, storage], #{ - type => local, - root => mk_root(TC, Config) - }) + emqx_common_test_helpers:load_config( + emqx_ft_schema, + iolist_to_binary([ + "file_transfer {" + " storage = {" + " type = \"local\"," + " root = \"", + mk_root(TC, Config), + "\"" + " }" + "}" + ]) + ) end ), Config. @@ -64,7 +70,7 @@ mk_root(TC, Config) -> t_gc_triggers_periodically(_Config) -> Interval = 500, - ok = application:set_env(emqx_ft, gc_interval, Interval), + ok = emqx_config:put([file_transfer, storage, gc, interval], Interval), ok = emqx_ft_storage_fs_gc:reset(emqx_ft_conf:storage()), ?check_trace( timer:sleep(Interval * 3), @@ -165,8 +171,8 @@ t_gc_complete_transfers(_Config) -> ). t_gc_incomplete_transfers(_Config) -> - _ = application:set_env(emqx_ft, min_segments_ttl, 0), - _ = application:set_env(emqx_ft, max_segments_ttl, 4), + ok = emqx_config:put([file_transfer, storage, gc, minimum_segments_ttl], 0), + ok = emqx_config:put([file_transfer, storage, gc, maximum_segments_ttl], 4), Storage = emqx_ft_conf:storage(), Transfers = [ { @@ -195,7 +201,7 @@ t_gc_incomplete_transfers(_Config) -> ?check_trace( begin % 2. Enable periodic GC every 0.5 seconds. - ok = application:set_env(emqx_ft, gc_interval, 500), + ok = emqx_config:put([file_transfer, storage, gc, interval], 500), ok = emqx_ft_storage_fs_gc:reset(Storage), % 3. First we need the first transfer to be collected. {ok, _} = ?block_until( @@ -241,8 +247,8 @@ t_gc_incomplete_transfers(_Config) -> ). t_gc_handling_errors(_Config) -> - _ = application:set_env(emqx_ft, min_segments_ttl, 0), - _ = application:set_env(emqx_ft, max_segments_ttl, 0), + ok = emqx_config:put([file_transfer, storage, gc, minimum_segments_ttl], 0), + ok = emqx_config:put([file_transfer, storage, gc, maximum_segments_ttl], 0), Storage = emqx_ft_conf:storage(), Transfer1 = {<<"client1">>, mk_file_id()}, Transfer2 = {<<"client2">>, mk_file_id()}, From 0c821cd3bd323fa159b4c202bfca046e40607da5 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 15 Mar 2023 13:57:42 +0300 Subject: [PATCH 066/156] test(ft): inject configs through hocon subsystem So that relevant parts of config would be initialized with defaults. --- apps/emqx_ft/test/emqx_ft_SUITE.erl | 11 ++++------- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 5 ++--- apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 5 ++--- .../emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl | 16 +++------------- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 7 ++++--- 5 files changed, 15 insertions(+), 29 deletions(-) diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index bb67d3c56..9c9087732 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -58,9 +58,8 @@ end_per_suite(_Config) -> set_special_configs(Config) -> fun (emqx_ft) -> - ok = emqx_config:put([file_transfer, storage], #{ - type => local, root => emqx_ft_test_helpers:ft_root(Config, node()) - }); + Root = emqx_ft_test_helpers:ft_root(Config, node()), + emqx_ft_test_helpers:load_config(#{storage => #{type => local, root => Root}}); (_) -> ok end. @@ -109,10 +108,8 @@ mk_cluster_specs(Config) -> {conf, [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]]}, {env_handler, fun (emqx_ft) -> - ok = emqx_config:put([file_transfer, storage], #{ - type => local, - root => emqx_ft_test_helpers:ft_root(Config, node()) - }); + Root = emqx_ft_test_helpers:ft_root(Config, node()), + emqx_ft_test_helpers:load_config(#{storage => #{type => local, root => Root}}); (_) -> ok end} diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index 239edb267..7b191e229 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -41,9 +41,8 @@ end_per_suite(_Config) -> set_special_configs(Config) -> fun (emqx_ft) -> - ok = emqx_config:put([file_transfer, storage], #{ - type => local, root => emqx_ft_test_helpers:ft_root(Config, node()) - }); + Root = emqx_ft_test_helpers:ft_root(Config, node()), + emqx_ft_test_helpers:load_config(#{storage => #{type => local, root => Root}}); (_) -> ok end. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index 3bda8042c..5551cce27 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -48,9 +48,8 @@ end_per_suite(_Config) -> set_special_configs(Config) -> fun (emqx_ft) -> - ok = emqx_config:put([file_transfer, storage], #{ - type => local, root => emqx_ft_test_helpers:ft_root(Config, node()) - }); + Root = emqx_ft_test_helpers:ft_root(Config, node()), + emqx_ft_test_helpers:load_config(#{storage => #{type => local, root => Root}}); (_) -> ok end. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index a83d915e6..37ac7eedf 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -40,19 +40,9 @@ init_per_testcase(TC, Config) -> ok = emqx_common_test_helpers:start_app( emqx_ft, fun(emqx_ft) -> - emqx_common_test_helpers:load_config( - emqx_ft_schema, - iolist_to_binary([ - "file_transfer {" - " storage = {" - " type = \"local\"," - " root = \"", - mk_root(TC, Config), - "\"" - " }" - "}" - ]) - ) + emqx_ft_test_helpers:load_config(#{ + storage => #{type => local, root => mk_root(TC, Config)} + }) end ), Config. diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index f054be762..b756f8034 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -30,9 +30,7 @@ start_additional_node(Config, Name) -> {configure_gen_rpc, true}, {env_handler, fun (emqx_ft) -> - ok = emqx_config:put([file_transfer, storage], #{ - type => local, root => ft_root(Config, node()) - }); + load_config(#{storage => #{type => local, root => ft_root(Config, node())}}); (_) -> ok end} @@ -45,6 +43,9 @@ stop_additional_node(Node) -> ok = emqx_common_test_helpers:stop_slave(Node), ok. +load_config(Config) -> + emqx_common_test_helpers:load_config(emqx_ft_schema, #{file_transfer => Config}). + tcp_port(Node) -> {_, Port} = rpc:call(Node, emqx_config, get, [[listeners, tcp, default, bind]]), Port. From 0d395460809ce5c7cdc3b7a400124097201d0b5c Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Mar 2023 14:24:52 +0300 Subject: [PATCH 067/156] feat(wdgraph): add `fold/3` which folds over graph edges --- apps/emqx/src/emqx_wdgraph.erl | 24 +++++++++++++++++++++++- apps/emqx/test/emqx_wdgraph_tests.erl | 20 ++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_wdgraph.erl b/apps/emqx/src/emqx_wdgraph.erl index 3361c52d1..bd7f58e7c 100644 --- a/apps/emqx/src/emqx_wdgraph.erl +++ b/apps/emqx/src/emqx_wdgraph.erl @@ -27,6 +27,8 @@ -export([find_edge/3]). -export([get_edges/2]). +-export([fold/3]). + -export([find_shortest_path/3]). -export_type([t/0]). @@ -38,7 +40,7 @@ -type label() :: term(). -opaque t() :: t(gnode(), label()). --opaque t(Node, Label) :: gb_trees:tree({Node}, {Node, weight(), Label}). +-opaque t(Node, Label) :: gb_trees:tree({Node}, [{Node, weight(), Label}]). %% @@ -72,6 +74,26 @@ find_edge(From, To, G) -> get_edges(Node, G) -> tree_lookup({Node}, G, []). +-spec fold(FoldFun, Acc, t(Node, Label)) -> Acc when + FoldFun :: fun((Node, _Edge :: {Node, weight(), Label}, Acc) -> Acc). +fold(FoldFun, Acc, G) -> + fold_iterator(FoldFun, Acc, gb_trees:iterator(G)). + +fold_iterator(FoldFun, AccIn, It) -> + case gb_trees:next(It) of + {{Node}, Edges = [_ | _], ItNext} -> + AccNext = lists:foldl( + fun(Edge = {_To, _Weight, _Label}, Acc) -> + FoldFun(Node, Edge, Acc) + end, + AccIn, + Edges + ), + fold_iterator(FoldFun, AccNext, ItNext); + none -> + AccIn + end. + % Find the shortest path between two nodes, if any. If the path exists, return list % of edge labels along that path. % This is a Dijkstra shortest path algorithm. It is one-way right now, for diff --git a/apps/emqx/test/emqx_wdgraph_tests.erl b/apps/emqx/test/emqx_wdgraph_tests.erl index ece87b966..c159a0f49 100644 --- a/apps/emqx/test/emqx_wdgraph_tests.erl +++ b/apps/emqx/test/emqx_wdgraph_tests.erl @@ -40,6 +40,26 @@ edges_nodes_test_() -> ?_assertEqual([{baz, 1, "cheapest"}, {foo, 0, "free"}], emqx_wdgraph:get_edges(bar, G5)) ]. +fold_test_() -> + G1 = emqx_wdgraph:new(), + G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1), + G3 = emqx_wdgraph:insert_edge(bar, baz, 1, "cheapest", G2), + G4 = emqx_wdgraph:insert_edge(bar, foo, 0, "free", G3), + G5 = emqx_wdgraph:insert_edge(foo, bar, 100, "luxury", G4), + [ + ?_assertEqual( + % 100 + 0 + 1 + 101, + emqx_wdgraph:fold(fun(_From, {_, Weight, _}, Acc) -> Weight + Acc end, 0, G5) + ), + ?_assertEqual( + [bar, baz, foo], + lists:usort( + emqx_wdgraph:fold(fun(From, {To, _, _}, Acc) -> [From, To | Acc] end, [], G5) + ) + ) + ]. + nonexistent_nodes_path_test_() -> G1 = emqx_wdgraph:new(), G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1), From 4f2600b9f1e31c167442bbf0f07e8649742ec3a8 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Mar 2023 15:15:43 +0300 Subject: [PATCH 068/156] feat(ft-gc): treat all transfer as incomplete Since the concept of _complete transfers_ is being split out into the _export_ concept, we lose knowledge of completeness in the GC. Instead of asking exporters for transfer statuses we just treat all transfer as incomplete when GCing. --- apps/emqx_ft/src/emqx_ft_assembler.erl | 12 +- apps/emqx_ft/src/emqx_ft_assembly.erl | 19 +++ apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 48 +++--- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 141 +++++++++--------- 4 files changed, 121 insertions(+), 99 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 38ccf13ac..ff845fee9 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -137,20 +137,10 @@ pread(Node, Segment, St) -> %% maybe_garbage_collect(ok, St = #st{storage = Storage, transfer = Transfer}) -> - Nodes = get_coverage_nodes(St), + Nodes = emqx_ft_assembly:nodes(St#st.assembly), emqx_ft_storage_fs_gc:collect(Storage, Transfer, Nodes); maybe_garbage_collect({error, _}, _St) -> ok. -get_coverage_nodes(St) -> - Coverage = emqx_ft_assembly:coverage(St#st.assembly), - ordsets:to_list( - lists:foldl( - fun({Node, _Segment}, Acc) -> ordsets:add_element(Node, Acc) end, - ordsets:new(), - Coverage - ) - ). - segsize(#{fragment := {segment, Info}}) -> maps:get(size, Info). diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index bea320bbf..d0998d6ec 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -22,6 +22,7 @@ -export([status/1]). -export([filemeta/1]). +-export([nodes/1]). -export([coverage/1]). -export([properties/1]). @@ -108,6 +109,24 @@ filemeta(Asm) -> coverage(#asm{coverage = Coverage}) -> Coverage. +-spec nodes(t()) -> [node()]. +nodes(#asm{meta = Meta, segs = Segs}) -> + S1 = orddict:fold( + fun(_Meta, {Node, _Fragment}, Acc) -> + ordsets:add_element(Node, Acc) + end, + ordsets:new(), + Meta + ), + S2 = emqx_wdgraph:fold( + fun(_Offset, {_End, _, {Node, _Fragment}}, Acc) -> + ordsets:add_element(Node, Acc) + end, + ordsets:new(), + Segs + ), + ordsets:to_list(ordsets:union(S1, S2)). + properties(#asm{properties = Properties}) -> Properties. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 98b048108..58c5dbfdf 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -80,11 +80,15 @@ handle_call(Call, From, St) -> ?SLOG(error, #{msg => "unexpected_call", call => Call, from => From}), {noreply, St}. -% TODO -% handle_cast({collect, Transfer, [Node | Rest]}, St) -> -% ok = do_collect_transfer(Transfer, Node, St), -% ok = collect(self(), Transfer, Rest), -% {noreply, St}; +handle_cast({collect, Transfer, [Node | Rest]}, St) -> + ok = do_collect_transfer(Transfer, Node, St), + case Rest of + [_ | _] -> + gen_server:cast(self(), {collect, Transfer, Rest}); + [] -> + ok + end, + {noreply, St}; handle_cast(reset, St) -> {noreply, reset_timer(St)}; handle_cast(Cast, St) -> @@ -95,10 +99,13 @@ handle_info({timeout, TRef, collect}, St = #st{next_gc_timer = TRef}) -> StNext = do_collect_garbage(St), {noreply, start_timer(StNext#st{next_gc_timer = undefined})}. -% do_collect_transfer(Transfer, Node, St = #st{storage = Storage}) when Node == node() -> -% Stats = try_collect_transfer(Storage, Transfer, complete, init_gcstats()), -% ok = maybe_report(Stats, St), -% ok. +do_collect_transfer(Transfer, Node, St = #st{storage = Storage}) when Node == node() -> + Stats = try_collect_transfer(Storage, Transfer, complete, init_gcstats()), + ok = maybe_report(Stats, St), + ok; +do_collect_transfer(_Transfer, _Node, _St = #st{}) -> + % TODO + ok. maybe_collect_garbage(_CalledAt, St = #st{last_gc = undefined}) -> do_collect_garbage(St); @@ -149,21 +156,13 @@ collect_garbage(Storage, Transfers, Stats) -> ) ). -try_collect_transfer(Storage, Transfer, #{status := complete}, Stats) -> - % File transfer is complete. - % We should be good to delete fragments and temporary files with their respective - % directories altogether. - % TODO: file expiration - {_, Stats1} = collect_fragments(Storage, Transfer, Stats), - {_, Stats2} = collect_tempfiles(Storage, Transfer, Stats1), - Stats2; -try_collect_transfer(Storage, Transfer, #{status := incomplete}, Stats) -> - % File transfer is still incomplete. +try_collect_transfer(Storage, Transfer, TransferInfo = #{}, Stats) -> + % File transfer might still be incomplete. % Any outdated fragments and temporary files should be collectable. As a kind of % heuristic we only delete transfer directory itself only if it is also outdated % _and was empty at the start of GC_, as a precaution against races between % writers and GCs. - TTL = get_segments_ttl(Storage, Transfer), + TTL = get_segments_ttl(Storage, TransferInfo), Cutoff = erlang:system_time(second) - TTL, {FragCleaned, Stats1} = collect_outdated_fragments(Storage, Transfer, Cutoff, Stats), {TempCleaned, Stats2} = collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats1), @@ -173,7 +172,14 @@ try_collect_transfer(Storage, Transfer, #{status := incomplete}, Stats) -> collect_transfer_directory(Storage, Transfer, Stats2); false -> Stats2 - end. + end; +try_collect_transfer(Storage, Transfer, complete, Stats) -> + % File transfer is complete. + % We should be good to delete fragments and temporary files with their respective + % directories altogether. + {_, Stats1} = collect_fragments(Storage, Transfer, Stats), + {_, Stats2} = collect_tempfiles(Storage, Transfer, Stats1), + Stats2. collect_fragments(Storage, Transfer, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, fragment), diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index 37ac7eedf..89e9eb970 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -45,9 +45,11 @@ init_per_testcase(TC, Config) -> }) end ), + ok = snabbkaffe:start_trace(), Config. end_per_testcase(_TC, _Config) -> + ok = snabbkaffe:stop(), ok = application:stop(emqx_ft), ok. @@ -126,20 +128,29 @@ t_gc_complete_transfers(_Config) -> emqx_ft_storage_fs_gc:collect(Storage) ), % 2. Complete just the first transfer - ?assertEqual( - ok, - complete_transfer(Storage, T1, S1) + {ok, {ok, Event}} = ?wait_async_action( + ?assertEqual(ok, complete_transfer(Storage, T1, S1)), + #{?snk_kind := garbage_collection}, + 1000 ), ?assertMatch( - #gcstats{ - files = Files, - directories = 2, - space = Space, - errors = #{} = Es + #{ + stats := #gcstats{ + files = Files, + directories = 2, + space = Space, + errors = #{} = Es + } } when Files == ?NSEGS(S1, SS1) andalso Space > S1 andalso map_size(Es) == 0, - emqx_ft_storage_fs_gc:collect(Storage) + Event ), % 3. Complete rest of transfers + {ok, Sub} = snabbkaffe_collector:subscribe( + ?match_event(#{?snk_kind := garbage_collection}), + 2, + 1000, + 0 + ), ?assertEqual( [ok, ok], emqx_misc:pmap( @@ -147,18 +158,19 @@ t_gc_complete_transfers(_Config) -> [{T2, S2}, {T3, S3}] ) ), - ?assertMatch( - #gcstats{ - files = Files, - directories = 4, - space = Space, - errors = #{} = Es - } when - Files == (?NSEGS(S2, SS2) + ?NSEGS(S3, SS3)) andalso - Space > (S2 + S3) andalso - map_size(Es) == 0, - emqx_ft_storage_fs_gc:collect(Storage) - ). + {ok, Events} = snabbkaffe_collector:receive_events(Sub), + CFiles = lists:sum([Stats#gcstats.files || #{stats := Stats} <- Events]), + CDirectories = lists:sum([Stats#gcstats.directories || #{stats := Stats} <- Events]), + CSpace = lists:sum([Stats#gcstats.space || #{stats := Stats} <- Events]), + CErrors = lists:foldl( + fun maps:merge/2, + #{}, + [Stats#gcstats.errors || #{stats := Stats} <- Events] + ), + ?assertEqual(?NSEGS(S2, SS2) + ?NSEGS(S3, SS3), CFiles), + ?assertEqual(2 + 2, CDirectories), + ?assertMatch(Space when Space > S2 + S3, CSpace), + ?assertMatch(Errors when map_size(Errors) == 0, CErrors). t_gc_incomplete_transfers(_Config) -> ok = emqx_config:put([file_transfer, storage, gc, minimum_segments_ttl], 0), @@ -188,52 +200,47 @@ t_gc_incomplete_transfers(_Config) -> ], % 1. Start transfers, send all the segments but don't trigger completion. _ = emqx_misc:pmap(fun(Transfer) -> start_transfer(Storage, Transfer) end, Transfers), - ?check_trace( - begin - % 2. Enable periodic GC every 0.5 seconds. - ok = emqx_config:put([file_transfer, storage, gc, interval], 500), - ok = emqx_ft_storage_fs_gc:reset(Storage), - % 3. First we need the first transfer to be collected. - {ok, _} = ?block_until( - #{ - ?snk_kind := garbage_collection, - stats := #gcstats{ - files = Files, - directories = 4, - space = Space - } - } when Files == (?NSEGS(S1, SS1)) andalso Space > S1, - 5000, - 0 - ), - % 4. Then the second one. - {ok, _} = ?block_until( - #{ - ?snk_kind := garbage_collection, - stats := #gcstats{ - files = Files, - directories = 4, - space = Space - } - } when Files == (?NSEGS(S2, SS2)) andalso Space > S2, - 5000, - 0 - ), - % 5. Then transfers 3 and 4 because 3rd has too big TTL and 4th has no specific TTL. - {ok, _} = ?block_until( - #{ - ?snk_kind := garbage_collection, - stats := #gcstats{ - files = Files, - directories = 4 * 2, - space = Space - } - } when Files == (?NSEGS(S3, SS3) + ?NSEGS(S4, SS4)) andalso Space > S3 + S4, - 5000, - 0 - ) - end, - [] + % 2. Enable periodic GC every 0.5 seconds. + ok = emqx_config:put([file_transfer, storage, gc, interval], 500), + ok = emqx_ft_storage_fs_gc:reset(Storage), + % 3. First we need the first transfer to be collected. + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = Files, + directories = 4, + space = Space + } + } when Files == (?NSEGS(S1, SS1)) andalso Space > S1, + 5000, + 0 + ), + % 4. Then the second one. + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = Files, + directories = 4, + space = Space + } + } when Files == (?NSEGS(S2, SS2)) andalso Space > S2, + 5000, + 0 + ), + % 5. Then transfers 3 and 4 because 3rd has too big TTL and 4th has no specific TTL. + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = Files, + directories = 4 * 2, + space = Space + } + } when Files == (?NSEGS(S3, SS3) + ?NSEGS(S4, SS4)) andalso Space > S3 + S4, + 5000, + 0 ). t_gc_handling_errors(_Config) -> From 45f00e14a9df3a5ca889f2fc8da6fa601b0987fb Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Mar 2023 15:18:25 +0300 Subject: [PATCH 069/156] feat(ft): ensure that clientid is always binary For the sake of simplicity (e.g. transfer ids are now easier to compare). --- apps/emqx_ft/src/emqx_ft.erl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 2ccd5db15..518807d9f 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -51,7 +51,7 @@ -type bytes() :: non_neg_integer(). %% MQTT Client ID --type clientid() :: emqx_types:clientid(). +-type clientid() :: binary(). -type fileid() :: binary(). -type transfer() :: {clientid(), fileid()}. @@ -327,7 +327,7 @@ assemble(Transfer, FinalSize) -> transfer(Msg, FileId) -> ClientId = Msg#message.from, - {ClientId, FileId}. + {clientid_to_binary(ClientId), FileId}. on_complete(Op, {ChanPid, PacketId}, Transfer, Result) -> ?SLOG(debug, #{ @@ -421,3 +421,8 @@ parse_checksum(Checksum) when is_binary(Checksum) andalso byte_size(Checksum) =: end; parse_checksum(_Checksum) -> {error, invalid_checksum}. + +clientid_to_binary(A) when is_atom(A) -> + atom_to_binary(A); +clientid_to_binary(B) when is_binary(B) -> + B. From 2880c8f4eb6c69ffdcdeec3cccc481761448098d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 21 Mar 2023 00:11:30 +0300 Subject: [PATCH 070/156] fix(ft): unwrap error details when reader fails to start --- apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl index 934e2888c..fff6b7830 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl @@ -33,7 +33,12 @@ start_child(CallerPid, Filename) -> start => {emqx_ft_storage_fs_reader, start_link, [CallerPid, Filename]}, restart => temporary }, - supervisor:start_child(?MODULE, Childspec). + case supervisor:start_child(?MODULE, Childspec) of + {ok, Pid} -> + {ok, Pid}; + {error, {Reason, _Child}} -> + {error, Reason} + end. init(_) -> SupFlags = #{ From 4132f5a5fb3127878532d436613a20026e8da7b4 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 10 Mar 2023 15:27:50 +0300 Subject: [PATCH 071/156] feat(ft): introduce exporter concept in local storage backend The exporter is responsible for keeping fully transferred and successfully assembled files. This was on the local storage itself before. This abstraction is needed to give us an ability to support S3 destinations more easily, just by swapping the storage exporter. Also implement local filesystem exporter and reimplement parts of the `emqx_ft` API on top of it. --- apps/emqx_ft/etc/emqx_ft.conf | 3 + apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 38 +- apps/emqx_ft/src/emqx_ft.erl | 8 +- apps/emqx_ft/src/emqx_ft_api.erl | 174 ++++++--- apps/emqx_ft/src/emqx_ft_assembler.erl | 59 ++- apps/emqx_ft/src/emqx_ft_fs_util.erl | 101 +++++ apps/emqx_ft/src/emqx_ft_schema.erl | 26 +- apps/emqx_ft/src/emqx_ft_storage.erl | 35 +- .../src/emqx_ft_storage_exporter_fs.erl | 352 ++++++++++++++++++ apps/emqx_ft/src/emqx_ft_storage_fs.erl | 281 +++----------- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 22 +- apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl | 22 +- .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 29 +- apps/emqx_ft/test/emqx_ft_SUITE.erl | 67 ++-- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 66 ++-- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 48 ++- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 11 +- .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 69 +--- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 12 +- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 15 +- 20 files changed, 936 insertions(+), 502 deletions(-) create mode 100644 apps/emqx_ft/src/emqx_ft_fs_util.erl create mode 100644 apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl diff --git a/apps/emqx_ft/etc/emqx_ft.conf b/apps/emqx_ft/etc/emqx_ft.conf index 250dca6a9..8d921e79c 100644 --- a/apps/emqx_ft/etc/emqx_ft.conf +++ b/apps/emqx_ft/etc/emqx_ft.conf @@ -1,5 +1,8 @@ file_transfer { storage { type = local + exporter { + type = local + } } } diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf index dd2d2a1dc..15c42dcfa 100644 --- a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -13,7 +13,7 @@ emqx_ft_schema { local_type { desc { - en: "Use local file system to store uploaded files and temporary data." + en: "Use local file system to store uploaded fragments and temporary data." zh: "使用本地文件系统来存储上传的文件和临时数据。" } label: { @@ -24,7 +24,7 @@ emqx_ft_schema { local_storage_root { desc { - en: "File system path to keep uploaded files and temporary data." + en: "File system path to keep uploaded fragments and temporary data." zh: "保存上传文件和临时数据的文件系统路径。" } label: { @@ -33,6 +33,40 @@ emqx_ft_schema { } } + local_storage_exporter { + desc { + en: "Exporter for the local file system storage backend.
" + "Exporter defines where and how fully transferred and assembled files are stored." + zh: "" + } + label: { + en: "Local Storage Exporter" + zh: "" + } + } + + local_storage_exporter_type { + desc { + en: "Type of the Exporter to use." + zh: "" + } + label: { + en: "Local Storage Exporter Type" + zh: "" + } + } + + local_storage_exporter_root { + desc { + en: "File system path to keep uploaded files." + zh: "" + } + label: { + en: "Local Filesystem Exporter Root" + zh: "" + } + } + local_storage_gc { desc { en: "Garbage collection settings for the intermediate and temporary files in the local file system." diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 518807d9f..b7c8f0eac 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -33,7 +33,8 @@ ]). -export([ - decode_filemeta/1 + decode_filemeta/1, + encode_filemeta/1 ]). -export([on_complete/4]). @@ -114,6 +115,11 @@ decode_filemeta(Map) when is_map(Map) -> {error, {invalid_filemeta, Error}} end. +encode_filemeta(Meta = #{}) -> + % TODO: Looks like this should be hocon's responsibility. + Schema = emqx_ft_schema:schema(filemeta), + hocon_tconf:make_serializable(Schema, emqx_map_lib:binary_key_map(Meta), #{}). + %%-------------------------------------------------------------------- %% Hooks %%-------------------------------------------------------------------- diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index ddc6e761a..143d629de 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -40,6 +40,10 @@ '/file_transfer/file'/2 ]). +-export([ + mk_file_uri/3 +]). + -import(hoconsc, [mk/2, ref/1, ref/2]). namespace() -> "file_transfer". @@ -69,6 +73,11 @@ schema("/file_transfer/files") -> } }; schema("/file_transfer/file") -> + % TODO + % This is conceptually another API, because this logic is inherent only to the + % local filesystem exporter. Ideally, we won't even publish it if `emqx_ft` is + % configured with another exporter. Accordingly, things that look too specific + % for this module (i.e. `parse_filepath/1`) should go away in another API module. #{ 'operationId' => '/file_transfer/file', get => #{ @@ -77,8 +86,7 @@ schema("/file_transfer/file") -> description => ?DESC("file_get"), parameters => [ ref(file_node), - ref(file_clientid), - ref(file_id) + ref(file_ref) ], responses => #{ 200 => <<"Operation success">>, @@ -91,32 +99,40 @@ schema("/file_transfer/file") -> }. '/file_transfer/files'(get, #{}) -> - case emqx_ft_storage:ready_transfers() of + case emqx_ft_storage:exports() of {ok, Transfers} -> - FormattedTransfers = lists:map( - fun({Id, Info}) -> - #{id => Id, info => format_file_info(Info)} - end, - Transfers - ), - {200, #{<<"files">> => FormattedTransfers}}; + {200, #{<<"files">> => lists:map(fun format_export_info/1, Transfers)}}; {error, _} -> {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} end. '/file_transfer/file'(get, #{query_string := Query}) -> - case emqx_ft_storage:get_ready_transfer(Query) of - {ok, FileData} -> - {200, - #{ - <<"content-type">> => <<"application/data">>, - <<"content-disposition">> => <<"attachment">> - }, - FileData}; - {error, enoent} -> - {404, error_msg('NOT_FOUND', <<"Not found">>)}; - {error, Error} -> - ?SLOG(warning, #{msg => "get_ready_transfer_fail", error => Error}), + try + Node = parse_node(maps:get(<<"node">>, Query)), + Filepath = parse_filepath(maps:get(<<"fileref">>, Query)), + case emqx_ft_storage_fs_proto_v1:read_export_file(Node, Filepath, self()) of + {ok, ReaderPid} -> + FileData = emqx_ft_storage_fs_reader:table(ReaderPid), + {200, + #{ + <<"content-type">> => <<"application/data">>, + <<"content-disposition">> => <<"attachment">> + }, + FileData}; + {error, enoent} -> + {404, error_msg('NOT_FOUND', <<"Not found">>)}; + {error, Error} -> + ?SLOG(warning, #{msg => "get_ready_transfer_fail", error => Error}), + {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} + end + catch + throw:{invalid, Param} -> + {404, + error_msg( + 'NOT_FOUND', + iolist_to_binary(["Invalid query parameter: ", Param]) + )}; + error:{erpc, noconnection} -> {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} end. @@ -124,46 +140,100 @@ error_msg(Code, Msg) -> #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. -spec fields(hocon_schema:name()) -> hocon_schema:fields(). +fields(file_ref) -> + [ + {fileref, + hoconsc:mk(binary(), #{ + in => query, + desc => <<"File reference">>, + example => <<"file1">>, + required => true + })} + ]; fields(file_node) -> - Desc = <<"File Node">>, - Meta = #{ - in => query, desc => Desc, example => <<"emqx@127.0.0.1">>, required => false - }, - [{node, hoconsc:mk(binary(), Meta)}]; -fields(file_clientid) -> - Desc = <<"File ClientId">>, - Meta = #{ - in => query, desc => Desc, example => <<"client1">>, required => false - }, - [{clientid, hoconsc:mk(binary(), Meta)}]; -fields(file_id) -> - Desc = <<"File">>, - Meta = #{ - in => query, desc => Desc, example => <<"file1">>, required => false - }, - [{fileid, hoconsc:mk(binary(), Meta)}]. + [ + {node, + hoconsc:mk(binary(), #{ + in => query, + desc => <<"Node under which the file is located">>, + example => atom_to_list(node()), + required => true + })} + ]. roots() -> [ file_node, - file_clientid, - file_id + file_ref + ]. + +mk_file_uri(_Options, Node, Filepath) -> + % TODO: qualify with `?BASE_PATH` + [ + "/file_transfer/file?", + uri_string:compose_query([ + {"node", atom_to_list(Node)}, + {"fileref", Filepath} + ]) ]. %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- -format_file_info(#{path := Path, size := Size, timestamp := Timestamp}) -> - #{ - path => Path, +format_export_info( + Info = #{ + name := Name, + size := Size, + uri := URI, + timestamp := Timestamp, + transfer := {ClientId, FileId} + } +) -> + Res = #{ + name => iolist_to_binary(Name), size => Size, - timestamp => format_datetime(Timestamp) - }. + timestamp => format_timestamp(Timestamp), + clientid => ClientId, + fileid => FileId, + uri => iolist_to_binary(URI) + }, + case Info of + #{meta := Meta} -> + Res#{metadata => emqx_ft:encode_filemeta(Meta)}; + #{} -> + Res + end. -format_datetime({{Year, Month, Day}, {Hour, Minute, Second}}) -> - iolist_to_binary( - io_lib:format("~4..0w-~2..0w-~2..0wT~2..0w:~2..0w:~2..0w", [ - Year, Month, Day, Hour, Minute, Second - ]) - ). +format_timestamp(Timestamp) -> + iolist_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])). + +parse_node(NodeBin) -> + case emqx_misc:safe_to_existing_atom(NodeBin) of + {ok, Node} -> + Node; + {error, _} -> + throw({invalid, NodeBin}) + end. + +parse_filepath(PathBin) -> + case filename:pathtype(PathBin) of + relative -> + ok; + absolute -> + throw({invalid, PathBin}) + end, + PathComponents = filename:split(PathBin), + case lists:any(fun is_special_component/1, PathComponents) of + false -> + filename:join(PathComponents); + true -> + throw({invalid, PathBin}) + end. + +is_special_component(<<".", _/binary>>) -> + true; +is_special_component([$. | _]) -> + true; +is_special_component(_) -> + false. diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index ff845fee9..5489a232a 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -22,13 +22,13 @@ -export([callback_mode/0]). -export([init/1]). -export([handle_event/4]). +-export([terminate/3]). -record(st, { storage :: _Storage, transfer :: emqx_ft:transfer(), assembly :: emqx_ft_assembly:t(), - file :: {file:filename(), io:device(), term()} | undefined, - hash + export :: _Export | undefined }). -define(NAME(Transfer), {n, l, {?MODULE, Transfer}}). @@ -47,11 +47,11 @@ callback_mode() -> handle_event_function. init({Storage, Transfer, Size}) -> + _ = erlang:process_flag(trap_exit, true), St = #st{ storage = Storage, transfer = Transfer, - assembly = emqx_ft_assembly:new(Size), - hash = crypto:hash_init(sha256) + assembly = emqx_ft_assembly:new(Size) }, {ok, idle, St}. @@ -61,10 +61,10 @@ handle_event(info, kickoff, idle, St) -> % We could wait for this message and handle it at the end of the assembling rather than at % the beginning, however it would make error handling much more messier. {next_state, list_local_fragments, St, ?internal([])}; -handle_event(internal, _, list_local_fragments, St = #st{assembly = Asm}) -> +handle_event(internal, _, list_local_fragments, St = #st{}) -> % TODO: what we do with non-transients errors here (e.g. `eacces`)? {ok, Fragments} = emqx_ft_storage_fs:list(St#st.storage, St#st.transfer, fragment), - NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(Asm, node(), Fragments)), + NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(St#st.assembly, node(), Fragments)), NSt = St#st{assembly = NAsm}, case emqx_ft_assembly:status(NAsm) of complete -> @@ -110,8 +110,8 @@ handle_event(internal, _, start_assembling, St = #st{assembly = Asm}) -> Filemeta = emqx_ft_assembly:filemeta(Asm), Coverage = emqx_ft_assembly:coverage(Asm), % TODO: better error handling - {ok, Handle} = emqx_ft_storage_fs:open_file(St#st.storage, St#st.transfer, Filemeta), - {next_state, {assemble, Coverage}, St#st{file = Handle}, ?internal([])}; + {ok, Export} = export_start(Filemeta, St), + {next_state, {assemble, Coverage}, St#st{export = Export}, ?internal([])}; handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % TODO % Currently, race is possible between getting segment info from the remote node and @@ -119,15 +119,17 @@ handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % TODO: pipelining % TODO: better error handling {ok, Content} = pread(Node, Segment, St), - {ok, NHandle} = emqx_ft_storage_fs:write(St#st.file, Content), - {next_state, {assemble, Rest}, St#st{file = NHandle}, ?internal([])}; + {ok, NExport} = export_write(St#st.export, Content), + {next_state, {assemble, Rest}, St#st{export = NExport}, ?internal([])}; handle_event(internal, _, {assemble, []}, St = #st{}) -> {next_state, complete, St, ?internal([])}; -handle_event(internal, _, complete, St = #st{assembly = Asm, file = Handle}) -> - Filemeta = emqx_ft_assembly:filemeta(Asm), - Result = emqx_ft_storage_fs:complete(St#st.storage, St#st.transfer, Filemeta, Handle), +handle_event(internal, _, complete, St = #st{}) -> + Result = export_complete(St#st.export), ok = maybe_garbage_collect(Result, St), - {stop, {shutdown, Result}}. + {stop, {shutdown, Result}, St#st{export = undefined}}. + +terminate(_Reason, _StateName, #st{export = Export}) -> + Export /= undefined andalso export_discard(Export). pread(Node, Segment, St) when Node =:= node() -> emqx_ft_storage_fs:pread(St#st.storage, St#st.transfer, Segment, 0, segsize(Segment)); @@ -136,8 +138,33 @@ pread(Node, Segment, St) -> %% -maybe_garbage_collect(ok, St = #st{storage = Storage, transfer = Transfer}) -> - Nodes = emqx_ft_assembly:nodes(St#st.assembly), +export_start(Filemeta, #st{storage = Storage, transfer = Transfer}) -> + {ExporterMod, Exporter} = emqx_ft_storage_fs:exporter(Storage), + case ExporterMod:start_export(Exporter, Transfer, Filemeta) of + {ok, Export} -> + {ok, {ExporterMod, Export}}; + {error, _} = Error -> + Error + end. + +export_write({ExporterMod, Export}, Content) -> + case ExporterMod:write(Export, Content) of + {ok, ExportNext} -> + {ok, {ExporterMod, ExportNext}}; + {error, _} = Error -> + Error + end. + +export_complete({ExporterMod, Export}) -> + ExporterMod:complete(Export). + +export_discard({ExporterMod, Export}) -> + ExporterMod:discard(Export). + +%% + +maybe_garbage_collect(ok, #st{storage = Storage, transfer = Transfer, assembly = Asm}) -> + Nodes = emqx_ft_assembly:nodes(Asm), emqx_ft_storage_fs_gc:collect(Storage, Transfer, Nodes); maybe_garbage_collect({error, _}, _St) -> ok. diff --git a/apps/emqx_ft/src/emqx_ft_fs_util.erl b/apps/emqx_ft/src/emqx_ft_fs_util.erl new file mode 100644 index 000000000..198f4ccc5 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_fs_util.erl @@ -0,0 +1,101 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_fs_util). + +-include_lib("snabbkaffe/include/trace.hrl"). + +-export([read_decode_file/2]). + +-export([fold/4]). + +-type glob() :: ['*' | globfun()]. +-type globfun() :: + fun((_Filename :: file:name()) -> boolean()). +-type foldfun(Acc) :: + fun( + ( + _Filepath :: file:name(), + _Info :: file:file_info() | {error, _IoError}, + _Stack :: [file:name()], + Acc + ) -> Acc + ). + +%% + +-spec read_decode_file(file:name(), fun((binary()) -> Value)) -> + {ok, Value} | {error, _IoError}. +read_decode_file(Filepath, DecodeFun) -> + case file:read_file(Filepath) of + {ok, Content} -> + safe_decode(Content, DecodeFun); + {error, _} = Error -> + Error + end. + +safe_decode(Content, DecodeFun) -> + try + {ok, DecodeFun(Content)} + catch + C:E:Stacktrace -> + ?tp(warning, "safe_decode_failed", #{ + class => C, + exception => E, + stacktrace => Stacktrace + }), + {error, corrupted} + end. + +-spec fold(foldfun(Acc), Acc, _Root :: file:name(), glob()) -> + Acc. +fold(Fun, Acc, Root, Glob) -> + fold(Fun, Acc, [], Root, Glob, []). + +fold(Fun, AccIn, Path, Root, [Glob | Rest], Stack) when Glob == '*' orelse is_function(Glob) -> + case file:list_dir(filename:join(Root, Path)) of + {ok, Filenames} -> + lists:foldl( + fun(FN, Acc) -> + case matches_glob(Glob, FN) of + true when Path == [] -> + fold(Fun, Acc, FN, Root, Rest, [FN | Stack]); + true -> + fold(Fun, Acc, filename:join(Path, FN), Root, Rest, [FN | Stack]); + false -> + Acc + end + end, + AccIn, + Filenames + ); + {error, enotdir} -> + AccIn; + {error, Reason} -> + Fun(Path, {error, Reason}, Stack, AccIn) + end; +fold(Fun, AccIn, Filepath, Root, [], Stack) -> + case file:read_link_info(filename:join(Root, Filepath), [{time, posix}, raw]) of + {ok, Info} -> + Fun(Filepath, Info, Stack, AccIn); + {error, Reason} -> + Fun(Filepath, {error, Reason}, Stack, AccIn) + end. + +matches_glob('*', _) -> + true; +matches_glob(FilterFun, Filename) when is_function(FilterFun) -> + FilterFun(Filename). diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index d2b2b9299..37e2adafc 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -66,12 +66,33 @@ fields(local_storage) -> desc => ?DESC("local_storage_root"), required => false }}, + {exporter, #{ + type => hoconsc:union([ + ?REF(local_storage_exporter) + ]), + desc => ?DESC("local_storage_exporter"), + required => true + }}, {gc, #{ - type => hoconsc:ref(?MODULE, local_storage_gc), + type => ?REF(local_storage_gc), desc => ?DESC("local_storage_gc"), required => false }} ]; +fields(local_storage_exporter) -> + [ + {type, #{ + type => local, + default => local, + required => false, + desc => ?DESC("local_storage_exporter_type") + }}, + {root, #{ + type => binary(), + desc => ?DESC("local_storage_exporter_root"), + required => false + }} + ]; fields(local_storage_gc) -> [ {interval, #{ @@ -101,12 +122,15 @@ desc(file_transfer) -> "File transfer settings"; desc(local_storage) -> "File transfer local storage settings"; +desc(local_storage_exporter) -> + "Exporter settings for the File transfer local storage backend"; desc(local_storage_gc) -> "Garbage collection settings for the File transfer local storage backend". schema(filemeta) -> #{ roots => [ + % TODO nonempty {name, hoconsc:mk(string(), #{required => true})}, {size, hoconsc:mk(non_neg_integer())}, {expire_at, hoconsc:mk(non_neg_integer())}, diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 0b8c38736..1d1c08ce9 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -24,8 +24,7 @@ store_segment/2, assemble/2, - ready_transfers/0, - get_ready_transfer/1, + exports/0, with_storage_type/3 ] @@ -34,12 +33,19 @@ -type storage() :: emqx_config:config(). -export_type([assemble_callback/0]). +-export_type([export_data/0]). -type assemble_callback() :: fun((ok | {error, term()}) -> any()). --type ready_transfer_id() :: term(). --type ready_transfer_info() :: map(). --type ready_transfer_data() :: binary() | qlc:query_handle(). +-type export_info() :: #{ + transfer := emqx_ft:transfer(), + name := file:name(), + size := _Bytes :: non_neg_integer(), + uri => uri_string:uri_string(), + meta => emqx_ft:filemeta() +}. + +-type export_data() :: binary() | qlc:query_handle(). %%-------------------------------------------------------------------- %% Behaviour @@ -57,10 +63,9 @@ ok | {async, pid()} | {error, term()}. -callback assemble(storage(), emqx_ft:transfer(), _Size :: emqx_ft:bytes()) -> ok | {async, pid()} | {error, term()}. --callback ready_transfers(storage()) -> - {ok, [{ready_transfer_id(), ready_transfer_info()}]} | {error, term()}. --callback get_ready_transfer(storage(), ready_transfer_id()) -> - {ok, ready_transfer_data()} | {error, term()}. + +-callback exports(storage()) -> + {ok, [export_info()]} | {error, term()}. %%-------------------------------------------------------------------- %% API @@ -95,15 +100,11 @@ assemble(Transfer, Size) -> Mod = mod(), Mod:assemble(storage(), Transfer, Size). --spec ready_transfers() -> {ok, [{ready_transfer_id(), ready_transfer_info()}]} | {error, term()}. -ready_transfers() -> +-spec exports() -> + {ok, [export_info()]} | {error, term()}. +exports() -> Mod = mod(), - Mod:ready_transfers(storage()). - --spec get_ready_transfer(ready_transfer_id()) -> {ok, ready_transfer_data()} | {error, term()}. -get_ready_transfer(ReadyTransferId) -> - Mod = mod(), - Mod:get_ready_transfer(storage(), ReadyTransferId). + Mod:exports(storage()). -spec with_storage_type(atom(), atom(), list(term())) -> any(). with_storage_type(Type, Fun, Args) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl new file mode 100644 index 000000000..e0bffd444 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -0,0 +1,352 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_exporter_fs). + +-include_lib("kernel/include/file.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% Exporter API +-export([start_export/3]). +-export([write/2]). +-export([complete/1]). +-export([discard/1]). + +-export([list_local/1]). +-export([list_local/2]). +-export([start_reader/3]). + +-export([list/1]). +% -export([list/2]). + +-export_type([export/0]). + +-type options() :: _TODO. +-type transfer() :: emqx_ft:transfer(). +-type filemeta() :: emqx_ft:filemeta(). +-type exportinfo() :: #{ + transfer := transfer(), + name := file:name(), + uri := uri_string:uri_string(), + timestamp := emqx_datetime:epoch_second(), + size := _Bytes :: non_neg_integer(), + meta => filemeta() +}. + +-type file_error() :: emqx_ft_storage_fs:file_error(). + +-opaque export() :: #{ + path := file:name(), + handle := io:device(), + result := file:name(), + meta := filemeta(), + hash := crypto:hash_state() +}. + +-type reader() :: pid(). + +-define(TEMPDIR, "tmp"). +-define(MANIFEST, ".MANIFEST.json"). + +%% NOTE +%% Bucketing of resulting files to accomodate the storage backend for considerably +%% large (e.g. > 10s of millions) amount of files. +-define(BUCKET_HASH, sha). + +%% 2 symbols = at most 256 directories on the upper level +-define(BUCKET1_LEN, 2). +%% 2 symbols = at most 256 directories on the second level +-define(BUCKET2_LEN, 2). + +-define(SLOG_UNEXPECTED(RelFilepath, Fileinfo, Options), + ?SLOG(notice, "filesystem_object_unexpected", #{ + relpath => RelFilepath, + fileinfo => Fileinfo, + options => Options + }) +). + +-define(SLOG_INACCESSIBLE(RelFilepath, Reason, Options), + ?SLOG(warning, "filesystem_object_inaccessible", #{ + relpath => RelFilepath, + reason => Reason, + options => Options + }) +). + +%% + +-spec start_export(options(), transfer(), filemeta()) -> + {ok, export()} | {error, file_error()}. +start_export(Options, Transfer, Filemeta = #{name := Filename}) -> + TempFilepath = mk_temp_absfilepath(Options, Transfer, Filename), + ResultFilepath = mk_absfilepath(Options, Transfer, result, Filename), + _ = filelib:ensure_dir(TempFilepath), + case file:open(TempFilepath, [write, raw, binary]) of + {ok, Handle} -> + {ok, #{ + path => TempFilepath, + handle => Handle, + result => ResultFilepath, + meta => Filemeta, + hash => init_checksum(Filemeta) + }}; + {error, _} = Error -> + Error + end. + +-spec write(export(), iodata()) -> + {ok, export()} | {error, file_error()}. +write(Export = #{handle := Handle, hash := Ctx}, IoData) -> + case file:write(Handle, IoData) of + ok -> + {ok, Export#{hash := update_checksum(Ctx, IoData)}}; + {error, _} = Error -> + Error + end. + +-spec complete(export()) -> + ok | {error, {checksum, _Algo, _Computed}} | {error, file_error()}. +complete( + Export = #{ + path := Filepath, + handle := Handle, + result := ResultFilepath, + meta := FilemetaIn, + hash := Ctx + } +) -> + case verify_checksum(Ctx, FilemetaIn) of + {ok, Filemeta} -> + ok = file:close(Handle), + _ = filelib:ensure_dir(ResultFilepath), + _ = file:write_file(mk_manifest_filename(ResultFilepath), encode_filemeta(Filemeta)), + file:rename(Filepath, ResultFilepath); + {error, _} = Error -> + _ = discard(Export), + Error + end. + +-spec discard(export()) -> + ok. +discard(#{path := Filepath, handle := Handle}) -> + ok = file:close(Handle), + file:delete(Filepath). + +%% + +-spec list_local(options(), transfer()) -> + {ok, [exportinfo(), ...]} | {error, file_error()}. +list_local(Options, Transfer) -> + TransferRoot = mk_absdir(Options, Transfer, result), + case + emqx_ft_fs_util:fold( + fun + (_Path, {error, Reason}, [], []) -> + {error, Reason}; + (_Path, Fileinfo = #file_info{type = regular}, [Filename | _], Acc) -> + RelFilepath = filename:join(mk_result_reldir(Transfer) ++ [Filename]), + Info = mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo), + [Info | Acc]; + (RelFilepath, Fileinfo = #file_info{}, _, Acc) -> + ?SLOG_UNEXPECTED(RelFilepath, Fileinfo, Options), + Acc; + (RelFilepath, {error, Reason}, _, Acc) -> + ?SLOG_INACCESSIBLE(RelFilepath, Reason, Options), + Acc + end, + [], + TransferRoot, + [fun filter_manifest/1] + ) + of + Infos = [_ | _] -> + {ok, Infos}; + [] -> + {error, enoent}; + {error, Reason} -> + {error, Reason} + end. + +-spec list_local(options()) -> + {ok, #{transfer() => [exportinfo(), ...]}}. +list_local(Options) -> + Pattern = [ + _Bucket1 = '*', + _Bucket2 = '*', + _Rest = '*', + _ClientId = '*', + _FileId = '*', + fun filter_manifest/1 + ], + Root = get_storage_root(Options), + {ok, + emqx_ft_fs_util:fold( + fun(RelFilepath, Info, Stack, Acc) -> + read_exportinfo(Options, RelFilepath, Info, Stack, Acc) + end, + [], + Root, + Pattern + )}. + +filter_manifest(?MANIFEST) -> + % Filename equals `?MANIFEST`, there should also be a manifest for it. + false; +filter_manifest(Filename) -> + ?MANIFEST =/= string:find(Filename, ?MANIFEST, trailing). + +read_exportinfo(Options, RelFilepath, Fileinfo = #file_info{type = regular}, Stack, Acc) -> + [Filename, FileId, ClientId | _] = Stack, + Transfer = dirnames_to_transfer(ClientId, FileId), + Info = mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo), + [Info | Acc]; +read_exportinfo(Options, RelFilepath, Fileinfo = #file_info{}, _Stack, Acc) -> + ?SLOG_UNEXPECTED(RelFilepath, Fileinfo, Options), + Acc; +read_exportinfo(Options, RelFilepath, {error, Reason}, _Stack, Acc) -> + ?SLOG_INACCESSIBLE(RelFilepath, Reason, Options), + Acc. + +mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo) -> + Root = get_storage_root(Options), + try_read_filemeta( + filename:join(Root, mk_manifest_filename(RelFilepath)), + #{ + transfer => Transfer, + name => Filename, + uri => mk_export_uri(Options, RelFilepath), + timestamp => Fileinfo#file_info.mtime, + size => Fileinfo#file_info.size, + path => filename:join(Root, RelFilepath) + } + ). + +try_read_filemeta(Filepath, Info) -> + case emqx_ft_fs_util:read_decode_file(Filepath, fun decode_filemeta/1) of + {ok, Filemeta} -> + Info#{meta => Filemeta}; + {error, Reason} -> + ?SLOG(warning, "filemeta_inaccessible", #{ + path => Filepath, + reason => Reason + }), + Info + end. + +mk_export_uri(Options, RelFilepath) -> + % emqx_ft_storage_exporter_fs_api:mk_export_uri(Options, RelFilepath). + emqx_ft_api:mk_file_uri(Options, node(), RelFilepath). + +-spec start_reader(options(), file:name(), _Caller :: pid()) -> + {ok, reader()} | {error, enoent}. +start_reader(Options, Filepath, CallerPid) -> + Root = get_storage_root(Options), + case filelib:safe_relative_path(Filepath, Root) of + SafeFilepath when SafeFilepath /= unsafe -> + AbsFilepath = filename:join(Root, SafeFilepath), + emqx_ft_storage_fs_reader:start_supervised(CallerPid, AbsFilepath); + unsafe -> + {error, enoent} + end. + +%% + +-spec list(options()) -> + {ok, [exportinfo(), ...]} | {error, file_error()}. +list(_Options) -> + Nodes = mria_mnesia:running_nodes(), + Results = emqx_ft_storage_fs_proto_v1:list_exports(Nodes), + {GoodResults, BadResults} = lists:partition( + fun + ({_Node, {ok, {ok, _}}}) -> true; + (_) -> false + end, + lists:zip(Nodes, Results) + ), + length(BadResults) > 0 andalso + ?SLOG(warning, #{msg => "list_remote_exports_failed", failures => BadResults}), + {ok, [File || {_Node, {ok, {ok, Files}}} <- GoodResults, File <- Files]}. + +%% + +init_checksum(#{checksum := {Algo, _}}) -> + crypto:hash_init(Algo); +init_checksum(#{}) -> + crypto:hash_init(sha256). + +update_checksum(Ctx, IoData) -> + crypto:hash_update(Ctx, IoData). + +verify_checksum(Ctx, Filemeta = #{checksum := {Algo, Digest}}) -> + case crypto:hash_final(Ctx) of + Digest -> + {ok, Filemeta}; + Mismatch -> + {error, {checksum, Algo, binary:encode_hex(Mismatch)}} + end; +verify_checksum(Ctx, Filemeta = #{}) -> + Digest = crypto:hash_final(Ctx), + {ok, Filemeta#{checksum => {sha256, Digest}}}. + +%% + +-define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). + +encode_filemeta(Meta) -> + emqx_json:encode(?PRELUDE(_Vsn = 1, emqx_ft:encode_filemeta(Meta))). + +decode_filemeta(Binary) when is_binary(Binary) -> + ?PRELUDE(_Vsn = 1, Map) = emqx_json:decode(Binary, [return_maps]), + case emqx_ft:decode_filemeta(Map) of + {ok, Meta} -> + Meta; + {error, Reason} -> + error(Reason) + end. + +mk_manifest_filename(Filename) when is_list(Filename) -> + Filename ++ ?MANIFEST; +mk_manifest_filename(Filename) when is_binary(Filename) -> + <>. + +mk_temp_absfilepath(Options, Transfer, Filename) -> + Unique = erlang:unique_integer([positive]), + TempFilename = integer_to_list(Unique) ++ "." ++ Filename, + filename:join(mk_absdir(Options, Transfer, temporary), TempFilename). + +mk_absdir(Options, _Transfer, temporary) -> + filename:join([get_storage_root(Options), ?TEMPDIR]); +mk_absdir(Options, Transfer, result) -> + filename:join([get_storage_root(Options) | mk_result_reldir(Transfer)]). + +mk_absfilepath(Options, Transfer, What, Filename) -> + filename:join(mk_absdir(Options, Transfer, What), Filename). + +mk_result_reldir(Transfer = {ClientId, FileId}) -> + Hash = mk_transfer_hash(Transfer), + << + Bucket1:?BUCKET1_LEN/binary, + Bucket2:?BUCKET2_LEN/binary, + BucketRest/binary + >> = binary:encode_hex(Hash), + [Bucket1, Bucket2, BucketRest, ClientId, FileId]. + +mk_transfer_hash(Transfer) -> + crypto:hash(?BUCKET_HASH, term_to_binary(Transfer)). + +get_storage_root(Options) -> + maps:get(root, Options, filename:join([emqx:data_dir(), "ft", "exports"])). diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index b8aef5276..4a613dbcf 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -29,6 +29,7 @@ -export([child_spec/1]). +% Segments-related API -export([store_filemeta/3]). -export([store_segment/3]). -export([read_filemeta/2]). @@ -43,22 +44,20 @@ -export([get_subdir/2]). -export([get_subdir/3]). --export([ready_transfers_local/1]). --export([get_ready_transfer_local/3]). +-export([exporter/1]). --export([ready_transfers/1]). --export([get_ready_transfer/2]). - --export([open_file/3]). --export([complete/4]). --export([write/2]). --export([discard/1]). +% Exporter-specific API +-export([exports/1]). +-export([exports_local/1]). +-export([exports_local/2]). -export_type([storage/0]). -export_type([filefrag/1]). -export_type([filefrag/0]). -export_type([transferinfo/0]). +-export_type([file_error/0]). + -type transfer() :: emqx_ft:transfer(). -type offset() :: emqx_ft:offset(). -type filemeta() :: emqx_ft:filemeta(). @@ -70,8 +69,7 @@ }. -type transferinfo() :: #{ - status := complete | incomplete, - result => [filefrag({result, #{}})] + filemeta => filemeta() }. % TODO naming @@ -85,16 +83,20 @@ -type filefrag() :: filefrag( {filemeta, filemeta()} | {segment, segmentinfo()} - | {result, #{}} ). -define(FRAGDIR, frags). -define(TEMPDIR, tmp). --define(RESULTDIR, result). -define(MANIFEST, "MANIFEST.json"). -define(SEGMENT, "SEG"). -type storage() :: #{ + root => file:name(), + exporter => exporter() +}. + +-type exporter() :: #{ + type := local, root => file:name() }. @@ -138,7 +140,9 @@ store_filemeta(Storage, Transfer, Meta) -> % about it too much now. {error, conflict}; {error, Reason} when Reason =:= notfound; Reason =:= corrupted; Reason =:= enoent -> - write_file_atomic(Storage, Transfer, Filepath, encode_filemeta(Meta)) + write_file_atomic(Storage, Transfer, Filepath, encode_filemeta(Meta)); + {error, _} = Error -> + Error end. %% Store a segment in the backing filesystem. @@ -153,17 +157,17 @@ store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> write_file_atomic(Storage, Transfer, Filepath, Content). -spec read_filemeta(storage(), transfer()) -> - {ok, filefrag({filemeta, filemeta()})} | {error, corrupted} | {error, file_error()}. + {ok, filemeta()} | {error, corrupted} | {error, file_error()}. read_filemeta(Storage, Transfer) -> Filepath = mk_filepath(Storage, Transfer, get_subdirs_for(fragment), ?MANIFEST), read_file(Filepath, fun decode_filemeta/1). --spec list(storage(), transfer(), _What :: fragment | result) -> +-spec list(storage(), transfer(), _What :: fragment) -> % Some lower level errors? {error, notfound}? % Result will contain zero or only one filemeta. {ok, [filefrag({filemeta, filemeta()} | {segment, segmentinfo()})]} | {error, file_error()}. -list(Storage, Transfer, What) -> +list(Storage, Transfer, What = fragment) -> Dirname = mk_filedir(Storage, Transfer, get_subdirs_for(What)), case file:list_dir(Dirname) of {ok, Filenames} -> @@ -172,18 +176,13 @@ list(Storage, Transfer, What) -> % extremely bad luck is needed for that, e.g. concurrent assemblers with % different filemetas from different nodes). This might be unexpected for a % client given the current protocol, yet might be helpful in the future. - {ok, filtermap_files(get_filefrag_fun_for(What), Dirname, Filenames)}; + {ok, filtermap_files(fun mk_filefrag/2, Dirname, Filenames)}; {error, enoent} -> {ok, []}; {error, _} = Error -> Error end. -get_filefrag_fun_for(fragment) -> - fun mk_filefrag/2; -get_filefrag_fun_for(result) -> - fun mk_result_filefrag/2. - -spec pread(storage(), transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> {ok, _Content :: iodata()} | {error, eof} | {error, file_error()}. pread(_Storage, _Transfer, Frag, Offset, Size) -> @@ -213,102 +212,28 @@ assemble(Storage, Transfer, Size) -> {ok, Pid} = emqx_ft_assembler_sup:ensure_child(Storage, Transfer, Size), {async, Pid}. -get_ready_transfer(_Storage, ReadyTransferId) -> - case parse_ready_transfer_id(ReadyTransferId) of - {ok, {Node, Transfer}} -> - try - case emqx_ft_storage_fs_proto_v1:get_ready_transfer(Node, self(), Transfer) of - {ok, ReaderPid} -> - {ok, emqx_ft_storage_fs_reader:table(ReaderPid)}; - {error, _} = Error -> - Error - end - catch - error:Exc:Stacktrace -> - ?SLOG(warning, #{ - msg => "get_ready_transfer_error", - node => Node, - transfer => Transfer, - exception => Exc, - stacktrace => Stacktrace - }), - {error, Exc}; - C:Exc:Stacktrace -> - ?SLOG(warning, #{ - msg => "get_ready_transfer_fail", - class => C, - node => Node, - transfer => Transfer, - exception => Exc, - stacktrace => Stacktrace - }), - {error, {C, Exc}} - end; - {error, _} = Error -> - Error +%% + +-spec exporter(storage()) -> {module(), _ExporterOptions}. +exporter(Storage) -> + case maps:get(exporter, Storage) of + #{type := local} = Options -> + {emqx_ft_storage_exporter_fs, Options} end. -get_ready_transfer_local(Storage, CallerPid, Transfer) -> - Dirname = mk_filedir(Storage, Transfer, get_subdirs_for(result)), - case file:list_dir(Dirname) of - {ok, [Filename | _]} -> - FullFilename = filename:join([Dirname, Filename]), - emqx_ft_storage_fs_reader:start_supervised(CallerPid, FullFilename); - {error, _} = Error -> - Error - end. +exports(Storage) -> + {ExporterMod, ExporterOpts} = exporter(Storage), + ExporterMod:list(ExporterOpts). -ready_transfers(_Storage) -> - Nodes = mria_mnesia:running_nodes(), - Results = emqx_ft_storage_fs_proto_v1:ready_transfers(Nodes), - {GoodResults, BadResults} = lists:partition( - fun - ({ok, _}) -> true; - (_) -> false - end, - Results - ), - case {GoodResults, BadResults} of - {[], _} -> - ?SLOG(warning, #{msg => "ready_transfers", failures => BadResults}), - {error, no_nodes}; - {_, []} -> - {ok, [File || {ok, Files} <- GoodResults, File <- Files]}; - {_, _} -> - ?SLOG(warning, #{msg => "ready_transfers", failures => BadResults}), - {ok, [File || {ok, Files} <- GoodResults, File <- Files]} - end. +exports_local(Storage) -> + {ExporterMod, ExporterOpts} = exporter(Storage), + ExporterMod:list_local(ExporterOpts). -ready_transfers_local(Storage) -> - {ok, Transfers} = transfers(Storage), - lists:filtermap( - fun - ({Transfer, #{status := complete, result := [Result | _]}}) -> - {true, {ready_transfer_id(Transfer), maps:without([fragment], Result)}}; - (_) -> - false - end, - maps:to_list(Transfers) - ). +exports_local(Storage, Transfer) -> + {ExporterMod, ExporterOpts} = exporter(Storage), + ExporterMod:list_local(ExporterOpts, Transfer). -ready_transfer_id({ClientId, FileId}) -> - #{ - <<"node">> => atom_to_binary(node()), - <<"clientid">> => ClientId, - <<"fileid">> => FileId - }. - -parse_ready_transfer_id(#{ - <<"node">> := NodeBin, <<"clientid">> := ClientId, <<"fileid">> := FileId -}) -> - case emqx_misc:safe_to_existing_atom(NodeBin) of - {ok, Node} -> - {ok, {Node, {ClientId, FileId}}}; - {error, _} -> - {error, {invalid_node, NodeBin}} - end; -parse_ready_transfer_id(#{}) -> - {error, invalid_file_id}. +%% -spec transfers(storage()) -> {ok, #{transfer() => transferinfo()}}. @@ -345,17 +270,16 @@ transfers(Storage, ClientId, AccIn) -> end. read_transferinfo(Storage, Transfer, Acc) -> - case list(Storage, Transfer, result) of - {ok, Result = [_ | _]} -> - Info = #{status => complete, result => Result}, - Acc#{Transfer => Info}; - {ok, []} -> - Info = #{status => incomplete}, - Acc#{Transfer => Info}; - {error, _Reason} -> - ?tp(warning, "list_result_failed", #{ + case read_filemeta(Storage, Transfer) of + {ok, Filemeta} -> + Acc#{Transfer => #{filemeta => Filemeta}}; + {error, enoent} -> + Acc#{Transfer => #{}}; + {error, Reason} -> + ?tp(warning, "read_transferinfo_failed", #{ storage => Storage, - transfer => Transfer + transfer => Transfer, + reason => Reason }), Acc end. @@ -365,7 +289,7 @@ read_transferinfo(Storage, Transfer, Acc) -> get_subdir(Storage, Transfer) -> mk_filedir(Storage, Transfer, []). --spec get_subdir(storage(), transfer(), fragment | temporary | result) -> +-spec get_subdir(storage(), transfer(), fragment | temporary) -> file:name(). get_subdir(Storage, Transfer, What) -> mk_filedir(Storage, Transfer, get_subdirs_for(What)). @@ -373,84 +297,12 @@ get_subdir(Storage, Transfer, What) -> get_subdirs_for(fragment) -> [?FRAGDIR]; get_subdirs_for(temporary) -> - [?TEMPDIR]; -get_subdirs_for(result) -> - [?RESULTDIR]. - -%% - --type handle() :: {file:name(), io:device(), crypto:hash_state()}. - --spec open_file(storage(), transfer(), filemeta()) -> - {ok, handle()} | {error, file_error()}. -open_file(Storage, Transfer, Filemeta) -> - Filename = maps:get(name, Filemeta), - TempFilepath = mk_temp_filepath(Storage, Transfer, Filename), - _ = filelib:ensure_dir(TempFilepath), - case file:open(TempFilepath, [write, raw, binary]) of - {ok, Handle} -> - % TODO: preserve filemeta - {ok, {TempFilepath, Handle, init_checksum(Filemeta)}}; - {error, _} = Error -> - Error - end. - --spec write(handle(), iodata()) -> - {ok, handle()} | {error, file_error()}. -write({Filepath, IoDevice, Ctx}, IoData) -> - case file:write(IoDevice, IoData) of - ok -> - {ok, {Filepath, IoDevice, update_checksum(Ctx, IoData)}}; - {error, _} = Error -> - Error - end. - --spec complete(storage(), transfer(), filemeta(), handle()) -> - ok | {error, {checksum, _Algo, _Computed}} | {error, file_error()}. -complete(Storage, Transfer, Filemeta = #{name := Filename}, Handle = {Filepath, IoDevice, Ctx}) -> - TargetFilepath = mk_filepath(Storage, Transfer, get_subdirs_for(result), Filename), - case verify_checksum(Ctx, Filemeta) of - ok -> - ok = file:close(IoDevice), - mv_temp_file(Filepath, TargetFilepath); - {error, _} = Error -> - _ = discard(Handle), - Error - end. - --spec discard(handle()) -> - ok. -discard({Filepath, IoDevice, _Ctx}) -> - ok = file:close(IoDevice), - file:delete(Filepath). - -init_checksum(#{checksum := {Algo, _}}) -> - crypto:hash_init(Algo); -init_checksum(#{}) -> - undefined. - -update_checksum(Ctx, IoData) when Ctx /= undefined -> - crypto:hash_update(Ctx, IoData); -update_checksum(undefined, _IoData) -> - undefined. - -verify_checksum(Ctx, #{checksum := {Algo, Digest}}) when Ctx /= undefined -> - case crypto:hash_final(Ctx) of - Digest -> - ok; - Mismatch -> - {error, {checksum, Algo, binary:encode_hex(Mismatch)}} - end; -verify_checksum(undefined, _) -> - ok. + [?TEMPDIR]. -define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). encode_filemeta(Meta) -> - % TODO: Looks like this should be hocon's responsibility. - Schema = emqx_ft_schema:schema(filemeta), - Term = hocon_tconf:make_serializable(Schema, emqx_map_lib:binary_key_map(Meta), #{}), - emqx_json:encode(?PRELUDE(_Vsn = 1, Term)). + emqx_json:encode(?PRELUDE(_Vsn = 1, emqx_ft:encode_filemeta(Meta))). decode_filemeta(Binary) when is_binary(Binary) -> ?PRELUDE(_Vsn = 1, Map) = emqx_json:decode(Binary, [return_maps]), @@ -490,33 +342,12 @@ try_list_dir(Dirname) -> end. get_storage_root(Storage) -> - maps:get(root, Storage, filename:join(emqx:data_dir(), "file_transfer")). + maps:get(root, Storage, filename:join([emqx:data_dir(), "ft", "transfers"])). -include_lib("kernel/include/file.hrl"). -read_file(Filepath) -> - file:read_file(Filepath). - read_file(Filepath, DecodeFun) -> - case read_file(Filepath) of - {ok, Content} -> - safe_decode(Content, DecodeFun); - {error, _} = Error -> - Error - end. - -safe_decode(Content, DecodeFun) -> - try - {ok, DecodeFun(Content)} - catch - C:E:Stacktrace -> - ?tp(warning, "safe_decode_failed", #{ - class => C, - exception => E, - stacktrace => Stacktrace - }), - {error, corrupted} - end. + emqx_ft_fs_util:read_decode_file(Filepath, DecodeFun). write_file_atomic(Storage, Transfer, Filepath, Content) when is_binary(Content) -> TempFilepath = mk_temp_filepath(Storage, Transfer, filename:basename(Filepath)), @@ -574,12 +405,6 @@ mk_filefrag(_Dirname, _Filename) -> }), false. -mk_result_filefrag(Dirname, Filename) -> - % NOTE - % Any file in the `?RESULTDIR` subdir is currently considered the result of - % the file transfer. - mk_filefrag(Dirname, Filename, result, fun(_, _) -> {ok, #{}} end). - mk_filefrag(Dirname, Filename, Tag, Fun) -> Filepath = filename:join(Dirname, Filename), % TODO error handling? diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 58c5dbfdf..20e4f468d 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -308,28 +308,18 @@ is_same_filepath(P1, P2) when is_binary(P1) -> filepath_to_binary(S) -> unicode:characters_to_binary(S, unicode, file:native_name_encoding()). -get_segments_ttl(Storage, Transfer) -> +get_segments_ttl(Storage, TransferInfo) -> {MinTTL, MaxTTL} = emqx_ft_conf:segments_ttl(Storage), - clamp(MinTTL, MaxTTL, try_get_filemeta_ttl(Storage, Transfer)). + clamp(MinTTL, MaxTTL, try_get_filemeta_ttl(TransferInfo)). -try_get_filemeta_ttl(Storage, Transfer) -> - case emqx_ft_storage_fs:read_filemeta(Storage, Transfer) of - {ok, Filemeta} -> - maps:get(segments_ttl, Filemeta, undefined); - {error, _} -> - undefined - end. +try_get_filemeta_ttl(#{filemeta := Filemeta}) -> + maps:get(segments_ttl, Filemeta, undefined); +try_get_filemeta_ttl(#{}) -> + undefined. clamp(Min, Max, V) -> min(Max, max(Min, V)). -% try_collect(_Subject, ok = Result, Then, _Stats) -> -% Then(Result); -% try_collect(_Subject, {ok, Result}, Then, _Stats) -> -% Then(Result); -% try_collect(Subject, {error, _} = Error, _Then, Stats) -> -% register_gcstat_error(Subject, Error, Stats). - %% init_gcstats() -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl index 7e19dd322..dbd0cd6dc 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl @@ -23,8 +23,8 @@ -export([ list_local/2, pread_local/4, - get_ready_transfer_local/2, - ready_transfers_local/0 + list_exports_local/0, + read_export_file_local/2 ]). list_local(Transfer, What) -> @@ -33,8 +33,18 @@ list_local(Transfer, What) -> pread_local(Transfer, Frag, Offset, Size) -> emqx_ft_storage:with_storage_type(local, pread, [Transfer, Frag, Offset, Size]). -get_ready_transfer_local(CallerPid, Transfer) -> - emqx_ft_storage:with_storage_type(local, get_ready_transfer_local, [CallerPid, Transfer]). +list_exports_local() -> + case emqx_ft_storage:with_storage_type(local, exporter, []) of + {emqx_ft_storage_exporter_fs, Options} -> + emqx_ft_storage_exporter_fs:list_local(Options); + InvalidExporter -> + {error, {invalid_exporter, InvalidExporter}} + end. -ready_transfers_local() -> - emqx_ft_storage:with_storage_type(local, ready_transfers_local, []). +read_export_file_local(Filepath, CallerPid) -> + case emqx_ft_storage:with_storage_type(local, exporter, []) of + {emqx_ft_storage_exporter_fs, Options} -> + emqx_ft_storage_exporter_fs:start_reader(Options, Filepath, CallerPid); + InvalidExporter -> + {error, {invalid_exporter, InvalidExporter}} + end. diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index e2c4c93d7..f152928fe 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -22,8 +22,10 @@ -export([multilist/3]). -export([pread/5]). --export([ready_transfers/1]). --export([get_ready_transfer/3]). + +%% TODO: These should be defined in a separate BPAPI +-export([list_exports/1]). +-export([read_export_file/3]). -type offset() :: emqx_ft:offset(). -type transfer() :: emqx_ft:transfer(). @@ -44,19 +46,16 @@ multilist(Nodes, Transfer, What) -> pread(Node, Transfer, Frag, Offset, Size) -> erpc:call(Node, emqx_ft_storage_fs_proxy, pread_local, [Transfer, Frag, Offset, Size]). --spec ready_transfers([node()]) -> - [ - {ok, [{emqx_ft_storage:ready_transfer_id(), emqx_ft_storage:ready_transfer_info()}]} - | {error, term()} - | {exit, term()} - | {throw, term()} - ]. -ready_transfers(Nodes) -> - erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, ready_transfers_local, []). +%% --spec get_ready_transfer(node(), pid(), emqx_ft_storage:ready_transfer_id()) -> - {ok, emqx_ft_storage:ready_transfer_data()} +-spec list_exports([node()]) -> + emqx_rpc:erpc_multicall([emqx_ft_storage:export_info()]). +list_exports(Nodes) -> + erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, list_exports_local, []). + +-spec read_export_file(node(), file:name(), pid()) -> + {ok, emqx_ft_storage:export_data()} | {error, term()} | no_return(). -get_ready_transfer(Node, CallerPid, ReadyTransferId) -> - erpc:call(Node, emqx_ft_storage_fs_proxy, get_ready_transfer_local, [CallerPid, ReadyTransferId]). +read_export_file(Node, Filepath, CallerPid) -> + erpc:call(Node, emqx_ft_storage_fs_proxy, read_export_file_local, [Filepath, CallerPid]). diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 9c9087732..5fb384c16 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -58,8 +58,9 @@ end_per_suite(_Config) -> set_special_configs(Config) -> fun (emqx_ft) -> - Root = emqx_ft_test_helpers:ft_root(Config, node()), - emqx_ft_test_helpers:load_config(#{storage => #{type => local, root => Root}}); + emqx_ft_test_helpers:load_config(#{ + storage => emqx_ft_test_helpers:local_storage(Config) + }); (_) -> ok end. @@ -108,8 +109,9 @@ mk_cluster_specs(Config) -> {conf, [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]]}, {env_handler, fun (emqx_ft) -> - Root = emqx_ft_test_helpers:ft_root(Config, node()), - emqx_ft_test_helpers:load_config(#{storage => #{type => local, root => Root}}); + emqx_ft_test_helpers:load_config(#{ + storage => emqx_ft_test_helpers:local_storage(Config) + }); (_) -> ok end} @@ -194,11 +196,10 @@ t_simple_transfer(Config) -> emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) ), - [ReadyTransferId] = list_ready_transfers(?config(clientid, Config)), - {ok, TableQH} = emqx_ft_storage:get_ready_transfer(ReadyTransferId), + [Export] = list_exports(?config(clientid, Config)), ?assertEqual( - iolist_to_binary(Data), - iolist_to_binary(qlc:eval(TableQH)) + {ok, iolist_to_binary(Data)}, + read_export(Export) ). t_meta_conflict(Config) -> @@ -424,11 +425,10 @@ t_switch_node(Config) -> %% Now check consistency of the file - [ReadyTransferId] = list_ready_transfers(ClientId), - {ok, TableQH} = emqx_ft_storage:get_ready_transfer(ReadyTransferId), + [Export] = list_exports(ClientId), ?assertEqual( - iolist_to_binary(Data), - iolist_to_binary(qlc:eval(TableQH)) + {ok, iolist_to_binary(Data)}, + read_export(Export) ). t_assemble_crash(Config) -> @@ -501,27 +501,30 @@ t_unreliable_migrating_client(Config) -> ], _Context = run_commands(Commands, Context), - ReadyTransferIds = list_ready_transfers(?config(clientid, Config)), + Exports = list_exports(?config(clientid, Config)), % NOTE % The cluster had 2 assemblers running on two different nodes, because client sent `fin` % twice. This is currently expected, files must be identical anyway. - Node1Bin = atom_to_binary(Node1), - NodeSelfBin = atom_to_binary(NodeSelf), + Node1Str = atom_to_list(Node1), + NodeSelfStr = atom_to_list(NodeSelf), ?assertMatch( - [#{<<"node">> := Node1Bin}, #{<<"node">> := NodeSelfBin}], - lists:sort(ReadyTransferIds) + [#{"node" := Node1Str}, #{"node" := NodeSelfStr}], + lists:map( + fun(#{uri := URIString}) -> + #{query := QS} = uri_string:parse(URIString), + uri_string:dissect_query(QS) + end, + lists:sort(Exports) + ) ), [ - begin - {ok, TableQH} = emqx_ft_storage:get_ready_transfer(Id), - ?assertEqual( - Payload, - iolist_to_binary(qlc:eval(TableQH)) - ) - end - || Id <- ReadyTransferIds + ?assertEqual( + {ok, Payload}, + read_export(Export) + ) + || Export <- Exports ]. run_commands(Commands, Context) -> @@ -620,10 +623,10 @@ meta(FileName, Data) -> size => byte_size(FullData) }. -list_ready_transfers(ClientId) -> - {ok, ReadyTransfers} = emqx_ft_storage:ready_transfers(), - [ - Id - || {#{<<"clientid">> := CId} = Id, _Info} <- ReadyTransfers, - CId == ClientId - ]. +list_exports(ClientId) -> + {ok, Exports} = emqx_ft_storage:exports(), + [Export || Export = #{transfer := {CId, _}} <- Exports, CId == ClientId]. + +read_export(#{path := AbsFilepath}) -> + % TODO: only works for the local filesystem exporter right now + file:read_file(AbsFilepath). diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index 7b191e229..aff4d864c 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -24,7 +24,7 @@ -include_lib("emqx/include/asserts.hrl"). --import(emqx_mgmt_api_test_util, [request/3, uri/1]). +-import(emqx_mgmt_api_test_util, [uri/1]). all() -> emqx_common_test_helpers:all(?MODULE). @@ -41,8 +41,9 @@ end_per_suite(_Config) -> set_special_configs(Config) -> fun (emqx_ft) -> - Root = emqx_ft_test_helpers:ft_root(Config, node()), - emqx_ft_test_helpers:load_config(#{storage => #{type => local, root => Root}}); + emqx_ft_test_helpers:load_config(#{ + storage => emqx_ft_test_helpers:local_storage(Config) + }); (_) -> ok end. @@ -61,40 +62,25 @@ t_list_ready_transfers(Config) -> ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, <<"data">>, node()), - {ok, 200, Response} = request(get, uri(["file_transfer", "files"])), - - #{<<"files">> := Files} = emqx_json:decode(Response, [return_maps]), + {ok, 200, #{<<"files">> := Files}} = + request(get, uri(["file_transfer", "files"]), fun json/1), ?assertInclude( - #{<<"id">> := #{<<"clientid">> := ClientId, <<"fileid">> := <<"f1">>}}, + #{<<"clientid">> := ClientId, <<"fileid">> := <<"f1">>}, Files ). -%% This shouldn't happen in real life -%% but we need to test it anyway -t_list_ready_transfers_no_nodes(_Config) -> - _ = meck:new(mria_mnesia, [passthrough]), - _ = meck:expect(mria_mnesia, running_nodes, fun() -> [] end), - - ?assertMatch( - {ok, 503, _}, - request(get, uri(["file_transfer", "files"])) - ). - t_download_transfer(Config) -> ClientId = client_id(Config), ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, <<"data">>, node()), ?assertMatch( - {ok, 503, _}, + {ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}}, request( get, - uri(["file_transfer", "file"]) ++ - query(#{ - clientid => ClientId, - fileid => <<"f1">> - }) + uri(["file_transfer", "file"]) ++ query(#{fileref => <<"f1">>}), + fun json/1 ) ), @@ -104,8 +90,7 @@ t_download_transfer(Config) -> get, uri(["file_transfer", "file"]) ++ query(#{ - clientid => ClientId, - fileid => <<"f1">>, + fileref => <<"f1">>, node => <<"nonode@nohost">> }) ) @@ -117,22 +102,16 @@ t_download_transfer(Config) -> get, uri(["file_transfer", "file"]) ++ query(#{ - clientid => ClientId, - fileid => <<"unknown_file">>, + fileref => <<"unknown_file">>, node => node() }) ) ), - {ok, 200, Response} = request( - get, - uri(["file_transfer", "file"]) ++ - query(#{ - clientid => ClientId, - fileid => <<"f1">>, - node => node() - }) - ), + {ok, 200, #{<<"files">> := [File]}} = + request(get, uri(["file_transfer", "files"]), fun json/1), + + {ok, 200, Response} = request(get, uri([]) ++ maps:get(<<"uri">>, File)), ?assertEqual( <<"data">>, @@ -147,7 +126,18 @@ client_id(Config) -> atom_to_binary(?config(tc, Config), utf8). request(Method, Url) -> - request(Method, Url, []). + emqx_mgmt_api_test_util:request(Method, Url, []). + +request(Method, Url, Decoder) when is_function(Decoder) -> + case emqx_mgmt_api_test_util:request(Method, Url, []) of + {ok, Code, Body} -> + {ok, Code, Decoder(Body)}; + Otherwise -> + Otherwise + end. + +json(Body) when is_binary(Body) -> + emqx_json:decode(Body, [return_maps]). query(Params) -> KVs = lists:map(fun({K, V}) -> uri_encode(K) ++ "=" ++ uri_encode(V) end, maps:to_list(Params)), diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 8df471deb..4c7c0f08c 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -47,6 +47,7 @@ init_per_testcase(TC, Config) -> {ok, Pid} = emqx_ft_assembler_sup:start_link(), [ {storage_root, "file_transfer_root"}, + {exports_root, "file_transfer_exports"}, {file_id, atom_to_binary(TC)}, {assembler_sup, Pid} | Config @@ -85,11 +86,12 @@ t_assemble_empty_transfer(Config) -> ), Status = complete_assemble(Storage, Transfer, 0), ?assertEqual({shutdown, ok}, Status), - {ok, [Result = #{size := Size = 0}]} = emqx_ft_storage_fs:list(Storage, Transfer, result), - ?assertEqual( - {error, eof}, - emqx_ft_storage_fs:pread(Storage, Transfer, Result, 0, Size) - ), + {ok, [_Result = #{size := _Size = 0}]} = + emqx_ft_storage_fs:exports_local(Storage, Transfer), + % ?assertEqual( + % {error, eof}, + % emqx_ft_storage_fs:pread(Storage, Transfer, Result, 0, Size) + % ), ok. t_assemble_complete_local_transfer(Config) -> @@ -133,12 +135,13 @@ t_assemble_complete_local_transfer(Config) -> {ok, [ #{ size := TransferSize, - fragment := {result, #{}} + meta := #{} } ]}, - emqx_ft_storage_fs:list(Storage, Transfer, result) + emqx_ft_storage_fs:exports_local(Storage, Transfer) ), - {ok, [#{path := AssemblyFilename}]} = emqx_ft_storage_fs:list(Storage, Transfer, result), + {ok, [#{path := AssemblyFilename}]} = + emqx_ft_storage_fs:exports_local(Storage, Transfer), ?assertMatch( {ok, #file_info{type = regular, size = TransferSize}}, file:read_file_info(AssemblyFilename) @@ -191,18 +194,23 @@ complete_assemble(Storage, Transfer, Size, Timeout) -> t_list_transfers(Config) -> Storage = storage(Config), + {ok, Exports} = emqx_ft_storage_fs:exports_local(Storage), ?assertMatch( - {ok, #{ - {?CLIENTID1, <<"t_assemble_empty_transfer">>} := #{ - status := complete, - result := [#{path := _, size := 0, fragment := {result, _}}] + [ + #{ + transfer := {?CLIENTID2, <<"t_assemble_complete_local_transfer">>}, + path := _, + size := Size, + meta := #{name := "topsecret.pdf"} }, - {?CLIENTID2, <<"t_assemble_complete_local_transfer">>} := #{ - status := complete, - result := [#{path := _, size := Size, fragment := {result, _}}] + #{ + transfer := {?CLIENTID1, <<"t_assemble_empty_transfer">>}, + path := _, + size := 0, + meta := #{name := "important.pdf"} } - }} when Size > 0, - emqx_ft_storage_fs:transfers(Storage) + ] when Size > 0, + lists:sort(Exports) ). %% @@ -232,5 +240,9 @@ mk_fileid() -> storage(Config) -> #{ type => local, - root => ?config(storage_root, Config) + root => ?config(storage_root, Config), + exporter => #{ + type => local, + root => ?config(exports_root, Config) + } }. diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index 314b4a5f2..e43abf095 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -56,7 +56,16 @@ t_update_config(_Config) -> {ok, _}, emqx_conf:update( [file_transfer], - #{<<"storage">> => #{<<"type">> => <<"local">>, <<"root">> => <<"/tmp/path">>}}, + #{ + <<"storage">> => #{ + <<"type">> => <<"local">>, + <<"root">> => <<"/tmp/path">>, + <<"exporter">> => #{ + <<"type">> => <<"local">>, + <<"root">> => <<"/tmp/exports">> + } + } + }, #{} ) ), diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index 5551cce27..5b3f56b7d 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -22,11 +22,8 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). --include_lib("emqx/include/asserts.hrl"). - all() -> [ - {group, single_node}, {group, cluster} ]. @@ -34,7 +31,6 @@ all() -> groups() -> [ - {single_node, [sequence], emqx_common_test_helpers:all(?MODULE) -- ?CLUSTER_CASES}, {cluster, [sequence], ?CLUSTER_CASES} ]. @@ -48,8 +44,9 @@ end_per_suite(_Config) -> set_special_configs(Config) -> fun (emqx_ft) -> - Root = emqx_ft_test_helpers:ft_root(Config, node()), - emqx_ft_test_helpers:load_config(#{storage => #{type => local, root => Root}}); + emqx_ft_test_helpers:load_config(#{ + storage => emqx_ft_test_helpers:local_storage(Config) + }); (_) -> ok end. @@ -74,47 +71,19 @@ end_per_group(_Group, _Config) -> %% Tests %%-------------------------------------------------------------------- -t_invalid_ready_transfer_id(Config) -> - ?assertMatch( - {error, _}, - emqx_ft_storage_fs:get_ready_transfer(storage(Config), #{ - <<"clientid">> => client_id(Config), - <<"fileid">> => <<"fileid">>, - <<"node">> => atom_to_binary('nonexistent@127.0.0.1') - }) - ), - ?assertMatch( - {error, _}, - emqx_ft_storage_fs:get_ready_transfer(storage(Config), #{ - <<"clientid">> => client_id(Config), - <<"fileid">> => <<"fileid">>, - <<"node">> => <<"nonexistent_as_atom@127.0.0.1">> - }) - ), - ?assertMatch( - {error, _}, - emqx_ft_storage_fs:get_ready_transfer(storage(Config), #{ - <<"clientid">> => client_id(Config), - <<"fileid">> => <<"nonexistent_file">>, - <<"node">> => node() - }) - ). - t_multinode_ready_transfers(Config) -> Node1 = ?config(additional_node, Config), - ok = emqx_ft_test_helpers:upload_file(<<"c1">>, <<"f1">>, <<"data">>, Node1), + ok = emqx_ft_test_helpers:upload_file(<<"c/1">>, <<"f:1">>, "fn1", <<"data">>, Node1), Node2 = node(), - ok = emqx_ft_test_helpers:upload_file(<<"c2">>, <<"f2">>, <<"data">>, Node2), + ok = emqx_ft_test_helpers:upload_file(<<"c/2">>, <<"f:2">>, "fn2", <<"data">>, Node2), - ?assertInclude( - #{<<"clientid">> := <<"c1">>, <<"fileid">> := <<"f1">>}, - ready_transfer_ids(Config) - ), - - ?assertInclude( - #{<<"clientid">> := <<"c2">>, <<"fileid">> := <<"f2">>}, - ready_transfer_ids(Config) + ?assertMatch( + [ + #{transfer := {<<"c/1">>, <<"f:1">>}, name := "fn1"}, + #{transfer := {<<"c/2">>, <<"f:2">>}, name := "fn2"} + ], + lists:sort(list_exports(Config)) ). %%-------------------------------------------------------------------- @@ -127,13 +96,13 @@ client_id(Config) -> storage(Config) -> #{ type => local, - root => ft_root(Config) + root => emqx_ft_test_helpers:root(Config, node(), ["transfers"]), + exporter => #{ + type => local, + root => emqx_ft_test_helpers:root(Config, node(), ["exports"]) + } }. -ft_root(Config) -> - emqx_ft_test_helpers:ft_root(Config, node()). - -ready_transfer_ids(Config) -> - {ok, ReadyTransfers} = emqx_ft_storage_fs:ready_transfers(storage(Config)), - {ReadyTransferIds, _} = lists:unzip(ReadyTransfers), - ReadyTransferIds. +list_exports(Config) -> + {ok, Exports} = emqx_ft_storage_fs:exports(storage(Config)), + Exports. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index 89e9eb970..25cd200f1 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -41,7 +41,14 @@ init_per_testcase(TC, Config) -> emqx_ft, fun(emqx_ft) -> emqx_ft_test_helpers:load_config(#{ - storage => #{type => local, root => mk_root(TC, Config)} + storage => #{ + type => local, + root => emqx_ft_test_helpers:root(Config, node(), [TC, transfers]), + exporter => #{ + type => local, + root => emqx_ft_test_helpers:root(Config, node(), [TC, exports]) + } + } }) end ), @@ -53,9 +60,6 @@ end_per_testcase(_TC, _Config) -> ok = application:stop(emqx_ft), ok. -mk_root(TC, Config) -> - filename:join([?config(priv_dir, Config), "file_transfer", TC, atom_to_list(node())]). - %% -define(NSEGS(Filesize, SegmentSize), (ceil(Filesize / SegmentSize) + 1)). diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index b756f8034..e62c74d81 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -30,7 +30,7 @@ start_additional_node(Config, Name) -> {configure_gen_rpc, true}, {env_handler, fun (emqx_ft) -> - load_config(#{storage => #{type => local, root => ft_root(Config, node())}}); + load_config(#{storage => local_storage(Config)}); (_) -> ok end} @@ -43,6 +43,13 @@ stop_additional_node(Node) -> ok = emqx_common_test_helpers:stop_slave(Node), ok. +local_storage(Config) -> + #{ + type => local, + root => root(Config, node(), [transfers]), + exporter => #{type => local, root => root(Config, node(), [exports])} + }. + load_config(Config) -> emqx_common_test_helpers:load_config(emqx_ft_schema, #{file_transfer => Config}). @@ -50,10 +57,8 @@ tcp_port(Node) -> {_, Port} = rpc:call(Node, emqx_config, get, [[listeners, tcp, default, bind]]), Port. -ft_root(Config, Node) -> - filename:join([ - ?config(priv_dir, Config), <<"file_transfer">>, atom_to_binary(Node) - ]). +root(Config, Node, Tail) -> + filename:join([?config(priv_dir, Config), "file_transfer", Node | Tail]). upload_file(ClientId, FileId, Data, Node) -> Port = tcp_port(Node), From 9aec01e7a3283b8b0021ce217e891e40d321809d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 23 Mar 2023 14:17:56 +0300 Subject: [PATCH 072/156] fix(ft-asm): use regular map for meta fragments There's actually no need to use an ordered data structure at all. --- apps/emqx_ft/src/emqx_ft_assembly.erl | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index d0998d6ec..dead1132e 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -37,10 +37,7 @@ status :: status(), coverage :: coverage() | undefined, properties :: properties() | undefined, - meta :: orddict:orddict( - filemeta(), - {node(), filefrag({filemeta, filemeta()})} - ), + meta :: #{filemeta() => {node(), filefrag({filemeta, filemeta()})}}, segs :: emqx_wdgraph:t(emqx_ft:offset(), {node(), filefrag({segment, segmentinfo()})}), size :: emqx_ft:bytes() }). @@ -63,7 +60,7 @@ new(Size) -> #asm{ status = {incomplete, {missing, filemeta}}, - meta = orddict:new(), + meta = #{}, segs = emqx_wdgraph:new(), size = Size }. @@ -111,7 +108,7 @@ coverage(#asm{coverage = Coverage}) -> -spec nodes(t()) -> [node()]. nodes(#asm{meta = Meta, segs = Segs}) -> - S1 = orddict:fold( + S1 = maps:fold( fun(_Meta, {Node, _Fragment}, Acc) -> ordsets:add_element(Node, Acc) end, @@ -131,7 +128,7 @@ properties(#asm{properties = Properties}) -> Properties. status(meta, #asm{meta = Meta}) -> - status(meta, orddict:to_list(Meta)); + status(meta, maps:to_list(Meta)); status(meta, [{Meta, {_Node, _Frag}}]) -> {complete, Meta}; status(meta, []) -> @@ -150,7 +147,7 @@ status(coverage, #asm{segs = Segments, size = Size}) -> append_filemeta(Asm, Node, Fragment = #{fragment := {filemeta, Meta}}) -> Asm#asm{ - meta = orddict:store(Meta, {Node, Fragment}, Asm#asm.meta) + meta = maps:put(Meta, {Node, Fragment}, Asm#asm.meta) }. append_segmentinfo(Asm, _Node, #{fragment := {segment, #{size := 0}}}) -> From 2707b4500fbe705ec6835699c8d54d4b3f9aceef Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 24 Mar 2023 00:16:17 +0300 Subject: [PATCH 073/156] feat(ft-fs): ensure filsystem-safe handling of client input Also restrict the filenames clients may specify in a filemeta to some safe subset, as a future proofing measure. --- apps/emqx_ft/src/emqx_ft_fs_util.erl | 84 +++++++++++++++++++ apps/emqx_ft/src/emqx_ft_schema.erl | 10 ++- .../src/emqx_ft_storage_exporter_fs.erl | 11 ++- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 18 ++-- apps/emqx_ft/test/emqx_ft_SUITE.erl | 41 +++++++++ apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 4 +- apps/emqx_ft/test/emqx_ft_fs_util_tests.erl | 65 ++++++++++++++ apps/emqx_ft/test/emqx_ft_test_helpers.erl | 7 +- 8 files changed, 225 insertions(+), 15 deletions(-) create mode 100644 apps/emqx_ft/test/emqx_ft_fs_util_tests.erl diff --git a/apps/emqx_ft/src/emqx_ft_fs_util.erl b/apps/emqx_ft/src/emqx_ft_fs_util.erl index 198f4ccc5..75fb47c27 100644 --- a/apps/emqx_ft/src/emqx_ft_fs_util.erl +++ b/apps/emqx_ft/src/emqx_ft_fs_util.erl @@ -18,6 +18,10 @@ -include_lib("snabbkaffe/include/trace.hrl"). +-export([is_filename_safe/1]). +-export([escape_filename/1]). +-export([unescape_filename/1]). + -export([read_decode_file/2]). -export([fold/4]). @@ -35,6 +39,86 @@ ) -> Acc ). +-define(IS_UNSAFE(C), + ((C) =:= $% orelse + (C) =:= $: orelse + (C) =:= $\\ orelse + (C) =:= $/) +). + +-define(IS_PRINTABLE(C), + % NOTE: See `io_lib:printable_unicode_list/1` + (((C) >= 32 andalso (C) =< 126) orelse + ((C) >= 16#A0 andalso (C) < 16#D800) orelse + ((C) > 16#DFFF andalso (C) < 16#FFFE) orelse + ((C) > 16#FFFF andalso (C) =< 16#10FFFF)) +). + +%% + +-spec is_filename_safe(file:filename_all()) -> ok | {error, atom()}. +is_filename_safe(FN) when is_binary(FN) -> + is_filename_safe(unicode:characters_to_list(FN)); +is_filename_safe("") -> + {error, empty}; +is_filename_safe(FN) when FN == "." orelse FN == ".." -> + {error, special}; +is_filename_safe(FN) -> + verify_filename_safe(FN). + +verify_filename_safe([$% | Rest]) -> + verify_filename_safe(Rest); +verify_filename_safe([C | _]) when ?IS_UNSAFE(C) -> + {error, unsafe}; +verify_filename_safe([C | _]) when not ?IS_PRINTABLE(C) -> + {error, nonprintable}; +verify_filename_safe([_ | Rest]) -> + verify_filename_safe(Rest); +verify_filename_safe([]) -> + ok. + +-spec escape_filename(binary()) -> file:name(). +escape_filename(Name) when Name == <<".">> orelse Name == <<"..">> -> + lists:reverse(percent_encode(Name, "")); +escape_filename(Name) -> + escape(Name, ""). + +escape(<>, Acc) when ?IS_UNSAFE(C) -> + escape(Rest, percent_encode(<>, Acc)); +escape(<>, Acc) when not ?IS_PRINTABLE(C) -> + escape(Rest, percent_encode(<>, Acc)); +escape(<>, Acc) -> + escape(Rest, [C | Acc]); +escape(<<>>, Acc) -> + lists:reverse(Acc). + +-spec unescape_filename(file:name()) -> binary(). +unescape_filename(Name) -> + unescape(Name, <<>>). + +unescape([$%, A, B | Rest], Acc) -> + unescape(Rest, percent_decode(A, B, Acc)); +unescape([C | Rest], Acc) -> + unescape(Rest, <>); +unescape([], Acc) -> + Acc. + +percent_encode(<>, Acc) -> + percent_encode(Rest, [dec2hex(B), dec2hex(A), $% | Acc]); +percent_encode(<<>>, Acc) -> + Acc. + +percent_decode(A, B, Acc) -> + <>. + +dec2hex(X) when (X >= 0) andalso (X =< 9) -> X + $0; +dec2hex(X) when (X >= 10) andalso (X =< 15) -> X + $A - 10. + +hex2dec(X) when (X >= $0) andalso (X =< $9) -> X - $0; +hex2dec(X) when (X >= $A) andalso (X =< $F) -> X - $A + 10; +hex2dec(X) when (X >= $a) andalso (X =< $f) -> X - $a + 10; +hex2dec(_) -> error(badarg). + %% -spec read_decode_file(file:name(), fun((binary()) -> Value)) -> diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 37e2adafc..5c0a764a2 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -130,8 +130,11 @@ desc(local_storage_gc) -> schema(filemeta) -> #{ roots => [ - % TODO nonempty - {name, hoconsc:mk(string(), #{required => true})}, + {name, + hoconsc:mk(string(), #{ + required => true, + validator => validator(filename) + })}, {size, hoconsc:mk(non_neg_integer())}, {expire_at, hoconsc:mk(non_neg_integer())}, {checksum, hoconsc:mk({atom(), binary()}, #{converter => converter(checksum)})}, @@ -140,6 +143,9 @@ schema(filemeta) -> ] }. +validator(filename) -> + fun emqx_ft_fs_util:is_filename_safe/1. + converter(checksum) -> fun (undefined, #{}) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index e0bffd444..6456e2ba5 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -343,7 +343,16 @@ mk_result_reldir(Transfer = {ClientId, FileId}) -> Bucket2:?BUCKET2_LEN/binary, BucketRest/binary >> = binary:encode_hex(Hash), - [Bucket1, Bucket2, BucketRest, ClientId, FileId]. + [ + Bucket1, + Bucket2, + BucketRest, + emqx_ft_fs_util:escape_filename(ClientId), + emqx_ft_fs_util:escape_filename(FileId) + ]. + +dirnames_to_transfer(ClientId, FileId) -> + {emqx_ft_fs_util:unescape_filename(ClientId), emqx_ft_fs_util:unescape_filename(FileId)}. mk_transfer_hash(Transfer) -> crypto:hash(?BUCKET_HASH, term_to_binary(Transfer)). diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 4a613dbcf..6c5b6962c 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -250,12 +250,12 @@ transfers(Storage) -> )}. transfers(Storage, ClientId, AccIn) -> - Dirname = mk_client_filedir(Storage, ClientId), + Dirname = filename:join(get_storage_root(Storage), ClientId), case file:list_dir(Dirname) of {ok, FileIds} -> lists:foldl( fun(FileId, Acc) -> - Transfer = {filename_to_binary(ClientId), filename_to_binary(FileId)}, + Transfer = dirnames_to_transfer(ClientId, FileId), read_transferinfo(Storage, Transfer, Acc) end, AccIn, @@ -327,10 +327,15 @@ break_segment_filename(Filename) -> end. mk_filedir(Storage, {ClientId, FileId}, SubDirs) -> - filename:join([get_storage_root(Storage), ClientId, FileId | SubDirs]). + filename:join([ + get_storage_root(Storage), + emqx_ft_fs_util:escape_filename(ClientId), + emqx_ft_fs_util:escape_filename(FileId) + | SubDirs + ]). -mk_client_filedir(Storage, ClientId) -> - filename:join([get_storage_root(Storage), ClientId]). +dirnames_to_transfer(ClientId, FileId) -> + {emqx_ft_fs_util:unescape_filename(ClientId), emqx_ft_fs_util:unescape_filename(FileId)}. mk_filepath(Storage, Transfer, SubDirs, Filename) -> filename:join(mk_filedir(Storage, Transfer, SubDirs), Filename). @@ -432,6 +437,3 @@ read_frag_filemeta(_Filename, Filepath) -> read_frag_segmentinfo(Filename, _Filepath) -> break_segment_filename(Filename). - -filename_to_binary(S) when is_list(S) -> unicode:characters_to_binary(S); -filename_to_binary(B) when is_binary(B) -> B. diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 5fb384c16..1a886c00d 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -165,6 +165,29 @@ t_invalid_fileid(Config) -> emqtt:publish(C, <<"$file//init">>, <<>>, 1) ). +t_invalid_filename(Config) -> + C = ?config(client, Config), + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_init_topic(<<"f1">>), emqx_json:encode(meta(".", <<>>)), 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_init_topic(<<"f2">>), emqx_json:encode(meta("..", <<>>)), 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_init_topic(<<"f2">>), emqx_json:encode(meta("../nice", <<>>)), 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_init_topic(<<"f3">>), emqx_json:encode(meta("/etc/passwd", <<>>)), 1) + ), + ?assertRCName( + success, + emqtt:publish(C, mk_init_topic(<<"f4">>), emqx_json:encode(meta("146%", <<>>)), 1) + ). + t_simple_transfer(Config) -> C = ?config(client, Config), @@ -202,6 +225,24 @@ t_simple_transfer(Config) -> read_export(Export) ). +t_nasty_clientids_fileids(_Config) -> + Transfers = [ + {<<".">>, <<".">>}, + {<<"🌚"/utf8>>, <<"🌝"/utf8>>}, + {<<"../..">>, <<"😤"/utf8>>}, + {<<"/etc/passwd">>, <<"whitehat">>}, + {<<"; rm -rf / ;">>, <<"whitehat">>} + ], + + ok = lists:foreach( + fun({ClientId, FileId}) -> + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "justfile", ClientId), + [Export] = list_exports(ClientId), + ?assertEqual({ok, ClientId}, read_export(Export)) + end, + Transfers + ). + t_meta_conflict(Config) -> C = ?config(client, Config), diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index aff4d864c..56622f3cd 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -60,7 +60,7 @@ end_per_testcase(_Case, _Config) -> t_list_ready_transfers(Config) -> ClientId = client_id(Config), - ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, <<"data">>, node()), + ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, "f1", <<"data">>), {ok, 200, #{<<"files">> := Files}} = request(get, uri(["file_transfer", "files"]), fun json/1), @@ -73,7 +73,7 @@ t_list_ready_transfers(Config) -> t_download_transfer(Config) -> ClientId = client_id(Config), - ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, <<"data">>, node()), + ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, "f1", <<"data">>), ?assertMatch( {ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}}, diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_tests.erl b/apps/emqx_ft/test/emqx_ft_fs_util_tests.erl new file mode 100644 index 000000000..1939e74c6 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_tests.erl @@ -0,0 +1,65 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_fs_util_tests). + +-include_lib("eunit/include/eunit.hrl"). + +filename_safe_test_() -> + [ + ?_assertEqual(ok, emqx_ft_fs_util:is_filename_safe("im.safe")), + ?_assertEqual(ok, emqx_ft_fs_util:is_filename_safe(<<"im.safe">>)), + ?_assertEqual(ok, emqx_ft_fs_util:is_filename_safe(<<".safe.100%">>)), + ?_assertEqual(ok, emqx_ft_fs_util:is_filename_safe(<<"safe.as.🦺"/utf8>>)) + ]. + +filename_unsafe_test_() -> + [ + ?_assertEqual({error, empty}, emqx_ft_fs_util:is_filename_safe("")), + ?_assertEqual({error, special}, emqx_ft_fs_util:is_filename_safe(".")), + ?_assertEqual({error, special}, emqx_ft_fs_util:is_filename_safe("..")), + ?_assertEqual({error, special}, emqx_ft_fs_util:is_filename_safe(<<"..">>)), + ?_assertEqual({error, unsafe}, emqx_ft_fs_util:is_filename_safe(<<".././..">>)), + ?_assertEqual({error, unsafe}, emqx_ft_fs_util:is_filename_safe("/etc/passwd")), + ?_assertEqual({error, unsafe}, emqx_ft_fs_util:is_filename_safe("../cookie")), + ?_assertEqual({error, unsafe}, emqx_ft_fs_util:is_filename_safe("C:$cookie")), + ?_assertEqual({error, nonprintable}, emqx_ft_fs_util:is_filename_safe([1, 2, 3])), + ?_assertEqual({error, nonprintable}, emqx_ft_fs_util:is_filename_safe(<<4, 5, 6>>)), + ?_assertEqual({error, nonprintable}, emqx_ft_fs_util:is_filename_safe([$a, 16#7F, $z])) + ]. + +-define(NAMES, [ + {"just.file", <<"just.file">>}, + {".hidden", <<".hidden">>}, + {".~what", <<".~what">>}, + {"100%25.file", <<"100%.file">>}, + {"%2E%2E", <<"..">>}, + {"...", <<"...">>}, + {"%2Fetc%2Fpasswd", <<"/etc/passwd">>}, + {"%01%02%0A ", <<1, 2, 10, 32>>} +]). + +escape_filename_test_() -> + [ + ?_assertEqual(Filename, emqx_ft_fs_util:escape_filename(Input)) + || {Filename, Input} <- ?NAMES + ]. + +unescape_filename_test_() -> + [ + ?_assertEqual(Input, emqx_ft_fs_util:unescape_filename(Filename)) + || {Filename, Input} <- ?NAMES + ]. diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index e62c74d81..a38242629 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -60,14 +60,17 @@ tcp_port(Node) -> root(Config, Node, Tail) -> filename:join([?config(priv_dir, Config), "file_transfer", Node | Tail]). -upload_file(ClientId, FileId, Data, Node) -> +upload_file(ClientId, FileId, Name, Data) -> + upload_file(ClientId, FileId, Name, Data, node()). + +upload_file(ClientId, FileId, Name, Data, Node) -> Port = tcp_port(Node), Size = byte_size(Data), {ok, C1} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, Port}]), {ok, _} = emqtt:connect(C1), Meta = #{ - name => FileId, + name => unicode:characters_to_binary(Name), expire_at => erlang:system_time(_Unit = second) + 3600, size => Size }, From 64f15f1fdbeef96573c3e6370d26e2c7f6a7fc17 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 27 Mar 2023 20:42:19 +0300 Subject: [PATCH 074/156] fix(ft-gc): ensure directories of complete transfers are GCed --- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 62 ++++++++++++------- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 19 +++++- 2 files changed, 57 insertions(+), 24 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 20e4f468d..eb274a4d5 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -169,7 +169,7 @@ try_collect_transfer(Storage, Transfer, TransferInfo = #{}, Stats) -> % TODO: collect empty directories separately case FragCleaned and TempCleaned of true -> - collect_transfer_directory(Storage, Transfer, Stats2); + collect_transfer_directory(Storage, Transfer, Cutoff, Stats2); false -> Stats2 end; @@ -191,18 +191,32 @@ collect_tempfiles(Storage, Transfer, Stats) -> collect_outdated_fragments(Storage, Transfer, Cutoff, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, fragment), - Filter = fun(_Filepath, #file_info{mtime = ModifiedAt}) -> ModifiedAt < Cutoff end, - maybe_collect_directory(Dirname, Filter, Stats). + maybe_collect_directory(Dirname, filter_older_than(Cutoff), Stats). collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, temporary), - Filter = fun(_Filepath, #file_info{mtime = ModifiedAt}) -> ModifiedAt < Cutoff end, - maybe_collect_directory(Dirname, Filter, Stats). + maybe_collect_directory(Dirname, filter_older_than(Cutoff), Stats). -collect_transfer_directory(Storage, Transfer, Stats) -> +collect_transfer_directory(Storage, Transfer, Cutoff, Stats) -> Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer), - StatsNext = collect_empty_directory(Dirname, Stats), - collect_parents(Dirname, get_storage_root(Storage), StatsNext). + Filter = + case Stats of + #gcstats{directories = 0} -> + % Nothing were collected, this is a leftover from a past complete transfer GC. + filter_older_than(Cutoff); + #gcstats{} -> + % Usual incomplete transfer GC, collect directories unconditionally. + true + end, + case collect_empty_directory(Dirname, Filter, Stats) of + {true, StatsNext} -> + collect_parents(Dirname, get_storage_root(Storage), StatsNext); + {false, StatsNext} -> + StatsNext + end. + +filter_older_than(Cutoff) -> + fun(_Filepath, #file_info{mtime = ModifiedAt}) -> ModifiedAt =< Cutoff end. collect_parents(Dirname, Until, Stats) -> Parent = filename:dirname(Dirname), @@ -218,14 +232,6 @@ collect_parents(Dirname, Until, Stats) -> register_gcstat_error({directory, Parent}, Reason, Stats) end. -% collect_outdated_fragment(#{path := Filepath, fileinfo := Fileinfo}, Cutoff, Stats) -> -% case Fileinfo#file_info.mtime of -% ModifiedAt when ModifiedAt < Cutoff -> -% collect_filepath(Filepath, Fileinfo, Stats); -% _ -> -% Stats -% end. - maybe_collect_directory(Dirpath, Filter, Stats) -> case filelib:is_dir(Dirpath) of true -> @@ -263,10 +269,10 @@ collect_directory(Dirpath, Fileinfo, Filter, Stats) -> case file:list_dir(Dirpath) of {ok, Filenames} -> {Clean, StatsNext} = collect_files(Dirpath, Filenames, Filter, Stats), - case Clean andalso filter_filepath(Filter, Dirpath, Fileinfo) of + case Clean of true -> - {true, collect_empty_directory(Dirpath, StatsNext)}; - _ -> + collect_empty_directory(Dirpath, Fileinfo, Filter, StatsNext); + false -> {false, StatsNext} end; {error, Reason} -> @@ -284,13 +290,23 @@ collect_files(Dirname, Filenames, Filter, Stats) -> Filenames ). -collect_empty_directory(Dirpath, Stats) -> - case file:del_dir(Dirpath) of +collect_empty_directory(Dirpath, Filter, Stats) -> + case file:read_link_info(Dirpath, [{time, posix}, raw]) of + {ok, Dirinfo} -> + collect_empty_directory(Dirpath, Dirinfo, Filter, Stats); + {error, Reason} -> + {Reason == enoent, register_gcstat_error({directory, Dirpath}, Reason, Stats)} + end. + +collect_empty_directory(Dirpath, Dirinfo, Filter, Stats) -> + case filter_filepath(Filter, Dirpath, Dirinfo) andalso file:del_dir(Dirpath) of + false -> + {false, Stats}; ok -> ?tp(garbage_collected_directory, #{path => Dirpath}), - account_gcstat_directory(Stats); + {true, account_gcstat_directory(Stats)}; {error, Reason} -> - register_gcstat_error({directory, Dirpath}, Reason, Stats) + {false, register_gcstat_error({directory, Dirpath}, Reason, Stats)} end. filter_filepath(Filter, _, _) when is_boolean(Filter) -> diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index 25cd200f1..53a22b8b6 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -104,6 +104,10 @@ t_gc_triggers_manually(_Config) -> t_gc_complete_transfers(_Config) -> Storage = emqx_ft_conf:storage(), + ok = emqx_config:put([file_transfer, storage, gc, minimum_segments_ttl], 0), + ok = emqx_config:put([file_transfer, storage, gc, maximum_segments_ttl], 3), + ok = emqx_config:put([file_transfer, storage, gc, interval], 500), + ok = emqx_ft_storage_fs_gc:reset(Storage), Transfers = [ { T1 = {<<"client1">>, mk_file_id()}, @@ -174,7 +178,20 @@ t_gc_complete_transfers(_Config) -> ?assertEqual(?NSEGS(S2, SS2) + ?NSEGS(S3, SS3), CFiles), ?assertEqual(2 + 2, CDirectories), ?assertMatch(Space when Space > S2 + S3, CSpace), - ?assertMatch(Errors when map_size(Errors) == 0, CErrors). + ?assertMatch(Errors when map_size(Errors) == 0, CErrors), + % 4. Ensure that empty transfer directories will be eventually collected + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = 0, + directories = 6, + space = 0 + } + }, + 5000, + 0 + ). t_gc_incomplete_transfers(_Config) -> ok = emqx_config:put([file_transfer, storage, gc, minimum_segments_ttl], 0), From 54e54cc63dc2f614127e37aa6fd9e6059fd59f0a Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 27 Mar 2023 23:44:23 +0300 Subject: [PATCH 075/156] feat(api): provide utility to qualify API URIs --- apps/emqx_dashboard/src/emqx_dashboard.erl | 8 +++----- apps/emqx_dashboard/src/emqx_dashboard_swagger.erl | 12 ++++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 6f0c8334a..53b661ea8 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -41,8 +41,6 @@ -include_lib("emqx/include/http_api.hrl"). -include_lib("emqx/include/emqx_release.hrl"). --define(BASE_PATH, "/api/v5"). - -define(EMQX_MIDDLE, emqx_dashboard_middleware). %%-------------------------------------------------------------------- @@ -61,7 +59,7 @@ start_listeners(Listeners) -> GlobalSpec = #{ openapi => "3.0.0", info => #{title => "EMQX API", version => ?EMQX_API_VERSION}, - servers => [#{url => ?BASE_PATH}], + servers => [#{url => emqx_dashboard_swagger:base_path()}], components => #{ schemas => #{}, 'securitySchemes' => #{ @@ -78,11 +76,11 @@ start_listeners(Listeners) -> {"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, {emqx_mgmt_api_status:path(), emqx_mgmt_api_status, []}, - {?BASE_PATH ++ "/[...]", emqx_dashboard_bad_api, []}, + {emqx_dashboard_swagger:relative_uri("/[...]"), emqx_dashboard_bad_api, []}, {'_', cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}} ], BaseMinirest = #{ - base_path => ?BASE_PATH, + base_path => emqx_dashboard_swagger:base_path(), modules => minirest_api:find_api_modules(apps()), authorization => Authorization, security => [#{'basicAuth' => []}, #{'bearerAuth' => []}], diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index e2872c0d7..95f28e387 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -19,12 +19,16 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). +-define(BASE_PATH, "/api/v5"). + %% API -export([spec/1, spec/2]). -export([namespace/0, namespace/1, fields/1]). -export([schema_with_example/2, schema_with_examples/2]). -export([error_codes/1, error_codes/2]). -export([file_schema/1]). +-export([base_path/0]). +-export([relative_uri/1]). -export([filter_check_request/2, filter_check_request_and_translate_body/2]). @@ -177,6 +181,14 @@ error_codes(Codes = [_ | _], MsgDesc) -> })} ]. +-spec base_path() -> uri_string:uri_string(). +base_path() -> + ?BASE_PATH. + +-spec relative_uri(uri_string:uri_string()) -> uri_string:uri_string(). +relative_uri(Uri) -> + base_path() ++ Uri. + file_schema(FileName) -> #{ content => #{ From b6b044f42947fb0702006dce70feb4afbc7c1490 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 27 Mar 2023 22:58:25 +0300 Subject: [PATCH 076/156] feat(ft): move out exporter concept into dedicated modules --- apps/emqx/priv/bpapi.versions | 1 + .../test/emqx_dashboard_api_test_helpers.erl | 8 +- apps/emqx_ft/i18n/emqx_ft_api_i18n.conf | 4 + apps/emqx_ft/src/emqx_ft_api.erl | 133 +------------ apps/emqx_ft/src/emqx_ft_assembler.erl | 43 ++--- apps/emqx_ft/src/emqx_ft_storage.erl | 14 +- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 97 ++++++++++ .../src/emqx_ft_storage_exporter_fs.erl | 14 +- .../src/emqx_ft_storage_exporter_fs_api.erl | 180 ++++++++++++++++++ .../src/emqx_ft_storage_exporter_fs_proxy.erl | 46 +++++ apps/emqx_ft/src/emqx_ft_storage_fs.erl | 23 +-- apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl | 20 +- .../emqx_ft_storage_exporter_fs_proto_v1.erl | 51 +++++ .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 18 -- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 4 +- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 22 ++- 16 files changed, 437 insertions(+), 241 deletions(-) create mode 100644 apps/emqx_ft/src/emqx_ft_storage_exporter.erl create mode 100644 apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl create mode 100644 apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl create mode 100644 apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 0e0957adf..d597646d6 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -13,6 +13,7 @@ {emqx_dashboard,1}. {emqx_delayed,1}. {emqx_exhook,1}. +{emqx_ft_storage_exporter_fs,1}. {emqx_ft_storage_fs,1}. {emqx_ft_storage_fs_reader,1}. {emqx_gateway_api_listeners,1}. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl index 8df130897..f70f464b3 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl @@ -25,11 +25,12 @@ request/4, multipart_formdata_request/3, multipart_formdata_request/4, + host/0, uri/0, uri/1 ]). --define(HOST, "http://127.0.0.1:18083/"). +-define(HOST, "http://127.0.0.1:18083"). -define(API_VERSION, "v5"). -define(BASE_PATH, "api"). @@ -94,10 +95,13 @@ request(Username, Method, Url, Body) -> {error, Reason} end. +host() -> + ?HOST. + uri() -> uri([]). uri(Parts) when is_list(Parts) -> NParts = [E || E <- Parts], - ?HOST ++ to_list(filename:join([?BASE_PATH, ?API_VERSION | NParts])). + host() ++ "/" ++ to_list(filename:join([?BASE_PATH, ?API_VERSION | NParts])). auth_header(Username) -> Password = <<"public">>, diff --git a/apps/emqx_ft/i18n/emqx_ft_api_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_api_i18n.conf index 0bda935f8..6ac01ea23 100644 --- a/apps/emqx_ft/i18n/emqx_ft_api_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_api_i18n.conf @@ -11,6 +11,10 @@ emqx_ft_api { } } +} + +emqx_ft_storage_exporter_fs_api { + file_get { desc { en: "Get a file by its id." diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index 143d629de..eeb047b16 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -30,18 +30,12 @@ ]). -export([ - fields/1, roots/0 ]). %% API callbacks -export([ - '/file_transfer/files'/2, - '/file_transfer/file'/2 -]). - --export([ - mk_file_uri/3 + '/file_transfer/files'/2 ]). -import(hoconsc, [mk/2, ref/1, ref/2]). @@ -53,8 +47,7 @@ api_spec() -> paths() -> [ - "/file_transfer/files", - "/file_transfer/file" + "/file_transfer/files" ]. schema("/file_transfer/files") -> @@ -71,31 +64,6 @@ schema("/file_transfer/files") -> ) } } - }; -schema("/file_transfer/file") -> - % TODO - % This is conceptually another API, because this logic is inherent only to the - % local filesystem exporter. Ideally, we won't even publish it if `emqx_ft` is - % configured with another exporter. Accordingly, things that look too specific - % for this module (i.e. `parse_filepath/1`) should go away in another API module. - #{ - 'operationId' => '/file_transfer/file', - get => #{ - tags => [<<"file_transfer">>], - summary => <<"Download a particular file">>, - description => ?DESC("file_get"), - parameters => [ - ref(file_node), - ref(file_ref) - ], - responses => #{ - 200 => <<"Operation success">>, - 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Not found">>), - 503 => emqx_dashboard_swagger:error_codes( - ['SERVICE_UNAVAILABLE'], <<"Service unavailable">> - ) - } - } }. '/file_transfer/files'(get, #{}) -> @@ -106,76 +74,11 @@ schema("/file_transfer/file") -> {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} end. -'/file_transfer/file'(get, #{query_string := Query}) -> - try - Node = parse_node(maps:get(<<"node">>, Query)), - Filepath = parse_filepath(maps:get(<<"fileref">>, Query)), - case emqx_ft_storage_fs_proto_v1:read_export_file(Node, Filepath, self()) of - {ok, ReaderPid} -> - FileData = emqx_ft_storage_fs_reader:table(ReaderPid), - {200, - #{ - <<"content-type">> => <<"application/data">>, - <<"content-disposition">> => <<"attachment">> - }, - FileData}; - {error, enoent} -> - {404, error_msg('NOT_FOUND', <<"Not found">>)}; - {error, Error} -> - ?SLOG(warning, #{msg => "get_ready_transfer_fail", error => Error}), - {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} - end - catch - throw:{invalid, Param} -> - {404, - error_msg( - 'NOT_FOUND', - iolist_to_binary(["Invalid query parameter: ", Param]) - )}; - error:{erpc, noconnection} -> - {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} - end. - error_msg(Code, Msg) -> #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. --spec fields(hocon_schema:name()) -> hocon_schema:fields(). -fields(file_ref) -> - [ - {fileref, - hoconsc:mk(binary(), #{ - in => query, - desc => <<"File reference">>, - example => <<"file1">>, - required => true - })} - ]; -fields(file_node) -> - [ - {node, - hoconsc:mk(binary(), #{ - in => query, - desc => <<"Node under which the file is located">>, - example => atom_to_list(node()), - required => true - })} - ]. - roots() -> - [ - file_node, - file_ref - ]. - -mk_file_uri(_Options, Node, Filepath) -> - % TODO: qualify with `?BASE_PATH` - [ - "/file_transfer/file?", - uri_string:compose_query([ - {"node", atom_to_list(Node)}, - {"fileref", Filepath} - ]) - ]. + []. %%-------------------------------------------------------------------- %% Helpers @@ -207,33 +110,3 @@ format_export_info( format_timestamp(Timestamp) -> iolist_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])). - -parse_node(NodeBin) -> - case emqx_misc:safe_to_existing_atom(NodeBin) of - {ok, Node} -> - Node; - {error, _} -> - throw({invalid, NodeBin}) - end. - -parse_filepath(PathBin) -> - case filename:pathtype(PathBin) of - relative -> - ok; - absolute -> - throw({invalid, PathBin}) - end, - PathComponents = filename:split(PathBin), - case lists:any(fun is_special_component/1, PathComponents) of - false -> - filename:join(PathComponents); - true -> - throw({invalid, PathBin}) - end. - -is_special_component(<<".", _/binary>>) -> - true; -is_special_component([$. | _]) -> - true; -is_special_component(_) -> - false. diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 5489a232a..a86116b02 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -106,11 +106,15 @@ handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> {error, _} = Error -> {stop, {shutdown, Error}} end; -handle_event(internal, _, start_assembling, St = #st{assembly = Asm}) -> - Filemeta = emqx_ft_assembly:filemeta(Asm), - Coverage = emqx_ft_assembly:coverage(Asm), +handle_event(internal, _, start_assembling, St = #st{}) -> + Filemeta = emqx_ft_assembly:filemeta(St#st.assembly), + Coverage = emqx_ft_assembly:coverage(St#st.assembly), % TODO: better error handling - {ok, Export} = export_start(Filemeta, St), + {ok, Export} = emqx_ft_storage_exporter:start_export( + St#st.storage, + St#st.transfer, + Filemeta + ), {next_state, {assemble, Coverage}, St#st{export = Export}, ?internal([])}; handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % TODO @@ -119,17 +123,17 @@ handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> % TODO: pipelining % TODO: better error handling {ok, Content} = pread(Node, Segment, St), - {ok, NExport} = export_write(St#st.export, Content), + {ok, NExport} = emqx_ft_storage_exporter:write(St#st.export, Content), {next_state, {assemble, Rest}, St#st{export = NExport}, ?internal([])}; handle_event(internal, _, {assemble, []}, St = #st{}) -> {next_state, complete, St, ?internal([])}; handle_event(internal, _, complete, St = #st{}) -> - Result = export_complete(St#st.export), + Result = emqx_ft_storage_exporter:complete(St#st.export), ok = maybe_garbage_collect(Result, St), {stop, {shutdown, Result}, St#st{export = undefined}}. terminate(_Reason, _StateName, #st{export = Export}) -> - Export /= undefined andalso export_discard(Export). + Export /= undefined andalso emqx_ft_storage_exporter:discard(Export). pread(Node, Segment, St) when Node =:= node() -> emqx_ft_storage_fs:pread(St#st.storage, St#st.transfer, Segment, 0, segsize(Segment)); @@ -138,31 +142,6 @@ pread(Node, Segment, St) -> %% -export_start(Filemeta, #st{storage = Storage, transfer = Transfer}) -> - {ExporterMod, Exporter} = emqx_ft_storage_fs:exporter(Storage), - case ExporterMod:start_export(Exporter, Transfer, Filemeta) of - {ok, Export} -> - {ok, {ExporterMod, Export}}; - {error, _} = Error -> - Error - end. - -export_write({ExporterMod, Export}, Content) -> - case ExporterMod:write(Export, Content) of - {ok, ExportNext} -> - {ok, {ExporterMod, ExportNext}}; - {error, _} = Error -> - Error - end. - -export_complete({ExporterMod, Export}) -> - ExporterMod:complete(Export). - -export_discard({ExporterMod, Export}) -> - ExporterMod:discard(Export). - -%% - maybe_garbage_collect(ok, #st{storage = Storage, transfer = Transfer, assembly = Asm}) -> Nodes = emqx_ft_assembly:nodes(Asm), emqx_ft_storage_fs_gc:collect(Storage, Transfer, Nodes); diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 1d1c08ce9..05a053fbf 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -26,6 +26,7 @@ exports/0, + with_storage_type/2, with_storage_type/3 ] ). @@ -33,7 +34,9 @@ -type storage() :: emqx_config:config(). -export_type([assemble_callback/0]). +-export_type([export_info/0]). -export_type([export_data/0]). +-export_type([reader/0]). -type assemble_callback() :: fun((ok | {error, term()}) -> any()). @@ -46,6 +49,7 @@ }. -type export_data() :: binary() | qlc:query_handle(). +-type reader() :: pid(). %%-------------------------------------------------------------------- %% Behaviour @@ -106,11 +110,17 @@ exports() -> Mod = mod(), Mod:exports(storage()). --spec with_storage_type(atom(), atom(), list(term())) -> any(). +-spec with_storage_type(atom(), atom() | function()) -> any(). +with_storage_type(Type, Fun) -> + with_storage_type(Type, Fun, []). + +-spec with_storage_type(atom(), atom() | function(), list(term())) -> any(). with_storage_type(Type, Fun, Args) -> Storage = storage(), case Storage of - #{type := Type} -> + #{type := Type} when is_function(Fun) -> + apply(Fun, [Storage | Args]); + #{type := Type} when is_atom(Fun) -> Mod = mod(Storage), apply(Mod, Fun, [Storage | Args]); disabled -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl new file mode 100644 index 000000000..97e260e3a --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -0,0 +1,97 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Filesystem storage exporter +%% +%% This is conceptually a part of the Filesystem storage backend that defines +%% how and where complete tranfers are assembled into files and stored. + +-module(emqx_ft_storage_exporter). + +%% Export API +-export([start_export/3]). +-export([write/2]). +-export([complete/1]). +-export([discard/1]). + +%% Listing API +-export([list/1]). +% TODO +% -export([list/2]). + +-export([exporter/1]). + +-type storage() :: emxt_ft_storage_fs:storage(). +-type transfer() :: emqx_ft:transfer(). +-type filemeta() :: emqx_ft:filemeta(). + +-type options() :: map(). +-type export() :: term(). + +-callback start_export(options(), transfer(), filemeta()) -> + {ok, export()} | {error, _Reason}. + +-callback write(ExportSt :: export(), iodata()) -> + {ok, ExportSt :: export()} | {error, _Reason}. + +-callback complete(ExportSt :: export()) -> + ok | {error, _Reason}. + +-callback discard(ExportSt :: export()) -> + ok | {error, _Reason}. + +-callback list(options()) -> + {ok, [emqx_ft_storage:export_info()]} | {error, _Reason}. + +%% + +start_export(Storage, Transfer, Filemeta) -> + {ExporterMod, Exporter} = exporter(Storage), + case ExporterMod:start_export(Exporter, Transfer, Filemeta) of + {ok, ExportSt} -> + {ok, {ExporterMod, ExportSt}}; + {error, _} = Error -> + Error + end. + +write({ExporterMod, ExportSt}, Content) -> + case ExporterMod:write(ExportSt, Content) of + {ok, ExportStNext} -> + {ok, {ExporterMod, ExportStNext}}; + {error, _} = Error -> + Error + end. + +complete({ExporterMod, ExportSt}) -> + ExporterMod:complete(ExportSt). + +discard({ExporterMod, ExportSt}) -> + ExporterMod:discard(ExportSt). + +%% + +list(Storage) -> + {ExporterMod, ExporterOpts} = exporter(Storage), + ExporterMod:list(ExporterOpts). + +%% + +-spec exporter(storage()) -> {module(), _ExporterOptions}. +exporter(Storage) -> + case maps:get(exporter, Storage) of + #{type := local} = Options -> + {emqx_ft_storage_exporter_fs, Options} + end. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index 6456e2ba5..7cf0ef6d5 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -30,6 +30,7 @@ -export([start_reader/3]). -export([list/1]). +% TODO % -export([list/2]). -export_type([export/0]). @@ -228,7 +229,7 @@ mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo) -> #{ transfer => Transfer, name => Filename, - uri => mk_export_uri(Options, RelFilepath), + uri => mk_export_uri(RelFilepath), timestamp => Fileinfo#file_info.mtime, size => Fileinfo#file_info.size, path => filename:join(Root, RelFilepath) @@ -247,15 +248,14 @@ try_read_filemeta(Filepath, Info) -> Info end. -mk_export_uri(Options, RelFilepath) -> - % emqx_ft_storage_exporter_fs_api:mk_export_uri(Options, RelFilepath). - emqx_ft_api:mk_file_uri(Options, node(), RelFilepath). +mk_export_uri(RelFilepath) -> + emqx_ft_storage_exporter_fs_api:mk_export_uri(node(), RelFilepath). -spec start_reader(options(), file:name(), _Caller :: pid()) -> {ok, reader()} | {error, enoent}. -start_reader(Options, Filepath, CallerPid) -> +start_reader(Options, RelFilepath, CallerPid) -> Root = get_storage_root(Options), - case filelib:safe_relative_path(Filepath, Root) of + case filelib:safe_relative_path(RelFilepath, Root) of SafeFilepath when SafeFilepath /= unsafe -> AbsFilepath = filename:join(Root, SafeFilepath), emqx_ft_storage_fs_reader:start_supervised(CallerPid, AbsFilepath); @@ -269,7 +269,7 @@ start_reader(Options, Filepath, CallerPid) -> {ok, [exportinfo(), ...]} | {error, file_error()}. list(_Options) -> Nodes = mria_mnesia:running_nodes(), - Results = emqx_ft_storage_fs_proto_v1:list_exports(Nodes), + Results = emqx_ft_storage_exporter_fs_proto_v1:list_exports(Nodes), {GoodResults, BadResults} = lists:partition( fun ({_Node, {ok, {ok, _}}}) -> true; diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl new file mode 100644 index 000000000..b7ad86436 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl @@ -0,0 +1,180 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_exporter_fs_api). + +-behaviour(minirest_api). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% Swagger specs from hocon schema +-export([ + api_spec/0, + paths/0, + schema/1, + namespace/0 +]). + +-export([ + fields/1, + roots/0 +]). + +%% API callbacks +-export([ + '/file_transfer/file'/2 +]). + +-export([mk_export_uri/2]). + +%% + +namespace() -> "file_transfer". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + [ + "/file_transfer/file" + ]. + +schema("/file_transfer/file") -> + #{ + 'operationId' => '/file_transfer/file', + get => #{ + tags => [<<"file_transfer">>], + summary => <<"Download a particular file">>, + description => ?DESC("file_get"), + parameters => [ + hoconsc:ref(file_node), + hoconsc:ref(file_ref) + ], + responses => #{ + 200 => <<"Operation success">>, + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Not found">>), + 503 => emqx_dashboard_swagger:error_codes( + ['SERVICE_UNAVAILABLE'], <<"Service unavailable">> + ) + } + } + }. + +roots() -> + [ + file_node, + file_ref + ]. + +-spec fields(hocon_schema:name()) -> hocon_schema:fields(). +fields(file_ref) -> + [ + {fileref, + hoconsc:mk(binary(), #{ + in => query, + desc => <<"File reference">>, + example => <<"file1">>, + required => true + })} + ]; +fields(file_node) -> + [ + {node, + hoconsc:mk(binary(), #{ + in => query, + desc => <<"Node under which the file is located">>, + example => atom_to_list(node()), + required => true + })} + ]. + +'/file_transfer/file'(get, #{query_string := Query}) -> + try + Node = parse_node(maps:get(<<"node">>, Query)), + Filepath = parse_filepath(maps:get(<<"fileref">>, Query)), + case emqx_ft_storage_exporter_fs_proto_v1:read_export_file(Node, Filepath, self()) of + {ok, ReaderPid} -> + FileData = emqx_ft_storage_fs_reader:table(ReaderPid), + {200, + #{ + <<"content-type">> => <<"application/data">>, + <<"content-disposition">> => <<"attachment">> + }, + FileData}; + {error, enoent} -> + {404, error_msg('NOT_FOUND', <<"Not found">>)}; + {error, Error} -> + ?SLOG(warning, #{msg => "get_ready_transfer_fail", error => Error}), + {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} + end + catch + throw:{invalid, Param} -> + {404, + error_msg( + 'NOT_FOUND', + iolist_to_binary(["Invalid query parameter: ", Param]) + )}; + error:{erpc, noconnection} -> + {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} + end. + +error_msg(Code, Msg) -> + #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. + +-spec mk_export_uri(node(), file:name()) -> + uri_string:uri_string(). +mk_export_uri(Node, Filepath) -> + emqx_dashboard_swagger:relative_uri([ + "/file_transfer/file?", + uri_string:compose_query([ + {"node", atom_to_list(Node)}, + {"fileref", Filepath} + ]) + ]). + +%% + +parse_node(NodeBin) -> + case emqx_misc:safe_to_existing_atom(NodeBin) of + {ok, Node} -> + Node; + {error, _} -> + throw({invalid, NodeBin}) + end. + +parse_filepath(PathBin) -> + case filename:pathtype(PathBin) of + relative -> + ok; + absolute -> + throw({invalid, PathBin}) + end, + PathComponents = filename:split(PathBin), + case lists:any(fun is_special_component/1, PathComponents) of + false -> + filename:join(PathComponents); + true -> + throw({invalid, PathBin}) + end. + +is_special_component(<<".", _/binary>>) -> + true; +is_special_component([$. | _]) -> + true; +is_special_component(_) -> + false. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl new file mode 100644 index 000000000..bdb64f543 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl @@ -0,0 +1,46 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% This methods are called via rpc by `emqx_ft_storage_exporter_fs` +%% They populate the call with actual storage which may be configured differently +%% on a concrete node. + +-module(emqx_ft_storage_exporter_fs_proxy). + +-export([ + list_exports_local/0, + read_export_file_local/2 +]). + +list_exports_local() -> + emqx_ft_storage:with_storage_type(local, fun(Storage) -> + case emqx_ft_storage_exporter:exporter(Storage) of + {emqx_ft_storage_exporter_fs, Options} -> + emqx_ft_storage_exporter_fs:list_local(Options); + InvalidExporter -> + {error, {invalid_exporter, InvalidExporter}} + end + end). + +read_export_file_local(Filepath, CallerPid) -> + emqx_ft_storage:with_storage_type(local, fun(Storage) -> + case emqx_ft_storage_exporter:exporter(Storage) of + {emqx_ft_storage_exporter_fs, Options} -> + emqx_ft_storage_exporter_fs:start_reader(Options, Filepath, CallerPid); + InvalidExporter -> + {error, {invalid_exporter, InvalidExporter}} + end + end). diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 6c5b6962c..b328cfd2b 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -44,12 +44,7 @@ -export([get_subdir/2]). -export([get_subdir/3]). --export([exporter/1]). - -% Exporter-specific API -export([exports/1]). --export([exports_local/1]). --export([exports_local/2]). -export_type([storage/0]). -export_type([filefrag/1]). @@ -214,24 +209,8 @@ assemble(Storage, Transfer, Size) -> %% --spec exporter(storage()) -> {module(), _ExporterOptions}. -exporter(Storage) -> - case maps:get(exporter, Storage) of - #{type := local} = Options -> - {emqx_ft_storage_exporter_fs, Options} - end. - exports(Storage) -> - {ExporterMod, ExporterOpts} = exporter(Storage), - ExporterMod:list(ExporterOpts). - -exports_local(Storage) -> - {ExporterMod, ExporterOpts} = exporter(Storage), - ExporterMod:list_local(ExporterOpts). - -exports_local(Storage, Transfer) -> - {ExporterMod, ExporterOpts} = exporter(Storage), - ExporterMod:list_local(ExporterOpts, Transfer). + emqx_ft_storage_exporter:list(Storage). %% diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl index dbd0cd6dc..597f84091 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl @@ -22,9 +22,7 @@ -export([ list_local/2, - pread_local/4, - list_exports_local/0, - read_export_file_local/2 + pread_local/4 ]). list_local(Transfer, What) -> @@ -32,19 +30,3 @@ list_local(Transfer, What) -> pread_local(Transfer, Frag, Offset, Size) -> emqx_ft_storage:with_storage_type(local, pread, [Transfer, Frag, Offset, Size]). - -list_exports_local() -> - case emqx_ft_storage:with_storage_type(local, exporter, []) of - {emqx_ft_storage_exporter_fs, Options} -> - emqx_ft_storage_exporter_fs:list_local(Options); - InvalidExporter -> - {error, {invalid_exporter, InvalidExporter}} - end. - -read_export_file_local(Filepath, CallerPid) -> - case emqx_ft_storage:with_storage_type(local, exporter, []) of - {emqx_ft_storage_exporter_fs, Options} -> - emqx_ft_storage_exporter_fs:start_reader(Options, Filepath, CallerPid); - InvalidExporter -> - {error, {invalid_exporter, InvalidExporter}} - end. diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl new file mode 100644 index 000000000..ee7a857be --- /dev/null +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl @@ -0,0 +1,51 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_exporter_fs_proto_v1). + +-behaviour(emqx_bpapi). + +-export([introduced_in/0]). + +-export([list_exports/1]). +-export([read_export_file/3]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.17". + +-spec list_exports([node()]) -> + emqx_rpc:erpc_multicall([emqx_ft_storage:export_info()]). +list_exports(Nodes) -> + erpc:multicall( + Nodes, + emqx_ft_storage_exporter_fs_proxy, + list_exports_local, + [] + ). + +-spec read_export_file(node(), file:name(), pid()) -> + {ok, emqx_ft_storage:reader()} + | {error, term()} + | no_return(). +read_export_file(Node, Filepath, CallerPid) -> + erpc:call( + Node, + emqx_ft_storage_exporter_fs_proxy, + read_export_file_local, + [Filepath, CallerPid] + ). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index f152928fe..3b91684e6 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -23,10 +23,6 @@ -export([multilist/3]). -export([pread/5]). -%% TODO: These should be defined in a separate BPAPI --export([list_exports/1]). --export([read_export_file/3]). - -type offset() :: emqx_ft:offset(). -type transfer() :: emqx_ft:transfer(). -type filefrag() :: emqx_ft_storage_fs:filefrag(). @@ -45,17 +41,3 @@ multilist(Nodes, Transfer, What) -> {ok, [filefrag()]} | {error, term()} | no_return(). pread(Node, Transfer, Frag, Offset, Size) -> erpc:call(Node, emqx_ft_storage_fs_proxy, pread_local, [Transfer, Frag, Offset, Size]). - -%% - --spec list_exports([node()]) -> - emqx_rpc:erpc_multicall([emqx_ft_storage:export_info()]). -list_exports(Nodes) -> - erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, list_exports_local, []). - --spec read_export_file(node(), file:name(), pid()) -> - {ok, emqx_ft_storage:export_data()} - | {error, term()} - | no_return(). -read_export_file(Node, Filepath, CallerPid) -> - erpc:call(Node, emqx_ft_storage_fs_proxy, read_export_file_local, [Filepath, CallerPid]). diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index 56622f3cd..a1bef2a2a 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -24,7 +24,7 @@ -include_lib("emqx/include/asserts.hrl"). --import(emqx_mgmt_api_test_util, [uri/1]). +-import(emqx_dashboard_api_test_helpers, [host/0, uri/1]). all() -> emqx_common_test_helpers:all(?MODULE). @@ -111,7 +111,7 @@ t_download_transfer(Config) -> {ok, 200, #{<<"files">> := [File]}} = request(get, uri(["file_transfer", "files"]), fun json/1), - {ok, 200, Response} = request(get, uri([]) ++ maps:get(<<"uri">>, File)), + {ok, 200, Response} = request(get, host() ++ maps:get(<<"uri">>, File)), ?assertEqual( <<"data">>, diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 4c7c0f08c..7c5d14b38 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -86,8 +86,7 @@ t_assemble_empty_transfer(Config) -> ), Status = complete_assemble(Storage, Transfer, 0), ?assertEqual({shutdown, ok}, Status), - {ok, [_Result = #{size := _Size = 0}]} = - emqx_ft_storage_fs:exports_local(Storage, Transfer), + {ok, [_Result = #{size := _Size = 0}]} = list_exports(Config, Transfer), % ?assertEqual( % {error, eof}, % emqx_ft_storage_fs:pread(Storage, Transfer, Result, 0, Size) @@ -138,10 +137,9 @@ t_assemble_complete_local_transfer(Config) -> meta := #{} } ]}, - emqx_ft_storage_fs:exports_local(Storage, Transfer) + list_exports(Config, Transfer) ), - {ok, [#{path := AssemblyFilename}]} = - emqx_ft_storage_fs:exports_local(Storage, Transfer), + {ok, [#{path := AssemblyFilename}]} = list_exports(Config, Transfer), ?assertMatch( {ok, #file_info{type = regular, size = TransferSize}}, file:read_file_info(AssemblyFilename) @@ -193,8 +191,7 @@ complete_assemble(Storage, Transfer, Size, Timeout) -> %% t_list_transfers(Config) -> - Storage = storage(Config), - {ok, Exports} = emqx_ft_storage_fs:exports_local(Storage), + {ok, Exports} = list_exports(Config), ?assertMatch( [ #{ @@ -237,6 +234,17 @@ inspect_file(Filename) -> mk_fileid() -> integer_to_binary(erlang:system_time(millisecond)). +list_exports(Config) -> + {emqx_ft_storage_exporter_fs, Options} = exporter(Config), + emqx_ft_storage_exporter_fs:list_local(Options). + +list_exports(Config, Transfer) -> + {emqx_ft_storage_exporter_fs, Options} = exporter(Config), + emqx_ft_storage_exporter_fs:list_local(Options, Transfer). + +exporter(Config) -> + emqx_ft_storage_exporter:exporter(storage(Config)). + storage(Config) -> #{ type => local, From 23cd78b8d63913efc1b6dc0e896d5d3d42015444 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 28 Mar 2023 13:26:49 +0300 Subject: [PATCH 077/156] refactor(ft-asm): turn state data record into a map Which should be more future-proof. --- apps/emqx_ft/src/emqx_ft_assembler.erl | 114 +++++++++++------- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 3 + 2 files changed, 75 insertions(+), 42 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index a86116b02..17fed012a 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -24,12 +24,12 @@ -export([handle_event/4]). -export([terminate/3]). --record(st, { - storage :: _Storage, - transfer :: emqx_ft:transfer(), - assembly :: emqx_ft_assembly:t(), - export :: _Export | undefined -}). +-type stdata() :: #{ + storage := emqx_ft_storage_fs:storage(), + transfer := emqx_ft:transfer(), + assembly := emqx_ft_assembly:t(), + export => emqx_ft_storage_exporter:export() +}. -define(NAME(Transfer), {n, l, {?MODULE, Transfer}}). -define(REF(Transfer), {via, gproc, ?NAME(Transfer)}). @@ -41,31 +41,48 @@ start_link(Storage, Transfer, Size) -> %% +-type state() :: + idle + | list_local_fragments + | {list_remote_fragments, [node()]} + | start_assembling + | {assemble, [{node(), emqx_ft_storage_fs:filefrag()}]} + | complete. + -define(internal(C), {next_event, internal, C}). callback_mode() -> handle_event_function. +-spec init(_Args) -> {ok, state(), stdata()}. init({Storage, Transfer, Size}) -> _ = erlang:process_flag(trap_exit, true), - St = #st{ - storage = Storage, - transfer = Transfer, - assembly = emqx_ft_assembly:new(Size) + St = #{ + storage => Storage, + transfer => Transfer, + assembly => emqx_ft_assembly:new(Size) }, {ok, idle, St}. +-spec handle_event(info | internal, _, state(), stdata()) -> + {next_state, state(), stdata(), {next_event, internal, _}} + | {stop, {shutdown, ok | {error, _}}, stdata()}. handle_event(info, kickoff, idle, St) -> % NOTE % Someone's told us to start the work, which usually means that it has set up a monitor. % We could wait for this message and handle it at the end of the assembling rather than at % the beginning, however it would make error handling much more messier. {next_state, list_local_fragments, St, ?internal([])}; -handle_event(internal, _, list_local_fragments, St = #st{}) -> +handle_event( + internal, + _, + list_local_fragments, + St = #{storage := Storage, transfer := Transfer, assembly := Asm} +) -> % TODO: what we do with non-transients errors here (e.g. `eacces`)? - {ok, Fragments} = emqx_ft_storage_fs:list(St#st.storage, St#st.transfer, fragment), - NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(St#st.assembly, node(), Fragments)), - NSt = St#st{assembly = NAsm}, + {ok, Fragments} = emqx_ft_storage_fs:list(Storage, Transfer, fragment), + NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(Asm, node(), Fragments)), + NSt = St#{assembly := NAsm}, case emqx_ft_assembly:status(NAsm) of complete -> {next_state, start_assembling, NSt, ?internal([])}; @@ -76,27 +93,32 @@ handle_event(internal, _, list_local_fragments, St = #st{}) -> {error, _} = Error -> {stop, {shutdown, Error}} end; -handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> +handle_event( + internal, + _, + {list_remote_fragments, Nodes}, + St = #{transfer := Transfer, assembly := Asm} +) -> % TODO % Async would better because we would not need to wait for some lagging nodes if % the coverage is already complete. % TODO: portable "storage" ref - Results = emqx_ft_storage_fs_proto_v1:multilist(Nodes, St#st.transfer, fragment), + Results = emqx_ft_storage_fs_proto_v1:multilist(Nodes, Transfer, fragment), NodeResults = lists:zip(Nodes, Results), NAsm = emqx_ft_assembly:update( lists:foldl( fun - ({Node, {ok, {ok, Fragments}}}, Asm) -> - emqx_ft_assembly:append(Asm, Node, Fragments); - ({_Node, _Result}, Asm) -> + ({Node, {ok, {ok, Fragments}}}, Acc) -> + emqx_ft_assembly:append(Acc, Node, Fragments); + ({_Node, _Result}, Acc) -> % TODO: log? - Asm + Acc end, - St#st.assembly, + Asm, NodeResults ) ), - NSt = St#st{assembly = NAsm}, + NSt = St#{assembly := NAsm}, case emqx_ft_assembly:status(NAsm) of complete -> {next_state, start_assembling, NSt, ?internal([])}; @@ -106,43 +128,51 @@ handle_event(internal, _, {list_remote_fragments, Nodes}, St) -> {error, _} = Error -> {stop, {shutdown, Error}} end; -handle_event(internal, _, start_assembling, St = #st{}) -> - Filemeta = emqx_ft_assembly:filemeta(St#st.assembly), - Coverage = emqx_ft_assembly:coverage(St#st.assembly), +handle_event( + internal, + _, + start_assembling, + St = #{storage := Storage, transfer := Transfer, assembly := Asm} +) -> + Filemeta = emqx_ft_assembly:filemeta(Asm), + Coverage = emqx_ft_assembly:coverage(Asm), % TODO: better error handling {ok, Export} = emqx_ft_storage_exporter:start_export( - St#st.storage, - St#st.transfer, + Storage, + Transfer, Filemeta ), - {next_state, {assemble, Coverage}, St#st{export = Export}, ?internal([])}; -handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #st{}) -> + {next_state, {assemble, Coverage}, St#{export => Export}, ?internal([])}; +handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #{export := Export}) -> % TODO % Currently, race is possible between getting segment info from the remote node and % this node garbage collecting the segment itself. % TODO: pipelining % TODO: better error handling {ok, Content} = pread(Node, Segment, St), - {ok, NExport} = emqx_ft_storage_exporter:write(St#st.export, Content), - {next_state, {assemble, Rest}, St#st{export = NExport}, ?internal([])}; -handle_event(internal, _, {assemble, []}, St = #st{}) -> + {ok, NExport} = emqx_ft_storage_exporter:write(Export, Content), + {next_state, {assemble, Rest}, St#{export := NExport}, ?internal([])}; +handle_event(internal, _, {assemble, []}, St = #{}) -> {next_state, complete, St, ?internal([])}; -handle_event(internal, _, complete, St = #st{}) -> - Result = emqx_ft_storage_exporter:complete(St#st.export), +handle_event(internal, _, complete, St = #{export := Export}) -> + Result = emqx_ft_storage_exporter:complete(Export), ok = maybe_garbage_collect(Result, St), - {stop, {shutdown, Result}, St#st{export = undefined}}. + {stop, {shutdown, Result}, maps:remove(export, St)}. -terminate(_Reason, _StateName, #st{export = Export}) -> - Export /= undefined andalso emqx_ft_storage_exporter:discard(Export). +-spec terminate(_Reason, state(), stdata()) -> _. +terminate(_Reason, _StateName, #{export := Export}) -> + emqx_ft_storage_exporter:discard(Export); +terminate(_Reason, _StateName, #{}) -> + ok. -pread(Node, Segment, St) when Node =:= node() -> - emqx_ft_storage_fs:pread(St#st.storage, St#st.transfer, Segment, 0, segsize(Segment)); -pread(Node, Segment, St) -> - emqx_ft_storage_fs_proto_v1:pread(Node, St#st.transfer, Segment, 0, segsize(Segment)). +pread(Node, Segment, #{storage := Storage, transfer := Transfer}) when Node =:= node() -> + emqx_ft_storage_fs:pread(Storage, Transfer, Segment, 0, segsize(Segment)); +pread(Node, Segment, #{transfer := Transfer}) -> + emqx_ft_storage_fs_proto_v1:pread(Node, Transfer, Segment, 0, segsize(Segment)). %% -maybe_garbage_collect(ok, #st{storage = Storage, transfer = Transfer, assembly = Asm}) -> +maybe_garbage_collect(ok, #{storage := Storage, transfer := Transfer, assembly := Asm}) -> Nodes = emqx_ft_assembly:nodes(Asm), emqx_ft_storage_fs_gc:collect(Storage, Transfer, Nodes); maybe_garbage_collect({error, _}, _St) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index 97e260e3a..954423fba 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -34,6 +34,9 @@ -export([exporter/1]). +-export_type([options/0]). +-export_type([export/0]). + -type storage() :: emxt_ft_storage_fs:storage(). -type transfer() :: emqx_ft:transfer(). -type filemeta() :: emqx_ft:filemeta(). From c24c7eca34ca76e0dd0d727336e2c63bdfc5176b Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 28 Mar 2023 14:38:30 +0300 Subject: [PATCH 078/156] fix(ft-test): unbreak testcase by inhibiting local fs storage GC --- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 31 +++++++++++++++------- apps/emqx_ft/test/emqx_ft_SUITE.erl | 23 +++++++--------- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index eb274a4d5..258f3a7f3 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -81,12 +81,17 @@ handle_call(Call, From, St) -> {noreply, St}. handle_cast({collect, Transfer, [Node | Rest]}, St) -> - ok = do_collect_transfer(Transfer, Node, St), - case Rest of - [_ | _] -> - gen_server:cast(self(), {collect, Transfer, Rest}); - [] -> - ok + case gc_enabled(St) of + true -> + ok = do_collect_transfer(Transfer, Node, St), + case Rest of + [_ | _] -> + gen_server:cast(self(), {collect, Transfer, Rest}); + [] -> + ok + end; + false -> + skip end, {noreply, St}; handle_cast(reset, St) -> @@ -127,9 +132,14 @@ maybe_report(#gcstats{errors = Errors}, #st{storage = Storage}) when map_size(Er maybe_report(#gcstats{} = _Stats, #st{storage = _Storage}) -> ?tp(garbage_collection, #{stats => _Stats, storage => _Storage}). -start_timer(St = #st{next_gc_timer = undefined}) -> - Delay = emqx_ft_conf:gc_interval(St#st.storage), - St#st{next_gc_timer = emqx_misc:start_timer(Delay, collect)}. +start_timer(St = #st{storage = Storage, next_gc_timer = undefined}) -> + case emqx_ft_conf:gc_interval(Storage) of + Delay when Delay > 0 -> + St#st{next_gc_timer = emqx_misc:start_timer(Delay, collect)}; + 0 -> + ?SLOG(warning, #{msg => "periodic_gc_disabled"}), + St + end. reset_timer(St = #st{next_gc_timer = undefined}) -> start_timer(St); @@ -137,6 +147,9 @@ reset_timer(St = #st{next_gc_timer = TRef}) -> ok = emqx_misc:cancel_timer(TRef), start_timer(St#st{next_gc_timer = undefined}). +gc_enabled(St) -> + emqx_ft_conf:gc_interval(St#st.storage) > 0. + %% collect_garbage(Storage) -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 1a886c00d..c493a78c5 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -58,8 +58,12 @@ end_per_suite(_Config) -> set_special_configs(Config) -> fun (emqx_ft) -> + Storage = emqx_ft_test_helpers:local_storage(Config), emqx_ft_test_helpers:load_config(#{ - storage => emqx_ft_test_helpers:local_storage(Config) + % NOTE + % Inhibit local fs GC to simulate it isn't fast enough to collect + % complete transfers. + storage => Storage#{gc => #{interval => 0}} }); (_) -> ok @@ -107,14 +111,7 @@ mk_cluster_specs(Config) -> {env, [{emqx, boot_modules, [broker, listeners]}]}, {apps, [emqx_ft]}, {conf, [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]]}, - {env_handler, fun - (emqx_ft) -> - emqx_ft_test_helpers:load_config(#{ - storage => emqx_ft_test_helpers:local_storage(Config) - }); - (_) -> - ok - end} + {env_handler, set_special_configs(Config)} ], emqx_common_test_helpers:emqx_cluster( Specs, @@ -549,22 +546,20 @@ t_unreliable_migrating_client(Config) -> % twice. This is currently expected, files must be identical anyway. Node1Str = atom_to_list(Node1), NodeSelfStr = atom_to_list(NodeSelf), + % TODO: this testcase is specific to local fs storage backend ?assertMatch( [#{"node" := Node1Str}, #{"node" := NodeSelfStr}], lists:map( fun(#{uri := URIString}) -> #{query := QS} = uri_string:parse(URIString), - uri_string:dissect_query(QS) + maps:from_list(uri_string:dissect_query(QS)) end, lists:sort(Exports) ) ), [ - ?assertEqual( - {ok, Payload}, - read_export(Export) - ) + ?assertEqual({ok, Payload}, read_export(Export)) || Export <- Exports ]. From 45e3b62dc424e8b3e155863ab67c5846bbe53203 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 28 Mar 2023 17:48:10 +0300 Subject: [PATCH 079/156] refactor(ft): prefer plain `files` over `exports` in generic APIs So there's no more `exports` magically becomings `files` in the REST API which was slightly confusing. --- apps/emqx_ft/src/emqx_ft_api.erl | 8 ++++---- apps/emqx_ft/src/emqx_ft_storage.erl | 18 +++++++++--------- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 4 ++-- .../emqx_ft_storage_exporter_fs_proto_v1.erl | 2 +- apps/emqx_ft/test/emqx_ft_SUITE.erl | 14 +++++++------- apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 8 ++++---- 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index eeb047b16..a667454b6 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -67,9 +67,9 @@ schema("/file_transfer/files") -> }. '/file_transfer/files'(get, #{}) -> - case emqx_ft_storage:exports() of - {ok, Transfers} -> - {200, #{<<"files">> => lists:map(fun format_export_info/1, Transfers)}}; + case emqx_ft_storage:files() of + {ok, Files} -> + {200, #{<<"files">> => lists:map(fun format_file_info/1, Files)}}; {error, _} -> {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} end. @@ -84,7 +84,7 @@ roots() -> %% Helpers %%-------------------------------------------------------------------- -format_export_info( +format_file_info( Info = #{ name := Name, size := Size, diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 05a053fbf..bff149ce2 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -24,7 +24,7 @@ store_segment/2, assemble/2, - exports/0, + files/0, with_storage_type/2, with_storage_type/3 @@ -34,13 +34,13 @@ -type storage() :: emqx_config:config(). -export_type([assemble_callback/0]). --export_type([export_info/0]). +-export_type([file_info/0]). -export_type([export_data/0]). -export_type([reader/0]). -type assemble_callback() :: fun((ok | {error, term()}) -> any()). --type export_info() :: #{ +-type file_info() :: #{ transfer := emqx_ft:transfer(), name := file:name(), size := _Bytes :: non_neg_integer(), @@ -68,8 +68,8 @@ -callback assemble(storage(), emqx_ft:transfer(), _Size :: emqx_ft:bytes()) -> ok | {async, pid()} | {error, term()}. --callback exports(storage()) -> - {ok, [export_info()]} | {error, term()}. +-callback files(storage()) -> + {ok, [file_info()]} | {error, term()}. %%-------------------------------------------------------------------- %% API @@ -104,11 +104,11 @@ assemble(Transfer, Size) -> Mod = mod(), Mod:assemble(storage(), Transfer, Size). --spec exports() -> - {ok, [export_info()]} | {error, term()}. -exports() -> +-spec files() -> + {ok, [file_info()]} | {error, term()}. +files() -> Mod = mod(), - Mod:exports(storage()). + Mod:files(storage()). -spec with_storage_type(atom(), atom() | function()) -> any(). with_storage_type(Type, Fun) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index 954423fba..a7d1cd2e2 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -57,7 +57,7 @@ ok | {error, _Reason}. -callback list(options()) -> - {ok, [emqx_ft_storage:export_info()]} | {error, _Reason}. + {ok, [emqx_ft_storage:file_info()]} | {error, _Reason}. %% diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index b328cfd2b..10ca263da 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -44,7 +44,7 @@ -export([get_subdir/2]). -export([get_subdir/3]). --export([exports/1]). +-export([files/1]). -export_type([storage/0]). -export_type([filefrag/1]). @@ -209,7 +209,7 @@ assemble(Storage, Transfer, Size) -> %% -exports(Storage) -> +files(Storage) -> emqx_ft_storage_exporter:list(Storage). %% diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl index ee7a857be..4c64011de 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl @@ -29,7 +29,7 @@ introduced_in() -> "5.0.17". -spec list_exports([node()]) -> - emqx_rpc:erpc_multicall([emqx_ft_storage:export_info()]). + emqx_rpc:erpc_multicall([emqx_ft_storage:file_info()]). list_exports(Nodes) -> erpc:multicall( Nodes, diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index c493a78c5..3cfcebf93 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -216,7 +216,7 @@ t_simple_transfer(Config) -> emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) ), - [Export] = list_exports(?config(clientid, Config)), + [Export] = list_files(?config(clientid, Config)), ?assertEqual( {ok, iolist_to_binary(Data)}, read_export(Export) @@ -234,7 +234,7 @@ t_nasty_clientids_fileids(_Config) -> ok = lists:foreach( fun({ClientId, FileId}) -> ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "justfile", ClientId), - [Export] = list_exports(ClientId), + [Export] = list_files(ClientId), ?assertEqual({ok, ClientId}, read_export(Export)) end, Transfers @@ -463,7 +463,7 @@ t_switch_node(Config) -> %% Now check consistency of the file - [Export] = list_exports(ClientId), + [Export] = list_files(ClientId), ?assertEqual( {ok, iolist_to_binary(Data)}, read_export(Export) @@ -539,7 +539,7 @@ t_unreliable_migrating_client(Config) -> ], _Context = run_commands(Commands, Context), - Exports = list_exports(?config(clientid, Config)), + Exports = list_files(?config(clientid, Config)), % NOTE % The cluster had 2 assemblers running on two different nodes, because client sent `fin` @@ -659,9 +659,9 @@ meta(FileName, Data) -> size => byte_size(FullData) }. -list_exports(ClientId) -> - {ok, Exports} = emqx_ft_storage:exports(), - [Export || Export = #{transfer := {CId, _}} <- Exports, CId == ClientId]. +list_files(ClientId) -> + {ok, Files} = emqx_ft_storage:files(), + [File || File = #{transfer := {CId, _}} <- Files, CId == ClientId]. read_export(#{path := AbsFilepath}) -> % TODO: only works for the local filesystem exporter right now diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index 5b3f56b7d..e1574da0d 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -83,7 +83,7 @@ t_multinode_ready_transfers(Config) -> #{transfer := {<<"c/1">>, <<"f:1">>}, name := "fn1"}, #{transfer := {<<"c/2">>, <<"f:2">>}, name := "fn2"} ], - lists:sort(list_exports(Config)) + lists:sort(list_files(Config)) ). %%-------------------------------------------------------------------- @@ -103,6 +103,6 @@ storage(Config) -> } }. -list_exports(Config) -> - {ok, Exports} = emqx_ft_storage_fs:exports(storage(Config)), - Exports. +list_files(Config) -> + {ok, Files} = emqx_ft_storage_fs:files(storage(Config)), + Files. From 6ad7ce55d2490451eee5be6abe577d59a1d86a07 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 28 Mar 2023 18:49:02 +0300 Subject: [PATCH 080/156] fix(ft-conf): separate local segments storage settings To make things less ambiguous. --- apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 17 +++++++++--- apps/emqx_ft/src/emqx_ft_conf.erl | 17 +++++++++--- apps/emqx_ft/src/emqx_ft_schema.erl | 26 +++++++++++++------ apps/emqx_ft/src/emqx_ft_storage_fs.erl | 10 +++---- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 8 +++--- apps/emqx_ft/test/emqx_ft_SUITE.erl | 5 +++- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 4 ++- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 13 ++++++++-- .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 9 +------ .../test/emqx_ft_storage_fs_gc_SUITE.erl | 26 +++++++++++-------- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 9 +++++-- 11 files changed, 96 insertions(+), 48 deletions(-) diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf index 15c42dcfa..941611424 100644 --- a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -22,13 +22,24 @@ emqx_ft_schema { } } - local_storage_root { + local_storage_segments { + desc { + en: "Settings for local segments storage, which include uploaded transfer fragments and temporary data." + zh: "保存上传文件和临时数据的文件系统路径。" + } + label: { + en: "Local Segments Storage" + zh: "本地存储根" + } + } + + local_storage_segments_root { desc { en: "File system path to keep uploaded fragments and temporary data." zh: "保存上传文件和临时数据的文件系统路径。" } label: { - en: "Local Storage Root" + en: "Local Segments Storage Filesystem Root" zh: "本地存储根" } } @@ -67,7 +78,7 @@ emqx_ft_schema { } } - local_storage_gc { + local_storage_segments_gc { desc { en: "Garbage collection settings for the intermediate and temporary files in the local file system." zh: "" diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index e925ee376..b0e73cda4 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -22,6 +22,7 @@ %% Accessors -export([storage/0]). +-export([segments_root/1]). -export([gc_interval/1]). -export([segments_ttl/1]). @@ -48,17 +49,27 @@ storage() -> emqx_config:get([file_transfer, storage], disabled). +-spec segments_root(_Storage) -> file:name(). +segments_root(_Storage) -> + Conf = assert_storage(local), + case emqx_map_lib:deep_find([segments, root], Conf) of + {ok, Root} -> + Root; + {not_found, _, _} -> + filename:join([emqx:data_dir(), file_transfer, segments]) + end. + -spec gc_interval(_Storage) -> milliseconds(). gc_interval(_Storage) -> Conf = assert_storage(local), - emqx_map_lib:deep_get([gc, interval], Conf). + emqx_map_lib:deep_get([segments, gc, interval], Conf). -spec segments_ttl(_Storage) -> {_Min :: seconds(), _Max :: seconds()}. segments_ttl(_Storage) -> Conf = assert_storage(local), { - emqx_map_lib:deep_get([gc, minimum_segments_ttl], Conf), - emqx_map_lib:deep_get([gc, maximum_segments_ttl], Conf) + emqx_map_lib:deep_get([segments, gc, minimum_segments_ttl], Conf), + emqx_map_lib:deep_get([segments, gc, maximum_segments_ttl], Conf) }. assert_storage(Type) -> diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 5c0a764a2..2abbe4c45 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -61,9 +61,9 @@ fields(local_storage) -> required => false, desc => ?DESC("local_type") }}, - {root, #{ - type => binary(), - desc => ?DESC("local_storage_root"), + {segments, #{ + type => ?REF(local_storage_segments), + desc => ?DESC("local_storage_segments"), required => false }}, {exporter, #{ @@ -72,10 +72,18 @@ fields(local_storage) -> ]), desc => ?DESC("local_storage_exporter"), required => true + }} + ]; +fields(local_storage_segments) -> + [ + {root, #{ + type => binary(), + desc => ?DESC("local_storage_segments_root"), + required => false }}, {gc, #{ - type => ?REF(local_storage_gc), - desc => ?DESC("local_storage_gc"), + type => ?REF(local_storage_segments_gc), + desc => ?DESC("local_storage_segments_gc"), required => false }} ]; @@ -93,7 +101,7 @@ fields(local_storage_exporter) -> required => false }} ]; -fields(local_storage_gc) -> +fields(local_storage_segments_gc) -> [ {interval, #{ type => emqx_schema:duration_ms(), @@ -122,10 +130,12 @@ desc(file_transfer) -> "File transfer settings"; desc(local_storage) -> "File transfer local storage settings"; +desc(local_storage_segments) -> + "File transfer local segments storage settings"; desc(local_storage_exporter) -> "Exporter settings for the File transfer local storage backend"; -desc(local_storage_gc) -> - "Garbage collection settings for the File transfer local storage backend". +desc(local_storage_segments_gc) -> + "Garbage collection settings for the File transfer local segments storage". schema(filemeta) -> #{ diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 10ca263da..f0bc9ac67 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -220,7 +220,7 @@ transfers(Storage) -> % TODO `Continuation` % There might be millions of transfers on the node, we need a protocol and % storage schema to iterate through them effectively. - ClientIds = try_list_dir(get_storage_root(Storage)), + ClientIds = try_list_dir(get_segments_root(Storage)), {ok, lists:foldl( fun(ClientId, Acc) -> transfers(Storage, ClientId, Acc) end, @@ -229,7 +229,7 @@ transfers(Storage) -> )}. transfers(Storage, ClientId, AccIn) -> - Dirname = filename:join(get_storage_root(Storage), ClientId), + Dirname = filename:join(get_segments_root(Storage), ClientId), case file:list_dir(Dirname) of {ok, FileIds} -> lists:foldl( @@ -307,7 +307,7 @@ break_segment_filename(Filename) -> mk_filedir(Storage, {ClientId, FileId}, SubDirs) -> filename:join([ - get_storage_root(Storage), + get_segments_root(Storage), emqx_ft_fs_util:escape_filename(ClientId), emqx_ft_fs_util:escape_filename(FileId) | SubDirs @@ -325,8 +325,8 @@ try_list_dir(Dirname) -> {error, _} -> [] end. -get_storage_root(Storage) -> - maps:get(root, Storage, filename:join([emqx:data_dir(), "ft", "transfers"])). +get_segments_root(Storage) -> + emqx_ft_conf:segments_root(Storage). -include_lib("kernel/include/file.hrl"). diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 258f3a7f3..63b0ab500 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -65,7 +65,7 @@ collect(Storage, Transfer, Nodes) -> mk_server_ref(Storage) -> % TODO - {via, gproc, {n, l, {?MODULE, get_storage_root(Storage)}}}. + {via, gproc, {n, l, {?MODULE, get_segments_root(Storage)}}}. %% @@ -223,7 +223,7 @@ collect_transfer_directory(Storage, Transfer, Cutoff, Stats) -> end, case collect_empty_directory(Dirname, Filter, Stats) of {true, StatsNext} -> - collect_parents(Dirname, get_storage_root(Storage), StatsNext); + collect_parents(Dirname, get_segments_root(Storage), StatsNext); {false, StatsNext} -> StatsNext end. @@ -373,5 +373,5 @@ register_gcstat_error(Subject, Error, Stats = #gcstats{errors = Errors}) -> %% -get_storage_root(Storage) -> - maps:get(root, Storage, filename:join(emqx:data_dir(), "file_transfer")). +get_segments_root(Storage) -> + emqx_ft_conf:segments_root(Storage). diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 3cfcebf93..e365cba85 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -63,7 +63,10 @@ set_special_configs(Config) -> % NOTE % Inhibit local fs GC to simulate it isn't fast enough to collect % complete transfers. - storage => Storage#{gc => #{interval => 0}} + storage => emqx_map_lib:deep_merge( + Storage, + #{segments => #{gc => #{interval => 0}}} + ) }); (_) -> ok diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 7c5d14b38..4b5610f51 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -248,7 +248,9 @@ exporter(Config) -> storage(Config) -> #{ type => local, - root => ?config(storage_root, Config), + segments => #{ + root => ?config(storage_root, Config) + }, exporter => #{ type => local, root => ?config(exports_root, Config) diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index e43abf095..3811c1ef4 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -59,7 +59,12 @@ t_update_config(_Config) -> #{ <<"storage">> => #{ <<"type">> => <<"local">>, - <<"root">> => <<"/tmp/path">>, + <<"segments">> => #{ + <<"root">> => <<"/tmp/path">>, + <<"gc">> => #{ + <<"interval">> => <<"5m">> + } + }, <<"exporter">> => #{ <<"type">> => <<"local">>, <<"root">> => <<"/tmp/exports">> @@ -71,5 +76,9 @@ t_update_config(_Config) -> ), ?assertEqual( <<"/tmp/path">>, - emqx_config:get([file_transfer, storage, root]) + emqx_config:get([file_transfer, storage, segments, root]) + ), + ?assertEqual( + 5 * 60 * 1000, + emqx_ft_conf:gc_interval(emqx_ft_conf:storage()) ). diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index e1574da0d..e3decf0f5 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -94,14 +94,7 @@ client_id(Config) -> atom_to_binary(?config(tc, Config), utf8). storage(Config) -> - #{ - type => local, - root => emqx_ft_test_helpers:root(Config, node(), ["transfers"]), - exporter => #{ - type => local, - root => emqx_ft_test_helpers:root(Config, node(), ["exports"]) - } - }. + emqx_ft_test_helpers:local_storage(Config). list_files(Config) -> {ok, Files} = emqx_ft_storage_fs:files(storage(Config)), diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index 53a22b8b6..90039cd96 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -20,7 +20,6 @@ -compile(nowarn_export_all). -include_lib("emqx_ft/include/emqx_ft_storage_fs.hrl"). --include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). -include_lib("snabbkaffe/include/test_macros.hrl"). @@ -43,7 +42,9 @@ init_per_testcase(TC, Config) -> emqx_ft_test_helpers:load_config(#{ storage => #{ type => local, - root => emqx_ft_test_helpers:root(Config, node(), [TC, transfers]), + segments => #{ + root => emqx_ft_test_helpers:root(Config, node(), [TC, segments]) + }, exporter => #{ type => local, root => emqx_ft_test_helpers:root(Config, node(), [TC, exports]) @@ -66,7 +67,7 @@ end_per_testcase(_TC, _Config) -> t_gc_triggers_periodically(_Config) -> Interval = 500, - ok = emqx_config:put([file_transfer, storage, gc, interval], Interval), + ok = set_gc_config(interval, Interval), ok = emqx_ft_storage_fs_gc:reset(emqx_ft_conf:storage()), ?check_trace( timer:sleep(Interval * 3), @@ -104,9 +105,9 @@ t_gc_triggers_manually(_Config) -> t_gc_complete_transfers(_Config) -> Storage = emqx_ft_conf:storage(), - ok = emqx_config:put([file_transfer, storage, gc, minimum_segments_ttl], 0), - ok = emqx_config:put([file_transfer, storage, gc, maximum_segments_ttl], 3), - ok = emqx_config:put([file_transfer, storage, gc, interval], 500), + ok = set_gc_config(minimum_segments_ttl, 0), + ok = set_gc_config(maximum_segments_ttl, 3), + ok = set_gc_config(interval, 500), ok = emqx_ft_storage_fs_gc:reset(Storage), Transfers = [ { @@ -194,8 +195,8 @@ t_gc_complete_transfers(_Config) -> ). t_gc_incomplete_transfers(_Config) -> - ok = emqx_config:put([file_transfer, storage, gc, minimum_segments_ttl], 0), - ok = emqx_config:put([file_transfer, storage, gc, maximum_segments_ttl], 4), + ok = set_gc_config(minimum_segments_ttl, 0), + ok = set_gc_config(maximum_segments_ttl, 4), Storage = emqx_ft_conf:storage(), Transfers = [ { @@ -222,7 +223,7 @@ t_gc_incomplete_transfers(_Config) -> % 1. Start transfers, send all the segments but don't trigger completion. _ = emqx_misc:pmap(fun(Transfer) -> start_transfer(Storage, Transfer) end, Transfers), % 2. Enable periodic GC every 0.5 seconds. - ok = emqx_config:put([file_transfer, storage, gc, interval], 500), + ok = set_gc_config(interval, 500), ok = emqx_ft_storage_fs_gc:reset(Storage), % 3. First we need the first transfer to be collected. {ok, _} = ?block_until( @@ -265,8 +266,8 @@ t_gc_incomplete_transfers(_Config) -> ). t_gc_handling_errors(_Config) -> - ok = emqx_config:put([file_transfer, storage, gc, minimum_segments_ttl], 0), - ok = emqx_config:put([file_transfer, storage, gc, maximum_segments_ttl], 0), + ok = set_gc_config(minimum_segments_ttl, 0), + ok = set_gc_config(maximum_segments_ttl, 0), Storage = emqx_ft_conf:storage(), Transfer1 = {<<"client1">>, mk_file_id()}, Transfer2 = {<<"client2">>, mk_file_id()}, @@ -322,6 +323,9 @@ t_gc_handling_errors(_Config) -> %% +set_gc_config(Name, Value) -> + emqx_config:put([file_transfer, storage, segments, gc, Name], Value). + start_transfer(Storage, {Transfer, Meta, Gen}) -> ?assertEqual( ok, diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index a38242629..b8ee45b15 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -46,8 +46,13 @@ stop_additional_node(Node) -> local_storage(Config) -> #{ type => local, - root => root(Config, node(), [transfers]), - exporter => #{type => local, root => root(Config, node(), [exports])} + segments => #{ + root => root(Config, node(), [segments]) + }, + exporter => #{ + type => local, + root => root(Config, node(), [exports]) + } }. load_config(Config) -> From bef5cc9c0f3b10ac5d415383d06dff3b2d6b4366 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 29 Mar 2023 14:22:34 +0300 Subject: [PATCH 081/156] fix(fs-fold): avoid folding through symlinked directories Also a testsuite that verifies multilevel fold behaviour. --- apps/emqx_ft/src/emqx_ft_fs_util.erl | 23 ++- apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl | 159 ++++++++++++++++++ .../emqx_ft_fs_util_SUITE_data/a/b/foo/42 | 0 .../test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я | 1 + .../test/emqx_ft_fs_util_SUITE_data/a/link | 1 + .../emqx_ft_fs_util_SUITE_data/c/bar/中文 | 1 + .../test/emqx_ft_fs_util_SUITE_data/c/link | 1 + .../emqx_ft_fs_util_SUITE_data/d/e/baz/needle | 1 + .../emqx_ft_fs_util_SUITE_data/d/haystack | 1 + 9 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl create mode 100644 apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/42 create mode 100644 apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я create mode 120000 apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/link create mode 100644 apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/bar/中文 create mode 120000 apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/link create mode 100644 apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/e/baz/needle create mode 100644 apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/haystack diff --git a/apps/emqx_ft/src/emqx_ft_fs_util.erl b/apps/emqx_ft/src/emqx_ft_fs_util.erl index 75fb47c27..df9135816 100644 --- a/apps/emqx_ft/src/emqx_ft_fs_util.erl +++ b/apps/emqx_ft/src/emqx_ft_fs_util.erl @@ -17,12 +17,14 @@ -module(emqx_ft_fs_util). -include_lib("snabbkaffe/include/trace.hrl"). +-include_lib("kernel/include/file.hrl"). -export([is_filename_safe/1]). -export([escape_filename/1]). -export([unescape_filename/1]). -export([read_decode_file/2]). +-export([read_info/1]). -export([fold/4]). @@ -144,13 +146,20 @@ safe_decode(Content, DecodeFun) -> {error, corrupted} end. +-spec read_info(file:name_all()) -> + {ok, file:file_info()} | {error, file:posix() | badarg}. +read_info(AbsPath) -> + % NOTE + % Be aware that this function is occasionally mocked in `emqx_ft_fs_util_SUITE`. + file:read_link_info(AbsPath, [{time, posix}, raw]). + -spec fold(foldfun(Acc), Acc, _Root :: file:name(), glob()) -> Acc. fold(Fun, Acc, Root, Glob) -> fold(Fun, Acc, [], Root, Glob, []). fold(Fun, AccIn, Path, Root, [Glob | Rest], Stack) when Glob == '*' orelse is_function(Glob) -> - case file:list_dir(filename:join(Root, Path)) of + case list_dir(filename:join(Root, Path)) of {ok, Filenames} -> lists:foldl( fun(FN, Acc) -> @@ -172,7 +181,7 @@ fold(Fun, AccIn, Path, Root, [Glob | Rest], Stack) when Glob == '*' orelse is_fu Fun(Path, {error, Reason}, Stack, AccIn) end; fold(Fun, AccIn, Filepath, Root, [], Stack) -> - case file:read_link_info(filename:join(Root, Filepath), [{time, posix}, raw]) of + case ?MODULE:read_info(filename:join(Root, Filepath)) of {ok, Info} -> Fun(Filepath, Info, Stack, AccIn); {error, Reason} -> @@ -183,3 +192,13 @@ matches_glob('*', _) -> true; matches_glob(FilterFun, Filename) when is_function(FilterFun) -> FilterFun(Filename). + +list_dir(AbsPath) -> + case ?MODULE:read_info(AbsPath) of + {ok, #file_info{type = directory}} -> + file:list_dir(AbsPath); + {ok, #file_info{}} -> + {error, enotdir}; + {error, Reason} -> + {error, Reason} + end. diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl new file mode 100644 index 000000000..81a483651 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl @@ -0,0 +1,159 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_fs_util_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("kernel/include/file.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +t_fold_single_level(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {"a", #file_info{type = directory}, ["a"]}, + {"c", #file_info{type = directory}, ["c"]}, + {"d", #file_info{type = directory}, ["d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*'])) + ). + +t_fold_multi_level(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {"a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]}, + {"a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]}, + {"d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*'])) + ), + ?assertMatch( + [ + {"a/b/foo", #file_info{type = directory}, ["foo", "b", "a"]}, + {"c/bar/中文", #file_info{type = regular}, ["中文", "bar", "c"]}, + {"d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*'])) + ). + +t_fold_no_glob(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [{"", #file_info{type = directory}, []}], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, [])) + ). + +t_fold_glob_too_deep(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*', '*'])) + ). + +t_fold_invalid_root(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [], + sort(emqx_ft_fs_util:fold(fun cons/4, [], filename:join([Root, "a", "link"]), ['*'])) + ), + ?assertMatch( + [], + sort(emqx_ft_fs_util:fold(fun cons/4, [], filename:join([Root, "d", "haystack"]), ['*'])) + ). + +t_fold_filter_unicode(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {"a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]}, + {"d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', fun is_latin1/1])) + ), + ?assertMatch( + [ + {"a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', is_not(fun is_latin1/1)])) + ). + +t_fold_filter_levels(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {"a/b/foo", #file_info{type = directory}, ["foo", "b", "a"]}, + {"d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, [fun is_letter/1, fun is_letter/1, '*'])) + ). + +t_fold_errors(Config) -> + Root = ?config(data_dir, Config), + ok = meck:new(emqx_ft_fs_util, [passthrough]), + ok = meck:expect(emqx_ft_fs_util, read_info, fun(AbsFilepath) -> + ct:pal("read_info(~p)", [AbsFilepath]), + Filename = filename:basename(AbsFilepath), + case Filename of + "b" -> {error, eacces}; + "link" -> {error, enotsup}; + "bar" -> {error, enotdir}; + "needle" -> {error, ebusy}; + _ -> meck:passthrough([AbsFilepath]) + end + end), + ?assertMatch( + [ + {"a/b", {error, eacces}, ["b", "a"]}, + {"a/link", {error, enotsup}, ["link", "a"]}, + {"c/link", {error, enotsup}, ["link", "c"]}, + {"d/e/baz/needle", {error, ebusy}, ["needle", "baz", "e", "d"]} + ], + sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*'])) + ). + +%% + +is_not(F) -> + fun(X) -> not F(X) end. + +is_latin1(Filename) -> + case unicode:characters_to_binary(Filename, unicode, latin1) of + {error, _, _} -> + false; + _ -> + true + end. + +is_letter(Filename) -> + case Filename of + [_] -> + true; + _ -> + false + end. + +cons(Path, Info, Stack, Acc) -> + [{Path, Info, Stack} | Acc]. + +sort(L) when is_list(L) -> + lists:sort(L). diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/42 b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/42 new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я new file mode 100644 index 000000000..ac31ffd53 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я @@ -0,0 +1 @@ +Ты diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/link b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/link new file mode 120000 index 000000000..1b271d838 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/link @@ -0,0 +1 @@ +../c \ No newline at end of file diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/bar/中文 b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/bar/中文 new file mode 100644 index 000000000..2e11eb72f --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/bar/中文 @@ -0,0 +1 @@ +Zhōngwén diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/link b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/link new file mode 120000 index 000000000..82f488f26 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/link @@ -0,0 +1 @@ +../a \ No newline at end of file diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/e/baz/needle b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/e/baz/needle new file mode 100644 index 000000000..d755762d1 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/e/baz/needle @@ -0,0 +1 @@ +haystack diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/haystack b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/haystack new file mode 100644 index 000000000..a6b681bf4 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/haystack @@ -0,0 +1 @@ +needle From 11edfc1c6a9981176f95a9dfb2efca4beba3ba70 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 29 Mar 2023 14:37:29 +0300 Subject: [PATCH 082/156] fix(ft): comment out some clauses as "unreachable" --- .../src/emqx_ft_storage_exporter_fs_proxy.erl | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl index bdb64f543..943c053ff 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl @@ -29,9 +29,11 @@ list_exports_local() -> emqx_ft_storage:with_storage_type(local, fun(Storage) -> case emqx_ft_storage_exporter:exporter(Storage) of {emqx_ft_storage_exporter_fs, Options} -> - emqx_ft_storage_exporter_fs:list_local(Options); - InvalidExporter -> - {error, {invalid_exporter, InvalidExporter}} + emqx_ft_storage_exporter_fs:list_local(Options) + % NOTE + % This case clause is currently deemed unreachable by dialyzer. + % InvalidExporter -> + % {error, {invalid_exporter, InvalidExporter}} end end). @@ -39,8 +41,10 @@ read_export_file_local(Filepath, CallerPid) -> emqx_ft_storage:with_storage_type(local, fun(Storage) -> case emqx_ft_storage_exporter:exporter(Storage) of {emqx_ft_storage_exporter_fs, Options} -> - emqx_ft_storage_exporter_fs:start_reader(Options, Filepath, CallerPid); - InvalidExporter -> - {error, {invalid_exporter, InvalidExporter}} + emqx_ft_storage_exporter_fs:start_reader(Options, Filepath, CallerPid) + % NOTE + % This case clause is currently deemed unreachable by dialyzer. + % InvalidExporter -> + % {error, {invalid_exporter, InvalidExporter}} end end). From 258fabbf8b3b37fa2643af2ed978f6a0f37ab322 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 29 Mar 2023 14:38:41 +0300 Subject: [PATCH 083/156] fix(fs-exp): reply with error when listing failed everywhere --- .../src/emqx_ft_storage_exporter_fs.erl | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index 7cf0ef6d5..647d84124 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -266,20 +266,32 @@ start_reader(Options, RelFilepath, CallerPid) -> %% -spec list(options()) -> - {ok, [exportinfo(), ...]} | {error, file_error()}. + {ok, [exportinfo(), ...]} | {error, [{node(), _Reason}]}. list(_Options) -> Nodes = mria_mnesia:running_nodes(), - Results = emqx_ft_storage_exporter_fs_proto_v1:list_exports(Nodes), - {GoodResults, BadResults} = lists:partition( + Replies = emqx_ft_storage_exporter_fs_proto_v1:list_exports(Nodes), + {Results, Errors} = lists:foldl( fun - ({_Node, {ok, {ok, _}}}) -> true; - (_) -> false + ({_Node, {ok, {ok, Files}}}, {Acc, Errors}) -> + {Files ++ Acc, Errors}; + ({Node, {ok, {error, _} = Error}}, {Acc, Errors}) -> + {Acc, [{Node, Error} | Errors]}; + ({Node, Error}, {Acc, Errors}) -> + {Acc, [{Node, Error} | Errors]} end, - lists:zip(Nodes, Results) + {[], []}, + lists:zip(Nodes, Replies) ), - length(BadResults) > 0 andalso - ?SLOG(warning, #{msg => "list_remote_exports_failed", failures => BadResults}), - {ok, [File || {_Node, {ok, {ok, Files}}} <- GoodResults, File <- Files]}. + length(Errors) > 0 andalso + ?SLOG(warning, #{msg => "list_remote_exports_failed", errors => Errors}), + case Results of + [_ | _] -> + {ok, Results}; + [] when Errors =:= [] -> + {ok, Results}; + [] -> + {error, Errors} + end. %% From 28d87ca62dab2fdf482de96212c31d9436290b68 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 30 Mar 2023 11:47:02 +0300 Subject: [PATCH 084/156] fix(ft-fs): move default config to the backend impl --- apps/emqx_ft/src/emqx_ft_conf.erl | 11 ----------- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 20 ++++++++++++++------ apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 2 +- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index b0e73cda4..aafcd5ad3 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -22,7 +22,6 @@ %% Accessors -export([storage/0]). --export([segments_root/1]). -export([gc_interval/1]). -export([segments_ttl/1]). @@ -49,16 +48,6 @@ storage() -> emqx_config:get([file_transfer, storage], disabled). --spec segments_root(_Storage) -> file:name(). -segments_root(_Storage) -> - Conf = assert_storage(local), - case emqx_map_lib:deep_find([segments, root], Conf) of - {ok, Root} -> - Root; - {not_found, _, _} -> - filename:join([emqx:data_dir(), file_transfer, segments]) - end. - -spec gc_interval(_Storage) -> milliseconds(). gc_interval(_Storage) -> Conf = assert_storage(local), diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index f0bc9ac67..4e5cc9236 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -41,6 +41,7 @@ % GC API % TODO: This is quickly becomes hairy. +-export([get_root/1]). -export([get_subdir/2]). -export([get_subdir/3]). @@ -220,7 +221,7 @@ transfers(Storage) -> % TODO `Continuation` % There might be millions of transfers on the node, we need a protocol and % storage schema to iterate through them effectively. - ClientIds = try_list_dir(get_segments_root(Storage)), + ClientIds = try_list_dir(get_root(Storage)), {ok, lists:foldl( fun(ClientId, Acc) -> transfers(Storage, ClientId, Acc) end, @@ -229,7 +230,7 @@ transfers(Storage) -> )}. transfers(Storage, ClientId, AccIn) -> - Dirname = filename:join(get_segments_root(Storage), ClientId), + Dirname = filename:join(get_root(Storage), ClientId), case file:list_dir(Dirname) of {ok, FileIds} -> lists:foldl( @@ -263,6 +264,16 @@ read_transferinfo(Storage, Transfer, Acc) -> Acc end. +-spec get_root(storage()) -> + file:name(). +get_root(Storage) -> + case emqx_map_lib:deep_find([segments, root], Storage) of + {ok, Root} -> + Root; + {not_found, _, _} -> + filename:join([emqx:data_dir(), file_transfer, segments]) + end. + -spec get_subdir(storage(), transfer()) -> file:name(). get_subdir(Storage, Transfer) -> @@ -307,7 +318,7 @@ break_segment_filename(Filename) -> mk_filedir(Storage, {ClientId, FileId}, SubDirs) -> filename:join([ - get_segments_root(Storage), + get_root(Storage), emqx_ft_fs_util:escape_filename(ClientId), emqx_ft_fs_util:escape_filename(FileId) | SubDirs @@ -325,9 +336,6 @@ try_list_dir(Dirname) -> {error, _} -> [] end. -get_segments_root(Storage) -> - emqx_ft_conf:segments_root(Storage). - -include_lib("kernel/include/file.hrl"). read_file(Filepath, DecodeFun) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 63b0ab500..2ab30e88b 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -374,4 +374,4 @@ register_gcstat_error(Subject, Error, Stats = #gcstats{errors = Errors}) -> %% get_segments_root(Storage) -> - emqx_ft_conf:segments_root(Storage). + emqx_ft_storage_fs:get_root(Storage). From 31b441a46e1edc6340a200fc11d3e2091fc666b0 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 17 Mar 2023 01:30:31 +0200 Subject: [PATCH 085/156] feat(s3): add S3 client application --- .../docker-compose-minio-tcp.yaml | 21 + .../docker-compose-minio-tls.yaml | 23 + .../docker-compose-toxiproxy.yaml | 16 + .ci/docker-compose-file/toxiproxy.json | 12 + apps/emqx_machine/src/emqx_machine.app.src | 2 +- apps/emqx_machine/src/emqx_machine_boot.erl | 3 +- apps/emqx_s3/BSL.txt | 94 +++ apps/emqx_s3/README.md | 133 +++++ apps/emqx_s3/docker-ct | 2 + apps/emqx_s3/docs/s3_app.png | Bin 0 -> 202227 bytes apps/emqx_s3/rebar.config | 6 + apps/emqx_s3/src/emqx_s3.app.src | 14 + apps/emqx_s3/src/emqx_s3.erl | 65 +++ apps/emqx_s3/src/emqx_s3_app.erl | 16 + apps/emqx_s3/src/emqx_s3_client.erl | 293 ++++++++++ apps/emqx_s3/src/emqx_s3_profile_conf.erl | 390 +++++++++++++ .../src/emqx_s3_profile_http_pool_clients.erl | 35 ++ .../src/emqx_s3_profile_http_pools.erl | 123 ++++ apps/emqx_s3/src/emqx_s3_profile_sup.erl | 48 ++ .../src/emqx_s3_profile_uploader_sup.erl | 73 +++ apps/emqx_s3/src/emqx_s3_schema.erl | 143 +++++ apps/emqx_s3/src/emqx_s3_sup.erl | 47 ++ apps/emqx_s3/src/emqx_s3_uploader.erl | 318 +++++++++++ apps/emqx_s3/test/certs/ca.crt | 29 + apps/emqx_s3/test/emqx_s3_SUITE.erl | 66 +++ apps/emqx_s3/test/emqx_s3_client_SUITE.erl | 104 ++++ .../test/emqx_s3_profile_conf_SUITE.erl | 293 ++++++++++ apps/emqx_s3/test/emqx_s3_schema_SUITE.erl | 154 +++++ apps/emqx_s3/test/emqx_s3_test_helpers.erl | 135 +++++ apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl | 535 ++++++++++++++++++ rebar.config.erl | 3 +- scripts/ct/run.sh | 12 +- 32 files changed, 3204 insertions(+), 4 deletions(-) create mode 100644 .ci/docker-compose-file/docker-compose-minio-tcp.yaml create mode 100644 .ci/docker-compose-file/docker-compose-minio-tls.yaml create mode 100644 apps/emqx_s3/BSL.txt create mode 100644 apps/emqx_s3/README.md create mode 100644 apps/emqx_s3/docker-ct create mode 100644 apps/emqx_s3/docs/s3_app.png create mode 100644 apps/emqx_s3/rebar.config create mode 100644 apps/emqx_s3/src/emqx_s3.app.src create mode 100644 apps/emqx_s3/src/emqx_s3.erl create mode 100644 apps/emqx_s3/src/emqx_s3_app.erl create mode 100644 apps/emqx_s3/src/emqx_s3_client.erl create mode 100644 apps/emqx_s3/src/emqx_s3_profile_conf.erl create mode 100644 apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl create mode 100644 apps/emqx_s3/src/emqx_s3_profile_http_pools.erl create mode 100644 apps/emqx_s3/src/emqx_s3_profile_sup.erl create mode 100644 apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl create mode 100644 apps/emqx_s3/src/emqx_s3_schema.erl create mode 100644 apps/emqx_s3/src/emqx_s3_sup.erl create mode 100644 apps/emqx_s3/src/emqx_s3_uploader.erl create mode 100644 apps/emqx_s3/test/certs/ca.crt create mode 100644 apps/emqx_s3/test/emqx_s3_SUITE.erl create mode 100644 apps/emqx_s3/test/emqx_s3_client_SUITE.erl create mode 100644 apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl create mode 100644 apps/emqx_s3/test/emqx_s3_schema_SUITE.erl create mode 100644 apps/emqx_s3/test/emqx_s3_test_helpers.erl create mode 100644 apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl diff --git a/.ci/docker-compose-file/docker-compose-minio-tcp.yaml b/.ci/docker-compose-file/docker-compose-minio-tcp.yaml new file mode 100644 index 000000000..93e1c4ead --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-minio-tcp.yaml @@ -0,0 +1,21 @@ +version: '3.7' + +services: + minio: + hostname: minio + image: quay.io/minio/minio:RELEASE.2023-03-20T20-16-18Z + command: server --address ":9000" --console-address ":9001" /minio-data + expose: + - "9000" + - "9001" + ports: + - "9000:9000" + - "9001:9001" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 5s + retries: 3 + networks: + emqx_bridge: + diff --git a/.ci/docker-compose-file/docker-compose-minio-tls.yaml b/.ci/docker-compose-file/docker-compose-minio-tls.yaml new file mode 100644 index 000000000..2e7a6bea5 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-minio-tls.yaml @@ -0,0 +1,23 @@ +version: '3.7' + +services: + minio_tls: + hostname: minio-tls + image: quay.io/minio/minio:RELEASE.2023-03-20T20-16-18Z + command: server --certs-dir /etc/certs --address ":9100" --console-address ":9101" /minio-data + volumes: + - ./certs/server.crt:/etc/certs/public.crt + - ./certs/server.key:/etc/certs/private.key + expose: + - "9100" + - "9101" + ports: + - "9100:9100" + - "9101:9101" + healthcheck: + test: ["CMD", "curl", "-k", "-f", "https://localhost:9100/minio/health/live"] + interval: 30s + timeout: 5s + retries: 3 + networks: + emqx_bridge: diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index 9a1d08ba6..0cf689921 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -13,18 +13,34 @@ services: volumes: - "./toxiproxy.json:/config/toxiproxy.json" ports: + # Toxiproxy management API - 8474:8474 + # InfluxDB - 8086:8086 + # InfluxDB TLS - 8087:8087 + # MySQL - 13306:3306 + # MySQL TLS - 13307:3307 + # PostgreSQL - 15432:5432 + # PostgreSQL TLS - 15433:5433 + # TDEngine - 16041:6041 + # DynamoDB - 18000:8000 + # RocketMQ - 19876:9876 + # Cassandra - 19042:9042 + # Cassandra TLS - 19142:9142 + # S3 + - 19000:19000 + # S3 TLS + - 19100:19100 command: - "-host=0.0.0.0" - "-config=/config/toxiproxy.json" diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index 708cbf1ef..d8dd8a166 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -95,5 +95,17 @@ "listen": "0.0.0.0:9142", "upstream": "cassandra:9142", "enabled": true + }, + { + "name": "minio_tcp", + "listen": "0.0.0.0:19000", + "upstream": "minio:9000", + "enabled": true + }, + { + "name": "minio_tls", + "listen": "0.0.0.0:19100", + "upstream": "minio-tls:9100", + "enabled": true } ] diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 0bee30e35..6bd36aab5 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index 82b3d602f..feeb1ba75 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -146,7 +146,8 @@ basic_reboot_apps() -> emqx_authz, emqx_slow_subs, emqx_auto_subscribe, - emqx_plugins + emqx_plugins, + emqx_s3 ], case emqx_release:edition() of ce -> CE; diff --git a/apps/emqx_s3/BSL.txt b/apps/emqx_s3/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_s3/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_s3/README.md b/apps/emqx_s3/README.md new file mode 100644 index 000000000..3f049d1a8 --- /dev/null +++ b/apps/emqx_s3/README.md @@ -0,0 +1,133 @@ +# emqx_s3 + +EMQX S3 Application + +## Description + +This application provides functionality for uploading files to S3. + +## Usage + +The steps to integrate this application are: +* Integrate S3 configuration schema where needed. +* On _client_ application start: + * Call `emqx_s3:start_profile(ProfileName, ProfileConfig)` with configuration. + * Add `emqx_config_handler` hook to call `emqx_s3:start_profile(ProfileName, ProfileConfig)` when configuration is updated. +* On _client_ application stop, call `emqx_s3:stop_profile(ProfileName)`. + +`ProfileName` is a unique name used to distinguish different sets of S3 settings. Each profile has its own connection pool and configuration. + +To use S3 from a _client_ application: +* Create an uploader process with `{ok, Pid} = emqx_s3:start_uploader(ProfileName, #{key => MyKey})`. +* Write data with `emqx_s3_uploader:write(Pid, <<"data">>)`. +* Finish the uploader with `emqx_s3_uploader:complete(Pid)` or `emqx_s3_uploader:abort(Pid)`. + +### Configuration + +Example of integrating S3 configuration schema into a _client_ application `emqx_someapp`. + +```erlang +-module(emqx_someapp_schema). + +... + +roots() -> [someapp] +... + +fields(someapp) -> + [ + {other_setting, ...}, + {s3_settings, + mk( + hoconsc:ref(emqx_s3_schema, s3), + #{ + desc => ?DESC("s3_settings"), + required => true + } + )} + ]; +... + +``` + +### Application start and config hooks + +```erlang +-module(emqx_someapp_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +-export([ + pre_config_update/3, + post_config_update/5 +]). + +start(_StartType, _StartArgs) -> + ProfileConfig = emqx_config:get([someapp, s3_settings]), + ProfileName = someapp, + ok = emqx_s3:start_profile(ProfileName, ProfileConfig), + ok = emqx_config_handler:add_handler([someapp], ?MODULE). + +stop(_State) -> + ok = emqx_conf:remove_handler([someapp]), + ProfileName = someapp, + ok = emqx_s3:stop_profile(ProfileName). + +pre_config_update(_Path, NewConfig, _OldConfig) -> + {ok, NewConfig}. + +post_config_update(Path, _Req, NewConfig, _OldConfig, _AppEnvs) -> + NewProfileConfig = maps:get(s3_settings, NewConfig), + ProfileName = someapp, + %% more graceful error handling may be needed + ok = emqx_s3:update_profile(ProfileName, NewProfileConfig). + +``` + +### Uploader usage + +```erlang +-module(emqx_someapp_logic). +... + +-spec do_upload_data(Key :: string(), Data :: binary()) -> ok. +do_upload_data(Key, Data) -> + ProfileName = someapp, + {ok, Pid} = emqx_s3:start_uploader(ProfileName, #{key => Key}), + ok = emqx_s3_uploader:write(Pid, Data), + ok = emqx_s3_uploader:complete(Pid). + +``` + +## Design + +![Design](./docs/s3_app.png) + +* Each profile has its own supervisor `emqx_s3_profile_sup`. +* Under each profile supervisor, there is a + * `emqx_s3_profile_uploader_sup` supervisor for uploader processes. + * `emqx_s3_profile_conf` server for managing profile configuration. + +When an uploader process is started, it checkouts the actual S3 configuration for the profile from the `emqx_s3_profile_conf` server. It uses the obtained configuration and connection pool to upload data to S3 till the termination, even if the configuration is updated. + +`emqx_s3_profile_conf`: +* Keeps actual S3 configuration for the profile and creates a connection pool for the actual configuration. +* Creates a new connection pool when the configuration is updated. +* Keeps track of uploaders using connection pools. +* Drops connection pools when no uploaders are using it or after a timeout. + +The code is designed to allow a painless transition from `ehttpc` pool to any other HTTP pool/client. + +## Possible performance improvements + +One of the downsides of the current implementation is that there is a lot of message passing between the uploader client and the actual sockets. + +A possible improvement could be: +* Use a process-less HTTP client, like [Mint](https://github.com/elixir-mint/mint). +* Use a resource pool, like [NimblePool](https://github.com/dashbitco/nimble_pool) to manage the HTTP connections. It temporarily grants sockets to its clients. +* Do the buffering logic locally in the uploader client. +* Use `emqx_s3_client` directly from the uploader client. + +In this case, the data will be directly sent to the socket, without being sent to any intermediate processes. diff --git a/apps/emqx_s3/docker-ct b/apps/emqx_s3/docker-ct new file mode 100644 index 000000000..a5a001815 --- /dev/null +++ b/apps/emqx_s3/docker-ct @@ -0,0 +1,2 @@ +minio +toxiproxy diff --git a/apps/emqx_s3/docs/s3_app.png b/apps/emqx_s3/docs/s3_app.png new file mode 100644 index 0000000000000000000000000000000000000000..372f16d371d0b75b7d796736906eb1de004d1deb GIT binary patch literal 202227 zcmeFZXH-+&w>OFi(nOkopi-2k^xk_7q4zE|5K0Iw^sa&;#i*ck1px(VQluBfhJXg7 zN>xOp_YQXjlIQ=NbKX0~JMM@3=?sS<+1Y!oHGi|uHCOH!=xI`tGm{e#5Kuz2)QkuS zNUsnO5U-M+0asdMhN!@wQ+`I8Dg>q7muA7YM*iv+{$4>axTgyNr>N@5J5CWHH*Y_G zPEj>Z5fKL@QotGR;Ogt(!4a2VXh`Q#oEArVo2AyIx2NfTjdPElnc zA@CwBE+8!?WOZ`CgR2VyH=wwH5Ey_**TD&n@b~s};}lf^pCJf;7f_2eP*a7C?3;)*;SbALi9b8Xn(K9p%4${%o3h|J1zv1O8 z>SAhOazX>j#n%t+jlhwN9T3cqeb_%3>2h+_)!WM%&8{KDJ63JDulhFa`LKn{9fw#x&-bz0O4dBI1X(qKP@p)BTao1^B_%6RaHGv zEq}|CDJ9KN2A<-6J{kt%zJ3A0PRa(7P(MEr4@noOv{aBoh>O06k6W;trg^ZAxtOX6 z(gKXEqN}N?>#8dx;j8Yh8v+kOxGB2=MyO~3gkbJYe(Ks@KGLFy5U7(83S16$)l=3+ zC_6~1I{Ls2)!-pIFimi$u6~e?YoLf1(p=Ll$W#NV>}%l$fr)D%bu@fD{hU-pObi{I z{4BlQrS-jp%$>Cj5g{&q`s#j`(*9n4;%Y|XK45Gw7j;w9Krwv-A9G)6gFtaHGYxNP z2|Y)VkYFPhBWF{TvWT;}uuo8su$e1%DaMYf%FY2UUV18U*8oc^OUV!^e?3V(Rb!Dr zA5mo_xJ6V%%2dnG$z0viSk%Ws&A?IG00s$=3XqmU`6IPN)x}g`a4E2Sl$M%@2+Bvw z#Y@=)h=z)}xsjDg5F9FQ}{LcHCL{LQr`#l@8o9^yJ7Uiun-M&M2l zL+ofmaA7AEEmdI;LtPV;iH@qSsT9mBSYKHRr5)hzBLd*m@G#X1Hult!P_xh#LZKub z4Sc=4j4aep5F;VUARtUYEkdM4MAT4TqQPdGfQu^Pz81<3ZYT|eTCfNB2vJit^KcXO zkkSoAB24t5x_FcUmx9?K-m|f;_4u3CTK+13pR9w8V5_5 zn3<^uYnbT)-BCe$d1;uc2m2U02L||>IEE+-`DytC8k+?pG{s&0oz+4DlywYDE!3rT zJWf(p6^Mjb z1_2UGq$RcCT81zcLnA$54K*uaKUaT!M;(|p+}zw3p{nZb{ryD%(pGM+BF^3hfC1*tC_`rmOvS@fKR8%f z+eO_$OTpedqa`N{^nvq!Y)oQuzr}Xwh2VbU)cxgZ>Z_x z;%F>|3{rN}azW}_7@4cMqf|ZJ4TQzi1J$6KC^chKHDJtD5SCgFLTXl0zU~s9YUWU2 ztu>59)DXJC`tC@L5LY9Fr;nxu+)_GFTTMs{A`M4~>$+HJS*od_y!E^df|Yf2L_9+f znue|-mfjMou7S>uAzogt!VWI3l71l;8m5|hU_^6CEeCyNVLu5q4?{04LuaHJ(%-_< zM;sL*Ze#>E)fX{$u+l`Rn+1e8s<|1NhUmFyYeJ-?4Pg$ZzD6iVD-)QHyEaVQR2eB2 zV&vhV07y*O9~r{1h@tp>OmagBEG5`rqWo> zqO?W)AVQAjo@!QFMt;)f7Lt0RlB%9kJ`PZ6X{dyzi-(1S4^T%HlR)f(U0p=g9Ka`) z5N8R0QJ8M9G0IdA;pyq33A4aLqyzQPH}KX#X#;ef%*BM&v;rJ0#WZw~N=3}V$y3A4Kwr`{zzPNNbagjSg@XIke4M>uCMbjuGDOT=!pSfQ=IbfuVc;BW zgmQ2&PjWU>-e`kX!wtThKju(E7 zI+vQA7?MkNLIzt(MD!_DjY}hsz2&Z@bq198caN98O0|8Kmuit8E5qMsM^WQl9Q->Q zDy?AqT=7YjkT~H^Ir>wXm79%m*gcUHaqK)aXb z1g~74Q@gC{!WDFcru4Z-hNgOYw~vQcdO&cCw2=vqYBU4UxfbNM9p%msN~30n$#V-; zZRwpxQLPQc(L}_aO3OPHbQ=qOvu2532OoDXg${iSN9dK4*^r;`gos*82QbaC&xSM_ zM*Jso`8{s-dcyNpOa>mj#tBQ*RYJ{kOc6!-k(N@U^?U^ia{(?x#G1a)3q!Q!v&L~P z>)G#hx8>Zo9aKp+>*TBrzgppWXx|M$3yNb)IR^-u{doG-@eD}SC>0{fRjukH{LkMCKW`F&h>EO@qpNJd-5V?*|PvLlxYizyGm=t8U6DDY# z^wc}K^pj*1@0F<5OT@Pm&!xdXPHRxg*KeIc{-bP`47o@5r{}++Sc(2bZl}07s$tcl z=n>#%HF~-cKdkhzBxjf(vQh-P6*kq(mR`zvD3YeiJl`DkYA9swZP}lZ7rl>+7vDh7 zY_iNp^r^+;q)veR>k(48_}#B`peh3*+H%K3Ldb<0v-2eeR8EvIx{v%Uvs}$VGUa$! zdgru^XYHq#RuM1sUU>0Rs0u?FSM2eEXq|r==tb3fo=%u^&45*RD6N+y? z%;tSEeoWuiwN6Wa!oh9|ux5o0qCbzpnm^ELOC7KI4yi$W3aKcgGH%UP6qlr(x1}J( z!WUJBMb-DHA77&$eZ&FZ{~B#VwbFbdu>{ecKq)_(HM{X+<&Emtz_A(B1AL!n_z>YF zMvCj$;z>&UvdeCm~eFRfN@!X8C%U3?nx1Y1HZY23)Y=MR9?TJkxqrqwOC2F7tO0;BcP$0y11zEYr`fweX z%cq2L=hShASd!@3hUL~^$Yp*^MbkxP8e&2g(lBwCbVRuGJ$0FgKdlO*a->hFafYb% z6d3Cq$9?L?^I&k7VKQdyWVwWZ`0K4d4RC^iRGo;YxP2fA%&(Mlnw`6Bj}3xI{XA(9 zo6ygh8vz}*c@KML3hhF4#&7+98T%vq)rbeHl&3cbEucoSV9UFZ5vbi&9eQNZL(2xoK<58(RSd^(Y1jTypfjP8eMyoB8)I2~Q; zjFt*JZ_oDiLkXSe$*4nEtFf^EEu;k)=k2V>XPQ_Z^5{mX+{82vUc|E<`%qgxEn1TB zEK$eqN=bIucR?sLIUB|PWUqoS3}MIe_H$c3o?IQ8E- zz6+=m7DUvcThlloV82`O+p&~BlG4ItK8HnM8W z8xRmNlk?}8f7*0FjKu^uHE?1~0v_@gdU5ktBEW7eKV9iWV38V8hjF4WIH;+>C_#_? zRgUlHu?CP;&Pq|^AhV$XkUi3&jY|fQ?LH20JQ17j3TEa!+ru8KY(n={}SF% zmy!a=w7KrQ)dY}*e~}SB88yWV5IfY(`fAG>5HP;O9fD<0G=_*rji)Q239pUH;GxSp z*V%)w0_-GTaN%S$lLoL@=Q`OtRWJ$*S5^v687|`nzKcFQd8Op>tf1QW~+?Wth49KYV znxi*fMh(D2BoDN;#<1Awz9c(;!bVr%OQjxY6RsEl$l@T}&c~2d5(CIGl1}3xqX&@L ze&X=i#%gG?B{%G3)c;qt1|H*o)%t(c`hV5>|0lJ!dY%%oeJES)q1eLA%>2?{L(kX? zsBAI+=N(*VmD>*tsy1EoPJQS4_p}>DGiNeh4!jSC%06zQDXIj<>w2uO%&}G%0^LW- zG!ez&{8I-o7#8=?ihK8hRvqM#J+DIGeWJTztNCopF6I`7vWt~zn~RY*8RpAV6fZEv zKa6fY%W;PbXS7=Ru+xUBdcOX|oW8u6A+!3}*uSr$kRf0>Sy6AG$n`eyUg?Nmo~6QD z{lj^>CUJzPf#;9V*IsvJSY_F%L9jT{bnFhJlvaQ(J@4I-jkl#mz?LR))!Zv*sb4}} z4*z4v&r&xy1)Ytabn0_>daF}PHm*-4D!SFG$X5?%1dE;nLNWfG*VML`4+5*EZx<6Y z)~`*@RR5NdPZtOocr`FvrpbDK|8Q}pCAI&;=B6Dmid){N2#!J2#s+~ze5ikdK&sh!8iD{$WJe4Gj_N-r==+T!4LT7 z+8doyHovwKBtwTryc@L(FNInx567D;e5Hj7^j3{~w;U84?0G$9%!#y;;BEhC&TaN; zHZd`Acek@gd?{>!)^c+)qpYP*<+AJx-3$rSgs0iuj%?-GyEgbB>N5@U#gU=j{)*Gx z*%mSLc>C~&FN_P_CiN1;MsB)y#~b{eah(gaID+3-XXLHBVRn!EAkOijdo9&L^6-%F ziBH)A&KfCLsCE-4eVpXZ;cQ>~6)%SzK4F&LH*8w_eNlORZhcVc;?|hT0o6q*YoE3o z=f+f*^nvTlQ>}4uYM$@do@m; zIn!^pyVGOja$xU`Bt>}(eeve-gYS&HADBMOJ(*1IJo7iz)?a2J?olo+aU{BLjPn>ET2BNlYkobYO(AN7@;)(e3x*zTQX-;ep$=w_1jWT z*m`8c2;0OoV^*OoqM0kcr)G0aFn3X2q7UL~mpg4{@d7^c^UOT!m#}i59De_cHys(eH{RMy8;-og zc%|ffc=%}eTmWE5#l%`uc*I!I)^C~V`GoYlA<2JQ62@4LLV9QH<~~$;jN27v%bFDC z1m!m$t|sL6IF4u+m$ICv_;K)uf<8Q=raiv%FgR(kamOlLA=t?HqrBvxiDnKG_EoXi zKkvxkg?&I$;gR8G&F&biKupAe@HgVu6CPY(Vt?lioa#S)S5&3FS zK47S2>Dv;+PI&L!H>GUKdi7frw@6t1-&NHA#k6&0_ueJZlRMZi7*Ke;T(h}U`b1qaxr9$S?!}1>;`C?aZ$?(eZqa9o)aNrXzf+|8ucR}XDI0o zIekgW&qxiYiz&_ z|A700X6xC%)+}f3TBgt~b1|>=$s=kkFHN0(wnFzKsK>bS6s81mmIx;9SA|o>NaD&L zM@3>DkJfMeUVqZRD?c#4p=NONATFC%f}Z`|NN7jm(Kq(J`LCX&OA&`2LpeFG#vNwD zcRF0Hf7ShbI^*oc7YNWVwb@9{J2DJ6?mmvZCrSbOi@#Gp;(dx%bjdlQ;fI+AHA8Gz z0z^rUpy3kZX1xXBWe&;m7rv<3TTiA97_!GS!EOKD6nuS>x z+cs=6%fhOX6kkRPNx?fVP@N48NE+Cl+*?}DX+U6RxmoDv16!YJ&b{kR7sT+?RFEtk zXeI`&PPRt=5f6MhVm0a#XgKuc{0A0kzuWPzyIX7;77u17X0$12Vo~i@Oycq4x$NAn8I3=W-)@nS zmp7EWdC9`E6Ebvjo%M4;@ox0G;!C~HWXH;M36$sBCV$Z~<1)+h+$rCqTes78P>hYE zl&s^3`c0#(m4PFQ!}X7%-fp)_i-{CJP|z%fZ3n-q(@QRRTSCpw{PFelDNlHu+iTg? z1qz>$>Kv4VJ-Yql4IE7QCTqazJ_8Dq`d8 z%2=CohS#UJD+)7tGafxf^D~!NhrbGlePQ-<90Fm_NFh-y(RYD;{04sd$~ZNaJX_ z|40s)7fDa|yWs^jJ^eM8!+!ZN`>#s3^{_S{CDr^uR!_{dEA-gb+7-62gc~fUcip7+ zT2Z;_sHiwn;*?R+Vx}IL<(tfCxa74r+m4Orw$+2_e;ZWZ?w#@OQIY?VrN1z3Pxs8Nb|4n1s}?BD$EI zJe8XPp@@9zLs3VDB51dw>SnoD-gIBKDIASEC4>iM&nq7?QB+lIBj^X%sCcR16oRZG8~A1@xERsQiibdtW7 zs8dz#bZQH51rdkAw|yxo+xjYNOs(RVCzc1Qrml~|;IT|4jql{z#>vx0ig@1Kq=K;t zIUoh_!6-e2bM=uSeB#mO4J)exKC8P9Y4Yd3N;BHgi@nAi!>kyzZVyNUG1O1itk=K#S&qKl*7FQ_(7U80ec`HD+}{@TRR^1e8b#YZadTg*s`uv z?)U=LtAKMxgh zy_)c8cXh2NCWfjZwo^j3$InUeg_{{s1Mla>q`MNlTII4uo)eWIzOP0Ws>9%B&u@`B zDejlPC}NM8%Br1L{8ooohW$I~{Ob5bys({ivxPa>sDHdVK7nJAogS4Qh%8mS? zi(h3e#=2mFdmlKD&e^@JqJBEq)3d@vl^wp|#9u#1`$$AQRPG=yy*xF*XqVNc!iO4b zL!t^nz#fzy)Q+n*MIVtl`>-w7-iZ8u9ol8>&s;kWr?N#3CAJ+JvbRra=H$0rQuOoc zb4wtnneW^*=hk^LUHzM!#*^77Bgwz#fQmAe&kbSiQ}N_A2M8`xdSS9%b#dQ62C%Shd=0D_h-ms^(&8-*}R&+@R$3cRYVvVW`R> z($Y8Ea<2AMhc2K%aJ61 zvq#FP?mnW}(|?rSkz9CWpxgDiTm2|cL^0_Yn`La2-?FM(h12n`_mmh&2p1^!A2H;} z#>1YQnh!+BT^9ZDL*Vs1xs_{^U7G?x?EZ($XL(6kb$0NoM^^FFqu;B#Sodag%*SM` zFE(fF-gwo3S&C_qG>=Du5aMrsG)a%|j|2V0Z=Z`_8CGw-q#SPAd0))DF<)-KdE522 z;+Q|;qEJ}SuQ~VLuNrT2f_SzYx)#d~KIpx+JxaR~(eDoJvAOQBbg22qd&b7Z=c4PE zZAe+;cBY2BtWD3kNS&YKLDjRFW;{MUKl2`SPFwM-ZTJ43BmN)(@_^U;gCZBZyduZ^ zWD_rKx(PwwQ3lEz`p9iaojM)8VK9@9H*R;781`5nz4Q3c|MJLjrWD1laxEuEUwTt$ zG~(x%X}0&_a=L3@dTWecO0(!6JuPTd&&oX1G}M_t$rhAA{h!dA2$$?F+p!g&j{cU` zgZj|j>!fhDxm@7KSa@h-#KANpfAcS+zi*bm#6-C>Tp6q+uW4+rRe{a&iXEsvP(X(w3jSy#bTIL?mzQm9fh#t^bTM@D81LQZTi|gW4+ze}6}l7`Ut{%fI!Qmz<)&6Fn&@AOX4P zH9_u^TvP{0=|4M6VAHl9>xsO$PGuOj){w-t6wsT`NB^UzjTP9xxkO zxU_H6r9yPk|8<+>2utK4KYv5mfaNf+@f+*g*g#@%$)+%`>4$nUKgmsyV{({?5Bv%Dyl>OALHG< zEPE`^svG>zM(9+^KF0R~jBoXvj&Y$OuN4+d#4L_@%dKTH+B}U2y=-B&r;*J3qUoql zQfjke=DkN9y;=IS+{zp_F=?wLQx%jlW~9olyZ7GuL4;989ZVq5VdVAIY=fiG+H(U~ zP$471^Og~~r-`_wmOOq*8-CI^Tds} zNt}Xa%B%dzSw#KTHU1xPnjv3OeYvolGyA~<1vlS;F9Xl1L)8YqP=aVa7akn*dy(fl zXM?PFb@N!z!nFPDT@OUCR)Ro{TJL0k&DGr{+Wm-wtmIF$!0!I}ZPV8aGP zgTwL~h|~+_5Rp}UD!WuJ72oIC0DtnP-yQSk968_**v28l^M~%^s*?S_Uhu{3O69Vg zh)kBVA?hR5wersrL%heNr#U#g~6=|8GK1#%3*JPu=qMg~|`7T&l0hd^{uNmD?*} zo$8pi5%LKJzpydC%vD>Y^mel%Cc3Nt#*10;gby`E1i)NmlG8l%=xS+=QI6KGI@nA4 zP}Rm2S;Uha!tzDQUVfua0_OuCUQL-k)RbO%pZo%K)X_a^17e);2i0j_yO+Fka=e=Z z-W+e#jgk3=^5RESGoW?BWgPXJ#s#G)RcQe}jc*)_9Ga>1e4Ls$L46 z;>1=ahQj`@(w2?r*1e@>^+J}&cb+%2N6r#|$o4JW^=w|?qux|+p}b3ZVZC;OX8bUG zT3u$w`vXy|OPStdkxq{syL2ok%VScyX2(e}%fJ7YmGifb zbi0tFrdqK-%i$R?mv~mETQm~JDvd>XKTrqTKmBdjlY7%)3OsT>G^h3ZX+R)U_dB~` z_IcnqbK$|R7tMK2wG5`Cai2o?dws>vscq#?ggu`Sk35Su@d!M6o7BJKdJvGh6;JtQ znD+e0;v4Hahx{q2Gla(X&oD>ht7xRy0**>i*P>O|VS;UIYFOyXPGNN~VsExFp%h;K zJ8|IbcTlc)KD#B$5~Al3u*9^_l0(@Q5xKQ?gLik|^C12d$)W_btt=~WH2dOgnd>>j zTJioD@rj43>S3nxj}J9XrKOpU!@i#&?32p2Z&$_j9`qC2Tp+EAez7&i!g@9gTQX_$ zBpmIZFSQYE?D0{-74iJQNP-S&Cd$$!ps>4tbcy^d$2Xu zkLlzqG_8O2VrRIzOyfd;Jjvhj7KEbXQZJ0@=Sk29SpW6pdL9^#@^>?K@P=EjYsCdDr`fy+8=xNOk^zZz- zA{zZ$=9n4*&?qvY!f@vl&T3u9mZq;Xn^BeKQ@1T{5BXAizvQp2!?hnaagS)2tS!qUB&XaHeX@v{ECIpiUqlFlH1 zhOie+HjJ0&4p1cLF}pr=5Vo`b1O0Mog>er73RNZN_fCo|Y=-fz4Y*BiqDDqQQ;m0Y z5%nYx_HTL(3+5vUKAT+8#R8X0+FY}Xp34fB0Rb4WWI98brOO|EGR8y+Shs&ce_Bru z|Lh)1WW|#9E^CK&1?$N=&tF+)!S&6YB6>^{y+uIC#9+@R^!8KU?1`WRT~}g{DZIn& z$&KA(y{{2-e7lxb6cY{Yf2s>9c3`8XH{+t!Z6yytc@5XQIh>rONPvxKcqi$Ca&8cK z*I`}3{XeH(!tVPY@)LlTxC^9TkgIMoC--Dyy9UHoZZ6yeTgu!RE$ZVz$v_a1LM@ht z?+aQ7y$6rSetZ=nvZ4bZiL=`shk9Vsyb5f+XjyBxQ4w~4E(V^lyYeS;nM;kO6i1*S z*bT;RkBIQKD%szDpo#0eTQ&jPCjTeGRc_wQt^23-%p7datpC29%KkMSY~{QnTfr@z zG+!6&Xb+31gi`Z>4H(T8vw2){3flHaaB)>@D1fagl7b+`0lX2-luJ)KUKYqYLF0g~ zoP#WAK6`sdjP|&1Nl675B@Uu47kpIK1fVXEl&m@%-4A+AJayv-@M`A;p2wup499{E zr**}bH3c!DpfmKtNo11I~jo{tXt3kSxDamQ@tzc16TL89@ zBL)Jtx8My-Dp>6D$vyhmMiG$eqf`Lr>EIls*g>Ojg1sCq()7-gMzScS5FnyD-Uv4) z!dGNM)Pw2NWw>q-rD32U`TSR_NTR2}NdMX8Gl*{u{kIx_-K!-a_@8Qa^tSgMT@4-J zSEklg<>To%I6%vY%2%5+INUJ20O}Aw)lO1&R)uo8SsQc@7K^_>$%`zC_cXvBl8 z)LtW;(OR|y_NHfudy@~_YUjJ@b{s?~$rD|0FN>2Iz;`p6kX`a=D?j?}TX!yQ9s~87 zdBSVzl_yNf_O-J0UQXn9`oKgZg`Av_S>bTT_J8`iKgIw%$me`V;)+lMu&rbCC&u7^ z_V3|(YZoduWUTU?LP@T2a1i}CzwQ26Y;^bI1quY{vJJ1VW8yT#?Epo2083@l$+NP3 z1^B2m9lB$`u2c=oWl<`}4Bk0^A^&gftLZ~hZc>EJWQ_-tX(3^KvY#P=wj$;FisMt< zb~r#j7JxN-{!TOleRQ#SNBuv0?@r&oCnZs~uUH>osS_URA0VWnqSC4x?T|O#`-Uus zxA2D#wsU{5i{}m=4IAK^CapB?d|(8wt=TzU>eyetnnRij7ZEs{O%$!g1$cXY zY?6y?sQKICl>Ftr@>`l^-HG8njk&m4(Y~*bZ){B|hu>J?kC?)BQbc=#VC5rh%uO8N zf_FBcIrP7FkzJ>L)muNxRqVd=Tt=Fi(}8^=sg>kG^S9Wm<5f}bsV)x|X2{;BA;Ya? z8`~iq$48e3#bW)LtK!5fVApGevB4KnjQFW_lZS&>l7^#~#Ei3end73!_FcE7!o`Bg zJ4KdJ_{L}*>;j#K-Ksu-(d6_Lp8g-(5(UWqOJ}$cZw&MaCj%q#vxfVt;6TItg)mFB#1>^YsWp7Z^aV!Y=(L-ecx&9L}YFVw5o zR$Aej3!8`7H_7NQyZW6;^L21I;mhp_>(x7qC(iBzHr`A%X!kJ!V?Ka$9w!*U8(*-~ z|CcwGAWCXPzFYe|6B;mA`FMBpPkN$8uQrh!?FFzEk(jYu_pmdOImCa7QjR>!02dv# zDg*9g8yGWbfjF4RkR7{MHV3S~J2T^p^Oh-y=)7*${-%nj4pk4m|5R(Zz8dg);xGBQ zpMBzfEA*#Af6;&<`k{xXo9DdGN^Wym(0PTY_%6p-fbb>Iohgm$1SQ2XR4EIq8+3AM zc!^mjzZ5aK+{CASS4glt#hAtdz>w|?{V)~+U3z1SPM}+zQ4u>8hmYf}6b7(r%)gR~7qr0oEUR)JbTYeiimh9g75j$yYMUmx-D4%J;hI}cYoERV zyB#(Nv{_*5YREHu7xWDP_cj_nr30))a+PcR=CPVPL!u_v`CJle@R`^%LeV9i&!hI8 zVthZIW=UB1S@O4-6snE1c2F-{PQ*gS|KJXqwE3P*tlMw?a&ux$HfHbyc_g-@b!3OL zejOM(t;i%d_G4IT04ygB`zO$I@KKix4Hrz(POml1W3m{RioQ1p=qanT2T%A$kPQ+{ zTwK!a*L+a;<*Z!V#qZ13Kcf;zHo`ju$)4exn#Qrjr20(h#sXr*oTDXg^QxW(eiOC}%Oaml2QDewXv>=~$&SO`EwX zKrvWVsCcwm+?*kyI=&y6uCV9+k}E<&T~Q~8=4KXXYnS-N+}-SKztkomOMMZi!fXbh z-#d1Ti10f$7{)gtM73rDq9+dUsUYe3YeH_zgPWqmI?{4w`w?g}-c=!#Droy9en0cD zp?SgkvY_nIpm^b<2q{gRJ`oTOfg$f7T%uKm0eYReOvR4dVcDP{O>n+uT}6+J8jY^n z8yEckKOFAT(!B~NC49m7QT+E>;`oej9YlJT|C*%Cz@XGU&I?94Qq1^XWil+oE@XB} zUk3*1+dJCh%_1dj&}63tcXY#V9kCf)jZWl{zY34ZFBPxXNY@*&#Hc~WwRUaOnKi6& zSqnB2URG*Nt=TK%unzUGqnfIwR(wzuB$Y6h!SZhZSw0K8=Kgv-$qMZi9E9hu0eg4f zVE*Lb6UaZLu9O$%XS_Xi;tvvtyqM&7fsD z;{H7|jWZyvPZJYwWy_&HbNTu#DMS-e&!N8X>>5tv!2vFEn!o=#z(vZzhm6Vh!%Z|| zfzg8Gad=IHl)uC8qwLniWk9{Z$K^ z7^?6u=fCxyKao5PwiCR5O?3_rNwhnK4g0Kp@}k+09p(Vd^3Yikf8y#m-BR);EQW^I zVi?g3(8bDtCZoQYO2I#Z#U4;}-J7netrX4Ii=26%O;5%{DuJk<1g%Aiqbo{@j8I=| z_)Fe~>{hCOUVbE0CYOl&pANudTm6|hL_0(o!vTSsN7U zL%MDu%r29AAgAuAF{TT> z_^`@@ll$}6iZpE*8_dw8CpDA1j4s|1Taj&x5ZLl6b|>J{TT*Ap1P8?RR;*)LbCePAZOLDFD|>%Fi9_OFQ&DT_wHD0P!uQHj{M~%hUhO@~ zM)#RxayUlKxB5AHqj&}+F1}0sWP|?4tEXd)fFH{3KL$@ex|s7a&nzJ%_GRF3U(T#J z#6#Q40JHVNo1YTzZorhS{L#G}S=VQm(0WMTDrpVBd--dewgR(4`crmlm>|ezY1|O=dVk<4)-SoJK zfgn1cI4MQU1S15A)Y_d2XdLTCFFI0}Br#23{v!^}6mq<9RY-$&eXr;}s`e{HId|J+ zA(hXW`pGEtO9*aJ;+PH!EYxiv*T?TR1YIq^o6pV3&km1-4206{e5rO@KwPrE+n$_#lJlN0Y{cGV9fvlZ{r04xE9$yY#YTG-owgPVd*!{Nb;bn0L#L z@*GVM`)(QgUY<*^%IUjn_!|nza65w|j01Rcm6*YQonSSvheDEX59~eQ$Y5$K^wNh& zVi_HxK#V_dcg)>vHWFP2@fpd;4y4&lVoooCS%^M6aNq_P_M*|H z8-3B`CpxsO0i4o*j!3ahL@QMt{t2ASlAsEq_VZ(Bp+4t6w0hn;$X#jhUy{*hJwX9U zxD9>ZWVZCaZ)bM~B4dYqO?{T^lfeCYw+F>@hdI^^0t`q z_QTr5jgc(9LB~UQDPVZ+e83=h&_E9ZtE7sxT?p_NVOc zrzd?Wm!k0n?jfw!X9VYc$4TQvp;B@=frB+f@wQ^l5_gBEVJ)Srs)Zru6T167@xSaQ z(cde`%>0(P&JPZM{^MAky23oez2aOZ`>Z;#8H}c%i|K;>~Q(hV(k!Kio}A%f9Z-8 z&+_AZ{`Q8EYnce?wjX1rYCH%+FV=X<0pio*qE6311 zmn_ck%rH8U0QC$qhqUqzsP-seR$Pp#lzRzG``Pg4 z1w5bFn4(*5sgruO4^eqC5M@wCdznQmLhG;F#G#BKy{GV{2JG$8^`!IcvkZ_jh*g6C z|Hmf#N}n!93Bo8tZ|M~) zZGvfOhh{Qu9=V5h%x5=2OW4|AqvZ>nkYue86HL+kZH>+ol>hq?m=v$*c0!_t;-A_O zDwu3Zdd(p_WJIf81x-2juixbWx)M%QbGC&vn;mWl8UJ3$^rY$Ud$h*wSG9uQsUEBw z!t!2(zhy*MCFeJBY}~U&=xi&5@%^^k>qipfV0{dZ2osIre_bI;>42C=iARux8(P_L zi%WZ04Z}U1bvq9)Z&)eLB`18#v2ou&EC0|`Dkr(nLK&lr?oV3>#)ISFgJwLflbt!n z3|(>omN!@-p&HkPwpS<9e-xyL!tc6~gNU$*I^`|i#zm78fk|1t6Oclmy6t2A%+n&T zC;clBB#0&_o=>pPU>UCo;Apa%+zC?%8$-UY4OhL{%Xwgoi451c#=cqI9z(>d^XIbc z;;V8hCVH-iNNs!y6YUFTHsTTzx2iIjU_+}Q5tv#k^D$B}ixvGFp))j_nsX9q`i=g78b;tP?|CK^#-1ZqYj~R%j z9G?&VlV}#{v=;RLFa+_G**X1i&O0d4zCNe>bb+@GP4U{1)LI`vq!h) zo)4R3IEnQ7GUIl7NK1VRXEHrP77Dgw3I=T)g%sG1)iY3oqQ9 zZ!OT;(GBSCn&Y~pB238Hwz_OBp`^d0e`z^oKx{FZ2o?(ISt;X=D3 z5wF%Cp4r;+3vG9+xxdf9jL00^T?lXg@E%>#!ZgnO_8AfH&14qu9c9&rC&)e&1dG@~ zUBdVOfZS9G2i5sUH>s=-lVABp@4RKAdp32mIzxIo8Etomp35S-{o7pmbWg-c9r^R1rEgTFW6wCPh?Jm{)3JFBMy>rC&+BXcv=DuHLm+z`O$j15Zd4gzF zmWfjtXZ*2}nQ}Rro}-EDQ*(K>V4IhORE;xnO^SbReila;z1^-ZWb2l6#dG4pmsjnV z*2LSN3ws8i4INyPfU1Y>aq{WJGx9WAWpN=HPISK#8wG9pY9!z&&5a<0-o8bGGQUaB z=AR_q0h22~CB>@{R2!k|zxcU^t57!)p7Bc3W3IdMUrPcegJ{0feDB%!#V_z4`BmRflsg8hcH8YC3f)V zgY@cqbSIUA6zmU17^M)U;}eA!4CTtzZof~5W^;SKdKE3hw|ODwFHhybn~rwb;E>z8 ziiVirA&OWAT;Tx=)5OClde@QlBMH+cezJ0b8@`3tTO^<{T`Bg->g}mRw1j42M-Mr8 zb6L|~GBgZ4zU3&&W#Q2N7hPii)JgQ|@Y%a7?Kj(1Zp|*PP}5)~5&S;K`77HGy&Uo7 z07s(v#q6NTXv#}!BeE6kryCl6WjN}6RN$l-%rdRL8JmGx9z!DpcK9ygsW=wH@8<`_ zGEX5KAe+;h8k1RL-y6@&yiAIsXICR8Gs~nNFS+4COeUyZ8L=?(y^g~buHNeFNKD2n zu_kwe;)9+5kI7?>Pb@jNdyEZEOetFzSXaKZY!yCpL{qy7zR;Aq^<38p-bt{pfSROn zSc#_mDwm-aE7$UV5v`Y6Wo7ZlAvWua#Nd>}l_h;$Vl7%X@uOMjsf^(b zDTrCTeFlBh`ildXI>LXOfG^7Sry#Y^%J$PE^NYa^Z?mfxUhmXA;hlH&qUF^oZ|RVn zqqBBspF@w4<}7%7ZqWR>0w;z@q|!uMti&9!@V9+4nv0-Fpr2AFzj2dTU7|BjzB%#W z5E|b8>*XCLKf*)QCN$j~;Q#Vz`pQ%)nt0 zDScQjyE^ojS4h7^iT%2sTdDoi@`?QCW|8Iw`Cn}+RMu<%3?(_75P^VN0C^a#7o=ZsHrpc8kvO78%Mse4l0%?j7k= z+!DpF3@0W}hT$0YH`YDE%vZaFH5{Fp;25G1=i%-TQ}_0suJa;;Ml_q{15-Wx2bUJizckk*C(;b9 z(of4VhUzqHD`a9fc|Kx`iBF(RlqaV-T4hgNWaH(NvUtu!Z2fb&dhl(k^>(Z5D}NHt z)20pdd~fYz)6<~!lB2qX0sj($e7jci*MVr~@SvP2KmR}MePvXYTNkdBl*9%B6{Vz+kd*FDK^g(2Tcn#U zASK<6bcd95iy|%Eps?wX?z=YXiEn(j#{GML9LE`Z4Bqvwxn?}`nRCq-@&ci4Q}b2; z=RX{6I36^Ha`Tayr1Uz$1hsZ*Uz2`S{Wrnb{Zz`NmseQQ2Elx&_2fzg-ZSEQ8hQF` z;D*GFPQHz=TWuE^mqz(kbVgnbVgW(76B*Rv?_3gp@RH&I=;$X?b@(#~zgV6Cib_ZJHRz?I)9KyAwI9erDAbbDj>g)>bk!Gm{BD4#N) zaAKPXl!2V0o@!DT8xyx=_w$=S1y8ui<>h#LklRORHNXAJG+qmkt90qe!vkGRp%r@ktrCo7pW?sux>x)nlaeE zbKvw=n%Zi5SQ}q*-ykf5}z zAbIy*KHFc5v#7KPBzIwAY{1$mkj}IplaA8#zaO8+Z%M%`JL-K~?%(?c*UB+Yvf)@i zQ6&QZPiG+96Mw-NCt_B!FVl3MsG)j5x0-L5)H-sVVx_ca_>x3!y!d*DN%hriT6>%C zV-#6cwS}L_hzO-9lhsP;xV?_b8XsA56P`bm(hltm!q=jI7Jxxy?98aRqYH`%^A4xz z*F$nzgrR;^6R0LhcypVgeee!4Jzm*e z>Uw`15Rei5+Nek@>HPEh`5Q6{ZCStwNeX>7Zzl3Dsow>=u14Pn;7?*{9d18y9haj$ z+&dG$|CI^NPz&Tva=BrjuEqL9n62{EkQmX=gnxMt&xXI!1(B4zQvP=R(AAi8m1`d zPhW^MC_$6l&s|Bmi5dOS`f_4hu9v{WEt^2zXg98Z13R9hcWIOleC*C_CKG{k#$q9U z(>;$7G7&K&`@Cjow1JrF%S~>}fp!x72jyh9e&^uVB^>Iq#4MV2qrIXVA8|ZtDzgI) z$-?tmMk6yM8)z3MG@iB{i)nXs0^PWcKoM2;8Z_jubPEyn%0XqA+3$~1#}sf zGB$#R?g6`!5+yW60*5x=4+x zr!sccEV3xRh2*f)hRzQ~&3nhff@uCg4zq{KtZC;$XU*=#g75-+m>&zL_pS)=mMJOa z39*^G|A={plIiLn@3$jH>f)HtRxm%jW&-g|^J!aSZELlMR9S$`bC6@PgMqJqMtiH{ zlcye#|9Je&M>%m>gGMm2tF=BFA1F$eSWA=GqlVyF&l=_dnDxR7JrT8a4hFOUB9v(D zJ4((vvM}3&D=tnCwFXl%dWrx+NfeaMhSB)QBbwg4&=!$MOP-PA_8{HGu)K7#Q|*QA z!=n|O!#QtrOEyZ6>F)2j<~jj(zk$>>9uks9tlpSKjVwZI#qg=Q6+A48)c2zI3!N~hV z@}8zwRg&khwmAlu&A7_;Z%v8l!AEnP3# z7Qyv=VY^P0-p0O4*hn`nOgP;Y@vGDa5u~J_mIvc@3oxoPC+nDwT|}Q1VBQ+$nXa{< zO7ph{eSDEdOgO9^gMIDU>wAcV2=2gecTu6d-TEiJ<-Iy-5*ED+YmO zMN-P5IC^{gMEuZz@#K6wfSjRJ(XVhnhYNL;QP}KKVBE4CeWP5{h6wJNx8!iVBOYQADUW7{5eS)iA}q zxIClKD9W&o%u;OkznE2yfL?@!!57RE(^u$3O_O#N8T!6piOPB7^B*Ai>*y${^evtU z;$xp*Dd5R+Q&aUj;O;24U_byvTU|qG1qmHIr4a^q-jrzd#nE;%kk7C`TOYpbk7hyO zM3w>d+4s*LA4<01K#iyztQH>UURi6HaO>*h65P9e8RGEm!kH98g?MngY(mp9jIj|n z?W9G3U3)=z|F}FAYiaTy+Cw&9)e;kl9SNsXb!Bn+fbx)+~SL)#Xsv_+p~#MZ7yza z{8h`mej`eHeT&LDrZ3j#8ZycPgN1gFftozm7nHujhE(6Onctc51tWaZji3&2(pRP| z`~GYo0p<9UN5qH~;POXH5+Z^V@-hSMlK>1HLVC1aZRyf3V~zl#d+w{)s1bnBp+V$~ z3ML@cfV6G5$9MUH)Y+G11Ru5SU##;6Iv=f>wzP@NsrF%_BB04y5TG-ZGlZXBpggS} zW`ds#3k^d3$w&}n1B_TT2*gP}E<)8A}!K~SR0rDym zu`ub0TM24#j8lPD^dJeLHRo|vZ>&z)B2v=5cmK8(sfregNP)Ov+`hF>DZXZXgjS+0 zR?w*SH+ftbv9AYySe)kc(J)@V{$MBZRgp1{F^GwZ2+lB4u^ey*I|F5aWXf=!RMdx+v?FP zh!7Y-HXMg+9S+I}!BiJ%vQP*nBHFOdyi;UojJ=L?wi}hE@S9$(ugkT-W281Cpb5qx zV)}O8RC6AXRaTT%(ttWq7g~Agi^KRIIF$VC*9w8?6hHtPZbi&&gwpXThkKgxVH^rV><#* zQqwSJ;RpUXF62Lry1;vYS1>`|imLRUm4#+?QB8!9O-Z}ber+9a z1YRad=n53fVlD1audeLmOxAG(V2&vQn>d}YGx9>&n0r3AGf4A@FW^eMQp6{YvhfZf zNy8>HK?MBxq25?MDa2aiax7}T`VS2LlreuFKp~(o+A$)l!}rD1e9~ut#FASUBAH_B zE3z6>f z6BFwQhsc1&sOrCvcFdf98L0+3Yf5oDz;stt$Z>AgRTR7CknRE=iz@eHnlEpG8cck~ zFYd`{(TcppWe_TJsJ1+NHuSB$i+AEy*tPC+5R@62OLUgBDBt2(TK;wU`_IDZEv^e# zbfmIt$@31iw}G-Ky=y^q9Qh8-m{{Vw z7Pay=pRICP0Tm_p9a+}i1$fm`h#I9%%9Yu&R&{oY=jEf$@GsAXO#gwsaZM#!MirVQ zhCCuKec@9*!dant8f!kFu)j)YK?>^s&<*ta6sxbL4`;?nBDeZ<(JDXwRh~2kJ0U#v z!#N8#p6YnhhIBUY={nR5)Cr+Vr%9XBe7T%5^Lbrb<9WyJZ`1()zXZq=`Jw#COX}60 zp320x6mXtdjkd}sOVHm>MW7Sjm7+upMWTAnXqHTwgyp1V#ni6(4SwY(@q@7T2kJ8U zq8slUv$m;bh}9~l*x+X6E#%T=hxex!U=^`ir#C2)T6$7?NH#m|&BI?I_?L#=fIh># zg>K?Gr8tC@Z%r2hC_6nA$2j}KtXxW~Wd=l$-3pDMkVID&U$+n)RM@o*!!Op=MGvG$ zW3qFdK|SsSW8ezNyPTPkz=dWEeGz#obt*H|7ipw5RFPDCrhOJ&VmH3I1;OWcu&R0 zn5^66^iS_e9Z0qdQ>|@Wt!u(ItkPU?EVeI-ssz>#e%8S@Dn{8F(sJjXTa?bZ#%3i( zuE{RC*<8J2;1>q|c#}##X5Vh10PX9Ci92fw%{KzkR=IL+dZFFcyU>3s_p|lBmwkjT z{BSf0SQHUb7*@HYO{81}3ya^s{-?$Wk#R$xQzcyWSnz(G)O{q^B%m;aYP4ZU@^(V$ zSnoCC0=yeN-5naV7;`c^)etn!$Ie=uoDrbj!w$wWGb!o1*Xk(?N-ukr7I=pqFYeZg zQ@rUy4PVZE-jZyZAT-`sMs1hb(?A+*c~JRH1pyHaS*qnxyMs>_qn|XIaUj!wNVgDi z`E80z+jcc89v-zUt2CMwm4|Xs)H3E#Yves?Rz>_r z>|YNOY2o9M(W+aXJ}%W~VIXg~ZHevruUtd`H{cPkW!+ChI&}03s;Y?LjGbWT3cVkg zN0=CAdV~0%1@tp8(a7|nLN2@JQVVX=+Y$E9<+KCU32)t(7ivL5TjpedQRZRKl_Lw!Y6BMl<6`2 zSDo({oOr08ipWR3+%oQqyw}}t?DqwA`Ao9{Q3O2a{!j8ka!S!Ya8D*YOGtBfO#@z> zuibHA?X)>(&aTW5UYUkyliL^fv)APjCMA|>y*1v4|O4nD;T>e60|Lh&O6l2`{3F>^;?Z zKTo#>W#@rwsb=ZTD@%W;>}A%$dSP%{n052t29k+r#E+t(`_Q@WNv5cn5gbYjHN2z; zE~*YWfO$3YK&Y0I!dXAu^raT^Yswi$Gw&=?s^^re7nQ?e@2^8aCyFDz&QNJ>mMkSd z#&sd?I4hL=I3Vxl5ZE=#7dJz)W(Vohwm};tg?7^a2M| zluACXGBc|S^epV>j`0Y)!E0VPnA5bY*wR-~_C@|@!hgP{@vFfO8ib7Jf)PG~OF#5u z8)$_B!lEjV1069`9}`y(k>`?<0@;2ZX?hct>VSTu*@dyay>|+S;GX2EW9@N{QRA;$8GM{C#WsJ>Ub>y`4LeL1eK<$JHC6vakih(BON*-s=EW zr*xouRF#u;P%N0yuCl&+C-rKM&< zedoRPul(@_O1B$a42Vz2S6C4VF>y;Dk=##w3zh>SCT$T*Qlt>cc)77H4o#xOqO&{> zjGR4qNn;9cs+-4$_ET0x+81Q0dkoRk{Ttjrqg0xOOF}7v#7pu9R1O2Tlr>-eIU2Go zP`IL9LNJ?*c;|*=O;VN#=9x!!)>8!b^sivE-i(N6vRY(jQCTyTR@K=rDv|!B%AdW3 z%oGtm+!MJupARmm;t`VRQptTn0NGRSja@1QTEPtoiZ`SZ5dzUv0}?}1k>=8Mx}fj$ zx>jkZI&UuhkWxO{-ce0kWa*D&uVpN>j3RR_2UbDsVc=*8fFS5OUxNNIDFyH2$mArN zFbYU(RJ%mx_uJ0m%0$x39zSY6x!kJ1esY%L{6lb$V2#{NGO|GDPIiz-|JDdixJGc2v3GGmGl2^47iPsqdfFEoVqEuhNJ;49NLygw z6p@>WviiOHC#ej?Z~Obf-+iR_AUm};cDrFna!0tIGk_$5j}wX##eC(=Rn)0bNMh_G zvfL=2$9)OP$ArIF`ZK6PAPU1HVq&j5dOsZBj6V`;_~2p>+PY^6 z$(o$q&KJ%31JP_Bmq&FWAI9xxXEkH0iyn^}Z06R&&yEzOFJ?bFi-^kMzyNg>|1aeQ zj{Eism%B{iD+Ps;+t&E8mju}U>;59#ys`_)mes*^keyf|TEY8$SNyjzif@6-BJa81 zpP$oZ-u%&bpeDFs&A8jWwfo*d7eb~R!>QasRNA9AX;$Oyw(BQ`TrZh8N=5UkCd+~< zBx0DWmg33vMZXp!(10EizMq~o%M>SX`;XH`R@QGRa~Y|ArZvY~v~v=fh>8bQco=>c zibC%9Nqs`?J9GgFXEHv4{z4UVw_ESKv`rBqnoa{nlLkMjS#mQnUMMpq#jyqAaU)l~ z4C@smb8aTDj23;q!h$)PUi1f>_BN-eJVqe#eHnplJD!SJ2TA}Wj$5hXB?2y z2En)cEds#3wZ`M>Z6qaCem+CA{zSwi-XKW&W)x|!FWxB9n&Bb&f{^syk4GugdqHo< zd#C4Fxtj0oVmGDB6Q6?C?z^~ov;tvK_W4=LFrSJK<(s2W7$relNn;gwCUo&6 zxI6l|VNU%hIwhW-lvtVdv8mF1MC!o#`1p&L_;@8T_BC0&RQ$>x|AreK%gs@ zrB2vHJrB7rcDYoz$A@9J}3hPKynk zg0+DY7drX@r9o6itRi}+?tf8W|dw<~-q zCqI}<1*DuG$ytmP7A)?Gyw@}+vG(#MHgc`HwHTu=8S02C-tu)f=ol|rQ%10n9son2 zkcAIr>O+Jm6lXr%L$v$dNU+OoH#+=hz&G&82GFL5Ias$C+!cG*R2>v@)i8GDQ^O>a zN?cT18Xmf|P1#j3*+hw8Jm;9@i*|03U2X!u~%T#r+KDAd%Iw^cIMJtr&&#F z?LGJ$h}3ia)$S@+yHgRr_JxKAdlsAA?(@RLD$FCiRJf%LY-*La5lTB57BTGz5w_8y z34wC&^c@cl4FR~Z7CV9)&kxt(Wt4#a1g>0(UnmC-Zb{L_QZFr(YThcW4zx+~XELB2 ztISvB?^V4#I29@h5^DGfOU_ZhB-l{~T6*k6GQ{SEaXSp|IPH>sh6|lj>x?*j$$HUF zK{sMG8g81^-M>fT069uS1s>+~#SJHq;PP?1v-5Z(M&#b0V!ZQBP-8;0qwt1;bKQm+ z0yB8=;N0AMAvozmyT>A+5t0UqQ?E0tQd9JHz8xV~7Ch-L!eWKV+aQp5{{d6ulmj-$ z(P}5$@B1AKw8Z)AE6elEDnavhH^ljTg!|Os8{yIqf1C##WKPz&WdZ8KcZ^O~BoUZ9 zz+$8(Bpw)Vge>98hxTymN+9-Vn6Z%j*y|ja3EDV=kbAY0C725GN6WoC5hIKOwAu4B zr#a;rt?A?em!|=xvm>;)jO2j2@+0jAc_E=WtF&N(vtR736)%JYz-^g@^qfLz@_HX} zgW05DA4Xswagt%xzsk|@%@~S<$dMn_HgxyF6z4b35XXKQ;yiHq6OTxva>GPE^`KZ} zC^+-XX(iDaeU|r)^HSsE%ucjB*BimKBS5gh==em02WjAbTUnO{U=>`i+2B3wpPO#% za-SIpVgAh|@KNKskiM?L*Di1jF(o%;edE!cZ_T2?WKy`LZ&Q*0yEZovwlT&@ zN&}c@Q?x7ZL#5nzm%b=9FQ{nw*}Z<9{VJ(!zx34;9l_h(6BCA!Z2LeNh@eWIPm-_P zXm4xXjcke#u@gBtQIWvVImkx?;(=@@y2cf_g`^3i@t?uJ0IgUAL0al z*~O>SDb@~#8gV)tXb<}J>Iq4h!@k~Ah)uF$cUUZSs~BTOkFXiso3fUzeVzjH$5}#O zSnB<*YW0Y-{&9;Br)$oLfHm#zM3;~382rnD$I0=3eH2N^8|8SBzr{y!jGtNmEQG%X z8mMj^tW1KKfM3S13(;fcBFaNYs6la$2^etdsDz#@oF|FAS8_nAQW>E)*C>6vHd0ce zRj>AqX{5mPY?FRZfxpHCvP2#(j<6S+@IQe+mGu9FKaV)sc2x1xkOWa~prr$a_V!VT z``roI;(*ZP?35CX2>BfEy~%Yr{gzC(mQlt9%}}A`5QO?K(>m$Y$kT*7fUwbA`q{2iV^==n-HvBV?=3+iWrDF?rFfhOA_OSyxVQ{v|E(S4zaq6VNrH@ z8Igm0zSBw ze*JFqYuNaF(^UQJDaRWL?=#*5Cjridz>)q>xClev>8$U%`EHp4_u=)ypa%b<0>Zfw zrr`0_&JLOg2!fVxfoQ*>iOmO{gzkH2Ne)yDVPN6JTREdhV zzuW9N_+s~UOZx2-<5Dhlh%U0fM6C$(Yx?e7a@0rFotZ3 zc@y9p(96jqAiMuz_dcBJ^ox4 zt3f_qoMDvi7AD(+@5@fx_rD7>3NB2Ar2JPOi(MXwS|{aLCxNBVj&=0AliSAFeMKrh z>C9ZBr8#kB&O8(_iB9Uq-AMs@<0LBM|Jw#Poi$Q_iej?UU*ULR`X^3+F4-DeDZAX17>E7qd>OeMc0|eRJ6*v1wkc)6Z ze!MG5rXPI0rX$6FU-KD7+kb%|u~Q^)BSE+}lG)@YrG-~dY`V_1+^|WV2R?b*^E1v+ z3gr65h(OS?fPsd$E9Cw*lJr|(vnJ+b^S{oIQTD%|U%r^{=Wo?mYoh~n541j72?O_v z%K{zC=^F!yh9*I)pDm3*y|)wu{7gV5!1nizDn$6>ss9fi{}&3GPQ4lJFT{cZ;Gc{Z zOOD`#e0~q&q8+{KeUKe>0(jJe{s$bT3ju9UKu2>{Z#g7(v;K-#I!&0(hs;W&?*_1t zp*52dKB6_IGd1M>+zwswPdkyc&1Y3Xu8wBktC8T8oDd3$mqX~~c%?Y^$%3dP@an`$ zZ=>tv-4z1Ed_Y0;7Q!N%%3rhy83fAh?Hz|=*<22*X5_+{`R{9yf7oyergn%1;CBl9 z`cw$wSxw59nKj$j?<(K%TC~Hs{@SDw25wry{Xg&pFy}o8$PJIKJ%v^6(-k;r_@ACa zG!RXN#xI+Vr`js#;@;-gpIs0#o@qvzcoc^3FRolZsJj_Wo^5^fK-gUYJsiG`XJ8xn z|JU32dAVy%Q~hl7-pI6LIv+^x@>-bJxz=i(1h^XYSaQ7l2@#B>@~1jr(Z^5j07!=m z&i(J(;&_mFfVMQR&t>X$=D>e*x+Zv#l8;W^Z5BOLU)zdICr_J1h6G;Kv|Ik`sJ|xXo0sF^70KtUBc@~TRlx{|KA)?f|&0= zV2ILZtDD^bAf{vZwhyhq1R`&DlYVRV$WG_U8nXGLYewkcfQbLE52!t7@1l-$vl;=e zVJ#?@f`cDCoS#4{J$y|`5odvQxH)Z@a{tDyA0M_H#PWtlzX7czZyfnS2KMPurULRb z0qq%ags;QBVh*T7@z9a}b8I2HmP>|PI%pkxA}WBT2nQdx^0qK?gp1>dTC*{}U#jGpJzT;g4C z_nc#H1^g~0hBoT_O&gH9;^1+NQ>Y>M!c&r9LJsgzixo~?p?|tmLCs6YruMrO9h#vi zndul?N1zY{0gX{R(nPQuB8~Fn{VB9%}^8s^TgjaSU_LaU!`CfZq`sPEiGu)jL z;Q8wfB!Ax;6as%)QZsFv1^+YL%JsX8s+`7m9Vq5QuY=iQ-xC!KP#7VDkQ1-LOCoCM z0!~N{Ek^5(+0>4ogG`a1rM73ydMz03C`?D;AhJ)kjPWs zj1k2x69d{Sd-IJkh^2?fKv%j`<=i>!DDI@;y8-oC1Sx1o@005RfgrO#rvlp2{3&V{ zS@zd{^?Pkx*O`B9S8p!_{O&F0s~^JED1HN~41e4QkAeTMdcO*^K=pJRXSM1KBXqr1 zx&1`Q4^{=GV(8Tm$6AiFxNI7Eo))TK`0extV#@&m;|UF_V5c<`#Q(W zKCOb|=^@@rVdj+M$l1Fnb6vYD`%wCyUe#ae0uA4W?YVW`sHieRyhlcq_%pRuAhQD;`rE%C)OFH9>@w>RfhF20AY_nOO$H0WIb( zRmv42s0R&Sz|LlrTnnWe?MrVa4!bFUkG6+f!`;Ww%+<56Gy=KwpLaoy6bllU+CPbl zFwxH1f!c>aw6ViRp;84`iOtuF1DX9znuf%*0tJ12Tjx~D(x|sd!6+R z?~DlX;HEMnr~-9fx@=}#j5*6Gr<#OkSsiiqmzZQtSt@a2zWZy+U{tF!@EpLTA&ssS zru7W)do5~W`q+OFNZ^0~`A=*Y$2|LuS3&r!32^u!ZT6SJ6uQ|eW0%`{|NWU{rbKV8 zae~wpk$=UHS9;#Z8%=anznKW{8f44H*~td8%<8$OAbQmj9ePvEKr|PZ^^`I8cmvs~ zX)VZ$scNgqGS6c3&1yFtq$aC%wPM|a%mTG3X%35#(DisjC)U#947RpEy#R5?7GDWB zl|9Bj292K{&GoI*n-5RRk2p?pml(7ylzHk_bl)Z2OAP){ZSS$$?dFIrDKc zsaqk?HAh)JqwD6T>s#l8{#(+9;>duKr5_xR485=du7PL~6DGQJ_4TOj1h>>!N^(C3 zOsCoLg}dtNhm&y05d+CVg8&vHATCG!9ZHV1!6cdDG*6w9vn`UM2~w|>={sCpojAOU zcO1_9^LaLl_JVJTSyHrh=@k#FgoO@P;>-@!U^Y7*^3KnTI_wf4*)K|)JZVlk%ExJR z5Zd2ft}s3$P3N=<8{BeOTkBO%4ZG|e*7n+dTetGlVmR9Z2aEJ|=CX%?HciT?ynG_Z zSVu1D;Ex);!B2voPIwTxvzyrW3cv@JvO5ePZ{B5Hq3=ySd9iPka~{mJU5d$EPrpTc zmE9{6b!%X613!iMC(=!Amf(l{!+gHd73xYM`Qxz50MiAHTd_( zwwa_J!d*@R^4kr&86^!rHt^=*7O3mSHRMY}`e?Vt!lqW1=!hg8v!6$p)aw{(XK>X; zAS?TTeXjFb#a>^oK?$T(=B!4dLFUmVeyI!*8IUNTWCTfNw%+mhJROV` zifUz20pF@Ol)4se1zy@U*2WD0^ko9*bGRlLaqSU^OKLn%TtY(m%9_}2FTcH#Qj@GZ=BxxC!fWziXCgyYJ%xb+ej_L<% z92tRV5w)+^pBvcQ`-h%x*6=l+pF8tkjNgGNKCeF-e(;dQ(Np(qy`NP!H>r0zo%Hqg zlPZC8(~C>DREcSBDVv7#fyRCi5ka`FVm!%r?9*7LohBR>bKPd#@j`1zJeXrOJD z?VsL_(V5()6z@4T4eHyT96UMsY!EsUQWK{1-Q#fNr%`Z93{@(zVBg79%-C!J!zs>d zPtHQfk7~1f&rx0uXX2-*()=7P{f4^TSTa?zy}n_hR_45xjg#?(z?zDJ@35t@?(A?q zndfbohfLRfw~PX{;vAQ>{y~QeR~~clu+hiA&o4lo%H}r6 zw*vFv?rOL53|^0LS|rZ~R2(XZPcK#uaVK6;tT?6t8UuL)BnMNEu`B-o8PNUOelR+} zUh>M?Gs)<1E~z~Ia{=PX$w}SMp^AgqBxYW&eG~T+AGoikgtSh}dA(e$7sr*n)u@Rx zBRG7rQDNgX!ZhOK;Mr8YjpFJs#Ky`%&Ude>Sa33hN%nU`mZnCJkxR+{wlXmw@uuQLD_Z_}nw?P3^BI7Y)H@<^$mo z((>MmfoROlT-uD-<^+rk%r=b*=3W6&-Y+(cd-bTkyPWR&o3)PVJt?GZ;B%>V9DmWZ z@_2b$U2k+oQ@G7C?U6=tVwL$i5c|k?oX}SSXY)}uRZi_5L6t6T_&6)o#HZY8{O3LE z=3bUGFq0ZQ2-BwRhbWh=8`#ODo$NitUr9V2@ zK%FrebR3ar1TuFJ_4&&vitQ-F+42JoDG==Ep90Kzr;cBaGa0RJh6ZFi2)rO|a(stW z{0k4kyYLLpTp13H>of<`uduEdp}WmTvja=8v^Dq3^Yw2jBe!0vi`@}4`!Gq8qW$CJ z_!vbz5UdQQu`CtvsWLqm*2S-1zYZGaxGL;-(CXssGl?L;o_sJgAbo?yio=SjN#6Yw z(wnkZeviv+x#&lD521w<2r(;{lBvv3-$}n^k%ky2HNFbrw3*F1EXH{B_Li5F$>GP3 z9gm~tTsx&pSf+f{b9orJ#ZK*p%x3Jjkrd;eERX43DoWlj%iTLs9h=FSRiFEDzM%%l z!-M%4qSV*j4c}*Nc~S$8&FoC<*{RhA&53z$AMH4AmQwkE1}6OupZ>hPd=nyRu>g7Yfct!I z%lH+t5a}E^jyUaF$eV2go5DSZf$38h?*W5QCJD&OevJXV-Y4DxNMj`98j2H_+qAClB z3F)Njj~JhN?IB?g&ewlhnOF1Y_9~=>Ecls_jHDwpCVMn%tGuRLbmk8;RS(~YEM~k; zMZJu*!~@Yf7y_MD*{_8){6|>f$sI$^eau#Yp!#IE{w~@TaIWi!!u2fn#C%h54B-zj zL<hc8uLNMi#X9Vt95)3xAuDay#lP4byCvlNs~^1C^o+iaASfH@=u}~wnD2t9^>DkvmTf2_BhWTQHlm7JlwX!<2_y`47H^;@xe8QYlC%IZ zCj0o-f_vV4183@F)&Js12q|H*aZa55O5 zy-ZQ~OU74?BME`ir-cLON3qGL36H`_`b zl()$H9vc(*eLdBMox>ZgR$*LwFzzJ|^A9!=S@?Te@5&!;4Lmden%HczAGJ1AB?+vi zcCL7&ul^bHQ2#7Z@^TM(MNDh}ZmG%AX0@#O#xxJ@Z@pCbw_Y~%d(>(22 z_d&jqd@WUybjvglKS<&6!wtlb!*?i*fB20JwDDJsMyb_YrMFlVE4>akOADUmD?{Fa zxT@zg`#NNA;vKWs07xE;K)e{fYNeSo2TAR1Grf&(2c%wA%2iIK-y5vk8gg1J_q@jQ zHfxxrXtu|OJjM0QUx5jGHmxhjiE;=#1pGl39~3h0m}9E zY@_fCDf2VkS=TCSwK5D5P1zbf2Q#k5i&IJyK7rGhevXqaL^Lj~e0M*p`+Q2|+e^YT z*q=VOz2`ndpitZ2c=%rHfn(NQ1#uvn*JZ6HD(CbBU4O&9_DWvIv-MI&^Kz%wLhh2r zC8p=oZr%bX&WU}z7aFh%X`B9GVaBQ3yMmW_%ag}l+kGb4R3O4647b!eD_s)@S8}OA zc#xh{kUXebo;y9>r&NuG7ez$e=>*ETNh2E^1wbdE@NL{{YfCsaB>{Ln z}m0g<5;v}+-;XSfk<3ici1=f(?OyPANK`Knw9T3 zg0DQe6z>W?-hIBQ=7s}1s?klk<1qqt!JFfY&>Pq~vbR0DdLozJs=%2U|a1_S$K z?g>7+(j7ajXGx@d8A=B{45Z)r6aDkE-@%{3G(a5>1T_N9#gMDjAo1O33wYT8SMMeM zgFxolcfdV&;D=0rvG4A5*96ZBfKrpzfHwr~>IjQ=udVO@8Buu&DO+%55ERsdrP(GM z_k-E}9MXurIF|XU3|<3~+ZeiE^i%u1jxtXsGq+Ovw!Z)@(p1nAZP$GlW;!0ftB}BT z(EKba)q{`cA=W!YJswuU?r01PqoikwgVKk*aZ{AXOFn3BH zy*#bhtJhpg9~3xStG8L^Wl)Ll&9ju$WBoeM+ciAaFsUca`4 zEsOt!C1{t_^!z-{^q(oQ&D>^!PF=jwUGo?T5l?-S#(E-uz*8Drv912q%PG|{Y)0VX z$HSCw**<%NXBC9`!kN^4{p+Kb8HG&iC5py!4xTn~Zf{6D4&`AmkoG*i`syYYg-P-C zToMnWz`O1ai$N~`g0cL>?)z0JB+T>*37cu?t<_ZZDm22$bbn_`c7Jtk-DXU@v6Sv- z_ggU&)s2umI7PPIe5Msab0P1NqOet<{Wes&R|}=oAi#3@?KwQNVou7TcPq@!m_9lxGpF~EIpPkkb{x0oh@znBZU)Nf(v@1_(F!+fNGAo1x#$ zKD{quv}xz@IO}zX^@k*}2REr@9$)6We){HFbA%3uy+Lpo5!rugVR55aQbIL<2RumE6zt7ZE61(k^Bo$v= z=YR#yIS|k`9b|&DsLCf3xM=|QT+jGYJK};u7AJg;@xqi}o{7OhQebVl69(dPTkE+2 zQM_Iu$YV>BWwC)nNKms!hsmPdc?A`l8(tdQcR9x2VM@RB#+Wuin0#Hc z^VQLdGYLJ^r?wUtho3jcHE}|{I#mMEmh1Bkw5Xfv{9LkqT&hItfKMM{Om;o#C=9|1 zR3{+rtU%qRC&}(OEWbDd*+u7fY4`kb+Eo4k=3QEXweRhP}9Q+s1nCd6_@>DZzK&&v zt!i$O{$_!hlX_`|C)@i5DQPC27b*!~2PP&|wrjfK9JL=eyk^&JL%OR;Auz@3AJ~5w z3O<4L`%*&>+N(y@ayzik(vK+F+Y6ny!!`=)=a*>@+H$GPzZq9xq(MqWdo^m~EGZoAnXsIMcZwk76HEuhW)*x;VHd-ZjV z@z+sgX?w312l+9tjY_Y^p6149CGu7|cW!&l0=4*m~p-}N=`%Z%i!>s6S@ zjdY-gnE^hK+l!dfxIuyY`+Zrr2Hm{oM zjH?@WS_Os3Vv{uqq!Lq>L2D5|AB!@YR=hW_bHu0x&&&Q zC^n(r46c5aHZKz_;J-H{0^~gRi|UtYE5$j!sNvgekUdbgc9tpY*>7Imfej#c+n&8vRA^t zSMZk2O{a*wo%D6%0vs-SSxqP#pMiG!y=D1OC%_8IoV%>gd0F(X#S}cX*)h%r)z2Q+pvp30Vl6^ZKU1yS-SaVhLRzyP*HpKj}6sZ^56lWB93SQ)0q1W69BL zhKU&@gB`FT$>?J_cZGvFo?UKL;n3ehKS7r-(dWYhwKe{9WaKvtxRu*U0qU(O>-*DZ z*NsiVN3c8mXeFSlVCmh3q9D_lhZD}|MwpY58u}ZO@4D_!M(^XBiG_s5ktFFoEc z)xJzm_4rBNxOYm)F?A&bs&qpWxfx=}Dx(h#-ipJZ%Bo38w29-Y-6ZhRQ<8BwOaG(Wv=-^p7g$Axe?*q+_e6QZKpEq`xmz6ryz23 zxPK0Db?)KsGa)*x)8d-udE`3&I|s4#R6HO+lWo$@Q`p3Qw5JzlHYK~=H&nu5&h0t2 zFEBPyv)$5a?%6^v9?GV;LE@tCLG!e=A-1)Waf0!1hL$?8rLv-uvBtmz?k7zbTjJfW z3ZJ%H+ePxpxivpOuCmhr<*i01?4gfOonj`~qi*eVU~cmi@<&&dO6%6HUv{zo&OvgU zV0vj=(VDsxZAcHdU*Wy%p4_^#TUO_|o$x5hE?G0>J1B5SL>b61^iYmhnoaecd|PMc zFGNsJhMn_)0xp%>n)7BS`n~a7GEG=Lhjz-9yraAFnnp6}lxGHk3D<{e?Honqzl;)nf)OX*r zH5rUSk`rlb`5)AMcRZEh|2M}u;m97@A%%=6+2bU8?>$aZC^HHX=OiK-8QCL+%Ff=W zh>{s)bE1f{vPV4EeYC#6=llEb`RDn^>(x1(`?{{r_1WWnpF5ws**@XKdy72)HUpihO- z*FEx^#X)V_`(<*Q>`tcddzRY$2Qx+~TZ!vZLl&2CDG&5Z^=#aTO3lsy)?Z~`VeiS45hnk~vZrs5xN|;#c(?MyM#K?m4OXa@WfbUg zT9ucU+ptvQp;Y6F^Kjm5lecxANXl9-L;6I;JJ_+J06Xt;2lll(YOQUXF`VO-j>%3}7B zTf_EgJ-G?5o$I|HSX%W`3M$oRPPlFil5ET^j1+rC9aUHt;PCn$-gWgnsBfwbtbLp1 zZ4e*XCu$J;LgAQcOixS40$xOtfWv~I{diA8!S|H6^yenMw$5tqS5Tv;$eO`EYNT({~LH=#>y(>|p)9D|Gr({IO7H!dJ&re9d(;2T&d z%Kg)C_BM3T@0!`(tXvsJcpVFDO;mgeTpJZS+To)kZ~e|HIqWQ?Zq1E8!dFmCGsj7l ztcZL_pXj9Ag>ujqwD|rr-F;(^IYGx(@kM4?`>+)+-d--l@!E=}r|(J&(+5--_IE%N zf^p&$)pZ5;>h8vy3l+|~*KQ9D^#ANqvy&yKdFFCg{Mn}hb`Oh*Z&1FJLEuUc#`biv zjA(JH@7(JRt-|Riqvuz@b??utO20_{z}9h+>X{1<v-xD=pdkPg**F%uIU!Ud+?i z1zY)PO;DCVIr0;=9OEt55SGQhXvGe{*P#~NpT4{;t7xW@WlE+71r!^o0J<{7M`48)3>UUbnUy8YgKw?^Lez6NT! zbaA`S(>WTk;q{u_hUVXO3X31^FRV^Pve_(0>Ag?a#G`;KOZQdN^hW3BaIWQ=U2f>s z!n-ZVgN^@SHb2qfiW{(U{su=JqtVA^-I`FYM}Ae#8)>A0nw6}x9(n)n1+WVc>-?&W zLZ~bAODvU2migQG1#eF*P0$%l1uZr_de^P^VQW;V(EZ_Hv<>a2XNd|upVTU&IX6J3 z$8~gbI=HcWq?f-04+03hy><#cf%OwhP1GX;e#i!`0nS>G-#b1`PnDxyHSH{#7$i~tcDA-^n!(B&0bm>d!Dt|%5HH}C4YRJk$ugc z`s|;B%oqWU!Qe4r|d%tX|$!*@3 z$-H?3IAvfqBp<}D_t=lrY;o>vhrFTsm4&gMP9Q3--+$(HCTPC_nK8#M=N;cwQ{odY z5wP?tO*I2pyyGOrn67m0n{RAgb=Ad(Cw0_q-T=6<%c_{6!{e%EOq9Jt)VBz19kXU=auyi4r? z-;ZM&9Jxmyn9%XGbtIp)k=GUW{c_H8=8b~jL*ZUe*A1`0hKnuDbHfdJD_+mT$u1{^ zfXvW{z`J;h{2MRxW$depn|8&wYr~zr5}%J}AiN}dg+IP#ze;C5d|hhXVkp~YVR_iA zI8^1HHmLMGSk@>%_4d2_PPpmmTkW64l4WRl70l$Gx6EX#f)+G-TF`2;FMILNTOPRL zV9B-P?GBkwDX>{&@cEq+lP3C?C4gNzXq9WTP|Li{13IKiA4o{P?pCO=8J%*^;9R`f zr2@3qv~Xu%lu16Lz_Hjcj~#jHuisZhgnv|lYO7$7c`lgWNx!wgwoxFHe(iGjx{&2k zg6^J=*38O#x0t52Ic1x%Z)smWpN&2dv90k_8ZW3_T1vXm`+>=RLrl*o<;h*(Oc(a_ z@=~uh)!)_j?_8hF&NR6-kiy7mHcV9-PIgdH-|gFfv-hrX-?Z)M(CF6M+au#%v3vj% zVcLW#PSMu<$=5`_sDU*z+s^^(UpCmX1z{}<<$fzCIQpHA%p6;Jj$CO+mgS^`(bTgtIN-A#ELDKQW4s|@|oUWj~d^~D0RU?)qV1Rjsk@hmEWxTAL{0nN+gDk=&poSZp;VFli6>tTqlCH)xRCgI_9*xd^E~;JpG-I7!WmUQR ztfoxyV*!iN*^6`jrv*8bCA_A#zr6KI>t^am&7ZHFumu&Tz-Mc1O??y6Ps|#J- zpjs;jS}A^RQpGn$-YaCGfy~~s0KzfMUVS9U48J1Af z!0PXs67x#a{+?FE5k@)biE8IF^CituMZo5zzTEjRGA_`h)-CA$E`1}>shNIi!7Ia7 z(rQ`x&WL-xVYi&Z$E^`tN>(*`x?f2zQ%x`5siRF-U%yyPh>LstGv%oH6QMGfZH&T< ztYk^&dehLvZ1aA@u|CeSTaCWyH->A_UoeC7BO!L*fKg71R#n`-9km!>Y3O?YZ&gaw z_fJ(ysRdM}@_*bINdP5ffBJ8~Iw7>;r9Xv}dXG^4OzXb5p>dY>0_c&t?zz#WTeERL z(9ouKo!_9!Pv?!E;$9Z$zIryaN6I_D-nr}7RT`P{J)gs;fK}MBuT@d}MhnMHhwmjk z^I@GPv=9#|)F{ovl1YXsA76r+r3-Rv+{>YqD(KxGN5| zR$j^=+W$Ucm!4W*Iub`cQ7r3T)4JsZ+$g6FnKf%2P@+s>TbVx8;AeyTwx9ZqS5~>w z)v9VPt?$|;Q#w&A5KVNpW0QJF4dUZ%rw$6mUmkns|7xB zTDOs31L{92Te1w#7W=0xW%I>b8bcq|Gt&O}@iG5;*|09luZ|*--o1R0`P?eI%+H&F ztFP}ly-)$FzqI(PXmMRNaLKj$$G+8zq@3RBp+FeSN^r zs^5O`wppaL|2v!7>J6e>KbFPQeYQ)x9OK3_o&{T0hI87}8Ad5S_gk&t0QD^1(+c71 ze91pu6?-*v^^)aI#6c;E7itedO5%iq;_1{K$90<)xDX-Lw zZO{`A2ECRi(jKe5gj7^~X&~s!vVm&ycH7ooWh1nc1R7Ib#xKu3arAbbph)}C5&0=~v3^2BRa7!kjk->L`%dkaKhz{7uK4NA zv?b`c>eBICnZu4OQB%$&9H12q#b0+hupeK^D_K0+>WlJRxj^^ z1hzQn1H8HWlGcE9<-FUs6TK4hGM{uRWWKAv-Q=Kw7nI+Qssvv4 z*n88m=dUhyH=6KTeED7Ty7#N*l|KlNTyMJ(>z5OnocXupb&*i=0uq+17|wYSNLcpI zx2t=}a=@@B%GUzJ9&X&?7`~Y{>!g0`SN0Zz*?}e3k6&wAiw{GUAP9Emzu(05W#QN`G~E$xSn$vS%i5jVG(R`*fj34w)4}^x{`!vTRvs&g z$(;qZXj3s!tMuBn^Z~}CylKbIB-UHV@zEGb$>~m}I&JTkBG0(XmbITpzV1!0R4MFg z>ZHzRYR9EryvN6*j66+F1p>wJkC)u06{YqPoKCBP+BZnKJqP&*?^uD3%eFqfPMNpX z#w1tL6V7y8&R7ash@>`YxEblRGp4S*`DNzpoNr50wyWe?e|}RHQ^1;Ex%^$u8}S)| zvjEIaN;~+X9m$31^$Tk3ppt!Iy$zJm20{KSW$8z}lRW5ZDN=uh-SVLmyT{S;f{u2Q zBl8KLGK_RMFgpQd_Tt6W6Ht=i3$@sC@}|pWvSn$(M=_ee2D6B;Wa-0U zLI=(+amIJ=F)#oJI=v%i@|ZcuN}x2(;l`R$H&f2AQIF`c!W~3I|ITx^2)VOv58EW) zeNLSRO=L&>5^is~OvoVgwy0V5V!FN?EWvL%u!(0S+WJl#BrB7w$OONs1pZAV`^~bB zd@g&*6-%rjDb>dh4PBlJ65(WuLUydT5U(DzvFEpC2IakZPC9*Px%94N#CXr1?|Afc z(AxYYx6ygv=f)_j%~qB#x7q5^CxS(d8VOY+BHMY-7MpVR5;yo`eQCC)dO2jE_PXP? zD=|(}pu|&D_r2*PDHTZDDs*M(u~4a>Vv<`E8G9zrKCm`@ z+g|f_8$2?hoeYX+UL=Y)B6xXLIXogdf*#C+M!W0ZJ0gEVsht4!x+mKpd_3Zusi9i^ zY(FPu{pE;CyW|?w^gnhAnN;3WIkcp$bqvjqW_QV*9j9MF$@8SdGd%NFCmmF1hg2QW!Y7f!3wQXRj>e z0z3b7+h4N%YloQL=b$q|84|9;PeG#9d~JPn#DWEgmV}jq%ANZXn(yaK6z6spnVdhg zGwQx?&hS5L-1jr(NR0)@$cYupa7L{}r=j2XtwNfHr~;OswKpV!cgrcTE(3X1$JJ`kJ_JRj8_yB7MnpqT=%VODC@3^#skn z0_WoD*sxvs4(Ijj`uWZG3UD#%7MJFCcHI3^Hn!Mj7%4t^G|qi+&kBu9jZfu`1?{tq zKQA{LJ+YX5RZsP#TzG6MPIDi!CpqrPQW91N_9PM56N$3m&9rbbb&lW42z3(y-o2YJ zZfhU|dr3n!yrI*QkM6j7b5`$Q;3ab<$`)}R%+EweK>-l(1X*u&D%ZoMr4t-XgVi@K z+n!o(Tl6D0xfsjRQCE`sLD)QmJ2osvpPO@Hh}WlH&NJXugVWHqI^WCVV^+=px~}NV z_z$qPdsRn=hNJ0UX8!f5!-13Sf9?(#sxrLktJATCEo9sWf2+_BY&}eTagX4df_FJdG!v zi~zwrjX%}cxHR_8OLvHi4^A(af-s!2`O}pLX=F@afRRYEmDu0$Svwl6Mp5P$5Z29^ zZY<9BYs}1=9*mit)kG?BO&yeCE(P2(;D!|)%(oX&o<(rScmh}KwC;j#$V$_;b>4NC zDTejE0u_)r4wpsvSH};s32a3Ao2KhZ_cb>x+x59}9)${wnUr984(>^>w=7pznu135 zZ(Yj1B@s=%EKPp~Bl9~Rx(v?h!pSilirP9@fB6*bKW6*E+F$^Yi^dlkd2u<)r3-gG z$~}A+q*ueb$2nv{{`;Uge*wD2sAJGn+`)|23$b@k=>gm=Aye+tnruh|ppHSWnSP!w z=wQ@@##i!7G@TFg-L8xVFJ{4kUVq2MC=TJ@sdpcE_4u{62hh(@gf^g|q)YwbU{sBu zv}LEXp=mWRrRDO}2mpkdABQ?iOw4(C1{K&r+m9WXIlOf?%^LzefkMAn8hpeCZ;uo- zW=AOezHpfAJq$i349vwc8$-io;H*L{WGP^Z2Z9?=7W6;8{GlTp9iaOi^rwV_NQlX|}NG;J87ZlHhajxZ7vPSHX+;G-;uUx%G2XP|Y{{ z7MQaIb2Ve7kAEY2ftPR%SS?5yLQ`=ELcmsbUxwD>Kngq^(*zJ$7qXbJ7|QE&017yt zFp-ALk8kzhaoQp4Re;w67?1UZ!QS6skREE7Z^Az8gr5<*bP^&hRMW|-j}cO()n~d0 zgOcjXZ$f3R1AI>ts8z%w1YifuH=hIJdk8|6A7kdd7ZOLLjZLYNQt# z63dS{4|*E}VV_FGw`-z#Lw>Zay4wi|M!xaLX~{nvY{UT*A@}756LQw9F89uHHN#4m zUvF!%oX$8cC;OBNOGNJRSg_*diC@q}`juetZ0PUcS@q<&02}bm*WPtG<)0ieqmm*0 zsfIfPv)UVe?s=-qP>Q{a&0lAsD8tzoNqO8h2h#=+7YqnsF0rr4l|GO?;i6%KhH{8w z&x0nS5!mQgP1jxU@6PDQ;4WOi2z1%+5$M|@J&rB?0FKN@wlwF;z*8n~s_kfuBRtTe z$)LMorN(*jHUNPtU(#+ia^hb;iIAgG;ff$2tt1NR(mYq?LL5M?AY<4;ws@QguWEK> zN{vV9N+5eA5$qBWxa8kPjK&*+Dbhu1IeNQ{Xz!dImjBAWdX%`EOvrM*>_qprDfjH_ z-k-AGYyZ#IB0wS?%%3~=NPks?WzhZ_KYtvfG!V5g5t!u2nxS+)+)M!P$q%z&SW*J5 zBMzRo_0Pft$Ot$!hM3O)&uFQVNV3+DO_7e)aAnE~6sfE*n;qYONdh*DVwFuH$0Shq zYgzhpQ-f|1*V|p^fOm5rH&xu#do;C@pHDI1D+Dbi-?zlTs#&i|9DN=FR5${mk44bm zc}}Z-j!U8uCc*e(LqiAY$wSw#evLKO~X$UCWkODyJ?c%_bfr_V^Cb% z8*UmotcQ8Z(zDGflDz9yve(4I}0AKf_-bp+;0In@)3izaIwsb;J zUX{xTk|Ujz%{0QL06Pdk3Vv%>PJkSh*J&&@TB#Bz)9%_Qo^gHf4JHD~{=>VC(>6hipY+ZowD{$6`BU3E;|D=pgOT!^`u8 zfXMcrYL6_Q0qEE(r!=Tjnvw>@y`9F4xxY0HUF4-*%Eb7Q{^-}Euxr!rX?WMRy0{dx ze={W@I?U|th$k=14Y?@>RPAUQcYFxy2W+|Z$hT8wfcbWL@xUSEw}TGWNi*s7UsMtX zH48)N!9p`m7dt;a0J?d^I#7%NnH&do2*NBy+P&uE7uRb5!0jk9vrUr>sCOO3=jXV% zi51%fyYAAZAiv+oe1nCzfZMa4$wL8MagU%Y1pb%XYiIHpe_$=(SpMo$<>g{X!Z5kl zHPq4Xxnfj_S3-6MBFrEuLsZ;RD!W$sWrF|+#+ttt(Ay%|P0h`*ltln7d>tGYcf#M6 zD;ac)-k!Oe4(5hYFCYdrEy7FvFH4_f`YLNbjNA3t(|M%-w06ip@3~2Ti7n-ad>fmJ z8-=+x&rN#QM@**IC>f1jkiraz%Q}|qg4{p2vsF3GheleiX=OYtwk%eZaP?d#zuW95 z{U}XVb;DipaOEZ<%xu}Jln=HH5Lb~;Kz~{n^ZI$NKw?Qq-z?Xa0rzfG8@dQe{{Mx; zmdOt@OB4@3)codumr=paaAh+JxbFa7;XmP@PIw03sX@0>Ad)CX8jAx^Y+BPNj+{l< za(#mOX7MwJiOAq)1YQQ1q5nnqj53I7G2H#15LXt+nQ9B_OWb((|AukpiwLfJtXqF= z;7oeX&+^Y&d7JX7>}g$B_+B zY)1dfh0rY3;Ka+?o1Rc9Eb@(kcDh9X01F{!5)oSgB;3lV`w{a$P&}{7T~Bz`xmP{* z%S(?S(K8@}14x~z%l{yT^~lH024&K=%s7ln0e86_pq!ln9tu=)hyDbJ^w_@df2EKD zGH}P`ATW3IV*uRRYVb&hZ1X&3b28eqpybF$J|1h&#du1<8{3y0%S{sI1fP~`4CdjgIBNlM9wg;3!DTNLlrQd3>b!()z1Nw*}s>JfNM{_6bqRbQ!G1q zTP65Bgr^opw0w^ocsa9lU<_XC(U1eE3O)s-H)CIddv**$NShj z@IJ06z8G?w^paS6#zwpkP(FUjR<8a)c+j_!StdDib^6;k6A@CP1URoPkDh=t580=zE@kS2}F zr2X%kJAm)JKHH=oXU_OC_T?ksC+jN%tmQj@k##TBaH=aONr)CK$qT3S8vx8DCc5{$ zKq+M7;-w?b2EAnsVW_>~y}#aEKX})rs%DXF(yOu!vrW4ME@UYBv1aBU62hMtnw<~B zm7IhbfmcMh(DBbZUz$wNIi_qy!0LEicQMiU9YjJ1;&wN!#}R^X&9E8f{h&bw1?KlC zH^{H;Oez_Q<0a5b=eldct93#zLroqQS5~ zyYZ^W@K(f=3GR0vMsb6;?@elbyvtQaS)xGu(nERt{BN|!%OOv}YefS1O1!^VHb~!k zG&|r~8$b2C7vSLTY)H=xf)Msl=`$pj2n;Qgh`#au^BOg>T$CTUJY{o6?j)~Z+hgcd zz?YS@kL1ZV!GPRkX6gMpz6^EcFrodiEH}#ZSMtr(q=^8SE6e*g;F6zkFa`ac&F?TC z6~Kwcz%O(7&4hsL&j^pN?RZY6z!3n@yxSl&Z#L1C2;emUsA*QAs+75K66&>s0e9g>#`9!)bX zjvG1xv{)_>aR-YJul~W4CyNI#p!+Ila6S4)ce;4ck>Gk#9%XTpZ32Mm$vW=Hrt=w^ zzgh5Ke`I+hULMTy_53%N1bfJ{W~Ie3@tgn@3IUD~AN2maWCmeLZ?L5A${o&Q=wPs9 zs6iGbfcgYWc6&zVn}H=o<>%zD%}GmNkiQssf`z+C>@pu5c510xM??VNv{74+T~NXs zye9&HJ0vGnc<^3eg0P!3Y2g>d7!m9x6aTl_CdcE~6U$ZJuf%qp3JEu%y)EzaRUVcM z;Jq(drfmZVUw~;Xb;8vrV0O*^b>)1~@wKN3Ifb|9%|NS$qAu_lXSH|X#$4?rh zkN#LxUiRiuaxmYH0@_$HEeOJ>byFC5c7^e5f!X=BD`sYeHu=tpS4_%(-bnDf6&y4p zCQp%tvG}-C+*CDJ6=_v|qgl0Z`?%gav%-O+sU3n$IUZ7JiS+F?7a{Epdvj4IF}(ZZ zVFcwlu8UyC;l6><-`5CW(qYrrDW;ny$;lpkl!z;TD#~&p^I9#Mj#h$U&(TrO8Q?ki z+JLtqtkFLC3GiF>4FZPr-b=$J=yc*kDYvVf)>MaF=9 zjf~HFJvR2XYv1pac3snM+;_LuqJZv|jMJ?x6{>f)IA;zlc(_|Ng_7q67C)jHj?{$MsTQWLM1Q{z1q!2 zm#7s{Zv+E>BSx|Ko?_eOr>Hu3AbgZ^lvHSN{y3&zAFJ~2p3AW!=-8Y>ccs(PXh zi;VKlaw_c8?Z~<;Q%(2AZ(rMu7!N2&AR`wkEFb3cWMhBX2crY%`sO)!{tKBitvRSY z#5PGYMi#qiaZ&IoCWH);Zr*1I{m3Z-RcJVRJZJb-=DTz5u#bl%4i&w{zMBQ$iynL8AlnM&(N7Hg4^Y9ebV9RB#ruxJbv`CrK+#KlF zJxthPwx`u~mrzzb9zFHsY1A&%hhbfBt(IEz|KsUQ=0};4qhtjG-^%3AL3>IE`?hqP zS4YO{ph@tS_?u|-Q0ytAMkbzk=gLkK0?Q1+Qb`hXWJji`d{SX!T6|&8C{JZN`ovg~(@uOH@HzV-a7bn4E3%c%m3d96qGl_$U zx(@x9$2!95i2aH4DvrZQJ5zB=xF+lXHhG1qe);5q=n$?GRgAbns!KY#*!Eu@T^+dR zk2l62+0tt}+{%mQzFkata9UPAnw@~wJ-?AB@x z+!XR)UP|>|md_Xgr@X)zv% zIM`ie0L3ct33y|>JFDurJ=V9Oo>cpA2cpH45~9H$hT+dejc)mr2xc>SUF+1u-X>=! z*<}rrO^g;XcguZ@fz309B|Ng%3bo4>W*ugWd@QL0PLkU@TuNZbw{Rx7Dp5DlN`jD2 zU!fiTG};UAh#Ok-aispM0+9SM2G~Jr>2;O$SN^JeZ%P>S@l7y3*iKHS#R1jCHSs>U zQT-iq=4v{W3WF)>1uk48{}>e^cC`_4LY!5$=# zNDgc|X2nu0>teYYSYo|e_Bnx2%Wx6mN+O^j1#E9#?uTLhv0pG_*fUn58RvhQkOMjR zhPZNM+y}_fffB|ca5r#ND7h+#ZzAcc$p(>wtku;9>BD;Ka3SS4<7vzf2+vnQaKazL zrw?U7_Gj!ec5_N$uVkOPL~h9~b;{HC-y3(T$7S}ZY`FfBVh@3(zSRpCMlly5sb)Ry zp)#ByQ7IKw80Cs!ETMUnMSsh{OpCjYL!n8~PXC2v$%V-76YV8G{Mc=GNHvWSyVjSw z*xmP>Ec()M(W$>I3%EO!3~1#Z*PpER&V0aHSYy9o*LJf?YTm3Dp)?R~hdu&po5(HRkKQwGWUNj{oc3}n~A|50j z(o21@pRI?*g_Zf|`%|k4Vod_qg{>ndi`EL+dNOVJMP;xxn7i1w*gi~y1HlLTwF=f6 zsZCl>89=0+{_gN$OJYPjPloC#aCR^JtIxAaVWnRzT^iQO@P;#y6%~@;1O>Qa>_v<@rT2d8 z4Oe$0zy39>5vIZDff0l35E8l;EZ;ysf$S+KJveFJsn-afH%gulV8EWiWafRb&NBny z#oueWcD#hugUy4lwzWsxQNuc83XQ~+6wZjv$stFP1ypi}g_8f%gkxioE{ClV))Xx#{eQV071YQo zQjv};F|EMUtzoF-=hGY+JIHa5A4HT26p=7AqNJ$W3gbIF2A~fVzmE-0SCCNcCIC< zo>OC=Hv(DcM7*MaKkT>OX#-INZ4I9$*(28`-bG5eS<5W5BUrgSL_Lh>@_>#bzBL^3 zkFX%c_0UK04tQ~N@igVdZb)~H!J>&N@)DRYs;H)d|JlQj^})Qwkh6Vw^wR;N+V`%= z2L!jr!3e)Gd)vU+O@tyKqf<2UIWKI4$Wzj!+yRHCw|mxPyiMAax-y}~3Y%)@%7$vp^m4r@&=L_A8} z!j@};<*vCmj*d84LhEr|bHNv%sG`$f{}0{X)Zi8|c0@ZQd|%{_G@GHY_7uiOH|dG& zj>wvn8KJOD$d35Rq=qBrk3yzoj8g$OJJHuS6S(=aA>Nv6eYKv#{CFY@v$p}_BfN;T zRZoSywBy@V6`ha&vCK2v7H%Eah1$L43`x@qei}jUmV0DJiHsb@P!|nR*iobqRm+W2 zo<<+^uBC=xH_RdT#K_AhdUJ!#Eur3yyP(Y>59G zEimz%^4%s7_4(s)Ur-EO)~O=w2?9P|ozXZRl+eboV#w~@^8N2@TN*Fb{}EIE0$j)i8wRgk98_yo4czTsOcBEbvMq z5e>nr7NiIyR9OZvQs98g63?7{ZnX2Z2_?5?luH+PrCFG%}U~3V&h;_KFj(g0WjZgHAEYwxRBuuFw zO7!fK^5uG7&7l1q{2V*=legO9gN6Qk~3!G%GOjxP?H&f@3 zr)ocvZyRi=0;U*u@`HltEYuLZlC(=t!~L+ekp$6-@O_*TI8|AEFC)$XcQNE`kMh54 z-iWJ0WtN8wn+;J9kTL3KJ%P=@WI!3OmpiSCnRp!BG!H}S zU7zb1ECN>Xyn_Toe}*pc0>6GVP6UwcE{+t(;{1-b>grRq4_^Op#8+FiA)X$u#Z^gE z4qdbbBQvsqdKX?T^T!Q$yEm7JJmZy&Ik%w;9mvnGoNypNxGnKz;!vWxhkUMQgt4ga z8vf|ox(mDBVym~yx7(UDW10m12^o8+u*rQ7gWO?fDvuGhLHBf%e@PF`f>yz2aAr(h zfM}!#x4-(_{r&Zg4E9Aa=_4gH+%xn}n*{BuL%T^)A^>~~Hkx;i!wmc>D3^f}qm$}M za>soxSaM(R?TtNq^dH6;7NYUlm#31W?3kD~XNRmmb1E)k_U2$cuFMcRDJ5N4_Y+wx zDK>|#KX1&z8XWDuUd$%~x0hQ3o%vaO4u&%lj>Zu)Iu+uqpD}aaqs!GhZx@H8E4u2+ zAN7w5in@R>Rrg_ovhwUXR zAa;r8EX}kkKT^kVe6;XC#Q44m-V9fY3&#;#LKn`$ILI0{&oXu5PNsvnYBSL90Z3)B z=`P%RR6*kvK`c;E$CDh42=|Xf`a_ToQYGKzFsDs|Z4$!7N}zV% ztnV_X6NJnQzD+ViTt+fMRJt1A(v1c-E`vg7IEVkjC+}fL=aJCibxQfg&SU@bM~(;!`{cVs=EoR$l?`4M*mz9Zx7y%3agjKySO2W7 zkH&^$y0IsF4DfPL*z!YIlw|49l7^**_{0!%1F#7iTy9iyY@5ev2Buw)L#CRS-ZceM zZW@T}_oq$Ine%-Kvbp(~-sdvNdt_C>NH7)H57^ggEZdKaV`x;!H!38A)c$edvu0r{ zBzn2??=HJr0ln>n8N9Yo->k2;GGs*fv1}t1gjf4>- zLE12*T-cqbCHL38PokuXwaKQRqe2nWu#Z~N{Muh{nwgLKEfdk~7`6`h*2jOcKKxHG zhh2bwgu8j#eO^e~Z6w51eE4W60Xo7S(G0A5Dcs(qf3Usc=#p}&;hEG@C>i&Skc`W6 zf?4IdGBat0zc$q4BU)q9N@HJnMAgjad-zoEd1|2BMBIP<9h;f(ftI;IRb*cMw}#MZ&SDc^U~|j8$w)|^#!H~ zm(YzKjN(oi5iJj3W!DEe2W={$1Wv77fJcx!V^&*`q;Dnv>2ats8l)fysu%hrZi-jO z)8NhVSqD){7OELur28({opkrvg@$2}t%JV^=<>D)!S=ODqHLRlxrHR9cD5yxI^=rtdef@QozpdUI4&S`q^(K zOu!bn)L>I2g2aM(#hx?jnIg;7-(J9OYZc8n*;ATeCCEFZG(DAs&axmV;-ZgdBw~GA zt{^LBZ1#mkx(0t!0IdEs991wm@L)VB@O>P;oGP@RQrsM_5EOh~2y>2n0-Q7mGs|U+ zx@Z+@buaz-P$W>1w~NUk8ej{^gkq7@rX=8vWF)W5`i?4>!d|&rkPl0N-3`DL*_)Bb zxK*zFYRqT_3Xp=r^X1TAW2*0lwJ`L|_hCugxsF~(I@sW6fJ?Tu>iUASU>V$B5u8(R zU}73>`^Mz+A*(82a<0=4V+j{7MWzk!AQ2+(M^Xa~S0t?PIpTJ(KQ`!z+I9BU9}Hk` zS->59-CXckF5BtRy8D^clI8)U(pMw6o!RAa=El|y#rX?OoPYNca}>*n@yDoPU;m)5 zd;ntSw{W064pEUt@mnn9f`UyaV;I}@$=<4(=0SfZ$RJG?Fvo$t=pV*fmcATC$z?BP ztzVxOdm<&rc(g?&g;V7=p&+1d*qQO$Y>+QmC&|$XEdC~ zDrZ>p1d#f-Cq})2#lvzU?^7#tpPLt3X3l8as0%!$5caKTFpKF`9nK3k_G}ZsI7?E& z8TysrT;?KjHC`i2Z@%ic;apz4xV?Cje!CLbwgFhkxLmY-XqWLb&JS~s{l~{P+PrOd z>jX#oW#45KLv{*?tsnj7@uiXS&s&^+6yv_Y1s8{PKS>RHc!*O-=i`azfMiUAG@Dm% zy7Yg4`CZW;B(@yfb?-wyIkOp-AC^RtBrl(>j(#N)y|5=6RmgE>y!wz>yI*P0O$N66 z3(E$(+ACT=J|J0Z1zo-_?ATBo z&1Rm`!bhxDd|MfAikt_SFpcZ(y;Xk*`?V04jB`UV@Ul;g#K}V*uoD(EHZTPFsHD;@ zrPxsPa42D8$@-C}r{wkJsuHw;g^bmyo(1esgt+|O&`DoV>g#sJgQKWhaSAQr)K^xp>5~LjakxW_4f`P9eNv~>F&@8lgOYBtH|k_svTc_ z4pE(wp_*urYLG>$3j1@|L>K-efLoNW1*-cZ@5kX{aqYO1xLxOvVDj2ofpxN=NAIT6 z4Hl3bYq~w9KdXFhMfH{Y#X|Rol7ua9s0~bpT0VI7wJ~bme{9V_usv)(_{!@eHkoZr zYoD9ger_1L#PwFha0i+-QPoSN6fbUF`IWN6Dv|BV&WG2-JlwfYp9abRgdYrAD!u*k zxO*yET>;?U3Sk?>?{d3k?5|$7dJAX0b)nE90ForgrKwqc=X;z>M7QPi=;KOroq}^lRY{~h2F$Vtomen*NA-@&sn<5kgcdLJ> z#rwte@2XcEfIOOFZ(-uF)n0ygU+^SVfz?#9))n|o{R7Jcjtx5JLZZ96$_z{ugE?O9 z86Lg0dcFTrx5A-4a^(cHTy_sW2j-vj%QaRx5+;16)wG&sB9%N$v8yg{zUxcy>Z8_t z5I4~-vR&!E=x6fVh?T2xty@Me`8kZI)K53wI+@0;(1YWc`jZLT8M z7G%t_h1zJx1;Iy<7oFy9WQxL8lDm+(96qe>qvTD=t03x|rL}zSpkzXjwZ}{+?$0*0 z@iLB_yI+KQWLmJ)*h2UM*hz7$3RdlXC&Ej@l|dFNqt4NT#60ET;p*(hA~`F-L7T-; znJx1aHB9T9p)PPY+ocyTgA#Wky}1zMQDtJKCXwdWQxp6WVa26-@97IE-ahzcAi$;{ zGx3A;{7PFsB%d4hreda45VCUKjo0Vbpgt#9!SA zd&Z9z*w;qO#@*ek>VUh^9k4o*Jd#8rD{_;6cG1b0#5%~&$W<_M>8Tnv|9$hT;OQ=G zgCImOVB7Z@!(QFtSO036M$DI^4!~;hK*);+~L)NXfA!HnhhB`m zhzoxl%@@2qy_Y`RF^FKevgoK9xv>Sf&-=1~*u{e?2p0*gjf9j)jpVKa7w36MbX6J6 znSF)krxYUOAOeseGIR&nfWPR9Zn}|h43PbrxJ4+?i^+_Gbeo70?L@N0c|#1%yHqIV z0oMf7cRjgwvIuL->$ihbwd+athr?U$>22hF$-78CH~8b_fw@8@BSc6fNDmRu#Iw8_ zWz^>?gL&Gl*rTO-6w3@)p%j)5S_VJQ;O)Y=Wid`CI^GK0mL6s`e*+Zu8nS@sdtxP; z4C_YMK{ldA>?}{C4NH_qj9o==R9vEMRO0VV-_gK8oS1 zFl7PdC=Uos_+!ptCvw;dbcv`s&;g+!Yz5A#m*G1O+$x8~3m1l8J>t+LMj(Ye?6(hp(ME4T6_qh?T)Y<4dh#_L zwfp`)?2hz%XEP$;OIeBidYq>=yAhtvIR+_zm@;gs+;oMl>ly8&k5hVCLj+=UXY5U zndG@yzNx~BATtO95$Z1zGki_P-Nrq4-XHR{J9|B`kCW7mG&HApfqUqTQ8Yn}-sGZO zH75CMBeLZny%KX4`x<+{6^r`-;40@z_v+e4SX4GD0X7z1v;CsQ^J z^8b-EVX~`x@Pb5{h{<4nr7(N=Z;c2Za}i-bG{>)LE^sSyrsCuE9C($7Q$bzBLCC3w z2eo#zJAR{+<&`buywt+8^-sDr%CJji7iGvnJhGXIdkM0E3Ao%>@m9bZU+Zof63oIS zAuc0CG>^QRAgR>*@G|9ZG+M5T=H#kuyVsUt6I$T<$%E3v@v4&NFVlKcn5Latu zB=F`Kme8=G!{p0g4mT&Qu6cWL2*j%KYFwrk;iyzmIG=JJ7YNcLwH; zgi<>W+Gr(w1VdjrYBcN;#2KvIG>iVZqz_4~drN4ur5Bo&?y&1+JOa_A5&Yrok9W(q zcs?AP4$E8JzZs2I6|RNh=Y{b!BT~x#h)u1N>b_`enKkwUNd`fS=o0gih^P$+S_Hvo zAiKP*uq}d@MlYydzoH1(;U;#|U*6OzoDGr7-w;X!1SSGvQb2A?n}^&HUxE+agWSx6 z=q$z=dmJQjgfe+JQjm1nqMuT=)!aP@U#c)d$uMSxc1;D9&H+#`;0l865c9zjiI>m3 zgVYspR3-7+cnNf*s<+e8Ehdn_oLF&O1!hYMD~~lY@(F@fUbP-r|C{ydfnm*uuK1(X z(6`H9o4SsK(h4(Aa35&Mr-arHDU!kSOyF2t!7u|%i8Kv@oX=5I;XENnWgP=>Yni0@ zGX3jDHBd8!%zd*BJ6l37j8wRa-=L+-^7&Q@S8*V)~xko|10a>vf;Z(o5v8Fr|Sfw$vu0PryB#zmAvPXM(7gxrK!5&w?R z%Y(dwE62%(+biJ(Eg+#gMAiYTkKIl_@RA5k6T?wbrlLAY z*R&EQL8E;EqK9MBAvikNdinh8aPa6_2t9U(jM?!s4@0I6T>s;4VR&UA)8>rP=k2mdrF%b0JXME*)M=tAB=8x_{A|oPbZ&_W2 zKM8ghp~+$eH!p|1hBUzOgb8*-v=Unoxab7zgMip^(Vx-1y4m3fwco6^mj6xeY~P|< z^`l68)l{V=PWRI;F?RG{T&U-Y3qiRAsTcN|S?%0sLok$L-4GXMESKE4A;=~U08V!! z%2AV2-sB1n6)dyI7t{HZ4qVN6c7+>UjS)ReEm>=o{P}lRr6`Z;|CvR#4$GQp^sgcB zAW1e#_LG|EP07E*gcfWkECY=(ct+N`eiW!xfWb*`AT}-y)&(D?00>{LW`1ef8B>LM zzl#4sr-~aCfmAjf$-vuF5QbRvaMuScfl{LvcmDS;kbt>Wfl1hQ6@-M=qa6OGrECdA zMP>gEMg)WP4IJdk#AIWi6!*t@`Uw{g%$OEQU^F;p59 zK{{b?KI;HEa=IvvivRJ#$Td74HwS>G7hwpe|JxnS$9RDZcc2I~D}sN5KinBW*L&^v zAAq9%y6}#%`KW%>Mr5_sqsv#?3Gn|Z3>qmGNQkg9vHq;BnYTg(Yqf&8WH~`Or;8nZ zrx(x_cRr>UL-I|Xuj1EpP4qgj^^K?C1|O0cF?dUI(ho56X#Dzf(UHBlZDB2FK9?t2 z2D6|w`6s_wQVdf_Qq~g>-{$=rjIyvO;M=hCPye7HinjrKTg1W?8;{O(g@0K?2!a3Z zF+d2#`0Z{eSHrOm%#XW-{oennNQ9@H-o^TPp9Sd4zPRYirjSe0)5P!Hh*=GA1P_c# zL{cHnE&*!GtG~sen?O0CFcNTmgf3hb3V4$57(u;?gKe$B$8rn57;8NJ1I?wI|00mv zENOuDnS{j{+<`Ze&Yla!yGk}v4YIQ7m-Ba1hyh0lt%LML_G3H2e;8bJr{XYAF|`=y z;18FUzBv)F0XtK;@)fiw&5IG?d1fIM+Ozf-BZGfp>YxTX5anNg?P&*IbNsdo5KXoz zf*I=-JQ5L`z~@l~1i*lI0zaU2x$I&~{7Vcp)4E$wW*|9HnNv=eu7o>=mRtrVmM|Iv z(8Vfg1R*RI=8n+W^$U~^arwvTfy~h3fE;)qVO>kZy6}f9;EVP#@idX1*IWsBbB42tK~WUCz2;IE~L@@U;TM0C&lg#VEh4-lD)Vv*MA2 z!ss|bI&6g`J;T8C(<3&VNd^!DQfKYUE@D6stQQLx-et|j&;B2ms=ESXh~ii9ftM@& zWk!RC>%<=tZ10l$G$v7M;s+^-j*Zkrr2Pkx7ote~jhZy{>1kGTd_Yzd3i*GYqZ=U# zFUvzx-zs9eEM?~YAM~`1#+=-nG;zW*9ZG!wQFcHAXhYP{vyM}Vw7s&jC5Sg-skJa- zKA?0lea3hTuqpsG6@srDH%IA?CptYIX{$h<&n;@lbdmd_(M5pA!^!zKV}jcOQbZ#* z6}~mt3H8+%t?c6LJpzqPAfiL^WY~eoQyJyLy8grJ2G#|g9lpf!eHLloe3xdqkyETS4Evk_D}&_gTF82 zDlnWXqDLjWJ6FH97tsg0SxASpW^D;A1J+53lEtHi516ZeLo7=Y!U3y7;JXGU z9c3d}CA^Rb6~*M%xWHkl2yemtlRKl9-a8x1w6y()?BlhkmCfZrk~RLVF%<<>C|?P{ z=MTmyFAV-eR0U~B`$+Y;3PxpK?3*Aml-kn(k%0P7G< zh2PGLr?a~J(oz87_0Q@AWX)7?CQV{j!*0UV3z!zKIGs>^ZPVgjAW1x%>nq?JeS{z# zXC~f`O=!Bv0qjU7^$yK37`51IrDJZ0D$a4j6d(t+)$fRce1ZBO?W%+`uHjs(G>!6M=+TFhlqMx8EqMCh7}G|qrR%({$B{|E=H~PR}BlU z^z^CPL2WFkzCDmACoL!1KmiIU3G`SB2!I)3b#F*6(GQhO05g4tZ4J-FIznjV3DBwc zJy`mGAQDIqp3`kR-MWFD{|5oN;~t^cCSyB0CdEMSaw?jW zaw^u^FEd&TJkWxcIq|g5^ds}5HR8Qv$bllUop5p9q}2qD{ii%@071e5 z4ORW2uh6G%fa_ZaHO&)sqya7c+HDpBytNyr_k>eJLJ#gUR1v$lUoZ9i;B~w+b&9SJ zcKk0pLE{=rKY6+iTmVjoxuZ(NregODMrI+Ob0ln6IWRi44w`=B7;gU<ra#u%_+J zXTNZ)+vBPDt0Y4q{Q_GK8l$f^Ge<{?R*3xU#Rz9RC^!lzPQ})GmNG!CaNBCk&Z6Rp zA0`ST5zT}-gB$3)62X z#JFDtyIS2b7$pojrXSNVg5P_i63R3`Y`PBs2y{ePHWEH^ocG>ODdYJs8mQtQ1Xk<3 z+=IFZa~jj!)6+YWSIlzgfiy3hPP>>33tx_S%>!A~?J|_3l850EN9{xJ97uXHko0Uy zKkK7k6YTsEyk@Axy;OwxKQbc7qTs6o_F|wyv|0n=xa`z7|1&ferb9~yb0l z2y)lkv_WD3uCNfYf3F~{6Lz6z=&#HA;0ph4pQ*B!8 z4Dc@kBT4#Dp#md>;up{uSquRt5c5e^a#9I;%Nnv2M;sA@O}qq^NSL=!nUxA&e~lNl zdLi^4{$2?|)Hs=>#B}9+w0?~cWPO0KwvI7eEd4zX-6hV{ii>)mvdvM1Eud5MWOUZ{q zVw3_)sSpts6D&~&s#?0^Fq;@S#IX>)oRkB;A@Bl#fobRfv=s!+5k}vw4+9^yMSq!i z`o|tG+MrhcKSR6fb?xCym4JQURN2?goq8ybnRL1SPBT)sVlA z1V{u%h+#6xR=|{OO@~a;ee$z0FvX~~zozJz26l1&dM-HJ3rOXkp{DH{ka#M=gFZ{cson>yu7ka2I+8+v?KO|1%qc$6O#3tc5~+7VbE0bUhw3 z^Hv5HQP1BcQ0&Xef{%KI#mdQDyx{KFIDz&{FeJfT!FIuvXg_1=_qmieLBMDGoZkR+ zqsg+S7)eEqh89Bpb*gG(VDcqYUNVqB9wL5XEjOI|Zs#z91p{Bsa(CXi=ZH{l<#OnT zb_rGp(y@uK42tSR2ZW%3KX&qYvlc_GRZXrXOh4(}N5pq~k3{u6OOdypmsw=>F0jW! z4Jxn7c=hCs?z{KKXs#T$pu8&g;djR*-fw^2^9g~J1)2y?!iiJspY+Il?_YYKAlIi? zBH|31$XD_{zR^!vX`1)nuZe}w6=oli;GA>BvtdUmSK<#gm<#lIg%oy2mTqL0Jl4-# zd~>uijaHH~1UeGo6V@AjIm2hJHQn5&Y7fxDZE$+l!PlSXrE z1525+tJN*`%al0@?zf-+kf8qX`9Z-=uFT*Oo7Cqi`Dr0``RSpbxKe^!W@FVlMn2E1 z&_Pwq?N(F`fmzVD0CUhD<$AyYZCUF!nNZ6aJS&hXC`qx#`1Eb=SJxf8<5FM6)7+6K z&_@{1?9Rg;kFJl@t9wwIPnM?K;bpufJ`;cI`iJ;ibY|bSH;i_fSyIB^Xa}oOgXv>Z-UL5sRnN7>#7nEG zWL==c!H>t_Iw7d>?MuboydJ7Mg`z(#=voJmwUUELs&Z>cC%10Ap;=JHTdz*`Ux?72uyw^^#t%bz^?_OA;u~)LzCYi+h zN~8iEKoRk})A!9A^dYJcB&2L6bbZmUXFGv5m!@lduWy@Tx15{&NF+pOWw~`3Fn;gJ z${xMCdzqr@%>6Y~e{rlB(tm_9zCG(A>AYw{xj|c28{%4w*1Jx*(C?Zy8f6ePUc9D*WoRo!BJWv{j z2xhlfYO@;2BiITkA&@5y#h5D}#4E>SwOZH{zdKS<7Y=&funGT zjDx$GTUQOR-LANZl#{s7xLEaDNSK05Rtgxjflzm&>_Qj!(e*lydIjZ8mR1M0DP2@T zi(7G8%<$2XZ2iBfQ=;>zQ^l59>9e31g^EEkd7nI73s}ipXIm>Sui;~#v)v*BO>kd+ z;e*uGf|Z4D`!L>w=IrZs=`lO#8FYwhf93dEe)pHq&oe=d~oO@zb7y z@AkVxEflUNWj|=Pbf_$@;b0S7*?>+HV4r5j?*052C_M`i5L^)4fKCW*+D-}60o|4` z`G&Ezj+OU_Q1X{NcWD``2BEUuG~e?ss_-QL$dr8*PEGxf|h08BNaD zkuM_+eHRgToXr}vTUE#-8#6Qraxv)J2@0>A zRck8cxveiA7cViH`()jo1voE;?z(MmO`NDR6+8XBD-o4#W3oC0X@PFcH&GSmTtfyB}m-Z=#W!Ml%eO|rd)~%2} zsa~(5?tg|0wuKkbaMw##0YrM+*Zp?Za+|>_@(xSEo~6JquNh58(BlZ0ELz>zJq}23 zHGx0)kdjo8`}NE8ixsHG?aoKp)V$)l-?ZOEN&fl{Z-QFUl!2G*N|(}E%IAn^b@VMT zdtDwtI3wKQSvJ>=pMc`F5|0RBL7(4?&PwF8+Xh{de1i5KZ9h1u=p{Tjq=hm7c%V(+ z8nwr(`YcIFzvp$_K&15o%!9xnpR#^0rpsqBn#7bcr(D$0g5a{@bVSaRJ(yM`Q9-7a zFI83mOixSZo-2j8CznXZKfHfg)o1lB7Ty(Lo~w9D~@Fp;Q}KgIzBF6pK+OYF$=w_SRjHbh}+pj zi22jqK)KTszU$YEG%^~`gV%rV)<7bqy-ef38YAlQILeE&+&?Qu?t6y~{q2-Tqu3%e z?=FPCO5wdY!FBjkqpgr#TFro$^6aYHTwNORVd^D)3KLijBoS-V6k#+bQlxizqU_AWMrKg zrD7uqVqq}iAt{aLd6mprmc&G64$-SeF$8l)<{P`x0z=`OWu*qv^Unlm5;NTeKpI%{$p#qtX+ zwDABffMxp5JUa7S|E53P^KEan)sEIpTn#uw5{B&O`g`4{a)yRQ-|Y(0{HYSt9PS;( zNIYnH2|>Qw*&PV8pkvN`Sugf={}bK>_1_b`>=JU0r}Agb9SLD1j@xWY@jyg)Bd43@CA3ku2d1030SEyh+dZDBnLKW`Kr0=la~zf&X`38XR=e_ zO%bY$nF|mfuP~iKA~*XrjVVT7ExpD0d^l(j0=HJG)7{HnBe3hG#Rf&Q12W&o`6c!n zQ=WZBL4i{SZ+t(C-e>j6IV{1MFQGq+q47-W=WkXgOa1Jp2-kwmdzfC=XySwFrVAya!*sr|K?+R2l*&~**l+b_~ zYy}Nn=vX!k&6UhaU9_-{stB=F`7Uma#xGsQpF>}IZ`^{Dw_mH%y2}NUf*YQGk;`aa zg}QUz_D+D|s$NX-XD}nL6m)6#Xxb@y{Fo?qmF7=!DYJ%AXxuK3s2tFLJw{1rvOjW) zn*F`>OXp*f`{(0EIbZToBiOG={gA?Xjncwv~ETD!Hy49;{X9yt4$ASv`h&e0E~2eKSx$?`Yph3 zD&`wC()~jP+EX*%?dN5jx~4V$vgA;Y(k?vW1$*m#kBzYT=E?fW)tg{rdnJ91Ac}MF zXW!p5b5tmd$RZ-IIU^zw&eQ@5+n^dvBs3**#9zTWDgB)*ZT=2<;=*eA?GY4d+Al=( z5_HIKe|;wL2pgR7dT_GJwIX_NYOLsvL9)}b!J=p1!XgB15-H&YNqQh-Fp^1w{_8@PY)%612MRq9>R}+ zI%A;oVnDyNqB_dg^R*~Bcqtck*?0}XfMCb~o7Pfh7lEP$AcNsh8Ynj0)&;kwetk2) z1l5U*p}B5LGF>gl7*L2sHU3)UP+8B0U$G6c#OJIVo7C%~E6+5FKuGQMczuzVb}qoe zjyjlJw0f_THRbR9o0`~3i&$1jO!17khZ!vW?tncQLM#%j^}O$-uMfWFgJp3rcHwni zm#!nt*|*NTpr#CZgIl36|BSLtk|h*-u5?N4|M5e2ixGJvo3cwJWo*btjEnX@EJSfKp6iyz!g z1RE4-WtdMGIk0@DuzIQEOD{BG?zA2S%ZHi~7a@ofI>F-&h4UzZm9qK9^asX(MBmvN z`!J_7}AOba>S6)AhSM%Tt+Lli3@lX6AS8ur`e1_w2MT081jZSb{Jx zdGdgyC&EJ7o6n3tPE`B~?M(!l5nyl0#`sbWNjY-EvE|#IxzW~z=lE2<%Z)6OFfV{ zmC(Cra8^ohT)7meyz{ZIXcn>dU2?=WeYt3R31FZt7SgAhI@6DNHzH+_j!_+?P)Odk z{hw~a;xaU-z~}jKiaq}O(W4LX6xvDXx6l(@=8naLXSKGivJ0LH&JY_d#-Wwg1hx09 zjdjtSV6D>&9f%qG*DO(&5t)h4$$s@eyl6JYVs`ArLTSI*#+#1wk*=!|#XIrZ1mbAL_hPtuO)E35Bk}2j^wVLYNul zaqs|Zz%|9!Ob{@s16~GUgRZ2(j@4)F3-S!C{4zlB>i>{uDpBtTp6yG6tzH|c+m?Kp z5P(b51DIZ%^&-f}L+1~@)Ov5nH)xHXGL^9Xss6O;%9=E-gFq!CV?vC$+S~@DB`NId zMH%B;!%zLscDeMgAB^NIUy^D{C%3+bK(}D_3al+ z@kl)!qls3!2q^DQCiP@CPNv@8_Hmx8Gy5#-ol7XEuL5>h9Pv%l1$(+Wd?4uqHmQl; zgCS?MS)DXZ7Z@v!6RzN=6PV&SzBqQg#R4&4tQ>p(XM~`~j;VJV{Gu;u8^&WjueR+p z*m;BDg3_CP#brSB=jIP?55>}AdMmdX26se63A^XDA@o0hnHhOBSn1=e9lPM*$*t@$tUW^XMj2gqm51lOFM z++{ziA;A`5yExyDY*0wxN#=_|B9L&@d0ntUXB>=BZ(h0u`0Id(Ag|VsU5bk%4v`_U z=Yc$S#tS*sjQ^S70@Ggro8?F|HXuuNG*RcSVvcqnV1>7fd=bZf&gO!2v28@Py6e0Y0pq>y5|ks!_WtEl*mTUewZ$~B2+E7$ z4?R`|IFJqGGcaIt+o~Y>pUyeT;WD=?@0V-5nm(hp?T7&d?_@qV!^H_q=8qB1fd_2+ zGJcE+>RoB!r9nTUczJGRj^l;M;{NGSP&uP zwV@dDkjGDggX152J$vlsb@RBuL^00kO`dd~391K0nn$*Go-REl;a>%>s;Zg_oa*z- z?CN^>r&Frc&&Jbp&2QlFy3IA^DQeM9dhU#ryp+%8{ZcoAkvA}Uh@zm(ur6(8n1X_o zUs8rI=9}wqERr9UjTamZJ43R3OkJkhcqSMJ!#@y0w;PLxdcmr(g-3&Qgx$zl;6h}> z1k>)YXxOdougMHy&_TDXA!@^HWfV{GUUVh0zk=a|Pu6P(DJhjvi94G__DH9@Sq6uA6K+O(= z|MW{p;jfdh<1kl4_`p{X8#j8xTsmVE25rETXR&VZ1UJKeAr|1~%yzK&QPh!K_ys1IQd1qNLTG$! zuoB9h+EV*W)SVZ47Yke)-%;i&?zznS=wMs8*&eT$Pe#FPIWxX8EBvPRiCFM6SgKkM zb9yQgTsRbC1&fBKBje~XvKFl1(`Y1jY1tZv3K9tlp`$gu6CEJw5s+o2 zk~~_}XC)<>^CCC{??%sd8*J`)Fdarb{K@A?9l8V z?Qf9I2T<7j(f@lNndJ2W=U$FdHK@j{<$KoZblP1H)KhkhRU>Q9P{b`OQps zA++Foc*Qa@wAxv+s@1kvU+(gYojt#IlAjf*&640LdV8rn2yR|AMZpRKT6oKGn-;_N z8T}lVsFEj|4w>&Bvd3Yaz14n%)M;3b4QE>P!c=v2Bg`wXbEKiWoh|?`ZG3xbGt6O5 zjz-Ii7jB>UU{+#(6`(Tp^E6WvzCpT!;d?*9WNsI^&aRtrGzBzIKD;up*O|=l1bK^P z<5F-bEYU$CPBZzX-i4x*k)E(_T=T2JymQKsO366N8Bu|-Rk%O>hl=404Cp=H0K z4$LpJDW|ilU05J_ge0(Dw*EMF@ExI=+2@6wdhly~$sF>!0dY^?LT(j|bX1-`aaMC# zSt3SaV{5MKdD<({H)t82CZX5_=s)_wdf-Njt=u$HKr7gI*RX^Ywv~v9y^Jwjjo?wu z%@Det>OI=4-zr&=BNCT=*CH>(QTYE;9A`!JSt_Ku)LMp@@PEB`{#u+K2HnLC3_Wsf z{G2gOTOcl{e_v3PvqqwGj)sHu6VeyHKr+n*Q(^NFB-VzWU5K5O8N&q2f+gc+OSwY9 z2ZhA*&u%W@z`BivIl+tSDCHpK0rwGcxj2(Voi${BW%$+Ki}*pBVbX{JGMrY=UN7u) zR>oSxMJg+GtOz^{ zG)3SF@~xE8tk3h4wqgHR=mg$Bq2m$@i~G9BtJu-cK}XAm$m=i~xVe0=X&|N46HYjJ zIUPD4wukUG>HYe(%bxfvGw?(cvhJLRN)g1_q`@V%Tbcbwmo5U>5Og033&fD!9q@pc zP9D$DisL7vvZtPR-a)*6Fr&7q$D&>M^-5ni&tJoMuW3GT_`QB}UkJ}PrZAvK^X}a0 zrW6yt_cP2gNF&*Sly2H z?*{>g6GU+HiibJ>JE$W;_>x$QVIfjKRWPu+@p2%PRto#C!`9(5v^G-A2TBtmv(?tW z>f!kL`Nyas?z0W*7kvn%Ls7wEo39CmU&PG~@@_2ix*zYvf-7bigI8}t8bxKZ1#`mI zY#-k5e@TpQlmL5*M`c0_#dQ{)mHxJH64Kck8gBvBDDFKDw}POv6RAX@BPKPH^{&s$ zhMRX@)poyX6;2d7VXAXFjVT`4W^?F&-}9^ucW-(FeecG>UejBnlKXSlsXB7c>;2(Q zll9McXscMg8VUcS0oh>QcPoAla{=g?Q_$JjM|WRQ zs&09GGwGG`ovlKi`5iHz`Rx%^Saj@+&~OSLnl`(AQD>0$Cu+TSe#s+kNM-G$p~7ND z{lsHg$m&xj?OnH?y9^uK=q%R*3>k;NG`z>qQtHbz{p?xuak9JvTrgcKHfZfEvZd{K zdSZWSckz@Wb6W7!2N5phtV$|eq@BXE+x9K#(M^mpLN`X?E0q_H@OZnA6eXkpBvate$M>`l8y&_MVDzwjrW|8fDe$s_!M#VgCrx_m~lD%QIkTFny9 zYt0-l&bpcp5O)t)F#9LdV7G9iI|Za!?J7}Ok$yK?`wyIcaodO!iysy zy`#9S6;o+9t4fSXo;i)<7Zwv=aogCuQ*^hKg?Mf3S>q*>oOz710JZf%jAn&P|F!rs zUHZ1p{_6Cg$J`c>@6*WdU4fJ1r2!v{me4r;h_K`RHBm=SL5Z(ShYmV#2!Cgp|7+u{ zO|tSlB9&`K8vg~WyRZMml7m%mg22~5F@@;xeR^~Xg_*w~{Wafga@u%-+r-oZ_mr!M zKMz*w5&oWRe?f88>A;(V@KzYJS4j+Sl7O`H(~p#{YeP~6K@R)H3SZ}L3M`k6#GD=u zh&jqP>E};;ZR(5vXgx4+cms0@;Q{YMxKN@DV0GgQZC5Vx94)d3VU%;p6oj$v;T6r{Osw!xq~mLAd- ze^d5pcWT(|krX;;{Vei|SlX|JoZJU57rwRb)VbIYhKhoiMuo>QvZ0`mR%} zPW$H8tCxzQE~Q(amnC#KOmx0$gwj3aO;T75t0CS0TtCC`$t!KD;aplg%a{euv~bV* zx$`M{sj2sm^NUB=8x5WsgRWI*)eEemWKIzb>Ja2@L3`8bVjvYcMV1!^e^f4i)r8&J zd{IvmMnVDMk4O#HkPfPLWiO8CB^z3b*)o%$w|AR%n4UW}Y|X4o2R)c5qb7n#8wiW2kOAZm6yA=>_k+iqy`@Gq!>+xp#?6J? z#)aG#9nYT%(AQd=ffN3#k1bjr_nv+@#qB0rkR(}?sAT5-SvY6qH#>SsCh!D_Pu{6^ zf{YoM?291X`!Uw+AwhS?xtOkavVq`}-j&$+0Q-4`^8jypdaXVY-rB*p=n8B*&8|s2KXa#cZlM$!3iMi^RCw==KTfwj(}wGP zx|KELbUH852 zDU-TQGc$R5IZb^`bM9<&xM~r)q1Ji4zNq*9_$)kvO2GK-+2rv)vz6v=$xxc}35&}D zhg(P0YXoFW@_}#K=_;vq!=1bvL=21Rax_&7{7jTp)r#gIy}s`0_}}?=GK{2uSuo8WN_eytm*30S9lRt zg3|O)@jUO#9%&(dT!Bpf=8N=tt9+b@)9cRE;WfqsXMSM?+F5J{D;wvNFJB^oM6Qp4 zxcQ{+SM%>5SA4eS@LyvqgD3j9FFjuJfqn2s{rJ&rB5xI`sCD;%uY!U?cLHx6IsMQh zUFv&BH|;j4YlD6U`roQ^(UyF zp~X0^pa<7%e+agt52J2QSuB1W=}u)AqN`g89r@PzV6~@XNm$Ii)|A_z5V0g-pG&+N zzPB93Xd?cWAMVy|Tz=!`4&!yfP;Lcc%dQ}4Jo!?ih+gV}ceQzZ`f3e5H7*Ajr()yl zBr2`^u|G17%$M`UXr$`Hw?x&}5DT*2)xE}y=Z7r5UiM)$WvwykK5}YZnNMVmH5NI- zvk2#Jwbj4C%19xe+ayY<%GI22;l5oT>b}ykZJq7cq`EZ4RVIguqA4}o!5>?CNceWX z)$EAE?RNsvlgyFWUh21Egts!NG>yGJ^nJ>$u$xdfZQhEcyDhm-BTT@|rM;MrqA}!2 zp+1*ACSmUiiq-3i%{JTrQ5Y4fgJKDJ`2efY;@c>t;8kit1GqbxD$qfQn3WzmtI@c`1T|v;gczcia52y9H%OYZL}E;SF8T? zMV??s<|tI{G2vTOTO_w(LL z&KYialxxf-WttRVQsFDR+xYTS;50lsKqt}&ry+1xQPWN&r!4}TM@pTYg9Bjcr4 z4NY}2BdcQ)Wo>m+J0CyI#xxFEd_w5u+Lz$i4puiuqVnw~w&M61#FoWWqJ3YE^n9oL zDHxe1QTqD9cFa<Ma8JbiZW)!>tv1K7GuQt zp?hv%mG1%frkbGB=hSP2z0{?sM23`2kKVGYHlwD+W)1iA{tUzmP&|+44QY&CLbcQdbHV(+ zw5UgEwX;D_@~&~l$$EM2(2P%Xv{J}tp<1<0zt8yY$&7qyv1!74kkB@Bbt(5s?x+Uswr>Zlz_mA=-lXQ+m!|To03Z=T!+jd1`B4k9aB(6U}CuSID-^J}hS8>%^ z9~$$gH1FFIt0s&2h>MqvML8IjzMBbk{|UeM;#z}MzyE`P>w6s)ZC$rp?E*No zQ;1~8_C9vFQ{Q=q$3x4yw=&}1mnuW3@kj8(@5rQ#jBhqtv7WM{g#kHZ*=tW+kJ|0# zTW+4XTKFd~C|z@!!1E#e^R&1BSqVdrp*voX*%f6{9ZV=0MY(0KR=Tu@R?Wam zQNk_C6QbR;@B24cD5$1(i+)&st$UTReM5S))hGNxC<2WaF(2tD@40~&3&OrmR`kW81=g0kx)taQXuzEl0o5Au3GGu_nz*%bYD{l z*N2_kJ2#%S4NVqE;mHK-(eROICo-R}0l48;hl3Z3>|?#nh>`K3PpA`w#R%crLYT{3 z4t{uB0*O&7iSZj2YwO#O76q_tI5v1lrHD7^58~}3`TfO;){CA8pR)vO$oSU!P@$Z7 z>0Kt?n`Fp7<7zl<)trmmxn@>O^3}tjr8q6(F_mD_@j3JD!Ly0}C$6*GZV~d%zs2V@ zy`(lACf=r!^M_iC#NyHJwCR2ty`4Kge{eV`_ucN?kKV+j-4t7v=pYg0lDaqa6pwl!OLcbix8gmC=8CIh zN_pfB_lUplibEhg{QFD5tbhG5{Jy#}8&x9d%$Csnm-m63^P|}FTx?1BqDj9pyx0vQ zjRz1ci10(cx^SZcrd9f)n)tROJ9){kaKDcxu7ZZ7xcjTh6JaIkuW@|*=cY50WmL~% z@Fyzgd%|f6e>RMtOf`70)9EJ>vOXC5x_cZarzdWM^~vB%zU!{AnaS2s;!iUd)(||V zQL&P|Ov2BF-|a%>o^(l8+16Lp-Cy#i(H1njrq+zpCi1C;)z7dFarfoBpCPWTw8u*W z8YA&#X&N~E(?#~2q^?uV!f5KKX2V9ZPDf7b7(^})c8Y0cYmIOR*dew_D?;D$ik{?OO3eQxyRH#)|TrpMSsq=>IEk+7mqGI_h z0YmXy_ar+~cxL-orMr^^TYZg{S=lV`KELr-ThD2}LlvFQZ5t}AdlA{Qj#H0d&yz|cr0bk>^?H2;!+j_Vf2d5TmTWA1z`B(s zsXH-BZPNHa{;N%VDWyrgaZAZWMlgl=ic5tf!HdMC12hqtRsb5)T<~ z@tBA1p$$XPdYmgSllJPu%qj1a;N ze~8#DH)7U4cylui4pq>xQ)&51*51>vO4ZU6zX>cb6Ir82yUQ+wa%Z z$?zz#-KF}<`GUxsSnJ*~ky_8SMVI~cv1GXim-AzoOEXQHW+f4NQj>&iY|4W_} z6gxHsS4@mi^h(?X@|+b(7)lrLDkB+>E2g zisq^D^?-ifmoAL;t*dBf!>GoRQ}>NJ*&ms$6dk7qFT2U_DR&<94y0{P;$|l4i4c%k zS=?=^Oz3@EOwm-k_7UUsBwpUlGHAE|PJbmqBdLN{t;@5I-G_Iztol+F9#+xFTZA<3 z3hjAB;=KKm9d`Xd-u+ipk9v+x-yk+|>%@U(f*7enh*9f{(>MVy@%Q^@O+D0JPw_s! zRsEEzrK2=86R_d)Wx)0P49oFeD<%7p#!cO>v*aofM>twKve|Fh>005Wkr?F*)n&nw zIB_Lo(?)N6+0mWH);ER&{nMTYtj9DTG^#7j%fwu+#iPMx=0CadYnb6V=p&xv-Qaa2V~IizemVbW)9!f%BTpqz>+4C*){Q=* zZqVfmi8k>h*7-O+w3(&TV(Ynkf+2f#yQX$lgMD@;|0;X13|HOCvFD?Aqc?qmdE6O3 zVqI-`bp9}~>yq$k=lh|!frNh72qB4u->$hg|6r$R^B*L3r)Jsx}iK#ZfN zc{pOTN~cy=llo`WagI);*22&-{qFnm#(~rx9e0mEA9uS=rd0$?fa-ORP)^g4H>nVo zp9p;%`@LpaAF{MZ;SOY$N4_QfaBuRL3CQ98S54n}3*NCt^If4)%Gyw}tG`_uId=>S9?p%gKH$U6RJ zu5G4vJok;z+nQsNPKK)b5rYnc=eOq_x{i~QtyJs|Jt%EDlcW9ml@Dr`@d>q$!JLst zYL6XG?oA0gZHnYCubp+cOR0O(IyWo}Xfw-4_o)0vAP1e5&Gbv}-8}`3xW`N1bn^Os zK052eR+nM9BJqK%&y~SG^3?5&I{uRoCnp@J_r$bHUm!sfMf};&$!zBR9l?Q5vs>?z zbs|{K#rru&uC)eMe&^2Bg5EgdNb&2$)9a^x zBn`S!$d+$ZS{~0n-(wAlPhA)q+LXBE@?&e5Wa<>ggT=bXJJZ$sws4?Eg_o}aZS~S< z>zf6~jRUr(^AF9qI%V%In-0!-Q!Sax)-7|e0 z9qP>GLdMpeS~k{Xuxd8(W?b!CYRp%LExH#!pD#IkpA6C0Vu!2IG9^Pu)c9Pmh%H8q$f$(pLxC1iDSTMx*G;2X$(y#WxWbZ zlNlzr>)dp{YiaL0&vD`Ax}iqjco;rjkR$EHTDa7t&3^Z}zF`@ta@@LoH0q(V{os~0P50^Gf9M`r4x?f1Q_Jq2eEUxEkh=n%yiym~*2%7--qqSM*F1ArWuuNq zCYo=x9+gb0FeyQtaol@1%{Xhmy`t@{5r4r^K|BtR?nl=HbKG3g55z^v%r`Na5b$eo zaxWO@m}Vk=@uy|mj|_C0S?TU@o9Zx*s;dUSBV>5D7_Atpltj|N(6=_$Kyry!S}s!k z6GDnZ!uV_l+vWJjqif)#zDG@U=<&DbYz}1Kt)|a$vK2kL)*W&e!{{BDi*NxB}&7 z`YxMCsa4`cxP0WUC0Z})0qU%MNUK}maq6c$Y<0k$OJ`8YdE*^?x5^-u^{%-{@pudQ zm)urvy-Mp5y2ti^Y^i@dIG2!Z{g7@KL+&DPCi_}Mz*{MTg_D-UBrJZ_sN5+KkD-$Y ztJ2x(+}lE;Y)bisV@&ID)Ni&Z%bO!%t?MVzY@CrS_c|Z<@2OsmsW{DNmdN@1Gp1GB zMj-3iN<&o>9+|O+-P8P4h7`xaeH0JvMKgzxX8qTlVduFsjFiP|?!>G|o^xkVkBGF5b5Q*OF5IyQBL822W z5+zZBAQD6fB2lA82~mUS5?~?;H9}sQ|)>;C`xV=0^=51Gny3k7va8|3a>^O=#DCAYQf+7GPE1EUJeB(yJ+ zJ=@aSJn>BxshDs_Jqv@Nm<@4e4StL(WYG)$JhVc!R>X5~qbtOMcDVy*^xv1r971P7r3>cF6gJuw$ zm6Ug-d%Uwg7`yf5al^;vf|go`z8~EWuNd%tFFuoP^HpUiBxL{@xF36o_D6wt)X$1> zi@X;mgU0zYKhzE%PH-1%pG35q3{g;T5l-z=cg3Qvj#5zvI#4E51fx`vkwGih={QFm zOBZ=THUk`aVuo+q4u@w+eeqai8A}WeuylReP2vAqp?N=6t^Wx*bwct|F$>jH;ylCl z5AvJ)s--2v5~8e=evCdJ2Td-Dd~2e6@l}4$eSu*)u@-eeahmb50$*&MS?ikwkpRYqr9Dxx-^Y!U$7D_-1^{P&KmRX@*hV;|a0Zw=eUkRR#|90>g@dn^tm_i@xH_ z`umm-xv=`;=Vmeb?zAn=1P-exoX*(=?X0_b-eD& zSBN^9Fn>aDFwHYcT_PWzp6=e+CU9rkaBBLFH%8UzI_k}xA6I{4mUe#&5_DKB7sj36 zzJ1niBJ#dR=b7~w4=%R4xr;Vo<>D&pS7OmpTVHs5$DRB-OL|&Akme1)49P$hs%^=d z)_CU$P+_>?@8xHNj)8_Jm|X~M-jkY)rQNKZ1IOkm{QDNSK(_?qLY-&AKW|b~#MQU* zHIVjp^Qrf15E~<|f7-DxEXqjW!!v0xaI#Leuncy~4vaR~|8x2V0H}$Z5WoND^`R8; z&h2MA9!@+Fk9@5TpOqu8SI}@e=wvJZS{?}EYj&Mf`Tco{L}LTfmfWu&cITDT6zv_^ z#z$9uV=iB*Dg3|NIXccr%UR7qX1m>51y`YF2d0s*x0qL7qTt!pj%`#*vaX%WH|RLa zM`~I5zes;h_Jp>&}@{fy}l_Qnd+Jx7+pLyx-ex!b;&)@UnQx!Ta-R>KiJFZTAQmc4rf% zF;QQW*%ei5=aOg~8d~4;sb|TCk!{lVEQ?DeM=cZP+ z%TL-0gLmN&s844Q4me5OFHNdUkMZ~#R|-|t;BNk{y=mgnwACT*ldir-2Toow@eJs_ zZ2;|yOB>wLcJ?=$(@VzXg{{pAe~Ms>g4q963n0!J>3dhYNj0+B%Hn%WcA9mdOQY&% zR_TBv?Z>Kw_G|$PxRZB01%VSC))P#gTb8ax-C^G@RCvjp$vZkG^Kp9PeWI`Zb4YM> zE0QmxK(3f0kcV79k)qKsV5K-zRQP(V0^x$ zSBeLI>OUrv8a|28a>hQ!1m*Z$X#{KG^-WOC4XY2Rq=gdSIOJstv`~nN%9>y zcs8o?7k;SVWBEIkV64UUxG;3G{zrl=Ix*I&!JnRrL;igbM{um)WXP0E`uOxzrLW9f z$7|DhW}7DuzB(%=6rKo4jm+}|ISoCvGby)uvt7!B9{)9Jnlsayt%MQ@RlV=J88!aH zEbSS=H{wrY#V+}F74m{Z8pQkLXWwlHxTw}+TvCO8f76Y`I3L~=G%Xv00N#Ev=8s6X zlgC<*&e85RzoLFRBPoq6Wx9fxRjh3fdqJ{2okk>&dW!GbW@ar2dNotu9+}9gp1snO zD+WJqsJSY?0r)>fY$FO1w7t`kvJ6o2d6Xv`@<3@T-fme-D+rLl>VROy3qm?_=>)?PNM; z?P9*7#^Cos_qB8`wQTvY!_3*B;hLZ!w6mwy63@leY8d~v=TTMS=}^^w1@-NzsAM5^g8MZ_I^ z`dNILBW7gQQI)Xx!}LamS}c>q4W#!SmfCk6gDiuyoiEr-ve9d&yX^U^$wW%tA{GU# z!g-CcH`sCt9n0vSe)><wkx2GCq4)@uUO7F}-JeQL4)v-G->*_qtkSn~De4!Y=bTMnDB!>%4BJy#OzcLlt04lwVif_GQ#H<7hi6i?3HbywxoBg}y-Dc|WLCB5i=i_`2%K zc8!*Bki(CmL?@$(OL|g*n25$E#{{tVA0?0(kak^;G9fFtGs|rH(lhlVP4(=Dv+qN; zgceKT@tK0jzovwINsngk9zr?uv1^)`wu{Tx3e6NcpLc|F@f6;=GfZ;@lpktk;FG@A zb_{2RW{GTZ$Oq|VOo@*5@E42K*KxVD_vVMS12EZ!5?;DpE*93aoRT9xE6hb2_ZGwx zew}w;=%D@G)cdX8H?Zu|)OJLy#Cu?0A7%@|caqrZ9r^DnboJh(0vl@fWpxpq)}+%7 z^{tL-*qmUnN!I@Wutzh%dGCB)B zC-rqT#4#3lH;dPYt%o8pP|Pms{1rR+YJEddJ5@GH)`=nQHpR=#UhiiPs-}l#9hoxf zPUxcW>6YQd>$q8|B)guJs+0th>xCKFC=cbd9D^#CZeecMemyJy%|P*9h2Za^>E-Cs zE}M~BgKwXkYSYPxd3v&OefOk4n!eGGMUT>A??sHHc&HgY@^rp0RvM|AcX9J+jpz-t zjc8P%-{GX`=H1z#Ek}z?jF0&+o|HE^S9S1uk|2W}>DBQWzS0YxWgZT^9}WZ9^{u7_ zY$uvrC9s!cN9H(qF!r?xAJRI0jlMMg6u+pXE1uq99@D{A={g@+^XNhHQDnVahodH; zA3Yc14k#3Z$@~7MhwDguQKsdZ+Z`%kJuNA2(^0-s=jmnUI+IAyFYxeq^dVG5nLs#4 z75e%~pn3O;O->55Oe&7zv_4kjWg2Sc*Q^$w8Jg>b^}fbLfBw!p0W4Lzu_J}W+QVH@x9X7h$nj)9vWAB8r%uhBhCKk6|fe(P^~ zuvHlH*!-u&pm5NO;H;OoUmmD>+4lKnDcmU!Y_jiro5H-AH}>JJf=NSARlCbB+Md4i z&MmR7OyJC1=PjNky%`Mac(8(RezV^1@w2u*OLXmXA?eQAc+rVaOqb{5l_yM8c6Te1 z9bzBb)YQ|Ye6ZWxd3XQUg2gZE08_zH+BuZD{*bw@;bJy=4)(@3s;zg0mZbM2&i@x{ zU0D-G9hKLvba*9^w$?QJxi_1zX@L=2oC><^Z;LNoQ5Iko%mbRA_c4h>-#ohF`8nfs=Q8;f+L=GU<{ z#j3|CT}{^OLUl!>oA1RTZ+!CYVkgmFND_~FDf`;s&x{s#KXsZuuZ_v!zW9GL}M%0OY2`6-%St{9HDY!6b3pv-I6nezyX@5MH;Ud7&RswV~pkUT( zu;tIV+Eh}=SNOA$2%oJ%gD^2SaN5%ZFyC&e$??{0nZ>gzs@sd7(VS63zrRiQH>>@W zvDb_&j!7!I!QVwk3aVTwB<_DQYB_9K+NW61fAEZ*YPBQC-}g`NC%FuF}LLqDQQKiNS(^K===7sFOMx>#ox*oR+CM5pm23!gCT9lJ|@ZM;A(}lg7=Vk zc~mFwo9CBeQNbZtwfDm&ECTR2)}%k)d%R8iS1kvBJE}&4V!mR}&mRTeMQ%T6c48KH z=qfnSd4yABJCGiB?!d37Y)QmS53#65>M@8EZ{MVxxf{z9ZOFK=ZIizEim(OQk zRyYKw^iW;ziJktnSwEPnN?0dab!m>8kHEI{La@xczEmkLEl@NJqHNOGBID*1Ier@h zM7@U5>46XizK^h`47VmhhwaHyjyPd$?N6)wf30BEQQnPpe@ZBy`*c4PVz}NOs>XKv zCrS0EZ@-nl2`914T?eokxp3E-QDW3uUf)qhY?#)uap_Kbq zmObki<8*HAecxB~@%)fmfdXY93N=R&y}$1>ektvXU>SZP9g-%N7!=%>^y%@n zc@5@U_OIz@kL*aP6CSF6C~WEcAieCH>-|E@-#+f?4yZ>z8_=tVXa<+qBKwE^`*sw5 zZkW{RkBUEe2P*W?Qd8|YwJs4?t*k98H)~NUA31`v%SyZyWp1BH?{^BAw0s5!zv-BB zHjg7{jAEZp){LFX46mZ1OA{+}niebFak|_6HaNH12VF!W#>9K);8lv}SrT6k_atZi zH&bQyHiCIuoV4jk`N`O==$FBf-k`Wt?}AWLzg^&|;EG6ICsuHjblbn}Ml=DBwo0|8 z5F0qZEN4%vqHz@Y%YD0F?8ia+Q&X~cJksL$I4C1IyU5KlcXO10GdN&-BQ>o>UeV`- zZcbxy|2z{0;%Ij)K zGN^7X9S8Tyz6^R)aDaTK$Mt90TDNQX* zrgEqqCZ6|x_>q^5eCRf+x)UpLH@#*fy>9zUcTn(PiBQAupi4iE7gh%v;MjM{?&r*E z<3FYskYncpx<=6R*llBMAV)~9(qR)aXSiaHj#2QNzw&B`7`3N%9I<`*rhhWL7;SdR zVO^V;3SbRSTSV2nLpePk#*lOx;ZNUuuYG5k7@q*vRRbKc}j;wa& zb*>89uBW0Q-?dl9YDp&!C_Aqie0x&%o90}b@<$`V0~HaH#YG`jlEj>NZzl1W;bT6x zix|~(5jY#TR|&2Hs*dQLb6%H|T2AO@5NVXe6O0jmVkvMqL>P4zNdHw>uO?h@JPD$BPW z-}u#=fb3vd1;;ij#X};IbOSg4yg1%_isuSSV=WOGtM8u;5eogiKF(qM6J1Z5?@G{E z=n$Y4Cn&~m11dDm1j{Xk7517n+6lc{{Mr^^yZXc)R6)6Ej{@_X^$m8@{;^k%z ztX+;Xa0&Gcuzb&Ib>OG@OZ-KcTr5&$r!2h!>B{|){#v%Yx_sfciI#QFb>1X>|0$uo z7rJJQ28z68fBG@<_dT(ABK6z5LlaI{-c%i!^?jJmz=hmKkx>WVR#`o37zE?ea3fT< zp2Ig}ogRzcp!Fxgjlb97wYiUQva2O{m<%c#%8(yaGQJWkzQ}Oc>n_=bT6~(fXxyxtOwuh~^ zytV5NyT*U?J-kx4?sz$KD}s3`6rQ5i`Z>{V7ueD zjY)afNhodCcw+Qk*wnt>@3bBmDNXa;i;ABZuN&4eTtHBOgnVgmeZcXx_aPD>Q>61K z5^lHZ5ujk%?9D!0VSOoE)wo3e?(p^Z$U#tqhA!n$FGn+7Q$NCHpDVeN*hCel{5A<3 zyRltMR7}GufDGTprVpNA8$>@d<4aq6>q}vK)5W`~GFms9P8u9;NOpexk>DcbzB6a` zK9uW%;-~E=vX%Mgo#~^0v#lBDyk$+c?(v*iA@UIo+d41+`%VL_?k4rX4ee(~XFkmh z9hZ{dcmKUTaXs#!1&hD&@G3s1*VASlt^HIdVm+k@NQ!rYBp#mq{mfZ^hxUTVtz-l{}gH2)w6K zQ|9^3n{eQH6JCGPSs(M`&!W}5QHg|rl;RDwVg6r{Cg|6oX7})E7|rc`Cgie zOGZJrqsVnI5xWdnzyJ}?Qs1FKzv0He=bRg1*-LAC1BR}3VR?hUH5=uQx!YUD-FI(- zW22?r3A;M)W5xT%%xl}J7w9FK@yRM+^z{mq!N$La{x-odQEUtKk zjrt8QIBC;G7(a7+d<8wMD}F?q(d+I?vvC2$9Zd$35wl>+p@J*)P6mZXhwK{1vGR;k zeiR?RSx(;Q1;%w+>6?P9fY{S~j_DF6B7}?T+tzd`Q?Qt3hdROz+++z4;vz#H5f+Hz zWMDtl-yS{0Lp$}zRus$%_HdAr!i|O;${zb`qhok(2NEbV!Ja9w#!P6pR-dMi2SDY} zF_E)*o%{ti)5Q3}pLQP>mnQRDqqz*(z}FCycyE0hICDdUDd2&@c+-CKi%T?ewYQB* z(Fc@qj6|bS#aO;#-z0<4%g(`HLE^-Y@^WpNLV)u&-d3I?yQb3p}T)gb` zKKb9;83)5cTJ6FKdVNFz74Ae`?ex~3GCF6+I-R5S3M{d>mBbJ-Q8_|c1S6$wOQzL) z_eS)CVe#-R$^tvO{Z?lnrQo~Dz-@zu6-JD$v{T4?mr|4$ zPwDCXpVLzvb`kRglvjVCuqDE6o+RG=3~Wjr6W$ApM(k;v>vBI&7Bce{@OpHg_ePPxc`(@NsBR?ERq1ri4jWVnPMPUJMKU@u>7nBAKQ z+l6!S!q1`lsVHtD_CAtPekK9B@QtSb<+IrOCny~Ho2F`~4w@CP4oM;?kpp@X5Igps z4u;(j04mTFkIDz2vXKrnFAs081Z{FE>4D~z*RU(j8ORD5{E7j0Zwqp%+B=MZ@~vpt zqHt?4VW(^};cwj`fq)`g48r*OEhbzTEFD(asoMPcEKTecFtobBL4aDgfZu2%+gVD8R#e9kS-4@n>Nl5hJ9aZA&NpcB(KqfZbDscXbrq zw@cYAF=^%y=fVQ~!y=sFwoIJL%5=wh#um4CFs zzqs}=0s=C9G{6edlw-Ge>moD44Wq`8rtUu|(d%F#LCmP36Zpn4}gNzL$0gtQWXZG z*)#_{o@vc4C(G*`KVMs5Y5^-=Wh4v%8J0?g@VZyKMNQ_Kk%6_zv2e%?xWlVp=C{5- z5-A>_?lnMM(g54_$DAA;P5-WS0ho#w?L`K$QS>i(?Bw%L!YiqjZ@{Jq9xp2uN*IHG z-lI-QU!s7qe*Ibh$5HGL)B^#uX#fpJh+8Yh4!fKW>kTjJwEoi>q9Q>Z6LC${6Wof1 z@<2!^1s4Gu%>}@t56E=)I+djxq>EBRpE)n-58mSGX4uw%jaSCnc;5xGZ9#e?M9;Jm ztoFOdghll^0ef!JE|A8ZhZVsepRtW&=v*+u+MWZ0ncjZ=zc4Y5%aU+W9r7AhT*e9Y zg<_x#m5kxy(YyJNU_2H*N|5AjIS^g1Xhia1ixjrF5X-VM>~sj2$3NdWQnLgFlyM{* zQUmSQ(-s6Y+LYiOR0I&x_$lZD>ja$%5`7ZB^Hqbst?2L_?l-|^7eOH`8vc^BWPp-_ zphoiNogQD@7NnhY;gH5=Q58gA)abC)ZY6-Gf(eN?f&=PoZ<#;hEyMq+1-S6n1RBBk z?)T}wfwv>?gK1~DJ9bXd-T>l;m(}fs!9ZC1tFP^)w-@9lpc|)bh-dH*K`F>$U6}uZ zT%i~ZSV|hHb^rtamZxQqlHF_mGzQ!kGpM|Y-h%eDql5Qp&ix3o>ClFaGY|fy^?#d5 zWGSC&&EXE+U6?n?1__RHjA5slO{^GDS;-kr4U>hXyi=vde3-%f?R9|ZN3~ut=)tb@ za=I3!;-*dk%K{AY-}b)wZvd-VPJVq{X zSqDcO0$9Ba3GT_fq(oL9TzhrSAI1Z7vRjyYcn=b{B%(v@-3ja<fIgUA#j z&~V7$Z|uT*>cbpM|6N;-|KcjE*A0F>;Za2rR>DC9bTL>&Ula6GvGqW#--?MIeYGNJ zBDf|p0WtJK_&)3@e55eLKfv~m-x3>X7PPJO3TW}XKt^pH7fS(Ya4U@D*INL#pjM zWbGHYQmo}9_0QM};5sgB<*z|=#k5+C9qL%a@V}lU_*fS6Vr>;5!@Ou;AU`8CRA^?B)`F3L|Q*cqI z3~c|m&0+ql7QQKa?7;VSLjl*YLm4v(?HICJO?A}IOegs|)lYR)_4@!NdCYA~XO>A) z$$4vHW(8{t!J+Bx#Qz07Q~ZiV?y6pYcwwI8*{2DVODYBAILz348+aDeSwnY2pdtBa zFpIC0L&#>_G$SFYIut{2-YbupHIALeNc)~5D9wfd__QfL?Jm$&MgSD8tH60GBbuRL zV9K;pJC;)xrv$?ix-%;`XlLD30}@HW$q_OWWR;(1MHbwb%N;pwm86Q0QAu5Txe(=)oyJD zLsX88fiD&R?>b)nD{?=zlfQxXXz1k?M9MWUIR#wW4tY2`kkuY6ya;x|$wQl;n7>Q; zltCAfO%-t{PdJEy5IBpxGw&6#(-o&veXL`x`OSo)dv20=gq;3qdfVrpMn)vz1!&uq zCN>x%FWJS$$oSd7>8vi`tqo>f%F|9+mq0KS$=B}I-gOi9xI*SCO{@7iS)F|FU~rzY zFU$h|9$u;6_66{pJ+X$(ucwa(k3{!yPboiI2HmfRpzTQ8*PdLoBL zqpj0E2B5Zn4+}1^{q287vq6GL&OeIjy0?qw<6B(Yql}t8RO?BAg1vv`HXw*nc7Os3 z@ue}>70?CN$y$m)w9A*e-wx=I4I1tIr4FWznC4ww*lcM=G1^0jPlIUquqg%OmK0M> zmu}P1Y3)5_$LN}Oq9o}mjlDHcZOIXRI%mXWf(Gj>$xE0+j z07^Q^1A4D%-RVCvHzRiId;3efRRnT%RgSLbZ*>=;3p}gjoi}tgcmESZZVDfVQv~&L z!c66BLka~>P!jeaY7vE;^+MHxKIHjP*?^i@{f1hmZ>BNFB(R#>vodUuYBT$lpAs?xXq;Cq` z3(z|48Hgi13U6IF{^9Yx%?+un{G)1>JRb2k3;N%)L*M%9vj>9B+5+2cSKZ?kfxFz} z4sVAEIH&#~uIv-x7j*)HazpJgw@ThA0Pfr0-Y7{0aRRyGd(T6csUN&lNTzoeU1JgZ zTo7#dpO)d*y*oR3w=;G}p6@U#DT-?U|Dzq|p)0pw3oh0#U`CUh>|azM?cf$AQsxq> z{|>vbcdH-)ICk^^DPy<6d1BG?12^XgvC|}wUQBZWQ-SnCk3l z7Vs}cwE_uTp}8bp_gF4JE+HJulVyOWz)6giguu5YzQKF75Y3Pb$X~d2&S=fbC+rOW z#^G@OJ@K}0iHh`2M=vJazKsM8TKw#l!sI^38fLE@Iz2uE@sSL89#qtZ%xPhue=5*U z>qzJ1!Z9XSl8-Ys*MjN|M4&x+3a%=;(LtSY;jWlCRQ2rz3}xdSP8#-zN%u{bgxpOG zTnXA=X)u`e#ixi)vOG2sXF=2((_8U305(HFkLE*WH7=lBr$Sn9Bb%#A5+Vo%p@67c zn1T(0z;JgE@n9_LS0?M_a4O~~1q!8a3c?4V>7A_-;w(&XaGag0p!8U^pc4#7&*=_V zM&o>7&972yyrT$%qhZN)LQUyHEfWnaDk=d3Wk;|muSeCDW!`_Wp>>V$!n#s2Ekh^) z03oYU%BA;O7 zDN!YjvWy&w+7#eQ(Zg0@*e2Ve2aKtBoJAJM3fg*-i`@Rl7Vapa=!7nIB&TwP7?TI- z4MXq@_R+f2t#LM0KSs{N;C;mkVR>Hqooz8I%;d+=)=&{RuJRVm?G)(IaST|RT>9;) zg)o?UFnS2lO88CdJ?wN4Gy%>&XG;s_P~(C=?JP<|m|jIMRDLw6Cq$LZZCQU zsm%rrP|CfdofO$F;~1-QCY+8UhZq60_Yw$KhrMv_oj`27FkvgT0T0)r(T-L*E_ojh z8SWx-(xo*I7ip=cIa16?jCubsY)P@D`L#^Ng=V z7&>c2-it6CyYhJ2?6HlIxFu#gL|824m>3BTf`c0?=+nof}S#W>pHK zz2>UBFYy^oxDvOs67WonjhrK1w?$eC$SVxmCA17Z_zc3EJz;27u6NSFsv^uC0EBJ6 z=3_#Er>u}z0{5=z+lDZbmGAP$(4ttwJwcC6>kN|0Hc$2e49&(RAnOr#JB!4zv890i zEp|8@{_Fj5z1w^B>SUVfV!zp$)jf}ms}HmM6*^o5r)=NVly*bHnWg|uFlV~H{MqO` z?%Z2mKdk@t!^*PRgJhRo{^~mtBBF^zdeoi%ksYF#AbUmNFu+SQK(AH(ovMUR6D81Q zc*u%{N?z|-9IDzSWEqw<`&N48Gz~e4Mi<=;i4ON9Ijoafm}UeopQu62ea-8I;0e3= z{Y?C%>I#7FqEHI?&4SsDVh?t$mM`*1mEW-7pP$hyr%@1g+KteV7;SsC=O!sn!eaVT za|=31%617r!>_=b0$^oK{1eKBoryk%(f4Kfw8tG zKsu&Aj&1YTTwujD?3`CzVYYVlnzr`PXdCR|6|mqkid|M^W7zN;DS0gr1#d_o1}VOO zOoC}1m>r$DPbdhHswYU0TS*gF-f#Rk5h-a?pbUk+&}e}fG?0k zIafg$avGbSChru_k=Lf*JIASU2V=009FPa7)uM_j*$x=Yt0?)>;io<{%5}>EG(Z*kI$echtQ>At69@O{aA=n$EMZ zt`&CFF?2{`jNp95`YZr!u*&&qBz zs|2O)w?hZeuNI+p!TO~UC&{h z$=&dAbfG1MPRx{LVCM`T-oFevz0;qj(^MEIw2ISISmh_#j(%M+$uIKOkK$0m-*^zR zP>bYygmZ5mH_>MWCtn+2xa$DLv5%-3i3`#$u!!*HMCa0`{yz-VL~tPL3*)F2(aha6 zAXmnW8-p#F59!IC2Ib(@P;H=RN|xTy`e5iZ2o~F`AAJF7S4lS##G)g4-x3xWnh32a zRY4a0Gy*mlB4gSCUPdhub`Utd_=IO^ZPw0Dr3EDX-{dPFLI-w4HZ6;L5BfhKF2f`d zXemV=v;Xh3N7cH~t)F3v#O}1pB(N>P_48~%Wj7oV74VTW=ob2t;IQHM_X%#?28&KL zv?azJj0rNqS(CS!WZ=NR)I(8&u;p!igf`_g1MC_M%SF{q!&n&7HNe#B{{L0%OzWYf z=$5Fk0bMcHm#;d#lYmwalWJ#Ni1cRk(=+gU^OJiQe* zyBEYyuA(d|%BP_*FL0fghw<79y_NO)`!rzs-?H8nXAfuNB{1Y+RK zN&Zw`O^iYe)53}buW#`1*29~1aH`zQER{Z`9Vb=Oru_4#8zPMi@L_JP~sYH$~ zR=F_~t%NyE)&F@i>SZoTB%J2`q~sj870oB%fpdf?Wf+`rfL|Gf$=&LGOs$L#KoC-M z%9M3C3LtqKCMVWZ5AJb}TF6Gh5YE(#d`qM{wU!Cz56gQMUlwCh=9aOY@8p|m4ZVp(rx?N`kV_B! zG!tQnE?RUf>b18J2toH*qZ z!t$}p41V<{*iRp(3T0>z$T10*c7zeXoGcEY;}@PQrv ztln&Jh~V(|q$qXm@pN%CAxirRSgp3f5qzy}(j2eZSjsqg?+Vq@88GoBTPX%r(@zx5 zF<6Vf+-R`kIRR1Qfg#r)#CZCJK=h6pEc`nXrOuZmaZ@J=mqBx=j(`5#sn7W{o%DD{ zaFFIe;pY|3H>!ZaC}ngm;Ol_QdIB??5YG$PP2+W|H1<$icJTLt-8E9XKy08EFL>+| z;;;dw%B2&gBWpO&DtTAb7YMSZwG4kDLsDx%w?>hf1Q1XJO-m-htCOP4tHYslV2W2L z)41GvG7@lU;2FL_wxvYlJ$ExHS5*i9wav6w1_DAj|E}ZfvA(Auww&9&@ZF00Jx3bH{ zdQ{sr?qzCTO~95+q@~~z+3u=8gKA~pC1>!q@(TpLleyYJGJfdOZ}dtG3{gsrjx}QT z>t(hHzr+1Tq-%(?`o}9Lvhts6$n0~@?f+hzjPNqY$>bQxG|5W?8Ti{W#LPhDjBdft zrG=JRM7%xN%qjW89dh|+l1C^ZDz>IIAJ42;ZTlsiaU^cp%1e6>#o)S!VcI zZCfdwcie9)iCzc5mB#uuzltqxk4&D94uiDdFPfoa{TPc!7>ZX!wtAYep-iI9H2DC;FN=jkeu)R z))oVt33oEKi%Q<&cSH;rq+K!*b&cCK&kg)cG7l@60QVr_(wTxKhKI|@L)^py115@@ z?2JOydfse!dwfUk<@C)3LP(Ilh+4~@Pp+3O-c4j)_g8GR8ih3SvGFvYPIAMwyi{*K zLnIpI1_t<*rxa)t9DbL3^bd?9&D;Ce#XcrAd@D*k`XZ3O0qd&o#a zbE`vhzpLdYKCCAX^u_yG!rnfW7=SeQF>)NUBst8~Xg{ioyjm0W*L3@@8N1bOonPB6 zv`lUf1#?q_0jRT)4b&{*_E-hZC3c;%_m75`Ru+BnD^!IZ_zQ--@M1_!fnWoHV2h-9 zw~^ql4;o+=;;JDPLz)goCr=MvShf`#7J`l4?} z-ajbG4%WGp?V=eEP?8=Un=-sV(Mp6WXRWHi2uF$vRNS1gg&4j{=<9_cJe2AlbHoc* zOkhS?z*_sS;g7Z!U9gCNFG}yCQ0Ce>KM}~fAFGS3r`$-!O+0kWKrw4q z|0JmuyxWo#6x!Q*3GnO=f<@<`qND@=NN-GiApI3a)S3`44Cr2x=-Jy8a?dMS#^wqw zEd+BsoSnC_eO#iz&4yX(YU0Qx;Dj^{(BuSWE$kYB14?2`fR>wI)Vrv|t25TdwnPdr zqY-h68R>*8FO3fMzmD+=M{b0s%WB(z*?kc)M2Z!E^o=`55^R$sr)MJPa6 zK9^;pR!K#^_KtfKA*Ca$H&BBYP6&2eLjNV59|hPPp>rb^_$V^41euFlzt$w8o{v*^ z=vI>I9WJjGv(mzXDo4PcOXF>M_+mgG8R2c6`^DqZ>-QbHoBII= z-U5PYXi=^fkaZ=0nbSDZKLn|g$G&bR@^J8+YI%l%yJ9Q!0vOblnXeUH(zZ1bsBnrbv^`w zb+^6^f&iqt30Oy>gjYQ|dS$Nm=sZe!?@h;>QOE}Q5?kBxpf~UdAl&+#!8Pe~fJy2F zZ?G0)j$AtA+-5aPKWM}+L<8oXhqx#dp)4H^$0uceH5<^H@pW9LA#LJvCegV{{6$yk z^FMPJL;@b^#`Zm-2CyLy#Ppbc3&o&Z$--EEytIa`bz}_fouzZs4lATJ^M@M3?!&^4k$AIN^85#fN%nChMKsNWdQE|A@xl zR?srvCOY9-x}kGXr&`X~=`ZWiMyh~Y-0y#Ug%qA0FkdjrnWH>k&{=BtW0Nv$=9~P0 z*XaVL5NzBN!p>TL(~VZhx0#Lwf8(zan&H~7H8pPN5CZO^#CkAi);{c{E)t@bQS|#%H+7 zo3;WgoOue0BA>V2ftOK6gYOM8vf_S{{cXllN9snS#5ZXEL8RrmrQqGMG*=noO7Kt= zV5qV5(wWy+5$`4@D4uEOTzbY;I8j-qmpy#o^S6lQdRT;zipwgw4uf7G=B@n=z&P;T=UU_9UA`B{FPOE) z0et5_3oLdZ?&gRL`A7_@h4xe4xJ?Hc3Hp#u#kJ&T%UCO;?U!DvU?Pa-$H2T>A3&g@ z9j&Z;7rA5BYPV)LA9Z}g@4$?R@SL1J#EK$e_s541%X61cZYm7?s}|t#^C?$MF_6SWi4=jy>;MB>Y8*e__PhUX1<3)T858WFTQ9Bn{}WRsVe>{xTJQC z-s{WzBz*W^Tt^WAxr=3C8)AXtJ77|((W-~aFuS|3J~Yt4pK+29TYe%atY9w}3*w6$UK79Dy0u$gjurfX>x-fIm?+JhgMoC z-?9WivWVjyIYOVLe1QF`s?CQ!x;A-%K?oo$Q?H$tU1N`_xow^-eQfBGgv7I2##P$5#Qjm^|wz<|oy2F`! zmrkQ6G)e@p|5uJk;5{JqE4(Qw=!g(u->gHHu&GwlurrZV z%5X0lQig)sQ}ET(*^x^lLapSGLeH7%0uyp0BdlhSRvnndui?GTcN(i4BMI44zN!sW z;fd8B+jtZGW&Kyk{eQY5VFVPvUb*KkauqR1Ren%7wX$%xY{LFVTUp8sP%m+!F%Ut{ zxca*_qS}JGScQuLnVe)JiJ|mPvfpyOD#3|oA9jy^symk6^g(X!Rh_+50dP*bE(ec$s2DvY!Y}g7fb_N&P@cwyJU0pp+ z)scrRu`F<%XfS^>f9vOnv%Sw!Lt4&xw;a#Zjd#h;sJ9=}EQkn$1lKi>wa~$~3@Z?R zDq9k*2+u#m)_4-skA+0PmVQA{GzFBD32yPT9mIa;$Dm*+S$};I37pGSuvmw(=p+rJ z))TF776UEZ6ZCISK9}YY{RCU60UrELie%G*)d=F0U2wupC`gm~ME!1RxOi7lXAh(y zc2sS%5%jv!2PN-nzN~%2oK;0@szg$ZPlb`_jA2CIQxU);yD-tckD$eUCA{hZkF>ZJ%W7Fj5UanKoqa=Xd&+znQ+Dose>ow^j zNp=Mc6LM6Pl=T%opyG5UGp zNdq~0BscgJ8c7kg!7CcBXq!e@1kv+TCf*_P^~OX##~!aDkWycF3RVD&EdhW?X_mS9 z{xx|p=ab@+JJ^tvL%R_P1fn)HBja(9(ZC;#0wr$<6oA=7CW4e2?4x8XxkgN}C4kwM zZoFKdI|_E~)%5-wkHoAUBV8wU``>nY_XV~{x-o_W`+YP|LbbBo5x$=_8$m*^=&v?# z-#PDGM-mJXM>{fRqB1g=xMByq*MW=&m!mQOH!`?I&RFIX*Xc|d#lB-}llW$LLtyf+ zfpuhEL{Er3Z56>^z5lAY#4SGXhGC{#_|^YI*IP$L`GswxGSuK845%O=ozhAQ(m9}l zN;je)9nw9tN;ybKBOnb!ONSzglq21!NQZQD_Ji;5UEli7Ie)m8x@Ml)d*6NC*S()o z?w}Ier|gm?^5ByA#DFU9R ze8c*WbSEu`U7m@Myy1%I?++z+_b?qNIJABm#OG=@i|=!+^W&N)gp5RFwFUs;E1`Dv zb0?qukO}*h`3)G86kfxwIUa#w`q&tNQPh(eno4o3;M1?bazAumIf)DkjAyE-Da^5c zuz>W-k8kC}@J@Q=<@~U9V{`xKdoQ4ip{(-Woc+jF=8wjwG;7yB#4HV#U3Rdl0Yz{z zAr+rd@elIrKl%Fl&u$-GM(f1MQUt^KZgDdfQ!R|=(aZ*iitMBeL!LS5;>sU>xB>Fp zn5-`0Y*A8lxC7X56+u1FO%9ADKycPx>QFCU0^aEXF!<|MQPPR>nbkQj);=A3JbFp< z(Fk_q)#%goBQdKdZM~n-ph(b$8EM`dbQKygfR7~G>WuxhMXlO8Uw4m6Xbr|h zdjzzO8t6i54&^IcSa8x#tppV9UgaXNY5<5J?+f(F5g^VyT$r7YYyyIZ-J}kMvX|Ow zXT7_IAlG9@c;AhhVAzcn+YrYWvSDGTqSy(3RSj12DW0#xDRy~awUn~Gk;@I!3_jfm z0RTRy3x0nIyw&PWaY)n=)p}KGK`TsB)msI(zJ@QL9{=OtVr0R)p}|UKDHY-K9YgQ| zWpKvgO$G!Mc4=?z+1$to$VI!v8?@P~UJ|=21WM_ph)--|!7+X^%zOp1`o%EwGX{eT z=6a3cO6y=n@-SjhP#B`lp5ujtvku7AgS3PM>@YU=#gl7<)L5LN&btLmloj={U20|$ ztPfRT+Q^^is((|XyYAnDRI{PesZ38`YWMlrg8M|$;w5gl_90z#r0S`7euU%6;==mQ zV*dKzlfkgDJuDx~)Od13V%6V$R*neGtqI~e?OrA4`Xl%JTMF)d$P5_Q^!(Wz3 z63xj*&%gKtwP#t2Fziq?V?+tmHLtOM^*?y`FZTFM~FYyP-0&4Mq$$HAYB2^i6#8U1Wb77mBwQ8Jz(( z1yJ7Ix7jBcGxb5ehVf8jv&NJr4FwGTM%BzhmoO_u^2`mx&-K7moDZORia^3@9LeTG z1~d35LsSLsnd9K$v+GPYuwgX`Q{1&$s>R5Bx7gGFiSbv^RRk0qXTth^y$jhU^Sck{ z?Mn_l9r!NEg`73tmdj%tYoY94{w44z&XQ*?ETMdOhEsX@rBKFSe+kQ_OJ27vM}pZ_ z+x@^!`Z$4L*X4;fCLc~IZMZlL3j^_YLD{(!}kDXRf|;q7ei=X0HZOz7^8 z7gFjFvRjwU_fb%|oYfKlBf#0Jdz-RaUqVqiNoYJgU__ngQjii~jQEyyncd9ov7u@O z2BSk2mB5R4RNFs9U!D2Y7;|5JXyY32BG9(JQ>dyV+dc{0$&MkC5gI|TY)i}!q{2w7 z;Ved;KYw}K_jATiorGNR(Hyi$vORF6x#L7Te{AQrI=e zBy8)I#C*Z~AKNv%ffgpDDCTZwNZTtG{K6g^yUofF^cY z3A>saB2r1#qO+jWRB6NKRsx!pgE~6FL^}y%dHr%Bw*z{HQ%AQ3bMzGkOLX(Pr00LN zpZu)oINXZ4ZBuXK9)LMM*_%#2T)sP4q^oi-#j53QMATf2b>-mWozI2SnGS8A;|70I zd9iY{sJ3XF7{@ig{Y`j@!(Z~dR7Soj$M{aU&G>@sLTeDQ47JyiI6V*pQVe|=)V=&j zgUY97B^QGT>&|0-Gl4g7{9~B+YV{9s^am2+opQiuRzNN!igG;1xOM3Mp%G3Gm6Au# zI{T|uqrow5J2+=jFHd;3vS;J`T+~=%-!lc_9cfbpjhMqoP~O49U((C|mQS>qe9Rs7n!ohvMyoH7T@t)va6wn=2%DrW6$pn!Tse@>dp#dBrN zW4qmgaVdO#MW}1j5*_=4f2okq!>A1B)gGP4b~f_c2?sR9KWRUh!_O8S+@(!s*_-d* z5h7wxnkBki{a|^=py*zM=wZQ%MM>rCQhlROfV5%P?8#?-fcvPS>5VU#JBl4r@CRBU5r3n%ap)7)%)FqmIVzD6fN)9({Mk%o|HEGln-pM+qMK5;8J~=3u#|X?o?j)1%?hWzLZ$d+plEv%v?$J2Hho zJFwJi8dCfIQQk+k)kd{%InD%2y5dG`$w_Z(_vkum{XHBn^UbP^@yi+W*JqkVC;C{Y zi5BVDS(t}TNiht5spZgWn0B^WdG|wLEn&;^-9g!RT$7psGOKXz$}u&441x1baW84Q(7wr|Mb=Du-LJtTQhm26rZ3^=CP!WdaBkdqhHKTeIhDr5 zD~a%;K&R&g5}&+)e=mn|g5r&~R4pEK(ll0{(Od)G!X!|Jid9C*RS*WVQmfb*U3qlZ zP~$~{9olV@^3p2XsDxi}PvXjfrR%xRQ=(Ee1lQaZ6~071G1XyW{wf!;-h8MXzTz3D z;{4S~` z!%e2Q{QL)cY`L{ZEAKx2z5T}(#f}zJXT3hFxiJC*$+lvD)8}awH>NqMjRFE|7Au-9 zdy?`J)32B09k#4DRh_*yT>5E9zMF1H7~v;rZ1_MVYdu)S=jtWP_xQB>zQ%eF#&HHrhHBWCp|totsj);%e85 zP^8`_dq&rF1RjArD7fJFK9k+JvcS1`1*M|y%^Uhl<5%qJj|nTbp4U}HIpnyA)+CU! zKsjTGENw9S8xWpwDR*q_Z>AtVnT?HAM=t_LzvrQ}B^5z?$I(S5-#^yIpG_39=~WZ7 zjNT%hlrWj|YxtE;xA-2fQ~M(KcFTd=q~EIT)bhvA#_ufobcX%~>s8%`uJ}tXB@t%k z;aT{dHb;|U1NIu*`P&&^=NV5jf7MSdo~-QVoobA))}!|tMdd=Cmra_6?@W~byc2MC zQrj!#A2{WGfLON@@byfmF8(lwWA-l2(DqCqijg+8t+vrAaO}IYVY(yLawpR4q~=uP z#LZG^Fk(%`G53{&bUsd6=q$A;+r6Qus0-Wno^NNL?yT4PPUwDZ4ziP18)?3lS{ENo zh|WY)^TG+pPLs8pyH#uWp5G4L+>o;^8S_PHN7}{2*U`&`7|lU=?L!tU@KT_NJ1Rvm zk)pot({KRSr76aJV-b3hOy!dx8 zPsRri&z9=*NS!UQK&H9erJ{bP9jQ|ntooGxkGx*o8r`TB26uycr6ApE^?m02^Yx5n zuctM1{MU=~)@o&5=`FSCJ+=7orW{nG>V>h`J2qpgbKC zO*Qh16|#+=v^PUcL#NPps_{TyrWO(OiYmUq?mG(~_j6#ztG1)>*voY5jNgK{KC&5F zOdi#9ZB+CA@~NF{4@ESswhJTJx|463M$w84O$w}0TMmD5q-!(_=>I#erA3B~9;ZLE zqCdfk5j7ysKJR}UFuQ%EZFEuR&b7mX4x480Y}(G`^_TEd604r1A~Dlb`OBQgf3!*W zy>P<%s%?%3;GISy@S=;aNB?xEm(d#-|329-=GohU(T!I%wjQ10#>v0?hkKObp6*=| z*lVYqB%3nQ-IX*tar8F!Fx^W|E*;bS*ixZZs$QFzeyEz5c3QO;#Aja)@|PFnEWYA6 z2LnEnBHgld#ARL*q}J72&z17LU9q%^ZndQQ3`0PUA*j-i%YHm#kIuMb5^>s1(Qr6WL&m@cC+u z@I{_sQ}-HwK3Ehvs`HsVnHU$?0DqiF&qP`RJm_0(Uv19jNcMo`y@0 zaJQ%#9O~q}a|^y4ui5HfS1DX@aufA5gz`Q?-Z>-_5!uS^pU?EGh*~)RkH|RZde0h* ziNnEMW9Q~K0n?u6K268fb4~V%d(A^7x*9u6gM4QO!+tw0W!qabqVXBy2g}|^L)z7S z%R^fxEk7H1M4p(rtHTU0WDVCfJ-JGO`5^o;$a>u9UUErJ+W2S`>VSl0#b<=7ekEEVk#-7XBt=^`O-&i& z%Mc1sD;!*H=EVBJV)R|dm`l`FBcq#)H;=LFmR$Yr#C(&iQp#@8;^fbRei^>I zl%|Xf{|9399)I{RhLI|t`r3C;UTaJ9VU&?|Bt4io`X1i0tv%`O24@-AH+?Q43hKIO zFUvO|_stCqCFcz>Tdp*$yoI8Y^}Heg#efSiLqNKjoG353l5AX=WE}Y1Srz<0QyGX2 zn_XYqD7ipGwkXu*Rk1-=Y=Q}T)%cpZNIQJd@3No^^tmvYPF8D>E6CVp_G;cgNvi3O z_p8Fi{J`&9Jt|~46|KoA^<4bavbu~;E8@pze9!7Uy4x;v_-sQ+=Sk6$j`oGN>02-4 z2#UA_PbV@A3bW&_7ryfYEf=5MZg~w zGrbkPmtEV^B+nyek~PNEuCnfz7gs<1%l_y~|Bq_tJU4DKk)7eU5@spOJPG<6!El9i zhHP~WpRVy<7>w!)yv9tEg;9<`a(CwYylsh20*L`GXplB6szI^Gpt#s#_k;1T&!;&F z%c4IJ+1^F>C%fJ<7x|i9MHWj5{?TC3oyqd_!FE%l6)7 z*wg@kMw(Oa+F)~{<=u>I{#Ah;lUm7BMt|_q4ggO` zxc5mmBG!0Bjd~jdr_3v>o9v~}7KrJGckZtr=t!28`S+#INHHEB>+RfLBw0U*b39X7 z_Q0OG=DZE)9pqn*7s$q*tkEqU62}a=d>bC_o7VWj-joiN38vr0WFiFroUFV4Je`{* zEDGDm;Mv^?x`LqM#X(hm{=XaY*(5QByjaA|$tc)(AxG9j@x-4iovkLumL; z_%!b^r@3|*hkNx+`_CcL{8m_}woYOu9zu8%0MVEIWM;R%F4av>nq-<)mKYYDspomi zm>SjSskqi@iwdOAx5f>aZ@Fj~p3WAY-B~8QT|6nd@Wj5{Z#?z$BRrbFoB+?ghy0I( zQBEb^1(8m&v3-aVMU1wb-j(Io-@O3fUl@4eV-Y~Q%KWwcSoZ@`UjJWv=mNSBko-29 zL1K)aW9xa2`q(9+`#`-i@MP1>UpNA1`cOcIwWfb780vo6J9_rTtdzKoi4EWg-;4^N3;I}ZF7BjBS z3JcI?a9SweGmeK5?~ZLxpBWTt_#@4yCPF2igr^7}5c8Zq@aLubI6}_hHEkFlt}it2 zMSZ+lt#ImSC}8&>g(+y7rupF;-v77&bo1b{VxBagnJ-f|Rcc;SU*u`?7!PGTT&Vt- z;YBZ=rD%WUxHg47k!HC4fOExa_(9h=QA&;0ulGD&ZDe%e)lFm*cXg+qD|U#QcUE?z zqWfP#av=I&Ib0D}#PEHbDf4=ALrEqOyDdOkew_t0v(fO2BQD<)l1c>@-Kb@`1dfy; zy24|qFKBsL+2a+r+0&twUB)B)Y~2Ta-Cy7FV`~wtdU_q)tlL#7OYf$v4*{(bq;Dv^ zhM;HW!8=88`Vkjd?P~o&0@7Ow!~u%VpR6ykJke2rUzXjN=eL#%nG|cBXy|v;CM6;* z^74vK%%J(D&57Mz|F--6dz1TSy2jyQz|y2eAjkdu)a>$xlfGsf8T*m%n-c0{w>zcY z)ta2X`$_DNH5-boGr5Xad&h;Wo?ux@o;3iz>He^fs_0I)Id-KOU@#~MG%BCzY`hWQSY65g|a$$3^S z%+cx&QhKXm<=Q$4RiTJ3SB>f6ZBX>`xseqWJAM4vsYM_hL^tw~rpf1y0fh^^+5NC% z(sgA~a+&c=Z}YF*$ln3+nFu>@kQ0ia)`t2UOh_K|2>`V;hHVj>hj<~=79x}qpm*;~ zIr3dwZh^%b)Qd)Rd3gQ0Q1Af6SJ*lUK_T@AP2Aubd5jNQo=Ktgl&f0FeRcW)qIl$} zgR<~{+B&xgmk8yF-U5tGS3Pm%wjoXoQ!LtWHW*;y&*OgAFgXbBFA`z|gP(45@I}O~ z*RBqjS}iR*KN`rCld85GylY!(&+|-s^8JZp0)S`%#ww--5OwXbXMY%1<=TF#RUeGvNK_1t9aL=~ZM(0;~p+q>NqWHoG z`$l%EKfU^yeadHf2{J5Um1^GxC3r6M=SEd*142?0Dw-O+XJm zJFgcRcpuQtt`pLt4gp_x#HvTZ2%hCadpnvnyAR|xW_zDW!4Op z^+5SecCh=5c3J9so)bJFHhF`H*AH%a;*z3SY&Crmg6{5pZgYbmxXz`lJ{rG)@#!?Ml`YNtNCLT=7r@<=_`HIYEB;N1hGb5Qa#;46VMuEYCOYnk7}LQjidxoFj8ZD+GLVcGUc=j2Uce}w{{`q)r#WOv&QPNc zxK#8W9!wok%wheKO5iL_|J?*a+3^LXKWks!wm4$s1*+_@FG8cKUI!vxuz}XDH zIbx|=v02nM(?%Y;N~t0SEoh2Zid^CzK*8%RW~2*sTkiiyywIz1D!(t|6tOf^Qgqrk z`F6+OdMcJlKG`axoE2$}c7x%C?3U(HiPvnjSzx4Jx~?x?2Zgd+t8@VNLJ**8?wSHw zOOpg!kV09=HSk;JRU@L8V05a71IBO6^uvU^J!do6+_ zOMS@P|Cx}S*VXKdypNP&Bs^wa-P;gnxOTQv5cpz@36kyWSG|o7;{sCORhCZPUv_&& z^&hXHnMUlMnCV8dL22dt()!h(#e3>oVPFvC&QT;iTJ@qG1)NFPS?U;fUdf1s1KREd zwERV37j;2Bcw0IE%4!IL`8s5w0RO*~@kC~)_#A28Z>>H#7homqX-f<|^}hn>;cO$c zzyHa7g^>5ohqN5~LK22!{zZJ$`iB`j59z0lhW&hCwZHt_TbY_}Jxh{UA9yI$yLRQj z#WFNnM=Hl875|?E7gZ#4Pi5Z-MZ_CfT~tl?k~!K$e`BPbcvtUA!4!@LFv$u7wPnQB zamo&X9v_gfc6jI~6hz!oH-io(FJ40X=f zUSaq?uKf;2wU<0DK>QI}LX&`5$xFUYskOWc(+y>sm4EKpPJ~}#12`=LIFS%ojmhA5 z{=k0r{uoZ<{h2|^-Ka;vZ`Bj*x_%QJ(X|z*yU-rtdQ%YMxn@EZ)+mJ3W9e7N@DYva z;tpDwG|ljMXr7@%7|>fI6-@q2S*>~V8;~;b61cyq3iomCT_}$iufn3G?3$De-O*GK zxpS7&`qReVj+2ZbP7m;g=yz>$hnB@2G#&2J+yR?)p@8(7!|x<~ivF-km!xeBaI|vM z+-o08R@ni^_9Xk0D6lXq^dSB>0lS~k_VNA2o9L5@ZRhm9&RIxPmIziXg}^KeH?!W+ z1R;_EeLfsw_5v!WF2Q*zuHCSi<{ug_ON^G&}pm21rn04 z+=m_6oSE*956I@?%lO7}Bz}sFXOee#WNjV-N>E%;Bv3!}@kKrMkZcW!RHVGBP7s{A zZR9#0@Ellcz?@8$?5F1UHDe|~+<-_T=*RH?6u1wBg%q|E?MN~wGAKW~4}#@SVZ-g} zSQxP?E!Uzb>Wu&Q3vp3EBOW0XGc27or70U9`{U0wE4d9liQr-}GT++_FT`x}z>&R) z-}-~k2;+WL62w^cgW10g`#)%e%3RDIYP*W9hGFJA2}K1Kc~OJ8%lCOZV?X)?G0BrX zTLH`xb3%8wZvrE?NKfZS96wxyQ_1RG-wm{b;`o4kYawGQ``3i%s$DFCSx@lgdC=Aj zg$|Pn{su8(Jg>t&UC69Q!%hp!r5sUvsEhJJRm}SvvMq+na-m;2=O#1vg4FN;iS4L8 zVW;Nx#ahl}TEJ%Uk78dZ2m*y5Nzn`g0CWLFRvs&`nt@+Ym~jI9&OYViT|jwn)cNn6 zn9MQ^^#}=$Y*y{5Wv>NM0#6&*S^?2Z$mC?bHF1A2Yw@nPnBa=4nf01M@z=K%NItR* zm_p#F{^OlCwptB7)C^{sB$?00*uLBch6tKh@kXE*e_>V~ zWz2SxFp+huA_<{It%jBP#P>!}2iJ-5k3JAzlmndEVP3!<}cs(&tUTzxrO=XbYsC0%LkQvy(1{zu8jAV z)Wt=;prr{$#RY~^*1WFDW%F?e=q65?bhB$CIC|=pvIzIa)x2woM)Mb*XHwwWs-$@8<1AmHyLpb7j&JdV@2!W8VX~QEFZfor?8JMIB zlqDe$B_TjAHEdm$2Z^(@qk{cJIv-)4UGLH?CIz`JJVo7b!i$+J6JVpa5nK_mky_8_G-wlFl z)v=OE?!V6Sgwi)8m( z6GF;s#5d&L`6Ye8K56>(sT`u}r4+o&hRpXCWWGdT3X2q4T@p>c?*^sKx!?%Lz^U=n z`y*w!W<#8yNzG{vbZScoDUM_1OHE6DH}Hx4tvs?ZCjmLyZ{1e4d?p?98H>E$gN(&`1ww+Nkx8inIfkM|efg6!(HZWM(=- z=|WcE%ym$QAUB}hn5YJ|zQ*v!KCzV`9Op$@++r~0G;fW>LsMQ9PJA*OqRl=ku*O5k zVBWGfZ1Qd1X=Tk0!9nnmjW_P#8<6>Mb$rF!LdCzp#k*w z9B4xB@J}j5o=IkXE;Xs8CR3mw=gz|mz#Iz*d3oJ14Q=h*tp==tnoIh#$H2QlV{`?n zyNaY^U5YYHUlX2SpHkV?5rT{XiZ;bM_hj4nv16gyG9O~~UzmhV%NOJV}}QVSazw>^^+pVQT{C?FTp#hXP8;iMUc&klzPY zh0GbCp%?FWlRz5!-#oTlWlZXnG@DoTQPZpeP4St3^HPT!aSO)G}Y^G#S6`@?#@Re7>)PiO=(WgfSjSi_d>8 zlq8PW2QUM(o(O?$O#Map`X?{5Z7F}SoS$M=+t+|`=0igv>j;9!JlIzV#+4h^m5~p8 zxvi6Y^hVMlJyS$wM&(!-!Q0O4f-e99VFVxnUw}a)Sf1*ilx9@HynQm4H7jcC7tZv zjZFy9s#QVq6ry!kDEMxeY?JIpYl6uLlZ755K|Ap}r^S~@(B7xyz{Uf182tz4hzs)V z9B*fV(n5|B7+9P*_fhs2|5>ca(v<4Y3pAI(vBs5U5_nEG|$b z)pik=p=jx9+rFEE#YP2cvs7P%6E{$3xD8@T+OxHtvLJq0v7-U9qUxL&)1#t3^^ys8AgVw(5L(t3CMW^1*LylWch&4nBW!td>jw{2U}^e ze+wy_gfXF22J-~)Pmsqs{=;i8<-q-QZKA5-w%F@!x3LS?pa3Xu7JyPgNeBwkHZ1=p zp;p^xr^tSS%%#FUy$1S~Z%bL-Jz96x{B)L@!N@x|gxOm{^(}@OBo;JVU$`C%^m8Mn zu@~W&@L7}RxEb_-Hp zlFU+|I&+^g0L1sdFfxo}bV39~N!YgXFx=MbWOT9!Cc9?>o-=7UWV^2iC23}mA zK=cv;@>q?(uO8I+S98xL^+^vLPb8-RyK(I;T#EYGx7-hpZrOza6(*s^Ht6t1n#Js* z%cTVt`B9D>zQ?2j0Wirrs9VUK0r= z%rS}!aXuOb?n=*mH(`1R*Okv=1MIcn=zlY@(6x~pAEqZKb~ROtex0T z=hBNoCvEo?m~dc0EK*XgAbCfAF#tOp(z1(?<%dPFt#Vlt@74a)p&>r#0@;qI*&unB z%WaY)$eh`c;uC&ULxX8PkIfDUeeOdT0#Y>=SR}zcOIWzetDFFJ34L4>$#q5rD2@cd z&RnPyYTaax@8U!->0u-wQAuBGsz^U-{p$@TEi`hF5y9Zw;@y;f(D1Vj$mBFDDGZsA z56!+sZlGIIGXj!gjH@*f3?uc6P)^Lm4=W-_Fdb=k^VR}^k%u2Y3vd>}(q^Jz_aP*8Qoz_Nz5-`hVXF}NCIlzrdxKUxLes$-yaLFIzQ5gCe zh|3Xw8vA)5NfZOYVISLNJVf8;_r(_FOtSMO&VtKK8qgbfE%SNXQ9zMZBU(>Et2U4> zW>Rk}A4H?i>`x*|@xckmGKGNeBlYYIgz*gV*CkO2S4$Jx(>HBwMO zxhHmuVEA8|=q^4;Cp=vQ19JxkcV=*5bn;qfvscw zo@QD2K2_b<;>&|oS&2fFH|eqN?9 zd)g2)xg)9da5&!`;}3ZNB`WMIT$6F`HOZYx%zZu=Msef61Z2;dLk8KSoS8!?IwYm6 zL9u#z7jjU5ZyRo&CgKj&!p)Co=`3J z8-76T*%@Cxu8Pw=kh!A*nd@T#Crpmd1lP*P5_$_679um4&dCfFO6OAB*lmnM5D_{D zx=~~-@)iT|5=B7VRlUC4os+#MJ#zxJ{k#fo-1H;HtpHbeTA=l~A1w9>`VxTBWKr~! zvMq_*#W_}Nes6xa35f!AxqzMk;rF*gXL=}wo)QqzK{eYEVK}sUs8PagXudi@gbJjbXqX#56f9><)xE*Y1aF9-B>I=7>ba}&f+DZVU9xy;^a=YVjB8~B zJT5WV1qppG3zniv%WZuR74SjpP=kct78CAP0Xk6O=^71ZXZN59W5ar|ksO~>8Yyt_ zDnRXA{@j{l)fZs(G;5pbwv?MCXVWLtFmy3j>*~1%0Um~Cm0*b5PfV-j5TKXJoId*y z<^p~!11QL5!PoLlW;6DJGd^o-R04P2dC=lD%#6|M{Q#h<$lTFP0sV`5B6Zz1#Baax-u1}BM=9zz; z&JZ1j&ZXWZi40mIE>C+X0VNLqTN43PkmoCIo{#MwA<_B5Z1E6|tG{b%%G_(o_{Ly0 z(LdWT&Cj)wL@Esp+hpitAb>A1tSS(Guf_*TjRB$7gG%JH42nD0es{~tzO0}ud_(Rx zkZ^O9%*Mg)>BXLbGUe$m4JRaT0w7Bz@MaW!MDb6xSB4kbg5duQ!wKerk{DdANX|J6 z9ZxHH#WfR-xdb{OK=2*HP_dwj2I_d_u1jOsGmkAx9m!{yn<1t}`R;-; zCLeQZp%S`;Q0EeF6V`U-nU+kx2p{n|1sa;?YxQAKgZZe_{sfg)E)6s=0-_J$v0%Fq zO2_*Rl@y*#h~W27r1T3D>O~4zC9?-Pm zH+)Mx->G)1JWXOA$A4S^Vt3Y0a+qXE0TVtZwRys5J}{j-EO;w z$!5zgnAz?%I)4`3J#uq;LFjWThJhZ{ML0wT_HNt?qq0S#lOM9F2dqXEC|aN;XIO83K=)n{?XK-I`CcG^Y*83P@p_QfeU0N zd(_WUl`qDbO*wxZ_5#%F`Sujbtr!T7k?mT0a;*(s;vC`77&QTfOG?fE|8#a69idzi zG$D0QzR@JORYIG9jMrB)@8wA|MwkQSx%2!`Fq| z&R1dX>WL?9qw*Nz{C~B;!2KChjEPe?n6{6rmb~sW4{7bGlLuSl+h5Kjm(YqpZQ(Le zRPvr5GOm)xxsQ6OxGmQwiau*;UK%cAJT+TEgZsW)0e@J;zkS$yZZOMx+1{{6O=4{z z8_pWy834YHj$gZ%qIFsPi;+RwoA>G!DO%FsJb6{cn6sfPI}&cp=t^VDb|wV*st!%o`lK%Qu0qt~IXd7F zD7mFlMmx&7*{>G2w;2o~ef5{$gs|NSaK9nob8wKS6aDmg1 z3*_ti>vC1Rd$Pb*{nm)U!+$b3Cb{!O>;RR_W-UW~+%^6E_Cq(AKis?&!Q4m+a28e2 zp>IN@(+jGWk<$LrmKR^2sQkFv4cHm6)w0uc4x8MpuFK8gWYX__hK_Q*6hMKM?pGcE zgc;ulR1BAjC=QyExrtvramPzKf1zIr}V3tcM%u4mFl8A46bn-N!yTFIG5h>LX?oSoYGzL0=w5U8@# zWw-Bdb!|3d9=R$^+g5%3tfLqS{6#2o|M-= kv}&?Qa8ZP1>Y`sbAwk{og&pYB;} z<5#?le~ZRH@(;C9c8up2O4LchT)aU9suuVam=%3uRP;VgIml{6W=23ZS_#Zwhr$$=!V$~ju^1oNpA4Dz!*n`#cZ+eUDxwzL}U z@a^UuU96C^8Np+K8!=Dh?A%!pcGM3`H1WK@vph`x-}pF!nt!1Ai19ZxkXPb(L(auB>hTUJ=dQ8JnMu(n27@{DqgHAc(14S*;T2E@ET+lYJpp-wj=o zqc70aIzakg03;!=bJtFb0Q~}Epe`Yc%VbH7c?6xh^%c)g+6tbJaHZA+TU(_;y8ygtK!99zs20yB^>rDJ8tHwjr74L>>e$Pu|v{ zF%LFdFw3P?evA(8?r$qArBim-PAW^HAD;HrxR4`Y*93TZ6UH3*UciZB9R>LH!A# zXiknK|F5RLF1ofyLOowVof-V*wzO-A73gcvWwS zrj)P5w{-WzQ99Dj4!kOIc@+O&Q{pjzJ5qMX&ORPtHySd7Bk><;M>FtX+68c zL2mpTc^(q=;VU%Y5*7!uBM;bYut23a**{Xqf3^5KEmtQY6r*}Fd%**48UhJD3Xj-~#rVD}Mo;=zJPoD-P{>`wPvA-DN74{sIexi^mna!_%w z)Lg;v>tYV6MC1&`VVCHZB87Unx=}AZ2gqfPHM5nY9#&fRxPGEgx$?JLB;zHMe2;P) zA5>v>x!Cgb(38> zBbU*?5jpG;g0`P|OC~($3^bl;1;fGH1BU=3C5Pm4aqIn^62n>t5C<}7MY=Sm_AC3O zf4sTzXyBa^!oIn#MEs5M(PCc4K)HpA6la0g)*lgXE3YpY2ltbm;pO_}(yc$Oe^}H2Z|K7)$QFCmF@z+= zEWaxlL{2QM2NAat?{9o0(%Ato;Q|=}k{%sE)J|+jSpRg8>=;~ZCHF6KFvaQtPxvBj zOz+YPbYtJ|ypnLG^SI1aEQ_yuwXJmK4%gz@MW1>_#>p-&vbZYgs~nV4eWd&`Fl!Q) zyQkI_hpFFHntE4rd&k{-^#xzb5g_od`);%)OH>~d88*l^|F|z{{VPgrsL^ZdNmCep$1Y5${=nh6*6 z^@)$+@9zwnncIP}h0>22DUV+gDIRG_!hXKs#Deb z)1{6CN>4tTkKBh`=?Sh8IaZUy?>qu8vw-Pjz~Hho;s3u!#2+A7jt`g1#-3$IaZkB- zUVk20+I(nJQib6&Zq7LY!xl@Um3g(U!PO$eMvp8yqqR>CcWieO*dwqj7q3*e^Wl26 zKo;praX${jB>#$6o@lgm=ka3kSN%IL1$|NhgbJ1x1Jn8Y*`aic(SM^m!J(>)*ncOO z(CZFo_<0zRWugMM`1t0%7wf;>FuidDBNy^bpv91ee!qh`|G(2;IAc&Ed$CM}eWA>jD$%Q=n}5ks|~BNj2*F$_}N z{Dp>uaJnCR(}r%7n@>55X+QbxP3Bz^o521|T5sA*cW60r_uFVAo}#?^XE0Ztdh~SM zuJT^ISpM(iy!=sYqGeTr1@Nl;5ogJWh9xz0+T(qjmZ1= zYn5%Bx`6h!MTXz*_os=*I#0#h<*BGmJk6C_PU@s*0`VI&bg$-PV-;HbPlU}{FZk`H zX=4N&Q|xQr^4I;QbR3N~1GYT3-MJY~LBhf-P+(0KWA{UD3x*e*kw_uR^=gg{XMJwe zg0M5!S$uFm&_@kpeln;*zM-BJTmf+eK$3xi`p&;cqxwcnTyl$Q5`?*>C*S8m!d(Hx z(+VEiB6SH9+{bdp2hK!hjG}rae|sh|@-M@Tsukq=FY**?F6y~InBs3Xti14P zrY+d*`0s8@VV6CypE!G4zLA#h)fJXzLb77QV7QP~??5^bYB%B-sCWy5?}nV?bdT6{ zBTESKPAAsu0 z)KhIIzx#w?#6+lUP`B0;uWKBbZE<#tbp4qB_lpP+vr+5(r$wjfN>_ue%TumNK;`qX zpWv#RDq+&zoc=3;5@h3O_7&nhWl-FKX^s&#ZSAYf;$Gh_vMJt(Fw@3A++O_Ub^>f@ zdu^oey|K?~b^UA@UE@8K=3(L25fFKa$j3#WHa+S@ukIkNq6Pi%0^wro|~4XZ|0b3L>=&0}h`;lgyq)BlIG z?+&N>{r^9XgJU0?h(oqQ$jUfoh)`MCBO{bi_Bi%2;$%x^RI-W4mYt$x@0AJ}+4FZF zy+5D#`~AIsfBe4J@48Mdb>!Ty`+kncb3E?n^~Y@TQ@;20IE%>7iM`6bi4sOxiql(_ zWuv!eC*1pr;X0w@9DDOs7NL69sh6$8GyOKzOMODFkSZDSwvN4V7^o*94A>5jVD;%D zH9gwsWHZ}qKCt}wZ9+fIr*f zRX+*z+mTL)7WqEDm(OxvW(AC04L2ulF{@mJ->+L0e;`Uqxt9Yw3^+O3pCWh|IEuDF z%(l^JZzU@M8zyUWOeRh$G}0GxT)kB7*OF{}C-%-cIlmg|M?V*2&1+n8-X4WybiTe& zfAF*LH2_`h?59V+wM*IPUzIh75I~F01&6`pj<;?J`C0f=3VK?0R-nSFOoHLnPGOHf zKk$$#9sR~7z^r6RsQ0p_g(d!9c%+KzJ1%0e$e4CAy)5t593NAqYYU2% zK!N_cRrcEb{y_CRGrtlsCQIMJed)N{SL+n5K8TK3je6v_@^kjTW+*fEDdtcNR39DW zyheiu<0sC$jIn@R(NlqQ5a1K*l5p%XtQME*U3#fAX)IgQ+u9t2U-dz-Zb24fF-Wp< z|3ryW_O|{avXbCExR9+CIHUS}wjonb)6zMGGJ-fwz1$>Aw8YdWgYf%Tqls$<3|{DL)S#US_=D$@JoQh-l$gGK4qt%fQmZ)Gi_| zR?9nCZ@s-g`3BfG0}?eGkJjX>vU}uHzWc4Dk3LB)d0B6zEbYyNP`)_pf&#aUMy)QHtu)%QUZ{yX z@9sKyc>AFCyr}in<>@c8=*@MMUQQ4J>5BHR;Obhzk$ba4Gw)whe!ZR6Y?h_hKjrFz z+j6jTq$5Jk)hofW)x`BSEj1SFGc+@2ZvYVH33R4*yxcATR&+Ecqb{#TlKifx@3RyM z&Jv=;p&q33X(b|xyG|ZzKg84P?!oWIS0`^7X`E*ihgMNz0DSm`JlXHey0YKdeSrHj zUE1Vuazt=vM)q6XQkpq~m*>p5_=&-|2Oz6Im46Vxd-ZSnd4CE~#MoN|^Y~2yiD|z| zPkUplX2B`D0VfBD@%`qffkXjv0vx^1qy`GPEj}h2lwCa$O(&qhgiZ=)~y_UDG_?G#Vo;B`eLhvIzH%8lMDK2wQ>9(sPI7#i$M&E>;k7u{JJye{)UOqiDvdbOE0>{y zw|9Y$+emIIyDO1AC`77bh;1E+Bp1t;5Z{>(>%-&6H~mBpUnI>3*n@V^3>^t}feD zs(Zvw+ITG0EiKrH9`-aeNq7hOPfw6ysBh7*)#u{7v@7_zOx<(A&5${pyx|oE0fU7( zKDe54t+SvDA+b>2g?l{Jf958-2n6-So8GC=z{hqn_0>LlIL+ilh<*{TcMe54O&t9M zi#nnfPDjl4^}kkd*Nfcf$167X*T_h*u_{M~YNyfM3Q~+ZQ~1LlUsQgw zQB&u((W!>PIl{3c&(gU05kVm~Pp~(st#t3YVRv8a6~6FGE~1R4zogh1d;YTLFB4h; zeX`{tslMCw4g=g@3mHmWO{(rC@jJ|t=QHiBPEnjf0%3bZp@EcpYrg0v`6GNNhe<*a z4b>QI@BQ?&cRIvddwY0Hd85*wb^P!k%Q--Wnp~Nk1?$EZnZW6>KlQL`wpDg^*TAJ% z{m7|X&)(_U+1M3C7Hr`+xt14ldf+EFL;H_A=@5kzc6WKX^YbR0eRmPX*X8k?h;)YP?|%_Jo4?}?jeyV+_EVv}KG{FbMCm~`4S%RW;+2dW2f#END+MHb zC_Fy?=zGv}Nw*pJrwl1O1{Sqy{B9pa5|{c@m{C^Wy~Cc|MxGn-FmNywr}k{B#;M2ai=h_~RS?RKWCN+{9_)*XsuqOu3Sq?{AMrWl&gd4vCR%af36 zu3Iy@M*&-&FQ?>E_G{G?LyClKROq?nx`MZuI71vKR_FGkU)|iCWF-1Rig?puOU=>M z;=eK1?)u{k*J5U(VE@g7Yjn>r#LBmk+}cYYZXH%uFGjY{-_}MYDQFI{w?3uP+}=H%4WAj)99b z+L*YAbwk;W$lCHADs{(mt+s~B`5*9p4tf6Lx-q6pu|n^=3gMj>q@4lr-3lVo73}R3yn!)TPqJ@;TEP;ABx00mPkHBdM1yy1aXK=#J-0=PqsZ^kP0h_Dc@6QYTTXJJd6 zc;@Ug_t{&+PEDIXD|Ncet-G?wyRmdqW;GY5zvYU-!nMCOZ3ycmhJf?@NjumJE99<&7#ftE+^(1$&DHewV$!1fHH8 z^IM0evS5b+MsF~yw0%r1Y45SvLzmQQE)b0X|MehAz~Bb)g@rOR5mFc-p{mPE-RG*H zF*21~8R4n`j0Vw#oGs(^Tv*~Jud{l@D|lzdC<9WijKexGehFe`F<^RxGv zfZIajD>TgA6f{ond8nprqfRzFC)4rs*ZxwFnN@H!>pOH?0jG}mSna!ku>PLI_m(i+ z+cSf(yxXyfy-zwl416DP{&F|F(aSmg#%A4t+#>2Q{F@r5g%;G6j^o+!?~B#W6S40P zDd+cQ=doGa$(-uc3j&GS{&ak1j4B$m?boHuG=)b#-&MlgKxa!iiL(c@2!6foN*A^g1Q+q1L z_bF4hunn)y16=npNJ=GldgR`ldRggw51Wh^0toW0)#OXY;om-{7Y@5+3vm&NeSQz( zA*A+!;|pS{>id&XV^xby7j#Aujr-r2_YM!0MC5@DBr>h@sM<{fZZ9t}cqELJ;dZ^1 z+pDS9gMp`ypNNoh3?UNM^a?dZ&hrX#IQ6G~7Zp|`jOb3_DfCbZCsSA++FT;^+FQ@r zyxt>$X;;M{=b}`i=?faIHt+m45D{1VMp$khcuc4DR7|&&i@AM$_Wi0=EXXhxj;QD$ znK<%_=@SwkA@J0C68Vz)XR3x5`nZsbY@C(BF0=`Z3yq&s)r~0q3n;FBhXv?Xsn$q_ z=QEiY!%3)2=G*lD9H%Qc_Qq@b(Vl~Q)7v2$tEG?Ltq{3pV)6eV3GlTBHz51hY?rQ; zHp;E?6I7VGnW3C5To%kx1G+eYvY(2X0liKPwXx~f(6CnwC8A`_b{eg~enD7?1pb<+Ew_&N z-q%_wIZ^B0nfZ7Icrex7Uz3Br1f=wD;&|LInKzn5(&QbX+g{&w$2t*l^h-Ycj+GUc z>P-ppzH}jComT$MNPC>${$>FgwMc9X+jtE89IwQ~~77;%x`p z%d1D66dUTHgy(K;7z->ThpQh<-)FGz>QjlLRe1OCN15lE*;V{bP0C&jV&K8!-Fj{I z;N2oRSuUK*5xRgZDDL^$Mzn~>Io!T3gf5nzZBY=4j>AnE$DI@UE^f$XTE4h*c0B1e z5#$)}Xt%!hs_lPz0aD`~CiAV&^m71a!pn~LHgcsch1IHliP&DBtvS2VsIDp!P z2+tr44hAi@?K~b&+C!N+CSSg#n&@A6v8PdwP~|#p`)?uHHHx<~BH`BJunSPwM|M#X zsx}F!n&g*zL}ZK>@-7~sylE7wu`H2AMwQZwzK1)`+q8`~-2KJ?0)CyXW{HXZ5e9d& z7t-E(Fja5VU{LFJ7bB^lDZ;e%b5P)>I*bZmjT=(D70eRvILdhAnPd>q$}v$H`+(b} zX6JF06{~2WrlXtfyu-GAnZ-}lh{G~vea9xv8holuo!G5F&r{cxhRAGGC>e)+zlfCm z+I6WYa9Nfz!nVs7loJioSUKH8fG4|2s3;<1FX!kUX^65KsF1cFW|I$y0rFF@%c3=+ z&Q0<-{luTj2($LIOW`!rCKo{;5OC_hB@A++SK^AIxewO6ge)!+-c^NJ_XAJW5ncR8 zA#%Bpul2P2bQHvLDT)`_0>_=WxE2q!Ho@?>Klnjd9t`2@1+?H_IetD8aI}o8^u%z; zT|eCMShzvs$rUKZDL*JZ=H9&`H<2(gaR#YuQd%D2F`A@x&H9TR^k$~BB1Ksak8c3_&)l9{T<8q>$TxP51#AfD0ZcZ@<5?b zlRLTT5)bBp(|_%pDdW9z7nZwxF^yt@II#N;mO~{9t|esNSX4ob5mu<+zz7|LxOBQM zLd<$7yHZCh?F@*vd~(otc|i$3I`r4h)tRTOkJOl;ol*I z+LfzIg$%sZo(BQ^>kJ##{=Ow@&2%pX5haWyO5@^0Qy^PBf#MN1hLVFHbKf~Hqv+*2 zwubX+NafzviCQcVTW9QEjX{h z1sOfRw^Bt+sr(RWp3-$Tkrsy~nh$|)%=kNgb7YN%E40C#A)ySa`G1BKa?;{?Z=TjW zZ;H8f#OuVS+L-YLo1RB8KrU#`(M%%oN_l2@l_fp)B$GcHgBwNcPGub9gbXrG_?LsY zQCOb3hHg$~#kEfjpUaJ}C0@KGGF0Llf?ywNDSv(qu0~JLfM~gpu=hQimV?pQkc28` z!^#mS17(tU>ek#$l*$W2tszGgGB-vwIPMuyp1hG13I}6>&!Pi4+-%J(L)B6W#U2qg z?{G@U(aPYH{y+Yi9J-lxYy#n=v+|m9bbM+PiZxk}b7I3jE9WLu)q?)w$tI@SLU0-Lvwz{`#9C}Z0cU+a;%0{q%f?tvV=Dpn94Xm&HIiiJ8 zs2U6nM&dvY#CHl$C_P9KCS)v3Q*)hNafJc=#d_s$HEZr9Ryz>s z_R{&Byn$l9-#_l3zHSjx{I&Hy+fvIq?%2;>5FJeOI5LO$B7qKbvoS=R>y7BMth4tZ z4R~Y_VSJQ3>2ItPJf2QL8P1`a4Wyzi1LbNgCCJwu#;Kmbi>DYu$viW7bdw|bTMXEz zdZ8FhYKNi-!juPx1awGMo~x7apz)%_xa3Rbo1 z_Em_YKNe33j8gq$2>|Z}{7PBsr|4G}DxvEyw8+!@Ph0}dWJEfM9}%Ye8d2Sj^b?5U?mCM^33nhNsH%MQNv6sDl8@Uy{dtmK9AM11 z9;6C#(Q`Q4OeEGu5gP;N0KFBs(eKTSczm^7P}siU5M{FsgrUk3h&tg`p>|VCdy}B$ zcWe;v?J{)vC*01>hy-1@7MEijXXbU1@5Ctl(=p}HsFpnpu5aewaRa=M56e@1ilGpj z4e}FTwhzP`xi3&z7M3-6(EM5y_8-Z!18Z#)b-BS1iN;Wp;Pb*~8251p4MxESH=97# z@a>@%)qFBh@;m_qI+ObG3L2ug5!@y`*?qxf_^7hg$O8BUU{|Ge9HaujSrr-9pQD@R zk7J!Qywt%c3rxr`F&UFI3$yrr)omZW{xS#I;j4FQxWMpmGrw@$DxG0a7G)w1xvmvX zc(;C?7LT0dnsd+>^S>4d=E37eADb*&syv9r$NE@%<2`%B0qHms3~o+gR@7BU7NqRv z{W%T@1=I_Mlt+tln=rpqhT3w@q!L)Z(h6PgPpiaP-qz*Q2~^xp%WD~>W&KP2*=DBJ z_+?-?Jt_u9FK8Cred|R*1({OE?zQlxFP$S1EK2&jdMK$UQ?+XYy&L4#c|DL_EvwPN7mJbF)KMR7}g+|J|i#gwI@u#HV6c~G+syAsco<5Tsz>+l{c zh#AkZ#mItfrw!`w*EtH{e`D!_|66Xgj}f{@|& zUz(kal%V{E4~O=%HivqpC(%4Xds4gf2Cs)`g}PjL;R5ESM`C~Ps>BF8&Y2`J1p)b= zF@wev=ctcU@+t~rk`!Sf%_CrvGKdY7W~7Kd2f-24QcOOjC2hfQO2Kz_KCx$Wxc{g{ z32u_|BYmeIOwgN-nqR^i;4{H?FJ@l-d6$s*P1X4}ZM^<53-br}16uX@)#SRK9PHqf zdkMB>gg7Z9Mg(}!(TBlH;-e;R^oSLi*AhqUoNrHx4!o6!@J(D%hrT9LN?sCuuC@{U z>UJ^o!{xTQOyKzF`$xiXYmA}cEuvxVI@RMUEP6g!b@%E>$p;@K7t!C@AWm#H*fuqt zBRP5RJCMysf=|^-Azfrh0f_f%FQ;|urSX(LC2KTp#Ry86)mWB1WSs_006C{RA1Ach zH|nH~nIOVaK^(0+-aqc*tB-Gf3-KA1$+D$kXQWQJiJ3`SzVQ<0QIXUD7<%106Q6P7 zJP*YAL_%z9_r-Wi{R)<@6bvOr-6Rc#+J8(Vm<|psb}%#24P#I21Vf%m{$dD2&YnY) zK{$(3>xa@=T(~<|d!eB-erV8EbrPsO&p|NIW0$_!! zGXVEt7J|Ttr8>%fCkMJ(K&-}XNOkUe7}4nS^}Xs#AA$ffXpDF&E&T_N{8^nYA{!C( zR@D!Ipi&^1RQ6G5m4$TUORLKNOPS&4`48FG9Aqq;40p}MPm@f4c~w7j#n zjU)-e&owZ)N$Vrz4Rrmv${JC)7F3sN4-Z8&p1y{c%D`|NnA9-VLtIRT_QnNdnN`_5 z;B#MaAWs8_?4%GN&%`TJPhlaC#;;!6JfOU#qGX5LfYfAbaB1>UPhoK^>L70yX-f`giC^u2VT?EU#zLu z;XzW6bK|PXJPi}c9Lb8XxFQB3T-+Sw#RCvc{e`7aux%SkC^?}NHx7HquyX z($jYznJJY2)?hhRa3E~PItdZh3kAluO2YeAUu1!C%i^%uKK2$t>qG9~r_S9H_+X*d zEE!Y+2_Jjceal=shLXykl7wnPD{&qd$e>z*ShHK!xa)(5Hl*o-XJ?13BLiulhe>t2 zAs6Wst!1xX{VP>~6HWqfuof*U+A~Ql z78LRtviDK7)jC%P*rrn4r=cH!Piv7FdZLR3Kv#`48@+lxlxS?ZSGVn;Up*PW`ymdZ z{z16lE9L^QYIAVxBsq=CVu5j-|GM*tvB%ll%ZReglZ)DQn2l+bq1tIC2OH>S%gQ}S zm~*~{v=0gOpBmPgX$$~MMl{t+F@vY&gz`PZkSlww`-v5mKqY@6Ww2TQFxMb|k8Gx# zkcA{Eb*LfVy}X}B&aUd`bY{s|FIco6OW~s(Atm_O+ZkfiW&A;Dkf#tySoSL(b|CKR z?^x2$1UwGN>_+EP(I@D3V$RQZuV7aQ?<9w?e0Hg|pkaCDvL|@D_8hkx|9Q31;2x7$ z>}4dl69D&_K@z06cR5+`1b+2-L!QB0NWdKXh)mH0R50~-5{p;G--=}vcH(#%au6*E z52eI$Ki$TL`z>)->^ShE5$kced94DU+XBy_&786u8Xc)@$rAo!EkG7r3&_qbp*ZcU zODj0pS^vRh=K>_qOtJaip}PZQ9B&1p({X(Wb3Yz^mH0N^db%@;uT%I_M#xp>XhoPd z&9^=2{ z4Alx0jeT|)&5A`cAiB=0B!@tF8Y5J7Zz#ZsEX@!CrvJ^B06}p9ZVvh&GL$nr5ciqX zZd{TQxXn`Lw8%fMp8LX-irT!xcg68t{N}*TFVrncg8( zg{l&BLf1tgd^tMJ3SQW@Zi~~g3T8=L$-RN^ZBGA*ym5PCaUbVdeEt0(Dh>P^#K~{b;%%8h#67;5VzX;Ug#JMcpv6_H_nU;D;My< z*EZZ-s4)+qH~AHW1YP_xq)8$&p;%4Am(J7LKY9dABG-_TY<~+d7F`SXQpWUqI*f0g z;PshNlKvK0AyT_k#sIN_V9C(ezKJpD`zQf{xdy#?xQ2&bh6sk@1s?2&oe$Xg8t6GK zO(W6i>U7<6>BQ;BqTTupcxcRcl&nD9bNvVj-HJN_B2j6O-KA6}Z>8vOkQaC(MA^!I zS2aO!A8CHNL^xDB5vp_;JJ-|Rw6Rs-x1)^Dey8R5s{QKYc=TiPJ=@S^teV!L&-xE|j1 z7k=E=qVe&5^|_)M`b^(KdD9W22+kWMR90`t(ubVh7e%`B$zjgR?QnowFEG3~IIJ^! ze)OC}2{ka6dN)B~ctaH>w=_sf&hmjbFR~+H{4<7Ju)`Sk4D+{At2Mubx2fY!;e%7a zNl?ZSg8PbXzJ6+jShpyLQcsaux(v-tF0F_K^_%Rjxp`^}Sr}p}zn?pn!kjaFP)o1P z6=GI-Z_Hubueo3E(`z1`@|T?YGQC|hAJF!>L>7?nJ~J$K)!6PQm$vu(<9WBhoesU{ zeZ&gccl$d>sva@C5Zt56pSqZY5cC>-Hj&lf)bgzMkR3froJ6j`l8b2BGv(w6{V{Jd zijI1abZz^d-jm%y88{i4m>i=o&`4ff2)%c_z*l5gOTsK3&SUC0HG3h7ifn=nL}KXn z7o0P4&>s*<5;Cz3g-?tx_Ib{g2i0Hur@KJny`{z^A#&EG!phyH zuTPbutjPZmwh(!%c2J$-EbcxRbQrfb_vPb&WZ*pQ{Ub*q2}XLtrWVU#*6p=%s{c>#X-B3a(125RYFPc6;AqMV~hc5{ke4ZkxqbODu*XXm?8B^pso`N98SkCI=;^S z=l*3qq5bdlNX1?m439u?VK?GJP*`+|#b%-dwigbfMZ2_c#>g-UaI`8qPO8ujxQv~E z>tXKd;dKfl3S|r;LwyHY920CUMbPc@FMkMN$ps5T3?g&xsuLWtPjuN#?UcmrE`M-L zK}CD|z#m}}TPy1)8ve=DrA;G`P#v8!>Vd3}=2F()d~< zZ|qRTRCtw`w$BJk$tKg*Sn2yjEKMM5zt-Jjnl1XU^2d5 zFFg-B@fBg{HiaO~H`ycnQ$SPZHqqxlNqC>pgW9?UWFLK=$jd-XVB&^DdKO@!$BM?yhJt~+oWLtHMur}S2?zNH5yfCj!qML&eB z*;$ZnR0}<<6e{7jj?#wS#3gs~`cheqW5-l9H+}@?f2;U%;`Al#m=3ia`Y9ag1U!;Q z#G6;cW{(WFmv361AAJfOm2lAOD?eY!4baC)Obkbz2Z@Eb*M~N}4kleT*e~r}LRW1> zT)4%&zJZ4J|E9>!T$!0R@(z3OD%Ozz9b+``t!=sXHOCYAtk#VKJv#{J%sebphP3_H zK%vWia-vq-mC!J~-7A-&yLN0%kxXXw2%HZ8Y?_3Oy4!X}op)2c{#y)BV5Kk9mhrFoZ4iIcyp#Wuqi^pPE!9RJGKdPruqd=0Pd*7{Xj+NUe-eZuh zv!Y@8ew|yAggmoGl0N;Pzz)D-xH6w|S^nV^@)Q-wek{P7bX~3~?+gm>B1Rtha0yQA zDY3|{?ZW8%^EL5z?bd3EeiYOauZ+DTy%Zv+8pJ^$y!O_-ceNrAm3SO07HuV?(?4{ya#Deo;bML(qu-{2|0X|DRtvD=_0ivc}ypW#{Ui= z!1s_`T8=GBhsyIPhwf0FZ%KDT&jJxMb&Eqb_B*zr=Ub|{lQw#gEYH53tdJIVG4`Tn z-eRZMRh9*Lw{7_}P1m0@nN%cFHrB|ed*KBVt-oGfzO8r&QnlrD!j*_w&{R57ZcRkN zDuV$H2-ee${%TorvY@6Gr5Hw%g9?K38kDdc9Ung?SPdHT{58=6etNKLb^Pr$P}h_Z zj46lSsDfe8+~Qd+#mKBqDmjpY&`y zmU)TQe(Kq^LhXjXoLlgsX`ZeY>b6IgW+Z`2xm`@edHxY%;M6n-eT!Z3^sX~^E2Z-n z&OXfB%_}xigbb((Vqo&5h}F)mKj!&1zjN$}`pY6c%Y65ZZr5&D5+r|X6lYSHe)l(B z3%p%&BCe`+eo=#=+VujS`FPeOrejh`kFcy`N3VS{q|;UgtWH<75HUN~9*wI6fKvW* z@>hU1!YFZ{<>pI|E59FY`BAXR#ej|^_FFGBy;iG+O~SoTj=X94ZYozzSK3j6o~T7P z^UiQeR+;7(TIpXGzvN%PpmmN<=$BNwe4yM|wR7}RrmvvtROati{|kY;lb(z+US)Ui zjYvg7_M<~T1eo^vi$iy4?eDw@POV_D5$^gX4$-=eYz=;8_Q5xfEcWBK{~;syIS~L< zJTMxkJUVI(h?nAD3}pLp)5dN7AxqR*xRcfV(bu5-WUueuT$ylFhyOk8XUwN&ZT_kN z@7Q%wsiCwj=5`Z=eb(*2`j-kep1*>2p+=)iSNe$2pSlv2@E%tAlS_R0e9M_nSy*)_ zMiG>8#hgYEpu;(iM3tgT;PxwYFahb@?E47puEc|>7%+0npdstRHHu!)uu_OmL9AJA6ELV z5yh%P6=&+?rY$?#()L~rUfXj+X)CHDp;L|WJP7d%1DPQ2hEjlZ#ys4*Z43nF$rW5b zWs*QwmVA>e2Auia_=6@;`RC(+dF*fAeRFS6Bk4ZkL-HkN3KmIpGpf7!ye~+g9FWnL z^Dqa>rctq<82b8flTgWMja-<2`8ASQSqEwUv-ojCb73z%GZIx?Xc_J0<=55T zuq6HiPlA{NsqpT#B>F$S0L03B+R9DXYW!ow&fOPZg>okfx2eBgz5$Z*DCjTlvc&}} z;JH~X*O|Y*9BMQB#X6lZ?;5znX&URS2W^nv?fBw-{qwJAkG(u!o&w(7tp(-gVAaRxEaz2g$%8iL zJLemAr$h;xbE>G=TS1wa3W3O0b-8jO*v7TSbB$HRrV9<4$E3Cg?NvkTJVD{|^=-k< z0B-$&E-?A$zi>S3V19>h%d1!hUT9^13)~f!D!PzQ2gX1 zQV_CUh?Rgd-MT9p7V=~C zP!Zhc%N;&)KU^N5`#3J~RwN1*H)*`P1S4Xo#J?FeONuK*#?*Qry#Yv)(N2_@Y5@tl z^(OcsQqPjfh=6s86VC>1k-4DD3AM3Wy>R1&*H-zP3>76PP(BBy;9J|xV zVq0qbmI6=%S`Ezo!>l_`uCM53qwo?h8PkJ~&gxxo214ALspw0_)vvx#3rVoAsq;mGzRaAvACh;J9(6<5FK1l!yzwZdf+n8a z>Pwz_SBg-Ue=i^&SZ#g(JW}ial3$m-9yD9Qewe>|=P?`s=S_}gsdp!u8T%Dt~T zMx3CZ_hpBqx5&2`ciV!rtV!63dt`{5X79r9VP}cn+WEI1VlGtIw<~ zU*)ko4pk*asBf(wgN~v8B_33v2h)(=6uKFq!H8#=v>pN&z4;j z8WuFG$JlgpWEOVk)Uz~3M?$QAe95=}oU4+z&u>sd<{U)`AADC~;i#~m zs+SrnHD|lqq%D6;>wmCS-UM2bt~ZIBeeU7_=ABGU3co;pzQ@yGtjo}qibm#f7zXvh zX1&y4!sKE6zYS0`4;*asvC>%AOA)3t@)n+dKYXP*EE&mG2*EY!3_b)M;QRw{(GIcNYJeB^y#F2ZI1f`Q|^^*JfGydI93wKZ_jOsqvS8V}n`XayR` z+O7*V%GclZ=ual>`uyUNJYGj9MGWXffuIB9!y^fzre}-9v~)M!+KJKf@T|nBMMO$ za4gLk5|rFv30PPf4_$wM)8djt$hX;iRH4mHdW=wr@{4@*Yk~=|lW(Rmb2^3=4VIYJ zaZCz0%bm7b+t`G| z$~1$BM(?XZ75h_dpu<+!;g@n^F|S?qn^5ip@fEuXj02AXo&a^l(q%GLG5K~I$hApQLWDNvLB;>&@|wbKkMERSF2^a;P(h$(_l z<71V4_kZmzuDmN8e)f1-Qy94_V`&NDjBak=ZM?Q?cH95Wa^`x9^}nna&VV_(Oj z4*kY!%H|MCy7sGAMQfhr{8>#y8?pC$Ektr_*j7EX4p}p_5T2R(-qhMH(1SreRtfrq zn6~Stj_J{(akqu8q!5fIsyThfj%FsRJFE%M2CDNZnTh?y&c==$@a8({>X_{FW7lDy zdia#-(i(ePfeq6ww^DgaO2ug$B=hkU@NN|xOOuw3=fObY1Al&$OW%9il= zz+YWAor=@Ixa_T4CJ`uTMh|?Z?4}mJKOy~THv74!jo4NX5)xh9xP-4{i3g}zV^7U zOw`hNZz|tt+ytj&HP;ro<3+=BV}(vJwEK7G%;rn~+3`Ss=M-S?$oZ^`|SM)tS+b+~dbr>WNlr&63)U*N8JP$Bt);@@Br z)I`bg(h)h2kL)gbS8-gz%DD2Ex@N}W&I;uOp9u705TMjbDA^5h)QUfN_M+;9eIiS% zPV!-IAu-E@FM@682+nsazpX^x!{^b(#)W!`)jsZ}6A|=4HIIbWZk#Uwkdv~27NAxT z4A|GNAbD+QmZ>_co99qH5&jVZ!HAAHm80Gmw~i>i zcB4d{*>3o;8I3FY`KuXlPgrQn{{UiGG%@ksw&_@PeN*7b3tXDREvSS0;S<5LApDTH zNOp(j{&Z8gUEdKuz+k#xE~I_}?&7}scPu4Us+=WZ_|EyL_`4cb2OM0QxoM$jiIqNX z1-cC8NBJ;WwDZj`lTIffQ{bf1%cpMO|Iclr-+vHfLyNsgY6bRP z8MXl^(KT(#=Kz2s1xy#+QG&KSG9?fwb8J%*6YI{AJO{p2NzLh#CJh~?S)oErjgPqe zm55ZA@8D~_TXNTjB0zh14!D#d^{S^J4g@^{<5>4D>?`-#KPUEJm~8UE z<9AM1y*%xMk*o%^n=m>6!YS`hRJng0mxH(rs1%ta!xZ}}Cc<9pGmIg%z%iP6FVYPP z8kCr5z>xH}RR4-JbYDAbDux2e`UvHhmzrt8piS@E=5n0;Nm&Z2KNOC084*8uv@>t7 zs`ToB#$zr+gt@3~FXow)`|Dl6Jp)8TVnAfRy&4h{vT8(_8+^OcRtq$8Ub1cmm!CWV z$XV^9ipd7J5#<~`huHlIMsN_1#WE9_u_J8@{+6KQu`QR{O~!k(Eps-rzQ*OUlVv>;c>jDW+Q|Q&*1dZjYCz~+^}Z*RBD`MO z(3waF5>G!qrMU7_Z#SvTkAniYy^t*^gdFk)A7S=vQdt%3Z@VzqUy5H~qHQclSrj1i z9tI_Uk$?S4SreHHz3eS*p817gfRyS}#?=TM8=X;ctk|$z(KH$$uk97E)kR}F-#;t) zOO5>;EZhLkTlnEA!gA?T#w#<~qX+0jUaS+T4^5CFz_o*12BC1r;gYHpBS!gv`W~?v zdyoof+yA`;!ag*VfCRPn9%NN?a}>kW{6Qd4IKuQ1FkP;95L`>Rh=^}TX=+F$UNrh5 zsQG>1WgNAv~T$tnj3?0DYdY8}N zMgFJFZo7cjbi(djjn8Y!{s1BJBI;;MMdqNj0Jr6gPRL$@>xOfN+0M8v$ zHxq1OuVSE93JMC?A`+jvz|+Q_e&od%5-OW#;I@d-N@@G~ zpOsoTsLDUk0x}bi@rfvq;0A-JI5^i9cexrN>@pRm9x$MzUb@hw3tI*sR}E=b@YVxt z*5k=H&%P+rmP*AAy;FJu+H_0JwkzJ2;&ks;2h0uLGCk}AobHuz%QRQ)o21`f zS(GV^Af8=u?A~Q{BO)qvo@UxjVsbOz)hpYXiGkQ&l2%lthpXB1~JQuUU(Gs_8P}2M*FcL9mzZ>$E^E9wKK!4 ze#mXNEFXhV?q=#`t6zP#CUgf2q_FUM~t^L^w_)Zgyg2D;^Z?vCIitWlR08L_LnOgwLiTO2ISGY?; zf6<;z``evHzglV@wSBxuy$|1zqZG>7uHXCFeVyD=g@}INe1V$h3$QPbX4~Ifyv4+6 z!@?wA%|s=3!|l6}72LmwV@EAry!al+(K&G^3}Bm#@2j)SCBVtWUd+AKA5tiBo^G5J z?8jzB)A9uaLbK)CRD%a$Y#J+2pBK~T*1WS6VQWSLJ{tc@@B?>CtbhuI_d)iq@a{+Y zU5B$t^5E&pqFdZ1Pqucw`CLy}8fWx2gn+BV+|2x%`t&$I|jA-w7T*;Rxlk z^A{2{i9q5ts!C0OHV8@H53QcN7l%FBF)5t(NoUVW~7=y=2gVy&^ zpy;l0BICL73cz60a|zWT;;uW-_Yb!eS7VlK9MYAs|3YK!Q?N#&bPKXB5+4od^CGkT@y>T{@0O=OlRF z@4|z4a^zCVkZ9x9g`;O>+uLx%0QyVM9e3_K5x%OVMErP zjC7A|Mjg34n?KtB&7a;30ZKn$dvv&YoG;T`)4w!DdC;1mtT z;ODjB5-03N$TrIEJ!Ee1_aW7>ZXXiL$mAUB{>_850@QyTzpvhzQ zoH1NJUgp${`7i3hKTA~>S3oz6p>sPExTXM|fgY8YoL3-!pik{0ALL)RljD$KdvC`4 z?WJq6CQ=jf#Mfd(@haE)Tq*i1@tvSLtZ3Rs%8ixV<;)pi|6G3)0_cJn85@9cMhBZ= zW`6AZIrxD06f{Z0?Hoz9x1&a)%*{>&{`LyT;& zdx1H8b#!PO1N!@MzdsD_T1y1Vv8r)_rH}uj6aBXy48AEl2TF)V)pIc^phA12d4=uS zMs~P_0C7zU`fE{%UWvHi(Hc}$>zFmE(7Y0v&oSQ1b^@t0klqe5ZPhyW3R}34kz)nt1er2Ga*FHv%?%KU2_<0ia6VDK4 z!GxCn#Wp2-VRJ40ax89Z+lM$23_x-ipLhN*H~BgW$Lg^k%^`RP^K@OCVfk2m_PAFJ zcPWO*njbM1O)~Cw>bmQ_;m~K zBLGcL_->2g8Mouq%-DOI>FU2E3rJa;CTKNHWhy5C>J&#Ud`M&`^J?QWbDSj?JcxDW!;=*s&W))G2{ayr5Pwf(%FfeO@xl-IPumsyBa%df*V;KVO(M zAnlh$tw_PYvT^;9_W9=$tnW1tP_3V0?I05UT>L;n>;cc{vV$$R#m@0)+e$d(`+XXA z3}Z`TN7I9jG>jq-CB>a4Vt#JF&bA!zDe-*c zavdyVeNOb@zt#ZO;}QgxA&9b~0dGum{gZ|< z6FC*w7$@G;J!cs}D={`A29Pu z6_D zcfyu@XuEY68t;&f=rTKWm{(-|LXfMK7gkraA^}=S_L|27Q02r4FmZ5xK<_D={OHtR zCGyukz@<)U5jKG3}XXFUsMhFbTkJy(sMP>Vg~vzx478BBk=XsK1lq!5>xK>+>V6^|ICg}?Lex8A|HH(K0b23`qtfeHvSt% zHTE~5pR!YH3diAWC5>N2_*;wL17GwzDvea(&%r5`?3M?k*Cs0{?wfbLUDlI~d?d49 zW&Ey$>%{!zIUlg5nOD<7laG<^ETy<3?N1^!_?ruoJg_)}HME!C2H zH?!~G(n;)lOZNji^;tODLO_8yZeZ zrL1{2qChHQbNll0@|Q@xY#V^h^x98_v#(eD&ccprNVpAl*2qXm;2%mrUU`XBk8ylF z4g&I=VHqN>P!#CNPT_$=ss*H_au(q7o>a3tU7!BnBAT1EYV8>J`xExC%YfclMUG0( z0E5#ZI#dWjK01d^CPQ3>g@5-%iSfzkv^KXna&LlY)(piX7j8>5ccwcPp7banhY(P8 zd~bdo?@V_t+Ph~QAhfjUAkG`&VpnEMwXKX)jkJVr zKUqLu>%a}X68#h0-!1iWzK|j!e9^4yi>d##b}kxYciNVK3%p2h&fB_tBc0eOcbFxC z^IKi)Blr8UZC>YxWRlckj|X4KKXtC{7%CepG?W4-UXiR%S+g3Pt;TTJV)CV%Q_*-F z*KaY%M}42!X*-I~*3Hdj9l<|T#XrZ{v7JlT$~j1)EX$bwg~N>@Rkq6}@_IuB^M zx%)H5)0NH&C@Ld`Y{(g&o>S+277Fh}Muz8p`4CVDfx^Q-<$Rq@Q|&q1ZI%j6${9+J z_{ovvHVfji4lFtNqV&M%!A{>mZ0`WH&oub$kN2h!=>wOZ{sOy7<`a&czLlxLo(8e_ z_n)WJ@{kg5MBZ!?dwDdp0h#Tt;XDjSNb4;j5MA2$@x(txmoL*4B^emL%JsZEl4rc$ z1`pnFDex@)NlF`2y71q<0EugmtjA`cqr)~(*tjpEo^;%kh7gUR&eo5K_L)G4UNON7I@7Gws^v1q&QJKRqCOBP0aR!<0g4{C&xY%#G*21|9Pe z%SwmtxkLMQA6E;|o!ucP4?`;@F+TnzNO0b9c{Ed$RU-GI`xDQzMyU&2ljEg{@oMMT z=jI)Gxdl59)NjEgz_j_7+NF*b(evCI2$4KJ_ey2CK|ekG;BkLKZZ6ddGb>c0Hc z6M?t-%!*sXEFm%IInT0MTFS9OobHqL?Sf7G(klm{glNbUQ+zlg)!zGsrr1SDnL7$i zvd@_ALXbm>uOo5wO$F# zb}W0_u^M9ZHZ^5oMklx;>QfQZLNt(+OM>&CM74LY8Z&SQGS~x7iAnO*w`>^6EoL)B zgq$}?Ett1qh0O*7sZx$d4NQUk51iXPM+d)je3H3K`L>2P;QNL9CReddYXx&FK2LDy zeO_#A>{hKL?{|!i>~BhxK?brrROI-nL%MWr2akXlQw;`7Ig00gok}<^Z3dG)lnrP* z+O~U7Eqd291fDF7OE{YY+HyQW3;wc_vz<}iO@b;hS??=*$O^n0Hc@|(_dbGYApiqO z`t5F#ucd%gqYsvm|e%ppHA)2#^OHpme7zs z{1f;^ihn|7XOxb7cH{L!PNt9{BfN90v+&zZ%i*zmL+`~V@ZJhLZTm(T2xq%QgtP!Q znpAIWM)gylxopfYH^5O1#=vV8E=KIkw_j4*)&?3wJoXJ>c(4_YPdz7@9vmT^Hy-<> zS!yOd^1&jKRSJA`6H(y;{3kfG7N~-ugdYPo>3)P8x35sW8p$rNIkqiu<$1op`*3nX2qfhdJzX za7v=uEEHit3QBGO8Ws#iAHy&~v0hcXpYwH;gW)pK1oI&#i`=&34@XuO!7O=+F1+EP zszz!U)$9>f*o|7e?2}BcxJ*>G=k2)EvT3aI6BG>LQdzbMpXTNcsm;}6P;GqbeU~<( z^Rl(eqBiL1Vv5kcg8=E%+{UaH&qDkMB`BUntr?8C%<7ln5t~{Hujti= zZ%kkzQupmcVGki;B8~w}(*38QpQZ0})?IdXW$VMX9~Q(`I`Ll+RcaE_*+U}WmB6Ea zgSn558mz(AwQ^rXUg9jnw>WC|AwD3;@v{tcBNZZ@Qiv0OJkS2MK|Q!qcj;K?@gYVo zW?Y+$QF*&s1BHtTfR+=9(9#9Aj472o08m}BfS<+jT6^k`PTWDpcCa$6l)1bCb! z++5IQ^R#kAm$+3upBv4@ZsA1zpe-cUqLnPNYy2KU4SD4%RdUDcc+VU-;)I%t*X>x* z zooMnStGQYSso~Eav|!#}@LLMXtdhG928)4fcEKdg6P-AEdGe7>S7LElRDuY_uPqSIg?DGELk7iUg)3b=J)3Vg5tN}fF=?9B}cku za!*#l$b4X$&8f`|A-)S~gT#947zLf8KEQG}uU=<(?Ce|?~8!}ARgoYfs@~6>UPrc z7Nr@rRzgHLzJ&Jnqw?0f;rv;mrp=$%(^PXT8M7ns{>R#$$>ynEIySoLl0* zRKD3)Y`=Sn?3@f)9=vnue9Wbh^vEbBG^tDfbdydr8}7HjK}OtgdX7D!C2ak60}d)C zWFF2t8IzVLA%^LMBlR>o6fd8ACTJFe)mOlqgkJQr;yu&t>h|e3yF-ftZW}?MRbsUb zVpe3ggPbO<+PBiF_{k&PDhFj(R~3?FvL9w zuhUkqhm9o9oF@Ok_}~RxD3wqpCW-t*ECPlu$&W%$>C;!QvPVKrUpK|tR+F{eA`7@l zuklXSj7~0Fdzqt7wV_O5HAz(J(2V*$wP3`QfZgI?P?z@Q!E?`QlzEY}*q;vjJizS| zd}O_g6mD=Ct5xd?|3TqCqXIj}-3|zT+Va47ZF_i6EwY>F{9^I1 zR`ZRxPhTRV2qKs=$SUk6Q#gyJE71`wm3^W32Ko-Vyj*M6z})gMLEvn^>}r8cfIH14 zG;;i?Kf`u=I^fw?65bk2p+IyepC_(GN5K^@21U~5fNUUlSPki3Me=VVtFvEOdOwOi zM;khqZ~u*MQki!960pa;UVq>n$Je)d=H@Ly{(s^r(#a}mW!e{N= z#8AD*Y5eg`pkG7)NYx4C;iZ*Kg8C0+Tclw^1+fU0SWaDyAX?>Q{0^kKBRS+hsRgBT zu3T<7PpgI|7jo#lvE8njxUdN6Mi^5LJ)QY{hufn5y!d{u0^POl1sEntRL&ZnFq2LOAx3Y8sy za;W8+od39!@D+>Tt67CTrS;$8_|d%={9!TdxwtJapL~4-Lg^co!j^AEAa=!`BeKVt z!ebYPq>MwEpcXrsDzU>a_w|w4&E#LN^jeUB4wLtw0PC8O*H_MSArY$)hzX7p;3Et0 ztx5Q=Qd8wXcKwK`48`Z?3l>=Hz`CGZ4=NC~lgBTS{E@mbiu;cz(``{;l9NwPZp78c zq!GRrCDEa@)@@%Wz20-*gr*5d#Ph4_Bj2Ri!?`#jJ$!I?{~z_z-=7~6B3xclL&IYj zxm)Q%{J;zjpdM>h6f_E>ksfz-u}des@i-6@V$=|(@S}wq474PCdlDyeo3rESsg*!l zt1Oi|g)E>8v>q$aRu)k$(ixeAK0i&Lx2-Rw7V!v#)gGIUjZTkOm{2eb`Mo^DUT&`& zX`DO#K=jaOjTD6lBX!5qIia^mOrKKF3^jK+pU_Gyjfn17y*G(nov!Y??I)o$aBpEt z@$~9t<06%cmW`S^uI;a8(*2fWtDPUQyZE`B$pb!#MDAlew>mC-TtQNjVpr+&Sxm9_ z29n?wB&$uBS>>UaVFw)Lb$uN6Q%ixM7p$N?;aOXX?)pY9tpBEzYQ1f_oMQqYNwjRx zVQYcWN{0B=w|04X&0IWq(MjkXw%9o``*{EMPZsIOk8vVttq^Lfr~BbYJ?qU8ngjrh zxWJee;J=Y?pSp;PL|yU^y=kAd{w$ktjMh zT9Y~`W0Z|T+~W-~TPxj5>VhS?9$(qL9zvJ6wx^~A)QGRMVQN~(H+??*8`UC;zB5(M6^wl$Gr zEe7hBN?+g5dN-(z0e-L9WAf0XOnCat$5!j7k@zDi{i~CM^K|M zd6<~c@V&6$nG5Nu%^0hOrT8Vm)KR-q`jhDUgHGr7QD&9N-nd$}i7FN|NuO2?wVy8J z?FKOQNHY)YCfKWnJfM|KymT*it5+Z+Yd$1nO?L|TWN#1bg!=jR)vPF}1~3e2CzYR5 zTE!Hpj^P~Q5tcFA9;l8=J-CBCJ=EfUtXr=1aqrVMb#gMat=V#xdU@f0!U52z35z~%P!%s1`W zAi)a(*H6n~)hgd40kd!Dq@(OpDzlJInBZN=m+6Oh0Q-eMi*sD?8)6K=!bnb=D0HMZ znVFI#wqmAuabKbF*@MKF%e=p1(hfuM;``+)r_Ea}i7Bc{LLXc1oFbT0{`}K_jWC{% z;+CqVKk=iw=PLb>H0H!yDK3b%S?YEo4#pV0TFevMJ9HgYN30_ul>J@v&=zOx=y8eC zEak*h9oS0b6BIy7>n~bf_K0QH%9yL}!b8J3l4DP<#(PhoqQbo0khh4$aM1V+v`c(p zlMPJ4-4V>*unmWmY?%M5alp2rs8kbsf4=9bUZ^TKIIPz8q`AYc5s!LE6PKm}&Q~yW z>%wJQs#;;0dp>KAAefBr)orGCaruRUM06cT8p2u&KPb5#uL*$6UL#o28gvSDd}c{g z0}7T5*&T1rzH@u;hNDj9pnF$#`CnI_=zEid-{;x|n)Bi}zrj>KWC}5ZYF&sM3pqmE?=MDLc_DrL&X&SETrR6 zr^bCb8~c}BpmfD6Yu3Nq?Qi%R4b15h;kn1299~~Y`5n#v`bi^-sjU!NycElbNHoH% zE;zRbQWQO-k;R|jy4IGh=i+{x&D|&V`7EeuVANIxg>jfFq5 zs3Dn?oyf3L!Ns{%lG&<#@s8 z;SA?0l~$KT{fWZ89Q!jY{(0>SM$iV5X{d;f{F8xZg`M92JJF$1Bw-}x74@pJ-y?T6 zrY|_>PU`_OjwE?WSsY)(#=05`M@s<=K0Y-M>VCG#zJ0PY9rIm2{-qTNM23w)OmrMfOwawZHtkh zEPz#!j`9Igrm&H1&zLP=U?&&91aA@%6N3zM4g5?u9OSq*O20KsP=_HlG3ndAB{xtx zntqXfT> z+5fb^&Jg>^sqrORIPrH-qobcrn3O2RB#~~S;kQ2&AGT1RnyFBX=S0->jvQxGy!(sg zTeWAGqw2*=NJGtf^gi>SG8C7Ew; zFZX1`$IzVQRhw?dI{m6PyC-Tp-YFWKpMd#*l7p4je^lwdDxy2ia8$DGLkTLVhiX3k z3a-4I&m?#beMQ!B1`c zOylILU940QYPPPC8%BVmDj}o7ORS%-dPoxRE=Zb4kymr{PA$(jHg@9S%Yrw|Q85a%~vLw z#Si^4dFI8;&K}Fhw7K8UB+T7S`9QS)QgOqh8*ZNGFiotX{XpY-IPZ71VwVG~da(mE zmgI>uP$U$Y;+bi2zl#`?ay={M`Ui~->V4Kwz+;0S@}`5o{IvzFuICC!`XTm&~KKs zRJn+{#T?qvMV!f1eIB)Y@5m-)LUfr-2xWfP$6Ni>%jAmzo;A-*1QI5PEri=9Dz5w$ z@esD&8TuN4x#y>O^dpx{Znd zhpRfB3z%i&E|k7fJn6n^SRAhnC48#;s^`#!O}ox`&(@%7zm2U?LsB2+A^KKlmDi-B!o_xSi7L?TP0U zSsF}qyY)nyl=iCA>#5I4>-|2R&-3rK;HsULg!ef%+S z^Pn_sxPotxI(an?huK) zEV^h+lHSAHEdSC1pbDKt9aS`vm?qPc&T)LEF~l6&`E5_hb3EUBk5qWavwACz7V6|d z8)1femDWeZt$O7a%iho7TSVxd+o6}=JvI2P&jy#{#5@}+)dEqF%Pq!q4NkUmzpe-_ z^S4IOYCnYd`Fy=ftFWF)jVaLcSaqzlh;>sIhI%&RVVAuaAS{11@+_fdOO1(?V4_I{ ze}0SBoaL&}tM4rS$|tSdYSeu`)b_&l1O+R)I^Sn=bql)ks{NR1qr}eKd1ElA8|A8S zd7@sdJD~`ab=_ozcrAZ1dnpb?(AkaogV%_?d1oA);S>y4&R!}{RoILni+H+a4p7#c zEWT`)%h##b?9pAyHZMw?r=n&Z8hcVTb`(_j1e=aq3<_%$T3I!E`*>Pd7jNb$SwJAd zy`g^!E%?kG3XL?{vG?|I8Fq+b5RPBhmWK*Xw%M0HRw^z{^;a&p9hUd5N!;>qw4lBo zq!QRPELdc*UUSyji3Oi;^E2)AX+ilva!^3K&mTIsL4j|F-yETeN%erhhPt$*#H?0= z;i6=bk>BQ*c8(=`cR#esC)$I_>7Y;|@H5tYN*S@ecM6+Y0A^u>Q+J=4EK~HsfpAg% z;_Q_8) zPz&%8VV_xWyLJ`HSZHk!K3(CkN*1;}k9GYV2HR9-&1vVn+P9!l|O$m7#8RlN1+I11wQ z$F!}F0IP1N*El%%1zp|v1kdt#SG|-gwyvxN`&mE|G|C{|baW<2oAAV%wgM+QfvQ~o z8SFc8LfznFrH**UoGG4*q=GhQJ%%DWg~tfTCil5%vWM*u-c6G3`*Jl&e_X@_O${+* z2(i?^euz(kd71Xsa*GCQuk(djY;Yw4KkZ+nz8)i~!KAA?BIgmiXZz@RLQ;5mxbrYF z>_t1<1Se(%1&WpjR+XrjaJG+sx&Wg$YtH+CZMI7J$|t0nr0Vu-a2wnqp8~uC#Zn8z z6!FLblvxhglLxV0FVWvGuM*?-yt*ay{(l#N_*5(ls39WiX&z+u4eKzQ9ccY(PFb#iqA28f+O~blMDZ6yXWeY7=^uN^CO;?fA}K zWK5KS;Y#=&ul(N3!WAhh3RROC>=#1S9ydl0kUISlF^u1%pc1%A0LnR2G~l%8LL))h zfQn0YZpXiIGe=d~9r7&!-&z*6@bC`8I8@+d@)c2#jC4;bYonm2HD)cvKP1qm4rigR zmbyjJNpVVgh?%T!iL~o2zKEXgFrwYozE9`sH1+mIa_irc`fIzQ3Bi1E7Eas8NPbR) zzjr{W-sX_;uHPQlyD?`-!wlO3^!-f1M8Ah` zCi^yt0KKoc(9Nobei?k)gc<&G!n~$oysV9dQkT18UhM&uR{c=)AG){u&g6j7PDpeT z$|K9$FmN~*d*9)=SUOD!Va7iNp-@dv*DGLnGFNVQcc_9&Pd#;^73($i z)uaDMXLYf_jMpxc3^gzU^PTS|wgDfm`+154|0#oS{BJk%U^s zSvpkxq$Dbe{2L4~OIx&k~3q->&a5WNy#oxYh;dY30z|a^@65UzS5(! zy2|`Pi?iw=3=Ny@;oyHO>l{K^eJJ@R!7#m>1iz&q8T>`ki*Xri+atCM(V4AdpL3a z2vvB9ico`snX>K(v#P_)Uo9bv%we+NfBuPF?n{HI%)z&Kr=^knrhn^UlU{rdN}uRn zzHKesrRYN7mrp=%egCLV^?iAS7`}b0k{T?JdMtmQW@-~&BUo9-@1?`7J-8oQ7%*I- z+Mb=X`1m?jE@879our8-J}~qXAKfRo_uex4k~|0bpHBzMPlbv}m)^=eh=Gb93VuJr ziSgjD;hpIlci%8|n%zF}tgpH}@8Ub;`bq{ppnaEf26U0fnbjSyp2;FPU6JwZm?)S2 z-jdGeg$QCC;k6FiEC%K9sO#5c<=7qA(kg!rL-E|Hx%|~4%w~Xu_fOA-(ARV@#`!I! zi_kkH6sEazNfHUvnV}~{ZcB->hr1&0-KOu}-@-ee>Vb6eqMYlo&YfRw^^2Ryvqs&H zbSg{7qqNk^h}%D@?8~cl_DooQR&3bl-KU}idwTnuZ_>qqwA zy#UO`IhyazIf_K#i9J7a-aPw5E`v{jTx#Ai!z7UWvVY43|48MvDLJNeF#hYl?vVt% z3EKnxFTWR5HD1lwr=ZK2^F?YqEQf`@eRVq1x6p2)I4xq0pE>is8zy4cX}+OQ_uVa8 z-h_PeuMvT$%;UV_NErMliUq1Mr~@7>^q$jO3dv6d>)JN`NFJ)8(n(>;-t|s@EM_h{ z?Pykl@P|^S2Vo4t?a3;G+#!>9)YP!r5@wleb$hBM(Bkh7u2)*lI?aCWoTt6khVPi{ z`mp+j&sP!K!NoNm1;(pZ@7eT|K1Z>j{h<*=KqFqXFb~OsY2sSHHKMNS3yq4)y}*yo zt5TH0UvDlpREpM!ia`rt8iyykQPvXK!Rd>P^=dLlOyzNUhJ6&o>;u_Cd>fmmH$s@% zjI)&Of172wxO}m+oiap0ufk59J9}OLhn3P`#3otv-~I_1f{ib|dvZG90hChe_Yxey zT;p>|n)`vfV+2|oEsC6nGPM>j_R)Y>_{fE?fg{E6F1qZ=v(4(aZqFzn|@w3kGJY-02(!Bfph? z?|<{pvEavy;>NA#IOnScCmR{&hWiQ>M_bHCA}bxJpPmq`(;?I{^q`S3fp(ooUW&>> zO)}15Gq8?cF`Y$o-1qKkwrax5n=G-C*TPOE(ym;-S1+(A2wh>%fxfE_qLw*faJ}B5 ze^|$EI1;c0bP&(9k*`Dfi>ii(Oq(SP90UvB#wVpP1$J9A)19KY8QNnAvh5Q#wD zT)w-bd3Nr1SCXWt>7BeU3xT-7frf;n$Lh zHaFqME}Aa8pJ9DBd9$Y(eq5>}kW_kBQ*eP5zP8u~m##hjazjeUq?|OB3oDuDr0wRV~nr9#&*Tm`Wz5HP^Y0+RJxCH5_8{ zSfU9@!Y}1A4-lS@Ltk$gSFGy|~XT!*YBfBDt1&Pnw}Z+Y)HzE^o^ zHAy>4i?41Xz$ZEib+3nxn2mP6xOmd-t1@r#W;9cXQh@CM+8^qA8@LX+JtP$ou%Al5 zT}O_mK(?TZUu=@+1U7BH0U}A_9V+e$S^tW?#q7z6nhF-~6VY(*X5)i0iWW)v5INm1 zyX2G{#d9OrdgkV1r!=JO>yw!dBu6{1qtZAsbifArHlm6qTm;ADC*+CKSSdN|9mCPG z9(@}&5=k!q1ML+8q+{>=;uQ%P-fHlBMe-1|Y#Kil;zw?$i6KY7KK6NLB)%SHA$O5U z6xtf1^($9V&%{Kijy1~cpqZ~R-0X^sHyeI|xp27=8qhHLE!yj@SK zu4my#lH{XmO6{$H?W3KnWFYKbOY_yMtw|xa^#*1j0)DCSK1-R#Uy0rcF6Ninma{0Q zH>$0&6Q6*h1vGD5@NT9U5j~b$cT*8W!93NP>!O|o-8nitG%D%hdpu|N#Q1+S)~780 zu{)fI-EmNV1?*k@&2Jo-yTq3q1<+IC;kUL3@;qfGL=G`G}>N5H?H} z?DH{mas8UPw-6KuR%Wu-yGW|;B_vF{JNUnLbp44z5g1d+3lvn`1LEzd zF4nX2+rGx_-1&O7N;*0^hDavkcUsl!J)GNh6&)|xfye!dGSzR38LL<$hIS;Zv)r*K z<%wES(vB9uXh1PyktroaRfJwX@!`kM=2TT+O|SiM^v2f0&+M$GW0G!TzUW~YjTpjK z^0}P)B=8pfk42>y0Dx9Tr7HyRv*&fbMn!PU?|)g(iZ(-f65JNyq3|GcLZ@Z$gDmwt z8orPBT2WpJ*>E&1B5BchbPMcIrlOGgU6hdoKrrg%P5v6Q>zim$jBv%U8&R1tpKd4U*mn{s=G9OQBfjyE(OJn}VN@F8veilYb2N5Q2tL1MD9bd<-uC<66B-6HrRP{fajLf`VR`=aWoO$J_ zy9En1-l_)8AA#fP%n{Yax@;!@Rek0slr04@Doc2baCSe5PhU1;#8nQKSU(f@$gP?= zo=1~?4-=d?dM5g>)qwAUgc9Rl31ub`l~MWYXcF;a1APDHWO821Nv4+FRzTNbNEdMo#`oe51 z%k2eQyZ~WaCi~)dj3DAzd+_hIh+86XeOrPqfp`=0?^yM4H|m8kQCN8R(ZUdyGjuOVMI;57W)pm>Z)C@zTzRSi3PRC~fxAJcj;aAo_75a_y-NZvTELQ!$5L?^;fpo75~Jqx^yM@2Il5i;Ylv} zVXeL$Mqq%?;`XcFAn{Iw-~M>(qfrfZ3#)v)LR`$!`C!9oeETX+7Z$25HtYDX&{V1k zhO@8kB zk50Dhs<&Ji0;qbLqNV-ffhQr13|7EOk{wERAu6{S)BsATzF5N%r7bLj$OAk{v}@gi zVv4R*!^gS@00wQrL(Q-&NpF9=L@Zi&@~r9)2!Z4`48XYDJK=IVU*8+Qchr84anFHh z>l6)}=JI5StKQIQZMYemS{t*uVO6rp40&vIm0^6Eou#qtmtWM3oFvt49ls_JsPI%Z zM|Gmm*+{J^Y6?7Dp7Q0*J{FwOvhu%KCjmm06EgPQ{K+cyJbumrWs2BWvU?Tw3CL(M zs5d;67#X=%@W>Bj1}UV_{`4^4S-{OFCGoo;`dqbWGU^nLA2zZW0(0FP-^G<5i;1!D3-m{gRd$HRuR1GFX~$4NF=tIqV{i`l z;>GWH2Gf0~Ui6ZFq!gmqSQKMmFjA76r|W_f)KKYJjSwgiCc2`!dWn?GkK=NYOIW7O zX4(4cU=jFdZ`FCGzE;2V2G$eJ7{j|f_J?|*e}HP5xqHa7=mduD^LM;=dtZ28^i=9r zdu0IE4O;?bMmcQ1tu1Rly9s>^N+;j*tBNEgkqdRBqb^XrX+$UZoaB9!g#7NQ8uE(y z?5!;^1_PbRNeYJPo%k4YajEM9`QUMyf&z-Hdvl=igs~(#e!mPVezu37V6*XVt2oVO zYkt1vM?1>XUH@D=6uv(f7fFgdi$-AWx0zB)Jqr9GA-y;YkQBF{rj73UV57l8yf}(P zU?5Jt7XE?ba}gFCKWQv(v3t)Rn3##stoLPPz@84y5^3H?W0XlYmNZOYN2@_s^q)qV z*kDDgVI?gqwcS9=I7iM&^yWg_kZ-n>w?G9snlCllmKqTSqs=pMiaWzh7oyA@w@`V7 zweto2@|(5@(N=FS5!>#WVp#kU2$4{hZ=SM-+E3wBZcel(9i0zkDO2N+-Itc7_AUb> zh0`rZPOpyk!st~-P-J9e;=~{gOi$y|vl2ZN0FlB`oc)f>>PGNXVbR8ldX1<*W;9dY zF7nFicb%P_bTAEiEn3TPuVJgPzTbK4Tl>Z@$G)QmOK^Xj?_COU7TV{egu8dGelg3p zD(Jaar(Y!4@!&}R+vNXK4rHaBi>HxXq8rOrP2;uJUb}MDdo%Nz4FpRKW578`F6vdK z6++#DiAq3tAO!Za?OWUC^zMDN_t!eKIhox!&bC1s;YHYzrC-4*;I?x8~}iKI4Q zo;%3`>?|Px7{sEl029UnYexE8B4Zz=iAaG+b>{W0LE^7m>p7pwgTwx)`u**S{TQ$x zQcGicPga0=yJ5eggnn|~1{t~o7A2To94F+QJ5);tsu6aPUy>FNO0!mb;#Y(4B#i)% z$t@C(g*W$tUB$-Dle`y--V)+7-!b+n+}CQZxLKp|tY|Q|Wu;M&2-Y5g1K`}1bt`R~DLI=>k9NRU4UU^8S@ z>+nUs(Nat81f3D7z;eU#tH0P>OUKyJDABAuPAiDEcai zG?K5TpgN^5a{yf*3r(s5Kz1_5$6FI6LF%#Z&1pjMyE@?ew+`#qucB)HVw zUj<9M0{~wFP6(dQQJ#^E7PBiYehAWmBctzK_#qSpv!lqF!UPr{*&bF><-~6QDI@dq1Y#mB_}}pD2P{9!(hMy z)R?@A0RzK*0Wi!~FC~q{;|Obe^zN+NE-%Ue&^lSRI|trcF?WBc=nb9^BDG|HR$K%S z=O4QNzXsV9g3^mYfanb{#X%1cUCQ{0`JzIc27ednCHvK=Jf=V99uD?_qfR5R{6JTc z)7@6Qu1kj`?_*WKY-3hSJ+2kYX|T|`YJs|N+t??9sQ9w|gl!sa{_UVxqY=5}DIH6> zD)WEQ`nh6I?hC;P$)ufgMJBFUplAjS5p}4eWFeP~4gOJwPMH`4fJ!&BPxPYu;{0^+ zy_qa#5a-m*CGZh11kV}+l@nE;5NP@7D}8lx$yXY>`yv^%feV3S(tdwN$G>tYUp$Z` zoEk^IqC@eyW1Pa)@PR%QKgozLslvdW-G29MHOceJ=X^a2I7Oc?tR9DJDMU&SCBWtQ zAyrwb3~)Q1spRITAaP0H=ZM&f!2^xScoDy$=UubwP&vg>G}K2)gZok40_&OY+geBfSxn z?mma`6IL9Wuhgo8Rf7ijFWU`-(EgDX{;f{eFD;75qYseN&oHaln@-kq2i*{)YR#)hll-A^}BxFR1V(>qI3K`3BZ7XGr^VlHC=KrwEQLLZZ78Dc|+A3}x zu64D88eLcdGM9aFnV*>IDZpSdzQu#GJ9(@qOx^-%{z{{U_wN6zs6b^0^$}0QAKqNCv^NaX#*^Q|7M$HOaM={qJQ0?Q~CY0r}!!RK7hjbqy_zhkr?c7 z$QI3cYy4Zc5(cq&M<>>HiQ$=tI|_2+P0)Mb{9RXG$Y4-KUPKX^XaF|Ul+KfLkR#N1KDDD*V(H{yrH73q z56qxmM#t5yw56ivRMtY6od@U2>RQ3E(<-FFt?9zi$JT^~hcQ_3+m&fcgh6r9aYo}b zF@b0?zCWxx=s#ZNUuF*ul|w+;>fx>%uLBAo4<34RFEJZ1`EUUzr=-cN&iumJ-1eQwcN1L`7dig5?5+F)Ft%e{6ii48z%Y&(~1 zQ0uZsyYKW1_@QX}7e0kwwPalHafkM6Mg`$Tu)_rQ^JDr4u!%*B6Y#74%RVEc5;6dD zl8C%bl+@(Dnvey7Z5kPZ3KiZMnfcdwssL`27F3b_4*JO|hqMy`y?d+il(H<)YLqo%d zPQ~`~Em#~I!#Qbyb}a|YB$&}!y8z6}T2z3!G>kT=jSFb9pw!B0r@MxeE9F5cRW&{q z)nTmDcTQ)od+Sdl4CT5Xs$%^YrB?X>wi>mbqGGwqn{b?XlGSq}W(>@MXFvH?ZZ+1~ zEBSr}ko?D2YCr&)XXc^YYUB}b)6aMxe-$z9i*fzd zuffubWfx(J*o$#R77Z$C?OKllO<0*io+f<%NbUPc3b3ZihL)I0;5D9f(A?%PhUk@B zYB1W?YPMYJ^9Ub2I5>E_Je#@H*j2Bg`}6EfMcyT1(0340O4m8duudhfrqI~%OKR);LKO7m253q0t(n*x_wSlyH<$Vay9YgsGXk#P#Ez;yh_)*bDe~Q$XK^s3Iw zdXbic2=n0~)IJZr@|!KP5>}!|z+ow{zA*wv(q951Moz(Grf(x z_JS%YB;Jwp4Hp@SO#`iSSd#t!L+{|j~^eFnR5K-a9=T*~d+w!3G$2-I7VPPcIMp7G30>kSTlclU~vJ zV}dq54kGsAWC{0q6l}8FBqY%C=HpqLT3nlv3R7Y4M;5DffHA*ncL&|&yFWI`!9k*| z-`SV5iG=RHgTH$L+7C(-#$JoB$8LKWBL{la*6k7VyXg&vWXuO}aCR$}{n0b=AH3MX z0A$S*aSTlHFhuOKShbmdTVTJ zy|+nXkdw498&!|aPb_AljF4&PF%44ES5p-QS|KBq&wpS|`1=6H_*TClz-_M)fR3{t zXWV}4aAQOfFi;-)zyWpP2+$Hz=XCQ#_qKzawUXlRU0jV@urt6TJhELmyW@S;g3aY7 zdtch{gG5mKUC$>Z22QHo+a(x?Ad}$OD|jcyp_3)cZP7BE_YN8Xz;^RG20#sJE_P~L z&2+|FJ8`KPzgaO78@sKggtRncQ3R+di;UFvv@jiexsR3hiEc*MI;}q|2ehBRJgw(P z%DdXJ4U|-3NTkQBOwvY+47jGEX)eoM0#0BX8hKs8xSEB7D^KIob`-aT%10Qo-o&l& zY7Bj4vK+KtZ7i57-)q6x7AjMVKW?mpi0Dr0CNE9awqMq*(wh6sxDVqUI5yGSJIe-| z&(!L%k>rmMjVYfxO7Au?CG1y3>awjD3>N*2B9<1}bD;k(UdmVe86vuMkdq-5EB9Zm zD?|2)`vLWGE;d&kcO8B`VE^CcL{FiS%3lEK8OZ4J) zxZ|_-VUX8#Dd(__tPU-Ep(+(&MR?dY$o;lps&N zj#*&ytod2#>EEX~nq6*SZs!)i_(@o9J1w+QF~Nya;p~-t1^5{ETtbQ$d8{ij^>!;a zK`G(`8X+r@bg==Jq+Z_?|6zq<(7O>pU0uJ6m{L<;(o>V1+;ky&Kh#)EZ+xlYV&|X* z9ngB4JnS}E?ui#re)-(=l^+gUJ10des4zJL68V?0FGS?h8PK zNZZ}F?*cXUi6N;dWgC$|ohmo$J#pk+y{L}UJ1>G+*OqO~pZCUb{_Kq|MrtvS{z6g> z4#NKb5K40fklS`mau;8N+fHe|Q{6p>iNLWKLNdhCpb6#kuPVYR& z_d9d%lWM>0zNGY!iV_hk(kXwripwEZFuCmZxkR{f+?Ml>!Bahf;f;(LD75ez6|K z)**isWbA6K=YP2cBWb}DLW|!^e zle#Vow1q3|#wwC3EkNV9@YTChpFQP+a#PO_#C^Jg1^&enN7`_T@vaA(3=^?7j15`P zF+;DDnjHn^2RZFQ{M*j)NFyA{E~fvIt4 zJZD8oq3M8>_Rh+Cwo$nn@L|bYoxHsG^-BlqNSWPf)UM2T(f99ve3paqxYxCcy`E9% z>Zv)!&6kXUdk;3@smEXrOV7_NV`C7~c#viYKV28v*n8eNb_-`df#Rb1l?uzTYIt0# zLi7bVj5=Q+;lLi3W>Bd3qGbh(caUZ7*w@r;3wlopt%OlBal@~8X8~4OS&~>ADovTr zZ^r~jgM+=JR1kbFg5!O+?Jr?bXj(T?I5gRx^_|I=d;?+|=U@-L*-ug=m8zV-fsC1loU{y?vmZTJce@!HyO5SELL zJrW@qc0+BlMHpU@8ZUNz=s4(I$lRz)!xaH9rEG0B6coRE{N}m{m&mXtky4r^!ZPw+ ziH`ORcgG#Y0&*S;c<&)^YT3%TevR4UQQKj)lZt9iwaYBCKZV(Q(=)q2W-73#BpHw& zpIfUrTCwD@o~%!cEz~}9!+daj^fLCDEdYW6;F@*Q6}YYoV=i0scV-b4X9z*y>9J6` z*}l1&XK_p$d)y@KBM)iXEC314uA@HO*?`T#P8XA^b^ZpxP_UPvq5 z?$>M~zokr=hC$TYdSP;<=_Pv&l9TCB+cKX5103Z(c{F5idQsSQPp8H2hOb_u;De7e zLoN0WrF)YvThIw_SPUhV+gb3Q=w{mjUOpk*A!)*PJbAN!bFHOAUHudW&9M%g%W!o) zHqbR?d6UPi?;eN{s&e>~Px`Y}=#dqXEh2ZfHTt$b#UNq}U&PlIn>6xt?Gj3)RWfXD z!rK6k&0m>r+5|@NREFs6pY$t(Z>j~e?H((!WIP!W$7VGJac179F%tA(&+#TlYoAkp zf}d#4;{9Z40ee-zGX=DR=zNitdU$)QM+uRKemH%CO3g;kW7E8HU=cO+!eXyn^E`u3 zx%}G9*qzGiGG&q&kBGnf#x%I*E3Or3^!{J-m z;JxD_fJMRs_RCzEL0&zU@Cd#DN=90%Rc`#ma6|A=tLw$jooF>ekK^(Yz3Dwql35<) ziWWEZq}`!i0ns0P?EEDO-*ar0TitT}o2^_9W_7@US4tjh1CL0Hy*&R6ob~9^F*pX! ztY2^wF>ZgimVJt&(=qsgvL3jHDSwn&T!6s)OFk_ri@DzR^C81Vv4G3G?Pfi5aUMEP ziV{XCmV+;7fI~P9lcUwNw8-JG+8{s2rK}Qr<|(R^y%#C+k&_g598afZ*hQ^T2*fU{ zZnYMxM&Vj@2?aAafjV9Jtv*01;SYxGCZ3Znt~XMRWtrQ=fg`SWzh%C_Ib8EqUbrK_ zeEQpta2nE)O9b&X0OSTYw|u;VjHnsz?>qU>|2X~tDZmb!C^#)as$!JTiRBY*u;3(x zHG$LD3x_cR4m%fyS4_MCT-14JL*s>&sg`qJQ87qu734lS^l`5F@|Vr?nlDtK@)Avg zD1xaP>W+)kbIhN`-Zk-erUlyLcqUBkWrpLk@>4CDIfUxyIs+#t$y9@cAt-FLNqXf`!obGl|-(1$qIRdQN{o7awYI*Ed2M zt0R7&k1yNJ=)1!=Wl+xS1)RVWE4x*r@cV7yKch7nrEeyb$*MO@KRMSChgk7hYq)Hg z25!{Qk=<$DtE?2P0?q;6v2)hrFsVk{?;OnoCy=0of*|H>&N4_>T5;$GCrh3aD+;V| zBf^jDC0BjdKuY|jIHougEDbvIA3f0`wCy7)9mh{0cDpDwl?h{3xbx!EW$H4w{i0T!q>=`c*W;MgpV{2ok}m z+o&jj_ers@X+#kMK>IoO2p$4RvwYkWr*e+f*Sbf6_xY`hsGPZ7_|TmFVpw!^Cb-kh zW%o0ggyU-2TqP~_^1*DhWiL41`GIWM0XdHL`g%UC_{YH&K!J8Rk(#xiZGAlx z-2$ffzJ$qUt^!e=ITa+phO7WLQf30&CZG1adEQsH!z2%rxl@NXyTs51j^n}o0uu+z z;d?uqs7x^>D}nmcY+G~{EhdZT!vWNtJb223Y7Y4Yda#&TWwA$FGFP*hky)v<`NAH}*R|=(L7S!f z)X|p75Z|um3_+Y7mApV$jvlJkZj}z8L z!Ko!)65!NLi)ghc(+F`7E_|HSLXhaWo_L=<&u!94jhI8-VTU@YKZ>}_>od5$qZjNb ztc?LY(>CxjFB`HzZg}bi*=Ml5k;5zoZk7W&R&3yYY}jt({swGZ-Pk$l4BJaulAfc> zTOtO17C&Tg(sLAG`l9PXr*T_U^2rgxV<^wyg+tOaiv=o;rl5koG|CbTlf&Ms6@BHB zC)x;=y<)u+fY+&@z{?f@$Ed>2RyNy{P2xa z07j3XpqQP@@X(Bm%Y?rp``NawJm~2PX-K3%h^wJeQl%}m*)MHveIO%X-jqp36j^*8 zF0s^Im}{*IY8lVqM{PI!zjy^%PiGMm5!kX5HbLz%l_9w|3tAGxFIgD4w+C3KMEp>~ zpsYIs1msD#tFafWfVZ!t_zeZ>lQ7@ECl+`QCaR7i%7qq$>~nJ)J*^#V!HPzOf%`YP z8X|Rmwl9>9nl_Wuq9pa{Z(DgGU{`*wu?LGl&bh6V%evsYK)xHd%dp z?J3l2Q4fj|z#RS(-S^_8<^i&~JEfR~^~f8j>z+>}SfktpflmKkdij$F@6rOeooBVz zUsMX!MPT{%Pbz=C?Ej991Vbn>-&Bamut@tha^q!oXZ-;Ek%-UMPgIYcZfw{nx_Df%-Tng~)?wboHd7^yVN%f5TkK#BK!M*UR)daQPoQN(c$LR)mc`~@O)Opla$Xm|> z>0C?Krn@h~NKnatGR?7D$|npwY!J4E_(v+Rm3VxSUc>Ka#*ir*Ki>CH+RzY#p7%p| zqJhi1C0CJ^w>~dc-5<2PVX>*s;wFH}pEv1CdChh-e3gHDStSN>P{9@3cfgu??=I22 zn$?$id~NgVn*}cu#iI$V|3srxtcgi^VGlI2rZp;uCW zYMXiya~=yl&f;!Pau6KxVQcY9l{F~+_Ux$(FAj{}8hxozU|Qx``>~7{m+a_W3x%P( zy_2oUWl)r-@lgp?Z>#9^BmRNE18i{w&~f+{TY#@MLf>Nla&nOYT2$?&-S%7IP2>sG4@UQ!}+(q{mNg2*gD%fHas zq!wf=!_RCF>lMi$D3S1>yeVv1i1x+o0loCm?qN)Zp=u*8ED2S2jm`XA3XAylawq0k zRnoTReOWP*S$rn)Sz~JOj$ld)>EB>7Fx%+3OWHqF9iUDyU37p_4aNZzKa{u|cRz5a zYeL(G;}$g|h@-+j-A*7xi))RHH(YYh9g}bCU=@2!e`M(GO`_v}ksG<|o))UG6UtVcnj*IGLwxuUjE&2#_s zK)?Sn?SWpf7`n>`+q@-_VlMGI{+o+^>koja$yLyaP}gc)2_PoA0*)rS?Y2Kvp;2S_ z`p7=ivzDuy)tg|EZIdep6dmo!QilVP1maly%oxJ1P z7`W`LlR!$Z?hKz9s?(F{Nx6z&>nTj2Qa^( zuaWWCo0uecn;sqS5vMk7w(gIB^+}Z;6pGmO_x@~Or<6`Po!XNJIlErheiq%+;p-jT z9UGtmePvEcXlL01GlGZ232LRiX7p4nU}o?yrocfdrrEs&E|%w;i}h+#a1IA^QMY3j%2Apomv7OQh&$M$2}#o>&6Q4lL6+nwa?+x@5{$X-V%o5tsX$@ANol^` z^yTzDuzTFE7Wla69&4rsL&e5U-FOi%29vu~Ci)5c%Xzu_NEW_m&ndZ4^iD9|p^enh_GxTWy@xe|ZsG&qG~ z5}QV%zrsdtEvzT`qion>eb+V0Hfk7hL_DVibeS0{=PW#aho+e{ez{h!za`Qc{gxpL zL11n;0uo3B5(uyo{HX+@ML&6BgXktRJWDJAT2!Bz3{D)II0 z7pNr76lts6eP1^7Dl6V?o71rIt+2@t9XN`YjvkWj3M^FW& z{2dVy1OnNZyeg%SL^;KDxvxK2>HiL)L`gvNahkqW#xOWN0*xKxKVK0d1%~c6x$ZVZ z0xDuD_@aIW&isvty`l#>>n=?nc<`iy89)nV5^J|i)quop$4!0~Is83w zx7;P2c>3mix*G-h-PTfw2egXJb(c7rd#XIIU zE%;pH{{a=>4PlC^Ukp6cld65icf1Fd_f=>1Mc_ejlEn6ZKX8^9&?GAF?=K6SgS@%m&(`z5*;fi0>T27p zo0IMnN;D;PrR4nm_8)%>*#QFYcK3}zP>Z3@z3@Hr-<;|KhUW|S7YIkebvLtL{ikhU zC83T_x;>cA4H}BkazJqUH^D#r_RHb^a=3pR^N+v%a=5=7?vFL&Z}0NU;r?>Ce`~`J zzx|4Ee?_>^Zsb>l`~TAjcM%-Vj(BL_H&gX-tjIVl#0AY;>!WwHsL?c0nksRj|Eue_ z`tBOPr`a*8A~(%)%qZgBes@1p=&?(Ns;1`fr8&L(?f= z7<_J9tG@n;-u*#JBv89YpH@3Lz^}7ULsN}S2jm!Ucw?Ty6AQPT&*D5JyY&Q5iJ|Y( zm;vHH@Q%Ae^SW0V2;1lFZPlyb6(rFSP!>Z#9ijvVbzM!4EGLpMNwJ8AoFnWL4i*iu z`{yz$Wr8ca%-!Jl4JZXuu=kA zSzUp$PTR1z^}ePS=du|7069lu9Uz)D=Jum!o>*nNp=bWXkCXxbF>^omLooPs(^9<( zt*qAo-I2KhwgF&YWC`-8zroM|Lx@cxarynJ#l6@RhassZxTpjiLRFInkk1k>UmP^A zd5D3qucX{n1+TdGc^34lLy2-)1q>SJVw#eTP8el6Yy>&S_%$%a=epLbn8XrINB-P0 z(4zSQ94k}F#v?c2M;51mJRwQp1ZvmhJ(^$ueyxQG`BN-n86Zc|7W2+K=!9gp)QvzKpcEW_3q=Y{9;hAXLx=f`z|#-FpniG^4TA{Cv6J(%EwPyQ8SQt)kW^D&j|C3F zCA0_--+$Rk7@F6Q8Gx{F$tP!kSETv3$w4b?Qz{r#c(+?#`z#H2PjMUM97QL9Xs~N0 zNAMYZLk>wyL!c-58hGn_PiB6Fet+81uh8#%SNJRR`^%>odx-yaN5JoY`6~?kSKRR{ z4E)oV{LY$RVc@@11pQYS_?J)l{mZW~@VhwWR~Y!6h7kM;1HTKZe<6V1hibnN!0$Qo z3jzFBBL52k{N9iJLID2>K>R`gzta%WUkKnoHRRbZ1n~Rt>=y#~Jx6|R{(hI!{M!8e z-mg9r{k8e~9a{8j^Y=Tr2=;69_q(j-*XHl{d#b-Sf4}3%{|B4Dy*=o&5)%8n$HsZ- zbiP)u5`sI^>_*?w@qJ9tln3@QgCbNy`l~My?n>5U#$E|oGZ*uI`rCvXp|!XKQd~?h zKc0pp<$i7Q25A_b23wuQZ-2LXr^nSQs~f=xWD5mxJ%epq2(Z z?l(20YBilwlq4OC^W4b9YuTT#f9Es*`4|J(n-*WUkDBwtn^7TjB@>V3P3MK?kDwRW z&d%frMSHQ4jwOuykcM}P1^S6zRJDdw$d`L8%FU@XfCK(Fh5n;-#1#~;Oq)@)bt+tq z^B?aah?=C)%gvck){-eqv2|RISVCed- zz@vNUaXy{b&3YLADYbtnJ@}_sbx`DK^*K@w#qv!^9x(}eO|CyDD|~{7*>zu*4JV5qwV32JRjPAyXS2v z9S09h3LRmVsf3)951l?=;6;DmBU7@&Fgkl_I)QLHH*>l`>2O~QQ!-83ur}G2Ca-2I zb9tkV%9Fy)W3!moj9WYU)ori3x&_g6yZNBvnSm$Sql`?@)jiKdlYvdWzRfdXQ!-|? zza_hHY$He8j3X5mI+tqtRFTMf#F1Dwo#z@Ahrl(3NRliSBdMk7OI`e?&V}+(7g)2T z1CHNlq)Mc7Gu~#+3Z+yAp(4bc$5)%s|ruAO3VQ`Jxa${7C$zuX<<;!MTUuF;Wc$;x^ zLaWEMd*H}P@v=>~!H=Q?(}T+8)P7r}G?P~e*FVLrmL}WXmXCComyfjHnnz5hO9ve* zMsae5PvEVNkzno!@gG!pAZV44t4@4AFLr_lIvi*xHBITp0ek0tk#Z}@@PD!BdiOp4>XM$2}$azDYTgx4ZI#f30-m%;s9!XDlOCSj&Qf^Vn{+O z`DMG+r07~2DDU7=$VXT4rF6I}%E!2k%&RdHPoyGM@el0m*O$fijthe^qi*`aj$OSrQTxhD4ibK zK0g+79(uZ+r9e#sD=Pf?spfU9)fr{SLQ?XO>7*Lh6apiqV~Dtdou9P6bx%-V5Z7ApWpR8_$EwQ4=x&a zA(~mCE*+wRIo80)O~sdy(y{Lff;Riwv!xM%+QX~<6vO=z5(oEehG`~kWTTKr-R-X( z<}wXN|LMZNe=oHNm=!qX>7fH>X~rQ*vrOgHK*u(f&nfL%#09~o!CxB-CDRSRJdEp- z$B^>bZ47#WY0y}AI?r+-g(^5!LM!~uB$q{tOZAXEC3IQiGtsLkGiR7VMpoRna|ljq zCJ9ZM6X{!gx1ZX$4E`%rZ=9(-Y%sAzw1Zdl3ryu@IUGCG$v#zKlohca+K7#9cq<+O zy%40;0JnY=xMcrnJR`fsfDV2sl!ck9o8MHURlAgBmW7X5gI+0TxU5afasFehqj?Ly z{^4vNhpiPNoOof8;dA5gn_Zp0)*D9uQGW-Pf`n6uTKvOuUA5cYV=he3t~)J`@I9Tl z$>GrT6_-z8s<_gv$6|ecR&(lO^@JyqCr3U?rWk;8|| zSd#C>{6B#c_@^iZ6#X}-m`*ntCDU_KR9LXvjKCeR@u_A{IcfLD2-jDiCM5NeNsG*l zbm@xhlQ>SpG~XkmgU$){O-pxQU7#u{*`;i<9N?$@*LKPOE|H=ZfH9A!9&9q5K4NB* zqd8`w(Z0(j>@cBjy=9|Ao)w-tfMmfiW4~^zrzE)X(E$jTN>E+!Ylsx`p-bZz_TFP| z5@;h2LDexLm-P$71+pu)>SdGr@kO?I&h;3v2RX^L2b&fIwW+5LrAz1`+F$4o`$X#_ zf&vqRBWkiI#+y1_+EJe5btU2wG=sRKfR+Z`xG^|zw39@9La#98d$(npZ+0)DVdQzs z)82&p86<<%PIiK>Cm!`pR{O)3x{pGyAs#O+?>{Qqu9W_#Fuw~uVU?&_Z9Oh8w;Wxf zWG@dA(~XR$0tItLpE>A};)RN_pb(Lj2UL~sB}+Xk)qxp~FjpEHqayhzvP~sNr&i$R zPtW@iUF6TDFU~&ESazMxP7OlZYD#wuUt4sYgZ6dtI$-GSFt<15UYYmbQPeGrNfFJx zqSCaQWY%AOU%)kAAF7UHK4z;HJ%E>g=^!ABkpxGJ@mMU8YUX|GS?C0d3Ik~(m;1+kiUssS({JMG*~*w5!xQB zcFi^XGGnY`+eax6?Sf5+fd%nfF19b-XR%-mbiG2O-VZ`G7XD<~ln$YU_Ee^^CN$J4 z9KE_4oZvNs6f1XjOtSsD5b;5)r-^GJ1}a2`&7zttrm`^)Ds*Bcp4v&YM)TX;Vc{?0 z0?}bf5IvJNG3<1(Ww3(Tq5fF5!c1s!a%gpNRpRcFWMl_9B;mv;uccT)v5e3lRPrX! zpY2L*$V)xGODCJIn98Ur0pq6dd(nTax$OLCC44Fw+!j2qtzNI*2H80wY7HFLVr~np zn#58zY^!Ic+lG|wB?i6rgBL2V%rC%Lm!1TyM5_e{MW-}t4hyKcr>Gn8(nC7$K+t5Q zJQb6*uvE^e5ElHVg|cc-lEZS{iAPwh%!qifczt-isUccgTB&Bgw}p!2KIkd<$psFiEMIf%p`#(?{6C@I;xY~EpKLinC3z2 z!$Hm`AR6m--KLt+7R-kEoz?E^wnal~Fae({Fg>PozZ=I!(MF~;yZfD1O;;Tf)k^XB z8)(Bf+yyAVI&PZead+bk2{_m2M1RSE^gE*Coxo4vbuIsH&B-T(gqyQ$S~m_}pL550 z#F}e38hGb8!^d~jocR)2>+hIJgb$%wpe$@IICK%}FE|d--D}H>G=!&fyfA5@`M4^VZ}T9CNdf zKyLI#zSM8NloL$H_~*dDhH)KSId)ct?g`JgSWpN>^zTaaK`%&awt%bHmY?-Jc3;p= zj%J0YG=_2MQiTV<)trc?q`hv9hDkX_dHKnS{y7TrqwC(mg?3w&#*>N;|IyM%1NwGK zF;2T_*Sw)?p8FEa_5|U-V55{a&KgY6B&FA}bErQhikP1;jXHKlEfPjOekzcZnn(dc zNgwbC`Mj3|IVnA-ss77m^AMwndX2;);n%8MY-mV5>{O&O|QUs zZOCwi3?#PTUZf|vIfpege&Ak95!wsF4o+U#>cedex%;uI>x2CQG~J~|Sp{=*QC%Nf zA(d1fw3mD;c#FT|;K4)#cdifExBjGF!@oXkO#`bVE=6FT5Q9ZhBYR^3EMz4VMmKZ* zBUUyB;f~-qo;k@G#L(KoKG?X9v}kKs$@SW7Uv84Pd0GdKDZ z-M2SoR(*ZdaTHx8TeWTN^NOEDr4S8PFBA{vA7{=)I!FyU zOKHxL@^X_rdk86Q!64S|UvpXA;THqo}oWxr^VaJX$<{0s{aXe%qNR zS3vgpQfA7(xII?s22%OV|Ay25T4|{xfNdJ|KW(&&h#k~;d?`RC1+3d#MpQkqEy-4!e?J^;4KHUJQLpo%?#%@&3uc?_UxN@{NZ9?NP_8aPCPSR4n>lh27c) zi&#;wXJ|>kvAA|aIzg+TFfldL>NGf@3}Q4`D{x4$TD77l{w-lLBD#+w$?0P@Q%qt6 zX*Xv~o{46mCLUy0;(kNQ2Va7$`SOs!efNZZ<21;qS=0ugloItQg;ihMX3uc8 zfHIs2p53hLZV4F0mO2!g{i7ENn|*9|n;2I{t1z*Jawj0Q=$piSFD@gFtt2X4hL%6) zO>Ye-&ZCh_QRBdH9dsxF3c^Bl^etd})aAtmsXh!TqCJ>|{Qzf)z1B=Iv(0vMD*x?` zr9u5CL-)`du$D*nmCZ}1{45P+y0-$7W2zj`{-Yn86)c4Z6)ru#1FhPQ_gVNBNX;H* zoPqfe4Cb$n8GyV=g2=k9J192W$3!C>(xRtz;?upbPzjkL&5F|PEbEPqs;eN)?l}6y zEfaqq?H1-vebvc!q>Qu=O<+=DmBGw*QAQTD#}s-K)sxZ#43S>sB#Y;? zs;_9`*^$E|rN-JL0Ry_g0g;@7Ik~|88h1!pG)0K^oFP1OSn=eg%o|p|g->rQtkkQE z-(N;M&yef3xd=Ge;47n6Vz(751d|rhrcIM$W4F@HptFZrav>|A%fjH z|G;6%7~luew6_;7qmT(Z6HAvUsNqhbov?Pk6Z57Q!F@Q{BtFEy2==67=(bJvClBK;lz9X zB&n>31;(Za4c{C4Ka4`p6C;Yv_qyYezoleUY|VJRTK$NmvC{RDDXeKVy~C7Y{g#6q zDY|9_3(_1{S$BcHOAxky3eD+)Nd`5*c2?EuWkv=8tRuLKp*HQr?o1ov_fcr%G#erjf z0QyL#5Ypb}yQKiBWd5w3++yB5Zp@0{+GPu;n2w|U5^0dIeujToz`Az@1xbu-#>J7l zamj|OMA&CnR`_%3K&IL8ei%{Le6J6V9ZmQ;?xJ9_PjfmW^~}8dFk@88w~bnOTkzeI zvJ&TnD*wUuf2|x=3}|h{5{`T{v#d9#wmB^NtzYO*tD`Ag&2^ab%9{vce`5^q@9(ma zSHZP>069-Q&3H!7Z{{vXbA$vT*5n@7nAM;OHUUHV$DLf8Svy z3_03k`cBZ^$ur=f8<&#jS}RM6LO>p;rV_zCvCYRS`HMePaD>s_XRRu2bciIj?P!2< zYs&brPnd+OrK{cNHRZCFXu|lmN5=f_w!CuY%aDy%Mx(7eG+1Fh;ic)mNOp;TAwVu$ zTQ}_#?yY}u>wmxx(NT14Dv68pOYW9o^_a#TW-ldEb?)-YJU>gU@aN(s)3fKb(;kaY zVj;W`%G4`Tq&>ynD>`L!SVhh50EerUZjx#FhpK{I%+Im?wvb-D zS3G`P%zJg0UbfP*-IL1zc#pnMa=0iB*@s`A1pXtUCU)ZVN{kS~GGhW)4w7f8D9Qx4 zi_UeJdMKbUaaBn{j=9k2(xk9-8Lm_A&u$21W73u3s}7&IVJf%tYbNnDx{B0_ep5jaWDT&%LO_{>>i-9K_2OQ%jm zvE9}?O{AX!G~0^Mo`WUo59n%4`^?R4NkqV1)4N4C;IyLSvM{jUm$x$;9DF=n`-YLL zy>dQ7$O_s&UXK->@Wk@UEY~HZwVwLJtLZJFWA)1X#WaS`&rv%a8j1N|psoBm>=Opz z>Z0!Y3);}*h0vGoY_oBD8=hG$eCYY-2g{F_*5^}B{O{6v*lIL>5oyUjPY;>ngczg7 zJM%~8K;g2;{`rHG+@^UBFBX0t?ntZ&Km^bWr z6*J6t3asr498Qa&=g%xqqe+tT(hGg3YHEO1Mx&y|vvhD@s;tVec zk>C_0P1sYc1ac`~?xVxvb~tZeart9Nh7pU!D}UpDbeQkv1rbY0`M824tV52|Fs-x8 zPUg4FkrQ8JXsbuzT{iC7Q9b7%O}&hO;XXk^j*A?q3co@t!wT}vf-!*%;^Pbr(F%cVh-1DVN zv%!-5oW#lMTt+eWaFJzkNZj)`SRUH*vos$TmiUDf!UR&}1(*~C4|y#je}nO#{ZO-l zFgSrzNrt5uv#cyv6(}|`K(8lW*nqa{r5TxAQejI*twL8~d*FTXOPx1hNF2fIo!>Tg z@7!iL)r1d6oCMiHI**SM-v;X}nPim>b9V-7o&H2~3w?K|I(pP?z0Imju+}+9{0ZZw zkil~y<%yJsfw;JQERJryI+3)7VU|@GVT`v5J2-3Z^2i0*Hn!ccB2r0zrvJmsszb_P z6-6DQ)OVG9Y)jRA?EQ+3iI4fzAM?`Vxo^;Y=N{F%wL}cQH7|BamOM3 z?7dV-f78i;;`O=T5tZ`sBpid)Plkb$x~0N7<>ex9_nElH>`|9E>tfQHwbs1|b~o-I z;m7Sgu?rOw!|R^`ev=h#`4W@L*>MXo^IP3a&Po6?yqm(tsDtI5dLnM&9~sTl;kYMD z^G=$H+c~gW7gmRj5Nu;5j>MhtObi#hEv7>G7 zzz9}%xc^&WV_CfPmt);>Z%*!y*lsR@v+FfR+*|srKZQw(aBS@~+Bf7>&m65=4tSVt z)#x6T;Wzl}?tkv`&?(rdHgRzum=~eqLNx8oV|wroDGG3`?o9~YIezs#=cI90`eY~C z(pA-U=e&oOYb&`@>0BOe-ueq>TG`3VAXIR23exN&gS7QaDEquVSTkAdYy0)A@zG<{ z?$Vi8ZEn-mZifa#NYY>$_;8q0!|G6y4bq?Kcg5+fkXp$}#2`=bF#dO6Id_a4 zPPE(D3gC<09o$I0P}qL7f9DzaGO{+K6yRPYo-9(c>Ec1SYdJ!mBjhVu$C@5Qak~3sy3uPmb8+Webl2ZIE;t!Cs;9=0Nt1Kzhg`8=9XhY( zmd>8N7e{r;ZTAZ^+sk10wU`%LjW|W;FJ6LEnOW{=(+~UYx*&kA z1n449j}AFh*6~bq6&-uiV;S?Xq9=c~dlMcbFl}UE^5_xWYWd;uUWX4;6~NdoNncLgUC& z#|S^0>WIAMar8M>jj{C|2T%23mQs~op%U_FV5-hz^TauOf5t8F4u}1{jwSM^abR1| zY^u=OAl9ZjF_FtBpeAxp(qV7rkI*;DX4(y5^*%<-ud6_Qv}q^@Tn3$3w^xp2B)N58JJDEc}t9 zz*yamis`6WUCrYaD*fhUQlUBXs1%Obt+4>NL;MC@%fqU@ZsRA*1IKiSx-jlx3bWM} z^&2*i|#xTpVT;5 zlUX&zo(CeO9At>n%(=P4l2MAU=46Sf)#Y;?9?o^D>t)d=$eYH9DofSM#qLColl4t1 zza@{8z@w1_+eY;KQEB}ZC6S23DfpMghB@JCH0bIeNmk|SFR`-P?1sbWuEiW(M)-Gc z5K0R5!c30G6YBEjRn;2HrElyeofL>vmm6Pth%_~bW1P!tENML1&+KUAjefI>)E*^? zkg5~$$e-I))7m6mk2%@9(}A#25^y}Es%GSjl<$c1K!#&ZMlFdD1&ib*&UHi&y=mAo z4aa=1aFh0Z=9H))Qbad^J8(snC-BTfHYCP4uJ)&ZLw(;09nM$=ZzP-?29M2-dZ6Kd z&^kY-`*bCT?j7GltyVsRrytaJO*+g-g522nayXC8zN+c(vU?#>N&9$h-jVT_EW?X-4u_;B)B?Wspib$J>LTJRG%5wH6)PN<7&-KLvVAUUPSQy@gRgZ2@#l3bH^h4V^7 z>CQn-z=(KVfk;8SpiuSJ*UM85I-6fkEy@oWpuO0&3=YIUmXn}kX3D(WP78@&{RW%D z0P6p6JHMPN#T2=-717t|o5v;Z^bF%^&YcG}>+7+I*eVb}^!iyh6v|q!-n228@7&PU z)K*h-n0|AqU$~O7_{lIyIDO280%OIO1x7K1hK9+fM_zB;gq^P7{XDCCLn5p`c$VGN z6_*5Z3C1F4|E5{2b|v(LH4X=ry!G?zHVPmo5GER|FtZTMn{^45LTakoO}%z|B?S#b zo=Hdfl)|E3H%6K#Zv_v#)86KrgaqCCioT&6@3!PWSbUtVl~8i;K5sSG3G@9ckd}DF z_=n0<$%XA@uJ^F}pY35^Rk~FMZgTtB-nLtx-L@+g)u5}^$L;UrwVCjjlf;mNJ!e9ZX3=71 zMib3Kq0Yh8o|eJd>$LFl<1-w={vqPSVP2_6-7UqHU zK9F%D3*Xwohc~d*Inc3X_3BRRzNE#B3!qaVZOt)tP787F!GR<#l?_Fj*{ai0+A{pe zC0yRLwPM~am5mnd6j}#!?krqMI%>tgbv`t4P8+%H_7-Z1AfUFYLaj)k@Ebag zg%3FJONO_kBDqhYRk3tBJ<=82*kqTCYS_t$YG|$^*WrWIHEhxN+zW&ni>rz>Y_Z4r z2NyZ)rA`G% zS~TUsee#jb2agK!_6#F<;gM>e86iPZSS0~NnxuE}jGDZk`NEs9?kgN~uMy2SdXfYT zbTB)SLoHhe3cn^>g)A-C#(wcd1$Id4wt`U9P0MI#E>gRVA zZ+1dUBU__Zmb4I9MSTAvq%g7Njxh2SC|Y~`~1 zMJ0=U$wxH%O_hs2!T4HNIib_Y4m6As-_GL7RlCr@3ZK9zGGn&N8*wXrQhLf~1DLQDWPlS2i86?e51b`lr1_*kxDM+{sp7mAyZ^X{UUZxopxy^ljvMQ7 zGw(;V)JOMfRnE-;lpcaA3>M`O5Mi)x%Z15k{=u{2XN{>mXh*bb^IwK1JF8hkrS590 zj^=*w$o~SJKIjoMv`TvS4{fxjOzV-|;hZBu1cl{Kcv;_gd@Y0&Z8QecVAM_5m+rlQ z04;@%YF}?C>2pR;D2;~m74@R{_=oPw)t|s5oZ)YCsX$<&uoQA2M`CAfad!oiW%O8{CYBIuOYzD32n z*JNGwTfM)$<=@?i@Sf5~P0nzod$6E@byvd&UloK1`EOabAk)l`emKNPnok+YYOPFd zud;(>^jDvIkVS@Fq0gSWjV8$yvLZ}m=5c^umU=}(t=MeGvjs7zoJ0(L0`vG)Y%r*J zy8#IG3{A;i(sxvIGo`RN`$@3TE4!ktXsXp0Z#zgv)o)~64V3d&5^aPqQFN?#A4Knf z0IRrHH(bJbIQNdqRO|DB*blF5(EX*xyIz-kPH@wt37j6vSoEe0naP9)^y@rH| zrp|78Q-gN0VcpcSTBS>Ft4s{~fx*3^Rs+|dx^qW+v?UG zJl!7jr)a@Qes8rl^c{uHDTL$opoq#@?$bXB?cuQfSL-t^t}hJ%FZzHcR2kUrK{@;6JECe;n% z9wsrR>r?H+Z8_kK8-y4T9gFyd=pA;NX{pNc! zu>(Ci!jg`hCm)_euJXUZ-~_q8omwu6QFNJ4-*R2hI$tLp2fv)HPtarr^KyyvRmiOF zQZWci!I>T47J4;+!5jo>rDC+oTQ9N7WQ&S=M~MlkE(#ktxAM(pi2Xl>U3)y!`y220 z)rq5$Iz_q9+;f?0YAz=mRt+g3j94PKV(z!3G9he4BMK3@q_Bpe!&nE+<>VSVA@}97 z$hF^R)%o*$+v~O0{@CmF`EJkie%{aXyq{;^X&`UyJ8Bz+6$n4RhuxofJi_mi|L9J0 z?Yn-EUv+O#)phhuqMKK-vbnKq4?0id2TFVk_R8BtEte=c53betvQG4+ykWj6EV?o+ z-{-1j46y)l-LRhD1{Fl9t$e&ifp-n6%Zww|H)e7x!tnJ1Mkjc}l{?;+8vqUVEi)hx z>^W8fmza#O-v2}l!0mfRyec~OgPaii6aI+G%-QC?*6u~>ko{IZFMr){sO;8b#{XW%$DiCu@B zL4rQBJ|6Sp)fJ#VU{R!qGG9!$ z2!D;;5agIzEs5u#V3tXJ$?Ei>WJ1XuVkh}BT}3fJOD9Q#l+mETSK$*ZS>^R5LZx|F z+C-gSh(0x#m}O}g16kNf&hvm{BX*k&E)LAVGB!NeEgG!wQ@g};`x8)39ekq0^~&1% zeSRV)_%7->74K^Q4y{K{P!YZT*HnN>Az^E~Q$uO654Rsl%Q4#gA|ow!pDzc5BO|MdN;vdRpe~m4#fB@BV12lfsaFJ=!r>dVb47U zTj2H2^B2bH2W=!FPvfn<&i!mOy@7kZ*=hLhuNdi*vh5*W@fjg?B6kdj`H_|5TYYu^ z>`X=x3hs~TC<^HylUC45(2l+V9DS9?#nY>FGa8QBQV5*l6b%TUK`d=Iqup$1;1zWn z5zes72q6`?ufzf{5R2P)A=CQoqxLyt0U(#zCCexRH|Cd|oRrC)=>) zpXI)a#f;9k!IEj5kDz=JSauA!51r}u(&VV?nNJLXW5fg(@-OD*H^=X=HX-57I8SDH zIW$9xt4nv0Pcx{nd}(!OZHI!;*>b5+WC6I21tNQXpLc9(AWkQzczu2d6!d3CCz>mQ zeS-K398Ny;*&*(@MDL?|c(HHoQaX0(HtzZSbGuOSxSw&x*6-YtDl8Ri zVk|-}VhxSBxW6Lpd(e+++nL6584RIQWV~_j_=KIe47fmC21n!M$p zlG&@t*;k~-5Vw}?; zMyk4lthR02rHVoQaqd4k$Z!jQZ!-E&<-Z<%q?CVTI@l^&Ac}l|3uh*sB(;L+9=lyc zxviS*d1L}QqC0v4d9Rti{DC=@jVB9!qa0ej2!=G3>-p!w91zl@v6Wv1*8@tztLfiy zcVgDc9Ex&zf4>WCpr3ikqB-a)fdz3!adZTPtlI^5FU}*%hD_y%8 zt5+|fehSgIJ!`->(sN`!+T~JY!3#XopI@~`i<1JO!2d$28H#-MNBlhIy5B^@Q zGpsl^M=2S_T0bPq3UN4hTCYqr^SPJaRRN$KmDO-5RD5lN9m+Zngrlbl2Xh1$9fDoCr%7`uK0Ogv@CB zCf&5A45t{|-&gYbl0JpF*ertx<4gsgFZ4A-y?;L}=RA>LGx|QOT`x^ekQ(S(qyO^# z;&JUvW^gZbdT@FM(gU+Qw(^_w5{865NLDK6_D);z*1$wWWQmP!3qQeMl<1JS2_K;9 zf8jXFWpoh*fR-K>4LwcT{+lg;ytp~Fva$-!+1*}iY-^J~W_aHgj3 zh~}VwF*BokbIS4jTP^=FpSOyTOVQU2Kk))S6>HH?cwS9g&NL+@iibKF^(S>+8>p~G zF;dXs6zlmk*^WAYG;;MzcxN!)hxO{6hcFEL)ina!ZKSG9dJ2^d_DZS+?KlG4mD3Nb zqKYBCM09aT_Wk3H%?$Sf$Gt9cs-h@|-*JUC)RK6)>@UFt(U563Ka6i6rCIK2k;uIR zIO;LTW%}z4`~aK6Hv^ff88gF~n67%FX^42Ng?=%t8LMl+t?qei4%Ny~L2aKk5E4lc zy%Jd+OF(O5w?4IAIb5$5RAZ?N0e$e685ZzC;p*Vp ze{KkCUBty74m>Y@?tZ<@Z0K}yAJq>#+fO?N>vFlpjweQ*>>Be}@;KZx?(_L4@NH-S zWxlY@SAEv-fMp~WK2?c19G57gq#$xI*DPayuG#V9#^>9nmk$4GbYpW)#Ce1)&{8MY ziw>N4borzi$tDX+9xN&nsY3{_Sx7^B=vQtZWoZ5|d$H_72&0&kSD}yT8X)33%PY8Q zKAtfP8Ws(3wRkYZ*gV(SkUzyn3WF(cGj~{1A07_IaJr=r322HecF5hjk=s?PJxl!y zQg7ZC^$HOC#jEX4t&6<8O7(PL!7ik{GJ^;ODWR~wCxwWxHjXgjW4y4o>jk{}B|6FP zVKn~5gKKWA(Q8gDmupv9AHf`}^~K3)Tm3I9L{1H8o&@#~=!?(}bb7`$YcYA|aWyDQ zSs3yYGZN>y>1dZ`&e)d@8slc}hysEzmdBk8rm3%ZrQbC&XZ*7{2+Elfy;D0`IHW18 zS~|9-OOvL7$-NYTZLF>>KEV47^gfoQE9`It?}g99qyFOY}Dz{ zI%!P4=3g;)W1maq<`S4sNEtmR=0Y26h@D)4!w znV{KjObI|&)hPLvSuz8}V%epQCiM??=006AB>2xJmP)VZEHQ^CkJid{dG;;^BpD;K z;8UCTHJ6q=W$9%ahZehUr45BNyo6?1L(nCHm@Dylh4!(FV5E3jPu(Dq@)X-$mAc{g zMcgwVz!C0}6l`~&I0l)SH#Gn!yU(!A>@*%bJXI7LDfgG0PKK1nf8k-MfD)tc({rMmHtFb2Jm=U#BJk zIY5xsA|i_EKAHkAH-PnLz^B-rhs|XKkB8pd1M8=x;3dqfUHsHt|IS@f*;=c^UHJog gAk4dWa!1rx0avvWt=7jm<=emmgEm8zo^ZPHfA+x|3;+NC literal 0 HcmV?d00001 diff --git a/apps/emqx_s3/rebar.config b/apps/emqx_s3/rebar.config new file mode 100644 index 000000000..f8e4d4e42 --- /dev/null +++ b/apps/emqx_s3/rebar.config @@ -0,0 +1,6 @@ +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {erlcloud, {git, "https://github.com/savonarola/erlcloud", {tag, "3.6.7-emqx-1"}}} +]}. + +{project_plugins, [erlfmt]}. diff --git a/apps/emqx_s3/src/emqx_s3.app.src b/apps/emqx_s3/src/emqx_s3.app.src new file mode 100644 index 000000000..7864ffb29 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3.app.src @@ -0,0 +1,14 @@ +{application, emqx_s3, [ + {description, "EMQX S3"}, + {vsn, "5.0.6"}, + {modules, []}, + {registered, [emqx_s3_sup]}, + {applications, [ + kernel, + stdlib, + gproc, + erlcloud, + ehttpc + ]}, + {mod, {emqx_s3_app, []}} +]}. diff --git a/apps/emqx_s3/src/emqx_s3.erl b/apps/emqx_s3/src/emqx_s3.erl new file mode 100644 index 000000000..6d2577dca --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3.erl @@ -0,0 +1,65 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3). + +-include_lib("emqx/include/types.hrl"). + +-export([ + start_profile/2, + stop_profile/1, + update_profile/2, + start_uploader/2, + with_client/2 +]). + +-export_type([ + profile_id/0, + profile_config/0 +]). + +-type profile_id() :: term(). + +%% TODO: define fields +-type profile_config() :: map(). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec start_profile(profile_id(), profile_config()) -> ok_or_error(term()). +start_profile(ProfileId, ProfileConfig) -> + case emqx_s3_sup:start_profile(ProfileId, ProfileConfig) of + {ok, _} -> + ok; + {error, _} = Error -> + Error + end. + +-spec stop_profile(profile_id()) -> ok_or_error(term()). +stop_profile(ProfileId) -> + emqx_s3_sup:stop_profile(ProfileId). + +-spec update_profile(profile_id(), profile_config()) -> ok_or_error(term()). +update_profile(ProfileId, ProfileConfig) -> + emqx_s3_profile_conf:update_config(ProfileId, ProfileConfig). + +-spec start_uploader(profile_id(), emqx_s3_uploader:opts()) -> + supervisor:start_ret() | {error, profile_not_found}. +start_uploader(ProfileId, Opts) -> + emqx_s3_profile_uploader_sup:start_uploader(ProfileId, Opts). + +-spec with_client(profile_id(), fun((emqx_s3_client:client()) -> Result)) -> + {error, profile_not_found} | Result. +with_client(ProfileId, Fun) when is_function(Fun, 1) -> + case emqx_s3_profile_conf:checkout_config(ProfileId) of + {ok, ClientConfig, _UploadConfig} -> + try + Fun(emqx_s3_client:create(ClientConfig)) + after + emqx_s3_profile_conf:checkin_config(ProfileId) + end; + {error, _} = Error -> + Error + end. diff --git a/apps/emqx_s3/src/emqx_s3_app.erl b/apps/emqx_s3/src/emqx_s3_app.erl new file mode 100644 index 000000000..8d8b0f7b9 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_app.erl @@ -0,0 +1,16 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +start(_Type, _Args) -> + {ok, Sup} = emqx_s3_sup:start_link(), + {ok, Sup}. + +stop(_State) -> + ok. diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl new file mode 100644 index 000000000..01d677922 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -0,0 +1,293 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_client). + +-include_lib("emqx/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("erlcloud/include/erlcloud_aws.hrl"). + +-compile(nowarn_export_all). +-compile(export_all). + +-export([ + create/1, + + put_object/3, + + start_multipart/2, + upload_part/5, + complete_multipart/4, + abort_multipart/3, + list/2, + + format/1 +]). + +-export_type([client/0]). + +-type s3_bucket_acl() :: + private + | public_read + | public_read_write + | authenticated_read + | bucket_owner_read + | bucket_owner_full_control. + +-type headers() :: #{binary() => binary()}. + +-type key() :: string(). +-type part_number() :: non_neg_integer(). +-type upload_id() :: string(). +-type etag() :: string(). + +-type upload_options() :: list({acl, s3_bucket_acl()}). + +-opaque client() :: #{ + aws_config := aws_config(), + options := upload_options(), + bucket := string(), + headers := headers() +}. + +-type config() :: #{ + scheme := string(), + host := string(), + port := part_number(), + bucket := string(), + headers := headers(), + acl := s3_bucket_acl(), + access_key_id := string() | undefined, + secret_access_key := string() | undefined, + http_pool := ecpool:pool_name(), + request_timeout := timeout() +}. + +-type s3_options() :: list({string(), string()}). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec create(config()) -> client(). +create(Config) -> + #{ + aws_config => aws_config(Config), + upload_options => upload_options(Config), + bucket => maps:get(bucket, Config), + headers => headers(Config) + }. + +-spec put_object(client(), key(), iodata()) -> ok_or_error(term()). +put_object( + #{bucket := Bucket, upload_options := Options, headers := Headers, aws_config := AwsConfig}, + Key, + Value +) -> + try erlcloud_s3:put_object(Bucket, Key, Value, Options, Headers, AwsConfig) of + Props when is_list(Props) -> + ok + catch + error:{aws_error, Reason} -> + ?SLOG(debug, #{msg => "put_object_fail", key => Key, reason => Reason}), + {error, Reason} + end. + +-spec start_multipart(client(), key()) -> ok_or_error(upload_id(), term()). +start_multipart( + #{bucket := Bucket, upload_options := Options, headers := Headers, aws_config := AwsConfig}, + Key +) -> + case erlcloud_s3:start_multipart(Bucket, Key, Options, Headers, AwsConfig) of + {ok, Props} -> + {ok, proplists:get_value(uploadId, Props)}; + {error, Reason} -> + ?SLOG(debug, #{msg => "start_multipart_fail", key => Key, reason => Reason}), + {error, Reason} + end. + +-spec upload_part(client(), key(), upload_id(), part_number(), iodata()) -> + ok_or_error(etag(), term()). +upload_part( + #{bucket := Bucket, headers := Headers, aws_config := AwsConfig}, + Key, + UploadId, + PartNumber, + Value +) -> + case erlcloud_s3:upload_part(Bucket, Key, UploadId, PartNumber, Value, Headers, AwsConfig) of + {ok, Props} -> + {ok, proplists:get_value(etag, Props)}; + {error, Reason} -> + ?SLOG(debug, #{msg => "upload_part_fail", key => Key, reason => Reason}), + {error, Reason} + end. + +-spec complete_multipart(client(), key(), upload_id(), [etag()]) -> ok_or_error(term()). +complete_multipart( + #{bucket := Bucket, headers := Headers, aws_config := AwsConfig}, Key, UploadId, ETags +) -> + case erlcloud_s3:complete_multipart(Bucket, Key, UploadId, ETags, Headers, AwsConfig) of + ok -> + ok; + {error, Reason} -> + ?SLOG(debug, #{msg => "complete_multipart_fail", key => Key, reason => Reason}), + {error, Reason} + end. + +-spec abort_multipart(client(), key(), upload_id()) -> ok_or_error(term()). +abort_multipart(#{bucket := Bucket, headers := Headers, aws_config := AwsConfig}, Key, UploadId) -> + case erlcloud_s3:abort_multipart(Bucket, Key, UploadId, [], Headers, AwsConfig) of + ok -> + ok; + {error, Reason} -> + ?SLOG(debug, #{msg => "abort_multipart_fail", key => Key, reason => Reason}), + {error, Reason} + end. + +-spec list(client(), s3_options()) -> ok_or_error(term()). +list(#{bucket := Bucket, aws_config := AwsConfig}, Options) -> + try + {ok, erlcloud_s3:list_objects(Bucket, Options, AwsConfig)} + catch + error:{aws_error, Reason} -> + ?SLOG(debug, #{msg => "list_objects_fail", bucket => Bucket, reason => Reason}), + {error, Reason} + end. + +-spec format(client()) -> term(). +format(#{aws_config := AwsConfig} = Client) -> + Client#{aws_config => AwsConfig#aws_config{secret_access_key = "***"}}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +upload_options(Config) -> + [ + {acl, maps:get(acl, Config)} + ]. + +headers(#{headers := Headers}) -> + maps:to_list(Headers). + +aws_config(#{ + scheme := Scheme, + host := Host, + port := Port, + headers := Headers, + access_key_id := AccessKeyId, + secret_access_key := SecretAccessKey, + http_pool := HttpPool, + request_timeout := Timeout +}) -> + #aws_config{ + s3_scheme = Scheme, + s3_host = Host, + s3_port = Port, + s3_bucket_access_method = path, + + access_key_id = AccessKeyId, + secret_access_key = SecretAccessKey, + + http_client = request_fun(Headers, HttpPool), + timeout = Timeout + }. + +-type http_headers() :: [{binary(), binary()}]. +-type http_pool() :: term(). + +-spec request_fun(http_headers(), http_pool()) -> erlcloud_httpc:request_fun(). +request_fun(CustomHeaders, HttpPool) -> + fun(Url, Method, Headers, Body, Timeout, _Config) -> + with_path_and_query_only(Url, fun(PathQuery) -> + JoinedHeaders = join_headers(Headers, CustomHeaders), + Request = make_request(Method, PathQuery, JoinedHeaders, Body), + ehttpc_request(HttpPool, Method, Request, Timeout) + end) + end. + +ehttpc_request(HttpPool, Method, Request, Timeout) -> + try ehttpc:request(HttpPool, Method, Request, Timeout) of + {ok, StatusCode, RespHeaders} -> + {ok, {{StatusCode, undefined}, string_headers(RespHeaders), undefined}}; + {ok, StatusCode, RespHeaders, RespBody} -> + {ok, {{StatusCode, undefined}, string_headers(RespHeaders), RespBody}}; + {error, Reason} -> + ?SLOG(error, #{ + msg => "s3_ehttpc_request_fail", + reason => Reason, + timeout => Timeout, + pool => HttpPool, + method => Method + }), + {error, Reason} + catch + error:badarg -> + ?SLOG(error, #{ + msg => "s3_ehttpc_request_fail", + reason => badarg, + timeout => Timeout, + pool => HttpPool, + method => Method + }), + {error, no_ehttpc_pool}; + error:Reason -> + ?SLOG(error, #{ + msg => "s3_ehttpc_request_fail", + reason => Reason, + timeout => Timeout, + pool => HttpPool, + method => Method + }), + {error, Reason} + end. + +-define(IS_BODY_EMPTY(Body), (Body =:= undefined orelse Body =:= <<>>)). +-define(NEEDS_BODY(Method), (Method =:= get orelse Method =:= head orelse Method =:= delete)). + +make_request(Method, PathQuery, Headers, Body) when + ?IS_BODY_EMPTY(Body) andalso ?NEEDS_BODY(Method) +-> + {PathQuery, Headers}; +make_request(_Method, PathQuery, Headers, Body) when ?IS_BODY_EMPTY(Body) -> + {PathQuery, [{<<"content-length">>, <<"0">>} | Headers], <<>>}; +make_request(_Method, PathQuery, Headers, Body) -> + {PathQuery, Headers, Body}. + +format_request({PathQuery, Headers, _Body}) -> {PathQuery, Headers, <<"...">>}. + +join_headers(Headers, CustomHeaders) -> + MapHeaders = lists:foldl( + fun({K, V}, MHeaders) -> + maps:put(to_binary(K), V, MHeaders) + end, + #{}, + Headers ++ maps:to_list(CustomHeaders) + ), + maps:to_list(MapHeaders). + +with_path_and_query_only(Url, Fun) -> + case string:split(Url, "//", leading) of + [_Scheme, UrlRem] -> + case string:split(UrlRem, "/", leading) of + [_HostPort, PathQuery] -> + Fun([$/ | PathQuery]); + _ -> + {error, {invalid_url, Url}} + end; + _ -> + {error, {invalid_url, Url}} + end. + +to_binary(Val) when is_list(Val) -> list_to_binary(Val); +to_binary(Val) when is_binary(Val) -> Val. + +string_headers(Hdrs) -> + [{string:to_lower(to_list_string(K)), to_list_string(V)} || {K, V} <- Hdrs]. + +to_list_string(Val) when is_binary(Val) -> + binary_to_list(Val); +to_list_string(Val) when is_list(Val) -> + Val. diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl new file mode 100644 index 000000000..09e945edc --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -0,0 +1,390 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_conf). + +-behaviour(gen_server). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-export([ + start_link/2, + child_spec/2 +]). + +-export([ + checkout_config/1, + checkout_config/2, + checkin_config/1, + checkin_config/2, + + update_config/2, + update_config/3 +]). + +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). + +%% For test purposes +-export([ + client_config/2, + start_http_pool/2, + id/1 +]). + +-define(DEFAULT_CALL_TIMEOUT, 5000). + +-define(DEFAULT_HTTP_POOL_TIMEOUT, 60000). +-define(DEAFULT_HTTP_POOL_CLEANUP_INTERVAL, 60000). + +-spec child_spec(emqx_s3:profile_id(), emqx_s3:profile_config()) -> supervisor:child_spec(). +child_spec(ProfileId, ProfileConfig) -> + #{ + id => ProfileId, + start => {?MODULE, start_link, [ProfileId, ProfileConfig]}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [?MODULE] + }. + +-spec start_link(emqx_s3:profile_id(), emqx_s3:profile_config()) -> gen_server:start_ret(). +start_link(ProfileId, ProfileConfig) -> + gen_server:start_link(?MODULE, [ProfileId, ProfileConfig], []). + +-spec update_config(emqx_s3:profile_id(), emqx_s3:profile_config()) -> ok_or_error(term()). +update_config(ProfileId, ProfileConfig) -> + update_config(ProfileId, ProfileConfig, ?DEFAULT_CALL_TIMEOUT). + +-spec update_config(emqx_s3:profile_id(), emqx_s3:profile_config(), timeout()) -> + ok_or_error(term()). +update_config(ProfileId, ProfileConfig, Timeout) -> + case gproc:where({n, l, id(ProfileId)}) of + undefined -> + {error, profile_not_found}; + Pid -> + gen_server:call(Pid, {update_config, ProfileConfig}, Timeout) + end. + +-spec checkout_config(emqx_s3:profile_id()) -> + {ok, emqx_s3_client:config(), emqx_s3_uploader:config()} | {error, profile_not_found}. +checkout_config(ProfileId) -> + checkout_config(ProfileId, ?DEFAULT_CALL_TIMEOUT). + +-spec checkout_config(emqx_s3:profile_id(), timeout()) -> + {ok, emqx_s3_client:config(), emqx_s3_uploader:config()} | {error, profile_not_found}. +checkout_config(ProfileId, Timeout) -> + case gproc:where({n, l, id(ProfileId)}) of + undefined -> + {error, profile_not_found}; + Pid -> + gen_server:call(Pid, {checkout_config, self()}, Timeout) + end. + +-spec checkin_config(emqx_s3:profile_id()) -> ok | {error, profile_not_found}. +checkin_config(ProfileId) -> + checkin_config(ProfileId, ?DEFAULT_CALL_TIMEOUT). + +-spec checkin_config(emqx_s3:profile_id(), timeout()) -> ok | {error, profile_not_found}. +checkin_config(ProfileId, Timeout) -> + case gproc:where({n, l, id(ProfileId)}) of + undefined -> + {error, profile_not_found}; + Pid -> + gen_server:call(Pid, {checkin_config, self()}, Timeout) + end. + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([ProfileId, ProfileConfig]) -> + _ = process_flag(trap_exit, true), + ok = cleanup_orphaned_pools(ProfileId), + case start_http_pool(ProfileId, ProfileConfig) of + {ok, PoolName} -> + true = gproc:reg({n, l, id(ProfileId)}, ignored), + HttpPoolCleanupInterval = http_pool_cleanup_interval(ProfileConfig), + {ok, #{ + profile_id => ProfileId, + profile_config => ProfileConfig, + client_config => client_config(ProfileConfig, PoolName), + uploader_config => uploader_config(ProfileConfig), + pool_name => PoolName, + pool_clients => emqx_s3_profile_http_pool_clients:create_table(), + %% We don't expose these options to users currently, but use in tests + http_pool_timeout => http_pool_timeout(ProfileConfig), + http_pool_cleanup_interval => HttpPoolCleanupInterval, + + outdated_pool_cleanup_tref => erlang:send_after( + HttpPoolCleanupInterval, self(), cleanup_outdated + ) + }}; + {error, Reason} -> + {stop, Reason} + end. + +handle_call( + {checkout_config, Pid}, + _From, + #{ + client_config := ClientConfig, + uploader_config := UploaderConfig + } = State +) -> + ok = register_client(Pid, State), + {reply, {ok, ClientConfig, UploaderConfig}, State}; +handle_call({checkin_config, Pid}, _From, State) -> + ok = unregister_client(Pid, State), + {reply, ok, State}; +handle_call( + {update_config, NewProfileConfig}, + _From, + #{profile_id := ProfileId} = State +) -> + case update_http_pool(ProfileId, NewProfileConfig, State) of + {ok, PoolName} -> + NewState = State#{ + profile_config => NewProfileConfig, + client_config => client_config(NewProfileConfig, PoolName), + uploader_config => uploader_config(NewProfileConfig), + http_pool_timeout => http_pool_timeout(NewProfileConfig), + http_pool_cleanup_interval => http_pool_cleanup_interval(NewProfileConfig), + pool_name => PoolName + }, + {reply, ok, NewState}; + {error, Reason} -> + {reply, {error, Reason}, State} + end; +handle_call(_Request, _From, State) -> + {reply, {error, not_implemented}, State}. + +handle_cast(_Request, State) -> + {noreply, State}. + +handle_info({'DOWN', _Ref, process, Pid, _Reason}, State) -> + ok = unregister_client(Pid, State), + {noreply, State}; +handle_info(cleanup_outdated, #{http_pool_cleanup_interval := HttpPoolCleanupInterval} = State0) -> + %% Maybe cleanup asynchoronously + ok = cleanup_outdated_pools(State0), + State1 = State0#{ + outdated_pool_cleanup_tref => erlang:send_after( + HttpPoolCleanupInterval, self(), cleanup_outdated + ) + }, + {noreply, State1}; +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, #{profile_id := ProfileId}) -> + lists:foreach( + fun(PoolName) -> + ok = stop_http_pool(ProfileId, PoolName) + end, + emqx_s3_profile_http_pools:all(ProfileId) + ). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +id(ProfileId) -> + {?MODULE, ProfileId}. + +client_config(ProfileConfig, PoolName) -> + HTTPOpts = maps:get(transport_options, ProfileConfig, #{}), + #{ + scheme => scheme(HTTPOpts), + host => maps:get(host, ProfileConfig), + port => maps:get(port, ProfileConfig), + headers => maps:get(headers, HTTPOpts, #{}), + acl => maps:get(acl, ProfileConfig), + bucket => maps:get(bucket, ProfileConfig), + access_key_id => maps:get(access_key_id, ProfileConfig, undefined), + secret_access_key => maps:get(secret_access_key, ProfileConfig, undefined), + request_timeout => maps:get(request_timeout, HTTPOpts, undefined), + http_pool => PoolName + }. + +uploader_config(#{max_part_size := MaxPartSize, min_part_size := MinPartSize} = _ProfileConfig) -> + #{ + min_part_size => MinPartSize, + max_part_size => MaxPartSize + }. + +scheme(#{ssl := #{enable := true}}) -> "https://"; +scheme(_TransportOpts) -> "http://". + +start_http_pool(ProfileId, ProfileConfig) -> + HttpConfig = http_config(ProfileConfig), + PoolName = pool_name(ProfileId), + case do_start_http_pool(PoolName, HttpConfig) of + ok -> + ok = emqx_s3_profile_http_pools:register(ProfileId, PoolName), + ok = ?tp(debug, "s3_start_http_pool", #{pool_name => PoolName, profile_id => ProfileId}), + {ok, PoolName}; + {error, _} = Error -> + Error + end. + +update_http_pool(ProfileId, ProfileConfig, #{pool_name := OldPoolName} = State) -> + HttpConfig = http_config(ProfileConfig), + OldHttpConfig = old_http_config(State), + case OldHttpConfig =:= HttpConfig of + true -> + {ok, OldPoolName}; + false -> + PoolName = pool_name(ProfileId), + case do_start_http_pool(PoolName, HttpConfig) of + ok -> + ok = set_old_pool_outdated(State), + ok = emqx_s3_profile_http_pools:register(ProfileId, PoolName), + {ok, PoolName}; + {error, _} = Error -> + Error + end + end. + +pool_name(ProfileId) -> + iolist_to_binary([ + <<"s3-http-">>, + ProfileId, + <<"-">>, + integer_to_binary(erlang:system_time(millisecond)), + <<"-">>, + integer_to_binary(erlang:unique_integer([positive])) + ]). + +old_http_config(#{profile_config := ProfileConfig}) -> http_config(ProfileConfig). + +set_old_pool_outdated(#{ + profile_id := ProfileId, pool_name := PoolName, http_pool_timeout := HttpPoolTimeout +}) -> + _ = emqx_s3_profile_http_pools:set_outdated(ProfileId, PoolName, HttpPoolTimeout), + ok. + +cleanup_orphaned_pools(ProfileId) -> + lists:foreach( + fun(PoolName) -> + ok = stop_http_pool(ProfileId, PoolName) + end, + emqx_s3_profile_http_pools:all(ProfileId) + ). + +register_client(Pid, #{profile_id := ProfileId, pool_clients := PoolClients, pool_name := PoolName}) -> + MRef = monitor(process, Pid), + ok = emqx_s3_profile_http_pool_clients:register(PoolClients, Pid, MRef, PoolName), + _ = emqx_s3_profile_http_pools:register_client(ProfileId, PoolName), + ok. + +unregister_client( + Pid, + #{ + profile_id := ProfileId, pool_clients := PoolClients, pool_name := PoolName + } +) -> + case emqx_s3_profile_http_pool_clients:unregister(PoolClients, Pid) of + undefined -> + ok; + {MRef, PoolName} -> + true = erlang:demonitor(MRef, [flush]), + _ = emqx_s3_profile_http_pools:unregister_client(ProfileId, PoolName), + ok; + {MRef, OutdatedPoolName} -> + true = erlang:demonitor(MRef, [flush]), + ClientNum = emqx_s3_profile_http_pools:unregister_client(ProfileId, OutdatedPoolName), + maybe_stop_outdated_pool(ProfileId, OutdatedPoolName, ClientNum) + end. + +maybe_stop_outdated_pool(ProfileId, OutdatedPoolName, 0) -> + ok = stop_http_pool(ProfileId, OutdatedPoolName); +maybe_stop_outdated_pool(_ProfileId, _OutdatedPoolName, _ClientNum) -> + ok. + +cleanup_outdated_pools(#{profile_id := ProfileId}) -> + lists:foreach( + fun(PoolName) -> + ok = stop_http_pool(ProfileId, PoolName) + end, + emqx_s3_profile_http_pools:outdated(ProfileId) + ). + +%%-------------------------------------------------------------------- +%% HTTP Pool implementation dependent functions +%%-------------------------------------------------------------------- + +http_config( + #{ + host := Host, + port := Port, + transport_options := #{ + pool_type := PoolType, + pool_size := PoolSize, + enable_pipelining := EnablePipelining, + connect_timeout := ConnectTimeout + } = HTTPOpts + } +) -> + {Transport, TransportOpts} = + case scheme(HTTPOpts) of + "http://" -> + {tcp, []}; + "https://" -> + SSLOpts = emqx_tls_lib:to_client_opts(maps:get(ssl, HTTPOpts)), + {tls, SSLOpts} + end, + NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), + [ + {host, Host}, + {port, Port}, + {connect_timeout, ConnectTimeout}, + {keepalive, 30000}, + {pool_type, PoolType}, + {pool_size, PoolSize}, + {transport, Transport}, + {transport_opts, NTransportOpts}, + {enable_pipelining, EnablePipelining} + ]. + +http_pool_cleanup_interval(ProfileConfig) -> + maps:get( + http_pool_cleanup_interval, ProfileConfig, ?DEAFULT_HTTP_POOL_CLEANUP_INTERVAL + ). + +http_pool_timeout(ProfileConfig) -> + maps:get( + http_pool_timeout, ProfileConfig, ?DEFAULT_HTTP_POOL_TIMEOUT + ). + +stop_http_pool(ProfileId, PoolName) -> + case ehttpc_sup:stop_pool(PoolName) of + ok -> + ok; + {error, Reason} -> + ?SLOG(error, #{msg => "ehttpc_pool_stop_fail", pool_name => PoolName, reason => Reason}), + ok + end, + ok = emqx_s3_profile_http_pools:unregister(ProfileId, PoolName), + ok = ?tp(debug, "s3_stop_http_pool", #{pool_name => PoolName}). + +do_start_http_pool(PoolName, HttpConfig) -> + case ehttpc_sup:start_pool(PoolName, HttpConfig) of + {ok, _} -> + ok; + {error, _} = Error -> + Error + end. diff --git a/apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl b/apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl new file mode 100644 index 000000000..b4e640f7c --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl @@ -0,0 +1,35 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_http_pool_clients). + +-export([ + create_table/0, + + register/4, + unregister/2 +]). + +-define(TAB, ?MODULE). + +-spec create_table() -> ok. +create_table() -> + ets:new(?TAB, [ + private, + set + ]). + +-spec register(ets:tid(), pid(), reference(), emqx_s3_profile_http_pools:pool_name()) -> true. +register(Tab, Pid, MRef, PoolName) -> + true = ets:insert(Tab, {Pid, {MRef, PoolName}}), + ok. + +-spec unregister(ets:tid(), pid()) -> emqx_s3_profile_http_pools:pool_name() | undefined. +unregister(Tab, Pid) -> + case ets:take(Tab, Pid) of + [{Pid, {MRef, PoolName}}] -> + {MRef, PoolName}; + [] -> + undefined + end. diff --git a/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl b/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl new file mode 100644 index 000000000..e1b36c3be --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl @@ -0,0 +1,123 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_http_pools). + +-include_lib("stdlib/include/ms_transform.hrl"). + +-export([ + create_table/0, + + register/2, + unregister/2, + + register_client/2, + unregister_client/2, + + set_outdated/3, + + outdated/1, + all/1 +]). + +-export_type([pool_name/0]). + +-define(TAB, ?MODULE). + +-type pool_name() :: ecpool:pool_name(). + +-type pool_key() :: {emqx_s3:profile_id(), pool_name()}. + +-record(pool, { + key :: pool_key(), + client_count = 0 :: integer(), + deadline = undefined :: undefined | integer(), + extra = #{} :: map() +}). + +-spec create_table() -> ok. +create_table() -> + _ = ets:new(?TAB, [ + named_table, + public, + ordered_set, + {keypos, #pool.key}, + {read_concurrency, true}, + {write_concurrency, true} + ]), + ok. + +-spec register(emqx_s3:profile_id(), pool_name()) -> + ok. +register(ProfileId, PoolName) -> + Key = key(ProfileId, PoolName), + true = ets:insert(?TAB, #pool{ + key = Key, + client_count = 0, + deadline = undefined, + extra = #{} + }), + ok. + +-spec unregister(emqx_s3:profile_id(), pool_name()) -> + ok. +unregister(ProfileId, PoolName) -> + Key = key(ProfileId, PoolName), + true = ets:delete(?TAB, Key), + ok. + +-spec register_client(emqx_s3:profile_id(), pool_name()) -> + integer(). +register_client(ProfileId, PoolName) -> + Key = key(ProfileId, PoolName), + ets:update_counter(?TAB, Key, {#pool.client_count, 1}). + +-spec unregister_client(emqx_s3:profile_id(), pool_name()) -> + integer(). +unregister_client(ProfileId, PoolName) -> + Key = key(ProfileId, PoolName), + try + ets:update_counter(?TAB, Key, {#pool.client_count, -1}) + catch + error:badarg -> + undefined + end. + +-spec set_outdated(emqx_s3:profile_id(), pool_name(), integer()) -> + ok. +set_outdated(ProfileId, PoolName, Timeout) -> + Key = key(ProfileId, PoolName), + Now = erlang:monotonic_time(millisecond), + ets:update_element(?TAB, Key, {#pool.deadline, Now + Timeout}). + +-spec outdated(emqx_s3:profile_id()) -> + [pool_name()]. +outdated(ProfileId) -> + Now = erlang:monotonic_time(millisecond), + MS = ets:fun2ms( + fun(#pool{key = {ProfileId_, PoolName}, deadline = Deadline_}) when + ProfileId_ =:= ProfileId andalso + Deadline_ =/= undefined andalso Deadline_ < Now + -> + PoolName + end + ), + ets:select(?TAB, MS). + +-spec all(emqx_s3:profile_id()) -> + [pool_name()]. +all(ProfileId) -> + MS = ets:fun2ms( + fun(#pool{key = {ProfileId_, PoolName}}) when ProfileId_ =:= ProfileId -> + PoolName + end + ), + ets:select(?TAB, MS). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +key(ProfileId, PoolName) -> + {ProfileId, PoolName}. diff --git a/apps/emqx_s3/src/emqx_s3_profile_sup.erl b/apps/emqx_s3/src/emqx_s3_profile_sup.erl new file mode 100644 index 000000000..c39fc9f4b --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_profile_sup.erl @@ -0,0 +1,48 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_sup). + +-behaviour(supervisor). + +-include_lib("emqx/include/types.hrl"). + +-export([ + start_link/2, + child_spec/2 +]). + +-export([init/1]). + +-spec start_link(emqx_s3:profile_id(), emqx_s3:profile_config()) -> supervisor:start_ret(). +start_link(ProfileId, ProfileConfig) -> + supervisor:start_link(?MODULE, [ProfileId, ProfileConfig]). + +-spec child_spec(emqx_s3:profile_id(), emqx_s3:profile_config()) -> supervisor:child_spec(). +child_spec(ProfileId, ProfileConfig) -> + #{ + id => ProfileId, + start => {?MODULE, start_link, [ProfileId, ProfileConfig]}, + restart => permanent, + shutdown => 5000, + type => supervisor, + modules => [?MODULE] + }. + +%%-------------------------------------------------------------------- +%% supervisor callbacks +%%------------------------------------------------------------------- + +init([ProfileId, ProfileConfig]) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 5 + }, + ChildSpecs = [ + %% Order matters + emqx_s3_profile_conf:child_spec(ProfileId, ProfileConfig), + emqx_s3_profile_uploader_sup:child_spec(ProfileId) + ], + {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl b/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl new file mode 100644 index 000000000..1cd155a77 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl @@ -0,0 +1,73 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_uploader_sup). + +-behaviour(supervisor). + +-include_lib("emqx/include/types.hrl"). + +-export([ + start_link/1, + child_spec/1, + id/1, + start_uploader/2 +]). + +-export([init/1]). + +-export_type([id/0]). + +-type id() :: {?MODULE, emqx_s3:profile_id()}. + +-spec start_link(emqx_s3:profile_id()) -> supervisor:start_ret(). +start_link(ProfileId) -> + supervisor:start_link(?MODULE, [ProfileId]). + +-spec child_spec(emqx_s3:profile_id()) -> supervisor:child_spec(). +child_spec(ProfileId) -> + #{ + id => id(ProfileId), + start => {?MODULE, start_link, [ProfileId]}, + restart => permanent, + shutdown => 5000, + type => supervisor, + modules => [?MODULE] + }. + +-spec id(emqx_s3:profile_id()) -> id(). +id(ProfileId) -> + {?MODULE, ProfileId}. + +-spec start_uploader(emqx_s3:profile_id(), emqx_s3_uploader:opts()) -> + supervisor:start_ret() | {error, profile_not_found}. +start_uploader(ProfileId, Opts) -> + Id = id(ProfileId), + case gproc:where({n, l, Id}) of + undefined -> {error, profile_not_found}; + Pid -> supervisor:start_child(Pid, [Opts]) + end. + +%%-------------------------------------------------------------------- +%% supervisor callbacks +%%------------------------------------------------------------------- + +init([ProfileId]) -> + true = gproc:reg({n, l, id(ProfileId)}, ignored), + SupFlags = #{ + strategy => simple_one_for_one, + intensity => 10, + period => 5 + }, + ChildSpecs = [ + #{ + id => emqx_s3_uploader, + start => {emqx_s3_uploader, start_link, [ProfileId]}, + restart => temporary, + shutdown => 5000, + type => worker, + modules => [emqx_s3_uploader] + } + ], + {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl new file mode 100644 index 000000000..ceb0d1dd4 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -0,0 +1,143 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_schema). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-import(hoconsc, [mk/2, ref/1]). + +-export([roots/0, fields/1, namespace/0, tags/0]). + +-export([translate/1]). + +roots() -> + [s3]. + +namespace() -> "s3". + +tags() -> + [<<"S3">>]. + +fields(s3) -> + [ + {access_key_id, + mk( + string(), + #{ + desc => ?DESC("access_key_id"), + required => false + } + )}, + {secret_access_key, + mk( + string(), + #{ + desc => ?DESC("secret_access_key"), + required => false + } + )}, + {bucket, + mk( + string(), + #{ + desc => ?DESC("bucket"), + required => true + } + )}, + {host, + mk( + string(), + #{ + desc => ?DESC("host"), + required => true + } + )}, + {port, + mk( + pos_integer(), + #{ + desc => ?DESC("port"), + required => true + } + )}, + {min_part_size, + mk( + emqx_schema:bytesize(), + #{ + default => "5mb", + desc => ?DESC("min_part_size"), + required => true, + validator => fun part_size_validator/1 + } + )}, + {max_part_size, + mk( + emqx_schema:bytesize(), + #{ + default => "5gb", + desc => ?DESC("max_part_size"), + required => true, + validator => fun part_size_validator/1 + } + )}, + {acl, + mk( + hoconsc:enum([ + private, + public_read, + public_read_write, + authenticated_read, + bucket_owner_read, + bucket_owner_full_control + ]), + #{ + default => private, + desc => ?DESC("acl"), + required => true + } + )}, + {transport_options, + mk( + ref(transport_options), + #{ + desc => ?DESC("transport_options"), + required => false + } + )} + ]; +fields(transport_options) -> + props_without( + [base_url, max_retries, retry_interval, request], emqx_connector_http:fields(config) + ) ++ + props_with( + [headers, max_retries, request_timeout], emqx_connector_http:fields("request") + ). + +translate(Conf) -> + Options = #{atom_key => true}, + #{s3 := TranslatedConf} = hocon_tconf:check_plain( + emqx_s3_schema, #{<<"s3">> => Conf}, Options, [s3] + ), + TranslatedConf. + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +props_with(Keys, Proplist) -> + lists:filter(fun({K, _}) -> lists:member(K, Keys) end, Proplist). + +props_without(Keys, Proplist) -> + lists:filter(fun({K, _}) -> not lists:member(K, Keys) end, Proplist). + +part_size_validator(PartSizeLimit) -> + case + PartSizeLimit >= 5 * 1024 * 1024 andalso + PartSizeLimit =< 5 * 1024 * 1024 * 1024 + of + true -> ok; + false -> {error, "must be at least 5mb and less than 5gb"} + end. diff --git a/apps/emqx_s3/src/emqx_s3_sup.erl b/apps/emqx_s3/src/emqx_s3_sup.erl new file mode 100644 index 000000000..0f6b0160b --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_sup.erl @@ -0,0 +1,47 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_sup). + +-behaviour(supervisor). + +-include_lib("emqx/include/types.hrl"). + +-export([ + start_link/0, + start_profile/2, + stop_profile/1 +]). + +-export([init/1]). + +-spec start_link() -> supervisor:start_ret(). +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +-spec start_profile(emqx_s3:profile_id(), emqx_s3:profile_config()) -> supervisor:startchild_ret(). +start_profile(ProfileId, ProfileConfig) -> + supervisor:start_child(?MODULE, emqx_s3_profile_sup:child_spec(ProfileId, ProfileConfig)). + +-spec stop_profile(emqx_s3:profile_id()) -> ok_or_error(term()). +stop_profile(ProfileId) -> + case supervisor:terminate_child(?MODULE, ProfileId) of + ok -> + supervisor:delete_child(?MODULE, ProfileId); + {error, Reason} -> + {error, Reason} + end. + +%%-------------------------------------------------------------------- +%% supervisor callbacks +%%------------------------------------------------------------------- + +init([]) -> + ok = emqx_s3_profile_http_pools:create_table(), + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 5 + }, + {ok, {SupFlags, []}}. diff --git a/apps/emqx_s3/src/emqx_s3_uploader.erl b/apps/emqx_s3/src/emqx_s3_uploader.erl new file mode 100644 index 000000000..4e3fe15f2 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_uploader.erl @@ -0,0 +1,318 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_uploader). + +-include_lib("emqx/include/types.hrl"). + +-behaviour(gen_statem). + +-export([ + start_link/2, + + write/2, + complete/1, + abort/1 +]). + +-export([ + init/1, + callback_mode/0, + handle_event/4, + terminate/3, + code_change/4, + format_status/1, + format_status/2 +]). + +-export_type([opts/0, config/0]). + +-type opts() :: #{ + name := string() +}. + +-type config() :: #{ + min_part_size := pos_integer() +}. + +-type data() :: #{ + profile_id := emqx_s3:profile_id(), + client := emqx_s3_client:client(), + key := emqx_s3_client:key(), + buffer := iodata(), + buffer_size := non_neg_integer(), + min_part_size := pos_integer(), + max_part_size := pos_integer(), + upload_id := undefined | emqx_s3_client:upload_id(), + etags := [emqx_s3_client:etag()], + part_number := emqx_s3_client:part_number() +}. + +%% 5MB +-define(DEFAULT_MIN_PART_SIZE, 5242880). +%% 5GB +-define(DEFAULT_MAX_PART_SIZE, 5368709120). + +-spec start_link(emqx_s3:profile_id(), opts()) -> gen_statem:start_ret(). +start_link(ProfileId, #{key := Key} = Opts) when is_list(Key) -> + gen_statem:start_link(?MODULE, [ProfileId, Opts], []). + +-spec write(pid(), binary()) -> ok_or_error(term()). +write(Pid, WriteData) when is_binary(WriteData) -> + write(Pid, WriteData, infinity). + +-spec write(pid(), binary(), timeout()) -> ok_or_error(term()). +write(Pid, WriteData, Timeout) when is_binary(WriteData) -> + gen_statem:call(Pid, {write, wrap(WriteData)}, Timeout). + +-spec complete(pid()) -> ok_or_error(term()). +complete(Pid) -> + complete(Pid, infinity). + +-spec complete(pid(), timeout()) -> ok_or_error(term()). +complete(Pid, Timeout) -> + gen_statem:call(Pid, complete, Timeout). + +-spec abort(pid()) -> ok_or_error(term()). +abort(Pid) -> + abort(Pid, infinity). + +-spec abort(pid(), timeout()) -> ok_or_error(term()). +abort(Pid, Timeout) -> + gen_statem:call(Pid, abort, Timeout). + +%%-------------------------------------------------------------------- +%% gen_statem callbacks +%%-------------------------------------------------------------------- + +callback_mode() -> handle_event_function. + +init([ProfileId, #{key := Key}]) -> + process_flag(trap_exit, true), + {ok, ClientConfig, UploaderConfig} = emqx_s3_profile_conf:checkout_config(ProfileId), + Client = emqx_s3_client:create(ClientConfig), + {ok, upload_not_started, #{ + profile_id => ProfileId, + client => Client, + key => Key, + buffer => [], + buffer_size => 0, + min_part_size => maps:get(min_part_size, UploaderConfig, ?DEFAULT_MIN_PART_SIZE), + max_part_size => maps:get(max_part_size, UploaderConfig, ?DEFAULT_MAX_PART_SIZE), + upload_id => undefined, + etags => [], + part_number => 1 + }}. + +handle_event({call, From}, {write, WriteDataWrapped}, State, Data0) -> + WriteData = unwrap(WriteDataWrapped), + case is_valid_part(WriteData, Data0) of + true -> + handle_write(State, From, WriteData, Data0); + false -> + {keep_state_and_data, {reply, From, {error, {too_large, byte_size(WriteData)}}}} + end; +handle_event({call, From}, complete, upload_not_started, Data0) -> + case put_object(Data0) of + ok -> + {stop_and_reply, normal, {reply, From, ok}}; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data0} + end; +handle_event({call, From}, complete, upload_started, Data0) -> + case complete_upload(Data0) of + {ok, Data1} -> + {stop_and_reply, normal, {reply, From, ok}, Data1}; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data0} + end; +handle_event({call, From}, abort, upload_not_started, _Data) -> + {stop_and_reply, normal, {reply, From, ok}}; +handle_event({call, From}, abort, upload_started, Data0) -> + case abort_upload(Data0) of + ok -> + {stop_and_reply, normal, {reply, From, ok}}; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data0} + end. + +handle_write(upload_not_started, From, WriteData, Data0) -> + Data1 = append_buffer(Data0, WriteData), + case maybe_start_upload(Data1) of + not_started -> + {keep_state, Data1, {reply, From, ok}}; + {started, Data2} -> + case upload_part(Data2) of + {ok, Data3} -> + {next_state, upload_started, Data3, {reply, From, ok}}; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data2} + end; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data1} + end; +handle_write(upload_started, From, WriteData, Data0) -> + Data1 = append_buffer(Data0, WriteData), + case maybe_upload_part(Data1) of + {ok, Data2} -> + {keep_state, Data2, {reply, From, ok}}; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data1} + end. + +terminate(Reason, _State, #{client := Client, upload_id := UploadId, key := Key}) when + (UploadId =/= undefined) andalso (Reason =/= normal) +-> + emqx_s3_client:abort_multipart(Client, Key, UploadId); +terminate(_Reason, _State, _Data) -> + ok. + +code_change(_OldVsn, StateName, State, _Extra) -> + {ok, StateName, State}. + +format_status(#{data := #{client := Client} = Data} = Status) -> + Status#{ + data => Data#{ + client => emqx_s3_client:format(Client), + buffer => [<<"...">>] + } + }. + +format_status(_Opt, [PDict, State, #{client := Client} = Data]) -> + #{ + data => Data#{ + client => emqx_s3_client:format(Client), + buffer => [<<"...">>] + }, + state => State, + pdict => PDict + }. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +-spec maybe_start_upload(data()) -> not_started | {started, data()} | {error, term()}. +maybe_start_upload(#{buffer_size := BufferSize, min_part_size := MinPartSize} = Data) -> + case BufferSize >= MinPartSize of + true -> + start_upload(Data); + false -> + not_started + end. + +-spec start_upload(data()) -> {started, data()} | {error, term()}. +start_upload(#{client := Client, key := Key} = Data) -> + case emqx_s3_client:start_multipart(Client, Key) of + {ok, UploadId} -> + NewData = Data#{upload_id => UploadId}, + {started, NewData}; + {error, _} = Error -> + Error + end. + +-spec maybe_upload_part(data()) -> ok_or_error(data(), term()). +maybe_upload_part(#{buffer_size := BufferSize, min_part_size := MinPartSize} = Data) -> + case BufferSize >= MinPartSize of + true -> + upload_part(Data); + false -> + % ct:print("buffer size: ~p, max part size: ~p, no upload", [BufferSize, MinPartSize]), + {ok, Data} + end. + +-spec upload_part(data()) -> ok_or_error(data(), term()). +upload_part(#{buffer_size := 0} = Data) -> + {ok, Data}; +upload_part( + #{ + client := Client, + key := Key, + upload_id := UploadId, + buffer := Buffer, + part_number := PartNumber, + etags := ETags + } = Data +) -> + case emqx_s3_client:upload_part(Client, Key, UploadId, PartNumber, lists:reverse(Buffer)) of + {ok, ETag} -> + % ct:print("upload part ~p, etag: ~p", [PartNumber, ETag]), + NewData = Data#{ + buffer => [], + buffer_size => 0, + part_number => PartNumber + 1, + etags => [{PartNumber, ETag} | ETags] + }, + {ok, NewData}; + {error, _} = Error -> + % ct:print("upload part ~p failed: ~p", [PartNumber, Error]), + Error + end. + +-spec complete_upload(data()) -> ok_or_error(term()). +complete_upload( + #{ + client := Client, + key := Key, + upload_id := UploadId + } = Data0 +) -> + case upload_part(Data0) of + {ok, #{etags := ETags} = Data1} -> + case emqx_s3_client:complete_multipart(Client, Key, UploadId, lists:reverse(ETags)) of + ok -> + {ok, Data1}; + {error, _} = Error -> + Error + end; + {error, _} = Error -> + Error + end. + +-spec abort_upload(data()) -> ok_or_error(term()). +abort_upload( + #{ + client := Client, + key := Key, + upload_id := UploadId + } +) -> + case emqx_s3_client:abort_multipart(Client, Key, UploadId) of + ok -> + ok; + {error, _} = Error -> + Error + end. + +-spec put_object(data()) -> ok_or_error(term()). +put_object( + #{ + client := Client, + key := Key, + buffer := Buffer + } +) -> + case emqx_s3_client:put_object(Client, Key, lists:reverse(Buffer)) of + ok -> + ok; + {error, _} = Error -> + Error + end. + +-spec append_buffer(data(), binary()) -> data(). +append_buffer(#{buffer := Buffer, buffer_size := BufferSize} = Data, WriteData) -> + Data#{ + buffer => [WriteData | Buffer], + buffer_size => BufferSize + byte_size(WriteData) + }. + +-compile({inline, [wrap/1, unwrap/1]}). +wrap(Data) -> + fun() -> Data end. + +unwrap(WrappedData) -> + WrappedData(). + +is_valid_part(WriteData, #{max_part_size := MaxPartSize, buffer_size := BufferSize}) -> + BufferSize + byte_size(WriteData) =< MaxPartSize. diff --git a/apps/emqx_s3/test/certs/ca.crt b/apps/emqx_s3/test/certs/ca.crt new file mode 100644 index 000000000..8a9dafccd --- /dev/null +++ b/apps/emqx_s3/test/certs/ca.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE5DCCAswCCQCF3o0gIdaNDjANBgkqhkiG9w0BAQsFADA0MRIwEAYDVQQKDAlF +TVFYIFRlc3QxHjAcBgNVBAMMFUNlcnRpZmljYXRlIEF1dGhvcml0eTAeFw0yMTEy +MzAwODQxMTFaFw00OTA1MTcwODQxMTFaMDQxEjAQBgNVBAoMCUVNUVggVGVzdDEe +MBwGA1UEAwwVQ2VydGlmaWNhdGUgQXV0aG9yaXR5MIICIjANBgkqhkiG9w0BAQEF +AAOCAg8AMIICCgKCAgEAqmqSrxyH16j63QhqGLT1UO8I+m6BM3HfnJQM8laQdtJ0 +WgHqCh0/OphH3S7v4SfF4fNJDEJWMWuuzJzU9cTqHPLzhvo3+ZHcMIENgtY2p2Cf +7AQjEqFViEDyv2ZWNEe76BJeShntdY5NZr4gIPar99YGG/Ln8YekspleV+DU38rE +EX9WzhgBr02NN9z4NzIxeB+jdvPnxcXs3WpUxzfnUjOQf/T1tManvSdRbFmKMbxl +A8NLYK3oAYm8EbljWUINUNN6loqYhbigKv8bvo5S4xvRqmX86XB7sc0SApngtNcg +O0EKn8z/KVPDskE+8lMfGMiU2e2Tzw6Rph57mQPOPtIp5hPiKRik7ST9n0p6piXW +zRLplJEzSjf40I1u+VHmpXlWI/Fs8b1UkDSMiMVJf0LyWb4ziBSZOY2LtZzWHbWj +LbNgxQcwSS29tKgUwfEFmFcm+iOM59cPfkl2IgqVLh5h4zmKJJbfQKSaYb5fcKRf +50b1qsN40VbR3Pk/0lJ0/WqgF6kZCExmT1qzD5HJES/5grjjKA4zIxmHOVU86xOF +ouWvtilVR4PGkzmkFvwK5yRhBUoGH/A9BurhqOc0QCGay1kqHQFA6se4JJS+9KOS +x8Rn1Nm6Pi7sd6Le3cKmHTlyl5a/ofKqTCX2Qh+v/7y62V1V1wnoh3ipRjdPTnMC +AwEAATANBgkqhkiG9w0BAQsFAAOCAgEARCqaocvlMFUQjtFtepO2vyG1krn11xJ0 +e7md26i+g8SxCCYqQ9IqGmQBg0Im8fyNDKRN/LZoj5+A4U4XkG1yya91ZIrPpWyF +KUiRAItchNj3g1kHmI2ckl1N//6Kpx3DPaS7qXZaN3LTExf6Ph+StE1FnS0wVF+s +tsNIf6EaQ+ZewW3pjdlLeAws3jvWKUkROc408Ngvx74zbbKo/zAC4tz8oH9ZcpsT +WD8enVVEeUQKI6ItcpZ9HgTI9TFWgfZ1vYwvkoRwNIeabYI62JKmLEo2vGfGwWKr +c+GjnJ/tlVI2DpPljfWOnQ037/7yyJI/zo65+HPRmGRD6MuW/BdPDYOvOZUTcQKh +kANi5THSbJJgZcG3jb1NLebaUQ1H0zgVjn0g3KhUV+NJQYk8RQ7rHtB+MySqTKlM +kRkRjfTfR0Ykxpks7Mjvsb6NcZENf08ZFPd45+e/ptsxpiKu4e4W4bV7NZDvNKf9 +0/aD3oGYNMiP7s+KJ1lRSAjnBuG21Yk8FpzG+yr8wvJhV8aFgNQ5wIH86SuUTmN0 +5bVzFEIcUejIwvGoQEctNHBlOwHrb7zmB6OwyZeMapdXBQ+9UDhYg8ehDqdDOdfn +wsBcnjD2MwNhlE1hjL+tZWLNwSHiD6xx3LvNoXZu2HK8Cp3SOrkE69cFghYMIZZb +T+fp6tNL6LE= +-----END CERTIFICATE----- diff --git a/apps/emqx_s3/test/emqx_s3_SUITE.erl b/apps/emqx_s3/test/emqx_s3_SUITE.erl new file mode 100644 index 000000000..287dcb597 --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_SUITE.erl @@ -0,0 +1,66 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(emqx_s3), + Config. + +end_per_suite(_Config) -> + ok = application:stop(emqx_s3). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_start_stop_update(_Config) -> + ProfileId = <<"test">>, + ProfileConfig = profile_config(), + + ?assertMatch( + ok, + emqx_s3:start_profile(ProfileId, ProfileConfig) + ), + + ?assertMatch( + {error, _}, + emqx_s3:start_profile(ProfileId, ProfileConfig) + ), + + ?assertEqual( + ok, + emqx_s3:update_profile(ProfileId, ProfileConfig) + ), + + ?assertMatch( + {error, _}, + emqx_s3:update_profile(<<"unknown">>, ProfileConfig) + ), + + ?assertEqual( + ok, + emqx_s3:stop_profile(ProfileId) + ), + + ?assertMatch( + {error, _}, + emqx_s3:stop_profile(ProfileId) + ). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +profile_config() -> + emqx_s3_test_helpers:base_config(tcp). diff --git a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl new file mode 100644 index 000000000..3d0d7bb18 --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl @@ -0,0 +1,104 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_client_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(PROFILE_ID, atom_to_binary(?MODULE)). + +all() -> + [ + {group, tcp}, + {group, tls} + ]. + +groups() -> + AllCases = emqx_common_test_helpers:all(?MODULE), + [ + {tcp, [], AllCases}, + {tls, [], AllCases} + ]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(emqx_s3), + Config. + +end_per_suite(_Config) -> + ok = application:stop(emqx_s3). + +init_per_group(ConnType, Config) -> + [{conn_type, ConnType} | Config]. +end_per_group(_ConnType, _Config) -> + ok. + +init_per_testcase(_TestCase, Config0) -> + ConnType = ?config(conn_type, Config0), + + Bucket = emqx_s3_test_helpers:unique_bucket(), + TestAwsConfig = emqx_s3_test_helpers:aws_config(ConnType), + ok = erlcloud_s3:create_bucket(Bucket, TestAwsConfig), + Config1 = [ + {key, emqx_s3_test_helpers:unique_key()}, + {bucket, Bucket} + | Config0 + ], + {ok, PoolName} = emqx_s3_profile_conf:start_http_pool(?PROFILE_ID, profile_config(Config1)), + [{ehttpc_pool_name, PoolName} | Config1]. + +end_per_testcase(_TestCase, Config) -> + ok = ehttpc_sup:stop_pool(?config(ehttpc_pool_name, Config)). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_multipart_upload(Config) -> + Key = ?config(key, Config), + + Client = client(Config), + + {ok, UploadId} = emqx_s3_client:start_multipart(Client, Key), + + Data = data(6_000_000), + + {ok, Etag1} = emqx_s3_client:upload_part(Client, Key, UploadId, 1, Data), + {ok, Etag2} = emqx_s3_client:upload_part(Client, Key, UploadId, 2, Data), + + ok = emqx_s3_client:complete_multipart( + Client, Key, UploadId, [{1, Etag1}, {2, Etag2}] + ). + +t_simple_put(Config) -> + Key = ?config(key, Config), + + Client = client(Config), + + Data = data(6_000_000), + + ok = emqx_s3_client:put_object(Client, Key, Data). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +client(Config) -> + ClientConfig = emqx_s3_profile_conf:client_config( + profile_config(Config), ?config(ehttpc_pool_name, Config) + ), + emqx_s3_client:create(ClientConfig). + +profile_config(Config) -> + maps:put( + bucket, + ?config(bucket, Config), + emqx_s3_test_helpers:base_config(?config(conn_type, Config)) + ). + +data(Size) -> + iolist_to_binary([$a || _ <- lists:seq(1, Size)]). diff --git a/apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl b/apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl new file mode 100644 index 000000000..ce53525be --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl @@ -0,0 +1,293 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_conf_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-define(assertWaitEvent(Code, EventMatch, Timeout), + ?assertMatch( + {_, {ok, EventMatch}}, + ?wait_async_action( + Code, + EventMatch, + Timeout + ) + ) +). + +all() -> emqx_common_test_helpers:all(?MODULE). + +suite() -> [{timetrap, {minutes, 1}}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(emqx_s3), + Config. + +end_per_suite(_Config) -> + ok = application:stop(emqx_s3). + +init_per_testcase(_TestCase, Config) -> + ok = snabbkaffe:start_trace(), + TestAwsConfig = emqx_s3_test_helpers:aws_config(tcp), + + Bucket = emqx_s3_test_helpers:unique_bucket(), + ok = erlcloud_s3:create_bucket(Bucket, TestAwsConfig), + + ProfileBaseConfig = emqx_s3_test_helpers:base_config(tcp), + ProfileConfig = ProfileBaseConfig#{bucket => Bucket}, + ok = emqx_s3:start_profile(profile_id(), ProfileConfig), + + [{profile_config, ProfileConfig} | Config]. + +end_per_testcase(_TestCase, _Config) -> + ok = snabbkaffe:stop(), + _ = emqx_s3:stop_profile(profile_id()). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_regular_outdated_pool_cleanup(Config) -> + _ = process_flag(trap_exit, true), + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + [OldPool] = emqx_s3_profile_http_pools:all(profile_id()), + + ProfileBaseConfig = ?config(profile_config, Config), + ProfileConfig = emqx_map_lib:deep_put( + [transport_options, pool_size], ProfileBaseConfig, 16 + ), + ok = emqx_s3:update_profile(profile_id(), ProfileConfig), + + ?assertEqual( + 2, + length(emqx_s3_profile_http_pools:all(profile_id())) + ), + + ?assertWaitEvent( + ok = emqx_s3_uploader:abort(Pid), + #{?snk_kind := "s3_stop_http_pool", pool_name := OldPool}, + 1000 + ), + + [NewPool] = emqx_s3_profile_http_pools:all(profile_id()), + + ?assertWaitEvent( + ok = emqx_s3:stop_profile(profile_id()), + #{?snk_kind := "s3_stop_http_pool", pool_name := NewPool}, + 1000 + ), + + ?assertEqual( + 0, + length(emqx_s3_profile_http_pools:all(profile_id())) + ). + +t_timeout_pool_cleanup(Config) -> + _ = process_flag(trap_exit, true), + + %% We restart the profile to set `http_pool_timeout` value suitable for test + ok = emqx_s3:stop_profile(profile_id()), + ProfileBaseConfig = ?config(profile_config, Config), + ProfileConfig = ProfileBaseConfig#{ + http_pool_timeout => 500, + http_pool_cleanup_interval => 100 + }, + ok = emqx_s3:start_profile(profile_id(), ProfileConfig), + + %% Start uploader + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + ok = emqx_s3_uploader:write(Pid, <<"data">>), + + [OldPool] = emqx_s3_profile_http_pools:all(profile_id()), + + NewProfileConfig = emqx_map_lib:deep_put( + [transport_options, pool_size], ProfileConfig, 16 + ), + + %% We update profile to create new pool and wait for the old one to be stopped by timeout + ?assertWaitEvent( + ok = emqx_s3:update_profile(profile_id(), NewProfileConfig), + #{?snk_kind := "s3_stop_http_pool", pool_name := OldPool}, + 1000 + ), + + %% The uploader now has no valid pool and should fail + ?assertMatch( + {error, _}, + emqx_s3_uploader:complete(Pid) + ). + +t_checkout_no_profile(_Config) -> + ?assertEqual( + {error, profile_not_found}, + emqx_s3_profile_conf:checkout_config(<<"no_such_profile">>) + ). + +t_httpc_pool_start_error(Config) -> + %% `ehhtpc_pool`s are lazy so it is difficult to trigger an error + %% passing some bad connection options. + %% So we emulate some unknown crash with `meck`. + meck:new(ehttpc_pool, [passthrough]), + meck:expect(ehttpc_pool, init, fun(_) -> meck:raise(error, badarg) end), + + ?assertMatch( + {error, _}, + emqx_s3:start_profile(<<"profile">>, ?config(profile_config, Config)) + ). + +t_httpc_pool_update_error(Config) -> + %% `ehhtpc_pool`s are lazy so it is difficult to trigger an error + %% passing some bad connection options. + %% So we emulate some unknown crash with `meck`. + meck:new(ehttpc_pool, [passthrough]), + meck:expect(ehttpc_pool, init, fun(_) -> meck:raise(error, badarg) end), + + ProfileBaseConfig = ?config(profile_config, Config), + NewProfileConfig = emqx_map_lib:deep_put( + [transport_options, pool_size], ProfileBaseConfig, 16 + ), + + ?assertMatch( + {error, _}, + emqx_s3:start_profile(<<"profile">>, NewProfileConfig) + ). + +t_orphaned_pools_cleanup(_Config) -> + ProfileId = profile_id(), + Pid = gproc:where({n, l, emqx_s3_profile_conf:id(ProfileId)}), + + %% We kill conf and wait for it to restart + %% and create a new pool + ?assertWaitEvent( + exit(Pid, kill), + #{?snk_kind := "s3_start_http_pool", profile_id := ProfileId}, + 1000 + ), + + %% We should still have only one pool + ?assertEqual( + 1, + length(emqx_s3_profile_http_pools:all(ProfileId)) + ). + +t_orphaned_pools_cleanup_non_graceful(_Config) -> + ProfileId = profile_id(), + Pid = gproc:where({n, l, emqx_s3_profile_conf:id(ProfileId)}), + + %% We stop pool, conf server should not fail when attempting to stop it once more + [PoolName] = emqx_s3_profile_http_pools:all(ProfileId), + ok = ehttpc_pool:stop_pool(PoolName), + + %% We kill conf and wait for it to restart + %% and create a new pool + ?assertWaitEvent( + exit(Pid, kill), + #{?snk_kind := "s3_start_http_pool", profile_id := ProfileId}, + 1000 + ), + + %% We should still have only one pool + ?assertEqual( + 1, + length(emqx_s3_profile_http_pools:all(ProfileId)) + ). + +t_checkout_client(Config) -> + ProfileId = profile_id(), + Key = emqx_s3_test_helpers:unique_key(), + Caller = self(), + Pid = spawn_link(fun() -> + emqx_s3:with_client( + ProfileId, + fun(Client) -> + receive + put_object -> + Caller ! {put_object, emqx_s3_client:put_object(Client, Key, <<"data">>)} + end, + receive + list_objects -> + Caller ! {list_objects, emqx_s3_client:list(Client, [])} + end + end + ), + Caller ! client_released, + receive + stop -> ok + end + end), + + %% Ask spawned process to put object + Pid ! put_object, + receive + {put_object, ok} -> ok + after 1000 -> + ct:fail("put_object fail") + end, + + %% Now change config for the profile + ProfileBaseConfig = ?config(profile_config, Config), + NewProfileConfig0 = ProfileBaseConfig#{bucket => <<"new_bucket">>}, + NewProfileConfig1 = emqx_map_lib:deep_put( + [transport_options, pool_size], NewProfileConfig0, 16 + ), + ok = emqx_s3:update_profile(profile_id(), NewProfileConfig1), + + %% We should have two pools now, because the old one is still in use + %% by the spawned process + ?assertEqual( + 2, + length(emqx_s3_profile_http_pools:all(ProfileId)) + ), + + %% Ask spawned process to list objects + Pid ! list_objects, + receive + {list_objects, Result} -> + {ok, OkResult} = Result, + Contents = proplists:get_value(contents, OkResult), + ?assertEqual(1, length(Contents)), + ?assertEqual(Key, proplists:get_value(key, hd(Contents))) + after 1000 -> + ct:fail("list_objects fail") + end, + + %% Wait till spawned process releases client + receive + client_released -> ok + after 1000 -> + ct:fail("client not released") + end, + + %% We should have only one pool now, because the old one is released + ?assertEqual( + 1, + length(emqx_s3_profile_http_pools:all(ProfileId)) + ). + +t_unknown_messages(_Config) -> + Pid = gproc:where({n, l, emqx_s3_profile_conf:id(profile_id())}), + + Pid ! unknown, + ok = gen_server:cast(Pid, unknown), + + ?assertEqual( + {error, not_implemented}, + gen_server:call(Pid, unknown) + ). + +%%-------------------------------------------------------------------- +%% Test helpers +%%-------------------------------------------------------------------- + +profile_id() -> + <<"test">>. diff --git a/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl b/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl new file mode 100644 index 000000000..bba1a5ba8 --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl @@ -0,0 +1,154 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_schema_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_minimal_config(_Config) -> + ?assertMatch( + #{ + bucket := "bucket", + host := "s3.us-east-1.endpoint.com", + port := 443, + acl := private, + min_part_size := 5242880, + transport_options := + #{ + connect_timeout := 15000, + enable_pipelining := 100, + pool_size := 8, + pool_type := random, + ssl := #{enable := false} + } + }, + emqx_s3_schema:translate(#{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443 + }) + ). + +t_full_config(_Config) -> + ?assertMatch( + #{ + access_key_id := "access_key_id", + acl := public_read, + bucket := "bucket", + host := "s3.us-east-1.endpoint.com", + min_part_size := 10485760, + port := 443, + secret_access_key := "secret_access_key", + transport_options := + #{ + connect_timeout := 30000, + enable_pipelining := 200, + headers := #{<<"x-amz-acl">> := <<"public-read">>}, + max_retries := 3, + pool_size := 10, + pool_type := random, + request_timeout := 10000, + ssl := + #{ + cacertfile := <<"cacertfile.crt">>, + certfile := <<"server.crt">>, + ciphers := ["ECDHE-RSA-AES256-GCM-SHA384"], + depth := 10, + enable := true, + keyfile := <<"server.key">>, + reuse_sessions := true, + secure_renegotiate := true, + server_name_indication := "some-host", + verify := verify_peer, + versions := ['tlsv1.2'] + } + } + }, + emqx_s3_schema:translate(#{ + <<"access_key_id">> => <<"access_key_id">>, + <<"secret_access_key">> => <<"secret_access_key">>, + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"min_part_size">> => <<"10mb">>, + <<"acl">> => <<"public_read">>, + <<"transport_options">> => #{ + <<"connect_timeout">> => 30000, + <<"enable_pipelining">> => 200, + <<"pool_size">> => 10, + <<"pool_type">> => <<"random">>, + <<"ssl">> => #{ + <<"enable">> => true, + <<"keyfile">> => <<"server.key">>, + <<"certfile">> => <<"server.crt">>, + <<"cacertfile">> => <<"cacertfile.crt">>, + <<"server_name_indication">> => <<"some-host">>, + <<"verify">> => <<"verify_peer">>, + <<"versions">> => [<<"tlsv1.2">>], + <<"ciphers">> => [<<"ECDHE-RSA-AES256-GCM-SHA384">>] + }, + <<"request_timeout">> => <<"10s">>, + <<"max_retries">> => 3, + <<"headers">> => #{ + <<"x-amz-acl">> => <<"public-read">> + } + } + }) + ). + +t_invalid_limits(_Config) -> + ?assertException( + throw, + {emqx_s3_schema, [#{kind := validation_error, path := "s3.min_part_size"}]}, + emqx_s3_schema:translate(#{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"min_part_size">> => <<"1mb">> + }) + ), + + ?assertException( + throw, + {emqx_s3_schema, [#{kind := validation_error, path := "s3.min_part_size"}]}, + emqx_s3_schema:translate(#{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"min_part_size">> => <<"100000gb">> + }) + ), + + ?assertException( + throw, + {emqx_s3_schema, [#{kind := validation_error, path := "s3.max_part_size"}]}, + emqx_s3_schema:translate(#{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"max_part_size">> => <<"1mb">> + }) + ), + + ?assertException( + throw, + {emqx_s3_schema, [#{kind := validation_error, path := "s3.max_part_size"}]}, + emqx_s3_schema:translate(#{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"max_part_size">> => <<"100000gb">> + }) + ). diff --git a/apps/emqx_s3/test/emqx_s3_test_helpers.erl b/apps/emqx_s3/test/emqx_s3_test_helpers.erl new file mode 100644 index 000000000..c74e78a4d --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_test_helpers.erl @@ -0,0 +1,135 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_test_helpers). + +-compile(nowarn_export_all). +-compile(export_all). + +-define(ACCESS_KEY_ID, "minioadmin"). +-define(SECRET_ACCESS_KEY, "minioadmin"). + +-define(TOXIPROXY_HOST, "toxiproxy"). +-define(TOXIPROXY_PORT, 8474). + +-define(TCP_HOST, ?TOXIPROXY_HOST). +-define(TCP_PORT, 19000). +-define(TLS_HOST, ?TOXIPROXY_HOST). +-define(TLS_PORT, 19100). + +-include_lib("erlcloud/include/erlcloud_aws.hrl"). + +-export([ + aws_config/1, + base_raw_config/1, + base_config/1, + + unique_key/0, + unique_bucket/0, + + with_failure/3 +]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +aws_config(tcp) -> + erlcloud_s3_new( + ?ACCESS_KEY_ID, + ?SECRET_ACCESS_KEY, + ?TCP_HOST, + ?TCP_PORT, + "http://" + ); +aws_config(tls) -> + erlcloud_s3_new( + ?ACCESS_KEY_ID, + ?SECRET_ACCESS_KEY, + ?TLS_HOST, + ?TLS_PORT, + "https://" + ). + +base_raw_config(tcp) -> + #{ + <<"bucket">> => <<"bucket">>, + <<"access_key_id">> => bin(?ACCESS_KEY_ID), + <<"secret_access_key">> => bin(?SECRET_ACCESS_KEY), + <<"host">> => ?TCP_HOST, + <<"port">> => ?TCP_PORT, + <<"max_part_size">> => 10 * 1024 * 1024, + <<"transport_options">> => + #{ + <<"request_timeout">> => 2000 + } + }; +base_raw_config(tls) -> + #{ + <<"bucket">> => <<"bucket">>, + <<"access_key_id">> => bin(?ACCESS_KEY_ID), + <<"secret_access_key">> => bin(?SECRET_ACCESS_KEY), + <<"host">> => ?TLS_HOST, + <<"port">> => ?TLS_PORT, + <<"max_part_size">> => 10 * 1024 * 1024, + <<"transport_options">> => + #{ + <<"request_timeout">> => 2000, + <<"ssl">> => #{ + <<"enable">> => true, + <<"cacertfile">> => bin(cert_path("ca.crt")), + <<"server_name_indication">> => <<"authn-server">>, + <<"verify">> => <<"verify_peer">> + } + } + }. + +base_config(ConnType) -> + emqx_s3_schema:translate(base_raw_config(ConnType)). + +unique_key() -> + "key-" ++ integer_to_list(erlang:system_time(millisecond)) ++ "-" ++ + integer_to_list(erlang:unique_integer([positive])). + +unique_bucket() -> + "bucket-" ++ integer_to_list(erlang:system_time(millisecond)) ++ "-" ++ + integer_to_list(erlang:unique_integer([positive])). + +with_failure(_ConnType, ehttpc_500, Fun) -> + try + meck:new(ehttpc, [passthrough, no_history]), + meck:expect(ehttpc, request, fun(_, _, _, _) -> {ok, 500, []} end), + Fun() + after + meck:unload(ehttpc) + end; +with_failure(ConnType, FailureType, Fun) -> + emqx_common_test_helpers:with_failure( + FailureType, + toxproxy_name(ConnType), + ?TOXIPROXY_HOST, + ?TOXIPROXY_PORT, + Fun + ). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +toxproxy_name(tcp) -> "minio_tcp"; +toxproxy_name(tls) -> "minio_tls". + +cert_path(FileName) -> + Dir = code:lib_dir(emqx_s3, test), + filename:join([Dir, <<"certs">>, FileName]). + +bin(String) when is_list(String) -> list_to_binary(String); +bin(Binary) when is_binary(Binary) -> Binary. + +erlcloud_s3_new(AccessKeyId, SecretAccessKey, Host, Port, Scheme) -> + AwsConfig = erlcloud_s3:new(AccessKeyId, SecretAccessKey, Host, Port), + AwsConfig#aws_config{ + s3_scheme = Scheme, + s3_bucket_access_method = path + }. diff --git a/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl new file mode 100644 index 000000000..ef1d916c6 --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl @@ -0,0 +1,535 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_uploader_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(assertProcessExited(Reason, Pid), + receive + {'DOWN', _, _, Pid, Reason} -> + % ct:print("uploader process exited with reason: ~p", [R]), + ok + after 3000 -> + ct:fail("uploader process did not exit") + end +). + +-define(assertObjectEqual(Value, AwsConfig, Bucket, Key), + ?assertEqual( + Value, + proplists:get_value( + content, + erlcloud_s3:get_object( + Bucket, + Key, + AwsConfig + ) + ) + ) +). + +all() -> + [ + {group, tcp}, + {group, tls} + ]. + +groups() -> + [ + {tcp, [ + {group, common_cases}, + {group, tcp_cases} + ]}, + {tls, [ + {group, common_cases}, + {group, tls_cases} + ]}, + {common_cases, [], [ + t_happy_path_simple_put, + t_happy_path_multi, + t_abort_multi, + t_abort_simple_put, + + {group, noconn_errors}, + {group, timeout_errors}, + {group, http_errors} + ]}, + + {tcp_cases, [ + t_config_switch, + t_config_switch_http_settings, + t_too_large, + t_no_profile + ]}, + + {tls_cases, [ + t_tls_error + ]}, + + {noconn_errors, [{group, transport_errors}]}, + {timeout_errors, [{group, transport_errors}]}, + {http_errors, [{group, transport_errors}]}, + + {transport_errors, [ + t_start_multipart_error, + t_upload_part_error, + t_complete_multipart_error, + t_abort_multipart_error, + t_put_object_error + ]} + ]. + +suite() -> [{timetrap, {minutes, 1}}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(emqx_s3), + Config. + +end_per_suite(_Config) -> + ok = application:stop(emqx_s3). + +init_per_group(Group, Config) when Group =:= tcp orelse Group =:= tls -> + [{conn_type, Group} | Config]; +init_per_group(noconn_errors, Config) -> + [{failure, down} | Config]; +init_per_group(timeout_errors, Config) -> + [{failure, timeout} | Config]; +init_per_group(http_errors, Config) -> + [{failure, ehttpc_500} | Config]; +init_per_group(_ConnType, Config) -> + Config. + +end_per_group(_ConnType, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + ok = snabbkaffe:start_trace(), + ConnType = ?config(conn_type, Config), + TestAwsConfig = emqx_s3_test_helpers:aws_config(ConnType), + + Bucket = emqx_s3_test_helpers:unique_bucket(), + ok = erlcloud_s3:create_bucket(Bucket, TestAwsConfig), + + ProfileBaseConfig = emqx_s3_test_helpers:base_config(ConnType), + ProfileConfig = ProfileBaseConfig#{bucket => Bucket}, + ok = emqx_s3:start_profile(profile_id(), ProfileConfig), + + [{bucket, Bucket}, {test_aws_config, TestAwsConfig}, {profile_config, ProfileConfig} | Config]. + +end_per_testcase(_TestCase, _Config) -> + ok = snabbkaffe:stop(), + _ = emqx_s3:stop_profile(profile_id()). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_happy_path_simple_put(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + Data = data($a, 1024, 10), + + lists:foreach( + fun(Chunk) -> + ?assertEqual( + ok, + emqx_s3_uploader:write(Pid, Chunk) + ) + end, + Data + ), + + ok = emqx_s3_uploader:complete(Pid), + + ?assertProcessExited( + normal, + Pid + ), + + ?assertObjectEqual( + iolist_to_binary(Data), + ?config(test_aws_config, Config), + ?config(bucket, Config), + Key + ). + +t_happy_path_multi(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + Data = data($a, 1024 * 1024, 10), + + lists:foreach( + fun(Chunk) -> + ?assertEqual( + ok, + emqx_s3_uploader:write(Pid, Chunk) + ) + end, + Data + ), + + ok = emqx_s3_uploader:complete(Pid), + + ?assertProcessExited( + normal, + Pid + ), + + ?assertObjectEqual( + iolist_to_binary(Data), + ?config(test_aws_config, Config), + ?config(bucket, Config), + Key + ). + +t_abort_multi(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 6 * 1024 * 1024, 1), + + ok = emqx_s3_uploader:write(Pid, Data), + + ?assertMatch( + [], + list_objects(Config) + ), + + ok = emqx_s3_uploader:abort(Pid), + + ?assertMatch( + [], + list_objects(Config) + ), + + ?assertProcessExited( + normal, + Pid + ). + +t_abort_simple_put(_Config) -> + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 10 * 1024, 1), + + ok = emqx_s3_uploader:write(Pid, Data), + + ok = emqx_s3_uploader:abort(Pid), + + ?assertProcessExited( + normal, + Pid + ). + +t_config_switch(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + OldBucket = ?config(bucket, Config), + {ok, Pid0} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + [Data0, Data1] = data($a, 6 * 1024 * 1024, 2), + + ok = emqx_s3_uploader:write(Pid0, Data0), + + %% Switch to the new config, but without changing HTTP settings + ProfileConfig = ?config(profile_config, Config), + NewBucket = emqx_s3_test_helpers:unique_bucket(), + ok = erlcloud_s3:create_bucket(NewBucket, ?config(test_aws_config, Config)), + NewProfileConfig = ProfileConfig#{bucket => NewBucket}, + + ok = emqx_s3:update_profile(profile_id(), NewProfileConfig), + + %% Already started uploader should be OK and use previous config + ok = emqx_s3_uploader:write(Pid0, Data1), + ok = emqx_s3_uploader:complete(Pid0), + + ?assertObjectEqual( + iolist_to_binary([Data0, Data1]), + ?config(test_aws_config, Config), + OldBucket, + Key + ), + + %% Now check that new uploader uses new config + {ok, Pid1} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + ok = emqx_s3_uploader:write(Pid1, Data0), + ok = emqx_s3_uploader:complete(Pid1), + + ?assertObjectEqual( + iolist_to_binary(Data0), + ?config(test_aws_config, Config), + NewBucket, + Key + ). + +t_config_switch_http_settings(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + OldBucket = ?config(bucket, Config), + {ok, Pid0} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + [Data0, Data1] = data($a, 6 * 1024 * 1024, 2), + + ok = emqx_s3_uploader:write(Pid0, Data0), + + %% Switch to the new config, completely changing HTTP settings (tcp -> tls) + NewBucket = emqx_s3_test_helpers:unique_bucket(), + NewTestAwsConfig = emqx_s3_test_helpers:aws_config(tls), + ok = erlcloud_s3:create_bucket(NewBucket, NewTestAwsConfig), + NewProfileConfig0 = emqx_s3_test_helpers:base_config(tls), + NewProfileConfig1 = NewProfileConfig0#{bucket => NewBucket}, + + ok = emqx_s3:update_profile(profile_id(), NewProfileConfig1), + + %% Already started uploader should be OK and use previous config + ok = emqx_s3_uploader:write(Pid0, Data1), + ok = emqx_s3_uploader:complete(Pid0), + + ?assertObjectEqual( + iolist_to_binary([Data0, Data1]), + ?config(test_aws_config, Config), + OldBucket, + Key + ), + + %% Now check that new uploader uses new config + {ok, Pid1} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + ok = emqx_s3_uploader:write(Pid1, Data0), + ok = emqx_s3_uploader:complete(Pid1), + + ?assertObjectEqual( + iolist_to_binary(Data0), + NewTestAwsConfig, + NewBucket, + Key + ). + +t_start_multipart_error(Config) -> + _ = process_flag(trap_exit, true), + + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 6 * 1024 * 1024, 1), + + emqx_s3_test_helpers:with_failure( + ?config(conn_type, Config), + ?config(failure, Config), + fun() -> + ?assertMatch( + {error, _}, + emqx_s3_uploader:write(Pid, Data) + ) + end + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_upload_part_error(Config) -> + _ = process_flag(trap_exit, true), + + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data0, Data1] = data($a, 6 * 1024 * 1024, 2), + + ok = emqx_s3_uploader:write(Pid, Data0), + + emqx_s3_test_helpers:with_failure( + ?config(conn_type, Config), + ?config(failure, Config), + fun() -> + ?assertMatch( + {error, _}, + emqx_s3_uploader:write(Pid, Data1) + ) + end + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_abort_multipart_error(Config) -> + _ = process_flag(trap_exit, true), + + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 6 * 1024 * 1024, 1), + + ok = emqx_s3_uploader:write(Pid, Data), + + emqx_s3_test_helpers:with_failure( + ?config(conn_type, Config), + ?config(failure, Config), + fun() -> + ?assertMatch( + {error, _}, + emqx_s3_uploader:abort(Pid) + ) + end + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_complete_multipart_error(Config) -> + _ = process_flag(trap_exit, true), + + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 6 * 1024 * 1024, 1), + + ok = emqx_s3_uploader:write(Pid, Data), + + emqx_s3_test_helpers:with_failure( + ?config(conn_type, Config), + ?config(failure, Config), + fun() -> + ?assertMatch( + {error, _}, + emqx_s3_uploader:complete(Pid) + ) + end + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_put_object_error(Config) -> + _ = process_flag(trap_exit, true), + + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + %% Little data to avoid multipart upload + [Data] = data($a, 1024, 1), + + emqx_s3_test_helpers:with_failure( + ?config(conn_type, Config), + ?config(failure, Config), + fun() -> + ok = emqx_s3_uploader:write(Pid, Data), + ?assertMatch( + {error, _}, + emqx_s3_uploader:complete(Pid) + ) + end + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_too_large(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 1024, 1), + + [DataLarge] = data($a, 20 * 1024 * 1024, 1), + + ?assertMatch( + {error, {too_large, _}}, + emqx_s3_uploader:write(Pid, DataLarge) + ), + + ok = emqx_s3_uploader:write(Pid, Data), + ok = emqx_s3_uploader:complete(Pid), + + ?assertProcessExited( + normal, + Pid + ), + + ?assertObjectEqual( + iolist_to_binary(Data), + ?config(test_aws_config, Config), + ?config(bucket, Config), + Key + ). + +t_tls_error(Config) -> + _ = process_flag(trap_exit, true), + + ProfileBaseConfig = ?config(profile_config, Config), + ProfileConfig = emqx_map_lib:deep_put( + [transport_options, ssl, server_name_indication], ProfileBaseConfig, "invalid-hostname" + ), + ok = emqx_s3:update_profile(profile_id(), ProfileConfig), + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 6 * 1024 * 1024, 1), + + ?assertMatch( + {error, _}, + emqx_s3_uploader:write(Pid, Data) + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_no_profile(_Config) -> + Key = emqx_s3_test_helpers:unique_key(), + ?assertMatch( + {error, profile_not_found}, + emqx_s3:start_uploader(<<"no-profile">>, #{key => Key}) + ). + +%%-------------------------------------------------------------------- +%% Test helpers +%%-------------------------------------------------------------------- + +profile_id() -> + <<"test">>. + +data(Byte, ChunkSize, ChunkCount) -> + Chunk = iolist_to_binary([Byte || _ <- lists:seq(1, ChunkSize)]), + [Chunk || _ <- lists:seq(1, ChunkCount)]. + +list_objects(Config) -> + Props = erlcloud_s3:list_objects(?config(bucket, Config), [], ?config(test_aws_config, Config)), + proplists:get_value(contents, Props). diff --git a/rebar.config.erl b/rebar.config.erl index ea0016ca9..51a6946dc 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -401,7 +401,8 @@ relx_apps(ReleaseType, Edition) -> emqx_psk, emqx_slow_subs, emqx_plugins, - emqx_ft + emqx_ft, + emqx_s3 ] ++ [quicer || is_quicer_supported()] ++ [bcrypt || provide_bcrypt_release(ReleaseType)] ++ diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 82823720d..9644ec8b9 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -91,6 +91,12 @@ if [ "${WHICH_APP}" = 'novalue' ]; then exit 1 fi +if [ ! -d "${WHICH_APP}" ]; then + echo "must provide an existing path for --app arg" + help + exit 1 +fi + if [[ "${WHICH_APP}" == lib-ee* && (-z "${PROFILE+x}" || "${PROFILE}" != emqx-enterprise) ]]; then echo 'You are trying to run an enterprise test case without the emqx-enterprise profile.' echo 'This will most likely not work.' @@ -172,10 +178,14 @@ for dep in ${CT_DEPS}; do ;; rocketmq) FILES+=( '.ci/docker-compose-file/docker-compose-rocketmq.yaml' ) - ;; + ;; cassandra) FILES+=( '.ci/docker-compose-file/docker-compose-cassandra.yaml' ) ;; + minio) + FILES+=( '.ci/docker-compose-file/docker-compose-minio-tcp.yaml' + '.ci/docker-compose-file/docker-compose-minio-tls.yaml' ) + ;; *) echo "unknown_ct_dependency $dep" exit 1 From 818a5cacf298e1b4b981618effee40c298f762ac Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 30 Mar 2023 22:05:21 +0300 Subject: [PATCH 086/156] feat(ft-s3): add initial integration --- apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 13 +- apps/emqx_ft/src/emqx_ft.erl | 2 +- apps/emqx_ft/src/emqx_ft_assembler.erl | 21 +- apps/emqx_ft/src/emqx_ft_schema.erl | 186 +++++++++++------- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 47 +++-- .../src/emqx_ft_storage_exporter_fs.erl | 12 +- .../src/emqx_ft_storage_exporter_s3.erl | 69 +++++++ apps/emqx_ft/src/emqx_ft_storage_fs.erl | 16 +- apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf | 76 +++++++ apps/emqx_s3/src/emqx_s3.erl | 28 ++- apps/emqx_s3/src/emqx_s3_schema.erl | 4 +- 11 files changed, 362 insertions(+), 112 deletions(-) create mode 100644 apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl create mode 100644 apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf index 941611424..c33ea447e 100644 --- a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -58,7 +58,7 @@ emqx_ft_schema { local_storage_exporter_type { desc { - en: "Type of the Exporter to use." + en: "Exporter type for the exporter to the local file system" zh: "" } label: { @@ -67,6 +67,17 @@ emqx_ft_schema { } } + s3_exporter_type { + desc { + en: "Exporter type for the exporter to S3" + zh: "" + } + label: { + en: "S3 Exporter Type" + zh: "" + } + } + local_storage_exporter_root { desc { en: "File system path to keep uploaded files." diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index b7c8f0eac..e3f6cbe1c 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -336,7 +336,7 @@ transfer(Msg, FileId) -> {clientid_to_binary(ClientId), FileId}. on_complete(Op, {ChanPid, PacketId}, Transfer, Result) -> - ?SLOG(debug, #{ + ?SLOG(warning, #{ msg => "on_complete", operation => Op, packet_id => PacketId, diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 17fed012a..3a352cd10 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -136,13 +136,12 @@ handle_event( ) -> Filemeta = emqx_ft_assembly:filemeta(Asm), Coverage = emqx_ft_assembly:coverage(Asm), - % TODO: better error handling - {ok, Export} = emqx_ft_storage_exporter:start_export( - Storage, - Transfer, - Filemeta - ), - {next_state, {assemble, Coverage}, St#{export => Export}, ?internal([])}; + case emqx_ft_storage_exporter:start_export(Storage, Transfer, Filemeta) of + {ok, Export} -> + {next_state, {assemble, Coverage}, St#{export => Export}, ?internal([])}; + {error, _} = Error -> + {stop, {shutdown, Error}} + end; handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #{export := Export}) -> % TODO % Currently, race is possible between getting segment info from the remote node and @@ -150,8 +149,12 @@ handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #{export := % TODO: pipelining % TODO: better error handling {ok, Content} = pread(Node, Segment, St), - {ok, NExport} = emqx_ft_storage_exporter:write(Export, Content), - {next_state, {assemble, Rest}, St#{export := NExport}, ?internal([])}; + case emqx_ft_storage_exporter:write(Export, Content) of + {ok, NExport} -> + {next_state, {assemble, Rest}, St#{export := NExport}, ?internal([])}; + {error, _} = Error -> + {stop, {shutdown, Error}, maps:remove(export, St)} + end; handle_event(internal, _, {assemble, []}, St = #{}) -> {next_state, complete, St, ?internal([])}; handle_event(internal, _, complete, St = #{export := Export}) -> diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 2abbe4c45..df28de218 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -16,7 +16,7 @@ -module(emqx_ft_schema). --behaviour(hocon_schema). +% -behaviour(hocon_schema). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). @@ -35,7 +35,7 @@ -reflect_type([json_value/0]). -%% +-import(hoconsc, [ref/1, ref/2, mk/2]). namespace() -> file_transfer. @@ -46,84 +46,130 @@ roots() -> [file_transfer]. fields(file_transfer) -> [ - {storage, #{ - type => hoconsc:union([ - hoconsc:ref(?MODULE, local_storage) - ]), - desc => ?DESC("storage") - }} + {storage, + mk( + hoconsc:union([ + ref(local_storage) + ]), + #{ + required => true, + desc => ?DESC("storage") + } + )} ]; fields(local_storage) -> [ - {type, #{ - type => local, - default => local, - required => false, - desc => ?DESC("local_type") - }}, - {segments, #{ - type => ?REF(local_storage_segments), - desc => ?DESC("local_storage_segments"), - required => false - }}, - {exporter, #{ - type => hoconsc:union([ - ?REF(local_storage_exporter) - ]), - desc => ?DESC("local_storage_exporter"), - required => true - }} + {type, + mk( + local, + #{ + default => local, + required => false, + desc => ?DESC("local_type") + } + )}, + {segments, + mk( + ref(local_storage_segments), + #{ + desc => ?DESC("local_storage_segments"), + required => false + } + )}, + {exporter, + mk( + hoconsc:union([ + ref(local_storage_exporter), + ref(s3_exporter) + ]), + #{ + desc => ?DESC("local_storage_exporter"), + required => true + } + )} ]; fields(local_storage_segments) -> [ - {root, #{ - type => binary(), - desc => ?DESC("local_storage_segments_root"), - required => false - }}, - {gc, #{ - type => ?REF(local_storage_segments_gc), - desc => ?DESC("local_storage_segments_gc"), - required => false - }} + {root, + mk( + binary(), + #{ + desc => ?DESC("local_storage_segments_root"), + required => false + } + )}, + {gc, + mk( + ref(local_storage_segments_gc), #{ + desc => ?DESC("local_storage_segments_gc"), + required => false + } + )} ]; fields(local_storage_exporter) -> [ - {type, #{ - type => local, - default => local, - required => false, - desc => ?DESC("local_storage_exporter_type") - }}, - {root, #{ - type => binary(), - desc => ?DESC("local_storage_exporter_root"), - required => false - }} + {type, + mk( + local, + #{ + default => local, + required => false, + desc => ?DESC("local_storage_exporter_type") + } + )}, + {root, + mk( + binary(), + #{ + desc => ?DESC("local_storage_exporter_root"), + required => false + } + )} ]; +fields(s3_exporter) -> + [ + {type, + mk( + s3, + #{ + default => s3, + required => false, + desc => ?DESC("s3_exporter_type") + } + )} + ] ++ + emqx_s3_schema:fields(s3); fields(local_storage_segments_gc) -> [ - {interval, #{ - type => emqx_schema:duration_ms(), - desc => ?DESC("storage_gc_interval"), - required => false, - default => "1h" - }}, - {maximum_segments_ttl, #{ - type => emqx_schema:duration_s(), - desc => ?DESC("storage_gc_max_segments_ttl"), - required => false, - default => "24h" - }}, - {minimum_segments_ttl, #{ - type => emqx_schema:duration_s(), - % desc => ?DESC("storage_gc_min_segments_ttl"), - required => false, - default => "5m", - % NOTE - % This setting does not seem to be useful to an end-user. - hidden => true - }} + {interval, + mk( + emqx_schema:duration_ms(), + #{ + desc => ?DESC("storage_gc_interval"), + required => false, + default => "1h" + } + )}, + {maximum_segments_ttl, + mk( + emqx_schema:duration_s(), + #{ + desc => ?DESC("storage_gc_max_segments_ttl"), + required => false, + default => "24h" + } + )}, + {minimum_segments_ttl, + mk( + emqx_schema:duration_s(), + #{ + required => false, + default => "5m", + % NOTE + % This setting does not seem to be useful to an end-user. + hidden => true + } + )} ]. desc(file_transfer) -> @@ -133,7 +179,9 @@ desc(local_storage) -> desc(local_storage_segments) -> "File transfer local segments storage settings"; desc(local_storage_exporter) -> - "Exporter settings for the File transfer local storage backend"; + "Local Exporter settings for the File transfer local storage backend"; +desc(s3_exporter) -> + "S3 Exporter settings for the File transfer local storage backend"; desc(local_storage_segments_gc) -> "Garbage collection settings for the File transfer local segments storage". diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index a7d1cd2e2..297762d28 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -34,42 +34,47 @@ -export([exporter/1]). --export_type([options/0]). -export_type([export/0]). -type storage() :: emxt_ft_storage_fs:storage(). -type transfer() :: emqx_ft:transfer(). -type filemeta() :: emqx_ft:filemeta(). --type options() :: map(). --type export() :: term(). +-type exporter_conf() :: map(). +-type export_st() :: term(). +-opaque export() :: {module(), export_st()}. --callback start_export(options(), transfer(), filemeta()) -> - {ok, export()} | {error, _Reason}. +-callback start_export(exporter_conf(), transfer(), filemeta()) -> + {ok, export_st()} | {error, _Reason}. --callback write(ExportSt :: export(), iodata()) -> - {ok, ExportSt :: export()} | {error, _Reason}. +%% Exprter must discard the export itself in case of error +-callback write(ExportSt :: export_st(), iodata()) -> + {ok, ExportSt :: export_st()} | {error, _Reason}. --callback complete(ExportSt :: export()) -> +-callback complete(ExportSt :: export_st()) -> ok | {error, _Reason}. --callback discard(ExportSt :: export()) -> +-callback discard(ExportSt :: export_st()) -> ok | {error, _Reason}. --callback list(options()) -> +-callback list(storage()) -> {ok, [emqx_ft_storage:file_info()]} | {error, _Reason}. %% +-spec start_export(storage(), transfer(), filemeta()) -> + {ok, export()} | {error, _Reason}. start_export(Storage, Transfer, Filemeta) -> - {ExporterMod, Exporter} = exporter(Storage), - case ExporterMod:start_export(Exporter, Transfer, Filemeta) of + {ExporterMod, ExporterConf} = exporter(Storage), + case ExporterMod:start_export(ExporterConf, Transfer, Filemeta) of {ok, ExportSt} -> {ok, {ExporterMod, ExportSt}}; {error, _} = Error -> Error end. +-spec write(export(), iodata()) -> + {ok, export()} | {error, _Reason}. write({ExporterMod, ExportSt}, Content) -> case ExporterMod:write(ExportSt, Content) of {ok, ExportStNext} -> @@ -78,23 +83,31 @@ write({ExporterMod, ExportSt}, Content) -> Error end. +-spec complete(export()) -> + ok | {error, _Reason}. complete({ExporterMod, ExportSt}) -> ExporterMod:complete(ExportSt). +-spec discard(export()) -> + ok | {error, _Reason}. discard({ExporterMod, ExportSt}) -> ExporterMod:discard(ExportSt). -%% - +-spec list(storage()) -> + {ok, [emqx_ft_storage:file_info()]} | {error, _Reason}. list(Storage) -> {ExporterMod, ExporterOpts} = exporter(Storage), ExporterMod:list(ExporterOpts). -%% - -spec exporter(storage()) -> {module(), _ExporterOptions}. exporter(Storage) -> case maps:get(exporter, Storage) of #{type := local} = Options -> - {emqx_ft_storage_exporter_fs, Options} + {emqx_ft_storage_exporter_fs, without_type(Options)}; + #{type := s3} = Options -> + {emqx_ft_storage_exporter_s3, without_type(Options)} end. + +-spec without_type(exporter_conf()) -> exporter_conf(). +without_type(#{type := _} = Options) -> + maps:without([type], Options). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index 647d84124..64c0e325a 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -20,21 +20,22 @@ -include_lib("emqx/include/logger.hrl"). %% Exporter API +-behaviour(emqx_ft_storage_exporter). + -export([start_export/3]). -export([write/2]). -export([complete/1]). -export([discard/1]). +-export([list/1]). +%% Internal API for RPC -export([list_local/1]). -export([list_local/2]). -export([start_reader/3]). --export([list/1]). % TODO % -export([list/2]). --export_type([export/0]). - -type options() :: _TODO. -type transfer() :: emqx_ft:transfer(). -type filemeta() :: emqx_ft:filemeta(). @@ -49,7 +50,7 @@ -type file_error() :: emqx_ft_storage_fs:file_error(). --opaque export() :: #{ +-type export() :: #{ path := file:name(), handle := io:device(), result := file:name(), @@ -116,6 +117,7 @@ write(Export = #{handle := Handle, hash := Ctx}, IoData) -> ok -> {ok, Export#{hash := update_checksum(Ctx, IoData)}}; {error, _} = Error -> + _ = discard(Export), Error end. @@ -370,4 +372,4 @@ mk_transfer_hash(Transfer) -> crypto:hash(?BUCKET_HASH, term_to_binary(Transfer)). get_storage_root(Options) -> - maps:get(root, Options, filename:join([emqx:data_dir(), "ft", "exports"])). + maps:get(root, Options, filename:join([emqx:data_dir(), file_transfer, exports])). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl new file mode 100644 index 000000000..300f0dc85 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -0,0 +1,69 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_exporter_s3). + +-include_lib("emqx/include/logger.hrl"). + +%% Exporter API +-export([start_export/3]). +-export([write/2]). +-export([complete/1]). +-export([discard/1]). +-export([list/1]). + +-type options() :: emqx_s3:profile_config(). +-type transfer() :: emqx_ft:transfer(). +-type filemeta() :: emqx_ft:filemeta(). +-type exportinfo() :: #{ + transfer := transfer(), + name := file:name(), + uri := uri_string:uri_string(), + timestamp := emqx_datetime:epoch_second(), + size := _Bytes :: non_neg_integer(), + meta => filemeta() +}. + +-type export_st() :: #{ + pid := pid(), + meta := filemeta() +}. + +-spec start_export(options(), transfer(), filemeta()) -> + {ok, export_st()} | {error, term()}. +start_export(_Options, _Transfer, Filemeta = #{name := Filename}) -> + Pid = spawn(fun() -> Filename end), + #{meta => Filemeta, pid => Pid}. + +-spec write(export_st(), iodata()) -> + {ok, export_st()} | {error, term()}. +write(ExportSt, _IoData) -> + {ok, ExportSt}. + +-spec complete(export_st()) -> + ok | {error, term()}. +complete(_ExportSt) -> + ok. + +-spec discard(export_st()) -> + ok. +discard(_ExportSt) -> + ok. + +-spec list(options()) -> + {ok, [exportinfo()]} | {error, term()}. +list(_Options) -> + {ok, []}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 4e5cc9236..d871d1f32 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -86,14 +86,18 @@ -define(MANIFEST, "MANIFEST.json"). -define(SEGMENT, "SEG"). --type storage() :: #{ - root => file:name(), - exporter => exporter() +-type segments() :: #{ + root := file:name(), + gc := #{ + interval := non_neg_integer(), + maximum_segments_ttl := non_neg_integer(), + minimum_segments_ttl := non_neg_integer() + } }. --type exporter() :: #{ - type := local, - root => file:name() +-type storage() :: #{ + segments := segments(), + exporter := emqx_ft_storage_exporter:exporter() }. -type file_error() :: diff --git a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf new file mode 100644 index 000000000..4e0870bae --- /dev/null +++ b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf @@ -0,0 +1,76 @@ +emqx_s3_schema { + access_key_id { + desc { + en: "The access key id of the S3 bucket." + } + label { + en: "Access Key ID" + } + } + secret_access_key { + desc { + en: "The secret access key of the S3 bucket." + } + label { + en: "Secret Access Key" + } + } + bucket { + desc { + en: "The name of the S3 bucket." + } + label { + en: "Bucket" + } + } + host { + desc { + en: "The host of the S3 endpoint." + } + label { + en: "S3 endpoint Host" + } + } + port { + desc { + en: "The port of the S3 endpoint." + } + label { + en: "S3 endpoint port" + } + } + min_part_size { + desc { + en: """The minimum part size for multipart uploads. +Uploaded data will be accumulated in memory until this size is reached.""" + } + label { + en: "Minimum multipart upload part size" + } + } + max_part_size { + desc { + en: """The maximum part size for multipart uploads. +S3 uploader won't try to upload parts larger than this size.""" + } + label { + en: "Maximum multipart upload part size" + } + } + acl { + desc { + en: "The ACL to use for the uploaded objects." + } + label { + en: "ACL" + } + } + transport_options { + desc { + en: "Options for the HTTP transport layer used by the S3 client." + } + label { + en: "Transport Options" + } + } +} diff --git a/apps/emqx_s3/src/emqx_s3.erl b/apps/emqx_s3/src/emqx_s3.erl index 6d2577dca..7c78de979 100644 --- a/apps/emqx_s3/src/emqx_s3.erl +++ b/apps/emqx_s3/src/emqx_s3.erl @@ -21,8 +21,32 @@ -type profile_id() :: term(). -%% TODO: define fields --type profile_config() :: map(). +-type acl() :: + private + | public_read + | public_read_write + | authenticated_read + | bucket_owner_read + | bucket_owner_full_control. + +-type transport_options() :: #{ + connect_timeout => pos_integer(), + enable_pipelining => pos_integer(), + headers => map(), + max_retries => pos_integer(), + pool_size => pos_integer(), + pool_type => atom(), + ssl => map() +}. + +-type profile_config() :: #{ + bucket := string(), + host := string(), + port := pos_integer(), + acl => acl(), + min_part_size => pos_integer(), + transport_options => transport_options() +}. %%-------------------------------------------------------------------- %% API diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index ceb0d1dd4..5d76e7120 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -7,7 +7,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). --import(hoconsc, [mk/2, ref/1]). +-import(hoconsc, [mk/2, ref/2]). -export([roots/0, fields/1, namespace/0, tags/0]). @@ -101,7 +101,7 @@ fields(s3) -> )}, {transport_options, mk( - ref(transport_options), + ref(?MODULE, transport_options), #{ desc => ?DESC("transport_options"), required => false From 83612236487af0bd22269e22f5acf72767825fa7 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 30 Mar 2023 23:09:13 +0300 Subject: [PATCH 087/156] feat(ft-s3): extract checksum verification --- apps/emqx_ft/src/emqx_ft.erl | 6 +- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 69 +++++++++++++++--- .../src/emqx_ft_storage_exporter_fs.erl | 70 ++++++------------- 3 files changed, 84 insertions(+), 61 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index e3f6cbe1c..064b6e066 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -45,7 +45,8 @@ bytes/0, offset/0, filemeta/0, - segment/0 + segment/0, + checksum/0 ]). %% Number of bytes @@ -57,6 +58,7 @@ -type fileid() :: binary(). -type transfer() :: {clientid(), fileid()}. -type offset() :: bytes(). +-type checksum() :: {_Algo :: atom(), _Digest :: binary()}. -type filemeta() :: #{ %% Display name @@ -68,7 +70,7 @@ %% currently do not condider that an error (or, specifically, a signal that %% the resulting file is corrupted during transmission). size => _Bytes :: non_neg_integer(), - checksum => {sha256, <<_:256>>}, + checksum => checksum(), expire_at := emqx_datetime:epoch_second(), %% TTL of individual segments %% Somewhat confusing that we won't know it on the nodes where the filemeta diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index 297762d28..444d686a8 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -39,10 +39,21 @@ -type storage() :: emxt_ft_storage_fs:storage(). -type transfer() :: emqx_ft:transfer(). -type filemeta() :: emqx_ft:filemeta(). +-type checksum() :: emqx_ft:checksum(). -type exporter_conf() :: map(). -type export_st() :: term(). --opaque export() :: {module(), export_st()}. +-type hash_state() :: term(). +-opaque export() :: #{ + mod := module(), + st := export_st(), + hash := hash_state(), + filemeta := filemeta() +}. + +%%------------------------------------------------------------------------------ +%% Behaviour +%%------------------------------------------------------------------------------ -callback start_export(exporter_conf(), transfer(), filemeta()) -> {ok, export_st()} | {error, _Reason}. @@ -51,7 +62,7 @@ -callback write(ExportSt :: export_st(), iodata()) -> {ok, ExportSt :: export_st()} | {error, _Reason}. --callback complete(ExportSt :: export_st()) -> +-callback complete(_ExportSt :: export_st(), _Checksum :: checksum()) -> ok | {error, _Reason}. -callback discard(ExportSt :: export_st()) -> @@ -60,7 +71,9 @@ -callback list(storage()) -> {ok, [emqx_ft_storage:file_info()]} | {error, _Reason}. -%% +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ -spec start_export(storage(), transfer(), filemeta()) -> {ok, export()} | {error, _Reason}. @@ -68,29 +81,43 @@ start_export(Storage, Transfer, Filemeta) -> {ExporterMod, ExporterConf} = exporter(Storage), case ExporterMod:start_export(ExporterConf, Transfer, Filemeta) of {ok, ExportSt} -> - {ok, {ExporterMod, ExportSt}}; + {ok, #{ + mod => ExporterMod, + st => ExportSt, + hash => init_checksum(Filemeta), + filemeta => Filemeta + }}; {error, _} = Error -> Error end. -spec write(export(), iodata()) -> {ok, export()} | {error, _Reason}. -write({ExporterMod, ExportSt}, Content) -> +write(#{mod := ExporterMod, st := ExportSt, hash := Hash} = Export, Content) -> case ExporterMod:write(ExportSt, Content) of {ok, ExportStNext} -> - {ok, {ExporterMod, ExportStNext}}; + {ok, Export#{ + st := ExportStNext, + hash := update_checksum(Hash, Content) + }}; {error, _} = Error -> Error end. -spec complete(export()) -> ok | {error, _Reason}. -complete({ExporterMod, ExportSt}) -> - ExporterMod:complete(ExportSt). +complete(#{mod := ExporterMod, st := ExportSt, hash := Hash, filemeta := Filemeta}) -> + case verify_checksum(Hash, Filemeta) of + {ok, Checksum} -> + ExporterMod:complete(ExportSt, Checksum); + {error, _} = Error -> + _ = ExporterMod:discard(ExportSt), + Error + end. -spec discard(export()) -> ok | {error, _Reason}. -discard({ExporterMod, ExportSt}) -> +discard(#{mod := ExporterMod, st := ExportSt}) -> ExporterMod:discard(ExportSt). -spec list(storage()) -> @@ -99,7 +126,10 @@ list(Storage) -> {ExporterMod, ExporterOpts} = exporter(Storage), ExporterMod:list(ExporterOpts). --spec exporter(storage()) -> {module(), _ExporterOptions}. +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + exporter(Storage) -> case maps:get(exporter, Storage) of #{type := local} = Options -> @@ -111,3 +141,22 @@ exporter(Storage) -> -spec without_type(exporter_conf()) -> exporter_conf(). without_type(#{type := _} = Options) -> maps:without([type], Options). + +init_checksum(#{checksum := {Algo, _}}) -> + crypto:hash_init(Algo); +init_checksum(#{}) -> + crypto:hash_init(sha256). + +update_checksum(Ctx, IoData) -> + crypto:hash_update(Ctx, IoData). + +verify_checksum(Ctx, #{checksum := {Algo, Digest} = Checksum}) -> + case crypto:hash_final(Ctx) of + Digest -> + {ok, Checksum}; + Mismatch -> + {error, {checksum, Algo, binary:encode_hex(Mismatch)}} + end; +verify_checksum(Ctx, #{}) -> + Digest = crypto:hash_final(Ctx), + {ok, {sha256, Digest}}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index 64c0e325a..fd1956009 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -24,7 +24,7 @@ -export([start_export/3]). -export([write/2]). --export([complete/1]). +-export([complete/2]). -export([discard/1]). -export([list/1]). @@ -50,12 +50,11 @@ -type file_error() :: emqx_ft_storage_fs:file_error(). --type export() :: #{ +-type export_st() :: #{ path := file:name(), handle := io:device(), result := file:name(), - meta := filemeta(), - hash := crypto:hash_state() + meta := filemeta() }. -type reader() :: pid(). @@ -92,7 +91,7 @@ %% -spec start_export(options(), transfer(), filemeta()) -> - {ok, export()} | {error, file_error()}. + {ok, export_st()} | {error, file_error()}. start_export(Options, Transfer, Filemeta = #{name := Filename}) -> TempFilepath = mk_temp_absfilepath(Options, Transfer, Filename), ResultFilepath = mk_absfilepath(Options, Transfer, result, Filename), @@ -103,47 +102,41 @@ start_export(Options, Transfer, Filemeta = #{name := Filename}) -> path => TempFilepath, handle => Handle, result => ResultFilepath, - meta => Filemeta, - hash => init_checksum(Filemeta) + meta => Filemeta }}; {error, _} = Error -> Error end. --spec write(export(), iodata()) -> - {ok, export()} | {error, file_error()}. -write(Export = #{handle := Handle, hash := Ctx}, IoData) -> +-spec write(export_st(), iodata()) -> + {ok, export_st()} | {error, file_error()}. +write(ExportSt = #{handle := Handle}, IoData) -> case file:write(Handle, IoData) of ok -> - {ok, Export#{hash := update_checksum(Ctx, IoData)}}; + {ok, ExportSt}; {error, _} = Error -> - _ = discard(Export), + _ = discard(ExportSt), Error end. --spec complete(export()) -> +-spec complete(export_st(), emqx_ft:checksum()) -> ok | {error, {checksum, _Algo, _Computed}} | {error, file_error()}. complete( - Export = #{ + #{ path := Filepath, handle := Handle, result := ResultFilepath, - meta := FilemetaIn, - hash := Ctx - } + meta := FilemetaIn + }, + Checksum ) -> - case verify_checksum(Ctx, FilemetaIn) of - {ok, Filemeta} -> - ok = file:close(Handle), - _ = filelib:ensure_dir(ResultFilepath), - _ = file:write_file(mk_manifest_filename(ResultFilepath), encode_filemeta(Filemeta)), - file:rename(Filepath, ResultFilepath); - {error, _} = Error -> - _ = discard(Export), - Error - end. + Filemeta = FilemetaIn#{checksum => Checksum}, + ok = file:close(Handle), + _ = filelib:ensure_dir(ResultFilepath), + _ = file:write_file(mk_manifest_filename(ResultFilepath), encode_filemeta(Filemeta)), + file:rename(Filepath, ResultFilepath). --spec discard(export()) -> +-spec discard(export_st()) -> ok. discard(#{path := Filepath, handle := Handle}) -> ok = file:close(Handle), @@ -297,27 +290,6 @@ list(_Options) -> %% -init_checksum(#{checksum := {Algo, _}}) -> - crypto:hash_init(Algo); -init_checksum(#{}) -> - crypto:hash_init(sha256). - -update_checksum(Ctx, IoData) -> - crypto:hash_update(Ctx, IoData). - -verify_checksum(Ctx, Filemeta = #{checksum := {Algo, Digest}}) -> - case crypto:hash_final(Ctx) of - Digest -> - {ok, Filemeta}; - Mismatch -> - {error, {checksum, Algo, binary:encode_hex(Mismatch)}} - end; -verify_checksum(Ctx, Filemeta = #{}) -> - Digest = crypto:hash_final(Ctx), - {ok, Filemeta#{checksum => {sha256, Digest}}}. - -%% - -define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). encode_filemeta(Meta) -> From 5ac3543a768a75abb0904aadd7afe7fef3469d0e Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 31 Mar 2023 00:32:11 +0300 Subject: [PATCH 088/156] feat(ft-s3): integrate exporter configs --- apps/emqx_ft/src/emqx_ft_conf.erl | 10 ++++- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 44 +++++++++++++++++-- .../emqx_ft_storage_exporter_fs.erl | 34 +++++++++++++- .../emqx_ft_storage_exporter_fs_api.erl | 0 .../emqx_ft_storage_exporter_fs_proxy.erl | 0 .../emqx_ft_storage_exporter_s3.erl | 33 ++++++++++++++ 6 files changed, 114 insertions(+), 7 deletions(-) rename apps/emqx_ft/src/{ => emqx_ft_storage_exporter}/emqx_ft_storage_exporter_fs.erl (91%) rename apps/emqx_ft/src/{ => emqx_ft_storage_exporter}/emqx_ft_storage_exporter_fs_api.erl (100%) rename apps/emqx_ft/src/{ => emqx_ft_storage_exporter}/emqx_ft_storage_exporter_fs_proxy.erl (100%) rename apps/emqx_ft/src/{ => emqx_ft_storage_exporter}/emqx_ft_storage_exporter_s3.erl (68%) diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index aafcd5ad3..6ce10408e 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -75,6 +75,10 @@ assert_storage(Type) -> -spec load() -> ok. load() -> + ok = emqx_ft_storage_exporter:update_exporter( + undefined, + emqx_config:get([file_transfer, storage]) + ), emqx_conf:add_handler([file_transfer], ?MODULE). -spec unload() -> ok. @@ -98,5 +102,7 @@ pre_config_update(_, Req, _Config) -> emqx_config:app_envs() ) -> ok | {ok, Result :: any()} | {error, Reason :: term()}. -post_config_update(_, _Req, _NewConfig, _OldConfig, _AppEnvs) -> - ok. +post_config_update(_Path, _Req, NewConfig, OldConfig, _AppEnvs) -> + OldStorageConfig = maps:get(storage, OldConfig, undefined), + NewStorageConfig = maps:get(storage, NewConfig, undefined), + emqx_ft_storage_exporter:update_exporter(OldStorageConfig, NewStorageConfig). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index 444d686a8..61241bd6f 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -29,9 +29,11 @@ %% Listing API -export([list/1]). -% TODO -% -export([list/2]). +%% Lifecycle API +-export([update_exporter/2]). + +%% Internal API -export([exporter/1]). -export_type([export/0]). @@ -71,6 +73,17 @@ -callback list(storage()) -> {ok, [emqx_ft_storage:file_info()]} | {error, _Reason}. +%% Lifecycle callbacks + +-callback start(exporter_conf()) -> + ok | {error, _Reason}. + +-callback stop(exporter_conf()) -> + ok. + +-callback update(exporter_conf(), exporter_conf()) -> + ok | {error, _Reason}. + %%------------------------------------------------------------------------------ %% API %%------------------------------------------------------------------------------ @@ -126,6 +139,32 @@ list(Storage) -> {ExporterMod, ExporterOpts} = exporter(Storage), ExporterMod:list(ExporterOpts). +%% Lifecycle + +-spec update_exporter(emqx_config:config(), emqx_config:config()) -> ok | {error, term()}. +update_exporter( + #{exporter := #{type := OldType}} = OldConfig, + #{exporter := #{type := OldType}} = NewConfig +) -> + {ExporterMod, OldExporterOpts} = exporter(OldConfig), + {_NewExporterMod, NewExporterOpts} = exporter(NewConfig), + ExporterMod:update(OldExporterOpts, NewExporterOpts); +update_exporter( + #{exporter := _} = OldConfig, + #{exporter := _} = NewConfig +) -> + {OldExporterMod, OldExporterOpts} = exporter(OldConfig), + {NewExporterMod, NewExporterOpts} = exporter(NewConfig), + ok = OldExporterMod:stop(OldExporterOpts), + NewExporterMod:start(NewExporterOpts); +update_exporter(undefined, NewConfig) -> + {ExporterMod, ExporterOpts} = exporter(NewConfig), + ExporterMod:start(ExporterOpts); +update_exporter(OldConfig, undefined) -> + {ExporterMod, ExporterOpts} = exporter(OldConfig), + ExporterMod:stop(ExporterOpts); +update_exporter(_, _) -> + ok. %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ @@ -138,7 +177,6 @@ exporter(Storage) -> {emqx_ft_storage_exporter_s3, without_type(Options)} end. --spec without_type(exporter_conf()) -> exporter_conf(). without_type(#{type := _} = Options) -> maps:without([type], Options). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs.erl similarity index 91% rename from apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl rename to apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs.erl index fd1956009..f84189f7f 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs.erl @@ -28,6 +28,12 @@ -export([discard/1]). -export([list/1]). +-export([ + start/1, + stop/1, + update/2 +]). + %% Internal API for RPC -export([list_local/1]). -export([list_local/2]). @@ -88,7 +94,9 @@ }) ). -%% +%%-------------------------------------------------------------------- +%% Exporter behaviour +%%-------------------------------------------------------------------- -spec start_export(options(), transfer(), filemeta()) -> {ok, export_st()} | {error, file_error()}. @@ -142,7 +150,25 @@ discard(#{path := Filepath, handle := Handle}) -> ok = file:close(Handle), file:delete(Filepath). -%% +%%-------------------------------------------------------------------- +%% Exporter behaviour (lifecycle) +%%-------------------------------------------------------------------- + +%% FS Exporter does not have require any stateful entities, +%% so lifecycle callbacks are no-op. + +-spec start(options()) -> ok. +start(_Options) -> ok. + +-spec stop(options()) -> ok. +stop(_Options) -> ok. + +-spec update(options(), options()) -> ok. +update(_OldOptions, _NewOptions) -> ok. + +%%-------------------------------------------------------------------- +%% Internal API +%%-------------------------------------------------------------------- -spec list_local(options(), transfer()) -> {ok, [exportinfo(), ...]} | {error, file_error()}. @@ -199,6 +225,10 @@ list_local(Options) -> Pattern )}. +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + filter_manifest(?MANIFEST) -> % Filename equals `?MANIFEST`, there should also be a manifest for it. false; diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs_api.erl similarity index 100% rename from apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl rename to apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs_api.erl diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs_proxy.erl similarity index 100% rename from apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl rename to apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs_proxy.erl diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_s3.erl similarity index 68% rename from apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl rename to apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_s3.erl index 300f0dc85..977ff576a 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_s3.erl @@ -25,6 +25,12 @@ -export([discard/1]). -export([list/1]). +-export([ + start/1, + stop/1, + update/2 +]). + -type options() :: emqx_s3:profile_config(). -type transfer() :: emqx_ft:transfer(). -type filemeta() :: emqx_ft:filemeta(). @@ -42,6 +48,10 @@ meta := filemeta() }. +%%-------------------------------------------------------------------- +%% Exporter behaviour +%%-------------------------------------------------------------------- + -spec start_export(options(), transfer(), filemeta()) -> {ok, export_st()} | {error, term()}. start_export(_Options, _Transfer, Filemeta = #{name := Filename}) -> @@ -67,3 +77,26 @@ discard(_ExportSt) -> {ok, [exportinfo()]} | {error, term()}. list(_Options) -> {ok, []}. + +%%-------------------------------------------------------------------- +%% Exporter behaviour (lifecycle) +%%-------------------------------------------------------------------- + +-spec start(options()) -> ok | {error, term()}. +start(Options) -> + emqx_s3:start_profile(s3_profile_id(), Options). + +-spec stop(options()) -> ok. +stop(_Options) -> + ok = emqx_s3:stop_profile(s3_profile_id()). + +-spec update(options(), options()) -> ok. +update(_OldOptions, NewOptions) -> + emqx_s3:update_profile(s3_profile_id(), NewOptions). + +%%-------------------------------------------------------------------- +%% Internal functions +%% ------------------------------------------------------------------- + +s3_profile_id() -> + atom_to_binary(?MODULE). From 43f973742033fa6eb14e6bdf278facc27f13731c Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 3 Apr 2023 20:35:47 +0300 Subject: [PATCH 089/156] feat(ft-s3): integrate list API --- apps/emqx_ft/src/emqx_ft.erl | 1 - apps/emqx_ft/src/emqx_ft_api.erl | 8 +- .../emqx_ft_storage_exporter_s3.erl | 102 ---------- .../emqx_ft_storage_exporter_fs.erl | 0 .../emqx_ft_storage_exporter_fs_api.erl | 0 .../emqx_ft_storage_exporter_fs_proxy.erl | 0 .../src/emqx_ft_storage_exporter_s3.erl | 188 ++++++++++++++++++ apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf | 12 +- apps/emqx_s3/src/emqx_s3.erl | 3 + apps/emqx_s3/src/emqx_s3_client.erl | 89 +++++---- apps/emqx_s3/src/emqx_s3_profile_conf.erl | 1 + apps/emqx_s3/src/emqx_s3_schema.erl | 9 + apps/emqx_s3/src/emqx_s3_uploader.erl | 49 +++-- apps/emqx_s3/test/emqx_s3_client_SUITE.erl | 33 +++ 14 files changed, 333 insertions(+), 162 deletions(-) delete mode 100644 apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_s3.erl rename apps/emqx_ft/src/{emqx_ft_storage_exporter => }/emqx_ft_storage_exporter_fs.erl (100%) rename apps/emqx_ft/src/{emqx_ft_storage_exporter => }/emqx_ft_storage_exporter_fs_api.erl (100%) rename apps/emqx_ft/src/{emqx_ft_storage_exporter => }/emqx_ft_storage_exporter_fs_proxy.erl (100%) create mode 100644 apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 064b6e066..c5522cdbd 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -118,7 +118,6 @@ decode_filemeta(Map) when is_map(Map) -> end. encode_filemeta(Meta = #{}) -> - % TODO: Looks like this should be hocon's responsibility. Schema = emqx_ft_schema:schema(filemeta), hocon_tconf:make_serializable(Schema, emqx_map_lib:binary_key_map(Meta), #{}). diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index a667454b6..390c10557 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -69,6 +69,7 @@ schema("/file_transfer/files") -> '/file_transfer/files'(get, #{}) -> case emqx_ft_storage:files() of {ok, Files} -> + ?SLOG(warning, #{msg => "files", files => Files}), {200, #{<<"files">> => lists:map(fun format_file_info/1, Files)}}; {error, _} -> {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} @@ -94,7 +95,7 @@ format_file_info( } ) -> Res = #{ - name => iolist_to_binary(Name), + name => format_name(Name), size => Size, timestamp => format_timestamp(Timestamp), clientid => ClientId, @@ -110,3 +111,8 @@ format_file_info( format_timestamp(Timestamp) -> iolist_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])). + +format_name(NameBin) when is_binary(NameBin) -> + NameBin; +format_name(Name) when is_list(Name) -> + iolist_to_binary(Name). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_s3.erl deleted file mode 100644 index 977ff576a..000000000 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_s3.erl +++ /dev/null @@ -1,102 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_ft_storage_exporter_s3). - --include_lib("emqx/include/logger.hrl"). - -%% Exporter API --export([start_export/3]). --export([write/2]). --export([complete/1]). --export([discard/1]). --export([list/1]). - --export([ - start/1, - stop/1, - update/2 -]). - --type options() :: emqx_s3:profile_config(). --type transfer() :: emqx_ft:transfer(). --type filemeta() :: emqx_ft:filemeta(). --type exportinfo() :: #{ - transfer := transfer(), - name := file:name(), - uri := uri_string:uri_string(), - timestamp := emqx_datetime:epoch_second(), - size := _Bytes :: non_neg_integer(), - meta => filemeta() -}. - --type export_st() :: #{ - pid := pid(), - meta := filemeta() -}. - -%%-------------------------------------------------------------------- -%% Exporter behaviour -%%-------------------------------------------------------------------- - --spec start_export(options(), transfer(), filemeta()) -> - {ok, export_st()} | {error, term()}. -start_export(_Options, _Transfer, Filemeta = #{name := Filename}) -> - Pid = spawn(fun() -> Filename end), - #{meta => Filemeta, pid => Pid}. - --spec write(export_st(), iodata()) -> - {ok, export_st()} | {error, term()}. -write(ExportSt, _IoData) -> - {ok, ExportSt}. - --spec complete(export_st()) -> - ok | {error, term()}. -complete(_ExportSt) -> - ok. - --spec discard(export_st()) -> - ok. -discard(_ExportSt) -> - ok. - --spec list(options()) -> - {ok, [exportinfo()]} | {error, term()}. -list(_Options) -> - {ok, []}. - -%%-------------------------------------------------------------------- -%% Exporter behaviour (lifecycle) -%%-------------------------------------------------------------------- - --spec start(options()) -> ok | {error, term()}. -start(Options) -> - emqx_s3:start_profile(s3_profile_id(), Options). - --spec stop(options()) -> ok. -stop(_Options) -> - ok = emqx_s3:stop_profile(s3_profile_id()). - --spec update(options(), options()) -> ok. -update(_OldOptions, NewOptions) -> - emqx_s3:update_profile(s3_profile_id(), NewOptions). - -%%-------------------------------------------------------------------- -%% Internal functions -%% ------------------------------------------------------------------- - -s3_profile_id() -> - atom_to_binary(?MODULE). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl similarity index 100% rename from apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs.erl rename to apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs_api.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl similarity index 100% rename from apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs_api.erl rename to apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl similarity index 100% rename from apps/emqx_ft/src/emqx_ft_storage_exporter/emqx_ft_storage_exporter_fs_proxy.erl rename to apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl new file mode 100644 index 000000000..1c97520e3 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -0,0 +1,188 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_exporter_s3). + +-include_lib("emqx/include/logger.hrl"). + +%% Exporter API +-export([start_export/3]). +-export([write/2]). +-export([complete/2]). +-export([discard/1]). +-export([list/1]). + +-export([ + start/1, + stop/1, + update/2 +]). + +-type options() :: emqx_s3:profile_config(). +-type transfer() :: emqx_ft:transfer(). +-type filemeta() :: emqx_ft:filemeta(). +-type exportinfo() :: #{ + transfer := transfer(), + name := file:name(), + uri := uri_string:uri_string(), + timestamp := emqx_datetime:epoch_second(), + size := _Bytes :: non_neg_integer(), + filemeta => filemeta() +}. + +-type export_st() :: #{ + pid := pid(), + filemeta := filemeta(), + transfer := transfer() +}. + +-define(S3_PROFILE_ID, <<"emqx_ft_storage_exporter_s3">>). +-define(FILEMETA_VSN, <<"1">>). +-define(S3_LIST_LIMIT, 500). + +%%-------------------------------------------------------------------- +%% Exporter behaviour +%%-------------------------------------------------------------------- + +-spec start_export(options(), transfer(), filemeta()) -> + {ok, export_st()} | {error, term()}. +start_export(_Options, Transfer, Filemeta) -> + Options = #{ + key => s3_key(Transfer, Filemeta), + headers => s3_headers(Transfer, Filemeta) + }, + case emqx_s3:start_uploader(?S3_PROFILE_ID, Options) of + {ok, Pid} -> + {ok, #{filemeta => Filemeta, pid => Pid}}; + {error, _Reason} = Error -> + Error + end. + +-spec write(export_st(), iodata()) -> + {ok, export_st()} | {error, term()}. +write(#{pid := Pid} = ExportSt, IoData) -> + case emqx_s3_uploader:write(Pid, IoData) of + ok -> + {ok, ExportSt}; + {error, _Reason} = Error -> + Error + end. + +-spec complete(export_st(), emqx_ft:checksum()) -> + ok | {error, term()}. +complete(#{pid := Pid} = _ExportSt, _Checksum) -> + emqx_s3_uploader:complete(Pid). + +-spec discard(export_st()) -> + ok. +discard(#{pid := Pid} = _ExportSt) -> + emqx_s3_uploader:abort(Pid). + +-spec list(options()) -> + {ok, [exportinfo()]} | {error, term()}. +list(Options) -> + emqx_s3:with_client(?S3_PROFILE_ID, fun(Client) -> list(Client, Options) end). + +%%-------------------------------------------------------------------- +%% Exporter behaviour (lifecycle) +%%-------------------------------------------------------------------- + +-spec start(options()) -> ok | {error, term()}. +start(Options) -> + emqx_s3:start_profile(?S3_PROFILE_ID, Options). + +-spec stop(options()) -> ok. +stop(_Options) -> + ok = emqx_s3:stop_profile(?S3_PROFILE_ID). + +-spec update(options(), options()) -> ok. +update(_OldOptions, NewOptions) -> + emqx_s3:update_profile(?S3_PROFILE_ID, NewOptions). + +%%-------------------------------------------------------------------- +%% Internal functions +%% ------------------------------------------------------------------- + +s3_key({ClientId, FileId} = _Transfer, #{name := Filename}) -> + filename:join([ + emqx_ft_fs_util:escape_filename(ClientId), + emqx_ft_fs_util:escape_filename(FileId), + Filename + ]). + +s3_headers({ClientId, FileId}, Filemeta) -> + #{ + %% The ClientID MUST be a UTF-8 Encoded String + <<"x-amz-meta-clientid">> => ClientId, + %% It [Topic Name] MUST be a UTF-8 Encoded String + <<"x-amz-meta-fileid">> => FileId, + <<"x-amz-meta-filemeta">> => emqx_json:encode(emqx_ft:encode_filemeta(Filemeta)), + <<"x-amz-meta-filemeta-vsn">> => ?FILEMETA_VSN + }. + +list(Client, Options) -> + case list_key_info(Client, Options) of + {ok, KeyInfos} -> + {ok, + lists:map( + fun(KeyInfo) -> key_info_to_exportinfo(Client, KeyInfo, Options) end, KeyInfos + )}; + {error, _Reason} = Error -> + Error + end. + +list_key_info(Client, Options) -> + list_key_info(Client, Options, _Marker = [], _Acc = []). + +list_key_info(Client, Options, Marker, Acc) -> + ListOptions = [{max_keys, ?S3_LIST_LIMIT}] ++ Marker, + case emqx_s3_client:list(Client, ListOptions) of + {ok, Result} -> + ?SLOG(warning, #{msg => "list_key_info", result => Result}), + KeyInfos = proplists:get_value(contents, Result, []), + case proplists:get_value(is_truncated, Result, false) of + true -> + NewMarker = [{marker, proplists:get_value(next_marker, Result)}], + list_key_info(Client, Options, NewMarker, [KeyInfos | Acc]); + false -> + {ok, lists:append(lists:reverse([KeyInfos | Acc]))} + end; + {error, _Reason} = Error -> + Error + end. + +key_info_to_exportinfo(Client, KeyInfo, _Options) -> + Key = proplists:get_value(key, KeyInfo), + {Transfer, Name} = parse_transfer_and_name(Key), + #{ + transfer => Transfer, + name => unicode:characters_to_binary(Name), + uri => emqx_s3_client:uri(Client, Key), + timestamp => datetime_to_epoch_second(proplists:get_value(last_modified, KeyInfo)), + size => proplists:get_value(size, KeyInfo) + }. + +-define(EPOCH_START, 62167219200). + +datetime_to_epoch_second(DateTime) -> + calendar:datetime_to_gregorian_seconds(DateTime) - ?EPOCH_START. + +parse_transfer_and_name(Key) -> + [ClientId, FileId, Name] = string:split(Key, "/", all), + Transfer = { + emqx_ft_fs_util:unescape_filename(ClientId), emqx_ft_fs_util:unescape_filename(FileId) + }, + {Transfer, Name}. diff --git a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf index 4e0870bae..99004d62a 100644 --- a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf +++ b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf @@ -39,9 +39,17 @@ emqx_s3_schema { en: "S3 endpoint port" } } + url_expire_time { + desc { + en: "The time in seconds for which the signed URLs to the S3 objects are valid." + } + label { + en: "Signed URL expiration time" + } + } min_part_size { desc { - en: """The minimum part size for multipart uploads. + en: """The minimum part size for multipart uploads.
Uploaded data will be accumulated in memory until this size is reached.""" } label { @@ -50,7 +58,7 @@ Uploaded data will be accumulated in memory until this size is reached.""" } max_part_size { desc { - en: """The maximum part size for multipart uploads. + en: """The maximum part size for multipart uploads.
S3 uploader won't try to upload parts larger than this size.""" } label { diff --git a/apps/emqx_s3/src/emqx_s3.erl b/apps/emqx_s3/src/emqx_s3.erl index 7c78de979..8798b608d 100644 --- a/apps/emqx_s3/src/emqx_s3.erl +++ b/apps/emqx_s3/src/emqx_s3.erl @@ -41,8 +41,11 @@ -type profile_config() :: #{ bucket := string(), + access_key_id => string(), + secret_access_key => string(), host := string(), port := pos_integer(), + url_expire_time := pos_integer(), acl => acl(), min_part_size => pos_integer(), transport_options => transport_options() diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index 01d677922..a4d470a97 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -8,9 +8,6 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("erlcloud/include/erlcloud_aws.hrl"). --compile(nowarn_export_all). --compile(export_all). - -export([ create/1, @@ -21,11 +18,16 @@ complete_multipart/4, abort_multipart/3, list/2, + uri/2, - format/1 + format/1, + format_request/1 ]). --export_type([client/0]). +-export_type([ + client/0, + headers/0 +]). -type s3_bucket_acl() :: private @@ -35,7 +37,7 @@ | bucket_owner_read | bucket_owner_full_control. --type headers() :: #{binary() => binary()}. +-type headers() :: #{binary() | string() => binary() | string()}. -type key() :: string(). -type part_number() :: non_neg_integer(). @@ -58,6 +60,7 @@ bucket := string(), headers := headers(), acl := s3_bucket_acl(), + url_expire_time := pos_integer(), access_key_id := string() | undefined, secret_access_key := string() | undefined, http_pool := ecpool:pool_name(), @@ -76,6 +79,7 @@ create(Config) -> aws_config => aws_config(Config), upload_options => upload_options(Config), bucket => maps:get(bucket, Config), + url_expire_time => maps:get(url_expire_time, Config), headers => headers(Config) }. @@ -85,7 +89,7 @@ put_object( Key, Value ) -> - try erlcloud_s3:put_object(Bucket, Key, Value, Options, Headers, AwsConfig) of + try erlcloud_s3:put_object(Bucket, key(Key), Value, Options, Headers, AwsConfig) of Props when is_list(Props) -> ok catch @@ -99,7 +103,7 @@ start_multipart( #{bucket := Bucket, upload_options := Options, headers := Headers, aws_config := AwsConfig}, Key ) -> - case erlcloud_s3:start_multipart(Bucket, Key, Options, Headers, AwsConfig) of + case erlcloud_s3:start_multipart(Bucket, key(Key), Options, Headers, AwsConfig) of {ok, Props} -> {ok, proplists:get_value(uploadId, Props)}; {error, Reason} -> @@ -116,7 +120,9 @@ upload_part( PartNumber, Value ) -> - case erlcloud_s3:upload_part(Bucket, Key, UploadId, PartNumber, Value, Headers, AwsConfig) of + case + erlcloud_s3:upload_part(Bucket, key(Key), UploadId, PartNumber, Value, Headers, AwsConfig) + of {ok, Props} -> {ok, proplists:get_value(etag, Props)}; {error, Reason} -> @@ -126,9 +132,12 @@ upload_part( -spec complete_multipart(client(), key(), upload_id(), [etag()]) -> ok_or_error(term()). complete_multipart( - #{bucket := Bucket, headers := Headers, aws_config := AwsConfig}, Key, UploadId, ETags + #{bucket := Bucket, headers := Headers, aws_config := AwsConfig}, + Key, + UploadId, + ETags ) -> - case erlcloud_s3:complete_multipart(Bucket, Key, UploadId, ETags, Headers, AwsConfig) of + case erlcloud_s3:complete_multipart(Bucket, key(Key), UploadId, ETags, Headers, AwsConfig) of ok -> ok; {error, Reason} -> @@ -138,7 +147,7 @@ complete_multipart( -spec abort_multipart(client(), key(), upload_id()) -> ok_or_error(term()). abort_multipart(#{bucket := Bucket, headers := Headers, aws_config := AwsConfig}, Key, UploadId) -> - case erlcloud_s3:abort_multipart(Bucket, Key, UploadId, [], Headers, AwsConfig) of + case erlcloud_s3:abort_multipart(Bucket, key(Key), UploadId, [], Headers, AwsConfig) of ok -> ok; {error, Reason} -> @@ -156,6 +165,10 @@ list(#{bucket := Bucket, aws_config := AwsConfig}, Options) -> {error, Reason} end. +-spec uri(client(), key()) -> iodata(). +uri(#{bucket := Bucket, aws_config := AwsConfig, url_expire_time := ExpireTime}, Key) -> + erlcloud_s3:make_get_url(ExpireTime, Bucket, key(Key), AwsConfig). + -spec format(client()) -> term(). format(#{aws_config := AwsConfig} = Client) -> Client#{aws_config => AwsConfig#aws_config{secret_access_key = "***"}}. @@ -170,13 +183,14 @@ upload_options(Config) -> ]. headers(#{headers := Headers}) -> - maps:to_list(Headers). + string_headers(maps:to_list(Headers)); +headers(#{}) -> + []. aws_config(#{ scheme := Scheme, host := Host, port := Port, - headers := Headers, access_key_id := AccessKeyId, secret_access_key := SecretAccessKey, http_pool := HttpPool, @@ -187,23 +201,29 @@ aws_config(#{ s3_host = Host, s3_port = Port, s3_bucket_access_method = path, + s3_bucket_after_host = true, access_key_id = AccessKeyId, secret_access_key = SecretAccessKey, - http_client = request_fun(Headers, HttpPool), + http_client = request_fun(HttpPool), timeout = Timeout }. --type http_headers() :: [{binary(), binary()}]. -type http_pool() :: term(). --spec request_fun(http_headers(), http_pool()) -> erlcloud_httpc:request_fun(). -request_fun(CustomHeaders, HttpPool) -> +-spec request_fun(http_pool()) -> erlcloud_httpc:request_fun(). +request_fun(HttpPool) -> fun(Url, Method, Headers, Body, Timeout, _Config) -> with_path_and_query_only(Url, fun(PathQuery) -> - JoinedHeaders = join_headers(Headers, CustomHeaders), - Request = make_request(Method, PathQuery, JoinedHeaders, Body), + Request = make_request(Method, PathQuery, binary_headers(Headers), Body), + ?SLOG(warning, #{ + msg => "s3_ehttpc_request", + timeout => Timeout, + pool => HttpPool, + method => Method, + request => Request + }), ehttpc_request(HttpPool, Method, Request, Timeout) end) end. @@ -211,9 +231,9 @@ request_fun(CustomHeaders, HttpPool) -> ehttpc_request(HttpPool, Method, Request, Timeout) -> try ehttpc:request(HttpPool, Method, Request, Timeout) of {ok, StatusCode, RespHeaders} -> - {ok, {{StatusCode, undefined}, string_headers(RespHeaders), undefined}}; + {ok, {{StatusCode, undefined}, erlcloud_string_headers(RespHeaders), undefined}}; {ok, StatusCode, RespHeaders, RespBody} -> - {ok, {{StatusCode, undefined}, string_headers(RespHeaders), RespBody}}; + {ok, {{StatusCode, undefined}, erlcloud_string_headers(RespHeaders), RespBody}}; {error, Reason} -> ?SLOG(error, #{ msg => "s3_ehttpc_request_fail", @@ -258,16 +278,6 @@ make_request(_Method, PathQuery, Headers, Body) -> format_request({PathQuery, Headers, _Body}) -> {PathQuery, Headers, <<"...">>}. -join_headers(Headers, CustomHeaders) -> - MapHeaders = lists:foldl( - fun({K, V}, MHeaders) -> - maps:put(to_binary(K), V, MHeaders) - end, - #{}, - Headers ++ maps:to_list(CustomHeaders) - ), - maps:to_list(MapHeaders). - with_path_and_query_only(Url, Fun) -> case string:split(Url, "//", leading) of [_Scheme, UrlRem] -> @@ -281,13 +291,22 @@ with_path_and_query_only(Url, Fun) -> {error, {invalid_url, Url}} end. +string_headers(Headers) -> + [{to_list_string(K), to_list_string(V)} || {K, V} <- Headers]. + +erlcloud_string_headers(Headers) -> + [{string:to_lower(K), V} || {K, V} <- string_headers(Headers)]. + +binary_headers(Headers) -> + [{to_binary(K), V} || {K, V} <- Headers]. + to_binary(Val) when is_list(Val) -> list_to_binary(Val); to_binary(Val) when is_binary(Val) -> Val. -string_headers(Hdrs) -> - [{string:to_lower(to_list_string(K)), to_list_string(V)} || {K, V} <- Hdrs]. - to_list_string(Val) when is_binary(Val) -> binary_to_list(Val); to_list_string(Val) when is_list(Val) -> Val. + +key(Characters) -> + binary_to_list(unicode:characters_to_binary(Characters)). diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index 09e945edc..3c5eb8f36 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -211,6 +211,7 @@ client_config(ProfileConfig, PoolName) -> scheme => scheme(HTTPOpts), host => maps:get(host, ProfileConfig), port => maps:get(port, ProfileConfig), + url_expire_time => maps:get(url_expire_time, ProfileConfig), headers => maps:get(headers, HTTPOpts, #{}), acl => maps:get(acl, ProfileConfig), bucket => maps:get(bucket, ProfileConfig), diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index 5d76e7120..6db9a5886 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -63,6 +63,15 @@ fields(s3) -> required => true } )}, + {url_expire_time, + mk( + emqx_schema:duration_s(), + #{ + default => "1h", + desc => ?DESC("url_expire_time"), + required => false + } + )}, {min_part_size, mk( emqx_schema:bytesize(), diff --git a/apps/emqx_s3/src/emqx_s3_uploader.erl b/apps/emqx_s3/src/emqx_s3_uploader.erl index 4e3fe15f2..07ae5bd19 100644 --- a/apps/emqx_s3/src/emqx_s3_uploader.erl +++ b/apps/emqx_s3/src/emqx_s3_uploader.erl @@ -12,8 +12,13 @@ start_link/2, write/2, + write/3, + complete/1, - abort/1 + complete/2, + + abort/1, + abort/2 ]). -export([ @@ -26,14 +31,11 @@ format_status/2 ]). --export_type([opts/0, config/0]). +-export_type([opts/0]). -type opts() :: #{ - name := string() -}. - --type config() :: #{ - min_part_size := pos_integer() + key := string(), + headers => emqx_s3_client:headers() }. -type data() :: #{ @@ -58,12 +60,12 @@ start_link(ProfileId, #{key := Key} = Opts) when is_list(Key) -> gen_statem:start_link(?MODULE, [ProfileId, Opts], []). --spec write(pid(), binary()) -> ok_or_error(term()). -write(Pid, WriteData) when is_binary(WriteData) -> +-spec write(pid(), iodata()) -> ok_or_error(term()). +write(Pid, WriteData) -> write(Pid, WriteData, infinity). --spec write(pid(), binary(), timeout()) -> ok_or_error(term()). -write(Pid, WriteData, Timeout) when is_binary(WriteData) -> +-spec write(pid(), iodata(), timeout()) -> ok_or_error(term()). +write(Pid, WriteData, Timeout) -> gen_statem:call(Pid, {write, wrap(WriteData)}, Timeout). -spec complete(pid()) -> ok_or_error(term()). @@ -88,10 +90,10 @@ abort(Pid, Timeout) -> callback_mode() -> handle_event_function. -init([ProfileId, #{key := Key}]) -> +init([ProfileId, #{key := Key} = Opts]) -> process_flag(trap_exit, true), {ok, ClientConfig, UploaderConfig} = emqx_s3_profile_conf:checkout_config(ProfileId), - Client = emqx_s3_client:create(ClientConfig), + Client = client(ClientConfig, Opts), {ok, upload_not_started, #{ profile_id => ProfileId, client => Client, @@ -111,7 +113,7 @@ handle_event({call, From}, {write, WriteDataWrapped}, State, Data0) -> true -> handle_write(State, From, WriteData, Data0); false -> - {keep_state_and_data, {reply, From, {error, {too_large, byte_size(WriteData)}}}} + {keep_state_and_data, {reply, From, {error, {too_large, iolist_size(WriteData)}}}} end; handle_event({call, From}, complete, upload_not_started, Data0) -> case put_object(Data0) of @@ -218,7 +220,6 @@ maybe_upload_part(#{buffer_size := BufferSize, min_part_size := MinPartSize} = D true -> upload_part(Data); false -> - % ct:print("buffer size: ~p, max part size: ~p, no upload", [BufferSize, MinPartSize]), {ok, Data} end. @@ -237,7 +238,6 @@ upload_part( ) -> case emqx_s3_client:upload_part(Client, Key, UploadId, PartNumber, lists:reverse(Buffer)) of {ok, ETag} -> - % ct:print("upload part ~p, etag: ~p", [PartNumber, ETag]), NewData = Data#{ buffer => [], buffer_size => 0, @@ -246,7 +246,6 @@ upload_part( }, {ok, NewData}; {error, _} = Error -> - % ct:print("upload part ~p failed: ~p", [PartNumber, Error]), Error end. @@ -260,7 +259,11 @@ complete_upload( ) -> case upload_part(Data0) of {ok, #{etags := ETags} = Data1} -> - case emqx_s3_client:complete_multipart(Client, Key, UploadId, lists:reverse(ETags)) of + case + emqx_s3_client:complete_multipart( + Client, Key, UploadId, lists:reverse(ETags) + ) + of ok -> {ok, Data1}; {error, _} = Error -> @@ -300,11 +303,11 @@ put_object( Error end. --spec append_buffer(data(), binary()) -> data(). +-spec append_buffer(data(), iodata()) -> data(). append_buffer(#{buffer := Buffer, buffer_size := BufferSize} = Data, WriteData) -> Data#{ buffer => [WriteData | Buffer], - buffer_size => BufferSize + byte_size(WriteData) + buffer_size => BufferSize + iolist_size(WriteData) }. -compile({inline, [wrap/1, unwrap/1]}). @@ -315,4 +318,8 @@ unwrap(WrappedData) -> WrappedData(). is_valid_part(WriteData, #{max_part_size := MaxPartSize, buffer_size := BufferSize}) -> - BufferSize + byte_size(WriteData) =< MaxPartSize. + BufferSize + iolist_size(WriteData) =< MaxPartSize. + +client(Config, Opts) -> + Headers = maps:get(headers, Opts, #{}), + emqx_s3_client:create(Config#{headers => Headers}). diff --git a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl index 3d0d7bb18..f5a507653 100644 --- a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl @@ -83,6 +83,39 @@ t_simple_put(Config) -> ok = emqx_s3_client:put_object(Client, Key, Data). +t_list(Config) -> + Key = ?config(key, Config), + + Client = client(Config), + + ok = emqx_s3_client:put_object(Client, Key, <<"data">>), + + {ok, List} = emqx_s3_client:list(Client, Key), + + [KeyInfo] = proplists:get_value(contents, List), + ?assertMatch( + #{ + key := Key, + size := 4, + etag := _, + last_modified := _ + }, + maps:from_list(KeyInfo) + ). + +t_url(Config) -> + Key = ?config(key, Config), + + Client = client(Config), + ok = emqx_s3_client:put_object(Client, Key, <<"data">>), + + Url = emqx_s3_client:url(Client, Key), + + ?assertMatch( + {ok, {{_StatusLine, 200, "OK"}, _Headers, "data"}}, + httpc:request(Url) + ). + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- From 9bb72ee020b5ed4350f1210aed259357b30d5360 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 3 Apr 2023 20:44:56 +0300 Subject: [PATCH 090/156] feat(ft-s3): fix behaviour specification --- apps/emqx_ft/src/emqx_ft_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index df28de218..697220e1e 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -16,7 +16,7 @@ -module(emqx_ft_schema). -% -behaviour(hocon_schema). +-behaviour(hocon_schema). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). From ca22f11161411bc5d92f2420bc5e2818626a282f Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 3 Apr 2023 20:46:59 +0300 Subject: [PATCH 091/156] feat(ft-s3): fix logging --- apps/emqx_ft/src/emqx_ft.erl | 2 +- apps/emqx_s3/src/emqx_s3_client.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index c5522cdbd..45ec2b933 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -337,7 +337,7 @@ transfer(Msg, FileId) -> {clientid_to_binary(ClientId), FileId}. on_complete(Op, {ChanPid, PacketId}, Transfer, Result) -> - ?SLOG(warning, #{ + ?SLOG(debug, #{ msg => "on_complete", operation => Op, packet_id => PacketId, diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index a4d470a97..b84022357 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -217,7 +217,7 @@ request_fun(HttpPool) -> fun(Url, Method, Headers, Body, Timeout, _Config) -> with_path_and_query_only(Url, fun(PathQuery) -> Request = make_request(Method, PathQuery, binary_headers(Headers), Body), - ?SLOG(warning, #{ + ?SLOG(debug, #{ msg => "s3_ehttpc_request", timeout => Timeout, pool => HttpPool, From 3ffa01e16085eed166faaeeb74710e1a4d6961c2 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 3 Apr 2023 20:47:41 +0300 Subject: [PATCH 092/156] feat(ft-s3): fix logging level --- apps/emqx_ft/src/emqx_ft_api.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index 390c10557..1ea710848 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -69,7 +69,6 @@ schema("/file_transfer/files") -> '/file_transfer/files'(get, #{}) -> case emqx_ft_storage:files() of {ok, Files} -> - ?SLOG(warning, #{msg => "files", files => Files}), {200, #{<<"files">> => lists:map(fun format_file_info/1, Files)}}; {error, _} -> {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} From 07ae50a54b2aed23385c119d6db74f97c2159d95 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 4 Apr 2023 12:53:58 +0300 Subject: [PATCH 093/156] fix(ft-s3): fix release settings --- apps/emqx_ft/src/emqx_ft.app.src | 3 ++- apps/emqx_machine/src/emqx_machine_boot.erl | 13 +++++++++---- mix.exs | 5 +++-- rebar.config.erl | 8 ++++---- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.app.src b/apps/emqx_ft/src/emqx_ft.app.src index 80b4b47dd..058fe984a 100644 --- a/apps/emqx_ft/src/emqx_ft.app.src +++ b/apps/emqx_ft/src/emqx_ft.app.src @@ -6,7 +6,8 @@ {applications, [ kernel, stdlib, - gproc + gproc, + emqx_s3 ]}, {env, []}, {modules, []} diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index feeb1ba75..83824db40 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -146,12 +146,17 @@ basic_reboot_apps() -> emqx_authz, emqx_slow_subs, emqx_auto_subscribe, - emqx_plugins, - emqx_s3 + emqx_plugins ], case emqx_release:edition() of - ce -> CE; - ee -> CE ++ [] + ce -> + CE; + ee -> + CE ++ + [ + emqx_s3, + emqx_ft + ] end. sorted_reboot_apps() -> diff --git a/mix.exs b/mix.exs index 9a8c22e6b..689253a76 100644 --- a/mix.exs +++ b/mix.exs @@ -293,7 +293,6 @@ defmodule EMQXUmbrella.MixProject do emqx_psk: :permanent, emqx_slow_subs: :permanent, emqx_plugins: :permanent, - emqx_ft: :permanent, emqx_mix: :none ] ++ if(enable_quicer?(), do: [quicer: :permanent], else: []) ++ @@ -308,7 +307,9 @@ defmodule EMQXUmbrella.MixProject do emqx_license: :permanent, emqx_ee_conf: :load, emqx_ee_connector: :permanent, - emqx_ee_bridge: :permanent + emqx_ee_bridge: :permanent, + emqx_s3: :permanent, + emqx_ft: :permanent ], else: [] ) diff --git a/rebar.config.erl b/rebar.config.erl index 51a6946dc..158f66cd6 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -400,9 +400,7 @@ relx_apps(ReleaseType, Edition) -> emqx_prometheus, emqx_psk, emqx_slow_subs, - emqx_plugins, - emqx_ft, - emqx_s3 + emqx_plugins ] ++ [quicer || is_quicer_supported()] ++ [bcrypt || provide_bcrypt_release(ReleaseType)] ++ @@ -424,7 +422,9 @@ relx_apps_per_edition(ee) -> emqx_license, {emqx_ee_conf, load}, emqx_ee_connector, - emqx_ee_bridge + emqx_ee_bridge, + emqx_s3, + emqx_ft ]; relx_apps_per_edition(ce) -> []. From 71965d90e41978e9aebdcbf9e32b4281fae1ce14 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 4 Apr 2023 12:55:26 +0300 Subject: [PATCH 094/156] fix(ft-s3): synchronize erlcloud version across apps --- lib-ee/emqx_ee_connector/rebar.config | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index e754bd573..9e4135fac 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -1,12 +1,12 @@ {erl_opts, [debug_info]}. {deps, [ - {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, - {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, - {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, - {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, - {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag,"3.5.16-emqx-1"}}}, - {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, - {emqx, {path, "../../apps/emqx"}} + {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, + {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, + {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, + {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, + {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag, "3.6.7-emqx-1"}}}, + {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, + {emqx, {path, "../../apps/emqx"}} ]}. {shell, [ From 4fbabe5a761461340678a6604ab7f6be35422444 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 4 Apr 2023 13:05:02 +0300 Subject: [PATCH 095/156] fix(ft-s3): fix atom and variable naming --- apps/emqx_s3/src/emqx_s3_client.erl | 2 +- apps/emqx_s3/src/emqx_s3_profile_http_pools.erl | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index b84022357..99e2de4da 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -105,7 +105,7 @@ start_multipart( ) -> case erlcloud_s3:start_multipart(Bucket, key(Key), Options, Headers, AwsConfig) of {ok, Props} -> - {ok, proplists:get_value(uploadId, Props)}; + {ok, proplists:get_value('uploadId', Props)}; {error, Reason} -> ?SLOG(debug, #{msg => "start_multipart_fail", key => Key, reason => Reason}), {error, Reason} diff --git a/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl b/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl index e1b36c3be..73774624e 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl @@ -96,11 +96,11 @@ set_outdated(ProfileId, PoolName, Timeout) -> outdated(ProfileId) -> Now = erlang:monotonic_time(millisecond), MS = ets:fun2ms( - fun(#pool{key = {ProfileId_, PoolName}, deadline = Deadline_}) when - ProfileId_ =:= ProfileId andalso - Deadline_ =/= undefined andalso Deadline_ < Now + fun(#pool{key = {CurProfileId, CurPoolName}, deadline = CurDeadline}) when + CurProfileId =:= ProfileId andalso + CurDeadline =/= undefined andalso CurDeadline < Now -> - PoolName + CurPoolName end ), ets:select(?TAB, MS). @@ -109,8 +109,8 @@ outdated(ProfileId) -> [pool_name()]. all(ProfileId) -> MS = ets:fun2ms( - fun(#pool{key = {ProfileId_, PoolName}}) when ProfileId_ =:= ProfileId -> - PoolName + fun(#pool{key = {CurProfileId, CurPoolName}}) when CurProfileId =:= ProfileId -> + CurPoolName end ), ets:select(?TAB, MS). From c7865e5eaea74a60f1f8a214040bb3e5d0ed8517 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 4 Apr 2023 14:19:58 +0300 Subject: [PATCH 096/156] fix(ft-s3): fix dependency dependencies --- apps/emqx_s3/rebar.config | 2 +- lib-ee/emqx_ee_connector/rebar.config | 2 +- mix.exs | 9 +- rebar.config | 157 ++++++++++++++------------ 4 files changed, 96 insertions(+), 74 deletions(-) diff --git a/apps/emqx_s3/rebar.config b/apps/emqx_s3/rebar.config index f8e4d4e42..92ab7895e 100644 --- a/apps/emqx_s3/rebar.config +++ b/apps/emqx_s3/rebar.config @@ -1,6 +1,6 @@ {deps, [ {emqx, {path, "../../apps/emqx"}}, - {erlcloud, {git, "https://github.com/savonarola/erlcloud", {tag, "3.6.7-emqx-1"}}} + {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.7-emqx-1"}}} ]}. {project_plugins, [erlfmt]}. diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index 9e4135fac..f36eb0ad6 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -4,7 +4,7 @@ {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, - {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag, "3.6.7-emqx-1"}}}, + {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.7-emqx-1"}}}, {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, {emqx, {path, "../../apps/emqx"}} ]}. diff --git a/mix.exs b/mix.exs index 689253a76..8cda3b9c0 100644 --- a/mix.exs +++ b/mix.exs @@ -93,7 +93,14 @@ defmodule EMQXUmbrella.MixProject do github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true}, # in conflict by grpc and eetcd {:gpb, "4.19.5", override: true, runtime: false}, - {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true} + {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true}, + {:erlcloud, github: "emqx/erlcloud", tag: "3.6.7-emqx-1", override: true}, + # erlcloud's rebar.config requires rebar3 and does not support Mix, + # so it tries to fetch deps from git. We need to override this. + {:lhttpc, "1.6.2", override: true}, + {:eini, "1.2.9", override: true}, + {:base16, "1.0.0", override: true} + # end of erlcloud's deps ] ++ emqx_apps(profile_info, version) ++ enterprise_deps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep() diff --git a/rebar.config b/rebar.config index 0cc143e71..5cedb47ff 100644 --- a/rebar.config +++ b/rebar.config @@ -7,24 +7,35 @@ %% with rebar.config.erl module. Final result is written to %% rebar.config.rendered if environment DEBUG is set. -{edoc_opts, [{preprocess,true}]}. -{erl_opts, [warn_unused_vars,warn_shadow_vars,warn_unused_import, - warn_obsolete_guard,compressed, nowarn_unused_import, - {d, snk_kind, msg} - ]}. +{edoc_opts, [{preprocess, true}]}. +{erl_opts, [ + warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard, + compressed, + nowarn_unused_import, + {d, snk_kind, msg} +]}. -{xref_checks,[undefined_function_calls,undefined_functions,locals_not_used, - deprecated_function_calls,warnings_as_errors,deprecated_functions]}. +{xref_checks, [ + undefined_function_calls, + undefined_functions, + locals_not_used, + deprecated_function_calls, + warnings_as_errors, + deprecated_functions +]}. %% Check the forbidden mnesia calls: -{xref_queries, - [ {"E || \"mnesia\":\"dirty_delete.*\"/\".*\" : Fun", []} - , {"E || \"mnesia\":\"transaction\"/\".*\" : Fun", []} - , {"E || \"mnesia\":\"async_dirty\"/\".*\" : Fun", []} - , {"E || \"mnesia\":\"clear_table\"/\".*\" : Fun", []} - , {"E || \"mnesia\":\"create_table\"/\".*\" : Fun", []} - , {"E || \"mnesia\":\"delete_table\"/\".*\" : Fun", []} - ]}. +{xref_queries, [ + {"E || \"mnesia\":\"dirty_delete.*\"/\".*\" : Fun", []}, + {"E || \"mnesia\":\"transaction\"/\".*\" : Fun", []}, + {"E || \"mnesia\":\"async_dirty\"/\".*\" : Fun", []}, + {"E || \"mnesia\":\"clear_table\"/\".*\" : Fun", []}, + {"E || \"mnesia\":\"create_table\"/\".*\" : Fun", []}, + {"E || \"mnesia\":\"delete_table\"/\".*\" : Fun", []} +]}. {dialyzer, [ {warnings, [unmatched_returns, error_handling]}, @@ -32,72 +43,76 @@ {plt_prefix, "emqx_dialyzer"}, {plt_apps, all_apps}, {statistics, true} - ] -}. +]}. {cover_opts, [verbose]}. {cover_export_enabled, true}. {cover_excl_mods, - [ %% generated protobuf modules - emqx_exproto_pb, - emqx_exhook_pb, - %% taken almost as-is from OTP - emqx_ssl_crl_cache - ]}. + %% generated protobuf modules + [ + emqx_exproto_pb, + emqx_exhook_pb, + %% taken almost as-is from OTP + emqx_ssl_crl_cache + ]}. {provider_hooks, [{pre, [{release, {relup_helper, gen_appups}}]}]}. -{post_hooks,[]}. +{post_hooks, []}. -{deps, - [ {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}} - , {redbug, "2.0.8"} - , {covertool, {git, "https://github.com/zmstone/covertool", {tag, "2.0.4.1"}}} - , {gpb, "4.19.5"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps - , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}} - , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}} - , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.7"}}} - , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} - , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} - , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}} - , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-9"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}} - , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} - , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} - , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}} - , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.9"}}} - , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} - , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}} - , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.5"}}} - , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x - , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} - , {getopt, "1.0.2"} - , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.0"}}} - , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} - , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} - , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} - , {telemetry, "1.1.0"} - , {hackney, {git, "https://github.com/emqx/hackney.git", {tag, "1.18.1-1"}}} - ]}. +{deps, [ + {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}}, + {redbug, "2.0.8"}, + {covertool, {git, "https://github.com/zmstone/covertool", {tag, "2.0.4.1"}}}, + %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps + {gpb, "4.19.5"}, + {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}}, + {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}}, + {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.7"}}}, + {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}, + {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, + {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, + {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, + {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-9"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}}, + {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, + {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}}, + {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}}, + {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}}, + {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.9"}}}, + {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}}, + {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}}, + % NOTE: depends on recon 2.5.x + {observer_cli, "1.7.1"}, + {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}, + {getopt, "1.0.2"}, + {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.0"}}}, + {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, + {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}, + {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}, + {telemetry, "1.1.0"}, + {hackney, {git, "https://github.com/emqx/hackney.git", {tag, "1.18.1-1"}}} +]}. {xref_ignores, - [ %% schema registry is for enterprise - {emqx_schema_registry,get_all_schemas,0}, - {emqx_schema_api,format_schema,1}, - {emqx_schema_api,make_schema_params,1}, - {emqx_schema_parser,decode,3}, - {emqx_schema_parser,encode,3}, - {emqx_schema_registry,add_schema,1}, - emqx_exhook_pb, % generated code for protobuf - emqx_exproto_pb % generated code for protobuf -]}. + %% schema registry is for enterprise + [ + {emqx_schema_registry, get_all_schemas, 0}, + {emqx_schema_api, format_schema, 1}, + {emqx_schema_api, make_schema_params, 1}, + {emqx_schema_parser, decode, 3}, + {emqx_schema_parser, encode, 3}, + {emqx_schema_registry, add_schema, 1}, + % generated code for protobuf + emqx_exhook_pb, + % generated code for protobuf + emqx_exproto_pb + ]}. -{project_plugins, - [ erlfmt, +{project_plugins, [ + erlfmt, {rebar3_hex, "7.0.2"}, - {rebar3_sbom, - {git, "https://github.com/emqx/rebar3_sbom.git", {tag, "v0.6.1-1"}}} + {rebar3_sbom, {git, "https://github.com/emqx/rebar3_sbom.git", {tag, "v0.6.1-1"}}} ]}. From 7d13862da54ad5e4931d31c8a5d783488cfead03 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 4 Apr 2023 15:25:53 +0300 Subject: [PATCH 097/156] fix(ft-s3): use uploader specific headers only for object creation --- apps/emqx_s3/src/emqx_s3_client.erl | 21 +++++++++++++++++++-- apps/emqx_s3/src/emqx_s3_uploader.erl | 17 +++++++++-------- apps/emqx_s3/test/emqx_s3_client_SUITE.erl | 2 +- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index 99e2de4da..8e54d90f9 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -12,8 +12,10 @@ create/1, put_object/3, + put_object/4, start_multipart/2, + start_multipart/3, upload_part/5, complete_multipart/4, abort_multipart/3, @@ -84,12 +86,18 @@ create(Config) -> }. -spec put_object(client(), key(), iodata()) -> ok_or_error(term()). +put_object(Client, Key, Value) -> + put_object(Client, #{}, Key, Value). + +-spec put_object(client(), headers(), key(), iodata()) -> ok_or_error(term()). put_object( #{bucket := Bucket, upload_options := Options, headers := Headers, aws_config := AwsConfig}, + SpecialHeaders, Key, Value ) -> - try erlcloud_s3:put_object(Bucket, key(Key), Value, Options, Headers, AwsConfig) of + AllHeaders = join_headers(Headers, SpecialHeaders), + try erlcloud_s3:put_object(Bucket, key(Key), Value, Options, AllHeaders, AwsConfig) of Props when is_list(Props) -> ok catch @@ -99,11 +107,17 @@ put_object( end. -spec start_multipart(client(), key()) -> ok_or_error(upload_id(), term()). +start_multipart(Client, Key) -> + start_multipart(Client, #{}, Key). + +-spec start_multipart(client(), headers(), key()) -> ok_or_error(upload_id(), term()). start_multipart( #{bucket := Bucket, upload_options := Options, headers := Headers, aws_config := AwsConfig}, + SpecialHeaders, Key ) -> - case erlcloud_s3:start_multipart(Bucket, key(Key), Options, Headers, AwsConfig) of + AllHeaders = join_headers(Headers, SpecialHeaders), + case erlcloud_s3:start_multipart(Bucket, key(Key), Options, AllHeaders, AwsConfig) of {ok, Props} -> {ok, proplists:get_value('uploadId', Props)}; {error, Reason} -> @@ -300,6 +314,9 @@ erlcloud_string_headers(Headers) -> binary_headers(Headers) -> [{to_binary(K), V} || {K, V} <- Headers]. +join_headers(Headers, SpecialHeaders) -> + Headers ++ string_headers(maps:to_list(SpecialHeaders)). + to_binary(Val) when is_list(Val) -> list_to_binary(Val); to_binary(Val) when is_binary(Val) -> Val. diff --git a/apps/emqx_s3/src/emqx_s3_uploader.erl b/apps/emqx_s3/src/emqx_s3_uploader.erl index 07ae5bd19..f6414669d 100644 --- a/apps/emqx_s3/src/emqx_s3_uploader.erl +++ b/apps/emqx_s3/src/emqx_s3_uploader.erl @@ -93,10 +93,11 @@ callback_mode() -> handle_event_function. init([ProfileId, #{key := Key} = Opts]) -> process_flag(trap_exit, true), {ok, ClientConfig, UploaderConfig} = emqx_s3_profile_conf:checkout_config(ProfileId), - Client = client(ClientConfig, Opts), + Client = client(ClientConfig), {ok, upload_not_started, #{ profile_id => ProfileId, client => Client, + headers => maps:get(headers, Opts, #{}), key => Key, buffer => [], buffer_size => 0, @@ -205,8 +206,8 @@ maybe_start_upload(#{buffer_size := BufferSize, min_part_size := MinPartSize} = end. -spec start_upload(data()) -> {started, data()} | {error, term()}. -start_upload(#{client := Client, key := Key} = Data) -> - case emqx_s3_client:start_multipart(Client, Key) of +start_upload(#{client := Client, key := Key, headers := Headers} = Data) -> + case emqx_s3_client:start_multipart(Client, Headers, Key) of {ok, UploadId} -> NewData = Data#{upload_id => UploadId}, {started, NewData}; @@ -293,10 +294,11 @@ put_object( #{ client := Client, key := Key, - buffer := Buffer + buffer := Buffer, + headers := Headers } ) -> - case emqx_s3_client:put_object(Client, Key, lists:reverse(Buffer)) of + case emqx_s3_client:put_object(Client, Headers, Key, lists:reverse(Buffer)) of ok -> ok; {error, _} = Error -> @@ -320,6 +322,5 @@ unwrap(WrappedData) -> is_valid_part(WriteData, #{max_part_size := MaxPartSize, buffer_size := BufferSize}) -> BufferSize + iolist_size(WriteData) =< MaxPartSize. -client(Config, Opts) -> - Headers = maps:get(headers, Opts, #{}), - emqx_s3_client:create(Config#{headers => Headers}). +client(Config) -> + emqx_s3_client:create(Config). diff --git a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl index f5a507653..ec7d5ebcf 100644 --- a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl @@ -109,7 +109,7 @@ t_url(Config) -> Client = client(Config), ok = emqx_s3_client:put_object(Client, Key, <<"data">>), - Url = emqx_s3_client:url(Client, Key), + Url = emqx_s3_client:uri(Client, Key), ?assertMatch( {ok, {{_StatusLine, 200, "OK"}, _Headers, "data"}}, From d7a85242de46fa47f4b0716ce8f644bb8c6672bc Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 4 Apr 2023 17:12:52 +0300 Subject: [PATCH 098/156] fix(ft-s3): fix s3 listing --- apps/emqx/src/emqx_channel.erl | 7 ------- apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl | 5 ++++- apps/emqx_s3/src/emqx_s3_client.erl | 11 +++++++++++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index c67c02d66..9d4b7eac2 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -641,13 +641,6 @@ process_connect( %%-------------------------------------------------------------------- process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) -> - ?SLOG( - warning, - #{ - packet => Packet, - packet_id => PacketId - } - ), case pipeline( [ diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index 1c97520e3..3258a5cd1 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -155,7 +155,7 @@ list_key_info(Client, Options, Marker, Acc) -> KeyInfos = proplists:get_value(contents, Result, []), case proplists:get_value(is_truncated, Result, false) of true -> - NewMarker = [{marker, proplists:get_value(next_marker, Result)}], + NewMarker = next_marker(KeyInfos), list_key_info(Client, Options, NewMarker, [KeyInfos | Acc]); false -> {ok, lists:append(lists:reverse([KeyInfos | Acc]))} @@ -164,6 +164,9 @@ list_key_info(Client, Options, Marker, Acc) -> Error end. +next_marker(KeyInfos) -> + [{marker, proplists:get_value(key, lists:last(KeyInfos))}]. + key_info_to_exportinfo(Client, KeyInfo, _Options) -> Key = proplists:get_value(key, KeyInfo), {Transfer, Name} = parse_transfer_and_name(Key), diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index 8e54d90f9..8639e0a25 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -245,8 +245,19 @@ request_fun(HttpPool) -> ehttpc_request(HttpPool, Method, Request, Timeout) -> try ehttpc:request(HttpPool, Method, Request, Timeout) of {ok, StatusCode, RespHeaders} -> + ?SLOG(debug, #{ + msg => "s3_ehttpc_request_ok", + status_code => StatusCode, + headers => RespHeaders + }), {ok, {{StatusCode, undefined}, erlcloud_string_headers(RespHeaders), undefined}}; {ok, StatusCode, RespHeaders, RespBody} -> + ?SLOG(debug, #{ + msg => "s3_ehttpc_request_ok", + status_code => StatusCode, + headers => RespHeaders, + body => RespBody + }), {ok, {{StatusCode, undefined}, erlcloud_string_headers(RespHeaders), RespBody}}; {error, Reason} -> ?SLOG(error, #{ From be99242e3279ed4a7dff72353002cda3ee0e83f1 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 4 Apr 2023 23:11:33 +0300 Subject: [PATCH 099/156] fix(ft-s3): fix review comments --- .ci/docker-compose-file/.env | 1 + .../docker-compose-minio-tcp.yaml | 2 +- .../docker-compose-minio-tls.yaml | 2 +- .github/workflows/run_test_cases.yaml | 1 + apps/emqx_ft/src/emqx_ft_conf.erl | 2 + .../src/emqx_ft_storage_exporter_s3.erl | 5 +- apps/emqx_s3/src/emqx_s3.erl | 19 +-- apps/emqx_s3/src/emqx_s3.hrl | 13 ++ apps/emqx_s3/src/emqx_s3_client.erl | 122 ++++++++++++------ apps/emqx_s3/src/emqx_s3_profile_conf.erl | 45 +++---- .../src/emqx_s3_profile_uploader_sup.erl | 14 +- apps/emqx_s3/src/emqx_s3_uploader.erl | 14 +- 12 files changed, 151 insertions(+), 89 deletions(-) create mode 100644 apps/emqx_s3/src/emqx_s3.hrl diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env index 956750e00..9675937d8 100644 --- a/.ci/docker-compose-file/.env +++ b/.ci/docker-compose-file/.env @@ -7,5 +7,6 @@ INFLUXDB_TAG=2.5.0 TDENGINE_TAG=3.0.2.4 DYNAMO_TAG=1.21.0 CASSANDRA_TAG=3.11.6 +MINIO_TAG=RELEASE.2023-03-20T20-16-18Z TARGET=emqx/emqx diff --git a/.ci/docker-compose-file/docker-compose-minio-tcp.yaml b/.ci/docker-compose-file/docker-compose-minio-tcp.yaml index 93e1c4ead..fa78e4426 100644 --- a/.ci/docker-compose-file/docker-compose-minio-tcp.yaml +++ b/.ci/docker-compose-file/docker-compose-minio-tcp.yaml @@ -3,7 +3,7 @@ version: '3.7' services: minio: hostname: minio - image: quay.io/minio/minio:RELEASE.2023-03-20T20-16-18Z + image: quay.io/minio/minio:${MINIO_TAG} command: server --address ":9000" --console-address ":9001" /minio-data expose: - "9000" diff --git a/.ci/docker-compose-file/docker-compose-minio-tls.yaml b/.ci/docker-compose-file/docker-compose-minio-tls.yaml index 2e7a6bea5..4999cccb5 100644 --- a/.ci/docker-compose-file/docker-compose-minio-tls.yaml +++ b/.ci/docker-compose-file/docker-compose-minio-tls.yaml @@ -3,7 +3,7 @@ version: '3.7' services: minio_tls: hostname: minio-tls - image: quay.io/minio/minio:RELEASE.2023-03-20T20-16-18Z + image: quay.io/minio/minio:${MINIO_TAG} command: server --certs-dir /etc/certs --address ":9100" --console-address ":9101" /minio-data volumes: - ./certs/server.crt:/etc/certs/public.crt diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 8702cd849..3360bec6c 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -168,6 +168,7 @@ jobs: REDIS_TAG: "7.0" INFLUXDB_TAG: "2.5.0" TDENGINE_TAG: "3.0.2.4" + MINIO_TAG: "RELEASE.2023-03-20T20-16-18Z" PROFILE: ${{ matrix.profile }} CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }} diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 6ce10408e..0e8bcc193 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -20,6 +20,8 @@ -behaviour(emqx_config_handler). +-include_lib("emqx/include/logger.hrl"). + %% Accessors -export([storage/0]). -export([gc_interval/1]). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index 3258a5cd1..24c3c52e5 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -49,7 +49,7 @@ transfer := transfer() }. --define(S3_PROFILE_ID, <<"emqx_ft_storage_exporter_s3">>). +-define(S3_PROFILE_ID, ?MODULE). -define(FILEMETA_VSN, <<"1">>). -define(S3_LIST_LIMIT, 500). @@ -66,6 +66,7 @@ start_export(_Options, Transfer, Filemeta) -> }, case emqx_s3:start_uploader(?S3_PROFILE_ID, Options) of {ok, Pid} -> + true = erlang:link(Pid), {ok, #{filemeta => Filemeta, pid => Pid}}; {error, _Reason} = Error -> Error @@ -151,7 +152,7 @@ list_key_info(Client, Options, Marker, Acc) -> ListOptions = [{max_keys, ?S3_LIST_LIMIT}] ++ Marker, case emqx_s3_client:list(Client, ListOptions) of {ok, Result} -> - ?SLOG(warning, #{msg => "list_key_info", result => Result}), + ?SLOG(debug, #{msg => "list_key_info", result => Result}), KeyInfos = proplists:get_value(contents, Result, []), case proplists:get_value(is_truncated, Result, false) of true -> diff --git a/apps/emqx_s3/src/emqx_s3.erl b/apps/emqx_s3/src/emqx_s3.erl index 8798b608d..15f63fbd5 100644 --- a/apps/emqx_s3/src/emqx_s3.erl +++ b/apps/emqx_s3/src/emqx_s3.erl @@ -16,10 +16,11 @@ -export_type([ profile_id/0, - profile_config/0 + profile_config/0, + acl/0 ]). --type profile_id() :: term(). +-type profile_id() :: atom() | binary(). -type acl() :: private @@ -30,9 +31,9 @@ | bucket_owner_full_control. -type transport_options() :: #{ + headers => map(), connect_timeout => pos_integer(), enable_pipelining => pos_integer(), - headers => map(), max_retries => pos_integer(), pool_size => pos_integer(), pool_type => atom(), @@ -51,12 +52,14 @@ transport_options => transport_options() }. +-define(IS_PROFILE_ID(ProfileId), (is_atom(ProfileId) orelse is_binary(ProfileId))). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- -spec start_profile(profile_id(), profile_config()) -> ok_or_error(term()). -start_profile(ProfileId, ProfileConfig) -> +start_profile(ProfileId, ProfileConfig) when ?IS_PROFILE_ID(ProfileId) -> case emqx_s3_sup:start_profile(ProfileId, ProfileConfig) of {ok, _} -> ok; @@ -65,21 +68,21 @@ start_profile(ProfileId, ProfileConfig) -> end. -spec stop_profile(profile_id()) -> ok_or_error(term()). -stop_profile(ProfileId) -> +stop_profile(ProfileId) when ?IS_PROFILE_ID(ProfileId) -> emqx_s3_sup:stop_profile(ProfileId). -spec update_profile(profile_id(), profile_config()) -> ok_or_error(term()). -update_profile(ProfileId, ProfileConfig) -> +update_profile(ProfileId, ProfileConfig) when ?IS_PROFILE_ID(ProfileId) -> emqx_s3_profile_conf:update_config(ProfileId, ProfileConfig). -spec start_uploader(profile_id(), emqx_s3_uploader:opts()) -> supervisor:start_ret() | {error, profile_not_found}. -start_uploader(ProfileId, Opts) -> +start_uploader(ProfileId, Opts) when ?IS_PROFILE_ID(ProfileId) -> emqx_s3_profile_uploader_sup:start_uploader(ProfileId, Opts). -spec with_client(profile_id(), fun((emqx_s3_client:client()) -> Result)) -> {error, profile_not_found} | Result. -with_client(ProfileId, Fun) when is_function(Fun, 1) -> +with_client(ProfileId, Fun) when is_function(Fun, 1) andalso ?IS_PROFILE_ID(ProfileId) -> case emqx_s3_profile_conf:checkout_config(ProfileId) of {ok, ClientConfig, _UploadConfig} -> try diff --git a/apps/emqx_s3/src/emqx_s3.hrl b/apps/emqx_s3/src/emqx_s3.hrl new file mode 100644 index 000000000..9a2cf8d2f --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3.hrl @@ -0,0 +1,13 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-define(VIA_GPROC(Id), {via, gproc, {n, l, Id}}). + +-define(SAFE_CALL_VIA_GPROC(Id, Message, Timeout, NoProcError), + try gen_server:call(?VIA_GPROC(Id), Message, Timeout) of + Result -> Result + catch + exit:{noproc, _} -> {error, NoProcError} + end +). diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index 8639e0a25..c84f32dc3 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -31,22 +31,14 @@ headers/0 ]). --type s3_bucket_acl() :: - private - | public_read - | public_read_write - | authenticated_read - | bucket_owner_read - | bucket_owner_full_control. - --type headers() :: #{binary() | string() => binary() | string()}. +-type headers() :: #{binary() | string() => iodata()}. -type key() :: string(). -type part_number() :: non_neg_integer(). -type upload_id() :: string(). -type etag() :: string(). --type upload_options() :: list({acl, s3_bucket_acl()}). +-type upload_options() :: list({acl, emqx_s3:acl()}). -opaque client() :: #{ aws_config := aws_config(), @@ -61,11 +53,11 @@ port := part_number(), bucket := string(), headers := headers(), - acl := s3_bucket_acl(), + acl := emqx_s3:acl(), url_expire_time := pos_integer(), access_key_id := string() | undefined, secret_access_key := string() | undefined, - http_pool := ecpool:pool_name(), + http_pool := ehttpc:pool_name(), request_timeout := timeout() }. @@ -97,7 +89,7 @@ put_object( Value ) -> AllHeaders = join_headers(Headers, SpecialHeaders), - try erlcloud_s3:put_object(Bucket, key(Key), Value, Options, AllHeaders, AwsConfig) of + try erlcloud_s3:put_object(Bucket, erlcloud_key(Key), Value, Options, AllHeaders, AwsConfig) of Props when is_list(Props) -> ok catch @@ -117,9 +109,9 @@ start_multipart( Key ) -> AllHeaders = join_headers(Headers, SpecialHeaders), - case erlcloud_s3:start_multipart(Bucket, key(Key), Options, AllHeaders, AwsConfig) of + case erlcloud_s3:start_multipart(Bucket, erlcloud_key(Key), Options, AllHeaders, AwsConfig) of {ok, Props} -> - {ok, proplists:get_value('uploadId', Props)}; + {ok, response_property('uploadId', Props)}; {error, Reason} -> ?SLOG(debug, #{msg => "start_multipart_fail", key => Key, reason => Reason}), {error, Reason} @@ -135,10 +127,12 @@ upload_part( Value ) -> case - erlcloud_s3:upload_part(Bucket, key(Key), UploadId, PartNumber, Value, Headers, AwsConfig) + erlcloud_s3:upload_part( + Bucket, erlcloud_key(Key), UploadId, PartNumber, Value, Headers, AwsConfig + ) of {ok, Props} -> - {ok, proplists:get_value(etag, Props)}; + {ok, response_property(etag, Props)}; {error, Reason} -> ?SLOG(debug, #{msg => "upload_part_fail", key => Key, reason => Reason}), {error, Reason} @@ -151,7 +145,11 @@ complete_multipart( UploadId, ETags ) -> - case erlcloud_s3:complete_multipart(Bucket, key(Key), UploadId, ETags, Headers, AwsConfig) of + case + erlcloud_s3:complete_multipart( + Bucket, erlcloud_key(Key), UploadId, ETags, Headers, AwsConfig + ) + of ok -> ok; {error, Reason} -> @@ -161,7 +159,7 @@ complete_multipart( -spec abort_multipart(client(), key(), upload_id()) -> ok_or_error(term()). abort_multipart(#{bucket := Bucket, headers := Headers, aws_config := AwsConfig}, Key, UploadId) -> - case erlcloud_s3:abort_multipart(Bucket, key(Key), UploadId, [], Headers, AwsConfig) of + case erlcloud_s3:abort_multipart(Bucket, erlcloud_key(Key), UploadId, [], Headers, AwsConfig) of ok -> ok; {error, Reason} -> @@ -181,7 +179,7 @@ list(#{bucket := Bucket, aws_config := AwsConfig}, Options) -> -spec uri(client(), key()) -> iodata(). uri(#{bucket := Bucket, aws_config := AwsConfig, url_expire_time := ExpireTime}, Key) -> - erlcloud_s3:make_get_url(ExpireTime, Bucket, key(Key), AwsConfig). + erlcloud_s3:make_get_url(ExpireTime, Bucket, erlcloud_key(Key), AwsConfig). -spec format(client()) -> term(). format(#{aws_config := AwsConfig} = Client) -> @@ -197,7 +195,7 @@ upload_options(Config) -> ]. headers(#{headers := Headers}) -> - string_headers(maps:to_list(Headers)); + headers_user_to_erlcloud_request(Headers); headers(#{}) -> []. @@ -230,7 +228,9 @@ aws_config(#{ request_fun(HttpPool) -> fun(Url, Method, Headers, Body, Timeout, _Config) -> with_path_and_query_only(Url, fun(PathQuery) -> - Request = make_request(Method, PathQuery, binary_headers(Headers), Body), + Request = make_request( + Method, PathQuery, headers_erlcloud_request_to_ehttpc(Headers), Body + ), ?SLOG(debug, #{ msg => "s3_ehttpc_request", timeout => Timeout, @@ -243,6 +243,13 @@ request_fun(HttpPool) -> end. ehttpc_request(HttpPool, Method, Request, Timeout) -> + ?SLOG(debug, #{ + msg => "s3_ehttpc_request", + timeout => Timeout, + pool => HttpPool, + method => Method, + request => format_request(Request) + }), try ehttpc:request(HttpPool, Method, Request, Timeout) of {ok, StatusCode, RespHeaders} -> ?SLOG(debug, #{ @@ -250,7 +257,9 @@ ehttpc_request(HttpPool, Method, Request, Timeout) -> status_code => StatusCode, headers => RespHeaders }), - {ok, {{StatusCode, undefined}, erlcloud_string_headers(RespHeaders), undefined}}; + {ok, { + {StatusCode, undefined}, headers_ehttpc_to_erlcloud_response(RespHeaders), undefined + }}; {ok, StatusCode, RespHeaders, RespBody} -> ?SLOG(debug, #{ msg => "s3_ehttpc_request_ok", @@ -258,7 +267,9 @@ ehttpc_request(HttpPool, Method, Request, Timeout) -> headers => RespHeaders, body => RespBody }), - {ok, {{StatusCode, undefined}, erlcloud_string_headers(RespHeaders), RespBody}}; + {ok, { + {StatusCode, undefined}, headers_ehttpc_to_erlcloud_response(RespHeaders), RespBody + }}; {error, Reason} -> ?SLOG(error, #{ msg => "s3_ehttpc_request_fail", @@ -290,10 +301,10 @@ ehttpc_request(HttpPool, Method, Request, Timeout) -> end. -define(IS_BODY_EMPTY(Body), (Body =:= undefined orelse Body =:= <<>>)). --define(NEEDS_BODY(Method), (Method =:= get orelse Method =:= head orelse Method =:= delete)). +-define(NEEDS_NO_BODY(Method), (Method =:= get orelse Method =:= head orelse Method =:= delete)). make_request(Method, PathQuery, Headers, Body) when - ?IS_BODY_EMPTY(Body) andalso ?NEEDS_BODY(Method) + ?IS_BODY_EMPTY(Body) andalso ?NEEDS_NO_BODY(Method) -> {PathQuery, Headers}; make_request(_Method, PathQuery, Headers, Body) when ?IS_BODY_EMPTY(Body) -> @@ -301,7 +312,8 @@ make_request(_Method, PathQuery, Headers, Body) when ?IS_BODY_EMPTY(Body) -> make_request(_Method, PathQuery, Headers, Body) -> {PathQuery, Headers, Body}. -format_request({PathQuery, Headers, _Body}) -> {PathQuery, Headers, <<"...">>}. +format_request({PathQuery, Headers, _Body}) -> {PathQuery, Headers, <<"...">>}; +format_request({PathQuery, Headers}) -> {PathQuery, Headers}. with_path_and_query_only(Url, Fun) -> case string:split(Url, "//", leading) of @@ -316,17 +328,41 @@ with_path_and_query_only(Url, Fun) -> {error, {invalid_url, Url}} end. -string_headers(Headers) -> - [{to_list_string(K), to_list_string(V)} || {K, V} <- Headers]. +%% We need some header conversions to tie the emqx_s3, erlcloud and ehttpc APIs together. -erlcloud_string_headers(Headers) -> - [{string:to_lower(K), V} || {K, V} <- string_headers(Headers)]. +%% The request header flow is: -binary_headers(Headers) -> - [{to_binary(K), V} || {K, V} <- Headers]. +%% UserHeaders -> [emqx_s3_client API] -> ErlcloudRequestHeaders0 -> +%% -> [erlcloud API] -> ErlcloudRequestHeaders1 -> [emqx_s3_client injected request_fun] -> +%% -> EhttpcRequestHeaders -> [ehttpc API] -join_headers(Headers, SpecialHeaders) -> - Headers ++ string_headers(maps:to_list(SpecialHeaders)). +%% The response header flow is: + +%% [ehttpc API] -> EhttpcResponseHeaders -> [emqx_s3_client injected request_fun] -> +%% -> ErlcloudResponseHeaders0 -> [erlcloud API] -> [emqx_s3_client API] + +%% UserHeders (emqx_s3 API headers) are maps with string/binary keys. +%% ErlcloudRequestHeaders are lists of tuples with string keys and iodata values +%% ErlcloudResponseHeders are lists of tuples with lower case string keys and iodata values. +%% EhttpcHeaders are lists of tuples with binary keys and iodata values. + +%% Users provide headers as a map, but erlcloud expects a list of tuples with string keys and values. +headers_user_to_erlcloud_request(UserHeaders) -> + [{to_list_string(K), V} || {K, V} <- maps:to_list(UserHeaders)]. + +%% Ehttpc returns operates on headers as a list of tuples with binary keys. +%% Erlcloud expects a list of tuples with string values and lowcase string keys +%% from the underlying http library. +headers_ehttpc_to_erlcloud_response(EhttpcHeaders) -> + [{string:to_lower(to_list_string(K)), to_list_string(V)} || {K, V} <- EhttpcHeaders]. + +%% Ehttpc expects a list of tuples with binary keys. +%% Erlcloud provides a list of tuples with string keys. +headers_erlcloud_request_to_ehttpc(ErlcloudHeaders) -> + [{to_binary(K), V} || {K, V} <- ErlcloudHeaders]. + +join_headers(ErlcloudHeaders, UserSpecialHeaders) -> + ErlcloudHeaders ++ headers_user_to_erlcloud_request(UserSpecialHeaders). to_binary(Val) when is_list(Val) -> list_to_binary(Val); to_binary(Val) when is_binary(Val) -> Val. @@ -336,5 +372,19 @@ to_list_string(Val) when is_binary(Val) -> to_list_string(Val) when is_list(Val) -> Val. -key(Characters) -> +erlcloud_key(Characters) -> binary_to_list(unicode:characters_to_binary(Characters)). + +response_property(Name, Props) -> + case proplists:get_value(Name, Props) of + undefined -> + %% This schould not happen for valid S3 implementations + ?SLOG(error, #{ + msg => "missing_s3_response_property", + name => Name, + props => Props + }), + error({missing_s3_response_property, Name}); + Value -> + Value + end. diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index 3c5eb8f36..3ab023d4d 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -11,6 +11,8 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include("src/emqx_s3.hrl"). + -export([ start_link/2, child_spec/2 @@ -47,6 +49,10 @@ -define(DEFAULT_HTTP_POOL_TIMEOUT, 60000). -define(DEAFULT_HTTP_POOL_CLEANUP_INTERVAL, 60000). +-define(SAFE_CALL_VIA_GPROC(ProfileId, Message, Timeout), + ?SAFE_CALL_VIA_GPROC(id(ProfileId), Message, Timeout, profile_not_found) +). + -spec child_spec(emqx_s3:profile_id(), emqx_s3:profile_config()) -> supervisor:child_spec(). child_spec(ProfileId, ProfileConfig) -> #{ @@ -60,7 +66,7 @@ child_spec(ProfileId, ProfileConfig) -> -spec start_link(emqx_s3:profile_id(), emqx_s3:profile_config()) -> gen_server:start_ret(). start_link(ProfileId, ProfileConfig) -> - gen_server:start_link(?MODULE, [ProfileId, ProfileConfig], []). + gen_server:start_link(?VIA_GPROC(id(ProfileId)), ?MODULE, [ProfileId, ProfileConfig], []). -spec update_config(emqx_s3:profile_id(), emqx_s3:profile_config()) -> ok_or_error(term()). update_config(ProfileId, ProfileConfig) -> @@ -69,12 +75,7 @@ update_config(ProfileId, ProfileConfig) -> -spec update_config(emqx_s3:profile_id(), emqx_s3:profile_config(), timeout()) -> ok_or_error(term()). update_config(ProfileId, ProfileConfig, Timeout) -> - case gproc:where({n, l, id(ProfileId)}) of - undefined -> - {error, profile_not_found}; - Pid -> - gen_server:call(Pid, {update_config, ProfileConfig}, Timeout) - end. + ?SAFE_CALL_VIA_GPROC(ProfileId, {update_config, ProfileConfig}, Timeout). -spec checkout_config(emqx_s3:profile_id()) -> {ok, emqx_s3_client:config(), emqx_s3_uploader:config()} | {error, profile_not_found}. @@ -84,12 +85,7 @@ checkout_config(ProfileId) -> -spec checkout_config(emqx_s3:profile_id(), timeout()) -> {ok, emqx_s3_client:config(), emqx_s3_uploader:config()} | {error, profile_not_found}. checkout_config(ProfileId, Timeout) -> - case gproc:where({n, l, id(ProfileId)}) of - undefined -> - {error, profile_not_found}; - Pid -> - gen_server:call(Pid, {checkout_config, self()}, Timeout) - end. + ?SAFE_CALL_VIA_GPROC(ProfileId, {checkout_config, self()}, Timeout). -spec checkin_config(emqx_s3:profile_id()) -> ok | {error, profile_not_found}. checkin_config(ProfileId) -> @@ -97,12 +93,7 @@ checkin_config(ProfileId) -> -spec checkin_config(emqx_s3:profile_id(), timeout()) -> ok | {error, profile_not_found}. checkin_config(ProfileId, Timeout) -> - case gproc:where({n, l, id(ProfileId)}) of - undefined -> - {error, profile_not_found}; - Pid -> - gen_server:call(Pid, {checkin_config, self()}, Timeout) - end. + ?SAFE_CALL_VIA_GPROC(ProfileId, {checkin_config, self()}, Timeout). %%-------------------------------------------------------------------- %% gen_server callbacks @@ -110,10 +101,9 @@ checkin_config(ProfileId, Timeout) -> init([ProfileId, ProfileConfig]) -> _ = process_flag(trap_exit, true), - ok = cleanup_orphaned_pools(ProfileId), + ok = cleanup_profile_pools(ProfileId), case start_http_pool(ProfileId, ProfileConfig) of {ok, PoolName} -> - true = gproc:reg({n, l, id(ProfileId)}, ignored), HttpPoolCleanupInterval = http_pool_cleanup_interval(ProfileConfig), {ok, #{ profile_id => ProfileId, @@ -188,12 +178,7 @@ handle_info(_Info, State) -> {noreply, State}. terminate(_Reason, #{profile_id := ProfileId}) -> - lists:foreach( - fun(PoolName) -> - ok = stop_http_pool(ProfileId, PoolName) - end, - emqx_s3_profile_http_pools:all(ProfileId) - ). + cleanup_profile_pools(ProfileId). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -263,12 +248,14 @@ update_http_pool(ProfileId, ProfileConfig, #{pool_name := OldPoolName} = State) pool_name(ProfileId) -> iolist_to_binary([ <<"s3-http-">>, - ProfileId, + profile_id_to_bin(ProfileId), <<"-">>, integer_to_binary(erlang:system_time(millisecond)), <<"-">>, integer_to_binary(erlang:unique_integer([positive])) ]). +profile_id_to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); +profile_id_to_bin(Bin) when is_binary(Bin) -> Bin. old_http_config(#{profile_config := ProfileConfig}) -> http_config(ProfileConfig). @@ -278,7 +265,7 @@ set_old_pool_outdated(#{ _ = emqx_s3_profile_http_pools:set_outdated(ProfileId, PoolName, HttpPoolTimeout), ok. -cleanup_orphaned_pools(ProfileId) -> +cleanup_profile_pools(ProfileId) -> lists:foreach( fun(PoolName) -> ok = stop_http_pool(ProfileId, PoolName) diff --git a/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl b/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl index 1cd155a77..fb7b93a15 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl @@ -7,6 +7,9 @@ -behaviour(supervisor). -include_lib("emqx/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-include("src/emqx_s3.hrl"). -export([ start_link/1, @@ -23,7 +26,7 @@ -spec start_link(emqx_s3:profile_id()) -> supervisor:start_ret(). start_link(ProfileId) -> - supervisor:start_link(?MODULE, [ProfileId]). + supervisor:start_link(?VIA_GPROC(id(ProfileId)), ?MODULE, [ProfileId]). -spec child_spec(emqx_s3:profile_id()) -> supervisor:child_spec(). child_spec(ProfileId) -> @@ -43,10 +46,10 @@ id(ProfileId) -> -spec start_uploader(emqx_s3:profile_id(), emqx_s3_uploader:opts()) -> supervisor:start_ret() | {error, profile_not_found}. start_uploader(ProfileId, Opts) -> - Id = id(ProfileId), - case gproc:where({n, l, Id}) of - undefined -> {error, profile_not_found}; - Pid -> supervisor:start_child(Pid, [Opts]) + try supervisor:start_child(?VIA_GPROC(id(ProfileId)), [Opts]) of + Result -> Result + catch + exit:{noproc, _} -> {error, profile_not_found} end. %%-------------------------------------------------------------------- @@ -54,7 +57,6 @@ start_uploader(ProfileId, Opts) -> %%------------------------------------------------------------------- init([ProfileId]) -> - true = gproc:reg({n, l, id(ProfileId)}, ignored), SupFlags = #{ strategy => simple_one_for_one, intensity => 10, diff --git a/apps/emqx_s3/src/emqx_s3_uploader.erl b/apps/emqx_s3/src/emqx_s3_uploader.erl index f6414669d..8327462c7 100644 --- a/apps/emqx_s3/src/emqx_s3_uploader.erl +++ b/apps/emqx_s3/src/emqx_s3_uploader.erl @@ -56,13 +56,15 @@ %% 5GB -define(DEFAULT_MAX_PART_SIZE, 5368709120). +-define(DEFAULT_TIMEOUT, 30000). + -spec start_link(emqx_s3:profile_id(), opts()) -> gen_statem:start_ret(). start_link(ProfileId, #{key := Key} = Opts) when is_list(Key) -> gen_statem:start_link(?MODULE, [ProfileId, Opts], []). -spec write(pid(), iodata()) -> ok_or_error(term()). write(Pid, WriteData) -> - write(Pid, WriteData, infinity). + write(Pid, WriteData, ?DEFAULT_TIMEOUT). -spec write(pid(), iodata(), timeout()) -> ok_or_error(term()). write(Pid, WriteData, Timeout) -> @@ -70,7 +72,7 @@ write(Pid, WriteData, Timeout) -> -spec complete(pid()) -> ok_or_error(term()). complete(Pid) -> - complete(Pid, infinity). + complete(Pid, ?DEFAULT_TIMEOUT). -spec complete(pid(), timeout()) -> ok_or_error(term()). complete(Pid, Timeout) -> @@ -78,7 +80,7 @@ complete(Pid, Timeout) -> -spec abort(pid()) -> ok_or_error(term()). abort(Pid) -> - abort(Pid, infinity). + abort(Pid, ?DEFAULT_TIMEOUT). -spec abort(pid(), timeout()) -> ok_or_error(term()). abort(Pid, Timeout) -> @@ -237,7 +239,7 @@ upload_part( etags := ETags } = Data ) -> - case emqx_s3_client:upload_part(Client, Key, UploadId, PartNumber, lists:reverse(Buffer)) of + case emqx_s3_client:upload_part(Client, Key, UploadId, PartNumber, Buffer) of {ok, ETag} -> NewData = Data#{ buffer => [], @@ -298,7 +300,7 @@ put_object( headers := Headers } ) -> - case emqx_s3_client:put_object(Client, Headers, Key, lists:reverse(Buffer)) of + case emqx_s3_client:put_object(Client, Headers, Key, Buffer) of ok -> ok; {error, _} = Error -> @@ -308,7 +310,7 @@ put_object( -spec append_buffer(data(), iodata()) -> data(). append_buffer(#{buffer := Buffer, buffer_size := BufferSize} = Data, WriteData) -> Data#{ - buffer => [WriteData | Buffer], + buffer => [Buffer, WriteData], buffer_size => BufferSize + iolist_size(WriteData) }. From 820e06d756bc81a2e2f9b8ddf7e65cc7d5af2817 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 00:02:26 +0300 Subject: [PATCH 100/156] fix(ft-s3): make controllable use of max_retries and request_timeout arguments --- apps/emqx_s3/src/emqx_s3_client.erl | 38 +++++++++++--------- apps/emqx_s3/src/emqx_s3_profile_conf.erl | 1 + apps/emqx_s3/test/emqx_s3_test_helpers.erl | 2 +- apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl | 1 - 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index c84f32dc3..2ba837e66 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -58,11 +58,15 @@ access_key_id := string() | undefined, secret_access_key := string() | undefined, http_pool := ehttpc:pool_name(), - request_timeout := timeout() + request_timeout := timeout() | undefined, + max_retries := non_neg_integer() | undefined }. -type s3_options() :: list({string(), string()}). +-define(DEFAULT_REQUEST_TIMEOUT, 30000). +-define(DEFAULT_MAX_RETRIES, 2). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -206,7 +210,8 @@ aws_config(#{ access_key_id := AccessKeyId, secret_access_key := SecretAccessKey, http_pool := HttpPool, - request_timeout := Timeout + request_timeout := Timeout, + max_retries := MaxRetries }) -> #aws_config{ s3_scheme = Scheme, @@ -218,39 +223,37 @@ aws_config(#{ access_key_id = AccessKeyId, secret_access_key = SecretAccessKey, - http_client = request_fun(HttpPool), - timeout = Timeout + http_client = request_fun(HttpPool, with_default(MaxRetries, ?DEFAULT_MAX_RETRIES)), + + %% This value will be transparently passed to ehttpc + timeout = with_default(Timeout, ?DEFAULT_REQUEST_TIMEOUT), + %% We rely on retry mechanism of ehttpc + retry_num = 1 }. -type http_pool() :: term(). --spec request_fun(http_pool()) -> erlcloud_httpc:request_fun(). -request_fun(HttpPool) -> +-spec request_fun(http_pool(), non_neg_integer()) -> erlcloud_httpc:request_fun(). +request_fun(HttpPool, MaxRetries) -> fun(Url, Method, Headers, Body, Timeout, _Config) -> with_path_and_query_only(Url, fun(PathQuery) -> Request = make_request( Method, PathQuery, headers_erlcloud_request_to_ehttpc(Headers), Body ), - ?SLOG(debug, #{ - msg => "s3_ehttpc_request", - timeout => Timeout, - pool => HttpPool, - method => Method, - request => Request - }), - ehttpc_request(HttpPool, Method, Request, Timeout) + ehttpc_request(HttpPool, Method, Request, Timeout, MaxRetries) end) end. -ehttpc_request(HttpPool, Method, Request, Timeout) -> +ehttpc_request(HttpPool, Method, Request, Timeout, MaxRetries) -> ?SLOG(debug, #{ msg => "s3_ehttpc_request", timeout => Timeout, pool => HttpPool, method => Method, + max_retries => MaxRetries, request => format_request(Request) }), - try ehttpc:request(HttpPool, Method, Request, Timeout) of + try ehttpc:request(HttpPool, Method, Request, Timeout, MaxRetries) of {ok, StatusCode, RespHeaders} -> ?SLOG(debug, #{ msg => "s3_ehttpc_request_ok", @@ -388,3 +391,6 @@ response_property(Name, Props) -> Value -> Value end. + +with_default(undefined, Default) -> Default; +with_default(Value, _Default) -> Value. diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index 3ab023d4d..13efb9d74 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -203,6 +203,7 @@ client_config(ProfileConfig, PoolName) -> access_key_id => maps:get(access_key_id, ProfileConfig, undefined), secret_access_key => maps:get(secret_access_key, ProfileConfig, undefined), request_timeout => maps:get(request_timeout, HTTPOpts, undefined), + max_retries => maps:get(max_retries, HTTPOpts, undefined), http_pool => PoolName }. diff --git a/apps/emqx_s3/test/emqx_s3_test_helpers.erl b/apps/emqx_s3/test/emqx_s3_test_helpers.erl index c74e78a4d..2edd52609 100644 --- a/apps/emqx_s3/test/emqx_s3_test_helpers.erl +++ b/apps/emqx_s3/test/emqx_s3_test_helpers.erl @@ -99,7 +99,7 @@ unique_bucket() -> with_failure(_ConnType, ehttpc_500, Fun) -> try meck:new(ehttpc, [passthrough, no_history]), - meck:expect(ehttpc, request, fun(_, _, _, _) -> {ok, 500, []} end), + meck:expect(ehttpc, request, fun(_, _, _, _, _) -> {ok, 500, []} end), Fun() after meck:unload(ehttpc) diff --git a/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl index ef1d916c6..5fda42fbe 100644 --- a/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl @@ -13,7 +13,6 @@ -define(assertProcessExited(Reason, Pid), receive {'DOWN', _, _, Pid, Reason} -> - % ct:print("uploader process exited with reason: ~p", [R]), ok after 3000 -> ct:fail("uploader process did not exit") From d333187c5a0ccbe563a10e24f729f040cad3ac70 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 00:25:31 +0300 Subject: [PATCH 101/156] docs(s3): update readme --- apps/emqx_s3/README.md | 2 ++ apps/emqx_s3/docs/s3_app.png | Bin 202227 -> 256486 bytes 2 files changed, 2 insertions(+) diff --git a/apps/emqx_s3/README.md b/apps/emqx_s3/README.md index 3f049d1a8..4ce1b0c0a 100644 --- a/apps/emqx_s3/README.md +++ b/apps/emqx_s3/README.md @@ -112,6 +112,8 @@ do_upload_data(Key, Data) -> When an uploader process is started, it checkouts the actual S3 configuration for the profile from the `emqx_s3_profile_conf` server. It uses the obtained configuration and connection pool to upload data to S3 till the termination, even if the configuration is updated. +Other processes (`emqx_XXX`) can also checkout the actual S3 configuration for the profile from the `emqx_s3_profile_conf` server. + `emqx_s3_profile_conf`: * Keeps actual S3 configuration for the profile and creates a connection pool for the actual configuration. * Creates a new connection pool when the configuration is updated. diff --git a/apps/emqx_s3/docs/s3_app.png b/apps/emqx_s3/docs/s3_app.png index 372f16d371d0b75b7d796736906eb1de004d1deb..cb7758844e0b12ffe5a46b34df2ddbec94287120 100644 GIT binary patch literal 256486 zcmeEv1wd5W_BS9Pph$@bf&vN%NOyyPf=EbrNDduCOIsL-q)19icZigUbfa8_lHd>kKX|xV z`MG(tcgGum%<2=X1$_{RITHeSQVhgo%vY_KS3%<+RLQSo~Z*Um= zR8#>!G{OHI>;@b>27HX*qlCS^wW)^bWjP3#%{d+}K2|PXaQL*GjH>)aI!;OO*&1SH z3Vxk8HMX*YUm|JYXlDbCoa5l&U}cB@3l6FnnHf1kP;LQVnJLuBe0Nps=ai(+^SFCJ zRW7M&I4c>b@_R_{uExdG(FtN_idEBJT!1k~N$boZ#aot?QgcpxVomoy!_1h_H$ zKQ2iGDZv+8BQT*I64Obs(@Aj9NpaBebME|PhaWk+`%`M?Gyl$UUc_-gjos%#5Dd_E zlHul3KCh^%;db6yQc{6S2CDV_vyn5faXas5WN#sFXRL^zXNZZZ)9$rMa@g5fLm~Ei4;tIq+L{_8zvs;JvT&D-NMbtL*@63^a2WM8a*&Oiq^UI=(nvZYo&%8g z?_c|MnT}s2>i_3L!D&C87-Eh@cvk&O9u=nbp~oYg4kHZ=jd1p#(DN2rCJ zxt*<%we+{6XTRYesP(ho#$2?s1K{65y`?D>>W)y9M$S;Xy)t3y27zjVk?gFTJm3uc zs}|xy_-83Ntl_ub`2wf$&KFyN`Q58_E`ooBU-k9ex69zCq>$GcJ3G3371n*8vVfPX8$2 z{N-?iu+@Kg+#p;n6#Z^$*Y`-fzD46=DYW@XN3tV|?Njqyh#~s1rGBGmc zH!}um#0lzXXJv|VhS!+i^s*T^v->=Iqu zUq|E-s_iGNvv;(E$5DS-7VhOBC(>;~W#Lcl{V&Ht4utcGB4You_dm7c2z~sU_8u7n z{`D0%f+Bwubol;tlm8!b`Tk`H0PPM}@PBy-APDmJGXR{(zzt!;|I8NP4;JG;PsE4R z?7wXS5I*f+UjC5@#Xrgf{Oc9~Bnbb=WWm1-9Uu*V57U99$Um_i&i2-JMkc0?f7K`g z8TcV=2p7_(phOXzNHYD}fglIM;Xz^ME{stW!~a+0&ST1N!u_+zouBhE7ccLBDRSpV za_FCMjDsJMK}F@*SNGu8iQPH4b~%K?GGyKb8N7q)l-Un#1SJZP?L;kT?9Haw+S!5( z&cX{G5Ky#|K%B6Fn3%vzDnG5De1i`>zq>oF-#1Ug z&4ciaklf}*=w=l89=>{xQk+17p7t`2Ubu`!(bfNq7XqxRDALm0`Sy z6bvfE1b)4=%s)=8A+6#M4E*~@CJ%x-NIoO^hsx)l8BQcVej&j`)ZJ0+MtLAaUdx269aB+4S_dA>;w`h$ojW5{C~s3B3lp;a?FF2PgFVnStSHH z?iZ5dAG;J(5Dpd!13>2s=m7dh9u7*6h>?^1Zd;HU#0_4m-s4WuadMfNnQRVHT^tYq`xyA>$ z%Z2QEU}r^Yzq4qw|PeviHf(ABs@sNdAYTz{J^7=j`w zn)nAS{ad;hHFyHjZfs(Dng23C$lgc`oZ;f-65xXChyRAQ=H^8xIi%Z#6yo2KcBBgb zg@hduwf!-|{!PHOfJh@n|KAh#uZrN?WxtECBM9}I!p@1Xc7J)di|Z$5{hLMjz3l`$ z>wmxM3D@>;%lO+E_`4{4E=2tKN2!lLPs!|G#?IfzSwKh=inCy90fpKd|7EKjNH+l) zB>W^sKzazMNjYSq?$_}ekT&i|*4cy29e-K=3GB_bar`W+`X^!Fk7;m(?ERG)$ia?m zMMOy8pP7R{=EeV`p_a+7QONjO_CV!*GqwN{h@ti$S7J1o+SH?1loV78x+Zc-;Rs^HnB0nwL@``{-fI2bmyO6 zFA;{UWBiuTBE6(vM>}&NwDTVm$^JV+`;Qv<{!3YJu75S_&H1-Ug8a@|?_X8ef98rI zLtIo>>=zXFzh(Htht$r0{^*Ab(Jg{vUnPv8&PMR}DZnGxAB^qarzMD+pBH$2J9%oP zD~%LN6sydK@Wg&?IYox+C~Q`AhME{bK`X%j7|qWCe@Va|P9r58h12|8NI3lBy#6sA z{6AhJUt_}k6JGzs{QhZpjezZ+$=QbYrN9fzC~@`g-(tdzP}9E|StCR6zkJmhNf8v2 zfXt%(S(^R6-!l0p?)KyU6ZVSzA%cwTQTFqc;D3;>yh$h-(B>>TLeMG`2FjQ{`s5v z;E5~HUI^aL_mdY3B7>o zo9^KD81b6B->O&r2UFkQ+1H<719ZTny!q~@l>1k=;&AW8w*LpP{W@>D`)BL_U48N2 zS&X?kk?9enApflJj7(vp3i2OnW`D|#A74WASJb=$NIwY4us^ZpMcB$emFC^;yGEu$ z{ts2h|A^{Elxu%8p4zPx{Pk4tKW--a&!s_rU)3uh@E20OKY8!RPBg2jsriEr{*?mQ z-J4{Yy3yCm<#;ThjahnW&Atm_WacY zR2#8#7Wr2hc26KKfWORj_x=2uabaiB(5TU5r6g3{^rr^*yQ!+J)qW-mc|fR?XSfe5 zNLh z8}Tbwg*>^UPN{H}UTImFI?Guavmihg`ecP~7DCO9Ilw^J+ig0jxQ^q|#U5k-w%l%H z@hCbbp*Y&tAJpU+*PJP#In&PTCC<~4cd8sO?w-F)+UlA7Xhl|XHqG$_kGkHnFzUd(5B1x@<@&4?^HM3~G-a6v3G8}4OG>mUQ9B`A<(puP< znVH|Y5ZY?LN~vq!If+Y{xBp>T=|eQ7Lw?`Ji*sUfa&jICi-?E_yg8&ui$+SrDUpWr zsQ1i!436r)Lzqllirw+caZ3}VRS7C(HYB37hi-nm9R6weg5UG!{bJ&Lt~1+sf_w3p^sJ6Q>L0RRppR}i@RhPV(^oD{uPF00zN}O` z-@*|4jox$VzRrrWvkYBFd@WlOo(lph1}HAimw)hKgTs{l-bd+^xeiDxP5M z0vKD5E4n)N&@dT5KnT5Zzs@L05@z%jSA2Fu6uLae>x8~L&CG0d$-&{Xl9(zs1MH<( zP&sC!$1H84Rpn&78pX5J@-J>wd-;jx_XTqTPt@kAACK$X_%lvqC-^+!I7%nR(W#W3 ztyV5(R8x8GjzUBG5^)x}0r&EH5(x%#gUIv=s>#Z<3vmmyd!z%0TIq&33txVgh=>>0 z?E{$LsdEp|x}t@7A06EPZ3=#>IGPP_`!WO4Qc{|j+1Pq4^W)YGDJ9ZWb&Z;Z&3o-t zs0j%-<9S5|Mj8W?G`?fY){#70lk~F>Qf6>OR@dhZ7XgKI{qoQmW}XSUK8KNiMLimZ zEg{Yd`jf`S#%pwRbT2B1-DqJW*O@uoy|VHt3*tify6OVP-h6j^oKunsL8jo|bo*kB z;md2oRmJ7kI>9}vZw!}$P}k|4V@uU{^+1I$~p)z zj4%|}(94)`PcsM+LLKL)UJjL7c135Y9=EEj4XTg({_bw_)J4!IHdoZIG2n~s3Q8gg z_3+_XG95Tfk;BFDFdZEo{ZIJ6RbTRaaE!^D1bX4wtB_%xmcQ{7FE% z@EHVka&QvQJrFOeQAs*?_lVtK`I17~5Ny8pFgqpS#69tF&tfbrEL8``mVq`E6sOhdN-qjrwCi=5g)fcPbrs5u5s6bP_ziDrXBV~X zzx1)w#1-`tnJ2!L8} z)vrA5HMQZMbOUub93O7#2@y3a;X;OSScZl`J-DK0@UL#-8E&8q{30d(Lc!yNg-!=S z`S|*yFkXKJ#?joRkt?)so;XRie5aiW#l^*5YiORox`jGS4nmTuy8wL^lUb-;Bm4Aq`Al;ZwTD0Ike$^OAGnGgS5R0Od6<-R@X@RIJh;UjI1?Nf zCz_j;m31SChK=psdWmh^#kC;Ri#c#%UMKYG6?8m2);0jB<2a~JH^<@#zQTRrJGk(W z*XH39(}sw**~XvG37}_CVxTT?XD!O~gcNS|Z{BDsX=@9{)Tq>_UL`r{D^7`_rKNS? z_U+qG$7%5J@J6%0jLsj*fZ>R=gkd2Uh|I?q5d4Fknjg#cgQuq%*Q&!R?^O0@-g zSlC16U|$u>0RiScgWw=Z7N-^ydv|8Z#0oP`nVlep7#%Z_n0n+eDcLbpCFA=J6G|?G zFJfc0bOi82wOWG$J#B62ll2eN9H#bT_hZPOQKds4O_*sX1;Uvt&OVY%(=;%kyM2M( z5mjQu6OQ-=dx|U(!|&-6X~I78GWz41U!(Q{b?Z6@4vDO z3j>|;^l6zdp9S__H!O{AMg9~(N&3~RSA(Gf=lm7wI`B=*%!1A^GIEf-N%INW-*o?b zy(C^!iP3$Rol3&Y?9=m*kPstUW!Ml=n-fq>nc@I)6##O_uf_}r1}q*!g_bg-;owJE zMa6gnpenhvyl7xylv)}5bhY70-}@TAzWcGBpD{o0%s|l0C3&{yJ;pQ7a_u_V7Yf{a zD2?U^!=&KGc^w?$Vs|4!1?r^S)9nY=b4j~jlP~#}ik@!89?96{z=k3m9s`PPCE1;; zN*O{9*QkBPMQ{imGxnP(p!oP`7(-;O1cd;dHY|)r4E_p%PlFzR2;-Lvx(W!sH+m2y zO(+jlWQa3587v57KD}_P5p~3{1p4~^KB-gh;kWs0z>OL6fo}u{bp)J;U>c<8>+<<` z?%csjNlAH;KJ!ucI^QjQa|Q!&8I3Nsnb%uejrJb-uc(sv^3EAEeFy#ggJxk@FG z24hICS9V^b*L-)d;|HAy*SHJ`U9pai86(k)2k=GU&Vio=mj8+yGZRzdU?hO_o;4*B zTC`FhmA@d<3k>+-FWMEdQjX4K-PZRG1!aFX6n-Zu9E)CANyWF?XKjP&5)CN#0ZVw2 zP`4n6AivzK8Fm_#p=FxX$;z$+Cv;E34zIofX26lTL~75&ccvdgPv{tMCr_n4S%4j5 zh~y2h?wY!~19u4uJnU@AiBRvzuTfJT85$m5{J5;F%u`r|4yIbFPc0J{IZ~2>ob{zOL-j*ViX^>(;FqL&m2jnNC+Qb=eF>8!2!wr7{J+k2NL~sV~X( z5~zi6UxCM}P*v}g2JPFu60iei#v@>khZWJ%&evdJL`6j%xNzY@xk~;! ztWaq(RHoDvV$iJEU#>s%swmW4x)lG6h=@paR@M~jW(Z%Ee!R`=8$X~Z7fcNEP>3kVBy(5r_f5jmENE`)kQ_tBX^kD2v|VB|i*#kc4L_}n<`>?Sed=t(Ui3eamd!wuhS29;)gf|HS zp3&^;m2cGa^})gsT6LWPkpmOblQ^%hA2;&plmB-v$S^hfWR*OKWoA|vTEuRg>*uN7 zANzRZ@Ob!hm+uw=zWN>jH-2K@4&ww|NqDGonNk364~T-f{WLQKR|Q~FZx|twwEOp} zpB*~JoD}b^6L6AHBz(-~2ZB2+5Zn!>e^NF05VJSpx~We+5+^T3o_dq_Ruyye&`A&| zOMyU{ecDt)!VeQ;?fvn+zQZmVd^ND}4jiCkV5mLF#=>Hw@9yr-mKzlAK;!HCz*52E zoyP2T`*#eDVJ5UuV|a%Z3(`HGA|oP>XRFA(se3}kDY8cq@8XY`1@8Trn4CGy!ZLjy z=43C7Nhnk4tJV~_UY#C)z+*2nI_coIHo%Gn2J^@fGBQ^csiBc>XU5Nhd+4@v(Xa?Q zzR);6_a@hFO2EN&_HGCf&9D`Eh5$&gkPD+;Qg%r^(|)p&f~bz{entQ5;mCwgcc)Ck zFlLvJbn@}2a=0rBelPJf`C5^lCJpT;TMB&O%CQ7TMDUhYR8%mWfI6=Tw|k-r*<7SJ zw_5B5A(VDxBzH2t?FSQ%hCzheE6Vct?HHlUt($l6651D_b;ZO83oU|sqY(MAbb>^H z;nno+3I~lF`X| z=~HZr2jlPr0hAkopolySR{6koD(JfeCLEYwq-*fLBL01(1D67(?wJJB&3(IEocv(a z>b&O(Xny4pxeFIt8G-!Sf1l1aN-05J+DH)CDo!z)}?0QHUeuz$iyTY8%(6i&zS*Jr= z4#oM=46~v~WIOd-Y17#=)dU5<{tQ8P8lbS!o8^4*Q8|mWA&I2aQ-`yUzC9UCj9zF> zb;4IZP=sM`NEAXlJeoj4la{Q@N%>t)XXh^B074E_)WdG@No*W$lh`u}Y7f7M5xGuI z(rJco7}?kq9i5u$oquM}(J_gc4A&1oIv!~{MV!&_jPRzMuj%B6560$u?pN~!qa;Bk z$oojsM7mA@FcUGJ*=<~VhU}U#CycAH!h9(Q-yFmEB#kflpLYF%!q5|Oa@;*Q-Bn&tfNYg;MH0cv*+l|$Hau>N)`Yyo#Kg$!G2By9-$}z< zkgxNJREYp=C*b(`+pP^8g>qWukT}jazp9h@2@gtfu3n8DaI0Bi7lfi7P{BR3z zVgFt&BVJ<+N4Ecpp{_I6>$R(gQf`X4m~w(_%2!E2e5#SgQ14|F5!svpYFU@eIW~uO zm=0ZKHQw`;{zK*0du^mDQo1(Hs=B4xtIHT+Cfzzc3JbVYtTsR18_iD@uUGWa(TcW+ z`Yn4~%qyprsAHSY0DYF2>jVsMdzfln2RcRl6{G-x*02b_fqc+}bj=lhUP7D)q-fvgABRX=Z}n^U)vX0bbubH#2FkG7 zX;yklaB}`g&6;V9l4G`KjK-GhlDBP%s9_@!QHk+;^+F6r$*ulRRkuR}6zE5iKbpoa z@temE{)Cd4A@m9HOr3chCFgn>A?=h${SthixHdAO=@RX^qwfmV05Oy^{!QCB^~lKP z?GU1$SUy{_yg_KkBa4Z;PSrFTUvXv3#UTe@&sy31(S*HsOTx5X(OuM+3sPYgRC`Pj z|Hh%SSkn9_jKQ=d!od|1SgJbabklbd3+au>x7IDG)a~V@pCHy*CJ3MX2Yj*xNDQzz z89#&xyAQCxEp8#;T)G(TO*_nqz97-}Gr9SQfxks&=cwCSmzCWdt$NSE?;yb*1HL0b zb9uhgpLD8@u&m4}kyBu#6XM*IK|c{Dk0mm;H*!)w85|mV5-=Dd*AvC}xS#pLp|nrW zhqLkzfLhuGEv+bS%g)ow0sL+)j4G*b?ul)kzH!34Y;k7r?V{>-toLHu`GWPCrwv$S zy2Cezap%Pd78Y2duYrpXAId0)J78wKd0-tI*+X<-@=w*U6SQ_nt@Kd{G=8bbS{tefx-P zrh4{BK?Cc;3EzNojSK4g<+1e~QhZrpVY3fXCbOKo7Ut(q@%0>bPNW&O@MF;_c>`-p zGJWQcRbc=AY)6lt>!4bY@VpE;r^(GlSlf%X&3ma%BA&}(s|#Zdzy~tf`10ATC)&1L z_VwLk4Es-9$-A=LXWmB1I%C5FEw=9Cg!JY;ih;JvKKod?`YF0S*FbDxDYtryNoc0b zcwp=U+8$&9v3p5tsPkOG)wNNiO!M_~Q5^n@HCcEyUM4r1@CMWA4X;HH>FJU?CIWnZ zz~Y8=mRe03&M|&AAHIoGW>Di1*Hh`~x^R)Jo!4{Oy06?hlhVEhy**Vvde~N4TAI_m znR206^nguw7Uxx&Oda2`Z0BPjf-T1Fohjs}hnWn;DBC_b(8Gce&1V-;<+WZPLMLZA zRdC(cw&L@-(u3*U-h$KnXH^tM=pyw!(#M^^J@aNUV}%`_aNz6_^%^dWWyNC!S7Xgn z!~K%=eG^nB#(a-=(k7s%wc4wU1$jn`XiurHvX_nK$a6@c`JKkc~HM;ad`(FZATY#8b5JaJ4ZvU%HGxo@z0!CuU5dm(=NL58g0Oyx?>l_Y(G?9_w~ z10PVH?#|{Kw0KulsrCGO+v+Qxfq6C}uYu_PbJeA53$(*De!!tVeE9H9Z~KVrzM+_B zB^4;_CkFDvL(63h2AU9%HyCUF)~}d9WgaE^;j`Wu7t;iVvB^KI+Y>r39=Vpb6MwQ!?p0Rum>r% zKd-ixo7(`t&{k)JXU6Gs?v-egTPU2 zVrC+Ldt*TnGF0&d)Y#8g__(`2*?70T89(mHEl<)XG+R-aoRV^FkY#{iNG2$^CyJG; zCs~$+E28}2vcY;{kEwjmgX2l52`YNErTQ8=_?E>MZD&i~+CG(W<$62LljGH475SZO z;w!EPl9Xl0+~#C|Tn~|Wf&#j~f-`Zxgx7&A-BPU~tqOJZ+_%;W@-^vI=DrK7dgA*O zJIs7+kM%Bhx@!^HIQh=A712-eFHqN}Ie zD6{dPeWsC7-dbX?X*%+*OI}m*KIwVSbv(Eha-PoIdTg+E;FTnSh{;*u2@9UsB<7s5 zvpH336HJd{fB@-5u<9|5#k$5(x()lygK5+iK{_s?$pt7_dc5;msXHVzMJU`I#;DTB zlefLK-T;Wd@!*o;m zmZI=ysSdkMzJ#7K{q24q@L`?G`^$TxCbnbW$OpaAHd?5@sh{QAx?9%Jos z&n)4&ntkp0#^;R_TXYOO#_8J3z4{lUdPUc#jN`&9x7H@-`^@99;+bav$5MNr9bl_4 zj8i-Av+-G3TiY@y-?EFziP8)Ev3zOea=%qm;i&z4CNFFe$B{R~?%7_BPEJh=&t)_m z-n%om=;*EAiw^2x91cq$k?ZafPz!u3&fL54@x(T}nD29*O&T7c_5~&$C;7AKh<)Ma zPBNmkl4W1tS__bVA*!zQdg~#!%%_%kyR)a&JM87<{gNYCbRIgvRu^vsV_)}Jgb^K- zf>4ijswnc@0Hq*a^w8X@wfZZYO?>a(08t_1>UC-nF~4GIN%6X!@IJ4e-Ey4g42O@; z>S$zByUns0!CA=TPwjGS*@Xt$baMMty4{{n+jnc3M=fYgmwgNh)haZLGC25rRi_#a zMmhClEY6ItxJRJX%>9OthQ;`-MfaU9_AYQ(=i>uKp=oX>D(&jv3V^pyWbyqEulEP> zf52HCPm8}unrbialiy*LY*6j}$y86h1+O3S#j3^^o&K?^N(Xi5W`9wR5B}5f;-k%) z+k$p1)UZtBVQzg8q2u#!!@2OaE^z0FCqX|wYDNy7fjF6-u zyy44gS_)ky_=FRH4GnQ&iGBsIpVeq#d={SV0_h?aFS8^4i|cD=6yl35kILTGx>~|v z{Q6DOJyD8f_b5^3$$=+M>@~^Kl$IW!-0KMt`5n3ZI32<-SDK#rxgs1` zV!e|UGq&jGF=N4t3QHAv^D%>Bt*_%taIu;Av4Tq z$-eXPrS@tcpF-}3*fCSj?g3~iVdGPr(pFGnUa%OJdVBta*WoO;Bnc*YEUU?(%BQmX zt@A>dGt>NxevEX82W%=pq?af?^H;~~Pzh@arlRdcv*Y8+P#&6A)c#B)TqRVbi zhI|6m$|yW(b<<=Xt8N)!Gn*b|C4@TbI=9xeerk#yjWUnJRj+AJ2*=JT>I+a#})tVO0Ib;S`+A5Y`AG5oO z9R~!AP34)~n0_wqbXs#Mt7{rf4$vk8)^>L#?k+uSt%=m%*T49%)!=#&o|?P_!{+Rr zA=Pe|w>G6w2JJ=h6PaD*S$O8lz8bKlTm@}uxt68H4#j9ot^T6^h-UKdB2qNJe$1>q z6+{&+sqHM9-vi*CzPeZ&6nMnVVuG_!mx?A$s=-kh^8-oVbA|FdgLloHZ_OG`-SrK? zB>r5Xp;6lQfuLOQu4rk;SaAQMR(8%C;Am{r3ZLxqKeD!=5Fa^S$nK@EzCK&s0G3I^ z5Pzkh?>sZ_c*dEvDdqU$erBHW!gF2`sn?D#MkQ_O#cx77Fb=N?t3_8*pQR=YCOUcU zf$;=a&uz9Ue$Dr?8lL%a65ZT%!gizXnE_i*+QU!b@YBFvhB(VU=zVx7-7lHfrvE{t zWM(&PMRapfMPMrLMdM`oMieP){(ceexgyXS3S&|^j*INr1D1R0i=WK#E8U!JzQaQvKh z4tq@7{Ah!8q*zC43dPa`^$tV%r!LbMAIk;kcPo1=e}rj5qxjbyEhs44@sw&Qoi0jo zc>%92+JOEni?jLDyOlo638AdaZktOz`&&}F?sA8jsm-<92V?t9NicYZGg;Ms{#Y9w za=#zicQ&(LEg2mAAPChG!xz$;oV7fy&|+G>3>_EPOsN}!_VMc}D^=&_>c~f~6f&8P zYi1u|g_-1Hr1X^Ne_T$LW3wzi9cx^fBf9m)c%fi(ji40sk-At1^r=_;P5S7&-^F`Z zm}`h#s$xB`<1~T861X&C!GWhK*PXAccgJ_?x_Ahde)PhVtaoM7Yc4h{3@+®}? zdo6xxe4cSm_j2O9qI?6Uk$V1P5Bcok7ALwIu59*OjXI)4o^;jPAv?~m}uFCCqtUrrg-lA332dINr z+AfVunU8%ec}`eqm1t@?-_^wDf^;P3VP^_LVT8g1-Pt-R6&1Se9UYlh3$ySS_ZNHB ziA}C+H_5!yL`GBp94qnG_yP_E{R9-zVIk zg?9m%Y53RYj)$aY#g76Kx;u2d<>b8B8pU9Au3pA}C; zQjD`PxiGU-r&s4$lnZfRJS^WT+cBa|Bc~&;_QbaAJ$75(gOZRC;!wd-gYEuE5YTCm z-@|S=4m=3UXBC>clBnIP@O;}`mG~QCm-I5WF)x#Iu>I%@>3r)q-fgg3yoQ}YXC6te zoOm~8*p^2EbVv*X`}xK8`S{GjJf_hx(xox)F zA_Mfn^$DS?FYjVsfJU9xp2wQ9O23{dlPvOp)AUxlMG`~{pAJ&!e|&jXtFrip=Tt+7 zR<4d8S*Cy3bsb>%jr6ys3S0b1!={o-o5|J!R7>0D?x+VA&U09m)RdyTuFb+ znVdiS4HSg)8taej#B#77K`%a(>UO-lyaYLa@3>8b(Cs3Kp=^LL#hbQO%}NjFyi-ZX zPwKX&rz+jQ?!=9m(xm;g`#2PDrQwdP6hCFrPwMfr#r!R{(TN}rcZfiJZV+OTuhFHwcb5#zl-Z6 z{bw5bgN}`V={vMH&jGLO<%QSzB31$+Jr9v#~pyESz-7B!LygB39Q9T2-XwpSkd&+I#ia<^4XMsG*&eSh-;45M7THqy$0c8Ah;7VCN_B-y zP!v|>h|LvEgy-NM%YhhLjl8*^nxUd6zfaLJRjyCEJge^^kCjQ|0}lFAq(DT}*Ea1X z7_uprUXmzYosI}EIWBso493Q!&d;Fh)+Ff5s*!eRdGrGg>3@(fafaRtT_&Epc}eLPgCf z`RxshpY=snc980MuZ=T2nR|Yq$)zEZEi&Mc;AzKTcAa^+pX8t?J~f5sr-~V?S->hC zc|GE@=Ry--Q(wH-a-X*6qfxx|m07!zeE;L`nXsjgK|O$TIvV5DoW8U_{JI|81$KTo8^5Mf}fsMHr@;oPPw5PM|-c`{HcUO6N!lUIW;Oc%V2DHbY7v5a# z=nAqkxUOK(Gfj&-1_Mf4SATY>^6s?~8Gi)|;lg^k33qNvb&Zb6wl%DC`)svclymvA z6O%voOiHkP#1+&+7s$q09*s1(J{_W7Q4Uf_O;*~MF4SInyOjG@`m$Ef7x(X)iEH99 zKz6mo-MkZ*gc=sce&NvJ+rIFWJuvzCuzUiHB{oAf=gX_*Yo%j%0!d|FFX&Gjar!HC zJJl)qrN~7-yiC-uo}6(NEkL0WqF<`?)B1v9`VEz?7sIB$%89x(X z4~Xbb!c2T#TWA#Jq9wQZhyizOeBeo8-zi-pES8A`=C?3Q!LfKB?`*lZpDUcJw1@)7 zvYXdjS_?`a@|-q+rCzdr_)zE|KT>fAw zOr#JwlUCV#X@PIk;ymgp*FpA!-7l@Eh`Wt2Z0m*w^QJ}N{@92!TC{pkU3lKjwk6%b z<$(~6Uq5CWj7viQVD^0ZMQ!;kZFiXf+5VjjjhcoY>42)_1&WZixuD{!@u|0Y+>+$C zJ5Llm4DL#OOK?I|xR;NoGeOx#z82o<%iCtDP+l37pPc;CP0y)`=h-07!DRLRDXY(x z{1<$_SlP=&t4kD|y-22OFJWJI*0YrnT2*{GVvG2ZA0NKO=9iE6+KV76P8Bi%3N8(D ztWREIb`%MQs4KTmUi883Fbj__3d(#}7;&eg+G>_Bh<4F0`fE-)p;WMV`bt968 z1YTO3Z(lpl95x=}L!r>&-u`*K%&aAj{caQ|SmKYd}H1XM&g-rW#3@ z`&5FOyv5UbfCnoUq-EF&F)nUzZH{(n7~m6c&b=T2Il$4D8o(aSoXTbG1y>eKBR2X+ zE!UQ`Y+YPREZY2NB(LAhd7e03#}I$mE9mhw$mtzC@}PCBwmhDfB_W}O#d-9YjW)H! zJv9}eT}Pc<;!4Zk8*q8uIiDaJpzzF6VeJIJwk0O_z$3APS*erp69oJ4LoDdPHR*3M zVv-FQ1YyYyFN&>tSlvYNV_`iH2a5TaVaZ~7+%-w)EH>}1EXcDbq^LbEDhS>3?K+Y} zteo6VakcG&6h%gR>X`6itE6<~LdS_)>d`z_5tL4i9G~Ryr-cE`pY?EycPGn+4$ph6 z`KR9JIjr`UKyNVT@nac5m#L$aX+Z`77V2|ylByIkg&BI|YzMXd>W@PU1QW+P4UU}Q z5{W2>44>Bl5kYlY?K`6zRQ}bcVJlO{UAZUm1;*4V=aC?7km>2jWn`AG)Z*lsJ13C;wZ_DTl*IqWjyg~~a;TcG#Um7VqD8E>g zp#rjea<>PW#7x}MrXSzzUn$x1Q7k#PYX#Ma)nTmqItHZ2$L;J@Ipyw$aq{7Xk z*9N*gY%^Q&XF*_HmW{k^s$5>rIy+X|CwKJWTx5?Ndnq!6b z2PhcV2Jy6TI$tC6?6{1pjg{fP@JX#ISFRUwjbYtMY*73*56nr{tBOqaqxJMijiyvtN<0C_Dhx84*EX}U4uQsRxZ*G zQ8%(2D9eX=WKrO>pHQW!k~OuLVkQ_Tux5GsxS^!jcCa7wT2cJ_hu0yUDv{b94iI0N zOx0^!zCkaSfSuy)%2vnGKN5 zNu>=_mn_)dQ$zqFmSwHn{d#NinWEzYJl5(RnkUHvypFLvN~ht^E>C&J=iHW|@%|xp zQ_+d?d)Tldw&m2GTU+PlBJ(;@=CVqRUSJn4KGNs9|6tOM2bR`T?J*e=L?sA)TsWa# zWNf~tNe#}bM52TY6>e?o+rEU*5418-HA^3vG|-Mh_3wo(`N6j}SYwdrXX$oBJ{WpI zWhH9{F_xApdIawsVQ(Fs0QD|^`*MoAIqiw0v5W47yjJ7opk7&bIJmb65{fG{zOodh z`?kc!>RL}tXwG6{i|~@xBTo|(~6Qphbc z)5kA}kwXT$9@ZG4vuHSiPh4kU0swkt*n~1UzDKU&;0w~N+zAvA1R>?eH<#com1{VH z6v{sGuF(+b}@EJfupcf5)y6b1`*ck4JHP``(v^Awq zlsn5-FB>0x6~40x#6H-&eQ3kj=ox}fj3Qu3lECc}FrnS(#rIe6 zj&XiHMG79RA0_+xy(;*i5KF#x&9@XYgbJGYFtju$(u2GSRIg42>?eqTm*jbE2Q=Bn zpbN4J{O>W{XDe4mCNgs)Kq*(6I}DzE52lec0fnpZu1t;dMFhLWpD!xn`xG9%PX*?l z_b7`?W|mBcV^8YVOQTMlmo#~{Dx=B`AgnmDxxRLHTgT&xO@mpxZLXQ)<;ZS_7(cz8 z?vsHth6jX3Z{Hj;NCE|TXE0G+Qbu@`#p$7NM|=SA@dNz`8crvZA%UQL6sL5v=(@GW z&CeBGq#aTWL^yb-gkVxU46MA!T6s+h#v>XAe17Md-ujfJM{jZX4L~(hF4E`3K@o^E z!*hk^cs>t!1K%l0-wcI1rO^k+4+C~QlXY$g2G}u8g)v}sf%LP3+HNo2u(zKHb}XJ0 z*yV%SSb*Ab=8A(_JQciHQ%ALMj;-=!f?pb>P>gS%Wk+IEOy}LvsS2*Aci<#Cb(v5Rv-ej>kKgCvE1m_`c{$uzY|Si}#4Vy_Spgz&}8l)rBec#ON(bt++iu|si` z<=4fj;9?<#XnLs@FYs4Tb#mM;eujfcyw;dwXxBlWcbb}YsyfW~jd1CL#wukOxyWt9 zH9iY*21rjql=1q7+MVIVY{30{8SN;g2ugkPozQYCcCg=$$6)aY`cpIshMfJoiy2@G zzLP-G6*J(Y$Bu#E(%hpONtMW`(Fq)ZRr`7Jf+Vc0yiitS*NR0$;2;%LH`apdbVK&0(X8OQ;H>};&Nw(|hh68X*%CHkq%vTZX@x(|+pn30o975?A?oFen69kVcHZpLAS5Rf|jmfVf0C}Ge!(! zP52@^o9P&pG&N5P2ncjDQk+`Qr96fn35e&|chx=pDQ0X1cl4cCc3^QEydUrE^t6Kw zvVn&Lqf93se;RoucZ~y4as~a{uNC#^GrMi*M@G7E&2;04@*?s&(B&x`Z(I=O%0IXR zH*o+rBI^9W6L7dWbW?Gf0USi(l=}gg0Jw=8lkKvDl0LXKVP}J-&0u*kb5{%{Zv*se zpZQ~nf0%3%0@s6=36y}uu|JR{|4(JMKypV)$QP7+a0T5hILU9#!b3sZdUAGnS0BJC zF*#iO>}Mmqj@&ymbfh&n^g75qHNRbGgjJl}$)pI*HZq=sZycp1&9nUwC{uNFM7sRM zTbf}bpdgON>826fO$hBUzqv)DR0zk~q1}}LtmP%BXlwTm%+Jrq!Lcp<4cjN+*#fF* zkS+LP+*d|BSFEoE&)AO`7QZwtGLoEMoESO{>0`;3?aHX9M9cXzGU!W#$MqWJlJgv zX7P1*d`%=qAkI!VV^_=-_f2+-UA__8=L0U1DIV8bDFz=56vYDIe&$s`Jxm--Fl`5p z%9E_SU8?0)M}1E+ZU9(()E`FqF%r)ruEs4PO2frbw~+$W)G&D*mNln|DVcdMC!fgS!v!P!ZbS0rRHfB89?5 zSSHXnJ45Bs&5`6}l9?os!qz! zK>uP~$hS_QVOn(C#&U}Suq~bjJZCgy$92!MoyfmJ)#HB;t{&XxRYag*&K}tOVi@@c z=DY=6gqC+Z7Hb$WK4XGJ3j;{z)T_1=AXp|&EeP&0|56G*;zFDaq$cWx*tE4Rm* zPL>7lq7%%}70=$dJ9gn*E|C&=kR#O&0egYFbo>M@&KD(M6bKjw86`h$7O9x|ds7ou z0!kB#gIexY?zZ3B`@`?I zKp$c{8N+^U5iZ6%{JAL)sG9#IpA18R5CD$E%uf_ff^@ReXU>GhP_le7p=jty@xB0) z*aZ2P3uXH3Akl%GZC_RToLcmf2Z{XCLrG?6i&V!LGx9? zx||M|kcNhaP_P9!E$FuQx2~X08bB3RBCr=4{9b9M5`oJ!F-y(7bKsFtH&&1o8cD31 z*Vb^~6*+HWFza};==_~VVmCGFhy;>`ZucciL)~5Bp&n)iKv=UZC(o*d^DZCPlpoGJ z(A%7$(57m0Zy{?cNW2~nxGQ)YZvoTR!Tzo%>sUahw**Wuckd!~40^dwFozroH3 z#lMl_JZJz1Tx3Uc0^o}u1r~VmLULZ7l7acEU=E2GoDv<@Le^P;b+%{q-8@1z%sbAF z4`fF|r`v?eWBVxlFyUhJEP-;jT;_Ke2Rw?ip_}yZVHxU#D3l$czB*W7`rT1U?{IF~ z8%JaUvVeDIumZy95{Sa1gpNSh=Xvg8RD)+?dKV0}ia#(vnfdgd6V^R&(`Bf^uwoxRmqRTdh8u z9VQw+C&xhc^&8@XHCv&Bf*>GwrPKKqIJr%5mAV+U3MhM?0dqkqETBJ20i-$F$X}t& zJ>S_8urOBFAZT}IlDHR+6bWEr`Pu`b=CAvdJ3|4QCvhKkPzPfIn9!q#OTvM~3Xcru zag*2c;r^ZUT`ywVn)UH*j)<_sw?!@bZJ#z$*u21st#7C|X^41dk`^0%W037WY3$Z) zqUy@?#hqq-t`eIEXnh;94IH-mtS#00}VRsn01mWsp^yX;p1e zYtZv~U`uc|ZBR5|@x-nub5#3SW^QcvA8Y9DqI)@s0JAT6Wf4F@VqIb;&Q%;h61kmz zZR!}bh8_pKFTmV|w$KmAzF1mcAJ5jgJ@S72{F@5@sQGD)hNezHe7&@63k#)-`!(!Gq4+U+d-2fk@Cl+UkLRt3ne7so~-0`3anUo-M~58SDP%qzByViAvqFN z_y2f%>#!)h=v`P5kWxTE6p#=B0Rd?Nfx!T!k&+f^knR={P!Lc_fgvPDx;qu=?(XjH zI{TRc&wGC7`o8P?>-*<@y=I2}?7iY%Yu#(_{Y>fYHZT01LJ#1R^A2#3RbA;S7@SG? zo%#}ggjQb5L7eBfjCJ2>&a7N_eTBApf6|n>@0biYK1vOW?10O$nrC)IpVY7Pkt2S-jkG`qf?qxdGU8ewhy54mt zqa5}R?lUr^15^KVU>!8KN`M9nx{rj208qBkM@>MQC&Zbao_?XN9cKooy*tash5i^a z>FS`R?YM)3lkLGZrABVjoiIh_3LF6`AiD=a(Ld-*xyD%bOGs=6T?0cEpFG>WGVgZg zWE+b8a%W*!=$3SjIp;w!?@S}EG_La~33;RL=HH`f54oo;OZo7nVk7HkS0t5=ruJB7 zi>O#MWTz7M8U=8lAKfyMXR`%6)54eXfA~qwFq&4I6^m;+o`UnN5ge|X`6J$=HeQmh z8;lDBN9c}|HH9rp*#W}6?8UaiAn@4E&Yz!oJ0tqUH6P4yx7Y&)!@;q^5cp5lIfQZS zqSL!#Ep|pYUi+XA3D2-cmqRnBO4dO%V~CL|3i4j++v_Ts9(+$!+OVld^<&`{`)*bf5fEKL3x>nAzHzN4gMVKo}2QZHLwNeGVB%ez5_(e~Tr|f?`+BK6eelfIPhj=k2 z-x|ELR1p?3=EA6q_qfN!eK?-2{_-7x21=IQ(3rk6>b0^BU&&jM=9qkCfT}IkDR@e-|Vah$ zv|6$yjhG53JCoxx8P}U#1C+|o+wRStRto{ZkQV-SoS#Q}hxQE<&NV`etUf@ghC<3q zXse{_OCVgeQP9F*9|1sOAitVGe+N4Mvk8izs&h~Qz8PlFH=(n&bY!GKX1cC87dg5`^aIA zWy-BR?$pYVsZuJ#Qz6^hyL{7G@gI)L4m-Q7=Ej>^`cxYljCW(ZvKe-R#G+I1WK~wQ z0%g9+Rhx5Y1^igg+ZI?tlh;m}i%W0WbE2}(NfzzN(yksCD)~74bZ;gilhmz3I%2KF zIYdYF&?`W$&S_@n>GW(|e9E%ES40T~oVKlkzp3n|c{^k0hS;Dxn`zyV5nBtr4l-R@ zPlbWaUZY%xJ2=|I=K>m6xm-CKzO($1)9WaM>{9m+@x@tTm$K_0C>RHK7H0?o|wv$Lq8j#SPm?kEjTT}mcqzNHPBTI z)y@q~5!f2`n~AmzjC2L8T)9S;1HY^OWn(N=lZU#xnNPV{GP?LeI@goMI0~ zIWx-`Mx1t&bK1^Ma$c@BRkpaS0krxsk#5K~n{jo8(dj|@JeS#~pOr#i^lzA=ic#|M ziT!6T!~XUMBM26djVDY6+EHlsNAGoMhCi=GngO_B1?(kI{gVt%4j#KDL)Td$Q?m7l z_gd!7PNtN%1BIaw4s8hlf`Q#b110^d*S|2}&-%6-{B)@bL;w$k4p-;5)SNHS8k14; zxTkTrlHg|G0~a^JF3M#q8h&7kXaU#Vc}niW+XPss0bWp_VB!U{6UDHz)sWj9zH3?g z-Tw7N{X}N|ZRlv%qEw*Jk#|2c56kOa&*hS%=$FOM@|l98n~cdEI)x0gQcMCzbGlkv zQlog0Ym#YvbH`0~R7Sw%!#pJ_^b%q{11Gkcdcjp(FU`#K9HsvQA4eF+PCC6OoOM_P zZx;A7kb%o10s|z)gf^2;Hc9}3WVU5_*3H%Jb)MyQ=@g4t9FkSlFHBG<~>VGK1d^|oD&9k z)!P{(HW+NPt_waAVepa08$neZ)Hojep3-;!kk{Ua4g5PwYvK2v%DlNm&kQh_w!k+F zSZR>mnq!tXBkM%pFi%6@Y*UhsSLD|*^fy#P(8a7CPvSZ2mQ#yv^<>D`ydpTK^VLB7 zbBO^(ir1sm{CsX~hzRcDkm|3N5*tCPSFIYtV4CfN1pLoTrufsi}S` zqj?LycW&Ap+o4hb=oJ($buWwx$1clhMX3H>IO0xC)UNjKoT&0lU9L2$ma%HGC2jnU zQBvevAC<`bEDSRbY}|2ka_y~5^{>W(d*Si}9sjyI<#7jO?jA37v5??XLHY|cW9g!B z^1cTt*?{9($QTFy;~-X*$*W(^>G|bv875ci3?=37Pf z0UWhQW_(R%6Vjt!S^1#X>2~IsA~{YP<9^qG!an$5^dp02D3SE4`DwIoY?tSmi>9>D zNfgrwbClrIj9U&V2ctS|GGSHh42?e>t*3wH2JnuRI?mrO-aMpH+`SEV_IoUNK(kdI zSaI^)&T#O-qTg2K6KtvxIP1(S*l#(Cm}{dmeRSM5M`bd~-a4bph`HDfbkgOVcA z9;nS7Eumk!z^7OwK3Oo`{*DjfQ}s)OdI+D00T%tNdMnQh2n4IG<>In6H&@~Ei+q0B z=*7Ip%eIDH0WNN>C6hkQ$1tVzeQ`400nM8%T5SGgdXar^!G^-T)2rAN6`2FV_S@)u zXOku$lv{=}WxhpbP#RW53(W<+2}x)mS_`@=dhME-ftU9-`IuY&FnR=(Nhi5)38|PI zW++CtE)OY&vK`gCXocl=#eOT$qhbrmm0mPE9WC!;-`sBE)G4uYAInh*=+De4=~*y1 zdAdp|@REZwx494C7Kh#Rour)Lg0|jcv*G2qw8+(4G|KA2f43En95)6SJ}G_fRFZFh zspY48%zmwYMqtBjaE9o`9;(o8-|QJ;^7CfVfnVI`0GLB?hZPdqVO7p(ZsqJtdhAf>dKoaaHs1Rf9Yjs5`th?4x|e@@ezWA7i-rWNXc@7R|E)gjbP zg4nNT3~xrZyYTr%3mMdC$H@Q}oAHBb&9-}O4S}fK{W*QTxIlG2$@J*Tbniqw-CTcA zisT_+>&{5(=l6~hJ`4VN{L}cl;y6M)!IITC!6rzbT!A@dH>&;T8;*BHvDC| z2ZvClqQW_fRZ;NvsmjCoJmpl^neVArn@(ug(mKViJ-bHECoR2bZK3KG}2JMXC zK=1zvJ`z=H0DQP3ztoyT;4}1vQxljJwG#jzztJy(>Yx|iU@JchzrN+B`pi>+x$>YB z-7_bL?6bGYb^ht~agMv6BUXgn#_p2yrty-%^Z0{U>Y2|5@7`*woQ|BOtJHzVI9lIQr|n9EM_yF!hmd=H_DDf_s=NzQy#qn%DlUGw+*#K}9TgblsoV2Ylrtr!c3IhnQ`Y@0}iuXW$sX`N-+(v3{1(`oh>cxm&IhT8MUJB zIe~40LFn2-Y5yId54Z`Q6^~#@gdc4Mj?tA`5`hE2+TusQ8%)4A4^S7yfn$zfTiy4d zp!i4<-%kg)*DeP_j7|T?5~D{T?Vo)|CosR&O3v0ap4uAdwB9wUW}KoyS4!u6+_38I zdM9SDYlp7I5w++S(J)hx_ui*(ga7YYFhaaxBce*~&V3LIMk98=A3JZj46JYQcNIbo zTFKcK<`JgOta>+QmICk({06M^&`km3x|fGGg|0;n%N%;czMu_P1`q~}OhPQ$v3J9x z4`ZG}ZdC3GeZ*@B+K{(ULj{mQQOvUP?TplSF+iPXv%vqKG;I%}49fZZZJ*h?GTH74 zsy3=&#JuRz7pQT<*_Frw3v^sjr zp9FL>m5v%g^o<&5xVPK5HWe5gy4UqhJo^p`7g`}2YPdMg02z9r81f1Plt=~5J_?te z#>_d6foN?cci`^~`P7cF*ZGd}zcM!h5uK zQ_yL#8?>|c-8sda^?n}P_U`Kab)j4h;g>i<{ zO>EOiPE~wvw3mi`z}n6Pl{Uke}=>vHhHSS20AXEpxPK9oV$4mdTSz;&u#Q9(cPp zZU-WbRmb}&-G!gUKNaCn9;>Z!w&s4uT={O0TE#zSNl&Hh>_{(&Ysk{zrG$h;|E*V! z)Mm{|uVSnrvt1okkum(CAJAs znf4%tF%JJiUBQ)-cW_oMvHyLrZaw<_ka@G&i0VE3G2th(B+bF*m8xd+^I7pFF9WMA ztr~iG6>-@{9BPVloO*aS1Di6!cuWQAgXbuNRMsjO-A@jDwuM%8`An-|3(k!{a$E+) z04PR7DJox5Xqj;G*nrj4R~N{mLU1R-&fW#H+v9&bz*zIcG*T@zVyla>@dnC#TfDYP zvspdZ1AbhZ*tUVFug3rKb#L^C{%+-dBl*miT(&f;G{!a|dNTgTcik^eEf;L9CXEbg zTsn-37CJWe8kyN18`UDxErV8O*5gZ>+6nRNg-myGDUS-4*f>HWou(-mWGcAg2l*Ie zij*mf48jji!-Q|X1e^f-o`k*n!HP16+4jR9g4Ry!k5pf#v|%F}SBiBKK+=L)RAKeuWpKT3}RJsm^Ek=`p8c8DC%E zn7e@{K85v9ygM7Qd4mX3^{E#-t;69`J9rjQ!B=$$>THJEHd=~hF3_P7LT#sT-)lrH z%SUWQo&%b_odZgKF)5;J-7fLNCZEWoJ~Ow!TB;O!=;0Z0&u(?>7qbtzlZusgF|ND5 zGV9?5Hc6yg-L{lsn)&TVerthag)h!3H1t*$Nw`=5>EN?xw)@}0A4UVxh%o7pGCQWxlLFb?VWs5TWZ13 zc}`6y=o~{WQ?_5==z776jpgGg79Di-MMv_8@!Kzv^f|mL=H%6vPi`xQqN+W(@%-#r zf=8*7v^n7~dC=m6KYRD}!DZgc80k>#y606a01WJ!Bs*kK$Y^}?SMpCH^Bcv+D5!>j z4!3BhHjUMhXZ){O(&qO9LY*^4zPjjp-~@t?%lhb0()aHL?G|51Y&!e6J|&R6zb>@> z;$U)1X4<={EtzA8cZ~0q6P0^Pa(zNw?vtllnQYjsu6C`-f-^z*q(4rq#AF_ZeBe1f z4od#XwKC(55oiFYF>=Fdp?~iPP&GA#2%tWjqq^uA){@7=7zW$l0z+9Jj=Agfidn(x zq}jDxWu%>`4K2=S+r%?<4#n;4bPkq-@b&6rB_#D|uQ`|R{6^$QPqhiVh4E^+YC@z~ zLFs&=^hB0 zm4~h~{#JuOt|VFp(dWdjpNu z;79sV>!F#sf(Y?Y`U^Nz3Bq=4Lo722`~Ti206RrZdPwHK+p2c_x2%@BTsRt@@8gx{ z8>qCET{sux^lY_a3v(;oevU1SXJwbyz3?jhqFG?U3o_jsMuRp7TpEY1#;S5JuJ8`( zvz*pGa4aw!r6`_Hn`+rxy>q4;qBqY~WOI^u<%2{nV6{uiS>t$h4i$3O`hCvm+)uOV zZdDZ9?i+TbyxQPmmh8^%BHSEorc=DmeUeW)6KZm%U9opCDEvpszp?a$KbJTrzdO=w z$h->HP`anxU^aX&I-?=Z)SinZJ)_L+)9_;(g%BN;xFlAwx%%A9R*p=-z3}h;b=$0$ z57{!#cNY{A5xtffHO5X9eO;}FDMO^U3t&V#gP}B(uYd*z1A`b2uI&tet_>WSc7Qg+ zX8p4g%UeL6E21WcUtm9>?uDUT0O~GB*a_}OT>hB_pH6K4o-9BRVlw3T$*=8nosAViYwhKyfNypL-znD*+*!!Kz3{A4Ragg|= z>1!#iekL9J7F`|~HcZpkC~+3?E>x6xe~;;9ksNovi3T zFWbo@0Nfl#WS15WE$ULsfK!tBh1 z(_^lviA!p;=xAsHCBJ-YKU15(n@y-5jWDM^2Fwk(k+Ys%V;7edpOn>wWP`f;-LK1u z9mRzTDsHWV%T)nB;;ZHO5{PO6F&kefU$7ss^B?HNaeAJo6YifE(K(I1*QkRZ$Su+D zRZxM5VyqSo2%0Tb^!~eu!$0EEg0b0~7uLM#Rq`dS;GAsW?kPYrjmj5HBexQUpDn0` zs6anZKGGT@idJv7>n+(Fl$bX4{-|sxSpNk0%s|Nma-WG>5W#Jd+Mw-#dtK0WKosb8 zXTpkNiOdO>?5{r}N2SDI56lfnQa$h6-JErAWu9E0zaOUQYMpN9Txdq?b7!FE9cY_m zlsXpSm2ej}x^-Z}(^6hBMDxQaot6`gUH4s^^!eUaYBcYGeq&O_elTA7f-lpNRLs_A zn@8*8A2o2vm=28(5IGLNuO=OKAAi{Mtm}pKeQ}JDP_5=-4BrVjs8B)}7fiMGv}&my z7E%-`&%Cm_!d9%bT4LZb>3Ozax}3Q(!r zI)PkA-nf9RyszQ?P`z}yEJ$zkDyLLPs(}`qWY6K1af9U~S8{=oIvz(d#3Hb^baBYD_=|u*YWOUdU=30~s9tu&8Dm ztTLELQq$+9@6SpwGP8sTs^$gXu}{IHpyL4NEm{{gt<#6{!uCkr6jzS5Tz0dgEM4?! z657F`^iqTfqbWaTI$mNrL@Mm4P>Jj7Pm`oRkx3KD%Bl;bq)M$bFg$emZ`!5 z?(a#vIp3+bX;!@Ygs8rg$6x^`mn#mW3+r8Hx%O0VLKSRB-?uT$NBrP^-+1ZicMV$t zb^_ubmx(@=;@tg2`1cd_ecv=gd{$q+_u5h>ub+RzppA{i`-G))?-n*k)%&-ve=lb| z8naGGpWTJ8R@goraZYJfYpZ@LWA9}>dbVIa*L%{E+!l3`uF|)HSKdNm<1664R9A^b z{-9)tziy@EWY+6Qgw4gsVWvA0bFnP}-%7`>lf=Rm4IPV10@j|aveP-(i1tA`BzgIf zu}4xx3}wFO7NOx!V9ru$5j;fHu=kfd2GB-C5PCEpLz+TaaG8~*-Rs}bg3rOlNLU1h z2$;Nib8X1iN#7TiN0W_y)vjj;mveEpPb4bIcioz+eb?aYzk{l$b_WsrPQfAwUWy~% z^_VqRHOtZ5ijDXzUB7gL{E75T;UtG=YT&waY3LRyhuYCnAM?yZr;S{;sIBs>&1E+4 z!aHY+lWlFn+rQRxl*`S=FYnC>pE15Goo21s$aEe$dg^jdb8fft>?mygz0{h5%MwYZ z<5P!4&KN4nydL43?ryWceQpN&FO60lrQ=qg%Jl0UMwz>6_3s(A-Tvh|R<>?3u^=;; z{%E&}^qBfx#SBkwYnLNjzS2f+<8`rp`Oa81wt2K$^LSwIF<)BY>Gss6sofpxw(D0{ zD#bQ-TtpCs<$F`wnG7q!OZ@DyH$9Cl$#>;ku84e@r{>icLto$m(yD((EmXoP^oR8$ z0hNZbi#BaL+GP>MP2%ii4a3Ewg(N2vUJp4O2Vp^RskysI5eE?x*hG9OO$EoLkP@e7 zZSN+}Ro2=GTJQH6t6W#x{UunfHnycT!D>GeyKj9wKy!7KJfrN`hejxRg*;3{I5(x^ zK4|Zoeqv^KJ?7N9W?y20) z50!Ez%&JkB;SSD;Y>A9hQ@O%c*-nYT)4lbwr-3+@bcp`SD|^oB*^LUrv$*^Yklf#5hzLarkGVwcWWD(Ld-RV(o{b=5} z7AAt&e^g!`Dwwrz=Q2G)+Ip}m+0KAz5uG*U=^IlcR!|c&vLS6xX}ImW62<1_iQk|4 zh-Yv)Yb)b17S@sv8I) zs2|FCB#nii949k)CC6V^Vc5Q*{MzXtH4wjv>SSM~g8c>YmFGT`2N9E%tt+%^?gkDg zCYnmGhx5Ci@hcf9oVT^DDJ$kGm&rA4hli>K zdCU5m+cHZg#U|=W1+DcJlUuLotTSb7ci$!>&(m-}1j-f|%X_OoleD@;vDvBfheoW7 zDb0OosP%Y$lgd^2vB_8+aU8A!A#^RC&vaJn~^u#6r5KzSrVR6U?HZ`+y1|GoU>gWHlRUWVn^!E>Dw`0$@iP1x2 zr`Mks&0NNJvABQsT_9kyhUf34o;I^0)_3JCCx2$u%!r_aToz-_>00U)TOGDjhBA)+ zV<)?_77g~5w=*5Q4I6Z8r{#e2)wpN z!lRM%y#HN;Qtk(_`1z-KC5xL>KIFaX^4-Q$QlTHy&lXN||0s|2F$NYdVfy_ERXdX> z6pu7n!sBy4ru>^JeKIU`S*P z)brg%-;FG|0b&|U@1}^vURy($ys;EV^Cwt$D6 zCty$QeH2!=kcN}kr~)RTa<{MY)RvT)pSv>4=^}8>(*a7DEpEb12wdZwp_#X|8@V^~ zK0eS@;?eqn&mJ_fsMCR&ZdkAhq&w;MfGu^oOogolk^oKh zFnYjg5D?E^f$iZLIItY)9Pa5$f zN8c}n8MGUzU}?{Qn*g4;*ueeI@dXw;f@aoSP+H;CXLgxae+ab*a9jmkRe`@jXh-vL z1_fN+xWLQG-b@}cD(_cS@AaOw>V-i^C*j?iCA$ai& zn9wtvxZ98od<72b>6@6Wh=C4I5g#z0MjMq0jOxb5{vrkZG+VBe06E!TQb0;Pstok! z>P}z*+8*5L;77y7E@C%vfgS>@L<4tMyn79T&g12Nf+rhZ{;+B=ESY8%J@44j0==@# zkToeNT=xiL&mbUe$Mk58l`s3vPGasLDJ7L3XMT8LP1ITMK{@Ar)ngW?h<#{y(1;IA zgPx9ho{Q13FI^5yMpPZ4`5I*1v>##C7$EwYt>i@^1sDw0U5kdCH8qSHlymx=EM;be zPr#y;g5?+>09OJomu*$r?|_aDuboU_!nei|F79wM<|_N@k?wfgnOSL}}<<2gZ607p;W^IB;=(mle`7WpQjU z;Y!{Fq3PXQwqgRPhPUi}gbY%hPq02ndtd&4U#17RPLKD2R>fn@t>OPq`T%m_QIMlTx)>1V z6|h!;OA$}8?%f5ezw6Zwj>=$hy#Ue|sp;%a0fJUcT;{a`ix!h!{D@vlKYp^^|4ZW& zQV$nifD|T}XXi}|9iEj#!#f^cz)!h_pohe2fhW4vERS6R|HmXwPM6wtD@hOO$)o@Y zO>11^;R|T3rlbTm#?D?lfyi-z{sJM}!2JKqHli#VYbJcTdA(fi#(3V9bzpu0e)Fkn zhj)}h`eZRtd#T3nkM@yS-&Z9Q=yFxauvw_*btNwxkAsB3rZI25WCRB+^8eWcvG{M_ zM2}zuQK7lSh#NW3Jb*k7`U`FR0QR7#>M&yF1$>sd@fQ(z)U8sOBkjYF>u0rYmZbL{ zI)Nv2MDpuA_uynzz0+D|VcW00kfiOC7^(QU(F`3+QuR%bUf)N>Fb(qEexsND)^&72 zW&um*D!Kw0Y8g8PL5DW93J5sqp`0_@cu?|w?ENFx(TM2%v8~X~81w`&D3!{esCs+l?!!WudIMv|Jh8HsuLuMA zxvtc~AI|nlHzD&8Fqfy;Sp3&&=H8$Y0lh`suk-`biOmMK?#}}EZi6(dE2$8R$6q5# zVOPaUWaLvVJ>zOo zowUTCXaUv*$1iuz^_&N0*qIU7O9*Kgj9^Rz4pI2`e#Y)*1+h#Z8y9lc5HMQ$1Cv%b zIJaDUh`m#eCtzAOfvFuMW*RBm#N`F9S(X*bk*;rWK#j5j^k6K*vhxus72pK&vdFka>No9j?faD*=!XQaZ3nou zf#rI$kh0TVwB&I;1tghJcO+mE=xn&c&@j z48($^$BTvy{DST^u@t1pE8s_DviGEx#lRD(SdMyGWkE%(d0(j3tS&Lq%b#~Jc6X@+ z$Y2R4I55>uc%z=57A2BA8Sodb+Ch61AlwkR%XclT=o5YTtMj{D$?du-tRughny zEMf5FY1y(Mrbkgu*8*tX2z85eGLKv^V9%}YHxV^`Qkd}jm($98L2#$%*%@Bmm{Dagra396+m!H)Zdv!QdFw*#h=53(!}bCGKcR|XI_PTl?=KzR5P zJn7|PsU4I|amVuOxuWT{GoGF`E+svc`6ILAC}8c3QK0#_{O+Jqq~w`aQF3{HFDG)A z4?*b@l#NmXNGuhAw7dQ+*)c<2FDD8IW?9t_WLGP_`e6qcQ_s@UGP*u`V_#stiJw+! zuX*u~{lVtU#ISYSwe!-*#MRaID{88S6=0EDCcmy;sKEmai}}`?z~dhH$M^f5qL9B@ zf=T-`|G;E>06^;_!L6Y8<89%w3^qT&HPAe1ud{#dn;RP&KS{SxLCF<5aV~;bRNB3F zZti*F;~(W&SwkDf4E9W`YspR9Qz{nlJa}-`-KSb6x-&J&Caq3qK zw7>*(qL`FJ>yZGhgUP2BT8A984yLf$T@WP}O?C0np5jj5vT0=54^ngYIZd=cLrwq9 zB={W>n+N8_VDz*Wwa-(le~@Nezd+Uou0M*{1oLYDR~U4qT0_W8Ho9Gd;P6EpSMC5BpZ6Op^@(4{0)GbcYr2gu!0R4}F2OdATmh)3P|Idk zfol*8i>kD*F27pyih^JA_=X*2(Rt8S3HBJ%a@yVnE1w3>4wgvFT|mUj^Cb3myu^Wo zlikMWT}pr8JTr(gktz!yoazI>Z$#`I3nU|k;{+$H^ zKWxZu1I8cl8vNbao)SLNi(o@Q(Sw@WCl^b2KrIXCBUU9_4h5_Pw()G+g}xWK08GE; z0E=zx=_t4iSt`sL%fk)B3hyM?H9(2a%dN4J0odbRPz3A4|oC5z>LabL#ni2muOoW z`$%ha%XCoD5IurT z)*H?Wk#Cx-RzR$ zQOlSpc!!e~kMdU(Kk$?F!o@^sENUVm|6n83)uY7wjfj6bJAWp=xEG*R#h1-Re63%;A2?qf3H+Sd+D$`kzPK5@0QO*eL%7 z0=n(7Noco0zQJxMz&i2(%Edo|Jn22`ps2*bC@?J&c3J&9P$CP6K*ri~TdUt=sGdgFAOV= zbDUTtZBq9&B;%@Iz`sOU7F~#dNkmv&R4eq z)KqQDZy}^hNyBa>tpEosksrClcrXPr4IzImC_et8mdyq7N)OCza#T#^SSImNQy-A% zPoC9!j9@^84JV-7oqIV@B8tKYm#7g4gE!82(9jnNKTRYRtrSmfLs1z_@J%;ZQ2b5X z3T$BfQbm_o$J{)uxMS3nIhmN%E8J$6+i%-74#b^Q?l>WfLG%b3An7msMs(-nQ)7E@ z%oCrv&)9awF-#!tiN_OOBkFhB*!DG4GvzpOpGfj1k+lYbZkGi@_r58eI|uxog9oeo zfvmThl~xS`&*k|lh`PxPP3c`C{ftdCv0cci{m*LSGB`;vT{P@=Kzb5o{->8aItsPy4FEW|S zXj9PNgyk(=xFyJjs0-a48RxmO<7WS$7GnffpU@O-p)V?K+87J^ZhsUF5u=!{#A zDN#JBJS;Pw92OdL9@FB150ND%SzlBHSoave4z;}CO$eg!CPR_5^q@9?>WV5u$VitK z-l6MbJG>hizVg$BMFX%v+27?>)#l=te`16Ui+wyr4(zc3sA2!Q0mTcrLZ`p~b?BO^ z2m<>$PZ+95ge-whokEKI88Br6?mB{B;3r4(z7OI!`-6^Xm-$;tuE>C-X!}uVif5zS zxs{Cp!MX!KbiPE}mtOpw0q`WdC z=)GT5aBDtW3iq!i(VE!bv$~e4;L<_TC1(Fgkh2mu$r+m0?{SgDaXe*Iv78lM8*Oz{zTy@+(zqD~2w z0jaVBld_|+ZJq6T;_#o^2lasP1lR%TXi=0)SzKuKY?94&#a=PwRN5;+SU%GacmYs@ zabeR?JqV+*&re6E_JC!q#b0HCXUW_0dHg5Cya`Z^(VG8t#6Q%qA8 z_+~sONV>_6U9+%_wL1>J`{2ae}W^hgW%N~XN;y3LVFn2Q*Ma)7S7KD=O;HAr{R{}F$jHC+8*lE3e{W1=r zjd2H+K@Eb+vHO&4iRJ<+nLSz~b2s$g;t9;;g`A=B)>X_Y7`UQb!xId6?dNRxN8rmh zSOb!{WvYy&g{gzzD2RAu8y2W%(2dF@ag~~3E zy=r@MOHWT>HqP{^W{KHytJpw|3HD=uQJ1)$=t&av7}2Ta2grF*KL-;cPW>r`nk5jj zE3VDxqh?!fS`kErk3S${AiXL99jSx}__X5i&&cYzrfF0;FARS6!wrs$Y7P1HmcYQ+ zxKXE1RN({DZ-hg#A!Xsb`FTYCKPf$EEFajM{{YBF^__E5j+Kk}$oFf`H9rdqaSnb} z&_C$T`LNi!kHJr+GUqz&aEuXMJ4KP%$-Gaz$5<^*>Bw7AQiLk$Zvi&2>pn3YMLmJJ zzIlrVr$G#dxwD;Ftzqu`^^V6yAum*ueP<1<-GR)a@&_$eGnS2al$AT1;Ad@`buPCZ zEADg$Dn>JSs)=nulE5*syZ#PhqpyhsY1vf}Ak|*|7r1mVK1P|b!0URs?QWSs4Xyv3 zuNyuFKb^9)3kk&uW=5;KBz9cKQP6<(fY`y8Qi%@yqwGJ)7?707(OV^k3m!>^`8@@u2J78P<}5IQ_S8WuEZE@@c3kzE5n64szAMH40~09dO?$Az^nlFqxV-O8NJp4*AUxAO9OBD3MbzJG6<>(jDDoz2zQ$pOr3 zA~iZa7DY~vkR{e8K(@xxQ>Tks#xNMR*CNELhiIDncEz8pmP`VPEc8ItTSvZ!yq;j1+o^(ya!n#nWwUqS9*E5)a6w)EeE35JvYj3r zNbWBHjl@_UVMutPoGJ>m5njC$+*-A7jg{TFZU;T*>GVy29vng!_9g9#ru00c%`VCy znr&O5KxE%2s^^zodaH%%!Tqw$dZttn+diJtV$Nd)%%3HdTIsgCs68D2C3_&{ef5Iv zF-b#k7;01~g&Q15xz1+gkA?F4ScvpvqY*X-bgz6hPEzp{$)`0EovgbRKCHZJh&nY- z{!FFbLcFGC(g;Gglw6#6zP(?6WZ~%xn%Kq_77CARQ{>{4<8vX6RXG3?u3cI&Mw$#H zKvyiA)6_19@W`B{$&kSG+z5H9D@_^5uR-fa1R;q#Hfn~;pI=zu#Pbp6LUqTJ2tI+m zmgsZIv^@@(VYY-`QR0!D5nR|rwoh%Wd-b8hz(niXUhb1_mp5Bp(Iu5*#ora?ia#c) z3KJvQmj-~7bn-aYE+|2Q`7$~A(J22hYk9lXY`KhCV)qhYNqqk-=^-Gmf`0$Aq!ip> z5AUM4@^eS?tFcdA*zR1x{&Tg*bg009kA6}xv}lLOdx3&i(UTSKRy#5+DsK-jo%3yV zj9*n!x_z~L_XpeS9`hk*4aC^iKnxTS=mFQu#pkSi!LJj9KdoYt?(unE91%qQ+LqH9 zxzolY3P{c>!;Sffb-ueXXHlV;Nrtgw&6i~+e(~|<$m<)ADeQp$(Hsf9<9DQ5 z+<--%U*!@g!JOH}+}*h}7jI_nSh3nJdTf1rn_&KzgzM=Rki&>b%4)76pIAt=|KelC zj2>^-n}@DHmJF;hQjS~mweQVe@!$dr?o9Jj`WI9Q!F(#a;ely>Gz`4g4tc?yQjmq# zgJvRQT}oj|8^~Z-&5s0>g~Zd;J(8C#UEKQ)C^NS_b=FP>g;}kZv@3rHaS(L5s#ne} z;Njwk%<{x+y6Q+{DD zfbXs{T7TuJJ2Gw;fXev!K)ySpexkIi(ViTlXX=h98N&sg)M53br3{}bqSx~sE{bS( z^s;`s9LLAj>>JOC8(}}#KH^aBH_kXoJ#JN8*-=l;uhA)0{=O-orr7O<9s!7??&U)x zLF5vVR3Mskbx4{uHv$woZFc9)nM)qx0I5U5m%Ax|ugrFfOFRvCJa+g@GYqQF@F>{r zici}9n@e73x7i=j>~j@66$k2);8a z_T?9*_(kze2HnAw)W{}%$7}Mgb-xgIkG=X5Z_-y=sE~pVWJCXc^}8+j!=B`&VOo;; zZ-pKeSNou-&pYoy~LeV?BssaeWg<&=LEf@ z+Ca+u|H-6tplz` zk>#IqnZE*{-PqoB=Ypks zXy;L^WbW&4@zlQPm5q08+ALR8VmvAx%z&PzRFQvj)5h4;Xs9E5RWa%#^`M+`IW+-B zBZ1C=EkgVtTp*4?#|y@Z?tSekwg`88tLlId(8q3%LLhR^z|D0?Kbd1qS^Z{j?k)~- zFZj@9K}Tps*4oyALc*#%fHB|BbE`b5kzG9JR!*5)E5dL6sfhsMT#;ddWq91JMv6($JTI~-X#lV(ScoXNW zhbBYB;_pZFODcAZpj(|8h7IdcqGT}322beX zIW2*@tpnvMJ|&G9lt5j9Kz9cCx1K?!^Jn#EIeOz)`QvsDA-iE6`b~gXbLD zmmUN8DjfTyS*jV5{AyJkpRD*J;}~$eJOk=zjWkAIJoN!aBk$Xhz(l`Wlv2uLU$ef; zyB4%cBBQev)Va;yOkXLCl{b+X8V+z=cCa=<%%Cq4&tDe|VUEM)=uTc8M(-q^H^BJS zil7f?>r6u^NulK3P-C0L7COL*Al#(7UVFo}8a(s$a*i zJnhfScL2-zVcTtt27KZ%8&amA)LISPl?Y|Q9uydpk+D$rHt%%%&7xh_@`|pQ=$wE0=D^bESLp)! z3~8ipz^xbYds* zTuoJORp3<}PUo~d$do716hBDmjW=wSF=%Ft%h?8HdfY@!LLHcwRk97w39(IMQn!(P zo;5c^YIcGKb2=?xvX7oZ#4_NwWxb={N;LJvO!hsIS?H6-y$YBmvgQ`K3){Mf&i_;``5c26`{d?teBu?DERc<5JS}s<(>Ug@u*^6xK_+Hc*${T#en*Y2tH4plV$ubX1 z5x~uYoxC^rt?V&KIygoxgU; z#Tbp$A9|v`5P)=q;Qra_ELCGq+)@Uk8OO*9}uchYj%MZ)MGXCHQ$fIE&CNA7h${H>cp2?j!o5x^$h^ykqv|0w&NJ`^U9kRClTEXlD{vJg*^BmSZt_yY*F z31={GHJF*o;jAyO(TL4MA1B-zxkscOJi~dEy?uqsoHzUXidpV|n$6UJy-*AVZ@DP@ zQDN^^xhO6<^V>STp4fesZCB-5x{3~CKzcEy%q?>>+~OHN)@@X$>tZiPt1mDTlDK81 zK-g{m^$9J(idOOWRqrkbW^vvHl`8@SNa+tFQ{DFIOyGx#!OQ2fmVEZ6V*J%R$;!a5 zk*QadbRu4;klS`R5G7NXTYF7Ej>wRk=>|7215tb}2P@ls8f$t9?7K@ftUi=)O$IHL zryYP3^B4qMFi_hl+UeSF=MdX*Jbnc-X7OaQK@^Ou(r3xaGTPaOhlI}lTvXRfK9aMV zIaPXVJU~sZZ9nPoKpBnsisT4lO?|=yG;KdB+k@s8%DthEzz%uopVv@v)VT5%j<#{4 zgGcdq?~~uByxnRaEw{i%KiFMuv*Q-NF2_pv}F(~`o zb)EoJmJ}D|EIklL&vXtI+9fQ>XRv;wyUVO4??H-kz9~WU{=<~4wkI_4#Sv;VCE%_Y zn&*}&ULV9a=JZ}Cmu0kf{$g&g059DNm^*|o>}_}}nHY~bbe!*!cvAodB0R6B@?MT) zr-%ooB&MSe(!Dt~QWo6th6SPX1CC^5D4{VwH~k;A_U66^lSjCl=c8;9#a6ZaiTKCL zJthT?mBnbHxLyGzSHwF+UnIBCCRA3m~+;oH<2b3oT?Jk^) zNo|lcD(c7sRL2gGqQB&k1nLIv@(+V!po^#b;$j1Yut*a#on8TL%sj~<0p|l$y%7vQ4xuX$TCAtX*p5ZDk7;!*|KjliA<5Eb4toy z6xl{0yOBvMYoW0-#4xsDF#F8!^`_2!-{0^3`P|p_yMDj_?yKt@UFUM%ulM`)dM=OW z^Z9r_U*O9}UzcrMi|9xf_@4@5+Kd0v$+As?j4m*1BZ-Ls0 zZT=nG^T+xG7dM^tfNf^9xk+RXoiNYo(#nnmtd?yZ%ysDCv#)9HT)}m$*|T`8d^|Zr z=alNv4qP2qwA3*(*nHXZaBm=VaFF0{ z0(G9-Pe69NtRQn&U=Hog?gg%2@W{2gOdGNvY**r$!)15OXo6rpc`>7Bm}H0R+B+ZS z)gPjD9D43UJC!Hr6uj%&Vv>A)$D_^8X1mziL~fbA6jU3b_-Hp57IexZ?nY14l1iRW(~e>g01S^XmHOVN*nDKH@1?fA>*f{XH#5d~*< z?fvuCYl-cm+T*50OX;AtFE3`$sI<-A(~~XI0c2v$THFAcI<*&9!0u~nd3X*^@h;+S zxYT5v9ZK3Ay` zAMYE6fhdP~R!UUwP-gg!SLPOf`U^e%($sLy>$S9nxti!E#WNS{d~HIr#4wTV$;^oC~z4P7JV znNxxyP!w#i#qZf!ErWzIi9Nr1aXdiT*+%V4_FvuLf6jC4g%}Ztfs*9TO&y>VebIRx z?csa#TlLvssYgY(rh|d5Zvl)?iS4>aGhaKM%dWX7u|G6*z>KpI;-F2U+)g9bJJtjK z(pn7xl*K_v{JH0+zfJZGW-%|eds@A=KV2=N=U8vDnjVNcAo3K`cd9XH8C+_0VYy35 zKuuxNpksTB-r{CMsBCamRK#%0rhm3Q8iFdCkm~G zu+gi*W2M+r%+11xWKTNv;4dx!2!M~5>^XlJWTzJnYkJqhjMY2z-fXVaT_^BNOvDi4 zwf>Vf_^&0OPuY@oh#rvH+Zk>U?j$Ma$5EmSSRmprhu1{anBEPgm&)Mny$8Ra;O4T9y+N_ty!V2^J(om9{QO(d=ztyRtT( zH&=BB+tt=>oil&iYO3qd36Ul3Q-}w9c=(8d)8u1c5>q40h71a!I*P7zlp(lvs4iln z4`yfZ*S-pkpIen#2CPY;wpWpg3>0w7%mvfMTJ}y}2JWWb5NNfdlws%Ukc5Oq=iHMCL0{sqlX5X=9U@n+6{A)Wz)6 z=gEK6>i>#5dIc6xUJ*Jz`s!-YzeODreTvz++F$cnU}Bq_>)+tR9(4tK35bh4#=g{w z{&T$O_Q_aD@ZEjtr+9Z-j$6&<7lID|HLd-H3HJ#^*>6Z|tMCXk_a6^5kmKVowXPB- zY!v{r;l(9#;em6qUvSNQ2jq#m@MO$lVQo6r3TQ^^%TNzPq@~PV0j~^ zdzOj4dT^Awy+$iI+WNbKfK=qn(dqzXC}vWo+P=W8KTZ2@QI$Zk@85qSTDo)67WFiE z;!fu_2tsBl+=Z{qzUln?nH2X3`p(siudDVWa=&1Ug*frEN!NY9g2YyNk4}^AAn*7l zcUfEd>HOD#y#u8nks}XBtx_&C0eP~*?$hhYvi~5w{=%WLn>->qcA+wfD&Bp7u`cxa zqtZ#-(UK5?WUT5LkDhs9_(N|8znCV9`)(u~CsVwaY zZB9_Q(=?wtJT?Qq^J84CNEfhLV((3CiIX>-1-b?MbfrX%?DETMMehFr63yH1TLI%2 z^-=u8LmLU>MmtW7lah;-$}egDPvq+aaihSZ^Ti8Clwts<_}zKy*~7Q$j|%mbCY$4R zC!T{Q4eLzUI$8%s;QKI!vc;wCcxIegCwZHBj;7r^sizx? zM?b@yzu=s9J@;1VC%14VqX7Nt!vZJMfLNc`6A9d58pUu8k~rANZ_+LrkT%E<&UVm( z(kk^8f*=FC=Nj=JGq%=7aegOQxYO9GvLeJO?V(Nck>_)I&Y2uv9mK~uYR(NJgSWd2 z7_$84e1A!KT`B#FkYuaO9Rep)p#}Fb0@O?)0@xF6NBL2-fYns@F(|vDYX!(kIgI3D zP^$NH?kihGNUI=G&|Ez_^NK)%s8J#|ZZYJ^lLWKDYYFwQU263p_N*&5?MV!f+;7RK zz@QH%t2LQ;AiLU{uZlK#b?!f$O;LbZatq(9uI|d{x0AT zjMRN`P&?A@$PPUU=wexo%Z@ubD&uTs&PLdR&B;GrupgK{#j}|a3B}2s^)VZNi7S9) z0P!C+!{qGjxC^$^y3)|8a$@JUoUS9-l}(eIHQ>FQ@sXp_v#h3T6Wa)BAgkSC(VF|n zezQl3*(2zq4q*R?RLZYk9B;O8YTe~6VCD8Q80wnSI|K*^2^x=%{Avw94;3*eixWMS zKccN13MzL&jW~^(R(|@BNxCMkcTFxj@9&A{d1qGC9>(4Z)krY%5CVm+*teg^oPJW>G=8$15$y}0;xctG5?9L0Uq zPevBcz?*-uljvJw7|~Ac|CnySSpzsCsIlykyvG+{=k%uj8f7KOPf8qQQmekc?1D(> zU7CRLTDE8eb*rt4f*Hgvh&vT@J#Fq{ZH+}Q>zpdO)Dhc6e1>@o zZ~JYl=0%vN2T3mlG=?NYN?hhqKlLR<$jHLV-ft_eHXj^5pfrt-opdO2+Hy(v`!65~ zWMVVqWKIRQZWY=p4aWRcZMD2*Ba>^Mxhr4McB-oByWQ!xQc+9bAXamNKX!J+fK%U9 ze)=_DO-2J*`tI+}Pe;?BuB}4ZNr{g|7aRvL<-c4KaDaCKX!@h)LMb9DJKD8*#I9Who=lJ|xnxJ~}zjRQM~r+@9#TksK|pBy1R*(AdVu&1|N0}vJOzMhJMypTIn~FuF1t~5g!54xgU?47ThfD(Ev{8@u!Pl?@x#@ z>Z=ZxeLDlz$W{eMv&+_8=^qt+0rCib|L{@3FLcC6t>hU2^O^V-9#smq>sq(Pb&($$ zYunZI*Xwo*QAT~320|D_3980Xi#m=l$D|8Wflvz*blwjkQM$R~YbHtX?~O((tQG(G zAr;UhL|Fa?3N`mHjKn-@(=|v(QE{y#0P#~ z4pxa`Yzr6&E9|(euO7Wh(KZswt3f@+nj~ON)4+6tHCiZitQTK^4UriSUib>4rG%lb zEux!5wq^6)J9mS7pI1-yKPkaO79=M9ce{oU3XKYXU06TO*tX;pmE){+^Kny?WgnFP zG*g8~Ipg59;O0Q&;ufv2<7=tRO~0D02qFM49l1MYD=0Fcxw`Y$cl?{3C(CLe?><1@ zH_w@??*tPz+Qu$@0=e2#3@Bv`dLcml_)AlI5dKIA*Mn%-q4evVZ9tYO`OL1_0#Ng? zm4+Jmmef~Q3s?JhtAT22$HBQSkQ8wK%MD5jT=oT8t!f>0#~E1_4SkZbTG%fR;!)7F zwkz?tN7G%f*pb!o9xd`mxzV+G;F`)j%_~TODd07}<=>N+{&Z4xdDVbOzsA$V0?-Vn zxwA#zWE}$IT$=d7KKB0miXmmd-6CQmuH(pSmJp3FNuf&-GBKJ0t*Jya+$^D^gE} zdM_Ryzvlka)T)_fBG(?Q^zR?63b%D^{hYa4iNV1?{?uWxf(*3Xr*?SFfERr&wYwT% zzYD6%z&5P>(eXO(F;^rTq)(3HcAo!3)gY=1f%q_D(!=q-PXFA+e`IRO3 zPtbL>Z}9Lf5D!B3q|Er=Sd;R%AO;7={(4`Q3Q)R-X+TD2ju(Ne&(=`tufY}Pp?>$| zN3|lGCT-S^$@1SF(+*uCrrX}165r7Vp7iNV{j}wiE5vKR^mh?*ZAnMhvw=;dv^Nh4 zirC6R8xS`Lnz%s=^?z^q%5|bD5xMV@ZCql4z*n0+0fPT_ceT|o!T%fT+N=9xYtPRi z8#~Q1U#yzxnOI<^HGmgB1md^JhY3|syA_2@C~hSQS^nZ*oBvtIOxmibwvwSns>@p7x&7a! z*O-C8>YM}5y?i3eH5z*E!~|fEhKa12W&yE(W16D`M={;tK3wm+zZ!OP!3$?W3bT4( zK+!&evYg(=D@4UC08>_5D6H1xK>lCs?ef9j+d*XQ_aTuuXv^Wp>38x0+UbBrf!0q= zqmu5O@rRg!O#aH!8S9VQBg<|BqeYe%IbicY`pm7i|A4bVgA?V3_8NzRuljK!I|?*z zuWE^%5Hv*rYZeCJ$dc!a!-6+ujz+?O0oDDzSt_T!#@FZvNNT7eqZp)Rn6dUVQN*}< zW>d6C2v_ls*1e#H0X}Xu4+jZkM*w*GD~D5Th(iQ*!Tb+ub1h1G3o>(Vqrq%(K{LJ91o4tOF0aGEp_vk^!WLx$USKiT z|41z;hv)WG9S2beq-x!V{)JM69=S}%bnP7}C46hWsf1^NRMfE0D2AB7r~2y7C0U2y z&7c4_v6;$pbpoHcYh;=5vt?NDKQbWT+oe^FK-=lG*s7c){|gu1rT;tbtiO#Rx1pPq zr*9yKl=l5H>tK%--MxDgvKTR|l_YQMRk`qj%1AS9YkT@k9;~_@K$?fQh*G}@r~fbf zdDGn)r#o(App%9@Q|IM9W``Etl&g9%@PXLTXS1WrY9+U)_e8A|szFJJkik{2khosF z45UUaE8vA5*v@ac(e|4>(pD8#Na4S?gH!Wr`-(pJ0W7S5ej8`wjeo^DaZxvlp-~Mw z_#KraezjH2{PDoE6j-t&N!DYcfBpf^5Ww}Hjz?r0FykBkX>l{tTsygA|KjBE*iTUV z^+JUa;1Ht@FDHS^Q>1Hz)$KzjSXQwgzckFm;&6;%T$Q?QDMAT&#X#8UFv7(P?FeWr5k z@2{y{=eBMG3b8s0=QCLL=sUfwYhllME#UO~zD@M)HP3O^)v$!VyEyXK(AldJJW?CJo3?Crp2z4Y`D2UZ0*LZ~%@eLeM@lu+6} zt%|wLib<=76$prq|2G^0OZ=x`cmTB@eA8W_FP57cJJUlxyp_%voH`>55Wyw8(tDsX zRD;sPT|K{sjgvupZr2aWHMTMk4}FRF3}@yTm1%?AG8(Zkngjz=jnLe!MWc@iVygpm zpovE&7yE@VD)2WEJ)H;ZPOLg8`GVKr-o#VgV%9-O+NoOJ#PhJo@U%gbt`iyw87UY_ zp#_C@>brT4*7pHfE5q^8OL93C$&VnqGeP8RqMKsvRh8-GEHFRsyufR5u+^X)e62CM zPYt|Y`gzJRU}tR!W{fXM2)OgoOl|yrmz&Ez1eH18rsEDw(iH_v2LHy%iY8M7v3|tl z^O6V5kHosIJPsLor*Mwl?%KJ-(cUN{O2UOt6b}dk}71BrU$G;&TV| zCojYD&XYdRxx?~^!zQ*kA6!WaCQS6@fhLT6b{E3F7=X1aw1GjZk9LQ zkxE}`Jg4111ukmB^SX5!PUAw|LfYb;?Je?Y#xJE-D21U=>@yqby}G}D?M>#kSY-L7 z&hP%Y#l!#g{SHjp7JZa%MuRxN%UBHR*}Vbq>{7|DKN3`yS*O*CY-B#21pwinKu-4v zFm-z`Rd*Qzv*3AZ<-N0Hu72t_;c)%Tpwze8^IwJ1ytVV{$0yP5U}1wcVq%}Ui^UH;8d{3y(k2(^Di9RvR5X-w7iH7YO% zGUoZA&EdNz^~|F9OHIg_{=qqTL)bJ9N&ZTUEDol%;&*}J0@$?cO5Mjbp#g>!QwI2r z!T6@w#IWFI7yedM5FwECAh^oEqJMDy+|*O^;5h6sdl=8zF%TGiIMHf>$YFID4eJq0;4`Ddz_!Nsg^OIJb>7V(~yC z+=TSGF$2F)hKn6@P(DbbF>1Z$%Jocc`Z-Phq!DpEnPTy~jL2`J(hdXw?n;&mdhR;PLRTp)3_N0$J zIA~J9`WU(&8Ba#HiMwpw7{MbYMj!T<{#g9goIQV95cR-hg5lfR?I6!Jq-(j$R{Ol| zl_yOJ*Xf!}hw6xySGIb6KX`ax{zG}q0jHpaa(9$G_TZ!t2E)x@a*Bt-rVIUBx*fRN zYv#Ij#D|#^LWlWd@{%zA2W|t&Qlj+@pY>plU`$PFD8s{B z4-DT-dA5Rr1_2ttg9FZ@YnKV1bI_4#BaTd(;$7qA6AE%olGEq>o?-(G)(vO&nl(%^UIIWkt2 zh9B$>uJlIsZ*mBH3#CV81pvPBH(bt7iUn{^3?!2WHLRwT?|~@iXn|_71<2?+BhQ~X zI=GeszXuWpJDgQpr=f=a+h;k&?7IpkUOVf*MSeoHhj>2?6e&G4VPlGUp1HAz>>OE{ta9T#~sJ@SgFn;or?+6Mj zu8>Mk{M|5MP#>9AVzA(z!5OG@C4>*UPmpuaD?eUX)4Z@@1NfL=G^49yFWK&4RbUe1 z{m=n=`brO_jxO%t%p)Ac4$tllmm^L|?5SN|Zi=C&QDZ}K^oJ@iX#{6V=MI{I>2IH* zCGJNmI0uX%XfT=e))^d5sOx8W$M%{1d6}K_sBr%tcWhb%C4Wx_odxH&93;6Ko4#c* z$=0U3ltVx5kH_g$vdSAZUDEo=JhlvJTqk{F`vo2iFVBa6EWtG-___m3b}9B$j9{w; z_t{1oP)q2X`Y40OiJ6Z=bNNOon5!vF_9?zw?wp!n8t7bvxwToVYORCYmg5Hi6$keQ zw)trcfGWpwtJj7leS(@-@X6&hO}DK~#W>9_921j>{>DbiDx?xWACUTW)!_Th0yPe{*%#gEn_!oB^{e8e`(~}!$Mkg?YFsc90GBJ|1hn=&VgfC z!@uC8DRAB|oK}t4%3J>2`8;2|9jU%LhtUNYLA1Ps4xjCBkarw1D5zt_UAj_WjarIq zYfx;nNOTncOuw%%jkTE#Tr%hWB7w`1H~LOtHNI7Lmn<)E(Y ztkDM|fEl8Fr+y^D?3~85=Xqy)mQMrYJhWg%`g9eI1VIy+uO@g?_WOhH)xWp^00y~rFRzW$ybl!bTFp?AeKkN^Qlns? z=B@_)g*QqrJeJhe!nmb!JXgEsM<^SkgI*~8U3Y0Y9w#|ni02h#+DJ@CFn;K?R(sd4P1)0X(>?{cK%&REVG#w4AvTi7ES@xig*{kW}F7dJ7lA6g*2?|37x!aph*upL#}5sWI=8HS2+- zj?t)HBQZl#`a!eVOEOZh-O+#kL+3F$Uy6L0Scr;lRYBgZTuXQ1FBB@>XW zQwm@o2XUi(d@1gop?RPV`@(@mUAmT#S1%6TrIoS0}@y5ZE~eY zE(~juuQ#)4$A*E>+HLXjpNCuxz_ITqKZ&z6;fpc0688I2_#m?r7<*em^hDp6hwa;ROm~`0?P?dn_W7k;Dt8-Z%u&1C zP4PGRtll@kwkz(ATbDIAOUXxVhOwz^g<;;O9K2J(zU41TCOHgRqr;$HiNG1MpI>*5 zus4c0>ZlXmJ>3B2h`aA(EM;UE?Kip9n?=5HwZN^wot!0T31*)w+|LhDK?Yu~m`lzm z^5?R3B2PUH18yluYUnxR94@mn2g!Ty+wzx=#!4-RjMnBM?~>cFsT(^_dj7L{l8D>F zOn{M3hNl3O92}^84OV4WM$C%fMrsJ6;mKV32{inCi2ddOkfj<6Y(6M@S-azjLBlx( zzo<5Amd*s_&@HoYu#u&phjK0WJ_C`CUT*H&`#T6+gjCTXy^W-MSk%!smJz%AfXl2_ zWkA%}J-o=j<%$B@c!pM?5@$ ztg4%1Mv=k4l~mWAQW7MT4S{N4_HK}D7^irwQ6&Bgsa0Ge78+Yj9bpgE_KbbDHr1`+ z&BD*R&x4qXFX_NxU>fP=Jz*)n?EYcb-_3KN_#%RoI$zjXr%!ZN2@3C7%344yGjVr> z5g{0AyAt1GfqdZ!a%aTkBliOT7IYomi`1WA2YXjJ{EoB(cXn?F?gUvlT)39{l9XuE z`97^})}PP4M_p8~7S2S43}!T7QNEx5ygOy%Vu-xK?@0T8a6$%-vNl|qB)nqN?$F}x z{{Es$jCfy|!J)pT#tg!2m1P{F!g$Xu<~`Jt*EQ8RX{liuq1(mr$KQ6Gyl(SM|OCRX5oiuHdHO?GG}lVMSm_^+wz3(<+c7 zdsK(L2DvR(nx0gv4zR2GVR_sE$iV8K0bB!!AR?F+#zjGnEK;m=TqiR!$T($CJu+HV z$wOtw!{@+{NyHa5ulcbS;K$rH1Xosp(nOd@Q}^xBtox&JCd0hXI1mvR8cg5C*6`Cv z%ec^U3$-|x2(KQhGyNGTpF8ul6u8@OR9$X9H#_a5GI{$1F(%wQ1UKpzvEX;skIgGX zi zty6BiJ8U?uggl>fB!e@_Zs{RVt~3UHm}R_HBVdN<4@RfwMmk<+u#OtCsL}BmDl3bxU&!B~Kg$r; zG4jnAoeoKZyvnJftFrKNTB*9Wj^`2-;90Q-D)+a-K@K3jK;ryizEoO3m~@axtUGtu&;?2~65G$?wW5 ziUgL+Si){>cN1>LzJ(oSc z9bIX|XFtq*l-ao*jdV%@6E(qUI^IKkw0cQZpzRGJz65NcEO4G6k0rZ`S$Op<4~uSH#=PGi)d!YHlvrgm#!36G|D9)n0IJbSu!}eXh$;6 z`6sR8;`(pEtnHJMT&dnq zJPka{?*Hsru7atrs=7GHvwV|Of7l>-n-n(U0j7BZ?D&`GmD08JXDKvU!nCaG(IN1& z!R#&;ZoJTnlbknRNSL09tS@od8iRWE0OHclnkviE+1% zFPWV~Y}5RZ%Z~1fnlOdv!{AmE;Rfy#Z-acv)k{&@`&|^Qz&#VhsZMEJgcp=_Juv-O zklfR&v)`EShoD5PA^Su|O4*sy1EKBcUW1wmhD^iY9D-X>bu&IHBWt5^?%BQh;qgr$ z64NhL6!-)8$R4J)85D`kqlqcs5AG}~T`H1F+c)MkN4;MgtRrwk7{RCdF1_)YDvvK& z;eWsk%RDe;6(4Mu?&su<4LO?}C>}tC(@v5euBSf}?MNIRp5M|gi5(idw_nQ}+(tto z`sY57^z(Hb%E-A1V15_>qMp>C=T+F-7dCvn`83QGMsfKH&g?jM9>i0iM%aGi0mw?3 zM|G*A#+9s86M_3*Zh+js25|q4c^)!rwFYQNMm6QN0Yp}fN-!x^M$b zYAl+QbqEEQKJYiy(P!fiIc>Q{(=Oiyb5~e@ispr8HfInLllU4OPOLrzbGGm9y_A2B zSkN_cFo?prI6uo@xJPBE&p8x)TY``>2j4%i6S`^PyC^I$eJ2b}SSH3oS2IwXmC*7B zhCNe8pGwLPdcEJ-h;reL!DFtY!VXsuBjqvBg$Z}ung&cglEPs0>fs?L!Ed8#li!QD zjuq{rtzzwGG0b-i&@J9=V9NyUy)t)IzJ3oy$(o4oZ6A@s9L5ZNcjUG zSx|Z@&3O{G))H{$XEUvS<}JCj4KXLii|0VHqkp;A1_1-!E`@D0`!Qv=Xfv$Ly=P*m zT>F7mv{hu<x>ix_u$gzL`R1O<@B$a9BnO>iThTV=Wc!VQv=_Ltd&z-w#Hvn2P)xoUG= zXht?P>YONL@pCLrt|L%kZ&xxLn!xw;y=7!&6|$#462opY*@N@!4~G0_f zHL0q2dWDN?lY@sUz#|>#fY4n|<&cBnCFQlDx#g!5O6Ywq0BnuyH5d@{TA`n z5@U$@H1DF%sq~#2{5Q-_8$P)H!u{t)#DFr6Qkkn1t>Z_Y2UyDo{L>W+ULp(k0`e=9 zmWlf}e{#yy_KDfAGgdQ5SMZ1U^i|-zx^_&1_EV_dzc#$56q+4E5{hiIp-TV@tffqy zQHTL|2srGko#LocROXF$%`kD+$6k4zym3^-#hVq=!jPj!9=f8GZ_=s z|IZ6e=4*gpT#t2n{J!8m1&?MWQcJU;51rdM@Cnee{^hqq6GK5c6{51RQdT)g8=b>J23c9 z0R;UhF?Un&rZkXwfBIhkEpJ%8M*t~KDL>CzFnPx+WeqUw3x+iccXa#)j0$hVrk#U- z7>Gb~ouobfsR6I$fE{wyTn}RwyT%T= zZMB7Wp0Y104D#*Mvytrfxeey*>jfBa@dBVhoLMYd;|TrN@#@n%qx{56$GMg4r(>{# zfGH6Gs1IMy83DrU5v+{}wgrqy}ru5=Xcsln)oyuI`d?R$%olEMd zyMdE}6@TV)BheZ;`4GynJSV>9M#MnIOKryUNzTpaOw%iL7HL_n?CT8pSDc5`8i<2< zcx=z)P^@6uKKO4{)|q>v-qj$1$1Rc`tr4Us=t_0_%!mn!?B4y$kbfE`F-V|x%1Nh| z`PV7e1%73cr11O;(gA&!^1T}NOCj`TmO;H0l;-*T-f9`$6L~D}dd$fq(YM84=!$8^ ze>L+J`s3L;C+Yjgbv^GNc~lW!ePqW<=17so9>06+&54=!l;$Kwe&4xO(7|_|AYJJ3 zWG1zhp6Z!bJJ{KLfdcNaBNviMTdr1lr$P5G=Jl7OBJ+~m9E&G_a93!pY0E1Ao@KSczR9($_9uvOoOFC^lRof&uRvpkjr6B+vVVq~ z)s-+qjVOD;RLptT?_gA}mPj^gHI+aaG@$a?XugheaElEgd}xJrg^i+z59@&Zm^vm( ziW+9n)~Lcua!0J@mR|D?4rL=|a4^}8Mn9YZRo?!glnStGh+#5eYkd^)|<(t$fPzh}BZ4aWW9`zVbVThjhi5*{+|NVqd>jA!*VyU^RH z4rp$9t$gk`nm-f7lfVRx3+#(yk9QxW<4dbgnw-8;c-v;Sc?DH@YY z>|w4;#rLvzD$E5>Jhye6R7LDZecq*j4@#Tn^qy*}VNvo4aGvj#ij^|lqb>tqC$BLD zB>R|LU**t>@$TO89K?zbp@~mh?>ZkjuQ)_^$%*wUzpx+gBEeOz8cy%dnt?49qa~VZ zM-vjTK4Tw;F-WJ-@wZ&Zy!XR=UT`9PrQZczqT{mMgpAGLF#h)CDh=d@afWU>ssymu zmdJx9bRnFRpHr4#S`lN`ry+<~>pW%^EzAM2x%8-E@W9k78!vTOL@G;f@HG&*li2do zu|V#+sOhF}``f@Kxqb8hYq}}>w)4+9j7#8_l-E*r-L;%}(c*g(@LC3ME z6?pW;GQQ&w`)l#xp5IMYNOUO&aHn05D@G~C08dsU^%0pR%rWl7VsjqOC3O5_o)iwv zM#p%)tNh+G?4uAtP9eo8z-18Z&)62Y8jRin6bD*aUeIqNBR-7)Dy${c`)xm!A-8BT%qErmIyV+BKEs2a|g)RZJz| zLXR;e9*$ds~MC6oZuOGU)b?VBi4~jiKs59bw*n#?NaWJqCaef5%Si2 zJhK`twqM6+FNm}cbSGUJAukXGv6mF{Hdad1?J4po9ZT>zbH_v8XRib;BZE0#Jgps+ z!T&b0S79FR^N>e)!mT02D&q6o52Z>bzqkMw;$5@K(bPxXo_5W%N(4Wh&{ju6ynDLz5bu?P z!)z^IEytnBY1iaZtjq(PKyb5H6Q0sa%Ecf&o!(Ib{Q}>p&HL9!FuY$!H~u!9CS{n* z`etc`atZk;19Rr`i70AHtNm6sjvwL^G9g25xJBmw$fLCTP4E9&9~nrakQ^_2oYbqhXahAZRG6d_P$N&)Ka&+zQ*pqU^J?*?fd77aq;7dmH?-u|rsN$!1j~ILaj%iQlgaOowJE zRXY{;k4WrqM;(%LzL&o{k^<@~O2e1QOknL*!xppPPNq-xY=~?#B##z%e|OXoF)GBf zUAwc!D6<@&NK<7Lb--;Dd3;$Sm6>PgMq^-4q97HN6e+9bW9h#y75g--L*Ws zCOf{1@H5I7my(2(qT|_Vc}-K=JeEPbvHAcX7F@uKEO(sU!sFtt3LC8bhI>|axftTa z&YpZb=uMw?(1M#z9BbcbjUHMM-x%1kAEq>#H8MKJ7+QvBoO}CBZDZvHK;T53PpBLk zrP%n%oC!%d5y>usKqc7qBlWGD!tuP(= zqjn`9X0GUPkJXXWzVZ5Zr||a?Arw$GvSLo6ZTTxNeK>{#BeBNzeDsf6I_~z-y&PGK zQ_BZt;;a^=BS0dtS#!7C`s;k&U!7!!=W$J2mD{DIB94ZPD-D?*a8W_r-B!ao?Z5nf zgU|7daFS2{f`MOFSP$#IX|A;MQ`S9n%=4Z=(WnDRRbTGeP_Zgh1^8ol@>f625omZ{ zYxB@&h!g1YyUO+s>nJD0uoun01!aF8Cz`D_V)EGAYw5n(t~KcQN;zPwM6=1t=XO!+ z@HTOS6!t%$nKC?<)ikR1X@f&R81Zw4BpY6V;sq)h@DMX4{ek)TCb^W+FDIiNo%WIm z@CXM*iD#>LNz;gqq_~o<;3Au@Mk-2>cW*q@Xj#zMl+&mkTSjS`ubt)Q#8emIn?4LT z3v(LV?2hi*kYg3n$&aVwgT^v_LLj7=io5r)0~(b z4o&hme0S{eimxtgkNMY1@gwx%Kd2V~JZw`0fp3Bb$MedHin`Pk(vF@?u_=T9fS|1KI zn6|FawdP53^uq81^ms~xv?U?{(IH20RzmZuJ?Q{m26bmV8p?&eL}Wdg~)s=+em4 z!*CdE?qO~I%m^ifDp4uNTd>7fn(sitHowjg1cYTI(b$Il)==V)x z8)4M@Hb@!R@(A0OZ;R8g#e5t!q}ZiXh!M6g{KAsesiZ|qx(b296D!=Va5GI2gRwD*1^-|=1Gh` z#fQd|BjI*h{we{VY8&<#^n1PErY>oea3kLNImPbU+vCP6%)0h!h1Q$`Xu=>$wHwCcz=sHrwy0#2$!y_sIVegxVHo-VlOC3hv zXlCaQ1-`2CO|ofTq%-#w{SJXgF;7`0_YWHy49s`*JMimlemkhJ-tq5E^x#vA73{XosrjYF0f-4f&$nJDF64*kxNm*v1rsH z(v~08M&Fa-^3}sP2a0q^IzE22l#J0V&Zru9^uV=_Yra>CT+I-nABhC*;0$r6gpx+P z&!cY3qsK4Bmn$};gnpjRvz%=7D$BHqh;WU#?>f-N|7_QtG4|a(_04@&G6;6E9yIa9 zh5A?!a#?;xX1(e7wCqO9d^Fue8MXwo=T>`_zkn%&n)yRQ_rJ$k3CZ?u5)Ih)u$f%# z75Sj|a6;)Z)nE|M4rtwo5d5ZalnY2{dgGa$BV!E@LPjha1~=9b<4RV7f(MW@Vex|s z8`4n4%!KOG0N!vHUXr}a`^!A(JxO&4lONL9-U6u{)QSx0t5mhi@_c->6`nuHzeA9! z8@yfr#g%TXS;K1O4^0M-BszFYxEy4+_12_|Ol3&q~Fvx8jsb#e~BVdt^W8wat>rx9`n8<-MZS17`1w^s6^vgFB;9EbKOR`1nAtbAv%28u z&33{Z{_>{lvQ3JI7!#p&@yH`n{0cS%7=oRg4b(fn3kw#(3GId;Fwi1mTZeTjQ4;*f%WGG4nPo}< zt=LC#>d+8M6VrV6e6z5V)Fu^kSGdvFM92^*P(xfk%B( zBg9*UG+!-Q#-ceE6(jxrTKs{V*xH6n>Qaf92<|K))tRHag22w z4$2T}io<$^eUyQp7P>sTQd!b2l@mGlcoSQu9Oq2UFCGje+atCPfJu$C25HPd5i5tt z>lY@}ZbXlqjZ~H}GNt%gi}OwPE0k^-;7uckl+u|WD(kxq&MM@~<>OHhWC37a(F2bJC|V)0Q<4Dp7`q|0jSj z4FQC603c`sfDjb9^uSE?ALkSIh?ChrE007O(lozKZ|wy=bGc)ObRI z&s+a2^PG?YMb5o-nGC}+Y{Z$HW0Jp7oq3u1bSS}d%5rj}#lj#3*KwGY=e>~i3<3#cP%b084> zoWVH*Q5&}K7M{}!dAzFfblJJkVc{F!mr|VB->6^4G90+a_QL)1{Rv8F_)70=Rd@3!W7JT%9LXVl zp(+?;gv*YVr1Sr#VzKBWjBn3!VlnVgI%U|3^+AE)GQ?}6>D-Gd4QXw1myF>;Lfp32?Ip_2&Lq%?Wn z8JBN)MwXerQ76!0p@vL?8ERoq_e#*eg5R`_OGXPNUzCNmXz>nP* zgDr}0VzfURl1X3A#^W9+M62@`Ul46wa}0Wd#F)DLLGJfrXCm;vLO@mjd#didjnrCEAPvyOQ1!1R&s>o9622BT2J zk)*l4$j8+gK^cyia_)5g1h^Dc`DV@_nZ4vQ&plf~KndHsVA*KK|HIjv$5S0{aii?E z+Jkz z_PMkl+9o=NJz}T7`PuygwQJA>b>9#7oYLCM#zUZe+sfJ#o{ZC&x|mMHZ6u%QyHrqfAw+BGq&wHE-|RIf`^VFY=7-l_8HYV=UOd$|lIbxg`ht)9#IK}! z_v*Sbcfo+g$RwKgq3){fZq!&{Y|E*OcAconWozW)#gyF<=OaVV>E%(WN^?XQN(A<-LnR~qB!nViS zf#s2?n+rGF8Mt$*ZxliC6g%rKJ@=cR@3`*!71TA-xnNp`W!WAJli2QGlns()6F+Dl zRV2}7m^AJkms(oFHMQS(>(uPs&%X`*FOLfD4jPwPPf8j5kyyXL+O++3;oSoF_SQsR zI*j(7m6`A?@ar5W=m!wk33ZEmM$%nH!^8frrgLZPyIHc|a{~cT5!6Sdn6+m8WQ*C5 z;|I6DHBA;Jo2)gpXS$n%R;tRk*7`FSf|!-^GPmah$1;3=FaK_8_x859xx5(d(Tp9* zl>6b+zxX3gQO+B8)ceNbF<*C=1)qmreG}8~6_%@hAA@`9L%p!oH8u4GaqdhRu11as zKJB(*d`OGkYnE9d_!1+*0xf2WBI>ITcwUE~>=qJ|y*x0!(cQ{iAii^F)1hU6mrB{# z4CV;H!Y?0+)8(akSr51_q(*G54Z|SI7`0&B1j;_sNB88PhN#v6`T?;+)!Q!7+|8Z4 zu=}1Cl!&1^sZZ{cwx3aj)VyF<1kwtJRw?gfz-2eV(lmoTe!KH}mAC&^j4f&V{eCM=d-!IkPPp)hc5KRr<~7P9cWh9N{_$=!8z z?a#=zc-rE45cQI-Z-|-y++bYhDdvogf##CGr*@}{H!c2Ij=e4C_VsJ)m*Jp}TZXBZ zDZc;svmni#bZXg2kxuT1XHP|w{^e$i<;uOwqOxKNubDHvHEz7n&t$6}hN>!w&cMlu zF^AZj1GlX7x7rYF)9szRpz|7ggRYz`yQkmUvgJXA#BJY0Tb=C8*HswA8rn2w-#h>y}UKizLv3GW%ELr`**ElU*UzC zf9AXKoAo$*rq2+ybvwWue`npj5(Q8HeEl0|wH3zxsJ=bmhoU0)Lz3;i&@TOMmMT45 zm$;yCZFl;k2ea)zJ=#`nvH*m;L>OAlZk{rhhvtq*6O~+;TO}1>BUS~?Z+k^*USZ;S z832kjK6X2{Br3LSc3EJ$cZxW3^9^`W{Kg{#(4mri($I9+Pt9U&)-oJa*3e>WR1$zhsnDHe4(;Vdalsc{=8ml`H!y4FK?0W4bJJB$ea?Ya`wRO}$?GGN;<1-~Pc8EmrfNzjAm1k&@XyRPU1!||dgV+>af34fhsBM5QOC{O>q4f-In z8<*qVk9c5z>J#=M;Adipmd*Q|f@bHBgr0}*lhT9eM#|H_UVmG1=2EBQpqc;XFl%$8 z|Klr{O&qtr?xO_=*@d+LJ1zB5$*;+SE;e9#*TNxRmYU`sykW4*;1@lmJcpW7b4ws2 z3B;*Dg~yjnWcy>oxJtRRXt?VG@A?FKY_WgnEFyn0ehiS>@R~|{3G{<~*o&GuY|E;G z)bD6tCZvti>~`nRgUQI&9$Xz+k*{t;i#hDYeuT9YqmQ}Yv|_%Vjp=Lr^#R==Lgg$mkNrs$h z9*f;*z0z2YzU;nA*PFChvEroJ_Q?|`zc1^LG>znTkHpVTY#fV@lq1)K@<>9XD_aFy zfvzY*T%W=5RIl*399)3D1pN<%J#+i%zXa73znVE6Z+L32fIbAU8njL(Xul&}HK5ny z2>P*2jh+cOMf6bqkMrrT=5q)=wbw1q2e&w)O1X=ue`Cq4HOT^C(LrV~GYKbgORKw> z{=nSziTT$1J|lw1RK5bqE%2;*3~ydt-FxN9Q!s-tE#hNo>@cH{62X;R+QHN$5q%$5 zgmN*c>GFii(lYx-U$8b zDG-v)#BMw~w=7NrrpvvyZ;1RwT4PNf@4B&(w0)12)H%Pl~ z8gS(c@Z#>n>bUL{`N+{?S4(ZN8U$H*Z=-NR47tXC7$|wG)t=aa_ z>`{0hBbWRU{gENxmDo0XD4@5=I`*mDgmctb??&lI>Grg== z+3RaL!H<8TMWtz~Ja1v;)^}^Yw(bvCx!IcNDKR4-E@B`TQ3(N`hW90GaayeiW4^hO zHdSrx<^^tyE&r&Bm(lkjXUSVyzWx2v7*FBRjqaSLYVuhgfU~sgeMZn-J3ejD-(Dy} z)oGxtdB$on++t4WLeWVvV~VM)Be}U+EJsF9SNT??N6>8Smh3%(zK#Xgks8atq?yBv zzRu*9jm>j(QP>Xv*{qW%x{d{$Qa#Hrq|5nVgy1H4{$7!?zr@$a?8fT*i+`|eiz$3` znb-Of@lw}-C%o|1r&S(XVgI;zXJjH;gg5|bq zW2Umkpd+po3w{VmI>N+df5m}^6nI`lj8Fyb^&l>_H|n%_+~HJT(muXP`Wl{HpvCAS zuVO+_jr*{DA$MZ$W4{;E7PTYI@seV2bh2&pkV?F8s-b)rcBzV1*ajnpue|r}2hW~YX_I0h-;{{H^T{XQ%sH1kzDmxXa<3zyTB(b+FsS3H7fWcMm^q(y|+5dA)=H~VPSuGCx7 z`ZV?P)%}QEd&uC%buh^EzVFssOnqYW%r9`;_u#YOx}eDBjvYV0MxWR7(}ur(ChQburNX zPw7gZox1Zs^o4hu7=+HZ_~xgXatPxM;Im=U%&9G;K1am9 z8vZE}1i^*6F8c~P9eb<)C?18rq(~!F$xBuzG4R<~@!-i+Wb~P7?~)NK`=v(0@WlZK zLm4^f;6wmh8j?g098VMYWSa_K-B!{^@6{HL;KRW;H%!3^53fpNQRXzYfu03r*T~G_ zaQNo~UgB5dZ)s7n$|Ypcms;rjf@a+b#OXcwaJg@ zzt42_0z5U5Am^ykRls+nlSv5IvYH)ynBy=>xlhAv-+~F$)g}ix&D?XGuIKPMMiTj! zJae``;d8^`b2&N&ftb%vImNro#|je-l=2KF9|bbJR)Oa_t3CUMgC;4ZW!;R^dhlTN zfNP{!1U?CR1LS)sWwUsSK_+XZk|?4X~mRp`DeAB&*4OggHct$sLargJ$Cuv|$q-T*krPK%vp40Dn6jSrK zMG~GY0fTMsfNFpZ?vgU#jHAHo!WpV`SGsbL$;(8p$@h2OeEeMx zf;sxclze*c67Nf<_-;MS84tL4IYnlv;)Be)H4keuU5*?3IM;s#bSKSl0v^bwTCE_T- za+5L-BvN`id=DoZZdnudR9cL9V;L1v#CDX`^dR6%5~hc4Kx4Qp5TLVQNSNfy_;Cbk z6=XO*3rG3UYGw-})zLwBI&1 z%~xEFrN9S5vVQf6i=QTsAiOKBFnPoTq4oVt?Nd2Ruz^>}iMPb3vU#Q)FK=ZQG68ik zI?h3MBClQmJ;Q&svkH=uy@^%c`!U7P($Xj9Pn+2845UBzRXsS73!3eRaaM=m;qJW+ zxfe*vMqnz=hLMNi403fHkOozVYyH0we=Udn*5Lv5f)5)|j%@+5@7vdN6|Ec zTl4{uec#Tj*}V6HLH4OxtRu}~Ov2$(WoDsE@WqSNR_i1aM4$qVxc{DXcP?Cl)WN|N zHaD9OwovBa1fN$b3O)KkI^W?-LE!^7mJtG4A{`$1Il=RJ@c)TM+}ju32)%i3s zh&=hBq6zJUr?TP;b+1?=kL*bMt)gia2{NM1D2rF7#q#T*&3Wz>zy-|>lRipz2kybQ zooVYz5CCnPqT_k@KGo0wNAf#b*iRE!u(b{t6)RhK?ENz;KDm(}plW^};=}$?jHdwjcLy57 z$^M-#NZ7fA`q1Yz*aV&L==SMX7ProI^P|E*+F6aiOW!nm;d(iM~=2737b(G6owChv?v^z(_ zy1UugP~+1C+H&37;`3cxg;Jij|I7=wj*D#J|Dv+gZcp%%BH0jV>jn6ChkkgpxQvx= z`G^klou}xJJyhhVfv#f%C$ed`9~mi8L%i{Pgx%}y z;E=_8qLCY2ZrvJAY)(#2_Qd82wt=aj#A1{Cr$Kehd*uRy zM?--u9G;Q^?) zDl3YtuJ?e%Di~bgJ`bxpNcpz3le)H|KP`t3iXSsz=We)IK<`%%cvI&^?8h6Zb9>T@ zJB|%MUV;H>iTk1ZOB?H{aWyTNCi2!8&Sqrr-~qficsF%A{p4s zz4#{ZZ&sdO-yp|D8!#uUw>f2I2^n-DpehK*@AaY-dYnG>7PwM!c|)B?6P#A5*+<4? zr_~s?=#r?}TrE01QY@fmmyb=ESx^Zx55ukHdl#W>svma8+!*(#4cVKqRA{Grfr^;g zyL9r`DXSW{fxq>deyU{)c({c7A|FSgnLwIU8*p{x27t$r#*XFrcKaX_oZIk`@snM{ zX7|J3o_-8aWr5G`3D`id7gd!djPTIHZ(LawmmO9(#DCEj#MA@ z_TGB^=~&xkbrXvC@^S(v7-m`uWex-t!qQ)%|8o`P6Kv%^UEbo_d)2Zaa*-nHz;C@ zIZ}_MvegCJ>Z|SImGpuwEns#cy$8c!0zBt8E3ctYPWZ{KHM^58bE`Re8y9#iL8Sei zoNUn>hvCc~s(t^Wt(14}X|$+^|0WXH4YbX_R{q84-Qn|>;{g6X)o$al+p$IU3zYR2 z-#FN|owkJgPjX`P!K!Osz|!wvDR%%%IpSxLN`T!24Vk6xGMAuZXLICUHK@$N)owWy zKL`t`)v9)(=_-wY;`jI0UiQhMuF=vB6JoBJwH3_MtmMh}$NueA)26QC1-%?zDYFCW zStFocLZziPm;*n({Gqy;R~9a;(p;!5Kp%;gohwPuhyK0tGsCK&2}V!TtSd`bR;d&l zj{5&f=}R9kUAph>vwC|7{#PqF{&d;4+$djKkP^gYCn@a! zq&Ku#+lmhojOagRhBsiX`(Z%RE?8PFbQCv-n8M4Pa-J3&ExCtC(BE9HS4^Xor&f%n zxN3}&X9VFJio8}o;`~E2Ov)Yv*d_ zURiX1m%V|y04Kx`Em3%Ms&HFRzUON8nO*0=e}c_773}9B9W+?C)WJ=C4|9z!VXBknC4*yMW~6H{!ac= zPfAxu^-Rj_Sj#R4&l6=kJRpn|U}n9ck8F-01JiVlsm(-wz^N5fxihI6LvW20<>2+u z{!Nx@IXzr?1bxz-MILysAz1e5n};r@j{~!vrI(_zmsNVvr=UYe+m3|N{$$u|I^^&J2D_hgT0@O_(xAL zFyko@Uze_~G8h?2DOdOoPrX%)~4oYFyfyg&nKyw5~|7v6{y=HKW z4l3i5BuiX+E`;Dxc2*JOeSJo|K8F28qiyfMbYR&1$IH%%*<72s9mJmk-v&>dJo~>8 zPOv;AH242SICYwnlhV`o&i0pOI-1dEJPdvoXDlBbPTFe*RVX&jlyBN>B~+1poHl z84j`=2cU!O(;gkMsU&d5T!Z5EJ@|I!57dE!LHuMn8;!)ATEyTY5V8`1Z?mwi`YrYq zJ-uGeD0Db{@j;b03fB-R$tvDU_6{INpq3kU&D})dhzp~cKIw)9<8ZrM{tuUprT&NCaIKbb6i%?+$-pJtA3a{pW^Z|@SfsgZd zbU2a#5Y(NnLa-H1m%6c~&Pbaz+!vIBvXj{*z>kk2b1npTdWCI}*}HxMqp}BX8kN7S z7nOQ=1Mj&rx%xHqCJ~N0e()tZGi3x9()9Z$c1n~w2E(4T|Gzrk5>7CgmV62THA3g6 z3g7PIC9RH-$rYOfa(vbdlZzQyffjEreN`tC{>c?+6T;&+~tLDgEaL^lh-{j3#RcjiWlvSl`+ZfjtDqr;;}}Uk zl1y4S2JcXFS{=aq8Ro6q@nU;IcO<8)7?$)TBj(dH_&LQdRJ04V+ph1+RqDqAqss%m zB99c&9pL_mluZi9nsf<&Cr*=_nW-AIxE>QQP12|{j7SGP&3(n*{=|U<>mBP1!7AKG z2v)lg*grdszB&bLt>c*{t>6|ZIQraz*#R)bd^$lc%@CkUo;Y#ho%)=zl_Mg?*ITksU) zFDmJC#RMzuJaS#5`Yx7(A37I;)hN~S1*jIbSA`rXz1qJxP**Y2;HH6pUfwe|=WNm( zxRrGN=W+5qhf7P!*u!zZqe=Pc_P+>Y^Mf*mstWu#T==>`v+3t;9wb|jEvEb+LX34B zJGO%HsY0ipXzj<0w0D#?Y;t~rsgy^ z?z#fzYys;1O~cNPhVNf|KHO#C?QCKtX)Al;gpFNZcmRMhB&icZM(f zx~fA(Ho@^?RtZ~`v#u-Xe|DIh3f>VNW$GJza)|8{(uqjF%*xSSo; zRXOw;T?M@zK9VjFAy?fE3{u|2^*?dk`)!c7rYR?vU`7{A=NE)IO%IU zQhh=qGF;UHU3E(RJYa^W!S`7jYH;*}clSK;vL%atG;S;W`NWG#VTXl1hv$0#Fdh7N zVWDPOf6sJE8i0E}QeK=lEn2coHAqWmkemjTK@its7&xz_J*!5*lowV2E$C&I5=^WCfY#O8(G+OCTr3JK;hu_7M1f zqS=<{4#x5aYNwvc=%Iwcqs=j44Pf#-4$2!5D7+n2@ucnbm2-r@qo>yTiCO^#DSR`T%S$h>x#i`1`GXl+rM2b3s0YWO70bf z`CmL;v>=HMyc3wd*-N4c+(w-l`JbO2g$fb@ znW~!;V0VZcjTSTZLdR8msvdLF>bpW=^~{km_v>++O5%ZKFBNAY(=xu_Sg~kZ(Pz)Z zj*20-C`we(Lu4GI50)WCq$~tJIWjmHFAN!!ew0zo94JIfNZ`ELBwH+a;VA^?Pd6dZ zW5rngm~Nyb^;42WgaW@GPQdi#Cj%spc}<9IC_KS{M!YGD`CrV2!oQ#jZKBPjpx)Qq zq!P}c_4S2o%aKdM=HxHhOWo_LF);>?P>E{Ky`t85zVcoXZTA(fYsT*nNh0$p;kw&{ z)G8Qjk33{Zigs<=a&~gjP{y@b9W0CiD<*_8(mQbOZ4(47U^^Y)A3q}#y4cjcOGJ|sf zk1Izv;^<9Yt9y1A_eSE%(8OQYjKt>*KYJ$PWZ-gqjz9hukHOkqGzuZj-9vexLMZdr zf9>Uv31zP?2XxaM|J}$hvgA{2ZP9j6oxHmz`QwBCM{WGk4IwokfwndnOT?K4qelF7 z=7(sG(SfZya7dGsBz^f>hsR=6TW-EeWPnB1_3VkGmAtDeguj7zgFcBW_NCVh>brL2 z1DXV4$-@Er;;iPtO!|T1fZwJ1mE5 zT)^Kq(w*S#Qlz8|`P-)vOZ`niXM0X)CEd(!nf*gcW0y@0hS#7u#j0{r_fj@V#aE)y zX&r?%-IKiKA04$XPjxIoRv}5f8RL7j3p-J^R6=c)!tx zhGZb>E1Z{?eVH8!{ddKreEs6eoU^}aRWuICvBJz#SCY)Pyhf>0rCsJZ!+zOTO@gQE z8E)%p{dxdrOWEZY(kDvH5kbkhLhXtPqd>J%e=OOKKLE!`t@H?1D!FBY;YBfrWE`*G>LiM* z@3Ilxshr?;dv6lmHl|VF zD?kmj=PtiUMDJGoQvOI^eL96ah&mh7TlCDN(Sb$!50j8OfnG7)d(dGyw%&|b;q$b^ zYM_Ar0u3%f+Wu}LV(2-ijQLa3GGClMUU6@)x7R~!rR*!*B%zbE!(#_nQZI@Ov21@OYU+#MYGrigv24CoDzK@#<)-jxyUYE$B;d#2#af$U(Ve})9?9W<@;6xLC^ z_;MnvzW7$PF;jks7H_EjMBIL=^mlBeS9aaU`A@Ldv^~$QlT#?4&zFSGbp)o*L$(`m_|+|=d<(m| z!`S{*e4JR^|I1^kLua*rkFxI42s(SPw0sQ_bOJJE@1s2~af0MBecjukCi3;OQ!)jf z0o<_tBsc8W86fiQlW`f`g9bvGm1wl zze1d;VdZu%f8&s`&y{$|-f_yK3fp}_mWN^@%REZwiZ>4;aSJ{JeAd;5;JW5`lJuT; zN;;&8F}aZQkHsNcpz&zxDJXQ2(Z()dtB!=jzU*X_@WV7meV!fd8nB#>-p|VQv1foAldPd z(zT{RrH&tyy;k_QoRQrO#LK3~mzNT?ml{5XS2Y6sh@+GXVXrPPKT4$9A5?06E-#Um za0)r^t6Ek|#KX|`|6}Kh!`G|a4ZpADF*Ogs2b^`q6Jm_C7cknGnGGx^aWO2KWdm3Nq>|H z=vazQw0f5MdIH12RI{@m$=(z6@w(i5WF71u)_2zrHPfEZ6#0R8AxW34awvNBM1N`? zOiEcP0j^YG($>wx<-DPi%-pn!Cr8=mYTi~ zXHpTJ#&CwyIkMUJ*~jXWuWhOLpO8_JD!y`qbq7fe>F9P^`rX-~j|K~#7-1Oxh2g+d zTJi$8JZJ!O_vq;4@M~KO!153d-`)`HAxab&R<^ak^ty5Se&wXypz~Yt;7s{d2PDf*AeTzM_++703FMZ7(6R z4BK5m=+6@=Z`*H{RtFl@i_7w%W7x+9V@^rQh>v9nTuiTZ3G`;KDyZitO{{GW-)jmk zx7&~wX^dEXdOb?4`sB5qT+LZ0;8=dN4@VqT06B}`^6$w&r{)?^M38|Nc@}m$d64}= zB>Z}Q{xi*)RX4rq<4EQN92P*%9B2$;%b4{pJUJOKosfDnApd~^(`#cm-W_Hc;qOJ= zfl3Kh(Xz~4u_bacp_1E4JP@r--eN@FS%_La{xFK8AVd=|D*bCC&q+iQ3J{h3dma&+ z_tj&2^^}Bo^EaoW8JUDG0_5=xIwh^Gb2L72k9T=*YdKp$*VWlrP>zK>pE3=`jGo#~ z7U+J5d7Qx$-E+;vif7u>4Kvoeu?Q!jEJxmNAb$xo>s}mpmLzR8Pgf(3^G0ORndqNM z4cIw@Jx{_i>&kMtKylp#U2!5N=0R~w?!XBczd9S|+&(mD*2qH|9iMWQ^m#Gn}YMQG`#Y0m2^VoLE7sROPc4rdH z*VIInE^Za%yto=pfiFP3|AhYM^=ymlbw8ZC-g05sS1O&cnlJ#WsKd@sD}=X84wQ(W zapgGnG?daI+>G=|S*5)3+z?7+LyrSZ(=}3-Vj_r3T{gMIn51DTQD13#m3~-rh1sv> zxj5wjvhuH0MGOs=gF5z2s`i%t=C;@Dr{9u7SqEoe>I$;tuHqKG*ym=BR6@`5u6*T$q3l=S z;-{>7Rvn=_-6Z~|^6?j8$`F{4YmZiA1jGO8!@w5GUD!}o3pIneArY@7wqommPvC+< zE4hOYy%yyJlr_kC2`=+`;qf*1VX=tc#>Q!x0{$ zmRe`mGelh8Qe}?+5zWf8NHor)UP015#@|1FUZN@tjXO^fz0agWbPsn3FT1f?UvwP! zn??!hMO=Q$1St>;wW1r_ugpGlJDJa#T5ckJ=SKDT;CwC3R}uJIpS3))$H12}jf4X* z2$}Hwy%#v>kY|{c*Y-S*-Mdiv`z>uMZ6;+wNq&%R;pbRqM--?P({oJou?YrDVs~8_ zg;c??QL%o}p*|us+6Y{(?i?!68bdzCHaMW+ls zPL+;xY`5z*2|qL#A>DmI7a-xW11B+D)o&5ezUJr@h2w&o>D${aq{38Q*rR&MZtkd+ zj!Js;16&8{MWV3r`4P-F2@t5i(m$3lv%jCsSAWUy{hnYv?De$bU#^h>aGuw|#yPmd znbD~VHq3|=Z+_0q4hac|t0Hofd^#mWmWT;9|7Bs~V?K8eS3RXNL7bHyenb8XLA#w)1ORamQ$6r$fzoge~p1OjnOp3o1NL||$1Wto14^Uw4_ zF3Z9avD0SW&%)rVAn-y6`oIh66NEg{YJsWPN?&IqP(z{TWhJaiU=PAMx#4V%AduHl ztA{yTOD9=k`JgZh(Xl-Eja{6)0N%UoB$;vUs9nvJhSBnX$GFmc0p|i?ZVU~wDO@kY zLQkIk`$jtDnMq(=s8~G-jC+wd2^+X8^WvdHXHI%0!7=Yeh%r!M!K0IB{dKy7kQDxJ z2`1;P3eFEw*Qd(-i;Z$({p;te<^D#Bmfk3JvYI);iuK-xLi2* zN<-9|I zUqXk9wRn>r>19@2~56H_rmiXW0;K}x(w5#7y4Gq0*XbC^h`P&{8G6cL9{IzpiW~*7 zdM^G!IejWVe#DghD;cwc#fVG75OQ|!l;mXUC2h$1Zb9%u^x0Zz za>w7WPK`d#>g_lrQ?RS<*+YXhC<#1x?j`DGyzMK_;y87T>AaaQw#by0anbr6}Na{^eNx_k$qZMA% zZ0GwiEQhUhL(kBBjkBir7k78wOpm)zFGh+o;41PQL%tuWJKNDFD+Yu>%b8=|@?r!R zW_A#$x=%PX7_#rva=M&8pF^I(6|B4Lnm z@XX2DJIAQ8(pl5&h&eyo!twAdO%}&Lv!sCzO+POD*qQ}05(b+r6y*&qx$FGad7HVn ze@T+4)GL}T5=VZKxV#duCk@2UQ{BDKN1R?UC~F;?lUfp(mO7TT3p^4YPDFZP zrUrAGzl+HQHKnIJAq4#Lf|rFH%lA;rvE?4=9D2oq^}Yt`+B7+-=;N%5v~LjQg;`Mr zv(3dSI#H@6`i~_K zr}TOv@Q$a@sB#+plJq~#^!k$3qHK96@q|S6m>r}};sGiyyog)`a!Nbk$$6iu|8hX0 z(7RpdHa@>u0i#0;ZX)ouCkZW8%32j-knTqE$;p^r+04zv00AoLe_k8zGHP{?-LR{G z5&KtONdU@wNciQE9KmPcOhNy@Sq%~p58d-gt_3K(_ z+9RlgqJPrH4g$$==3bY~-hYw_q;JV}rLaEnT;cfvbP7-z9EW=umsti}>gQvc$0J2E zIE+0b`>|%*=%Jz69dfEd-iMM$uapUvsP{@+g~lhJD66+ z0A&Q9NJZRNy7kHW?PAJ5>kZ|kHF31iF6s)>`VQ(2uXOzPPBVI^9AAm2Uzi+@x8h8s zz-NJKinv?}(--ua7N_5V7d+R6TD`gSpKDFO_2QT}(2I971*r37;m{*wv@HS&{n z5mS#agnuLHBr`L<)RdOo8@91d=_v~a^mCKL1}2>G6nIATX0a41(VUv$>Lr509_l$+ zt=V8@OmFic-_Fuw5mJ4cT@Hl}^uTt*d!sk61dss49cKh%y%<7UbSH%oy@7BNW3e-K zdZ<}tOy85)(DVm|ZzI#)J<1fHe$+=jqv>6EcA6;HPs726RA7gEEY*JBXvT6Kdot9G zSk*2%Ep^Sh9h4!C7M1wjh4Lcv&i$iAEkjPv-sNu!%YR>T9kpyN*vy)F>0094z0{jq z`EeFZuMJ!D*{+uqyE9Hl1bgTx660+oW^eZP-z`bFr|+!ZCEWhmRs)S;5!4B~Hz7-m z5^Y4{RMJAQjQK(O>RkoQfzeFt5ZgQ09+%+Jp*@O`>wNbvl-;i~M{sn~!cV%J50yi` zZujk@h)VA zQp~QMlZ8Fa1GN9|@mJ9t`<%_7oAovKdBl0Mrc^vDO-FS5&vq$OoAfyd3CJ<3B2=t9 z$EVMJpsEIssi&-tSXkd?jgiYS)}WkQaeQ0ut7{9WiJuLI!F;WADYb zgfqO*I24omd4{FW-e#uvj5a1-M5no*DOAjBH@hs}1u;{r5(TTz@9AU42G(tKjRCSH zs#(;-GOnt&?9C15;oPAng8OAlStNK@W0LAWIt(OQj^D^EwnV*{r%st5NooD|1&vEB zq#WT)ig^zB25X+X28UzSC=CVy9H6HfY~Oqq7Skq?UVC8 zt2$B5#|bFT{n|MVh)>X)8d6jS5l3)VS3#|Ve|8d&bL7oC$)c1;O+(CqPfe=EZI%nT zEGa+vE(6^yjGr(4a}K}L6t4yL-t4|U6@LGhI;eKb)sJ6>F)#NyU6~#ByQC}Y<>67r z;Zp*vSV10V2LXr$azJVyZrD{d3XQVt`8qQ}+}%f&-jNid0`@(YYW&=hri(znxOjZA zYH$yBb#`Kp3j!tVoI*BD@k*d)5+Uwac6Q{KOx!ZKiJhs~!$42FJm4VkH^y}HPxxMr z!Vj*_^PppYU3NlJVR^bGNT7^^P9Vd-N2*9_d5R$3`_|z^n_=4SRd|1LlnS)u(Wcrh zam3v&uMMB~{gzfp*|J%fyjYXVzYD~~U^25ct)?WnaGqHxL!6-&vH))kcqF~>RI!z2 zKOXkw4oAH>=;l#Yev3mKkESC2GEoT9E@U$`GzbCGqpZ0YM{PsEg_l&Aw zgg!POHUftKaaEoOdc_IKUjPl>UWoWRH+%JpS4_ zhmH?LH*jkrq))4};Da(gpVP9sj$i!(*2~@4!uLCVNk<_w`H<%##>m6*Q%^TEb}A}I1ViXNaJx~%n{!5z}b?Tz~_d>=3X@esq3_k0R8r5R*wNa zNva`19th7d!F_4D6GRA$=>!3(chg!Mykxax@Au_e&DW^X+VM!mS7G*lrunCP+Y4t- zP@>M!(s_8ianw4e%@{&XxC2CP*z%{l;3xJZz5on>W>}vR^o9EI zvqT$s;k|wOVVzJ|Vx-U$B_>s)7Duf&i2`q8#Fl@XFYMBit|sU|=+VkUu3BY_n(RaW z0h_gdN*|+P2Fr4wl~ec!s!E~u(iV?67L~*_!mNpXg|SWvc{yPvlf8@YJWhgOFu|z- zD^mb5Sw*L$=~8CK#;3fHW(iO7y6u?V3e^Rt<7bY@fS>tuzsbz22YqQjOB=iYd9!*9 zi-A&MBGbG6H3kyI(^GwlV7M~gGe=Z-*%D`%F)2k%y&3qtto1;l1GEo@O(fOIV)whz z6Kg5%$Tq(qYF138=43E&zt;ge_dyaLpFvgIsvs(H-_4Ijaa{l2cAXX8*57f^I0p3S zDWkrm<7_5DB>-oxLs!FnJ_YR;%YL$Kb8RiTs{|yWX1S-r@#F{Y5?U~O^B}^?7Qh_s_}J)>WEp>dun zcN+OdDi?-mfjpiI%_yGi-M-_DMsJ7%9@%89vrWBX)^aL;WYwwgmnb^*GPB2r;9(mJSzpB9z1RFi=72X%(bJQfNk_EJ4~CRbs-v5q^nv^Hrj~Eg{AJ^3uV7s!;;rg=;I9R4HdT1 zd2}?6n12c>gZhTz;FDW3E)}qAZHdDz%dEGv^43vOKmTz7R?VC%+BP&(WlGFWzw>L$ zTc4`)^053dI{_aMzi>q6I4S^P4w4xxEN=82PfA0w3wLR*HjA-24_=NE)A~~7ps#Vd zZ~J@flNT5$3PMmfo_is5tqQ)W@&~E$b(|a78nf%mWV>|yS00HE*1>|*5B9bAG=HYg`n$dH z;rO|`DR2&*n6$vZ=co30j!As5bnA-Rztxz$pEHaC1`l{Qj%Cn>SA3BU8d(j$X}I0A zK6~o!snugZ!$Ahxbk*WGy(5E4zg$VZRt+;5w{P^nGIPiM*8zYYKevmd`Ys5G%IxKZ zE}G~E=Fl9dRWmP5n(aE0N&%@zKws{ffwoIB-2RWwA(~y3wl^!6{VLPt<7OmhhEA@H zB7U)_DrYK2V!Ag5@`@`sD@h90`ufi{3#A8ID~ryoTHUQ9*E=Du2M-mV-Q?iV->Szw zL(Mc&nmYX88!a|eapKwCOb@vXLSsMU=!MFjAsQ|x*My4`O4dc(4P0(1C{2ro#KDQ)(4|Y z>%IRj$XWP2EUhaAIvEq&`fpbW^R8CQ&9w)_1;y=7e(R-vBIm4O#;1ATP}a;`9cbny zOu!z2s)GHut&2W8xlnMPo_}=+stBMo;iZSpX|^_wZYa-#Aw}0{d#4EN)0KO#$(G7P z0}$j0vR?eDIr)ucm)4*8MyD~6a8gNIE9PZO57co>ar;FU$x+Mg<+BIK^E%sYQQ56g zwh?N!ZQ?B|<>==*oMp0Xwj0aoY`fF0%N|oep$A#>TeXf+JT+LKnUo#iD;}8*G|W3a zS(-cu+02P)A%vD%K9ln4a;?>pYirFQV>Ba=PBB!DROHCIq0(u4@e2d@lcs=WRjvy5 zI9^q6^p?6fEw|$Ai^WED@uCgE(RkZrPo$Enk>NHTI{U8;6_(Wu+C{jH)Za)mm!Qdu zgE~hM9mPbKC8DN#TW+mdTwn6R@7|JNU>R1S_R;b1csZJQ_CM#6!2NH`>i|^}i;6AE@YWFHAo3{W{xB5uK&g;4xYo59H zAk+{~E7(!8iy={wl(>~w-m43o&=w~H^D?+cZ!C8pph#x*@$I?`1RVbR8SUA%)29zr zP5obFy#-X%(H1uf2vQ;-Ac`~s(n`wEA|f>)NVhaYDBT?@(j7xccXuix-7z#GF*HcZ zQ16WQ-fw;Hy|r{LUE=@5?!A9|A5xspisGgm5GA_}pkr!So;C#dPp0XZAVB9DK-R9C z%A+|17_g=ogx!BYOJ$cLk2D=1=m}Z=I$dlx9*+R>0eklT$H*!X&~;yI9tq~ZR&>wO z%z=b<*#q>3`wW;g1jPY9&hM42 zcO$)^qTpW#T!e|b2JnFgoC)(6D;sjUYKiGtxVc%n)grc(L6mn~qr(iC=y(Fj#-2)geV z*yB8O<{lKECgNLMJkbs4%`t0p%cy8)TFpfJ@bE8yzJ+*cmyM2{xe~3S^GLsi@5UGV z3G(&-`e956fB31*-=7(OPm4%WUdN~X zix!ZvUm6JBmk)MHuYr!kO4 zg97}SvA1NWyLehp3;$T)p*MqKeLs6nM+AnPRu-*jyx%gsI!?my@5O0z-ke!Zm4Gxu zRKO$Q%ebQ6hk@p9*kB-5nDuMyLoyQXp7J~}A;cbdtmwwwL@<^^!?CY8{S}`Xpex%g zk;^Tt|1)__J?*#kzbY=^o&7>W>C6_%CX;2yW|f=P;XNdP2t zIqCJH-hXfkmy`l%D>ih?8od;VU$Mi!Dp;X)nSREOr)K#&IWBmhem~*LjVKCm5%w5a zfu{4}-=<^hY*}nP8}5U$YJ-Q|sI~J+%Am9X5+JrV+!cS{xB$`oz6E8#28(f_7gJf9 z4lB<8q5k&Yj1)B5=zn!>B>jj>qyFu!I59oroFXF>1KLBb0#qyrq|Nl38L)N|UKb0FvCa|UOrx2j%)c~kOc-L(~ z|GQMZxWngCVNmMDJ|unM6{o*x%*p{-<0}sjk~9aC9?UdUIvGZ!{fQWYl1T#8*F3aD z^;x3)LBxJrhCk?k!-oa1(8EhU7DLdp>%a`^I^=)w<~o!O$N7n&u@bI!Xun{2Bm300 z)td-}CPvKu-=bn=CzCmV5A{Ib=-`bDb10at@+w0Fgisw5U|LElr1CZpP8p;3>cYjs z$=pB@asl52XrC%DgP;%0m3#zdBi&f%b%RgzD1n?${bwp-kmL?ZJ1oqN225J&R}m|V za2B~$`wvN$T6XTbjHL!tT^aV4`WldRb(|UnAnV9Xbi2;~T99M$MK^=~UOx-Urt_%p zy}LU3&&xBy@+Vt53*MS6_qM)dCO}Ia=mB#;_kaObHNeUZg_mr^A-n{supZ)R4YCC? zQegpP9ANo-`|*Q96_OEYfN7yL(qdZbm|)*Mv$3; zwYBlNz{^cA+Z_f4DU=$A7|E?|ZR&aI|DVh4#pnfwB-=P8XaL_=mR!5Ry;WqIqVF;yfUZ!30nk0ly6rP%{S< z;TMEc-uoL10QocjcSqKL`U3&rUzPOv)n#M|@MDsq)Yx%G!SfnbFBl}mos+JTwk zQl7NYYY!-iy)@O{A)>%>eZx zx^4XykY*zloOg*muO0gkz_?5or0RW)rvEg-j;IUXps&Tmzu~Kz-VB>vZV=!vNV-!B zt0X-^L!|{k7exj5sjvN;dGkcGgK4|3v^81?Jr(R}L1#F61VTz;EB>YL=^pmtPwViN zzvCr#V>zEb_mDI8sw98KjPHq(kE!HkX87^!?N52~_!m?p_!LCcj^ymUahB=0_3ll3 zNbR|1&okkRv}&X)s&p9Xv$5lV14TKhSKpx z@H>H=OE!VuBFWjLk{FoSzwb?>W+3i2N6zbM-<0xyIG0fpvuvz(6+N@S_G0b8^=#u{ zBYe@}h=t89rpe3l#jhNDXJ0C1)QH|s@*Wm8Jf>r1VigB;*V^|`kf1*pLkW`bEex8H z82+o+<|Q<&v!jKU%kNb4^}<%Uf6n01`(a=`!`Cv-hnS_v>u1;-%Z{++$p!?HNYZ;_ z*=Q$<>y~}Nz^Wu9Y5216nTb@A{simd)Uc}UqiO#2p&qHf5k83%fkvx-y-onMj02zS z1y~b{4X}NbnZ+nx!z-h%h~O%(ZbrV0`;(px<_3z`nAjwE0Uh%~tVw1lZ$gm(sFYiZ zHWOAEMP}eWmoxH7L*QMkRqFo_8)DOq>dbF!h0>#8;HKMLgUp2I0Yf4yGw^Erc`A1=;@!LM&$pP6BMJ?Zd_CJyV1 zAvuV`$g(G<2rKje)Q@$wUh7Z3=X7YMtxb-FNB>J0ESb*nknB|lCDy4O)J7W_7E^_X zO>$7-w^Uz|ogEuW2hEG{b47o6VfL9L_T^*n;f$hQ9g>4|zbmoVjt}o(le~@wu^PFQ zL-G%>Tjuqj+W*K}NC8N%qg)mO=!dUpD54>LZO5uR)%CLry>&?)oE_b1B0C6k>6ydt z-`Map6=XUdwn4`%3%kP0npA>YfKcyWfLnqVI4Wb4RA5gF##&(d^8T`9p!%(u=7PF(P6dSGfI787$4h3|H}$|+pWn# z@ZcyazrTC6l^(8$fuv;=UA;_vIZ2GWr9nTX8UM8A3|e{g=rP!R?VqVz#?DGj5P~ zgeEgH{rZCO4a;J_kpm0wC4ttQ%%CX{7BPapn?nqdqA!OZHjetBw zUl9EUkp6&c>2Io(IWZ#e#LjFb3cDP#Ed)g!b>O}TmpK( zcz>aOM*jQqeE}o~IT&*??o~F5RG5bN6}EZheZBAI*}>11#^d+2;6ZS5MmWjPJkba) zE0OZ!Cc`f#Od?^c=}+LM*x41N&^JZmbf_<wE^jJ@KLCaY z8gMUR^TS#Z-YU4LAJWSO;9Iiijl$q~ejPr5i9`v&LGdw*Q)3s4Wlm_58<@|~YK-Ky ziqT#w*+6n&Dpc-(U=I==0o^OYt(NKt#qugZc7CmTj4k>EtV7F5rS_^r77M%G`>kTa zZ}cbk%A?CIUXS!5*33SDW&AAABAxeEksL5Zg)JP-6hN;TVfk!ty&PdPB7WL0Ltl-B zO~|`FxBxD;2?O(1IuNq=svXy~am!Rs|b z+!;tj0fh+-2DP7I+1<-*&P^@)riaA<79ufo*=NLuG}a3a+yR$?7Xui8UOlyz5irZ# zl<>-sKCm9>Uc}w)*~u-Yd^$~*WCdX#ZYkDog$}^q*%)xbOD*-OAU`3T4Nl}>(=!1 zW&ajhCNuEVb`D!}ASjDDwH)Y6omyW+NQW!nTq4-*w)@TDm+gESVM@)-!%T z&L*eVd#;8L5jMPK?&mj*y^;RAI-rUgTa>fH{${@V<;$hi2WTR&0Ab{dot-DdOhu~? zTv@QPaNqFYKLdM8NaFPS4LGqAP&WDy=SD%W5>Q_-^f@-2*MSE6#fEut%6TI5cuuX_ zq6++-6%*2bUYJsXgSn4Sa(QYnm{b`F?&iE8$v}@lij_wrTo`R_uE~Jxx2IGwV$VMY zrS;8?We;8_2Rn&{)FCcDo^%b?DE z0*POP!#pk56)6BaN_G%H;68c}tu%`xEu=O!R%gbZeL!D=EvZaT@4rjHR`cW94$yYh z2=~)PbA$qOl58fLBbqLUc+$aqegUJ~;9wY7jM#8D0w8s8tWHYjXNe$P4mB$aM>|i- z{}ezMt`Iz?Ej8!74;wyqXJC!=pdEv5pScZ>_7o)IZvTg zeuFaj_J#HLFC+(HexPiw(~va5hWO6Y|BeRD51D8|{Vyx*_12D$_Qsl-jDa0>&^n9_ zvCT>agDvAdXYRP40_}A)#3oSzevsQ;lJ~&Qvr$D6~io?*I5@sIuCY6-LaU=dM zEZG6t?BIGDQJG4inQ9~K={Fv}7rUCmtG}edF93@fxVb))NR~j#xm&Dwod2#?Uj514 znllwlWO}{Bx;AIu9^*Jv%7skRoO@v7@Fd|nXOyc@8;7CKX>QH-7}VN8OU)puRkH2kRHV=YJksD{pl)(-nrh$w zQ%I<{vYUo!P)F#fR)I0>Vhw4yItXR&=y`@@GDL9%8msPHPQ zD;DrzeWZj2yL#ahA?J3vqRE1UI!_Vj+x{%w)SM?x_PQb^y=_gI{$g8~5u+1K#3KeFoi#Jnpvv56c1mPMbh3 z;;k}Cj(qCwd;@e|#G#1K5smIr>{O!RllIF?wRFeiqcJKFU>#9sz}Q(;@~!dD55tt# zBxLU=z~)?IIdlzJEfPw>2Qt;z>G;Yh^1;nK|Mr|36Z_qLU}*R#a9J1*#uqbk6z0kf zheBKrTjbk~uX6N}+p!H{7f@T36UOtcu<}(2*=yO^8ha+qnif6zL@uQfwjovTKQokL z6*^3g3Aev-x1$^y%T{U~OqZEfT4H>OZ`Sjb&ARH`UySs&3z(Y$-OJE1#VJYxCdr)tW`2>Xs#6XY@+m>wcI$!SHFd zvP6u;&0qZp;ZrwZRMN@-Vsm3P{oUATuUPHtZbjgiNXNr~QX~NOPq-&Y@<#Wf|2y)9 z{%4(~5qT;BtH;3v6ftJyrv269=@bhIilG@z$Cs%7a@E$i814kQaKQ%ylcnq=)k3JLYG$V{(4R|4#lxQve=2^gVigH}|_4$SU`BAyV z+CgXJ$7U~yE7x*z@uJrgN2hyNQlH6%S@K}lr*q{F8(E>fQn)na{DaJSvtoW;#xa#SPUm}R_uUA^ z+2BmRO4C2(yj8qV!G9-@rt;mxo6Dm}ubW+j>c#=n;bnT53ys+DwapJIC)6&&@WoOr zE**9M@%Qf4Gn+1F2Ga9waKO~^_PCV@38}jxB`PiX-Oeo#E6F$eYim*?z5SCWdw31N zDO0mKIY;jMu9ml7$GiybzS-wms46{pGb%EC>ALT`R=_CR#s5j-=2Gph=r2j(zfpg$ zX=um0VxPa(Z@WV2Rue54hpt^~+b^|wtF8>BunbI;DTr()I?Mi8m?}5m5vtoTY7?{` zYaOeR8Gmd}b~Ulzer52P+OdY(N#!`hKO3H_UvO9WII2JZK{9@}bVbja?iE0NZw+bJ zx;QLxWiwT};@?dd6%x+V=c-?M8-D$FCGtu_lP<-9^#aHDX4h2>VG<$o(y5iwD#9};6=DM`=n=p$s16V)c0yW+Gxskapp&e@@E21 z^z#BHnkxLAHl{>0>5-p$&w5*C_*T=lYIBcDz0H1_kgcuZ3@2B-inkV6{mr5&cL5bs zdSdx@il)k8h9j>(I^_A5fDNa#tkM95$5=&o&c&;~FrI+~v1XSflT?GrS7j|IoteMu zRNBw?A6hwS^KFj0el%@Ajrr2-nVD>HlH$Bt)=c-&%}%iWDq()IK9w4>mv$SgEvLoe(MMC7^w-ZCkujcubd*H1i$-aw+z2y!)4AFFt z3v5$4wvrGTX=hCn1^PnE?ybO}7HSvruH`C3`taKJ-8!88jdR#5OP2yGI_g+nSv!u& z?L1RiNeGAaFWp>nt+0r(?b|Ot)^n_)2y{)9n@ViT8}Xtk(0I~>%N;v z(SN`H^h_vqWmvepPETw9-R8({T@aR=a5P<$eNIcufpDSIX`{oLcjS7}x12vrrhmRR zs%E>0p-tP72jilLfsYiccLW`^h~lw(Z3xPFV^(3dX8WE%Ngh(s+cK?0w_5bw;-HQJ z>#oRI3_*O+s!pVMry9$b81;Tb0q3!U2=V&Cxp||u^5gTVBUy>(T)*ds#b=n#BB>a+ zHF${a6$*w+e0+6lYoH%bX5?4>4J(~0=Us=5W~$uP{vO}U?R#Ae6L}q4Rxg8n&{PLw znUyKoCDLC@dDOJp1Z+fE6p#4LOUR;Ev%CH1GnXIm!=azf4(>;=9H*IT)2Z0VJ`Nr+ z`!=45*I&Aa{)YvSN6fj7hyg<}Ev^DJ4c~+Sr>e=cksAx|jv={6r0ttl+y}zg$H=W2 zzV)8(jf7_7sFC(&`FJ*F0iF`J-R1@Qd~ylq-(NOk^ZPGncwSF#sjueg+Q%eEiZLtP z=X|D{2-n}Wv6{}ws@ZmGwQR6boO+;LYg229A=^pg{AeorTlQ^PA~7r09E)@}mX_Vr z+vZe)c8|L=-lu(`g}nNluJxAI-TX7BuWj(W@7a|(6gefTd3({@;>YEp=Y?4BmdHSm z>rkn4*}PzZtEvFvpf|{qHttNQ-Tvat1K;X{Euj@CQ}x0q}XOJ|5w~2 zZRloQTqqSJ%Ur4^kx*pbaH7DmoA+&f_z`aYO<`pl|r zGkm}O(@rk|uNC$fx6#d09+)p~V9iq8BXro}jz{wJ*C?=+`8Xw9+kYE-Hs#;@VuM#g zHhyF;7x*CpptHFJ_GhPuTM9?nO;*1wQE8ev4wGBUi2a8S&7#lNe8{jyo<@Qi{aqp` zn8?OL-BQ_J#*k%kpOe^iYgAGy5?ZQ6&ptbHVX4wbU}A|Vgq4+W*xPGa-1T)s9XX6L zC#JT;C^L(x;H2I$8;Y(1H%?c z{-7+wVc{LA0@L+e1pM&VlZQgQ3*B@!go@|t_Suz{Q>oLx zQaH`lIu<*%?e!HnSKqC3Ew=t(HTW?z!g-*TSW&?E!S4Iq+J@R;BQn+TXw4C;;ZTva z^;53zwf&p<3{9bVgRv`N=}9VLrLiU4uc$ITT)utQLr}TtTBs=y-4Z)>vTTlf*%E#f zf3jNYT%4ody|+9P=i@!U5NO&Ltd5V!9I`9zo6 zgJ-k0#TH0GcOU!BvwEMKwL1{2%pz9=Y!Ej3YI|FxH>#O6sM)XU(1uyH=G@UdW&c}C zhzs$!1%k@CDBCO?(0f2ZsIKj;29K{2zo|J6j^m=Tq33$Ajqq|# z4{<)3wi?SY!$zE(?Ow1X82Ag3DX(|gc{t+j*K#_x{f2nt{n&(Xq+D)M{P*%mDqbCP z#bo2ALCZusB8ydOD)N~IAGPOhCcgT&sFk^s7(!`L_8r%4(P#Z8#Nzjz?*c3SwJ~&2 zxq05wXamJD7kujtS-!-R6P{6XT$JMp-B{Hd4zk73O3M>QHtnL9--#~*gbBly4Wv7@ zX1iW@i#%G4mZinR8S56~g%VJF-a(lXzaV-fp{P@1n|{zTAwTG(B|ge6XuC+J+Irf4 zqcpMC_egR2)pjBA4JGo36vff%{z^)`yaKfdP8Az zfkGO)z@vej4@b-P742v~hDjE52HfVqU+yuCJ+@;^zU{H4R_DcB=s21SoPu#hzk#i- zTpFJ(!}%XJu2q|*wPV#s0_N9L5!y{Y`7_o^%`09PCz@aVS!tNMm7*PC21a#f<16oo zAXodoH+)kXuGz~bvZrWKm|9*i|c7Axi)HC)>Ip_ z*1F95kOP#)c9)0B z^+Yc(?=&K%3f)eH56>+!dIkz(P0>+U?Oy$o*>!$)rUgf|Emf_;+h8?fVv!R%-dFO8 zrSzV@mj-#_{06je{-3~|>Kjzu8W{1E0@+SNr6nQm65qvs)=xEaefdsDblaDvJkU2*9Ty;%`4P1f<-!y<9a(hWc z3q-*1gU(WkxeRaX0uHs^rrMnSs5aFi8>^X%6@%o{5i(xqv6c1OU=GTjhZgD4Nbh5d zSG!~5`|H=sWh(uP2JEb9aerN^Pcvf|wP3DYlZmNi+QrqH{U0%z2EA%lM!u>OcFqlH zOR>VLh`4R(k#OJix8nIRn?D7tcBJ!Q(GjV3oAd~$hM(ABb*=S-PLov-A8pJc9*LVV zn~OZaQ`#$4IXZ*Z4&Oh?4C(=K3@i4xxe^0;J%xG9@`K~gDAde8t*JSOHgIa&YGE!2+ zFrHIgRebg;dceyjZy-f{u4U>fLN|BYHtQy$8az*7;dIa%sAzN$E0*#4Mi)2eRzSI} zn>rM-_IT!KS+>e&(&OQjeNy93$MT`*7qZmc`EZ{iYNv|kI@!oBVR$#FrR8%YUSVM3 z;^2`#|2SLzF0U89=e$wU&0&SX?P=Rd(RAb?mIA-__Y-Fbo3-S4UG zn!89{N-z%}5-5*+Afm0KPl=Mq-Cf4_lx6Z(YQ`qo^(_{L)+>9Onm?J&k#5G(cQM`& znl6oyknp-GohTq7z_-GsziRTfL_TTFh)C17*f|^X>jqEuQKy%?thgT+1~=Bpea!#e zB(o$YeUYLj=zS1~zo#W#e^O*9R>PI%(lTay4Sb0p#q_6rHeD9sRTS*KKMEZg>wLp3 zpi`BnnQB92g67ajXjF8dG1GCex;z5nTDJ@hV<7jES(-%0ubeZL- z;5+M6WY##Bckcc#&w=e(fsHi2NCDprLC;SUWsYzDwrB-Hn*`EaNXD}C<92khwh-px z;&M)@qBN&Q3XCqjx8d(jrW%1o-o#br;x06=j%;mAT z#Eb7MXxM(*c$afDe)o;S{Xss=PIY@s;yPrUehR{`cL@ng}}ul&27B*h{pXFQ7&;_yJTC z7}ZUKO_M+`ZkacG`V9iSS9rkZ`1 zgVe+hO7lr)J*1)AyHw$BKOi`AEnbPdZncetDG5C$TI1CE^P+(xqAd@7&S?-Y51)Ek zx#qy^T3AGv2p{P^A(l8%QI*i#pKeksEAowRJX_7DyWg58aJ;mgIB9M3LZ9k*Yo3pM zqAHTO&X--i>;ruEt(ufhkK}g&6A*|4l`FIIe2UgnXy`d4r#bakSrI)3OlYC_Tg@o!xHqML%9<$aHIx+ITw{)Q6&nzo#9{DMuF2&$24*R8Sbg~iAWq_5hPNQrSHOz_dzSGGuy?Y3OBdJv4&~FJ2oT6T3|@oRDRiU2fVJ* zznH+g#_>u|*|raUZ}$635oe+vsz;=+Mu)Vuvsq-)_!{hml22B%Nih@BVq5-R60kI= zXS?WEbEQYT#fID2EzLV!RN*@BoYmp}K5iUEvW`aEjIe3g!wrWr$B`M{oF_%WP2|&^ zfv2@Chrf?l^s-k9&--&c1__l|^9T2Bp3*%{*DQwqzzuJ@%5fna-$m5uP9K%GSq;ID zCMMqJ=UeJBZ{3fgni?-%EZOJsPVZ0m`Y3GJDNV)LCv0F?rP8!Og345_zaGDJmSM%O7&11H>-_KB_1#?)}Y{?<4aJl1Ja4uEi6>-UAQI9aNsPm6vaeH zm%I(4B7UJPl3mN=jgpZm{nO0x5V1#$0d(6MrLlK4Lw)W~axL%&nb1*clrSy%UbH%G zjmmi}%&65cjmRMR`9r@n_%`TGvK;TjE-JnY3j_JAHbKCegX^`&B|o`jj!D#pz2T*a zJ-(*fzn%_CcCtifFl6t?fOKm}{^`ZO@6OE`N{V!ubKCE1(T8V#v!{)A*CXc}r)QGL z;y%0Q$mY4(utM@-^o`h8!FXt)4R`TEx)X=s(b<(QBL0c`i~A*7^td#~`@@^3>k42SA(5KukMkOJu?2BS;zADjg#(gt~mp+xB;d z&`OKzc**;8_jm^}X1nWixh!in-H`RRK##$zMQq1H;<}d7Fg9H%`1-;Lb!p{bH|WR& z2321;CziEJ!xc-7iQ^ZJeuehzq%fY+M=#fDR%L>SAC7)lk4}FJ?#>HMWv&cNs(98rfC8i-wHOnj? zNIx|aAxA0Hd-V<4qJD+yA6|7ut)b#O~%1H)4LRv#WQeSe~n$a*1% z5NdxtRjQrNp<*uaD$tM$qZ+|MA;r4OnKztlSBuEeKDK>xtUWr4LP5Xk&M`w`h#*~C zrAPe_opjn=g7sY0H|j64aW0OxPyHh8r1Ka0O8$)do_)6SKKmANwpOhy9us-CbC2k^ z)^NPoM$VkLIPV9iwk2ILk=kbq*=Si>=P7VVX@rM7;!rP457VxXsIEgix;ap^@=#C3 z$|!van=I-*KRgX_ULLPynhieTex~2a&zW@8sLTQYk#!%b9v7p_61YTntly7t1ZKu+ zvUiD+9;5^JWnSRXsBe%c9~U|Mr>W@I?PYM6n!rexcI_rM@(1++(9?2l&H7N!ow;%C za-jl`=$WeiE&AFI3$Gfl`&`!7kcmUFyc93sy1SiCK#|1KW+TS9-%Q>2L<#Kv7_X3f zA{xI_(WIn}7(Lfl+4mZ+-AJQCuFids4OE${w;|gNs&Smd<%OY24(fbbaZh6+7Np@8 zWDkLRVgUDb5)m$yZ4KM*&3Tw*Yg-Cl5ttfO)a1HI67{JPeeUtDY)JdmWPLcrzRq?0 zf3dJU$h2rHX7KH9O*YJ!y^*7Jrzfw0xqB;&mhRqrcp9O!LPx`zsOKZsmI%Tv-LdN; zmC*desg@`C86P~#R(Aaxj8{UFNm9j<1YG*G8h83sko+)9mG53ADxZFR#H=)Bn1ROz3tiz?hk?FU`D=f-sbBcy2F62t6cFtJrR)(KYSfr zJ^$|38?NodWbIjd-;RjmUbskZhdp`Isc@=dID|7B2-Wx~SKs4l)z$olt0p_u#icp` z3W@;Hf|@~f7az^OPz%ZE7yCIbnA`p+4U2q+b6ukA8NAvdl#=llgpsMn}Dcu zd4Kt{?TUEPxVbLAPQC4|a?6a0Gfc2#KU~-D(POdwkMkZZ_gV8JK*L*{=$TB%%|C0< zptcRIGC!Y9`dqFbGv!M{aH#VwySf<8rQH5tYtMuTDBs|APd@AKXk8Zd8Wj!XF7FoCQs4WD%E zx{~R!boM*(MW0`U)s#JOu|0mnl#HvEG;QuGGL50I*+6~!Jslp|OV%?U>&xr8ebrJ^ z%P@0}4s&9N7;1+}wb@|zqXL+%FHnNUv!ul zB=7b7GQz2OFv%q8(Ri&tsUKbu;IxoCRn-xWuSNR2*J&e|6-I*t2LJIJZk$^nRQh5G zohMAQM@Af)hNj$clb%q+NcR3A>LLsR3H3dr9Af-If z^8oy=0Z*5djj8+U!=>>s4q+ItM#0yYg}jG-TW4hH*%U3K6_c}P;*lQk2;0=Y&$(eT z%_@U#dy`n5Pc$@{Va3gsl)GLF6fsw`3Zyh`KnKs%D~v~o2{{wus2oz|$3J04zAcZh z(3g^8-F6SV@ASD_mohQXCu>ce5;@D+nS+q6A`pRDJIT$XDfF= zZAwW}r86i}k(x=3W&Tm6GknsUqn#v+VhdydPM@aV?&B`WxUpZ?6sylmGpKR`amV+r zS>LZN(K^)*g(bRhE|5pzh0m9a{#l<}i)h{xxJg16Y1EJjVvE!uos#3<u`<)D*LE;}!atGy&sZStt7ZvZD+ zY7bC=r@cbUZr`D~(azvo>>sAR*g~@WS7;ibLkGGU3ibLV*kqZyd^_psb%W-j)_F*Y zVXHv`SFa#Er(|?_{Za0)zW3vcc$@R}V*8xw!Y&6K-4YWz_(1`vMBG}Zt6tJ(kx?GF za#ZIe1NLFGNlzQ;YCAV*6RfckKy~=nF)M@ekPxEnU#V&)-y1-|^Ai5^xOd#_c;wl7 zt>%)443y1BeoTthb3<0dd|{gF_{hXyrRH9Kik~`TXPa0C#YEpeWOlT~9?`ZQ6|bv- zPL9NGzrWy<_-`HbV+1sE&}YI4qna2@zka*FQDifGH?B(ZN0_bR6TUU3puQoO2Hlsttu=v&M)9ZskYW19t z`K&BUtY>$KuILtvm(E{;+OxwC@*Rf2G+o5i2_j?cGKQ+^>S}PEn#j-VY!|Vf&QP|DX0EPh$@hwxm6C(IoB9z!$z$G< z%-Gh8Xz-lJ2D{FPTx8g!ZsZ_P<)KOG`VG9+M%#s~v@s=^Vgqp{CMDYi!vVRERyRx{ z!n8d0%MhPK3U**sb-@$2j<#sy`Cb<2u2_YaB2jx6FXmFFLw%NZ=kF72!ata610c`= z+9lb_ncMKb7(^UqFhvfWw6@sxH|LSX`n1n=S_q5$oY^lnmeVhfumm~U87{phHpE+r zE4O<7=R6Zx^fO$nWFKCyQwKnMH$H%p+$tJ&TpvL}Kc<;bqH9w})GsW4bL=>C(P_Y@ zxB7(5N9Wm*1A8F!G^)otGCVOX)gTRax4x!%GU%g2h{rW*o5#vo4mR`0r=YW_?fR;I zDYYF`R+)u?UWCoP>_!;+a4mTs!7YV~(-5<5X3b7E3%&EhDVoD~#nx&Q4TkHPVkY78 z$ePOq8{zAqA3owW_WJeabJlDQPSGFDwuVC~;o~FNRkN-z9`Deg@22NC@EwM{a5~$l z&buOgUYy&+~xyW~= zZ|02O?>7&)1S;u*;ZFh!SSqpx-=26Xm746(!0{kms>y$Wl_bxglx6bsUx#)}IsLI}4xUGV2;+9;>t^JMsW==B@0|K2T#GfOU3@ za_08M34OaqGi$q5=&sPItHZ21yPaxiT36D1xCJvC__5i@ZR~7Oh4qY!yOfQ}bhPw4 z@Rp!uw&G}{$(FiECw6emg3lnW6V-oQ;SK3qFiQk85%;E>={i;+qT++Uaoh(c+!U@^ zr&82hdQWl}L-`e~p#n3CgaVe*j3PE2>Kw*pL4i5@$1J22091GZP=#^odak_v^=@C+praWvq_xd7%c+@RBHsAN zbnn7y}L2%i7?v}*zszL<0^d%%+gOkL z!kJb^`8iwk+H;4y0OCE}$Oc?j(2vkOaB92q-}ExghmmrSqD`D=mvY%?J-*G7Nr9fspR|4_nkebEsv1`D4&{xs^wXnBZEsjxV;gHr~G%|2cD9zkY$)F(ch#@9eS)nX7o-!?jkYHsq%K zKP*7YMO#hN^NC!Ja!EbTh$af-!K$o1`P%KN7+DLM+pCG75I(!9*d3S8s<37qpQ|x5 zGMvseE0_r4gtaE6j3IB+KLN~nc?v?U+_Lw@VGTfKBr6E(D&{ZM04L*C2s6 zjZ<=#YeQN{OoRqE8)HKH%pRo8F+^DdAuGJU=J5gFjcwz&o33491rom&8!66D6GH3h zkrl_OK2@SA(tM|_wHRM-BdPf>QQ_M)y`!_Y0MP?Vv{pLpMLRPTsNFyA3R7}j)y2u} zz7;FpRx^Ju4+KA)KK(iOXiBb_1(~`sRQkbxz{{ubyusqG}9 z0SuLP1BKQB7b8fzr+ew!GODfH+0xBo0_{0@$+~BA`@A>RuD?fjOG6@3ESnZKR95`G zn|dA*Y>1wdWvR1* zcyVm}ZKty4BB=(?snoglXhxYA?N{*6nXw)Q4%gc29#nl)AOJBRN;u&Xs^cFHEVCF~ zAAVV{s^~N=ZA3M_HQ$=C5o$)II)0Ec^;XtEDqpMhqye32OF6m9_C-LAsNAZ4&RgX7 z`Wa+Nsz$a+uTF^v4lb2$Z!9IY)+&6akXZ22uYWsX2ZgxOP*tHrllA>s_npJr$e@Kh z@vE=+7+WJyTTuPwy!5uXSxgavHEq9#3N+^Gnqu70Hy1GC$>*OnvhNKa18l=QYpr#CGh9uac<@Lt(CnIm1|(GW2XN{!TMwiOo1 zFD5}=GNv+B5c(<vrfGL86n8oJj%mlp`j|;o-T@Q zjEo>^UgABgOn2QruZckT`drT-%~Z-qhTCQ<;veRX;AZv ziWS-ZUP6Lbg|V+7RVEIT?Qq9@0Ubj4J*1DS0WM_@ej|9K7Pyp~5JZXFLeRUCH@P}Q zY)*ekS)Lo?h}-Rnsd$_aI(Ljl_hqWp2y5wXSUv9%FUj?jqjLTyDywK zaVQ28Y#~k@>_j||d7ZXP6u6&r?5G?CpwLBK)Bg4F<&MbP_6a4k@0w;e5)72K)bT@Q zkJCNuuYUCtIBX(RNs~YbJGQ-5XFhu+drNAn;yW#BxH9MsZSr%Sn!s=Y{+Fli-Zy;y z_h6Q1B~u7jf*VJWHS~Dqna}JzSuqK~h3r(;)jwB;+c+sFU~#X7JCT+cB|srAH*K{# zX$?0!8~FfnYdJ}>l*$M7@+s^)rMCA^+C6Vpi%4z)@}(O>l>6R3i59A%W6^d!PzQ&njKR$3|jqW0l&H*5)xQ#K+O- zx&~!bqeHiWVCB9yG7-Gvj2m&5&hJDb|JPSoMnT=9=^=Nq>%(Z--vc*zYlcGXLl4wynhy zk<~w;-Lg)ql}iXgCGQfE+oOIOwTX&Ho1GGmE|IG*g$SvAvri zj-)7bHV!-}OcW}x#(J)S;PC!78#A!z-2*ZHHlJ_yx?R^HIO2G^EpR^?75-xG0gDL# z+Z_J~95y+K@spDllY+}X7xrI=lUx$UY;W$JOdXZ%65H>LDcm*!wns9P?2Ii8i+#L* z9sK#Kzi>P=de@XHqAKKeA}uu^XG?sRcSKXy+F~|XY4vjmyb^|&2D^!q+MCUf_Y)H4n3#H(~%*lPpN&6AJiFs+iM3X#>Aex{o_DQPImYwOfUxI0%Ulqv zr|&#RZGSN2T>Sd~A?&^5ss7*pap!c@q2UN=I|!MH%H~*Q9x{`ejAYM@a7bAd8QGMK zWEEwLin3=JktBO>*}v;~dc8jH_wV-od~e@By4~a)k8wS&eLb%GqhG7=TZ;F1YdwS7 zp>5MpyY82UO)fe0MyyHST`u_+iz=m?eUL22N^_(zM2+Ba2f&#<9{D<3X(iYSVejCa{5<(IR7_JHSCs^H3YQAK%u&n^DpKXk+hO*L-L5pJDe zEU9;F?bcR4tge}lt@2v5m^UPucR93u*vH{nAKTjLHP>HM>q679VXYuvEGRxm;q9e1 z(}r(`oX9ZT$5+l0Y?KcOoxY%Ht++?D{Lai%(9#W% z-r?m)a!Q31vHT&4<+g_mJzI$j6JrJ5jcy`aL7zr)3R?yIo*`G1$DiM1Davc?Nn8Cn zes6{;cCeK(#hdxc5k8(;qJ(^dE|>l2G22_+&wemXKUG`3%d)u|oapK4C^GY~bEN0D zOy5}!F(^T^O&v)feJkSa8~&73p8m(GHqD^c$)kn$sn^uX>fMqr;xxY7>oHM$x=Jl) zy)SDRzBDb(KmEcm(ao;UN>uZd&dcyOotDg}q;#zZtv0)6QVl)M~^W*>`#4$deM>)8MbG$-xrz{IyJ6+43DKKSd^ z)JNyv54*6HNTEN_<1we3$_j6%0Lv~~-KZ?!$={pQKb>IHX0?+Tx45L{_4~T%8%^c3 z94~bvp)R6A8Jk|k@5ITm#)fG&0lQ%U_UM4#y3ezQeX4LR`-{2m+Y3nzL#LoJvyjX* z{QKv%cRZ$IwbuZ8e;(i7ay9NKd=gh;nK3Kf#;MuSzYiMGU}n4K$b+K1v4!eBI}@2D z6d_61dCY&@TihuHcR|Y9$6%@BUbap?7hJhY$h-QzUGW}K`UITmlXs2aw}^JF7<<;H zv-5wMFM_5*`|hw_RvbSFD2iLSQNEfL5m1HavFvC`1`yEBj|Dk{h=AJ77`}0WAo84S zfpli*?WprVOoomfQ;pyD zBRPtVD`$G)**(kQEe5Iqv!cE3mR(|zcUb>k!lgj_&_cuK4<+^!E{n4bOUWKBxl>yu zm%9Q083vkHFTusFJ!luMdBT%e7Bq@}u#%G`Z&{BmfmK9?hJ4V%DWroluXWogw*E*$Qs!>+2kG09AnUrARcA z1W%T`dYjxqtpy$8tte>7RR-7VZ!ZHrTo1k(w=!x6t#2`;6;yJvma$yHZ_$K%@F#xD zP`y+N~~E3j2*& zErOsAwuS}LvPak+OGBIb{+?pfc8ACZiR30S=uj6Scc}ZB>~EV9=vlE`LBRpilUT|< zf?)uIQ=1vS@ilM?eIq6Bq9ztUS;LsQNq3Q2G=XC&-Ndx{;aRU%8I1~E3$aBJxsZ)-U zNK!D(dSPA;-23oHa`NsMEo(u5sy7CSnZQ_Yl~H7+97^#EU99ErfyWuZ{{8j6bWS5^ z3@E<$C!P}9>JMt093uCWgH%G3Dc%q{Z?;BPge9Ah`E!Ks$WowKP39s}e`WB<5XFxH z@dZ#HDfzguozi#`x`GA}3wVy9z@w~j9{k_2D)?Z=iHMOhnNZ&8a{>&CT$U!d9FN0hz|u^*lC)U?t^|oTz9v?Q@`To5K1{2-@$H=&kl|(p>ac5H6!xbY}-3ZIAFpywsN_rMa z6H<2=fxlov6s|O_9vaPuoMgvGeqKbc5MdFH{*J=F^wEFiZ|x``Ka1whT8U zu^2TF|G{sG46N!%u@D_B41fo~omq|V$U4>(#-q=s{{oCu0lyz&`0b7yIr5!;Jw}WB zq(I7^S1q4vZ%!iC%|V)M-5jzg=4APJ4|K|aKJX_Jb@7919OLx!`$!`QJOv+OwjyNs z4+1z5@JFD$xguz51^A{Pk67qs{3*0Z@vFwewA&P6h=wXrB@}=Ay2f+LCVQf2lAj6V z&rIMKKk8}`jXiS)*n#;HQ7IbMuBGU73f6|8k3cH=#g|gZIH!O@NTxwn3{as-TUW2> z_>n%LZM4T6P$d#T79Mz6%7W{V90h_VmB+RWPzUnAuz&kii3^-qxF)4scYdRXck7ywBcC6+8w9Gj}>vOfVrbgN=ew#R@1ph zxQ<*5yB{6;8Ki+mJ%LaUpNGUKf;DVes*bnGIM;?(Orz8<|PKKi~0JBxk zcHiVUA{G&_y9ms%3!($|xqiqI%f4J4ie3$;^HVve6E4C!NQ3fmTfA~gj}mPEkfqa3>&5wXi@L*Jo zI7lT9M#?NH5f=UitI*(WSV4xw97g8(>u&)Zb&~1Tp`*x&ZNtxG%S%$m0FV9M04@?t zVr;B17St1Rc+J%Yr%_TMWQ#YoCEl|86Hjv;(UgZl@AAN=ztoyrVTF;OfOuZWiOC}q zW9rRu^TiZ?DBtdwZ_I;2sDZ*QovU5<5ZfvOwzcUv>I>*E5hH&SfX~o;J=HpiFae-; zx*&KV`~C}H<-fEK+>Q71gS+wcD9lO5L5aqN!~Af*INE_w_X9BV4LDM&PDfLCA5t`3 z=7}}gcAepdGuM-DxkBMr0l#0cL4Lafp2KjMXaO~*17EG(&Gug&NnhijVamDPiE4S7AUNn~nb|f(Zoju;% zZaRj7$Fp4)FDJ#afvHdj7_3bH0QXaV)z}`X#7^Q_Jr_ADl^MOWXrm+0zrXQ5J?XFG zMTZVZB;$qs0dFu;ZU0oXIjzg0QhF>0;=kA-w5mAYtptQXHknw$h+?ktEO1uV{W(vS8#>-(CG?6t}ri?qsp>BK=AumyzM#CFrM!K}?3#jo2fw zARHp>WFQD%sy6Rx;}x@uSd2W7N}a1&l-8k&Lf^8i>T(kf)r#mFgZizDH(&#=IIwT10xkG061>_ z;Jws)U!?l_(8%*dX@$k>{SyziGS=IM5zqVVtQee0A#Yr#sa7T#+)g6@( zJvbwKM7v}}@c>{Rnh(x;kcOg>CZ6ObeT;==Bl9s2ZF?I9;s(w8!%?5p3Kn4*^hK35 z{Ds2#7xRcpbm0J*<%M~l^>tb4a{MSF7U$nYew789BuYs&4*_r!M%IbkTqiuB93P6h z;a~?Am=hwuC=DVXxX@J8pr&z;JK-4J;7H%-JDSB~%6`V!=Jmgrxo=grCk7FI_|)Ey zN-h7uwpS4N98}cauQFTm%e6mwTNIY#{d#|-LXul03*`+MwV+@m#T$l7;{*TmJC%W&(~grX3KFJl5I zMDPSS_AQiN1X_j50F^mwhBdGyvLu?{{a5!{fK3kGKQH;ToAgS;`&R&|9F7(td76?zhJs(@N&2I=GDk~q;WxJyQ|E?)9NL}}QW1f_M zyU`Tq@B$_g(z!2!aKHkbWysHmf;fj@k`G1Z%^+Jz7b! zE#sU}d02}VaNWOHY(YqHDSrKAD9IYrlhJZ9-Gl|sFAePF8I;H=;5>-dIMWji4xqlz z>FE4>7vO)aZ<==X_m4#T)V`i?IWA@NVs8|#&(+tGHN)096I?6}vH2Z=7@ zMRDKjgAun*81YJI-R!Q1lib`q9cNA#K1zCggi)J)S_^kPAMq2hatyEUICjiT#((9K z=HN&Mm#Z!o6zIdN&a8+G=7JvYXlSgZRfvWz^uO+(!#%*FRNX^)+^Itff+!M+4lVLC z5-TEdm2x(i0*x49@dJ5Bo|Q*tH+~vM*hY7M&mN-a)iUAIG6R)EsGo>00SIJZTrny@ z&}!-yr%HB0b-gviO~8l&%cTE};Y`gG>=LkoA_cOVD`6=Bu7i|M!z0nh-M+8ahb?~C znI(!c{^-d26AYP9sw)XuZ{8(EG7JqMYg$65c)#mb8S2cJO`_L9y=ptu!>6r4`M%Iz1P=vba2M_buw40XkIkFxMd)?%s~!RWk7$>P4*+`i3*iK%C8Ba?ILim{Z(Xb zy4!e@0P4%H0lC9bLXet z>&PszCCH1+-oObEVFz0x7Uh3bl)Dm%6q(sjYX-?6qF>^iW0`i6o!%Ej)Dm%RG*2G{ z1+Abop9YO@(vwd#7qm|I(+9!KgNJ|zXJ5+t6vC!4(5)xYE-Jx$LSa0#Onj$MWFk{}{k#* z{Y7(AM6aPyA2FPSVXz3{8mNI7jfVPKIHgYHlz3aVar2-kIMx_>xVe=d96EJ-N60K_ zd-sIPZhstkCXg=V?ANFZB_VtPaM#RdTJf$H3ix~y-LH%T45U6-&1VdFAm?w}K@$Ev z^;bIDZM@>{*XPX6K6^uH!#+WTB7y8cbH9Z@+$jW+DBz)uWch#4{v=)9x}OwNo`?>$ z9jL`)!1!b4b2N10DP-a1gcDa@Ns7m=A+|9Bg72st{21%tO+R?zvE8@1X<5S+H z1*OW$3;x7`Jf=c&&IY0xuy9B;14IN-pv>&bxd6|&Cw6>nkf$Y7r=wkz9m!58iqRyy zj3h=zl#Q#0JD5E)Ad!Jm|)x+V-Yd zn|RUZYe%eu1CRp>cI~Y`wPB*^x8K5`S7N~8{FNF9P@s`Fm-~;R)<7iTm|Y$~?g9}) zgnh`jd&mJO1!@;rim|HR)$f?Mk-^`ZYswV6cB*g=N+-k^!G~tV;?4k|KVbF(Tx;9R z^<_)JCC9^l&dGp^Hy;XHuhNQe#fKy5N-MZy!$u{Vj*3=hL2L*o}USC3lnz?E(eyHX^Suc z2m=7Z)sv0tKbVS#Z7+YVKjCWnluRtB#f;MhSx zC1At{oWuvs29b^XYnE}4Gni9F@rpNI#w)4sveAES6nG8(#P?@e zL&-_zTOGgSA7`I-AiYlto$f0>cPeHqp1gRV5|aIatKtXm{1B zAH|W$^$~y>tEo^jn7<&*XW>;Ei@!*e{~!$)9K~oL(d@hAiHhU@q%_EG*%V$Z`KCXN?FDGBse1&nJ?6Omyz} z*44+~Eehh{=pQb{tXiyka+UNR{u%V9fWu9cuyBL zHZ(*$mko^HJ$Ni)DTb%A@Q?)n2g1ChpCEGUxv4pjj`T;8J1?VY`pbIb`5f3RKY-6b zqF_d!rUk-g6Iv)Nyo-rZ9*C1Tfs}X-Y3Bu#0mMBr!$CLzXO%oAjt36z!z-a|V1Zr4 zC9${Fi=(YumwxZ_D0jkL7$`N4c5$TMpsU`Xb;u;yO+2rfv9q0dE>-v@`n8yKZ4P|n z5wKN9P-8mnd>NeRoL2aCk4&8=$WOp$*ASM|wnsp#caWUn&+mKP&0XxN9TxONLCUxx zq&SUWPQsGcIA(@wF2&pB9$A8P{aYj+;y%ZCrMeI8Lba%~eMp$_vqOG$rf)1aSWq9a zI{%%0!Q;-tPW-rvQy8wL1IuKmqQ?CpJ@1Zw7Tr=!D z!r^JXJ+nJv;~MdS%YhsAP)KO8opkqgaefzj0zkX*0mpJY3$H_zcNMJlL-msn2sqH+ z9@&!!PyP64*IQt&k)QldqOiED+J@`_6|f8Hh?RP`m~3>%N59jmF4&eR zZfL{~T7wgwRS0KM-~;PuMf)sN^ZK>YvtR}hcvq%Zb;ofEk-)qcK4eT3eC1YsZ2)M5 z$X?FCUjN>lC6PjreFFqJo0Q+yM^(?{710pjFQg19Ou0>EvJD7))!%RTY&>!+dgXkl zGWHabvjM$?a0+=g?&%uOJv7eQe+%s;t3Qk^Y>|Jmi3_9`=?~j7zDbkMpYK4Y z-|12EQU$_-4kRqdtuO$a8$n-J8lCYVal9t(3|A&xmET49IzJ-^>la1!n&EoGSwSae zh}*3*`no1oT@i}~^v5|MUcx`PxrBeyTWRVf2TkhprGT}KxeO?aCQ8jRwDUROJ$n?n zsK8_bNV*d#7ouno<*)u`Rc(c|XW7=!44?rA62t{T@52NDs}1nC*R-C3?({=+Y+XUQ zZTrRtT|?S8Mf`6@Gpk1O@7YfD>zp42vfF`(?9?Aox63+!L?_B5JbNn`jsGNG{P4(R zM(=Xt51ta$dT`*_P$eb*&A%JDi{!j4Da}RT`_~Yr<*kg|-%W_z33G^y)E)%aeb5#4 zagR|xe(4Gj)5USo_mLjB6aUFAkPEVvN*3nk!4DFFqw?LFj*>U=f0ob0-kNgv)6WRS zjQ0<#UoGJU7gYm1KaR%FX|&iV(oTPIQ;}&|Gv%u-9t5@WKb`<{DCk2_oe>lh4@ltLe2jt@VC z;^vpNeC9HqF&A%R{Lc#y2*3X?r3RYjhDm@3Ou!0J4m~k>0=984N@mYLSy(63K(+?4ff6A(EyR+P@G&Iy;8ll{Ut7xG z-5e#;X$Of~@;wWVgL9b&GC$ZQ`8ay4_*9b74tuOEm0?l?(S^Con4$Bw@qxar|L zavpMg#Z)~%^T!AOmN{_qP$TA0iwUb|4VOgxzbxab_iYSky+2P>2=sNQYsOgr4I^;n zFhx(U@L@z2p+L$Zr2FMfDXopu!{ifCIwDY}R?gja02w4OVV^!Cg+gL&cxZ_K;(tXQ z_;4p66emaXI3?V@j!H;C?9)G6JADWd!@u>DFK2Xb*j~}qm0O+@3H57+ zenTdta5u=8`4p3(!gA)6of}f8aZ;dXieY3KZ{$O}rq=Wn0&16_1H>{9+ zEdONmCzhjg@`#(OGtQn|{ot`9B&!CRgj5Z`*c%pYJ2xZ1oSD%b^=KEJ2+{32HIJu7 zdBvY5H)m*hmU^i35V*)Rz-g3!>QnkPSyda(A(f;Oc>Dry!iXQNSptbLq66o>o9pvk zu%_Jz&lwsW2_y``Plt=0`{G<&o%H2n|GpRy4yLnK*ETyrGfwG?^4ybmzsWD_;4L6+ zDW{tmTOaUDb6UqrQec309D6an6c2SQ@JKWX_s}BuvRveX>7IN7ANU2tZ~_g@&B$JQ zkReBFu3|An5+7lQOJrsFpZh8#(jD@2+&>D4pNN%Ll3U$e7up>!pdG2kFS)H9G~gZK zdiuoYlh+qlJ9Xq4ABB;d2QTXBx6$5qN>k}aJi;Kw>dE(+`jUG0NSZ1xC&AhtqV9o9 zmxUSc?I+02F(g@l|3nNZ?7~R;l-SQD?*xLIL-UBirH)8#1ISYku3?DU{O9blIJ(nD z4)S+7oFJ5};cc4tpvs3+KzMAq#7P??&KsVT#9QW?^w@iI)x(olWycux(OUchgZ{A#wH^fOsa@mL8GNrls*M1{=*;u2u$ z5Xzhs-P~Dcmt^Jh{4q#fRs*^FKt|tVqPixX>A5ml561BB+X9nw_w$G!@}#oKaks-| z(zDLpUNJ^t?y;=}U#-Od(Dp} z;P;>kgdOGZ6ew&!5JjqHkbkWPv-(+d5b-{X;UAB>NiFx$CM*j1s|9w__%QEI5OKi0 z{RFxcM}SfsO*<#sxy&{#<5H2aTCD28~u=kQoo2Y<5($4Vokr;fx+XXVCn(W(~RvCD{-MCFVc6< zJCvP%Q~b8CI9*5`HMec|jbm}PU(Kv| zpFOiF?dXbELyL!))A9pkSl~Y5jbQqz018Y56<*kpoxI~%WXJcubTEgXLrZurTCn_aBZA&y>g;xD@y<+}YVfN;sg3(2_CI`q$N)zk^|wW`TMxPCJ1ys^$SMOLlVX`>CGS_c=Zn0=oE}rBjGT$*+ataU(Aa zg|Kpg6ncrNbjogM6t*T2*4acu(HBVILLWRno{OSlZ^-=AU}NSQ;uO_;;xAeK2#Io; zA&v}AV?V6=b}dwFJ&^3UVMkO+e<$P~z14{<9L_zeXLB z^!?Mm*yC*|J;Jw41hmNHyK@ck`q{q9Z=s)!pAY_Q zENV_rc+q@3!=RAKv<>NSHuf&P8gJWO!@$_ARp$9kwAZHaK}U|frf&BwjvtLN)|cv5 z#q3xUgzTbvNv0 zYF2$-DM@JyM-q+&Po|vXebTqPk@F;yUHs0~Q;H4m?Kj033*1&bJ^svgn|=SJ9sFix zTcU3k@7*GLZR3!5+S4<`+Ad|@%Y};rTfU8snJ4jHMSap4Dv8|Z)O^*nUTL&X)EOn2 z)oDwZXJ=Hs`eeCYrPKIGml#<*=3}#A-D&+;(!-ataw|5f`=#7Un_+%KCrxwY2vZ*M zVb;f9dEL_-hG9MBUE)S-n$P-_TRQQg92~-!EVAo3F3oP15DeX+htWgNyxi_Vv+>Dgtx6 zO8~cKY+LG)`@U6j=jlVPTo2>TzKkWBNsqw0ZN{SeX-8 zi#G2l(00~)b62&&W4UrOBStCoR57t)x1n(VNI@Z!N;w}x@@#_R^XZ~7w_r&w*@Gvx zXyPQSt-ep~Jv7107&aNV;zNBWu-=wkj>CRyk}hfMNKAxo20Ai2cDYiEMKj|EUeuhW zW|K^{*PNP~svGUb$B!in9|vm1V$X!e7l{kTjen$<@1jOzl@bWZ7(R~n6Q(8H4c2^X zy{bf2{--walq|ybgkRX2t1e2>K)(vfUz`o8oq?gc^;TEbx&PFXn(k@yyuTE97| z1XjiKD2h#2JW}!9q_*mL?N$&imhWveTQ$PwY*^)dC|a{+BX|03U(w>|6jKAkn3QS3 zQ7en|`g|jHK>NO5F~2M(;4{5Hqsk)qcF#+{>9Q|wdZEOy{_Xhs*tD6D6KYRxs)-Pe zU$Qx?%$SEW?A!A7xi#Q5n){?+DX%X+zmOqmTBLMqL$YJ0?oz?d)ZB2KqW0o==E(MzuWN{%Px8?$cc5oqgR3_Trxt zoS(jCojos!wK+xBON%o2$ElJ2Dly;csYIgZ>CM{}8w({I^i{3K^_aG_i*}lyj!yOE zd+3}lc{l3XyqMOTY||;=DzR4n`L0*Z`OUgVbo;svKcB6fkT0Do+ZCLvdOgl%_1;@E zZbKn@YbD}KlXXjzO0R{BM%?5&$Iqvqe7cS7yv-bPQzF*}4EXkhm4B*m>wDoFF1u?i zw&co}+G(68rnr&fe9JU>a`N_i(4Sv-#eEcoN$~PEtptTmrwVwz5NZ_YrZ!6 zu4@$(eq9gk#e&E@{q8EyZ)I{_*^#Z{*YYZ|>~pQVyjOA)9lz$XXKR&v>o|@!UmrXHBQLVz&J{brZXDNA3-O{Sa6e^oE0DUR-L#wsffCO4AyH z+Sb)We7)seAHFWwaVFRqop7mtQg9_<#+xxo(tX!?-E-!0Z>e^n+-LUkAm0xz+|vKv z1$h5f?!j_Z>s@PemTgh}_K9xQqVNWqE8X?C-~IW~`jC{SYMwv)#g!i&8)6cFWAgjL zp2@B1Ji|&OpW;^I<|#%4pNEM{V=rpzmx6tFYUh>ggdQexh3#y+E-fZ9+%#MI{78K6 zOG-IBYU9W%njZO}Kaa0(YKfE0{CfF7WMb}A#@xjhkww`prz|A{XQcKri<*as&mn9X z#Nel;K|()!tC{l{m2jUh4sPCk$p9O&qF4o2xWMvDG%n~2YBMZlLn_m1frUU-?p~Yl zLx%i~8`xo47bT8EW{0UYg7@@l)LqcM^!t^;d~^3t<<8vVC*>I{Efzu5M!)I(Hx5lc zt-G7Hzf0=Q?t0U{rtjLz9#)E_7Y0lPmnsEsb}r35iZ~)>m*g1cW9d>rmj}=!9v+`% z)^APts(gOYDrv;ZCEYt3IG)y$B?E zTzgkb@3lSm-Pm@xAUX5&*%#bC=hg=Pr1oDN-A(ItQ#BhMDv07q@mN;nta2~#P*dK0 z_dQ#FSU%IE(74e(&;L8Ko>GVF_4Aw_%~=~6p1ePdQgX_gD=xn3{vlm^b*>DTBy;T2 zk%Ne)RYU8TI_)qkG!?YXz_6!dwv|K~^p`QVoA>FZwF}8Qau_ z7j$~wsW{a+uh$FR+>MQm={YOw z!TSTBHgrZ>Dl{6pg(i+)7)lH@8~u96@K@Eshr)EP*2fw7)IwYXFDX{Of4RsuE%44p zJ~MOYyl&q55}k#+Wqmort`Rq#y{~+-yNO#;maE~3s`Y`2*}!p^jtN8aue_NV{i)GU zTS}+TPJGKRO#JGaGjV6^=2(Nh*IPx8=E2`{QFq3?S@zwrXJrUW;WlSfdpZ&Jp7S4v z?##$VXKtzRj`wSzMtxUhdh;7+-yBJ*2qI*WG9n)fB>gr>PgD7Axg?pYdFgA(Ub2ou z6YzT2Hnyxmxz!8`OduPGZ!$KG?ow%r$43Qnz}1c?jnh98F4&L#BLm8l@!|8+m|o7W zziuv&4Pp2d^oCeRQA0z+bwQDz&OS-`BWZLr0#uZx)caeu&5m1x_ns%u%aHgs`VepQ z*{6A%h|}q7YrXSBTaaoyV_k|n z&B)xiUXTXivF`)!9L8Qq)`=G3#Y$p0^>C2vMrC{o#S>!qU%lfMj8?} z8eDzjOapQe=Pj&C@&eoEz*$diyZ4jFmIjuKFEa?qA+zxrf805(>0HKT4AXTu<&LfBsUQqCI#IIPhn%w=Qvb3kz9y@kD+L zKB9)!HTHg2Kc4oeiMSxXXZkqe7ks6+cFMm z`E>k%z+lBEYr_}luUUbQ1oPLKMQ18oiHo;JQl15)2?~#1)kf=b^`gvcg@v~xhv8*DybAomGfO?Y4cckEloF)J~0;;GwLm=bKJHI z*Us5VFx@|kzwd=pho))w>aQw~LF_!Z)r0`*U zv}>a7z=g?U9`0M&qv^UkE4o%?%}=f5Glh(I)my|oDke4uml&15w3M?wxL$cXX?eoU zJM};kLsEuX@;P%xiQ5}lq%@X@LJ@f$DLIZ)UjJI>QAdje&l~O99d|b~oVwbJ>r;!R zqMzo*H+DJ)Dt@|r5J(@FaN%BL5qPo@vwENol76f+SgTfxsc%c zbCc`S^TuCZ^?x`oN>%wX32A-~^OnB;&W>4g^sCv^5Sj94YZnR+3W%15vZqFh5tsdu zWPqPP1ih7tjM5%u{(=5f5R)?jt=F-U&ymnu?gI)t3TC&he+qjFo#)4B*5(-Z5^>>axSO!^!Z`7pLM`R*hIH;tWqeZvHJHczcEvPs$=b4fAZ5GoumJW97eB4 zXK^J9>&Cykb^HBpRp!8xC%$)k?s{)oC5`0YSrsy_TAY?Cot_489gd^{ZN94KVKzk` ztV`P+YdUozm2(0gJ5W_WWM{oIGQ@|FSb5Lm3i%6s55^+L6a)8IyWjSxu;%mb4>J8e z>oa**Wc+oP-VCIW28|(D+mHL3^CBNh1;YP|92w$<0}FyE3_hF#orU2j=oq28^a-(J zSV5FrCjXVNce2%>c<0pxMJM)1@fu#aM?&P`BoYdf2+UODDOYSnN^J`H(<5SSKBt>T zjXp{kkK#&*Jn^AEx$tE$v3T3{fw+75PC`yyRPU#>Z$`Y6+fT~Os<#(6nrbsz!J}-< z^RnUCKensiBDuO+;~KC15($9yt|U;4g;i{Ebm}G)HYND{v?*s7aCUq|xcKLfm^GE^ z#=!S;@3RWoxkbI#W@wzg{}O5l8L?Jjn#uQa_iFhv>ea0A&gaLS%V#^*h?}37FON3J zyA*`Bd&4tkt?x#CoL2dJH-z-#`MCTH`{&-vl}e2~^Y8h-o0nhvnKN+0)xLR0p=HKf z*R*wJ)t0mGLSIAB%`zI>FUd|M12G>$YFPI+{o?z$MgvI1jWdgS{Fb*TYv8*5r~;u8 zO743=Lliz-$)UrX*={dhm3kMD><$10gDzrQpJ1xvObKK!x&NF_dafpeNk*Ves zoQPQQDrOte(o%}PpLG-+so(E-I0=N>gEo<;73f)I?fZ^4_`?3q8b*PbbBPN*cOwlI z`%)Ub>f;!@qU5*SdwtD&27?#dJ9fW{ddBCD7g&2LZ*IF~t!%XU(KO^7+fSFT=V%2l zw#D-(#Ans_zEw%KxLH>6vvSnD&$w#GVS($IT#0wsJwf)VgW6KH2t8_!ncQ_=PXE7GxvX@$#yy$Hp&@cX?NjO}x@4Mi(Vr@ABLZ z+}%F&`^!#>xP$(W-Ivefp4+((<^)iTR;Vje9XhcU79clrcMjD0$aH9VXtP`PoW~Ef z&jes@;>bbTM}P3?WZ>{-{W9yFYqs^Q{ScrRuxWlH48M(6xnqyH5=7*e5&Y zXgObN&zh^Y%fS37KC#QO|J74u)Vf)(cl;yM$1mNEb<6**c0b40clo|e)zYN`-SSh- zmYF$GcNY*hqEX6#clUaQlFy%?VI5^Z9v5s5^mTua`MI$FyF0KV3uOErF_P1w1iU5M z%)a3m^B&kSiXFnH!NJ!TknP_<9IXdu-(1GhlStlJU_^=@<|G-W>(*`@39c_HD*CZ% zKb6fspZMz(Bt>mF%5O7NrlAy{$ZYeeWC|7W&pIuJaMkMiJ3;sYR)DRcC~ z(*sEOSO?bJ|KzEzy?$ug*XC1oZJPK&+E{iyx6~*@jDN!s#0=T649eejqE@|nRxl<^ z7RT6Lf6gt4==0=2w{Q@#Hkc9g<}BAN$0x0$vpBYS|3SLSMRA=#E09Xbu`)o!kct zOzAW_A?1C@di?%VhJxx6BDrD0 zh055pGbV*|>P3(DL_n2PZT(q8ESzv6)wMTS72+$t=%Ql@xCDP!CE%Od-xF6d8^6{I z4*&UC9Bou(KQ&{Da8Vvr|J~XX1=E%19uB7!(D5SnzUkpOlKsb9V8Rnjk2nId{`6Pz$fV~ubWdx}P;!jmcre^X;inEAasG~&)%Wxbv1x&9#R^yKK_jUa3)-512C zpun6=A8UxXb!(K}HY($HEKGd$jLDpBix<;~@Qb5Xt7J)&1L2aP=XpwvlA^k{9$-WC z(J;^1z~aRn$&C6{jleeC(4eHb(fM@O#mKLRzJ)Ged|1K`^7cl(8J%uUcXx}T-Yh+S z!i`rCBt6a6FPh)aDq_W&5Lqny9~CMWM0mAtt4v}z14|i~3oGHIN^39(9IUxPu5M)H#loIZ8I@uuT%LPUS|JVxCTcL#UF4CECo6+LzP| zJ{o*;c+utiu@Awre+`)*3-H5Bhu253^baU{6RAJN=}mHk^N3s7H+*-z4Pyx7Ugn=k zicWwFm)y)XekvE%g2tot4Xgt)kENzq;pSybw#)LicLQnj51H7 zO)9{AF(;c>*=OZDol!w&cFhlza*CffO9xi7!$eSW%=(rt4Bsa623TIXO!1HAx3Cy6 z(~17afP>c)cACnl=|+DSt!=MWUoClW z_~Y|*4bit3{_MB7{~TfT5dJp`$lyj)X^{yoC!38@Y1UWn?L`>$6-LAS^@|IZS}&U4 zBRWD3Da!`<*X!U19}vP$OFUFhrO|w6bLt>knuWAXpLXl@5%?=zw9b$Uo743;!ETWf z$v?0HeE$j0xok@|%rZaL;u{2+Mdd}x zrVm0s5{ufzSINmOTU%;oWoguy2|OoPB_XNyk;7lpAhlHzsp85ZhNosv(<*+DM>lgZ z(84)Y8qbAEWSezS%NpiLsBCIb`yTet;;IO0dJpv-kos;IT-?)rpv`HW7dAp9g zq3e4Sr+pBU+-s7<)XsU-(S!jZZ3Nb=;d|PTCkeur;0*e)>dAa|w3b!KN2Uru^#{_H zjSI|!QHgcK^eH7@yff_zB=s&6sg#U6dDOq-o)7ti^hjZD)3yD~95w#ZXuZ2BT%|9< z39u=mrO)MW;Ux|+fdb9hhrcD!sha(26q``uN6S%^a{l#T3D2M$M#*)O=WnKL zQ4yw1aXEGZP04OT88#gFMm&hu0Uyh!jlp}$CEjLDleT>O~yQB!oizTs-trAPb> zFJDxf;s@hMnn#NmBP;6#Kld1e1Z}cXkkGUWG0pYFuyL(t1UdKPa+%nDF}XnG8~A55 zb5q=knduYDe}&2pb5@P9IH=H=EYbYP$zgF=PE_XCG~-O}UiqxD@V&9-i~%G~{|>5| zH9Ps221nu){gqbUYrbl5@#eGseH~j92TuCX$?dJ_Q|+8m2FDoCM08w%NbmbPVq$^; z5M@Lr%o;gTclI_zI=fH>y%26|jaiL|7C|jOgwy5m(G;)Nt~9t_=4!I}OOOs)C`Q{5 z@Ma+NF|YO27d|x5aLM6i0(eSzgd8x5A0sUSl)g}VY3wbOn2)OXNKCX+j+0U3#Df{a7RHtktq3b{;wR?Ck^Y3Jjc>r&h?4GJgVw1nr^E!z|xBI+@{@;!numHU&FvmT5sM++y7L4F*Mhd}Z(>)LRf&>`4| z^Q4mk<5empDeKPsq=W68hcZ9yO0d8)m{GLM6n+Ya)#DVU1cFt~Y(vcc^wPt*!xbN) z5sENnGRg5iK=L6*XSb6qh;0ZW4b!JSd!vO7NF=bY*Xzx;_?%kgE!+yCBRGV(lVo2@ z$YtU%!E6sA2(N62Iwr4<>N4MXoQ>P8T>Une^pUPW^Xw<1ukx&-3=C3{;Cg)kJ7|-7 z($L}Nmj+<6Av(O5=EBYDzn4+)X)w3Uap9+@ywN&N8c$5Ra`f7N@{7Tx9JDfFgdtqZ*vfe>vtNEbTs?u$m#GZgGgL1A;Dq zRaXd%X}%v#TKbivM8sllJUTP=fUu0cHc`JdHNw3y7gKd1Epxn9?4cNv^Xv${(NAOj z|B?2VQB`(P+o)_&36(|>!A&VSpG^n)F-O?p34WgoStFR?R1Ox$5 zKv2qW-k{I(yzlwW7-x+0$HKkud(Ac1%x^T>X=?do+EC zl?OovJ0pB8;|ps$f*huSO#eNb(9W3$Gi~A!jonjiHJCfOz)_$T+TTE$I!&=yzDzmD z;Xrny84O)kw3WT;h(HL&e3Z%lI6ZvJ)T;+nUOih{e8-|O{9N669KpR3sS6D&4!03{J9B6k3Eeo=|<5fz`rZ7L>9Ht z8Kj4viN31t)Xiy#D}_8dRww5onjQA@9y#GVGB(5c4POkWu0lL)Q`+s)7MDt&-#uS;}=6=23RYMGpI8Ev1D$ zDRLLQ4)u^CXBB()Xiz73%~G$)}fe_WBSL zQ>Ob(it5DWeM&d{dG$Qg{#kEB0=S3w;Pyxe-Px!P>pE+b`R%^}C!A(?xi1e@9}rhX zSh$A^xCVUeMT`~^A;GAFM#DH(xS1?Pj$A#S5Mr5l z@JcA!Gt|g8LedO5A!UMcRevi5Iz?ydQX2{ceU2c9JUvp<`w;DY1`z6fv=TNf7(;e~ zdyMD;C`@Y*JC;};OJve<^*sKx99pM_bvDy7rdlhMmpA_&k`Ml3grnmXWof*BbpC@U zf**+FAN&9vj2|7qX4Dt%;CAt${_T+mQ@X}`J?WD0fN>fD6n-VVS?men(!vO6J$yb4 zQQ9HAO5r1D0;GNzOb0gV74MA0>3!S$LKfnL<674rMX_D05?@d(E-q$zW+sB*5L?M_ zP5xdNR*kwXl8T=i9h3ejI+|Ry(OF(j2|j~mxbd6}5kZvfQ5)J|3?9)@HGiZ$(D}}y zdm!owhxF%rX~=@uE@zBe>=6q7R=al4Ujl9U!sIvK_RCN95W(dd-a{9!5rV|E2*m=Z zD!qo$; zA?ytUlG0>dt9Ua{4kZlijqX0s&rN*oxT+Lk`}l3Vc^X1gr$H*T0U0a!M99ow_rE`q zlH}v{_xC=k@|IR_wKbM-Xf~4fmC5lcCFtPjFu&+vMUyh4EbNK2g{~r|)Na2a- z(YP%|vvg3?L1}_yzA;?)zl}yr3%hm=@5u_Q2w0sJ%3`gvhrLKDGBxWA?(PclAjjl) z>&;gn2;B=CT4^+c97R!Tr2b+7PT-P>$f;SuMqO8Jos5b3 zZB6H_qfUZ9c92Tk0;C<0_4rlS2Z%uBo?CDM6zwK{bd_XWLIQD;x7SPBI{&2=xyJLo zGr26sZApVspe*e2QBn_(-pT1~>Lp<&i6fHIB_~mMj8n16s`AVeW)4e-T}>otE}{YR=0;f_$u4i56rxW%YU&%mtwEppd{3N zSR5(YuGbN8Ni7#^V@p}vkxrxzp-XFP(n(YRq#DWEaUzw)Xf+-4>F3R9HZGKqTp7R01L31F3WF;IuM;d#eTy_x*F3Uw;_Au*#B4cQ<=7D$Zo%+(sIyYS8aBwF*-!Sj?2krh)v$z|Y=7 zzkmAxm;wB4uq#JIMM{qEF9}UbfV?NzreG%jE2i!W>(6g)ew|QI8yVc+)mBNo{G|s% zU((DNN@bln@h#^F0GXJNRtkEmkR}^MCz;H{Yxwm17b8pGBS`b+P=`96e*=&TVp@z~ ze2<}buhq{1X=RO6PDDdKGDKpWhUf}PROj-(wMEB)IYs2U3a~JamEa^GOTpp0Er2}v z=cU#zOmOzzw{;gvW;PjH0sX7+vjAb|5=o$=q-Bx5np(mek`Ls+QF`p%UNh8arul#^LWW}>vn$nyzr!w?zoToHbzs_lLe!yR zy(iAMGiGt@;MhRi5e>GC!B+4T{#pDpLIdYyq5#}hx3BSjB=`=5l51s57H+HlHg1>D z;+E}?inBmZJ3SqQJ%>lNL{o_SPY;-FOw`6KjTL5Z`qwdT;J*&1*g^ZrPP~3k=_utHVvzdQM-%{Blp)$O2YcqtQ$8p+p*@U zwx`6Vgbfv5<3r>!L~$AFlPVJ1p8g+>rP3el5R|^az|GLA9=+%137hcMU~nHHeFY3m zNQt}Yg5`&EogAKS#x#!b)5&T3Dsd=UFSZ-+mzL2Xt&3&Pc$+;pkKcnovL z=B%9%J-olAh~OahZgcmOB_kLO`WCCtiSo*Wr0Dv^R#nk)Yhnr~0b8m-!Kw!_2=}1Q zwo@W5P1iZgI8D@Ty%aW*W4CZ%)%B&KF$1E&AG7&pf7Zb{o^&f5w%?y!&vQ<{SJ(Bh#r2$&x{x>3I2Akx5 zN!(F7JlILc=0O!JTiKBVGfRr1oV4_fD=p_plZehpvm(zRJ%j4dcuYt|jA#(NQCkq` zve)%q%weSY`KOL#K^Q5;_xthF+^epvGZQ;&R)I)#u}UopuhKN1mINR1Pz=T-^+!}4 z+i!?$1RP=nq5uv4zHtIJin9)<`@49aTnk(3%k>BU3QzkD2liKcbQE+lK(_X}SP(lp zk^(Q~Z3wxWxVgF~zt`H!IIV1O^PNT)_8TvEtWcl;)I8>*j_-xUm&#uXn2~1FgRgTNrS497%DKeSU`ekWO{&}qb z3pibzXYH--9RPy%qmPdSk**!S@AT2bQ4n|UX84gYkTM{L;U0~}pzX`R69S$a4(^}R z?#w)?os9mq_BkwGVa)gUZmv!~X6W&yW8ypCUgny=44f263z}Sf_asL0Yu(F-UeAx5 z>ZloWN(nMp6`aXZ2($R|rf?`ng@Q)FLb)%7(g>R@dNaKzP4Wu-&U^Ui@eX$xA83cp zJOadxs43Y}8i~IOzn;WZ+;NQ!7kvIomi+_G&bXgnQzxZ_o<6(pW)YokrA36)G=IWB zm9S~Gi!00Ee1~6@pz2XD3+SoPL)zh7>ZOvn4>vsgt(=EGSR-vGEvIcXW*P-hD-# zR{CtugGSKmVTk{0M&%QrWkN{eNu+fXKaccCSnj@e^*|Hog22rN_~5yuRN={zd7e2oo_>4b2}7*dHHnnW!$sW}Ru7D$M33IK$>LD)`- ztc#Rsx*`%k-so8V`101ujB_eBHdeVkoNl&prG>)#_uDIGaW`9jy>5E8{k5?>Msl>4#tRoB!2S>A{HVg@37m3KRXqDI`fFD_0bLvnWosnF!`m)Tdx&8mpjLxZ%XkUS z%H9KQLGq%dww3d~cLh{05VWaa|5U8bXX(_u&>4`=*DKcVl-^#75i7AybB#ClTz*>l zR^RBYL~DlcUVW-dpLkw%T20!WNrOqBZAYBfTCa=5d@$yR^?)<+6}97bMvZPy#?-tP zq2oZ?l|}MkUN@GsE-D^tjtf?EyF{R_K*$qwuxFb1>`I?0lmq_|$7IpQa6Vs9@#)jz zp)9CEWJz&Qt0YxLjr+O~(`Gq2?bJ!M;7>D4*v3jOjq;N%0sAO31mm&N_~V^XZ+hSf zT2>~-hsvIbzB}KQ#FwjwACeX=mo|Ql_SxN<#r|v##>nih#|YWI>FD_^k{x69_KC=Jh>bsd^X#_|Z?X2@rZQ(1c6F=Cv1+4n><-aJs8}s45%l7rz;b zx9?UCK~-N?ya9FCT4Mif!fxF|g6+iH=6YnT*Q(5Wl7+i1+AFS6;`+!U*P+x1BVOd@pn&<=Ln#H{RRxRZ+#630)d4^W9zaAX$A8Ks+1F6n@^@ zu-1u3Pfu@d@sydD*7{~-s1RI#)Z|d{iu&p6ij`vSwIMY=6-&!pWB1v(N}Itf`DZ8T zBsv}k(R1E-{ZOT8V=m=p*U#^tW}l9j3g-SeDgE=>;Io@Hg*BH>inpx~3(p%m4Eu1J zOMxj1arW9ukAE+@y;NoSZR$Cpt&-r?g>>m(GZZfM@w3AY`kUh}5!E+8Nb~2#2Q@C& zj9K)hiswDZSe01qU{WYsdH2Lf)ygn*C4?>g{D{r)kTcv=*3@~=74{&7Kc2TYbx@_g zFGI#`Yh#&f`Tf(FW9Pgsr@PN3wMhV>lqKvGpU9fcf<#)oNL-y__}QeZwU1mgq`i9< z3m@>r6>xHLT1>ZvtimPjtgj`e{0`3)6_*NFoYA-~N)Ik`a0_rJ7PRf2BBGJElcoHRIgkKq6d&9gQa&|QXgQVK9 zg%kdX6u~L;IUn7uT%7|y0CS0XZ~Y@!Xiz1WnaWx3MWvknYU{IKy*Foy%UYuMU8bX;RZpZ~ z{{_M223H@;rPwyz=`&)tGMjc*yKT!FXQI{Yc1z{4&OdHYgksF8(j4N&cWZ++mcIzK z+8cS)$K6*G4$x7QGbDOdXdSh0oGvq?oS@vBynH3cQUDUy1Id!IBX!y73hTK<2wFmI zr*I;0@(EC}Woz_DRo}T=zosI+MIM|D=)&Td7!~h*_3!n-zM{I4n9F93Dq4v(yEEbH z{PKioB5##-zerMaQz(WONP=-(SCUIGoK&`XdEjtNl7(y;!P{nYR3So76kiukD^pA0 zopX)T79NfPiFVfKr`&A8SzNRmEq~m!IqB)#D?AcxWi_2l<}dJ> zT)ltICMY@$x!Mfxai$r3yb>U~J0N1wK`Q<^)kS@dqt_$1B3bf|Q`*|SdCIg8gaMRk zC8-t*w(+&PJoRR)o9CL>gN9V(MF z{bWUEYd%}s(9o{IZiA)w1<(;I*NksHKO*41Y9ScdJCn0g_`qZFNo}$dC!4{7%Cp~l z+bfSUZ*&)lE9A@UeUHd99v@UX=lE{Hfm3N=vCnmJ#BlNnM9XtOk_4?)Kli2ej>znM zyU!7kzh>i>|2f?gb4J)M_O!Mj`+|h-sP<#k0H)oEV)7yI&h|5`CXH2RGsAOV#Md~; zv+*qEdsb_F9(MK`&ezmz%A=`Ld~}B8&ZK+3yGHlqaz#7+)2jso!VaV7*OHyJc`^0s zYO51)3ZIq^?tYmRDt-gO3tO_3Hf>f+N=EXvjY5y?y!9N%GTx-?Ef=cBGI~E_SEJUF z)u*-cHh0n4E#kxkSrXs8vF2eE9pafD*CS@ zKu02o^L7-t2q-XIn^K2%x-2@xueU^rrlv>fW#!N@iw%FrxncyRBSzS!hwX}medyr0 za6|=LsU2A=tXE%;Wt%Vv{MreVVVV=RcqD9VSb7!Tl$ps4a%*(Y;*4ZVsc}Dk{D5VQwm}HzP3sdjqSqIL-LO+WXE9{VfKU_BBPG-2FS*2l1EAE_goKE7+ z(qKGas-Sgvwe4^oeE!*vb3H^$6x)vDb+?%n@-6V#&F<_Pe@;qZ=42@o9QnTX^d{ZN zo&!!i`dMvRLhPGTk%ZEWjoRPTkIm*>mLDQS97JtpRZ+3opRVoydEYz(u3TatS3VU8 zSqiP{IucVv%76iJX^(fnd=(ocMkNtE(85-d{9cSX^o4?;GVYk}-L1LB6{NP&u_l;j zC6}_f`80M+Lv+GoZjju8^H{yDp@c?vh3zo6is#q*{Iaa!2G>P(5oSk2AB-S1izrp{ zNtvBnYn3|K%H8Uc6f5x|au?2X-aFshvqnbe6tP3mPWO+V5J*V9{hFq7cVleCpxx+Z zb6f{w_Vw64YqbenrBCOn2{0ge*wm4S7|zSK`h_z!<}Z)x*!7FV#S1CEAL`WKN(XB5 zg$0lHh;ui)4%->L*m{?HFFw@p{akMf8^mT;*yRy2lZ#9BjS;04mA;KvTU2vn`n~qd zwOcuio+g#SvBw8{Py6^QVOPVLKm8Gz0C`7jh03uyk70Iwzb-L$y1#}P&b<$m{n~8( zWc@*=RHwy;k!VEuqMzsOxf>VfiuYk+qWJe%B>?qZQXTswb0+Y9iZl{7=d<-k#y2ie z16Yyc$%WnZVdpk7r8-<7M_S(O$7*lz=asLhjVvhBSTU*WzFWf?J;KSiJ8N`v7bksx zd+U&6#=s7chyzM{v1AdD^kvodBYjO@Aq=-`K*EL>4+w$!@lZLLMKy(XP=%#F;QZ_EownjIolu7 z2(o_qZdhgYXsLN~GS3W>tw$C)JUR;!yUkd{__id;=SN6yDW8;>M-PL0TZxnQ-g;v& z0zp+IJDIgY{>ro8lgg6d6NPv6;w8QmWIrU$wVwdTX93B~`IUOIG+pX5{MR2W_`cP< z(ut8=*5%7EZT1MU@~@Gs%{1=!5X7`T0Wmp^DU2W;`j3Q z^21p=$M5&W|0#7A5{`1KLnt92ay#@N<&}lORT8s!oR{R#bdas~A!gVA8YF*3GAAuu zh+d0ZQVpUn6AY>ZFmh2%)T`W5RpN_lvoKpro?xCh+d%!7_;geStIc2F^0}|lc~-UR z0?%^1%AA&gif_Bk@RD4zrE-c6c55a^(*h(~8lbc-1)=ZUQ}wK25}6&edlC;5Dc(fPFkzy3*9 z-Ws$gnE7yI_-qwIwxTS7ok#ZxUw|kgx~Sv|Ox!U43=GfkGMen@6sw9MZuLS#2s@*S zqEo8YGgOU}9lJUBaq1?WBf#Y9RnFC@vV-i}Dj37j8MLM#pMUH1BfUSq-wO>L;vsI0 zxRkkz4y$-#m;N!R{^}6x3Xn~E6Z`}qRHstn@qJvg5*SptdGv#xI6XISkSsagGX*w4 z2;q{4EH$)kJFP5+%66j@$MMBp(GySrv8_66TPV=W2?#c- zW=Pr~Z@RJ8OeDsZemCov$6Qa6VrHe|+akTZwj>_f1QocF7BR3)riI6IlA=UDc&!;e zM)Ep&1!hNmthRe$bv3JJe5ukav~}ZJpQPoR0+U?%gmC5h|{=@fI0_pSyoIqzwh3TygsL7o}f{@b2?O4$lmGdzxUppIB|}u zB2{BEizckqYN?)H-&4uiG%&Fw{KV>@*t~(SRV~-UaC-leS-@V&|Lvjn!vnUb!!4iWi zb&hj}C-CZSk)x;h{EZdZxJ2sXlB;)&_>H`Asb45kmy++ZN)$F_X-DyF=B@TZ&S~+% zdsEo5(`Rde&7j6139(*TNx=>&5K5kP>ERDqdA9Y1i|M(IBt&Mj{>SJe&335x}a31qMfVZ}oYW1J9rwXjLNQo;p$G|l;QcoY^02pl~hdc^V6J00&6p!974es8lOD80ohK{uAPo8gs6Qg z>%#(A&il5*{q6LgCBF>o3&bpw!~~UXj~0QIU&;G0T5)4`+@)XG%y)C*j+KewB*sW2 zE2hf`%rpOp-Ei5mhw!J^Pz{Ov`0e4%YFm*fv^Sgh8Nzt3mbWgM1hDkpf~nCD+tSzj zi8&L6ZHL-|)sNBN&2-o7{$y*#K3kx%nfdv(z;IbuCGFY%FoqPI?qa)$>ABTczZ^47C#h6Ugi7Y51?pU&rw^eUr1rvP-8y>WjIP?w(5BFEv{jKdO{cvg8&6C=JR*77 zn3HK?(m}rn!1hVC*b|Uurn&Rgb4d2)Sk{YwxDui^;>5!F*)v&1&0zFH{?_CP(eZ0@ zgLPL6OA=-B`|3Mk8WlN|rT0FcnHL)OnZ>nYyz`{!r~W`bnBfwMWo$|juB%KWnKh{_bWaaC zyCJR*`{j+w#^?sZ_cj@w0 zqvA8i>TYM-P;;3CS(SjZqM?l|&=nb8^$ntPOI*nRu1nv**UKAc=iB$gM&*)8<7+MV z8i+sJ1Ptb#8-uR@02+%T{l7FE=sn1{cBwA3vGxw=*4sJ;YbX$r5C3WPLev2`Y>8Rb z$VG-Jf5=FVUBx0^;PN5V2E!ApFTtLRgV{kG{_-Ng6~NzE5%t&2(s@-!q@owkd+K7x zimNAGzp9ak$-;nkSpwTu&_!&Qhl_FIDOB>2E)EIHWD6)EWu1E&L?U|bwiE1d7PK1w zQoo?ch@Q$jPIp}g5A~e#pHr1N0DDWMk^1YphF?%e#YveI1cgXMP)!n zPFb(=8Te~5!L>0W80vc$E!Cd0v>yI3^*S&P^yxdZY+iG6ozcmO#yg@kX~6mN^TpT2 zCJWoP^mthKTpuXt(Ad-3vSgOe_qzOOBhgfWEf>6_;9*Os{S5}jnBRf*zW5*O6({}1 zDZHQ?tRa5rv-UsQOez9paPzR!izlEVlz%xW;uSM})ek=LZMdxdmUcLaONF*<38JkY+Bsj`1wc{XJ z21JT{`oj|4bfQv;BjVWtVp+hqqFM5ONGGYke&gr2k8@0=NXKL5520yaxabpj)-`#R z9_%K~3sO0(;pG3gDL+{{So2vT0?J6}mEoBS0>cER&La(PnUf*Fu5N5%G@Mvk$rI}~ z9Smh}K=qc>StO?Q7j%Os4? zxuyNQu%u*8hAkr9C;bm5^JMKz5G`k2bo^-~4=D@(+0Zf368IRArWEYBKFmL=v zt>L0Mgsf+AmY@zzx*4>%8-S4~ z?@9k9Yx(p%t-P5ZzE9S=Kspbdm?pTPx`M~77FzlO4sBFJr8j74q_?z5W~Rr0uhCLr z{dxc4kjJ|4xo^Q?IIuy06cWi9Ny}t$^Sh$_G&%Q?-=Sf zpILr%Ocw^Bs>pW(UWh}GY5!~e00mwbm5f~z{?Vimnat=(IePjZZ6G)bdVj6WU$5=K zA(Ge|;)xohrY?ecqvtVdIf?6(lq~(Gh%TL7 zr1!WRePO*Ld#L4x5sk=}=49rpI#EDs-tT{hxbU_hzp+&pRs6!gn85=X__spKl*m~5 z>gB%}Gbkq#a3Q03VxJH%hFDI#VutHYau#pJ5S|1~Lz{y>TeEieh^-Y{*QopF4yq)= zwkKX`R<{hxr=<3ISvqrq(@XyU>L2{D){JA^BE^k95OzF zcM4zMYvZQD_XTI$dV4P0`fexQ(qK%&AEFjj+6k&_Y{8f#emw`)HIf8{BhrY?=vERs zjX)*-gpDvyHyG1$&4QGmj!8BIeJ6DuNue?$!3x^D+4Z>5jVX_v^kfpVqyZ)w_w(BaJRQntKfy6= zgTRn9gXG9$A{rLB9p<0???8I~eFQXzP{R1t_>nbR^wdlew+#_3rA(L2+jcQltNSil z_{}G@H8$F3*iI1KKpvPp=S>2!vW%aKW*MrO^iOZabn?f;=7qtz@k=Dn~& zJM%v4oF+A?EXqd((;zILf9Pl;%SCj^KHBRF^%Fw^T7n$--nNhX8|r_70uXcQc!C%H zk*}XD7OD*jZ1GFXX#7-?*H{(IM-Bt5>f3~4eRzb--G~QNLQm`(h`!HOB5`?EvpH4p zA;a|<=Us*o#jlA~4Ce?&Gf3#VA~MM2+0Na2K}y;oh}ShTOLI#;5Qdc~qjCG?i)o5m zL>djrmB-aQ6y!NYe+-r#@ZkR19RI5n&@yrTiSv>sV3s*s>oR{B>s?Ut1i*k^%%U>% zd9yb(A0l8(G!cHC%x`ii0kEgqY=GAKs9_ou!OhBt@-s~F!J~h53l?PqS3tOOi5M}W zlLXAf8Z2AB*B6scJm$Fh_}aC9ExmsM=tHQXJWdl$kmH$X`eB;=9A)r81pz#08XgoD z<)k+^pl@PA_dU2uilC0|8VlRA-|B($Gtx)bAIMe^=r(JyB)?Umom~v23Yh8-o2T44 zU9s;<6d0#+XHrcVwRNKIy{i7_KpjF2BFsRT5)|e~)-iWc0+))SmXAk>%F%~5KbrwM z)iPX}{1?mA-@!nq<$QEUc$({;9Jd z+ky`kL5`F;{xk+C#Z3QV8nYBm3@Kg#+~ZkpByc z;Kf~BD4lXmfQ(De=?f^4MuV1aF-D0ad>bVMs09L8P?gp>q;hu*`Twot-wd7ly42!Q z+#U?TJl@Llf1iLn?C-kL3Eym#a-4{r2eO=Jgv`N~IA z=bd7_FN;_6df&^) zX`r1ptsy8ocZGW8Fy6Lc=B3$2i!>A7tQZpi8w3#X`B9}|RVyLp@*>ES`$JqqlUkta zL)9BZtqa9>C*w1TS6d7Y#>xJDWaKgMQtp0Tk)+D3kzX!)FbEKk67IiHL5A%3S6JMR zd|MhqtCjaXNyH+~aZ?^r7Kx2>il`c|i9hb)bZu)g+!Qg$L2{V}ro;x5`#W@Z8yxK` zQQtaaIhTc%7)7@KpLKeHLe2A_&bcU`fSgX6|K+8=g+rAr^%vU<3}s7Drhk> zZv0TUVdqN-A>E(X_cxG$gbws?hGOnYg47MgEsNgDw%~3$vu&XhFF|Z%ECWUEDQ$gj zHzyTgYNdV4ofi7RC!f8I#xvI7-Q?u2z35n2{qD(S`RxNHazb*MGAr)X&BF1Fu=dHa zBu9~axrCCcMI~$(Pqe0tgh@Q_mTWxt&g_ao{iBYzq_;G$O)ib7e=eOkdB<1F&$DK6 zR1vdur)l0g`3@+~WYTRw@Ls-*s#jHSxN4->7oV8jWPI^bhs9sxv#)1=@aHT)bV6)@ zCFg3_J9xx1-9kbbh?3pxYifl63$FS;)o(qPCSl?EE)E71zd|6azUH6zmd(uNM8B{g^?#KT-lYkNaDw#(@8?9PY}mS&)7UoV zH8LwY3|8m14=$X9h&QhF%MUJcqu887L!u3Nikdw}5jBduIf_=Tw9!|A8P~*1z{;U$$-YoH?5j%<01(o$Gk>0OZ?cb)a%&scygNfe9 z=0Aw=Q&sO9x^1Y6Nf~o?c57c?X7xT7#o_h&9@XC-|H0XVi=wg4?iZrZWfr%+xxQ%f zaP+~z3_0Tj9L;|Gbv)0?pE(Bd4Ydk)E6>-I74!C;clNWP3rzI7$ODPb47fn z{A^JRg?ekvxy-L=((7bSeVO6cj3vO1RQF^9CC?~=;K>-PC z$9o4%^7Dl){t=JH#7WYQ!~O-9u9d+hOY-_`n?XkhJEgchCJge1gNLD4P!VgrZ5o>5;rqd%qsd^gfl8i*h{U29=Y@A`CuYJ7=DEI&|B`=a&ba z6ag+3+Y7tPn9s7P>!#fq;_Ny(Iwa-pN+>%^xvd%Xy*Isl98hr%02U#$pLMc7PuvSJz&ai;e%Ev;U^b{-Z^^5%%* zuPUQMXBR0)D%vw_3bb1KCO%%oI=6?Ik`kQu_v%n{3*+R{li0AIWns?qqeCH))_;T~ zj~Lg`L`A8RI=2}TNce^p5=EAHy0*rYOC!gSyhcno}T*J zA9aFnkQ=D3Rjf_fPGKMQziq z<*=(q|BjG}z%OZBX)i&+0W|aV9=Lj+``hOz=jsy<8<{hK@6%T{PXaE&{L1u zKV*a{3GenG&n}k7uDT)PUA|}CF(;JgDMs{t?f!m(d_xm$(kXT%W~E21L#;x_=g_}i z1UYM25cSyVJTNoac&N*p;v^u>Ag-`GS|tNnv!8dO=}=z6*}=I7$_}yHE(moF1x~%N z?qcxtZhf8naQ`Hb z!k6w%=guLZ6#D4DLWkI*17WKKj|R<$t_8b!vtLQruD=e0ZGiD6lpQ0jK0YZ~Pp6zZpPx&8fpkD*nOdJ3|`q#^filPDM*w`Xc{|UMyTyb&2 z{b`M^i>v@W)p&QKzWy6@##|GrpxT+olfH%tPFfdzKWIq3nn@+_#atGJqN+#S_<6Wc*Gj2jQJ?t zQv%6L0YmxSSH^H%5>)*XE!n>Ir`o{9rkU4`1IEQm<#^8;a^_xA+oek#G$sq~+Qq%K z3VXu=a-c+}vb`jgA5^i-*3p#G(=Ri16v;+mR8&H+OQTvX+g~LIH_g0%=>3A8OV2g{ zu^QErg;d_xI4(M_ad_DOf5t`M=ayLTX|i&a!*SJ zBvfdv4Iww4XQT_*b~&${XBxm`gbYkwn3NhJ;@n%J1O;mcaB;EzlW6Y^99u<5RiA_! z(z(ak6+f|aTw3dwNdPExdu+{c4QyLCfD3t`-BmAoNTr!9?Mr`V9L4xq-gnUf@5e2W zb>LxJc@?lkS7l$+r_RWv=4`!GRT+A>vvv4U%5gOb;TPIs{nW+(Yf+?JC>X8lP!mb5 zVbn#oj0w@3!f`_@%o_+-(=QX>g2t}*vq^Fz6y+C!nxZ!nSNfknzIun{BDYcgzLBs4 zn2w3}kGIYU^g{#sfa;15Pe-|3lG*#2o$B`9Hp^OT%;e4l3*esDj-m@|;ZhN3t3(4j zE{SX$FkRfb$ngfGQMXl!W4!YdQ$dL*X3tEMMI66)d_eFV1vxef3Niu`9K>;vLfi2v zkni(wKIJ$pIx<><@!ea?nC%hlk4|grQoZ=7oFT8v=QUWqQ9jS0g`$_^u^w|5%5{2wlBc`57Ckl45Xp(()7tKZ* z9|sVh>LOD*Yra+OG+|g&RMa!V*YY!~Kq!~77j~J?$e}{v$?c_W&Tgc?t_^&6AQ0wq z8&4U+A( z*Y1yzS)hf)h`Rtd)~9Q+gDHZtxy7i~i6hCDqXnQ7FO0L7%U_<1`4k~DY4Kd*_s&}H zX|)8azWE&E&;1#EaLrE~M3SBGKJ5L_|GZAu?C~Rdb#*m%HBQx-Nhw^tWbAc=W`^|K z58W5>s=8EERNDIb`u1iBAj;VFtzKTq<0`05Hy*>Uhw5)+wvM%9VI`VMz*Ae|Y9}t5 zQwOX5^-^H&WjZP91BmJ0pbl(s0GiZjFvNi&K+LoUaEp{FvZs-a6b+E z)Q(rayUMv+FrKY(q_|Kgzbo_N{VzRt-e@U>Y`mQ_Hgi^rd!TDGG)I{R_Lr^wtc!Xx zYEXqOjd}Bf@3Z=i*25&~%9uWJ1gRrFpLxGW?bGW=G0%3FZ`$Gt4q1xZvGu0s%>H(2 zSjdYV8GYJFm8PpY#BIoFZot6he$89ceYx&acu1{PPojAxf3Cg6Gt*E17m&2df{Lg?L5u`DB@RNC=+Z$pA zErAXg-hM&|*Zm{O32wY2{o2nU<-jfZ4I+elR9WXIGwv&tZ`j>4tifDCZljsO2Lw%t z(iP*|0k~Fxbf~{(5M6FqERCUcs(X%2lk%+Tj0KeLd+3S+UuVd|Rl87GPDVVlh4Oh!`I)&6y3yKXDkI+# zp6#$O*to;p5I%ZMLk`+zzPnGsp>$eg4CPhEsa|;x(@~g`{X4BPlFyspi)v`uSyf{a zU681@?a`B{V$9_+$X_`@Bf!j`ZdE$8qyuoSCoZO>Y2!O@9%O4CmuHiiWpXiMziy#p zd6Cgmo9yt5H~-PxKYx@VY}3Xv8U<%41y_LJGb!%6cE-T;X(I9BG0P#yyk+?kwp}t` zzMPgr3QBkJjt}pO-(9yWV=sWfAr1)~G$ZBn$(R)LKlMfb#R4!h({Sz8;d-+pq&ahQ zbMuO=%mBv)txn1QBd3hfF>IP5H%8SlK&!Av$SoS8aWN(9IUgXX+Eg6)_3b4|?>0bp zB7AqUIZoYKA5;n}9meT~4~?WPzAJm$WP}in7XJ{9Qp$XPemtf_ulad!3DYmNnF^3H zcK#Wd32yoBF(hYtoM&xC*y185y*Eujc<<9@54mzv)d1d5!qz}$BHK|SBmPs0w0DEM z%4Qv)uDc4fvS$DbFH~lWYn0j;&88-%Y5?wym8lM<>lH^r;V8D7r#4aDZ0qqdEKk65 zxI}wk>N_Jo~PO)fR_I*aGpUMh)Madlp^$xVQ4}N7}wzD0Lwb4twQH z_`f{?*plarq-uk<(@V4p=kjVl{JMtApMCy3l%N=a`c4Vrr3mNbXz+<=^%6=)d|kOU zp0PXIX#=c*`SKe|Ys*=B#CQ~>DDwuHWB3N|E%Bc^r13&VF}6S$J#UeVp14@#?6TlC z$9-uNaDAbncYOfj?5I*j2AA1t3aRNhyL9OT-C{6HAy3E@f&%r|tky18)|Ji?Eg2ZB zAyD2rr*nG03cAxJM-^pdy=W@Q8d#m})^Ou=On&@jeOR-sY9oxdL7CothO9p;#;QiU zQ&~_~JVn)x6K(4WE+hwn$svH~7^3u78LO8vtWt(VoM)5)R8@e><}nkq?mG(*>Ny$G zt2kTe3uifx0Bk31^d?xV8%z><&04d|K~fRTyH~rLD7SxI-zBR0GV}Wa`FVf#QwksBU|Wb3(Ro=EmWxDgFA@p%TO@B6Wn!)*sPiwLep@{ETI|FH1DX~ zP+P5Htt%O9hgV&aE793h-`efRyH(an%JS>1<2zZ?oBjo$J7$^wXJe{A?#@}SBQ&R)*AaS?~nYXFRD z1X2{jL#iCcNG7ce|6dT)Gp^!Q5)-%E>9VroYuwl7I!!&7sfY$`q0%*ecKLVzWLoQ( z?hoIyzZG0;F|)B+K(K6=EvGmbM=u{r%uRabJ+Gp z9#uJ0xMTLrHN*A&=Si2Z01EZ)GuTe;_h?Q(^%V!utPl1^n+spdK1?S@;ycKmkNdiF z)_&W5tj6(0sWA4}vpgAMXr3cChW?LVqNv#GGGivFRjf-~!X~5A#Eyo;|M^KqV4reC zW@O!yd$G!lKU@jz3KuyR8d+|4u$T1XM811U-NX328ke5T&5Y5fs(c)0o$jvqzS=g1 zJiV6u&4`&VPl{uWK`zxk7(2n(Ib^|%ix(Gfv)F)81b9AHBY;g{s|GoL+5$LDu1XB+ z+~QqQ;6>d(x-bA2Cui+40j}%b56Vzf?W9fx3YH^3mcR=E%t@kA=gqi$x@o;R7tPBVT=?9Jx!`Q@!o~<-RI2B#Dz6iF8U{5bhWjw zqmGvf-^1)#wSmeWy}$9n*VmxpdO$tl#QR<+_xPn!p^}`(ERKdI%EiZR*N&#O{G2An zL0&T5Fz_i4lV;bVZ!mx`IJJxQd0ZTR@9p;f?y5;J^-!s9h!mm4BHW{y#2exKX2U$= z)rPUPwug#b7vIgPf&Q8T335~f3z8I$i|(p3hE>EwdnkYtO;(aNcn2M>GKqg^by&Cu0y{joE^SU!IGeCt^| zE?4*dH7E0Yvymc01h{%J-@?~`@u%xO3|XpB;KTpNfp_P)eCuLsxIq7Efg#ou2XfI=UXzC4qfym3FbW9( zhoqs6IC{Uy5pKK^xHToama7YwMjwirG|F23_U4j_gKP0h%XXix{E*v_3R{SNz-kS| zb8mGMwTYiyWLpNd{A7q;Zso2=u_cdb)2ObU#QMOcSs*dQK!W=ArwwWM*t%);y%>=b znRIuj{9o&sJVi`+#fP!ACcVrrBV{+YMgXrS_Fnn7f2K}Oe#CR8|6xWEEu@EgY`d~M zXY`SaU@cHuccU-^A)inA_Ylm-yBZFv)RqiCS+sE?lsETwQ!ClAS<=4%`R^^~0$uE? zC{O8;G@+)f$Bc~=DS1ENbx`2!OOkvw9h^UQ7`e1U5dO^WD*M z8*>z0`^*$BCdaCth(<~TUO z4SzKs4aBO6&=vvJO?O-*Tx5QIYq6{w)@|K4XorDR_K9wxwnDY@TO_-KOz+Vnb6~sv zAI{zaDhjS^8CHfACP0qv{Vsj)6?h)fv|tL9AN z)S|7>5_H=g8brJ8nw`S#cJNbG)qJwbd0~>>JefFWN7YqUogIp0gwBZZZFjJ4fu`hk zx0JD26T`>${V5u=od|ALqm(g9mr|Bb7z+oSV)l;-rt;%Kc!}Fe4*B?7Bfi^=55@VEpalPEg0s#+jV)appVnM_A)#b!9aZU1a zL8M?b_R@m13fzadxg8)hZeRl9}u#m)5{ty#q4j%T|>%R;DIP z;a}}YA=Z~7BXjx5y4bO7X+nrE>eXkz?5h0?Jv5HGi#)*{-H;?5!@;DKLFiY6x7EWF2Abu$N_t|=# z+yR~i5i{x5vDd-|7&Be&xhfb5gD)(LQnO;JcSpnl|l zB8&Mb45=m}Xfh^FSk0--(Ts1LOjkBVcLkxl;c>0p0njhyL$I>)RtZR(wZDb2IVEE1cUmX{<>6#MDcn8HyW$ws3eC zQhHux&=d;xUNq__59!Ykv0mEaO3eJg* zb#x*dbc=P9r-p@84N2i{d(1r6y2|F3q|tKUKJ%i;lV$ zeebDrbD{?6%CJ43?&;la|})>;y0 z32<3Zm8@NDp`OZKTUU@Q_7sArrUOjblF6O(c|^hx!9x05yPHGU$dgV^u?t4Rvl0v< zA4ihC^aBd#E&cQn;Q}s?FVBRXpXbdyVF;KFUYe6wPB(pYw3ed2oVF&VF`?WsfW9+U zk)GB#60rzN$3^uXiN!``d(@O`=|ek{F0jv-03a)1X1G-iH|f833HCM)#9?k@{3#3A zKD8pokjK;RkJGPGW4ZuYiN0UJ=iY{<7E!pXWOog}oV8-z?(T>j2a$Qt2k0mcPdSg& zS+i(QYpvSXxEWKhu4pEYi9JLmljo-0)_y5uWhxgJEN>O~d)EBFj@S`QA}{CV{B#X` zAsnT;U8)WuJG?)~`^F*FG9@EK*%p-nW#dGj@PuNBYEQaWrGvb@{EwBVzMEEODSq)G zg3N(;xR;Yc3_xgs0!f{F9OjVCt1p>;_NCdb;`=p6?j8vM+1=kCCLnsT!R z)7SqUk>Qkqhuu0+u4jODD$@QuY=yKw85oXu;b><7=Gb8Q`O~r*fl7KV(C~AQ&nfRv z2WXPa27^_eIJB5R?)r{$SB>}3hQYcwJP2B<8}=mhMU!rSTStaTf%k4jB5*>I@A$5*2Qclld4BLt|kIE zU?-FbXQ$Ee3Qbfunf`W~E@7(O0?LZUn8^p6^PKu0F=^W+s^8s~5E21uDj+G4?heuRD_AU+j4psjVPQFGvPw)%}5Ig($DojE&n3y@a-8r{z+1c0| zc?Y#y>WB@~mC^8*L8j~PI1ut3lEw%pC_?URtNmcxgj0* zuEjpifrjqS}N@N>d%h)g*3=M$|plV9F6EX!b0-;8&V zCOM`4bL!x_Ekm8wtz;d^3FNBOgi9s%3d2N5kYKV8;*Dvuoh)<9_8(yp7AP6N@HCw1 zfXQP{1w$u7J!|+poK70Hj*5on*Gnfq>$smaVk1e}OsQcAS3h#w`d)ea*^HDq-ZOG7xQnGc2xb3d^HpK3|Nx4F}6HBPA~h)>FYTTz8?f(c%i zRO6%$*GlZ8au&WHkuO;@Iu@+1+Lk$zXkIA5Aq0H<$4j;ewWx7o)#E-=z(d8( zjlW*T5aMQFYgBC{&|x&3F0tKIAXYem%awds_u}-JT{x8u65IEeCBGq5hYnl2WN@); zl%|1`9R3x{GT46u{&@Mu7}bEbC7GZ+pP-Q4u0sbib;w-tW}RcO^6uSMK(_>bC;Jk~0+ z5&kyr7;44#Ta$|(F6*!Tt&gaufdTox?J4vDQ&wOn^jd{^Ft<8`p&kFZbh=V%Nry8$ zknh(t(N<%sPGq-b+36MTy(D9E3~*JAt=%CFn&;p6gj4yPdu-m2WpyVnl|+Y}3G^{b z1G{$|Us)sI?^-&u`?80QPYNJr-g)jHcLQT|3y>w6G(e+rzAx#EldF3P{np1neqAs# zUT!ym{$OT}`T;-LTeT`!PKi01)V5fc6_hHrD1kZ=pPGY&v3PoS?b(sWV6Fxv#wy?W z!^5sFJ#~V)W2@Q-#**RZ85SChGor`4OAv)PXYJ_CF-=+l*En;~*PK7=u3D@fXlfri z(zcPr3jK1w;R*~k9i}N-J(P7O=a7~!QvXl^`0YpSxe5EKW~mg0Au=2i%aiDG5EuxY zXkIu`QgFC%uBzlv5Lj+HK)>)*kQwEneOzd+#rTUgdMe6Mnet8~O6rp+&mB(Czz z=kPfpPrd-;%&h51^VeX)CH-BsXg46GB`2XeppuM4%&`ZLp(ycuGn_J24 zhW!DF23=&f^D()#pV5k;3Y)*GLC>b14)Q&~ZX>I7{wOmyA|TW92~jL1GvVZRa24OY zp&FaV=l)C`*}yfYw+QRWd-ByU?eDR=eX4r8+^9;EsC;fobNb;}vadQiyz=<+CUfwZ z6)EFpm-EnDuRXK!IC6)buQsaw4fBGq^Ja37W*`0Cu&|w#QK_?&RoM}TN(;j^EPaEI zjHT~(OC35vZ?75iAxPsUxdc!c%pH8DyJ~eyrWHa3fAG+MlD1Df^3JiUh;YSaVQ4tM?n+kWmPP@#IxD^N!#!d&-Nc1=?Ng0 z6u1FKHac&8gXr64VfDrjC9CK>xJqfGTW*c6X2&{B<2p}oT%GOeU5#^Ed7px(96(_f z%Dd^~C=h(LiJ#i4T43z%>bCWpF`v1gf&T5kETrOk)2##`yN;Pu=T$Y_DjVgelf$;j zLx+yN-)PNKwx=z4a;|Q|#jkVyyu_WmU(PbOWoB?B)t9U{>&IfUP$-D z9Y=_hT*B3D^5wzuN{nc34?o3RHGaP^b9l+sN24WyyCdt#qiE|261q8E*JkM-VOX(4 zT&J)dPWUk<8Q1c*VG-+ziN6haN!;BW{|oL=6xx8(sfisSf3->XO1>aOeGc@-I(ZlYiBpcL!l8o=w2%#^LV{pfJ8Fa zog{3v&3)4GmCo|uw)u8Bfx5E{k@M8~+KZ9=y}*S;0}z2uNp)9BPd;{jxNKoOEG42- zV#4l=5pgXPMhT}~=C0%zSE6*;5GHzEn4PdF;v;3c0vA|wcKlS0wvfAB1bN(R#AGi@8p^D_O ziNp;K_)cv~ZTfiPdh>u_+c%kW3-{cwVXH4S8C==ML7tE!wHAQYEBn4 z%I8OK;>Hp3?-iTmN(A*Wvn^8emlSLAOv|Th!61h_iLwU6>D7vtQRDlCwt`2u2whD;Qx zXn#6d4wpv;!LC`OS^Q#@ZTS&<=74Qji1qzx4Qe@d$QZ63WD2oRTf0-+P{gRIb~MGx z=7?RRQ7LW2 zrQx<2iepA*Ac>1pJXYGGIK^Gl{>$dn;+$QF%D!*irkT+(aw=lNWzAnHb3Um7$`bam z&HZfVgb@5imP2QVUP#?n{5-39r5IAn(TbhrT@)Um9@QqWc($lnNE}m2yiZ|X&s{+= zRV^E;96>(@HJ_4T%hF}IzPzXom&|;1*+SR{s&cw7d01{aoy96{2N3<{~=&}Cg|~Q zgii8=farWx^+07Tle0Q0=LEk)7hqtT`f9JHcy$6FcPhj!27lD(^(9oi1X~bv17Z%4PDk_S~(c-0}qibzpuU`p`S5W=3 zGaD_GUAhKE^B1+Aot`i0^Owgn)oCPQ@T_P*-XjxyMp;qVVN^ zumH3C=RdaRelRdIaqfK~kxCj;JAbC*p^G{-l47XajJ-7WaaU8{cc%EP}tW*zi=`iPB(cD9QKV8>y!_n z90c9N>-$aoHWB3g8}W6v*1+=8CMA7KpK-h%f~8JHt&|aZ#fVKhQJCy>dqKlI_NRKM zDJS#(ZVriE*vH%@~qkjx+NM zYy_bujfn*oOP*i1>L!wj6DD9Gx(vm&mZ2RH*coDTKp|HKDu8QHSGx_rCB`bqjDv)CAqv%X1JCDV09hJodl zm|0Da&)SdC3cBrFGG8FIMqq4(uwo)h+4?m+=j71l7tYWSjHDAJIJg)-r3YivXDk>{ z;H)O?zZgF6`p(Q>D~%NGp;IW>`FP<;{*SW01#`;WS{l?Q6`&s*JSl5ZbVE68568?! z5(T}-=c(pe6iqLcnkegABh;!M>l?wkqQxN|>$a4x8b(Eczh1aqvoMPxk{jMvfmF7< zmMd_cv9qB}T2Bx1p!9Mu$|-is;{w*MQ^E4B3I>rVA34szFT5|u@2yH<$%={p zaf~6?U%`*@atn>m9;g+%1Z1i174x08^yROQ_q&cK&SC~U6Yr70G9%Sw7%R$<|LB+e zt@uoYiG;wACSxgyxUA?_giBS4u2KE5(?Tz7Zsp_E$nKP?XQHHR>ex_+Q&E?%*htklhV%CNBgE*@ z8obrKG(-P?y`F@-cBbm7VQXroVIL8oNpT93Sjn~kd9(jlrz!NOyGQd={sw6TxTybb zRg=K8qT}aW>~;YL#iq#`mcrL{zrnvn;(UmkX=!HLQ zwu%SjBU|FO9eKR|t-Z}uN|t<7j3?qe6r&?<)xcP|tw${;>jb&yZ<`y_%#&qK3r*Bm zjaxXj1BZtw^UgfKW+`VSxV0ps?C3CUC_|B^`^>yi)##+=i61%&4~vah^KA2mmfVt` z5{4&hbQj8SE+TDWarcc67Na$CB!-K#zo2FdD7WI;U1_fI&}4Kdl@zM6!dlC$AG z-)vzOSbc=gm+E+g9B&{V15aLe+`-+jyI2^qsr;@qfQrta5^h}nY<1AnV&9Tv-_3~u zo{X6_#=&9SPkqM@m3<@5RbP@mbz$OJPOUg|XgtxX?Yu{^YMg>b+`5#JMyG_KS{#E% zjV50`a4ba&AOELtd&e8fBp~sLj>1bZgL}jscO~iJlh+I!DpbHV~EaD__q(X26|eRNMpGSbdFCjp7o?e@TQt2M*}%TS;aUPNn-RJ?TCtl}l5^ z*R7|p_C9NouH7QBb6MOjZMXQg09o4U9Ga3mJ{^r&yx&v20Wf@twQDN_3vjoz`-r!B` z{jG!3pqN~f$fcjx{iO=4_d|h4BhkQ5+7{JpPn=z3-~+AboaoICde%JU*4$~+fh!hK zIf(W-D9*KxL=2AIjDuH>GNnp|t9DkT44OxYt%Mzcv(gr>VIk2Y@!~%UuRr4D;Nr#= zvxSp;DK1DyIHgm=rQY}uj!pbmgNyxf%n^ZhAhp@r7U&xv^0BZ!5)X{kGldR%$15Dk z`f{!ny_xsk;r-F^3MANk>%}1#k_nzdk3k~rxt?7v(PoJ*63Q!=Lt$-EE}R2mB>}aQ z_lGV%6OJLBoKsKw<3l)AZC&>J=2up8={f!0J3>RrK3cKpFq7FGZ7L15i|h*+2=wKB zBH*LJZ#;j5lZXSl@fX2Wy*tV8*@8t9AS>k9=78fHS#zKH`lUQ zwTalaEOc0%c!-X=j_=8j=W{ICL33^A3NRv?;3>jcA%x5X0W^3#?30E0eEjR_HbZ;L z!>H_ayN{JGz*CXha;$X|L$f>Ez_n~Pk^$s3qH}1G@mT0=@@MH<<+l4mH|diK!*QH+ z^+Z>`{*_W-4d>49vWo@&DM4S&5RWQze zNM=X^5P<~B(WdKE45d?iqi^%lmUgfIa8Q5hTs5VO&5nZ zS&$LrrL!j^-K`M!Ug$3TfXr7GwwfR^7|3djej@&z+hS=&w_V02iqiQnLAo^b+b#Ye=quNxfpp z>F%PAjdE(CD8qd;+@eZj(>OUUGpANE(A+r`w{7Wk$-#hU(r#>A&8mF_5^j0OF&{vO zil51`)~XiVhacAJcR$r}-SHKhYo>A!#fNU{o$-MfV?OV?e^})Xh-AWVo>vzab)Q>A zgl`;FPZ+mo(09gE&+NF}(Y!Z)t1<(s6=&qe>%(Os{0Q zk3)abEXW@M@tEk)4&;m<22cCOzvn!<8_+RF`T zvz#U`TYshXhAP@g?r-YTcIm^RIc;d0mgQ}z<5D01`(b6~aP$a-c-)d8Fs*!!yL}WV z(e*CcGtLycH;Kb@ImXZ|Q2%LjJZx-p0K}J)HUbx_3e-0=J>wPLh`?O`W37K_4KF@l z9$9>@@J&3%PwWUWHS|_#l%f~Z<@!~}P^y~qZ*QYxlfzI zj(l9oX9FvMo0b1;q&YUsE8;qv$Kd(~rSq9ydWE%;j{X*N*b3~D4pSt_` zx}&r8mXwXsrU$(~4L*+vabhJE1#G-X2EO3DJd_xBSgkCCk`y1krDTUX=3f38A}u6y zy6JpkIbv2@-SJC3D!oiLpKteP-$M6nT$PGC>Fczb85TVO zSRgqC)q{7{P4${PN`O)vmoVhq2;n94%FAE?!RE>8GJLnM+V9P7SwTQB^yo<2k-X3G zn$3cvSww3`W3p1nRR_C59!vUtG0{}TP_8Q%;)euk;OF2KhA!M?H$a^x*}fJu9?anG z+_R)*2+t0djc_w{8K7>T8={rUMoMNr^rf&vJww1>b1EQPN*0beB0yH}v7-t{OTxFu z1mY%@XC-4p z-LJN(-*O8W)p}YfL17&YIU6qV7!9%O4w%12)K^q%4tDUjt&bB&+tYAMThV2b6NT;w zPFM$vj!GZ@{$!sy!vXLiBAt76ik*F+Qy$0F z3DjKDy2xiQX2=VGz1=zi%=MPYPc_ccPkLsnCltB`M48IwyvrD;?JAik_N2ymb*hL; z2N3SrYU>LNg&@t{E&)T_PE{Fq%;Wo2N0cB;JF(Hpk^LfREPGkmc{1f&#K$A8+nH=h z!gsRe^-ACIrdgPkuYRulk-=dyELJLBI1{x@6)N7T&fXU-N->lcV;Q7mNk1kL)b7&a z8?pCp3~JHT&YCpk8_|3@X3hC33B?l?ZAZg$i=QY`%_w^5k|4B$^DDr|;AHq0ExFy{ zw7U&R26i-~eci)R;A-iX(7A1WMR{18#`uNhlpDTdo5(a-Gn-^g(AGGJUIVesFzn+3 zGs{4DIJ`V-oKcfwT7KDGZojO&m;Q;0mRm7yyK{_kM1}jDhs^ZKkBx}$-ctA~;eHc4 z_p0@#3AUEtL5;}NnPlsrz>vM;Vosf*HtkBbVH$@|en@mQW>m$(y1_>vrR6l@YolJO zV7Sf55d?QYr$P<~-65SPkYE79eesJAoe&r;5ys_ z_~SXkpaWDIndC``iCY1KVscW_i_v0+?6oDEm~w*;9jYu*@geOY--qdMr)6aX1Ei}G zvS^umvW^G?sY@8ext8R`vPzz%(ghWrZkR7L?ENr;DzO$b={p!!gj#Ri^@c*B_&ygZ zl3wT{XtLLxMg@1x(?qwTpAXfJ=2=GE{Q()TkxbGglYOBJL#BtSMaC<1X@%CjP+!z+0GHxGy6JX#~-Jm}gZm?D5O2%jKje}}K_eAU)8WFTf-T99~4E|0r#P_YGXVQJC)7#v#sh$cR9BrSzHa5@;XFMN zT4o~Csl8*(7+PeKQL)R{Jpg~0Q>PA#SIB9OlpQM7Xd`lHJIQhfc-0QglhE|JB;nM@ zob)`R&U<%xFE~F=0E_aAZ%&bV>EjOK`egn`B0C%N=j7yptIQ}|p7PzEJOO`BD&axc z@ekEZ9b_k z2+?CYNU?i!rW;=t^rNN9yJ-Les!g~O1X?oHpR)gcDqisrq23BmTpuscj z&e;n|dJynK37ODO+eJ#NlXDcA0hNnpYl{8?;)2k~=YsF@6qb^#Db5NlCN@yT_ER0& zPw=aW?AkC{z+mIQ{1C4!9B?RjtR%wV6B}kA#&+J&a*vmHy~r>p*Nt(w%b!7_D1AW< zK4o*t>Rj$RPd*$uHL`G!hW>vO(l;I25Y-bUIUsIH&=?Y}zG?U2=o$-{w*A*idB7Sv z0b;I8UYIxEoQm=Yr(!zaDDT9(Ty-B(@|6zZ-w1YB9F@mx%4t44J=!BrN;tJC|N2M# zJ2bp^{ViDMlZSwXM(T5HB+E9UhvDxt#4*qTj`m(>`2+9?7-a^u|Mm|~NYG^xdvVX; z5Rs72^Iw5%!J-oD+n4`uU~s~4+RE)lHT{k-+~G~Vih&H}he?Jyo_Pvo3+Lt+LJCs`NER*t0(8&cc~xsuHtB1N%T#Se(AiT zFFZ{J&<2)#1e1A0V^ddRLfpnmx=t>=d3bs=-XwntTj7WKg;Nuv<^<x0aX@&jPf=2D#N`p&~?Z$^h%MY|BLq6uez2B4NU`tQrhjCAy z>MauGQ#62Z7}0a&UX8C=w*RrD2KI`)>*sr6uQgz=hqSYRzDtNY`k%daE`E?pg-pW_ z#arn38{Bx>fv6IjWzQuMBa6O-TtOyG0vX!9XK-S8nQbY_Vy5l#G}i+(*v>p))fOCa zD!}xMpRNSJW7qd)fHsMx0t|b93sX8!PLdNJ9-=Q={GW?XqGGp1&2bFUNr0T;8So5Pt~7Uj&U_y)QGKylpbZPI z;0%}d-h3w5y4dJ0*wqZ+F93Eqe#qM@{`aF7qK*VAN!J~LM)SRR`A3;P*na%1hwU;B zcQUTZrPOku?w?*hEU`ci_4^_J#|_8i)EA;=#5jfrGOG3cXg`p6i=?O3E^;ezD|=ls zk^}XD9uDL1%+F@o$o?%kw^P89^$5oefXKvq^=C=EpI@KEYtYUq3WXq>vMYeevzGcA z_a$g-Q0`-0 zG?R^`e|22nDVZlMOFbj`jq&x67q&<0W0IZFd})_zXYY zPpcc}T>Nmw1XRc;C<7K!F6pIiZNIK1EdBE^dOP3a>nVb4olp|~1(opTIe1G%eRb4m zUE1-Fpe)2;qs;w=mr-#{Odm;( z=8Yt>uFgELu7@X#$z2{4H~Df)vkD!hdgvNF@|P(8vJx2B1+>8TFzeUE0iU#F|B;*d zy~F9189XN)YWYSw%`vyNPN0H@77nW{wA3_?tAq0ta^iS$;&Q-+*v%JsP(QV4C`E66XfD>y1%Fk1LgsmJ^8@ z2R%2^#DP)d9x9C392<0oS8EIxo1i32l`|vE+;f6FnK8(A=8xl;OAr*AEqbY&gG?b& zALCf|K>79=F>(vosMIdt~E+ z4~&2Lce+sU7MmvX6c+DP+qQNW-~7p_TT3F}*~6*T1%2Fqe*U^N%WUjG)ugXXzw4yX zS^{l8 ztTB3(zB>DxKWTKaSbcAV|GG@%<%3VNR%Co=e)y)*ob{n^fguup0OlJ25RxkQG2-kFxtaKq+n~4~0eqS)%l<`k1{{$q z=*RQFI3nBA9PP^1s0dGxAMWtsoxNj21FQ-HKjL}rf8d{`%YBYRd~%6>nr*SOJf+Qx z0~31rT>R}7uo(Ghz|m4#PbRl>Iu7rl(;KdSTnw@=tr4D|h|;+4z7SB#c{HbZbP34p zH=RdJ7Zv+GG$8M$G7ma#0KFY(7Yni7jV6N(Al$dn*!<$jq-U_s~dv#!Aoth^}peL>T*~Vx^P=D^^u-c|H;iq_o6yIg4 zi>iSJNRS3f^D0_3xy5tL!1&@vOS1r9xrm&cg+p;Ve{ zX8j7kcYW2km*k!La4XO%%5PA?L;>+c-4}E{&Rh`CVkc2j8`xmkw14c1lc(0rpEVtQ z5k-uIruTvD*cm!~dPoS6^)&P}H~t3;5LYMxWl!xdf`Q3mn6jh8ZA|Q}fo(ZL09D?3 z8R+8R34Gh)Pq@REVzle#Z^UrU^_Tq0N2ORde~NmMtq)==i>ABG7Y^YO5kQ9)qNV%x zyMJ_4i_g+e3;D*@nby|rufVXvK6?D)3z`h&Ai~nim2#ZnPhWatPA5jsI~*0|G=ru}?R^0{;#r7}R;8`x~&PbZID@M4;+`@T!cfFQTLJtXgAaxl?X}4)CRa z;rza5{~1vddUc(E*TgVY(|lWsI5jl2om~8P)5rgJ8womC`U>SlDp9}Um`?IvSh-9= z+VXgn!o4*g?#^Z^#=S)qtd)X zu0u$Oaco-Zk|}P3eu%C|BfK+&~(4O zA3Rv>a&pxr3`i{A5c>F8=NBgma*~q1zeO6);6~Qh54Bv3mFPJ6zY%69aiv%B9AuVyMrDkYL}y()lS_VT%^0YN-E z>#7dA4q8Up%U^9^SIun2W0by_p{-`cgf4e83#aa^a}izG`#30l&keeP@i1WT`hUJe zXns0#S|BbLalhSSXD9G6VVD(ZY2rl*L>hyJ&pEB*)Mq5HWREgSZ@RQilHdLjHe8eQ z(`ceH4C#ktGtQ|^L7@?5HOD5-n)9^$vJEn$qih?IlAPorFFfOkf-auK+(+;P-J&Em z&~Gs9#UD@C>8%e==tL~^IKf-thFn)?iYJKNhwq5^UAvk6Bgq1cvDR~x^$5cmjz=rT zxpt-GvC)x~uc@R1D-xX+E78mYS;~a6*qFez#rsY&62iWgq{O~E^x;e}DtmON`U+x@ zDX4PlXuvT-E0gp7`0XDGK=;3puDF=xM4;a|ogj%AutxHj4LuzlaLbI#iSm?U?XV3Q_($YMeukWlT!}wB{6wv|VZ_X% zVsu=w)S(pla9~9mfa$}UH{1s8#^$|e=jXPASZmMUG_EB@pjKo+BQ?i)qA$8tPgBUR`SesUJB4fC#_p_LQ5# zD;QffPQ<%Imvb^WQXX_|3+vV?onM*|+UiFDrSO_Rbx2XKPQO7_G z1i=o7zM2m;Lb(NU#P{sC*G=}ny+_9zUg1|uLu27gw~uZ3SygazRe-OdW-6Q2J>!BR z!_>nAskQWbP@T7kdg~4c=Ed@D<~U17nu>}BXS~W-Ohxs?)5pS=i?QH~oR;%k{37UVC|~<4*R7ftFtean zCd=0V5k!?>TF~?1_wfo`3f)KKf%>A=!_;31a+dOsY!uJM%ILPw4&PiJggF|0ju_qO zxcpF{cUE_K zoYr$_c^uYDPfrF+2mWcQUu7&0%PIWbgZ7kL<`*ePq6ooTv0kA6TOO!`ozam3=N{bG z5?R@f;9Q7|s8-DR+1ZSXW1v-6Q+6m@tBp#O4a(9H9KgQ?@}TsG*61%$&98Xd5;c23 zc{j)L{$A4&qEZbq7s~6J^U3og<|MmU29&Amm1R?s2I(kX?8(W)ZCh}*+fAAn7gS#X zusd|TU35B)%J{vg^`l%;6Oc$Cz&IAsk3e@P;_W>FFJ`+C2EpgdXV>DY*^$@w+~uy6 zy{yDju-Z9rNms*MCc(JPBjc;5|K3caV|av|;RwT8|b z*EsvC%!yX$6V<=DAucE6!IgV1S=}hiXi%>(tH8%l=?}mfk({fxEAFkJ5J!0QLy~L^ zmAujV`5RChK8MLJ92x`3^o`ayw@W+;)PEAX;vI8~JM3c{u^wxJU!TQXeZkof+5=IN zVK+}Q13PFdAuk0Y8XCArjeX;dSl!vQ6UZ`cTx(;!PV4U_7=c-rX3+>za!9n&iD+|YMe zH{K|Fk}Q8qY1++RRgIfnj9^^$=HEmhwJ-)3JDe06=~#1iJ_yy|53@&kjE8KpQmmb& zZQSl)?i~S*TAayZ7vD_)csJu^kZO8n6h+JpKOav*4DF=jI>|SW(Mh5xfw3zwJ3J1@ z+?(tpUzqw>h0cZ1hqn|a7-35GTdVof3Mr*NkNLH;8L`Q^JVwJ460cAeyK^x$Q;zl# zd5_VZ34c zsNy8*RKe_+lnMoPb885AOqSw=7w!oR!xe^bF5Kt9^M(e>;4}!=`Cnp2!F6f40EYRg zU@gqdSU9GJ4Rs!7L26K-M_Yf*-JW}r9o*Uu?5(H5kplt*kFA%7YXVarf|L<&G6Up! z4;`Gh>q4B4Q3@omoSo`iwVf?YW3WnZp zl!h3Cu=BzQ;JsGgaNDtJy$7t6YF$pJxlS|D4;9Z9KwKml1Rhx|8SDZflBX+(0?A{D z{_BohOD4Rb=ZGxz5O*UV8FLYmK;G&y#=0ZK(Mc!)*GRFqr^1RKI@Otu`)UA9I3dX- zWDoI>y$U*twf~rPmx@oc-A;ZQ-i=Y8$73s)A*{Cj&6|7ClP?i_F9 z#c0OdyY{9}5y8*%7QYm3ivW%Ji*Gc~V1(T_;``yRaa$6c7s8&P%oCYx{OR)wG^`sS z9NGf7(Ek6w-SPy`uF{(C4BF=Zmiw@TgLITZX((ztt^fAg;q2PU9bI@}oQ}p}y>e}j z>_`Gr?fLaLjyPrX(*&un23iNyK6I6}DLbMzEl7`sbW9$Ayb0c4s{!{{2gnFFM+o}* zf}aq;09DV{Vw^|Y9S!r($qckL9e{dZ{<9e6GpfCn0S1m(FiNtGbIR=@qecmv9-jJZ zpaHJBvfwc;%@MGrBhE(1cphg;q|ps;hi!~xbuYJ-+f%L(v-2 zH2R`@-6y9mSay4^R7y}>9>cTNS-guU&xlrPU!cEuu#xg_+k6_(?1F$7A`Bp(v?iTq z32epCb8L%@d!HZeZ!-In+lUWt7NCIurB+rsm4GWJT`4pC5DrG&7yzC)6d1c*4mueM z1egePs{yl%JU~4z_k0x@%#B;Q?EE=4L-EJ#*0ZHK&W}YR(F+D`*5u)M&unfw? z6o+%;V;#*{yp#8!{;}F^HERPw=ELtJYQEa`-Nyu@JL1nh($msf5E4qMY?ruc(me%K zCOyRC&#N_B29Rk>-*xW%ecd-&NWcaIJ0A%{nrR=2G#moP;rXS+pl~q7kYSk^5X0UF zG++{bFN4c8RD7U-?I9P;?&R*OJpmQRHR}LSOJDqBh_(m=7GZ)57CSY3oKdA+#|N*+0o>eGjH4i4w4H|%~`{r3n3 z(rg6qkorVlzRn<=7oxTRHD~0c>(0`BB(EdK*ms#&KDDM#_a1_g@a%DD$}ALMBD@xm z;8qHHjda`(lTQHOi+r_pi~TdbDPBT{Z61nTD6A<;c7tytSk@$wh(7_60L|!Oj?D_0*wJ&a2O> zo+s3Wob6lP2MxTwY-8gWs)~n<=;w48Alb9rEqERZsHIxNH7u7kUX@~x0umsi*itZw zb_B3%Dy(dydA>BFEDBm_=_ul|efmX9;e8fM=n1zr5d-UYch1;5?3R-w+5o%hEyqus z=1F>K=f}0@_!dh%Ru!E+fD4b29k3M-Srx=9WMCbDA&Uluog5(noReGw3LVxiv9XDZ zw7@v;H%9GDQ!}F)?g=+m8}Kd}U`g+(@K$i2<8y?ax)k(QDo4HFxkOAf9sAjr~WQ(^2+(OZkFRZG5`F$RdK91uOB(iI?u>@j{ZN9dV+fSA_}u-wRJtFwc*G9E&U~ztNS3d6N2$`mf;AA$IEf) zWJK^ds*(lYQm`4ninm^F-CpPn`N&+uWy#BEuk1*mvAkIfFpWqsz%`|!AF$TZlW4vX z9(O`hmEGfd6ThsnsCj>*4R}ARoom>*Qo8+003p^GcEUnO6wxrv+V$ZIrby#R;nWxRX=^cPT{#)A zo9eln9TzcAnj5C-I#P52#YoIsAW$eE<_w3j4<~ux4~H`$*u%7eI(9_+dPw+vt@2l; zg%6h$#bvqo|87tdaz_^d&E2&?12ur}p^D8^oR5}Mi9VQ=&4XbuNoCpd zUen`rcVQn;<)w`jbd6OBrvbt}jV|s*L{W1xL}07sq0OjCD$V-r!E6xcc81`N^c_3X z!K|o!z!VlYZr3N1U&AJy0w2rXYtw##B@bAWT%VLB+rFvCg+w;6b)+mpObPjSf|nf| zKM>1JtJlX6k3r-JD`-zgzJ>KUUqQ4uT}8+$+ydRj6YxW z&>gUf{J;5dDmqE7STEHv(7f}s@vOhu%Bx+UENub8r0*F|0S#Mq`T5y?EvnoWrdDhk zW?a`wX34XfRVYmp2dLJR(3EAQ?|SzDfmMvq7Y+h>3ipGVg?%vmIqw>kz_@ws9-wWH z{xzHz01_3}pc^og-lQ+rEM6k>g-N52Sx$MAn-+tZmeY){i6HLkP%fu1lLe0eMeP-z z{^uT6KEkB8P0=JUt0$-XQz@e%%B%%SU&EPonC!tsSXXoP%jtj?G5?qf@PMaVCXZb; zEoLCNZQ{BR1$%)d>+TIzHgzpQ-vqQo#O%G?{X&u1@?ULLi%pfl*jc$zrYN4z@d}LvM-RSnWN#>y zOzh8TEl9Cv3A8U`NVVU-{G3yX?|^gjmV=e85FuHd>dW9Gz(%BfYs7sc4R8 z)IIV$UL|v&PhxCLc*c6j+xC2rmCExov z+)oaM_h8u?rGUg~2k@Tz;u~&vat0w2b%wgpu+}5kW^oY@N|Z$FHrXc7P!<_@&hFET z$L}KrJ!(dvGGl;PQuMe;#)k*ZLy2o|($j94yjcBi@}uQ;i9zd^;fUhr;mV#oYk3WB zLCvn-ekeBar0g{gXHthz!0*`sBV{4uwn@qb)P4;cWwX}{m5_cxRbEf3Hv#EiONgw1 z$$|)$24FMH$ISz??bpXjImd`shAb+7d|;SsBC}%i*zsG~09#izWQaI7bQA#n12c~{ z3&S3#CB?Z0$vRl_5EhqKd?`KpvKN+xslYRozC=Cs&jXB&~h?G>0J@&XXP&_i^*^%E&9r}f{)xUbTS+O zTzdR&k^E;%dlinQq^~jhf!s}RO^cSLOta5IAIFDu&HikK#6><0;W)jYb|o{K&E(eV z3Q#^Ffc&Bz+ofveiEG{X|Hao^hgF%i?ZdL&Ac6=Aill^ybPCcS(nv^2gOoJVT?&X& z(!Eh)3kWD(n^NiSMx-05%_hHfJMX;DJkI<3{uqbjn2~j_Yh4}ZRpm5Seo|sMvZ_Gd ziFdte9I#B6tFI2$8d$#0;pRoII8e0ae*aaRl@z1TtO{@|2N3B<+QMR6fZLrTAIpad zj%WlQo)OlaR3UP9ICqyO`!P%K^N(HKRZ~qp3kg)dFO}jner+EMYYxv0k|MbDhl>n! z7COs0FUs8ruUc0KEcQFI8gT1eu}yY<6O?d5Zo1<>y!1pLdZ48fFwv*Wv+tqos+c#M z{m42xn5V-o0W);Br1L%SSCiVz!s?*TlvZ32VsWd*xn1m9+E@oI7*liY1b>EeDp0c* zWr`=MVU{)aovG=^x*y&=I@*_^_V}jGmZw#eewo3t1dM5nuvYL-&P}Cv``TR;EDwyu zF}o@-Pqh0BC@-S?jQKQCFBF*O5Rv{Ws>9H3XOA=-u#(h~V+u3Re>_%K+B5*d3QK10 z)8m=*c;HiUubA=TW}{1iL#7R&n{%U0EA}Y>((?1TOy_YhVGKD_JV+3QDDZyP)K<8^ zT>9)gUddD{A3?>+O=q(dFiPUpm!mrmybQ}15{zlY6Com66|{rmv3vy%W*1FW^KU&# za*7pIKHx}<@zKwZ8eK#iiWv$mz?`g>Ze};ikC3l9t%Q9R^)sggAT}54KJ}b`QV%Wf z98^}z6+{EqUr`3&IO$?Qham3!lZIgK(I)nU5|zpkB@*}kbAt#+`el-ZvHu4U%>3x? zkDqV%AjnArU(7;mR;hfyve>CU@M*LYP62b!#7b0bc0F=##Qf%az5t)o~&R6;$$)uPw^!SU2hG1F|@q zlRtuF=NUA*$PcDSM?Xf5EU6FwFkQ1^p3QDYbo)hsQj5%6lWXD`DsUSOMwDSfe)Y|E zZ<19XPlV+C*N)pjb-x$=KFR;5>c&9h*7*IQXv?cstL9^WVK9xFVoq);3W(-iu8ds~L4U9}Knt zko6%OEPjt2Q#HBva^KUVw#WQargdjPjynL=n8}~W4wgQ7U67tV6a!d^aYk~GMhiH_ zT^{_C4lkdN<`u_V^{xOd%0tU)3l?)!Qv&qP8?FU6ig=wDrm8m+41JljR+tR~l|WtU zW0)W#z@`CRe=+ARf%^h3Ji|-hG(j>`OfpHh9~8J^&`LAFW{FDyzwal;K_bJ(g^@~) zsp-`A&4~zy%_QN1iD<49_d3Tv^;&pm1whVIE*pDv|R(Hxeb} z0YSMmq-z`jxN986R|ui>;UhB^!IhhK)n((g9=n|aM;M^!tbNY`_q60uXA6asKRN;) z$D@4hF+BNlj;Uy+uZo@`LhMuQTL5mC&m@`0LaC`^d{zDEeeegXk-`lxci{7BeF}`V z%S1QfbHuJe8?txCt&5U~M(>a4nZR@)JJeF3SM>yp<(c^w-Ke<9rgMkyQ<-@eInT&= zwc8dudeTX`s&2?PcVL~8>N4} z>X$Zq1w_*z{XcCbcr?RYs7}|FeR-riO?(g{o+EnQ!R%krHm(*Zi&UlucsYR>e~Qlz zB%96gAJ`ADwL-o@lW?`mB#2)-Ta0niAh>8CD4eJ1Rt2@s{_X_;yBEMVk;6)^Ull#Yk3pGV0_%4y`O zhps8)GwB0t<3($7QF2Zr|L5QD+J=_T4%m`Kh#o!n`1PIiv3t86c4GGh7cYS+bEOFY zZN(1Zf`6j8pRZd{h$)HK0`QK?0?S!TqZ`;RVbxOx0Ayh<*d6#`?Etx!+zkO{;w)}{ z6$$|vMFn6a>Wg*X7KSc0_DGUF4q?RO`Qt@E`0Mjy5vqhFmUKRNEDw+4S zGoU~R@Hoxu(pSsv+H#($osYEmEau8uVu*1XqLTo}b5_{MkUxoDf(h;*FgKf&NZayYMVo z_vVAXf}1;@AkEHQ}`(Y*UO*)&3de|Zlqbx2{`DTOjc#o)SG3ekPL?(e9> z65c4CodT1~XX0b-Q?87=R&u@{!~-SQ65yb1pwq8I>O~D_0wtM3CCu!iGI{gBIlF7w z&MgU~!xCcp-$^((U~nBR5ZmqSKpAHg)$Y8Z1Yy7zuG=8iE?$V&ePhRjG`?x8liRAP z)GrnSM#(m2T|t08x?dU{b%p_32f|BAC6GV^e79-K!D6!oi#{T-OH1+MG+lK-%#Km4 zNE|R3u++(xV{v-^Q9k9MWXAV6)NTnWc38RrG8ORqz7^6q^a4_Jc|w z!f3(w>hl2_Vfg15mXf9ipTyN+K%Cvpf%o#?$rGB zIee!ox{-D;kwEjOY~Y_wE7GQxIaLtrXb+4^eoGZ3C>O zxpq8)tiE9k5P7?pbR;YRT62HBpPw35M>$vm^>uIUH2G9fg!e=f0LPF10*goD=OTo< zPcR_q83$A&ot^A7o*o#l{{pQIIq(4g*E~?p5vSS}hfW2*=tM@tKrix4!_F z`PNEW0R(yf`P=wL9{m{|-j@|hBaqQ)hq>YliAbF5b`RwVFwGEMT1@j_3t$XX@<6w~ zko+T*!X<%bYg%%q7cyd=Sc)%!JtEPV-KCDB{k5)m>*|Fhf_BrFY8Tmc?#t$L+LjGz z0{cY2T!%@HMHXxj(1I|O+kw5c$`e|7KwJjU-6?v2L!`b} z2U-O+bBnPuIa`W!^(;Rs=0CNJQY^~U$ZC3C?UT_RGVK*keqWI@58&<^J5!Ccw8yb@ zX$Fx~cf3;V#R6cdcV&B-5^t|LefRwYinT|tKr2Lfw*w#WFQTlmC|v()X98XHKdgQL z9MiYT_$GhGA{PRjK~_9FGrrMDk5JHTd-$Ro74Nb$NP={Losr(*k}nmhgk1MrB4%E? zq@XSn-#rxRQt$7Ny1@q3K?1eGM=UPwfHcpqlCMXpr^vW{0npZY4e)PktiS%iSQEQi z(;h{%H5u zs;ci+(m$)^#IEowVrfCb)s(7n^luXAK@A|;zJb=n&hmZ_1M6g!&)ZW&#dATi>ZQnK za7|2=H{h&}617_5>yRPpZSHqQQ@^d1>IJ3C*!c`<0V%wqTXcV(6j5Z6piD99R& z8exS>5;>?tmD0PxQ2ni2`a-}wC^rjJLC@feH@ct3w!Y)!yL(z#Oyy3u9J{4Vg8sAp zUwjsH@;llTUjDxdi4OqwP&EE$(OqZgnK4Wmx&dIFS5gZ7dMv_857#ZOpaE2a9=a!2 z{;TGSCkfhuO?DN5BmA!dUJFQsMCtkOCn2E*qE<>hME_-Z8n8k6fxR=(;tCY;{i~)6HU76@KSg*5g1-r%$v+!25(EKi z&!f8>-8?ulBp4|?i}|fVUzBwkaz^`%sU|dR7LakkSbbCgenuDqQ6bs@YebE)5}*GmPtAzORfRb^b2b| z%9o&I#4VUB>NWuTB_SmxZGP;O;K4zp3FKaFT8w#;=FgY|8+1pseaZbltIl&^yhoP3 z;iz-qchhFDKhDFynwo>SmV%rUj=Xz7=YsG=IcBUMgfz4wZ2!+{1jE0Jd)T{w{Q3yc z+uxM5_ONc?kw!W*J{qL-=Na$1p{K3g_hw^qsQt zWXZ2=+1HnT+_UC|JF+&QcPW)GB*OeEU}4kAWs!zZfM^8}U z$^ZrO)K~t*zmF&QKaN)h6I`=h9<(zrn13_D$t%;P>BnUu;aQ_zWtv_a&(5A!47)Cl zcnaC2q%}I+fLz&Uu#P%?6Y1KMl-p8%^(QI)Vl6tBL9c1y>2!Qwu2Hhj^w2-Byh4Tr zPrdfz*uxbGuvc7BEU0nN0n`RMf#iSV2Fu)El5c!)0ZRtxF*1<3rv)SATdf~|hWvtR zf226BwF7Oxs;Hfx`MBAxYs1lf-4XqC% zN){g@{lGUlXjOFdUjJnTBufuTdK0osmi)+Wdydmc+HI++SF_zC2q3~f^rfb|s}H4Q zs3DSVz5tyN0u!a4)I2s}Vbpxhv11(U5yvtrmisMofUZm?Z$N!5yPZ9ky_7VI)AXkn0K4k z>r890Jw?4>XZWc8ye@iGNg(mHaP6I$o?VA0RSckQMbm$4X;$emSSSPL7TFC8b?87Q zDmr;+H7K9#q5XkWwQCt*2qI5t=%qE-|7`84Pg3DfYH41K^#*`OFW&t4YUX3R#co>)s#7@;%t{^tU6UkW0|x7l0vy^`X_V=# z=W5G413)v-`okAn1r=h6*sydLlx4d7tYxe>Z`nFvDfi~eK6)dSrJrPQSp?gn`f0S3dQ_kpj(@3Gmu;=93R~BH3Ulo1K8ej`cP> zg6*wlXMB0MkbFvWi=cqgumvMBNP?C~4zyUvDp9AlWbK7*s!LAB3 zNo9CCWvUoy@HYijQu6??1^WW*9CzC>7D-D5_#2toEb!M$9{Q%&u|fU1+`c*d>w{ni z($i5nKV``a0h74TgQCcI%o7V3Y@I~rv*uE(4&^X*8;&b~4L3aCcaPJ%&kP`52*6j} zrp3|t0p1Y%?1IBF^oWF5O-iDlAMo14qv2`mrwn+)D;BTPcNQ<>y3(VhV@qQQo_s*5 zv@sYtcLi35c2rf+|0XBR<^!zh8xy4;mi!zqB~!+lQ2|| zTG^^TX8LK5neEPaVnwQ>prM*wu49#JX8!2my`iiFP3zDPvFZ%O+KcB$=}AkANkVR$ zC1A~iM|#h$=XX}LN4Hb=ajI8j)sk2Jk_@BbJ2U-`Pu=tLWpm|7cCMLm%H?lhGnQ%# zAS{uzKWNUCPyLM&uiJn)jbv@dP~&@8`iN)5AOcvGJ;+F|jeD$rr;6k-sJ2;8>|1M3 z?4zgGYs&?sy8s)ynJ9w3pCp2IaY|Wfxqr;VuX#KiyOnrhx5|V{+65R|Em-GLUyaa6 zfD$oZ8f$(LFI}|`Rbj6`vZK3Za#Fi~3Fx{bTpJ-f-7&k#^2J3ydySD5VJE1w?^r-$$#E$j>2xrGFxqv}6mf%7}}C9aMYnD*h#i@FZjHCS4$xhIzL zQHfaD1MPG@YFw_>P1_ed-d)CbQ-~pB@9*yes}K>ESw!&|W+~_(Z_m_vv%rq?GF0ZZ+Qct(cSa2=jrA^`WL7j@e5wLr(2MwzmBYk8QLQ zI^X3()c_q_66o91*ecViFYQ*TO-Y%7CtU(hI(tc+{bE{d%LKJ$iA!_q;@6f}q;rb! z>$pVAu@@p0Ou*IvFv=!wfG}~nPfF|WZvtbl6BUwH1S|o@Ut!seNq}m||#EJ#b zs9UHaKxz6ww!IWxE)z2FlNEj)k5U*z%muo0A{$L2LWS8422|(BO5BHZ`S&1 z42sc~4CBfEJInA9bwQiP%HHmnDyheIi*^-y>b)Bvqsq_hNU@wHY)WvMU-kiu2o@}@ zza%Tm2#Z6vwHg6fnVkRrA?$7UkNvR^-5H?$H>%pdqo^m>M1=-$r_uWhjY>CN@igRF zRlbWe`b`~m>|`yKzL9k!p$#F1iB*3-?JF!S{6#*W)u=@ITmBC4-SIOC2ncxCManG$ zPw(dh58X5pEh^_+fK=5AJ|`lCr^_XSSX;7*;Ci>bvcEWTkCln(X~V)_)Zl=qQ3GSI z7p=)x>Vn?pIjo?dpufNW8GCyPra&nS`jYS)+GJkcm)RGVs!63jp_+q2?JMtFHY614JgM!n~tZ zgFpUag&Rvs%m}vuf@}fMK`MZ?R@CB);d5d@2o)uNBgST_c(@hi@;q|4Sa@Y?&ugn! zoUv;s>ym_WJON*M)t66_AFo1&P{M!1j?T#JM165L%#E`PNT4FP_9Zv_&5LO#+LC;_ z7GXA`-in{eEog|;!2J^)^lr*EOk9`a5Og@6Gn>!Lq|>+WXsZ>S=qDZoky8Kl*7h|g z9qL#}Csxt``Nc;)qWX$E$OYsV<)}NcGH1|BjHPTWSc0>pC{SM@MV`fJzE{m{zSl}V z`A!#7i?~?~4NcE=qBj%1dT;vP?uP+~r|812hr`U0Xs=x8$=j8`mEqF}9-`N@fI?Sz z*+cLAFnok~VC9y?X;mRRGU3Wy)Kk8XyARs2Dq-4QJ`rmTE|3m;3)|S(=%1Xl$(tS= zG%f!!P(+DadB;c&XzVcPd3s~EZowcf@r6{z%^1ekR|s_erypSq?U&D($@7Bd)F= zth5}QDBI-dD$X4d8;M%j(DZ8gV-TvdZ=jngUt1nDnPs_S=1`n)pT=m?;+t^fge zmhnRFPJoPI?c=l4KH;{9Q86P91_*irEG3|w6tIgAk8GIk4?*dLJi3fVrQ|8O`qe(b zf3WliL=I97P2!AVRM4B>xhc)IMwF*^BfzX9zRmb}lbJ_|ntS24_aT>KJK0hKYJ^8w zg#CD3bXvH4)ylf{A`c=#xv11pzsS&Ilv(4&dlfYH(_1d}gacMsyB<2xu`vu-i|4HS z=H(Mj-qe%za6wF1q1TD9$HuAFD z-`7g$sif|MbjBy~ctgaBBNSfP!;;6^@Ll^qDcRcEdeCQ}BJiG4FVAZ+EVl)6@ht?F zvlle?9lKm|b)bpT2=GJg2d@hnD_{=&ZYSNHcDJbUw(N{RCD%C2!Z6L=&6}fXD_^EZEXoX@YdHoe*rgidCHd-=6k<8f1Dhd?q*5-qh`lP}$B6PXx+p-) z1N3uTO|&@jI)Z0Qpy@>VWf~uaXt3=%_wExT8A*1rEyt?$+Kz<~waDnq9BMO}?2kda zuxhG){9W@DMI}$l%F|!mjzg;575SxDKq?CrEQw%Aq^%vG@mwex2MQb*z+L5d1Xmh0 zU9S*o!71^acVtngwD(<-{fb&QOReY%=zjWa$@4!@Q@$ykw!+Mu9N3UnaVD*l=orPc z;zZ9Mq92=1x9Z`Z4K&h_4D zqa))seV%o!wL5Q8S06d)hAvpVd1jPTFT9)OZ9Vd05xr*nKz`GES=nq&e`_t@QgX}= z?=$DBd40CN_2O=+)D(Lprr!R12)e6Zh0L8YLVDnmXord;eZ8rDFw`sV7`xqN^R=*d zT_KT|d8Qv+3U~v93v!9~lWcQ*P^K)aQM^^AoBUl@_hwPguq(k0pb4o1YU@OKExk7*aGS2F>ecRN_f(@%)H`cJ>3sub(iRfmrg%3`6++OA(wWme= z3u*`hx#qBxdxn@J18$7a)L2;UTwBZSIG*#ZTb7i)_xLA&z1BPLSlMY1spxQWI=A7b z_OY!%Eo5w+%WCS|5>)M-@GMN%*fH6y<>*$YsUIfUhCUe(S|k_z zET6VZ^)^coc}(T&ux;vercqy|JFMgNiqf}%+FbX6NOy{FnoTig*>k=5Hfc=Cb2uu<7!H$cisArE$kZlGejZjxfFyWShj+E58Nfi zc9)0+*x7gI@?sOyEJ#(8zlohS6xRwD#uo5E^{nyq0N|9*kVi`46(k4+=0l`3HER3e zRg&BRVc3EguE4k3&f!1~Q? zN%RbZ%s!mjl5=ofo63iMz#lO*@>GEuGbiFhJwTH;BX$@YrstRW)8vd>fur|FF28}- z(8*C`i9ysp-{@0npLt)*wC~jI<%wr|qzl8x)Ewq-tCu2D<|}Vg8c`n)`-ZeDvsZH8 z@NwzCz#>wrM9jEB$FJv4lqM>LI#S{CS5Z!=H<~(`bgWk6ndCipNJ9D zBOU%Ni4FzxtXkNKJuGO-03|H?8t42wuyj2irq7wvaKW)Y!ouMX&RF^r?^yqu84egnn_yPQ`C|KYw1ZV3PocD0(wMiB_hB{bNPFb3*MILaXxz z`F82JJiY0DTu8?%i_7_!)&TJ+5CBL#6gFVq-in-#^*eVdg|Rj zj~M0XH%GFI9yD|=%?DZ{la5wj3{3HuuGv;7YaILCJ{_?!UK*0YueWd38*`ZJdN>7` zG9>Q~$g$3x7w7s)FosKc~%bWUBqJl6}st# zvj*fsJ^ck{I{QUsMlkWFPgxUCZL~>6KAP2nZ_jrACZbzu|-Yd$cp3FQKv~Lw0loazVxVgBr-L-bJ zmvNitVzgkps`qyEW0oDy|cg{6&t39}`I&DXPACIOXyoDIL;XyS-UlnVrMVUcPG=1knA~$AiX47>jlW zu>s_>8S#%ui=J4YdagdW51m_;+i74-KUif0qeu18ZTHx*pA;Z8os-> zR(N}rZ@*4dok4(!6vi61_#x#YMVmLZY-}5G;(4Z8d;R(G#z4R8+(3Wr3}1ik7M}!i_W?{={AG%Cpeg=6Q$13cdPxu?q?mbs4_8Q<6O?8Z*P zDdF0U<-G*0rH-igh*gbX(q-r7s)P>FOVrbAU;4$7{D$Wt>!nl8i>Yr5D^*=jn=6h7 zm%cA8wo=EG^VM^ZF3%f&9&+;KI~~~xx8J;7t#7RC{pnOfz!{<6xsv?OD(%|Z&5Aam zs(oRB`Pa9#So45D+A=OJNCJWM-g8o6N$Pp|su=SC2&C_S1hVkAKync{TlEpDmzI{w zzR&QRZ~}KMgV*wtrX(k4foM~w;=LYV5fB0V?;mgB$tfvP=hTCe;2*T%9?KOAhI?KT zJe{lOHf`-S7?HXRAnOfxO;1Q=lGk*5suT}ImS2myJoj5R5IFG{>D=nT7EI$@psCs& zo*^kZo@^zUFAPEO>5gT)cpo+%7k3=??VNdSCDU#Btrv0I#PzgyoA5K)CAe*6I+f@?zWdJFLt9t)*sl+mj0MAT1>TOVgto8=Q~> zF;eO}AM<}N#JqF!Uf1TM2DN8T@Nk)aOh~J~blJNYNg0P-o53iOU(dG5owfq}QyXC^ z(W5QzG4sK^9Ily3G?4(XVMn|+zkVvugQaKu^)p6uk4?9K(B|R@f{83eK7FLu+p3FK zPv58d9()I)k&Z5foih|UgBLeDs*P@bZ;5?gp1R>DUN?$1~^#NsGryHM_)`pnnW z*x&z-shF=pch9s%MvnHJ7*urDEKFKx%`UV5cPiuWKg6wIbs5BJcgf1t zGP5&$f$Yfh@;1RmRtUIn9DZwh{(;M6=sR&;Wjf@cK0e44L+z!5mF*c?r-i7UX9CB2 zb6s21UI#APCIv1}2%P&J-DXza?)2|EnV|?|k`xum6&I)!5?J4zGH%_e?uwv36G!^h z?{Z+;ucoAg$y*nQG?{82&Ay+BO`;-9aw@9(MV<6{VC1@&b@{z6^6Zx3iSU@2$-d!^ zm5(nODTqaUdH0=nd;A8EkG@?X0YFbTrFlT_2*r1A>LgYkQqJ>BZf3vLDxn>%g{cIj zRynRO#|Eh?QKVjf{=65NCaa~Dpthe={P+EVv4&CzE4aX4r;?Gs45d#VAc-tI#?tUk zO+8naFS|R6E`zrx%4~e6W>=3HaqQ;`Ng7uYIOE;554xerDJFK*dg|CYdkmJ{q}3It zocyew0b9aUB5}~mzY<=7rW~^R`UM<}pP1liEV|(IYTQ6~-6&kh#g#o)!Ot~V&tRCp2%Rigk`aoY zeHg3j@vA*P(A^oLs<>xo>^5<#+`3#)T><}u-hVOCZFE2IkJCaqf`g4P#4>CW~3^w&#tHGIWI>_&v3!q)uY7(vjf)fuWMJYjz8-t zSSa8(OsGdqqRi6Fy{i414b8gBLQ)U|SDpI>DL;=W?I6b@Z{+f-Jl`zwTa{eG7Mwbo zFl=LL@tqMC#Ry$$Q|34``!$(TSo1{b+)pM6QD3*@Hs54;a#xd-Rli!jPRQfQ7X#p` zww{GJC+K<}l7`*q8A>BOmDALFp`u@O8w82~yiipX{`6LtjM$~JXpFGzy_!g#bM#C_ z=lK{FTHA&UP{Lpcc%CM;1%IV&iyz>CWq!+ zoF|9Iv3-Gz>uq3UEEf5~sT0GcR+0ims4)U7Zxnh~oO({Liav=r`o(14A-L7cDlbD_ ziNw6<786@78%jO18`KG3{8G|=9OQlMzqQZ2JD4lezGYWAZYtd0DKq^ABq*vJ#w|bS zE?@`=$ORW)1yXv=pDm7%FYm3yX~w_Eta(|;i<^I{?3ZAi?_KV$z4CJ-j(?KjvCtOz z*lx46A1xc>W3G{KGJaKGpS73YZM3q3EQ52mHqb6feFrkTy@ZiM8k&YC1=R+Ao#)n} zljiKLkr(hYF)M5GY-XZ8=m$n;95d&g=NMmu%3;sN-F6Y_j2LVB@UpL+713|F#h9iIX`Swfn z0%RB|icri5%(_+a%1zz&*DfPPmJ3jvxzl8LPKSTfn>8hJT7H8XU|9~9~T}HTN1P!6N72du^i)I9sbvnoKZp)oNUdQN+KV2)Y0>*D(uCKb8fx!8w zWyUZXXe%kZ5q2-{4x5FR0P&m9H#{qnwkO{8aONren08^;Z{t%{Obry>9!H2Xy0!4# zcT;beh|b8){_ZW90QHFc(*y@R!6I<)Cv11W{VrjBeO2YYKFN=X+lg$5^Cmu3_E|kpKsPcTOe)N?l1l2yZ&#STjGcH+40H5g zhe8}Try_C2^8WVkPJzx};-k?86qxDJU#h zwTn#yu{A#6p|C$qE=B~Y23xXWN3%N*7Z*KyS3}p9;b?N!vV1&8JNC#E? zG4;WtlkM8j>H0v0l)aV>;9c&HaAMwsqPMFqH#Poy{F3$hJ<{#L!NG6k%QyGm*=o@C zwv|(d>LIFaoLZWm&-?xaNo4T`W?*0nRwJr4f-$vrdy@du22>P*RpF)&P|Cf+gC{{# z>yN&hSw~i&J^0{11pVu$*K{i6+gT$!vt{=JF#;nM`^NcKV@^tiW7rDDApSRZJn6j$ zQq}K(v|Iz7^FjC_2E{wbB9^jPOzI2h*8qS4Z-c6H zTmY8pt>z^I^nqjP+()Ph@b5!FLFNmvHaFdr((j#KYtq91BiK1a?d#o`!!B==NUeUcj$MM|*pv3Tpq<_DQ ztb)Hkvh>arM~m0F#G{iftm+v~9U>W;d)smygGR_h>FHi;EKnJ&(#AQ)%`eK;4CmH7(AZALH1U~=$UWp&_fJbGshN<04)Fy_ld+usgp zdPBKXBpQzA^(7v)`{LPmR)!Yq_lTXQ|5gSK1mJ+FWu(;zNF`h0t3kU@RP-T!t^+<3 z-)Uxi!`s!Xvhp(GfOyY-^T|JmHz=P`?~eo%8PR5ynrSK_$It86jtNDyJv;d+L6m(D zN0eaZd47(EpI3QLE?kt$7h+Llv^+LN4ETNGF3^>Zk1nR^Jj&|r@23k2 z^o#+L)aT*n zr+HU-{7DYNEx2j}O0wB)nRz60?>p1ZsTFvJ97A+$1nh!Cu`qiDmbTN|Q3^UUi&-Il zal7o?zO#(99R~W))OJLL>%SBT%`OZsY*zR7@kNwPKV3gF-4T||Rrtfpdlj5{jY`HB z5qr{bv7n!ecfjAGaRJm-V{|DN|Bw4m0$r65$1fu$htY08d3)u+1(f+zpfX~fg+kju zs0)4LEcV`(8i9Ae-?T(La@&^=v4Zam(3#`iK6l0C*lFUv(!Vv-fMoPKi|aVd3jbyA zwsdMEMV|`?&lqJ44(FMyx|V#?RWwrN4iZX-<3dWSmb+Z6SFUHB`+SAl5Gargd6>bG zI}AS`uK4AT^WVW+cSu4zLSrb%-`F3=)txUrI?Xjbh5uv^^8~jo7PaNK`>-rlEIL$# zsN#X)nreYias9wHHvW5BPv+!Hr{6r82TF$qWAWn1)o>Ac=?p_Gpaqm^K1T%vLQD>Q z)?a@Zy%n$WXDp!5PPbWk@36l;?e_G=_Zt>IJ8g_YuWH_u5G~)7wKg8a_r{CaZ<};m zU0FPXi*a3tN}8tNG4Jknv=3Q!EouMCfciH5tsy(;^@+Dv4*kj6%F^=P!}mXQ4Al(b zsn@qfwLvLOZAu!{y8j_Gpv@#|p>3-0z^6gJ;?qFE8pwsPsdNSPacUz z6HW{r_yOq8D>yekE_z3BLf~)?D*&!asJQ7b4odRN6p=tBEXI8KtD5_d z`;Ol0HH>HUIe-51Tj0M1^eC+zY(>>tD%dEbl?bD_6Fe?IzS%-LT{?ete?`k82;#d&0-Oc zM>>n-Yb2v*^U`gUl7~Vs2K2)CSi`KgV=5NXG$-+cnQdR;2zfLr!$pHOpD8`M5D*5k zn#sSeUL6yrSlo@$Sm1?3B7hq@HblxYBPM;y>}bpTF!{W3W3H3tvrgdoBK zCovfMJsMEa`-)kekdyzTLB<5;G?j#jsxXambi1LAmC?G^picJFjr$7JZ2WibPOO)8m8 zp{lmfkFnElp-|0@kiwJ9i1bG{(~}L}`tvCZ@niAb`}tOm6_OX)j;Hrs#|`|pUTV;N z&>g-%VPLpwf$;CZ&oq9CP@k&bNJE$Spd`W}3N8b#mFDmGd4s)YNqe1e9NH@I2vpaL zST)&ubEQhcDoRU9W?sE{(+$)lkrHrd9F$W3z&rmDe?dZde~kl{29AjnvamJg1uuC( zKZYpgPPV)R!;k>ug*Y`BU;WgzvzouU+9bv|rs#couzwKJM0;rketv#hkGvo%lU?V7s0+81nK!Zke*K#3BO{c z0n2&%UkdjERz4ipGtbw}eJt~`b6%Z`=2p~FP~U?wM2@1!)D`SNMDbBrhCXg3V?)zN zFCWct@+A+^qF_cdhy%+YreQVU>J4S(DWfi&QW&Bf=jS#z{TDmkUl`ZzW90ol!7q^bk%11PJ-D_)>QW5~Og)$db;{Mc~yD_-iVn8>$>;skB`W9B@mU)tGlTJnfYJtfUZw|c@Mt++U|JMXKQL4 z-N4=#c^`sx)a@rW6~y_VWGnjiC?pVTYVyT*cE zCT(KCqShkhMIxO`za}g3fEV-muk7^RagL0M$y}yFJS$nzgykFYcF=Ug`}B(=XF&w+5<_3JdNN!}t04x}O%Cw!srtmGn2R8yT1K zonB)&j?torWmpaHPhw+@K#c>aO9Q!LyB>lOv5*=VQ~}}Dd7?e;g=SR=mK1-RufGhLcruK9aGPDb)xOSGv^7JEiWM}a~U>}0M5Lnu zh0Zk_xPT1Cd8gnzOtt(C{N)4OWjZD-Wz_M@AXTqs3lZxk?{05{o?k7fYp;O0yZu*vO z|Fc@gq~19w_`N-wBQiFQRm}h*w%N55rU&1-3%O`5Nx)r%qkg^5O-&ws!eYM)L&+9j~YsI3;)&rt9r4xjXU#9B$ zL<(CG#h#Z%(R>^iUaC~!#+n%}xZ4|Q#0X1^4*$5W*X5mFfDsm=kBO&IF#vo~gdd<3 zhhUQkP-3Hd;;gM2i+jLZS|21J&`W(-RuMj9Q%!swJ5_SLjNb1ww5vqwoi(ZrmY!u ze}4BRcQn3&wIyHw*K^1IF~=S`{iAax~1JKS1prciGhA!py~oW9KDyY z1y_dj7sd1!@j+*%x&*La%o$Ja{PS%f(Y*#g8B5@$?D6Mm8`Z`dI9L{#Y&$FF)+@uYWMdZ{6{Vq_=)VNKjC*SIuus zk(cNpkLZeOWCP=j7v2Wl2ePyWM6!1pnwEELl(Z*{+53LrR#xf}#Ts0{PZ3KB*C-kp zvLaV?b-}JBBq3hr{VJ|3Rw&R(ZsZ~q)V~J!6-ZS@V@voo*?T~$2xP(7MF7d7pCial z9h%NN!kcj2`*PueL47|d|68O$10)9#-x}1b^3aD3h8Qg`JeZ?db2BxL4eGNBd*Syv zhEzNc9eN#}z2sF?G>snm$YNH_@&o%k0_5Ea>rai{D55_v0>>EM*}BlV)VV}P*3L^G z0ud(-e3Zv_$YtmL>!Wq-AslqPRpl@nLoHqV%|DSC#h(Ox4p8<)Seg1fo2bqbY29A{ z410sDOf&QS?8VSBuZXO71oGBvgs*md3-wtlV+df=m);oqdW8}U7AQAjfoc&+S3Jdr=F_rx;?v?Et>+<#$ z&43`)o6@!v!)G-23R2Kci8d+*y0urlrnT(7YIRJfOS*X0_JP6(>iZP=>+^;QIFnFF z^8q0i1rQWaylOVf0_j$S?^Hr6(wri_H z`D1k)_?fI>)qG9I_^N!cwDv6*FE~=m3)0wo(!>n3z=T6T zXQ-0$^~!Y^A6L^nrk0_3?~ndafEDDOe2?~S!XKVF=nWOG+DJOmeT}xD>075iL*tE> zcQ=LPTROTy)6dP1&Wd&rr0=pizGL&wTG$3?`4!w4R zp*M76Es)oPGT;?>)&!!cj{gdDfQ5BXZ$Mz=Qi$#z=4MOH6vnt-vve&TQ=fPUTo3JA zWe!g@QSxUalMxXS21Sj=B@Znb6nG>(bm!wlHTQ8A4slq+iaC?#W@qwOAkGpdC34UP zz0DZm=O@x2#}2EhR&-H%H`Z2Aiht8#u-+fK*I!%{+xfkIN%)?rQ@^>DznYrZ2Jl`5J-c_idzc9~6e3A*kW^=v8o8W5ybU7~=$ zdjUF^Bus>@GKeQeutRqqz%FCglG?%Wh#xu%{HgKCZalZMJGj|-3R=29$DE~%4y>OA zB04>5B7_d~lf!j@p4ewG&j8!E=korS6Ge(;n4#t**ZYJifOMUN??U^h z3N*>X3+e_q>T*Kb5Vs*EXR6H;aD*-S>icJ|kqlK#KukDT&FGZI{0&lR$oDO_*Py&4 z^yV~D;N+do1D9m#8kKwSxSlx>uc$)Su=E=gVgC8}aH4Ivv#RdJG#mnQUMB#`DAXML z%ff4?zEO(_KX&>J&Bw$a?E1}3_wX~>!?jk&*;F=IRXqvqDMZePJ@^XbDlw1?`%B7|Q$g~%2>;2#3(za>&M|GOq=4JD@Tbx@Rm?!xuJzlPMYF2t z5OPaok5RbXdke9_8s~bYSvWi|j-frjj`WJgkrUg?Oz8Th(#ni$_FuvE?M`X313``p z^vH@5dIo+2ij@1YCSxFHpgWK`ci@Y}C->#aJ=#7^W&OIGQ>0fAm`n!XKD&yBz)e{?@E5ngJBIqpHi-0E}Q%id}WvpmBRF z)Au7iw<(vMZlSGKfMUp)-6(0PRjH>E>(OV2Ht4U}x@B=J?5BUa<#${kNkRD?qNSx6 zsUBM-c(KW$FDcTfq@DqlPxfRGLrc(e+GWISB8R{>1&LZ9O2Sz7$Aal5zfOL;k z0DK5jfR4AAdVlz-Wcie z&v-hG2&7lyRJYW;x8)0e6twK~Tg!1NayzxV)gS9u%?48_dPNBD$c_Q%jxN3zZZkd= zdH}g-2LPKTwHSCT+!zD%o>s$~AXGH)0;|~9*JCxb^ffefZOf8}T58_0;b+DLX4baw z%32XIb-nk?bha+nYAm3JJ=VY#s zPz6T2jI-Lb?K);YM%i?^_q}nt463`2%>cmeX&zJ)iXMuK?k4rTVutjPIZFw^4+$2m z*JR9nYj>P8(Mv!rT-js0e=%p=m%v;Qz#c9H;I?xD(g6P$Pf2Szu=VL<)Lls>!te5d{AZf55m!w2e}st*%xF`-=Tr;i9Xm#gdq{wETv9 zz;7_THH4lks{Dx;6Z#xTrTHhjI?N)sW&a%k02O35-}(-f?nd6QyJmQ<(;QC0$e8Z_ zLC>*`!Lfs_qBX_7j>)7YBCailF{NP4nag=~Fc-pvZ3WOSEGz}3(cY(PhWtt4?bsqK zryKx*?Ps~Y*G5?alXXGaF!%mb@bdydgiR1TnI&{=XF?_lK2GG6U`Ors8|-@l5GO^x z>>FBos&|!A?}?qRC=;XXCIRC?@J{VKJ;G|N763_b2eswXGpO*gc+VZcv6>D%=D$Ca zpLujhNFcovSh0s12LOa3s;Ir60#1vqTrHIJXJL<4KW&oRWY4Vi8D965yEi!ykzNb_><-XU#|^qiM$$#k|av zznERBKZG0K?|9rnlHK^EWU0ZrNl+NQkB=**l}iUKt7Df4;cN{QW)p*Ib4Rd=Uz z=YMt3Ver`r;qwWYI$%lub9V^!(t5n5a z)8Ei%>w@cqgy7a5=y@qZD+8*)=2xBRqk#*r#!b?ULuUpz9MXplAvU#VJl`ihFmMC{ z7rvV8Z@93n`S%AAmAr13H?YgBpS@@L(0z>o>j%wPGEdzn_cyO=I^hx}QnoG*8jdip z&wNh%3;Igg29u|CjsEb3wuof`lhwKx3J1Tv{SqZzx7#X>u#-5%? z(UR%wx`5?r+~C&|7rFjPQO!Zn9}hMOwA<1nwlz=QNVJY5`2J%4%7KH%yxf zS82$dp{P{!c#5^Z(vh)L_@3I`!dL$bC8oWF@f^LTP{Ogk{MhoB4|-Y&WbKt7$fzT5 z`@ARCdkd~F)lgtHCbdJ_?23()9=?s|pBXQ4zoq>U3Y0GprrJ1V?n2Ws-6r=Lx3YDd z?iVYdIj_-yao$rY7$AUTna5VC>5&+jAdlTrf_~kfPEV(fUj}94H(VmqY?UVrok=x! z?W_F&cG<8X^<9(W<%ivqNmPUP#H(|C+-4eb=5lgS#O1-OyWU;qhbFJJD4VXwqZJdD zhJHGoV+MwPRG127i--L#cVX8D={aJVZqvMz0;*6=Eg%6-1R1n6bch!W8UfzU3He%`#RX}oe* zXQWKe_c7?7X>=uMr`t%`*_4O($6y5gO1Vk(^>3cl$?N1> zq^_0Pdh%j2`EKhM-)=;oe@Uj_#dN`adN|w|I-m&OISqZmhQMxXb{(L#EBDyd_^Mdq zGW`0e1*n=t|xV{5n?P+?=45hCSE|@ z8|PG=`@Uk|k+-|1Q%*JOvh4`&s~mX&O;F`OjclT+taEJjQB{9H z?(C%7HGbiVb(wV&(xvcwve4oCCLZyo{uz#UH>WsbhPGpeD@Jf(ss;py3(74lt3 zqvc}+e{#>M@637ob*sz=@5-Q+8>ad$xa)#Q5w6(9l2TP$Tf4W^JSG%(02-1G=}iby(wTr2UPJ%JmH*yFXqOp59Qm(9u_TI^|BWHH!0G#UK`B!Vs4fZn~(o_j>KIx;VDn- zS6Uw`*l7a2qJ(6HoXre6L-au~v(1G9ndv{PP9uQJQSz}*x*w&9Z@bzjsD8cKoH;uai(H=l4xOeQ~by|^>d z^1$AwDOb4Fk}J?j<*m?y>45f=kx~~k)<4@q#UT)els70X2Bp1nw%A}}lxN!>0A z-^wuzri#&QJJ9y?IyJNJZJdy4haOrbB_KTZg@>eTMB?Xk_Uf%CTyygH?xmi$(nWik zV9NU%N4Or`J?icbdL!tmJo^#v{j!SAP@OT|?Bv!#+yC=|T8C6WS*PQ+NI`WIa6;O@ z<^B1w@BK!l>1QpC$siI+$_#fEZ~kF@AttZF>Ts|kx6j*8!ao9|wuQZ#s7em@RzVc! zMpp6X%?)PgfQ&9M7X0$SE#XkUmzmfM&Hv=mcDxsp9&cmVCTQlDWd{!16B0{Vg<4Du zwDtdFB@*LFj`S>z@-#6j&**%t+tvr>6Py}&u9kBk$I^cLAsVmp!w2=8NQ{i^sJg`@ zcAB1fZY6^vJG6%~mB-;NVX{u(l+7_n5zw~}pDZ>rl^)ncVJhv=`u+~opUOmS=`>0e zr|)e0;D1Hu<(?KYaq~p$$#8+p-En+N-O=-ue#M1}(?v?wtycC7j1;47PYJAexi=yH z??hvLe-vf(<-SOE{jqnbGa#IbvkY3wpdH^2Xc{?Sp9=*AS6wn{&)zn; z^MK#G>Ec_Op~ivBH6alVgz-$pVg4NGM6QBS%fdi_x4}X+ucieN{!$tlyWOwL{24s# zKiUoL4EOa!lIWU-+Mh_auAf+6<-sXbf!Wo#c@P?#Z8ARXe9uI#9{i2Z$wHqhp>OQA zQ`lMzXCt}V`WJ6@1{Z&KX`VDl#>61d4aKRs#h;{ysOpyVVM(xedms3v-`b9@>Uc@; z##V(n9L~tBF~8K_y?^YC^uYGccdk24+D4>3$0c8C-`o4K)Xe(Ehuw|A-Rb6lxG0qq zoKw3ls#1S~tVtK82+r__ou^;O@Tzq!6E8*xZq?w}8y&+!)tf=f4sWH9bNq4Zl zQ=6Q7Zj#>(ln@-g`anuKjWaR#SN+|dIJ_Rag0EoB6j%sbXfmk8ApkoC=7q5ADd%43 z06LVO?3Q@8EcmC=)ucbVHONg4ZU^+C(5Uh-P6n3JIHj3TKf90j?Xr%-e?sC0R`R+D zCtH657L%yZ2C|3wn|H0diVY$Dr}f$U3Ybq0uY^yzOM zCH1mHo1MVZ2-gxy9y&r3O}VSCC$x>XXhT!c;#PBV@$vN$NDH~~c9Fcgm`yId&pavF zctD8hzUZp9uxUHWbDK!rk|-6TL)rn-7n}SrRv#)V=j#%Ds5sD3 zrJVUG8`|ipgoc6jnwmM9s@Zjwg)zNa$5KA22~_#*ZIO1{bD@83p4v9(=JyJ@hzf-d zg=YU87)wNTb>vR{YDfib6+9r+S`B8^K7*9%KjXuyr+Y4SKK3ly=RW>P+Bjhn)UfI0 zJdo2xaBm}=F#dY`yguZob*GLef^&N?e6rw#Ie`HJw_F zj;)Ip3E`rIt{J~&-WLuhNW10hFAd@X1 z{BhdGD^_o5LH!T*9N_O+{@p%@0O(&!2@cJM@E_0|u?>v6r zL{l6*o#Fw(-<{9}aZ7P+q(YYU?-93JFgE0OI<}B7NdB-U+ZuD_Wp2p%gE0!GrZF2? z1Y=Km2h%`r`#(=dB_?~*4sQ_e5^DOMUagVYuzkNFCYi^;u1wwrVn)9#IJbpLue1CYqixx7j0icsRToW@T{9nC#E~r0>vZ{N?uc&XI=Z z>1wOHF$LfQ?>+td4fFSVT7BY304TA(#pbCY5`>uXl&=*k!u@V$3LCs_sw~B zuAm5R+>y^Dlo>Kf32NSbAg1V!l_*ur^lSh#^!8YV%Atv2@b)WuwtXWLM?{f{vM^0) zdFy-T)4b$n=1e@ry(7r22nBYx#iT^OJk|~tthhpcyzB7~*PpjfGd6xvJFT1Bia6{b z2kQRB${qo)YsxA13M|#z=f^r&>m6Q3h16V_=G8ls znN*PqmuPUn4I?}m8jYECCfh5{H@0JG`y!YP93MXv<2g^LkeU8@$3U0kK3snLjx*76 zHgcZ$tj$?$mPrNl_`yDvY)XR=d6;fx!d!G(MTj)?V>e^umsW~*jDrT1b+7n2(98JK zFl$|12=4%4ht{3z8rjT8>?x$DMk+WGzI@u3_tjF)qO_2bGsF-YoL)FBY~QWr@AL7K zQCr^8PZQ-2UKw4Ek`-XFYm>>jUgnP#k9$%4NO~H2|36g0-hp6V@~}`u1%&8~Q>mO} z_s4|TDE-uFykFcP{Uy6}S7k&{;B)LH+QbLvyT&|a{ybKr*9iyv{uC&u@bBlK5aaIf z7l~xG&>RM&Q*qhG$?);drr%G*Ifjy-4YjYq9S{DAf~L3AgC1pQj?$h@klwoi|}GqV$R z_Q9jM(-S=htZMgb!>uSPIL<`VtX8lR_lLPd@I}5MDkxM?Igr^fz+}pv`Vb9I1@8%# zqtRn!yvNLUK#S8~o*8b?KlLD9qbE75ePb)N&71rHiF?dSI`<~WA3{sik9nP!& z38F+hagtzL2X?#d9Cnh-OgV{U$~O1}TI8NNbRq`r1hL^8P>?QA`U1K6dXd$lfO4%1 z(HT8>%~$J@g$W`CMIghv6Q{U7G%0V|IY`F+awLZZx&qudrmo;v&K_Zx8Uq*W%1JfFVJ{8#gJaPTF&xFBN6_t7=XPEI~M4Ee_Foz}t9LL>WfNT@2< z(ox-EH%waxZbv_B_p)TSBuT2+d)wCd5hy!=G+0-x(#!q9Bhz}}7iP0E#bXBzoqB>G z<>W8T3@87nY}_`_o_gSP>(UZ`1}anEk{SrdJU;9x z{mk(7l|I(^DWLDo{&<9C|2FBM*(DB| zNUQOD99>9wrLRXsO6B@ld7LW#9@Dyc37t{ATLz77Etu}&loJ%wZJ-(qkS4aRTkv26 zm=2K2EqlzGrHzQKyBlbj(S~=|Az9_pi%&t;Z`>|yRS!mv+H4W+>t3S*FoY?|K6VGJ zkXKHT%b(ot-ja{M!)K7=n6szq35yuhYz}t^t$T17$s8K<_hX|dk=f!l?xvC@U?G(X zNvOlg-FPe;uq?r#4+w$OTB#vDl|m!0E4tP}6hZ%FC;85#T<|iI~9?Axo3R9zLY6 z^7`zv#cAxm&(-{;?$;TUJxVrPEed#}yrCA&k5n|2RC+`UWFeh$VEB=AD!X!LJ7TSr z-xc7oW0**arIhGKmo_!=7Z<;;^K*nU$ZnHueA5aghv)=N$GyugmCPrH zO;T4%UHMpM4)D7F33xRg%aP!J;uc@;m6taF^PGVB<+ZFH9(<5F&0s5!52t({WdN<3 z;N%4yqzt2^c)>f%vHW+M1$(}L#0zV8e8P*efD;}d{RG=BSUV#}&roL*48C1Lu|d-J z7ZhEWS5`9qw7@Q{DuJG7lqtsgiIG`qdb9BRNZ?*I_TBb)!tObA&hW%#4PcOw^+cq4XML&yq z{>1~N_n{=V=uNEg;nee0XMk&x8(LaFP(-5Vmf_vKrSlyVPFCtIGo(+0+Q=QFtGt8exo#-Z0$ z9ot5a!$n0!8N^A6sGzh%p1vLTJ|&1jEr%2TxU*nT=U(rbhNQR=hrvGHBDtcZt9aw!BUgmymGy7(rrwR zP}$qbpdDq0)Fc02wA2y8zUB2bH$fHT{EG0EVDnRfip0!aa3txc))0TUA)@HNMJ%wRcrH`<$=yuo|jsu=OQ?o~&_YG`- zvj)3J8v77g+7aa8p$xmh&E+=2D;fWlDq5Olq*SQAH%mb}FYabAGwZx;F-)#tU$qbF+d{3msgSJH_XesXd+(S?HEK&c@Ud%+B88OU@h^ zspd~MxDqvyYjwSXOX}YJKi*^rcioGIR)5@YYj)gkym7>=4o(Oin94aQmj|JK4}DUF z@QW|)?Cni$ztaVceR0j|`zFRlQ!YJK+is#V1U(6xG6#Z6h9_MKKYqcXxv_rs_Td%* zOHG1*VRz(l=uy<-+qw~T1(l=bvsrTM?)Oj&vpAv|xfjsi<$O%X8pLs5bAv9r>L8T3 z_q}L4Pw|@!X~xvx^T_P7yoct%v)6C9O17vWRW>1}>q#?^`I-R_uOn)rF`S9k0<|}J z1z(32Y@^WHj80%^U0P`a5d=rwqT5F;OD6S4<0u^D(Kqq~h8uJuWvR{`N#1+z$Q^GD z(g(BfRVo|)`dVWI|J&0BR(BPM>}zgFHrVq0l|AUU(UnK~ndgkNKg z9#i&9e!-xR=2PRok29{cRQ49*Q($z^pZWCA?i6>_4G;y@bdPiqcbuELGs8ZZ><~+& z)5Npli5a#XVjvbu%nK@@%l5}>Q1N_Gq<(8>WO{%33|$EQS3Go^^Ya+vJ{G1XdrZ=I zQIHs7`3FecamHnPj%-5h*z(Xu_K}nOug<9t67{6<$jRF_&6a!HP3=S==i|D`SS@Xh+eQXM*7)6n5!4O4`ntPWN`BeTpI#tPkp>qk9>fR=3VwbuQX5Tq`~{i3IFy3+pYqhu zhmC=GXJld|O}mpS0}kyJ7c=1|RCRS92@09_B0K3F{a|@JgXmU;JC^_MA!JL%@~Urc ztE?Y>#biYxx;Gyu@&(7b1zLOWpiU`P^_9pAkA{VDAa9L9G1S}9pkh3>o|FNaUxGn^ zRDJ(MfuUI91Lq|d?@$W%7PtJPauhaH>pzHLDN-7J7#ijFK+FmMWL?A8{acHm%f~@} zI0yXCUykmpxTF8-!TEp#q9Tt#-qiJ;)PXZEbs%JEo0F?Ih~M<; z*ETH|qbJb6Iw!^e#lAE&jcQI|%HFh#L=LR)9`)~oT(^&=c|GUc7r2=&MIrwa>9es& zKI7lPZ+qjm3eAGaMfKUm^i`*@NV}O|dsl(*@C>{69RrwnBVxJbUxs>SGq<)w1pkbW zR%c?GJDG5^Ju{*3!jL+<(?o9>lZfGe{lb;;^*Dnd+sYBdIGBprc>3JK-rV8)L00TO zNE<{w=2A4L{*FL?^U%w#X-P4c(4}zLAcO=Nt9BpK8TO7I$hi|+u7@{#EsZV7GIkCBoxQ8{X(KX@CQ{NTOQ{0vPTa zl$Ux7H8#jEDlK79^QQ&N&@Ufm=qrsn2GZn?LYEdnQ;k7YoRd`WyS?bi{|Ka3X9`4@@FWH*nr1Fo&|mtQ zo9;-{5wAy`A{yM&9|QHE_|A(5XX-aNsfz$o6xx>AlFg5ltKG zdZR0$-yI}RvH{|}7waPcuO+msUPY58UPANP56M`M!RyROs}J_T`Q1Ns^j-T-m*~%3fkqN1|@G%6m%^T7wbr~ z7`P~HEc%?>c^ozgiZ(-wxiW!>nm~aFh4$^&su8ll?;FO1(2@t=SuCAOlh?^&H*P}Rr~b{X;F&C- z^1*i}qT(m&&1HAKL=PcgEn)prZ9^s|7D! z-xAFS&7_}(so`SwTg>*Je#g)mI4XM^{gHd;?vta?D8W=6(L3qD4{9~_qZrOAbS-zI zfULO~9pz+%R=L&h!LnD0Js}ZHmL;!Yro4@APF_NZfj%%YpiHp=EP# zz)k)r`XyXLiL%QUpVD=jKZ<5|_R5nNjL_77gx62$s$fW2Rh5u5vrqJ0gsg!m78Vv0 zJCHU;6F-Y5WZgHRERZuY2QALSsWCAqP)#=AH06MfwB(z1h@SiT{k=r$6jX3;++(Vn z-|z;j8>d^_WxK3G7oS7vD2rXcVUsKL8G4VhYO`Kf+VqaJi7k%TpF>;r<$5yZ1JZk5 zQIa7f)TTG{!&L+rmp?vDalS~Y;AQiscH6N@KPG^W2K1|?S7>Nr&n>g78)(Do>gvRN z4P%0R+0Wnqk7Qr%%?|Y&;(^h#7RcX!t}zW>2_dK5DVdO#X6sqXY{Rx#5zlfyri$!9 zU|C^IRaA4@)QPGBT9J>j9%Iq`C6B#v`-_)cX2Z3&WlBmC6c#5FRLe-IEJ69~pEyb* zAt(lEu+X$173yWVSowiwR+Ni4IBm)*J6*YXu?#_HdALhB(3i>F%Fev|ZK>=62}751 z1f77#QmLf)3%&Ke1-A{kU(HNRRJGeHvFub}DzKd+`AFT-898-)Bd#1x zLsJdT0Dh1ZC2>9TyUM>382}I=qp-0_YmOCKhOIVDbPrep2`Gslws84I05IQgoV|UN z7SH>?SfBf(?oQH}p`UqSK&&#v-Xf(UD!16M8@wP1IdPx_^F3M zUxAV7Q$)8c8Cjgf-LE1azkZi)j^j_q9v;Q(D~&s`<)c2qUkT|*b*`&Ciebd%N~j|a zO{a=b1l!miT(U3VPl5@Y{2h%xzxnRjYyRXC z=B0?Ct;J-xUE&DE;$?f#kWS~RxpY_Xvh@j9`iL)?WA@M8P;?vTedh4t!!9RJ*2uYe z^}V=9dqC-*@FWKJE8Famk{4@$& zzNoCMIzc^z6j#QJmKpq~NM@SvOVxDl3bs1V%;Yz`6lox^H^QX}jCl+~wS{sP&?E6- zJ`HGqSYBv`KJnrCP88(wX!UbKvknB&I5MTX$#83Ijr;9H8pZv?Yt^#g-trHnK_MHrqvZ zF|IvEr}JkN^}-I;4P4NHgWCUK>>cFHzDLT_wJ~nEx6DgL77WM50}M>>HF8TSD;42A zW@vh%qdC%$+Xns_1>pmfr~vtr#n1jZ0X)pz_^eFOaj+%a4w_|EoGsMy=jV?yR>NfO zEKTgqIeZn0<(8T*=y1|$bPN)L>+bFSBSUXq5<{N2G#rI+HrE#9@btMWy;wK{Sh$1A{(ooVYc(eV;t8gA8Mg5iY4g8Td0(GHVw=4$e z7wY5pfL!S)`9)vP*=>BVSL35_W4X@84&|rm2DK)!Ex!HG46!iu)syH&FJRme{t#Y? zKgyA2p)~IujZ;GPB}I~o9s?9MwOxX6x!2#v5B!Jm(Byb|_s9yQtA;(PHIroam+KQC zV_=k#hFrFQF4aZQvp9K^PlGSdYld?pzbfF?%YEhVs?R_7&{-08oSmr_PPF<<<(3PA z#c2+??5zpM*;)$2d>?b*C<8wA%UvYsyUAJqg>%+-g<=#1W#wc?j;TE)*l|{-=R5vg z%G|?8Re>ccxit44ZT`L2F^YaLAb2mCu?k_N;`M}{xuxZ&Q+v&kJ2Qx46|TbaUDr^V zRdKR%>9H9Y{NiyB45y2WuLq~glM>!6)dtY-s{WDfqU#JlW_nDsR+p8%X0jjONxl-t$7VsE zT8Q(G{b4vz$S8!8eE+@mNe%_lUZt;B5aYLmDZvCcxln#Mt!rY_lyUOu-FXaq{#p57 z900FlI5Ur2KVs=9s}|jE)kxUSQzVRNX_zoVR1jHhe zGL)>*bQhftd}K-RXCBeYAt7Ns{w!KuflMjWpk@4< znZhyae}_EETdH=^|9*qgujqFTO+Vk|{KzC6IH*PGsH55-xK zqGEZ0q^NQ}2@wv@sD_B%CB$?_>)rj2#LfTo)v%F-nQ>W+;SKSiMZT!Vcy^4+MgJnd z+?&gy!3aKDt^xE|JI(mOUjHA8%44auj?FR73RH*l>G9K~VH6a0k?Z3){@srw^n=os z-)0OIN)%bXMuz`7HGPxz&obB%T0%txhVzp|Y*7_^XIxP}<3Q3n?Xa(X5*Gi2zA;T? zf-e;B9H@Krau;1!nyG$Mv6;-j)6Fxs$97<{tTMIqSmQjn2gx>LFkYVdBJFlE3dhuE zCu5Ntc@{|Wn=B`vV`?%g{IjaF_-oaT+n1wu9Z3gl0h?FU3FRKCFdk>)o7>ho@n^A* zH&M`yv|%_ki>EEbRbtK424eA-!!*$UIUX5T;(#Na9-taNRj&NHHWMS43j6;vu|3j6F z7&KQ(+_PtwKHS@*ytB)kO1E6BG)4=#&UY!>K}&SF66(@tTW*d)^L&q)lBUoN+V)`% zq?gqk)*teq(;?xze{DB`yYH9dP1dV?3QwZm-L%gCd0*u~kf?FV?$*j$|EMbuiw^wK zk}UMrzr^9M@|};|iW3`ogvK{mRWO z8T!hj#M|N8;TAsj=aT-^@R+>=TpcylRz{cZr_lEzySJH4CjQUMfXuOael=}RgXO1U9Poua@^{>JpbFmL+kKws42v^T)viN zLBQb}Z=2TTI<_^&hU?T8K6A(l4nM22Tb)|mSSE$e*@8aacy{2~ml^d4mmfr5x)yAq zP-d*(@GPz+?W8nu>V@GkJt$PDj;VQ68JWeInBKe#VdD1Pm4`hy<(}bvI>AavwQTy+ zE;94`n1rm;`&&Hf4SnX%OEs%CnSKvnX10IX&4KZlA8W8YDXDp)I`i^pjmc^Ek(o_f z-z&!!D$5anUw9B>7(#jl+j`AL`x9AeOU2#6ijPI-`lszi{Cn2_H=37BEAMk65v};i z)a*3}>I0ROv2vB~i{T_;k{+hN$YEgIy4O!@mqEPL$Bu|TVlBWEJe{W7@lkwq?^xZw zU4NgR!#4sb3mu)E#kx;Xtn{(XBs?xHchOpEYHDLGEiI44kKsKLvMk_c9sUDZ_f?g; z#&onlMzi&P|Dqq~P`N)It_)_5@FE?}mZ_SVjM`{+;Dy;Nn~Sk zut)UaInh@b_H1e60&7u&*5A=dE3E(D_wgZAaErFx{vtMv2Zn*Pl~THDpDwjDrK}LX z?;p~wIDh9;4}zv?u9Ocz9m*7YJv?>hl~dcd^t|l}OeC>bwZh@QoV}Px^GIlU0Zdh} z$A$#{R^nFQh11cmIm7;6Y;T4RS9fRggUza%D^C1(P5Yf-1b@?kLU`_D0_I>ghP6@I zZ5_{KX0KyTFZAZS@h4E8E zE%q^QTwhsFySJv;{{qJV=;5H(oo%0XT9dy7Auie$ z2e+)PiZ;}(|A2blVsZCofmgYYfft=4*L{mA48GQ}`R`U(1~nW2BWS)qy{?Y|c?b`3 znc;GL^Jt;3nkqUQEa%mUg?|9*%ZYssDEtYE9RnY;d3ys`?Kwv(sjMwmhn0dZmH?BI zc_tapk(mU~`&+R3l7n__moEEb&et)UHol!7cjfs5WP#8dHKDc*8Q^0kW=g{$y}5=5 z_fG$nX^i;lFq4pG(ZfbA=3+cFi)tSBi#d&9sqFo!cN)WZhLy|$)>v`=Pcl@f?}aw1 zx$y~Zqy-Iqh4c=6X_*IBhI1AN_(PPS<}LJY?U_Y>6mx9fFTr|xhdQhp&WUGgFF|%K zcq0%b&vPz5hl)b#hy8(W5W}m&xt|gdb{_qA=hy}FmNepeSZZTv?@;CbsO@X85Cx+q zVD+J`rqJwSV&gsZ!{WuA+-4bL<@IS9z#Bu!>mUFH1FN6HKf^gqoRBlhmNL$BU27^j zuA9ICisW%Lb8nu(^9}~MolubL8>3sogGiJiv64k63@H8x(sK@ z_WcIW+s@Ec5d*5L3W`8 zn*YI#YIECXWv)PXwy_eCyF(XEEouEUdpEU2<2uCZSosx1+I;~8+E+JJ%wY`woyzE2 zbQSxLhQ_!TwpwDVzlo6rMxqiIHJ)cAf42SgmvEmrqBhg61*N(GD2Q7$df9UjVU$tb z4h}_3pWk5X_|>FW1rveexgy|HHRb;N@Jjpk*Y{&BQDI6l4m4|sca?h{go@p-U3ll# zqRs;Q?D?~#MJs5S7T5I6iJ&DzOGmm67?1%mwD@162^~XdRw`&$y@Z4)_N5L zU*RNiaI-v6K)5&9zOi3x>{t9|;I_bvu{!h8+lE`+JwP9KO`plpCP1HGA&P5$BdQUrXP_>g%C9j2&F} zom`!W6m%4_ocj=)8yVNo26cXGT_d-PGr*gX?N7Vl~Fx}8TbZ5ZKwm82z?8V+d zNK*F&TNm!OF!D+twt_t{#6z5Ih#lP<5Uw42Z3P<4FBxqL`c-VZm(AR0_#$cwiR05- z))ARug>A7v7+lG*EAwOe$&j3h`BN+>zZ%8hIe%Vsr(ogwK}RCdwvsBdpKbk^mw3~Q z_s^Oe_g{wqV8TNWUe8cr`FE+Ll~3hXRAqrL zFs;A_K9919%F}NHemmRQ(WC!avawC3t+hY!<%k@>J!hG>4tUUrPs>FAh^I7-iHQ6a zVCCTfWS70Rd^86Gmk-9Rw$LHF-1|8D3QrTMra=As%tHqv@2R`uedwRr87ahAnR2eV z4WkeftMRDNe3VB^3(eZ)q6(%RTpLfM-U=VM0zrqzeTdsOM;*c&(mG!(vJDk#pP0v1Fv3{v5F+XAkKrrv?}vuns2!XPl&^<9t2iPSttRq31ZiJuNP z^kfC(6QN?%0b`-GRa0^v7>J-CIGOVO^n-Mb#pO`_K6BE5GlGBq8HgZSoZ-wCPybS> zv<8Q-ws`-zHD=o(W+ zz|L_JAR?oR{^B)#dy_=XRo8^+0@mc6&q#(;VaIr{uC(3my8n)qhhX#(zAW3<<)`!hDj=i&%NJS{>fR1?X@#OIMr zaL&obcwRqW!3!s2c=E`c=f7IvJw`d04Sn?($2{f;G^FYJp{Y@6@K0mtnyy~kF-$4< zgL4{;#8=Oerl#YSB0R&BGdyf>;GU!VqlBKzFir zL}u`%sI4lzy?6nO-u1Ht*!1ow5aU;PU#i-wxQMSikE|-!(4}4=*w}TFh#p0NZhjK> zM_6+I3X7Nr{HA6aN>enqg?N1Wr(TCP{A z25^I~jmuUdX;VmjllS&^GGdp5H~$TnC+r5X_E_q9GGaJU<##=QZ>poTE?J$2=p2JQ zb{%v2?=Osd4Noq_-WR@I^nvgL@WBUpb0TohNz)0M)r5h&QLJen7CpKM+HuCg_3^G> zh5pkGvh$a*x z{)J-M65H3^mTeSaleD@Jr!&*Zr;Q1Ywq+--ER}RS`&o0jr#HQkYDv) zmgcMO-F5hr2qyL}3>8ct3ry(Z7Grg!RBBb=Ac02t`?1zxJ^qD&ek?;Q>*>t@t+F#9 z;*joB{f0>8a%$06Nh9plM0Q!k6SEJ&rFvJe<3ZPzh(O9SBGhs*rjLn%idy2gOFm%m zs6Uu^%-dr;^VC9n&QXi^wwJ2dQ#HjU&;Kh7_JAwLQ;1F{il>LWfumG6=M=2Im*c@4 z(@MYD^I(3QQXRZbDR)CLcSBX7=f7hwnHHmqxMy!HG>H2$3ijoBx6G%NwL;3vaWVf5 zBn5XJ`5Sp7AvI>a=&HXhCZ2P3e0+h4Y=r545Lu9l3n0#a%pJ5Ubn1d2W}c|3tlu2>xRuE8N#jwHOH-5>Q|rD4zV>f%fpBf(byXRgi!la<<;4- z02~EzY(IW}TKqBUS>9ruNYd`su$;~9u0REl?%E_WuH8>FV~EB@JQ^(Uu56Pn(TNJH zPv?pHGZ=Tcbmszuewd61O{lX>^{Ul+{3?BoXl3aU*apJib%^e;x&{)eut&sx@7S7Q zXAUF`b>=sI>5Phvj zG(W3=9^Z|mef>^ExMA5VG z!`}}(poF7v`+_G2wy10ju3Kk5=(qxO>i@wLR5ZG9!{97V@QQS&Sm?p71X2I!T8GQ- z1uk0l0i%e7XAT^-CL+@YLYx5=uepd`;tHC1sY?G&8Ogq9vR^x=E)J=ST401lAN;nH z$XNZ&`cu$xB?2+b6%vTarsZ#9Isi6t(>|h^g^!0lf82g(Vs*9rTpbwd`osHwDSUi3 zI_L#&H-=gyke?wZj*7`9JMZOHH5+qk zo+RS)FIoS7V|EIlJaa%~;Tu7)qbku4KP_z71vLHoo7;+tgx_Oh=h62lA&KP-HR z^?)bHZakHrCrJM8S?x&wjGOJgstw=H)!#&`s{=n>*@Uel@)IfS@hxVjlvdZ?ZO0+R zl=j`R<7Dj*-suXvna;IFL@zyf*kP>*O>FR<0$5Bu+f z?05Jp`x%ft;5j*~TVBSYj)XPvMiGDnkMxEsY35grPKGr)g?g_Mi6BGV#*c&m#d#Lw zLTbZaZ-{kWm7w`Xh|Ts3B}uGZRi^>+SbWxg@Xxvz(-gnNfhHlf2fz$AtU3o6hD+@q zc2aFW?nx__!*ZpC39Ii0D9r_ED1T-xxQGO7%MC9_Te+v(_6!P!5j6XzjkVU0dlWms zm}v`LAct8XhDl3DD)kiD-DUU~yS8TE=}kKBrYzK*k@D=uSLGuNXr zq-adKb9Lvo^Z+2`z6P8{oaYQfIm4$Z70*0U(E*JiIfquI93=K(3I9Z$bofeDN(;|p z6T_~IuE6i}bl-zy66-MOM9<~pO$j^+AR9 zwrhPAO9&xCz91@zA4-u58($3IbR~%=PeEr7igf0dK6Y6LuCcf6;TdBP{ zJB>tQS|wf*jkUXBaZ!o@OsVNnY*x9JX*S6h{!ovRJ|C`rwO`gf+EIw)=nk>;J&dJ&>F2NT1 zVD}{xmP^hjH^)Sp+t|oCABz+?c<}X}g12)N|%1C{CTr=wSO>AO;*vmgufzF8U@3kieE<~gA z^NR-_ib&Slnmah8?q=QDuj+4f=H3VSyYGVh4?XzMYQA}FkNHc!u5OmCEJ`8{qZ^wV zUHVQsV3nkHuI0`z37*4Y-R$#DGv21dYyMvN*p646~4L8!f8vLI8wxpot+mwK#+3Aftzi#)Wm1g!dsAcFHY>9PLd>3JU zucvy4-V`EvL(U~X)lMwm}tfW+QsH25S=;fljr!}LCg9Er( zZC|hZ_oGKG0!qf%zK2&F6;}1>c;%$gd_!YjaKI7CHR^i_+`#tHwO@&wg)1yyfBOmE zkb5{jwR z7*kHtbvNY>u12lC%Sk~HZP}VSUvh%Cp#196>cYQOiTq@gV(Js=ua%7l`9j4^We=SD zLMCAQ#i>l{$2+C`jjWfYfAwAF`ZaJF{j0AjY!l7Sr<0pPzTS=On&^LS+V{T3{U=_@ z!)(67l(?jSY=C73egs*@uVm*7!I1;X6)!S=U69g@jyb57D;i|qEe{ium9m{}?%!~B zpknO(tC#`R4Ya#{{~Y_a`{pBm!SO#&UZtO2w;-vO2^cg~Tgm=gZH2!?{Wbl}`#0R? zL&}P~^8$`I*WVW55SSc!f2q9d_StrBW?z>?9)lMHL6fanoS7fOI5Xws#GgEn3_LP% z($jw=nP2zSW!)|MYgD9si~ymwGv*1$JmcqILXn?0aIzQQu}g)mNjuwa;_0}+r1u{9 zzp1l3>_Z9({@dfL&pK}6Ylq*pi>5df+!;_|e|699y{U7uzJulZPg!fREe+7o8k0`+ zvkCg7>!yrn~Yw3fyAo zn-$rnv@SGHqzAq!C8j`^D=xGG%p=Bf;}}tKouk}1l01Qt`*TFjSstfa62MR)Hk@=6 z8|LCPq7q7yL_>Pj^H)JycE5L{#K6%Fv@$Y7xnf!DY}1jM7y3*a29GK)L|*>kj(H01 zfbxdI9-`v-_$>cE&0cqDRsR`1YEf0GGuRND>f-{YO@U%bFI}Vhnw(rDhik1bbzW~| zFW%#@TXCY9dEe==%pW}`<_=>lUk>;pu3*jlu;q`R_t$y&Q?lHvhP*~HEfp$NLoX|Z zI9FSKZ_9dcT+hU&!5vql|8wM=jMw1}VPeOQ;v)6shCG!gs?R_CS|qu8(Yg9%V85Y~ zQ=W7B0+KrYb%S4&v&;LNmD!qt0;P=WpN>?foqq5+$f50>cz6g!ZE-!D_>zzRP5mRJ=EZyXML_m`MDymIcS^>)=yu|C~*Ff)a%r$YBq2G}9E9v=g07a^rgW zojtcEY0{2Tr-F1ZoZ^V>NEfRCrNsvGPdF5f!gf7CzIG`B=LSLBPRyE*I{ zZJ}Go#w0+&O8#t#t{ra6zwO4+RdcU%+UNbJyb<-d?KL}Aord6p2!eO%f<$F%U`TMU z+Q4>GO(FP8Z(KMRz@%kpH>h4r%-!t>_C7bwyqW8aGb`!nNYxF&a7$?Tqforv(xKHafrLk@|DR|DE3{~d&HINCeT z6E(>v6s~b-dRxgx)<1f3fAoIQni>pNiyBYg)$DPl$WdttV$FQtO6wZ&%fa12Y3%t} z31Wi@tP$?@<_E*0n7E$jM_b^gYrg(|rM#{555`o)E;&xWudy$FXnXngiA zaIQgrwMEb5-7;0~9V?1v$BQi8E5SLK5BE7FlV{O02IU3$hphkY-~KbmX?hAgI*sK6 zBANJ$Tm+RrsT<0PQ&|}UPA{N{GSf#k!3u8JZ5!AtZYymMi#o-6%f7cu+SdL2zZnMN zVu>WSW?@pshEYD*$Th zSIb)+s?3}1F~*!@jcpHJYNxH8<)1Xb?t6D%$8@J@AsSyhkPHvpQ_DsC?=b|%Cz#0Y)3^dZoU^!T5R`6fky3EpfF$Y9LnxoSy!V~Tw(giLP!!nQTn-UZ#a~R5 z`)0K;rWt!IM#iGSPS6Iw<7hC&!ajv z#Zld3sK;8Wusnk*pES3@wvX^4^pB51FS4=a!tTj!^epKu4rxb}ad%5{!{k%n=aN~8 zF-aF={q0O84Z|HqyR;6YP>IS+N47&ug?Do-KaH zDu`jcIc*%$OuimiX>WgRy^$>|&C*x0=_?zRjCbDhUJE?CejA-XKK$8_z(Bj@c+bad zvauHvT)JJ|MjC_Rmt}G>{<->~?!ozeWg(I-7mE??OGe0*l_0tSViCkn`GFq>t9IJe zNo*6ov)`sOd*QhigKn$@runtYz{e^c{cv*jJ7YLeLaY$P`J($cfbRErb}Av!CR_lx zJ=`wU9m2T`$LYt7*DR>#Khu~u1TG5TWxisNaKKDUxQ!IInKFS!XzHn z7$?g-45A|6Zf(b|BzOkG%7-nL6;&OV$CoGmiRwtlEArxFXBUnYdX z2dynjZOA<0=zW zR*GGZP!~UzhgN5Ah(q?@2aznT#{MuJ6L1Cio%WS13#6E&nDB~`E8wl3{LNv?OaK|X zCVhw41G-Fz{iRu`StadOisn<5`wl$^f8y=f7%>RKA`36?w}rO7HP=wtdrA4aKdnK~ z@lO_C#RX>6kh680nA#E2gZ02wuxTV!~tI?z)d|H0|Lg^$o>>I>h}S_dEk!$g@|OnjS1)Sn(EVz ztQICohW=3^e(qJp=%)7fP(>`V*u9w|YqK@UV)CDx#zCaUkU!uuuL*em$XeTUkT{1o(uZSVa~2Z= zPdq|7UtrD}m7#5wyWf*6J%(RDxO&<6bWJL;Rd*}!o1vbdb)YNZBSfx;;dKWIyux(X z=CZFM8U!95ckiAQA~MxP`2Tyw$3V}WP$+rM2p|+~R$oQD`Q(z;ozL3%2PK6? zK{8epKnY-jO>hf9+6yeLWZ-P97+Z-&-8;2Q3~1NCHN!HcGWSJO+~|yGK!o%USRy5e zi!wXh@O;irsT5vl$^* zJuQ?_#iDRxK=6C#et$Sd7*Vlb>pE0x-INWI0lZm31FP-?qRih2dmH7(aNJS0lLC$1 zOlki7wUPTLt?g8)b8p7oSzXw+0s1oz0R^V>6dO%htAf9v&fIf;K;*-X)!6&o){CU6 zQ)00+0bsh%wSEvOaK(A98s{3umgOed_9AzY2BZrh2n@p<_z=APeU>--GY|0|S5pej z=nSO8MN`Q){yK3eYhW+gT{>X-q|sPdyj1h(0H>iNf!eO?%iH|zM*jE!zXQk>S`Y73 z$nRdICiv=k-_`M23!|fp{Q3>PyFhW7b6?@fIIoSkX=CZ1fGIFd_~SAE$GQHI2ovJU zeg|s}wMt5fb(!!mU01RY{^%3E|HkG13Wwk-Sr^zzgkqe2PvOeanWc1+<&_}02wEUw zW9uOEtVvE_q-P%{#m7rod&jtRV(F>b9x}n7qt`u|I*=U2>~5$n;+dF^+bK}r(ttqy z+f0{m8qUY4^FXoXlQa5pVO5NquF%#xm53%^0a5wGKO)Pa%{1I6OP7Qn(KrV*=E@TIX#KCMT&bmDs(y6ixL3| z6)yLe(+7RJbwf9PK{*bM9 zUMJlot8_j4-eSho4AT9gY{5XP{%v@G`%rj$+Vmn^s_HM%33-+??!RGZ0umd8AwP^k zfo!WWIjQx5n`r=3?RxPVgiWitAGZ6@Ql=!;wxQ#!CaSWqPAJ>zn$a_4Sw}u$JYZ2E zDWh_jd*bc)S6L{Z-+BH&kB99g_ipL^^({ibpb`opE&}EiWpBoVd`REx^11cowugY$ zp%yRbrT2(q`sVDku3qY8jeknIIgCx6@X}E}`rC&HPbLRpchO1Jp{_MLV^+XE-oJN9 zl$!0xR=UG(HZX+KZgFsA>XjgAQ#uT0Uz}EN(V*Nm1HhDe;l8IZ2kA8`$hB8x;T}4w zU&5*^(x(5&4$%RHDp9$@ysc0&T4d2! zT|Ra88KT}t5yXCf3FlmsVSF%1@X0~oq2srq-|=CA9az%{rT0+Lj^#BX6_^r-E50~X z5^uA$19ON98Y-Lvf$GmKUzM{(6$j`1We?=R9tebL?1T0oj1XPUUag#JFwA8)ufJ6O zK^P=1E0oEwF4FW=wnp)|DhwfHjU$%->^OWGHUYv(MLrdSPRfUZ$Sw9n^nRK<#QGvL zGtQ^|AsDg*xL6jJfcyCMS`L^S;1wesvetUMuaa_7yWkQ4K8@Tycn`9RQAa^YInsKb z$@q_bz25%@DaE>P1?@ZTSGa71MC<%75AEnGpLFrzz~Qt;%f&#X%BPw!0FrJ-r{`Ap zpaLwMH_jEaul33Ufqg`}{F04|unn&tGyE3Vy&~zWSisE3f1FDhN@}4KC0ObsuJ0-w z)E~U4DZrV^A-pqO*4E?i5>v(hVg3ntf+aa&@3zS$n138Es}e_p(ZklP%3l)>kYHW! zZ-V#c;0nrpIR}4Mc}+Fk{{R2X+i-X^>b!uDhqL3divP1L?6S|S3plNs;~gud(t!h? zpHLeaatJ6*S@A#A{Rj*M_l${!L z%c6Noy<15Zk>}OmG{SBbhng)bk4LQyJsc^EBVFzN(suU~wKb{T%aF%4Iv`P;y&0o& z>Npjv!%ZgdX%CN&&%uO@l7b3Cmdg3QA#0~iy<1C>s!KK$_y32N7&fR#&?-}x@q%_; zmyB1RAycJgsVgYnPUn|}dEd?rn&QQQg8 zEQlsvc$@rUIjzzHeDbt~g7%MppZT*usH?s2L`X6!Gq$k*N&0^*5Hc2P$Ip$2&C+N< zRdeIv6$_*5$WY%F?(z>OuLek#=z81)+H6fZ-Lr5b-6Q{ev#*DFn{e!BD0bQ@Z)`g^ zG@n1!Hkk92n&iX+y^z_Btd(qlwc3w&l)HGOyMZz|A(6HQ>r`{nxpOdS`f`eA{3EF7 zrD8W_56W|3{AUi#8u1}uBE`t9Vu}ykWne+ZJ6p=%Y-3^Lf7w(m6X|Bqs8d9GrA^Xg zdt=vYS~DS=D}7L+*}l(?)~YJ^HD`OS{9*a7Sj?bWb-c zzwMz3k|CQXCB0)E=D>TXwwNhfW@>P4kv;k>lb(|qlZeCS?yN$P^^H4_Zd}IZ>?lDU zqSTlXn`ff=Kd$zy@vpf7P9<9=1Fu`ZRAO6#!);U(N+`P8&Ia+#P$@*6iue)JzPh~7 z_LA6uw8QW0+}X3nH(=U`%CTy7DdAHpQn*L$22Czx(IPID(lCe1XFVL9&Kl>4IM4La z)Tl^G1VxHn;!lrfOo_L@66|-syG-zE5dHB;m1Ko>q@@YDNCuKyBnv6pEF>lEVAP_$ zM@yTQB=oCD1N*kn0Ev{JTEfSoO&{M+>F#x%n%aVCF#Io4;@t|cn4;Yu{2SxxbUi;wRR^8WK667p zIx+FSRen~fU^_nWqmya`-TfnhB+(exNt31zoI#rVw-&8oMD{&;$`<67#m&~mO_x@! zMrg}qn)~n-9Cme)u=VSry0`RMTY@jW#g2ssOP)|gmGj2+E_q@aEn>uY(HjHH=gq$L zM`V^S7qHfNpzx9HDh}f(_toTj*aq5ZCsK0SxfSHxUrcp1AEDw1{N!bB*y2&Ab@?8t z9ZS$4|3{3rPH7x+@Z&iZey`T!ldMXd7NQyMRCavzC~0`+#XX00kUHjqLJD0CWMcV_ zMhXW>cs73*neE8s>%$DB4Jz%3 zQj%m4p6St%z8i2q;pu74JWE z_CIU`B!5e9fIdrA=F@Ith;!%2GJYIVW^dM0h?^^;7qcI%IHo-s^?y$IN>*{ z8u4zPlL9DAs#uziG;NEzL>ATs<}?ytNtapt7K^prn8FNxWi2hnVpH_#s)uFMjP+5i z;emO5c5WNf=zSb6x+%-NQMF^5CXGRX^OdrA7V6sOuHx!WHye4EPMSQgm&(&$-#aK{C$xZcl|a@)pNH?e89 z!|TQd2JJ#0EuUDDtTN>}0DKphUBHzsYYtyHYH4@_#iFDlv5t{C8vaBfOm~$`F&a5_ z{ch{5lM4bl)y#~->thnf+}GFr)i+EfiZc277BdBh5SRv!s$9B#Z|sAsx(f!LDWKd& z6cQXz4L5P9b_H;cptDg6zQ>(cdgeW83akhm>IK*y-)*z!rwR1e+gdIl&bdy3L+gAJ zJ;yPxMmpGdo}JbZUP1>=)p+szXpGanjn5N#Lr+aps+!(Bp}II!Fe5%Bjxs!^yOCvG z+-~Gn^~`Rj73Nlfy$Q@vXJ%>6WLeg*NRJNKo8(#N0@AtO{tAkJ#F#%t(n8ifjzT|r z3L{tfaw5fB0-+)whdkZ5Anq~~C$m#O)bey7j|4{ zxN(3ux>$e(wh$SSiCo<%ULAw2r_o=!}NiI;vcBko*! zj~8awxi+tETyo(;WPRG^X3rX{zr0J&VlFBc*H&0rn2GOx?fE|C-N=+Daw^#>eYHw@ zl*KS2BevZv^<38THzbo#{Q^(=_CVY7^FY2xdrDEVMa*XH+o{qUj1A^_ISV=5L*=Zu zSfr8I&$|Ced7)b(sOCq`TRMbYfY+U0NHxgz{15LJhQxiSAYi-M+%1|GqcRaHp+g)BR_@ES5V^{kw z-s?jrt`UFX46iXhS;iP2FVB-Am@ZUjeJg&Ek&qm3OnDx4596_ z7W>@HnS>tH^R~@)7fSmCNWa6PvOeWYm94{iig<0g(LbHs5_L%?fnDD{y39RG9N0Qe z{(oh$ehPK+{W#fi*RS!o=Zr%z*n~l^VW#ma|@Akyv1B)!;*Zn?PMOD9+-DE46?VyeD>M!{;5HDre&;G94C~C$p9OF!tHv@SI@^FSRfP}L68R9K;NG&YHOBeXMCDYn7|Eev34aGT>M1lItGTd zm~W4F#alZ!&2h&ECC~PD;;_s!E(#*i)#FO#DBPm)EoX1fktqDK-$%^m!obCuX}Xn- zQC<8gsns%8K&uBiPG zs${`=u)e7K%&~5+g;CEaM(>2WXDxOBlRu zbX}PG{N1`}fxyqQc#BV`@Uh|Q`;=d01kf~o8ovvtiaW=DSUO!yM{30kV>)K!87qVo z@orlPq?ek0#vPwwgyav3Tt-J^{4i`Ouv=8#qgG_>ja_f7#tL{VtR);bY4-9csO&QK z;%M*e0(<*`qT}r=(st=6(2c9l`L^df@X&(g`EXHBq;sgd8vnz5Dd)x=TEigj_WQo- z#>*eSN!FoTjS{aO4wf#T-dOz}d}}TjDLIm9m^1m})A$n@V-frV%A47#@}%9@2x(?p zSa+?w#r1oKW;=7v$GMK9IN0^{4cadve=XD!`!IF@g<}z5!v@6!dZ7$+(aFgF7uQ4% z>#<>87kyNh=*4|>>JGZ-*!q&8i*m&fj@%~b!{8vM>5?}RXOi) z1Bxxfxsi(u<^Rh<{|J`6p>UDzX1V9(>2X(f_Ru&3af{Dy+l>bxbGnmJdL)BCK<_vy zZ6FRodmQJS;&-RvaYljGm#Q`3v=>1&&B+ zxfRN*xpO^)Ej~+BsM*X=O1_XB?|%39Qr%{qLd?ZY4gO(pu zRLAyVgUcI%6a57%aE3NuT{FSy=Z393!ZQJi(YC|ogO2%pfh!+ft|m*pe6uwBTviT9N_ zl@{htI51{Sb)L~B;qAfF|5PUhxB^{Ud}wXu;DG62|F6Mbw$eJe!tjM-pdxs-Ix}O6 zBRcSstytki(EqltO~tFd!XX(wFF!CHZBg5_3~n!axfR+M<9S17dKg0{Vs40!dEA#Y z7$Mz$vO`+WX1^`+_F}Lb52KMj_T;(tvV(9p4G$~a3n_kQ<_Ri*b8zB>lj`$ub$ca} zEW{N+?cDg5k|?@uv-({Q<`>lCFDKQCQ$?;QoOl27k3RYSrI3mORo=l!ZD~KnX4bX_ z!?{eUt~L?q!2d%8L^(~GCJxQ(LBcw$WpdZytFZ+K4r>?M$3zqCZXji#87-r9z{Ycf zdw#>mSpF2EJkqFrm5Pg2jh+Mqlu57z_LX+#rxg_HSQ_4f>6i^Q+)g}IMO|Y|4(?@= zI_hfQshR_BHuF`9F%(ak_+J;AfD7_hkU)KOQOJC5m11-vQsu5xH({y!&GKS%Ci|v} z^8(W7FXXPr%ub(uVX_@l?c9OgWby()U&n(rpjq>MAb4HI$XC{eS~LVxYb(4G^;sfc zmoc%Zp~-o$_P9q4(+&u;=6wcTe=)1v1he||Cp1Arri_3O%a2jVON=*#4)t?*OB>&R zubf~;9_O_%Hs8~N8C$reWzfF#0de5e{{LDd;Q%$3z(f5u_fu^9*AYHo18zS!YPFcH z89L}YByh&|pNRuY@QTpy5Bz$&ML|NLx4yXgl~hObank26MGZ^s(7gd6zB0s#IbnB5U zA2>3u1q`8Aos}R2`&kK01FOFx-m3m-*K47f9?$%QQ*gwQj_iB4ZF+w(P}dp*$Ld|g zdF24H6Q6eqpv+Sd&+M4^?{G)t-6Z#XlNcF zdv`MXcI%gp;t#?IAt^yqqZYbr9O1`E3g`!bDb9SP3Exg8gymG2QVLv56>KRIVSizj zvLO%Rq_7;HadP)lFK4X55<3+(I8kM%9MHtH0UrlQYf16|$QW*`g!%yEw=5FIhA8B1?C}p&ydQ6W2)D=|>e6 zAUbEbY1_ITOu%;^x%smv`*fh=26c@g{nx9k7k1HxvuVVTAqypG)TKNfI74jWG}|w} zxu`Wx8m-U)B^|MEw%>yL@C!#R?`Pt#%X!*QSDGe{TB;A8Nlc_$-bK-+Q&{;bFnCzU zaLe}XxrMeTz0gO5g=##2x;WtQBRAVFz`7EI^di%omS(*;&+H4P_szX`gy#$~mP7R} zZgR$YZ}d>4`JH(L70B&0uJBv}mEf$ve{Sn{JXG+JAIpVY)MKizJ@o_PJ@wt5zCu*& z-fv@s@$bk-(}i5^k;==QkldI3>m4c4+66BG#;=YN`F8s+WMGu}{F&0cxpo#K2?G;9 zZHvBG#BJ^^g#A2_zB$s~+<-$OzEyU9bcjNqDgH~LSDjh~E3OXxjba^GLuzy0=wcUC+F^czxQ$V^?DwXRGo%95dLekDqK)L&07{whqsmF z#L~Fwidq9la0 zfMABPO4!^|xitTzwBz$pvUgnBE1zwv%6c2e_Qv+<8nmynz6)1lASF`?VFZO*n5cuS zjw5A>t8U76cy{&-d7Vzec4nak%sp6PW+TPA@jL+1UghOo3{@hzvt_@1w^?e%Q1}nS zg)_14%_CCyrHXp4cc-&Q;f&1y^>Zs7rK^A4z0}*Ra+)MVqLE6r*`%Aw%?r(*nY%!c zOBZhH6!nW60raqr(~kSYVk<_MF;&Sa=pdoNt&Alp$vv2EqR zP{`8&@|_JfwA_*njWPn{GxqaUKg=q4>9F+5#=^6gB{t-7?!#>zADs;OXN5s#DgI1! z|6@o?c5X2jA?Ojk4%rmijxS zmdA@6W zcGuq{qu?QW)!*l~%itI3F?RirKpKnrNQ6JcJzl4FwHcsLa55Lum80tj+Nb9DDCdf{Ij;qi1DY?`lytm)3`ww6(wB+VOr^&`T!9byR^5JP?;+LT zsAYJP{hY7S?A?>_E7apXg@P=`>ofV*_&ClBcJebZ$y1c?M7>DL1bLq?c)aLa3uq`Z zfZG{-i?AbB3k^SqQff}Npfs1<$l2!V>+k{hqjj%xPEc8o@w?KIo+2mnfnc`6L&J#q zyl?=47lRc!yf~X3&6dOR-lm#6XRYM;%ADK}LhKhH#Fpo*4dO(YTGW>=*IpS{rrIfJ z&`;oHM*wiA^IIXNv-E+D9YFXlp&U*INem$9#d3?^FItvKP^?T_26MBl2OB?E-?E8= zR3iu2w{POw0fj-3VNbB#XhjOhcDr*lb=`u=%H4X!XkT|o+>M$?1m5#f>p{n9_6O-1 z*2N_q>+J$NohtN`{$(HvrExD&5N&gpgJ2tVtr)n2GNkzLB{!Uo?sE)2N+1f)4O-%?`3Ht07ft0_L*6m! zTs^P(I-td9x6sx}>M_bYI!Y_9hI-;`ABP$4>kug#CjVq^8VA+0FZoAy@in zsJR0er1=iVZZ@#1B`#m5adjV|GR2N+7u}=^Ory7VU%rIu)Vp_LgtRJe6cExU2cW~K zu--3`n|=H0U3q(#D08aR)JW!5t%f`DY_ntl#`pyla1|i~B-=Z~f7ruj(qy&kC2^LhpRUQ>#f`$d9BWy(9?yrW2ObDj7w9`DCg2WNokG9(F@>IK z2zjwoKjB2_eSPT-`G4pHG0Uw1=t8PD#DR)oSu)m(8$Q0nZ7~&}70QGUiw!3&RK1k2 zC2N#il##$zs>2MWCCdvQTsG~C(j=KkVtcs{;0HGuU1+Waj}&~wpFt{yb0V;)AuT0_ zSz9>IPvB{gKM{`PA@M@0{Lz=P=BMNtuMYiw?>|4ljevEZAlpDw7j zSUc2OC&$s4rUeT~lfT$^i-^QrdO4M(q)ZNBSNENrvz5x^r>RA$WLa-K-_4szvdKm!fsj<9?SEtWVjJP`c*0R7 zc`d}#`C_a8T%KdDN06#APkkdFiK>i^@f+r zkpK2j7?E^$j!;Gn4*?p(NL1rvD{1jRmJ1$yMT)r;B0MPsvOP7wJ`d$AJ>PB#aQyo? zd!5gi^v_>tEsZ^Nfp3_2a85P!yLp54hWmhX8pMA52L=0QJa_aM1m~mtMec3nUo`7t zl}DIeJNH2m$MrI~)788+_zqCqa{iDwV}rtkobt&|;+*cCNjr@I;2Zqre-S5?9E$-^ zWX)DN)Gt}+2aAx+uOtEkLoeZI^LhEhn=dpSg%-*-H$R?pwne>UeCffjgq%!i=UY`8 za7yKXG=C?FEaMH%aHQSX36id-!|mc-y% zW-020uXKa3@ces0)1D+6%4F%lV?h;*j4T0x!T5InLte03RWJS18rK>czD#Ivlm3P; zge2U}u{YpqcY+5?kRw|C`l=J#^<7%*wUy(PfT-~|RmOf5c;KXZ->@ZR{f@B*X^e7B z57?m7@tHQ;Uo2xpbW)z;>=$GebQFI4DpWw18;$U0TPS$sXRO%`Z!%JM3aLt;+wxP! z)U~7t=G|nHbikQ7@ahsl?%|sWPIh0XIaSL;l7HehilHELPhZXBK{_bLoIH%+oHvQ$ zjA(1Hx^;b{mXoyo{DZ^caUf>In7)qBx0nuUT3j9Jr@otcon>AeGeg*KU7PNbg{q3J z+?Wx|#5Fl>78ywB!>-KdYa&npO zD$W#_1)0wTn-T8dOPE(R{vu)3q-BW$kHZ5an6X6RW|>g*jOV<= zvhb5TSPO84a;HHUT!>^w>j;kt@X!>V7xf_;rB1y+Dz+1=Z zAZdxdKrrQdpPyxiOt}~c<`)^86*FDuL!67uOu1PY!M}8X%e*djdGWOwpbGAMuYPX& zy;{L&k{#n)%-NjP+Qw&DZ)$ywsN8?V92J+zjd$&$;82pgd;@$-ojx$adaUJhkZk81 z+#>gWRNqyqbQ3VA2YhV6C0Fx+OP@q8(8~G=>l>{%fYKO3u-j%4QG=dgG@3ja_(`(1 zGLXRd)hV0$5l!wNUrczV5}--7(QwF9YE+Va*B@193_CTe&h7?dm{_HI%X!}XO8~v( zZFd5#?%T2qn-Y&Rrs)>R2*`-mk&|1uE;Vytm?)oCZ-!m+MnBfZp`INixu3ud5K=-k zTR8`uMz#OMni&6WgVNT*uEeOh7r^jzbgY*n8u(6`H znNyYSMpAfSO^nC)Aup|;{C|049HA^&`o_%f@w7((1|N+-5>mpegmBpyZPLmtC8RFr z&WD;-zW#cr_Z3SyKQLrP7G(N^#M?%jPi-@p1|rCg`&{ov z$>MI>nhs^y|Kr_{(DMN6vWF|<@V5NQur)#_Y(9EQ$z3g~x|@T2F`92=Z@kfJ>WeoV zX{K}8sS++dc?(!)m&!QinH&!f1B^xs@4jgEXwLZ|AysroyqQ5zJkmtF(b<_K!3cWu zP7)FzNi9uK|C5Z#BXS+lM&?|BtL1zzmZ&%eF&u-#DE?TG3zt=3Vv_Yy_azk4fBstS z1UtlV;ohTSPq#=y83UkPU0^f4%eSU`Kb~Hh_7usD^)Swe$8rpgS#e4;GIe{~I~smZ zGYX4f`q^2u*6AUHAMFBH9TKLu4eRP#cT zV?$;p18kgOnC$DiV{vq4E3Nne2j*y4@;7*%OC`(jM;V6Bq)TMJWsi(B&--YFP*HBa zD290bnl4vJBv~n4t=Ule}wer?XIe?x7him0pxB#NaQN#eI@yoiA*7+w}seJZRpg4KAjEs kqeP?dACkPJ(~sY?=lXcKbMEZIF7S_%yqa9*6_dyRA85Y)#sB~S literal 202227 zcmeFZXH-+&w>OFi(nOkopi-2k^xk_7q4zE|5K0Iw^sa&;#i*ck1px(VQluBfhJXg7 zN>xOp_YQXjlIQ=NbKX0~JMM@3=?sS<+1Y!oHGi|uHCOH!=xI`tGm{e#5Kuz2)QkuS zNUsnO5U-M+0asdMhN!@wQ+`I8Dg>q7muA7YM*iv+{$4>axTgyNr>N@5J5CWHH*Y_G zPEj>Z5fKL@QotGR;Ogt(!4a2VXh`Q#oEArVo2AyIx2NfTjdPElnc zA@CwBE+8!?WOZ`CgR2VyH=wwH5Ey_**TD&n@b~s};}lf^pCJf;7f_2eP*a7C?3;)*;SbALi9b8Xn(K9p%4${%o3h|J1zv1O8 z>SAhOazX>j#n%t+jlhwN9T3cqeb_%3>2h+_)!WM%&8{KDJ63JDulhFa`LKn{9fw#x&-bz0O4dBI1X(qKP@p)BTao1^B_%6RaHGv zEq}|CDJ9KN2A<-6J{kt%zJ3A0PRa(7P(MEr4@noOv{aBoh>O06k6W;trg^ZAxtOX6 z(gKXEqN}N?>#8dx;j8Yh8v+kOxGB2=MyO~3gkbJYe(Ks@KGLFy5U7(83S16$)l=3+ zC_6~1I{Ls2)!-pIFimi$u6~e?YoLf1(p=Ll$W#NV>}%l$fr)D%bu@fD{hU-pObi{I z{4BlQrS-jp%$>Cj5g{&q`s#j`(*9n4;%Y|XK45Gw7j;w9Krwv-A9G)6gFtaHGYxNP z2|Y)VkYFPhBWF{TvWT;}uuo8su$e1%DaMYf%FY2UUV18U*8oc^OUV!^e?3V(Rb!Dr zA5mo_xJ6V%%2dnG$z0viSk%Ws&A?IG00s$=3XqmU`6IPN)x}g`a4E2Sl$M%@2+Bvw z#Y@=)h=z)}xsjDg5F9FQ}{LcHCL{LQr`#l@8o9^yJ7Uiun-M&M2l zL+ofmaA7AEEmdI;LtPV;iH@qSsT9mBSYKHRr5)hzBLd*m@G#X1Hult!P_xh#LZKub z4Sc=4j4aep5F;VUARtUYEkdM4MAT4TqQPdGfQu^Pz81<3ZYT|eTCfNB2vJit^KcXO zkkSoAB24t5x_FcUmx9?K-m|f;_4u3CTK+13pR9w8V5_5 zn3<^uYnbT)-BCe$d1;uc2m2U02L||>IEE+-`DytC8k+?pG{s&0oz+4DlywYDE!3rT zJWf(p6^Mjb z1_2UGq$RcCT81zcLnA$54K*uaKUaT!M;(|p+}zw3p{nZb{ryD%(pGM+BF^3hfC1*tC_`rmOvS@fKR8%f z+eO_$OTpedqa`N{^nvq!Y)oQuzr}Xwh2VbU)cxgZ>Z_x z;%F>|3{rN}azW}_7@4cMqf|ZJ4TQzi1J$6KC^chKHDJtD5SCgFLTXl0zU~s9YUWU2 ztu>59)DXJC`tC@L5LY9Fr;nxu+)_GFTTMs{A`M4~>$+HJS*od_y!E^df|Yf2L_9+f znue|-mfjMou7S>uAzogt!VWI3l71l;8m5|hU_^6CEeCyNVLu5q4?{04LuaHJ(%-_< zM;sL*Ze#>E)fX{$u+l`Rn+1e8s<|1NhUmFyYeJ-?4Pg$ZzD6iVD-)QHyEaVQR2eB2 zV&vhV07y*O9~r{1h@tp>OmagBEG5`rqWo> zqO?W)AVQAjo@!QFMt;)f7Lt0RlB%9kJ`PZ6X{dyzi-(1S4^T%HlR)f(U0p=g9Ka`) z5N8R0QJ8M9G0IdA;pyq33A4aLqyzQPH}KX#X#;ef%*BM&v;rJ0#WZw~N=3}V$y3A4Kwr`{zzPNNbagjSg@XIke4M>uCMbjuGDOT=!pSfQ=IbfuVc;BW zgmQ2&PjWU>-e`kX!wtThKju(E7 zI+vQA7?MkNLIzt(MD!_DjY}hsz2&Z@bq198caN98O0|8Kmuit8E5qMsM^WQl9Q->Q zDy?AqT=7YjkT~H^Ir>wXm79%m*gcUHaqK)aXb z1g~74Q@gC{!WDFcru4Z-hNgOYw~vQcdO&cCw2=vqYBU4UxfbNM9p%msN~30n$#V-; zZRwpxQLPQc(L}_aO3OPHbQ=qOvu2532OoDXg${iSN9dK4*^r;`gos*82QbaC&xSM_ zM*Jso`8{s-dcyNpOa>mj#tBQ*RYJ{kOc6!-k(N@U^?U^ia{(?x#G1a)3q!Q!v&L~P z>)G#hx8>Zo9aKp+>*TBrzgppWXx|M$3yNb)IR^-u{doG-@eD}SC>0{fRjukH{LkMCKW`F&h>EO@qpNJd-5V?*|PvLlxYizyGm=t8U6DDY# z^wc}K^pj*1@0F<5OT@Pm&!xdXPHRxg*KeIc{-bP`47o@5r{}++Sc(2bZl}07s$tcl z=n>#%HF~-cKdkhzBxjf(vQh-P6*kq(mR`zvD3YeiJl`DkYA9swZP}lZ7rl>+7vDh7 zY_iNp^r^+;q)veR>k(48_}#B`peh3*+H%K3Ldb<0v-2eeR8EvIx{v%Uvs}$VGUa$! zdgru^XYHq#RuM1sUU>0Rs0u?FSM2eEXq|r==tb3fo=%u^&45*RD6N+y? z%;tSEeoWuiwN6Wa!oh9|ux5o0qCbzpnm^ELOC7KI4yi$W3aKcgGH%UP6qlr(x1}J( z!WUJBMb-DHA77&$eZ&FZ{~B#VwbFbdu>{ecKq)_(HM{X+<&Emtz_A(B1AL!n_z>YF zMvCj$;z>&UvdeCm~eFRfN@!X8C%U3?nx1Y1HZY23)Y=MR9?TJkxqrqwOC2F7tO0;BcP$0y11zEYr`fweX z%cq2L=hShASd!@3hUL~^$Yp*^MbkxP8e&2g(lBwCbVRuGJ$0FgKdlO*a->hFafYb% z6d3Cq$9?L?^I&k7VKQdyWVwWZ`0K4d4RC^iRGo;YxP2fA%&(Mlnw`6Bj}3xI{XA(9 zo6ygh8vz}*c@KML3hhF4#&7+98T%vq)rbeHl&3cbEucoSV9UFZ5vbi&9eQNZL(2xoK<58(RSd^(Y1jTypfjP8eMyoB8)I2~Q; zjFt*JZ_oDiLkXSe$*4nEtFf^EEu;k)=k2V>XPQ_Z^5{mX+{82vUc|E<`%qgxEn1TB zEK$eqN=bIucR?sLIUB|PWUqoS3}MIe_H$c3o?IQ8E- zz6+=m7DUvcThlloV82`O+p&~BlG4ItK8HnM8W z8xRmNlk?}8f7*0FjKu^uHE?1~0v_@gdU5ktBEW7eKV9iWV38V8hjF4WIH;+>C_#_? zRgUlHu?CP;&Pq|^AhV$XkUi3&jY|fQ?LH20JQ17j3TEa!+ru8KY(n={}SF% zmy!a=w7KrQ)dY}*e~}SB88yWV5IfY(`fAG>5HP;O9fD<0G=_*rji)Q239pUH;GxSp z*V%)w0_-GTaN%S$lLoL@=Q`OtRWJ$*S5^v687|`nzKcFQd8Op>tf1QW~+?Wth49KYV znxi*fMh(D2BoDN;#<1Awz9c(;!bVr%OQjxY6RsEl$l@T}&c~2d5(CIGl1}3xqX&@L ze&X=i#%gG?B{%G3)c;qt1|H*o)%t(c`hV5>|0lJ!dY%%oeJES)q1eLA%>2?{L(kX? zsBAI+=N(*VmD>*tsy1EoPJQS4_p}>DGiNeh4!jSC%06zQDXIj<>w2uO%&}G%0^LW- zG!ez&{8I-o7#8=?ihK8hRvqM#J+DIGeWJTztNCopF6I`7vWt~zn~RY*8RpAV6fZEv zKa6fY%W;PbXS7=Ru+xUBdcOX|oW8u6A+!3}*uSr$kRf0>Sy6AG$n`eyUg?Nmo~6QD z{lj^>CUJzPf#;9V*IsvJSY_F%L9jT{bnFhJlvaQ(J@4I-jkl#mz?LR))!Zv*sb4}} z4*z4v&r&xy1)Ytabn0_>daF}PHm*-4D!SFG$X5?%1dE;nLNWfG*VML`4+5*EZx<6Y z)~`*@RR5NdPZtOocr`FvrpbDK|8Q}pCAI&;=B6Dmid){N2#!J2#s+~ze5ikdK&sh!8iD{$WJe4Gj_N-r==+T!4LT7 z+8doyHovwKBtwTryc@L(FNInx567D;e5Hj7^j3{~w;U84?0G$9%!#y;;BEhC&TaN; zHZd`Acek@gd?{>!)^c+)qpYP*<+AJx-3$rSgs0iuj%?-GyEgbB>N5@U#gU=j{)*Gx z*%mSLc>C~&FN_P_CiN1;MsB)y#~b{eah(gaID+3-XXLHBVRn!EAkOijdo9&L^6-%F ziBH)A&KfCLsCE-4eVpXZ;cQ>~6)%SzK4F&LH*8w_eNlORZhcVc;?|hT0o6q*YoE3o z=f+f*^nvTlQ>}4uYM$@do@m; zIn!^pyVGOja$xU`Bt>}(eeve-gYS&HADBMOJ(*1IJo7iz)?a2J?olo+aU{BLjPn>ET2BNlYkobYO(AN7@;)(e3x*zTQX-;ep$=w_1jWT z*m`8c2;0OoV^*OoqM0kcr)G0aFn3X2q7UL~mpg4{@d7^c^UOT!m#}i59De_cHys(eH{RMy8;-og zc%|ffc=%}eTmWE5#l%`uc*I!I)^C~V`GoYlA<2JQ62@4LLV9QH<~~$;jN27v%bFDC z1m!m$t|sL6IF4u+m$ICv_;K)uf<8Q=raiv%FgR(kamOlLA=t?HqrBvxiDnKG_EoXi zKkvxkg?&I$;gR8G&F&biKupAe@HgVu6CPY(Vt?lioa#S)S5&3FS zK47S2>Dv;+PI&L!H>GUKdi7frw@6t1-&NHA#k6&0_ueJZlRMZi7*Ke;T(h}U`b1qaxr9$S?!}1>;`C?aZ$?(eZqa9o)aNrXzf+|8ucR}XDI0o zIekgW&qxiYiz&_ z|A700X6xC%)+}f3TBgt~b1|>=$s=kkFHN0(wnFzKsK>bS6s81mmIx;9SA|o>NaD&L zM@3>DkJfMeUVqZRD?c#4p=NONATFC%f}Z`|NN7jm(Kq(J`LCX&OA&`2LpeFG#vNwD zcRF0Hf7ShbI^*oc7YNWVwb@9{J2DJ6?mmvZCrSbOi@#Gp;(dx%bjdlQ;fI+AHA8Gz z0z^rUpy3kZX1xXBWe&;m7rv<3TTiA97_!GS!EOKD6nuS>x z+cs=6%fhOX6kkRPNx?fVP@N48NE+Cl+*?}DX+U6RxmoDv16!YJ&b{kR7sT+?RFEtk zXeI`&PPRt=5f6MhVm0a#XgKuc{0A0kzuWPzyIX7;77u17X0$12Vo~i@Oycq4x$NAn8I3=W-)@nS zmp7EWdC9`E6Ebvjo%M4;@ox0G;!C~HWXH;M36$sBCV$Z~<1)+h+$rCqTes78P>hYE zl&s^3`c0#(m4PFQ!}X7%-fp)_i-{CJP|z%fZ3n-q(@QRRTSCpw{PFelDNlHu+iTg? z1qz>$>Kv4VJ-Yql4IE7QCTqazJ_8Dq`d8 z%2=CohS#UJD+)7tGafxf^D~!NhrbGlePQ-<90Fm_NFh-y(RYD;{04sd$~ZNaJX_ z|40s)7fDa|yWs^jJ^eM8!+!ZN`>#s3^{_S{CDr^uR!_{dEA-gb+7-62gc~fUcip7+ zT2Z;_sHiwn;*?R+Vx}IL<(tfCxa74r+m4Orw$+2_e;ZWZ?w#@OQIY?VrN1z3Pxs8Nb|4n1s}?BD$EI zJe8XPp@@9zLs3VDB51dw>SnoD-gIBKDIASEC4>iM&nq7?QB+lIBj^X%sCcR16oRZG8~A1@xERsQiibdtW7 zs8dz#bZQH51rdkAw|yxo+xjYNOs(RVCzc1Qrml~|;IT|4jql{z#>vx0ig@1Kq=K;t zIUoh_!6-e2bM=uSeB#mO4J)exKC8P9Y4Yd3N;BHgi@nAi!>kyzZVyNUG1O1itk=K#S&qKl*7FQ_(7U80ec`HD+}{@TRR^1e8b#YZadTg*s`uv z?)U=LtAKMxgh zy_)c8cXh2NCWfjZwo^j3$InUeg_{{s1Mla>q`MNlTII4uo)eWIzOP0Ws>9%B&u@`B zDejlPC}NM8%Br1L{8ooohW$I~{Ob5bys({ivxPa>sDHdVK7nJAogS4Qh%8mS? zi(h3e#=2mFdmlKD&e^@JqJBEq)3d@vl^wp|#9u#1`$$AQRPG=yy*xF*XqVNc!iO4b zL!t^nz#fzy)Q+n*MIVtl`>-w7-iZ8u9ol8>&s;kWr?N#3CAJ+JvbRra=H$0rQuOoc zb4wtnneW^*=hk^LUHzM!#*^77Bgwz#fQmAe&kbSiQ}N_A2M8`xdSS9%b#dQ62C%Shd=0D_h-ms^(&8-*}R&+@R$3cRYVvVW`R> z($Y8Ea<2AMhc2K%aJ61 zvq#FP?mnW}(|?rSkz9CWpxgDiTm2|cL^0_Yn`La2-?FM(h12n`_mmh&2p1^!A2H;} z#>1YQnh!+BT^9ZDL*Vs1xs_{^U7G?x?EZ($XL(6kb$0NoM^^FFqu;B#Sodag%*SM` zFE(fF-gwo3S&C_qG>=Du5aMrsG)a%|j|2V0Z=Z`_8CGw-q#SPAd0))DF<)-KdE522 z;+Q|;qEJ}SuQ~VLuNrT2f_SzYx)#d~KIpx+JxaR~(eDoJvAOQBbg22qd&b7Z=c4PE zZAe+;cBY2BtWD3kNS&YKLDjRFW;{MUKl2`SPFwM-ZTJ43BmN)(@_^U;gCZBZyduZ^ zWD_rKx(PwwQ3lEz`p9iaojM)8VK9@9H*R;781`5nz4Q3c|MJLjrWD1laxEuEUwTt$ zG~(x%X}0&_a=L3@dTWecO0(!6JuPTd&&oX1G}M_t$rhAA{h!dA2$$?F+p!g&j{cU` zgZj|j>!fhDxm@7KSa@h-#KANpfAcS+zi*bm#6-C>Tp6q+uW4+rRe{a&iXEsvP(X(w3jSy#bTIL?mzQm9fh#t^bTM@D81LQZTi|gW4+ze}6}l7`Ut{%fI!Qmz<)&6Fn&@AOX4P zH9_u^TvP{0=|4M6VAHl9>xsO$PGuOj){w-t6wsT`NB^UzjTP9xxkO zxU_H6r9yPk|8<+>2utK4KYv5mfaNf+@f+*g*g#@%$)+%`>4$nUKgmsyV{({?5Bv%Dyl>OALHG< zEPE`^svG>zM(9+^KF0R~jBoXvj&Y$OuN4+d#4L_@%dKTH+B}U2y=-B&r;*J3qUoql zQfjke=DkN9y;=IS+{zp_F=?wLQx%jlW~9olyZ7GuL4;989ZVq5VdVAIY=fiG+H(U~ zP$471^Og~~r-`_wmOOq*8-CI^Tds} zNt}Xa%B%dzSw#KTHU1xPnjv3OeYvolGyA~<1vlS;F9Xl1L)8YqP=aVa7akn*dy(fl zXM?PFb@N!z!nFPDT@OUCR)Ro{TJL0k&DGr{+Wm-wtmIF$!0!I}ZPV8aGP zgTwL~h|~+_5Rp}UD!WuJ72oIC0DtnP-yQSk968_**v28l^M~%^s*?S_Uhu{3O69Vg zh)kBVA?hR5wersrL%heNr#U#g~6=|8GK1#%3*JPu=qMg~|`7T&l0hd^{uNmD?*} zo$8pi5%LKJzpydC%vD>Y^mel%Cc3Nt#*10;gby`E1i)NmlG8l%=xS+=QI6KGI@nA4 zP}Rm2S;Uha!tzDQUVfua0_OuCUQL-k)RbO%pZo%K)X_a^17e);2i0j_yO+Fka=e=Z z-W+e#jgk3=^5RESGoW?BWgPXJ#s#G)RcQe}jc*)_9Ga>1e4Ls$L46 z;>1=ahQj`@(w2?r*1e@>^+J}&cb+%2N6r#|$o4JW^=w|?qux|+p}b3ZVZC;OX8bUG zT3u$w`vXy|OPStdkxq{syL2ok%VScyX2(e}%fJ7YmGifb zbi0tFrdqK-%i$R?mv~mETQm~JDvd>XKTrqTKmBdjlY7%)3OsT>G^h3ZX+R)U_dB~` z_IcnqbK$|R7tMK2wG5`Cai2o?dws>vscq#?ggu`Sk35Su@d!M6o7BJKdJvGh6;JtQ znD+e0;v4Hahx{q2Gla(X&oD>ht7xRy0**>i*P>O|VS;UIYFOyXPGNN~VsExFp%h;K zJ8|IbcTlc)KD#B$5~Al3u*9^_l0(@Q5xKQ?gLik|^C12d$)W_btt=~WH2dOgnd>>j zTJioD@rj43>S3nxj}J9XrKOpU!@i#&?32p2Z&$_j9`qC2Tp+EAez7&i!g@9gTQX_$ zBpmIZFSQYE?D0{-74iJQNP-S&Cd$$!ps>4tbcy^d$2Xu zkLlzqG_8O2VrRIzOyfd;Jjvhj7KEbXQZJ0@=Sk29SpW6pdL9^#@^>?K@P=EjYsCdDr`fy+8=xNOk^zZz- zA{zZ$=9n4*&?qvY!f@vl&T3u9mZq;Xn^BeKQ@1T{5BXAizvQp2!?hnaagS)2tS!qUB&XaHeX@v{ECIpiUqlFlH1 zhOie+HjJ0&4p1cLF}pr=5Vo`b1O0Mog>er73RNZN_fCo|Y=-fz4Y*BiqDDqQQ;m0Y z5%nYx_HTL(3+5vUKAT+8#R8X0+FY}Xp34fB0Rb4WWI98brOO|EGR8y+Shs&ce_Bru z|Lh)1WW|#9E^CK&1?$N=&tF+)!S&6YB6>^{y+uIC#9+@R^!8KU?1`WRT~}g{DZIn& z$&KA(y{{2-e7lxb6cY{Yf2s>9c3`8XH{+t!Z6yytc@5XQIh>rONPvxKcqi$Ca&8cK z*I`}3{XeH(!tVPY@)LlTxC^9TkgIMoC--Dyy9UHoZZ6yeTgu!RE$ZVz$v_a1LM@ht z?+aQ7y$6rSetZ=nvZ4bZiL=`shk9Vsyb5f+XjyBxQ4w~4E(V^lyYeS;nM;kO6i1*S z*bT;RkBIQKD%szDpo#0eTQ&jPCjTeGRc_wQt^23-%p7datpC29%KkMSY~{QnTfr@z zG+!6&Xb+31gi`Z>4H(T8vw2){3flHaaB)>@D1fagl7b+`0lX2-luJ)KUKYqYLF0g~ zoP#WAK6`sdjP|&1Nl675B@Uu47kpIK1fVXEl&m@%-4A+AJayv-@M`A;p2wup499{E zr**}bH3c!DpfmKtNo11I~jo{tXt3kSxDamQ@tzc16TL89@ zBL)Jtx8My-Dp>6D$vyhmMiG$eqf`Lr>EIls*g>Ojg1sCq()7-gMzScS5FnyD-Uv4) z!dGNM)Pw2NWw>q-rD32U`TSR_NTR2}NdMX8Gl*{u{kIx_-K!-a_@8Qa^tSgMT@4-J zSEklg<>To%I6%vY%2%5+INUJ20O}Aw)lO1&R)uo8SsQc@7K^_>$%`zC_cXvBl8 z)LtW;(OR|y_NHfudy@~_YUjJ@b{s?~$rD|0FN>2Iz;`p6kX`a=D?j?}TX!yQ9s~87 zdBSVzl_yNf_O-J0UQXn9`oKgZg`Av_S>bTT_J8`iKgIw%$me`V;)+lMu&rbCC&u7^ z_V3|(YZoduWUTU?LP@T2a1i}CzwQ26Y;^bI1quY{vJJ1VW8yT#?Epo2083@l$+NP3 z1^B2m9lB$`u2c=oWl<`}4Bk0^A^&gftLZ~hZc>EJWQ_-tX(3^KvY#P=wj$;FisMt< zb~r#j7JxN-{!TOleRQ#SNBuv0?@r&oCnZs~uUH>osS_URA0VWnqSC4x?T|O#`-Uus zxA2D#wsU{5i{}m=4IAK^CapB?d|(8wt=TzU>eyetnnRij7ZEs{O%$!g1$cXY zY?6y?sQKICl>Ftr@>`l^-HG8njk&m4(Y~*bZ){B|hu>J?kC?)BQbc=#VC5rh%uO8N zf_FBcIrP7FkzJ>L)muNxRqVd=Tt=Fi(}8^=sg>kG^S9Wm<5f}bsV)x|X2{;BA;Ya? z8`~iq$48e3#bW)LtK!5fVApGevB4KnjQFW_lZS&>l7^#~#Ei3end73!_FcE7!o`Bg zJ4KdJ_{L}*>;j#K-Ksu-(d6_Lp8g-(5(UWqOJ}$cZw&MaCj%q#vxfVt;6TItg)mFB#1>^YsWp7Z^aV!Y=(L-ecx&9L}YFVw5o zR$Aej3!8`7H_7NQyZW6;^L21I;mhp_>(x7qC(iBzHr`A%X!kJ!V?Ka$9w!*U8(*-~ z|CcwGAWCXPzFYe|6B;mA`FMBpPkN$8uQrh!?FFzEk(jYu_pmdOImCa7QjR>!02dv# zDg*9g8yGWbfjF4RkR7{MHV3S~J2T^p^Oh-y=)7*${-%nj4pk4m|5R(Zz8dg);xGBQ zpMBzfEA*#Af6;&<`k{xXo9DdGN^Wym(0PTY_%6p-fbb>Iohgm$1SQ2XR4EIq8+3AM zc!^mjzZ5aK+{CASS4glt#hAtdz>w|?{V)~+U3z1SPM}+zQ4u>8hmYf}6b7(r%)gR~7qr0oEUR)JbTYeiimh9g75j$yYMUmx-D4%J;hI}cYoERV zyB#(Nv{_*5YREHu7xWDP_cj_nr30))a+PcR=CPVPL!u_v`CJle@R`^%LeV9i&!hI8 zVthZIW=UB1S@O4-6snE1c2F-{PQ*gS|KJXqwE3P*tlMw?a&ux$HfHbyc_g-@b!3OL zejOM(t;i%d_G4IT04ygB`zO$I@KKix4Hrz(POml1W3m{RioQ1p=qanT2T%A$kPQ+{ zTwK!a*L+a;<*Z!V#qZ13Kcf;zHo`ju$)4exn#Qrjr20(h#sXr*oTDXg^QxW(eiOC}%Oaml2QDewXv>=~$&SO`EwX zKrvWVsCcwm+?*kyI=&y6uCV9+k}E<&T~Q~8=4KXXYnS-N+}-SKztkomOMMZi!fXbh z-#d1Ti10f$7{)gtM73rDq9+dUsUYe3YeH_zgPWqmI?{4w`w?g}-c=!#Droy9en0cD zp?SgkvY_nIpm^b<2q{gRJ`oTOfg$f7T%uKm0eYReOvR4dVcDP{O>n+uT}6+J8jY^n z8yEckKOFAT(!B~NC49m7QT+E>;`oej9YlJT|C*%Cz@XGU&I?94Qq1^XWil+oE@XB} zUk3*1+dJCh%_1dj&}63tcXY#V9kCf)jZWl{zY34ZFBPxXNY@*&#Hc~WwRUaOnKi6& zSqnB2URG*Nt=TK%unzUGqnfIwR(wzuB$Y6h!SZhZSw0K8=Kgv-$qMZi9E9hu0eg4f zVE*Lb6UaZLu9O$%XS_Xi;tvvtyqM&7fsD z;{H7|jWZyvPZJYwWy_&HbNTu#DMS-e&!N8X>>5tv!2vFEn!o=#z(vZzhm6Vh!%Z|| zfzg8Gad=IHl)uC8qwLniWk9{Z$K^ z7^?6u=fCxyKao5PwiCR5O?3_rNwhnK4g0Kp@}k+09p(Vd^3Yikf8y#m-BR);EQW^I zVi?g3(8bDtCZoQYO2I#Z#U4;}-J7netrX4Ii=26%O;5%{DuJk<1g%Aiqbo{@j8I=| z_)Fe~>{hCOUVbE0CYOl&pANudTm6|hL_0(o!vTSsN7U zL%MDu%r29AAgAuAF{TT> z_^`@@ll$}6iZpE*8_dw8CpDA1j4s|1Taj&x5ZLl6b|>J{TT*Ap1P8?RR;*)LbCePAZOLDFD|>%Fi9_OFQ&DT_wHD0P!uQHj{M~%hUhO@~ zM)#RxayUlKxB5AHqj&}+F1}0sWP|?4tEXd)fFH{3KL$@ex|s7a&nzJ%_GRF3U(T#J z#6#Q40JHVNo1YTzZorhS{L#G}S=VQm(0WMTDrpVBd--dewgR(4`crmlm>|ezY1|O=dVk<4)-SoJK zfgn1cI4MQU1S15A)Y_d2XdLTCFFI0}Br#23{v!^}6mq<9RY-$&eXr;}s`e{HId|J+ zA(hXW`pGEtO9*aJ;+PH!EYxiv*T?TR1YIq^o6pV3&km1-4206{e5rO@KwPrE+n$_#lJlN0Y{cGV9fvlZ{r04xE9$yY#YTG-owgPVd*!{Nb;bn0L#L z@*GVM`)(QgUY<*^%IUjn_!|nza65w|j01Rcm6*YQonSSvheDEX59~eQ$Y5$K^wNh& zVi_HxK#V_dcg)>vHWFP2@fpd;4y4&lVoooCS%^M6aNq_P_M*|H z8-3B`CpxsO0i4o*j!3ahL@QMt{t2ASlAsEq_VZ(Bp+4t6w0hn;$X#jhUy{*hJwX9U zxD9>ZWVZCaZ)bM~B4dYqO?{T^lfeCYw+F>@hdI^^0t`q z_QTr5jgc(9LB~UQDPVZ+e83=h&_E9ZtE7sxT?p_NVOc zrzd?Wm!k0n?jfw!X9VYc$4TQvp;B@=frB+f@wQ^l5_gBEVJ)Srs)Zru6T167@xSaQ z(cde`%>0(P&JPZM{^MAky23oez2aOZ`>Z;#8H}c%i|K;>~Q(hV(k!Kio}A%f9Z-8 z&+_AZ{`Q8EYnce?wjX1rYCH%+FV=X<0pio*qE6311 zmn_ck%rH8U0QC$qhqUqzsP-seR$Pp#lzRzG``Pg4 z1w5bFn4(*5sgruO4^eqC5M@wCdznQmLhG;F#G#BKy{GV{2JG$8^`!IcvkZ_jh*g6C z|Hmf#N}n!93Bo8tZ|M~) zZGvfOhh{Qu9=V5h%x5=2OW4|AqvZ>nkYue86HL+kZH>+ol>hq?m=v$*c0!_t;-A_O zDwu3Zdd(p_WJIf81x-2juixbWx)M%QbGC&vn;mWl8UJ3$^rY$Ud$h*wSG9uQsUEBw z!t!2(zhy*MCFeJBY}~U&=xi&5@%^^k>qipfV0{dZ2osIre_bI;>42C=iARux8(P_L zi%WZ04Z}U1bvq9)Z&)eLB`18#v2ou&EC0|`Dkr(nLK&lr?oV3>#)ISFgJwLflbt!n z3|(>omN!@-p&HkPwpS<9e-xyL!tc6~gNU$*I^`|i#zm78fk|1t6Oclmy6t2A%+n&T zC;clBB#0&_o=>pPU>UCo;Apa%+zC?%8$-UY4OhL{%Xwgoi451c#=cqI9z(>d^XIbc z;;V8hCVH-iNNs!y6YUFTHsTTzx2iIjU_+}Q5tv#k^D$B}ixvGFp))j_nsX9q`i=g78b;tP?|CK^#-1ZqYj~R%j z9G?&VlV}#{v=;RLFa+_G**X1i&O0d4zCNe>bb+@GP4U{1)LI`vq!h) zo)4R3IEnQ7GUIl7NK1VRXEHrP77Dgw3I=T)g%sG1)iY3oqQ9 zZ!OT;(GBSCn&Y~pB238Hwz_OBp`^d0e`z^oKx{FZ2o?(ISt;X=D3 z5wF%Cp4r;+3vG9+xxdf9jL00^T?lXg@E%>#!ZgnO_8AfH&14qu9c9&rC&)e&1dG@~ zUBdVOfZS9G2i5sUH>s=-lVABp@4RKAdp32mIzxIo8Etomp35S-{o7pmbWg-c9r^R1rEgTFW6wCPh?Jm{)3JFBMy>rC&+BXcv=DuHLm+z`O$j15Zd4gzF zmWfjtXZ*2}nQ}Rro}-EDQ*(K>V4IhORE;xnO^SbReila;z1^-ZWb2l6#dG4pmsjnV z*2LSN3ws8i4INyPfU1Y>aq{WJGx9WAWpN=HPISK#8wG9pY9!z&&5a<0-o8bGGQUaB z=AR_q0h22~CB>@{R2!k|zxcU^t57!)p7Bc3W3IdMUrPcegJ{0feDB%!#V_z4`BmRflsg8hcH8YC3f)V zgY@cqbSIUA6zmU17^M)U;}eA!4CTtzZof~5W^;SKdKE3hw|ODwFHhybn~rwb;E>z8 ziiVirA&OWAT;Tx=)5OClde@QlBMH+cezJ0b8@`3tTO^<{T`Bg->g}mRw1j42M-Mr8 zb6L|~GBgZ4zU3&&W#Q2N7hPii)JgQ|@Y%a7?Kj(1Zp|*PP}5)~5&S;K`77HGy&Uo7 z07s(v#q6NTXv#}!BeE6kryCl6WjN}6RN$l-%rdRL8JmGx9z!DpcK9ygsW=wH@8<`_ zGEX5KAe+;h8k1RL-y6@&yiAIsXICR8Gs~nNFS+4COeUyZ8L=?(y^g~buHNeFNKD2n zu_kwe;)9+5kI7?>Pb@jNdyEZEOetFzSXaKZY!yCpL{qy7zR;Aq^<38p-bt{pfSROn zSc#_mDwm-aE7$UV5v`Y6Wo7ZlAvWua#Nd>}l_h;$Vl7%X@uOMjsf^(b zDTrCTeFlBh`ildXI>LXOfG^7Sry#Y^%J$PE^NYa^Z?mfxUhmXA;hlH&qUF^oZ|RVn zqqBBspF@w4<}7%7ZqWR>0w;z@q|!uMti&9!@V9+4nv0-Fpr2AFzj2dTU7|BjzB%#W z5E|b8>*XCLKf*)QCN$j~;Q#Vz`pQ%)nt0 zDScQjyE^ojS4h7^iT%2sTdDoi@`?QCW|8Iw`Cn}+RMu<%3?(_75P^VN0C^a#7o=ZsHrpc8kvO78%Mse4l0%?j7k= z+!DpF3@0W}hT$0YH`YDE%vZaFH5{Fp;25G1=i%-TQ}_0suJa;;Ml_q{15-Wx2bUJizckk*C(;b9 z(of4VhUzqHD`a9fc|Kx`iBF(RlqaV-T4hgNWaH(NvUtu!Z2fb&dhl(k^>(Z5D}NHt z)20pdd~fYz)6<~!lB2qX0sj($e7jci*MVr~@SvP2KmR}MePvXYTNkdBl*9%B6{Vz+kd*FDK^g(2Tcn#U zASK<6bcd95iy|%Eps?wX?z=YXiEn(j#{GML9LE`Z4Bqvwxn?}`nRCq-@&ci4Q}b2; z=RX{6I36^Ha`Tayr1Uz$1hsZ*Uz2`S{Wrnb{Zz`NmseQQ2Elx&_2fzg-ZSEQ8hQF` z;D*GFPQHz=TWuE^mqz(kbVgnbVgW(76B*Rv?_3gp@RH&I=;$X?b@(#~zgV6Cib_ZJHRz?I)9KyAwI9erDAbbDj>g)>bk!Gm{BD4#N) zaAKPXl!2V0o@!DT8xyx=_w$=S1y8ui<>h#LklRORHNXAJG+qmkt90qe!vkGRp%r@ktrCo7pW?sux>x)nlaeE zbKvw=n%Zi5SQ}q*-ykf5}z zAbIy*KHFc5v#7KPBzIwAY{1$mkj}IplaA8#zaO8+Z%M%`JL-K~?%(?c*UB+Yvf)@i zQ6&QZPiG+96Mw-NCt_B!FVl3MsG)j5x0-L5)H-sVVx_ca_>x3!y!d*DN%hriT6>%C zV-#6cwS}L_hzO-9lhsP;xV?_b8XsA56P`bm(hltm!q=jI7Jxxy?98aRqYH`%^A4xz z*F$nzgrR;^6R0LhcypVgeee!4Jzm*e z>Uw`15Rei5+Nek@>HPEh`5Q6{ZCStwNeX>7Zzl3Dsow>=u14Pn;7?*{9d18y9haj$ z+&dG$|CI^NPz&Tva=BrjuEqL9n62{EkQmX=gnxMt&xXI!1(B4zQvP=R(AAi8m1`d zPhW^MC_$6l&s|Bmi5dOS`f_4hu9v{WEt^2zXg98Z13R9hcWIOleC*C_CKG{k#$q9U z(>;$7G7&K&`@Cjow1JrF%S~>}fp!x72jyh9e&^uVB^>Iq#4MV2qrIXVA8|ZtDzgI) z$-?tmMk6yM8)z3MG@iB{i)nXs0^PWcKoM2;8Z_jubPEyn%0XqA+3$~1#}sf zGB$#R?g6`!5+yW60*5x=4+x zr!sccEV3xRh2*f)hRzQ~&3nhff@uCg4zq{KtZC;$XU*=#g75-+m>&zL_pS)=mMJOa z39*^G|A={plIiLn@3$jH>f)HtRxm%jW&-g|^J!aSZELlMR9S$`bC6@PgMqJqMtiH{ zlcye#|9Je&M>%m>gGMm2tF=BFA1F$eSWA=GqlVyF&l=_dnDxR7JrT8a4hFOUB9v(D zJ4((vvM}3&D=tnCwFXl%dWrx+NfeaMhSB)QBbwg4&=!$MOP-PA_8{HGu)K7#Q|*QA z!=n|O!#QtrOEyZ6>F)2j<~jj(zk$>>9uks9tlpSKjVwZI#qg=Q6+A48)c2zI3!N~hV z@}8zwRg&khwmAlu&A7_;Z%v8l!AEnP3# z7Qyv=VY^P0-p0O4*hn`nOgP;Y@vGDa5u~J_mIvc@3oxoPC+nDwT|}Q1VBQ+$nXa{< zO7ph{eSDEdOgO9^gMIDU>wAcV2=2gecTu6d-TEiJ<-Iy-5*ED+YmO zMN-P5IC^{gMEuZz@#K6wfSjRJ(XVhnhYNL;QP}KKVBE4CeWP5{h6wJNx8!iVBOYQADUW7{5eS)iA}q zxIClKD9W&o%u;OkznE2yfL?@!!57RE(^u$3O_O#N8T!6piOPB7^B*Ai>*y${^evtU z;$xp*Dd5R+Q&aUj;O;24U_byvTU|qG1qmHIr4a^q-jrzd#nE;%kk7C`TOYpbk7hyO zM3w>d+4s*LA4<01K#iyztQH>UURi6HaO>*h65P9e8RGEm!kH98g?MngY(mp9jIj|n z?W9G3U3)=z|F}FAYiaTy+Cw&9)e;kl9SNsXb!Bn+fbx)+~SL)#Xsv_+p~#MZ7yza z{8h`mej`eHeT&LDrZ3j#8ZycPgN1gFftozm7nHujhE(6Onctc51tWaZji3&2(pRP| z`~GYo0p<9UN5qH~;POXH5+Z^V@-hSMlK>1HLVC1aZRyf3V~zl#d+w{)s1bnBp+V$~ z3ML@cfV6G5$9MUH)Y+G11Ru5SU##;6Iv=f>wzP@NsrF%_BB04y5TG-ZGlZXBpggS} zW`ds#3k^d3$w&}n1B_TT2*gP}E<)8A}!K~SR0rDym zu`ub0TM24#j8lPD^dJeLHRo|vZ>&z)B2v=5cmK8(sfregNP)Ov+`hF>DZXZXgjS+0 zR?w*SH+ftbv9AYySe)kc(J)@V{$MBZRgp1{F^GwZ2+lB4u^ey*I|F5aWXf=!RMdx+v?FP zh!7Y-HXMg+9S+I}!BiJ%vQP*nBHFOdyi;UojJ=L?wi}hE@S9$(ugkT-W281Cpb5qx zV)}O8RC6AXRaTT%(ttWq7g~Agi^KRIIF$VC*9w8?6hHtPZbi&&gwpXThkKgxVH^rV><#* zQqwSJ;RpUXF62Lry1;vYS1>`|imLRUm4#+?QB8!9O-Z}ber+9a z1YRad=n53fVlD1audeLmOxAG(V2&vQn>d}YGx9>&n0r3AGf4A@FW^eMQp6{YvhfZf zNy8>HK?MBxq25?MDa2aiax7}T`VS2LlreuFKp~(o+A$)l!}rD1e9~ut#FASUBAH_B zE3z6>f z6BFwQhsc1&sOrCvcFdf98L0+3Yf5oDz;stt$Z>AgRTR7CknRE=iz@eHnlEpG8cck~ zFYd`{(TcppWe_TJsJ1+NHuSB$i+AEy*tPC+5R@62OLUgBDBt2(TK;wU`_IDZEv^e# zbfmIt$@31iw}G-Ky=y^q9Qh8-m{{Vw z7Pay=pRICP0Tm_p9a+}i1$fm`h#I9%%9Yu&R&{oY=jEf$@GsAXO#gwsaZM#!MirVQ zhCCuKec@9*!dant8f!kFu)j)YK?>^s&<*ta6sxbL4`;?nBDeZ<(JDXwRh~2kJ0U#v z!#N8#p6YnhhIBUY={nR5)Cr+Vr%9XBe7T%5^Lbrb<9WyJZ`1()zXZq=`Jw#COX}60 zp320x6mXtdjkd}sOVHm>MW7Sjm7+upMWTAnXqHTwgyp1V#ni6(4SwY(@q@7T2kJ8U zq8slUv$m;bh}9~l*x+X6E#%T=hxex!U=^`ir#C2)T6$7?NH#m|&BI?I_?L#=fIh># zg>K?Gr8tC@Z%r2hC_6nA$2j}KtXxW~Wd=l$-3pDMkVID&U$+n)RM@o*!!Op=MGvG$ zW3qFdK|SsSW8ezNyPTPkz=dWEeGz#obt*H|7ipw5RFPDCrhOJ&VmH3I1;OWcu&R0 zn5^66^iS_e9Z0qdQ>|@Wt!u(ItkPU?EVeI-ssz>#e%8S@Dn{8F(sJjXTa?bZ#%3i( zuE{RC*<8J2;1>q|c#}##X5Vh10PX9Ci92fw%{KzkR=IL+dZFFcyU>3s_p|lBmwkjT z{BSf0SQHUb7*@HYO{81}3ya^s{-?$Wk#R$xQzcyWSnz(G)O{q^B%m;aYP4ZU@^(V$ zSnoCC0=yeN-5naV7;`c^)etn!$Ie=uoDrbj!w$wWGb!o1*Xk(?N-ukr7I=pqFYeZg zQ@rUy4PVZE-jZyZAT-`sMs1hb(?A+*c~JRH1pyHaS*qnxyMs>_qn|XIaUj!wNVgDi z`E80z+jcc89v-zUt2CMwm4|Xs)H3E#Yves?Rz>_r z>|YNOY2o9M(W+aXJ}%W~VIXg~ZHevruUtd`H{cPkW!+ChI&}03s;Y?LjGbWT3cVkg zN0=CAdV~0%1@tp8(a7|nLN2@JQVVX=+Y$E9<+KCU32)t(7ivL5TjpedQRZRKl_Lw!Y6BMl<6`2 zSDo({oOr08ipWR3+%oQqyw}}t?DqwA`Ao9{Q3O2a{!j8ka!S!Ya8D*YOGtBfO#@z> zuibHA?X)>(&aTW5UYUkyliL^fv)APjCMA|>y*1v4|O4nD;T>e60|Lh&O6l2`{3F>^;?Z zKTo#>W#@rwsb=ZTD@%W;>}A%$dSP%{n052t29k+r#E+t(`_Q@WNv5cn5gbYjHN2z; zE~*YWfO$3YK&Y0I!dXAu^raT^Yswi$Gw&=?s^^re7nQ?e@2^8aCyFDz&QNJ>mMkSd z#&sd?I4hL=I3Vxl5ZE=#7dJz)W(Vohwm};tg?7^a2M| zluACXGBc|S^epV>j`0Y)!E0VPnA5bY*wR-~_C@|@!hgP{@vFfO8ib7Jf)PG~OF#5u z8)$_B!lEjV1069`9}`y(k>`?<0@;2ZX?hct>VSTu*@dyay>|+S;GX2EW9@N{QRA;$8GM{C#WsJ>Ub>y`4LeL1eK<$JHC6vakih(BON*-s=EW zr*xouRF#u;P%N0yuCl&+C-rKM&< zedoRPul(@_O1B$a42Vz2S6C4VF>y;Dk=##w3zh>SCT$T*Qlt>cc)77H4o#xOqO&{> zjGR4qNn;9cs+-4$_ET0x+81Q0dkoRk{Ttjrqg0xOOF}7v#7pu9R1O2Tlr>-eIU2Go zP`IL9LNJ?*c;|*=O;VN#=9x!!)>8!b^sivE-i(N6vRY(jQCTyTR@K=rDv|!B%AdW3 z%oGtm+!MJupARmm;t`VRQptTn0NGRSja@1QTEPtoiZ`SZ5dzUv0}?}1k>=8Mx}fj$ zx>jkZI&UuhkWxO{-ce0kWa*D&uVpN>j3RR_2UbDsVc=*8fFS5OUxNNIDFyH2$mArN zFbYU(RJ%mx_uJ0m%0$x39zSY6x!kJ1esY%L{6lb$V2#{NGO|GDPIiz-|JDdixJGc2v3GGmGl2^47iPsqdfFEoVqEuhNJ;49NLygw z6p@>WviiOHC#ej?Z~Obf-+iR_AUm};cDrFna!0tIGk_$5j}wX##eC(=Rn)0bNMh_G zvfL=2$9)OP$ArIF`ZK6PAPU1HVq&j5dOsZBj6V`;_~2p>+PY^6 z$(o$q&KJ%31JP_Bmq&FWAI9xxXEkH0iyn^}Z06R&&yEzOFJ?bFi-^kMzyNg>|1aeQ zj{Eism%B{iD+Ps;+t&E8mju}U>;59#ys`_)mes*^keyf|TEY8$SNyjzif@6-BJa81 zpP$oZ-u%&bpeDFs&A8jWwfo*d7eb~R!>QasRNA9AX;$Oyw(BQ`TrZh8N=5UkCd+~< zBx0DWmg33vMZXp!(10EizMq~o%M>SX`;XH`R@QGRa~Y|ArZvY~v~v=fh>8bQco=>c zibC%9Nqs`?J9GgFXEHv4{z4UVw_ESKv`rBqnoa{nlLkMjS#mQnUMMpq#jyqAaU)l~ z4C@smb8aTDj23;q!h$)PUi1f>_BN-eJVqe#eHnplJD!SJ2TA}Wj$5hXB?2y z2En)cEds#3wZ`M>Z6qaCem+CA{zSwi-XKW&W)x|!FWxB9n&Bb&f{^syk4GugdqHo< zd#C4Fxtj0oVmGDB6Q6?C?z^~ov;tvK_W4=LFrSJK<(s2W7$relNn;gwCUo&6 zxI6l|VNU%hIwhW-lvtVdv8mF1MC!o#`1p&L_;@8T_BC0&RQ$>x|AreK%gs@ zrB2vHJrB7rcDYoz$A@9J}3hPKynk zg0+DY7drX@r9o6itRi}+?tf8W|dw<~-q zCqI}<1*DuG$ytmP7A)?Gyw@}+vG(#MHgc`HwHTu=8S02C-tu)f=ol|rQ%10n9son2 zkcAIr>O+Jm6lXr%L$v$dNU+OoH#+=hz&G&82GFL5Ias$C+!cG*R2>v@)i8GDQ^O>a zN?cT18Xmf|P1#j3*+hw8Jm;9@i*|03U2X!u~%T#r+KDAd%Iw^cIMJtr&&#F z?LGJ$h}3ia)$S@+yHgRr_JxKAdlsAA?(@RLD$FCiRJf%LY-*La5lTB57BTGz5w_8y z34wC&^c@cl4FR~Z7CV9)&kxt(Wt4#a1g>0(UnmC-Zb{L_QZFr(YThcW4zx+~XELB2 ztISvB?^V4#I29@h5^DGfOU_ZhB-l{~T6*k6GQ{SEaXSp|IPH>sh6|lj>x?*j$$HUF zK{sMG8g81^-M>fT069uS1s>+~#SJHq;PP?1v-5Z(M&#b0V!ZQBP-8;0qwt1;bKQm+ z0yB8=;N0AMAvozmyT>A+5t0UqQ?E0tQd9JHz8xV~7Ch-L!eWKV+aQp5{{d6ulmj-$ z(P}5$@B1AKw8Z)AE6elEDnavhH^ljTg!|Os8{yIqf1C##WKPz&WdZ8KcZ^O~BoUZ9 zz+$8(Bpw)Vge>98hxTymN+9-Vn6Z%j*y|ja3EDV=kbAY0C725GN6WoC5hIKOwAu4B zr#a;rt?A?em!|=xvm>;)jO2j2@+0jAc_E=WtF&N(vtR736)%JYz-^g@^qfLz@_HX} zgW05DA4Xswagt%xzsk|@%@~S<$dMn_HgxyF6z4b35XXKQ;yiHq6OTxva>GPE^`KZ} zC^+-XX(iDaeU|r)^HSsE%ucjB*BimKBS5gh==em02WjAbTUnO{U=>`i+2B3wpPO#% za-SIpVgAh|@KNKskiM?L*Di1jF(o%;edE!cZ_T2?WKy`LZ&Q*0yEZovwlT&@ zN&}c@Q?x7ZL#5nzm%b=9FQ{nw*}Z<9{VJ(!zx34;9l_h(6BCA!Z2LeNh@eWIPm-_P zXm4xXjcke#u@gBtQIWvVImkx?;(=@@y2cf_g`^3i@t?uJ0IgUAL0al z*~O>SDb@~#8gV)tXb<}J>Iq4h!@k~Ah)uF$cUUZSs~BTOkFXiso3fUzeVzjH$5}#O zSnB<*YW0Y-{&9;Br)$oLfHm#zM3;~382rnD$I0=3eH2N^8|8SBzr{y!jGtNmEQG%X z8mMj^tW1KKfM3S13(;fcBFaNYs6la$2^etdsDz#@oF|FAS8_nAQW>E)*C>6vHd0ce zRj>AqX{5mPY?FRZfxpHCvP2#(j<6S+@IQe+mGu9FKaV)sc2x1xkOWa~prr$a_V!VT z``roI;(*ZP?35CX2>BfEy~%Yr{gzC(mQlt9%}}A`5QO?K(>m$Y$kT*7fUwbA`q{2iV^==n-HvBV?=3+iWrDF?rFfhOA_OSyxVQ{v|E(S4zaq6VNrH@ z8Igm0zSBw ze*JFqYuNaF(^UQJDaRWL?=#*5Cjridz>)q>xClev>8$U%`EHp4_u=)ypa%b<0>Zfw zrr`0_&JLOg2!fVxfoQ*>iOmO{gzkH2Ne)yDVPN6JTREdhV zzuW9N_+s~UOZx2-<5Dhlh%U0fM6C$(Yx?e7a@0rFotZ3 zc@y9p(96jqAiMuz_dcBJ^ox4 zt3f_qoMDvi7AD(+@5@fx_rD7>3NB2Ar2JPOi(MXwS|{aLCxNBVj&=0AliSAFeMKrh z>C9ZBr8#kB&O8(_iB9Uq-AMs@<0LBM|Jw#Poi$Q_iej?UU*ULR`X^3+F4-DeDZAX17>E7qd>OeMc0|eRJ6*v1wkc)6Z ze!MG5rXPI0rX$6FU-KD7+kb%|u~Q^)BSE+}lG)@YrG-~dY`V_1+^|WV2R?b*^E1v+ z3gr65h(OS?fPsd$E9Cw*lJr|(vnJ+b^S{oIQTD%|U%r^{=Wo?mYoh~n541j72?O_v z%K{zC=^F!yh9*I)pDm3*y|)wu{7gV5!1nizDn$6>ss9fi{}&3GPQ4lJFT{cZ;Gc{Z zOOD`#e0~q&q8+{KeUKe>0(jJe{s$bT3ju9UKu2>{Z#g7(v;K-#I!&0(hs;W&?*_1t zp*52dKB6_IGd1M>+zwswPdkyc&1Y3Xu8wBktC8T8oDd3$mqX~~c%?Y^$%3dP@an`$ zZ=>tv-4z1Ed_Y0;7Q!N%%3rhy83fAh?Hz|=*<22*X5_+{`R{9yf7oyergn%1;CBl9 z`cw$wSxw59nKj$j?<(K%TC~Hs{@SDw25wry{Xg&pFy}o8$PJIKJ%v^6(-k;r_@ACa zG!RXN#xI+Vr`js#;@;-gpIs0#o@qvzcoc^3FRolZsJj_Wo^5^fK-gUYJsiG`XJ8xn z|JU32dAVy%Q~hl7-pI6LIv+^x@>-bJxz=i(1h^XYSaQ7l2@#B>@~1jr(Z^5j07!=m z&i(J(;&_mFfVMQR&t>X$=D>e*x+Zv#l8;W^Z5BOLU)zdICr_J1h6G;Kv|Ik`sJ|xXo0sF^70KtUBc@~TRlx{|KA)?f|&0= zV2ILZtDD^bAf{vZwhyhq1R`&DlYVRV$WG_U8nXGLYewkcfQbLE52!t7@1l-$vl;=e zVJ#?@f`cDCoS#4{J$y|`5odvQxH)Z@a{tDyA0M_H#PWtlzX7czZyfnS2KMPurULRb z0qq%ags;QBVh*T7@z9a}b8I2HmP>|PI%pkxA}WBT2nQdx^0qK?gp1>dTC*{}U#jGpJzT;g4C z_nc#H1^g~0hBoT_O&gH9;^1+NQ>Y>M!c&r9LJsgzixo~?p?|tmLCs6YruMrO9h#vi zndul?N1zY{0gX{R(nPQuB8~Fn{VB9%}^8s^TgjaSU_LaU!`CfZq`sPEiGu)jL z;Q8wfB!Ax;6as%)QZsFv1^+YL%JsX8s+`7m9Vq5QuY=iQ-xC!KP#7VDkQ1-LOCoCM z0!~N{Ek^5(+0>4ogG`a1rM73ydMz03C`?D;AhJ)kjPWs zj1k2x69d{Sd-IJkh^2?fKv%j`<=i>!DDI@;y8-oC1Sx1o@005RfgrO#rvlp2{3&V{ zS@zd{^?Pkx*O`B9S8p!_{O&F0s~^JED1HN~41e4QkAeTMdcO*^K=pJRXSM1KBXqr1 zx&1`Q4^{=GV(8Tm$6AiFxNI7Eo))TK`0extV#@&m;|UF_V5c<`#Q(W zKCOb|=^@@rVdj+M$l1Fnb6vYD`%wCyUe#ae0uA4W?YVW`sHieRyhlcq_%pRuAhQD;`rE%C)OFH9>@w>RfhF20AY_nOO$H0WIb( zRmv42s0R&Sz|LlrTnnWe?MrVa4!bFUkG6+f!`;Ww%+<56Gy=KwpLaoy6bllU+CPbl zFwxH1f!c>aw6ViRp;84`iOtuF1DX9znuf%*0tJ12Tjx~D(x|sd!6+R z?~DlX;HEMnr~-9fx@=}#j5*6Gr<#OkSsiiqmzZQtSt@a2zWZy+U{tF!@EpLTA&ssS zru7W)do5~W`q+OFNZ^0~`A=*Y$2|LuS3&r!32^u!ZT6SJ6uQ|eW0%`{|NWU{rbKV8 zae~wpk$=UHS9;#Z8%=anznKW{8f44H*~td8%<8$OAbQmj9ePvEKr|PZ^^`I8cmvs~ zX)VZ$scNgqGS6c3&1yFtq$aC%wPM|a%mTG3X%35#(DisjC)U#947RpEy#R5?7GDWB zl|9Bj292K{&GoI*n-5RRk2p?pml(7ylzHk_bl)Z2OAP){ZSS$$?dFIrDKc zsaqk?HAh)JqwD6T>s#l8{#(+9;>duKr5_xR485=du7PL~6DGQJ_4TOj1h>>!N^(C3 zOsCoLg}dtNhm&y05d+CVg8&vHATCG!9ZHV1!6cdDG*6w9vn`UM2~w|>={sCpojAOU zcO1_9^LaLl_JVJTSyHrh=@k#FgoO@P;>-@!U^Y7*^3KnTI_wf4*)K|)JZVlk%ExJR z5Zd2ft}s3$P3N=<8{BeOTkBO%4ZG|e*7n+dTetGlVmR9Z2aEJ|=CX%?HciT?ynG_Z zSVu1D;Ex);!B2voPIwTxvzyrW3cv@JvO5ePZ{B5Hq3=ySd9iPka~{mJU5d$EPrpTc zmE9{6b!%X613!iMC(=!Amf(l{!+gHd73xYM`Qxz50MiAHTd_( zwwa_J!d*@R^4kr&86^!rHt^=*7O3mSHRMY}`e?Vt!lqW1=!hg8v!6$p)aw{(XK>X; zAS?TTeXjFb#a>^oK?$T(=B!4dLFUmVeyI!*8IUNTWCTfNw%+mhJROV` zifUz20pF@Ol)4se1zy@U*2WD0^ko9*bGRlLaqSU^OKLn%TtY(m%9_}2FTcH#Qj@GZ=BxxC!fWziXCgyYJ%xb+ej_L<% z92tRV5w)+^pBvcQ`-h%x*6=l+pF8tkjNgGNKCeF-e(;dQ(Np(qy`NP!H>r0zo%Hqg zlPZC8(~C>DREcSBDVv7#fyRCi5ka`FVm!%r?9*7LohBR>bKPd#@j`1zJeXrOJD z?VsL_(V5()6z@4T4eHyT96UMsY!EsUQWK{1-Q#fNr%`Z93{@(zVBg79%-C!J!zs>d zPtHQfk7~1f&rx0uXX2-*()=7P{f4^TSTa?zy}n_hR_45xjg#?(z?zDJ@35t@?(A?q zndfbohfLRfw~PX{;vAQ>{y~QeR~~clu+hiA&o4lo%H}r6 zw*vFv?rOL53|^0LS|rZ~R2(XZPcK#uaVK6;tT?6t8UuL)BnMNEu`B-o8PNUOelR+} zUh>M?Gs)<1E~z~Ia{=PX$w}SMp^AgqBxYW&eG~T+AGoikgtSh}dA(e$7sr*n)u@Rx zBRG7rQDNgX!ZhOK;Mr8YjpFJs#Ky`%&Ude>Sa33hN%nU`mZnCJkxR+{wlXmw@uuQLD_Z_}nw?P3^BI7Y)H@<^$mo z((>MmfoROlT-uD-<^+rk%r=b*=3W6&-Y+(cd-bTkyPWR&o3)PVJt?GZ;B%>V9DmWZ z@_2b$U2k+oQ@G7C?U6=tVwL$i5c|k?oX}SSXY)}uRZi_5L6t6T_&6)o#HZY8{O3LE z=3bUGFq0ZQ2-BwRhbWh=8`#ODo$NitUr9V2@ zK%FrebR3ar1TuFJ_4&&vitQ-F+42JoDG==Ep90Kzr;cBaGa0RJh6ZFi2)rO|a(stW z{0k4kyYLLpTp13H>of<`uduEdp}WmTvja=8v^Dq3^Yw2jBe!0vi`@}4`!Gq8qW$CJ z_!vbz5UdQQu`CtvsWLqm*2S-1zYZGaxGL;-(CXssGl?L;o_sJgAbo?yio=SjN#6Yw z(wnkZeviv+x#&lD521w<2r(;{lBvv3-$}n^k%ky2HNFbrw3*F1EXH{B_Li5F$>GP3 z9gm~tTsx&pSf+f{b9orJ#ZK*p%x3Jjkrd;eERX43DoWlj%iTLs9h=FSRiFEDzM%%l z!-M%4qSV*j4c}*Nc~S$8&FoC<*{RhA&53z$AMH4AmQwkE1}6OupZ>hPd=nyRu>g7Yfct!I z%lH+t5a}E^jyUaF$eV2go5DSZf$38h?*W5QCJD&OevJXV-Y4DxNMj`98j2H_+qAClB z3F)Njj~JhN?IB?g&ewlhnOF1Y_9~=>Ecls_jHDwpCVMn%tGuRLbmk8;RS(~YEM~k; zMZJu*!~@Yf7y_MD*{_8){6|>f$sI$^eau#Yp!#IE{w~@TaIWi!!u2fn#C%h54B-zj zL<hc8uLNMi#X9Vt95)3xAuDay#lP4byCvlNs~^1C^o+iaASfH@=u}~wnD2t9^>DkvmTf2_BhWTQHlm7JlwX!<2_y`47H^;@xe8QYlC%IZ zCj0o-f_vV4183@F)&Js12q|H*aZa55O5 zy-ZQ~OU74?BME`ir-cLON3qGL36H`_`b zl()$H9vc(*eLdBMox>ZgR$*LwFzzJ|^A9!=S@?Te@5&!;4Lmden%HczAGJ1AB?+vi zcCL7&ul^bHQ2#7Z@^TM(MNDh}ZmG%AX0@#O#xxJ@Z@pCbw_Y~%d(>(22 z_d&jqd@WUybjvglKS<&6!wtlb!*?i*fB20JwDDJsMyb_YrMFlVE4>akOADUmD?{Fa zxT@zg`#NNA;vKWs07xE;K)e{fYNeSo2TAR1Grf&(2c%wA%2iIK-y5vk8gg1J_q@jQ zHfxxrXtu|OJjM0QUx5jGHmxhjiE;=#1pGl39~3h0m}9E zY@_fCDf2VkS=TCSwK5D5P1zbf2Q#k5i&IJyK7rGhevXqaL^Lj~e0M*p`+Q2|+e^YT z*q=VOz2`ndpitZ2c=%rHfn(NQ1#uvn*JZ6HD(CbBU4O&9_DWvIv-MI&^Kz%wLhh2r zC8p=oZr%bX&WU}z7aFh%X`B9GVaBQ3yMmW_%ag}l+kGb4R3O4647b!eD_s)@S8}OA zc#xh{kUXebo;y9>r&NuG7ez$e=>*ETNh2E^1wbdE@NL{{YfCsaB>{Ln z}m0g<5;v}+-;XSfk<3ici1=f(?OyPANK`Knw9T3 zg0DQe6z>W?-hIBQ=7s}1s?klk<1qqt!JFfY&>Pq~vbR0DdLozJs=%2U|a1_S$K z?g>7+(j7ajXGx@d8A=B{45Z)r6aDkE-@%{3G(a5>1T_N9#gMDjAo1O33wYT8SMMeM zgFxolcfdV&;D=0rvG4A5*96ZBfKrpzfHwr~>IjQ=udVO@8Buu&DO+%55ERsdrP(GM z_k-E}9MXurIF|XU3|<3~+ZeiE^i%u1jxtXsGq+Ovw!Z)@(p1nAZP$GlW;!0ftB}BT z(EKba)q{`cA=W!YJswuU?r01PqoikwgVKk*aZ{AXOFn3BH zy*#bhtJhpg9~3xStG8L^Wl)Ll&9ju$WBoeM+ciAaFsUca`4 zEsOt!C1{t_^!z-{^q(oQ&D>^!PF=jwUGo?T5l?-S#(E-uz*8Drv912q%PG|{Y)0VX z$HSCw**<%NXBC9`!kN^4{p+Kb8HG&iC5py!4xTn~Zf{6D4&`AmkoG*i`syYYg-P-C zToMnWz`O1ai$N~`g0cL>?)z0JB+T>*37cu?t<_ZZDm22$bbn_`c7Jtk-DXU@v6Sv- z_ggU&)s2umI7PPIe5Msab0P1NqOet<{Wes&R|}=oAi#3@?KwQNVou7TcPq@!m_9lxGpF~EIpPkkb{x0oh@znBZU)Nf(v@1_(F!+fNGAo1x#$ zKD{quv}xz@IO}zX^@k*}2REr@9$)6We){HFbA%3uy+Lpo5!rugVR55aQbIL<2RumE6zt7ZE61(k^Bo$v= z=YR#yIS|k`9b|&DsLCf3xM=|QT+jGYJK};u7AJg;@xqi}o{7OhQebVl69(dPTkE+2 zQM_Iu$YV>BWwC)nNKms!hsmPdc?A`l8(tdQcR9x2VM@RB#+Wuin0#Hc z^VQLdGYLJ^r?wUtho3jcHE}|{I#mMEmh1Bkw5Xfv{9LkqT&hItfKMM{Om;o#C=9|1 zR3{+rtU%qRC&}(OEWbDd*+u7fY4`kb+Eo4k=3QEXweRhP}9Q+s1nCd6_@>DZzK&&v zt!i$O{$_!hlX_`|C)@i5DQPC27b*!~2PP&|wrjfK9JL=eyk^&JL%OR;Auz@3AJ~5w z3O<4L`%*&>+N(y@ayzik(vK+F+Y6ny!!`=)=a*>@+H$GPzZq9xq(MqWdo^m~EGZoAnXsIMcZwk76HEuhW)*x;VHd-ZjV z@z+sgX?w312l+9tjY_Y^p6149CGu7|cW!&l0=4*m~p-}N=`%Z%i!>s6S@ zjdY-gnE^hK+l!dfxIuyY`+Zrr2Hm{oM zjH?@WS_Os3Vv{uqq!Lq>L2D5|AB!@YR=hW_bHu0x&&&Q zC^n(r46c5aHZKz_;J-H{0^~gRi|UtYE5$j!sNvgekUdbgc9tpY*>7Imfej#c+n&8vRA^t zSMZk2O{a*wo%D6%0vs-SSxqP#pMiG!y=D1OC%_8IoV%>gd0F(X#S}cX*)h%r)z2Q+pvp30Vl6^ZKU1yS-SaVhLRzyP*HpKj}6sZ^56lWB93SQ)0q1W69BL zhKU&@gB`FT$>?J_cZGvFo?UKL;n3ehKS7r-(dWYhwKe{9WaKvtxRu*U0qU(O>-*DZ z*NsiVN3c8mXeFSlVCmh3q9D_lhZD}|MwpY58u}ZO@4D_!M(^XBiG_s5ktFFoEc z)xJzm_4rBNxOYm)F?A&bs&qpWxfx=}Dx(h#-ipJZ%Bo38w29-Y-6ZhRQ<8BwOaG(Wv=-^p7g$Axe?*q+_e6QZKpEq`xmz6ryz23 zxPK0Db?)KsGa)*x)8d-udE`3&I|s4#R6HO+lWo$@Q`p3Qw5JzlHYK~=H&nu5&h0t2 zFEBPyv)$5a?%6^v9?GV;LE@tCLG!e=A-1)Waf0!1hL$?8rLv-uvBtmz?k7zbTjJfW z3ZJ%H+ePxpxivpOuCmhr<*i01?4gfOonj`~qi*eVU~cmi@<&&dO6%6HUv{zo&OvgU zV0vj=(VDsxZAcHdU*Wy%p4_^#TUO_|o$x5hE?G0>J1B5SL>b61^iYmhnoaecd|PMc zFGNsJhMn_)0xp%>n)7BS`n~a7GEG=Lhjz-9yraAFnnp6}lxGHk3D<{e?Honqzl;)nf)OX*r zH5rUSk`rlb`5)AMcRZEh|2M}u;m97@A%%=6+2bU8?>$aZC^HHX=OiK-8QCL+%Ff=W zh>{s)bE1f{vPV4EeYC#6=llEb`RDn^>(x1(`?{{r_1WWnpF5ws**@XKdy72)HUpihO- z*FEx^#X)V_`(<*Q>`tcddzRY$2Qx+~TZ!vZLl&2CDG&5Z^=#aTO3lsy)?Z~`VeiS45hnk~vZrs5xN|;#c(?MyM#K?m4OXa@WfbUg zT9ucU+ptvQp;Y6F^Kjm5lecxANXl9-L;6I;JJ_+J06Xt;2lll(YOQUXF`VO-j>%3}7B zTf_EgJ-G?5o$I|HSX%W`3M$oRPPlFil5ET^j1+rC9aUHt;PCn$-gWgnsBfwbtbLp1 zZ4e*XCu$J;LgAQcOixS40$xOtfWv~I{diA8!S|H6^yenMw$5tqS5Tv;$eO`EYNT({~LH=#>y(>|p)9D|Gr({IO7H!dJ&re9d(;2T&d z%Kg)C_BM3T@0!`(tXvsJcpVFDO;mgeTpJZS+To)kZ~e|HIqWQ?Zq1E8!dFmCGsj7l ztcZL_pXj9Ag>ujqwD|rr-F;(^IYGx(@kM4?`>+)+-d--l@!E=}r|(J&(+5--_IE%N zf^p&$)pZ5;>h8vy3l+|~*KQ9D^#ANqvy&yKdFFCg{Mn}hb`Oh*Z&1FJLEuUc#`biv zjA(JH@7(JRt-|Riqvuz@b??utO20_{z}9h+>X{1<v-xD=pdkPg**F%uIU!Ud+?i z1zY)PO;DCVIr0;=9OEt55SGQhXvGe{*P#~NpT4{;t7xW@WlE+71r!^o0J<{7M`48)3>UUbnUy8YgKw?^Lez6NT! zbaA`S(>WTk;q{u_hUVXO3X31^FRV^Pve_(0>Ag?a#G`;KOZQdN^hW3BaIWQ=U2f>s z!n-ZVgN^@SHb2qfiW{(U{su=JqtVA^-I`FYM}Ae#8)>A0nw6}x9(n)n1+WVc>-?&W zLZ~bAODvU2migQG1#eF*P0$%l1uZr_de^P^VQW;V(EZ_Hv<>a2XNd|upVTU&IX6J3 z$8~gbI=HcWq?f-04+03hy><#cf%OwhP1GX;e#i!`0nS>G-#b1`PnDxyHSH{#7$i~tcDA-^n!(B&0bm>d!Dt|%5HH}C4YRJk$ugc z`s|;B%oqWU!Qe4r|d%tX|$!*@3 z$-H?3IAvfqBp<}D_t=lrY;o>vhrFTsm4&gMP9Q3--+$(HCTPC_nK8#M=N;cwQ{odY z5wP?tO*I2pyyGOrn67m0n{RAgb=Ad(Cw0_q-T=6<%c_{6!{e%EOq9Jt)VBz19kXU=auyi4r? z-;ZM&9Jxmyn9%XGbtIp)k=GUW{c_H8=8b~jL*ZUe*A1`0hKnuDbHfdJD_+mT$u1{^ zfXvW{z`J;h{2MRxW$depn|8&wYr~zr5}%J}AiN}dg+IP#ze;C5d|hhXVkp~YVR_iA zI8^1HHmLMGSk@>%_4d2_PPpmmTkW64l4WRl70l$Gx6EX#f)+G-TF`2;FMILNTOPRL zV9B-P?GBkwDX>{&@cEq+lP3C?C4gNzXq9WTP|Li{13IKiA4o{P?pCO=8J%*^;9R`f zr2@3qv~Xu%lu16Lz_Hjcj~#jHuisZhgnv|lYO7$7c`lgWNx!wgwoxFHe(iGjx{&2k zg6^J=*38O#x0t52Ic1x%Z)smWpN&2dv90k_8ZW3_T1vXm`+>=RLrl*o<;h*(Oc(a_ z@=~uh)!)_j?_8hF&NR6-kiy7mHcV9-PIgdH-|gFfv-hrX-?Z)M(CF6M+au#%v3vj% zVcLW#PSMu<$=5`_sDU*z+s^^(UpCmX1z{}<<$fzCIQpHA%p6;Jj$CO+mgS^`(bTgtIN-A#ELDKQW4s|@|oUWj~d^~D0RU?)qV1Rjsk@hmEWxTAL{0nN+gDk=&poSZp;VFli6>tTqlCH)xRCgI_9*xd^E~;JpG-I7!WmUQR ztfoxyV*!iN*^6`jrv*8bCA_A#zr6KI>t^am&7ZHFumu&Tz-Mc1O??y6Ps|#J- zpjs;jS}A^RQpGn$-YaCGfy~~s0KzfMUVS9U48J1Af z!0PXs67x#a{+?FE5k@)biE8IF^CituMZo5zzTEjRGA_`h)-CA$E`1}>shNIi!7Ia7 z(rQ`x&WL-xVYi&Z$E^`tN>(*`x?f2zQ%x`5siRF-U%yyPh>LstGv%oH6QMGfZH&T< ztYk^&dehLvZ1aA@u|CeSTaCWyH->A_UoeC7BO!L*fKg71R#n`-9km!>Y3O?YZ&gaw z_fJ(ysRdM}@_*bINdP5ffBJ8~Iw7>;r9Xv}dXG^4OzXb5p>dY>0_c&t?zz#WTeERL z(9ouKo!_9!Pv?!E;$9Z$zIryaN6I_D-nr}7RT`P{J)gs;fK}MBuT@d}MhnMHhwmjk z^I@GPv=9#|)F{ovl1YXsA76r+r3-Rv+{>YqD(KxGN5| zR$j^=+W$Ucm!4W*Iub`cQ7r3T)4JsZ+$g6FnKf%2P@+s>TbVx8;AeyTwx9ZqS5~>w z)v9VPt?$|;Q#w&A5KVNpW0QJF4dUZ%rw$6mUmkns|7xB zTDOs31L{92Te1w#7W=0xW%I>b8bcq|Gt&O}@iG5;*|09luZ|*--o1R0`P?eI%+H&F ztFP}ly-)$FzqI(PXmMRNaLKj$$G+8zq@3RBp+FeSN^r zs^5O`wppaL|2v!7>J6e>KbFPQeYQ)x9OK3_o&{T0hI87}8Ad5S_gk&t0QD^1(+c71 ze91pu6?-*v^^)aI#6c;E7itedO5%iq;_1{K$90<)xDX-Lw zZO{`A2ECRi(jKe5gj7^~X&~s!vVm&ycH7ooWh1nc1R7Ib#xKu3arAbbph)}C5&0=~v3^2BRa7!kjk->L`%dkaKhz{7uK4NA zv?b`c>eBICnZu4OQB%$&9H12q#b0+hupeK^D_K0+>WlJRxj^^ z1hzQn1H8HWlGcE9<-FUs6TK4hGM{uRWWKAv-Q=Kw7nI+Qssvv4 z*n88m=dUhyH=6KTeED7Ty7#N*l|KlNTyMJ(>z5OnocXupb&*i=0uq+17|wYSNLcpI zx2t=}a=@@B%GUzJ9&X&?7`~Y{>!g0`SN0Zz*?}e3k6&wAiw{GUAP9Emzu(05W#QN`G~E$xSn$vS%i5jVG(R`*fj34w)4}^x{`!vTRvs&g z$(;qZXj3s!tMuBn^Z~}CylKbIB-UHV@zEGb$>~m}I&JTkBG0(XmbITpzV1!0R4MFg z>ZHzRYR9EryvN6*j66+F1p>wJkC)u06{YqPoKCBP+BZnKJqP&*?^uD3%eFqfPMNpX z#w1tL6V7y8&R7ash@>`YxEblRGp4S*`DNzpoNr50wyWe?e|}RHQ^1;Ex%^$u8}S)| zvjEIaN;~+X9m$31^$Tk3ppt!Iy$zJm20{KSW$8z}lRW5ZDN=uh-SVLmyT{S;f{u2Q zBl8KLGK_RMFgpQd_Tt6W6Ht=i3$@sC@}|pWvSn$(M=_ee2D6B;Wa-0U zLI=(+amIJ=F)#oJI=v%i@|ZcuN}x2(;l`R$H&f2AQIF`c!W~3I|ITx^2)VOv58EW) zeNLSRO=L&>5^is~OvoVgwy0V5V!FN?EWvL%u!(0S+WJl#BrB7w$OONs1pZAV`^~bB zd@g&*6-%rjDb>dh4PBlJ65(WuLUydT5U(DzvFEpC2IakZPC9*Px%94N#CXr1?|Afc z(AxYYx6ygv=f)_j%~qB#x7q5^CxS(d8VOY+BHMY-7MpVR5;yo`eQCC)dO2jE_PXP? zD=|(}pu|&D_r2*PDHTZDDs*M(u~4a>Vv<`E8G9zrKCm`@ z+g|f_8$2?hoeYX+UL=Y)B6xXLIXogdf*#C+M!W0ZJ0gEVsht4!x+mKpd_3Zusi9i^ zY(FPu{pE;CyW|?w^gnhAnN;3WIkcp$bqvjqW_QV*9j9MF$@8SdGd%NFCmmF1hg2QW!Y7f!3wQXRj>e z0z3b7+h4N%YloQL=b$q|84|9;PeG#9d~JPn#DWEgmV}jq%ANZXn(yaK6z6spnVdhg zGwQx?&hS5L-1jr(NR0)@$cYupa7L{}r=j2XtwNfHr~;OswKpV!cgrcTE(3X1$JJ`kJ_JRj8_yB7MnpqT=%VODC@3^#skn z0_WoD*sxvs4(Ijj`uWZG3UD#%7MJFCcHI3^Hn!Mj7%4t^G|qi+&kBu9jZfu`1?{tq zKQA{LJ+YX5RZsP#TzG6MPIDi!CpqrPQW91N_9PM56N$3m&9rbbb&lW42z3(y-o2YJ zZfhU|dr3n!yrI*QkM6j7b5`$Q;3ab<$`)}R%+EweK>-l(1X*u&D%ZoMr4t-XgVi@K z+n!o(Tl6D0xfsjRQCE`sLD)QmJ2osvpPO@Hh}WlH&NJXugVWHqI^WCVV^+=px~}NV z_z$qPdsRn=hNJ0UX8!f5!-13Sf9?(#sxrLktJATCEo9sWf2+_BY&}eTagX4df_FJdG!v zi~zwrjX%}cxHR_8OLvHi4^A(af-s!2`O}pLX=F@afRRYEmDu0$Svwl6Mp5P$5Z29^ zZY<9BYs}1=9*mit)kG?BO&yeCE(P2(;D!|)%(oX&o<(rScmh}KwC;j#$V$_;b>4NC zDTejE0u_)r4wpsvSH};s32a3Ao2KhZ_cb>x+x59}9)${wnUr984(>^>w=7pznu135 zZ(Yj1B@s=%EKPp~Bl9~Rx(v?h!pSilirP9@fB6*bKW6*E+F$^Yi^dlkd2u<)r3-gG z$~}A+q*ueb$2nv{{`;Uge*wD2sAJGn+`)|23$b@k=>gm=Aye+tnruh|ppHSWnSP!w z=wQ@@##i!7G@TFg-L8xVFJ{4kUVq2MC=TJ@sdpcE_4u{62hh(@gf^g|q)YwbU{sBu zv}LEXp=mWRrRDO}2mpkdABQ?iOw4(C1{K&r+m9WXIlOf?%^LzefkMAn8hpeCZ;uo- zW=AOezHpfAJq$i349vwc8$-io;H*L{WGP^Z2Z9?=7W6;8{GlTp9iaOi^rwV_NQlX|}NG;J87ZlHhajxZ7vPSHX+;G-;uUx%G2XP|Y{{ z7MQaIb2Ve7kAEY2ftPR%SS?5yLQ`=ELcmsbUxwD>Kngq^(*zJ$7qXbJ7|QE&017yt zFp-ALk8kzhaoQp4Re;w67?1UZ!QS6skREE7Z^Az8gr5<*bP^&hRMW|-j}cO()n~d0 zgOcjXZ$f3R1AI>ts8z%w1YifuH=hIJdk8|6A7kdd7ZOLLjZLYNQt# z63dS{4|*E}VV_FGw`-z#Lw>Zay4wi|M!xaLX~{nvY{UT*A@}756LQw9F89uHHN#4m zUvF!%oX$8cC;OBNOGNJRSg_*diC@q}`juetZ0PUcS@q<&02}bm*WPtG<)0ieqmm*0 zsfIfPv)UVe?s=-qP>Q{a&0lAsD8tzoNqO8h2h#=+7YqnsF0rr4l|GO?;i6%KhH{8w z&x0nS5!mQgP1jxU@6PDQ;4WOi2z1%+5$M|@J&rB?0FKN@wlwF;z*8n~s_kfuBRtTe z$)LMorN(*jHUNPtU(#+ia^hb;iIAgG;ff$2tt1NR(mYq?LL5M?AY<4;ws@QguWEK> zN{vV9N+5eA5$qBWxa8kPjK&*+Dbhu1IeNQ{Xz!dImjBAWdX%`EOvrM*>_qprDfjH_ z-k-AGYyZ#IB0wS?%%3~=NPks?WzhZ_KYtvfG!V5g5t!u2nxS+)+)M!P$q%z&SW*J5 zBMzRo_0Pft$Ot$!hM3O)&uFQVNV3+DO_7e)aAnE~6sfE*n;qYONdh*DVwFuH$0Shq zYgzhpQ-f|1*V|p^fOm5rH&xu#do;C@pHDI1D+Dbi-?zlTs#&i|9DN=FR5${mk44bm zc}}Z-j!U8uCc*e(LqiAY$wSw#evLKO~X$UCWkODyJ?c%_bfr_V^Cb% z8*UmotcQ8Z(zDGflDz9yve(4I}0AKf_-bp+;0In@)3izaIwsb;J zUX{xTk|Ujz%{0QL06Pdk3Vv%>PJkSh*J&&@TB#Bz)9%_Qo^gHf4JHD~{=>VC(>6hipY+ZowD{$6`BU3E;|D=pgOT!^`u8 zfXMcrYL6_Q0qEE(r!=Tjnvw>@y`9F4xxY0HUF4-*%Eb7Q{^-}Euxr!rX?WMRy0{dx ze={W@I?U|th$k=14Y?@>RPAUQcYFxy2W+|Z$hT8wfcbWL@xUSEw}TGWNi*s7UsMtX zH48)N!9p`m7dt;a0J?d^I#7%NnH&do2*NBy+P&uE7uRb5!0jk9vrUr>sCOO3=jXV% zi51%fyYAAZAiv+oe1nCzfZMa4$wL8MagU%Y1pb%XYiIHpe_$=(SpMo$<>g{X!Z5kl zHPq4Xxnfj_S3-6MBFrEuLsZ;RD!W$sWrF|+#+ttt(Ay%|P0h`*ltln7d>tGYcf#M6 zD;ac)-k!Oe4(5hYFCYdrEy7FvFH4_f`YLNbjNA3t(|M%-w06ip@3~2Ti7n-ad>fmJ z8-=+x&rN#QM@**IC>f1jkiraz%Q}|qg4{p2vsF3GheleiX=OYtwk%eZaP?d#zuW95 z{U}XVb;DipaOEZ<%xu}Jln=HH5Lb~;Kz~{n^ZI$NKw?Qq-z?Xa0rzfG8@dQe{{Mx; zmdOt@OB4@3)codumr=paaAh+JxbFa7;XmP@PIw03sX@0>Ad)CX8jAx^Y+BPNj+{l< za(#mOX7MwJiOAq)1YQQ1q5nnqj53I7G2H#15LXt+nQ9B_OWb((|AukpiwLfJtXqF= z;7oeX&+^Y&d7JX7>}g$B_+B zY)1dfh0rY3;Ka+?o1Rc9Eb@(kcDh9X01F{!5)oSgB;3lV`w{a$P&}{7T~Bz`xmP{* z%S(?S(K8@}14x~z%l{yT^~lH024&K=%s7ln0e86_pq!ln9tu=)hyDbJ^w_@df2EKD zGH}P`ATW3IV*uRRYVb&hZ1X&3b28eqpybF$J|1h&#du1<8{3y0%S{sI1fP~`4CdjgIBNlM9wg;3!DTNLlrQd3>b!()z1Nw*}s>JfNM{_6bqRbQ!G1q zTP65Bgr^opw0w^ocsa9lU<_XC(U1eE3O)s-H)CIddv**$NShj z@IJ06z8G?w^paS6#zwpkP(FUjR<8a)c+j_!StdDib^6;k6A@CP1URoPkDh=t580=zE@kS2}F zr2X%kJAm)JKHH=oXU_OC_T?ksC+jN%tmQj@k##TBaH=aONr)CK$qT3S8vx8DCc5{$ zKq+M7;-w?b2EAnsVW_>~y}#aEKX})rs%DXF(yOu!vrW4ME@UYBv1aBU62hMtnw<~B zm7IhbfmcMh(DBbZUz$wNIi_qy!0LEicQMiU9YjJ1;&wN!#}R^X&9E8f{h&bw1?KlC zH^{H;Oez_Q<0a5b=eldct93#zLroqQS5~ zyYZ^W@K(f=3GR0vMsb6;?@elbyvtQaS)xGu(nERt{BN|!%OOv}YefS1O1!^VHb~!k zG&|r~8$b2C7vSLTY)H=xf)Msl=`$pj2n;Qgh`#au^BOg>T$CTUJY{o6?j)~Z+hgcd zz?YS@kL1ZV!GPRkX6gMpz6^EcFrodiEH}#ZSMtr(q=^8SE6e*g;F6zkFa`ac&F?TC z6~Kwcz%O(7&4hsL&j^pN?RZY6z!3n@yxSl&Z#L1C2;emUsA*QAs+75K66&>s0e9g>#`9!)bX zjvG1xv{)_>aR-YJul~W4CyNI#p!+Ila6S4)ce;4ck>Gk#9%XTpZ32Mm$vW=Hrt=w^ zzgh5Ke`I+hULMTy_53%N1bfJ{W~Ie3@tgn@3IUD~AN2maWCmeLZ?L5A${o&Q=wPs9 zs6iGbfcgYWc6&zVn}H=o<>%zD%}GmNkiQssf`z+C>@pu5c510xM??VNv{74+T~NXs zye9&HJ0vGnc<^3eg0P!3Y2g>d7!m9x6aTl_CdcE~6U$ZJuf%qp3JEu%y)EzaRUVcM z;Jq(drfmZVUw~;Xb;8vrV0O*^b>)1~@wKN3Ifb|9%|NS$qAu_lXSH|X#$4?rh zkN#LxUiRiuaxmYH0@_$HEeOJ>byFC5c7^e5f!X=BD`sYeHu=tpS4_%(-bnDf6&y4p zCQp%tvG}-C+*CDJ6=_v|qgl0Z`?%gav%-O+sU3n$IUZ7JiS+F?7a{Epdvj4IF}(ZZ zVFcwlu8UyC;l6><-`5CW(qYrrDW;ny$;lpkl!z;TD#~&p^I9#Mj#h$U&(TrO8Q?ki z+JLtqtkFLC3GiF>4FZPr-b=$J=yc*kDYvVf)>MaF=9 zjf~HFJvR2XYv1pac3snM+;_LuqJZv|jMJ?x6{>f)IA;zlc(_|Ng_7q67C)jHj?{$MsTQWLM1Q{z1q!2 zm#7s{Zv+E>BSx|Ko?_eOr>Hu3AbgZ^lvHSN{y3&zAFJ~2p3AW!=-8Y>ccs(PXh zi;VKlaw_c8?Z~<;Q%(2AZ(rMu7!N2&AR`wkEFb3cWMhBX2crY%`sO)!{tKBitvRSY z#5PGYMi#qiaZ&IoCWH);Zr*1I{m3Z-RcJVRJZJb-=DTz5u#bl%4i&w{zMBQ$iynL8AlnM&(N7Hg4^Y9ebV9RB#ruxJbv`CrK+#KlF zJxthPwx`u~mrzzb9zFHsY1A&%hhbfBt(IEz|KsUQ=0};4qhtjG-^%3AL3>IE`?hqP zS4YO{ph@tS_?u|-Q0ytAMkbzk=gLkK0?Q1+Qb`hXWJji`d{SX!T6|&8C{JZN`ovg~(@uOH@HzV-a7bn4E3%c%m3d96qGl_$U zx(@x9$2!95i2aH4DvrZQJ5zB=xF+lXHhG1qe);5q=n$?GRgAbns!KY#*!Eu@T^+dR zk2l62+0tt}+{%mQzFkata9UPAnw@~wJ-?AB@x z+!XR)UP|>|md_Xgr@X)zv% zIM`ie0L3ct33y|>JFDurJ=V9Oo>cpA2cpH45~9H$hT+dejc)mr2xc>SUF+1u-X>=! z*<}rrO^g;XcguZ@fz309B|Ng%3bo4>W*ugWd@QL0PLkU@TuNZbw{Rx7Dp5DlN`jD2 zU!fiTG};UAh#Ok-aispM0+9SM2G~Jr>2;O$SN^JeZ%P>S@l7y3*iKHS#R1jCHSs>U zQT-iq=4v{W3WF)>1uk48{}>e^cC`_4LY!5$=# zNDgc|X2nu0>teYYSYo|e_Bnx2%Wx6mN+O^j1#E9#?uTLhv0pG_*fUn58RvhQkOMjR zhPZNM+y}_fffB|ca5r#ND7h+#ZzAcc$p(>wtku;9>BD;Ka3SS4<7vzf2+vnQaKazL zrw?U7_Gj!ec5_N$uVkOPL~h9~b;{HC-y3(T$7S}ZY`FfBVh@3(zSRpCMlly5sb)Ry zp)#ByQ7IKw80Cs!ETMUnMSsh{OpCjYL!n8~PXC2v$%V-76YV8G{Mc=GNHvWSyVjSw z*xmP>Ec()M(W$>I3%EO!3~1#Z*PpER&V0aHSYy9o*LJf?YTm3Dp)?R~hdu&po5(HRkKQwGWUNj{oc3}n~A|50j z(o21@pRI?*g_Zf|`%|k4Vod_qg{>ndi`EL+dNOVJMP;xxn7i1w*gi~y1HlLTwF=f6 zsZCl>89=0+{_gN$OJYPjPloC#aCR^JtIxAaVWnRzT^iQO@P;#y6%~@;1O>Qa>_v<@rT2d8 z4Oe$0zy39>5vIZDff0l35E8l;EZ;ysf$S+KJveFJsn-afH%gulV8EWiWafRb&NBny z#oueWcD#hugUy4lwzWsxQNuc83XQ~+6wZjv$stFP1ypi}g_8f%gkxioE{ClV))Xx#{eQV071YQo zQjv};F|EMUtzoF-=hGY+JIHa5A4HT26p=7AqNJ$W3gbIF2A~fVzmE-0SCCNcCIC< zo>OC=Hv(DcM7*MaKkT>OX#-INZ4I9$*(28`-bG5eS<5W5BUrgSL_Lh>@_>#bzBL^3 zkFX%c_0UK04tQ~N@igVdZb)~H!J>&N@)DRYs;H)d|JlQj^})Qwkh6Vw^wR;N+V`%= z2L!jr!3e)Gd)vU+O@tyKqf<2UIWKI4$Wzj!+yRHCw|mxPyiMAax-y}~3Y%)@%7$vp^m4r@&=L_A8} z!j@};<*vCmj*d84LhEr|bHNv%sG`$f{}0{X)Zi8|c0@ZQd|%{_G@GHY_7uiOH|dG& zj>wvn8KJOD$d35Rq=qBrk3yzoj8g$OJJHuS6S(=aA>Nv6eYKv#{CFY@v$p}_BfN;T zRZoSywBy@V6`ha&vCK2v7H%Eah1$L43`x@qei}jUmV0DJiHsb@P!|nR*iobqRm+W2 zo<<+^uBC=xH_RdT#K_AhdUJ!#Eur3yyP(Y>59G zEimz%^4%s7_4(s)Ur-EO)~O=w2?9P|ozXZRl+eboV#w~@^8N2@TN*Fb{}EIE0$j)i8wRgk98_yo4czTsOcBEbvMq z5e>nr7NiIyR9OZvQs98g63?7{ZnX2Z2_?5?luH+PrCFG%}U~3V&h;_KFj(g0WjZgHAEYwxRBuuFw zO7!fK^5uG7&7l1q{2V*=legO9gN6Qk~3!G%GOjxP?H&f@3 zr)ocvZyRi=0;U*u@`HltEYuLZlC(=t!~L+ekp$6-@O_*TI8|AEFC)$XcQNE`kMh54 z-iWJ0WtN8wn+;J9kTL3KJ%P=@WI!3OmpiSCnRp!BG!H}S zU7zb1ECN>Xyn_Toe}*pc0>6GVP6UwcE{+t(;{1-b>grRq4_^Op#8+FiA)X$u#Z^gE z4qdbbBQvsqdKX?T^T!Q$yEm7JJmZy&Ik%w;9mvnGoNypNxGnKz;!vWxhkUMQgt4ga z8vf|ox(mDBVym~yx7(UDW10m12^o8+u*rQ7gWO?fDvuGhLHBf%e@PF`f>yz2aAr(h zfM}!#x4-(_{r&Zg4E9Aa=_4gH+%xn}n*{BuL%T^)A^>~~Hkx;i!wmc>D3^f}qm$}M za>soxSaM(R?TtNq^dH6;7NYUlm#31W?3kD~XNRmmb1E)k_U2$cuFMcRDJ5N4_Y+wx zDK>|#KX1&z8XWDuUd$%~x0hQ3o%vaO4u&%lj>Zu)Iu+uqpD}aaqs!GhZx@H8E4u2+ zAN7w5in@R>Rrg_ovhwUXR zAa;r8EX}kkKT^kVe6;XC#Q44m-V9fY3&#;#LKn`$ILI0{&oXu5PNsvnYBSL90Z3)B z=`P%RR6*kvK`c;E$CDh42=|Xf`a_ToQYGKzFsDs|Z4$!7N}zV% ztnV_X6NJnQzD+ViTt+fMRJt1A(v1c-E`vg7IEVkjC+}fL=aJCibxQfg&SU@bM~(;!`{cVs=EoR$l?`4M*mz9Zx7y%3agjKySO2W7 zkH&^$y0IsF4DfPL*z!YIlw|49l7^**_{0!%1F#7iTy9iyY@5ev2Buw)L#CRS-ZceM zZW@T}_oq$Ine%-Kvbp(~-sdvNdt_C>NH7)H57^ggEZdKaV`x;!H!38A)c$edvu0r{ zBzn2??=HJr0ln>n8N9Yo->k2;GGs*fv1}t1gjf4>- zLE12*T-cqbCHL38PokuXwaKQRqe2nWu#Z~N{Muh{nwgLKEfdk~7`6`h*2jOcKKxHG zhh2bwgu8j#eO^e~Z6w51eE4W60Xo7S(G0A5Dcs(qf3Usc=#p}&;hEG@C>i&Skc`W6 zf?4IdGBat0zc$q4BU)q9N@HJnMAgjad-zoEd1|2BMBIP<9h;f(ftI;IRb*cMw}#MZ&SDc^U~|j8$w)|^#!H~ zm(YzKjN(oi5iJj3W!DEe2W={$1Wv77fJcx!V^&*`q;Dnv>2ats8l)fysu%hrZi-jO z)8NhVSqD){7OELur28({opkrvg@$2}t%JV^=<>D)!S=ODqHLRlxrHR9cD5yxI^=rtdef@QozpdUI4&S`q^(K zOu!bn)L>I2g2aM(#hx?jnIg;7-(J9OYZc8n*;ATeCCEFZG(DAs&axmV;-ZgdBw~GA zt{^LBZ1#mkx(0t!0IdEs991wm@L)VB@O>P;oGP@RQrsM_5EOh~2y>2n0-Q7mGs|U+ zx@Z+@buaz-P$W>1w~NUk8ej{^gkq7@rX=8vWF)W5`i?4>!d|&rkPl0N-3`DL*_)Bb zxK*zFYRqT_3Xp=r^X1TAW2*0lwJ`L|_hCugxsF~(I@sW6fJ?Tu>iUASU>V$B5u8(R zU}73>`^Mz+A*(82a<0=4V+j{7MWzk!AQ2+(M^Xa~S0t?PIpTJ(KQ`!z+I9BU9}Hk` zS->59-CXckF5BtRy8D^clI8)U(pMw6o!RAa=El|y#rX?OoPYNca}>*n@yDoPU;m)5 zd;ntSw{W064pEUt@mnn9f`UyaV;I}@$=<4(=0SfZ$RJG?Fvo$t=pV*fmcATC$z?BP ztzVxOdm<&rc(g?&g;V7=p&+1d*qQO$Y>+QmC&|$XEdC~ zDrZ>p1d#f-Cq})2#lvzU?^7#tpPLt3X3l8as0%!$5caKTFpKF`9nK3k_G}ZsI7?E& z8TysrT;?KjHC`i2Z@%ic;apz4xV?Cje!CLbwgFhkxLmY-XqWLb&JS~s{l~{P+PrOd z>jX#oW#45KLv{*?tsnj7@uiXS&s&^+6yv_Y1s8{PKS>RHc!*O-=i`azfMiUAG@Dm% zy7Yg4`CZW;B(@yfb?-wyIkOp-AC^RtBrl(>j(#N)y|5=6RmgE>y!wz>yI*P0O$N66 z3(E$(+ACT=J|J0Z1zo-_?ATBo z&1Rm`!bhxDd|MfAikt_SFpcZ(y;Xk*`?V04jB`UV@Ul;g#K}V*uoD(EHZTPFsHD;@ zrPxsPa42D8$@-C}r{wkJsuHw;g^bmyo(1esgt+|O&`DoV>g#sJgQKWhaSAQr)K^xp>5~LjakxW_4f`P9eNv~>F&@8lgOYBtH|k_svTc_ z4pE(wp_*urYLG>$3j1@|L>K-efLoNW1*-cZ@5kX{aqYO1xLxOvVDj2ofpxN=NAIT6 z4Hl3bYq~w9KdXFhMfH{Y#X|Rol7ua9s0~bpT0VI7wJ~bme{9V_usv)(_{!@eHkoZr zYoD9ger_1L#PwFha0i+-QPoSN6fbUF`IWN6Dv|BV&WG2-JlwfYp9abRgdYrAD!u*k zxO*yET>;?U3Sk?>?{d3k?5|$7dJAX0b)nE90ForgrKwqc=X;z>M7QPi=;KOroq}^lRY{~h2F$Vtomen*NA-@&sn<5kgcdLJ> z#rwte@2XcEfIOOFZ(-uF)n0ygU+^SVfz?#9))n|o{R7Jcjtx5JLZZ96$_z{ugE?O9 z86Lg0dcFTrx5A-4a^(cHTy_sW2j-vj%QaRx5+;16)wG&sB9%N$v8yg{zUxcy>Z8_t z5I4~-vR&!E=x6fVh?T2xty@Me`8kZI)K53wI+@0;(1YWc`jZLT8M z7G%t_h1zJx1;Iy<7oFy9WQxL8lDm+(96qe>qvTD=t03x|rL}zSpkzXjwZ}{+?$0*0 z@iLB_yI+KQWLmJ)*h2UM*hz7$3RdlXC&Ej@l|dFNqt4NT#60ET;p*(hA~`F-L7T-; znJx1aHB9T9p)PPY+ocyTgA#Wky}1zMQDtJKCXwdWQxp6WVa26-@97IE-ahzcAi$;{ zGx3A;{7PFsB%d4hreda45VCUKjo0Vbpgt#9!SA zd&Z9z*w;qO#@*ek>VUh^9k4o*Jd#8rD{_;6cG1b0#5%~&$W<_M>8Tnv|9$hT;OQ=G zgCImOVB7Z@!(QFtSO036M$DI^4!~;hK*);+~L)NXfA!HnhhB`m zhzoxl%@@2qy_Y`RF^FKevgoK9xv>Sf&-=1~*u{e?2p0*gjf9j)jpVKa7w36MbX6J6 znSF)krxYUOAOeseGIR&nfWPR9Zn}|h43PbrxJ4+?i^+_Gbeo70?L@N0c|#1%yHqIV z0oMf7cRjgwvIuL->$ihbwd+athr?U$>22hF$-78CH~8b_fw@8@BSc6fNDmRu#Iw8_ zWz^>?gL&Gl*rTO-6w3@)p%j)5S_VJQ;O)Y=Wid`CI^GK0mL6s`e*+Zu8nS@sdtxP; z4C_YMK{ldA>?}{C4NH_qj9o==R9vEMRO0VV-_gK8oS1 zFl7PdC=Uos_+!ptCvw;dbcv`s&;g+!Yz5A#m*G1O+$x8~3m1l8J>t+LMj(Ye?6(hp(ME4T6_qh?T)Y<4dh#_L zwfp`)?2hz%XEP$;OIeBidYq>=yAhtvIR+_zm@;gs+;oMl>ly8&k5hVCLj+=UXY5U zndG@yzNx~BATtO95$Z1zGki_P-Nrq4-XHR{J9|B`kCW7mG&HApfqUqTQ8Yn}-sGZO zH75CMBeLZny%KX4`x<+{6^r`-;40@z_v+e4SX4GD0X7z1v;CsQ^J z^8b-EVX~`x@Pb5{h{<4nr7(N=Z;c2Za}i-bG{>)LE^sSyrsCuE9C($7Q$bzBLCC3w z2eo#zJAR{+<&`buywt+8^-sDr%CJji7iGvnJhGXIdkM0E3Ao%>@m9bZU+Zof63oIS zAuc0CG>^QRAgR>*@G|9ZG+M5T=H#kuyVsUt6I$T<$%E3v@v4&NFVlKcn5Latu zB=F`Kme8=G!{p0g4mT&Qu6cWL2*j%KYFwrk;iyzmIG=JJ7YNcLwH; zgi<>W+Gr(w1VdjrYBcN;#2KvIG>iVZqz_4~drN4ur5Bo&?y&1+JOa_A5&Yrok9W(q zcs?AP4$E8JzZs2I6|RNh=Y{b!BT~x#h)u1N>b_`enKkwUNd`fS=o0gih^P$+S_Hvo zAiKP*uq}d@MlYydzoH1(;U;#|U*6OzoDGr7-w;X!1SSGvQb2A?n}^&HUxE+agWSx6 z=q$z=dmJQjgfe+JQjm1nqMuT=)!aP@U#c)d$uMSxc1;D9&H+#`;0l865c9zjiI>m3 zgVYspR3-7+cnNf*s<+e8Ehdn_oLF&O1!hYMD~~lY@(F@fUbP-r|C{ydfnm*uuK1(X z(6`H9o4SsK(h4(Aa35&Mr-arHDU!kSOyF2t!7u|%i8Kv@oX=5I;XENnWgP=>Yni0@ zGX3jDHBd8!%zd*BJ6l37j8wRa-=L+-^7&Q@S8*V)~xko|10a>vf;Z(o5v8Fr|Sfw$vu0PryB#zmAvPXM(7gxrK!5&w?R z%Y(dwE62%(+biJ(Eg+#gMAiYTkKIl_@RA5k6T?wbrlLAY z*R&EQL8E;EqK9MBAvikNdinh8aPa6_2t9U(jM?!s4@0I6T>s;4VR&UA)8>rP=k2mdrF%b0JXME*)M=tAB=8x_{A|oPbZ&_W2 zKM8ghp~+$eH!p|1hBUzOgb8*-v=Unoxab7zgMip^(Vx-1y4m3fwco6^mj6xeY~P|< z^`l68)l{V=PWRI;F?RG{T&U-Y3qiRAsTcN|S?%0sLok$L-4GXMESKE4A;=~U08V!! z%2AV2-sB1n6)dyI7t{HZ4qVN6c7+>UjS)ReEm>=o{P}lRr6`Z;|CvR#4$GQp^sgcB zAW1e#_LG|EP07E*gcfWkECY=(ct+N`eiW!xfWb*`AT}-y)&(D?00>{LW`1ef8B>LM zzl#4sr-~aCfmAjf$-vuF5QbRvaMuScfl{LvcmDS;kbt>Wfl1hQ6@-M=qa6OGrECdA zMP>gEMg)WP4IJdk#AIWi6!*t@`Uw{g%$OEQU^F;p59 zK{{b?KI;HEa=IvvivRJ#$Td74HwS>G7hwpe|JxnS$9RDZcc2I~D}sN5KinBW*L&^v zAAq9%y6}#%`KW%>Mr5_sqsv#?3Gn|Z3>qmGNQkg9vHq;BnYTg(Yqf&8WH~`Or;8nZ zrx(x_cRr>UL-I|Xuj1EpP4qgj^^K?C1|O0cF?dUI(ho56X#Dzf(UHBlZDB2FK9?t2 z2D6|w`6s_wQVdf_Qq~g>-{$=rjIyvO;M=hCPye7HinjrKTg1W?8;{O(g@0K?2!a3Z zF+d2#`0Z{eSHrOm%#XW-{oennNQ9@H-o^TPp9Sd4zPRYirjSe0)5P!Hh*=GA1P_c# zL{cHnE&*!GtG~sen?O0CFcNTmgf3hb3V4$57(u;?gKe$B$8rn57;8NJ1I?wI|00mv zENOuDnS{j{+<`Ze&Yla!yGk}v4YIQ7m-Ba1hyh0lt%LML_G3H2e;8bJr{XYAF|`=y z;18FUzBv)F0XtK;@)fiw&5IG?d1fIM+Ozf-BZGfp>YxTX5anNg?P&*IbNsdo5KXoz zf*I=-JQ5L`z~@l~1i*lI0zaU2x$I&~{7Vcp)4E$wW*|9HnNv=eu7o>=mRtrVmM|Iv z(8Vfg1R*RI=8n+W^$U~^arwvTfy~h3fE;)qVO>kZy6}f9;EVP#@idX1*IWsBbB42tK~WUCz2;IE~L@@U;TM0C&lg#VEh4-lD)Vv*MA2 z!ss|bI&6g`J;T8C(<3&VNd^!DQfKYUE@D6stQQLx-et|j&;B2ms=ESXh~ii9ftM@& zWk!RC>%<=tZ10l$G$v7M;s+^-j*Zkrr2Pkx7ote~jhZy{>1kGTd_Yzd3i*GYqZ=U# zFUvzx-zs9eEM?~YAM~`1#+=-nG;zW*9ZG!wQFcHAXhYP{vyM}Vw7s&jC5Sg-skJa- zKA?0lea3hTuqpsG6@srDH%IA?CptYIX{$h<&n;@lbdmd_(M5pA!^!zKV}jcOQbZ#* z6}~mt3H8+%t?c6LJpzqPAfiL^WY~eoQyJyLy8grJ2G#|g9lpf!eHLloe3xdqkyETS4Evk_D}&_gTF82 zDlnWXqDLjWJ6FH97tsg0SxASpW^D;A1J+53lEtHi516ZeLo7=Y!U3y7;JXGU z9c3d}CA^Rb6~*M%xWHkl2yemtlRKl9-a8x1w6y()?BlhkmCfZrk~RLVF%<<>C|?P{ z=MTmyFAV-eR0U~B`$+Y;3PxpK?3*Aml-kn(k%0P7G< zh2PGLr?a~J(oz87_0Q@AWX)7?CQV{j!*0UV3z!zKIGs>^ZPVgjAW1x%>nq?JeS{z# zXC~f`O=!Bv0qjU7^$yK37`51IrDJZ0D$a4j6d(t+)$fRce1ZBO?W%+`uHjs(G>!6M=+TFhlqMx8EqMCh7}G|qrR%({$B{|E=H~PR}BlU z^z^CPL2WFkzCDmACoL!1KmiIU3G`SB2!I)3b#F*6(GQhO05g4tZ4J-FIznjV3DBwc zJy`mGAQDIqp3`kR-MWFD{|5oN;~t^cCSyB0CdEMSaw?jW zaw^u^FEd&TJkWxcIq|g5^ds}5HR8Qv$bllUop5p9q}2qD{ii%@071e5 z4ORW2uh6G%fa_ZaHO&)sqya7c+HDpBytNyr_k>eJLJ#gUR1v$lUoZ9i;B~w+b&9SJ zcKk0pLE{=rKY6+iTmVjoxuZ(NregODMrI+Ob0ln6IWRi44w`=B7;gU<ra#u%_+J zXTNZ)+vBPDt0Y4q{Q_GK8l$f^Ge<{?R*3xU#Rz9RC^!lzPQ})GmNG!CaNBCk&Z6Rp zA0`ST5zT}-gB$3)62X z#JFDtyIS2b7$pojrXSNVg5P_i63R3`Y`PBs2y{ePHWEH^ocG>ODdYJs8mQtQ1Xk<3 z+=IFZa~jj!)6+YWSIlzgfiy3hPP>>33tx_S%>!A~?J|_3l850EN9{xJ97uXHko0Uy zKkK7k6YTsEyk@Axy;OwxKQbc7qTs6o_F|wyv|0n=xa`z7|1&ferb9~yb0l z2y)lkv_WD3uCNfYf3F~{6Lz6z=&#HA;0ph4pQ*B!8 z4Dc@kBT4#Dp#md>;up{uSquRt5c5e^a#9I;%Nnv2M;sA@O}qq^NSL=!nUxA&e~lNl zdLi^4{$2?|)Hs=>#B}9+w0?~cWPO0KwvI7eEd4zX-6hV{ii>)mvdvM1Eud5MWOUZ{q zVw3_)sSpts6D&~&s#?0^Fq;@S#IX>)oRkB;A@Bl#fobRfv=s!+5k}vw4+9^yMSq!i z`o|tG+MrhcKSR6fb?xCym4JQURN2?goq8ybnRL1SPBT)sVlA z1V{u%h+#6xR=|{OO@~a;ee$z0FvX~~zozJz26l1&dM-HJ3rOXkp{DH{ka#M=gFZ{cson>yu7ka2I+8+v?KO|1%qc$6O#3tc5~+7VbE0bUhw3 z^Hv5HQP1BcQ0&Xef{%KI#mdQDyx{KFIDz&{FeJfT!FIuvXg_1=_qmieLBMDGoZkR+ zqsg+S7)eEqh89Bpb*gG(VDcqYUNVqB9wL5XEjOI|Zs#z91p{Bsa(CXi=ZH{l<#OnT zb_rGp(y@uK42tSR2ZW%3KX&qYvlc_GRZXrXOh4(}N5pq~k3{u6OOdypmsw=>F0jW! z4Jxn7c=hCs?z{KKXs#T$pu8&g;djR*-fw^2^9g~J1)2y?!iiJspY+Il?_YYKAlIi? zBH|31$XD_{zR^!vX`1)nuZe}w6=oli;GA>BvtdUmSK<#gm<#lIg%oy2mTqL0Jl4-# zd~>uijaHH~1UeGo6V@AjIm2hJHQn5&Y7fxDZE$+l!PlSXrE z1525+tJN*`%al0@?zf-+kf8qX`9Z-=uFT*Oo7Cqi`Dr0``RSpbxKe^!W@FVlMn2E1 z&_Pwq?N(F`fmzVD0CUhD<$AyYZCUF!nNZ6aJS&hXC`qx#`1Eb=SJxf8<5FM6)7+6K z&_@{1?9Rg;kFJl@t9wwIPnM?K;bpufJ`;cI`iJ;ibY|bSH;i_fSyIB^Xa}oOgXv>Z-UL5sRnN7>#7nEG zWL==c!H>t_Iw7d>?MuboydJ7Mg`z(#=voJmwUUELs&Z>cC%10Ap;=JHTdz*`Ux?72uyw^^#t%bz^?_OA;u~)LzCYi+h zN~8iEKoRk})A!9A^dYJcB&2L6bbZmUXFGv5m!@lduWy@Tx15{&NF+pOWw~`3Fn;gJ z${xMCdzqr@%>6Y~e{rlB(tm_9zCG(A>AYw{xj|c28{%4w*1Jx*(C?Zy8f6ePUc9D*WoRo!BJWv{j z2xhlfYO@;2BiITkA&@5y#h5D}#4E>SwOZH{zdKS<7Y=&funGT zjDx$GTUQOR-LANZl#{s7xLEaDNSK05Rtgxjflzm&>_Qj!(e*lydIjZ8mR1M0DP2@T zi(7G8%<$2XZ2iBfQ=;>zQ^l59>9e31g^EEkd7nI73s}ipXIm>Sui;~#v)v*BO>kd+ z;e*uGf|Z4D`!L>w=IrZs=`lO#8FYwhf93dEe)pHq&oe=d~oO@zb7y z@AkVxEflUNWj|=Pbf_$@;b0S7*?>+HV4r5j?*052C_M`i5L^)4fKCW*+D-}60o|4` z`G&Ezj+OU_Q1X{NcWD``2BEUuG~e?ss_-QL$dr8*PEGxf|h08BNaD zkuM_+eHRgToXr}vTUE#-8#6Qraxv)J2@0>A zRck8cxveiA7cViH`()jo1voE;?z(MmO`NDR6+8XBD-o4#W3oC0X@PFcH&GSmTtfyB}m-Z=#W!Ml%eO|rd)~%2} zsa~(5?tg|0wuKkbaMw##0YrM+*Zp?Za+|>_@(xSEo~6JquNh58(BlZ0ELz>zJq}23 zHGx0)kdjo8`}NE8ixsHG?aoKp)V$)l-?ZOEN&fl{Z-QFUl!2G*N|(}E%IAn^b@VMT zdtDwtI3wKQSvJ>=pMc`F5|0RBL7(4?&PwF8+Xh{de1i5KZ9h1u=p{Tjq=hm7c%V(+ z8nwr(`YcIFzvp$_K&15o%!9xnpR#^0rpsqBn#7bcr(D$0g5a{@bVSaRJ(yM`Q9-7a zFI83mOixSZo-2j8CznXZKfHfg)o1lB7Ty(Lo~w9D~@Fp;Q}KgIzBF6pK+OYF$=w_SRjHbh}+pj zi22jqK)KTszU$YEG%^~`gV%rV)<7bqy-ef38YAlQILeE&+&?Qu?t6y~{q2-Tqu3%e z?=FPCO5wdY!FBjkqpgr#TFro$^6aYHTwNORVd^D)3KLijBoS-V6k#+bQlxizqU_AWMrKg zrD7uqVqq}iAt{aLd6mprmc&G64$-SeF$8l)<{P`x0z=`OWu*qv^Unlm5;NTeKpI%{$p#qtX+ zwDABffMxp5JUa7S|E53P^KEan)sEIpTn#uw5{B&O`g`4{a)yRQ-|Y(0{HYSt9PS;( zNIYnH2|>Qw*&PV8pkvN`Sugf={}bK>_1_b`>=JU0r}Agb9SLD1j@xWY@jyg)Bd43@CA3ku2d1030SEyh+dZDBnLKW`Kr0=la~zf&X`38XR=e_ zO%bY$nF|mfuP~iKA~*XrjVVT7ExpD0d^l(j0=HJG)7{HnBe3hG#Rf&Q12W&o`6c!n zQ=WZBL4i{SZ+t(C-e>j6IV{1MFQGq+q47-W=WkXgOa1Jp2-kwmdzfC=XySwFrVAya!*sr|K?+R2l*&~**l+b_~ zYy}Nn=vX!k&6UhaU9_-{stB=F`7Uma#xGsQpF>}IZ`^{Dw_mH%y2}NUf*YQGk;`aa zg}QUz_D+D|s$NX-XD}nL6m)6#Xxb@y{Fo?qmF7=!DYJ%AXxuK3s2tFLJw{1rvOjW) zn*F`>OXp*f`{(0EIbZToBiOG={gA?Xjncwv~ETD!Hy49;{X9yt4$ASv`h&e0E~2eKSx$?`Yph3 zD&`wC()~jP+EX*%?dN5jx~4V$vgA;Y(k?vW1$*m#kBzYT=E?fW)tg{rdnJ91Ac}MF zXW!p5b5tmd$RZ-IIU^zw&eQ@5+n^dvBs3**#9zTWDgB)*ZT=2<;=*eA?GY4d+Al=( z5_HIKe|;wL2pgR7dT_GJwIX_NYOLsvL9)}b!J=p1!XgB15-H&YNqQh-Fp^1w{_8@PY)%612MRq9>R}+ zI%A;oVnDyNqB_dg^R*~Bcqtck*?0}XfMCb~o7Pfh7lEP$AcNsh8Ynj0)&;kwetk2) z1l5U*p}B5LGF>gl7*L2sHU3)UP+8B0U$G6c#OJIVo7C%~E6+5FKuGQMczuzVb}qoe zjyjlJw0f_THRbR9o0`~3i&$1jO!17khZ!vW?tncQLM#%j^}O$-uMfWFgJp3rcHwni zm#!nt*|*NTpr#CZgIl36|BSLtk|h*-u5?N4|M5e2ixGJvo3cwJWo*btjEnX@EJSfKp6iyz!g z1RE4-WtdMGIk0@DuzIQEOD{BG?zA2S%ZHi~7a@ofI>F-&h4UzZm9qK9^asX(MBmvN z`!J_7}AOba>S6)AhSM%Tt+Lli3@lX6AS8ur`e1_w2MT081jZSb{Jx zdGdgyC&EJ7o6n3tPE`B~?M(!l5nyl0#`sbWNjY-EvE|#IxzW~z=lE2<%Z)6OFfV{ zmC(Cra8^ohT)7meyz{ZIXcn>dU2?=WeYt3R31FZt7SgAhI@6DNHzH+_j!_+?P)Odk z{hw~a;xaU-z~}jKiaq}O(W4LX6xvDXx6l(@=8naLXSKGivJ0LH&JY_d#-Wwg1hx09 zjdjtSV6D>&9f%qG*DO(&5t)h4$$s@eyl6JYVs`ArLTSI*#+#1wk*=!|#XIrZ1mbAL_hPtuO)E35Bk}2j^wVLYNul zaqs|Zz%|9!Ob{@s16~GUgRZ2(j@4)F3-S!C{4zlB>i>{uDpBtTp6yG6tzH|c+m?Kp z5P(b51DIZ%^&-f}L+1~@)Ov5nH)xHXGL^9Xss6O;%9=E-gFq!CV?vC$+S~@DB`NId zMH%B;!%zLscDeMgAB^NIUy^D{C%3+bK(}D_3al+ z@kl)!qls3!2q^DQCiP@CPNv@8_Hmx8Gy5#-ol7XEuL5>h9Pv%l1$(+Wd?4uqHmQl; zgCS?MS)DXZ7Z@v!6RzN=6PV&SzBqQg#R4&4tQ>p(XM~`~j;VJV{Gu;u8^&WjueR+p z*m;BDg3_CP#brSB=jIP?55>}AdMmdX26se63A^XDA@o0hnHhOBSn1=e9lPM*$*t@$tUW^XMj2gqm51lOFM z++{ziA;A`5yExyDY*0wxN#=_|B9L&@d0ntUXB>=BZ(h0u`0Id(Ag|VsU5bk%4v`_U z=Yc$S#tS*sjQ^S70@Ggro8?F|HXuuNG*RcSVvcqnV1>7fd=bZf&gO!2v28@Py6e0Y0pq>y5|ks!_WtEl*mTUewZ$~B2+E7$ z4?R`|IFJqGGcaIt+o~Y>pUyeT;WD=?@0V-5nm(hp?T7&d?_@qV!^H_q=8qB1fd_2+ zGJcE+>RoB!r9nTUczJGRj^l;M;{NGSP&uP zwV@dDkjGDggX152J$vlsb@RBuL^00kO`dd~391K0nn$*Go-REl;a>%>s;Zg_oa*z- z?CN^>r&Frc&&Jbp&2QlFy3IA^DQeM9dhU#ryp+%8{ZcoAkvA}Uh@zm(ur6(8n1X_o zUs8rI=9}wqERr9UjTamZJ43R3OkJkhcqSMJ!#@y0w;PLxdcmr(g-3&Qgx$zl;6h}> z1k>)YXxOdougMHy&_TDXA!@^HWfV{GUUVh0zk=a|Pu6P(DJhjvi94G__DH9@Sq6uA6K+O(= z|MW{p;jfdh<1kl4_`p{X8#j8xTsmVE25rETXR&VZ1UJKeAr|1~%yzK&QPh!K_ys1IQd1qNLTG$! zuoB9h+EV*W)SVZ47Yke)-%;i&?zznS=wMs8*&eT$Pe#FPIWxX8EBvPRiCFM6SgKkM zb9yQgTsRbC1&fBKBje~XvKFl1(`Y1jY1tZv3K9tlp`$gu6CEJw5s+o2 zk~~_}XC)<>^CCC{??%sd8*J`)Fdarb{K@A?9l8V z?Qf9I2T<7j(f@lNndJ2W=U$FdHK@j{<$KoZblP1H)KhkhRU>Q9P{b`OQps zA++Foc*Qa@wAxv+s@1kvU+(gYojt#IlAjf*&640LdV8rn2yR|AMZpRKT6oKGn-;_N z8T}lVsFEj|4w>&Bvd3Yaz14n%)M;3b4QE>P!c=v2Bg`wXbEKiWoh|?`ZG3xbGt6O5 zjz-Ii7jB>UU{+#(6`(Tp^E6WvzCpT!;d?*9WNsI^&aRtrGzBzIKD;up*O|=l1bK^P z<5F-bEYU$CPBZzX-i4x*k)E(_T=T2JymQKsO366N8Bu|-Rk%O>hl=404Cp=H0K z4$LpJDW|ilU05J_ge0(Dw*EMF@ExI=+2@6wdhly~$sF>!0dY^?LT(j|bX1-`aaMC# zSt3SaV{5MKdD<({H)t82CZX5_=s)_wdf-Njt=u$HKr7gI*RX^Ywv~v9y^Jwjjo?wu z%@Det>OI=4-zr&=BNCT=*CH>(QTYE;9A`!JSt_Ku)LMp@@PEB`{#u+K2HnLC3_Wsf z{G2gOTOcl{e_v3PvqqwGj)sHu6VeyHKr+n*Q(^NFB-VzWU5K5O8N&q2f+gc+OSwY9 z2ZhA*&u%W@z`BivIl+tSDCHpK0rwGcxj2(Voi${BW%$+Ki}*pBVbX{JGMrY=UN7u) zR>oSxMJg+GtOz^{ zG)3SF@~xE8tk3h4wqgHR=mg$Bq2m$@i~G9BtJu-cK}XAm$m=i~xVe0=X&|N46HYjJ zIUPD4wukUG>HYe(%bxfvGw?(cvhJLRN)g1_q`@V%Tbcbwmo5U>5Og033&fD!9q@pc zP9D$DisL7vvZtPR-a)*6Fr&7q$D&>M^-5ni&tJoMuW3GT_`QB}UkJ}PrZAvK^X}a0 zrW6yt_cP2gNF&*Sly2H z?*{>g6GU+HiibJ>JE$W;_>x$QVIfjKRWPu+@p2%PRto#C!`9(5v^G-A2TBtmv(?tW z>f!kL`Nyas?z0W*7kvn%Ls7wEo39CmU&PG~@@_2ix*zYvf-7bigI8}t8bxKZ1#`mI zY#-k5e@TpQlmL5*M`c0_#dQ{)mHxJH64Kck8gBvBDDFKDw}POv6RAX@BPKPH^{&s$ zhMRX@)poyX6;2d7VXAXFjVT`4W^?F&-}9^ucW-(FeecG>UejBnlKXSlsXB7c>;2(Q zll9McXscMg8VUcS0oh>QcPoAla{=g?Q_$JjM|WRQ zs&09GGwGG`ovlKi`5iHz`Rx%^Saj@+&~OSLnl`(AQD>0$Cu+TSe#s+kNM-G$p~7ND z{lsHg$m&xj?OnH?y9^uK=q%R*3>k;NG`z>qQtHbz{p?xuak9JvTrgcKHfZfEvZd{K zdSZWSckz@Wb6W7!2N5phtV$|eq@BXE+x9K#(M^mpLN`X?E0q_H@OZnA6eXkpBvate$M>`l8y&_MVDzwjrW|8fDe$s_!M#VgCrx_m~lD%QIkTFny9 zYt0-l&bpcp5O)t)F#9LdV7G9iI|Za!?J7}Ok$yK?`wyIcaodO!iysy zy`#9S6;o+9t4fSXo;i)<7Zwv=aogCuQ*^hKg?Mf3S>q*>oOz710JZf%jAn&P|F!rs zUHZ1p{_6Cg$J`c>@6*WdU4fJ1r2!v{me4r;h_K`RHBm=SL5Z(ShYmV#2!Cgp|7+u{ zO|tSlB9&`K8vg~WyRZMml7m%mg22~5F@@;xeR^~Xg_*w~{Wafga@u%-+r-oZ_mr!M zKMz*w5&oWRe?f88>A;(V@KzYJS4j+Sl7O`H(~p#{YeP~6K@R)H3SZ}L3M`k6#GD=u zh&jqP>E};;ZR(5vXgx4+cms0@;Q{YMxKN@DV0GgQZC5Vx94)d3VU%;p6oj$v;T6r{Osw!xq~mLAd- ze^d5pcWT(|krX;;{Vei|SlX|JoZJU57rwRb)VbIYhKhoiMuo>QvZ0`mR%} zPW$H8tCxzQE~Q(amnC#KOmx0$gwj3aO;T75t0CS0TtCC`$t!KD;aplg%a{euv~bV* zx$`M{sj2sm^NUB=8x5WsgRWI*)eEemWKIzb>Ja2@L3`8bVjvYcMV1!^e^f4i)r8&J zd{IvmMnVDMk4O#HkPfPLWiO8CB^z3b*)o%$w|AR%n4UW}Y|X4o2R)c5qb7n#8wiW2kOAZm6yA=>_k+iqy`@Gq!>+xp#?6J? z#)aG#9nYT%(AQd=ffN3#k1bjr_nv+@#qB0rkR(}?sAT5-SvY6qH#>SsCh!D_Pu{6^ zf{YoM?291X`!Uw+AwhS?xtOkavVq`}-j&$+0Q-4`^8jypdaXVY-rB*p=n8B*&8|s2KXa#cZlM$!3iMi^RCw==KTfwj(}wGP zx|KELbUH852 zDU-TQGc$R5IZb^`bM9<&xM~r)q1Ji4zNq*9_$)kvO2GK-+2rv)vz6v=$xxc}35&}D zhg(P0YXoFW@_}#K=_;vq!=1bvL=21Rax_&7{7jTp)r#gIy}s`0_}}?=GK{2uSuo8WN_eytm*30S9lRt zg3|O)@jUO#9%&(dT!Bpf=8N=tt9+b@)9cRE;WfqsXMSM?+F5J{D;wvNFJB^oM6Qp4 zxcQ{+SM%>5SA4eS@LyvqgD3j9FFjuJfqn2s{rJ&rB5xI`sCD;%uY!U?cLHx6IsMQh zUFv&BH|;j4YlD6U`roQ^(UyF zp~X0^pa<7%e+agt52J2QSuB1W=}u)AqN`g89r@PzV6~@XNm$Ii)|A_z5V0g-pG&+N zzPB93Xd?cWAMVy|Tz=!`4&!yfP;Lcc%dQ}4Jo!?ih+gV}ceQzZ`f3e5H7*Ajr()yl zBr2`^u|G17%$M`UXr$`Hw?x&}5DT*2)xE}y=Z7r5UiM)$WvwykK5}YZnNMVmH5NI- zvk2#Jwbj4C%19xe+ayY<%GI22;l5oT>b}ykZJq7cq`EZ4RVIguqA4}o!5>?CNceWX z)$EAE?RNsvlgyFWUh21Egts!NG>yGJ^nJ>$u$xdfZQhEcyDhm-BTT@|rM;MrqA}!2 zp+1*ACSmUiiq-3i%{JTrQ5Y4fgJKDJ`2efY;@c>t;8kit1GqbxD$qfQn3WzmtI@c`1T|v;gczcia52y9H%OYZL}E;SF8T? zMV??s<|tI{G2vTOTO_w(LL z&KYialxxf-WttRVQsFDR+xYTS;50lsKqt}&ry+1xQPWN&r!4}TM@pTYg9Bjcr4 z4NY}2BdcQ)Wo>m+J0CyI#xxFEd_w5u+Lz$i4puiuqVnw~w&M61#FoWWqJ3YE^n9oL zDHxe1QTqD9cFa<Ma8JbiZW)!>tv1K7GuQt zp?hv%mG1%frkbGB=hSP2z0{?sM23`2kKVGYHlwD+W)1iA{tUzmP&|+44QY&CLbcQdbHV(+ zw5UgEwX;D_@~&~l$$EM2(2P%Xv{J}tp<1<0zt8yY$&7qyv1!74kkB@Bbt(5s?x+Uswr>Zlz_mA=-lXQ+m!|To03Z=T!+jd1`B4k9aB(6U}CuSID-^J}hS8>%^ z9~$$gH1FFIt0s&2h>MqvML8IjzMBbk{|UeM;#z}MzyE`P>w6s)ZC$rp?E*No zQ;1~8_C9vFQ{Q=q$3x4yw=&}1mnuW3@kj8(@5rQ#jBhqtv7WM{g#kHZ*=tW+kJ|0# zTW+4XTKFd~C|z@!!1E#e^R&1BSqVdrp*voX*%f6{9ZV=0MY(0KR=Tu@R?Wam zQNk_C6QbR;@B24cD5$1(i+)&st$UTReM5S))hGNxC<2WaF(2tD@40~&3&OrmR`kW81=g0kx)taQXuzEl0o5Au3GGu_nz*%bYD{l z*N2_kJ2#%S4NVqE;mHK-(eROICo-R}0l48;hl3Z3>|?#nh>`K3PpA`w#R%crLYT{3 z4t{uB0*O&7iSZj2YwO#O76q_tI5v1lrHD7^58~}3`TfO;){CA8pR)vO$oSU!P@$Z7 z>0Kt?n`Fp7<7zl<)trmmxn@>O^3}tjr8q6(F_mD_@j3JD!Ly0}C$6*GZV~d%zs2V@ zy`(lACf=r!^M_iC#NyHJwCR2ty`4Kge{eV`_ucN?kKV+j-4t7v=pYg0lDaqa6pwl!OLcbix8gmC=8CIh zN_pfB_lUplibEhg{QFD5tbhG5{Jy#}8&x9d%$Csnm-m63^P|}FTx?1BqDj9pyx0vQ zjRz1ci10(cx^SZcrd9f)n)tROJ9){kaKDcxu7ZZ7xcjTh6JaIkuW@|*=cY50WmL~% z@Fyzgd%|f6e>RMtOf`70)9EJ>vOXC5x_cZarzdWM^~vB%zU!{AnaS2s;!iUd)(||V zQL&P|Ov2BF-|a%>o^(l8+16Lp-Cy#i(H1njrq+zpCi1C;)z7dFarfoBpCPWTw8u*W z8YA&#X&N~E(?#~2q^?uV!f5KKX2V9ZPDf7b7(^})c8Y0cYmIOR*dew_D?;D$ik{?OO3eQxyRH#)|TrpMSsq=>IEk+7mqGI_h z0YmXy_ar+~cxL-orMr^^TYZg{S=lV`KELr-ThD2}LlvFQZ5t}AdlA{Qj#H0d&yz|cr0bk>^?H2;!+j_Vf2d5TmTWA1z`B(s zsXH-BZPNHa{;N%VDWyrgaZAZWMlgl=ic5tf!HdMC12hqtRsb5)T<~ z@tBA1p$$XPdYmgSllJPu%qj1a;N ze~8#DH)7U4cylui4pq>xQ)&51*51>vO4ZU6zX>cb6Ir82yUQ+wa%Z z$?zz#-KF}<`GUxsSnJ*~ky_8SMVI~cv1GXim-AzoOEXQHW+f4NQj>&iY|4W_} z6gxHsS4@mi^h(?X@|+b(7)lrLDkB+>E2g zisq^D^?-ifmoAL;t*dBf!>GoRQ}>NJ*&ms$6dk7qFT2U_DR&<94y0{P;$|l4i4c%k zS=?=^Oz3@EOwm-k_7UUsBwpUlGHAE|PJbmqBdLN{t;@5I-G_Iztol+F9#+xFTZA<3 z3hjAB;=KKm9d`Xd-u+ipk9v+x-yk+|>%@U(f*7enh*9f{(>MVy@%Q^@O+D0JPw_s! zRsEEzrK2=86R_d)Wx)0P49oFeD<%7p#!cO>v*aofM>twKve|Fh>005Wkr?F*)n&nw zIB_Lo(?)N6+0mWH);ER&{nMTYtj9DTG^#7j%fwu+#iPMx=0CadYnb6V=p&xv-Qaa2V~IizemVbW)9!f%BTpqz>+4C*){Q=* zZqVfmi8k>h*7-O+w3(&TV(Ynkf+2f#yQX$lgMD@;|0;X13|HOCvFD?Aqc?qmdE6O3 zVqI-`bp9}~>yq$k=lh|!frNh72qB4u->$hg|6r$R^B*L3r)Jsx}iK#ZfN zc{pOTN~cy=llo`WagI);*22&-{qFnm#(~rx9e0mEA9uS=rd0$?fa-ORP)^g4H>nVo zp9p;%`@LpaAF{MZ;SOY$N4_QfaBuRL3CQ98S54n}3*NCt^If4)%Gyw}tG`_uId=>S9?p%gKH$U6RJ zu5G4vJok;z+nQsNPKK)b5rYnc=eOq_x{i~QtyJs|Jt%EDlcW9ml@Dr`@d>q$!JLst zYL6XG?oA0gZHnYCubp+cOR0O(IyWo}Xfw-4_o)0vAP1e5&Gbv}-8}`3xW`N1bn^Os zK052eR+nM9BJqK%&y~SG^3?5&I{uRoCnp@J_r$bHUm!sfMf};&$!zBR9l?Q5vs>?z zbs|{K#rru&uC)eMe&^2Bg5EgdNb&2$)9a^x zBn`S!$d+$ZS{~0n-(wAlPhA)q+LXBE@?&e5Wa<>ggT=bXJJZ$sws4?Eg_o}aZS~S< z>zf6~jRUr(^AF9qI%V%In-0!-Q!Sax)-7|e0 z9qP>GLdMpeS~k{Xuxd8(W?b!CYRp%LExH#!pD#IkpA6C0Vu!2IG9^Pu)c9Pmh%H8q$f$(pLxC1iDSTMx*G;2X$(y#WxWbZ zlNlzr>)dp{YiaL0&vD`Ax}iqjco;rjkR$EHTDa7t&3^Z}zF`@ta@@LoH0q(V{os~0P50^Gf9M`r4x?f1Q_Jq2eEUxEkh=n%yiym~*2%7--qqSM*F1ArWuuNq zCYo=x9+gb0FeyQtaol@1%{Xhmy`t@{5r4r^K|BtR?nl=HbKG3g55z^v%r`Na5b$eo zaxWO@m}Vk=@uy|mj|_C0S?TU@o9Zx*s;dUSBV>5D7_Atpltj|N(6=_$Kyry!S}s!k z6GDnZ!uV_l+vWJjqif)#zDG@U=<&DbYz}1Kt)|a$vK2kL)*W&e!{{BDi*NxB}&7 z`YxMCsa4`cxP0WUC0Z})0qU%MNUK}maq6c$Y<0k$OJ`8YdE*^?x5^-u^{%-{@pudQ zm)urvy-Mp5y2ti^Y^i@dIG2!Z{g7@KL+&DPCi_}Mz*{MTg_D-UBrJZ_sN5+KkD-$Y ztJ2x(+}lE;Y)bisV@&ID)Ni&Z%bO!%t?MVzY@CrS_c|Z<@2OsmsW{DNmdN@1Gp1GB zMj-3iN<&o>9+|O+-P8P4h7`xaeH0JvMKgzxX8qTlVduFsjFiP|?!>G|o^xkVkBGF5b5Q*OF5IyQBL822W z5+zZBAQD6fB2lA82~mUS5?~?;H9}sQ|)>;C`xV=0^=51Gny3k7va8|3a>^O=#DCAYQf+7GPE1EUJeB(yJ+ zJ=@aSJn>BxshDs_Jqv@Nm<@4e4StL(WYG)$JhVc!R>X5~qbtOMcDVy*^xv1r971P7r3>cF6gJuw$ zm6Ug-d%Uwg7`yf5al^;vf|go`z8~EWuNd%tFFuoP^HpUiBxL{@xF36o_D6wt)X$1> zi@X;mgU0zYKhzE%PH-1%pG35q3{g;T5l-z=cg3Qvj#5zvI#4E51fx`vkwGih={QFm zOBZ=THUk`aVuo+q4u@w+eeqai8A}WeuylReP2vAqp?N=6t^Wx*bwct|F$>jH;ylCl z5AvJ)s--2v5~8e=evCdJ2Td-Dd~2e6@l}4$eSu*)u@-eeahmb50$*&MS?ikwkpRYqr9Dxx-^Y!U$7D_-1^{P&KmRX@*hV;|a0Zw=eUkRR#|90>g@dn^tm_i@xH_ z`umm-xv=`;=Vmeb?zAn=1P-exoX*(=?X0_b-eD& zSBN^9Fn>aDFwHYcT_PWzp6=e+CU9rkaBBLFH%8UzI_k}xA6I{4mUe#&5_DKB7sj36 zzJ1niBJ#dR=b7~w4=%R4xr;Vo<>D&pS7OmpTVHs5$DRB-OL|&Akme1)49P$hs%^=d z)_CU$P+_>?@8xHNj)8_Jm|X~M-jkY)rQNKZ1IOkm{QDNSK(_?qLY-&AKW|b~#MQU* zHIVjp^Qrf15E~<|f7-DxEXqjW!!v0xaI#Leuncy~4vaR~|8x2V0H}$Z5WoND^`R8; z&h2MA9!@+Fk9@5TpOqu8SI}@e=wvJZS{?}EYj&Mf`Tco{L}LTfmfWu&cITDT6zv_^ z#z$9uV=iB*Dg3|NIXccr%UR7qX1m>51y`YF2d0s*x0qL7qTt!pj%`#*vaX%WH|RLa zM`~I5zes;h_Jp>&}@{fy}l_Qnd+Jx7+pLyx-ex!b;&)@UnQx!Ta-R>KiJFZTAQmc4rf% zF;QQW*%ei5=aOg~8d~4;sb|TCk!{lVEQ?DeM=cZP+ z%TL-0gLmN&s844Q4me5OFHNdUkMZ~#R|-|t;BNk{y=mgnwACT*ldir-2Toow@eJs_ zZ2;|yOB>wLcJ?=$(@VzXg{{pAe~Ms>g4q963n0!J>3dhYNj0+B%Hn%WcA9mdOQY&% zR_TBv?Z>Kw_G|$PxRZB01%VSC))P#gTb8ax-C^G@RCvjp$vZkG^Kp9PeWI`Zb4YM> zE0QmxK(3f0kcV79k)qKsV5K-zRQP(V0^x$ zSBeLI>OUrv8a|28a>hQ!1m*Z$X#{KG^-WOC4XY2Rq=gdSIOJstv`~nN%9>y zcs8o?7k;SVWBEIkV64UUxG;3G{zrl=Ix*I&!JnRrL;igbM{um)WXP0E`uOxzrLW9f z$7|DhW}7DuzB(%=6rKo4jm+}|ISoCvGby)uvt7!B9{)9Jnlsayt%MQ@RlV=J88!aH zEbSS=H{wrY#V+}F74m{Z8pQkLXWwlHxTw}+TvCO8f76Y`I3L~=G%Xv00N#Ev=8s6X zlgC<*&e85RzoLFRBPoq6Wx9fxRjh3fdqJ{2okk>&dW!GbW@ar2dNotu9+}9gp1snO zD+WJqsJSY?0r)>fY$FO1w7t`kvJ6o2d6Xv`@<3@T-fme-D+rLl>VROy3qm?_=>)?PNM; z?P9*7#^Cos_qB8`wQTvY!_3*B;hLZ!w6mwy63@leY8d~v=TTMS=}^^w1@-NzsAM5^g8MZ_I^ z`dNILBW7gQQI)Xx!}LamS}c>q4W#!SmfCk6gDiuyoiEr-ve9d&yX^U^$wW%tA{GU# z!g-CcH`sCt9n0vSe)><wkx2GCq4)@uUO7F}-JeQL4)v-G->*_qtkSn~De4!Y=bTMnDB!>%4BJy#OzcLlt04lwVif_GQ#H<7hi6i?3HbywxoBg}y-Dc|WLCB5i=i_`2%K zc8!*Bki(CmL?@$(OL|g*n25$E#{{tVA0?0(kak^;G9fFtGs|rH(lhlVP4(=Dv+qN; zgceKT@tK0jzovwINsngk9zr?uv1^)`wu{Tx3e6NcpLc|F@f6;=GfZ;@lpktk;FG@A zb_{2RW{GTZ$Oq|VOo@*5@E42K*KxVD_vVMS12EZ!5?;DpE*93aoRT9xE6hb2_ZGwx zew}w;=%D@G)cdX8H?Zu|)OJLy#Cu?0A7%@|caqrZ9r^DnboJh(0vl@fWpxpq)}+%7 z^{tL-*qmUnN!I@Wutzh%dGCB)B zC-rqT#4#3lH;dPYt%o8pP|Pms{1rR+YJEddJ5@GH)`=nQHpR=#UhiiPs-}l#9hoxf zPUxcW>6YQd>$q8|B)guJs+0th>xCKFC=cbd9D^#CZeecMemyJy%|P*9h2Za^>E-Cs zE}M~BgKwXkYSYPxd3v&OefOk4n!eGGMUT>A??sHHc&HgY@^rp0RvM|AcX9J+jpz-t zjc8P%-{GX`=H1z#Ek}z?jF0&+o|HE^S9S1uk|2W}>DBQWzS0YxWgZT^9}WZ9^{u7_ zY$uvrC9s!cN9H(qF!r?xAJRI0jlMMg6u+pXE1uq99@D{A={g@+^XNhHQDnVahodH; zA3Yc14k#3Z$@~7MhwDguQKsdZ+Z`%kJuNA2(^0-s=jmnUI+IAyFYxeq^dVG5nLs#4 z75e%~pn3O;O->55Oe&7zv_4kjWg2Sc*Q^$w8Jg>b^}fbLfBw!p0W4Lzu_J}W+QVH@x9X7h$nj)9vWAB8r%uhBhCKk6|fe(P^~ zuvHlH*!-u&pm5NO;H;OoUmmD>+4lKnDcmU!Y_jiro5H-AH}>JJf=NSARlCbB+Md4i z&MmR7OyJC1=PjNky%`Mac(8(RezV^1@w2u*OLXmXA?eQAc+rVaOqb{5l_yM8c6Te1 z9bzBb)YQ|Ye6ZWxd3XQUg2gZE08_zH+BuZD{*bw@;bJy=4)(@3s;zg0mZbM2&i@x{ zU0D-G9hKLvba*9^w$?QJxi_1zX@L=2oC><^Z;LNoQ5Iko%mbRA_c4h>-#ohF`8nfs=Q8;f+L=GU<{ z#j3|CT}{^OLUl!>oA1RTZ+!CYVkgmFND_~FDf`;s&x{s#KXsZuuZ_v!zW9GL}M%0OY2`6-%St{9HDY!6b3pv-I6nezyX@5MH;Ud7&RswV~pkUT( zu;tIV+Eh}=SNOA$2%oJ%gD^2SaN5%ZFyC&e$??{0nZ>gzs@sd7(VS63zrRiQH>>@W zvDb_&j!7!I!QVwk3aVTwB<_DQYB_9K+NW61fAEZ*YPBQC-}g`NC%FuF}LLqDQQKiNS(^K===7sFOMx>#ox*oR+CM5pm23!gCT9lJ|@ZM;A(}lg7=Vk zc~mFwo9CBeQNbZtwfDm&ECTR2)}%k)d%R8iS1kvBJE}&4V!mR}&mRTeMQ%T6c48KH z=qfnSd4yABJCGiB?!d37Y)QmS53#65>M@8EZ{MVxxf{z9ZOFK=ZIizEim(OQk zRyYKw^iW;ziJktnSwEPnN?0dab!m>8kHEI{La@xczEmkLEl@NJqHNOGBID*1Ier@h zM7@U5>46XizK^h`47VmhhwaHyjyPd$?N6)wf30BEQQnPpe@ZBy`*c4PVz}NOs>XKv zCrS0EZ@-nl2`914T?eokxp3E-QDW3uUf)qhY?#)uap_Kbq zmObki<8*HAecxB~@%)fmfdXY93N=R&y}$1>ektvXU>SZP9g-%N7!=%>^y%@n zc@5@U_OIz@kL*aP6CSF6C~WEcAieCH>-|E@-#+f?4yZ>z8_=tVXa<+qBKwE^`*sw5 zZkW{RkBUEe2P*W?Qd8|YwJs4?t*k98H)~NUA31`v%SyZyWp1BH?{^BAw0s5!zv-BB zHjg7{jAEZp){LFX46mZ1OA{+}niebFak|_6HaNH12VF!W#>9K);8lv}SrT6k_atZi zH&bQyHiCIuoV4jk`N`O==$FBf-k`Wt?}AWLzg^&|;EG6ICsuHjblbn}Ml=DBwo0|8 z5F0qZEN4%vqHz@Y%YD0F?8ia+Q&X~cJksL$I4C1IyU5KlcXO10GdN&-BQ>o>UeV`- zZcbxy|2z{0;%Ij)K zGN^7X9S8Tyz6^R)aDaTK$Mt90TDNQX* zrgEqqCZ6|x_>q^5eCRf+x)UpLH@#*fy>9zUcTn(PiBQAupi4iE7gh%v;MjM{?&r*E z<3FYskYncpx<=6R*llBMAV)~9(qR)aXSiaHj#2QNzw&B`7`3N%9I<`*rhhWL7;SdR zVO^V;3SbRSTSV2nLpePk#*lOx;ZNUuuYG5k7@q*vRRbKc}j;wa& zb*>89uBW0Q-?dl9YDp&!C_Aqie0x&%o90}b@<$`V0~HaH#YG`jlEj>NZzl1W;bT6x zix|~(5jY#TR|&2Hs*dQLb6%H|T2AO@5NVXe6O0jmVkvMqL>P4zNdHw>uO?h@JPD$BPW z-}u#=fb3vd1;;ij#X};IbOSg4yg1%_isuSSV=WOGtM8u;5eogiKF(qM6J1Z5?@G{E z=n$Y4Cn&~m11dDm1j{Xk7517n+6lc{{Mr^^yZXc)R6)6Ej{@_X^$m8@{;^k%z ztX+;Xa0&Gcuzb&Ib>OG@OZ-KcTr5&$r!2h!>B{|){#v%Yx_sfciI#QFb>1X>|0$uo z7rJJQ28z68fBG@<_dT(ABK6z5LlaI{-c%i!^?jJmz=hmKkx>WVR#`o37zE?ea3fT< zp2Ig}ogRzcp!Fxgjlb97wYiUQva2O{m<%c#%8(yaGQJWkzQ}Oc>n_=bT6~(fXxyxtOwuh~^ zytV5NyT*U?J-kx4?sz$KD}s3`6rQ5i`Z>{V7ueD zjY)afNhodCcw+Qk*wnt>@3bBmDNXa;i;ABZuN&4eTtHBOgnVgmeZcXx_aPD>Q>61K z5^lHZ5ujk%?9D!0VSOoE)wo3e?(p^Z$U#tqhA!n$FGn+7Q$NCHpDVeN*hCel{5A<3 zyRltMR7}GufDGTprVpNA8$>@d<4aq6>q}vK)5W`~GFms9P8u9;NOpexk>DcbzB6a` zK9uW%;-~E=vX%Mgo#~^0v#lBDyk$+c?(v*iA@UIo+d41+`%VL_?k4rX4ee(~XFkmh z9hZ{dcmKUTaXs#!1&hD&@G3s1*VASlt^HIdVm+k@NQ!rYBp#mq{mfZ^hxUTVtz-l{}gH2)w6K zQ|9^3n{eQH6JCGPSs(M`&!W}5QHg|rl;RDwVg6r{Cg|6oX7})E7|rc`Cgie zOGZJrqsVnI5xWdnzyJ}?Qs1FKzv0He=bRg1*-LAC1BR}3VR?hUH5=uQx!YUD-FI(- zW22?r3A;M)W5xT%%xl}J7w9FK@yRM+^z{mq!N$La{x-odQEUtKk zjrt8QIBC;G7(a7+d<8wMD}F?q(d+I?vvC2$9Zd$35wl>+p@J*)P6mZXhwK{1vGR;k zeiR?RSx(;Q1;%w+>6?P9fY{S~j_DF6B7}?T+tzd`Q?Qt3hdROz+++z4;vz#H5f+Hz zWMDtl-yS{0Lp$}zRus$%_HdAr!i|O;${zb`qhok(2NEbV!Ja9w#!P6pR-dMi2SDY} zF_E)*o%{ti)5Q3}pLQP>mnQRDqqz*(z}FCycyE0hICDdUDd2&@c+-CKi%T?ewYQB* z(Fc@qj6|bS#aO;#-z0<4%g(`HLE^-Y@^WpNLV)u&-d3I?yQb3p}T)gb` zKKb9;83)5cTJ6FKdVNFz74Ae`?ex~3GCF6+I-R5S3M{d>mBbJ-Q8_|c1S6$wOQzL) z_eS)CVe#-R$^tvO{Z?lnrQo~Dz-@zu6-JD$v{T4?mr|4$ zPwDCXpVLzvb`kRglvjVCuqDE6o+RG=3~Wjr6W$ApM(k;v>vBI&7Bce{@OpHg_ePPxc`(@NsBR?ERq1ri4jWVnPMPUJMKU@u>7nBAKQ z+l6!S!q1`lsVHtD_CAtPekK9B@QtSb<+IrOCny~Ho2F`~4w@CP4oM;?kpp@X5Igps z4u;(j04mTFkIDz2vXKrnFAs081Z{FE>4D~z*RU(j8ORD5{E7j0Zwqp%+B=MZ@~vpt zqHt?4VW(^};cwj`fq)`g48r*OEhbzTEFD(asoMPcEKTecFtobBL4aDgfZu2%+gVD8R#e9kS-4@n>Nl5hJ9aZA&NpcB(KqfZbDscXbrq zw@cYAF=^%y=fVQ~!y=sFwoIJL%5=wh#um4CFs zzqs}=0s=C9G{6edlw-Ge>moD44Wq`8rtUu|(d%F#LCmP36Zpn4}gNzL$0gtQWXZG z*)#_{o@vc4C(G*`KVMs5Y5^-=Wh4v%8J0?g@VZyKMNQ_Kk%6_zv2e%?xWlVp=C{5- z5-A>_?lnMM(g54_$DAA;P5-WS0ho#w?L`K$QS>i(?Bw%L!YiqjZ@{Jq9xp2uN*IHG z-lI-QU!s7qe*Ibh$5HGL)B^#uX#fpJh+8Yh4!fKW>kTjJwEoi>q9Q>Z6LC${6Wof1 z@<2!^1s4Gu%>}@t56E=)I+djxq>EBRpE)n-58mSGX4uw%jaSCnc;5xGZ9#e?M9;Jm ztoFOdghll^0ef!JE|A8ZhZVsepRtW&=v*+u+MWZ0ncjZ=zc4Y5%aU+W9r7AhT*e9Y zg<_x#m5kxy(YyJNU_2H*N|5AjIS^g1Xhia1ixjrF5X-VM>~sj2$3NdWQnLgFlyM{* zQUmSQ(-s6Y+LYiOR0I&x_$lZD>ja$%5`7ZB^Hqbst?2L_?l-|^7eOH`8vc^BWPp-_ zphoiNogQD@7NnhY;gH5=Q58gA)abC)ZY6-Gf(eN?f&=PoZ<#;hEyMq+1-S6n1RBBk z?)T}wfwv>?gK1~DJ9bXd-T>l;m(}fs!9ZC1tFP^)w-@9lpc|)bh-dH*K`F>$U6}uZ zT%i~ZSV|hHb^rtamZxQqlHF_mGzQ!kGpM|Y-h%eDql5Qp&ix3o>ClFaGY|fy^?#d5 zWGSC&&EXE+U6?n?1__RHjA5slO{^GDS;-kr4U>hXyi=vde3-%f?R9|ZN3~ut=)tb@ za=I3!;-*dk%K{AY-}b)wZvd-VPJVq{X zSqDcO0$9Ba3GT_fq(oL9TzhrSAI1Z7vRjyYcn=b{B%(v@-3ja<fIgUA#j z&~V7$Z|uT*>cbpM|6N;-|KcjE*A0F>;Za2rR>DC9bTL>&Ula6GvGqW#--?MIeYGNJ zBDf|p0WtJK_&)3@e55eLKfv~m-x3>X7PPJO3TW}XKt^pH7fS(Ya4U@D*INL#pjM zWbGHYQmo}9_0QM};5sgB<*z|=#k5+C9qL%a@V}lU_*fS6Vr>;5!@Ou;AU`8CRA^?B)`F3L|Q*cqI z3~c|m&0+ql7QQKa?7;VSLjl*YLm4v(?HICJO?A}IOegs|)lYR)_4@!NdCYA~XO>A) z$$4vHW(8{t!J+Bx#Qz07Q~ZiV?y6pYcwwI8*{2DVODYBAILz348+aDeSwnY2pdtBa zFpIC0L&#>_G$SFYIut{2-YbupHIALeNc)~5D9wfd__QfL?Jm$&MgSD8tH60GBbuRL zV9K;pJC;)xrv$?ix-%;`XlLD30}@HW$q_OWWR;(1MHbwb%N;pwm86Q0QAu5Txe(=)oyJD zLsX88fiD&R?>b)nD{?=zlfQxXXz1k?M9MWUIR#wW4tY2`kkuY6ya;x|$wQl;n7>Q; zltCAfO%-t{PdJEy5IBpxGw&6#(-o&veXL`x`OSo)dv20=gq;3qdfVrpMn)vz1!&uq zCN>x%FWJS$$oSd7>8vi`tqo>f%F|9+mq0KS$=B}I-gOi9xI*SCO{@7iS)F|FU~rzY zFU$h|9$u;6_66{pJ+X$(ucwa(k3{!yPboiI2HmfRpzTQ8*PdLoBL zqpj0E2B5Zn4+}1^{q287vq6GL&OeIjy0?qw<6B(Yql}t8RO?BAg1vv`HXw*nc7Os3 z@ue}>70?CN$y$m)w9A*e-wx=I4I1tIr4FWznC4ww*lcM=G1^0jPlIUquqg%OmK0M> zmu}P1Y3)5_$LN}Oq9o}mjlDHcZOIXRI%mXWf(Gj>$xE0+j z07^Q^1A4D%-RVCvHzRiId;3efRRnT%RgSLbZ*>=;3p}gjoi}tgcmESZZVDfVQv~&L z!c66BLka~>P!jeaY7vE;^+MHxKIHjP*?^i@{f1hmZ>BNFB(R#>vodUuYBT$lpAs?xXq;Cq` z3(z|48Hgi13U6IF{^9Yx%?+un{G)1>JRb2k3;N%)L*M%9vj>9B+5+2cSKZ?kfxFz} z4sVAEIH&#~uIv-x7j*)HazpJgw@ThA0Pfr0-Y7{0aRRyGd(T6csUN&lNTzoeU1JgZ zTo7#dpO)d*y*oR3w=;G}p6@U#DT-?U|Dzq|p)0pw3oh0#U`CUh>|azM?cf$AQsxq> z{|>vbcdH-)ICk^^DPy<6d1BG?12^XgvC|}wUQBZWQ-SnCk3l z7Vs}cwE_uTp}8bp_gF4JE+HJulVyOWz)6giguu5YzQKF75Y3Pb$X~d2&S=fbC+rOW z#^G@OJ@K}0iHh`2M=vJazKsM8TKw#l!sI^38fLE@Iz2uE@sSL89#qtZ%xPhue=5*U z>qzJ1!Z9XSl8-Ys*MjN|M4&x+3a%=;(LtSY;jWlCRQ2rz3}xdSP8#-zN%u{bgxpOG zTnXA=X)u`e#ixi)vOG2sXF=2((_8U305(HFkLE*WH7=lBr$Sn9Bb%#A5+Vo%p@67c zn1T(0z;JgE@n9_LS0?M_a4O~~1q!8a3c?4V>7A_-;w(&XaGag0p!8U^pc4#7&*=_V zM&o>7&972yyrT$%qhZN)LQUyHEfWnaDk=d3Wk;|muSeCDW!`_Wp>>V$!n#s2Ekh^) z03oYU%BA;O7 zDN!YjvWy&w+7#eQ(Zg0@*e2Ve2aKtBoJAJM3fg*-i`@Rl7Vapa=!7nIB&TwP7?TI- z4MXq@_R+f2t#LM0KSs{N;C;mkVR>Hqooz8I%;d+=)=&{RuJRVm?G)(IaST|RT>9;) zg)o?UFnS2lO88CdJ?wN4Gy%>&XG;s_P~(C=?JP<|m|jIMRDLw6Cq$LZZCQU zsm%rrP|CfdofO$F;~1-QCY+8UhZq60_Yw$KhrMv_oj`27FkvgT0T0)r(T-L*E_ojh z8SWx-(xo*I7ip=cIa16?jCubsY)P@D`L#^Ng=V z7&>c2-it6CyYhJ2?6HlIxFu#gL|824m>3BTf`c0?=+nof}S#W>pHK zz2>UBFYy^oxDvOs67WonjhrK1w?$eC$SVxmCA17Z_zc3EJz;27u6NSFsv^uC0EBJ6 z=3_#Er>u}z0{5=z+lDZbmGAP$(4ttwJwcC6>kN|0Hc$2e49&(RAnOr#JB!4zv890i zEp|8@{_Fj5z1w^B>SUVfV!zp$)jf}ms}HmM6*^o5r)=NVly*bHnWg|uFlV~H{MqO` z?%Z2mKdk@t!^*PRgJhRo{^~mtBBF^zdeoi%ksYF#AbUmNFu+SQK(AH(ovMUR6D81Q zc*u%{N?z|-9IDzSWEqw<`&N48Gz~e4Mi<=;i4ON9Ijoafm}UeopQu62ea-8I;0e3= z{Y?C%>I#7FqEHI?&4SsDVh?t$mM`*1mEW-7pP$hyr%@1g+KteV7;SsC=O!sn!eaVT za|=31%617r!>_=b0$^oK{1eKBoryk%(f4Kfw8tG zKsu&Aj&1YTTwujD?3`CzVYYVlnzr`PXdCR|6|mqkid|M^W7zN;DS0gr1#d_o1}VOO zOoC}1m>r$DPbdhHswYU0TS*gF-f#Rk5h-a?pbUk+&}e}fG?0k zIafg$avGbSChru_k=Lf*JIASU2V=009FPa7)uM_j*$x=Yt0?)>;io<{%5}>EG(Z*kI$echtQ>At69@O{aA=n$EMZ zt`&CFF?2{`jNp95`YZr!u*&&qBz zs|2O)w?hZeuNI+p!TO~UC&{h z$=&dAbfG1MPRx{LVCM`T-oFevz0;qj(^MEIw2ISISmh_#j(%M+$uIKOkK$0m-*^zR zP>bYygmZ5mH_>MWCtn+2xa$DLv5%-3i3`#$u!!*HMCa0`{yz-VL~tPL3*)F2(aha6 zAXmnW8-p#F59!IC2Ib(@P;H=RN|xTy`e5iZ2o~F`AAJF7S4lS##G)g4-x3xWnh32a zRY4a0Gy*mlB4gSCUPdhub`Utd_=IO^ZPw0Dr3EDX-{dPFLI-w4HZ6;L5BfhKF2f`d zXemV=v;Xh3N7cH~t)F3v#O}1pB(N>P_48~%Wj7oV74VTW=ob2t;IQHM_X%#?28&KL zv?azJj0rNqS(CS!WZ=NR)I(8&u;p!igf`_g1MC_M%SF{q!&n&7HNe#B{{L0%OzWYf z=$5Fk0bMcHm#;d#lYmwalWJ#Ni1cRk(=+gU^OJiQe* zyBEYyuA(d|%BP_*FL0fghw<79y_NO)`!rzs-?H8nXAfuNB{1Y+RK zN&Zw`O^iYe)53}buW#`1*29~1aH`zQER{Z`9Vb=Oru_4#8zPMi@L_JP~sYH$~ zR=F_~t%NyE)&F@i>SZoTB%J2`q~sj870oB%fpdf?Wf+`rfL|Gf$=&LGOs$L#KoC-M z%9M3C3LtqKCMVWZ5AJb}TF6Gh5YE(#d`qM{wU!Cz56gQMUlwCh=9aOY@8p|m4ZVp(rx?N`kV_B! zG!tQnE?RUf>b18J2toH*qZ z!t$}p41V<{*iRp(3T0>z$T10*c7zeXoGcEY;}@PQrv ztln&Jh~V(|q$qXm@pN%CAxirRSgp3f5qzy}(j2eZSjsqg?+Vq@88GoBTPX%r(@zx5 zF<6Vf+-R`kIRR1Qfg#r)#CZCJK=h6pEc`nXrOuZmaZ@J=mqBx=j(`5#sn7W{o%DD{ zaFFIe;pY|3H>!ZaC}ngm;Ol_QdIB??5YG$PP2+W|H1<$icJTLt-8E9XKy08EFL>+| z;;;dw%B2&gBWpO&DtTAb7YMSZwG4kDLsDx%w?>hf1Q1XJO-m-htCOP4tHYslV2W2L z)41GvG7@lU;2FL_wxvYlJ$ExHS5*i9wav6w1_DAj|E}ZfvA(Auww&9&@ZF00Jx3bH{ zdQ{sr?qzCTO~95+q@~~z+3u=8gKA~pC1>!q@(TpLleyYJGJfdOZ}dtG3{gsrjx}QT z>t(hHzr+1Tq-%(?`o}9Lvhts6$n0~@?f+hzjPNqY$>bQxG|5W?8Ti{W#LPhDjBdft zrG=JRM7%xN%qjW89dh|+l1C^ZDz>IIAJ42;ZTlsiaU^cp%1e6>#o)S!VcI zZCfdwcie9)iCzc5mB#uuzltqxk4&D94uiDdFPfoa{TPc!7>ZX!wtAYep-iI9H2DC;FN=jkeu)R z))oVt33oEKi%Q<&cSH;rq+K!*b&cCK&kg)cG7l@60QVr_(wTxKhKI|@L)^py115@@ z?2JOydfse!dwfUk<@C)3LP(Ilh+4~@Pp+3O-c4j)_g8GR8ih3SvGFvYPIAMwyi{*K zLnIpI1_t<*rxa)t9DbL3^bd?9&D;Ce#XcrAd@D*k`XZ3O0qd&o#a zbE`vhzpLdYKCCAX^u_yG!rnfW7=SeQF>)NUBst8~Xg{ioyjm0W*L3@@8N1bOonPB6 zv`lUf1#?q_0jRT)4b&{*_E-hZC3c;%_m75`Ru+BnD^!IZ_zQ--@M1_!fnWoHV2h-9 zw~^ql4;o+=;;JDPLz)goCr=MvShf`#7J`l4?} z-ajbG4%WGp?V=eEP?8=Un=-sV(Mp6WXRWHi2uF$vRNS1gg&4j{=<9_cJe2AlbHoc* zOkhS?z*_sS;g7Z!U9gCNFG}yCQ0Ce>KM}~fAFGS3r`$-!O+0kWKrw4q z|0JmuyxWo#6x!Q*3GnO=f<@<`qND@=NN-GiApI3a)S3`44Cr2x=-Jy8a?dMS#^wqw zEd+BsoSnC_eO#iz&4yX(YU0Qx;Dj^{(BuSWE$kYB14?2`fR>wI)Vrv|t25TdwnPdr zqY-h68R>*8FO3fMzmD+=M{b0s%WB(z*?kc)M2Z!E^o=`55^R$sr)MJPa6 zK9^;pR!K#^_KtfKA*Ca$H&BBYP6&2eLjNV59|hPPp>rb^_$V^41euFlzt$w8o{v*^ z=vI>I9WJjGv(mzXDo4PcOXF>M_+mgG8R2c6`^DqZ>-QbHoBII= z-U5PYXi=^fkaZ=0nbSDZKLn|g$G&bR@^J8+YI%l%yJ9Q!0vOblnXeUH(zZ1bsBnrbv^`w zb+^6^f&iqt30Oy>gjYQ|dS$Nm=sZe!?@h;>QOE}Q5?kBxpf~UdAl&+#!8Pe~fJy2F zZ?G0)j$AtA+-5aPKWM}+L<8oXhqx#dp)4H^$0uceH5<^H@pW9LA#LJvCegV{{6$yk z^FMPJL;@b^#`Zm-2CyLy#Ppbc3&o&Z$--EEytIa`bz}_fouzZs4lATJ^M@M3?!&^4k$AIN^85#fN%nChMKsNWdQE|A@xl zR?srvCOY9-x}kGXr&`X~=`ZWiMyh~Y-0y#Ug%qA0FkdjrnWH>k&{=BtW0Nv$=9~P0 z*XaVL5NzBN!p>TL(~VZhx0#Lwf8(zan&H~7H8pPN5CZO^#CkAi);{c{E)t@bQS|#%H+7 zo3;WgoOue0BA>V2ftOK6gYOM8vf_S{{cXllN9snS#5ZXEL8RrmrQqGMG*=noO7Kt= zV5qV5(wWy+5$`4@D4uEOTzbY;I8j-qmpy#o^S6lQdRT;zipwgw4uf7G=B@n=z&P;T=UU_9UA`B{FPOE) z0et5_3oLdZ?&gRL`A7_@h4xe4xJ?Hc3Hp#u#kJ&T%UCO;?U!DvU?Pa-$H2T>A3&g@ z9j&Z;7rA5BYPV)LA9Z}g@4$?R@SL1J#EK$e_s541%X61cZYm7?s}|t#^C?$MF_6SWi4=jy>;MB>Y8*e__PhUX1<3)T858WFTQ9Bn{}WRsVe>{xTJQC z-s{WzBz*W^Tt^WAxr=3C8)AXtJ77|((W-~aFuS|3J~Yt4pK+29TYe%atY9w}3*w6$UK79Dy0u$gjurfX>x-fIm?+JhgMoC z-?9WivWVjyIYOVLe1QF`s?CQ!x;A-%K?oo$Q?H$tU1N`_xow^-eQfBGgv7I2##P$5#Qjm^|wz<|oy2F`! zmrkQ6G)e@p|5uJk;5{JqE4(Qw=!g(u->gHHu&GwlurrZV z%5X0lQig)sQ}ET(*^x^lLapSGLeH7%0uyp0BdlhSRvnndui?GTcN(i4BMI44zN!sW z;fd8B+jtZGW&Kyk{eQY5VFVPvUb*KkauqR1Ren%7wX$%xY{LFVTUp8sP%m+!F%Ut{ zxca*_qS}JGScQuLnVe)JiJ|mPvfpyOD#3|oA9jy^symk6^g(X!Rh_+50dP*bE(ec$s2DvY!Y}g7fb_N&P@cwyJU0pp+ z)scrRu`F<%XfS^>f9vOnv%Sw!Lt4&xw;a#Zjd#h;sJ9=}EQkn$1lKi>wa~$~3@Z?R zDq9k*2+u#m)_4-skA+0PmVQA{GzFBD32yPT9mIa;$Dm*+S$};I37pGSuvmw(=p+rJ z))TF776UEZ6ZCISK9}YY{RCU60UrELie%G*)d=F0U2wupC`gm~ME!1RxOi7lXAh(y zc2sS%5%jv!2PN-nzN~%2oK;0@szg$ZPlb`_jA2CIQxU);yD-tckD$eUCA{hZkF>ZJ%W7Fj5UanKoqa=Xd&+znQ+Dose>ow^j zNp=Mc6LM6Pl=T%opyG5UGp zNdq~0BscgJ8c7kg!7CcBXq!e@1kv+TCf*_P^~OX##~!aDkWycF3RVD&EdhW?X_mS9 z{xx|p=ab@+JJ^tvL%R_P1fn)HBja(9(ZC;#0wr$<6oA=7CW4e2?4x8XxkgN}C4kwM zZoFKdI|_E~)%5-wkHoAUBV8wU``>nY_XV~{x-o_W`+YP|LbbBo5x$=_8$m*^=&v?# z-#PDGM-mJXM>{fRqB1g=xMByq*MW=&m!mQOH!`?I&RFIX*Xc|d#lB-}llW$LLtyf+ zfpuhEL{Er3Z56>^z5lAY#4SGXhGC{#_|^YI*IP$L`GswxGSuK845%O=ozhAQ(m9}l zN;je)9nw9tN;ybKBOnb!ONSzglq21!NQZQD_Ji;5UEli7Ie)m8x@Ml)d*6NC*S()o z?w}Ier|gm?^5ByA#DFU9R ze8c*WbSEu`U7m@Myy1%I?++z+_b?qNIJABm#OG=@i|=!+^W&N)gp5RFwFUs;E1`Dv zb0?qukO}*h`3)G86kfxwIUa#w`q&tNQPh(eno4o3;M1?bazAumIf)DkjAyE-Da^5c zuz>W-k8kC}@J@Q=<@~U9V{`xKdoQ4ip{(-Woc+jF=8wjwG;7yB#4HV#U3Rdl0Yz{z zAr+rd@elIrKl%Fl&u$-GM(f1MQUt^KZgDdfQ!R|=(aZ*iitMBeL!LS5;>sU>xB>Fp zn5-`0Y*A8lxC7X56+u1FO%9ADKycPx>QFCU0^aEXF!<|MQPPR>nbkQj);=A3JbFp< z(Fk_q)#%goBQdKdZM~n-ph(b$8EM`dbQKygfR7~G>WuxhMXlO8Uw4m6Xbr|h zdjzzO8t6i54&^IcSa8x#tppV9UgaXNY5<5J?+f(F5g^VyT$r7YYyyIZ-J}kMvX|Ow zXT7_IAlG9@c;AhhVAzcn+YrYWvSDGTqSy(3RSj12DW0#xDRy~awUn~Gk;@I!3_jfm z0RTRy3x0nIyw&PWaY)n=)p}KGK`TsB)msI(zJ@QL9{=OtVr0R)p}|UKDHY-K9YgQ| zWpKvgO$G!Mc4=?z+1$to$VI!v8?@P~UJ|=21WM_ph)--|!7+X^%zOp1`o%EwGX{eT z=6a3cO6y=n@-SjhP#B`lp5ujtvku7AgS3PM>@YU=#gl7<)L5LN&btLmloj={U20|$ ztPfRT+Q^^is((|XyYAnDRI{PesZ38`YWMlrg8M|$;w5gl_90z#r0S`7euU%6;==mQ zV*dKzlfkgDJuDx~)Od13V%6V$R*neGtqI~e?OrA4`Xl%JTMF)d$P5_Q^!(Wz3 z63xj*&%gKtwP#t2Fziq?V?+tmHLtOM^*?y`FZTFM~FYyP-0&4Mq$$HAYB2^i6#8U1Wb77mBwQ8Jz(( z1yJ7Ix7jBcGxb5ehVf8jv&NJr4FwGTM%BzhmoO_u^2`mx&-K7moDZORia^3@9LeTG z1~d35LsSLsnd9K$v+GPYuwgX`Q{1&$s>R5Bx7gGFiSbv^RRk0qXTth^y$jhU^Sck{ z?Mn_l9r!NEg`73tmdj%tYoY94{w44z&XQ*?ETMdOhEsX@rBKFSe+kQ_OJ27vM}pZ_ z+x@^!`Z$4L*X4;fCLc~IZMZlL3j^_YLD{(!}kDXRf|;q7ei=X0HZOz7^8 z7gFjFvRjwU_fb%|oYfKlBf#0Jdz-RaUqVqiNoYJgU__ngQjii~jQEyyncd9ov7u@O z2BSk2mB5R4RNFs9U!D2Y7;|5JXyY32BG9(JQ>dyV+dc{0$&MkC5gI|TY)i}!q{2w7 z;Ved;KYw}K_jATiorGNR(Hyi$vORF6x#L7Te{AQrI=e zBy8)I#C*Z~AKNv%ffgpDDCTZwNZTtG{K6g^yUofF^cY z3A>saB2r1#qO+jWRB6NKRsx!pgE~6FL^}y%dHr%Bw*z{HQ%AQ3bMzGkOLX(Pr00LN zpZu)oINXZ4ZBuXK9)LMM*_%#2T)sP4q^oi-#j53QMATf2b>-mWozI2SnGS8A;|70I zd9iY{sJ3XF7{@ig{Y`j@!(Z~dR7Soj$M{aU&G>@sLTeDQ47JyiI6V*pQVe|=)V=&j zgUY97B^QGT>&|0-Gl4g7{9~B+YV{9s^am2+opQiuRzNN!igG;1xOM3Mp%G3Gm6Au# zI{T|uqrow5J2+=jFHd;3vS;J`T+~=%-!lc_9cfbpjhMqoP~O49U((C|mQS>qe9Rs7n!ohvMyoH7T@t)va6wn=2%DrW6$pn!Tse@>dp#dBrN zW4qmgaVdO#MW}1j5*_=4f2okq!>A1B)gGP4b~f_c2?sR9KWRUh!_O8S+@(!s*_-d* z5h7wxnkBki{a|^=py*zM=wZQ%MM>rCQhlROfV5%P?8#?-fcvPS>5VU#JBl4r@CRBU5r3n%ap)7)%)FqmIVzD6fN)9({Mk%o|HEGln-pM+qMK5;8J~=3u#|X?o?j)1%?hWzLZ$d+plEv%v?$J2Hho zJFwJi8dCfIQQk+k)kd{%InD%2y5dG`$w_Z(_vkum{XHBn^UbP^@yi+W*JqkVC;C{Y zi5BVDS(t}TNiht5spZgWn0B^WdG|wLEn&;^-9g!RT$7psGOKXz$}u&441x1baW84Q(7wr|Mb=Du-LJtTQhm26rZ3^=CP!WdaBkdqhHKTeIhDr5 zD~a%;K&R&g5}&+)e=mn|g5r&~R4pEK(ll0{(Od)G!X!|Jid9C*RS*WVQmfb*U3qlZ zP~$~{9olV@^3p2XsDxi}PvXjfrR%xRQ=(Ee1lQaZ6~071G1XyW{wf!;-h8MXzTz3D z;{4S~` z!%e2Q{QL)cY`L{ZEAKx2z5T}(#f}zJXT3hFxiJC*$+lvD)8}awH>NqMjRFE|7Au-9 zdy?`J)32B09k#4DRh_*yT>5E9zMF1H7~v;rZ1_MVYdu)S=jtWP_xQB>zQ%eF#&HHrhHBWCp|totsj);%e85 zP^8`_dq&rF1RjArD7fJFK9k+JvcS1`1*M|y%^Uhl<5%qJj|nTbp4U}HIpnyA)+CU! zKsjTGENw9S8xWpwDR*q_Z>AtVnT?HAM=t_LzvrQ}B^5z?$I(S5-#^yIpG_39=~WZ7 zjNT%hlrWj|YxtE;xA-2fQ~M(KcFTd=q~EIT)bhvA#_ufobcX%~>s8%`uJ}tXB@t%k z;aT{dHb;|U1NIu*`P&&^=NV5jf7MSdo~-QVoobA))}!|tMdd=Cmra_6?@W~byc2MC zQrj!#A2{WGfLON@@byfmF8(lwWA-l2(DqCqijg+8t+vrAaO}IYVY(yLawpR4q~=uP z#LZG^Fk(%`G53{&bUsd6=q$A;+r6Qus0-Wno^NNL?yT4PPUwDZ4ziP18)?3lS{ENo zh|WY)^TG+pPLs8pyH#uWp5G4L+>o;^8S_PHN7}{2*U`&`7|lU=?L!tU@KT_NJ1Rvm zk)pot({KRSr76aJV-b3hOy!dx8 zPsRri&z9=*NS!UQK&H9erJ{bP9jQ|ntooGxkGx*o8r`TB26uycr6ApE^?m02^Yx5n zuctM1{MU=~)@o&5=`FSCJ+=7orW{nG>V>h`J2qpgbKC zO*Qh16|#+=v^PUcL#NPps_{TyrWO(OiYmUq?mG(~_j6#ztG1)>*voY5jNgK{KC&5F zOdi#9ZB+CA@~NF{4@ESswhJTJx|463M$w84O$w}0TMmD5q-!(_=>I#erA3B~9;ZLE zqCdfk5j7ysKJR}UFuQ%EZFEuR&b7mX4x480Y}(G`^_TEd604r1A~Dlb`OBQgf3!*W zy>P<%s%?%3;GISy@S=;aNB?xEm(d#-|329-=GohU(T!I%wjQ10#>v0?hkKObp6*=| z*lVYqB%3nQ-IX*tar8F!Fx^W|E*;bS*ixZZs$QFzeyEz5c3QO;#Aja)@|PFnEWYA6 z2LnEnBHgld#ARL*q}J72&z17LU9q%^ZndQQ3`0PUA*j-i%YHm#kIuMb5^>s1(Qr6WL&m@cC+u z@I{_sQ}-HwK3Ehvs`HsVnHU$?0DqiF&qP`RJm_0(Uv19jNcMo`y@0 zaJQ%#9O~q}a|^y4ui5HfS1DX@aufA5gz`Q?-Z>-_5!uS^pU?EGh*~)RkH|RZde0h* ziNnEMW9Q~K0n?u6K268fb4~V%d(A^7x*9u6gM4QO!+tw0W!qabqVXBy2g}|^L)z7S z%R^fxEk7H1M4p(rtHTU0WDVCfJ-JGO`5^o;$a>u9UUErJ+W2S`>VSl0#b<=7ekEEVk#-7XBt=^`O-&i& z%Mc1sD;!*H=EVBJV)R|dm`l`FBcq#)H;=LFmR$Yr#C(&iQp#@8;^fbRei^>I zl%|Xf{|9399)I{RhLI|t`r3C;UTaJ9VU&?|Bt4io`X1i0tv%`O24@-AH+?Q43hKIO zFUvO|_stCqCFcz>Tdp*$yoI8Y^}Heg#efSiLqNKjoG353l5AX=WE}Y1Srz<0QyGX2 zn_XYqD7ipGwkXu*Rk1-=Y=Q}T)%cpZNIQJd@3No^^tmvYPF8D>E6CVp_G;cgNvi3O z_p8Fi{J`&9Jt|~46|KoA^<4bavbu~;E8@pze9!7Uy4x;v_-sQ+=Sk6$j`oGN>02-4 z2#UA_PbV@A3bW&_7ryfYEf=5MZg~w zGrbkPmtEV^B+nyek~PNEuCnfz7gs<1%l_y~|Bq_tJU4DKk)7eU5@spOJPG<6!El9i zhHP~WpRVy<7>w!)yv9tEg;9<`a(CwYylsh20*L`GXplB6szI^Gpt#s#_k;1T&!;&F z%c4IJ+1^F>C%fJ<7x|i9MHWj5{?TC3oyqd_!FE%l6)7 z*wg@kMw(Oa+F)~{<=u>I{#Ah;lUm7BMt|_q4ggO` zxc5mmBG!0Bjd~jdr_3v>o9v~}7KrJGckZtr=t!28`S+#INHHEB>+RfLBw0U*b39X7 z_Q0OG=DZE)9pqn*7s$q*tkEqU62}a=d>bC_o7VWj-joiN38vr0WFiFroUFV4Je`{* zEDGDm;Mv^?x`LqM#X(hm{=XaY*(5QByjaA|$tc)(AxG9j@x-4iovkLumL; z_%!b^r@3|*hkNx+`_CcL{8m_}woYOu9zu8%0MVEIWM;R%F4av>nq-<)mKYYDspomi zm>SjSskqi@iwdOAx5f>aZ@Fj~p3WAY-B~8QT|6nd@Wj5{Z#?z$BRrbFoB+?ghy0I( zQBEb^1(8m&v3-aVMU1wb-j(Io-@O3fUl@4eV-Y~Q%KWwcSoZ@`UjJWv=mNSBko-29 zL1K)aW9xa2`q(9+`#`-i@MP1>UpNA1`cOcIwWfb780vo6J9_rTtdzKoi4EWg-;4^N3;I}ZF7BjBS z3JcI?a9SweGmeK5?~ZLxpBWTt_#@4yCPF2igr^7}5c8Zq@aLubI6}_hHEkFlt}it2 zMSZ+lt#ImSC}8&>g(+y7rupF;-v77&bo1b{VxBagnJ-f|Rcc;SU*u`?7!PGTT&Vt- z;YBZ=rD%WUxHg47k!HC4fOExa_(9h=QA&;0ulGD&ZDe%e)lFm*cXg+qD|U#QcUE?z zqWfP#av=I&Ib0D}#PEHbDf4=ALrEqOyDdOkew_t0v(fO2BQD<)l1c>@-Kb@`1dfy; zy24|qFKBsL+2a+r+0&twUB)B)Y~2Ta-Cy7FV`~wtdU_q)tlL#7OYf$v4*{(bq;Dv^ zhM;HW!8=88`Vkjd?P~o&0@7Ow!~u%VpR6ykJke2rUzXjN=eL#%nG|cBXy|v;CM6;* z^74vK%%J(D&57Mz|F--6dz1TSy2jyQz|y2eAjkdu)a>$xlfGsf8T*m%n-c0{w>zcY z)ta2X`$_DNH5-boGr5Xad&h;Wo?ux@o;3iz>He^fs_0I)Id-KOU@#~MG%BCzY`hWQSY65g|a$$3^S z%+cx&QhKXm<=Q$4RiTJ3SB>f6ZBX>`xseqWJAM4vsYM_hL^tw~rpf1y0fh^^+5NC% z(sgA~a+&c=Z}YF*$ln3+nFu>@kQ0ia)`t2UOh_K|2>`V;hHVj>hj<~=79x}qpm*;~ zIr3dwZh^%b)Qd)Rd3gQ0Q1Af6SJ*lUK_T@AP2Aubd5jNQo=Ktgl&f0FeRcW)qIl$} zgR<~{+B&xgmk8yF-U5tGS3Pm%wjoXoQ!LtWHW*;y&*OgAFgXbBFA`z|gP(45@I}O~ z*RBqjS}iR*KN`rCld85GylY!(&+|-s^8JZp0)S`%#ww--5OwXbXMY%1<=TF#RUeGvNK_1t9aL=~ZM(0;~p+q>NqWHoG z`$l%EKfU^yeadHf2{J5Um1^GxC3r6M=SEd*142?0Dw-O+XJm zJFgcRcpuQtt`pLt4gp_x#HvTZ2%hCadpnvnyAR|xW_zDW!4Op z^+5SecCh=5c3J9so)bJFHhF`H*AH%a;*z3SY&Crmg6{5pZgYbmxXz`lJ{rG)@#!?Ml`YNtNCLT=7r@<=_`HIYEB;N1hGb5Qa#;46VMuEYCOYnk7}LQjidxoFj8ZD+GLVcGUc=j2Uce}w{{`q)r#WOv&QPNc zxK#8W9!wok%wheKO5iL_|J?*a+3^LXKWks!wm4$s1*+_@FG8cKUI!vxuz}XDH zIbx|=v02nM(?%Y;N~t0SEoh2Zid^CzK*8%RW~2*sTkiiyywIz1D!(t|6tOf^Qgqrk z`F6+OdMcJlKG`axoE2$}c7x%C?3U(HiPvnjSzx4Jx~?x?2Zgd+t8@VNLJ**8?wSHw zOOpg!kV09=HSk;JRU@L8V05a71IBO6^uvU^J!do6+_ zOMS@P|Cx}S*VXKdypNP&Bs^wa-P;gnxOTQv5cpz@36kyWSG|o7;{sCORhCZPUv_&& z^&hXHnMUlMnCV8dL22dt()!h(#e3>oVPFvC&QT;iTJ@qG1)NFPS?U;fUdf1s1KREd zwERV37j;2Bcw0IE%4!IL`8s5w0RO*~@kC~)_#A28Z>>H#7homqX-f<|^}hn>;cO$c zzyHa7g^>5ohqN5~LK22!{zZJ$`iB`j59z0lhW&hCwZHt_TbY_}Jxh{UA9yI$yLRQj z#WFNnM=Hl875|?E7gZ#4Pi5Z-MZ_CfT~tl?k~!K$e`BPbcvtUA!4!@LFv$u7wPnQB zamo&X9v_gfc6jI~6hz!oH-io(FJ40X=f zUSaq?uKf;2wU<0DK>QI}LX&`5$xFUYskOWc(+y>sm4EKpPJ~}#12`=LIFS%ojmhA5 z{=k0r{uoZ<{h2|^-Ka;vZ`Bj*x_%QJ(X|z*yU-rtdQ%YMxn@EZ)+mJ3W9e7N@DYva z;tpDwG|ljMXr7@%7|>fI6-@q2S*>~V8;~;b61cyq3iomCT_}$iufn3G?3$De-O*GK zxpS7&`qReVj+2ZbP7m;g=yz>$hnB@2G#&2J+yR?)p@8(7!|x<~ivF-km!xeBaI|vM z+-o08R@ni^_9Xk0D6lXq^dSB>0lS~k_VNA2o9L5@ZRhm9&RIxPmIziXg}^KeH?!W+ z1R;_EeLfsw_5v!WF2Q*zuHCSi<{ug_ON^G&}pm21rn04 z+=m_6oSE*956I@?%lO7}Bz}sFXOee#WNjV-N>E%;Bv3!}@kKrMkZcW!RHVGBP7s{A zZR9#0@Ellcz?@8$?5F1UHDe|~+<-_T=*RH?6u1wBg%q|E?MN~wGAKW~4}#@SVZ-g} zSQxP?E!Uzb>Wu&Q3vp3EBOW0XGc27or70U9`{U0wE4d9liQr-}GT++_FT`x}z>&R) z-}-~k2;+WL62w^cgW10g`#)%e%3RDIYP*W9hGFJA2}K1Kc~OJ8%lCOZV?X)?G0BrX zTLH`xb3%8wZvrE?NKfZS96wxyQ_1RG-wm{b;`o4kYawGQ``3i%s$DFCSx@lgdC=Aj zg$|Pn{su8(Jg>t&UC69Q!%hp!r5sUvsEhJJRm}SvvMq+na-m;2=O#1vg4FN;iS4L8 zVW;Nx#ahl}TEJ%Uk78dZ2m*y5Nzn`g0CWLFRvs&`nt@+Ym~jI9&OYViT|jwn)cNn6 zn9MQ^^#}=$Y*y{5Wv>NM0#6&*S^?2Z$mC?bHF1A2Yw@nPnBa=4nf01M@z=K%NItR* zm_p#F{^OlCwptB7)C^{sB$?00*uLBch6tKh@kXE*e_>V~ zWz2SxFp+huA_<{It%jBP#P>!}2iJ-5k3JAzlmndEVP3!<}cs(&tUTzxrO=XbYsC0%LkQvy(1{zu8jAV z)Wt=;prr{$#RY~^*1WFDW%F?e=q65?bhB$CIC|=pvIzIa)x2woM)Mb*XHwwWs-$@8<1AmHyLpb7j&JdV@2!W8VX~QEFZfor?8JMIB zlqDe$B_TjAHEdm$2Z^(@qk{cJIv-)4UGLH?CIz`JJVo7b!i$+J6JVpa5nK_mky_8_G-wlFl z)v=OE?!V6Sgwi)8m( z6GF;s#5d&L`6Ye8K56>(sT`u}r4+o&hRpXCWWGdT3X2q4T@p>c?*^sKx!?%Lz^U=n z`y*w!W<#8yNzG{vbZScoDUM_1OHE6DH}Hx4tvs?ZCjmLyZ{1e4d?p?98H>E$gN(&`1ww+Nkx8inIfkM|efg6!(HZWM(=- z=|WcE%ym$QAUB}hn5YJ|zQ*v!KCzV`9Op$@++r~0G;fW>LsMQ9PJA*OqRl=ku*O5k zVBWGfZ1Qd1X=Tk0!9nnmjW_P#8<6>Mb$rF!LdCzp#k*w z9B4xB@J}j5o=IkXE;Xs8CR3mw=gz|mz#Iz*d3oJ14Q=h*tp==tnoIh#$H2QlV{`?n zyNaY^U5YYHUlX2SpHkV?5rT{XiZ;bM_hj4nv16gyG9O~~UzmhV%NOJV}}QVSazw>^^+pVQT{C?FTp#hXP8;iMUc&klzPY zh0GbCp%?FWlRz5!-#oTlWlZXnG@DoTQPZpeP4St3^HPT!aSO)G}Y^G#S6`@?#@Re7>)PiO=(WgfSjSi_d>8 zlq8PW2QUM(o(O?$O#Map`X?{5Z7F}SoS$M=+t+|`=0igv>j;9!JlIzV#+4h^m5~p8 zxvi6Y^hVMlJyS$wM&(!-!Q0O4f-e99VFVxnUw}a)Sf1*ilx9@HynQm4H7jcC7tZv zjZFy9s#QVq6ry!kDEMxeY?JIpYl6uLlZ755K|Ap}r^S~@(B7xyz{Uf182tz4hzs)V z9B*fV(n5|B7+9P*_fhs2|5>ca(v<4Y3pAI(vBs5U5_nEG|$b z)pik=p=jx9+rFEE#YP2cvs7P%6E{$3xD8@T+OxHtvLJq0v7-U9qUxL&)1#t3^^ys8AgVw(5L(t3CMW^1*LylWch&4nBW!td>jw{2U}^e ze+wy_gfXF22J-~)Pmsqs{=;i8<-q-QZKA5-w%F@!x3LS?pa3Xu7JyPgNeBwkHZ1=p zp;p^xr^tSS%%#FUy$1S~Z%bL-Jz96x{B)L@!N@x|gxOm{^(}@OBo;JVU$`C%^m8Mn zu@~W&@L7}RxEb_-Hp zlFU+|I&+^g0L1sdFfxo}bV39~N!YgXFx=MbWOT9!Cc9?>o-=7UWV^2iC23}mA zK=cv;@>q?(uO8I+S98xL^+^vLPb8-RyK(I;T#EYGx7-hpZrOza6(*s^Ht6t1n#Js* z%cTVt`B9D>zQ?2j0Wirrs9VUK0r= z%rS}!aXuOb?n=*mH(`1R*Okv=1MIcn=zlY@(6x~pAEqZKb~ROtex0T z=hBNoCvEo?m~dc0EK*XgAbCfAF#tOp(z1(?<%dPFt#Vlt@74a)p&>r#0@;qI*&unB z%WaY)$eh`c;uC&ULxX8PkIfDUeeOdT0#Y>=SR}zcOIWzetDFFJ34L4>$#q5rD2@cd z&RnPyYTaax@8U!->0u-wQAuBGsz^U-{p$@TEi`hF5y9Zw;@y;f(D1Vj$mBFDDGZsA z56!+sZlGIIGXj!gjH@*f3?uc6P)^Lm4=W-_Fdb=k^VR}^k%u2Y3vd>}(q^Jz_aP*8Qoz_Nz5-`hVXF}NCIlzrdxKUxLes$-yaLFIzQ5gCe zh|3Xw8vA)5NfZOYVISLNJVf8;_r(_FOtSMO&VtKK8qgbfE%SNXQ9zMZBU(>Et2U4> zW>Rk}A4H?i>`x*|@xckmGKGNeBlYYIgz*gV*CkO2S4$Jx(>HBwMO zxhHmuVEA8|=q^4;Cp=vQ19JxkcV=*5bn;qfvscw zo@QD2K2_b<;>&|oS&2fFH|eqN?9 zd)g2)xg)9da5&!`;}3ZNB`WMIT$6F`HOZYx%zZu=Msef61Z2;dLk8KSoS8!?IwYm6 zL9u#z7jjU5ZyRo&CgKj&!p)Co=`3J z8-76T*%@Cxu8Pw=kh!A*nd@T#Crpmd1lP*P5_$_679um4&dCfFO6OAB*lmnM5D_{D zx=~~-@)iT|5=B7VRlUC4os+#MJ#zxJ{k#fo-1H;HtpHbeTA=l~A1w9>`VxTBWKr~! zvMq_*#W_}Nes6xa35f!AxqzMk;rF*gXL=}wo)QqzK{eYEVK}sUs8PagXudi@gbJjbXqX#56f9><)xE*Y1aF9-B>I=7>ba}&f+DZVU9xy;^a=YVjB8~B zJT5WV1qppG3zniv%WZuR74SjpP=kct78CAP0Xk6O=^71ZXZN59W5ar|ksO~>8Yyt_ zDnRXA{@j{l)fZs(G;5pbwv?MCXVWLtFmy3j>*~1%0Um~Cm0*b5PfV-j5TKXJoId*y z<^p~!11QL5!PoLlW;6DJGd^o-R04P2dC=lD%#6|M{Q#h<$lTFP0sV`5B6Zz1#Baax-u1}BM=9zz; z&JZ1j&ZXWZi40mIE>C+X0VNLqTN43PkmoCIo{#MwA<_B5Z1E6|tG{b%%G_(o_{Ly0 z(LdWT&Cj)wL@Esp+hpitAb>A1tSS(Guf_*TjRB$7gG%JH42nD0es{~tzO0}ud_(Rx zkZ^O9%*Mg)>BXLbGUe$m4JRaT0w7Bz@MaW!MDb6xSB4kbg5duQ!wKerk{DdANX|J6 z9ZxHH#WfR-xdb{OK=2*HP_dwj2I_d_u1jOsGmkAx9m!{yn<1t}`R;-; zCLeQZp%S`;Q0EeF6V`U-nU+kx2p{n|1sa;?YxQAKgZZe_{sfg)E)6s=0-_J$v0%Fq zO2_*Rl@y*#h~W27r1T3D>O~4zC9?-Pm zH+)Mx->G)1JWXOA$A4S^Vt3Y0a+qXE0TVtZwRys5J}{j-EO;w z$!5zgnAz?%I)4`3J#uq;LFjWThJhZ{ML0wT_HNt?qq0S#lOM9F2dqXEC|aN;XIO83K=)n{?XK-I`CcG^Y*83P@p_QfeU0N zd(_WUl`qDbO*wxZ_5#%F`Sujbtr!T7k?mT0a;*(s;vC`77&QTfOG?fE|8#a69idzi zG$D0QzR@JORYIG9jMrB)@8wA|MwkQSx%2!`Fq| z&R1dX>WL?9qw*Nz{C~B;!2KChjEPe?n6{6rmb~sW4{7bGlLuSl+h5Kjm(YqpZQ(Le zRPvr5GOm)xxsQ6OxGmQwiau*;UK%cAJT+TEgZsW)0e@J;zkS$yZZOMx+1{{6O=4{z z8_pWy834YHj$gZ%qIFsPi;+RwoA>G!DO%FsJb6{cn6sfPI}&cp=t^VDb|wV*st!%o`lK%Qu0qt~IXd7F zD7mFlMmx&7*{>G2w;2o~ef5{$gs|NSaK9nob8wKS6aDmg1 z3*_ti>vC1Rd$Pb*{nm)U!+$b3Cb{!O>;RR_W-UW~+%^6E_Cq(AKis?&!Q4m+a28e2 zp>IN@(+jGWk<$LrmKR^2sQkFv4cHm6)w0uc4x8MpuFK8gWYX__hK_Q*6hMKM?pGcE zgc;ulR1BAjC=QyExrtvramPzKf1zIr}V3tcM%u4mFl8A46bn-N!yTFIG5h>LX?oSoYGzL0=w5U8@# zWw-Bdb!|3d9=R$^+g5%3tfLqS{6#2o|M-= kv}&?Qa8ZP1>Y`sbAwk{og&pYB;} z<5#?le~ZRH@(;C9c8up2O4LchT)aU9suuVam=%3uRP;VgIml{6W=23ZS_#Zwhr$$=!V$~ju^1oNpA4Dz!*n`#cZ+eUDxwzL}U z@a^UuU96C^8Np+K8!=Dh?A%!pcGM3`H1WK@vph`x-}pF!nt!1Ai19ZxkXPb(L(auB>hTUJ=dQ8JnMu(n27@{DqgHAc(14S*;T2E@ET+lYJpp-wj=o zqc70aIzakg03;!=bJtFb0Q~}Epe`Yc%VbH7c?6xh^%c)g+6tbJaHZA+TU(_;y8ygtK!99zs20yB^>rDJ8tHwjr74L>>e$Pu|v{ zF%LFdFw3P?evA(8?r$qArBim-PAW^HAD;HrxR4`Y*93TZ6UH3*UciZB9R>LH!A# zXiknK|F5RLF1ofyLOowVof-V*wzO-A73gcvWwS zrj)P5w{-WzQ99Dj4!kOIc@+O&Q{pjzJ5qMX&ORPtHySd7Bk><;M>FtX+68c zL2mpTc^(q=;VU%Y5*7!uBM;bYut23a**{Xqf3^5KEmtQY6r*}Fd%**48UhJD3Xj-~#rVD}Mo;=zJPoD-P{>`wPvA-DN74{sIexi^mna!_%w z)Lg;v>tYV6MC1&`VVCHZB87Unx=}AZ2gqfPHM5nY9#&fRxPGEgx$?JLB;zHMe2;P) zA5>v>x!Cgb(38> zBbU*?5jpG;g0`P|OC~($3^bl;1;fGH1BU=3C5Pm4aqIn^62n>t5C<}7MY=Sm_AC3O zf4sTzXyBa^!oIn#MEs5M(PCc4K)HpA6la0g)*lgXE3YpY2ltbm;pO_}(yc$Oe^}H2Z|K7)$QFCmF@z+= zEWaxlL{2QM2NAat?{9o0(%Ato;Q|=}k{%sE)J|+jSpRg8>=;~ZCHF6KFvaQtPxvBj zOz+YPbYtJ|ypnLG^SI1aEQ_yuwXJmK4%gz@MW1>_#>p-&vbZYgs~nV4eWd&`Fl!Q) zyQkI_hpFFHntE4rd&k{-^#xzb5g_od`);%)OH>~d88*l^|F|z{{VPgrsL^ZdNmCep$1Y5${=nh6*6 z^@)$+@9zwnncIP}h0>22DUV+gDIRG_!hXKs#Deb z)1{6CN>4tTkKBh`=?Sh8IaZUy?>qu8vw-Pjz~Hho;s3u!#2+A7jt`g1#-3$IaZkB- zUVk20+I(nJQib6&Zq7LY!xl@Um3g(U!PO$eMvp8yqqR>CcWieO*dwqj7q3*e^Wl26 zKo;praX${jB>#$6o@lgm=ka3kSN%IL1$|NhgbJ1x1Jn8Y*`aic(SM^m!J(>)*ncOO z(CZFo_<0zRWugMM`1t0%7wf;>FuidDBNy^bpv91ee!qh`|G(2;IAc&Ed$CM}eWA>jD$%Q=n}5ks|~BNj2*F$_}N z{Dp>uaJnCR(}r%7n@>55X+QbxP3Bz^o521|T5sA*cW60r_uFVAo}#?^XE0Ztdh~SM zuJT^ISpM(iy!=sYqGeTr1@Nl;5ogJWh9xz0+T(qjmZ1= zYn5%Bx`6h!MTXz*_os=*I#0#h<*BGmJk6C_PU@s*0`VI&bg$-PV-;HbPlU}{FZk`H zX=4N&Q|xQr^4I;QbR3N~1GYT3-MJY~LBhf-P+(0KWA{UD3x*e*kw_uR^=gg{XMJwe zg0M5!S$uFm&_@kpeln;*zM-BJTmf+eK$3xi`p&;cqxwcnTyl$Q5`?*>C*S8m!d(Hx z(+VEiB6SH9+{bdp2hK!hjG}rae|sh|@-M@Tsukq=FY**?F6y~InBs3Xti14P zrY+d*`0s8@VV6CypE!G4zLA#h)fJXzLb77QV7QP~??5^bYB%B-sCWy5?}nV?bdT6{ zBTESKPAAsu0 z)KhIIzx#w?#6+lUP`B0;uWKBbZE<#tbp4qB_lpP+vr+5(r$wjfN>_ue%TumNK;`qX zpWv#RDq+&zoc=3;5@h3O_7&nhWl-FKX^s&#ZSAYf;$Gh_vMJt(Fw@3A++O_Ub^>f@ zdu^oey|K?~b^UA@UE@8K=3(L25fFKa$j3#WHa+S@ukIkNq6Pi%0^wro|~4XZ|0b3L>=&0}h`;lgyq)BlIG z?+&N>{r^9XgJU0?h(oqQ$jUfoh)`MCBO{bi_Bi%2;$%x^RI-W4mYt$x@0AJ}+4FZF zy+5D#`~AIsfBe4J@48Mdb>!Ty`+kncb3E?n^~Y@TQ@;20IE%>7iM`6bi4sOxiql(_ zWuv!eC*1pr;X0w@9DDOs7NL69sh6$8GyOKzOMODFkSZDSwvN4V7^o*94A>5jVD;%D zH9gwsWHZ}qKCt}wZ9+fIr*f zRX+*z+mTL)7WqEDm(OxvW(AC04L2ulF{@mJ->+L0e;`Uqxt9Yw3^+O3pCWh|IEuDF z%(l^JZzU@M8zyUWOeRh$G}0GxT)kB7*OF{}C-%-cIlmg|M?V*2&1+n8-X4WybiTe& zfAF*LH2_`h?59V+wM*IPUzIh75I~F01&6`pj<;?J`C0f=3VK?0R-nSFOoHLnPGOHf zKk$$#9sR~7z^r6RsQ0p_g(d!9c%+KzJ1%0e$e4CAy)5t593NAqYYU2% zK!N_cRrcEb{y_CRGrtlsCQIMJed)N{SL+n5K8TK3je6v_@^kjTW+*fEDdtcNR39DW zyheiu<0sC$jIn@R(NlqQ5a1K*l5p%XtQME*U3#fAX)IgQ+u9t2U-dz-Zb24fF-Wp< z|3ryW_O|{avXbCExR9+CIHUS}wjonb)6zMGGJ-fwz1$>Aw8YdWgYf%Tqls$<3|{DL)S#US_=D$@JoQh-l$gGK4qt%fQmZ)Gi_| zR?9nCZ@s-g`3BfG0}?eGkJjX>vU}uHzWc4Dk3LB)d0B6zEbYyNP`)_pf&#aUMy)QHtu)%QUZ{yX z@9sKyc>AFCyr}in<>@c8=*@MMUQQ4J>5BHR;Obhzk$ba4Gw)whe!ZR6Y?h_hKjrFz z+j6jTq$5Jk)hofW)x`BSEj1SFGc+@2ZvYVH33R4*yxcATR&+Ecqb{#TlKifx@3RyM z&Jv=;p&q33X(b|xyG|ZzKg84P?!oWIS0`^7X`E*ihgMNz0DSm`JlXHey0YKdeSrHj zUE1Vuazt=vM)q6XQkpq~m*>p5_=&-|2Oz6Im46Vxd-ZSnd4CE~#MoN|^Y~2yiD|z| zPkUplX2B`D0VfBD@%`qffkXjv0vx^1qy`GPEj}h2lwCa$O(&qhgiZ=)~y_UDG_?G#Vo;B`eLhvIzH%8lMDK2wQ>9(sPI7#i$M&E>;k7u{JJye{)UOqiDvdbOE0>{y zw|9Y$+emIIyDO1AC`77bh;1E+Bp1t;5Z{>(>%-&6H~mBpUnI>3*n@V^3>^t}feD zs(Zvw+ITG0EiKrH9`-aeNq7hOPfw6ysBh7*)#u{7v@7_zOx<(A&5${pyx|oE0fU7( zKDe54t+SvDA+b>2g?l{Jf958-2n6-So8GC=z{hqn_0>LlIL+ilh<*{TcMe54O&t9M zi#nnfPDjl4^}kkd*Nfcf$167X*T_h*u_{M~YNyfM3Q~+ZQ~1LlUsQgw zQB&u((W!>PIl{3c&(gU05kVm~Pp~(st#t3YVRv8a6~6FGE~1R4zogh1d;YTLFB4h; zeX`{tslMCw4g=g@3mHmWO{(rC@jJ|t=QHiBPEnjf0%3bZp@EcpYrg0v`6GNNhe<*a z4b>QI@BQ?&cRIvddwY0Hd85*wb^P!k%Q--Wnp~Nk1?$EZnZW6>KlQL`wpDg^*TAJ% z{m7|X&)(_U+1M3C7Hr`+xt14ldf+EFL;H_A=@5kzc6WKX^YbR0eRmPX*X8k?h;)YP?|%_Jo4?}?jeyV+_EVv}KG{FbMCm~`4S%RW;+2dW2f#END+MHb zC_Fy?=zGv}Nw*pJrwl1O1{Sqy{B9pa5|{c@m{C^Wy~Cc|MxGn-FmNywr}k{B#;M2ai=h_~RS?RKWCN+{9_)*XsuqOu3Sq?{AMrWl&gd4vCR%af36 zu3Iy@M*&-&FQ?>E_G{G?LyClKROq?nx`MZuI71vKR_FGkU)|iCWF-1Rig?puOU=>M z;=eK1?)u{k*J5U(VE@g7Yjn>r#LBmk+}cYYZXH%uFGjY{-_}MYDQFI{w?3uP+}=H%4WAj)99b z+L*YAbwk;W$lCHADs{(mt+s~B`5*9p4tf6Lx-q6pu|n^=3gMj>q@4lr-3lVo73}R3yn!)TPqJ@;TEP;ABx00mPkHBdM1yy1aXK=#J-0=PqsZ^kP0h_Dc@6QYTTXJJd6 zc;@Ug_t{&+PEDIXD|Ncet-G?wyRmdqW;GY5zvYU-!nMCOZ3ycmhJf?@NjumJE99<&7#ftE+^(1$&DHewV$!1fHH8 z^IM0evS5b+MsF~yw0%r1Y45SvLzmQQE)b0X|MehAz~Bb)g@rOR5mFc-p{mPE-RG*H zF*21~8R4n`j0Vw#oGs(^Tv*~Jud{l@D|lzdC<9WijKexGehFe`F<^RxGv zfZIajD>TgA6f{ond8nprqfRzFC)4rs*ZxwFnN@H!>pOH?0jG}mSna!ku>PLI_m(i+ z+cSf(yxXyfy-zwl416DP{&F|F(aSmg#%A4t+#>2Q{F@r5g%;G6j^o+!?~B#W6S40P zDd+cQ=doGa$(-uc3j&GS{&ak1j4B$m?boHuG=)b#-&MlgKxa!iiL(c@2!6foN*A^g1Q+q1L z_bF4hunn)y16=npNJ=GldgR`ldRggw51Wh^0toW0)#OXY;om-{7Y@5+3vm&NeSQz( zA*A+!;|pS{>id&XV^xby7j#Aujr-r2_YM!0MC5@DBr>h@sM<{fZZ9t}cqELJ;dZ^1 z+pDS9gMp`ypNNoh3?UNM^a?dZ&hrX#IQ6G~7Zp|`jOb3_DfCbZCsSA++FT;^+FQ@r zyxt>$X;;M{=b}`i=?faIHt+m45D{1VMp$khcuc4DR7|&&i@AM$_Wi0=EXXhxj;QD$ znK<%_=@SwkA@J0C68Vz)XR3x5`nZsbY@C(BF0=`Z3yq&s)r~0q3n;FBhXv?Xsn$q_ z=QEiY!%3)2=G*lD9H%Qc_Qq@b(Vl~Q)7v2$tEG?Ltq{3pV)6eV3GlTBHz51hY?rQ; zHp;E?6I7VGnW3C5To%kx1G+eYvY(2X0liKPwXx~f(6CnwC8A`_b{eg~enD7?1pb<+Ew_&N z-q%_wIZ^B0nfZ7Icrex7Uz3Br1f=wD;&|LInKzn5(&QbX+g{&w$2t*l^h-Ycj+GUc z>P-ppzH}jComT$MNPC>${$>FgwMc9X+jtE89IwQ~~77;%x`p z%d1D66dUTHgy(K;7z->ThpQh<-)FGz>QjlLRe1OCN15lE*;V{bP0C&jV&K8!-Fj{I z;N2oRSuUK*5xRgZDDL^$Mzn~>Io!T3gf5nzZBY=4j>AnE$DI@UE^f$XTE4h*c0B1e z5#$)}Xt%!hs_lPz0aD`~CiAV&^m71a!pn~LHgcsch1IHliP&DBtvS2VsIDp!P z2+tr44hAi@?K~b&+C!N+CSSg#n&@A6v8PdwP~|#p`)?uHHHx<~BH`BJunSPwM|M#X zsx}F!n&g*zL}ZK>@-7~sylE7wu`H2AMwQZwzK1)`+q8`~-2KJ?0)CyXW{HXZ5e9d& z7t-E(Fja5VU{LFJ7bB^lDZ;e%b5P)>I*bZmjT=(D70eRvILdhAnPd>q$}v$H`+(b} zX6JF06{~2WrlXtfyu-GAnZ-}lh{G~vea9xv8holuo!G5F&r{cxhRAGGC>e)+zlfCm z+I6WYa9Nfz!nVs7loJioSUKH8fG4|2s3;<1FX!kUX^65KsF1cFW|I$y0rFF@%c3=+ z&Q0<-{luTj2($LIOW`!rCKo{;5OC_hB@A++SK^AIxewO6ge)!+-c^NJ_XAJW5ncR8 zA#%Bpul2P2bQHvLDT)`_0>_=WxE2q!Ho@?>Klnjd9t`2@1+?H_IetD8aI}o8^u%z; zT|eCMShzvs$rUKZDL*JZ=H9&`H<2(gaR#YuQd%D2F`A@x&H9TR^k$~BB1Ksak8c3_&)l9{T<8q>$TxP51#AfD0ZcZ@<5?b zlRLTT5)bBp(|_%pDdW9z7nZwxF^yt@II#N;mO~{9t|esNSX4ob5mu<+zz7|LxOBQM zLd<$7yHZCh?F@*vd~(otc|i$3I`r4h)tRTOkJOl;ol*I z+LfzIg$%sZo(BQ^>kJ##{=Ow@&2%pX5haWyO5@^0Qy^PBf#MN1hLVFHbKf~Hqv+*2 zwubX+NafzviCQcVTW9QEjX{h z1sOfRw^Bt+sr(RWp3-$Tkrsy~nh$|)%=kNgb7YN%E40C#A)ySa`G1BKa?;{?Z=TjW zZ;H8f#OuVS+L-YLo1RB8KrU#`(M%%oN_l2@l_fp)B$GcHgBwNcPGub9gbXrG_?LsY zQCOb3hHg$~#kEfjpUaJ}C0@KGGF0Llf?ywNDSv(qu0~JLfM~gpu=hQimV?pQkc28` z!^#mS17(tU>ek#$l*$W2tszGgGB-vwIPMuyp1hG13I}6>&!Pi4+-%J(L)B6W#U2qg z?{G@U(aPYH{y+Yi9J-lxYy#n=v+|m9bbM+PiZxk}b7I3jE9WLu)q?)w$tI@SLU0-Lvwz{`#9C}Z0cU+a;%0{q%f?tvV=Dpn94Xm&HIiiJ8 zs2U6nM&dvY#CHl$C_P9KCS)v3Q*)hNafJc=#d_s$HEZr9Ryz>s z_R{&Byn$l9-#_l3zHSjx{I&Hy+fvIq?%2;>5FJeOI5LO$B7qKbvoS=R>y7BMth4tZ z4R~Y_VSJQ3>2ItPJf2QL8P1`a4Wyzi1LbNgCCJwu#;Kmbi>DYu$viW7bdw|bTMXEz zdZ8FhYKNi-!juPx1awGMo~x7apz)%_xa3Rbo1 z_Em_YKNe33j8gq$2>|Z}{7PBsr|4G}DxvEyw8+!@Ph0}dWJEfM9}%Ye8d2Sj^b?5U?mCM^33nhNsH%MQNv6sDl8@Uy{dtmK9AM11 z9;6C#(Q`Q4OeEGu5gP;N0KFBs(eKTSczm^7P}siU5M{FsgrUk3h&tg`p>|VCdy}B$ zcWe;v?J{)vC*01>hy-1@7MEijXXbU1@5Ctl(=p}HsFpnpu5aewaRa=M56e@1ilGpj z4e}FTwhzP`xi3&z7M3-6(EM5y_8-Z!18Z#)b-BS1iN;Wp;Pb*~8251p4MxESH=97# z@a>@%)qFBh@;m_qI+ObG3L2ug5!@y`*?qxf_^7hg$O8BUU{|Ge9HaujSrr-9pQD@R zk7J!Qywt%c3rxr`F&UFI3$yrr)omZW{xS#I;j4FQxWMpmGrw@$DxG0a7G)w1xvmvX zc(;C?7LT0dnsd+>^S>4d=E37eADb*&syv9r$NE@%<2`%B0qHms3~o+gR@7BU7NqRv z{W%T@1=I_Mlt+tln=rpqhT3w@q!L)Z(h6PgPpiaP-qz*Q2~^xp%WD~>W&KP2*=DBJ z_+?-?Jt_u9FK8Cred|R*1({OE?zQlxFP$S1EK2&jdMK$UQ?+XYy&L4#c|DL_EvwPN7mJbF)KMR7}g+|J|i#gwI@u#HV6c~G+syAsco<5Tsz>+l{c zh#AkZ#mItfrw!`w*EtH{e`D!_|66Xgj}f{@|& zUz(kal%V{E4~O=%HivqpC(%4Xds4gf2Cs)`g}PjL;R5ESM`C~Ps>BF8&Y2`J1p)b= zF@wev=ctcU@+t~rk`!Sf%_CrvGKdY7W~7Kd2f-24QcOOjC2hfQO2Kz_KCx$Wxc{g{ z32u_|BYmeIOwgN-nqR^i;4{H?FJ@l-d6$s*P1X4}ZM^<53-br}16uX@)#SRK9PHqf zdkMB>gg7Z9Mg(}!(TBlH;-e;R^oSLi*AhqUoNrHx4!o6!@J(D%hrT9LN?sCuuC@{U z>UJ^o!{xTQOyKzF`$xiXYmA}cEuvxVI@RMUEP6g!b@%E>$p;@K7t!C@AWm#H*fuqt zBRP5RJCMysf=|^-Azfrh0f_f%FQ;|urSX(LC2KTp#Ry86)mWB1WSs_006C{RA1Ach zH|nH~nIOVaK^(0+-aqc*tB-Gf3-KA1$+D$kXQWQJiJ3`SzVQ<0QIXUD7<%106Q6P7 zJP*YAL_%z9_r-Wi{R)<@6bvOr-6Rc#+J8(Vm<|psb}%#24P#I21Vf%m{$dD2&YnY) zK{$(3>xa@=T(~<|d!eB-erV8EbrPsO&p|NIW0$_!! zGXVEt7J|Ttr8>%fCkMJ(K&-}XNOkUe7}4nS^}Xs#AA$ffXpDF&E&T_N{8^nYA{!C( zR@D!Ipi&^1RQ6G5m4$TUORLKNOPS&4`48FG9Aqq;40p}MPm@f4c~w7j#n zjU)-e&owZ)N$Vrz4Rrmv${JC)7F3sN4-Z8&p1y{c%D`|NnA9-VLtIRT_QnNdnN`_5 z;B#MaAWs8_?4%GN&%`TJPhlaC#;;!6JfOU#qGX5LfYfAbaB1>UPhoK^>L70yX-f`giC^u2VT?EU#zLu z;XzW6bK|PXJPi}c9Lb8XxFQB3T-+Sw#RCvc{e`7aux%SkC^?}NHx7HquyX z($jYznJJY2)?hhRa3E~PItdZh3kAluO2YeAUu1!C%i^%uKK2$t>qG9~r_S9H_+X*d zEE!Y+2_Jjceal=shLXykl7wnPD{&qd$e>z*ShHK!xa)(5Hl*o-XJ?13BLiulhe>t2 zAs6Wst!1xX{VP>~6HWqfuof*U+A~Ql z78LRtviDK7)jC%P*rrn4r=cH!Piv7FdZLR3Kv#`48@+lxlxS?ZSGVn;Up*PW`ymdZ z{z16lE9L^QYIAVxBsq=CVu5j-|GM*tvB%ll%ZReglZ)DQn2l+bq1tIC2OH>S%gQ}S zm~*~{v=0gOpBmPgX$$~MMl{t+F@vY&gz`PZkSlww`-v5mKqY@6Ww2TQFxMb|k8Gx# zkcA{Eb*LfVy}X}B&aUd`bY{s|FIco6OW~s(Atm_O+ZkfiW&A;Dkf#tySoSL(b|CKR z?^x2$1UwGN>_+EP(I@D3V$RQZuV7aQ?<9w?e0Hg|pkaCDvL|@D_8hkx|9Q31;2x7$ z>}4dl69D&_K@z06cR5+`1b+2-L!QB0NWdKXh)mH0R50~-5{p;G--=}vcH(#%au6*E z52eI$Ki$TL`z>)->^ShE5$kced94DU+XBy_&786u8Xc)@$rAo!EkG7r3&_qbp*ZcU zODj0pS^vRh=K>_qOtJaip}PZQ9B&1p({X(Wb3Yz^mH0N^db%@;uT%I_M#xp>XhoPd z&9^=2{ z4Alx0jeT|)&5A`cAiB=0B!@tF8Y5J7Zz#ZsEX@!CrvJ^B06}p9ZVvh&GL$nr5ciqX zZd{TQxXn`Lw8%fMp8LX-irT!xcg68t{N}*TFVrncg8( zg{l&BLf1tgd^tMJ3SQW@Zi~~g3T8=L$-RN^ZBGA*ym5PCaUbVdeEt0(Dh>P^#K~{b;%%8h#67;5VzX;Ug#JMcpv6_H_nU;D;My< z*EZZ-s4)+qH~AHW1YP_xq)8$&p;%4Am(J7LKY9dABG-_TY<~+d7F`SXQpWUqI*f0g z;PshNlKvK0AyT_k#sIN_V9C(ezKJpD`zQf{xdy#?xQ2&bh6sk@1s?2&oe$Xg8t6GK zO(W6i>U7<6>BQ;BqTTupcxcRcl&nD9bNvVj-HJN_B2j6O-KA6}Z>8vOkQaC(MA^!I zS2aO!A8CHNL^xDB5vp_;JJ-|Rw6Rs-x1)^Dey8R5s{QKYc=TiPJ=@S^teV!L&-xE|j1 z7k=E=qVe&5^|_)M`b^(KdD9W22+kWMR90`t(ubVh7e%`B$zjgR?QnowFEG3~IIJ^! ze)OC}2{ka6dN)B~ctaH>w=_sf&hmjbFR~+H{4<7Ju)`Sk4D+{At2Mubx2fY!;e%7a zNl?ZSg8PbXzJ6+jShpyLQcsaux(v-tF0F_K^_%Rjxp`^}Sr}p}zn?pn!kjaFP)o1P z6=GI-Z_Hubueo3E(`z1`@|T?YGQC|hAJF!>L>7?nJ~J$K)!6PQm$vu(<9WBhoesU{ zeZ&gccl$d>sva@C5Zt56pSqZY5cC>-Hj&lf)bgzMkR3froJ6j`l8b2BGv(w6{V{Jd zijI1abZz^d-jm%y88{i4m>i=o&`4ff2)%c_z*l5gOTsK3&SUC0HG3h7ifn=nL}KXn z7o0P4&>s*<5;Cz3g-?tx_Ib{g2i0Hur@KJny`{z^A#&EG!phyH zuTPbutjPZmwh(!%c2J$-EbcxRbQrfb_vPb&WZ*pQ{Ub*q2}XLtrWVU#*6p=%s{c>#X-B3a(125RYFPc6;AqMV~hc5{ke4ZkxqbODu*XXm?8B^pso`N98SkCI=;^S z=l*3qq5bdlNX1?m439u?VK?GJP*`+|#b%-dwigbfMZ2_c#>g-UaI`8qPO8ujxQv~E z>tXKd;dKfl3S|r;LwyHY920CUMbPc@FMkMN$ps5T3?g&xsuLWtPjuN#?UcmrE`M-L zK}CD|z#m}}TPy1)8ve=DrA;G`P#v8!>Vd3}=2F()d~< zZ|qRTRCtw`w$BJk$tKg*Sn2yjEKMM5zt-Jjnl1XU^2d5 zFFg-B@fBg{HiaO~H`ycnQ$SPZHqqxlNqC>pgW9?UWFLK=$jd-XVB&^DdKO@!$BM?yhJt~+oWLtHMur}S2?zNH5yfCj!qML&eB z*;$ZnR0}<<6e{7jj?#wS#3gs~`cheqW5-l9H+}@?f2;U%;`Al#m=3ia`Y9ag1U!;Q z#G6;cW{(WFmv361AAJfOm2lAOD?eY!4baC)Obkbz2Z@Eb*M~N}4kleT*e~r}LRW1> zT)4%&zJZ4J|E9>!T$!0R@(z3OD%Ozz9b+``t!=sXHOCYAtk#VKJv#{J%sebphP3_H zK%vWia-vq-mC!J~-7A-&yLN0%kxXXw2%HZ8Y?_3Oy4!X}op)2c{#y)BV5Kk9mhrFoZ4iIcyp#Wuqi^pPE!9RJGKdPruqd=0Pd*7{Xj+NUe-eZuh zv!Y@8ew|yAggmoGl0N;Pzz)D-xH6w|S^nV^@)Q-wek{P7bX~3~?+gm>B1Rtha0yQA zDY3|{?ZW8%^EL5z?bd3EeiYOauZ+DTy%Zv+8pJ^$y!O_-ceNrAm3SO07HuV?(?4{ya#Deo;bML(qu-{2|0X|DRtvD=_0ivc}ypW#{Ui= z!1s_`T8=GBhsyIPhwf0FZ%KDT&jJxMb&Eqb_B*zr=Ub|{lQw#gEYH53tdJIVG4`Tn z-eRZMRh9*Lw{7_}P1m0@nN%cFHrB|ed*KBVt-oGfzO8r&QnlrD!j*_w&{R57ZcRkN zDuV$H2-ee${%TorvY@6Gr5Hw%g9?K38kDdc9Ung?SPdHT{58=6etNKLb^Pr$P}h_Z zj46lSsDfe8+~Qd+#mKBqDmjpY&`y zmU)TQe(Kq^LhXjXoLlgsX`ZeY>b6IgW+Z`2xm`@edHxY%;M6n-eT!Z3^sX~^E2Z-n z&OXfB%_}xigbb((Vqo&5h}F)mKj!&1zjN$}`pY6c%Y65ZZr5&D5+r|X6lYSHe)l(B z3%p%&BCe`+eo=#=+VujS`FPeOrejh`kFcy`N3VS{q|;UgtWH<75HUN~9*wI6fKvW* z@>hU1!YFZ{<>pI|E59FY`BAXR#ej|^_FFGBy;iG+O~SoTj=X94ZYozzSK3j6o~T7P z^UiQeR+;7(TIpXGzvN%PpmmN<=$BNwe4yM|wR7}RrmvvtROati{|kY;lb(z+US)Ui zjYvg7_M<~T1eo^vi$iy4?eDw@POV_D5$^gX4$-=eYz=;8_Q5xfEcWBK{~;syIS~L< zJTMxkJUVI(h?nAD3}pLp)5dN7AxqR*xRcfV(bu5-WUueuT$ylFhyOk8XUwN&ZT_kN z@7Q%wsiCwj=5`Z=eb(*2`j-kep1*>2p+=)iSNe$2pSlv2@E%tAlS_R0e9M_nSy*)_ zMiG>8#hgYEpu;(iM3tgT;PxwYFahb@?E47puEc|>7%+0npdstRHHu!)uu_OmL9AJA6ELV z5yh%P6=&+?rY$?#()L~rUfXj+X)CHDp;L|WJP7d%1DPQ2hEjlZ#ys4*Z43nF$rW5b zWs*QwmVA>e2Auia_=6@;`RC(+dF*fAeRFS6Bk4ZkL-HkN3KmIpGpf7!ye~+g9FWnL z^Dqa>rctq<82b8flTgWMja-<2`8ASQSqEwUv-ojCb73z%GZIx?Xc_J0<=55T zuq6HiPlA{NsqpT#B>F$S0L03B+R9DXYW!ow&fOPZg>okfx2eBgz5$Z*DCjTlvc&}} z;JH~X*O|Y*9BMQB#X6lZ?;5znX&URS2W^nv?fBw-{qwJAkG(u!o&w(7tp(-gVAaRxEaz2g$%8iL zJLemAr$h;xbE>G=TS1wa3W3O0b-8jO*v7TSbB$HRrV9<4$E3Cg?NvkTJVD{|^=-k< z0B-$&E-?A$zi>S3V19>h%d1!hUT9^13)~f!D!PzQ2gX1 zQV_CUh?Rgd-MT9p7V=~C zP!Zhc%N;&)KU^N5`#3J~RwN1*H)*`P1S4Xo#J?FeONuK*#?*Qry#Yv)(N2_@Y5@tl z^(OcsQqPjfh=6s86VC>1k-4DD3AM3Wy>R1&*H-zP3>76PP(BBy;9J|xV zVq0qbmI6=%S`Ezo!>l_`uCM53qwo?h8PkJ~&gxxo214ALspw0_)vvx#3rVoAsq;mGzRaAvACh;J9(6<5FK1l!yzwZdf+n8a z>Pwz_SBg-Ue=i^&SZ#g(JW}ial3$m-9yD9Qewe>|=P?`s=S_}gsdp!u8T%Dt~T zMx3CZ_hpBqx5&2`ciV!rtV!63dt`{5X79r9VP}cn+WEI1VlGtIw<~ zU*)ko4pk*asBf(wgN~v8B_33v2h)(=6uKFq!H8#=v>pN&z4;j z8WuFG$JlgpWEOVk)Uz~3M?$QAe95=}oU4+z&u>sd<{U)`AADC~;i#~m zs+SrnHD|lqq%D6;>wmCS-UM2bt~ZIBeeU7_=ABGU3co;pzQ@yGtjo}qibm#f7zXvh zX1&y4!sKE6zYS0`4;*asvC>%AOA)3t@)n+dKYXP*EE&mG2*EY!3_b)M;QRw{(GIcNYJeB^y#F2ZI1f`Q|^^*JfGydI93wKZ_jOsqvS8V}n`XayR` z+O7*V%GclZ=ual>`uyUNJYGj9MGWXffuIB9!y^fzre}-9v~)M!+KJKf@T|nBMMO$ za4gLk5|rFv30PPf4_$wM)8djt$hX;iRH4mHdW=wr@{4@*Yk~=|lW(Rmb2^3=4VIYJ zaZCz0%bm7b+t`G| z$~1$BM(?XZ75h_dpu<+!;g@n^F|S?qn^5ip@fEuXj02AXo&a^l(q%GLG5K~I$hApQLWDNvLB;>&@|wbKkMERSF2^a;P(h$(_l z<71V4_kZmzuDmN8e)f1-Qy94_V`&NDjBak=ZM?Q?cH95Wa^`x9^}nna&VV_(Oj z4*kY!%H|MCy7sGAMQfhr{8>#y8?pC$Ektr_*j7EX4p}p_5T2R(-qhMH(1SreRtfrq zn6~Stj_J{(akqu8q!5fIsyThfj%FsRJFE%M2CDNZnTh?y&c==$@a8({>X_{FW7lDy zdia#-(i(ePfeq6ww^DgaO2ug$B=hkU@NN|xOOuw3=fObY1Al&$OW%9il= zz+YWAor=@Ixa_T4CJ`uTMh|?Z?4}mJKOy~THv74!jo4NX5)xh9xP-4{i3g}zV^7U zOw`hNZz|tt+ytj&HP;ro<3+=BV}(vJwEK7G%;rn~+3`Ss=M-S?$oZ^`|SM)tS+b+~dbr>WNlr&63)U*N8JP$Bt);@@Br z)I`bg(h)h2kL)gbS8-gz%DD2Ex@N}W&I;uOp9u705TMjbDA^5h)QUfN_M+;9eIiS% zPV!-IAu-E@FM@682+nsazpX^x!{^b(#)W!`)jsZ}6A|=4HIIbWZk#Uwkdv~27NAxT z4A|GNAbD+QmZ>_co99qH5&jVZ!HAAHm80Gmw~i>i zcB4d{*>3o;8I3FY`KuXlPgrQn{{UiGG%@ksw&_@PeN*7b3tXDREvSS0;S<5LApDTH zNOp(j{&Z8gUEdKuz+k#xE~I_}?&7}scPu4Us+=WZ_|EyL_`4cb2OM0QxoM$jiIqNX z1-cC8NBJ;WwDZj`lTIffQ{bf1%cpMO|Iclr-+vHfLyNsgY6bRP z8MXl^(KT(#=Kz2s1xy#+QG&KSG9?fwb8J%*6YI{AJO{p2NzLh#CJh~?S)oErjgPqe zm55ZA@8D~_TXNTjB0zh14!D#d^{S^J4g@^{<5>4D>?`-#KPUEJm~8UE z<9AM1y*%xMk*o%^n=m>6!YS`hRJng0mxH(rs1%ta!xZ}}Cc<9pGmIg%z%iP6FVYPP z8kCr5z>xH}RR4-JbYDAbDux2e`UvHhmzrt8piS@E=5n0;Nm&Z2KNOC084*8uv@>t7 zs`ToB#$zr+gt@3~FXow)`|Dl6Jp)8TVnAfRy&4h{vT8(_8+^OcRtq$8Ub1cmm!CWV z$XV^9ipd7J5#<~`huHlIMsN_1#WE9_u_J8@{+6KQu`QR{O~!k(Eps-rzQ*OUlVv>;c>jDW+Q|Q&*1dZjYCz~+^}Z*RBD`MO z(3waF5>G!qrMU7_Z#SvTkAniYy^t*^gdFk)A7S=vQdt%3Z@VzqUy5H~qHQclSrj1i z9tI_Uk$?S4SreHHz3eS*p817gfRyS}#?=TM8=X;ctk|$z(KH$$uk97E)kR}F-#;t) zOO5>;EZhLkTlnEA!gA?T#w#<~qX+0jUaS+T4^5CFz_o*12BC1r;gYHpBS!gv`W~?v zdyoof+yA`;!ag*VfCRPn9%NN?a}>kW{6Qd4IKuQ1FkP;95L`>Rh=^}TX=+F$UNrh5 zsQG>1WgNAv~T$tnj3?0DYdY8}N zMgFJFZo7cjbi(djjn8Y!{s1BJBI;;MMdqNj0Jr6gPRL$@>xOfN+0M8v$ zHxq1OuVSE93JMC?A`+jvz|+Q_e&od%5-OW#;I@d-N@@G~ zpOsoTsLDUk0x}bi@rfvq;0A-JI5^i9cexrN>@pRm9x$MzUb@hw3tI*sR}E=b@YVxt z*5k=H&%P+rmP*AAy;FJu+H_0JwkzJ2;&ks;2h0uLGCk}AobHuz%QRQ)o21`f zS(GV^Af8=u?A~Q{BO)qvo@UxjVsbOz)hpYXiGkQ&l2%lthpXB1~JQuUU(Gs_8P}2M*FcL9mzZ>$E^E9wKK!4 ze#mXNEFXhV?q=#`t6zP#CUgf2q_FUM~t^L^w_)Zgyg2D;^Z?vCIitWlR08L_LnOgwLiTO2ISGY?; zf6<;z``evHzglV@wSBxuy$|1zqZG>7uHXCFeVyD=g@}INe1V$h3$QPbX4~Ifyv4+6 z!@?wA%|s=3!|l6}72LmwV@EAry!al+(K&G^3}Bm#@2j)SCBVtWUd+AKA5tiBo^G5J z?8jzB)A9uaLbK)CRD%a$Y#J+2pBK~T*1WS6VQWSLJ{tc@@B?>CtbhuI_d)iq@a{+Y zU5B$t^5E&pqFdZ1Pqucw`CLy}8fWx2gn+BV+|2x%`t&$I|jA-w7T*;Rxlk z^A{2{i9q5ts!C0OHV8@H53QcN7l%FBF)5t(NoUVW~7=y=2gVy&^ zpy;l0BICL73cz60a|zWT;;uW-_Yb!eS7VlK9MYAs|3YK!Q?N#&bPKXB5+4od^CGkT@y>T{@0O=OlRF z@4|z4a^zCVkZ9x9g`;O>+uLx%0QyVM9e3_K5x%OVMErP zjC7A|Mjg34n?KtB&7a;30ZKn$dvv&YoG;T`)4w!DdC;1mtT z;ODjB5-03N$TrIEJ!Ee1_aW7>ZXXiL$mAUB{>_850@QyTzpvhzQ zoH1NJUgp${`7i3hKTA~>S3oz6p>sPExTXM|fgY8YoL3-!pik{0ALL)RljD$KdvC`4 z?WJq6CQ=jf#Mfd(@haE)Tq*i1@tvSLtZ3Rs%8ixV<;)pi|6G3)0_cJn85@9cMhBZ= zW`6AZIrxD06f{Z0?Hoz9x1&a)%*{>&{`LyT;& zdx1H8b#!PO1N!@MzdsD_T1y1Vv8r)_rH}uj6aBXy48AEl2TF)V)pIc^phA12d4=uS zMs~P_0C7zU`fE{%UWvHi(Hc}$>zFmE(7Y0v&oSQ1b^@t0klqe5ZPhyW3R}34kz)nt1er2Ga*FHv%?%KU2_<0ia6VDK4 z!GxCn#Wp2-VRJ40ax89Z+lM$23_x-ipLhN*H~BgW$Lg^k%^`RP^K@OCVfk2m_PAFJ zcPWO*njbM1O)~Cw>bmQ_;m~K zBLGcL_->2g8Mouq%-DOI>FU2E3rJa;CTKNHWhy5C>J&#Ud`M&`^J?QWbDSj?JcxDW!;=*s&W))G2{ayr5Pwf(%FfeO@xl-IPumsyBa%df*V;KVO(M zAnlh$tw_PYvT^;9_W9=$tnW1tP_3V0?I05UT>L;n>;cc{vV$$R#m@0)+e$d(`+XXA z3}Z`TN7I9jG>jq-CB>a4Vt#JF&bA!zDe-*c zavdyVeNOb@zt#ZO;}QgxA&9b~0dGum{gZ|< z6FC*w7$@G;J!cs}D={`A29Pu z6_D zcfyu@XuEY68t;&f=rTKWm{(-|LXfMK7gkraA^}=S_L|27Q02r4FmZ5xK<_D={OHtR zCGyukz@<)U5jKG3}XXFUsMhFbTkJy(sMP>Vg~vzx478BBk=XsK1lq!5>xK>+>V6^|ICg}?Lex8A|HH(K0b23`qtfeHvSt% zHTE~5pR!YH3diAWC5>N2_*;wL17GwzDvea(&%r5`?3M?k*Cs0{?wfbLUDlI~d?d49 zW&Ey$>%{!zIUlg5nOD<7laG<^ETy<3?N1^!_?ruoJg_)}HME!C2H zH?!~G(n;)lOZNji^;tODLO_8yZeZ zrL1{2qChHQbNll0@|Q@xY#V^h^x98_v#(eD&ccprNVpAl*2qXm;2%mrUU`XBk8ylF z4g&I=VHqN>P!#CNPT_$=ss*H_au(q7o>a3tU7!BnBAT1EYV8>J`xExC%YfclMUG0( z0E5#ZI#dWjK01d^CPQ3>g@5-%iSfzkv^KXna&LlY)(piX7j8>5ccwcPp7banhY(P8 zd~bdo?@V_t+Ph~QAhfjUAkG`&VpnEMwXKX)jkJVr zKUqLu>%a}X68#h0-!1iWzK|j!e9^4yi>d##b}kxYciNVK3%p2h&fB_tBc0eOcbFxC z^IKi)Blr8UZC>YxWRlckj|X4KKXtC{7%CepG?W4-UXiR%S+g3Pt;TTJV)CV%Q_*-F z*KaY%M}42!X*-I~*3Hdj9l<|T#XrZ{v7JlT$~j1)EX$bwg~N>@Rkq6}@_IuB^M zx%)H5)0NH&C@Ld`Y{(g&o>S+277Fh}Muz8p`4CVDfx^Q-<$Rq@Q|&q1ZI%j6${9+J z_{ovvHVfji4lFtNqV&M%!A{>mZ0`WH&oub$kN2h!=>wOZ{sOy7<`a&czLlxLo(8e_ z_n)WJ@{kg5MBZ!?dwDdp0h#Tt;XDjSNb4;j5MA2$@x(txmoL*4B^emL%JsZEl4rc$ z1`pnFDex@)NlF`2y71q<0EugmtjA`cqr)~(*tjpEo^;%kh7gUR&eo5K_L)G4UNON7I@7Gws^v1q&QJKRqCOBP0aR!<0g4{C&xY%#G*21|9Pe z%SwmtxkLMQA6E;|o!ucP4?`;@F+TnzNO0b9c{Ed$RU-GI`xDQzMyU&2ljEg{@oMMT z=jI)Gxdl59)NjEgz_j_7+NF*b(evCI2$4KJ_ey2CK|ekG;BkLKZZ6ddGb>c0Hc z6M?t-%!*sXEFm%IInT0MTFS9OobHqL?Sf7G(klm{glNbUQ+zlg)!zGsrr1SDnL7$i zvd@_ALXbm>uOo5wO$F# zb}W0_u^M9ZHZ^5oMklx;>QfQZLNt(+OM>&CM74LY8Z&SQGS~x7iAnO*w`>^6EoL)B zgq$}?Ett1qh0O*7sZx$d4NQUk51iXPM+d)je3H3K`L>2P;QNL9CReddYXx&FK2LDy zeO_#A>{hKL?{|!i>~BhxK?brrROI-nL%MWr2akXlQw;`7Ig00gok}<^Z3dG)lnrP* z+O~U7Eqd291fDF7OE{YY+HyQW3;wc_vz<}iO@b;hS??=*$O^n0Hc@|(_dbGYApiqO z`t5F#ucd%gqYsvm|e%ppHA)2#^OHpme7zs z{1f;^ihn|7XOxb7cH{L!PNt9{BfN90v+&zZ%i*zmL+`~V@ZJhLZTm(T2xq%QgtP!Q znpAIWM)gylxopfYH^5O1#=vV8E=KIkw_j4*)&?3wJoXJ>c(4_YPdz7@9vmT^Hy-<> zS!yOd^1&jKRSJA`6H(y;{3kfG7N~-ugdYPo>3)P8x35sW8p$rNIkqiu<$1op`*3nX2qfhdJzX za7v=uEEHit3QBGO8Ws#iAHy&~v0hcXpYwH;gW)pK1oI&#i`=&34@XuO!7O=+F1+EP zszz!U)$9>f*o|7e?2}BcxJ*>G=k2)EvT3aI6BG>LQdzbMpXTNcsm;}6P;GqbeU~<( z^Rl(eqBiL1Vv5kcg8=E%+{UaH&qDkMB`BUntr?8C%<7ln5t~{Hujti= zZ%kkzQupmcVGki;B8~w}(*38QpQZ0})?IdXW$VMX9~Q(`I`Ll+RcaE_*+U}WmB6Ea zgSn558mz(AwQ^rXUg9jnw>WC|AwD3;@v{tcBNZZ@Qiv0OJkS2MK|Q!qcj;K?@gYVo zW?Y+$QF*&s1BHtTfR+=9(9#9Aj472o08m}BfS<+jT6^k`PTWDpcCa$6l)1bCb! z++5IQ^R#kAm$+3upBv4@ZsA1zpe-cUqLnPNYy2KU4SD4%RdUDcc+VU-;)I%t*X>x* z zooMnStGQYSso~Eav|!#}@LLMXtdhG928)4fcEKdg6P-AEdGe7>S7LElRDuY_uPqSIg?DGELk7iUg)3b=J)3Vg5tN}fF=?9B}cku za!*#l$b4X$&8f`|A-)S~gT#947zLf8KEQG}uU=<(?Ce|?~8!}ARgoYfs@~6>UPrc z7Nr@rRzgHLzJ&Jnqw?0f;rv;mrp=$%(^PXT8M7ns{>R#$$>ynEIySoLl0* zRKD3)Y`=Sn?3@f)9=vnue9Wbh^vEbBG^tDfbdydr8}7HjK}OtgdX7D!C2ak60}d)C zWFF2t8IzVLA%^LMBlR>o6fd8ACTJFe)mOlqgkJQr;yu&t>h|e3yF-ftZW}?MRbsUb zVpe3ggPbO<+PBiF_{k&PDhFj(R~3?FvL9w zuhUkqhm9o9oF@Ok_}~RxD3wqpCW-t*ECPlu$&W%$>C;!QvPVKrUpK|tR+F{eA`7@l zuklXSj7~0Fdzqt7wV_O5HAz(J(2V*$wP3`QfZgI?P?z@Q!E?`QlzEY}*q;vjJizS| zd}O_g6mD=Ct5xd?|3TqCqXIj}-3|zT+Va47ZF_i6EwY>F{9^I1 zR`ZRxPhTRV2qKs=$SUk6Q#gyJE71`wm3^W32Ko-Vyj*M6z})gMLEvn^>}r8cfIH14 zG;;i?Kf`u=I^fw?65bk2p+IyepC_(GN5K^@21U~5fNUUlSPki3Me=VVtFvEOdOwOi zM;khqZ~u*MQki!960pa;UVq>n$Je)d=H@Ly{(s^r(#a}mW!e{N= z#8AD*Y5eg`pkG7)NYx4C;iZ*Kg8C0+Tclw^1+fU0SWaDyAX?>Q{0^kKBRS+hsRgBT zu3T<7PpgI|7jo#lvE8njxUdN6Mi^5LJ)QY{hufn5y!d{u0^POl1sEntRL&ZnFq2LOAx3Y8sy za;W8+od39!@D+>Tt67CTrS;$8_|d%={9!TdxwtJapL~4-Lg^co!j^AEAa=!`BeKVt z!ebYPq>MwEpcXrsDzU>a_w|w4&E#LN^jeUB4wLtw0PC8O*H_MSArY$)hzX7p;3Et0 ztx5Q=Qd8wXcKwK`48`Z?3l>=Hz`CGZ4=NC~lgBTS{E@mbiu;cz(``{;l9NwPZp78c zq!GRrCDEa@)@@%Wz20-*gr*5d#Ph4_Bj2Ri!?`#jJ$!I?{~z_z-=7~6B3xclL&IYj zxm)Q%{J;zjpdM>h6f_E>ksfz-u}des@i-6@V$=|(@S}wq474PCdlDyeo3rESsg*!l zt1Oi|g)E>8v>q$aRu)k$(ixeAK0i&Lx2-Rw7V!v#)gGIUjZTkOm{2eb`Mo^DUT&`& zX`DO#K=jaOjTD6lBX!5qIia^mOrKKF3^jK+pU_Gyjfn17y*G(nov!Y??I)o$aBpEt z@$~9t<06%cmW`S^uI;a8(*2fWtDPUQyZE`B$pb!#MDAlew>mC-TtQNjVpr+&Sxm9_ z29n?wB&$uBS>>UaVFw)Lb$uN6Q%ixM7p$N?;aOXX?)pY9tpBEzYQ1f_oMQqYNwjRx zVQYcWN{0B=w|04X&0IWq(MjkXw%9o``*{EMPZsIOk8vVttq^Lfr~BbYJ?qU8ngjrh zxWJee;J=Y?pSp;PL|yU^y=kAd{w$ktjMh zT9Y~`W0Z|T+~W-~TPxj5>VhS?9$(qL9zvJ6wx^~A)QGRMVQN~(H+??*8`UC;zB5(M6^wl$Gr zEe7hBN?+g5dN-(z0e-L9WAf0XOnCat$5!j7k@zDi{i~CM^K|M zd6<~c@V&6$nG5Nu%^0hOrT8Vm)KR-q`jhDUgHGr7QD&9N-nd$}i7FN|NuO2?wVy8J z?FKOQNHY)YCfKWnJfM|KymT*it5+Z+Yd$1nO?L|TWN#1bg!=jR)vPF}1~3e2CzYR5 zTE!Hpj^P~Q5tcFA9;l8=J-CBCJ=EfUtXr=1aqrVMb#gMat=V#xdU@f0!U52z35z~%P!%s1`W zAi)a(*H6n~)hgd40kd!Dq@(OpDzlJInBZN=m+6Oh0Q-eMi*sD?8)6K=!bnb=D0HMZ znVFI#wqmAuabKbF*@MKF%e=p1(hfuM;``+)r_Ea}i7Bc{LLXc1oFbT0{`}K_jWC{% z;+CqVKk=iw=PLb>H0H!yDK3b%S?YEo4#pV0TFevMJ9HgYN30_ul>J@v&=zOx=y8eC zEak*h9oS0b6BIy7>n~bf_K0QH%9yL}!b8J3l4DP<#(PhoqQbo0khh4$aM1V+v`c(p zlMPJ4-4V>*unmWmY?%M5alp2rs8kbsf4=9bUZ^TKIIPz8q`AYc5s!LE6PKm}&Q~yW z>%wJQs#;;0dp>KAAefBr)orGCaruRUM06cT8p2u&KPb5#uL*$6UL#o28gvSDd}c{g z0}7T5*&T1rzH@u;hNDj9pnF$#`CnI_=zEid-{;x|n)Bi}zrj>KWC}5ZYF&sM3pqmE?=MDLc_DrL&X&SETrR6 zr^bCb8~c}BpmfD6Yu3Nq?Qi%R4b15h;kn1299~~Y`5n#v`bi^-sjU!NycElbNHoH% zE;zRbQWQO-k;R|jy4IGh=i+{x&D|&V`7EeuVANIxg>jfFq5 zs3Dn?oyf3L!Ns{%lG&<#@s8 z;SA?0l~$KT{fWZ89Q!jY{(0>SM$iV5X{d;f{F8xZg`M92JJF$1Bw-}x74@pJ-y?T6 zrY|_>PU`_OjwE?WSsY)(#=05`M@s<=K0Y-M>VCG#zJ0PY9rIm2{-qTNM23w)OmrMfOwawZHtkh zEPz#!j`9Igrm&H1&zLP=U?&&91aA@%6N3zM4g5?u9OSq*O20KsP=_HlG3ndAB{xtx zntqXfT> z+5fb^&Jg>^sqrORIPrH-qobcrn3O2RB#~~S;kQ2&AGT1RnyFBX=S0->jvQxGy!(sg zTeWAGqw2*=NJGtf^gi>SG8C7Ew; zFZX1`$IzVQRhw?dI{m6PyC-Tp-YFWKpMd#*l7p4je^lwdDxy2ia8$DGLkTLVhiX3k z3a-4I&m?#beMQ!B1`c zOylILU940QYPPPC8%BVmDj}o7ORS%-dPoxRE=Zb4kymr{PA$(jHg@9S%Yrw|Q85a%~vLw z#Si^4dFI8;&K}Fhw7K8UB+T7S`9QS)QgOqh8*ZNGFiotX{XpY-IPZ71VwVG~da(mE zmgI>uP$U$Y;+bi2zl#`?ay={M`Ui~->V4Kwz+;0S@}`5o{IvzFuICC!`XTm&~KKs zRJn+{#T?qvMV!f1eIB)Y@5m-)LUfr-2xWfP$6Ni>%jAmzo;A-*1QI5PEri=9Dz5w$ z@esD&8TuN4x#y>O^dpx{Znd zhpRfB3z%i&E|k7fJn6n^SRAhnC48#;s^`#!O}ox`&(@%7zm2U?LsB2+A^KKlmDi-B!o_xSi7L?TP0U zSsF}qyY)nyl=iCA>#5I4>-|2R&-3rK;HsULg!ef%+S z^Pn_sxPotxI(an?huK) zEV^h+lHSAHEdSC1pbDKt9aS`vm?qPc&T)LEF~l6&`E5_hb3EUBk5qWavwACz7V6|d z8)1femDWeZt$O7a%iho7TSVxd+o6}=JvI2P&jy#{#5@}+)dEqF%Pq!q4NkUmzpe-_ z^S4IOYCnYd`Fy=ftFWF)jVaLcSaqzlh;>sIhI%&RVVAuaAS{11@+_fdOO1(?V4_I{ ze}0SBoaL&}tM4rS$|tSdYSeu`)b_&l1O+R)I^Sn=bql)ks{NR1qr}eKd1ElA8|A8S zd7@sdJD~`ab=_ozcrAZ1dnpb?(AkaogV%_?d1oA);S>y4&R!}{RoILni+H+a4p7#c zEWT`)%h##b?9pAyHZMw?r=n&Z8hcVTb`(_j1e=aq3<_%$T3I!E`*>Pd7jNb$SwJAd zy`g^!E%?kG3XL?{vG?|I8Fq+b5RPBhmWK*Xw%M0HRw^z{^;a&p9hUd5N!;>qw4lBo zq!QRPELdc*UUSyji3Oi;^E2)AX+ilva!^3K&mTIsL4j|F-yETeN%erhhPt$*#H?0= z;i6=bk>BQ*c8(=`cR#esC)$I_>7Y;|@H5tYN*S@ecM6+Y0A^u>Q+J=4EK~HsfpAg% z;_Q_8) zPz&%8VV_xWyLJ`HSZHk!K3(CkN*1;}k9GYV2HR9-&1vVn+P9!l|O$m7#8RlN1+I11wQ z$F!}F0IP1N*El%%1zp|v1kdt#SG|-gwyvxN`&mE|G|C{|baW<2oAAV%wgM+QfvQ~o z8SFc8LfznFrH**UoGG4*q=GhQJ%%DWg~tfTCil5%vWM*u-c6G3`*Jl&e_X@_O${+* z2(i?^euz(kd71Xsa*GCQuk(djY;Yw4KkZ+nz8)i~!KAA?BIgmiXZz@RLQ;5mxbrYF z>_t1<1Se(%1&WpjR+XrjaJG+sx&Wg$YtH+CZMI7J$|t0nr0Vu-a2wnqp8~uC#Zn8z z6!FLblvxhglLxV0FVWvGuM*?-yt*ay{(l#N_*5(ls39WiX&z+u4eKzQ9ccY(PFb#iqA28f+O~blMDZ6yXWeY7=^uN^CO;?fA}K zWK5KS;Y#=&ul(N3!WAhh3RROC>=#1S9ydl0kUISlF^u1%pc1%A0LnR2G~l%8LL))h zfQn0YZpXiIGe=d~9r7&!-&z*6@bC`8I8@+d@)c2#jC4;bYonm2HD)cvKP1qm4rigR zmbyjJNpVVgh?%T!iL~o2zKEXgFrwYozE9`sH1+mIa_irc`fIzQ3Bi1E7Eas8NPbR) zzjr{W-sX_;uHPQlyD?`-!wlO3^!-f1M8Ah` zCi^yt0KKoc(9Nobei?k)gc<&G!n~$oysV9dQkT18UhM&uR{c=)AG){u&g6j7PDpeT z$|K9$FmN~*d*9)=SUOD!Va7iNp-@dv*DGLnGFNVQcc_9&Pd#;^73($i z)uaDMXLYf_jMpxc3^gzU^PTS|wgDfm`+154|0#oS{BJk%U^s zSvpkxq$Dbe{2L4~OIx&k~3q->&a5WNy#oxYh;dY30z|a^@65UzS5(! zy2|`Pi?iw=3=Ny@;oyHO>l{K^eJJ@R!7#m>1iz&q8T>`ki*Xri+atCM(V4AdpL3a z2vvB9ico`snX>K(v#P_)Uo9bv%we+NfBuPF?n{HI%)z&Kr=^knrhn^UlU{rdN}uRn zzHKesrRYN7mrp=%egCLV^?iAS7`}b0k{T?JdMtmQW@-~&BUo9-@1?`7J-8oQ7%*I- z+Mb=X`1m?jE@879our8-J}~qXAKfRo_uex4k~|0bpHBzMPlbv}m)^=eh=Gb93VuJr ziSgjD;hpIlci%8|n%zF}tgpH}@8Ub;`bq{ppnaEf26U0fnbjSyp2;FPU6JwZm?)S2 z-jdGeg$QCC;k6FiEC%K9sO#5c<=7qA(kg!rL-E|Hx%|~4%w~Xu_fOA-(ARV@#`!I! zi_kkH6sEazNfHUvnV}~{ZcB->hr1&0-KOu}-@-ee>Vb6eqMYlo&YfRw^^2Ryvqs&H zbSg{7qqNk^h}%D@?8~cl_DooQR&3bl-KU}idwTnuZ_>qqwA zy#UO`IhyazIf_K#i9J7a-aPw5E`v{jTx#Ai!z7UWvVY43|48MvDLJNeF#hYl?vVt% z3EKnxFTWR5HD1lwr=ZK2^F?YqEQf`@eRVq1x6p2)I4xq0pE>is8zy4cX}+OQ_uVa8 z-h_PeuMvT$%;UV_NErMliUq1Mr~@7>^q$jO3dv6d>)JN`NFJ)8(n(>;-t|s@EM_h{ z?Pykl@P|^S2Vo4t?a3;G+#!>9)YP!r5@wleb$hBM(Bkh7u2)*lI?aCWoTt6khVPi{ z`mp+j&sP!K!NoNm1;(pZ@7eT|K1Z>j{h<*=KqFqXFb~OsY2sSHHKMNS3yq4)y}*yo zt5TH0UvDlpREpM!ia`rt8iyykQPvXK!Rd>P^=dLlOyzNUhJ6&o>;u_Cd>fmmH$s@% zjI)&Of172wxO}m+oiap0ufk59J9}OLhn3P`#3otv-~I_1f{ib|dvZG90hChe_Yxey zT;p>|n)`vfV+2|oEsC6nGPM>j_R)Y>_{fE?fg{E6F1qZ=v(4(aZqFzn|@w3kGJY-02(!Bfph? z?|<{pvEavy;>NA#IOnScCmR{&hWiQ>M_bHCA}bxJpPmq`(;?I{^q`S3fp(ooUW&>> zO)}15Gq8?cF`Y$o-1qKkwrax5n=G-C*TPOE(ym;-S1+(A2wh>%fxfE_qLw*faJ}B5 ze^|$EI1;c0bP&(9k*`Dfi>ii(Oq(SP90UvB#wVpP1$J9A)19KY8QNnAvh5Q#wD zT)w-bd3Nr1SCXWt>7BeU3xT-7frf;n$Lh zHaFqME}Aa8pJ9DBd9$Y(eq5>}kW_kBQ*eP5zP8u~m##hjazjeUq?|OB3oDuDr0wRV~nr9#&*Tm`Wz5HP^Y0+RJxCH5_8{ zSfU9@!Y}1A4-lS@Ltk$gSFGy|~XT!*YBfBDt1&Pnw}Z+Y)HzE^o^ zHAy>4i?41Xz$ZEib+3nxn2mP6xOmd-t1@r#W;9cXQh@CM+8^qA8@LX+JtP$ou%Al5 zT}O_mK(?TZUu=@+1U7BH0U}A_9V+e$S^tW?#q7z6nhF-~6VY(*X5)i0iWW)v5INm1 zyX2G{#d9OrdgkV1r!=JO>yw!dBu6{1qtZAsbifArHlm6qTm;ADC*+CKSSdN|9mCPG z9(@}&5=k!q1ML+8q+{>=;uQ%P-fHlBMe-1|Y#Kil;zw?$i6KY7KK6NLB)%SHA$O5U z6xtf1^($9V&%{Kijy1~cpqZ~R-0X^sHyeI|xp27=8qhHLE!yj@SK zu4my#lH{XmO6{$H?W3KnWFYKbOY_yMtw|xa^#*1j0)DCSK1-R#Uy0rcF6Ninma{0Q zH>$0&6Q6*h1vGD5@NT9U5j~b$cT*8W!93NP>!O|o-8nitG%D%hdpu|N#Q1+S)~780 zu{)fI-EmNV1?*k@&2Jo-yTq3q1<+IC;kUL3@;qfGL=G`G}>N5H?H} z?DH{mas8UPw-6KuR%Wu-yGW|;B_vF{JNUnLbp44z5g1d+3lvn`1LEzd zF4nX2+rGx_-1&O7N;*0^hDavkcUsl!J)GNh6&)|xfye!dGSzR38LL<$hIS;Zv)r*K z<%wES(vB9uXh1PyktroaRfJwX@!`kM=2TT+O|SiM^v2f0&+M$GW0G!TzUW~YjTpjK z^0}P)B=8pfk42>y0Dx9Tr7HyRv*&fbMn!PU?|)g(iZ(-f65JNyq3|GcLZ@Z$gDmwt z8orPBT2WpJ*>E&1B5BchbPMcIrlOGgU6hdoKrrg%P5v6Q>zim$jBv%U8&R1tpKd4U*mn{s=G9OQBfjyE(OJn}VN@F8veilYb2N5Q2tL1MD9bd<-uC<66B-6HrRP{fajLf`VR`=aWoO$J_ zy9En1-l_)8AA#fP%n{Yax@;!@Rek0slr04@Doc2baCSe5PhU1;#8nQKSU(f@$gP?= zo=1~?4-=d?dM5g>)qwAUgc9Rl31ub`l~MWYXcF;a1APDHWO821Nv4+FRzTNbNEdMo#`oe51 z%k2eQyZ~WaCi~)dj3DAzd+_hIh+86XeOrPqfp`=0?^yM4H|m8kQCN8R(ZUdyGjuOVMI;57W)pm>Z)C@zTzRSi3PRC~fxAJcj;aAo_75a_y-NZvTELQ!$5L?^;fpo75~Jqx^yM@2Il5i;Ylv} zVXeL$Mqq%?;`XcFAn{Iw-~M>(qfrfZ3#)v)LR`$!`C!9oeETX+7Z$25HtYDX&{V1k zhO@8kB zk50Dhs<&Ji0;qbLqNV-ffhQr13|7EOk{wERAu6{S)BsATzF5N%r7bLj$OAk{v}@gi zVv4R*!^gS@00wQrL(Q-&NpF9=L@Zi&@~r9)2!Z4`48XYDJK=IVU*8+Qchr84anFHh z>l6)}=JI5StKQIQZMYemS{t*uVO6rp40&vIm0^6Eou#qtmtWM3oFvt49ls_JsPI%Z zM|Gmm*+{J^Y6?7Dp7Q0*J{FwOvhu%KCjmm06EgPQ{K+cyJbumrWs2BWvU?Tw3CL(M zs5d;67#X=%@W>Bj1}UV_{`4^4S-{OFCGoo;`dqbWGU^nLA2zZW0(0FP-^G<5i;1!D3-m{gRd$HRuR1GFX~$4NF=tIqV{i`l z;>GWH2Gf0~Ui6ZFq!gmqSQKMmFjA76r|W_f)KKYJjSwgiCc2`!dWn?GkK=NYOIW7O zX4(4cU=jFdZ`FCGzE;2V2G$eJ7{j|f_J?|*e}HP5xqHa7=mduD^LM;=dtZ28^i=9r zdu0IE4O;?bMmcQ1tu1Rly9s>^N+;j*tBNEgkqdRBqb^XrX+$UZoaB9!g#7NQ8uE(y z?5!;^1_PbRNeYJPo%k4YajEM9`QUMyf&z-Hdvl=igs~(#e!mPVezu37V6*XVt2oVO zYkt1vM?1>XUH@D=6uv(f7fFgdi$-AWx0zB)Jqr9GA-y;YkQBF{rj73UV57l8yf}(P zU?5Jt7XE?ba}gFCKWQv(v3t)Rn3##stoLPPz@84y5^3H?W0XlYmNZOYN2@_s^q)qV z*kDDgVI?gqwcS9=I7iM&^yWg_kZ-n>w?G9snlCllmKqTSqs=pMiaWzh7oyA@w@`V7 zweto2@|(5@(N=FS5!>#WVp#kU2$4{hZ=SM-+E3wBZcel(9i0zkDO2N+-Itc7_AUb> zh0`rZPOpyk!st~-P-J9e;=~{gOi$y|vl2ZN0FlB`oc)f>>PGNXVbR8ldX1<*W;9dY zF7nFicb%P_bTAEiEn3TPuVJgPzTbK4Tl>Z@$G)QmOK^Xj?_COU7TV{egu8dGelg3p zD(Jaar(Y!4@!&}R+vNXK4rHaBi>HxXq8rOrP2;uJUb}MDdo%Nz4FpRKW578`F6vdK z6++#DiAq3tAO!Za?OWUC^zMDN_t!eKIhox!&bC1s;YHYzrC-4*;I?x8~}iKI4Q zo;%3`>?|Px7{sEl029UnYexE8B4Zz=iAaG+b>{W0LE^7m>p7pwgTwx)`u**S{TQ$x zQcGicPga0=yJ5eggnn|~1{t~o7A2To94F+QJ5);tsu6aPUy>FNO0!mb;#Y(4B#i)% z$t@C(g*W$tUB$-Dle`y--V)+7-!b+n+}CQZxLKp|tY|Q|Wu;M&2-Y5g1K`}1bt`R~DLI=>k9NRU4UU^8S@ z>+nUs(Nat81f3D7z;eU#tH0P>OUKyJDABAuPAiDEcai zG?K5TpgN^5a{yf*3r(s5Kz1_5$6FI6LF%#Z&1pjMyE@?ew+`#qucB)HVw zUj<9M0{~wFP6(dQQJ#^E7PBiYehAWmBctzK_#qSpv!lqF!UPr{*&bF><-~6QDI@dq1Y#mB_}}pD2P{9!(hMy z)R?@A0RzK*0Wi!~FC~q{;|Obe^zN+NE-%Ue&^lSRI|trcF?WBc=nb9^BDG|HR$K%S z=O4QNzXsV9g3^mYfanb{#X%1cUCQ{0`JzIc27ednCHvK=Jf=V99uD?_qfR5R{6JTc z)7@6Qu1kj`?_*WKY-3hSJ+2kYX|T|`YJs|N+t??9sQ9w|gl!sa{_UVxqY=5}DIH6> zD)WEQ`nh6I?hC;P$)ufgMJBFUplAjS5p}4eWFeP~4gOJwPMH`4fJ!&BPxPYu;{0^+ zy_qa#5a-m*CGZh11kV}+l@nE;5NP@7D}8lx$yXY>`yv^%feV3S(tdwN$G>tYUp$Z` zoEk^IqC@eyW1Pa)@PR%QKgozLslvdW-G29MHOceJ=X^a2I7Oc?tR9DJDMU&SCBWtQ zAyrwb3~)Q1spRITAaP0H=ZM&f!2^xScoDy$=UubwP&vg>G}K2)gZok40_&OY+geBfSxn z?mma`6IL9Wuhgo8Rf7ijFWU`-(EgDX{;f{eFD;75qYseN&oHaln@-kq2i*{)YR#)hll-A^}BxFR1V(>qI3K`3BZ7XGr^VlHC=KrwEQLLZZ78Dc|+A3}x zu64D88eLcdGM9aFnV*>IDZpSdzQu#GJ9(@qOx^-%{z{{U_wN6zs6b^0^$}0QAKqNCv^NaX#*^Q|7M$HOaM={qJQ0?Q~CY0r}!!RK7hjbqy_zhkr?c7 z$QI3cYy4Zc5(cq&M<>>HiQ$=tI|_2+P0)Mb{9RXG$Y4-KUPKX^XaF|Ul+KfLkR#N1KDDD*V(H{yrH73q z56qxmM#t5yw56ivRMtY6od@U2>RQ3E(<-FFt?9zi$JT^~hcQ_3+m&fcgh6r9aYo}b zF@b0?zCWxx=s#ZNUuF*ul|w+;>fx>%uLBAo4<34RFEJZ1`EUUzr=-cN&iumJ-1eQwcN1L`7dig5?5+F)Ft%e{6ii48z%Y&(~1 zQ0uZsyYKW1_@QX}7e0kwwPalHafkM6Mg`$Tu)_rQ^JDr4u!%*B6Y#74%RVEc5;6dD zl8C%bl+@(Dnvey7Z5kPZ3KiZMnfcdwssL`27F3b_4*JO|hqMy`y?d+il(H<)YLqo%d zPQ~`~Em#~I!#Qbyb}a|YB$&}!y8z6}T2z3!G>kT=jSFb9pw!B0r@MxeE9F5cRW&{q z)nTmDcTQ)od+Sdl4CT5Xs$%^YrB?X>wi>mbqGGwqn{b?XlGSq}W(>@MXFvH?ZZ+1~ zEBSr}ko?D2YCr&)XXc^YYUB}b)6aMxe-$z9i*fzd zuffubWfx(J*o$#R77Z$C?OKllO<0*io+f<%NbUPc3b3ZihL)I0;5D9f(A?%PhUk@B zYB1W?YPMYJ^9Ub2I5>E_Je#@H*j2Bg`}6EfMcyT1(0340O4m8duudhfrqI~%OKR);LKO7m253q0t(n*x_wSlyH<$Vay9YgsGXk#P#Ez;yh_)*bDe~Q$XK^s3Iw zdXbic2=n0~)IJZr@|!KP5>}!|z+ow{zA*wv(q951Moz(Grf(x z_JS%YB;Jwp4Hp@SO#`iSSd#t!L+{|j~^eFnR5K-a9=T*~d+w!3G$2-I7VPPcIMp7G30>kSTlclU~vJ zV}dq54kGsAWC{0q6l}8FBqY%C=HpqLT3nlv3R7Y4M;5DffHA*ncL&|&yFWI`!9k*| z-`SV5iG=RHgTH$L+7C(-#$JoB$8LKWBL{la*6k7VyXg&vWXuO}aCR$}{n0b=AH3MX z0A$S*aSTlHFhuOKShbmdTVTJ zy|+nXkdw498&!|aPb_AljF4&PF%44ES5p-QS|KBq&wpS|`1=6H_*TClz-_M)fR3{t zXWV}4aAQOfFi;-)zyWpP2+$Hz=XCQ#_qKzawUXlRU0jV@urt6TJhELmyW@S;g3aY7 zdtch{gG5mKUC$>Z22QHo+a(x?Ad}$OD|jcyp_3)cZP7BE_YN8Xz;^RG20#sJE_P~L z&2+|FJ8`KPzgaO78@sKggtRncQ3R+di;UFvv@jiexsR3hiEc*MI;}q|2ehBRJgw(P z%DdXJ4U|-3NTkQBOwvY+47jGEX)eoM0#0BX8hKs8xSEB7D^KIob`-aT%10Qo-o&l& zY7Bj4vK+KtZ7i57-)q6x7AjMVKW?mpi0Dr0CNE9awqMq*(wh6sxDVqUI5yGSJIe-| z&(!L%k>rmMjVYfxO7Au?CG1y3>awjD3>N*2B9<1}bD;k(UdmVe86vuMkdq-5EB9Zm zD?|2)`vLWGE;d&kcO8B`VE^CcL{FiS%3lEK8OZ4J) zxZ|_-VUX8#Dd(__tPU-Ep(+(&MR?dY$o;lps&N zj#*&ytod2#>EEX~nq6*SZs!)i_(@o9J1w+QF~Nya;p~-t1^5{ETtbQ$d8{ij^>!;a zK`G(`8X+r@bg==Jq+Z_?|6zq<(7O>pU0uJ6m{L<;(o>V1+;ky&Kh#)EZ+xlYV&|X* z9ngB4JnS}E?ui#re)-(=l^+gUJ10des4zJL68V?0FGS?h8PK zNZZ}F?*cXUi6N;dWgC$|ohmo$J#pk+y{L}UJ1>G+*OqO~pZCUb{_Kq|MrtvS{z6g> z4#NKb5K40fklS`mau;8N+fHe|Q{6p>iNLWKLNdhCpb6#kuPVYR& z_d9d%lWM>0zNGY!iV_hk(kXwripwEZFuCmZxkR{f+?Ml>!Bahf;f;(LD75ez6|K z)**isWbA6K=YP2cBWb}DLW|!^e zle#Vow1q3|#wwC3EkNV9@YTChpFQP+a#PO_#C^Jg1^&enN7`_T@vaA(3=^?7j15`P zF+;DDnjHn^2RZFQ{M*j)NFyA{E~fvIt4 zJZD8oq3M8>_Rh+Cwo$nn@L|bYoxHsG^-BlqNSWPf)UM2T(f99ve3paqxYxCcy`E9% z>Zv)!&6kXUdk;3@smEXrOV7_NV`C7~c#viYKV28v*n8eNb_-`df#Rb1l?uzTYIt0# zLi7bVj5=Q+;lLi3W>Bd3qGbh(caUZ7*w@r;3wlopt%OlBal@~8X8~4OS&~>ADovTr zZ^r~jgM+=JR1kbFg5!O+?Jr?bXj(T?I5gRx^_|I=d;?+|=U@-L*-ug=m8zV-fsC1loU{y?vmZTJce@!HyO5SELL zJrW@qc0+BlMHpU@8ZUNz=s4(I$lRz)!xaH9rEG0B6coRE{N}m{m&mXtky4r^!ZPw+ ziH`ORcgG#Y0&*S;c<&)^YT3%TevR4UQQKj)lZt9iwaYBCKZV(Q(=)q2W-73#BpHw& zpIfUrTCwD@o~%!cEz~}9!+daj^fLCDEdYW6;F@*Q6}YYoV=i0scV-b4X9z*y>9J6` z*}l1&XK_p$d)y@KBM)iXEC314uA@HO*?`T#P8XA^b^ZpxP_UPvq5 z?$>M~zokr=hC$TYdSP;<=_Pv&l9TCB+cKX5103Z(c{F5idQsSQPp8H2hOb_u;De7e zLoN0WrF)YvThIw_SPUhV+gb3Q=w{mjUOpk*A!)*PJbAN!bFHOAUHudW&9M%g%W!o) zHqbR?d6UPi?;eN{s&e>~Px`Y}=#dqXEh2ZfHTt$b#UNq}U&PlIn>6xt?Gj3)RWfXD z!rK6k&0m>r+5|@NREFs6pY$t(Z>j~e?H((!WIP!W$7VGJac179F%tA(&+#TlYoAkp zf}d#4;{9Z40ee-zGX=DR=zNitdU$)QM+uRKemH%CO3g;kW7E8HU=cO+!eXyn^E`u3 zx%}G9*qzGiGG&q&kBGnf#x%I*E3Or3^!{J-m z;JxD_fJMRs_RCzEL0&zU@Cd#DN=90%Rc`#ma6|A=tLw$jooF>ekK^(Yz3Dwql35<) ziWWEZq}`!i0ns0P?EEDO-*ar0TitT}o2^_9W_7@US4tjh1CL0Hy*&R6ob~9^F*pX! ztY2^wF>ZgimVJt&(=qsgvL3jHDSwn&T!6s)OFk_ri@DzR^C81Vv4G3G?Pfi5aUMEP ziV{XCmV+;7fI~P9lcUwNw8-JG+8{s2rK}Qr<|(R^y%#C+k&_g598afZ*hQ^T2*fU{ zZnYMxM&Vj@2?aAafjV9Jtv*01;SYxGCZ3Znt~XMRWtrQ=fg`SWzh%C_Ib8EqUbrK_ zeEQpta2nE)O9b&X0OSTYw|u;VjHnsz?>qU>|2X~tDZmb!C^#)as$!JTiRBY*u;3(x zHG$LD3x_cR4m%fyS4_MCT-14JL*s>&sg`qJQ87qu734lS^l`5F@|Vr?nlDtK@)Avg zD1xaP>W+)kbIhN`-Zk-erUlyLcqUBkWrpLk@>4CDIfUxyIs+#t$y9@cAt-FLNqXf`!obGl|-(1$qIRdQN{o7awYI*Ed2M zt0R7&k1yNJ=)1!=Wl+xS1)RVWE4x*r@cV7yKch7nrEeyb$*MO@KRMSChgk7hYq)Hg z25!{Qk=<$DtE?2P0?q;6v2)hrFsVk{?;OnoCy=0of*|H>&N4_>T5;$GCrh3aD+;V| zBf^jDC0BjdKuY|jIHougEDbvIA3f0`wCy7)9mh{0cDpDwl?h{3xbx!EW$H4w{i0T!q>=`c*W;MgpV{2ok}m z+o&jj_ers@X+#kMK>IoO2p$4RvwYkWr*e+f*Sbf6_xY`hsGPZ7_|TmFVpw!^Cb-kh zW%o0ggyU-2TqP~_^1*DhWiL41`GIWM0XdHL`g%UC_{YH&K!J8Rk(#xiZGAlx z-2$ffzJ$qUt^!e=ITa+phO7WLQf30&CZG1adEQsH!z2%rxl@NXyTs51j^n}o0uu+z z;d?uqs7x^>D}nmcY+G~{EhdZT!vWNtJb223Y7Y4Yda#&TWwA$FGFP*hky)v<`NAH}*R|=(L7S!f z)X|p75Z|um3_+Y7mApV$jvlJkZj}z8L z!Ko!)65!NLi)ghc(+F`7E_|HSLXhaWo_L=<&u!94jhI8-VTU@YKZ>}_>od5$qZjNb ztc?LY(>CxjFB`HzZg}bi*=Ml5k;5zoZk7W&R&3yYY}jt({swGZ-Pk$l4BJaulAfc> zTOtO17C&Tg(sLAG`l9PXr*T_U^2rgxV<^wyg+tOaiv=o;rl5koG|CbTlf&Ms6@BHB zC)x;=y<)u+fY+&@z{?f@$Ed>2RyNy{P2xa z07j3XpqQP@@X(Bm%Y?rp``NawJm~2PX-K3%h^wJeQl%}m*)MHveIO%X-jqp36j^*8 zF0s^Im}{*IY8lVqM{PI!zjy^%PiGMm5!kX5HbLz%l_9w|3tAGxFIgD4w+C3KMEp>~ zpsYIs1msD#tFafWfVZ!t_zeZ>lQ7@ECl+`QCaR7i%7qq$>~nJ)J*^#V!HPzOf%`YP z8X|Rmwl9>9nl_Wuq9pa{Z(DgGU{`*wu?LGl&bh6V%evsYK)xHd%dp z?J3l2Q4fj|z#RS(-S^_8<^i&~JEfR~^~f8j>z+>}SfktpflmKkdij$F@6rOeooBVz zUsMX!MPT{%Pbz=C?Ej991Vbn>-&Bamut@tha^q!oXZ-;Ek%-UMPgIYcZfw{nx_Df%-Tng~)?wboHd7^yVN%f5TkK#BK!M*UR)daQPoQN(c$LR)mc`~@O)Opla$Xm|> z>0C?Krn@h~NKnatGR?7D$|npwY!J4E_(v+Rm3VxSUc>Ka#*ir*Ki>CH+RzY#p7%p| zqJhi1C0CJ^w>~dc-5<2PVX>*s;wFH}pEv1CdChh-e3gHDStSN>P{9@3cfgu??=I22 zn$?$id~NgVn*}cu#iI$V|3srxtcgi^VGlI2rZp;uCW zYMXiya~=yl&f;!Pau6KxVQcY9l{F~+_Ux$(FAj{}8hxozU|Qx``>~7{m+a_W3x%P( zy_2oUWl)r-@lgp?Z>#9^BmRNE18i{w&~f+{TY#@MLf>Nla&nOYT2$?&-S%7IP2>sG4@UQ!}+(q{mNg2*gD%fHas zq!wf=!_RCF>lMi$D3S1>yeVv1i1x+o0loCm?qN)Zp=u*8ED2S2jm`XA3XAylawq0k zRnoTReOWP*S$rn)Sz~JOj$ld)>EB>7Fx%+3OWHqF9iUDyU37p_4aNZzKa{u|cRz5a zYeL(G;}$g|h@-+j-A*7xi))RHH(YYh9g}bCU=@2!e`M(GO`_v}ksG<|o))UG6UtVcnj*IGLwxuUjE&2#_s zK)?Sn?SWpf7`n>`+q@-_VlMGI{+o+^>koja$yLyaP}gc)2_PoA0*)rS?Y2Kvp;2S_ z`p7=ivzDuy)tg|EZIdep6dmo!QilVP1maly%oxJ1P z7`W`LlR!$Z?hKz9s?(F{Nx6z&>nTj2Qa^( zuaWWCo0uecn;sqS5vMk7w(gIB^+}Z;6pGmO_x@~Or<6`Po!XNJIlErheiq%+;p-jT z9UGtmePvEcXlL01GlGZ232LRiX7p4nU}o?yrocfdrrEs&E|%w;i}h+#a1IA^QMY3j%2Apomv7OQh&$M$2}#o>&6Q4lL6+nwa?+x@5{$X-V%o5tsX$@ANol^` z^yTzDuzTFE7Wla69&4rsL&e5U-FOi%29vu~Ci)5c%Xzu_NEW_m&ndZ4^iD9|p^enh_GxTWy@xe|ZsG&qG~ z5}QV%zrsdtEvzT`qion>eb+V0Hfk7hL_DVibeS0{=PW#aho+e{ez{h!za`Qc{gxpL zL11n;0uo3B5(uyo{HX+@ML&6BgXktRJWDJAT2!Bz3{D)II0 z7pNr76lts6eP1^7Dl6V?o71rIt+2@t9XN`YjvkWj3M^FW& z{2dVy1OnNZyeg%SL^;KDxvxK2>HiL)L`gvNahkqW#xOWN0*xKxKVK0d1%~c6x$ZVZ z0xDuD_@aIW&isvty`l#>>n=?nc<`iy89)nV5^J|i)quop$4!0~Is83w zx7;P2c>3mix*G-h-PTfw2egXJb(c7rd#XIIU zE%;pH{{a=>4PlC^Ukp6cld65icf1Fd_f=>1Mc_ejlEn6ZKX8^9&?GAF?=K6SgS@%m&(`z5*;fi0>T27p zo0IMnN;D;PrR4nm_8)%>*#QFYcK3}zP>Z3@z3@Hr-<;|KhUW|S7YIkebvLtL{ikhU zC83T_x;>cA4H}BkazJqUH^D#r_RHb^a=3pR^N+v%a=5=7?vFL&Z}0NU;r?>Ce`~`J zzx|4Ee?_>^Zsb>l`~TAjcM%-Vj(BL_H&gX-tjIVl#0AY;>!WwHsL?c0nksRj|Eue_ z`tBOPr`a*8A~(%)%qZgBes@1p=&?(Ns;1`fr8&L(?f= z7<_J9tG@n;-u*#JBv89YpH@3Lz^}7ULsN}S2jm!Ucw?Ty6AQPT&*D5JyY&Q5iJ|Y( zm;vHH@Q%Ae^SW0V2;1lFZPlyb6(rFSP!>Z#9ijvVbzM!4EGLpMNwJ8AoFnWL4i*iu z`{yz$Wr8ca%-!Jl4JZXuu=kA zSzUp$PTR1z^}ePS=du|7069lu9Uz)D=Jum!o>*nNp=bWXkCXxbF>^omLooPs(^9<( zt*qAo-I2KhwgF&YWC`-8zroM|Lx@cxarynJ#l6@RhassZxTpjiLRFInkk1k>UmP^A zd5D3qucX{n1+TdGc^34lLy2-)1q>SJVw#eTP8el6Yy>&S_%$%a=epLbn8XrINB-P0 z(4zSQ94k}F#v?c2M;51mJRwQp1ZvmhJ(^$ueyxQG`BN-n86Zc|7W2+K=!9gp)QvzKpcEW_3q=Y{9;hAXLx=f`z|#-FpniG^4TA{Cv6J(%EwPyQ8SQt)kW^D&j|C3F zCA0_--+$Rk7@F6Q8Gx{F$tP!kSETv3$w4b?Qz{r#c(+?#`z#H2PjMUM97QL9Xs~N0 zNAMYZLk>wyL!c-58hGn_PiB6Fet+81uh8#%SNJRR`^%>odx-yaN5JoY`6~?kSKRR{ z4E)oV{LY$RVc@@11pQYS_?J)l{mZW~@VhwWR~Y!6h7kM;1HTKZe<6V1hibnN!0$Qo z3jzFBBL52k{N9iJLID2>K>R`gzta%WUkKnoHRRbZ1n~Rt>=y#~Jx6|R{(hI!{M!8e z-mg9r{k8e~9a{8j^Y=Tr2=;69_q(j-*XHl{d#b-Sf4}3%{|B4Dy*=o&5)%8n$HsZ- zbiP)u5`sI^>_*?w@qJ9tln3@QgCbNy`l~My?n>5U#$E|oGZ*uI`rCvXp|!XKQd~?h zKc0pp<$i7Q25A_b23wuQZ-2LXr^nSQs~f=xWD5mxJ%epq2(Z z?l(20YBilwlq4OC^W4b9YuTT#f9Es*`4|J(n-*WUkDBwtn^7TjB@>V3P3MK?kDwRW z&d%frMSHQ4jwOuykcM}P1^S6zRJDdw$d`L8%FU@XfCK(Fh5n;-#1#~;Oq)@)bt+tq z^B?aah?=C)%gvck){-eqv2|RISVCed- zz@vNUaXy{b&3YLADYbtnJ@}_sbx`DK^*K@w#qv!^9x(}eO|CyDD|~{7*>zu*4JV5qwV32JRjPAyXS2v z9S09h3LRmVsf3)951l?=;6;DmBU7@&Fgkl_I)QLHH*>l`>2O~QQ!-83ur}G2Ca-2I zb9tkV%9Fy)W3!moj9WYU)ori3x&_g6yZNBvnSm$Sql`?@)jiKdlYvdWzRfdXQ!-|? zza_hHY$He8j3X5mI+tqtRFTMf#F1Dwo#z@Ahrl(3NRliSBdMk7OI`e?&V}+(7g)2T z1CHNlq)Mc7Gu~#+3Z+yAp(4bc$5)%s|ruAO3VQ`Jxa${7C$zuX<<;!MTUuF;Wc$;x^ zLaWEMd*H}P@v=>~!H=Q?(}T+8)P7r}G?P~e*FVLrmL}WXmXCComyfjHnnz5hO9ve* zMsae5PvEVNkzno!@gG!pAZV44t4@4AFLr_lIvi*xHBITp0ek0tk#Z}@@PD!BdiOp4>XM$2}$azDYTgx4ZI#f30-m%;s9!XDlOCSj&Qf^Vn{+O z`DMG+r07~2DDU7=$VXT4rF6I}%E!2k%&RdHPoyGM@el0m*O$fijthe^qi*`aj$OSrQTxhD4ibK zK0g+79(uZ+r9e#sD=Pf?spfU9)fr{SLQ?XO>7*Lh6apiqV~Dtdou9P6bx%-V5Z7ApWpR8_$EwQ4=x&a zA(~mCE*+wRIo80)O~sdy(y{Lff;Riwv!xM%+QX~<6vO=z5(oEehG`~kWTTKr-R-X( z<}wXN|LMZNe=oHNm=!qX>7fH>X~rQ*vrOgHK*u(f&nfL%#09~o!CxB-CDRSRJdEp- z$B^>bZ47#WY0y}AI?r+-g(^5!LM!~uB$q{tOZAXEC3IQiGtsLkGiR7VMpoRna|ljq zCJ9ZM6X{!gx1ZX$4E`%rZ=9(-Y%sAzw1Zdl3ryu@IUGCG$v#zKlohca+K7#9cq<+O zy%40;0JnY=xMcrnJR`fsfDV2sl!ck9o8MHURlAgBmW7X5gI+0TxU5afasFehqj?Ly z{^4vNhpiPNoOof8;dA5gn_Zp0)*D9uQGW-Pf`n6uTKvOuUA5cYV=he3t~)J`@I9Tl z$>GrT6_-z8s<_gv$6|ecR&(lO^@JyqCr3U?rWk;8|| zSd#C>{6B#c_@^iZ6#X}-m`*ntCDU_KR9LXvjKCeR@u_A{IcfLD2-jDiCM5NeNsG*l zbm@xhlQ>SpG~XkmgU$){O-pxQU7#u{*`;i<9N?$@*LKPOE|H=ZfH9A!9&9q5K4NB* zqd8`w(Z0(j>@cBjy=9|Ao)w-tfMmfiW4~^zrzE)X(E$jTN>E+!Ylsx`p-bZz_TFP| z5@;h2LDexLm-P$71+pu)>SdGr@kO?I&h;3v2RX^L2b&fIwW+5LrAz1`+F$4o`$X#_ zf&vqRBWkiI#+y1_+EJe5btU2wG=sRKfR+Z`xG^|zw39@9La#98d$(npZ+0)DVdQzs z)82&p86<<%PIiK>Cm!`pR{O)3x{pGyAs#O+?>{Qqu9W_#Fuw~uVU?&_Z9Oh8w;Wxf zWG@dA(~XR$0tItLpE>A};)RN_pb(Lj2UL~sB}+Xk)qxp~FjpEHqayhzvP~sNr&i$R zPtW@iUF6TDFU~&ESazMxP7OlZYD#wuUt4sYgZ6dtI$-GSFt<15UYYmbQPeGrNfFJx zqSCaQWY%AOU%)kAAF7UHK4z;HJ%E>g=^!ABkpxGJ@mMU8YUX|GS?C0d3Ik~(m;1+kiUssS({JMG*~*w5!xQB zcFi^XGGnY`+eax6?Sf5+fd%nfF19b-XR%-mbiG2O-VZ`G7XD<~ln$YU_Ee^^CN$J4 z9KE_4oZvNs6f1XjOtSsD5b;5)r-^GJ1}a2`&7zttrm`^)Ds*Bcp4v&YM)TX;Vc{?0 z0?}bf5IvJNG3<1(Ww3(Tq5fF5!c1s!a%gpNRpRcFWMl_9B;mv;uccT)v5e3lRPrX! zpY2L*$V)xGODCJIn98Ur0pq6dd(nTax$OLCC44Fw+!j2qtzNI*2H80wY7HFLVr~np zn#58zY^!Ic+lG|wB?i6rgBL2V%rC%Lm!1TyM5_e{MW-}t4hyKcr>Gn8(nC7$K+t5Q zJQb6*uvE^e5ElHVg|cc-lEZS{iAPwh%!qifczt-isUccgTB&Bgw}p!2KIkd<$psFiEMIf%p`#(?{6C@I;xY~EpKLinC3z2 z!$Hm`AR6m--KLt+7R-kEoz?E^wnal~Fae({Fg>PozZ=I!(MF~;yZfD1O;;Tf)k^XB z8)(Bf+yyAVI&PZead+bk2{_m2M1RSE^gE*Coxo4vbuIsH&B-T(gqyQ$S~m_}pL550 z#F}e38hGb8!^d~jocR)2>+hIJgb$%wpe$@IICK%}FE|d--D}H>G=!&fyfA5@`M4^VZ}T9CNdf zKyLI#zSM8NloL$H_~*dDhH)KSId)ct?g`JgSWpN>^zTaaK`%&awt%bHmY?-Jc3;p= zj%J0YG=_2MQiTV<)trc?q`hv9hDkX_dHKnS{y7TrqwC(mg?3w&#*>N;|IyM%1NwGK zF;2T_*Sw)?p8FEa_5|U-V55{a&KgY6B&FA}bErQhikP1;jXHKlEfPjOekzcZnn(dc zNgwbC`Mj3|IVnA-ss77m^AMwndX2;);n%8MY-mV5>{O&O|QUs zZOCwi3?#PTUZf|vIfpege&Ak95!wsF4o+U#>cedex%;uI>x2CQG~J~|Sp{=*QC%Nf zA(d1fw3mD;c#FT|;K4)#cdifExBjGF!@oXkO#`bVE=6FT5Q9ZhBYR^3EMz4VMmKZ* zBUUyB;f~-qo;k@G#L(KoKG?X9v}kKs$@SW7Uv84Pd0GdKDZ z-M2SoR(*ZdaTHx8TeWTN^NOEDr4S8PFBA{vA7{=)I!FyU zOKHxL@^X_rdk86Q!64S|UvpXA;THqo}oWxr^VaJX$<{0s{aXe%qNR zS3vgpQfA7(xII?s22%OV|Ay25T4|{xfNdJ|KW(&&h#k~;d?`RC1+3d#MpQkqEy-4!e?J^;4KHUJQLpo%?#%@&3uc?_UxN@{NZ9?NP_8aPCPSR4n>lh27c) zi&#;wXJ|>kvAA|aIzg+TFfldL>NGf@3}Q4`D{x4$TD77l{w-lLBD#+w$?0P@Q%qt6 zX*Xv~o{46mCLUy0;(kNQ2Va7$`SOs!efNZZ<21;qS=0ugloItQg;ihMX3uc8 zfHIs2p53hLZV4F0mO2!g{i7ENn|*9|n;2I{t1z*Jawj0Q=$piSFD@gFtt2X4hL%6) zO>Ye-&ZCh_QRBdH9dsxF3c^Bl^etd})aAtmsXh!TqCJ>|{Qzf)z1B=Iv(0vMD*x?` zr9u5CL-)`du$D*nmCZ}1{45P+y0-$7W2zj`{-Yn86)c4Z6)ru#1FhPQ_gVNBNX;H* zoPqfe4Cb$n8GyV=g2=k9J192W$3!C>(xRtz;?upbPzjkL&5F|PEbEPqs;eN)?l}6y zEfaqq?H1-vebvc!q>Qu=O<+=DmBGw*QAQTD#}s-K)sxZ#43S>sB#Y;? zs;_9`*^$E|rN-JL0Ry_g0g;@7Ik~|88h1!pG)0K^oFP1OSn=eg%o|p|g->rQtkkQE z-(N;M&yef3xd=Ge;47n6Vz(751d|rhrcIM$W4F@HptFZrav>|A%fjH z|G;6%7~luew6_;7qmT(Z6HAvUsNqhbov?Pk6Z57Q!F@Q{BtFEy2==67=(bJvClBK;lz9X zB&n>31;(Za4c{C4Ka4`p6C;Yv_qyYezoleUY|VJRTK$NmvC{RDDXeKVy~C7Y{g#6q zDY|9_3(_1{S$BcHOAxky3eD+)Nd`5*c2?EuWkv=8tRuLKp*HQr?o1ov_fcr%G#erjf z0QyL#5Ypb}yQKiBWd5w3++yB5Zp@0{+GPu;n2w|U5^0dIeujToz`Az@1xbu-#>J7l zamj|OMA&CnR`_%3K&IL8ei%{Le6J6V9ZmQ;?xJ9_PjfmW^~}8dFk@88w~bnOTkzeI zvJ&TnD*wUuf2|x=3}|h{5{`T{v#d9#wmB^NtzYO*tD`Ag&2^ab%9{vce`5^q@9(ma zSHZP>069-Q&3H!7Z{{vXbA$vT*5n@7nAM;OHUUHV$DLf8Svy z3_03k`cBZ^$ur=f8<&#jS}RM6LO>p;rV_zCvCYRS`HMePaD>s_XRRu2bciIj?P!2< zYs&brPnd+OrK{cNHRZCFXu|lmN5=f_w!CuY%aDy%Mx(7eG+1Fh;ic)mNOp;TAwVu$ zTQ}_#?yY}u>wmxx(NT14Dv68pOYW9o^_a#TW-ldEb?)-YJU>gU@aN(s)3fKb(;kaY zVj;W`%G4`Tq&>ynD>`L!SVhh50EerUZjx#FhpK{I%+Im?wvb-D zS3G`P%zJg0UbfP*-IL1zc#pnMa=0iB*@s`A1pXtUCU)ZVN{kS~GGhW)4w7f8D9Qx4 zi_UeJdMKbUaaBn{j=9k2(xk9-8Lm_A&u$21W73u3s}7&IVJf%tYbNnDx{B0_ep5jaWDT&%LO_{>>i-9K_2OQ%jm zvE9}?O{AX!G~0^Mo`WUo59n%4`^?R4NkqV1)4N4C;IyLSvM{jUm$x$;9DF=n`-YLL zy>dQ7$O_s&UXK->@Wk@UEY~HZwVwLJtLZJFWA)1X#WaS`&rv%a8j1N|psoBm>=Opz z>Z0!Y3);}*h0vGoY_oBD8=hG$eCYY-2g{F_*5^}B{O{6v*lIL>5oyUjPY;>ngczg7 zJM%~8K;g2;{`rHG+@^UBFBX0t?ntZ&Km^bWr z6*J6t3asr498Qa&=g%xqqe+tT(hGg3YHEO1Mx&y|vvhD@s;tVec zk>C_0P1sYc1ac`~?xVxvb~tZeart9Nh7pU!D}UpDbeQkv1rbY0`M824tV52|Fs-x8 zPUg4FkrQ8JXsbuzT{iC7Q9b7%O}&hO;XXk^j*A?q3co@t!wT}vf-!*%;^Pbr(F%cVh-1DVN zv%!-5oW#lMTt+eWaFJzkNZj)`SRUH*vos$TmiUDf!UR&}1(*~C4|y#je}nO#{ZO-l zFgSrzNrt5uv#cyv6(}|`K(8lW*nqa{r5TxAQejI*twL8~d*FTXOPx1hNF2fIo!>Tg z@7!iL)r1d6oCMiHI**SM-v;X}nPim>b9V-7o&H2~3w?K|I(pP?z0Imju+}+9{0ZZw zkil~y<%yJsfw;JQERJryI+3)7VU|@GVT`v5J2-3Z^2i0*Hn!ccB2r0zrvJmsszb_P z6-6DQ)OVG9Y)jRA?EQ+3iI4fzAM?`Vxo^;Y=N{F%wL}cQH7|BamOM3 z?7dV-f78i;;`O=T5tZ`sBpid)Plkb$x~0N7<>ex9_nElH>`|9E>tfQHwbs1|b~o-I z;m7Sgu?rOw!|R^`ev=h#`4W@L*>MXo^IP3a&Po6?yqm(tsDtI5dLnM&9~sTl;kYMD z^G=$H+c~gW7gmRj5Nu;5j>MhtObi#hEv7>G7 zzz9}%xc^&WV_CfPmt);>Z%*!y*lsR@v+FfR+*|srKZQw(aBS@~+Bf7>&m65=4tSVt z)#x6T;Wzl}?tkv`&?(rdHgRzum=~eqLNx8oV|wroDGG3`?o9~YIezs#=cI90`eY~C z(pA-U=e&oOYb&`@>0BOe-ueq>TG`3VAXIR23exN&gS7QaDEquVSTkAdYy0)A@zG<{ z?$Vi8ZEn-mZifa#NYY>$_;8q0!|G6y4bq?Kcg5+fkXp$}#2`=bF#dO6Id_a4 zPPE(D3gC<09o$I0P}qL7f9DzaGO{+K6yRPYo-9(c>Ec1SYdJ!mBjhVu$C@5Qak~3sy3uPmb8+Webl2ZIE;t!Cs;9=0Nt1Kzhg`8=9XhY( zmd>8N7e{r;ZTAZ^+sk10wU`%LjW|W;FJ6LEnOW{=(+~UYx*&kA z1n449j}AFh*6~bq6&-uiV;S?Xq9=c~dlMcbFl}UE^5_xWYWd;uUWX4;6~NdoNncLgUC& z#|S^0>WIAMar8M>jj{C|2T%23mQs~op%U_FV5-hz^TauOf5t8F4u}1{jwSM^abR1| zY^u=OAl9ZjF_FtBpeAxp(qV7rkI*;DX4(y5^*%<-ud6_Qv}q^@Tn3$3w^xp2B)N58JJDEc}t9 zz*yamis`6WUCrYaD*fhUQlUBXs1%Obt+4>NL;MC@%fqU@ZsRA*1IKiSx-jlx3bWM} z^&2*i|#xTpVT;5 zlUX&zo(CeO9At>n%(=P4l2MAU=46Sf)#Y;?9?o^D>t)d=$eYH9DofSM#qLColl4t1 zza@{8z@w1_+eY;KQEB}ZC6S23DfpMghB@JCH0bIeNmk|SFR`-P?1sbWuEiW(M)-Gc z5K0R5!c30G6YBEjRn;2HrElyeofL>vmm6Pth%_~bW1P!tENML1&+KUAjefI>)E*^? zkg5~$$e-I))7m6mk2%@9(}A#25^y}Es%GSjl<$c1K!#&ZMlFdD1&ib*&UHi&y=mAo z4aa=1aFh0Z=9H))Qbad^J8(snC-BTfHYCP4uJ)&ZLw(;09nM$=ZzP-?29M2-dZ6Kd z&^kY-`*bCT?j7GltyVsRrytaJO*+g-g522nayXC8zN+c(vU?#>N&9$h-jVT_EW?X-4u_;B)B?Wspib$J>LTJRG%5wH6)PN<7&-KLvVAUUPSQy@gRgZ2@#l3bH^h4V^7 z>CQn-z=(KVfk;8SpiuSJ*UM85I-6fkEy@oWpuO0&3=YIUmXn}kX3D(WP78@&{RW%D z0P6p6JHMPN#T2=-717t|o5v;Z^bF%^&YcG}>+7+I*eVb}^!iyh6v|q!-n228@7&PU z)K*h-n0|AqU$~O7_{lIyIDO280%OIO1x7K1hK9+fM_zB;gq^P7{XDCCLn5p`c$VGN z6_*5Z3C1F4|E5{2b|v(LH4X=ry!G?zHVPmo5GER|FtZTMn{^45LTakoO}%z|B?S#b zo=Hdfl)|E3H%6K#Zv_v#)86KrgaqCCioT&6@3!PWSbUtVl~8i;K5sSG3G@9ckd}DF z_=n0<$%XA@uJ^F}pY35^Rk~FMZgTtB-nLtx-L@+g)u5}^$L;UrwVCjjlf;mNJ!e9ZX3=71 zMib3Kq0Yh8o|eJd>$LFl<1-w={vqPSVP2_6-7UqHU zK9F%D3*Xwohc~d*Inc3X_3BRRzNE#B3!qaVZOt)tP787F!GR<#l?_Fj*{ai0+A{pe zC0yRLwPM~am5mnd6j}#!?krqMI%>tgbv`t4P8+%H_7-Z1AfUFYLaj)k@Ebag zg%3FJONO_kBDqhYRk3tBJ<=82*kqTCYS_t$YG|$^*WrWIHEhxN+zW&ni>rz>Y_Z4r z2NyZ)rA`G% zS~TUsee#jb2agK!_6#F<;gM>e86iPZSS0~NnxuE}jGDZk`NEs9?kgN~uMy2SdXfYT zbTB)SLoHhe3cn^>g)A-C#(wcd1$Id4wt`U9P0MI#E>gRVA zZ+1dUBU__Zmb4I9MSTAvq%g7Njxh2SC|Y~`~1 zMJ0=U$wxH%O_hs2!T4HNIib_Y4m6As-_GL7RlCr@3ZK9zGGn&N8*wXrQhLf~1DLQDWPlS2i86?e51b`lr1_*kxDM+{sp7mAyZ^X{UUZxopxy^ljvMQ7 zGw(;V)JOMfRnE-;lpcaA3>M`O5Mi)x%Z15k{=u{2XN{>mXh*bb^IwK1JF8hkrS590 zj^=*w$o~SJKIjoMv`TvS4{fxjOzV-|;hZBu1cl{Kcv;_gd@Y0&Z8QecVAM_5m+rlQ z04;@%YF}?C>2pR;D2;~m74@R{_=oPw)t|s5oZ)YCsX$<&uoQA2M`CAfad!oiW%O8{CYBIuOYzD32n z*JNGwTfM)$<=@?i@Sf5~P0nzod$6E@byvd&UloK1`EOabAk)l`emKNPnok+YYOPFd zud;(>^jDvIkVS@Fq0gSWjV8$yvLZ}m=5c^umU=}(t=MeGvjs7zoJ0(L0`vG)Y%r*J zy8#IG3{A;i(sxvIGo`RN`$@3TE4!ktXsXp0Z#zgv)o)~64V3d&5^aPqQFN?#A4Knf z0IRrHH(bJbIQNdqRO|DB*blF5(EX*xyIz-kPH@wt37j6vSoEe0naP9)^y@rH| zrp|78Q-gN0VcpcSTBS>Ft4s{~fx*3^Rs+|dx^qW+v?UG zJl!7jr)a@Qes8rl^c{uHDTL$opoq#@?$bXB?cuQfSL-t^t}hJ%FZzHcR2kUrK{@;6JECe;n% z9wsrR>r?H+Z8_kK8-y4T9gFyd=pA;NX{pNc! zu>(Ci!jg`hCm)_euJXUZ-~_q8omwu6QFNJ4-*R2hI$tLp2fv)HPtarr^KyyvRmiOF zQZWci!I>T47J4;+!5jo>rDC+oTQ9N7WQ&S=M~MlkE(#ktxAM(pi2Xl>U3)y!`y220 z)rq5$Iz_q9+;f?0YAz=mRt+g3j94PKV(z!3G9he4BMK3@q_Bpe!&nE+<>VSVA@}97 z$hF^R)%o*$+v~O0{@CmF`EJkie%{aXyq{;^X&`UyJ8Bz+6$n4RhuxofJi_mi|L9J0 z?Yn-EUv+O#)phhuqMKK-vbnKq4?0id2TFVk_R8BtEte=c53betvQG4+ykWj6EV?o+ z-{-1j46y)l-LRhD1{Fl9t$e&ifp-n6%Zww|H)e7x!tnJ1Mkjc}l{?;+8vqUVEi)hx z>^W8fmza#O-v2}l!0mfRyec~OgPaii6aI+G%-QC?*6u~>ko{IZFMr){sO;8b#{XW%$DiCu@B zL4rQBJ|6Sp)fJ#VU{R!qGG9!$ z2!D;;5agIzEs5u#V3tXJ$?Ei>WJ1XuVkh}BT}3fJOD9Q#l+mETSK$*ZS>^R5LZx|F z+C-gSh(0x#m}O}g16kNf&hvm{BX*k&E)LAVGB!NeEgG!wQ@g};`x8)39ekq0^~&1% zeSRV)_%7->74K^Q4y{K{P!YZT*HnN>Az^E~Q$uO654Rsl%Q4#gA|ow!pDzc5BO|MdN;vdRpe~m4#fB@BV12lfsaFJ=!r>dVb47U zTj2H2^B2bH2W=!FPvfn<&i!mOy@7kZ*=hLhuNdi*vh5*W@fjg?B6kdj`H_|5TYYu^ z>`X=x3hs~TC<^HylUC45(2l+V9DS9?#nY>FGa8QBQV5*l6b%TUK`d=Iqup$1;1zWn z5zes72q6`?ufzf{5R2P)A=CQoqxLyt0U(#zCCexRH|Cd|oRrC)=>) zpXI)a#f;9k!IEj5kDz=JSauA!51r}u(&VV?nNJLXW5fg(@-OD*H^=X=HX-57I8SDH zIW$9xt4nv0Pcx{nd}(!OZHI!;*>b5+WC6I21tNQXpLc9(AWkQzczu2d6!d3CCz>mQ zeS-K398Ny;*&*(@MDL?|c(HHoQaX0(HtzZSbGuOSxSw&x*6-YtDl8Ri zVk|-}VhxSBxW6Lpd(e+++nL6584RIQWV~_j_=KIe47fmC21n!M$p zlG&@t*;k~-5Vw}?; zMyk4lthR02rHVoQaqd4k$Z!jQZ!-E&<-Z<%q?CVTI@l^&Ac}l|3uh*sB(;L+9=lyc zxviS*d1L}QqC0v4d9Rti{DC=@jVB9!qa0ej2!=G3>-p!w91zl@v6Wv1*8@tztLfiy zcVgDc9Ex&zf4>WCpr3ikqB-a)fdz3!adZTPtlI^5FU}*%hD_y%8 zt5+|fehSgIJ!`->(sN`!+T~JY!3#XopI@~`i<1JO!2d$28H#-MNBlhIy5B^@Q zGpsl^M=2S_T0bPq3UN4hTCYqr^SPJaRRN$KmDO-5RD5lN9m+Zngrlbl2Xh1$9fDoCr%7`uK0Ogv@CB zCf&5A45t{|-&gYbl0JpF*ertx<4gsgFZ4A-y?;L}=RA>LGx|QOT`x^ekQ(S(qyO^# z;&JUvW^gZbdT@FM(gU+Qw(^_w5{865NLDK6_D);z*1$wWWQmP!3qQeMl<1JS2_K;9 zf8jXFWpoh*fR-K>4LwcT{+lg;ytp~Fva$-!+1*}iY-^J~W_aHgj3 zh~}VwF*BokbIS4jTP^=FpSOyTOVQU2Kk))S6>HH?cwS9g&NL+@iibKF^(S>+8>p~G zF;dXs6zlmk*^WAYG;;MzcxN!)hxO{6hcFEL)ina!ZKSG9dJ2^d_DZS+?KlG4mD3Nb zqKYBCM09aT_Wk3H%?$Sf$Gt9cs-h@|-*JUC)RK6)>@UFt(U563Ka6i6rCIK2k;uIR zIO;LTW%}z4`~aK6Hv^ff88gF~n67%FX^42Ng?=%t8LMl+t?qei4%Ny~L2aKk5E4lc zy%Jd+OF(O5w?4IAIb5$5RAZ?N0e$e685ZzC;p*Vp ze{KkCUBty74m>Y@?tZ<@Z0K}yAJq>#+fO?N>vFlpjweQ*>>Be}@;KZx?(_L4@NH-S zWxlY@SAEv-fMp~WK2?c19G57gq#$xI*DPayuG#V9#^>9nmk$4GbYpW)#Ce1)&{8MY ziw>N4borzi$tDX+9xN&nsY3{_Sx7^B=vQtZWoZ5|d$H_72&0&kSD}yT8X)33%PY8Q zKAtfP8Ws(3wRkYZ*gV(SkUzyn3WF(cGj~{1A07_IaJr=r322HecF5hjk=s?PJxl!y zQg7ZC^$HOC#jEX4t&6<8O7(PL!7ik{GJ^;ODWR~wCxwWxHjXgjW4y4o>jk{}B|6FP zVKn~5gKKWA(Q8gDmupv9AHf`}^~K3)Tm3I9L{1H8o&@#~=!?(}bb7`$YcYA|aWyDQ zSs3yYGZN>y>1dZ`&e)d@8slc}hysEzmdBk8rm3%ZrQbC&XZ*7{2+Elfy;D0`IHW18 zS~|9-OOvL7$-NYTZLF>>KEV47^gfoQE9`It?}g99qyFOY}Dz{ zI%!P4=3g;)W1maq<`S4sNEtmR=0Y26h@D)4!w znV{KjObI|&)hPLvSuz8}V%epQCiM??=006AB>2xJmP)VZEHQ^CkJid{dG;;^BpD;K z;8UCTHJ6q=W$9%ahZehUr45BNyo6?1L(nCHm@Dylh4!(FV5E3jPu(Dq@)X-$mAc{g zMcgwVz!C0}6l`~&I0l)SH#Gn!yU(!A>@*%bJXI7LDfgG0PKK1nf8k-MfD)tc({rMmHtFb2Jm=U#BJk zIY5xsA|i_EKAHkAH-PnLz^B-rhs|XKkB8pd1M8=x;3dqfUHsHt|IS@f*;=c^UHJog gAk4dWa!1rx0avvWt=7jm<=emmgEm8zo^ZPHfA+x|3;+NC From 18aa39f892ba0fcdb7bedbb29bdcd46b43fe10dd Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 01:04:11 +0300 Subject: [PATCH 102/156] fix(ft): add emqx_ft_schema to ee schemas --- apps/emqx_conf/src/emqx_conf_schema.erl | 3 +-- lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src | 2 +- lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl | 5 ++++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 1a0f8bbb1..58bcf9700 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -63,8 +63,7 @@ emqx_psk_schema, emqx_limiter_schema, emqx_slow_subs_schema, - emqx_mgmt_api_key_schema, - emqx_ft_schema + emqx_mgmt_api_key_schema ]). %% root config should not have a namespace diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src index 324e7e308..771fdcb27 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_conf, [ {description, "EMQX Enterprise Edition configuration schema"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl index 5137574e3..10c6c0379 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl @@ -8,7 +8,10 @@ -export([namespace/0, roots/0, fields/1, translations/0, translation/1]). --define(EE_SCHEMA_MODULES, [emqx_license_schema]). +-define(EE_SCHEMA_MODULES, [ + emqx_license_schema, + emqx_ft_schema +]). namespace() -> emqx_conf_schema:namespace(). From 2f1970adbcb21387521cb8cbc092878202804676 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 13:36:34 +0300 Subject: [PATCH 103/156] fix(ft): make ee schema aware of ee fields --- apps/emqx_ft/src/emqx_ft_schema.erl | 5 ++++- lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 697220e1e..270543bc9 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -35,7 +35,7 @@ -reflect_type([json_value/0]). --import(hoconsc, [ref/1, ref/2, mk/2]). +-import(hoconsc, [ref/2, mk/2]). namespace() -> file_transfer. @@ -217,3 +217,6 @@ converter(checksum) -> _ = byte_size(Hex) =:= 64 orelse throw({expected_length, 64}), {sha256, binary:decode_hex(Hex)} end. + +ref(Ref) -> + ref(?MODULE, Ref). diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl index 10c6c0379..8e2800c68 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl @@ -26,10 +26,24 @@ roots() -> ). fields(Name) -> - emqx_conf_schema:fields(Name). + ee_fields(?EE_SCHEMA_MODULES, Name). translations() -> emqx_conf_schema:translations(). translation(Name) -> emqx_conf_schema:translation(Name). + +%%------------------------------------------------------------------------------ +%% helpers +%%------------------------------------------------------------------------------ + +ee_fields([EEMod | EEMods], Name) -> + case lists:member(Name, apply(EEMod, roots, [])) of + true -> + apply(EEMod, fields, [Name]); + false -> + ee_fields(EEMods, Name) + end; +ee_fields([], Name) -> + emqx_conf_schema:fields(Name). From f6461fe2872c66861fc086ed0230d0fe544e6d6d Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 15:46:28 +0300 Subject: [PATCH 104/156] fix(ft): fix typings --- apps/emqx_ft/src/emqx_ft_storage.erl | 1 + apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl | 10 +--------- apps/emqx_s3/src/emqx_s3_client.erl | 15 ++++++++------- .../src/emqx_s3_profile_http_pool_clients.erl | 5 +++-- apps/emqx_s3/src/emqx_s3_profile_http_pools.erl | 3 ++- apps/emqx_s3/src/emqx_s3_uploader.erl | 5 +++-- 6 files changed, 18 insertions(+), 21 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index bff149ce2..79538a9c7 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -44,6 +44,7 @@ transfer := emqx_ft:transfer(), name := file:name(), size := _Bytes :: non_neg_integer(), + timestamp := emqx_datetime:epoch_second(), uri => uri_string:uri_string(), meta => emqx_ft:filemeta() }. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index f84189f7f..ae8315510 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -45,15 +45,7 @@ -type options() :: _TODO. -type transfer() :: emqx_ft:transfer(). -type filemeta() :: emqx_ft:filemeta(). --type exportinfo() :: #{ - transfer := transfer(), - name := file:name(), - uri := uri_string:uri_string(), - timestamp := emqx_datetime:epoch_second(), - size := _Bytes :: non_neg_integer(), - meta => filemeta() -}. - +-type exportinfo() :: emqx_ft_storage:file_info(). -type file_error() :: emqx_ft_storage_fs:file_error(). -type export_st() :: #{ diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index 2ba837e66..b6578a7e1 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -32,19 +32,20 @@ ]). -type headers() :: #{binary() | string() => iodata()}. +-type erlcloud_headers() :: list({string(), iodata()}). -type key() :: string(). -type part_number() :: non_neg_integer(). -type upload_id() :: string(). -type etag() :: string(). - -type upload_options() :: list({acl, emqx_s3:acl()}). -opaque client() :: #{ aws_config := aws_config(), - options := upload_options(), + upload_options := upload_options(), bucket := string(), - headers := headers() + headers := erlcloud_headers(), + url_expire_time := non_neg_integer() }. -type config() :: #{ @@ -62,7 +63,7 @@ max_retries := non_neg_integer() | undefined }. --type s3_options() :: list({string(), string()}). +-type s3_options() :: proplists:proplist(). -define(DEFAULT_REQUEST_TIMEOUT, 30000). -define(DEFAULT_MAX_RETRIES, 2). @@ -171,10 +172,10 @@ abort_multipart(#{bucket := Bucket, headers := Headers, aws_config := AwsConfig} {error, Reason} end. --spec list(client(), s3_options()) -> ok_or_error(term()). +-spec list(client(), s3_options()) -> ok_or_error(proplists:proplist(), term()). list(#{bucket := Bucket, aws_config := AwsConfig}, Options) -> - try - {ok, erlcloud_s3:list_objects(Bucket, Options, AwsConfig)} + try erlcloud_s3:list_objects(Bucket, Options, AwsConfig) of + Result -> {ok, Result} catch error:{aws_error, Reason} -> ?SLOG(debug, #{msg => "list_objects_fail", bucket => Bucket, reason => Reason}), diff --git a/apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl b/apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl index b4e640f7c..19e4d2ddb 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl @@ -20,12 +20,13 @@ create_table() -> set ]). --spec register(ets:tid(), pid(), reference(), emqx_s3_profile_http_pools:pool_name()) -> true. +-spec register(ets:tid(), pid(), reference(), emqx_s3_profile_http_pools:pool_name()) -> ok. register(Tab, Pid, MRef, PoolName) -> true = ets:insert(Tab, {Pid, {MRef, PoolName}}), ok. --spec unregister(ets:tid(), pid()) -> emqx_s3_profile_http_pools:pool_name() | undefined. +-spec unregister(ets:tid(), pid()) -> + {reference(), emqx_s3_profile_http_pools:pool_name()} | undefined. unregister(Tab, Pid) -> case ets:take(Tab, Pid) of [{Pid, {MRef, PoolName}}] -> diff --git a/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl b/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl index 73774624e..944f2037d 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl @@ -89,7 +89,8 @@ unregister_client(ProfileId, PoolName) -> set_outdated(ProfileId, PoolName, Timeout) -> Key = key(ProfileId, PoolName), Now = erlang:monotonic_time(millisecond), - ets:update_element(?TAB, Key, {#pool.deadline, Now + Timeout}). + _ = ets:update_element(?TAB, Key, {#pool.deadline, Now + Timeout}), + ok. -spec outdated(emqx_s3:profile_id()) -> [pool_name()]. diff --git a/apps/emqx_s3/src/emqx_s3_uploader.erl b/apps/emqx_s3/src/emqx_s3_uploader.erl index 8327462c7..595612f62 100644 --- a/apps/emqx_s3/src/emqx_s3_uploader.erl +++ b/apps/emqx_s3/src/emqx_s3_uploader.erl @@ -48,7 +48,8 @@ max_part_size := pos_integer(), upload_id := undefined | emqx_s3_client:upload_id(), etags := [emqx_s3_client:etag()], - part_number := emqx_s3_client:part_number() + part_number := emqx_s3_client:part_number(), + headers := emqx_s3_client:headers() }. %% 5MB @@ -252,7 +253,7 @@ upload_part( Error end. --spec complete_upload(data()) -> ok_or_error(term()). +-spec complete_upload(data()) -> ok_or_error(data(), term()). complete_upload( #{ client := Client, From 6e50b168a07c1c7175009b335ed82ca9d269d15b Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 16:54:45 +0300 Subject: [PATCH 105/156] fix(ft): add empty strings for zh translations --- apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf index 99004d62a..c26fa477a 100644 --- a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf +++ b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf @@ -2,83 +2,103 @@ emqx_s3_schema { access_key_id { desc { en: "The access key id of the S3 bucket." + zh: "" } label { en: "Access Key ID" + zh: "" } } secret_access_key { desc { en: "The secret access key of the S3 bucket." + zh: "" } label { en: "Secret Access Key" + zh: "" } } bucket { desc { en: "The name of the S3 bucket." + zh: "" } label { en: "Bucket" + zh: "" } } host { desc { en: "The host of the S3 endpoint." + zh: "" } label { en: "S3 endpoint Host" + zh: "" } } port { desc { en: "The port of the S3 endpoint." + zh: "" } label { en: "S3 endpoint port" + zh: "" } } url_expire_time { desc { en: "The time in seconds for which the signed URLs to the S3 objects are valid." + zh: "" } label { en: "Signed URL expiration time" + zh: "" } } min_part_size { desc { en: """The minimum part size for multipart uploads.
Uploaded data will be accumulated in memory until this size is reached.""" + zh: "" } label { en: "Minimum multipart upload part size" + zh: "" } } max_part_size { desc { en: """The maximum part size for multipart uploads.
S3 uploader won't try to upload parts larger than this size.""" + zh: "" } label { en: "Maximum multipart upload part size" + zh: "" } } acl { desc { en: "The ACL to use for the uploaded objects." + zh: "" } label { en: "ACL" + zh: "" } } transport_options { desc { en: "Options for the HTTP transport layer used by the S3 client." + zh: "" } label { en: "Transport Options" + zh: "" } } } From 7bcb60a84a7fdb81048bb9086d2519de9817139a Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 18:18:03 +0300 Subject: [PATCH 106/156] fix(ft): fix config update tests --- apps/emqx_ft/etc/emqx_ft.conf | 7 +++++++ apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 23 ++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/apps/emqx_ft/etc/emqx_ft.conf b/apps/emqx_ft/etc/emqx_ft.conf index 8d921e79c..2058d24f5 100644 --- a/apps/emqx_ft/etc/emqx_ft.conf +++ b/apps/emqx_ft/etc/emqx_ft.conf @@ -1,7 +1,14 @@ file_transfer { storage { type = local + segments { + root = "{{ platform_data_dir }}/file_transfer/segments" + gc { + interval = 60000 + } + } exporter { + root = "{{ platform_data_dir }}/file_transfer/exports" type = local } } diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index 3811c1ef4..5788e4171 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -25,10 +25,31 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft]), + _ = emqx_config:save_schema_mod_and_names(emqx_ft_schema), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], fun set_special_config/1), {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. +set_special_config(emqx_ft) -> + emqx_config:put( + [file_transfer], + #{ + storage => #{ + type => local, + segments => #{ + gc => #{ + interval => 60000 + } + }, + exporter => #{ + type => local + } + } + } + ); +set_special_config(_) -> + ok. + end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), ok. From 8b13d249ebf66b5dd4691ff5a3c2a0b9f5167e1b Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 18:44:10 +0300 Subject: [PATCH 107/156] fix(ft): fix detection of ee apps in mix --- apps/emqx_ft/BSL.txt | 94 ++++++++++++++++++++++++++++++++++++++++++++ mix.exs | 33 ++++++++++++---- 2 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 apps/emqx_ft/BSL.txt diff --git a/apps/emqx_ft/BSL.txt b/apps/emqx_ft/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_ft/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/mix.exs b/mix.exs index 8cda3b9c0..e5e91b491 100644 --- a/mix.exs +++ b/mix.exs @@ -107,13 +107,12 @@ defmodule EMQXUmbrella.MixProject do end defp emqx_apps(profile_info, version) do - apps = umbrella_apps() ++ enterprise_apps(profile_info) + apps = community_apps() ++ enterprise_apps(profile_info) set_emqx_app_system_env(apps, profile_info, version) end - defp umbrella_apps() do - "apps/*" - |> Path.wildcard() + defp community_apps() do + community_app_paths() |> Enum.map(fn path -> app = path @@ -125,9 +124,7 @@ defmodule EMQXUmbrella.MixProject do end defp enterprise_apps(_profile_info = %{edition_type: :enterprise}) do - "lib-ee/*" - |> Path.wildcard() - |> Enum.filter(&File.dir?/1) + enterprise_app_paths() |> Enum.map(fn path -> app = path @@ -142,6 +139,28 @@ defmodule EMQXUmbrella.MixProject do [] end + defp community_app_paths() do + "apps/*" + |> Path.wildcard() + |> Enum.filter(&File.dir?/1) + |> Enum.reject(&File.exists?(Path.join(&1, "BSL.txt"))) + end + + defp enterprise_app_paths() do + lib_ee_paths = + "lib-ee/*" + |> Path.wildcard() + |> Enum.filter(&File.dir?/1) + + apps_paths = + "apps/*" + |> Path.wildcard() + |> Enum.filter(&File.dir?/1) + |> Enum.filter(&File.exists?(Path.join(&1, "BSL.txt"))) + + lib_ee_paths ++ apps_paths + end + defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do [ {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"}, From 918bdcae7d4cbf556a158c335552dbf0d7eb5845 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 21:10:07 +0300 Subject: [PATCH 108/156] fix(ft): fix schema descriptions --- apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf | 16 +++++++++++++++- apps/emqx_ft/src/emqx_ft_schema.erl | 1 + apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf index c33ea447e..15cf17a39 100644 --- a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf +++ b/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf @@ -119,9 +119,23 @@ emqx_ft_schema { zh: "" } label: { - en: "GC Interval" + en: "Max segment TTL" zh: "" } } + storage_gc_min_segments_ttl { + desc { + en: "Minimum TTL of a segment kept in the local file system.
" + "This is a hard limit: no segment will be garbage collected before reaching this TTL, " + "even if some file transfer specifies a TTL less than that." + zh: "" + } + label: { + en: "Min segment TTL" + zh: "" + } + } + + } diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 270543bc9..28970f238 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -163,6 +163,7 @@ fields(local_storage_segments_gc) -> mk( emqx_schema:duration_s(), #{ + desc => ?DESC("storage_gc_min_segments_ttl"), required => false, default => "5m", % NOTE diff --git a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf index c26fa477a..9e9a52a41 100644 --- a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf +++ b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf @@ -1,7 +1,7 @@ emqx_s3_schema { access_key_id { desc { - en: "The access key id of the S3 bucket." + en: "The access key ID of the S3 bucket." zh: "" } label { From 4a144044b7c9a394a42d920bc2fc5d9a02aac4b5 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Apr 2023 21:10:32 +0300 Subject: [PATCH 109/156] fix(ft): set default ft config in tests --- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 24 +++---------------- apps/emqx_ft/test/emqx_ft_responder_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 15 +++++++----- 3 files changed, 13 insertions(+), 28 deletions(-) diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index 5788e4171..c616681dd 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -26,30 +26,12 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> _ = emqx_config:save_schema_mod_and_names(emqx_ft_schema), - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], fun set_special_config/1), + ok = emqx_common_test_helpers:start_apps( + [emqx_conf, emqx_ft], emqx_ft_test_helpers:env_handler(Config) + ), {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. -set_special_config(emqx_ft) -> - emqx_config:put( - [file_transfer], - #{ - storage => #{ - type => local, - segments => #{ - gc => #{ - interval => 60000 - } - }, - exporter => #{ - type => local - } - } - } - ); -set_special_config(_) -> - ok. - end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), ok. diff --git a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl index e447ba03b..751861206 100644 --- a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl @@ -24,7 +24,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_ft]), + ok = emqx_common_test_helpers:start_apps([emqx_ft], emqx_ft_test_helpers:env_handler(Config)), Config. end_per_suite(_Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index b8ee45b15..1e78f072e 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -28,12 +28,7 @@ start_additional_node(Config, Name) -> {apps, [emqx_ft]}, {join_to, node()}, {configure_gen_rpc, true}, - {env_handler, fun - (emqx_ft) -> - load_config(#{storage => local_storage(Config)}); - (_) -> - ok - end} + {env_handler, env_handler(Config)} ] ). @@ -43,6 +38,14 @@ stop_additional_node(Node) -> ok = emqx_common_test_helpers:stop_slave(Node), ok. +env_handler(Config) -> + fun + (emqx_ft) -> + load_config(#{storage => local_storage(Config)}); + (_) -> + ok + end. + local_storage(Config) -> #{ type => local, From bd7250cb1392619eb8d091f62991426d3c1e4b60 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 6 Apr 2023 00:04:10 +0300 Subject: [PATCH 110/156] fix(s3): fix hash pool type --- .../src/emqx_ft_storage_exporter_s3.erl | 44 ++++++---- apps/emqx_s3/src/emqx_s3_client.erl | 86 ++++++++++++------- apps/emqx_s3/src/emqx_s3_profile_conf.erl | 4 + apps/emqx_s3/test/emqx_s3_client_SUITE.erl | 36 ++++++-- 4 files changed, 117 insertions(+), 53 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index 24c3c52e5..63df71179 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -137,10 +137,11 @@ s3_headers({ClientId, FileId}, Filemeta) -> list(Client, Options) -> case list_key_info(Client, Options) of {ok, KeyInfos} -> - {ok, - lists:map( - fun(KeyInfo) -> key_info_to_exportinfo(Client, KeyInfo, Options) end, KeyInfos - )}; + MaybeExportInfos = lists:map( + fun(KeyInfo) -> key_info_to_exportinfo(Client, KeyInfo, Options) end, KeyInfos + ), + ExportInfos = [ExportInfo || {ok, ExportInfo} <- MaybeExportInfos], + {ok, ExportInfos}; {error, _Reason} = Error -> Error end. @@ -170,14 +171,18 @@ next_marker(KeyInfos) -> key_info_to_exportinfo(Client, KeyInfo, _Options) -> Key = proplists:get_value(key, KeyInfo), - {Transfer, Name} = parse_transfer_and_name(Key), - #{ - transfer => Transfer, - name => unicode:characters_to_binary(Name), - uri => emqx_s3_client:uri(Client, Key), - timestamp => datetime_to_epoch_second(proplists:get_value(last_modified, KeyInfo)), - size => proplists:get_value(size, KeyInfo) - }. + case parse_transfer_and_name(Key) of + {ok, {Transfer, Name}} -> + {ok, #{ + transfer => Transfer, + name => unicode:characters_to_binary(Name), + uri => emqx_s3_client:uri(Client, Key), + timestamp => datetime_to_epoch_second(proplists:get_value(last_modified, KeyInfo)), + size => proplists:get_value(size, KeyInfo) + }}; + {error, _Reason} = Error -> + Error + end. -define(EPOCH_START, 62167219200). @@ -185,8 +190,13 @@ datetime_to_epoch_second(DateTime) -> calendar:datetime_to_gregorian_seconds(DateTime) - ?EPOCH_START. parse_transfer_and_name(Key) -> - [ClientId, FileId, Name] = string:split(Key, "/", all), - Transfer = { - emqx_ft_fs_util:unescape_filename(ClientId), emqx_ft_fs_util:unescape_filename(FileId) - }, - {Transfer, Name}. + case string:split(Key, "/", all) of + [ClientId, FileId, Name] -> + Transfer = { + emqx_ft_fs_util:unescape_filename(ClientId), + emqx_ft_fs_util:unescape_filename(FileId) + }, + {ok, {Transfer, Name}}; + _ -> + {error, invalid_key} + end. diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index b6578a7e1..bfe87a602 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -38,6 +38,8 @@ -type part_number() :: non_neg_integer(). -type upload_id() :: string(). -type etag() :: string(). +-type http_pool() :: ehttpc:pool_name(). +-type pool_type() :: random | hash. -type upload_options() :: list({acl, emqx_s3:acl()}). -opaque client() :: #{ @@ -59,6 +61,7 @@ access_key_id := string() | undefined, secret_access_key := string() | undefined, http_pool := ehttpc:pool_name(), + pool_type := pool_type(), request_timeout := timeout() | undefined, max_retries := non_neg_integer() | undefined }. @@ -79,7 +82,8 @@ create(Config) -> upload_options => upload_options(Config), bucket => maps:get(bucket, Config), url_expire_time => maps:get(url_expire_time, Config), - headers => headers(Config) + headers => headers(Config), + pool_type => maps:get(pool_type, Config) }. -spec put_object(client(), key(), iodata()) -> ok_or_error(term()). @@ -211,6 +215,7 @@ aws_config(#{ access_key_id := AccessKeyId, secret_access_key := SecretAccessKey, http_pool := HttpPool, + pool_type := PoolType, request_timeout := Timeout, max_retries := MaxRetries }) -> @@ -224,7 +229,9 @@ aws_config(#{ access_key_id = AccessKeyId, secret_access_key = SecretAccessKey, - http_client = request_fun(HttpPool, with_default(MaxRetries, ?DEFAULT_MAX_RETRIES)), + http_client = request_fun( + HttpPool, PoolType, with_default(MaxRetries, ?DEFAULT_MAX_RETRIES) + ), %% This value will be transparently passed to ehttpc timeout = with_default(Timeout, ?DEFAULT_REQUEST_TIMEOUT), @@ -232,55 +239,63 @@ aws_config(#{ retry_num = 1 }. --type http_pool() :: term(). - --spec request_fun(http_pool(), non_neg_integer()) -> erlcloud_httpc:request_fun(). -request_fun(HttpPool, MaxRetries) -> +-spec request_fun(http_pool(), pool_type(), non_neg_integer()) -> erlcloud_httpc:request_fun(). +request_fun(HttpPool, PoolType, MaxRetries) -> fun(Url, Method, Headers, Body, Timeout, _Config) -> with_path_and_query_only(Url, fun(PathQuery) -> Request = make_request( Method, PathQuery, headers_erlcloud_request_to_ehttpc(Headers), Body ), - ehttpc_request(HttpPool, Method, Request, Timeout, MaxRetries) + case pick_worker_safe(HttpPool, PoolType) of + {ok, Worker} -> + ehttpc_request(Worker, Method, Request, Timeout, MaxRetries); + {error, Reason} -> + ?SLOG(error, #{ + msg => "s3_request_fun_fail", + reason => Reason, + http_pool => HttpPool, + pool_type => PoolType, + method => Method, + request => Request, + timeout => Timeout, + max_retries => MaxRetries + }), + {error, Reason} + end end) end. ehttpc_request(HttpPool, Method, Request, Timeout, MaxRetries) -> - ?SLOG(debug, #{ - msg => "s3_ehttpc_request", - timeout => Timeout, - pool => HttpPool, - method => Method, - max_retries => MaxRetries, - request => format_request(Request) - }), - try ehttpc:request(HttpPool, Method, Request, Timeout, MaxRetries) of - {ok, StatusCode, RespHeaders} -> - ?SLOG(debug, #{ - msg => "s3_ehttpc_request_ok", - status_code => StatusCode, - headers => RespHeaders - }), - {ok, { - {StatusCode, undefined}, headers_ehttpc_to_erlcloud_response(RespHeaders), undefined - }}; - {ok, StatusCode, RespHeaders, RespBody} -> + try timer:tc(fun() -> ehttpc:request(HttpPool, Method, Request, Timeout, MaxRetries) end) of + {Time, {ok, StatusCode, RespHeaders}} -> ?SLOG(debug, #{ msg => "s3_ehttpc_request_ok", status_code => StatusCode, headers => RespHeaders, - body => RespBody + time => Time + }), + {ok, { + {StatusCode, undefined}, headers_ehttpc_to_erlcloud_response(RespHeaders), undefined + }}; + {Time, {ok, StatusCode, RespHeaders, RespBody}} -> + ?SLOG(debug, #{ + msg => "s3_ehttpc_request_ok", + status_code => StatusCode, + headers => RespHeaders, + body => RespBody, + time => Time }), {ok, { {StatusCode, undefined}, headers_ehttpc_to_erlcloud_response(RespHeaders), RespBody }}; - {error, Reason} -> + {Time, {error, Reason}} -> ?SLOG(error, #{ msg => "s3_ehttpc_request_fail", reason => Reason, timeout => Timeout, pool => HttpPool, - method => Method + method => Method, + time => Time }), {error, Reason} catch @@ -304,6 +319,19 @@ ehttpc_request(HttpPool, Method, Request, Timeout, MaxRetries) -> {error, Reason} end. +pick_worker_safe(HttpPool, PoolType) -> + try + {ok, pick_worker(HttpPool, PoolType)} + catch + error:badarg -> + {error, no_ehttpc_pool} + end. + +pick_worker(HttpPool, random) -> + ehttpc_pool:pick_worker(HttpPool); +pick_worker(HttpPool, hash) -> + ehttpc_pool:pick_worker(HttpPool, self()). + -define(IS_BODY_EMPTY(Body), (Body =:= undefined orelse Body =:= <<>>)). -define(NEEDS_NO_BODY(Method), (Method =:= get orelse Method =:= head orelse Method =:= delete)). diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index 13efb9d74..e11dac530 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -204,6 +204,7 @@ client_config(ProfileConfig, PoolName) -> secret_access_key => maps:get(secret_access_key, ProfileConfig, undefined), request_timeout => maps:get(request_timeout, HTTPOpts, undefined), max_retries => maps:get(max_retries, HTTPOpts, undefined), + pool_type => maps:get(pool_type, HTTPOpts, random), http_pool => PoolName }. @@ -371,9 +372,12 @@ stop_http_pool(ProfileId, PoolName) -> ok = ?tp(debug, "s3_stop_http_pool", #{pool_name => PoolName}). do_start_http_pool(PoolName, HttpConfig) -> + ?SLOG(warning, #{msg => "s3_start_http_pool", pool_name => PoolName, config => HttpConfig}), case ehttpc_sup:start_pool(PoolName, HttpConfig) of {ok, _} -> + ?SLOG(warning, #{msg => "s3_start_http_pool_success", pool_name => PoolName}), ok; {error, _} = Error -> + ?SLOG(error, #{msg => "s3_start_http_pool_fail", pool_name => PoolName, error => Error}), Error end. diff --git a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl index ec7d5ebcf..4af803c67 100644 --- a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl @@ -20,9 +20,15 @@ all() -> groups() -> AllCases = emqx_common_test_helpers:all(?MODULE), + PoolGroups = [ + {group, pool_random}, + {group, pool_hash} + ], [ - {tcp, [], AllCases}, - {tls, [], AllCases} + {tcp, [], PoolGroups}, + {tls, [], PoolGroups}, + {pool_random, [], AllCases}, + {pool_hash, [], AllCases} ]. init_per_suite(Config) -> @@ -32,8 +38,17 @@ init_per_suite(Config) -> end_per_suite(_Config) -> ok = application:stop(emqx_s3). -init_per_group(ConnType, Config) -> - [{conn_type, ConnType} | Config]. +init_per_group(ConnTypeGroup, Config) when ConnTypeGroup =:= tcp; ConnTypeGroup =:= tls -> + [{conn_type, ConnTypeGroup} | Config]; +init_per_group(PoolTypeGroup, Config) when + PoolTypeGroup =:= pool_random; PoolTypeGroup =:= pool_hash +-> + PoolType = + case PoolTypeGroup of + pool_random -> random; + pool_hash -> hash + end, + [{pool_type, PoolType} | Config]. end_per_group(_ConnType, _Config) -> ok. @@ -127,11 +142,18 @@ client(Config) -> emqx_s3_client:create(ClientConfig). profile_config(Config) -> - maps:put( + ProfileConfig0 = emqx_s3_test_helpers:base_config(?config(conn_type, Config)), + ProfileConfig1 = maps:put( bucket, ?config(bucket, Config), - emqx_s3_test_helpers:base_config(?config(conn_type, Config)) - ). + ProfileConfig0 + ), + ProfileConfig2 = emqx_map_lib:deep_put( + [transport_options, pool_type], + ProfileConfig1, + ?config(pool_type, Config) + ), + ProfileConfig2. data(Size) -> iolist_to_binary([$a || _ <- lists:seq(1, Size)]). From eae2478678999ca4b569365fe04deb8aebc49972 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 6 Apr 2023 13:20:41 +0300 Subject: [PATCH 111/156] fix(ft): add descriptions for missing ft schema fields --- apps/emqx_s3/src/emqx_s3_schema.erl | 7 +++- .../emqx_ee_conf/src/emqx_ee_conf_schema.erl | 34 +++++++++++-------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index 6db9a5886..bdd88a9e8 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -9,7 +9,7 @@ -import(hoconsc, [mk/2, ref/2]). --export([roots/0, fields/1, namespace/0, tags/0]). +-export([roots/0, fields/1, namespace/0, tags/0, desc/1]). -export([translate/1]). @@ -125,6 +125,11 @@ fields(transport_options) -> [headers, max_retries, request_timeout], emqx_connector_http:fields("request") ). +desc(s3) -> + "S3 connection options"; +desc(transport_options) -> + "Options for the HTTP transport layer used by the S3 client". + translate(Conf) -> Options = #{atom_key => true}, #{s3 := TranslatedConf} = hocon_tconf:check_plain( diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl index 8e2800c68..907db70d9 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl @@ -6,10 +6,11 @@ -behaviour(hocon_schema). --export([namespace/0, roots/0, fields/1, translations/0, translation/1]). +-export([namespace/0, roots/0, fields/1, translations/0, translation/1, desc/1]). -define(EE_SCHEMA_MODULES, [ emqx_license_schema, + emqx_s3_schema, emqx_ft_schema ]). @@ -17,16 +18,10 @@ namespace() -> emqx_conf_schema:namespace(). roots() -> - lists:foldl( - fun(Module, Roots) -> - Roots ++ apply(Module, roots, []) - end, - emqx_conf_schema:roots(), - ?EE_SCHEMA_MODULES - ). + emqx_conf_schema:roots() ++ ee_roots(). fields(Name) -> - ee_fields(?EE_SCHEMA_MODULES, Name). + ee_delegate(fields, ?EE_SCHEMA_MODULES, Name). translations() -> emqx_conf_schema:translations(). @@ -34,16 +29,27 @@ translations() -> translation(Name) -> emqx_conf_schema:translation(Name). +desc(Name) -> + ee_delegate(desc, ?EE_SCHEMA_MODULES, Name). + %%------------------------------------------------------------------------------ %% helpers %%------------------------------------------------------------------------------ -ee_fields([EEMod | EEMods], Name) -> +ee_roots() -> + lists:flatmap( + fun(Module) -> + apply(Module, roots, []) + end, + ?EE_SCHEMA_MODULES + ). + +ee_delegate(Method, [EEMod | EEMods], Name) -> case lists:member(Name, apply(EEMod, roots, [])) of true -> - apply(EEMod, fields, [Name]); + EEMod:Method(Name); false -> - ee_fields(EEMods, Name) + ee_delegate(Method, EEMods, Name) end; -ee_fields([], Name) -> - emqx_conf_schema:fields(Name). +ee_delegate(Method, [], Name) -> + emqx_conf_schema:Method(Name). From 52f3189779070099ea8a6fa091660ff347ef28bb Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 6 Apr 2023 13:42:06 +0300 Subject: [PATCH 112/156] fix(ft): fix default ft config --- apps/emqx_ft/etc/emqx_ft.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_ft/etc/emqx_ft.conf b/apps/emqx_ft/etc/emqx_ft.conf index 2058d24f5..ec48d4e69 100644 --- a/apps/emqx_ft/etc/emqx_ft.conf +++ b/apps/emqx_ft/etc/emqx_ft.conf @@ -4,7 +4,7 @@ file_transfer { segments { root = "{{ platform_data_dir }}/file_transfer/segments" gc { - interval = 60000 + interval = "1h" } } exporter { From d21daae7aeb93fffc568b58b8ee7b7e88d3a6fb2 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 6 Apr 2023 13:43:17 +0300 Subject: [PATCH 113/156] fix(ft): fix dynamic function calls --- lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl index 907db70d9..08a6aa971 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl @@ -47,9 +47,9 @@ ee_roots() -> ee_delegate(Method, [EEMod | EEMods], Name) -> case lists:member(Name, apply(EEMod, roots, [])) of true -> - EEMod:Method(Name); + apply(EEMod, Method, [Name]); false -> ee_delegate(Method, EEMods, Name) end; ee_delegate(Method, [], Name) -> - emqx_conf_schema:Method(Name). + apply(emqx_conf_schema, Method, [Name]). From 6fbc7dc822eef92a69e8c36096fab701ea436e76 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 6 Apr 2023 14:11:02 +0300 Subject: [PATCH 114/156] fix(ft): remove s3 roots from ee schema --- lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl index 08a6aa971..df3382df1 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl @@ -10,7 +10,6 @@ -define(EE_SCHEMA_MODULES, [ emqx_license_schema, - emqx_s3_schema, emqx_ft_schema ]). From 7eeba326195b7183a73528edd5d438c78d3047e7 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 6 Apr 2023 21:53:12 +0300 Subject: [PATCH 115/156] fix(s3): fix typings --- apps/emqx_ft/src/emqx_ft.erl | 2 +- apps/emqx_s3/src/emqx_s3_client.erl | 9 +++++---- apps/emqx_s3/src/emqx_s3_profile_conf.erl | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 45ec2b933..c055680b0 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -82,7 +82,7 @@ -type segment() :: {offset(), _Content :: binary()}. -define(STORE_SEGMENT_TIMEOUT, 10000). --define(ASSEMBLE_TIMEOUT, 60000). +-define(ASSEMBLE_TIMEOUT, 300000). %%-------------------------------------------------------------------- %% API for app diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index bfe87a602..c01b1b240 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -47,7 +47,8 @@ upload_options := upload_options(), bucket := string(), headers := erlcloud_headers(), - url_expire_time := non_neg_integer() + url_expire_time := non_neg_integer(), + pool_type := pool_type() }. -type config() :: #{ @@ -60,7 +61,7 @@ url_expire_time := pos_integer(), access_key_id := string() | undefined, secret_access_key := string() | undefined, - http_pool := ehttpc:pool_name(), + http_pool := http_pool(), pool_type := pool_type(), request_timeout := timeout() | undefined, max_retries := non_neg_integer() | undefined @@ -268,7 +269,7 @@ request_fun(HttpPool, PoolType, MaxRetries) -> ehttpc_request(HttpPool, Method, Request, Timeout, MaxRetries) -> try timer:tc(fun() -> ehttpc:request(HttpPool, Method, Request, Timeout, MaxRetries) end) of {Time, {ok, StatusCode, RespHeaders}} -> - ?SLOG(debug, #{ + ?SLOG(info, #{ msg => "s3_ehttpc_request_ok", status_code => StatusCode, headers => RespHeaders, @@ -278,7 +279,7 @@ ehttpc_request(HttpPool, Method, Request, Timeout, MaxRetries) -> {StatusCode, undefined}, headers_ehttpc_to_erlcloud_response(RespHeaders), undefined }}; {Time, {ok, StatusCode, RespHeaders, RespBody}} -> - ?SLOG(debug, #{ + ?SLOG(info, #{ msg => "s3_ehttpc_request_ok", status_code => StatusCode, headers => RespHeaders, diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index e11dac530..006d06899 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -337,7 +337,7 @@ http_config( SSLOpts = emqx_tls_lib:to_client_opts(maps:get(ssl, HTTPOpts)), {tls, SSLOpts} end, - NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), + % NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), [ {host, Host}, {port, Port}, @@ -346,7 +346,7 @@ http_config( {pool_type, PoolType}, {pool_size, PoolSize}, {transport, Transport}, - {transport_opts, NTransportOpts}, + {transport_opts, TransportOpts}, {enable_pipelining, EnablePipelining} ]. From b2fd2dcbc114122336b196c6109c71955bc8ebd1 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 7 Apr 2023 01:58:25 +0300 Subject: [PATCH 116/156] fix(ft): make configs user friendlier --- apps/emqx_ft/src/emqx_ft_conf.erl | 27 +++++++++++++++++----- apps/emqx_ft/src/emqx_ft_schema.erl | 17 +++++++++++--- apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf | 10 ++++++++ apps/emqx_s3/src/emqx_s3.erl | 1 + apps/emqx_s3/src/emqx_s3_profile_conf.erl | 9 ++++++-- apps/emqx_s3/src/emqx_s3_schema.erl | 17 +++++++++++--- 6 files changed, 67 insertions(+), 14 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 0e8bcc193..5ca0b85ed 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -42,25 +42,40 @@ -type milliseconds() :: non_neg_integer(). -type seconds() :: non_neg_integer(). +%% 5 minutes (s) +-define(DEFAULT_MIN_SEGMENTS_TTL, 300). +%% 1 day (s) +-define(DEFAULT_MAX_SEGMENTS_TTL, 86400). +%% 1 minute (ms) +-define(DEFAULT_GC_INTERVAL, 60000). + %%-------------------------------------------------------------------- %% Accessors %%-------------------------------------------------------------------- --spec storage() -> _Storage | disabled. +-spec storage() -> _Storage. storage() -> - emqx_config:get([file_transfer, storage], disabled). + emqx_config:get([file_transfer, storage]). -spec gc_interval(_Storage) -> milliseconds(). gc_interval(_Storage) -> Conf = assert_storage(local), - emqx_map_lib:deep_get([segments, gc, interval], Conf). + emqx_map_lib:deep_get([segments, gc, interval], Conf, ?DEFAULT_GC_INTERVAL). -spec segments_ttl(_Storage) -> {_Min :: seconds(), _Max :: seconds()}. segments_ttl(_Storage) -> Conf = assert_storage(local), { - emqx_map_lib:deep_get([segments, gc, minimum_segments_ttl], Conf), - emqx_map_lib:deep_get([segments, gc, maximum_segments_ttl], Conf) + emqx_map_lib:deep_get( + [segments, gc, minimum_segments_ttl], + Conf, + ?DEFAULT_MIN_SEGMENTS_TTL + ), + emqx_map_lib:deep_get( + [segments, gc, maximum_segments_ttl], + Conf, + ?DEFAULT_MAX_SEGMENTS_TTL + ) }. assert_storage(Type) -> @@ -79,7 +94,7 @@ assert_storage(Type) -> load() -> ok = emqx_ft_storage_exporter:update_exporter( undefined, - emqx_config:get([file_transfer, storage]) + storage() ), emqx_conf:add_handler([file_transfer], ?MODULE). diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 28970f238..f22b93c6d 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -52,8 +52,9 @@ fields(file_transfer) -> ref(local_storage) ]), #{ - required => true, - desc => ?DESC("storage") + required => false, + desc => ?DESC("storage"), + default => default_storage() } )} ]; @@ -184,7 +185,9 @@ desc(local_storage_exporter) -> desc(s3_exporter) -> "S3 Exporter settings for the File transfer local storage backend"; desc(local_storage_segments_gc) -> - "Garbage collection settings for the File transfer local segments storage". + "Garbage collection settings for the File transfer local segments storage"; +desc(_) -> + undefined. schema(filemeta) -> #{ @@ -221,3 +224,11 @@ converter(checksum) -> ref(Ref) -> ref(?MODULE, Ref). + +default_storage() -> + #{ + <<"type">> => <<"local">>, + <<"exporter">> => #{ + <<"type">> => <<"local">> + } + }. diff --git a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf index 9e9a52a41..c391fa352 100644 --- a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf +++ b/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf @@ -101,4 +101,14 @@ S3 uploader won't try to upload parts larger than this size.""" zh: "" } } + ipv6_probe { + desc { + en: "Whether to probe for IPv6 support." + zh: "" + } + label { + en: "IPv6 probe" + zh: "" + } + } } diff --git a/apps/emqx_s3/src/emqx_s3.erl b/apps/emqx_s3/src/emqx_s3.erl index 15f63fbd5..0c2592736 100644 --- a/apps/emqx_s3/src/emqx_s3.erl +++ b/apps/emqx_s3/src/emqx_s3.erl @@ -37,6 +37,7 @@ max_retries => pos_integer(), pool_size => pos_integer(), pool_type => atom(), + ipv6_probe => boolean(), ssl => map() }. diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index 006d06899..3d9d6ed3f 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -337,7 +337,7 @@ http_config( SSLOpts = emqx_tls_lib:to_client_opts(maps:get(ssl, HTTPOpts)), {tls, SSLOpts} end, - % NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), + NTransportOpts = maybe_ipv6_probe(TransportOpts, maps:get(ipv6_probe, HTTPOpts, true)), [ {host, Host}, {port, Port}, @@ -346,10 +346,15 @@ http_config( {pool_type, PoolType}, {pool_size, PoolSize}, {transport, Transport}, - {transport_opts, TransportOpts}, + {transport_opts, NTransportOpts}, {enable_pipelining, EnablePipelining} ]. +maybe_ipv6_probe(TransportOpts, true) -> + emqx_misc:ipv6_probe(TransportOpts); +maybe_ipv6_probe(TransportOpts, false) -> + TransportOpts. + http_pool_cleanup_interval(ProfileConfig) -> maps:get( http_pool_cleanup_interval, ProfileConfig, ?DEAFULT_HTTP_POOL_CLEANUP_INTERVAL diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index bdd88a9e8..4d017deed 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -118,9 +118,20 @@ fields(s3) -> )} ]; fields(transport_options) -> - props_without( - [base_url, max_retries, retry_interval, request], emqx_connector_http:fields(config) - ) ++ + [ + {ipv6_probe, + mk( + boolean(), + #{ + default => true, + desc => ?DESC("ipv6_probe"), + required => false + } + )} + ] ++ + props_without( + [base_url, max_retries, retry_interval, request], emqx_connector_http:fields(config) + ) ++ props_with( [headers, max_retries, request_timeout], emqx_connector_http:fields("request") ). From 0d86ef8d3a87a98efa6479d57b61aa853a5f41ef Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 7 Apr 2023 21:05:09 +0300 Subject: [PATCH 117/156] fix(ft-s3): use AWS4 signed urls for S3 export URIs This will ensure that the S3 export URIs are valid in all AWS regions. --- apps/emqx_s3/rebar.config | 2 +- apps/emqx_s3/src/emqx_s3_client.erl | 2 +- apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl | 57 ++++++++++++++++++++ lib-ee/emqx_ee_connector/rebar.config | 2 +- mix.exs | 4 +- 5 files changed, 62 insertions(+), 5 deletions(-) diff --git a/apps/emqx_s3/rebar.config b/apps/emqx_s3/rebar.config index 92ab7895e..b1483e028 100644 --- a/apps/emqx_s3/rebar.config +++ b/apps/emqx_s3/rebar.config @@ -1,6 +1,6 @@ {deps, [ {emqx, {path, "../../apps/emqx"}}, - {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.7-emqx-1"}}} + {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.8-emqx-1"}}} ]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index c01b1b240..6bb6c2e3b 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -189,7 +189,7 @@ list(#{bucket := Bucket, aws_config := AwsConfig}, Options) -> -spec uri(client(), key()) -> iodata(). uri(#{bucket := Bucket, aws_config := AwsConfig, url_expire_time := ExpireTime}, Key) -> - erlcloud_s3:make_get_url(ExpireTime, Bucket, erlcloud_key(Key), AwsConfig). + erlcloud_s3:make_presigned_v4_url(ExpireTime, Bucket, get, erlcloud_key(Key), [], AwsConfig). -spec format(client()) -> term(). format(#{aws_config := AwsConfig} = Client) -> diff --git a/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl index 5fda42fbe..e09ea2773 100644 --- a/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl @@ -54,6 +54,8 @@ groups() -> t_happy_path_multi, t_abort_multi, t_abort_simple_put, + t_signed_url_download, + t_signed_nonascii_url_download, {group, noconn_errors}, {group, timeout_errors}, @@ -193,6 +195,40 @@ t_happy_path_multi(Config) -> Key ). +t_signed_url_download(_Config) -> + Prefix = emqx_s3_test_helpers:unique_key(), + Key = Prefix ++ "/ascii.txt", + + {ok, Data} = upload(Key, 1024, 5), + + SignedUrl = emqx_s3:with_client(profile_id(), fun(Client) -> + emqx_s3_client:uri(Client, Key) + end), + + {ok, {_, _, Body}} = httpc:request(get, {SignedUrl, []}, [], []), + + ?assertEqual( + iolist_to_binary(Data), + iolist_to_binary(Body) + ). + +t_signed_nonascii_url_download(_Config) -> + Prefix = emqx_s3_test_helpers:unique_key(), + Key = Prefix ++ "/unicode-🫠.txt", + + {ok, Data} = upload(Key, 1024 * 1024, 8), + + SignedUrl = emqx_s3:with_client(profile_id(), fun(Client) -> + emqx_s3_client:uri(Client, Key) + end), + + {ok, {_, _, Body}} = httpc:request(get, {SignedUrl, []}, [], []), + + ?assertEqual( + iolist_to_binary(Data), + iolist_to_binary(Body) + ). + t_abort_multi(Config) -> Key = emqx_s3_test_helpers:unique_key(), {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), @@ -532,3 +568,24 @@ data(Byte, ChunkSize, ChunkCount) -> list_objects(Config) -> Props = erlcloud_s3:list_objects(?config(bucket, Config), [], ?config(test_aws_config, Config)), proplists:get_value(contents, Props). + +upload(Key, ChunkSize, ChunkCount) -> + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + Data = data($a, ChunkSize, ChunkCount), + + ok = lists:foreach( + fun(Chunk) -> ?assertEqual(ok, emqx_s3_uploader:write(Pid, Chunk)) end, + Data + ), + + ok = emqx_s3_uploader:complete(Pid), + + ok = ?assertProcessExited( + normal, + Pid + ), + + {ok, Data}. diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index f36eb0ad6..654b1bf54 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -4,7 +4,7 @@ {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, - {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.7-emqx-1"}}}, + {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.8-emqx-1"}}}, {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, {emqx, {path, "../../apps/emqx"}} ]}. diff --git a/mix.exs b/mix.exs index e5e91b491..4f3d5a217 100644 --- a/mix.exs +++ b/mix.exs @@ -94,10 +94,10 @@ defmodule EMQXUmbrella.MixProject do # in conflict by grpc and eetcd {:gpb, "4.19.5", override: true, runtime: false}, {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true}, - {:erlcloud, github: "emqx/erlcloud", tag: "3.6.7-emqx-1", override: true}, + {:erlcloud, github: "emqx/erlcloud", tag: "3.6.8-emqx-1", override: true}, # erlcloud's rebar.config requires rebar3 and does not support Mix, # so it tries to fetch deps from git. We need to override this. - {:lhttpc, "1.6.2", override: true}, + {:lhttpc, github: "https://github.com/erlcloud/lhttpc", tag: "1.6.2", override: true}, {:eini, "1.2.9", override: true}, {:base16, "1.0.0", override: true} # end of erlcloud's deps From 8daa38ef06a7091cde479f4ecec12e3d71482829 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 7 Apr 2023 21:58:27 +0300 Subject: [PATCH 118/156] feat(ft-s3): store metadata in ASCII-safe format Also ensure consistent encoding and decoding filenames throughout the `emqx_ft` application. --- apps/emqx_ft/src/emqx_ft_schema.erl | 14 ++++- .../src/emqx_ft_storage_exporter_s3.erl | 5 +- apps/emqx_ft/test/emqx_ft_SUITE.erl | 56 +++++++++---------- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 6 +- mix.exs | 2 +- 5 files changed, 49 insertions(+), 34 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index f22b93c6d..beb43adc3 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -195,7 +195,8 @@ schema(filemeta) -> {name, hoconsc:mk(string(), #{ required => true, - validator => validator(filename) + validator => validator(filename), + converter => converter(unicode_string) })}, {size, hoconsc:mk(non_neg_integer())}, {expire_at, hoconsc:mk(non_neg_integer())}, @@ -220,6 +221,17 @@ converter(checksum) -> _ = is_binary(Hex) orelse throw({expected_type, string}), _ = byte_size(Hex) =:= 64 orelse throw({expected_length, 64}), {sha256, binary:decode_hex(Hex)} + end; +converter(unicode_string) -> + fun + (undefined, #{}) -> + undefined; + (Str, #{make_serializable := true}) -> + _ = is_list(Str) orelse throw({expected_type, string}), + unicode:characters_to_binary(Str); + (Str, #{}) -> + _ = is_binary(Str) orelse throw({expected_type, string}), + unicode:characters_to_list(Str) end. ref(Ref) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index 63df71179..259cb4016 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -130,10 +130,13 @@ s3_headers({ClientId, FileId}, Filemeta) -> <<"x-amz-meta-clientid">> => ClientId, %% It [Topic Name] MUST be a UTF-8 Encoded String <<"x-amz-meta-fileid">> => FileId, - <<"x-amz-meta-filemeta">> => emqx_json:encode(emqx_ft:encode_filemeta(Filemeta)), + <<"x-amz-meta-filemeta">> => s3_header_filemeta(Filemeta), <<"x-amz-meta-filemeta-vsn">> => ?FILEMETA_VSN }. +s3_header_filemeta(Filemeta) -> + emqx_json:encode(emqx_ft:encode_filemeta(Filemeta), [force_utf8, uescape]). + list(Client, Options) -> case list_key_info(Client, Options) of {ok, KeyInfos} -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index e365cba85..c3ee7f271 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -169,39 +169,38 @@ t_invalid_filename(Config) -> C = ?config(client, Config), ?assertRCName( unspecified_error, - emqtt:publish(C, mk_init_topic(<<"f1">>), emqx_json:encode(meta(".", <<>>)), 1) + emqtt:publish(C, mk_init_topic(<<"f1">>), encode_meta(meta(".", <<>>)), 1) ), ?assertRCName( unspecified_error, - emqtt:publish(C, mk_init_topic(<<"f2">>), emqx_json:encode(meta("..", <<>>)), 1) + emqtt:publish(C, mk_init_topic(<<"f2">>), encode_meta(meta("..", <<>>)), 1) ), ?assertRCName( unspecified_error, - emqtt:publish(C, mk_init_topic(<<"f2">>), emqx_json:encode(meta("../nice", <<>>)), 1) + emqtt:publish(C, mk_init_topic(<<"f2">>), encode_meta(meta("../nice", <<>>)), 1) ), ?assertRCName( unspecified_error, - emqtt:publish(C, mk_init_topic(<<"f3">>), emqx_json:encode(meta("/etc/passwd", <<>>)), 1) + emqtt:publish(C, mk_init_topic(<<"f3">>), encode_meta(meta("/etc/passwd", <<>>)), 1) ), ?assertRCName( success, - emqtt:publish(C, mk_init_topic(<<"f4">>), emqx_json:encode(meta("146%", <<>>)), 1) + emqtt:publish(C, mk_init_topic(<<"f4">>), encode_meta(meta("146%", <<>>)), 1) ). t_simple_transfer(Config) -> C = ?config(client, Config), - Filename = <<"topsecret.pdf">>, + Filename = "topsecret.pdf", FileId = <<"f1">>, Data = [<<"first">>, <<"second">>, <<"third">>], Meta = #{size := Filesize} = meta(Filename, Data), - MetaPayload = emqx_json:encode(Meta), ?assertRCName( success, - emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1) ), lists:foreach( @@ -246,23 +245,21 @@ t_nasty_clientids_fileids(_Config) -> t_meta_conflict(Config) -> C = ?config(client, Config), - Filename = <<"topsecret.pdf">>, + Filename = "topsecret.pdf", FileId = <<"f1">>, Meta = meta(Filename, [<<"x">>]), - MetaPayload = emqx_json:encode(Meta), ?assertRCName( success, - emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1) ), - ConflictMeta = Meta#{name => <<"conflict.pdf">>}, - ConflictMetaPayload = emqx_json:encode(ConflictMeta), + ConflictMeta = Meta#{name => "conflict.pdf"}, ?assertRCName( unspecified_error, - emqtt:publish(C, mk_init_topic(FileId), ConflictMetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), encode_meta(ConflictMeta), 1) ). t_no_meta(Config) -> @@ -284,17 +281,16 @@ t_no_meta(Config) -> t_no_segment(Config) -> C = ?config(client, Config), - Filename = <<"topsecret.pdf">>, + Filename = "topsecret.pdf", FileId = <<"f1">>, Data = [<<"first">>, <<"second">>, <<"third">>], Meta = #{size := Filesize} = meta(Filename, Data), - MetaPayload = emqx_json:encode(Meta), ?assertRCName( success, - emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) + emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1) ), lists:foreach( @@ -335,13 +331,13 @@ t_invalid_meta(Config) -> t_invalid_checksum(Config) -> C = ?config(client, Config), - Filename = <<"topsecret.pdf">>, + Filename = "topsecret.pdf", FileId = <<"f1">>, Data = [<<"first">>, <<"second">>, <<"third">>], Meta = #{size := Filesize} = meta(Filename, Data), - MetaPayload = emqx_json:encode(Meta#{checksum => sha256hex(<<"invalid">>)}), + MetaPayload = encode_meta(Meta#{checksum => {sha256, sha256(<<"invalid">>)}}), ?assertRCName( success, @@ -366,7 +362,7 @@ t_invalid_checksum(Config) -> t_corrupted_segment_retry(Config) -> C = ?config(client, Config), - Filename = <<"corruption.pdf">>, + Filename = "corruption.pdf", FileId = <<"4242-4242">>, Data = [<<"first">>, <<"second">>, <<"third">>], @@ -379,11 +375,11 @@ t_corrupted_segment_retry(Config) -> Checksum1, Checksum2, Checksum3 - ] = [sha256hex(S) || S <- Data], + ] = [binary:encode_hex(sha256(S)) || S <- Data], Meta = #{size := Filesize} = meta(Filename, Data), - ?assertRCName(success, emqtt:publish(C, mk_init_topic(FileId), emqx_json:encode(Meta), 1)), + ?assertRCName(success, emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1)), ?assertRCName( success, @@ -421,7 +417,7 @@ t_switch_node(Config) -> {ok, C1} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, AdditionalNodePort}]), {ok, _} = emqtt:connect(C1), - Filename = <<"multinode_upload.txt">>, + Filename = "multinode_upload.txt", FileId = <<"f1">>, Data = [<<"first">>, <<"second">>, <<"third">>], @@ -430,11 +426,10 @@ t_switch_node(Config) -> %% First, publist metadata and the first segment to the additional node Meta = #{size := Filesize} = meta(Filename, Data), - MetaPayload = emqx_json:encode(Meta), ?assertRCName( success, - emqtt:publish(C1, mk_init_topic(FileId), MetaPayload, 1) + emqtt:publish(C1, mk_init_topic(FileId), encode_meta(Meta), 1) ), ?assertRCName( success, @@ -593,7 +588,7 @@ disown_mqtt_client(Context = #{}) -> send_filemeta(Meta, Context = #{client := Client, fileid := FileId}) -> ?assertRCName( success, - emqtt:publish(Client, mk_init_topic(FileId), emqx_json:encode(Meta), 1) + emqtt:publish(Client, mk_init_topic(FileId), encode_meta(Meta), 1) ), Context. @@ -650,18 +645,21 @@ with_offsets(Items) -> ), List. -sha256hex(Data) -> - binary:encode_hex(crypto:hash(sha256, Data)). +sha256(Data) -> + crypto:hash(sha256, Data). meta(FileName, Data) -> FullData = iolist_to_binary(Data), #{ name => FileName, - checksum => sha256hex(FullData), + checksum => {sha256, sha256(FullData)}, expire_at => erlang:system_time(_Unit = second) + 3600, size => byte_size(FullData) }. +encode_meta(Meta) -> + emqx_json:encode(emqx_ft:encode_filemeta(Meta)). + list_files(ClientId) -> {ok, Files} = emqx_ft_storage:files(), [File || File = #{transfer := {CId, _}} <- Files, CId == ClientId]. diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index 1e78f072e..b7004dcc6 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -78,11 +78,13 @@ upload_file(ClientId, FileId, Name, Data, Node) -> {ok, C1} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, Port}]), {ok, _} = emqtt:connect(C1), Meta = #{ - name => unicode:characters_to_binary(Name), + name => Name, expire_at => erlang:system_time(_Unit = second) + 3600, size => Size }, - MetaPayload = emqx_json:encode(Meta), + MetaPayload = emqx_json:encode(emqx_ft:encode_filemeta(Meta)), + + ct:pal("MetaPayload = ~ts", [MetaPayload]), MetaTopic = <<"$file/", FileId/binary, "/init">>, {ok, _} = emqtt:publish(C1, MetaTopic, MetaPayload, 1), diff --git a/mix.exs b/mix.exs index 4f3d5a217..cd6d08fd1 100644 --- a/mix.exs +++ b/mix.exs @@ -97,7 +97,7 @@ defmodule EMQXUmbrella.MixProject do {:erlcloud, github: "emqx/erlcloud", tag: "3.6.8-emqx-1", override: true}, # erlcloud's rebar.config requires rebar3 and does not support Mix, # so it tries to fetch deps from git. We need to override this. - {:lhttpc, github: "https://github.com/erlcloud/lhttpc", tag: "1.6.2", override: true}, + {:lhttpc, github: "erlcloud/lhttpc", tag: "1.6.2", override: true}, {:eini, "1.2.9", override: true}, {:base16, "1.0.0", override: true} # end of erlcloud's deps From 9224b339becf5b85f49e3d1547fb99553e80a1b1 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 7 Apr 2023 22:14:06 +0300 Subject: [PATCH 119/156] fix(dep): upgrade `rulesql` back to 0.1.5 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 5cedb47ff..908adefdb 100644 --- a/rebar.config +++ b/rebar.config @@ -82,7 +82,7 @@ {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.9"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}}, - {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}}, + {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.5"}}}, % NOTE: depends on recon 2.5.x {observer_cli, "1.7.1"}, {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}, From 37a520d79773ad66a4c7641fbcbdd6b453651e8c Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 10 Apr 2023 16:02:34 +0300 Subject: [PATCH 120/156] fix(i18n): relocate i18n files to rel/i18n --- .../i18n/emqx_ft_api_i18n.conf => rel/i18n/emqx_ft_api.hocon | 0 .../emqx_ft_schema_i18n.conf => rel/i18n/emqx_ft_schema.hocon | 0 .../emqx_s3_schema_i18n.conf => rel/i18n/emqx_s3_schema.hocon | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename apps/emqx_ft/i18n/emqx_ft_api_i18n.conf => rel/i18n/emqx_ft_api.hocon (100%) rename apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf => rel/i18n/emqx_ft_schema.hocon (100%) rename apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf => rel/i18n/emqx_s3_schema.hocon (100%) diff --git a/apps/emqx_ft/i18n/emqx_ft_api_i18n.conf b/rel/i18n/emqx_ft_api.hocon similarity index 100% rename from apps/emqx_ft/i18n/emqx_ft_api_i18n.conf rename to rel/i18n/emqx_ft_api.hocon diff --git a/apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf b/rel/i18n/emqx_ft_schema.hocon similarity index 100% rename from apps/emqx_ft/i18n/emqx_ft_schema_i18n.conf rename to rel/i18n/emqx_ft_schema.hocon diff --git a/apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf b/rel/i18n/emqx_s3_schema.hocon similarity index 100% rename from apps/emqx_s3/i18n/emqx_s3_schema_i18n.conf rename to rel/i18n/emqx_s3_schema.hocon From e22c1c01ec3dad342bc12651a4a7104b31ef5ee0 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 13 Apr 2023 19:47:12 +0300 Subject: [PATCH 121/156] feat(s3): make acl optional --- apps/emqx_s3/src/emqx_s3_client.erl | 8 +++++--- apps/emqx_s3/src/emqx_s3_profile_conf.erl | 2 +- apps/emqx_s3/src/emqx_s3_schema.erl | 2 +- apps/emqx_s3/test/emqx_s3_client_SUITE.erl | 13 ++++++++++++- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index 6bb6c2e3b..e3e1d84af 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -199,10 +199,12 @@ format(#{aws_config := AwsConfig} = Client) -> %% Internal functions %%-------------------------------------------------------------------- -upload_options(Config) -> +upload_options(#{acl := Acl}) when Acl =/= undefined -> [ - {acl, maps:get(acl, Config)} - ]. + {acl, Acl} + ]; +upload_options(#{}) -> + []. headers(#{headers := Headers}) -> headers_user_to_erlcloud_request(Headers); diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index 3d9d6ed3f..3d66823a7 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -198,7 +198,7 @@ client_config(ProfileConfig, PoolName) -> port => maps:get(port, ProfileConfig), url_expire_time => maps:get(url_expire_time, ProfileConfig), headers => maps:get(headers, HTTPOpts, #{}), - acl => maps:get(acl, ProfileConfig), + acl => maps:get(acl, ProfileConfig, undefined), bucket => maps:get(bucket, ProfileConfig), access_key_id => maps:get(access_key_id, ProfileConfig, undefined), secret_access_key => maps:get(secret_access_key, ProfileConfig, undefined), diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index 4d017deed..2d714bb7d 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -105,7 +105,7 @@ fields(s3) -> #{ default => private, desc => ?DESC("acl"), - required => true + required => false } )}, {transport_options, diff --git a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl index 4af803c67..cb55bc083 100644 --- a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl @@ -60,7 +60,8 @@ init_per_testcase(_TestCase, Config0) -> ok = erlcloud_s3:create_bucket(Bucket, TestAwsConfig), Config1 = [ {key, emqx_s3_test_helpers:unique_key()}, - {bucket, Bucket} + {bucket, Bucket}, + {aws_config, TestAwsConfig} | Config0 ], {ok, PoolName} = emqx_s3_profile_conf:start_http_pool(?PROFILE_ID, profile_config(Config1)), @@ -131,6 +132,16 @@ t_url(Config) -> httpc:request(Url) ). +t_no_acl(Config) -> + Key = ?config(key, Config), + + ClientConfig = emqx_s3_profile_conf:client_config( + profile_config(Config), ?config(ehttpc_pool_name, Config) + ), + Client = emqx_s3_client:create(maps:without([acl], ClientConfig)), + + ok = emqx_s3_client:put_object(Client, Key, <<"data">>). + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- From 92ff2b7d6ee8a11d11f9f1d67b564fcb5c1e9ad5 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 13 Apr 2023 19:47:32 +0300 Subject: [PATCH 122/156] feat(ft): make timeouts configurable --- apps/emqx_ft/src/emqx_ft.erl | 9 +++----- apps/emqx_ft/src/emqx_ft_conf.erl | 12 ++++++++++ apps/emqx_ft/src/emqx_ft_schema.erl | 27 ++++++++++++++++++++++ rel/i18n/emqx_ft_schema.hocon | 36 +++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 6 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index c055680b0..6ac6596ff 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -81,9 +81,6 @@ -type segment() :: {offset(), _Content :: binary()}. --define(STORE_SEGMENT_TIMEOUT, 10000). --define(ASSEMBLE_TIMEOUT, 300000). - %%-------------------------------------------------------------------- %% API for app %%-------------------------------------------------------------------- @@ -212,7 +209,7 @@ on_init(PacketId, Msg, Transfer, Meta) -> Callback = fun(Result) -> ?MODULE:on_complete("store_filemeta", PacketKey, Transfer, Result) end, - with_responder(PacketKey, Callback, ?STORE_SEGMENT_TIMEOUT, fun() -> + with_responder(PacketKey, Callback, emqx_ft_conf:init_timeout(), fun() -> case store_filemeta(Transfer, Meta) of % Stored, ack through the responder right away ok -> @@ -245,7 +242,7 @@ on_segment(PacketId, Msg, Transfer, Offset, Checksum) -> Callback = fun(Result) -> ?MODULE:on_complete("store_segment", PacketKey, Transfer, Result) end, - with_responder(PacketKey, Callback, ?STORE_SEGMENT_TIMEOUT, fun() -> + with_responder(PacketKey, Callback, emqx_ft_conf:store_segment_timeout(), fun() -> case store_segment(Transfer, Segment) of ok -> emqx_ft_responder:ack(PacketKey, ok); @@ -271,7 +268,7 @@ on_fin(PacketId, Msg, Transfer, FinalSize, Checksum) -> Callback = fun(Result) -> ?MODULE:on_complete("assemble", FinPacketKey, Transfer, Result) end, - with_responder(FinPacketKey, Callback, ?ASSEMBLE_TIMEOUT, fun() -> + with_responder(FinPacketKey, Callback, emqx_ft_conf:assemble_timeout(), fun() -> case assemble(Transfer, FinalSize) of %% Assembling completed, ack through the responder right away % ok -> diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 5ca0b85ed..14a79b94c 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -26,6 +26,9 @@ -export([storage/0]). -export([gc_interval/1]). -export([segments_ttl/1]). +-export([init_timeout/0]). +-export([store_segment_timeout/0]). +-export([assemble_timeout/0]). %% Load/Unload -export([ @@ -86,6 +89,15 @@ assert_storage(Type) -> error({inapplicable, Conf}) end. +init_timeout() -> + emqx_config:get([file_transfer, init_timeout]). + +assemble_timeout() -> + emqx_config:get([file_transfer, assemble_timeout]). + +store_segment_timeout() -> + emqx_config:get([file_transfer, store_segment_timeout]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index beb43adc3..bc780874a 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -46,6 +46,33 @@ roots() -> [file_transfer]. fields(file_transfer) -> [ + {init_timeout, + mk( + emqx_schema:duration_ms(), + #{ + desc => ?DESC("init_timeout"), + required => false, + default => "10s" + } + )}, + {store_segment_timeout, + mk( + emqx_schema:duration_ms(), + #{ + desc => ?DESC("store_segment_timeout"), + required => false, + default => "5m" + } + )}, + {assemble_timeout, + mk( + emqx_schema:duration_ms(), + #{ + desc => ?DESC("assemble_timeout"), + required => false, + default => "5m" + } + )}, {storage, mk( hoconsc:union([ diff --git a/rel/i18n/emqx_ft_schema.hocon b/rel/i18n/emqx_ft_schema.hocon index 15cf17a39..9df02e920 100644 --- a/rel/i18n/emqx_ft_schema.hocon +++ b/rel/i18n/emqx_ft_schema.hocon @@ -1,5 +1,41 @@ emqx_ft_schema { + init_timeout { + desc { + en: "Timeout for initializing the file transfer.
" + "After reaching the timeout, `init` message will be acked with an error" + zh: "" + } + label { + en: "File Transfer Init Timeout" + zh: "" + } + } + + assemble_timeout { + desc { + en: "Timeout for assembling and exporting file segments into a final file.
" + "After reaching the timeout, `fin` message will be acked with an error" + zh: "" + } + label { + en: "File Assemble Timeout" + zh: "" + } + } + + store_segment_timeout { + desc { + en: "Timeout for storing a file segment.
" + "After reaching the timeout, message with the segment will be acked with an error" + zh: "" + } + label { + en: "Store Segment Timeout" + zh: "" + } + } + storage { desc { en: "Storage settings for file transfer." From 04523b3f8153e8c35b80c30e43e385cce8c8c4a5 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 19 Apr 2023 18:48:40 +0300 Subject: [PATCH 123/156] fix(ft): restrict max filename length in transfers Reject transfers with too long filenames right away, during `init` handling, to avoid having to deal with filesystem errors later. --- apps/emqx_ft/src/emqx_ft.erl | 6 +++--- apps/emqx_ft/src/emqx_ft_schema.erl | 15 ++++++++++++++- apps/emqx_ft/test/emqx_ft_SUITE.erl | 11 ++++++++++- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 6ac6596ff..45207baa9 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -110,8 +110,8 @@ decode_filemeta(Map) when is_map(Map) -> Meta = hocon_tconf:check_plain(Schema, Map, #{atom_key => true, required => false}), {ok, Meta} catch - throw:Error -> - {error, {invalid_filemeta, Error}} + throw:{_Schema, Errors} -> + {error, {invalid_filemeta, Errors}} end. encode_filemeta(Meta = #{}) -> @@ -381,7 +381,7 @@ do_validate([{filemeta, Payload} | Rest], Parsed) -> {ok, Meta} -> do_validate(Rest, [Meta | Parsed]); {error, Reason} -> - {error, {invalid_filemeta, Reason}} + {error, Reason} end; do_validate([{offset, Offset} | Rest], Parsed) -> case string:to_integer(Offset) of diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index bc780874a..ce3710cc1 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -35,6 +35,13 @@ -reflect_type([json_value/0]). +%% NOTE +%% This is rather conservative limit, mostly dictated by the filename limitations +%% on most filesystems. Even though, say, S3 does not have such limitations, it's +%% still useful to have a limit on the filename length, to avoid having to deal with +%% limits in the storage backends. +-define(MAX_FILENAME_BYTELEN, 255). + -import(hoconsc, [ref/2, mk/2]). namespace() -> file_transfer. @@ -234,7 +241,13 @@ schema(filemeta) -> }. validator(filename) -> - fun emqx_ft_fs_util:is_filename_safe/1. + [ + fun(Value) -> + Bin = unicode:characters_to_binary(Value), + byte_size(Bin) =< ?MAX_FILENAME_BYTELEN orelse {error, max_length_exceeded} + end, + fun emqx_ft_fs_util:is_filename_safe/1 + ]. converter(checksum) -> fun diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index c3ee7f271..3649d02f8 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -183,9 +183,18 @@ t_invalid_filename(Config) -> unspecified_error, emqtt:publish(C, mk_init_topic(<<"f3">>), encode_meta(meta("/etc/passwd", <<>>)), 1) ), + ?assertRCName( + unspecified_error, + emqtt:publish( + C, + mk_init_topic(<<"f4">>), + encode_meta(meta(lists:duplicate(1000, $A), <<>>)), + 1 + ) + ), ?assertRCName( success, - emqtt:publish(C, mk_init_topic(<<"f4">>), encode_meta(meta("146%", <<>>)), 1) + emqtt:publish(C, mk_init_topic(<<"f5">>), encode_meta(meta("146%", <<>>)), 1) ). t_simple_transfer(Config) -> From f06300cbed896c290c37b370026c6631cfb85a88 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 19 Apr 2023 19:05:12 +0300 Subject: [PATCH 124/156] fix(s3): mark S3 secrets as `sensitive` in schema --- apps/emqx_s3/src/emqx_s3_schema.erl | 9 +++++++-- apps/emqx_s3/test/emqx_s3_schema_SUITE.erl | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index 2d714bb7d..23e69ec5d 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -12,6 +12,7 @@ -export([roots/0, fields/1, namespace/0, tags/0, desc/1]). -export([translate/1]). +-export([translate/2]). roots() -> [s3]. @@ -36,7 +37,8 @@ fields(s3) -> string(), #{ desc => ?DESC("secret_access_key"), - required => false + required => false, + sensitive => true } )}, {bucket, @@ -142,7 +144,10 @@ desc(transport_options) -> "Options for the HTTP transport layer used by the S3 client". translate(Conf) -> - Options = #{atom_key => true}, + translate(Conf, #{}). + +translate(Conf, OptionsIn) -> + Options = maps:merge(#{atom_key => true}, OptionsIn), #{s3 := TranslatedConf} = hocon_tconf:check_plain( emqx_s3_schema, #{<<"s3">> => Conf}, Options, [s3] ), diff --git a/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl b/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl index bba1a5ba8..89ec8a958 100644 --- a/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl @@ -108,6 +108,25 @@ t_full_config(_Config) -> }) ). +t_sensitive_config_hidden(_Config) -> + ?assertMatch( + #{ + access_key_id := "access_key_id", + secret_access_key := <<"******">> + }, + emqx_s3_schema:translate( + #{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"access_key_id">> => <<"access_key_id">>, + <<"secret_access_key">> => <<"secret_access_key">> + }, + % NOTE: this is what Config API handler is doing + #{obfuscate_sensitive_values => true} + ) + ). + t_invalid_limits(_Config) -> ?assertException( throw, From 12a8c070d4b2bec79e613ee386f6fa2746743135 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 21 Apr 2023 17:46:37 +0300 Subject: [PATCH 125/156] feat(ft): fix dependencies in mix.exs --- mix.exs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index f94755b09..d05a1b225 100644 --- a/mix.exs +++ b/mix.exs @@ -184,7 +184,14 @@ defmodule EMQXUmbrella.MixProject do {:brod, github: "kafka4beam/brod", tag: "3.16.8"}, {:snappyer, "1.2.8", override: true}, {:crc32cer, "0.1.8", override: true}, - {:supervisor3, "1.1.12", override: true} + {:supervisor3, "1.1.12", override: true}, + {:erlcloud, github: "emqx/erlcloud", tag: "3.6.8-emqx-1", override: true}, + # erlcloud's rebar.config requires rebar3 and does not support Mix, + # so it tries to fetch deps from git. We need to override this. + {:lhttpc, github: "erlcloud/lhttpc", tag: "1.6.2", override: true}, + {:eini, "1.2.9", override: true}, + {:base16, "1.0.0", override: true} + # end of erlcloud's deps ] end @@ -374,7 +381,8 @@ defmodule EMQXUmbrella.MixProject do emqx_bridge_rocketmq: :permanent, emqx_bridge_tdengine: :permanent, emqx_bridge_timescale: :permanent, - emqx_ee_schema_registry: :permanent + emqx_ee_schema_registry: :permanent, + emqx_ft: :permanent ], else: [] ) From ebb75b275e6509fb7bf864c0802d23f2c74b022e Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 21 Apr 2023 17:50:10 +0300 Subject: [PATCH 126/156] feat(ft): update app versions --- apps/emqx_machine/src/emqx_machine.app.src | 2 +- lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 6bd36aab5..a44d2b36e 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src index 771fdcb27..3df18ce7a 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_conf, [ {description, "EMQX Enterprise Edition configuration schema"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [ kernel, From b951de4c6ea4ef2dd5a61958739b83289169e191 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 21 Apr 2023 19:14:24 +0300 Subject: [PATCH 127/156] feat(ft): convert hocon files to the new format --- rel/i18n/emqx_ft_api.hocon | 24 +---- rel/i18n/emqx_ft_schema.hocon | 197 +++++++--------------------------- rel/i18n/emqx_s3_schema.hocon | 144 ++++++------------------- 3 files changed, 76 insertions(+), 289 deletions(-) diff --git a/rel/i18n/emqx_ft_api.hocon b/rel/i18n/emqx_ft_api.hocon index 6ac01ea23..0c67db554 100644 --- a/rel/i18n/emqx_ft_api.hocon +++ b/rel/i18n/emqx_ft_api.hocon @@ -1,29 +1,13 @@ emqx_ft_api { - file_list { - desc { - en: "List all uploaded files." - zh: "列出所有上传的文件。" - } - label: { - en: "List all uploaded files" - zh: "列出所有上传的文件" - } - } +file_list.desc: +"""List all uploaded files.""" } emqx_ft_storage_exporter_fs_api { - file_get { - desc { - en: "Get a file by its id." - zh: "根据文件 id 获取文件。" - } - label: { - en: "Get a file by its id" - zh: "根据文件 id 获取文件" - } - } +file_get.desc: +"""Get a file by its id.""" } diff --git a/rel/i18n/emqx_ft_schema.hocon b/rel/i18n/emqx_ft_schema.hocon index 9df02e920..28c93e1ef 100644 --- a/rel/i18n/emqx_ft_schema.hocon +++ b/rel/i18n/emqx_ft_schema.hocon @@ -1,177 +1,56 @@ emqx_ft_schema { - init_timeout { - desc { - en: "Timeout for initializing the file transfer.
" - "After reaching the timeout, `init` message will be acked with an error" - zh: "" - } - label { - en: "File Transfer Init Timeout" - zh: "" - } - } +init_timeout.desc: +"""Timeout for initializing the file transfer.
+After reaching the timeout, `init` message will be acked with an error""" - assemble_timeout { - desc { - en: "Timeout for assembling and exporting file segments into a final file.
" - "After reaching the timeout, `fin` message will be acked with an error" - zh: "" - } - label { - en: "File Assemble Timeout" - zh: "" - } - } +assemble_timeout.desc: +"""Timeout for assembling and exporting file segments into a final file.
+After reaching the timeout, `fin` message will be acked with an error""" - store_segment_timeout { - desc { - en: "Timeout for storing a file segment.
" - "After reaching the timeout, message with the segment will be acked with an error" - zh: "" - } - label { - en: "Store Segment Timeout" - zh: "" - } - } +store_segment_timeout.desc: +"""Timeout for storing a file segment.
+After reaching the timeout, message with the segment will be acked with an error""" - storage { - desc { - en: "Storage settings for file transfer." - zh: "文件传输的存储设置。" - } - label: { - en: "Storage settings" - zh: "存储设置" - } - } +storage.desc: +"""Storage settings for file transfer.""" - local_type { - desc { - en: "Use local file system to store uploaded fragments and temporary data." - zh: "使用本地文件系统来存储上传的文件和临时数据。" - } - label: { - en: "Local Storage" - zh: "本地存储" - } - } +local_type.desc: +"""Use local file system to store uploaded fragments and temporary data.""" - local_storage_segments { - desc { - en: "Settings for local segments storage, which include uploaded transfer fragments and temporary data." - zh: "保存上传文件和临时数据的文件系统路径。" - } - label: { - en: "Local Segments Storage" - zh: "本地存储根" - } - } +local_storage_segments.desc: +"""Settings for local segments storage, which include uploaded transfer fragments and temporary data.""" - local_storage_segments_root { - desc { - en: "File system path to keep uploaded fragments and temporary data." - zh: "保存上传文件和临时数据的文件系统路径。" - } - label: { - en: "Local Segments Storage Filesystem Root" - zh: "本地存储根" - } - } +local_storage_segments_root.desc: +"""File system path to keep uploaded fragments and temporary data.""" - local_storage_exporter { - desc { - en: "Exporter for the local file system storage backend.
" - "Exporter defines where and how fully transferred and assembled files are stored." - zh: "" - } - label: { - en: "Local Storage Exporter" - zh: "" - } - } +local_storage_exporter.desc: +"""Exporter for the local file system storage backend.
+Exporter defines where and how fully transferred and assembled files are stored.""" - local_storage_exporter_type { - desc { - en: "Exporter type for the exporter to the local file system" - zh: "" - } - label: { - en: "Local Storage Exporter Type" - zh: "" - } - } +local_storage_exporter_type.desc: +"""Exporter type for the exporter to the local file system""" - s3_exporter_type { - desc { - en: "Exporter type for the exporter to S3" - zh: "" - } - label: { - en: "S3 Exporter Type" - zh: "" - } - } +s3_exporter_type.desc: +"""Exporter type for the exporter to S3""" - local_storage_exporter_root { - desc { - en: "File system path to keep uploaded files." - zh: "" - } - label: { - en: "Local Filesystem Exporter Root" - zh: "" - } - } +local_storage_exporter_root.desc: +"""File system path to keep uploaded files.""" - local_storage_segments_gc { - desc { - en: "Garbage collection settings for the intermediate and temporary files in the local file system." - zh: "" - } - label: { - en: "Local Storage GC" - zh: "" - } - } +local_storage_segments_gc.desc: +"""Garbage collection settings for the intermediate and temporary files in the local file system.""" - storage_gc_interval { - desc { - en: "Interval of periodic garbage collection." - zh: "" - } - label: { - en: "GC Interval" - zh: "" - } - } +storage_gc_interval.desc: +"""Interval of periodic garbage collection.""" - storage_gc_max_segments_ttl { - desc { - en: "Maximum TTL of a segment kept in the local file system.
" - "This is a hard limit: no segment will outlive this TTL, even if some file transfer specifies a " - "TTL more than that." - zh: "" - } - label: { - en: "Max segment TTL" - zh: "" - } - } - - storage_gc_min_segments_ttl { - desc { - en: "Minimum TTL of a segment kept in the local file system.
" - "This is a hard limit: no segment will be garbage collected before reaching this TTL, " - "even if some file transfer specifies a TTL less than that." - zh: "" - } - label: { - en: "Min segment TTL" - zh: "" - } - } +storage_gc_max_segments_ttl.desc: +"""Maximum TTL of a segment kept in the local file system.
+This is a hard limit: no segment will outlive this TTL, even if some file transfer specifies a +TTL more than that.""" +storage_gc_min_segments_ttl.desc: +"""Minimum TTL of a segment kept in the local file system.
+This is a hard limit: no segment will be garbage collected before reaching this TTL, +even if some file transfer specifies a TTL less than that.""" } diff --git a/rel/i18n/emqx_s3_schema.hocon b/rel/i18n/emqx_s3_schema.hocon index c391fa352..df4b973fa 100644 --- a/rel/i18n/emqx_s3_schema.hocon +++ b/rel/i18n/emqx_s3_schema.hocon @@ -1,114 +1,38 @@ emqx_s3_schema { - access_key_id { - desc { - en: "The access key ID of the S3 bucket." - zh: "" - } - label { - en: "Access Key ID" - zh: "" - } - } - secret_access_key { - desc { - en: "The secret access key of the S3 bucket." - zh: "" - } - label { - en: "Secret Access Key" - zh: "" - } - } - bucket { - desc { - en: "The name of the S3 bucket." - zh: "" - } - label { - en: "Bucket" - zh: "" - } - } - host { - desc { - en: "The host of the S3 endpoint." - zh: "" - } - label { - en: "S3 endpoint Host" - zh: "" - } - } - port { - desc { - en: "The port of the S3 endpoint." - zh: "" - } - label { - en: "S3 endpoint port" - zh: "" - } - } - url_expire_time { - desc { - en: "The time in seconds for which the signed URLs to the S3 objects are valid." - zh: "" - } - label { - en: "Signed URL expiration time" - zh: "" - } - } - min_part_size { - desc { - en: """The minimum part size for multipart uploads.
+ +access_key_id.desc: +"""The access key ID of the S3 bucket.""" + +secret_access_key.desc: +"""The secret access key of the S3 bucket.""" + +bucket.desc: +"""The name of the S3 bucket.""" + +host.desc: +"""The host of the S3 endpoint.""" + +port.desc: +"""The port of the S3 endpoint.""" + +url_expire_time.desc: +"""The time in seconds for which the signed URLs to the S3 objects are valid.""" + +min_part_size.desc: +"""The minimum part size for multipart uploads.
Uploaded data will be accumulated in memory until this size is reached.""" - zh: "" - } - label { - en: "Minimum multipart upload part size" - zh: "" - } - } - max_part_size { - desc { - en: """The maximum part size for multipart uploads.
+ +max_part_size.desc: +"""The maximum part size for multipart uploads.
S3 uploader won't try to upload parts larger than this size.""" - zh: "" - } - label { - en: "Maximum multipart upload part size" - zh: "" - } - } - acl { - desc { - en: "The ACL to use for the uploaded objects." - zh: "" - } - label { - en: "ACL" - zh: "" - } - } - transport_options { - desc { - en: "Options for the HTTP transport layer used by the S3 client." - zh: "" - } - label { - en: "Transport Options" - zh: "" - } - } - ipv6_probe { - desc { - en: "Whether to probe for IPv6 support." - zh: "" - } - label { - en: "IPv6 probe" - zh: "" - } - } + +acl.desc: +"""The ACL to use for the uploaded objects.""" + +transport_options.desc: +"""Options for the HTTP transport layer used by the S3 client.""" + +ipv6_probe.desc: +"""Whether to probe for IPv6 support.""" + } From 69c4ba2a62a05f54d4e7dbd8985952d2785ba111 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Sat, 22 Apr 2023 22:50:19 +0300 Subject: [PATCH 128/156] feat(ft): use new utils application --- apps/emqx_ft/src/emqx_ft.erl | 4 ++-- apps/emqx_ft/src/emqx_ft_api.erl | 2 +- apps/emqx_ft/src/emqx_ft_conf.erl | 6 +++--- apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl | 4 ++-- apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl | 4 ++-- apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 8 ++++---- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 4 ++-- apps/emqx_ft/test/emqx_ft_SUITE.erl | 6 +++--- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl | 6 +++--- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 2 +- apps/emqx_s3/src/emqx_s3_profile_conf.erl | 2 +- apps/emqx_s3/test/emqx_s3_client_SUITE.erl | 2 +- apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl | 8 ++++---- apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl | 2 +- 16 files changed, 32 insertions(+), 32 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 45207baa9..95982b849 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -98,7 +98,7 @@ unhook() -> %%-------------------------------------------------------------------- decode_filemeta(Payload) when is_binary(Payload) -> - case emqx_json:safe_decode(Payload, [return_maps]) of + case emqx_utils_json:safe_decode(Payload, [return_maps]) of {ok, Map} -> decode_filemeta(Map); {error, Error} -> @@ -116,7 +116,7 @@ decode_filemeta(Map) when is_map(Map) -> encode_filemeta(Meta = #{}) -> Schema = emqx_ft_schema:schema(filemeta), - hocon_tconf:make_serializable(Schema, emqx_map_lib:binary_key_map(Meta), #{}). + hocon_tconf:make_serializable(Schema, emqx_utils_maps:binary_key_map(Meta), #{}). %%-------------------------------------------------------------------- %% Hooks diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index 1ea710848..85fd8d8cb 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -75,7 +75,7 @@ schema("/file_transfer/files") -> end. error_msg(Code, Msg) -> - #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. + #{code => Code, message => emqx_utils:readable_error_msg(Msg)}. roots() -> []. diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 14a79b94c..dd9806e95 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -63,18 +63,18 @@ storage() -> -spec gc_interval(_Storage) -> milliseconds(). gc_interval(_Storage) -> Conf = assert_storage(local), - emqx_map_lib:deep_get([segments, gc, interval], Conf, ?DEFAULT_GC_INTERVAL). + emqx_utils_maps:deep_get([segments, gc, interval], Conf, ?DEFAULT_GC_INTERVAL). -spec segments_ttl(_Storage) -> {_Min :: seconds(), _Max :: seconds()}. segments_ttl(_Storage) -> Conf = assert_storage(local), { - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [segments, gc, minimum_segments_ttl], Conf, ?DEFAULT_MIN_SEGMENTS_TTL ), - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [segments, gc, maximum_segments_ttl], Conf, ?DEFAULT_MAX_SEGMENTS_TTL diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index ae8315510..4b05c9a58 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -315,10 +315,10 @@ list(_Options) -> -define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). encode_filemeta(Meta) -> - emqx_json:encode(?PRELUDE(_Vsn = 1, emqx_ft:encode_filemeta(Meta))). + emqx_utils_json:encode(?PRELUDE(_Vsn = 1, emqx_ft:encode_filemeta(Meta))). decode_filemeta(Binary) when is_binary(Binary) -> - ?PRELUDE(_Vsn = 1, Map) = emqx_json:decode(Binary, [return_maps]), + ?PRELUDE(_Vsn = 1, Map) = emqx_utils_json:decode(Binary, [return_maps]), case emqx_ft:decode_filemeta(Map) of {ok, Meta} -> Meta; diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl index b7ad86436..f1a8c6dae 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl @@ -134,7 +134,7 @@ fields(file_node) -> end. error_msg(Code, Msg) -> - #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. + #{code => Code, message => emqx_utils:readable_error_msg(Msg)}. -spec mk_export_uri(node(), file:name()) -> uri_string:uri_string(). @@ -150,7 +150,7 @@ mk_export_uri(Node, Filepath) -> %% parse_node(NodeBin) -> - case emqx_misc:safe_to_existing_atom(NodeBin) of + case emqx_utils:safe_to_existing_atom(NodeBin) of {ok, Node} -> Node; {error, _} -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index 259cb4016..adf000346 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -135,7 +135,7 @@ s3_headers({ClientId, FileId}, Filemeta) -> }. s3_header_filemeta(Filemeta) -> - emqx_json:encode(emqx_ft:encode_filemeta(Filemeta), [force_utf8, uescape]). + emqx_utils_json:encode(emqx_ft:encode_filemeta(Filemeta), [force_utf8, uescape]). list(Client, Options) -> case list_key_info(Client, Options) of diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index d871d1f32..1b1d9ecf7 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -271,7 +271,7 @@ read_transferinfo(Storage, Transfer, Acc) -> -spec get_root(storage()) -> file:name(). get_root(Storage) -> - case emqx_map_lib:deep_find([segments, root], Storage) of + case emqx_utils_maps:deep_find([segments, root], Storage) of {ok, Root} -> Root; {not_found, _, _} -> @@ -296,10 +296,10 @@ get_subdirs_for(temporary) -> -define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). encode_filemeta(Meta) -> - emqx_json:encode(?PRELUDE(_Vsn = 1, emqx_ft:encode_filemeta(Meta))). + emqx_utils_json:encode(?PRELUDE(_Vsn = 1, emqx_ft:encode_filemeta(Meta))). decode_filemeta(Binary) when is_binary(Binary) -> - ?PRELUDE(_Vsn = 1, Map) = emqx_json:decode(Binary, [return_maps]), + ?PRELUDE(_Vsn = 1, Map) = emqx_utils_json:decode(Binary, [return_maps]), case emqx_ft:decode_filemeta(Map) of {ok, Meta} -> Meta; @@ -347,7 +347,7 @@ read_file(Filepath, DecodeFun) -> write_file_atomic(Storage, Transfer, Filepath, Content) when is_binary(Content) -> TempFilepath = mk_temp_filepath(Storage, Transfer, filename:basename(Filepath)), - Result = emqx_misc:pipeline( + Result = emqx_utils:pipeline( [ fun filelib:ensure_dir/1, fun write_contents/2, diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 2ab30e88b..0f61e65b7 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -135,7 +135,7 @@ maybe_report(#gcstats{} = _Stats, #st{storage = _Storage}) -> start_timer(St = #st{storage = Storage, next_gc_timer = undefined}) -> case emqx_ft_conf:gc_interval(Storage) of Delay when Delay > 0 -> - St#st{next_gc_timer = emqx_misc:start_timer(Delay, collect)}; + St#st{next_gc_timer = emqx_utils:start_timer(Delay, collect)}; 0 -> ?SLOG(warning, #{msg => "periodic_gc_disabled"}), St @@ -144,7 +144,7 @@ start_timer(St = #st{storage = Storage, next_gc_timer = undefined}) -> reset_timer(St = #st{next_gc_timer = undefined}) -> start_timer(St); reset_timer(St = #st{next_gc_timer = TRef}) -> - ok = emqx_misc:cancel_timer(TRef), + ok = emqx_utils:cancel_timer(TRef), start_timer(St#st{next_gc_timer = undefined}). gc_enabled(St) -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 3649d02f8..151b8e5fe 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -63,7 +63,7 @@ set_special_configs(Config) -> % NOTE % Inhibit local fs GC to simulate it isn't fast enough to collect % complete transfers. - storage => emqx_map_lib:deep_merge( + storage => emqx_utils_maps:deep_merge( Storage, #{segments => #{gc => #{interval => 0}}} ) @@ -325,7 +325,7 @@ t_invalid_meta(Config) -> %% Invalid schema Meta = #{foo => <<"bar">>}, - MetaPayload = emqx_json:encode(Meta), + MetaPayload = emqx_utils_json:encode(Meta), ?assertRCName( unspecified_error, emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) @@ -667,7 +667,7 @@ meta(FileName, Data) -> }. encode_meta(Meta) -> - emqx_json:encode(emqx_ft:encode_filemeta(Meta)). + emqx_utils_json:encode(emqx_ft:encode_filemeta(Meta)). list_files(ClientId) -> {ok, Files} = emqx_ft_storage:files(), diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index a1bef2a2a..5f3b213fb 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -137,7 +137,7 @@ request(Method, Url, Decoder) when is_function(Decoder) -> end. json(Body) when is_binary(Body) -> - emqx_json:decode(Body, [return_maps]). + emqx_utils_json:decode(Body, [return_maps]). query(Params) -> KVs = lists:map(fun({K, V}) -> uri_encode(K) ++ "=" ++ uri_encode(V) end, maps:to_list(Params)), diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index 90039cd96..abf2749ed 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -127,7 +127,7 @@ t_gc_complete_transfers(_Config) -> } ], % 1. Start all transfers - TransferSizes = emqx_misc:pmap( + TransferSizes = emqx_utils:pmap( fun(Transfer) -> start_transfer(Storage, Transfer) end, Transfers ), @@ -162,7 +162,7 @@ t_gc_complete_transfers(_Config) -> ), ?assertEqual( [ok, ok], - emqx_misc:pmap( + emqx_utils:pmap( fun({Transfer, Size}) -> complete_transfer(Storage, Transfer, Size) end, [{T2, S2}, {T3, S3}] ) @@ -221,7 +221,7 @@ t_gc_incomplete_transfers(_Config) -> } ], % 1. Start transfers, send all the segments but don't trigger completion. - _ = emqx_misc:pmap(fun(Transfer) -> start_transfer(Storage, Transfer) end, Transfers), + _ = emqx_utils:pmap(fun(Transfer) -> start_transfer(Storage, Transfer) end, Transfers), % 2. Enable periodic GC every 0.5 seconds. ok = set_gc_config(interval, 500), ok = emqx_ft_storage_fs_gc:reset(Storage), diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index b7004dcc6..704e55454 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -82,7 +82,7 @@ upload_file(ClientId, FileId, Name, Data, Node) -> expire_at => erlang:system_time(_Unit = second) + 3600, size => Size }, - MetaPayload = emqx_json:encode(emqx_ft:encode_filemeta(Meta)), + MetaPayload = emqx_utils_json:encode(emqx_ft:encode_filemeta(Meta)), ct:pal("MetaPayload = ~ts", [MetaPayload]), diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index 3d66823a7..87f006bcb 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -351,7 +351,7 @@ http_config( ]. maybe_ipv6_probe(TransportOpts, true) -> - emqx_misc:ipv6_probe(TransportOpts); + emqx_utils:ipv6_probe(TransportOpts); maybe_ipv6_probe(TransportOpts, false) -> TransportOpts. diff --git a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl index cb55bc083..434510867 100644 --- a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl @@ -159,7 +159,7 @@ profile_config(Config) -> ?config(bucket, Config), ProfileConfig0 ), - ProfileConfig2 = emqx_map_lib:deep_put( + ProfileConfig2 = emqx_utils_maps:deep_put( [transport_options, pool_type], ProfileConfig1, ?config(pool_type, Config) diff --git a/apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl b/apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl index ce53525be..4b70ba015 100644 --- a/apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl @@ -62,7 +62,7 @@ t_regular_outdated_pool_cleanup(Config) -> [OldPool] = emqx_s3_profile_http_pools:all(profile_id()), ProfileBaseConfig = ?config(profile_config, Config), - ProfileConfig = emqx_map_lib:deep_put( + ProfileConfig = emqx_utils_maps:deep_put( [transport_options, pool_size], ProfileBaseConfig, 16 ), ok = emqx_s3:update_profile(profile_id(), ProfileConfig), @@ -110,7 +110,7 @@ t_timeout_pool_cleanup(Config) -> [OldPool] = emqx_s3_profile_http_pools:all(profile_id()), - NewProfileConfig = emqx_map_lib:deep_put( + NewProfileConfig = emqx_utils_maps:deep_put( [transport_options, pool_size], ProfileConfig, 16 ), @@ -153,7 +153,7 @@ t_httpc_pool_update_error(Config) -> meck:expect(ehttpc_pool, init, fun(_) -> meck:raise(error, badarg) end), ProfileBaseConfig = ?config(profile_config, Config), - NewProfileConfig = emqx_map_lib:deep_put( + NewProfileConfig = emqx_utils_maps:deep_put( [transport_options, pool_size], ProfileBaseConfig, 16 ), @@ -237,7 +237,7 @@ t_checkout_client(Config) -> %% Now change config for the profile ProfileBaseConfig = ?config(profile_config, Config), NewProfileConfig0 = ProfileBaseConfig#{bucket => <<"new_bucket">>}, - NewProfileConfig1 = emqx_map_lib:deep_put( + NewProfileConfig1 = emqx_utils_maps:deep_put( [transport_options, pool_size], NewProfileConfig0, 16 ), ok = emqx_s3:update_profile(profile_id(), NewProfileConfig1), diff --git a/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl index e09ea2773..6ba0e3ed9 100644 --- a/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl @@ -526,7 +526,7 @@ t_tls_error(Config) -> _ = process_flag(trap_exit, true), ProfileBaseConfig = ?config(profile_config, Config), - ProfileConfig = emqx_map_lib:deep_put( + ProfileConfig = emqx_utils_maps:deep_put( [transport_options, ssl, server_name_indication], ProfileBaseConfig, "invalid-hostname" ), ok = emqx_s3:update_profile(profile_id(), ProfileConfig), From 4574597175649d8e11308698fcbda39784e221f4 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Sat, 22 Apr 2023 23:39:03 +0300 Subject: [PATCH 129/156] feat(ft): update minirest --- mix.exs | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index d05a1b225..2925c6580 100644 --- a/mix.exs +++ b/mix.exs @@ -58,7 +58,7 @@ defmodule EMQXUmbrella.MixProject do {:ekka, github: "emqx/ekka", tag: "0.14.6", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true}, - {:minirest, github: "emqx/minirest", tag: "1.3.8", override: true}, + {:minirest, github: "emqx/minirest", tag: "1.3.9", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, diff --git a/rebar.config b/rebar.config index de520f124..b1640991e 100644 --- a/rebar.config +++ b/rebar.config @@ -65,7 +65,7 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.8"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.9"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} From 5efd590ca47afddd12282ff064974ed6e4b0b7dc Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 24 Apr 2023 20:28:58 +0300 Subject: [PATCH 130/156] feat(ft): properly propagate config updates Ensure that: * Storage config might be removed. * Local FS GC process is set up when Local FS storage is configured. * Local FS GC process gets its timer reset on config updates. * Storage / exporter gets chosen based on `type` only. * Exporter config updates propagated as before. Also employ `emqx_ft_schema:translate/1` instead of duplicating defaults where applicable. --- apps/emqx/src/emqx_maybe.erl | 3 + apps/emqx_ft/src/emqx_ft.erl | 21 ++- apps/emqx_ft/src/emqx_ft_assembler.erl | 2 +- apps/emqx_ft/src/emqx_ft_conf.erl | 54 +++----- apps/emqx_ft/src/emqx_ft_schema.erl | 71 +++++++--- apps/emqx_ft/src/emqx_ft_storage.erl | 98 +++++++++++--- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 46 +++---- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 9 ++ apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 125 ++++++++++-------- apps/emqx_ft/src/emqx_ft_sup.erl | 2 +- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 28 ++-- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 84 ++++++++++++ .../test/emqx_ft_storage_fs_gc_SUITE.erl | 12 +- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 15 ++- 14 files changed, 376 insertions(+), 194 deletions(-) diff --git a/apps/emqx/src/emqx_maybe.erl b/apps/emqx/src/emqx_maybe.erl index 0b919f7ab..2629bc737 100644 --- a/apps/emqx/src/emqx_maybe.erl +++ b/apps/emqx/src/emqx_maybe.erl @@ -23,6 +23,9 @@ -export([define/2]). -export([apply/2]). +-type t(T) :: maybe(T). +-export_type([t/1]). + -spec to_list(maybe(A)) -> [A]. to_list(undefined) -> []; diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 95982b849..d500a6344 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -301,8 +301,8 @@ store_filemeta(Transfer, Segment) -> emqx_ft_storage:store_filemeta(Transfer, Segment) catch C:E:S -> - ?SLOG(error, #{ - msg => "start_store_filemeta_failed", class => C, reason => E, stacktrace => S + ?tp(error, "start_store_filemeta_failed", #{ + class => C, reason => E, stacktrace => S }), {error, {internal_error, E}} end. @@ -312,8 +312,8 @@ store_segment(Transfer, Segment) -> emqx_ft_storage:store_segment(Transfer, Segment) catch C:E:S -> - ?SLOG(error, #{ - msg => "start_store_segment_failed", class => C, reason => E, stacktrace => S + ?tp(error, "start_store_segment_failed", #{ + class => C, reason => E, stacktrace => S }), {error, {internal_error, E}} end. @@ -323,8 +323,8 @@ assemble(Transfer, FinalSize) -> emqx_ft_storage:assemble(Transfer, FinalSize) catch C:E:S -> - ?SLOG(error, #{ - msg => "start_assemble_failed", class => C, reason => E, stacktrace => S + ?tp(error, "start_assemble_failed", #{ + class => C, reason => E, stacktrace => S }), {error, {internal_error, E}} end. @@ -334,8 +334,7 @@ transfer(Msg, FileId) -> {clientid_to_binary(ClientId), FileId}. on_complete(Op, {ChanPid, PacketId}, Transfer, Result) -> - ?SLOG(debug, #{ - msg => "on_complete", + ?tp(debug, "on_complete", #{ operation => Op, packet_id => PacketId, transfer => Transfer @@ -344,15 +343,13 @@ on_complete(Op, {ChanPid, PacketId}, Transfer, Result) -> {Mode, ok} when Mode == ack orelse Mode == down -> erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS}); {Mode, {error, _} = Reason} when Mode == ack orelse Mode == down -> - ?SLOG(error, #{ - msg => Op ++ "_failed", + ?tp(error, Op ++ "_failed", #{ transfer => Transfer, reason => Reason }), erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}); timeout -> - ?SLOG(error, #{ - msg => Op ++ "_timed_out", + ?tp(error, Op ++ "_timed_out", #{ transfer => Transfer }), erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 3a352cd10..767930f98 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -159,7 +159,7 @@ handle_event(internal, _, {assemble, []}, St = #{}) -> {next_state, complete, St, ?internal([])}; handle_event(internal, _, complete, St = #{export := Export}) -> Result = emqx_ft_storage_exporter:complete(Export), - ok = maybe_garbage_collect(Result, St), + _ = maybe_garbage_collect(Result, St), {stop, {shutdown, Result}, maps:remove(export, St)}. -spec terminate(_Reason, state(), stdata()) -> _. diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index dd9806e95..1e531ecdb 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -45,49 +45,28 @@ -type milliseconds() :: non_neg_integer(). -type seconds() :: non_neg_integer(). -%% 5 minutes (s) --define(DEFAULT_MIN_SEGMENTS_TTL, 300). -%% 1 day (s) --define(DEFAULT_MAX_SEGMENTS_TTL, 86400). -%% 1 minute (ms) --define(DEFAULT_GC_INTERVAL, 60000). - %%-------------------------------------------------------------------- %% Accessors %%-------------------------------------------------------------------- -spec storage() -> _Storage. storage() -> - emqx_config:get([file_transfer, storage]). + emqx_config:get([file_transfer, storage], undefined). --spec gc_interval(_Storage) -> milliseconds(). -gc_interval(_Storage) -> - Conf = assert_storage(local), - emqx_utils_maps:deep_get([segments, gc, interval], Conf, ?DEFAULT_GC_INTERVAL). +-spec gc_interval(_Storage) -> emqx_maybe:t(milliseconds()). +gc_interval(Conf = #{type := local}) -> + emqx_utils_maps:deep_get([segments, gc, interval], Conf); +gc_interval(_) -> + undefined. --spec segments_ttl(_Storage) -> {_Min :: seconds(), _Max :: seconds()}. -segments_ttl(_Storage) -> - Conf = assert_storage(local), +-spec segments_ttl(_Storage) -> emqx_maybe:t({_Min :: seconds(), _Max :: seconds()}). +segments_ttl(Conf = #{type := local}) -> { - emqx_utils_maps:deep_get( - [segments, gc, minimum_segments_ttl], - Conf, - ?DEFAULT_MIN_SEGMENTS_TTL - ), - emqx_utils_maps:deep_get( - [segments, gc, maximum_segments_ttl], - Conf, - ?DEFAULT_MAX_SEGMENTS_TTL - ) - }. - -assert_storage(Type) -> - case storage() of - Conf = #{type := Type} -> - Conf; - Conf -> - error({inapplicable, Conf}) - end. + emqx_utils_maps:deep_get([segments, gc, minimum_segments_ttl], Conf), + emqx_utils_maps:deep_get([segments, gc, maximum_segments_ttl], Conf) + }; +segments_ttl(_) -> + undefined. init_timeout() -> emqx_config:get([file_transfer, init_timeout]). @@ -104,10 +83,7 @@ store_segment_timeout() -> -spec load() -> ok. load() -> - ok = emqx_ft_storage_exporter:update_exporter( - undefined, - storage() - ), + ok = emqx_ft_storage:on_config_update(undefined, storage()), emqx_conf:add_handler([file_transfer], ?MODULE). -spec unload() -> ok. @@ -134,4 +110,4 @@ pre_config_update(_, Req, _Config) -> post_config_update(_Path, _Req, NewConfig, OldConfig, _AppEnvs) -> OldStorageConfig = maps:get(storage, OldConfig, undefined), NewStorageConfig = maps:get(storage, NewConfig, undefined), - emqx_ft_storage_exporter:update_exporter(OldStorageConfig, NewStorageConfig). + emqx_ft_storage:on_config_update(OldStorageConfig, NewStorageConfig). diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index ce3710cc1..27c593b6c 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -25,6 +25,8 @@ -export([schema/1]). +-export([translate/1]). + -type json_value() :: null | boolean() @@ -82,13 +84,25 @@ fields(file_transfer) -> )}, {storage, mk( - hoconsc:union([ - ref(local_storage) - ]), + hoconsc:union( + fun + (all_union_members) -> + [ + % NOTE: by default storage is disabled + undefined, + ref(local_storage) + ]; + ({value, #{<<"type">> := <<"local">>}}) -> + [ref(local_storage)]; + ({value, #{<<"type">> := _}}) -> + throw(#{field_name => type, expected => "local"}); + (_) -> + [undefined] + end + ), #{ required => false, - desc => ?DESC("storage"), - default => default_storage() + desc => ?DESC("storage") } )} ]; @@ -108,18 +122,38 @@ fields(local_storage) -> ref(local_storage_segments), #{ desc => ?DESC("local_storage_segments"), - required => false + required => false, + default => #{ + <<"gc">> => #{} + } } )}, {exporter, mk( - hoconsc:union([ - ref(local_storage_exporter), - ref(s3_exporter) - ]), + hoconsc:union( + fun + (all_union_members) -> + [ + ref(local_storage_exporter), + ref(s3_exporter) + ]; + ({value, #{<<"type">> := <<"local">>}}) -> + [ref(local_storage_exporter)]; + ({value, #{<<"type">> := <<"s3">>}}) -> + [ref(s3_exporter)]; + ({value, #{<<"type">> := _}}) -> + throw(#{field_name => type, expected => "local | s3"}); + ({value, _}) -> + % NOTE: default + [ref(local_storage_exporter)] + end + ), #{ desc => ?DESC("local_storage_exporter"), - required => true + required => true, + default => #{ + <<"type">> => <<"local">> + } } )} ]; @@ -277,10 +311,11 @@ converter(unicode_string) -> ref(Ref) -> ref(?MODULE, Ref). -default_storage() -> - #{ - <<"type">> => <<"local">>, - <<"exporter">> => #{ - <<"type">> => <<"local">> - } - }. +translate(Conf) -> + [Root] = roots(), + maps:get( + Root, + hocon_tconf:check_plain( + ?MODULE, #{atom_to_binary(Root) => Conf}, #{atom_key => true}, [Root] + ) + ). diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 79538a9c7..fee16cd09 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -27,7 +27,9 @@ files/0, with_storage_type/2, - with_storage_type/3 + with_storage_type/3, + + on_config_update/2 ] ). @@ -90,26 +92,35 @@ child_spec() -> -spec store_filemeta(emqx_ft:transfer(), emqx_ft:filemeta()) -> ok | {async, pid()} | {error, term()}. store_filemeta(Transfer, FileMeta) -> - Mod = mod(), - Mod:store_filemeta(storage(), Transfer, FileMeta). + with_storage(store_filemeta, [Transfer, FileMeta]). -spec store_segment(emqx_ft:transfer(), emqx_ft:segment()) -> ok | {async, pid()} | {error, term()}. store_segment(Transfer, Segment) -> - Mod = mod(), - Mod:store_segment(storage(), Transfer, Segment). + with_storage(store_segment, [Transfer, Segment]). -spec assemble(emqx_ft:transfer(), emqx_ft:bytes()) -> ok | {async, pid()} | {error, term()}. assemble(Transfer, Size) -> - Mod = mod(), - Mod:assemble(storage(), Transfer, Size). + with_storage(assemble, [Transfer, Size]). -spec files() -> {ok, [file_info()]} | {error, term()}. files() -> - Mod = mod(), - Mod:files(storage()). + with_storage(files, []). + +-spec with_storage(atom() | function()) -> any(). +with_storage(Fun) -> + with_storage(Fun, []). + +-spec with_storage(atom() | function(), list(term())) -> any(). +with_storage(Fun, Args) -> + case storage() of + Storage = #{} -> + apply_storage(Storage, Fun, Args); + undefined -> + {error, disabled} + end. -spec with_storage_type(atom(), atom() | function()) -> any(). with_storage_type(Type, Fun) -> @@ -117,17 +128,61 @@ with_storage_type(Type, Fun) -> -spec with_storage_type(atom(), atom() | function(), list(term())) -> any(). with_storage_type(Type, Fun, Args) -> - Storage = storage(), - case Storage of - #{type := Type} when is_function(Fun) -> - apply(Fun, [Storage | Args]); - #{type := Type} when is_atom(Fun) -> - Mod = mod(Storage), - apply(Mod, Fun, [Storage | Args]); - disabled -> - {error, disabled}; - _ -> - {error, {invalid_storage_type, Type}} + with_storage(fun(Storage) -> + case Storage of + #{type := Type} -> + apply_storage(Storage, Fun, Args); + _ -> + {error, {invalid_storage_type, Storage}} + end + end). + +apply_storage(Storage, Fun, Args) when is_atom(Fun) -> + apply(mod(Storage), Fun, [Storage | Args]); +apply_storage(Storage, Fun, Args) when is_function(Fun) -> + apply(Fun, [Storage | Args]). + +%% + +-spec on_config_update(_Old :: emqx_maybe:t(storage()), _New :: emqx_maybe:t(storage())) -> + ok. +on_config_update(Storage, Storage) -> + ok; +on_config_update(#{type := Type} = StorageOld, #{type := Type} = StorageNew) -> + ok = (mod(StorageNew)):on_config_update(StorageOld, StorageNew); +on_config_update(StorageOld, StorageNew) -> + _ = emqx_maybe:apply(fun on_storage_stop/1, StorageOld), + _ = emqx_maybe:apply(fun on_storage_start/1, StorageNew), + _ = emqx_maybe:apply( + fun(Storage) -> (mod(Storage)):on_config_update(StorageOld, StorageNew) end, + StorageNew + ), + ok. + +on_storage_start(Storage = #{type := _}) -> + lists:foreach( + fun(ChildSpec) -> + {ok, _Child} = supervisor:start_child(emqx_ft_sup, ChildSpec) + end, + child_spec(Storage) + ). + +on_storage_stop(Storage = #{type := _}) -> + lists:foreach( + fun(#{id := ChildId}) -> + _ = supervisor:terminate_child(emqx_ft_sup, ChildId), + ok = supervisor:delete_child(emqx_ft_sup, ChildId) + end, + child_spec(Storage) + ). + +child_spec(Storage) -> + try + Mod = mod(Storage), + Mod:child_spec(Storage) + catch + error:disabled -> []; + error:undef -> [] end. %%-------------------------------------------------------------------- @@ -144,7 +199,6 @@ mod(Storage) -> case Storage of #{type := local} -> emqx_ft_storage_fs; - disabled -> + undefined -> error(disabled) - % emqx_ft_storage_dummy end. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index 61241bd6f..72128cb40 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -31,7 +31,7 @@ -export([list/1]). %% Lifecycle API --export([update_exporter/2]). +-export([on_config_update/2]). %% Internal API -export([exporter/1]). @@ -141,30 +141,28 @@ list(Storage) -> %% Lifecycle --spec update_exporter(emqx_config:config(), emqx_config:config()) -> ok | {error, term()}. -update_exporter( - #{exporter := #{type := OldType}} = OldConfig, - #{exporter := #{type := OldType}} = NewConfig -) -> - {ExporterMod, OldExporterOpts} = exporter(OldConfig), - {_NewExporterMod, NewExporterOpts} = exporter(NewConfig), - ExporterMod:update(OldExporterOpts, NewExporterOpts); -update_exporter( - #{exporter := _} = OldConfig, - #{exporter := _} = NewConfig -) -> - {OldExporterMod, OldExporterOpts} = exporter(OldConfig), - {NewExporterMod, NewExporterOpts} = exporter(NewConfig), - ok = OldExporterMod:stop(OldExporterOpts), - NewExporterMod:start(NewExporterOpts); -update_exporter(undefined, NewConfig) -> - {ExporterMod, ExporterOpts} = exporter(NewConfig), - ExporterMod:start(ExporterOpts); -update_exporter(OldConfig, undefined) -> - {ExporterMod, ExporterOpts} = exporter(OldConfig), - ExporterMod:stop(ExporterOpts); -update_exporter(_, _) -> +-spec on_config_update(storage(), storage()) -> ok | {error, term()}. +on_config_update(StorageOld, StorageNew) -> + on_exporter_update( + emqx_maybe:apply(fun exporter/1, StorageOld), + emqx_maybe:apply(fun exporter/1, StorageNew) + ). + +on_exporter_update(Config, Config) -> + ok; +on_exporter_update({ExporterMod, ConfigOld}, {ExporterMod, ConfigNew}) -> + ExporterMod:update(ConfigOld, ConfigNew); +on_exporter_update(ExporterOld, ExporterNew) -> + _ = emqx_maybe:apply(fun stop_exporter/1, ExporterOld), + _ = emqx_maybe:apply(fun start_exporter/1, ExporterNew), ok. + +start_exporter({ExporterMod, ExporterOpts}) -> + ok = ExporterMod:start(ExporterOpts). + +stop_exporter({ExporterMod, ExporterOpts}) -> + ok = ExporterMod:stop(ExporterOpts). + %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 1b1d9ecf7..5754d2cfc 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -47,6 +47,8 @@ -export([files/1]). +-export([on_config_update/2]). + -export_type([storage/0]). -export_type([filefrag/1]). -export_type([filefrag/0]). @@ -219,6 +221,13 @@ files(Storage) -> %% +on_config_update(StorageOld, StorageNew) -> + % NOTE: this will reset GC timer, frequent changes would postpone GC indefinitely + ok = emqx_ft_storage_fs_gc:reset(StorageNew), + emqx_ft_storage_exporter:on_config_update(StorageOld, StorageNew). + +%% + -spec transfers(storage()) -> {ok, #{transfer() => transferinfo()}}. transfers(Storage) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 0f61e65b7..692a270e3 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -29,8 +29,9 @@ -export([start_link/1]). --export([collect/1]). +-export([collect/0]). -export([collect/3]). +-export([reset/0]). -export([reset/1]). -behaviour(gen_server). @@ -40,38 +41,43 @@ -export([handle_info/2]). -record(st, { - storage :: emqx_ft_storage_fs:storage(), next_gc_timer :: maybe(reference()), last_gc :: maybe(gcstats()) }). -type gcstats() :: #gcstats{}. +-define(IS_ENABLED(INTERVAL), (is_integer(INTERVAL) andalso INTERVAL > 0)). + %% start_link(Storage) -> - gen_server:start_link(mk_server_ref(Storage), ?MODULE, Storage, []). + gen_server:start_link(mk_server_ref(global), ?MODULE, Storage, []). --spec collect(emqx_ft_storage_fs:storage()) -> gcstats(). -collect(Storage) -> - gen_server:call(mk_server_ref(Storage), {collect, erlang:system_time()}, infinity). +-spec collect() -> gcstats(). +collect() -> + gen_server:call(mk_server_ref(global), {collect, erlang:system_time()}, infinity). + +-spec reset() -> ok. +reset() -> + reset(emqx_ft_conf:storage()). -spec reset(emqx_ft_storage_fs:storage()) -> ok. reset(Storage) -> - gen_server:cast(mk_server_ref(Storage), reset). + gen_server:cast(mk_server_ref(global), {reset, gc_interval(Storage)}). collect(Storage, Transfer, Nodes) -> - gen_server:cast(mk_server_ref(Storage), {collect, Transfer, Nodes}). + gc_enabled(Storage) andalso cast_collect(mk_server_ref(global), Storage, Transfer, Nodes). -mk_server_ref(Storage) -> +mk_server_ref(Name) -> % TODO - {via, gproc, {n, l, {?MODULE, get_segments_root(Storage)}}}. + {via, gproc, {n, l, {?MODULE, Name}}}. %% init(Storage) -> - St = #st{storage = Storage}, - {ok, start_timer(St)}. + St = #st{}, + {ok, start_timer(gc_interval(Storage), St)}. handle_call({collect, CalledAt}, _From, St) -> StNext = maybe_collect_garbage(CalledAt, St), @@ -80,22 +86,17 @@ handle_call(Call, From, St) -> ?SLOG(error, #{msg => "unexpected_call", call => Call, from => From}), {noreply, St}. -handle_cast({collect, Transfer, [Node | Rest]}, St) -> - case gc_enabled(St) of - true -> - ok = do_collect_transfer(Transfer, Node, St), - case Rest of - [_ | _] -> - gen_server:cast(self(), {collect, Transfer, Rest}); - [] -> - ok - end; - false -> - skip +handle_cast({collect, Storage, Transfer, [Node | Rest]}, St) -> + ok = do_collect_transfer(Storage, Transfer, Node, St), + case Rest of + [_ | _] -> + cast_collect(self(), Storage, Transfer, Rest); + [] -> + ok end, {noreply, St}; -handle_cast(reset, St) -> - {noreply, reset_timer(St)}; +handle_cast({reset, Interval}, St) -> + {noreply, start_timer(Interval, cancel_timer(St))}; handle_cast(Cast, St) -> ?SLOG(error, #{msg => "unexpected_cast", cast => Cast}), {noreply, St}. @@ -104,14 +105,17 @@ handle_info({timeout, TRef, collect}, St = #st{next_gc_timer = TRef}) -> StNext = do_collect_garbage(St), {noreply, start_timer(StNext#st{next_gc_timer = undefined})}. -do_collect_transfer(Transfer, Node, St = #st{storage = Storage}) when Node == node() -> +do_collect_transfer(Storage, Transfer, Node, St = #st{}) when Node == node() -> Stats = try_collect_transfer(Storage, Transfer, complete, init_gcstats()), ok = maybe_report(Stats, St), ok; -do_collect_transfer(_Transfer, _Node, _St = #st{}) -> +do_collect_transfer(_Storage, _Transfer, _Node, _St = #st{}) -> % TODO ok. +cast_collect(Ref, Storage, Transfer, Nodes) -> + gen_server:cast(Ref, {collect, Storage, Transfer, Nodes}). + maybe_collect_garbage(_CalledAt, St = #st{last_gc = undefined}) -> do_collect_garbage(St); maybe_collect_garbage(CalledAt, St = #st{last_gc = #gcstats{finished_at = FinishedAt}}) -> @@ -119,36 +123,41 @@ maybe_collect_garbage(CalledAt, St = #st{last_gc = #gcstats{finished_at = Finish true -> St; false -> - reset_timer(do_collect_garbage(St)) + start_timer(do_collect_garbage(cancel_timer(St))) end. -do_collect_garbage(St = #st{storage = Storage}) -> - Stats = collect_garbage(Storage), - ok = maybe_report(Stats, St), - St#st{last_gc = Stats}. +do_collect_garbage(St = #st{}) -> + emqx_ft_storage:with_storage_type(local, fun(Storage) -> + Stats = collect_garbage(Storage), + ok = maybe_report(Stats, Storage), + St#st{last_gc = Stats} + end). -maybe_report(#gcstats{errors = Errors}, #st{storage = Storage}) when map_size(Errors) > 0 -> +maybe_report(#gcstats{errors = Errors}, Storage) when map_size(Errors) > 0 -> ?tp(warning, "garbage_collection_errors", #{errors => Errors, storage => Storage}); -maybe_report(#gcstats{} = _Stats, #st{storage = _Storage}) -> +maybe_report(#gcstats{} = _Stats, _Storage) -> ?tp(garbage_collection, #{stats => _Stats, storage => _Storage}). -start_timer(St = #st{storage = Storage, next_gc_timer = undefined}) -> - case emqx_ft_conf:gc_interval(Storage) of - Delay when Delay > 0 -> - St#st{next_gc_timer = emqx_utils:start_timer(Delay, collect)}; - 0 -> - ?SLOG(warning, #{msg => "periodic_gc_disabled"}), - St - end. +start_timer(St) -> + start_timer(gc_interval(emqx_ft_conf:storage()), St). -reset_timer(St = #st{next_gc_timer = undefined}) -> - start_timer(St); -reset_timer(St = #st{next_gc_timer = TRef}) -> +start_timer(Interval, St = #st{next_gc_timer = undefined}) when ?IS_ENABLED(Interval) -> + St#st{next_gc_timer = emqx_utils:start_timer(Interval, collect)}; +start_timer(Interval, St) -> + ?SLOG(warning, #{msg => "periodic_gc_disabled", interval => Interval}), + St. + +cancel_timer(St = #st{next_gc_timer = undefined}) -> + St; +cancel_timer(St = #st{next_gc_timer = TRef}) -> ok = emqx_utils:cancel_timer(TRef), - start_timer(St#st{next_gc_timer = undefined}). + St#st{next_gc_timer = undefined}. -gc_enabled(St) -> - emqx_ft_conf:gc_interval(St#st.storage) > 0. +gc_enabled(Storage) -> + ?IS_ENABLED(gc_interval(Storage)). + +gc_interval(Storage) -> + emqx_ft_conf:gc_interval(Storage). %% @@ -175,8 +184,13 @@ try_collect_transfer(Storage, Transfer, TransferInfo = #{}, Stats) -> % heuristic we only delete transfer directory itself only if it is also outdated % _and was empty at the start of GC_, as a precaution against races between % writers and GCs. - TTL = get_segments_ttl(Storage, TransferInfo), - Cutoff = erlang:system_time(second) - TTL, + Cutoff = + case get_segments_ttl(Storage, TransferInfo) of + TTL when is_integer(TTL) -> + erlang:system_time(second) - TTL; + undefined -> + 0 + end, {FragCleaned, Stats1} = collect_outdated_fragments(Storage, Transfer, Cutoff, Stats), {TempCleaned, Stats2} = collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats1), % TODO: collect empty directories separately @@ -338,16 +352,17 @@ filepath_to_binary(S) -> unicode:characters_to_binary(S, unicode, file:native_name_encoding()). get_segments_ttl(Storage, TransferInfo) -> - {MinTTL, MaxTTL} = emqx_ft_conf:segments_ttl(Storage), - clamp(MinTTL, MaxTTL, try_get_filemeta_ttl(TransferInfo)). + clamp(emqx_ft_conf:segments_ttl(Storage), try_get_filemeta_ttl(TransferInfo)). try_get_filemeta_ttl(#{filemeta := Filemeta}) -> maps:get(segments_ttl, Filemeta, undefined); try_get_filemeta_ttl(#{}) -> undefined. -clamp(Min, Max, V) -> - min(Max, max(Min, V)). +clamp({Min, Max}, V) -> + min(Max, max(Min, V)); +clamp(undefined, V) -> + V. %% diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index 3c28eae30..8d388814c 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -61,5 +61,5 @@ init([]) -> modules => [emqx_ft_responder_sup] }, - ChildSpecs = [Responder, AssemblerSup, FileReaderSup | emqx_ft_storage:child_spec()], + ChildSpecs = [Responder, AssemblerSup, FileReaderSup], {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 4b5610f51..a7323fc0e 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -46,8 +46,8 @@ init_per_testcase(TC, Config) -> ok = snabbkaffe:start_trace(), {ok, Pid} = emqx_ft_assembler_sup:start_link(), [ - {storage_root, "file_transfer_root"}, - {exports_root, "file_transfer_exports"}, + {storage_root, <<"file_transfer_root">>}, + {exports_root, <<"file_transfer_exports">>}, {file_id, atom_to_binary(TC)}, {assembler_sup, Pid} | Config @@ -246,13 +246,17 @@ exporter(Config) -> emqx_ft_storage_exporter:exporter(storage(Config)). storage(Config) -> - #{ - type => local, - segments => #{ - root => ?config(storage_root, Config) - }, - exporter => #{ - type => local, - root => ?config(exports_root, Config) - } - }. + maps:get( + storage, + emqx_ft_schema:translate(#{ + <<"storage">> => #{ + <<"type">> => <<"local">>, + <<"segments">> => #{ + <<"root">> => ?config(storage_root, Config) + }, + <<"exporter">> => #{ + <<"root">> => ?config(exports_root, Config) + } + } + }) + ). diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index c616681dd..89b0e895d 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -21,6 +21,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). +-include_lib("snabbkaffe/include/test_macros.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). @@ -85,3 +86,86 @@ t_update_config(_Config) -> 5 * 60 * 1000, emqx_ft_conf:gc_interval(emqx_ft_conf:storage()) ). + +t_remove_restore_config(Config) -> + ?assertMatch( + {ok, _}, + emqx_conf:update([file_transfer, storage], #{<<"type">> => <<"local">>}, #{}) + ), + ?assertEqual( + 60 * 60 * 1000, + emqx_ft_conf:gc_interval(emqx_ft_conf:storage()) + ), + % Verify that transfers work + ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <>), + ?assertMatch( + {ok, _}, + emqx_conf:remove([file_transfer, storage], #{}) + ), + ?assertEqual( + undefined, + emqx_ft_conf:storage() + ), + ?assertEqual( + undefined, + emqx_ft_conf:gc_interval(emqx_ft_conf:storage()) + ), + ClientId = gen_clientid(), + Client = emqx_ft_test_helpers:start_client(ClientId), + % Verify that transfers fail cleanly when storage is disabled + ?check_trace( + ?assertMatch( + {ok, #{reason_code_name := unspecified_error}}, + emqtt:publish( + Client, + <<"$file/f2/init">>, + emqx_utils_json:encode(emqx_ft:encode_filemeta(#{name => "f2", size => 42})), + 1 + ) + ), + fun(Trace) -> + ?assertMatch( + [#{transfer := {ClientId, <<"f2">>}, reason := {error, disabled}}], + ?of_kind("store_filemeta_failed", Trace) + ) + end + ), + % Restore local storage backend + Root = iolist_to_binary(emqx_ft_test_helpers:root(Config, node(), [segments])), + ?assertMatch( + {ok, _}, + emqx_conf:update( + [file_transfer, storage], + #{ + <<"type">> => <<"local">>, + <<"segments">> => #{ + <<"root">> => Root, + <<"gc">> => #{<<"interval">> => <<"1s">>} + } + }, + #{} + ) + ), + % Verify that GC is getting triggered eventually + ?check_trace( + ?block_until(#{?snk_kind := garbage_collection}, 5000, 0), + fun(Trace) -> + ?assertMatch( + [ + #{ + ?snk_kind := garbage_collection, + storage := #{ + type := local, + segments := #{root := Root} + } + } + ], + ?of_kind(garbage_collection, Trace) + ) + end + ), + % Verify that transfers work again + ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <>). + +gen_clientid() -> + emqx_base62:encode(emqx_guid:gen()). diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index abf2749ed..065e9ae0a 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -68,7 +68,7 @@ end_per_testcase(_TC, _Config) -> t_gc_triggers_periodically(_Config) -> Interval = 500, ok = set_gc_config(interval, Interval), - ok = emqx_ft_storage_fs_gc:reset(emqx_ft_conf:storage()), + ok = emqx_ft_storage_fs_gc:reset(), ?check_trace( timer:sleep(Interval * 3), fun(Trace) -> @@ -92,7 +92,7 @@ t_gc_triggers_manually(_Config) -> ?assertMatch( #gcstats{files = 0, directories = 0, space = 0, errors = #{} = Errors} when map_size(Errors) == 0, - emqx_ft_storage_fs_gc:collect(emqx_ft_conf:storage()) + emqx_ft_storage_fs_gc:collect() ), fun(Trace) -> [Event] = ?of_kind(garbage_collection, Trace), @@ -108,7 +108,7 @@ t_gc_complete_transfers(_Config) -> ok = set_gc_config(minimum_segments_ttl, 0), ok = set_gc_config(maximum_segments_ttl, 3), ok = set_gc_config(interval, 500), - ok = emqx_ft_storage_fs_gc:reset(Storage), + ok = emqx_ft_storage_fs_gc:reset(), Transfers = [ { T1 = {<<"client1">>, mk_file_id()}, @@ -134,7 +134,7 @@ t_gc_complete_transfers(_Config) -> ?assertEqual([S1, S2, S3], TransferSizes), ?assertMatch( #gcstats{files = 0, directories = 0, errors = #{} = Es} when map_size(Es) == 0, - emqx_ft_storage_fs_gc:collect(Storage) + emqx_ft_storage_fs_gc:collect() ), % 2. Complete just the first transfer {ok, {ok, Event}} = ?wait_async_action( @@ -224,7 +224,7 @@ t_gc_incomplete_transfers(_Config) -> _ = emqx_utils:pmap(fun(Transfer) -> start_transfer(Storage, Transfer) end, Transfers), % 2. Enable periodic GC every 0.5 seconds. ok = set_gc_config(interval, 500), - ok = emqx_ft_storage_fs_gc:reset(Storage), + ok = emqx_ft_storage_fs_gc:reset(), % 3. First we need the first transfer to be collected. {ok, _} = ?block_until( #{ @@ -304,7 +304,7 @@ t_gc_handling_errors(_Config) -> {directory, DirTransfer2} := eexist } } when Files == ?NSEGS(Size, SegSize) * 2 andalso Space > Size * 2, - emqx_ft_storage_fs_gc:collect(Storage) + emqx_ft_storage_fs_gc:collect() ), fun(Trace) -> ?assertMatch( diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index 704e55454..11ddf191b 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -68,15 +68,22 @@ tcp_port(Node) -> root(Config, Node, Tail) -> filename:join([?config(priv_dir, Config), "file_transfer", Node | Tail]). +start_client(ClientId) -> + start_client(ClientId, node()). + +start_client(ClientId, Node) -> + Port = tcp_port(Node), + {ok, Client} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, Port}]), + {ok, _} = emqtt:connect(Client), + Client. + upload_file(ClientId, FileId, Name, Data) -> upload_file(ClientId, FileId, Name, Data, node()). upload_file(ClientId, FileId, Name, Data, Node) -> - Port = tcp_port(Node), - Size = byte_size(Data), + C1 = start_client(ClientId, Node), - {ok, C1} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, Port}]), - {ok, _} = emqtt:connect(C1), + Size = byte_size(Data), Meta = #{ name => Name, expire_at => erlang:system_time(_Unit = second) + 3600, From a420c92d28562bb27be0084789eaaf2543398f9b Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 24 Apr 2023 20:37:50 +0300 Subject: [PATCH 131/156] fix(ft): fix typing issue --- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 5754d2cfc..823407307 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -98,6 +98,7 @@ }. -type storage() :: #{ + type := 'local', segments := segments(), exporter := emqx_ft_storage_exporter:exporter() }. From 811e449357dda9d2b874459c29c5d86e8c2997da Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 25 Apr 2023 19:16:14 +0300 Subject: [PATCH 132/156] feat(ft-conf): provide global killswitch --- apps/emqx_ft/src/emqx_ft.erl | 9 +- apps/emqx_ft/src/emqx_ft_app.erl | 2 - apps/emqx_ft/src/emqx_ft_conf.erl | 34 ++++- apps/emqx_ft/src/emqx_ft_schema.erl | 22 ++-- apps/emqx_ft/test/emqx_ft_SUITE.erl | 1 + apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 12 +- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 117 ++++++++++++++---- .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 12 +- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 2 +- rel/i18n/emqx_ft_schema.hocon | 6 + 10 files changed, 148 insertions(+), 69 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index d500a6344..42611e537 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -198,8 +198,7 @@ on_file_command(PacketId, FileId, Msg, FileCommand) -> end. on_init(PacketId, Msg, Transfer, Meta) -> - ?SLOG(info, #{ - msg => "on_init", + ?tp(info, "file_transfer_init", #{ mqtt_msg => Msg, packet_id => PacketId, transfer => Transfer, @@ -229,8 +228,7 @@ on_abort(_Msg, _FileId) -> ?RC_SUCCESS. on_segment(PacketId, Msg, Transfer, Offset, Checksum) -> - ?SLOG(info, #{ - msg => "on_segment", + ?tp(info, "file_transfer_segment", #{ mqtt_msg => Msg, packet_id => PacketId, transfer => Transfer, @@ -255,8 +253,7 @@ on_segment(PacketId, Msg, Transfer, Offset, Checksum) -> end). on_fin(PacketId, Msg, Transfer, FinalSize, Checksum) -> - ?SLOG(info, #{ - msg => "on_fin", + ?tp(info, "file_transfer_fin", #{ mqtt_msg => Msg, packet_id => PacketId, transfer => Transfer, diff --git a/apps/emqx_ft/src/emqx_ft_app.erl b/apps/emqx_ft/src/emqx_ft_app.erl index 9b1513b46..0bac6b592 100644 --- a/apps/emqx_ft/src/emqx_ft_app.erl +++ b/apps/emqx_ft/src/emqx_ft_app.erl @@ -22,11 +22,9 @@ start(_StartType, _StartArgs) -> {ok, Sup} = emqx_ft_sup:start_link(), - ok = emqx_ft:hook(), ok = emqx_ft_conf:load(), {ok, Sup}. stop(_State) -> ok = emqx_ft_conf:unload(), - ok = emqx_ft:unhook(), ok. diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 1e531ecdb..90b59c8d1 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -23,6 +23,7 @@ -include_lib("emqx/include/logger.hrl"). %% Accessors +-export([enabled/0]). -export([storage/0]). -export([gc_interval/1]). -export([segments_ttl/1]). @@ -49,6 +50,10 @@ %% Accessors %%-------------------------------------------------------------------- +-spec enabled() -> boolean(). +enabled() -> + emqx_config:get([file_transfer, enable], false). + -spec storage() -> _Storage. storage() -> emqx_config:get([file_transfer, storage], undefined). @@ -83,7 +88,7 @@ store_segment_timeout() -> -spec load() -> ok. load() -> - ok = emqx_ft_storage:on_config_update(undefined, storage()), + ok = on_config_update(#{}, emqx_config:get([file_transfer], #{})), emqx_conf:add_handler([file_transfer], ?MODULE). -spec unload() -> ok. @@ -107,7 +112,26 @@ pre_config_update(_, Req, _Config) -> emqx_config:app_envs() ) -> ok | {ok, Result :: any()} | {error, Reason :: term()}. -post_config_update(_Path, _Req, NewConfig, OldConfig, _AppEnvs) -> - OldStorageConfig = maps:get(storage, OldConfig, undefined), - NewStorageConfig = maps:get(storage, NewConfig, undefined), - emqx_ft_storage:on_config_update(OldStorageConfig, NewStorageConfig). +post_config_update([file_transfer | _], _Req, NewConfig, OldConfig, _AppEnvs) -> + on_config_update(OldConfig, NewConfig). + +on_config_update(OldConfig, NewConfig) -> + lists:foreach( + fun(ConfKey) -> + on_config_update( + ConfKey, + maps:get(ConfKey, OldConfig, undefined), + maps:get(ConfKey, NewConfig, undefined) + ) + end, + [storage, enable] + ). + +on_config_update(_, Config, Config) -> + ok; +on_config_update(storage, OldConfig, NewConfig) -> + ok = emqx_ft_storage:on_config_update(OldConfig, NewConfig); +on_config_update(enable, _, true) -> + ok = emqx_ft:hook(); +on_config_update(enable, _, false) -> + ok = emqx_ft:unhook(). diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 27c593b6c..e2eebbbb8 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -55,6 +55,15 @@ roots() -> [file_transfer]. fields(file_transfer) -> [ + {enable, + mk( + boolean(), + #{ + desc => ?DESC("enable"), + required => false, + default => false + } + )}, {init_timeout, mk( emqx_schema:duration_ms(), @@ -87,22 +96,19 @@ fields(file_transfer) -> hoconsc:union( fun (all_union_members) -> - [ - % NOTE: by default storage is disabled - undefined, - ref(local_storage) - ]; + [ref(local_storage)]; ({value, #{<<"type">> := <<"local">>}}) -> [ref(local_storage)]; ({value, #{<<"type">> := _}}) -> throw(#{field_name => type, expected => "local"}); - (_) -> - [undefined] + ({value, _}) -> + [ref(local_storage)] end ), #{ required => false, - desc => ?DESC("storage") + desc => ?DESC("storage"), + default => #{<<"type">> => <<"local">>} } )} ]; diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 151b8e5fe..d3a3aee21 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -63,6 +63,7 @@ set_special_configs(Config) -> % NOTE % Inhibit local fs GC to simulate it isn't fast enough to collect % complete transfers. + enable => true, storage => emqx_utils_maps:deep_merge( Storage, #{segments => #{gc => #{interval => 0}}} diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index 5f3b213fb..523026d5a 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -30,7 +30,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> ok = emqx_mgmt_api_test_util:init_suite( - [emqx_conf, emqx_ft], set_special_configs(Config) + [emqx_conf, emqx_ft], emqx_ft_test_helpers:env_handler(Config) ), {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. @@ -38,16 +38,6 @@ end_per_suite(_Config) -> ok = emqx_mgmt_api_test_util:end_suite([emqx_ft, emqx_conf]), ok. -set_special_configs(Config) -> - fun - (emqx_ft) -> - emqx_ft_test_helpers:load_config(#{ - storage => emqx_ft_test_helpers:local_storage(Config) - }); - (_) -> - ok - end. - init_per_testcase(Case, Config) -> [{tc, Case} | Config]. end_per_testcase(_Case, _Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index 89b0e895d..106c34702 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -26,22 +26,21 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - _ = emqx_config:save_schema_mod_and_names(emqx_ft_schema), - ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_ft], emqx_ft_test_helpers:env_handler(Config) - ), - {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. end_per_suite(_Config) -> - ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), ok. init_per_testcase(_Case, Config) -> + % NOTE: running each testcase with clean config + _ = emqx_config:save_schema_mod_and_names(emqx_ft_schema), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], fun(_) -> ok end), + {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. end_per_testcase(_Case, _Config) -> - ok. + ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok = emqx_config:erase(file_transfer). %%-------------------------------------------------------------------- %% Tests @@ -61,6 +60,7 @@ t_update_config(_Config) -> emqx_conf:update( [file_transfer], #{ + <<"enable">> => true, <<"storage">> => #{ <<"type">> => <<"local">>, <<"segments">> => #{ @@ -87,10 +87,14 @@ t_update_config(_Config) -> emqx_ft_conf:gc_interval(emqx_ft_conf:storage()) ). -t_remove_restore_config(Config) -> +t_disable_restore_config(Config) -> ?assertMatch( {ok, _}, - emqx_conf:update([file_transfer, storage], #{<<"type">> => <<"local">>}, #{}) + emqx_conf:update( + [file_transfer], + #{<<"enable">> => true, <<"storage">> => #{<<"type">> => <<"local">>}}, + #{} + ) ), ?assertEqual( 60 * 60 * 1000, @@ -98,24 +102,29 @@ t_remove_restore_config(Config) -> ), % Verify that transfers work ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <>), + % Verify that clearing storage settings reverts config to defaults ?assertMatch( {ok, _}, - emqx_conf:remove([file_transfer, storage], #{}) + emqx_conf:update( + [file_transfer], + #{<<"enable">> => false, <<"storage">> => undefined}, + #{} + ) ), ?assertEqual( - undefined, + false, + emqx_ft_conf:enabled() + ), + ?assertMatch( + #{type := local, exporter := #{type := local}}, emqx_ft_conf:storage() ), - ?assertEqual( - undefined, - emqx_ft_conf:gc_interval(emqx_ft_conf:storage()) - ), ClientId = gen_clientid(), Client = emqx_ft_test_helpers:start_client(ClientId), % Verify that transfers fail cleanly when storage is disabled ?check_trace( ?assertMatch( - {ok, #{reason_code_name := unspecified_error}}, + {ok, #{reason_code_name := no_matching_subscribers}}, emqtt:publish( Client, <<"$file/f2/init">>, @@ -124,23 +133,24 @@ t_remove_restore_config(Config) -> ) ), fun(Trace) -> - ?assertMatch( - [#{transfer := {ClientId, <<"f2">>}, reason := {error, disabled}}], - ?of_kind("store_filemeta_failed", Trace) - ) + ?assertMatch([], ?of_kind("file_transfer_init", Trace)) end ), + ok = emqtt:stop(Client), % Restore local storage backend Root = iolist_to_binary(emqx_ft_test_helpers:root(Config, node(), [segments])), ?assertMatch( {ok, _}, emqx_conf:update( - [file_transfer, storage], + [file_transfer], #{ - <<"type">> => <<"local">>, - <<"segments">> => #{ - <<"root">> => Root, - <<"gc">> => #{<<"interval">> => <<"1s">>} + <<"enable">> => true, + <<"storage">> => #{ + <<"type">> => <<"local">>, + <<"segments">> => #{ + <<"root">> => Root, + <<"gc">> => #{<<"interval">> => <<"1s">>} + } } }, #{} @@ -167,5 +177,62 @@ t_remove_restore_config(Config) -> % Verify that transfers work again ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <>). +t_switch_exporter(_Config) -> + ?assertMatch( + {ok, _}, + emqx_conf:update( + [file_transfer], + #{<<"enable">> => true}, + #{} + ) + ), + ?assertMatch( + #{type := local, exporter := #{type := local}}, + emqx_ft_conf:storage() + ), + % Verify that switching to a different exporter works + ?assertMatch( + {ok, _}, + emqx_conf:update( + [file_transfer, storage, exporter], + #{ + <<"type">> => <<"s3">>, + <<"bucket">> => <<"emqx">>, + <<"host">> => <<"https://localhost">>, + <<"port">> => 9000, + <<"transport_options">> => #{ + <<"ipv6_probe">> => false + } + }, + #{} + ) + ), + ?assertMatch( + #{type := local, exporter := #{type := s3}}, + emqx_ft_conf:storage() + ), + % Verify that switching back to local exporter works + ?assertMatch( + {ok, _}, + emqx_conf:remove( + [file_transfer, storage, exporter], + #{} + ) + ), + ?assertMatch( + {ok, _}, + emqx_conf:update( + [file_transfer, storage, exporter], + #{<<"type">> => <<"local">>}, + #{} + ) + ), + ?assertMatch( + #{type := local, exporter := #{type := local}}, + emqx_ft_conf:storage() + ), + % Verify that transfers work + ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <>). + gen_clientid() -> emqx_base62:encode(emqx_guid:gen()). diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index e3decf0f5..d4c13f7d1 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -35,22 +35,12 @@ groups() -> ]. init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_ft], set_special_configs(Config)), + ok = emqx_common_test_helpers:start_apps([emqx_ft], emqx_ft_test_helpers:env_handler(Config)), Config. end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_ft]), ok. -set_special_configs(Config) -> - fun - (emqx_ft) -> - emqx_ft_test_helpers:load_config(#{ - storage => emqx_ft_test_helpers:local_storage(Config) - }); - (_) -> - ok - end. - init_per_testcase(Case, Config) -> [{tc, Case} | Config]. end_per_testcase(_Case, _Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index 11ddf191b..89e349fae 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -41,7 +41,7 @@ stop_additional_node(Node) -> env_handler(Config) -> fun (emqx_ft) -> - load_config(#{storage => local_storage(Config)}); + load_config(#{enable => true, storage => local_storage(Config)}); (_) -> ok end. diff --git a/rel/i18n/emqx_ft_schema.hocon b/rel/i18n/emqx_ft_schema.hocon index 28c93e1ef..e7e551289 100644 --- a/rel/i18n/emqx_ft_schema.hocon +++ b/rel/i18n/emqx_ft_schema.hocon @@ -1,5 +1,11 @@ emqx_ft_schema { +enable.desc: +"""Enable the File Transfer feature.
+Enabling File Transfer implies reserving special MQTT topics in order to serve the protocol.
+This toggle does not have an effect neither on the availability of the File Transfer REST API, nor +on storage-dependent background activities (e.g. garbage collection).""" + init_timeout.desc: """Timeout for initializing the file transfer.
After reaching the timeout, `init` message will be acked with an error""" From a79cf681f7d405bc63fcb90ac045af1f18f5b799 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 27 Apr 2023 18:42:06 +0300 Subject: [PATCH 133/156] fix(s3): drop default from `acl` config field --- apps/emqx_s3/src/emqx_s3_client.erl | 2 +- apps/emqx_s3/src/emqx_s3_schema.erl | 1 - apps/emqx_s3/test/emqx_s3_schema_SUITE.erl | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index e3e1d84af..3bc5861c6 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -57,7 +57,7 @@ port := part_number(), bucket := string(), headers := headers(), - acl := emqx_s3:acl(), + acl := emqx_s3:acl() | undefined, url_expire_time := pos_integer(), access_key_id := string() | undefined, secret_access_key := string() | undefined, diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index 23e69ec5d..5866f8c2b 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -105,7 +105,6 @@ fields(s3) -> bucket_owner_full_control ]), #{ - default => private, desc => ?DESC("acl"), required => false } diff --git a/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl b/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl index 89ec8a958..63f659da0 100644 --- a/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl @@ -23,7 +23,6 @@ t_minimal_config(_Config) -> bucket := "bucket", host := "s3.us-east-1.endpoint.com", port := 443, - acl := private, min_part_size := 5242880, transport_options := #{ From 573bb22ada94110485dc3660a4033981d188cadb Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 30 Mar 2023 19:18:47 +0300 Subject: [PATCH 134/156] feat(ft-fs): introduce fs iterators concept + forward seeks In order to support paging over filesystem contents, to serve REST APIs effectively. --- apps/emqx_ft/src/emqx_ft_fs_iterator.erl | 219 ++++++++++++++++++++ apps/emqx_ft/src/emqx_ft_fs_util.erl | 66 ++---- apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl | 113 +++++++++- 3 files changed, 342 insertions(+), 56 deletions(-) create mode 100644 apps/emqx_ft/src/emqx_ft_fs_iterator.erl diff --git a/apps/emqx_ft/src/emqx_ft_fs_iterator.erl b/apps/emqx_ft/src/emqx_ft_fs_iterator.erl new file mode 100644 index 000000000..5c8857ab0 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_fs_iterator.erl @@ -0,0 +1,219 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_fs_iterator). + +-export([new/2]). +-export([next/1]). +-export([next_leaf/1]). + +-export([seek/3]). + +-export([fold/3]). + +-export_type([t/0]). +-export_type([glob/0]). +-export_type([pathstack/0]). + +-type root() :: file:name(). +-type glob() :: ['*' | globfun()]. +-type globfun() :: + fun((_Filename :: file:name()) -> boolean()) + | fun((_Filename :: file:name(), pathstack()) -> boolean()). + +% A path stack is a list of path components, in reverse order. +-type pathstack() :: [file:name(), ...]. + +-opaque t() :: #{ + root := root(), + queue := [_PathStack :: [file:name()]], + head := glob(), + stack := [{[pathstack()], glob()}] +}. + +-type entry() :: entry_leaf() | entry_node(). +-type entry_leaf() :: + {leaf, file:name(), file:file_info() | {error, file:posix()}, pathstack()}. +-type entry_node() :: + {node, file:name(), {error, file:posix()}, pathstack()}. + +-spec new(root(), glob()) -> + t(). +new(Root, Glob) -> + #{ + root => Root, + queue => [[]], + head => Glob, + stack => [] + }. + +-spec next(t()) -> + {entry(), t()} | none. +next(It = #{queue := [PathStack | Rest], head := []}) -> + {emit(PathStack, It), It#{queue => Rest}}; +next(It = #{queue := [PathStack | Rest], head := [Pat | _], root := Root}) -> + Filepath = mk_filepath(PathStack), + case emqx_ft_fs_util:list_dir(filename:join(Root, Filepath)) of + {ok, Filenames} -> + Sorted = lists:sort(Filenames), + Matches = [[Fn | PathStack] || Fn <- Sorted, matches_glob(Pat, Fn, [Fn | PathStack])], + ItNext = windup(It), + next(ItNext#{queue => Matches}); + {error, _} = Error -> + {{node, Filepath, Error, PathStack}, It#{queue => Rest}} + end; +next(It = #{queue := []}) -> + unwind(It). + +windup(It = #{queue := [_ | Rest], head := [Pat | Glob], stack := Stack}) -> + % NOTE + % Preserve unfinished paths and glob in the stack, so that we can resume traversal + % when the lower levels of the tree are exhausted. + It#{ + head => Glob, + stack => [{Rest, [Pat | Glob]} | Stack] + }. + +unwind(It = #{stack := [{Queue, Glob} | StackRest]}) -> + % NOTE + % Resume traversal of unfinished paths from the upper levels of the tree. + next(It#{ + queue => Queue, + head => Glob, + stack => StackRest + }); +unwind(#{stack := []}) -> + none. + +emit(PathStack, #{root := Root}) -> + Filepath = mk_filepath(PathStack), + case emqx_ft_fs_util:read_info(filename:join(Root, Filepath)) of + {ok, Fileinfo} -> + {leaf, Filepath, Fileinfo, PathStack}; + {error, _} = Error -> + {leaf, Filepath, Error, PathStack} + end. + +mk_filepath([]) -> + ""; +mk_filepath(PathStack) -> + filename:join(lists:reverse(PathStack)). + +matches_glob('*', _, _) -> + true; +matches_glob(FilterFun, Filename, _PathStack) when is_function(FilterFun, 1) -> + FilterFun(Filename); +matches_glob(FilterFun, Filename, PathStack) when is_function(FilterFun, 2) -> + FilterFun(Filename, PathStack). + +%% + +-spec next_leaf(t()) -> + {entry_leaf(), t()} | none. +next_leaf(It) -> + case next(It) of + {{leaf, _, _, _} = Leaf, ItNext} -> + {Leaf, ItNext}; + {{node, _Filename, _Error, _PathStack}, ItNext} -> + % NOTE + % Intentionally skipping intermediate traversal errors here, for simplicity. + next_leaf(ItNext); + none -> + none + end. + +%% + +-spec seek([file:name()], root(), glob()) -> + t(). +seek(PathSeek, Root, Glob) -> + SeekGlob = mk_seek_glob(PathSeek, Glob), + SeekStack = lists:reverse(PathSeek), + case next_leaf(new(Root, SeekGlob)) of + {{leaf, _Filepath, _Info, SeekStack}, It} -> + fixup_glob(Glob, It); + {{leaf, _Filepath, _Info, Successor}, It = #{queue := Queue}} -> + fixup_glob(Glob, It#{queue => [Successor | Queue]}); + none -> + none(Root) + end. + +mk_seek_glob(PathSeek, Glob) -> + % NOTE + % The seek glob is a glob that skips all the nodes / leaves that are lexicographically + % smaller than the seek path. For example, if the seek path is ["a", "b", "c"], and + % the glob is ['*', '*', '*', '*'], then the seek glob is: + % [ fun(Path) -> Path >= ["a"] end, + % fun(Path) -> Path >= ["a", "b"] end, + % fun(Path) -> Path >= ["a", "b", "c"] end, + % '*' + % ] + L = min(length(PathSeek), length(Glob)), + merge_glob([mk_seek_pat(lists:sublist(PathSeek, N)) || N <- lists:seq(1, L)], Glob). + +mk_seek_pat(PathSeek) -> + % NOTE + % The `PathStack` and `PathSeek` are of the same length here. + fun(_Filename, PathStack) -> lists:reverse(PathStack) >= PathSeek end. + +merge_glob([Pat | SeekRest], [PatOrig | Rest]) -> + [merge_pat(Pat, PatOrig) | merge_glob(SeekRest, Rest)]; +merge_glob([], [PatOrig | Rest]) -> + [PatOrig | merge_glob([], Rest)]; +merge_glob([], []) -> + []. + +merge_pat(Pat, PatOrig) -> + fun(Filename, PathStack) -> + Pat(Filename, PathStack) andalso matches_glob(PatOrig, Filename, PathStack) + end. + +fixup_glob(Glob, It = #{head := [], stack := Stack}) -> + % NOTE + % Restoring original glob through the stack. Strictly speaking, this is not usually + % necessary, it's a kind of optimization. + fixup_glob(Glob, lists:reverse(Stack), It#{stack => []}). + +fixup_glob(Glob = [_ | Rest], [{Queue, _} | StackRest], It = #{stack := Stack}) -> + fixup_glob(Rest, StackRest, It#{stack => [{Queue, Glob} | Stack]}); +fixup_glob(Rest, [], It) -> + It#{head => Rest}. + +%% + +-spec fold(fun((entry(), Acc) -> Acc), Acc, t()) -> + Acc. +fold(FoldFun, Acc, It) -> + case next(It) of + {Entry, ItNext} -> + fold(FoldFun, FoldFun(Entry, Acc), ItNext); + none -> + Acc + end. + +%% + +-spec none(root()) -> + t(). +none(Root) -> + % NOTE + % The _none_ iterator is a valid iterator, but it will never yield any entries. + #{ + root => Root, + queue => [], + head => [], + stack => [] + }. diff --git a/apps/emqx_ft/src/emqx_ft_fs_util.erl b/apps/emqx_ft/src/emqx_ft_fs_util.erl index df9135816..b731d3270 100644 --- a/apps/emqx_ft/src/emqx_ft_fs_util.erl +++ b/apps/emqx_ft/src/emqx_ft_fs_util.erl @@ -25,18 +25,16 @@ -export([read_decode_file/2]). -export([read_info/1]). +-export([list_dir/1]). -export([fold/4]). --type glob() :: ['*' | globfun()]. --type globfun() :: - fun((_Filename :: file:name()) -> boolean()). -type foldfun(Acc) :: fun( ( _Filepath :: file:name(), - _Info :: file:file_info() | {error, _IoError}, - _Stack :: [file:name()], + _Info :: file:file_info() | {error, file:posix()}, + _Stack :: emqx_ft_fs_iterator:pathstack(), Acc ) -> Acc ). @@ -153,46 +151,8 @@ read_info(AbsPath) -> % Be aware that this function is occasionally mocked in `emqx_ft_fs_util_SUITE`. file:read_link_info(AbsPath, [{time, posix}, raw]). --spec fold(foldfun(Acc), Acc, _Root :: file:name(), glob()) -> - Acc. -fold(Fun, Acc, Root, Glob) -> - fold(Fun, Acc, [], Root, Glob, []). - -fold(Fun, AccIn, Path, Root, [Glob | Rest], Stack) when Glob == '*' orelse is_function(Glob) -> - case list_dir(filename:join(Root, Path)) of - {ok, Filenames} -> - lists:foldl( - fun(FN, Acc) -> - case matches_glob(Glob, FN) of - true when Path == [] -> - fold(Fun, Acc, FN, Root, Rest, [FN | Stack]); - true -> - fold(Fun, Acc, filename:join(Path, FN), Root, Rest, [FN | Stack]); - false -> - Acc - end - end, - AccIn, - Filenames - ); - {error, enotdir} -> - AccIn; - {error, Reason} -> - Fun(Path, {error, Reason}, Stack, AccIn) - end; -fold(Fun, AccIn, Filepath, Root, [], Stack) -> - case ?MODULE:read_info(filename:join(Root, Filepath)) of - {ok, Info} -> - Fun(Filepath, Info, Stack, AccIn); - {error, Reason} -> - Fun(Filepath, {error, Reason}, Stack, AccIn) - end. - -matches_glob('*', _) -> - true; -matches_glob(FilterFun, Filename) when is_function(FilterFun) -> - FilterFun(Filename). - +-spec list_dir(file:name_all()) -> + {ok, [file:name()]} | {error, file:posix() | badarg}. list_dir(AbsPath) -> case ?MODULE:read_info(AbsPath) of {ok, #file_info{type = directory}} -> @@ -202,3 +162,19 @@ list_dir(AbsPath) -> {error, Reason} -> {error, Reason} end. + +-spec fold(foldfun(Acc), Acc, _Root :: file:name(), emqx_ft_fs_iterator:glob()) -> + Acc. +fold(FoldFun, Acc, Root, Glob) -> + fold(FoldFun, Acc, emqx_ft_fs_iterator:new(Root, Glob)). + +fold(FoldFun, Acc, It) -> + case emqx_ft_fs_iterator:next(It) of + {{node, _Path, {error, enotdir}, _PathStack}, ItNext} -> + fold(FoldFun, Acc, ItNext); + {{_Type, Path, Info, PathStack}, ItNext} -> + AccNext = FoldFun(Path, Info, PathStack, Acc), + fold(FoldFun, AccNext, ItNext); + none -> + Acc + end. diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl index 81a483651..e4aa70f81 100644 --- a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl @@ -34,7 +34,7 @@ t_fold_single_level(Config) -> {"c", #file_info{type = directory}, ["c"]}, {"d", #file_info{type = directory}, ["d"]} ], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*'])) + sort(fold(fun cons/4, [], Root, ['*'])) ). t_fold_multi_level(Config) -> @@ -45,7 +45,7 @@ t_fold_multi_level(Config) -> {"a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]}, {"d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} ], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*'])) + sort(fold(fun cons/4, [], Root, ['*', '*', '*', '*'])) ), ?assertMatch( [ @@ -53,32 +53,32 @@ t_fold_multi_level(Config) -> {"c/bar/中文", #file_info{type = regular}, ["中文", "bar", "c"]}, {"d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]} ], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*'])) + sort(fold(fun cons/4, [], Root, ['*', '*', '*'])) ). t_fold_no_glob(Config) -> Root = ?config(data_dir, Config), ?assertMatch( [{"", #file_info{type = directory}, []}], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, [])) + sort(fold(fun cons/4, [], Root, [])) ). t_fold_glob_too_deep(Config) -> Root = ?config(data_dir, Config), ?assertMatch( [], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*', '*'])) + sort(fold(fun cons/4, [], Root, ['*', '*', '*', '*', '*'])) ). t_fold_invalid_root(Config) -> Root = ?config(data_dir, Config), ?assertMatch( [], - sort(emqx_ft_fs_util:fold(fun cons/4, [], filename:join([Root, "a", "link"]), ['*'])) + sort(fold(fun cons/4, [], filename:join([Root, "a", "link"]), ['*'])) ), ?assertMatch( [], - sort(emqx_ft_fs_util:fold(fun cons/4, [], filename:join([Root, "d", "haystack"]), ['*'])) + sort(fold(fun cons/4, [], filename:join([Root, "d", "haystack"]), ['*'])) ). t_fold_filter_unicode(Config) -> @@ -88,13 +88,13 @@ t_fold_filter_unicode(Config) -> {"a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]}, {"d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} ], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', fun is_latin1/1])) + sort(fold(fun cons/4, [], Root, ['*', '*', '*', fun is_latin1/1])) ), ?assertMatch( [ {"a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]} ], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', is_not(fun is_latin1/1)])) + sort(fold(fun cons/4, [], Root, ['*', '*', '*', is_not(fun is_latin1/1)])) ). t_fold_filter_levels(Config) -> @@ -104,7 +104,7 @@ t_fold_filter_levels(Config) -> {"a/b/foo", #file_info{type = directory}, ["foo", "b", "a"]}, {"d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]} ], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, [fun is_letter/1, fun is_letter/1, '*'])) + sort(fold(fun cons/4, [], Root, [fun is_letter/1, fun is_letter/1, '*'])) ). t_fold_errors(Config) -> @@ -128,11 +128,99 @@ t_fold_errors(Config) -> {"c/link", {error, enotsup}, ["link", "c"]}, {"d/e/baz/needle", {error, ebusy}, ["needle", "baz", "e", "d"]} ], - sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*'])) + sort(fold(fun cons/4, [], Root, ['*', '*', '*', '*'])) + ). + +t_seek_fold(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {leaf, "a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]}, + {leaf, "a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]}, + {leaf, "d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} + | _Nodes + ], + sort( + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:seek(["a", "a"], Root, ['*', '*', '*', '*']) + ) + ) + ), + ?assertMatch( + [ + {leaf, "a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]}, + {leaf, "d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} + | _Nodes + ], + sort( + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:seek(["a", "b", "foo", "42"], Root, ['*', '*', '*', '*']) + ) + ) + ), + ?assertMatch( + [ + {leaf, "d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} + | _Nodes + ], + sort( + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:seek(["c", "d", "e", "f"], Root, ['*', '*', '*', '*']) + ) + ) + ). + +t_seek_empty(Config) -> + Root = ?config(data_dir, Config), + ?assertEqual( + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:new(Root, ['*', '*', '*', '*']) + ), + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:seek([], Root, ['*', '*', '*', '*']) + ) + ). + +t_seek_past_end(Config) -> + Root = ?config(data_dir, Config), + ?assertEqual( + none, + emqx_ft_fs_iterator:next( + emqx_ft_fs_iterator:seek(["g", "h"], Root, ['*', '*', '*', '*']) + ) + ). + +t_seek_with_filter(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {leaf, "d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]} + | _Nodes + ], + sort( + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:seek(["a", "link"], Root, ['*', fun is_letter/1, '*']) + ) + ) ). %% +fold(FoldFun, Acc, Root, Glob) -> + emqx_ft_fs_util:fold(FoldFun, Acc, Root, Glob). + is_not(F) -> fun(X) -> not F(X) end. @@ -155,5 +243,8 @@ is_letter(Filename) -> cons(Path, Info, Stack, Acc) -> [{Path, Info, Stack} | Acc]. +cons(Entry, Acc) -> + [Entry | Acc]. + sort(L) when is_list(L) -> lists:sort(L). From ed3756ea0928965c628a518015b5c5d0c79f01ef Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Sat, 1 Apr 2023 00:08:46 +0300 Subject: [PATCH 135/156] feat(ft-api): add paging support through cursors --- apps/emqx_ft/src/emqx_ft_api.erl | 131 +++++++- apps/emqx_ft/src/emqx_ft_fs_iterator.erl | 16 + apps/emqx_ft/src/emqx_ft_storage.erl | 29 +- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 16 +- .../src/emqx_ft_storage_exporter_fs.erl | 301 ++++++++++++------ .../src/emqx_ft_storage_exporter_fs_proxy.erl | 6 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 6 +- .../emqx_ft_storage_exporter_fs_proto_v1.erl | 13 +- apps/emqx_ft/test/emqx_ft_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 97 +++++- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 2 +- .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 2 +- rel/i18n/emqx_ft_api.hocon | 3 + 13 files changed, 482 insertions(+), 142 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index 85fd8d8cb..7e1ed97ad 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -19,7 +19,6 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). --include_lib("emqx/include/logger.hrl"). %% Swagger specs from hocon schema -export([ @@ -30,12 +29,14 @@ ]). -export([ - roots/0 + roots/0, + fields/1 ]). %% API callbacks -export([ - '/file_transfer/files'/2 + '/file_transfer/files'/2, + '/file_transfer/files/:clientid/:fileid'/2 ]). -import(hoconsc, [mk/2, ref/1, ref/2]). @@ -47,7 +48,8 @@ api_spec() -> paths() -> [ - "/file_transfer/files" + "/file_transfer/files", + "/file_transfer/files/:clientid/:fileid" ]. schema("/file_transfer/files") -> @@ -57,29 +59,133 @@ schema("/file_transfer/files") -> tags => [<<"file_transfer">>], summary => <<"List all uploaded files">>, description => ?DESC("file_list"), + parameters => [ + ref(following), + ref(emqx_dashboard_swagger, limit) + ], responses => #{ 200 => <<"Operation success">>, + 400 => emqx_dashboard_swagger:error_codes( + ['BAD_REQUEST'], <<"Invalid cursor">> + ), 503 => emqx_dashboard_swagger:error_codes( - ['SERVICE_UNAVAILABLE'], <<"Service unavailable">> + ['SERVICE_UNAVAILABLE'], error_desc('SERVICE_UNAVAILABLE') + ) + } + } + }; +schema("/file_transfer/files/:clientid/:fileid") -> + #{ + 'operationId' => '/file_transfer/files/:clientid/:fileid', + get => #{ + tags => [<<"file_transfer">>], + summary => <<"List files uploaded in a specific transfer">>, + description => ?DESC("file_list_transfer"), + parameters => [ + ref(client_id), + ref(file_id) + ], + responses => #{ + 200 => <<"Operation success">>, + 404 => emqx_dashboard_swagger:error_codes( + ['FILES_NOT_FOUND'], error_desc('FILES_NOT_FOUND') + ), + 503 => emqx_dashboard_swagger:error_codes( + ['SERVICE_UNAVAILABLE'], error_desc('SERVICE_UNAVAILABLE') ) } } }. -'/file_transfer/files'(get, #{}) -> - case emqx_ft_storage:files() of - {ok, Files} -> - {200, #{<<"files">> => lists:map(fun format_file_info/1, Files)}}; - {error, _} -> - {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} +'/file_transfer/files'(get, #{ + query_string := QueryString +}) -> + try + Limit = limit(QueryString), + Query = + case maps:get(<<"following">>, QueryString, undefined) of + undefined -> + #{limit => Limit}; + Cursor -> + #{limit => Limit, following => Cursor} + end, + case emqx_ft_storage:files(Query) of + {ok, Page} -> + {200, format_page(Page)}; + {error, _} -> + {503, error_msg('SERVICE_UNAVAILABLE')} + end + catch + error:{badarg, cursor} -> + {400, error_msg('BAD_REQUEST', <<"Invalid cursor">>)} end. +'/file_transfer/files/:clientid/:fileid'(get, #{ + bindings := #{clientid := ClientId, fileid := FileId} +}) -> + Transfer = {ClientId, FileId}, + case emqx_ft_storage:files(#{transfer => Transfer}) of + {ok, Page} -> + {200, format_page(Page)}; + {error, [{_Node, enoent} | _]} -> + {404, error_msg('FILES_NOT_FOUND')}; + {error, _} -> + {503, error_msg('SERVICE_UNAVAILABLE')} + end. + +format_page(#{items := Files, cursor := Cursor}) -> + #{ + <<"files">> => lists:map(fun format_file_info/1, Files), + <<"cursor">> => Cursor + }; +format_page(#{items := Files}) -> + #{ + <<"files">> => lists:map(fun format_file_info/1, Files) + }. + +error_msg(Code) -> + #{code => Code, message => error_desc(Code)}. + error_msg(Code, Msg) -> #{code => Code, message => emqx_utils:readable_error_msg(Msg)}. +error_desc('FILES_NOT_FOUND') -> + <<"Files requested for this transfer could not be found">>; +error_desc('SERVICE_UNAVAILABLE') -> + <<"Service unavailable">>. + roots() -> []. +-spec fields(hocon_schema:name()) -> [hoconsc:field()]. +fields(client_id) -> + [ + {clientid, + mk(binary(), #{ + in => path, + desc => <<"MQTT Client ID">>, + required => true + })} + ]; +fields(file_id) -> + [ + {fileid, + mk(binary(), #{ + in => path, + desc => <<"File ID">>, + required => true + })} + ]; +fields(following) -> + [ + {following, + mk(binary(), #{ + in => query, + desc => <<"Cursor to start listing files from">>, + required => false + })} + ]. + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- @@ -115,3 +221,6 @@ format_name(NameBin) when is_binary(NameBin) -> NameBin; format_name(Name) when is_list(Name) -> iolist_to_binary(Name). + +limit(QueryString) -> + maps:get(<<"limit">>, QueryString, emqx_mgmt:default_row_limit()). diff --git a/apps/emqx_ft/src/emqx_ft_fs_iterator.erl b/apps/emqx_ft/src/emqx_ft_fs_iterator.erl index 5c8857ab0..7fb8d8634 100644 --- a/apps/emqx_ft/src/emqx_ft_fs_iterator.erl +++ b/apps/emqx_ft/src/emqx_ft_fs_iterator.erl @@ -23,6 +23,7 @@ -export([seek/3]). -export([fold/3]). +-export([fold_n/4]). -export_type([t/0]). -export_type([glob/0]). @@ -204,6 +205,21 @@ fold(FoldFun, Acc, It) -> Acc end. +%% NOTE +%% Passing negative `N` is allowed, in which case the iterator will be exhausted +%% completely, like in `fold/3`. +-spec fold_n(fun((entry(), Acc) -> Acc), Acc, t(), _N :: integer()) -> + {Acc, {more, t()} | none}. +fold_n(_FoldFun, Acc, It, 0) -> + {Acc, {more, It}}; +fold_n(FoldFun, Acc, It, N) -> + case next(It) of + {Entry, ItNext} -> + fold_n(FoldFun, FoldFun(Entry, Acc), ItNext, N - 1); + none -> + {Acc, none} + end. + %% -spec none(root()) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index fee16cd09..5364211a4 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -25,6 +25,7 @@ assemble/2, files/0, + files/1, with_storage_type/2, with_storage_type/3, @@ -36,12 +37,27 @@ -type storage() :: emqx_config:config(). -export_type([assemble_callback/0]). + +-export_type([query/1]). +-export_type([page/2]). -export_type([file_info/0]). -export_type([export_data/0]). -export_type([reader/0]). -type assemble_callback() :: fun((ok | {error, term()}) -> any()). +-type query(Cursor) :: + #{transfer => emqx_ft:transfer()} + | #{ + limit => non_neg_integer(), + following => Cursor + }. + +-type page(Item, Cursor) :: #{ + items := [Item], + cursor => Cursor +}. + -type file_info() :: #{ transfer := emqx_ft:transfer(), name := file:name(), @@ -71,8 +87,8 @@ -callback assemble(storage(), emqx_ft:transfer(), _Size :: emqx_ft:bytes()) -> ok | {async, pid()} | {error, term()}. --callback files(storage()) -> - {ok, [file_info()]} | {error, term()}. +-callback files(storage(), query(Cursor)) -> + {ok, page(file_info(), Cursor)} | {error, term()}. %%-------------------------------------------------------------------- %% API @@ -105,9 +121,14 @@ assemble(Transfer, Size) -> with_storage(assemble, [Transfer, Size]). -spec files() -> - {ok, [file_info()]} | {error, term()}. + {ok, page(file_info(), _)} | {error, term()}. files() -> - with_storage(files, []). + files(#{}). + +-spec files(query(Cursor)) -> + {ok, page(file_info(), Cursor)} | {error, term()}. +files(Query) -> + with_storage(files, [Query]). -spec with_storage(atom() | function()) -> any(). with_storage(Fun) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index 72128cb40..fb44093c1 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -17,7 +17,7 @@ %% Filesystem storage exporter %% %% This is conceptually a part of the Filesystem storage backend that defines -%% how and where complete tranfers are assembled into files and stored. +%% how and where complete transfers are assembled into files and stored. -module(emqx_ft_storage_exporter). @@ -28,7 +28,7 @@ -export([discard/1]). %% Listing API --export([list/1]). +-export([list/2]). %% Lifecycle API -export([on_config_update/2]). @@ -70,8 +70,8 @@ -callback discard(ExportSt :: export_st()) -> ok | {error, _Reason}. --callback list(storage()) -> - {ok, [emqx_ft_storage:file_info()]} | {error, _Reason}. +-callback list(exporter_conf(), emqx_ft_storage:query(Cursor)) -> + {ok, emqx_ft_storage:page(emqx_ft_storage:file_info(), Cursor)} | {error, _Reason}. %% Lifecycle callbacks @@ -133,11 +133,11 @@ complete(#{mod := ExporterMod, st := ExportSt, hash := Hash, filemeta := Filemet discard(#{mod := ExporterMod, st := ExportSt}) -> ExporterMod:discard(ExportSt). --spec list(storage()) -> - {ok, [emqx_ft_storage:file_info()]} | {error, _Reason}. -list(Storage) -> +-spec list(storage(), emqx_ft_storage:query(Cursor)) -> + {ok, emqx_ft_storage:page(emqx_ft_storage:file_info(), Cursor)} | {error, _Reason}. +list(Storage, Query) -> {ExporterMod, ExporterOpts} = exporter(Storage), - ExporterMod:list(ExporterOpts). + ExporterMod:list(ExporterOpts, Query). %% Lifecycle diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index 4b05c9a58..9109dadbb 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -37,12 +37,23 @@ %% Internal API for RPC -export([list_local/1]). -export([list_local/2]). +-export([list_local_transfer/2]). -export([start_reader/3]). -% TODO -% -export([list/2]). +-export([list/2]). + +-export_type([export_st/0]). +-export_type([options/0]). + +-type options() :: #{ + root => file:name(), + _ => _ +}. + +-type query() :: emqx_ft_storage:query(cursor()). +-type page(T) :: emqx_ft_storage:page(T, cursor()). +-type cursor() :: iodata(). --type options() :: _TODO. -type transfer() :: emqx_ft:transfer(). -type filemeta() :: emqx_ft:filemeta(). -type exportinfo() :: emqx_ft_storage:file_info(). @@ -70,22 +81,6 @@ %% 2 symbols = at most 256 directories on the second level -define(BUCKET2_LEN, 2). --define(SLOG_UNEXPECTED(RelFilepath, Fileinfo, Options), - ?SLOG(notice, "filesystem_object_unexpected", #{ - relpath => RelFilepath, - fileinfo => Fileinfo, - options => Options - }) -). - --define(SLOG_INACCESSIBLE(RelFilepath, Reason, Options), - ?SLOG(warning, "filesystem_object_inaccessible", #{ - relpath => RelFilepath, - reason => Reason, - options => Options - }) -). - %%-------------------------------------------------------------------- %% Exporter behaviour %%-------------------------------------------------------------------- @@ -162,33 +157,33 @@ update(_OldOptions, _NewOptions) -> ok. %% Internal API %%-------------------------------------------------------------------- --spec list_local(options(), transfer()) -> - {ok, [exportinfo(), ...]} | {error, file_error()}. -list_local(Options, Transfer) -> - TransferRoot = mk_absdir(Options, Transfer, result), - case - emqx_ft_fs_util:fold( - fun - (_Path, {error, Reason}, [], []) -> - {error, Reason}; - (_Path, Fileinfo = #file_info{type = regular}, [Filename | _], Acc) -> - RelFilepath = filename:join(mk_result_reldir(Transfer) ++ [Filename]), - Info = mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo), - [Info | Acc]; - (RelFilepath, Fileinfo = #file_info{}, _, Acc) -> - ?SLOG_UNEXPECTED(RelFilepath, Fileinfo, Options), - Acc; - (RelFilepath, {error, Reason}, _, Acc) -> - ?SLOG_INACCESSIBLE(RelFilepath, Reason, Options), - Acc - end, - [], - TransferRoot, - [fun filter_manifest/1] - ) - of +-type local_query() :: emqx_ft_storage:query({transfer(), file:name()}). + +-spec list_local_transfer(options(), transfer()) -> + {ok, [exportinfo()]} | {error, file_error()}. +list_local_transfer(Options, Transfer) -> + It = emqx_ft_fs_iterator:new( + mk_absdir(Options, Transfer, result), + [fun filter_manifest/1] + ), + Result = emqx_ft_fs_iterator:fold( + fun + ({leaf, _Path, Fileinfo = #file_info{type = regular}, [Filename | _]}, Acc) -> + RelFilepath = filename:join(mk_result_reldir(Transfer) ++ [Filename]), + Info = mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo), + [Info | Acc]; + ({node, _Path, {error, Reason}, []}, []) -> + {error, Reason}; + (Entry, Acc) -> + ok = log_invalid_entry(Options, Entry), + Acc + end, + [], + It + ), + case Result of Infos = [_ | _] -> - {ok, Infos}; + {ok, lists:reverse(Infos)}; [] -> {error, enoent}; {error, Reason} -> @@ -196,9 +191,17 @@ list_local(Options, Transfer) -> end. -spec list_local(options()) -> - {ok, #{transfer() => [exportinfo(), ...]}}. + {ok, [exportinfo()]} | {error, file_error()}. list_local(Options) -> - Pattern = [ + list_local(Options, #{}). + +-spec list_local(options(), local_query()) -> + {ok, [exportinfo()]} | {error, file_error()}. +list_local(Options, #{transfer := Transfer}) -> + list_local_transfer(Options, Transfer); +list_local(Options, #{} = Query) -> + Root = get_storage_root(Options), + Glob = [ _Bucket1 = '*', _Bucket2 = '*', _Rest = '*', @@ -206,16 +209,30 @@ list_local(Options) -> _FileId = '*', fun filter_manifest/1 ], - Root = get_storage_root(Options), - {ok, - emqx_ft_fs_util:fold( - fun(RelFilepath, Info, Stack, Acc) -> - read_exportinfo(Options, RelFilepath, Info, Stack, Acc) - end, - [], - Root, - Pattern - )}. + It = + case Query of + #{following := Cursor} -> + emqx_ft_fs_iterator:seek(mk_path_seek(Cursor), Root, Glob); + #{} -> + emqx_ft_fs_iterator:new(Root, Glob) + end, + % NOTE + % In the rare case when some transfer contain more than one file, the paging mechanic + % here may skip over some files, when the cursor is transfer-only. + Limit = maps:get(limit, Query, -1), + {Exports, _} = emqx_ft_fs_iterator:fold_n( + fun(Entry, Acc) -> read_exportinfo(Options, Entry, Acc) end, + [], + It, + Limit + ), + {ok, Exports}. + +mk_path_seek(#{transfer := Transfer, name := Filename}) -> + mk_result_reldir(Transfer) ++ [Filename]; +mk_path_seek(#{transfer := Transfer}) -> + % NOTE: Any bitstring is greater than any list. + mk_result_reldir(Transfer) ++ [<<>>]. %%-------------------------------------------------------------------- %% Helpers @@ -227,16 +244,21 @@ filter_manifest(?MANIFEST) -> filter_manifest(Filename) -> ?MANIFEST =/= string:find(Filename, ?MANIFEST, trailing). -read_exportinfo(Options, RelFilepath, Fileinfo = #file_info{type = regular}, Stack, Acc) -> - [Filename, FileId, ClientId | _] = Stack, +read_exportinfo( + Options, + {leaf, RelFilepath, Fileinfo = #file_info{type = regular}, [Filename, FileId, ClientId | _]}, + Acc +) -> + % NOTE + % There might be more than one file for a single transfer (though + % extremely bad luck is needed for that, e.g. concurrent assemblers with + % different filemetas from different nodes). This might be unexpected for a + % client given the current protocol, yet might be helpful in the future. Transfer = dirnames_to_transfer(ClientId, FileId), Info = mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo), [Info | Acc]; -read_exportinfo(Options, RelFilepath, Fileinfo = #file_info{}, _Stack, Acc) -> - ?SLOG_UNEXPECTED(RelFilepath, Fileinfo, Options), - Acc; -read_exportinfo(Options, RelFilepath, {error, Reason}, _Stack, Acc) -> - ?SLOG_INACCESSIBLE(RelFilepath, Reason, Options), +read_exportinfo(Options, Entry, Acc) -> + ok = log_invalid_entry(Options, Entry), Acc. mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo) -> @@ -268,6 +290,19 @@ try_read_filemeta(Filepath, Info) -> mk_export_uri(RelFilepath) -> emqx_ft_storage_exporter_fs_api:mk_export_uri(node(), RelFilepath). +log_invalid_entry(Options, {_Type, RelFilepath, Fileinfo = #file_info{}, _Stack}) -> + ?SLOG(notice, "filesystem_object_unexpected", #{ + relpath => RelFilepath, + fileinfo => Fileinfo, + options => Options + }); +log_invalid_entry(Options, {_Type, RelFilepath, {error, Reason}, _Stack}) -> + ?SLOG(warning, "filesystem_object_inaccessible", #{ + relpath => RelFilepath, + reason => Reason, + options => Options + }). + -spec start_reader(options(), file:name(), _Caller :: pid()) -> {ok, reader()} | {error, enoent}. start_reader(Options, RelFilepath, CallerPid) -> @@ -282,32 +317,112 @@ start_reader(Options, RelFilepath, CallerPid) -> %% --spec list(options()) -> - {ok, [exportinfo(), ...]} | {error, [{node(), _Reason}]}. -list(_Options) -> - Nodes = mria_mnesia:running_nodes(), - Replies = emqx_ft_storage_exporter_fs_proto_v1:list_exports(Nodes), - {Results, Errors} = lists:foldl( - fun - ({_Node, {ok, {ok, Files}}}, {Acc, Errors}) -> - {Files ++ Acc, Errors}; - ({Node, {ok, {error, _} = Error}}, {Acc, Errors}) -> - {Acc, [{Node, Error} | Errors]}; - ({Node, Error}, {Acc, Errors}) -> - {Acc, [{Node, Error} | Errors]} - end, - {[], []}, - lists:zip(Nodes, Replies) - ), - length(Errors) > 0 andalso - ?SLOG(warning, #{msg => "list_remote_exports_failed", errors => Errors}), - case Results of - [_ | _] -> - {ok, Results}; - [] when Errors =:= [] -> - {ok, Results}; - [] -> - {error, Errors} +-spec list(options(), query()) -> + {ok, page(exportinfo())} | {error, [{node(), _Reason}]}. +list(_Options, Query = #{transfer := _Transfer}) -> + case list(Query) of + #{items := Exports = [_ | _]} -> + {ok, #{items => Exports}}; + #{items := [], errors := NodeErrors} -> + {error, NodeErrors} + end; +list(_Options, Query) -> + Result = list(Query), + case Result of + #{errors := NodeErrors} -> + ?SLOG(warning, "list_exports_errors", #{ + query => Query, + errors => NodeErrors + }); + #{} -> + ok + end, + case Result of + #{items := Exports, cursor := Cursor} -> + {ok, #{items => lists:reverse(Exports), cursor => encode_cursor(Cursor)}}; + #{items := Exports} -> + {ok, #{items => lists:reverse(Exports)}} + end. + +list(QueryIn) -> + {Nodes, NodeQuery} = decode_query(QueryIn, lists:sort(mria_mnesia:running_nodes())), + list_nodes(NodeQuery, Nodes, #{items => []}). + +list_nodes(Query, Nodes = [Node | Rest], Acc) -> + case emqx_ft_storage_exporter_fs_proto_v1:list_exports([Node], Query) of + [{ok, Result}] -> + list_accumulate(Result, Query, Nodes, Acc); + [Failure] -> + ?SLOG(warning, #{ + msg => "list_remote_exports_failed", + node => Node, + query => Query, + failure => Failure + }), + list_next(Query, Rest, Acc) + end; +list_nodes(_Query, [], Acc) -> + Acc. + +list_accumulate({ok, Exports}, Query, [Node | Rest], Acc = #{items := EAcc}) -> + NExports = length(Exports), + AccNext = Acc#{items := Exports ++ EAcc}, + case Query of + #{limit := Limit} when NExports < Limit -> + list_next(Query#{limit => Limit - NExports}, Rest, AccNext); + #{limit := _} -> + AccNext#{cursor => mk_cursor(Node, Exports)}; + #{} -> + list_next(Query, Rest, AccNext) + end; +list_accumulate({error, Reason}, Query, [Node | Rest], Acc) -> + EAcc = maps:get(errors, Acc, []), + list_next(Query, Rest, Acc#{errors => [{Node, Reason} | EAcc]}). + +list_next(Query, Nodes, Acc) -> + list_nodes(maps:remove(following, Query), Nodes, Acc). + +decode_query(Query = #{following := Cursor}, Nodes) -> + {Node, NodeCursor} = decode_cursor(Cursor), + {skip_query_nodes(Node, Nodes), Query#{following => NodeCursor}}; +decode_query(Query = #{}, Nodes) -> + {Nodes, Query}. + +skip_query_nodes(CNode, Nodes) -> + lists:dropwhile(fun(N) -> N < CNode end, Nodes). + +mk_cursor(Node, [_Last = #{transfer := Transfer, name := Name} | _]) -> + {Node, #{transfer => Transfer, name => Name}}. + +encode_cursor({Node, #{transfer := {ClientId, FileId}, name := Name}}) -> + emqx_utils_json:encode(#{ + <<"n">> => Node, + <<"cid">> => ClientId, + <<"fid">> => FileId, + <<"fn">> => unicode:characters_to_binary(Name) + }). + +decode_cursor(Cursor) -> + try + #{ + <<"n">> := NodeIn, + <<"cid">> := ClientId, + <<"fid">> := FileId, + <<"fn">> := NameIn + } = emqx_utils_json:decode(Cursor), + true = is_binary(ClientId), + true = is_binary(FileId), + Node = binary_to_existing_atom(NodeIn), + Name = unicode:characters_to_list(NameIn), + true = is_list(Name), + {Node, #{transfer => {ClientId, FileId}, name => Name}} + catch + error:{_, invalid_json} -> + error({badarg, cursor}); + error:{badmatch, _} -> + error({badarg, cursor}); + error:badarg -> + error({badarg, cursor}) end. %% @@ -352,9 +467,9 @@ mk_result_reldir(Transfer = {ClientId, FileId}) -> BucketRest/binary >> = binary:encode_hex(Hash), [ - Bucket1, - Bucket2, - BucketRest, + binary_to_list(Bucket1), + binary_to_list(Bucket2), + binary_to_list(BucketRest), emqx_ft_fs_util:escape_filename(ClientId), emqx_ft_fs_util:escape_filename(FileId) ]. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl index 943c053ff..13160bfc6 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl @@ -21,15 +21,15 @@ -module(emqx_ft_storage_exporter_fs_proxy). -export([ - list_exports_local/0, + list_exports_local/1, read_export_file_local/2 ]). -list_exports_local() -> +list_exports_local(Query) -> emqx_ft_storage:with_storage_type(local, fun(Storage) -> case emqx_ft_storage_exporter:exporter(Storage) of {emqx_ft_storage_exporter_fs, Options} -> - emqx_ft_storage_exporter_fs:list_local(Options) + emqx_ft_storage_exporter_fs:list_local(Options, Query) % NOTE % This case clause is currently deemed unreachable by dialyzer. % InvalidExporter -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 823407307..bd88727c0 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -45,7 +45,7 @@ -export([get_subdir/2]). -export([get_subdir/3]). --export([files/1]). +-export([files/2]). -export([on_config_update/2]). @@ -217,8 +217,8 @@ assemble(Storage, Transfer, Size) -> %% -files(Storage) -> - emqx_ft_storage_exporter:list(Storage). +files(Storage, Query) -> + emqx_ft_storage_exporter:list(Storage, Query). %% diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl index 4c64011de..9ca6db786 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl @@ -20,7 +20,7 @@ -export([introduced_in/0]). --export([list_exports/1]). +-export([list_exports/2]). -export([read_export_file/3]). -include_lib("emqx/include/bpapi.hrl"). @@ -28,14 +28,17 @@ introduced_in() -> "5.0.17". --spec list_exports([node()]) -> - emqx_rpc:erpc_multicall([emqx_ft_storage:file_info()]). -list_exports(Nodes) -> +-spec list_exports([node()], emqx_ft_storage:query(_LocalCursor)) -> + emqx_rpc:erpc_multicall( + {ok, [emqx_ft_storage:file_info()]} + | {error, file:posix() | disabled | {invalid_storage_type, _}} + ). +list_exports(Nodes, Query) -> erpc:multicall( Nodes, emqx_ft_storage_exporter_fs_proxy, list_exports_local, - [] + [Query] ). -spec read_export_file(node(), file:name(), pid()) -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index d3a3aee21..929665ca9 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -671,7 +671,7 @@ encode_meta(Meta) -> emqx_utils_json:encode(emqx_ft:encode_filemeta(Meta)). list_files(ClientId) -> - {ok, Files} = emqx_ft_storage:files(), + {ok, #{items := Files}} = emqx_ft_storage:files(), [File || File = #{transfer := {CId, _}} <- Files, CId == ClientId]. read_export(#{path := AbsFilepath}) -> diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index 523026d5a..d9abd5a22 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -47,17 +47,31 @@ end_per_testcase(_Case, _Config) -> %% Tests %%-------------------------------------------------------------------- -t_list_ready_transfers(Config) -> +t_list_files(Config) -> ClientId = client_id(Config), + FileId = <<"f1">>, - ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, "f1", <<"data">>), + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "f1", <<"data">>), {ok, 200, #{<<"files">> := Files}} = - request(get, uri(["file_transfer", "files"]), fun json/1), + request_json(get, uri(["file_transfer", "files"])), - ?assertInclude( - #{<<"clientid">> := ClientId, <<"fileid">> := <<"f1">>}, - Files + ?assertMatch( + [#{<<"clientid">> := ClientId, <<"fileid">> := <<"f1">>}], + [File || File = #{<<"clientid">> := CId} <- Files, CId == ClientId] + ), + + {ok, 200, #{<<"files">> := FilesTransfer}} = + request_json(get, uri(["file_transfer", "files", ClientId, FileId])), + + ?assertMatch( + [#{<<"clientid">> := ClientId, <<"fileid">> := <<"f1">>}], + FilesTransfer + ), + + ?assertMatch( + {ok, 404, #{<<"code">> := <<"FILES_NOT_FOUND">>}}, + request_json(get, uri(["file_transfer", "files", ClientId, <<"no-such-file">>])) ). t_download_transfer(Config) -> @@ -67,10 +81,9 @@ t_download_transfer(Config) -> ?assertMatch( {ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}}, - request( + request_json( get, - uri(["file_transfer", "file"]) ++ query(#{fileref => <<"f1">>}), - fun json/1 + uri(["file_transfer", "file"]) ++ query(#{fileref => <<"f1">>}) ) ), @@ -99,7 +112,7 @@ t_download_transfer(Config) -> ), {ok, 200, #{<<"files">> := [File]}} = - request(get, uri(["file_transfer", "files"]), fun json/1), + request_json(get, uri(["file_transfer", "files"])), {ok, 200, Response} = request(get, host() ++ maps:get(<<"uri">>, File)), @@ -108,6 +121,58 @@ t_download_transfer(Config) -> Response ). +t_list_files_paging(Config) -> + ClientId = client_id(Config), + NFiles = 20, + Uploads = [{mk_file_id("file:", N), mk_file_name(N)} || N <- lists:seq(1, NFiles)], + ok = lists:foreach( + fun({FileId, Name}) -> + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, Name, <<"data">>) + end, + Uploads + ), + + ?assertMatch( + {ok, 200, #{<<"files">> := [_, _, _], <<"cursor">> := _}}, + request_json(get, uri(["file_transfer", "files"]) ++ query(#{limit => 3})) + ), + + {ok, 200, #{<<"files">> := Files}} = + request_json(get, uri(["file_transfer", "files"]) ++ query(#{limit => 100})), + + ?assert(length(Files) >= NFiles), + + ?assertNotMatch( + {ok, 200, #{<<"cursor">> := _}}, + request_json(get, uri(["file_transfer", "files"]) ++ query(#{limit => 100})) + ), + + ?assertMatch( + {ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}}, + request_json(get, uri(["file_transfer", "files"]) ++ query(#{limit => 0})) + ), + + ?assertMatch( + {ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}}, + request_json( + get, + uri(["file_transfer", "files"]) ++ query(#{following => <<"whatsthat!?">>}) + ) + ), + + PageThrough = fun PageThrough(Query, Acc) -> + case request_json(get, uri(["file_transfer", "files"]) ++ query(Query)) of + {ok, 200, #{<<"files">> := FilesPage, <<"cursor">> := Cursor}} -> + PageThrough(Query#{following => Cursor}, Acc ++ FilesPage); + {ok, 200, #{<<"files">> := FilesPage}} -> + Acc ++ FilesPage + end + end, + + ?assertEqual(Files, PageThrough(#{limit => 1}, [])), + ?assertEqual(Files, PageThrough(#{limit => 8}, [])), + ?assertEqual(Files, PageThrough(#{limit => NFiles}, [])). + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- @@ -115,13 +180,19 @@ t_download_transfer(Config) -> client_id(Config) -> atom_to_binary(?config(tc, Config), utf8). +mk_file_id(Prefix, N) -> + iolist_to_binary([Prefix, integer_to_list(N)]). + +mk_file_name(N) -> + "file." ++ integer_to_list(N). + request(Method, Url) -> emqx_mgmt_api_test_util:request(Method, Url, []). -request(Method, Url, Decoder) when is_function(Decoder) -> +request_json(Method, Url) -> case emqx_mgmt_api_test_util:request(Method, Url, []) of {ok, Code, Body} -> - {ok, Code, Decoder(Body)}; + {ok, Code, json(Body)}; Otherwise -> Otherwise end. @@ -138,6 +209,8 @@ uri_encode(T) -> to_list(A) when is_atom(A) -> atom_to_list(A); +to_list(A) when is_integer(A) -> + integer_to_list(A); to_list(B) when is_binary(B) -> binary_to_list(B); to_list(L) when is_list(L) -> diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index a7323fc0e..24a4593c2 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -240,7 +240,7 @@ list_exports(Config) -> list_exports(Config, Transfer) -> {emqx_ft_storage_exporter_fs, Options} = exporter(Config), - emqx_ft_storage_exporter_fs:list_local(Options, Transfer). + emqx_ft_storage_exporter_fs:list_local_transfer(Options, Transfer). exporter(Config) -> emqx_ft_storage_exporter:exporter(storage(Config)). diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index d4c13f7d1..e0675d167 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -87,5 +87,5 @@ storage(Config) -> emqx_ft_test_helpers:local_storage(Config). list_files(Config) -> - {ok, Files} = emqx_ft_storage_fs:files(storage(Config)), + {ok, #{items := Files}} = emqx_ft_storage_fs:files(storage(Config), #{}), Files. diff --git a/rel/i18n/emqx_ft_api.hocon b/rel/i18n/emqx_ft_api.hocon index 0c67db554..bf6c22411 100644 --- a/rel/i18n/emqx_ft_api.hocon +++ b/rel/i18n/emqx_ft_api.hocon @@ -3,6 +3,9 @@ emqx_ft_api { file_list.desc: """List all uploaded files.""" +file_list_transfer.desc +"""List a file uploaded during specified transfer, identified by client id and file id.""" + } emqx_ft_storage_exporter_fs_api { From 75cceffa06d09d319c4288ed07b300751cfa97db Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Sat, 1 Apr 2023 00:17:55 +0300 Subject: [PATCH 136/156] fix(ft-test): rename testcases for consistency --- apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index e0675d167..5635d981b 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -27,7 +27,7 @@ all() -> {group, cluster} ]. --define(CLUSTER_CASES, [t_multinode_ready_transfers]). +-define(CLUSTER_CASES, [t_multinode_exports]). groups() -> [ @@ -61,7 +61,7 @@ end_per_group(_Group, _Config) -> %% Tests %%-------------------------------------------------------------------- -t_multinode_ready_transfers(Config) -> +t_multinode_exports(Config) -> Node1 = ?config(additional_node, Config), ok = emqx_ft_test_helpers:upload_file(<<"c/1">>, <<"f:1">>, "fn1", <<"data">>, Node1), From a9866fede4cb4feba41a78d79885843ade0ad729 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 25 Apr 2023 14:42:26 +0300 Subject: [PATCH 137/156] feat(ft-api): support paging in S3 storage exporter --- .../src/emqx_ft_storage_exporter_s3.erl | 118 ++++++++++++------ 1 file changed, 82 insertions(+), 36 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index adf000346..c7110c74a 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -23,7 +23,7 @@ -export([write/2]). -export([complete/2]). -export([discard/1]). --export([list/1]). +-export([list/2]). -export([ start/1, @@ -43,6 +43,10 @@ filemeta => filemeta() }. +-type query() :: emqx_ft_storage:query(cursor()). +-type page(T) :: emqx_ft_storage:page(T, cursor()). +-type cursor() :: iodata(). + -type export_st() :: #{ pid := pid(), filemeta := filemeta(), @@ -92,10 +96,10 @@ complete(#{pid := Pid} = _ExportSt, _Checksum) -> discard(#{pid := Pid} = _ExportSt) -> emqx_s3_uploader:abort(Pid). --spec list(options()) -> - {ok, [exportinfo()]} | {error, term()}. -list(Options) -> - emqx_s3:with_client(?S3_PROFILE_ID, fun(Client) -> list(Client, Options) end). +-spec list(options(), query()) -> + {ok, page(exportinfo())} | {error, term()}. +list(Options, Query) -> + emqx_s3:with_client(?S3_PROFILE_ID, fun(Client) -> list(Client, Options, Query) end). %%-------------------------------------------------------------------- %% Exporter behaviour (lifecycle) @@ -117,12 +121,11 @@ update(_OldOptions, NewOptions) -> %% Internal functions %% ------------------------------------------------------------------- -s3_key({ClientId, FileId} = _Transfer, #{name := Filename}) -> - filename:join([ - emqx_ft_fs_util:escape_filename(ClientId), - emqx_ft_fs_util:escape_filename(FileId), - Filename - ]). +s3_key(Transfer, #{name := Filename}) -> + s3_prefix(Transfer) ++ "/" ++ Filename. + +s3_prefix({ClientId, FileId} = _Transfer) -> + emqx_ft_fs_util:escape_filename(ClientId) ++ "/" ++ emqx_ft_fs_util:escape_filename(FileId). s3_headers({ClientId, FileId}, Filemeta) -> #{ @@ -137,54 +140,97 @@ s3_headers({ClientId, FileId}, Filemeta) -> s3_header_filemeta(Filemeta) -> emqx_utils_json:encode(emqx_ft:encode_filemeta(Filemeta), [force_utf8, uescape]). -list(Client, Options) -> - case list_key_info(Client, Options) of - {ok, KeyInfos} -> - MaybeExportInfos = lists:map( - fun(KeyInfo) -> key_info_to_exportinfo(Client, KeyInfo, Options) end, KeyInfos - ), - ExportInfos = [ExportInfo || {ok, ExportInfo} <- MaybeExportInfos], - {ok, ExportInfos}; +list(Client, _Options, #{transfer := Transfer}) -> + case list_key_info(Client, [{prefix, s3_prefix(Transfer)}, {max_keys, ?S3_LIST_LIMIT}]) of + {ok, {Exports, _Marker}} -> + {ok, #{items => Exports}}; + {error, _Reason} = Error -> + Error + end; +list(Client, _Options, Query) -> + Limit = maps:get(limit, Query, undefined), + Marker = emqx_maybe:apply(fun decode_cursor/1, maps:get(cursor, Query, undefined)), + case list_pages(Client, Marker, Limit, []) of + {ok, {Exports, undefined}} -> + {ok, #{items => Exports}}; + {ok, {Exports, NextMarker}} -> + {ok, #{items => Exports, cursor => encode_cursor(NextMarker)}}; {error, _Reason} = Error -> Error end. -list_key_info(Client, Options) -> - list_key_info(Client, Options, _Marker = [], _Acc = []). +list_pages(Client, Marker, Limit, Acc) -> + MaxKeys = min(?S3_LIST_LIMIT, Limit), + ListOptions = [{marker, Marker} || Marker =/= undefined], + case list_key_info(Client, [{max_keys, MaxKeys} | ListOptions]) of + {ok, {Exports, NextMarker}} -> + list_accumulate(Client, Limit, NextMarker, [Exports | Acc]); + {error, _Reason} = Error -> + Error + end. -list_key_info(Client, Options, Marker, Acc) -> - ListOptions = [{max_keys, ?S3_LIST_LIMIT}] ++ Marker, +list_accumulate(_Client, _Limit, undefined, Acc) -> + {ok, {flatten_pages(Acc), undefined}}; +list_accumulate(Client, undefined, Marker, Acc) -> + list_pages(Client, Marker, undefined, Acc); +list_accumulate(Client, Limit, Marker, Acc = [Exports | _]) -> + case Limit - length(Exports) of + 0 -> + {ok, {flatten_pages(Acc), Marker}}; + Left -> + list_pages(Client, Marker, Left, Acc) + end. + +flatten_pages(Pages) -> + lists:append(lists:reverse(Pages)). + +list_key_info(Client, ListOptions) -> case emqx_s3_client:list(Client, ListOptions) of {ok, Result} -> ?SLOG(debug, #{msg => "list_key_info", result => Result}), KeyInfos = proplists:get_value(contents, Result, []), - case proplists:get_value(is_truncated, Result, false) of - true -> - NewMarker = next_marker(KeyInfos), - list_key_info(Client, Options, NewMarker, [KeyInfos | Acc]); - false -> - {ok, lists:append(lists:reverse([KeyInfos | Acc]))} - end; + Exports = lists:filtermap( + fun(KeyInfo) -> key_info_to_exportinfo(Client, KeyInfo) end, KeyInfos + ), + Marker = + case proplists:get_value(is_truncated, Result, false) of + true -> + next_marker(KeyInfos); + false -> + undefined + end, + {ok, {Exports, Marker}}; {error, _Reason} = Error -> Error end. -next_marker(KeyInfos) -> - [{marker, proplists:get_value(key, lists:last(KeyInfos))}]. +encode_cursor(Key) -> + unicode:characters_to_binary(Key). -key_info_to_exportinfo(Client, KeyInfo, _Options) -> +decode_cursor(Cursor) -> + case unicode:characters_to_list(Cursor) of + Key when is_list(Key) -> + Key; + _ -> + error({badarg, cursor}) + end. + +next_marker(KeyInfos) -> + proplists:get_value(key, lists:last(KeyInfos)). + +key_info_to_exportinfo(Client, KeyInfo) -> Key = proplists:get_value(key, KeyInfo), case parse_transfer_and_name(Key) of {ok, {Transfer, Name}} -> - {ok, #{ + {true, #{ transfer => Transfer, name => unicode:characters_to_binary(Name), uri => emqx_s3_client:uri(Client, Key), timestamp => datetime_to_epoch_second(proplists:get_value(last_modified, KeyInfo)), size => proplists:get_value(size, KeyInfo) }}; - {error, _Reason} = Error -> - Error + {error, _Reason} -> + false end. -define(EPOCH_START, 62167219200). From bd9f129bb84953e3848b1db449470bb3dbb6aa73 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 25 Apr 2023 14:44:00 +0300 Subject: [PATCH 138/156] test(ft-api): also run testcases under cluster setup --- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 80 +++++++++++++++++++++---- 1 file changed, 68 insertions(+), 12 deletions(-) diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index d9abd5a22..4efa31205 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -22,11 +22,19 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). --include_lib("emqx/include/asserts.hrl"). - -import(emqx_dashboard_api_test_helpers, [host/0, uri/1]). -all() -> emqx_common_test_helpers:all(?MODULE). +all() -> + [ + {group, single}, + {group, cluster} + ]. + +groups() -> + [ + {single, [], emqx_common_test_helpers:all(?MODULE)}, + {cluster, [], emqx_common_test_helpers:all(?MODULE)} + ]. init_per_suite(Config) -> ok = emqx_mgmt_api_test_util:init_suite( @@ -38,6 +46,41 @@ end_per_suite(_Config) -> ok = emqx_mgmt_api_test_util:end_suite([emqx_ft, emqx_conf]), ok. +init_per_group(Group = cluster, Config) -> + Cluster = mk_cluster_specs(Config), + ct:pal("Starting ~p", [Cluster]), + Nodes = [ + emqx_common_test_helpers:start_slave(Name, Opts#{join_to => node()}) + || {Name, Opts} <- Cluster + ], + [{group, Group}, {cluster_nodes, Nodes} | Config]; +init_per_group(Group, Config) -> + [{group, Group} | Config]. + +end_per_group(cluster, Config) -> + ok = lists:foreach( + fun emqx_ft_test_helpers:stop_additional_node/1, + ?config(cluster_nodes, Config) + ); +end_per_group(_Group, _Config) -> + ok. + +mk_cluster_specs(Config) -> + Specs = [ + {core, emqx_ft_api_SUITE1, #{listener_ports => [{tcp, 2883}]}}, + {core, emqx_ft_api_SUITE2, #{listener_ports => [{tcp, 3883}]}} + ], + CommOpts = [ + {env, [{emqx, boot_modules, [broker, listeners]}]}, + {apps, [emqx_ft]}, + {conf, [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]]}, + {env_handler, emqx_ft_test_helpers:env_handler(Config)} + ], + emqx_common_test_helpers:emqx_cluster( + Specs, + CommOpts + ). + init_per_testcase(Case, Config) -> [{tc, Case} | Config]. end_per_testcase(_Case, _Config) -> @@ -51,7 +94,8 @@ t_list_files(Config) -> ClientId = client_id(Config), FileId = <<"f1">>, - ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "f1", <<"data">>), + Node = lists:last(cluster(Config)), + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "f1", <<"data">>, Node), {ok, 200, #{<<"files">> := Files}} = request_json(get, uri(["file_transfer", "files"])), @@ -76,14 +120,16 @@ t_list_files(Config) -> t_download_transfer(Config) -> ClientId = client_id(Config), + FileId = <<"f1">>, - ok = emqx_ft_test_helpers:upload_file(ClientId, <<"f1">>, "f1", <<"data">>), + Node = lists:last(cluster(Config)), + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "f1", <<"data">>, Node), ?assertMatch( {ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}}, request_json( get, - uri(["file_transfer", "file"]) ++ query(#{fileref => <<"f1">>}) + uri(["file_transfer", "file"]) ++ query(#{fileref => FileId}) ) ), @@ -93,7 +139,7 @@ t_download_transfer(Config) -> get, uri(["file_transfer", "file"]) ++ query(#{ - fileref => <<"f1">>, + fileref => FileId, node => <<"nonode@nohost">> }) ) @@ -112,7 +158,7 @@ t_download_transfer(Config) -> ), {ok, 200, #{<<"files">> := [File]}} = - request_json(get, uri(["file_transfer", "files"])), + request_json(get, uri(["file_transfer", "files", ClientId, FileId])), {ok, 200, Response} = request(get, host() ++ maps:get(<<"uri">>, File)), @@ -124,10 +170,14 @@ t_download_transfer(Config) -> t_list_files_paging(Config) -> ClientId = client_id(Config), NFiles = 20, - Uploads = [{mk_file_id("file:", N), mk_file_name(N)} || N <- lists:seq(1, NFiles)], + Nodes = cluster(Config), + Uploads = [ + {mk_file_id("file:", N), mk_file_name(N), pick(N, Nodes)} + || N <- lists:seq(1, NFiles) + ], ok = lists:foreach( - fun({FileId, Name}) -> - ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, Name, <<"data">>) + fun({FileId, Name, Node}) -> + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, Name, <<"data">>, Node) end, Uploads ), @@ -177,8 +227,11 @@ t_list_files_paging(Config) -> %% Helpers %%-------------------------------------------------------------------- +cluster(Config) -> + [node() | proplists:get_value(cluster_nodes, Config, [])]. + client_id(Config) -> - atom_to_binary(?config(tc, Config), utf8). + iolist_to_binary(io_lib:format("~s.~s", [?config(group, Config), ?config(tc, Config)])). mk_file_id(Prefix, N) -> iolist_to_binary([Prefix, integer_to_list(N)]). @@ -215,3 +268,6 @@ to_list(B) when is_binary(B) -> binary_to_list(B); to_list(L) when is_list(L) -> L. + +pick(N, List) -> + lists:nth(1 + (N rem length(List)), List). From cff7788b2e6e3114449cd8cbc98bf43c6482f3c6 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 27 Apr 2023 23:23:52 +0300 Subject: [PATCH 139/156] chore(ft): add s3 exporter tests --- .../src/emqx_dashboard_swagger.erl | 66 ++++-- apps/emqx_ft/docker-ct | 1 + apps/emqx_ft/etc/emqx_ft.conf | 15 -- apps/emqx_ft/src/emqx_ft_api.erl | 15 +- apps/emqx_ft/src/emqx_ft_conf.erl | 44 ++-- apps/emqx_ft/src/emqx_ft_storage.erl | 51 ++--- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 8 +- .../src/emqx_ft_storage_exporter_fs_api.erl | 4 +- .../src/emqx_ft_storage_exporter_s3.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 23 ++ apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 31 +++ apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 5 +- .../emqx_ft_storage_exporter_s3_SUITE.erl | 199 ++++++++++++++++++ .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 4 +- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 15 +- .../test/emqx_ft_storage_fs_reader_SUITE.erl | 2 +- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 54 +++-- apps/emqx_s3/test/emqx_s3_test_helpers.erl | 15 +- rel/i18n/emqx_ft_schema.hocon | 4 +- 19 files changed, 437 insertions(+), 121 deletions(-) create mode 100644 apps/emqx_ft/docker-ct create mode 100644 apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index e42d9c3ae..6422d627c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -29,6 +29,7 @@ -export([file_schema/1]). -export([base_path/0]). -export([relative_uri/1]). +-export([compose_filters/2]). -export([filter_check_request/2, filter_check_request_and_translate_body/2]). @@ -82,14 +83,30 @@ -type request() :: #{bindings => map(), query_string => map(), body => map()}. -type request_meta() :: #{module => module(), path => string(), method => atom()}. --type filter_result() :: {ok, request()} | {400, 'BAD_REQUEST', binary()}. --type filter() :: fun((request(), request_meta()) -> filter_result()). +%% More exact types are defined in minirest.hrl, but we don't want to include it +%% because it defines a lot of types and they may clash with the types declared locally. +-type status_code() :: pos_integer(). +-type error_code() :: atom() | binary(). +-type error_message() :: binary(). +-type response_body() :: term(). +-type headers() :: map(). + +-type response() :: + status_code() + | {status_code()} + | {status_code(), response_body()} + | {status_code(), headers(), response_body()} + | {status_code(), error_code(), error_message()}. + +-type filter_result() :: {ok, request()} | response(). +-type filter() :: emqx_maybe:t(fun((request(), request_meta()) -> filter_result())). -type spec_opts() :: #{ check_schema => boolean() | filter(), translate_body => boolean(), schema_converter => fun((hocon_schema:schema(), Module :: atom()) -> map()), - i18n_lang => atom() | string() | binary() + i18n_lang => atom() | string() | binary(), + filter => filter() }. -type route_path() :: string() | binary(). @@ -115,9 +132,9 @@ spec(Module, Options) -> lists:foldl( fun(Path, {AllAcc, AllRefsAcc}) -> {OperationId, Specs, Refs} = parse_spec_ref(Module, Path, Options), - CheckSchema = support_check_schema(Options), + Opts = #{filter => filter(Options)}, { - [{filename:join("/", Path), Specs, OperationId, CheckSchema} | AllAcc], + [{filename:join("/", Path), Specs, OperationId, Opts} | AllAcc], Refs ++ AllRefsAcc } end, @@ -204,6 +221,21 @@ file_schema(FileName) -> } }. +-spec compose_filters(filter(), filter()) -> filter(). +compose_filters(undefined, Filter2) -> + Filter2; +compose_filters(Filter1, undefined) -> + Filter1; +compose_filters(Filter1, Filter2) -> + fun(Request, RequestMeta) -> + case Filter1(Request, RequestMeta) of + {ok, Request1} -> + Filter2(Request1, RequestMeta); + Response -> + Response + end + end. + %%------------------------------------------------------------------------------ %% Private functions %%------------------------------------------------------------------------------ @@ -235,14 +267,22 @@ check_only(Schema, Map, Opts) -> _ = hocon_tconf:check_plain(Schema, Map, Opts), Map. -support_check_schema(#{check_schema := true, translate_body := true}) -> - #{filter => fun ?MODULE:filter_check_request_and_translate_body/2}; -support_check_schema(#{check_schema := true}) -> - #{filter => fun ?MODULE:filter_check_request/2}; -support_check_schema(#{check_schema := Filter}) when is_function(Filter, 2) -> - #{filter => Filter}; -support_check_schema(_) -> - #{filter => undefined}. +filter(Options) -> + CheckSchemaFilter = check_schema_filter(Options), + CustomFilter = custom_filter(Options), + compose_filters(CheckSchemaFilter, CustomFilter). + +custom_filter(Options) -> + maps:get(filter, Options, undefined). + +check_schema_filter(#{check_schema := true, translate_body := true}) -> + fun ?MODULE:filter_check_request_and_translate_body/2; +check_schema_filter(#{check_schema := true}) -> + fun ?MODULE:filter_check_request/2; +check_schema_filter(#{check_schema := Filter}) when is_function(Filter, 2) -> + Filter; +check_schema_filter(_) -> + undefined. parse_spec_ref(Module, Path, Options) -> Schema = diff --git a/apps/emqx_ft/docker-ct b/apps/emqx_ft/docker-ct new file mode 100644 index 000000000..36f9d86d3 --- /dev/null +++ b/apps/emqx_ft/docker-ct @@ -0,0 +1 @@ +minio diff --git a/apps/emqx_ft/etc/emqx_ft.conf b/apps/emqx_ft/etc/emqx_ft.conf index ec48d4e69..e69de29bb 100644 --- a/apps/emqx_ft/etc/emqx_ft.conf +++ b/apps/emqx_ft/etc/emqx_ft.conf @@ -1,15 +0,0 @@ -file_transfer { - storage { - type = local - segments { - root = "{{ platform_data_dir }}/file_transfer/segments" - gc { - interval = "1h" - } - } - exporter { - root = "{{ platform_data_dir }}/file_transfer/exports" - type = local - } - } -} diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index 7e1ed97ad..61a6a2c93 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -33,6 +33,9 @@ fields/1 ]). +%% Minirest filter for checking if file transfer is enabled +-export([check_ft_enabled/2]). + %% API callbacks -export([ '/file_transfer/files'/2, @@ -44,7 +47,9 @@ namespace() -> "file_transfer". api_spec() -> - emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + emqx_dashboard_swagger:spec(?MODULE, #{ + check_schema => true, filter => fun ?MODULE:check_ft_enabled/2 + }). paths() -> [ @@ -97,6 +102,14 @@ schema("/file_transfer/files/:clientid/:fileid") -> } }. +check_ft_enabled(Params, _Meta) -> + case emqx_ft_conf:enabled() of + true -> + {ok, Params}; + false -> + {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} + end. + '/file_transfer/files'(get, #{ query_string := QueryString }) -> diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 90b59c8d1..61f639271 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -56,7 +56,7 @@ enabled() -> -spec storage() -> _Storage. storage() -> - emqx_config:get([file_transfer, storage], undefined). + emqx_config:get([file_transfer, storage]). -spec gc_interval(_Storage) -> emqx_maybe:t(milliseconds()). gc_interval(Conf = #{type := local}) -> @@ -88,11 +88,12 @@ store_segment_timeout() -> -spec load() -> ok. load() -> - ok = on_config_update(#{}, emqx_config:get([file_transfer], #{})), + ok = maybe_start(), emqx_conf:add_handler([file_transfer], ?MODULE). -spec unload() -> ok. unload() -> + ok = stop(), emqx_conf:remove_handler([file_transfer]). %%-------------------------------------------------------------------- @@ -115,23 +116,26 @@ pre_config_update(_, Req, _Config) -> post_config_update([file_transfer | _], _Req, NewConfig, OldConfig, _AppEnvs) -> on_config_update(OldConfig, NewConfig). -on_config_update(OldConfig, NewConfig) -> - lists:foreach( - fun(ConfKey) -> - on_config_update( - ConfKey, - maps:get(ConfKey, OldConfig, undefined), - maps:get(ConfKey, NewConfig, undefined) - ) - end, - [storage, enable] - ). - -on_config_update(_, Config, Config) -> +on_config_update(#{enable := false}, #{enable := false}) -> ok; -on_config_update(storage, OldConfig, NewConfig) -> - ok = emqx_ft_storage:on_config_update(OldConfig, NewConfig); -on_config_update(enable, _, true) -> +on_config_update(#{enable := true, storage := OldStorage}, #{enable := false}) -> + ok = emqx_ft_storage:on_config_update(OldStorage, undefined), + ok = emqx_ft:unhook(); +on_config_update(#{enable := false}, #{enable := true, storage := NewStorage}) -> + ok = emqx_ft_storage:on_config_update(undefined, NewStorage), ok = emqx_ft:hook(); -on_config_update(enable, _, false) -> - ok = emqx_ft:unhook(). +on_config_update(#{enable := true, storage := OldStorage}, #{enable := true, storage := NewStorage}) -> + ok = emqx_ft_storage:on_config_update(OldStorage, NewStorage). + +maybe_start() -> + case emqx_config:get([file_transfer]) of + #{enable := true, storage := Storage} -> + ok = emqx_ft_storage:on_config_update(undefined, Storage), + ok = emqx_ft:hook(); + _ -> + ok + end. + +stop() -> + ok = emqx_ft:unhook(), + ok = emqx_ft_storage:on_config_update(storage(), undefined). diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 5364211a4..5ec342585 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -90,6 +90,12 @@ -callback files(storage(), query(Cursor)) -> {ok, page(file_info(), Cursor)} | {error, term()}. +-callback start(emqx_config:config()) -> any(). +-callback stop(emqx_config:config()) -> any(). + +-callback on_config_update(_OldConfig :: emqx_config:config(), _NewConfig :: emqx_config:config()) -> + any(). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -167,49 +173,28 @@ apply_storage(Storage, Fun, Args) when is_function(Fun) -> -spec on_config_update(_Old :: emqx_maybe:t(storage()), _New :: emqx_maybe:t(storage())) -> ok. -on_config_update(Storage, Storage) -> +on_config_update(#{type := _} = Storage, #{type := _} = Storage) -> ok; on_config_update(#{type := Type} = StorageOld, #{type := Type} = StorageNew) -> ok = (mod(StorageNew)):on_config_update(StorageOld, StorageNew); -on_config_update(StorageOld, StorageNew) -> +on_config_update(StorageOld, StorageNew) when + (StorageOld =:= undefined orelse is_map_key(type, StorageOld)) andalso + (StorageNew =:= undefined orelse is_map_key(type, StorageNew)) +-> _ = emqx_maybe:apply(fun on_storage_stop/1, StorageOld), _ = emqx_maybe:apply(fun on_storage_start/1, StorageNew), - _ = emqx_maybe:apply( - fun(Storage) -> (mod(Storage)):on_config_update(StorageOld, StorageNew) end, - StorageNew - ), ok. -on_storage_start(Storage = #{type := _}) -> - lists:foreach( - fun(ChildSpec) -> - {ok, _Child} = supervisor:start_child(emqx_ft_sup, ChildSpec) - end, - child_spec(Storage) - ). - -on_storage_stop(Storage = #{type := _}) -> - lists:foreach( - fun(#{id := ChildId}) -> - _ = supervisor:terminate_child(emqx_ft_sup, ChildId), - ok = supervisor:delete_child(emqx_ft_sup, ChildId) - end, - child_spec(Storage) - ). - -child_spec(Storage) -> - try - Mod = mod(Storage), - Mod:child_spec(Storage) - catch - error:disabled -> []; - error:undef -> [] - end. - %%-------------------------------------------------------------------- -%% Local FS API +%% Local API %%-------------------------------------------------------------------- +on_storage_start(Storage) -> + (mod(Storage)):start(Storage). + +on_storage_stop(Storage) -> + (mod(Storage)):stop(Storage). + storage() -> emqx_ft_conf:storage(). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index fb44093c1..e000fe5c6 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -153,14 +153,14 @@ on_exporter_update(Config, Config) -> on_exporter_update({ExporterMod, ConfigOld}, {ExporterMod, ConfigNew}) -> ExporterMod:update(ConfigOld, ConfigNew); on_exporter_update(ExporterOld, ExporterNew) -> - _ = emqx_maybe:apply(fun stop_exporter/1, ExporterOld), - _ = emqx_maybe:apply(fun start_exporter/1, ExporterNew), + _ = emqx_maybe:apply(fun stop/1, ExporterOld), + _ = emqx_maybe:apply(fun start/1, ExporterNew), ok. -start_exporter({ExporterMod, ExporterOpts}) -> +start({ExporterMod, ExporterOpts}) -> ok = ExporterMod:start(ExporterOpts). -stop_exporter({ExporterMod, ExporterOpts}) -> +stop({ExporterMod, ExporterOpts}) -> ok = ExporterMod:stop(ExporterOpts). %%------------------------------------------------------------------------------ diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl index f1a8c6dae..a278b01a5 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl @@ -47,7 +47,9 @@ namespace() -> "file_transfer". api_spec() -> - emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + emqx_dashboard_swagger:spec(?MODULE, #{ + check_schema => true, filter => fun emqx_ft_api:check_ft_enabled/2 + }). paths() -> [ diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index c7110c74a..5c5aade86 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -149,7 +149,7 @@ list(Client, _Options, #{transfer := Transfer}) -> end; list(Client, _Options, Query) -> Limit = maps:get(limit, Query, undefined), - Marker = emqx_maybe:apply(fun decode_cursor/1, maps:get(cursor, Query, undefined)), + Marker = emqx_maybe:apply(fun decode_cursor/1, maps:get(following, Query, undefined)), case list_pages(Client, Marker, Limit, []) of {ok, {Exports, undefined}} -> {ok, #{items => Exports}}; diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index bd88727c0..cdc86d218 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -48,6 +48,8 @@ -export([files/2]). -export([on_config_update/2]). +-export([start/1]). +-export([stop/1]). -export_type([storage/0]). -export_type([filefrag/1]). @@ -227,6 +229,27 @@ on_config_update(StorageOld, StorageNew) -> ok = emqx_ft_storage_fs_gc:reset(StorageNew), emqx_ft_storage_exporter:on_config_update(StorageOld, StorageNew). +start(Storage) -> + ok = lists:foreach( + fun(ChildSpec) -> + {ok, _Child} = supervisor:start_child(emqx_ft_sup, ChildSpec) + end, + child_spec(Storage) + ), + ok = emqx_ft_storage_exporter:on_config_update(undefined, Storage), + ok. + +stop(Storage) -> + ok = emqx_ft_storage_exporter:on_config_update(Storage, undefined), + ok = lists:foreach( + fun(#{id := ChildId}) -> + _ = supervisor:terminate_child(emqx_ft_sup, ChildId), + ok = supervisor:delete_child(emqx_ft_sup, ChildId) + end, + child_spec(Storage) + ), + ok. + %% -spec transfers(storage()) -> diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index 4efa31205..f69e13a6d 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -83,6 +83,8 @@ mk_cluster_specs(Config) -> init_per_testcase(Case, Config) -> [{tc, Case} | Config]. +end_per_testcase(t_ft_disabled, _Config) -> + emqx_config:put([file_transfer, enable], true); end_per_testcase(_Case, _Config) -> ok. @@ -223,6 +225,35 @@ t_list_files_paging(Config) -> ?assertEqual(Files, PageThrough(#{limit => 8}, [])), ?assertEqual(Files, PageThrough(#{limit => NFiles}, [])). +t_ft_disabled(_Config) -> + ?assertMatch( + {ok, 200, _}, + request_json(get, uri(["file_transfer", "files"])) + ), + + ?assertMatch( + {ok, 400, _}, + request_json( + get, + uri(["file_transfer", "file"]) ++ query(#{fileref => <<"f1">>}) + ) + ), + + ok = emqx_config:put([file_transfer, enable], false), + + ?assertMatch( + {ok, 503, _}, + request_json(get, uri(["file_transfer", "files"])) + ), + + ?assertMatch( + {ok, 503, _}, + request_json( + get, + uri(["file_transfer", "file"]) ++ query(#{fileref => <<"f1">>, node => node()}) + ) + ). + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index 106c34702..bc9eb5d98 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -32,9 +32,10 @@ end_per_suite(_Config) -> ok. init_per_testcase(_Case, Config) -> - % NOTE: running each testcase with clean config _ = emqx_config:save_schema_mod_and_names(emqx_ft_schema), - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], fun(_) -> ok end), + ok = emqx_common_test_helpers:start_apps( + [emqx_conf, emqx_ft], emqx_ft_test_helpers:env_handler(Config) + ), {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. diff --git a/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl new file mode 100644 index 000000000..86de74ee0 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl @@ -0,0 +1,199 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_exporter_s3_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-define(assertS3Data(Data, Url), + case httpc:request(Url) of + {ok, {{_StatusLine, 200, "OK"}, _Headers, Body}} -> + ?assertEqual(Data, list_to_binary(Body), "S3 data mismatch"); + OtherResponse -> + ct:fail("Unexpected response: ~p", [OtherResponse]) + end +). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Config. +end_per_suite(_Config) -> + ok. + +set_special_configs(Config) -> + fun + (emqx_ft) -> + Storage = emqx_ft_test_helpers:local_storage(Config, #{ + exporter => s3, bucket_name => ?config(bucket_name, Config) + }), + emqx_ft_test_helpers:load_config(#{<<"enable">> => true, <<"storage">> => Storage}); + (_) -> + ok + end. + +init_per_testcase(Case, Config0) -> + ClientId = atom_to_binary(Case), + BucketName = create_bucket(), + Config1 = [{bucket_name, BucketName}, {clientid, ClientId} | Config0], + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], set_special_configs(Config1)), + Config1. +end_per_testcase(_Case, _Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok. + +%%-------------------------------------------------------------------- +%% Test Cases +%%------------------------------------------------------------------- + +t_happy_path(Config) -> + ClientId = ?config(clientid, Config), + + FileId = <<"🌚"/utf8>>, + Name = "cool_name", + Data = <<"data"/utf8>>, + + ?assertEqual( + ok, + emqx_ft_test_helpers:upload_file(ClientId, FileId, Name, Data) + ), + + {ok, #{items := [#{uri := Uri}]}} = emqx_ft_storage:files(), + + ?assertS3Data(Data, Uri), + + Key = binary_to_list(ClientId) ++ "/" ++ binary_to_list(FileId) ++ "/" ++ Name, + Meta = erlcloud_s3:get_object_metadata( + ?config(bucket_name, Config), Key, emqx_ft_test_helpers:aws_config() + ), + + ?assertEqual( + ClientId, + metadata_field("clientid", Meta) + ), + + ?assertEqual( + FileId, + metadata_field("fileid", Meta) + ), + + NameBin = list_to_binary(Name), + ?assertMatch( + #{ + <<"name">> := NameBin, + <<"size">> := 4 + }, + emqx_utils_json:decode(metadata_field("filemeta", Meta), [return_maps]) + ). + +t_upload_error(Config) -> + ClientId = ?config(clientid, Config), + + FileId = <<"🌚"/utf8>>, + Name = "cool_name", + Data = <<"data"/utf8>>, + + {ok, _} = emqx_conf:update( + [file_transfer, storage, exporter, bucket], <<"invalid-bucket">>, #{} + ), + + ?assertEqual( + {error, unspecified_error}, + emqx_ft_test_helpers:upload_file(ClientId, FileId, Name, Data) + ). + +t_paging(Config) -> + ClientId = ?config(clientid, Config), + N = 1050, + + FileId = fun integer_to_binary/1, + Name = "cool_name", + Data = fun integer_to_binary/1, + + ok = lists:foreach( + fun(I) -> + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId(I), Name, Data(I)) + end, + lists:seq(1, N) + ), + + {ok, #{items := [#{uri := Uri}]}} = emqx_ft_storage:files(#{transfer => {ClientId, FileId(123)}}), + + ?assertS3Data(Data(123), Uri), + + lists:foreach( + fun(PageSize) -> + Pages = file_pages(#{limit => PageSize}), + ?assertEqual( + expected_page_count(PageSize, N), + length(Pages) + ), + FileIds = [ + FId + || #{transfer := {_, FId}} <- lists:concat(Pages) + ], + ?assertEqual( + lists:sort([FileId(I) || I <- lists:seq(1, N)]), + lists:sort(FileIds) + ) + end, + %% less than S3 limit, greater than S3 limit + [20, 550] + ). + +t_invalid_cursor(_Config) -> + InvalidUtf8 = <<16#80>>, + ?assertError( + {badarg, cursor}, + emqx_ft_storage:files(#{following => InvalidUtf8}) + ). + +%%-------------------------------------------------------------------- +%% Helper Functions +%%-------------------------------------------------------------------- + +expected_page_count(PageSize, Total) -> + case Total rem PageSize of + 0 -> Total div PageSize; + _ -> Total div PageSize + 1 + end. + +file_pages(Query) -> + case emqx_ft_storage:files(Query) of + {ok, #{items := Items, cursor := NewCursor}} -> + [Items] ++ file_pages(Query#{following => NewCursor}); + {ok, #{items := Items}} -> + [Items]; + {error, Error} -> + ct:fail("Failed to download files: ~p", [Error]) + end. + +metadata_field(Field, Meta) -> + Key = "x-amz-meta-" ++ Field, + case lists:keyfind(Key, 1, Meta) of + {Key, Value} -> list_to_binary(Value); + false -> false + end. + +create_bucket() -> + BucketName = emqx_s3_test_helpers:unique_bucket(), + _ = application:ensure_all_started(lhttpc), + ok = erlcloud_s3:create_bucket(BucketName, emqx_ft_test_helpers:aws_config()), + BucketName. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index 5635d981b..2acb57a8e 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -84,7 +84,9 @@ client_id(Config) -> atom_to_binary(?config(tc, Config), utf8). storage(Config) -> - emqx_ft_test_helpers:local_storage(Config). + RawConfig = #{<<"storage">> => emqx_ft_test_helpers:local_storage(Config)}, + #{storage := Storage} = emqx_ft_schema:translate(RawConfig), + Storage. list_files(Config) -> {ok, #{items := Files}} = emqx_ft_storage_fs:files(storage(Config), #{}), diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index 065e9ae0a..04aedf8f3 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -40,14 +40,15 @@ init_per_testcase(TC, Config) -> emqx_ft, fun(emqx_ft) -> emqx_ft_test_helpers:load_config(#{ - storage => #{ - type => local, - segments => #{ - root => emqx_ft_test_helpers:root(Config, node(), [TC, segments]) + <<"enable">> => true, + <<"storage">> => #{ + <<"type">> => <<"local">>, + <<"segments">> => #{ + <<"root">> => emqx_ft_test_helpers:root(Config, node(), [TC, segments]) }, - exporter => #{ - type => local, - root => emqx_ft_test_helpers:root(Config, node(), [TC, exports]) + <<"exporter">> => #{ + <<"type">> => <<"local">>, + <<"root">> => emqx_ft_test_helpers:root(Config, node(), [TC, exports]) } } }) diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl index 0ac5d2844..217205f6f 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl @@ -25,7 +25,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_ft]), + ok = emqx_common_test_helpers:start_apps([emqx_ft], emqx_ft_test_helpers:env_handler(Config)), Config. end_per_suite(_Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index 89e349fae..1482223d8 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -21,6 +21,9 @@ -include_lib("common_test/include/ct.hrl"). +-define(S3_HOST, <<"minio">>). +-define(S3_PORT, 9000). + start_additional_node(Config, Name) -> emqx_common_test_helpers:start_slave( Name, @@ -41,32 +44,41 @@ stop_additional_node(Node) -> env_handler(Config) -> fun (emqx_ft) -> - load_config(#{enable => true, storage => local_storage(Config)}); + load_config(#{<<"enable">> => true, <<"storage">> => local_storage(Config)}); (_) -> ok end. local_storage(Config) -> + local_storage(Config, #{exporter => local}). + +local_storage(Config, Opts) -> #{ - type => local, - segments => #{ - root => root(Config, node(), [segments]) - }, - exporter => #{ - type => local, - root => root(Config, node(), [exports]) - } + <<"type">> => <<"local">>, + <<"segments">> => #{<<"root">> => root(Config, node(), [segments])}, + <<"exporter">> => exporter(Config, Opts) + }. + +exporter(Config, #{exporter := local}) -> + #{<<"type">> => <<"local">>, <<"root">> => root(Config, node(), [exports])}; +exporter(_Config, #{exporter := s3, bucket_name := BucketName}) -> + BaseConfig = emqx_s3_test_helpers:base_raw_config(tcp), + BaseConfig#{ + <<"bucket">> => list_to_binary(BucketName), + <<"type">> => <<"s3">>, + <<"host">> => ?S3_HOST, + <<"port">> => ?S3_PORT }. load_config(Config) -> - emqx_common_test_helpers:load_config(emqx_ft_schema, #{file_transfer => Config}). + emqx_common_test_helpers:load_config(emqx_ft_schema, #{<<"file_transfer">> => Config}). tcp_port(Node) -> {_, Port} = rpc:call(Node, emqx_config, get, [[listeners, tcp, default, bind]]), Port. root(Config, Node, Tail) -> - filename:join([?config(priv_dir, Config), "file_transfer", Node | Tail]). + iolist_to_binary(filename:join([?config(priv_dir, Config), "file_transfer", Node | Tail])). start_client(ClientId) -> start_client(ClientId, node()). @@ -94,9 +106,21 @@ upload_file(ClientId, FileId, Name, Data, Node) -> ct:pal("MetaPayload = ~ts", [MetaPayload]), MetaTopic = <<"$file/", FileId/binary, "/init">>, - {ok, _} = emqtt:publish(C1, MetaTopic, MetaPayload, 1), - {ok, _} = emqtt:publish(C1, <<"$file/", FileId/binary, "/0">>, Data, 1), + {ok, #{reason_code_name := success}} = emqtt:publish(C1, MetaTopic, MetaPayload, 1), + {ok, #{reason_code_name := success}} = emqtt:publish( + C1, <<"$file/", FileId/binary, "/0">>, Data, 1 + ), FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Size))/binary>>, - {ok, _} = emqtt:publish(C1, FinTopic, <<>>, 1), - ok = emqtt:stop(C1). + FinResult = + case emqtt:publish(C1, FinTopic, <<>>, 1) of + {ok, #{reason_code_name := success}} -> + ok; + {ok, #{reason_code_name := Error}} -> + {error, Error} + end, + ok = emqtt:stop(C1), + FinResult. + +aws_config() -> + emqx_s3_test_helpers:aws_config(tcp, binary_to_list(?S3_HOST), ?S3_PORT). diff --git a/apps/emqx_s3/test/emqx_s3_test_helpers.erl b/apps/emqx_s3/test/emqx_s3_test_helpers.erl index 2edd52609..a73f618af 100644 --- a/apps/emqx_s3/test/emqx_s3_test_helpers.erl +++ b/apps/emqx_s3/test/emqx_s3_test_helpers.erl @@ -36,19 +36,24 @@ %%-------------------------------------------------------------------- aws_config(tcp) -> + aws_config(tcp, ?TCP_HOST, ?TCP_PORT); +aws_config(tls) -> + aws_config(tls, ?TLS_HOST, ?TLS_PORT). + +aws_config(tcp, Host, Port) -> erlcloud_s3_new( ?ACCESS_KEY_ID, ?SECRET_ACCESS_KEY, - ?TCP_HOST, - ?TCP_PORT, + Host, + Port, "http://" ); -aws_config(tls) -> +aws_config(tls, Host, Port) -> erlcloud_s3_new( ?ACCESS_KEY_ID, ?SECRET_ACCESS_KEY, - ?TLS_HOST, - ?TLS_PORT, + Host, + Port, "https://" ). diff --git a/rel/i18n/emqx_ft_schema.hocon b/rel/i18n/emqx_ft_schema.hocon index e7e551289..64b9bd67e 100644 --- a/rel/i18n/emqx_ft_schema.hocon +++ b/rel/i18n/emqx_ft_schema.hocon @@ -3,8 +3,8 @@ emqx_ft_schema { enable.desc: """Enable the File Transfer feature.
Enabling File Transfer implies reserving special MQTT topics in order to serve the protocol.
-This toggle does not have an effect neither on the availability of the File Transfer REST API, nor -on storage-dependent background activities (e.g. garbage collection).""" +This toggle also affects the availability of the File Transfer REST API and +storage-dependent background activities (e.g. garbage collection).""" init_timeout.desc: """Timeout for initializing the file transfer.
From 8ac881a140a68a0440e383ab14e904554746ce97 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 9 May 2023 14:17:50 +0500 Subject: [PATCH 140/156] chore(ft): handle multiple/concurrent fins more gracefully --- apps/emqx_ft/src/emqx_ft.erl | 4 +- apps/emqx_ft/src/emqx_ft_assembler.erl | 5 + .../src/emqx_ft_storage_exporter_fs.erl | 4 +- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 51 +++++++- apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl | 6 +- .../src/proto/emqx_ft_storage_fs_proto_v1.erl | 6 + apps/emqx_ft/test/emqx_ft_SUITE.erl | 110 +++++++++++++++--- 7 files changed, 163 insertions(+), 23 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 42611e537..dba087484 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -268,8 +268,8 @@ on_fin(PacketId, Msg, Transfer, FinalSize, Checksum) -> with_responder(FinPacketKey, Callback, emqx_ft_conf:assemble_timeout(), fun() -> case assemble(Transfer, FinalSize) of %% Assembling completed, ack through the responder right away - % ok -> - % emqx_ft_responder:ack(FinPacketKey, ok); + ok -> + emqx_ft_responder:ack(FinPacketKey, ok); %% Assembling started, packet will be acked by the responder {async, Pid} -> ok = emqx_ft_responder:kickoff(FinPacketKey, Pid), diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 767930f98..19f6c48d1 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -24,6 +24,8 @@ -export([handle_event/4]). -export([terminate/3]). +-export([where/1]). + -type stdata() :: #{ storage := emqx_ft_storage_fs:storage(), transfer := emqx_ft:transfer(), @@ -39,6 +41,9 @@ start_link(Storage, Transfer, Size) -> gen_statem:start_link(?REF(Transfer), ?MODULE, {Storage, Transfer, Size}, []). +where(Transfer) -> + gproc:where(?NAME(Transfer)). + %% -type state() :: diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index 9109dadbb..208c6fda3 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -324,7 +324,9 @@ list(_Options, Query = #{transfer := _Transfer}) -> #{items := Exports = [_ | _]} -> {ok, #{items => Exports}}; #{items := [], errors := NodeErrors} -> - {error, NodeErrors} + {error, NodeErrors}; + #{items := []} -> + {ok, #{items => []}} end; list(_Options, Query) -> Result = list(Query), diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index cdc86d218..010d004a1 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -35,6 +35,7 @@ -export([read_filemeta/2]). -export([list/3]). -export([pread/5]). +-export([lookup_local_assembler/1]). -export([assemble/3]). -export([transfers/1]). @@ -211,11 +212,15 @@ pread(_Storage, _Transfer, Frag, Offset, Size) -> end. -spec assemble(storage(), transfer(), emqx_ft:bytes()) -> - {async, _Assembler :: pid()} | {error, _TODO}. + {async, _Assembler :: pid()} | ok | {error, _TODO}. assemble(Storage, Transfer, Size) -> - % TODO: ask cluster if the transfer is already assembled - {ok, Pid} = emqx_ft_assembler_sup:ensure_child(Storage, Transfer, Size), - {async, Pid}. + LookupSources = [ + fun() -> lookup_local_assembler(Transfer) end, + fun() -> lookup_remote_assembler(Transfer) end, + fun() -> check_if_already_exported(Storage, Transfer) end, + fun() -> ensure_local_assembler(Storage, Transfer, Size) end + ], + lookup_assembler(LookupSources). %% @@ -252,6 +257,44 @@ stop(Storage) -> %% +lookup_assembler([LastSource]) -> + LastSource(); +lookup_assembler([Source | Sources]) -> + case Source() of + {error, not_found} -> lookup_assembler(Sources); + Result -> Result + end. + +check_if_already_exported(Storage, Transfer) -> + case files(Storage, #{transfer => Transfer}) of + {ok, #{items := [_ | _]}} -> ok; + _ -> {error, not_found} + end. + +lookup_local_assembler(Transfer) -> + case emqx_ft_assembler:where(Transfer) of + Pid when is_pid(Pid) -> {async, Pid}; + _ -> {error, not_found} + end. + +lookup_remote_assembler(Transfer) -> + Nodes = emqx:running_nodes() -- [node()], + Assemblers = lists:flatmap( + fun + ({ok, {async, Pid}}) -> [Pid]; + (_) -> [] + end, + emqx_ft_storage_fs_proto_v1:list_assemblers(Nodes, Transfer) + ), + case Assemblers of + [Pid | _] -> {async, Pid}; + _ -> {error, not_found} + end. + +ensure_local_assembler(Storage, Transfer, Size) -> + {ok, Pid} = emqx_ft_assembler_sup:ensure_child(Storage, Transfer, Size), + {async, Pid}. + -spec transfers(storage()) -> {ok, #{transfer() => transferinfo()}}. transfers(Storage) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl index 597f84091..68cd4c2fd 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl @@ -22,7 +22,8 @@ -export([ list_local/2, - pread_local/4 + pread_local/4, + lookup_local_assembler/1 ]). list_local(Transfer, What) -> @@ -30,3 +31,6 @@ list_local(Transfer, What) -> pread_local(Transfer, Frag, Offset, Size) -> emqx_ft_storage:with_storage_type(local, pread, [Transfer, Frag, Offset, Size]). + +lookup_local_assembler(Transfer) -> + emqx_ft_storage:with_storage_type(local, lookup_local_assembler, [Transfer]). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index 3b91684e6..d09b9ec9a 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -22,6 +22,7 @@ -export([multilist/3]). -export([pread/5]). +-export([list_assemblers/2]). -type offset() :: emqx_ft:offset(). -type transfer() :: emqx_ft:transfer(). @@ -41,3 +42,8 @@ multilist(Nodes, Transfer, What) -> {ok, [filefrag()]} | {error, term()} | no_return(). pread(Node, Transfer, Frag, Offset, Size) -> erpc:call(Node, emqx_ft_storage_fs_proxy, pread_local, [Transfer, Frag, Offset, Size]). + +-spec list_assemblers([node()], transfer()) -> + emqx_rpc:erpc_multicall([pid()]). +list_assemblers(Nodes, Transfer) -> + erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, lookup_local_assembler, [Transfer]). diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 929665ca9..ae550eb12 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -44,7 +44,8 @@ groups() -> group_cluster() -> [ t_switch_node, - t_unreliable_migrating_client + t_unreliable_migrating_client, + t_concurrent_fins ]. init_per_suite(Config) -> @@ -549,21 +550,11 @@ t_unreliable_migrating_client(Config) -> Exports = list_files(?config(clientid, Config)), - % NOTE - % The cluster had 2 assemblers running on two different nodes, because client sent `fin` - % twice. This is currently expected, files must be identical anyway. Node1Str = atom_to_list(Node1), - NodeSelfStr = atom_to_list(NodeSelf), % TODO: this testcase is specific to local fs storage backend ?assertMatch( - [#{"node" := Node1Str}, #{"node" := NodeSelfStr}], - lists:map( - fun(#{uri := URIString}) -> - #{query := QS} = uri_string:parse(URIString), - maps:from_list(uri_string:dissect_query(QS)) - end, - lists:sort(Exports) - ) + [#{"node" := Node1Str}], + fs_exported_file_attributes(Exports) ), [ @@ -571,6 +562,84 @@ t_unreliable_migrating_client(Config) -> || Export <- Exports ]. +t_concurrent_fins(Config) -> + NodeSelf = node(), + [Node1, Node2] = ?config(cluster_nodes, Config), + + ClientId = ?config(clientid, Config), + FileId = emqx_guid:to_hexstr(emqx_guid:gen()), + Filename = "migratory-birds-in-southern-hemisphere-2013.pdf", + Filesize = 100, + Gen = emqx_ft_content_gen:new({{ClientId, FileId}, Filesize}, 16), + Payload = iolist_to_binary(emqx_ft_content_gen:consume(Gen, fun({Chunk, _, _}) -> Chunk end)), + Meta = meta(Filename, Payload), + + %% Send filemeta and segments to Node1 + Context0 = #{ + clientid => ClientId, + fileid => FileId, + filesize => Filesize, + payload => Payload + }, + + Context1 = run_commands( + [ + {fun connect_mqtt_client/2, [Node1]}, + {fun send_filemeta/2, [Meta]}, + {fun send_segment/3, [0, 100]}, + {fun stop_mqtt_client/1, []} + ], + Context0 + ), + + %% Now send fins concurrently to the 3 nodes + Self = self(), + Nodes = [Node1, Node2, NodeSelf], + FinSenders = lists:map( + fun(Node) -> + %% takeovers and disconnects will happen due to concurrency + _ = erlang:process_flag(trap_exit, true), + _Context = run_commands( + [ + {fun connect_mqtt_client/2, [Node]}, + {fun send_finish/1, []} + ], + Context1 + ), + Self ! {done, Node} + end, + Nodes + ), + ok = lists:foreach( + fun(F) -> + _Pid = spawn_link(F) + end, + FinSenders + ), + ok = lists:foreach( + fun(Node) -> + receive + {done, Node} -> ok + after 1000 -> + ct:fail("Node ~p did not send finish successfully", [Node]) + end + end, + Nodes + ), + + %% Only one node should have the file + Exports = list_files(?config(clientid, Config)), + ?assertMatch( + [#{"node" := _Node}], + fs_exported_file_attributes(Exports) + ). + +%%------------------------------------------------------------------------------ +%% Command helpers +%%------------------------------------------------------------------------------ + +%% Command runners + run_commands(Commands, Context) -> lists:foldl(fun run_command/2, Context, Commands). @@ -578,6 +647,8 @@ run_command({Command, Args}, Context) -> ct:pal("COMMAND ~p ~p", [erlang:fun_info(Command, name), Args]), erlang:apply(Command, Args ++ [Context]). +%% Commands + connect_mqtt_client(Node, ContextIn) -> Context = #{clientid := ClientId} = disown_mqtt_client(ContextIn), NodePort = emqx_ft_test_helpers:tcp_port(Node), @@ -623,9 +694,18 @@ send_finish(Context = #{client := Client, fileid := FileId, filesize := Filesize ), Context. -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ %% Helpers -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ + +fs_exported_file_attributes(FSExports) -> + lists:map( + fun(#{uri := URIString}) -> + #{query := QS} = uri_string:parse(URIString), + maps:from_list(uri_string:dissect_query(QS)) + end, + lists:sort(FSExports) + ). mk_init_topic(FileId) -> <<"$file/", FileId/binary, "/init">>. From 5e5f854ce1a157551c54b3d9cccccbdf6eb6f1fd Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 5 May 2023 23:34:45 +0300 Subject: [PATCH 141/156] feat(ft-conf): simplify schema of storage / exporter backends Assumption is this changes will make `emqx_ft` config schema user-friendlier and also more future-proof. --- apps/emqx_ft/src/emqx_ft_conf.erl | 30 +++-- apps/emqx_ft/src/emqx_ft_schema.erl | 112 +++++++--------- apps/emqx_ft/src/emqx_ft_storage.erl | 123 ++++++++---------- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 11 +- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 7 +- apps/emqx_ft/test/emqx_ft_SUITE.erl | 18 +-- apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl | 19 +-- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 84 ++++++------ .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 2 +- .../test/emqx_ft_storage_fs_gc_SUITE.erl | 22 ++-- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 20 +-- rel/i18n/emqx_ft_schema.hocon | 16 +-- 12 files changed, 228 insertions(+), 236 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 61f639271..74f2df27d 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -54,24 +54,26 @@ enabled() -> emqx_config:get([file_transfer, enable], false). --spec storage() -> _Storage. +-spec storage() -> emqx_config:config(). storage() -> emqx_config:get([file_transfer, storage]). --spec gc_interval(_Storage) -> emqx_maybe:t(milliseconds()). -gc_interval(Conf = #{type := local}) -> - emqx_utils_maps:deep_get([segments, gc, interval], Conf); -gc_interval(_) -> - undefined. +-spec gc_interval(emqx_ft_storage_fs:storage()) -> + emqx_maybe:t(milliseconds()). +gc_interval(Storage) -> + emqx_utils_maps:deep_get([segments, gc, interval], Storage, undefined). --spec segments_ttl(_Storage) -> emqx_maybe:t({_Min :: seconds(), _Max :: seconds()}). -segments_ttl(Conf = #{type := local}) -> - { - emqx_utils_maps:deep_get([segments, gc, minimum_segments_ttl], Conf), - emqx_utils_maps:deep_get([segments, gc, maximum_segments_ttl], Conf) - }; -segments_ttl(_) -> - undefined. +-spec segments_ttl(emqx_ft_storage_fs:storage()) -> + emqx_maybe:t({_Min :: seconds(), _Max :: seconds()}). +segments_ttl(Storage) -> + Min = emqx_utils_maps:deep_get([segments, gc, minimum_segments_ttl], Storage, undefined), + Max = emqx_utils_maps:deep_get([segments, gc, maximum_segments_ttl], Storage, undefined), + case is_integer(Min) andalso is_integer(Max) of + true -> + {Min, Max}; + false -> + undefined + end. init_timeout() -> emqx_config:get([file_transfer, init_timeout]). diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index e2eebbbb8..0440bdcb9 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -93,36 +93,30 @@ fields(file_transfer) -> )}, {storage, mk( - hoconsc:union( - fun - (all_union_members) -> - [ref(local_storage)]; - ({value, #{<<"type">> := <<"local">>}}) -> - [ref(local_storage)]; - ({value, #{<<"type">> := _}}) -> - throw(#{field_name => type, expected => "local"}); - ({value, _}) -> - [ref(local_storage)] - end - ), + ref(storage_backend), #{ + desc => ?DESC("storage_backend"), required => false, - desc => ?DESC("storage"), - default => #{<<"type">> => <<"local">>} + validator => validator(backend), + default => #{ + <<"local">> => #{} + } + } + )} + ]; +fields(storage_backend) -> + [ + {local, + mk( + ref(local_storage), + #{ + desc => ?DESC("local_storage"), + required => {false, recursively} } )} ]; fields(local_storage) -> [ - {type, - mk( - local, - #{ - default => local, - required => false, - desc => ?DESC("local_type") - } - )}, {segments, mk( ref(local_storage_segments), @@ -136,29 +130,13 @@ fields(local_storage) -> )}, {exporter, mk( - hoconsc:union( - fun - (all_union_members) -> - [ - ref(local_storage_exporter), - ref(s3_exporter) - ]; - ({value, #{<<"type">> := <<"local">>}}) -> - [ref(local_storage_exporter)]; - ({value, #{<<"type">> := <<"s3">>}}) -> - [ref(s3_exporter)]; - ({value, #{<<"type">> := _}}) -> - throw(#{field_name => type, expected => "local | s3"}); - ({value, _}) -> - % NOTE: default - [ref(local_storage_exporter)] - end - ), + ref(local_storage_exporter_backend), #{ - desc => ?DESC("local_storage_exporter"), - required => true, + desc => ?DESC("local_storage_exporter_backend"), + required => false, + validator => validator(backend), default => #{ - <<"type">> => <<"local">> + <<"local">> => #{} } } )} @@ -181,17 +159,27 @@ fields(local_storage_segments) -> } )} ]; -fields(local_storage_exporter) -> +fields(local_storage_exporter_backend) -> [ - {type, + {local, mk( - local, + ref(local_storage_exporter), #{ - default => local, - required => false, - desc => ?DESC("local_storage_exporter_type") + desc => ?DESC("local_storage_exporter"), + required => {false, recursively} } )}, + {s3, + mk( + ref(s3_exporter), + #{ + desc => ?DESC("s3_exporter"), + required => {false, recursively} + } + )} + ]; +fields(local_storage_exporter) -> + [ {root, mk( binary(), @@ -202,18 +190,7 @@ fields(local_storage_exporter) -> )} ]; fields(s3_exporter) -> - [ - {type, - mk( - s3, - #{ - default => s3, - required => false, - desc => ?DESC("s3_exporter_type") - } - )} - ] ++ - emqx_s3_schema:fields(s3); + emqx_s3_schema:fields(s3); fields(local_storage_segments_gc) -> [ {interval, @@ -287,7 +264,16 @@ validator(filename) -> byte_size(Bin) =< ?MAX_FILENAME_BYTELEN orelse {error, max_length_exceeded} end, fun emqx_ft_fs_util:is_filename_safe/1 - ]. + ]; +validator(backend) -> + fun(Config) -> + case maps:keys(Config) of + [_Type] -> + ok; + _Conflicts = [_ | _] -> + {error, multiple_conflicting_backends} + end + end. converter(checksum) -> fun diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 5ec342585..79fc2dcfc 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -18,8 +18,6 @@ -export( [ - child_spec/0, - store_filemeta/2, store_segment/2, assemble/2, @@ -30,11 +28,17 @@ with_storage_type/2, with_storage_type/3, + backend/0, on_config_update/2 ] ). --type storage() :: emqx_config:config(). +-type type() :: local. +-type backend() :: {type(), storage()}. +-type storage() :: config(). +-type config() :: emqx_config:config(). + +-export_type([backend/0]). -export_type([assemble_callback/0]). @@ -100,31 +104,20 @@ %% API %%-------------------------------------------------------------------- --spec child_spec() -> - [supervisor:child_spec()]. -child_spec() -> - try - Mod = mod(), - Mod:child_spec(storage()) - catch - error:disabled -> []; - error:undef -> [] - end. - -spec store_filemeta(emqx_ft:transfer(), emqx_ft:filemeta()) -> ok | {async, pid()} | {error, term()}. store_filemeta(Transfer, FileMeta) -> - with_storage(store_filemeta, [Transfer, FileMeta]). + dispatch(store_filemeta, [Transfer, FileMeta]). -spec store_segment(emqx_ft:transfer(), emqx_ft:segment()) -> ok | {async, pid()} | {error, term()}. store_segment(Transfer, Segment) -> - with_storage(store_segment, [Transfer, Segment]). + dispatch(store_segment, [Transfer, Segment]). -spec assemble(emqx_ft:transfer(), emqx_ft:bytes()) -> ok | {async, pid()} | {error, term()}. assemble(Transfer, Size) -> - with_storage(assemble, [Transfer, Size]). + dispatch(assemble, [Transfer, Size]). -spec files() -> {ok, page(file_info(), _)} | {error, term()}. @@ -134,77 +127,75 @@ files() -> -spec files(query(Cursor)) -> {ok, page(file_info(), Cursor)} | {error, term()}. files(Query) -> - with_storage(files, [Query]). + dispatch(files, [Query]). --spec with_storage(atom() | function()) -> any(). -with_storage(Fun) -> - with_storage(Fun, []). - --spec with_storage(atom() | function(), list(term())) -> any(). -with_storage(Fun, Args) -> - case storage() of - Storage = #{} -> - apply_storage(Storage, Fun, Args); - undefined -> +-spec dispatch(atom(), list(term())) -> any(). +dispatch(Fun, Args) when is_atom(Fun) -> + case backend() of + {Type, Storage} -> + apply(mod(Type), Fun, [Storage | Args]); + _ -> {error, disabled} end. +%% + -spec with_storage_type(atom(), atom() | function()) -> any(). with_storage_type(Type, Fun) -> with_storage_type(Type, Fun, []). -spec with_storage_type(atom(), atom() | function(), list(term())) -> any(). with_storage_type(Type, Fun, Args) -> - with_storage(fun(Storage) -> - case Storage of - #{type := Type} -> - apply_storage(Storage, Fun, Args); - _ -> - {error, {invalid_storage_type, Storage}} - end - end). - -apply_storage(Storage, Fun, Args) when is_atom(Fun) -> - apply(mod(Storage), Fun, [Storage | Args]); -apply_storage(Storage, Fun, Args) when is_function(Fun) -> - apply(Fun, [Storage | Args]). + case backend() of + {Type, Storage} when is_atom(Fun) -> + apply(mod(Type), Fun, [Storage | Args]); + {Type, Storage} when is_function(Fun) -> + apply(Fun, [Storage | Args]); + {_, _} = Backend -> + {error, {invalid_storage_backend, Backend}}; + _ -> + {error, disabled} + end. %% --spec on_config_update(_Old :: emqx_maybe:t(storage()), _New :: emqx_maybe:t(storage())) -> +-spec backend() -> backend(). +backend() -> + backend(emqx_ft_conf:storage()). + +-spec on_config_update(_Old :: emqx_maybe:t(config()), _New :: emqx_maybe:t(config())) -> ok. -on_config_update(#{type := _} = Storage, #{type := _} = Storage) -> +on_config_update(ConfigOld, ConfigNew) -> + on_backend_update( + emqx_maybe:apply(fun backend/1, ConfigOld), + emqx_maybe:apply(fun backend/1, ConfigNew) + ). + +on_backend_update({Type, _} = Backend, {Type, _} = Backend) -> ok; -on_config_update(#{type := Type} = StorageOld, #{type := Type} = StorageNew) -> - ok = (mod(StorageNew)):on_config_update(StorageOld, StorageNew); -on_config_update(StorageOld, StorageNew) when - (StorageOld =:= undefined orelse is_map_key(type, StorageOld)) andalso - (StorageNew =:= undefined orelse is_map_key(type, StorageNew)) +on_backend_update({Type, StorageOld}, {Type, StorageNew}) -> + ok = (mod(Type)):on_config_update(StorageOld, StorageNew); +on_backend_update(BackendOld, BackendNew) when + (BackendOld =:= undefined orelse is_tuple(BackendOld)) andalso + (BackendNew =:= undefined orelse is_tuple(BackendNew)) -> - _ = emqx_maybe:apply(fun on_storage_stop/1, StorageOld), - _ = emqx_maybe:apply(fun on_storage_start/1, StorageNew), + _ = emqx_maybe:apply(fun on_storage_stop/1, BackendOld), + _ = emqx_maybe:apply(fun on_storage_start/1, BackendNew), ok. %%-------------------------------------------------------------------- %% Local API %%-------------------------------------------------------------------- -on_storage_start(Storage) -> - (mod(Storage)):start(Storage). +-spec backend(config()) -> backend(). +backend(#{local := Storage}) -> + {local, Storage}. -on_storage_stop(Storage) -> - (mod(Storage)):stop(Storage). +on_storage_start({Type, Storage}) -> + (mod(Type)):start(Storage). -storage() -> - emqx_ft_conf:storage(). +on_storage_stop({Type, Storage}) -> + (mod(Type)):stop(Storage). -mod() -> - mod(storage()). - -mod(Storage) -> - case Storage of - #{type := local} -> - emqx_ft_storage_fs; - undefined -> - error(disabled) - end. +mod(local) -> + emqx_ft_storage_fs. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index e000fe5c6..e25ab158e 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -169,15 +169,12 @@ stop({ExporterMod, ExporterOpts}) -> exporter(Storage) -> case maps:get(exporter, Storage) of - #{type := local} = Options -> - {emqx_ft_storage_exporter_fs, without_type(Options)}; - #{type := s3} = Options -> - {emqx_ft_storage_exporter_s3, without_type(Options)} + #{local := Options} -> + {emqx_ft_storage_exporter_fs, Options}; + #{s3 := Options} -> + {emqx_ft_storage_exporter_s3, Options} end. -without_type(#{type := _} = Options) -> - maps:without([type], Options). - init_checksum(#{checksum := {Algo, _}}) -> crypto:hash_init(Algo); init_checksum(#{}) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 692a270e3..713649759 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -58,9 +58,9 @@ start_link(Storage) -> collect() -> gen_server:call(mk_server_ref(global), {collect, erlang:system_time()}, infinity). --spec reset() -> ok. +-spec reset() -> ok | {error, _}. reset() -> - reset(emqx_ft_conf:storage()). + emqx_ft_storage:with_storage_type(local, fun reset/1). -spec reset(emqx_ft_storage_fs:storage()) -> ok. reset(Storage) -> @@ -139,7 +139,8 @@ maybe_report(#gcstats{} = _Stats, _Storage) -> ?tp(garbage_collection, #{stats => _Stats, storage => _Storage}). start_timer(St) -> - start_timer(gc_interval(emqx_ft_conf:storage()), St). + Interval = emqx_ft_storage:with_storage_type(local, fun gc_interval/1), + start_timer(Interval, St). start_timer(Interval, St = #st{next_gc_timer = undefined}) when ?IS_ENABLED(Interval) -> St#st{next_gc_timer = emqx_utils:start_timer(Interval, collect)}; diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 929665ca9..6b675e0c0 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -58,16 +58,16 @@ end_per_suite(_Config) -> set_special_configs(Config) -> fun (emqx_ft) -> - Storage = emqx_ft_test_helpers:local_storage(Config), + % NOTE + % Inhibit local fs GC to simulate it isn't fast enough to collect + % complete transfers. + Storage = emqx_utils_maps:deep_merge( + emqx_ft_test_helpers:local_storage(Config), + #{<<"local">> => #{<<"segments">> => #{<<"gc">> => #{<<"interval">> => 0}}}} + ), emqx_ft_test_helpers:load_config(#{ - % NOTE - % Inhibit local fs GC to simulate it isn't fast enough to collect - % complete transfers. - enable => true, - storage => emqx_utils_maps:deep_merge( - Storage, - #{segments => #{gc => #{interval => 0}}} - ) + <<"enable">> => true, + <<"storage">> => Storage }); (_) -> ok diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl index 24a4593c2..c1deeb3bc 100644 --- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -246,16 +246,19 @@ exporter(Config) -> emqx_ft_storage_exporter:exporter(storage(Config)). storage(Config) -> - maps:get( - storage, + emqx_utils_maps:deep_get( + [storage, local], emqx_ft_schema:translate(#{ <<"storage">> => #{ - <<"type">> => <<"local">>, - <<"segments">> => #{ - <<"root">> => ?config(storage_root, Config) - }, - <<"exporter">> => #{ - <<"root">> => ?config(exports_root, Config) + <<"local">> => #{ + <<"segments">> => #{ + <<"root">> => ?config(storage_root, Config) + }, + <<"exporter">> => #{ + <<"local">> => #{ + <<"root">> => ?config(exports_root, Config) + } + } } } }) diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index bc9eb5d98..1f53f88af 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -34,7 +34,12 @@ end_per_suite(_Config) -> init_per_testcase(_Case, Config) -> _ = emqx_config:save_schema_mod_and_names(emqx_ft_schema), ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_ft], emqx_ft_test_helpers:env_handler(Config) + [emqx_conf, emqx_ft], fun + (emqx_ft) -> + emqx_ft_test_helpers:load_config(#{}); + (_) -> + ok + end ), {ok, _} = emqx:update_config([rpc, port_discovery], manual), Config. @@ -52,7 +57,7 @@ t_update_config(_Config) -> {error, #{kind := validation_error}}, emqx_conf:update( [file_transfer], - #{<<"storage">> => #{<<"type">> => <<"unknown">>}}, + #{<<"storage">> => #{<<"unknown">> => #{<<"foo">> => 42}}}, #{} ) ), @@ -63,16 +68,18 @@ t_update_config(_Config) -> #{ <<"enable">> => true, <<"storage">> => #{ - <<"type">> => <<"local">>, - <<"segments">> => #{ - <<"root">> => <<"/tmp/path">>, - <<"gc">> => #{ - <<"interval">> => <<"5m">> + <<"local">> => #{ + <<"segments">> => #{ + <<"root">> => <<"/tmp/path">>, + <<"gc">> => #{ + <<"interval">> => <<"5m">> + } + }, + <<"exporter">> => #{ + <<"local">> => #{ + <<"root">> => <<"/tmp/exports">> + } } - }, - <<"exporter">> => #{ - <<"type">> => <<"local">>, - <<"root">> => <<"/tmp/exports">> } } }, @@ -81,11 +88,15 @@ t_update_config(_Config) -> ), ?assertEqual( <<"/tmp/path">>, - emqx_config:get([file_transfer, storage, segments, root]) + emqx_config:get([file_transfer, storage, local, segments, root]) ), ?assertEqual( 5 * 60 * 1000, - emqx_ft_conf:gc_interval(emqx_ft_conf:storage()) + emqx_ft_storage:with_storage_type(local, fun emqx_ft_conf:gc_interval/1) + ), + ?assertEqual( + {5 * 60, 24 * 60 * 60}, + emqx_ft_storage:with_storage_type(local, fun emqx_ft_conf:segments_ttl/1) ). t_disable_restore_config(Config) -> @@ -93,13 +104,13 @@ t_disable_restore_config(Config) -> {ok, _}, emqx_conf:update( [file_transfer], - #{<<"enable">> => true, <<"storage">> => #{<<"type">> => <<"local">>}}, + #{<<"enable">> => true, <<"storage">> => #{<<"local">> => #{}}}, #{} ) ), ?assertEqual( 60 * 60 * 1000, - emqx_ft_conf:gc_interval(emqx_ft_conf:storage()) + emqx_ft_storage:with_storage_type(local, fun emqx_ft_conf:gc_interval/1) ), % Verify that transfers work ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <>), @@ -117,7 +128,7 @@ t_disable_restore_config(Config) -> emqx_ft_conf:enabled() ), ?assertMatch( - #{type := local, exporter := #{type := local}}, + #{local := #{exporter := #{local := _}}}, emqx_ft_conf:storage() ), ClientId = gen_clientid(), @@ -147,10 +158,11 @@ t_disable_restore_config(Config) -> #{ <<"enable">> => true, <<"storage">> => #{ - <<"type">> => <<"local">>, - <<"segments">> => #{ - <<"root">> => Root, - <<"gc">> => #{<<"interval">> => <<"1s">>} + <<"local">> => #{ + <<"segments">> => #{ + <<"root">> => Root, + <<"gc">> => #{<<"interval">> => <<"1s">>} + } } } }, @@ -165,10 +177,7 @@ t_disable_restore_config(Config) -> [ #{ ?snk_kind := garbage_collection, - storage := #{ - type := local, - segments := #{root := Root} - } + storage := #{segments := #{root := Root}} } ], ?of_kind(garbage_collection, Trace) @@ -188,48 +197,49 @@ t_switch_exporter(_Config) -> ) ), ?assertMatch( - #{type := local, exporter := #{type := local}}, + #{local := #{exporter := #{local := _}}}, emqx_ft_conf:storage() ), % Verify that switching to a different exporter works ?assertMatch( {ok, _}, emqx_conf:update( - [file_transfer, storage, exporter], + [file_transfer, storage, local, exporter], #{ - <<"type">> => <<"s3">>, - <<"bucket">> => <<"emqx">>, - <<"host">> => <<"https://localhost">>, - <<"port">> => 9000, - <<"transport_options">> => #{ - <<"ipv6_probe">> => false + <<"s3">> => #{ + <<"bucket">> => <<"emqx">>, + <<"host">> => <<"https://localhost">>, + <<"port">> => 9000, + <<"transport_options">> => #{ + <<"ipv6_probe">> => false + } } }, #{} ) ), ?assertMatch( - #{type := local, exporter := #{type := s3}}, + #{local := #{exporter := #{s3 := _}}}, emqx_ft_conf:storage() ), % Verify that switching back to local exporter works ?assertMatch( {ok, _}, emqx_conf:remove( - [file_transfer, storage, exporter], + [file_transfer, storage, local, exporter], #{} ) ), ?assertMatch( {ok, _}, emqx_conf:update( - [file_transfer, storage, exporter], - #{<<"type">> => <<"local">>}, + [file_transfer, storage, local, exporter], + #{<<"local">> => #{}}, #{} ) ), ?assertMatch( - #{type := local, exporter := #{type := local}}, + #{local := #{exporter := #{local := #{}}}}, emqx_ft_conf:storage() ), % Verify that transfers work diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index 2acb57a8e..50925cfb9 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -85,7 +85,7 @@ client_id(Config) -> storage(Config) -> RawConfig = #{<<"storage">> => emqx_ft_test_helpers:local_storage(Config)}, - #{storage := Storage} = emqx_ft_schema:translate(RawConfig), + #{storage := #{local := Storage}} = emqx_ft_schema:translate(RawConfig), Storage. list_files(Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl index 04aedf8f3..a7ffd5675 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -36,19 +36,19 @@ end_per_suite(_Config) -> ok. init_per_testcase(TC, Config) -> + SegmentsRoot = emqx_ft_test_helpers:root(Config, node(), [TC, segments]), + ExportsRoot = emqx_ft_test_helpers:root(Config, node(), [TC, exports]), ok = emqx_common_test_helpers:start_app( emqx_ft, fun(emqx_ft) -> emqx_ft_test_helpers:load_config(#{ <<"enable">> => true, <<"storage">> => #{ - <<"type">> => <<"local">>, - <<"segments">> => #{ - <<"root">> => emqx_ft_test_helpers:root(Config, node(), [TC, segments]) - }, - <<"exporter">> => #{ - <<"type">> => <<"local">>, - <<"root">> => emqx_ft_test_helpers:root(Config, node(), [TC, exports]) + <<"local">> => #{ + <<"segments">> => #{<<"root">> => SegmentsRoot}, + <<"exporter">> => #{ + <<"local">> => #{<<"root">> => ExportsRoot} + } } } }) @@ -105,7 +105,7 @@ t_gc_triggers_manually(_Config) -> ). t_gc_complete_transfers(_Config) -> - Storage = emqx_ft_conf:storage(), + {local, Storage} = emqx_ft_storage:backend(), ok = set_gc_config(minimum_segments_ttl, 0), ok = set_gc_config(maximum_segments_ttl, 3), ok = set_gc_config(interval, 500), @@ -198,7 +198,7 @@ t_gc_complete_transfers(_Config) -> t_gc_incomplete_transfers(_Config) -> ok = set_gc_config(minimum_segments_ttl, 0), ok = set_gc_config(maximum_segments_ttl, 4), - Storage = emqx_ft_conf:storage(), + {local, Storage} = emqx_ft_storage:backend(), Transfers = [ { {<<"client43"/utf8>>, <<"file-🦕"/utf8>>}, @@ -269,7 +269,7 @@ t_gc_incomplete_transfers(_Config) -> t_gc_handling_errors(_Config) -> ok = set_gc_config(minimum_segments_ttl, 0), ok = set_gc_config(maximum_segments_ttl, 0), - Storage = emqx_ft_conf:storage(), + {local, Storage} = emqx_ft_storage:backend(), Transfer1 = {<<"client1">>, mk_file_id()}, Transfer2 = {<<"client2">>, mk_file_id()}, Filemeta = #{name => "oops.pdf"}, @@ -325,7 +325,7 @@ t_gc_handling_errors(_Config) -> %% set_gc_config(Name, Value) -> - emqx_config:put([file_transfer, storage, segments, gc, Name], Value). + emqx_config:put([file_transfer, storage, local, segments, gc, Name], Value). start_transfer(Storage, {Transfer, Meta, Gen}) -> ?assertEqual( diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index 1482223d8..2eb6d84db 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -54,20 +54,22 @@ local_storage(Config) -> local_storage(Config, Opts) -> #{ - <<"type">> => <<"local">>, - <<"segments">> => #{<<"root">> => root(Config, node(), [segments])}, - <<"exporter">> => exporter(Config, Opts) + <<"local">> => #{ + <<"segments">> => #{<<"root">> => root(Config, node(), [segments])}, + <<"exporter">> => exporter(Config, Opts) + } }. exporter(Config, #{exporter := local}) -> - #{<<"type">> => <<"local">>, <<"root">> => root(Config, node(), [exports])}; + #{<<"local">> => #{<<"root">> => root(Config, node(), [exports])}}; exporter(_Config, #{exporter := s3, bucket_name := BucketName}) -> BaseConfig = emqx_s3_test_helpers:base_raw_config(tcp), - BaseConfig#{ - <<"bucket">> => list_to_binary(BucketName), - <<"type">> => <<"s3">>, - <<"host">> => ?S3_HOST, - <<"port">> => ?S3_PORT + #{ + <<"s3">> => BaseConfig#{ + <<"bucket">> => list_to_binary(BucketName), + <<"host">> => ?S3_HOST, + <<"port">> => ?S3_PORT + } }. load_config(Config) -> diff --git a/rel/i18n/emqx_ft_schema.hocon b/rel/i18n/emqx_ft_schema.hocon index 64b9bd67e..13bbc6970 100644 --- a/rel/i18n/emqx_ft_schema.hocon +++ b/rel/i18n/emqx_ft_schema.hocon @@ -18,11 +18,11 @@ store_segment_timeout.desc: """Timeout for storing a file segment.
After reaching the timeout, message with the segment will be acked with an error""" -storage.desc: +storage_backend.desc: """Storage settings for file transfer.""" -local_type.desc: -"""Use local file system to store uploaded fragments and temporary data.""" +local_storage.desc: +"""Local file system backend to store uploaded fragments and temporary data.""" local_storage_segments.desc: """Settings for local segments storage, which include uploaded transfer fragments and temporary data.""" @@ -30,15 +30,15 @@ local_storage_segments.desc: local_storage_segments_root.desc: """File system path to keep uploaded fragments and temporary data.""" -local_storage_exporter.desc: +local_storage_exporter_backend.desc: """Exporter for the local file system storage backend.
Exporter defines where and how fully transferred and assembled files are stored.""" -local_storage_exporter_type.desc: -"""Exporter type for the exporter to the local file system""" +local_storage_exporter.desc: +"""Exporter to the local file system.""" -s3_exporter_type.desc: -"""Exporter type for the exporter to S3""" +s3_exporter.desc: +"""Exporter to the S3 API compatible object storage.""" local_storage_exporter_root.desc: """File system path to keep uploaded files.""" From 079b8e94765027cd25345812a7b0105b67460f66 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 5 May 2023 23:37:28 +0300 Subject: [PATCH 142/156] fix(ft): silence warnings when some root is not yet `mkdir`ed --- apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index 9109dadbb..9f77c8afb 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -257,6 +257,9 @@ read_exportinfo( Transfer = dirnames_to_transfer(ClientId, FileId), Info = mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo), [Info | Acc]; +read_exportinfo(_Options, {node, _Root = "", {error, enoent}, []}, Acc) -> + % NOTE: Root directory does not exist, this is not an error. + Acc; read_exportinfo(Options, Entry, Acc) -> ok = log_invalid_entry(Options, Entry), Acc. From 9381c895bb86798d7fa19fc9e4d0cb6280b947ac Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 9 May 2023 12:19:55 +0500 Subject: [PATCH 143/156] fix(ft): fix dialyzer issues --- apps/emqx_ft/src/emqx_ft_storage.erl | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index 79fc2dcfc..d35229bfc 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -131,12 +131,8 @@ files(Query) -> -spec dispatch(atom(), list(term())) -> any(). dispatch(Fun, Args) when is_atom(Fun) -> - case backend() of - {Type, Storage} -> - apply(mod(Type), Fun, [Storage | Args]); - _ -> - {error, disabled} - end. + {Type, Storage} = backend(), + apply(mod(Type), Fun, [Storage | Args]). %% @@ -152,9 +148,7 @@ with_storage_type(Type, Fun, Args) -> {Type, Storage} when is_function(Fun) -> apply(Fun, [Storage | Args]); {_, _} = Backend -> - {error, {invalid_storage_backend, Backend}}; - _ -> - {error, disabled} + {error, {invalid_storage_backend, Backend}} end. %% From 45875bfee56dfc68bd52d7db785c8c4255445bfd Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 9 May 2023 14:24:50 +0500 Subject: [PATCH 144/156] fix(ft): update s3 exporter tests to the new config --- apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl index 86de74ee0..e717fe262 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl @@ -111,7 +111,7 @@ t_upload_error(Config) -> Data = <<"data"/utf8>>, {ok, _} = emqx_conf:update( - [file_transfer, storage, exporter, bucket], <<"invalid-bucket">>, #{} + [file_transfer, storage, local, exporter, s3, bucket], <<"invalid-bucket">>, #{} ), ?assertEqual( From 244188982bde8a9a9c09e995a76e620b31e2d294 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 9 May 2023 17:08:58 +0500 Subject: [PATCH 145/156] fix(ft): add missing translations --- apps/emqx_ft/src/emqx_ft_schema.erl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 0440bdcb9..26eedb1d6 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -237,6 +237,10 @@ desc(s3_exporter) -> "S3 Exporter settings for the File transfer local storage backend"; desc(local_storage_segments_gc) -> "Garbage collection settings for the File transfer local segments storage"; +desc(local_storage_exporter_backend) -> + "Exporter for the local file system storage backend"; +desc(storage_backend) -> + "Storage backend settings for file transfer"; desc(_) -> undefined. From 04a5ab4498bbe7baa4f695382fdafc97a961fdbe Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 10 May 2023 11:40:05 +0300 Subject: [PATCH 146/156] chore: add changelog entry --- changes/ee/feat-9927.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ee/feat-9927.en.md diff --git a/changes/ee/feat-9927.en.md b/changes/ee/feat-9927.en.md new file mode 100644 index 000000000..c20a0c51e --- /dev/null +++ b/changes/ee/feat-9927.en.md @@ -0,0 +1 @@ +Introduce support for the File Transfer over MQTT feature as described in [EIP-0021](https://github.com/emqx/eip), with support to publish transferred files either to the node-local file system or to the S3 API compatible remote object storage. From 6b688d6646c5be7b871c7071e585516c358f7e04 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 10 May 2023 16:25:43 +0300 Subject: [PATCH 147/156] fix(ft): anticipate repeated `kickoff`s + fix testcase --- apps/emqx_ft/src/emqx_ft_assembler.erl | 2 + apps/emqx_ft/test/emqx_ft_SUITE.erl | 99 ++++++++++++++++---------- 2 files changed, 63 insertions(+), 38 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 19f6c48d1..7a14199ad 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -78,6 +78,8 @@ handle_event(info, kickoff, idle, St) -> % We could wait for this message and handle it at the end of the assembling rather than at % the beginning, however it would make error handling much more messier. {next_state, list_local_fragments, St, ?internal([])}; +handle_event(info, kickoff, _, _St) -> + keep_state_and_data; handle_event( internal, _, diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 97f0cbbcc..7d64f9716 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -37,15 +37,27 @@ all() -> groups() -> [ - {single_node, [], emqx_common_test_helpers:all(?MODULE) -- group_cluster()}, - {cluster, [], group_cluster()} - ]. - -group_cluster() -> - [ - t_switch_node, - t_unreliable_migrating_client, - t_concurrent_fins + {single_node, [parallel], [ + t_assemble_crash, + t_corrupted_segment_retry, + t_invalid_checksum, + t_invalid_fileid, + t_invalid_filename, + t_invalid_meta, + t_invalid_topic_format, + t_meta_conflict, + t_nasty_clientids_fileids, + t_no_meta, + t_no_segment, + t_simple_transfer + ]}, + {cluster, [], [ + t_switch_node, + t_unreliable_migrating_client, + {g_concurrent_fins, [{repeat_until_any_fail, 8}], [ + t_concurrent_fins + ]} + ]} ]. init_per_suite(Config) -> @@ -563,10 +575,15 @@ t_unreliable_migrating_client(Config) -> ]. t_concurrent_fins(Config) -> + ct:timetrap({seconds, 10}), + NodeSelf = node(), [Node1, Node2] = ?config(cluster_nodes, Config), - ClientId = ?config(clientid, Config), + ClientId = iolist_to_binary([ + ?config(clientid, Config), + integer_to_list(erlang:unique_integer()) + ]), FileId = emqx_guid:to_hexstr(emqx_guid:gen()), Filename = "migratory-birds-in-southern-hemisphere-2013.pdf", Filesize = 100, @@ -593,46 +610,52 @@ t_concurrent_fins(Config) -> ), %% Now send fins concurrently to the 3 nodes - Self = self(), Nodes = [Node1, Node2, NodeSelf], - FinSenders = lists:map( + SendFin = fun(Node) -> + run_commands( + [ + {fun connect_mqtt_client/2, [Node]}, + {fun send_finish/1, []} + ], + Context1 + ) + end, + + PidMons = lists:map( fun(Node) -> - %% takeovers and disconnects will happen due to concurrency - _ = erlang:process_flag(trap_exit, true), - _Context = run_commands( - [ - {fun connect_mqtt_client/2, [Node]}, - {fun send_finish/1, []} - ], - Context1 - ), - Self ! {done, Node} + erlang:spawn_monitor(fun F() -> + _ = erlang:process_flag(trap_exit, true), + try + SendFin(Node) + catch + C:E -> + % NOTE: random delay to avoid livelock conditions + ct:pal("Node ~p did not send finish successfully: ~p:~p", [Node, C, E]), + ok = timer:sleep(rand:uniform(10)), + F() + end + end) end, Nodes ), ok = lists:foreach( - fun(F) -> - _Pid = spawn_link(F) - end, - FinSenders - ), - ok = lists:foreach( - fun(Node) -> + fun({Pid, MRef}) -> receive - {done, Node} -> ok - after 1000 -> - ct:fail("Node ~p did not send finish successfully", [Node]) + {'DOWN', MRef, process, Pid, normal} -> ok end end, - Nodes + PidMons ), %% Only one node should have the file - Exports = list_files(?config(clientid, Config)), - ?assertMatch( - [#{"node" := _Node}], - fs_exported_file_attributes(Exports) - ). + Exports = list_files(ClientId), + case fs_exported_file_attributes(Exports) of + [#{"node" := _Node}] -> + ok; + [#{"node" := _Node} | _] = Files -> + % ...But we can't really guarantee that + ct:comment({multiple_files_on_different_nodes, Files}) + end. %%------------------------------------------------------------------------------ %% Command helpers From 5fdcfad60c305ae9a9b4c8562592e2c6489e0b7a Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 11 May 2023 16:18:45 +0500 Subject: [PATCH 148/156] fix(ft): synchronize erlcloud deps --- apps/emqx_bridge_dynamo/rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_dynamo/rebar.config b/apps/emqx_bridge_dynamo/rebar.config index fbccb5c9a..d3ba1093d 100644 --- a/apps/emqx_bridge_dynamo/rebar.config +++ b/apps/emqx_bridge_dynamo/rebar.config @@ -1,6 +1,6 @@ %% -*- mode: erlang; -*- {erl_opts, [debug_info]}. -{deps, [ {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag, "3.5.16-emqx-1"}}} +{deps, [ {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.8-emqx-1"}}} , {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} From 84db57f8dfc5fe0314789573ab819d49d288491d Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 11 May 2023 16:23:59 +0500 Subject: [PATCH 149/156] fix(ft): synchronize erlcloud mix deps --- mix.exs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/mix.exs b/mix.exs index 3336743bc..42f12cac8 100644 --- a/mix.exs +++ b/mix.exs @@ -192,13 +192,6 @@ defmodule EMQXUmbrella.MixProject do {:snappyer, "1.2.8", override: true}, {:crc32cer, "0.1.8", override: true}, {:supervisor3, "1.1.12", override: true}, - {:erlcloud, github: "emqx/erlcloud", tag: "3.5.16-emqx-1", override: true}, - # erlcloud's rebar.config requires rebar3 and does not support Mix, - # so it tries to fetch deps from git. We need to override this. - {:lhttpc, tag: "1.6.2", override: true}, - {:eini, "1.2.9", override: true}, - {:base16, "1.0.0", override: true}, - # end of erlcloud's deps {:opentsdb, github: "emqx/opentsdb-client-erl", tag: "v0.5.1", override: true}, # The following two are dependencies of rabbit_common. They are needed here to # make mix not complain about conflicting versions From be3a0ce974ab1976cc9640707b4880a5d55224af Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 16 May 2023 16:26:26 +0300 Subject: [PATCH 150/156] fix(maybe): correct `apply/2` typespec Co-authored-by: ieQu1 <99872536+ieQu1@users.noreply.github.com> --- apps/emqx/src/emqx_maybe.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_maybe.erl b/apps/emqx/src/emqx_maybe.erl index 2629bc737..5b5d5e94b 100644 --- a/apps/emqx/src/emqx_maybe.erl +++ b/apps/emqx/src/emqx_maybe.erl @@ -45,7 +45,7 @@ define(Term, _) -> Term. %% @doc Apply a function to a maybe argument. --spec apply(fun((maybe(A)) -> maybe(A)), maybe(A)) -> +-spec apply(fun((A) -> maybe(A)), maybe(A)) -> maybe(A). apply(_Fun, undefined) -> undefined; From 91ce1c2a9080760a0360489842535789089e09c9 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 16 May 2023 16:27:33 +0300 Subject: [PATCH 151/156] fix(ft-schema): make description more natural Co-authored-by: ieQu1 <99872536+ieQu1@users.noreply.github.com> --- rel/i18n/emqx_ft_schema.hocon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rel/i18n/emqx_ft_schema.hocon b/rel/i18n/emqx_ft_schema.hocon index 13bbc6970..bafda331a 100644 --- a/rel/i18n/emqx_ft_schema.hocon +++ b/rel/i18n/emqx_ft_schema.hocon @@ -41,7 +41,7 @@ s3_exporter.desc: """Exporter to the S3 API compatible object storage.""" local_storage_exporter_root.desc: -"""File system path to keep uploaded files.""" +"""Directory where the uploaded files are kept.""" local_storage_segments_gc.desc: """Garbage collection settings for the intermediate and temporary files in the local file system.""" From a7595ff46877c12ff4e31b9f6a7f2d8c4bc28250 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 16 May 2023 16:45:07 +0300 Subject: [PATCH 152/156] chore: fixup copyright years --- apps/emqx_ft/include/emqx_ft_storage_fs.hrl | 2 +- apps/emqx_ft/src/emqx_ft.erl | 3 +-- apps/emqx_ft/src/emqx_ft_api.erl | 2 +- apps/emqx_ft/src/emqx_ft_app.erl | 2 +- apps/emqx_ft/src/emqx_ft_assembler.erl | 2 +- apps/emqx_ft/src/emqx_ft_assembler_sup.erl | 2 +- apps/emqx_ft/src/emqx_ft_assembly.erl | 2 +- apps/emqx_ft/src/emqx_ft_conf.erl | 2 +- apps/emqx_ft/src/emqx_ft_fs_iterator.erl | 2 +- apps/emqx_ft/src/emqx_ft_fs_util.erl | 2 +- apps/emqx_ft/src/emqx_ft_responder.erl | 2 +- apps/emqx_ft/src/emqx_ft_responder_sup.erl | 2 +- apps/emqx_ft/src/emqx_ft_schema.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl | 2 +- apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl | 2 +- apps/emqx_ft/src/emqx_ft_sup.erl | 2 +- .../emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl | 2 +- apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl | 2 +- apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl | 2 +- 27 files changed, 27 insertions(+), 28 deletions(-) diff --git a/apps/emqx_ft/include/emqx_ft_storage_fs.hrl b/apps/emqx_ft/include/emqx_ft_storage_fs.hrl index 72ebe586a..81ab9cfad 100644 --- a/apps/emqx_ft/include/emqx_ft_storage_fs.hrl +++ b/apps/emqx_ft/include/emqx_ft_storage_fs.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index dba087484..898203b51 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -17,7 +17,6 @@ -module(emqx_ft). -include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index 61a6a2c93..3fd279c76 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_app.erl b/apps/emqx_ft/src/emqx_ft_app.erl index 0bac6b592..299683e43 100644 --- a/apps/emqx_ft/src/emqx_ft_app.erl +++ b/apps/emqx_ft/src/emqx_ft_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 7a14199ad..873efc6ff 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl index bdefdac47..4ba65c290 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl index dead1132e..d765a2bd2 100644 --- a/apps/emqx_ft/src/emqx_ft_assembly.erl +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 74f2df27d..2e994925c 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_fs_iterator.erl b/apps/emqx_ft/src/emqx_ft_fs_iterator.erl index 7fb8d8634..7a58c5b38 100644 --- a/apps/emqx_ft/src/emqx_ft_fs_iterator.erl +++ b/apps/emqx_ft/src/emqx_ft_fs_iterator.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_fs_util.erl b/apps/emqx_ft/src/emqx_ft_fs_util.erl index b731d3270..9028722aa 100644 --- a/apps/emqx_ft/src/emqx_ft_fs_util.erl +++ b/apps/emqx_ft/src/emqx_ft_fs_util.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_responder.erl b/apps/emqx_ft/src/emqx_ft_responder.erl index cbbfbe687..c2c62e1c2 100644 --- a/apps/emqx_ft/src/emqx_ft_responder.erl +++ b/apps/emqx_ft/src/emqx_ft_responder.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_responder_sup.erl b/apps/emqx_ft/src/emqx_ft_responder_sup.erl index 23d4f55fa..fb3932425 100644 --- a/apps/emqx_ft/src/emqx_ft_responder_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_responder_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 26eedb1d6..09e9ab0a5 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index d35229bfc..4e1060d88 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index e25ab158e..591173615 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index 71f2f2748..6738d6fef 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl index a278b01a5..abb774f82 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl index 13160bfc6..50e02db6f 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index 5c5aade86..b9f07d5c0 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl index 713649759..4e9a6d56c 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl index 68cd4c2fd..e6358cb14 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl index 4b1c4acb8..513872edd 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl index fff6b7830..8c8aea6b3 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl index 8d388814c..0308668ab 100644 --- a/apps/emqx_ft/src/emqx_ft_sup.erl +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl index 9ca6db786..222891f54 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl index d09b9ec9a..989a48555 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl index f8fe02d36..ea089111d 100644 --- a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. From e3b822c1a046ee14ab7e6a5a9d3df9d132214f7d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 16 May 2023 16:45:41 +0300 Subject: [PATCH 153/156] chore: remove empty header Co-authored-by: ieQu1 <99872536+ieQu1@users.noreply.github.com> --- apps/emqx_ft/include/emqx_ft.hrl | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 apps/emqx_ft/include/emqx_ft.hrl diff --git a/apps/emqx_ft/include/emqx_ft.hrl b/apps/emqx_ft/include/emqx_ft.hrl deleted file mode 100644 index e46d79490..000000000 --- a/apps/emqx_ft/include/emqx_ft.hrl +++ /dev/null @@ -1,15 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- From 6f8f21106b9ef8dc45c2418c124ce45a438d891f Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 16 May 2023 16:46:05 +0300 Subject: [PATCH 154/156] fix(assert): use unpredictable binding names in macros Also translate macro vars to ALLCAPS for consistency. --- apps/emqx/include/asserts.hrl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx/include/asserts.hrl b/apps/emqx/include/asserts.hrl index 4b0afd67b..1be725d2d 100644 --- a/apps/emqx/include/asserts.hrl +++ b/apps/emqx/include/asserts.hrl @@ -30,16 +30,16 @@ ) ). --define(assertInclude(Pattern, List), +-define(assertInclude(PATTERN, LIST), ?assert( lists:any( - fun(El) -> - case El of - Pattern -> true; + fun(X__Elem_) -> + case X__Elem_ of + PATTERN -> true; _ -> false end end, - List + LIST ) ) ). From b71955e3685aebf3636e3b05e813fe28379c782b Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 17 May 2023 00:29:15 +0300 Subject: [PATCH 155/156] fix(ft): bump application versions --- apps/emqx/src/emqx.app.src | 2 +- apps/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- apps/emqx_machine/src/emqx_machine.app.src | 2 +- lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 5ca8fc797..be68b438f 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.25"}, + {vsn, "5.0.26"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index bd022f226..02fbdfb74 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.20"}, + {vsn, "5.0.21"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index a44d2b36e..7cf0e4b53 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.3"}, + {vsn, "0.2.4"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src index 3df18ce7a..599b0798c 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_conf, [ {description, "EMQX Enterprise Edition configuration schema"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [ kernel, From c95ef71fb54493aaf49a766809fc3f816ffe3a8d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 17 May 2023 12:49:20 +0300 Subject: [PATCH 156/156] chore(ft): provide more details in README.md --- apps/emqx_ft/README.md | 89 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 6 deletions(-) diff --git a/apps/emqx_ft/README.md b/apps/emqx_ft/README.md index c483b3169..a479754d2 100644 --- a/apps/emqx_ft/README.md +++ b/apps/emqx_ft/README.md @@ -1,9 +1,86 @@ -emqx_ft -===== +# EMQX File Transfer -EMQX file transfer over MQTT +EMQX File Transfer application enables the _File Transfer over MQTT_ feature described in [EIP-0021](https://github.com/emqx/eip), and provides support to publish transferred files either to the node-local file system or to the S3 API compatible remote object storage. -Build ------ +## Usage - $ rebar3 compile +As almost any other EMQX application, `emqx_ft` is configured via the EMQX configuration system. The following snippet is the minimal configuration that will enable File Transfer over MQTT. + +``` +file_transfer { + enabled = true +} +``` + +The configuration above will make File Transfer available to all MQTT clients, and will use the default storage backend, which in turn uses node-local file system both for temporary storage and for the final destination of the transferred files. + +## Configuration + +Every configuration parameter is described in the `emqx_ft_schema` module. + +The most important configuration parameter is `storage`, which defines the storage backend to use. Currently, only `local` storage backend is available, which stores all the temporary data accumulating during file transfers in the node-local file system. Those go into `${EMQX_DATA_DIR}/file_transfer` directory by default, but can be configured via `local.storage.segments.root` parameter. The final destination of the transferred files on the other hand is defined by `local.storage.exporter` parameter, and currently can be either `local` or `s3`. + +### Local Exporter + +The `local` exporter is the default one, and it stores the transferred files in the node-local file system. The final destination directory is defined by `local.storage.exporter.local.root` parameter, and defaults to `${EMQX_DATA_DIR}/file_transfer/exports` directory. + +``` +file_transfer { + enabled = true + storage { + local { + exporter { + local { root = "/var/lib/emqx/transfers" } + } + } + } +} +``` + +Important to note that even though the transferred files go into the node-local file system, the File Transfer API provides a cluster-wide view of the transferred files, and any file can be downloaded from any node in the cluster. + +### S3 Exporter + +The `s3` exporter stores the transferred files in the S3 API compatible remote object storage. The destination bucket is defined by `local.storage.exporter.s3.bucket` parameter. + +This snippet configures File Transfer to store the transferred files in the `my-bucket` bucket in the `us-east-1` region of the AWS S3 service. + +``` +file_transfer { + enabled = true + storage { + local { + exporter { + s3 { + host = "s3.us-east-1.amazonaws.com" + port = "443" + access_key_id = "AKIA27EZDDM9XLINWXFE" + secret_access_key = "..." + bucket = "my-bucket" + } + } + } + } +} + +``` + +## API + +### MQTT + +When enabled, File Transfer application reserves MQTT topics starting with `$file/` prefix for the purpose of serving the File Transfer protocol, as described in [EIP-0021](https://github.com/emqx/eip). + +### REST + +Application publishes a basic set of APIs, to: +* List all the transferred files available for download. +* Configure the application, including the storage backend. +* (When using `local` storage exporter) Download the transferred files. + +Switching to the `s3` storage exporter is possible at any time, but the files transferred before the switch will not be +available for download anymore. Though, the files will still be available in the node-local file system. + +## Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md).